diff --git a/tierb-fixed-moonep-full/hf_watcher.log b/tierb-fixed-moonep-full/hf_watcher.log index 07895d1438dc9492d95d458f6254045728d2a6ce..bcaf2c5c0d0e1f95fcb1ea4035d64cc5191822c3 100644 --- a/tierb-fixed-moonep-full/hf_watcher.log +++ b/tierb-fixed-moonep-full/hf_watcher.log @@ -3,3 +3,4 @@ Watching /workspace/tierb-fixed-moonep-full -> HF every 600s (kill to stop) [2026-08-29T23:33:50+00:00] backup pass uploaded: /workspace/tierb-fixed-moonep-full -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb-fixed-moonep-full https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +[2026-08-29T23:43:52+00:00] backup pass diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..323128407648a9b56db4ec22fbec3c84a4308c09 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 783 +1, 725 +2, 727 +3, 743 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 5835 +1, 5823 +2, 5847 +3, 5823 +4, 5847 +5, 6371 +6, 6157 +7, 6133 +0, 26773 +1, 26773 +2, 26773 +3, 26773 +4, 26773 +5, 26773 +6, 26773 +7, 26773 +0, 30417 +1, 30547 +2, 30375 +3, 30411 +4, 30385 +5, 30397 +6, 30509 +7, 30555 +0, 40611 +1, 40689 +2, 40543 +3, 40531 +4, 40527 +5, 40537 +6, 40685 +7, 40681 +0, 41131 +1, 41215 +2, 41065 +3, 41065 +4, 41065 +5, 41061 +6, 41197 +7, 41215 +0, 41135 +1, 41219 +2, 41069 +3, 41069 +4, 41069 +5, 41065 +6, 41201 +7, 41219 +0, 45269 +1, 45355 +2, 45205 +3, 45203 +4, 45203 +5, 45201 +6, 45335 +7, 45355 +0, 108983 +1, 109077 +2, 108899 +3, 108903 +4, 108917 +5, 108909 +6, 109067 +7, 109085 +0, 109391 +1, 109477 +2, 109325 +3, 109325 +4, 109325 +5, 109323 +6, 109457 +7, 109475 +0, 109391 +1, 109477 +2, 109325 +3, 109325 +4, 109325 +5, 109323 +6, 109457 +7, 109477 +0, 109781 +1, 109867 +2, 109715 +3, 109715 +4, 109715 +5, 109713 +6, 109847 +7, 109867 +0, 109905 +1, 109991 +2, 109857 +3, 109845 +4, 109859 +5, 109865 +6, 109987 +7, 110007 +0, 111071 +1, 111111 +2, 110953 +3, 110975 +4, 110979 +5, 110973 +6, 111125 +7, 111153 +0, 125077 +1, 125157 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125157 +0, 125077 +1, 125157 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125157 +0, 125077 +1, 125157 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125157 +0, 125077 +1, 125157 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125157 +0, 123195 +1, 123663 +2, 123319 +3, 123513 +4, 123515 +5, 123509 +6, 123525 +7, 123663 +0, 121637 +1, 122629 +2, 121835 +3, 122479 +4, 121831 +5, 122087 +6, 121965 +7, 122241 +0, 4 +1, 120553 +2, 4 +3, 120403 +4, 4 +5, 120399 +6, 111043 +7, 120553 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 120550 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..2c839ae09adb11f7d259373ada47d25387c12dfb --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5fe011819bcb4972f24a12af3ac6966638ae643e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log @@ -0,0 +1,7040 @@ +[23:39:25] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:39:36.090000 225018 torch/distributed/run.py:766] +W0829 23:39:36.090000 225018 torch/distributed/run.py:766] ***************************************** +W0829 23:39:36.090000 225018 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:39:36.090000 225018 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 0142 is connected to is connected to is connected to is connected to 7777 peer ranks. [Gloo] Rank peer ranks. 3 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W829 23:39:54.986665777 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W829 23:39:54.986755850 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:39:54.986807370 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:39:54.986857717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:39:54.986865101 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:39:54.986864241 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:39:54.986873841 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:39:54.149554361 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.591 seconds +time to initialize megatron (seconds): 8.089 +[after megatron is initialized] datetime: 2026-08-29 23:40:01 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.input_layernorm.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.input_layernorm.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:40:05 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:40:05 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3903.41, 3917.61) + train/valid/test-data-iterators-setup ..........: (64.57, 70.23) +[before the start of training step] datetime: 2026-08-29 23:40:05 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 200.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.01 MiB + total_size : 204.01 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [3840, 4480, 5632, 5760, 7424, 5760, 5376, 6400, 5248, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[12, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1], [48, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1], [55, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4992, 7296, 3072, 3968, 4480, 3456, 3840, 5376, 11136, 2176, 0, 0, 0, 0, 0, 0] | duplicated experts [[32, 34, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [16, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [5760, 5120, 6912, 3712, 9472, 4608, 8960, 1792, 3584, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[16, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [29, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled +Number of parameters in transformer block in billions: 13.53 [2026-08-29 23:40:31] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25773.2 | throughput per GPU (TFLOP/s/GPU): 5.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092134E+01 | loss scale: 1.0 | grad norm: 30.617 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 34478.52001953125 | max allocated: 93366.11376953125 | reserved: 95432.0 | max reserved: 95432.0 + [2026-08-29 23:40:32] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 676.5 | throughput per GPU (TFLOP/s/GPU): 188.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092013E+01 | loss scale: 1.0 | grad norm: 30.576 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65592 dup_rows=24120 dup_rows_remote=21482 dedup_ratio=0.245361 router_tokens_per_expert=[1289, 725, 579, 645, 603, 895, 1343, 163, 713, 776, 1286, 454, 120, 1381, 444, 935, 1059, 849, 692, 805, 621, 557, 1063, 547, 816, 795, 303, 362, 884, 1042, 477, 936, 1153, 444, 925, 823, 495, 387, 886, 773, 496, 581, 364, 773, 1370, 1553, 789, 323, 474, 709, 671, 866, 1163, 1664, 831, 565, 564, 959, 1231, 801, 567, 377, 1115, 301] payload_rows_per_dst=[9400, 9150, 9482, 8592, 8860, 9530, 10008, 9162] dup_rows_per_dst=[3084, 3068, 2904, 2638, 2912, 2968, 3878, 2668] combine_rows_per_src=[9400, 9150, 9482, 8592, 8860, 9530, 10008, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=64970 dup_rows=23808 dup_rows_remote=20796 dedup_ratio=0.242188 router_tokens_per_expert=[1462, 727, 526, 508, 587, 929, 1263, 252, 646, 1190, 1201, 409, 171, 1269, 447, 1010, 1350, 856, 726, 857, 616, 493, 997, 548, 886, 738, 312, 451, 824, 1060, 438, 907, 758, 433, 762, 831, 430, 394, 937, 778, 407, 531, 427, 805, 1399, 1543, 805, 352, 625, 640, 783, 789, 1039, 1921, 730, 560, 494, 860, 1327, 734, 559, 413, 1182, 248] payload_rows_per_dst=[9608, 7758, 9854, 8630, 9734, 9526, 10150, 9236] dup_rows_per_dst=[2900, 2390, 3032, 2602, 3450, 3012, 4024, 2398] combine_rows_per_src=[9608, 7758, 9854, 8630, 9734, 9526, 10150, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74664 payload_rows_remote=64274 dup_rows=23640 dup_rows_remote=19050 dedup_ratio=0.240479 router_tokens_per_expert=[1409, 822, 529, 582, 533, 924, 1297, 244, 831, 1200, 1296, 527, 147, 1389, 434, 1010, 1133, 869, 760, 831, 580, 417, 982, 597, 836, 812, 305, 477, 878, 803, 455, 910, 857, 503, 794, 879, 515, 376, 862, 721, 456, 571, 330, 516, 1398, 1614, 831, 311, 523, 708, 846, 859, 1120, 1609, 743, 627, 520, 847, 1287, 792, 473, 433, 1159, 253] payload_rows_per_dst=[9578, 10390, 9580, 10096, 8390, 9546, 7970, 9114] dup_rows_per_dst=[3102, 4590, 2758, 4074, 2624, 2508, 1570, 2414] combine_rows_per_src=[9578, 10390, 9580, 10096, 8390, 9546, 7970, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=64640 dup_rows=23840 dup_rows_remote=20956 dedup_ratio=0.242513 router_tokens_per_expert=[1435, 738, 561, 531, 658, 807, 1415, 257, 770, 1060, 1298, 394, 140, 1433, 503, 907, 1053, 814, 803, 796, 650, 511, 1063, 664, 725, 853, 400, 476, 888, 1016, 464, 914, 783, 483, 740, 747, 546, 373, 884, 755, 507, 549, 382, 669, 1436, 1471, 610, 338, 635, 653, 723, 939, 964, 1608, 700, 462, 571, 988, 1323, 789, 660, 458, 1114, 295] payload_rows_per_dst=[9670, 9428, 9824, 10314, 8068, 9238, 8552, 9370] dup_rows_per_dst=[3134, 3582, 2884, 4056, 2554, 2686, 1918, 3026] combine_rows_per_src=[9670, 9428, 9824, 10314, 8068, 9238, 8552, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65302 dup_rows=24018 dup_rows_remote=20910 dedup_ratio=0.244324 router_tokens_per_expert=[1462, 698, 491, 635, 630, 922, 1358, 189, 646, 1216, 1198, 462, 135, 1314, 480, 944, 1162, 894, 754, 869, 590, 486, 946, 604, 797, 682, 314, 403, 833, 1119, 402, 948, 1012, 435, 732, 895, 437, 377, 972, 698, 460, 561, 384, 497, 1377, 1493, 737, 353, 610, 729, 723, 842, 990, 1705, 798, 553, 557, 1043, 1338, 854, 499, 389, 1192, 327] payload_rows_per_dst=[9504, 8712, 9770, 8384, 8984, 9262, 10042, 9628] dup_rows_per_dst=[3266, 3102, 2840, 2612, 3108, 2462, 3858, 2770] combine_rows_per_src=[9504, 8712, 9770, 8384, 8984, 9262, 10042, 9628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268664832 dispatch_meta_bytes=348296 combine_payload_bytes=268664832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269013128 ingress_bytes=303267840 link_bytes=303267840 token_bytes=537677960 weight_movement_bytes=34603008 total_nvlink_bytes=572280968 weight_to_token_ratio=0.064356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141172467e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460184 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532577304 weight_movement_bytes=34603008 total_nvlink_bytes=567180312 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74628 payload_rows_remote=64670 dup_rows=23676 dup_rows_remote=19918 dedup_ratio=0.240845 router_tokens_per_expert=[1465, 689, 529, 603, 517, 862, 1329, 196, 816, 1148, 1300, 418, 160, 1272, 476, 965, 1028, 897, 749, 781, 650, 470, 1019, 571, 790, 796, 255, 489, 864, 1005, 379, 934, 996, 438, 863, 875, 537, 396, 944, 778, 454, 501, 352, 483, 1545, 1554, 827, 359, 557, 679, 747, 794, 1165, 1557, 823, 536, 477, 1039, 1377, 764, 524, 397, 1140, 252] payload_rows_per_dst=[9428, 8142, 9590, 8512, 10090, 9608, 9958, 9300] dup_rows_per_dst=[2952, 2424, 2740, 2512, 4108, 2542, 3758, 2640] combine_rows_per_src=[9428, 8142, 9590, 8512, 10090, 9608, 9958, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263266304 dispatch_meta_bytes=333296 combine_payload_bytes=263266304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263599600 ingress_bytes=332472320 link_bytes=332472320 token_bytes=526865904 weight_movement_bytes=69206016 total_nvlink_bytes=596071920 weight_to_token_ratio=0.131354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251066574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=342384 combine_payload_bytes=264765440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265107824 ingress_bytes=368574464 link_bytes=368574464 token_bytes=529873264 weight_movement_bytes=103809024 total_nvlink_bytes=633682288 weight_to_token_ratio=0.195913 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.386916035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=344848 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267821840 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535298832 weight_movement_bytes=34603008 total_nvlink_bytes=569901840 weight_to_token_ratio=0.064642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74718 payload_rows_remote=65190 dup_rows=23586 dup_rows_remote=20314 dedup_ratio=0.239929 router_tokens_per_expert=[1373, 788, 657, 602, 593, 848, 1295, 182, 681, 865, 1243, 562, 155, 1403, 521, 1001, 1177, 908, 694, 855, 614, 479, 999, 529, 707, 806, 302, 401, 879, 955, 379, 970, 881, 506, 757, 906, 479, 381, 999, 842, 486, 541, 371, 498, 1591, 1578, 795, 348, 471, 627, 730, 885, 1071, 1582, 711, 540, 459, 1075, 1391, 809, 515, 413, 1188, 303] payload_rows_per_dst=[9528, 10694, 9582, 10186, 8684, 9772, 6816, 9456] dup_rows_per_dst=[3272, 4310, 2804, 3776, 2818, 2644, 1112, 2850] combine_rows_per_src=[9528, 10694, 9582, 10186, 8684, 9772, 6816, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73804 payload_rows_remote=63076 dup_rows=24500 dup_rows_remote=20956 dedup_ratio=0.249227 router_tokens_per_expert=[1400, 680, 536, 590, 471, 863, 1297, 229, 775, 1252, 1297, 437, 117, 1438, 424, 980, 1210, 817, 821, 834, 608, 511, 858, 573, 806, 854, 316, 480, 799, 1040, 465, 900, 862, 442, 737, 921, 423, 360, 851, 926, 499, 539, 383, 431, 1512, 1537, 729, 308, 602, 733, 733, 849, 1123, 1615, 812, 538, 615, 925, 1304, 765, 493, 433, 1211, 263] payload_rows_per_dst=[7570, 8086, 8392, 8652, 10036, 9892, 10448, 10728] dup_rows_per_dst=[2308, 2478, 2268, 2668, 3884, 3108, 4242, 3544] combine_rows_per_src=[7570, 8086, 8392, 8652, 10036, 9892, 10448, 10728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=338352 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265226672 ingress_bytes=368697344 link_bytes=368697344 token_bytes=530114992 weight_movement_bytes=103809024 total_nvlink_bytes=633924016 weight_to_token_ratio=0.195824 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387378423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76584 payload_rows_remote=66962 dup_rows=21720 dup_rows_remote=18284 dedup_ratio=0.220947 router_tokens_per_expert=[258, 530, 706, 563, 1255, 590, 976, 645, 303, 441, 1141, 289, 1087, 559, 719, 1914, 884, 1200, 871, 301, 912, 1229, 1403, 552, 669, 816, 966, 1802, 233, 851, 525, 667, 274, 349, 405, 713, 1203, 277, 303, 979, 1518, 798, 42, 663, 263, 511, 62, 629, 3185, 509, 1451, 126, 823, 1114, 774, 916, 393, 814, 786, 569, 1470, 415, 516, 445] payload_rows_per_dst=[11268, 7598, 9024, 9622, 11964, 9198, 8970, 8940] dup_rows_per_dst=[3606, 1480, 2874, 3436, 3412, 2580, 2456, 1876] combine_rows_per_src=[11268, 7598, 9024, 9622, 11964, 9198, 8970, 8940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=342016 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267360256 ingress_bytes=336224256 link_bytes=336224256 token_bytes=534378496 weight_movement_bytes=69206016 total_nvlink_bytes=603584512 weight_to_token_ratio=0.129507 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265184807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75686 payload_rows_remote=65900 dup_rows=22618 dup_rows_remote=20066 dedup_ratio=0.230082 router_tokens_per_expert=[248, 375, 673, 534, 1213, 560, 947, 537, 292, 349, 856, 260, 1205, 468, 913, 2229, 848, 1284, 972, 732, 871, 892, 1353, 707, 663, 776, 876, 1903, 246, 873, 536, 581, 324, 345, 455, 578, 1056, 224, 333, 978, 1639, 554, 19, 827, 376, 507, 37, 857, 3273, 388, 1396, 64, 588, 1144, 760, 784, 557, 965, 585, 580, 1742, 403, 599, 443] payload_rows_per_dst=[8110, 10230, 9576, 9710, 7620, 9786, 10890, 9764] dup_rows_per_dst=[2064, 2914, 3036, 3198, 1280, 2552, 4938, 2636] combine_rows_per_src=[8110, 10230, 9576, 9710, 7620, 9786, 10890, 9764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76332 payload_rows_remote=66692 dup_rows=21972 dup_rows_remote=19114 dedup_ratio=0.223511 router_tokens_per_expert=[209, 447, 439, 502, 1234, 752, 929, 608, 242, 371, 1236, 204, 1245, 631, 924, 2231, 834, 1195, 904, 484, 905, 906, 1231, 667, 595, 799, 914, 1890, 187, 845, 487, 555, 399, 398, 487, 618, 1091, 360, 380, 925, 1529, 777, 40, 774, 340, 491, 50, 769, 3039, 339, 1400, 97, 705, 1088, 787, 860, 510, 956, 497, 641, 1825, 456, 539, 383] payload_rows_per_dst=[10908, 9640, 9056, 7690, 12052, 9212, 8382, 9392] dup_rows_per_dst=[3794, 2858, 2734, 2062, 3342, 2790, 2170, 2222] combine_rows_per_src=[10908, 9640, 9056, 7690, 12052, 9212, 8382, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=258359296 dispatch_meta_bytes=336128 combine_payload_bytes=258359296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258695424 ingress_bytes=292962304 link_bytes=292962304 token_bytes=517054720 weight_movement_bytes=34603008 total_nvlink_bytes=551657728 weight_to_token_ratio=0.066923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102393565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77128 payload_rows_remote=66978 dup_rows=21176 dup_rows_remote=17516 dedup_ratio=0.215413 router_tokens_per_expert=[221, 420, 467, 593, 1270, 525, 795, 538, 316, 397, 1243, 324, 1240, 612, 877, 2007, 1011, 1243, 1017, 972, 1011, 1073, 1302, 578, 591, 800, 851, 1994, 230, 871, 556, 463, 391, 339, 464, 529, 1010, 269, 376, 976, 1624, 608, 54, 768, 299, 430, 28, 586, 3007, 427, 1437, 137, 663, 990, 716, 890, 595, 896, 665, 501, 1550, 402, 630, 487] payload_rows_per_dst=[10754, 8092, 10150, 9546, 11694, 9230, 8438, 9224] dup_rows_per_dst=[2918, 1926, 3660, 3166, 3028, 2168, 2082, 2228] combine_rows_per_src=[10754, 8092, 10150, 9546, 11694, 9230, 8438, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76386 payload_rows_remote=64368 dup_rows=21918 dup_rows_remote=18826 dedup_ratio=0.222961 router_tokens_per_expert=[183, 602, 197, 599, 1321, 655, 888, 641, 302, 324, 1144, 225, 1069, 550, 866, 2271, 940, 1233, 894, 426, 951, 1044, 1293, 765, 621, 769, 777, 1848, 151, 789, 441, 730, 345, 317, 436, 567, 1094, 284, 389, 829, 1579, 811, 28, 841, 443, 516, 49, 617, 3294, 462, 1334, 92, 714, 1153, 801, 859, 557, 882, 717, 484, 1844, 376, 462, 467] payload_rows_per_dst=[8456, 9954, 9378, 9138, 12018, 9490, 8630, 9322] dup_rows_per_dst=[2114, 3150, 3128, 3114, 3092, 2864, 2200, 2256] combine_rows_per_src=[8456, 9954, 9378, 9138, 12018, 9490, 8630, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=340984 combine_payload_bytes=274276352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274617336 ingress_bytes=343482368 link_bytes=343482368 token_bytes=548893688 weight_movement_bytes=69206016 total_nvlink_bytes=618099704 weight_to_token_ratio=0.126083 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292496498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269926400 dispatch_meta_bytes=343864 combine_payload_bytes=269926400 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270270264 ingress_bytes=339132416 link_bytes=339132416 token_bytes=540196664 weight_movement_bytes=69206016 total_nvlink_bytes=609402680 weight_to_token_ratio=0.128113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276127979e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273170432 dispatch_meta_bytes=343224 combine_payload_bytes=273170432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273513656 ingress_bytes=342376448 link_bytes=342376448 token_bytes=546684088 weight_movement_bytes=69206016 total_nvlink_bytes=615890104 weight_to_token_ratio=0.126592 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288335010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274341888 dispatch_meta_bytes=337976 combine_payload_bytes=274341888 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=274679864 ingress_bytes=378150912 link_bytes=378150912 token_bytes=549021752 weight_movement_bytes=103809024 total_nvlink_bytes=652830776 weight_to_token_ratio=0.189080 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422951438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263651328 dispatch_meta_bytes=332776 combine_payload_bytes=263651328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263984104 ingress_bytes=298254336 link_bytes=298254336 token_bytes=527635432 weight_movement_bytes=34603008 total_nvlink_bytes=562238440 weight_to_token_ratio=0.065581 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122307055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74954 payload_rows_remote=64654 dup_rows=23350 dup_rows_remote=19140 dedup_ratio=0.237528 router_tokens_per_expert=[204, 428, 331, 536, 1299, 614, 957, 594, 262, 269, 1226, 308, 1147, 583, 855, 2289, 801, 1125, 899, 563, 916, 1015, 1466, 709, 607, 736, 837, 1843, 164, 883, 519, 693, 289, 320, 493, 612, 971, 254, 394, 921, 1573, 796, 39, 739, 344, 454, 41, 788, 3212, 449, 1404, 104, 788, 1142, 838, 722, 613, 890, 682, 492, 1693, 431, 526, 460] payload_rows_per_dst=[7826, 10448, 9052, 9440, 7300, 9656, 10300, 10932] dup_rows_per_dst=[2100, 3430, 3004, 3124, 1208, 2824, 4210, 3450] combine_rows_per_src=[7826, 10448, 9052, 9440, 7300, 9656, 10300, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76108 payload_rows_remote=65204 dup_rows=22196 dup_rows_remote=18818 dedup_ratio=0.225789 router_tokens_per_expert=[274, 516, 328, 529, 1242, 562, 925, 494, 271, 352, 1316, 246, 1194, 587, 864, 2271, 841, 1346, 920, 523, 1000, 976, 1432, 708, 666, 844, 893, 1729, 216, 878, 535, 663, 363, 319, 486, 655, 1015, 330, 328, 929, 1485, 794, 33, 729, 326, 454, 48, 889, 3011, 437, 1233, 147, 611, 1045, 811, 948, 524, 921, 446, 494, 1843, 384, 539, 434] payload_rows_per_dst=[10904, 10100, 9414, 9306, 9676, 9266, 8312, 9130] dup_rows_per_dst=[3378, 3018, 3214, 2968, 2312, 3114, 2152, 2040] combine_rows_per_src=[10904, 10100, 9414, 9306, 9676, 9266, 8312, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76612 payload_rows_remote=68726 dup_rows=21692 dup_rows_remote=20404 dedup_ratio=0.220662 router_tokens_per_expert=[105, 757, 141, 448, 326, 440, 259, 1380, 34, 986, 258, 1803, 527, 1708, 344, 739, 1917, 358, 707, 1048, 341, 272, 1616, 200, 465, 1410, 2817, 1023, 597, 642, 286, 164, 681, 624, 2034, 1184, 1791, 944, 1954, 570, 240, 967, 169, 111, 1569, 1712, 205, 407, 944, 894, 643, 450, 647, 550, 216, 218, 1662, 470, 306, 344, 368, 678, 710, 772] payload_rows_per_dst=[10412, 10238, 9814, 7886, 10280, 8574, 10294, 9114] dup_rows_per_dst=[2934, 2560, 3104, 1288, 2996, 2186, 2738, 3886] combine_rows_per_src=[10412, 10238, 9814, 7886, 10280, 8574, 10294, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=335176 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265157960 ingress_bytes=403234816 link_bytes=403234816 token_bytes=529980744 weight_movement_bytes=138412032 total_nvlink_bytes=668392776 weight_to_token_ratio=0.261164 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517340150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75342 payload_rows_remote=64076 dup_rows=22962 dup_rows_remote=19336 dedup_ratio=0.233582 router_tokens_per_expert=[253, 446, 345, 623, 1291, 640, 843, 611, 273, 357, 1175, 237, 1140, 562, 928, 2061, 945, 1203, 991, 707, 858, 943, 1499, 627, 612, 771, 963, 1820, 259, 821, 512, 634, 353, 314, 413, 552, 923, 319, 384, 895, 1682, 607, 51, 751, 396, 476, 36, 630, 3123, 390, 1471, 118, 760, 1008, 782, 793, 586, 938, 762, 515, 1655, 390, 670, 459] payload_rows_per_dst=[8104, 10220, 8532, 9624, 7082, 10310, 10204, 11266] dup_rows_per_dst=[2000, 3246, 2472, 3160, 1224, 3264, 3970, 3626] combine_rows_per_src=[8104, 10220, 8532, 9624, 7082, 10310, 10204, 11266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74928 payload_rows_remote=64200 dup_rows=23376 dup_rows_remote=20122 dedup_ratio=0.237793 router_tokens_per_expert=[135, 793, 103, 366, 263, 874, 249, 1847, 26, 1118, 223, 1873, 661, 1382, 313, 582, 1789, 508, 610, 1113, 413, 295, 1765, 167, 434, 1525, 2796, 1082, 687, 590, 212, 117, 918, 628, 1974, 1214, 1908, 1022, 1887, 441, 354, 756, 117, 120, 1339, 1687, 104, 725, 952, 773, 637, 293, 486, 351, 176, 187, 1527, 632, 301, 415, 283, 770, 544, 720] payload_rows_per_dst=[7454, 9830, 7806, 10518, 9734, 10728, 9410, 9448] dup_rows_per_dst=[1806, 2526, 1936, 4368, 2528, 3254, 2074, 4884] combine_rows_per_src=[7454, 9830, 7806, 10518, 9734, 10728, 9410, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75384 payload_rows_remote=65764 dup_rows=22920 dup_rows_remote=20396 dedup_ratio=0.233154 router_tokens_per_expert=[123, 811, 75, 413, 269, 524, 237, 2048, 33, 1247, 282, 1707, 718, 1122, 403, 560, 2352, 424, 557, 1019, 447, 265, 1597, 145, 556, 1622, 3071, 1242, 764, 483, 345, 140, 672, 727, 1782, 1396, 1472, 969, 1979, 434, 312, 904, 115, 144, 1368, 1725, 133, 417, 784, 1045, 724, 306, 584, 440, 197, 181, 1311, 554, 255, 321, 377, 789, 540, 594] payload_rows_per_dst=[11060, 9620, 11554, 8382, 9050, 8336, 10156, 7226] dup_rows_per_dst=[4082, 2524, 5002, 1922, 2910, 1900, 2324, 2256] combine_rows_per_src=[11060, 9620, 11554, 8382, 9050, 8336, 10156, 7226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=336088 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411672 ingress_bytes=301678592 link_bytes=301678592 token_bytes=534487256 weight_movement_bytes=34603008 total_nvlink_bytes=569090264 weight_to_token_ratio=0.064741 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135192255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75742 payload_rows_remote=66108 dup_rows=22562 dup_rows_remote=19020 dedup_ratio=0.229513 router_tokens_per_expert=[114, 842, 121, 382, 258, 567, 240, 1704, 30, 1405, 304, 1850, 667, 1274, 367, 794, 2046, 507, 669, 955, 480, 311, 1338, 195, 537, 1496, 2974, 1261, 739, 408, 387, 93, 688, 653, 1789, 1122, 1746, 1059, 1709, 504, 324, 981, 125, 135, 1469, 1807, 187, 458, 736, 1027, 764, 383, 556, 299, 189, 203, 1400, 564, 255, 392, 354, 772, 540, 647] payload_rows_per_dst=[10792, 10274, 9634, 8356, 10880, 8728, 9560, 7518] dup_rows_per_dst=[3612, 3108, 3542, 1486, 4068, 2244, 2172, 2330] combine_rows_per_src=[10792, 10274, 9634, 8356, 10880, 8728, 9560, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76024 payload_rows_remote=66652 dup_rows=22280 dup_rows_remote=20056 dedup_ratio=0.226644 router_tokens_per_expert=[85, 713, 79, 430, 267, 514, 273, 1930, 24, 969, 249, 2017, 653, 1285, 353, 599, 1916, 531, 651, 950, 359, 307, 1514, 125, 509, 1789, 3073, 1160, 648, 417, 333, 110, 760, 722, 1734, 1203, 1936, 716, 1789, 461, 265, 1062, 157, 132, 1448, 1924, 104, 491, 921, 953, 764, 292, 466, 439, 157, 107, 1694, 554, 263, 423, 298, 768, 556, 761] payload_rows_per_dst=[10608, 9758, 8492, 8704, 9372, 9936, 9720, 9434] dup_rows_per_dst=[3390, 2540, 2404, 1958, 2224, 3040, 2056, 4668] combine_rows_per_src=[10608, 9758, 8492, 8704, 9372, 9936, 9720, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281501696 dispatch_meta_bytes=356520 combine_payload_bytes=281501696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281858216 ingress_bytes=350707712 link_bytes=350707712 token_bytes=563359912 weight_movement_bytes=69206016 total_nvlink_bytes=632565928 weight_to_token_ratio=0.122845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319684886e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262455296 dispatch_meta_bytes=333648 combine_payload_bytes=262455296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262788944 ingress_bytes=297058304 link_bytes=297058304 token_bytes=525244240 weight_movement_bytes=34603008 total_nvlink_bytes=559847248 weight_to_token_ratio=0.065880 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117806483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262963200 dispatch_meta_bytes=337288 combine_payload_bytes=262963200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263300488 ingress_bytes=297566208 link_bytes=297566208 token_bytes=526263688 weight_movement_bytes=34603008 total_nvlink_bytes=560866696 weight_to_token_ratio=0.065752 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119717685e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269369344 dispatch_meta_bytes=344640 combine_payload_bytes=269369344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269713984 ingress_bytes=338575360 link_bytes=338575360 token_bytes=539083328 weight_movement_bytes=69206016 total_nvlink_bytes=608289344 weight_to_token_ratio=0.128377 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274031822e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270778368 dispatch_meta_bytes=340512 combine_payload_bytes=270778368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271118880 ingress_bytes=339984384 link_bytes=339984384 token_bytes=541897248 weight_movement_bytes=69206016 total_nvlink_bytes=611103264 weight_to_token_ratio=0.127711 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273006592 dispatch_meta_bytes=346832 combine_payload_bytes=273006592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=273353424 ingress_bytes=446021632 link_bytes=446021632 token_bytes=546360016 weight_movement_bytes=173015040 total_nvlink_bytes=719375056 weight_to_token_ratio=0.316669 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.678343494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=65288 dup_rows=23334 dup_rows_remote=20918 dedup_ratio=0.237366 router_tokens_per_expert=[102, 729, 85, 436, 304, 539, 241, 2046, 23, 1203, 226, 2000, 729, 1258, 407, 640, 2109, 504, 519, 1019, 524, 306, 1685, 163, 510, 1630, 3017, 1275, 680, 512, 407, 100, 541, 570, 1882, 1253, 1600, 708, 1905, 434, 323, 792, 137, 162, 1163, 1919, 199, 438, 734, 977, 642, 344, 523, 589, 203, 132, 1495, 606, 259, 415, 286, 737, 525, 731] payload_rows_per_dst=[7376, 10106, 7596, 11288, 8342, 11234, 9682, 9346] dup_rows_per_dst=[1588, 2866, 1844, 4974, 1870, 3250, 2416, 4526] combine_rows_per_src=[7376, 10106, 7596, 11288, 8342, 11234, 9682, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75316 payload_rows_remote=64902 dup_rows=22988 dup_rows_remote=19300 dedup_ratio=0.233846 router_tokens_per_expert=[90, 921, 121, 427, 319, 506, 280, 1479, 53, 1170, 206, 1827, 580, 1412, 363, 652, 1862, 527, 643, 947, 548, 308, 1408, 168, 474, 1652, 2908, 1107, 926, 477, 332, 125, 884, 725, 1918, 1096, 1784, 783, 1891, 544, 331, 902, 127, 169, 1324, 1659, 179, 436, 840, 915, 669, 459, 520, 453, 216, 158, 1468, 534, 305, 488, 455, 719, 563, 820] payload_rows_per_dst=[10414, 10004, 11210, 8318, 8966, 8162, 10074, 8168] dup_rows_per_dst=[3688, 2522, 5180, 1868, 2934, 2092, 2168, 2536] combine_rows_per_src=[10414, 10004, 11210, 8318, 8966, 8162, 10074, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75384 payload_rows_remote=67974 dup_rows=22920 dup_rows_remote=20930 dedup_ratio=0.233154 router_tokens_per_expert=[1661, 847, 1049, 45, 59, 291, 1991, 1979, 141, 292, 577, 354, 890, 695, 321, 1696, 2591, 194, 920, 922, 617, 859, 154, 1300, 178, 311, 949, 344, 576, 1190, 2451, 643, 1692, 785, 166, 142, 551, 746, 133, 479, 1114, 2351, 618, 756, 286, 476, 550, 604, 1306, 703, 611, 304, 129, 1072, 364, 2075, 752, 83, 230, 521, 1003, 671, 519, 273] payload_rows_per_dst=[9462, 10252, 8428, 7410, 10690, 9986, 9716, 9440] dup_rows_per_dst=[2400, 3564, 1504, 1990, 3880, 3524, 3412, 2646] combine_rows_per_src=[9462, 10252, 8428, 7410, 10690, 9986, 9716, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=344824 combine_payload_bytes=267419648 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267764472 ingress_bytes=336625664 link_bytes=336625664 token_bytes=535184120 weight_movement_bytes=69206016 total_nvlink_bytes=604390136 weight_to_token_ratio=0.129313 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266695273e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73434 payload_rows_remote=62456 dup_rows=24870 dup_rows_remote=20798 dedup_ratio=0.252991 router_tokens_per_expert=[1900, 806, 1051, 54, 93, 192, 1919, 1888, 85, 197, 639, 319, 904, 595, 374, 1507, 3177, 206, 693, 1258, 445, 1068, 111, 1412, 125, 295, 811, 231, 864, 1172, 2905, 626, 1587, 798, 205, 121, 472, 750, 231, 380, 1021, 2147, 318, 686, 333, 398, 449, 619, 1272, 861, 663, 339, 97, 1191, 216, 2323, 891, 8, 186, 298, 1062, 553, 536, 219] payload_rows_per_dst=[11200, 10978, 12360, 6878, 7288, 7986, 7826, 8918] dup_rows_per_dst=[5414, 4072, 6160, 1370, 1800, 2176, 1452, 2426] combine_rows_per_src=[11200, 10978, 12360, 6878, 7288, 7986, 7826, 8918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75604 payload_rows_remote=65944 dup_rows=22700 dup_rows_remote=19144 dedup_ratio=0.230916 router_tokens_per_expert=[1721, 671, 776, 60, 99, 227, 1895, 1714, 151, 228, 760, 287, 949, 695, 271, 1695, 2345, 331, 843, 1190, 610, 887, 78, 1131, 175, 266, 1254, 212, 668, 1304, 2672, 791, 1480, 1168, 192, 114, 572, 636, 171, 370, 963, 2761, 574, 757, 246, 438, 477, 392, 991, 1084, 615, 239, 100, 1275, 318, 2579, 830, 26, 234, 301, 845, 757, 521, 170] payload_rows_per_dst=[8512, 8240, 8214, 10686, 10602, 9660, 10840, 8850] dup_rows_per_dst=[2024, 1832, 1926, 3998, 3494, 3556, 3562, 2308] combine_rows_per_src=[8512, 8240, 8214, 10686, 10602, 9660, 10840, 8850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75040 payload_rows_remote=66016 dup_rows=23264 dup_rows_remote=20698 dedup_ratio=0.236654 router_tokens_per_expert=[1855, 773, 898, 56, 105, 284, 1920, 1874, 113, 262, 477, 350, 1082, 720, 532, 1481, 2752, 267, 757, 1110, 518, 1171, 108, 1097, 199, 426, 1154, 255, 814, 1323, 2475, 1007, 1101, 829, 223, 215, 563, 543, 253, 600, 1080, 2062, 294, 723, 377, 422, 450, 491, 1229, 826, 624, 203, 149, 1025, 334, 2133, 795, 97, 210, 424, 1072, 647, 623, 320] payload_rows_per_dst=[10140, 10922, 9024, 8320, 9842, 9590, 7960, 9242] dup_rows_per_dst=[3542, 4062, 2566, 2600, 2782, 2982, 1756, 2974] combine_rows_per_src=[10140, 10922, 9024, 8320, 9842, 9590, 7960, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=336808 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266175400 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532013992 weight_movement_bytes=34603008 total_nvlink_bytes=566617000 weight_to_token_ratio=0.065042 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75160 payload_rows_remote=65758 dup_rows=23144 dup_rows_remote=19952 dedup_ratio=0.235433 router_tokens_per_expert=[1716, 952, 781, 45, 180, 262, 1844, 1896, 144, 298, 705, 270, 1079, 542, 293, 1392, 2359, 287, 508, 1170, 493, 903, 42, 1027, 166, 367, 1251, 192, 795, 1294, 2922, 1137, 1292, 721, 198, 184, 278, 581, 232, 452, 1560, 2413, 497, 649, 269, 478, 379, 337, 1035, 959, 809, 201, 109, 871, 345, 2620, 811, 30, 355, 321, 995, 934, 707, 218] payload_rows_per_dst=[9296, 8054, 7524, 11388, 9402, 9924, 10230, 9342] dup_rows_per_dst=[2368, 1392, 1336, 4860, 3192, 3240, 3668, 3088] combine_rows_per_src=[9296, 8054, 7524, 11388, 9402, 9924, 10230, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73692 payload_rows_remote=64838 dup_rows=24612 dup_rows_remote=20502 dedup_ratio=0.250366 router_tokens_per_expert=[143, 793, 114, 450, 278, 551, 271, 1944, 36, 1179, 259, 1753, 1013, 1267, 283, 674, 1911, 608, 642, 806, 538, 387, 1393, 219, 510, 1679, 2871, 1165, 675, 533, 363, 94, 562, 678, 1719, 1302, 1627, 1062, 1822, 420, 315, 967, 120, 131, 1449, 1716, 144, 567, 967, 959, 707, 320, 561, 548, 185, 139, 1370, 543, 291, 327, 250, 731, 587, 664] payload_rows_per_dst=[7378, 7894, 7250, 11598, 9152, 11096, 10470, 8854] dup_rows_per_dst=[1710, 1934, 1936, 5390, 2998, 3544, 2990, 4110] combine_rows_per_src=[7378, 7894, 7250, 11598, 9152, 11096, 10470, 8854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278421504 dispatch_meta_bytes=355616 combine_payload_bytes=278421504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278777120 ingress_bytes=347627520 link_bytes=347627520 token_bytes=557198624 weight_movement_bytes=69206016 total_nvlink_bytes=626404640 weight_to_token_ratio=0.124203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308094371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270106624 dispatch_meta_bytes=340352 combine_payload_bytes=270106624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270446976 ingress_bytes=339312640 link_bytes=339312640 token_bytes=540553600 weight_movement_bytes=69206016 total_nvlink_bytes=609759616 weight_to_token_ratio=0.128028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276806147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=255819776 dispatch_meta_bytes=333016 combine_payload_bytes=255819776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256152792 ingress_bytes=290422784 link_bytes=290422784 token_bytes=511972568 weight_movement_bytes=34603008 total_nvlink_bytes=546575576 weight_to_token_ratio=0.067588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092837555e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270401536 dispatch_meta_bytes=346856 combine_payload_bytes=270401536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270748392 ingress_bytes=339607552 link_bytes=339607552 token_bytes=541149928 weight_movement_bytes=69206016 total_nvlink_bytes=610355944 weight_to_token_ratio=0.127887 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277915878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269344768 dispatch_meta_bytes=342840 combine_payload_bytes=269344768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269687608 ingress_bytes=303947776 link_bytes=303947776 token_bytes=539032376 weight_movement_bytes=34603008 total_nvlink_bytes=573635384 weight_to_token_ratio=0.064195 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143731011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=341360 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265917808 ingress_bytes=300179456 link_bytes=300179456 token_bytes=531494256 weight_movement_bytes=34603008 total_nvlink_bytes=566097264 weight_to_token_ratio=0.065105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129551126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75542 payload_rows_remote=64940 dup_rows=22762 dup_rows_remote=19214 dedup_ratio=0.231547 router_tokens_per_expert=[1982, 883, 734, 74, 88, 249, 2132, 1540, 121, 265, 773, 318, 845, 525, 324, 1586, 2585, 231, 942, 1151, 628, 843, 76, 1159, 234, 387, 1105, 205, 787, 1257, 2767, 668, 1221, 708, 224, 148, 542, 701, 163, 428, 1228, 2266, 601, 605, 322, 446, 442, 815, 1237, 912, 678, 245, 122, 1298, 330, 2253, 826, 30, 245, 330, 960, 612, 486, 264] payload_rows_per_dst=[8956, 8010, 8280, 10832, 10052, 9602, 10602, 9208] dup_rows_per_dst=[2144, 1504, 1780, 3988, 3388, 3848, 3548, 2562] combine_rows_per_src=[8956, 8010, 8280, 10832, 10052, 9602, 10602, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72394 payload_rows_remote=61834 dup_rows=25910 dup_rows_remote=20966 dedup_ratio=0.263570 router_tokens_per_expert=[2116, 739, 637, 92, 105, 266, 1717, 1607, 121, 217, 620, 393, 1089, 616, 242, 1911, 2524, 284, 732, 1218, 552, 1051, 101, 1188, 169, 382, 1186, 227, 713, 1238, 2495, 784, 1348, 718, 154, 186, 289, 597, 270, 452, 1144, 2382, 697, 654, 293, 701, 502, 505, 1133, 983, 534, 258, 141, 1078, 457, 2190, 1056, 39, 231, 307, 1024, 683, 620, 194] payload_rows_per_dst=[10560, 10936, 13274, 7306, 6484, 7110, 7684, 9040] dup_rows_per_dst=[4944, 4472, 6790, 2092, 1544, 1882, 1484, 2702] combine_rows_per_src=[10560, 10936, 13274, 7306, 6484, 7110, 7684, 9040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75004 payload_rows_remote=65468 dup_rows=23300 dup_rows_remote=19818 dedup_ratio=0.237020 router_tokens_per_expert=[972, 1013, 1100, 305, 608, 573, 898, 712, 104, 688, 853, 627, 528, 1184, 1883, 161, 286, 3518, 60, 968, 33, 885, 2268, 268, 425, 430, 2127, 1494, 1265, 2018, 766, 2, 65, 240, 18, 1719, 358, 351, 472, 633, 285, 186, 283, 1083, 2422, 21, 877, 1003, 359, 423, 734, 1247, 1056, 895, 504, 813, 168, 867, 93, 288, 2083, 83, 116, 385] payload_rows_per_dst=[9818, 8728, 10210, 9536, 8352, 9758, 8668, 9934] dup_rows_per_dst=[2544, 3328, 2908, 3482, 2814, 2562, 3394, 2268] combine_rows_per_src=[9818, 8728, 10210, 9536, 8352, 9758, 8668, 9934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=336616 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266330856 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532325096 weight_movement_bytes=103809024 total_nvlink_bytes=636134120 weight_to_token_ratio=0.195011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=253272064 dispatch_meta_bytes=331200 combine_payload_bytes=253272064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=253603264 ingress_bytes=357081088 link_bytes=357081088 token_bytes=506875328 weight_movement_bytes=103809024 total_nvlink_bytes=610684352 weight_to_token_ratio=0.204802 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343667387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76894 payload_rows_remote=67646 dup_rows=21410 dup_rows_remote=19692 dedup_ratio=0.217794 router_tokens_per_expert=[406, 618, 1475, 431, 335, 358, 585, 1235, 125, 1126, 936, 445, 214, 1122, 2251, 121, 237, 4159, 26, 907, 51, 807, 2650, 215, 605, 495, 2382, 1640, 1473, 2473, 661, 0, 52, 111, 11, 1781, 317, 345, 475, 599, 203, 138, 207, 1159, 2136, 14, 1054, 572, 278, 353, 550, 1302, 1070, 568, 556, 591, 91, 1278, 81, 137, 1954, 31, 47, 527] payload_rows_per_dst=[9048, 9362, 8616, 10346, 11492, 9248, 8126, 10656] dup_rows_per_dst=[1838, 3318, 1170, 4166, 4208, 1718, 2410, 2582] combine_rows_per_src=[9048, 9362, 8616, 10346, 11492, 9248, 8126, 10656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75954 payload_rows_remote=66718 dup_rows=22350 dup_rows_remote=18960 dedup_ratio=0.227356 router_tokens_per_expert=[251, 803, 1349, 503, 431, 431, 552, 1174, 131, 862, 680, 487, 311, 1157, 2087, 88, 243, 4328, 21, 928, 50, 545, 2407, 286, 409, 535, 2339, 1912, 1441, 2519, 777, 0, 37, 70, 16, 2124, 310, 310, 447, 441, 183, 115, 248, 1172, 2227, 18, 1048, 712, 309, 369, 598, 1679, 1097, 642, 496, 663, 64, 1405, 105, 399, 1410, 32, 48, 321] payload_rows_per_dst=[10976, 9236, 12806, 7852, 6380, 9482, 8810, 10412] dup_rows_per_dst=[3670, 3390, 4810, 2296, 1130, 1964, 2896, 2194] combine_rows_per_src=[10976, 9236, 12806, 7852, 6380, 9482, 8810, 10412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75664 payload_rows_remote=63966 dup_rows=22640 dup_rows_remote=18358 dedup_ratio=0.230306 router_tokens_per_expert=[320, 732, 1263, 389, 379, 708, 682, 1162, 213, 883, 790, 441, 314, 1105, 1872, 80, 257, 4222, 31, 900, 65, 607, 2356, 319, 523, 117, 2305, 1934, 1496, 2300, 521, 3, 37, 105, 21, 2010, 364, 375, 319, 537, 143, 117, 328, 1372, 2123, 31, 1325, 1128, 379, 375, 770, 1107, 1118, 815, 365, 604, 87, 1434, 135, 232, 1319, 94, 70, 624] payload_rows_per_dst=[9062, 8442, 7876, 10500, 11698, 9784, 8156, 10146] dup_rows_per_dst=[2208, 2954, 1194, 3298, 4282, 3350, 2910, 2444] combine_rows_per_src=[9062, 8442, 7876, 10500, 11698, 9784, 8156, 10146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75316 payload_rows_remote=63154 dup_rows=22988 dup_rows_remote=18844 dedup_ratio=0.233846 router_tokens_per_expert=[370, 740, 1211, 388, 399, 534, 780, 965, 164, 936, 716, 444, 280, 1374, 1797, 97, 238, 3580, 32, 1062, 52, 455, 2415, 319, 448, 77, 2331, 1935, 2016, 2174, 575, 1, 67, 112, 8, 2302, 410, 294, 393, 587, 258, 187, 253, 1818, 1915, 38, 1453, 1091, 412, 356, 575, 1287, 945, 1106, 580, 734, 154, 569, 166, 236, 1465, 55, 53, 368] payload_rows_per_dst=[8708, 9452, 12162, 10418, 6870, 10016, 8854, 8836] dup_rows_per_dst=[2066, 3516, 4144, 2996, 1476, 4010, 3136, 1644] combine_rows_per_src=[8708, 9452, 12162, 10418, 6870, 10016, 8854, 8836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=341144 combine_payload_bytes=268156928 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=268498072 ingress_bytes=441171968 link_bytes=441171968 token_bytes=536655000 weight_movement_bytes=173015040 total_nvlink_bytes=709670040 weight_to_token_ratio=0.322395 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.660094598e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=63326 dup_rows=24084 dup_rows_remote=19420 dedup_ratio=0.244995 router_tokens_per_expert=[1994, 696, 824, 74, 120, 275, 1976, 1504, 128, 192, 762, 309, 783, 685, 334, 1814, 2572, 308, 732, 1165, 495, 860, 89, 920, 294, 334, 1166, 226, 774, 1269, 2361, 869, 1375, 751, 249, 143, 566, 618, 317, 341, 1099, 2754, 506, 523, 224, 513, 477, 523, 1185, 903, 1009, 322, 75, 879, 435, 2474, 843, 51, 189, 312, 907, 634, 799, 252] payload_rows_per_dst=[6530, 8290, 7658, 10454, 10266, 9668, 10460, 10894] dup_rows_per_dst=[812, 1724, 1480, 4132, 3598, 3570, 4104, 4664] combine_rows_per_src=[6530, 8290, 7658, 10454, 10266, 9668, 10460, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277078016 dispatch_meta_bytes=349352 combine_payload_bytes=277078016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277427368 ingress_bytes=346284032 link_bytes=346284032 token_bytes=554505384 weight_movement_bytes=69206016 total_nvlink_bytes=623711400 weight_to_token_ratio=0.124807 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303038934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273276928 dispatch_meta_bytes=342712 combine_payload_bytes=273276928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273619640 ingress_bytes=307879936 link_bytes=307879936 token_bytes=546896568 weight_movement_bytes=34603008 total_nvlink_bytes=581499576 weight_to_token_ratio=0.063272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158527413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258678784 dispatch_meta_bytes=327992 combine_payload_bytes=258678784 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259006776 ingress_bytes=362487808 link_bytes=362487808 token_bytes=517685560 weight_movement_bytes=103809024 total_nvlink_bytes=621494584 weight_to_token_ratio=0.200525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364012439e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262004736 dispatch_meta_bytes=329296 combine_payload_bytes=262004736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262334032 ingress_bytes=296607744 link_bytes=296607744 token_bytes=524338768 weight_movement_bytes=34603008 total_nvlink_bytes=558941776 weight_to_token_ratio=0.065994 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116111062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259383296 dispatch_meta_bytes=330984 combine_payload_bytes=259383296 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259714280 ingress_bytes=328589312 link_bytes=328589312 token_bytes=519097576 weight_movement_bytes=69206016 total_nvlink_bytes=588303592 weight_to_token_ratio=0.133320 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236455127e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75632 payload_rows_remote=67294 dup_rows=22672 dup_rows_remote=19592 dedup_ratio=0.230632 router_tokens_per_expert=[402, 854, 970, 467, 672, 400, 697, 1147, 201, 712, 525, 487, 289, 1159, 2405, 108, 287, 3948, 41, 821, 57, 532, 2394, 256, 651, 383, 1890, 1902, 1499, 2487, 864, 1, 33, 189, 6, 2142, 383, 381, 418, 451, 260, 149, 195, 1210, 1891, 30, 1627, 1195, 267, 398, 837, 1177, 803, 806, 455, 966, 124, 1199, 59, 156, 1259, 87, 88, 403] payload_rows_per_dst=[9078, 8962, 7792, 10336, 11686, 9600, 8338, 9840] dup_rows_per_dst=[2140, 2810, 984, 3810, 4216, 3514, 3080, 2118] combine_rows_per_src=[9078, 8962, 7792, 10336, 11686, 9600, 8338, 9840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75032 payload_rows_remote=62930 dup_rows=23272 dup_rows_remote=19040 dedup_ratio=0.236735 router_tokens_per_expert=[426, 696, 1067, 348, 779, 519, 709, 1154, 185, 538, 952, 548, 247, 1261, 2160, 162, 254, 4002, 34, 944, 31, 434, 2895, 277, 544, 538, 2469, 1637, 1750, 2125, 613, 0, 38, 157, 7, 2202, 459, 399, 453, 405, 243, 121, 199, 1108, 2185, 29, 1133, 779, 362, 466, 552, 1635, 948, 667, 322, 691, 112, 1124, 73, 212, 1232, 92, 81, 368] payload_rows_per_dst=[12102, 8886, 13028, 7938, 6670, 8564, 8430, 9414] dup_rows_per_dst=[4232, 3220, 5926, 2226, 1570, 1818, 2856, 1424] combine_rows_per_src=[12102, 8886, 13028, 7938, 6670, 8564, 8430, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=65804 dup_rows=20848 dup_rows_remote=17120 dedup_ratio=0.212077 router_tokens_per_expert=[273, 1246, 544, 33, 582, 747, 227, 3291, 680, 124, 181, 2176, 401, 611, 1637, 373, 2851, 2824, 64, 156, 1683, 40, 1371, 41, 91, 47, 582, 956, 0, 564, 1501, 60, 652, 31, 379, 846, 10, 67, 3889, 2273, 1651, 163, 43, 186, 16, 496, 1187, 276, 121, 308, 2079, 2589, 660, 1084, 75, 519, 122, 822, 1130, 63, 64, 693, 10, 691] payload_rows_per_dst=[8198, 9644, 10348, 11652, 7340, 9566, 10938, 9770] dup_rows_per_dst=[1724, 2722, 2010, 3728, 1176, 2434, 3932, 3122] combine_rows_per_src=[8198, 9644, 10348, 11652, 7340, 9566, 10938, 9770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275636224 dispatch_meta_bytes=347544 combine_payload_bytes=275636224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275983768 ingress_bytes=310239232 link_bytes=310239232 token_bytes=551619992 weight_movement_bytes=34603008 total_nvlink_bytes=586223000 weight_to_token_ratio=0.062730 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167405254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77278 payload_rows_remote=65982 dup_rows=21026 dup_rows_remote=17336 dedup_ratio=0.213888 router_tokens_per_expert=[235, 1001, 592, 51, 559, 674, 193, 3712, 601, 136, 205, 2082, 273, 913, 1224, 220, 3623, 3758, 63, 297, 1518, 39, 1658, 12, 141, 67, 575, 464, 0, 644, 1696, 48, 661, 15, 126, 842, 6, 79, 3524, 2632, 1663, 141, 4, 229, 7, 542, 956, 239, 122, 477, 1821, 2666, 684, 833, 38, 230, 145, 729, 1045, 30, 26, 647, 5, 714] payload_rows_per_dst=[6032, 8772, 11204, 6514, 12094, 11296, 10696, 10670] dup_rows_per_dst=[578, 2536, 3486, 756, 3676, 3690, 3046, 3258] combine_rows_per_src=[6032, 8772, 11204, 6514, 12094, 11296, 10696, 10670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=257761280 dispatch_meta_bytes=327880 combine_payload_bytes=257761280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258089160 ingress_bytes=292364288 link_bytes=292364288 token_bytes=515850440 weight_movement_bytes=34603008 total_nvlink_bytes=550453448 weight_to_token_ratio=0.067080 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100143278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75932 payload_rows_remote=67612 dup_rows=22372 dup_rows_remote=20368 dedup_ratio=0.227580 router_tokens_per_expert=[203, 975, 776, 32, 677, 999, 98, 3877, 460, 111, 130, 1797, 283, 720, 1195, 466, 3355, 2919, 72, 212, 1666, 51, 1828, 45, 163, 50, 739, 1003, 1, 493, 1260, 61, 753, 22, 400, 662, 41, 104, 3623, 2401, 1628, 124, 19, 365, 8, 582, 945, 270, 187, 723, 1913, 2660, 448, 774, 96, 623, 95, 747, 907, 41, 33, 591, 19, 631] payload_rows_per_dst=[13412, 8320, 10608, 11134, 7446, 6800, 8110, 10102] dup_rows_per_dst=[7182, 2004, 2978, 3652, 1320, 1082, 1418, 2736] combine_rows_per_src=[13412, 8320, 10608, 11134, 7446, 6800, 8110, 10102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269533184 dispatch_meta_bytes=331696 combine_payload_bytes=269533184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269864880 ingress_bytes=304136192 link_bytes=304136192 token_bytes=539398064 weight_movement_bytes=34603008 total_nvlink_bytes=574001072 weight_to_token_ratio=0.064151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144440006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77642 payload_rows_remote=69874 dup_rows=20662 dup_rows_remote=19262 dedup_ratio=0.210185 router_tokens_per_expert=[187, 1051, 843, 42, 628, 917, 248, 3832, 336, 89, 223, 2321, 257, 1432, 1369, 363, 3515, 2549, 45, 132, 1178, 67, 1770, 72, 100, 62, 650, 597, 6, 783, 1362, 55, 826, 35, 233, 494, 26, 113, 3798, 2583, 1550, 126, 36, 168, 11, 374, 783, 448, 166, 790, 1652, 2446, 447, 787, 69, 448, 132, 667, 1187, 54, 41, 724, 10, 847] payload_rows_per_dst=[7020, 9658, 9518, 11042, 7768, 11068, 10614, 10954] dup_rows_per_dst=[812, 3122, 2108, 3236, 1400, 3588, 2996, 3400] combine_rows_per_src=[7020, 9658, 9518, 11042, 7768, 11068, 10614, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76636 payload_rows_remote=65502 dup_rows=21668 dup_rows_remote=18882 dedup_ratio=0.220418 router_tokens_per_expert=[247, 877, 714, 74, 479, 866, 276, 3654, 490, 127, 88, 1788, 355, 831, 1330, 754, 3473, 2412, 62, 404, 2365, 78, 1480, 159, 93, 62, 865, 748, 9, 571, 1242, 123, 826, 75, 314, 505, 21, 146, 3673, 1936, 1667, 125, 29, 352, 34, 657, 1081, 673, 407, 537, 1245, 2891, 296, 855, 127, 596, 77, 779, 675, 54, 30, 827, 31, 515] payload_rows_per_dst=[13092, 8958, 11134, 11154, 6712, 7640, 7552, 10394] dup_rows_per_dst=[6478, 2568, 2786, 3618, 934, 1596, 1160, 2528] combine_rows_per_src=[13092, 8958, 11134, 11154, 6712, 7640, 7552, 10394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74714 payload_rows_remote=63886 dup_rows=23590 dup_rows_remote=20026 dedup_ratio=0.239970 router_tokens_per_expert=[491, 714, 1197, 288, 596, 345, 720, 1056, 133, 963, 770, 420, 305, 1269, 2380, 110, 212, 3861, 26, 974, 58, 468, 3508, 358, 493, 98, 2234, 1811, 1280, 2141, 541, 0, 42, 170, 8, 1991, 335, 407, 397, 519, 211, 147, 164, 1465, 1974, 36, 1579, 788, 336, 342, 428, 1626, 1068, 701, 586, 768, 228, 950, 103, 205, 1255, 28, 103, 372] payload_rows_per_dst=[8876, 9384, 4354, 7578, 11580, 9852, 12262, 10828] dup_rows_per_dst=[1938, 3316, 324, 1526, 3880, 3064, 5978, 3564] combine_rows_per_src=[8876, 9384, 4354, 7578, 11580, 9852, 12262, 10828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270262272 dispatch_meta_bytes=333272 combine_payload_bytes=270262272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270595544 ingress_bytes=304865280 link_bytes=304865280 token_bytes=540857816 weight_movement_bytes=34603008 total_nvlink_bytes=575460824 weight_to_token_ratio=0.063978 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147183505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276938752 dispatch_meta_bytes=351920 combine_payload_bytes=276938752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277290672 ingress_bytes=311541760 link_bytes=311541760 token_bytes=554229424 weight_movement_bytes=34603008 total_nvlink_bytes=588832432 weight_to_token_ratio=0.062434 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172306562e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=286203904 dispatch_meta_bytes=356544 combine_payload_bytes=286203904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286560448 ingress_bytes=355409920 link_bytes=355409920 token_bytes=572764352 weight_movement_bytes=69206016 total_nvlink_bytes=641970368 weight_to_token_ratio=0.120828 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337378916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=268296192 dispatch_meta_bytes=337536 combine_payload_bytes=268296192 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268633728 ingress_bytes=406708224 link_bytes=406708224 token_bytes=536929920 weight_movement_bytes=138412032 total_nvlink_bytes=675341952 weight_to_token_ratio=0.257784 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530410304e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261677056 dispatch_meta_bytes=335648 combine_payload_bytes=261677056 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262012704 ingress_bytes=330883072 link_bytes=330883072 token_bytes=523689760 weight_movement_bytes=69206016 total_nvlink_bytes=592895776 weight_to_token_ratio=0.132151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245086362e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75236 payload_rows_remote=64426 dup_rows=23068 dup_rows_remote=19894 dedup_ratio=0.234660 router_tokens_per_expert=[255, 843, 665, 55, 582, 757, 177, 3998, 515, 85, 191, 2097, 374, 920, 1142, 578, 3054, 2730, 67, 229, 1346, 24, 1753, 43, 102, 30, 792, 845, 2, 847, 1480, 40, 686, 45, 375, 760, 27, 58, 3806, 2958, 1801, 157, 26, 295, 13, 414, 1262, 200, 150, 706, 1996, 2144, 680, 927, 47, 342, 139, 509, 770, 63, 58, 566, 19, 535] payload_rows_per_dst=[6056, 8900, 7320, 6944, 14078, 11936, 10810, 9192] dup_rows_per_dst=[612, 2904, 1096, 1332, 7320, 4396, 3174, 2234] combine_rows_per_src=[6056, 8900, 7320, 6944, 14078, 11936, 10810, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75298 payload_rows_remote=62204 dup_rows=23006 dup_rows_remote=16636 dedup_ratio=0.234029 router_tokens_per_expert=[132, 908, 579, 70, 777, 686, 253, 3954, 435, 191, 129, 2105, 360, 540, 1761, 484, 3429, 3008, 55, 154, 1650, 57, 1813, 52, 137, 53, 664, 591, 0, 889, 1810, 41, 728, 32, 268, 627, 16, 92, 3667, 2563, 1588, 225, 19, 258, 24, 368, 1019, 180, 202, 259, 1946, 2373, 443, 915, 105, 342, 148, 547, 1123, 106, 34, 624, 12, 532] payload_rows_per_dst=[13094, 11176, 11468, 11642, 7478, 6628, 7496, 6316] dup_rows_per_dst=[6370, 3786, 4628, 4062, 1174, 734, 928, 1324] combine_rows_per_src=[13094, 11176, 11468, 11642, 7478, 6628, 7496, 6316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78808 payload_rows_remote=68092 dup_rows=19496 dup_rows_remote=17170 dedup_ratio=0.198324 router_tokens_per_expert=[1161, 55, 233, 4622, 316, 2, 116, 1260, 552, 906, 961, 359, 4379, 1901, 26, 1411, 667, 20, 18, 944, 35, 408, 969, 596, 18, 749, 128, 737, 263, 123, 23, 101, 16, 277, 936, 396, 712, 1, 137, 412, 492, 5340, 39, 2274, 2374, 41, 2237, 287, 238, 1798, 159, 906, 2442, 1709, 892, 63, 266, 58, 12, 44, 33, 3, 474, 25] payload_rows_per_dst=[11812, 7642, 6132, 10716, 8784, 10764, 11580, 11378] dup_rows_per_dst=[3718, 1378, 1182, 2326, 1738, 3188, 4834, 1132] combine_rows_per_src=[11812, 7642, 6132, 10716, 8784, 10764, 11580, 11378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263888896 dispatch_meta_bytes=337280 combine_payload_bytes=263888896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264226176 ingress_bytes=333094912 link_bytes=333094912 token_bytes=528115072 weight_movement_bytes=69206016 total_nvlink_bytes=597321088 weight_to_token_ratio=0.131043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253409338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80850 payload_rows_remote=69308 dup_rows=17454 dup_rows_remote=14252 dedup_ratio=0.177551 router_tokens_per_expert=[1717, 36, 95, 4787, 864, 5, 198, 1045, 535, 823, 757, 430, 5090, 2706, 9, 1453, 707, 19, 34, 960, 33, 600, 920, 660, 23, 739, 170, 1056, 557, 29, 34, 14, 13, 227, 880, 170, 359, 0, 29, 371, 891, 5680, 5, 1988, 3157, 18, 1528, 236, 89, 617, 183, 808, 2554, 715, 325, 4, 225, 37, 29, 29, 39, 1, 821, 19] payload_rows_per_dst=[7114, 7048, 12956, 12284, 8840, 11542, 8974, 12092] dup_rows_per_dst=[806, 966, 4484, 3140, 1572, 3202, 1616, 1668] combine_rows_per_src=[7114, 7048, 12956, 12284, 8840, 11542, 8974, 12092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=254787584 dispatch_meta_bytes=315360 combine_payload_bytes=254787584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255102944 ingress_bytes=323993600 link_bytes=323993600 token_bytes=509890528 weight_movement_bytes=69206016 total_nvlink_bytes=579096544 weight_to_token_ratio=0.135727 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219161833e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75528 payload_rows_remote=62142 dup_rows=22776 dup_rows_remote=14460 dedup_ratio=0.231689 router_tokens_per_expert=[1278, 65, 344, 4384, 730, 3, 158, 903, 371, 1172, 795, 450, 4186, 2043, 25, 1809, 727, 93, 29, 1031, 27, 624, 901, 1071, 28, 736, 311, 1167, 404, 24, 43, 45, 27, 286, 1263, 328, 790, 5, 87, 395, 805, 5341, 31, 2029, 2973, 56, 1656, 205, 169, 537, 127, 726, 2435, 657, 899, 19, 233, 33, 31, 56, 56, 1, 902, 17] payload_rows_per_dst=[11772, 13386, 7278, 5040, 11834, 5294, 9402, 11522] dup_rows_per_dst=[3958, 8316, 1728, 476, 4532, 212, 1736, 1818] combine_rows_per_src=[11772, 13386, 7278, 5040, 11834, 5294, 9402, 11522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278904832 dispatch_meta_bytes=341048 combine_payload_bytes=278904832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279245880 ingress_bytes=313507840 link_bytes=313507840 token_bytes=558150712 weight_movement_bytes=34603008 total_nvlink_bytes=592753720 weight_to_token_ratio=0.061996 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179704762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79062 payload_rows_remote=70902 dup_rows=19242 dup_rows_remote=17710 dedup_ratio=0.195740 router_tokens_per_expert=[1410, 59, 174, 4090, 701, 9, 137, 1376, 367, 687, 783, 345, 5207, 2347, 17, 1465, 729, 39, 14, 1478, 48, 1113, 617, 854, 19, 847, 357, 929, 279, 46, 28, 38, 21, 363, 788, 177, 597, 1, 106, 381, 771, 5988, 17, 2089, 2412, 62, 2015, 250, 129, 357, 103, 938, 1693, 1558, 272, 35, 203, 146, 11, 72, 60, 2, 915, 11] payload_rows_per_dst=[8468, 6762, 10792, 12272, 8160, 11460, 8362, 12786] dup_rows_per_dst=[1902, 974, 4126, 3228, 1532, 3642, 1808, 2030] combine_rows_per_src=[8468, 6762, 10792, 12272, 8160, 11460, 8362, 12786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77830 payload_rows_remote=70788 dup_rows=20474 dup_rows_remote=18756 dedup_ratio=0.208272 router_tokens_per_expert=[1437, 92, 368, 3846, 1142, 33, 113, 975, 353, 1055, 830, 392, 3997, 1875, 29, 1821, 629, 95, 120, 898, 37, 810, 759, 1032, 13, 798, 344, 1186, 374, 68, 32, 49, 32, 193, 819, 495, 758, 4, 113, 765, 1046, 6061, 17, 2582, 2835, 62, 1219, 262, 159, 531, 230, 779, 2044, 690, 348, 24, 300, 86, 3, 26, 74, 3, 964, 26] payload_rows_per_dst=[11934, 9854, 7042, 10310, 11206, 6588, 8004, 12892] dup_rows_per_dst=[4078, 3174, 1718, 3094, 3906, 704, 1606, 2194] combine_rows_per_src=[11934, 9854, 7042, 10310, 11206, 6588, 8004, 12892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283885568 dispatch_meta_bytes=334240 combine_payload_bytes=283885568 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=284219808 ingress_bytes=422297600 link_bytes=422297600 token_bytes=568105376 weight_movement_bytes=138412032 total_nvlink_bytes=706517408 weight_to_token_ratio=0.243638 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589071871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75504 payload_rows_remote=65682 dup_rows=22800 dup_rows_remote=20318 dedup_ratio=0.231934 router_tokens_per_expert=[108, 835, 829, 75, 639, 1025, 311, 3514, 562, 100, 170, 2266, 384, 1032, 1601, 502, 3245, 2396, 43, 214, 1959, 57, 1463, 70, 99, 109, 812, 825, 2, 441, 1553, 78, 694, 27, 220, 539, 25, 102, 3812, 2170, 1677, 172, 4, 366, 6, 447, 1365, 374, 299, 377, 1538, 2520, 647, 1014, 116, 415, 144, 492, 691, 60, 69, 766, 10, 675] payload_rows_per_dst=[6960, 9678, 6850, 6672, 13134, 11526, 10862, 9822] dup_rows_per_dst=[684, 3556, 762, 1166, 5254, 4324, 4572, 2482] combine_rows_per_src=[6960, 9678, 6850, 6672, 13134, 11526, 10862, 9822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=254533632 dispatch_meta_bytes=306408 combine_payload_bytes=254533632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=254840040 ingress_bytes=358342656 link_bytes=358342656 token_bytes=509373672 weight_movement_bytes=103809024 total_nvlink_bytes=613182696 weight_to_token_ratio=0.203797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348414566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290414592 dispatch_meta_bytes=354448 combine_payload_bytes=290414592 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=290769040 ingress_bytes=359620608 link_bytes=359620608 token_bytes=581183632 weight_movement_bytes=69206016 total_nvlink_bytes=650389648 weight_to_token_ratio=0.119078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353223396e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289947648 dispatch_meta_bytes=358176 combine_payload_bytes=289947648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290305824 ingress_bytes=324550656 link_bytes=324550656 token_bytes=580253472 weight_movement_bytes=34603008 total_nvlink_bytes=614856480 weight_to_token_ratio=0.059634 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221257990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79226 payload_rows_remote=68768 dup_rows=19078 dup_rows_remote=15690 dedup_ratio=0.194071 router_tokens_per_expert=[1827, 33, 297, 3762, 669, 2, 215, 1238, 439, 729, 825, 252, 4983, 1740, 17, 836, 661, 27, 19, 1070, 26, 553, 864, 1149, 8, 882, 271, 962, 337, 45, 49, 46, 26, 448, 1521, 184, 650, 0, 31, 499, 719, 5567, 10, 2040, 2466, 49, 1834, 312, 181, 1709, 179, 705, 2193, 1205, 717, 34, 219, 74, 6, 54, 17, 4, 651, 15] payload_rows_per_dst=[7354, 5596, 12118, 11954, 9396, 10556, 10458, 11794] dup_rows_per_dst=[1208, 600, 4144, 3212, 2254, 2852, 3388, 1420] combine_rows_per_src=[7354, 5596, 12118, 11954, 9396, 10556, 10458, 11794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269033472 dispatch_meta_bytes=344000 combine_payload_bytes=269033472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269377472 ingress_bytes=303636480 link_bytes=303636480 token_bytes=538410944 weight_movement_bytes=34603008 total_nvlink_bytes=573013952 weight_to_token_ratio=0.064269 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142559629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73708 payload_rows_remote=61956 dup_rows=24596 dup_rows_remote=20418 dedup_ratio=0.250203 router_tokens_per_expert=[1451, 92, 189, 4376, 820, 27, 112, 898, 353, 725, 853, 636, 4701, 2811, 48, 1866, 732, 40, 26, 1451, 40, 524, 622, 722, 45, 852, 466, 1348, 523, 61, 60, 47, 20, 349, 929, 94, 685, 2, 65, 301, 265, 4819, 15, 2321, 2245, 70, 1632, 260, 122, 783, 59, 577, 1714, 2397, 939, 25, 201, 155, 6, 87, 52, 3, 434, 9] payload_rows_per_dst=[11752, 14224, 6932, 5998, 10848, 11112, 10400, 2442] dup_rows_per_dst=[4178, 9762, 1382, 806, 3174, 2316, 2832, 146] combine_rows_per_src=[11752, 14224, 6932, 5998, 10848, 11112, 10400, 2442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74686 payload_rows_remote=65150 dup_rows=23618 dup_rows_remote=19696 dedup_ratio=0.240255 router_tokens_per_expert=[1306, 359, 22, 67, 2361, 37, 399, 584, 66, 274, 4811, 40, 165, 1588, 724, 1130, 2277, 72, 150, 48, 275, 845, 14, 306, 131, 1262, 437, 495, 525, 7128, 1593, 19, 569, 22, 820, 401, 1267, 274, 151, 25, 1518, 935, 0, 74, 161, 619, 301, 24, 2783, 287, 28, 155, 695, 258, 2144, 1608, 255, 792, 37, 580, 13, 591, 1501, 754] payload_rows_per_dst=[7938, 11012, 13972, 9536, 6908, 6314, 7962, 11044] dup_rows_per_dst=[2332, 3466, 5878, 3922, 1114, 950, 2388, 3568] combine_rows_per_src=[7938, 11012, 13972, 9536, 6908, 6314, 7962, 11044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281673728 dispatch_meta_bytes=337832 combine_payload_bytes=281673728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282011560 ingress_bytes=316276736 link_bytes=316276736 token_bytes=563685288 weight_movement_bytes=34603008 total_nvlink_bytes=598288296 weight_to_token_ratio=0.061387 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190123895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77194 payload_rows_remote=61862 dup_rows=21110 dup_rows_remote=15754 dedup_ratio=0.214742 router_tokens_per_expert=[1174, 702, 77, 28, 2336, 16, 275, 320, 20, 156, 5108, 46, 151, 1574, 455, 1320, 2974, 78, 61, 89, 285, 296, 11, 465, 291, 999, 415, 740, 401, 7262, 2128, 16, 262, 49, 1627, 451, 1732, 255, 85, 10, 1399, 671, 0, 32, 114, 725, 129, 12, 2475, 341, 4, 243, 376, 335, 2801, 1192, 151, 519, 9, 618, 2, 612, 1268, 384] payload_rows_per_dst=[7786, 6136, 7584, 8348, 13610, 15332, 8508, 9890] dup_rows_per_dst=[2070, 1308, 934, 1632, 5548, 5356, 2076, 2186] combine_rows_per_src=[7786, 6136, 7584, 8348, 13610, 15332, 8508, 9890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=253771776 dispatch_meta_bytes=329496 combine_payload_bytes=253771776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=254101272 ingress_bytes=357580800 link_bytes=357580800 token_bytes=507873048 weight_movement_bytes=103809024 total_nvlink_bytes=611682072 weight_to_token_ratio=0.204400 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345547763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75794 payload_rows_remote=61892 dup_rows=22510 dup_rows_remote=17042 dedup_ratio=0.228984 router_tokens_per_expert=[1734, 665, 64, 62, 1442, 46, 213, 302, 13, 231, 5741, 29, 39, 1501, 474, 1657, 1983, 51, 155, 86, 230, 605, 17, 680, 215, 1450, 668, 615, 297, 7176, 1884, 15, 409, 45, 1122, 411, 1919, 315, 87, 10, 1138, 866, 2, 124, 108, 788, 141, 24, 2796, 251, 9, 34, 137, 435, 2171, 1325, 77, 750, 15, 1061, 10, 471, 1149, 612] payload_rows_per_dst=[11072, 13902, 11296, 8656, 7456, 5506, 10744, 7162] dup_rows_per_dst=[4714, 5468, 3940, 1632, 1180, 876, 3572, 1128] combine_rows_per_src=[11072, 13902, 11296, 8656, 7456, 5506, 10744, 7162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=339384 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267193784 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534048184 weight_movement_bytes=138412032 total_nvlink_bytes=672460216 weight_to_token_ratio=0.259175 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74898 payload_rows_remote=62034 dup_rows=23406 dup_rows_remote=18974 dedup_ratio=0.238098 router_tokens_per_expert=[2331, 696, 24, 82, 1717, 24, 642, 245, 14, 155, 5262, 23, 51, 1397, 741, 1263, 2701, 88, 73, 55, 397, 380, 10, 165, 308, 1340, 331, 439, 221, 7289, 2205, 3, 288, 61, 832, 388, 1231, 272, 294, 20, 1246, 824, 4, 61, 119, 517, 604, 14, 3115, 276, 3, 25, 123, 297, 2013, 1145, 102, 865, 11, 1083, 32, 508, 1636, 471] payload_rows_per_dst=[8896, 6200, 6696, 13588, 12864, 8534, 6494, 11626] dup_rows_per_dst=[2626, 1088, 1042, 7082, 4432, 1846, 1270, 4020] combine_rows_per_src=[8896, 6200, 6696, 13588, 12864, 8534, 6494, 11626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=60960 dup_rows=21984 dup_rows_remote=15484 dedup_ratio=0.223633 router_tokens_per_expert=[1340, 667, 57, 40, 1239, 32, 414, 148, 41, 393, 4628, 29, 24, 1859, 470, 938, 2145, 118, 131, 94, 418, 433, 12, 339, 312, 1519, 292, 680, 462, 7240, 2576, 91, 329, 275, 1101, 334, 1587, 499, 288, 21, 2137, 720, 8, 23, 200, 344, 118, 32, 3089, 745, 26, 222, 248, 411, 2228, 955, 179, 820, 10, 485, 7, 481, 1422, 627] payload_rows_per_dst=[6418, 12532, 15360, 9722, 7468, 6438, 8574, 9808] dup_rows_per_dst=[1456, 4232, 6500, 2142, 1400, 726, 2804, 2724] combine_rows_per_src=[6418, 12532, 15360, 9722, 7468, 6438, 8574, 9808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253386752 dispatch_meta_bytes=310464 combine_payload_bytes=253386752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253697216 ingress_bytes=287989760 link_bytes=287989760 token_bytes=507083968 weight_movement_bytes=34603008 total_nvlink_bytes=541686976 weight_to_token_ratio=0.068239 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.083682282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=67506 dup_rows=19650 dup_rows_remote=17808 dedup_ratio=0.199890 router_tokens_per_expert=[1436, 55, 255, 4420, 1137, 4, 188, 1050, 505, 725, 847, 547, 4094, 1813, 64, 1623, 775, 38, 78, 1094, 26, 840, 842, 824, 39, 900, 334, 977, 280, 27, 45, 28, 46, 318, 860, 274, 785, 2, 157, 303, 726, 5146, 20, 2624, 2519, 91, 1910, 252, 257, 620, 131, 827, 2096, 1069, 811, 49, 253, 89, 9, 37, 82, 6, 860, 13] payload_rows_per_dst=[7238, 7550, 12010, 10458, 8888, 11328, 10034, 11148] dup_rows_per_dst=[1012, 1072, 4280, 2990, 1640, 3544, 3270, 1842] combine_rows_per_src=[7238, 7550, 12010, 10458, 8888, 11328, 10034, 11148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253509632 dispatch_meta_bytes=315736 combine_payload_bytes=253509632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=253825368 ingress_bytes=357318656 link_bytes=357318656 token_bytes=507335000 weight_movement_bytes=103809024 total_nvlink_bytes=611144024 weight_to_token_ratio=0.204616 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344561336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=254091264 dispatch_meta_bytes=324032 combine_payload_bytes=254091264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254415296 ingress_bytes=288694272 link_bytes=288694272 token_bytes=508506560 weight_movement_bytes=34603008 total_nvlink_bytes=543109568 weight_to_token_ratio=0.068048 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086333304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=249692160 dispatch_meta_bytes=305776 combine_payload_bytes=249692160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249997936 ingress_bytes=284295168 link_bytes=284295168 token_bytes=499690096 weight_movement_bytes=34603008 total_nvlink_bytes=534293104 weight_to_token_ratio=0.069249 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069779830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76362 payload_rows_remote=68412 dup_rows=21942 dup_rows_remote=20172 dedup_ratio=0.223206 router_tokens_per_expert=[1504, 496, 21, 74, 2044, 24, 244, 552, 14, 163, 5462, 16, 65, 1397, 462, 1620, 2694, 51, 76, 40, 294, 488, 20, 325, 352, 1075, 584, 828, 216, 6874, 1737, 7, 418, 51, 1401, 565, 1559, 263, 89, 10, 1333, 751, 0, 46, 176, 615, 252, 7, 2497, 325, 11, 53, 242, 352, 2599, 1245, 99, 833, 35, 972, 8, 546, 1326, 654] payload_rows_per_dst=[7664, 6322, 7012, 8078, 13904, 14836, 7950, 10596] dup_rows_per_dst=[2254, 1152, 964, 1520, 5666, 5272, 1770, 3344] combine_rows_per_src=[7664, 6322, 7012, 8078, 13904, 14836, 7950, 10596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276504576 dispatch_meta_bytes=341256 combine_payload_bytes=276504576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276845832 ingress_bytes=311107584 link_bytes=311107584 token_bytes=553350408 weight_movement_bytes=34603008 total_nvlink_bytes=587953416 weight_to_token_ratio=0.062534 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170672792e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76396 payload_rows_remote=61010 dup_rows=21908 dup_rows_remote=14764 dedup_ratio=0.222860 router_tokens_per_expert=[1702, 354, 29, 48, 1161, 27, 691, 155, 16, 266, 5313, 20, 129, 1292, 668, 1417, 3364, 74, 69, 103, 203, 649, 12, 288, 174, 1463, 309, 797, 289, 7098, 2032, 17, 283, 47, 1246, 367, 1307, 418, 252, 10, 1166, 1603, 3, 71, 53, 577, 88, 10, 2462, 537, 32, 20, 388, 281, 2547, 1121, 126, 917, 24, 583, 5, 613, 1325, 441] payload_rows_per_dst=[15386, 13398, 8384, 8612, 6928, 6030, 10770, 6888] dup_rows_per_dst=[7144, 4844, 1140, 1550, 932, 1112, 4006, 1180] combine_rows_per_src=[15386, 13398, 8384, 8612, 6928, 6030, 10770, 6888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78776 payload_rows_remote=69188 dup_rows=19528 dup_rows_remote=17306 dedup_ratio=0.198649 router_tokens_per_expert=[4046, 343, 405, 491, 24, 359, 1392, 1032, 89, 0, 0, 1814, 1, 958, 3, 403, 58, 8, 1253, 74, 195, 17, 70, 178, 761, 180, 3014, 6, 10, 3820, 854, 29, 218, 1553, 110, 2214, 297, 676, 693, 0, 35, 3063, 78, 2, 3536, 320, 22, 2612, 409, 51, 14, 441, 470, 930, 18, 11, 119, 2020, 439, 749, 16, 503, 5386, 260] payload_rows_per_dst=[10740, 10528, 12246, 9588, 9456, 9256, 9864, 7098] dup_rows_per_dst=[3342, 3648, 2232, 2222, 2066, 3008, 1896, 1114] combine_rows_per_src=[10740, 10528, 12246, 9588, 9456, 9256, 9864, 7098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76818 payload_rows_remote=67648 dup_rows=21486 dup_rows_remote=19488 dedup_ratio=0.218567 router_tokens_per_expert=[4493, 276, 150, 706, 4, 535, 603, 1721, 61, 0, 0, 1815, 1, 2072, 0, 721, 34, 2, 1717, 9, 227, 38, 53, 101, 1082, 57, 2170, 7, 0, 4015, 825, 17, 88, 1051, 34, 2163, 26, 347, 702, 0, 48, 3423, 41, 8, 4052, 240, 5, 1819, 169, 14, 23, 215, 185, 696, 12, 3, 35, 1965, 317, 491, 1, 1020, 5750, 697] payload_rows_per_dst=[6946, 9008, 10116, 10796, 11996, 9170, 9412, 9374] dup_rows_per_dst=[1044, 2214, 1998, 3668, 5812, 1998, 1326, 3426] combine_rows_per_src=[6946, 9008, 10116, 10796, 11996, 9170, 9412, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280215552 dispatch_meta_bytes=354336 combine_payload_bytes=280215552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280569888 ingress_bytes=349421568 link_bytes=349421568 token_bytes=560785440 weight_movement_bytes=69206016 total_nvlink_bytes=629991456 weight_to_token_ratio=0.123409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314845230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=249896960 dispatch_meta_bytes=303096 combine_payload_bytes=249896960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250200056 ingress_bytes=284499968 link_bytes=284499968 token_bytes=500097016 weight_movement_bytes=34603008 total_nvlink_bytes=534700024 weight_to_token_ratio=0.069193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.070550475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283394048 dispatch_meta_bytes=345976 combine_payload_bytes=283394048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283740024 ingress_bytes=352600064 link_bytes=352600064 token_bytes=567134072 weight_movement_bytes=69206016 total_nvlink_bytes=636340088 weight_to_token_ratio=0.122028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326805654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79382 payload_rows_remote=68662 dup_rows=18922 dup_rows_remote=15838 dedup_ratio=0.192485 router_tokens_per_expert=[4286, 418, 544, 555, 34, 332, 749, 1839, 41, 0, 0, 1193, 2, 613, 1, 659, 18, 4, 1167, 35, 447, 49, 40, 131, 959, 63, 2737, 0, 10, 4393, 1029, 25, 100, 1299, 30, 1696, 258, 1107, 582, 0, 42, 3456, 90, 16, 3902, 167, 16, 1991, 209, 10, 13, 355, 672, 959, 37, 5, 200, 1186, 287, 578, 0, 750, 5845, 921] payload_rows_per_dst=[12356, 10720, 12904, 8410, 8540, 9264, 10488, 6700] dup_rows_per_dst=[5158, 3084, 2568, 1236, 1604, 2292, 1836, 1144] combine_rows_per_src=[12356, 10720, 12904, 8410, 8540, 9264, 10488, 6700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77544 payload_rows_remote=65850 dup_rows=20760 dup_rows_remote=14902 dedup_ratio=0.211182 router_tokens_per_expert=[4458, 427, 113, 752, 50, 334, 1442, 1868, 59, 0, 0, 2186, 0, 1251, 5, 575, 46, 15, 806, 16, 580, 21, 62, 151, 577, 72, 2449, 3, 0, 3408, 977, 32, 209, 2076, 77, 1613, 128, 413, 719, 1, 34, 4014, 37, 17, 3847, 148, 11, 1939, 187, 8, 24, 275, 270, 697, 97, 1, 132, 1565, 264, 332, 4, 540, 5866, 872] payload_rows_per_dst=[8278, 11116, 12598, 8506, 11694, 10020, 9428, 5904] dup_rows_per_dst=[1694, 3852, 2528, 1550, 5858, 2380, 1384, 1514] combine_rows_per_src=[8278, 11116, 12598, 8506, 11694, 10020, 9428, 5904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78022 payload_rows_remote=65168 dup_rows=20282 dup_rows_remote=16600 dedup_ratio=0.206319 router_tokens_per_expert=[4155, 347, 410, 682, 13, 286, 949, 2405, 73, 5, 4, 936, 39, 902, 11, 503, 13, 15, 2062, 25, 450, 51, 55, 156, 1047, 275, 2050, 2, 3, 3958, 1187, 26, 253, 1183, 236, 2054, 188, 805, 678, 7, 61, 2735, 195, 7, 3526, 229, 6, 1871, 174, 30, 82, 405, 422, 778, 103, 9, 314, 2064, 442, 494, 4, 342, 5441, 919] payload_rows_per_dst=[12668, 10378, 12854, 8148, 8794, 8462, 9492, 7226] dup_rows_per_dst=[5826, 2484, 3682, 1032, 2014, 1746, 1566, 1932] combine_rows_per_src=[12668, 10378, 12854, 8148, 8794, 8462, 9492, 7226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277086208 dispatch_meta_bytes=348544 combine_payload_bytes=277086208 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277434752 ingress_bytes=380895232 link_bytes=380895232 token_bytes=554520960 weight_movement_bytes=103809024 total_nvlink_bytes=658329984 weight_to_token_ratio=0.187205 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433278093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76824 payload_rows_remote=66626 dup_rows=21480 dup_rows_remote=18538 dedup_ratio=0.218506 router_tokens_per_expert=[1812, 575, 18, 49, 2122, 36, 494, 168, 47, 343, 4747, 44, 120, 1475, 561, 1032, 2128, 42, 95, 100, 129, 651, 9, 647, 231, 1719, 672, 786, 305, 6789, 2426, 3, 367, 58, 1312, 316, 1791, 349, 251, 3, 1392, 514, 2, 61, 140, 538, 136, 16, 2535, 283, 19, 134, 317, 327, 3071, 810, 118, 758, 15, 603, 12, 776, 1165, 588] payload_rows_per_dst=[8260, 5988, 6658, 10010, 7820, 14664, 13226, 10198] dup_rows_per_dst=[2288, 1256, 944, 2274, 1074, 4512, 6190, 2942] combine_rows_per_src=[8260, 5988, 6658, 10010, 7820, 14664, 13226, 10198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281239552 dispatch_meta_bytes=338000 combine_payload_bytes=281239552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281577552 ingress_bytes=315842560 link_bytes=315842560 token_bytes=562817104 weight_movement_bytes=34603008 total_nvlink_bytes=597420112 weight_to_token_ratio=0.061482 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188490126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269721600 dispatch_meta_bytes=323008 combine_payload_bytes=269721600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270044608 ingress_bytes=304324608 link_bytes=304324608 token_bytes=539766208 weight_movement_bytes=34603008 total_nvlink_bytes=574369216 weight_to_token_ratio=0.064107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145149000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=327072 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255200 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534183328 weight_movement_bytes=34603008 total_nvlink_bytes=568786336 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272900096 dispatch_meta_bytes=340656 combine_payload_bytes=272900096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273240752 ingress_bytes=307503104 link_bytes=307503104 token_bytes=546140848 weight_movement_bytes=34603008 total_nvlink_bytes=580743856 weight_to_token_ratio=0.063359 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157109424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75050 payload_rows_remote=64842 dup_rows=23254 dup_rows_remote=21222 dedup_ratio=0.236552 router_tokens_per_expert=[4239, 355, 315, 628, 18, 256, 902, 1032, 43, 0, 0, 1647, 0, 1777, 1, 765, 56, 1, 1453, 28, 455, 39, 12, 224, 696, 140, 2690, 2, 8, 3633, 1321, 60, 140, 1163, 52, 1809, 197, 716, 487, 0, 16, 4353, 31, 9, 4111, 126, 7, 2137, 223, 13, 7, 241, 303, 1116, 25, 3, 117, 1762, 202, 661, 1, 537, 4892, 899] payload_rows_per_dst=[6180, 7234, 4382, 12144, 8526, 13718, 10208, 12658] dup_rows_per_dst=[832, 1232, 154, 4956, 1718, 6846, 2032, 5484] combine_rows_per_src=[6180, 7234, 4382, 12144, 8526, 13718, 10208, 12658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79400 payload_rows_remote=67142 dup_rows=18904 dup_rows_remote=14910 dedup_ratio=0.192301 router_tokens_per_expert=[4537, 260, 185, 443, 28, 422, 775, 1476, 56, 0, 0, 1861, 4, 1001, 1, 343, 52, 23, 881, 11, 402, 51, 85, 167, 606, 175, 2726, 3, 7, 4176, 807, 32, 378, 1596, 134, 1616, 278, 317, 640, 0, 25, 3875, 50, 22, 3979, 244, 6, 2384, 275, 26, 81, 263, 664, 655, 189, 7, 216, 1323, 511, 594, 1, 280, 6160, 767] payload_rows_per_dst=[12258, 11064, 13068, 7766, 8302, 10154, 10576, 6212] dup_rows_per_dst=[3994, 3820, 2596, 946, 1616, 3058, 1702, 1172] combine_rows_per_src=[12258, 11064, 13068, 7766, 8302, 10154, 10576, 6212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=69216 dup_rows=19650 dup_rows_remote=16186 dedup_ratio=0.199890 router_tokens_per_expert=[538, 180, 155, 1677, 376, 1261, 899, 40, 26, 287, 767, 53, 34, 279, 49, 253, 43, 17, 4, 705, 4, 2, 464, 1218, 552, 3829, 845, 758, 2503, 3174, 3369, 234, 32, 1036, 1138, 237, 1451, 1079, 63, 137, 93, 1803, 124, 5, 1265, 4, 0, 4, 5240, 467, 17, 2775, 58, 21, 26, 249, 11, 3180, 52, 668, 3198, 0, 1, 123] payload_rows_per_dst=[10550, 9914, 12098, 9438, 8662, 10350, 6546, 11096] dup_rows_per_dst=[3322, 1240, 3296, 3464, 1684, 2594, 680, 3370] combine_rows_per_src=[10550, 9914, 12098, 9438, 8662, 10350, 6546, 11096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77124 payload_rows_remote=65724 dup_rows=21180 dup_rows_remote=18508 dedup_ratio=0.215454 router_tokens_per_expert=[228, 100, 170, 1195, 151, 981, 1083, 8, 4, 378, 717, 75, 72, 677, 35, 247, 30, 11, 5, 637, 7, 1, 287, 1109, 929, 4258, 520, 888, 2530, 4349, 3394, 323, 13, 1767, 804, 285, 1800, 150, 34, 87, 53, 1641, 71, 2, 919, 0, 0, 1, 5863, 505, 22, 2911, 57, 15, 13, 195, 2, 2806, 290, 45, 3330, 0, 0, 72] payload_rows_per_dst=[6592, 10626, 4026, 9830, 11022, 11400, 13324, 10304] dup_rows_per_dst=[1240, 2300, 148, 2668, 3528, 2672, 5838, 2786] combine_rows_per_src=[6592, 10626, 4026, 9830, 11022, 11400, 13324, 10304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=265937088 ingress_bytes=334798848 link_bytes=334798848 token_bytes=531529920 weight_movement_bytes=69206016 total_nvlink_bytes=600735936 weight_to_token_ratio=0.130202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259821112e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=328208 combine_payload_bytes=275013632 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=275341840 ingress_bytes=413425664 link_bytes=413425664 token_bytes=550355472 weight_movement_bytes=138412032 total_nvlink_bytes=688767504 weight_to_token_ratio=0.251496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555687490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283508736 dispatch_meta_bytes=341608 combine_payload_bytes=283508736 prefetch_bytes=34603008 reduce_bytes=207618048 egress_bytes=283850344 ingress_bytes=525729792 link_bytes=525729792 token_bytes=567359080 weight_movement_bytes=242221056 total_nvlink_bytes=809580136 weight_to_token_ratio=0.426927 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.978278883e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77970 payload_rows_remote=69320 dup_rows=20334 dup_rows_remote=18862 dedup_ratio=0.206848 router_tokens_per_expert=[703, 104, 616, 1030, 195, 2077, 641, 47, 103, 510, 592, 59, 165, 967, 126, 92, 35, 20, 11, 1093, 12, 2, 395, 1166, 1175, 3485, 1692, 199, 2763, 2493, 3274, 176, 45, 1792, 843, 259, 1674, 253, 68, 127, 127, 1782, 111, 12, 740, 0, 0, 0, 5490, 645, 27, 2593, 105, 56, 46, 192, 6, 3121, 332, 282, 2282, 0, 1, 123] payload_rows_per_dst=[8768, 10222, 10420, 12526, 8650, 8852, 8216, 10316] dup_rows_per_dst=[2058, 1976, 2998, 6032, 1472, 1678, 2142, 1978] combine_rows_per_src=[8768, 10222, 10420, 12526, 8650, 8852, 8216, 10316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79248 payload_rows_remote=69170 dup_rows=19056 dup_rows_remote=17346 dedup_ratio=0.193848 router_tokens_per_expert=[639, 149, 195, 432, 111, 983, 746, 12, 15, 371, 658, 61, 77, 898, 56, 150, 17, 18, 1, 1005, 19, 6, 589, 1802, 1907, 3608, 1117, 326, 2546, 3563, 2996, 379, 57, 1666, 1058, 265, 2186, 264, 46, 82, 104, 2484, 27, 7, 735, 0, 0, 2, 4734, 521, 17, 2571, 33, 1, 44, 205, 9, 2730, 445, 143, 3119, 1, 0, 144] payload_rows_per_dst=[10178, 10078, 11978, 9648, 9370, 11270, 6168, 10558] dup_rows_per_dst=[2348, 1710, 4404, 2902, 1878, 2574, 616, 2624] combine_rows_per_src=[10178, 10078, 11978, 9648, 9370, 11270, 6168, 10558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77528 payload_rows_remote=64544 dup_rows=20776 dup_rows_remote=14822 dedup_ratio=0.211344 router_tokens_per_expert=[952, 115, 451, 1173, 252, 1684, 909, 26, 17, 732, 1078, 69, 50, 860, 59, 118, 129, 115, 5, 1485, 72, 3, 581, 1851, 1093, 3422, 1195, 728, 2729, 2957, 2674, 492, 80, 1740, 1015, 246, 980, 187, 51, 98, 146, 1829, 52, 44, 948, 2, 20, 2, 5228, 615, 99, 1690, 75, 31, 56, 269, 6, 2943, 259, 397, 1846, 3, 2, 117] payload_rows_per_dst=[12344, 10710, 12984, 9714, 7418, 9786, 5084, 9488] dup_rows_per_dst=[4128, 2100, 5954, 2760, 1376, 2214, 586, 1658] combine_rows_per_src=[12344, 10710, 12984, 9714, 7418, 9786, 5084, 9488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269205504 dispatch_meta_bytes=336928 combine_payload_bytes=269205504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269542432 ingress_bytes=303808512 link_bytes=303808512 token_bytes=538747936 weight_movement_bytes=34603008 total_nvlink_bytes=573350944 weight_to_token_ratio=0.064229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143206972e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72116 payload_rows_remote=57080 dup_rows=26188 dup_rows_remote=13878 dedup_ratio=0.266398 router_tokens_per_expert=[4425, 277, 646, 734, 11, 443, 663, 720, 23, 0, 0, 1631, 2, 1580, 3, 722, 14, 8, 1444, 12, 430, 70, 103, 108, 753, 200, 2859, 3, 4, 3859, 956, 34, 235, 1133, 78, 1652, 110, 1108, 838, 0, 40, 3194, 46, 17, 3534, 204, 14, 2237, 232, 34, 11, 429, 383, 708, 16, 6, 204, 1556, 376, 488, 3, 669, 5928, 932] payload_rows_per_dst=[6150, 6884, 4252, 12316, 8494, 6384, 12600, 15036] dup_rows_per_dst=[838, 1038, 126, 5020, 1814, 548, 4494, 12310] combine_rows_per_src=[6150, 6884, 4252, 12316, 8494, 6384, 12600, 15036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283934720 dispatch_meta_bytes=352728 combine_payload_bytes=283934720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284287448 ingress_bytes=318537728 link_bytes=318537728 token_bytes=568222168 weight_movement_bytes=34603008 total_nvlink_bytes=602825176 weight_to_token_ratio=0.060897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198631826e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264372224 dispatch_meta_bytes=317464 combine_payload_bytes=264372224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264689688 ingress_bytes=298975232 link_bytes=298975232 token_bytes=529061912 weight_movement_bytes=34603008 total_nvlink_bytes=563664920 weight_to_token_ratio=0.065404 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125019729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283320320 dispatch_meta_bytes=346064 combine_payload_bytes=283320320 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283666384 ingress_bytes=352526336 link_bytes=352526336 token_bytes=566986704 weight_movement_bytes=69206016 total_nvlink_bytes=636192720 weight_to_token_ratio=0.122059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326528222e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233799680 dispatch_meta_bytes=283832 combine_payload_bytes=233799680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=234083512 ingress_bytes=303005696 link_bytes=303005696 token_bytes=467883192 weight_movement_bytes=69206016 total_nvlink_bytes=537089208 weight_to_token_ratio=0.147913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140186040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77802 payload_rows_remote=65516 dup_rows=20502 dup_rows_remote=15214 dedup_ratio=0.208557 router_tokens_per_expert=[601, 174, 306, 552, 164, 2047, 828, 9, 56, 360, 494, 13, 47, 1354, 53, 272, 34, 19, 4, 991, 4, 0, 493, 1690, 746, 4000, 681, 568, 2957, 2937, 3237, 491, 78, 1490, 948, 255, 1989, 539, 40, 117, 108, 1994, 104, 4, 585, 6, 0, 1, 4428, 370, 48, 3355, 53, 21, 28, 208, 11, 3036, 469, 324, 2250, 0, 0, 111] payload_rows_per_dst=[8012, 10910, 6168, 8774, 11840, 9628, 12286, 10184] dup_rows_per_dst=[1350, 2388, 302, 2120, 4986, 1850, 5288, 2218] combine_rows_per_src=[8012, 10910, 6168, 8774, 11840, 9628, 12286, 10184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78586 payload_rows_remote=66052 dup_rows=19718 dup_rows_remote=15302 dedup_ratio=0.200582 router_tokens_per_expert=[179, 1058, 315, 288, 1038, 4858, 10, 24, 485, 977, 258, 10, 285, 714, 2610, 146, 30, 107, 121, 1859, 0, 296, 330, 7, 327, 196, 296, 167, 4900, 383, 70, 2630, 48, 175, 97, 62, 252, 938, 0, 5, 2058, 2447, 3, 4902, 1141, 10, 52, 2002, 0, 3817, 29, 670, 29, 2, 45, 1441, 105, 104, 1316, 2221, 100, 30, 39, 38] payload_rows_per_dst=[6916, 8900, 10400, 12534, 11058, 10600, 10282, 7896] dup_rows_per_dst=[864, 2718, 2860, 4416, 1900, 4178, 1784, 998] combine_rows_per_src=[6916, 8900, 10400, 12534, 11058, 10600, 10282, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78176 payload_rows_remote=65624 dup_rows=20128 dup_rows_remote=14524 dedup_ratio=0.204753 router_tokens_per_expert=[490, 305, 252, 2170, 610, 1544, 923, 30, 32, 90, 551, 53, 66, 375, 66, 130, 71, 13, 3, 1181, 6, 7, 363, 1593, 1130, 3141, 973, 797, 1677, 3772, 2754, 373, 140, 1365, 548, 240, 2813, 222, 87, 138, 193, 1763, 57, 26, 1394, 0, 1, 0, 5083, 542, 80, 3349, 46, 17, 24, 156, 12, 2623, 217, 45, 2333, 6, 0, 91] payload_rows_per_dst=[12552, 8854, 12430, 10124, 9354, 8436, 7574, 8852] dup_rows_per_dst=[5604, 1340, 4210, 3172, 1752, 1592, 854, 1604] combine_rows_per_src=[12552, 8854, 12430, 10124, 9354, 8436, 7574, 8852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79710 payload_rows_remote=71140 dup_rows=18594 dup_rows_remote=16434 dedup_ratio=0.189148 router_tokens_per_expert=[59, 800, 295, 289, 1989, 4120, 51, 6, 116, 1835, 192, 22, 466, 357, 2409, 143, 45, 66, 27, 3349, 0, 99, 289, 0, 300, 14, 173, 199, 5688, 431, 27, 2850, 48, 91, 68, 36, 349, 1024, 0, 1, 1467, 1473, 1, 4723, 1242, 13, 24, 1145, 0, 4287, 50, 656, 35, 0, 23, 2027, 93, 16, 1256, 1941, 209, 4, 116, 28] payload_rows_per_dst=[6538, 8856, 12000, 7316, 10970, 8570, 11570, 13890] dup_rows_per_dst=[440, 2224, 3990, 672, 1710, 2160, 2586, 4812] combine_rows_per_src=[6538, 8856, 12000, 7316, 10970, 8570, 11570, 13890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=270548992 dispatch_meta_bytes=325416 combine_payload_bytes=270548992 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270874408 ingress_bytes=339755008 link_bytes=339755008 token_bytes=541423400 weight_movement_bytes=69206016 total_nvlink_bytes=610629416 weight_to_token_ratio=0.127822 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278470743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268353536 dispatch_meta_bytes=322920 combine_payload_bytes=268353536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268676456 ingress_bytes=337559552 link_bytes=337559552 token_bytes=537029992 weight_movement_bytes=69206016 total_nvlink_bytes=606236008 weight_to_token_ratio=0.128868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270209418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79392 payload_rows_remote=67626 dup_rows=18912 dup_rows_remote=16692 dedup_ratio=0.192383 router_tokens_per_expert=[243, 736, 479, 289, 1752, 4483, 27, 19, 118, 1473, 566, 32, 617, 433, 1744, 233, 145, 29, 201, 2433, 0, 395, 170, 13, 426, 230, 140, 316, 5344, 511, 4, 2255, 18, 141, 90, 0, 525, 855, 1, 0, 2063, 1539, 1, 5363, 865, 50, 69, 1512, 1, 3690, 28, 686, 31, 4, 78, 2513, 67, 1, 784, 1991, 245, 12, 58, 15] payload_rows_per_dst=[6570, 8206, 11766, 7064, 11766, 9690, 11202, 13128] dup_rows_per_dst=[520, 2226, 3972, 700, 2220, 2508, 2860, 3906] combine_rows_per_src=[6570, 8206, 11766, 7064, 11766, 9690, 11202, 13128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268795904 dispatch_meta_bytes=320592 combine_payload_bytes=268795904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269116496 ingress_bytes=303398912 link_bytes=303398912 token_bytes=537912400 weight_movement_bytes=34603008 total_nvlink_bytes=572515408 weight_to_token_ratio=0.064328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141665680e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76422 payload_rows_remote=70824 dup_rows=21882 dup_rows_remote=21366 dedup_ratio=0.222595 router_tokens_per_expert=[119, 1063, 687, 495, 2188, 4447, 44, 8, 342, 1973, 630, 80, 379, 436, 1510, 323, 370, 94, 181, 2008, 18, 159, 227, 0, 316, 174, 110, 289, 3200, 390, 4, 3185, 21, 178, 52, 47, 1053, 2098, 3, 35, 1043, 1218, 0, 4779, 1050, 14, 124, 1318, 0, 4205, 28, 1284, 267, 2, 48, 2204, 112, 28, 603, 1565, 233, 4, 74, 13] payload_rows_per_dst=[13144, 10530, 5598, 11642, 11962, 6282, 12222, 5042] dup_rows_per_dst=[4958, 3252, 516, 3694, 4570, 816, 3854, 222] combine_rows_per_src=[13144, 10530, 5598, 11642, 11962, 6282, 12222, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291389440 dispatch_meta_bytes=350296 combine_payload_bytes=291389440 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=291739736 ingress_bytes=429801472 link_bytes=429801472 token_bytes=583129176 weight_movement_bytes=138412032 total_nvlink_bytes=721541208 weight_to_token_ratio=0.237361 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.617308337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76566 payload_rows_remote=65310 dup_rows=21738 dup_rows_remote=17992 dedup_ratio=0.221130 router_tokens_per_expert=[84, 1052, 199, 295, 1879, 4719, 11, 3, 201, 2097, 571, 23, 321, 474, 2216, 211, 98, 39, 110, 2174, 0, 215, 654, 1, 263, 126, 114, 100, 5192, 274, 4, 3317, 11, 108, 46, 2, 587, 1109, 0, 0, 1535, 1387, 1, 5099, 767, 52, 87, 985, 0, 4269, 40, 386, 181, 1, 70, 1834, 39, 63, 736, 2539, 142, 9, 26, 4] payload_rows_per_dst=[14398, 11256, 6184, 13878, 11592, 6162, 6386, 6710] dup_rows_per_dst=[8590, 3746, 398, 4902, 2332, 692, 672, 406] combine_rows_per_src=[14398, 11256, 6184, 13878, 11592, 6162, 6386, 6710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75188 payload_rows_remote=65594 dup_rows=23116 dup_rows_remote=21442 dedup_ratio=0.235148 router_tokens_per_expert=[852, 192, 264, 1103, 179, 934, 1079, 26, 30, 670, 635, 50, 45, 1057, 55, 196, 181, 25, 3, 892, 6, 12, 910, 1415, 1106, 4230, 866, 348, 1951, 3641, 2453, 246, 28, 2107, 837, 245, 1239, 1006, 147, 96, 149, 1774, 66, 19, 1055, 0, 0, 0, 5680, 466, 39, 2513, 53, 82, 37, 228, 7, 2718, 271, 100, 2419, 0, 2, 117] payload_rows_per_dst=[7732, 11216, 6400, 4604, 10698, 10788, 14156, 9594] dup_rows_per_dst=[1526, 2720, 488, 528, 4614, 2620, 8946, 1674] combine_rows_per_src=[7732, 11216, 6400, 4604, 10698, 10788, 14156, 9594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=276996096 dispatch_meta_bytes=337272 combine_payload_bytes=276996096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277333368 ingress_bytes=311599104 link_bytes=311599104 token_bytes=554329464 weight_movement_bytes=34603008 total_nvlink_bytes=588932472 weight_to_token_ratio=0.062423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172522343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290095104 dispatch_meta_bytes=368760 combine_payload_bytes=290095104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290463864 ingress_bytes=324698112 link_bytes=324698112 token_bytes=580558968 weight_movement_bytes=34603008 total_nvlink_bytes=615161976 weight_to_token_ratio=0.059603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221812855e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=333208 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267842968 ingress_bytes=302112768 link_bytes=302112768 token_bytes=535352728 weight_movement_bytes=34603008 total_nvlink_bytes=569955736 weight_to_token_ratio=0.064636 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136826024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268673024 dispatch_meta_bytes=348144 combine_payload_bytes=268673024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=269021168 ingress_bytes=303276032 link_bytes=303276032 token_bytes=537694192 weight_movement_bytes=34603008 total_nvlink_bytes=572297200 weight_to_token_ratio=0.064354 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141203293e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80498 payload_rows_remote=73248 dup_rows=17806 dup_rows_remote=16220 dedup_ratio=0.181132 router_tokens_per_expert=[2, 2483, 469, 4324, 127, 34, 262, 8, 112, 7, 142, 20, 399, 5, 1540, 48, 831, 1, 448, 19, 521, 445, 32, 1758, 179, 1820, 15, 141, 69, 1439, 467, 288, 3505, 184, 290, 397, 586, 5, 538, 1193, 353, 22, 714, 364, 12, 65, 3, 165, 257, 228, 4154, 1375, 7054, 42, 1719, 0, 1109, 52, 0, 4139, 624, 128, 343, 1077] payload_rows_per_dst=[12412, 10972, 7256, 7250, 10576, 14672, 6130, 11230] dup_rows_per_dst=[3006, 1882, 854, 1586, 2820, 2832, 1112, 3714] combine_rows_per_src=[12412, 10972, 7256, 7250, 10576, 14672, 6130, 11230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66240 dup_rows=21160 dup_rows_remote=18580 dedup_ratio=0.215251 router_tokens_per_expert=[251, 562, 307, 298, 1452, 4374, 94, 3, 290, 2060, 511, 32, 664, 762, 1608, 210, 34, 21, 181, 2364, 0, 251, 394, 0, 336, 134, 131, 117, 5025, 391, 13, 3144, 26, 171, 99, 57, 547, 1290, 0, 0, 1444, 1978, 2, 5347, 983, 33, 87, 1451, 1, 3931, 33, 499, 31, 0, 113, 1828, 64, 33, 853, 2013, 133, 38, 35, 18] payload_rows_per_dst=[5510, 9062, 11422, 7796, 7066, 12692, 10904, 12692] dup_rows_per_dst=[424, 3212, 3816, 736, 882, 5778, 2580, 3732] combine_rows_per_src=[5510, 9062, 11422, 7796, 7066, 12692, 10904, 12692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75620 payload_rows_remote=61366 dup_rows=22684 dup_rows_remote=13850 dedup_ratio=0.230754 router_tokens_per_expert=[107, 729, 176, 283, 2302, 3984, 19, 1, 208, 1992, 1220, 45, 233, 252, 2055, 114, 37, 91, 138, 2994, 0, 264, 178, 2, 460, 254, 212, 38, 5020, 159, 4, 2780, 12, 112, 15, 53, 491, 2549, 0, 0, 1530, 1179, 0, 5549, 732, 47, 25, 879, 0, 3943, 33, 600, 13, 0, 64, 1641, 95, 11, 722, 2250, 183, 3, 39, 31] payload_rows_per_dst=[14254, 11300, 6930, 13640, 12918, 5802, 4514, 6262] dup_rows_per_dst=[8834, 3296, 478, 4214, 4644, 624, 188, 406] combine_rows_per_src=[14254, 11300, 6930, 13640, 12918, 5802, 4514, 6262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79094 payload_rows_remote=64938 dup_rows=19210 dup_rows_remote=16688 dedup_ratio=0.195414 router_tokens_per_expert=[2, 2155, 827, 4192, 114, 33, 370, 0, 91, 10, 92, 81, 405, 0, 1687, 52, 1607, 2, 547, 31, 669, 302, 48, 909, 210, 2248, 10, 69, 75, 1106, 458, 312, 4215, 107, 265, 366, 405, 0, 854, 1827, 152, 10, 1192, 122, 0, 3, 4, 158, 433, 284, 5689, 1083, 6698, 0, 760, 0, 580, 71, 0, 3639, 436, 35, 300, 750] payload_rows_per_dst=[6496, 6910, 7162, 12388, 11696, 14156, 10822, 9464] dup_rows_per_dst=[506, 1006, 1068, 4972, 4382, 2522, 2596, 2158] combine_rows_per_src=[6496, 6910, 7162, 12388, 11696, 14156, 10822, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=300023808 dispatch_meta_bytes=357872 combine_payload_bytes=300023808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300381680 ingress_bytes=334626816 link_bytes=334626816 token_bytes=600405488 weight_movement_bytes=34603008 total_nvlink_bytes=635008496 weight_to_token_ratio=0.057633 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259173769e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271319040 dispatch_meta_bytes=339280 combine_payload_bytes=271319040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271658320 ingress_bytes=340525056 link_bytes=340525056 token_bytes=542977360 weight_movement_bytes=69206016 total_nvlink_bytes=612183376 weight_to_token_ratio=0.127457 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281368371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=66832 dup_rows=20392 dup_rows_remote=16100 dedup_ratio=0.207438 router_tokens_per_expert=[3, 2037, 501, 3756, 77, 123, 54, 2, 139, 5, 361, 58, 597, 19, 2316, 9, 2524, 4, 745, 128, 410, 482, 19, 2463, 267, 1045, 10, 69, 539, 1892, 177, 533, 3649, 102, 224, 177, 211, 20, 1514, 1789, 130, 15, 1461, 178, 15, 27, 28, 146, 144, 109, 4151, 1011, 5932, 0, 1119, 0, 498, 65, 0, 2729, 449, 126, 137, 1632] payload_rows_per_dst=[5824, 12202, 10640, 11422, 11080, 13160, 4174, 9410] dup_rows_per_dst=[344, 3108, 2910, 4580, 4292, 2704, 592, 1862] combine_rows_per_src=[5824, 12202, 10640, 11422, 11080, 13160, 4174, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=326504 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312552 ingress_bytes=300589056 link_bytes=300589056 token_bytes=532298600 weight_movement_bytes=34603008 total_nvlink_bytes=566901608 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131092418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=251355136 dispatch_meta_bytes=300864 combine_payload_bytes=251355136 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251656000 ingress_bytes=320561152 link_bytes=320561152 token_bytes=503011136 weight_movement_bytes=69206016 total_nvlink_bytes=572217152 weight_to_token_ratio=0.137583 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206245808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78222 payload_rows_remote=67392 dup_rows=20082 dup_rows_remote=16034 dedup_ratio=0.204285 router_tokens_per_expert=[24, 2326, 208, 3403, 164, 100, 171, 26, 151, 11, 1161, 99, 382, 4, 2409, 63, 706, 35, 658, 81, 861, 738, 48, 2341, 351, 762, 38, 45, 214, 1121, 452, 319, 3520, 285, 263, 365, 469, 2, 1764, 2547, 100, 9, 486, 364, 8, 14, 21, 385, 812, 277, 3844, 1655, 6462, 5, 891, 1, 570, 50, 0, 2405, 373, 135, 156, 1442] payload_rows_per_dst=[10852, 12404, 10830, 5630, 10242, 13628, 6214, 8422] dup_rows_per_dst=[1992, 3844, 4048, 974, 4246, 2070, 1068, 1840] combine_rows_per_src=[10852, 12404, 10830, 5630, 10242, 13628, 6214, 8422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79790 payload_rows_remote=67622 dup_rows=18514 dup_rows_remote=15842 dedup_ratio=0.188334 router_tokens_per_expert=[2, 1762, 122, 5119, 157, 43, 142, 11, 197, 22, 274, 160, 902, 0, 1411, 89, 1365, 3, 387, 50, 337, 439, 33, 2701, 244, 1069, 5, 65, 191, 1038, 321, 250, 3076, 147, 314, 236, 289, 5, 2303, 2484, 94, 9, 1241, 356, 8, 11, 0, 149, 896, 185, 4365, 1366, 6797, 0, 596, 0, 981, 37, 0, 2562, 431, 118, 227, 958] payload_rows_per_dst=[12452, 12168, 12222, 5500, 9128, 14374, 5282, 8664] dup_rows_per_dst=[2264, 2672, 4560, 866, 2428, 2956, 804, 1964] combine_rows_per_src=[12452, 12168, 12222, 5500, 9128, 14374, 5282, 8664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=60654 dup_rows=23808 dup_rows_remote=19448 dedup_ratio=0.242188 router_tokens_per_expert=[130, 629, 244, 331, 1762, 4150, 75, 1, 167, 1062, 422, 13, 514, 237, 2259, 86, 113, 141, 118, 3791, 0, 318, 356, 0, 164, 143, 242, 280, 5870, 204, 20, 2208, 22, 140, 53, 18, 244, 880, 0, 0, 1196, 1625, 0, 5524, 1000, 14, 304, 1697, 0, 3622, 48, 509, 16, 0, 181, 2778, 22, 23, 648, 2221, 213, 11, 68, 25] payload_rows_per_dst=[5890, 7684, 12608, 4848, 2672, 14420, 12532, 13842] dup_rows_per_dst=[454, 1836, 5366, 252, 42, 7330, 4168, 4360] combine_rows_per_src=[5890, 7684, 12608, 4848, 2672, 14420, 12532, 13842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=273743872 dispatch_meta_bytes=331728 combine_payload_bytes=273743872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=274075600 ingress_bytes=377552896 link_bytes=377552896 token_bytes=547819472 weight_movement_bytes=103809024 total_nvlink_bytes=651628496 weight_to_token_ratio=0.189495 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420701152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276037632 dispatch_meta_bytes=333704 combine_payload_bytes=276037632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276371336 ingress_bytes=310640640 link_bytes=310640640 token_bytes=552408968 weight_movement_bytes=34603008 total_nvlink_bytes=587011976 weight_to_token_ratio=0.062640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168915720e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276979712 dispatch_meta_bytes=333856 combine_payload_bytes=276979712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277313568 ingress_bytes=311582720 link_bytes=311582720 token_bytes=554293280 weight_movement_bytes=34603008 total_nvlink_bytes=588896288 weight_to_token_ratio=0.062427 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172460691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=248438784 dispatch_meta_bytes=320408 combine_payload_bytes=248438784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248759192 ingress_bytes=283041792 link_bytes=283041792 token_bytes=497197976 weight_movement_bytes=34603008 total_nvlink_bytes=531800984 weight_to_token_ratio=0.069596 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.065063477e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80910 payload_rows_remote=67436 dup_rows=17394 dup_rows_remote=14774 dedup_ratio=0.176941 router_tokens_per_expert=[1945, 93, 2, 0, 17, 752, 0, 19, 1780, 768, 67, 0, 647, 2471, 590, 507, 1777, 3038, 0, 817, 1894, 2033, 25, 665, 45, 0, 5468, 58, 114, 669, 94, 1599, 325, 131, 592, 196, 201, 0, 957, 35, 1973, 776, 98, 180, 38, 5846, 38, 35, 39, 1884, 3, 131, 309, 840, 1795, 1, 63, 64, 4031, 151, 140, 304, 14, 8] payload_rows_per_dst=[10056, 10244, 10596, 13474, 13338, 5700, 8432, 9070] dup_rows_per_dst=[1676, 3416, 3826, 2620, 3228, 576, 1572, 480] combine_rows_per_src=[10056, 10244, 10596, 13474, 13338, 5700, 8432, 9070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73520 payload_rows_remote=59004 dup_rows=24784 dup_rows_remote=15150 dedup_ratio=0.252116 router_tokens_per_expert=[0, 1729, 767, 4690, 59, 32, 269, 12, 158, 7, 87, 102, 805, 2, 2458, 18, 1690, 9, 729, 41, 383, 1001, 89, 2325, 231, 1415, 5, 125, 458, 1562, 252, 255, 2729, 159, 83, 327, 267, 6, 895, 2214, 240, 19, 860, 210, 12, 3, 2, 165, 400, 121, 3219, 1713, 6619, 2, 848, 0, 468, 68, 0, 3438, 473, 92, 386, 1349] payload_rows_per_dst=[5364, 6614, 10434, 12568, 9922, 4238, 14516, 9864] dup_rows_per_dst=[372, 660, 2100, 5418, 3438, 478, 9634, 2684] combine_rows_per_src=[5364, 6614, 10434, 12568, 9922, 4238, 14516, 9864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76822 payload_rows_remote=63054 dup_rows=21482 dup_rows_remote=14094 dedup_ratio=0.218526 router_tokens_per_expert=[1334, 26, 1, 0, 44, 734, 0, 45, 2303, 763, 4, 0, 739, 3436, 391, 496, 2777, 3891, 0, 238, 1825, 1763, 37, 483, 25, 0, 5856, 102, 112, 336, 30, 1097, 470, 271, 428, 216, 25, 0, 1355, 21, 999, 429, 112, 56, 13, 5580, 109, 10, 3, 1902, 0, 32, 548, 1228, 1628, 0, 29, 102, 4283, 124, 90, 170, 3, 28] payload_rows_per_dst=[10648, 8060, 10760, 7820, 4770, 13768, 8676, 12320] dup_rows_per_dst=[1502, 1332, 3486, 756, 802, 7388, 2006, 4210] combine_rows_per_src=[10648, 8060, 10760, 7820, 4770, 13768, 8676, 12320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72230 payload_rows_remote=57122 dup_rows=26074 dup_rows_remote=13612 dedup_ratio=0.265238 router_tokens_per_expert=[5, 3502, 696, 4377, 120, 54, 87, 12, 146, 12, 115, 50, 615, 0, 2381, 66, 814, 0, 472, 110, 313, 345, 24, 2468, 378, 1075, 25, 46, 199, 1383, 372, 665, 2575, 117, 106, 352, 323, 1, 986, 1396, 111, 5, 1382, 310, 2, 43, 20, 91, 214, 212, 5175, 1595, 6274, 5, 971, 2, 615, 149, 3, 3340, 471, 201, 160, 1018] payload_rows_per_dst=[15108, 13120, 10944, 6964, 5714, 5780, 5156, 9444] dup_rows_per_dst=[12462, 4000, 3298, 1322, 848, 832, 842, 2470] combine_rows_per_src=[15108, 13120, 10944, 6964, 5714, 5780, 5156, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81070 payload_rows_remote=73976 dup_rows=17234 dup_rows_remote=15560 dedup_ratio=0.175313 router_tokens_per_expert=[702, 129, 27, 2, 38, 708, 0, 20, 2181, 764, 13, 0, 597, 2856, 794, 172, 3136, 5780, 0, 363, 1461, 836, 51, 560, 112, 1, 6067, 148, 56, 371, 63, 918, 605, 486, 229, 175, 36, 1, 900, 124, 1594, 649, 328, 57, 47, 5848, 241, 30, 225, 1792, 1, 41, 353, 1121, 1352, 0, 49, 61, 3376, 127, 62, 265, 44, 7] payload_rows_per_dst=[13010, 8694, 10190, 13712, 7094, 10908, 7926, 9536] dup_rows_per_dst=[1802, 2406, 2624, 1760, 1674, 3024, 1844, 2100] combine_rows_per_src=[13010, 8694, 10190, 13712, 7094, 10908, 7926, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276217856 dispatch_meta_bytes=328840 combine_payload_bytes=276217856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276546696 ingress_bytes=380026880 link_bytes=380026880 token_bytes=552764552 weight_movement_bytes=103809024 total_nvlink_bytes=656573576 weight_to_token_ratio=0.187800 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.430010555e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82710 payload_rows_remote=72744 dup_rows=15594 dup_rows_remote=13266 dedup_ratio=0.158630 router_tokens_per_expert=[1038, 127, 38, 2, 75, 785, 0, 10, 1684, 495, 151, 3, 605, 2690, 1020, 715, 2622, 4874, 28, 326, 1101, 673, 49, 1348, 43, 13, 5827, 167, 206, 208, 32, 767, 345, 222, 98, 279, 41, 7, 1399, 26, 2096, 441, 98, 45, 48, 6406, 386, 68, 147, 1435, 3, 24, 44, 751, 2272, 2, 116, 111, 3723, 231, 213, 298, 26, 29] payload_rows_per_dst=[11996, 10598, 9966, 13094, 14212, 5988, 8056, 8800] dup_rows_per_dst=[1902, 4128, 2328, 1432, 3434, 376, 1300, 694] combine_rows_per_src=[11996, 10598, 9966, 13094, 14212, 5988, 8056, 8800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=241680384 dispatch_meta_bytes=296616 combine_payload_bytes=241680384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=241977000 ingress_bytes=380092416 link_bytes=380092416 token_bytes=483657384 weight_movement_bytes=138412032 total_nvlink_bytes=622069416 weight_to_token_ratio=0.286178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.430257161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=258269184 dispatch_meta_bytes=308592 combine_payload_bytes=258269184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258577776 ingress_bytes=327475200 link_bytes=327475200 token_bytes=516846960 weight_movement_bytes=69206016 total_nvlink_bytes=586052976 weight_to_token_ratio=0.133900 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232262814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233971712 dispatch_meta_bytes=282936 combine_payload_bytes=233971712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=234254648 ingress_bytes=303177728 link_bytes=303177728 token_bytes=468226360 weight_movement_bytes=69206016 total_nvlink_bytes=537432376 weight_to_token_ratio=0.147805 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140833383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80276 payload_rows_remote=68694 dup_rows=18028 dup_rows_remote=13018 dedup_ratio=0.183390 router_tokens_per_expert=[1202, 347, 2, 1, 40, 647, 0, 22, 2188, 943, 37, 0, 655, 3225, 805, 443, 2353, 3849, 0, 280, 847, 1239, 54, 1189, 54, 2, 5650, 70, 309, 318, 19, 1296, 611, 238, 362, 126, 201, 0, 1378, 6, 1404, 388, 165, 69, 24, 6749, 394, 54, 143, 1403, 0, 71, 55, 1046, 1566, 0, 72, 81, 3902, 144, 144, 251, 4, 15] payload_rows_per_dst=[10420, 11582, 9452, 13234, 14800, 4644, 7392, 8752] dup_rows_per_dst=[1800, 5010, 2472, 2202, 4542, 352, 1176, 474] combine_rows_per_src=[10420, 11582, 9452, 13234, 14800, 4644, 7392, 8752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=303005696 dispatch_meta_bytes=358144 combine_payload_bytes=303005696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303363840 ingress_bytes=337608704 link_bytes=337608704 token_bytes=606369536 weight_movement_bytes=34603008 total_nvlink_bytes=640972544 weight_to_token_ratio=0.057066 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270394373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72324 payload_rows_remote=61128 dup_rows=25980 dup_rows_remote=22062 dedup_ratio=0.264282 router_tokens_per_expert=[11, 3253, 632, 4550, 184, 92, 250, 3, 144, 12, 86, 121, 1139, 0, 1853, 16, 1247, 0, 672, 71, 681, 327, 107, 1884, 627, 618, 50, 55, 68, 1648, 460, 356, 4010, 193, 246, 439, 471, 3, 1121, 680, 134, 2, 1299, 239, 8, 11, 5, 253, 261, 601, 4001, 1574, 5963, 1, 1121, 4, 445, 22, 4, 2361, 409, 79, 496, 1479] payload_rows_per_dst=[8094, 6118, 8734, 12794, 6266, 3648, 15474, 11196] dup_rows_per_dst=[756, 624, 1244, 4070, 1226, 254, 13888, 3918] combine_rows_per_src=[8094, 6118, 8734, 12794, 6266, 3648, 15474, 11196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=297959424 dispatch_meta_bytes=344040 combine_payload_bytes=297959424 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=298303464 ingress_bytes=401768448 link_bytes=401768448 token_bytes=596262888 weight_movement_bytes=103809024 total_nvlink_bytes=700071912 weight_to_token_ratio=0.174099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511822325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281370624 dispatch_meta_bytes=326848 combine_payload_bytes=281370624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281697472 ingress_bytes=350576640 link_bytes=350576640 token_bytes=563068096 weight_movement_bytes=69206016 total_nvlink_bytes=632274112 weight_to_token_ratio=0.122909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319191673e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82196 payload_rows_remote=68222 dup_rows=16108 dup_rows_remote=12444 dedup_ratio=0.163859 router_tokens_per_expert=[86, 17, 33, 3, 1, 2507, 1666, 557, 14, 0, 267, 274, 1, 18, 19, 0, 1015, 19, 1598, 2091, 5301, 867, 26, 10, 1363, 0, 210, 1084, 4, 5943, 88, 523, 9, 1286, 1647, 5, 0, 762, 86, 0, 23, 21, 2114, 37, 1, 61, 1001, 0, 325, 136, 4744, 504, 4003, 176, 47, 734, 3975, 21, 0, 168, 1513, 3, 143, 2] payload_rows_per_dst=[7920, 11282, 9350, 13974, 12796, 6398, 10108, 10368] dup_rows_per_dst=[1820, 506, 1902, 3664, 4282, 910, 1742, 1282] combine_rows_per_src=[7920, 11282, 9350, 13974, 12796, 6398, 10108, 10368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=250380288 dispatch_meta_bytes=332760 combine_payload_bytes=250380288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250713048 ingress_bytes=284983296 link_bytes=284983296 token_bytes=501093336 weight_movement_bytes=34603008 total_nvlink_bytes=535696344 weight_to_token_ratio=0.069055 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072369200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75168 payload_rows_remote=62362 dup_rows=23136 dup_rows_remote=17066 dedup_ratio=0.235352 router_tokens_per_expert=[917, 270, 0, 4, 50, 884, 0, 15, 2597, 1272, 44, 0, 1161, 2960, 835, 536, 3198, 3605, 0, 273, 797, 1112, 17, 348, 135, 0, 5979, 120, 110, 478, 62, 1101, 578, 188, 283, 246, 128, 0, 905, 33, 1165, 521, 636, 44, 20, 5493, 83, 28, 201, 1628, 1, 84, 401, 1120, 2026, 3, 21, 80, 3803, 105, 122, 308, 16, 2] payload_rows_per_dst=[9814, 9966, 8302, 4936, 4232, 13454, 12806, 11658] dup_rows_per_dst=[1676, 2924, 1900, 364, 490, 6536, 6070, 3176] combine_rows_per_src=[9814, 9966, 8302, 4936, 4232, 13454, 12806, 11658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80834 payload_rows_remote=68236 dup_rows=17470 dup_rows_remote=13278 dedup_ratio=0.177714 router_tokens_per_expert=[26, 4, 14, 0, 2, 3509, 698, 1064, 146, 0, 221, 131, 0, 21, 1, 0, 1175, 128, 974, 1946, 6716, 367, 96, 176, 2036, 0, 92, 1227, 1, 4521, 53, 455, 5, 399, 860, 0, 0, 1212, 103, 0, 21, 17, 3010, 88, 0, 145, 593, 0, 934, 518, 4382, 404, 5230, 160, 116, 611, 3196, 32, 0, 138, 1100, 1, 76, 1] payload_rows_per_dst=[8914, 13820, 8524, 6952, 6138, 12598, 12126, 11762] dup_rows_per_dst=[1720, 652, 1200, 776, 1370, 4192, 3876, 3684] combine_rows_per_src=[8914, 13820, 8524, 6952, 6138, 12598, 12126, 11762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83030 payload_rows_remote=72768 dup_rows=15274 dup_rows_remote=13858 dedup_ratio=0.155375 router_tokens_per_expert=[975, 91, 15, 0, 2, 868, 0, 27, 870, 452, 14, 0, 438, 2594, 777, 241, 3468, 3861, 0, 458, 962, 960, 27, 465, 60, 0, 6277, 106, 165, 473, 50, 1015, 298, 329, 578, 140, 44, 0, 1662, 28, 2410, 390, 244, 106, 21, 6142, 534, 40, 400, 1802, 2, 37, 40, 1061, 1421, 0, 39, 130, 4794, 139, 274, 306, 15, 15] payload_rows_per_dst=[10262, 8690, 10016, 14344, 14252, 6964, 7854, 10648] dup_rows_per_dst=[1416, 2082, 2664, 1948, 4190, 526, 1672, 776] combine_rows_per_src=[10262, 8690, 10016, 14344, 14252, 6964, 7854, 10648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=322664 combine_payload_bytes=279437312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279759976 ingress_bytes=348643328 link_bytes=348643328 token_bytes=559197288 weight_movement_bytes=69206016 total_nvlink_bytes=628403304 weight_to_token_ratio=0.123760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311916775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81172 payload_rows_remote=67762 dup_rows=17132 dup_rows_remote=12502 dedup_ratio=0.174276 router_tokens_per_expert=[20, 139, 19, 0, 0, 3357, 930, 943, 4, 25, 266, 208, 4, 50, 53, 1, 1197, 15, 1201, 2015, 6408, 1253, 136, 28, 874, 1, 25, 1377, 2, 5147, 43, 840, 57, 1024, 1164, 1, 0, 607, 305, 2, 44, 31, 2631, 113, 1, 300, 973, 0, 669, 269, 5860, 330, 3196, 177, 132, 603, 3156, 46, 1, 123, 709, 3, 42, 2] payload_rows_per_dst=[8928, 13320, 9636, 5898, 13410, 13114, 9160, 7706] dup_rows_per_dst=[1888, 718, 2054, 426, 4630, 5366, 1592, 458] combine_rows_per_src=[8928, 13320, 9636, 5898, 13410, 13114, 9160, 7706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=255434752 dispatch_meta_bytes=317712 combine_payload_bytes=255434752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255752464 ingress_bytes=290037760 link_bytes=290037760 token_bytes=511187216 weight_movement_bytes=34603008 total_nvlink_bytes=545790224 weight_to_token_ratio=0.067691 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091388741e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82340 payload_rows_remote=74710 dup_rows=15964 dup_rows_remote=15076 dedup_ratio=0.162394 router_tokens_per_expert=[32, 210, 13, 8, 5, 4384, 876, 1402, 33, 182, 253, 177, 3, 80, 3, 2, 964, 74, 2140, 1322, 6359, 606, 141, 48, 1380, 1, 65, 869, 27, 3516, 123, 577, 55, 473, 422, 4, 13, 1298, 260, 1, 55, 224, 2213, 396, 97, 57, 608, 0, 763, 427, 4551, 656, 4869, 225, 413, 436, 2860, 11, 1, 341, 1458, 25, 61, 4] payload_rows_per_dst=[11860, 13316, 7630, 11106, 11142, 6638, 12192, 8456] dup_rows_per_dst=[4072, 868, 888, 2010, 3012, 662, 3386, 1066] combine_rows_per_src=[11860, 13316, 7630, 11106, 11142, 6638, 12192, 8456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=326056 combine_payload_bytes=279494656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279820712 ingress_bytes=314097664 link_bytes=314097664 token_bytes=559315368 weight_movement_bytes=34603008 total_nvlink_bytes=593918376 weight_to_token_ratio=0.061867 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181924223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=298057728 dispatch_meta_bytes=346504 combine_payload_bytes=298057728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298404232 ingress_bytes=332660736 link_bytes=332660736 token_bytes=596461960 weight_movement_bytes=34603008 total_nvlink_bytes=631064968 weight_to_token_ratio=0.058014 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251775568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81254 payload_rows_remote=67906 dup_rows=17050 dup_rows_remote=16694 dedup_ratio=0.173442 router_tokens_per_expert=[36, 32, 2, 0, 0, 2967, 1164, 1494, 26, 3, 98, 177, 1, 43, 18, 1, 911, 30, 1263, 3007, 6485, 690, 160, 44, 1957, 0, 107, 1097, 1, 4687, 24, 970, 11, 588, 1326, 1, 1, 524, 221, 0, 55, 16, 2438, 169, 2, 67, 1388, 0, 579, 377, 4325, 345, 4293, 183, 30, 344, 2445, 18, 1, 132, 1709, 11, 57, 1] payload_rows_per_dst=[11650, 13348, 5568, 13564, 11364, 7322, 10496, 7942] dup_rows_per_dst=[5754, 356, 628, 4122, 2630, 948, 1806, 806] combine_rows_per_src=[11650, 13348, 5568, 13564, 11364, 7322, 10496, 7942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277553152 dispatch_meta_bytes=321056 combine_payload_bytes=277553152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277874208 ingress_bytes=312156160 link_bytes=312156160 token_bytes=555427360 weight_movement_bytes=34603008 total_nvlink_bytes=590030368 weight_to_token_ratio=0.062300 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174618499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76228 payload_rows_remote=64298 dup_rows=22076 dup_rows_remote=18396 dedup_ratio=0.224569 router_tokens_per_expert=[961, 30, 9, 6, 54, 418, 0, 21, 2831, 931, 3, 0, 745, 3346, 770, 337, 1586, 4035, 0, 356, 1053, 1277, 110, 662, 139, 0, 4828, 161, 259, 149, 101, 909, 414, 474, 476, 237, 146, 0, 1841, 35, 1485, 987, 283, 31, 3, 5287, 870, 104, 13, 1856, 2, 124, 245, 1270, 2423, 0, 78, 93, 3761, 176, 81, 232, 5, 33] payload_rows_per_dst=[9458, 7756, 7976, 5920, 6264, 12930, 13994, 11930] dup_rows_per_dst=[866, 1416, 1746, 688, 982, 5170, 7528, 3680] combine_rows_per_src=[9458, 7756, 7976, 5920, 6264, 12930, 13994, 11930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=306012160 dispatch_meta_bytes=359144 combine_payload_bytes=306012160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306371304 ingress_bytes=409821184 link_bytes=409821184 token_bytes=612383464 weight_movement_bytes=103809024 total_nvlink_bytes=716192488 weight_to_token_ratio=0.169516 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.542124122e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278142976 dispatch_meta_bytes=338400 combine_payload_bytes=278142976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278481376 ingress_bytes=312745984 link_bytes=312745984 token_bytes=556624352 weight_movement_bytes=34603008 total_nvlink_bytes=591227360 weight_to_token_ratio=0.062166 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176837960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81614 payload_rows_remote=69478 dup_rows=16690 dup_rows_remote=15388 dedup_ratio=0.169779 router_tokens_per_expert=[100, 110, 0, 0, 623, 201, 140, 0, 64, 138, 3133, 6, 1194, 2618, 811, 0, 241, 6, 762, 134, 0, 4, 0, 2671, 437, 498, 5755, 0, 0, 26, 0, 3, 204, 4, 350, 688, 0, 0, 1191, 18, 2589, 1890, 134, 7324, 18, 0, 961, 4047, 1845, 10, 134, 137, 140, 5, 552, 3052, 1021, 977, 31, 1841, 3, 2, 242, 67] payload_rows_per_dst=[14964, 9652, 11908, 12136, 7182, 6562, 12138, 7072] dup_rows_per_dst=[2032, 1692, 3822, 1302, 1508, 842, 4196, 1296] combine_rows_per_src=[14964, 9652, 11908, 12136, 7182, 6562, 12138, 7072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77090 payload_rows_remote=69474 dup_rows=21214 dup_rows_remote=19530 dedup_ratio=0.215800 router_tokens_per_expert=[12, 196, 0, 0, 633, 287, 67, 0, 59, 59, 3545, 0, 1486, 1866, 1208, 0, 163, 0, 328, 35, 0, 0, 0, 1834, 789, 724, 4397, 0, 0, 63, 1, 12, 243, 0, 239, 771, 0, 0, 2053, 28, 2457, 2275, 281, 7334, 39, 0, 1873, 3793, 1717, 19, 244, 104, 350, 3, 362, 1522, 1473, 1954, 57, 1740, 9, 0, 391, 57] payload_rows_per_dst=[6492, 8170, 10242, 10404, 13620, 7616, 11618, 8928] dup_rows_per_dst=[790, 1186, 2064, 1568, 7374, 1684, 4114, 2434] combine_rows_per_src=[6492, 8170, 10242, 10404, 13620, 7616, 11618, 8928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81262 payload_rows_remote=69856 dup_rows=17042 dup_rows_remote=14818 dedup_ratio=0.173360 router_tokens_per_expert=[100, 15, 10, 2, 6, 2646, 1112, 774, 74, 11, 140, 103, 1, 321, 5, 0, 979, 27, 883, 2465, 6781, 239, 89, 35, 2784, 0, 59, 1135, 1, 5333, 37, 614, 9, 441, 1724, 2, 2, 1825, 202, 0, 53, 13, 1869, 42, 1, 41, 985, 2, 712, 230, 4746, 486, 3833, 214, 59, 368, 3167, 8, 0, 186, 1108, 0, 43, 0] payload_rows_per_dst=[7656, 13896, 8366, 8096, 6858, 12710, 11406, 12274] dup_rows_per_dst=[1674, 974, 1070, 1164, 1552, 3968, 2224, 4416] combine_rows_per_src=[7656, 13896, 8366, 8096, 6858, 12710, 11406, 12274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263364608 dispatch_meta_bytes=330776 combine_payload_bytes=263364608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263695384 ingress_bytes=297967616 link_bytes=297967616 token_bytes=527059992 weight_movement_bytes=34603008 total_nvlink_bytes=561663000 weight_to_token_ratio=0.065653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121228151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79366 payload_rows_remote=69914 dup_rows=18938 dup_rows_remote=15216 dedup_ratio=0.192647 router_tokens_per_expert=[24, 16, 13, 0, 2, 2399, 1020, 1317, 6, 11, 262, 68, 11, 99, 9, 0, 2156, 23, 2263, 1796, 6437, 1155, 210, 21, 1015, 0, 158, 1187, 3, 5922, 53, 419, 33, 468, 881, 2, 1, 454, 403, 0, 26, 31, 2419, 174, 6, 108, 867, 0, 610, 240, 4611, 462, 3851, 180, 180, 281, 3038, 10, 5, 136, 1384, 172, 43, 1] payload_rows_per_dst=[9452, 13172, 10838, 14456, 11456, 6606, 4822, 8564] dup_rows_per_dst=[3722, 634, 4468, 5346, 2250, 656, 848, 1014] combine_rows_per_src=[9452, 13172, 10838, 14456, 11456, 6606, 4822, 8564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284581888 dispatch_meta_bytes=339464 combine_payload_bytes=284581888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284921352 ingress_bytes=353787904 link_bytes=353787904 token_bytes=569503240 weight_movement_bytes=69206016 total_nvlink_bytes=638709256 weight_to_token_ratio=0.121520 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331275400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82188 payload_rows_remote=73056 dup_rows=16116 dup_rows_remote=13972 dedup_ratio=0.163940 router_tokens_per_expert=[3, 174, 0, 0, 830, 635, 174, 7, 25, 52, 4351, 0, 1001, 1481, 1021, 0, 253, 6, 291, 79, 14, 1, 0, 1709, 299, 547, 6455, 0, 0, 5, 2, 10, 303, 0, 865, 919, 0, 37, 1463, 146, 1293, 1905, 922, 7315, 25, 0, 1435, 4361, 786, 3, 49, 33, 486, 2, 173, 2350, 882, 1281, 138, 1800, 130, 0, 317, 308] payload_rows_per_dst=[15236, 6554, 10900, 13296, 9132, 6508, 12592, 7970] dup_rows_per_dst=[3040, 606, 2528, 1340, 2144, 842, 3874, 1742] combine_rows_per_src=[15236, 6554, 10900, 13296, 9132, 6508, 12592, 7970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284565504 dispatch_meta_bytes=356016 combine_payload_bytes=284565504 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=284921520 ingress_bytes=492183552 link_bytes=492183552 token_bytes=569487024 weight_movement_bytes=207618048 total_nvlink_bytes=777105072 weight_to_token_ratio=0.364570 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.852047082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80120 payload_rows_remote=68564 dup_rows=18184 dup_rows_remote=15240 dedup_ratio=0.184977 router_tokens_per_expert=[47, 436, 1, 9, 1624, 689, 71, 79, 175, 70, 3621, 3, 1278, 1898, 806, 3, 243, 3, 889, 95, 5, 0, 1, 1584, 697, 912, 5914, 4, 3, 65, 1, 8, 295, 4, 508, 684, 2, 17, 1054, 122, 2274, 1945, 384, 6586, 71, 0, 2302, 4430, 1111, 11, 236, 131, 221, 17, 299, 1216, 1216, 948, 35, 1318, 13, 6, 321, 141] payload_rows_per_dst=[14656, 12396, 11556, 12938, 7728, 8074, 5946, 6826] dup_rows_per_dst=[4428, 3312, 2944, 2986, 1534, 1272, 538, 1170] combine_rows_per_src=[14656, 12396, 11556, 12938, 7728, 8074, 5946, 6826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286130176 dispatch_meta_bytes=338696 combine_payload_bytes=286130176 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=286468872 ingress_bytes=424542208 link_bytes=424542208 token_bytes=572599048 weight_movement_bytes=138412032 total_nvlink_bytes=711011080 weight_to_token_ratio=0.241726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.597518150e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286367744 dispatch_meta_bytes=340520 combine_payload_bytes=286367744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286708264 ingress_bytes=320970752 link_bytes=320970752 token_bytes=573076008 weight_movement_bytes=34603008 total_nvlink_bytes=607679016 weight_to_token_ratio=0.060381 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207787100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80892 payload_rows_remote=66636 dup_rows=17412 dup_rows_remote=10838 dedup_ratio=0.177124 router_tokens_per_expert=[11, 438, 0, 0, 481, 90, 75, 2, 18, 42, 4016, 0, 1360, 3217, 1762, 0, 103, 1, 265, 73, 0, 0, 0, 1321, 650, 616, 5200, 0, 7, 36, 0, 14, 274, 0, 218, 591, 0, 1, 1069, 33, 1473, 1759, 97, 7205, 30, 0, 1621, 5842, 756, 23, 129, 204, 149, 4, 230, 2626, 1517, 714, 47, 2089, 10, 0, 445, 198] payload_rows_per_dst=[14678, 14256, 12870, 12578, 6910, 3420, 7610, 8570] dup_rows_per_dst=[1926, 6574, 2340, 3414, 980, 76, 632, 1470] combine_rows_per_src=[14678, 14256, 12870, 12578, 6910, 3420, 7610, 8570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299237376 dispatch_meta_bytes=348112 combine_payload_bytes=299237376 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=299585488 ingress_bytes=368443392 link_bytes=368443392 token_bytes=598822864 weight_movement_bytes=69206016 total_nvlink_bytes=668028880 weight_to_token_ratio=0.115570 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.386422822e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=280838144 dispatch_meta_bytes=335216 combine_payload_bytes=280838144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281173360 ingress_bytes=315441152 link_bytes=315441152 token_bytes=562011504 weight_movement_bytes=34603008 total_nvlink_bytes=596614512 weight_to_token_ratio=0.061570 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186979660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71528 payload_rows_remote=58704 dup_rows=26776 dup_rows_remote=21094 dedup_ratio=0.272380 router_tokens_per_expert=[176, 4, 37, 4, 1, 2655, 703, 2057, 63, 13, 219, 361, 2, 16, 43, 4, 1812, 36, 878, 3088, 6001, 811, 393, 72, 1056, 0, 201, 1230, 20, 3925, 70, 798, 8, 612, 1424, 13, 0, 399, 251, 0, 184, 12, 2592, 253, 6, 94, 1042, 0, 517, 79, 4421, 283, 4039, 233, 311, 416, 3377, 18, 0, 327, 1359, 2, 130, 1] payload_rows_per_dst=[9086, 1440, 15074, 6026, 4532, 11862, 10684, 12824] dup_rows_per_dst=[2188, 2, 11108, 724, 882, 4354, 1836, 5682] combine_rows_per_src=[9086, 1440, 15074, 6026, 4532, 11862, 10684, 12824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272941056 dispatch_meta_bytes=309896 combine_payload_bytes=272941056 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273250952 ingress_bytes=342147072 link_bytes=342147072 token_bytes=546192008 weight_movement_bytes=69206016 total_nvlink_bytes=615398024 weight_to_token_ratio=0.126706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287471887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83558 payload_rows_remote=71712 dup_rows=14746 dup_rows_remote=10822 dedup_ratio=0.150004 router_tokens_per_expert=[2950, 1, 6, 1510, 24, 1, 3399, 1150, 6567, 7572, 124, 18, 0, 123, 13, 260, 387, 17, 1250, 134, 108, 740, 24, 0, 781, 947, 15, 1, 458, 3, 89, 2880, 0, 3, 65, 182, 9, 167, 6, 1, 103, 366, 447, 618, 1951, 484, 72, 324, 2252, 10, 1, 173, 7, 534, 0, 16, 564, 84, 32, 0, 5, 2212, 3600, 3312] payload_rows_per_dst=[9952, 8240, 9958, 11846, 15178, 7656, 10618, 10110] dup_rows_per_dst=[1332, 548, 1986, 3924, 832, 1074, 2166, 2884] combine_rows_per_src=[9952, 8240, 9958, 11846, 15178, 7656, 10618, 10110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79942 payload_rows_remote=71632 dup_rows=18362 dup_rows_remote=16066 dedup_ratio=0.186788 router_tokens_per_expert=[2577, 1, 3, 585, 46, 1, 3934, 462, 5602, 8115, 102, 1, 0, 289, 16, 178, 188, 2, 1932, 236, 532, 858, 49, 1, 1319, 1367, 52, 6, 102, 0, 60, 522, 0, 2, 63, 453, 0, 227, 5, 2, 452, 564, 242, 683, 1847, 864, 408, 243, 2625, 0, 2, 4, 8, 265, 0, 2, 935, 52, 62, 1, 5, 2071, 3768, 4159] payload_rows_per_dst=[7080, 1168, 12084, 13414, 16256, 8310, 10948, 10682] dup_rows_per_dst=[270, 4, 3830, 4646, 1478, 2296, 2732, 3106] combine_rows_per_src=[7080, 1168, 12084, 13414, 16256, 8310, 10948, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=240451584 dispatch_meta_bytes=319192 combine_payload_bytes=240451584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240770776 ingress_bytes=275054592 link_bytes=275054592 token_bytes=481222360 weight_movement_bytes=34603008 total_nvlink_bytes=515825368 weight_to_token_ratio=0.071906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.035008286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72986 payload_rows_remote=61600 dup_rows=25318 dup_rows_remote=21580 dedup_ratio=0.257548 router_tokens_per_expert=[27, 210, 0, 2, 605, 127, 294, 6, 13, 99, 3293, 0, 1606, 1839, 1951, 0, 162, 2, 100, 20, 3, 1, 0, 1577, 690, 775, 5842, 0, 1, 9, 6, 29, 238, 0, 646, 667, 0, 1, 1602, 5, 1233, 2558, 679, 7235, 31, 0, 1832, 4433, 1095, 35, 46, 174, 370, 0, 455, 2094, 1072, 719, 101, 2173, 7, 1, 262, 99] payload_rows_per_dst=[2440, 9114, 7632, 3718, 10366, 14342, 11386, 13988] dup_rows_per_dst=[102, 1902, 1052, 286, 3132, 9526, 3738, 5580] combine_rows_per_src=[2440, 9114, 7632, 3718, 10366, 14342, 11386, 13988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79922 payload_rows_remote=66212 dup_rows=18382 dup_rows_remote=16176 dedup_ratio=0.186991 router_tokens_per_expert=[126, 135, 4, 6, 614, 484, 95, 1, 50, 72, 4038, 28, 802, 1760, 554, 2, 185, 1, 260, 82, 4, 1, 2, 2421, 405, 412, 5838, 0, 0, 5, 0, 24, 247, 0, 654, 755, 0, 2, 1240, 28, 3466, 4320, 546, 6493, 137, 0, 1352, 3829, 906, 1, 183, 133, 523, 5, 73, 1501, 1749, 1453, 130, 607, 2, 0, 294, 112] payload_rows_per_dst=[13710, 12154, 11002, 14592, 11208, 3894, 5926, 7436] dup_rows_per_dst=[2206, 2458, 2568, 5708, 3284, 176, 724, 1258] combine_rows_per_src=[13710, 12154, 11002, 14592, 11208, 3894, 5926, 7436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81374 payload_rows_remote=66544 dup_rows=16930 dup_rows_remote=15638 dedup_ratio=0.172221 router_tokens_per_expert=[3619, 1, 3, 720, 17, 1, 3156, 326, 5917, 7348, 144, 4, 18, 246, 9, 155, 396, 18, 1529, 136, 403, 213, 249, 15, 616, 721, 26, 31, 46, 2, 82, 266, 0, 19, 184, 183, 4, 315, 8, 0, 283, 363, 106, 717, 2909, 1590, 216, 576, 3113, 10, 7, 11, 12, 291, 0, 3, 1124, 239, 42, 1, 11, 2254, 3989, 4139] payload_rows_per_dst=[8864, 1128, 11362, 12794, 14830, 10652, 10420, 11324] dup_rows_per_dst=[510, 24, 2834, 2620, 1292, 2868, 2786, 3996] combine_rows_per_src=[8864, 1128, 11362, 12794, 14830, 10652, 10420, 11324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=293732352 dispatch_meta_bytes=330136 combine_payload_bytes=293732352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294062488 ingress_bytes=328335360 link_bytes=328335360 token_bytes=587794840 weight_movement_bytes=34603008 total_nvlink_bytes=622397848 weight_to_token_ratio=0.058869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235499527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=293404672 dispatch_meta_bytes=350792 combine_payload_bytes=293404672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=293755464 ingress_bytes=362610688 link_bytes=362610688 token_bytes=587160136 weight_movement_bytes=69206016 total_nvlink_bytes=656366152 weight_to_token_ratio=0.117866 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364474826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76816 payload_rows_remote=67634 dup_rows=21488 dup_rows_remote=19894 dedup_ratio=0.218587 router_tokens_per_expert=[2806, 21, 25, 1028, 463, 28, 3976, 331, 5196, 7057, 171, 1, 1, 52, 33, 348, 329, 12, 1272, 359, 320, 139, 18, 3, 1172, 1022, 46, 14, 133, 21, 57, 473, 0, 24, 379, 585, 21, 598, 12, 1, 730, 992, 164, 711, 1973, 1851, 484, 648, 2817, 236, 1, 1, 45, 517, 0, 53, 1011, 631, 92, 0, 80, 1280, 3352, 2936] payload_rows_per_dst=[10678, 13460, 9182, 5098, 8620, 10898, 9266, 9614] dup_rows_per_dst=[2804, 5454, 1594, 778, 1424, 4208, 1948, 3278] combine_rows_per_src=[10678, 13460, 9182, 5098, 8620, 10898, 9266, 9614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=252313600 dispatch_meta_bytes=332720 combine_payload_bytes=252313600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252646320 ingress_bytes=286916608 link_bytes=286916608 token_bytes=504959920 weight_movement_bytes=34603008 total_nvlink_bytes=539562928 weight_to_token_ratio=0.068526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079644097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271204352 dispatch_meta_bytes=329552 combine_payload_bytes=271204352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271533904 ingress_bytes=305807360 link_bytes=305807360 token_bytes=542738256 weight_movement_bytes=34603008 total_nvlink_bytes=577341264 weight_to_token_ratio=0.063756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150728476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272564224 dispatch_meta_bytes=328728 combine_payload_bytes=272564224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272892952 ingress_bytes=307167232 link_bytes=307167232 token_bytes=545457176 weight_movement_bytes=34603008 total_nvlink_bytes=580060184 weight_to_token_ratio=0.063439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155845565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71248 payload_rows_remote=55024 dup_rows=27056 dup_rows_remote=11702 dedup_ratio=0.275228 router_tokens_per_expert=[3102, 11, 0, 871, 25, 2, 3692, 455, 5482, 7670, 72, 3, 0, 74, 4, 119, 274, 4, 1845, 394, 236, 208, 38, 5, 478, 1571, 32, 14, 261, 1, 53, 394, 0, 16, 291, 500, 1, 607, 14, 0, 261, 658, 314, 917, 2270, 1086, 351, 418, 4304, 2, 5, 8, 3, 206, 0, 6, 689, 42, 79, 7, 4, 1300, 3815, 3588] payload_rows_per_dst=[12582, 16224, 10518, 4998, 2616, 9226, 8910, 6174] dup_rows_per_dst=[4390, 15354, 2666, 610, 242, 2668, 158, 968] combine_rows_per_src=[12582, 16224, 10518, 4998, 2616, 9226, 8910, 6174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277028864 dispatch_meta_bytes=350112 combine_payload_bytes=277028864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277378976 ingress_bytes=311631872 link_bytes=311631872 token_bytes=554407840 weight_movement_bytes=34603008 total_nvlink_bytes=589010848 weight_to_token_ratio=0.062414 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172645646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66700 payload_rows_remote=51786 dup_rows=31604 dup_rows_remote=23954 dedup_ratio=0.321493 router_tokens_per_expert=[55, 62, 0, 7, 502, 108, 62, 6, 26, 156, 2666, 2, 1571, 2246, 1185, 0, 95, 11, 122, 46, 20, 2, 0, 2142, 404, 576, 6050, 2, 0, 24, 10, 3, 274, 1, 407, 848, 0, 22, 1336, 195, 1927, 3018, 622, 6857, 4, 0, 1112, 5166, 1126, 17, 264, 160, 418, 2, 122, 1861, 1761, 1062, 225, 1825, 7, 1, 153, 198] payload_rows_per_dst=[1586, 7276, 4708, 1986, 9712, 16054, 10464, 14914] dup_rows_per_dst=[18, 1002, 168, 52, 2490, 17416, 2808, 7650] combine_rows_per_src=[1586, 7276, 4708, 1986, 9712, 16054, 10464, 14914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=225378304 dispatch_meta_bytes=266904 combine_payload_bytes=225378304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=225645208 ingress_bytes=329187328 link_bytes=329187328 token_bytes=451023512 weight_movement_bytes=103809024 total_nvlink_bytes=554832536 weight_to_token_ratio=0.230163 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238705414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79168 payload_rows_remote=68944 dup_rows=19136 dup_rows_remote=16750 dedup_ratio=0.194661 router_tokens_per_expert=[0, 105, 1119, 2592, 406, 1139, 0, 70, 0, 10, 0, 43, 2036, 176, 3601, 2556, 7, 3082, 0, 0, 12, 0, 88, 0, 126, 322, 568, 88, 62, 157, 975, 0, 281, 214, 1620, 1942, 107, 3774, 0, 0, 407, 0, 73, 1142, 8, 0, 1951, 364, 270, 5, 101, 2172, 1947, 4520, 489, 1, 0, 232, 0, 12, 4007, 4121, 29, 23] payload_rows_per_dst=[9342, 12204, 11654, 10224, 11754, 7136, 8378, 8476] dup_rows_per_dst=[1520, 4640, 3764, 2386, 4122, 754, 1592, 358] combine_rows_per_src=[9342, 12204, 11654, 10224, 11754, 7136, 8378, 8476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78484 payload_rows_remote=71462 dup_rows=19820 dup_rows_remote=18784 dedup_ratio=0.201619 router_tokens_per_expert=[1, 207, 1079, 1695, 571, 1679, 0, 67, 0, 3, 0, 47, 1151, 140, 2316, 1752, 8, 2917, 0, 0, 2, 2, 29, 1, 208, 255, 647, 99, 22, 358, 574, 0, 217, 76, 1260, 2972, 112, 4524, 1, 0, 274, 2, 94, 1310, 9, 0, 1544, 796, 154, 1, 877, 2474, 1253, 4417, 545, 0, 0, 287, 0, 9, 5649, 4448, 14, 3] payload_rows_per_dst=[8264, 8964, 10946, 12700, 9512, 7022, 11920, 9156] dup_rows_per_dst=[1434, 2246, 3806, 2924, 2816, 1036, 5192, 366] combine_rows_per_src=[8264, 8964, 10946, 12700, 9512, 7022, 11920, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=212115456 dispatch_meta_bytes=302960 combine_payload_bytes=212115456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=212418416 ingress_bytes=246718464 link_bytes=246718464 token_bytes=424533872 weight_movement_bytes=34603008 total_nvlink_bytes=459136880 weight_to_token_ratio=0.081508 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.283817176e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80034 payload_rows_remote=67900 dup_rows=18270 dup_rows_remote=15506 dedup_ratio=0.185852 router_tokens_per_expert=[3056, 3, 3, 687, 73, 7, 2324, 448, 5378, 7392, 132, 11, 0, 155, 40, 139, 204, 6, 870, 219, 479, 417, 30, 4, 420, 1198, 43, 34, 573, 0, 68, 677, 0, 3, 69, 184, 0, 486, 1, 0, 735, 546, 302, 1319, 3400, 1288, 191, 1147, 4849, 2, 12, 23, 5, 233, 0, 1, 826, 94, 32, 0, 9, 2043, 2555, 3707] payload_rows_per_dst=[8036, 938, 9972, 12874, 14914, 12420, 12134, 8746] dup_rows_per_dst=[518, 16, 1900, 3908, 1356, 5436, 2764, 2372] combine_rows_per_src=[8036, 938, 9972, 12874, 14914, 12420, 12134, 8746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65870 payload_rows_remote=51850 dup_rows=32434 dup_rows_remote=23718 dedup_ratio=0.329936 router_tokens_per_expert=[2823, 3, 2, 727, 29, 52, 3590, 369, 4760, 7743, 85, 3, 0, 124, 10, 315, 175, 20, 1817, 425, 822, 225, 221, 3, 339, 1708, 1, 8, 77, 1, 50, 462, 0, 4, 58, 576, 2, 248, 50, 2, 345, 203, 252, 1250, 3773, 1474, 281, 171, 2299, 6, 1, 86, 22, 285, 1, 6, 1188, 64, 60, 1, 40, 1238, 4460, 3717] payload_rows_per_dst=[14020, 16284, 11734, 4720, 1838, 7156, 5300, 4818] dup_rows_per_dst=[8716, 18716, 3116, 572, 42, 796, 112, 364] combine_rows_per_src=[14020, 16284, 11734, 4720, 1838, 7156, 5300, 4818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80224 payload_rows_remote=68732 dup_rows=18080 dup_rows_remote=14896 dedup_ratio=0.183919 router_tokens_per_expert=[1, 445, 1685, 2508, 544, 1969, 0, 4, 0, 31, 0, 87, 1745, 161, 2071, 1300, 8, 2324, 0, 0, 14, 0, 64, 0, 59, 624, 412, 213, 6, 109, 775, 0, 564, 608, 724, 1275, 40, 4125, 0, 2, 703, 2, 188, 2390, 9, 0, 4083, 547, 213, 3, 380, 2382, 376, 3558, 448, 2, 1, 86, 0, 20, 3774, 5474, 16, 0] payload_rows_per_dst=[11016, 8750, 9544, 9934, 11492, 12218, 6294, 10976] dup_rows_per_dst=[3296, 2040, 2392, 2010, 3184, 3626, 1314, 218] combine_rows_per_src=[11016, 8750, 9544, 9934, 11492, 12218, 6294, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282394624 dispatch_meta_bytes=342776 combine_payload_bytes=282394624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282737400 ingress_bytes=316997632 link_bytes=316997632 token_bytes=565132024 weight_movement_bytes=34603008 total_nvlink_bytes=599735032 weight_to_token_ratio=0.061230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192836569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292708352 dispatch_meta_bytes=360984 combine_payload_bytes=292708352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293069336 ingress_bytes=327311360 link_bytes=327311360 token_bytes=585777688 weight_movement_bytes=34603008 total_nvlink_bytes=620380696 weight_to_token_ratio=0.059072 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231646297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=69196 dup_rows=17856 dup_rows_remote=15074 dedup_ratio=0.181641 router_tokens_per_expert=[10, 259, 1438, 3021, 990, 1817, 0, 150, 17, 75, 0, 93, 1072, 28, 2720, 1156, 1, 1977, 0, 8, 5, 4, 48, 32, 120, 267, 502, 763, 16, 69, 665, 5, 501, 52, 1226, 617, 170, 4039, 0, 0, 393, 0, 72, 3355, 28, 8, 1973, 738, 268, 17, 390, 2331, 459, 4942, 457, 8, 1, 266, 0, 26, 4674, 4685, 123, 5] payload_rows_per_dst=[11756, 8704, 11252, 11468, 10468, 10640, 6434, 9726] dup_rows_per_dst=[3614, 1618, 2782, 2694, 2742, 2494, 1426, 486] combine_rows_per_src=[11756, 8704, 11252, 11468, 10468, 10640, 6434, 9726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=333624 combine_payload_bytes=278118400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278452024 ingress_bytes=312721408 link_bytes=312721408 token_bytes=556570424 weight_movement_bytes=34603008 total_nvlink_bytes=591173432 weight_to_token_ratio=0.062172 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176745482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=212377600 dispatch_meta_bytes=302272 combine_payload_bytes=212377600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=212679872 ingress_bytes=281583616 link_bytes=281583616 token_bytes=425057472 weight_movement_bytes=69206016 total_nvlink_bytes=494263488 weight_to_token_ratio=0.162816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.059576478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281526272 dispatch_meta_bytes=334512 combine_payload_bytes=281526272 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281860784 ingress_bytes=385335296 link_bytes=385335296 token_bytes=563387056 weight_movement_bytes=103809024 total_nvlink_bytes=667196080 weight_to_token_ratio=0.184259 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449985697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79822 payload_rows_remote=70868 dup_rows=18482 dup_rows_remote=16408 dedup_ratio=0.188009 router_tokens_per_expert=[0, 275, 1135, 3359, 148, 2847, 0, 50, 0, 13, 0, 6, 1212, 251, 2280, 1752, 1, 3220, 0, 0, 2, 3, 35, 2, 93, 205, 464, 70, 37, 78, 880, 0, 170, 134, 1037, 571, 19, 5146, 0, 3, 276, 0, 210, 2861, 0, 0, 2244, 604, 228, 33, 162, 2790, 1112, 4013, 487, 4, 0, 310, 0, 5, 5021, 3282, 12, 0] payload_rows_per_dst=[12012, 8954, 10972, 11644, 11630, 9810, 7988, 6812] dup_rows_per_dst=[3616, 2074, 3580, 2052, 2530, 2580, 1644, 406] combine_rows_per_src=[12012, 8954, 10972, 11644, 11630, 9810, 7988, 6812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76628 payload_rows_remote=63302 dup_rows=21676 dup_rows_remote=15134 dedup_ratio=0.220500 router_tokens_per_expert=[2625, 9, 8, 622, 72, 12, 4512, 928, 4729, 7268, 108, 8, 0, 68, 110, 79, 225, 2, 1453, 452, 337, 207, 52, 0, 478, 1228, 39, 1, 167, 11, 110, 988, 0, 11, 172, 1094, 11, 114, 22, 1, 448, 403, 182, 1671, 2640, 709, 178, 467, 4217, 3, 3, 49, 110, 124, 0, 3, 759, 77, 58, 2, 20, 2016, 3170, 3510] payload_rows_per_dst=[7776, 746, 5148, 12100, 14962, 9656, 12914, 13326] dup_rows_per_dst=[776, 0, 420, 3402, 2424, 2984, 5128, 6542] combine_rows_per_src=[7776, 746, 5148, 12100, 14962, 9656, 12914, 13326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=283426816 dispatch_meta_bytes=337080 combine_payload_bytes=283426816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283763896 ingress_bytes=318029824 link_bytes=318029824 token_bytes=567190712 weight_movement_bytes=34603008 total_nvlink_bytes=601793720 weight_to_token_ratio=0.061008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196720624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=290275328 dispatch_meta_bytes=349104 combine_payload_bytes=290275328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290624432 ingress_bytes=324878336 link_bytes=324878336 token_bytes=580899760 weight_movement_bytes=34603008 total_nvlink_bytes=615502768 weight_to_token_ratio=0.059568 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222491024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82664 payload_rows_remote=71086 dup_rows=15640 dup_rows_remote=14034 dedup_ratio=0.159098 router_tokens_per_expert=[440, 251, 131, 310, 0, 56, 115, 208, 0, 3, 4, 4754, 0, 1941, 6615, 10, 101, 135, 484, 590, 3148, 5963, 70, 56, 0, 5175, 533, 431, 25, 1, 394, 33, 0, 0, 1, 1347, 0, 2540, 1113, 0, 0, 395, 14, 387, 1425, 1809, 26, 305, 2979, 14, 0, 65, 0, 1, 1155, 339, 160, 436, 268, 1175, 369, 2, 3, 847] payload_rows_per_dst=[14014, 11472, 9348, 11578, 8144, 7782, 8084, 12242] dup_rows_per_dst=[2238, 1952, 2464, 1606, 1858, 940, 1022, 3560] combine_rows_per_src=[14014, 11472, 9348, 11578, 8144, 7782, 8084, 12242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80012 payload_rows_remote=68754 dup_rows=18292 dup_rows_remote=16096 dedup_ratio=0.186076 router_tokens_per_expert=[366, 236, 180, 12, 0, 182, 331, 647, 0, 5, 6, 4184, 1, 2205, 6311, 11, 111, 26, 322, 675, 3618, 5038, 200, 4, 1, 4059, 501, 171, 8, 0, 401, 18, 0, 0, 1, 750, 0, 2776, 2263, 0, 0, 467, 29, 704, 1499, 3438, 11, 579, 2448, 0, 0, 145, 0, 4, 1429, 138, 201, 565, 162, 394, 11, 0, 0, 1308] payload_rows_per_dst=[12352, 5780, 8870, 9350, 11718, 11258, 8590, 12094] dup_rows_per_dst=[2330, 524, 1042, 968, 5216, 2196, 2752, 3264] combine_rows_per_src=[12352, 5780, 8870, 9350, 11718, 11258, 8590, 12094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76164 payload_rows_remote=62080 dup_rows=22140 dup_rows_remote=13220 dedup_ratio=0.225220 router_tokens_per_expert=[0, 243, 2451, 2658, 250, 2655, 0, 28, 4, 18, 0, 72, 1773, 226, 2794, 1635, 12, 3755, 0, 0, 35, 7, 25, 2, 78, 256, 488, 133, 39, 112, 752, 0, 316, 273, 817, 1376, 185, 4174, 0, 0, 251, 4, 58, 1947, 8, 3, 1850, 499, 9, 3, 247, 2051, 935, 4512, 504, 6, 0, 212, 0, 5, 5391, 2942, 16, 57] payload_rows_per_dst=[9406, 10260, 8442, 8364, 5744, 10556, 14084, 9308] dup_rows_per_dst=[1848, 2784, 716, 1324, 1084, 3106, 8920, 2358] combine_rows_per_src=[9406, 10260, 8442, 8364, 5744, 10556, 14084, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=259284992 dispatch_meta_bytes=313744 combine_payload_bytes=259284992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259598736 ingress_bytes=363094016 link_bytes=363094016 token_bytes=518883728 weight_movement_bytes=103809024 total_nvlink_bytes=622692752 weight_to_token_ratio=0.200062 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366293551e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81556 payload_rows_remote=70854 dup_rows=16748 dup_rows_remote=13962 dedup_ratio=0.170369 router_tokens_per_expert=[0, 147, 1709, 2815, 606, 1452, 0, 15, 0, 37, 0, 14, 2151, 64, 2070, 2347, 8, 1592, 0, 0, 5, 3, 146, 1, 263, 131, 432, 300, 22, 125, 752, 0, 131, 165, 1566, 512, 99, 5575, 0, 0, 367, 5, 287, 1761, 4, 9, 2340, 276, 208, 2, 313, 1634, 643, 4125, 417, 72, 0, 219, 0, 3, 6222, 4900, 81, 9] payload_rows_per_dst=[10702, 10434, 9682, 13444, 12818, 8700, 5768, 10008] dup_rows_per_dst=[2786, 2932, 2078, 3050, 3278, 1398, 810, 416] combine_rows_per_src=[10702, 10434, 9682, 13444, 12818, 8700, 5768, 10008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80148 payload_rows_remote=67526 dup_rows=18156 dup_rows_remote=11762 dedup_ratio=0.184692 router_tokens_per_expert=[303, 241, 166, 12, 0, 33, 198, 101, 0, 2, 85, 3716, 0, 2646, 5857, 2, 65, 2, 239, 347, 2703, 4918, 359, 24, 84, 5396, 727, 373, 5, 1, 701, 1, 0, 0, 1, 1232, 13, 4273, 950, 0, 0, 548, 20, 352, 775, 2232, 91, 798, 3716, 18, 5, 77, 0, 2, 1266, 114, 142, 164, 548, 281, 799, 0, 0, 1428] payload_rows_per_dst=[12318, 6440, 6900, 12204, 12622, 8452, 9014, 12198] dup_rows_per_dst=[1504, 384, 578, 2372, 6394, 1180, 1382, 4362] combine_rows_per_src=[12318, 6440, 6900, 12204, 12622, 8452, 9014, 12198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=291168256 dispatch_meta_bytes=340480 combine_payload_bytes=291168256 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=291508736 ingress_bytes=394977280 link_bytes=394977280 token_bytes=582676992 weight_movement_bytes=103809024 total_nvlink_bytes=686486016 weight_to_token_ratio=0.178159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486267706e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281616384 dispatch_meta_bytes=339400 combine_payload_bytes=281616384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281955784 ingress_bytes=316219392 link_bytes=316219392 token_bytes=563572168 weight_movement_bytes=34603008 total_nvlink_bytes=598175176 weight_to_token_ratio=0.061399 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189908114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=254279680 dispatch_meta_bytes=301200 combine_payload_bytes=254279680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254580880 ingress_bytes=323485696 link_bytes=323485696 token_bytes=508860560 weight_movement_bytes=69206016 total_nvlink_bytes=578066576 weight_to_token_ratio=0.136002 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217250631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81508 payload_rows_remote=68740 dup_rows=16796 dup_rows_remote=11742 dedup_ratio=0.170858 router_tokens_per_expert=[552, 164, 426, 10, 7, 66, 671, 608, 40, 3, 20, 4005, 35, 1134, 6010, 2, 279, 4, 341, 611, 2173, 5178, 92, 233, 24, 5495, 294, 266, 7, 32, 348, 6, 3, 2, 0, 239, 50, 3253, 1115, 0, 4, 410, 0, 875, 1416, 1375, 65, 1898, 4119, 1, 7, 146, 0, 56, 1624, 109, 469, 682, 108, 1079, 252, 0, 1, 658] payload_rows_per_dst=[13744, 9546, 12768, 11792, 8056, 9870, 10234, 5498] dup_rows_per_dst=[3284, 932, 5054, 1152, 1268, 2216, 1890, 1000] combine_rows_per_src=[13744, 9546, 12768, 11792, 8056, 9870, 10234, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=290217984 dispatch_meta_bytes=339264 combine_payload_bytes=290217984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=290557248 ingress_bytes=394027008 link_bytes=394027008 token_bytes=580775232 weight_movement_bytes=103809024 total_nvlink_bytes=684584256 weight_to_token_ratio=0.178742 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482691909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81244 payload_rows_remote=70380 dup_rows=17060 dup_rows_remote=16008 dedup_ratio=0.173543 router_tokens_per_expert=[388, 213, 186, 11, 0, 28, 1007, 110, 0, 2, 29, 4616, 9, 1302, 5838, 0, 68, 2, 190, 356, 3193, 5669, 124, 60, 8, 5205, 350, 376, 16, 0, 480, 1, 0, 0, 2, 812, 0, 2191, 638, 0, 0, 361, 10, 1256, 1940, 1861, 67, 1656, 3583, 14, 2, 288, 0, 9, 1930, 63, 404, 290, 249, 803, 166, 0, 0, 720] payload_rows_per_dst=[12928, 10864, 13478, 11474, 6244, 11524, 10044, 4688] dup_rows_per_dst=[2634, 1052, 5846, 1398, 1042, 2778, 1734, 576] combine_rows_per_src=[12928, 10864, 13478, 11474, 6244, 11524, 10044, 4688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276586496 dispatch_meta_bytes=317152 combine_payload_bytes=276586496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276903648 ingress_bytes=311189504 link_bytes=311189504 token_bytes=553490144 weight_movement_bytes=34603008 total_nvlink_bytes=588093152 weight_to_token_ratio=0.062518 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170981051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71866 payload_rows_remote=56592 dup_rows=26438 dup_rows_remote=12478 dedup_ratio=0.268941 router_tokens_per_expert=[0, 191, 878, 2480, 780, 1990, 0, 7, 0, 11, 0, 11, 1617, 236, 2398, 1577, 19, 3255, 0, 0, 6, 14, 52, 1, 146, 325, 443, 297, 36, 95, 701, 0, 251, 153, 835, 1221, 105, 4252, 0, 0, 395, 0, 259, 3078, 1, 0, 1616, 785, 93, 3, 169, 2404, 1141, 4022, 438, 0, 2, 303, 0, 25, 4481, 5549, 3, 2] payload_rows_per_dst=[6906, 9758, 6614, 3788, 4688, 12576, 12262, 15274] dup_rows_per_dst=[786, 1942, 80, 298, 442, 4652, 4278, 13960] combine_rows_per_src=[6906, 9758, 6614, 3788, 4688, 12576, 12262, 15274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281559040 dispatch_meta_bytes=321928 combine_payload_bytes=281559040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281880968 ingress_bytes=350765056 link_bytes=350765056 token_bytes=563440008 weight_movement_bytes=69206016 total_nvlink_bytes=632646024 weight_to_token_ratio=0.122828 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319900667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288276480 dispatch_meta_bytes=345552 combine_payload_bytes=288276480 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=288622032 ingress_bytes=426688512 link_bytes=426688512 token_bytes=576898512 weight_movement_bytes=138412032 total_nvlink_bytes=715310544 weight_to_token_ratio=0.239924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605594519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84172 payload_rows_remote=72206 dup_rows=14132 dup_rows_remote=13932 dedup_ratio=0.143758 router_tokens_per_expert=[0, 0, 0, 2071, 257, 22, 0, 3, 714, 256, 1, 0, 1, 44, 3119, 1860, 36, 0, 75, 192, 4940, 2833, 0, 1, 79, 2174, 116, 370, 3953, 2404, 1, 1848, 205, 0, 67, 1092, 150, 91, 484, 3, 0, 0, 127, 6, 0, 0, 19, 0, 273, 0, 5931, 3564, 548, 0, 140, 2345, 332, 1293, 3423, 0, 0, 0, 755, 934] payload_rows_per_dst=[7862, 10168, 13438, 11306, 10554, 11966, 11350, 7528] dup_rows_per_dst=[648, 1822, 2716, 2678, 1536, 200, 2390, 2142] combine_rows_per_src=[7862, 10168, 13438, 11306, 10554, 11966, 11350, 7528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80582 payload_rows_remote=68682 dup_rows=17722 dup_rows_remote=13918 dedup_ratio=0.180278 router_tokens_per_expert=[1, 93, 0, 2206, 449, 23, 10, 33, 972, 1120, 0, 0, 5, 64, 1037, 1324, 13, 17, 85, 159, 4247, 2209, 2, 4, 193, 1549, 101, 574, 2851, 3590, 0, 1845, 380, 0, 84, 2338, 282, 126, 1340, 1, 2, 0, 391, 21, 1, 0, 16, 0, 338, 0, 4173, 3829, 273, 1, 510, 2219, 721, 2458, 2875, 1, 1, 0, 1219, 776] payload_rows_per_dst=[9866, 7336, 11408, 11900, 11310, 8914, 11452, 8396] dup_rows_per_dst=[1514, 1708, 2064, 3804, 3494, 294, 2888, 1956] combine_rows_per_src=[9866, 7336, 11408, 11900, 11310, 8914, 11452, 8396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81520 payload_rows_remote=69934 dup_rows=16784 dup_rows_remote=12680 dedup_ratio=0.170736 router_tokens_per_expert=[346, 504, 202, 113, 0, 178, 400, 268, 2, 22, 150, 3575, 0, 1832, 7309, 2, 36, 6, 314, 241, 3258, 6254, 102, 6, 7, 5973, 221, 262, 41, 1, 945, 2, 0, 0, 0, 990, 1, 3020, 1764, 0, 0, 368, 28, 460, 1166, 2071, 39, 741, 3144, 10, 1, 191, 0, 11, 781, 132, 140, 284, 185, 325, 59, 0, 5, 664] payload_rows_per_dst=[3460, 15222, 7416, 12704, 9276, 8454, 11586, 13402] dup_rows_per_dst=[562, 3412, 510, 2200, 2274, 1292, 4104, 2430] combine_rows_per_src=[3460, 15222, 7416, 12704, 9276, 8454, 11586, 13402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231800832 dispatch_meta_bytes=276280 combine_payload_bytes=231800832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=232077112 ingress_bytes=301006848 link_bytes=301006848 token_bytes=463877944 weight_movement_bytes=69206016 total_nvlink_bytes=533083960 weight_to_token_ratio=0.149190 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79340 payload_rows_remote=67942 dup_rows=18964 dup_rows_remote=17278 dedup_ratio=0.192912 router_tokens_per_expert=[245, 232, 169, 2, 0, 203, 234, 88, 0, 2, 58, 4104, 0, 1921, 5369, 28, 128, 33, 585, 618, 3151, 3638, 25, 5, 24, 3318, 809, 1153, 12, 0, 1190, 1, 0, 0, 9, 1693, 288, 2507, 2359, 1, 0, 371, 3, 1368, 1097, 2495, 57, 748, 3756, 9, 6, 225, 0, 32, 722, 407, 319, 506, 629, 837, 41, 0, 4, 1318] payload_rows_per_dst=[11398, 11468, 11904, 8700, 10288, 10478, 9040, 6064] dup_rows_per_dst=[1686, 2692, 5234, 1608, 3426, 1800, 1274, 1244] combine_rows_per_src=[11398, 11468, 11904, 8700, 10288, 10478, 9040, 6064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81532 payload_rows_remote=69184 dup_rows=16772 dup_rows_remote=14314 dedup_ratio=0.170614 router_tokens_per_expert=[0, 9, 0, 2865, 418, 5, 2, 108, 636, 1114, 19, 0, 2, 115, 1102, 909, 164, 20, 381, 137, 4446, 1984, 21, 0, 302, 1457, 578, 1524, 5273, 1904, 3, 488, 364, 0, 83, 481, 286, 73, 829, 14, 1, 0, 182, 1, 0, 1, 1, 0, 356, 1, 4409, 2092, 408, 2, 852, 1247, 1690, 2127, 5890, 1, 16, 14, 743, 1002] payload_rows_per_dst=[13922, 6416, 12166, 10050, 12348, 9038, 8602, 8990] dup_rows_per_dst=[4672, 1378, 2140, 2462, 2458, 152, 1314, 2196] combine_rows_per_src=[13922, 6416, 12166, 10050, 12348, 9038, 8602, 8990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=281321472 dispatch_meta_bytes=330400 combine_payload_bytes=281321472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=281651872 ingress_bytes=419733504 link_bytes=419733504 token_bytes=562973344 weight_movement_bytes=138412032 total_nvlink_bytes=701385376 weight_to_token_ratio=0.245859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.579423384e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=295755776 dispatch_meta_bytes=344552 combine_payload_bytes=295755776 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296100328 ingress_bytes=364961792 link_bytes=364961792 token_bytes=591856104 weight_movement_bytes=69206016 total_nvlink_bytes=661062120 weight_to_token_ratio=0.116930 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373321841e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=286449664 dispatch_meta_bytes=330456 combine_payload_bytes=286449664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286780120 ingress_bytes=321052672 link_bytes=321052672 token_bytes=573229784 weight_movement_bytes=34603008 total_nvlink_bytes=607832792 weight_to_token_ratio=0.060365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208095358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80636 payload_rows_remote=73548 dup_rows=17668 dup_rows_remote=17022 dedup_ratio=0.179728 router_tokens_per_expert=[0, 43, 5, 2970, 315, 89, 2, 393, 930, 1357, 12, 0, 0, 130, 2696, 2329, 97, 8, 212, 210, 2424, 733, 12, 4, 205, 1895, 173, 885, 3760, 1817, 13, 2537, 359, 9, 179, 503, 142, 169, 779, 29, 22, 2, 160, 71, 0, 0, 48, 1, 489, 2, 5614, 1637, 181, 0, 571, 2023, 937, 1661, 4671, 4, 71, 5, 1366, 1191] payload_rows_per_dst=[12730, 10924, 7088, 11340, 10234, 11540, 8650, 8130] dup_rows_per_dst=[4246, 4020, 646, 3340, 1624, 296, 1156, 2340] combine_rows_per_src=[12730, 10924, 7088, 11340, 10234, 11540, 8650, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=340880 combine_payload_bytes=278290432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278631312 ingress_bytes=312893440 link_bytes=312893440 token_bytes=556921744 weight_movement_bytes=34603008 total_nvlink_bytes=591524752 weight_to_token_ratio=0.062133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177392825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78490 payload_rows_remote=66650 dup_rows=19814 dup_rows_remote=13690 dedup_ratio=0.201558 router_tokens_per_expert=[0, 2, 0, 2846, 335, 75, 0, 14, 1207, 1142, 11, 0, 4, 196, 1417, 1736, 54, 7, 110, 661, 3743, 843, 3, 2, 334, 674, 75, 1045, 2505, 3269, 6, 1636, 179, 0, 55, 678, 27, 62, 913, 3, 2, 0, 105, 9, 0, 1, 15, 3, 634, 1, 5383, 3954, 952, 0, 570, 1616, 1348, 811, 5083, 0, 1, 0, 1527, 1268] payload_rows_per_dst=[12856, 11840, 9474, 6474, 7652, 11840, 10454, 7900] dup_rows_per_dst=[3854, 6124, 1372, 1066, 1192, 2258, 1938, 2010] combine_rows_per_src=[12856, 11840, 9474, 6474, 7652, 11840, 10454, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283377664 dispatch_meta_bytes=333992 combine_payload_bytes=283377664 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283711656 ingress_bytes=352583680 link_bytes=352583680 token_bytes=567089320 weight_movement_bytes=69206016 total_nvlink_bytes=636295336 weight_to_token_ratio=0.122037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326744003e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=301252608 dispatch_meta_bytes=362280 combine_payload_bytes=301252608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301614888 ingress_bytes=405061632 link_bytes=405061632 token_bytes=602867496 weight_movement_bytes=103809024 total_nvlink_bytes=706676520 weight_to_token_ratio=0.172192 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78410 payload_rows_remote=65650 dup_rows=19894 dup_rows_remote=16972 dedup_ratio=0.202372 router_tokens_per_expert=[456, 263, 262, 103, 0, 30, 443, 369, 0, 8, 86, 3452, 0, 1267, 6900, 21, 327, 4, 382, 644, 1634, 5823, 436, 19, 12, 6407, 247, 620, 24, 29, 501, 12, 0, 0, 4, 1244, 9, 3242, 738, 2, 0, 273, 65, 857, 1533, 1737, 131, 1193, 3078, 6, 0, 426, 0, 15, 1485, 315, 92, 53, 135, 432, 120, 0, 8, 1178] payload_rows_per_dst=[3394, 14180, 6230, 8106, 8740, 12764, 12236, 12760] dup_rows_per_dst=[458, 2384, 662, 1092, 1738, 5320, 5318, 2922] combine_rows_per_src=[3394, 14180, 6230, 8106, 8740, 12764, 12236, 12760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=272998400 dispatch_meta_bytes=321360 combine_payload_bytes=272998400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273319760 ingress_bytes=307601408 link_bytes=307601408 token_bytes=546318160 weight_movement_bytes=34603008 total_nvlink_bytes=580921168 weight_to_token_ratio=0.063339 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157479334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84806 payload_rows_remote=74686 dup_rows=13498 dup_rows_remote=11008 dedup_ratio=0.137309 router_tokens_per_expert=[0, 122, 9, 112, 147, 1693, 9, 0, 0, 5500, 0, 611, 3, 469, 19, 424, 0, 1115, 2597, 1, 1672, 584, 0, 7, 5, 6192, 2, 104, 6938, 176, 0, 0, 864, 0, 46, 0, 313, 60, 11, 6, 29, 583, 631, 3533, 0, 1, 1519, 9, 0, 0, 333, 576, 0, 0, 623, 2280, 4502, 3067, 47, 0, 0, 1, 1607, 0] payload_rows_per_dst=[11368, 11832, 6000, 12622, 14658, 10120, 6796, 11410] dup_rows_per_dst=[1820, 2220, 758, 336, 1818, 2490, 828, 3228] combine_rows_per_src=[11368, 11832, 6000, 12622, 14658, 10120, 6796, 11410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83868 payload_rows_remote=73640 dup_rows=14436 dup_rows_remote=13998 dedup_ratio=0.146851 router_tokens_per_expert=[0, 127, 770, 514, 339, 1424, 14, 1, 0, 5889, 0, 857, 27, 432, 3, 910, 0, 1328, 2743, 3, 2349, 1263, 0, 0, 0, 4762, 0, 20, 6551, 550, 1, 0, 613, 0, 38, 0, 102, 15, 67, 75, 129, 481, 708, 2516, 0, 2, 803, 27, 0, 0, 232, 200, 0, 0, 1430, 2674, 5278, 1852, 55, 1, 0, 0, 977, 0] payload_rows_per_dst=[13142, 13002, 8502, 10228, 13822, 7814, 7862, 9496] dup_rows_per_dst=[3792, 3234, 1384, 438, 1100, 1518, 1210, 1760] combine_rows_per_src=[13142, 13002, 8502, 10228, 13822, 7814, 7862, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81358 payload_rows_remote=69220 dup_rows=16946 dup_rows_remote=13914 dedup_ratio=0.172384 router_tokens_per_expert=[0, 0, 2, 2770, 402, 7, 3, 11, 242, 1008, 3, 0, 1, 62, 1892, 1265, 31, 2, 94, 447, 3282, 3372, 5, 0, 363, 682, 184, 1090, 3122, 2065, 0, 1666, 180, 0, 95, 1254, 154, 126, 1184, 39, 0, 15, 109, 5, 0, 5, 11, 1, 466, 3, 5668, 4307, 754, 0, 320, 1236, 1095, 1889, 4804, 3, 2, 3, 628, 723] payload_rows_per_dst=[6206, 7790, 11110, 9818, 10030, 11448, 12138, 12818] dup_rows_per_dst=[184, 1156, 2358, 2282, 2278, 180, 3032, 5476] combine_rows_per_src=[6206, 7790, 11110, 9818, 10030, 11448, 12138, 12818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=268902400 dispatch_meta_bytes=330488 combine_payload_bytes=268902400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269232888 ingress_bytes=303505408 link_bytes=303505408 token_bytes=538135288 weight_movement_bytes=34603008 total_nvlink_bytes=572738296 weight_to_token_ratio=0.064302 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142066416e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79586 payload_rows_remote=67540 dup_rows=18718 dup_rows_remote=15758 dedup_ratio=0.190409 router_tokens_per_expert=[0, 14, 0, 2435, 391, 3, 8, 27, 311, 640, 1, 0, 5, 87, 2181, 872, 145, 18, 66, 214, 3921, 2445, 0, 0, 13, 1405, 88, 837, 3716, 3708, 0, 1247, 136, 1, 70, 504, 351, 316, 1036, 89, 7, 0, 40, 7, 0, 18, 100, 0, 373, 2, 5745, 3580, 708, 0, 443, 1418, 1187, 2011, 4625, 1, 0, 5, 367, 1214] payload_rows_per_dst=[12046, 11516, 11562, 6426, 10696, 13578, 5574, 8188] dup_rows_per_dst=[2960, 4094, 2056, 754, 1742, 5416, 314, 1382] combine_rows_per_src=[12046, 11516, 11562, 6426, 10696, 13578, 5574, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=305913856 dispatch_meta_bytes=342776 combine_payload_bytes=305913856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306256632 ingress_bytes=375119872 link_bytes=375119872 token_bytes=612170488 weight_movement_bytes=69206016 total_nvlink_bytes=681376504 weight_to_token_ratio=0.113050 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411545879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81384 payload_rows_remote=68234 dup_rows=16920 dup_rows_remote=15598 dedup_ratio=0.172119 router_tokens_per_expert=[0, 104, 289, 327, 119, 807, 1, 0, 1, 3463, 4, 332, 51, 444, 23, 121, 0, 2308, 2943, 6, 2200, 1006, 1, 10, 0, 4850, 10, 1087, 6053, 287, 0, 0, 459, 37, 113, 0, 216, 31, 246, 81, 214, 1651, 980, 4277, 2, 0, 674, 30, 0, 0, 397, 1161, 0, 0, 445, 2944, 2733, 3131, 30, 0, 1, 1, 2448, 3] payload_rows_per_dst=[7890, 7756, 9128, 11154, 13150, 11800, 8394, 12112] dup_rows_per_dst=[870, 1122, 1934, 1314, 1322, 3856, 1500, 5002] combine_rows_per_src=[7890, 7756, 9128, 11154, 13150, 11800, 8394, 12112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=301629440 dispatch_meta_bytes=350552 combine_payload_bytes=301629440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=301979992 ingress_bytes=405438464 link_bytes=405438464 token_bytes=603609432 weight_movement_bytes=103809024 total_nvlink_bytes=707418456 weight_to_token_ratio=0.171980 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525632300e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283525120 dispatch_meta_bytes=332536 combine_payload_bytes=283525120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=283857656 ingress_bytes=387334144 link_bytes=387334144 token_bytes=567382776 weight_movement_bytes=103809024 total_nvlink_bytes=671191800 weight_to_token_ratio=0.182961 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457507201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80110 payload_rows_remote=68868 dup_rows=18194 dup_rows_remote=15014 dedup_ratio=0.185079 router_tokens_per_expert=[2, 186, 193, 204, 59, 1091, 16, 1, 39, 5297, 0, 512, 85, 974, 6, 565, 3, 1140, 2680, 20, 4122, 1623, 9, 12, 0, 4222, 11, 140, 5313, 461, 0, 2, 304, 15, 8, 32, 493, 87, 51, 76, 307, 528, 1851, 5479, 1, 6, 2091, 12, 1, 3, 495, 344, 3, 0, 405, 1703, 1828, 808, 46, 2, 0, 7, 3116, 62] payload_rows_per_dst=[6284, 11976, 11242, 9246, 11674, 13456, 5470, 10762] dup_rows_per_dst=[876, 2980, 3180, 426, 1084, 7094, 438, 2116] combine_rows_per_src=[6284, 11976, 11242, 9246, 11674, 13456, 5470, 10762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=276643840 dispatch_meta_bytes=333192 combine_payload_bytes=276643840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276977032 ingress_bytes=311246848 link_bytes=311246848 token_bytes=553620872 weight_movement_bytes=34603008 total_nvlink_bytes=588223880 weight_to_token_ratio=0.062503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171196832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78004 payload_rows_remote=67274 dup_rows=20300 dup_rows_remote=17940 dedup_ratio=0.206502 router_tokens_per_expert=[0, 249, 89, 236, 237, 1848, 5, 0, 0, 4679, 0, 504, 20, 693, 8, 641, 0, 1090, 2303, 1, 1384, 596, 0, 35, 0, 6666, 1, 77, 7084, 170, 0, 0, 552, 2, 49, 0, 128, 60, 28, 22, 51, 227, 766, 3580, 0, 0, 1284, 16, 2, 1, 229, 438, 0, 0, 892, 4621, 4478, 2213, 77, 0, 0, 5, 808, 7] payload_rows_per_dst=[11872, 10730, 11684, 15186, 2280, 10052, 10600, 5600] dup_rows_per_dst=[2412, 2360, 4502, 6678, 166, 1796, 1766, 620] combine_rows_per_src=[11872, 10730, 11684, 15186, 2280, 10052, 10600, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=279486464 dispatch_meta_bytes=335328 combine_payload_bytes=279486464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279821792 ingress_bytes=314089472 link_bytes=314089472 token_bytes=559308256 weight_movement_bytes=34603008 total_nvlink_bytes=593911264 weight_to_token_ratio=0.061868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181893397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80324 payload_rows_remote=66948 dup_rows=17980 dup_rows_remote=12504 dedup_ratio=0.182902 router_tokens_per_expert=[0, 26, 0, 2063, 208, 48, 7, 17, 70, 743, 1, 5, 0, 49, 3268, 1870, 39, 0, 531, 518, 2689, 3262, 1, 2, 276, 660, 660, 668, 2243, 3634, 29, 2114, 161, 1, 69, 581, 207, 230, 1319, 28, 0, 2, 98, 32, 0, 0, 48, 0, 846, 2, 5412, 3050, 405, 0, 77, 2324, 580, 1894, 5030, 11, 4, 3, 685, 352] payload_rows_per_dst=[4620, 10172, 7932, 11294, 9050, 10980, 12900, 13376] dup_rows_per_dst=[118, 1840, 774, 3310, 2106, 204, 4152, 5476] combine_rows_per_src=[4620, 10172, 7932, 11294, 9050, 10980, 12900, 13376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282083328 dispatch_meta_bytes=335528 combine_payload_bytes=282083328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=282418856 ingress_bytes=351289344 link_bytes=351289344 token_bytes=564502184 weight_movement_bytes=69206016 total_nvlink_bytes=633708200 weight_to_token_ratio=0.122597 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321873520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81274 payload_rows_remote=65038 dup_rows=17030 dup_rows_remote=11770 dedup_ratio=0.173238 router_tokens_per_expert=[2275, 211, 0, 6019, 0, 4, 190, 263, 0, 55, 178, 4, 0, 0, 94, 2220, 0, 1, 6, 0, 0, 22, 175, 4665, 4, 0, 2, 0, 2708, 0, 0, 0, 21, 1588, 411, 49, 5881, 0, 1765, 4643, 0, 0, 4, 913, 19, 0, 8047, 0, 46, 113, 0, 0, 0, 24, 0, 0, 251, 0, 0, 3245, 7, 1318, 38, 1673] payload_rows_per_dst=[6068, 9556, 14414, 5426, 6444, 16236, 11996, 11134] dup_rows_per_dst=[464, 1450, 6716, 2, 1076, 5260, 132, 1930] combine_rows_per_src=[6068, 9556, 14414, 5426, 6444, 16236, 11996, 11134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=275554304 dispatch_meta_bytes=340856 combine_payload_bytes=275554304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275895160 ingress_bytes=344760320 link_bytes=344760320 token_bytes=551449464 weight_movement_bytes=69206016 total_nvlink_bytes=620655480 weight_to_token_ratio=0.125498 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297305329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84624 payload_rows_remote=74838 dup_rows=13680 dup_rows_remote=11204 dedup_ratio=0.139160 router_tokens_per_expert=[1953, 577, 20, 5527, 0, 32, 218, 243, 10, 150, 62, 11, 0, 0, 16, 1361, 0, 3, 53, 0, 0, 204, 50, 4073, 3, 0, 1, 0, 2590, 0, 0, 0, 65, 1731, 1375, 60, 5037, 0, 2681, 5866, 0, 0, 11, 1091, 12, 0, 8081, 0, 60, 181, 1, 0, 0, 65, 18, 0, 502, 8, 0, 2545, 1, 1081, 38, 1485] payload_rows_per_dst=[13536, 13156, 8504, 9786, 9326, 10104, 10520, 9692] dup_rows_per_dst=[3604, 1796, 262, 2476, 2210, 1500, 204, 1628] combine_rows_per_src=[13536, 13156, 8504, 9786, 9326, 10104, 10520, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84818 payload_rows_remote=72366 dup_rows=13486 dup_rows_remote=8486 dedup_ratio=0.137187 router_tokens_per_expert=[3, 95, 302, 76, 319, 1511, 39, 0, 7, 6128, 0, 418, 14, 420, 1, 453, 0, 1059, 2161, 4, 2006, 795, 1, 11, 0, 6122, 13, 169, 7691, 394, 0, 0, 428, 3, 33, 0, 135, 17, 73, 117, 286, 109, 631, 3320, 1, 0, 342, 8, 0, 0, 364, 606, 0, 5, 308, 3388, 4392, 3045, 86, 1, 1, 0, 1241, 0] payload_rows_per_dst=[11502, 6108, 6918, 12872, 15856, 8372, 12452, 10738] dup_rows_per_dst=[1972, 664, 834, 524, 1138, 1022, 5000, 2332] combine_rows_per_src=[11502, 6108, 6918, 12872, 15856, 8372, 12452, 10738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274219008 dispatch_meta_bytes=317808 combine_payload_bytes=274219008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274536816 ingress_bytes=343425024 link_bytes=343425024 token_bytes=548755824 weight_movement_bytes=69206016 total_nvlink_bytes=617961840 weight_to_token_ratio=0.126114 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292280717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74808 payload_rows_remote=65524 dup_rows=23496 dup_rows_remote=21920 dedup_ratio=0.239014 router_tokens_per_expert=[3, 68, 376, 250, 35, 2603, 10, 0, 0, 5141, 0, 909, 10, 1640, 3, 200, 0, 2449, 2129, 0, 4104, 292, 0, 2, 0, 3393, 12, 83, 4136, 368, 1, 0, 378, 1, 283, 0, 255, 337, 318, 23, 320, 735, 560, 3704, 2, 0, 372, 22, 3, 0, 744, 430, 8, 0, 597, 4956, 2085, 3396, 21, 0, 0, 7, 1378, 0] payload_rows_per_dst=[9284, 14252, 14716, 8910, 2758, 5306, 11240, 8342] dup_rows_per_dst=[1576, 6942, 10022, 290, 432, 736, 2236, 1262] combine_rows_per_src=[9284, 14252, 14716, 8910, 2758, 5306, 11240, 8342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85024 payload_rows_remote=78994 dup_rows=13280 dup_rows_remote=13120 dedup_ratio=0.135091 router_tokens_per_expert=[910, 142, 31, 5819, 46, 47, 536, 191, 39, 239, 19, 7, 1, 0, 12, 1526, 0, 2, 9, 0, 14, 247, 4, 3783, 1, 7, 7, 3, 2688, 0, 0, 6, 2, 2396, 685, 11, 5642, 1, 2398, 5826, 0, 0, 0, 316, 33, 0, 7880, 3, 16, 425, 0, 0, 0, 125, 0, 0, 326, 0, 0, 2436, 14, 1531, 66, 2684] payload_rows_per_dst=[12932, 13434, 7924, 5424, 6030, 16010, 11918, 11352] dup_rows_per_dst=[2512, 1904, 194, 0, 160, 5250, 498, 2762] combine_rows_per_src=[12932, 13434, 7924, 5424, 6030, 16010, 11918, 11352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=307232 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266702880 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533098528 weight_movement_bytes=69206016 total_nvlink_bytes=602304544 weight_to_token_ratio=0.129818 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306536448 dispatch_meta_bytes=344168 combine_payload_bytes=306536448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306880616 ingress_bytes=341139456 link_bytes=341139456 token_bytes=613417064 weight_movement_bytes=34603008 total_nvlink_bytes=648020072 weight_to_token_ratio=0.056410 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283680309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=296411136 dispatch_meta_bytes=323408 combine_payload_bytes=296411136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296734544 ingress_bytes=331014144 link_bytes=331014144 token_bytes=593145680 weight_movement_bytes=34603008 total_nvlink_bytes=627748688 weight_to_token_ratio=0.058338 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245579575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80584 payload_rows_remote=74894 dup_rows=17720 dup_rows_remote=17674 dedup_ratio=0.180257 router_tokens_per_expert=[1048, 265, 7, 5578, 54, 18, 841, 346, 3, 502, 23, 86, 31, 0, 16, 1243, 0, 15, 46, 1, 29, 20, 33, 2724, 2, 3, 286, 0, 3572, 1, 4, 8, 25, 1885, 241, 25, 6590, 4, 3707, 5989, 1, 7, 0, 202, 108, 61, 7929, 0, 11, 953, 0, 1, 0, 220, 0, 20, 430, 7, 8, 1316, 5, 1621, 82, 899] payload_rows_per_dst=[13006, 13706, 5690, 16158, 9510, 728, 14218, 7568] dup_rows_per_dst=[3308, 2080, 46, 7452, 2264, 30, 1372, 1168] combine_rows_per_src=[13006, 13706, 5690, 16158, 9510, 728, 14218, 7568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268386304 dispatch_meta_bytes=349776 combine_payload_bytes=268386304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268736080 ingress_bytes=302989312 link_bytes=302989312 token_bytes=537122384 weight_movement_bytes=34603008 total_nvlink_bytes=571725392 weight_to_token_ratio=0.064423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140124388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84504 payload_rows_remote=72504 dup_rows=13800 dup_rows_remote=12448 dedup_ratio=0.140381 router_tokens_per_expert=[1146, 59, 1, 6659, 4, 5, 242, 580, 8, 244, 85, 2, 0, 0, 22, 1321, 0, 0, 24, 0, 0, 13, 18, 4913, 4, 0, 0, 7, 3106, 0, 0, 0, 43, 894, 640, 72, 7068, 1, 2033, 4994, 0, 0, 2, 349, 5, 0, 8060, 1, 40, 369, 0, 1, 0, 11, 40, 1, 409, 0, 0, 2912, 2, 1484, 33, 1225] payload_rows_per_dst=[14162, 12000, 9886, 16240, 6672, 714, 14358, 10472] dup_rows_per_dst=[3230, 1352, 50, 6114, 694, 0, 702, 1658] combine_rows_per_src=[14162, 12000, 9886, 16240, 6672, 714, 14358, 10472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=323559424 dispatch_meta_bytes=368456 combine_payload_bytes=323559424 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=323927880 ingress_bytes=461971456 link_bytes=461971456 token_bytes=647487304 weight_movement_bytes=138412032 total_nvlink_bytes=785899336 weight_to_token_ratio=0.213768 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.738361397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306765824 dispatch_meta_bytes=370272 combine_payload_bytes=306765824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307136096 ingress_bytes=341368832 link_bytes=341368832 token_bytes=613901920 weight_movement_bytes=34603008 total_nvlink_bytes=648504928 weight_to_token_ratio=0.056366 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284543432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78006 payload_rows_remote=67428 dup_rows=20298 dup_rows_remote=17352 dedup_ratio=0.206482 router_tokens_per_expert=[0, 293, 201, 104, 144, 2068, 6, 0, 13, 5440, 3, 626, 29, 509, 49, 334, 0, 1806, 2112, 9, 1399, 436, 4, 96, 4, 3995, 12, 375, 6407, 272, 0, 0, 626, 1, 281, 0, 695, 276, 127, 110, 307, 1200, 758, 3142, 0, 17, 341, 53, 0, 4, 387, 367, 1, 0, 418, 4639, 4051, 3164, 305, 5, 0, 26, 1104, 1] payload_rows_per_dst=[11468, 2990, 6492, 7806, 14220, 9810, 14642, 10578] dup_rows_per_dst=[2176, 136, 1008, 274, 2826, 3062, 7870, 2946] combine_rows_per_src=[11468, 2990, 6492, 7806, 14220, 9810, 14642, 10578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296976384 dispatch_meta_bytes=339808 combine_payload_bytes=296976384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297316192 ingress_bytes=331579392 link_bytes=331579392 token_bytes=594292576 weight_movement_bytes=34603008 total_nvlink_bytes=628895584 weight_to_token_ratio=0.058226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247706558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86172 payload_rows_remote=76320 dup_rows=12132 dup_rows_remote=10388 dedup_ratio=0.123413 router_tokens_per_expert=[0, 2, 163, 1219, 5, 9, 6344, 1, 106, 218, 0, 0, 3494, 0, 1502, 1, 5, 7366, 1160, 0, 0, 1, 1, 55, 68, 2, 0, 0, 0, 18, 153, 6, 952, 750, 0, 4, 0, 42, 1001, 0, 26, 20, 4159, 0, 0, 2394, 3087, 0, 15, 7140, 0, 1594, 557, 0, 4546, 17, 0, 2, 934, 0, 11, 0, 0, 2] payload_rows_per_dst=[8730, 9660, 15130, 14510, 8676, 9852, 11130, 8484] dup_rows_per_dst=[1200, 982, 2046, 264, 2378, 1744, 2328, 1190] combine_rows_per_src=[8730, 9660, 15130, 14510, 8676, 9852, 11130, 8484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78740 payload_rows_remote=67336 dup_rows=19564 dup_rows_remote=19188 dedup_ratio=0.199015 router_tokens_per_expert=[1423, 44, 7, 5928, 0, 7, 328, 543, 3, 256, 64, 1, 5, 0, 60, 2225, 1, 5, 58, 0, 0, 87, 37, 5421, 3, 0, 0, 2, 1612, 2, 1, 0, 45, 2118, 1083, 31, 5385, 0, 1513, 4568, 3, 0, 4, 839, 0, 0, 7911, 36, 10, 463, 0, 10, 0, 14, 8, 0, 336, 4, 0, 2968, 11, 1642, 79, 1948] payload_rows_per_dst=[4528, 4964, 14828, 3236, 12354, 15956, 11404, 11470] dup_rows_per_dst=[176, 264, 8246, 4, 3336, 4656, 376, 2506] combine_rows_per_src=[4528, 4964, 14828, 3236, 12354, 15956, 11404, 11470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=276185088 dispatch_meta_bytes=339120 combine_payload_bytes=276185088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276524208 ingress_bytes=345391104 link_bytes=345391104 token_bytes=552709296 weight_movement_bytes=69206016 total_nvlink_bytes=621915312 weight_to_token_ratio=0.125212 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299678918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89464 payload_rows_remote=76118 dup_rows=8840 dup_rows_remote=8650 dedup_ratio=0.089925 router_tokens_per_expert=[1, 1, 115, 833, 3, 6, 6293, 5, 113, 191, 0, 0, 2943, 0, 1502, 2, 0, 7324, 992, 0, 0, 4, 1, 251, 178, 0, 0, 2, 0, 11, 6, 8, 1662, 1549, 3, 0, 0, 22, 2323, 0, 59, 6, 3904, 0, 0, 1129, 3988, 3, 80, 6563, 0, 748, 34, 1, 5207, 17, 0, 4, 1053, 1, 4, 0, 0, 7] payload_rows_per_dst=[12988, 8684, 15080, 13346, 9286, 9596, 11224, 9260] dup_rows_per_dst=[1526, 818, 2064, 190, 1832, 606, 950, 854] combine_rows_per_src=[12988, 8684, 15080, 13346, 9286, 9596, 11224, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79812 payload_rows_remote=66086 dup_rows=18492 dup_rows_remote=14826 dedup_ratio=0.188110 router_tokens_per_expert=[2022, 56, 1, 6139, 1, 2, 283, 192, 0, 335, 185, 28, 8, 0, 48, 1166, 0, 2, 14, 13, 0, 62, 30, 3580, 0, 0, 1, 0, 3185, 0, 0, 0, 11, 1646, 231, 54, 5676, 0, 4553, 5679, 0, 0, 0, 857, 4, 2, 7614, 0, 89, 182, 0, 18, 0, 8, 4, 3, 463, 3, 16, 2432, 0, 1402, 71, 781] payload_rows_per_dst=[13726, 13048, 7338, 15488, 12356, 1724, 7116, 9016] dup_rows_per_dst=[3666, 1850, 64, 6112, 5334, 2, 144, 1320] combine_rows_per_src=[13726, 13048, 7338, 15488, 12356, 1724, 7116, 9016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312606720 dispatch_meta_bytes=346832 combine_payload_bytes=312606720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312953552 ingress_bytes=416415744 link_bytes=416415744 token_bytes=625560272 weight_movement_bytes=103809024 total_nvlink_bytes=729369296 weight_to_token_ratio=0.165946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.566938920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88458 payload_rows_remote=80872 dup_rows=9846 dup_rows_remote=8628 dedup_ratio=0.100159 router_tokens_per_expert=[0, 38, 22, 1998, 0, 14, 4628, 6, 41, 103, 3, 0, 2096, 0, 1064, 4, 10, 6920, 495, 0, 0, 0, 13, 53, 34, 8, 0, 0, 0, 22, 75, 18, 2120, 735, 13, 13, 2, 13, 1503, 3, 28, 56, 5304, 0, 0, 2189, 5004, 2, 27, 6253, 0, 1071, 10, 0, 6072, 44, 0, 1, 1015, 0, 8, 1, 0, 0] payload_rows_per_dst=[11408, 6284, 14260, 12578, 7586, 12586, 12726, 11030] dup_rows_per_dst=[2004, 338, 722, 242, 1218, 2572, 1722, 1028] combine_rows_per_src=[11408, 6284, 14260, 12578, 7586, 12586, 12726, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=275808256 dispatch_meta_bytes=346096 combine_payload_bytes=275808256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276154352 ingress_bytes=310411264 link_bytes=310411264 token_bytes=551962608 weight_movement_bytes=34603008 total_nvlink_bytes=586565616 weight_to_token_ratio=0.062691 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168052596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86674 payload_rows_remote=73044 dup_rows=11630 dup_rows_remote=9804 dedup_ratio=0.118306 router_tokens_per_expert=[67, 91, 55, 2300, 8, 30, 6448, 9, 631, 472, 23, 0, 1219, 1, 888, 166, 0, 6617, 985, 6, 0, 9, 34, 285, 104, 56, 0, 30, 0, 16, 86, 29, 1587, 846, 0, 3, 41, 31, 1856, 0, 71, 70, 5201, 0, 5, 748, 4547, 8, 76, 6787, 0, 1395, 299, 18, 2974, 170, 7, 5, 1642, 0, 5, 68, 2, 25] payload_rows_per_dst=[14398, 6498, 13630, 13690, 7454, 11542, 8386, 11076] dup_rows_per_dst=[3618, 718, 1826, 526, 1274, 1084, 1058, 1526] combine_rows_per_src=[14398, 6498, 13630, 13690, 7454, 11542, 8386, 11076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=339072 combine_payload_bytes=311779328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312118400 ingress_bytes=346382336 link_bytes=346382336 token_bytes=623897728 weight_movement_bytes=34603008 total_nvlink_bytes=658500736 weight_to_token_ratio=0.055463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303408844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270688256 dispatch_meta_bytes=323648 combine_payload_bytes=270688256 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=271011904 ingress_bytes=374497280 link_bytes=374497280 token_bytes=541700160 weight_movement_bytes=103809024 total_nvlink_bytes=645509184 weight_to_token_ratio=0.191636 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409203115e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78734 payload_rows_remote=62728 dup_rows=19570 dup_rows_remote=7976 dedup_ratio=0.199076 router_tokens_per_expert=[0, 57, 48, 1040, 7, 39, 7445, 3, 347, 704, 3, 0, 3803, 0, 1355, 2, 4, 7586, 777, 0, 0, 0, 1, 175, 32, 0, 0, 0, 0, 8, 11, 1, 1305, 751, 5, 22, 0, 46, 1103, 0, 14, 10, 2680, 0, 1, 1002, 4489, 0, 13, 6550, 1, 1240, 9, 1, 5363, 6, 0, 12, 1078, 0, 0, 1, 0, 2] payload_rows_per_dst=[15220, 16006, 1904, 13112, 5802, 13928, 2536, 10226] dup_rows_per_dst=[2058, 11594, 10, 92, 662, 4212, 4, 938] combine_rows_per_src=[15220, 16006, 1904, 13112, 5802, 13928, 2536, 10226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=331251712 dispatch_meta_bytes=358000 combine_payload_bytes=331251712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331609712 ingress_bytes=365854720 link_bytes=365854720 token_bytes=662861424 weight_movement_bytes=34603008 total_nvlink_bytes=697464432 weight_to_token_ratio=0.052202 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376681858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=299188224 dispatch_meta_bytes=331392 combine_payload_bytes=299188224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=299519616 ingress_bytes=402997248 link_bytes=402997248 token_bytes=598707840 weight_movement_bytes=103809024 total_nvlink_bytes=702516864 weight_to_token_ratio=0.173388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516446200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79792 payload_rows_remote=68126 dup_rows=18512 dup_rows_remote=15550 dedup_ratio=0.188314 router_tokens_per_expert=[717, 71, 6, 4979, 1, 15, 447, 436, 3, 360, 74, 47, 2, 0, 65, 3321, 0, 1, 48, 2, 0, 8, 8, 5097, 12, 0, 17, 0, 2160, 3, 1, 0, 55, 1694, 109, 775, 6222, 8, 1902, 4671, 0, 0, 0, 562, 5, 0, 7593, 5, 25, 325, 32, 2, 0, 47, 20, 0, 408, 4, 1, 3161, 12, 2700, 100, 813] payload_rows_per_dst=[3230, 7310, 11546, 4384, 13122, 15624, 12910, 11666] dup_rows_per_dst=[156, 434, 2596, 2, 7416, 4510, 436, 2962] combine_rows_per_src=[3230, 7310, 11546, 4384, 13122, 15624, 12910, 11666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256933888 dispatch_meta_bytes=282816 combine_payload_bytes=256933888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257216704 ingress_bytes=291536896 link_bytes=291536896 token_bytes=514150592 weight_movement_bytes=34603008 total_nvlink_bytes=548753600 weight_to_token_ratio=0.067301 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097029869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85138 payload_rows_remote=70864 dup_rows=13166 dup_rows_remote=9472 dedup_ratio=0.133931 router_tokens_per_expert=[34, 0, 7341, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 1241, 59, 2872, 1150, 172, 1, 285, 2413, 0, 78, 20, 0, 0, 0, 520, 1, 2525, 36, 0, 0, 0, 0, 881, 10, 2, 2647, 8, 0, 0, 2146, 0, 0, 6830, 0, 2112, 0, 578, 3950, 177, 1419, 270, 0, 0, 426, 41, 6994, 6, 3, 0, 0] payload_rows_per_dst=[14682, 8720, 8204, 5490, 11816, 14274, 7746, 14206] dup_rows_per_dst=[78, 1606, 1810, 714, 3164, 3694, 1366, 734] combine_rows_per_src=[14682, 8720, 8204, 5490, 11816, 14274, 7746, 14206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83462 payload_rows_remote=69678 dup_rows=14842 dup_rows_remote=11302 dedup_ratio=0.150981 router_tokens_per_expert=[0, 0, 134, 923, 12, 13, 7500, 5, 77, 99, 0, 0, 1756, 0, 1471, 3, 32, 7359, 859, 0, 0, 0, 0, 128, 29, 7, 0, 0, 0, 43, 148, 124, 1280, 529, 0, 1, 1, 69, 873, 0, 39, 5, 2446, 0, 0, 2397, 3523, 0, 21, 7055, 0, 2360, 13, 0, 6255, 13, 1, 2, 1534, 2, 6, 3, 2, 0] payload_rows_per_dst=[2130, 6426, 15074, 14224, 15716, 4852, 13784, 11256] dup_rows_per_dst=[44, 386, 1682, 588, 4790, 30, 3540, 3782] combine_rows_per_src=[2130, 6426, 15074, 14224, 15716, 4852, 13784, 11256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=279044096 dispatch_meta_bytes=334704 combine_payload_bytes=279044096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279378800 ingress_bytes=313647104 link_bytes=313647104 token_bytes=558422896 weight_movement_bytes=34603008 total_nvlink_bytes=593025904 weight_to_token_ratio=0.061966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180228802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85422 payload_rows_remote=70836 dup_rows=12882 dup_rows_remote=10578 dedup_ratio=0.131042 router_tokens_per_expert=[104, 0, 7191, 0, 2, 0, 0, 2, 0, 2, 0, 0, 2, 2590, 1237, 13, 2700, 1370, 87, 8, 223, 2238, 0, 293, 38, 0, 0, 2, 28, 26, 1153, 139, 2, 0, 0, 0, 2100, 10, 2, 943, 6, 27, 0, 642, 0, 1, 7059, 3, 2045, 12, 1925, 4821, 930, 1519, 235, 0, 6, 326, 44, 7034, 6, 3, 0, 3] payload_rows_per_dst=[14396, 10690, 7200, 14586, 12300, 1356, 10634, 14260] dup_rows_per_dst=[202, 2398, 1238, 2304, 3456, 2, 2698, 584] combine_rows_per_src=[14396, 10690, 7200, 14586, 12300, 1356, 10634, 14260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77600 payload_rows_remote=63128 dup_rows=20704 dup_rows_remote=16602 dedup_ratio=0.210612 router_tokens_per_expert=[2, 0, 133, 1454, 2, 64, 6582, 1, 63, 89, 9, 0, 4431, 0, 762, 9, 1, 6148, 652, 0, 1, 1, 13, 155, 162, 6, 0, 1, 2, 24, 4, 13, 1579, 575, 0, 12, 0, 0, 1966, 0, 46, 108, 3385, 0, 0, 1254, 4991, 0, 15, 5359, 0, 1081, 4, 5, 6754, 7, 0, 0, 1185, 0, 20, 8, 0, 14] payload_rows_per_dst=[14472, 14222, 1634, 10920, 7296, 15586, 2218, 11252] dup_rows_per_dst=[4102, 6702, 12, 222, 968, 7508, 6, 1184] combine_rows_per_src=[14472, 14222, 1634, 10920, 7296, 15586, 2218, 11252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290258944 dispatch_meta_bytes=321344 combine_payload_bytes=290258944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290580288 ingress_bytes=359464960 link_bytes=359464960 token_bytes=580839232 weight_movement_bytes=69206016 total_nvlink_bytes=650045248 weight_to_token_ratio=0.119148 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352637705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85864 payload_rows_remote=75390 dup_rows=12440 dup_rows_remote=10998 dedup_ratio=0.126546 router_tokens_per_expert=[82, 0, 6997, 0, 2, 0, 0, 13, 0, 8, 72, 0, 5, 2939, 2571, 268, 1794, 746, 619, 64, 294, 3356, 0, 754, 444, 0, 1, 18, 290, 1, 2190, 119, 0, 0, 0, 21, 1137, 28, 2, 564, 152, 13, 2, 732, 68, 0, 7040, 0, 1741, 0, 812, 4206, 344, 1044, 734, 2, 3, 38, 120, 6608, 67, 24, 3, 0] payload_rows_per_dst=[14028, 11594, 9374, 4942, 10474, 14228, 7808, 13416] dup_rows_per_dst=[160, 3720, 2292, 1276, 1442, 1694, 1546, 310] combine_rows_per_src=[14028, 11594, 9374, 4942, 10474, 14228, 7808, 13416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285401088 dispatch_meta_bytes=323920 combine_payload_bytes=285401088 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285725008 ingress_bytes=389210112 link_bytes=389210112 token_bytes=571126096 weight_movement_bytes=103809024 total_nvlink_bytes=674935120 weight_to_token_ratio=0.181762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.464566317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81256 payload_rows_remote=73504 dup_rows=17048 dup_rows_remote=15704 dedup_ratio=0.173421 router_tokens_per_expert=[18, 3, 7283, 0, 62, 0, 0, 30, 0, 1, 0, 0, 12, 2247, 1165, 176, 2746, 1525, 140, 6, 321, 2408, 0, 148, 365, 3, 0, 14, 390, 18, 2930, 14, 4, 0, 20, 39, 870, 36, 9, 456, 14, 1, 14, 927, 3, 0, 6931, 0, 1116, 11, 2045, 2128, 2198, 3282, 535, 49, 13, 374, 292, 5742, 0, 3, 3, 12] payload_rows_per_dst=[14624, 10056, 7752, 15016, 6616, 1906, 13134, 12152] dup_rows_per_dst=[168, 2638, 1344, 6314, 508, 12, 5338, 726] combine_rows_per_src=[14624, 10056, 7752, 15016, 6616, 1906, 13134, 12152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290144256 dispatch_meta_bytes=325656 combine_payload_bytes=290144256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290469912 ingress_bytes=324747264 link_bytes=324747264 token_bytes=580614168 weight_movement_bytes=34603008 total_nvlink_bytes=615217176 weight_to_token_ratio=0.059597 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221997810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81646 payload_rows_remote=69988 dup_rows=16658 dup_rows_remote=13116 dedup_ratio=0.169454 router_tokens_per_expert=[5, 1, 6404, 0, 11, 0, 0, 1, 0, 29, 1, 0, 4, 2267, 965, 128, 4206, 655, 316, 9, 268, 3847, 0, 502, 198, 0, 2, 7, 651, 4, 2464, 32, 0, 2, 23, 0, 784, 11, 4, 2930, 114, 2, 0, 1207, 0, 0, 5960, 1, 1220, 3, 878, 3624, 273, 1082, 1067, 5, 0, 170, 58, 6739, 11, 3, 3, 1] payload_rows_per_dst=[12812, 11658, 9424, 13740, 11402, 10160, 7618, 4832] dup_rows_per_dst=[32, 3542, 1770, 4896, 3354, 1514, 1438, 112] combine_rows_per_src=[12812, 11658, 9424, 13740, 11402, 10160, 7618, 4832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=258572288 dispatch_meta_bytes=318920 combine_payload_bytes=258572288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258891208 ingress_bytes=327778304 link_bytes=327778304 token_bytes=517463496 weight_movement_bytes=69206016 total_nvlink_bytes=586669512 weight_to_token_ratio=0.133741 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12844524440 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=6426141592 ingress_bytes=8044724224 link_bytes=8044724224 total_nvlink_bytes=14470865816 weight_to_token_ratio=0.126617 prefetch_share_of_ingress=0.103232 dedup_ratio=0.194857 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.316158678e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=308797440 dispatch_meta_bytes=345552 combine_payload_bytes=308797440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309142992 ingress_bytes=343400448 link_bytes=343400448 token_bytes=617940432 weight_movement_bytes=34603008 total_nvlink_bytes=652543440 weight_to_token_ratio=0.055997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292188240e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72734 payload_rows_remote=61308 dup_rows=25570 dup_rows_remote=21778 dedup_ratio=0.260111 router_tokens_per_expert=[0, 82, 88, 1876, 12, 115, 7367, 30, 176, 267, 1, 14, 3370, 0, 622, 43, 75, 7600, 702, 3, 0, 1, 14, 163, 294, 1, 0, 0, 2, 62, 7, 22, 950, 2144, 9, 3, 1, 30, 1105, 74, 55, 28, 3074, 0, 0, 1219, 3277, 1, 48, 6653, 0, 1708, 61, 6, 4417, 22, 1, 5, 1216, 1, 8, 2, 0, 25] payload_rows_per_dst=[4278, 8454, 11954, 3608, 15498, 2590, 14926, 11426] dup_rows_per_dst=[128, 532, 1276, 192, 7868, 16, 11766, 3792] combine_rows_per_src=[4278, 8454, 11954, 3608, 15498, 2590, 14926, 11426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=301072384 dispatch_meta_bytes=356832 combine_payload_bytes=301072384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301429216 ingress_bytes=370278400 link_bytes=370278400 token_bytes=602501600 weight_movement_bytes=69206016 total_nvlink_bytes=671707616 weight_to_token_ratio=0.114864 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393327809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13053284856 prefetch_bytes=622854144 reduce_bytes=1176502272 weight_movement_bytes=1799356416 egress_bytes=6530552312 ingress_bytes=8322088960 link_bytes=8322088960 total_nvlink_bytes=14852641272 weight_to_token_ratio=0.137847 prefetch_share_of_ingress=0.074843 dedup_ratio=0.186507 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=34 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.361536990e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286670848 dispatch_meta_bytes=332416 combine_payload_bytes=286670848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287003264 ingress_bytes=321273856 link_bytes=321273856 token_bytes=573674112 weight_movement_bytes=34603008 total_nvlink_bytes=608277120 weight_to_token_ratio=0.060318 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208927655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13022898152 prefetch_bytes=865075200 reduce_bytes=449839104 weight_movement_bytes=1314914304 egress_bytes=6515361768 ingress_bytes=7822450688 link_bytes=7822450688 total_nvlink_bytes=14337812456 weight_to_token_ratio=0.100969 prefetch_share_of_ingress=0.110589 dedup_ratio=0.190454 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=13 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.279793573e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84094 payload_rows_remote=73682 dup_rows=14210 dup_rows_remote=11814 dedup_ratio=0.144552 router_tokens_per_expert=[40, 0, 6465, 0, 2, 0, 6, 0, 1, 5, 2, 0, 0, 2508, 818, 32, 3169, 743, 79, 1, 211, 2996, 0, 971, 51, 0, 0, 0, 118, 2, 1934, 52, 1, 0, 4, 9, 770, 25, 5, 1794, 388, 0, 0, 1535, 0, 0, 6528, 1, 1780, 0, 1877, 4548, 114, 1621, 816, 0, 0, 301, 65, 6726, 3, 34, 1, 0] payload_rows_per_dst=[12938, 6444, 11558, 4146, 11174, 13712, 10412, 13710] dup_rows_per_dst=[88, 288, 4782, 168, 2746, 3192, 2396, 550] combine_rows_per_src=[12938, 6444, 11558, 4146, 11174, 13712, 10412, 13710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12930891872 prefetch_bytes=657457152 reduce_bytes=899678208 weight_movement_bytes=1557135360 egress_bytes=6469369952 ingress_bytes=8018657280 link_bytes=8018657280 total_nvlink_bytes=14488027232 weight_to_token_ratio=0.120420 prefetch_share_of_ingress=0.081991 dedup_ratio=0.198638 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=26 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311893991e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=251117568 dispatch_meta_bytes=332344 combine_payload_bytes=251117568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=251449912 ingress_bytes=320323584 link_bytes=320323584 token_bytes=502567480 weight_movement_bytes=69206016 total_nvlink_bytes=571773496 weight_to_token_ratio=0.137705 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205351858e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81782 payload_rows_remote=69216 dup_rows=16522 dup_rows_remote=15096 dedup_ratio=0.168070 router_tokens_per_expert=[163, 0, 5955, 0, 4, 0, 0, 7, 0, 1, 1, 0, 76, 1252, 2594, 26, 3287, 582, 394, 0, 180, 1635, 2, 963, 11, 0, 1, 0, 880, 1, 2093, 44, 3, 0, 1, 23, 1292, 2, 0, 1041, 46, 1, 0, 708, 12, 0, 6594, 0, 4129, 1, 642, 5411, 152, 849, 786, 6, 2, 75, 165, 7044, 5, 6, 4, 0] payload_rows_per_dst=[12566, 11534, 6506, 14292, 12562, 14264, 9548, 510] dup_rows_per_dst=[1426, 2940, 1006, 4956, 2984, 1358, 1848, 4] combine_rows_per_src=[12566, 11534, 6506, 14292, 12562, 14264, 9548, 510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12516481512 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6262028776 ingress_bytes=7603970048 link_bytes=7603970048 total_nvlink_bytes=13865998824 weight_to_token_ratio=0.107819 prefetch_share_of_ingress=0.091013 dedup_ratio=0.206489 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244049005e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=301801472 dispatch_meta_bytes=341984 combine_payload_bytes=301801472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=302143456 ingress_bytes=371007488 link_bytes=371007488 token_bytes=603944928 weight_movement_bytes=69206016 total_nvlink_bytes=673150944 weight_to_token_ratio=0.114590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396071309e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=283508736 dispatch_meta_bytes=337248 combine_payload_bytes=283508736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=283845984 ingress_bytes=387317760 link_bytes=387317760 token_bytes=567354720 weight_movement_bytes=103809024 total_nvlink_bytes=671163744 weight_to_token_ratio=0.182970 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457445549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12541974440 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=6274815912 ingress_bytes=7928102912 link_bytes=7928102912 total_nvlink_bytes=14202918824 weight_to_token_ratio=0.132431 prefetch_share_of_ingress=0.096021 dedup_ratio=0.208457 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.297078826e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70596 payload_rows_remote=55878 dup_rows=27708 dup_rows_remote=20228 dedup_ratio=0.281860 router_tokens_per_expert=[1, 1, 6037, 16, 20, 0, 2, 0, 0, 0, 13, 0, 1, 2548, 2518, 158, 4828, 1064, 371, 9, 744, 2405, 1, 439, 0, 0, 1, 44, 971, 76, 1571, 69, 13, 1, 0, 4, 1095, 4, 4, 1032, 380, 15, 1, 2703, 4, 0, 5363, 3, 1001, 0, 728, 4366, 62, 855, 368, 7, 2, 343, 60, 6791, 2, 20, 3, 14] payload_rows_per_dst=[92, 9176, 13782, 4324, 4110, 12638, 11756, 14718] dup_rows_per_dst=[12, 1300, 10262, 1140, 196, 4300, 3018, 7480] combine_rows_per_src=[92, 9176, 13782, 4324, 4110, 12638, 11756, 14718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12193263776 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6100390048 ingress_bytes=7546200064 link_bytes=7546200064 total_nvlink_bytes=13646590112 weight_to_token_ratio=0.119191 prefetch_share_of_ingress=0.087124 dedup_ratio=0.218455 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234597536e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=228876288 dispatch_meta_bytes=304424 combine_payload_bytes=228876288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=229180712 ingress_bytes=298082304 link_bytes=298082304 token_bytes=458057000 weight_movement_bytes=69206016 total_nvlink_bytes=527263016 weight_to_token_ratio=0.151086 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121659712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11832577128 prefetch_bytes=865075200 reduce_bytes=276824064 weight_movement_bytes=1141899264 egress_bytes=5920033896 ingress_bytes=7054442496 link_bytes=7054442496 total_nvlink_bytes=12974476392 weight_to_token_ratio=0.096505 prefetch_share_of_ingress=0.122628 dedup_ratio=0.239409 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=8 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29337 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.154143443e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none + [2026-08-29 23:40:32] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 669.6 | throughput per GPU (TFLOP/s/GPU): 190.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952379E+00 | loss scale: 1.0 | grad norm: 42.220 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=64930 dup_rows=22936 dup_rows_remote=19194 dedup_ratio=0.233317 router_tokens_per_expert=[2866, 512, 415, 593, 311, 1547, 928, 93, 2807, 630, 1195, 485, 31, 1185, 241, 516, 1189, 1153, 471, 888, 322, 334, 431, 631, 551, 764, 133, 396, 928, 850, 364, 1193, 1185, 355, 926, 1922, 590, 413, 1081, 507, 151, 422, 445, 189, 1387, 1970, 749, 135, 290, 607, 599, 783, 817, 1804, 716, 292, 454, 910, 1409, 534, 404, 444, 1473, 236] payload_rows_per_dst=[11084, 10438, 8682, 8440, 10020, 8688, 8882, 9134] dup_rows_per_dst=[3446, 3742, 2156, 1918, 3938, 2208, 2934, 2594] combine_rows_per_src=[11084, 10438, 8682, 8440, 10020, 8688, 8882, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75120 payload_rows_remote=64014 dup_rows=23184 dup_rows_remote=19516 dedup_ratio=0.235840 router_tokens_per_expert=[2833, 567, 404, 698, 290, 1440, 894, 110, 2647, 623, 1210, 530, 18, 1175, 285, 447, 1272, 1190, 504, 760, 299, 340, 477, 618, 528, 760, 130, 340, 945, 833, 447, 1442, 1325, 357, 957, 1957, 624, 354, 1000, 525, 172, 438, 440, 247, 1313, 1912, 760, 148, 312, 580, 613, 798, 973, 1545, 605, 363, 423, 905, 1378, 521, 393, 376, 1462, 320] payload_rows_per_dst=[6988, 9016, 9958, 11622, 8276, 11106, 9076, 9078] dup_rows_per_dst=[1818, 2340, 3476, 4894, 2008, 3668, 2502, 2478] combine_rows_per_src=[6988, 9016, 9958, 11622, 8276, 11106, 9076, 9078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75390 payload_rows_remote=66792 dup_rows=22914 dup_rows_remote=20866 dedup_ratio=0.233093 router_tokens_per_expert=[2551, 463, 394, 727, 312, 1397, 996, 115, 2581, 670, 1201, 555, 20, 1108, 283, 437, 1156, 1223, 472, 922, 365, 397, 396, 615, 538, 789, 130, 383, 959, 807, 442, 1275, 1305, 374, 956, 1804, 715, 390, 1066, 539, 187, 421, 415, 239, 1298, 1969, 667, 161, 271, 608, 619, 773, 1013, 1912, 653, 428, 410, 844, 1410, 507, 433, 428, 1417, 241] payload_rows_per_dst=[10890, 10210, 8710, 8598, 10224, 8404, 9442, 8912] dup_rows_per_dst=[3020, 3500, 2382, 2048, 4074, 2310, 3112, 2468] combine_rows_per_src=[10890, 10210, 8710, 8598, 10224, 8404, 9442, 8912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=65974 dup_rows=22524 dup_rows_remote=18818 dedup_ratio=0.229126 router_tokens_per_expert=[2732, 549, 426, 735, 333, 1360, 930, 115, 2756, 599, 1260, 526, 24, 1138, 271, 492, 1244, 1131, 405, 837, 280, 329, 419, 580, 517, 754, 131, 368, 971, 862, 366, 1042, 1281, 370, 998, 1834, 646, 376, 1047, 530, 182, 456, 417, 239, 1297, 1984, 680, 171, 319, 635, 585, 781, 949, 1974, 775, 403, 393, 742, 1531, 498, 387, 402, 1466, 322] payload_rows_per_dst=[9834, 10598, 8560, 9210, 9806, 9164, 9518, 9090] dup_rows_per_dst=[2906, 3534, 1890, 2432, 3706, 2340, 3324, 2392] combine_rows_per_src=[9834, 10598, 8560, 9210, 9806, 9164, 9518, 9090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=63654 dup_rows=23848 dup_rows_remote=18514 dedup_ratio=0.242594 router_tokens_per_expert=[2639, 493, 370, 736, 384, 1378, 948, 90, 2817, 602, 1182, 559, 22, 1187, 257, 572, 1234, 1164, 517, 766, 295, 309, 436, 595, 546, 893, 99, 382, 940, 795, 398, 1433, 1229, 318, 945, 1840, 649, 401, 992, 527, 216, 417, 398, 171, 1384, 1889, 718, 136, 321, 587, 542, 839, 1017, 1576, 732, 343, 389, 802, 1490, 603, 337, 467, 1547, 292] payload_rows_per_dst=[7000, 7700, 10380, 9406, 8186, 10920, 10062, 10802] dup_rows_per_dst=[1798, 1062, 4090, 2562, 2096, 3418, 3488, 5334] combine_rows_per_src=[7000, 7700, 10380, 9406, 8186, 10920, 10062, 10802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75260 payload_rows_remote=65986 dup_rows=23044 dup_rows_remote=20064 dedup_ratio=0.234416 router_tokens_per_expert=[2723, 647, 435, 686, 349, 1161, 891, 133, 2602, 612, 1157, 513, 22, 1132, 274, 584, 1224, 1164, 483, 783, 347, 313, 508, 577, 568, 876, 128, 371, 1017, 792, 381, 1243, 1326, 420, 890, 1780, 582, 364, 1140, 542, 175, 500, 437, 250, 1389, 1852, 730, 181, 292, 506, 538, 880, 1005, 1834, 700, 372, 351, 748, 1406, 591, 400, 394, 1553, 328] payload_rows_per_dst=[6774, 7560, 11298, 11748, 8290, 11148, 9274, 9168] dup_rows_per_dst=[1830, 1028, 4704, 4450, 2238, 3440, 2980, 2374] combine_rows_per_src=[6774, 7560, 11298, 11748, 8290, 11148, 9274, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=336496 combine_payload_bytes=265953280 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266289776 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532243056 weight_movement_bytes=103809024 total_nvlink_bytes=636052080 weight_to_token_ratio=0.195041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=66754 dup_rows=22936 dup_rows_remote=20654 dedup_ratio=0.233317 router_tokens_per_expert=[2785, 551, 337, 676, 357, 1273, 1113, 104, 2470, 611, 1136, 559, 24, 1178, 275, 656, 1303, 1061, 551, 892, 304, 344, 395, 598, 631, 771, 112, 421, 856, 895, 398, 1119, 1200, 362, 1002, 1707, 669, 392, 1093, 561, 199, 423, 403, 228, 1335, 1855, 710, 165, 351, 527, 653, 725, 972, 1596, 631, 274, 377, 997, 1479, 513, 488, 516, 1506, 487] payload_rows_per_dst=[11128, 10272, 8614, 8342, 10044, 8364, 8900, 9704] dup_rows_per_dst=[3264, 3546, 2282, 2064, 3928, 2272, 2558, 3022] combine_rows_per_src=[11128, 10272, 8614, 8342, 10044, 8364, 8900, 9704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262201344 dispatch_meta_bytes=334120 combine_payload_bytes=262201344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262535464 ingress_bytes=296804352 link_bytes=296804352 token_bytes=524736808 weight_movement_bytes=34603008 total_nvlink_bytes=559339816 weight_to_token_ratio=0.065944 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116850882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273580032 dispatch_meta_bytes=350632 combine_payload_bytes=273580032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273930664 ingress_bytes=308183040 link_bytes=308183040 token_bytes=547510696 weight_movement_bytes=34603008 total_nvlink_bytes=582113704 weight_to_token_ratio=0.063201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159667969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270229504 dispatch_meta_bytes=339168 combine_payload_bytes=270229504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270568672 ingress_bytes=374038528 link_bytes=374038528 token_bytes=540798176 weight_movement_bytes=103809024 total_nvlink_bytes=644607200 weight_to_token_ratio=0.191955 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407476868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76228 payload_rows_remote=64334 dup_rows=22076 dup_rows_remote=17446 dedup_ratio=0.224569 router_tokens_per_expert=[2657, 530, 382, 758, 328, 1441, 1026, 89, 2481, 608, 1234, 493, 15, 1196, 317, 535, 1365, 1200, 510, 822, 265, 340, 445, 614, 509, 799, 120, 394, 938, 766, 422, 1337, 1273, 327, 1035, 1598, 584, 380, 996, 593, 180, 456, 448, 224, 1387, 1805, 722, 205, 313, 588, 634, 910, 1011, 1635, 634, 401, 431, 758, 1393, 550, 407, 473, 1506, 359] payload_rows_per_dst=[11894, 10340, 8894, 8780, 9108, 8572, 9408, 9232] dup_rows_per_dst=[4630, 3418, 2228, 1790, 2362, 2282, 2844, 2522] combine_rows_per_src=[11894, 10340, 8894, 8780, 9108, 8572, 9408, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270278656 dispatch_meta_bytes=344200 combine_payload_bytes=270278656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270622856 ingress_bytes=304881664 link_bytes=304881664 token_bytes=540901512 weight_movement_bytes=34603008 total_nvlink_bytes=575504520 weight_to_token_ratio=0.063973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147245157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260726784 dispatch_meta_bytes=328672 combine_payload_bytes=260726784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261055456 ingress_bytes=295329792 link_bytes=295329792 token_bytes=521782240 weight_movement_bytes=34603008 total_nvlink_bytes=556385248 weight_to_token_ratio=0.066317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111302231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273424384 dispatch_meta_bytes=349632 combine_payload_bytes=273424384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273774016 ingress_bytes=308027392 link_bytes=308027392 token_bytes=547198400 weight_movement_bytes=34603008 total_nvlink_bytes=581801408 weight_to_token_ratio=0.063237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159082278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78576 payload_rows_remote=68700 dup_rows=19728 dup_rows_remote=17552 dedup_ratio=0.200684 router_tokens_per_expert=[130, 334, 132, 137, 1716, 264, 2932, 314, 29, 345, 636, 350, 913, 686, 103, 2964, 552, 1063, 1417, 1217, 427, 1649, 758, 422, 185, 351, 1172, 4749, 36, 1045, 86, 647, 127, 7, 10, 218, 541, 9, 269, 180, 1595, 653, 0, 468, 142, 107, 9, 1145, 6154, 222, 2157, 9, 357, 263, 922, 681, 1019, 1054, 651, 527, 1036, 160, 354, 345] payload_rows_per_dst=[9974, 9876, 10724, 12198, 13146, 6466, 7686, 8506] dup_rows_per_dst=[1944, 2176, 4286, 4344, 1884, 1772, 1536, 1786] combine_rows_per_src=[9974, 9876, 10724, 12198, 13146, 6466, 7686, 8506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78840 payload_rows_remote=66414 dup_rows=19464 dup_rows_remote=15048 dedup_ratio=0.197998 router_tokens_per_expert=[155, 390, 220, 183, 1781, 256, 2817, 282, 60, 368, 637, 420, 904, 714, 139, 2841, 594, 1092, 1346, 1356, 507, 1606, 820, 395, 220, 326, 1051, 4458, 49, 873, 88, 728, 186, 13, 21, 240, 551, 23, 288, 191, 1645, 644, 0, 486, 163, 128, 9, 888, 5956, 285, 1996, 17, 382, 335, 905, 783, 964, 1084, 677, 493, 1134, 225, 390, 374] payload_rows_per_dst=[10082, 9804, 9050, 6306, 13024, 12426, 7768, 10380] dup_rows_per_dst=[2086, 2362, 2614, 920, 1914, 4416, 1638, 3514] combine_rows_per_src=[10082, 9804, 9050, 6306, 13024, 12426, 7768, 10380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263512064 dispatch_meta_bytes=327120 combine_payload_bytes=263512064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263839184 ingress_bytes=367321088 link_bytes=367321088 token_bytes=527351248 weight_movement_bytes=103809024 total_nvlink_bytes=631160272 weight_to_token_ratio=0.196850 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382199682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78864 payload_rows_remote=67112 dup_rows=19440 dup_rows_remote=15838 dedup_ratio=0.197754 router_tokens_per_expert=[189, 425, 161, 146, 1768, 341, 2769, 306, 47, 361, 606, 432, 965, 656, 181, 2835, 504, 1035, 1416, 1315, 506, 1614, 802, 369, 195, 296, 1040, 4473, 33, 923, 85, 632, 136, 17, 21, 259, 569, 17, 211, 185, 1528, 682, 0, 501, 142, 154, 6, 1319, 5911, 243, 2099, 16, 294, 324, 1008, 800, 973, 1021, 706, 532, 992, 211, 450, 399] payload_rows_per_dst=[10190, 9870, 10668, 11752, 12828, 6898, 7838, 8820] dup_rows_per_dst=[2020, 2296, 4040, 3602, 1824, 1766, 1730, 2162] combine_rows_per_src=[10190, 9870, 10668, 11752, 12828, 6898, 7838, 8820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=65612 dup_rows=19818 dup_rows_remote=18014 dedup_ratio=0.201599 router_tokens_per_expert=[139, 340, 149, 161, 1828, 303, 2960, 246, 40, 312, 591, 390, 948, 659, 150, 2929, 629, 1046, 1608, 1219, 433, 1603, 791, 304, 205, 383, 976, 4621, 20, 902, 49, 704, 126, 18, 18, 248, 460, 14, 262, 155, 1528, 728, 0, 502, 182, 94, 10, 1321, 6038, 218, 2035, 5, 337, 271, 983, 767, 957, 1065, 604, 472, 1005, 209, 458, 424] payload_rows_per_dst=[9982, 9822, 9284, 7660, 12874, 10984, 7606, 10274] dup_rows_per_dst=[2270, 2216, 2776, 1464, 1804, 4342, 1626, 3320] combine_rows_per_src=[9982, 9822, 9284, 7660, 12874, 10984, 7606, 10274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76938 payload_rows_remote=64452 dup_rows=21366 dup_rows_remote=15794 dedup_ratio=0.217346 router_tokens_per_expert=[190, 414, 190, 150, 1638, 260, 2810, 338, 66, 430, 673, 433, 992, 664, 172, 3019, 588, 1170, 1354, 1344, 518, 1649, 847, 363, 220, 348, 1184, 4421, 45, 909, 81, 658, 176, 20, 44, 219, 573, 23, 292, 170, 1473, 627, 0, 461, 167, 146, 10, 856, 5764, 263, 1928, 12, 304, 301, 1122, 791, 1038, 1028, 688, 504, 1016, 189, 416, 393] payload_rows_per_dst=[9814, 10262, 7810, 8128, 5718, 12208, 12486, 10512] dup_rows_per_dst=[2166, 2636, 1308, 1470, 770, 4114, 5572, 3330] combine_rows_per_src=[9814, 10262, 7810, 8128, 5718, 12208, 12486, 10512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=281395200 dispatch_meta_bytes=345008 combine_payload_bytes=281395200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281740208 ingress_bytes=315998208 link_bytes=315998208 token_bytes=563135408 weight_movement_bytes=34603008 total_nvlink_bytes=597738416 weight_to_token_ratio=0.061447 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189075817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272031744 dispatch_meta_bytes=325848 combine_payload_bytes=272031744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272357592 ingress_bytes=306634752 link_bytes=306634752 token_bytes=544389336 weight_movement_bytes=34603008 total_nvlink_bytes=578992344 weight_to_token_ratio=0.063563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153841886e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274890752 dispatch_meta_bytes=331800 combine_payload_bytes=274890752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275222552 ingress_bytes=344096768 link_bytes=344096768 token_bytes=550113304 weight_movement_bytes=69206016 total_nvlink_bytes=619319320 weight_to_token_ratio=0.125803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294808436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64124 dup_rows=24044 dup_rows_remote=20544 dedup_ratio=0.244588 router_tokens_per_expert=[172, 371, 173, 142, 1837, 267, 3120, 280, 40, 384, 720, 466, 883, 536, 143, 2724, 597, 999, 1458, 1206, 435, 1559, 857, 422, 213, 318, 1027, 4567, 39, 930, 70, 734, 168, 8, 10, 245, 545, 25, 301, 190, 1706, 652, 0, 427, 150, 117, 4, 855, 5922, 288, 2095, 23, 289, 350, 1072, 762, 935, 951, 813, 463, 1077, 207, 440, 373] payload_rows_per_dst=[10202, 9416, 6032, 7996, 2798, 12540, 15140, 10136] dup_rows_per_dst=[2522, 2376, 588, 1582, 186, 4416, 8874, 3500] combine_rows_per_src=[10202, 9416, 6032, 7996, 2798, 12540, 15140, 10136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268746752 dispatch_meta_bytes=334504 combine_payload_bytes=268746752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269081256 ingress_bytes=303349760 link_bytes=303349760 token_bytes=537828008 weight_movement_bytes=34603008 total_nvlink_bytes=572431016 weight_to_token_ratio=0.064338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141480725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=320984 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264316376 ingress_bytes=333201408 link_bytes=333201408 token_bytes=528311768 weight_movement_bytes=69206016 total_nvlink_bytes=597517784 weight_to_token_ratio=0.130995 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253810073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78358 payload_rows_remote=67802 dup_rows=19946 dup_rows_remote=15512 dedup_ratio=0.202901 router_tokens_per_expert=[239, 349, 209, 155, 1794, 244, 2793, 320, 65, 411, 651, 445, 902, 706, 174, 2706, 673, 947, 1432, 1368, 526, 1453, 697, 399, 268, 440, 1083, 4185, 40, 954, 109, 599, 194, 15, 11, 232, 635, 12, 299, 213, 1606, 703, 0, 486, 156, 196, 10, 839, 5882, 288, 1990, 9, 283, 362, 1027, 748, 1061, 1121, 828, 564, 964, 276, 423, 383] payload_rows_per_dst=[9952, 9706, 10556, 11762, 12998, 6732, 7826, 8826] dup_rows_per_dst=[2254, 2414, 4434, 3594, 1988, 1260, 1588, 2414] combine_rows_per_src=[9952, 9706, 10556, 11762, 12998, 6732, 7826, 8826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262651904 dispatch_meta_bytes=338672 combine_payload_bytes=262651904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262990576 ingress_bytes=297254912 link_bytes=297254912 token_bytes=525642480 weight_movement_bytes=34603008 total_nvlink_bytes=560245488 weight_to_token_ratio=0.065830 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118546303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78738 payload_rows_remote=68364 dup_rows=19566 dup_rows_remote=17094 dedup_ratio=0.199036 router_tokens_per_expert=[139, 401, 213, 151, 1897, 276, 2723, 290, 47, 366, 668, 400, 1034, 611, 159, 2878, 546, 1040, 1407, 1468, 565, 1500, 814, 433, 243, 376, 866, 4469, 35, 1067, 87, 620, 193, 10, 21, 230, 546, 18, 227, 203, 1595, 635, 0, 489, 119, 117, 9, 1018, 5826, 248, 2024, 20, 403, 276, 1043, 681, 1001, 1032, 635, 494, 1127, 218, 449, 456] payload_rows_per_dst=[10374, 10178, 10374, 12038, 12790, 6490, 7716, 8778] dup_rows_per_dst=[2472, 2894, 3760, 3488, 1758, 1474, 1674, 2046] combine_rows_per_src=[10374, 10178, 10374, 12038, 12790, 6490, 7716, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=333256 combine_payload_bytes=277716992 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=278050248 ingress_bytes=450732032 link_bytes=450732032 token_bytes=555767240 weight_movement_bytes=173015040 total_nvlink_bytes=728782280 weight_to_token_ratio=0.311308 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.696068350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=65490 dup_rows=24274 dup_rows_remote=22756 dedup_ratio=0.246928 router_tokens_per_expert=[1, 172, 12, 196, 28, 35, 2111, 5283, 1, 702, 463, 1453, 2067, 129, 136, 78, 2841, 1191, 895, 772, 54, 5, 4694, 342, 255, 1017, 5254, 1053, 694, 223, 134, 26, 119, 423, 1678, 100, 3738, 452, 414, 148, 179, 346, 0, 52, 1350, 1876, 112, 46, 232, 1747, 272, 23, 46, 87, 280, 4, 1306, 52, 20, 39, 2, 737, 13, 942] payload_rows_per_dst=[12852, 8540, 14290, 13588, 6194, 6996, 5608, 5962] dup_rows_per_dst=[2824, 1518, 9378, 8338, 474, 926, 556, 260] combine_rows_per_src=[12852, 8540, 14290, 13588, 6194, 6996, 5608, 5962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79902 payload_rows_remote=68938 dup_rows=18402 dup_rows_remote=15888 dedup_ratio=0.187195 router_tokens_per_expert=[2, 210, 13, 208, 31, 43, 1876, 4776, 1, 677, 377, 1311, 1910, 135, 144, 83, 2669, 1457, 941, 947, 64, 3, 4857, 330, 289, 1132, 5013, 1073, 615, 233, 195, 27, 142, 452, 1592, 110, 3884, 470, 519, 162, 190, 378, 1, 74, 1187, 2019, 142, 79, 290, 1685, 286, 54, 49, 72, 345, 4, 1468, 62, 70, 59, 8, 648, 28, 981] payload_rows_per_dst=[9918, 9472, 6526, 6236, 11770, 10964, 12018, 12998] dup_rows_per_dst=[1842, 2362, 958, 892, 2892, 2514, 3266, 3676] combine_rows_per_src=[9918, 9472, 6526, 6236, 11770, 10964, 12018, 12998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280018944 dispatch_meta_bytes=341832 combine_payload_bytes=280018944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280360776 ingress_bytes=314621952 link_bytes=314621952 token_bytes=560379720 weight_movement_bytes=34603008 total_nvlink_bytes=594982728 weight_to_token_ratio=0.061749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183897076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80152 payload_rows_remote=71276 dup_rows=18152 dup_rows_remote=15528 dedup_ratio=0.184652 router_tokens_per_expert=[2, 294, 6, 185, 35, 76, 2012, 4918, 0, 648, 402, 1310, 1927, 132, 177, 114, 2967, 1438, 845, 1069, 56, 3, 4475, 349, 349, 1150, 5174, 991, 625, 180, 179, 19, 201, 392, 1460, 123, 3881, 455, 553, 157, 177, 377, 0, 61, 1183, 1992, 137, 84, 307, 1592, 250, 28, 69, 78, 348, 4, 1297, 60, 43, 67, 7, 611, 28, 1023] payload_rows_per_dst=[12204, 8026, 6644, 8876, 11762, 11364, 11362, 9914] dup_rows_per_dst=[2852, 1394, 876, 2624, 2682, 2592, 2940, 2192] combine_rows_per_src=[12204, 8026, 6644, 8876, 11762, 11364, 11362, 9914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78758 payload_rows_remote=67192 dup_rows=19546 dup_rows_remote=16506 dedup_ratio=0.198832 router_tokens_per_expert=[2, 298, 13, 221, 44, 27, 2083, 4706, 1, 740, 340, 1364, 1669, 197, 183, 126, 2872, 1437, 892, 848, 100, 3, 4596, 319, 389, 1092, 4775, 1106, 627, 243, 177, 39, 254, 469, 1711, 147, 3653, 436, 550, 175, 215, 431, 0, 79, 1310, 2026, 128, 99, 297, 1657, 267, 78, 56, 73, 270, 9, 1282, 83, 55, 70, 4, 678, 28, 1033] payload_rows_per_dst=[5094, 13518, 6372, 6444, 11854, 11436, 11566, 12474] dup_rows_per_dst=[282, 5134, 826, 902, 2936, 2884, 3040, 3542] combine_rows_per_src=[5094, 13518, 6372, 6444, 11854, 11436, 11566, 12474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268247040 dispatch_meta_bytes=352984 combine_payload_bytes=268247040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268600024 ingress_bytes=302850048 link_bytes=302850048 token_bytes=536847064 weight_movement_bytes=34603008 total_nvlink_bytes=571450072 weight_to_token_ratio=0.064456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139600349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80766 payload_rows_remote=69038 dup_rows=17538 dup_rows_remote=14736 dedup_ratio=0.178406 router_tokens_per_expert=[2, 191, 3, 227, 38, 49, 1904, 4992, 1, 610, 508, 1289, 2045, 127, 151, 102, 2671, 1468, 1019, 698, 70, 3, 4958, 406, 260, 1087, 5384, 1069, 653, 201, 127, 22, 104, 517, 1600, 99, 3689, 497, 622, 137, 183, 300, 0, 68, 1346, 1943, 121, 67, 228, 1566, 236, 28, 47, 56, 246, 4, 1311, 41, 33, 49, 2, 593, 23, 1061] payload_rows_per_dst=[12178, 8298, 6336, 6044, 11728, 11000, 11934, 13248] dup_rows_per_dst=[2634, 1368, 992, 794, 2802, 2398, 2804, 3746] combine_rows_per_src=[12178, 8298, 6336, 6044, 11728, 11000, 11934, 13248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282370048 dispatch_meta_bytes=339304 combine_payload_bytes=282370048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282709352 ingress_bytes=316973056 link_bytes=316973056 token_bytes=565079400 weight_movement_bytes=34603008 total_nvlink_bytes=599682408 weight_to_token_ratio=0.061236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192744091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=291946496 dispatch_meta_bytes=347216 combine_payload_bytes=291946496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292293712 ingress_bytes=361152512 link_bytes=361152512 token_bytes=584240208 weight_movement_bytes=69206016 total_nvlink_bytes=653446224 weight_to_token_ratio=0.118455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358987827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275218432 dispatch_meta_bytes=334792 combine_payload_bytes=275218432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275553224 ingress_bytes=309821440 link_bytes=309821440 token_bytes=550771656 weight_movement_bytes=34603008 total_nvlink_bytes=585374664 weight_to_token_ratio=0.062826 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165833136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79110 payload_rows_remote=66110 dup_rows=19194 dup_rows_remote=15440 dedup_ratio=0.195251 router_tokens_per_expert=[3, 243, 13, 226, 26, 43, 1934, 4857, 0, 590, 361, 1243, 1976, 127, 177, 81, 2852, 1533, 929, 1012, 58, 1, 4622, 333, 323, 1189, 5177, 982, 677, 259, 153, 16, 158, 390, 1444, 88, 3802, 429, 543, 142, 245, 344, 2, 94, 1290, 2106, 146, 51, 331, 1591, 262, 30, 45, 63, 338, 2, 1352, 47, 47, 55, 2, 663, 23, 1011] payload_rows_per_dst=[4788, 13706, 6740, 6346, 11508, 11460, 11562, 13000] dup_rows_per_dst=[188, 5118, 992, 852, 2484, 2800, 3006, 3754] combine_rows_per_src=[4788, 13706, 6740, 6346, 11508, 11460, 11562, 13000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282779648 dispatch_meta_bytes=335096 combine_payload_bytes=282779648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=283114744 ingress_bytes=386588672 link_bytes=386588672 token_bytes=565894392 weight_movement_bytes=103809024 total_nvlink_bytes=669703416 weight_to_token_ratio=0.183442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454702050e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78172 payload_rows_remote=69640 dup_rows=20132 dup_rows_remote=18142 dedup_ratio=0.204793 router_tokens_per_expert=[1, 355, 9, 171, 53, 51, 1815, 4539, 3, 616, 378, 1389, 2024, 196, 238, 148, 2835, 1464, 912, 995, 101, 4, 4335, 313, 322, 1109, 4724, 1074, 682, 238, 204, 41, 264, 414, 1480, 134, 3612, 620, 693, 171, 302, 370, 2, 104, 1259, 2109, 138, 80, 415, 1301, 227, 63, 53, 100, 388, 7, 1483, 114, 33, 66, 7, 667, 49, 1088] payload_rows_per_dst=[11434, 8368, 8532, 12830, 9988, 9438, 10938, 6644] dup_rows_per_dst=[2554, 1616, 1990, 6312, 2434, 2016, 2840, 370] combine_rows_per_src=[11434, 8368, 8532, 12830, 9988, 9438, 10938, 6644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73102 payload_rows_remote=60632 dup_rows=25202 dup_rows_remote=21012 dedup_ratio=0.256368 router_tokens_per_expert=[5, 352, 17, 132, 44, 65, 2041, 4768, 2, 751, 408, 1452, 1877, 144, 190, 109, 2529, 1266, 984, 793, 72, 5, 4104, 308, 394, 1146, 5028, 987, 631, 211, 179, 37, 173, 418, 1622, 107, 4271, 440, 495, 143, 250, 351, 1, 129, 1746, 2047, 128, 106, 297, 1404, 220, 46, 79, 78, 305, 3, 1345, 67, 78, 52, 6, 603, 17, 1094] payload_rows_per_dst=[12470, 8422, 14910, 12446, 6070, 7994, 4584, 6206] dup_rows_per_dst=[4190, 1444, 11942, 4780, 726, 1522, 280, 318] combine_rows_per_src=[12470, 8422, 14910, 12446, 6070, 7994, 4584, 6206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270786560 dispatch_meta_bytes=326200 combine_payload_bytes=270786560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271112760 ingress_bytes=305389568 link_bytes=305389568 token_bytes=541899320 weight_movement_bytes=34603008 total_nvlink_bytes=576502328 weight_to_token_ratio=0.063855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149156359e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=62292 dup_rows=23252 dup_rows_remote=20844 dedup_ratio=0.236532 router_tokens_per_expert=[871, 36, 1880, 21, 544, 147, 644, 472, 232, 121, 88, 29, 307, 6, 855, 157, 5939, 350, 228, 2, 523, 90, 0, 3792, 299, 0, 2221, 2345, 911, 1122, 5789, 81, 637, 1006, 28, 40, 943, 409, 935, 238, 344, 1489, 40, 2332, 178, 535, 0, 1352, 1, 138, 1754, 718, 38, 558, 34, 1526, 230, 0, 884, 97, 199, 1883, 142, 342] payload_rows_per_dst=[10630, 12760, 15482, 4562, 7206, 9344, 8252, 6816] dup_rows_per_dst=[3290, 2408, 10808, 264, 1266, 3196, 1282, 738] combine_rows_per_src=[10630, 12760, 15482, 4562, 7206, 9344, 8252, 6816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285245440 dispatch_meta_bytes=351128 combine_payload_bytes=285245440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285596568 ingress_bytes=389054464 link_bytes=389054464 token_bytes=570842008 weight_movement_bytes=103809024 total_nvlink_bytes=674651032 weight_to_token_ratio=0.181852 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463980627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77002 payload_rows_remote=67582 dup_rows=21302 dup_rows_remote=18032 dedup_ratio=0.216695 router_tokens_per_expert=[918, 49, 1686, 46, 589, 210, 619, 446, 219, 123, 95, 72, 322, 18, 883, 270, 5723, 304, 217, 2, 362, 110, 0, 3654, 269, 0, 2083, 2218, 810, 1186, 5778, 81, 800, 1069, 22, 43, 858, 331, 962, 240, 401, 1639, 59, 2128, 188, 515, 6, 1409, 0, 157, 1963, 926, 75, 621, 28, 1212, 341, 0, 807, 104, 254, 2215, 221, 196] payload_rows_per_dst=[10466, 12700, 3672, 8250, 8836, 9420, 11006, 12652] dup_rows_per_dst=[3096, 2396, 304, 1072, 2060, 3270, 4280, 4824] combine_rows_per_src=[10466, 12700, 3672, 8250, 8836, 9420, 11006, 12652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70488 payload_rows_remote=58724 dup_rows=27816 dup_rows_remote=22374 dedup_ratio=0.282959 router_tokens_per_expert=[870, 41, 1536, 26, 435, 154, 646, 409, 249, 106, 80, 44, 335, 13, 570, 105, 5655, 383, 312, 1, 532, 141, 1, 3602, 204, 2, 2111, 2385, 984, 1384, 5788, 81, 774, 1022, 28, 46, 859, 333, 978, 273, 388, 1736, 60, 2359, 197, 615, 2, 1201, 1, 158, 1931, 741, 75, 600, 38, 1457, 286, 0, 1008, 85, 303, 1917, 292, 204] payload_rows_per_dst=[7618, 2930, 2550, 14962, 7262, 9610, 11764, 13792] dup_rows_per_dst=[1198, 74, 190, 10334, 1364, 3506, 5442, 5708] combine_rows_per_src=[7618, 2930, 2550, 14962, 7262, 9610, 11764, 13792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=248348672 dispatch_meta_bytes=326576 combine_payload_bytes=248348672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248675248 ingress_bytes=317554688 link_bytes=317554688 token_bytes=497023920 weight_movement_bytes=69206016 total_nvlink_bytes=566229936 weight_to_token_ratio=0.139241 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.194932726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79826 payload_rows_remote=71970 dup_rows=18478 dup_rows_remote=17706 dedup_ratio=0.187968 router_tokens_per_expert=[915, 42, 1531, 19, 556, 142, 664, 351, 175, 121, 94, 50, 353, 29, 779, 188, 5617, 346, 344, 1, 500, 112, 1, 3813, 219, 0, 2203, 2405, 791, 1044, 5893, 57, 753, 972, 18, 59, 1039, 453, 967, 244, 307, 1472, 68, 2344, 182, 515, 4, 1379, 1, 127, 1900, 802, 54, 592, 30, 1227, 233, 0, 1019, 71, 275, 2208, 168, 314] payload_rows_per_dst=[10442, 13034, 9006, 7856, 14444, 9410, 8142, 7492] dup_rows_per_dst=[2808, 2330, 1228, 772, 5800, 3132, 1324, 1084] combine_rows_per_src=[10442, 13034, 9006, 7856, 14444, 9410, 8142, 7492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79006 payload_rows_remote=64116 dup_rows=19298 dup_rows_remote=12834 dedup_ratio=0.196309 router_tokens_per_expert=[850, 35, 1735, 30, 510, 144, 675, 451, 175, 97, 80, 44, 341, 13, 690, 209, 6026, 354, 281, 0, 743, 102, 0, 3571, 267, 0, 2094, 2212, 743, 994, 5873, 121, 866, 1120, 25, 54, 1101, 354, 893, 238, 332, 1500, 50, 2257, 196, 622, 0, 1469, 2, 121, 1830, 788, 35, 516, 39, 1459, 226, 0, 974, 94, 244, 1776, 153, 358] payload_rows_per_dst=[10462, 12746, 8648, 7688, 14890, 9454, 8348, 6770] dup_rows_per_dst=[2822, 2298, 1454, 750, 6464, 3398, 1232, 880] combine_rows_per_src=[10462, 12746, 8648, 7688, 14890, 9454, 8348, 6770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276815872 dispatch_meta_bytes=342456 combine_payload_bytes=276815872 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=277158328 ingress_bytes=311418880 link_bytes=311418880 token_bytes=553974200 weight_movement_bytes=34603008 total_nvlink_bytes=588577208 weight_to_token_ratio=0.062463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171844174e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=255148032 dispatch_meta_bytes=332544 combine_payload_bytes=255148032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255480576 ingress_bytes=289751040 link_bytes=289751040 token_bytes=510628608 weight_movement_bytes=34603008 total_nvlink_bytes=545231616 weight_to_token_ratio=0.067766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090309837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=240533504 dispatch_meta_bytes=324392 combine_payload_bytes=240533504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240857896 ingress_bytes=275136512 link_bytes=275136512 token_bytes=481391400 weight_movement_bytes=34603008 total_nvlink_bytes=515994408 weight_to_token_ratio=0.071881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.035316544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=294789120 dispatch_meta_bytes=358704 combine_payload_bytes=294789120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=295147824 ingress_bytes=433201152 link_bytes=433201152 token_bytes=589936944 weight_movement_bytes=138412032 total_nvlink_bytes=728348976 weight_to_token_ratio=0.234622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630101059e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262619136 dispatch_meta_bytes=307800 combine_payload_bytes=262619136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262926936 ingress_bytes=297222144 link_bytes=297222144 token_bytes=525546072 weight_movement_bytes=34603008 total_nvlink_bytes=560149080 weight_to_token_ratio=0.065842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118423000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77736 payload_rows_remote=65468 dup_rows=20568 dup_rows_remote=14964 dedup_ratio=0.209229 router_tokens_per_expert=[897, 53, 1588, 46, 606, 218, 493, 458, 270, 138, 124, 113, 318, 26, 904, 183, 5380, 301, 255, 8, 333, 159, 0, 3430, 349, 4, 2495, 2373, 772, 1154, 5691, 111, 767, 994, 18, 47, 1098, 274, 938, 245, 376, 1714, 80, 2276, 153, 451, 5, 1347, 6, 127, 1956, 1060, 58, 512, 40, 1282, 300, 0, 761, 83, 273, 2129, 245, 287] payload_rows_per_dst=[10382, 12966, 12268, 6972, 9868, 9374, 8632, 7274] dup_rows_per_dst=[3082, 2568, 5604, 706, 2846, 3430, 1450, 882] combine_rows_per_src=[10382, 12966, 12268, 6972, 9868, 9374, 8632, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70444 payload_rows_remote=56338 dup_rows=27860 dup_rows_remote=21244 dedup_ratio=0.283407 router_tokens_per_expert=[926, 45, 1621, 31, 565, 184, 491, 490, 245, 122, 74, 39, 329, 16, 739, 239, 5834, 346, 233, 2, 407, 127, 0, 3588, 272, 0, 2261, 2082, 830, 1254, 5758, 86, 580, 867, 10, 46, 750, 270, 1075, 229, 453, 1571, 53, 2297, 170, 582, 6, 1262, 0, 151, 2143, 880, 45, 478, 39, 1432, 273, 0, 1056, 111, 251, 2301, 185, 350] payload_rows_per_dst=[7544, 3512, 2100, 14936, 6594, 9564, 12088, 14106] dup_rows_per_dst=[1162, 94, 130, 10150, 1060, 3224, 5424, 6616] combine_rows_per_src=[7544, 3512, 2100, 14936, 6594, 9564, 12088, 14106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76500 payload_rows_remote=66040 dup_rows=21804 dup_rows_remote=18962 dedup_ratio=0.221802 router_tokens_per_expert=[902, 81, 1637, 25, 443, 163, 542, 512, 227, 129, 109, 53, 352, 10, 880, 203, 5670, 248, 259, 3, 403, 131, 1, 3399, 266, 1, 2316, 2346, 814, 1184, 5781, 135, 593, 961, 18, 69, 909, 385, 990, 247, 336, 1702, 67, 2390, 235, 482, 5, 1384, 6, 134, 1962, 769, 54, 580, 35, 1279, 256, 0, 946, 142, 302, 2209, 263, 217] payload_rows_per_dst=[10460, 13058, 15298, 7416, 7042, 7138, 8448, 7640] dup_rows_per_dst=[2842, 2430, 9562, 1614, 1302, 1834, 1190, 1030] combine_rows_per_src=[10460, 13058, 15298, 7416, 7042, 7138, 8448, 7640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=321728 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268478656 ingress_bytes=406568960 link_bytes=406568960 token_bytes=536635584 weight_movement_bytes=138412032 total_nvlink_bytes=675047616 weight_to_token_ratio=0.257926 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529886265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84026 payload_rows_remote=70800 dup_rows=14278 dup_rows_remote=12750 dedup_ratio=0.145243 router_tokens_per_expert=[4, 195, 1044, 0, 19, 523, 21, 300, 1, 392, 529, 210, 0, 0, 14, 73, 0, 1058, 0, 7163, 638, 177, 320, 811, 85, 0, 490, 4700, 109, 6158, 60, 0, 0, 1, 0, 7595, 2611, 1, 42, 594, 11, 1, 18, 984, 292, 0, 2616, 59, 0, 51, 1200, 37, 9, 367, 1287, 1519, 6, 563, 0, 13, 4076, 94, 2, 9] payload_rows_per_dst=[15592, 13226, 15138, 10134, 6050, 7562, 7388, 8936] dup_rows_per_dst=[3810, 1528, 5196, 754, 448, 400, 1552, 590] combine_rows_per_src=[15592, 13226, 15138, 10134, 6050, 7562, 7388, 8936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80248 payload_rows_remote=66336 dup_rows=18056 dup_rows_remote=12126 dedup_ratio=0.183675 router_tokens_per_expert=[11, 237, 1216, 0, 20, 559, 31, 355, 1, 428, 524, 254, 0, 0, 8, 64, 0, 1067, 0, 7085, 605, 184, 297, 899, 83, 2, 443, 4500, 162, 5816, 45, 0, 0, 4, 0, 7617, 2682, 3, 49, 642, 8, 8, 18, 1029, 348, 0, 2313, 74, 0, 57, 1346, 35, 7, 340, 1284, 1489, 6, 588, 2, 6, 4218, 68, 6, 9] payload_rows_per_dst=[4652, 12658, 5602, 9636, 15904, 13912, 8686, 9198] dup_rows_per_dst=[206, 1532, 502, 834, 6090, 5930, 2354, 608] combine_rows_per_src=[4652, 12658, 5602, 9636, 15904, 13912, 8686, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230760448 dispatch_meta_bytes=310328 combine_payload_bytes=230760448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231070776 ingress_bytes=265363456 link_bytes=265363456 token_bytes=461831224 weight_movement_bytes=34603008 total_nvlink_bytes=496434232 weight_to_token_ratio=0.074926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.985413214e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77662 payload_rows_remote=62332 dup_rows=20642 dup_rows_remote=13980 dedup_ratio=0.209981 router_tokens_per_expert=[18, 312, 915, 1, 23, 580, 23, 408, 6, 365, 657, 298, 2, 0, 20, 44, 0, 1294, 0, 7013, 469, 265, 321, 936, 61, 3, 605, 4750, 151, 5922, 70, 2, 2, 2, 0, 7609, 2545, 7, 53, 487, 22, 6, 63, 1084, 414, 0, 2484, 92, 0, 50, 1023, 61, 18, 346, 1168, 1317, 13, 535, 2, 38, 4042, 115, 6, 14] payload_rows_per_dst=[4376, 8642, 5958, 12408, 11058, 7736, 15330, 12154] dup_rows_per_dst=[184, 1094, 612, 3768, 2896, 594, 6662, 4832] combine_rows_per_src=[4376, 8642, 5958, 12408, 11058, 7736, 15330, 12154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270499840 dispatch_meta_bytes=340008 combine_payload_bytes=270499840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270839848 ingress_bytes=305102848 link_bytes=305102848 token_bytes=541339688 weight_movement_bytes=34603008 total_nvlink_bytes=575942696 weight_to_token_ratio=0.063921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148077454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80134 payload_rows_remote=64550 dup_rows=18170 dup_rows_remote=12140 dedup_ratio=0.184835 router_tokens_per_expert=[12, 168, 1186, 0, 14, 507, 20, 335, 1, 371, 531, 205, 0, 0, 11, 53, 0, 1213, 0, 7165, 530, 190, 259, 750, 95, 1, 561, 4963, 82, 6016, 57, 0, 0, 0, 0, 7653, 2807, 8, 54, 580, 5, 1, 33, 1064, 307, 0, 2622, 50, 0, 46, 1205, 31, 10, 306, 1137, 1240, 8, 469, 1, 8, 4088, 101, 2, 20] payload_rows_per_dst=[4696, 12984, 5438, 10724, 15584, 15128, 6706, 8874] dup_rows_per_dst=[378, 1392, 446, 794, 6030, 7366, 1244, 520] combine_rows_per_src=[4696, 12984, 5438, 10724, 15584, 15128, 6706, 8874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81632 payload_rows_remote=71012 dup_rows=16672 dup_rows_remote=15298 dedup_ratio=0.169596 router_tokens_per_expert=[7, 284, 1284, 0, 24, 606, 18, 365, 2, 475, 482, 263, 1, 0, 15, 40, 0, 1175, 0, 7116, 750, 181, 218, 762, 69, 5, 574, 4698, 145, 5661, 108, 3, 0, 1, 0, 7537, 2465, 3, 50, 634, 7, 1, 22, 1016, 498, 0, 2480, 76, 0, 60, 1109, 62, 11, 340, 1360, 1408, 5, 613, 0, 20, 3932, 95, 6, 10] payload_rows_per_dst=[15542, 12272, 10168, 10620, 5954, 11118, 7208, 8750] dup_rows_per_dst=[4708, 1606, 2770, 1374, 352, 3758, 1492, 612] combine_rows_per_src=[15542, 12272, 10168, 10620, 5954, 11118, 7208, 8750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=289996800 dispatch_meta_bytes=334200 combine_payload_bytes=289996800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290331000 ingress_bytes=324599808 link_bytes=324599808 token_bytes=580327800 weight_movement_bytes=34603008 total_nvlink_bytes=614930808 weight_to_token_ratio=0.059627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221442945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=271712256 dispatch_meta_bytes=313848 combine_payload_bytes=271712256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272026104 ingress_bytes=306315264 link_bytes=306315264 token_bytes=543738360 weight_movement_bytes=34603008 total_nvlink_bytes=578341368 weight_to_token_ratio=0.063639 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152639678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=255311872 dispatch_meta_bytes=305248 combine_payload_bytes=255311872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255617120 ingress_bytes=289914880 link_bytes=289914880 token_bytes=510928992 weight_movement_bytes=34603008 total_nvlink_bytes=545532000 weight_to_token_ratio=0.067726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090926353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264396800 dispatch_meta_bytes=306760 combine_payload_bytes=264396800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264703560 ingress_bytes=368205824 link_bytes=368205824 token_bytes=529100360 weight_movement_bytes=103809024 total_nvlink_bytes=632909384 weight_to_token_ratio=0.196199 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385528873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82632 payload_rows_remote=67686 dup_rows=15672 dup_rows_remote=10344 dedup_ratio=0.159424 router_tokens_per_expert=[7, 233, 1268, 1, 20, 491, 20, 423, 7, 546, 609, 440, 0, 0, 18, 50, 0, 1062, 0, 6989, 508, 143, 346, 1089, 73, 0, 428, 4915, 133, 5072, 63, 1, 0, 0, 0, 7422, 2728, 9, 82, 834, 12, 11, 46, 1156, 312, 0, 2656, 79, 0, 102, 848, 56, 11, 380, 1341, 1432, 10, 599, 2, 21, 3909, 117, 3, 19] payload_rows_per_dst=[15330, 11828, 14946, 10382, 6660, 8040, 6748, 8698] dup_rows_per_dst=[4440, 1656, 5328, 844, 646, 504, 1592, 662] combine_rows_per_src=[15330, 11828, 14946, 10382, 6660, 8040, 6748, 8698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290865152 dispatch_meta_bytes=345240 combine_payload_bytes=290865152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291210392 ingress_bytes=394674176 link_bytes=394674176 token_bytes=582075544 weight_movement_bytes=103809024 total_nvlink_bytes=685884568 weight_to_token_ratio=0.178343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75652 payload_rows_remote=59924 dup_rows=22652 dup_rows_remote=13546 dedup_ratio=0.230428 router_tokens_per_expert=[2, 222, 1104, 0, 16, 535, 20, 362, 1, 403, 596, 377, 0, 0, 5, 56, 1, 1113, 0, 7403, 521, 287, 356, 978, 95, 1, 549, 4590, 104, 5726, 46, 1, 0, 2, 0, 7538, 2864, 5, 58, 608, 12, 8, 38, 1023, 340, 0, 2369, 56, 1, 43, 713, 50, 31, 354, 1396, 1294, 12, 483, 2, 14, 4272, 84, 3, 9] payload_rows_per_dst=[4358, 2784, 5934, 12656, 11342, 7172, 15678, 15728] dup_rows_per_dst=[164, 92, 578, 1780, 3520, 520, 6892, 9106] combine_rows_per_src=[4358, 2784, 5934, 12656, 11342, 7172, 15678, 15728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83606 payload_rows_remote=69150 dup_rows=14698 dup_rows_remote=11784 dedup_ratio=0.149516 router_tokens_per_expert=[5, 512, 1970, 0, 354, 21, 63, 407, 131, 0, 9, 1340, 0, 1, 2431, 1345, 5735, 7964, 44, 2, 5198, 1534, 2209, 0, 0, 17, 375, 0, 0, 1, 27, 0, 16, 0, 0, 1, 0, 1, 3236, 138, 6784, 6, 292, 0, 0, 6, 0, 1597, 10, 16, 245, 3091, 8, 21, 0, 1143, 1, 26, 0, 12, 0, 709, 1, 97] payload_rows_per_dst=[6150, 14110, 3132, 15952, 6730, 14456, 11140, 11936] dup_rows_per_dst=[514, 6800, 28, 816, 54, 2914, 2346, 1226] combine_rows_per_src=[6150, 14110, 3132, 15952, 6730, 14456, 11140, 11936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=312120 combine_payload_bytes=277241856 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=277553976 ingress_bytes=381050880 link_bytes=381050880 token_bytes=554795832 weight_movement_bytes=103809024 total_nvlink_bytes=658604856 weight_to_token_ratio=0.187112 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433863784e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78112 payload_rows_remote=70016 dup_rows=20192 dup_rows_remote=18532 dedup_ratio=0.205404 router_tokens_per_expert=[1, 375, 2046, 0, 309, 22, 27, 209, 75, 0, 1, 1224, 0, 4, 2256, 1318, 5657, 8007, 49, 1, 5836, 1463, 2627, 0, 0, 15, 343, 1, 0, 0, 27, 0, 1, 0, 1, 1, 0, 0, 3440, 128, 6985, 4, 174, 1, 0, 2, 0, 1636, 4, 11, 270, 3376, 4, 17, 0, 548, 0, 23, 0, 7, 0, 519, 0, 107] payload_rows_per_dst=[13716, 8096, 16216, 772, 15504, 3556, 8512, 11740] dup_rows_per_dst=[3934, 1660, 7312, 0, 5608, 78, 714, 886] combine_rows_per_src=[13716, 8096, 16216, 772, 15504, 3556, 8512, 11740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82748 payload_rows_remote=67302 dup_rows=15556 dup_rows_remote=11134 dedup_ratio=0.158244 router_tokens_per_expert=[15, 315, 1188, 2, 23, 607, 20, 324, 5, 557, 623, 410, 1, 0, 21, 41, 0, 1113, 0, 6870, 426, 128, 416, 891, 54, 0, 689, 4846, 150, 5847, 58, 1, 0, 5, 0, 7440, 2197, 11, 47, 625, 34, 2, 58, 1069, 425, 0, 2631, 63, 0, 62, 874, 42, 14, 390, 1480, 1524, 8, 566, 0, 19, 3761, 147, 5, 12] payload_rows_per_dst=[15446, 12968, 14710, 10674, 5412, 7970, 7136, 8432] dup_rows_per_dst=[4422, 2042, 4978, 922, 358, 594, 1636, 604] combine_rows_per_src=[15446, 12968, 14710, 10674, 5412, 7970, 7136, 8432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=245448704 dispatch_meta_bytes=293880 combine_payload_bytes=245448704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245742584 ingress_bytes=280051712 link_bytes=280051712 token_bytes=491191288 weight_movement_bytes=34603008 total_nvlink_bytes=525794296 weight_to_token_ratio=0.070447 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.053812046e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85892 payload_rows_remote=75138 dup_rows=12412 dup_rows_remote=10260 dedup_ratio=0.126261 router_tokens_per_expert=[1, 422, 1682, 0, 324, 21, 31, 357, 123, 0, 13, 1468, 1, 8, 2879, 1323, 5621, 7932, 45, 1, 5493, 1312, 2336, 0, 0, 11, 370, 0, 0, 2, 42, 0, 16, 0, 0, 1, 0, 2, 3445, 197, 6653, 5, 329, 1, 0, 11, 0, 1830, 4, 10, 323, 2988, 4, 16, 0, 772, 0, 63, 0, 13, 1, 534, 1, 115] payload_rows_per_dst=[5344, 10100, 10792, 15880, 7122, 14188, 10754, 11712] dup_rows_per_dst=[332, 3204, 1236, 834, 200, 3470, 2152, 984] combine_rows_per_src=[5344, 10100, 10792, 15880, 7122, 14188, 10754, 11712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84048 payload_rows_remote=77926 dup_rows=14256 dup_rows_remote=14214 dedup_ratio=0.145020 router_tokens_per_expert=[0, 386, 2044, 0, 264, 21, 40, 217, 70, 0, 1, 1347, 0, 2, 2577, 1188, 5800, 8041, 53, 2, 5497, 1402, 2583, 0, 0, 5, 349, 0, 0, 1, 21, 0, 4, 0, 0, 1, 0, 0, 2930, 147, 7070, 4, 232, 0, 0, 14, 0, 1884, 3, 10, 267, 3199, 3, 11, 0, 734, 0, 43, 0, 19, 0, 573, 0, 93] payload_rows_per_dst=[10006, 10886, 2902, 16084, 6122, 14864, 11148, 12036] dup_rows_per_dst=[2202, 4214, 12, 750, 42, 3544, 2472, 1020] combine_rows_per_src=[10006, 10886, 2902, 16084, 6122, 14864, 11148, 12036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84616 payload_rows_remote=68528 dup_rows=13688 dup_rows_remote=12966 dedup_ratio=0.139242 router_tokens_per_expert=[0, 466, 2064, 0, 344, 36, 74, 223, 117, 0, 13, 1312, 0, 2, 2182, 1313, 5769, 8038, 48, 3, 5066, 1665, 2625, 0, 1, 12, 327, 0, 0, 0, 27, 0, 11, 0, 0, 2, 0, 3, 3057, 178, 6764, 4, 232, 1, 0, 16, 1, 1940, 11, 11, 234, 3467, 6, 31, 0, 691, 4, 39, 0, 31, 0, 586, 0, 105] payload_rows_per_dst=[12810, 8070, 3406, 16088, 8584, 12110, 11568, 11980] dup_rows_per_dst=[3736, 1808, 26, 722, 1306, 2418, 2584, 1088] combine_rows_per_src=[12810, 8070, 3406, 16088, 8584, 12110, 11568, 11980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283238400 dispatch_meta_bytes=323736 combine_payload_bytes=283238400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283562136 ingress_bytes=352444416 link_bytes=352444416 token_bytes=566800536 weight_movement_bytes=69206016 total_nvlink_bytes=636006552 weight_to_token_ratio=0.122099 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326219963e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=286785536 dispatch_meta_bytes=354192 combine_payload_bytes=286785536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287139728 ingress_bytes=321388544 link_bytes=321388544 token_bytes=573925264 weight_movement_bytes=34603008 total_nvlink_bytes=608528272 weight_to_token_ratio=0.060292 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209359217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275668992 dispatch_meta_bytes=313744 combine_payload_bytes=275668992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275982736 ingress_bytes=310272000 link_bytes=310272000 token_bytes=551651728 weight_movement_bytes=34603008 total_nvlink_bytes=586254736 weight_to_token_ratio=0.062726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167528557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=307765248 dispatch_meta_bytes=341592 combine_payload_bytes=307765248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308106840 ingress_bytes=342368256 link_bytes=342368256 token_bytes=615872088 weight_movement_bytes=34603008 total_nvlink_bytes=650475096 weight_to_token_ratio=0.056185 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288304184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=319184896 dispatch_meta_bytes=368560 combine_payload_bytes=319184896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319553456 ingress_bytes=353787904 link_bytes=353787904 token_bytes=638738352 weight_movement_bytes=34603008 total_nvlink_bytes=673341360 weight_to_token_ratio=0.054174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331275400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280690688 dispatch_meta_bytes=325976 combine_payload_bytes=280690688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281016664 ingress_bytes=315293696 link_bytes=315293696 token_bytes=561707352 weight_movement_bytes=34603008 total_nvlink_bytes=596310360 weight_to_token_ratio=0.061603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186424795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81826 payload_rows_remote=75716 dup_rows=16478 dup_rows_remote=15728 dedup_ratio=0.167623 router_tokens_per_expert=[4, 481, 1888, 0, 425, 23, 74, 470, 78, 0, 8, 1425, 1, 3, 2070, 1464, 5608, 7853, 19, 2, 4970, 1555, 2321, 0, 0, 32, 337, 0, 0, 1, 36, 0, 6, 0, 0, 1, 0, 3, 3523, 204, 6954, 4, 275, 2, 0, 3, 0, 1656, 12, 16, 249, 3224, 4, 46, 0, 875, 0, 52, 0, 25, 0, 728, 0, 142] payload_rows_per_dst=[12992, 8334, 6110, 12198, 15384, 3766, 11214, 11828] dup_rows_per_dst=[3678, 1764, 750, 612, 5998, 114, 2280, 1282] combine_rows_per_src=[12992, 8334, 6110, 12198, 15384, 3766, 11214, 11828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87542 payload_rows_remote=75008 dup_rows=10762 dup_rows_remote=9486 dedup_ratio=0.109477 router_tokens_per_expert=[0, 474, 1726, 0, 393, 27, 36, 465, 50, 0, 3, 1227, 0, 3, 2096, 1205, 6050, 8017, 23, 2, 5342, 1555, 2228, 0, 0, 7, 342, 0, 0, 2, 17, 0, 7, 0, 2, 0, 0, 0, 3534, 115, 6963, 11, 292, 0, 0, 4, 0, 1772, 10, 5, 236, 3133, 5, 22, 0, 896, 0, 51, 0, 17, 0, 670, 0, 117] payload_rows_per_dst=[5824, 7720, 12334, 16038, 7280, 14840, 10972, 12534] dup_rows_per_dst=[418, 1448, 1510, 732, 36, 3244, 2098, 1276] combine_rows_per_src=[5824, 7720, 12334, 16038, 7280, 14840, 10972, 12534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=310132736 dispatch_meta_bytes=365776 combine_payload_bytes=310132736 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=310498512 ingress_bytes=517750784 link_bytes=517750784 token_bytes=620631248 weight_movement_bytes=207618048 total_nvlink_bytes=828249296 weight_to_token_ratio=0.334527 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.948254518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86152 payload_rows_remote=74092 dup_rows=12152 dup_rows_remote=9250 dedup_ratio=0.123617 router_tokens_per_expert=[1990, 0, 7273, 12, 0, 0, 0, 4732, 0, 0, 918, 1725, 6722, 6218, 470, 14, 0, 0, 0, 505, 0, 5, 1859, 674, 0, 84, 116, 2161, 0, 0, 1, 0, 62, 1, 5548, 1, 0, 0, 0, 0, 4, 1901, 0, 20, 6, 0, 724, 94, 20, 0, 4647, 542, 30, 14, 0, 0, 52, 3, 0, 0, 2, 0, 2, 0] payload_rows_per_dst=[3984, 5772, 14318, 15220, 11218, 12060, 10124, 13456] dup_rows_per_dst=[20, 482, 4204, 4050, 6, 2902, 382, 106] combine_rows_per_src=[3984, 5772, 14318, 15220, 11218, 12060, 10124, 13456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=307232768 dispatch_meta_bytes=337976 combine_payload_bytes=307232768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307570744 ingress_bytes=341835776 link_bytes=341835776 token_bytes=614803512 weight_movement_bytes=34603008 total_nvlink_bytes=649406520 weight_to_token_ratio=0.056283 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286300505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81038 payload_rows_remote=67678 dup_rows=17266 dup_rows_remote=13486 dedup_ratio=0.175639 router_tokens_per_expert=[2015, 0, 7416, 7, 0, 0, 0, 4716, 0, 0, 865, 1571, 6613, 5664, 451, 19, 0, 0, 0, 342, 0, 3, 2379, 537, 0, 69, 116, 1956, 0, 0, 1, 0, 32, 0, 5725, 1, 0, 0, 0, 0, 1, 1808, 0, 25, 5, 0, 715, 94, 35, 0, 5254, 636, 18, 1, 0, 0, 55, 2, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[16206, 13360, 6070, 4258, 11510, 5136, 11260, 13238] dup_rows_per_dst=[12102, 3780, 452, 26, 6, 160, 628, 112] combine_rows_per_src=[16206, 13360, 6070, 4258, 11510, 5136, 11260, 13238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70724 payload_rows_remote=57780 dup_rows=27580 dup_rows_remote=23876 dedup_ratio=0.280558 router_tokens_per_expert=[2, 438, 2011, 0, 372, 34, 35, 367, 139, 0, 17, 1336, 0, 1, 2210, 1342, 5502, 7968, 54, 5, 5065, 1554, 2469, 0, 0, 13, 360, 0, 0, 0, 36, 4, 8, 0, 5, 3, 0, 3, 3465, 204, 6826, 9, 314, 2, 1, 3, 3, 2039, 11, 8, 273, 3232, 4, 27, 0, 620, 0, 71, 5, 18, 0, 536, 1, 127] payload_rows_per_dst=[12944, 8278, 16308, 826, 13690, 4562, 7902, 6214] dup_rows_per_dst=[3704, 1812, 16230, 0, 4644, 180, 448, 562] combine_rows_per_src=[12944, 8278, 16308, 826, 13690, 4562, 7902, 6214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86254 payload_rows_remote=75706 dup_rows=12050 dup_rows_remote=11612 dedup_ratio=0.122579 router_tokens_per_expert=[1922, 0, 7127, 10, 0, 0, 1, 4467, 0, 2, 988, 1678, 6942, 5954, 436, 11, 0, 0, 0, 602, 0, 11, 2239, 744, 0, 86, 109, 1887, 0, 1, 0, 0, 72, 1, 5594, 1, 0, 0, 0, 0, 1, 1767, 0, 49, 10, 0, 806, 82, 23, 0, 4840, 582, 29, 19, 0, 0, 48, 4, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[3852, 5784, 14332, 14966, 11330, 11548, 10548, 13894] dup_rows_per_dst=[14, 446, 4768, 3454, 6, 2816, 438, 108] combine_rows_per_src=[3852, 5784, 14332, 14966, 11330, 11548, 10548, 13894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86778 payload_rows_remote=75650 dup_rows=11526 dup_rows_remote=11516 dedup_ratio=0.117249 router_tokens_per_expert=[1778, 0, 7448, 11, 0, 0, 0, 5206, 0, 0, 837, 1666, 6815, 6118, 465, 11, 0, 0, 0, 292, 0, 2, 2431, 655, 0, 64, 97, 1446, 0, 1, 7, 0, 67, 0, 5499, 3, 0, 0, 0, 0, 12, 2131, 0, 44, 3, 0, 569, 76, 15, 1, 4796, 524, 0, 1, 0, 0, 55, 2, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[3556, 5510, 14456, 15386, 11128, 12824, 10280, 13638] dup_rows_per_dst=[22, 448, 4540, 2740, 10, 3258, 394, 114] combine_rows_per_src=[3556, 5510, 14456, 15386, 11128, 12824, 10280, 13638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86738 payload_rows_remote=71164 dup_rows=11566 dup_rows_remote=8270 dedup_ratio=0.117655 router_tokens_per_expert=[2022, 1, 7520, 7, 1, 0, 0, 4837, 0, 0, 1116, 1805, 6449, 6048, 500, 5, 0, 0, 3, 406, 0, 3, 2049, 678, 0, 78, 109, 1723, 0, 0, 5, 0, 62, 3, 5642, 3, 0, 0, 0, 0, 8, 1607, 0, 30, 2, 0, 730, 92, 28, 0, 4770, 707, 22, 14, 1, 0, 55, 2, 0, 0, 0, 0, 7, 2] payload_rows_per_dst=[4048, 8054, 12232, 15574, 11412, 12026, 10442, 12950] dup_rows_per_dst=[14, 1772, 3168, 3296, 8, 2586, 642, 80] combine_rows_per_src=[4048, 8054, 12232, 15574, 11412, 12026, 10442, 12950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=303480832 dispatch_meta_bytes=333368 combine_payload_bytes=303480832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303814200 ingress_bytes=338083840 link_bytes=338083840 token_bytes=607295032 weight_movement_bytes=34603008 total_nvlink_bytes=641898040 weight_to_token_ratio=0.056979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272182272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277209088 dispatch_meta_bytes=324656 combine_payload_bytes=277209088 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=277533744 ingress_bytes=415621120 link_bytes=415621120 token_bytes=554742832 weight_movement_bytes=138412032 total_nvlink_bytes=693154864 weight_to_token_ratio=0.249507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.563948814e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=236666880 dispatch_meta_bytes=326624 combine_payload_bytes=236666880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=236993504 ingress_bytes=271269888 link_bytes=271269888 token_bytes=473660384 weight_movement_bytes=34603008 total_nvlink_bytes=508263392 weight_to_token_ratio=0.073054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.020766750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310091776 dispatch_meta_bytes=349272 combine_payload_bytes=310091776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310441048 ingress_bytes=344694784 link_bytes=344694784 token_bytes=620532824 weight_movement_bytes=34603008 total_nvlink_bytes=655135832 weight_to_token_ratio=0.055763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297058722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=309862400 dispatch_meta_bytes=348664 combine_payload_bytes=309862400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310211064 ingress_bytes=344465408 link_bytes=344465408 token_bytes=620073464 weight_movement_bytes=34603008 total_nvlink_bytes=654676472 weight_to_token_ratio=0.055805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296195599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=291487744 dispatch_meta_bytes=317736 combine_payload_bytes=291487744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291805480 ingress_bytes=326090752 link_bytes=326090752 token_bytes=583293224 weight_movement_bytes=34603008 total_nvlink_bytes=617896232 weight_to_token_ratio=0.059324 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227053247e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85136 payload_rows_remote=79292 dup_rows=13168 dup_rows_remote=12534 dedup_ratio=0.133952 router_tokens_per_expert=[2101, 0, 7058, 10, 0, 0, 0, 4593, 0, 0, 970, 1943, 6731, 5882, 473, 10, 0, 0, 1, 546, 0, 17, 1773, 902, 0, 100, 131, 2187, 0, 0, 0, 0, 45, 7, 5438, 4, 0, 0, 0, 0, 2, 1893, 0, 56, 8, 0, 728, 107, 26, 0, 4651, 485, 146, 22, 0, 0, 90, 8, 0, 0, 8, 0, 0, 0] payload_rows_per_dst=[15298, 13904, 5844, 4870, 10958, 10468, 10298, 13496] dup_rows_per_dst=[5066, 4652, 634, 84, 30, 2162, 362, 178] combine_rows_per_src=[15298, 13904, 5844, 4870, 10958, 10468, 10298, 13496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79480 payload_rows_remote=65410 dup_rows=18824 dup_rows_remote=18750 dedup_ratio=0.191488 router_tokens_per_expert=[2074, 0, 7524, 6, 0, 0, 0, 4453, 0, 0, 1017, 1815, 7024, 6022, 408, 10, 0, 0, 0, 459, 0, 12, 2070, 659, 0, 105, 120, 2103, 0, 0, 2, 0, 12, 1, 4515, 3, 1, 0, 0, 0, 4, 2020, 0, 40, 4, 1, 788, 100, 37, 0, 5163, 502, 18, 12, 0, 0, 29, 8, 0, 0, 8, 0, 2, 1] payload_rows_per_dst=[4148, 5982, 6450, 8694, 14090, 11904, 14142, 14070] dup_rows_per_dst=[12, 518, 746, 1710, 6222, 2916, 6626, 74] combine_rows_per_src=[4148, 5982, 6450, 8694, 14090, 11904, 14142, 14070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=324780032 dispatch_meta_bytes=367304 combine_payload_bytes=324780032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325147336 ingress_bytes=359383040 link_bytes=359383040 token_bytes=649927368 weight_movement_bytes=34603008 total_nvlink_bytes=684530376 weight_to_token_ratio=0.053241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352329447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=73792 dup_rows=11968 dup_rows_remote=8034 dedup_ratio=0.121745 router_tokens_per_expert=[1184, 446, 0, 58, 0, 0, 0, 0, 1, 68, 7818, 0, 0, 2416, 7241, 30, 0, 0, 0, 0, 1065, 54, 0, 5, 34, 263, 17, 1180, 0, 8192, 1, 48, 2621, 1, 1149, 0, 5179, 0, 58, 0, 415, 2492, 0, 0, 1, 152, 0, 0, 161, 0, 5, 13, 51, 4, 5488, 0, 9, 887, 0, 0, 0, 0, 314, 31] payload_rows_per_dst=[3304, 4992, 14864, 16384, 7284, 12544, 11178, 15786] dup_rows_per_dst=[72, 38, 1866, 3086, 374, 3934, 266, 2332] combine_rows_per_src=[3304, 4992, 14864, 16384, 7284, 12544, 11178, 15786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267919360 dispatch_meta_bytes=336640 combine_payload_bytes=267919360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268256000 ingress_bytes=302522368 link_bytes=302522368 token_bytes=536175360 weight_movement_bytes=34603008 total_nvlink_bytes=570778368 weight_to_token_ratio=0.064537 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138367316e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71746 payload_rows_remote=55404 dup_rows=26558 dup_rows_remote=8704 dedup_ratio=0.270162 router_tokens_per_expert=[1298, 476, 0, 49, 0, 0, 0, 0, 0, 69, 7873, 0, 0, 2059, 7296, 52, 0, 0, 0, 0, 1004, 44, 0, 1, 36, 313, 5, 1615, 0, 8192, 1, 8, 2934, 0, 1029, 0, 5394, 0, 30, 0, 369, 1911, 0, 0, 0, 185, 0, 0, 106, 0, 2, 14, 80, 0, 5553, 0, 0, 868, 0, 0, 0, 0, 254, 32] payload_rows_per_dst=[16384, 16342, 2522, 3802, 14350, 4886, 11240, 2220] dup_rows_per_dst=[3646, 17854, 78, 154, 4424, 44, 270, 88] combine_rows_per_src=[16384, 16342, 2522, 3802, 14350, 4886, 11240, 2220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71238 payload_rows_remote=55042 dup_rows=27066 dup_rows_remote=13300 dedup_ratio=0.275330 router_tokens_per_expert=[1662, 0, 7409, 12, 0, 0, 0, 4828, 0, 1, 1007, 1822, 7045, 5933, 481, 21, 0, 0, 0, 213, 1, 15, 2015, 801, 0, 72, 99, 1815, 0, 0, 2, 0, 80, 9, 5087, 6, 1, 0, 0, 2, 10, 1654, 0, 66, 4, 0, 936, 99, 43, 0, 5203, 562, 56, 5, 0, 0, 65, 0, 1, 0, 7, 0, 2, 0] payload_rows_per_dst=[16196, 15718, 5674, 3940, 10350, 5302, 11288, 2770] dup_rows_per_dst=[13766, 12122, 416, 36, 20, 236, 450, 20] combine_rows_per_src=[16196, 15718, 5674, 3940, 10350, 5302, 11288, 2770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85676 payload_rows_remote=73792 dup_rows=12628 dup_rows_remote=11904 dedup_ratio=0.128459 router_tokens_per_expert=[1417, 508, 0, 115, 0, 0, 0, 0, 3, 80, 7657, 1, 0, 2404, 7299, 77, 0, 0, 0, 0, 1119, 49, 0, 9, 33, 445, 20, 1300, 0, 8190, 1, 47, 1975, 0, 1179, 0, 4660, 0, 55, 0, 412, 2586, 0, 5, 0, 180, 0, 0, 196, 0, 4, 7, 105, 6, 5718, 1, 9, 795, 0, 0, 0, 0, 461, 24] payload_rows_per_dst=[3996, 4548, 14950, 16382, 6124, 12136, 11884, 15656] dup_rows_per_dst=[84, 48, 2002, 3690, 294, 3550, 724, 2236] combine_rows_per_src=[3996, 4548, 14950, 16382, 6124, 12136, 11884, 15656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86498 payload_rows_remote=79376 dup_rows=11806 dup_rows_remote=11480 dedup_ratio=0.120097 router_tokens_per_expert=[1200, 393, 0, 59, 0, 0, 0, 2, 0, 73, 7812, 0, 0, 2449, 7212, 89, 0, 1, 0, 0, 1337, 31, 0, 1, 28, 335, 6, 1446, 2, 8191, 5, 11, 2745, 0, 956, 0, 5518, 0, 23, 0, 388, 1901, 0, 1, 0, 127, 0, 0, 127, 0, 1, 11, 94, 7, 5517, 0, 2, 845, 0, 0, 0, 0, 180, 26] payload_rows_per_dst=[6750, 5202, 14874, 13054, 7122, 12526, 11256, 15714] dup_rows_per_dst=[858, 20, 2290, 2694, 326, 3344, 258, 2016] combine_rows_per_src=[6750, 5202, 14874, 13054, 7122, 12526, 11256, 15714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87082 payload_rows_remote=83092 dup_rows=11222 dup_rows_remote=11142 dedup_ratio=0.114156 router_tokens_per_expert=[1015, 450, 0, 76, 0, 0, 1, 3, 0, 74, 7683, 0, 2, 2390, 7375, 23, 0, 0, 2, 0, 1144, 66, 0, 4, 51, 292, 17, 1622, 3, 8182, 1, 49, 2332, 0, 1361, 0, 5288, 1, 34, 0, 396, 1713, 0, 1, 5, 156, 0, 0, 194, 2, 1, 15, 80, 16, 5827, 0, 13, 880, 0, 1, 0, 1, 290, 20] payload_rows_per_dst=[16374, 4954, 15158, 3990, 7068, 12014, 11898, 15626] dup_rows_per_dst=[3080, 24, 2024, 80, 402, 3090, 372, 2150] combine_rows_per_src=[16374, 4954, 15158, 3990, 7068, 12014, 11898, 15626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=302252032 dispatch_meta_bytes=327304 combine_payload_bytes=302252032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302579336 ingress_bytes=336855040 link_bytes=336855040 token_bytes=604831368 weight_movement_bytes=34603008 total_nvlink_bytes=639434376 weight_to_token_ratio=0.057211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267558396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=226934784 dispatch_meta_bytes=256432 combine_payload_bytes=226934784 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=227191216 ingress_bytes=399949824 link_bytes=399949824 token_bytes=454126000 weight_movement_bytes=173015040 total_nvlink_bytes=627141040 weight_to_token_ratio=0.380985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504978989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=225452032 dispatch_meta_bytes=273368 combine_payload_bytes=225452032 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=225725400 ingress_bytes=363864064 link_bytes=363864064 token_bytes=451177432 weight_movement_bytes=138412032 total_nvlink_bytes=589589464 weight_to_token_ratio=0.306780 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369191179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=302252032 dispatch_meta_bytes=342784 combine_payload_bytes=302252032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302594816 ingress_bytes=336855040 link_bytes=336855040 token_bytes=604846848 weight_movement_bytes=34603008 total_nvlink_bytes=639449856 weight_to_token_ratio=0.057210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267558396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79180 payload_rows_remote=64344 dup_rows=19124 dup_rows_remote=17338 dedup_ratio=0.194539 router_tokens_per_expert=[1178, 544, 0, 41, 0, 0, 0, 0, 3, 115, 7691, 2, 0, 2631, 7227, 52, 0, 0, 0, 0, 1023, 59, 0, 2, 85, 298, 12, 1233, 0, 8188, 0, 20, 2301, 2, 1032, 0, 5054, 0, 45, 0, 392, 2895, 0, 1, 1, 202, 0, 0, 146, 0, 3, 11, 80, 13, 5304, 0, 13, 813, 0, 0, 1, 0, 405, 34] payload_rows_per_dst=[16378, 7642, 14836, 3140, 15260, 6906, 10868, 4150] dup_rows_per_dst=[3524, 1186, 1786, 156, 11860, 76, 246, 290] combine_rows_per_src=[16378, 7642, 14836, 3140, 15260, 6906, 10868, 4150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=325124096 dispatch_meta_bytes=363424 combine_payload_bytes=325124096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325487520 ingress_bytes=394330112 link_bytes=394330112 token_bytes=650611616 weight_movement_bytes=69206016 total_nvlink_bytes=719817632 weight_to_token_ratio=0.106371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.483832465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=340344832 dispatch_meta_bytes=376936 combine_payload_bytes=340344832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=340721768 ingress_bytes=409550848 link_bytes=409550848 token_bytes=681066600 weight_movement_bytes=69206016 total_nvlink_bytes=750272616 weight_to_token_ratio=0.101614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541106869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75990 payload_rows_remote=60284 dup_rows=22314 dup_rows_remote=20434 dedup_ratio=0.226990 router_tokens_per_expert=[1190, 454, 0, 69, 1, 0, 0, 0, 4, 62, 7764, 0, 1, 2253, 7406, 60, 0, 0, 0, 0, 1315, 52, 0, 13, 27, 277, 7, 937, 0, 8188, 0, 11, 2471, 1, 890, 0, 5348, 0, 35, 0, 380, 2801, 0, 3, 1, 185, 0, 0, 171, 0, 6, 12, 61, 0, 5667, 0, 9, 653, 0, 0, 0, 0, 333, 34] payload_rows_per_dst=[3380, 252, 7512, 16380, 6524, 12918, 13318, 15706] dup_rows_per_dst=[48, 2, 1084, 11490, 270, 4518, 3022, 1880] combine_rows_per_src=[3380, 252, 7512, 16380, 6524, 12918, 13318, 15706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=263553024 dispatch_meta_bytes=326728 combine_payload_bytes=263553024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263879752 ingress_bytes=298156032 link_bytes=298156032 token_bytes=527432776 weight_movement_bytes=34603008 total_nvlink_bytes=562035784 weight_to_token_ratio=0.065606 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121937145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=246923264 dispatch_meta_bytes=322872 combine_payload_bytes=246923264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247246136 ingress_bytes=281526272 link_bytes=281526272 token_bytes=494169400 weight_movement_bytes=34603008 total_nvlink_bytes=528772408 weight_to_token_ratio=0.070023 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.059360697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78790 payload_rows_remote=69844 dup_rows=19514 dup_rows_remote=17808 dedup_ratio=0.198507 router_tokens_per_expert=[3733, 687, 0, 196, 2, 0, 384, 0, 0, 0, 0, 1928, 0, 0, 3, 214, 0, 184, 1, 0, 443, 105, 0, 0, 0, 0, 829, 0, 0, 2347, 2601, 2, 0, 910, 0, 0, 0, 0, 0, 0, 0, 1945, 3, 1, 7863, 0, 0, 4990, 1, 0, 0, 0, 2396, 9, 747, 2, 60, 878, 0, 98, 0, 3, 7961, 7626] payload_rows_per_dst=[9076, 8476, 8928, 9710, 9906, 8946, 12926, 10822] dup_rows_per_dst=[928, 1590, 844, 1848, 886, 1706, 3364, 8348] combine_rows_per_src=[9076, 8476, 8928, 9710, 9906, 8946, 12926, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71386 payload_rows_remote=55028 dup_rows=26918 dup_rows_remote=22850 dedup_ratio=0.273824 router_tokens_per_expert=[1413, 560, 0, 71, 0, 0, 0, 1, 3, 87, 7672, 0, 0, 2252, 7394, 192, 1, 0, 1, 0, 1148, 71, 0, 8, 68, 478, 24, 1310, 0, 8168, 4, 27, 2107, 1, 985, 0, 5190, 0, 28, 0, 435, 2245, 0, 0, 26, 203, 0, 0, 196, 0, 4, 14, 99, 13, 5426, 0, 10, 790, 0, 0, 0, 0, 368, 59] payload_rows_per_dst=[16358, 16356, 2444, 3598, 13456, 5696, 11156, 2322] dup_rows_per_dst=[4068, 18844, 14, 224, 3166, 122, 348, 132] combine_rows_per_src=[16358, 16356, 2444, 3598, 13456, 5696, 11156, 2322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84828 payload_rows_remote=72220 dup_rows=13476 dup_rows_remote=10366 dedup_ratio=0.137085 router_tokens_per_expert=[3373, 753, 0, 211, 2, 0, 425, 0, 0, 0, 0, 1965, 0, 0, 0, 181, 0, 176, 0, 0, 208, 111, 0, 0, 0, 0, 807, 0, 0, 2522, 2758, 0, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 2025, 2, 0, 7881, 0, 0, 5090, 1, 0, 0, 0, 2176, 4, 567, 0, 59, 920, 0, 109, 0, 4, 7994, 7747] payload_rows_per_dst=[10652, 12608, 15992, 10074, 16070, 11820, 5456, 2156] dup_rows_per_dst=[2944, 3110, 986, 2100, 1854, 2414, 40, 28] combine_rows_per_src=[10652, 12608, 15992, 10074, 16070, 11820, 5456, 2156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70706 payload_rows_remote=58514 dup_rows=27598 dup_rows_remote=24266 dedup_ratio=0.280741 router_tokens_per_expert=[3357, 766, 0, 208, 12, 1, 493, 0, 0, 0, 0, 2120, 0, 0, 6, 150, 0, 100, 0, 0, 467, 105, 0, 1, 0, 0, 851, 0, 0, 2451, 2626, 2, 0, 886, 0, 0, 0, 0, 0, 0, 0, 2209, 2, 0, 7899, 0, 0, 4498, 4, 0, 1, 1, 2635, 17, 605, 1, 146, 914, 0, 129, 0, 0, 7926, 7563] payload_rows_per_dst=[8676, 4514, 1346, 9868, 1772, 15964, 12192, 16374] dup_rows_per_dst=[998, 38, 0, 1992, 0, 4256, 3332, 16982] combine_rows_per_src=[8676, 4514, 1346, 9868, 1772, 15964, 12192, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85164 payload_rows_remote=69092 dup_rows=13140 dup_rows_remote=11060 dedup_ratio=0.133667 router_tokens_per_expert=[3292, 781, 0, 224, 7, 1, 392, 0, 0, 0, 0, 1900, 0, 0, 3, 173, 0, 184, 0, 0, 272, 145, 0, 0, 0, 0, 822, 0, 0, 2349, 2495, 2, 0, 1144, 0, 1, 0, 0, 0, 0, 0, 1994, 5, 1, 7931, 0, 0, 5011, 2, 0, 0, 0, 2501, 1, 613, 0, 64, 1091, 0, 103, 0, 3, 7936, 7709] payload_rows_per_dst=[8492, 15594, 15882, 9654, 16072, 4000, 12968, 2502] dup_rows_per_dst=[902, 3976, 1192, 1682, 2080, 0, 3288, 20] combine_rows_per_src=[8492, 15594, 15882, 9654, 16072, 4000, 12968, 2502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84856 payload_rows_remote=75072 dup_rows=13448 dup_rows_remote=11722 dedup_ratio=0.136800 router_tokens_per_expert=[3596, 720, 0, 213, 14, 0, 401, 1, 0, 0, 0, 2047, 0, 1, 4, 159, 1, 190, 1, 0, 424, 118, 1, 0, 0, 0, 855, 0, 2, 2394, 2485, 19, 0, 860, 0, 0, 0, 1, 0, 0, 0, 1760, 10, 0, 7850, 0, 0, 4898, 1, 1, 0, 0, 2985, 10, 521, 0, 40, 952, 0, 119, 0, 9, 7924, 7565] payload_rows_per_dst=[8916, 15400, 15864, 9784, 15920, 4204, 12562, 2206] dup_rows_per_dst=[974, 4152, 1454, 1726, 1502, 258, 3348, 34] combine_rows_per_src=[8916, 15400, 15864, 9784, 15920, 4204, 12562, 2206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286081024 dispatch_meta_bytes=350608 combine_payload_bytes=286081024 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=286431632 ingress_bytes=424493056 link_bytes=424493056 token_bytes=572512656 weight_movement_bytes=138412032 total_nvlink_bytes=710924688 weight_to_token_ratio=0.241762 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.597333195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=225394688 dispatch_meta_bytes=311512 combine_payload_bytes=225394688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=225706200 ingress_bytes=259997696 link_bytes=259997696 token_bytes=451100888 weight_movement_bytes=34603008 total_nvlink_bytes=485703896 weight_to_token_ratio=0.076708 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.783503985e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=295813120 dispatch_meta_bytes=330344 combine_payload_bytes=295813120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296143464 ingress_bytes=330416128 link_bytes=330416128 token_bytes=591956584 weight_movement_bytes=34603008 total_nvlink_bytes=626559592 weight_to_token_ratio=0.058455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243329289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=239673344 dispatch_meta_bytes=331120 combine_payload_bytes=239673344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240004464 ingress_bytes=274276352 link_bytes=274276352 token_bytes=479677808 weight_movement_bytes=34603008 total_nvlink_bytes=514280816 weight_to_token_ratio=0.072138 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032079832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85836 payload_rows_remote=70012 dup_rows=12468 dup_rows_remote=11040 dedup_ratio=0.126831 router_tokens_per_expert=[3926, 669, 0, 199, 8, 0, 421, 0, 0, 0, 0, 2212, 0, 1, 2, 189, 0, 128, 2, 0, 482, 109, 0, 1, 0, 0, 848, 0, 0, 2587, 2348, 6, 0, 915, 0, 0, 0, 1, 0, 0, 0, 1861, 3, 0, 7691, 0, 0, 4645, 2, 0, 0, 1, 2573, 10, 648, 0, 110, 1128, 0, 104, 0, 0, 7904, 7418] payload_rows_per_dst=[9446, 15152, 15824, 9622, 15666, 11106, 6372, 2648] dup_rows_per_dst=[1000, 4492, 1428, 1956, 1548, 1912, 96, 36] combine_rows_per_src=[9446, 15152, 15824, 9622, 15666, 11106, 6372, 2648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=307494912 dispatch_meta_bytes=347176 combine_payload_bytes=307494912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307842088 ingress_bytes=342097920 link_bytes=342097920 token_bytes=615337000 weight_movement_bytes=34603008 total_nvlink_bytes=649940008 weight_to_token_ratio=0.056234 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287286932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283000832 dispatch_meta_bytes=320608 combine_payload_bytes=283000832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283321440 ingress_bytes=317603840 link_bytes=317603840 token_bytes=566322272 weight_movement_bytes=34603008 total_nvlink_bytes=600925280 weight_to_token_ratio=0.061101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.195117681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71628 payload_rows_remote=55244 dup_rows=26676 dup_rows_remote=9230 dedup_ratio=0.271362 router_tokens_per_expert=[3770, 810, 0, 175, 7, 0, 392, 0, 0, 0, 0, 2264, 0, 1, 4, 129, 0, 152, 1, 1, 431, 96, 0, 2, 0, 0, 803, 0, 0, 2525, 2605, 2, 0, 1053, 0, 1, 3, 0, 0, 0, 0, 1840, 4, 2, 7765, 0, 0, 4970, 1, 0, 0, 0, 2019, 1, 685, 1, 59, 897, 0, 90, 0, 1, 8046, 7544] payload_rows_per_dst=[9286, 4758, 1360, 9942, 2112, 15334, 12452, 16384] dup_rows_per_dst=[1022, 38, 6, 1928, 2, 3332, 2902, 17446] combine_rows_per_src=[9286, 4758, 1360, 9942, 2112, 15334, 12452, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286769152 dispatch_meta_bytes=324208 combine_payload_bytes=286769152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287093360 ingress_bytes=321372160 link_bytes=321372160 token_bytes=573862512 weight_movement_bytes=34603008 total_nvlink_bytes=608465520 weight_to_token_ratio=0.060298 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209297565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=226279424 dispatch_meta_bytes=257896 combine_payload_bytes=226279424 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=226537320 ingress_bytes=364691456 link_bytes=364691456 token_bytes=452816744 weight_movement_bytes=138412032 total_nvlink_bytes=591228776 weight_to_token_ratio=0.305669 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372304589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89732 payload_rows_remote=79150 dup_rows=8572 dup_rows_remote=6928 dedup_ratio=0.087199 router_tokens_per_expert=[1014, 2625, 5851, 785, 0, 6279, 286, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3745, 0, 0, 0, 376, 2, 240, 2004, 2798, 8192, 0, 155, 0, 0, 0, 0, 0, 1, 0, 15, 0, 1250, 4280, 0, 0, 583, 0, 0, 0, 0, 5252, 0, 910, 0, 0, 0, 0, 0, 65, 1589, 0, 850, 0, 0, 3] payload_rows_per_dst=[7850, 12560, 8238, 9362, 16384, 10582, 11520, 13236] dup_rows_per_dst=[1292, 2, 4, 1314, 32, 1644, 804, 3480] combine_rows_per_src=[7850, 12560, 8238, 9362, 16384, 10582, 11520, 13236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80290 payload_rows_remote=64124 dup_rows=18014 dup_rows_remote=9188 dedup_ratio=0.183248 router_tokens_per_expert=[3444, 744, 0, 231, 16, 0, 560, 0, 0, 1, 0, 2004, 0, 0, 5, 193, 0, 134, 1, 0, 456, 124, 0, 0, 0, 0, 937, 0, 0, 2394, 2622, 17, 1, 876, 1, 0, 0, 4, 0, 0, 0, 1434, 2, 1, 7814, 0, 0, 5089, 1, 0, 0, 2, 2971, 2, 527, 0, 98, 952, 0, 103, 0, 6, 7884, 7501] payload_rows_per_dst=[16166, 4344, 15788, 11642, 11682, 11406, 6966, 2296] dup_rows_per_dst=[8826, 62, 1410, 4776, 1232, 1646, 40, 22] combine_rows_per_src=[16166, 4344, 15788, 11642, 11682, 11406, 6966, 2296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81718 payload_rows_remote=69582 dup_rows=16586 dup_rows_remote=16568 dedup_ratio=0.168722 router_tokens_per_expert=[1047, 2466, 5837, 543, 0, 6068, 229, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 4312, 0, 0, 0, 348, 2, 220, 1851, 3333, 8192, 0, 174, 0, 0, 0, 0, 0, 3, 0, 18, 0, 1770, 4011, 0, 0, 556, 0, 0, 0, 0, 4746, 0, 772, 0, 0, 0, 0, 0, 53, 1615, 0, 977, 0, 0, 0] payload_rows_per_dst=[14704, 12136, 12344, 16384, 42, 10594, 10458, 5056] dup_rows_per_dst=[5540, 18, 3642, 4494, 0, 2080, 578, 234] combine_rows_per_src=[14704, 12136, 12344, 16384, 42, 10594, 10458, 5056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87746 payload_rows_remote=76334 dup_rows=10558 dup_rows_remote=9378 dedup_ratio=0.107402 router_tokens_per_expert=[813, 2463, 5684, 784, 0, 6604, 238, 0, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 375, 0, 82, 2107, 2908, 8192, 0, 126, 0, 0, 0, 1, 0, 3, 0, 53, 0, 1167, 4101, 0, 0, 569, 0, 0, 0, 4, 5177, 0, 1115, 0, 0, 0, 0, 0, 23, 1486, 0, 973, 0, 0, 21] payload_rows_per_dst=[3376, 13216, 8872, 11274, 16384, 10240, 11412, 12972] dup_rows_per_dst=[294, 26, 10, 4098, 114, 1434, 1180, 3402] combine_rows_per_src=[3376, 13216, 8872, 11274, 16384, 10240, 11412, 12972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87746 payload_rows_remote=82412 dup_rows=10558 dup_rows_remote=10530 dedup_ratio=0.107402 router_tokens_per_expert=[1128, 2484, 5794, 998, 0, 6088, 261, 0, 0, 0, 0, 1, 0, 10, 0, 2, 0, 0, 0, 3741, 0, 0, 0, 327, 0, 268, 2276, 3270, 8188, 0, 137, 0, 1, 0, 3, 0, 5, 0, 28, 1, 1616, 2891, 0, 0, 656, 2, 0, 2, 0, 5810, 0, 923, 0, 0, 1, 0, 1, 38, 1279, 0, 911, 0, 0, 11] payload_rows_per_dst=[8074, 12180, 11602, 5334, 16380, 9228, 12070, 12878] dup_rows_per_dst=[1668, 22, 3074, 28, 72, 1106, 1398, 3190] combine_rows_per_src=[8074, 12180, 11602, 5334, 16380, 9228, 12070, 12878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88884 payload_rows_remote=72500 dup_rows=9420 dup_rows_remote=9216 dedup_ratio=0.095825 router_tokens_per_expert=[1018, 2491, 5869, 701, 0, 5978, 370, 0, 1, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 360, 2, 167, 2093, 3164, 8192, 0, 142, 1, 0, 0, 0, 0, 3, 0, 99, 0, 1256, 4507, 0, 0, 562, 0, 0, 0, 0, 5767, 0, 794, 0, 0, 0, 0, 0, 38, 974, 1, 704, 0, 0, 4] payload_rows_per_dst=[7810, 11992, 11038, 5720, 16384, 10890, 12294, 12756] dup_rows_per_dst=[1350, 94, 2436, 324, 204, 1760, 828, 2424] combine_rows_per_src=[7810, 11992, 11038, 5720, 16384, 10890, 12294, 12756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=324198400 dispatch_meta_bytes=344312 combine_payload_bytes=324198400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324542712 ingress_bytes=358801408 link_bytes=358801408 token_bytes=648741112 weight_movement_bytes=34603008 total_nvlink_bytes=683344120 weight_to_token_ratio=0.053339 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350140812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=285007872 dispatch_meta_bytes=344600 combine_payload_bytes=285007872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285352472 ingress_bytes=319610880 link_bytes=319610880 token_bytes=570360344 weight_movement_bytes=34603008 total_nvlink_bytes=604963352 weight_to_token_ratio=0.060669 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202670011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=262651904 dispatch_meta_bytes=293248 combine_payload_bytes=262651904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262945152 ingress_bytes=297254912 link_bytes=297254912 token_bytes=525597056 weight_movement_bytes=34603008 total_nvlink_bytes=560200064 weight_to_token_ratio=0.065836 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118546303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312664064 dispatch_meta_bytes=342848 combine_payload_bytes=312664064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313006912 ingress_bytes=347267072 link_bytes=347267072 token_bytes=625670976 weight_movement_bytes=34603008 total_nvlink_bytes=660273984 weight_to_token_ratio=0.055305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306738035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86294 payload_rows_remote=74942 dup_rows=12010 dup_rows_remote=9090 dedup_ratio=0.122172 router_tokens_per_expert=[762, 2592, 6012, 1085, 0, 6813, 240, 0, 0, 0, 0, 0, 1, 33, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 337, 8, 254, 2140, 3081, 8192, 0, 209, 0, 0, 0, 0, 0, 5, 0, 9, 0, 1430, 3812, 0, 0, 536, 0, 0, 0, 0, 4124, 0, 987, 0, 0, 0, 0, 0, 45, 1583, 0, 1124, 0, 0, 20] payload_rows_per_dst=[13956, 13626, 11352, 5870, 15594, 9856, 9642, 6398] dup_rows_per_dst=[5636, 68, 2920, 144, 26, 1700, 842, 674] combine_rows_per_src=[13956, 13626, 11352, 5870, 15594, 9856, 9642, 6398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=326864 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297286864 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594246864 weight_movement_bytes=34603008 total_nvlink_bytes=628849872 weight_to_token_ratio=0.058230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=337559552 dispatch_meta_bytes=371768 combine_payload_bytes=337559552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=337931320 ingress_bytes=441368576 link_bytes=441368576 token_bytes=675490872 weight_movement_bytes=103809024 total_nvlink_bytes=779299896 weight_to_token_ratio=0.153679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.660834418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87312 payload_rows_remote=74104 dup_rows=10992 dup_rows_remote=7328 dedup_ratio=0.111816 router_tokens_per_expert=[936, 2468, 5826, 866, 0, 6074, 137, 0, 0, 0, 0, 0, 1, 24, 0, 0, 0, 1, 0, 3824, 0, 0, 0, 342, 3, 199, 2152, 2993, 8191, 0, 189, 1, 0, 0, 2, 0, 2, 0, 94, 0, 1279, 3909, 1, 0, 642, 0, 0, 0, 0, 5404, 0, 982, 0, 0, 0, 0, 0, 44, 1347, 0, 1215, 0, 0, 4] payload_rows_per_dst=[3628, 12152, 8334, 10280, 16382, 11698, 11630, 13208] dup_rows_per_dst=[250, 46, 0, 2482, 196, 3212, 1142, 3664] combine_rows_per_src=[3628, 12152, 8334, 10280, 16382, 11698, 11630, 13208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=306962432 dispatch_meta_bytes=336128 combine_payload_bytes=306962432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307298560 ingress_bytes=341565440 link_bytes=341565440 token_bytes=614260992 weight_movement_bytes=34603008 total_nvlink_bytes=648864000 weight_to_token_ratio=0.056333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285283252e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89448 payload_rows_remote=84294 dup_rows=8856 dup_rows_remote=8762 dedup_ratio=0.090088 router_tokens_per_expert=[0, 0, 0, 7573, 27, 2420, 0, 0, 375, 0, 7168, 0, 0, 0, 0, 130, 0, 1628, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 106, 0, 6245, 0, 0, 0, 0, 598, 0, 0, 0, 0, 711, 43, 8192, 1870, 0, 0, 7033, 0, 3571, 0, 118, 0, 0, 0, 497, 0, 0, 243, 370, 133, 47, 0, 0] payload_rows_per_dst=[4882, 10230, 15548, 12554, 16384, 5154, 10210, 14486] dup_rows_per_dst=[12, 270, 2946, 164, 1196, 94, 3008, 1166] combine_rows_per_src=[4882, 10230, 15548, 12554, 16384, 5154, 10210, 14486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=303529984 dispatch_meta_bytes=325728 combine_payload_bytes=303529984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303855712 ingress_bytes=338132992 link_bytes=338132992 token_bytes=607385696 weight_movement_bytes=34603008 total_nvlink_bytes=641988704 weight_to_token_ratio=0.056970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272367227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86524 payload_rows_remote=71718 dup_rows=11780 dup_rows_remote=10814 dedup_ratio=0.119832 router_tokens_per_expert=[0, 0, 0, 7465, 16, 2494, 0, 0, 408, 0, 7040, 0, 0, 0, 0, 144, 0, 1651, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 70, 0, 6316, 0, 0, 0, 0, 614, 0, 0, 0, 0, 752, 94, 8192, 1971, 0, 0, 6946, 0, 3599, 0, 132, 0, 0, 0, 469, 0, 0, 379, 254, 71, 61, 0, 0] payload_rows_per_dst=[15364, 14806, 3326, 12678, 15828, 13936, 7468, 3118] dup_rows_per_dst=[4586, 966, 0, 98, 1196, 3856, 932, 146] combine_rows_per_src=[15364, 14806, 3326, 12678, 15828, 13936, 7468, 3118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79064 payload_rows_remote=64424 dup_rows=19240 dup_rows_remote=11004 dedup_ratio=0.195719 router_tokens_per_expert=[1048, 2682, 5299, 765, 0, 6412, 306, 0, 3, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 3924, 0, 0, 0, 363, 10, 255, 2008, 2932, 8192, 0, 133, 0, 0, 0, 0, 0, 3, 2, 50, 0, 1648, 3946, 0, 0, 546, 0, 0, 0, 0, 5272, 0, 981, 0, 0, 0, 0, 0, 27, 1425, 0, 911, 0, 0, 3] payload_rows_per_dst=[14640, 10160, 11476, 16384, 110, 10482, 11238, 4574] dup_rows_per_dst=[8236, 6, 2962, 4812, 0, 1798, 1268, 158] combine_rows_per_src=[14640, 10160, 11476, 16384, 110, 10482, 11238, 4574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81880 payload_rows_remote=66466 dup_rows=16424 dup_rows_remote=9302 dedup_ratio=0.167074 router_tokens_per_expert=[0, 0, 0, 7552, 249, 2423, 0, 0, 451, 0, 7281, 0, 0, 0, 0, 165, 0, 1611, 43, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 99, 0, 5879, 0, 0, 1, 1, 1084, 0, 0, 0, 0, 916, 97, 8192, 1593, 0, 0, 6533, 0, 3388, 0, 133, 0, 0, 0, 466, 0, 0, 328, 398, 160, 98, 0, 0] payload_rows_per_dst=[5204, 1232, 15436, 11824, 5596, 13544, 15414, 13630] dup_rows_per_dst=[140, 0, 2976, 154, 820, 3808, 7122, 1404] combine_rows_per_src=[5204, 1232, 15436, 11824, 5596, 13544, 15414, 13630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90168 payload_rows_remote=73786 dup_rows=8136 dup_rows_remote=6164 dedup_ratio=0.082764 router_tokens_per_expert=[0, 0, 0, 7503, 41, 2630, 0, 0, 396, 0, 7088, 0, 0, 0, 0, 119, 0, 1890, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 131, 0, 5833, 0, 0, 0, 0, 986, 0, 0, 0, 0, 583, 88, 8191, 2107, 0, 0, 6956, 0, 3212, 0, 114, 0, 0, 0, 508, 0, 0, 289, 280, 117, 59, 0, 0] payload_rows_per_dst=[5282, 14810, 15522, 11764, 16382, 5432, 6714, 14262] dup_rows_per_dst=[60, 396, 3310, 180, 1972, 124, 954, 1140] combine_rows_per_src=[5282, 14810, 15522, 11764, 16382, 5432, 6714, 14262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90540 payload_rows_remote=78968 dup_rows=7764 dup_rows_remote=7654 dedup_ratio=0.078979 router_tokens_per_expert=[0, 0, 4, 7454, 8, 2581, 0, 2, 431, 0, 7153, 0, 0, 1, 0, 114, 0, 1496, 55, 1, 2, 0, 0, 1, 0, 0, 0, 0, 4, 82, 2, 5753, 1, 0, 1, 0, 852, 2, 1, 0, 0, 666, 165, 8187, 2190, 0, 0, 6693, 0, 3733, 0, 77, 0, 0, 0, 550, 0, 0, 326, 334, 132, 98, 0, 0] payload_rows_per_dst=[7820, 14920, 12236, 11572, 16380, 5858, 7808, 13946] dup_rows_per_dst=[1162, 478, 1990, 110, 1708, 184, 912, 1220] combine_rows_per_src=[7820, 14920, 12236, 11572, 16380, 5858, 7808, 13946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345268224 dispatch_meta_bytes=372224 combine_payload_bytes=345268224 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=345640448 ingress_bytes=518283264 link_bytes=518283264 token_bytes=690908672 weight_movement_bytes=173015040 total_nvlink_bytes=863923712 weight_to_token_ratio=0.250417 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.950258197e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=293756928 dispatch_meta_bytes=330128 combine_payload_bytes=293756928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294087056 ingress_bytes=362962944 link_bytes=362962944 token_bytes=587843984 weight_movement_bytes=69206016 total_nvlink_bytes=657050000 weight_to_token_ratio=0.117729 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365800337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88710 payload_rows_remote=85016 dup_rows=9594 dup_rows_remote=9568 dedup_ratio=0.097595 router_tokens_per_expert=[0, 0, 0, 7590, 225, 2252, 0, 0, 381, 0, 7106, 0, 0, 0, 0, 189, 0, 1781, 75, 4, 0, 0, 0, 0, 0, 1, 0, 0, 5, 116, 0, 5549, 0, 0, 0, 0, 909, 0, 0, 0, 1, 898, 123, 8192, 1916, 0, 0, 6902, 0, 3354, 0, 131, 0, 0, 0, 348, 0, 0, 530, 386, 121, 67, 0, 0] payload_rows_per_dst=[15456, 14836, 3694, 11212, 16384, 5732, 6940, 14456] dup_rows_per_dst=[4678, 516, 26, 130, 1818, 144, 726, 1556] combine_rows_per_src=[15456, 14836, 3694, 11212, 16384, 5732, 6940, 14456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=301712 combine_payload_bytes=263880704 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=264182416 ingress_bytes=436895744 link_bytes=436895744 token_bytes=528063120 weight_movement_bytes=173015040 total_nvlink_bytes=701078160 weight_to_token_ratio=0.327641 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644003512e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=323452928 dispatch_meta_bytes=346488 combine_payload_bytes=323452928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323799416 ingress_bytes=358055936 link_bytes=358055936 token_bytes=647252344 weight_movement_bytes=34603008 total_nvlink_bytes=681855352 weight_to_token_ratio=0.053461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347335661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302227456 dispatch_meta_bytes=319800 combine_payload_bytes=302227456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302547256 ingress_bytes=336830464 link_bytes=336830464 token_bytes=604774712 weight_movement_bytes=34603008 total_nvlink_bytes=639377720 weight_to_token_ratio=0.057216 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267465919e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=303072 combine_payload_bytes=272244736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272547808 ingress_bytes=306847744 link_bytes=306847744 token_bytes=544792544 weight_movement_bytes=34603008 total_nvlink_bytes=579395552 weight_to_token_ratio=0.063516 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154643357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82066 payload_rows_remote=67522 dup_rows=16238 dup_rows_remote=15148 dedup_ratio=0.165181 router_tokens_per_expert=[0, 0, 0, 7629, 144, 2599, 0, 0, 381, 0, 7151, 1, 0, 0, 0, 135, 0, 1630, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 6248, 0, 0, 1, 0, 614, 0, 0, 1, 0, 794, 75, 8180, 2038, 0, 0, 7017, 1, 3084, 0, 128, 0, 0, 0, 384, 0, 0, 272, 359, 126, 43, 0, 0] payload_rows_per_dst=[5298, 1034, 15668, 12576, 1232, 16370, 15344, 14544] dup_rows_per_dst=[188, 0, 2882, 122, 0, 5804, 6152, 1090] combine_rows_per_src=[5298, 1034, 15668, 12576, 1232, 16370, 15344, 14544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=348225536 dispatch_meta_bytes=378336 combine_payload_bytes=348225536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348603872 ingress_bytes=382828544 link_bytes=382828544 token_bytes=696829408 weight_movement_bytes=34603008 total_nvlink_bytes=731432416 weight_to_token_ratio=0.049658 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440552991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86588 payload_rows_remote=74822 dup_rows=11716 dup_rows_remote=11076 dedup_ratio=0.119181 router_tokens_per_expert=[0, 7981, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 7061, 0, 0, 0, 13, 373, 0, 747, 0, 1774, 0, 1, 0, 1154, 2036, 0, 6, 6402, 0, 0, 1406, 0, 0, 0, 239, 0, 5753, 0, 0, 450, 0, 0, 0, 0, 0, 0, 46, 0, 1533, 0, 466, 0, 0, 0, 0, 8183, 1669, 0, 0, 0] payload_rows_per_dst=[15962, 3686, 15340, 6270, 3250, 11766, 13944, 16370] dup_rows_per_dst=[32, 0, 4596, 124, 40, 640, 2950, 3334] combine_rows_per_src=[15962, 3686, 15340, 6270, 3250, 11766, 13944, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=276570112 dispatch_meta_bytes=330680 combine_payload_bytes=276570112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276900792 ingress_bytes=311173120 link_bytes=311173120 token_bytes=553470904 weight_movement_bytes=34603008 total_nvlink_bytes=588073912 weight_to_token_ratio=0.062520 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170919399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86444 payload_rows_remote=71696 dup_rows=11860 dup_rows_remote=8332 dedup_ratio=0.120646 router_tokens_per_expert=[0, 8008, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 6971, 0, 0, 0, 5, 463, 0, 878, 0, 1238, 0, 3, 0, 1182, 2213, 0, 0, 6037, 0, 0, 1161, 0, 0, 0, 224, 0, 6381, 0, 0, 453, 0, 0, 0, 0, 0, 0, 40, 0, 1573, 0, 411, 0, 0, 0, 0, 8185, 1556, 0, 0, 0] payload_rows_per_dst=[16016, 14748, 4908, 14314, 16370, 13058, 3918, 3112] dup_rows_per_dst=[6, 3528, 260, 4556, 2770, 610, 130, 0] combine_rows_per_src=[16016, 14748, 4908, 14314, 16370, 13058, 3918, 3112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86798 payload_rows_remote=75546 dup_rows=11506 dup_rows_remote=9400 dedup_ratio=0.117045 router_tokens_per_expert=[0, 8022, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 6904, 0, 0, 0, 1, 637, 0, 932, 0, 1594, 0, 1, 0, 1321, 2497, 0, 3, 5703, 0, 0, 1465, 0, 0, 0, 208, 0, 5875, 0, 0, 481, 0, 0, 0, 0, 0, 0, 32, 0, 1297, 0, 454, 0, 0, 0, 0, 8169, 1544, 0, 0, 0] payload_rows_per_dst=[16044, 8226, 11708, 6844, 11252, 11984, 12630, 8110] dup_rows_per_dst=[36, 1342, 2848, 800, 2106, 728, 2342, 1304] combine_rows_per_src=[16044, 8226, 11708, 6844, 11252, 11984, 12630, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87414 payload_rows_remote=80920 dup_rows=10890 dup_rows_remote=10654 dedup_ratio=0.110779 router_tokens_per_expert=[0, 7983, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 1, 6963, 2, 1, 0, 10, 442, 0, 1068, 0, 1738, 0, 3, 0, 1311, 2033, 1, 17, 5985, 0, 0, 1357, 0, 0, 0, 234, 0, 5579, 0, 0, 457, 1, 0, 0, 0, 2, 10, 16, 0, 1297, 2, 551, 0, 0, 0, 0, 8183, 1689, 0, 0, 0] payload_rows_per_dst=[15976, 14774, 5824, 6494, 16372, 11450, 13146, 3378] dup_rows_per_dst=[50, 3530, 694, 236, 3176, 624, 2580, 0] combine_rows_per_src=[15976, 14774, 5824, 6494, 16372, 11450, 13146, 3378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72248 payload_rows_remote=56232 dup_rows=26056 dup_rows_remote=13900 dedup_ratio=0.265055 router_tokens_per_expert=[0, 0, 0, 7656, 24, 2580, 0, 0, 425, 0, 6960, 0, 0, 0, 0, 160, 0, 1335, 37, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 130, 0, 5650, 1, 0, 9, 0, 690, 3, 0, 1, 2, 834, 323, 8186, 2144, 0, 0, 6623, 0, 3762, 0, 126, 0, 0, 0, 412, 0, 0, 508, 354, 145, 63, 0, 0] payload_rows_per_dst=[16016, 15496, 2738, 12182, 1402, 14568, 7762, 2084] dup_rows_per_dst=[12156, 8314, 6, 984, 6, 3696, 838, 56] combine_rows_per_src=[16016, 15496, 2738, 12182, 1402, 14568, 7762, 2084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80180 payload_rows_remote=66292 dup_rows=18124 dup_rows_remote=14726 dedup_ratio=0.184367 router_tokens_per_expert=[0, 7838, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1877, 0, 6295, 0, 1, 0, 10, 412, 0, 1221, 2, 2185, 0, 3, 0, 1430, 2312, 0, 11, 6581, 0, 0, 671, 1, 0, 0, 318, 0, 5570, 0, 0, 437, 0, 0, 0, 0, 0, 0, 49, 0, 1480, 0, 533, 0, 0, 0, 0, 8170, 1732, 0, 0, 0] payload_rows_per_dst=[1180, 9988, 14964, 10444, 1966, 11406, 13888, 16344] dup_rows_per_dst=[10, 1726, 5288, 2644, 14, 608, 3398, 4436] combine_rows_per_src=[1180, 9988, 14964, 10444, 1966, 11406, 13888, 16344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306470912 dispatch_meta_bytes=343592 combine_payload_bytes=306470912 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=306814504 ingress_bytes=341073920 link_bytes=341073920 token_bytes=613285416 weight_movement_bytes=34603008 total_nvlink_bytes=647888424 weight_to_token_ratio=0.056422 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283433702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=293666816 dispatch_meta_bytes=320112 combine_payload_bytes=293666816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293986928 ingress_bytes=362872832 link_bytes=362872832 token_bytes=587653744 weight_movement_bytes=69206016 total_nvlink_bytes=656859760 weight_to_token_ratio=0.117767 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365461253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86568 payload_rows_remote=80866 dup_rows=11736 dup_rows_remote=11408 dedup_ratio=0.119385 router_tokens_per_expert=[0, 7900, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 6625, 0, 0, 0, 7, 370, 0, 1253, 0, 1385, 0, 1, 0, 1325, 2080, 1, 5, 6601, 0, 0, 1088, 1, 0, 0, 272, 0, 5587, 0, 0, 458, 0, 0, 0, 0, 0, 0, 111, 0, 1722, 0, 762, 0, 0, 0, 0, 8171, 1688, 0, 0, 0] payload_rows_per_dst=[15806, 13938, 5702, 13720, 16348, 11574, 6104, 3376] dup_rows_per_dst=[70, 2714, 328, 4740, 2716, 516, 652, 0] combine_rows_per_src=[15806, 13938, 5702, 13720, 16348, 11574, 6104, 3376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309436416 dispatch_meta_bytes=339784 combine_payload_bytes=309436416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309776200 ingress_bytes=344039424 link_bytes=344039424 token_bytes=619212616 weight_movement_bytes=34603008 total_nvlink_bytes=653815624 weight_to_token_ratio=0.055882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=230326272 dispatch_meta_bytes=280528 combine_payload_bytes=230326272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=230606800 ingress_bytes=299532288 link_bytes=299532288 token_bytes=460933072 weight_movement_bytes=69206016 total_nvlink_bytes=530139088 weight_to_token_ratio=0.150143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127115885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70532 payload_rows_remote=54174 dup_rows=27772 dup_rows_remote=8806 dedup_ratio=0.282511 router_tokens_per_expert=[0, 7950, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1900, 0, 6942, 0, 0, 0, 13, 478, 0, 698, 0, 1668, 0, 1, 0, 1293, 1893, 0, 7, 6584, 0, 0, 953, 0, 0, 0, 268, 1, 5979, 0, 0, 443, 0, 0, 0, 0, 0, 0, 81, 0, 1563, 0, 710, 0, 0, 0, 0, 8163, 1548, 0, 0, 1] payload_rows_per_dst=[30, 3800, 15184, 6300, 2416, 12272, 14172, 16358] dup_rows_per_dst=[0, 0, 4414, 88, 28, 572, 3704, 18966] combine_rows_per_src=[30, 3800, 15184, 6300, 2416, 12272, 14172, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=331448320 dispatch_meta_bytes=366296 combine_payload_bytes=331448320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331814616 ingress_bytes=400654336 link_bytes=400654336 token_bytes=663262936 weight_movement_bytes=69206016 total_nvlink_bytes=732468952 weight_to_token_ratio=0.104342 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.507630011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=271532032 dispatch_meta_bytes=324072 combine_payload_bytes=271532032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271856104 ingress_bytes=306135040 link_bytes=306135040 token_bytes=543388136 weight_movement_bytes=34603008 total_nvlink_bytes=577991144 weight_to_token_ratio=0.063680 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151961510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=331227136 dispatch_meta_bytes=369096 combine_payload_bytes=331227136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331596232 ingress_bytes=365830144 link_bytes=365830144 token_bytes=662823368 weight_movement_bytes=34603008 total_nvlink_bytes=697426376 weight_to_token_ratio=0.052205 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376589380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=221896704 dispatch_meta_bytes=251920 combine_payload_bytes=221896704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222148624 ingress_bytes=291102720 link_bytes=291102720 token_bytes=444045328 weight_movement_bytes=69206016 total_nvlink_bytes=513251344 weight_to_token_ratio=0.155853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.095396100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75142 payload_rows_remote=64942 dup_rows=23162 dup_rows_remote=21380 dedup_ratio=0.235616 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 2491, 0, 4623, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2096, 760, 0, 0, 0, 24, 0, 0, 3666, 0, 6101, 8192, 0, 0, 480, 830, 0, 0, 0, 0, 0, 0, 0, 7811, 0, 0, 0, 165, 0, 215, 0, 0, 1043, 0] payload_rows_per_dst=[16384, 10200, 16384, 13930, 7372, 3188, 5000, 2684] dup_rows_per_dst=[0, 1782, 17094, 3988, 8, 128, 0, 162] combine_rows_per_src=[16384, 10200, 16384, 13930, 7372, 3188, 5000, 2684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87926 payload_rows_remote=75232 dup_rows=10378 dup_rows_remote=8506 dedup_ratio=0.105570 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2927, 0, 0, 1, 0, 2587, 0, 4063, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1820, 863, 0, 0, 0, 23, 0, 0, 3306, 2, 6014, 8192, 0, 0, 396, 873, 0, 0, 0, 1, 0, 0, 0, 7759, 0, 0, 0, 257, 0, 515, 0, 0, 1356, 0] payload_rows_per_dst=[16384, 10022, 0, 5234, 11690, 12694, 15518, 16384] dup_rows_per_dst=[0, 1008, 0, 142, 3098, 1872, 2, 4256] combine_rows_per_src=[16384, 10022, 0, 5234, 11690, 12694, 15518, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87852 payload_rows_remote=74666 dup_rows=10452 dup_rows_remote=6666 dedup_ratio=0.106323 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 2, 0, 2679, 0, 3920, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1805, 732, 0, 0, 0, 37, 0, 0, 4528, 1, 6247, 8191, 0, 0, 582, 921, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 237, 0, 216, 0, 0, 1071, 0] payload_rows_per_dst=[16382, 8732, 0, 10146, 13186, 7444, 15578, 16384] dup_rows_per_dst=[0, 628, 0, 2030, 3786, 960, 0, 3048] combine_rows_per_src=[16382, 8732, 0, 10146, 13186, 7444, 15578, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85028 payload_rows_remote=69068 dup_rows=13276 dup_rows_remote=12150 dedup_ratio=0.135050 router_tokens_per_expert=[0, 7957, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 0, 6700, 0, 0, 0, 11, 330, 0, 1001, 0, 1781, 0, 1, 0, 1273, 2523, 0, 3, 6565, 0, 1, 1273, 0, 0, 0, 286, 0, 5328, 0, 0, 492, 0, 0, 0, 0, 0, 1, 79, 0, 1619, 0, 674, 0, 0, 0, 1, 8180, 1607, 0, 0, 0] payload_rows_per_dst=[15960, 14050, 5796, 15128, 16360, 9960, 4560, 3214] dup_rows_per_dst=[1126, 2254, 450, 5602, 3120, 536, 186, 2] combine_rows_per_src=[15960, 14050, 5796, 15128, 16360, 9960, 4560, 3214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=345288 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266347720 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532350152 weight_movement_bytes=34603008 total_nvlink_bytes=566953160 weight_to_token_ratio=0.065000 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85200 payload_rows_remote=71798 dup_rows=13104 dup_rows_remote=9492 dedup_ratio=0.133301 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 0, 0, 3, 0, 2496, 0, 4914, 8189, 1, 2, 0, 0, 0, 0, 0, 1, 5, 0, 0, 1, 1968, 868, 0, 1, 0, 16, 0, 0, 3458, 0, 5664, 8186, 0, 0, 570, 957, 3, 0, 0, 0, 0, 0, 2, 7811, 0, 3, 0, 285, 2, 382, 2, 1, 957, 1] payload_rows_per_dst=[16376, 9162, 10068, 13402, 9348, 3008, 15628, 8208] dup_rows_per_dst=[0, 638, 5788, 3612, 1646, 52, 4, 1364] combine_rows_per_src=[16376, 9162, 10068, 13402, 9348, 3008, 15628, 8208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75574 payload_rows_remote=59976 dup_rows=22730 dup_rows_remote=22726 dedup_ratio=0.231222 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 0, 1, 0, 2686, 0, 3992, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1768, 1031, 0, 0, 0, 9, 0, 0, 4287, 0, 6044, 8189, 1, 0, 374, 1129, 0, 0, 0, 2, 0, 0, 0, 7799, 0, 0, 0, 222, 0, 256, 0, 0, 982, 0] payload_rows_per_dst=[14, 9010, 2, 5306, 12876, 16384, 15598, 16384] dup_rows_per_dst=[0, 736, 0, 296, 3700, 15076, 4, 2918] combine_rows_per_src=[14, 9010, 2, 5306, 12876, 16384, 15598, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=308150272 dispatch_meta_bytes=334952 combine_payload_bytes=308150272 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=308485224 ingress_bytes=411959296 link_bytes=411959296 token_bytes=616635496 weight_movement_bytes=103809024 total_nvlink_bytes=720444520 weight_to_token_ratio=0.168347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.550169665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84288 payload_rows_remote=67904 dup_rows=14016 dup_rows_remote=5286 dedup_ratio=0.142578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 1, 0, 2733, 0, 4365, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2020, 983, 0, 0, 0, 11, 0, 0, 3359, 2, 5581, 8192, 0, 0, 416, 787, 0, 0, 0, 1, 0, 0, 0, 7730, 0, 0, 0, 271, 0, 456, 0, 0, 1028, 0] payload_rows_per_dst=[16384, 10356, 16384, 13368, 6734, 2398, 15460, 3204] dup_rows_per_dst=[0, 1146, 8730, 3814, 10, 8, 2, 306] combine_rows_per_src=[16384, 10356, 16384, 13368, 6734, 2398, 15460, 3204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=305831936 dispatch_meta_bytes=325328 combine_payload_bytes=305831936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306157264 ingress_bytes=340434944 link_bytes=340434944 token_bytes=611989200 weight_movement_bytes=34603008 total_nvlink_bytes=646592208 weight_to_token_ratio=0.056542 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281029287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=282902528 dispatch_meta_bytes=324872 combine_payload_bytes=282902528 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=283227400 ingress_bytes=421314560 link_bytes=421314560 token_bytes=566129928 weight_movement_bytes=138412032 total_nvlink_bytes=704541960 weight_to_token_ratio=0.244488 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585372771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294084608 dispatch_meta_bytes=325160 combine_payload_bytes=294084608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294409768 ingress_bytes=328687616 link_bytes=328687616 token_bytes=588494376 weight_movement_bytes=34603008 total_nvlink_bytes=623097384 weight_to_token_ratio=0.058799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236825037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75328 payload_rows_remote=58944 dup_rows=22976 dup_rows_remote=20166 dedup_ratio=0.233724 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 0, 0, 4, 0, 2473, 0, 4003, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1929, 820, 0, 0, 0, 35, 0, 0, 4235, 3, 6554, 8190, 0, 0, 411, 944, 0, 0, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 163, 0, 371, 0, 0, 871, 0] payload_rows_per_dst=[0, 8758, 2, 5384, 12910, 16384, 15506, 16384] dup_rows_per_dst=[0, 592, 0, 118, 3642, 15814, 0, 2810] combine_rows_per_src=[0, 8758, 2, 5384, 12910, 16384, 15506, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=245661696 dispatch_meta_bytes=330808 combine_payload_bytes=245661696 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=245992504 ingress_bytes=349470720 link_bytes=349470720 token_bytes=491654200 weight_movement_bytes=103809024 total_nvlink_bytes=595463224 weight_to_token_ratio=0.211142 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315030185e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=278134784 dispatch_meta_bytes=292760 combine_payload_bytes=278134784 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278427544 ingress_bytes=381943808 link_bytes=381943808 token_bytes=556562328 weight_movement_bytes=103809024 total_nvlink_bytes=660371352 weight_to_token_ratio=0.186518 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437223801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82480 payload_rows_remote=70220 dup_rows=15824 dup_rows_remote=11498 dedup_ratio=0.160970 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 0, 0, 73, 69, 0, 3534, 3152, 0, 0, 0, 151, 0, 0, 5413, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 4940, 78, 0, 0, 7660, 0, 0, 0, 0, 1462, 0, 0] payload_rows_per_dst=[15320, 12260, 11706, 10870, 16140, 8300, 4960, 2924] dup_rows_per_dst=[0, 4326, 1950, 258, 9022, 0, 268, 0] combine_rows_per_src=[15320, 12260, 11706, 10870, 16140, 8300, 4960, 2924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91472 payload_rows_remote=75872 dup_rows=6832 dup_rows_remote=6832 dedup_ratio=0.069499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4676, 0, 0, 133, 101, 0, 3401, 3465, 0, 0, 0, 165, 0, 0, 5394, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 4907, 76, 0, 0, 7674, 0, 0, 0, 0, 1363, 0, 0] payload_rows_per_dst=[15348, 9352, 12038, 10872, 0, 15600, 12168, 16094] dup_rows_per_dst=[0, 0, 2162, 246, 0, 0, 1698, 2726] combine_rows_per_src=[15348, 9352, 12038, 10872, 0, 15600, 12168, 16094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=241434624 dispatch_meta_bytes=316440 combine_payload_bytes=241434624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241751064 ingress_bytes=276037632 link_bytes=276037632 token_bytes=483185688 weight_movement_bytes=34603008 total_nvlink_bytes=517788696 weight_to_token_ratio=0.071614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.038707386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94052 payload_rows_remote=79708 dup_rows=4252 dup_rows_remote=4252 dedup_ratio=0.043254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5141, 0, 0, 73, 51, 0, 3635, 3225, 0, 0, 0, 123, 0, 0, 4954, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 7559, 0, 0, 1, 0, 0, 0, 0, 1990, 0, 5028, 66, 0, 0, 7630, 0, 0, 0, 0, 1587, 0, 0] payload_rows_per_dst=[15260, 10282, 12022, 9984, 14344, 15118, 12464, 4578] dup_rows_per_dst=[0, 0, 1946, 170, 0, 2, 1704, 430] combine_rows_per_src=[15260, 10282, 12022, 9984, 14344, 15118, 12464, 4578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=287621120 dispatch_meta_bytes=326872 combine_payload_bytes=287621120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287947992 ingress_bytes=322224128 link_bytes=322224128 token_bytes=575569112 weight_movement_bytes=34603008 total_nvlink_bytes=610172120 weight_to_token_ratio=0.060120 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212503452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77440 payload_rows_remote=61060 dup_rows=20864 dup_rows_remote=20864 dedup_ratio=0.212240 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 0, 2, 0, 2561, 0, 3734, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1902, 804, 0, 0, 0, 16, 0, 0, 4133, 0, 6057, 8190, 0, 0, 561, 886, 0, 0, 0, 2, 0, 0, 0, 7796, 0, 0, 0, 265, 0, 359, 0, 0, 1280, 0] payload_rows_per_dst=[16380, 16372, 16384, 13770, 8290, 2842, 4, 3398] dup_rows_per_dst=[0, 9162, 7468, 3764, 8, 52, 0, 410] combine_rows_per_src=[16380, 16372, 16384, 13770, 8290, 2842, 4, 3398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=310771712 dispatch_meta_bytes=330816 combine_payload_bytes=310771712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=311102528 ingress_bytes=414580736 link_bytes=414580736 token_bytes=621874240 weight_movement_bytes=103809024 total_nvlink_bytes=725683264 weight_to_token_ratio=0.166929 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560033933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80590 payload_rows_remote=67972 dup_rows=17714 dup_rows_remote=15940 dedup_ratio=0.180196 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4848, 0, 0, 150, 68, 1, 3182, 3409, 6, 0, 0, 163, 0, 0, 5154, 0, 0, 1, 0, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 2, 0, 0, 1, 0, 1840, 0, 5285, 70, 0, 0, 7698, 0, 0, 0, 0, 1538, 0, 0] payload_rows_per_dst=[6468, 9696, 7194, 12944, 0, 15436, 12618, 16234] dup_rows_per_dst=[0, 0, 504, 3626, 0, 0, 1774, 11810] combine_rows_per_src=[6468, 9696, 7194, 12944, 0, 15436, 12618, 16234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94362 payload_rows_remote=84304 dup_rows=3942 dup_rows_remote=3750 dedup_ratio=0.040100 router_tokens_per_expert=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5060, 3, 0, 113, 121, 3, 3234, 3660, 0, 0, 1, 169, 0, 1, 4947, 1, 6, 1, 0, 0, 0, 8094, 1, 0, 2, 2, 0, 0, 0, 7569, 0, 0, 0, 0, 0, 0, 0, 1975, 1, 4872, 76, 0, 0, 7655, 1, 0, 0, 1, 1579, 0, 0] payload_rows_per_dst=[15316, 10128, 12268, 10058, 16192, 15138, 12102, 3160] dup_rows_per_dst=[0, 0, 1996, 192, 6, 0, 1746, 2] combine_rows_per_src=[15316, 10128, 12268, 10058, 16192, 15138, 12102, 3160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=326483968 dispatch_meta_bytes=335840 combine_payload_bytes=326483968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326819808 ingress_bytes=395689984 link_bytes=395689984 token_bytes=653303776 weight_movement_bytes=69206016 total_nvlink_bytes=722509792 weight_to_token_ratio=0.105932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488949554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=250101760 dispatch_meta_bytes=327696 combine_payload_bytes=250101760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250429456 ingress_bytes=284704768 link_bytes=284704768 token_bytes=500531216 weight_movement_bytes=34603008 total_nvlink_bytes=535134224 weight_to_token_ratio=0.069133 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071321121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93582 payload_rows_remote=81730 dup_rows=4722 dup_rows_remote=2232 dedup_ratio=0.048035 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4682, 0, 0, 153, 98, 0, 3505, 3415, 0, 0, 0, 256, 0, 0, 5457, 0, 1, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 7675, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 5152, 92, 0, 0, 7618, 0, 0, 0, 0, 1100, 0, 0] payload_rows_per_dst=[15236, 9364, 11852, 11024, 16000, 15350, 12556, 2200] dup_rows_per_dst=[0, 0, 2490, 404, 0, 0, 1828, 0] combine_rows_per_src=[15236, 9364, 11852, 11024, 16000, 15350, 12556, 2200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278413312 dispatch_meta_bytes=335648 combine_payload_bytes=278413312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278748960 ingress_bytes=347619328 link_bytes=347619328 token_bytes=557162272 weight_movement_bytes=69206016 total_nvlink_bytes=626368288 weight_to_token_ratio=0.124212 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308063546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70188 payload_rows_remote=53804 dup_rows=28116 dup_rows_remote=9266 dedup_ratio=0.286011 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 0, 0, 172, 76, 0, 3176, 3593, 1, 0, 1, 204, 0, 0, 4976, 0, 2, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 1, 0, 0, 0, 0, 2125, 0, 4912, 89, 0, 0, 7741, 0, 0, 0, 0, 1803, 0, 0] payload_rows_per_dst=[0, 8722, 5028, 13426, 0, 13360, 13268, 16384] dup_rows_per_dst=[0, 0, 152, 4124, 0, 0, 4990, 18850] combine_rows_per_src=[0, 8722, 5028, 13426, 0, 13360, 13268, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=345309184 dispatch_meta_bytes=352216 combine_payload_bytes=345309184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345661400 ingress_bytes=379912192 link_bytes=379912192 token_bytes=690970584 weight_movement_bytes=34603008 total_nvlink_bytes=725573592 weight_to_token_ratio=0.050079 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429578993e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88944 payload_rows_remote=78156 dup_rows=9360 dup_rows_remote=9210 dedup_ratio=0.095215 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 314, 200, 14, 0, 0, 0, 5353, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 7166, 0, 0, 7258, 0, 0, 0, 0, 0, 0, 0, 534, 112, 0, 0, 3544, 0, 0, 3446, 7162, 207, 0, 0, 192, 0, 0, 0, 0, 0, 4771, 0, 285, 0, 0, 0, 0, 0, 561, 0, 0, 24] payload_rows_per_dst=[16022, 10788, 14706, 11784, 8142, 14872, 9690, 2940] dup_rows_per_dst=[1024, 150, 2154, 0, 238, 5238, 422, 134] combine_rows_per_src=[16022, 10788, 14706, 11784, 8142, 14872, 9690, 2940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89788 payload_rows_remote=82010 dup_rows=8516 dup_rows_remote=8010 dedup_ratio=0.086629 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 292, 121, 11, 0, 0, 0, 5214, 0, 0, 0, 8041, 0, 29, 0, 0, 0, 0, 6465, 0, 0, 7405, 0, 0, 0, 0, 0, 0, 0, 818, 135, 0, 0, 3069, 0, 0, 3600, 7121, 380, 0, 0, 162, 0, 0, 0, 0, 0, 5349, 0, 324, 0, 0, 0, 0, 0, 608, 0, 0, 8] payload_rows_per_dst=[9752, 10444, 6690, 14810, 15212, 7778, 10812, 14290] dup_rows_per_dst=[496, 6, 28, 0, 5762, 506, 534, 1184] combine_rows_per_src=[9752, 10444, 6690, 14810, 15212, 7778, 10812, 14290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=334766080 dispatch_meta_bytes=335848 combine_payload_bytes=334766080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335101928 ingress_bytes=403972096 link_bytes=403972096 token_bytes=669868008 weight_movement_bytes=69206016 total_nvlink_bytes=739074024 weight_to_token_ratio=0.103313 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520114475e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=220381184 dispatch_meta_bytes=252280 combine_payload_bytes=220381184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=220633464 ingress_bytes=289587200 link_bytes=289587200 token_bytes=441014648 weight_movement_bytes=69206016 total_nvlink_bytes=510220664 weight_to_token_ratio=0.156925 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089693320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88910 payload_rows_remote=74574 dup_rows=9394 dup_rows_remote=3056 dedup_ratio=0.095561 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 241, 241, 6, 0, 0, 0, 5587, 0, 0, 0, 8047, 0, 1, 0, 0, 0, 0, 7043, 0, 0, 7166, 0, 0, 0, 0, 1, 0, 0, 554, 110, 0, 0, 3699, 0, 0, 3601, 7011, 205, 0, 0, 144, 0, 0, 0, 0, 0, 4486, 0, 301, 0, 0, 0, 0, 0, 684, 0, 0, 24] payload_rows_per_dst=[16118, 11178, 2140, 14332, 14336, 7558, 9128, 14120] dup_rows_per_dst=[940, 8, 0, 2, 6338, 342, 446, 1318] combine_rows_per_src=[16118, 11178, 2140, 14332, 14336, 7558, 9128, 14120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=320126976 dispatch_meta_bytes=349464 combine_payload_bytes=320126976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320476440 ingress_bytes=354729984 link_bytes=354729984 token_bytes=640603416 weight_movement_bytes=34603008 total_nvlink_bytes=675206424 weight_to_token_ratio=0.054016 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334820372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=335912960 dispatch_meta_bytes=360080 combine_payload_bytes=335912960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=336273040 ingress_bytes=405118976 link_bytes=405118976 token_bytes=672186000 weight_movement_bytes=69206016 total_nvlink_bytes=741392016 weight_to_token_ratio=0.102957 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75982 payload_rows_remote=60626 dup_rows=22322 dup_rows_remote=22322 dedup_ratio=0.227071 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4509, 0, 0, 144, 164, 0, 3705, 3476, 9, 0, 0, 182, 0, 0, 4871, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 1, 0, 0, 0, 0, 7575, 0, 0, 0, 0, 0, 0, 0, 2016, 0, 5299, 64, 0, 0, 7678, 0, 0, 0, 0, 1456, 0, 0] payload_rows_per_dst=[15356, 15308, 12452, 9860, 16062, 0, 4032, 2912] dup_rows_per_dst=[0, 8866, 2544, 246, 10538, 0, 128, 0] combine_rows_per_src=[15356, 15308, 12452, 9860, 16062, 0, 4032, 2912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89238 payload_rows_remote=77948 dup_rows=9066 dup_rows_remote=8342 dedup_ratio=0.092224 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 309, 216, 24, 0, 0, 0, 4700, 0, 0, 0, 8037, 0, 2, 0, 0, 0, 0, 6758, 0, 1, 7172, 0, 0, 0, 0, 0, 0, 0, 746, 139, 0, 0, 3156, 0, 0, 3556, 7111, 264, 0, 0, 131, 0, 0, 0, 0, 0, 5468, 0, 539, 0, 0, 0, 0, 0, 808, 0, 0, 14] payload_rows_per_dst=[1048, 9418, 16078, 14344, 12448, 10288, 11290, 14324] dup_rows_per_dst=[4, 30, 0, 2, 3974, 2790, 724, 1542] combine_rows_per_src=[1048, 9418, 16078, 14344, 12448, 10288, 11290, 14324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94222 payload_rows_remote=79702 dup_rows=4082 dup_rows_remote=4082 dedup_ratio=0.041524 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 332, 306, 13, 0, 1, 3, 5302, 0, 0, 2, 8004, 0, 33, 2, 0, 0, 0, 6486, 0, 0, 7260, 0, 0, 0, 0, 0, 0, 0, 823, 113, 1, 0, 3372, 0, 0, 2961, 7172, 312, 1, 0, 143, 0, 0, 0, 0, 0, 5354, 0, 435, 0, 0, 3, 0, 0, 707, 0, 2, 8] payload_rows_per_dst=[16080, 10624, 12976, 14520, 8170, 6418, 11028, 14406] dup_rows_per_dst=[1206, 18, 66, 0, 448, 416, 550, 1378] combine_rows_per_src=[16080, 10624, 12976, 14520, 8170, 6418, 11028, 14406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=305455104 dispatch_meta_bytes=310520 combine_payload_bytes=305455104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305765624 ingress_bytes=340058112 link_bytes=340058112 token_bytes=611220728 weight_movement_bytes=34603008 total_nvlink_bytes=645823736 weight_to_token_ratio=0.056613 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279611299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=248324096 dispatch_meta_bytes=331792 combine_payload_bytes=248324096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248655888 ingress_bytes=282927104 link_bytes=282927104 token_bytes=496979984 weight_movement_bytes=34603008 total_nvlink_bytes=531582992 weight_to_token_ratio=0.069627 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064631915e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94462 payload_rows_remote=81416 dup_rows=3842 dup_rows_remote=3810 dedup_ratio=0.039083 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 365, 155, 9, 0, 0, 0, 4778, 0, 0, 0, 7978, 0, 16, 0, 0, 0, 0, 6523, 0, 0, 7155, 0, 0, 0, 0, 0, 0, 0, 829, 91, 0, 0, 3616, 1, 0, 3327, 7220, 364, 0, 0, 235, 0, 0, 0, 0, 0, 5657, 0, 286, 0, 0, 0, 0, 0, 546, 0, 0, 1] payload_rows_per_dst=[15958, 9568, 13046, 14310, 8426, 7262, 11434, 14458] dup_rows_per_dst=[1038, 6, 32, 0, 648, 590, 452, 1076] combine_rows_per_src=[15958, 9568, 13046, 14310, 8426, 7262, 11434, 14458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=319275008 dispatch_meta_bytes=345160 combine_payload_bytes=319275008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319620168 ingress_bytes=353878016 link_bytes=353878016 token_bytes=638895176 weight_movement_bytes=34603008 total_nvlink_bytes=673498184 weight_to_token_ratio=0.054161 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331614485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87340 payload_rows_remote=73150 dup_rows=10964 dup_rows_remote=9596 dedup_ratio=0.111532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 259, 177, 19, 0, 0, 0, 5073, 0, 0, 0, 8013, 0, 17, 0, 0, 0, 0, 6983, 0, 0, 7100, 0, 0, 0, 0, 0, 0, 0, 604, 136, 0, 0, 3240, 0, 0, 3895, 7057, 205, 0, 0, 214, 0, 0, 0, 0, 0, 5130, 0, 308, 0, 0, 0, 0, 0, 705, 0, 0, 17] payload_rows_per_dst=[868, 10170, 16026, 14200, 7666, 9862, 14358, 14190] dup_rows_per_dst=[4, 14, 34, 0, 294, 2358, 6892, 1368] combine_rows_per_src=[868, 10170, 16026, 14200, 7666, 9862, 14358, 14190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=326459392 dispatch_meta_bytes=335136 combine_payload_bytes=326459392 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=326794528 ingress_bytes=395665408 link_bytes=395665408 token_bytes=653253920 weight_movement_bytes=69206016 total_nvlink_bytes=722459936 weight_to_token_ratio=0.105940 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488857077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92268 payload_rows_remote=85514 dup_rows=6036 dup_rows_remote=6034 dedup_ratio=0.061401 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6368, 0, 0, 0, 8192, 0, 5115, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5658, 0, 1, 3377, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7606, 0, 0, 0, 0, 4469, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 400, 15288, 13040, 11316, 6754, 15406, 13682] dup_rows_per_dst=[0, 0, 932, 208, 0, 2, 1360, 3534] combine_rows_per_src=[16382, 400, 15288, 13040, 11316, 6754, 15406, 13682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92958 payload_rows_remote=82048 dup_rows=5346 dup_rows_remote=5346 dedup_ratio=0.054382 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6740, 0, 0, 0, 8192, 0, 5455, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5174, 0, 0, 3897, 0, 0, 7219, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 1, 4211, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10910, 14636, 13670, 10348, 12590, 5998, 8422] dup_rows_per_dst=[0, 0, 964, 154, 0, 4226, 2, 0] combine_rows_per_src=[16384, 10910, 14636, 13670, 10348, 12590, 5998, 8422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=333479936 dispatch_meta_bytes=340904 combine_payload_bytes=333479936 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=333820840 ingress_bytes=506494976 link_bytes=506494976 token_bytes=667300776 weight_movement_bytes=173015040 total_nvlink_bytes=840315816 weight_to_token_ratio=0.259276 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.905899818e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299622400 dispatch_meta_bytes=330984 combine_payload_bytes=299622400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299953384 ingress_bytes=334225408 link_bytes=334225408 token_bytes=599575784 weight_movement_bytes=34603008 total_nvlink_bytes=634178792 weight_to_token_ratio=0.057712 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=350265344 dispatch_meta_bytes=366192 combine_payload_bytes=350265344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350631536 ingress_bytes=419471360 link_bytes=419471360 token_bytes=700896880 weight_movement_bytes=69206016 total_nvlink_bytes=770102896 weight_to_token_ratio=0.098739 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578436957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96710 payload_rows_remote=86350 dup_rows=1594 dup_rows_remote=1594 dedup_ratio=0.016215 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6848, 0, 0, 0, 8192, 0, 5493, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5180, 0, 0, 3607, 0, 0, 7259, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10986, 14738, 13962, 10360, 7214, 14956, 8110] dup_rows_per_dst=[0, 0, 778, 816, 0, 0, 0, 0] combine_rows_per_src=[16384, 10986, 14738, 13962, 10360, 7214, 14956, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=336068608 dispatch_meta_bytes=349576 combine_payload_bytes=336068608 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=336418184 ingress_bytes=474480640 link_bytes=474480640 token_bytes=672486792 weight_movement_bytes=138412032 total_nvlink_bytes=810898824 weight_to_token_ratio=0.205821 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.785432449e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80372 payload_rows_remote=64270 dup_rows=17932 dup_rows_remote=16288 dedup_ratio=0.182414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 437, 407, 19, 0, 0, 0, 5559, 0, 0, 0, 8029, 0, 10, 0, 0, 0, 0, 7154, 0, 0, 7363, 0, 0, 0, 0, 0, 0, 0, 479, 111, 0, 0, 2511, 0, 0, 3200, 7184, 464, 0, 0, 165, 0, 0, 0, 0, 0, 5012, 0, 260, 0, 0, 0, 0, 0, 782, 0, 0, 6] payload_rows_per_dst=[16102, 16212, 14308, 0, 6072, 15966, 10136, 1576] dup_rows_per_dst=[1644, 9670, 20, 0, 130, 6060, 408, 0] combine_rows_per_src=[16102, 16212, 14308, 0, 6072, 15966, 10136, 1576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86978 payload_rows_remote=71752 dup_rows=11326 dup_rows_remote=11326 dedup_ratio=0.115214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6343, 0, 0, 0, 8192, 0, 4860, 0, 520, 0, 0, 0, 6, 0, 0, 0, 0, 519, 0, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 5057, 0, 0, 3807, 0, 0, 7511, 1, 0, 0, 0, 0, 0, 7610, 0, 0, 0, 3, 4711, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 2696, 13220, 10114, 14862, 15226, 14476] dup_rows_per_dst=[0, 0, 138, 528, 0, 5994, 0, 4666] combine_rows_per_src=[0, 16384, 2696, 13220, 10114, 14862, 15226, 14476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97326 payload_rows_remote=84212 dup_rows=978 dup_rows_remote=854 dedup_ratio=0.009949 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 1, 7, 6352, 0, 0, 0, 8188, 0, 5847, 0, 539, 0, 0, 0, 4, 0, 0, 0, 0, 254, 0, 0, 1, 12, 0, 2, 0, 0, 0, 2, 0, 5453, 0, 0, 3939, 0, 0, 7144, 0, 0, 1, 0, 0, 0, 7571, 3, 0, 0, 5, 3823, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 11700, 14540, 13114, 10910, 7880, 15152, 7650] dup_rows_per_dst=[2, 8, 834, 124, 4, 0, 6, 0] combine_rows_per_src=[16380, 11700, 14540, 13114, 10910, 7880, 15152, 7650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=353689600 dispatch_meta_bytes=351776 combine_payload_bytes=353689600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=354041376 ingress_bytes=388292608 link_bytes=388292608 token_bytes=707730976 weight_movement_bytes=34603008 total_nvlink_bytes=742333984 weight_to_token_ratio=0.048893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.461113824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263249920 dispatch_meta_bytes=322232 combine_payload_bytes=263249920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263572152 ingress_bytes=297852928 link_bytes=297852928 token_bytes=526822072 weight_movement_bytes=34603008 total_nvlink_bytes=561425080 weight_to_token_ratio=0.065683 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120796589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96618 payload_rows_remote=81314 dup_rows=1686 dup_rows_remote=732 dedup_ratio=0.017151 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6484, 0, 0, 0, 8192, 0, 5468, 0, 539, 0, 0, 0, 1, 0, 0, 0, 0, 534, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5378, 0, 0, 2562, 0, 0, 7589, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 1, 4916, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10936, 15304, 13308, 10756, 5124, 14974, 9832] dup_rows_per_dst=[0, 0, 954, 730, 0, 0, 2, 0] combine_rows_per_src=[16384, 10936, 15304, 13308, 10756, 5124, 14974, 9832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85940 payload_rows_remote=71194 dup_rows=12364 dup_rows_remote=7724 dedup_ratio=0.125773 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5235, 0, 0, 0, 8191, 0, 5652, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5475, 0, 0, 4023, 0, 0, 7798, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 0, 0, 4036, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[0, 16382, 1266, 11356, 10950, 16018, 15222, 14746] dup_rows_per_dst=[0, 0, 0, 100, 0, 7624, 0, 4640] combine_rows_per_src=[0, 16382, 1266, 11356, 10950, 16018, 15222, 14746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=293896192 dispatch_meta_bytes=332312 combine_payload_bytes=293896192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=294228504 ingress_bytes=397705216 link_bytes=397705216 token_bytes=588124696 weight_movement_bytes=103809024 total_nvlink_bytes=691933720 weight_to_token_ratio=0.176509 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496532710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=344932352 dispatch_meta_bytes=340264 combine_payload_bytes=344932352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345272616 ingress_bytes=379535360 link_bytes=379535360 token_bytes=690204968 weight_movement_bytes=34603008 total_nvlink_bytes=724807976 weight_to_token_ratio=0.050134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428161005e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95194 payload_rows_remote=81884 dup_rows=3110 dup_rows_remote=2282 dedup_ratio=0.031637 router_tokens_per_expert=[0, 0, 3306, 0, 0, 0, 1212, 5, 0, 7664, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 643, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6397, 6197, 7562, 7050] payload_rows_per_dst=[8124, 15328, 14100, 12906, 13140, 13310, 15124, 3162] dup_rows_per_dst=[922, 162, 0, 1198, 0, 828, 0, 0] combine_rows_per_src=[8124, 15328, 14100, 12906, 13140, 13310, 15124, 3162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92632 payload_rows_remote=83448 dup_rows=5672 dup_rows_remote=5668 dedup_ratio=0.057699 router_tokens_per_expert=[0, 0, 3475, 0, 0, 0, 1230, 0, 0, 7682, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 8, 0, 0, 0, 520, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6647, 6125, 7421, 7131] payload_rows_per_dst=[11612, 9184, 14262, 12776, 16304, 13652, 14842, 0] dup_rows_per_dst=[3978, 4, 0, 980, 0, 710, 0, 0] combine_rows_per_src=[11612, 9184, 14262, 12776, 16304, 13652, 14842, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=333062144 dispatch_meta_bytes=328184 combine_payload_bytes=333062144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333390328 ingress_bytes=367665152 link_bytes=367665152 token_bytes=666452472 weight_movement_bytes=34603008 total_nvlink_bytes=701055480 weight_to_token_ratio=0.051921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383494368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=291610624 dispatch_meta_bytes=315672 combine_payload_bytes=291610624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291926296 ingress_bytes=326213632 link_bytes=326213632 token_bytes=583536920 weight_movement_bytes=34603008 total_nvlink_bytes=618139928 weight_to_token_ratio=0.059299 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227515635e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95624 payload_rows_remote=79286 dup_rows=2680 dup_rows_remote=2680 dedup_ratio=0.027262 router_tokens_per_expert=[0, 0, 3371, 0, 0, 0, 1395, 0, 0, 7546, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 1, 0, 0, 0, 510, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6803, 6145, 7488, 7037] payload_rows_per_dst=[8602, 15094, 14074, 12678, 16338, 13860, 14978, 0] dup_rows_per_dst=[930, 40, 0, 934, 0, 776, 0, 0] combine_rows_per_src=[8602, 15094, 14074, 12678, 16338, 13860, 14978, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82360 payload_rows_remote=65976 dup_rows=15944 dup_rows_remote=15940 dedup_ratio=0.162191 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 6522, 0, 0, 0, 8191, 0, 5201, 0, 527, 0, 0, 0, 13, 1, 0, 0, 0, 557, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5227, 0, 2, 4004, 0, 0, 7077, 0, 0, 0, 0, 0, 0, 7548, 0, 0, 0, 0, 4273, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14404, 14378, 3910, 15614, 9124, 0, 8546] dup_rows_per_dst=[4, 6194, 858, 64, 7932, 892, 0, 0] combine_rows_per_src=[16384, 14404, 14378, 3910, 15614, 9124, 0, 8546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=341803008 dispatch_meta_bytes=356464 combine_payload_bytes=341803008 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=342159472 ingress_bytes=411009024 link_bytes=411009024 token_bytes=683962480 weight_movement_bytes=69206016 total_nvlink_bytes=753168496 weight_to_token_ratio=0.101184 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546593868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=335396864 dispatch_meta_bytes=336664 combine_payload_bytes=335396864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335733528 ingress_bytes=369999872 link_bytes=369999872 token_bytes=671130392 weight_movement_bytes=34603008 total_nvlink_bytes=705733400 weight_to_token_ratio=0.051559 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86714 payload_rows_remote=77594 dup_rows=11590 dup_rows_remote=11590 dedup_ratio=0.117900 router_tokens_per_expert=[0, 0, 3912, 0, 0, 0, 1367, 2, 0, 7613, 0, 0, 1, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 729, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 2, 0, 0, 0, 0, 0, 553, 24, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6719, 5890, 7320, 6790] payload_rows_per_dst=[9520, 15234, 13580, 12258, 0, 10670, 9120, 16332] dup_rows_per_dst=[1042, 136, 0, 982, 0, 664, 0, 8766] combine_rows_per_src=[9520, 15234, 13580, 12258, 0, 10670, 9120, 16332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95184 payload_rows_remote=82798 dup_rows=3120 dup_rows_remote=1952 dedup_ratio=0.031738 router_tokens_per_expert=[0, 0, 3232, 0, 0, 0, 1436, 10, 0, 7599, 1, 0, 0, 1, 0, 73, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 891, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 1, 1, 6, 0, 0, 4, 504, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 6562, 5876, 7541, 7234] payload_rows_per_dst=[8268, 15204, 14470, 12386, 16326, 13438, 15086, 6] dup_rows_per_dst=[1088, 144, 0, 1168, 0, 720, 0, 0] combine_rows_per_src=[8268, 15204, 14470, 12386, 16326, 13438, 15086, 6] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=324755456 dispatch_meta_bytes=327864 combine_payload_bytes=324755456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325083320 ingress_bytes=393961472 link_bytes=393961472 token_bytes=649838776 weight_movement_bytes=69206016 total_nvlink_bytes=719044792 weight_to_token_ratio=0.106497 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482445303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=270237696 dispatch_meta_bytes=327664 combine_payload_bytes=270237696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270565360 ingress_bytes=304840704 link_bytes=304840704 token_bytes=540803056 weight_movement_bytes=34603008 total_nvlink_bytes=575406064 weight_to_token_ratio=0.063984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147091027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94950 payload_rows_remote=81458 dup_rows=3354 dup_rows_remote=3354 dedup_ratio=0.034119 router_tokens_per_expert=[0, 0, 3672, 0, 0, 0, 1594, 14, 0, 7738, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 2, 0, 0, 0, 621, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6418, 6057, 7258, 6746] payload_rows_per_dst=[9222, 15476, 13492, 12692, 16256, 13296, 14516, 0] dup_rows_per_dst=[1338, 148, 0, 1050, 0, 818, 0, 0] combine_rows_per_src=[9222, 15476, 13492, 12692, 16256, 13296, 14516, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317825024 dispatch_meta_bytes=356736 combine_payload_bytes=317825024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318181760 ingress_bytes=352428032 link_bytes=352428032 token_bytes=636006784 weight_movement_bytes=34603008 total_nvlink_bytes=670609792 weight_to_token_ratio=0.054407 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326158312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=43504 payload_rows_remote=27120 dup_rows=54800 dup_rows_remote=1148 dedup_ratio=0.557454 router_tokens_per_expert=[0, 0, 3096, 0, 0, 0, 1427, 0, 0, 7735, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 4, 1, 0, 0, 626, 28, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6298, 6499, 7537, 6899] payload_rows_per_dst=[8112, 11896, 144, 2038, 3608, 1318, 4, 16384] dup_rows_per_dst=[934, 140, 0, 74, 0, 0, 0, 53652] combine_rows_per_src=[8112, 11896, 144, 2038, 3608, 1318, 4, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=339140608 dispatch_meta_bytes=339000 combine_payload_bytes=339140608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339479608 ingress_bytes=373743616 link_bytes=373743616 token_bytes=678620216 weight_movement_bytes=34603008 total_nvlink_bytes=713223224 weight_to_token_ratio=0.050990 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406367138e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93560 payload_rows_remote=77176 dup_rows=4744 dup_rows_remote=4740 dedup_ratio=0.048258 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 112, 1, 0, 1, 241, 3245, 0, 308, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7183, 8177, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 4669, 0, 0, 40, 0, 0, 0, 0, 0, 0, 133, 0, 22, 0, 0, 0] payload_rows_per_dst=[16362, 8796, 11728, 14500, 22, 16384, 9414, 16354] dup_rows_per_dst=[0, 116, 3400, 910, 0, 4, 4, 310] combine_rows_per_src=[16362, 8796, 11728, 14500, 22, 16384, 9414, 16354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97370 payload_rows_remote=81148 dup_rows=934 dup_rows_remote=692 dedup_ratio=0.009501 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8110, 0, 122, 0, 0, 0, 301, 2629, 0, 367, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7340, 8178, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 5120, 0, 0, 40, 0, 0, 0, 0, 0, 0, 169, 0, 17, 0, 3, 0] payload_rows_per_dst=[16384, 16222, 6580, 14746, 16356, 16384, 10320, 378] dup_rows_per_dst=[2, 242, 14, 672, 0, 4, 0, 0] combine_rows_per_src=[16384, 16222, 6580, 14746, 16356, 16384, 10320, 378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=333651968 dispatch_meta_bytes=339248 combine_payload_bytes=333651968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333991216 ingress_bytes=368254976 link_bytes=368254976 token_bytes=667643184 weight_movement_bytes=34603008 total_nvlink_bytes=702246192 weight_to_token_ratio=0.051829 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385713828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=111083520 dispatch_meta_bytes=113072 combine_payload_bytes=111083520 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=111196592 ingress_bytes=284098560 link_bytes=284098560 token_bytes=222280112 weight_movement_bytes=173015040 total_nvlink_bytes=395295152 weight_to_token_ratio=0.778365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069040009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96874 payload_rows_remote=96874 dup_rows=1430 dup_rows_remote=1430 dedup_ratio=0.014547 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8110, 0, 170, 0, 3, 0, 318, 2602, 0, 335, 0, 0, 0, 0, 452, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7334, 8178, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 5103, 0, 0, 30, 0, 0, 0, 0, 0, 0, 115, 0, 10, 0, 0, 0] payload_rows_per_dst=[16384, 16232, 6494, 14762, 0, 16384, 10260, 16358] dup_rows_per_dst=[2, 334, 16, 812, 0, 12, 6, 248] combine_rows_per_src=[16384, 16232, 6494, 14762, 0, 16384, 10260, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87992 payload_rows_remote=71668 dup_rows=10312 dup_rows_remote=2024 dedup_ratio=0.104899 router_tokens_per_expert=[0, 0, 3068, 0, 0, 0, 1515, 8, 0, 7715, 0, 0, 1, 0, 0, 56, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 535, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6545, 6277, 7317, 7085] payload_rows_per_dst=[16324, 114, 14172, 12932, 16332, 13482, 14634, 2] dup_rows_per_dst=[8288, 0, 10, 1252, 0, 762, 0, 0] combine_rows_per_src=[16324, 114, 14172, 12932, 16332, 13482, 14634, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=316112896 dispatch_meta_bytes=327664 combine_payload_bytes=316112896 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316440560 ingress_bytes=419921920 link_bytes=419921920 token_bytes=632553456 weight_movement_bytes=103809024 total_nvlink_bytes=736362480 weight_to_token_ratio=0.164111 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580132378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=332382208 dispatch_meta_bytes=327360 combine_payload_bytes=332382208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332709568 ingress_bytes=401588224 link_bytes=401588224 token_bytes=665091776 weight_movement_bytes=69206016 total_nvlink_bytes=734297792 weight_to_token_ratio=0.104055 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511144156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75886 payload_rows_remote=63952 dup_rows=22418 dup_rows_remote=19074 dedup_ratio=0.228048 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8092, 0, 144, 1, 2, 0, 259, 3249, 0, 625, 0, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6962, 8157, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 4708, 0, 0, 48, 0, 0, 0, 0, 0, 0, 82, 0, 16, 0, 7, 0] payload_rows_per_dst=[4, 294, 16372, 3964, 16384, 10618, 11934, 16316] dup_rows_per_dst=[0, 0, 8078, 176, 10604, 8, 3344, 208] combine_rows_per_src=[4, 294, 16372, 3964, 16384, 10618, 11934, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96464 payload_rows_remote=81920 dup_rows=1840 dup_rows_remote=712 dedup_ratio=0.018717 router_tokens_per_expert=[0, 0, 3, 1, 1, 0, 2, 0, 0, 0, 0, 8100, 1, 177, 0, 1, 2, 353, 2735, 2, 424, 0, 1, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7190, 8177, 8187, 0, 0, 7, 0, 0, 0, 3, 0, 8183, 4771, 0, 0, 23, 1, 0, 0, 0, 0, 0, 118, 0, 35, 1, 6, 1] payload_rows_per_dst=[16380, 16208, 7004, 14544, 0, 16380, 9586, 16362] dup_rows_per_dst=[8, 350, 30, 1128, 0, 6, 4, 314] combine_rows_per_src=[16380, 16208, 7004, 14544, 0, 16380, 9586, 16362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=396795904 dispatch_meta_bytes=393216 combine_payload_bytes=396795904 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=397189120 ingress_bytes=535207936 link_bytes=535207936 token_bytes=793985024 weight_movement_bytes=138412032 total_nvlink_bytes=932397056 weight_to_token_ratio=0.174326 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.013944375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=293552128 dispatch_meta_bytes=294768 combine_payload_bytes=293552128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293846896 ingress_bytes=328155136 link_bytes=328155136 token_bytes=587399024 weight_movement_bytes=34603008 total_nvlink_bytes=622002032 weight_to_token_ratio=0.058909 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234821358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=261947392 dispatch_meta_bytes=332104 combine_payload_bytes=261947392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262279496 ingress_bytes=296550400 link_bytes=296550400 token_bytes=524226888 weight_movement_bytes=34603008 total_nvlink_bytes=558829896 weight_to_token_ratio=0.066008 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115895281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96642 payload_rows_remote=88662 dup_rows=1662 dup_rows_remote=1626 dedup_ratio=0.016907 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8117, 0, 111, 0, 4, 0, 217, 3422, 0, 369, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7193, 8178, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 4285, 0, 0, 33, 0, 0, 0, 0, 0, 0, 200, 0, 52, 0, 5, 0] payload_rows_per_dst=[16384, 16240, 7980, 14520, 2282, 16382, 8632, 14222] dup_rows_per_dst=[6, 224, 36, 1024, 0, 2, 4, 366] combine_rows_per_src=[16384, 16240, 7980, 14520, 2282, 16382, 8632, 14222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66906 payload_rows_remote=50572 dup_rows=31398 dup_rows_remote=31154 dedup_ratio=0.319397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8121, 0, 246, 0, 8, 0, 336, 2799, 0, 399, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 8167, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 4720, 0, 0, 43, 0, 0, 0, 0, 0, 0, 85, 0, 31, 0, 6, 0] payload_rows_per_dst=[2, 502, 16372, 924, 16384, 4, 16384, 16334] dup_rows_per_dst=[0, 6, 6938, 0, 14688, 0, 9522, 244] combine_rows_per_src=[2, 502, 16372, 924, 16384, 4, 16384, 16334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=330528 combine_payload_bytes=335544320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335874848 ingress_bytes=370147328 link_bytes=370147328 token_bytes=671419168 weight_movement_bytes=34603008 total_nvlink_bytes=706022176 weight_to_token_ratio=0.051537 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93064 payload_rows_remote=77472 dup_rows=5240 dup_rows_remote=3746 dedup_ratio=0.053304 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7648, 547, 1210, 1055, 0, 0, 1, 0, 0, 362, 0, 0, 0, 0, 7614, 0, 0, 0, 0, 0, 405, 0, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 6663, 133, 0, 8154, 3, 0, 2239, 65, 0, 0, 0, 0, 4471] payload_rows_per_dst=[16308, 15228, 5514, 13358, 15288, 15592, 290, 11486] dup_rows_per_dst=[6, 140, 110, 694, 732, 1494, 0, 2064] combine_rows_per_src=[16308, 15228, 5514, 13358, 15288, 15592, 290, 11486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85578 payload_rows_remote=69606 dup_rows=12726 dup_rows_remote=12580 dedup_ratio=0.129456 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7520, 487, 987, 848, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0, 442, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 6794, 112, 0, 8141, 13, 0, 2367, 88, 0, 0, 0, 0, 4425] payload_rows_per_dst=[16282, 15972, 16168, 13624, 15480, 1528, 250, 6274] dup_rows_per_dst=[0, 146, 10584, 740, 836, 0, 0, 420] combine_rows_per_src=[16282, 15972, 16168, 13624, 15480, 1528, 250, 6274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=363159552 dispatch_meta_bytes=361152 combine_payload_bytes=363159552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=363520704 ingress_bytes=397762560 link_bytes=397762560 token_bytes=726680256 weight_movement_bytes=34603008 total_nvlink_bytes=761283264 weight_to_token_ratio=0.047618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496748491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=207142912 dispatch_meta_bytes=326904 combine_payload_bytes=207142912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=207469816 ingress_bytes=241745920 link_bytes=241745920 token_bytes=414612728 weight_movement_bytes=34603008 total_nvlink_bytes=449215736 weight_to_token_ratio=0.083459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.096704348e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92932 payload_rows_remote=77664 dup_rows=5372 dup_rows_remote=4510 dedup_ratio=0.054647 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7423, 452, 1083, 889, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 7576, 0, 1, 0, 0, 0, 488, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 6583, 134, 0, 8118, 10, 0, 2497, 154, 0, 0, 0, 0, 4853] payload_rows_per_dst=[16236, 15578, 4766, 13190, 15268, 15168, 288, 12438] dup_rows_per_dst=[2, 68, 86, 706, 862, 1078, 0, 2570] combine_rows_per_src=[16236, 15578, 4766, 13190, 15268, 15168, 288, 12438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=317325312 dispatch_meta_bytes=324872 combine_payload_bytes=317325312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317650184 ingress_bytes=351928320 link_bytes=351928320 token_bytes=634975496 weight_movement_bytes=34603008 total_nvlink_bytes=669578504 weight_to_token_ratio=0.054495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324277936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85948 payload_rows_remote=69564 dup_rows=12356 dup_rows_remote=12354 dedup_ratio=0.125692 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 373, 0, 2, 5, 407, 2793, 0, 426, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7285, 8178, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 4402, 0, 0, 43, 0, 0, 0, 0, 0, 0, 126, 0, 11, 0, 8, 0] payload_rows_per_dst=[16384, 16366, 7242, 14682, 16362, 5732, 8890, 290] dup_rows_per_dst=[2, 6172, 20, 940, 5222, 0, 0, 0] combine_rows_per_src=[16384, 16366, 7242, 14682, 16362, 5732, 8890, 290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285106176 dispatch_meta_bytes=328744 combine_payload_bytes=285106176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285434920 ingress_bytes=319709184 link_bytes=319709184 token_bytes=570541096 weight_movement_bytes=34603008 total_nvlink_bytes=605144104 weight_to_token_ratio=0.060649 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203039921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67138 payload_rows_remote=50808 dup_rows=31166 dup_rows_remote=4652 dedup_ratio=0.317037 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7501, 414, 1545, 969, 0, 0, 18, 0, 0, 368, 0, 0, 0, 0, 7814, 0, 3, 0, 0, 0, 431, 0, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 6603, 164, 0, 8116, 4, 0, 2197, 129, 0, 0, 0, 0, 4225] payload_rows_per_dst=[684, 4924, 5720, 12080, 868, 15478, 16330, 11054] dup_rows_per_dst=[0, 18, 136, 648, 0, 1802, 26514, 2048] combine_rows_per_src=[684, 4924, 5720, 12080, 868, 15478, 16330, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92978 payload_rows_remote=79840 dup_rows=5326 dup_rows_remote=4584 dedup_ratio=0.054179 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 1, 81, 0, 0, 0, 1, 0, 1, 0, 3, 1, 7608, 632, 1398, 984, 2, 0, 23, 0, 0, 386, 0, 0, 0, 0, 7575, 0, 2, 0, 0, 0, 432, 1, 1, 971, 0, 3, 0, 1, 0, 2, 2, 0, 7682, 6531, 146, 0, 8127, 14, 1, 2179, 115, 0, 0, 0, 0, 4245] payload_rows_per_dst=[16256, 15366, 5894, 13138, 15240, 15594, 324, 11166] dup_rows_per_dst=[2, 164, 146, 742, 780, 1578, 0, 1914] combine_rows_per_src=[16256, 15366, 5894, 13138, 15240, 15594, 324, 11166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=318111744 dispatch_meta_bytes=328696 combine_payload_bytes=318111744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=318440440 ingress_bytes=387317760 link_bytes=387317760 token_bytes=636552184 weight_movement_bytes=69206016 total_nvlink_bytes=705758200 weight_to_token_ratio=0.108720 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457445549e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91444 payload_rows_remote=81044 dup_rows=6860 dup_rows_remote=3982 dedup_ratio=0.069784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 565, 1460, 1014, 0, 0, 15, 0, 0, 400, 0, 0, 0, 0, 7595, 0, 0, 0, 0, 0, 324, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 7576, 6768, 95, 0, 8107, 0, 0, 1679, 59, 0, 0, 0, 0, 4525] payload_rows_per_dst=[16214, 15152, 10400, 13602, 15232, 9690, 190, 10964] dup_rows_per_dst=[0, 108, 2878, 764, 606, 942, 0, 1562] combine_rows_per_src=[16214, 15152, 10400, 13602, 15232, 9690, 190, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=52830 payload_rows_remote=39454 dup_rows=45474 dup_rows_remote=38966 dedup_ratio=0.462585 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 524, 1359, 990, 0, 0, 16, 0, 0, 375, 0, 0, 0, 0, 7650, 0, 1, 0, 0, 0, 544, 0, 0, 786, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 6849, 150, 0, 8153, 2, 1, 2053, 114, 0, 0, 0, 0, 4334] payload_rows_per_dst=[4, 90, 5602, 780, 1090, 15504, 16384, 13376] dup_rows_per_dst=[0, 0, 144, 2, 0, 1292, 37528, 6508] combine_rows_per_src=[4, 90, 5602, 780, 1090, 15504, 16384, 13376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=284934144 dispatch_meta_bytes=327672 combine_payload_bytes=284934144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285261816 ingress_bytes=319537152 link_bytes=319537152 token_bytes=570195960 weight_movement_bytes=34603008 total_nvlink_bytes=604798968 weight_to_token_ratio=0.060686 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202392578e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=208109568 dispatch_meta_bytes=221840 combine_payload_bytes=208109568 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=208331408 ingress_bytes=346521600 link_bytes=346521600 token_bytes=416440976 weight_movement_bytes=138412032 total_nvlink_bytes=554853008 weight_to_token_ratio=0.332369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303932884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=327024640 dispatch_meta_bytes=337696 combine_payload_bytes=327024640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327362336 ingress_bytes=361627648 link_bytes=361627648 token_bytes=654386976 weight_movement_bytes=34603008 total_nvlink_bytes=688989984 weight_to_token_ratio=0.052879 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360775726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=331956224 dispatch_meta_bytes=340104 combine_payload_bytes=331956224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332296328 ingress_bytes=401162240 link_bytes=401162240 token_bytes=664252552 weight_movement_bytes=69206016 total_nvlink_bytes=733458568 weight_to_token_ratio=0.104186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.509541213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92370 payload_rows_remote=75986 dup_rows=5934 dup_rows_remote=5064 dedup_ratio=0.060364 router_tokens_per_expert=[0, 3956, 0, 0, 0, 0, 0, 0, 0, 1466, 780, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 0, 0, 0, 886, 117, 7882, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 435, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 4384, 0, 0, 0, 5530, 0] payload_rows_per_dst=[7912, 4154, 7510, 15766, 12004, 16384, 16384, 12256] dup_rows_per_dst=[0, 338, 436, 232, 702, 870, 0, 3356] combine_rows_per_src=[7912, 4154, 7510, 15766, 12004, 16384, 16384, 12256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=161603584 dispatch_meta_bytes=313680 combine_payload_bytes=161603584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=161917264 ingress_bytes=230809600 link_bytes=230809600 token_bytes=323520848 weight_movement_bytes=69206016 total_nvlink_bytes=392726864 weight_to_token_ratio=0.213915 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.685179431e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81480 payload_rows_remote=65150 dup_rows=16824 dup_rows_remote=12666 dedup_ratio=0.171143 router_tokens_per_expert=[0, 3710, 0, 0, 0, 0, 0, 0, 0, 1633, 570, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 543, 143, 8041, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 0, 333, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 4344, 0, 0, 0, 5640, 0] payload_rows_per_dst=[16384, 16330, 13684, 16384, 12100, 666, 0, 5932] dup_rows_per_dst=[7420, 4158, 4248, 286, 712, 0, 0, 0] combine_rows_per_src=[16384, 16330, 13684, 16384, 12100, 666, 0, 5932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92616 payload_rows_remote=81126 dup_rows=5688 dup_rows_remote=5214 dedup_ratio=0.057861 router_tokens_per_expert=[0, 3800, 0, 0, 0, 0, 0, 0, 0, 1691, 500, 0, 0, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 636, 84, 8073, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 0, 452, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 4618, 0, 0, 0, 5224, 0] payload_rows_per_dst=[7600, 4922, 8386, 15030, 11490, 16384, 16384, 12420] dup_rows_per_dst=[0, 582, 404, 166, 474, 904, 0, 3158] combine_rows_per_src=[7600, 4922, 8386, 15030, 11490, 16384, 16384, 12420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=311238656 dispatch_meta_bytes=324200 combine_payload_bytes=311238656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=311562856 ingress_bytes=415047680 link_bytes=415047680 token_bytes=622801512 weight_movement_bytes=103809024 total_nvlink_bytes=726610536 weight_to_token_ratio=0.166681 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.561791006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=311264 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165664 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534020064 weight_movement_bytes=34603008 total_nvlink_bytes=568623072 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92340 payload_rows_remote=75956 dup_rows=5964 dup_rows_remote=5964 dedup_ratio=0.060669 router_tokens_per_expert=[0, 3991, 0, 1, 0, 0, 0, 0, 0, 1131, 660, 0, 0, 0, 0, 0, 0, 3225, 0, 0, 0, 0, 0, 512, 91, 7969, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 531, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2868, 0, 5138, 0, 0, 0, 5721, 0] payload_rows_per_dst=[7984, 3228, 7162, 15938, 12426, 16384, 16384, 12834] dup_rows_per_dst=[0, 354, 312, 182, 876, 1062, 0, 3178] combine_rows_per_src=[7984, 3228, 7162, 15938, 12426, 16384, 16384, 12834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82910 payload_rows_remote=66600 dup_rows=15394 dup_rows_remote=15394 dedup_ratio=0.156596 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7524, 413, 1607, 1096, 0, 0, 20, 0, 0, 363, 0, 0, 0, 0, 7588, 0, 1, 0, 0, 0, 422, 0, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 6735, 87, 0, 8155, 16, 0, 2148, 82, 0, 0, 0, 0, 4438] payload_rows_per_dst=[16310, 15222, 16352, 13532, 15272, 1564, 206, 4452] dup_rows_per_dst=[0, 128, 13804, 704, 750, 0, 0, 8] combine_rows_per_src=[16310, 15222, 16352, 13532, 15272, 1564, 206, 4452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=332292096 dispatch_meta_bytes=345360 combine_payload_bytes=332292096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332637456 ingress_bytes=366895104 link_bytes=366895104 token_bytes=664929552 weight_movement_bytes=34603008 total_nvlink_bytes=699532560 weight_to_token_ratio=0.052040 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380596739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89178 payload_rows_remote=89060 dup_rows=9126 dup_rows_remote=9126 dedup_ratio=0.092834 router_tokens_per_expert=[0, 3599, 1, 0, 0, 0, 0, 0, 0, 1498, 539, 1, 1, 2, 1, 0, 1, 3738, 0, 0, 0, 0, 0, 533, 57, 8026, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 0, 1, 0, 678, 0, 0, 571, 0, 0, 1, 0, 8190, 3, 3, 0, 0, 0, 0, 2, 0, 0, 3756, 0, 4290, 0, 0, 0, 5470, 0] payload_rows_per_dst=[7200, 16274, 10800, 118, 11768, 16382, 16382, 10254] dup_rows_per_dst=[0, 3862, 2052, 0, 530, 1148, 4, 1530] combine_rows_per_src=[7200, 16274, 10800, 118, 11768, 16382, 16382, 10254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87218 payload_rows_remote=73552 dup_rows=11086 dup_rows_remote=6674 dedup_ratio=0.112773 router_tokens_per_expert=[0, 3840, 0, 0, 0, 0, 0, 0, 0, 1138, 1014, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 0, 0, 607, 102, 7786, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 0, 577, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 4727, 0, 0, 0, 5537, 0] payload_rows_per_dst=[9900, 16070, 13666, 1572, 11904, 11660, 16382, 6064] dup_rows_per_dst=[1708, 3806, 4412, 14, 576, 570, 0, 0] combine_rows_per_src=[9900, 16070, 13666, 1572, 11904, 11660, 16382, 6064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87936 payload_rows_remote=75022 dup_rows=10368 dup_rows_remote=3034 dedup_ratio=0.105469 router_tokens_per_expert=[0, 4150, 0, 9, 0, 0, 0, 0, 0, 1444, 643, 0, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 674, 152, 7882, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 752, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3015, 0, 4395, 0, 0, 0, 5433, 0] payload_rows_per_dst=[8300, 3802, 7770, 15766, 9170, 13830, 16384, 12914] dup_rows_per_dst=[18, 372, 356, 302, 602, 1384, 0, 7334] combine_rows_per_src=[8300, 3802, 7770, 15766, 9170, 13830, 16384, 12914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=311115776 dispatch_meta_bytes=327680 combine_payload_bytes=311115776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311443456 ingress_bytes=345718784 link_bytes=345718784 token_bytes=622559232 weight_movement_bytes=34603008 total_nvlink_bytes=657162240 weight_to_token_ratio=0.055582 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300911952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=272793600 dispatch_meta_bytes=327976 combine_payload_bytes=272793600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273121576 ingress_bytes=307396608 link_bytes=307396608 token_bytes=545915176 weight_movement_bytes=34603008 total_nvlink_bytes=580518184 weight_to_token_ratio=0.063385 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156708688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=364789760 dispatch_meta_bytes=392744 combine_payload_bytes=364789760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365182504 ingress_bytes=468598784 link_bytes=468598784 token_bytes=729972264 weight_movement_bytes=103809024 total_nvlink_bytes=833781288 weight_to_token_ratio=0.142210 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.763299499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=301268992 dispatch_meta_bytes=320904 combine_payload_bytes=301268992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301589896 ingress_bytes=335872000 link_bytes=335872000 token_bytes=602858888 weight_movement_bytes=34603008 total_nvlink_bytes=637461896 weight_to_token_ratio=0.057398 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95370 payload_rows_remote=80652 dup_rows=2934 dup_rows_remote=2608 dedup_ratio=0.029846 router_tokens_per_expert=[1588, 0, 0, 8192, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 2090, 0, 8107, 0, 0, 7327, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5746, 920] payload_rows_per_dst=[4826, 16384, 16166, 10496, 4180, 14718, 16214, 12386] dup_rows_per_dst=[50, 1448, 10, 152, 0, 326, 0, 948] combine_rows_per_src=[4826, 16384, 16166, 10496, 4180, 14718, 16214, 12386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=307290112 dispatch_meta_bytes=312224 combine_payload_bytes=307290112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307602336 ingress_bytes=411099136 link_bytes=411099136 token_bytes=614892448 weight_movement_bytes=103809024 total_nvlink_bytes=718701472 weight_to_token_ratio=0.168825 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546932953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80692 payload_rows_remote=78616 dup_rows=17612 dup_rows_remote=17612 dedup_ratio=0.179159 router_tokens_per_expert=[1678, 0, 0, 8192, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 5003, 0, 0, 0, 0, 1718, 0, 8115, 0, 0, 7435, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5669, 1213] payload_rows_per_dst=[16384, 2076, 16384, 10006, 14640, 6608, 2026, 12568] dup_rows_per_dst=[4862, 0, 18, 0, 11428, 108, 0, 1196] combine_rows_per_src=[16384, 2076, 16384, 10006, 14640, 6608, 2026, 12568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95822 payload_rows_remote=92366 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.025248 router_tokens_per_expert=[1672, 0, 0, 8192, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5347, 0, 0, 0, 0, 1728, 0, 8103, 0, 0, 7444, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6070, 988] payload_rows_per_dst=[4712, 16384, 16384, 10694, 3456, 14912, 16206, 13074] dup_rows_per_dst=[38, 1224, 16, 0, 0, 162, 0, 1042] combine_rows_per_src=[4712, 16384, 16384, 10694, 3456, 14912, 16206, 13074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=330350592 dispatch_meta_bytes=333040 combine_payload_bytes=330350592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330683632 ingress_bytes=364953600 link_bytes=364953600 token_bytes=661034224 weight_movement_bytes=34603008 total_nvlink_bytes=695637232 weight_to_token_ratio=0.052347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373291016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=322011136 dispatch_meta_bytes=384912 combine_payload_bytes=322011136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322396048 ingress_bytes=356614144 link_bytes=356614144 token_bytes=644407184 weight_movement_bytes=34603008 total_nvlink_bytes=679010192 weight_to_token_ratio=0.053697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341910314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=378331136 dispatch_meta_bytes=379392 combine_payload_bytes=378331136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=378710528 ingress_bytes=482140160 link_bytes=482140160 token_bytes=757041664 weight_movement_bytes=103809024 total_nvlink_bytes=860850688 weight_to_token_ratio=0.137125 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.814254607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86230 payload_rows_remote=70172 dup_rows=12074 dup_rows_remote=12072 dedup_ratio=0.122823 router_tokens_per_expert=[1928, 0, 0, 8192, 0, 0, 0, 1139, 3, 0, 0, 0, 0, 0, 805, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 4606, 0, 0, 0, 0, 2130, 0, 8029, 0, 0, 7502, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 745] payload_rows_per_dst=[6002, 16384, 44, 16384, 4260, 15044, 16058, 12054] dup_rows_per_dst=[132, 1616, 0, 9212, 0, 284, 2, 828] combine_rows_per_src=[6002, 16384, 44, 16384, 4260, 15044, 16058, 12054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=65880 dup_rows=16040 dup_rows_remote=8546 dedup_ratio=0.163167 router_tokens_per_expert=[0, 3745, 0, 2, 0, 0, 0, 0, 0, 1377, 628, 0, 0, 0, 0, 0, 0, 2867, 0, 0, 0, 0, 0, 962, 81, 8021, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 2, 916, 0, 0, 338, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 4724, 0, 0, 0, 5567, 0] payload_rows_per_dst=[16384, 16236, 13366, 16384, 12144, 676, 0, 7074] dup_rows_per_dst=[7494, 3816, 3740, 164, 826, 0, 0, 0] combine_rows_per_src=[16384, 16236, 13366, 16384, 12144, 676, 0, 7074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95078 payload_rows_remote=84986 dup_rows=3226 dup_rows_remote=3224 dedup_ratio=0.032817 router_tokens_per_expert=[1812, 0, 0, 8189, 0, 2, 0, 845, 0, 0, 0, 1, 1, 0, 940, 0, 0, 0, 0, 0, 0, 8187, 0, 8, 1, 3, 0, 0, 5043, 0, 0, 0, 0, 1926, 0, 8102, 0, 0, 7249, 0, 0, 2, 0, 203, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5681, 951] payload_rows_per_dst=[5268, 16382, 16374, 10092, 3852, 14608, 16208, 12294] dup_rows_per_dst=[50, 1880, 16, 2, 0, 304, 0, 974] combine_rows_per_src=[5268, 16382, 16374, 10092, 3852, 14608, 16208, 12294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91836 payload_rows_remote=75452 dup_rows=6468 dup_rows_remote=6460 dedup_ratio=0.065796 router_tokens_per_expert=[1598, 0, 0, 8192, 0, 0, 0, 1106, 0, 0, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 4692, 0, 0, 0, 0, 2098, 0, 7886, 0, 0, 7463, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 5903, 1055] payload_rows_per_dst=[13962, 4618, 16384, 9384, 4196, 14962, 15772, 12558] dup_rows_per_dst=[4372, 304, 8, 0, 0, 396, 0, 1388] combine_rows_per_src=[13962, 4618, 16384, 9384, 4196, 14962, 15772, 12558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=287424512 dispatch_meta_bytes=328976 combine_payload_bytes=287424512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287753488 ingress_bytes=322027520 link_bytes=322027520 token_bytes=575178000 weight_movement_bytes=34603008 total_nvlink_bytes=609781008 weight_to_token_ratio=0.060161 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211763632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90710 payload_rows_remote=78018 dup_rows=7594 dup_rows_remote=6384 dedup_ratio=0.077250 router_tokens_per_expert=[1953, 0, 0, 8192, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 8191, 0, 15, 0, 0, 0, 0, 5031, 0, 0, 0, 0, 1710, 0, 7907, 0, 0, 7600, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5781, 1111] payload_rows_per_dst=[5422, 16384, 30, 10872, 14256, 15236, 15818, 12692] dup_rows_per_dst=[56, 1454, 0, 1504, 3120, 250, 0, 1210] combine_rows_per_src=[5422, 16384, 30, 10872, 14256, 15236, 15818, 12692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=269844480 dispatch_meta_bytes=297704 combine_payload_bytes=269844480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270142184 ingress_bytes=304447488 link_bytes=304447488 token_bytes=539986664 weight_movement_bytes=34603008 total_nvlink_bytes=574589672 weight_to_token_ratio=0.064081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145611387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=348102656 dispatch_meta_bytes=352840 combine_payload_bytes=348102656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348455496 ingress_bytes=382705664 link_bytes=382705664 token_bytes=696558152 weight_movement_bytes=34603008 total_nvlink_bytes=731161160 weight_to_token_ratio=0.049677 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440090603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309051392 dispatch_meta_bytes=327648 combine_payload_bytes=309051392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=309379040 ingress_bytes=412860416 link_bytes=412860416 token_bytes=618430432 weight_movement_bytes=103809024 total_nvlink_bytes=722239456 weight_to_token_ratio=0.167859 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553560507e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91594 payload_rows_remote=82338 dup_rows=6710 dup_rows_remote=6394 dedup_ratio=0.068258 router_tokens_per_expert=[0, 0, 61, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3555, 0, 0, 0, 1719, 0, 8192, 0, 15, 78, 0, 0, 0, 0, 2, 505, 0, 0, 303, 8192, 0, 0, 8192, 6, 0, 0, 1309, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15802, 13116, 9046, 3468, 8138, 9256, 16384, 16384] dup_rows_per_dst=[122, 394, 1440, 0, 462, 316, 3976, 0] combine_rows_per_src=[15802, 13116, 9046, 3468, 8138, 9256, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=319561728 dispatch_meta_bytes=337608 combine_payload_bytes=319561728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=319899336 ingress_bytes=388767744 link_bytes=388767744 token_bytes=639461064 weight_movement_bytes=69206016 total_nvlink_bytes=708667080 weight_to_token_ratio=0.108226 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462901722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91972 payload_rows_remote=75592 dup_rows=6332 dup_rows_remote=6040 dedup_ratio=0.064412 router_tokens_per_expert=[0, 0, 64, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3265, 0, 0, 0, 2016, 0, 8192, 0, 35, 88, 0, 0, 0, 0, 0, 491, 0, 0, 326, 8192, 0, 0, 8192, 1, 0, 0, 1650, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15832, 16380, 6530, 16384, 16384, 16384, 4078, 0] dup_rows_per_dst=[126, 292, 0, 4102, 1158, 654, 0, 0] combine_rows_per_src=[15832, 16380, 6530, 16384, 16384, 16384, 4078, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93068 payload_rows_remote=76684 dup_rows=5236 dup_rows_remote=4302 dedup_ratio=0.053263 router_tokens_per_expert=[0, 0, 64, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 1865, 0, 8192, 0, 23, 69, 0, 0, 0, 0, 4, 394, 0, 0, 212, 8192, 0, 0, 8192, 1, 0, 0, 1456, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[15874, 16384, 7456, 3776, 16384, 426, 16384, 16384] dup_rows_per_dst=[128, 302, 0, 0, 934, 0, 3870, 2] combine_rows_per_src=[15874, 16384, 7456, 3776, 16384, 426, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=337256448 dispatch_meta_bytes=354928 combine_payload_bytes=337256448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=337611376 ingress_bytes=441065472 link_bytes=441065472 token_bytes=674867824 weight_movement_bytes=103809024 total_nvlink_bytes=778676848 weight_to_token_ratio=0.153821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.659693862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77316 payload_rows_remote=60932 dup_rows=20988 dup_rows_remote=16988 dedup_ratio=0.213501 router_tokens_per_expert=[0, 0, 61, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 1693, 0, 8192, 0, 21, 72, 0, 0, 0, 0, 1, 457, 0, 0, 238, 8192, 0, 0, 8192, 4, 0, 0, 1595, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2634, 244, 16384, 7842, 1060, 16384, 16384, 16384] dup_rows_per_dst=[18, 0, 11574, 1204, 0, 4192, 4000, 0] combine_rows_per_src=[2634, 244, 16384, 7842, 1060, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=309624832 dispatch_meta_bytes=326528 combine_payload_bytes=309624832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309951360 ingress_bytes=344227840 link_bytes=344227840 token_bytes=619576192 weight_movement_bytes=34603008 total_nvlink_bytes=654179200 weight_to_token_ratio=0.055849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295301649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=314097664 dispatch_meta_bytes=323944 combine_payload_bytes=314097664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314421608 ingress_bytes=348700672 link_bytes=348700672 token_bytes=628519272 weight_movement_bytes=34603008 total_nvlink_bytes=663122280 weight_to_token_ratio=0.055055 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312132556e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63830 payload_rows_remote=47446 dup_rows=34474 dup_rows_remote=18690 dedup_ratio=0.350688 router_tokens_per_expert=[1753, 0, 0, 8192, 0, 0, 0, 815, 3, 0, 0, 0, 0, 0, 880, 0, 0, 0, 0, 0, 0, 8192, 0, 43, 0, 0, 0, 0, 4753, 0, 0, 0, 0, 1741, 0, 8090, 0, 0, 7627, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5668, 1199] payload_rows_per_dst=[16384, 1764, 16384, 9506, 16384, 386, 0, 3022] dup_rows_per_dst=[15784, 2, 86, 0, 18532, 0, 0, 70] combine_rows_per_src=[16384, 1764, 16384, 9506, 16384, 386, 0, 3022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92934 payload_rows_remote=78196 dup_rows=5370 dup_rows_remote=2170 dedup_ratio=0.054626 router_tokens_per_expert=[6, 0, 54, 0, 7838, 0, 0, 2, 1, 0, 0, 2, 123, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 3493, 0, 0, 2, 1821, 0, 8185, 0, 46, 75, 0, 0, 1, 1, 7, 508, 1, 0, 251, 8188, 0, 4, 8189, 5, 0, 0, 1639, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[15698, 16382, 6986, 14738, 16382, 520, 5846, 16382] dup_rows_per_dst=[102, 248, 0, 3200, 1180, 0, 640, 0] combine_rows_per_src=[15698, 16382, 6986, 14738, 16382, 520, 5846, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92256 payload_rows_remote=85164 dup_rows=6048 dup_rows_remote=6048 dedup_ratio=0.061523 router_tokens_per_expert=[0, 0, 131, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 1742, 0, 8192, 0, 24, 91, 0, 0, 0, 0, 4, 419, 0, 0, 276, 8192, 0, 0, 8192, 34, 0, 0, 1084, 0, 0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15672, 16384, 7092, 16384, 16384, 620, 3338, 16382] dup_rows_per_dst=[254, 1226, 0, 3532, 1028, 2, 6, 0] combine_rows_per_src=[15672, 16384, 7092, 16384, 16384, 620, 3338, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=249577472 dispatch_meta_bytes=311680 combine_payload_bytes=249577472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249889152 ingress_bytes=284180480 link_bytes=284180480 token_bytes=499466624 weight_movement_bytes=34603008 total_nvlink_bytes=534069632 weight_to_token_ratio=0.069280 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069348268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70406 payload_rows_remote=54022 dup_rows=27898 dup_rows_remote=27896 dedup_ratio=0.283793 router_tokens_per_expert=[0, 0, 40, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 4253, 0, 0, 0, 1498, 0, 8190, 0, 10, 87, 0, 0, 0, 0, 1, 620, 0, 0, 295, 8192, 0, 0, 8192, 1, 0, 0, 1071, 0, 0, 0, 0, 462, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[80, 364, 16378, 3016, 1416, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 8506, 0, 0, 16328, 3062, 2] combine_rows_per_src=[80, 364, 16378, 3016, 1416, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=194338816 dispatch_meta_bytes=264544 combine_payload_bytes=194338816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=194603360 ingress_bytes=263544832 link_bytes=263544832 token_bytes=388942176 weight_movement_bytes=69206016 total_nvlink_bytes=458148192 weight_to_token_ratio=0.177934 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.916979857e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=320290816 dispatch_meta_bytes=321464 combine_payload_bytes=320290816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320612280 ingress_bytes=389496832 link_bytes=389496832 token_bytes=640903096 weight_movement_bytes=69206016 total_nvlink_bytes=710109112 weight_to_token_ratio=0.107982 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465645222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=348831744 dispatch_meta_bytes=364848 combine_payload_bytes=348831744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=349196592 ingress_bytes=418037760 link_bytes=418037760 token_bytes=698028336 weight_movement_bytes=69206016 total_nvlink_bytes=767234352 weight_to_token_ratio=0.099145 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.573042436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91484 payload_rows_remote=75384 dup_rows=6820 dup_rows_remote=6794 dedup_ratio=0.069377 router_tokens_per_expert=[0, 0, 3761, 0, 0, 0, 0, 0, 0, 226, 8192, 0, 0, 414, 5124, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 150, 0, 0, 0, 0, 0, 768, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4647, 0, 8192, 0, 0, 8067, 0, 0, 0, 0, 0, 121, 6523, 0, 0, 0] payload_rows_per_dst=[9562, 1264, 16140, 16384, 12814, 16100, 6080, 13140] dup_rows_per_dst=[1282, 16, 132, 3350, 1690, 26, 176, 148] combine_rows_per_src=[9562, 1264, 16140, 16384, 12814, 16100, 6080, 13140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=221274112 dispatch_meta_bytes=327672 combine_payload_bytes=221274112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=221601784 ingress_bytes=255877120 link_bytes=255877120 token_bytes=442875896 weight_movement_bytes=34603008 total_nvlink_bytes=477478904 weight_to_token_ratio=0.078133 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.628450028e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83052 payload_rows_remote=66668 dup_rows=15252 dup_rows_remote=3130 dedup_ratio=0.155151 router_tokens_per_expert=[0, 0, 3393, 0, 0, 0, 0, 2, 0, 364, 8192, 0, 0, 251, 5446, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 48, 0, 0, 0, 0, 0, 708, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4485, 0, 8192, 0, 0, 8140, 1, 0, 0, 0, 0, 132, 6686, 0, 0, 0] payload_rows_per_dst=[13084, 16384, 16280, 2984, 4420, 16384, 2, 13514] dup_rows_per_dst=[2676, 12122, 196, 2, 108, 26, 0, 122] combine_rows_per_src=[13084, 16384, 16280, 2984, 4420, 16384, 2, 13514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89768 payload_rows_remote=76194 dup_rows=8536 dup_rows_remote=6342 dedup_ratio=0.086833 router_tokens_per_expert=[0, 0, 3211, 0, 0, 0, 0, 1, 0, 440, 8192, 0, 0, 163, 5506, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 118, 0, 0, 0, 0, 0, 1008, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4189, 0, 8192, 0, 0, 8086, 0, 0, 0, 0, 0, 145, 6617, 0, 0, 0] payload_rows_per_dst=[12674, 1192, 16174, 16384, 13574, 16384, 0, 13386] dup_rows_per_dst=[2128, 14, 290, 3718, 2194, 54, 0, 138] combine_rows_per_src=[12674, 1192, 16174, 16384, 13574, 16384, 0, 13386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=308772864 dispatch_meta_bytes=328712 combine_payload_bytes=308772864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309101576 ingress_bytes=343375872 link_bytes=343375872 token_bytes=617874440 weight_movement_bytes=34603008 total_nvlink_bytes=652477448 weight_to_token_ratio=0.056003 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292095762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67434 payload_rows_remote=51050 dup_rows=30870 dup_rows_remote=6254 dedup_ratio=0.314026 router_tokens_per_expert=[0, 0, 2903, 0, 0, 0, 0, 1, 0, 403, 8192, 0, 0, 288, 5087, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1555, 105, 0, 0, 0, 0, 0, 1450, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4437, 0, 8191, 0, 0, 7942, 2, 0, 0, 0, 0, 166, 6629, 0, 0, 0] payload_rows_per_dst=[5808, 1372, 304, 16384, 13732, 68, 16384, 13382] dup_rows_per_dst=[0, 10, 0, 3320, 2716, 0, 24616, 208] combine_rows_per_src=[5808, 1372, 304, 16384, 13732, 68, 16384, 13382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=273072128 dispatch_meta_bytes=279192 combine_payload_bytes=273072128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273351320 ingress_bytes=376881152 link_bytes=376881152 token_bytes=546423448 weight_movement_bytes=103809024 total_nvlink_bytes=650232472 weight_to_token_ratio=0.189979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418173434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=312090624 dispatch_meta_bytes=330144 combine_payload_bytes=312090624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312420768 ingress_bytes=346693632 link_bytes=346693632 token_bytes=624511392 weight_movement_bytes=34603008 total_nvlink_bytes=659114400 weight_to_token_ratio=0.055408 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304580226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92182 payload_rows_remote=76394 dup_rows=6122 dup_rows_remote=5948 dedup_ratio=0.062276 router_tokens_per_expert=[0, 0, 94, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 2103, 0, 8192, 0, 34, 76, 0, 0, 0, 0, 4, 425, 0, 0, 212, 8192, 0, 0, 8192, 15, 0, 0, 1679, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15788, 16382, 6912, 16384, 16384, 16384, 3948, 0] dup_rows_per_dst=[174, 206, 0, 4274, 1010, 454, 4, 0] combine_rows_per_src=[15788, 16382, 6912, 16384, 16384, 16384, 3948, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91066 payload_rows_remote=82544 dup_rows=7238 dup_rows_remote=5880 dedup_ratio=0.073629 router_tokens_per_expert=[0, 0, 3563, 0, 0, 1, 0, 1, 0, 419, 8185, 0, 0, 283, 5208, 1, 0, 4, 0, 95, 3, 0, 0, 0, 0, 0, 0, 0, 1736, 66, 0, 0, 0, 2, 0, 899, 1590, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 4025, 0, 8189, 1, 0, 8099, 2, 0, 2, 0, 0, 137, 6630, 0, 3, 0] payload_rows_per_dst=[12714, 10574, 16216, 8522, 13282, 16378, 6, 13374] dup_rows_per_dst=[2466, 926, 186, 1358, 2118, 14, 0, 170] combine_rows_per_src=[12714, 10574, 16216, 8522, 13282, 16378, 6, 13374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86412 payload_rows_remote=70702 dup_rows=11892 dup_rows_remote=11648 dedup_ratio=0.120972 router_tokens_per_expert=[0, 0, 2805, 0, 0, 0, 0, 2, 0, 240, 8192, 0, 0, 292, 5017, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 152, 0, 0, 0, 0, 0, 1033, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4384, 0, 8192, 0, 0, 7855, 0, 0, 0, 0, 0, 178, 6924, 0, 0, 0] payload_rows_per_dst=[12146, 16384, 15710, 3728, 8086, 16384, 0, 13974] dup_rows_per_dst=[2236, 8308, 244, 8, 824, 42, 0, 230] combine_rows_per_src=[12146, 16384, 15710, 3728, 8086, 16384, 0, 13974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=209100800 dispatch_meta_bytes=229216 combine_payload_bytes=209100800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=209330016 ingress_bytes=347512832 link_bytes=347512832 token_bytes=418430816 weight_movement_bytes=138412032 total_nvlink_bytes=556842848 weight_to_token_ratio=0.330788 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307662810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14163122856 prefetch_bytes=553648128 reduce_bytes=934281216 weight_movement_bytes=1487929344 egress_bytes=7085447848 ingress_bytes=8565604352 link_bytes=8565604352 total_nvlink_bytes=15651052200 weight_to_token_ratio=0.105057 prefetch_share_of_ingress=0.064636 dedup_ratio=0.112843 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=27 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030543 gemm_inner_pad_upper_bound=20828 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.401377373e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67352 payload_rows_remote=54384 dup_rows=30952 dup_rows_remote=30886 dedup_ratio=0.314860 router_tokens_per_expert=[0, 0, 3238, 0, 0, 0, 0, 0, 0, 251, 8192, 0, 0, 228, 5880, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1350, 94, 0, 0, 0, 0, 0, 1093, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4807, 0, 8192, 0, 0, 8029, 0, 0, 0, 0, 0, 70, 6447, 0, 0, 0] payload_rows_per_dst=[6476, 950, 192, 16384, 13970, 28, 16384, 12968] dup_rows_per_dst=[0, 8, 0, 2888, 2318, 0, 25672, 66] combine_rows_per_src=[6476, 950, 192, 16384, 13970, 28, 16384, 12968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14332769448 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=7170266280 ingress_bytes=8477417472 link_bytes=8477417472 total_nvlink_bytes=15647683752 weight_to_token_ratio=0.091742 prefetch_share_of_ingress=0.085718 dedup_ratio=0.103607 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=20955 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.386949542e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13328822400 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6668234880 ingress_bytes=8079310848 link_bytes=8079310848 total_nvlink_bytes=14747545728 weight_to_token_ratio=0.106440 prefetch_share_of_ingress=0.077092 dedup_ratio=0.155343 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=21463 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.321817230e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312909824 dispatch_meta_bytes=329368 combine_payload_bytes=312909824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=313239192 ingress_bytes=416718848 link_bytes=416718848 token_bytes=626149016 weight_movement_bytes=103809024 total_nvlink_bytes=729958040 weight_to_token_ratio=0.165790 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.568079476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=338100224 dispatch_meta_bytes=353696 combine_payload_bytes=338100224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338453920 ingress_bytes=372703232 link_bytes=372703232 token_bytes=676554144 weight_movement_bytes=34603008 total_nvlink_bytes=711157152 weight_to_token_ratio=0.051146 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.402452257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289595392 dispatch_meta_bytes=329400 combine_payload_bytes=289595392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289924792 ingress_bytes=324198400 link_bytes=324198400 token_bytes=579520184 weight_movement_bytes=34603008 total_nvlink_bytes=614123192 weight_to_token_ratio=0.059710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219932479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12615232216 prefetch_bytes=761266176 reduce_bytes=449839104 weight_movement_bytes=1211105280 egress_bytes=6311324376 ingress_bytes=7515013120 link_bytes=7515013120 total_nvlink_bytes=13826337496 weight_to_token_ratio=0.096003 prefetch_share_of_ingress=0.101299 dedup_ratio=0.184925 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=13 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=20955 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229495190e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=222756864 dispatch_meta_bytes=341080 combine_payload_bytes=222756864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=223097944 ingress_bytes=291962880 link_bytes=291962880 token_bytes=445854808 weight_movement_bytes=69206016 total_nvlink_bytes=515060824 weight_to_token_ratio=0.155221 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.098632813e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14652624888 prefetch_bytes=761266176 reduce_bytes=553648128 weight_movement_bytes=1314914304 egress_bytes=7330295800 ingress_bytes=8637243392 link_bytes=8637243392 total_nvlink_bytes=15967539192 weight_to_token_ratio=0.089739 prefetch_share_of_ingress=0.088138 dedup_ratio=0.102674 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=16 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030109 gemm_inner_pad_upper_bound=21717 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.413097893e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14228604152 prefetch_bytes=657457152 reduce_bytes=1072693248 weight_movement_bytes=1730150400 egress_bytes=7118210296 ingress_bytes=8840544256 link_bytes=8840544256 total_nvlink_bytes=15958754552 weight_to_token_ratio=0.121597 prefetch_share_of_ingress=0.074368 dedup_ratio=0.117412 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=31 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21082 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.446358970e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11456024440 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=5731536760 ingress_bytes=7004798976 link_bytes=7004798976 total_nvlink_bytes=12736335736 weight_to_token_ratio=0.111759 prefetch_share_of_ingress=0.108678 dedup_ratio=0.236215 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=21971 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.146021505e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82954 payload_rows_remote=70280 dup_rows=15350 dup_rows_remote=12972 dedup_ratio=0.156148 router_tokens_per_expert=[0, 0, 3292, 0, 0, 0, 0, 2, 0, 292, 8192, 0, 0, 300, 4987, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 92, 0, 0, 0, 0, 0, 1087, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4232, 0, 8192, 0, 0, 8050, 0, 0, 0, 0, 0, 213, 6640, 0, 0, 0] payload_rows_per_dst=[12674, 16384, 16100, 2880, 6104, 16384, 8984, 3444] dup_rows_per_dst=[2378, 12396, 140, 10, 354, 32, 0, 40] combine_rows_per_src=[12674, 16384, 16100, 2880, 6104, 16384, 8984, 3444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=287866880 dispatch_meta_bytes=333008 combine_payload_bytes=287866880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288199888 ingress_bytes=322469888 link_bytes=322469888 token_bytes=576066768 weight_movement_bytes=34603008 total_nvlink_bytes=610669776 weight_to_token_ratio=0.060068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213428228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12034199296 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6020722432 ingress_bytes=7397597184 link_bytes=7397597184 total_nvlink_bytes=13418319616 weight_to_token_ratio=0.115016 prefetch_share_of_ingress=0.098230 dedup_ratio=0.198791 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.031087 gemm_inner_pad_upper_bound=20320 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210285333e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-29 23:40:33] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 670.6 | throughput per GPU (TFLOP/s/GPU): 190.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477225E+00 | loss scale: 1.0 | grad norm: 7.147 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76560 payload_rows_remote=66692 dup_rows=21744 dup_rows_remote=19146 dedup_ratio=0.221191 router_tokens_per_expert=[2706, 548, 255, 988, 401, 808, 569, 185, 2558, 276, 423, 923, 39, 918, 534, 448, 732, 1110, 437, 553, 366, 263, 385, 968, 983, 940, 312, 397, 1025, 504, 1157, 1053, 1135, 625, 834, 1439, 1334, 800, 841, 664, 328, 671, 455, 351, 1026, 1530, 751, 171, 154, 436, 515, 1026, 716, 1925, 557, 365, 390, 649, 1417, 561, 223, 672, 1633, 1194] payload_rows_per_dst=[10166, 9312, 9454, 9926, 9868, 8316, 8952, 10566] dup_rows_per_dst=[2754, 2926, 3052, 2816, 2598, 2250, 2436, 2912] combine_rows_per_src=[10166, 9312, 9454, 9926, 9868, 8316, 8952, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75488 payload_rows_remote=65878 dup_rows=22816 dup_rows_remote=19834 dedup_ratio=0.232096 router_tokens_per_expert=[2635, 583, 277, 1111, 424, 798, 646, 176, 2683, 287, 383, 1096, 51, 786, 539, 471, 867, 1154, 400, 547, 409, 298, 404, 965, 1085, 909, 330, 422, 1086, 434, 1127, 1055, 1118, 667, 915, 1344, 1238, 845, 919, 586, 271, 648, 447, 319, 972, 1473, 805, 218, 159, 406, 551, 1131, 619, 1644, 507, 364, 371, 793, 1305, 572, 197, 605, 1555, 1150] payload_rows_per_dst=[11320, 9610, 9340, 9874, 8244, 8246, 10492, 8362] dup_rows_per_dst=[4456, 2982, 3436, 3022, 1856, 2060, 3380, 1624] combine_rows_per_src=[11320, 9610, 9340, 9874, 8244, 8246, 10492, 8362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74040 payload_rows_remote=63412 dup_rows=24264 dup_rows_remote=20772 dedup_ratio=0.246826 router_tokens_per_expert=[2867, 561, 226, 1147, 573, 858, 442, 100, 2603, 262, 404, 1136, 31, 820, 569, 538, 617, 1149, 426, 502, 366, 256, 398, 865, 1049, 833, 272, 358, 971, 446, 1124, 1006, 1211, 588, 802, 1476, 1382, 898, 826, 594, 229, 612, 439, 391, 927, 1510, 801, 155, 173, 459, 489, 1155, 825, 1749, 579, 269, 451, 637, 1315, 606, 198, 660, 1710, 1261] payload_rows_per_dst=[6370, 7658, 7654, 9362, 12064, 11374, 8930, 10628] dup_rows_per_dst=[1444, 2224, 1972, 2312, 5866, 4488, 2466, 3492] combine_rows_per_src=[6370, 7658, 7654, 9362, 12064, 11374, 8930, 10628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76846 payload_rows_remote=67326 dup_rows=21458 dup_rows_remote=19020 dedup_ratio=0.218282 router_tokens_per_expert=[2897, 616, 275, 1161, 479, 744, 414, 130, 2931, 236, 365, 1242, 41, 920, 454, 485, 709, 1118, 370, 547, 322, 266, 417, 1027, 1036, 909, 261, 364, 904, 403, 1136, 966, 1210, 592, 728, 1527, 1313, 799, 753, 588, 265, 684, 453, 322, 910, 1487, 821, 184, 158, 429, 489, 1107, 752, 2056, 515, 283, 393, 687, 1347, 611, 157, 637, 1668, 1082] payload_rows_per_dst=[10624, 10056, 9478, 9520, 9712, 8072, 9078, 10306] dup_rows_per_dst=[2808, 3292, 3128, 2438, 2254, 2180, 2500, 2858] combine_rows_per_src=[10624, 10056, 9478, 9520, 9712, 8072, 9078, 10306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75404 payload_rows_remote=64856 dup_rows=22900 dup_rows_remote=18944 dedup_ratio=0.232951 router_tokens_per_expert=[2774, 577, 229, 1165, 503, 789, 515, 130, 2764, 274, 435, 970, 21, 934, 526, 451, 749, 1165, 423, 618, 330, 293, 368, 981, 985, 882, 288, 437, 812, 538, 1141, 914, 1264, 591, 856, 1429, 1410, 813, 773, 533, 283, 593, 424, 337, 913, 1541, 822, 198, 155, 412, 547, 1020, 697, 1959, 540, 333, 423, 667, 1308, 593, 154, 636, 1777, 1170] payload_rows_per_dst=[7212, 9576, 9480, 9400, 9942, 10548, 8838, 10408] dup_rows_per_dst=[1870, 3174, 3232, 2594, 2538, 3956, 2488, 3048] combine_rows_per_src=[7212, 9576, 9480, 9400, 9942, 10548, 8838, 10408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76090 payload_rows_remote=66230 dup_rows=22214 dup_rows_remote=18960 dedup_ratio=0.225972 router_tokens_per_expert=[2738, 560, 275, 1263, 475, 882, 497, 119, 2640, 346, 409, 1023, 30, 840, 519, 415, 794, 1242, 467, 565, 369, 298, 378, 941, 989, 921, 215, 427, 899, 590, 1239, 1065, 1229, 602, 815, 1503, 1409, 738, 807, 578, 255, 634, 456, 312, 942, 1523, 782, 161, 164, 476, 460, 1142, 738, 1744, 496, 300, 413, 644, 1326, 544, 125, 656, 1574, 1174] payload_rows_per_dst=[11208, 9636, 9860, 9836, 8614, 8036, 9726, 9174] dup_rows_per_dst=[4200, 2808, 3254, 2854, 1952, 2094, 3026, 2026] combine_rows_per_src=[11208, 9636, 9860, 9836, 8614, 8036, 9726, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75710 payload_rows_remote=66950 dup_rows=22594 dup_rows_remote=20356 dedup_ratio=0.229838 router_tokens_per_expert=[2874, 533, 209, 1322, 396, 803, 476, 167, 2629, 265, 346, 1167, 24, 743, 501, 455, 754, 1245, 280, 598, 370, 226, 408, 900, 1072, 944, 267, 392, 906, 437, 1234, 1074, 1260, 656, 857, 1449, 1329, 755, 805, 653, 316, 676, 448, 392, 962, 1517, 837, 211, 146, 435, 543, 1135, 720, 1712, 501, 307, 423, 665, 1270, 612, 127, 604, 1591, 1221] payload_rows_per_dst=[6210, 9560, 9442, 9764, 9950, 11704, 8760, 10320] dup_rows_per_dst=[1602, 2700, 3018, 2888, 2680, 4762, 2238, 2706] combine_rows_per_src=[6210, 9560, 9442, 9764, 9950, 11704, 8760, 10320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273170432 dispatch_meta_bytes=343352 combine_payload_bytes=273170432 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273513784 ingress_bytes=376979456 link_bytes=376979456 token_bytes=546684216 weight_movement_bytes=103809024 total_nvlink_bytes=650493240 weight_to_token_ratio=0.189888 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418543344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269836288 dispatch_meta_bytes=342848 combine_payload_bytes=269836288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270179136 ingress_bytes=339042304 link_bytes=339042304 token_bytes=540015424 weight_movement_bytes=69206016 total_nvlink_bytes=609221440 weight_to_token_ratio=0.128156 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275788895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259735552 dispatch_meta_bytes=336736 combine_payload_bytes=259735552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260072288 ingress_bytes=328941568 link_bytes=328941568 token_bytes=519807840 weight_movement_bytes=69206016 total_nvlink_bytes=589013856 weight_to_token_ratio=0.133138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237780638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275767296 dispatch_meta_bytes=345384 combine_payload_bytes=275767296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276112680 ingress_bytes=344973312 link_bytes=344973312 token_bytes=551879976 weight_movement_bytes=69206016 total_nvlink_bytes=621085992 weight_to_token_ratio=0.125400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298106800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=335200 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265985376 ingress_bytes=300253184 link_bytes=300253184 token_bytes=531635552 weight_movement_bytes=34603008 total_nvlink_bytes=566238560 weight_to_token_ratio=0.065088 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129828559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=340760 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271618840 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542896920 weight_movement_bytes=34603008 total_nvlink_bytes=577499928 weight_to_token_ratio=0.063738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274227200 dispatch_meta_bytes=349224 combine_payload_bytes=274227200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274576424 ingress_bytes=308830208 link_bytes=308830208 token_bytes=548803624 weight_movement_bytes=34603008 total_nvlink_bytes=583406632 weight_to_token_ratio=0.063052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162103210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75492 payload_rows_remote=63930 dup_rows=22812 dup_rows_remote=18314 dedup_ratio=0.232056 router_tokens_per_expert=[2929, 581, 190, 1087, 445, 766, 434, 142, 2750, 261, 456, 1030, 27, 904, 522, 398, 670, 1138, 388, 563, 345, 311, 453, 987, 899, 817, 290, 391, 889, 515, 1139, 956, 1339, 604, 866, 1397, 1456, 841, 860, 573, 267, 724, 459, 347, 898, 1430, 828, 156, 150, 463, 546, 1105, 741, 2069, 506, 284, 418, 677, 1397, 541, 180, 677, 1633, 1047] payload_rows_per_dst=[11562, 9684, 9450, 9346, 8222, 8032, 10922, 8274] dup_rows_per_dst=[4498, 3106, 3054, 2352, 1944, 2186, 4072, 1600] combine_rows_per_src=[11562, 9684, 9450, 9346, 8222, 8032, 10922, 8274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76902 payload_rows_remote=65762 dup_rows=21402 dup_rows_remote=19802 dedup_ratio=0.217712 router_tokens_per_expert=[324, 890, 370, 586, 2342, 105, 4292, 326, 1105, 646, 364, 981, 552, 1141, 62, 1408, 600, 1018, 890, 1900, 40, 630, 29, 391, 98, 772, 953, 2305, 69, 702, 286, 2395, 135, 55, 562, 211, 597, 27, 350, 141, 884, 578, 319, 830, 192, 158, 301, 1212, 2266, 245, 1805, 435, 429, 165, 919, 160, 841, 1012, 1335, 1277, 649, 540, 1052, 898] payload_rows_per_dst=[7856, 9290, 8770, 11372, 11140, 9104, 10008, 9362] dup_rows_per_dst=[2030, 3228, 2226, 3788, 1600, 2514, 2840, 3176] combine_rows_per_src=[7856, 9290, 8770, 11372, 11140, 9104, 10008, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75682 payload_rows_remote=66040 dup_rows=22622 dup_rows_remote=19428 dedup_ratio=0.230123 router_tokens_per_expert=[304, 964, 359, 621, 2297, 70, 4512, 335, 1260, 629, 386, 1042, 577, 1103, 55, 1366, 565, 954, 976, 1799, 43, 559, 14, 400, 116, 796, 1000, 2187, 50, 654, 229, 2270, 142, 35, 601, 235, 573, 24, 408, 149, 677, 572, 312, 905, 264, 155, 325, 1317, 1975, 262, 1835, 375, 431, 188, 948, 139, 796, 1000, 1414, 1313, 721, 590, 997, 982] payload_rows_per_dst=[10040, 9642, 10588, 5152, 11540, 10430, 9792, 8498] dup_rows_per_dst=[4400, 3194, 4192, 752, 1818, 4078, 2514, 1674] combine_rows_per_src=[10040, 9642, 10588, 5152, 11540, 10430, 9792, 8498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76774 payload_rows_remote=66074 dup_rows=21530 dup_rows_remote=16774 dedup_ratio=0.219014 router_tokens_per_expert=[354, 933, 376, 663, 2426, 93, 4775, 286, 1355, 598, 318, 959, 568, 1049, 52, 1337, 586, 1092, 1273, 1914, 21, 535, 10, 378, 111, 738, 891, 1953, 43, 580, 253, 2583, 39, 18, 681, 213, 512, 26, 270, 149, 419, 531, 323, 832, 222, 156, 263, 1346, 1972, 219, 1478, 540, 477, 162, 826, 124, 1000, 994, 1455, 1453, 716, 529, 1098, 1006] payload_rows_per_dst=[7994, 8598, 8800, 11048, 11664, 8540, 9430, 10700] dup_rows_per_dst=[2268, 2010, 2818, 3256, 1702, 2554, 2166, 4756] combine_rows_per_src=[7994, 8598, 8800, 11048, 11664, 8540, 9430, 10700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=261857280 dispatch_meta_bytes=328976 combine_payload_bytes=261857280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262186256 ingress_bytes=331063296 link_bytes=331063296 token_bytes=524043536 weight_movement_bytes=69206016 total_nvlink_bytes=593249552 weight_to_token_ratio=0.132062 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245764530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77190 payload_rows_remote=65860 dup_rows=21114 dup_rows_remote=17326 dedup_ratio=0.214783 router_tokens_per_expert=[318, 825, 332, 663, 2235, 120, 4431, 271, 1257, 665, 242, 1062, 585, 1070, 48, 1389, 548, 944, 903, 2069, 37, 554, 25, 422, 101, 718, 948, 2407, 30, 595, 213, 2547, 97, 37, 575, 223, 551, 29, 353, 138, 779, 538, 324, 854, 180, 190, 301, 1284, 2207, 233, 1639, 448, 502, 189, 983, 127, 938, 933, 1463, 1304, 581, 562, 1026, 990] payload_rows_per_dst=[7658, 9508, 8802, 11330, 11200, 9300, 9876, 9516] dup_rows_per_dst=[1870, 3128, 2202, 3788, 1668, 2526, 2780, 3152] combine_rows_per_src=[7658, 9508, 8802, 11330, 11200, 9300, 9876, 9516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76470 payload_rows_remote=67328 dup_rows=21834 dup_rows_remote=18860 dedup_ratio=0.222107 router_tokens_per_expert=[347, 901, 293, 644, 2372, 89, 4521, 324, 1031, 543, 319, 1119, 547, 1045, 63, 1498, 557, 997, 1308, 2093, 22, 659, 29, 393, 130, 722, 927, 2166, 44, 646, 246, 2382, 102, 20, 507, 185, 528, 16, 380, 121, 812, 532, 287, 813, 224, 153, 280, 1146, 2400, 241, 1939, 406, 476, 156, 871, 131, 972, 1010, 1319, 1006, 582, 493, 1082, 985] payload_rows_per_dst=[9502, 9326, 9142, 8716, 11222, 8966, 10328, 9268] dup_rows_per_dst=[4206, 3004, 2974, 2042, 1538, 2690, 2912, 2468] combine_rows_per_src=[9502, 9326, 9142, 8716, 11222, 8966, 10328, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76812 payload_rows_remote=67608 dup_rows=21492 dup_rows_remote=18748 dedup_ratio=0.218628 router_tokens_per_expert=[307, 882, 349, 547, 2257, 115, 4649, 325, 1028, 598, 334, 1177, 564, 1034, 42, 1346, 531, 997, 1225, 1890, 16, 614, 16, 322, 94, 704, 877, 2475, 33, 643, 212, 2420, 96, 42, 524, 162, 531, 11, 389, 138, 563, 603, 299, 867, 215, 154, 285, 1372, 2489, 227, 1592, 405, 488, 151, 864, 120, 877, 988, 1616, 1263, 590, 536, 1168, 904] payload_rows_per_dst=[7646, 9140, 8760, 11350, 11546, 9204, 9838, 9328] dup_rows_per_dst=[1918, 3106, 2462, 3566, 1538, 2744, 2834, 3324] combine_rows_per_src=[7646, 9140, 8760, 11350, 11546, 9204, 9838, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269361152 dispatch_meta_bytes=342256 combine_payload_bytes=269361152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269703408 ingress_bytes=303964160 link_bytes=303964160 token_bytes=539064560 weight_movement_bytes=34603008 total_nvlink_bytes=573667568 weight_to_token_ratio=0.064191 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143792663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77122 payload_rows_remote=67084 dup_rows=21182 dup_rows_remote=18354 dedup_ratio=0.215474 router_tokens_per_expert=[299, 887, 401, 665, 2212, 89, 4673, 317, 1155, 670, 279, 1018, 525, 1170, 76, 1385, 525, 988, 875, 1789, 26, 555, 16, 354, 135, 658, 831, 2104, 60, 651, 216, 2483, 119, 28, 579, 270, 492, 31, 427, 125, 713, 588, 286, 916, 240, 142, 280, 1318, 2141, 225, 1897, 527, 485, 153, 896, 109, 825, 982, 1547, 1488, 593, 562, 1112, 969] payload_rows_per_dst=[7800, 9618, 8250, 10900, 11666, 9292, 10038, 9558] dup_rows_per_dst=[1940, 2938, 2006, 3376, 1822, 2768, 2828, 3504] combine_rows_per_src=[7800, 9618, 8250, 10900, 11666, 9292, 10038, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270499840 dispatch_meta_bytes=341872 combine_payload_bytes=270499840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270841712 ingress_bytes=339705856 link_bytes=339705856 token_bytes=541341552 weight_movement_bytes=69206016 total_nvlink_bytes=610547568 weight_to_token_ratio=0.127842 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278285788e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270639104 dispatch_meta_bytes=331392 combine_payload_bytes=270639104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270970496 ingress_bytes=374448128 link_bytes=374448128 token_bytes=541609600 weight_movement_bytes=103809024 total_nvlink_bytes=645418624 weight_to_token_ratio=0.191668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409018160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269762560 dispatch_meta_bytes=332744 combine_payload_bytes=269762560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270095304 ingress_bytes=373571584 link_bytes=373571584 token_bytes=539857864 weight_movement_bytes=103809024 total_nvlink_bytes=643666888 weight_to_token_ratio=0.192290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405719796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275775488 dispatch_meta_bytes=344752 combine_payload_bytes=275775488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276120240 ingress_bytes=310378496 link_bytes=310378496 token_bytes=551895728 weight_movement_bytes=34603008 total_nvlink_bytes=586498736 weight_to_token_ratio=0.062698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167929293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276922368 dispatch_meta_bytes=345424 combine_payload_bytes=276922368 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277267792 ingress_bytes=346128384 link_bytes=346128384 token_bytes=554190160 weight_movement_bytes=69206016 total_nvlink_bytes=623396176 weight_to_token_ratio=0.124878 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302453243e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274776064 dispatch_meta_bytes=341752 combine_payload_bytes=274776064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275117816 ingress_bytes=343982080 link_bytes=343982080 token_bytes=549893880 weight_movement_bytes=69206016 total_nvlink_bytes=619099896 weight_to_token_ratio=0.125853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294376874e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72846 payload_rows_remote=60492 dup_rows=25458 dup_rows_remote=16636 dedup_ratio=0.258972 router_tokens_per_expert=[370, 954, 342, 661, 2412, 84, 4672, 316, 1142, 626, 329, 946, 527, 1043, 50, 1498, 577, 977, 1273, 1996, 44, 610, 15, 348, 115, 784, 881, 2316, 44, 559, 257, 2334, 85, 36, 509, 159, 535, 10, 301, 128, 629, 598, 288, 810, 209, 152, 305, 1340, 2410, 244, 1813, 397, 458, 174, 838, 146, 875, 1020, 1390, 1076, 577, 551, 1097, 890] payload_rows_per_dst=[12354, 9398, 11248, 6806, 6030, 10256, 8200, 8554] dup_rows_per_dst=[8822, 3176, 5064, 1062, 610, 3338, 1920, 1466] combine_rows_per_src=[12354, 9398, 11248, 6806, 6030, 10256, 8200, 8554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77776 payload_rows_remote=66982 dup_rows=20528 dup_rows_remote=17646 dedup_ratio=0.208822 router_tokens_per_expert=[526, 302, 532, 1313, 767, 76, 1122, 781, 1077, 1110, 1423, 733, 704, 1461, 158, 274, 3022, 878, 4325, 330, 743, 51, 1401, 1854, 342, 722, 625, 214, 210, 1193, 913, 924, 372, 367, 872, 1108, 322, 361, 564, 208, 1186, 514, 40, 265, 287, 761, 356, 115, 2664, 1260, 470, 983, 102, 1063, 1064, 422, 623, 739, 377, 31, 282, 315, 16, 937] payload_rows_per_dst=[8610, 8884, 8530, 10098, 10794, 12060, 8658, 10142] dup_rows_per_dst=[2228, 2150, 1984, 3034, 2882, 3638, 2070, 2542] combine_rows_per_src=[8610, 8884, 8530, 10098, 10794, 12060, 8658, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76158 payload_rows_remote=65980 dup_rows=22146 dup_rows_remote=18300 dedup_ratio=0.225281 router_tokens_per_expert=[510, 295, 508, 1367, 959, 62, 1051, 692, 1203, 774, 1751, 752, 621, 1502, 177, 232, 3172, 1094, 4449, 371, 588, 35, 1339, 1638, 342, 750, 616, 186, 207, 1231, 904, 904, 338, 322, 866, 1286, 263, 431, 481, 259, 1191, 583, 27, 265, 275, 751, 311, 64, 2740, 1151, 446, 934, 127, 1058, 1177, 339, 612, 896, 358, 41, 213, 296, 19, 750] payload_rows_per_dst=[10176, 10178, 12264, 8536, 11044, 6336, 8390, 9234] dup_rows_per_dst=[3988, 3846, 4762, 1744, 2928, 598, 2074, 2206] combine_rows_per_src=[10176, 10178, 12264, 8536, 11044, 6336, 8390, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75056 payload_rows_remote=64972 dup_rows=23248 dup_rows_remote=20684 dedup_ratio=0.236491 router_tokens_per_expert=[692, 283, 438, 1476, 1067, 76, 1047, 610, 1214, 853, 1943, 680, 680, 1585, 127, 264, 3098, 852, 4396, 237, 662, 42, 1381, 1667, 376, 650, 496, 146, 253, 1152, 769, 1140, 414, 278, 889, 1035, 272, 463, 415, 296, 1082, 518, 20, 209, 264, 656, 366, 82, 3053, 1256, 395, 936, 156, 1138, 946, 435, 522, 1083, 293, 31, 245, 297, 15, 740] payload_rows_per_dst=[9026, 8754, 7888, 10482, 6498, 9308, 13016, 10084] dup_rows_per_dst=[2352, 2052, 1794, 3368, 1626, 2086, 7406, 2564] combine_rows_per_src=[9026, 8754, 7888, 10482, 6498, 9308, 13016, 10084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77982 payload_rows_remote=67636 dup_rows=20322 dup_rows_remote=16932 dedup_ratio=0.206726 router_tokens_per_expert=[568, 291, 552, 1454, 949, 48, 1178, 780, 1089, 991, 1974, 722, 664, 1507, 139, 310, 3415, 785, 4270, 322, 466, 25, 1511, 1728, 423, 636, 470, 194, 218, 1175, 759, 1019, 367, 288, 978, 1248, 228, 466, 501, 247, 1054, 485, 15, 257, 252, 616, 351, 46, 2658, 1259, 402, 945, 121, 1016, 1004, 399, 623, 1040, 283, 39, 233, 298, 6, 795] payload_rows_per_dst=[9186, 8836, 7972, 10346, 10900, 11628, 8446, 10668] dup_rows_per_dst=[2454, 2008, 1702, 3390, 3062, 3064, 1846, 2796] combine_rows_per_src=[9186, 8836, 7972, 10346, 10900, 11628, 8446, 10668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77430 payload_rows_remote=68880 dup_rows=20874 dup_rows_remote=18714 dedup_ratio=0.212341 router_tokens_per_expert=[406, 370, 338, 1246, 779, 64, 1252, 744, 1170, 869, 1809, 775, 637, 1495, 106, 355, 3655, 969, 4375, 288, 587, 45, 1585, 1783, 462, 658, 593, 171, 253, 1372, 964, 730, 384, 221, 786, 1013, 431, 438, 244, 147, 1222, 457, 15, 237, 216, 754, 273, 80, 3138, 1149, 246, 1035, 89, 883, 1074, 363, 544, 923, 332, 34, 205, 298, 17, 999] payload_rows_per_dst=[9910, 10346, 8550, 8684, 10934, 9846, 8066, 11094] dup_rows_per_dst=[3324, 4044, 2160, 1764, 2670, 2380, 1612, 2920] combine_rows_per_src=[9910, 10346, 8550, 8684, 10934, 9846, 8066, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274358272 dispatch_meta_bytes=338512 combine_payload_bytes=274358272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274696784 ingress_bytes=308961280 link_bytes=308961280 token_bytes=549055056 weight_movement_bytes=34603008 total_nvlink_bytes=583658064 weight_to_token_ratio=0.063023 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162596423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=247775232 dispatch_meta_bytes=308512 combine_payload_bytes=247775232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248083744 ingress_bytes=316981248 link_bytes=316981248 token_bytes=495858976 weight_movement_bytes=69206016 total_nvlink_bytes=565064992 weight_to_token_ratio=0.139568 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192774917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76992 payload_rows_remote=65070 dup_rows=21312 dup_rows_remote=17040 dedup_ratio=0.216797 router_tokens_per_expert=[648, 292, 522, 1463, 908, 60, 1171, 694, 1280, 1037, 1843, 615, 652, 1403, 144, 224, 3479, 853, 4308, 323, 602, 22, 1381, 1730, 403, 648, 445, 136, 191, 1227, 831, 961, 330, 266, 810, 1118, 296, 524, 418, 172, 1250, 499, 12, 273, 253, 646, 303, 60, 3150, 1326, 350, 889, 125, 965, 945, 347, 619, 797, 333, 20, 259, 301, 11, 989] payload_rows_per_dst=[9126, 8756, 8096, 10140, 6468, 11708, 11922, 10776] dup_rows_per_dst=[2390, 1954, 1726, 3230, 1400, 3500, 4272, 2840] combine_rows_per_src=[9126, 8756, 8096, 10140, 6468, 11708, 11922, 10776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78186 payload_rows_remote=65868 dup_rows=20118 dup_rows_remote=16482 dedup_ratio=0.204651 router_tokens_per_expert=[564, 243, 391, 1226, 854, 46, 1243, 834, 954, 1071, 1914, 758, 634, 1436, 115, 274, 3411, 819, 4695, 257, 434, 19, 1545, 1833, 423, 565, 595, 231, 260, 1283, 811, 819, 375, 284, 1029, 1147, 374, 357, 283, 180, 1177, 546, 14, 276, 210, 694, 325, 40, 2836, 1193, 423, 951, 76, 944, 1140, 315, 624, 988, 270, 23, 270, 310, 5, 916] payload_rows_per_dst=[8660, 8406, 8068, 10534, 10800, 12318, 8496, 10904] dup_rows_per_dst=[2142, 2078, 1746, 3268, 2646, 3636, 1872, 2730] combine_rows_per_src=[8660, 8406, 8068, 10534, 10800, 12318, 8496, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270254080 dispatch_meta_bytes=337120 combine_payload_bytes=270254080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270591200 ingress_bytes=339460096 link_bytes=339460096 token_bytes=540845280 weight_movement_bytes=69206016 total_nvlink_bytes=610051296 weight_to_token_ratio=0.127959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277361012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=342624 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266467936 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532593248 weight_movement_bytes=34603008 total_nvlink_bytes=567196256 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277037056 dispatch_meta_bytes=338272 combine_payload_bytes=277037056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277375328 ingress_bytes=311640064 link_bytes=311640064 token_bytes=554412384 weight_movement_bytes=34603008 total_nvlink_bytes=589015392 weight_to_token_ratio=0.062414 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172676472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282132480 dispatch_meta_bytes=350376 combine_payload_bytes=282132480 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=282482856 ingress_bytes=455147520 link_bytes=455147520 token_bytes=564615336 weight_movement_bytes=173015040 total_nvlink_bytes=737630376 weight_to_token_ratio=0.306430 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.712683475e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=328440 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266855160 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533381880 weight_movement_bytes=69206016 total_nvlink_bytes=602587896 weight_to_token_ratio=0.129749 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269795328 dispatch_meta_bytes=329400 combine_payload_bytes=269795328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270124728 ingress_bytes=304398336 link_bytes=304398336 token_bytes=539920056 weight_movement_bytes=34603008 total_nvlink_bytes=574523064 weight_to_token_ratio=0.064089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145426432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75254 payload_rows_remote=67638 dup_rows=23050 dup_rows_remote=21654 dedup_ratio=0.234477 router_tokens_per_expert=[152, 65, 8, 1549, 1227, 2779, 683, 127, 1045, 869, 303, 325, 448, 184, 730, 258, 525, 261, 4, 4, 1113, 169, 478, 134, 3399, 1313, 128, 742, 1039, 372, 1282, 1475, 15, 262, 2060, 428, 4557, 216, 1259, 266, 629, 389, 900, 712, 354, 241, 1, 483, 751, 472, 914, 391, 3793, 79, 597, 159, 697, 43, 40, 1340, 46, 1597, 145, 2126] payload_rows_per_dst=[9706, 8366, 10014, 9892, 7616, 11322, 8912, 9426] dup_rows_per_dst=[3474, 2800, 2160, 2810, 1396, 5210, 2558, 2642] combine_rows_per_src=[9706, 8366, 10014, 9892, 7616, 11322, 8912, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75426 payload_rows_remote=64850 dup_rows=22878 dup_rows_remote=18722 dedup_ratio=0.232727 router_tokens_per_expert=[546, 287, 515, 1405, 756, 63, 1216, 838, 1034, 906, 1901, 745, 637, 1305, 122, 296, 3575, 846, 4769, 301, 439, 37, 1624, 1740, 294, 612, 513, 148, 248, 1272, 845, 911, 382, 269, 927, 1011, 395, 390, 414, 137, 1058, 470, 11, 227, 190, 724, 352, 67, 2901, 1136, 511, 967, 87, 934, 1001, 443, 641, 1115, 211, 38, 228, 277, 11, 881] payload_rows_per_dst=[10576, 11102, 13782, 8240, 11020, 5752, 8464, 6490] dup_rows_per_dst=[4156, 4910, 7280, 1446, 2632, 446, 1694, 314] combine_rows_per_src=[10576, 11102, 13782, 8240, 11020, 5752, 8464, 6490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74828 payload_rows_remote=63510 dup_rows=23476 dup_rows_remote=18746 dedup_ratio=0.238810 router_tokens_per_expert=[120, 64, 4, 1738, 1141, 3092, 408, 129, 1140, 823, 393, 214, 472, 428, 526, 218, 428, 218, 6, 8, 975, 181, 578, 116, 3359, 1392, 168, 736, 838, 341, 1078, 1522, 10, 163, 2545, 473, 4577, 154, 1223, 263, 610, 421, 978, 627, 324, 324, 2, 492, 737, 478, 1008, 397, 3810, 61, 958, 146, 727, 45, 30, 1285, 18, 1292, 126, 1994] payload_rows_per_dst=[10358, 11318, 10670, 7972, 13376, 5634, 6624, 8876] dup_rows_per_dst=[3034, 4730, 3886, 1360, 5440, 1922, 946, 2158] combine_rows_per_src=[10358, 11318, 10670, 7972, 13376, 5634, 6624, 8876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=65914 dup_rows=22998 dup_rows_remote=20236 dedup_ratio=0.233948 router_tokens_per_expert=[96, 63, 2, 1816, 1156, 3130, 361, 93, 894, 738, 277, 158, 511, 505, 792, 188, 207, 239, 6, 1, 1235, 128, 464, 129, 3547, 1471, 6, 818, 1040, 369, 985, 1388, 18, 188, 2385, 467, 5181, 163, 1037, 271, 545, 393, 886, 603, 346, 198, 0, 800, 781, 429, 929, 474, 4071, 86, 682, 148, 577, 49, 33, 1038, 36, 1427, 108, 1990] payload_rows_per_dst=[10284, 11754, 9954, 9392, 13072, 6326, 6114, 8410] dup_rows_per_dst=[3150, 4514, 1958, 2762, 5150, 2414, 944, 2106] combine_rows_per_src=[10284, 11754, 9954, 9392, 13072, 6326, 6114, 8410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71624 payload_rows_remote=59804 dup_rows=26680 dup_rows_remote=20992 dedup_ratio=0.271403 router_tokens_per_expert=[6, 46, 1, 2073, 1007, 3098, 275, 116, 833, 760, 299, 165, 451, 488, 554, 157, 150, 252, 1, 5, 1080, 135, 531, 127, 3437, 1795, 11, 804, 744, 329, 866, 1776, 9, 131, 2751, 625, 4942, 101, 888, 246, 566, 375, 871, 528, 289, 252, 3, 632, 963, 426, 984, 413, 4007, 86, 844, 192, 631, 64, 40, 1286, 18, 1270, 122, 2225] payload_rows_per_dst=[6334, 5958, 9592, 9672, 3810, 11564, 12874, 11820] dup_rows_per_dst=[714, 1456, 1844, 2978, 190, 5352, 8458, 5688] combine_rows_per_src=[6334, 5958, 9592, 9672, 3810, 11564, 12874, 11820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75664 payload_rows_remote=66304 dup_rows=22640 dup_rows_remote=21042 dedup_ratio=0.230306 router_tokens_per_expert=[129, 32, 6, 1616, 1468, 3372, 705, 117, 1015, 753, 382, 133, 422, 115, 992, 220, 507, 238, 7, 2, 1033, 138, 269, 159, 3126, 1388, 78, 869, 1121, 404, 1238, 1306, 5, 165, 2206, 387, 4834, 191, 1213, 236, 546, 416, 950, 625, 209, 307, 3, 576, 626, 398, 999, 462, 4122, 136, 472, 102, 568, 50, 35, 925, 43, 1437, 237, 2311] payload_rows_per_dst=[10702, 12290, 9360, 9930, 13360, 5586, 5642, 8794] dup_rows_per_dst=[4188, 4018, 1598, 2878, 5114, 1678, 748, 2418] combine_rows_per_src=[10702, 12290, 9360, 9930, 13360, 5586, 5642, 8794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277045248 dispatch_meta_bytes=357168 combine_payload_bytes=277045248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277402416 ingress_bytes=380854272 link_bytes=380854272 token_bytes=554447664 weight_movement_bytes=103809024 total_nvlink_bytes=658256688 weight_to_token_ratio=0.187230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433123964e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72144 payload_rows_remote=59590 dup_rows=26160 dup_rows_remote=18484 dedup_ratio=0.266113 router_tokens_per_expert=[103, 62, 6, 1725, 1288, 3362, 595, 51, 923, 751, 297, 150, 388, 235, 686, 121, 347, 219, 1, 10, 1059, 132, 375, 124, 3495, 1460, 24, 860, 1045, 406, 1220, 1415, 6, 179, 2333, 406, 5121, 228, 873, 198, 435, 440, 978, 624, 283, 227, 2, 607, 849, 396, 1013, 433, 4113, 86, 748, 144, 682, 55, 27, 1123, 32, 1219, 132, 2225] payload_rows_per_dst=[7872, 5678, 9796, 10916, 3792, 11850, 12554, 9686] dup_rows_per_dst=[1666, 1424, 1728, 4522, 260, 5584, 7676, 3300] combine_rows_per_src=[7872, 5678, 9796, 10916, 3792, 11850, 12554, 9686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=62712 dup_rows=23904 dup_rows_remote=18450 dedup_ratio=0.243164 router_tokens_per_expert=[103, 55, 2, 1615, 1290, 2788, 832, 91, 923, 728, 365, 173, 400, 146, 923, 96, 404, 260, 2, 1, 1260, 147, 255, 137, 3142, 1495, 63, 765, 1366, 379, 1210, 1420, 4, 113, 2315, 446, 5011, 160, 1012, 245, 473, 452, 643, 753, 356, 290, 0, 593, 712, 365, 1165, 408, 4439, 90, 605, 96, 541, 25, 55, 1103, 17, 1364, 103, 2362] payload_rows_per_dst=[9904, 6100, 9576, 10214, 6046, 11688, 11950, 8922] dup_rows_per_dst=[3648, 1408, 1640, 3182, 722, 5454, 5632, 2218] combine_rows_per_src=[9904, 6100, 9576, 10214, 6046, 11688, 11950, 8922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=334288 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265959888 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531585488 weight_movement_bytes=34603008 total_nvlink_bytes=566188496 weight_to_token_ratio=0.065094 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=260136960 dispatch_meta_bytes=329024 combine_payload_bytes=260136960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260465984 ingress_bytes=294739968 link_bytes=294739968 token_bytes=520602944 weight_movement_bytes=34603008 total_nvlink_bytes=555205952 weight_to_token_ratio=0.066467 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109082771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269983744 dispatch_meta_bytes=344600 combine_payload_bytes=269983744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270328344 ingress_bytes=373792768 link_bytes=373792768 token_bytes=540312088 weight_movement_bytes=103809024 total_nvlink_bytes=644121112 weight_to_token_ratio=0.192128 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406552093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=244957184 dispatch_meta_bytes=323184 combine_payload_bytes=244957184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245280368 ingress_bytes=279560192 link_bytes=279560192 token_bytes=490237552 weight_movement_bytes=34603008 total_nvlink_bytes=524840560 weight_to_token_ratio=0.070584 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051962496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271581184 dispatch_meta_bytes=349384 combine_payload_bytes=271581184 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=271930568 ingress_bytes=340787200 link_bytes=340787200 token_bytes=543511752 weight_movement_bytes=69206016 total_nvlink_bytes=612717768 weight_to_token_ratio=0.127331 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282354798e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=244080640 dispatch_meta_bytes=312296 combine_payload_bytes=244080640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=244392936 ingress_bytes=313286656 link_bytes=313286656 token_bytes=488473576 weight_movement_bytes=69206016 total_nvlink_bytes=557679592 weight_to_token_ratio=0.141678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178872465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=256868352 dispatch_meta_bytes=324648 combine_payload_bytes=256868352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257193000 ingress_bytes=291471360 link_bytes=291471360 token_bytes=514061352 weight_movement_bytes=34603008 total_nvlink_bytes=548664360 weight_to_token_ratio=0.067313 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096783262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74668 payload_rows_remote=63918 dup_rows=23636 dup_rows_remote=18512 dedup_ratio=0.240438 router_tokens_per_expert=[1702, 2444, 1712, 101, 237, 8, 1, 212, 297, 1901, 6, 859, 146, 1378, 384, 109, 837, 545, 92, 526, 626, 355, 487, 1315, 1381, 622, 96, 1279, 4, 1478, 8, 3536, 497, 1634, 539, 403, 2663, 15, 771, 475, 10, 1371, 675, 1195, 4, 631, 326, 1878, 624, 2728, 300, 322, 1225, 472, 660, 537, 928, 29, 360, 827, 553, 1265, 413, 138] payload_rows_per_dst=[9694, 8228, 10160, 7980, 10750, 9804, 10236, 7816] dup_rows_per_dst=[3140, 1932, 3784, 1756, 5124, 2376, 3500, 2024] combine_rows_per_src=[9694, 8228, 10160, 7980, 10750, 9804, 10236, 7816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75280 payload_rows_remote=65330 dup_rows=23024 dup_rows_remote=19704 dedup_ratio=0.234212 router_tokens_per_expert=[121, 13, 3, 1462, 1373, 2888, 665, 110, 1105, 754, 366, 99, 409, 284, 844, 146, 452, 273, 0, 7, 1329, 187, 241, 129, 3098, 1344, 159, 857, 962, 419, 1171, 1548, 4, 146, 2590, 413, 4871, 196, 1002, 258, 440, 450, 661, 670, 371, 280, 1, 487, 688, 347, 1059, 415, 4238, 86, 574, 167, 562, 31, 68, 1269, 29, 1655, 111, 2195] payload_rows_per_dst=[9950, 12102, 10718, 8362, 13622, 5184, 5884, 9458] dup_rows_per_dst=[3320, 4388, 3810, 1462, 5338, 1536, 788, 2382] combine_rows_per_src=[9950, 12102, 10718, 8362, 13622, 5184, 5884, 9458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261808128 dispatch_meta_bytes=329720 combine_payload_bytes=261808128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262137848 ingress_bytes=331014144 link_bytes=331014144 token_bytes=523945976 weight_movement_bytes=69206016 total_nvlink_bytes=593151992 weight_to_token_ratio=0.132086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245579575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=65794 dup_rows=23570 dup_rows_remote=21060 dedup_ratio=0.239766 router_tokens_per_expert=[2375, 2309, 1590, 144, 243, 2, 0, 304, 389, 1983, 0, 660, 114, 2142, 366, 71, 960, 376, 201, 257, 859, 276, 553, 1214, 1366, 668, 75, 980, 0, 1269, 13, 3940, 528, 1741, 598, 261, 1879, 3, 649, 462, 18, 1090, 623, 1197, 0, 983, 202, 1782, 849, 3064, 122, 341, 1201, 499, 492, 476, 1211, 12, 280, 634, 453, 1354, 316, 133] payload_rows_per_dst=[9954, 8940, 9832, 11984, 6956, 11698, 8032, 7338] dup_rows_per_dst=[3980, 2510, 3318, 4638, 1528, 3916, 2232, 1448] combine_rows_per_src=[9954, 8940, 9832, 11984, 6956, 11698, 8032, 7338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75286 payload_rows_remote=66040 dup_rows=23018 dup_rows_remote=20358 dedup_ratio=0.234151 router_tokens_per_expert=[2458, 2563, 1599, 86, 33, 0, 0, 285, 345, 1914, 1, 803, 161, 2214, 398, 75, 1103, 370, 118, 406, 682, 260, 403, 1029, 1427, 574, 79, 1301, 2, 1380, 10, 3779, 474, 1764, 876, 309, 2347, 0, 829, 382, 21, 814, 648, 1056, 0, 559, 193, 1652, 477, 3323, 171, 345, 1145, 309, 518, 498, 1111, 19, 235, 718, 582, 1499, 301, 119] payload_rows_per_dst=[10166, 9264, 10120, 9246, 9918, 8216, 10712, 7644] dup_rows_per_dst=[3882, 2558, 3316, 2660, 4548, 1670, 2860, 1524] combine_rows_per_src=[10166, 9264, 10120, 9246, 9918, 8216, 10712, 7644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74974 payload_rows_remote=64052 dup_rows=23330 dup_rows_remote=18788 dedup_ratio=0.237325 router_tokens_per_expert=[1649, 2369, 1656, 79, 132, 0, 2, 279, 313, 2173, 0, 962, 174, 1395, 443, 66, 882, 549, 20, 771, 664, 336, 436, 1300, 1395, 639, 98, 1087, 3, 1550, 11, 3756, 333, 1530, 443, 550, 2774, 0, 780, 457, 8, 1566, 607, 1621, 0, 470, 250, 1591, 385, 2882, 171, 258, 1138, 221, 755, 524, 1076, 5, 280, 800, 467, 1353, 549, 119] payload_rows_per_dst=[9422, 8562, 10922, 12214, 6652, 9820, 9824, 7558] dup_rows_per_dst=[2910, 2490, 4542, 4864, 1534, 2406, 2844, 1740] combine_rows_per_src=[9422, 8562, 10922, 12214, 6652, 9820, 9824, 7558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74794 payload_rows_remote=62266 dup_rows=23510 dup_rows_remote=17862 dedup_ratio=0.239156 router_tokens_per_expert=[2914, 1963, 1517, 94, 22, 0, 0, 239, 351, 1786, 0, 685, 149, 2652, 561, 82, 1025, 344, 163, 223, 691, 241, 416, 843, 1073, 579, 83, 844, 1, 1210, 4, 4200, 732, 2055, 1086, 170, 2053, 0, 696, 476, 6, 777, 564, 842, 1, 846, 218, 1867, 709, 3454, 114, 307, 1138, 389, 350, 459, 1237, 8, 284, 762, 627, 1621, 216, 133] payload_rows_per_dst=[7378, 11160, 6640, 7206, 10442, 8726, 10714, 12528] dup_rows_per_dst=[910, 5298, 1252, 1666, 4094, 1516, 3126, 5648] combine_rows_per_src=[7378, 11160, 6640, 7206, 10442, 8726, 10714, 12528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74726 payload_rows_remote=66290 dup_rows=23578 dup_rows_remote=21640 dedup_ratio=0.239848 router_tokens_per_expert=[2189, 2338, 1626, 65, 29, 0, 0, 234, 347, 1938, 0, 683, 177, 2200, 374, 48, 852, 473, 69, 573, 821, 266, 601, 1294, 969, 649, 84, 1090, 6, 1487, 2, 3811, 435, 1605, 749, 335, 2338, 0, 940, 435, 11, 899, 577, 1576, 0, 601, 220, 1303, 447, 3462, 191, 377, 1267, 260, 526, 436, 991, 12, 254, 934, 741, 1277, 526, 132] payload_rows_per_dst=[9680, 9130, 7792, 6992, 9990, 8436, 10664, 12042] dup_rows_per_dst=[3282, 2404, 2106, 1582, 3684, 1938, 3268, 5314] combine_rows_per_src=[9680, 9130, 7792, 6992, 9990, 8436, 10664, 12042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74768 payload_rows_remote=64732 dup_rows=23536 dup_rows_remote=20852 dedup_ratio=0.239421 router_tokens_per_expert=[2322, 2293, 1618, 48, 69, 3, 0, 244, 291, 1811, 0, 856, 125, 1763, 367, 80, 1007, 458, 30, 524, 717, 233, 520, 1228, 1339, 659, 73, 1105, 1, 1469, 5, 3679, 413, 1755, 724, 309, 2818, 4, 760, 424, 19, 1451, 569, 1407, 1, 612, 235, 1633, 520, 2974, 159, 262, 988, 260, 711, 486, 1031, 2, 420, 771, 486, 1439, 457, 115] payload_rows_per_dst=[8116, 10128, 7448, 7636, 10482, 9542, 10036, 11380] dup_rows_per_dst=[1212, 4324, 1986, 1666, 3932, 2312, 2684, 5420] combine_rows_per_src=[8116, 10128, 7448, 7636, 10482, 9542, 10036, 11380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=340136 combine_payload_bytes=267591680 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267931816 ingress_bytes=371400704 link_bytes=371400704 token_bytes=535523496 weight_movement_bytes=103809024 total_nvlink_bytes=639332520 weight_to_token_ratio=0.193846 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397550949e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269492224 dispatch_meta_bytes=347416 combine_payload_bytes=269492224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269839640 ingress_bytes=304095232 link_bytes=304095232 token_bytes=539331864 weight_movement_bytes=34603008 total_nvlink_bytes=573934872 weight_to_token_ratio=0.064159 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144285876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270499840 dispatch_meta_bytes=345592 combine_payload_bytes=270499840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270845432 ingress_bytes=374308864 link_bytes=374308864 token_bytes=541345272 weight_movement_bytes=103809024 total_nvlink_bytes=645154296 weight_to_token_ratio=0.191761 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408494121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262356992 dispatch_meta_bytes=331360 combine_payload_bytes=262356992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262688352 ingress_bytes=296960000 link_bytes=296960000 token_bytes=525045344 weight_movement_bytes=34603008 total_nvlink_bytes=559648352 weight_to_token_ratio=0.065905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117436573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=255041536 dispatch_meta_bytes=320512 combine_payload_bytes=255041536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255362048 ingress_bytes=289644544 link_bytes=289644544 token_bytes=510403584 weight_movement_bytes=34603008 total_nvlink_bytes=545006592 weight_to_token_ratio=0.067795 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089909101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=271523840 dispatch_meta_bytes=351720 combine_payload_bytes=271523840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271875560 ingress_bytes=306126848 link_bytes=306126848 token_bytes=543399400 weight_movement_bytes=34603008 total_nvlink_bytes=578002408 weight_to_token_ratio=0.063679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151930684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342336 combine_payload_bytes=265142272 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265484608 ingress_bytes=334348288 link_bytes=334348288 token_bytes=530626880 weight_movement_bytes=69206016 total_nvlink_bytes=599832896 weight_to_token_ratio=0.130423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78754 payload_rows_remote=68284 dup_rows=19550 dup_rows_remote=16688 dedup_ratio=0.198873 router_tokens_per_expert=[121, 1869, 1365, 1902, 773, 1902, 33, 124, 392, 733, 103, 1248, 2945, 168, 169, 2276, 498, 192, 315, 568, 57, 43, 410, 808, 63, 1341, 525, 168, 50, 193, 251, 1928, 437, 378, 32, 1861, 134, 2, 855, 22, 1, 379, 20, 207, 2556, 1973, 163, 1024, 1020, 560, 573, 413, 118, 13, 1532, 473, 351, 3670, 487, 1668, 2883, 802, 787, 225] payload_rows_per_dst=[10906, 8352, 10208, 9730, 10470, 9756, 8138, 11194] dup_rows_per_dst=[1534, 1826, 2914, 3046, 2862, 2890, 1266, 3212] combine_rows_per_src=[10906, 8352, 10208, 9730, 10470, 9756, 8138, 11194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279691264 dispatch_meta_bytes=339888 combine_payload_bytes=279691264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280031152 ingress_bytes=314294272 link_bytes=314294272 token_bytes=559722416 weight_movement_bytes=34603008 total_nvlink_bytes=594325424 weight_to_token_ratio=0.061822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182664043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78872 payload_rows_remote=68918 dup_rows=19432 dup_rows_remote=16504 dedup_ratio=0.197673 router_tokens_per_expert=[107, 1970, 1372, 1612, 479, 2107, 32, 85, 365, 810, 104, 1126, 3024, 346, 29, 2185, 696, 3, 144, 721, 21, 5, 352, 1143, 51, 1223, 522, 154, 92, 4, 217, 2208, 348, 392, 53, 1800, 160, 4, 766, 2, 0, 383, 6, 150, 2350, 1868, 160, 1004, 937, 630, 485, 306, 95, 7, 1778, 438, 494, 3729, 473, 2209, 3082, 743, 869, 122] payload_rows_per_dst=[10256, 8068, 10744, 9954, 10464, 9168, 10968, 9250] dup_rows_per_dst=[1332, 1862, 2884, 2928, 2634, 2674, 3424, 1694] combine_rows_per_src=[10256, 8068, 10744, 9954, 10464, 9168, 10968, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=64012 dup_rows=23904 dup_rows_remote=19714 dedup_ratio=0.243164 router_tokens_per_expert=[2191, 2417, 2057, 64, 23, 0, 0, 291, 378, 2139, 0, 784, 159, 1809, 363, 56, 895, 557, 36, 624, 830, 282, 420, 1316, 1186, 652, 91, 1314, 4, 1259, 4, 3597, 435, 1409, 687, 469, 2583, 0, 940, 395, 10, 968, 681, 1306, 0, 467, 206, 1417, 518, 3155, 204, 315, 1239, 316, 695, 499, 970, 4, 283, 722, 813, 1138, 373, 137] payload_rows_per_dst=[10388, 8992, 10810, 11668, 7008, 11810, 6246, 7478] dup_rows_per_dst=[4190, 2384, 4276, 4546, 1662, 4118, 1326, 1402] combine_rows_per_src=[10388, 8992, 10810, 11668, 7008, 11810, 6246, 7478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77738 payload_rows_remote=68006 dup_rows=20566 dup_rows_remote=17348 dedup_ratio=0.209208 router_tokens_per_expert=[131, 1287, 1424, 2246, 364, 1948, 19, 203, 365, 929, 121, 1037, 3420, 440, 67, 1903, 585, 57, 205, 991, 10, 18, 260, 1568, 152, 1104, 544, 166, 147, 161, 197, 2035, 407, 435, 149, 1944, 187, 81, 513, 4, 3, 431, 8, 108, 2137, 1930, 171, 1141, 1480, 604, 532, 234, 292, 19, 1952, 412, 488, 3189, 532, 1427, 2571, 603, 844, 220] payload_rows_per_dst=[10924, 9732, 10622, 9394, 9012, 9088, 11726, 7240] dup_rows_per_dst=[1804, 3218, 3144, 2134, 2042, 2770, 4466, 988] combine_rows_per_src=[10924, 9732, 10622, 9394, 9012, 9088, 11726, 7240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79030 payload_rows_remote=69008 dup_rows=19274 dup_rows_remote=16836 dedup_ratio=0.196065 router_tokens_per_expert=[111, 2810, 1480, 1769, 995, 2216, 61, 107, 381, 597, 74, 1341, 3208, 98, 81, 2703, 563, 32, 270, 704, 25, 59, 669, 851, 24, 1445, 545, 134, 73, 8, 211, 1866, 543, 371, 39, 1715, 134, 1, 802, 11, 13, 261, 16, 120, 2493, 1919, 134, 1014, 871, 429, 513, 334, 208, 17, 1163, 354, 499, 3057, 498, 1437, 2860, 862, 805, 148] payload_rows_per_dst=[11504, 8640, 10022, 10490, 10882, 9368, 10660, 7464] dup_rows_per_dst=[1974, 1910, 2438, 3742, 2766, 2572, 2838, 1034] combine_rows_per_src=[11504, 8640, 10022, 10490, 10882, 9368, 10660, 7464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76514 payload_rows_remote=63304 dup_rows=21790 dup_rows_remote=15326 dedup_ratio=0.221659 router_tokens_per_expert=[111, 1600, 1288, 2043, 174, 1940, 4, 112, 463, 998, 155, 1068, 2761, 528, 37, 1978, 533, 15, 217, 811, 15, 14, 200, 1393, 92, 839, 520, 141, 183, 23, 275, 2216, 276, 391, 129, 2091, 127, 27, 579, 4, 3, 502, 8, 101, 2476, 1559, 138, 1017, 1462, 607, 515, 233, 196, 3, 2633, 431, 484, 3285, 499, 1752, 3123, 569, 1012, 173] payload_rows_per_dst=[7018, 10020, 7538, 9482, 10334, 9126, 9786, 13210] dup_rows_per_dst=[446, 4314, 978, 2296, 2436, 2482, 2374, 6464] combine_rows_per_src=[7018, 10020, 7538, 9482, 10334, 9126, 9786, 13210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79076 payload_rows_remote=69634 dup_rows=19228 dup_rows_remote=16750 dedup_ratio=0.195597 router_tokens_per_expert=[113, 3101, 1318, 2065, 448, 1931, 41, 79, 502, 662, 92, 1449, 3298, 158, 41, 2536, 475, 0, 161, 687, 11, 32, 476, 894, 39, 1403, 563, 156, 97, 27, 326, 1912, 319, 431, 48, 1607, 120, 0, 746, 13, 1, 304, 10, 142, 2524, 1700, 60, 1219, 1159, 516, 695, 638, 79, 6, 1553, 306, 433, 2767, 496, 1811, 2402, 897, 860, 197] payload_rows_per_dst=[10460, 8670, 9410, 11118, 10698, 9442, 8424, 10854] dup_rows_per_dst=[1530, 2210, 1596, 4130, 2466, 2478, 1480, 3338] combine_rows_per_src=[10460, 8670, 9410, 11118, 10698, 9442, 8424, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77782 payload_rows_remote=70106 dup_rows=20522 dup_rows_remote=19130 dedup_ratio=0.208761 router_tokens_per_expert=[99, 1786, 1446, 1946, 547, 2268, 57, 51, 416, 638, 108, 1274, 3422, 188, 134, 2279, 624, 52, 169, 702, 26, 12, 654, 760, 26, 1034, 523, 151, 100, 2, 301, 2259, 456, 424, 48, 1741, 106, 3, 920, 18, 6, 359, 19, 99, 2233, 1546, 99, 1169, 1105, 659, 568, 253, 146, 6, 1364, 433, 455, 3568, 486, 1770, 3042, 901, 950, 146] payload_rows_per_dst=[9606, 9012, 10326, 9540, 11206, 8748, 7676, 11668] dup_rows_per_dst=[1496, 2788, 2808, 2824, 3070, 2312, 1392, 3832] combine_rows_per_src=[9606, 9012, 10326, 9540, 11206, 8748, 7676, 11668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282288128 dispatch_meta_bytes=341688 combine_payload_bytes=282288128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282629816 ingress_bytes=316891136 link_bytes=316891136 token_bytes=564917944 weight_movement_bytes=34603008 total_nvlink_bytes=599520952 weight_to_token_ratio=0.061253 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192435833e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282656768 dispatch_meta_bytes=343376 combine_payload_bytes=282656768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283000144 ingress_bytes=317259776 link_bytes=317259776 token_bytes=565656912 weight_movement_bytes=34603008 total_nvlink_bytes=600259920 weight_to_token_ratio=0.061173 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193822996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=341416 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278893992 ingress_bytes=347758592 link_bytes=347758592 token_bytes=557446568 weight_movement_bytes=69206016 total_nvlink_bytes=626652584 weight_to_token_ratio=0.124148 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308587585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262193152 dispatch_meta_bytes=334904 combine_payload_bytes=262193152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262528056 ingress_bytes=366002176 link_bytes=366002176 token_bytes=524721208 weight_movement_bytes=103809024 total_nvlink_bytes=628530232 weight_to_token_ratio=0.197837 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377236723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259293184 dispatch_meta_bytes=314520 combine_payload_bytes=259293184 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=259607704 ingress_bytes=397705216 link_bytes=397705216 token_bytes=518900888 weight_movement_bytes=138412032 total_nvlink_bytes=657312920 weight_to_token_ratio=0.266741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496532710e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76666 payload_rows_remote=65078 dup_rows=21638 dup_rows_remote=18416 dedup_ratio=0.220113 router_tokens_per_expert=[687, 875, 364, 11, 153, 66, 649, 3946, 1079, 40, 777, 1, 413, 528, 3730, 270, 6, 6, 37, 2309, 149, 4, 134, 71, 40, 976, 1977, 0, 1239, 481, 51, 2453, 276, 123, 1188, 34, 1862, 241, 7, 78, 31, 1635, 1369, 356, 102, 332, 34, 525, 1433, 416, 382, 101, 2026, 87, 13, 1617, 246, 462, 32, 4074, 488, 1374, 2524, 2162] payload_rows_per_dst=[11140, 5566, 10670, 9174, 11588, 9706, 8912, 9910] dup_rows_per_dst=[2362, 918, 2910, 2718, 3222, 3968, 3238, 2302] combine_rows_per_src=[11140, 5566, 10670, 9174, 11588, 9706, 8912, 9910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285220864 dispatch_meta_bytes=345536 combine_payload_bytes=285220864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285566400 ingress_bytes=319823872 link_bytes=319823872 token_bytes=570787264 weight_movement_bytes=34603008 total_nvlink_bytes=605390272 weight_to_token_ratio=0.060623 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203471482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=287154176 dispatch_meta_bytes=356944 combine_payload_bytes=287154176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287511120 ingress_bytes=321757184 link_bytes=321757184 token_bytes=574665296 weight_movement_bytes=34603008 total_nvlink_bytes=609268304 weight_to_token_ratio=0.060214 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210746380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=333976 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266893464 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533452952 weight_movement_bytes=34603008 total_nvlink_bytes=568055960 weight_to_token_ratio=0.064866 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77800 payload_rows_remote=67086 dup_rows=20504 dup_rows_remote=16312 dedup_ratio=0.208577 router_tokens_per_expert=[806, 1419, 265, 17, 137, 73, 480, 4355, 1324, 103, 247, 1, 448, 249, 4316, 106, 7, 10, 57, 1332, 157, 3, 114, 79, 48, 1030, 1907, 0, 1992, 199, 37, 2930, 356, 117, 854, 24, 2027, 279, 0, 114, 6, 1052, 1035, 356, 20, 210, 43, 426, 1277, 320, 80, 25, 2023, 76, 4, 3418, 156, 222, 18, 3637, 382, 1739, 2240, 2368] payload_rows_per_dst=[12036, 11404, 9164, 10714, 6806, 8980, 10020, 8676] dup_rows_per_dst=[3068, 2184, 1628, 4192, 736, 3176, 4426, 1094] combine_rows_per_src=[12036, 11404, 9164, 10714, 6806, 8980, 10020, 8676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77156 payload_rows_remote=67894 dup_rows=21148 dup_rows_remote=18924 dedup_ratio=0.215129 router_tokens_per_expert=[637, 720, 785, 6, 132, 60, 463, 3796, 643, 22, 1095, 13, 439, 1004, 4812, 186, 0, 2, 83, 2008, 93, 16, 155, 75, 29, 1043, 2502, 0, 1230, 243, 47, 2135, 381, 65, 1267, 33, 1859, 135, 1, 27, 37, 1503, 592, 351, 42, 417, 136, 563, 1426, 309, 270, 27, 2539, 64, 14, 1086, 282, 1045, 27, 3311, 394, 1336, 3117, 2022] payload_rows_per_dst=[10882, 12556, 9262, 11408, 6562, 8398, 8896, 9192] dup_rows_per_dst=[2316, 3872, 2224, 5014, 974, 3154, 2574, 1020] combine_rows_per_src=[10882, 12556, 9262, 11408, 6562, 8398, 8896, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77020 payload_rows_remote=66020 dup_rows=21284 dup_rows_remote=19574 dedup_ratio=0.216512 router_tokens_per_expert=[702, 1355, 124, 1, 180, 126, 771, 3340, 1132, 75, 68, 12, 323, 7, 4592, 146, 4, 19, 26, 1304, 218, 5, 65, 17, 24, 960, 1930, 0, 1500, 216, 61, 2631, 518, 160, 563, 15, 2668, 616, 2, 219, 6, 884, 2072, 159, 32, 254, 11, 329, 1447, 436, 194, 4, 1926, 38, 20, 3984, 136, 759, 39, 3506, 405, 1670, 1834, 2312] payload_rows_per_dst=[11308, 11000, 9756, 9412, 8342, 9378, 10016, 7808] dup_rows_per_dst=[2930, 1710, 2756, 3638, 1180, 3378, 5042, 650] combine_rows_per_src=[11308, 11000, 9756, 9412, 8342, 9378, 10016, 7808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76134 payload_rows_remote=63372 dup_rows=22170 dup_rows_remote=17418 dedup_ratio=0.225525 router_tokens_per_expert=[128, 2098, 1392, 1866, 501, 1880, 10, 124, 550, 748, 63, 1202, 3360, 218, 22, 2678, 587, 2, 242, 938, 9, 20, 296, 977, 39, 1314, 536, 156, 74, 2, 187, 1440, 373, 435, 26, 1665, 128, 2, 845, 7, 6, 283, 3, 138, 2777, 1901, 60, 1433, 914, 550, 619, 572, 100, 2, 1625, 402, 402, 3265, 558, 1802, 2817, 857, 777, 149] payload_rows_per_dst=[12762, 12410, 10088, 6872, 6366, 11010, 10478, 6148] dup_rows_per_dst=[4752, 5272, 2584, 624, 596, 4420, 3208, 714] combine_rows_per_src=[12762, 12410, 10088, 6872, 6366, 11010, 10478, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76476 payload_rows_remote=68246 dup_rows=21828 dup_rows_remote=18628 dedup_ratio=0.222046 router_tokens_per_expert=[610, 1113, 313, 12, 101, 79, 323, 4142, 781, 77, 831, 1, 397, 662, 4672, 179, 5, 4, 99, 1733, 70, 6, 184, 65, 18, 1124, 1886, 0, 1406, 367, 55, 2279, 302, 75, 1149, 43, 2024, 164, 0, 31, 22, 1378, 875, 337, 26, 276, 15, 507, 1696, 289, 195, 31, 2303, 194, 7, 1625, 248, 491, 13, 3867, 445, 1580, 2926, 2424] payload_rows_per_dst=[10292, 6166, 9890, 7788, 12928, 8230, 9376, 11806] dup_rows_per_dst=[2110, 674, 2176, 1924, 3992, 3200, 3304, 4448] combine_rows_per_src=[10292, 6166, 9890, 7788, 12928, 8230, 9376, 11806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75382 payload_rows_remote=62246 dup_rows=22922 dup_rows_remote=16282 dedup_ratio=0.233175 router_tokens_per_expert=[701, 1176, 95, 4, 140, 158, 678, 4312, 1322, 129, 46, 0, 394, 64, 4685, 95, 1, 9, 26, 759, 165, 10, 47, 13, 37, 1020, 1525, 1, 2066, 131, 38, 3015, 577, 220, 370, 9, 2462, 368, 0, 200, 3, 770, 2013, 265, 31, 206, 39, 185, 1470, 295, 34, 3, 1656, 32, 2, 4781, 78, 156, 30, 3689, 374, 1862, 1766, 2344] payload_rows_per_dst=[5550, 10244, 8462, 7556, 13034, 9746, 7654, 13136] dup_rows_per_dst=[354, 2480, 976, 1790, 4748, 3598, 2336, 6640] combine_rows_per_src=[5550, 10244, 8462, 7556, 13034, 9746, 7654, 13136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75814 payload_rows_remote=66078 dup_rows=22490 dup_rows_remote=18580 dedup_ratio=0.228780 router_tokens_per_expert=[579, 955, 299, 13, 83, 64, 566, 4305, 982, 55, 958, 0, 477, 641, 4667, 165, 5, 6, 60, 1875, 82, 24, 185, 58, 33, 844, 2429, 0, 1135, 201, 58, 2380, 350, 64, 1212, 20, 1881, 213, 0, 51, 8, 1091, 1002, 424, 19, 200, 85, 470, 1412, 353, 90, 22, 2284, 114, 23, 2525, 289, 878, 28, 3175, 447, 1650, 2497, 2091] payload_rows_per_dst=[4774, 11234, 9146, 7692, 12936, 8388, 9736, 11908] dup_rows_per_dst=[344, 3932, 1794, 1708, 3980, 2970, 3910, 3852] combine_rows_per_src=[4774, 11234, 9146, 7692, 12936, 8388, 9736, 11908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274784256 dispatch_meta_bytes=333592 combine_payload_bytes=274784256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275117848 ingress_bytes=378593280 link_bytes=378593280 token_bytes=549902104 weight_movement_bytes=103809024 total_nvlink_bytes=653711128 weight_to_token_ratio=0.188777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424616033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278093824 dispatch_meta_bytes=347272 combine_payload_bytes=278093824 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=278441096 ingress_bytes=347299840 link_bytes=347299840 token_bytes=556534920 weight_movement_bytes=69206016 total_nvlink_bytes=625740936 weight_to_token_ratio=0.124352 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306861338e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270417920 dispatch_meta_bytes=342376 combine_payload_bytes=270417920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270760296 ingress_bytes=339623936 link_bytes=339623936 token_bytes=541178216 weight_movement_bytes=69206016 total_nvlink_bytes=610384232 weight_to_token_ratio=0.127880 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277977529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77160 payload_rows_remote=65836 dup_rows=21144 dup_rows_remote=16666 dedup_ratio=0.215088 router_tokens_per_expert=[262, 420, 651, 51, 406, 3371, 1, 803, 172, 679, 304, 4521, 227, 24, 322, 1113, 1191, 800, 29, 261, 513, 308, 8, 1865, 99, 145, 207, 291, 1288, 440, 292, 26, 751, 332, 1, 235, 1173, 99, 1466, 40, 317, 5, 2172, 74, 66, 151, 933, 11, 216, 403, 364, 253, 2055, 12, 1035, 1595, 3804, 0, 2817, 1, 5277, 1223, 918, 263] payload_rows_per_dst=[9610, 11896, 7864, 13370, 11324, 10098, 7986, 5012] dup_rows_per_dst=[2320, 2828, 2086, 2760, 4478, 2454, 3880, 338] combine_rows_per_src=[9610, 11896, 7864, 13370, 11324, 10098, 7986, 5012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259571712 dispatch_meta_bytes=323160 combine_payload_bytes=259571712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259894872 ingress_bytes=363380736 link_bytes=363380736 token_bytes=519466584 weight_movement_bytes=103809024 total_nvlink_bytes=623275608 weight_to_token_ratio=0.199838 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367372455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279535616 dispatch_meta_bytes=347496 combine_payload_bytes=279535616 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279883112 ingress_bytes=348741632 link_bytes=348741632 token_bytes=559418728 weight_movement_bytes=69206016 total_nvlink_bytes=628624744 weight_to_token_ratio=0.123711 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312286685e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=254959616 dispatch_meta_bytes=314112 combine_payload_bytes=254959616 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=255273728 ingress_bytes=393371648 link_bytes=393371648 token_bytes=510233344 weight_movement_bytes=138412032 total_nvlink_bytes=648645376 weight_to_token_ratio=0.271272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.480225842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270655488 dispatch_meta_bytes=338632 combine_payload_bytes=270655488 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270994120 ingress_bytes=339861504 link_bytes=339861504 token_bytes=541649608 weight_movement_bytes=69206016 total_nvlink_bytes=610855624 weight_to_token_ratio=0.127769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278871478e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=269664256 dispatch_meta_bytes=330008 combine_payload_bytes=269664256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269994264 ingress_bytes=304267264 link_bytes=304267264 token_bytes=539658520 weight_movement_bytes=34603008 total_nvlink_bytes=574261528 weight_to_token_ratio=0.064120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144933219e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77378 payload_rows_remote=63592 dup_rows=20926 dup_rows_remote=17792 dedup_ratio=0.212870 router_tokens_per_expert=[269, 375, 1302, 46, 420, 4159, 5, 1117, 172, 340, 300, 4314, 195, 56, 382, 808, 967, 687, 68, 227, 735, 388, 2, 1345, 11, 100, 59, 350, 1368, 390, 247, 0, 536, 447, 4, 251, 873, 99, 639, 19, 361, 0, 2382, 90, 91, 119, 661, 186, 145, 458, 34, 199, 1975, 12, 905, 1535, 4147, 1, 3401, 0, 5935, 1397, 819, 227] payload_rows_per_dst=[11376, 10904, 7128, 13786, 10984, 11198, 7302, 4700] dup_rows_per_dst=[4010, 2230, 1710, 3134, 3046, 3384, 3224, 188] combine_rows_per_src=[11376, 10904, 7128, 13786, 10984, 11198, 7302, 4700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77588 payload_rows_remote=69160 dup_rows=20716 dup_rows_remote=18460 dedup_ratio=0.210734 router_tokens_per_expert=[249, 334, 641, 40, 487, 3687, 1, 1001, 187, 774, 467, 4472, 169, 211, 372, 834, 1425, 1098, 15, 162, 811, 437, 4, 1390, 15, 148, 40, 415, 1187, 452, 386, 2, 428, 138, 3, 122, 1076, 144, 988, 14, 360, 4, 2138, 69, 165, 152, 1016, 8, 127, 362, 305, 199, 2165, 25, 1103, 1876, 4248, 0, 1832, 0, 5647, 1433, 822, 270] payload_rows_per_dst=[10188, 11902, 8428, 13522, 10920, 9294, 8448, 4886] dup_rows_per_dst=[2692, 3070, 2256, 3062, 3402, 2194, 3876, 164] combine_rows_per_src=[10188, 11902, 8428, 13522, 10920, 9294, 8448, 4886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76930 payload_rows_remote=69252 dup_rows=21374 dup_rows_remote=20538 dedup_ratio=0.217428 router_tokens_per_expert=[255, 378, 3224, 20, 343, 3473, 11, 568, 186, 239, 223, 2748, 327, 139, 309, 86, 705, 557, 743, 271, 145, 304, 0, 1099, 21, 26, 172, 172, 964, 313, 272, 0, 735, 656, 0, 129, 479, 37, 357, 6, 309, 0, 3649, 90, 41, 161, 468, 413, 19, 561, 48, 147, 2374, 2, 424, 1364, 4520, 1, 4756, 0, 6697, 1700, 498, 218] payload_rows_per_dst=[11538, 7678, 6480, 14242, 11170, 13896, 7284, 4642] dup_rows_per_dst=[5006, 836, 1168, 3032, 2668, 5878, 2594, 192] combine_rows_per_src=[11538, 7678, 6480, 14242, 11170, 13896, 7284, 4642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75148 payload_rows_remote=67666 dup_rows=23156 dup_rows_remote=22162 dedup_ratio=0.235555 router_tokens_per_expert=[256, 350, 781, 97, 459, 3491, 0, 1145, 245, 652, 368, 4047, 164, 254, 377, 914, 1114, 1338, 25, 237, 992, 235, 2, 1448, 24, 125, 31, 364, 905, 545, 358, 17, 646, 346, 0, 265, 920, 193, 882, 23, 326, 5, 2423, 75, 184, 120, 1079, 26, 277, 436, 65, 166, 1844, 13, 1016, 1958, 4054, 1, 2004, 0, 5783, 1328, 1131, 203] payload_rows_per_dst=[5648, 11204, 11782, 6978, 11086, 7482, 12968, 8000] dup_rows_per_dst=[1186, 2838, 5324, 992, 3572, 994, 6916, 1334] combine_rows_per_src=[5648, 11204, 11782, 6978, 11086, 7482, 12968, 8000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74944 payload_rows_remote=62214 dup_rows=23360 dup_rows_remote=17252 dedup_ratio=0.237630 router_tokens_per_expert=[647, 1067, 448, 3, 123, 60, 471, 4334, 801, 62, 516, 0, 373, 394, 4901, 135, 4, 6, 36, 1702, 79, 12, 284, 49, 21, 871, 1618, 2, 1353, 250, 42, 2477, 347, 90, 936, 18, 1977, 213, 3, 54, 12, 1508, 971, 331, 26, 232, 41, 419, 1647, 274, 244, 22, 2199, 41, 5, 2266, 200, 473, 19, 4109, 459, 1780, 2631, 2464] payload_rows_per_dst=[12730, 12032, 12112, 10102, 6464, 8740, 7098, 5666] dup_rows_per_dst=[6108, 2332, 5378, 3474, 812, 3294, 1766, 196] combine_rows_per_src=[12730, 12032, 12112, 10102, 6464, 8740, 7098, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67932 payload_rows_remote=58766 dup_rows=30372 dup_rows_remote=25012 dedup_ratio=0.308960 router_tokens_per_expert=[246, 352, 805, 87, 353, 4241, 2, 975, 247, 551, 294, 4480, 123, 156, 390, 977, 1149, 1041, 42, 220, 577, 300, 16, 1644, 16, 185, 92, 327, 944, 404, 260, 0, 443, 261, 1, 153, 1069, 114, 861, 15, 369, 6, 2288, 31, 99, 77, 877, 95, 436, 434, 370, 196, 2006, 21, 1069, 1676, 4070, 7, 2850, 0, 5327, 1323, 913, 199] payload_rows_per_dst=[4730, 11584, 12406, 4254, 5388, 6936, 9166, 13468] dup_rows_per_dst=[910, 2852, 6054, 202, 1446, 748, 5360, 12800] combine_rows_per_src=[4730, 11584, 12406, 4254, 5388, 6936, 9166, 13468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67290 payload_rows_remote=51002 dup_rows=31014 dup_rows_remote=9820 dedup_ratio=0.315491 router_tokens_per_expert=[252, 381, 3240, 36, 321, 4238, 9, 515, 209, 202, 125, 3561, 331, 0, 368, 184, 715, 644, 332, 209, 384, 290, 2, 831, 14, 27, 159, 197, 826, 369, 177, 0, 609, 832, 0, 176, 319, 102, 345, 4, 335, 4, 3321, 74, 18, 171, 365, 719, 35, 496, 8, 24, 1941, 6, 326, 1033, 4791, 1, 5083, 2, 6591, 1511, 552, 210] payload_rows_per_dst=[7464, 8984, 11630, 3394, 4192, 9270, 6068, 16288] dup_rows_per_dst=[2044, 976, 3660, 144, 582, 744, 1670, 21194] combine_rows_per_src=[7464, 8984, 11630, 3394, 4192, 9270, 6068, 16288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260472832 dispatch_meta_bytes=325536 combine_payload_bytes=260472832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260798368 ingress_bytes=295075840 link_bytes=295075840 token_bytes=521271200 weight_movement_bytes=34603008 total_nvlink_bytes=555874208 weight_to_token_ratio=0.066382 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110346630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283279360 dispatch_meta_bytes=350480 combine_payload_bytes=283279360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283629840 ingress_bytes=317882368 link_bytes=317882368 token_bytes=566909200 weight_movement_bytes=34603008 total_nvlink_bytes=601512208 weight_to_token_ratio=0.061038 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196165759e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283656192 dispatch_meta_bytes=359160 combine_payload_bytes=283656192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284015352 ingress_bytes=352862208 link_bytes=352862208 token_bytes=567671544 weight_movement_bytes=69206016 total_nvlink_bytes=636877560 weight_to_token_ratio=0.121912 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327792081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78358 payload_rows_remote=67778 dup_rows=19946 dup_rows_remote=17998 dedup_ratio=0.202901 router_tokens_per_expert=[68, 77, 376, 1514, 153, 882, 211, 0, 152, 416, 10, 175, 838, 11, 463, 225, 1784, 360, 781, 1501, 8, 490, 2250, 1020, 945, 1053, 321, 2988, 14, 7, 97, 824, 928, 525, 1, 121, 41, 670, 145, 3833, 1426, 5, 1127, 174, 20, 31, 768, 777, 39, 554, 121, 1, 2145, 5775, 158, 123, 4015, 698, 186, 231, 2758, 123, 591, 1028] payload_rows_per_dst=[11470, 12964, 9002, 9392, 10580, 9798, 5938, 9214] dup_rows_per_dst=[3122, 3166, 2886, 3106, 1948, 3358, 344, 2016] combine_rows_per_src=[11470, 12964, 9002, 9392, 10580, 9798, 5938, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277159936 dispatch_meta_bytes=359312 combine_payload_bytes=277159936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277519248 ingress_bytes=311762944 link_bytes=311762944 token_bytes=554679184 weight_movement_bytes=34603008 total_nvlink_bytes=589282192 weight_to_token_ratio=0.062384 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173138859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=254828544 dispatch_meta_bytes=317864 combine_payload_bytes=254828544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255146408 ingress_bytes=324034560 link_bytes=324034560 token_bytes=509974952 weight_movement_bytes=69206016 total_nvlink_bytes=579180968 weight_to_token_ratio=0.135705 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219315962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=240705536 dispatch_meta_bytes=335112 combine_payload_bytes=240705536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241040648 ingress_bytes=275308544 link_bytes=275308544 token_bytes=481746184 weight_movement_bytes=34603008 total_nvlink_bytes=516349192 weight_to_token_ratio=0.071828 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.035963887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=208904192 dispatch_meta_bytes=243288 combine_payload_bytes=208904192 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=209147480 ingress_bytes=381919232 link_bytes=381919232 token_bytes=418051672 weight_movement_bytes=173015040 total_nvlink_bytes=591066712 weight_to_token_ratio=0.413860 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437131323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277618688 dispatch_meta_bytes=343104 combine_payload_bytes=277618688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277961792 ingress_bytes=312221696 link_bytes=312221696 token_bytes=555580480 weight_movement_bytes=34603008 total_nvlink_bytes=590183488 weight_to_token_ratio=0.062283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174865106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78320 payload_rows_remote=68294 dup_rows=19984 dup_rows_remote=16498 dedup_ratio=0.203288 router_tokens_per_expert=[47, 84, 420, 1418, 26, 691, 198, 0, 293, 397, 27, 51, 1318, 4, 209, 220, 1679, 285, 878, 1795, 1, 569, 2226, 1001, 1333, 720, 415, 3293, 5, 1, 100, 889, 948, 546, 1, 146, 12, 541, 137, 3628, 1665, 0, 984, 94, 6, 87, 917, 586, 12, 287, 176, 0, 2100, 5943, 61, 169, 4185, 803, 166, 148, 2645, 168, 378, 1020] payload_rows_per_dst=[11426, 13132, 9310, 10026, 10336, 9786, 5342, 8962] dup_rows_per_dst=[2712, 3792, 3106, 3486, 1582, 3344, 268, 1694] combine_rows_per_src=[11426, 13132, 9310, 10026, 10336, 9786, 5342, 8962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78206 payload_rows_remote=68316 dup_rows=20098 dup_rows_remote=16446 dedup_ratio=0.204447 router_tokens_per_expert=[57, 90, 318, 1564, 39, 813, 199, 0, 84, 429, 11, 225, 207, 2, 270, 270, 1857, 405, 778, 1791, 0, 617, 1857, 1323, 888, 1033, 283, 3055, 1, 0, 89, 796, 692, 534, 0, 102, 8, 1009, 117, 2814, 997, 0, 1294, 119, 30, 25, 672, 1217, 24, 276, 164, 0, 2615, 6377, 125, 21, 4051, 695, 234, 223, 2881, 214, 592, 1679] payload_rows_per_dst=[11560, 13208, 9890, 9150, 9120, 9342, 6036, 9900] dup_rows_per_dst=[2702, 2542, 3652, 3140, 1432, 3080, 414, 3136] combine_rows_per_src=[11560, 13208, 9890, 9150, 9120, 9342, 6036, 9900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76512 payload_rows_remote=63900 dup_rows=21792 dup_rows_remote=17012 dedup_ratio=0.221680 router_tokens_per_expert=[46, 26, 491, 176, 27, 929, 193, 0, 291, 663, 55, 0, 3736, 3, 436, 35, 1821, 29, 130, 1005, 0, 377, 2392, 1189, 1209, 271, 83, 4067, 30, 0, 333, 999, 274, 613, 0, 213, 85, 683, 274, 4786, 2649, 6, 1296, 279, 0, 596, 1373, 22, 3, 530, 119, 0, 1969, 3477, 158, 296, 2255, 884, 126, 224, 4260, 56, 244, 360] payload_rows_per_dst=[7174, 12612, 9926, 11168, 11766, 10188, 6002, 7676] dup_rows_per_dst=[1112, 4780, 2846, 5484, 2090, 3368, 148, 1964] combine_rows_per_src=[7174, 12612, 9926, 11168, 11766, 10188, 6002, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76496 payload_rows_remote=67312 dup_rows=21808 dup_rows_remote=18990 dedup_ratio=0.221842 router_tokens_per_expert=[64, 84, 339, 1662, 46, 674, 170, 0, 235, 348, 11, 125, 311, 1, 263, 261, 1882, 454, 1012, 1482, 0, 514, 1858, 949, 1401, 905, 356, 2604, 0, 0, 94, 917, 1502, 360, 0, 100, 6, 736, 114, 2938, 1033, 1, 1060, 78, 37, 12, 743, 1179, 17, 145, 112, 0, 2429, 6297, 147, 29, 4425, 672, 214, 176, 2696, 342, 623, 1907] payload_rows_per_dst=[11942, 3100, 9400, 7680, 9916, 9184, 14834, 10440] dup_rows_per_dst=[2986, 314, 3186, 2082, 1596, 2818, 6006, 2820] combine_rows_per_src=[11942, 3100, 9400, 7680, 9916, 9184, 14834, 10440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=61872 dup_rows=24032 dup_rows_remote=16850 dedup_ratio=0.244466 router_tokens_per_expert=[239, 401, 1026, 67, 397, 3677, 2, 851, 202, 540, 318, 3960, 202, 33, 344, 480, 1027, 1214, 42, 114, 885, 289, 2, 1591, 102, 93, 58, 361, 1350, 400, 393, 2, 383, 251, 0, 243, 1066, 126, 768, 6, 335, 1, 2673, 81, 111, 149, 681, 26, 143, 469, 71, 234, 2018, 6, 972, 1868, 4180, 1, 2810, 0, 6270, 1304, 1010, 234] payload_rows_per_dst=[12400, 7390, 8018, 12272, 10762, 10644, 7886, 4900] dup_rows_per_dst=[7182, 1132, 2310, 3160, 3284, 3090, 3676, 198] combine_rows_per_src=[12400, 7390, 8018, 12272, 10762, 10644, 7886, 4900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74744 payload_rows_remote=59006 dup_rows=23560 dup_rows_remote=14698 dedup_ratio=0.239665 router_tokens_per_expert=[68, 73, 296, 1231, 110, 657, 206, 0, 233, 391, 14, 57, 400, 8, 145, 387, 1866, 520, 882, 2060, 1, 434, 1686, 1037, 753, 729, 480, 2758, 4, 1, 92, 816, 895, 389, 0, 90, 9, 714, 112, 3595, 1386, 1, 970, 72, 15, 48, 754, 938, 31, 231, 118, 0, 2424, 6516, 121, 101, 4187, 691, 246, 164, 3390, 390, 537, 1622] payload_rows_per_dst=[11238, 3082, 9922, 5010, 10134, 9046, 15738, 10574] dup_rows_per_dst=[2418, 188, 3678, 740, 1474, 2694, 8862, 3506] combine_rows_per_src=[11238, 3082, 9922, 5010, 10134, 9046, 15738, 10574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279732224 dispatch_meta_bytes=339168 combine_payload_bytes=279732224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280071392 ingress_bytes=348938240 link_bytes=348938240 token_bytes=559803616 weight_movement_bytes=69206016 total_nvlink_bytes=629009632 weight_to_token_ratio=0.123626 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313026505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279822336 dispatch_meta_bytes=339048 combine_payload_bytes=279822336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=280161384 ingress_bytes=383631360 link_bytes=383631360 token_bytes=559983720 weight_movement_bytes=103809024 total_nvlink_bytes=663792744 weight_to_token_ratio=0.185379 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443573923e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75850 payload_rows_remote=63884 dup_rows=22454 dup_rows_remote=17038 dedup_ratio=0.228414 router_tokens_per_expert=[0, 331, 291, 9, 1688, 1630, 41, 54, 52, 1552, 44, 133, 43, 0, 5, 1368, 525, 13, 7534, 1757, 222, 1003, 1320, 483, 930, 1218, 88, 315, 9, 5437, 194, 1884, 1581, 320, 94, 1068, 3, 29, 2857, 32, 522, 13, 1191, 278, 14, 396, 756, 3, 3586, 2, 155, 502, 491, 0, 9, 6, 1182, 14, 184, 2, 320, 30, 3142, 197] payload_rows_per_dst=[7302, 15300, 7794, 7308, 11966, 9360, 8356, 8464] dup_rows_per_dst=[786, 6162, 2852, 1968, 5416, 2446, 1146, 1678] combine_rows_per_src=[7302, 15300, 7794, 7308, 11966, 9360, 8356, 8464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=261734400 dispatch_meta_bytes=323648 combine_payload_bytes=261734400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262058048 ingress_bytes=296337408 link_bytes=296337408 token_bytes=523792448 weight_movement_bytes=34603008 total_nvlink_bytes=558395456 weight_to_token_ratio=0.066062 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115093809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=61192 dup_rows=24052 dup_rows_remote=17100 dedup_ratio=0.244670 router_tokens_per_expert=[41, 36, 382, 176, 35, 869, 188, 0, 296, 591, 36, 2, 3546, 1, 453, 27, 1780, 39, 312, 1673, 0, 347, 1999, 1124, 1419, 690, 110, 3754, 45, 0, 254, 959, 476, 477, 8, 116, 56, 531, 224, 4501, 2418, 3, 1210, 118, 3, 356, 1121, 46, 0, 543, 111, 0, 2018, 4405, 148, 175, 3142, 803, 241, 119, 3899, 36, 266, 398] payload_rows_per_dst=[4010, 8986, 4912, 6308, 12082, 10918, 13976, 13060] dup_rows_per_dst=[296, 918, 696, 646, 2582, 3630, 8332, 6952] combine_rows_per_src=[4010, 8986, 4912, 6308, 12082, 10918, 13976, 13060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275709952 dispatch_meta_bytes=345208 combine_payload_bytes=275709952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276055160 ingress_bytes=310312960 link_bytes=310312960 token_bytes=551765112 weight_movement_bytes=34603008 total_nvlink_bytes=586368120 weight_to_token_ratio=0.062713 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167682686e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253427712 dispatch_meta_bytes=314888 combine_payload_bytes=253427712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253742600 ingress_bytes=322633728 link_bytes=322633728 token_bytes=507170312 weight_movement_bytes=69206016 total_nvlink_bytes=576376328 weight_to_token_ratio=0.136455 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214044744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=241688576 dispatch_meta_bytes=294816 combine_payload_bytes=241688576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=241983392 ingress_bytes=310894592 link_bytes=310894592 token_bytes=483671968 weight_movement_bytes=69206016 total_nvlink_bytes=552877984 weight_to_token_ratio=0.143085 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169871321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261668864 dispatch_meta_bytes=323688 combine_payload_bytes=261668864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261992552 ingress_bytes=296271872 link_bytes=296271872 token_bytes=523661416 weight_movement_bytes=34603008 total_nvlink_bytes=558264424 weight_to_token_ratio=0.066079 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114847202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=250642432 dispatch_meta_bytes=313168 combine_payload_bytes=250642432 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250955600 ingress_bytes=354451456 link_bytes=354451456 token_bytes=501598032 weight_movement_bytes=103809024 total_nvlink_bytes=605407056 weight_to_token_ratio=0.206957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333772293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74736 payload_rows_remote=64610 dup_rows=23568 dup_rows_remote=18970 dedup_ratio=0.239746 router_tokens_per_expert=[0, 221, 156, 2, 2037, 1148, 9, 11, 16, 1976, 51, 153, 10, 176, 7, 1624, 517, 5, 7915, 938, 243, 1226, 970, 559, 723, 557, 24, 244, 20, 5925, 85, 2341, 1687, 402, 83, 1169, 0, 6, 1624, 3, 140, 0, 1632, 233, 2, 328, 1236, 5, 3484, 1, 23, 255, 503, 0, 67, 9, 1954, 3, 196, 5, 153, 4, 3964, 92] payload_rows_per_dst=[6616, 15972, 6592, 10126, 11214, 6244, 7814, 10158] dup_rows_per_dst=[552, 7884, 2324, 4598, 3848, 908, 870, 2584] combine_rows_per_src=[6616, 15972, 6592, 10126, 11214, 6244, 7814, 10158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65208 dup_rows=23924 dup_rows_remote=20004 dedup_ratio=0.243368 router_tokens_per_expert=[1, 152, 167, 2, 1027, 2173, 22, 21, 24, 1219, 20, 88, 42, 77, 8, 1748, 623, 3, 7646, 2747, 147, 891, 1453, 682, 1051, 991, 138, 245, 13, 5752, 105, 2801, 1447, 440, 155, 376, 0, 14, 2889, 5, 252, 3, 860, 248, 6, 316, 510, 0, 4115, 0, 51, 475, 281, 0, 7, 7, 872, 0, 107, 12, 320, 9, 3076, 220] payload_rows_per_dst=[14570, 6846, 9172, 13992, 8904, 4204, 8712, 7980] dup_rows_per_dst=[6130, 1328, 3920, 8200, 1748, 186, 1160, 1252] combine_rows_per_src=[14570, 6846, 9172, 13992, 8904, 4204, 8712, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82294 payload_rows_remote=73816 dup_rows=16010 dup_rows_remote=15356 dedup_ratio=0.162862 router_tokens_per_expert=[0, 0, 2, 0, 2095, 57, 1, 5, 30, 3388, 321, 291, 7, 308, 0, 221, 1284, 12, 8038, 8, 340, 2178, 195, 280, 394, 1, 0, 72, 0, 4045, 79, 596, 3132, 221, 105, 2679, 2, 1, 98, 3, 4, 0, 3717, 287, 0, 563, 2951, 84, 2669, 0, 38, 0, 198, 0, 0, 52, 2450, 0, 129, 38, 104, 4, 5302, 73] payload_rows_per_dst=[16170, 8478, 7154, 9470, 10498, 11882, 5810, 12832] dup_rows_per_dst=[4226, 654, 1440, 904, 1984, 3330, 104, 3368] combine_rows_per_src=[16170, 8478, 7154, 9470, 10498, 11882, 5810, 12832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78834 payload_rows_remote=65614 dup_rows=19470 dup_rows_remote=16436 dedup_ratio=0.198059 router_tokens_per_expert=[0, 232, 185, 3, 1118, 2135, 6, 10, 14, 1546, 26, 73, 30, 8, 6, 2064, 474, 12, 7504, 2232, 138, 777, 1212, 717, 1016, 799, 29, 216, 0, 5877, 89, 2910, 1420, 409, 84, 547, 0, 1, 2824, 5, 178, 2, 1032, 260, 23, 194, 561, 0, 4317, 0, 12, 553, 273, 0, 10, 1, 1379, 0, 144, 2, 247, 13, 3024, 179] payload_rows_per_dst=[6872, 8468, 15572, 8040, 9122, 13220, 9124, 8416] dup_rows_per_dst=[506, 1736, 7890, 2078, 1458, 3034, 1208, 1560] combine_rows_per_src=[6872, 8468, 15572, 8040, 9122, 13220, 9124, 8416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76460 payload_rows_remote=65022 dup_rows=21844 dup_rows_remote=19146 dedup_ratio=0.222209 router_tokens_per_expert=[69, 87, 383, 1369, 23, 788, 235, 0, 216, 446, 6, 73, 819, 1, 503, 210, 1752, 282, 686, 1722, 1, 441, 1557, 1505, 1695, 1071, 317, 2929, 2, 1, 92, 1108, 653, 525, 0, 136, 11, 715, 140, 3653, 1398, 2, 994, 159, 8, 40, 712, 562, 27, 173, 121, 0, 2357, 5486, 227, 127, 4114, 656, 320, 178, 3228, 198, 426, 1417] payload_rows_per_dst=[11438, 12676, 11438, 13046, 10050, 6064, 5786, 5962] dup_rows_per_dst=[2698, 2844, 4454, 7840, 1616, 1686, 278, 428] combine_rows_per_src=[11438, 12676, 11438, 13046, 10050, 6064, 5786, 5962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82746 payload_rows_remote=72550 dup_rows=15558 dup_rows_remote=13024 dedup_ratio=0.158264 router_tokens_per_expert=[4087, 1434, 2056, 249, 159, 59, 1930, 18, 924, 0, 44, 151, 253, 365, 7, 7, 847, 0, 266, 19, 898, 127, 5, 36, 2658, 2, 858, 228, 2, 245, 653, 190, 1462, 3762, 565, 3566, 107, 2474, 335, 4, 129, 30, 4213, 302, 12, 1900, 307, 8, 0, 356, 1872, 0, 74, 1174, 187, 1, 1245, 4189, 12, 59, 1, 1304, 328, 397] payload_rows_per_dst=[9788, 12010, 10842, 8922, 10196, 11806, 6870, 12312] dup_rows_per_dst=[2022, 3312, 1728, 750, 2534, 1996, 458, 2758] combine_rows_per_src=[9788, 12010, 10842, 8922, 10196, 11806, 6870, 12312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=334320 combine_payload_bytes=264642560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264976880 ingress_bytes=403054592 link_bytes=403054592 token_bytes=529619440 weight_movement_bytes=138412032 total_nvlink_bytes=668031472 weight_to_token_ratio=0.261342 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516661981e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75678 payload_rows_remote=61144 dup_rows=22626 dup_rows_remote=14476 dedup_ratio=0.230164 router_tokens_per_expert=[0, 279, 238, 2, 1237, 1762, 29, 25, 47, 1833, 21, 83, 20, 2, 1, 2394, 417, 11, 7493, 2066, 113, 888, 1191, 758, 830, 1057, 86, 276, 38, 5472, 102, 3183, 1099, 357, 87, 684, 0, 33, 2819, 5, 255, 0, 784, 250, 9, 348, 677, 3, 4048, 0, 58, 663, 340, 0, 10, 4, 1106, 3, 224, 6, 266, 11, 2822, 227] payload_rows_per_dst=[6542, 7712, 8562, 8634, 8950, 12750, 14534, 7994] dup_rows_per_dst=[602, 1090, 4874, 2510, 1218, 2846, 8150, 1336] combine_rows_per_src=[6542, 7712, 8562, 8634, 8950, 12750, 14534, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=340848 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267432816 ingress_bytes=405504000 link_bytes=405504000 token_bytes=534524784 weight_movement_bytes=138412032 total_nvlink_bytes=672936816 weight_to_token_ratio=0.258944 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525878906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=302350336 dispatch_meta_bytes=356688 combine_payload_bytes=302350336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302707024 ingress_bytes=336953344 link_bytes=336953344 token_bytes=605057360 weight_movement_bytes=34603008 total_nvlink_bytes=639660368 weight_to_token_ratio=0.057190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267928307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=61454 dup_rows=23912 dup_rows_remote=17646 dedup_ratio=0.243245 router_tokens_per_expert=[0, 8, 19, 0, 2082, 185, 11, 32, 32, 2619, 124, 254, 5, 321, 2, 612, 568, 33, 8036, 121, 319, 2065, 353, 362, 486, 103, 12, 42, 0, 5071, 91, 1515, 2599, 258, 86, 2894, 4, 4, 139, 11, 49, 2, 3719, 286, 5, 453, 2053, 38, 2803, 0, 4, 18, 164, 0, 0, 7, 3196, 1, 120, 7, 122, 4, 4551, 72] payload_rows_per_dst=[4596, 7300, 6026, 4044, 10180, 13196, 16112, 12938] dup_rows_per_dst=[78, 638, 1616, 454, 1810, 7098, 5952, 6266] combine_rows_per_src=[4596, 7300, 6026, 4044, 10180, 13196, 16112, 12938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268754944 dispatch_meta_bytes=328200 combine_payload_bytes=268754944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269083144 ingress_bytes=303357952 link_bytes=303357952 token_bytes=537838088 weight_movement_bytes=34603008 total_nvlink_bytes=572441096 weight_to_token_ratio=0.064337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141511551e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=336672 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666784 ingress_bytes=300933120 link_bytes=300933120 token_bytes=532996896 weight_movement_bytes=34603008 total_nvlink_bytes=567599904 weight_to_token_ratio=0.064922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=297164800 dispatch_meta_bytes=342296 combine_payload_bytes=297164800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=297507096 ingress_bytes=435576832 link_bytes=435576832 token_bytes=594671896 weight_movement_bytes=138412032 total_nvlink_bytes=733083928 weight_to_token_ratio=0.232754 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.639040552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=250445824 dispatch_meta_bytes=302480 combine_payload_bytes=250445824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250748304 ingress_bytes=285048832 link_bytes=285048832 token_bytes=501194128 weight_movement_bytes=34603008 total_nvlink_bytes=535797136 weight_to_token_ratio=0.069041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072615807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251715584 dispatch_meta_bytes=316400 combine_payload_bytes=251715584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252031984 ingress_bytes=286318592 link_bytes=286318592 token_bytes=503747568 weight_movement_bytes=34603008 total_nvlink_bytes=538350576 weight_to_token_ratio=0.068691 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077393811e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82588 payload_rows_remote=74240 dup_rows=15716 dup_rows_remote=15210 dedup_ratio=0.159871 router_tokens_per_expert=[4594, 1148, 1968, 105, 195, 44, 1439, 81, 614, 0, 16, 163, 75, 298, 28, 0, 530, 0, 135, 26, 967, 95, 14, 47, 2361, 0, 867, 240, 0, 134, 791, 34, 1321, 3236, 340, 2949, 74, 3259, 432, 0, 132, 3, 4721, 362, 0, 2007, 190, 0, 1, 265, 3021, 0, 106, 669, 294, 2, 1300, 5366, 0, 47, 1, 673, 847, 525] payload_rows_per_dst=[8324, 12054, 11336, 8348, 8412, 12618, 8324, 13172] dup_rows_per_dst=[1636, 3324, 1480, 506, 1820, 2212, 392, 4346] combine_rows_per_src=[8324, 12054, 11336, 8348, 8412, 12618, 8324, 13172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77702 payload_rows_remote=68274 dup_rows=20602 dup_rows_remote=19098 dedup_ratio=0.209574 router_tokens_per_expert=[3296, 1170, 2132, 199, 109, 133, 1459, 1, 783, 0, 13, 58, 81, 277, 0, 0, 964, 1, 229, 19, 795, 108, 11, 43, 2465, 4, 691, 263, 1, 142, 661, 182, 1505, 3440, 1364, 3885, 48, 3349, 47, 1, 130, 5, 5111, 379, 4, 1767, 541, 0, 0, 692, 1181, 0, 35, 1799, 300, 3, 937, 4834, 0, 21, 3, 786, 431, 264] payload_rows_per_dst=[8928, 12282, 9428, 8208, 10962, 15348, 7282, 5264] dup_rows_per_dst=[1478, 3720, 1504, 610, 2738, 9504, 738, 310] combine_rows_per_src=[8928, 12282, 9428, 8208, 10962, 15348, 7282, 5264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79158 payload_rows_remote=71686 dup_rows=19146 dup_rows_remote=18304 dedup_ratio=0.194763 router_tokens_per_expert=[4675, 742, 2404, 1, 350, 6, 1524, 1468, 129, 0, 0, 683, 62, 622, 354, 0, 16, 0, 0, 352, 1794, 124, 76, 698, 814, 0, 1593, 247, 0, 1, 2439, 0, 795, 984, 3, 1117, 0, 1323, 630, 9, 17, 0, 1456, 2, 187, 2896, 539, 0, 23, 124, 6292, 0, 482, 62, 238, 141, 554, 7765, 0, 0, 0, 4, 1575, 760] payload_rows_per_dst=[11300, 7472, 12484, 14762, 5352, 9106, 13184, 5498] dup_rows_per_dst=[3654, 842, 2986, 7688, 936, 1212, 1540, 288] combine_rows_per_src=[11300, 7472, 12484, 14762, 5352, 9106, 13184, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79850 payload_rows_remote=72920 dup_rows=18454 dup_rows_remote=18034 dedup_ratio=0.187724 router_tokens_per_expert=[4044, 443, 2234, 174, 1, 43, 1104, 1, 731, 1, 17, 65, 341, 247, 0, 0, 1390, 0, 381, 18, 788, 88, 7, 10, 2222, 0, 711, 245, 0, 170, 748, 34, 1329, 3251, 739, 4027, 142, 4048, 50, 0, 106, 18, 5518, 352, 0, 1753, 258, 0, 0, 398, 1340, 0, 51, 1621, 274, 0, 1204, 4586, 0, 50, 0, 766, 806, 207] payload_rows_per_dst=[7550, 8084, 11182, 7748, 13890, 6930, 12166, 12300] dup_rows_per_dst=[450, 1222, 2270, 512, 6780, 420, 3862, 2938] combine_rows_per_src=[7550, 8084, 11182, 7748, 13890, 6930, 12166, 12300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77264 payload_rows_remote=61390 dup_rows=21040 dup_rows_remote=14158 dedup_ratio=0.214030 router_tokens_per_expert=[0, 134, 63, 3, 2078, 1256, 12, 12, 38, 2376, 48, 135, 15, 44, 9, 1703, 520, 5, 7820, 866, 221, 1122, 1063, 481, 833, 713, 20, 243, 0, 5938, 89, 2133, 2107, 416, 95, 867, 1, 0, 1501, 7, 242, 1, 1935, 277, 4, 377, 963, 1, 3981, 1, 23, 404, 208, 0, 3, 6, 1786, 0, 202, 8, 149, 10, 3481, 103] payload_rows_per_dst=[15874, 7600, 8902, 11544, 8628, 6872, 8302, 9542] dup_rows_per_dst=[6882, 1136, 3614, 4434, 1360, 728, 950, 1936] combine_rows_per_src=[15874, 7600, 8902, 11544, 8628, 6872, 8302, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304087040 dispatch_meta_bytes=357800 combine_payload_bytes=304087040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304444840 ingress_bytes=338690048 link_bytes=338690048 token_bytes=608531880 weight_movement_bytes=34603008 total_nvlink_bytes=643134888 weight_to_token_ratio=0.056863 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274463384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81944 payload_rows_remote=75158 dup_rows=16360 dup_rows_remote=16000 dedup_ratio=0.166423 router_tokens_per_expert=[367, 9, 47, 72, 434, 465, 119, 1626, 102, 6402, 168, 134, 254, 3923, 0, 1513, 2890, 90, 0, 2, 75, 145, 677, 369, 541, 247, 5531, 547, 34, 396, 39, 31, 477, 0, 2, 479, 73, 1824, 150, 568, 6098, 58, 352, 0, 340, 3282, 1252, 0, 957, 40, 773, 1821, 274, 170, 124, 655, 1457, 135, 17, 26, 87, 258, 33, 121] payload_rows_per_dst=[5632, 13766, 12626, 13012, 6786, 9206, 8094, 12822] dup_rows_per_dst=[646, 3380, 3716, 1720, 360, 1362, 1534, 3642] combine_rows_per_src=[5632, 13766, 12626, 13012, 6786, 9206, 8094, 12822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=279650304 dispatch_meta_bytes=349488 combine_payload_bytes=279650304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279999792 ingress_bytes=314253312 link_bytes=314253312 token_bytes=559650096 weight_movement_bytes=34603008 total_nvlink_bytes=594253104 weight_to_token_ratio=0.061830 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182509914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79616 payload_rows_remote=67444 dup_rows=18688 dup_rows_remote=14378 dedup_ratio=0.190104 router_tokens_per_expert=[3174, 1415, 2172, 204, 107, 141, 1591, 16, 746, 0, 68, 158, 206, 282, 2, 3, 659, 2, 743, 63, 612, 80, 24, 23, 3211, 1, 427, 215, 2, 227, 609, 85, 1262, 4119, 1463, 3799, 210, 3055, 50, 0, 53, 18, 4827, 367, 0, 2191, 159, 1, 0, 494, 1423, 0, 19, 1105, 373, 0, 936, 3630, 0, 117, 0, 1209, 707, 297] payload_rows_per_dst=[9394, 9892, 9506, 8946, 13184, 5438, 12172, 11084] dup_rows_per_dst=[1898, 1276, 1254, 608, 6494, 140, 4310, 2708] combine_rows_per_src=[9394, 9892, 9506, 8946, 13184, 5438, 12172, 11084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=293625856 dispatch_meta_bytes=359960 combine_payload_bytes=293625856 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=293985816 ingress_bytes=362831872 link_bytes=362831872 token_bytes=587611672 weight_movement_bytes=69206016 total_nvlink_bytes=656817688 weight_to_token_ratio=0.117775 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365307124e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81090 payload_rows_remote=65730 dup_rows=17214 dup_rows_remote=12406 dedup_ratio=0.175110 router_tokens_per_expert=[5478, 771, 1873, 12, 279, 21, 1092, 455, 353, 0, 0, 510, 82, 274, 134, 0, 172, 0, 15, 423, 1525, 97, 40, 383, 1118, 4, 1368, 165, 0, 3, 1918, 2, 1793, 2224, 49, 1401, 5, 2160, 631, 2, 47, 0, 2384, 9, 44, 2868, 358, 0, 16, 114, 5628, 1, 189, 167, 344, 67, 462, 7444, 0, 2, 0, 43, 1609, 524] payload_rows_per_dst=[7672, 6286, 13114, 8526, 9724, 6452, 13956, 15360] dup_rows_per_dst=[1334, 868, 3152, 630, 2358, 200, 3864, 4808] combine_rows_per_src=[7672, 6286, 13114, 8526, 9724, 6452, 13956, 15360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298680320 dispatch_meta_bytes=363816 combine_payload_bytes=298680320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299044136 ingress_bytes=367886336 link_bytes=367886336 token_bytes=597724456 weight_movement_bytes=69206016 total_nvlink_bytes=666930472 weight_to_token_ratio=0.115782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384326665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251453440 dispatch_meta_bytes=302192 combine_payload_bytes=251453440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251755632 ingress_bytes=286056448 link_bytes=286056448 token_bytes=503209072 weight_movement_bytes=34603008 total_nvlink_bytes=537812080 weight_to_token_ratio=0.068765 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076407384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307847168 dispatch_meta_bytes=364632 combine_payload_bytes=307847168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308211800 ingress_bytes=342450176 link_bytes=342450176 token_bytes=616058968 weight_movement_bytes=34603008 total_nvlink_bytes=650661976 weight_to_token_ratio=0.056168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288612443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=276250624 dispatch_meta_bytes=327288 combine_payload_bytes=276250624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276577912 ingress_bytes=310853632 link_bytes=310853632 token_bytes=552828536 weight_movement_bytes=34603008 total_nvlink_bytes=587431544 weight_to_token_ratio=0.062593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169717191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269230080 dispatch_meta_bytes=312544 combine_payload_bytes=269230080 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=269542624 ingress_bytes=373039104 link_bytes=373039104 token_bytes=538772704 weight_movement_bytes=103809024 total_nvlink_bytes=642581728 weight_to_token_ratio=0.192677 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403716116e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82436 payload_rows_remote=69998 dup_rows=15868 dup_rows_remote=14978 dedup_ratio=0.161418 router_tokens_per_expert=[455, 0, 44, 74, 297, 123, 85, 1222, 14, 6854, 179, 11, 161, 3909, 0, 1385, 2910, 108, 1, 0, 76, 117, 708, 62, 426, 25, 5812, 209, 4, 181, 0, 7, 478, 0, 0, 502, 48, 3281, 121, 410, 5258, 82, 539, 0, 190, 4673, 2178, 0, 897, 21, 770, 1361, 322, 78, 68, 1200, 643, 29, 0, 3, 81, 78, 299, 83] payload_rows_per_dst=[13764, 4334, 12684, 12438, 8704, 12092, 7396, 11024] dup_rows_per_dst=[3398, 312, 3098, 890, 976, 3232, 2038, 1924] combine_rows_per_src=[13764, 4334, 12684, 12438, 8704, 12092, 7396, 11024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82020 payload_rows_remote=72216 dup_rows=16284 dup_rows_remote=14194 dedup_ratio=0.165649 router_tokens_per_expert=[264, 0, 10, 102, 484, 281, 114, 2064, 80, 6087, 143, 322, 251, 3958, 0, 1055, 2731, 127, 0, 0, 60, 200, 779, 70, 788, 110, 5746, 450, 26, 340, 36, 6, 769, 0, 0, 660, 139, 876, 203, 513, 7378, 8, 195, 8, 394, 3635, 364, 0, 1378, 32, 794, 2784, 255, 105, 135, 251, 1065, 98, 5, 0, 269, 31, 8, 116] payload_rows_per_dst=[14426, 6886, 9804, 12996, 10782, 2476, 9658, 14992] dup_rows_per_dst=[4386, 772, 2090, 2008, 2174, 96, 1810, 2948] combine_rows_per_src=[14426, 6886, 9804, 12996, 10782, 2476, 9658, 14992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78624 payload_rows_remote=66046 dup_rows=19680 dup_rows_remote=17994 dedup_ratio=0.200195 router_tokens_per_expert=[562, 0, 0, 59, 100, 46, 125, 1026, 58, 6574, 381, 5, 1, 5734, 0, 953, 70, 74, 0, 0, 99, 1, 919, 0, 422, 254, 4672, 0, 1, 121, 0, 0, 265, 0, 0, 0, 0, 7388, 1, 378, 345, 132, 3018, 0, 66, 4784, 6598, 0, 418, 0, 3, 82, 0, 156, 0, 3132, 17, 0, 0, 0, 0, 0, 32, 80] payload_rows_per_dst=[13912, 12578, 2322, 10364, 15806, 15692, 7032, 918] dup_rows_per_dst=[3072, 1686, 4, 576, 9826, 3936, 550, 30] combine_rows_per_src=[13912, 12578, 2322, 10364, 15806, 15692, 7032, 918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84870 payload_rows_remote=74932 dup_rows=13434 dup_rows_remote=12350 dedup_ratio=0.136658 router_tokens_per_expert=[280, 0, 0, 73, 555, 211, 60, 2261, 97, 6567, 251, 91, 128, 2967, 1, 1286, 3418, 167, 0, 0, 29, 256, 617, 38, 526, 29, 5729, 495, 47, 253, 3, 18, 475, 0, 0, 948, 108, 1370, 173, 531, 7168, 39, 208, 0, 404, 4140, 720, 0, 712, 50, 924, 2468, 508, 148, 268, 360, 508, 65, 6, 5, 214, 63, 23, 93] payload_rows_per_dst=[6308, 13782, 11996, 12750, 6770, 9938, 8888, 14438] dup_rows_per_dst=[572, 3060, 2988, 1450, 440, 1084, 1988, 1852] combine_rows_per_src=[6308, 13782, 11996, 12750, 6770, 9938, 8888, 14438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75198 payload_rows_remote=61490 dup_rows=23106 dup_rows_remote=13938 dedup_ratio=0.235046 router_tokens_per_expert=[4778, 1035, 2445, 209, 134, 34, 1151, 27, 483, 1, 2, 156, 90, 260, 6, 0, 470, 0, 78, 74, 838, 99, 109, 52, 2633, 2, 953, 245, 0, 136, 982, 2, 1218, 2772, 149, 2924, 24, 4060, 69, 2, 98, 8, 4249, 343, 1, 2015, 747, 0, 0, 264, 2765, 0, 101, 1132, 328, 3, 919, 5817, 0, 42, 0, 457, 908, 253] payload_rows_per_dst=[13708, 12564, 5532, 14668, 2882, 12386, 8478, 4980] dup_rows_per_dst=[9168, 3096, 506, 6872, 42, 2536, 708, 178] combine_rows_per_src=[13708, 12564, 5532, 14668, 2882, 12386, 8478, 4980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76576 payload_rows_remote=69382 dup_rows=21728 dup_rows_remote=20414 dedup_ratio=0.221029 router_tokens_per_expert=[433, 0, 59, 1, 1458, 666, 1081, 132, 0, 1250, 152, 47, 312, 83, 804, 14, 6167, 47, 4002, 592, 503, 2553, 143, 11, 3, 4658, 2, 585, 659, 2206, 1165, 128, 2134, 411, 1134, 23, 532, 0, 9, 11, 35, 383, 148, 627, 8, 3, 356, 4075, 1218, 0, 0, 1773, 533, 1, 50, 1388, 858, 4, 292, 2090, 10, 2, 56, 1072] payload_rows_per_dst=[6902, 13522, 9118, 13358, 7194, 10228, 7986, 8268] dup_rows_per_dst=[758, 4136, 3110, 5454, 1314, 3096, 1940, 1920] combine_rows_per_src=[6902, 13522, 9118, 13358, 7194, 10228, 7986, 8268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77566 payload_rows_remote=62650 dup_rows=20738 dup_rows_remote=13246 dedup_ratio=0.210958 router_tokens_per_expert=[156, 0, 74, 56, 1032, 339, 42, 1899, 101, 6406, 100, 144, 282, 3185, 7, 1292, 2945, 136, 0, 0, 24, 410, 577, 82, 593, 1, 5572, 456, 16, 259, 9, 14, 435, 0, 0, 613, 146, 1220, 154, 597, 7088, 18, 334, 6, 280, 2981, 717, 1, 1010, 43, 1253, 2215, 375, 157, 221, 480, 1858, 50, 4, 19, 195, 366, 23, 84] payload_rows_per_dst=[6404, 13630, 11816, 2416, 5980, 7982, 14916, 14422] dup_rows_per_dst=[792, 3034, 2902, 524, 350, 692, 7492, 4952] combine_rows_per_src=[6404, 13630, 11816, 2416, 5980, 7982, 14916, 14422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=286711808 dispatch_meta_bytes=339904 combine_payload_bytes=286711808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287051712 ingress_bytes=355917824 link_bytes=355917824 token_bytes=573763520 weight_movement_bytes=69206016 total_nvlink_bytes=642969536 weight_to_token_ratio=0.120618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339290118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79472 payload_rows_remote=76746 dup_rows=18832 dup_rows_remote=18736 dedup_ratio=0.191569 router_tokens_per_expert=[733, 0, 2, 80, 33, 49, 123, 1249, 47, 7085, 190, 0, 6, 5266, 1, 1022, 688, 64, 2, 0, 66, 1, 814, 0, 340, 187, 5497, 21, 2, 124, 3, 0, 274, 1, 0, 47, 3, 6623, 5, 373, 1204, 74, 2039, 0, 92, 5274, 5312, 1, 238, 4, 42, 258, 31, 99, 30, 3226, 36, 4, 0, 0, 5, 2, 77, 83] payload_rows_per_dst=[4236, 4272, 12452, 12710, 13468, 15662, 13946, 2726] dup_rows_per_dst=[302, 598, 1350, 476, 1184, 9922, 4904, 96] combine_rows_per_src=[4236, 4272, 12452, 12710, 13468, 15662, 13946, 2726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=295796736 dispatch_meta_bytes=345640 combine_payload_bytes=295796736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296142376 ingress_bytes=330399744 link_bytes=330399744 token_bytes=591939112 weight_movement_bytes=34603008 total_nvlink_bytes=626542120 weight_to_token_ratio=0.058457 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243267637e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270524416 dispatch_meta_bytes=336160 combine_payload_bytes=270524416 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270860576 ingress_bytes=408936448 link_bytes=408936448 token_bytes=541384992 weight_movement_bytes=138412032 total_nvlink_bytes=679797024 weight_to_token_ratio=0.255663 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538794932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=284188672 dispatch_meta_bytes=359184 combine_payload_bytes=284188672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284547856 ingress_bytes=318791680 link_bytes=318791680 token_bytes=568736528 weight_movement_bytes=34603008 total_nvlink_bytes=603339536 weight_to_token_ratio=0.060842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199587427e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306921472 dispatch_meta_bytes=349128 combine_payload_bytes=306921472 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=307270600 ingress_bytes=410730496 link_bytes=410730496 token_bytes=614192072 weight_movement_bytes=103809024 total_nvlink_bytes=718001096 weight_to_token_ratio=0.169017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545545790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=251863040 dispatch_meta_bytes=301712 combine_payload_bytes=251863040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252164752 ingress_bytes=321069056 link_bytes=321069056 token_bytes=504027792 weight_movement_bytes=69206016 total_nvlink_bytes=573233808 weight_to_token_ratio=0.137306 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208157010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=256614400 dispatch_meta_bytes=303584 combine_payload_bytes=256614400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256917984 ingress_bytes=291217408 link_bytes=291217408 token_bytes=513532384 weight_movement_bytes=34603008 total_nvlink_bytes=548135392 weight_to_token_ratio=0.067382 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.095827661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=314351616 dispatch_meta_bytes=381928 combine_payload_bytes=314351616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314733544 ingress_bytes=348954624 link_bytes=348954624 token_bytes=629085160 weight_movement_bytes=34603008 total_nvlink_bytes=663688168 weight_to_token_ratio=0.055005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313088157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80170 payload_rows_remote=67076 dup_rows=18134 dup_rows_remote=12136 dedup_ratio=0.184469 router_tokens_per_expert=[337, 0, 57, 0, 574, 566, 1592, 57, 0, 875, 37, 20, 168, 24, 698, 9, 5663, 18, 5205, 476, 269, 1527, 159, 17, 0, 5614, 2, 660, 987, 2969, 728, 57, 1355, 350, 1951, 0, 549, 0, 4, 9, 23, 232, 86, 654, 2, 2, 306, 5241, 931, 2, 0, 1907, 374, 0, 14, 2082, 696, 0, 256, 1317, 0, 0, 10, 1434] payload_rows_per_dst=[7936, 11938, 12562, 13094, 7302, 11652, 9032, 6654] dup_rows_per_dst=[1372, 3050, 2780, 5998, 1134, 1440, 1588, 772] combine_rows_per_src=[7936, 11938, 12562, 13094, 7302, 11652, 9032, 6654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77796 payload_rows_remote=65862 dup_rows=20508 dup_rows_remote=15678 dedup_ratio=0.208618 router_tokens_per_expert=[388, 0, 67, 0, 1223, 479, 727, 44, 6, 1947, 150, 29, 157, 51, 862, 6, 6698, 3, 4248, 344, 1110, 2551, 124, 2, 0, 4214, 1, 402, 639, 2092, 1418, 63, 1616, 95, 872, 32, 461, 0, 5, 137, 67, 586, 131, 340, 3, 0, 310, 3434, 1254, 2, 0, 1810, 492, 0, 243, 1206, 784, 0, 493, 3854, 0, 1, 18, 861] payload_rows_per_dst=[11408, 14562, 11934, 8198, 5526, 8438, 7774, 9956] dup_rows_per_dst=[2876, 5250, 4830, 1032, 910, 1304, 2240, 2066] combine_rows_per_src=[11408, 14562, 11934, 8198, 5526, 8438, 7774, 9956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81488 payload_rows_remote=80966 dup_rows=16816 dup_rows_remote=16808 dedup_ratio=0.171061 router_tokens_per_expert=[273, 0, 86, 0, 2, 180, 4518, 100, 0, 3, 0, 0, 1, 0, 0, 0, 261, 0, 4931, 155, 90, 410, 138, 4, 0, 7790, 0, 77, 3535, 2966, 5, 0, 7, 0, 4427, 0, 1034, 2, 0, 78, 0, 0, 0, 630, 0, 0, 263, 7300, 4, 0, 0, 3393, 530, 0, 5, 2219, 154, 0, 15, 26, 60, 17, 0, 3463] payload_rows_per_dst=[16260, 522, 11104, 11068, 9984, 15138, 10132, 7280] dup_rows_per_dst=[9638, 8, 352, 2098, 1112, 1248, 2170, 190] combine_rows_per_src=[16260, 522, 11104, 11068, 9984, 15138, 10132, 7280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78448 payload_rows_remote=72128 dup_rows=19856 dup_rows_remote=19120 dedup_ratio=0.201986 router_tokens_per_expert=[2655, 1468, 3338, 5220, 1790, 84, 26, 3626, 1722, 1261, 242, 299, 0, 0, 577, 38, 95, 945, 1, 608, 55, 12, 0, 539, 664, 15, 161, 84, 0, 0, 1188, 806, 224, 9, 12, 68, 722, 191, 832, 2, 3127, 381, 61, 730, 1698, 8, 1, 1262, 929, 5655, 3, 59, 16, 514, 0, 0, 119, 18, 77, 1018, 3077, 603, 0, 217] payload_rows_per_dst=[12172, 7180, 12048, 9716, 6320, 7036, 12074, 11902] dup_rows_per_dst=[4880, 1098, 2902, 2374, 736, 1246, 2954, 3666] combine_rows_per_src=[12172, 7180, 12048, 9716, 6320, 7036, 12074, 11902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77464 payload_rows_remote=68136 dup_rows=20840 dup_rows_remote=19382 dedup_ratio=0.211995 router_tokens_per_expert=[374, 0, 69, 1, 1425, 508, 964, 46, 0, 1616, 117, 36, 486, 57, 897, 7, 6889, 15, 4063, 526, 583, 2253, 113, 14, 0, 4267, 0, 446, 630, 1736, 1505, 4, 2304, 504, 662, 2, 542, 0, 0, 22, 64, 389, 120, 512, 1, 1, 376, 3930, 1675, 3, 0, 1658, 511, 0, 123, 1450, 558, 0, 619, 2481, 0, 1, 1, 996] payload_rows_per_dst=[6226, 14638, 6260, 8778, 10470, 9328, 9196, 12568] dup_rows_per_dst=[548, 5572, 748, 1764, 3066, 1458, 2814, 4870] combine_rows_per_src=[6226, 14638, 6260, 8778, 10470, 9328, 9196, 12568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80788 payload_rows_remote=65212 dup_rows=17516 dup_rows_remote=12638 dedup_ratio=0.178182 router_tokens_per_expert=[438, 0, 3, 82, 255, 149, 118, 1784, 104, 7398, 442, 59, 59, 3893, 1, 1654, 2950, 245, 0, 0, 67, 53, 913, 0, 550, 81, 5906, 271, 35, 140, 3, 1, 422, 0, 0, 288, 167, 2889, 101, 561, 5400, 77, 617, 5, 174, 4816, 1465, 0, 913, 27, 436, 1230, 239, 82, 43, 1059, 294, 10, 0, 0, 12, 12, 13, 146] payload_rows_per_dst=[15576, 11842, 7078, 12842, 10966, 4896, 6744, 10844] dup_rows_per_dst=[4878, 4372, 1378, 1132, 3732, 36, 1314, 674] combine_rows_per_src=[15576, 11842, 7078, 12842, 10966, 4896, 6744, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=316848 combine_payload_bytes=274743296 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=275060144 ingress_bytes=413155328 link_bytes=413155328 token_bytes=549803440 weight_movement_bytes=138412032 total_nvlink_bytes=688215472 weight_to_token_ratio=0.251748 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554670238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74718 payload_rows_remote=63518 dup_rows=23586 dup_rows_remote=19496 dedup_ratio=0.239929 router_tokens_per_expert=[245, 0, 42, 0, 1352, 557, 824, 43, 0, 1841, 115, 43, 552, 54, 1031, 12, 6607, 24, 3586, 474, 495, 2157, 43, 3, 0, 3912, 8, 494, 537, 1987, 2068, 123, 2429, 382, 918, 6, 498, 0, 0, 119, 37, 960, 133, 580, 0, 0, 294, 2909, 2167, 3, 0, 1740, 325, 0, 81, 1342, 554, 0, 462, 3182, 0, 1, 14, 787] payload_rows_per_dst=[5750, 6868, 11846, 6250, 11972, 8416, 11200, 12416] dup_rows_per_dst=[376, 3676, 4512, 210, 4556, 1410, 4090, 4756] combine_rows_per_src=[5750, 6868, 11846, 6250, 11972, 8416, 11200, 12416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77902 payload_rows_remote=63964 dup_rows=20402 dup_rows_remote=16624 dedup_ratio=0.207540 router_tokens_per_expert=[136, 0, 82, 1, 44, 116, 4068, 176, 0, 85, 2, 0, 33, 0, 58, 0, 1163, 0, 5477, 208, 37, 486, 110, 0, 0, 7583, 0, 90, 3564, 2267, 61, 0, 35, 160, 4508, 0, 816, 1, 0, 7, 1, 30, 29, 684, 0, 1, 260, 7139, 11, 0, 0, 3189, 232, 0, 67, 2754, 238, 0, 81, 26, 28, 4, 0, 3004] payload_rows_per_dst=[8880, 316, 3698, 7282, 16226, 14902, 12660, 13938] dup_rows_per_dst=[366, 40, 310, 148, 9994, 1386, 4380, 3778] combine_rows_per_src=[8880, 316, 3698, 7282, 16226, 14902, 12660, 13938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269770752 dispatch_meta_bytes=326160 combine_payload_bytes=269770752 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270096912 ingress_bytes=373579776 link_bytes=373579776 token_bytes=539867664 weight_movement_bytes=103809024 total_nvlink_bytes=643676688 weight_to_token_ratio=0.192286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405750621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331636736 dispatch_meta_bytes=391096 combine_payload_bytes=331636736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332027832 ingress_bytes=366239744 link_bytes=366239744 token_bytes=663664568 weight_movement_bytes=34603008 total_nvlink_bytes=698267576 weight_to_token_ratio=0.052139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378130672e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295436288 dispatch_meta_bytes=364992 combine_payload_bytes=295436288 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=295801280 ingress_bytes=364642304 link_bytes=364642304 token_bytes=591237568 weight_movement_bytes=69206016 total_nvlink_bytes=660443584 weight_to_token_ratio=0.117053 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372119634e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279085056 dispatch_meta_bytes=350072 combine_payload_bytes=279085056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279435128 ingress_bytes=348291072 link_bytes=348291072 token_bytes=558520184 weight_movement_bytes=69206016 total_nvlink_bytes=627726200 weight_to_token_ratio=0.123910 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310591264e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=311400 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267419752 ingress_bytes=301711360 link_bytes=301711360 token_bytes=534528104 weight_movement_bytes=34603008 total_nvlink_bytes=569131112 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135315558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=260169728 dispatch_meta_bytes=332056 combine_payload_bytes=260169728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260501784 ingress_bytes=294772736 link_bytes=294772736 token_bytes=520671512 weight_movement_bytes=34603008 total_nvlink_bytes=555274520 weight_to_token_ratio=0.066458 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109206074e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=261996544 dispatch_meta_bytes=322352 combine_payload_bytes=261996544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262318896 ingress_bytes=296599552 link_bytes=296599552 token_bytes=524315440 weight_movement_bytes=34603008 total_nvlink_bytes=558918448 weight_to_token_ratio=0.065997 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116080236e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81388 payload_rows_remote=69252 dup_rows=16916 dup_rows_remote=12728 dedup_ratio=0.172078 router_tokens_per_expert=[3123, 2157, 2393, 6053, 2053, 21, 4, 2349, 1228, 1201, 229, 601, 2, 0, 422, 37, 29, 368, 0, 175, 12, 11, 0, 388, 410, 1, 125, 68, 0, 0, 2546, 1306, 40, 0, 0, 231, 677, 372, 753, 0, 3706, 302, 17, 487, 1826, 15, 0, 2727, 677, 6005, 1, 95, 2, 500, 0, 0, 21, 2, 28, 902, 1768, 632, 0, 54] payload_rows_per_dst=[10814, 6288, 12462, 12136, 7632, 8978, 12566, 10512] dup_rows_per_dst=[2826, 1152, 1610, 4188, 828, 1770, 1994, 2548] combine_rows_per_src=[10814, 6288, 12462, 12136, 7632, 8978, 12566, 10512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75920 payload_rows_remote=63506 dup_rows=22384 dup_rows_remote=19780 dedup_ratio=0.227702 router_tokens_per_expert=[2432, 1414, 4229, 5397, 1121, 39, 3, 3949, 1749, 1171, 205, 264, 0, 0, 654, 18, 10, 592, 0, 325, 5, 19, 0, 1161, 317, 7, 100, 50, 0, 0, 1915, 1438, 167, 24, 0, 128, 632, 194, 717, 0, 4126, 393, 4, 241, 1699, 8, 0, 1085, 601, 5437, 1, 2, 26, 731, 0, 0, 14, 29, 208, 901, 2271, 647, 2, 280] payload_rows_per_dst=[12786, 11250, 12414, 11976, 5936, 5884, 5282, 10392] dup_rows_per_dst=[5896, 4096, 2604, 3930, 616, 976, 1090, 3176] combine_rows_per_src=[12786, 11250, 12414, 11976, 5936, 5884, 5282, 10392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77818 payload_rows_remote=73424 dup_rows=20486 dup_rows_remote=19884 dedup_ratio=0.208394 router_tokens_per_expert=[2, 1, 147, 507, 792, 893, 0, 350, 93, 2149, 0, 7061, 0, 2, 9, 7558, 72, 2649, 7, 1092, 100, 21, 2791, 0, 114, 145, 92, 690, 607, 1, 159, 770, 1245, 596, 40, 159, 9, 309, 140, 0, 536, 4, 354, 695, 170, 1616, 2595, 124, 552, 410, 34, 952, 437, 1330, 1674, 344, 61, 2327, 761, 17, 259, 269, 34, 2225] payload_rows_per_dst=[7506, 11502, 7092, 15986, 4394, 12384, 9688, 9266] dup_rows_per_dst=[1540, 3464, 790, 4286, 602, 5386, 1778, 2640] combine_rows_per_src=[7506, 11502, 7092, 15986, 4394, 12384, 9688, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75230 payload_rows_remote=59160 dup_rows=23074 dup_rows_remote=15634 dedup_ratio=0.234721 router_tokens_per_expert=[3931, 5817, 386, 1410, 2160, 13, 0, 14, 6, 142, 0, 3504, 134, 4, 255, 12, 0, 5, 353, 0, 0, 0, 2, 17, 0, 1, 7, 0, 0, 0, 3218, 797, 6, 0, 0, 2508, 10, 26, 1217, 0, 1605, 931, 0, 4, 1624, 1, 0, 7444, 549, 7698, 0, 248, 1, 1108, 5, 0, 0, 2, 411, 1053, 18, 493, 0, 2] payload_rows_per_dst=[5946, 16070, 3504, 9600, 13380, 14396, 2892, 9442] dup_rows_per_dst=[526, 7440, 70, 1656, 5788, 4286, 930, 2378] combine_rows_per_src=[5946, 16070, 3504, 9600, 13380, 14396, 2892, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77258 payload_rows_remote=68376 dup_rows=21046 dup_rows_remote=18620 dedup_ratio=0.214091 router_tokens_per_expert=[1878, 1615, 3657, 6058, 1059, 24, 16, 4446, 2201, 1153, 333, 268, 0, 0, 578, 48, 43, 811, 0, 380, 22, 28, 0, 1181, 426, 1, 106, 125, 0, 0, 1685, 967, 175, 5, 0, 61, 1012, 444, 505, 0, 4412, 202, 1, 722, 1503, 3, 0, 1094, 596, 4955, 4, 14, 1, 495, 0, 0, 0, 4, 8, 510, 2652, 587, 0, 78] payload_rows_per_dst=[5338, 7792, 13442, 9162, 10310, 8882, 12690, 9642] dup_rows_per_dst=[108, 1370, 3604, 2024, 2968, 2426, 6754, 1792] combine_rows_per_src=[5338, 7792, 13442, 9162, 10310, 8882, 12690, 9642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80222 payload_rows_remote=66370 dup_rows=18082 dup_rows_remote=12926 dedup_ratio=0.183940 router_tokens_per_expert=[508, 0, 76, 0, 286, 632, 1934, 36, 1, 564, 118, 17, 85, 68, 546, 2, 5702, 16, 4957, 452, 476, 1519, 144, 6, 0, 6032, 1, 463, 918, 2646, 400, 1, 898, 270, 1664, 8, 899, 1, 0, 32, 30, 64, 52, 574, 2, 3, 289, 5773, 320, 5, 0, 1857, 589, 0, 153, 2627, 1041, 1, 269, 1261, 20, 5, 0, 1839] payload_rows_per_dst=[13852, 12016, 12212, 10780, 6352, 8020, 9154, 7836] dup_rows_per_dst=[5156, 2190, 2928, 2792, 1192, 840, 1948, 1036] combine_rows_per_src=[13852, 12016, 12212, 10780, 6352, 8020, 9154, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78294 payload_rows_remote=66968 dup_rows=20010 dup_rows_remote=14874 dedup_ratio=0.203552 router_tokens_per_expert=[2678, 1214, 3064, 5648, 1681, 44, 39, 4872, 2510, 1330, 275, 229, 0, 0, 552, 34, 116, 911, 0, 347, 27, 29, 0, 793, 901, 4, 101, 144, 0, 0, 1629, 971, 209, 9, 0, 91, 629, 162, 547, 2, 4489, 218, 24, 1072, 1470, 1, 0, 677, 666, 3957, 1, 3, 51, 489, 0, 0, 26, 20, 1, 1039, 2285, 631, 0, 240] payload_rows_per_dst=[3522, 8376, 12752, 6460, 12774, 12200, 11326, 10884] dup_rows_per_dst=[6, 1484, 2990, 1040, 2696, 3702, 5136, 2956] combine_rows_per_src=[3522, 8376, 12752, 6460, 12774, 12200, 11326, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=283656192 dispatch_meta_bytes=327920 combine_payload_bytes=283656192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283984112 ingress_bytes=318259200 link_bytes=318259200 token_bytes=567640304 weight_movement_bytes=34603008 total_nvlink_bytes=602243312 weight_to_token_ratio=0.060959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197583748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80248 payload_rows_remote=71432 dup_rows=18056 dup_rows_remote=16172 dedup_ratio=0.183675 router_tokens_per_expert=[3163, 5902, 582, 3416, 3200, 3, 1, 374, 153, 272, 13, 2383, 41, 6, 285, 15, 1, 41, 176, 3, 0, 4, 0, 47, 12, 1, 72, 15, 3, 0, 3108, 1213, 27, 5, 0, 1122, 106, 66, 1090, 1, 1847, 387, 0, 26, 1833, 10, 0, 6679, 592, 7494, 3, 221, 0, 946, 5, 0, 0, 6, 160, 1030, 374, 616, 0, 1] payload_rows_per_dst=[6408, 6120, 7210, 7786, 12984, 15144, 15780, 8816] dup_rows_per_dst=[0, 216, 166, 1062, 4402, 6420, 3906, 1884] combine_rows_per_src=[6408, 6120, 7210, 7786, 12984, 15144, 15780, 8816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=300744704 dispatch_meta_bytes=373232 combine_payload_bytes=300744704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301117936 ingress_bytes=335347712 link_bytes=335347712 token_bytes=601862640 weight_movement_bytes=34603008 total_nvlink_bytes=636465648 weight_to_token_ratio=0.057493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260120576 dispatch_meta_bytes=333144 combine_payload_bytes=260120576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260453720 ingress_bytes=294723584 link_bytes=294723584 token_bytes=520574296 weight_movement_bytes=34603008 total_nvlink_bytes=555177304 weight_to_token_ratio=0.066471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109021119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=242319360 dispatch_meta_bytes=299176 combine_payload_bytes=242319360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242618536 ingress_bytes=276922368 link_bytes=276922368 token_bytes=484937896 weight_movement_bytes=34603008 total_nvlink_bytes=519540904 weight_to_token_ratio=0.071356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.042036577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=347984 combine_payload_bytes=280068096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=280416080 ingress_bytes=349274112 link_bytes=349274112 token_bytes=560484176 weight_movement_bytes=69206016 total_nvlink_bytes=629690192 weight_to_token_ratio=0.123475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314290365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=271851520 dispatch_meta_bytes=317184 combine_payload_bytes=271851520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272168704 ingress_bytes=306454528 link_bytes=306454528 token_bytes=544020224 weight_movement_bytes=34603008 total_nvlink_bytes=578623232 weight_to_token_ratio=0.063606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153163717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=274300928 dispatch_meta_bytes=327368 combine_payload_bytes=274300928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274628296 ingress_bytes=343506944 link_bytes=343506944 token_bytes=548929224 weight_movement_bytes=69206016 total_nvlink_bytes=618135240 weight_to_token_ratio=0.126075 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292588976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=292585472 dispatch_meta_bytes=350416 combine_payload_bytes=292585472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292935888 ingress_bytes=327188480 link_bytes=327188480 token_bytes=585521360 weight_movement_bytes=34603008 total_nvlink_bytes=620124368 weight_to_token_ratio=0.059098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231183909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79720 payload_rows_remote=63536 dup_rows=18584 dup_rows_remote=15090 dedup_ratio=0.189046 router_tokens_per_expert=[0, 4, 859, 437, 472, 672, 1, 316, 38, 2233, 0, 6324, 0, 0, 7, 7859, 137, 1898, 4, 1965, 11, 21, 4604, 0, 42, 80, 65, 920, 294, 0, 127, 452, 2155, 563, 4, 155, 7, 470, 79, 0, 209, 2, 225, 333, 209, 1626, 965, 97, 266, 376, 28, 635, 385, 1445, 899, 150, 52, 3489, 681, 15, 222, 142, 0, 3426] payload_rows_per_dst=[14428, 4542, 7134, 16184, 6338, 12678, 7092, 11324] dup_rows_per_dst=[3742, 14, 938, 3494, 528, 3862, 1276, 4730] combine_rows_per_src=[14428, 4542, 7134, 16184, 6338, 12678, 7092, 11324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82866 payload_rows_remote=77250 dup_rows=15438 dup_rows_remote=14938 dedup_ratio=0.157043 router_tokens_per_expert=[1404, 226, 54, 531, 1915, 334, 0, 0, 0, 135, 460, 96, 4638, 12, 710, 320, 0, 508, 95, 4323, 0, 667, 339, 44, 2163, 0, 501, 2778, 3, 2730, 380, 611, 5253, 517, 1279, 499, 776, 423, 137, 706, 0, 434, 534, 232, 520, 174, 914, 67, 2930, 0, 392, 385, 652, 578, 1266, 580, 582, 2847, 220, 0, 7, 0, 178, 93] payload_rows_per_dst=[9404, 11752, 10906, 10078, 5616, 13474, 11092, 10544] dup_rows_per_dst=[2082, 990, 3474, 1500, 500, 2782, 2474, 1636] combine_rows_per_src=[9404, 11752, 10906, 10078, 5616, 13474, 11092, 10544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78682 payload_rows_remote=69772 dup_rows=19622 dup_rows_remote=17362 dedup_ratio=0.199605 router_tokens_per_expert=[0, 9, 122, 521, 505, 1020, 0, 462, 42, 1536, 0, 6568, 1, 1, 32, 8003, 6, 3043, 17, 915, 91, 32, 2030, 0, 25, 187, 9, 774, 679, 0, 240, 320, 1532, 404, 16, 132, 1, 45, 116, 0, 103, 0, 291, 295, 250, 1140, 3485, 101, 633, 738, 54, 788, 76, 1741, 1747, 125, 106, 4207, 550, 105, 293, 181, 1, 2706] payload_rows_per_dst=[14192, 3212, 8910, 16080, 4406, 10074, 9908, 11900] dup_rows_per_dst=[4222, 12, 2260, 4394, 86, 2354, 1896, 4398] combine_rows_per_src=[14192, 3212, 8910, 16080, 4406, 10074, 9908, 11900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82512 payload_rows_remote=73374 dup_rows=15792 dup_rows_remote=14220 dedup_ratio=0.160645 router_tokens_per_expert=[0, 1, 5528, 2, 286, 52, 0, 541, 0, 3757, 0, 368, 0, 0, 1, 7810, 0, 11, 1, 6409, 1, 0, 5817, 0, 0, 1, 0, 186, 8, 0, 0, 2, 5880, 55, 0, 0, 0, 1210, 0, 0, 2, 0, 0, 5, 160, 3181, 38, 190, 13, 54, 2, 122, 1, 3126, 4, 0, 0, 3342, 0, 0, 80, 0, 0, 905] payload_rows_per_dst=[12576, 9138, 15014, 12614, 12364, 6904, 6596, 7306] dup_rows_per_dst=[980, 1572, 9464, 206, 1926, 248, 48, 1348] combine_rows_per_src=[12576, 9138, 15014, 12614, 12364, 6904, 6596, 7306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78576 payload_rows_remote=66488 dup_rows=19728 dup_rows_remote=15040 dedup_ratio=0.200684 router_tokens_per_expert=[1, 1, 104, 604, 1369, 745, 0, 300, 20, 1070, 0, 6944, 0, 0, 8, 7753, 106, 2473, 13, 759, 50, 8, 2296, 0, 222, 379, 51, 1303, 413, 1, 192, 577, 886, 525, 7, 211, 5, 237, 364, 0, 537, 8, 403, 457, 153, 1735, 2705, 94, 638, 762, 35, 853, 517, 1590, 2216, 177, 105, 2558, 840, 54, 258, 228, 0, 2232] payload_rows_per_dst=[5828, 14112, 6296, 10612, 9204, 12088, 10968, 9468] dup_rows_per_dst=[420, 1972, 522, 3804, 2632, 4688, 2608, 3082] combine_rows_per_src=[5828, 14112, 6296, 10612, 9204, 12088, 10968, 9468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79042 payload_rows_remote=66236 dup_rows=19262 dup_rows_remote=14702 dedup_ratio=0.195943 router_tokens_per_expert=[5, 1, 26, 930, 790, 616, 0, 314, 34, 1643, 0, 7025, 0, 0, 5, 7547, 60, 2703, 12, 549, 10, 19, 2264, 0, 167, 193, 172, 998, 470, 2, 143, 988, 763, 870, 14, 306, 7, 227, 231, 2, 639, 0, 506, 831, 112, 1060, 2903, 90, 406, 323, 30, 1390, 922, 807, 2318, 302, 73, 2407, 632, 12, 254, 311, 1, 2717] payload_rows_per_dst=[5140, 14352, 6372, 5748, 15516, 9582, 12806, 9526] dup_rows_per_dst=[224, 3062, 334, 518, 4418, 2858, 4560, 3288] combine_rows_per_src=[5140, 14352, 6372, 5748, 15516, 9582, 12806, 9526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=59640 dup_rows=23992 dup_rows_remote=11588 dedup_ratio=0.244059 router_tokens_per_expert=[2757, 3726, 3197, 5469, 1844, 12, 1, 2285, 796, 441, 215, 760, 1, 0, 299, 7, 12, 229, 0, 53, 13, 4, 0, 398, 175, 5, 172, 15, 0, 0, 2374, 1085, 32, 0, 0, 355, 362, 187, 805, 0, 3771, 447, 4, 101, 1898, 29, 0, 2883, 667, 6667, 14, 146, 3, 635, 0, 0, 0, 15, 285, 942, 1915, 613, 0, 31] payload_rows_per_dst=[14672, 14544, 10014, 11698, 9884, 4676, 1954, 6870] dup_rows_per_dst=[12404, 3828, 512, 3496, 1050, 282, 976, 1444] combine_rows_per_src=[14672, 14544, 10014, 11698, 9884, 4676, 1954, 6870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260243456 dispatch_meta_bytes=314504 combine_payload_bytes=260243456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260557960 ingress_bytes=294846464 link_bytes=294846464 token_bytes=520801416 weight_movement_bytes=34603008 total_nvlink_bytes=555404424 weight_to_token_ratio=0.066442 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109483507e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316416000 dispatch_meta_bytes=368752 combine_payload_bytes=316416000 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=316784752 ingress_bytes=454828032 link_bytes=454828032 token_bytes=633200752 weight_movement_bytes=138412032 total_nvlink_bytes=771612784 weight_to_token_ratio=0.218591 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.711481268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77746 payload_rows_remote=69020 dup_rows=20558 dup_rows_remote=18598 dedup_ratio=0.209127 router_tokens_per_expert=[0, 0, 4183, 26, 298, 196, 0, 529, 10, 3695, 1, 1473, 8, 0, 0, 8150, 0, 358, 19, 4764, 1, 1, 6151, 0, 8, 10, 1, 719, 12, 0, 8, 13, 5252, 310, 0, 13, 8, 762, 0, 0, 70, 0, 25, 2, 241, 3277, 149, 212, 58, 176, 3, 177, 71, 2239, 115, 15, 8, 3785, 42, 3, 197, 13, 0, 1295] payload_rows_per_dst=[9962, 9296, 9978, 1532, 16374, 7508, 14370, 8726] dup_rows_per_dst=[502, 1078, 308, 10, 12616, 444, 3640, 1960] combine_rows_per_src=[9962, 9296, 9978, 1532, 16374, 7508, 14370, 8726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=285786112 dispatch_meta_bytes=348536 combine_payload_bytes=285786112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286134648 ingress_bytes=389595136 link_bytes=389595136 token_bytes=571920760 weight_movement_bytes=103809024 total_nvlink_bytes=675729784 weight_to_token_ratio=0.181509 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466015132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=300539904 dispatch_meta_bytes=350376 combine_payload_bytes=300539904 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=300890280 ingress_bytes=438951936 link_bytes=438951936 token_bytes=601430184 weight_movement_bytes=138412032 total_nvlink_bytes=739842216 weight_to_token_ratio=0.230138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651740797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272334848 dispatch_meta_bytes=326112 combine_payload_bytes=272334848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272660960 ingress_bytes=306937856 link_bytes=306937856 token_bytes=544995808 weight_movement_bytes=34603008 total_nvlink_bytes=579598816 weight_to_token_ratio=0.063492 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154982442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271302656 dispatch_meta_bytes=323752 combine_payload_bytes=271302656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271626408 ingress_bytes=305905664 link_bytes=305905664 token_bytes=542929064 weight_movement_bytes=34603008 total_nvlink_bytes=577532072 weight_to_token_ratio=0.063734 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151098386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=244285440 dispatch_meta_bytes=284912 combine_payload_bytes=244285440 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=244570352 ingress_bytes=451903488 link_bytes=451903488 token_bytes=488855792 weight_movement_bytes=207618048 total_nvlink_bytes=696473840 weight_to_token_ratio=0.424702 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.700476444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=282705920 dispatch_meta_bytes=350472 combine_payload_bytes=282705920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283056392 ingress_bytes=351911936 link_bytes=351911936 token_bytes=565762312 weight_movement_bytes=69206016 total_nvlink_bytes=634968328 weight_to_token_ratio=0.122323 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324216284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85416 payload_rows_remote=77748 dup_rows=12888 dup_rows_remote=12244 dedup_ratio=0.131104 router_tokens_per_expert=[1452, 166, 122, 596, 876, 483, 0, 1, 0, 116, 473, 195, 5924, 7, 529, 127, 0, 150, 11, 3609, 0, 457, 142, 83, 3687, 0, 246, 2837, 4, 2701, 357, 712, 6499, 579, 1671, 142, 1345, 247, 128, 623, 2, 477, 332, 45, 413, 93, 469, 67, 3407, 0, 541, 341, 586, 566, 810, 594, 253, 2049, 610, 20, 3, 0, 29, 148] payload_rows_per_dst=[9234, 13814, 11216, 7668, 5780, 14572, 11114, 12018] dup_rows_per_dst=[1500, 928, 3090, 644, 348, 2222, 2576, 1580] combine_rows_per_src=[9234, 13814, 11216, 7668, 5780, 14572, 11114, 12018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79270 payload_rows_remote=66808 dup_rows=19034 dup_rows_remote=15414 dedup_ratio=0.193624 router_tokens_per_expert=[91, 7, 120, 52, 22, 17, 11, 14, 687, 46, 1576, 1, 1379, 83, 1218, 4, 1405, 1565, 1082, 116, 79, 788, 5, 4232, 1, 398, 127, 713, 6397, 132, 0, 1337, 2192, 5, 1, 0, 2999, 258, 2585, 1, 0, 10, 6125, 1613, 0, 1557, 51, 9, 2, 442, 3355, 0, 239, 17, 6, 63, 46, 154, 2704, 0, 860, 51, 0, 102] payload_rows_per_dst=[8978, 8638, 8700, 4964, 12462, 6370, 14250, 14908] dup_rows_per_dst=[154, 1350, 1380, 452, 3620, 110, 6248, 5720] combine_rows_per_src=[8978, 8638, 8700, 4964, 12462, 6370, 14250, 14908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81650 payload_rows_remote=68814 dup_rows=16654 dup_rows_remote=11706 dedup_ratio=0.169413 router_tokens_per_expert=[1823, 259, 51, 222, 2169, 419, 0, 0, 0, 235, 514, 53, 5197, 0, 693, 239, 0, 476, 0, 5032, 0, 605, 266, 2, 763, 0, 128, 2107, 10, 2511, 217, 934, 5951, 764, 2031, 166, 607, 997, 137, 426, 0, 584, 547, 99, 305, 126, 544, 59, 3584, 1, 602, 408, 155, 522, 722, 765, 454, 3279, 188, 1, 9, 5, 71, 118] payload_rows_per_dst=[10788, 12534, 12836, 6252, 6244, 12966, 11048, 8982] dup_rows_per_dst=[3160, 1328, 4948, 540, 880, 2534, 2470, 794] combine_rows_per_src=[10788, 12534, 12836, 6252, 6244, 12966, 11048, 8982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84040 payload_rows_remote=70724 dup_rows=14264 dup_rows_remote=11832 dedup_ratio=0.145101 router_tokens_per_expert=[2223, 135, 70, 218, 2088, 209, 0, 0, 0, 114, 329, 130, 4461, 3, 856, 288, 0, 780, 0, 4319, 0, 523, 319, 16, 1815, 0, 301, 2727, 0, 1774, 823, 771, 5227, 361, 1202, 137, 488, 320, 130, 616, 0, 501, 368, 39, 689, 179, 847, 24, 3866, 0, 854, 474, 188, 734, 667, 648, 660, 4303, 208, 0, 14, 0, 4, 112] payload_rows_per_dst=[9828, 11396, 10446, 10822, 3942, 13316, 12116, 12174] dup_rows_per_dst=[2462, 966, 1468, 1970, 162, 2432, 2746, 2058] combine_rows_per_src=[9828, 11396, 10446, 10822, 3942, 13316, 12116, 12174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77732 payload_rows_remote=64470 dup_rows=20572 dup_rows_remote=20338 dedup_ratio=0.209269 router_tokens_per_expert=[4, 236, 77, 1194, 24, 410, 0, 0, 45, 0, 99, 0, 6603, 0, 0, 1, 0, 0, 0, 54, 0, 4, 72, 1682, 6553, 0, 1, 844, 8, 4043, 3, 1, 8185, 1672, 5399, 0, 4385, 0, 123, 1, 0, 726, 28, 1, 1, 3, 2, 177, 961, 0, 474, 6, 348, 5, 5, 470, 2, 14, 717, 3474, 0, 2, 1, 12] payload_rows_per_dst=[11902, 13262, 10098, 9046, 16370, 1872, 4218, 10964] dup_rows_per_dst=[2786, 234, 1612, 852, 12362, 4, 320, 2402] combine_rows_per_src=[11902, 13262, 10098, 9046, 16370, 1872, 4218, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83188 payload_rows_remote=70278 dup_rows=15116 dup_rows_remote=10516 dedup_ratio=0.153768 router_tokens_per_expert=[2287, 46, 31, 336, 1853, 268, 2, 1, 0, 195, 338, 110, 3924, 13, 1505, 411, 0, 574, 6, 4187, 0, 539, 219, 2, 1411, 0, 705, 2506, 3, 1904, 629, 798, 4289, 151, 836, 359, 285, 466, 120, 936, 0, 423, 366, 140, 1098, 147, 666, 9, 4174, 0, 358, 624, 458, 1251, 1239, 651, 465, 4574, 140, 0, 5, 0, 15, 104] payload_rows_per_dst=[9288, 11620, 9810, 10906, 4360, 12266, 12910, 12028] dup_rows_per_dst=[2032, 1372, 1244, 2184, 274, 2010, 4600, 1400] combine_rows_per_src=[9288, 11620, 9810, 10906, 4360, 12266, 12910, 12028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318455808 dispatch_meta_bytes=359968 combine_payload_bytes=318455808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=318815776 ingress_bytes=422264832 link_bytes=422264832 token_bytes=637271584 weight_movement_bytes=103809024 total_nvlink_bytes=741080608 weight_to_token_ratio=0.162896 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.588948568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77714 payload_rows_remote=64250 dup_rows=20590 dup_rows_remote=17012 dedup_ratio=0.209452 router_tokens_per_expert=[0, 8, 1417, 137, 467, 825, 0, 393, 3, 1924, 0, 5274, 0, 0, 2, 8092, 14, 2067, 1, 2262, 7, 2, 3829, 0, 13, 120, 2, 656, 419, 8, 145, 55, 2701, 317, 0, 66, 0, 317, 49, 0, 17, 0, 118, 59, 212, 1736, 1372, 109, 872, 808, 35, 647, 108, 2559, 422, 84, 11, 4217, 354, 1, 250, 115, 0, 3454] payload_rows_per_dst=[13464, 16214, 14320, 2690, 6730, 6720, 9388, 8188] dup_rows_per_dst=[3578, 4762, 8572, 146, 170, 526, 1682, 1154] combine_rows_per_src=[13464, 16214, 14320, 2690, 6730, 6720, 9388, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=273645568 dispatch_meta_bytes=328888 combine_payload_bytes=273645568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273974456 ingress_bytes=377454592 link_bytes=377454592 token_bytes=547620024 weight_movement_bytes=103809024 total_nvlink_bytes=651429048 weight_to_token_ratio=0.189564 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420331242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76344 payload_rows_remote=62674 dup_rows=21960 dup_rows_remote=18214 dedup_ratio=0.223389 router_tokens_per_expert=[206, 328, 99, 963, 260, 603, 1, 1, 16, 19, 322, 1, 6988, 0, 31, 21, 0, 9, 0, 489, 0, 69, 207, 1018, 5935, 0, 1, 1793, 5, 4379, 9, 74, 8092, 921, 5235, 18, 3798, 28, 250, 14, 0, 630, 100, 0, 38, 1, 53, 145, 1728, 2, 398, 39, 562, 22, 22, 436, 48, 109, 897, 1655, 1, 0, 4, 59] payload_rows_per_dst=[6438, 3778, 3424, 14614, 12880, 13082, 8458, 13670] dup_rows_per_dst=[1182, 328, 160, 8248, 5300, 1434, 1562, 3746] combine_rows_per_src=[6438, 3778, 3424, 14614, 12880, 13082, 8458, 13670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=281862144 dispatch_meta_bytes=322080 combine_payload_bytes=281862144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282184224 ingress_bytes=316465152 link_bytes=316465152 token_bytes=564046368 weight_movement_bytes=34603008 total_nvlink_bytes=598649376 weight_to_token_ratio=0.061348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190832889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=289685504 dispatch_meta_bytes=330224 combine_payload_bytes=289685504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290015728 ingress_bytes=324288512 link_bytes=324288512 token_bytes=579701232 weight_movement_bytes=34603008 total_nvlink_bytes=614304240 weight_to_token_ratio=0.059691 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220271563e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264069120 dispatch_meta_bytes=339232 combine_payload_bytes=264069120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264408352 ingress_bytes=333275136 link_bytes=333275136 token_bytes=528477472 weight_movement_bytes=69206016 total_nvlink_bytes=597683488 weight_to_token_ratio=0.130954 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254087506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287858688 dispatch_meta_bytes=323176 combine_payload_bytes=287858688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288181864 ingress_bytes=322461696 link_bytes=322461696 token_bytes=576040552 weight_movement_bytes=34603008 total_nvlink_bytes=610643560 weight_to_token_ratio=0.060070 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213397402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263168000 dispatch_meta_bytes=325048 combine_payload_bytes=263168000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263493048 ingress_bytes=297771008 link_bytes=297771008 token_bytes=526661048 weight_movement_bytes=34603008 total_nvlink_bytes=561264056 weight_to_token_ratio=0.065703 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120488331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=256712704 dispatch_meta_bytes=323552 combine_payload_bytes=256712704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257036256 ingress_bytes=291315712 link_bytes=291315712 token_bytes=513748960 weight_movement_bytes=34603008 total_nvlink_bytes=548351968 weight_to_token_ratio=0.067354 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096197571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82440 payload_rows_remote=68234 dup_rows=15864 dup_rows_remote=12188 dedup_ratio=0.161377 router_tokens_per_expert=[124, 3, 86, 43, 14, 27, 0, 0, 570, 4, 1649, 0, 971, 102, 1569, 0, 1301, 1690, 2393, 63, 84, 384, 8, 5185, 0, 236, 26, 779, 6950, 344, 0, 1422, 1989, 0, 0, 0, 3751, 170, 3234, 0, 1, 0, 3802, 757, 0, 3340, 90, 0, 0, 437, 2889, 0, 373, 2, 20, 76, 148, 13, 1341, 0, 545, 117, 0, 30] payload_rows_per_dst=[10836, 8556, 9952, 14206, 13596, 10592, 9230, 5472] dup_rows_per_dst=[128, 1174, 1894, 3676, 4692, 1508, 2244, 548] combine_rows_per_src=[10836, 8556, 9952, 14206, 13596, 10592, 9230, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74688 payload_rows_remote=61946 dup_rows=23616 dup_rows_remote=17788 dedup_ratio=0.240234 router_tokens_per_expert=[738, 56, 307, 213, 946, 65, 1, 1371, 2085, 53, 85, 0, 284, 155, 8, 31, 627, 550, 606, 1127, 1231, 2, 193, 185, 757, 240, 130, 0, 1136, 95, 47, 5885, 1268, 1218, 224, 0, 1767, 29, 410, 4292, 678, 730, 74, 3128, 961, 140, 2106, 832, 0, 530, 12, 126, 1068, 304, 355, 44, 1817, 15, 384, 224, 76, 5748, 190, 1193] payload_rows_per_dst=[6616, 9614, 8128, 13210, 12742, 11556, 4440, 8382] dup_rows_per_dst=[778, 2626, 2714, 3370, 5828, 3942, 438, 3920] combine_rows_per_src=[6616, 9614, 8128, 13210, 12742, 11556, 4440, 8382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81306 payload_rows_remote=70128 dup_rows=16998 dup_rows_remote=12610 dedup_ratio=0.172913 router_tokens_per_expert=[0, 41, 260, 216, 10, 112, 6, 0, 383, 41, 1723, 0, 747, 30, 1305, 1, 1263, 1818, 1795, 245, 109, 288, 2, 5278, 0, 445, 210, 236, 7477, 113, 0, 1540, 1608, 4, 0, 2, 2397, 150, 3098, 11, 15, 27, 6293, 1552, 0, 1835, 25, 5, 28, 613, 2625, 0, 321, 85, 25, 39, 25, 51, 2391, 0, 126, 16, 9, 82] payload_rows_per_dst=[11400, 7526, 11178, 15828, 8826, 14190, 7032, 5326] dup_rows_per_dst=[446, 934, 4388, 4214, 1188, 5314, 440, 74] combine_rows_per_src=[11400, 7526, 11178, 15828, 8826, 14190, 7032, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79684 payload_rows_remote=73818 dup_rows=18620 dup_rows_remote=18538 dedup_ratio=0.189412 router_tokens_per_expert=[33, 16, 493, 236, 13, 27, 0, 0, 1191, 21, 2399, 1, 1205, 31, 966, 1, 1168, 1301, 1817, 65, 46, 440, 9, 4158, 0, 494, 317, 783, 7312, 7, 0, 925, 2672, 0, 0, 0, 2703, 196, 1698, 0, 0, 0, 6586, 1980, 0, 978, 15, 1, 11, 523, 3169, 0, 223, 8, 9, 72, 32, 38, 2189, 0, 422, 95, 4, 53] payload_rows_per_dst=[9536, 9566, 8422, 4576, 11714, 5866, 14616, 15388] dup_rows_per_dst=[416, 2064, 1270, 476, 2824, 82, 6586, 4902] combine_rows_per_src=[9536, 9566, 8422, 4576, 11714, 5866, 14616, 15388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82774 payload_rows_remote=71150 dup_rows=15530 dup_rows_remote=14828 dedup_ratio=0.157979 router_tokens_per_expert=[386, 3, 0, 0, 43, 242, 0, 0, 90, 0, 303, 0, 5, 412, 5353, 0, 590, 322, 3241, 11, 179, 3, 0, 7427, 0, 0, 2, 0, 7203, 4902, 0, 39, 9, 0, 0, 0, 2614, 0, 3379, 0, 0, 0, 18, 3, 0, 8000, 532, 0, 1, 16, 414, 0, 231, 0, 1145, 274, 1718, 0, 39, 0, 1, 0, 0, 2] payload_rows_per_dst=[15370, 11624, 11514, 15668, 5238, 16034, 3806, 3520] dup_rows_per_dst=[832, 702, 3936, 8624, 8, 1072, 356, 0] combine_rows_per_src=[15370, 11624, 11514, 15668, 5238, 16034, 3806, 3520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79536 payload_rows_remote=64532 dup_rows=18768 dup_rows_remote=12848 dedup_ratio=0.190918 router_tokens_per_expert=[0, 4, 340, 174, 16, 14, 4, 3, 597, 70, 2418, 0, 2392, 24, 658, 3, 1296, 1092, 1091, 67, 52, 956, 2, 2874, 0, 436, 131, 1247, 6722, 8, 0, 1185, 3701, 0, 0, 0, 2848, 208, 1540, 0, 1, 27, 7188, 2179, 0, 907, 0, 1, 11, 496, 2599, 0, 122, 9, 0, 37, 17, 36, 1908, 0, 1218, 147, 0, 76] payload_rows_per_dst=[6598, 10050, 8096, 5512, 12858, 6082, 15004, 15336] dup_rows_per_dst=[260, 2274, 1016, 502, 3736, 148, 5920, 4912] combine_rows_per_src=[6598, 10050, 8096, 5512, 12858, 6082, 15004, 15336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=279486464 dispatch_meta_bytes=321688 combine_payload_bytes=279486464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=279808152 ingress_bytes=383295488 link_bytes=383295488 token_bytes=559294616 weight_movement_bytes=103809024 total_nvlink_bytes=663103640 weight_to_token_ratio=0.185607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.442310064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=253730816 dispatch_meta_bytes=318936 combine_payload_bytes=253730816 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=254049752 ingress_bytes=357539840 link_bytes=357539840 token_bytes=507780568 weight_movement_bytes=103809024 total_nvlink_bytes=611589592 weight_to_token_ratio=0.204437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345393634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79196 payload_rows_remote=67020 dup_rows=19108 dup_rows_remote=15066 dedup_ratio=0.194377 router_tokens_per_expert=[1018, 501, 110, 196, 1749, 615, 0, 2, 0, 56, 473, 42, 6167, 3, 153, 107, 0, 95, 0, 2622, 0, 593, 120, 96, 3052, 0, 86, 2452, 8, 3260, 190, 702, 7602, 1076, 3918, 70, 2101, 376, 198, 220, 0, 808, 329, 15, 161, 45, 217, 94, 2957, 0, 823, 59, 257, 209, 252, 743, 153, 982, 567, 132, 0, 0, 201, 119] payload_rows_per_dst=[12176, 13568, 10662, 10422, 15660, 3162, 9422, 4124] dup_rows_per_dst=[4042, 434, 2910, 2558, 7626, 176, 1178, 184] combine_rows_per_src=[12176, 13568, 10662, 10422, 15660, 3162, 9422, 4124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287244288 dispatch_meta_bytes=330952 combine_payload_bytes=287244288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287575240 ingress_bytes=391053312 link_bytes=391053312 token_bytes=574819528 weight_movement_bytes=103809024 total_nvlink_bytes=678628552 weight_to_token_ratio=0.180594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471502131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89080 payload_rows_remote=73910 dup_rows=9224 dup_rows_remote=6728 dedup_ratio=0.093831 router_tokens_per_expert=[384, 4, 20, 37, 22, 124, 0, 0, 171, 3, 435, 0, 18, 180, 5241, 0, 655, 870, 3892, 407, 121, 1, 1, 7199, 0, 97, 34, 8, 7527, 2551, 0, 588, 111, 15, 1, 11, 2768, 3, 4484, 0, 5, 5, 236, 125, 0, 7390, 227, 0, 0, 24, 1002, 0, 421, 0, 236, 192, 899, 0, 384, 1, 1, 15, 0, 6] payload_rows_per_dst=[14996, 11318, 10986, 5938, 12004, 15034, 3634, 15170] dup_rows_per_dst=[584, 778, 908, 618, 2782, 470, 588, 2496] combine_rows_per_src=[14996, 11318, 10986, 5938, 12004, 15034, 3634, 15170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=302358528 dispatch_meta_bytes=369424 combine_payload_bytes=302358528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=302727952 ingress_bytes=371564544 link_bytes=371564544 token_bytes=605086480 weight_movement_bytes=69206016 total_nvlink_bytes=674292496 weight_to_token_ratio=0.114374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398167466e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=291430400 dispatch_meta_bytes=343912 combine_payload_bytes=291430400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291774312 ingress_bytes=326033408 link_bytes=326033408 token_bytes=583204712 weight_movement_bytes=34603008 total_nvlink_bytes=617807720 weight_to_token_ratio=0.059333 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226837466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264323072 dispatch_meta_bytes=309520 combine_payload_bytes=264323072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264632592 ingress_bytes=298926080 link_bytes=298926080 token_bytes=528955664 weight_movement_bytes=34603008 total_nvlink_bytes=563558672 weight_to_token_ratio=0.065418 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124834774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274513920 dispatch_meta_bytes=328344 combine_payload_bytes=274513920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274842264 ingress_bytes=309116928 link_bytes=309116928 token_bytes=549356184 weight_movement_bytes=34603008 total_nvlink_bytes=583959192 weight_to_token_ratio=0.062988 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163182114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=302735360 dispatch_meta_bytes=322552 combine_payload_bytes=302735360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303057912 ingress_bytes=337338368 link_bytes=337338368 token_bytes=605793272 weight_movement_bytes=34603008 total_nvlink_bytes=640396280 weight_to_token_ratio=0.057120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269377121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83356 payload_rows_remote=73420 dup_rows=14948 dup_rows_remote=12986 dedup_ratio=0.152059 router_tokens_per_expert=[66, 0, 264, 520, 996, 0, 6, 0, 53, 3092, 107, 501, 2022, 37, 251, 3, 3085, 0, 2789, 0, 36, 13, 2288, 1296, 1885, 0, 0, 573, 0, 215, 244, 0, 530, 111, 2330, 2232, 171, 0, 2, 573, 5577, 288, 190, 524, 1071, 1402, 3871, 3150, 11, 1276, 524, 15, 0, 348, 1, 5, 3, 10, 258, 596, 598, 2559, 257, 327] payload_rows_per_dst=[9788, 9982, 12610, 15070, 9936, 10616, 7372, 7982] dup_rows_per_dst=[1658, 2150, 1828, 1918, 1962, 2634, 1564, 1234] combine_rows_per_src=[9788, 9982, 12610, 15070, 9936, 10616, 7372, 7982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81874 payload_rows_remote=68230 dup_rows=16430 dup_rows_remote=13526 dedup_ratio=0.167135 router_tokens_per_expert=[495, 13, 391, 140, 2023, 25, 18, 600, 2186, 28, 0, 0, 131, 101, 3, 10, 630, 457, 219, 412, 687, 3, 261, 108, 621, 84, 48, 0, 935, 115, 47, 6424, 1530, 2400, 43, 0, 1314, 0, 270, 3819, 588, 654, 12, 5003, 669, 36, 2024, 423, 0, 279, 160, 53, 845, 847, 28, 31, 2623, 0, 368, 302, 78, 6582, 170, 786] payload_rows_per_dst=[12572, 13492, 13266, 13644, 9412, 8190, 4212, 7086] dup_rows_per_dst=[2428, 4590, 2294, 2904, 1750, 622, 274, 1568] combine_rows_per_src=[12572, 13492, 13266, 13644, 9412, 8190, 4212, 7086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79994 payload_rows_remote=67182 dup_rows=18310 dup_rows_remote=15586 dedup_ratio=0.186259 router_tokens_per_expert=[364, 104, 983, 455, 1353, 18, 0, 451, 2214, 72, 8, 0, 295, 81, 1, 1, 255, 1024, 285, 202, 79, 31, 234, 768, 878, 631, 31, 0, 536, 62, 123, 6470, 562, 1691, 129, 0, 854, 0, 40, 2327, 773, 1068, 53, 4890, 1194, 137, 1875, 431, 0, 773, 87, 118, 2378, 357, 327, 52, 2059, 9, 396, 387, 104, 6880, 241, 951] payload_rows_per_dst=[9814, 14186, 12812, 13716, 6070, 9484, 7336, 6576] dup_rows_per_dst=[2296, 4918, 2724, 3746, 482, 1578, 848, 1718] combine_rows_per_src=[9814, 14186, 12812, 13716, 6070, 9484, 7336, 6576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87472 payload_rows_remote=71104 dup_rows=10832 dup_rows_remote=9052 dedup_ratio=0.110189 router_tokens_per_expert=[5, 0, 1440, 1, 8128, 0, 0, 0, 889, 0, 0, 0, 1, 0, 0, 0, 1, 234, 0, 1, 0, 0, 1151, 2, 5, 3, 1, 0, 2, 205, 29, 4235, 3, 3716, 1, 0, 7, 0, 0, 18, 47, 236, 94, 7816, 1, 1, 48, 0, 0, 0, 938, 0, 2, 6032, 0, 28, 5326, 0, 5, 0, 2, 8184, 0, 314] payload_rows_per_dst=[16304, 16368, 16226, 11866, 7446, 838, 12996, 5428] dup_rows_per_dst=[2880, 1780, 2184, 2678, 8, 16, 1004, 282] combine_rows_per_src=[16304, 16368, 16226, 11866, 7446, 838, 12996, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73136 payload_rows_remote=60960 dup_rows=25168 dup_rows_remote=20066 dedup_ratio=0.256022 router_tokens_per_expert=[766, 14, 333, 167, 625, 48, 6, 1208, 1542, 78, 0, 0, 243, 297, 0, 1, 543, 691, 961, 617, 950, 1, 66, 511, 668, 351, 62, 0, 975, 4, 37, 6541, 1444, 1508, 88, 0, 2364, 0, 160, 4307, 627, 697, 9, 3360, 892, 254, 2045, 755, 0, 718, 9, 68, 2234, 200, 174, 27, 1065, 1, 829, 300, 86, 5374, 218, 1033] payload_rows_per_dst=[5624, 4258, 6972, 4396, 13110, 12176, 14146, 12454] dup_rows_per_dst=[710, 64, 1708, 612, 6632, 5102, 4982, 5358] combine_rows_per_src=[5624, 4258, 6972, 4396, 13110, 12176, 14146, 12454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=300728320 dispatch_meta_bytes=345624 combine_payload_bytes=300728320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301073944 ingress_bytes=369934336 link_bytes=369934336 token_bytes=601802264 weight_movement_bytes=69206016 total_nvlink_bytes=671008280 weight_to_token_ratio=0.114998 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392033124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73608 payload_rows_remote=59820 dup_rows=24696 dup_rows_remote=21034 dedup_ratio=0.251221 router_tokens_per_expert=[1480, 74, 97, 146, 415, 43, 13, 1100, 1023, 43, 0, 0, 263, 190, 7, 38, 1344, 559, 700, 739, 1399, 1, 21, 51, 1221, 289, 128, 0, 1893, 39, 19, 6169, 1938, 1091, 82, 0, 2667, 9, 434, 4800, 344, 586, 22, 2870, 706, 323, 2364, 1169, 0, 1013, 1, 93, 1291, 59, 85, 14, 870, 2, 686, 311, 77, 4769, 196, 776] payload_rows_per_dst=[6006, 3106, 7260, 6326, 14144, 11870, 13788, 11108] dup_rows_per_dst=[730, 22, 2368, 852, 7898, 4898, 3662, 4266] combine_rows_per_src=[6006, 3106, 7260, 6326, 14144, 11870, 13788, 11108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=279470080 dispatch_meta_bytes=327024 combine_payload_bytes=279470080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279797104 ingress_bytes=348676096 link_bytes=348676096 token_bytes=559267184 weight_movement_bytes=69206016 total_nvlink_bytes=628473200 weight_to_token_ratio=0.123744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312040079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275177472 dispatch_meta_bytes=331072 combine_payload_bytes=275177472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275508544 ingress_bytes=309780480 link_bytes=309780480 token_bytes=550686016 weight_movement_bytes=34603008 total_nvlink_bytes=585289024 weight_to_token_ratio=0.062836 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165679007e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78538 payload_rows_remote=64758 dup_rows=19766 dup_rows_remote=15776 dedup_ratio=0.201070 router_tokens_per_expert=[174, 29, 28, 65, 20, 151, 0, 0, 421, 28, 1237, 0, 126, 101, 2648, 0, 879, 1302, 2965, 85, 209, 30, 14, 6238, 0, 128, 30, 179, 7766, 786, 3, 1284, 659, 2, 0, 0, 3012, 45, 4028, 0, 0, 0, 2199, 382, 0, 5317, 141, 0, 0, 719, 2648, 0, 446, 55, 182, 125, 119, 4, 1954, 0, 147, 2, 0, 40] payload_rows_per_dst=[13780, 14826, 10782, 10828, 2656, 13542, 7646, 4478] dup_rows_per_dst=[3990, 6924, 3882, 2222, 100, 1890, 704, 54] combine_rows_per_src=[13780, 14826, 10782, 10828, 2656, 13542, 7646, 4478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=57928 dup_rows=24104 dup_rows_remote=13560 dedup_ratio=0.245199 router_tokens_per_expert=[21, 2, 1072, 37, 7250, 0, 31, 25, 1534, 1, 0, 6, 13, 29, 1, 0, 34, 286, 7, 55, 0, 0, 901, 29, 8, 33, 3, 0, 22, 178, 40, 5976, 88, 4114, 14, 0, 62, 0, 0, 294, 102, 486, 41, 7481, 74, 25, 457, 2, 0, 4, 441, 1, 55, 4363, 24, 22, 4774, 0, 31, 105, 22, 8053, 22, 401] payload_rows_per_dst=[14964, 3168, 2560, 558, 5650, 15434, 15594, 16272] dup_rows_per_dst=[1912, 0, 64, 10, 422, 4974, 6178, 10544] combine_rows_per_src=[14964, 3168, 2560, 558, 5650, 15434, 15594, 16272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=291241984 dispatch_meta_bytes=320624 combine_payload_bytes=291241984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291562608 ingress_bytes=325844992 link_bytes=325844992 token_bytes=582804592 weight_movement_bytes=34603008 total_nvlink_bytes=617407600 weight_to_token_ratio=0.059373 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226128472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=249692160 dispatch_meta_bytes=324104 combine_payload_bytes=249692160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250016264 ingress_bytes=318898176 link_bytes=318898176 token_bytes=499708424 weight_movement_bytes=69206016 total_nvlink_bytes=568914440 weight_to_token_ratio=0.138493 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199988163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=245022720 dispatch_meta_bytes=323416 combine_payload_bytes=245022720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245346136 ingress_bytes=279625728 link_bytes=279625728 token_bytes=490368856 weight_movement_bytes=34603008 total_nvlink_bytes=524971864 weight_to_token_ratio=0.070565 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.052209103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=322136 combine_payload_bytes=265248768 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=265570904 ingress_bytes=334454784 link_bytes=334454784 token_bytes=530819672 weight_movement_bytes=69206016 total_nvlink_bytes=600025688 weight_to_token_ratio=0.130376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258526426e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=237273088 dispatch_meta_bytes=285952 combine_payload_bytes=237273088 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=237559040 ingress_bytes=341082112 link_bytes=341082112 token_bytes=474832128 weight_movement_bytes=103809024 total_nvlink_bytes=578641152 weight_to_token_ratio=0.218623 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283464528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79506 payload_rows_remote=66166 dup_rows=18798 dup_rows_remote=13496 dedup_ratio=0.191223 router_tokens_per_expert=[935, 94, 80, 0, 319, 233, 4358, 0, 49, 841, 1452, 2060, 5, 4857, 1012, 2147, 3550, 12, 578, 2874, 137, 0, 1577, 487, 2, 144, 3521, 508, 1323, 110, 1009, 1, 0, 170, 65, 1397, 319, 973, 45, 2802, 302, 106, 34, 75, 608, 0, 8, 0, 115, 4433, 0, 480, 0, 374, 57, 69, 274, 0, 0, 74, 11, 214, 777, 1095] payload_rows_per_dst=[10788, 9728, 9646, 10766, 13340, 11616, 2174, 11448] dup_rows_per_dst=[1250, 3380, 1684, 2470, 5302, 2388, 16, 2308] combine_rows_per_src=[10788, 9728, 9646, 10766, 13340, 11616, 2174, 11448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83296 payload_rows_remote=70618 dup_rows=15008 dup_rows_remote=12972 dedup_ratio=0.152669 router_tokens_per_expert=[18, 0, 69, 1779, 1171, 0, 0, 0, 66, 3615, 65, 202, 2577, 31, 19, 65, 4565, 0, 1840, 0, 0, 0, 1729, 1108, 538, 0, 0, 56, 0, 393, 106, 1, 229, 35, 1436, 3756, 140, 0, 29, 855, 6890, 193, 250, 553, 1268, 1873, 3597, 1155, 12, 1830, 570, 6, 0, 1148, 0, 10, 20, 19, 20, 451, 1154, 1079, 404, 157] payload_rows_per_dst=[11596, 11148, 12678, 14906, 11136, 7538, 8582, 5712] dup_rows_per_dst=[3982, 2444, 2036, 1062, 1824, 736, 2028, 896] combine_rows_per_src=[11596, 11148, 12678, 14906, 11136, 7538, 8582, 5712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83150 payload_rows_remote=67908 dup_rows=15154 dup_rows_remote=13054 dedup_ratio=0.154154 router_tokens_per_expert=[112, 0, 854, 538, 533, 0, 0, 0, 25, 3831, 52, 401, 2453, 7, 42, 0, 2692, 0, 1821, 0, 3, 0, 4625, 1035, 1144, 0, 0, 197, 0, 555, 410, 0, 337, 172, 1520, 3669, 145, 0, 3, 552, 6365, 226, 155, 1603, 803, 600, 2405, 3138, 2, 2148, 203, 1, 0, 511, 0, 0, 6, 8, 107, 485, 1141, 1293, 82, 142] payload_rows_per_dst=[7766, 11184, 10226, 15242, 10996, 10658, 10978, 6100] dup_rows_per_dst=[1406, 2438, 876, 2100, 1800, 2104, 4002, 428] combine_rows_per_src=[7766, 11184, 10226, 15242, 10996, 10658, 10978, 6100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92480 payload_rows_remote=87050 dup_rows=5824 dup_rows_remote=5238 dedup_ratio=0.059245 router_tokens_per_expert=[0, 0, 7357, 3, 11, 0, 0, 0, 1, 72, 0, 0, 1869, 0, 0, 1, 1065, 0, 4, 0, 0, 0, 7494, 92, 0, 0, 0, 0, 0, 7, 102, 0, 1, 701, 7, 6927, 116, 0, 0, 3, 7771, 0, 17, 3430, 26, 4, 9, 48, 0, 5045, 3, 0, 0, 471, 0, 0, 0, 0, 1, 1, 6484, 5, 1, 3] payload_rows_per_dst=[14774, 5430, 10796, 15552, 14172, 6904, 11868, 12984] dup_rows_per_dst=[82, 586, 116, 208, 1338, 50, 3438, 6] combine_rows_per_src=[14774, 5430, 10796, 15552, 14172, 6904, 11868, 12984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=271015936 dispatch_meta_bytes=318648 combine_payload_bytes=271015936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271334584 ingress_bytes=305618944 link_bytes=305618944 token_bytes=542350520 weight_movement_bytes=34603008 total_nvlink_bytes=576953528 weight_to_token_ratio=0.063802 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150019482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76670 payload_rows_remote=63908 dup_rows=21634 dup_rows_remote=13474 dedup_ratio=0.220072 router_tokens_per_expert=[76, 0, 90, 1293, 664, 0, 1, 0, 76, 4063, 104, 444, 1502, 3, 62, 15, 3514, 0, 3263, 0, 0, 39, 2379, 942, 1307, 0, 0, 211, 0, 909, 224, 0, 519, 85, 2878, 1611, 95, 0, 1, 1161, 5896, 561, 205, 462, 930, 1671, 4473, 2282, 3, 1307, 389, 1, 1, 391, 0, 4, 0, 8, 63, 625, 394, 1511, 199, 245] payload_rows_per_dst=[10926, 10584, 13182, 5938, 9944, 12762, 7222, 6112] dup_rows_per_dst=[2268, 1954, 2334, 1376, 2756, 8160, 1728, 1058] combine_rows_per_src=[10926, 10584, 13182, 5938, 9944, 12762, 7222, 6112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80432 payload_rows_remote=74964 dup_rows=17872 dup_rows_remote=16986 dedup_ratio=0.181803 router_tokens_per_expert=[96, 0, 41, 1006, 840, 0, 0, 0, 61, 2826, 75, 1278, 1047, 3, 176, 1, 2284, 0, 4274, 0, 18, 33, 1637, 1131, 2521, 0, 0, 579, 0, 514, 136, 0, 530, 28, 3578, 906, 41, 0, 2, 1293, 4634, 477, 146, 165, 1028, 1355, 4915, 2620, 7, 916, 400, 6, 0, 210, 0, 1, 1, 3, 355, 790, 157, 3298, 237, 476] payload_rows_per_dst=[11782, 9664, 13606, 6318, 10272, 9370, 5468, 13952] dup_rows_per_dst=[2014, 1270, 1874, 1182, 2484, 2212, 886, 5950] combine_rows_per_src=[11782, 9664, 13606, 6318, 10272, 9370, 5468, 13952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=289251328 dispatch_meta_bytes=334360 combine_payload_bytes=289251328 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=289585688 ingress_bytes=393060352 link_bytes=393060352 token_bytes=578837016 weight_movement_bytes=103809024 total_nvlink_bytes=682646040 weight_to_token_ratio=0.179341 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479054461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=278151168 dispatch_meta_bytes=323848 combine_payload_bytes=278151168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278475016 ingress_bytes=312754176 link_bytes=312754176 token_bytes=556626184 weight_movement_bytes=34603008 total_nvlink_bytes=591229192 weight_to_token_ratio=0.062166 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176868786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80166 payload_rows_remote=68182 dup_rows=18138 dup_rows_remote=15452 dedup_ratio=0.184509 router_tokens_per_expert=[197, 13, 1401, 209, 4343, 9, 0, 301, 2323, 9, 0, 0, 85, 74, 1, 0, 150, 760, 138, 85, 135, 1, 535, 252, 68, 40, 11, 0, 243, 244, 47, 6238, 490, 2204, 124, 0, 507, 2, 28, 862, 538, 673, 17, 6456, 558, 66, 1898, 62, 0, 121, 362, 17, 978, 2613, 26, 86, 3317, 0, 202, 280, 85, 7564, 208, 896] payload_rows_per_dst=[11984, 15292, 14570, 14724, 6066, 6874, 7836, 2820] dup_rows_per_dst=[2686, 4820, 2454, 5692, 644, 750, 570, 522] combine_rows_per_src=[11984, 15292, 14570, 14724, 6066, 6874, 7836, 2820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356556800 dispatch_meta_bytes=369152 combine_payload_bytes=356556800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=356925952 ingress_bytes=391159808 link_bytes=391159808 token_bytes=713482752 weight_movement_bytes=34603008 total_nvlink_bytes=748085760 weight_to_token_ratio=0.048499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471902867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72958 payload_rows_remote=56958 dup_rows=25346 dup_rows_remote=14474 dedup_ratio=0.257833 router_tokens_per_expert=[17, 0, 5989, 30, 52, 7, 0, 0, 0, 675, 3, 2, 3393, 0, 15, 9, 1598, 1, 85, 0, 0, 0, 7098, 302, 0, 1, 0, 4, 0, 31, 394, 0, 5, 384, 33, 5091, 221, 0, 0, 82, 7897, 8, 93, 4170, 94, 4, 102, 712, 0, 4547, 76, 1, 11, 362, 0, 14, 2, 7, 1, 55, 5431, 26, 15, 2] payload_rows_per_dst=[12320, 7620, 3226, 856, 4668, 12602, 15666, 16000] dup_rows_per_dst=[74, 574, 234, 4, 346, 4690, 8552, 10872] combine_rows_per_src=[12320, 7620, 3226, 856, 4668, 12602, 15666, 16000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=261767168 dispatch_meta_bytes=309528 combine_payload_bytes=261767168 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=262076696 ingress_bytes=434782208 link_bytes=434782208 token_bytes=523843864 weight_movement_bytes=173015040 total_nvlink_bytes=696858904 weight_to_token_ratio=0.330280 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.636050446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307052544 dispatch_meta_bytes=367800 combine_payload_bytes=307052544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307420344 ingress_bytes=341655552 link_bytes=341655552 token_bytes=614472888 weight_movement_bytes=34603008 total_nvlink_bytes=649075896 weight_to_token_ratio=0.056313 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285622337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=279273472 dispatch_meta_bytes=334536 combine_payload_bytes=279273472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279608008 ingress_bytes=313876480 link_bytes=313876480 token_bytes=558881480 weight_movement_bytes=34603008 total_nvlink_bytes=593484488 weight_to_token_ratio=0.061915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181091925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78278 payload_rows_remote=67028 dup_rows=20026 dup_rows_remote=13534 dedup_ratio=0.203715 router_tokens_per_expert=[47, 303, 1091, 2, 0, 2534, 0, 27, 224, 151, 0, 2501, 6395, 167, 39, 29, 33, 365, 3383, 0, 0, 281, 0, 0, 676, 23, 1262, 86, 69, 147, 788, 20, 654, 0, 0, 0, 0, 3100, 0, 5117, 2523, 2048, 61, 2841, 0, 912, 686, 111, 0, 63, 208, 50, 41, 928, 149, 5488, 2, 233, 502, 0, 61, 1426, 1253, 52] payload_rows_per_dst=[7064, 8700, 12242, 6936, 11250, 10136, 12814, 9136] dup_rows_per_dst=[944, 886, 5308, 1030, 6492, 2308, 1040, 2018] combine_rows_per_src=[7064, 8700, 12242, 6936, 11250, 10136, 12814, 9136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233299968 dispatch_meta_bytes=285728 combine_payload_bytes=233299968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233585696 ingress_bytes=267902976 link_bytes=267902976 token_bytes=466885664 weight_movement_bytes=34603008 total_nvlink_bytes=501488672 weight_to_token_ratio=0.074115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.008097331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81030 payload_rows_remote=75294 dup_rows=17274 dup_rows_remote=16840 dedup_ratio=0.175720 router_tokens_per_expert=[822, 69, 64, 0, 82, 246, 5654, 0, 23, 470, 1359, 1317, 15, 4811, 1054, 3313, 5325, 195, 469, 861, 202, 7, 1290, 61, 61, 53, 1960, 503, 824, 109, 1736, 3, 25, 383, 0, 935, 932, 237, 6, 1028, 427, 48, 11, 124, 1268, 0, 16, 159, 524, 5301, 0, 873, 0, 1257, 101, 364, 169, 0, 0, 198, 0, 208, 558, 1042] payload_rows_per_dst=[12522, 11368, 5736, 8974, 13590, 11802, 12842, 4196] dup_rows_per_dst=[1352, 3734, 434, 1524, 4152, 1926, 3998, 154] combine_rows_per_src=[12522, 11368, 5736, 8974, 13590, 11802, 12842, 4196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81962 payload_rows_remote=70886 dup_rows=16342 dup_rows_remote=14302 dedup_ratio=0.166239 router_tokens_per_expert=[1144, 45, 26, 0, 176, 200, 5439, 0, 22, 682, 968, 970, 3, 4572, 1728, 2025, 4825, 1, 474, 1873, 145, 0, 2529, 301, 0, 175, 3073, 134, 593, 84, 2499, 0, 0, 393, 24, 972, 92, 822, 18, 1438, 265, 103, 2, 41, 861, 0, 0, 2, 78, 4532, 0, 228, 0, 1400, 20, 131, 166, 0, 0, 184, 5, 264, 557, 1848] payload_rows_per_dst=[12380, 7122, 9304, 11076, 13674, 11792, 9516, 7098] dup_rows_per_dst=[1680, 2644, 1342, 2040, 3494, 2930, 982, 1230] combine_rows_per_src=[12380, 7122, 9304, 11076, 13674, 11792, 9516, 7098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=274546688 dispatch_meta_bytes=322248 combine_payload_bytes=274546688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274868936 ingress_bytes=309149696 link_bytes=309149696 token_bytes=549415624 weight_movement_bytes=34603008 total_nvlink_bytes=584018632 weight_to_token_ratio=0.062981 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163305417e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92038 payload_rows_remote=81380 dup_rows=6266 dup_rows_remote=6184 dedup_ratio=0.063741 router_tokens_per_expert=[11, 0, 1, 0, 0, 2, 7361, 1, 0, 4, 0, 9, 121, 1234, 5218, 18, 8178, 0, 0, 9, 251, 0, 1721, 2, 0, 0, 80, 0, 3, 1, 7377, 0, 37, 319, 1, 9, 1, 2, 0, 13, 141, 0, 0, 0, 387, 0, 1, 0, 106, 232, 0, 1, 1, 8097, 0, 1, 1, 0, 0, 777, 0, 476, 3, 6944] payload_rows_per_dst=[14744, 10658, 15432, 14766, 1928, 3518, 16238, 14754] dup_rows_per_dst=[8, 82, 3648, 156, 78, 8, 638, 1648] combine_rows_per_src=[14744, 10658, 15432, 14766, 1928, 3518, 16238, 14754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=308404224 dispatch_meta_bytes=368536 combine_payload_bytes=308404224 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=308772760 ingress_bytes=412213248 link_bytes=412213248 token_bytes=617176984 weight_movement_bytes=103809024 total_nvlink_bytes=720986008 weight_to_token_ratio=0.168200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.551125266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80048 payload_rows_remote=70132 dup_rows=18256 dup_rows_remote=16512 dedup_ratio=0.185710 router_tokens_per_expert=[983, 131, 113, 0, 237, 361, 4650, 0, 9, 960, 1512, 1241, 0, 4038, 625, 2896, 3544, 10, 930, 2479, 132, 13, 1650, 371, 0, 50, 2686, 425, 908, 140, 901, 0, 0, 140, 8, 2035, 483, 637, 7, 2751, 158, 206, 8, 159, 553, 0, 77, 6, 115, 6280, 0, 689, 0, 388, 62, 139, 156, 0, 0, 31, 0, 257, 1336, 476] payload_rows_per_dst=[11174, 10222, 9504, 8924, 13696, 9916, 2768, 13844] dup_rows_per_dst=[1776, 3014, 1666, 1296, 5514, 1744, 18, 3228] combine_rows_per_src=[11174, 10222, 9504, 8924, 13696, 9916, 2768, 13844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=290349056 dispatch_meta_bytes=340752 combine_payload_bytes=290349056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290689808 ingress_bytes=359555072 link_bytes=359555072 token_bytes=581038864 weight_movement_bytes=69206016 total_nvlink_bytes=650244880 weight_to_token_ratio=0.119107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352976789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78560 payload_rows_remote=76082 dup_rows=19744 dup_rows_remote=19648 dedup_ratio=0.200846 router_tokens_per_expert=[1056, 166, 124, 0, 339, 278, 3459, 0, 68, 2032, 1302, 2042, 2, 4198, 294, 2973, 2008, 28, 593, 2404, 42, 5, 1087, 578, 0, 219, 3864, 262, 1735, 260, 641, 1, 0, 108, 20, 1804, 158, 1444, 1, 3877, 37, 77, 26, 138, 319, 0, 17, 7, 3, 5823, 0, 709, 0, 151, 35, 185, 339, 0, 0, 11, 0, 189, 1360, 254] payload_rows_per_dst=[9358, 11828, 8456, 11294, 12652, 9262, 2478, 13232] dup_rows_per_dst=[1486, 5010, 1018, 2670, 5780, 964, 96, 2720] combine_rows_per_src=[9358, 11828, 8456, 11294, 12652, 9262, 2478, 13232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84528 payload_rows_remote=77478 dup_rows=13776 dup_rows_remote=12958 dedup_ratio=0.140137 router_tokens_per_expert=[34, 0, 1544, 184, 289, 0, 0, 0, 42, 3427, 19, 143, 2905, 4, 15, 4, 4079, 0, 836, 0, 0, 5, 4740, 760, 236, 0, 0, 53, 0, 102, 498, 0, 132, 238, 585, 5833, 308, 0, 0, 239, 7397, 130, 116, 2347, 725, 316, 812, 1071, 3, 3429, 175, 2, 4, 857, 0, 5, 0, 6, 16, 401, 3633, 343, 81, 29] payload_rows_per_dst=[7050, 13940, 11742, 15356, 13092, 6846, 8048, 8454] dup_rows_per_dst=[818, 7336, 940, 1216, 1578, 422, 902, 564] combine_rows_per_src=[7050, 13940, 11742, 15356, 13092, 6846, 8048, 8454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333332480 dispatch_meta_bytes=350256 combine_payload_bytes=333332480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=333682736 ingress_bytes=437141504 link_bytes=437141504 token_bytes=667015216 weight_movement_bytes=103809024 total_nvlink_bytes=770824240 weight_to_token_ratio=0.155632 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644928287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78046 payload_rows_remote=65322 dup_rows=20258 dup_rows_remote=18842 dedup_ratio=0.206075 router_tokens_per_expert=[209, 11, 7, 0, 1, 12, 7210, 0, 9, 74, 44, 29, 1, 2119, 5639, 75, 7922, 38, 4, 191, 206, 0, 2407, 1, 4, 2, 269, 4, 1, 5, 6941, 0, 17, 390, 0, 41, 52, 9, 0, 76, 364, 2, 0, 20, 383, 0, 8, 17, 96, 551, 5, 75, 1, 7080, 3, 8, 2, 0, 2, 417, 0, 390, 17, 5691] payload_rows_per_dst=[3898, 10562, 5530, 14284, 1120, 13774, 16154, 12724] dup_rows_per_dst=[118, 70, 164, 884, 50, 3330, 14226, 1416] combine_rows_per_src=[3898, 10562, 5530, 14284, 1120, 13774, 16154, 12724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287260672 dispatch_meta_bytes=346576 combine_payload_bytes=287260672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287607248 ingress_bytes=356466688 link_bytes=356466688 token_bytes=574867920 weight_movement_bytes=69206016 total_nvlink_bytes=644073936 weight_to_token_ratio=0.120386 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341355449e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=311631872 dispatch_meta_bytes=382920 combine_payload_bytes=311631872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312014792 ingress_bytes=380837888 link_bytes=380837888 token_bytes=623646664 weight_movement_bytes=69206016 total_nvlink_bytes=692852680 weight_to_token_ratio=0.110970 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433062313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=317349888 dispatch_meta_bytes=361744 combine_payload_bytes=317349888 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=317711632 ingress_bytes=386555904 link_bytes=386555904 token_bytes=635061520 weight_movement_bytes=69206016 total_nvlink_bytes=704267536 weight_to_token_ratio=0.108975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454578746e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75734 payload_rows_remote=67076 dup_rows=22570 dup_rows_remote=22134 dedup_ratio=0.229594 router_tokens_per_expert=[0, 0, 3, 262, 87, 2816, 22, 146, 15, 5, 156, 135, 2005, 485, 712, 1042, 5327, 135, 335, 1052, 236, 2457, 126, 0, 234, 1143, 5211, 531, 48, 625, 0, 342, 938, 140, 0, 2, 3085, 60, 322, 0, 211, 145, 0, 85, 164, 1760, 18, 203, 2856, 941, 581, 440, 256, 2336, 59, 136, 2642, 39, 1215, 0, 866, 459, 3430, 70] payload_rows_per_dst=[12086, 12538, 7968, 12626, 8658, 4928, 11776, 5154] dup_rows_per_dst=[6730, 7226, 714, 3642, 436, 244, 3434, 144] combine_rows_per_src=[12086, 12538, 7968, 12626, 8658, 4928, 11776, 5154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75630 payload_rows_remote=66894 dup_rows=22674 dup_rows_remote=22142 dedup_ratio=0.230652 router_tokens_per_expert=[3, 1580, 1187, 8, 3, 1720, 13, 2, 35, 196, 0, 4245, 6646, 143, 2, 41, 4, 1403, 3191, 2, 0, 30, 4, 0, 1056, 36, 1671, 157, 84, 204, 498, 67, 49, 0, 0, 0, 0, 1478, 0, 2735, 1944, 4145, 60, 1353, 0, 1224, 301, 131, 0, 61, 130, 24, 22, 131, 611, 6790, 20, 464, 495, 0, 77, 1586, 1015, 75] payload_rows_per_dst=[9608, 14728, 8736, 8614, 6444, 7428, 7888, 12184] dup_rows_per_dst=[2540, 11776, 532, 1380, 2080, 616, 180, 3570] combine_rows_per_src=[9608, 14728, 8736, 8614, 6444, 7428, 7888, 12184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=336656 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267895568 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535454480 weight_movement_bytes=34603008 total_nvlink_bytes=570057488 weight_to_token_ratio=0.064624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80028 payload_rows_remote=71222 dup_rows=18276 dup_rows_remote=15844 dedup_ratio=0.185913 router_tokens_per_expert=[13, 350, 1910, 0, 0, 1844, 21, 18, 66, 194, 0, 2693, 6164, 101, 0, 6, 7, 1646, 1902, 0, 0, 140, 0, 0, 223, 18, 752, 56, 4, 171, 1857, 48, 339, 0, 0, 0, 0, 1953, 0, 3488, 3893, 4214, 35, 1787, 1, 2490, 262, 240, 0, 102, 140, 17, 218, 496, 157, 6393, 0, 194, 387, 0, 64, 1034, 975, 69] payload_rows_per_dst=[7162, 13522, 7108, 8806, 7394, 9992, 14244, 11800] dup_rows_per_dst=[1150, 6518, 282, 2432, 4166, 852, 802, 2074] combine_rows_per_src=[7162, 13522, 7108, 8806, 7394, 9992, 14244, 11800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=356840 combine_payload_bytes=274743296 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=275100136 ingress_bytes=309346304 link_bytes=309346304 token_bytes=549843432 weight_movement_bytes=34603008 total_nvlink_bytes=584446440 weight_to_token_ratio=0.062932 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164045238e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85692 payload_rows_remote=82286 dup_rows=12612 dup_rows_remote=11956 dedup_ratio=0.128296 router_tokens_per_expert=[0, 363, 1169, 0, 0, 6, 534, 0, 0, 1088, 0, 485, 394, 1, 0, 1, 0, 7434, 5, 0, 0, 1, 0, 0, 0, 0, 5, 2, 0, 276, 3555, 0, 0, 0, 0, 0, 0, 8, 0, 11, 62, 7918, 0, 10, 4328, 7956, 6, 1, 6, 1, 0, 0, 5151, 0, 5, 8182, 0, 86, 84, 0, 10, 2, 5, 1] payload_rows_per_dst=[16376, 3406, 14880, 16104, 26, 8678, 10318, 15904] dup_rows_per_dst=[4132, 656, 0, 7484, 12, 12, 8, 308] combine_rows_per_src=[16376, 3406, 14880, 16104, 26, 8678, 10318, 15904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=273997824 dispatch_meta_bytes=356144 combine_payload_bytes=273997824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274353968 ingress_bytes=308600832 link_bytes=308600832 token_bytes=548351792 weight_movement_bytes=34603008 total_nvlink_bytes=582954800 weight_to_token_ratio=0.063104 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161240086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80860 payload_rows_remote=70010 dup_rows=17444 dup_rows_remote=15676 dedup_ratio=0.177450 router_tokens_per_expert=[28, 280, 1078, 0, 0, 2306, 0, 28, 141, 114, 0, 3273, 6587, 127, 2, 13, 72, 329, 2273, 0, 0, 117, 0, 0, 637, 0, 2409, 26, 67, 82, 620, 21, 243, 0, 0, 0, 0, 3093, 0, 5312, 3305, 1672, 36, 2478, 0, 695, 380, 110, 0, 197, 349, 11, 10, 289, 351, 5457, 7, 221, 553, 0, 55, 2265, 1374, 59] payload_rows_per_dst=[6510, 7076, 11558, 8074, 14222, 10850, 12516, 10054] dup_rows_per_dst=[930, 264, 3390, 1040, 6882, 1768, 812, 2358] combine_rows_per_src=[6510, 7076, 11558, 8074, 14222, 10850, 12516, 10054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291725312 dispatch_meta_bytes=348264 combine_payload_bytes=291725312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292073576 ingress_bytes=326328320 link_bytes=326328320 token_bytes=583798888 weight_movement_bytes=34603008 total_nvlink_bytes=618401896 weight_to_token_ratio=0.059272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227947197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75056 payload_rows_remote=64502 dup_rows=23248 dup_rows_remote=22552 dedup_ratio=0.236491 router_tokens_per_expert=[18, 56, 745, 0, 0, 3632, 0, 67, 438, 79, 0, 1872, 5901, 88, 3, 75, 23, 57, 3015, 0, 0, 187, 0, 0, 267, 2, 3068, 169, 7, 40, 709, 3, 346, 0, 0, 0, 0, 3686, 0, 6098, 3164, 539, 19, 4177, 0, 236, 1148, 137, 0, 238, 174, 29, 2, 688, 264, 4230, 1, 14, 512, 0, 48, 1645, 1074, 162] payload_rows_per_dst=[8194, 5014, 6474, 8406, 16326, 12958, 10554, 7130] dup_rows_per_dst=[842, 96, 90, 596, 15736, 4332, 696, 860] combine_rows_per_src=[8194, 5014, 6474, 8406, 16326, 12958, 10554, 7130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79688 payload_rows_remote=65164 dup_rows=18616 dup_rows_remote=17370 dedup_ratio=0.189372 router_tokens_per_expert=[776, 21, 23, 0, 71, 126, 6868, 0, 0, 317, 351, 249, 5, 4624, 2336, 1183, 7206, 38, 45, 906, 212, 0, 2545, 61, 0, 6, 1097, 135, 204, 25, 3480, 0, 0, 403, 0, 489, 251, 118, 6, 700, 447, 9, 0, 56, 1518, 0, 13, 0, 796, 2993, 0, 260, 0, 4001, 4, 26, 40, 0, 0, 387, 0, 241, 139, 3345] payload_rows_per_dst=[14524, 7884, 15686, 6760, 3378, 11250, 12276, 7930] dup_rows_per_dst=[1246, 998, 9054, 420, 556, 2084, 3884, 374] combine_rows_per_src=[14524, 7884, 15686, 6760, 3378, 11250, 12276, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=337043456 dispatch_meta_bytes=376968 combine_payload_bytes=337043456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=337420424 ingress_bytes=440852480 link_bytes=440852480 token_bytes=674463880 weight_movement_bytes=103809024 total_nvlink_bytes=778272904 weight_to_token_ratio=0.153913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.658892391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82184 payload_rows_remote=66506 dup_rows=16120 dup_rows_remote=15620 dedup_ratio=0.163981 router_tokens_per_expert=[21, 578, 2257, 2, 0, 146, 341, 0, 0, 951, 3, 1000, 1512, 13, 0, 0, 0, 7214, 67, 0, 0, 0, 4, 0, 88, 23, 119, 17, 15, 248, 4045, 90, 0, 4, 2, 0, 1, 41, 3, 242, 790, 7734, 13, 55, 1717, 6994, 7, 66, 11, 0, 12, 0, 4213, 0, 26, 8112, 9, 89, 106, 0, 7, 77, 66, 1] payload_rows_per_dst=[6082, 3906, 14522, 11224, 3272, 11240, 16260, 15678] dup_rows_per_dst=[608, 28, 48, 3110, 338, 3000, 8488, 500] combine_rows_per_src=[6082, 3906, 14522, 11224, 3272, 11240, 16260, 15678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=286760960 dispatch_meta_bytes=342744 combine_payload_bytes=286760960 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=287103704 ingress_bytes=425172992 link_bytes=425172992 token_bytes=573864664 weight_movement_bytes=138412032 total_nvlink_bytes=712276696 weight_to_token_ratio=0.241193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599891740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264200192 dispatch_meta_bytes=348216 combine_payload_bytes=264200192 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264548408 ingress_bytes=333406208 link_bytes=333406208 token_bytes=528748600 weight_movement_bytes=69206016 total_nvlink_bytes=597954616 weight_to_token_ratio=0.130886 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254580719e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=330136 combine_payload_bytes=266911744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267241880 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534153624 weight_movement_bytes=69206016 total_nvlink_bytes=603359640 weight_to_token_ratio=0.129562 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77306 payload_rows_remote=70442 dup_rows=20998 dup_rows_remote=20554 dedup_ratio=0.213603 router_tokens_per_expert=[30, 2423, 2, 1066, 507, 1826, 950, 0, 5340, 160, 622, 535, 0, 26, 421, 1475, 0, 20, 29, 15, 0, 2175, 1356, 2, 0, 57, 2816, 1033, 15, 156, 0, 0, 323, 2622, 264, 158, 217, 2143, 125, 424, 198, 490, 4685, 266, 16, 125, 0, 1483, 235, 641, 0, 2057, 5, 1606, 29, 1169, 0, 1471, 1167, 1473, 1, 2702, 0, 0] payload_rows_per_dst=[7326, 6106, 8552, 9530, 6864, 11222, 14742, 12964] dup_rows_per_dst=[498, 372, 774, 2276, 444, 3304, 7422, 5908] combine_rows_per_src=[7326, 6106, 8552, 9530, 6864, 11222, 14742, 12964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82090 payload_rows_remote=72004 dup_rows=16214 dup_rows_remote=14744 dedup_ratio=0.164937 router_tokens_per_expert=[0, 64, 2, 301, 213, 1115, 114, 480, 94, 0, 185, 80, 521, 260, 132, 869, 5283, 129, 623, 1819, 122, 2982, 103, 0, 146, 1136, 6442, 156, 79, 307, 0, 156, 181, 486, 3, 10, 5331, 383, 29, 1, 3, 211, 1, 562, 535, 365, 0, 145, 5270, 1179, 476, 650, 249, 1327, 9, 162, 1973, 77, 534, 0, 290, 444, 3935, 418] payload_rows_per_dst=[11724, 11894, 10086, 14396, 11982, 12134, 6434, 3440] dup_rows_per_dst=[4670, 2954, 1470, 2448, 866, 2050, 1670, 86] combine_rows_per_src=[11724, 11894, 10086, 14396, 11982, 12134, 6434, 3440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82234 payload_rows_remote=67896 dup_rows=16070 dup_rows_remote=14038 dedup_ratio=0.163472 router_tokens_per_expert=[0, 16, 0, 141, 27, 1761, 21, 530, 49, 0, 179, 346, 1145, 928, 300, 402, 3320, 112, 260, 1369, 199, 3659, 169, 0, 215, 668, 6423, 97, 6, 322, 0, 454, 566, 372, 0, 0, 5221, 60, 219, 0, 209, 48, 0, 67, 450, 1046, 16, 156, 4033, 389, 778, 936, 257, 1356, 27, 147, 2883, 352, 672, 0, 618, 1565, 3195, 396] payload_rows_per_dst=[11638, 9456, 10502, 14338, 12042, 8864, 8506, 6888] dup_rows_per_dst=[5510, 3882, 1034, 2032, 834, 580, 1880, 318] combine_rows_per_src=[11638, 9456, 10502, 14338, 12042, 8864, 8506, 6888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=272408576 dispatch_meta_bytes=328504 combine_payload_bytes=272408576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272737080 ingress_bytes=307011584 link_bytes=307011584 token_bytes=545145656 weight_movement_bytes=34603008 total_nvlink_bytes=579748664 weight_to_token_ratio=0.063475 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155259874e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=288530432 dispatch_meta_bytes=363984 combine_payload_bytes=288530432 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=288894416 ingress_bytes=392339456 link_bytes=392339456 token_bytes=577424848 weight_movement_bytes=103809024 total_nvlink_bytes=681233872 weight_to_token_ratio=0.179779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476341787e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78732 payload_rows_remote=69758 dup_rows=19572 dup_rows_remote=18862 dedup_ratio=0.199097 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 769, 3802, 0, 228, 5, 4, 778, 0, 0, 25, 0, 6, 80, 2, 1591, 0, 0, 1, 2, 8187, 0, 3, 1, 0, 3, 2, 258, 0, 0, 8172, 0, 2, 0, 0, 4, 0, 0, 6863, 3, 0, 0, 8017, 2, 52, 474, 1, 7, 0, 0, 19, 5170, 1, 128, 1, 1433, 19, 3035] payload_rows_per_dst=[1586, 8974, 3356, 16380, 16350, 16242, 1068, 14776] dup_rows_per_dst=[32, 710, 2, 14, 518, 13532, 4, 4760] combine_rows_per_src=[1586, 8974, 3356, 16380, 16350, 16242, 1068, 14776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=294928384 dispatch_meta_bytes=346992 combine_payload_bytes=294928384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295275376 ingress_bytes=364134400 link_bytes=364134400 token_bytes=590203760 weight_movement_bytes=69206016 total_nvlink_bytes=659409776 weight_to_token_ratio=0.117258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370208432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=278102016 dispatch_meta_bytes=327736 combine_payload_bytes=278102016 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278429752 ingress_bytes=416514048 link_bytes=416514048 token_bytes=556531768 weight_movement_bytes=138412032 total_nvlink_bytes=694943800 weight_to_token_ratio=0.248705 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=35328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567308830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78134 payload_rows_remote=64700 dup_rows=20170 dup_rows_remote=14782 dedup_ratio=0.205180 router_tokens_per_expert=[0, 0, 10, 345, 73, 3481, 24, 102, 0, 0, 75, 118, 2621, 68, 568, 1329, 6580, 284, 476, 741, 73, 1937, 375, 0, 191, 838, 3813, 64, 9, 976, 0, 696, 1198, 18, 0, 0, 1831, 163, 407, 0, 133, 75, 0, 33, 37, 2428, 11, 193, 779, 1064, 508, 183, 244, 3792, 17, 45, 3039, 1, 1201, 0, 1182, 288, 4414, 1] payload_rows_per_dst=[11826, 8500, 15546, 5154, 6834, 5686, 13434, 11154] dup_rows_per_dst=[5072, 1058, 5386, 394, 400, 134, 5388, 2338] combine_rows_per_src=[11826, 8500, 15546, 5154, 6834, 5686, 13434, 11154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73868 payload_rows_remote=70390 dup_rows=24436 dup_rows_remote=24106 dedup_ratio=0.248576 router_tokens_per_expert=[0, 2, 2, 276, 64, 2408, 17, 105, 16, 0, 135, 74, 1770, 134, 403, 1172, 6224, 185, 291, 1211, 140, 2937, 173, 0, 168, 1427, 5112, 167, 89, 201, 0, 513, 615, 45, 0, 0, 3469, 74, 135, 0, 70, 324, 0, 37, 100, 1242, 0, 131, 2623, 2258, 424, 287, 231, 2469, 27, 101, 2751, 20, 847, 0, 553, 67, 4773, 63] payload_rows_per_dst=[11984, 8268, 11056, 9750, 8554, 3478, 13168, 7610] dup_rows_per_dst=[3982, 5472, 4934, 2120, 122, 330, 7156, 320] combine_rows_per_src=[11984, 8268, 11056, 9750, 8554, 3478, 13168, 7610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12907278888 prefetch_bytes=761266176 reduce_bytes=692060160 weight_movement_bytes=1453326336 egress_bytes=6457594408 ingress_bytes=7903010816 link_bytes=7903010816 total_nvlink_bytes=14360605224 weight_to_token_ratio=0.112597 prefetch_share_of_ingress=0.096326 dedup_ratio=0.205444 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=20 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.292973629e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=58306 dup_rows=24100 dup_rows_remote=13836 dedup_ratio=0.245158 router_tokens_per_expert=[21, 2070, 2264, 0, 0, 941, 122, 12, 24, 373, 0, 2843, 5130, 109, 16, 3, 12, 5216, 1320, 0, 0, 18, 1, 0, 191, 63, 649, 16, 11, 277, 1051, 384, 88, 0, 0, 0, 0, 662, 0, 1082, 2366, 6635, 26, 467, 78, 3892, 63, 252, 1, 32, 159, 0, 593, 20, 37, 7651, 12, 665, 468, 0, 25, 414, 309, 18] payload_rows_per_dst=[15898, 13118, 12216, 10874, 2476, 13484, 1672, 4466] dup_rows_per_dst=[10264, 8610, 918, 2194, 1188, 650, 12, 264] combine_rows_per_src=[15898, 13118, 12216, 10874, 2476, 13484, 1672, 4466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285728768 dispatch_meta_bytes=354480 combine_payload_bytes=285728768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286083248 ingress_bytes=320331776 link_bytes=320331776 token_bytes=571812016 weight_movement_bytes=34603008 total_nvlink_bytes=606415024 weight_to_token_ratio=0.060515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205382684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=317928 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265329128 ingress_bytes=334217216 link_bytes=334217216 token_bytes=530340328 weight_movement_bytes=69206016 total_nvlink_bytes=599546344 weight_to_token_ratio=0.130494 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257632477e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=288317440 dispatch_meta_bytes=377984 combine_payload_bytes=288317440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288695424 ingress_bytes=322920448 link_bytes=322920448 token_bytes=577012864 weight_movement_bytes=34603008 total_nvlink_bytes=611615872 weight_to_token_ratio=0.059969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215123649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76484 payload_rows_remote=60210 dup_rows=21820 dup_rows_remote=4216 dedup_ratio=0.221965 router_tokens_per_expert=[4, 21, 0, 2, 2, 2, 4, 1280, 2035, 0, 225, 9, 0, 1847, 4, 55, 157, 45, 24, 759, 2, 3041, 9, 0, 59, 151, 8108, 2, 0, 2, 0, 2, 0, 325, 0, 0, 8046, 0, 0, 0, 14, 79, 0, 41, 4604, 0, 2, 15, 7914, 27, 400, 623, 23, 25, 14, 76, 152, 4406, 0, 34, 1, 2208, 242, 2030] payload_rows_per_dst=[3024, 7378, 7722, 424, 16122, 9452, 16088, 16274] dup_rows_per_dst=[90, 972, 352, 8, 620, 58, 2116, 17604] combine_rows_per_src=[3024, 7378, 7722, 424, 16122, 9452, 16088, 16274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=238821376 dispatch_meta_bytes=288568 combine_payload_bytes=238821376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239109944 ingress_bytes=273424384 link_bytes=273424384 token_bytes=477931320 weight_movement_bytes=34603008 total_nvlink_bytes=512534328 weight_to_token_ratio=0.072402 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.028873945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76094 payload_rows_remote=66042 dup_rows=22210 dup_rows_remote=20694 dedup_ratio=0.225932 router_tokens_per_expert=[3, 1374, 0, 2701, 1841, 2486, 158, 0, 6830, 18, 5, 149, 0, 97, 325, 3408, 0, 53, 94, 54, 0, 936, 1945, 1, 0, 83, 849, 252, 67, 134, 176, 0, 309, 4753, 1093, 32, 180, 1280, 314, 197, 395, 791, 5535, 380, 14, 201, 0, 2436, 4, 533, 0, 2492, 2, 562, 51, 878, 0, 667, 662, 473, 8, 870, 0, 1] payload_rows_per_dst=[6344, 14974, 10052, 7184, 11708, 13902, 7824, 4106] dup_rows_per_dst=[408, 6690, 1516, 910, 2282, 7928, 1220, 1256] combine_rows_per_src=[6344, 14974, 10052, 7184, 11708, 13902, 7824, 4106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82104 payload_rows_remote=70956 dup_rows=16200 dup_rows_remote=13130 dedup_ratio=0.164795 router_tokens_per_expert=[27, 2022, 0, 3103, 1814, 3081, 650, 0, 4047, 13, 269, 319, 0, 4, 226, 1991, 0, 30, 40, 14, 0, 1333, 2216, 0, 0, 64, 2701, 1088, 11, 164, 0, 0, 1058, 4447, 332, 106, 31, 1447, 32, 209, 135, 376, 5623, 337, 57, 49, 0, 716, 228, 721, 0, 2119, 5, 1076, 1, 696, 2, 1079, 625, 943, 0, 1475, 0, 0] payload_rows_per_dst=[8266, 9280, 11120, 11148, 6132, 12504, 10320, 13334] dup_rows_per_dst=[760, 1118, 2352, 3070, 298, 2386, 2712, 3504] combine_rows_per_src=[8266, 9280, 11120, 11148, 6132, 12504, 10320, 13334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=246620160 dispatch_meta_bytes=257704 combine_payload_bytes=246620160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=246877864 ingress_bytes=350429184 link_bytes=350429184 token_bytes=493498024 weight_movement_bytes=103809024 total_nvlink_bytes=597307048 weight_to_token_ratio=0.210353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318636808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81796 payload_rows_remote=70728 dup_rows=16508 dup_rows_remote=14224 dedup_ratio=0.167928 router_tokens_per_expert=[0, 2, 0, 7908, 8166, 7681, 0, 0, 29, 0, 0, 6, 0, 0, 3, 2965, 0, 14, 0, 0, 0, 9, 1675, 0, 0, 90, 45, 363, 0, 0, 0, 0, 4882, 8178, 71, 0, 0, 6, 0, 0, 1, 13, 4607, 254, 0, 3, 0, 4, 1, 897, 0, 159, 0, 6, 0, 9, 1097, 1, 2, 2, 0, 3, 0, 0] payload_rows_per_dst=[16336, 11068, 15850, 15414, 16368, 2412, 2138, 2210] dup_rows_per_dst=[0, 2284, 3362, 944, 9906, 6, 6, 0] combine_rows_per_src=[16336, 11068, 15850, 15414, 16368, 2412, 2138, 2210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=270508032 dispatch_meta_bytes=346944 combine_payload_bytes=270508032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270854976 ingress_bytes=305111040 link_bytes=305111040 token_bytes=541363008 weight_movement_bytes=34603008 total_nvlink_bytes=575966016 weight_to_token_ratio=0.063918 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148108280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76270 payload_rows_remote=61984 dup_rows=22034 dup_rows_remote=15340 dedup_ratio=0.224141 router_tokens_per_expert=[70, 2095, 0, 427, 59, 1109, 1450, 0, 4305, 165, 480, 817, 0, 32, 750, 819, 0, 0, 86, 22, 0, 3113, 881, 0, 0, 49, 3138, 586, 25, 376, 0, 0, 350, 936, 227, 218, 38, 4018, 87, 419, 557, 588, 4058, 309, 41, 108, 0, 2097, 451, 468, 0, 1556, 2, 2505, 13, 1190, 0, 1350, 1249, 2322, 0, 3141, 0, 0] payload_rows_per_dst=[6822, 5746, 8562, 8710, 9932, 10458, 14286, 11754] dup_rows_per_dst=[526, 380, 496, 1856, 782, 5058, 6694, 6242] combine_rows_per_src=[6822, 5746, 8562, 8710, 9932, 10458, 14286, 11754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290635776 dispatch_meta_bytes=336344 combine_payload_bytes=290635776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290972120 ingress_bytes=325238784 link_bytes=325238784 token_bytes=581607896 weight_movement_bytes=34603008 total_nvlink_bytes=616210904 weight_to_token_ratio=0.059495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223847360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76406 payload_rows_remote=64408 dup_rows=21898 dup_rows_remote=17052 dedup_ratio=0.222758 router_tokens_per_expert=[37, 2346, 0, 1569, 492, 2158, 1019, 0, 5369, 269, 130, 483, 0, 72, 508, 858, 0, 16, 121, 58, 0, 2365, 1495, 0, 0, 65, 2249, 780, 4, 137, 0, 0, 341, 2402, 240, 304, 163, 2487, 102, 488, 419, 289, 5014, 281, 44, 63, 0, 2312, 61, 283, 0, 1933, 0, 1261, 1, 1758, 0, 1231, 1113, 1645, 0, 2317, 0, 0] payload_rows_per_dst=[7286, 4472, 10016, 8710, 7412, 11998, 14462, 12050] dup_rows_per_dst=[502, 168, 1232, 2076, 838, 4846, 6870, 5366] combine_rows_per_src=[7286, 4472, 10016, 8710, 7412, 11998, 14462, 12050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12861011016 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=6434444360 ingress_bytes=7983702016 link_bytes=7983702016 total_nvlink_bytes=14418146376 weight_to_token_ratio=0.121074 prefetch_share_of_ingress=0.091018 dedup_ratio=0.203802 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.306175128e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80912 payload_rows_remote=74062 dup_rows=17392 dup_rows_remote=15892 dedup_ratio=0.176921 router_tokens_per_expert=[0, 3, 5, 41, 29, 333, 95, 781, 903, 0, 206, 243, 358, 1086, 21, 348, 1346, 73, 63, 2471, 64, 4351, 109, 0, 160, 330, 7565, 51, 42, 155, 0, 34, 140, 494, 0, 0, 7184, 18, 56, 0, 99, 48, 0, 286, 2618, 208, 6, 43, 6988, 582, 645, 1051, 138, 405, 4, 266, 1073, 1277, 229, 16, 46, 838, 1815, 1313] payload_rows_per_dst=[6850, 7142, 13270, 13784, 13026, 14822, 5308, 6710] dup_rows_per_dst=[1500, 1880, 4906, 794, 940, 5770, 874, 728] combine_rows_per_src=[6850, 7142, 13270, 13784, 13026, 14822, 5308, 6710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289701888 dispatch_meta_bytes=339808 combine_payload_bytes=289701888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290041696 ingress_bytes=358907904 link_bytes=358907904 token_bytes=579743584 weight_movement_bytes=69206016 total_nvlink_bytes=648949600 weight_to_token_ratio=0.119373 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350541548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12889299432 prefetch_bytes=726663168 reduce_bytes=968884224 weight_movement_bytes=1695547392 egress_bytes=6448511464 ingress_bytes=8136335360 link_bytes=8136335360 total_nvlink_bytes=14584846824 weight_to_token_ratio=0.131547 prefetch_share_of_ingress=0.089311 dedup_ratio=0.188309 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=28 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.331146736e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=253886464 dispatch_meta_bytes=309296 combine_payload_bytes=253886464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254195760 ingress_bytes=288489472 link_bytes=288489472 token_bytes=508082224 weight_movement_bytes=34603008 total_nvlink_bytes=542685232 weight_to_token_ratio=0.068105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.085562658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263815168 dispatch_meta_bytes=325840 combine_payload_bytes=263815168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264141008 ingress_bytes=333021184 link_bytes=333021184 token_bytes=527956176 weight_movement_bytes=69206016 total_nvlink_bytes=597162192 weight_to_token_ratio=0.131083 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253131905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=303357952 dispatch_meta_bytes=359816 combine_payload_bytes=303357952 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=303717768 ingress_bytes=372563968 link_bytes=372563968 token_bytes=607075720 weight_movement_bytes=69206016 total_nvlink_bytes=676281736 weight_to_token_ratio=0.113999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401928218e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13257316736 prefetch_bytes=795869184 reduce_bytes=657457152 weight_movement_bytes=1453326336 egress_bytes=6632634752 ingress_bytes=8078008320 link_bytes=8078008320 total_nvlink_bytes=14710643072 weight_to_token_ratio=0.109624 prefetch_share_of_ingress=0.098523 dedup_ratio=0.184822 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=19 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.321604129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90974 payload_rows_remote=74864 dup_rows=7330 dup_rows_remote=6850 dedup_ratio=0.074565 router_tokens_per_expert=[16, 165, 0, 7623, 7833, 7361, 6, 0, 599, 0, 0, 6, 0, 17, 3, 4276, 0, 19, 25, 0, 26, 2, 2540, 0, 3, 85, 81, 300, 0, 0, 20, 0, 3123, 8006, 163, 1, 1, 0, 69, 11, 12, 82, 4445, 396, 25, 8, 0, 101, 3, 711, 0, 644, 0, 0, 5, 51, 193, 48, 45, 0, 0, 2, 1, 0] payload_rows_per_dst=[15792, 8590, 15550, 14818, 6556, 9786, 3772, 16110] dup_rows_per_dst=[248, 14, 4920, 882, 180, 352, 254, 480] combine_rows_per_src=[15792, 8590, 15550, 14818, 6556, 9786, 3772, 16110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12313653520 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6160625936 ingress_bytes=7294926848 link_bytes=7294926848 total_nvlink_bytes=13455552784 weight_to_token_ratio=0.092734 prefetch_share_of_ingress=0.090125 dedup_ratio=0.220384 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.193487932e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12795653216 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6401764448 ingress_bytes=7812612096 link_bytes=7812612096 total_nvlink_bytes=14214376544 weight_to_token_ratio=0.110875 prefetch_share_of_ingress=0.101870 dedup_ratio=0.209069 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.278183928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=306642944 dispatch_meta_bytes=326856 combine_payload_bytes=306642944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306969800 ingress_bytes=341245952 link_bytes=341245952 token_bytes=613612744 weight_movement_bytes=34603008 total_nvlink_bytes=648215752 weight_to_token_ratio=0.056392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284081045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67724 payload_rows_remote=52596 dup_rows=30580 dup_rows_remote=18518 dedup_ratio=0.311076 router_tokens_per_expert=[7, 774, 0, 5815, 5438, 4935, 201, 0, 4247, 16, 86, 53, 0, 32, 73, 3440, 0, 76, 21, 0, 0, 417, 1272, 0, 0, 84, 690, 1099, 4, 103, 10, 0, 1136, 6866, 611, 68, 9, 531, 53, 39, 46, 43, 5574, 377, 3, 10, 0, 592, 9, 647, 0, 1583, 0, 196, 2, 354, 0, 425, 452, 165, 0, 468, 0, 0] payload_rows_per_dst=[15128, 14480, 3852, 9670, 15276, 2002, 5126, 2190] dup_rows_per_dst=[12062, 12562, 98, 1106, 3326, 140, 456, 830] combine_rows_per_src=[15128, 14480, 3852, 9670, 15276, 2002, 5126, 2190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12139611232 prefetch_bytes=622854144 reduce_bytes=934281216 weight_movement_bytes=1557135360 egress_bytes=6073476192 ingress_bytes=7623270400 link_bytes=7623270400 total_nvlink_bytes=13696746592 weight_to_token_ratio=0.128269 prefetch_share_of_ingress=0.081704 dedup_ratio=0.214130 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=27 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.247206643e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=215433216 dispatch_meta_bytes=284456 combine_payload_bytes=215433216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=215717672 ingress_bytes=319242240 link_bytes=319242240 token_bytes=431150888 weight_movement_bytes=103809024 total_nvlink_bytes=534959912 weight_to_token_ratio=0.240772 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201282848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12107073888 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6057232736 ingress_bytes=7606976512 link_bytes=7606976512 total_nvlink_bytes=13664209248 weight_to_token_ratio=0.128614 prefetch_share_of_ingress=0.109172 dedup_ratio=0.219056 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244540878e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none + [2026-08-29 23:40:34] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 671.5 | throughput per GPU (TFLOP/s/GPU): 190.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.121702E+01 | loss scale: 1.0 | grad norm: 398.481 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75658 payload_rows_remote=63522 dup_rows=22646 dup_rows_remote=17372 dedup_ratio=0.230367 router_tokens_per_expert=[2054, 631, 228, 1181, 123, 1073, 573, 96, 3224, 501, 718, 404, 5, 2451, 734, 668, 465, 859, 589, 692, 132, 317, 142, 917, 598, 540, 392, 689, 861, 393, 751, 947, 883, 549, 1225, 1227, 1085, 339, 992, 583, 184, 508, 463, 135, 855, 1428, 406, 94, 166, 356, 528, 1165, 821, 2429, 280, 370, 1346, 546, 1622, 706, 138, 745, 1942, 1088] payload_rows_per_dst=[9590, 12136, 9282, 9852, 8364, 7034, 9722, 9678] dup_rows_per_dst=[3242, 5274, 2828, 2940, 2038, 1112, 2508, 2704] combine_rows_per_src=[9590, 12136, 9282, 9852, 8364, 7034, 9722, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76048 payload_rows_remote=65926 dup_rows=22256 dup_rows_remote=19308 dedup_ratio=0.226400 router_tokens_per_expert=[2005, 681, 258, 1109, 139, 1127, 524, 117, 2973, 556, 765, 389, 4, 2536, 712, 577, 558, 841, 688, 767, 152, 223, 177, 857, 623, 721, 456, 616, 883, 463, 772, 991, 931, 512, 1126, 1217, 956, 347, 896, 550, 208, 526, 548, 172, 864, 1409, 446, 64, 177, 425, 479, 1225, 789, 2241, 252, 457, 1394, 450, 1794, 684, 97, 737, 1959, 960] payload_rows_per_dst=[9054, 8710, 9556, 8834, 10122, 10642, 9708, 9422] dup_rows_per_dst=[2866, 2368, 2888, 2216, 2948, 3778, 2382, 2810] combine_rows_per_src=[9054, 8710, 9556, 8834, 10122, 10642, 9708, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75674 payload_rows_remote=66174 dup_rows=22630 dup_rows_remote=19774 dedup_ratio=0.230204 router_tokens_per_expert=[1972, 626, 243, 1128, 125, 1099, 496, 103, 2907, 649, 726, 438, 3, 2369, 655, 711, 599, 924, 609, 692, 151, 242, 186, 984, 624, 552, 457, 636, 861, 534, 844, 1062, 943, 616, 1500, 1118, 1036, 421, 1067, 547, 209, 532, 461, 182, 816, 1402, 444, 98, 134, 374, 482, 1092, 759, 2170, 258, 489, 1355, 570, 1619, 644, 126, 632, 1791, 1058] payload_rows_per_dst=[8798, 10490, 9500, 10392, 9196, 8480, 9434, 9384] dup_rows_per_dst=[2786, 3922, 2856, 3748, 2300, 2312, 2082, 2624] combine_rows_per_src=[8798, 10490, 9500, 10392, 9196, 8480, 9434, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76322 payload_rows_remote=65830 dup_rows=21982 dup_rows_remote=18346 dedup_ratio=0.223612 router_tokens_per_expert=[2079, 569, 212, 1165, 126, 1033, 508, 92, 3057, 611, 859, 377, 0, 2486, 731, 555, 579, 819, 676, 742, 142, 221, 178, 1042, 618, 649, 454, 748, 847, 464, 890, 964, 901, 585, 1274, 1122, 1124, 324, 886, 574, 160, 488, 454, 168, 828, 1401, 438, 70, 133, 348, 536, 1125, 770, 2395, 300, 394, 1303, 463, 1709, 594, 142, 697, 1924, 1029] payload_rows_per_dst=[8860, 8806, 9870, 8898, 10508, 10492, 9606, 9282] dup_rows_per_dst=[2708, 2432, 2776, 2370, 3072, 3636, 2396, 2592] combine_rows_per_src=[8860, 8806, 9870, 8898, 10508, 10492, 9606, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76264 payload_rows_remote=66440 dup_rows=22040 dup_rows_remote=19030 dedup_ratio=0.224202 router_tokens_per_expert=[2136, 681, 255, 987, 164, 1013, 586, 123, 2994, 558, 723, 428, 5, 2272, 665, 717, 526, 829, 648, 723, 158, 252, 185, 932, 570, 706, 380, 626, 907, 415, 879, 1048, 891, 582, 1233, 1104, 1080, 364, 962, 712, 173, 547, 433, 171, 798, 1329, 480, 74, 198, 427, 472, 1194, 799, 2145, 284, 455, 1396, 528, 1660, 580, 142, 712, 2065, 1071] payload_rows_per_dst=[8966, 8440, 9786, 9824, 9702, 10462, 9618, 9466] dup_rows_per_dst=[2924, 2296, 2850, 3010, 2382, 3536, 2330, 2712] combine_rows_per_src=[8966, 8440, 9786, 9824, 9702, 10462, 9618, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260186112 dispatch_meta_bytes=323576 combine_payload_bytes=260186112 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=260509688 ingress_bytes=398598144 link_bytes=398598144 token_bytes=520695800 weight_movement_bytes=138412032 total_nvlink_bytes=659107832 weight_to_token_ratio=0.265821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499892726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270032896 dispatch_meta_bytes=340936 combine_payload_bytes=270032896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270373832 ingress_bytes=373841920 link_bytes=373841920 token_bytes=540406728 weight_movement_bytes=103809024 total_nvlink_bytes=644215752 weight_to_token_ratio=0.192094 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406737048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271048704 dispatch_meta_bytes=343792 combine_payload_bytes=271048704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271392496 ingress_bytes=305651712 link_bytes=305651712 token_bytes=542441200 weight_movement_bytes=34603008 total_nvlink_bytes=577044208 weight_to_token_ratio=0.063791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150142785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269639680 dispatch_meta_bytes=336704 combine_payload_bytes=269639680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269976384 ingress_bytes=304242688 link_bytes=304242688 token_bytes=539616064 weight_movement_bytes=34603008 total_nvlink_bytes=574219072 weight_to_token_ratio=0.064125 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144840741e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272138240 dispatch_meta_bytes=341880 combine_payload_bytes=272138240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272480120 ingress_bytes=306741248 link_bytes=306741248 token_bytes=544618360 weight_movement_bytes=34603008 total_nvlink_bytes=579221368 weight_to_token_ratio=0.063536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154242622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75128 payload_rows_remote=65012 dup_rows=23176 dup_rows_remote=19952 dedup_ratio=0.235758 router_tokens_per_expert=[2111, 686, 299, 994, 139, 1228, 556, 122, 3062, 682, 665, 336, 4, 2525, 638, 700, 521, 809, 599, 686, 172, 297, 178, 978, 534, 713, 494, 686, 861, 414, 738, 874, 898, 517, 1114, 1283, 1026, 417, 959, 635, 190, 550, 487, 162, 810, 1418, 439, 80, 179, 398, 499, 1036, 734, 2466, 299, 414, 1453, 552, 1717, 591, 148, 611, 1796, 973] payload_rows_per_dst=[9346, 5296, 9106, 8426, 10956, 10700, 11182, 10116] dup_rows_per_dst=[2924, 754, 2664, 2202, 3490, 3696, 4222, 3224] combine_rows_per_src=[9346, 5296, 9106, 8426, 10956, 10700, 11182, 10116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76002 payload_rows_remote=66756 dup_rows=22302 dup_rows_remote=20174 dedup_ratio=0.226868 router_tokens_per_expert=[2145, 652, 255, 1063, 139, 1125, 586, 134, 3099, 466, 689, 440, 10, 2452, 638, 615, 533, 897, 613, 694, 163, 242, 164, 923, 624, 678, 443, 674, 930, 490, 842, 923, 909, 572, 1188, 1272, 1102, 409, 916, 621, 177, 542, 488, 165, 785, 1444, 491, 88, 180, 402, 523, 1109, 792, 1983, 279, 419, 1329, 495, 1699, 651, 103, 702, 1865, 1111] payload_rows_per_dst=[9166, 8420, 9348, 8938, 10638, 10692, 9246, 9554] dup_rows_per_dst=[3032, 2122, 2840, 2270, 3418, 3866, 2128, 2626] combine_rows_per_src=[9166, 8420, 9348, 8938, 10638, 10692, 9246, 9554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75416 payload_rows_remote=65034 dup_rows=22888 dup_rows_remote=19228 dedup_ratio=0.232829 router_tokens_per_expert=[2021, 613, 264, 1054, 163, 1140, 589, 105, 2860, 621, 847, 334, 12, 2477, 695, 573, 570, 850, 670, 721, 178, 231, 179, 931, 702, 784, 436, 739, 888, 451, 746, 981, 1007, 540, 1465, 1072, 1034, 479, 963, 562, 161, 491, 404, 161, 777, 1392, 442, 105, 164, 408, 516, 1077, 729, 2261, 297, 418, 1323, 530, 1627, 662, 171, 711, 1747, 1031] payload_rows_per_dst=[10382, 11812, 9386, 10490, 7618, 6782, 9526, 9420] dup_rows_per_dst=[3660, 5026, 2768, 3894, 1552, 1084, 2214, 2690] combine_rows_per_src=[10382, 11812, 9386, 10490, 7618, 6782, 9526, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76284 payload_rows_remote=69418 dup_rows=22020 dup_rows_remote=20596 dedup_ratio=0.223999 router_tokens_per_expert=[227, 462, 46, 177, 4639, 55, 2152, 427, 42, 619, 1178, 415, 671, 980, 40, 2929, 439, 1109, 340, 594, 266, 931, 370, 385, 420, 558, 657, 3735, 1, 133, 102, 689, 49, 1, 252, 7, 508, 6, 296, 108, 773, 511, 249, 586, 18, 496, 1, 262, 6632, 316, 3653, 0, 22, 198, 519, 989, 2912, 332, 1037, 606, 289, 144, 1295, 297] payload_rows_per_dst=[13758, 6866, 12108, 9986, 14458, 4462, 8160, 6486] dup_rows_per_dst=[8470, 1424, 4066, 2204, 1260, 1330, 1752, 1514] combine_rows_per_src=[13758, 6866, 12108, 9986, 14458, 4462, 8160, 6486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77570 payload_rows_remote=65348 dup_rows=20734 dup_rows_remote=16876 dedup_ratio=0.210917 router_tokens_per_expert=[186, 586, 27, 181, 4689, 52, 2286, 349, 83, 643, 1148, 403, 734, 1090, 41, 2768, 378, 1102, 537, 451, 367, 831, 342, 435, 434, 625, 685, 3617, 7, 147, 113, 788, 116, 1, 254, 31, 518, 3, 285, 91, 805, 519, 252, 661, 20, 545, 4, 253, 6551, 340, 3597, 0, 18, 199, 503, 946, 2496, 334, 922, 633, 351, 221, 1224, 334] payload_rows_per_dst=[11702, 6790, 12222, 10292, 14290, 8832, 4596, 8846] dup_rows_per_dst=[5542, 1494, 3858, 2540, 1410, 2290, 540, 3060] combine_rows_per_src=[11702, 6790, 12222, 10292, 14290, 8832, 4596, 8846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80668 payload_rows_remote=66922 dup_rows=17636 dup_rows_remote=16282 dedup_ratio=0.179403 router_tokens_per_expert=[253, 501, 38, 153, 4434, 92, 2068, 361, 62, 686, 1409, 456, 799, 816, 55, 2912, 380, 1061, 555, 756, 361, 828, 419, 408, 405, 545, 692, 3661, 1, 154, 147, 749, 89, 1, 267, 20, 526, 0, 268, 106, 753, 453, 255, 479, 23, 653, 4, 271, 6273, 404, 3388, 2, 20, 224, 485, 923, 2668, 355, 1068, 640, 454, 205, 1355, 303] payload_rows_per_dst=[6102, 10590, 8344, 10192, 13746, 11960, 9500, 10234] dup_rows_per_dst=[830, 3800, 1192, 2516, 1354, 2690, 1392, 3862] combine_rows_per_src=[6102, 10590, 8344, 10192, 13746, 11960, 9500, 10234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80886 payload_rows_remote=68540 dup_rows=17418 dup_rows_remote=14220 dedup_ratio=0.177185 router_tokens_per_expert=[200, 527, 47, 164, 4563, 81, 1901, 456, 90, 658, 1066, 426, 624, 1042, 51, 2825, 416, 1018, 509, 517, 334, 965, 326, 459, 397, 587, 788, 3813, 6, 138, 101, 579, 114, 0, 260, 9, 418, 0, 284, 109, 806, 527, 257, 684, 36, 581, 2, 316, 6552, 355, 3695, 1, 16, 201, 541, 926, 2684, 268, 891, 654, 348, 182, 1421, 340] payload_rows_per_dst=[5992, 10242, 7944, 10328, 14176, 12346, 9948, 9910] dup_rows_per_dst=[760, 3322, 1144, 2490, 1316, 3198, 1522, 3666] combine_rows_per_src=[5992, 10242, 7944, 10328, 14176, 12346, 9948, 9910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=339856 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266629008 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532918160 weight_movement_bytes=69206016 total_nvlink_bytes=602124176 weight_to_token_ratio=0.129862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79866 payload_rows_remote=69476 dup_rows=18438 dup_rows_remote=15740 dedup_ratio=0.187561 router_tokens_per_expert=[225, 429, 58, 192, 4685, 56, 2131, 464, 70, 867, 1164, 369, 720, 1119, 26, 2921, 509, 1027, 266, 620, 331, 888, 332, 389, 488, 551, 777, 3810, 0, 112, 113, 693, 92, 2, 251, 22, 464, 5, 233, 126, 789, 461, 247, 640, 23, 507, 1, 300, 6435, 352, 3224, 0, 33, 215, 506, 973, 2600, 417, 1004, 670, 366, 156, 1325, 311] payload_rows_per_dst=[8656, 8480, 9982, 10390, 14004, 12350, 6010, 9994] dup_rows_per_dst=[2218, 2268, 2686, 2698, 1256, 2956, 652, 3704] combine_rows_per_src=[8656, 8480, 9982, 10390, 14004, 12350, 6010, 9994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273432576 dispatch_meta_bytes=347720 combine_payload_bytes=273432576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273780296 ingress_bytes=308035584 link_bytes=308035584 token_bytes=547212872 weight_movement_bytes=34603008 total_nvlink_bytes=581815880 weight_to_token_ratio=0.063235 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159113104e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284336128 dispatch_meta_bytes=360056 combine_payload_bytes=284336128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284696184 ingress_bytes=353542144 link_bytes=353542144 token_bytes=569032312 weight_movement_bytes=69206016 total_nvlink_bytes=638238328 weight_to_token_ratio=0.121621 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330350625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=337048 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266716312 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533095576 weight_movement_bytes=34603008 total_nvlink_bytes=567698584 weight_to_token_ratio=0.064910 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267665408 dispatch_meta_bytes=328896 combine_payload_bytes=267665408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267994304 ingress_bytes=302268416 link_bytes=302268416 token_bytes=535659712 weight_movement_bytes=34603008 total_nvlink_bytes=570262720 weight_to_token_ratio=0.064599 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137411715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274112512 dispatch_meta_bytes=332816 combine_payload_bytes=274112512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274445328 ingress_bytes=308715520 link_bytes=308715520 token_bytes=548557840 weight_movement_bytes=34603008 total_nvlink_bytes=583160848 weight_to_token_ratio=0.063080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161671648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280739840 dispatch_meta_bytes=331040 combine_payload_bytes=280739840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281070880 ingress_bytes=315342848 link_bytes=315342848 token_bytes=561810720 weight_movement_bytes=34603008 total_nvlink_bytes=596413728 weight_to_token_ratio=0.061592 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186609750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284573696 dispatch_meta_bytes=340864 combine_payload_bytes=284573696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284914560 ingress_bytes=353779712 link_bytes=353779712 token_bytes=569488256 weight_movement_bytes=69206016 total_nvlink_bytes=638694272 weight_to_token_ratio=0.121523 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331244575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78616 payload_rows_remote=66552 dup_rows=19688 dup_rows_remote=13732 dedup_ratio=0.200277 router_tokens_per_expert=[0, 118, 65, 168, 1, 129, 347, 1880, 6, 1684, 236, 1671, 453, 327, 0, 1, 4632, 894, 5450, 70, 0, 0, 5456, 765, 189, 565, 2724, 2789, 3, 282, 255, 1, 26, 543, 551, 184, 3600, 265, 272, 94, 1267, 0, 2, 26, 477, 2007, 97, 0, 921, 8, 217, 516, 3, 1, 185, 1, 4317, 219, 3, 4, 0, 1312, 0, 873] payload_rows_per_dst=[13166, 12064, 10244, 10748, 9576, 6944, 4928, 10946] dup_rows_per_dst=[3162, 5956, 2278, 2868, 1494, 808, 612, 2510] combine_rows_per_src=[13166, 12064, 10244, 10748, 9576, 6944, 4928, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75338 payload_rows_remote=60168 dup_rows=22966 dup_rows_remote=14694 dedup_ratio=0.233622 router_tokens_per_expert=[237, 470, 42, 177, 4546, 85, 2089, 353, 82, 691, 1179, 384, 678, 971, 47, 2790, 428, 988, 412, 712, 267, 876, 422, 454, 471, 477, 783, 3781, 0, 123, 131, 650, 86, 2, 241, 24, 478, 3, 286, 104, 850, 511, 243, 671, 20, 472, 6, 343, 6636, 350, 3246, 0, 24, 183, 613, 993, 2585, 358, 1032, 690, 394, 181, 1400, 331] payload_rows_per_dst=[6092, 10498, 8012, 10410, 2888, 12144, 15170, 10124] dup_rows_per_dst=[814, 3146, 1106, 2422, 208, 3180, 8272, 3818] combine_rows_per_src=[6092, 10498, 8012, 10410, 2888, 12144, 15170, 10124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77870 payload_rows_remote=74360 dup_rows=20434 dup_rows_remote=20248 dedup_ratio=0.207865 router_tokens_per_expert=[0, 147, 93, 158, 1, 67, 413, 1991, 30, 1714, 346, 1444, 454, 355, 0, 0, 4343, 1034, 5498, 68, 1, 0, 5360, 745, 213, 719, 2752, 2429, 3, 287, 268, 0, 64, 529, 631, 344, 3528, 227, 136, 86, 1395, 0, 0, 22, 444, 2121, 97, 1, 1007, 7, 217, 573, 5, 0, 136, 0, 3965, 223, 22, 8, 0, 1446, 0, 985] payload_rows_per_dst=[13032, 11798, 3510, 10404, 9358, 7222, 11910, 10636] dup_rows_per_dst=[3428, 5574, 186, 2938, 1732, 938, 2976, 2662] combine_rows_per_src=[13032, 11798, 3510, 10404, 9358, 7222, 11910, 10636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75022 payload_rows_remote=64544 dup_rows=23282 dup_rows_remote=19444 dedup_ratio=0.236837 router_tokens_per_expert=[215, 459, 46, 139, 4280, 92, 2023, 326, 64, 616, 1233, 369, 640, 927, 40, 2962, 346, 1038, 489, 510, 306, 1116, 427, 428, 475, 528, 806, 3958, 3, 135, 124, 644, 51, 3, 245, 9, 470, 2, 276, 101, 809, 487, 246, 479, 16, 574, 0, 308, 6662, 388, 3362, 0, 33, 199, 534, 976, 2836, 385, 982, 582, 339, 175, 1603, 256] payload_rows_per_dst=[5900, 10426, 8072, 10616, 2144, 11806, 15580, 10478] dup_rows_per_dst=[700, 3276, 1248, 2730, 170, 2592, 8728, 3838] combine_rows_per_src=[5900, 10426, 8072, 10616, 2144, 11806, 15580, 10478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80466 payload_rows_remote=71184 dup_rows=17838 dup_rows_remote=16032 dedup_ratio=0.181458 router_tokens_per_expert=[0, 152, 71, 138, 1, 108, 447, 1887, 43, 1775, 262, 1637, 457, 350, 0, 0, 4337, 851, 5572, 60, 0, 0, 5028, 759, 170, 630, 2609, 3034, 1, 340, 264, 0, 43, 645, 642, 260, 3415, 214, 245, 80, 1527, 0, 0, 39, 456, 2040, 113, 0, 938, 6, 173, 571, 11, 0, 176, 0, 3931, 213, 10, 18, 0, 1413, 0, 990] payload_rows_per_dst=[12610, 8230, 5874, 10880, 9282, 11032, 11994, 10564] dup_rows_per_dst=[3054, 818, 646, 3216, 1806, 2812, 2900, 2586] combine_rows_per_src=[12610, 8230, 5874, 10880, 9282, 11032, 11994, 10564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81106 payload_rows_remote=67580 dup_rows=17198 dup_rows_remote=13264 dedup_ratio=0.174947 router_tokens_per_expert=[0, 168, 95, 185, 0, 146, 445, 1802, 17, 1718, 278, 1480, 450, 343, 0, 0, 4414, 708, 5525, 64, 1, 0, 5469, 920, 176, 662, 2693, 2705, 4, 428, 259, 0, 35, 461, 649, 321, 3296, 233, 166, 83, 1653, 0, 2, 29, 417, 2120, 95, 0, 1037, 7, 235, 476, 8, 0, 220, 0, 3942, 177, 11, 5, 0, 1326, 0, 993] payload_rows_per_dst=[5586, 7876, 11946, 10730, 9050, 13526, 11976, 10416] dup_rows_per_dst=[386, 696, 2088, 3124, 1438, 3934, 3040, 2492] combine_rows_per_src=[5586, 7876, 11946, 10730, 9050, 13526, 11976, 10416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76124 payload_rows_remote=62904 dup_rows=22180 dup_rows_remote=14632 dedup_ratio=0.225627 router_tokens_per_expert=[247, 529, 40, 153, 4356, 68, 1880, 415, 89, 725, 1157, 367, 593, 1090, 71, 2696, 496, 973, 436, 803, 366, 1016, 383, 461, 463, 538, 749, 3749, 2, 122, 112, 630, 104, 0, 261, 6, 524, 1, 288, 119, 765, 497, 257, 544, 25, 660, 8, 328, 6509, 377, 3226, 0, 30, 204, 535, 880, 2844, 347, 975, 566, 370, 239, 1558, 330] payload_rows_per_dst=[13220, 11888, 12076, 7198, 14254, 4848, 5622, 7018] dup_rows_per_dst=[7548, 3794, 4244, 1292, 1370, 1320, 860, 1752] combine_rows_per_src=[13220, 11888, 12076, 7198, 14254, 4848, 5622, 7018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81012 payload_rows_remote=70626 dup_rows=17292 dup_rows_remote=14498 dedup_ratio=0.175903 router_tokens_per_expert=[0, 192, 93, 161, 0, 124, 403, 1907, 22, 1636, 282, 1628, 449, 385, 0, 1, 4455, 886, 5488, 66, 2, 0, 5359, 867, 170, 634, 2471, 2646, 1, 396, 272, 0, 56, 468, 568, 354, 3516, 245, 164, 68, 1520, 0, 0, 23, 438, 2188, 116, 1, 1078, 9, 302, 522, 7, 0, 190, 0, 3785, 218, 10, 9, 0, 1364, 0, 937] payload_rows_per_dst=[13146, 8786, 9564, 10386, 9256, 7620, 12082, 10172] dup_rows_per_dst=[3332, 1562, 1446, 2794, 1622, 952, 3110, 2474] combine_rows_per_src=[13146, 8786, 9564, 10386, 9256, 7620, 12082, 10172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272596992 dispatch_meta_bytes=321136 combine_payload_bytes=272596992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272918128 ingress_bytes=307200000 link_bytes=307200000 token_bytes=545515120 weight_movement_bytes=34603008 total_nvlink_bytes=580118128 weight_to_token_ratio=0.063432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155968868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264372224 dispatch_meta_bytes=335952 combine_payload_bytes=264372224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264708176 ingress_bytes=333578240 link_bytes=333578240 token_bytes=529080400 weight_movement_bytes=69206016 total_nvlink_bytes=598286416 weight_to_token_ratio=0.130804 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255228062e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=246448128 dispatch_meta_bytes=299448 combine_payload_bytes=246448128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=246747576 ingress_bytes=350257152 link_bytes=350257152 token_bytes=493195704 weight_movement_bytes=103809024 total_nvlink_bytes=597004728 weight_to_token_ratio=0.210482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317989465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=304578560 dispatch_meta_bytes=378432 combine_payload_bytes=304578560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=304956992 ingress_bytes=477593600 link_bytes=477593600 token_bytes=609535552 weight_movement_bytes=173015040 total_nvlink_bytes=782550592 weight_to_token_ratio=0.283847 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.797146267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=348864 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291918528 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583488192 weight_movement_bytes=34603008 total_nvlink_bytes=618091200 weight_to_token_ratio=0.059304 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276807680 dispatch_meta_bytes=323376 combine_payload_bytes=276807680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277131056 ingress_bytes=311410688 link_bytes=311410688 token_bytes=553938736 weight_movement_bytes=34603008 total_nvlink_bytes=588541744 weight_to_token_ratio=0.062467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171813348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=257654784 dispatch_meta_bytes=310144 combine_payload_bytes=257654784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257964928 ingress_bytes=326860800 link_bytes=326860800 token_bytes=515619712 weight_movement_bytes=69206016 total_nvlink_bytes=584825728 weight_to_token_ratio=0.134219 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229950876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=289284096 dispatch_meta_bytes=340496 combine_payload_bytes=289284096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=289624592 ingress_bytes=358490112 link_bytes=358490112 token_bytes=578908688 weight_movement_bytes=69206016 total_nvlink_bytes=648114704 weight_to_token_ratio=0.119546 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348969431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77746 payload_rows_remote=75612 dup_rows=20558 dup_rows_remote=20538 dedup_ratio=0.209127 router_tokens_per_expert=[0, 0, 3, 2, 2040, 1038, 980, 119, 420, 89, 316, 1, 248, 0, 6603, 3, 3249, 1821, 134, 0, 1995, 272, 0, 67, 757, 1025, 743, 535, 1060, 637, 435, 9, 0, 7, 89, 33, 2589, 44, 8, 0, 1254, 5637, 16, 835, 495, 0, 1, 1657, 1, 0, 13, 567, 20, 1393, 0, 1, 2, 2, 670, 2, 2716, 5679, 5, 815] payload_rows_per_dst=[14940, 2134, 11018, 9200, 13082, 7220, 12564, 7588] dup_rows_per_dst=[6630, 20, 4058, 1202, 3732, 1296, 2784, 836] combine_rows_per_src=[14940, 2134, 11018, 9200, 13082, 7220, 12564, 7588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79634 payload_rows_remote=68704 dup_rows=18670 dup_rows_remote=15452 dedup_ratio=0.189921 router_tokens_per_expert=[0, 174, 79, 191, 0, 77, 437, 1748, 12, 1725, 231, 1576, 436, 355, 0, 2, 4349, 817, 5300, 53, 1, 1, 5563, 889, 190, 591, 2874, 3082, 3, 274, 300, 0, 39, 526, 456, 154, 4034, 196, 209, 95, 1516, 0, 1, 31, 426, 2089, 90, 0, 945, 4, 251, 405, 5, 0, 135, 0, 3763, 178, 14, 4, 1, 1305, 0, 950] payload_rows_per_dst=[5208, 8016, 12374, 6050, 12452, 13158, 11446, 10930] dup_rows_per_dst=[204, 658, 2274, 696, 5130, 3846, 2644, 3218] combine_rows_per_src=[5208, 8016, 12374, 6050, 12452, 13158, 11446, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79884 payload_rows_remote=68110 dup_rows=18420 dup_rows_remote=15502 dedup_ratio=0.187378 router_tokens_per_expert=[1, 171, 78, 195, 1, 74, 412, 1941, 31, 1747, 294, 1459, 403, 356, 0, 0, 4380, 955, 5443, 73, 2, 0, 5390, 902, 206, 640, 2735, 2582, 0, 302, 271, 0, 107, 512, 583, 385, 3476, 236, 151, 66, 1446, 0, 3, 40, 538, 2124, 113, 1, 880, 8, 232, 635, 9, 0, 137, 2, 3956, 183, 18, 8, 1, 1302, 1, 955] payload_rows_per_dst=[5502, 7848, 12154, 7632, 11316, 13300, 11774, 10358] dup_rows_per_dst=[244, 732, 2490, 1474, 4082, 3990, 2918, 2490] combine_rows_per_src=[5502, 7848, 12154, 7632, 11316, 13300, 11774, 10358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81314 payload_rows_remote=70622 dup_rows=16990 dup_rows_remote=13168 dedup_ratio=0.172831 router_tokens_per_expert=[0, 0, 7, 0, 2429, 1284, 983, 60, 455, 87, 449, 0, 248, 0, 6641, 2, 2937, 1779, 180, 1, 1992, 271, 0, 97, 721, 1075, 667, 602, 873, 516, 458, 4, 0, 5, 81, 39, 2526, 88, 12, 0, 1089, 5520, 30, 835, 317, 0, 0, 1990, 0, 0, 1, 496, 27, 1184, 0, 0, 4, 0, 613, 2, 2670, 5933, 2, 870] payload_rows_per_dst=[7856, 13940, 10692, 8506, 12966, 7136, 12696, 7522] dup_rows_per_dst=[1916, 1578, 3822, 1326, 3576, 1386, 2586, 800] combine_rows_per_src=[7856, 13940, 10692, 8506, 12966, 7136, 12696, 7522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81548 payload_rows_remote=68806 dup_rows=16756 dup_rows_remote=13518 dedup_ratio=0.170451 router_tokens_per_expert=[0, 0, 9, 0, 2260, 1181, 1010, 96, 505, 77, 442, 0, 252, 0, 6405, 0, 2895, 1877, 80, 0, 2168, 298, 0, 111, 782, 985, 671, 528, 1067, 648, 445, 13, 0, 5, 75, 26, 2446, 38, 15, 0, 967, 5385, 35, 725, 531, 0, 0, 1772, 0, 0, 7, 544, 54, 1332, 0, 0, 4, 0, 629, 0, 2758, 6100, 6, 893] payload_rows_per_dst=[7336, 13640, 10430, 9492, 12742, 6962, 13284, 7662] dup_rows_per_dst=[1776, 1722, 3718, 1496, 3238, 1098, 2790, 918] combine_rows_per_src=[7336, 13640, 10430, 9492, 12742, 6962, 13284, 7662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80602 payload_rows_remote=68400 dup_rows=17702 dup_rows_remote=12880 dedup_ratio=0.180074 router_tokens_per_expert=[0, 0, 11, 0, 2163, 1295, 934, 132, 421, 80, 348, 0, 255, 0, 6679, 7, 3040, 1690, 162, 0, 2175, 314, 0, 51, 771, 844, 726, 533, 1214, 711, 436, 42, 0, 1, 87, 26, 2004, 23, 19, 1, 1048, 5602, 32, 954, 532, 0, 0, 1771, 0, 0, 39, 516, 34, 1017, 0, 0, 2, 0, 672, 0, 2431, 6340, 3, 964] payload_rows_per_dst=[7212, 14284, 7414, 12658, 6578, 12202, 10796, 9458] dup_rows_per_dst=[1858, 1296, 1370, 3976, 598, 4822, 1814, 1968] combine_rows_per_src=[7212, 14284, 7414, 12658, 6578, 12202, 10796, 9458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=309706752 dispatch_meta_bytes=384600 combine_payload_bytes=309706752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310091352 ingress_bytes=378912768 link_bytes=378912768 token_bytes=619798104 weight_movement_bytes=69206016 total_nvlink_bytes=689004120 weight_to_token_ratio=0.111659 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425818241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75862 payload_rows_remote=63072 dup_rows=22442 dup_rows_remote=18988 dedup_ratio=0.228292 router_tokens_per_expert=[0, 229, 66, 156, 0, 131, 388, 1952, 32, 1626, 253, 1629, 475, 324, 0, 0, 4506, 899, 5356, 62, 3, 0, 5200, 784, 211, 630, 2792, 2838, 5, 433, 273, 0, 53, 519, 540, 280, 3439, 199, 224, 101, 1570, 0, 3, 22, 423, 1985, 107, 1, 984, 7, 192, 588, 6, 0, 220, 0, 3815, 177, 13, 13, 0, 1485, 0, 933] payload_rows_per_dst=[12790, 11940, 14398, 11008, 9080, 7358, 3842, 5446] dup_rows_per_dst=[3454, 5750, 6832, 3356, 1630, 864, 152, 404] combine_rows_per_src=[12790, 11940, 14398, 11008, 9080, 7358, 3842, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=336624 combine_payload_bytes=281411584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281748208 ingress_bytes=350617600 link_bytes=350617600 token_bytes=563159792 weight_movement_bytes=69206016 total_nvlink_bytes=632365808 weight_to_token_ratio=0.122889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319345802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=334448 combine_payload_bytes=278978560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279313008 ingress_bytes=313581568 link_bytes=313581568 token_bytes=558291568 weight_movement_bytes=34603008 total_nvlink_bytes=592894576 weight_to_token_ratio=0.061980 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179982195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81580 payload_rows_remote=72342 dup_rows=16724 dup_rows_remote=15320 dedup_ratio=0.170125 router_tokens_per_expert=[0, 0, 11, 0, 2181, 1194, 977, 115, 459, 91, 401, 0, 246, 0, 6484, 0, 3269, 2029, 83, 0, 1891, 256, 0, 140, 752, 1149, 698, 605, 1071, 579, 466, 1, 0, 11, 83, 21, 2261, 56, 14, 1, 886, 5209, 33, 795, 553, 0, 0, 1725, 0, 0, 29, 520, 43, 1391, 0, 0, 3, 0, 616, 0, 2742, 6192, 3, 817] payload_rows_per_dst=[7244, 13896, 11120, 9238, 12312, 6764, 13332, 7674] dup_rows_per_dst=[1712, 1466, 4216, 1404, 3000, 1220, 3018, 688] combine_rows_per_src=[7244, 13896, 11120, 9238, 12312, 6764, 13332, 7674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289267712 dispatch_meta_bytes=335160 combine_payload_bytes=289267712 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=289602872 ingress_bytes=427679744 link_bytes=427679744 token_bytes=578870584 weight_movement_bytes=138412032 total_nvlink_bytes=717282616 weight_to_token_ratio=0.239107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609324446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281829376 dispatch_meta_bytes=329296 combine_payload_bytes=281829376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282158672 ingress_bytes=316432384 link_bytes=316432384 token_bytes=563988048 weight_movement_bytes=34603008 total_nvlink_bytes=598591056 weight_to_token_ratio=0.061354 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190709586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280166400 dispatch_meta_bytes=325120 combine_payload_bytes=280166400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280491520 ingress_bytes=349372416 link_bytes=349372416 token_bytes=560657920 weight_movement_bytes=69206016 total_nvlink_bytes=629863936 weight_to_token_ratio=0.123437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314660275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258342912 dispatch_meta_bytes=328240 combine_payload_bytes=258342912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258671152 ingress_bytes=292945920 link_bytes=292945920 token_bytes=517014064 weight_movement_bytes=34603008 total_nvlink_bytes=551617072 weight_to_token_ratio=0.066929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102331913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296312832 dispatch_meta_bytes=350648 combine_payload_bytes=296312832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296663480 ingress_bytes=330915840 link_bytes=330915840 token_bytes=592976312 weight_movement_bytes=34603008 total_nvlink_bytes=627579320 weight_to_token_ratio=0.058355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245209665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84348 payload_rows_remote=70032 dup_rows=13956 dup_rows_remote=13518 dedup_ratio=0.141968 router_tokens_per_expert=[0, 557, 4123, 0, 0, 0, 0, 43, 0, 2, 0, 93, 129, 0, 0, 0, 0, 3236, 0, 3714, 2, 215, 5635, 3, 4, 0, 337, 11, 0, 1942, 0, 0, 1, 0, 2, 1629, 1664, 0, 3659, 319, 0, 1483, 1, 7492, 84, 0, 127, 0, 0, 1152, 7153, 0, 0, 437, 3277, 447, 0, 0, 1, 2, 176, 0, 0, 0] payload_rows_per_dst=[13108, 14316, 11906, 15132, 6366, 6608, 5446, 11466] dup_rows_per_dst=[3656, 438, 2434, 4440, 864, 598, 1364, 162] combine_rows_per_src=[13108, 14316, 11906, 15132, 6366, 6608, 5446, 11466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76858 payload_rows_remote=61760 dup_rows=21446 dup_rows_remote=12812 dedup_ratio=0.218160 router_tokens_per_expert=[0, 0, 10, 3, 2287, 979, 1025, 68, 500, 79, 377, 0, 243, 0, 6760, 1, 3208, 2106, 44, 1, 1961, 283, 0, 46, 686, 847, 691, 569, 1355, 631, 429, 8, 0, 5, 72, 33, 2339, 29, 22, 3, 941, 5534, 15, 771, 436, 3, 1, 1813, 1, 2, 53, 516, 41, 1107, 0, 2, 7, 0, 616, 0, 2488, 6182, 33, 890] payload_rows_per_dst=[7368, 14382, 6982, 10752, 4968, 13896, 3412, 15098] dup_rows_per_dst=[1376, 1538, 1272, 2796, 38, 5760, 32, 8634] combine_rows_per_src=[7368, 14382, 6982, 10752, 4968, 13896, 3412, 15098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77970 payload_rows_remote=74604 dup_rows=20334 dup_rows_remote=20302 dedup_ratio=0.206848 router_tokens_per_expert=[2, 2, 8, 1, 2201, 1319, 977, 62, 408, 90, 444, 0, 238, 2, 6367, 2, 3128, 1784, 179, 0, 2073, 288, 0, 90, 669, 873, 708, 570, 934, 598, 421, 0, 0, 9, 83, 50, 2455, 35, 27, 0, 1124, 5446, 42, 841, 638, 0, 0, 1845, 1, 1, 30, 497, 45, 1122, 3, 0, 15, 2, 620, 0, 2661, 6231, 3, 888] payload_rows_per_dst=[7404, 13650, 7486, 12170, 5290, 13792, 3366, 14812] dup_rows_per_dst=[1740, 1452, 1342, 3632, 28, 6080, 32, 6028] combine_rows_per_src=[7404, 13650, 7486, 12170, 5290, 13792, 3366, 14812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83966 payload_rows_remote=72712 dup_rows=14338 dup_rows_remote=12042 dedup_ratio=0.145854 router_tokens_per_expert=[0, 476, 3757, 0, 0, 2, 0, 47, 0, 1, 0, 61, 72, 0, 0, 0, 1, 3188, 0, 3314, 3, 269, 5507, 0, 27, 0, 369, 52, 0, 1989, 0, 0, 0, 0, 0, 1752, 1811, 1, 3779, 178, 0, 1580, 0, 7508, 58, 0, 150, 0, 0, 1550, 7117, 0, 0, 476, 3424, 438, 0, 0, 0, 1, 194, 0, 0, 0] payload_rows_per_dst=[12918, 14238, 11254, 15232, 6508, 3460, 9126, 11230] dup_rows_per_dst=[3204, 264, 2296, 4658, 976, 116, 2650, 174] combine_rows_per_src=[12918, 14238, 11254, 15232, 6508, 3460, 9126, 11230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85700 payload_rows_remote=74814 dup_rows=12604 dup_rows_remote=9284 dedup_ratio=0.128215 router_tokens_per_expert=[0, 531, 3846, 0, 0, 0, 0, 30, 0, 1, 0, 78, 96, 0, 0, 0, 0, 3304, 0, 3518, 1, 264, 5780, 7, 11, 0, 465, 9, 0, 1888, 0, 0, 0, 0, 0, 1724, 1540, 0, 3552, 287, 0, 1491, 0, 7543, 68, 0, 117, 0, 0, 1443, 7056, 0, 0, 497, 3365, 470, 0, 0, 0, 1, 169, 0, 0, 0] payload_rows_per_dst=[8226, 14116, 11700, 4694, 10886, 15416, 8928, 11734] dup_rows_per_dst=[588, 346, 2488, 52, 3320, 3022, 2622, 166] combine_rows_per_src=[8226, 14116, 11700, 4694, 10886, 15416, 8928, 11734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85920 payload_rows_remote=70498 dup_rows=12384 dup_rows_remote=9586 dedup_ratio=0.125977 router_tokens_per_expert=[0, 574, 3928, 0, 0, 2, 0, 38, 0, 0, 0, 94, 93, 0, 0, 0, 0, 3319, 0, 3030, 0, 236, 5778, 1, 8, 0, 346, 15, 0, 2047, 0, 0, 0, 0, 0, 1665, 1920, 0, 3438, 309, 0, 1416, 0, 7536, 86, 0, 72, 0, 0, 1374, 7010, 0, 0, 522, 3632, 476, 0, 0, 0, 1, 185, 0, 1, 0] payload_rows_per_dst=[8474, 14020, 11180, 4806, 11050, 15422, 9204, 11764] dup_rows_per_dst=[610, 374, 1992, 26, 3614, 2798, 2804, 166] combine_rows_per_src=[8474, 14020, 11180, 4806, 11050, 15422, 9204, 11764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286851072 dispatch_meta_bytes=334200 combine_payload_bytes=286851072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287185272 ingress_bytes=321454080 link_bytes=321454080 token_bytes=574036344 weight_movement_bytes=34603008 total_nvlink_bytes=608639352 weight_to_token_ratio=0.060280 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209605824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=252968960 dispatch_meta_bytes=298288 combine_payload_bytes=252968960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253267248 ingress_bytes=322174976 link_bytes=322174976 token_bytes=506236208 weight_movement_bytes=69206016 total_nvlink_bytes=575442224 weight_to_token_ratio=0.136707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212318497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78342 payload_rows_remote=63506 dup_rows=19962 dup_rows_remote=12862 dedup_ratio=0.203064 router_tokens_per_expert=[0, 0, 13, 0, 2194, 1238, 864, 122, 454, 88, 445, 0, 256, 0, 6537, 1, 3003, 1661, 182, 1, 1866, 296, 0, 82, 784, 1023, 667, 525, 1248, 544, 433, 4, 0, 8, 80, 33, 2367, 85, 20, 1, 1071, 5484, 14, 884, 494, 0, 0, 1872, 0, 0, 56, 496, 33, 1326, 0, 0, 1, 0, 747, 0, 2679, 5988, 12, 870] payload_rows_per_dst=[14836, 3186, 10138, 9134, 12894, 7352, 12988, 7814] dup_rows_per_dst=[7100, 120, 3226, 1322, 3262, 1318, 2810, 804] combine_rows_per_src=[14836, 3186, 10138, 9134, 12894, 7352, 12988, 7814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=305577984 dispatch_meta_bytes=379624 combine_payload_bytes=305577984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305957608 ingress_bytes=340180992 link_bytes=340180992 token_bytes=611535592 weight_movement_bytes=34603008 total_nvlink_bytes=646138600 weight_to_token_ratio=0.056584 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280073686e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83996 payload_rows_remote=69358 dup_rows=14308 dup_rows_remote=9634 dedup_ratio=0.145549 router_tokens_per_expert=[0, 510, 3515, 0, 0, 1, 0, 41, 0, 0, 0, 56, 73, 0, 0, 0, 0, 3351, 0, 3446, 0, 255, 5625, 2, 9, 0, 403, 68, 0, 2076, 0, 0, 0, 0, 0, 1658, 1858, 0, 3709, 235, 0, 1475, 0, 7572, 52, 0, 138, 0, 0, 1342, 7012, 0, 0, 462, 3585, 457, 0, 0, 0, 0, 166, 0, 0, 0] payload_rows_per_dst=[11958, 14030, 11744, 14638, 7078, 3988, 9122, 11438] dup_rows_per_dst=[2604, 252, 2364, 4674, 1414, 286, 2570, 144] combine_rows_per_src=[11958, 14030, 11744, 14638, 7078, 3988, 9122, 11438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=297828352 dispatch_meta_bytes=339016 combine_payload_bytes=297828352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=298167368 ingress_bytes=436240384 link_bytes=436240384 token_bytes=595995720 weight_movement_bytes=138412032 total_nvlink_bytes=734407752 weight_to_token_ratio=0.232237 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641537445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306438144 dispatch_meta_bytes=336392 combine_payload_bytes=306438144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306774536 ingress_bytes=375644160 link_bytes=375644160 token_bytes=613212680 weight_movement_bytes=69206016 total_nvlink_bytes=682418696 weight_to_token_ratio=0.112858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413518732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=288759808 dispatch_meta_bytes=320336 combine_payload_bytes=288759808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289080144 ingress_bytes=357965824 link_bytes=357965824 token_bytes=577839952 weight_movement_bytes=69206016 total_nvlink_bytes=647045968 weight_to_token_ratio=0.119767 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346996577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=260120576 dispatch_meta_bytes=305472 combine_payload_bytes=260120576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260426048 ingress_bytes=294723584 link_bytes=294723584 token_bytes=520546624 weight_movement_bytes=34603008 total_nvlink_bytes=555149632 weight_to_token_ratio=0.066474 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109021119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=284090368 dispatch_meta_bytes=315968 combine_payload_bytes=284090368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284406336 ingress_bytes=318693376 link_bytes=318693376 token_bytes=568496704 weight_movement_bytes=34603008 total_nvlink_bytes=603099712 weight_to_token_ratio=0.060868 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199217517e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72130 payload_rows_remote=56678 dup_rows=26174 dup_rows_remote=18770 dedup_ratio=0.266256 router_tokens_per_expert=[88, 7289, 87, 3, 5148, 2495, 4803, 0, 0, 2, 0, 1073, 170, 0, 0, 2894, 321, 1260, 814, 70, 4, 674, 5657, 0, 2, 1789, 323, 0, 0, 0, 479, 0, 688, 1, 2, 0, 1, 2, 257, 567, 5910, 0, 0, 2, 1042, 1, 86, 44, 0, 3, 1714, 86, 0, 0, 1, 726, 0, 1, 86, 0, 0, 2294, 3, 190] payload_rows_per_dst=[14892, 15452, 14548, 5072, 2886, 9768, 4846, 4666] dup_rows_per_dst=[10356, 7404, 6402, 114, 150, 1052, 214, 482] combine_rows_per_src=[14892, 15452, 14548, 5072, 2886, 9768, 4846, 4666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71838 payload_rows_remote=58714 dup_rows=26466 dup_rows_remote=22594 dedup_ratio=0.269226 router_tokens_per_expert=[0, 484, 3534, 1, 1, 22, 2, 24, 0, 1, 0, 83, 75, 0, 0, 0, 0, 3157, 0, 3800, 0, 198, 5849, 6, 5, 0, 374, 20, 0, 2033, 2, 0, 0, 0, 0, 1671, 1760, 3, 3672, 238, 0, 1616, 0, 7619, 54, 1, 129, 0, 0, 968, 7016, 1, 4, 450, 3596, 475, 0, 0, 0, 4, 203, 0, 0, 1] payload_rows_per_dst=[7544, 318, 1784, 4830, 13458, 15542, 15238, 13124] dup_rows_per_dst=[592, 0, 56, 38, 8830, 3296, 9782, 3872] combine_rows_per_src=[7544, 318, 1784, 4830, 13458, 15542, 15238, 13124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82990 payload_rows_remote=72274 dup_rows=15314 dup_rows_remote=10746 dedup_ratio=0.155782 router_tokens_per_expert=[0, 550, 3732, 4, 0, 6, 1, 67, 1, 2, 1, 57, 70, 0, 3, 0, 0, 3216, 0, 3372, 0, 249, 5407, 7, 15, 1, 432, 75, 0, 2122, 0, 3, 1, 0, 4, 1609, 1744, 0, 3890, 194, 3, 1434, 0, 7473, 49, 0, 139, 1, 1, 1453, 7000, 10, 0, 465, 3654, 464, 0, 2, 3, 0, 165, 0, 0, 1] payload_rows_per_dst=[8100, 10874, 10350, 5110, 11562, 15268, 10716, 11010] dup_rows_per_dst=[620, 204, 1934, 186, 4726, 2930, 4568, 146] combine_rows_per_src=[8100, 10874, 10350, 5110, 11562, 15268, 10716, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=232153088 dispatch_meta_bytes=301792 combine_payload_bytes=232153088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=232454880 ingress_bytes=266756096 link_bytes=266756096 token_bytes=464607968 weight_movement_bytes=34603008 total_nvlink_bytes=499210976 weight_to_token_ratio=0.074478 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.003781714e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79696 payload_rows_remote=65666 dup_rows=18608 dup_rows_remote=14504 dedup_ratio=0.189290 router_tokens_per_expert=[71, 7342, 70, 0, 5080, 2217, 4688, 24, 0, 0, 0, 890, 307, 0, 2, 2714, 335, 1150, 796, 64, 1, 1026, 5695, 0, 0, 1483, 336, 0, 0, 0, 458, 0, 1008, 0, 0, 0, 0, 0, 260, 678, 6040, 0, 0, 0, 983, 0, 71, 35, 0, 0, 2179, 60, 0, 0, 0, 644, 0, 0, 71, 0, 3, 2197, 0, 174] payload_rows_per_dst=[6714, 15566, 14030, 10706, 3546, 12966, 5534, 10634] dup_rows_per_dst=[1838, 6944, 4104, 1412, 346, 1292, 232, 2440] combine_rows_per_src=[6714, 15566, 14030, 10706, 3546, 12966, 5534, 10634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81040 payload_rows_remote=66154 dup_rows=17264 dup_rows_remote=13900 dedup_ratio=0.175618 router_tokens_per_expert=[72, 7240, 72, 0, 5205, 2123, 4986, 0, 0, 0, 0, 889, 179, 0, 0, 2703, 349, 1107, 756, 67, 0, 841, 5769, 0, 0, 1869, 348, 0, 0, 0, 449, 0, 916, 0, 0, 0, 0, 0, 238, 731, 5806, 0, 0, 0, 1213, 0, 72, 31, 0, 0, 1890, 74, 0, 0, 0, 731, 0, 0, 72, 0, 2, 2152, 0, 200] payload_rows_per_dst=[4534, 6258, 4406, 13122, 14886, 12728, 12814, 12292] dup_rows_per_dst=[0, 1284, 1834, 2182, 3364, 1516, 4114, 2970] combine_rows_per_src=[4534, 6258, 4406, 13122, 14886, 12728, 12814, 12292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80922 payload_rows_remote=68482 dup_rows=17382 dup_rows_remote=16150 dedup_ratio=0.176819 router_tokens_per_expert=[79, 7249, 76, 4, 5229, 2201, 5000, 0, 0, 0, 0, 1030, 241, 0, 0, 2776, 337, 1186, 835, 78, 1, 670, 5813, 0, 0, 1591, 338, 0, 0, 0, 431, 0, 1125, 0, 0, 0, 1, 0, 213, 805, 5649, 0, 0, 0, 1086, 1, 76, 24, 0, 0, 1907, 92, 0, 0, 0, 743, 0, 0, 76, 0, 3, 1998, 0, 188] payload_rows_per_dst=[4720, 6494, 4542, 12742, 14960, 12440, 12794, 12230] dup_rows_per_dst=[0, 1600, 1672, 1978, 3826, 1232, 4316, 2758] combine_rows_per_src=[4720, 6494, 4542, 12742, 14960, 12440, 12794, 12230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=240492544 dispatch_meta_bytes=325232 combine_payload_bytes=240492544 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=240817776 ingress_bytes=309698560 link_bytes=309698560 token_bytes=481310320 weight_movement_bytes=69206016 total_nvlink_bytes=550516336 weight_to_token_ratio=0.143787 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165370749e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84586 payload_rows_remote=71716 dup_rows=13718 dup_rows_remote=10486 dedup_ratio=0.139547 router_tokens_per_expert=[0, 560, 3836, 0, 0, 0, 0, 55, 0, 1, 0, 61, 83, 0, 0, 0, 0, 2907, 0, 3543, 0, 250, 5675, 6, 10, 0, 466, 10, 0, 1825, 0, 0, 0, 0, 0, 1793, 1709, 0, 3600, 324, 0, 1712, 0, 7583, 60, 0, 144, 0, 0, 1142, 7112, 0, 0, 473, 3566, 467, 0, 0, 0, 1, 178, 0, 0, 0] payload_rows_per_dst=[12870, 14224, 11924, 15354, 6644, 8556, 3444, 11570] dup_rows_per_dst=[3232, 290, 2898, 4434, 1008, 998, 720, 138] combine_rows_per_src=[12870, 14224, 11924, 15354, 6644, 8556, 3444, 11570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=296034304 dispatch_meta_bytes=332080 combine_payload_bytes=296034304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=296366384 ingress_bytes=399843328 link_bytes=399843328 token_bytes=592400688 weight_movement_bytes=103809024 total_nvlink_bytes=696209712 weight_to_token_ratio=0.175234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504578253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81502 payload_rows_remote=68316 dup_rows=16802 dup_rows_remote=14382 dedup_ratio=0.170919 router_tokens_per_expert=[82, 7319, 81, 2, 4836, 2365, 5139, 0, 0, 0, 0, 1047, 251, 0, 0, 2606, 371, 1274, 871, 86, 6, 513, 5715, 0, 0, 1808, 373, 0, 0, 0, 483, 0, 756, 0, 0, 0, 0, 0, 250, 553, 5916, 0, 0, 0, 1139, 0, 80, 37, 0, 0, 2056, 52, 0, 0, 0, 682, 0, 0, 81, 0, 1, 2115, 0, 206] payload_rows_per_dst=[5058, 8122, 6982, 13186, 12666, 12840, 10858, 11790] dup_rows_per_dst=[2, 2498, 2080, 2420, 2278, 1504, 3332, 2688] combine_rows_per_src=[5058, 8122, 6982, 13186, 12666, 12840, 10858, 11790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=268967936 dispatch_meta_bytes=320680 combine_payload_bytes=268967936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269288616 ingress_bytes=338173952 link_bytes=338173952 token_bytes=538256552 weight_movement_bytes=69206016 total_nvlink_bytes=607462568 weight_to_token_ratio=0.128574 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272521356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270966784 dispatch_meta_bytes=320216 combine_payload_bytes=270966784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271287000 ingress_bytes=305569792 link_bytes=305569792 token_bytes=542253784 weight_movement_bytes=34603008 total_nvlink_bytes=576856792 weight_to_token_ratio=0.063813 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149834527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280502272 dispatch_meta_bytes=338528 combine_payload_bytes=280502272 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=280840800 ingress_bytes=349708288 link_bytes=349708288 token_bytes=561343072 weight_movement_bytes=69206016 total_nvlink_bytes=630549088 weight_to_token_ratio=0.123286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315924134e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=293748736 dispatch_meta_bytes=328808 combine_payload_bytes=293748736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294077544 ingress_bytes=328351744 link_bytes=328351744 token_bytes=587826280 weight_movement_bytes=34603008 total_nvlink_bytes=622429288 weight_to_token_ratio=0.058866 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235561178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279822336 dispatch_meta_bytes=330792 combine_payload_bytes=279822336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280153128 ingress_bytes=314425344 link_bytes=314425344 token_bytes=559975464 weight_movement_bytes=34603008 total_nvlink_bytes=594578472 weight_to_token_ratio=0.061794 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183157256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77334 payload_rows_remote=60956 dup_rows=20970 dup_rows_remote=9954 dedup_ratio=0.213318 router_tokens_per_expert=[650, 0, 55, 0, 0, 0, 2, 259, 0, 1, 5642, 0, 2374, 7869, 0, 4130, 0, 3, 1, 790, 0, 0, 439, 122, 0, 54, 299, 1, 1, 86, 0, 0, 0, 0, 5408, 0, 2, 2, 0, 0, 0, 7446, 0, 0, 0, 138, 664, 547, 0, 0, 0, 94, 2942, 1015, 6, 1, 3, 1, 1, 0, 1, 1, 8102, 0] payload_rows_per_dst=[12774, 16378, 10246, 716, 14034, 15542, 7634, 10] dup_rows_per_dst=[442, 11016, 724, 166, 6088, 2048, 482, 4] combine_rows_per_src=[12774, 16378, 10246, 716, 14034, 15542, 7634, 10] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=249675776 dispatch_meta_bytes=283640 combine_payload_bytes=249675776 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=249959416 ingress_bytes=422690816 link_bytes=422690816 token_bytes=499635192 weight_movement_bytes=173015040 total_nvlink_bytes=672650232 weight_to_token_ratio=0.346283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.590551511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81998 payload_rows_remote=69414 dup_rows=16306 dup_rows_remote=13390 dedup_ratio=0.165873 router_tokens_per_expert=[70, 7377, 76, 5, 5368, 1805, 4823, 0, 0, 0, 1, 982, 154, 0, 0, 2986, 351, 1242, 901, 52, 1, 815, 5879, 0, 2, 1808, 356, 1, 0, 0, 386, 0, 736, 0, 0, 3, 1, 1, 237, 422, 6192, 0, 1, 0, 1057, 8, 75, 45, 0, 0, 1842, 56, 0, 0, 1, 652, 0, 0, 75, 0, 2, 2129, 1, 175] payload_rows_per_dst=[3908, 6762, 4980, 12732, 14986, 13192, 12854, 12584] dup_rows_per_dst=[4, 1484, 1744, 2020, 2568, 1564, 4006, 2916] combine_rows_per_src=[3908, 6762, 4980, 12732, 14986, 13192, 12854, 12584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80934 payload_rows_remote=68282 dup_rows=17370 dup_rows_remote=13098 dedup_ratio=0.176697 router_tokens_per_expert=[84, 7323, 86, 0, 4918, 2063, 4989, 6, 0, 0, 3, 1002, 219, 0, 0, 2747, 352, 1234, 714, 88, 2, 731, 5648, 0, 0, 1892, 350, 0, 0, 2, 468, 2, 797, 19, 0, 0, 2, 0, 249, 726, 5860, 0, 0, 0, 1066, 5, 89, 30, 0, 1, 2106, 55, 0, 0, 2, 650, 0, 1, 91, 0, 4, 2295, 4, 177] payload_rows_per_dst=[4478, 6502, 4492, 13068, 15098, 12626, 12652, 12018] dup_rows_per_dst=[0, 1440, 1750, 2338, 3134, 1474, 4272, 2962] combine_rows_per_src=[4478, 6502, 4492, 13068, 15098, 12626, 12652, 12018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91756 payload_rows_remote=81388 dup_rows=6548 dup_rows_remote=5860 dedup_ratio=0.066610 router_tokens_per_expert=[686, 0, 35, 0, 0, 0, 0, 344, 0, 0, 5753, 0, 2294, 7835, 0, 4260, 0, 0, 0, 680, 0, 0, 426, 162, 0, 71, 137, 0, 0, 70, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 1, 7390, 0, 0, 0, 130, 609, 427, 0, 0, 0, 61, 2783, 1394, 2, 0, 0, 0, 0, 0, 0, 0, 8121, 0] payload_rows_per_dst=[13248, 5002, 10368, 15200, 12132, 15220, 7822, 12764] dup_rows_per_dst=[388, 174, 688, 438, 2308, 1894, 658, 0] combine_rows_per_src=[13248, 5002, 10368, 15200, 12132, 15220, 7822, 12764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94470 payload_rows_remote=84098 dup_rows=3834 dup_rows_remote=3834 dedup_ratio=0.039001 router_tokens_per_expert=[694, 0, 16, 0, 0, 0, 0, 219, 0, 0, 6192, 0, 2370, 7843, 0, 4620, 0, 0, 0, 770, 0, 0, 433, 161, 0, 73, 82, 0, 0, 72, 0, 0, 0, 0, 5186, 0, 0, 0, 0, 0, 0, 7259, 0, 0, 0, 120, 580, 433, 0, 0, 0, 77, 2674, 1156, 2, 0, 0, 0, 0, 0, 0, 0, 8120, 0] payload_rows_per_dst=[13818, 4740, 11152, 15688, 10372, 15062, 7398, 16240] dup_rows_per_dst=[424, 0, 816, 452, 0, 1722, 420, 0] combine_rows_per_src=[13818, 4740, 11152, 15688, 10372, 15062, 7398, 16240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91322 payload_rows_remote=83504 dup_rows=6982 dup_rows_remote=6110 dedup_ratio=0.071025 router_tokens_per_expert=[680, 0, 34, 0, 0, 0, 0, 197, 0, 0, 6269, 0, 2225, 7838, 0, 4320, 0, 0, 0, 654, 0, 0, 448, 114, 0, 56, 138, 0, 0, 76, 0, 0, 0, 0, 5092, 0, 0, 0, 0, 0, 0, 7343, 0, 0, 0, 118, 669, 495, 0, 0, 0, 94, 3014, 1158, 4, 0, 0, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[13934, 4450, 10412, 15694, 10184, 7818, 12598, 16232] dup_rows_per_dst=[426, 0, 660, 522, 0, 872, 4502, 0] combine_rows_per_src=[13934, 4450, 10412, 15694, 10184, 7818, 12598, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59190 payload_rows_remote=43326 dup_rows=39114 dup_rows_remote=13178 dedup_ratio=0.397888 router_tokens_per_expert=[80, 7264, 78, 0, 5620, 1907, 5137, 13, 0, 0, 0, 919, 164, 0, 0, 2895, 307, 1090, 803, 86, 5, 721, 5561, 0, 0, 1831, 307, 0, 0, 0, 468, 0, 838, 1, 0, 0, 0, 0, 271, 695, 5866, 0, 0, 0, 952, 0, 78, 25, 0, 0, 1942, 62, 0, 0, 0, 729, 0, 0, 78, 0, 3, 2174, 0, 182] payload_rows_per_dst=[15864, 8182, 15036, 5058, 3246, 2106, 5260, 4438] dup_rows_per_dst=[25936, 2312, 9702, 154, 364, 4, 206, 436] combine_rows_per_src=[15864, 8182, 15036, 5058, 3246, 2106, 5260, 4438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94356 payload_rows_remote=78512 dup_rows=3948 dup_rows_remote=3392 dedup_ratio=0.040161 router_tokens_per_expert=[659, 0, 56, 0, 0, 0, 0, 317, 0, 0, 5743, 0, 2258, 7815, 0, 4705, 0, 0, 0, 680, 0, 0, 483, 130, 0, 67, 237, 0, 0, 81, 0, 0, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 7080, 0, 0, 0, 125, 718, 462, 0, 1, 5, 53, 2774, 982, 2, 0, 0, 0, 0, 0, 0, 0, 8111, 0] payload_rows_per_dst=[13224, 4516, 11236, 15844, 11216, 14830, 7268, 16222] dup_rows_per_dst=[326, 0, 760, 556, 0, 1940, 366, 0] combine_rows_per_src=[13224, 4516, 11236, 15844, 11216, 14830, 7268, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85558 payload_rows_remote=69994 dup_rows=12746 dup_rows_remote=5442 dedup_ratio=0.129659 router_tokens_per_expert=[248, 1, 3, 0, 249, 0, 0, 1, 0, 559, 1172, 2427, 1, 1, 4, 4, 2203, 1524, 0, 0, 7607, 0, 0, 1353, 0, 0, 0, 7266, 1, 322, 120, 0, 1, 1, 0, 0, 1285, 590, 343, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 7, 515, 7589, 6405, 0, 0, 0, 0, 1, 0, 1, 7340, 1] payload_rows_per_dst=[15220, 15564, 8412, 886, 14570, 15184, 14458, 1264] dup_rows_per_dst=[998, 7304, 1748, 0, 2680, 2, 14, 0] combine_rows_per_src=[15220, 15564, 8412, 886, 14570, 15184, 14458, 1264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284319744 dispatch_meta_bytes=331216 combine_payload_bytes=284319744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284650960 ingress_bytes=318922752 link_bytes=318922752 token_bytes=568970704 weight_movement_bytes=34603008 total_nvlink_bytes=603573712 weight_to_token_ratio=0.060817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200080640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279683072 dispatch_meta_bytes=325520 combine_payload_bytes=279683072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280008592 ingress_bytes=314286080 link_bytes=314286080 token_bytes=559691664 weight_movement_bytes=34603008 total_nvlink_bytes=594294672 weight_to_token_ratio=0.061825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182633217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=333365248 dispatch_meta_bytes=348992 combine_payload_bytes=333365248 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=333714240 ingress_bytes=402571264 link_bytes=402571264 token_bytes=667079488 weight_movement_bytes=69206016 total_nvlink_bytes=736285504 weight_to_token_ratio=0.103745 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514843257e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=344465408 dispatch_meta_bytes=351728 combine_payload_bytes=344465408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=344817136 ingress_bytes=379068416 link_bytes=379068416 token_bytes=689282544 weight_movement_bytes=34603008 total_nvlink_bytes=723885552 weight_to_token_ratio=0.050201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426403932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342032384 dispatch_meta_bytes=358456 combine_payload_bytes=342032384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=342390840 ingress_bytes=411238400 link_bytes=411238400 token_bytes=684423224 weight_movement_bytes=69206016 total_nvlink_bytes=753629240 weight_to_token_ratio=0.101116 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.547456992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=177463296 dispatch_meta_bytes=226016 combine_payload_bytes=177463296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=177689312 ingress_bytes=350478336 link_bytes=350478336 token_bytes=355152608 weight_movement_bytes=173015040 total_nvlink_bytes=528167648 weight_to_token_ratio=0.487157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318821763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321585152 dispatch_meta_bytes=327616 combine_payload_bytes=321585152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321912768 ingress_bytes=356188160 link_bytes=356188160 token_bytes=643497920 weight_movement_bytes=34603008 total_nvlink_bytes=678100928 weight_to_token_ratio=0.053773 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340307371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286695424 dispatch_meta_bytes=301744 combine_payload_bytes=286695424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286997168 ingress_bytes=321298432 link_bytes=321298432 token_bytes=573692592 weight_movement_bytes=34603008 total_nvlink_bytes=608295600 weight_to_token_ratio=0.060316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209020133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84030 payload_rows_remote=67804 dup_rows=14274 dup_rows_remote=9884 dedup_ratio=0.145203 router_tokens_per_expert=[645, 0, 25, 0, 0, 1, 0, 81, 0, 0, 6525, 1, 2119, 7822, 1, 4366, 0, 0, 0, 560, 0, 2, 447, 158, 1, 63, 103, 0, 1, 71, 1, 0, 1, 0, 4995, 0, 1, 1, 0, 0, 1, 7532, 0, 0, 0, 122, 601, 465, 0, 4, 7, 58, 3188, 1066, 1, 0, 4, 2, 0, 0, 0, 0, 8110, 0] payload_rows_per_dst=[1574, 12966, 11996, 15650, 9994, 2218, 13406, 16226] dup_rows_per_dst=[18, 0, 3308, 474, 2, 160, 5922, 4390] combine_rows_per_src=[1574, 12966, 11996, 15650, 9994, 2218, 13406, 16226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88838 payload_rows_remote=73164 dup_rows=9466 dup_rows_remote=3062 dedup_ratio=0.096293 router_tokens_per_expert=[645, 0, 52, 0, 3, 0, 1, 337, 0, 1, 5814, 0, 2252, 7823, 1, 4592, 1, 4, 0, 654, 0, 3, 465, 199, 2, 52, 148, 4, 0, 82, 0, 1, 0, 0, 5621, 0, 1, 0, 3, 0, 7, 7147, 1, 0, 0, 136, 629, 472, 0, 0, 1, 48, 2691, 1142, 7, 3, 0, 0, 6, 0, 1, 4, 8094, 2] payload_rows_per_dst=[13334, 2490, 11784, 15788, 11250, 2314, 15674, 16204] dup_rows_per_dst=[370, 0, 2070, 436, 0, 176, 6404, 10] combine_rows_per_src=[13334, 2490, 11784, 15788, 11250, 2314, 15674, 16204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89960 payload_rows_remote=81312 dup_rows=8344 dup_rows_remote=6570 dedup_ratio=0.084880 router_tokens_per_expert=[248, 1, 0, 0, 248, 0, 0, 2, 0, 811, 911, 2160, 0, 0, 6, 2, 2197, 1665, 0, 0, 7776, 0, 0, 1349, 0, 1, 0, 7421, 0, 342, 62, 0, 0, 0, 0, 0, 1155, 558, 302, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 138, 460, 7655, 6249, 1, 0, 0, 0, 0, 0, 0, 7430, 0] payload_rows_per_dst=[15552, 9570, 8648, 10608, 14210, 15314, 1198, 14860] dup_rows_per_dst=[998, 2752, 1774, 502, 2318, 0, 0, 0] combine_rows_per_src=[15552, 9570, 8648, 10608, 14210, 15314, 1198, 14860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91626 payload_rows_remote=77478 dup_rows=6678 dup_rows_remote=4096 dedup_ratio=0.067932 router_tokens_per_expert=[252, 0, 0, 0, 250, 0, 0, 1, 0, 681, 1051, 2324, 0, 0, 25, 4, 2436, 1334, 0, 0, 7650, 0, 0, 1216, 0, 3, 0, 7306, 0, 350, 102, 0, 0, 0, 0, 0, 1166, 564, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 398, 7691, 6315, 1, 0, 0, 0, 0, 0, 0, 7470, 0] payload_rows_per_dst=[15300, 7390, 8392, 14810, 14148, 15382, 1264, 14940] dup_rows_per_dst=[1006, 780, 1580, 712, 2582, 0, 18, 0] combine_rows_per_src=[15300, 7390, 8392, 14810, 14148, 15382, 1264, 14940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91988 payload_rows_remote=77192 dup_rows=6316 dup_rows_remote=6316 dedup_ratio=0.064250 router_tokens_per_expert=[255, 0, 0, 0, 255, 0, 0, 3, 0, 639, 938, 2096, 0, 0, 174, 6, 2364, 1480, 0, 0, 7661, 0, 0, 1472, 0, 0, 0, 7333, 0, 343, 132, 0, 0, 0, 0, 0, 1253, 731, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 32, 498, 7398, 6336, 0, 0, 0, 0, 0, 0, 0, 7353, 0] payload_rows_per_dst=[13496, 6870, 9892, 14926, 4562, 14796, 12740, 14706] dup_rows_per_dst=[966, 836, 2626, 690, 198, 0, 1000, 0] combine_rows_per_src=[13496, 6870, 9892, 14926, 4562, 14796, 12740, 14706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74884 payload_rows_remote=61224 dup_rows=23420 dup_rows_remote=23076 dedup_ratio=0.238241 router_tokens_per_expert=[642, 0, 105, 0, 0, 0, 0, 172, 0, 0, 6083, 0, 2289, 7884, 0, 4896, 0, 0, 0, 574, 0, 0, 403, 169, 0, 56, 206, 0, 0, 78, 0, 0, 0, 0, 5362, 0, 0, 0, 0, 0, 1, 7040, 0, 0, 0, 122, 615, 466, 0, 0, 0, 62, 2863, 950, 0, 0, 1, 0, 0, 0, 0, 0, 8113, 0] payload_rows_per_dst=[13660, 16384, 11312, 636, 10724, 14752, 7414, 2] dup_rows_per_dst=[344, 20188, 772, 44, 0, 1736, 336, 0] combine_rows_per_src=[13660, 16384, 11312, 636, 10724, 14752, 7414, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91132 payload_rows_remote=76310 dup_rows=7172 dup_rows_remote=6412 dedup_ratio=0.072957 router_tokens_per_expert=[245, 0, 0, 0, 245, 0, 0, 3, 0, 777, 989, 2162, 0, 0, 25, 2, 2139, 1663, 0, 0, 7710, 0, 0, 1261, 0, 1, 0, 7297, 0, 374, 118, 1, 0, 0, 0, 0, 1334, 610, 303, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 445, 7660, 6316, 3, 0, 0, 0, 0, 0, 0, 7458, 0] payload_rows_per_dst=[15420, 6852, 8424, 14822, 14460, 15322, 916, 14916] dup_rows_per_dst=[986, 1058, 1702, 760, 2666, 0, 0, 0] combine_rows_per_src=[15420, 6852, 8424, 14822, 14460, 15322, 916, 14916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77526 payload_rows_remote=61196 dup_rows=20778 dup_rows_remote=20134 dedup_ratio=0.211365 router_tokens_per_expert=[7336, 85, 3, 2362, 82, 3, 194, 0, 0, 0, 2, 320, 2, 0, 2, 8161, 0, 7037, 1, 0, 2, 0, 2, 0, 0, 3024, 0, 0, 0, 86, 1, 0, 3481, 1, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 6713, 318, 0, 0, 0, 0, 111, 2, 0, 17, 3, 0, 343, 613, 287, 542, 8011] payload_rows_per_dst=[14526, 16330, 2010, 16280, 6970, 13430, 5008, 2972] dup_rows_per_dst=[13314, 644, 2, 5964, 0, 0, 216, 638] combine_rows_per_src=[14526, 16330, 2010, 16280, 6970, 13430, 5008, 2972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277725184 dispatch_meta_bytes=310752 combine_payload_bytes=277725184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278035936 ingress_bytes=381534208 link_bytes=381534208 token_bytes=555761120 weight_movement_bytes=103809024 total_nvlink_bytes=659570144 weight_to_token_ratio=0.186787 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435682509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=299679744 dispatch_meta_bytes=304904 combine_payload_bytes=299679744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299984648 ingress_bytes=334282752 link_bytes=334282752 token_bytes=599664392 weight_movement_bytes=34603008 total_nvlink_bytes=634267400 weight_to_token_ratio=0.057704 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257879084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=333053952 dispatch_meta_bytes=351528 combine_payload_bytes=333053952 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=333405480 ingress_bytes=436862976 link_bytes=436862976 token_bytes=666459432 weight_movement_bytes=103809024 total_nvlink_bytes=770268456 weight_to_token_ratio=0.155762 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.643880208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317349888 dispatch_meta_bytes=326296 combine_payload_bytes=317349888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317676184 ingress_bytes=351952896 link_bytes=351952896 token_bytes=635026072 weight_movement_bytes=34603008 total_nvlink_bytes=669629080 weight_to_token_ratio=0.054491 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324370413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=316178432 dispatch_meta_bytes=334032 combine_payload_bytes=316178432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316512464 ingress_bytes=350781440 link_bytes=350781440 token_bytes=632690896 weight_movement_bytes=34603008 total_nvlink_bytes=667293904 weight_to_token_ratio=0.054692 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319962318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=250773504 dispatch_meta_bytes=337200 combine_payload_bytes=250773504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251110704 ingress_bytes=285376512 link_bytes=285376512 token_bytes=501884208 weight_movement_bytes=34603008 total_nvlink_bytes=536487216 weight_to_token_ratio=0.068946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.073848840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=312565760 dispatch_meta_bytes=330888 combine_payload_bytes=312565760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312896648 ingress_bytes=381771776 link_bytes=381771776 token_bytes=625462408 weight_movement_bytes=69206016 total_nvlink_bytes=694668424 weight_to_token_ratio=0.110648 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436576458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=250658816 dispatch_meta_bytes=325320 combine_payload_bytes=250658816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250984136 ingress_bytes=319864832 link_bytes=319864832 token_bytes=501642952 weight_movement_bytes=69206016 total_nvlink_bytes=570848968 weight_to_token_ratio=0.137959 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203625612e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70424 payload_rows_remote=55072 dup_rows=27880 dup_rows_remote=24432 dedup_ratio=0.283610 router_tokens_per_expert=[243, 4, 0, 2, 244, 0, 0, 2, 1, 884, 848, 2556, 2, 4, 131, 2, 1976, 1585, 0, 0, 7727, 2, 0, 1244, 0, 0, 0, 7397, 1, 354, 77, 0, 0, 0, 0, 0, 1187, 675, 280, 0, 2, 4, 0, 0, 0, 0, 0, 0, 1, 1, 0, 23, 460, 7516, 6293, 0, 0, 0, 0, 0, 0, 1, 7423, 0] payload_rows_per_dst=[504, 7966, 10630, 15592, 4252, 12, 16116, 15352] dup_rows_per_dst=[486, 890, 2998, 7554, 32, 0, 12472, 3448] combine_rows_per_src=[504, 7966, 10630, 15592, 4252, 12, 16116, 15352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79612 payload_rows_remote=65040 dup_rows=18692 dup_rows_remote=11306 dedup_ratio=0.190145 router_tokens_per_expert=[237, 1, 0, 3, 236, 1, 1, 2, 0, 705, 718, 2441, 0, 0, 47, 7, 2277, 1401, 4, 0, 7725, 0, 0, 1441, 0, 0, 0, 7361, 5, 352, 60, 0, 1, 0, 4, 5, 1213, 489, 393, 0, 3, 0, 0, 1, 2, 0, 0, 0, 2, 6, 0, 197, 496, 7571, 6329, 0, 1, 4, 3, 1, 0, 4, 7401, 1] payload_rows_per_dst=[486, 7102, 16048, 14852, 4070, 7254, 14572, 15228] dup_rows_per_dst=[476, 734, 5662, 704, 140, 2, 7386, 3588] combine_rows_per_src=[486, 7102, 16048, 14852, 4070, 7254, 14572, 15228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91354 payload_rows_remote=77026 dup_rows=6950 dup_rows_remote=6948 dedup_ratio=0.070699 router_tokens_per_expert=[7329, 70, 0, 2834, 57, 0, 175, 0, 0, 0, 0, 314, 0, 0, 0, 8192, 0, 7164, 1, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 57, 0, 0, 3463, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 6911, 230, 0, 0, 0, 0, 113, 0, 0, 3, 0, 0, 395, 428, 252, 453, 8042] payload_rows_per_dst=[6254, 16384, 14328, 16230, 6944, 13822, 14734, 2658] dup_rows_per_dst=[18, 628, 2, 5288, 0, 0, 610, 404] combine_rows_per_src=[6254, 16384, 14328, 16230, 6944, 13822, 14734, 2658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94110 payload_rows_remote=86080 dup_rows=4194 dup_rows_remote=4194 dedup_ratio=0.042664 router_tokens_per_expert=[7222, 72, 0, 2605, 64, 0, 208, 0, 0, 0, 0, 319, 0, 0, 0, 8133, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 63, 0, 0, 4007, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6679, 341, 0, 0, 0, 0, 61, 0, 0, 18, 0, 0, 384, 563, 213, 464, 7979] payload_rows_per_dst=[5890, 16266, 14298, 5326, 8030, 13358, 14562, 16380] dup_rows_per_dst=[8, 638, 0, 0, 0, 0, 686, 2862] combine_rows_per_src=[5890, 16266, 14298, 5326, 8030, 13358, 14562, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91338 payload_rows_remote=78428 dup_rows=6966 dup_rows_remote=6966 dedup_ratio=0.070862 router_tokens_per_expert=[7212, 75, 0, 2551, 69, 0, 219, 0, 0, 0, 0, 325, 0, 0, 0, 8175, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 0, 54, 0, 0, 3838, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 6455, 339, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 469, 661, 304, 476, 7986] payload_rows_per_dst=[5822, 11340, 14132, 5650, 10576, 12910, 14524, 16384] dup_rows_per_dst=[6, 426, 0, 0, 2378, 0, 748, 3408] combine_rows_per_src=[5822, 11340, 14132, 5650, 10576, 12910, 14524, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85254 payload_rows_remote=69870 dup_rows=13050 dup_rows_remote=12024 dedup_ratio=0.132751 router_tokens_per_expert=[255, 1, 0, 0, 255, 0, 0, 2, 0, 685, 932, 2237, 0, 0, 184, 0, 1989, 1514, 0, 0, 7692, 0, 0, 1381, 0, 3, 0, 7280, 0, 311, 129, 0, 0, 0, 1, 0, 1300, 599, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 192, 438, 7518, 6454, 1, 0, 0, 0, 0, 0, 1, 7443, 0] payload_rows_per_dst=[15384, 15514, 8168, 886, 14516, 15036, 15748, 2] dup_rows_per_dst=[1026, 7122, 1600, 0, 2900, 0, 402, 0] combine_rows_per_src=[15384, 15514, 8168, 886, 14516, 15036, 15748, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91356 payload_rows_remote=81654 dup_rows=6948 dup_rows_remote=4920 dedup_ratio=0.070679 router_tokens_per_expert=[7412, 81, 0, 2821, 93, 0, 205, 0, 0, 0, 0, 321, 0, 0, 2, 8192, 0, 7041, 0, 0, 0, 0, 0, 0, 0, 2487, 0, 0, 0, 69, 0, 0, 3720, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6607, 358, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 445, 491, 270, 460, 8003] payload_rows_per_dst=[6374, 16384, 14082, 9702, 7446, 13526, 14850, 8992] dup_rows_per_dst=[26, 646, 0, 2028, 0, 1800, 826, 1622] combine_rows_per_src=[6374, 16384, 14082, 9702, 7446, 13526, 14850, 8992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=59380 dup_rows=23882 dup_rows_remote=23082 dedup_ratio=0.242940 router_tokens_per_expert=[0, 6871, 0, 0, 1, 77, 0, 0, 0, 0, 3, 2, 1, 399, 0, 0, 0, 0, 0, 8188, 0, 2, 381, 8131, 0, 472, 1, 0, 0, 0, 0, 2474, 0, 0, 87, 2, 0, 4695, 7516, 0, 385, 0, 1, 0, 251, 0, 0, 2, 8061, 0, 0, 1007, 0, 3, 0, 2, 0, 2, 0, 134, 1, 0, 0, 0] payload_rows_per_dst=[13744, 15042, 16380, 16122, 9566, 1274, 2020, 274] dup_rows_per_dst=[154, 800, 17024, 5894, 2, 4, 4, 0] combine_rows_per_src=[13744, 15042, 16380, 16122, 9566, 1274, 2020, 274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=225574912 dispatch_meta_bytes=318016 combine_payload_bytes=225574912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=225892928 ingress_bytes=294780928 link_bytes=294780928 token_bytes=451467840 weight_movement_bytes=69206016 total_nvlink_bytes=520673856 weight_to_token_ratio=0.153291 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109236900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=305384 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266709224 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533113064 weight_movement_bytes=103809024 total_nvlink_bytes=636922088 weight_to_token_ratio=0.194722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=315498496 dispatch_meta_bytes=335896 combine_payload_bytes=315498496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=315834392 ingress_bytes=384704512 link_bytes=384704512 token_bytes=631332888 weight_movement_bytes=69206016 total_nvlink_bytes=700538904 weight_to_token_ratio=0.109619 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447612107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=352583680 dispatch_meta_bytes=361096 combine_payload_bytes=352583680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=352944776 ingress_bytes=387186688 link_bytes=387186688 token_bytes=705528456 weight_movement_bytes=34603008 total_nvlink_bytes=740131464 weight_to_token_ratio=0.049046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.456952336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=321241088 dispatch_meta_bytes=341576 combine_payload_bytes=321241088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321582664 ingress_bytes=390447104 link_bytes=390447104 token_bytes=642823752 weight_movement_bytes=69206016 total_nvlink_bytes=712029768 weight_to_token_ratio=0.107659 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469221019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286187520 dispatch_meta_bytes=327576 combine_payload_bytes=286187520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286515096 ingress_bytes=320790528 link_bytes=320790528 token_bytes=572702616 weight_movement_bytes=34603008 total_nvlink_bytes=607305624 weight_to_token_ratio=0.060421 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207108931e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=334454784 dispatch_meta_bytes=346296 combine_payload_bytes=334454784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334801080 ingress_bytes=369057792 link_bytes=369057792 token_bytes=669255864 weight_movement_bytes=34603008 total_nvlink_bytes=703858872 weight_to_token_ratio=0.051704 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=243220480 dispatch_meta_bytes=329848 combine_payload_bytes=243220480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243550328 ingress_bytes=277823488 link_bytes=277823488 token_bytes=486770808 weight_movement_bytes=34603008 total_nvlink_bytes=521373816 weight_to_token_ratio=0.071087 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.045427419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86750 payload_rows_remote=70374 dup_rows=11554 dup_rows_remote=8704 dedup_ratio=0.117533 router_tokens_per_expert=[7317, 69, 0, 2658, 42, 0, 227, 0, 0, 0, 0, 340, 0, 0, 9, 8159, 0, 7142, 7, 0, 0, 0, 1, 3, 0, 2406, 0, 0, 0, 49, 0, 0, 3998, 0, 0, 0, 0, 0, 7, 0, 0, 2, 1, 0, 8, 0, 2, 6686, 362, 1, 0, 1, 0, 42, 0, 0, 0, 1, 0, 406, 509, 233, 432, 8032] payload_rows_per_dst=[5990, 698, 14288, 4910, 16332, 13388, 14768, 16376] dup_rows_per_dst=[2, 0, 18, 0, 7996, 10, 678, 2850] combine_rows_per_src=[5990, 698, 14288, 4910, 16332, 13388, 14768, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86792 payload_rows_remote=72306 dup_rows=11512 dup_rows_remote=10674 dedup_ratio=0.117106 router_tokens_per_expert=[7218, 81, 0, 2551, 89, 1, 201, 0, 1, 0, 4, 318, 9, 0, 5, 8151, 0, 7091, 0, 0, 1, 4, 5, 1, 2, 2633, 0, 6, 0, 89, 1, 0, 3758, 2, 1, 0, 0, 1, 2, 6, 2, 1, 0, 7, 0, 2, 1, 6718, 352, 0, 0, 0, 0, 90, 2, 0, 3, 1, 0, 426, 600, 278, 471, 7966] payload_rows_per_dst=[5840, 662, 14198, 5460, 16318, 13456, 14486, 16372] dup_rows_per_dst=[6, 12, 6, 2, 7524, 6, 838, 3118] combine_rows_per_src=[5840, 662, 14198, 5460, 16318, 13456, 14486, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88500 payload_rows_remote=78768 dup_rows=9804 dup_rows_remote=7674 dedup_ratio=0.099731 router_tokens_per_expert=[0, 6978, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 8192, 0, 0, 319, 8131, 0, 470, 0, 0, 0, 0, 0, 2602, 0, 0, 71, 0, 0, 4185, 7556, 0, 341, 0, 0, 0, 261, 0, 0, 0, 8119, 0, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0] payload_rows_per_dst=[13958, 15166, 9732, 16238, 8512, 7506, 2752, 14636] dup_rows_per_dst=[106, 792, 2130, 6144, 0, 508, 0, 124] combine_rows_per_src=[13958, 15166, 9732, 16238, 8512, 7506, 2752, 14636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92764 payload_rows_remote=83972 dup_rows=5540 dup_rows_remote=5540 dedup_ratio=0.056356 router_tokens_per_expert=[0, 7042, 0, 0, 0, 73, 0, 1, 0, 0, 0, 0, 0, 454, 0, 1, 0, 0, 0, 8192, 0, 0, 344, 8114, 0, 436, 0, 6, 0, 0, 0, 2397, 0, 0, 72, 0, 0, 4324, 7613, 0, 377, 0, 0, 0, 282, 0, 0, 0, 8079, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0] payload_rows_per_dst=[14084, 15256, 688, 5174, 8792, 16228, 16158, 16384] dup_rows_per_dst=[148, 880, 0, 504, 0, 1318, 2456, 234] combine_rows_per_src=[14084, 15256, 688, 5174, 8792, 16228, 16158, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93132 payload_rows_remote=76850 dup_rows=5172 dup_rows_remote=3946 dedup_ratio=0.052612 router_tokens_per_expert=[0, 7013, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 387, 0, 2, 0, 0, 0, 8192, 0, 0, 324, 8141, 0, 453, 0, 0, 0, 0, 0, 2303, 0, 0, 76, 0, 0, 4715, 7518, 0, 373, 0, 0, 0, 240, 0, 0, 0, 8089, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0] payload_rows_per_dst=[14026, 15036, 648, 4998, 9580, 16282, 16178, 16384] dup_rows_per_dst=[178, 778, 0, 514, 2, 1226, 2264, 210] combine_rows_per_src=[14026, 15036, 648, 4998, 9580, 16282, 16178, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73524 payload_rows_remote=57148 dup_rows=24780 dup_rows_remote=6928 dedup_ratio=0.252075 router_tokens_per_expert=[7300, 78, 0, 2555, 55, 0, 193, 0, 0, 0, 0, 347, 0, 0, 0, 8147, 0, 6933, 0, 0, 0, 0, 0, 0, 0, 2800, 0, 0, 0, 58, 0, 0, 3893, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6769, 302, 0, 0, 0, 0, 83, 0, 0, 3, 0, 0, 435, 485, 233, 478, 8003] payload_rows_per_dst=[16376, 16308, 10940, 15998, 7790, 2598, 770, 2744] dup_rows_per_dst=[17852, 938, 0, 5466, 0, 0, 0, 524] combine_rows_per_src=[16376, 16308, 10940, 15998, 7790, 2598, 770, 2744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90098 payload_rows_remote=76344 dup_rows=8206 dup_rows_remote=8204 dedup_ratio=0.083476 router_tokens_per_expert=[0, 0, 0, 624, 1, 408, 0, 3, 6864, 1, 0, 2, 0, 2, 1, 8, 0, 0, 5708, 0, 31, 0, 0, 0, 193, 1, 0, 0, 1, 160, 2, 0, 301, 79, 7936, 8189, 8096, 1, 1, 0, 0, 0, 2383, 2, 1, 0, 0, 41, 1, 0, 3, 0, 0, 8102, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16122, 13754, 11422, 16198, 16380, 16206, 6, 10] dup_rows_per_dst=[1822, 2, 56, 708, 762, 4852, 4, 0] combine_rows_per_src=[16122, 13754, 11422, 16198, 16380, 16206, 6, 10] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92320 payload_rows_remote=85052 dup_rows=5984 dup_rows_remote=4500 dedup_ratio=0.060872 router_tokens_per_expert=[0, 6949, 0, 0, 0, 51, 0, 2, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 8192, 0, 0, 375, 8151, 0, 422, 0, 0, 0, 0, 0, 2322, 0, 0, 81, 0, 0, 4416, 7584, 0, 326, 0, 0, 0, 315, 0, 0, 0, 8096, 0, 0, 1228, 0, 0, 0, 0, 0, 2, 0, 167, 0, 0, 0, 0] payload_rows_per_dst=[13898, 15234, 750, 7268, 8994, 16302, 13490, 16384] dup_rows_per_dst=[106, 880, 0, 1484, 0, 1282, 1894, 338] combine_rows_per_src=[13898, 15234, 750, 7268, 8994, 16302, 13490, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288251904 dispatch_meta_bytes=316312 combine_payload_bytes=288251904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=288568216 ingress_bytes=392060928 link_bytes=392060928 token_bytes=576820120 weight_movement_bytes=103809024 total_nvlink_bytes=680629144 weight_to_token_ratio=0.179968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475293709e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=296165376 dispatch_meta_bytes=331920 combine_payload_bytes=296165376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296497296 ingress_bytes=330768384 link_bytes=330768384 token_bytes=592662672 weight_movement_bytes=34603008 total_nvlink_bytes=627265680 weight_to_token_ratio=0.058386 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244654800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=322633728 dispatch_meta_bytes=345768 combine_payload_bytes=322633728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=322979496 ingress_bytes=426442752 link_bytes=426442752 token_bytes=645613224 weight_movement_bytes=103809024 total_nvlink_bytes=749422248 weight_to_token_ratio=0.160791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.604669744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=343949312 dispatch_meta_bytes=358048 combine_payload_bytes=343949312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=344307360 ingress_bytes=447758336 link_bytes=447758336 token_bytes=688256672 weight_movement_bytes=103809024 total_nvlink_bytes=792065696 weight_to_token_ratio=0.150829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.684878571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=314777600 dispatch_meta_bytes=323184 combine_payload_bytes=314777600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315100784 ingress_bytes=349380608 link_bytes=349380608 token_bytes=629878384 weight_movement_bytes=34603008 total_nvlink_bytes=664481392 weight_to_token_ratio=0.054936 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314691100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=256304 combine_payload_bytes=234078208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=234334512 ingress_bytes=303284224 link_bytes=303284224 token_bytes=468412720 weight_movement_bytes=69206016 total_nvlink_bytes=537618736 weight_to_token_ratio=0.147746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141234119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=312705024 dispatch_meta_bytes=338192 combine_payload_bytes=312705024 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313043216 ingress_bytes=381911040 link_bytes=381911040 token_bytes=625748240 weight_movement_bytes=69206016 total_nvlink_bytes=694954256 weight_to_token_ratio=0.110597 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437100497e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=348372992 dispatch_meta_bytes=358208 combine_payload_bytes=348372992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348731200 ingress_bytes=382976000 link_bytes=382976000 token_bytes=697104192 weight_movement_bytes=34603008 total_nvlink_bytes=731707200 weight_to_token_ratio=0.049638 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.441107856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85100 payload_rows_remote=68720 dup_rows=13204 dup_rows_remote=13044 dedup_ratio=0.134318 router_tokens_per_expert=[0, 6972, 0, 0, 0, 56, 0, 3, 0, 0, 0, 1, 0, 423, 0, 1, 0, 0, 0, 8188, 0, 0, 330, 8131, 0, 422, 0, 3, 0, 0, 0, 2539, 0, 0, 83, 0, 1, 4304, 7561, 0, 352, 0, 1, 1, 250, 0, 0, 1, 8114, 0, 0, 1332, 0, 1, 0, 0, 0, 0, 0, 75, 0, 0, 1, 6] payload_rows_per_dst=[118, 850, 13980, 5442, 15836, 16264, 16230, 16380] dup_rows_per_dst=[0, 0, 624, 486, 7210, 1208, 3516, 160] combine_rows_per_src=[118, 850, 13980, 5442, 15836, 16264, 16230, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85874 payload_rows_remote=69640 dup_rows=12430 dup_rows_remote=10062 dedup_ratio=0.126444 router_tokens_per_expert=[0, 6901, 0, 0, 0, 48, 0, 0, 0, 10, 7, 0, 0, 422, 0, 5, 0, 0, 0, 8181, 0, 2, 301, 8117, 0, 478, 2, 2, 0, 0, 0, 2493, 0, 0, 88, 3, 1, 4458, 7542, 4, 350, 1, 8, 9, 290, 3, 0, 2, 8111, 0, 0, 1185, 0, 0, 0, 5, 0, 1, 1, 114, 0, 0, 5, 2] payload_rows_per_dst=[11560, 1678, 2782, 5472, 15506, 16266, 16234, 16376] dup_rows_per_dst=[86, 68, 76, 478, 7828, 1294, 2368, 232] combine_rows_per_src=[11560, 1678, 2782, 5472, 15506, 16266, 16234, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90864 payload_rows_remote=79192 dup_rows=7440 dup_rows_remote=7310 dedup_ratio=0.075684 router_tokens_per_expert=[0, 0, 0, 496, 0, 366, 0, 0, 6837, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 67, 0, 0, 0, 175, 2, 0, 0, 0, 145, 0, 0, 213, 70, 7981, 8192, 8115, 0, 0, 0, 0, 0, 2510, 8, 0, 0, 0, 21, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16236, 13676, 11672, 16234, 474, 16236, 0, 16336] dup_rows_per_dst=[1450, 2, 130, 640, 140, 5078, 0, 0] combine_rows_per_src=[16236, 13676, 11672, 16234, 474, 16236, 0, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95148 payload_rows_remote=94634 dup_rows=3156 dup_rows_remote=3014 dedup_ratio=0.032104 router_tokens_per_expert=[0, 0, 0, 780, 0, 441, 0, 0, 6937, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5494, 0, 30, 0, 0, 0, 208, 0, 0, 0, 0, 189, 0, 0, 257, 71, 7947, 8192, 8119, 0, 0, 0, 0, 0, 2346, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8120, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16164, 13876, 11004, 16238, 514, 4728, 16240, 16384] dup_rows_per_dst=[2172, 0, 44, 794, 142, 0, 0, 4] combine_rows_per_src=[16164, 13876, 11004, 16238, 514, 4728, 16240, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92396 payload_rows_remote=87158 dup_rows=5908 dup_rows_remote=5908 dedup_ratio=0.060099 router_tokens_per_expert=[0, 0, 0, 558, 0, 444, 0, 0, 6731, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5651, 0, 16, 0, 0, 0, 219, 10, 0, 0, 0, 190, 0, 0, 247, 76, 7967, 8192, 8116, 0, 0, 0, 0, 0, 2615, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3434, 13464, 11304, 11782, 14560, 5238, 16230, 16384] dup_rows_per_dst=[262, 0, 30, 588, 5028, 0, 0, 0] combine_rows_per_src=[3434, 13464, 11304, 11782, 14560, 5238, 16230, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72644 payload_rows_remote=58474 dup_rows=25660 dup_rows_remote=23692 dedup_ratio=0.261027 router_tokens_per_expert=[0, 6982, 0, 0, 0, 60, 0, 7, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 8192, 0, 0, 412, 8141, 0, 466, 0, 0, 0, 0, 0, 2473, 0, 0, 78, 0, 0, 4180, 7568, 0, 390, 0, 0, 0, 281, 0, 0, 0, 8104, 0, 0, 1370, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0] payload_rows_per_dst=[14170, 15136, 16384, 16208, 6476, 1342, 2740, 188] dup_rows_per_dst=[1968, 708, 17106, 5878, 0, 0, 0, 0] combine_rows_per_src=[14170, 15136, 16384, 16208, 6476, 1342, 2740, 188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92228 payload_rows_remote=76012 dup_rows=6076 dup_rows_remote=5400 dedup_ratio=0.061808 router_tokens_per_expert=[0, 0, 0, 535, 0, 455, 0, 0, 6816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5462, 0, 20, 0, 0, 0, 204, 7, 0, 0, 0, 127, 0, 0, 228, 81, 7992, 8192, 8108, 0, 0, 0, 0, 0, 2802, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16242, 13632, 10942, 16216, 456, 12400, 5956, 16384] dup_rows_per_dst=[1722, 0, 22, 676, 162, 3494, 0, 0] combine_rows_per_src=[16242, 13632, 10942, 16216, 456, 12400, 5956, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94242 payload_rows_remote=77860 dup_rows=4062 dup_rows_remote=4060 dedup_ratio=0.041321 router_tokens_per_expert=[0, 0, 24, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 8185, 1, 0, 0, 0, 0, 8189, 5811, 0, 0, 63, 1, 1, 0, 1639, 0, 3, 91, 3, 8187, 3, 2, 0, 0, 1, 2, 0, 1, 0, 4456, 86, 505, 0, 19, 2, 6562, 0, 0, 0, 762, 4546, 0, 0] payload_rows_per_dst=[13146, 16382, 16376, 11748, 16384, 8, 9888, 10310] dup_rows_per_dst=[30, 2, 0, 0, 3466, 8, 250, 306] combine_rows_per_src=[13146, 16382, 16376, 11748, 16384, 8, 9888, 10310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281477120 dispatch_meta_bytes=327056 combine_payload_bytes=281477120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281804176 ingress_bytes=316080128 link_bytes=316080128 token_bytes=563281296 weight_movement_bytes=34603008 total_nvlink_bytes=597884304 weight_to_token_ratio=0.061431 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189384075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=285245440 dispatch_meta_bytes=318808 combine_payload_bytes=285245440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285564248 ingress_bytes=354451456 link_bytes=354451456 token_bytes=570809688 weight_movement_bytes=69206016 total_nvlink_bytes=640015704 weight_to_token_ratio=0.121242 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333772293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324370432 dispatch_meta_bytes=346008 combine_payload_bytes=324370432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324716440 ingress_bytes=358973440 link_bytes=358973440 token_bytes=649086872 weight_movement_bytes=34603008 total_nvlink_bytes=683689880 weight_to_token_ratio=0.053310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350788155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=356999168 dispatch_meta_bytes=372264 combine_payload_bytes=356999168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=357371432 ingress_bytes=391602176 link_bytes=391602176 token_bytes=714370600 weight_movement_bytes=34603008 total_nvlink_bytes=748973608 weight_to_token_ratio=0.048438 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473567462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=387620864 dispatch_meta_bytes=390592 combine_payload_bytes=387620864 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=388011456 ingress_bytes=560635904 link_bytes=560635904 token_bytes=775632320 weight_movement_bytes=173015040 total_nvlink_bytes=948647360 weight_to_token_ratio=0.223063 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.109627772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=311345152 dispatch_meta_bytes=325648 combine_payload_bytes=311345152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311670800 ingress_bytes=345948160 link_bytes=345948160 token_bytes=623015952 weight_movement_bytes=34603008 total_nvlink_bytes=657618960 weight_to_token_ratio=0.055541 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301775075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=239509504 dispatch_meta_bytes=328664 combine_payload_bytes=239509504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=239838168 ingress_bytes=308715520 link_bytes=308715520 token_bytes=479347672 weight_movement_bytes=69206016 total_nvlink_bytes=548553688 weight_to_token_ratio=0.144375 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161671648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=318914560 dispatch_meta_bytes=327680 combine_payload_bytes=318914560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319242240 ingress_bytes=353517568 link_bytes=353517568 token_bytes=638156800 weight_movement_bytes=34603008 total_nvlink_bytes=672759808 weight_to_token_ratio=0.054223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330258148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72666 payload_rows_remote=56284 dup_rows=25638 dup_rows_remote=25624 dedup_ratio=0.260803 router_tokens_per_expert=[0, 0, 0, 806, 0, 475, 0, 0, 6864, 0, 0, 0, 0, 2, 0, 8, 1, 0, 5668, 0, 62, 1, 1, 0, 227, 1, 0, 0, 0, 100, 0, 0, 224, 71, 7986, 8190, 8113, 0, 0, 0, 0, 0, 2196, 0, 0, 0, 4, 46, 2, 0, 0, 0, 0, 8094, 0, 2, 6, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[2458, 2496, 13432, 656, 16370, 4490, 16382, 16382] dup_rows_per_dst=[104, 0, 4670, 0, 5062, 2, 15786, 14] combine_rows_per_src=[2458, 2496, 13432, 656, 16370, 4490, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81822 payload_rows_remote=65504 dup_rows=16482 dup_rows_remote=5100 dedup_ratio=0.167664 router_tokens_per_expert=[0, 0, 0, 475, 8, 419, 0, 8, 6782, 1, 0, 7, 1, 1, 4, 9, 0, 0, 5934, 0, 79, 6, 0, 0, 200, 3, 0, 0, 0, 123, 2, 0, 240, 80, 7974, 8181, 8089, 0, 0, 1, 0, 0, 2370, 3, 1, 5, 1, 27, 2, 0, 5, 0, 1, 8086, 0, 5, 10, 8, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[1800, 13594, 11910, 656, 16350, 4812, 16318, 16382] dup_rows_per_dst=[20, 16, 128, 0, 4916, 2, 11382, 18] combine_rows_per_src=[1800, 13594, 11910, 656, 16350, 4812, 16318, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85618 payload_rows_remote=69234 dup_rows=12686 dup_rows_remote=562 dedup_ratio=0.129049 router_tokens_per_expert=[0, 0, 145, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5868, 0, 0, 49, 0, 0, 0, 1237, 0, 0, 85, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4810, 75, 517, 0, 28, 0, 6409, 0, 2, 0, 735, 4612, 0, 0] payload_rows_per_dst=[13084, 16348, 4260, 11840, 2642, 16384, 10598, 10462] dup_rows_per_dst=[30, 0, 0, 30, 4, 12124, 262, 236] combine_rows_per_src=[13084, 16348, 4260, 11840, 2642, 16384, 10598, 10462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95066 payload_rows_remote=78682 dup_rows=3238 dup_rows_remote=3238 dedup_ratio=0.032939 router_tokens_per_expert=[0, 0, 151, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 5971, 0, 0, 45, 0, 0, 0, 1222, 0, 0, 72, 4, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4443, 70, 546, 0, 31, 0, 6442, 0, 0, 0, 771, 4807, 0, 0] payload_rows_per_dst=[13154, 16382, 16384, 12032, 16384, 0, 9920, 10810] dup_rows_per_dst=[36, 0, 0, 0, 2596, 0, 260, 346] combine_rows_per_src=[13154, 16382, 16384, 12032, 16384, 0, 9920, 10810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96064 payload_rows_remote=83440 dup_rows=2240 dup_rows_remote=448 dedup_ratio=0.022786 router_tokens_per_expert=[0, 0, 98, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6228, 0, 0, 73, 0, 0, 0, 1086, 0, 0, 79, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4768, 72, 552, 0, 16, 0, 6318, 0, 0, 0, 768, 4517, 0, 0] payload_rows_per_dst=[12812, 16384, 16384, 12602, 12624, 4298, 10558, 10402] dup_rows_per_dst=[22, 0, 0, 0, 1792, 0, 258, 168] combine_rows_per_src=[12812, 16384, 16384, 12602, 12624, 4298, 10558, 10402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95312 payload_rows_remote=83246 dup_rows=2992 dup_rows_remote=2988 dedup_ratio=0.030436 router_tokens_per_expert=[0, 0, 122, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5983, 0, 0, 52, 0, 0, 0, 1163, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4744, 81, 584, 0, 20, 0, 6317, 0, 7, 0, 719, 4693, 0, 0] payload_rows_per_dst=[12880, 16384, 16384, 12066, 16384, 0, 10550, 10664] dup_rows_per_dst=[18, 0, 0, 4, 2488, 0, 308, 174] combine_rows_per_src=[12880, 16384, 16384, 12066, 16384, 0, 10550, 10664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90338 payload_rows_remote=74122 dup_rows=7966 dup_rows_remote=5886 dedup_ratio=0.081034 router_tokens_per_expert=[0, 0, 0, 717, 0, 435, 0, 0, 6888, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5750, 0, 19, 0, 0, 0, 193, 16, 0, 0, 0, 138, 0, 0, 224, 78, 7996, 8192, 8112, 0, 0, 0, 0, 0, 2234, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 8106, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16216, 13792, 11504, 16224, 16384, 16212, 0, 6] dup_rows_per_dst=[2080, 0, 34, 694, 604, 4554, 0, 0] combine_rows_per_src=[16216, 13792, 11504, 16224, 16384, 16212, 0, 6] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65872 payload_rows_remote=49494 dup_rows=32432 dup_rows_remote=3848 dedup_ratio=0.329915 router_tokens_per_expert=[0, 0, 2, 0, 1672, 0, 0, 0, 2, 7867, 0, 0, 8098, 42, 235, 145, 149, 0, 2, 58, 0, 118, 0, 0, 0, 3, 1, 244, 1, 0, 8188, 0, 0, 87, 2, 0, 2, 0, 0, 1, 1, 1, 0, 0, 0, 0, 7472, 2, 6883, 5, 0, 1, 1, 0, 0, 0, 2, 0, 200, 10, 1, 0, 7653, 1] payload_rows_per_dst=[16380, 16378, 654, 15314, 180, 2766, 13778, 422] dup_rows_per_dst=[3344, 28584, 0, 490, 4, 2, 2, 6] combine_rows_per_src=[16380, 16378, 654, 15314, 180, 2766, 13778, 422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=230539264 dispatch_meta_bytes=327632 combine_payload_bytes=230539264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230866896 ingress_bytes=265142272 link_bytes=265142272 token_bytes=461406160 weight_movement_bytes=34603008 total_nvlink_bytes=496009168 weight_to_token_ratio=0.074995 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.977090238e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=268304384 dispatch_meta_bytes=282416 combine_payload_bytes=268304384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268586800 ingress_bytes=337510400 link_bytes=337510400 token_bytes=536891184 weight_movement_bytes=69206016 total_nvlink_bytes=606097200 weight_to_token_ratio=0.128901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270024463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=283582464 dispatch_meta_bytes=279184 combine_payload_bytes=283582464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283861648 ingress_bytes=352788480 link_bytes=352788480 token_bytes=567444112 weight_movement_bytes=69206016 total_nvlink_bytes=636650128 weight_to_token_ratio=0.121961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327514648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=322281472 dispatch_meta_bytes=327680 combine_payload_bytes=322281472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322609152 ingress_bytes=391487488 link_bytes=391487488 token_bytes=644890624 weight_movement_bytes=69206016 total_nvlink_bytes=714096640 weight_to_token_ratio=0.107314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473135900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=341770240 dispatch_meta_bytes=335552 combine_payload_bytes=341770240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=342105792 ingress_bytes=376373248 link_bytes=376373248 token_bytes=683876032 weight_movement_bytes=34603008 total_nvlink_bytes=718479040 weight_to_token_ratio=0.050598 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416262232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=340975616 dispatch_meta_bytes=344936 combine_payload_bytes=340975616 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=341320552 ingress_bytes=479387648 link_bytes=479387648 token_bytes=682296168 weight_movement_bytes=138412032 total_nvlink_bytes=820708200 weight_to_token_ratio=0.202862 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.803897126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303603712 dispatch_meta_bytes=320032 combine_payload_bytes=303603712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303923744 ingress_bytes=338206720 link_bytes=338206720 token_bytes=607527456 weight_movement_bytes=34603008 total_nvlink_bytes=642130464 weight_to_token_ratio=0.056957 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272644659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=202727424 dispatch_meta_bytes=213368 combine_payload_bytes=202727424 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=202940792 ingress_bytes=306536448 link_bytes=306536448 token_bytes=405668216 weight_movement_bytes=103809024 total_nvlink_bytes=509477240 weight_to_token_ratio=0.255896 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153471976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68226 payload_rows_remote=54716 dup_rows=30078 dup_rows_remote=20274 dedup_ratio=0.305969 router_tokens_per_expert=[0, 0, 19, 0, 8, 0, 0, 1, 2, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 1, 0, 8189, 1, 0, 0, 0, 0, 8186, 6292, 0, 0, 56, 0, 0, 0, 1167, 0, 0, 93, 4, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4752, 71, 556, 0, 16, 0, 6168, 0, 2, 0, 778, 4596, 0, 0] payload_rows_per_dst=[6798, 8, 4, 16376, 2526, 16378, 12626, 13510] dup_rows_per_dst=[18, 2, 4, 108, 2, 16378, 3762, 9804] combine_rows_per_src=[6798, 8, 4, 16376, 2526, 16378, 12626, 13510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72208 payload_rows_remote=58616 dup_rows=26096 dup_rows_remote=21660 dedup_ratio=0.265462 router_tokens_per_expert=[1, 0, 153, 8, 5, 0, 0, 3, 4, 1, 0, 1, 0, 8, 0, 0, 0, 0, 0, 1, 3, 8178, 5, 0, 0, 0, 2, 8179, 5963, 1, 0, 49, 1, 0, 4, 945, 1, 7, 98, 11, 8162, 2, 14, 0, 0, 2, 5, 0, 1, 0, 4846, 81, 532, 0, 34, 6, 6455, 0, 4, 1, 743, 4632, 0, 0] payload_rows_per_dst=[13202, 22, 16, 16362, 2126, 16382, 13592, 10506] dup_rows_per_dst=[48, 6, 2, 4998, 8, 16344, 4436, 254] combine_rows_per_src=[13202, 22, 16, 16362, 2126, 16382, 13592, 10506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97178 payload_rows_remote=82396 dup_rows=1126 dup_rows_remote=1122 dedup_ratio=0.011454 router_tokens_per_expert=[0, 0, 0, 0, 1719, 0, 0, 0, 0, 7909, 0, 0, 8116, 3, 244, 138, 178, 0, 0, 10, 0, 95, 0, 0, 0, 0, 0, 204, 0, 0, 8192, 0, 1, 79, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7391, 0, 6959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 2, 0, 0, 7697, 0] payload_rows_per_dst=[3438, 768, 15988, 16384, 16384, 14782, 13918, 15516] dup_rows_per_dst=[0, 2, 396, 408, 8, 4, 0, 308] combine_rows_per_src=[3438, 768, 15988, 16384, 16384, 14782, 13918, 15516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96496 payload_rows_remote=80112 dup_rows=1808 dup_rows_remote=1776 dedup_ratio=0.018392 router_tokens_per_expert=[0, 0, 0, 0, 1726, 0, 0, 0, 0, 7894, 0, 0, 8120, 18, 223, 159, 167, 0, 0, 22, 0, 45, 0, 0, 0, 0, 0, 242, 0, 0, 8192, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7550, 0, 6942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 9, 0, 0, 7612, 0] payload_rows_per_dst=[6406, 780, 15890, 12750, 16384, 15114, 13884, 15288] dup_rows_per_dst=[788, 20, 366, 376, 32, 18, 0, 208] combine_rows_per_src=[6406, 780, 15890, 12750, 16384, 15114, 13884, 15288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93844 payload_rows_remote=78704 dup_rows=4460 dup_rows_remote=4100 dedup_ratio=0.045369 router_tokens_per_expert=[0, 0, 0, 0, 1705, 0, 0, 0, 0, 7897, 0, 0, 8122, 23, 212, 145, 141, 0, 0, 56, 0, 97, 0, 0, 0, 0, 0, 226, 0, 0, 8192, 0, 2, 71, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 7414, 0, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 1, 0, 0, 7690, 0] payload_rows_per_dst=[16384, 1588, 15140, 2888, 15990, 14832, 14092, 12930] dup_rows_per_dst=[3410, 452, 360, 80, 2, 12, 0, 144] combine_rows_per_src=[16384, 1588, 15140, 2888, 15990, 14832, 14092, 12930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83086 payload_rows_remote=70080 dup_rows=15218 dup_rows_remote=15198 dedup_ratio=0.154806 router_tokens_per_expert=[0, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6064, 0, 0, 58, 0, 0, 0, 1159, 0, 0, 80, 8, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4751, 78, 657, 0, 32, 0, 6461, 0, 0, 0, 633, 4543, 0, 0] payload_rows_per_dst=[13006, 16384, 16384, 138, 16384, 0, 10706, 10084] dup_rows_per_dst=[20, 0, 12106, 0, 2494, 0, 330, 268] combine_rows_per_src=[13006, 16384, 16384, 138, 16384, 0, 10706, 10084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93410 payload_rows_remote=82972 dup_rows=4894 dup_rows_remote=4888 dedup_ratio=0.049784 router_tokens_per_expert=[2, 1, 1, 515, 0, 2, 0, 8187, 5216, 0, 0, 3, 0, 3, 0, 0, 3590, 6316, 4, 0, 3807, 0, 0, 1691, 1843, 0, 0, 458, 0, 0, 0, 0, 0, 342, 1, 0, 128, 0, 0, 0, 6, 1, 7588, 8185, 0, 0, 0, 3, 3, 0, 1, 0, 727, 1, 1, 0, 58, 0, 0, 0, 0, 468, 0, 0] payload_rows_per_dst=[16380, 10438, 9828, 10466, 16372, 15186, 1464, 13276] dup_rows_per_dst=[1036, 6, 742, 1750, 940, 10, 2, 408] combine_rows_per_src=[16380, 10438, 9828, 10466, 16372, 15186, 1464, 13276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94248 payload_rows_remote=93802 dup_rows=4056 dup_rows_remote=4056 dedup_ratio=0.041260 router_tokens_per_expert=[0, 0, 0, 0, 1687, 0, 0, 0, 0, 7916, 0, 0, 8112, 10, 223, 151, 173, 0, 0, 35, 0, 166, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7399, 0, 6979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 8, 0, 0, 7653, 0] payload_rows_per_dst=[16384, 756, 16168, 446, 16384, 14798, 13958, 15354] dup_rows_per_dst=[3374, 12, 412, 0, 4, 0, 0, 254] combine_rows_per_src=[16384, 756, 16168, 446, 16384, 14798, 13958, 15354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224116736 dispatch_meta_bytes=299960 combine_payload_bytes=224116736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=224416696 ingress_bytes=293322752 link_bytes=293322752 token_bytes=448533432 weight_movement_bytes=69206016 total_nvlink_bytes=517739448 weight_to_token_ratio=0.154294 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103749901e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=240091136 dispatch_meta_bytes=321104 combine_payload_bytes=240091136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240412240 ingress_bytes=274694144 link_bytes=274694144 token_bytes=480503376 weight_movement_bytes=34603008 total_nvlink_bytes=515106384 weight_to_token_ratio=0.072014 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.033651949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=337494016 dispatch_meta_bytes=334072 combine_payload_bytes=337494016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=337828088 ingress_bytes=406700032 link_bytes=406700032 token_bytes=675322104 weight_movement_bytes=69206016 total_nvlink_bytes=744528120 weight_to_token_ratio=0.102479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530379478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=322371584 dispatch_meta_bytes=331216 combine_payload_bytes=322371584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322702800 ingress_bytes=356974592 link_bytes=356974592 token_bytes=645074384 weight_movement_bytes=34603008 total_nvlink_bytes=679677392 weight_to_token_ratio=0.053642 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343266651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=328138752 dispatch_meta_bytes=327552 combine_payload_bytes=328138752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328466304 ingress_bytes=362741760 link_bytes=362741760 token_bytes=656605056 weight_movement_bytes=34603008 total_nvlink_bytes=691208064 weight_to_token_ratio=0.052700 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364968040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287047680 dispatch_meta_bytes=341112 combine_payload_bytes=287047680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287388792 ingress_bytes=321650688 link_bytes=321650688 token_bytes=574436472 weight_movement_bytes=34603008 total_nvlink_bytes=609039480 weight_to_token_ratio=0.060238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210345644e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=339853312 dispatch_meta_bytes=351440 combine_payload_bytes=339853312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=340204752 ingress_bytes=374456320 link_bytes=374456320 token_bytes=680058064 weight_movement_bytes=34603008 total_nvlink_bytes=714661072 weight_to_token_ratio=0.050882 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409048986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=384212992 dispatch_meta_bytes=391432 combine_payload_bytes=384212992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=384604424 ingress_bytes=453419008 link_bytes=453419008 token_bytes=768817416 weight_movement_bytes=69206016 total_nvlink_bytes=838023432 weight_to_token_ratio=0.090016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.706179224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91262 payload_rows_remote=75546 dup_rows=7042 dup_rows_remote=694 dedup_ratio=0.071635 router_tokens_per_expert=[1, 0, 0, 0, 2021, 0, 0, 0, 0, 7911, 0, 0, 8117, 15, 247, 140, 132, 1, 2, 19, 0, 69, 0, 1, 0, 1, 0, 184, 0, 0, 8188, 0, 0, 72, 0, 4, 0, 0, 1, 0, 0, 7, 0, 0, 2, 0, 7310, 0, 7028, 0, 1, 1, 0, 2, 0, 0, 0, 0, 128, 1, 0, 0, 7546, 0] payload_rows_per_dst=[4044, 784, 15976, 16380, 16380, 14636, 7346, 15716] dup_rows_per_dst=[0, 20, 294, 366, 8, 2, 4, 6348] combine_rows_per_src=[4044, 784, 15976, 16380, 16380, 14636, 7346, 15716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97182 payload_rows_remote=83308 dup_rows=1122 dup_rows_remote=1118 dedup_ratio=0.011414 router_tokens_per_expert=[7, 1, 4, 5, 1935, 0, 1, 0, 7, 7910, 0, 0, 8092, 16, 237, 132, 134, 0, 3, 73, 2, 23, 1, 1, 0, 5, 0, 233, 4, 0, 8179, 0, 0, 83, 5, 0, 8, 5, 0, 0, 0, 3, 0, 0, 0, 1, 7366, 4, 6930, 1, 0, 2, 0, 0, 0, 6, 2, 0, 98, 12, 0, 4, 7615, 2] payload_rows_per_dst=[3896, 768, 15864, 16374, 16368, 14744, 13874, 15294] dup_rows_per_dst=[10, 16, 430, 468, 18, 4, 4, 172] combine_rows_per_src=[3896, 768, 15864, 16374, 16368, 14744, 13874, 15294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88030 payload_rows_remote=80474 dup_rows=10274 dup_rows_remote=10274 dedup_ratio=0.104513 router_tokens_per_expert=[0, 0, 0, 488, 0, 0, 0, 8192, 5140, 0, 0, 0, 0, 0, 0, 0, 3530, 6241, 0, 0, 3966, 0, 0, 1479, 2148, 0, 0, 343, 0, 0, 0, 0, 0, 421, 0, 0, 103, 0, 0, 0, 0, 0, 7513, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 786, 0, 0, 0, 62, 0, 0, 0, 0, 548, 0, 0] payload_rows_per_dst=[8692, 13302, 9426, 11064, 9380, 7556, 15454, 13156] dup_rows_per_dst=[558, 5088, 592, 1850, 496, 0, 1144, 546] combine_rows_per_src=[8692, 13302, 9426, 11064, 9380, 7556, 15454, 13156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92640 payload_rows_remote=83638 dup_rows=5664 dup_rows_remote=5130 dedup_ratio=0.057617 router_tokens_per_expert=[0, 0, 0, 462, 0, 0, 0, 8192, 5468, 0, 0, 0, 0, 0, 0, 0, 3294, 6398, 0, 0, 3834, 0, 0, 1474, 2077, 0, 0, 497, 0, 0, 0, 0, 0, 453, 0, 0, 72, 0, 0, 0, 0, 0, 7559, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 84, 0, 0, 0, 0, 507, 0, 0] payload_rows_per_dst=[16384, 10936, 9002, 10774, 16338, 3016, 12714, 13476] dup_rows_per_dst=[924, 0, 534, 2042, 1050, 0, 612, 502] combine_rows_per_src=[16384, 10936, 9002, 10774, 16338, 3016, 12714, 13476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92274 payload_rows_remote=75926 dup_rows=6030 dup_rows_remote=4884 dedup_ratio=0.061340 router_tokens_per_expert=[0, 0, 0, 490, 0, 0, 0, 8192, 5291, 0, 0, 0, 0, 0, 0, 0, 3471, 6363, 0, 0, 3691, 0, 0, 1729, 1961, 0, 0, 445, 0, 0, 0, 0, 0, 463, 0, 0, 110, 0, 0, 0, 0, 0, 7519, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 93, 0, 0, 0, 0, 554, 0, 0] payload_rows_per_dst=[16384, 10582, 9686, 10372, 16348, 0, 15454, 13448] dup_rows_per_dst=[980, 0, 714, 1822, 1146, 0, 796, 572] combine_rows_per_src=[16384, 10582, 9686, 10372, 16348, 0, 15454, 13448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62762 payload_rows_remote=46378 dup_rows=35542 dup_rows_remote=31480 dedup_ratio=0.361552 router_tokens_per_expert=[0, 0, 0, 0, 2031, 0, 0, 0, 0, 7855, 0, 0, 8114, 5, 216, 137, 145, 0, 0, 18, 0, 20, 0, 0, 0, 0, 0, 244, 0, 0, 8192, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 7361, 0, 6973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 17, 0, 0, 7546, 0] payload_rows_per_dst=[16384, 16384, 366, 15092, 158, 6590, 7444, 344] dup_rows_per_dst=[4062, 30992, 0, 488, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 366, 15092, 158, 6590, 7444, 344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77794 payload_rows_remote=61928 dup_rows=20510 dup_rows_remote=14790 dedup_ratio=0.208639 router_tokens_per_expert=[0, 0, 5, 219, 1, 0, 3, 1, 0, 1, 0, 4281, 1, 0, 0, 0, 7824, 147, 3, 0, 1, 0, 3, 0, 1, 0, 7788, 0, 0, 0, 0, 8072, 4, 0, 0, 2, 0, 0, 0, 0, 6510, 0, 0, 8185, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 5523, 0, 0, 4, 0, 0, 567, 0, 0, 3] payload_rows_per_dst=[16380, 15866, 15898, 16324, 1130, 4, 11048, 1144] dup_rows_per_dst=[448, 5720, 58, 14280, 0, 0, 0, 4] combine_rows_per_src=[16380, 15866, 15898, 16324, 1130, 4, 11048, 1144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92426 payload_rows_remote=81678 dup_rows=5878 dup_rows_remote=4284 dedup_ratio=0.059794 router_tokens_per_expert=[0, 0, 0, 523, 0, 0, 0, 8192, 5311, 0, 0, 0, 0, 0, 0, 0, 3642, 6017, 0, 0, 3849, 0, 0, 1823, 1933, 0, 0, 389, 0, 0, 0, 0, 0, 357, 0, 0, 71, 0, 0, 0, 0, 0, 7487, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 84, 0, 0, 0, 0, 559, 0, 0] payload_rows_per_dst=[16384, 10622, 10148, 10748, 16376, 0, 15362, 12786] dup_rows_per_dst=[1046, 0, 782, 1594, 856, 0, 1066, 534] combine_rows_per_src=[16384, 10622, 10148, 10748, 16376, 0, 15362, 12786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=309436416 dispatch_meta_bytes=304960 combine_payload_bytes=309436416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=309741376 ingress_bytes=378642432 link_bytes=378642432 token_bytes=619177792 weight_movement_bytes=69206016 total_nvlink_bytes=688383808 weight_to_token_ratio=0.111771 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424800988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=341229568 dispatch_meta_bytes=337704 combine_payload_bytes=341229568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=341567272 ingress_bytes=410435584 link_bytes=410435584 token_bytes=682796840 weight_movement_bytes=69206016 total_nvlink_bytes=752002856 weight_to_token_ratio=0.101357 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544436060e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=329621504 dispatch_meta_bytes=362992 combine_payload_bytes=329621504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329984496 ingress_bytes=433430528 link_bytes=433430528 token_bytes=659606000 weight_movement_bytes=103809024 total_nvlink_bytes=763415024 weight_to_token_ratio=0.157380 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630964183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=310992896 dispatch_meta_bytes=323240 combine_payload_bytes=310992896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311316136 ingress_bytes=345595904 link_bytes=345595904 token_bytes=622309032 weight_movement_bytes=34603008 total_nvlink_bytes=656912040 weight_to_token_ratio=0.055604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300449564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=342581248 dispatch_meta_bytes=355072 combine_payload_bytes=342581248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=342936320 ingress_bytes=446390272 link_bytes=446390272 token_bytes=685517568 weight_movement_bytes=103809024 total_nvlink_bytes=789326592 weight_to_token_ratio=0.151432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.679730656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=189964288 dispatch_meta_bytes=311432 combine_payload_bytes=189964288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=190275720 ingress_bytes=224567296 link_bytes=224567296 token_bytes=380240008 weight_movement_bytes=34603008 total_nvlink_bytes=414843016 weight_to_token_ratio=0.091003 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.450286557e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=253657088 dispatch_meta_bytes=306872 combine_payload_bytes=253657088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253963960 ingress_bytes=288260096 link_bytes=288260096 token_bytes=507621048 weight_movement_bytes=34603008 total_nvlink_bytes=542224056 weight_to_token_ratio=0.068167 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.084699534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=334553088 dispatch_meta_bytes=343848 combine_payload_bytes=334553088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334896936 ingress_bytes=369156096 link_bytes=369156096 token_bytes=669450024 weight_movement_bytes=34603008 total_nvlink_bytes=704053032 weight_to_token_ratio=0.051689 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389104670e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78062 payload_rows_remote=64908 dup_rows=20242 dup_rows_remote=19686 dedup_ratio=0.205912 router_tokens_per_expert=[1, 0, 0, 614, 0, 0, 0, 8188, 5244, 0, 0, 0, 0, 0, 0, 1, 3759, 6223, 0, 1, 4359, 0, 0, 1233, 1768, 0, 1, 390, 0, 0, 0, 0, 0, 322, 2, 0, 75, 0, 0, 1, 2, 0, 7465, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 681, 0, 0, 0, 58, 0, 0, 0, 0, 574, 0, 0] payload_rows_per_dst=[1228, 10490, 9492, 11120, 800, 16384, 15394, 13154] dup_rows_per_dst=[2, 0, 494, 1916, 0, 16376, 898, 556] combine_rows_per_src=[1228, 10490, 9492, 11120, 800, 16384, 15394, 13154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80662 payload_rows_remote=65276 dup_rows=17642 dup_rows_remote=16500 dedup_ratio=0.179464 router_tokens_per_expert=[1, 1, 0, 496, 2, 1, 0, 8178, 5217, 0, 4, 2, 8, 10, 0, 1, 3639, 6278, 10, 0, 3759, 0, 6, 1644, 1966, 0, 0, 400, 3, 0, 0, 0, 0, 362, 1, 0, 70, 0, 1, 0, 3, 4, 7482, 8135, 1, 0, 0, 6, 0, 0, 0, 1, 772, 4, 4, 1, 121, 0, 11, 0, 0, 547, 0, 0] payload_rows_per_dst=[1002, 12478, 10832, 10510, 866, 16308, 15386, 13280] dup_rows_per_dst=[0, 3482, 2312, 1746, 2, 8324, 1142, 634] combine_rows_per_src=[1002, 12478, 10832, 10510, 866, 16308, 15386, 13280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96640 payload_rows_remote=80472 dup_rows=1664 dup_rows_remote=1658 dedup_ratio=0.016927 router_tokens_per_expert=[0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4123, 0, 0, 0, 0, 7894, 143, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 0, 0, 0, 0, 8084, 0, 0, 0, 3, 0, 0, 0, 0, 6504, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5589, 0, 0, 0, 0, 0, 629, 0, 0, 0] payload_rows_per_dst=[16384, 8246, 15980, 0, 16168, 13008, 11178, 15676] dup_rows_per_dst=[306, 0, 94, 0, 6, 0, 0, 1258] combine_rows_per_src=[16384, 8246, 15980, 0, 16168, 13008, 11178, 15676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94796 payload_rows_remote=80756 dup_rows=3508 dup_rows_remote=1318 dedup_ratio=0.035685 router_tokens_per_expert=[0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 7846, 190, 2, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 6780, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 0, 0, 0, 0, 462, 0, 0, 0] payload_rows_per_dst=[13764, 7818, 15956, 0, 16184, 14040, 11392, 15642] dup_rows_per_dst=[274, 0, 120, 0, 0, 2190, 0, 924] combine_rows_per_src=[13764, 7818, 15956, 0, 16184, 14040, 11392, 15642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93336 payload_rows_remote=77270 dup_rows=4968 dup_rows_remote=3914 dedup_ratio=0.050537 router_tokens_per_expert=[0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 4302, 0, 0, 0, 0, 7961, 113, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 8107, 0, 0, 0, 2, 0, 0, 0, 0, 6407, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5636, 0, 0, 0, 0, 0, 499, 0, 0, 0] payload_rows_per_dst=[16384, 12418, 16066, 4362, 16214, 5134, 11272, 11486] dup_rows_per_dst=[296, 2894, 1054, 0, 4, 0, 0, 720] combine_rows_per_src=[16384, 12418, 16066, 4362, 16214, 5134, 11272, 11486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86412 payload_rows_remote=70028 dup_rows=11892 dup_rows_remote=10972 dedup_ratio=0.120972 router_tokens_per_expert=[0, 0, 0, 460, 0, 0, 0, 8192, 5440, 0, 0, 0, 0, 0, 0, 0, 3471, 6183, 0, 0, 3840, 0, 0, 1555, 2102, 0, 0, 356, 0, 0, 0, 0, 0, 425, 0, 0, 84, 0, 0, 0, 0, 0, 7572, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 112, 0, 0, 0, 0, 494, 0, 0] payload_rows_per_dst=[16384, 10880, 15248, 9086, 16316, 15144, 1416, 1938] dup_rows_per_dst=[920, 0, 8472, 1444, 1018, 0, 0, 38] combine_rows_per_src=[16384, 10880, 15248, 9086, 16316, 15144, 1416, 1938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88902 payload_rows_remote=72698 dup_rows=9402 dup_rows_remote=5792 dedup_ratio=0.095642 router_tokens_per_expert=[1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1894, 0, 0, 0, 0, 2, 2458, 0, 0, 0, 1, 0, 3, 3, 0, 8011, 0, 0, 0, 631, 0, 4, 6, 3, 0, 2, 0, 0, 587, 0, 0, 0, 7995, 8184, 3040, 0, 0, 2, 0, 0, 33, 0, 8184, 0, 0, 0, 8098, 1, 4, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16204, 16384, 7348, 16384, 0, 15260, 948] dup_rows_per_dst=[2, 3610, 4912, 0, 810, 0, 66, 2] combine_rows_per_src=[16374, 16204, 16384, 7348, 16384, 0, 15260, 948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96798 payload_rows_remote=96798 dup_rows=1506 dup_rows_remote=1506 dedup_ratio=0.015320 router_tokens_per_expert=[0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 7903, 195, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 6621, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 0, 0, 0, 0, 538, 0, 0, 0] payload_rows_per_dst=[16384, 7924, 16034, 0, 16186, 13242, 11426, 15602] dup_rows_per_dst=[268, 0, 162, 0, 0, 0, 0, 1076] combine_rows_per_src=[16384, 7924, 16034, 0, 16186, 13242, 11426, 15602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=338376 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266201544 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532064712 weight_movement_bytes=34603008 total_nvlink_bytes=566667720 weight_to_token_ratio=0.065035 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=329613312 dispatch_meta_bytes=328520 combine_payload_bytes=329613312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329941832 ingress_bytes=364216320 link_bytes=364216320 token_bytes=659555144 weight_movement_bytes=34603008 total_nvlink_bytes=694158152 weight_to_token_ratio=0.052464 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370516690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=327104 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267697600 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535068096 weight_movement_bytes=34603008 total_nvlink_bytes=569671104 weight_to_token_ratio=0.064670 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=330776576 dispatch_meta_bytes=328296 combine_payload_bytes=330776576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=331104872 ingress_bytes=469188608 link_bytes=469188608 token_bytes=661881448 weight_movement_bytes=138412032 total_nvlink_bytes=800293480 weight_to_token_ratio=0.209119 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.765518959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=316497920 dispatch_meta_bytes=324736 combine_payload_bytes=316497920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316822656 ingress_bytes=385703936 link_bytes=385703936 token_bytes=633320576 weight_movement_bytes=69206016 total_nvlink_bytes=702526592 weight_to_token_ratio=0.109275 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451372859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286834688 dispatch_meta_bytes=324000 combine_payload_bytes=286834688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=287158688 ingress_bytes=390643712 link_bytes=390643712 token_bytes=573993376 weight_movement_bytes=103809024 total_nvlink_bytes=677802400 weight_to_token_ratio=0.180854 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469960839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=297771008 dispatch_meta_bytes=313960 combine_payload_bytes=297771008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298084968 ingress_bytes=332374016 link_bytes=332374016 token_bytes=595855976 weight_movement_bytes=34603008 total_nvlink_bytes=630458984 weight_to_token_ratio=0.058073 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250696664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=396484608 dispatch_meta_bytes=393216 combine_payload_bytes=396484608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=396877824 ingress_bytes=500293632 link_bytes=500293632 token_bytes=793362432 weight_movement_bytes=103809024 total_nvlink_bytes=897171456 weight_to_token_ratio=0.130847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.882564660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93680 payload_rows_remote=77296 dup_rows=4624 dup_rows_remote=3588 dedup_ratio=0.047038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1900, 0, 0, 0, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 726, 0, 0, 0, 7984, 8192, 3324, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 6024, 4148, 14872, 16384, 3244, 16384, 16240] dup_rows_per_dst=[0, 482, 0, 2984, 1036, 0, 122, 0] combine_rows_per_src=[16384, 6024, 4148, 14872, 16384, 3244, 16384, 16240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87202 payload_rows_remote=71522 dup_rows=11102 dup_rows_remote=9982 dedup_ratio=0.112935 router_tokens_per_expert=[0, 1, 0, 162, 1, 0, 0, 0, 3, 3, 0, 4126, 0, 0, 3, 0, 7953, 119, 4, 0, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 4, 0, 8092, 1, 1, 0, 9, 0, 3, 0, 1, 6585, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 0, 0, 0, 560, 0, 0, 0] payload_rows_per_dst=[328, 8264, 246, 15910, 16194, 16382, 14198, 15680] dup_rows_per_dst=[0, 6, 0, 4, 20, 5206, 4746, 1120] combine_rows_per_src=[328, 8264, 246, 15910, 16194, 16382, 14198, 15680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84286 payload_rows_remote=72838 dup_rows=14018 dup_rows_remote=14014 dedup_ratio=0.142598 router_tokens_per_expert=[0, 0, 15, 187, 4, 0, 2, 4, 0, 0, 0, 4017, 2, 11, 0, 2, 7947, 146, 12, 9, 0, 0, 6, 1, 1, 0, 7804, 0, 0, 0, 1, 8073, 10, 0, 0, 1, 0, 2, 0, 0, 6380, 0, 0, 8168, 0, 1, 0, 0, 3, 1, 1, 0, 0, 2, 5719, 0, 0, 11, 0, 0, 605, 0, 1, 3] payload_rows_per_dst=[410, 8060, 504, 15710, 16168, 16348, 11448, 15638] dup_rows_per_dst=[14, 4, 32, 0, 4, 12750, 4, 1210] combine_rows_per_src=[410, 8060, 504, 15710, 16168, 16348, 11448, 15638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93156 payload_rows_remote=81058 dup_rows=5148 dup_rows_remote=1770 dedup_ratio=0.052368 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 7972, 8192, 3074, 0, 0, 0, 0, 0, 65, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 3400, 4816, 16232, 7630, 12098, 16384, 16232] dup_rows_per_dst=[0, 0, 0, 1144, 496, 3378, 130, 0] combine_rows_per_src=[16364, 3400, 4816, 16232, 7630, 12098, 16384, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88856 payload_rows_remote=72472 dup_rows=9448 dup_rows_remote=4964 dedup_ratio=0.096110 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1962, 0, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 0, 8020, 8192, 2979, 0, 0, 0, 0, 0, 56, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16244, 16384, 7112, 16384, 0, 112, 16236] dup_rows_per_dst=[0, 3782, 4484, 0, 1182, 0, 0, 0] combine_rows_per_src=[16384, 16244, 16384, 7112, 16384, 0, 112, 16236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77274 payload_rows_remote=60890 dup_rows=21030 dup_rows_remote=20750 dedup_ratio=0.213928 router_tokens_per_expert=[0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 7859, 191, 0, 0, 0, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 8104, 0, 0, 0, 13, 0, 0, 0, 0, 6456, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 562, 0, 0, 0] payload_rows_per_dst=[16384, 15800, 15964, 16384, 26, 0, 11592, 1124] dup_rows_per_dst=[280, 5284, 136, 15330, 0, 0, 0, 0] combine_rows_per_src=[16384, 15800, 15964, 16384, 26, 0, 11592, 1124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92764 payload_rows_remote=76380 dup_rows=5540 dup_rows_remote=5538 dedup_ratio=0.056356 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 1, 3, 7648, 0, 2, 0, 3, 0, 0, 0, 4, 0, 0, 4962, 0, 0, 0, 0, 1750, 0, 1, 0, 0, 8186, 8186, 0, 5, 1, 0, 3444, 4, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 6753, 8188, 0, 0] payload_rows_per_dst=[16372, 16384, 15304, 16384, 12420, 10, 15890, 0] dup_rows_per_dst=[0, 2, 4, 2, 1006, 2, 4524, 0] combine_rows_per_src=[16372, 16384, 15304, 16384, 12420, 10, 15890, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89398 payload_rows_remote=82678 dup_rows=8906 dup_rows_remote=8906 dedup_ratio=0.090597 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 0, 7987, 8192, 2650, 0, 0, 0, 0, 0, 56, 0, 8192, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16222, 13680, 6720, 16384, 0, 3786, 16222] dup_rows_per_dst=[0, 4310, 3222, 0, 1362, 0, 12, 0] combine_rows_per_src=[16384, 16222, 13680, 6720, 16384, 0, 3786, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=298344448 dispatch_meta_bytes=347408 combine_payload_bytes=298344448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298691856 ingress_bytes=332947456 link_bytes=332947456 token_bytes=597036304 weight_movement_bytes=34603008 total_nvlink_bytes=631639312 weight_to_token_ratio=0.057958 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252854472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316604416 dispatch_meta_bytes=323536 combine_payload_bytes=316604416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316927952 ingress_bytes=351207424 link_bytes=351207424 token_bytes=633532368 weight_movement_bytes=34603008 total_nvlink_bytes=668135376 weight_to_token_ratio=0.054619 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321565262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292954112 dispatch_meta_bytes=326016 combine_payload_bytes=292954112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293280128 ingress_bytes=327557120 link_bytes=327557120 token_bytes=586234240 weight_movement_bytes=34603008 total_nvlink_bytes=620837248 weight_to_token_ratio=0.059026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232571072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=332013568 dispatch_meta_bytes=331312 combine_payload_bytes=332013568 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=332344880 ingress_bytes=470425600 link_bytes=470425600 token_bytes=664358448 weight_movement_bytes=138412032 total_nvlink_bytes=802770480 weight_to_token_ratio=0.208339 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.770173660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=296845312 dispatch_meta_bytes=309744 combine_payload_bytes=296845312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297155056 ingress_bytes=331448320 link_bytes=331448320 token_bytes=594000368 weight_movement_bytes=34603008 total_nvlink_bytes=628603376 weight_to_token_ratio=0.058254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247213344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=249405440 dispatch_meta_bytes=326560 combine_payload_bytes=249405440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249732000 ingress_bytes=284008448 link_bytes=284008448 token_bytes=499137440 weight_movement_bytes=34603008 total_nvlink_bytes=533740448 weight_to_token_ratio=0.069326 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.068700925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=312852480 dispatch_meta_bytes=327672 combine_payload_bytes=312852480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313180152 ingress_bytes=347455488 link_bytes=347455488 token_bytes=626032632 weight_movement_bytes=34603008 total_nvlink_bytes=660635640 weight_to_token_ratio=0.055273 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307447029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=338649088 dispatch_meta_bytes=366336 combine_payload_bytes=338649088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=339015424 ingress_bytes=407855104 link_bytes=407855104 token_bytes=677664512 weight_movement_bytes=69206016 total_nvlink_bytes=746870528 weight_to_token_ratio=0.102124 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.534725921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75670 payload_rows_remote=59320 dup_rows=22634 dup_rows_remote=22490 dedup_ratio=0.230245 router_tokens_per_expert=[3, 2, 0, 0, 2, 2, 1, 1, 1, 0, 1, 2098, 3, 1, 0, 0, 7, 2500, 0, 0, 0, 0, 0, 8, 6, 1, 8004, 1, 0, 1, 626, 0, 13, 9, 3, 0, 8, 1, 0, 641, 0, 7, 1, 7965, 8177, 2711, 0, 1, 2, 0, 1, 74, 0, 8170, 0, 0, 0, 8080, 4, 12, 0, 1, 1, 1] payload_rows_per_dst=[14, 4208, 5024, 16192, 1326, 16370, 16350, 16186] dup_rows_per_dst=[8, 0, 6, 1086, 24, 21354, 144, 12] combine_rows_per_src=[14, 4208, 5024, 16192, 1326, 16370, 16350, 16186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97122 payload_rows_remote=84344 dup_rows=1182 dup_rows_remote=0 dedup_ratio=0.012024 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5030, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6246, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15324, 16382, 12778, 0, 7378, 12492] dup_rows_per_dst=[0, 0, 0, 0, 1182, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15324, 16382, 12778, 0, 7378, 12492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74928 payload_rows_remote=58704 dup_rows=23376 dup_rows_remote=23374 dedup_ratio=0.237793 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 2, 0, 2, 0, 1837, 2, 0, 0, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 586, 0, 1, 0, 0, 1, 2, 0, 0, 762, 0, 1, 0, 7961, 8188, 3151, 0, 0, 0, 2, 0, 60, 0, 8189, 0, 0, 0, 8111, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[6, 3682, 4494, 16230, 1528, 16382, 16382, 16224] dup_rows_per_dst=[0, 0, 0, 1030, 4, 22220, 120, 2] combine_rows_per_src=[6, 3682, 4494, 16230, 1528, 16382, 16382, 16224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97208 payload_rows_remote=94840 dup_rows=1096 dup_rows_remote=1046 dedup_ratio=0.011149 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4843, 0, 0, 0, 0, 1893, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 3554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6642, 8192, 0, 0] payload_rows_per_dst=[16380, 16342, 12926, 16332, 12460, 2368, 7108, 13292] dup_rows_per_dst=[0, 0, 0, 0, 1012, 50, 0, 34] combine_rows_per_src=[16380, 16342, 12926, 16332, 12460, 2368, 7108, 13292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242974720 dispatch_meta_bytes=327240 combine_payload_bytes=242974720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=243301960 ingress_bytes=312180736 link_bytes=312180736 token_bytes=486276680 weight_movement_bytes=69206016 total_nvlink_bytes=555482696 weight_to_token_ratio=0.142318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174710977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93096 payload_rows_remote=77862 dup_rows=5208 dup_rows_remote=5208 dedup_ratio=0.052979 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5079, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 8192, 8189, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6709, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15234, 16378, 12876, 0, 15840, 0] dup_rows_per_dst=[0, 0, 0, 0, 1044, 0, 4164, 0] combine_rows_per_src=[16384, 16384, 15234, 16378, 12876, 0, 15840, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88972 payload_rows_remote=72588 dup_rows=9332 dup_rows_remote=9332 dedup_ratio=0.094930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 7998, 8192, 3096, 0, 0, 0, 0, 0, 41, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16228, 16384, 7354, 16382, 3258, 12982, 0] dup_rows_per_dst=[0, 3908, 4366, 0, 988, 0, 70, 0] combine_rows_per_src=[16384, 16228, 16384, 7354, 16382, 3258, 12982, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97002 payload_rows_remote=80618 dup_rows=1302 dup_rows_remote=1302 dedup_ratio=0.013245 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4975, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15160, 16384, 12646, 0, 7680, 12364] dup_rows_per_dst=[0, 0, 0, 0, 974, 0, 328, 0] combine_rows_per_src=[16384, 16384, 15160, 16384, 12646, 0, 7680, 12364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57298 payload_rows_remote=42488 dup_rows=41006 dup_rows_remote=41006 dedup_ratio=0.417135 router_tokens_per_expert=[8186, 0, 3, 0, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 7404, 7791, 1, 4917, 0, 8107, 0, 3, 0, 0, 0, 2, 0, 251, 1, 0, 0, 0, 2, 0, 0, 2400, 0, 346, 0, 1, 3, 0, 4, 3, 427, 0, 498, 0, 0, 1, 0, 1, 3, 0, 0, 2, 172, 418, 0, 5, 4, 0, 5] payload_rows_per_dst=[16384, 14810, 16382, 508, 5404, 1756, 856, 1198] dup_rows_per_dst=[16374, 0, 24410, 0, 92, 116, 0, 14] combine_rows_per_src=[16384, 14810, 16382, 508, 5404, 1756, 856, 1198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=345473024 dispatch_meta_bytes=337376 combine_payload_bytes=345473024 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=345810400 ingress_bytes=414679040 link_bytes=414679040 token_bytes=691283424 weight_movement_bytes=69206016 total_nvlink_bytes=760489440 weight_to_token_ratio=0.100112 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560403843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=240451584 dispatch_meta_bytes=328312 combine_payload_bytes=240451584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=240779896 ingress_bytes=309657600 link_bytes=309657600 token_bytes=481231480 weight_movement_bytes=69206016 total_nvlink_bytes=550437496 weight_to_token_ratio=0.143810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165216619e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=388464640 dispatch_meta_bytes=383544 combine_payload_bytes=388464640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=388848184 ingress_bytes=492273664 link_bytes=492273664 token_bytes=777312824 weight_movement_bytes=103809024 total_nvlink_bytes=881121848 weight_to_token_ratio=0.133549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.852386166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=318922752 dispatch_meta_bytes=332280 combine_payload_bytes=318922752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=319255032 ingress_bytes=388128768 link_bytes=388128768 token_bytes=638177784 weight_movement_bytes=69206016 total_nvlink_bytes=707383800 weight_to_token_ratio=0.108443 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460497307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=297320448 dispatch_meta_bytes=327680 combine_payload_bytes=297320448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297648128 ingress_bytes=331923456 link_bytes=331923456 token_bytes=594968576 weight_movement_bytes=34603008 total_nvlink_bytes=629571584 weight_to_token_ratio=0.058159 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.249001243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330211328 dispatch_meta_bytes=327680 combine_payload_bytes=330211328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330539008 ingress_bytes=364814336 link_bytes=364814336 token_bytes=660750336 weight_movement_bytes=34603008 total_nvlink_bytes=695353344 weight_to_token_ratio=0.052369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372766976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=174030848 dispatch_meta_bytes=333976 combine_payload_bytes=174030848 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=174364824 ingress_bytes=243236864 link_bytes=243236864 token_bytes=348395672 weight_movement_bytes=69206016 total_nvlink_bytes=417601688 weight_to_token_ratio=0.198642 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.152807371e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=52810 payload_rows_remote=45594 dup_rows=45494 dup_rows_remote=45442 dedup_ratio=0.462789 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 16, 0, 0, 0, 2, 0, 10, 7556, 1, 0, 0, 10, 0, 0, 0, 16, 0, 0, 4940, 1, 0, 0, 0, 1991, 1, 0, 0, 0, 8172, 8170, 3, 15, 8, 0, 3594, 28, 0, 0, 0, 0, 11, 1, 2, 0, 0, 0, 6415, 8177, 0, 1] payload_rows_per_dst=[12, 32, 26, 52, 12732, 16382, 7216, 16358] dup_rows_per_dst=[0, 10, 0, 0, 1134, 31466, 52, 12832] combine_rows_per_src=[12, 32, 26, 52, 12732, 16382, 7216, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94720 payload_rows_remote=79790 dup_rows=3584 dup_rows_remote=3276 dedup_ratio=0.036458 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7434, 7862, 0, 4891, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 2363, 0, 363, 2, 0, 0, 0, 0, 0, 323, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 400, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14868, 0, 16384, 14930, 16384, 16242, 15912] dup_rows_per_dst=[0, 0, 0, 504, 308, 1930, 0, 842] combine_rows_per_src=[0, 14868, 0, 16384, 14930, 16384, 16242, 15912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51308 payload_rows_remote=34926 dup_rows=46996 dup_rows_remote=32446 dedup_ratio=0.478068 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 1, 0, 2, 1, 0, 0, 0, 0, 1, 4620, 0, 0, 0, 0, 1801, 0, 1, 1, 1, 8190, 8181, 0, 0, 2, 0, 4028, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 6489, 8189, 0, 0] payload_rows_per_dst=[2, 0, 2, 8, 10472, 16384, 8058, 16382] dup_rows_per_dst=[2, 0, 0, 0, 802, 31642, 0, 14550] combine_rows_per_src=[2, 0, 2, 8, 10472, 16384, 8058, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94556 payload_rows_remote=78172 dup_rows=3748 dup_rows_remote=1558 dedup_ratio=0.038127 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7484, 7845, 0, 4774, 0, 8137, 0, 0, 0, 0, 0, 1, 0, 255, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 326, 1, 0, 0, 0, 0, 0, 528, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 372, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14968, 0, 16384, 14634, 16384, 16274, 15912] dup_rows_per_dst=[0, 0, 0, 512, 222, 2190, 0, 824] combine_rows_per_src=[0, 14968, 0, 16384, 14634, 16384, 16274, 15912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=186753024 dispatch_meta_bytes=364144 combine_payload_bytes=186753024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=187117168 ingress_bytes=221356032 link_bytes=221356032 token_bytes=373870192 weight_movement_bytes=34603008 total_nvlink_bytes=408473200 weight_to_token_ratio=0.092554 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.329449278e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85348 payload_rows_remote=68964 dup_rows=12956 dup_rows_remote=12956 dedup_ratio=0.131795 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5110, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6367, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 15748, 16382, 4638, 0, 15814, 0] dup_rows_per_dst=[0, 0, 8576, 0, 128, 0, 4252, 0] combine_rows_per_src=[16384, 16382, 15748, 16382, 4638, 0, 15814, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90848 payload_rows_remote=84878 dup_rows=7456 dup_rows_remote=5286 dedup_ratio=0.075846 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7464, 7918, 0, 4660, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 2546, 0, 388, 0, 0, 0, 0, 0, 0, 506, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 386, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14928, 5970, 12524, 11104, 1808, 16242, 11888] dup_rows_per_dst=[3990, 0, 2170, 370, 206, 142, 0, 578] combine_rows_per_src=[16384, 14928, 5970, 12524, 11104, 1808, 16242, 11888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94558 payload_rows_remote=78174 dup_rows=3746 dup_rows_remote=3256 dedup_ratio=0.038106 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7460, 7905, 0, 4548, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 2524, 0, 369, 1, 0, 0, 0, 0, 0, 584, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 375, 0, 0, 0, 0, 0] payload_rows_per_dst=[18, 14920, 0, 16384, 14618, 16382, 16258, 15978] dup_rows_per_dst=[0, 0, 0, 490, 266, 2116, 0, 874] combine_rows_per_src=[18, 14920, 0, 16384, 14618, 16382, 16258, 15978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74896 payload_rows_remote=62916 dup_rows=23408 dup_rows_remote=19952 dedup_ratio=0.238118 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4422, 0, 1, 0, 8183, 0, 0, 0, 2, 0, 7937, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 8187, 5, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4006, 0, 0, 3, 1, 0, 0, 0, 0, 2, 1, 0, 8187, 0, 5, 11, 0, 0] payload_rows_per_dst=[16374, 11980, 9790, 16380, 12, 16378, 3950, 32] dup_rows_per_dst=[0, 3456, 0, 11936, 0, 8014, 0, 2] combine_rows_per_src=[16374, 11980, 9790, 16380, 12, 16378, 3950, 32] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=326819840 dispatch_meta_bytes=332264 combine_payload_bytes=326819840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327152104 ingress_bytes=361422848 link_bytes=361422848 token_bytes=653971944 weight_movement_bytes=34603008 total_nvlink_bytes=688574952 weight_to_token_ratio=0.052912 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360005080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=143056896 dispatch_meta_bytes=269488 combine_payload_bytes=143056896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=143326384 ingress_bytes=246865920 link_bytes=246865920 token_bytes=286383280 weight_movement_bytes=103809024 total_nvlink_bytes=390192304 weight_to_token_ratio=0.362483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.289365826e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320192512 dispatch_meta_bytes=318920 combine_payload_bytes=320192512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320511432 ingress_bytes=354795520 link_bytes=354795520 token_bytes=640703944 weight_movement_bytes=34603008 total_nvlink_bytes=675306952 weight_to_token_ratio=0.054008 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335066978e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282476544 dispatch_meta_bytes=327680 combine_payload_bytes=282476544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282804224 ingress_bytes=317079552 link_bytes=317079552 token_bytes=565280768 weight_movement_bytes=34603008 total_nvlink_bytes=599883776 weight_to_token_ratio=0.061214 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193144827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=347660288 dispatch_meta_bytes=360656 combine_payload_bytes=347660288 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=348020944 ingress_bytes=486072320 link_bytes=486072320 token_bytes=695681232 weight_movement_bytes=138412032 total_nvlink_bytes=834093264 weight_to_token_ratio=0.198959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.829051008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=257703936 dispatch_meta_bytes=331472 combine_payload_bytes=257703936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258035408 ingress_bytes=292306944 link_bytes=292306944 token_bytes=515739344 weight_movement_bytes=34603008 total_nvlink_bytes=550342352 weight_to_token_ratio=0.067094 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099927498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320200704 dispatch_meta_bytes=325720 combine_payload_bytes=320200704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320526424 ingress_bytes=354803712 link_bytes=354803712 token_bytes=640727128 weight_movement_bytes=34603008 total_nvlink_bytes=675330136 weight_to_token_ratio=0.054006 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335097804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94536 payload_rows_remote=78338 dup_rows=3768 dup_rows_remote=3768 dedup_ratio=0.038330 router_tokens_per_expert=[8174, 0, 13, 0, 15, 0, 0, 8176, 0, 0, 0, 0, 0, 12, 0, 7474, 7826, 4, 4728, 0, 8083, 0, 0, 0, 0, 0, 2, 0, 236, 12, 0, 0, 1, 3, 0, 0, 2489, 0, 313, 1, 2, 1, 0, 10, 30, 538, 0, 485, 0, 0, 0, 1, 0, 15, 0, 0, 7, 90, 387, 0, 4, 14, 0, 6] payload_rows_per_dst=[34, 14958, 8, 16372, 14718, 16378, 16198, 15870] dup_rows_per_dst=[22, 14, 0, 476, 352, 2106, 0, 798] combine_rows_per_src=[34, 14958, 8, 16372, 14718, 16378, 16198, 15870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98216 payload_rows_remote=81832 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.000895 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4351, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 44, 0, 0] payload_rows_per_dst=[16380, 8702, 16384, 0, 16384, 8092, 15890, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 88] combine_rows_per_src=[16380, 8702, 16384, 0, 16384, 8092, 15890, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94704 payload_rows_remote=78696 dup_rows=3600 dup_rows_remote=2870 dedup_ratio=0.036621 router_tokens_per_expert=[8190, 0, 1, 0, 1, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 7542, 7915, 0, 4715, 0, 8116, 1, 2, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 1, 2, 0, 2455, 0, 240, 6, 1, 0, 0, 1, 0, 506, 0, 570, 0, 0, 1, 0, 0, 0, 0, 0, 0, 101, 348, 0, 3, 1, 0, 1] payload_rows_per_dst=[4, 6100, 8988, 16380, 14606, 16384, 16234, 16008] dup_rows_per_dst=[2, 0, 2, 490, 232, 2144, 0, 730] combine_rows_per_src=[4, 6100, 8988, 16380, 14606, 16384, 16234, 16008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320872448 dispatch_meta_bytes=328424 combine_payload_bytes=320872448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321200872 ingress_bytes=355475456 link_bytes=355475456 token_bytes=642073320 weight_movement_bytes=34603008 total_nvlink_bytes=676676328 weight_to_token_ratio=0.053893 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337625523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98162 payload_rows_remote=90860 dup_rows=142 dup_rows_remote=142 dedup_ratio=0.001444 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4801, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 7861, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 70, 0, 0] payload_rows_per_dst=[16374, 9602, 16384, 10, 16384, 7302, 15722, 16384] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 140] combine_rows_per_src=[16374, 9602, 16384, 10, 16384, 7302, 15722, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=45406 payload_rows_remote=29022 dup_rows=52898 dup_rows_remote=24190 dedup_ratio=0.538106 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7531, 7900, 0, 4151, 0, 8136, 0, 0, 0, 0, 0, 5, 0, 258, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 405, 0, 0, 0, 0, 0, 0, 401, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 281, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 2738, 16384, 526, 6522, 1974, 0, 878] dup_rows_per_dst=[28708, 0, 23990, 0, 90, 110, 0, 0] combine_rows_per_src=[16384, 2738, 16384, 526, 6522, 1974, 0, 878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96028 payload_rows_remote=79644 dup_rows=2276 dup_rows_remote=2276 dedup_ratio=0.023153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4947, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 15, 0, 0] payload_rows_per_dst=[16380, 9894, 16384, 5216, 11168, 10176, 15802, 11008] dup_rows_per_dst=[0, 0, 0, 0, 0, 2252, 0, 24] combine_rows_per_src=[16380, 9894, 16384, 5216, 11168, 10176, 15802, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=335183872 dispatch_meta_bytes=327680 combine_payload_bytes=335183872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335511552 ingress_bytes=404389888 link_bytes=404389888 token_bytes=670695424 weight_movement_bytes=69206016 total_nvlink_bytes=739901440 weight_to_token_ratio=0.103185 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521686592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75352 payload_rows_remote=59210 dup_rows=22952 dup_rows_remote=12788 dedup_ratio=0.233480 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7790, 7789, 328, 3, 0, 3, 0, 4, 2, 3194, 0, 2, 1636, 0, 7667, 1, 6670, 0, 118, 0, 0, 4, 0, 0, 0, 0, 0, 4626, 8133, 0, 0, 1, 0, 0, 0, 2, 0, 710, 456, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] payload_rows_per_dst=[15578, 16142, 13728, 5416, 15966, 2332, 5412, 778] dup_rows_per_dst=[0, 10164, 5880, 68, 6826, 6, 4, 4] combine_rows_per_src=[15578, 16142, 13728, 5416, 15966, 2332, 5412, 778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98202 payload_rows_remote=98202 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5112, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 50, 0, 0] payload_rows_per_dst=[16380, 10224, 16384, 0, 16384, 6992, 15454, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 100] combine_rows_per_src=[16380, 10224, 16384, 0, 16384, 6992, 15454, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322338816 dispatch_meta_bytes=326264 combine_payload_bytes=322338816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322665080 ingress_bytes=356941824 link_bytes=356941824 token_bytes=645003896 weight_movement_bytes=34603008 total_nvlink_bytes=679606904 weight_to_token_ratio=0.053648 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343143348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=372162560 dispatch_meta_bytes=364008 combine_payload_bytes=372162560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=372526568 ingress_bytes=406765568 link_bytes=406765568 token_bytes=744689128 weight_movement_bytes=34603008 total_nvlink_bytes=779292136 weight_to_token_ratio=0.046466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530626085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=118874112 dispatch_meta_bytes=212848 combine_payload_bytes=118874112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=119086960 ingress_bytes=222683136 link_bytes=222683136 token_bytes=237961072 weight_movement_bytes=103809024 total_nvlink_bytes=341770096 weight_to_token_ratio=0.436244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.379387133e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326221824 dispatch_meta_bytes=327680 combine_payload_bytes=326221824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=326549504 ingress_bytes=430030848 link_bytes=430030848 token_bytes=652771328 weight_movement_bytes=103809024 total_nvlink_bytes=756580352 weight_to_token_ratio=0.159028 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.618171461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=242524160 dispatch_meta_bytes=287992 combine_payload_bytes=242524160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=242812152 ingress_bytes=346333184 link_bytes=346333184 token_bytes=485336312 weight_movement_bytes=103809024 total_nvlink_bytes=589145336 weight_to_token_ratio=0.213891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303223889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=402235392 dispatch_meta_bytes=393216 combine_payload_bytes=402235392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=402628608 ingress_bytes=506044416 link_bytes=506044416 token_bytes=804864000 weight_movement_bytes=103809024 total_nvlink_bytes=908673024 weight_to_token_ratio=0.128977 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.904204397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98142 payload_rows_remote=82398 dup_rows=162 dup_rows_remote=158 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 14, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4727, 1, 0, 0, 8168, 0, 0, 0, 10, 0, 7858, 14, 0, 0, 0, 8174, 0, 0, 0, 1, 8174, 17, 0, 0, 2, 0, 4, 0, 0, 16, 1, 0, 3690, 0, 0, 5, 1, 0, 0, 0, 0, 10, 2, 0, 8175, 0, 17, 67, 0, 0] payload_rows_per_dst=[32, 9456, 16356, 16374, 16384, 7414, 15744, 16382] dup_rows_per_dst=[0, 4, 0, 2, 4, 8, 4, 140] combine_rows_per_src=[32, 9456, 16356, 16374, 16384, 7414, 15744, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96422 payload_rows_remote=86592 dup_rows=1882 dup_rows_remote=1882 dedup_ratio=0.019145 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7755, 7709, 332, 0, 0, 0, 0, 0, 0, 2754, 0, 0, 1666, 0, 7988, 0, 6765, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4915, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 561, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[15418, 664, 8372, 13536, 9830, 16382, 15976, 16244] dup_rows_per_dst=[0, 0, 468, 186, 0, 1228, 0, 0] combine_rows_per_src=[15418, 664, 8372, 13536, 9830, 16382, 15976, 16244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88350 payload_rows_remote=71968 dup_rows=9954 dup_rows_remote=5238 dedup_ratio=0.101257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4663, 2, 1, 0, 8189, 0, 0, 8, 0, 2, 7850, 1, 0, 0, 0, 8184, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 3787, 0, 2, 2, 0, 0, 0, 0, 0, 6, 3, 0, 8190, 0, 0, 69, 0, 0] payload_rows_per_dst=[2, 9332, 6604, 16370, 16378, 7578, 15704, 16382] dup_rows_per_dst=[0, 0, 16, 0, 5206, 0, 16, 4716] combine_rows_per_src=[2, 9332, 6604, 16370, 16378, 7578, 15704, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65688 payload_rows_remote=49306 dup_rows=32616 dup_rows_remote=32616 dedup_ratio=0.331787 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8192, 0, 0, 69, 0, 0] payload_rows_per_dst=[16382, 16384, 0, 16384, 0, 16384, 16, 138] dup_rows_per_dst=[0, 9270, 0, 15632, 0, 7714, 0, 0] combine_rows_per_src=[16382, 16384, 0, 16384, 0, 16384, 16, 138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96284 payload_rows_remote=79902 dup_rows=2020 dup_rows_remote=764 dedup_ratio=0.020549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 7818, 330, 0, 0, 0, 0, 0, 0, 2671, 0, 0, 1815, 0, 7749, 0, 6858, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 4823, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 629, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15636, 660, 8468, 13720, 9646, 16382, 15498, 16274] dup_rows_per_dst=[0, 0, 504, 260, 0, 1256, 0, 0] combine_rows_per_src=[15636, 660, 8468, 13720, 9646, 16382, 15498, 16274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=337502208 dispatch_meta_bytes=330224 combine_payload_bytes=337502208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337832432 ingress_bytes=372105216 link_bytes=372105216 token_bytes=675334640 weight_movement_bytes=34603008 total_nvlink_bytes=709937648 weight_to_token_ratio=0.051238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400201971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=354680832 dispatch_meta_bytes=353896 combine_payload_bytes=354680832 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=355034728 ingress_bytes=458489856 link_bytes=458489856 token_bytes=709715560 weight_movement_bytes=103809024 total_nvlink_bytes=813524584 weight_to_token_ratio=0.146268 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725260417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96966 payload_rows_remote=87992 dup_rows=1338 dup_rows_remote=846 dedup_ratio=0.013611 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 7797, 273, 0, 0, 0, 0, 0, 0, 2751, 0, 0, 1982, 0, 7675, 0, 6872, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4781, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 550, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[15594, 12272, 8974, 13744, 9562, 5226, 15350, 16244] dup_rows_per_dst=[0, 422, 492, 124, 0, 300, 0, 0] combine_rows_per_src=[15594, 12272, 8974, 13744, 9562, 5226, 15350, 16244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78984 payload_rows_remote=62602 dup_rows=19320 dup_rows_remote=16364 dedup_ratio=0.196533 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 1, 694, 388, 6, 0, 0, 0, 0, 0, 1090, 0, 1, 4273, 0, 0, 0, 1, 0, 0, 5, 3614, 0, 0, 0, 0, 2, 7757, 4, 0, 0, 0, 8185, 0, 8186, 0, 4, 1, 8186, 4, 4, 3, 0, 0, 2985, 0, 2, 3, 0, 0, 2, 0, 3, 433, 0, 0, 3228, 0, 86] payload_rows_per_dst=[16376, 16382, 15522, 15574, 1682, 16, 5980, 7452] dup_rows_per_dst=[1388, 2956, 7726, 7182, 0, 16, 4, 48] combine_rows_per_src=[16376, 16382, 15522, 15574, 1682, 16, 5980, 7452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96378 payload_rows_remote=82422 dup_rows=1926 dup_rows_remote=1694 dedup_ratio=0.019592 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 7706, 7752, 287, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 1588, 0, 7753, 0, 6978, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 4973, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 588, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15504, 574, 8284, 13956, 9946, 16358, 15506, 16250] dup_rows_per_dst=[2, 0, 452, 232, 0, 1240, 0, 0] combine_rows_per_src=[15504, 574, 8284, 13956, 9946, 16358, 15506, 16250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=294780928 dispatch_meta_bytes=308824 combine_payload_bytes=294780928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295089752 ingress_bytes=363986944 link_bytes=363986944 token_bytes=589870680 weight_movement_bytes=69206016 total_nvlink_bytes=659076696 weight_to_token_ratio=0.117324 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369653567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=201957376 dispatch_meta_bytes=327688 combine_payload_bytes=201957376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202285064 ingress_bytes=236560384 link_bytes=236560384 token_bytes=404242440 weight_movement_bytes=34603008 total_nvlink_bytes=438845448 weight_to_token_ratio=0.085600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.901576803e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=327278592 dispatch_meta_bytes=322664 combine_payload_bytes=327278592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327601256 ingress_bytes=361881600 link_bytes=361881600 token_bytes=654879848 weight_movement_bytes=34603008 total_nvlink_bytes=689482856 weight_to_token_ratio=0.052839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361731327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=360415232 dispatch_meta_bytes=355352 combine_payload_bytes=360415232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=360770584 ingress_bytes=429621248 link_bytes=429621248 token_bytes=721185816 weight_movement_bytes=69206016 total_nvlink_bytes=790391832 weight_to_token_ratio=0.095961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.616630169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=256417792 dispatch_meta_bytes=315864 combine_payload_bytes=256417792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256733656 ingress_bytes=291020800 link_bytes=291020800 token_bytes=513151448 weight_movement_bytes=34603008 total_nvlink_bytes=547754456 weight_to_token_ratio=0.067432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.095087841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=337600512 dispatch_meta_bytes=336464 combine_payload_bytes=337600512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=337936976 ingress_bytes=406806528 link_bytes=406806528 token_bytes=675537488 weight_movement_bytes=69206016 total_nvlink_bytes=744743504 weight_to_token_ratio=0.102446 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530780214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89442 payload_rows_remote=73454 dup_rows=8862 dup_rows_remote=3530 dedup_ratio=0.090149 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 681, 265, 0, 0, 0, 0, 0, 0, 1106, 0, 0, 4236, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 3482, 0, 0, 0, 2, 0, 0, 0, 0, 327, 0, 0, 3437, 0, 72] payload_rows_per_dst=[16384, 11070, 8472, 6202, 15988, 16384, 7310, 7632] dup_rows_per_dst=[1362, 1866, 0, 0, 5332, 4, 258, 40] combine_rows_per_src=[16384, 11070, 8472, 6202, 15988, 16384, 7310, 7632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95964 payload_rows_remote=80732 dup_rows=2340 dup_rows_remote=2330 dedup_ratio=0.023804 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 11, 0, 0, 7767, 7897, 269, 2, 0, 9, 0, 15, 8, 2623, 0, 11, 1658, 0, 7600, 2, 6737, 0, 62, 0, 0, 15, 0, 0, 0, 0, 0, 5170, 8109, 0, 0, 2, 0, 0, 0, 1, 3, 630, 516, 5, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 2, 8] payload_rows_per_dst=[5068, 11044, 8192, 13508, 10340, 16340, 15232, 16240] dup_rows_per_dst=[4, 264, 438, 124, 0, 1498, 10, 2] combine_rows_per_src=[5068, 11044, 8192, 13508, 10340, 16340, 15232, 16240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96176 payload_rows_remote=79910 dup_rows=2128 dup_rows_remote=2126 dedup_ratio=0.021647 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 1, 0, 7768, 7949, 259, 1, 0, 0, 1, 0, 0, 2593, 0, 2, 1594, 0, 7655, 0, 7101, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 4956, 8130, 0, 0, 0, 0, 0, 0, 2, 0, 539, 510, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 2, 1] payload_rows_per_dst=[4, 16056, 8026, 14214, 9912, 16384, 15314, 16266] dup_rows_per_dst=[0, 364, 352, 154, 0, 1254, 2, 2] combine_rows_per_src=[4, 16056, 8026, 14214, 9912, 16384, 15314, 16266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300867584 dispatch_meta_bytes=307936 combine_payload_bytes=300867584 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=301175520 ingress_bytes=508485632 link_bytes=508485632 token_bytes=602043104 weight_movement_bytes=207618048 total_nvlink_bytes=809661152 weight_to_token_ratio=0.344856 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.913390497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83218 payload_rows_remote=66834 dup_rows=15086 dup_rows_remote=15086 dedup_ratio=0.153463 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 711, 195, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 4227, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3123, 0, 76] payload_rows_per_dst=[7240, 2592, 8454, 7214, 16384, 16384, 15546, 9404] dup_rows_per_dst=[488, 390, 0, 0, 5902, 0, 6298, 2008] combine_rows_per_src=[7240, 2592, 8454, 7214, 16384, 16384, 15546, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75644 payload_rows_remote=60148 dup_rows=22660 dup_rows_remote=22660 dedup_ratio=0.230509 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 7748, 280, 0, 0, 0, 0, 0, 0, 2947, 0, 0, 1672, 0, 7891, 0, 6795, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 615, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15496, 16310, 16382, 9062, 16276, 2118, 0, 0] dup_rows_per_dst=[0, 13368, 9292, 0, 0, 0, 0, 0] combine_rows_per_src=[15496, 16310, 16382, 9062, 16276, 2118, 0, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89428 payload_rows_remote=80846 dup_rows=8876 dup_rows_remote=8876 dedup_ratio=0.090291 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 624, 191, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 4291, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 3721, 0, 71] payload_rows_per_dst=[16384, 16384, 8582, 13308, 4604, 16248, 5594, 8324] dup_rows_per_dst=[1248, 2716, 0, 4878, 0, 0, 0, 34] combine_rows_per_src=[16384, 16384, 8582, 13308, 4604, 16248, 5594, 8324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=330678272 dispatch_meta_bytes=332248 combine_payload_bytes=330678272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331010520 ingress_bytes=365281280 link_bytes=365281280 token_bytes=661688792 weight_movement_bytes=34603008 total_nvlink_bytes=696291800 weight_to_token_ratio=0.052295 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374524049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68534 payload_rows_remote=52150 dup_rows=29770 dup_rows_remote=14492 dedup_ratio=0.302836 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8185, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 4, 0, 1, 0, 0, 190, 0, 0, 6, 8185, 0, 0, 6, 466, 0, 0, 253, 7236, 0, 0, 0, 4, 43, 5, 8186, 0, 0, 3] payload_rows_per_dst=[16374, 16384, 1094, 16372, 16376, 392, 1444, 98] dup_rows_per_dst=[2, 15278, 0, 0, 14472, 0, 6, 12] combine_rows_per_src=[16374, 16384, 1094, 16372, 16376, 392, 1444, 98] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94296 payload_rows_remote=87492 dup_rows=4008 dup_rows_remote=4008 dedup_ratio=0.040771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 596, 241, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 3402, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3449, 0, 81] payload_rows_per_dst=[16384, 16384, 8348, 6804, 15658, 16384, 6610, 7724] dup_rows_per_dst=[1192, 2740, 0, 0, 0, 14, 0, 62] combine_rows_per_src=[16384, 16384, 8348, 6804, 15658, 16384, 6610, 7724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=327311360 dispatch_meta_bytes=328144 combine_payload_bytes=327311360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327639504 ingress_bytes=361914368 link_bytes=361914368 token_bytes=654950864 weight_movement_bytes=34603008 total_nvlink_bytes=689553872 weight_to_token_ratio=0.052833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361854630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=273752064 dispatch_meta_bytes=327680 combine_payload_bytes=273752064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274079744 ingress_bytes=342958080 link_bytes=342958080 token_bytes=547831808 weight_movement_bytes=69206016 total_nvlink_bytes=617037824 weight_to_token_ratio=0.126327 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290523645e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=246366208 dispatch_meta_bytes=331232 combine_payload_bytes=246366208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246697440 ingress_bytes=280969216 link_bytes=280969216 token_bytes=493063648 weight_movement_bytes=34603008 total_nvlink_bytes=527666656 weight_to_token_ratio=0.070180 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057264540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=331145216 dispatch_meta_bytes=358888 combine_payload_bytes=331145216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331504104 ingress_bytes=365748224 link_bytes=365748224 token_bytes=662649320 weight_movement_bytes=34603008 total_nvlink_bytes=697252328 weight_to_token_ratio=0.052219 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376281122e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213606400 dispatch_meta_bytes=266568 combine_payload_bytes=213606400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=213872968 ingress_bytes=282812416 link_bytes=282812416 token_bytes=427479368 weight_movement_bytes=69206016 total_nvlink_bytes=496685384 weight_to_token_ratio=0.161893 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064200353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=358367232 dispatch_meta_bytes=366000 combine_payload_bytes=358367232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=358733232 ingress_bytes=392970240 link_bytes=392970240 token_bytes=717100464 weight_movement_bytes=34603008 total_nvlink_bytes=751703472 weight_to_token_ratio=0.048254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478715376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97864 payload_rows_remote=95110 dup_rows=440 dup_rows_remote=440 dedup_ratio=0.004476 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 8192, 0, 0, 0, 198, 0, 0, 252, 7490, 0, 0, 0, 0, 83, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 1062, 16384, 16384, 2754, 13694, 15880, 15322] dup_rows_per_dst=[0, 0, 0, 0, 0, 274, 0, 166] combine_rows_per_src=[16384, 1062, 16384, 16384, 2754, 13694, 15880, 15322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84202 payload_rows_remote=68664 dup_rows=14102 dup_rows_remote=8260 dedup_ratio=0.143453 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 7, 563, 237, 15, 1, 0, 0, 0, 0, 1166, 0, 5, 4154, 0, 0, 0, 4, 0, 2, 15, 3854, 0, 0, 0, 0, 6, 7754, 13, 2, 0, 0, 8173, 0, 8172, 0, 11, 0, 8167, 15, 12, 1, 0, 1, 2909, 0, 4, 11, 0, 0, 11, 0, 2, 438, 0, 0, 3336, 0, 89] payload_rows_per_dst=[1144, 2362, 8326, 7748, 16360, 16368, 15538, 16356] dup_rows_per_dst=[0, 476, 0, 6, 14, 44, 5842, 7720] combine_rows_per_src=[1144, 2362, 8326, 7748, 16360, 16368, 15538, 16356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=389570560 dispatch_meta_bytes=382200 combine_payload_bytes=389570560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=389952760 ingress_bytes=458776576 link_bytes=458776576 token_bytes=779523320 weight_movement_bytes=69206016 total_nvlink_bytes=848729336 weight_to_token_ratio=0.088780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726339321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83628 payload_rows_remote=67248 dup_rows=14676 dup_rows_remote=7268 dedup_ratio=0.149292 router_tokens_per_expert=[0, 1, 0, 0, 1, 0, 0, 567, 239, 1, 1, 0, 0, 0, 1, 1118, 0, 1, 4014, 0, 0, 0, 0, 0, 0, 2, 3730, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 8189, 0, 8190, 1, 0, 0, 8189, 1, 2, 26, 0, 2, 3367, 0, 0, 0, 0, 0, 0, 1, 0, 385, 1, 0, 3242, 0, 76] payload_rows_per_dst=[1138, 2240, 8030, 7464, 16380, 16384, 15612, 16380] dup_rows_per_dst=[0, 480, 0, 0, 0, 54, 6734, 7408] combine_rows_per_src=[1138, 2240, 8030, 7464, 16380, 16384, 15612, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97654 payload_rows_remote=81270 dup_rows=650 dup_rows_remote=650 dedup_ratio=0.006612 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 8192, 0, 0, 0, 181, 0, 0, 245, 7554, 0, 0, 0, 0, 49, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 326, 15310, 98] dup_rows_per_dst=[0, 0, 0, 0, 650, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 326, 15310, 98] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97832 payload_rows_remote=81448 dup_rows=472 dup_rows_remote=90 dedup_ratio=0.004801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 8192, 0, 0, 0, 191, 0, 0, 255, 7524, 0, 0, 0, 0, 36, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16366, 0, 16384, 16374, 0, 16384, 15940, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 382, 0, 90] combine_rows_per_src=[16366, 0, 16384, 16374, 0, 16384, 15940, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79584 payload_rows_remote=63200 dup_rows=18720 dup_rows_remote=17528 dedup_ratio=0.190430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 596, 300, 0, 0, 0, 0, 0, 0, 1141, 0, 0, 4230, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 3482, 0, 7, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3409, 0, 78] payload_rows_per_dst=[16384, 16384, 16384, 15756, 0, 4, 6978, 7694] dup_rows_per_dst=[1192, 2882, 8460, 6168, 0, 0, 0, 18] combine_rows_per_src=[16384, 16384, 16384, 15756, 0, 4, 6978, 7694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281247744 dispatch_meta_bytes=307696 combine_payload_bytes=281247744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281555440 ingress_bytes=315850752 link_bytes=315850752 token_bytes=562803184 weight_movement_bytes=34603008 total_nvlink_bytes=597406192 weight_to_token_ratio=0.061483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188520952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81780 payload_rows_remote=71076 dup_rows=16524 dup_rows_remote=16246 dedup_ratio=0.168091 router_tokens_per_expert=[3, 0, 3, 2, 6, 0, 0, 7, 0, 419, 366, 0, 0, 0, 4706, 0, 0, 1, 0, 10, 6, 8184, 2, 0, 1, 3, 0, 0, 7974, 0, 3, 0, 5, 5, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 518, 0, 4, 0, 2703, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 7948, 1, 0] payload_rows_per_dst=[16384, 10704, 16380, 15954, 13678, 1040, 5408, 2232] dup_rows_per_dst=[24, 278, 16200, 8, 10, 4, 0, 0] combine_rows_per_src=[16384, 10704, 16380, 15954, 13678, 1040, 5408, 2232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 8192, 0, 0, 0, 234, 0, 0, 245, 7508, 0, 0, 0, 0, 27, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 356, 15974, 54] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 356, 15974, 54] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=275447808 dispatch_meta_bytes=298064 combine_payload_bytes=275447808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275745872 ingress_bytes=310050816 link_bytes=310050816 token_bytes=551193680 weight_movement_bytes=34603008 total_nvlink_bytes=585796688 weight_to_token_ratio=0.062778 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166696259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=332881920 dispatch_meta_bytes=327680 combine_payload_bytes=332881920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=333209600 ingress_bytes=402087936 link_bytes=402087936 token_bytes=666091520 weight_movement_bytes=69206016 total_nvlink_bytes=735297536 weight_to_token_ratio=0.103899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513024532e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333611008 dispatch_meta_bytes=326152 combine_payload_bytes=333611008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333937160 ingress_bytes=402817024 link_bytes=402817024 token_bytes=667548168 weight_movement_bytes=69206016 total_nvlink_bytes=736754184 weight_to_token_ratio=0.103672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515768032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=258867200 dispatch_meta_bytes=322912 combine_payload_bytes=258867200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259190112 ingress_bytes=293470208 link_bytes=293470208 token_bytes=518057312 weight_movement_bytes=34603008 total_nvlink_bytes=552660320 weight_to_token_ratio=0.066794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104304766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=291127296 dispatch_meta_bytes=349288 combine_payload_bytes=291127296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291476584 ingress_bytes=325730304 link_bytes=325730304 token_bytes=582603880 weight_movement_bytes=34603008 total_nvlink_bytes=617206888 weight_to_token_ratio=0.059394 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225696911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96958 payload_rows_remote=80708 dup_rows=1346 dup_rows_remote=1346 dedup_ratio=0.013692 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 392, 506, 0, 0, 0, 4339, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 2838, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0] payload_rows_per_dst=[16384, 10286, 20, 16008, 16250, 16384, 5676, 15950] dup_rows_per_dst=[10, 188, 0, 0, 0, 1148, 0, 0] combine_rows_per_src=[16384, 10286, 20, 16008, 16250, 16384, 5676, 15950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335872000 ingress_bytes=370147328 link_bytes=370147328 token_bytes=671416320 weight_movement_bytes=34603008 total_nvlink_bytes=706019328 weight_to_token_ratio=0.051537 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12307274192 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=6157302224 ingress_bytes=7534092288 link_bytes=7534092288 total_nvlink_bytes=13691394512 weight_to_token_ratio=0.112464 prefetch_share_of_ingress=0.091857 dedup_ratio=0.195244 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232616641e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=330579968 dispatch_meta_bytes=328216 combine_payload_bytes=330579968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330908184 ingress_bytes=399785984 link_bytes=399785984 token_bytes=661488152 weight_movement_bytes=69206016 total_nvlink_bytes=730694168 weight_to_token_ratio=0.104622 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504362472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65106 payload_rows_remote=49228 dup_rows=33198 dup_rows_remote=33194 dedup_ratio=0.337708 router_tokens_per_expert=[6, 0, 0, 1, 0, 0, 4, 0, 16, 0, 0, 0, 8169, 0, 0, 3, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 3, 0, 0, 0, 8166, 6, 0, 1, 13, 0, 0, 0, 0, 176, 0, 1, 18, 8172, 0, 0, 21, 441, 0, 1, 242, 7236, 0, 0, 0, 14, 71, 14, 8173, 0, 0, 1] payload_rows_per_dst=[22, 38, 16348, 20, 38, 16380, 15878, 16382] dup_rows_per_dst=[0, 0, 4, 0, 2, 16686, 4, 16502] combine_rows_per_src=[22, 38, 16348, 20, 38, 16380, 15878, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64044 payload_rows_remote=47662 dup_rows=34260 dup_rows_remote=17730 dedup_ratio=0.348511 router_tokens_per_expert=[1, 0, 1, 0, 0, 0, 0, 0, 2, 1, 0, 0, 8189, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 159, 0, 0, 2, 8189, 0, 0, 1, 207, 1, 0, 250, 7498, 0, 0, 0, 2, 74, 1, 8189, 0, 0, 1] payload_rows_per_dst=[4, 4, 15318, 2, 2, 16384, 15948, 16382] dup_rows_per_dst=[0, 2, 4, 0, 0, 16696, 1028, 16530] combine_rows_per_src=[4, 4, 15318, 2, 2, 16384, 15948, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88356 payload_rows_remote=72082 dup_rows=9948 dup_rows_remote=788 dedup_ratio=0.101196 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 478, 405, 0, 0, 0, 4476, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 2756, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0] payload_rows_per_dst=[16384, 10448, 16274, 16052, 0, 7708, 5512, 15978] dup_rows_per_dst=[10, 270, 9160, 0, 0, 508, 0, 0] combine_rows_per_src=[16384, 10448, 16274, 16052, 0, 7708, 5512, 15978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14910231896 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=7459018072 ingress_bytes=8939143168 link_bytes=8939143168 total_nvlink_bytes=16398161240 weight_to_token_ratio=0.099793 prefetch_share_of_ingress=0.077419 dedup_ratio=0.071431 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027065 gemm_inner_pad_upper_bound=23622 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.462490264e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97120 payload_rows_remote=80736 dup_rows=1184 dup_rows_remote=314 dedup_ratio=0.012044 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 406, 433, 0, 0, 0, 5134, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 2280, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0] payload_rows_per_dst=[16176, 11692, 0, 15996, 15922, 16384, 4716, 16234] dup_rows_per_dst=[8, 254, 0, 0, 0, 870, 52, 0] combine_rows_per_src=[16176, 11692, 0, 15996, 15922, 16384, 4716, 16234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67062 payload_rows_remote=50678 dup_rows=31242 dup_rows_remote=31242 dedup_ratio=0.317810 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 8192, 0, 0, 0, 191, 0, 0, 255, 7429, 0, 0, 0, 0, 142, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 16384, 16384, 350, 892, 284] dup_rows_per_dst=[0, 16384, 0, 0, 14858, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 16384, 16384, 350, 892, 284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=201637888 dispatch_meta_bytes=329688 combine_payload_bytes=201637888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201967576 ingress_bytes=270843904 link_bytes=270843904 token_bytes=403605464 weight_movement_bytes=69206016 total_nvlink_bytes=472811480 weight_to_token_ratio=0.171469 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019163806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96652 payload_rows_remote=80606 dup_rows=1652 dup_rows_remote=1652 dedup_ratio=0.016805 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6, 0, 468, 447, 0, 0, 0, 4700, 0, 0, 0, 0, 26, 0, 8192, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 688, 0, 0, 0, 2325, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0] payload_rows_per_dst=[16384, 11008, 7732, 16046, 8442, 16384, 4650, 16006] dup_rows_per_dst=[12, 222, 42, 0, 0, 1376, 0, 0] combine_rows_per_src=[16384, 11008, 7732, 16046, 8442, 16384, 4650, 16006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=195223552 dispatch_meta_bytes=261568 combine_payload_bytes=195223552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=195485120 ingress_bytes=264429568 link_bytes=264429568 token_bytes=390708672 weight_movement_bytes=69206016 total_nvlink_bytes=459914688 weight_to_token_ratio=0.177129 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.950271761e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=295247872 dispatch_meta_bytes=291480 combine_payload_bytes=295247872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295539352 ingress_bytes=364453888 link_bytes=364453888 token_bytes=590787224 weight_movement_bytes=69206016 total_nvlink_bytes=659993240 weight_to_token_ratio=0.117142 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371410640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=330694656 dispatch_meta_bytes=324200 combine_payload_bytes=330694656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331018856 ingress_bytes=365297664 link_bytes=365297664 token_bytes=661713512 weight_movement_bytes=34603008 total_nvlink_bytes=696316520 weight_to_token_ratio=0.052293 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374585701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=207577088 dispatch_meta_bytes=327680 combine_payload_bytes=207577088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=207904768 ingress_bytes=242180096 link_bytes=242180096 token_bytes=415481856 weight_movement_bytes=34603008 total_nvlink_bytes=450084864 weight_to_token_ratio=0.083284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.113042042e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=330162176 dispatch_meta_bytes=329032 combine_payload_bytes=330162176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=330491208 ingress_bytes=433971200 link_bytes=433971200 token_bytes=660653384 weight_movement_bytes=103809024 total_nvlink_bytes=764462408 weight_to_token_ratio=0.157131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.632998688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14490478968 prefetch_bytes=692060160 reduce_bytes=1141899264 weight_movement_bytes=1833959424 egress_bytes=7249127800 ingress_bytes=9075310592 link_bytes=9075310592 total_nvlink_bytes=16324438392 weight_to_token_ratio=0.126563 prefetch_share_of_ingress=0.076257 dedup_ratio=0.095688 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=33 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.484767962e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14582684664 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=7295196152 ingress_bytes=8740814848 link_bytes=8740814848 total_nvlink_bytes=16036011000 weight_to_token_ratio=0.099661 prefetch_share_of_ingress=0.075217 dedup_ratio=0.084874 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027609 gemm_inner_pad_upper_bound=23495 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.430042721e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82052 payload_rows_remote=65708 dup_rows=16252 dup_rows_remote=1814 dedup_ratio=0.165324 router_tokens_per_expert=[1, 0, 10, 9, 19, 0, 0, 20, 0, 447, 375, 0, 0, 0, 4245, 0, 0, 13, 0, 36, 18, 8168, 2, 0, 2, 9, 0, 1, 7983, 0, 10, 0, 12, 15, 0, 0, 0, 8118, 5, 0, 0, 0, 0, 0, 722, 0, 5, 0, 2878, 0, 0, 2, 0, 8158, 0, 0, 0, 0, 4, 0, 0, 7861, 4, 0] payload_rows_per_dst=[84, 9874, 110, 7280, 16276, 16346, 16344, 15738] dup_rows_per_dst=[34, 260, 28, 24, 24, 1444, 14438, 0] combine_rows_per_src=[84, 9874, 110, 7280, 16276, 16346, 16344, 15738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=15095439792 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=7551697328 ingress_bytes=8858656768 link_bytes=8858656768 total_nvlink_bytes=16410354096 weight_to_token_ratio=0.087107 prefetch_share_of_ingress=0.082029 dedup_ratio=0.075987 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=21463 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.449322271e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77050 payload_rows_remote=60886 dup_rows=21254 dup_rows_remote=21252 dedup_ratio=0.216207 router_tokens_per_expert=[0, 0, 0, 3, 2, 0, 0, 3, 0, 287, 391, 0, 0, 0, 5440, 3, 0, 2, 0, 2, 2, 8189, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 2, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 2028, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 3, 0, 0, 8079, 1, 0] payload_rows_per_dst=[12, 12046, 8, 0, 16062, 16378, 16380, 16164] dup_rows_per_dst=[4, 196, 4, 0, 0, 982, 20066, 2] combine_rows_per_src=[12, 12046, 8, 0, 16062, 16378, 16380, 16164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75864 payload_rows_remote=59480 dup_rows=22440 dup_rows_remote=22430 dedup_ratio=0.228271 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 517, 483, 0, 0, 0, 4469, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 674, 0, 0, 0, 2531, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0] payload_rows_per_dst=[16384, 14110, 16384, 10936, 11640, 1348, 5062, 0] dup_rows_per_dst=[10, 6468, 15962, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14110, 16384, 10936, 11640, 1348, 5062, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=270088 combine_payload_bytes=269139968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269410056 ingress_bytes=338345984 link_bytes=338345984 token_bytes=538550024 weight_movement_bytes=69206016 total_nvlink_bytes=607756040 weight_to_token_ratio=0.128504 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273168699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=249389056 dispatch_meta_bytes=328552 combine_payload_bytes=249389056 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=249717608 ingress_bytes=318595072 link_bytes=318595072 token_bytes=499106664 weight_movement_bytes=69206016 total_nvlink_bytes=568312680 weight_to_token_ratio=0.138660 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198847607e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=243630080 dispatch_meta_bytes=327640 combine_payload_bytes=243630080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243957720 ingress_bytes=278233088 link_bytes=278233088 token_bytes=487587800 weight_movement_bytes=34603008 total_nvlink_bytes=522190808 weight_to_token_ratio=0.070968 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.046968711e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12835076000 prefetch_bytes=726663168 reduce_bytes=484442112 weight_movement_bytes=1211105280 egress_bytes=6421280672 ingress_bytes=7624900608 link_bytes=7624900608 total_nvlink_bytes=14046181280 weight_to_token_ratio=0.094359 prefetch_share_of_ingress=0.095301 dedup_ratio=0.169004 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=14 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=23114 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.247473354e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11403580304 prefetch_bytes=761266176 reduce_bytes=415236096 weight_movement_bytes=1176502272 egress_bytes=5705397136 ingress_bytes=6874685440 link_bytes=6874685440 total_nvlink_bytes=12580082576 weight_to_token_ratio=0.103170 prefetch_share_of_ingress=0.110735 dedup_ratio=0.230751 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=12 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.124734254e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12074871512 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6041078488 ingress_bytes=7452516352 link_bytes=7452516352 total_nvlink_bytes=13493594840 weight_to_token_ratio=0.117494 prefetch_share_of_ingress=0.092863 dedup_ratio=0.197526 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.219270394e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-29 23:40:34] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 671.0 | throughput per GPU (TFLOP/s/GPU): 190.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.369346E+00 | loss scale: 1.0 | grad norm: 5.165 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76124 payload_rows_remote=64656 dup_rows=22180 dup_rows_remote=17168 dedup_ratio=0.225627 router_tokens_per_expert=[1445, 485, 336, 1046, 130, 1399, 1001, 181, 2364, 556, 918, 407, 15, 2511, 724, 745, 577, 792, 832, 629, 121, 371, 174, 1083, 451, 690, 688, 635, 880, 296, 801, 899, 766, 694, 1752, 926, 824, 247, 883, 798, 257, 536, 481, 258, 924, 1124, 337, 167, 306, 422, 476, 739, 719, 2066, 333, 601, 1897, 457, 1631, 834, 131, 677, 1722, 985] payload_rows_per_dst=[9194, 11468, 7696, 10368, 8562, 9280, 9542, 10014] dup_rows_per_dst=[2852, 5012, 1462, 3816, 1714, 2682, 2246, 2396] combine_rows_per_src=[9194, 11468, 7696, 10368, 8562, 9280, 9542, 10014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76492 payload_rows_remote=66594 dup_rows=21812 dup_rows_remote=18498 dedup_ratio=0.221883 router_tokens_per_expert=[1598, 598, 405, 998, 100, 1529, 889, 170, 2393, 558, 722, 288, 11, 2574, 706, 833, 552, 806, 749, 778, 113, 373, 179, 1132, 427, 677, 644, 385, 849, 262, 736, 890, 798, 676, 1736, 1018, 840, 220, 920, 738, 220, 528, 508, 202, 856, 1158, 337, 141, 294, 398, 551, 845, 744, 2140, 345, 637, 1951, 384, 1847, 737, 184, 743, 1649, 883] payload_rows_per_dst=[9492, 9860, 9454, 9898, 8714, 9374, 9578, 10122] dup_rows_per_dst=[3082, 3586, 2634, 3314, 1706, 2428, 2330, 2732] combine_rows_per_src=[9492, 9860, 9454, 9898, 8714, 9374, 9578, 10122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76158 payload_rows_remote=67618 dup_rows=22146 dup_rows_remote=20410 dedup_ratio=0.225281 router_tokens_per_expert=[1579, 514, 445, 977, 96, 1406, 944, 195, 2192, 579, 994, 244, 14, 2606, 727, 733, 595, 847, 823, 783, 140, 391, 160, 1106, 486, 648, 712, 421, 835, 366, 774, 954, 808, 712, 1778, 1010, 749, 223, 873, 763, 267, 542, 494, 246, 828, 1235, 415, 155, 333, 377, 493, 757, 742, 2010, 289, 676, 1972, 360, 1708, 721, 169, 656, 1583, 922] payload_rows_per_dst=[9536, 8402, 10970, 10264, 8540, 9412, 9286, 9748] dup_rows_per_dst=[2776, 2564, 3932, 3684, 1736, 2896, 2068, 2490] combine_rows_per_src=[9536, 8402, 10970, 10264, 8540, 9412, 9286, 9748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75112 payload_rows_remote=64838 dup_rows=23192 dup_rows_remote=19950 dedup_ratio=0.235921 router_tokens_per_expert=[1567, 576, 412, 997, 90, 1204, 807, 166, 2385, 575, 877, 285, 8, 2649, 694, 850, 562, 849, 804, 688, 156, 368, 173, 1190, 394, 653, 731, 426, 840, 286, 673, 790, 754, 649, 1772, 937, 815, 209, 918, 848, 231, 523, 486, 211, 812, 1222, 388, 170, 333, 442, 579, 710, 793, 2246, 347, 647, 2107, 392, 1631, 741, 152, 661, 1788, 913] payload_rows_per_dst=[8880, 5556, 10910, 7910, 12250, 9464, 9868, 10274] dup_rows_per_dst=[2758, 1022, 3968, 1676, 5364, 2836, 2326, 3242] combine_rows_per_src=[8880, 5556, 10910, 7910, 12250, 9464, 9868, 10274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76440 payload_rows_remote=68624 dup_rows=21864 dup_rows_remote=20378 dedup_ratio=0.222412 router_tokens_per_expert=[1495, 520, 296, 907, 113, 1464, 894, 145, 2346, 502, 882, 288, 11, 2717, 768, 851, 534, 806, 831, 727, 89, 381, 131, 1152, 408, 709, 615, 452, 843, 223, 759, 859, 825, 697, 1889, 952, 722, 240, 900, 823, 201, 565, 436, 218, 855, 1174, 394, 121, 304, 387, 508, 683, 797, 2261, 368, 596, 2219, 360, 1740, 751, 135, 690, 1658, 965] payload_rows_per_dst=[9122, 11372, 7816, 10070, 8756, 9694, 9554, 10056] dup_rows_per_dst=[2546, 5358, 1486, 3444, 1562, 2672, 2254, 2542] combine_rows_per_src=[9122, 11372, 7816, 10070, 8756, 9694, 9554, 10056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76780 payload_rows_remote=67060 dup_rows=21524 dup_rows_remote=18824 dedup_ratio=0.218953 router_tokens_per_expert=[1586, 526, 357, 1011, 135, 1258, 807, 267, 2344, 477, 734, 263, 17, 2517, 768, 852, 545, 813, 736, 760, 140, 418, 210, 1122, 437, 702, 659, 552, 902, 167, 689, 799, 759, 817, 1610, 1027, 855, 256, 847, 754, 247, 611, 525, 245, 937, 1162, 371, 142, 309, 398, 580, 722, 778, 2193, 283, 638, 1970, 421, 1728, 796, 158, 681, 1853, 909] payload_rows_per_dst=[9082, 8522, 10862, 8588, 10426, 9720, 9574, 10006] dup_rows_per_dst=[2812, 2388, 3660, 2088, 2562, 2700, 2228, 3086] combine_rows_per_src=[9082, 8522, 10862, 8588, 10426, 9720, 9574, 10006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75912 payload_rows_remote=65774 dup_rows=22392 dup_rows_remote=18558 dedup_ratio=0.227783 router_tokens_per_expert=[1448, 457, 432, 871, 131, 1442, 1008, 236, 2125, 624, 898, 302, 13, 2637, 759, 721, 541, 818, 766, 639, 139, 420, 177, 1064, 391, 682, 714, 624, 841, 239, 712, 878, 855, 709, 1575, 1043, 739, 266, 923, 733, 282, 577, 556, 229, 911, 1110, 346, 120, 380, 433, 525, 641, 766, 2172, 342, 640, 1922, 393, 1686, 816, 244, 765, 1839, 865] payload_rows_per_dst=[10138, 10372, 7742, 9870, 8736, 9832, 11424, 7798] dup_rows_per_dst=[3834, 3864, 1386, 3442, 1800, 2904, 4052, 1110] combine_rows_per_src=[10138, 10372, 7742, 9870, 8736, 9832, 11424, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264830976 dispatch_meta_bytes=327296 combine_payload_bytes=264830976 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265158272 ingress_bytes=403243008 link_bytes=403243008 token_bytes=529989248 weight_movement_bytes=138412032 total_nvlink_bytes=668401280 weight_to_token_ratio=0.261160 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517370975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272769024 dispatch_meta_bytes=340368 combine_payload_bytes=272769024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273109392 ingress_bytes=307372032 link_bytes=307372032 token_bytes=545878416 weight_movement_bytes=34603008 total_nvlink_bytes=580481424 weight_to_token_ratio=0.063390 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156616211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276963328 dispatch_meta_bytes=352112 combine_payload_bytes=276963328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277315440 ingress_bytes=346169344 link_bytes=346169344 token_bytes=554278768 weight_movement_bytes=69206016 total_nvlink_bytes=623484784 weight_to_token_ratio=0.124858 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302607373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=339152 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265915600 ingress_bytes=403988480 link_bytes=403988480 token_bytes=531492048 weight_movement_bytes=138412032 total_nvlink_bytes=669904080 weight_to_token_ratio=0.260422 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520176126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281083904 dispatch_meta_bytes=356008 combine_payload_bytes=281083904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281439912 ingress_bytes=315686912 link_bytes=315686912 token_bytes=562523816 weight_movement_bytes=34603008 total_nvlink_bytes=597126824 weight_to_token_ratio=0.061514 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187904435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274677760 dispatch_meta_bytes=343536 combine_payload_bytes=274677760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275021296 ingress_bytes=343883776 link_bytes=343883776 token_bytes=549699056 weight_movement_bytes=69206016 total_nvlink_bytes=618905072 weight_to_token_ratio=0.125898 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294006964e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269410304 dispatch_meta_bytes=337328 combine_payload_bytes=269410304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269747632 ingress_bytes=304013312 link_bytes=304013312 token_bytes=539157936 weight_movement_bytes=34603008 total_nvlink_bytes=573760944 weight_to_token_ratio=0.064180 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143977618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75558 payload_rows_remote=65830 dup_rows=22746 dup_rows_remote=20500 dedup_ratio=0.231384 router_tokens_per_expert=[1606, 582, 369, 963, 116, 1367, 900, 197, 2374, 553, 954, 267, 14, 2697, 756, 757, 514, 785, 743, 743, 96, 395, 194, 996, 401, 704, 606, 639, 827, 268, 653, 838, 787, 690, 1724, 1002, 590, 219, 954, 776, 207, 630, 521, 160, 919, 1137, 401, 172, 314, 381, 522, 890, 746, 2230, 350, 554, 2058, 442, 1723, 769, 205, 715, 1609, 881] payload_rows_per_dst=[9092, 7056, 10680, 8162, 11282, 9596, 9728, 9962] dup_rows_per_dst=[3108, 1902, 3646, 1710, 4594, 2814, 2246, 2726] combine_rows_per_src=[9092, 7056, 10680, 8162, 11282, 9596, 9728, 9962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77242 payload_rows_remote=65850 dup_rows=21062 dup_rows_remote=16398 dedup_ratio=0.214254 router_tokens_per_expert=[366, 537, 57, 332, 3864, 65, 470, 813, 22, 703, 2477, 321, 646, 808, 220, 2831, 425, 1084, 536, 288, 1249, 392, 1332, 983, 804, 528, 628, 2591, 1, 64, 71, 185, 122, 10, 270, 18, 710, 22, 550, 194, 1046, 531, 259, 718, 24, 1097, 0, 132, 5659, 791, 3565, 0, 16, 523, 300, 1597, 2050, 387, 1003, 393, 549, 118, 530, 275] payload_rows_per_dst=[10146, 11392, 10174, 12148, 13128, 6162, 5838, 8254] dup_rows_per_dst=[2878, 4664, 2388, 4726, 1982, 1452, 616, 2356] combine_rows_per_src=[10146, 11392, 10174, 12148, 13128, 6162, 5838, 8254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269639680 dispatch_meta_bytes=345320 combine_payload_bytes=269639680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269985000 ingress_bytes=304242688 link_bytes=304242688 token_bytes=539624680 weight_movement_bytes=34603008 total_nvlink_bytes=574227688 weight_to_token_ratio=0.064124 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144840741e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79356 payload_rows_remote=70580 dup_rows=18948 dup_rows_remote=17330 dedup_ratio=0.192749 router_tokens_per_expert=[267, 528, 56, 283, 3697, 96, 464, 710, 18, 616, 2496, 304, 562, 825, 200, 3071, 442, 994, 527, 320, 1149, 382, 1339, 1093, 754, 536, 746, 2846, 0, 67, 71, 177, 87, 7, 251, 13, 791, 12, 430, 261, 1046, 608, 252, 703, 21, 1111, 0, 124, 5652, 649, 3849, 0, 12, 475, 244, 1681, 2060, 384, 900, 348, 611, 118, 541, 275] payload_rows_per_dst=[9938, 7934, 10076, 8776, 13212, 9972, 11114, 8334] dup_rows_per_dst=[2264, 2108, 2416, 1618, 1796, 3900, 2706, 2140] combine_rows_per_src=[9938, 7934, 10076, 8776, 13212, 9972, 11114, 8334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79514 payload_rows_remote=66182 dup_rows=18790 dup_rows_remote=16694 dedup_ratio=0.191142 router_tokens_per_expert=[245, 574, 44, 277, 4042, 32, 382, 652, 23, 668, 2613, 281, 537, 848, 211, 3156, 508, 1071, 484, 277, 1234, 377, 1198, 1156, 717, 537, 686, 2715, 2, 51, 53, 158, 131, 0, 257, 16, 767, 25, 545, 220, 983, 527, 259, 732, 20, 1123, 0, 105, 5753, 739, 3656, 0, 14, 568, 214, 1447, 2145, 373, 926, 357, 576, 145, 497, 223] payload_rows_per_dst=[10172, 8192, 10234, 8334, 13332, 10108, 10826, 8316] dup_rows_per_dst=[2324, 2170, 2376, 1504, 2096, 3702, 2450, 2168] combine_rows_per_src=[10172, 8192, 10234, 8334, 13332, 10108, 10826, 8316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78836 payload_rows_remote=68798 dup_rows=19468 dup_rows_remote=17208 dedup_ratio=0.198039 router_tokens_per_expert=[270, 487, 40, 261, 4082, 78, 374, 689, 5, 611, 2519, 277, 568, 794, 175, 3276, 463, 1054, 545, 303, 1164, 380, 1249, 991, 784, 543, 689, 3003, 0, 41, 42, 158, 83, 3, 249, 3, 830, 7, 428, 131, 1010, 547, 252, 710, 16, 1015, 0, 116, 5992, 640, 3841, 0, 11, 461, 227, 1447, 2106, 406, 864, 364, 550, 109, 598, 221] payload_rows_per_dst=[10118, 8392, 10038, 11012, 13586, 9608, 7800, 8282] dup_rows_per_dst=[2444, 2212, 2260, 3624, 1866, 3570, 1338, 2154] combine_rows_per_src=[10118, 8392, 10038, 11012, 13586, 9608, 7800, 8282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75108 payload_rows_remote=63318 dup_rows=23196 dup_rows_remote=17442 dedup_ratio=0.235962 router_tokens_per_expert=[232, 588, 57, 270, 3912, 59, 519, 649, 14, 577, 2487, 239, 576, 774, 195, 3317, 428, 1072, 458, 287, 1141, 439, 1388, 945, 833, 548, 737, 2983, 0, 51, 44, 223, 80, 3, 270, 3, 806, 31, 456, 161, 955, 596, 271, 792, 25, 1103, 0, 123, 5909, 600, 3465, 0, 24, 419, 246, 1465, 2136, 349, 955, 338, 568, 120, 626, 215] payload_rows_per_dst=[5964, 10884, 9984, 9032, 3402, 10380, 13672, 11790] dup_rows_per_dst=[1210, 4238, 2332, 1806, 218, 3984, 3654, 5754] combine_rows_per_src=[5964, 10884, 9984, 9032, 3402, 10380, 13672, 11790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78826 payload_rows_remote=68860 dup_rows=19478 dup_rows_remote=15532 dedup_ratio=0.198140 router_tokens_per_expert=[330, 508, 61, 261, 3742, 108, 453, 847, 13, 699, 2349, 352, 607, 795, 175, 2995, 406, 1082, 504, 307, 1181, 315, 1349, 1058, 703, 567, 773, 2716, 3, 81, 96, 203, 50, 2, 282, 46, 817, 18, 455, 442, 1104, 662, 282, 821, 21, 958, 2, 111, 5309, 757, 3433, 0, 17, 463, 259, 1772, 2079, 379, 977, 410, 601, 172, 535, 277] payload_rows_per_dst=[10024, 7892, 10174, 8566, 12922, 9966, 10900, 8382] dup_rows_per_dst=[2596, 2088, 2230, 1718, 1920, 3946, 2502, 2478] combine_rows_per_src=[10024, 7892, 10174, 8566, 12922, 9966, 10900, 8382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76700 payload_rows_remote=65480 dup_rows=21604 dup_rows_remote=17466 dedup_ratio=0.219767 router_tokens_per_expert=[341, 585, 80, 297, 3775, 77, 371, 745, 17, 676, 2448, 298, 629, 821, 220, 3041, 521, 1096, 545, 336, 1096, 423, 1408, 977, 761, 532, 736, 2671, 4, 78, 97, 190, 117, 6, 256, 38, 856, 29, 438, 430, 1184, 600, 268, 733, 25, 1148, 4, 125, 5180, 746, 2989, 1, 16, 632, 307, 1688, 2103, 389, 981, 402, 644, 133, 561, 231] payload_rows_per_dst=[11220, 11558, 8910, 11244, 12662, 6480, 6172, 8454] dup_rows_per_dst=[4138, 4742, 2246, 3704, 2038, 1694, 608, 2434] combine_rows_per_src=[11220, 11558, 8910, 11244, 12662, 6480, 6172, 8454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269721600 dispatch_meta_bytes=328992 combine_payload_bytes=269721600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270050592 ingress_bytes=338927616 link_bytes=338927616 token_bytes=539772192 weight_movement_bytes=69206016 total_nvlink_bytes=608978208 weight_to_token_ratio=0.128213 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275357333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=289095680 dispatch_meta_bytes=351640 combine_payload_bytes=289095680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289447320 ingress_bytes=323698688 link_bytes=323698688 token_bytes=578543000 weight_movement_bytes=34603008 total_nvlink_bytes=613146008 weight_to_token_ratio=0.059811 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218052103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271081472 dispatch_meta_bytes=331504 combine_payload_bytes=271081472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271412976 ingress_bytes=305684480 link_bytes=305684480 token_bytes=542494448 weight_movement_bytes=34603008 total_nvlink_bytes=577097456 weight_to_token_ratio=0.063785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150266089e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=259350528 dispatch_meta_bytes=323040 combine_payload_bytes=259350528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259673568 ingress_bytes=293953536 link_bytes=293953536 token_bytes=519024096 weight_movement_bytes=34603008 total_nvlink_bytes=553627104 weight_to_token_ratio=0.066669 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106123491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=282050560 dispatch_meta_bytes=337568 combine_payload_bytes=282050560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282388128 ingress_bytes=316653568 link_bytes=316653568 token_bytes=564438688 weight_movement_bytes=34603008 total_nvlink_bytes=599041696 weight_to_token_ratio=0.061305 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191541884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=281796608 dispatch_meta_bytes=344024 combine_payload_bytes=281796608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=282140632 ingress_bytes=351002624 link_bytes=351002624 token_bytes=563937240 weight_movement_bytes=69206016 total_nvlink_bytes=633143256 weight_to_token_ratio=0.122719 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320794616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268206080 dispatch_meta_bytes=331784 combine_payload_bytes=268206080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268537864 ingress_bytes=337412096 link_bytes=337412096 token_bytes=536743944 weight_movement_bytes=69206016 total_nvlink_bytes=605949960 weight_to_token_ratio=0.128937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269654553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76228 payload_rows_remote=62578 dup_rows=22076 dup_rows_remote=17618 dedup_ratio=0.224569 router_tokens_per_expert=[359, 562, 43, 246, 4057, 57, 362, 748, 19, 565, 2600, 333, 568, 691, 143, 3035, 418, 1176, 551, 274, 1107, 340, 1400, 967, 780, 575, 636, 2952, 4, 62, 38, 199, 75, 1, 267, 7, 801, 15, 504, 170, 1026, 562, 267, 716, 11, 924, 1, 148, 5873, 715, 3733, 0, 17, 444, 254, 1493, 1985, 402, 921, 400, 598, 99, 574, 282] payload_rows_per_dst=[10260, 8024, 10174, 8834, 3828, 9778, 13650, 11680] dup_rows_per_dst=[2608, 1814, 2292, 1658, 316, 3602, 4458, 5328] combine_rows_per_src=[10260, 8024, 10174, 8834, 3828, 9778, 13650, 11680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77034 payload_rows_remote=62928 dup_rows=21270 dup_rows_remote=15910 dedup_ratio=0.216370 router_tokens_per_expert=[1, 745, 220, 167, 0, 191, 122, 345, 54, 541, 677, 1833, 690, 345, 1, 0, 1934, 1606, 1851, 77, 1, 0, 4088, 683, 827, 773, 3661, 3941, 3, 4, 277, 0, 139, 948, 574, 106, 5592, 1182, 836, 125, 656, 2, 16, 22, 880, 1351, 94, 84, 279, 2, 219, 528, 0, 0, 96, 8, 3826, 361, 5, 9, 0, 2761, 0, 2793] payload_rows_per_dst=[11928, 14106, 9496, 12914, 6602, 6292, 9788, 5908] dup_rows_per_dst=[4892, 5360, 2808, 4892, 1218, 1084, 652, 364] combine_rows_per_src=[11928, 14106, 9496, 12914, 6602, 6292, 9788, 5908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=70420 dup_rows=18000 dup_rows_remote=16774 dedup_ratio=0.183105 router_tokens_per_expert=[0, 738, 227, 116, 0, 158, 128, 345, 51, 613, 736, 1830, 562, 360, 0, 0, 1809, 1664, 1766, 55, 0, 0, 4242, 677, 665, 729, 3860, 4111, 1, 0, 300, 0, 121, 954, 563, 110, 5930, 975, 759, 107, 608, 0, 6, 17, 720, 1423, 94, 81, 252, 0, 187, 432, 0, 0, 93, 2, 3765, 256, 0, 8, 0, 2781, 0, 3165] payload_rows_per_dst=[9584, 7454, 13618, 9884, 7726, 11712, 9904, 10422] dup_rows_per_dst=[1370, 850, 7634, 1226, 2002, 2408, 512, 1998] combine_rows_per_src=[9584, 7454, 13618, 9884, 7726, 11712, 9904, 10422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80970 payload_rows_remote=67600 dup_rows=17334 dup_rows_remote=12388 dedup_ratio=0.176331 router_tokens_per_expert=[0, 723, 235, 156, 0, 174, 126, 373, 67, 571, 835, 1879, 675, 337, 0, 0, 1894, 1592, 1883, 82, 0, 0, 4321, 709, 796, 746, 3589, 3873, 2, 4, 277, 0, 116, 849, 486, 116, 5718, 1021, 745, 107, 687, 0, 4, 18, 862, 1320, 101, 97, 266, 0, 206, 460, 0, 0, 96, 7, 3790, 347, 0, 15, 0, 2873, 0, 2926] payload_rows_per_dst=[9606, 7822, 7906, 9494, 13370, 11434, 11054, 10284] dup_rows_per_dst=[1548, 906, 1684, 1334, 4946, 2490, 2388, 2038] combine_rows_per_src=[9606, 7822, 7906, 9494, 13370, 11434, 11054, 10284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=256319488 dispatch_meta_bytes=320784 combine_payload_bytes=256319488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=256640272 ingress_bytes=394731520 link_bytes=394731520 token_bytes=512959760 weight_movement_bytes=138412032 total_nvlink_bytes=651371792 weight_to_token_ratio=0.269830 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485342931e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77812 payload_rows_remote=62498 dup_rows=20492 dup_rows_remote=11924 dedup_ratio=0.208455 router_tokens_per_expert=[0, 737, 205, 174, 0, 133, 164, 277, 76, 610, 906, 1705, 594, 317, 0, 0, 2121, 1654, 1942, 60, 3, 0, 4149, 840, 686, 763, 3709, 4242, 0, 4, 318, 0, 116, 1025, 478, 91, 6007, 1115, 742, 101, 496, 0, 1, 15, 507, 1549, 63, 67, 132, 1, 373, 428, 0, 0, 51, 3, 3468, 299, 0, 15, 0, 2849, 0, 2771] payload_rows_per_dst=[9106, 7452, 7614, 6088, 9180, 11818, 11240, 15314] dup_rows_per_dst=[1210, 964, 1384, 622, 2406, 2062, 3276, 8568] combine_rows_per_src=[9106, 7452, 7614, 6088, 9180, 11818, 11240, 15314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80002 payload_rows_remote=68580 dup_rows=18302 dup_rows_remote=16156 dedup_ratio=0.186178 router_tokens_per_expert=[0, 577, 239, 142, 0, 164, 123, 418, 50, 645, 782, 1798, 658, 329, 0, 0, 2134, 1498, 1882, 67, 4, 0, 4031, 761, 798, 628, 3584, 3987, 4, 1, 295, 0, 159, 927, 678, 193, 5637, 1311, 857, 137, 524, 0, 12, 13, 907, 1196, 74, 71, 173, 1, 496, 484, 0, 0, 67, 7, 3503, 370, 0, 3, 0, 2773, 0, 2980] payload_rows_per_dst=[9044, 7756, 7220, 9414, 13406, 11422, 11374, 10366] dup_rows_per_dst=[1288, 768, 1204, 1206, 6392, 2146, 3412, 1886] combine_rows_per_src=[9044, 7756, 7220, 9414, 13406, 11422, 11374, 10366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76912 payload_rows_remote=65328 dup_rows=21392 dup_rows_remote=16686 dedup_ratio=0.217611 router_tokens_per_expert=[0, 628, 241, 164, 0, 221, 166, 369, 56, 640, 688, 1837, 621, 352, 2, 1, 1827, 1433, 1766, 80, 2, 2, 4140, 627, 636, 728, 3816, 4133, 2, 13, 308, 0, 141, 971, 584, 121, 5705, 1003, 795, 124, 627, 4, 5, 25, 850, 1748, 97, 93, 199, 2, 255, 467, 1, 0, 115, 9, 3544, 337, 3, 17, 5, 2994, 0, 2812] payload_rows_per_dst=[11584, 14298, 9026, 13644, 6384, 5872, 9756, 6348] dup_rows_per_dst=[4706, 5506, 2448, 5628, 1094, 1026, 620, 364] combine_rows_per_src=[11584, 14298, 9026, 13644, 6384, 5872, 9756, 6348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80464 payload_rows_remote=68704 dup_rows=17840 dup_rows_remote=12258 dedup_ratio=0.181478 router_tokens_per_expert=[0, 626, 219, 139, 0, 164, 120, 285, 47, 623, 827, 1850, 544, 393, 0, 0, 2032, 1570, 1990, 60, 3, 0, 4097, 670, 667, 740, 3731, 4287, 0, 1, 267, 0, 135, 973, 412, 76, 6119, 997, 608, 112, 606, 0, 3, 11, 583, 1540, 79, 72, 116, 0, 356, 425, 0, 0, 94, 9, 3642, 293, 1, 8, 0, 2858, 0, 3072] payload_rows_per_dst=[9862, 12134, 11760, 9676, 5760, 12092, 9674, 9506] dup_rows_per_dst=[1928, 3980, 5582, 1136, 866, 2270, 520, 1558] combine_rows_per_src=[9862, 12134, 11760, 9676, 5760, 12092, 9674, 9506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=257753088 dispatch_meta_bytes=315352 combine_payload_bytes=257753088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258068440 ingress_bytes=292356096 link_bytes=292356096 token_bytes=515821528 weight_movement_bytes=34603008 total_nvlink_bytes=550424536 weight_to_token_ratio=0.067083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100112453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=288440320 dispatch_meta_bytes=348776 combine_payload_bytes=288440320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288789096 ingress_bytes=357646336 link_bytes=357646336 token_bytes=577229416 weight_movement_bytes=69206016 total_nvlink_bytes=646435432 weight_to_token_ratio=0.119893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345794369e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=255991808 dispatch_meta_bytes=297688 combine_payload_bytes=255991808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256289496 ingress_bytes=359800832 link_bytes=359800832 token_bytes=512281304 weight_movement_bytes=103809024 total_nvlink_bytes=616090328 weight_to_token_ratio=0.202641 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353901564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276889600 dispatch_meta_bytes=319952 combine_payload_bytes=276889600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277209552 ingress_bytes=415301632 link_bytes=415301632 token_bytes=554099152 weight_movement_bytes=138412032 total_nvlink_bytes=692511184 weight_to_token_ratio=0.249797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.562746607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280903680 dispatch_meta_bytes=338944 combine_payload_bytes=280903680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281242624 ingress_bytes=315506688 link_bytes=315506688 token_bytes=562146304 weight_movement_bytes=34603008 total_nvlink_bytes=596749312 weight_to_token_ratio=0.061555 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187226267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=328056 combine_payload_bytes=267583488 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267911544 ingress_bytes=336789504 link_bytes=336789504 token_bytes=535495032 weight_movement_bytes=69206016 total_nvlink_bytes=604701048 weight_to_token_ratio=0.129237 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267311790e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=323848 combine_payload_bytes=281411584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281735432 ingress_bytes=385220608 link_bytes=385220608 token_bytes=563147016 weight_movement_bytes=103809024 total_nvlink_bytes=666956040 weight_to_token_ratio=0.184337 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81204 payload_rows_remote=69978 dup_rows=17100 dup_rows_remote=14192 dedup_ratio=0.173950 router_tokens_per_expert=[0, 659, 215, 149, 0, 177, 125, 263, 65, 557, 884, 2027, 578, 363, 0, 0, 2014, 1629, 2145, 61, 0, 0, 4092, 625, 690, 773, 3839, 4261, 0, 4, 266, 1, 188, 806, 366, 62, 6014, 1161, 631, 89, 568, 0, 3, 21, 888, 1175, 85, 87, 235, 0, 169, 463, 0, 0, 92, 2, 3736, 290, 1, 4, 0, 2848, 0, 2706] payload_rows_per_dst=[9370, 7864, 7668, 9844, 12484, 11956, 11226, 10792] dup_rows_per_dst=[1278, 1084, 1252, 1302, 4468, 2220, 2908, 2588] combine_rows_per_src=[9370, 7864, 7668, 9844, 12484, 11956, 11226, 10792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80108 payload_rows_remote=67622 dup_rows=18196 dup_rows_remote=13104 dedup_ratio=0.185099 router_tokens_per_expert=[0, 0, 469, 0, 1002, 699, 560, 350, 4524, 319, 521, 46, 345, 5, 6717, 39, 3028, 12, 727, 325, 534, 257, 0, 96, 732, 1137, 498, 448, 382, 2469, 506, 94, 0, 10, 76, 6, 359, 73, 22, 0, 20, 4088, 2, 406, 228, 1, 1, 4186, 1, 80, 8, 626, 5, 3112, 0, 0, 6, 2, 590, 1, 2817, 5053, 87, 445] payload_rows_per_dst=[11912, 12486, 8220, 10348, 8136, 8448, 13084, 7474] dup_rows_per_dst=[2620, 5092, 1738, 2184, 410, 1044, 4686, 422] combine_rows_per_src=[11912, 12486, 8220, 10348, 8136, 8448, 13084, 7474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84060 payload_rows_remote=73952 dup_rows=14244 dup_rows_remote=12284 dedup_ratio=0.144897 router_tokens_per_expert=[0, 0, 456, 0, 1086, 644, 591, 335, 4515, 296, 498, 0, 307, 0, 6959, 32, 3184, 18, 652, 307, 439, 205, 0, 71, 674, 1136, 522, 448, 417, 2297, 447, 93, 0, 3, 79, 6, 292, 46, 17, 0, 15, 4273, 0, 448, 194, 0, 0, 4460, 0, 39, 1, 592, 1, 3275, 0, 0, 2, 0, 542, 0, 2753, 5004, 74, 407] payload_rows_per_dst=[12282, 10310, 8204, 10108, 14106, 8772, 13082, 7196] dup_rows_per_dst=[2862, 986, 1548, 1960, 698, 1088, 4742, 360] combine_rows_per_src=[12282, 10310, 8204, 10108, 14106, 8772, 13082, 7196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84658 payload_rows_remote=70672 dup_rows=13646 dup_rows_remote=13006 dedup_ratio=0.138814 router_tokens_per_expert=[0, 0, 444, 0, 1171, 762, 533, 374, 4503, 280, 496, 0, 301, 0, 6890, 44, 2821, 14, 620, 316, 535, 234, 0, 75, 756, 1273, 484, 363, 376, 2016, 447, 92, 0, 3, 88, 5, 274, 43, 10, 0, 19, 4488, 0, 415, 241, 0, 0, 4588, 0, 0, 5, 594, 0, 3296, 0, 0, 0, 0, 558, 0, 2764, 5057, 69, 415] payload_rows_per_dst=[12356, 10276, 7642, 9908, 13986, 9240, 8626, 12624] dup_rows_per_dst=[3388, 972, 1588, 1706, 640, 1086, 886, 3380] combine_rows_per_src=[12356, 10276, 7642, 9908, 13986, 9240, 8626, 12624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78470 payload_rows_remote=65088 dup_rows=19834 dup_rows_remote=15896 dedup_ratio=0.201762 router_tokens_per_expert=[0, 0, 451, 0, 1204, 498, 553, 258, 4669, 240, 416, 0, 307, 1, 7090, 15, 3492, 38, 669, 298, 551, 197, 0, 47, 641, 1035, 539, 371, 380, 2183, 471, 48, 1, 5, 61, 4, 334, 50, 12, 0, 16, 4455, 2, 474, 155, 0, 0, 4820, 0, 26, 1, 540, 3, 3400, 0, 0, 11, 0, 444, 0, 2375, 4884, 67, 350] payload_rows_per_dst=[5350, 1800, 12656, 9678, 14346, 13408, 7850, 13382] dup_rows_per_dst=[578, 158, 7266, 1658, 768, 5378, 90, 3938] combine_rows_per_src=[5350, 1800, 12656, 9678, 14346, 13408, 7850, 13382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286629888 dispatch_meta_bytes=336680 combine_payload_bytes=286629888 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=286966568 ingress_bytes=355835904 link_bytes=355835904 token_bytes=573596456 weight_movement_bytes=69206016 total_nvlink_bytes=642802472 weight_to_token_ratio=0.120653 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338981860e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83756 payload_rows_remote=73246 dup_rows=14548 dup_rows_remote=11890 dedup_ratio=0.147990 router_tokens_per_expert=[2, 0, 480, 0, 1145, 486, 562, 313, 4331, 348, 569, 50, 357, 3, 6851, 29, 2802, 24, 666, 365, 542, 200, 0, 67, 817, 1087, 570, 397, 407, 2210, 422, 60, 0, 4, 70, 4, 331, 58, 13, 0, 18, 4505, 3, 463, 144, 1, 0, 4898, 0, 45, 0, 673, 1, 3366, 0, 0, 3, 0, 520, 2, 2735, 4612, 110, 411] payload_rows_per_dst=[10416, 9068, 8370, 10184, 13902, 10510, 8038, 13268] dup_rows_per_dst=[2456, 1094, 2174, 1756, 760, 2658, 132, 3518] combine_rows_per_src=[10416, 9068, 8370, 10184, 13902, 10510, 8038, 13268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83970 payload_rows_remote=75744 dup_rows=14334 dup_rows_remote=12718 dedup_ratio=0.145813 router_tokens_per_expert=[0, 0, 384, 0, 1083, 480, 574, 349, 4792, 278, 397, 0, 290, 2, 6931, 13, 3159, 16, 644, 305, 568, 187, 0, 42, 687, 1049, 526, 507, 379, 2665, 415, 89, 0, 2, 75, 1, 266, 34, 8, 1, 24, 4427, 0, 439, 170, 0, 0, 3979, 0, 10, 0, 653, 1, 3351, 0, 0, 2, 0, 548, 0, 2451, 5397, 86, 416] payload_rows_per_dst=[11554, 10722, 8226, 10162, 14038, 9014, 13612, 6642] dup_rows_per_dst=[2144, 822, 1616, 2472, 598, 1106, 5212, 364] combine_rows_per_src=[11554, 10722, 8226, 10162, 14038, 9014, 13612, 6642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72646 payload_rows_remote=60720 dup_rows=25658 dup_rows_remote=23182 dedup_ratio=0.261007 router_tokens_per_expert=[0, 0, 500, 1, 1055, 562, 584, 369, 4013, 334, 572, 50, 327, 4, 6750, 42, 3125, 28, 629, 331, 603, 265, 0, 72, 779, 1251, 512, 436, 439, 2486, 434, 112, 2, 5, 90, 8, 341, 77, 38, 6, 23, 4315, 2, 417, 222, 3, 0, 4130, 5, 54, 1, 647, 3, 3425, 0, 6, 9, 6, 669, 6, 2457, 4970, 92, 458] payload_rows_per_dst=[11926, 16084, 8342, 10624, 1118, 8342, 9214, 6996] dup_rows_per_dst=[2476, 16106, 1764, 2828, 16, 1068, 1002, 398] combine_rows_per_src=[11926, 16084, 8342, 10624, 1118, 8342, 9214, 6996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276979712 dispatch_meta_bytes=322904 combine_payload_bytes=276979712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277302616 ingress_bytes=380788736 link_bytes=380788736 token_bytes=554282328 weight_movement_bytes=103809024 total_nvlink_bytes=658091352 weight_to_token_ratio=0.187285 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432877358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=302907392 dispatch_meta_bytes=344944 combine_payload_bytes=302907392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303252336 ingress_bytes=337510400 link_bytes=337510400 token_bytes=606159728 weight_movement_bytes=34603008 total_nvlink_bytes=640762736 weight_to_token_ratio=0.057086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270024463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=323936 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266924384 ingress_bytes=370409472 link_bytes=370409472 token_bytes=533524832 weight_movement_bytes=103809024 total_nvlink_bytes=637333856 weight_to_token_ratio=0.194572 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393821023e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289472512 dispatch_meta_bytes=334712 combine_payload_bytes=289472512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289807224 ingress_bytes=324075520 link_bytes=324075520 token_bytes=579279736 weight_movement_bytes=34603008 total_nvlink_bytes=613882744 weight_to_token_ratio=0.059735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219470092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=300015616 dispatch_meta_bytes=340544 combine_payload_bytes=300015616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300356160 ingress_bytes=403824640 link_bytes=403824640 token_bytes=600371776 weight_movement_bytes=103809024 total_nvlink_bytes=704180800 weight_to_token_ratio=0.172908 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519559610e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310247424 dispatch_meta_bytes=353848 combine_payload_bytes=310247424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310601272 ingress_bytes=344850432 link_bytes=344850432 token_bytes=620848696 weight_movement_bytes=34603008 total_nvlink_bytes=655451704 weight_to_token_ratio=0.055735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297644413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=248709120 dispatch_meta_bytes=335608 combine_payload_bytes=248709120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249044728 ingress_bytes=283312128 link_bytes=283312128 token_bytes=497753848 weight_movement_bytes=34603008 total_nvlink_bytes=532356856 weight_to_token_ratio=0.069518 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.066080729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78706 payload_rows_remote=70770 dup_rows=19598 dup_rows_remote=19486 dedup_ratio=0.199361 router_tokens_per_expert=[0, 0, 384, 0, 997, 643, 580, 388, 4530, 358, 441, 11, 316, 1, 6845, 37, 3104, 34, 632, 371, 490, 255, 0, 59, 730, 1098, 501, 502, 418, 2562, 467, 86, 0, 3, 80, 1, 402, 56, 4, 0, 24, 4015, 2, 375, 181, 1, 0, 3973, 0, 67, 0, 654, 1, 3302, 0, 0, 4, 0, 576, 0, 2813, 5147, 126, 505] payload_rows_per_dst=[5300, 2104, 12382, 10202, 13920, 12900, 7936, 13962] dup_rows_per_dst=[684, 224, 6568, 2526, 862, 4242, 112, 4380] combine_rows_per_src=[5300, 2104, 12382, 10202, 13920, 12900, 7936, 13962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79298 payload_rows_remote=67816 dup_rows=19006 dup_rows_remote=16988 dedup_ratio=0.193339 router_tokens_per_expert=[2, 1506, 1387, 126, 155, 1950, 2, 2790, 0, 920, 618, 73, 88, 0, 1, 2, 0, 330, 0, 132, 0, 943, 5226, 13, 412, 0, 5954, 2, 563, 114, 68, 0, 56, 0, 0, 1035, 976, 0, 20, 1101, 0, 1768, 0, 4262, 71, 1, 510, 1, 0, 898, 2304, 0, 0, 6093, 5048, 458, 7, 86, 0, 442, 631, 0, 7, 0] payload_rows_per_dst=[14168, 11482, 12114, 2840, 9796, 11132, 10858, 6908] dup_rows_per_dst=[7334, 2018, 1174, 140, 2160, 2094, 3284, 802] combine_rows_per_src=[14168, 11482, 12114, 2840, 9796, 11132, 10858, 6908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84236 payload_rows_remote=71730 dup_rows=14068 dup_rows_remote=12376 dedup_ratio=0.143107 router_tokens_per_expert=[0, 1449, 1383, 95, 149, 1944, 0, 2631, 0, 901, 440, 60, 74, 0, 0, 0, 0, 483, 0, 157, 0, 768, 5329, 9, 352, 0, 6080, 1, 502, 95, 69, 0, 8, 0, 0, 874, 694, 0, 17, 1373, 0, 1810, 0, 4174, 72, 0, 442, 0, 0, 842, 2628, 0, 0, 6162, 5427, 477, 0, 64, 0, 495, 622, 0, 0, 0] payload_rows_per_dst=[7154, 11904, 12090, 12506, 9378, 11264, 7090, 12850] dup_rows_per_dst=[2886, 1900, 1402, 1692, 1816, 1732, 804, 1836] combine_rows_per_src=[7154, 11904, 12090, 12506, 9378, 11264, 7090, 12850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83674 payload_rows_remote=73860 dup_rows=14630 dup_rows_remote=12510 dedup_ratio=0.148824 router_tokens_per_expert=[0, 1443, 1426, 80, 138, 1941, 0, 2700, 0, 1082, 442, 59, 69, 0, 0, 0, 0, 351, 0, 123, 0, 764, 5278, 5, 431, 0, 5977, 7, 548, 103, 81, 0, 59, 0, 0, 885, 1055, 0, 38, 1230, 0, 1613, 0, 4150, 59, 0, 557, 0, 0, 864, 2447, 0, 0, 6106, 5406, 469, 2, 62, 0, 432, 670, 0, 0, 0] payload_rows_per_dst=[7078, 11926, 11944, 12378, 9814, 11004, 6730, 12800] dup_rows_per_dst=[2978, 2190, 1098, 1916, 2120, 1754, 830, 1744] combine_rows_per_src=[7078, 11926, 11944, 12378, 9814, 11004, 6730, 12800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289873920 dispatch_meta_bytes=361024 combine_payload_bytes=289873920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290234944 ingress_bytes=324476928 link_bytes=324476928 token_bytes=580108864 weight_movement_bytes=34603008 total_nvlink_bytes=614711872 weight_to_token_ratio=0.059649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220980558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73126 payload_rows_remote=60172 dup_rows=25178 dup_rows_remote=23256 dedup_ratio=0.256124 router_tokens_per_expert=[0, 1494, 1327, 90, 135, 2019, 1, 2761, 0, 904, 555, 42, 65, 0, 0, 0, 0, 358, 0, 174, 1, 559, 5536, 1, 306, 0, 6257, 1, 591, 95, 64, 0, 7, 0, 0, 827, 917, 0, 45, 1165, 0, 1609, 0, 4109, 36, 0, 543, 0, 0, 850, 2335, 0, 1, 6208, 5459, 475, 2, 39, 0, 498, 690, 0, 1, 0] payload_rows_per_dst=[4556, 3028, 4766, 14784, 11556, 6986, 14496, 12954] dup_rows_per_dst=[1538, 104, 292, 7394, 3926, 828, 9174, 1922] combine_rows_per_src=[4556, 3028, 4766, 14784, 11556, 6986, 14496, 12954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84118 payload_rows_remote=72016 dup_rows=14186 dup_rows_remote=12764 dedup_ratio=0.144307 router_tokens_per_expert=[0, 1451, 1434, 59, 109, 2035, 0, 2680, 0, 914, 425, 65, 86, 0, 0, 0, 0, 492, 0, 163, 0, 624, 5481, 2, 313, 0, 6008, 1, 599, 88, 61, 0, 55, 0, 0, 908, 770, 0, 22, 1329, 0, 1752, 0, 4302, 64, 0, 380, 0, 0, 1004, 2128, 0, 1, 6256, 5587, 440, 0, 68, 0, 391, 605, 0, 0, 0] payload_rows_per_dst=[7342, 12088, 12102, 12350, 9596, 11312, 6368, 12960] dup_rows_per_dst=[2834, 2066, 1422, 1790, 1932, 1684, 778, 1680] combine_rows_per_src=[7342, 12088, 12102, 12350, 9596, 11312, 6368, 12960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83670 payload_rows_remote=72640 dup_rows=14634 dup_rows_remote=12920 dedup_ratio=0.148865 router_tokens_per_expert=[0, 1378, 1524, 135, 122, 1585, 0, 2570, 0, 1080, 637, 71, 110, 0, 0, 0, 0, 418, 0, 138, 0, 857, 5404, 2, 426, 0, 5981, 14, 454, 87, 60, 0, 47, 0, 0, 855, 910, 0, 34, 1207, 0, 1726, 0, 4090, 70, 0, 486, 0, 0, 1100, 2596, 0, 0, 6086, 5205, 521, 1, 85, 0, 456, 618, 0, 6, 0] payload_rows_per_dst=[6694, 11824, 12206, 12358, 9340, 11030, 7508, 12710] dup_rows_per_dst=[2794, 2382, 1432, 1686, 1906, 1714, 926, 1794] combine_rows_per_src=[6694, 11824, 12206, 12358, 9340, 11030, 7508, 12710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78280 payload_rows_remote=63450 dup_rows=20024 dup_rows_remote=12314 dedup_ratio=0.203695 router_tokens_per_expert=[5, 1453, 1452, 115, 117, 1975, 1, 2617, 0, 746, 757, 70, 92, 0, 4, 2, 0, 450, 7, 164, 1, 814, 5252, 4, 341, 7, 6152, 2, 554, 130, 68, 7, 92, 5, 0, 852, 981, 0, 30, 1218, 2, 1633, 0, 4161, 69, 6, 375, 2, 1, 872, 2248, 2, 5, 6048, 5336, 479, 0, 83, 0, 459, 833, 0, 1, 0] payload_rows_per_dst=[14830, 11894, 11964, 2188, 9470, 10950, 14416, 2568] dup_rows_per_dst=[7710, 2120, 1420, 30, 2120, 1546, 4894, 184] combine_rows_per_src=[14830, 11894, 11964, 2188, 9470, 10950, 14416, 2568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=339216 combine_payload_bytes=277774336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278113552 ingress_bytes=312377344 link_bytes=312377344 token_bytes=555887888 weight_movement_bytes=34603008 total_nvlink_bytes=590490896 weight_to_token_ratio=0.062248 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175450797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=293806080 dispatch_meta_bytes=336424 combine_payload_bytes=293806080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294142504 ingress_bytes=363012096 link_bytes=363012096 token_bytes=587948584 weight_movement_bytes=69206016 total_nvlink_bytes=657154600 weight_to_token_ratio=0.117708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365985292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=246464512 dispatch_meta_bytes=333712 combine_payload_bytes=246464512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246798224 ingress_bytes=281067520 link_bytes=281067520 token_bytes=493262736 weight_movement_bytes=34603008 total_nvlink_bytes=527865744 weight_to_token_ratio=0.070151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057634450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=302530560 dispatch_meta_bytes=345480 combine_payload_bytes=302530560 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302876040 ingress_bytes=371736576 link_bytes=371736576 token_bytes=605406600 weight_movement_bytes=69206016 total_nvlink_bytes=674612616 weight_to_token_ratio=0.114313 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398814808e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=294977536 dispatch_meta_bytes=339120 combine_payload_bytes=294977536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295316656 ingress_bytes=364183552 link_bytes=364183552 token_bytes=590294192 weight_movement_bytes=69206016 total_nvlink_bytes=659500208 weight_to_token_ratio=0.117240 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370393387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=297533440 dispatch_meta_bytes=342240 combine_payload_bytes=297533440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297875680 ingress_bytes=366739456 link_bytes=366739456 token_bytes=595409120 weight_movement_bytes=69206016 total_nvlink_bytes=664615136 weight_to_token_ratio=0.116233 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380011048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259891200 dispatch_meta_bytes=303056 combine_payload_bytes=259891200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=260194256 ingress_bytes=363700224 link_bytes=363700224 token_bytes=520085456 weight_movement_bytes=103809024 total_nvlink_bytes=623894480 weight_to_token_ratio=0.199600 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368574663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83318 payload_rows_remote=74908 dup_rows=14986 dup_rows_remote=13248 dedup_ratio=0.152445 router_tokens_per_expert=[0, 1371, 1447, 91, 124, 2105, 0, 2797, 0, 893, 530, 93, 126, 0, 0, 0, 0, 370, 0, 133, 0, 697, 5236, 3, 335, 0, 6038, 3, 596, 110, 44, 0, 11, 0, 0, 1007, 705, 0, 18, 1215, 0, 1799, 0, 4246, 90, 0, 502, 0, 0, 937, 2484, 0, 1, 6154, 5251, 448, 1, 98, 0, 433, 610, 0, 0, 0] payload_rows_per_dst=[6186, 9750, 11666, 12506, 10576, 11352, 8410, 12872] dup_rows_per_dst=[2222, 1628, 1212, 1746, 2798, 1922, 1738, 1720] combine_rows_per_src=[6186, 9750, 11666, 12506, 10576, 11352, 8410, 12872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77194 payload_rows_remote=64234 dup_rows=21110 dup_rows_remote=17724 dedup_ratio=0.214742 router_tokens_per_expert=[172, 2712, 83, 191, 69, 2351, 1560, 176, 0, 0, 1, 527, 1527, 0, 1, 760, 334, 4248, 7261, 2, 2, 1501, 5357, 3, 1, 1388, 2273, 1, 0, 0, 1169, 0, 4086, 273, 0, 0, 0, 1, 363, 259, 292, 0, 0, 94, 0, 0, 218, 1022, 82, 571, 761, 435, 0, 0, 1, 1104, 37, 0, 87, 0, 51, 498, 0, 5247] payload_rows_per_dst=[11222, 12960, 13252, 8522, 11860, 3176, 5124, 11078] dup_rows_per_dst=[3406, 3386, 7510, 1142, 4044, 76, 784, 762] combine_rows_per_src=[11222, 12960, 13252, 8522, 11860, 3176, 5124, 11078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=73656 dup_rows=15852 dup_rows_remote=14430 dedup_ratio=0.161255 router_tokens_per_expert=[172, 2896, 80, 155, 69, 2225, 1759, 172, 0, 0, 0, 537, 1435, 0, 0, 798, 346, 4039, 7350, 31, 0, 1537, 5431, 0, 0, 1682, 2406, 0, 0, 0, 1021, 0, 3940, 194, 0, 0, 1, 0, 344, 252, 339, 0, 0, 81, 0, 0, 146, 962, 50, 598, 796, 550, 0, 0, 0, 1036, 3, 0, 80, 0, 36, 371, 0, 5232] payload_rows_per_dst=[11344, 13066, 3768, 8796, 9142, 14866, 10530, 10940] dup_rows_per_dst=[3712, 3336, 60, 1422, 1146, 2064, 3608, 504] combine_rows_per_src=[11344, 13066, 3768, 8796, 9142, 14866, 10530, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306823168 dispatch_meta_bytes=352624 combine_payload_bytes=306823168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307175792 ingress_bytes=410632192 link_bytes=410632192 token_bytes=613998960 weight_movement_bytes=103809024 total_nvlink_bytes=717807984 weight_to_token_ratio=0.169070 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545175880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80362 payload_rows_remote=67480 dup_rows=17942 dup_rows_remote=13250 dedup_ratio=0.182515 router_tokens_per_expert=[109, 3162, 64, 150, 67, 2028, 1612, 100, 0, 0, 0, 587, 1541, 0, 0, 786, 312, 4362, 7504, 4, 0, 1408, 5343, 0, 0, 1692, 2107, 0, 0, 0, 1357, 0, 3904, 204, 0, 0, 0, 0, 290, 181, 304, 0, 0, 66, 1, 0, 163, 1103, 58, 636, 873, 524, 0, 0, 0, 965, 1, 0, 63, 0, 1, 424, 1, 5095] payload_rows_per_dst=[5844, 4938, 8496, 12574, 8678, 15934, 11016, 12882] dup_rows_per_dst=[638, 890, 1012, 4062, 480, 2348, 3820, 4692] combine_rows_per_src=[5844, 4938, 8496, 12574, 8678, 15934, 11016, 12882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83790 payload_rows_remote=74410 dup_rows=14514 dup_rows_remote=13878 dedup_ratio=0.147644 router_tokens_per_expert=[186, 2779, 94, 171, 81, 2425, 1611, 189, 0, 0, 0, 503, 1387, 0, 0, 772, 325, 4252, 7345, 0, 0, 1448, 5418, 0, 0, 1510, 2251, 0, 0, 0, 1051, 0, 4129, 230, 0, 0, 0, 0, 364, 285, 267, 0, 0, 94, 0, 0, 219, 985, 93, 576, 795, 433, 0, 0, 0, 1009, 3, 0, 94, 0, 6, 455, 0, 5317] payload_rows_per_dst=[10960, 12992, 3546, 9036, 9380, 15744, 10938, 11194] dup_rows_per_dst=[3220, 3168, 0, 1480, 636, 2076, 3378, 556] combine_rows_per_src=[10960, 12992, 3546, 9036, 9380, 15744, 10938, 11194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77714 payload_rows_remote=72598 dup_rows=20590 dup_rows_remote=20040 dedup_ratio=0.209452 router_tokens_per_expert=[166, 2887, 76, 153, 62, 2243, 1404, 153, 0, 0, 0, 547, 1406, 0, 0, 793, 392, 4626, 7420, 21, 0, 1436, 5363, 0, 0, 1444, 2210, 0, 0, 0, 1170, 0, 3845, 273, 0, 0, 0, 0, 313, 172, 285, 0, 0, 76, 0, 0, 184, 1087, 55, 604, 952, 485, 0, 0, 0, 1031, 2, 0, 76, 0, 0, 350, 0, 5390] payload_rows_per_dst=[10976, 12992, 5116, 8468, 15438, 3202, 10296, 11226] dup_rows_per_dst=[3312, 3226, 550, 1180, 8608, 62, 3242, 410] combine_rows_per_src=[10976, 12992, 5116, 8468, 15438, 3202, 10296, 11226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83882 payload_rows_remote=67976 dup_rows=14422 dup_rows_remote=12250 dedup_ratio=0.146708 router_tokens_per_expert=[132, 2904, 70, 180, 61, 2506, 1348, 130, 0, 0, 0, 574, 1318, 0, 0, 805, 323, 4389, 7467, 8, 0, 1462, 5224, 0, 0, 1539, 2148, 0, 0, 0, 1248, 0, 4173, 297, 0, 0, 0, 0, 275, 172, 275, 0, 0, 70, 1, 0, 189, 1037, 62, 666, 960, 432, 0, 0, 0, 1044, 6, 2, 70, 0, 9, 388, 0, 5188] payload_rows_per_dst=[7826, 4954, 11566, 12098, 9400, 15906, 11282, 10850] dup_rows_per_dst=[1028, 1044, 1864, 3580, 434, 2172, 3824, 476] combine_rows_per_src=[7826, 4954, 11566, 12098, 9400, 15906, 11282, 10850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263102464 dispatch_meta_bytes=327832 combine_payload_bytes=263102464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263430296 ingress_bytes=297705472 link_bytes=297705472 token_bytes=526532760 weight_movement_bytes=34603008 total_nvlink_bytes=561135768 weight_to_token_ratio=0.065719 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120241724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76544 payload_rows_remote=65366 dup_rows=21760 dup_rows_remote=17876 dedup_ratio=0.221354 router_tokens_per_expert=[194, 2590, 91, 175, 56, 2443, 1784, 198, 0, 5, 2, 542, 1213, 0, 2, 801, 341, 4261, 7168, 9, 0, 1643, 5288, 8, 5, 1710, 2356, 8, 0, 1, 1104, 6, 3950, 362, 3, 0, 11, 8, 338, 231, 283, 2, 0, 98, 4, 2, 208, 849, 84, 602, 829, 471, 0, 1, 8, 1056, 45, 1, 89, 0, 20, 444, 0, 5149] payload_rows_per_dst=[11178, 12820, 15380, 8910, 9274, 2850, 5312, 10820] dup_rows_per_dst=[3884, 2886, 11480, 1470, 532, 42, 790, 676] combine_rows_per_src=[11178, 12820, 15380, 8910, 9274, 2850, 5312, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=301694976 dispatch_meta_bytes=352344 combine_payload_bytes=301694976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302047320 ingress_bytes=336297984 link_bytes=336297984 token_bytes=603742296 weight_movement_bytes=34603008 total_nvlink_bytes=638345304 weight_to_token_ratio=0.057314 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265462240e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276398080 dispatch_meta_bytes=322920 combine_payload_bytes=276398080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276721000 ingress_bytes=311001088 link_bytes=311001088 token_bytes=553119080 weight_movement_bytes=34603008 total_nvlink_bytes=587722088 weight_to_token_ratio=0.062560 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170272057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=304783360 dispatch_meta_bytes=353152 combine_payload_bytes=304783360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305136512 ingress_bytes=339386368 link_bytes=339386368 token_bytes=609919872 weight_movement_bytes=34603008 total_nvlink_bytes=644522880 weight_to_token_ratio=0.056734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277083580e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297361408 dispatch_meta_bytes=370552 combine_payload_bytes=297361408 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=297731960 ingress_bytes=504979456 link_bytes=504979456 token_bytes=595093368 weight_movement_bytes=207618048 total_nvlink_bytes=802711416 weight_to_token_ratio=0.348883 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.900197039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278429696 dispatch_meta_bytes=320904 combine_payload_bytes=278429696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278750600 ingress_bytes=313032704 link_bytes=313032704 token_bytes=557180296 weight_movement_bytes=34603008 total_nvlink_bytes=591783304 weight_to_token_ratio=0.062104 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177916864e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83840 payload_rows_remote=72956 dup_rows=14464 dup_rows_remote=10988 dedup_ratio=0.147135 router_tokens_per_expert=[171, 2855, 85, 147, 41, 2687, 1573, 173, 0, 0, 0, 534, 1523, 0, 0, 766, 353, 4195, 7251, 9, 0, 1580, 5410, 0, 0, 1487, 2345, 0, 0, 0, 1095, 0, 3939, 170, 0, 0, 0, 0, 291, 226, 356, 0, 0, 84, 0, 1, 178, 1023, 76, 498, 796, 519, 0, 0, 0, 1096, 14, 0, 85, 0, 10, 402, 0, 5108] payload_rows_per_dst=[8512, 4798, 12548, 12110, 8774, 15488, 10884, 10726] dup_rows_per_dst=[1242, 848, 2156, 3454, 478, 2298, 3476, 512] combine_rows_per_src=[8512, 4798, 12548, 12110, 8774, 15488, 10884, 10726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267739136 dispatch_meta_bytes=332968 combine_payload_bytes=267739136 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268072104 ingress_bytes=336945152 link_bytes=336945152 token_bytes=535811240 weight_movement_bytes=69206016 total_nvlink_bytes=605017256 weight_to_token_ratio=0.129161 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267897481e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83186 payload_rows_remote=71010 dup_rows=15118 dup_rows_remote=12224 dedup_ratio=0.153788 router_tokens_per_expert=[3162, 0, 0, 0, 0, 0, 0, 0, 0, 156, 166, 0, 2515, 6386, 0, 203, 0, 0, 2, 0, 1, 152, 87, 191, 0, 61, 2, 1, 0, 83, 0, 1, 1, 4495, 3462, 0, 0, 1, 0, 0, 677, 845, 1, 0, 0, 83, 2546, 1668, 0, 84, 0, 218, 89, 7093, 6049, 1, 642, 0, 0, 0, 0, 1, 8027, 0] payload_rows_per_dst=[16054, 12176, 13388, 14482, 7742, 11616, 6442, 1286] dup_rows_per_dst=[6324, 2894, 250, 0, 646, 4954, 50, 0] combine_rows_per_src=[16054, 12176, 13388, 14482, 7742, 11616, 6442, 1286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=298827776 dispatch_meta_bytes=335776 combine_payload_bytes=298827776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299163552 ingress_bytes=333430784 link_bytes=333430784 token_bytes=597991328 weight_movement_bytes=34603008 total_nvlink_bytes=632594336 weight_to_token_ratio=0.057865 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254673197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90856 payload_rows_remote=76494 dup_rows=7448 dup_rows_remote=7448 dedup_ratio=0.075765 router_tokens_per_expert=[3378, 0, 0, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2559, 6261, 0, 178, 0, 0, 0, 0, 0, 130, 80, 204, 0, 31, 0, 0, 0, 80, 0, 0, 0, 4711, 3285, 0, 0, 0, 0, 0, 704, 832, 0, 0, 0, 80, 2579, 1646, 0, 80, 0, 162, 92, 7070, 5987, 0, 636, 0, 0, 0, 0, 0, 8092, 0] payload_rows_per_dst=[6756, 12394, 13142, 14362, 6570, 8886, 12562, 16184] dup_rows_per_dst=[0, 3092, 208, 0, 0, 2796, 80, 1272] combine_rows_per_src=[6756, 12394, 13142, 14362, 6570, 8886, 12562, 16184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90762 payload_rows_remote=81692 dup_rows=7542 dup_rows_remote=6020 dedup_ratio=0.076721 router_tokens_per_expert=[3513, 0, 0, 0, 0, 0, 0, 0, 0, 150, 151, 0, 2279, 6291, 0, 216, 0, 0, 0, 0, 0, 130, 96, 232, 0, 80, 9, 0, 0, 94, 0, 0, 0, 4751, 3123, 0, 0, 0, 0, 0, 690, 897, 0, 0, 0, 94, 2387, 1482, 0, 94, 0, 150, 92, 7120, 6143, 0, 829, 0, 0, 0, 0, 0, 8059, 0] payload_rows_per_dst=[7026, 9180, 13194, 14606, 9070, 8686, 12882, 16118] dup_rows_per_dst=[0, 1568, 304, 0, 1522, 2414, 76, 1658] combine_rows_per_src=[7026, 9180, 13194, 14606, 9070, 8686, 12882, 16118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79282 payload_rows_remote=63098 dup_rows=19022 dup_rows_remote=14640 dedup_ratio=0.193502 router_tokens_per_expert=[3443, 0, 0, 0, 0, 0, 0, 0, 0, 112, 82, 0, 2499, 6324, 0, 112, 0, 0, 0, 0, 0, 168, 68, 146, 0, 38, 1, 0, 0, 63, 0, 0, 0, 4629, 3201, 0, 0, 0, 0, 0, 753, 966, 0, 0, 0, 63, 2540, 1662, 0, 63, 0, 153, 46, 7191, 6025, 0, 712, 0, 0, 0, 0, 0, 8092, 0] payload_rows_per_dst=[6886, 5532, 13236, 204, 12206, 9204, 15830, 16184] dup_rows_per_dst=[0, 78, 176, 0, 3454, 2764, 8168, 4382] combine_rows_per_src=[6886, 5532, 13236, 204, 12206, 9204, 15830, 16184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86710 payload_rows_remote=73392 dup_rows=11594 dup_rows_remote=11382 dedup_ratio=0.117940 router_tokens_per_expert=[3566, 0, 0, 0, 0, 0, 0, 0, 0, 137, 155, 0, 2511, 6333, 0, 137, 0, 0, 0, 0, 0, 162, 107, 163, 0, 27, 1, 0, 0, 76, 0, 0, 0, 4617, 3303, 0, 0, 0, 0, 0, 725, 968, 0, 0, 0, 76, 2642, 1475, 0, 76, 0, 188, 89, 7105, 6001, 0, 415, 0, 0, 0, 0, 0, 8097, 0] payload_rows_per_dst=[14064, 12274, 13318, 14418, 6606, 8958, 12592, 4480] dup_rows_per_dst=[5442, 2840, 212, 0, 0, 2814, 116, 170] combine_rows_per_src=[14064, 12274, 13318, 14418, 6606, 8958, 12592, 4480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89986 payload_rows_remote=81226 dup_rows=8318 dup_rows_remote=5948 dedup_ratio=0.084615 router_tokens_per_expert=[3475, 0, 0, 0, 0, 0, 0, 0, 0, 171, 130, 0, 2165, 6476, 0, 96, 0, 0, 0, 0, 0, 116, 70, 169, 0, 60, 0, 0, 0, 70, 0, 0, 0, 4484, 3660, 0, 0, 0, 0, 0, 755, 883, 0, 0, 0, 70, 2278, 1579, 0, 70, 0, 290, 60, 7170, 5948, 0, 898, 0, 0, 0, 0, 0, 8009, 0] payload_rows_per_dst=[6950, 5020, 13468, 14600, 12594, 8760, 12574, 16020] dup_rows_per_dst=[0, 104, 194, 0, 3694, 2370, 162, 1794] combine_rows_per_src=[6950, 5020, 13468, 14600, 12594, 8760, 12574, 16020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290856960 dispatch_meta_bytes=332936 combine_payload_bytes=290856960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291189896 ingress_bytes=360062976 link_bytes=360062976 token_bytes=582046856 weight_movement_bytes=69206016 total_nvlink_bytes=651252872 weight_to_token_ratio=0.118901 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354887991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=313319424 dispatch_meta_bytes=335768 combine_payload_bytes=313319424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313655192 ingress_bytes=347922432 link_bytes=347922432 token_bytes=626974616 weight_movement_bytes=34603008 total_nvlink_bytes=661577624 weight_to_token_ratio=0.055190 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309204102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=310952 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258760360 ingress_bytes=327655424 link_bytes=327655424 token_bytes=517209768 weight_movement_bytes=69206016 total_nvlink_bytes=586415784 weight_to_token_ratio=0.133806 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232940982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73526 payload_rows_remote=57496 dup_rows=24778 dup_rows_remote=18636 dedup_ratio=0.252055 router_tokens_per_expert=[3075, 0, 3, 4, 6, 0, 3, 13, 3, 144, 170, 1, 2467, 6201, 1, 186, 0, 0, 10, 0, 5, 149, 102, 211, 2, 51, 6, 2, 0, 90, 1, 0, 1, 4706, 3323, 0, 0, 7, 2, 0, 700, 1004, 10, 0, 0, 88, 2743, 1756, 8, 90, 11, 256, 113, 6893, 5960, 8, 571, 0, 0, 1, 8, 5, 7982, 0] payload_rows_per_dst=[16030, 15306, 3296, 14090, 13244, 9442, 952, 1166] dup_rows_per_dst=[6142, 9914, 196, 0, 5342, 3160, 20, 4] combine_rows_per_src=[16030, 15306, 3296, 14090, 13244, 9442, 952, 1166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=334610432 dispatch_meta_bytes=350848 combine_payload_bytes=334610432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334961280 ingress_bytes=403816448 link_bytes=403816448 token_bytes=669571712 weight_movement_bytes=69206016 total_nvlink_bytes=738777728 weight_to_token_ratio=0.103359 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519528784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300613632 dispatch_meta_bytes=339096 combine_payload_bytes=300613632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300952728 ingress_bytes=335216640 link_bytes=335216640 token_bytes=601566360 weight_movement_bytes=34603008 total_nvlink_bytes=636169368 weight_to_token_ratio=0.057522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=332701696 dispatch_meta_bytes=348696 combine_payload_bytes=332701696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333050392 ingress_bytes=367304704 link_bytes=367304704 token_bytes=665752088 weight_movement_bytes=34603008 total_nvlink_bytes=700355096 weight_to_token_ratio=0.051976 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382138031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88202 payload_rows_remote=74748 dup_rows=10102 dup_rows_remote=7758 dedup_ratio=0.102763 router_tokens_per_expert=[3315, 0, 0, 0, 0, 0, 0, 0, 0, 164, 177, 0, 2472, 6413, 0, 153, 0, 0, 1, 0, 0, 146, 96, 205, 0, 25, 0, 0, 0, 85, 0, 0, 0, 4425, 3382, 0, 0, 0, 0, 0, 685, 908, 0, 0, 1, 85, 2319, 1742, 0, 85, 0, 207, 86, 7051, 6131, 0, 750, 0, 0, 0, 0, 0, 8043, 0] payload_rows_per_dst=[6630, 5754, 13484, 11542, 12316, 8936, 13454, 16086] dup_rows_per_dst=[0, 178, 238, 0, 3298, 2544, 2344, 1500] combine_rows_per_src=[6630, 5754, 13484, 11542, 12316, 8936, 13454, 16086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=235503616 dispatch_meta_bytes=304528 combine_payload_bytes=235503616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235808144 ingress_bytes=270106624 link_bytes=270106624 token_bytes=471311760 weight_movement_bytes=34603008 total_nvlink_bytes=505914768 weight_to_token_ratio=0.073419 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016389481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306167808 dispatch_meta_bytes=330024 combine_payload_bytes=306167808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=306497832 ingress_bytes=444579840 link_bytes=444579840 token_bytes=612665640 weight_movement_bytes=138412032 total_nvlink_bytes=751077672 weight_to_token_ratio=0.225918 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.672918146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85032 payload_rows_remote=73464 dup_rows=13272 dup_rows_remote=10398 dedup_ratio=0.135010 router_tokens_per_expert=[252, 718, 1, 0, 252, 0, 0, 0, 0, 3831, 121, 5044, 0, 0, 1663, 5262, 89, 861, 1, 0, 573, 0, 0, 337, 658, 1053, 0, 6071, 0, 0, 0, 0, 1, 0, 0, 1, 4530, 1091, 309, 0, 15, 1, 1, 0, 0, 89, 0, 1, 4013, 1, 0, 6101, 420, 2243, 1132, 0, 0, 0, 0, 0, 0, 430, 1896, 90] payload_rows_per_dst=[12398, 11568, 9060, 13432, 10636, 10708, 6636, 10594] dup_rows_per_dst=[2250, 2874, 1538, 2132, 1228, 30, 956, 2264] combine_rows_per_src=[12398, 11568, 9060, 13432, 10636, 10708, 6636, 10594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86366 payload_rows_remote=72238 dup_rows=11938 dup_rows_remote=9514 dedup_ratio=0.121440 router_tokens_per_expert=[245, 663, 0, 1, 245, 0, 0, 0, 0, 3778, 122, 4955, 0, 0, 1530, 5098, 74, 826, 0, 0, 548, 0, 0, 378, 633, 1144, 0, 6498, 1, 0, 0, 0, 0, 0, 0, 0, 4664, 1071, 274, 0, 27, 0, 0, 0, 0, 92, 0, 0, 3949, 0, 0, 6134, 458, 2162, 1182, 0, 0, 0, 0, 0, 0, 487, 1821, 92] payload_rows_per_dst=[12406, 9958, 11700, 14128, 10844, 10380, 6590, 10360] dup_rows_per_dst=[2170, 902, 1862, 2424, 1174, 54, 1014, 2338] combine_rows_per_src=[12406, 9958, 11700, 14128, 10844, 10380, 6590, 10360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86854 payload_rows_remote=76608 dup_rows=11450 dup_rows_remote=10416 dedup_ratio=0.116475 router_tokens_per_expert=[251, 576, 0, 0, 251, 0, 0, 0, 0, 4125, 98, 5135, 0, 0, 1616, 5144, 69, 914, 0, 0, 562, 0, 0, 333, 622, 989, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 4334, 1026, 280, 0, 1, 0, 0, 0, 0, 92, 0, 0, 4000, 0, 0, 6178, 427, 2126, 1059, 0, 0, 0, 0, 0, 0, 544, 1976, 92] payload_rows_per_dst=[12480, 10660, 12114, 13824, 10246, 10472, 6312, 10746] dup_rows_per_dst=[2032, 1018, 1912, 2062, 1034, 2, 912, 2478] combine_rows_per_src=[12480, 10660, 12114, 13824, 10246, 10472, 6312, 10746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70436 payload_rows_remote=59678 dup_rows=27868 dup_rows_remote=25444 dedup_ratio=0.283488 router_tokens_per_expert=[264, 584, 0, 0, 264, 0, 0, 0, 0, 3886, 67, 5228, 0, 0, 1534, 5087, 65, 922, 0, 0, 572, 0, 0, 403, 540, 925, 0, 6428, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 1013, 302, 1, 88, 0, 0, 0, 0, 46, 0, 0, 4164, 0, 0, 6242, 469, 2089, 1065, 0, 0, 0, 0, 0, 0, 452, 1929, 46] payload_rows_per_dst=[1696, 3132, 12332, 2930, 10440, 13482, 15666, 10758] dup_rows_per_dst=[528, 70, 2048, 0, 1146, 4732, 16920, 2424] combine_rows_per_src=[1696, 3132, 12332, 2930, 10440, 13482, 15666, 10758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86694 payload_rows_remote=74556 dup_rows=11610 dup_rows_remote=9690 dedup_ratio=0.118103 router_tokens_per_expert=[244, 585, 0, 5, 245, 0, 0, 0, 0, 3746, 116, 5131, 0, 0, 1598, 5038, 86, 875, 0, 0, 566, 0, 0, 371, 687, 990, 0, 6605, 0, 0, 0, 0, 0, 0, 0, 0, 4724, 1038, 296, 0, 2, 0, 0, 0, 0, 90, 0, 0, 3832, 0, 0, 6151, 454, 2130, 1121, 0, 0, 0, 0, 0, 0, 420, 1917, 89] payload_rows_per_dst=[12396, 10050, 12138, 14190, 10940, 10256, 6434, 10290] dup_rows_per_dst=[2064, 870, 1920, 2374, 1176, 4, 976, 2226] combine_rows_per_src=[12396, 10050, 12138, 14190, 10940, 10256, 6434, 10290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=76018 dup_rows=11796 dup_rows_remote=11796 dedup_ratio=0.119995 router_tokens_per_expert=[274, 736, 0, 0, 274, 0, 0, 0, 0, 3577, 115, 5091, 0, 0, 1490, 5185, 110, 876, 0, 0, 566, 0, 0, 400, 634, 1233, 0, 6393, 0, 3, 0, 0, 0, 0, 0, 0, 4637, 899, 289, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4043, 0, 0, 5988, 469, 2312, 1136, 0, 0, 0, 0, 0, 0, 487, 1815, 60] payload_rows_per_dst=[12200, 9536, 12172, 14060, 10682, 10490, 6836, 10532] dup_rows_per_dst=[2344, 828, 1914, 2466, 968, 0, 998, 2278] combine_rows_per_src=[12200, 9536, 12172, 14060, 10682, 10490, 6836, 10532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300908544 dispatch_meta_bytes=335448 combine_payload_bytes=300908544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301243992 ingress_bytes=439320576 link_bytes=439320576 token_bytes=602152536 weight_movement_bytes=138412032 total_nvlink_bytes=740564568 weight_to_token_ratio=0.229862 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653127959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73494 payload_rows_remote=61070 dup_rows=24810 dup_rows_remote=22622 dedup_ratio=0.252380 router_tokens_per_expert=[250, 667, 15, 24, 248, 0, 0, 0, 1, 3644, 113, 5096, 4, 1, 1360, 5237, 91, 892, 10, 4, 588, 0, 0, 362, 657, 1342, 0, 6207, 7, 3, 0, 0, 2, 2, 1, 8, 4267, 1060, 302, 0, 37, 0, 0, 4, 0, 104, 0, 9, 4346, 13, 0, 6102, 451, 2033, 1142, 0, 0, 3, 7, 2, 1, 517, 1812, 104] payload_rows_per_dst=[12424, 15510, 2984, 4622, 10194, 10704, 9794, 7262] dup_rows_per_dst=[2188, 16558, 910, 180, 1090, 78, 2642, 1164] combine_rows_per_src=[12424, 15510, 2984, 4622, 10194, 10704, 9794, 7262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=313786368 dispatch_meta_bytes=348096 combine_payload_bytes=313786368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314134464 ingress_bytes=348389376 link_bytes=348389376 token_bytes=627920832 weight_movement_bytes=34603008 total_nvlink_bytes=662523840 weight_to_token_ratio=0.055107 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310961174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=244441088 dispatch_meta_bytes=340488 combine_payload_bytes=244441088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244781576 ingress_bytes=279044096 link_bytes=279044096 token_bytes=489222664 weight_movement_bytes=34603008 total_nvlink_bytes=523825672 weight_to_token_ratio=0.070731 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.050020468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=295886848 dispatch_meta_bytes=327008 combine_payload_bytes=295886848 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=296213856 ingress_bytes=399695872 link_bytes=399695872 token_bytes=592100704 weight_movement_bytes=103809024 total_nvlink_bytes=695909728 weight_to_token_ratio=0.175323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504023388e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=305381376 dispatch_meta_bytes=336984 combine_payload_bytes=305381376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305718360 ingress_bytes=339984384 link_bytes=339984384 token_bytes=611099736 weight_movement_bytes=34603008 total_nvlink_bytes=645702744 weight_to_token_ratio=0.056624 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311369728 dispatch_meta_bytes=351256 combine_payload_bytes=311369728 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=311720984 ingress_bytes=380575744 link_bytes=380575744 token_bytes=623090712 weight_movement_bytes=69206016 total_nvlink_bytes=692296728 weight_to_token_ratio=0.111069 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432075886e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78732 payload_rows_remote=67884 dup_rows=19572 dup_rows_remote=14802 dedup_ratio=0.199097 router_tokens_per_expert=[260, 733, 0, 36, 260, 0, 0, 0, 0, 3913, 138, 5026, 0, 0, 1663, 5024, 126, 781, 0, 0, 583, 0, 0, 334, 659, 1038, 0, 6426, 0, 0, 0, 0, 0, 0, 0, 0, 4542, 975, 263, 0, 14, 0, 0, 0, 0, 89, 0, 0, 3595, 0, 0, 6186, 419, 2274, 1095, 0, 0, 0, 0, 0, 0, 566, 2048, 86] payload_rows_per_dst=[7250, 4464, 11884, 8484, 12618, 12922, 10848, 10262] dup_rows_per_dst=[1478, 306, 1816, 996, 3888, 3990, 4770, 2328] combine_rows_per_src=[7250, 4464, 11884, 8484, 12618, 12922, 10848, 10262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=250142720 dispatch_meta_bytes=334768 combine_payload_bytes=250142720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250477488 ingress_bytes=284745728 link_bytes=284745728 token_bytes=500620208 weight_movement_bytes=34603008 total_nvlink_bytes=535223216 weight_to_token_ratio=0.069120 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071475251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278052864 dispatch_meta_bytes=330744 combine_payload_bytes=278052864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278383608 ingress_bytes=381861888 link_bytes=381861888 token_bytes=556436472 weight_movement_bytes=103809024 total_nvlink_bytes=660245496 weight_to_token_ratio=0.186560 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436915542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76780 payload_rows_remote=61138 dup_rows=21524 dup_rows_remote=19790 dedup_ratio=0.218953 router_tokens_per_expert=[6459, 83, 1, 212, 5065, 0, 358, 186, 0, 4, 1, 918, 1, 0, 0, 7764, 0, 1385, 4876, 0, 4, 0, 669, 0, 1, 767, 2, 0, 0, 152, 0, 1, 2087, 0, 0, 32, 2, 1386, 5103, 1, 0, 0, 0, 0, 0, 2, 0, 809, 1318, 0, 0, 0, 0, 2444, 42, 1, 2338, 2, 0, 237, 486, 3241, 89, 623] payload_rows_per_dst=[15320, 15642, 11570, 1842, 12530, 6138, 7410, 6328] dup_rows_per_dst=[10764, 1734, 2298, 4, 4692, 610, 200, 1222] combine_rows_per_src=[15320, 15642, 11570, 1842, 12530, 6138, 7410, 6328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83668 payload_rows_remote=77534 dup_rows=14636 dup_rows_remote=14384 dedup_ratio=0.148885 router_tokens_per_expert=[6666, 94, 0, 187, 5363, 0, 350, 194, 0, 0, 0, 1009, 0, 0, 0, 7772, 0, 1505, 4667, 0, 0, 0, 656, 0, 0, 878, 0, 0, 0, 153, 0, 0, 1830, 0, 0, 0, 0, 1363, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 892, 1207, 0, 0, 0, 0, 2234, 62, 0, 2156, 0, 0, 254, 493, 3363, 69, 640] payload_rows_per_dst=[2456, 10846, 11264, 15732, 6134, 13470, 12944, 10822] dup_rows_per_dst=[258, 834, 2392, 1874, 252, 1646, 4252, 3128] combine_rows_per_src=[2456, 10846, 11264, 15732, 6134, 13470, 12944, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79838 payload_rows_remote=65934 dup_rows=18466 dup_rows_remote=9880 dedup_ratio=0.187846 router_tokens_per_expert=[6768, 63, 0, 220, 5100, 0, 404, 114, 0, 2, 0, 791, 0, 0, 0, 7817, 0, 1538, 4667, 0, 0, 0, 629, 0, 0, 913, 0, 0, 0, 98, 0, 0, 2088, 0, 0, 9, 0, 1504, 5079, 0, 0, 0, 0, 0, 0, 0, 0, 788, 1334, 0, 0, 0, 0, 2004, 45, 0, 2278, 0, 0, 204, 542, 3273, 65, 815] payload_rows_per_dst=[1278, 11040, 3712, 15760, 7528, 13658, 12958, 13904] dup_rows_per_dst=[324, 746, 622, 1896, 872, 1454, 3966, 8586] combine_rows_per_src=[1278, 11040, 3712, 15760, 7528, 13658, 12958, 13904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82986 payload_rows_remote=67296 dup_rows=15318 dup_rows_remote=13444 dedup_ratio=0.155823 router_tokens_per_expert=[6692, 80, 0, 232, 4889, 0, 365, 186, 0, 0, 0, 929, 0, 0, 0, 7765, 0, 1691, 5148, 0, 0, 0, 722, 0, 0, 877, 0, 0, 0, 140, 0, 0, 1862, 0, 0, 2, 0, 1393, 5175, 0, 3, 0, 0, 0, 0, 0, 0, 786, 1296, 0, 0, 0, 0, 2071, 22, 0, 2027, 0, 0, 224, 525, 3399, 68, 583] payload_rows_per_dst=[12240, 1858, 12276, 15690, 11078, 13504, 8022, 8318] dup_rows_per_dst=[2260, 0, 2846, 1874, 3432, 1458, 1110, 2338] combine_rows_per_src=[12240, 1858, 12276, 15690, 11078, 13504, 8022, 8318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82392 payload_rows_remote=71050 dup_rows=15912 dup_rows_remote=13342 dedup_ratio=0.161865 router_tokens_per_expert=[6858, 76, 0, 232, 5044, 0, 391, 174, 0, 0, 0, 924, 0, 1, 0, 7724, 0, 1671, 4520, 0, 0, 0, 765, 0, 0, 894, 0, 0, 0, 140, 0, 0, 1761, 0, 0, 0, 1, 1508, 5072, 0, 0, 0, 0, 0, 0, 0, 0, 729, 1271, 0, 0, 0, 0, 2171, 60, 0, 2224, 0, 0, 221, 479, 3382, 85, 774] payload_rows_per_dst=[14162, 11880, 11342, 5770, 12300, 13796, 6850, 6292] dup_rows_per_dst=[4436, 1210, 2570, 506, 4384, 1378, 154, 1274] combine_rows_per_src=[14162, 11880, 11342, 5770, 12300, 13796, 6850, 6292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83694 payload_rows_remote=70364 dup_rows=14610 dup_rows_remote=13028 dedup_ratio=0.148621 router_tokens_per_expert=[6599, 70, 0, 212, 5516, 0, 363, 149, 0, 0, 0, 949, 0, 0, 0, 7798, 0, 1614, 4815, 0, 3, 0, 467, 0, 0, 890, 0, 0, 0, 140, 0, 0, 1954, 0, 0, 2, 0, 1325, 4584, 0, 0, 0, 0, 0, 0, 0, 0, 857, 1254, 0, 0, 0, 0, 2261, 42, 0, 2725, 0, 0, 213, 530, 2878, 110, 832] payload_rows_per_dst=[1356, 12010, 11280, 15736, 6284, 13330, 12526, 11172] dup_rows_per_dst=[232, 920, 2518, 1920, 278, 1582, 3756, 3404] combine_rows_per_src=[1356, 12010, 11280, 15736, 6284, 13330, 12526, 11172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=250421248 dispatch_meta_bytes=323712 combine_payload_bytes=250421248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250744960 ingress_bytes=319627264 link_bytes=319627264 token_bytes=501166208 weight_movement_bytes=69206016 total_nvlink_bytes=570372224 weight_to_token_ratio=0.138090 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202731662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72480 payload_rows_remote=56456 dup_rows=25824 dup_rows_remote=10418 dedup_ratio=0.262695 router_tokens_per_expert=[6510, 88, 0, 229, 4891, 3, 364, 208, 4, 9, 11, 924, 16, 0, 2, 7736, 0, 1657, 4493, 0, 70, 0, 703, 4, 0, 929, 0, 0, 0, 161, 7, 3, 1902, 0, 2, 48, 0, 1373, 5160, 10, 6, 4, 1, 1, 0, 12, 16, 704, 1360, 1, 0, 0, 3, 2343, 50, 10, 2166, 3, 0, 316, 502, 3422, 101, 614] payload_rows_per_dst=[16024, 15596, 11264, 2192, 12464, 1462, 7278, 6200] dup_rows_per_dst=[15406, 1808, 2590, 8, 4526, 26, 256, 1204] combine_rows_per_src=[16024, 15596, 11264, 2192, 12464, 1462, 7278, 6200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=317579264 dispatch_meta_bytes=367672 combine_payload_bytes=317579264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317946936 ingress_bytes=386785280 link_bytes=386785280 token_bytes=635526200 weight_movement_bytes=69206016 total_nvlink_bytes=704732216 weight_to_token_ratio=0.108896 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455441870e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=270065664 dispatch_meta_bytes=303256 combine_payload_bytes=270065664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270368920 ingress_bytes=339271680 link_bytes=339271680 token_bytes=540434584 weight_movement_bytes=69206016 total_nvlink_bytes=609640600 weight_to_token_ratio=0.128056 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276652018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275644416 dispatch_meta_bytes=322960 combine_payload_bytes=275644416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275967376 ingress_bytes=310247424 link_bytes=310247424 token_bytes=551611792 weight_movement_bytes=34603008 total_nvlink_bytes=586214800 weight_to_token_ratio=0.062731 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167436080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291020800 dispatch_meta_bytes=337568 combine_payload_bytes=291020800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=291358368 ingress_bytes=360226816 link_bytes=360226816 token_bytes=582379168 weight_movement_bytes=69206016 total_nvlink_bytes=651585184 weight_to_token_ratio=0.118833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355504508e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83206 payload_rows_remote=70526 dup_rows=15098 dup_rows_remote=11156 dedup_ratio=0.153585 router_tokens_per_expert=[6634, 85, 0, 253, 5167, 0, 363, 193, 0, 0, 0, 923, 0, 0, 0, 7750, 0, 1614, 4927, 0, 0, 0, 661, 0, 0, 825, 0, 0, 0, 175, 0, 0, 2044, 0, 0, 21, 0, 1388, 4908, 0, 1, 0, 0, 0, 0, 0, 0, 761, 1247, 0, 0, 0, 0, 2130, 26, 0, 2156, 0, 0, 264, 553, 3221, 125, 737] payload_rows_per_dst=[1496, 11360, 8068, 15670, 9638, 13400, 12680, 10894] dup_rows_per_dst=[292, 820, 1692, 1830, 1912, 1392, 3942, 3218] combine_rows_per_src=[1496, 11360, 8068, 15670, 9638, 13400, 12680, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288210944 dispatch_meta_bytes=333568 combine_payload_bytes=288210944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288544512 ingress_bytes=322813952 link_bytes=322813952 token_bytes=576755456 weight_movement_bytes=34603008 total_nvlink_bytes=611358464 weight_to_token_ratio=0.059996 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214722913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231243776 dispatch_meta_bytes=267496 combine_payload_bytes=231243776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=231511272 ingress_bytes=335052800 link_bytes=335052800 token_bytes=462755048 weight_movement_bytes=103809024 total_nvlink_bytes=566564072 weight_to_token_ratio=0.224328 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288874496 dispatch_meta_bytes=326728 combine_payload_bytes=288874496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289201224 ingress_bytes=323477504 link_bytes=323477504 token_bytes=578075720 weight_movement_bytes=34603008 total_nvlink_bytes=612678728 weight_to_token_ratio=0.059859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217219805e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83698 payload_rows_remote=67514 dup_rows=14606 dup_rows_remote=13132 dedup_ratio=0.148580 router_tokens_per_expert=[0, 8107, 206, 0, 0, 69, 0, 46, 0, 0, 1, 0, 0, 987, 0, 152, 0, 12, 0, 8100, 0, 2, 0, 7689, 0, 251, 0, 68, 0, 0, 1, 0, 0, 324, 83, 1, 1, 2, 7591, 0, 2526, 0, 2, 0, 108, 0, 0, 1, 7541, 1, 83, 0, 0, 0, 0, 2, 0, 0, 1496, 3612, 3, 0, 0, 84] payload_rows_per_dst=[16268, 16184, 16264, 640, 12694, 5264, 6430, 9954] dup_rows_per_dst=[5500, 1474, 3106, 0, 4002, 10, 78, 436] combine_rows_per_src=[16268, 16184, 16264, 640, 12694, 5264, 6430, 9954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92834 payload_rows_remote=77118 dup_rows=5470 dup_rows_remote=5142 dedup_ratio=0.055644 router_tokens_per_expert=[0, 8098, 170, 0, 0, 59, 0, 31, 0, 0, 0, 0, 0, 990, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 7697, 0, 251, 0, 81, 0, 0, 0, 0, 0, 321, 94, 0, 0, 0, 7607, 0, 2495, 0, 0, 0, 112, 0, 0, 0, 7585, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1404, 3718, 0, 0, 0, 94] payload_rows_per_dst=[9404, 16246, 0, 16196, 15716, 10054, 15170, 10048] dup_rows_per_dst=[284, 1434, 0, 664, 328, 2188, 188, 384] combine_rows_per_src=[9404, 16246, 0, 16196, 15716, 10054, 15170, 10048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82130 payload_rows_remote=67266 dup_rows=16174 dup_rows_remote=5912 dedup_ratio=0.164530 router_tokens_per_expert=[0, 8128, 166, 0, 0, 35, 0, 19, 0, 0, 0, 0, 0, 1047, 0, 98, 0, 0, 0, 8124, 0, 0, 0, 7736, 0, 264, 0, 64, 0, 0, 0, 0, 0, 286, 63, 0, 0, 0, 7520, 0, 2549, 0, 0, 0, 72, 0, 0, 0, 7574, 0, 63, 3, 0, 0, 0, 0, 0, 0, 1391, 3886, 0, 0, 0, 64] payload_rows_per_dst=[440, 2290, 15472, 16248, 1284, 16382, 15150, 14864] dup_rows_per_dst=[0, 0, 0, 656, 10, 5116, 130, 10262] combine_rows_per_src=[440, 2290, 15472, 16248, 1284, 16382, 15150, 14864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94906 payload_rows_remote=78682 dup_rows=3398 dup_rows_remote=2772 dedup_ratio=0.034566 router_tokens_per_expert=[0, 8112, 123, 0, 0, 59, 0, 23, 0, 0, 0, 0, 0, 1035, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 7725, 0, 244, 0, 69, 0, 0, 0, 0, 0, 272, 80, 0, 0, 0, 7569, 0, 2472, 0, 0, 0, 89, 0, 0, 0, 7574, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1349, 3945, 0, 0, 0, 80] payload_rows_per_dst=[16224, 16236, 0, 16224, 15626, 5114, 15148, 10334] dup_rows_per_dst=[410, 1564, 0, 626, 216, 8, 160, 414] combine_rows_per_src=[16224, 16236, 0, 16224, 15626, 5114, 15148, 10334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94936 payload_rows_remote=90810 dup_rows=3368 dup_rows_remote=3368 dedup_ratio=0.034261 router_tokens_per_expert=[0, 8116, 124, 0, 0, 64, 0, 16, 0, 0, 0, 0, 0, 1064, 0, 140, 0, 0, 0, 8116, 0, 0, 0, 7730, 0, 242, 0, 60, 0, 0, 0, 0, 0, 280, 76, 0, 0, 0, 7574, 0, 2503, 0, 0, 0, 100, 0, 0, 0, 7573, 0, 76, 0, 0, 0, 0, 0, 0, 0, 1468, 3755, 0, 0, 0, 75] payload_rows_per_dst=[16232, 16212, 4126, 12268, 15632, 5190, 15146, 10130] dup_rows_per_dst=[408, 1656, 0, 442, 228, 16, 152, 466] combine_rows_per_src=[16232, 16212, 4126, 12268, 15632, 5190, 15146, 10130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=276537344 dispatch_meta_bytes=322584 combine_payload_bytes=276537344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276859928 ingress_bytes=311140352 link_bytes=311140352 token_bytes=553397272 weight_movement_bytes=34603008 total_nvlink_bytes=588000280 weight_to_token_ratio=0.062528 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170796096e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91424 payload_rows_remote=75050 dup_rows=6880 dup_rows_remote=1754 dedup_ratio=0.069987 router_tokens_per_expert=[0, 8115, 207, 0, 0, 70, 0, 59, 0, 0, 0, 0, 0, 1050, 0, 140, 0, 2, 0, 8122, 0, 0, 0, 7673, 0, 274, 0, 61, 0, 0, 0, 0, 0, 331, 70, 0, 0, 0, 7517, 0, 2536, 0, 0, 0, 99, 0, 0, 0, 7522, 0, 70, 0, 0, 0, 0, 0, 0, 0, 1609, 3554, 0, 0, 0, 71] payload_rows_per_dst=[672, 4802, 12694, 16244, 15586, 16374, 15044, 10008] dup_rows_per_dst=[0, 234, 0, 670, 250, 5126, 140, 460] combine_rows_per_src=[672, 4802, 12694, 16244, 15586, 16374, 15044, 10008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65734 payload_rows_remote=49390 dup_rows=32570 dup_rows_remote=16976 dedup_ratio=0.331319 router_tokens_per_expert=[0, 8081, 231, 0, 0, 66, 1, 50, 0, 4, 9, 1, 0, 1009, 3, 154, 1, 17, 1, 8065, 4, 10, 0, 7649, 0, 248, 3, 53, 0, 1, 1, 0, 8, 295, 88, 9, 0, 1, 7540, 0, 2653, 1, 18, 1, 103, 6, 5, 19, 7538, 5, 88, 3, 0, 2, 0, 15, 0, 3, 1463, 3528, 2, 0, 2, 94] payload_rows_per_dst=[16344, 16112, 16360, 612, 802, 5552, 214, 9738] dup_rows_per_dst=[15594, 1546, 14912, 0, 0, 60, 12, 446] combine_rows_per_src=[16344, 16112, 16360, 612, 802, 5552, 214, 9738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275521536 dispatch_meta_bytes=292712 combine_payload_bytes=275521536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275814248 ingress_bytes=310124544 link_bytes=310124544 token_bytes=551335784 weight_movement_bytes=34603008 total_nvlink_bytes=585938792 weight_to_token_ratio=0.062762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166973692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=315875328 dispatch_meta_bytes=329040 combine_payload_bytes=315875328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=316204368 ingress_bytes=419684352 link_bytes=419684352 token_bytes=632079696 weight_movement_bytes=103809024 total_nvlink_bytes=735888720 weight_to_token_ratio=0.164234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.579238429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=322281472 dispatch_meta_bytes=325816 combine_payload_bytes=322281472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322607288 ingress_bytes=356884480 link_bytes=356884480 token_bytes=644888760 weight_movement_bytes=34603008 total_nvlink_bytes=679491768 weight_to_token_ratio=0.053657 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342927567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91296 payload_rows_remote=76324 dup_rows=7008 dup_rows_remote=6838 dedup_ratio=0.071289 router_tokens_per_expert=[0, 8107, 117, 0, 0, 89, 0, 56, 0, 0, 0, 0, 0, 982, 0, 175, 0, 4, 0, 8107, 0, 0, 0, 7681, 0, 260, 0, 40, 0, 0, 0, 0, 0, 286, 85, 0, 0, 0, 7594, 0, 2716, 0, 0, 0, 102, 0, 0, 0, 7486, 0, 85, 0, 0, 0, 0, 0, 0, 0, 1354, 3741, 0, 0, 0, 85] payload_rows_per_dst=[524, 2314, 15370, 16214, 15716, 16384, 14972, 9802] dup_rows_per_dst=[0, 0, 0, 600, 214, 5466, 170, 558] combine_rows_per_src=[524, 2314, 15370, 16214, 15716, 16384, 14972, 9802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=371957760 dispatch_meta_bytes=376712 combine_payload_bytes=371957760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372334472 ingress_bytes=475766784 link_bytes=475766784 token_bytes=744292232 weight_movement_bytes=103809024 total_nvlink_bytes=848101256 weight_to_token_ratio=0.139473 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.790272106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=307404800 dispatch_meta_bytes=307216 combine_payload_bytes=307404800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307712016 ingress_bytes=342007808 link_bytes=342007808 token_bytes=615116816 weight_movement_bytes=34603008 total_nvlink_bytes=649719824 weight_to_token_ratio=0.056254 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286947848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=202301440 dispatch_meta_bytes=265464 combine_payload_bytes=202301440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=202566904 ingress_bytes=271507456 link_bytes=271507456 token_bytes=404868344 weight_movement_bytes=69206016 total_nvlink_bytes=474074360 weight_to_token_ratio=0.170935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.021660699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312623104 dispatch_meta_bytes=332648 combine_payload_bytes=312623104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=312955752 ingress_bytes=381829120 link_bytes=381829120 token_bytes=625578856 weight_movement_bytes=69206016 total_nvlink_bytes=694784872 weight_to_token_ratio=0.110627 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436792239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82502 payload_rows_remote=66290 dup_rows=15802 dup_rows_remote=13460 dedup_ratio=0.160746 router_tokens_per_expert=[0, 0, 0, 495, 0, 0, 1, 2, 158, 69, 0, 0, 1114, 50, 1, 1, 0, 0, 2195, 0, 181, 0, 0, 1, 2375, 7956, 1, 2895, 0, 7884, 1, 0, 252, 4288, 0, 7313, 0, 7885, 0, 0, 0, 0, 323, 0, 0, 446, 1150, 7, 0, 90, 1, 2, 0, 0, 0, 0, 0, 1, 1754, 189, 70, 0, 0, 1] payload_rows_per_dst=[15778, 16212, 11126, 15518, 14686, 3576, 1584, 4022] dup_rows_per_dst=[988, 2342, 2204, 9540, 444, 276, 0, 8] combine_rows_per_src=[15778, 16212, 11126, 15518, 14686, 3576, 1584, 4022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=58090 dup_rows=23958 dup_rows_remote=13796 dedup_ratio=0.243713 router_tokens_per_expert=[0, 0, 0, 548, 0, 0, 0, 0, 100, 36, 0, 0, 1020, 5, 0, 0, 0, 0, 2117, 0, 190, 0, 0, 0, 2712, 8031, 0, 3333, 0, 7967, 0, 0, 185, 4052, 0, 7391, 0, 7922, 0, 0, 0, 0, 201, 0, 0, 438, 949, 1, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 87, 35, 0, 0, 0] payload_rows_per_dst=[1096, 2242, 7882, 0, 15854, 14954, 16062, 16256] dup_rows_per_dst=[0, 80, 1094, 0, 9526, 3006, 90, 10162] combine_rows_per_src=[1096, 2242, 7882, 0, 15854, 14954, 16062, 16256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88144 payload_rows_remote=87646 dup_rows=10160 dup_rows_remote=10160 dedup_ratio=0.103353 router_tokens_per_expert=[0, 0, 0, 459, 0, 0, 0, 0, 128, 59, 0, 0, 1078, 29, 0, 0, 0, 0, 1989, 0, 186, 0, 0, 0, 2528, 7945, 0, 3253, 0, 7864, 0, 0, 249, 4072, 0, 7288, 0, 7912, 0, 0, 0, 0, 298, 0, 0, 457, 1089, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 188, 59, 0, 0, 0] payload_rows_per_dst=[15824, 14084, 10600, 10152, 498, 14992, 15890, 6104] dup_rows_per_dst=[918, 1864, 1894, 1410, 0, 3272, 178, 624] combine_rows_per_src=[15824, 14084, 10600, 10152, 498, 14992, 15890, 6104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88784 payload_rows_remote=79210 dup_rows=9520 dup_rows_remote=8298 dedup_ratio=0.096842 router_tokens_per_expert=[0, 0, 0, 545, 0, 0, 0, 0, 134, 60, 0, 0, 1142, 11, 0, 0, 0, 0, 2127, 0, 317, 0, 0, 0, 2439, 7972, 0, 2959, 0, 7902, 0, 0, 232, 4088, 0, 7377, 0, 7936, 0, 0, 0, 0, 218, 0, 0, 424, 1165, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 79, 60, 0, 0, 0] payload_rows_per_dst=[15872, 16224, 10988, 9574, 4272, 11882, 15944, 4028] dup_rows_per_dst=[1090, 2274, 2076, 1222, 104, 2574, 178, 2] combine_rows_per_src=[15872, 16224, 10988, 9574, 4272, 11882, 15944, 4028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271523840 dispatch_meta_bytes=319000 combine_payload_bytes=271523840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271842840 ingress_bytes=306126848 link_bytes=306126848 token_bytes=543366680 weight_movement_bytes=34603008 total_nvlink_bytes=577969688 weight_to_token_ratio=0.063683 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=38272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151930684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90834 payload_rows_remote=80328 dup_rows=7470 dup_rows_remote=5612 dedup_ratio=0.075989 router_tokens_per_expert=[0, 0, 0, 531, 0, 0, 0, 0, 150, 64, 0, 0, 1164, 10, 0, 0, 0, 0, 2146, 0, 346, 0, 0, 0, 2669, 7976, 0, 3331, 0, 7914, 0, 0, 222, 3690, 0, 7292, 0, 7938, 0, 0, 0, 0, 245, 0, 0, 399, 1044, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 44, 65, 0, 0, 0] payload_rows_per_dst=[15876, 16232, 10506, 10486, 14636, 3158, 15952, 3988] dup_rows_per_dst=[1062, 2372, 1858, 1514, 392, 218, 54, 0] combine_rows_per_src=[15876, 16232, 10506, 10486, 14636, 3158, 15952, 3988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85056 payload_rows_remote=70154 dup_rows=13248 dup_rows_remote=10158 dedup_ratio=0.134766 router_tokens_per_expert=[0, 0, 0, 534, 0, 0, 0, 0, 183, 70, 0, 0, 1101, 9, 0, 0, 0, 0, 2663, 0, 14, 0, 0, 0, 2745, 7982, 0, 2203, 0, 7921, 0, 0, 238, 3916, 0, 7292, 0, 7909, 0, 0, 0, 0, 346, 0, 0, 432, 926, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 86, 70, 0, 0, 0] payload_rows_per_dst=[11024, 2562, 11010, 7990, 5462, 14902, 15964, 16142] dup_rows_per_dst=[716, 164, 2176, 540, 160, 3090, 120, 6282] combine_rows_per_src=[11024, 2562, 11010, 7990, 5462, 14902, 15964, 16142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81376 payload_rows_remote=65656 dup_rows=16928 dup_rows_remote=15794 dedup_ratio=0.172201 router_tokens_per_expert=[6, 1, 0, 557, 4, 0, 2, 19, 165, 67, 0, 14, 1289, 74, 7, 0, 0, 1, 1779, 0, 349, 14, 0, 3, 2695, 7917, 0, 2631, 13, 7836, 4, 5, 251, 4043, 0, 7166, 1, 7838, 9, 3, 1, 0, 353, 5, 10, 506, 1102, 10, 0, 104, 15, 1, 2, 1, 0, 3, 2, 16, 2008, 179, 66, 0, 0, 5] payload_rows_per_dst=[15720, 16154, 10568, 16050, 14416, 3686, 246, 4536] dup_rows_per_dst=[1134, 2750, 1810, 10480, 444, 288, 6, 16] combine_rows_per_src=[15720, 16154, 10568, 16050, 14416, 3686, 246, 4536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=237936640 dispatch_meta_bytes=287544 combine_payload_bytes=237936640 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=238224184 ingress_bytes=307142656 link_bytes=307142656 token_bytes=476160824 weight_movement_bytes=69206016 total_nvlink_bytes=545366840 weight_to_token_ratio=0.145342 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155753088e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81346 payload_rows_remote=65482 dup_rows=16958 dup_rows_remote=16786 dedup_ratio=0.172506 router_tokens_per_expert=[0, 0, 0, 603, 0, 0, 0, 0, 216, 90, 0, 0, 1054, 14, 0, 0, 0, 0, 1904, 0, 299, 0, 0, 0, 2445, 7932, 0, 3249, 0, 7889, 0, 0, 250, 4053, 1, 7367, 0, 7923, 0, 0, 0, 0, 280, 0, 0, 419, 1149, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 1753, 86, 90, 0, 0, 0] payload_rows_per_dst=[1206, 2540, 10810, 3828, 15864, 15020, 15864, 16214] dup_rows_per_dst=[0, 208, 1702, 0, 1546, 3410, 172, 9920] combine_rows_per_src=[1206, 2540, 10810, 3828, 15864, 15020, 15864, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=358998016 dispatch_meta_bytes=391224 combine_payload_bytes=358998016 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=359389240 ingress_bytes=497410048 link_bytes=497410048 token_bytes=718387256 weight_movement_bytes=138412032 total_nvlink_bytes=856799288 weight_to_token_ratio=0.192671 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871713966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324444160 dispatch_meta_bytes=350032 combine_payload_bytes=324444160 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=324794192 ingress_bytes=497459200 link_bytes=497459200 token_bytes=649238352 weight_movement_bytes=173015040 total_nvlink_bytes=822253392 weight_to_token_ratio=0.266489 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871898921e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=329023488 dispatch_meta_bytes=343760 combine_payload_bytes=329023488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329367248 ingress_bytes=363626496 link_bytes=363626496 token_bytes=658390736 weight_movement_bytes=34603008 total_nvlink_bytes=692993744 weight_to_token_ratio=0.052557 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368297230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287350784 dispatch_meta_bytes=321248 combine_payload_bytes=287350784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287672032 ingress_bytes=321953792 link_bytes=321953792 token_bytes=575022816 weight_movement_bytes=34603008 total_nvlink_bytes=609625824 weight_to_token_ratio=0.060177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211486200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=268926976 dispatch_meta_bytes=325800 combine_payload_bytes=268926976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269252776 ingress_bytes=303529984 link_bytes=303529984 token_bytes=538179752 weight_movement_bytes=34603008 total_nvlink_bytes=572782760 weight_to_token_ratio=0.064296 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142158894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=268214272 dispatch_meta_bytes=329072 combine_payload_bytes=268214272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268543344 ingress_bytes=302817280 link_bytes=302817280 token_bytes=536757616 weight_movement_bytes=34603008 total_nvlink_bytes=571360624 weight_to_token_ratio=0.064467 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139477046e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71294 payload_rows_remote=56088 dup_rows=27010 dup_rows_remote=27010 dedup_ratio=0.274760 router_tokens_per_expert=[0, 0, 4154, 89, 2239, 2921, 0, 334, 0, 0, 0, 0, 8, 1, 0, 0, 6, 0, 0, 0, 1, 8185, 0, 488, 0, 0, 58, 8189, 0, 0, 0, 70, 355, 0, 0, 629, 0, 1, 171, 405, 338, 1, 2, 5459, 0, 0, 2, 0, 7594, 0, 3, 1, 6120, 0, 91, 1, 30, 0, 355, 308, 464, 0, 1, 78] payload_rows_per_dst=[15436, 15206, 16380, 256, 2250, 11556, 192, 10018] dup_rows_per_dst=[7582, 0, 17358, 0, 872, 436, 0, 762] combine_rows_per_src=[15436, 15206, 16380, 256, 2250, 11556, 192, 10018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78686 payload_rows_remote=65700 dup_rows=19618 dup_rows_remote=16784 dedup_ratio=0.199565 router_tokens_per_expert=[0, 0, 3687, 65, 2474, 3149, 0, 296, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 8192, 0, 454, 0, 0, 48, 8192, 0, 0, 0, 35, 302, 0, 0, 546, 0, 0, 113, 476, 230, 0, 0, 5763, 0, 0, 0, 0, 7706, 0, 0, 0, 6293, 0, 43, 0, 9, 0, 352, 176, 473, 3, 0, 61] payload_rows_per_dst=[5078, 14, 922, 15404, 16384, 11860, 16038, 12986] dup_rows_per_dst=[592, 0, 0, 154, 2874, 126, 13038, 2834] combine_rows_per_src=[5078, 14, 922, 15404, 16384, 11860, 16038, 12986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92492 payload_rows_remote=76108 dup_rows=5812 dup_rows_remote=2792 dedup_ratio=0.059123 router_tokens_per_expert=[0, 0, 4076, 69, 2377, 2754, 0, 323, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 455, 0, 0, 44, 8192, 0, 0, 0, 59, 360, 0, 0, 633, 0, 0, 103, 414, 328, 0, 0, 5685, 0, 0, 0, 0, 7652, 0, 0, 0, 6170, 0, 61, 0, 16, 0, 373, 247, 442, 31, 0, 86] payload_rows_per_dst=[9416, 15322, 912, 16384, 16384, 11828, 12462, 9784] dup_rows_per_dst=[1630, 0, 0, 206, 3020, 198, 0, 758] combine_rows_per_src=[9416, 15322, 912, 16384, 16384, 11828, 12462, 9784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229736448 dispatch_meta_bytes=332392 combine_payload_bytes=229736448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230068840 ingress_bytes=264339456 link_bytes=264339456 token_bytes=459805288 weight_movement_bytes=34603008 total_nvlink_bytes=494408296 weight_to_token_ratio=0.075256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.946880919e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90982 payload_rows_remote=74598 dup_rows=7322 dup_rows_remote=5480 dedup_ratio=0.074483 router_tokens_per_expert=[0, 0, 3656, 86, 2515, 3240, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8192, 0, 509, 0, 0, 62, 8192, 0, 0, 0, 64, 245, 0, 0, 585, 0, 0, 89, 439, 307, 0, 0, 5507, 0, 0, 0, 0, 7694, 0, 0, 0, 6164, 0, 74, 0, 8, 0, 328, 195, 535, 15, 0, 76] payload_rows_per_dst=[10514, 15388, 16384, 15588, 2200, 12340, 9564, 9004] dup_rows_per_dst=[1902, 0, 1842, 240, 516, 2200, 0, 622] combine_rows_per_src=[10514, 15388, 16384, 15588, 2200, 12340, 9564, 9004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93052 payload_rows_remote=76668 dup_rows=5252 dup_rows_remote=5064 dedup_ratio=0.053426 router_tokens_per_expert=[0, 0, 3761, 74, 2323, 2894, 0, 339, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 532, 0, 0, 34, 8192, 0, 0, 0, 60, 398, 0, 0, 634, 0, 0, 136, 447, 312, 0, 0, 5739, 0, 0, 0, 0, 7640, 0, 0, 0, 6167, 0, 59, 0, 14, 0, 293, 314, 479, 33, 0, 78] payload_rows_per_dst=[9694, 15284, 8280, 16384, 9714, 11928, 12452, 9316] dup_rows_per_dst=[1566, 0, 502, 188, 2194, 174, 0, 628] combine_rows_per_src=[9694, 15284, 8280, 16384, 9714, 11928, 12452, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91908 payload_rows_remote=80046 dup_rows=6396 dup_rows_remote=6242 dedup_ratio=0.065063 router_tokens_per_expert=[0, 0, 4024, 110, 2151, 2968, 0, 322, 0, 0, 0, 0, 3, 0, 0, 0, 16, 0, 0, 0, 0, 8192, 0, 444, 0, 0, 17, 8192, 0, 0, 0, 70, 275, 0, 0, 596, 0, 0, 157, 460, 339, 0, 0, 5669, 0, 0, 0, 0, 7631, 0, 0, 0, 6508, 0, 59, 0, 35, 0, 241, 156, 449, 1, 0, 67] payload_rows_per_dst=[5402, 15268, 920, 16384, 16384, 11862, 13134, 12554] dup_rows_per_dst=[778, 0, 0, 174, 2976, 154, 0, 2314] combine_rows_per_src=[5402, 15268, 920, 16384, 16384, 11862, 13134, 12554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311738368 dispatch_meta_bytes=315600 combine_payload_bytes=311738368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312053968 ingress_bytes=346341376 link_bytes=346341376 token_bytes=623792336 weight_movement_bytes=34603008 total_nvlink_bytes=658395344 weight_to_token_ratio=0.055472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303254715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71324 payload_rows_remote=55736 dup_rows=26980 dup_rows_remote=19434 dedup_ratio=0.274455 router_tokens_per_expert=[1, 2, 4100, 100, 1870, 3009, 7, 340, 0, 0, 0, 4, 10, 10, 0, 5, 12, 0, 1, 0, 20, 8149, 10, 559, 0, 0, 84, 8165, 0, 2, 0, 66, 332, 4, 1, 640, 1, 12, 179, 410, 357, 16, 27, 5452, 0, 0, 15, 1, 7546, 0, 0, 1, 6238, 0, 74, 4, 56, 0, 389, 271, 473, 31, 8, 88] payload_rows_per_dst=[15588, 15134, 16376, 304, 2312, 11496, 158, 9956] dup_rows_per_dst=[7546, 16, 17456, 0, 846, 240, 0, 876] combine_rows_per_src=[15588, 15134, 16376, 304, 2312, 11496, 158, 9956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84528 payload_rows_remote=69990 dup_rows=13776 dup_rows_remote=7048 dedup_ratio=0.140137 router_tokens_per_expert=[0, 0, 3853, 126, 2314, 2832, 0, 350, 0, 0, 0, 0, 20, 0, 0, 0, 9, 0, 0, 0, 0, 8192, 0, 477, 0, 0, 38, 8192, 0, 0, 0, 89, 315, 0, 0, 654, 0, 0, 153, 375, 403, 0, 0, 5550, 0, 0, 0, 0, 7610, 0, 0, 0, 6269, 0, 70, 0, 30, 0, 331, 311, 506, 16, 0, 67] payload_rows_per_dst=[4912, 6672, 972, 16384, 16384, 11722, 14538, 12944] dup_rows_per_dst=[668, 0, 0, 254, 2994, 184, 6728, 2948] combine_rows_per_src=[4912, 6672, 972, 16384, 16384, 11722, 14538, 12944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=269107200 dispatch_meta_bytes=329936 combine_payload_bytes=269107200 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269437136 ingress_bytes=338313216 link_bytes=338313216 token_bytes=538544336 weight_movement_bytes=69206016 total_nvlink_bytes=607750352 weight_to_token_ratio=0.128506 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273045395e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305553408 dispatch_meta_bytes=320312 combine_payload_bytes=305553408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305873720 ingress_bytes=374759424 link_bytes=374759424 token_bytes=611427128 weight_movement_bytes=69206016 total_nvlink_bytes=680633144 weight_to_token_ratio=0.113188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410189542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=314032128 dispatch_meta_bytes=326928 combine_payload_bytes=314032128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=314359056 ingress_bytes=383238144 link_bytes=383238144 token_bytes=628391184 weight_movement_bytes=69206016 total_nvlink_bytes=697597200 weight_to_token_ratio=0.110132 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.442094283e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=327868416 dispatch_meta_bytes=345152 combine_payload_bytes=327868416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328213568 ingress_bytes=362471424 link_bytes=362471424 token_bytes=656081984 weight_movement_bytes=34603008 total_nvlink_bytes=690684992 weight_to_token_ratio=0.052742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363950787e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70142 payload_rows_remote=53796 dup_rows=28162 dup_rows_remote=4518 dedup_ratio=0.286479 router_tokens_per_expert=[1270, 0, 0, 0, 305, 88, 0, 0, 0, 694, 0, 0, 288, 0, 7338, 8106, 6425, 6263, 1, 83, 0, 0, 1, 1, 204, 1, 0, 251, 1, 117, 7498, 1241, 253, 83, 1, 428, 2, 0, 0, 0, 2, 0, 0, 1, 0, 407, 251, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 7544, 0, 0, 0, 0, 0] payload_rows_per_dst=[15612, 16346, 8846, 9440, 1528, 9864, 908, 7598] dup_rows_per_dst=[2710, 23644, 120, 1682, 6, 0, 0, 0] combine_rows_per_src=[15612, 16346, 8846, 9440, 1528, 9864, 908, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=228294656 dispatch_meta_bytes=300680 combine_payload_bytes=228294656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=228595336 ingress_bytes=332103680 link_bytes=332103680 token_bytes=456889992 weight_movement_bytes=103809024 total_nvlink_bytes=560699016 weight_to_token_ratio=0.227208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.249679411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=286679040 dispatch_meta_bytes=308152 combine_payload_bytes=286679040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286987192 ingress_bytes=425091072 link_bytes=425091072 token_bytes=573666232 weight_movement_bytes=138412032 total_nvlink_bytes=712078264 weight_to_token_ratio=0.241276 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599583481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=220348416 dispatch_meta_bytes=233256 combine_payload_bytes=220348416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=220581672 ingress_bytes=324157440 link_bytes=324157440 token_bytes=440930088 weight_movement_bytes=103809024 total_nvlink_bytes=544739112 weight_to_token_ratio=0.235432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219778350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93870 payload_rows_remote=78466 dup_rows=4434 dup_rows_remote=3580 dedup_ratio=0.045105 router_tokens_per_expert=[1277, 0, 0, 0, 277, 69, 0, 0, 0, 668, 0, 0, 302, 0, 7360, 8098, 6635, 6090, 0, 94, 0, 0, 0, 0, 181, 0, 0, 251, 0, 108, 7563, 1233, 250, 94, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7542, 0, 0, 0, 0, 0] payload_rows_per_dst=[12798, 1752, 12180, 5918, 15404, 14538, 16196, 15084] dup_rows_per_dst=[2138, 188, 188, 1064, 854, 2, 0, 0] combine_rows_per_src=[12798, 1752, 12180, 5918, 15404, 14538, 16196, 15084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94228 payload_rows_remote=79052 dup_rows=4076 dup_rows_remote=4076 dedup_ratio=0.041463 router_tokens_per_expert=[1168, 0, 0, 0, 271, 65, 0, 0, 0, 562, 0, 0, 241, 0, 7434, 8129, 6547, 6210, 0, 63, 0, 0, 0, 0, 199, 0, 0, 260, 0, 65, 7662, 1292, 264, 63, 0, 457, 0, 0, 0, 0, 1, 0, 0, 0, 0, 347, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0] payload_rows_per_dst=[3002, 1488, 12420, 16088, 15488, 14308, 16258, 15176] dup_rows_per_dst=[6, 118, 126, 2868, 948, 10, 0, 0] combine_rows_per_src=[3002, 1488, 12420, 16088, 15488, 14308, 16258, 15176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93712 payload_rows_remote=81648 dup_rows=4592 dup_rows_remote=4440 dedup_ratio=0.046712 router_tokens_per_expert=[1454, 0, 0, 0, 239, 85, 0, 0, 0, 640, 0, 0, 272, 0, 7301, 8116, 6578, 6032, 0, 76, 0, 0, 0, 0, 196, 0, 0, 211, 0, 76, 7489, 1316, 245, 76, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0] payload_rows_per_dst=[15588, 4194, 12064, 3176, 12806, 14490, 16232, 15162] dup_rows_per_dst=[2946, 260, 152, 422, 746, 66, 0, 0] combine_rows_per_src=[15588, 4194, 12064, 3176, 12806, 14490, 16232, 15162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94378 payload_rows_remote=80018 dup_rows=3926 dup_rows_remote=3926 dedup_ratio=0.039937 router_tokens_per_expert=[1142, 0, 0, 0, 319, 110, 0, 0, 0, 682, 0, 0, 292, 0, 7413, 8122, 6510, 6258, 0, 70, 0, 0, 0, 0, 214, 0, 0, 274, 0, 81, 7516, 1235, 275, 70, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 0, 0] payload_rows_per_dst=[3134, 1802, 12516, 15932, 15494, 14360, 16244, 14896] dup_rows_per_dst=[8, 146, 140, 2708, 924, 0, 0, 0] combine_rows_per_src=[3134, 1802, 12516, 15932, 15494, 14360, 16244, 14896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93738 payload_rows_remote=90720 dup_rows=4566 dup_rows_remote=4040 dedup_ratio=0.046448 router_tokens_per_expert=[1322, 0, 0, 0, 296, 74, 0, 0, 0, 689, 0, 0, 272, 0, 7360, 8112, 6614, 6039, 0, 80, 0, 0, 0, 0, 187, 0, 0, 244, 0, 100, 7522, 1241, 244, 80, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7555, 0, 0, 0, 0, 0] payload_rows_per_dst=[15630, 1758, 12078, 3018, 15392, 14528, 16224, 15110] dup_rows_per_dst=[2798, 164, 160, 526, 916, 2, 0, 0] combine_rows_per_src=[15630, 1758, 12078, 3018, 15392, 14528, 16224, 15110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=321396736 dispatch_meta_bytes=328184 combine_payload_bytes=321396736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321724920 ingress_bytes=355999744 link_bytes=355999744 token_bytes=643121656 weight_movement_bytes=34603008 total_nvlink_bytes=677724664 weight_to_token_ratio=0.053805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339598376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94558 payload_rows_remote=78344 dup_rows=3746 dup_rows_remote=3746 dedup_ratio=0.038106 router_tokens_per_expert=[1219, 0, 0, 0, 247, 126, 0, 0, 0, 713, 0, 0, 280, 0, 7312, 8107, 6652, 6109, 0, 85, 0, 0, 0, 0, 166, 0, 0, 260, 0, 105, 7454, 1244, 260, 85, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 0, 0, 0] payload_rows_per_dst=[3180, 1812, 12218, 15898, 15318, 14708, 16214, 15210] dup_rows_per_dst=[4, 174, 170, 2560, 838, 0, 0, 0] combine_rows_per_src=[3180, 1812, 12218, 15898, 15318, 14708, 16214, 15210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62494 payload_rows_remote=47020 dup_rows=35810 dup_rows_remote=32534 dedup_ratio=0.364278 router_tokens_per_expert=[1568, 0, 4, 1, 294, 91, 1, 0, 6, 739, 0, 0, 309, 0, 7261, 8063, 6452, 5851, 9, 88, 0, 0, 8, 7, 182, 18, 0, 248, 17, 172, 7416, 1296, 246, 89, 4, 470, 10, 2, 0, 0, 3, 0, 7, 11, 4, 412, 254, 23, 0, 0, 0, 3, 2, 0, 9, 0, 10, 0, 7468, 0, 9, 10, 0, 5] payload_rows_per_dst=[15474, 16330, 13312, 14286, 1636, 1390, 20, 46] dup_rows_per_dst=[3276, 28128, 1614, 2718, 6, 38, 8, 22] combine_rows_per_src=[15474, 16330, 13312, 14286, 1636, 1390, 20, 46] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=323796992 dispatch_meta_bytes=332512 combine_payload_bytes=323796992 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324129504 ingress_bytes=393003008 link_bytes=393003008 token_bytes=647926496 weight_movement_bytes=69206016 total_nvlink_bytes=717132512 weight_to_token_ratio=0.106812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478838680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=334430208 dispatch_meta_bytes=344352 combine_payload_bytes=334430208 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=334774560 ingress_bytes=438239232 link_bytes=438239232 token_bytes=669204768 weight_movement_bytes=103809024 total_nvlink_bytes=773013792 weight_to_token_ratio=0.155123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.649058949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327753728 dispatch_meta_bytes=335776 combine_payload_bytes=327753728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328089504 ingress_bytes=362356736 link_bytes=362356736 token_bytes=655843232 weight_movement_bytes=34603008 total_nvlink_bytes=690446240 weight_to_token_ratio=0.052761 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363519225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=371589120 dispatch_meta_bytes=379040 combine_payload_bytes=371589120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=371968160 ingress_bytes=440795136 link_bytes=440795136 token_bytes=743557280 weight_movement_bytes=69206016 total_nvlink_bytes=812763296 weight_to_token_ratio=0.093074 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.658676610e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79020 payload_rows_remote=63504 dup_rows=19284 dup_rows_remote=16474 dedup_ratio=0.196167 router_tokens_per_expert=[0, 0, 0, 973, 0, 0, 0, 84, 0, 0, 713, 1, 0, 2, 782, 0, 6508, 706, 2, 0, 7524, 0, 0, 1, 7117, 0, 0, 315, 0, 0, 0, 1, 0, 0, 0, 0, 7790, 0, 0, 0, 7665, 2, 163, 286, 0, 0, 0, 1, 2, 1, 0, 1, 7727, 2, 0, 0, 152, 0, 0, 0, 1, 630, 0, 0] payload_rows_per_dst=[13752, 15516, 16360, 14864, 1348, 15614, 10, 1556] dup_rows_per_dst=[1378, 2810, 14338, 2, 0, 744, 2, 10] combine_rows_per_src=[13752, 15516, 16360, 14864, 1348, 15614, 10, 1556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=320897024 dispatch_meta_bytes=328360 combine_payload_bytes=320897024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321225384 ingress_bytes=355500032 link_bytes=355500032 token_bytes=642122408 weight_movement_bytes=34603008 total_nvlink_bytes=676725416 weight_to_token_ratio=0.053888 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337718000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=192593920 dispatch_meta_bytes=318216 combine_payload_bytes=192593920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=192912136 ingress_bytes=227196928 link_bytes=227196928 token_bytes=385506056 weight_movement_bytes=34603008 total_nvlink_bytes=420109064 weight_to_token_ratio=0.089760 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.549237492e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93002 payload_rows_remote=77456 dup_rows=5302 dup_rows_remote=5302 dedup_ratio=0.053935 router_tokens_per_expert=[0, 0, 0, 961, 0, 0, 0, 92, 0, 0, 674, 0, 0, 0, 777, 0, 6637, 699, 0, 0, 7504, 0, 0, 0, 7093, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 7718, 0, 177, 296, 0, 0, 0, 0, 0, 0, 0, 0, 7713, 0, 0, 0, 153, 0, 0, 0, 0, 550, 0, 0] payload_rows_per_dst=[14042, 13382, 1398, 14856, 15546, 3344, 15426, 15008] dup_rows_per_dst=[1338, 2320, 0, 0, 0, 238, 0, 1406] combine_rows_per_src=[14042, 13382, 1398, 14856, 15546, 3344, 15426, 15008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260112384 dispatch_meta_bytes=319912 combine_payload_bytes=260112384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260432296 ingress_bytes=294715392 link_bytes=294715392 token_bytes=520544680 weight_movement_bytes=34603008 total_nvlink_bytes=555147688 weight_to_token_ratio=0.066475 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108990294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=66582 dup_rows=16492 dup_rows_remote=15258 dedup_ratio=0.167765 router_tokens_per_expert=[0, 0, 0, 940, 0, 0, 0, 62, 0, 0, 687, 0, 0, 0, 750, 0, 6595, 756, 0, 0, 7615, 0, 0, 0, 7119, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 0, 7806, 0, 110, 209, 0, 0, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 0, 98, 0, 0, 0, 0, 519, 0, 0] payload_rows_per_dst=[1880, 2386, 14436, 476, 15616, 15850, 15938, 15230] dup_rows_per_dst=[124, 488, 266, 0, 0, 400, 13980, 1234] combine_rows_per_src=[1880, 2386, 14436, 476, 15616, 15850, 15938, 15230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94344 payload_rows_remote=78686 dup_rows=3960 dup_rows_remote=3454 dedup_ratio=0.040283 router_tokens_per_expert=[0, 0, 0, 1004, 0, 0, 0, 71, 0, 0, 689, 0, 0, 0, 860, 0, 6351, 795, 0, 0, 7544, 0, 0, 0, 7095, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 0, 7688, 0, 126, 268, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 140, 0, 0, 0, 0, 670, 0, 0] payload_rows_per_dst=[13498, 2628, 1590, 14828, 15582, 15658, 15472, 15088] dup_rows_per_dst=[1354, 470, 0, 10, 0, 506, 0, 1620] combine_rows_per_src=[13498, 2628, 1590, 14828, 15582, 15658, 15472, 15088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92536 payload_rows_remote=87068 dup_rows=5768 dup_rows_remote=5590 dedup_ratio=0.058675 router_tokens_per_expert=[0, 0, 0, 815, 0, 0, 0, 75, 0, 0, 747, 0, 0, 0, 809, 0, 6508, 730, 0, 0, 7559, 0, 0, 0, 7037, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 7878, 0, 151, 205, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 138, 0, 0, 0, 0, 620, 0, 0] payload_rows_per_dst=[13734, 15914, 5468, 14328, 15632, 11446, 4714, 11300] dup_rows_per_dst=[1062, 2954, 178, 0, 0, 426, 0, 1148] combine_rows_per_src=[13734, 15914, 5468, 14328, 15632, 11446, 4714, 11300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=317259776 dispatch_meta_bytes=331032 combine_payload_bytes=317259776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317590808 ingress_bytes=386465792 link_bytes=386465792 token_bytes=634850584 weight_movement_bytes=69206016 total_nvlink_bytes=704056600 weight_to_token_ratio=0.109012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454239662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92588 payload_rows_remote=78018 dup_rows=5716 dup_rows_remote=5716 dedup_ratio=0.058146 router_tokens_per_expert=[0, 0, 0, 1198, 0, 0, 0, 80, 0, 0, 649, 0, 0, 0, 649, 0, 6452, 723, 0, 0, 7546, 0, 0, 0, 7073, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 7817, 0, 0, 0, 7632, 0, 161, 296, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 140, 0, 0, 0, 0, 665, 0, 0] payload_rows_per_dst=[13956, 15442, 1446, 14570, 15634, 730, 15718, 15092] dup_rows_per_dst=[1504, 2418, 0, 0, 0, 184, 0, 1610] combine_rows_per_src=[13956, 15442, 1446, 14570, 15634, 730, 15718, 15092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90362 payload_rows_remote=74798 dup_rows=7942 dup_rows_remote=6528 dedup_ratio=0.080790 router_tokens_per_expert=[0, 0, 0, 994, 0, 0, 0, 86, 0, 0, 750, 0, 0, 0, 816, 0, 6546, 742, 0, 0, 7504, 0, 0, 0, 7035, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 7707, 0, 129, 297, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 175, 0, 0, 0, 0, 531, 0, 0] payload_rows_per_dst=[5358, 2620, 10590, 9838, 15660, 15724, 15564, 15008] dup_rows_per_dst=[534, 512, 254, 2, 3272, 542, 1414, 1412] combine_rows_per_src=[5358, 2620, 10590, 9838, 15660, 15724, 15564, 15008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272719872 dispatch_meta_bytes=327360 combine_payload_bytes=272719872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273047232 ingress_bytes=307322880 link_bytes=307322880 token_bytes=545767104 weight_movement_bytes=34603008 total_nvlink_bytes=580370112 weight_to_token_ratio=0.063403 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156431256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77508 payload_rows_remote=64042 dup_rows=20796 dup_rows_remote=19472 dedup_ratio=0.211548 router_tokens_per_expert=[5, 3, 0, 979, 0, 0, 0, 87, 0, 0, 684, 11, 6, 26, 816, 5, 6321, 717, 21, 0, 7479, 0, 4, 1, 7005, 0, 0, 253, 2, 2, 0, 3, 0, 0, 5, 0, 7768, 1, 0, 1, 7674, 20, 161, 364, 1, 0, 0, 24, 1, 4, 0, 2, 7879, 17, 9, 0, 154, 0, 3, 0, 0, 633, 1, 0] payload_rows_per_dst=[13466, 15604, 16370, 14510, 14, 15916, 54, 1574] dup_rows_per_dst=[1324, 2840, 15610, 20, 0, 982, 12, 8] combine_rows_per_src=[13466, 15604, 16370, 14510, 14, 15916, 54, 1574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=322297856 dispatch_meta_bytes=328560 combine_payload_bytes=322297856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322626416 ingress_bytes=391503872 link_bytes=391503872 token_bytes=644924272 weight_movement_bytes=69206016 total_nvlink_bytes=714130288 weight_to_token_ratio=0.107309 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473197552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=356630528 dispatch_meta_bytes=370632 combine_payload_bytes=356630528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357001160 ingress_bytes=460439552 link_bytes=460439552 token_bytes=713631688 weight_movement_bytes=103809024 total_nvlink_bytes=817440712 weight_to_token_ratio=0.145466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.732596966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=319561728 dispatch_meta_bytes=334936 combine_payload_bytes=319561728 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=319896664 ingress_bytes=388767744 link_bytes=388767744 token_bytes=639458392 weight_movement_bytes=69206016 total_nvlink_bytes=708664408 weight_to_token_ratio=0.108226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462901722e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83738 payload_rows_remote=68192 dup_rows=14566 dup_rows_remote=14566 dedup_ratio=0.148173 router_tokens_per_expert=[83, 0, 2, 83, 2, 0, 1, 0, 7771, 0, 0, 0, 0, 1, 0, 1, 970, 0, 8190, 0, 0, 0, 1, 0, 467, 0, 0, 0, 0, 84, 0, 0, 2, 0, 0, 0, 0, 2, 1450, 0, 1300, 410, 4665, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6920, 27, 0, 2, 0, 0, 509, 8019, 0, 0] payload_rows_per_dst=[16384, 15546, 16382, 1102, 16042, 3410, 13850, 1022] dup_rows_per_dst=[336, 0, 11270, 0, 2904, 12, 44, 0] combine_rows_per_src=[16384, 15546, 16382, 1102, 16042, 3410, 13850, 1022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=306372608 dispatch_meta_bytes=325304 combine_payload_bytes=306372608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306697912 ingress_bytes=375578624 link_bytes=375578624 token_bytes=613070520 weight_movement_bytes=69206016 total_nvlink_bytes=682276536 weight_to_token_ratio=0.112884 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413272126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=262316032 dispatch_meta_bytes=334056 combine_payload_bytes=262316032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262650088 ingress_bytes=296919040 link_bytes=296919040 token_bytes=524966120 weight_movement_bytes=34603008 total_nvlink_bytes=559569128 weight_to_token_ratio=0.065915 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117282444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93688 payload_rows_remote=82188 dup_rows=4616 dup_rows_remote=2886 dedup_ratio=0.046956 router_tokens_per_expert=[94, 0, 0, 94, 0, 0, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 8192, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 94, 0, 1, 0, 0, 0, 0, 0, 0, 1320, 0, 1186, 353, 4713, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6915, 22, 0, 0, 0, 0, 556, 8029, 0, 0] payload_rows_per_dst=[16384, 15518, 5040, 16384, 11500, 8730, 13842, 6290] dup_rows_per_dst=[376, 0, 118, 1436, 1730, 634, 32, 290] combine_rows_per_src=[16384, 15518, 5040, 16384, 11500, 8730, 13842, 6290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=279314432 dispatch_meta_bytes=331032 combine_payload_bytes=279314432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279645464 ingress_bytes=348520448 link_bytes=348520448 token_bytes=558959896 weight_movement_bytes=69206016 total_nvlink_bytes=628165912 weight_to_token_ratio=0.123812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311454388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=336642048 dispatch_meta_bytes=340296 combine_payload_bytes=336642048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336982344 ingress_bytes=371245056 link_bytes=371245056 token_bytes=673624392 weight_movement_bytes=34603008 total_nvlink_bytes=708227400 weight_to_token_ratio=0.051368 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88250 payload_rows_remote=75152 dup_rows=10054 dup_rows_remote=7120 dedup_ratio=0.102275 router_tokens_per_expert=[70, 0, 0, 70, 0, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 0, 0, 853, 0, 8191, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 70, 0, 47, 0, 0, 0, 0, 0, 0, 1441, 0, 1221, 332, 5142, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6853, 21, 0, 0, 0, 0, 390, 8012, 0, 0] payload_rows_per_dst=[13774, 10442, 1706, 16382, 2882, 13098, 13722, 16244] dup_rows_per_dst=[248, 0, 0, 1174, 0, 2934, 26, 5672] combine_rows_per_src=[13774, 10442, 1706, 16382, 2882, 13098, 13722, 16244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68448 payload_rows_remote=52190 dup_rows=29856 dup_rows_remote=13452 dedup_ratio=0.303711 router_tokens_per_expert=[62, 0, 0, 63, 0, 0, 0, 0, 7802, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 8191, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 1490, 0, 1205, 304, 4652, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7036, 36, 0, 0, 0, 0, 465, 8064, 0, 0] payload_rows_per_dst=[250, 0, 2108, 16382, 2982, 16384, 14084, 16258] dup_rows_per_dst=[0, 0, 0, 1070, 0, 12322, 60, 16404] combine_rows_per_src=[250, 0, 2108, 16382, 2982, 16384, 14084, 16258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87880 payload_rows_remote=73724 dup_rows=10424 dup_rows_remote=3594 dedup_ratio=0.106038 router_tokens_per_expert=[76, 0, 0, 76, 0, 0, 0, 0, 7796, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 8192, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 0, 1320, 337, 4400, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7004, 32, 0, 0, 0, 0, 596, 8030, 0, 0] payload_rows_per_dst=[16384, 15592, 14156, 7156, 16060, 3312, 14028, 1192] dup_rows_per_dst=[304, 0, 6830, 438, 2806, 2, 44, 0] combine_rows_per_src=[16384, 15592, 14156, 7156, 16060, 3312, 14028, 1192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93250 payload_rows_remote=76866 dup_rows=5054 dup_rows_remote=3688 dedup_ratio=0.051412 router_tokens_per_expert=[80, 0, 0, 80, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 972, 0, 8192, 0, 0, 0, 0, 0, 603, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 1089, 413, 4627, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6940, 17, 0, 0, 0, 0, 526, 8038, 0, 0] payload_rows_per_dst=[16384, 15576, 10890, 16384, 16076, 2998, 13890, 1052] dup_rows_per_dst=[320, 0, 308, 1366, 3030, 6, 24, 0] combine_rows_per_src=[16384, 15576, 10890, 16384, 16076, 2998, 13890, 1052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68604 payload_rows_remote=54542 dup_rows=29700 dup_rows_remote=29678 dedup_ratio=0.302124 router_tokens_per_expert=[85, 0, 0, 85, 0, 0, 0, 0, 7763, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 8192, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 85, 0, 1, 0, 0, 0, 0, 0, 0, 1486, 0, 1308, 428, 4225, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7024, 18, 0, 0, 0, 0, 574, 7981, 0, 0] payload_rows_per_dst=[340, 0, 2248, 16384, 2972, 16384, 14062, 16214] dup_rows_per_dst=[0, 0, 0, 1334, 0, 11922, 22, 16422] combine_rows_per_src=[340, 0, 2248, 16384, 2972, 16384, 14062, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73038 payload_rows_remote=56658 dup_rows=25266 dup_rows_remote=24826 dedup_ratio=0.257019 router_tokens_per_expert=[87, 1, 28, 87, 6, 19, 22, 3, 7723, 0, 0, 0, 4, 6, 0, 7, 805, 1, 8164, 10, 3, 1, 12, 0, 540, 0, 0, 8, 2, 88, 0, 18, 16, 0, 0, 0, 0, 10, 1686, 0, 1198, 389, 4628, 8157, 0, 13, 0, 0, 6, 3, 0, 0, 0, 6, 6808, 17, 7, 22, 0, 8, 574, 7948, 1, 10] payload_rows_per_dst=[16380, 15620, 16366, 1278, 15942, 5684, 566, 1202] dup_rows_per_dst=[440, 10010, 10882, 34, 3378, 472, 8, 42] combine_rows_per_src=[16380, 15620, 16366, 1278, 15942, 5684, 566, 1202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=307822592 dispatch_meta_bytes=329088 combine_payload_bytes=307822592 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=308151680 ingress_bytes=411631616 link_bytes=411631616 token_bytes=615974272 weight_movement_bytes=103809024 total_nvlink_bytes=719783296 weight_to_token_ratio=0.168528 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548936632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=213770240 dispatch_meta_bytes=262568 combine_payload_bytes=213770240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=214032808 ingress_bytes=282976256 link_bytes=282976256 token_bytes=427803048 weight_movement_bytes=69206016 total_nvlink_bytes=497009064 weight_to_token_ratio=0.161771 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064816870e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93718 payload_rows_remote=77506 dup_rows=4586 dup_rows_remote=4256 dedup_ratio=0.046651 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 336, 0, 0, 247, 0, 1, 0, 0, 0, 0, 0, 7425, 0, 0, 1, 3423, 0, 2, 1, 0, 8023, 0, 0, 0, 69, 5029, 2, 0, 0, 0, 2, 0, 0, 18, 0, 2, 0, 5434, 8190, 2163, 5229, 0, 0, 0, 0, 531, 0, 2950, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70] payload_rows_per_dst=[15524, 16212, 9916, 5588, 16384, 12790, 6292, 11012] dup_rows_per_dst=[2, 330, 1546, 0, 40, 1998, 670, 0] combine_rows_per_src=[15524, 16212, 9916, 5588, 16384, 12790, 6292, 11012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301973504 dispatch_meta_bytes=309272 combine_payload_bytes=301973504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302282776 ingress_bytes=371179520 link_bytes=371179520 token_bytes=604256280 weight_movement_bytes=69206016 total_nvlink_bytes=673462296 weight_to_token_ratio=0.114531 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396718651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=314843136 dispatch_meta_bytes=322216 combine_payload_bytes=314843136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315165352 ingress_bytes=349446144 link_bytes=349446144 token_bytes=630008488 weight_movement_bytes=34603008 total_nvlink_bytes=664611496 weight_to_token_ratio=0.054925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314937707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94154 payload_rows_remote=77770 dup_rows=4150 dup_rows_remote=4118 dedup_ratio=0.042216 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 342, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 3472, 0, 0, 0, 0, 8004, 0, 0, 0, 59, 4835, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 5567, 8192, 2169, 5163, 0, 0, 0, 0, 565, 0, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] payload_rows_per_dst=[15692, 16196, 6944, 9788, 16384, 4338, 13516, 11296] dup_rows_per_dst=[0, 336, 0, 0, 32, 0, 3782, 0] combine_rows_per_src=[15692, 16196, 6944, 9788, 16384, 4338, 13516, 11296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=223404032 dispatch_meta_bytes=336880 combine_payload_bytes=223404032 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=223740912 ingress_bytes=292610048 link_bytes=292610048 token_bytes=447144944 weight_movement_bytes=69206016 total_nvlink_bytes=516350960 weight_to_token_ratio=0.154773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101068054e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=232071168 dispatch_meta_bytes=325936 combine_payload_bytes=232071168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=232397104 ingress_bytes=266674176 link_bytes=266674176 token_bytes=464468272 weight_movement_bytes=34603008 total_nvlink_bytes=499071280 weight_to_token_ratio=0.074500 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.003473455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317464576 dispatch_meta_bytes=327048 combine_payload_bytes=317464576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317791624 ingress_bytes=352067584 link_bytes=352067584 token_bytes=635256200 weight_movement_bytes=34603008 total_nvlink_bytes=669859208 weight_to_token_ratio=0.054471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324801975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318545920 dispatch_meta_bytes=327552 combine_payload_bytes=318545920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318873472 ingress_bytes=353148928 link_bytes=353148928 token_bytes=637419392 weight_movement_bytes=34603008 total_nvlink_bytes=672022400 weight_to_token_ratio=0.054286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328870985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92520 payload_rows_remote=88212 dup_rows=5784 dup_rows_remote=5708 dedup_ratio=0.058838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 334, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 7479, 0, 0, 0, 3265, 0, 0, 0, 0, 8052, 0, 0, 0, 70, 5300, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 5332, 8192, 2081, 5166, 0, 0, 0, 0, 509, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[15626, 14172, 6530, 11426, 16162, 4308, 13510, 10786] dup_rows_per_dst=[0, 352, 0, 1430, 24, 76, 3902, 0] combine_rows_per_src=[15626, 14172, 6530, 11426, 16162, 4308, 13510, 10786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70904 payload_rows_remote=55972 dup_rows=27400 dup_rows_remote=21322 dedup_ratio=0.278727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 306, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 7508, 0, 0, 0, 3706, 0, 0, 0, 0, 8066, 0, 0, 0, 35, 5011, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5387, 8192, 2171, 5054, 0, 0, 0, 0, 495, 0, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] payload_rows_per_dst=[612, 514, 16214, 16132, 2, 16384, 6114, 14932] dup_rows_per_dst=[0, 0, 6214, 9152, 0, 5282, 674, 6078] combine_rows_per_src=[612, 514, 16214, 16132, 2, 16384, 6114, 14932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95110 payload_rows_remote=87012 dup_rows=3194 dup_rows_remote=3194 dedup_ratio=0.032491 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 7563, 0, 0, 0, 4049, 0, 0, 0, 0, 8040, 0, 0, 0, 64, 4429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5596, 8192, 2311, 4883, 0, 0, 0, 0, 515, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[15776, 16230, 8098, 8986, 16384, 10858, 7464, 11314] dup_rows_per_dst=[0, 382, 0, 0, 0, 1594, 1218, 0] combine_rows_per_src=[15776, 16230, 8098, 8986, 16384, 10858, 7464, 11314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93828 payload_rows_remote=83986 dup_rows=4476 dup_rows_remote=4476 dedup_ratio=0.045532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 334, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 7493, 0, 0, 0, 3623, 0, 0, 0, 0, 8032, 0, 0, 0, 60, 4861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5260, 8192, 2184, 5136, 0, 0, 0, 0, 477, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[15654, 16224, 7246, 9842, 16384, 4368, 13452, 10658] dup_rows_per_dst=[0, 362, 0, 0, 0, 0, 4114, 0] combine_rows_per_src=[15654, 16224, 7246, 9842, 16384, 4368, 13452, 10658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75214 payload_rows_remote=61914 dup_rows=23090 dup_rows_remote=19230 dedup_ratio=0.234884 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 346, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 7549, 0, 0, 0, 3370, 0, 0, 0, 0, 8022, 0, 0, 0, 90, 4790, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5415, 8192, 2441, 5052, 0, 0, 0, 0, 501, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40] payload_rows_per_dst=[3980, 494, 14038, 16044, 6, 16384, 13300, 10968] dup_rows_per_dst=[0, 0, 4512, 9760, 0, 4882, 3860, 76] combine_rows_per_src=[3980, 494, 14038, 16044, 6, 16384, 13300, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91496 payload_rows_remote=76074 dup_rows=6808 dup_rows_remote=6792 dedup_ratio=0.069255 router_tokens_per_expert=[2, 1, 2, 0, 3, 0, 356, 0, 20, 255, 5, 0, 2, 0, 0, 0, 12, 7355, 0, 0, 3, 3559, 0, 5, 2, 2, 7968, 2, 1, 5, 66, 4944, 27, 32, 2, 0, 18, 0, 0, 11, 12, 20, 4, 5509, 8161, 2111, 5366, 8, 1, 0, 4, 518, 0, 2695, 0, 0, 0, 0, 0, 30, 0, 2, 0, 51] payload_rows_per_dst=[15422, 16162, 13464, 156, 16382, 12966, 5764, 11180] dup_rows_per_dst=[16, 338, 3582, 0, 120, 2076, 672, 4] combine_rows_per_src=[15422, 16162, 13464, 156, 16382, 12966, 5764, 11180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=361316352 dispatch_meta_bytes=375680 combine_payload_bytes=361316352 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=361692032 ingress_bytes=534331392 link_bytes=534331392 token_bytes=723008384 weight_movement_bytes=173015040 total_nvlink_bytes=896023424 weight_to_token_ratio=0.239299 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.010646011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=229261312 dispatch_meta_bytes=309176 combine_payload_bytes=229261312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=229570488 ingress_bytes=298467328 link_bytes=298467328 token_bytes=458831800 weight_movement_bytes=69206016 total_nvlink_bytes=528037816 weight_to_token_ratio=0.150831 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123108527e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95270 payload_rows_remote=78888 dup_rows=3034 dup_rows_remote=220 dedup_ratio=0.030863 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 78, 0, 0, 1, 83, 0, 7388, 1, 2, 1322, 1, 0, 1, 0, 1, 6521, 0, 0, 0, 0, 0, 272, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3291, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1, 141, 0, 0, 5475, 8183, 1, 0] payload_rows_per_dst=[14932, 16382, 13048, 11284, 16380, 6588, 16370, 286] dup_rows_per_dst=[0, 2814, 0, 214, 0, 2, 4, 0] combine_rows_per_src=[14932, 16382, 13048, 11284, 16380, 6588, 16370, 286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=356401152 dispatch_meta_bytes=360824 combine_payload_bytes=356401152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356761976 ingress_bytes=425607168 link_bytes=425607168 token_bytes=713163128 weight_movement_bytes=69206016 total_nvlink_bytes=782369144 weight_to_token_ratio=0.097041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601525509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97664 payload_rows_remote=81280 dup_rows=640 dup_rows_remote=640 dedup_ratio=0.006510 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 94, 0, 7366, 0, 0, 1430, 0, 0, 0, 0, 0, 6464, 0, 0, 0, 0, 0, 257, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 5267, 8192, 0, 0] payload_rows_per_dst=[14894, 3048, 12928, 16384, 16384, 6884, 16384, 10758] dup_rows_per_dst=[0, 0, 0, 514, 0, 0, 0, 126] combine_rows_per_src=[14894, 3048, 12928, 16384, 16384, 6884, 16384, 10758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=344006656 dispatch_meta_bytes=353848 combine_payload_bytes=344006656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=344360504 ingress_bytes=447815680 link_bytes=447815680 token_bytes=688367160 weight_movement_bytes=103809024 total_nvlink_bytes=792176184 weight_to_token_ratio=0.150805 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.685094352e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=253599744 dispatch_meta_bytes=324576 combine_payload_bytes=253599744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253924320 ingress_bytes=288202752 link_bytes=288202752 token_bytes=507524064 weight_movement_bytes=34603008 total_nvlink_bytes=542127072 weight_to_token_ratio=0.068180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.084483754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=311599104 dispatch_meta_bytes=331464 combine_payload_bytes=311599104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311930568 ingress_bytes=346202112 link_bytes=346202112 token_bytes=623529672 weight_movement_bytes=34603008 total_nvlink_bytes=658132680 weight_to_token_ratio=0.055495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302730676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=323125248 dispatch_meta_bytes=316432 combine_payload_bytes=323125248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323441680 ingress_bytes=392331264 link_bytes=392331264 token_bytes=646566928 weight_movement_bytes=69206016 total_nvlink_bytes=715772944 weight_to_token_ratio=0.107036 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476310961e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=332922880 dispatch_meta_bytes=327680 combine_payload_bytes=332922880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333250560 ingress_bytes=402128896 link_bytes=402128896 token_bytes=666173440 weight_movement_bytes=69206016 total_nvlink_bytes=735379456 weight_to_token_ratio=0.103886 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513178662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93394 payload_rows_remote=80590 dup_rows=4910 dup_rows_remote=694 dedup_ratio=0.049947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 70, 0, 7328, 0, 0, 1300, 0, 0, 0, 0, 0, 6511, 0, 0, 0, 0, 0, 277, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 5339, 8192, 0, 0] payload_rows_per_dst=[14776, 2740, 13022, 16384, 6468, 12804, 16304, 10896] dup_rows_per_dst=[0, 0, 0, 554, 0, 4216, 0, 140] combine_rows_per_src=[14776, 2740, 13022, 16384, 6468, 12804, 16304, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77734 payload_rows_remote=61350 dup_rows=20570 dup_rows_remote=8796 dedup_ratio=0.209249 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 63, 0, 7288, 0, 0, 1415, 0, 0, 0, 0, 0, 6440, 0, 0, 0, 0, 0, 171, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 5716, 8192, 0, 0] payload_rows_per_dst=[152, 15550, 2896, 16384, 9984, 16384, 0, 16384] dup_rows_per_dst=[0, 1982, 0, 342, 0, 6472, 0, 11774] combine_rows_per_src=[152, 15550, 2896, 16384, 9984, 16384, 0, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95210 payload_rows_remote=82392 dup_rows=3094 dup_rows_remote=3094 dedup_ratio=0.031474 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 75, 0, 7235, 0, 0, 1420, 0, 0, 0, 0, 0, 6409, 0, 0, 0, 0, 0, 201, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 5731, 8192, 0, 0] payload_rows_per_dst=[14674, 16384, 12818, 3292, 16384, 6488, 16384, 8786] dup_rows_per_dst=[4, 2990, 0, 32, 0, 0, 0, 68] combine_rows_per_src=[14674, 16384, 12818, 3292, 16384, 6488, 16384, 8786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97050 payload_rows_remote=87242 dup_rows=1254 dup_rows_remote=960 dedup_ratio=0.012756 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 80, 0, 7309, 0, 0, 1171, 0, 0, 0, 0, 0, 6483, 0, 0, 0, 0, 0, 220, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 5259, 8192, 0, 0] payload_rows_per_dst=[14758, 8288, 12966, 9808, 16384, 7672, 16384, 10790] dup_rows_per_dst=[0, 936, 0, 294, 0, 0, 0, 24] combine_rows_per_src=[14758, 8288, 12966, 9808, 16384, 7672, 16384, 10790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330096640 dispatch_meta_bytes=325136 combine_payload_bytes=330096640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330421776 ingress_bytes=364699648 link_bytes=364699648 token_bytes=660518416 weight_movement_bytes=34603008 total_nvlink_bytes=695121424 weight_to_token_ratio=0.052388 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372335415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78796 payload_rows_remote=78796 dup_rows=19508 dup_rows_remote=19508 dedup_ratio=0.198446 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 85, 0, 7353, 0, 0, 1224, 0, 0, 0, 0, 0, 6435, 0, 0, 0, 0, 0, 241, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 5486, 8192, 0, 0] payload_rows_per_dst=[9510, 7264, 12870, 16384, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 630, 0, 482, 0, 7006, 0, 11390] combine_rows_per_src=[9510, 7264, 12870, 16384, 0, 16384, 0, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91692 payload_rows_remote=77090 dup_rows=6612 dup_rows_remote=6610 dedup_ratio=0.067261 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 86, 0, 0, 0, 93, 0, 7214, 10, 31, 1357, 2, 0, 2, 1, 13, 6468, 3, 11, 1, 0, 0, 313, 0, 31, 8157, 0, 0, 0, 1, 0, 0, 8154, 2, 3, 0, 0, 3636, 0, 2, 31, 33, 0, 10, 44, 0, 0, 0, 0, 21, 5, 8, 235, 0, 3, 5009, 8157, 2, 1] payload_rows_per_dst=[14602, 16370, 14344, 5840, 16316, 7362, 16364, 494] dup_rows_per_dst=[2, 2926, 3412, 112, 4, 42, 110, 4] combine_rows_per_src=[14602, 16370, 14344, 5840, 16316, 7362, 16364, 494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=251289600 dispatch_meta_bytes=280584 combine_payload_bytes=251289600 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=251570184 ingress_bytes=355098624 link_bytes=355098624 token_bytes=502859784 weight_movement_bytes=103809024 total_nvlink_bytes=606668808 weight_to_token_ratio=0.206437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336207534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93014 payload_rows_remote=77128 dup_rows=5290 dup_rows_remote=5290 dedup_ratio=0.053813 router_tokens_per_expert=[7567, 0, 2, 0, 3, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 1185, 204, 8189, 0, 2, 0, 0, 0, 115, 0, 1385, 0, 0, 0, 568, 1, 0, 0, 0, 0, 0, 83, 0, 2, 0, 2, 7943, 7014, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6792, 6945, 0, 87, 2, 2, 334] payload_rows_per_dst=[16068, 15886, 16384, 15038, 1304, 13590, 13892, 852] dup_rows_per_dst=[72, 0, 3226, 1990, 0, 2, 0, 0] combine_rows_per_src=[16068, 15886, 16384, 15038, 1304, 13590, 13892, 852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94358 payload_rows_remote=77974 dup_rows=3946 dup_rows_remote=2596 dedup_ratio=0.040141 router_tokens_per_expert=[7639, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 1092, 156, 8192, 0, 0, 0, 0, 0, 113, 0, 1266, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6827, 7013, 0, 101, 0, 0, 326] payload_rows_per_dst=[16138, 15944, 2964, 15060, 16384, 0, 14026, 13842] dup_rows_per_dst=[74, 0, 48, 1808, 1350, 0, 0, 666] combine_rows_per_src=[16138, 15944, 2964, 15060, 16384, 0, 14026, 13842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337477632 dispatch_meta_bytes=341944 combine_payload_bytes=337477632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=337819576 ingress_bytes=406683648 link_bytes=406683648 token_bytes=675297208 weight_movement_bytes=69206016 total_nvlink_bytes=744503224 weight_to_token_ratio=0.102482 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530317827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=357343232 dispatch_meta_bytes=352808 combine_payload_bytes=357343232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=357696040 ingress_bytes=426549248 link_bytes=426549248 token_bytes=715039272 weight_movement_bytes=69206016 total_nvlink_bytes=784245288 weight_to_token_ratio=0.096786 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605070480e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=322748416 dispatch_meta_bytes=393216 combine_payload_bytes=322748416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323141632 ingress_bytes=357351424 link_bytes=357351424 token_bytes=645890048 weight_movement_bytes=34603008 total_nvlink_bytes=680493056 weight_to_token_ratio=0.053574 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344684639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=315760640 dispatch_meta_bytes=334800 combine_payload_bytes=315760640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316095440 ingress_bytes=350363648 link_bytes=350363648 token_bytes=631856080 weight_movement_bytes=34603008 total_nvlink_bytes=666459088 weight_to_token_ratio=0.054764 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318390201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=315916288 dispatch_meta_bytes=329672 combine_payload_bytes=315916288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316245960 ingress_bytes=350519296 link_bytes=350519296 token_bytes=632162248 weight_movement_bytes=34603008 total_nvlink_bytes=666765256 weight_to_token_ratio=0.054738 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318975892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=319381504 dispatch_meta_bytes=322280 combine_payload_bytes=319381504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319703784 ingress_bytes=353984512 link_bytes=353984512 token_bytes=639085288 weight_movement_bytes=34603008 total_nvlink_bytes=673688296 weight_to_token_ratio=0.054145 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332015221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94824 payload_rows_remote=87704 dup_rows=3480 dup_rows_remote=3480 dedup_ratio=0.035400 router_tokens_per_expert=[7571, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 1206, 243, 8192, 0, 0, 0, 0, 0, 87, 0, 1459, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 6971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6789, 6884, 0, 75, 0, 0, 323] payload_rows_per_dst=[15014, 15904, 4110, 8806, 16384, 7120, 13768, 13718] dup_rows_per_dst=[40, 0, 374, 1108, 1302, 0, 0, 656] combine_rows_per_src=[15014, 15904, 4110, 8806, 16384, 7120, 13768, 13718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66540 payload_rows_remote=50436 dup_rows=31764 dup_rows_remote=17526 dedup_ratio=0.323120 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 1197, 222, 8192, 0, 0, 0, 0, 0, 102, 0, 1063, 0, 0, 0, 495, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 8044, 7011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6843, 7201, 0, 63, 0, 0, 312] payload_rows_per_dst=[1078, 0, 14512, 2204, 16384, 16258, 0, 16104] dup_rows_per_dst=[0, 0, 2432, 126, 1116, 13852, 0, 14238] combine_rows_per_src=[1078, 0, 14512, 2204, 16384, 16258, 0, 16104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94040 payload_rows_remote=82660 dup_rows=4264 dup_rows_remote=2302 dedup_ratio=0.043376 router_tokens_per_expert=[7612, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 1129, 198, 8192, 0, 0, 0, 0, 0, 127, 0, 1026, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 7952, 6924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6905, 7272, 0, 76, 0, 0, 392] payload_rows_per_dst=[16172, 15904, 11380, 14828, 6888, 6774, 14544, 7550] dup_rows_per_dst=[96, 0, 1962, 1326, 458, 0, 0, 422] combine_rows_per_src=[16172, 15904, 11380, 14828, 6888, 6774, 14544, 7550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94466 payload_rows_remote=79536 dup_rows=3838 dup_rows_remote=2114 dedup_ratio=0.039042 router_tokens_per_expert=[7629, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 1101, 192, 8192, 0, 0, 0, 0, 0, 110, 0, 1225, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 6992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6886, 7074, 0, 80, 0, 0, 347] payload_rows_per_dst=[16156, 15932, 2984, 14930, 16384, 0, 14148, 13932] dup_rows_per_dst=[98, 0, 86, 1724, 1236, 0, 0, 694] combine_rows_per_src=[16156, 15932, 2984, 14930, 16384, 0, 14148, 13932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=359235584 dispatch_meta_bytes=364736 combine_payload_bytes=359235584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=359600320 ingress_bytes=463044608 link_bytes=463044608 token_bytes=718835904 weight_movement_bytes=103809024 total_nvlink_bytes=822644928 weight_to_token_ratio=0.144413 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.742399582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82300 payload_rows_remote=68272 dup_rows=16004 dup_rows_remote=16004 dedup_ratio=0.162801 router_tokens_per_expert=[7560, 0, 0, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 1122, 234, 8192, 0, 0, 0, 0, 0, 163, 0, 1284, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7890, 6930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6820, 7056, 0, 88, 0, 0, 354] payload_rows_per_dst=[1114, 2944, 15594, 2664, 16384, 15754, 14028, 13818] dup_rows_per_dst=[0, 0, 2676, 230, 1366, 10942, 0, 790] combine_rows_per_src=[1114, 2944, 15594, 2664, 16384, 15754, 14028, 13818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93292 payload_rows_remote=77238 dup_rows=5012 dup_rows_remote=4882 dedup_ratio=0.050985 router_tokens_per_expert=[7519, 0, 23, 0, 31, 0, 0, 519, 0, 0, 1, 1, 0, 0, 0, 0, 306, 3, 1126, 204, 8133, 0, 14, 0, 0, 0, 131, 0, 1180, 12, 1, 0, 555, 12, 25, 0, 2, 0, 0, 88, 0, 12, 0, 23, 7893, 6907, 0, 0, 0, 0, 3, 0, 0, 26, 0, 0, 11, 6971, 6920, 0, 92, 30, 7, 371] payload_rows_per_dst=[16054, 15788, 16334, 14860, 1358, 14006, 13894, 998] dup_rows_per_dst=[130, 2, 3238, 1602, 6, 6, 4, 24] combine_rows_per_src=[16054, 15788, 16334, 14860, 1358, 14006, 13894, 998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93920 payload_rows_remote=86304 dup_rows=4384 dup_rows_remote=4384 dedup_ratio=0.044596 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 0, 0, 0, 93, 0, 8054, 4831, 0, 0, 7286, 0, 0, 0, 0, 883, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 3808, 0, 7582, 0, 0, 0, 0, 93, 0, 0, 0, 0, 94, 0, 0, 0, 245, 1264, 0, 5614, 0, 0, 94, 0, 0] payload_rows_per_dst=[16108, 14994, 9848, 1766, 7616, 15164, 16196, 12228] dup_rows_per_dst=[0, 1804, 0, 0, 0, 186, 678, 1716] combine_rows_per_src=[16108, 14994, 9848, 1766, 7616, 15164, 16196, 12228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=206585856 dispatch_meta_bytes=271848 combine_payload_bytes=206585856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=206857704 ingress_bytes=310394880 link_bytes=310394880 token_bytes=413443560 weight_movement_bytes=103809024 total_nvlink_bytes=517252584 weight_to_token_ratio=0.251084 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167990945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87210 payload_rows_remote=72158 dup_rows=11094 dup_rows_remote=9224 dedup_ratio=0.112854 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1175, 1, 0, 0, 95, 1, 8031, 5003, 1, 0, 7282, 2, 0, 0, 0, 911, 0, 0, 0, 2, 8104, 3, 1, 0, 0, 3695, 0, 7566, 0, 1, 0, 0, 82, 0, 0, 0, 0, 84, 0, 0, 0, 243, 1250, 2, 5526, 0, 3, 83, 0, 0] payload_rows_per_dst=[16066, 15052, 10190, 15304, 16218, 11054, 654, 2672] dup_rows_per_dst=[0, 1870, 10, 1654, 7392, 164, 0, 4] combine_rows_per_src=[16066, 15052, 10190, 15304, 16218, 11054, 654, 2672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=338575360 dispatch_meta_bytes=339848 combine_payload_bytes=338575360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=338915208 ingress_bytes=407781376 link_bytes=407781376 token_bytes=677490568 weight_movement_bytes=69206016 total_nvlink_bytes=746696584 weight_to_token_ratio=0.102151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.534448489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=325779456 dispatch_meta_bytes=326600 combine_payload_bytes=325779456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326106056 ingress_bytes=360382464 link_bytes=360382464 token_bytes=651885512 weight_movement_bytes=34603008 total_nvlink_bytes=686488520 weight_to_token_ratio=0.053081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356090199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=353501184 dispatch_meta_bytes=362752 combine_payload_bytes=353501184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=353863936 ingress_bytes=422707200 link_bytes=422707200 token_bytes=707365120 weight_movement_bytes=69206016 total_nvlink_bytes=776571136 weight_to_token_ratio=0.097836 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.590613163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=279642112 dispatch_meta_bytes=337104 combine_payload_bytes=279642112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279979216 ingress_bytes=314245120 link_bytes=314245120 token_bytes=559621328 weight_movement_bytes=34603008 total_nvlink_bytes=594224336 weight_to_token_ratio=0.061833 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182479088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=316366848 dispatch_meta_bytes=328480 combine_payload_bytes=316366848 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316695328 ingress_bytes=385572864 link_bytes=385572864 token_bytes=633062176 weight_movement_bytes=69206016 total_nvlink_bytes=702268192 weight_to_token_ratio=0.109319 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450879646e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=295559168 dispatch_meta_bytes=325528 combine_payload_bytes=295559168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295884696 ingress_bytes=330162176 link_bytes=330162176 token_bytes=591443864 weight_movement_bytes=34603008 total_nvlink_bytes=626046872 weight_to_token_ratio=0.058506 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242373688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94052 payload_rows_remote=78894 dup_rows=4252 dup_rows_remote=4132 dedup_ratio=0.043254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 64, 0, 8072, 4964, 0, 0, 7270, 0, 0, 0, 0, 964, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 3632, 0, 7579, 0, 0, 0, 0, 60, 0, 0, 0, 0, 70, 0, 0, 0, 200, 1324, 0, 5270, 0, 0, 70, 0, 0] payload_rows_per_dst=[16144, 9532, 10052, 7844, 7264, 15158, 16244, 11814] dup_rows_per_dst=[0, 1316, 4, 758, 0, 120, 540, 1514] combine_rows_per_src=[16144, 9532, 10052, 7844, 7264, 15158, 16244, 11814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86712 payload_rows_remote=71438 dup_rows=11592 dup_rows_remote=2574 dedup_ratio=0.117920 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 0, 0, 49, 0, 8057, 5403, 0, 0, 7308, 0, 0, 0, 0, 833, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 3145, 0, 7682, 0, 0, 0, 0, 63, 0, 0, 0, 0, 63, 0, 0, 0, 144, 1315, 0, 6032, 0, 0, 63, 0, 0] payload_rows_per_dst=[0, 1732, 16114, 14686, 7284, 15364, 16258, 15274] dup_rows_per_dst=[0, 0, 98, 1596, 340, 126, 414, 9018] combine_rows_per_src=[0, 1732, 16114, 14686, 7284, 15364, 16258, 15274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91486 payload_rows_remote=80972 dup_rows=6818 dup_rows_remote=6818 dedup_ratio=0.069356 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 91, 0, 8074, 5166, 0, 0, 7214, 0, 0, 0, 0, 780, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 3534, 0, 7602, 0, 0, 0, 0, 68, 0, 0, 0, 0, 76, 0, 0, 0, 230, 1342, 0, 5793, 0, 0, 76, 0, 0] payload_rows_per_dst=[16148, 14798, 10514, 15336, 9574, 2444, 12164, 10508] dup_rows_per_dst=[0, 1610, 0, 1428, 1692, 24, 482, 1582] combine_rows_per_src=[16148, 14798, 10514, 15336, 9574, 2444, 12164, 10508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=323149824 dispatch_meta_bytes=332104 combine_payload_bytes=323149824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323481928 ingress_bytes=392355840 link_bytes=392355840 token_bytes=646631752 weight_movement_bytes=69206016 total_nvlink_bytes=715837768 weight_to_token_ratio=0.107025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476403439e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93426 payload_rows_remote=88932 dup_rows=4878 dup_rows_remote=4410 dedup_ratio=0.049622 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 0, 95, 0, 8046, 4704, 0, 0, 7247, 0, 0, 0, 0, 943, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 3638, 0, 7616, 0, 0, 0, 0, 80, 0, 0, 0, 0, 80, 0, 0, 0, 232, 1311, 0, 6047, 0, 0, 80, 0, 0] payload_rows_per_dst=[16092, 14718, 9594, 4494, 7276, 12180, 16224, 12848] dup_rows_per_dst=[0, 1618, 4, 468, 0, 136, 624, 2028] combine_rows_per_src=[16092, 14718, 9594, 4494, 7276, 12180, 16224, 12848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94910 payload_rows_remote=78582 dup_rows=3394 dup_rows_remote=3058 dedup_ratio=0.034526 router_tokens_per_expert=[0, 227, 0, 8055, 659, 94, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7913, 0, 0, 225, 0, 0, 0, 0, 0, 0, 3845, 0, 8192, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 168, 0, 0, 0, 161, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16366, 0, 16384, 7690, 16328, 6622, 15826, 15694] dup_rows_per_dst=[1704, 0, 450, 0, 336, 184, 0, 720] combine_rows_per_src=[16366, 0, 16384, 7690, 16328, 6622, 15826, 15694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87110 payload_rows_remote=70896 dup_rows=11194 dup_rows_remote=10502 dedup_ratio=0.113871 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 0, 89, 0, 8068, 4955, 0, 0, 7244, 0, 0, 0, 0, 811, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 3716, 0, 7512, 0, 0, 0, 0, 85, 0, 0, 0, 0, 85, 0, 0, 0, 261, 1238, 0, 5796, 0, 0, 85, 0, 0] payload_rows_per_dst=[1808, 2200, 15474, 14668, 9200, 15024, 16214, 12522] dup_rows_per_dst=[0, 0, 6604, 1442, 570, 170, 692, 1716] combine_rows_per_src=[1808, 2200, 15474, 14668, 9200, 15024, 16214, 12522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292610048 dispatch_meta_bytes=296048 combine_payload_bytes=292610048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292906096 ingress_bytes=361816064 link_bytes=361816064 token_bytes=585516144 weight_movement_bytes=69206016 total_nvlink_bytes=654722160 weight_to_token_ratio=0.118197 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361484720e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86746 payload_rows_remote=70810 dup_rows=11558 dup_rows_remote=11556 dedup_ratio=0.117574 router_tokens_per_expert=[0, 0, 25, 0, 5, 2, 0, 0, 1, 0, 0, 0, 3, 0, 1327, 17, 1, 0, 107, 0, 7937, 4776, 14, 1, 7182, 22, 0, 0, 0, 887, 4, 0, 0, 10, 8057, 29, 0, 0, 0, 3843, 9, 7517, 3, 33, 1, 0, 88, 0, 10, 11, 3, 88, 7, 0, 1, 340, 1299, 6, 5361, 0, 37, 88, 0, 0] payload_rows_per_dst=[15936, 15038, 9796, 15250, 16192, 10788, 904, 2842] dup_rows_per_dst=[2, 2022, 2, 1610, 7686, 202, 16, 18] combine_rows_per_src=[15936, 15038, 9796, 15250, 16192, 10788, 904, 2842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67336 payload_rows_remote=50956 dup_rows=30968 dup_rows_remote=2342 dedup_ratio=0.315023 router_tokens_per_expert=[0, 227, 1, 8072, 588, 82, 0, 0, 0, 0, 7853, 0, 1, 0, 2, 7912, 3, 0, 283, 0, 1, 0, 0, 0, 0, 3871, 0, 8189, 1, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 221, 0, 0, 0, 176, 0, 2, 0, 3097, 1, 0, 2, 0, 0, 1, 3, 0, 10, 0, 399, 0, 0, 0, 0, 1, 2] payload_rows_per_dst=[16348, 16380, 16384, 7742, 442, 6374, 2862, 804] dup_rows_per_dst=[1592, 28626, 568, 2, 0, 178, 2, 0] combine_rows_per_src=[16348, 16380, 16384, 7742, 442, 6374, 2862, 804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=331661312 dispatch_meta_bytes=351160 combine_payload_bytes=331661312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332012472 ingress_bytes=435470336 link_bytes=435470336 token_bytes=663673784 weight_movement_bytes=103809024 total_nvlink_bytes=767482808 weight_to_token_ratio=0.156416 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.638639816e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=364265472 dispatch_meta_bytes=373368 combine_payload_bytes=364265472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=364638840 ingress_bytes=433471488 link_bytes=433471488 token_bytes=728904312 weight_movement_bytes=69206016 total_nvlink_bytes=798110328 weight_to_token_ratio=0.094945 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.631118312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321871872 dispatch_meta_bytes=326560 combine_payload_bytes=321871872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322198432 ingress_bytes=391077888 link_bytes=391077888 token_bytes=644070304 weight_movement_bytes=69206016 total_nvlink_bytes=713276320 weight_to_token_ratio=0.107451 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471594608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290390016 dispatch_meta_bytes=325592 combine_payload_bytes=290390016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290715608 ingress_bytes=324993024 link_bytes=324993024 token_bytes=581105624 weight_movement_bytes=34603008 total_nvlink_bytes=615708632 weight_to_token_ratio=0.059547 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222922585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290037760 dispatch_meta_bytes=329464 combine_payload_bytes=290037760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290367224 ingress_bytes=324640768 link_bytes=324640768 token_bytes=580404984 weight_movement_bytes=34603008 total_nvlink_bytes=615007992 weight_to_token_ratio=0.059619 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221597074e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=208715776 dispatch_meta_bytes=213192 combine_payload_bytes=208715776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=208928968 ingress_bytes=312524800 link_bytes=312524800 token_bytes=417644744 weight_movement_bytes=103809024 total_nvlink_bytes=521453768 weight_to_token_ratio=0.248558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176005662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88598 payload_rows_remote=75858 dup_rows=9706 dup_rows_remote=5268 dedup_ratio=0.098735 router_tokens_per_expert=[0, 196, 0, 8043, 648, 70, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7897, 0, 0, 264, 0, 0, 2, 0, 0, 0, 4066, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 229, 0, 0, 0, 170, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 364, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5078, 0, 12720, 10192, 16380, 12740, 15794, 15694] dup_rows_per_dst=[468, 0, 412, 2002, 1652, 4438, 6, 728] combine_rows_per_src=[5078, 0, 12720, 10192, 16380, 12740, 15794, 15694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82674 payload_rows_remote=66944 dup_rows=15630 dup_rows_remote=15020 dedup_ratio=0.158997 router_tokens_per_expert=[0, 149, 0, 8055, 572, 37, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8081, 0, 0, 184, 0, 0, 0, 0, 0, 0, 3955, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 54, 0, 0, 0, 101, 0, 0, 0, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 305, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1442, 0, 368, 16384, 16374, 16214, 16162, 15730] dup_rows_per_dst=[74, 0, 0, 7910, 108, 6926, 2, 610] combine_rows_per_src=[1442, 0, 368, 16384, 16374, 16214, 16162, 15730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=310714368 dispatch_meta_bytes=324504 combine_payload_bytes=310714368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311038872 ingress_bytes=345317376 link_bytes=345317376 token_bytes=621753240 weight_movement_bytes=34603008 total_nvlink_bytes=656356248 weight_to_token_ratio=0.055654 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299401486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93384 payload_rows_remote=77000 dup_rows=4920 dup_rows_remote=4488 dedup_ratio=0.050049 router_tokens_per_expert=[0, 177, 0, 8066, 574, 58, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 7939, 0, 0, 216, 0, 0, 0, 0, 0, 0, 3633, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 64, 0, 0, 0, 167, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16324, 3434, 16384, 7266, 12982, 7418, 15878, 13698] dup_rows_per_dst=[1426, 2196, 432, 0, 86, 162, 0, 618] combine_rows_per_src=[16324, 3434, 16384, 7266, 12982, 7418, 15878, 13698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95264 payload_rows_remote=86896 dup_rows=3040 dup_rows_remote=3040 dedup_ratio=0.030924 router_tokens_per_expert=[0, 180, 0, 8081, 546, 58, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 7931, 0, 0, 198, 0, 0, 0, 0, 0, 0, 4184, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 172, 0, 0, 0, 166, 0, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 0, 16384, 8368, 16378, 6194, 15862, 15734] dup_rows_per_dst=[1386, 0, 396, 0, 344, 184, 0, 730] combine_rows_per_src=[16344, 0, 16384, 8368, 16378, 6194, 15862, 15734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97222 payload_rows_remote=94466 dup_rows=1082 dup_rows_remote=1082 dedup_ratio=0.011007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 405, 0, 1104, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 5693, 0, 0, 0, 0, 1378, 0, 8192, 0, 0, 0, 8192, 0, 0, 105, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 325, 0, 6319] payload_rows_per_dst=[16384, 3262, 12394, 16384, 2756, 16384, 16370, 13288] dup_rows_per_dst=[0, 670, 202, 0, 0, 210, 0, 0] combine_rows_per_src=[16384, 3262, 12394, 16384, 2756, 16384, 16370, 13288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=274202624 dispatch_meta_bytes=327856 combine_payload_bytes=274202624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274530480 ingress_bytes=308805632 link_bytes=308805632 token_bytes=548733104 weight_movement_bytes=34603008 total_nvlink_bytes=583336112 weight_to_token_ratio=0.063060 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162010732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82580 payload_rows_remote=66778 dup_rows=15724 dup_rows_remote=15722 dedup_ratio=0.159953 router_tokens_per_expert=[0, 198, 0, 8042, 663, 82, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7901, 0, 0, 268, 0, 0, 0, 0, 0, 0, 3839, 0, 8192, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 72, 0, 0, 0, 212, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 358, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1722, 0, 536, 16384, 16278, 16164, 15802, 15694] dup_rows_per_dst=[164, 0, 0, 7678, 144, 7020, 2, 716] combine_rows_per_src=[1722, 0, 536, 16384, 16278, 16164, 15802, 15694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64556 payload_rows_remote=48334 dup_rows=33748 dup_rows_remote=32104 dedup_ratio=0.343302 router_tokens_per_expert=[0, 209, 1, 7987, 647, 86, 3, 0, 0, 0, 7822, 0, 18, 1, 22, 7895, 30, 5, 233, 0, 20, 0, 0, 0, 0, 3330, 0, 8155, 1, 0, 12, 0, 0, 0, 8076, 0, 0, 0, 236, 0, 6, 0, 193, 0, 9, 1, 3511, 23, 5, 2, 1, 0, 4, 35, 0, 6, 0, 530, 0, 11, 0, 2, 0, 24] payload_rows_per_dst=[16222, 16370, 16376, 6684, 472, 7224, 88, 1120] dup_rows_per_dst=[1644, 31298, 510, 2, 0, 262, 18, 14] combine_rows_per_src=[16222, 16370, 16376, 6684, 472, 7224, 88, 1120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93882 payload_rows_remote=77498 dup_rows=4422 dup_rows_remote=298 dedup_ratio=0.044983 router_tokens_per_expert=[10, 2, 0, 0, 0, 0, 1, 8181, 427, 3, 1188, 0, 0, 0, 0, 450, 0, 0, 4, 0, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 5603, 1, 0, 0, 0, 1291, 0, 8189, 0, 2, 0, 8186, 1, 3, 82, 0, 0, 0, 0, 4, 2, 8174, 0, 2, 0, 1, 0, 0, 0, 335, 0, 6407] payload_rows_per_dst=[16368, 16384, 12294, 16378, 2584, 14268, 2120, 13486] dup_rows_per_dst=[20, 4124, 126, 0, 0, 146, 6, 0] combine_rows_per_src=[16368, 16384, 12294, 16378, 2584, 14268, 2120, 13486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315392000 dispatch_meta_bytes=325952 combine_payload_bytes=315392000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315717952 ingress_bytes=349995008 link_bytes=349995008 token_bytes=631109952 weight_movement_bytes=34603008 total_nvlink_bytes=665712960 weight_to_token_ratio=0.054829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317003038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=355926016 dispatch_meta_bytes=359744 combine_payload_bytes=355926016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356285760 ingress_bytes=425132032 link_bytes=425132032 token_bytes=712211776 weight_movement_bytes=69206016 total_nvlink_bytes=781417792 weight_to_token_ratio=0.097171 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599737610e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=386932736 dispatch_meta_bytes=382192 combine_payload_bytes=386932736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=387314928 ingress_bytes=490741760 link_bytes=490741760 token_bytes=774247664 weight_movement_bytes=103809024 total_nvlink_bytes=878056688 weight_to_token_ratio=0.134077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846621735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273522688 dispatch_meta_bytes=330000 combine_payload_bytes=273522688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273852688 ingress_bytes=308125696 link_bytes=308125696 token_bytes=547375376 weight_movement_bytes=34603008 total_nvlink_bytes=581978384 weight_to_token_ratio=0.063216 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159452188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=197976064 dispatch_meta_bytes=321752 combine_payload_bytes=197976064 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=198297816 ingress_bytes=301785088 link_bytes=301785088 token_bytes=396273880 weight_movement_bytes=103809024 total_nvlink_bytes=500082904 weight_to_token_ratio=0.261963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=317431808 dispatch_meta_bytes=311184 combine_payload_bytes=317431808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317742992 ingress_bytes=352034816 link_bytes=352034816 token_bytes=635174800 weight_movement_bytes=34603008 total_nvlink_bytes=669777808 weight_to_token_ratio=0.054478 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324678671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95346 payload_rows_remote=78962 dup_rows=2958 dup_rows_remote=2808 dedup_ratio=0.030090 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 8180, 509, 0, 949, 0, 0, 0, 0, 548, 0, 0, 2, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 0, 0, 0, 1350, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 11, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 323, 0, 6226] payload_rows_per_dst=[5092, 3214, 12640, 16384, 12154, 16384, 16380, 13098] dup_rows_per_dst=[2, 798, 172, 0, 1814, 150, 22, 0] combine_rows_per_src=[5092, 3214, 12640, 16384, 12154, 16384, 16380, 13098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75298 payload_rows_remote=61400 dup_rows=23006 dup_rows_remote=21148 dedup_ratio=0.234029 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 476, 0, 1128, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 1328, 0, 8192, 0, 0, 0, 8192, 0, 0, 96, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 312, 0, 6439] payload_rows_per_dst=[0, 3360, 11142, 0, 16384, 16384, 14130, 13898] dup_rows_per_dst=[0, 832, 110, 0, 20014, 192, 0, 1858] combine_rows_per_src=[0, 3360, 11142, 0, 16384, 16384, 14130, 13898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=323428352 dispatch_meta_bytes=327080 combine_payload_bytes=323428352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323755432 ingress_bytes=392634368 link_bytes=392634368 token_bytes=647183784 weight_movement_bytes=69206016 total_nvlink_bytes=716389800 weight_to_token_ratio=0.106934 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.477451517e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93810 payload_rows_remote=82126 dup_rows=4494 dup_rows_remote=4374 dedup_ratio=0.045715 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 468, 0, 1230, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 5312, 0, 0, 0, 0, 1350, 0, 8192, 0, 0, 0, 8192, 0, 0, 96, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 392, 0, 6461] payload_rows_per_dst=[16384, 16384, 11684, 16384, 2700, 192, 16376, 13706] dup_rows_per_dst=[0, 4374, 120, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 11684, 16384, 2700, 192, 16376, 13706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94532 payload_rows_remote=78148 dup_rows=3772 dup_rows_remote=3772 dedup_ratio=0.038371 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8186, 452, 0, 976, 0, 0, 0, 0, 493, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 5698, 0, 0, 0, 0, 1157, 0, 8192, 0, 0, 0, 8192, 0, 0, 113, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 346, 0, 6574] payload_rows_per_dst=[16372, 15656, 12432, 16384, 2314, 1150, 16384, 13840] dup_rows_per_dst=[0, 3640, 114, 0, 0, 6, 12, 0] combine_rows_per_src=[16372, 15656, 12432, 16384, 2314, 1150, 16384, 13840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93158 payload_rows_remote=90950 dup_rows=5146 dup_rows_remote=5122 dedup_ratio=0.052348 router_tokens_per_expert=[0, 0, 1433, 5179, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 7114, 0, 0, 0, 0, 0, 0, 0, 0, 666, 0, 1484, 0, 0, 0] payload_rows_per_dst=[12668, 16384, 16384, 16384, 2208, 0, 14228, 14902] dup_rows_per_dst=[556, 0, 688, 172, 24, 0, 0, 3706] combine_rows_per_src=[12668, 16384, 16384, 16384, 2208, 0, 14228, 14902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78898 payload_rows_remote=62540 dup_rows=19406 dup_rows_remote=19406 dedup_ratio=0.197408 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 463, 0, 1129, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 5764, 0, 0, 0, 0, 1100, 0, 8192, 0, 0, 0, 8192, 0, 0, 62, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 352, 0, 6493] payload_rows_per_dst=[0, 3464, 12566, 52, 16384, 16384, 16358, 13690] dup_rows_per_dst=[0, 668, 82, 0, 18532, 124, 0, 0] combine_rows_per_src=[0, 3464, 12566, 52, 16384, 16384, 16358, 13690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81470 payload_rows_remote=65138 dup_rows=16834 dup_rows_remote=5484 dedup_ratio=0.171244 router_tokens_per_expert=[56, 22, 0, 0, 0, 0, 4, 8093, 441, 36, 1610, 0, 0, 0, 0, 516, 0, 11, 10, 0, 0, 0, 1, 547, 0, 0, 0, 0, 0, 1, 3, 16, 5544, 18, 0, 0, 0, 1316, 0, 8149, 0, 29, 0, 8134, 24, 33, 64, 0, 0, 0, 0, 34, 20, 8015, 0, 19, 0, 9, 4, 1, 1, 348, 0, 6023] payload_rows_per_dst=[16332, 16376, 12124, 16326, 2662, 16104, 112, 1434] dup_rows_per_dst=[11350, 5098, 102, 12, 6, 226, 34, 6] combine_rows_per_src=[16332, 16376, 12124, 16326, 2662, 16104, 112, 1434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=251494400 dispatch_meta_bytes=330192 combine_payload_bytes=251494400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251824592 ingress_bytes=320700416 link_bytes=320700416 token_bytes=503318992 weight_movement_bytes=69206016 total_nvlink_bytes=572525008 weight_to_token_ratio=0.137499 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206769847e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=336388096 dispatch_meta_bytes=346000 combine_payload_bytes=336388096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336734096 ingress_bytes=370991104 link_bytes=370991104 token_bytes=673122192 weight_movement_bytes=34603008 total_nvlink_bytes=707725200 weight_to_token_ratio=0.051407 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396009657e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96610 payload_rows_remote=80226 dup_rows=1694 dup_rows_remote=1694 dedup_ratio=0.017232 router_tokens_per_expert=[0, 0, 1539, 5055, 0, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 0, 2, 390, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 67, 0, 0, 1, 0, 0, 3, 8182, 8187, 0, 3, 7128, 0, 2, 1, 0, 0, 0, 0, 3, 634, 2, 1482, 0, 0, 0] payload_rows_per_dst=[12592, 16384, 16376, 16374, 16380, 11744, 2526, 4234] dup_rows_per_dst=[596, 0, 772, 178, 134, 2, 4, 8] combine_rows_per_src=[12592, 16384, 16376, 16374, 16380, 11744, 2526, 4234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=320094208 dispatch_meta_bytes=327680 combine_payload_bytes=320094208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320421888 ingress_bytes=354697216 link_bytes=354697216 token_bytes=640516096 weight_movement_bytes=34603008 total_nvlink_bytes=675119104 weight_to_token_ratio=0.054024 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334697068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=372531200 dispatch_meta_bytes=384288 combine_payload_bytes=372531200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=372915488 ingress_bytes=441737216 link_bytes=441737216 token_bytes=745446688 weight_movement_bytes=69206016 total_nvlink_bytes=814652704 weight_to_token_ratio=0.092838 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.662221581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=256163840 dispatch_meta_bytes=327784 combine_payload_bytes=256163840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=256491624 ingress_bytes=325369856 link_bytes=325369856 token_bytes=512655464 weight_movement_bytes=69206016 total_nvlink_bytes=581861480 weight_to_token_ratio=0.134995 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224340574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=282488 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267087736 ingress_bytes=336011264 link_bytes=336011264 token_bytes=533892984 weight_movement_bytes=69206016 total_nvlink_bytes=603099000 weight_to_token_ratio=0.129625 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264383335e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91816 payload_rows_remote=85884 dup_rows=6488 dup_rows_remote=5274 dedup_ratio=0.065999 router_tokens_per_expert=[0, 0, 1329, 5385, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 1467, 0, 0, 0] payload_rows_per_dst=[12842, 16384, 10692, 15182, 112, 5932, 14288, 16384] dup_rows_per_dst=[586, 0, 442, 108, 0, 1214, 0, 4138] combine_rows_per_src=[12842, 16384, 10692, 15182, 112, 5932, 14288, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328605696 dispatch_meta_bytes=327680 combine_payload_bytes=328605696 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=328933376 ingress_bytes=397811712 link_bytes=397811712 token_bytes=657539072 weight_movement_bytes=69206016 total_nvlink_bytes=726745088 weight_to_token_ratio=0.105250 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496933446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=351780864 dispatch_meta_bytes=364632 combine_payload_bytes=351780864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=352145496 ingress_bytes=455589888 link_bytes=455589888 token_bytes=703926360 weight_movement_bytes=103809024 total_nvlink_bytes=807735384 weight_to_token_ratio=0.147471 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714348071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67932 payload_rows_remote=51548 dup_rows=30372 dup_rows_remote=26000 dedup_ratio=0.308960 router_tokens_per_expert=[0, 0, 1480, 5109, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 7174, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 1486, 0, 0, 0] payload_rows_per_dst=[2960, 0, 688, 86, 16384, 16384, 15046, 16384] dup_rows_per_dst=[0, 0, 0, 0, 100, 16380, 9520, 4372] combine_rows_per_src=[2960, 0, 688, 86, 16384, 16384, 15046, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96686 payload_rows_remote=80302 dup_rows=1618 dup_rows_remote=784 dedup_ratio=0.016459 router_tokens_per_expert=[0, 0, 1387, 5097, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 7075, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 1479, 0, 0, 0] payload_rows_per_dst=[12406, 16384, 16384, 16384, 16384, 0, 14150, 4594] dup_rows_per_dst=[562, 0, 834, 178, 44, 0, 0, 0] combine_rows_per_src=[12406, 16384, 16384, 16384, 16384, 0, 14150, 4594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96686 payload_rows_remote=80302 dup_rows=1618 dup_rows_remote=1438 dedup_ratio=0.016459 router_tokens_per_expert=[0, 0, 1504, 5033, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 7117, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 1479, 0, 0, 0] payload_rows_per_dst=[12504, 16384, 16384, 16384, 16384, 0, 14234, 4412] dup_rows_per_dst=[570, 0, 742, 180, 126, 0, 0, 0] combine_rows_per_src=[12504, 16384, 16384, 16384, 16384, 0, 14234, 4412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97772 payload_rows_remote=81412 dup_rows=532 dup_rows_remote=532 dedup_ratio=0.005412 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7847, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 345, 0, 0, 7056, 0, 0] payload_rows_per_dst=[16384, 620, 13996, 3532, 16360, 16384, 15694, 14802] dup_rows_per_dst=[0, 0, 0, 154, 0, 194, 184, 0] combine_rows_per_src=[16384, 620, 13996, 3532, 16360, 16384, 15694, 14802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76814 payload_rows_remote=62598 dup_rows=21490 dup_rows_remote=21484 dedup_ratio=0.218608 router_tokens_per_expert=[0, 0, 1495, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 7108, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 1526, 0, 0, 0] payload_rows_per_dst=[12526, 2868, 730, 170, 13536, 16384, 14216, 16384] dup_rows_per_dst=[586, 0, 0, 0, 60, 16384, 6, 4454] combine_rows_per_src=[12526, 2868, 730, 170, 13536, 16384, 14216, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93908 payload_rows_remote=80664 dup_rows=4396 dup_rows_remote=1290 dedup_ratio=0.044718 router_tokens_per_expert=[11, 0, 1811, 4926, 0, 0, 0, 2, 34, 0, 0, 0, 8149, 0, 0, 0, 17, 446, 0, 0, 0, 0, 0, 0, 112, 0, 11, 1, 0, 0, 0, 0, 8126, 1, 0, 0, 34, 0, 53, 0, 0, 7, 0, 4, 36, 8131, 8138, 0, 38, 6893, 0, 0, 4, 0, 1, 0, 0, 36, 641, 30, 1455, 0, 0, 4] payload_rows_per_dst=[13244, 13516, 16298, 16312, 16324, 13864, 86, 4264] dup_rows_per_dst=[3106, 0, 890, 212, 104, 16, 0, 68] combine_rows_per_src=[13244, 13516, 16298, 16312, 16324, 13864, 86, 4264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=211140608 dispatch_meta_bytes=310192 combine_payload_bytes=211140608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=211450800 ingress_bytes=245743616 link_bytes=245743616 token_bytes=422591408 weight_movement_bytes=34603008 total_nvlink_bytes=457194416 weight_to_token_ratio=0.081883 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.247134430e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328916992 dispatch_meta_bytes=324344 combine_payload_bytes=328916992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329241336 ingress_bytes=363520000 link_bytes=363520000 token_bytes=658158328 weight_movement_bytes=34603008 total_nvlink_bytes=692761336 weight_to_token_ratio=0.052576 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367896494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77120 payload_rows_remote=61584 dup_rows=21184 dup_rows_remote=7558 dedup_ratio=0.215495 router_tokens_per_expert=[0, 0, 1, 3, 3, 2, 0, 0, 2, 8189, 7854, 0, 0, 0, 320, 0, 0, 0, 0, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 8185, 810, 1, 0, 2, 3, 0, 0, 1, 0, 0, 8175, 0, 0, 0, 0, 100, 0, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 334, 0, 0, 6886, 0, 0] payload_rows_per_dst=[16384, 15536, 16382, 16380, 880, 200, 2818, 8540] dup_rows_per_dst=[12, 13626, 5900, 1612, 4, 2, 28, 0] combine_rows_per_src=[16384, 15536, 16382, 16380, 880, 200, 2818, 8540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328916992 dispatch_meta_bytes=326960 combine_payload_bytes=328916992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329243952 ingress_bytes=363520000 link_bytes=363520000 token_bytes=658160944 weight_movement_bytes=34603008 total_nvlink_bytes=692763952 weight_to_token_ratio=0.052575 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367896494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96246 payload_rows_remote=79862 dup_rows=2058 dup_rows_remote=1900 dedup_ratio=0.020935 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7848, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 344, 0, 0, 6831, 0, 0] payload_rows_per_dst=[16254, 810, 0, 16384, 16368, 16384, 15696, 14350] dup_rows_per_dst=[0, 4, 0, 1776, 0, 158, 120, 0] combine_rows_per_src=[16254, 810, 0, 16384, 16368, 16384, 15696, 14350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=333463552 dispatch_meta_bytes=327776 combine_payload_bytes=333463552 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=333791328 ingress_bytes=402669568 link_bytes=402669568 token_bytes=667254880 weight_movement_bytes=69206016 total_nvlink_bytes=736460896 weight_to_token_ratio=0.103718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515213167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256401408 dispatch_meta_bytes=336328 combine_payload_bytes=256401408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256737736 ingress_bytes=325607424 link_bytes=325607424 token_bytes=513139144 weight_movement_bytes=69206016 total_nvlink_bytes=582345160 weight_to_token_ratio=0.134868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225234523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330399744 dispatch_meta_bytes=327816 combine_payload_bytes=330399744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330727560 ingress_bytes=399605760 link_bytes=399605760 token_bytes=661127304 weight_movement_bytes=69206016 total_nvlink_bytes=730333320 weight_to_token_ratio=0.104679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.503684304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=252248064 dispatch_meta_bytes=276568 combine_payload_bytes=252248064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=252524632 ingress_bytes=356057088 link_bytes=356057088 token_bytes=504772696 weight_movement_bytes=103809024 total_nvlink_bytes=608581720 weight_to_token_ratio=0.205655 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339814157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=327114752 dispatch_meta_bytes=327048 combine_payload_bytes=327114752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327441800 ingress_bytes=361717760 link_bytes=361717760 token_bytes=654556552 weight_movement_bytes=34603008 total_nvlink_bytes=689159560 weight_to_token_ratio=0.052865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361114810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14785416016 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=7396658000 ingress_bytes=8876687360 link_bytes=8876687360 total_nvlink_bytes=16273345360 weight_to_token_ratio=0.100635 prefetch_share_of_ingress=0.089658 dedup_ratio=0.085695 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=23114 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.452272170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12705216000 prefetch_bytes=726663168 reduce_bytes=761266176 weight_movement_bytes=1487929344 egress_bytes=6356227584 ingress_bytes=7836917760 link_bytes=7836917760 total_nvlink_bytes=14193145344 weight_to_token_ratio=0.117112 prefetch_share_of_ingress=0.092723 dedup_ratio=0.165609 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=22 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.282160461e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95400 payload_rows_remote=80340 dup_rows=2904 dup_rows_remote=2068 dedup_ratio=0.029541 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7867, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 325, 0, 0, 7161, 0, 0] payload_rows_per_dst=[0, 16384, 0, 16384, 15454, 16384, 15734, 15060] dup_rows_per_dst=[0, 598, 0, 1244, 0, 134, 92, 836] combine_rows_per_src=[0, 16384, 0, 16384, 15454, 16384, 15734, 15060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97886 payload_rows_remote=81502 dup_rows=418 dup_rows_remote=418 dedup_ratio=0.004252 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7873, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 319, 0, 0, 7037, 0, 0] payload_rows_per_dst=[16384, 616, 16384, 3112, 16384, 14548, 15746, 14712] dup_rows_per_dst=[0, 0, 0, 104, 0, 134, 180, 0] combine_rows_per_src=[16384, 616, 16384, 3112, 16384, 14548, 15746, 14712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97954 payload_rows_remote=96558 dup_rows=350 dup_rows_remote=350 dedup_ratio=0.003560 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 325, 0, 0, 7024, 0, 0] payload_rows_per_dst=[16384, 608, 16384, 1396, 16364, 16384, 15736, 14698] dup_rows_per_dst=[0, 0, 0, 0, 0, 166, 184, 0] combine_rows_per_src=[16384, 608, 16384, 1396, 16364, 16384, 15736, 14698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14434221280 prefetch_bytes=830472192 reduce_bytes=588251136 weight_movement_bytes=1418723328 egress_bytes=7220993248 ingress_bytes=8631951360 link_bytes=8631951360 total_nvlink_bytes=15852944608 weight_to_token_ratio=0.098289 prefetch_share_of_ingress=0.096209 dedup_ratio=0.098068 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=17 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027391 gemm_inner_pad_upper_bound=24638 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.412232089e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95854 payload_rows_remote=80160 dup_rows=2450 dup_rows_remote=2278 dedup_ratio=0.024923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7847, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 345, 0, 0, 6967, 0, 0] payload_rows_per_dst=[0, 16384, 0, 16384, 16384, 16384, 15694, 14624] dup_rows_per_dst=[0, 700, 0, 1408, 0, 170, 172, 0] combine_rows_per_src=[0, 16384, 0, 16384, 16384, 16384, 15694, 14624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66956 payload_rows_remote=50582 dup_rows=31348 dup_rows_remote=31220 dedup_ratio=0.318888 router_tokens_per_expert=[0, 0, 22, 28, 40, 16, 0, 2, 1, 8143, 7774, 0, 0, 1, 313, 0, 0, 7, 0, 1, 35, 8136, 5, 0, 7, 21, 0, 0, 8145, 1024, 20, 0, 18, 34, 0, 1, 0, 0, 0, 8120, 1, 0, 4, 0, 267, 8, 59, 0, 6, 2, 0, 1, 0, 104, 0, 0, 0, 0, 340, 0, 0, 6445, 1, 0] payload_rows_per_dst=[16374, 16298, 16354, 16332, 70, 620, 226, 682] dup_rows_per_dst=[128, 16120, 12904, 2102, 36, 58, 0, 0] combine_rows_per_src=[16374, 16298, 16354, 16332, 70, 620, 226, 682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=329072640 dispatch_meta_bytes=329632 combine_payload_bytes=329072640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329402272 ingress_bytes=398278656 link_bytes=398278656 token_bytes=658474912 weight_movement_bytes=69206016 total_nvlink_bytes=727680928 weight_to_token_ratio=0.105100 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498690518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=333832192 dispatch_meta_bytes=327680 combine_payload_bytes=333832192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334159872 ingress_bytes=403038208 link_bytes=403038208 token_bytes=667992064 weight_movement_bytes=69206016 total_nvlink_bytes=737198080 weight_to_token_ratio=0.103603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516600329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=395501568 dispatch_meta_bytes=387632 combine_payload_bytes=395501568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=395889200 ingress_bytes=464707584 link_bytes=464707584 token_bytes=791390768 weight_movement_bytes=69206016 total_nvlink_bytes=860596784 weight_to_token_ratio=0.087449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.748657227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=328335360 dispatch_meta_bytes=329752 combine_payload_bytes=328335360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328665112 ingress_bytes=362938368 link_bytes=362938368 token_bytes=657000472 weight_movement_bytes=34603008 total_nvlink_bytes=691603480 weight_to_token_ratio=0.052668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365707860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=207183872 dispatch_meta_bytes=327208 combine_payload_bytes=207183872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=207511080 ingress_bytes=241786880 link_bytes=241786880 token_bytes=414694952 weight_movement_bytes=34603008 total_nvlink_bytes=449297960 weight_to_token_ratio=0.083442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.098245640e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11970878408 prefetch_bytes=830472192 reduce_bytes=692060160 weight_movement_bytes=1522532352 egress_bytes=5989030856 ingress_bytes=7504379904 link_bytes=7504379904 total_nvlink_bytes=13493410760 weight_to_token_ratio=0.127186 prefetch_share_of_ingress=0.110665 dedup_ratio=0.207255 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=20 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=23368 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227755540e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14652099648 prefetch_bytes=692060160 reduce_bytes=934281216 weight_movement_bytes=1626341376 egress_bytes=7329991744 ingress_bytes=8948449280 link_bytes=8948449280 total_nvlink_bytes=16278441024 weight_to_token_ratio=0.110997 prefetch_share_of_ingress=0.077339 dedup_ratio=0.095163 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=27 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=23114 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.464012793e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14792206848 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=7400057344 ingress_bytes=8776269824 link_bytes=8776269824 total_nvlink_bytes=16176327168 weight_to_token_ratio=0.093571 prefetch_share_of_ingress=0.067027 dedup_ratio=0.085962 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=21971 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.435843339e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13128109976 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6567907224 ingress_bytes=8013529088 link_bytes=8013529088 total_nvlink_bytes=14581436312 weight_to_token_ratio=0.110703 prefetch_share_of_ingress=0.082043 dedup_ratio=0.160875 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311054992e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11849455040 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=5928392128 ingress_bytes=7235977216 link_bytes=7235977216 total_nvlink_bytes=13164369344 weight_to_token_ratio=0.110968 prefetch_share_of_ingress=0.100424 dedup_ratio=0.213582 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.183843466e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-29 23:40:35] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 674.7 | throughput per GPU (TFLOP/s/GPU): 189.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.974985E+00 | loss scale: 1.0 | grad norm: 3.703 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76172 payload_rows_remote=66826 dup_rows=22132 dup_rows_remote=19814 dedup_ratio=0.225138 router_tokens_per_expert=[1391, 393, 538, 953, 185, 1179, 832, 707, 1610, 287, 737, 335, 132, 1645, 822, 922, 647, 910, 796, 622, 184, 480, 242, 1344, 531, 753, 819, 367, 752, 98, 772, 912, 872, 851, 1363, 913, 819, 238, 675, 981, 500, 611, 637, 511, 960, 963, 657, 403, 687, 550, 717, 628, 919, 1675, 528, 731, 1867, 398, 1495, 811, 150, 706, 1421, 1018] payload_rows_per_dst=[9340, 9606, 9772, 9198, 9346, 10462, 9768, 8680] dup_rows_per_dst=[3016, 3374, 2438, 2816, 2318, 3012, 3102, 2056] combine_rows_per_src=[9340, 9606, 9772, 9198, 9346, 10462, 9768, 8680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=61462 dup_rows=24026 dup_rows_remote=17464 dedup_ratio=0.244405 router_tokens_per_expert=[1309, 361, 559, 811, 189, 1361, 953, 616, 1507, 298, 763, 434, 135, 1652, 880, 952, 664, 859, 860, 728, 185, 447, 224, 1151, 499, 679, 788, 399, 824, 166, 693, 1063, 954, 877, 1419, 937, 586, 244, 740, 852, 508, 582, 620, 533, 922, 946, 582, 382, 632, 559, 642, 596, 1005, 1566, 531, 805, 1686, 488, 1684, 826, 151, 862, 1467, 959] payload_rows_per_dst=[9202, 7312, 8388, 8594, 11508, 7982, 8476, 12816] dup_rows_per_dst=[2724, 1620, 1848, 1628, 5014, 2168, 2462, 6562] combine_rows_per_src=[9202, 7312, 8388, 8594, 11508, 7982, 8476, 12816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75914 payload_rows_remote=65990 dup_rows=22390 dup_rows_remote=18912 dedup_ratio=0.227763 router_tokens_per_expert=[1283, 319, 621, 1042, 255, 1256, 898, 678, 1586, 262, 730, 357, 126, 1694, 905, 1012, 645, 918, 848, 695, 192, 454, 251, 1246, 495, 729, 693, 402, 801, 96, 774, 959, 855, 825, 1412, 842, 844, 264, 549, 989, 496, 662, 578, 661, 831, 920, 587, 400, 607, 485, 764, 714, 973, 1658, 602, 762, 1752, 372, 1568, 789, 161, 687, 1325, 996] payload_rows_per_dst=[9620, 9740, 10340, 9924, 8708, 10386, 9882, 7314] dup_rows_per_dst=[3084, 3604, 2982, 3478, 1628, 3020, 3248, 1346] combine_rows_per_src=[9620, 9740, 10340, 9924, 8708, 10386, 9882, 7314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75976 payload_rows_remote=67772 dup_rows=22328 dup_rows_remote=20180 dedup_ratio=0.227132 router_tokens_per_expert=[1341, 273, 505, 866, 169, 1222, 889, 500, 1692, 284, 702, 272, 106, 1942, 892, 1015, 732, 916, 966, 711, 199, 484, 237, 1306, 557, 711, 713, 484, 789, 144, 717, 946, 813, 782, 1505, 890, 759, 248, 750, 971, 448, 556, 596, 518, 999, 1091, 503, 427, 695, 544, 686, 531, 809, 1676, 468, 714, 1753, 372, 1623, 702, 265, 781, 1414, 981] payload_rows_per_dst=[9060, 10034, 8956, 8402, 10314, 8204, 9526, 11480] dup_rows_per_dst=[2470, 3776, 2146, 1720, 3122, 2148, 2720, 4226] combine_rows_per_src=[9060, 10034, 8956, 8402, 10314, 8204, 9526, 11480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76138 payload_rows_remote=66572 dup_rows=22166 dup_rows_remote=19116 dedup_ratio=0.225484 router_tokens_per_expert=[1316, 372, 619, 892, 203, 1288, 789, 627, 1540, 271, 646, 326, 147, 1624, 803, 951, 697, 914, 841, 628, 204, 471, 256, 1332, 535, 635, 731, 326, 744, 118, 839, 1023, 873, 936, 1469, 899, 755, 238, 699, 956, 397, 536, 608, 605, 877, 1049, 687, 377, 589, 512, 704, 644, 970, 1644, 604, 826, 1968, 435, 1633, 696, 159, 749, 1362, 988] payload_rows_per_dst=[9360, 9566, 10546, 10136, 8858, 10490, 9802, 7380] dup_rows_per_dst=[2852, 3050, 3078, 3702, 1854, 3048, 3184, 1398] combine_rows_per_src=[9360, 9566, 10546, 10136, 8858, 10490, 9802, 7380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75152 payload_rows_remote=66234 dup_rows=23152 dup_rows_remote=21086 dedup_ratio=0.235514 router_tokens_per_expert=[1348, 381, 669, 1082, 197, 1278, 767, 587, 1573, 351, 676, 322, 123, 1643, 759, 1037, 660, 1018, 767, 586, 238, 448, 243, 1392, 561, 688, 726, 278, 773, 114, 826, 1010, 853, 963, 1362, 865, 766, 215, 622, 893, 511, 598, 650, 641, 877, 976, 612, 401, 573, 553, 681, 724, 966, 1600, 673, 841, 1819, 342, 1513, 769, 174, 676, 1330, 992] payload_rows_per_dst=[9500, 8246, 8732, 8234, 11106, 8222, 8918, 12194] dup_rows_per_dst=[3118, 2444, 1972, 1718, 4250, 2310, 2066, 5274] combine_rows_per_src=[9500, 8246, 8732, 8234, 11106, 8222, 8918, 12194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76094 payload_rows_remote=65612 dup_rows=22210 dup_rows_remote=19162 dedup_ratio=0.225932 router_tokens_per_expert=[1283, 335, 657, 1003, 175, 1205, 842, 679, 1508, 243, 716, 309, 120, 1793, 897, 916, 743, 972, 800, 666, 192, 429, 270, 1254, 491, 709, 806, 372, 740, 94, 862, 1345, 875, 928, 1439, 884, 557, 246, 705, 905, 427, 593, 616, 590, 936, 952, 500, 385, 664, 495, 629, 621, 987, 1739, 498, 666, 1905, 435, 1598, 746, 171, 692, 1448, 894] payload_rows_per_dst=[9316, 9622, 10482, 10874, 8364, 10358, 9688, 7390] dup_rows_per_dst=[3042, 3382, 3048, 3774, 1836, 2836, 2910, 1382] combine_rows_per_src=[9316, 9622, 10482, 10874, 8364, 10358, 9688, 7390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273719296 dispatch_meta_bytes=346560 combine_payload_bytes=273719296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274065856 ingress_bytes=342925312 link_bytes=342925312 token_bytes=547785152 weight_movement_bytes=69206016 total_nvlink_bytes=616991168 weight_to_token_ratio=0.126338 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290400341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270295040 dispatch_meta_bytes=339608 combine_payload_bytes=270295040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270634648 ingress_bytes=304898048 link_bytes=304898048 token_bytes=540929688 weight_movement_bytes=34603008 total_nvlink_bytes=575532696 weight_to_token_ratio=0.063970 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147306808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=251748352 dispatch_meta_bytes=315704 combine_payload_bytes=251748352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=252064056 ingress_bytes=355557376 link_bytes=355557376 token_bytes=503812408 weight_movement_bytes=103809024 total_nvlink_bytes=607621432 weight_to_token_ratio=0.206047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337933781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277594112 dispatch_meta_bytes=351808 combine_payload_bytes=277594112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277945920 ingress_bytes=312197120 link_bytes=312197120 token_bytes=555540032 weight_movement_bytes=34603008 total_nvlink_bytes=590143040 weight_to_token_ratio=0.062287 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174772629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272678912 dispatch_meta_bytes=342752 combine_payload_bytes=272678912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273021664 ingress_bytes=376487936 link_bytes=376487936 token_bytes=545700576 weight_movement_bytes=103809024 total_nvlink_bytes=649509600 weight_to_token_ratio=0.190231 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416693793e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76036 payload_rows_remote=66484 dup_rows=22268 dup_rows_remote=19220 dedup_ratio=0.226522 router_tokens_per_expert=[1340, 351, 670, 918, 168, 1210, 929, 714, 1538, 251, 713, 316, 144, 1515, 938, 1049, 640, 945, 911, 712, 202, 445, 226, 1426, 623, 732, 672, 318, 892, 92, 822, 995, 811, 862, 1281, 829, 772, 212, 591, 962, 453, 629, 559, 533, 912, 1031, 743, 428, 600, 455, 690, 641, 1018, 1583, 546, 743, 1907, 411, 1541, 765, 157, 747, 1406, 917] payload_rows_per_dst=[9552, 9650, 10478, 10402, 8546, 10512, 9554, 7342] dup_rows_per_dst=[3048, 3278, 3098, 3704, 1532, 3146, 2998, 1464] combine_rows_per_src=[9552, 9650, 10478, 10402, 8546, 10512, 9554, 7342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271294464 dispatch_meta_bytes=349280 combine_payload_bytes=271294464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271643744 ingress_bytes=340500480 link_bytes=340500480 token_bytes=542938208 weight_movement_bytes=69206016 total_nvlink_bytes=612144224 weight_to_token_ratio=0.127466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281275894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268746752 dispatch_meta_bytes=339096 combine_payload_bytes=268746752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269085848 ingress_bytes=303349760 link_bytes=303349760 token_bytes=537832600 weight_movement_bytes=34603008 total_nvlink_bytes=572435608 weight_to_token_ratio=0.064338 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141480725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74640 payload_rows_remote=65240 dup_rows=23664 dup_rows_remote=20256 dedup_ratio=0.240723 router_tokens_per_expert=[595, 689, 413, 554, 2053, 236, 397, 995, 66, 914, 1874, 327, 694, 1113, 536, 1985, 881, 1276, 812, 316, 1364, 157, 1761, 1442, 671, 787, 898, 1121, 17, 129, 147, 159, 291, 45, 346, 53, 1248, 730, 893, 813, 1911, 1148, 603, 1192, 208, 1116, 15, 134, 1839, 1138, 1699, 2, 44, 1111, 214, 1454, 1311, 312, 1030, 541, 943, 619, 366, 404] payload_rows_per_dst=[8984, 8300, 9480, 8910, 9400, 9430, 11240, 8896] dup_rows_per_dst=[2880, 2748, 2122, 3364, 3408, 3224, 3762, 2156] combine_rows_per_src=[8984, 8300, 9480, 8910, 9400, 9430, 11240, 8896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74674 payload_rows_remote=66458 dup_rows=23630 dup_rows_remote=20770 dedup_ratio=0.240377 router_tokens_per_expert=[551, 837, 424, 531, 1637, 234, 379, 1135, 65, 921, 1739, 338, 796, 1129, 706, 1935, 899, 1317, 875, 488, 1501, 152, 1799, 1148, 728, 676, 711, 1167, 20, 128, 167, 142, 313, 52, 386, 89, 1097, 969, 940, 844, 2061, 1185, 768, 1217, 185, 903, 19, 162, 1724, 1095, 1523, 6, 60, 1206, 255, 1516, 1248, 267, 909, 527, 996, 635, 335, 385] payload_rows_per_dst=[8746, 8574, 10114, 8216, 9610, 9616, 11020, 8778] dup_rows_per_dst=[2710, 2814, 2646, 2860, 3640, 3384, 3750, 1826] combine_rows_per_src=[8746, 8574, 10114, 8216, 9610, 9616, 11020, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272318464 dispatch_meta_bytes=342816 combine_payload_bytes=272318464 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=272661280 ingress_bytes=306921472 link_bytes=306921472 token_bytes=544979744 weight_movement_bytes=34603008 total_nvlink_bytes=579582752 weight_to_token_ratio=0.063494 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154920790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72890 payload_rows_remote=62162 dup_rows=25414 dup_rows_remote=20848 dedup_ratio=0.258525 router_tokens_per_expert=[589, 689, 332, 521, 2185, 152, 403, 1131, 56, 1040, 2017, 395, 635, 1104, 519, 2042, 916, 1199, 843, 448, 1110, 199, 1609, 1423, 693, 725, 815, 1260, 27, 156, 167, 177, 335, 43, 321, 74, 1070, 626, 828, 783, 1800, 1040, 539, 918, 196, 1174, 8, 165, 2059, 1107, 1946, 0, 49, 1116, 246, 1544, 1391, 381, 978, 509, 895, 651, 392, 391] payload_rows_per_dst=[9234, 8578, 7882, 9598, 8992, 8878, 9000, 10728] dup_rows_per_dst=[2770, 2954, 1548, 4506, 3252, 2802, 3016, 4566] combine_rows_per_src=[9234, 8578, 7882, 9598, 8992, 8878, 9000, 10728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73666 payload_rows_remote=64580 dup_rows=24638 dup_rows_remote=21060 dedup_ratio=0.250631 router_tokens_per_expert=[636, 718, 447, 518, 2303, 138, 364, 1121, 50, 988, 1874, 380, 688, 1228, 563, 2145, 859, 1189, 870, 343, 1256, 213, 1710, 1312, 714, 778, 781, 1126, 24, 141, 155, 150, 304, 48, 306, 52, 1023, 748, 810, 851, 1722, 1184, 590, 1049, 193, 1129, 19, 187, 2013, 1051, 1953, 0, 33, 1151, 231, 1420, 1281, 375, 946, 510, 870, 551, 436, 334] payload_rows_per_dst=[9488, 9086, 11052, 8292, 9082, 9126, 8770, 8770] dup_rows_per_dst=[3002, 3578, 4938, 2866, 3258, 3020, 2140, 1836] combine_rows_per_src=[9488, 9086, 11052, 8292, 9082, 9126, 8770, 8770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73676 payload_rows_remote=64460 dup_rows=24628 dup_rows_remote=21382 dedup_ratio=0.250529 router_tokens_per_expert=[602, 712, 363, 547, 2142, 111, 320, 1375, 48, 934, 1848, 382, 631, 980, 517, 2050, 963, 1193, 722, 392, 1138, 202, 1653, 1585, 700, 800, 835, 1257, 34, 309, 111, 118, 253, 43, 316, 61, 1145, 633, 804, 776, 1911, 1212, 505, 1070, 164, 1227, 11, 131, 2244, 1015, 2087, 0, 39, 991, 200, 1193, 1348, 316, 1130, 502, 805, 627, 370, 449] payload_rows_per_dst=[9342, 8208, 7780, 10186, 9070, 9216, 9152, 10722] dup_rows_per_dst=[3002, 2472, 1440, 4618, 3092, 3246, 2412, 4346] combine_rows_per_src=[9342, 8208, 7780, 10186, 9070, 9216, 9152, 10722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65298 dup_rows=24044 dup_rows_remote=21552 dedup_ratio=0.244588 router_tokens_per_expert=[546, 837, 526, 470, 2021, 233, 386, 1183, 63, 944, 1941, 368, 691, 1069, 685, 2109, 854, 1322, 830, 361, 1575, 175, 1770, 1264, 720, 699, 825, 1038, 24, 144, 110, 143, 316, 46, 373, 91, 1165, 943, 818, 853, 1804, 1186, 651, 1051, 192, 925, 11, 169, 1762, 1091, 1802, 6, 51, 1116, 233, 1428, 1158, 330, 918, 474, 1002, 579, 315, 367] payload_rows_per_dst=[9252, 8714, 8676, 9464, 9838, 9176, 8962, 10178] dup_rows_per_dst=[3152, 2808, 1558, 4010, 3590, 2802, 2492, 3632] combine_rows_per_src=[9252, 8714, 8676, 9464, 9838, 9176, 8962, 10178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=341984 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267565024 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534788064 weight_movement_bytes=34603008 total_nvlink_bytes=569391072 weight_to_token_ratio=0.064704 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272211968 dispatch_meta_bytes=348912 combine_payload_bytes=272211968 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272560880 ingress_bytes=341417984 link_bytes=341417984 token_bytes=544772848 weight_movement_bytes=69206016 total_nvlink_bytes=613978864 weight_to_token_ratio=0.127036 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284728387e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=254615552 dispatch_meta_bytes=332040 combine_payload_bytes=254615552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254947592 ingress_bytes=289218560 link_bytes=289218560 token_bytes=509563144 weight_movement_bytes=34603008 total_nvlink_bytes=544166152 weight_to_token_ratio=0.067907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.088306157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=63758 dup_rows=24090 dup_rows_remote=20876 dedup_ratio=0.245056 router_tokens_per_expert=[686, 854, 417, 519, 1916, 140, 397, 1213, 50, 980, 1876, 359, 676, 1132, 584, 2026, 837, 1293, 910, 291, 1291, 131, 1713, 1378, 667, 693, 744, 1105, 31, 84, 121, 158, 309, 29, 360, 58, 1167, 831, 931, 823, 1920, 1074, 613, 1027, 186, 1102, 11, 133, 2065, 1136, 1810, 1, 22, 1176, 270, 1558, 1365, 258, 980, 517, 910, 558, 318, 362] payload_rows_per_dst=[9118, 8450, 10456, 8082, 9574, 9086, 10830, 8618] dup_rows_per_dst=[3166, 2864, 3214, 2550, 3494, 3156, 3728, 1918] combine_rows_per_src=[9118, 8450, 10456, 8082, 9574, 9086, 10830, 8618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342560 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264862240 ingress_bytes=333725696 link_bytes=333725696 token_bytes=529381920 weight_movement_bytes=69206016 total_nvlink_bytes=598587936 weight_to_token_ratio=0.130730 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255782927e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264028160 dispatch_meta_bytes=343368 combine_payload_bytes=264028160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264371528 ingress_bytes=298631168 link_bytes=298631168 token_bytes=528399688 weight_movement_bytes=34603008 total_nvlink_bytes=563002696 weight_to_token_ratio=0.065486 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123725043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=347400 combine_payload_bytes=267460608 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267808008 ingress_bytes=440475648 link_bytes=440475648 token_bytes=535268616 weight_movement_bytes=173015040 total_nvlink_bytes=708283656 weight_to_token_ratio=0.323230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657474402e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64436 dup_rows=24166 dup_rows_remote=20410 dedup_ratio=0.245829 router_tokens_per_expert=[620, 696, 495, 631, 1986, 204, 383, 1122, 100, 915, 1815, 325, 732, 1195, 646, 1878, 1016, 1292, 773, 561, 1469, 154, 1629, 1293, 616, 752, 810, 1115, 18, 196, 126, 129, 268, 35, 375, 99, 1125, 916, 858, 826, 1886, 1075, 681, 1124, 183, 937, 11, 174, 1555, 973, 1733, 2, 58, 1149, 278, 1472, 1303, 343, 996, 515, 1042, 584, 389, 495] payload_rows_per_dst=[9702, 12362, 11004, 8096, 7816, 9318, 6736, 9104] dup_rows_per_dst=[3756, 5794, 4394, 2686, 1188, 2824, 1294, 2230] combine_rows_per_src=[9702, 12362, 11004, 8096, 7816, 9318, 6736, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=261152768 dispatch_meta_bytes=338536 combine_payload_bytes=261152768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261491304 ingress_bytes=364961792 link_bytes=364961792 token_bytes=522644072 weight_movement_bytes=103809024 total_nvlink_bytes=626453096 weight_to_token_ratio=0.198623 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373321841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77466 payload_rows_remote=68990 dup_rows=20838 dup_rows_remote=18732 dedup_ratio=0.211975 router_tokens_per_expert=[1, 1298, 565, 225, 7, 1188, 599, 240, 378, 437, 1247, 1366, 1447, 645, 0, 167, 1051, 1789, 379, 266, 42, 69, 2122, 643, 847, 618, 1631, 1542, 45, 91, 711, 49, 317, 1370, 1014, 668, 4085, 2360, 1651, 271, 473, 86, 195, 96, 1706, 1573, 121, 536, 463, 2, 632, 828, 13, 6, 107, 383, 2309, 504, 24, 251, 0, 2455, 25, 2923] payload_rows_per_dst=[10936, 8778, 10206, 8994, 8476, 10358, 10202, 9516] dup_rows_per_dst=[3156, 2596, 2516, 2074, 2106, 3934, 2836, 1620] combine_rows_per_src=[10936, 8778, 10206, 8994, 8476, 10358, 10202, 9516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77262 payload_rows_remote=68322 dup_rows=21042 dup_rows_remote=19112 dedup_ratio=0.214050 router_tokens_per_expert=[2, 1139, 519, 256, 23, 1116, 653, 253, 438, 485, 1126, 1509, 1281, 655, 6, 301, 1081, 1473, 412, 276, 53, 68, 2435, 663, 655, 641, 1545, 1441, 33, 363, 680, 77, 360, 1637, 1048, 668, 3542, 1713, 1694, 339, 478, 117, 225, 139, 2013, 1486, 129, 703, 565, 63, 547, 827, 10, 20, 172, 441, 2230, 561, 43, 359, 58, 2268, 51, 2988] payload_rows_per_dst=[10954, 8924, 10086, 8940, 9038, 10192, 9602, 9526] dup_rows_per_dst=[2944, 2678, 2836, 1930, 2454, 3814, 2772, 1614] combine_rows_per_src=[10954, 8924, 10086, 8940, 9038, 10192, 9602, 9526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263929856 dispatch_meta_bytes=339384 combine_payload_bytes=263929856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264269240 ingress_bytes=298532864 link_bytes=298532864 token_bytes=528199096 weight_movement_bytes=34603008 total_nvlink_bytes=562802104 weight_to_token_ratio=0.065511 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123355133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75188 payload_rows_remote=62552 dup_rows=23116 dup_rows_remote=16672 dedup_ratio=0.235148 router_tokens_per_expert=[1, 1072, 588, 259, 6, 1225, 504, 260, 367, 421, 1024, 1671, 1246, 538, 0, 119, 1128, 1702, 683, 280, 45, 69, 2015, 595, 903, 766, 2065, 1964, 41, 29, 708, 47, 332, 1512, 1013, 519, 4049, 2096, 1495, 211, 703, 102, 190, 164, 1569, 1752, 160, 387, 450, 5, 395, 684, 8, 2, 229, 254, 2397, 628, 30, 234, 0, 2189, 19, 3033] payload_rows_per_dst=[6800, 8712, 9808, 10194, 6720, 10166, 10152, 12636] dup_rows_per_dst=[1030, 2060, 3226, 2852, 1424, 4080, 2000, 6444] combine_rows_per_src=[6800, 8712, 9808, 10194, 6720, 10166, 10152, 12636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75826 payload_rows_remote=65938 dup_rows=22478 dup_rows_remote=18200 dedup_ratio=0.228658 router_tokens_per_expert=[0, 1280, 598, 213, 5, 1014, 496, 234, 414, 435, 1217, 1404, 1692, 585, 2, 111, 1124, 1823, 530, 271, 44, 92, 2075, 664, 846, 670, 1705, 1525, 28, 13, 714, 52, 315, 1287, 1109, 579, 4375, 2337, 1458, 250, 622, 80, 192, 184, 1662, 1402, 109, 583, 441, 5, 626, 789, 8, 8, 126, 257, 2351, 585, 10, 214, 0, 2023, 26, 3263] payload_rows_per_dst=[11134, 9888, 8748, 10074, 10542, 7344, 10506, 7590] dup_rows_per_dst=[3072, 4278, 2052, 2704, 4128, 2324, 2764, 1156] combine_rows_per_src=[11134, 9888, 8748, 10074, 10542, 7344, 10506, 7590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77146 payload_rows_remote=67492 dup_rows=21158 dup_rows_remote=17878 dedup_ratio=0.215230 router_tokens_per_expert=[0, 1143, 550, 244, 9, 1280, 571, 235, 340, 416, 1152, 1621, 1706, 515, 1, 41, 1074, 1584, 624, 214, 25, 61, 1979, 618, 900, 654, 1870, 2086, 23, 6, 745, 32, 318, 1555, 1077, 428, 4520, 2042, 1558, 206, 665, 90, 156, 54, 1508, 1394, 120, 438, 446, 4, 471, 738, 3, 7, 173, 284, 2420, 560, 16, 161, 3, 2144, 10, 3264] payload_rows_per_dst=[9184, 8926, 9608, 10078, 8396, 9654, 10816, 10484] dup_rows_per_dst=[2448, 2658, 2750, 2554, 1888, 3280, 2476, 3104] combine_rows_per_src=[9184, 8926, 9608, 10078, 8396, 9654, 10816, 10484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76284 payload_rows_remote=65346 dup_rows=22020 dup_rows_remote=19286 dedup_ratio=0.223999 router_tokens_per_expert=[0, 1234, 588, 240, 4, 1112, 704, 211, 440, 438, 1241, 1281, 1286, 585, 5, 286, 1037, 1586, 463, 307, 40, 75, 2503, 798, 787, 618, 1550, 1440, 40, 34, 879, 71, 314, 1382, 1059, 593, 4560, 1972, 1457, 273, 509, 74, 170, 180, 1735, 1495, 116, 560, 516, 12, 508, 755, 14, 5, 127, 339, 2169, 438, 37, 323, 9, 2398, 30, 3140] payload_rows_per_dst=[6974, 8652, 10468, 8800, 8246, 10056, 10938, 12150] dup_rows_per_dst=[1212, 2472, 3150, 2038, 1910, 3566, 2734, 4938] combine_rows_per_src=[6974, 8652, 10468, 8800, 8246, 10056, 10938, 12150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282583040 dispatch_meta_bytes=350888 combine_payload_bytes=282583040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=282933928 ingress_bytes=420995072 link_bytes=420995072 token_bytes=565516968 weight_movement_bytes=138412032 total_nvlink_bytes=703929000 weight_to_token_ratio=0.244753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584170563e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279846912 dispatch_meta_bytes=349736 combine_payload_bytes=279846912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280196648 ingress_bytes=314449920 link_bytes=314449920 token_bytes=560043560 weight_movement_bytes=34603008 total_nvlink_bytes=594646568 weight_to_token_ratio=0.061786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183249734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77196 payload_rows_remote=67354 dup_rows=21108 dup_rows_remote=18054 dedup_ratio=0.214722 router_tokens_per_expert=[0, 1306, 536, 247, 14, 1074, 586, 212, 346, 402, 1102, 1584, 1305, 597, 1, 186, 1144, 1604, 429, 228, 49, 41, 2411, 542, 692, 721, 1938, 1823, 30, 12, 764, 29, 373, 1416, 986, 585, 4364, 2085, 1371, 217, 488, 94, 202, 88, 1951, 1319, 126, 564, 479, 2, 461, 847, 9, 9, 105, 315, 2211, 553, 20, 220, 1, 2560, 27, 3149] payload_rows_per_dst=[11060, 8730, 9842, 9674, 9938, 7738, 10590, 9624] dup_rows_per_dst=[3188, 2316, 3054, 2344, 3818, 2236, 2592, 1560] combine_rows_per_src=[11060, 8730, 9842, 9674, 9938, 7738, 10590, 9624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=256212992 dispatch_meta_bytes=316896 combine_payload_bytes=256212992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256529888 ingress_bytes=360022016 link_bytes=360022016 token_bytes=512742880 weight_movement_bytes=103809024 total_nvlink_bytes=616551904 weight_to_token_ratio=0.202458 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354733862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270082048 dispatch_meta_bytes=336552 combine_payload_bytes=270082048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270418600 ingress_bytes=304685056 link_bytes=304685056 token_bytes=540500648 weight_movement_bytes=34603008 total_nvlink_bytes=575103656 weight_to_token_ratio=0.064020 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146505337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276447232 dispatch_meta_bytes=341480 combine_payload_bytes=276447232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276788712 ingress_bytes=311050240 link_bytes=311050240 token_bytes=553235944 weight_movement_bytes=34603008 total_nvlink_bytes=587838952 weight_to_token_ratio=0.062547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170457012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267657216 dispatch_meta_bytes=338528 combine_payload_bytes=267657216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267995744 ingress_bytes=302260224 link_bytes=302260224 token_bytes=535652960 weight_movement_bytes=34603008 total_nvlink_bytes=570255968 weight_to_token_ratio=0.064600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137380889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=341632 combine_payload_bytes=275881984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276223616 ingress_bytes=345088000 link_bytes=345088000 token_bytes=552105600 weight_movement_bytes=69206016 total_nvlink_bytes=621311616 weight_to_token_ratio=0.125349 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298538362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73074 payload_rows_remote=62046 dup_rows=25230 dup_rows_remote=22046 dedup_ratio=0.256653 router_tokens_per_expert=[0, 1308, 573, 281, 14, 1244, 591, 235, 412, 398, 1107, 1515, 1745, 723, 5, 250, 1066, 1514, 419, 276, 40, 70, 2163, 679, 834, 563, 1590, 1415, 58, 83, 728, 56, 317, 1492, 1096, 795, 3730, 1969, 1792, 289, 575, 130, 176, 95, 1757, 1341, 128, 579, 479, 34, 754, 832, 10, 7, 203, 386, 2200, 482, 21, 352, 43, 2234, 39, 2860] payload_rows_per_dst=[11028, 11224, 8444, 11446, 11910, 7266, 5994, 5762] dup_rows_per_dst=[3184, 5412, 1510, 3608, 7310, 2296, 1330, 580] combine_rows_per_src=[11028, 11224, 8444, 11446, 11910, 7266, 5994, 5762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=254140416 dispatch_meta_bytes=336368 combine_payload_bytes=254140416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254476784 ingress_bytes=288743424 link_bytes=288743424 token_bytes=508617200 weight_movement_bytes=34603008 total_nvlink_bytes=543220208 weight_to_token_ratio=0.068033 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086518259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79966 payload_rows_remote=68836 dup_rows=18338 dup_rows_remote=16348 dedup_ratio=0.186544 router_tokens_per_expert=[152, 87, 1518, 6, 447, 618, 555, 654, 2532, 2451, 327, 276, 1104, 357, 4537, 207, 2678, 46, 2180, 352, 511, 262, 1, 379, 658, 1311, 523, 372, 474, 1432, 896, 414, 439, 298, 93, 246, 333, 236, 281, 97, 54, 3694, 64, 462, 169, 113, 2, 4998, 0, 143, 78, 1112, 90, 2085, 56, 65, 76, 7, 1569, 371, 1982, 1269, 70, 283] payload_rows_per_dst=[10362, 8360, 9502, 10060, 11130, 7800, 13534, 9218] dup_rows_per_dst=[2776, 1084, 3316, 2100, 1990, 1316, 3720, 2036] combine_rows_per_src=[10362, 8360, 9502, 10060, 11130, 7800, 13534, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79274 payload_rows_remote=68890 dup_rows=19030 dup_rows_remote=16706 dedup_ratio=0.193583 router_tokens_per_expert=[165, 174, 1538, 11, 423, 444, 594, 599, 2299, 2495, 314, 304, 1129, 363, 4203, 142, 2512, 35, 2535, 383, 490, 301, 0, 488, 586, 1292, 560, 407, 550, 1580, 921, 458, 386, 513, 75, 223, 433, 252, 495, 120, 126, 3228, 72, 476, 248, 134, 4, 4666, 0, 160, 75, 1065, 183, 1930, 82, 80, 64, 16, 1559, 673, 1786, 1122, 277, 334] payload_rows_per_dst=[9924, 8466, 9922, 10384, 11128, 9532, 10186, 9732] dup_rows_per_dst=[2570, 1028, 3566, 2324, 2272, 3192, 2148, 1930] combine_rows_per_src=[9924, 8466, 9922, 10384, 11128, 9532, 10186, 9732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78222 payload_rows_remote=69934 dup_rows=20082 dup_rows_remote=18974 dedup_ratio=0.204285 router_tokens_per_expert=[142, 75, 1442, 8, 554, 768, 499, 724, 3264, 2518, 308, 258, 1080, 356, 4501, 178, 2582, 31, 2025, 344, 456, 285, 0, 305, 609, 1379, 590, 357, 422, 1536, 715, 408, 372, 191, 80, 216, 364, 164, 301, 59, 67, 4136, 94, 517, 191, 88, 0, 4627, 0, 103, 71, 1143, 58, 1797, 61, 81, 182, 2, 1285, 502, 1924, 1376, 77, 304] payload_rows_per_dst=[11168, 8288, 9148, 10134, 10894, 13296, 6026, 9268] dup_rows_per_dst=[3784, 1108, 2908, 1898, 1602, 6144, 602, 2036] combine_rows_per_src=[11168, 8288, 9148, 10134, 10894, 13296, 6026, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76340 payload_rows_remote=63508 dup_rows=21964 dup_rows_remote=15642 dedup_ratio=0.223429 router_tokens_per_expert=[156, 50, 1436, 3, 638, 988, 545, 743, 3040, 2453, 367, 289, 976, 340, 4634, 147, 2958, 16, 2145, 329, 414, 209, 0, 335, 433, 1436, 456, 351, 408, 1338, 871, 531, 411, 212, 101, 216, 316, 263, 239, 96, 40, 3690, 74, 400, 214, 91, 0, 4217, 0, 106, 96, 1160, 84, 1861, 169, 174, 122, 0, 1548, 377, 2004, 1446, 57, 333] payload_rows_per_dst=[7356, 11438, 8856, 10612, 11120, 1506, 12620, 12832] dup_rows_per_dst=[1762, 3786, 2600, 2382, 1856, 142, 3114, 6322] combine_rows_per_src=[7356, 11438, 8856, 10612, 11120, 1506, 12620, 12832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79982 payload_rows_remote=71920 dup_rows=18322 dup_rows_remote=17010 dedup_ratio=0.186381 router_tokens_per_expert=[139, 88, 1354, 7, 548, 995, 453, 675, 3332, 2644, 345, 267, 975, 316, 4849, 168, 2779, 27, 1864, 359, 447, 237, 0, 332, 556, 1255, 534, 330, 467, 1303, 871, 511, 350, 227, 96, 214, 254, 214, 275, 97, 57, 3870, 53, 452, 157, 98, 0, 4452, 0, 122, 90, 1004, 73, 1847, 157, 161, 113, 5, 1481, 306, 1902, 1560, 87, 351] payload_rows_per_dst=[9208, 10036, 9214, 9800, 11474, 8062, 12924, 9264] dup_rows_per_dst=[2944, 2424, 2876, 1854, 1678, 1312, 2888, 2346] combine_rows_per_src=[9208, 10036, 9214, 9800, 11474, 8062, 12924, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79980 payload_rows_remote=66780 dup_rows=18324 dup_rows_remote=15308 dedup_ratio=0.186401 router_tokens_per_expert=[155, 89, 1535, 2, 457, 628, 569, 605, 2673, 2648, 310, 275, 957, 346, 4571, 222, 2755, 27, 2345, 363, 517, 313, 0, 363, 652, 1274, 525, 350, 523, 1444, 1037, 392, 400, 314, 82, 243, 337, 253, 416, 153, 51, 3982, 34, 458, 194, 93, 0, 4676, 0, 106, 63, 976, 115, 2077, 47, 48, 78, 6, 1419, 207, 1869, 1152, 107, 274] payload_rows_per_dst=[7004, 11386, 10042, 10078, 11400, 8170, 13200, 8700] dup_rows_per_dst=[1076, 3476, 3324, 2316, 2138, 1194, 3016, 1784] combine_rows_per_src=[7004, 11386, 10042, 10078, 11400, 8170, 13200, 8700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281952256 dispatch_meta_bytes=340736 combine_payload_bytes=281952256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282292992 ingress_bytes=316555264 link_bytes=316555264 token_bytes=564245248 weight_movement_bytes=34603008 total_nvlink_bytes=598848256 weight_to_token_ratio=0.061326 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191171974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78690 payload_rows_remote=68842 dup_rows=19614 dup_rows_remote=16436 dedup_ratio=0.199524 router_tokens_per_expert=[133, 113, 1417, 3, 506, 742, 553, 694, 2882, 2436, 301, 289, 1069, 373, 4539, 116, 2502, 21, 2412, 411, 494, 233, 0, 440, 543, 1248, 494, 379, 456, 1801, 788, 446, 386, 254, 87, 231, 359, 258, 321, 76, 47, 3436, 42, 389, 235, 61, 0, 4808, 0, 146, 83, 1183, 118, 1859, 64, 72, 52, 4, 1539, 284, 2206, 1225, 113, 380] payload_rows_per_dst=[10850, 8220, 9848, 9868, 11168, 12842, 6466, 9428] dup_rows_per_dst=[3236, 948, 3178, 2442, 1854, 5194, 584, 2178] combine_rows_per_src=[10850, 8220, 9848, 9868, 11168, 12842, 6466, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282173440 dispatch_meta_bytes=342384 combine_payload_bytes=282173440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282515824 ingress_bytes=316776448 link_bytes=316776448 token_bytes=564689264 weight_movement_bytes=34603008 total_nvlink_bytes=599292272 weight_to_token_ratio=0.061278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192004271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=286449664 dispatch_meta_bytes=355632 combine_payload_bytes=286449664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286805296 ingress_bytes=390258688 link_bytes=390258688 token_bytes=573254960 weight_movement_bytes=103809024 total_nvlink_bytes=677063984 weight_to_token_ratio=0.181087 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468512025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=260128768 dispatch_meta_bytes=316600 combine_payload_bytes=260128768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260445368 ingress_bytes=294731776 link_bytes=294731776 token_bytes=520574136 weight_movement_bytes=34603008 total_nvlink_bytes=555177144 weight_to_token_ratio=0.066471 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109051945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=294584320 dispatch_meta_bytes=355720 combine_payload_bytes=294584320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=294940040 ingress_bytes=398393344 link_bytes=398393344 token_bytes=589524360 weight_movement_bytes=103809024 total_nvlink_bytes=693333384 weight_to_token_ratio=0.176089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499122080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75928 payload_rows_remote=65214 dup_rows=22376 dup_rows_remote=18806 dedup_ratio=0.227620 router_tokens_per_expert=[138, 129, 1550, 12, 544, 763, 543, 687, 2776, 2168, 364, 288, 1078, 356, 4301, 208, 2395, 53, 2292, 360, 405, 274, 0, 445, 607, 1339, 499, 375, 525, 1369, 812, 500, 420, 380, 128, 232, 398, 243, 530, 112, 84, 3738, 77, 435, 203, 90, 1, 4673, 0, 138, 93, 1134, 131, 1860, 141, 141, 108, 5, 1465, 468, 1803, 1285, 133, 348] payload_rows_per_dst=[10714, 12710, 8030, 10050, 5812, 12902, 6448, 9262] dup_rows_per_dst=[3570, 5546, 1974, 2002, 788, 5700, 828, 1968] combine_rows_per_src=[10714, 12710, 8030, 10050, 5812, 12902, 6448, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=273530880 dispatch_meta_bytes=328352 combine_payload_bytes=273530880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273859232 ingress_bytes=308133888 link_bytes=308133888 token_bytes=547390112 weight_movement_bytes=34603008 total_nvlink_bytes=581993120 weight_to_token_ratio=0.063215 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159483014e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281976832 dispatch_meta_bytes=341112 combine_payload_bytes=281976832 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=282317944 ingress_bytes=385785856 link_bytes=385785856 token_bytes=564294776 weight_movement_bytes=103809024 total_nvlink_bytes=668103800 weight_to_token_ratio=0.183962 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451681118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=336080 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267452624 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534569168 weight_movement_bytes=34603008 total_nvlink_bytes=569172176 weight_to_token_ratio=0.064731 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79484 payload_rows_remote=67290 dup_rows=18820 dup_rows_remote=17326 dedup_ratio=0.191447 router_tokens_per_expert=[0, 483, 1252, 1137, 970, 2289, 248, 1815, 33, 1593, 1718, 76, 132, 351, 245, 22, 1782, 0, 0, 46, 50, 1345, 937, 3, 1083, 0, 3404, 100, 3722, 188, 214, 0, 149, 0, 0, 819, 209, 0, 8, 697, 72, 751, 0, 1075, 155, 0, 4173, 0, 0, 1023, 107, 310, 407, 4962, 4349, 791, 673, 184, 15, 1481, 765, 400, 20, 319] payload_rows_per_dst=[8486, 7144, 9744, 8368, 12194, 10870, 11414, 11264] dup_rows_per_dst=[3324, 1196, 3160, 1610, 1494, 1582, 2560, 3894] combine_rows_per_src=[8486, 7144, 9744, 8368, 12194, 10870, 11414, 11264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76552 payload_rows_remote=64256 dup_rows=21752 dup_rows_remote=16368 dedup_ratio=0.221273 router_tokens_per_expert=[0, 472, 1287, 1075, 870, 2036, 239, 1785, 9, 1601, 1752, 92, 136, 356, 223, 20, 1800, 0, 0, 18, 31, 1367, 980, 6, 1097, 0, 3626, 91, 3847, 128, 153, 0, 165, 0, 0, 1006, 196, 0, 3, 711, 74, 776, 0, 1109, 151, 0, 4301, 0, 0, 1049, 98, 283, 409, 4834, 4651, 713, 434, 185, 13, 1588, 666, 351, 9, 280] payload_rows_per_dst=[8148, 12296, 9508, 12006, 12074, 11222, 5050, 6248] dup_rows_per_dst=[3308, 5384, 2968, 5878, 1756, 1600, 54, 804] combine_rows_per_src=[8148, 12296, 9508, 12006, 12074, 11222, 5050, 6248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77422 payload_rows_remote=67848 dup_rows=20882 dup_rows_remote=17932 dedup_ratio=0.212423 router_tokens_per_expert=[0, 848, 1298, 1094, 942, 2084, 313, 1734, 88, 1321, 1920, 61, 132, 333, 260, 65, 1576, 1, 0, 53, 55, 1325, 799, 18, 1188, 0, 3664, 116, 3289, 226, 262, 0, 139, 0, 0, 695, 249, 0, 50, 744, 117, 738, 0, 1066, 158, 0, 4160, 0, 0, 941, 147, 456, 411, 4507, 3940, 859, 712, 196, 24, 1818, 1025, 529, 74, 332] payload_rows_per_dst=[8452, 10666, 9308, 9574, 11396, 10896, 6470, 10660] dup_rows_per_dst=[4006, 3996, 2514, 2950, 1372, 1582, 736, 3726] combine_rows_per_src=[8452, 10666, 9308, 9574, 11396, 10896, 6470, 10660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69072 payload_rows_remote=57322 dup_rows=29232 dup_rows_remote=25742 dedup_ratio=0.297363 router_tokens_per_expert=[0, 486, 1193, 1383, 904, 1894, 254, 1801, 14, 1321, 1689, 196, 99, 217, 248, 17, 1472, 1, 0, 32, 43, 1356, 850, 3, 1189, 0, 3396, 116, 3980, 159, 127, 0, 167, 0, 0, 1030, 310, 0, 5, 794, 86, 727, 0, 1213, 147, 0, 4237, 0, 0, 890, 136, 411, 258, 5386, 4532, 743, 467, 101, 12, 1732, 700, 338, 7, 283] payload_rows_per_dst=[7624, 6682, 9634, 3124, 3952, 11340, 14966, 11750] dup_rows_per_dst=[2682, 920, 3404, 58, 660, 1906, 16112, 3490] combine_rows_per_src=[7624, 6682, 9634, 3124, 3952, 11340, 14966, 11750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80356 payload_rows_remote=68744 dup_rows=17948 dup_rows_remote=16314 dedup_ratio=0.182576 router_tokens_per_expert=[0, 441, 1219, 1181, 834, 2034, 247, 1786, 8, 1641, 1735, 67, 153, 339, 246, 14, 1533, 6, 0, 51, 49, 1283, 1002, 1, 965, 0, 3124, 108, 3919, 154, 184, 0, 153, 0, 0, 988, 288, 0, 4, 790, 79, 719, 0, 1165, 161, 0, 4499, 0, 0, 902, 120, 423, 362, 5569, 4444, 653, 449, 135, 19, 1469, 729, 385, 16, 307] payload_rows_per_dst=[8190, 7172, 9108, 7834, 13438, 11612, 11482, 11520] dup_rows_per_dst=[3226, 1234, 2810, 1236, 2146, 1634, 2326, 3336] combine_rows_per_src=[8190, 7172, 9108, 7834, 13438, 11612, 11482, 11520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77712 payload_rows_remote=65610 dup_rows=20592 dup_rows_remote=16236 dedup_ratio=0.209473 router_tokens_per_expert=[0, 627, 1244, 1426, 891, 2062, 254, 1843, 28, 1311, 1537, 73, 152, 461, 253, 41, 1816, 0, 0, 49, 62, 1087, 846, 15, 967, 0, 3886, 95, 3685, 199, 267, 0, 141, 0, 0, 799, 169, 0, 39, 687, 128, 857, 7, 1057, 149, 0, 4293, 0, 2, 1063, 112, 355, 405, 4539, 4349, 819, 455, 208, 20, 1711, 866, 449, 25, 271] payload_rows_per_dst=[8844, 6584, 9214, 8746, 9406, 11390, 12102, 11426] dup_rows_per_dst=[3726, 1128, 2660, 1602, 1094, 2072, 4356, 3954] combine_rows_per_src=[8844, 6584, 9214, 8746, 9406, 11390, 12102, 11426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76728 payload_rows_remote=67444 dup_rows=21576 dup_rows_remote=18850 dedup_ratio=0.219482 router_tokens_per_expert=[0, 540, 1304, 1067, 894, 2088, 264, 1726, 17, 1549, 1780, 94, 203, 305, 253, 21, 1669, 1, 0, 41, 41, 1245, 915, 5, 1060, 0, 3562, 103, 4110, 155, 211, 0, 126, 0, 0, 882, 208, 0, 9, 623, 68, 831, 0, 1047, 187, 0, 4179, 0, 0, 976, 99, 361, 401, 4904, 4486, 719, 543, 246, 6, 1717, 693, 327, 15, 276] payload_rows_per_dst=[8172, 12196, 9284, 12270, 11978, 10970, 5054, 6804] dup_rows_per_dst=[3418, 5220, 2726, 6132, 1526, 1654, 58, 842] combine_rows_per_src=[8172, 12196, 9284, 12270, 11978, 10970, 5054, 6804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275619840 dispatch_meta_bytes=338464 combine_payload_bytes=275619840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275958304 ingress_bytes=310222848 link_bytes=310222848 token_bytes=551578144 weight_movement_bytes=34603008 total_nvlink_bytes=586181152 weight_to_token_ratio=0.062735 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167343602e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263192576 dispatch_meta_bytes=322496 combine_payload_bytes=263192576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263515072 ingress_bytes=297795584 link_bytes=297795584 token_bytes=526707648 weight_movement_bytes=34603008 total_nvlink_bytes=561310656 weight_to_token_ratio=0.065697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120580808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75258 payload_rows_remote=64262 dup_rows=23046 dup_rows_remote=17344 dedup_ratio=0.234436 router_tokens_per_expert=[0, 556, 1224, 1320, 904, 1933, 260, 1664, 46, 1595, 1719, 65, 123, 330, 249, 26, 1810, 1, 1, 54, 63, 1359, 900, 10, 1280, 0, 3376, 141, 3291, 168, 201, 1, 174, 0, 2, 902, 334, 0, 39, 774, 108, 859, 4, 1120, 154, 0, 4162, 1, 0, 969, 162, 481, 412, 4684, 4090, 758, 754, 164, 24, 1480, 843, 645, 47, 336] payload_rows_per_dst=[10996, 11810, 9450, 11408, 11896, 7048, 5398, 7252] dup_rows_per_dst=[5702, 4676, 2812, 5508, 1922, 926, 166, 1334] combine_rows_per_src=[10996, 11810, 9450, 11408, 11896, 7048, 5398, 7252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=332256 combine_payload_bytes=234790912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235123168 ingress_bytes=269393920 link_bytes=269393920 token_bytes=469914080 weight_movement_bytes=34603008 total_nvlink_bytes=504517088 weight_to_token_ratio=0.073637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.013707633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277905408 dispatch_meta_bytes=343120 combine_payload_bytes=277905408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=278248528 ingress_bytes=381714432 link_bytes=381714432 token_bytes=556153936 weight_movement_bytes=103809024 total_nvlink_bytes=659962960 weight_to_token_ratio=0.186655 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436360677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=281575424 dispatch_meta_bytes=340232 combine_payload_bytes=281575424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281915656 ingress_bytes=350781440 link_bytes=350781440 token_bytes=563491080 weight_movement_bytes=69206016 total_nvlink_bytes=632697096 weight_to_token_ratio=0.122817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319962318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268738560 dispatch_meta_bytes=327384 combine_payload_bytes=268738560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269065944 ingress_bytes=372547584 link_bytes=372547584 token_bytes=537804504 weight_movement_bytes=103809024 total_nvlink_bytes=641613528 weight_to_token_ratio=0.193024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401866566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276250624 dispatch_meta_bytes=345176 combine_payload_bytes=276250624 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276595800 ingress_bytes=345456640 link_bytes=345456640 token_bytes=552846424 weight_movement_bytes=69206016 total_nvlink_bytes=622052440 weight_to_token_ratio=0.125181 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299925525e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263217152 dispatch_meta_bytes=326424 combine_payload_bytes=263217152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263543576 ingress_bytes=367026176 link_bytes=367026176 token_bytes=526760728 weight_movement_bytes=103809024 total_nvlink_bytes=630569752 weight_to_token_ratio=0.197071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381089952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79822 payload_rows_remote=69410 dup_rows=18482 dup_rows_remote=17668 dedup_ratio=0.188009 router_tokens_per_expert=[166, 1849, 423, 1373, 69, 211, 1522, 515, 0, 0, 27, 589, 2655, 0, 0, 560, 348, 725, 6252, 0, 1, 532, 1235, 0, 0, 4507, 1105, 0, 2, 0, 2730, 0, 4390, 404, 13, 4, 194, 0, 476, 132, 1, 0, 0, 367, 42, 169, 3329, 1216, 636, 1573, 1060, 978, 8, 4, 0, 927, 803, 258, 282, 61, 108, 263, 209, 3849] payload_rows_per_dst=[9690, 7122, 14854, 9216, 10412, 11486, 7488, 9554] dup_rows_per_dst=[2566, 540, 3332, 1782, 814, 4452, 2884, 2112] combine_rows_per_src=[9690, 7122, 14854, 9216, 10412, 11486, 7488, 9554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=63300 dup_rows=20392 dup_rows_remote=14382 dedup_ratio=0.207438 router_tokens_per_expert=[169, 1862, 420, 1436, 92, 168, 1363, 490, 0, 0, 11, 588, 3047, 0, 0, 551, 350, 771, 6114, 0, 2, 675, 1500, 0, 0, 4392, 1198, 0, 2, 0, 2678, 0, 4303, 431, 45, 6, 120, 0, 431, 163, 0, 0, 0, 341, 5, 181, 3156, 1214, 557, 1432, 1163, 1038, 0, 0, 0, 872, 794, 228, 342, 37, 106, 199, 197, 3912] payload_rows_per_dst=[9600, 14612, 6286, 12054, 10204, 8366, 7248, 9542] dup_rows_per_dst=[2400, 6010, 310, 4486, 794, 1428, 2876, 2088] combine_rows_per_src=[9600, 14612, 6286, 12054, 10204, 8366, 7248, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76602 payload_rows_remote=65432 dup_rows=21702 dup_rows_remote=18018 dedup_ratio=0.220764 router_tokens_per_expert=[286, 1615, 352, 1632, 72, 127, 1324, 765, 0, 0, 10, 482, 2960, 0, 0, 707, 337, 681, 6257, 1, 0, 714, 1570, 0, 0, 4230, 1304, 0, 2, 0, 2345, 0, 4277, 329, 39, 0, 95, 0, 377, 136, 0, 0, 0, 389, 12, 157, 3301, 1314, 635, 1281, 1219, 1093, 0, 0, 0, 913, 736, 194, 241, 26, 97, 254, 195, 4069] payload_rows_per_dst=[7688, 7768, 6312, 6920, 15052, 8892, 12800, 11170] dup_rows_per_dst=[1428, 550, 294, 382, 7968, 1454, 5942, 3684] combine_rows_per_src=[7688, 7768, 6312, 6920, 15052, 8892, 12800, 11170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79218 payload_rows_remote=67442 dup_rows=19086 dup_rows_remote=13432 dedup_ratio=0.194153 router_tokens_per_expert=[142, 1828, 402, 1436, 75, 223, 1543, 436, 0, 0, 17, 562, 2821, 0, 0, 717, 337, 734, 6330, 0, 0, 587, 1238, 0, 0, 4705, 1168, 0, 3, 0, 2531, 1, 4363, 388, 48, 8, 158, 0, 438, 124, 1, 0, 0, 340, 23, 203, 3421, 1296, 507, 1353, 1409, 983, 2, 0, 0, 933, 674, 237, 114, 56, 84, 187, 166, 3800] payload_rows_per_dst=[7742, 7338, 13502, 8054, 10240, 10136, 11776, 10430] dup_rows_per_dst=[1350, 896, 3220, 1062, 834, 2786, 5654, 3284] combine_rows_per_src=[7742, 7338, 13502, 8054, 10240, 10136, 11776, 10430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79680 payload_rows_remote=66488 dup_rows=18624 dup_rows_remote=13070 dedup_ratio=0.189453 router_tokens_per_expert=[271, 1552, 382, 1898, 60, 141, 1545, 613, 0, 0, 33, 584, 2839, 0, 0, 621, 305, 745, 6290, 0, 1, 622, 1429, 0, 0, 4131, 1378, 0, 10, 0, 2836, 0, 4021, 404, 5, 2, 102, 0, 342, 148, 0, 0, 0, 434, 41, 121, 3260, 1386, 559, 1277, 1122, 1143, 0, 0, 0, 905, 634, 234, 360, 37, 79, 193, 156, 3901] payload_rows_per_dst=[10070, 7664, 14904, 7812, 9498, 13192, 7216, 9324] dup_rows_per_dst=[2854, 490, 3880, 636, 550, 5554, 2796, 1864] combine_rows_per_src=[10070, 7664, 14904, 7812, 9498, 13192, 7216, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79882 payload_rows_remote=67534 dup_rows=18422 dup_rows_remote=13950 dedup_ratio=0.187398 router_tokens_per_expert=[192, 1944, 416, 1406, 66, 234, 1812, 463, 0, 0, 50, 649, 2433, 0, 0, 414, 347, 789, 6198, 2, 1, 489, 1109, 0, 0, 4510, 1176, 0, 9, 0, 2715, 0, 4190, 442, 65, 4, 164, 0, 473, 141, 3, 0, 0, 346, 29, 189, 3200, 1102, 644, 1556, 1298, 1140, 5, 29, 0, 987, 727, 232, 441, 135, 107, 296, 275, 3508] payload_rows_per_dst=[10042, 6888, 14718, 12348, 9980, 8336, 8082, 9488] dup_rows_per_dst=[3024, 204, 3152, 4472, 978, 1402, 3236, 1954] combine_rows_per_src=[10042, 6888, 14718, 12348, 9980, 8336, 8082, 9488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78864 payload_rows_remote=71964 dup_rows=19440 dup_rows_remote=18634 dedup_ratio=0.197754 router_tokens_per_expert=[192, 1800, 440, 1567, 44, 271, 1495, 417, 0, 0, 41, 596, 2642, 0, 0, 573, 362, 731, 6321, 2, 0, 689, 1432, 0, 0, 4191, 1371, 0, 4, 0, 2472, 0, 4281, 347, 43, 0, 149, 0, 366, 133, 1, 0, 0, 340, 10, 181, 3430, 1195, 593, 1435, 1102, 1052, 0, 0, 0, 969, 810, 224, 273, 46, 63, 283, 175, 3998] payload_rows_per_dst=[9960, 14000, 6900, 12038, 9944, 8780, 7532, 9710] dup_rows_per_dst=[2492, 5072, 806, 4038, 694, 1534, 2770, 2034] combine_rows_per_src=[9960, 14000, 6900, 12038, 9944, 8780, 7532, 9710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284303360 dispatch_meta_bytes=348312 combine_payload_bytes=284303360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284651672 ingress_bytes=318906368 link_bytes=318906368 token_bytes=568955032 weight_movement_bytes=34603008 total_nvlink_bytes=603558040 weight_to_token_ratio=0.060819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200018989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78628 payload_rows_remote=66492 dup_rows=19676 dup_rows_remote=14414 dedup_ratio=0.200155 router_tokens_per_expert=[286, 1990, 440, 1691, 69, 225, 1604, 566, 1, 1, 31, 570, 2420, 0, 1, 556, 348, 723, 5995, 3, 2, 545, 1296, 0, 1, 4582, 1130, 0, 7, 1, 2609, 0, 3894, 383, 60, 10, 123, 1, 524, 176, 0, 1, 0, 405, 38, 200, 3529, 1080, 662, 1409, 957, 1003, 2, 0, 0, 969, 816, 234, 410, 103, 91, 318, 222, 3839] payload_rows_per_dst=[12136, 14242, 5588, 10998, 9478, 9074, 7246, 9866] dup_rows_per_dst=[5262, 4908, 246, 2006, 864, 1432, 2758, 2200] combine_rows_per_src=[12136, 14242, 5588, 10998, 9478, 9074, 7246, 9866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259276800 dispatch_meta_bytes=310728 combine_payload_bytes=259276800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259587528 ingress_bytes=293879808 link_bytes=293879808 token_bytes=518864328 weight_movement_bytes=34603008 total_nvlink_bytes=553467336 weight_to_token_ratio=0.066690 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.105846058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=333800 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268343272 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536352744 weight_movement_bytes=34603008 total_nvlink_bytes=570955752 weight_to_token_ratio=0.064515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276242432 dispatch_meta_bytes=323496 combine_payload_bytes=276242432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276565928 ingress_bytes=310845440 link_bytes=310845440 token_bytes=552808360 weight_movement_bytes=34603008 total_nvlink_bytes=587411368 weight_to_token_ratio=0.062595 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169686366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272334848 dispatch_meta_bytes=318232 combine_payload_bytes=272334848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272653080 ingress_bytes=306937856 link_bytes=306937856 token_bytes=544987928 weight_movement_bytes=34603008 total_nvlink_bytes=579590936 weight_to_token_ratio=0.063493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154982442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276619264 dispatch_meta_bytes=325936 combine_payload_bytes=276619264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=276945200 ingress_bytes=415031296 link_bytes=415031296 token_bytes=553564464 weight_movement_bytes=138412032 total_nvlink_bytes=691976496 weight_to_token_ratio=0.250038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.561729354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=294764544 dispatch_meta_bytes=362392 combine_payload_bytes=294764544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=295126936 ingress_bytes=398573568 link_bytes=398573568 token_bytes=589891480 weight_movement_bytes=103809024 total_nvlink_bytes=693700504 weight_to_token_ratio=0.175980 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499800249e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272351232 dispatch_meta_bytes=323624 combine_payload_bytes=272351232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272674856 ingress_bytes=341557248 link_bytes=341557248 token_bytes=545026088 weight_movement_bytes=69206016 total_nvlink_bytes=614232104 weight_to_token_ratio=0.126977 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285252427e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88308 payload_rows_remote=75670 dup_rows=9996 dup_rows_remote=8070 dedup_ratio=0.101685 router_tokens_per_expert=[1275, 83, 0, 5358, 0, 0, 215, 5, 0, 181, 127, 34, 860, 385, 387, 375, 0, 0, 148, 0, 0, 452, 415, 529, 0, 69, 414, 0, 64, 75, 0, 0, 69, 5850, 271, 0, 0, 0, 0, 1092, 785, 589, 0, 680, 4854, 75, 356, 4770, 387, 75, 0, 1139, 0, 5335, 6800, 0, 3565, 0, 0, 0, 0, 0, 1009, 0] payload_rows_per_dst=[11762, 13606, 11488, 14016, 12638, 13526, 2428, 8844] dup_rows_per_dst=[2110, 1762, 1140, 828, 1926, 1152, 774, 304] combine_rows_per_src=[11762, 13606, 11488, 14016, 12638, 13526, 2428, 8844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87992 payload_rows_remote=73828 dup_rows=10312 dup_rows_remote=8100 dedup_ratio=0.104899 router_tokens_per_expert=[1113, 63, 0, 5261, 0, 0, 178, 0, 0, 204, 141, 1, 887, 378, 294, 316, 0, 0, 154, 0, 0, 496, 403, 533, 0, 92, 355, 0, 78, 71, 0, 0, 76, 5895, 302, 0, 0, 0, 0, 1168, 792, 599, 1, 484, 4709, 74, 368, 4661, 421, 71, 0, 920, 3, 5967, 6781, 0, 3681, 0, 0, 0, 0, 0, 1159, 2] payload_rows_per_dst=[11460, 14164, 11324, 14044, 12704, 12940, 2322, 9034] dup_rows_per_dst=[1770, 2212, 1170, 710, 2178, 1114, 508, 650] combine_rows_per_src=[11460, 14164, 11324, 14044, 12704, 12940, 2322, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62350 payload_rows_remote=47570 dup_rows=35954 dup_rows_remote=29124 dedup_ratio=0.365743 router_tokens_per_expert=[1141, 49, 0, 5251, 0, 0, 148, 1, 0, 175, 216, 1, 836, 369, 253, 321, 0, 0, 191, 0, 0, 525, 404, 400, 0, 72, 422, 0, 170, 95, 0, 0, 57, 5557, 218, 0, 0, 0, 0, 943, 728, 583, 0, 532, 4336, 96, 353, 5290, 326, 95, 0, 753, 4, 6198, 6795, 0, 3534, 0, 0, 0, 0, 0, 1713, 1] payload_rows_per_dst=[4082, 3824, 7558, 1518, 5914, 8888, 15786, 14780] dup_rows_per_dst=[572, 518, 744, 0, 976, 5086, 21228, 6830] combine_rows_per_src=[4082, 3824, 7558, 1518, 5914, 8888, 15786, 14780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72802 payload_rows_remote=57528 dup_rows=25502 dup_rows_remote=10394 dedup_ratio=0.259420 router_tokens_per_expert=[1107, 86, 0, 5247, 0, 0, 179, 0, 0, 222, 108, 11, 818, 370, 355, 404, 0, 0, 204, 0, 0, 499, 401, 502, 0, 75, 258, 0, 48, 65, 0, 0, 92, 5826, 308, 0, 0, 0, 0, 1285, 772, 537, 4, 537, 4886, 65, 371, 4770, 376, 63, 0, 715, 0, 6275, 6711, 0, 3572, 0, 0, 0, 0, 0, 1027, 1] payload_rows_per_dst=[11516, 3852, 11484, 8280, 4834, 4094, 15274, 13468] dup_rows_per_dst=[1722, 724, 1268, 282, 1050, 478, 15108, 4870] combine_rows_per_src=[11516, 3852, 11484, 8280, 4834, 4094, 15274, 13468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84510 payload_rows_remote=78266 dup_rows=13794 dup_rows_remote=13038 dedup_ratio=0.140320 router_tokens_per_expert=[1193, 67, 0, 5113, 0, 0, 152, 0, 0, 194, 227, 0, 852, 307, 203, 312, 0, 0, 140, 0, 0, 511, 341, 355, 0, 60, 388, 0, 154, 90, 0, 0, 87, 5595, 180, 0, 0, 0, 0, 1099, 768, 396, 0, 620, 4636, 90, 330, 5026, 392, 90, 0, 989, 17, 6118, 6723, 0, 3569, 0, 0, 0, 0, 0, 1768, 0] payload_rows_per_dst=[11254, 9142, 11542, 14054, 12304, 6244, 10314, 9656] dup_rows_per_dst=[1796, 1194, 1204, 776, 1618, 756, 5432, 1018] combine_rows_per_src=[11254, 9142, 11542, 14054, 12304, 6244, 10314, 9656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88462 payload_rows_remote=76330 dup_rows=9842 dup_rows_remote=8050 dedup_ratio=0.100118 router_tokens_per_expert=[1122, 120, 0, 5229, 0, 0, 97, 0, 0, 199, 162, 0, 829, 398, 297, 492, 0, 0, 402, 0, 0, 515, 443, 554, 0, 44, 151, 0, 72, 82, 0, 0, 113, 5497, 262, 0, 0, 0, 0, 951, 759, 579, 0, 517, 4954, 82, 464, 5048, 340, 82, 0, 909, 0, 5840, 6757, 0, 3595, 0, 0, 0, 0, 0, 1195, 0] payload_rows_per_dst=[11598, 14264, 12132, 13910, 12098, 13194, 2282, 8984] dup_rows_per_dst=[1538, 2170, 1792, 302, 1548, 1516, 380, 596] combine_rows_per_src=[11598, 14264, 12132, 13910, 12098, 13194, 2282, 8984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=334960 combine_payload_bytes=309944320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310279280 ingress_bytes=379150336 link_bytes=379150336 token_bytes=620223600 weight_movement_bytes=69206016 total_nvlink_bytes=689429616 weight_to_token_ratio=0.111582 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426712190e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88230 payload_rows_remote=74250 dup_rows=10074 dup_rows_remote=9384 dedup_ratio=0.102478 router_tokens_per_expert=[1106, 99, 0, 5163, 0, 0, 181, 15, 0, 207, 131, 103, 879, 378, 402, 476, 0, 0, 137, 0, 0, 464, 414, 567, 1, 66, 344, 0, 78, 78, 0, 0, 79, 5588, 286, 0, 0, 0, 0, 1153, 814, 579, 1, 580, 4851, 69, 415, 4836, 402, 69, 0, 1178, 0, 5409, 6768, 0, 3772, 0, 0, 0, 0, 0, 1013, 1] payload_rows_per_dst=[11400, 13938, 11656, 13980, 12284, 13310, 2552, 9110] dup_rows_per_dst=[1728, 2032, 1180, 690, 1928, 1308, 746, 462] combine_rows_per_src=[11400, 13938, 11656, 13980, 12284, 13310, 2552, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87936 payload_rows_remote=76024 dup_rows=10368 dup_rows_remote=8284 dedup_ratio=0.105469 router_tokens_per_expert=[1343, 79, 0, 5347, 0, 1, 171, 57, 1, 201, 194, 58, 839, 414, 358, 383, 0, 0, 127, 0, 0, 467, 434, 605, 0, 70, 422, 0, 144, 92, 0, 0, 69, 5695, 339, 0, 1, 1, 0, 1047, 776, 605, 19, 883, 4594, 93, 379, 4678, 361, 91, 1, 1130, 1, 5003, 6729, 1, 3886, 0, 0, 0, 2, 0, 961, 0] payload_rows_per_dst=[11912, 13318, 11330, 14064, 12388, 13082, 2534, 9308] dup_rows_per_dst=[2084, 1584, 1292, 850, 1916, 1616, 636, 390] combine_rows_per_src=[11912, 13318, 11330, 14064, 12388, 13082, 2534, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=302399488 dispatch_meta_bytes=327712 combine_payload_bytes=302399488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302727200 ingress_bytes=337002496 link_bytes=337002496 token_bytes=605126688 weight_movement_bytes=34603008 total_nvlink_bytes=639729696 weight_to_token_ratio=0.057183 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268113262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=194846720 dispatch_meta_bytes=306776 combine_payload_bytes=194846720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=195153496 ingress_bytes=229449728 link_bytes=229449728 token_bytes=390000216 weight_movement_bytes=34603008 total_nvlink_bytes=424603224 weight_to_token_ratio=0.088726 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.634008542e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=235634688 dispatch_meta_bytes=271688 combine_payload_bytes=235634688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=235906376 ingress_bytes=339443712 link_bytes=339443712 token_bytes=471541064 weight_movement_bytes=103809024 total_nvlink_bytes=575350088 weight_to_token_ratio=0.220148 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277299361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=320577536 dispatch_meta_bytes=365216 combine_payload_bytes=320577536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320942752 ingress_bytes=424386560 link_bytes=424386560 token_bytes=641520288 weight_movement_bytes=103809024 total_nvlink_bytes=745329312 weight_to_token_ratio=0.161817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596932459e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312647680 dispatch_meta_bytes=337520 combine_payload_bytes=312647680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312985200 ingress_bytes=347250688 link_bytes=347250688 token_bytes=625632880 weight_movement_bytes=34603008 total_nvlink_bytes=660235888 weight_to_token_ratio=0.055309 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306676383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304128000 dispatch_meta_bytes=334536 combine_payload_bytes=304128000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304462536 ingress_bytes=338731008 link_bytes=338731008 token_bytes=608590536 weight_movement_bytes=34603008 total_nvlink_bytes=643193544 weight_to_token_ratio=0.056858 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274617513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311394304 dispatch_meta_bytes=337232 combine_payload_bytes=311394304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=311731536 ingress_bytes=380600320 link_bytes=380600320 token_bytes=623125840 weight_movement_bytes=69206016 total_nvlink_bytes=692331856 weight_to_token_ratio=0.111063 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432168363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86210 payload_rows_remote=71418 dup_rows=12094 dup_rows_remote=9550 dedup_ratio=0.123027 router_tokens_per_expert=[786, 7521, 0, 0, 331, 0, 63, 0, 0, 350, 63, 2849, 0, 58, 513, 1658, 69, 2596, 0, 0, 267, 63, 11, 498, 6572, 4944, 765, 372, 0, 0, 9, 42, 0, 384, 294, 0, 82, 821, 509, 6, 562, 0, 0, 69, 0, 4066, 0, 0, 5442, 0, 9, 3869, 462, 73, 4, 0, 0, 0, 0, 0, 0, 1622, 270, 208] payload_rows_per_dst=[15318, 9564, 5972, 11082, 14792, 8274, 8680, 12528] dup_rows_per_dst=[2084, 1418, 1036, 1182, 2544, 1120, 154, 2556] combine_rows_per_src=[15318, 9564, 5972, 11082, 14792, 8274, 8680, 12528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77708 payload_rows_remote=66296 dup_rows=20596 dup_rows_remote=16502 dedup_ratio=0.209513 router_tokens_per_expert=[936, 7490, 0, 0, 324, 0, 60, 0, 0, 349, 60, 2852, 0, 29, 530, 1680, 69, 2512, 0, 0, 231, 60, 55, 533, 6411, 4952, 642, 377, 0, 0, 0, 50, 0, 357, 255, 0, 84, 1241, 469, 11, 431, 0, 0, 92, 0, 3714, 0, 0, 5435, 0, 1, 3753, 491, 46, 0, 0, 0, 0, 0, 0, 0, 1951, 372, 247] payload_rows_per_dst=[15248, 11412, 5720, 14814, 4430, 7616, 13426, 5042] dup_rows_per_dst=[2372, 4094, 1200, 5544, 404, 858, 6026, 98] combine_rows_per_src=[15248, 11412, 5720, 14814, 4430, 7616, 13426, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78126 payload_rows_remote=65692 dup_rows=20178 dup_rows_remote=16742 dedup_ratio=0.205261 router_tokens_per_expert=[959, 7575, 0, 0, 370, 0, 42, 0, 0, 290, 42, 2919, 0, 57, 557, 1158, 56, 2779, 0, 0, 195, 42, 1, 384, 6664, 4715, 582, 436, 0, 0, 0, 45, 0, 375, 197, 0, 100, 1007, 524, 45, 512, 0, 0, 72, 0, 3713, 1, 0, 4971, 0, 2, 4289, 419, 93, 0, 0, 0, 0, 0, 0, 0, 2232, 391, 341] payload_rows_per_dst=[2234, 8646, 15770, 2438, 12446, 13466, 10692, 12434] dup_rows_per_dst=[508, 1400, 5928, 54, 2384, 4560, 1908, 3436] combine_rows_per_src=[2234, 8646, 15770, 2438, 12446, 13466, 10692, 12434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79126 payload_rows_remote=70852 dup_rows=19178 dup_rows_remote=19048 dedup_ratio=0.195089 router_tokens_per_expert=[851, 7535, 0, 0, 334, 0, 60, 0, 0, 306, 60, 2494, 0, 67, 546, 1611, 69, 2493, 0, 0, 219, 60, 50, 453, 6574, 4947, 696, 439, 0, 0, 0, 48, 0, 389, 226, 0, 69, 1174, 446, 30, 481, 0, 0, 75, 0, 4063, 0, 0, 5578, 0, 8, 3697, 437, 60, 0, 0, 0, 0, 0, 0, 0, 1932, 387, 188] payload_rows_per_dst=[1982, 8768, 16126, 2354, 14770, 13996, 8274, 12856] dup_rows_per_dst=[508, 1400, 5632, 12, 3046, 5136, 130, 3314] combine_rows_per_src=[1982, 8768, 16126, 2354, 14770, 13996, 8274, 12856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80514 payload_rows_remote=68788 dup_rows=17790 dup_rows_remote=14468 dedup_ratio=0.180969 router_tokens_per_expert=[852, 7467, 0, 0, 348, 0, 66, 0, 0, 300, 66, 2620, 0, 84, 577, 1291, 87, 2997, 0, 0, 211, 66, 4, 467, 6651, 4761, 813, 445, 0, 0, 4, 22, 0, 453, 221, 0, 97, 848, 527, 57, 496, 0, 0, 60, 0, 4198, 0, 0, 5501, 0, 10, 3414, 479, 63, 0, 0, 0, 0, 0, 0, 0, 1735, 483, 311] payload_rows_per_dst=[4208, 8550, 14534, 5984, 14816, 11726, 7748, 12948] dup_rows_per_dst=[830, 1326, 5558, 566, 2892, 3322, 184, 3112] combine_rows_per_src=[4208, 8550, 14534, 5984, 14816, 11726, 7748, 12948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79002 payload_rows_remote=72720 dup_rows=19302 dup_rows_remote=18264 dedup_ratio=0.196350 router_tokens_per_expert=[848, 7485, 0, 1, 360, 0, 90, 0, 0, 325, 90, 2651, 0, 58, 590, 1677, 111, 2691, 0, 0, 299, 90, 55, 414, 6611, 4883, 657, 445, 1, 0, 0, 40, 0, 368, 249, 0, 89, 1268, 490, 21, 435, 0, 0, 44, 0, 4045, 0, 0, 5218, 0, 0, 3581, 421, 82, 0, 0, 0, 0, 0, 0, 0, 1774, 400, 195] payload_rows_per_dst=[15318, 9380, 6282, 15908, 4596, 8180, 10524, 8814] dup_rows_per_dst=[2250, 1402, 1038, 9366, 374, 868, 2612, 1392] combine_rows_per_src=[15318, 9380, 6282, 15908, 4596, 8180, 10524, 8814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=292528128 dispatch_meta_bytes=323872 combine_payload_bytes=292528128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292852000 ingress_bytes=327131136 link_bytes=327131136 token_bytes=585380128 weight_movement_bytes=34603008 total_nvlink_bytes=619983136 weight_to_token_ratio=0.059112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230968129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85404 payload_rows_remote=73796 dup_rows=12900 dup_rows_remote=11052 dedup_ratio=0.131226 router_tokens_per_expert=[840, 7505, 0, 0, 361, 0, 53, 0, 0, 302, 53, 2445, 0, 57, 513, 1783, 76, 2478, 0, 0, 293, 53, 40, 501, 6496, 5119, 795, 463, 0, 0, 13, 29, 0, 310, 214, 0, 75, 887, 509, 61, 599, 0, 0, 66, 3, 4292, 0, 0, 5737, 0, 19, 3626, 471, 69, 0, 0, 0, 0, 0, 0, 0, 1355, 378, 213] payload_rows_per_dst=[15254, 8910, 5794, 11608, 14102, 8722, 8220, 12794] dup_rows_per_dst=[2264, 1396, 1088, 1848, 2384, 1198, 150, 2572] combine_rows_per_src=[15254, 8910, 5794, 11608, 14102, 8722, 8220, 12794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78518 payload_rows_remote=63174 dup_rows=19786 dup_rows_remote=17204 dedup_ratio=0.201274 router_tokens_per_expert=[1009, 7523, 0, 1, 373, 0, 57, 0, 0, 317, 54, 2865, 1, 61, 529, 1762, 69, 2328, 1, 2, 296, 54, 14, 549, 6541, 4730, 713, 418, 0, 1, 55, 44, 2, 456, 285, 0, 109, 808, 501, 1, 613, 0, 0, 69, 2, 3946, 0, 1, 5562, 1, 9, 3674, 451, 91, 0, 0, 0, 1, 2, 0, 0, 1556, 345, 300] payload_rows_per_dst=[15344, 13664, 5552, 13912, 4156, 8052, 13536, 4302] dup_rows_per_dst=[2582, 6974, 1074, 1632, 168, 1210, 6040, 106] combine_rows_per_src=[15344, 13664, 5552, 13912, 4156, 8052, 13536, 4302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=331192 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271879608 ingress_bytes=306151424 link_bytes=306151424 token_bytes=543428024 weight_movement_bytes=34603008 total_nvlink_bytes=578031032 weight_to_token_ratio=0.063675 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152023161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=269074432 dispatch_meta_bytes=329736 combine_payload_bytes=269074432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269404168 ingress_bytes=303677440 link_bytes=303677440 token_bytes=538478600 weight_movement_bytes=34603008 total_nvlink_bytes=573081608 weight_to_token_ratio=0.064261 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142713759e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=290209792 dispatch_meta_bytes=359600 combine_payload_bytes=290209792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290569392 ingress_bytes=359415808 link_bytes=359415808 token_bytes=580779184 weight_movement_bytes=69206016 total_nvlink_bytes=649985200 weight_to_token_ratio=0.119161 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352452750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281755648 dispatch_meta_bytes=333024 combine_payload_bytes=281755648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282088672 ingress_bytes=316358656 link_bytes=316358656 token_bytes=563844320 weight_movement_bytes=34603008 total_nvlink_bytes=598447328 weight_to_token_ratio=0.061370 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190432154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=297861120 dispatch_meta_bytes=363936 combine_payload_bytes=297861120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298225056 ingress_bytes=332464128 link_bytes=332464128 token_bytes=596086176 weight_movement_bytes=34603008 total_nvlink_bytes=630689184 weight_to_token_ratio=0.058050 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251035748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=302268416 dispatch_meta_bytes=339392 combine_payload_bytes=302268416 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=302607808 ingress_bytes=475283456 link_bytes=475283456 token_bytes=604876224 weight_movement_bytes=173015040 total_nvlink_bytes=777891264 weight_to_token_ratio=0.286034 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.788453381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=258760704 dispatch_meta_bytes=321512 combine_payload_bytes=258760704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=259082216 ingress_bytes=327966720 link_bytes=327966720 token_bytes=517842920 weight_movement_bytes=69206016 total_nvlink_bytes=587048936 weight_to_token_ratio=0.133643 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234112364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85160 payload_rows_remote=79496 dup_rows=13144 dup_rows_remote=11858 dedup_ratio=0.133708 router_tokens_per_expert=[600, 75, 981, 289, 7141, 0, 6992, 4752, 0, 149, 0, 652, 0, 202, 0, 3677, 245, 9, 3586, 0, 477, 52, 14, 69, 1, 523, 77, 0, 0, 139, 0, 3, 5772, 0, 0, 75, 120, 1112, 2168, 0, 144, 0, 0, 0, 0, 0, 0, 10, 1145, 0, 0, 0, 2, 832, 435, 0, 4191, 456, 0, 458, 1463, 64, 0, 0] payload_rows_per_dst=[3736, 12478, 9452, 15110, 5664, 14588, 13630, 10502] dup_rows_per_dst=[154, 4032, 1806, 360, 1286, 2, 2742, 2762] combine_rows_per_src=[3736, 12478, 9452, 15110, 5664, 14588, 13630, 10502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76114 payload_rows_remote=66016 dup_rows=22190 dup_rows_remote=21458 dedup_ratio=0.225728 router_tokens_per_expert=[591, 71, 1151, 199, 6914, 0, 6893, 4569, 0, 145, 0, 729, 0, 225, 0, 4316, 238, 0, 2927, 0, 449, 46, 12, 76, 5, 517, 74, 0, 0, 134, 0, 3, 5859, 0, 0, 74, 93, 1048, 2219, 0, 147, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 638, 477, 0, 4502, 466, 0, 480, 1728, 81, 0, 0] payload_rows_per_dst=[15094, 10098, 6856, 4494, 14130, 12258, 3636, 9548] dup_rows_per_dst=[10530, 732, 640, 160, 6960, 0, 706, 2462] combine_rows_per_src=[15094, 10098, 6856, 4494, 14130, 12258, 3636, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85190 payload_rows_remote=73866 dup_rows=13114 dup_rows_remote=9864 dedup_ratio=0.133403 router_tokens_per_expert=[698, 95, 790, 204, 6889, 0, 6812, 4557, 0, 190, 0, 867, 0, 322, 0, 3769, 253, 1, 3156, 0, 530, 33, 19, 57, 27, 478, 96, 0, 0, 138, 0, 0, 5744, 0, 0, 96, 108, 999, 2533, 0, 153, 0, 0, 0, 0, 0, 0, 3, 1099, 0, 0, 0, 0, 718, 431, 0, 4352, 410, 0, 556, 1795, 174, 0, 0] payload_rows_per_dst=[3426, 9354, 12684, 14722, 6084, 14088, 13508, 11324] dup_rows_per_dst=[148, 942, 4528, 380, 1388, 2, 2476, 3250] combine_rows_per_src=[3426, 9354, 12684, 14722, 6084, 14088, 13508, 11324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85800 payload_rows_remote=71970 dup_rows=12504 dup_rows_remote=10066 dedup_ratio=0.127197 router_tokens_per_expert=[607, 63, 1289, 245, 6987, 0, 7076, 4813, 0, 125, 0, 597, 0, 200, 0, 3472, 254, 83, 3470, 0, 434, 44, 77, 92, 5, 513, 65, 0, 0, 125, 0, 0, 5904, 0, 0, 65, 95, 944, 2597, 0, 155, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 2, 713, 347, 0, 4194, 346, 0, 469, 1448, 69, 0, 0] payload_rows_per_dst=[4138, 8318, 13286, 15128, 6176, 14284, 13830, 10640] dup_rows_per_dst=[270, 470, 5248, 440, 1226, 0, 2438, 2412] combine_rows_per_src=[4138, 8318, 13286, 15128, 6176, 14284, 13830, 10640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84998 payload_rows_remote=70984 dup_rows=13306 dup_rows_remote=13306 dedup_ratio=0.135356 router_tokens_per_expert=[599, 90, 1003, 150, 6829, 0, 6746, 4409, 0, 176, 0, 790, 0, 336, 0, 3787, 249, 0, 3270, 0, 443, 49, 20, 87, 57, 550, 90, 0, 0, 157, 0, 0, 5753, 0, 0, 90, 114, 980, 2595, 0, 177, 0, 0, 0, 0, 0, 0, 1, 1114, 0, 0, 0, 0, 717, 403, 0, 4407, 474, 0, 645, 1634, 161, 0, 0] payload_rows_per_dst=[3492, 9246, 12468, 14746, 6248, 14014, 13442, 11342] dup_rows_per_dst=[192, 932, 4586, 454, 1310, 0, 2532, 3300] combine_rows_per_src=[3492, 9246, 12468, 14746, 6248, 14014, 13442, 11342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85514 payload_rows_remote=78420 dup_rows=12790 dup_rows_remote=11962 dedup_ratio=0.130107 router_tokens_per_expert=[605, 82, 923, 289, 7037, 0, 7028, 4504, 0, 164, 0, 713, 0, 258, 0, 4306, 264, 0, 2968, 0, 552, 43, 21, 113, 4, 428, 82, 0, 0, 172, 0, 0, 5808, 0, 0, 82, 141, 1012, 2555, 0, 195, 0, 0, 0, 0, 0, 0, 0, 1071, 0, 0, 0, 3, 674, 447, 0, 4025, 397, 0, 494, 1620, 72, 0, 0] payload_rows_per_dst=[8748, 11404, 7094, 15064, 10664, 14464, 7320, 10756] dup_rows_per_dst=[1070, 2466, 828, 364, 4128, 0, 1474, 2460] combine_rows_per_src=[8748, 11404, 7094, 15064, 10664, 14464, 7320, 10756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=325615616 dispatch_meta_bytes=365416 combine_payload_bytes=325615616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325981032 ingress_bytes=394821632 link_bytes=394821632 token_bytes=651596648 weight_movement_bytes=69206016 total_nvlink_bytes=720802664 weight_to_token_ratio=0.106210 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485682015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86510 payload_rows_remote=71528 dup_rows=11794 dup_rows_remote=11478 dedup_ratio=0.119975 router_tokens_per_expert=[586, 69, 1401, 311, 6957, 0, 6955, 4924, 0, 138, 0, 654, 0, 226, 0, 3136, 260, 208, 3651, 0, 857, 42, 0, 79, 9, 483, 71, 0, 0, 122, 0, 9, 5720, 0, 0, 69, 151, 769, 2389, 0, 148, 0, 0, 0, 0, 0, 0, 22, 1214, 0, 0, 0, 1, 833, 228, 0, 4304, 469, 0, 366, 1242, 79, 0, 0] payload_rows_per_dst=[4458, 13482, 9358, 14982, 5492, 14254, 13564, 10920] dup_rows_per_dst=[276, 4674, 836, 316, 1264, 0, 2428, 2000] combine_rows_per_src=[4458, 13482, 9358, 14982, 5492, 14254, 13564, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57988 payload_rows_remote=41780 dup_rows=40316 dup_rows_remote=13952 dedup_ratio=0.410116 router_tokens_per_expert=[617, 91, 1656, 245, 6994, 1, 6825, 4670, 1, 181, 2, 719, 3, 300, 0, 3632, 251, 17, 3295, 0, 587, 51, 47, 69, 5, 522, 95, 0, 0, 145, 1, 24, 5867, 2, 0, 91, 125, 878, 2066, 2, 160, 1, 2, 2, 0, 3, 1, 6, 1056, 0, 1, 0, 2, 803, 426, 1, 4298, 451, 0, 428, 1290, 143, 1, 0] payload_rows_per_dst=[16208, 8946, 7860, 1384, 15008, 344, 3894, 4344] dup_rows_per_dst=[26364, 730, 774, 200, 11276, 6, 684, 282] combine_rows_per_src=[16208, 8946, 7860, 1384, 15008, 344, 3894, 4344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=270401536 dispatch_meta_bytes=349896 combine_payload_bytes=270401536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270751432 ingress_bytes=305004544 link_bytes=305004544 token_bytes=541152968 weight_movement_bytes=34603008 total_nvlink_bytes=575755976 weight_to_token_ratio=0.063943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147707544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=302555136 dispatch_meta_bytes=334920 combine_payload_bytes=302555136 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=302890056 ingress_bytes=371761152 link_bytes=371761152 token_bytes=605445192 weight_movement_bytes=69206016 total_nvlink_bytes=674651208 weight_to_token_ratio=0.114306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398907286e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=294789120 dispatch_meta_bytes=328144 combine_payload_bytes=294789120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295117264 ingress_bytes=329392128 link_bytes=329392128 token_bytes=589906384 weight_movement_bytes=34603008 total_nvlink_bytes=624509392 weight_to_token_ratio=0.058658 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.239476059e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=290750464 dispatch_meta_bytes=337160 combine_payload_bytes=290750464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291087624 ingress_bytes=325353472 link_bytes=325353472 token_bytes=581838088 weight_movement_bytes=34603008 total_nvlink_bytes=616441096 weight_to_token_ratio=0.059472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224278922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=321208320 dispatch_meta_bytes=361528 combine_payload_bytes=321208320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321569848 ingress_bytes=355811328 link_bytes=355811328 token_bytes=642778168 weight_movement_bytes=34603008 total_nvlink_bytes=677381176 weight_to_token_ratio=0.053834 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338889382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292978688 dispatch_meta_bytes=332024 combine_payload_bytes=292978688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293310712 ingress_bytes=327581696 link_bytes=327581696 token_bytes=586289400 weight_movement_bytes=34603008 total_nvlink_bytes=620892408 weight_to_token_ratio=0.059020 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232663550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=171130880 dispatch_meta_bytes=222928 combine_payload_bytes=171130880 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=171353808 ingress_bytes=344145920 link_bytes=344145920 token_bytes=342484688 weight_movement_bytes=173015040 total_nvlink_bytes=515499728 weight_to_token_ratio=0.505176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294993391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88776 payload_rows_remote=86698 dup_rows=9528 dup_rows_remote=9528 dedup_ratio=0.096924 router_tokens_per_expert=[0, 8117, 5147, 0, 0, 63, 0, 615, 0, 0, 0, 0, 0, 721, 3883, 138, 0, 75, 0, 315, 0, 0, 0, 1804, 64, 246, 0, 69, 0, 63, 139, 0, 217, 7912, 75, 0, 747, 0, 6793, 0, 1427, 0, 0, 0, 515, 0, 0, 0, 1218, 0, 77, 0, 0, 0, 0, 0, 0, 0, 7245, 541, 75, 115, 0, 736] payload_rows_per_dst=[1356, 9388, 13708, 16088, 2078, 15100, 16234, 14824] dup_rows_per_dst=[0, 96, 974, 898, 0, 2370, 2590, 2600] combine_rows_per_src=[1356, 9388, 13708, 16088, 2078, 15100, 16234, 14824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75006 payload_rows_remote=59006 dup_rows=23298 dup_rows_remote=15910 dedup_ratio=0.237000 router_tokens_per_expert=[0, 8121, 5311, 0, 0, 60, 0, 700, 0, 0, 0, 0, 0, 732, 3570, 136, 0, 77, 0, 315, 0, 0, 0, 1754, 81, 239, 0, 92, 0, 60, 152, 0, 267, 7866, 74, 0, 1081, 0, 6682, 0, 1330, 0, 0, 0, 450, 0, 0, 0, 1140, 0, 77, 8, 0, 0, 0, 0, 0, 0, 7256, 679, 71, 93, 0, 678] payload_rows_per_dst=[16242, 16000, 4292, 16036, 13690, 3544, 2302, 2900] dup_rows_per_dst=[12142, 7388, 0, 944, 2518, 16, 148, 142] combine_rows_per_src=[16242, 16000, 4292, 16036, 13690, 3544, 2302, 2900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86944 payload_rows_remote=71918 dup_rows=11360 dup_rows_remote=8366 dedup_ratio=0.115560 router_tokens_per_expert=[0, 8097, 5316, 0, 0, 42, 0, 605, 0, 0, 0, 0, 0, 734, 3467, 138, 0, 98, 0, 311, 0, 0, 0, 1655, 174, 254, 0, 72, 0, 42, 269, 0, 210, 7916, 96, 0, 1062, 0, 6590, 0, 1144, 0, 0, 0, 430, 0, 0, 0, 1322, 0, 97, 1, 0, 0, 0, 0, 0, 0, 7148, 981, 94, 105, 0, 682] payload_rows_per_dst=[1294, 8604, 13732, 1274, 16170, 14650, 16194, 15026] dup_rows_per_dst=[0, 74, 1028, 348, 2398, 1678, 2840, 2994] combine_rows_per_src=[1294, 8604, 13732, 1274, 16170, 14650, 16194, 15026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87442 payload_rows_remote=71184 dup_rows=10862 dup_rows_remote=8580 dedup_ratio=0.110494 router_tokens_per_expert=[0, 8129, 5359, 0, 1, 60, 0, 792, 0, 0, 0, 0, 0, 731, 3318, 125, 0, 65, 0, 346, 0, 0, 0, 2066, 48, 254, 0, 76, 0, 60, 111, 0, 200, 7892, 65, 0, 1097, 0, 6692, 0, 1162, 0, 0, 0, 498, 0, 0, 0, 1074, 0, 67, 0, 0, 0, 0, 0, 0, 0, 7312, 721, 63, 106, 0, 662] payload_rows_per_dst=[1706, 8342, 14194, 1002, 16064, 14914, 16258, 14962] dup_rows_per_dst=[0, 6, 1478, 96, 2444, 1790, 2282, 2766] combine_rows_per_src=[1706, 8342, 14194, 1002, 16064, 14914, 16258, 14962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=355115008 dispatch_meta_bytes=384904 combine_payload_bytes=355115008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=355499912 ingress_bytes=458924032 link_bytes=458924032 token_bytes=710614920 weight_movement_bytes=103809024 total_nvlink_bytes=814423944 weight_to_token_ratio=0.146083 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726894186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88248 payload_rows_remote=73324 dup_rows=10056 dup_rows_remote=8382 dedup_ratio=0.102295 router_tokens_per_expert=[0, 8102, 5491, 0, 0, 66, 0, 691, 0, 0, 0, 0, 0, 687, 3512, 156, 0, 90, 0, 334, 0, 0, 0, 1641, 161, 247, 0, 60, 0, 66, 251, 0, 269, 7883, 90, 0, 824, 0, 6709, 0, 1158, 0, 0, 0, 432, 0, 0, 0, 1356, 0, 91, 0, 0, 0, 0, 0, 0, 0, 7313, 673, 89, 115, 0, 595] payload_rows_per_dst=[1514, 8596, 14060, 9424, 8378, 14924, 16204, 15148] dup_rows_per_dst=[0, 114, 1052, 978, 922, 1674, 2894, 2422] combine_rows_per_src=[1514, 8596, 14060, 9424, 8378, 14924, 16204, 15148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85012 payload_rows_remote=71860 dup_rows=13292 dup_rows_remote=12272 dedup_ratio=0.135213 router_tokens_per_expert=[0, 8110, 5298, 0, 0, 90, 0, 645, 0, 0, 0, 0, 0, 732, 3456, 172, 0, 86, 0, 377, 0, 0, 0, 1774, 72, 264, 0, 45, 0, 90, 154, 0, 256, 7836, 82, 0, 921, 0, 6745, 0, 1455, 0, 0, 0, 382, 0, 0, 0, 1218, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7289, 645, 82, 100, 0, 693] payload_rows_per_dst=[16220, 14062, 13152, 15940, 13328, 6260, 2438, 3612] dup_rows_per_dst=[2368, 5150, 1020, 982, 2828, 466, 164, 314] combine_rows_per_src=[16220, 14062, 13152, 15940, 13328, 6260, 2438, 3612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=241688576 dispatch_meta_bytes=299664 combine_payload_bytes=241688576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241988240 ingress_bytes=276291584 link_bytes=276291584 token_bytes=483676816 weight_movement_bytes=34603008 total_nvlink_bytes=518279824 weight_to_token_ratio=0.071542 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.039662987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88418 payload_rows_remote=72398 dup_rows=9886 dup_rows_remote=8898 dedup_ratio=0.100566 router_tokens_per_expert=[0, 8123, 5165, 0, 15, 53, 0, 1247, 0, 0, 0, 0, 0, 675, 3355, 122, 0, 70, 0, 340, 0, 0, 0, 2010, 78, 261, 0, 74, 0, 53, 147, 0, 222, 7891, 69, 0, 694, 0, 6717, 0, 1241, 0, 0, 0, 491, 0, 0, 0, 1076, 0, 70, 0, 0, 0, 0, 0, 0, 0, 7207, 709, 69, 115, 0, 793] payload_rows_per_dst=[5028, 8302, 13762, 16020, 1970, 14986, 13526, 14824] dup_rows_per_dst=[670, 2, 1408, 988, 0, 1912, 1944, 2962] combine_rows_per_src=[5028, 8302, 13762, 16020, 1970, 14986, 13526, 14824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69560 payload_rows_remote=53366 dup_rows=28744 dup_rows_remote=5174 dedup_ratio=0.292399 router_tokens_per_expert=[0, 8094, 5359, 0, 3, 54, 2, 919, 0, 1, 4, 0, 0, 693, 3587, 146, 0, 96, 0, 315, 1, 2, 0, 1808, 140, 249, 0, 76, 0, 54, 231, 1, 185, 7892, 91, 1, 767, 3, 6710, 1, 1355, 2, 4, 0, 645, 2, 0, 2, 1044, 1, 94, 0, 0, 0, 0, 4, 0, 1, 7110, 441, 92, 139, 2, 729] payload_rows_per_dst=[16194, 10438, 4444, 16032, 13724, 4006, 2102, 2620] dup_rows_per_dst=[23570, 1742, 0, 1254, 1792, 14, 184, 188] combine_rows_per_src=[16194, 10438, 4444, 16032, 13724, 4006, 2102, 2620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294576128 dispatch_meta_bytes=321136 combine_payload_bytes=294576128 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=294897264 ingress_bytes=398385152 link_bytes=398385152 token_bytes=589473392 weight_movement_bytes=103809024 total_nvlink_bytes=693282416 weight_to_token_ratio=0.176105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499091254e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=319056 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291888720 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583458384 weight_movement_bytes=34603008 total_nvlink_bytes=618061392 weight_to_token_ratio=0.059307 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=300335104 dispatch_meta_bytes=326824 combine_payload_bytes=300335104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300661928 ingress_bytes=334938112 link_bytes=334938112 token_bytes=600997032 weight_movement_bytes=34603008 total_nvlink_bytes=635600040 weight_to_token_ratio=0.057576 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294338560 dispatch_meta_bytes=336528 combine_payload_bytes=294338560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294675088 ingress_bytes=328941568 link_bytes=328941568 token_bytes=589013648 weight_movement_bytes=34603008 total_nvlink_bytes=623616656 weight_to_token_ratio=0.058747 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237780638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=296542208 dispatch_meta_bytes=325184 combine_payload_bytes=296542208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296867392 ingress_bytes=331145216 link_bytes=331145216 token_bytes=593409600 weight_movement_bytes=34603008 total_nvlink_bytes=628012608 weight_to_token_ratio=0.058312 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246072788e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=218587136 dispatch_meta_bytes=234160 combine_payload_bytes=218587136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=218821296 ingress_bytes=322396160 link_bytes=322396160 token_bytes=437408432 weight_movement_bytes=103809024 total_nvlink_bytes=541217456 weight_to_token_ratio=0.237327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213150795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91664 payload_rows_remote=87852 dup_rows=6640 dup_rows_remote=6394 dedup_ratio=0.067546 router_tokens_per_expert=[0, 0, 0, 376, 0, 0, 0, 0, 135, 132, 59, 0, 2245, 75, 0, 0, 2, 0, 0, 0, 0, 69, 0, 0, 421, 7830, 0, 1159, 0, 3051, 0, 0, 150, 1612, 0, 0, 0, 7555, 0, 267, 2494, 194, 287, 0, 383, 1992, 3636, 0, 0, 64, 0, 183, 0, 0, 0, 0, 5, 0, 7216, 6593, 865, 0, 0, 102] payload_rows_per_dst=[15110, 9986, 14570, 8704, 3812, 9224, 15942, 14316] dup_rows_per_dst=[752, 2578, 4, 558, 246, 1476, 212, 814] combine_rows_per_src=[15110, 9986, 14570, 8704, 3812, 9224, 15942, 14316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81002 payload_rows_remote=70870 dup_rows=17302 dup_rows_remote=14732 dedup_ratio=0.176005 router_tokens_per_expert=[0, 0, 0, 347, 0, 0, 0, 0, 136, 191, 52, 0, 2175, 71, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 439, 7893, 0, 1112, 0, 3116, 0, 0, 145, 1446, 0, 0, 0, 7574, 0, 240, 2533, 196, 200, 0, 420, 2291, 3726, 0, 0, 81, 0, 165, 0, 0, 0, 0, 8, 0, 7102, 6484, 862, 0, 0, 71] payload_rows_per_dst=[15148, 10132, 14356, 16080, 13452, 9602, 492, 1740] dup_rows_per_dst=[694, 2570, 0, 9040, 3178, 1678, 0, 142] combine_rows_per_src=[15148, 10132, 14356, 16080, 13452, 9602, 492, 1740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=359841792 dispatch_meta_bytes=376984 combine_payload_bytes=359841792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=360218776 ingress_bytes=429047808 link_bytes=429047808 token_bytes=720060568 weight_movement_bytes=69206016 total_nvlink_bytes=789266584 weight_to_token_ratio=0.096111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.614472360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73054 payload_rows_remote=57180 dup_rows=25250 dup_rows_remote=12284 dedup_ratio=0.256856 router_tokens_per_expert=[0, 0, 0, 309, 0, 0, 0, 0, 99, 127, 49, 0, 2136, 95, 0, 1, 0, 0, 0, 0, 0, 57, 0, 0, 348, 7833, 0, 1495, 0, 2535, 0, 0, 191, 1574, 0, 0, 0, 7428, 0, 254, 2388, 260, 191, 0, 326, 2663, 4035, 0, 0, 174, 0, 164, 0, 0, 0, 0, 15, 0, 6985, 6493, 831, 0, 0, 96] payload_rows_per_dst=[618, 7040, 114, 3664, 15778, 14024, 15942, 15874] dup_rows_per_dst=[0, 1282, 0, 22, 3116, 7464, 400, 12966] combine_rows_per_src=[618, 7040, 114, 3664, 15778, 14024, 15942, 15874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86960 payload_rows_remote=70948 dup_rows=11344 dup_rows_remote=10984 dedup_ratio=0.115397 router_tokens_per_expert=[0, 0, 0, 332, 0, 0, 0, 0, 152, 214, 60, 0, 2166, 63, 0, 0, 4, 0, 0, 0, 0, 92, 0, 0, 405, 7881, 0, 1354, 0, 2941, 0, 0, 128, 1506, 0, 0, 0, 7573, 0, 259, 2647, 144, 285, 0, 373, 2144, 3311, 0, 0, 48, 0, 257, 0, 0, 0, 0, 7, 0, 7113, 6642, 988, 0, 0, 63] payload_rows_per_dst=[3458, 9432, 12340, 7314, 13582, 10232, 16012, 14590] dup_rows_per_dst=[132, 2500, 0, 500, 2424, 2540, 360, 2888] combine_rows_per_src=[3458, 9432, 12340, 7314, 13582, 10232, 16012, 14590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91682 payload_rows_remote=82174 dup_rows=6622 dup_rows_remote=5218 dedup_ratio=0.067362 router_tokens_per_expert=[0, 0, 0, 318, 0, 0, 0, 0, 153, 109, 34, 0, 2079, 90, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 422, 7805, 0, 1491, 0, 2760, 0, 0, 180, 1856, 0, 0, 0, 7412, 0, 249, 2329, 272, 273, 0, 392, 2190, 3826, 0, 0, 161, 0, 160, 0, 0, 0, 0, 56, 0, 7038, 6443, 873, 0, 0, 94] payload_rows_per_dst=[14824, 10116, 14250, 8710, 4218, 9508, 15926, 14130] dup_rows_per_dst=[636, 2466, 0, 636, 352, 1404, 326, 802] combine_rows_per_src=[14824, 10116, 14250, 8710, 4218, 9508, 15926, 14130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88528 payload_rows_remote=74304 dup_rows=9776 dup_rows_remote=9776 dedup_ratio=0.099447 router_tokens_per_expert=[0, 0, 0, 351, 0, 0, 0, 0, 203, 212, 36, 0, 2077, 82, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 419, 7811, 0, 1419, 0, 2880, 0, 0, 164, 1449, 0, 0, 0, 7455, 0, 264, 2336, 192, 313, 0, 339, 2189, 4042, 0, 0, 72, 0, 186, 0, 0, 0, 0, 33, 0, 6999, 6512, 922, 0, 0, 82] payload_rows_per_dst=[14910, 10394, 14224, 9236, 13422, 9290, 15142, 1910] dup_rows_per_dst=[702, 2910, 0, 1042, 3356, 1448, 154, 164] combine_rows_per_src=[14910, 10394, 14224, 9236, 13422, 9290, 15142, 1910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290283520 dispatch_meta_bytes=342408 combine_payload_bytes=290283520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290625928 ingress_bytes=324886528 link_bytes=324886528 token_bytes=580909448 weight_movement_bytes=34603008 total_nvlink_bytes=615512456 weight_to_token_ratio=0.059567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222521849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90264 payload_rows_remote=82418 dup_rows=8040 dup_rows_remote=7530 dedup_ratio=0.081787 router_tokens_per_expert=[0, 0, 0, 332, 0, 0, 0, 0, 227, 180, 91, 0, 2611, 69, 0, 0, 26, 0, 0, 0, 0, 79, 0, 0, 416, 7802, 0, 892, 0, 2870, 0, 0, 138, 1419, 0, 0, 0, 7519, 0, 355, 2296, 193, 336, 0, 402, 1890, 3625, 0, 0, 78, 0, 335, 0, 0, 0, 0, 1, 0, 6972, 6605, 1269, 0, 0, 124] payload_rows_per_dst=[15038, 10604, 14102, 7846, 4178, 8784, 15928, 13784] dup_rows_per_dst=[664, 3002, 52, 510, 276, 1450, 502, 1584] combine_rows_per_src=[15038, 10604, 14102, 7846, 4178, 8784, 15928, 13784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81272 payload_rows_remote=66338 dup_rows=17032 dup_rows_remote=16348 dedup_ratio=0.173258 router_tokens_per_expert=[0, 0, 0, 342, 2, 0, 0, 6, 219, 213, 55, 3, 2363, 92, 2, 1, 5, 0, 0, 0, 8, 71, 0, 0, 402, 7759, 0, 927, 3, 3054, 2, 2, 186, 1611, 0, 0, 0, 7459, 1, 263, 2196, 271, 305, 0, 361, 1955, 3698, 2, 0, 140, 3, 337, 0, 3, 0, 0, 0, 4, 7062, 6468, 1166, 0, 0, 130] payload_rows_per_dst=[14934, 10484, 14272, 16034, 13446, 8798, 962, 2342] dup_rows_per_dst=[684, 2808, 20, 8408, 3468, 1382, 4, 258] combine_rows_per_src=[14934, 10484, 14272, 16034, 13446, 8798, 962, 2342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=277856 combine_payload_bytes=234209280 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=234487136 ingress_bytes=338018304 link_bytes=338018304 token_bytes=468696416 weight_movement_bytes=103809024 total_nvlink_bytes=572505440 weight_to_token_ratio=0.221485 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271935665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290603008 dispatch_meta_bytes=327728 combine_payload_bytes=290603008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290930736 ingress_bytes=325206016 link_bytes=325206016 token_bytes=581533744 weight_movement_bytes=34603008 total_nvlink_bytes=616136752 weight_to_token_ratio=0.059503 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223724057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=336584704 dispatch_meta_bytes=349568 combine_payload_bytes=336584704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=336934272 ingress_bytes=405790720 link_bytes=405790720 token_bytes=673518976 weight_movement_bytes=69206016 total_nvlink_bytes=742724992 weight_to_token_ratio=0.102753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526957811e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304349184 dispatch_meta_bytes=336320 combine_payload_bytes=304349184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304685504 ingress_bytes=338952192 link_bytes=338952192 token_bytes=609034688 weight_movement_bytes=34603008 total_nvlink_bytes=643637696 weight_to_token_ratio=0.056816 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275449811e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91704 payload_rows_remote=75510 dup_rows=6600 dup_rows_remote=3332 dedup_ratio=0.067139 router_tokens_per_expert=[0, 0, 246, 315, 0, 7506, 0, 418, 0, 0, 0, 0, 75, 0, 0, 0, 681, 0, 0, 0, 0, 4949, 0, 6801, 0, 0, 94, 7664, 0, 0, 0, 63, 1439, 0, 0, 498, 0, 0, 144, 144, 207, 0, 0, 3491, 0, 0, 0, 0, 6517, 0, 755, 0, 6176, 0, 0, 0, 0, 64, 0, 397, 347, 86, 0, 75] payload_rows_per_dst=[1466, 13184, 10622, 15328, 16194, 7396, 13114, 14400] dup_rows_per_dst=[492, 0, 638, 314, 3268, 0, 748, 1140] combine_rows_per_src=[1466, 13184, 10622, 15328, 16194, 7396, 13114, 14400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=337584128 dispatch_meta_bytes=359792 combine_payload_bytes=337584128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=337943920 ingress_bytes=441393152 link_bytes=441393152 token_bytes=675528048 weight_movement_bytes=103809024 total_nvlink_bytes=779337072 weight_to_token_ratio=0.153671 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.660926896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271720448 dispatch_meta_bytes=330744 combine_payload_bytes=271720448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272051192 ingress_bytes=306323456 link_bytes=306323456 token_bytes=543771640 weight_movement_bytes=34603008 total_nvlink_bytes=578374648 weight_to_token_ratio=0.063635 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152670504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309288960 dispatch_meta_bytes=315368 combine_payload_bytes=309288960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309604328 ingress_bytes=343891968 link_bytes=343891968 token_bytes=618893288 weight_movement_bytes=34603008 total_nvlink_bytes=653496296 weight_to_token_ratio=0.055911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294037790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78962 payload_rows_remote=65804 dup_rows=19342 dup_rows_remote=19342 dedup_ratio=0.196757 router_tokens_per_expert=[0, 0, 239, 312, 0, 7565, 0, 419, 0, 0, 0, 0, 74, 0, 0, 0, 591, 0, 0, 0, 0, 5152, 0, 6805, 0, 0, 81, 7675, 0, 0, 0, 60, 1291, 0, 0, 523, 0, 0, 147, 166, 207, 0, 0, 3532, 0, 0, 0, 0, 6505, 0, 712, 0, 6108, 0, 0, 0, 0, 81, 0, 384, 348, 101, 0, 74] payload_rows_per_dst=[16232, 13158, 13926, 15758, 3140, 7478, 7604, 1666] dup_rows_per_dst=[11142, 0, 866, 5568, 1114, 0, 342, 310] combine_rows_per_src=[16232, 13158, 13926, 15758, 3140, 7478, 7604, 1666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79892 payload_rows_remote=65070 dup_rows=18412 dup_rows_remote=16820 dedup_ratio=0.187297 router_tokens_per_expert=[0, 0, 254, 311, 0, 7529, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 599, 0, 0, 0, 0, 5307, 0, 6963, 0, 0, 77, 7614, 0, 0, 0, 42, 1162, 0, 1, 632, 0, 0, 152, 168, 194, 0, 0, 3303, 0, 0, 0, 0, 6509, 0, 641, 0, 5922, 0, 0, 0, 0, 174, 0, 455, 337, 182, 0, 96] payload_rows_per_dst=[1486, 6172, 11196, 238, 16274, 15876, 13828, 14822] dup_rows_per_dst=[508, 0, 616, 0, 3014, 6346, 6336, 1592] combine_rows_per_src=[1486, 6172, 11196, 238, 16274, 15876, 13828, 14822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86156 payload_rows_remote=72966 dup_rows=12148 dup_rows_remote=11514 dedup_ratio=0.123576 router_tokens_per_expert=[0, 0, 254, 345, 0, 7522, 0, 399, 0, 0, 0, 0, 65, 0, 0, 0, 721, 0, 0, 0, 0, 5162, 0, 6939, 0, 0, 74, 7705, 0, 0, 0, 60, 1392, 0, 0, 521, 0, 0, 155, 140, 215, 0, 0, 3058, 0, 0, 0, 0, 6613, 0, 682, 0, 6230, 0, 0, 0, 0, 48, 0, 382, 336, 69, 0, 65] payload_rows_per_dst=[1490, 13356, 11134, 268, 16196, 15876, 13190, 14646] dup_rows_per_dst=[506, 0, 632, 0, 3264, 6080, 634, 1032] combine_rows_per_src=[1490, 13356, 11134, 268, 16196, 15876, 13190, 14646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86820 payload_rows_remote=73168 dup_rows=11484 dup_rows_remote=6506 dedup_ratio=0.116821 router_tokens_per_expert=[0, 0, 247, 334, 0, 7521, 0, 393, 0, 0, 1, 0, 90, 0, 0, 0, 689, 0, 0, 0, 0, 4973, 0, 6763, 0, 0, 97, 7594, 0, 0, 0, 66, 1261, 0, 1, 645, 0, 0, 177, 149, 243, 0, 0, 3542, 0, 0, 0, 0, 6508, 0, 659, 0, 6027, 0, 0, 0, 0, 161, 0, 436, 303, 182, 0, 90] payload_rows_per_dst=[1454, 13196, 10676, 4382, 16298, 13652, 12778, 14384] dup_rows_per_dst=[494, 2, 648, 72, 3210, 4978, 594, 1486] combine_rows_per_src=[1454, 13196, 10676, 4382, 16298, 13652, 12778, 14384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88406 payload_rows_remote=81196 dup_rows=9898 dup_rows_remote=9442 dedup_ratio=0.100688 router_tokens_per_expert=[0, 0, 264, 377, 0, 7415, 0, 442, 0, 0, 0, 0, 82, 0, 0, 0, 716, 0, 0, 0, 0, 5252, 0, 6653, 0, 0, 69, 7675, 0, 0, 0, 90, 1349, 0, 0, 575, 0, 0, 195, 126, 285, 0, 0, 3300, 0, 0, 0, 0, 6454, 0, 712, 0, 6087, 0, 0, 0, 0, 72, 0, 429, 361, 90, 0, 82] payload_rows_per_dst=[15934, 13072, 7210, 15350, 3258, 7170, 12946, 13466] dup_rows_per_dst=[6088, 0, 456, 318, 1232, 0, 652, 1152] combine_rows_per_src=[15934, 13072, 7210, 15350, 3258, 7170, 12946, 13466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=269533184 dispatch_meta_bytes=340584 combine_payload_bytes=269533184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269873768 ingress_bytes=304136192 link_bytes=304136192 token_bytes=539406952 weight_movement_bytes=34603008 total_nvlink_bytes=574009960 weight_to_token_ratio=0.064150 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144440006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92716 payload_rows_remote=77486 dup_rows=5588 dup_rows_remote=5296 dedup_ratio=0.056844 router_tokens_per_expert=[0, 0, 263, 337, 0, 7542, 0, 414, 0, 0, 4, 0, 69, 0, 0, 0, 828, 0, 0, 0, 0, 5066, 0, 6868, 0, 0, 93, 7615, 0, 0, 0, 53, 1498, 0, 0, 532, 0, 0, 148, 153, 201, 0, 0, 3034, 0, 0, 0, 0, 6400, 0, 774, 0, 6212, 0, 0, 0, 0, 78, 0, 399, 403, 99, 0, 69] payload_rows_per_dst=[13502, 12942, 11078, 15230, 5718, 6470, 13228, 14548] dup_rows_per_dst=[1062, 4, 710, 292, 1492, 0, 744, 1284] combine_rows_per_src=[13502, 12942, 11078, 15230, 5718, 6470, 13228, 14548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73266 payload_rows_remote=57096 dup_rows=25038 dup_rows_remote=14760 dedup_ratio=0.254700 router_tokens_per_expert=[1, 0, 250, 317, 0, 7490, 3, 434, 0, 0, 5, 0, 91, 4, 0, 0, 809, 0, 0, 0, 0, 4729, 3, 6619, 0, 0, 55, 7610, 0, 2, 0, 55, 1342, 11, 1, 605, 0, 3, 160, 194, 214, 6, 8, 3256, 0, 0, 6, 1, 6509, 0, 782, 0, 6388, 0, 0, 1, 1, 140, 0, 431, 351, 170, 1, 94] payload_rows_per_dst=[16170, 13208, 14130, 15854, 3468, 6958, 1566, 1912] dup_rows_per_dst=[10278, 10, 3766, 9332, 1164, 24, 0, 464] combine_rows_per_src=[16170, 13208, 14130, 15854, 3468, 6958, 1566, 1912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=327560 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854280 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533381000 weight_movement_bytes=34603008 total_nvlink_bytes=567984008 weight_to_token_ratio=0.064875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=298868736 dispatch_meta_bytes=337920 combine_payload_bytes=298868736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=299206656 ingress_bytes=402677760 link_bytes=402677760 token_bytes=598075392 weight_movement_bytes=103809024 total_nvlink_bytes=701884416 weight_to_token_ratio=0.173572 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515243993e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299696128 dispatch_meta_bytes=318696 combine_payload_bytes=299696128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300014824 ingress_bytes=334299136 link_bytes=334299136 token_bytes=599710952 weight_movement_bytes=34603008 total_nvlink_bytes=634313960 weight_to_token_ratio=0.057699 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332578816 dispatch_meta_bytes=362552 combine_payload_bytes=332578816 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=332941368 ingress_bytes=436387840 link_bytes=436387840 token_bytes=665520184 weight_movement_bytes=103809024 total_nvlink_bytes=769329208 weight_to_token_ratio=0.155982 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642092310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91860 payload_rows_remote=83288 dup_rows=6444 dup_rows_remote=6294 dedup_ratio=0.065552 router_tokens_per_expert=[3869, 0, 0, 0, 63, 0, 0, 0, 0, 456, 0, 0, 75, 0, 678, 5268, 69, 7357, 0, 75, 0, 0, 69, 0, 7577, 0, 0, 246, 0, 4305, 75, 7404, 4286, 75, 0, 0, 0, 0, 0, 0, 75, 0, 294, 0, 0, 4635, 246, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7864, 11244, 14714, 9102, 8572, 9582, 15628, 15154] dup_rows_per_dst=[0, 1710, 426, 150, 150, 918, 2748, 342] combine_rows_per_src=[7864, 11244, 14714, 9102, 8572, 9582, 15628, 15154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=317382656 dispatch_meta_bytes=331128 combine_payload_bytes=317382656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317713784 ingress_bytes=386588672 link_bytes=386588672 token_bytes=635096440 weight_movement_bytes=69206016 total_nvlink_bytes=704302456 weight_to_token_ratio=0.108969 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454702050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233865216 dispatch_meta_bytes=287424 combine_payload_bytes=233865216 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=234152640 ingress_bytes=303071232 link_bytes=303071232 token_bytes=468017856 weight_movement_bytes=69206016 total_nvlink_bytes=537223872 weight_to_token_ratio=0.147870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140432647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=341147648 dispatch_meta_bytes=358328 combine_payload_bytes=341147648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341505976 ingress_bytes=375750656 link_bytes=375750656 token_bytes=682653624 weight_movement_bytes=34603008 total_nvlink_bytes=717256632 weight_to_token_ratio=0.050689 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413919468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82072 payload_rows_remote=70386 dup_rows=16232 dup_rows_remote=14382 dedup_ratio=0.165120 router_tokens_per_expert=[4589, 0, 0, 0, 60, 0, 0, 0, 0, 444, 0, 0, 74, 0, 718, 5532, 92, 7320, 0, 74, 0, 0, 92, 0, 7540, 0, 0, 239, 0, 3911, 71, 7401, 3707, 71, 0, 0, 0, 0, 0, 0, 74, 0, 280, 0, 0, 4681, 239, 0, 0, 0, 0, 1783, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14088, 11686, 14640, 15774, 7414, 9668, 3566, 5236] dup_rows_per_dst=[3032, 1850, 516, 9696, 142, 880, 0, 116] combine_rows_per_src=[14088, 11686, 14640, 15774, 7414, 9668, 3566, 5236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86880 payload_rows_remote=71926 dup_rows=11424 dup_rows_remote=11052 dedup_ratio=0.116211 router_tokens_per_expert=[4817, 0, 0, 0, 42, 0, 0, 0, 0, 443, 0, 0, 95, 0, 574, 5407, 72, 7301, 0, 96, 0, 0, 72, 0, 7473, 0, 1, 254, 0, 3960, 95, 7402, 3610, 95, 0, 0, 0, 0, 0, 0, 96, 0, 299, 0, 0, 4725, 254, 0, 0, 0, 0, 1779, 0, 0, 0, 0, 0, 170, 20, 0, 0, 0, 0, 0] payload_rows_per_dst=[9718, 5586, 7122, 8430, 15732, 9840, 15498, 14954] dup_rows_per_dst=[0, 566, 244, 190, 6280, 908, 2864, 372] combine_rows_per_src=[9718, 5586, 7122, 8430, 15732, 9840, 15498, 14954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84294 payload_rows_remote=68656 dup_rows=14010 dup_rows_remote=10878 dedup_ratio=0.142517 router_tokens_per_expert=[3942, 0, 0, 0, 60, 0, 0, 0, 0, 411, 0, 0, 65, 0, 623, 5160, 75, 7391, 0, 65, 0, 0, 75, 0, 7610, 0, 0, 254, 0, 4395, 63, 7462, 4279, 63, 0, 0, 0, 0, 0, 0, 65, 0, 278, 0, 0, 4478, 254, 0, 0, 0, 0, 1923, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8004, 10910, 454, 9298, 15542, 9228, 15638, 15220] dup_rows_per_dst=[0, 1608, 152, 126, 7748, 922, 3132, 322] combine_rows_per_src=[8004, 10910, 454, 9298, 15542, 9228, 15638, 15220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91488 payload_rows_remote=81506 dup_rows=6816 dup_rows_remote=5922 dedup_ratio=0.069336 router_tokens_per_expert=[4369, 0, 0, 0, 66, 0, 0, 0, 0, 393, 0, 0, 90, 0, 647, 5622, 60, 7236, 0, 90, 0, 0, 60, 0, 7513, 0, 0, 247, 0, 4012, 90, 7536, 3499, 90, 0, 0, 0, 0, 0, 0, 90, 0, 299, 0, 0, 4802, 247, 0, 0, 0, 0, 1947, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8870, 11894, 14472, 8518, 6998, 9982, 15728, 15026] dup_rows_per_dst=[0, 1610, 420, 180, 180, 894, 3238, 294] combine_rows_per_src=[8870, 11894, 14472, 8518, 6998, 9982, 15728, 15026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85648 payload_rows_remote=70960 dup_rows=12656 dup_rows_remote=12320 dedup_ratio=0.128743 router_tokens_per_expert=[4466, 0, 0, 0, 90, 0, 0, 0, 0, 373, 0, 0, 82, 0, 585, 5389, 42, 7344, 0, 82, 0, 0, 44, 0, 7543, 0, 0, 264, 0, 4437, 82, 7505, 4013, 82, 0, 0, 0, 0, 0, 0, 81, 0, 301, 0, 0, 4263, 264, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14372, 11382, 14688, 4688, 8026, 14078, 3328, 15086] dup_rows_per_dst=[3614, 1476, 336, 128, 164, 6626, 0, 312] combine_rows_per_src=[14372, 11382, 14688, 4688, 8026, 14078, 3328, 15086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288301056 dispatch_meta_bytes=339072 combine_payload_bytes=288301056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=288640128 ingress_bytes=357507072 link_bytes=357507072 token_bytes=576941184 weight_movement_bytes=69206016 total_nvlink_bytes=646147200 weight_to_token_ratio=0.119953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345270330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87352 payload_rows_remote=85920 dup_rows=10952 dup_rows_remote=10946 dedup_ratio=0.111410 router_tokens_per_expert=[3098, 0, 0, 0, 70, 0, 0, 0, 0, 620, 0, 0, 69, 0, 686, 4747, 66, 7329, 0, 69, 0, 0, 66, 0, 7580, 0, 0, 261, 0, 4505, 69, 7427, 5544, 69, 0, 0, 0, 0, 0, 0, 69, 0, 285, 0, 0, 4112, 261, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[11220, 11074, 14658, 1432, 11088, 11298, 11422, 15160] dup_rows_per_dst=[1638, 2880, 402, 6, 138, 3478, 2084, 326] combine_rows_per_src=[11220, 11074, 14658, 1432, 11088, 11298, 11422, 15160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78266 payload_rows_remote=64852 dup_rows=20038 dup_rows_remote=17568 dedup_ratio=0.203837 router_tokens_per_expert=[3829, 0, 0, 0, 60, 0, 1, 0, 1, 543, 0, 0, 92, 0, 637, 5141, 70, 7290, 2, 91, 0, 0, 77, 0, 7482, 6, 2, 249, 4, 4052, 92, 7458, 4973, 91, 0, 0, 1, 0, 0, 0, 91, 0, 275, 4, 0, 4376, 249, 6, 0, 0, 0, 1712, 2, 0, 0, 1, 5, 182, 0, 0, 1, 4, 0, 0] payload_rows_per_dst=[13414, 11104, 14592, 16246, 9948, 9154, 3430, 378] dup_rows_per_dst=[2470, 1724, 468, 14340, 182, 848, 0, 6] combine_rows_per_src=[13414, 11104, 14592, 16246, 9948, 9154, 3430, 378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294608896 dispatch_meta_bytes=331912 combine_payload_bytes=294608896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294940808 ingress_bytes=329211904 link_bytes=329211904 token_bytes=589549704 weight_movement_bytes=34603008 total_nvlink_bytes=624152712 weight_to_token_ratio=0.058694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238797891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281214976 dispatch_meta_bytes=318136 combine_payload_bytes=281214976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281533112 ingress_bytes=315817984 link_bytes=315817984 token_bytes=562748088 weight_movement_bytes=34603008 total_nvlink_bytes=597351096 weight_to_token_ratio=0.061489 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188397648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=333848576 dispatch_meta_bytes=349712 combine_payload_bytes=333848576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334198288 ingress_bytes=368451584 link_bytes=368451584 token_bytes=668046864 weight_movement_bytes=34603008 total_nvlink_bytes=702649872 weight_to_token_ratio=0.051797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.386453648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88142 payload_rows_remote=73180 dup_rows=10162 dup_rows_remote=9882 dedup_ratio=0.103373 router_tokens_per_expert=[0, 0, 0, 459, 0, 0, 935, 0, 0, 4266, 7803, 0, 0, 0, 709, 0, 69, 327, 0, 132, 1405, 0, 0, 0, 964, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 7406, 140, 0, 75, 421, 0, 1989, 0, 0, 0, 0, 0, 2021, 0, 75, 0, 7870, 0, 69, 0, 4527, 0, 0, 0, 6504, 560, 78, 0] payload_rows_per_dst=[14428, 1418, 3522, 15614, 14962, 11486, 16384, 10328] dup_rows_per_dst=[1368, 0, 344, 2616, 280, 1866, 3686, 2] combine_rows_per_src=[14428, 1418, 3522, 15614, 14962, 11486, 16384, 10328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290652160 dispatch_meta_bytes=333120 combine_payload_bytes=290652160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290985280 ingress_bytes=359858176 link_bytes=359858176 token_bytes=581637440 weight_movement_bytes=69206016 total_nvlink_bytes=650843456 weight_to_token_ratio=0.118985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354117345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=351928320 dispatch_meta_bytes=387464 combine_payload_bytes=351928320 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=352315784 ingress_bytes=524943360 link_bytes=524943360 token_bytes=704244104 weight_movement_bytes=173015040 total_nvlink_bytes=877259144 weight_to_token_ratio=0.245675 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.975319602e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=329680 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265963472 ingress_bytes=300236800 link_bytes=300236800 token_bytes=531597264 weight_movement_bytes=34603008 total_nvlink_bytes=566200272 weight_to_token_ratio=0.065093 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129766907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299745280 dispatch_meta_bytes=332248 combine_payload_bytes=299745280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=300077528 ingress_bytes=368951296 link_bytes=368951296 token_bytes=599822808 weight_movement_bytes=69206016 total_nvlink_bytes=669028824 weight_to_token_ratio=0.115377 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388334024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74598 payload_rows_remote=58398 dup_rows=23706 dup_rows_remote=8038 dedup_ratio=0.241150 router_tokens_per_expert=[0, 0, 0, 516, 0, 0, 901, 0, 0, 3935, 7810, 0, 0, 0, 734, 0, 76, 337, 0, 136, 1762, 0, 0, 0, 866, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 134, 0, 71, 595, 0, 2119, 0, 0, 0, 0, 0, 1768, 0, 71, 0, 7823, 0, 76, 0, 4718, 0, 0, 0, 6415, 565, 28, 0] payload_rows_per_dst=[14332, 16200, 15910, 2344, 1024, 11430, 5266, 8092] dup_rows_per_dst=[1332, 15668, 4358, 0, 14, 1868, 454, 12] combine_rows_per_src=[14332, 16200, 15910, 2344, 1024, 11430, 5266, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81704 payload_rows_remote=66480 dup_rows=16600 dup_rows_remote=8130 dedup_ratio=0.168864 router_tokens_per_expert=[0, 0, 0, 395, 0, 0, 896, 0, 0, 3804, 7844, 0, 0, 0, 676, 0, 57, 317, 0, 99, 2121, 0, 0, 0, 836, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 139, 0, 95, 531, 0, 1969, 0, 0, 0, 0, 0, 1857, 0, 95, 0, 7813, 0, 57, 0, 4962, 0, 0, 0, 6239, 589, 57, 0] payload_rows_per_dst=[2438, 1352, 4876, 15690, 14904, 10870, 16350, 15224] dup_rows_per_dst=[144, 0, 312, 2366, 276, 1738, 3294, 8470] combine_rows_per_src=[2438, 1352, 4876, 15690, 14904, 10870, 16350, 15224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81916 payload_rows_remote=65532 dup_rows=16388 dup_rows_remote=13252 dedup_ratio=0.166707 router_tokens_per_expert=[0, 0, 0, 452, 0, 0, 893, 0, 0, 4248, 7780, 0, 0, 0, 695, 0, 92, 353, 0, 152, 1778, 0, 0, 0, 873, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 7438, 125, 0, 63, 525, 0, 1987, 0, 0, 0, 0, 0, 1765, 0, 63, 0, 7840, 0, 92, 0, 4527, 0, 0, 0, 6542, 527, 23, 0] payload_rows_per_dst=[2540, 1390, 4266, 15584, 15006, 11596, 16384, 15150] dup_rows_per_dst=[150, 0, 484, 2360, 246, 1924, 3136, 8088] combine_rows_per_src=[2540, 1390, 4266, 15584, 15006, 11596, 16384, 15150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88302 payload_rows_remote=77418 dup_rows=10002 dup_rows_remote=8348 dedup_ratio=0.101746 router_tokens_per_expert=[0, 0, 0, 452, 0, 0, 876, 0, 0, 3899, 7823, 0, 0, 0, 729, 0, 87, 356, 0, 153, 1938, 0, 0, 0, 890, 0, 0, 342, 0, 0, 0, 0, 0, 1, 0, 0, 7335, 156, 0, 90, 587, 0, 1783, 0, 0, 0, 0, 0, 2048, 0, 90, 0, 7803, 0, 87, 0, 4751, 0, 0, 0, 6368, 497, 11, 0] payload_rows_per_dst=[13906, 1458, 4584, 15646, 14850, 10884, 16384, 10590] dup_rows_per_dst=[1258, 0, 484, 2464, 314, 1654, 3672, 156] combine_rows_per_src=[13906, 1458, 4584, 15646, 14850, 10884, 16384, 10590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85758 payload_rows_remote=69790 dup_rows=12546 dup_rows_remote=7800 dedup_ratio=0.127625 router_tokens_per_expert=[0, 0, 0, 386, 0, 0, 958, 0, 0, 4067, 7761, 0, 0, 0, 686, 0, 113, 403, 0, 200, 1858, 0, 0, 0, 827, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 7397, 172, 0, 82, 380, 0, 1950, 0, 0, 0, 0, 0, 1942, 0, 82, 0, 7783, 0, 113, 0, 4570, 0, 0, 0, 6428, 546, 90, 0] payload_rows_per_dst=[14226, 15722, 15968, 2370, 11856, 10944, 4274, 10398] dup_rows_per_dst=[1318, 4374, 4746, 0, 244, 1850, 0, 14] combine_rows_per_src=[14226, 15722, 15968, 2370, 11856, 10944, 4274, 10398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=239198208 dispatch_meta_bytes=265744 combine_payload_bytes=239198208 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=239463952 ingress_bytes=343007232 link_bytes=343007232 token_bytes=478662160 weight_movement_bytes=103809024 total_nvlink_bytes=582471184 weight_to_token_ratio=0.216873 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290708600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272302080 dispatch_meta_bytes=298440 combine_payload_bytes=272302080 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=272600520 ingress_bytes=376111104 link_bytes=376111104 token_bytes=544902600 weight_movement_bytes=103809024 total_nvlink_bytes=648711624 weight_to_token_ratio=0.190509 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.415275805e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69760 payload_rows_remote=55596 dup_rows=28544 dup_rows_remote=27438 dedup_ratio=0.290365 router_tokens_per_expert=[0, 0, 0, 430, 0, 2, 799, 0, 0, 4579, 7753, 0, 4, 7, 700, 0, 69, 347, 3, 123, 1343, 0, 0, 0, 1146, 0, 0, 444, 1, 1, 0, 0, 1, 0, 4, 3, 7322, 150, 1, 91, 279, 3, 2220, 0, 3, 0, 0, 7, 1999, 1, 91, 2, 7807, 3, 70, 0, 4299, 0, 3, 0, 6404, 520, 118, 0] payload_rows_per_dst=[14164, 16242, 15874, 3184, 8804, 5886, 4326, 1280] dup_rows_per_dst=[1106, 22932, 3510, 0, 294, 694, 6, 2] combine_rows_per_src=[14164, 16242, 15874, 3184, 8804, 5886, 4326, 1280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=268419072 dispatch_meta_bytes=315136 combine_payload_bytes=268419072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268734208 ingress_bytes=337625088 link_bytes=337625088 token_bytes=537153280 weight_movement_bytes=69206016 total_nvlink_bytes=606359296 weight_to_token_ratio=0.128838 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270456025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88796 payload_rows_remote=73298 dup_rows=9508 dup_rows_remote=6940 dedup_ratio=0.096720 router_tokens_per_expert=[0, 0, 0, 471, 0, 0, 843, 0, 0, 4555, 7772, 0, 0, 0, 733, 0, 79, 351, 0, 144, 1276, 0, 0, 0, 946, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 7423, 123, 0, 69, 244, 0, 2008, 0, 0, 0, 0, 0, 2027, 0, 69, 0, 7852, 0, 79, 0, 4459, 0, 0, 0, 6483, 530, 260, 0] payload_rows_per_dst=[14398, 1542, 15706, 15498, 14984, 11556, 4618, 10494] dup_rows_per_dst=[1196, 6, 3360, 2568, 246, 2058, 70, 4] combine_rows_per_src=[14398, 1542, 15706, 15498, 14984, 11556, 4618, 10494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=317104128 dispatch_meta_bytes=343064 combine_payload_bytes=317104128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317447192 ingress_bytes=351707136 link_bytes=351707136 token_bytes=634551320 weight_movement_bytes=34603008 total_nvlink_bytes=669154328 weight_to_token_ratio=0.054531 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323445638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=285859840 dispatch_meta_bytes=310360 combine_payload_bytes=285859840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286170200 ingress_bytes=320462848 link_bytes=320462848 token_bytes=572030040 weight_movement_bytes=34603008 total_nvlink_bytes=606633048 weight_to_token_ratio=0.060492 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205875898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90660 payload_rows_remote=78740 dup_rows=7644 dup_rows_remote=5150 dedup_ratio=0.077759 router_tokens_per_expert=[551, 0, 0, 75, 321, 5977, 0, 0, 3804, 69, 112, 0, 0, 0, 0, 0, 0, 75, 1056, 0, 0, 75, 0, 0, 149, 0, 0, 0, 60, 75, 0, 0, 0, 0, 71, 0, 0, 0, 2514, 280, 8, 4342, 7665, 8129, 0, 0, 33, 0, 0, 0, 0, 75, 0, 0, 387, 7103, 4993, 0, 0, 0, 627, 314, 3, 209] payload_rows_per_dst=[13514, 7970, 16006, 16258, 11920, 82, 14356, 10554] dup_rows_per_dst=[334, 0, 1736, 568, 2494, 0, 774, 1738] combine_rows_per_src=[13514, 7970, 16006, 16258, 11920, 82, 14356, 10554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=227721216 dispatch_meta_bytes=332136 combine_payload_bytes=227721216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228053352 ingress_bytes=262324224 link_bytes=262324224 token_bytes=455774568 weight_movement_bytes=34603008 total_nvlink_bytes=490377576 weight_to_token_ratio=0.075921 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=38528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.871049361e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=300228608 dispatch_meta_bytes=320952 combine_payload_bytes=300228608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300549560 ingress_bytes=334831616 link_bytes=334831616 token_bytes=600778168 weight_movement_bytes=34603008 total_nvlink_bytes=635381176 weight_to_token_ratio=0.057597 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=322519040 dispatch_meta_bytes=335560 combine_payload_bytes=322519040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322854600 ingress_bytes=357122048 link_bytes=357122048 token_bytes=645373640 weight_movement_bytes=34603008 total_nvlink_bytes=679976648 weight_to_token_ratio=0.053617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343821516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84684 payload_rows_remote=69482 dup_rows=13620 dup_rows_remote=6760 dedup_ratio=0.138550 router_tokens_per_expert=[523, 0, 0, 74, 313, 6029, 0, 0, 3748, 76, 86, 0, 0, 0, 0, 0, 0, 74, 1238, 0, 0, 55, 0, 0, 166, 0, 0, 0, 29, 74, 0, 0, 0, 0, 82, 0, 0, 0, 2351, 285, 0, 4322, 7655, 8132, 0, 0, 20, 0, 0, 0, 0, 74, 0, 0, 421, 7121, 5074, 0, 0, 0, 621, 301, 12, 196] payload_rows_per_dst=[13586, 15202, 16002, 16264, 11870, 40, 990, 10730] dup_rows_per_dst=[292, 6860, 2042, 538, 2210, 0, 0, 1678] combine_rows_per_src=[13586, 15202, 16002, 16264, 11870, 40, 990, 10730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60844 payload_rows_remote=49122 dup_rows=37460 dup_rows_remote=34228 dedup_ratio=0.381063 router_tokens_per_expert=[580, 0, 0, 96, 349, 5775, 0, 0, 3934, 57, 101, 0, 0, 0, 0, 0, 0, 96, 1236, 0, 0, 89, 0, 0, 167, 0, 0, 0, 23, 96, 0, 0, 0, 0, 176, 0, 0, 0, 2458, 290, 0, 4143, 7699, 8150, 0, 0, 22, 0, 0, 0, 0, 96, 0, 0, 326, 7035, 5026, 0, 0, 0, 545, 424, 14, 149] payload_rows_per_dst=[1858, 8184, 2842, 572, 5020, 16384, 14262, 11722] dup_rows_per_dst=[192, 0, 0, 0, 828, 21006, 12202, 3232] combine_rows_per_src=[1858, 8184, 2842, 572, 5020, 16384, 14262, 11722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70430 payload_rows_remote=56004 dup_rows=27874 dup_rows_remote=26178 dedup_ratio=0.283549 router_tokens_per_expert=[520, 0, 0, 65, 319, 6029, 0, 0, 3889, 92, 147, 0, 0, 0, 0, 0, 0, 65, 961, 0, 0, 63, 0, 0, 142, 0, 0, 0, 91, 65, 0, 0, 0, 0, 120, 0, 0, 0, 2787, 309, 0, 4026, 7666, 8132, 0, 0, 28, 0, 0, 0, 0, 65, 0, 0, 376, 7148, 5021, 0, 0, 0, 614, 279, 19, 114] payload_rows_per_dst=[12172, 8256, 2178, 596, 5866, 16384, 14426, 10552] dup_rows_per_dst=[288, 0, 0, 0, 566, 23782, 1696, 1542] combine_rows_per_src=[12172, 8256, 2178, 596, 5866, 16384, 14426, 10552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86576 payload_rows_remote=72558 dup_rows=11728 dup_rows_remote=5392 dedup_ratio=0.119303 router_tokens_per_expert=[598, 0, 0, 90, 337, 5975, 0, 0, 3292, 87, 116, 0, 0, 0, 0, 0, 0, 89, 1410, 0, 0, 56, 0, 0, 148, 0, 0, 0, 19, 90, 0, 0, 0, 0, 172, 0, 0, 0, 2493, 323, 0, 4208, 7742, 8126, 0, 0, 23, 0, 0, 0, 0, 90, 0, 0, 392, 7059, 5144, 0, 0, 0, 581, 311, 2, 179] payload_rows_per_dst=[13672, 6990, 4564, 12532, 9616, 14018, 14298, 10886] dup_rows_per_dst=[328, 0, 258, 428, 2046, 6336, 784, 1548] combine_rows_per_src=[13672, 6990, 4564, 12532, 9616, 14018, 14298, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87250 payload_rows_remote=71248 dup_rows=11054 dup_rows_remote=9128 dedup_ratio=0.112447 router_tokens_per_expert=[607, 0, 0, 82, 346, 6028, 0, 0, 3769, 111, 82, 0, 0, 0, 0, 0, 0, 82, 1198, 0, 0, 79, 0, 0, 134, 0, 0, 0, 91, 82, 0, 0, 0, 0, 78, 0, 0, 0, 2668, 354, 0, 4188, 7605, 8102, 0, 0, 21, 0, 0, 0, 0, 82, 0, 0, 340, 7036, 4865, 0, 0, 0, 659, 285, 0, 178] payload_rows_per_dst=[13716, 11212, 16002, 16204, 11714, 42, 8092, 10268] dup_rows_per_dst=[410, 3182, 1926, 614, 2862, 0, 354, 1706] combine_rows_per_src=[13716, 11212, 16002, 16204, 11714, 42, 8092, 10268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284598272 dispatch_meta_bytes=304968 combine_payload_bytes=284598272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284903240 ingress_bytes=319201280 link_bytes=319201280 token_bytes=569501512 weight_movement_bytes=34603008 total_nvlink_bytes=604104520 weight_to_token_ratio=0.060760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201128719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=201203712 dispatch_meta_bytes=333400 combine_payload_bytes=201203712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=201537112 ingress_bytes=235806720 link_bytes=235806720 token_bytes=402740824 weight_movement_bytes=34603008 total_nvlink_bytes=437343832 weight_to_token_ratio=0.085919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.873217034e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=229392384 dispatch_meta_bytes=328728 combine_payload_bytes=229392384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=229721112 ingress_bytes=298598400 link_bytes=298598400 token_bytes=459113496 weight_movement_bytes=69206016 total_nvlink_bytes=528319512 weight_to_token_ratio=0.150738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84000 payload_rows_remote=70592 dup_rows=14304 dup_rows_remote=13966 dedup_ratio=0.145508 router_tokens_per_expert=[592, 0, 9, 91, 340, 5840, 1, 0, 3791, 70, 162, 0, 0, 0, 1, 2, 0, 91, 946, 6, 4, 91, 5, 0, 163, 0, 0, 3, 60, 91, 0, 0, 3, 0, 199, 0, 0, 0, 2843, 288, 15, 4472, 7605, 8126, 0, 6, 30, 0, 1, 2, 0, 91, 0, 0, 361, 7020, 4577, 8, 0, 2, 588, 395, 46, 115] payload_rows_per_dst=[13408, 15056, 15858, 16258, 12486, 102, 906, 9926] dup_rows_per_dst=[338, 7036, 1638, 628, 3124, 0, 4, 1536] combine_rows_per_src=[13408, 15056, 15858, 16258, 12486, 102, 906, 9926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90682 payload_rows_remote=74404 dup_rows=7622 dup_rows_remote=7056 dedup_ratio=0.077535 router_tokens_per_expert=[595, 0, 0, 69, 330, 6104, 0, 0, 3958, 79, 141, 0, 0, 0, 0, 0, 0, 69, 813, 0, 0, 69, 0, 0, 135, 0, 0, 0, 79, 69, 0, 0, 0, 0, 251, 0, 0, 0, 3392, 313, 7, 3885, 7675, 8139, 0, 0, 57, 0, 0, 0, 0, 69, 0, 0, 402, 7109, 4328, 0, 0, 0, 606, 377, 23, 9] payload_rows_per_dst=[13760, 8356, 15894, 16278, 12574, 128, 14356, 9336] dup_rows_per_dst=[436, 0, 1358, 566, 3108, 0, 804, 1350] combine_rows_per_src=[13760, 8356, 15894, 16278, 12574, 128, 14356, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=297197568 dispatch_meta_bytes=311800 combine_payload_bytes=297197568 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=297509368 ingress_bytes=470212608 link_bytes=470212608 token_bytes=594706936 weight_movement_bytes=173015040 total_nvlink_bytes=767721976 weight_to_token_ratio=0.290925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.769372189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291831808 dispatch_meta_bytes=321504 combine_payload_bytes=291831808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292153312 ingress_bytes=326434816 link_bytes=326434816 token_bytes=583985120 weight_movement_bytes=34603008 total_nvlink_bytes=618588128 weight_to_token_ratio=0.059253 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228347932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92704 payload_rows_remote=77026 dup_rows=5600 dup_rows_remote=5280 dedup_ratio=0.056966 router_tokens_per_expert=[805, 318, 0, 0, 0, 0, 4182, 0, 0, 509, 0, 0, 0, 0, 35, 0, 0, 150, 0, 0, 0, 1090, 0, 0, 0, 0, 7796, 231, 194, 0, 63, 7554, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 6192, 132, 7595, 75, 7370, 0, 3654, 0, 0, 371, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 69] payload_rows_per_dst=[9214, 15812, 15408, 976, 15678, 12538, 7762, 15316] dup_rows_per_dst=[1396, 868, 2180, 0, 320, 260, 438, 138] combine_rows_per_src=[9214, 15812, 15408, 976, 15678, 12538, 7762, 15316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=289144832 dispatch_meta_bytes=338232 combine_payload_bytes=289144832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=289483064 ingress_bytes=358350848 link_bytes=358350848 token_bytes=578627896 weight_movement_bytes=69206016 total_nvlink_bytes=647833912 weight_to_token_ratio=0.119604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348445391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=304758784 dispatch_meta_bytes=325840 combine_payload_bytes=304758784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305084624 ingress_bytes=339361792 link_bytes=339361792 token_bytes=609843408 weight_movement_bytes=34603008 total_nvlink_bytes=644446416 weight_to_token_ratio=0.056741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276991102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=315498496 dispatch_meta_bytes=329224 combine_payload_bytes=315498496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315827720 ingress_bytes=350101504 link_bytes=350101504 token_bytes=631326216 weight_movement_bytes=34603008 total_nvlink_bytes=665929224 weight_to_token_ratio=0.054810 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317403774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92358 payload_rows_remote=76588 dup_rows=5946 dup_rows_remote=5268 dedup_ratio=0.060486 router_tokens_per_expert=[840, 332, 0, 0, 0, 0, 4128, 0, 0, 413, 0, 0, 0, 0, 3, 0, 0, 145, 0, 0, 0, 1095, 0, 0, 0, 0, 7808, 239, 202, 0, 60, 7532, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 5989, 136, 7692, 79, 7426, 0, 3902, 0, 0, 401, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] payload_rows_per_dst=[9112, 15770, 15358, 12238, 15730, 11106, 8290, 4754] dup_rows_per_dst=[1488, 678, 2186, 742, 136, 216, 458, 42] combine_rows_per_src=[9112, 15770, 15358, 12238, 15730, 11106, 8290, 4754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74962 payload_rows_remote=60722 dup_rows=23342 dup_rows_remote=17388 dedup_ratio=0.237447 router_tokens_per_expert=[769, 324, 0, 0, 0, 0, 4188, 0, 0, 540, 0, 0, 0, 0, 14, 0, 0, 128, 0, 0, 0, 1063, 0, 0, 0, 0, 7810, 245, 314, 0, 60, 7579, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 6246, 152, 7617, 87, 7430, 0, 3397, 0, 0, 391, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75] payload_rows_per_dst=[9210, 1108, 2312, 16240, 2100, 14398, 14240, 15354] dup_rows_per_dst=[1352, 0, 70, 15776, 38, 2, 5954, 150] combine_rows_per_src=[9210, 1108, 2312, 16240, 2100, 14398, 14240, 15354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75750 payload_rows_remote=60276 dup_rows=22554 dup_rows_remote=22408 dedup_ratio=0.229431 router_tokens_per_expert=[759, 448, 0, 0, 0, 0, 4693, 0, 0, 376, 0, 0, 0, 0, 3, 0, 0, 191, 0, 0, 0, 992, 0, 0, 0, 0, 7747, 250, 192, 0, 42, 7415, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 6132, 99, 7694, 110, 7412, 0, 3450, 0, 0, 422, 0, 95, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72] payload_rows_per_dst=[10266, 758, 2336, 16186, 1028, 15242, 14460, 15474] dup_rows_per_dst=[1534, 0, 30, 15106, 0, 0, 5738, 146] combine_rows_per_src=[10266, 758, 2336, 16186, 1028, 15242, 14460, 15474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93024 payload_rows_remote=77660 dup_rows=5280 dup_rows_remote=3144 dedup_ratio=0.053711 router_tokens_per_expert=[783, 355, 0, 0, 0, 0, 4477, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1068, 0, 0, 0, 0, 7764, 264, 156, 0, 90, 7518, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 6046, 203, 7655, 88, 7285, 0, 3652, 0, 0, 390, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] payload_rows_per_dst=[9868, 15692, 15364, 3526, 15474, 9766, 7844, 15490] dup_rows_per_dst=[1362, 740, 2136, 156, 148, 246, 404, 88] combine_rows_per_src=[9868, 15692, 15364, 3526, 15474, 9766, 7844, 15490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88336 payload_rows_remote=81878 dup_rows=9968 dup_rows_remote=9806 dedup_ratio=0.101400 router_tokens_per_expert=[785, 439, 0, 0, 0, 0, 4398, 0, 0, 474, 0, 0, 0, 0, 1, 0, 0, 180, 0, 0, 0, 912, 0, 0, 0, 0, 7764, 246, 155, 0, 66, 7415, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 6336, 153, 7618, 91, 7306, 0, 3675, 0, 0, 397, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 60] payload_rows_per_dst=[9642, 12656, 5940, 11442, 15518, 6458, 11312, 15368] dup_rows_per_dst=[1602, 598, 512, 3278, 144, 162, 3552, 120] combine_rows_per_src=[9642, 12656, 5940, 11442, 15518, 6458, 11312, 15368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=313704448 dispatch_meta_bytes=327424 combine_payload_bytes=313704448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314031872 ingress_bytes=348307456 link_bytes=348307456 token_bytes=627736320 weight_movement_bytes=34603008 total_nvlink_bytes=662339328 weight_to_token_ratio=0.055123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310652916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=246890496 dispatch_meta_bytes=330736 combine_payload_bytes=246890496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247221232 ingress_bytes=281493504 link_bytes=281493504 token_bytes=494111728 weight_movement_bytes=34603008 total_nvlink_bytes=528714736 weight_to_token_ratio=0.070031 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.059237393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=248717312 dispatch_meta_bytes=312440 combine_payload_bytes=248717312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249029752 ingress_bytes=283320320 link_bytes=283320320 token_bytes=497747064 weight_movement_bytes=34603008 total_nvlink_bytes=532350072 weight_to_token_ratio=0.069519 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.066111555e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88096 payload_rows_remote=72540 dup_rows=10208 dup_rows_remote=3836 dedup_ratio=0.103841 router_tokens_per_expert=[794, 440, 0, 0, 0, 1, 4012, 0, 2, 576, 2, 0, 5, 0, 96, 0, 1, 183, 0, 0, 0, 878, 0, 1, 0, 0, 7746, 249, 243, 6, 54, 7434, 8, 4, 4, 0, 7, 0, 1, 541, 0, 5, 5916, 123, 7676, 92, 7383, 3, 4119, 0, 3, 318, 0, 91, 0, 0, 0, 0, 0, 9, 56, 0, 0, 70] payload_rows_per_dst=[15556, 15870, 15240, 1484, 15646, 15472, 8558, 270] dup_rows_per_dst=[6372, 984, 1754, 18, 250, 326, 504, 0] combine_rows_per_src=[15556, 15870, 15240, 1484, 15646, 15472, 8558, 270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92694 payload_rows_remote=91558 dup_rows=5610 dup_rows_remote=5610 dedup_ratio=0.057068 router_tokens_per_expert=[816, 361, 0, 0, 0, 0, 3988, 0, 0, 557, 0, 0, 0, 0, 35, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 0, 0, 7793, 260, 255, 0, 53, 7563, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 5855, 132, 7678, 69, 7439, 0, 3964, 0, 0, 362, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66] payload_rows_per_dst=[8844, 15794, 15406, 1136, 15772, 11898, 8382, 15462] dup_rows_per_dst=[1486, 976, 2072, 0, 322, 214, 408, 132] combine_rows_per_src=[8844, 15794, 15406, 1136, 15772, 11898, 8382, 15462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318095360 dispatch_meta_bytes=323216 combine_payload_bytes=318095360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318418576 ingress_bytes=352698368 link_bytes=352698368 token_bytes=636513936 weight_movement_bytes=34603008 total_nvlink_bytes=671116944 weight_to_token_ratio=0.054363 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327175564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96144 payload_rows_remote=79886 dup_rows=2160 dup_rows_remote=2010 dedup_ratio=0.021973 router_tokens_per_expert=[0, 0, 404, 0, 0, 0, 2816, 0, 0, 0, 17, 0, 7347, 0, 0, 7734, 75, 0, 0, 0, 0, 1149, 0, 0, 0, 0, 0, 8117, 0, 0, 8056, 0, 0, 0, 75, 0, 0, 8129, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 506, 4210, 0, 0] payload_rows_per_dst=[6302, 14728, 14188, 2500, 16258, 15882, 16234, 10052] dup_rows_per_dst=[138, 0, 1872, 0, 150, 0, 0, 0] combine_rows_per_src=[6302, 14728, 14188, 2500, 16258, 15882, 16234, 10052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=335372288 dispatch_meta_bytes=366736 combine_payload_bytes=335372288 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=335739024 ingress_bytes=508387328 link_bytes=508387328 token_bytes=671111312 weight_movement_bytes=173015040 total_nvlink_bytes=844126352 weight_to_token_ratio=0.257804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.913020587e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=297123840 dispatch_meta_bytes=305504 combine_payload_bytes=297123840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297429344 ingress_bytes=331726848 link_bytes=331726848 token_bytes=594553184 weight_movement_bytes=34603008 total_nvlink_bytes=629156192 weight_to_token_ratio=0.058200 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.248261423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=375021568 dispatch_meta_bytes=388672 combine_payload_bytes=375021568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=375410240 ingress_bytes=478830592 link_bytes=478830592 token_bytes=750431808 weight_movement_bytes=103809024 total_nvlink_bytes=854240832 weight_to_token_ratio=0.138332 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.801800969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=327213056 dispatch_meta_bytes=327584 combine_payload_bytes=327213056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=327540640 ingress_bytes=431022080 link_bytes=431022080 token_bytes=654753696 weight_movement_bytes=103809024 total_nvlink_bytes=758562720 weight_to_token_ratio=0.158547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621901387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91964 payload_rows_remote=79406 dup_rows=6340 dup_rows_remote=6340 dedup_ratio=0.064494 router_tokens_per_expert=[0, 0, 364, 0, 0, 0, 2778, 0, 0, 0, 21, 0, 7357, 0, 0, 7765, 74, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 8121, 0, 0, 8059, 0, 0, 0, 70, 0, 0, 8132, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 320, 0, 0, 472, 4652, 0, 0] payload_rows_per_dst=[16190, 12558, 16384, 15624, 16264, 3428, 628, 10888] dup_rows_per_dst=[4808, 0, 1392, 0, 140, 0, 0, 0] combine_rows_per_src=[16190, 12558, 16384, 15624, 16264, 3428, 628, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96898 payload_rows_remote=80640 dup_rows=1406 dup_rows_remote=1316 dedup_ratio=0.014303 router_tokens_per_expert=[0, 0, 433, 0, 0, 0, 2606, 0, 0, 0, 62, 0, 7420, 0, 0, 7676, 65, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 0, 8129, 0, 0, 8039, 0, 0, 0, 58, 0, 0, 8132, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 302, 0, 0, 430, 4447, 0, 0] payload_rows_per_dst=[5928, 14964, 2316, 16078, 14534, 15782, 16258, 11038] dup_rows_per_dst=[150, 0, 0, 0, 102, 0, 90, 1064] combine_rows_per_src=[5928, 14964, 2316, 16078, 14534, 15782, 16258, 11038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87574 payload_rows_remote=71190 dup_rows=10730 dup_rows_remote=170 dedup_ratio=0.109151 router_tokens_per_expert=[0, 0, 435, 0, 0, 0, 2918, 0, 0, 0, 16, 0, 7343, 0, 0, 7605, 96, 0, 0, 0, 0, 763, 0, 0, 0, 0, 0, 8097, 0, 0, 8107, 0, 0, 0, 94, 0, 0, 8149, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 428, 0, 0, 477, 4418, 0, 0] payload_rows_per_dst=[6562, 14118, 1718, 16214, 788, 15596, 16194, 16384] dup_rows_per_dst=[144, 0, 0, 0, 0, 2, 24, 10560] combine_rows_per_src=[6562, 14118, 1718, 16214, 788, 15596, 16194, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91442 payload_rows_remote=75058 dup_rows=6862 dup_rows_remote=5042 dedup_ratio=0.069804 router_tokens_per_expert=[0, 0, 389, 0, 0, 0, 2712, 0, 0, 0, 45, 0, 7395, 0, 0, 7608, 82, 0, 0, 0, 0, 1020, 0, 0, 0, 0, 0, 8110, 0, 0, 8000, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 336, 0, 0, 492, 4486, 0, 0] payload_rows_per_dst=[16130, 90, 16384, 0, 16204, 15786, 16220, 10628] dup_rows_per_dst=[4862, 0, 1820, 0, 164, 0, 16, 0] combine_rows_per_src=[16130, 90, 16384, 0, 16204, 15786, 16220, 10628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97990 payload_rows_remote=82362 dup_rows=314 dup_rows_remote=314 dedup_ratio=0.003194 router_tokens_per_expert=[0, 0, 408, 0, 0, 0, 2789, 0, 0, 0, 7, 0, 7372, 0, 0, 7571, 90, 0, 0, 0, 0, 901, 0, 0, 0, 0, 0, 8102, 0, 0, 8044, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 408, 0, 0, 481, 4515, 0, 0] payload_rows_per_dst=[6270, 14758, 1982, 16088, 16252, 15628, 16204, 10808] dup_rows_per_dst=[124, 0, 0, 0, 180, 0, 10, 0] combine_rows_per_src=[6270, 14758, 1982, 16088, 16252, 15628, 16204, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=325246976 dispatch_meta_bytes=342984 combine_payload_bytes=325246976 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325589960 ingress_bytes=429056000 link_bytes=429056000 token_bytes=650836936 weight_movement_bytes=103809024 total_nvlink_bytes=754645960 weight_to_token_ratio=0.159501 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.614503186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330301440 dispatch_meta_bytes=327824 combine_payload_bytes=330301440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330629264 ingress_bytes=364904448 link_bytes=364904448 token_bytes=660930704 weight_movement_bytes=34603008 total_nvlink_bytes=695533712 weight_to_token_ratio=0.052355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373106061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=291594240 dispatch_meta_bytes=285440 combine_payload_bytes=291594240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291879680 ingress_bytes=326197248 link_bytes=326197248 token_bytes=583473920 weight_movement_bytes=34603008 total_nvlink_bytes=618076928 weight_to_token_ratio=0.059305 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227453983e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92650 payload_rows_remote=92650 dup_rows=5654 dup_rows_remote=5654 dedup_ratio=0.057515 router_tokens_per_expert=[0, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 68, 0, 7460, 0, 0, 7666, 69, 0, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 8123, 0, 0, 8019, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 339, 0, 0, 457, 3079, 0, 0] payload_rows_per_dst=[8592, 11668, 16382, 0, 16278, 15734, 16246, 7750] dup_rows_per_dst=[1218, 0, 4146, 0, 138, 0, 152, 0] combine_rows_per_src=[8592, 11668, 16382, 0, 16278, 15734, 16246, 7750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74626 payload_rows_remote=58626 dup_rows=23678 dup_rows_remote=18792 dedup_ratio=0.240865 router_tokens_per_expert=[1, 0, 402, 0, 0, 0, 2689, 0, 0, 0, 87, 0, 7351, 0, 9, 7614, 93, 0, 0, 0, 2, 1712, 1, 6, 0, 0, 0, 8084, 0, 9, 8046, 1, 0, 0, 91, 0, 0, 8128, 8, 5, 1, 0, 214, 0, 1, 11, 6, 0, 0, 2, 0, 1, 0, 6, 5, 0, 0, 389, 0, 1, 497, 3679, 0, 0] payload_rows_per_dst=[16000, 15420, 16368, 16380, 850, 452, 28, 9128] dup_rows_per_dst=[4886, 0, 3352, 15420, 2, 14, 0, 4] combine_rows_per_src=[16000, 15420, 16368, 16380, 850, 452, 28, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=307437568 dispatch_meta_bytes=320400 combine_payload_bytes=307437568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307757968 ingress_bytes=342040576 link_bytes=342040576 token_bytes=615195536 weight_movement_bytes=34603008 total_nvlink_bytes=649798544 weight_to_token_ratio=0.056247 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287071151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337354752 dispatch_meta_bytes=330704 combine_payload_bytes=337354752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337685456 ingress_bytes=371957760 link_bytes=371957760 token_bytes=675040208 weight_movement_bytes=34603008 total_nvlink_bytes=709643216 weight_to_token_ratio=0.051261 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399647106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93674 payload_rows_remote=91452 dup_rows=4630 dup_rows_remote=4480 dedup_ratio=0.047099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 170, 0, 2089, 0, 150, 7978, 8192, 0, 0, 0, 0, 0, 697, 0, 7710, 0, 63, 0, 7892, 0, 0, 0, 0, 1111, 7187, 75, 0, 0, 0, 0, 93, 290, 0, 0, 0, 3425, 0, 47, 0, 0, 0, 0, 0, 0, 63, 0, 75, 0, 1088, 344] payload_rows_per_dst=[16384, 16126, 4478, 16234, 2222, 15910, 6944, 15376] dup_rows_per_dst=[0, 996, 0, 706, 150, 640, 0, 2138] combine_rows_per_src=[16384, 16126, 4478, 16234, 2222, 15910, 6944, 15376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=379494400 dispatch_meta_bytes=393216 combine_payload_bytes=379494400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=379887616 ingress_bytes=483303424 link_bytes=483303424 token_bytes=759382016 weight_movement_bytes=103809024 total_nvlink_bytes=863191040 weight_to_token_ratio=0.136702 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.818631875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=240132096 dispatch_meta_bytes=309672 combine_payload_bytes=240132096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240441768 ingress_bytes=274735104 link_bytes=274735104 token_bytes=480573864 weight_movement_bytes=34603008 total_nvlink_bytes=515176872 weight_to_token_ratio=0.072004 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.033806078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=374587392 dispatch_meta_bytes=383728 combine_payload_bytes=374587392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=374971120 ingress_bytes=478396416 link_bytes=478396416 token_bytes=749558512 weight_movement_bytes=103809024 total_nvlink_bytes=853367536 weight_to_token_ratio=0.138494 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.800167199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80238 payload_rows_remote=64056 dup_rows=18066 dup_rows_remote=16384 dedup_ratio=0.183777 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 497, 0, 2054, 0, 145, 7966, 8192, 0, 0, 0, 0, 0, 876, 0, 7710, 0, 60, 0, 7891, 0, 0, 0, 0, 680, 7091, 74, 0, 0, 0, 0, 93, 337, 0, 0, 0, 3368, 0, 76, 0, 0, 0, 0, 0, 0, 60, 0, 71, 0, 1093, 349] payload_rows_per_dst=[16384, 16182, 4398, 16236, 16264, 860, 6888, 3026] dup_rows_per_dst=[0, 1682, 0, 1056, 15208, 0, 0, 120] combine_rows_per_src=[16384, 16182, 4398, 16236, 16264, 860, 6888, 3026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68326 payload_rows_remote=52192 dup_rows=29978 dup_rows_remote=23496 dedup_ratio=0.304952 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 283, 0, 2115, 0, 128, 8016, 8192, 0, 0, 0, 0, 0, 749, 0, 7731, 0, 60, 0, 7898, 0, 0, 0, 0, 981, 7190, 65, 0, 0, 0, 0, 51, 244, 0, 1, 0, 3514, 0, 63, 0, 0, 0, 0, 0, 0, 60, 10, 63, 0, 972, 337] payload_rows_per_dst=[0, 954, 16384, 1618, 1962, 15916, 16134, 15358] dup_rows_per_dst=[0, 470, 20518, 0, 130, 472, 6482, 1906] combine_rows_per_src=[0, 954, 16384, 1618, 1962, 15916, 16134, 15358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76988 payload_rows_remote=61532 dup_rows=21316 dup_rows_remote=19258 dedup_ratio=0.216838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 488, 0, 1670, 0, 191, 7828, 8192, 0, 0, 0, 0, 0, 787, 0, 7717, 0, 42, 0, 7931, 0, 0, 0, 0, 657, 7231, 96, 0, 0, 0, 0, 102, 247, 0, 0, 0, 4001, 0, 54, 0, 0, 0, 0, 0, 0, 42, 0, 95, 0, 1052, 337] payload_rows_per_dst=[0, 1130, 16384, 5408, 6556, 15946, 16108, 15456] dup_rows_per_dst=[0, 630, 9484, 470, 624, 614, 7436, 2058] combine_rows_per_src=[0, 1130, 16384, 5408, 6556, 15946, 16108, 15456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92472 payload_rows_remote=87970 dup_rows=5832 dup_rows_remote=5832 dedup_ratio=0.059326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 190, 0, 2086, 0, 165, 7961, 8192, 0, 0, 0, 0, 0, 737, 0, 7659, 0, 90, 0, 7871, 0, 0, 0, 0, 1046, 7129, 82, 0, 0, 0, 0, 139, 269, 0, 0, 0, 3424, 0, 107, 0, 0, 0, 0, 0, 0, 90, 5, 82, 0, 1080, 361] payload_rows_per_dst=[16384, 16148, 4502, 16220, 13718, 14088, 7062, 4350] dup_rows_per_dst=[0, 928, 0, 752, 3208, 560, 0, 384] combine_rows_per_src=[16384, 16148, 4502, 16220, 13718, 14088, 7062, 4350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87498 payload_rows_remote=71666 dup_rows=10806 dup_rows_remote=10194 dedup_ratio=0.109924 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 403, 0, 1904, 0, 180, 7851, 8192, 0, 0, 0, 0, 0, 723, 0, 7733, 0, 66, 0, 7850, 0, 0, 0, 0, 858, 7161, 90, 0, 0, 0, 0, 105, 267, 0, 0, 0, 3513, 0, 84, 0, 0, 0, 0, 0, 1, 66, 1, 90, 0, 1232, 303] payload_rows_per_dst=[16370, 8136, 10828, 5786, 1716, 15832, 13526, 15304] dup_rows_per_dst=[0, 868, 1816, 186, 180, 612, 4740, 2404] combine_rows_per_src=[16370, 8136, 10828, 5786, 1716, 15832, 13526, 15304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=321760 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262695136 ingress_bytes=296976384 link_bytes=296976384 token_bytes=525068512 weight_movement_bytes=34603008 total_nvlink_bytes=559671520 weight_to_token_ratio=0.065902 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117498224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=213778432 dispatch_meta_bytes=302752 combine_payload_bytes=213778432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214081184 ingress_bytes=248381440 link_bytes=248381440 token_bytes=427859616 weight_movement_bytes=34603008 total_nvlink_bytes=462462624 weight_to_token_ratio=0.080875 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.346393624e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94606 payload_rows_remote=78360 dup_rows=3698 dup_rows_remote=3080 dedup_ratio=0.037618 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 1, 110, 0, 2030, 0, 138, 7976, 8192, 0, 0, 0, 0, 0, 654, 0, 7725, 0, 53, 0, 7874, 0, 0, 0, 0, 1538, 7237, 69, 0, 0, 0, 0, 4, 184, 0, 4, 0, 3510, 0, 1, 0, 0, 0, 0, 0, 0, 53, 0, 69, 0, 913, 345] payload_rows_per_dst=[16384, 16244, 4336, 16246, 3076, 15854, 7022, 15444] dup_rows_per_dst=[0, 874, 0, 618, 138, 278, 0, 1790] combine_rows_per_src=[16384, 16244, 4336, 16246, 3076, 15854, 7022, 15444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=252035072 dispatch_meta_bytes=323160 combine_payload_bytes=252035072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252358232 ingress_bytes=286638080 link_bytes=286638080 token_bytes=504393304 weight_movement_bytes=34603008 total_nvlink_bytes=538996312 weight_to_token_ratio=0.068603 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078596019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79828 payload_rows_remote=63448 dup_rows=18476 dup_rows_remote=18458 dedup_ratio=0.187948 router_tokens_per_expert=[0, 0, 10, 0, 10, 0, 1, 4, 0, 0, 438, 3, 0, 1, 172, 0, 2391, 1, 191, 7854, 8174, 0, 5, 0, 0, 0, 643, 0, 7685, 1, 55, 0, 7874, 2, 4, 0, 1, 1390, 7214, 91, 1, 5, 0, 8, 65, 282, 0, 0, 0, 2949, 0, 25, 0, 7, 0, 0, 4, 0, 72, 2, 91, 9, 1072, 345] payload_rows_per_dst=[16380, 16140, 5176, 16172, 16248, 716, 5962, 3034] dup_rows_per_dst=[18, 796, 0, 596, 16904, 6, 0, 156] combine_rows_per_src=[16380, 16140, 5176, 16172, 16248, 716, 5962, 3034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94520 payload_rows_remote=78426 dup_rows=3784 dup_rows_remote=3232 dedup_ratio=0.038493 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 68, 452, 8047, 1119, 0, 0, 7979, 0, 0, 7523, 64, 63, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 6750, 0, 197, 0, 6806, 75, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[15958, 15172, 3128, 254, 16094, 15632, 14520, 13762] dup_rows_per_dst=[2, 670, 150, 0, 552, 0, 2260, 150] combine_rows_per_src=[15958, 15172, 3128, 254, 16094, 15632, 14520, 13762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=360325120 dispatch_meta_bytes=375208 combine_payload_bytes=360325120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=360700328 ingress_bytes=498737152 link_bytes=498737152 token_bytes=721025448 weight_movement_bytes=138412032 total_nvlink_bytes=859437480 weight_to_token_ratio=0.191966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.876707751e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293543936 dispatch_meta_bytes=327440 combine_payload_bytes=293543936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293871376 ingress_bytes=328146944 link_bytes=328146944 token_bytes=587415312 weight_movement_bytes=34603008 total_nvlink_bytes=622018320 weight_to_token_ratio=0.058907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234790532e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320962560 dispatch_meta_bytes=325760 combine_payload_bytes=320962560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321288320 ingress_bytes=390168576 link_bytes=390168576 token_bytes=642250880 weight_movement_bytes=69206016 total_nvlink_bytes=711456896 weight_to_token_ratio=0.107755 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468172940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259883008 dispatch_meta_bytes=327624 combine_payload_bytes=259883008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260210632 ingress_bytes=294486016 link_bytes=294486016 token_bytes=520093640 weight_movement_bytes=34603008 total_nvlink_bytes=554696648 weight_to_token_ratio=0.066532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108127170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=321232896 dispatch_meta_bytes=326632 combine_payload_bytes=321232896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321559528 ingress_bytes=355835904 link_bytes=355835904 token_bytes=642792424 weight_movement_bytes=34603008 total_nvlink_bytes=677395432 weight_to_token_ratio=0.053832 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338981860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78692 payload_rows_remote=63578 dup_rows=19612 dup_rows_remote=18924 dedup_ratio=0.199504 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 85, 471, 8055, 1101, 0, 0, 7987, 0, 0, 7497, 81, 60, 0, 0, 0, 0, 156, 0, 0, 0, 0, 151, 0, 7810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 6721, 0, 197, 12, 7015, 71, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[15974, 15114, 16384, 13724, 614, 0, 2710, 14172] dup_rows_per_dst=[2, 688, 18660, 0, 0, 0, 120, 142] combine_rows_per_src=[15974, 15114, 16384, 13724, 614, 0, 2710, 14172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82534 payload_rows_remote=67936 dup_rows=15770 dup_rows_remote=13446 dedup_ratio=0.160421 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 52, 455, 8043, 1137, 0, 0, 8004, 0, 0, 7507, 48, 60, 0, 0, 0, 0, 111, 0, 0, 0, 0, 144, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 6781, 0, 205, 9, 6751, 63, 0, 0, 0, 63, 0, 0] payload_rows_per_dst=[2666, 2644, 3162, 14096, 16086, 15654, 14598, 13628] dup_rows_per_dst=[2, 88, 126, 12594, 510, 0, 2324, 126] combine_rows_per_src=[2666, 2644, 3162, 14096, 16086, 15654, 14598, 13628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79976 payload_rows_remote=65966 dup_rows=18328 dup_rows_remote=18138 dedup_ratio=0.186442 router_tokens_per_expert=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 181, 507, 8038, 1061, 0, 0, 7959, 0, 0, 7461, 174, 42, 0, 0, 0, 0, 269, 0, 0, 0, 0, 107, 0, 7722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 6690, 0, 179, 9, 6909, 95, 0, 0, 0, 96, 0, 0] payload_rows_per_dst=[20, 780, 3308, 16002, 16076, 15444, 14336, 14010] dup_rows_per_dst=[0, 0, 190, 15270, 752, 0, 1926, 190] combine_rows_per_src=[20, 780, 3308, 16002, 16076, 15444, 14336, 14010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94576 payload_rows_remote=79134 dup_rows=3728 dup_rows_remote=3728 dedup_ratio=0.037923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 169, 477, 8058, 1098, 0, 0, 7946, 0, 0, 7496, 161, 66, 0, 0, 0, 0, 254, 0, 0, 0, 0, 157, 0, 7721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 6512, 0, 158, 7, 6930, 90, 0, 0, 0, 90, 0, 0] payload_rows_per_dst=[15892, 15124, 3308, 454, 16116, 15442, 14200, 14040] dup_rows_per_dst=[0, 724, 180, 0, 822, 0, 1822, 180] combine_rows_per_src=[15892, 15124, 3308, 454, 16116, 15442, 14200, 14040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91916 payload_rows_remote=85282 dup_rows=6388 dup_rows_remote=1948 dedup_ratio=0.064982 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 0, 0, 0, 75, 467, 8065, 1157, 0, 0, 7938, 0, 0, 7431, 72, 90, 0, 0, 0, 0, 159, 0, 0, 0, 0, 213, 0, 7767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 6681, 0, 239, 24, 6774, 82, 0, 0, 0, 82, 0, 0] payload_rows_per_dst=[15876, 15042, 6634, 11786, 12694, 11512, 4660, 13712] dup_rows_per_dst=[0, 806, 4440, 0, 526, 0, 452, 164] combine_rows_per_src=[15876, 15042, 6634, 11786, 12694, 11512, 4660, 13712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=260415488 dispatch_meta_bytes=330008 combine_payload_bytes=260415488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260745496 ingress_bytes=295018496 link_bytes=295018496 token_bytes=521160984 weight_movement_bytes=34603008 total_nvlink_bytes=555763992 weight_to_token_ratio=0.066396 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110130850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=278265856 dispatch_meta_bytes=325528 combine_payload_bytes=278265856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278591384 ingress_bytes=347471872 link_bytes=347471872 token_bytes=556857240 weight_movement_bytes=69206016 total_nvlink_bytes=626063256 weight_to_token_ratio=0.124280 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307508681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93724 payload_rows_remote=93462 dup_rows=4580 dup_rows_remote=4580 dedup_ratio=0.046590 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 80, 421, 8060, 1216, 0, 0, 8001, 0, 0, 7462, 78, 53, 0, 0, 0, 0, 147, 0, 0, 0, 0, 140, 0, 7791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 6692, 0, 189, 47, 6341, 69, 0, 0, 0, 72, 0, 0] payload_rows_per_dst=[16002, 15030, 3296, 262, 16120, 15582, 14606, 12826] dup_rows_per_dst=[4, 720, 138, 0, 574, 0, 3006, 138] combine_rows_per_src=[16002, 15030, 3296, 262, 16120, 15582, 14606, 12826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=270196736 dispatch_meta_bytes=336416 combine_payload_bytes=270196736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270533152 ingress_bytes=339402752 link_bytes=339402752 token_bytes=540729888 weight_movement_bytes=69206016 total_nvlink_bytes=609935904 weight_to_token_ratio=0.127986 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277145232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96254 payload_rows_remote=88110 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.020854 router_tokens_per_expert=[0, 697, 315, 1194, 7985, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5282, 0, 0, 4530, 0, 0, 381, 0, 0, 0, 0, 2, 7802, 139, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 4072, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 288, 0, 0, 0, 0, 0, 7984, 75, 139, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3918, 10564, 9680, 15604, 8144, 15970, 16118, 16256] dup_rows_per_dst=[760, 0, 142, 282, 0, 394, 426, 46] combine_rows_per_src=[3918, 10564, 9680, 15604, 8144, 15970, 16118, 16256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78168 payload_rows_remote=62276 dup_rows=20136 dup_rows_remote=20122 dedup_ratio=0.204834 router_tokens_per_expert=[0, 0, 7, 8, 3, 4, 0, 0, 0, 0, 0, 2, 1, 0, 398, 11, 0, 0, 142, 499, 8040, 1182, 1, 0, 7931, 7, 0, 7439, 140, 54, 0, 0, 0, 7, 244, 8, 0, 0, 0, 125, 7, 7736, 1, 10, 0, 0, 0, 0, 1, 1, 1, 1696, 6730, 1, 220, 31, 6269, 94, 0, 0, 10, 91, 0, 0] payload_rows_per_dst=[15892, 15010, 16360, 13852, 756, 9022, 6394, 882] dup_rows_per_dst=[14, 692, 18840, 10, 12, 16, 370, 182] combine_rows_per_src=[15892, 15010, 16360, 13852, 756, 9022, 6394, 882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324132864 dispatch_meta_bytes=331448 combine_payload_bytes=324132864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324464312 ingress_bytes=393338880 link_bytes=393338880 token_bytes=648597176 weight_movement_bytes=69206016 total_nvlink_bytes=717803192 weight_to_token_ratio=0.106701 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.480102539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=349315072 dispatch_meta_bytes=348920 combine_payload_bytes=349315072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=349663992 ingress_bytes=418521088 link_bytes=418521088 token_bytes=698979064 weight_movement_bytes=69206016 total_nvlink_bytes=768185080 weight_to_token_ratio=0.099010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.574861160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=382820352 dispatch_meta_bytes=392168 combine_payload_bytes=382820352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=383212520 ingress_bytes=486629376 link_bytes=486629376 token_bytes=766032872 weight_movement_bytes=103809024 total_nvlink_bytes=869841896 weight_to_token_ratio=0.135515 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.831147165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=360898560 dispatch_meta_bytes=360640 combine_payload_bytes=360898560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=361259200 ingress_bytes=430104576 link_bytes=430104576 token_bytes=722157760 weight_movement_bytes=69206016 total_nvlink_bytes=791363776 weight_to_token_ratio=0.095832 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.618448893e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=255082496 dispatch_meta_bytes=329592 combine_payload_bytes=255082496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255412088 ingress_bytes=289685504 link_bytes=289685504 token_bytes=510494584 weight_movement_bytes=34603008 total_nvlink_bytes=545097592 weight_to_token_ratio=0.067783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090063230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66010 payload_rows_remote=55372 dup_rows=32294 dup_rows_remote=32294 dedup_ratio=0.328512 router_tokens_per_expert=[0, 645, 313, 1258, 7969, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 4193, 0, 0, 378, 0, 0, 0, 0, 0, 7790, 142, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 4442, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 268, 0, 0, 0, 0, 0, 7964, 71, 143, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10638, 15908, 5756, 16046, 314, 536, 428] dup_rows_per_dst=[14226, 0, 8814, 0, 9092, 162, 0, 0] combine_rows_per_src=[16384, 10638, 15908, 5756, 16046, 314, 536, 428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96448 payload_rows_remote=80344 dup_rows=1856 dup_rows_remote=1484 dedup_ratio=0.018880 router_tokens_per_expert=[0, 745, 346, 1176, 7994, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5378, 0, 0, 4297, 0, 0, 366, 0, 0, 0, 0, 0, 7798, 111, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 4259, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 249, 0, 0, 0, 0, 0, 7989, 63, 131, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4022, 10756, 9176, 15596, 8518, 15988, 16104, 16288] dup_rows_per_dst=[676, 0, 150, 222, 0, 376, 372, 60] combine_rows_per_src=[4022, 10756, 9176, 15596, 8518, 15988, 16104, 16288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95782 payload_rows_remote=79728 dup_rows=2522 dup_rows_remote=2486 dedup_ratio=0.025655 router_tokens_per_expert=[0, 541, 311, 1333, 7983, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5503, 0, 0, 3957, 0, 0, 385, 0, 0, 0, 0, 0, 7772, 258, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 4445, 0, 0, 0, 231, 0, 0, 0, 170, 0, 0, 246, 0, 0, 0, 0, 0, 7866, 95, 151, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3860, 11006, 8540, 15544, 8890, 15966, 15922, 16054] dup_rows_per_dst=[722, 0, 144, 516, 0, 802, 302, 36] combine_rows_per_src=[3860, 11006, 8540, 15544, 8890, 15966, 15922, 16054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93112 payload_rows_remote=82158 dup_rows=5192 dup_rows_remote=3214 dedup_ratio=0.052816 router_tokens_per_expert=[0, 701, 370, 1261, 7976, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5239, 0, 0, 4345, 0, 0, 359, 0, 0, 0, 0, 0, 7809, 149, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 4215, 0, 0, 0, 185, 0, 0, 0, 72, 0, 0, 264, 0, 0, 0, 0, 0, 7925, 82, 181, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5756, 10478, 10954, 12160, 8880, 13900, 14940, 16044] dup_rows_per_dst=[1278, 0, 1978, 232, 868, 458, 320, 58] combine_rows_per_src=[5756, 10478, 10954, 12160, 8880, 13900, 14940, 16044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95594 payload_rows_remote=79642 dup_rows=2710 dup_rows_remote=1892 dedup_ratio=0.027568 router_tokens_per_expert=[0, 709, 333, 1206, 7976, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 0, 0, 4465, 0, 0, 392, 0, 0, 0, 0, 4, 7796, 222, 0, 0, 0, 0, 0, 7840, 0, 0, 0, 4055, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 280, 0, 0, 0, 0, 0, 7854, 90, 157, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4000, 10396, 9594, 15592, 8110, 15952, 15888, 16062] dup_rows_per_dst=[828, 0, 120, 452, 0, 818, 380, 112] combine_rows_per_src=[4000, 10396, 9594, 15592, 8110, 15952, 15888, 16062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=226803712 dispatch_meta_bytes=350664 combine_payload_bytes=226803712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=227154376 ingress_bytes=261406720 link_bytes=261406720 token_bytes=453958088 weight_movement_bytes=34603008 total_nvlink_bytes=488561096 weight_to_token_ratio=0.076225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.836524424e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=329089024 dispatch_meta_bytes=327312 combine_payload_bytes=329089024 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=329416336 ingress_bytes=432898048 link_bytes=432898048 token_bytes=658505360 weight_movement_bytes=103809024 total_nvlink_bytes=762314384 weight_to_token_ratio=0.157643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.628960503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96330 payload_rows_remote=90456 dup_rows=1974 dup_rows_remote=1922 dedup_ratio=0.020081 router_tokens_per_expert=[7990, 0, 0, 0, 0, 0, 0, 1865, 387, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 306, 110, 64, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7445, 0, 0, 197, 0, 0, 0, 0, 246, 0, 7684, 0, 0, 0, 0, 0, 209, 75, 0, 0, 75] payload_rows_per_dst=[3730, 12990, 10864, 15980, 5874, 15144, 15368, 16380] dup_rows_per_dst=[0, 0, 572, 0, 52, 140, 492, 718] combine_rows_per_src=[3730, 12990, 10864, 15980, 5874, 15144, 15368, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96176 payload_rows_remote=80592 dup_rows=2128 dup_rows_remote=1832 dedup_ratio=0.021647 router_tokens_per_expert=[0, 709, 339, 1332, 8004, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5330, 0, 0, 4829, 0, 0, 370, 0, 0, 0, 0, 1, 7792, 147, 0, 0, 0, 0, 0, 7960, 0, 0, 0, 3566, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 251, 0, 0, 0, 0, 0, 7966, 69, 130, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4230, 10660, 10264, 15584, 7132, 16008, 16070, 16228] dup_rows_per_dst=[774, 0, 134, 296, 0, 470, 364, 90] combine_rows_per_src=[4230, 10660, 10264, 15584, 7132, 16008, 16070, 16228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=326565888 dispatch_meta_bytes=328856 combine_payload_bytes=326565888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326894744 ingress_bytes=361168896 link_bytes=361168896 token_bytes=653460632 weight_movement_bytes=34603008 total_nvlink_bytes=688063640 weight_to_token_ratio=0.052953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359049479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60138 payload_rows_remote=43778 dup_rows=38166 dup_rows_remote=17656 dedup_ratio=0.388245 router_tokens_per_expert=[0, 902, 318, 1216, 7958, 164, 1, 0, 2, 0, 5, 0, 4, 0, 7, 5204, 10, 0, 4903, 0, 7, 363, 0, 0, 0, 0, 6, 7751, 220, 0, 8, 0, 0, 0, 7832, 0, 0, 0, 3480, 0, 0, 0, 209, 0, 5, 1, 142, 6, 2, 265, 3, 0, 0, 11, 0, 7876, 91, 169, 0, 9, 0, 0, 1, 1] payload_rows_per_dst=[16360, 14804, 11232, 468, 15736, 442, 558, 538] dup_rows_per_dst=[20510, 9002, 1474, 0, 6888, 284, 4, 4] combine_rows_per_src=[16360, 14804, 11232, 468, 15736, 442, 558, 538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=336519168 dispatch_meta_bytes=341488 combine_payload_bytes=336519168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336860656 ingress_bytes=371122176 link_bytes=371122176 token_bytes=673379824 weight_movement_bytes=34603008 total_nvlink_bytes=707982832 weight_to_token_ratio=0.051387 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=326213632 dispatch_meta_bytes=326136 combine_payload_bytes=326213632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326539768 ingress_bytes=360816640 link_bytes=360816640 token_bytes=652753400 weight_movement_bytes=34603008 total_nvlink_bytes=687356408 weight_to_token_ratio=0.053011 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357723968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=370507776 dispatch_meta_bytes=369512 combine_payload_bytes=370507776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=370877288 ingress_bytes=439713792 link_bytes=439713792 token_bytes=741385064 weight_movement_bytes=69206016 total_nvlink_bytes=810591080 weight_to_token_ratio=0.093347 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654607599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=330104832 dispatch_meta_bytes=329696 combine_payload_bytes=330104832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=330434528 ingress_bytes=433913856 link_bytes=433913856 token_bytes=660539360 weight_movement_bytes=103809024 total_nvlink_bytes=764348384 weight_to_token_ratio=0.157158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.632782907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=179314688 dispatch_meta_bytes=245736 combine_payload_bytes=179314688 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=179560424 ingress_bytes=248520704 link_bytes=248520704 token_bytes=358875112 weight_movement_bytes=69206016 total_nvlink_bytes=428081128 weight_to_token_ratio=0.192842 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.351634016e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96864 payload_rows_remote=81404 dup_rows=1440 dup_rows_remote=932 dedup_ratio=0.014648 router_tokens_per_expert=[8021, 0, 0, 0, 0, 0, 0, 2028, 377, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 161, 28, 48, 0, 7750, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 7486, 0, 0, 184, 0, 0, 0, 0, 254, 0, 7730, 0, 0, 0, 0, 0, 197, 63, 0, 0, 65] payload_rows_per_dst=[4056, 13018, 474, 16042, 16254, 15190, 15460, 16370] dup_rows_per_dst=[0, 4, 0, 0, 128, 150, 508, 650] combine_rows_per_src=[4056, 13018, 474, 16042, 16254, 15190, 15460, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75676 payload_rows_remote=62974 dup_rows=22628 dup_rows_remote=22628 dedup_ratio=0.230184 router_tokens_per_expert=[7980, 0, 0, 0, 0, 0, 0, 2228, 421, 0, 5930, 0, 0, 0, 0, 0, 0, 0, 0, 180, 59, 81, 0, 7706, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7418, 0, 0, 217, 0, 0, 0, 0, 239, 0, 7655, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74] payload_rows_per_dst=[16222, 12702, 15896, 2882, 16384, 10386, 478, 726] dup_rows_per_dst=[16622, 0, 4914, 0, 966, 126, 0, 0] combine_rows_per_src=[16222, 12702, 15896, 2882, 16384, 10386, 478, 726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88982 payload_rows_remote=72600 dup_rows=9322 dup_rows_remote=8282 dedup_ratio=0.094828 router_tokens_per_expert=[7881, 0, 0, 0, 0, 0, 0, 2050, 326, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 0, 169, 96, 174, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7454, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7520, 0, 0, 0, 0, 0, 330, 95, 0, 0, 95] payload_rows_per_dst=[4100, 9316, 878, 15762, 16192, 10814, 15538, 16382] dup_rows_per_dst=[0, 0, 0, 0, 190, 64, 8028, 1040] combine_rows_per_src=[4100, 9316, 878, 15762, 16192, 10814, 15538, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95418 payload_rows_remote=79660 dup_rows=2886 dup_rows_remote=2440 dedup_ratio=0.029358 router_tokens_per_expert=[7948, 0, 0, 0, 0, 0, 0, 2116, 340, 0, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 221, 79, 72, 0, 7730, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7491, 0, 0, 180, 0, 0, 0, 0, 264, 0, 7645, 0, 0, 0, 0, 0, 199, 82, 0, 0, 82] payload_rows_per_dst=[8654, 12778, 15758, 15896, 6894, 15254, 9830, 10354] dup_rows_per_dst=[1212, 0, 446, 0, 368, 88, 354, 418] combine_rows_per_src=[8654, 12778, 15758, 15896, 6894, 15254, 9830, 10354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96554 payload_rows_remote=81440 dup_rows=1750 dup_rows_remote=1630 dedup_ratio=0.017802 router_tokens_per_expert=[7875, 0, 0, 0, 0, 0, 0, 2088, 392, 0, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 243, 77, 161, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7453, 0, 0, 164, 0, 0, 0, 0, 262, 0, 7544, 0, 0, 0, 0, 0, 294, 90, 0, 0, 89] payload_rows_per_dst=[4176, 12846, 962, 15750, 16206, 15114, 15116, 16384] dup_rows_per_dst=[0, 0, 0, 0, 188, 120, 496, 946] combine_rows_per_src=[4176, 12846, 962, 15750, 16206, 15114, 15116, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=333430784 dispatch_meta_bytes=329344 combine_payload_bytes=333430784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333760128 ingress_bytes=402636800 link_bytes=402636800 token_bytes=667190912 weight_movement_bytes=69206016 total_nvlink_bytes=736396928 weight_to_token_ratio=0.103727 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515089863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96194 payload_rows_remote=80624 dup_rows=2110 dup_rows_remote=2034 dedup_ratio=0.021464 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 40, 0, 0, 365, 8, 0, 0, 0, 63, 0, 130, 0, 0, 0, 0, 0, 0, 6214, 7785, 0, 18, 0, 0, 0, 20, 0, 79, 108, 0, 0, 0, 2819, 8192, 0, 0, 75, 0, 0, 206, 0, 75, 0, 0, 0, 6646, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16234, 16384, 12428, 15570, 5902, 16384, 13292] dup_rows_per_dst=[0, 80, 872, 260, 76, 110, 712, 0] combine_rows_per_src=[0, 16234, 16384, 12428, 15570, 5902, 16384, 13292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=257941504 dispatch_meta_bytes=342408 combine_payload_bytes=257941504 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=258283912 ingress_bytes=292544512 link_bytes=292544512 token_bytes=516225416 weight_movement_bytes=34603008 total_nvlink_bytes=550828424 weight_to_token_ratio=0.067031 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100821447e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96208 payload_rows_remote=80224 dup_rows=2096 dup_rows_remote=2096 dedup_ratio=0.021322 router_tokens_per_expert=[7992, 0, 0, 0, 0, 0, 0, 2132, 402, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 350, 30, 82, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 7449, 0, 0, 182, 0, 0, 0, 0, 263, 0, 7785, 0, 0, 0, 0, 0, 211, 69, 0, 0, 69] payload_rows_per_dst=[4264, 12288, 15622, 15984, 1014, 15130, 15570, 16336] dup_rows_per_dst=[0, 0, 740, 0, 0, 132, 526, 698] combine_rows_per_src=[4264, 12288, 15622, 15984, 1014, 15130, 15570, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=297369600 dispatch_meta_bytes=323528 combine_payload_bytes=297369600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297693128 ingress_bytes=331972608 link_bytes=331972608 token_bytes=595062728 weight_movement_bytes=34603008 total_nvlink_bytes=629665736 weight_to_token_ratio=0.058150 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.037500 gemm_active_groups=6 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.249186198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63008 payload_rows_remote=46910 dup_rows=35296 dup_rows_remote=16126 dedup_ratio=0.359049 router_tokens_per_expert=[7894, 7, 0, 0, 0, 0, 9, 2146, 359, 11, 5810, 0, 1, 0, 0, 0, 0, 1, 1, 269, 120, 145, 0, 7631, 0, 0, 0, 0, 0, 2, 0, 2, 612, 7, 1, 0, 2, 1, 0, 8174, 0, 11, 0, 7422, 7, 9, 199, 0, 0, 0, 0, 261, 4, 7578, 0, 5, 0, 0, 0, 262, 93, 5, 0, 91] payload_rows_per_dst=[16098, 12358, 16322, 8, 16366, 420, 534, 902] dup_rows_per_dst=[19170, 4, 14856, 0, 1228, 32, 6, 0] combine_rows_per_src=[16098, 12358, 16322, 8, 16366, 420, 534, 902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=326287360 dispatch_meta_bytes=328400 combine_payload_bytes=326287360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326615760 ingress_bytes=395493376 link_bytes=395493376 token_bytes=652903120 weight_movement_bytes=69206016 total_nvlink_bytes=722109136 weight_to_token_ratio=0.105997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488209734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=333578240 dispatch_meta_bytes=332280 combine_payload_bytes=333578240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=333910520 ingress_bytes=437387264 link_bytes=437387264 token_bytes=667488760 weight_movement_bytes=103809024 total_nvlink_bytes=771297784 weight_to_token_ratio=0.155522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.645853062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330235904 dispatch_meta_bytes=330632 combine_payload_bytes=330235904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=330566536 ingress_bytes=399441920 link_bytes=399441920 token_bytes=660802440 weight_movement_bytes=69206016 total_nvlink_bytes=730008456 weight_to_token_ratio=0.104730 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.503067787e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=328597504 dispatch_meta_bytes=329280 combine_payload_bytes=328597504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328926784 ingress_bytes=363200512 link_bytes=363200512 token_bytes=657524288 weight_movement_bytes=34603008 total_nvlink_bytes=692127296 weight_to_token_ratio=0.052626 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366694287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=192143360 dispatch_meta_bytes=252144 combine_payload_bytes=192143360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=192395504 ingress_bytes=261349376 link_bytes=261349376 token_bytes=384538864 weight_movement_bytes=69206016 total_nvlink_bytes=453744880 weight_to_token_ratio=0.179971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.834366615e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96112 payload_rows_remote=79728 dup_rows=2192 dup_rows_remote=1444 dedup_ratio=0.022298 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 10, 0, 0, 402, 12, 0, 0, 0, 60, 0, 132, 0, 0, 0, 0, 0, 0, 6487, 7796, 0, 0, 0, 0, 0, 58, 0, 65, 111, 0, 0, 0, 2751, 8192, 0, 0, 65, 0, 0, 244, 0, 65, 0, 0, 0, 6383, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16254, 16384, 12974, 15592, 5758, 16384, 12766] dup_rows_per_dst=[0, 20, 948, 264, 116, 96, 748, 0] combine_rows_per_src=[0, 16254, 16384, 12974, 15592, 5758, 16384, 12766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75264 payload_rows_remote=64284 dup_rows=23040 dup_rows_remote=23032 dedup_ratio=0.234375 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 4, 0, 0, 363, 0, 0, 0, 0, 60, 0, 150, 0, 0, 0, 0, 0, 0, 6086, 7784, 0, 3, 0, 0, 0, 3, 0, 80, 122, 0, 0, 0, 2816, 8192, 0, 0, 74, 0, 0, 232, 0, 74, 0, 0, 0, 6799, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10980, 846, 12276, 4184, 16384, 612, 13598] dup_rows_per_dst=[16236, 8, 0, 610, 2, 6036, 148, 0] combine_rows_per_src=[16384, 10980, 846, 12276, 4184, 16384, 612, 13598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92366 payload_rows_remote=77516 dup_rows=5938 dup_rows_remote=2030 dedup_ratio=0.060404 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 2, 0, 0, 382, 2, 0, 0, 0, 41, 0, 230, 0, 0, 0, 0, 0, 0, 6157, 7812, 0, 1, 0, 0, 0, 0, 0, 96, 135, 0, 0, 0, 2492, 8192, 0, 0, 95, 0, 0, 251, 0, 96, 0, 0, 0, 6879, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16194, 16384, 7566, 15624, 5364, 16384, 14850] dup_rows_per_dst=[0, 4, 850, 208, 2, 82, 884, 3908] combine_rows_per_src=[0, 16194, 16384, 7566, 15624, 5364, 16384, 14850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87220 payload_rows_remote=77686 dup_rows=11084 dup_rows_remote=10558 dedup_ratio=0.112752 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 396, 9, 0, 0, 0, 90, 0, 174, 0, 0, 0, 0, 0, 0, 6158, 7740, 0, 0, 0, 0, 0, 0, 0, 85, 160, 0, 0, 0, 2714, 8192, 0, 0, 82, 0, 0, 262, 0, 82, 0, 0, 0, 6706, 0, 0, 0, 0, 0] payload_rows_per_dst=[16304, 0, 9534, 12316, 15480, 9894, 10280, 13412] dup_rows_per_dst=[7230, 0, 526, 348, 0, 2374, 606, 0] combine_rows_per_src=[16304, 0, 9534, 12316, 15480, 9894, 10280, 13412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=326565888 dispatch_meta_bytes=324688 combine_payload_bytes=326565888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326890576 ingress_bytes=361168896 link_bytes=361168896 token_bytes=653456464 weight_movement_bytes=34603008 total_nvlink_bytes=688059472 weight_to_token_ratio=0.052954 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359049479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96056 payload_rows_remote=90592 dup_rows=2248 dup_rows_remote=2116 dedup_ratio=0.022868 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 1, 0, 0, 314, 1, 0, 0, 0, 65, 0, 240, 0, 0, 0, 0, 0, 0, 6014, 7819, 0, 12, 0, 0, 0, 0, 0, 97, 137, 0, 0, 0, 2564, 8192, 0, 0, 90, 0, 0, 245, 0, 90, 0, 0, 0, 6977, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16204, 16384, 12028, 15638, 5464, 16384, 13954] dup_rows_per_dst=[0, 2, 760, 480, 24, 132, 850, 0] combine_rows_per_src=[0, 16204, 16384, 12028, 15638, 5464, 16384, 13954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94900 payload_rows_remote=92612 dup_rows=3404 dup_rows_remote=3404 dedup_ratio=0.034627 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 349, 0, 0, 7878, 7059, 0, 0, 71, 41, 0, 0, 0, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 1887, 1117, 0, 0, 0, 0, 0, 0, 1144, 7341, 0, 8117, 0, 0, 0, 0, 75, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 6491, 0, 0, 0, 23] payload_rows_per_dst=[14682, 9582, 14292, 8888, 2288, 15756, 16234, 13178] dup_rows_per_dst=[698, 90, 0, 1790, 0, 334, 0, 492] combine_rows_per_src=[14682, 9582, 14292, 8888, 2288, 15756, 16234, 13178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263307264 dispatch_meta_bytes=349264 combine_payload_bytes=263307264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263656528 ingress_bytes=332513280 link_bytes=332513280 token_bytes=526963792 weight_movement_bytes=69206016 total_nvlink_bytes=596169808 weight_to_token_ratio=0.131330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251220703e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95940 payload_rows_remote=82818 dup_rows=2364 dup_rows_remote=2070 dedup_ratio=0.024048 router_tokens_per_expert=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 48, 0, 0, 465, 52, 0, 0, 0, 51, 0, 147, 0, 0, 0, 0, 0, 0, 6561, 7794, 0, 44, 0, 0, 0, 45, 0, 69, 29, 0, 0, 0, 2756, 8192, 0, 0, 69, 0, 0, 211, 0, 69, 0, 0, 0, 6237, 0, 0, 0, 0, 0] payload_rows_per_dst=[9634, 6654, 16384, 13122, 15588, 5700, 16384, 12474] dup_rows_per_dst=[0, 54, 1132, 294, 178, 8, 698, 0] combine_rows_per_src=[9634, 6654, 16384, 13122, 15588, 5700, 16384, 12474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62780 payload_rows_remote=46418 dup_rows=35524 dup_rows_remote=19370 dedup_ratio=0.361369 router_tokens_per_expert=[0, 1, 8171, 1, 1, 0, 0, 1, 11, 1, 0, 0, 8083, 0, 30, 1, 1, 358, 45, 0, 0, 0, 54, 0, 213, 2, 0, 0, 0, 0, 0, 6350, 7765, 0, 61, 0, 10, 0, 44, 0, 103, 38, 0, 2, 10, 2917, 8179, 0, 12, 93, 0, 0, 227, 0, 91, 0, 0, 12, 6252, 12, 0, 0, 0, 0] payload_rows_per_dst=[16362, 15562, 826, 12726, 230, 16384, 662, 28] dup_rows_per_dst=[16154, 54, 90, 404, 0, 18616, 184, 22] combine_rows_per_src=[16362, 15562, 826, 12726, 230, 16384, 662, 28] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=317505536 dispatch_meta_bytes=318184 combine_payload_bytes=317505536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317823720 ingress_bytes=386711552 link_bytes=386711552 token_bytes=635329256 weight_movement_bytes=69206016 total_nvlink_bytes=704535272 weight_to_token_ratio=0.108929 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455164437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=318201856 dispatch_meta_bytes=352976 combine_payload_bytes=318201856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318554832 ingress_bytes=352804864 link_bytes=352804864 token_bytes=636756688 weight_movement_bytes=34603008 total_nvlink_bytes=671359696 weight_to_token_ratio=0.054343 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327576300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=371064832 dispatch_meta_bytes=370832 combine_payload_bytes=371064832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=371435664 ingress_bytes=440270848 link_bytes=440270848 token_bytes=742500496 weight_movement_bytes=69206016 total_nvlink_bytes=811706512 weight_to_token_ratio=0.093207 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.656703756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=379338752 dispatch_meta_bytes=384064 combine_payload_bytes=379338752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=379722816 ingress_bytes=483147776 link_bytes=483147776 token_bytes=759061568 weight_movement_bytes=103809024 total_nvlink_bytes=862870592 weight_to_token_ratio=0.136760 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.818046185e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=339222528 dispatch_meta_bytes=339552 combine_payload_bytes=339222528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=339562080 ingress_bytes=408428544 link_bytes=408428544 token_bytes=678784608 weight_movement_bytes=69206016 total_nvlink_bytes=747990624 weight_to_token_ratio=0.101956 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536883730e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=190128128 dispatch_meta_bytes=263152 combine_payload_bytes=190128128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=190391280 ingress_bytes=259334144 link_bytes=259334144 token_bytes=380519408 weight_movement_bytes=69206016 total_nvlink_bytes=449725424 weight_to_token_ratio=0.181872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.758535058e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83818 payload_rows_remote=67560 dup_rows=14486 dup_rows_remote=14486 dedup_ratio=0.147359 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7846, 7267, 0, 0, 63, 4, 0, 0, 0, 0, 0, 0, 6895, 0, 0, 0, 0, 0, 0, 2123, 1105, 0, 0, 0, 0, 0, 0, 951, 7419, 0, 8129, 0, 0, 0, 0, 63, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 6545, 0, 0, 0, 31] payload_rows_per_dst=[6760, 134, 2952, 15942, 12798, 15692, 16258, 13282] dup_rows_per_dst=[248, 0, 0, 5048, 8362, 320, 0, 508] combine_rows_per_src=[6760, 134, 2952, 15942, 12798, 15692, 16258, 13282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14729325512 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=7368666056 ingress_bytes=8813985792 link_bytes=8813985792 total_nvlink_bytes=16182651848 weight_to_token_ratio=0.098669 prefetch_share_of_ingress=0.082444 dedup_ratio=0.101831 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=23241 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.442013867e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81206 payload_rows_remote=65342 dup_rows=17098 dup_rows_remote=3968 dedup_ratio=0.173930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 0, 0, 7878, 7122, 0, 0, 71, 56, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 0, 0, 0, 2160, 1025, 0, 0, 0, 0, 0, 0, 1233, 7264, 0, 8121, 0, 0, 0, 0, 71, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 6197, 0, 0, 0, 43] payload_rows_per_dst=[14528, 15864, 14340, 5738, 15582, 1584, 942, 12628] dup_rows_per_dst=[650, 13130, 0, 632, 2184, 24, 0, 478] combine_rows_per_src=[14528, 15864, 14340, 5738, 15582, 1584, 942, 12628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77048 payload_rows_remote=63618 dup_rows=21256 dup_rows_remote=19262 dedup_ratio=0.216227 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 279, 0, 0, 7882, 7105, 0, 0, 92, 40, 0, 0, 0, 0, 0, 0, 7160, 0, 0, 0, 0, 0, 0, 2186, 882, 0, 0, 0, 0, 0, 0, 1131, 7281, 0, 8097, 0, 0, 0, 0, 95, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 6431, 0, 0, 0, 34] payload_rows_per_dst=[558, 264, 0, 14534, 16304, 15764, 16194, 13430] dup_rows_per_dst=[0, 0, 0, 4016, 14840, 406, 0, 1994] combine_rows_per_src=[558, 264, 0, 14534, 16304, 15764, 16194, 13430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=276725760 dispatch_meta_bytes=328184 combine_payload_bytes=276725760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277053944 ingress_bytes=311328768 link_bytes=311328768 token_bytes=553779704 weight_movement_bytes=34603008 total_nvlink_bytes=588382712 weight_to_token_ratio=0.062485 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171505090e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95796 payload_rows_remote=81706 dup_rows=2508 dup_rows_remote=2508 dedup_ratio=0.025513 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 322, 0, 0, 7829, 6994, 0, 0, 78, 81, 0, 0, 0, 0, 0, 0, 7045, 0, 0, 0, 0, 0, 0, 2251, 1131, 0, 0, 0, 0, 0, 0, 1089, 7271, 0, 8110, 0, 0, 0, 0, 82, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 6371, 0, 0, 0, 22] payload_rows_per_dst=[14542, 14150, 14090, 6008, 2178, 15658, 16220, 12950] dup_rows_per_dst=[644, 156, 0, 756, 0, 424, 0, 528] combine_rows_per_src=[14542, 14150, 14090, 6008, 2178, 15658, 16220, 12950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91158 payload_rows_remote=75440 dup_rows=7146 dup_rows_remote=6672 dedup_ratio=0.072693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 297, 0, 0, 7859, 7106, 0, 0, 89, 42, 0, 0, 0, 0, 0, 0, 6668, 0, 0, 0, 0, 0, 0, 2387, 1246, 0, 0, 0, 0, 0, 0, 1153, 7183, 0, 8102, 0, 0, 0, 0, 90, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 6390, 0, 0, 0, 56] payload_rows_per_dst=[14366, 262, 13336, 15894, 2306, 15718, 16204, 13072] dup_rows_per_dst=[594, 0, 0, 5584, 0, 474, 0, 494] combine_rows_per_src=[14366, 262, 13336, 15894, 2306, 15718, 16204, 13072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96224 payload_rows_remote=90518 dup_rows=2080 dup_rows_remote=1418 dedup_ratio=0.021159 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 300, 0, 0, 7853, 7079, 0, 0, 69, 178, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 1943, 1241, 0, 0, 0, 0, 0, 0, 1161, 7196, 0, 8123, 0, 0, 0, 0, 69, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 6521, 0, 0, 0, 46] payload_rows_per_dst=[14392, 14514, 14066, 5706, 2322, 15706, 16246, 13272] dup_rows_per_dst=[600, 138, 0, 662, 0, 158, 0, 522] combine_rows_per_src=[14392, 14514, 14066, 5706, 2322, 15706, 16246, 13272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70654 payload_rows_remote=56264 dup_rows=27650 dup_rows_remote=26998 dedup_ratio=0.281270 router_tokens_per_expert=[0, 0, 8, 9, 12, 5, 308, 1, 0, 7859, 6990, 0, 0, 91, 70, 0, 0, 1, 0, 1, 11, 6619, 7, 0, 0, 4, 0, 0, 2166, 1376, 7, 0, 1, 12, 0, 0, 1469, 7178, 0, 8088, 0, 0, 0, 0, 98, 0, 45, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 340, 6317, 0, 1, 1, 54] payload_rows_per_dst=[14390, 15996, 15744, 6420, 16382, 266, 6, 1450] dup_rows_per_dst=[652, 14024, 9484, 686, 2758, 20, 0, 26] combine_rows_per_src=[14390, 15996, 15744, 6420, 16382, 266, 6, 1450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=260579328 dispatch_meta_bytes=331520 combine_payload_bytes=260579328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260910848 ingress_bytes=329785344 link_bytes=329785344 token_bytes=521490176 weight_movement_bytes=69206016 total_nvlink_bytes=590696192 weight_to_token_ratio=0.132708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240955700e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12892498840 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6449998744 ingress_bytes=7861223424 link_bytes=7861223424 total_nvlink_bytes=14311222168 weight_to_token_ratio=0.110043 prefetch_share_of_ingress=0.088035 dedup_ratio=0.162538 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.286136994e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=334667776 dispatch_meta_bytes=336856 combine_payload_bytes=334667776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335004632 ingress_bytes=403873792 link_bytes=403873792 token_bytes=669672408 weight_movement_bytes=69206016 total_nvlink_bytes=738878424 weight_to_token_ratio=0.103343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519744565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267640832 dispatch_meta_bytes=277240 combine_payload_bytes=267640832 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267918072 ingress_bytes=371449856 link_bytes=371449856 token_bytes=535558904 weight_movement_bytes=103809024 total_nvlink_bytes=639367928 weight_to_token_ratio=0.193833 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397735904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309002240 dispatch_meta_bytes=328448 combine_payload_bytes=309002240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309330688 ingress_bytes=343605248 link_bytes=343605248 token_bytes=618332928 weight_movement_bytes=34603008 total_nvlink_bytes=652935936 weight_to_token_ratio=0.055962 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292958886e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12243678968 prefetch_bytes=622854144 reduce_bytes=657457152 weight_movement_bytes=1280311296 egress_bytes=6125532920 ingress_bytes=7398457344 link_bytes=7398457344 total_nvlink_bytes=13523990264 weight_to_token_ratio=0.104569 prefetch_share_of_ingress=0.084187 dedup_ratio=0.197777 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=19 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.031196 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210426059e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=370761728 dispatch_meta_bytes=367744 combine_payload_bytes=370761728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=371129472 ingress_bytes=405364736 link_bytes=405364736 token_bytes=741891200 weight_movement_bytes=34603008 total_nvlink_bytes=776494208 weight_to_token_ratio=0.046642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525354867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=230457344 dispatch_meta_bytes=333048 combine_payload_bytes=230457344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230790392 ingress_bytes=265060352 link_bytes=265060352 token_bytes=461247736 weight_movement_bytes=34603008 total_nvlink_bytes=495850744 weight_to_token_ratio=0.075020 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.974007655e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14089759320 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=7048809048 ingress_bytes=8425070592 link_bytes=8425070592 total_nvlink_bytes=15473879640 weight_to_token_ratio=0.098236 prefetch_share_of_ingress=0.086250 dedup_ratio=0.126151 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.378385320e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12510724784 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=6259139248 ingress_bytes=7497293824 link_bytes=7497293824 total_nvlink_bytes=13756433072 weight_to_token_ratio=0.099571 prefetch_share_of_ingress=0.092308 dedup_ratio=0.195989 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226596221e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14157153400 prefetch_bytes=761266176 reduce_bytes=657457152 weight_movement_bytes=1418723328 egress_bytes=7082476664 ingress_bytes=8493400064 link_bytes=8493400064 total_nvlink_bytes=15575876728 weight_to_token_ratio=0.100212 prefetch_share_of_ingress=0.089630 dedup_ratio=0.116443 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=19 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=23876 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.389564377e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14595646720 prefetch_bytes=761266176 reduce_bytes=968884224 weight_movement_bytes=1730150400 egress_bytes=7301809408 ingress_bytes=9023987712 link_bytes=9023987712 total_nvlink_bytes=16325797120 weight_to_token_ratio=0.118539 prefetch_share_of_ingress=0.084360 dedup_ratio=0.106135 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=28 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030000 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.476371273e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11377699520 prefetch_bytes=657457152 reduce_bytes=692060160 weight_movement_bytes=1349517312 egress_bytes=5692402368 ingress_bytes=7034814464 link_bytes=7034814464 total_nvlink_bytes=12727216832 weight_to_token_ratio=0.118611 prefetch_share_of_ingress=0.093458 dedup_ratio=0.243285 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=20 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21590 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.150932195e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-29 23:40:36] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 670.1 | throughput per GPU (TFLOP/s/GPU): 190.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.782619E+00 | loss scale: 1.0 | grad norm: 5.638 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76100 payload_rows_remote=66492 dup_rows=22204 dup_rows_remote=18912 dedup_ratio=0.225871 router_tokens_per_expert=[1310, 284, 781, 948, 258, 840, 821, 1052, 1435, 184, 576, 323, 315, 1344, 1015, 1258, 644, 1028, 794, 572, 245, 725, 278, 1464, 577, 727, 809, 327, 724, 73, 877, 1080, 749, 944, 1175, 694, 676, 232, 555, 991, 606, 607, 515, 833, 863, 916, 468, 544, 717, 602, 773, 501, 984, 1406, 870, 724, 1686, 446, 1442, 774, 176, 783, 1280, 932] payload_rows_per_dst=[9276, 9608, 10770, 10198, 9600, 10552, 8588, 7508] dup_rows_per_dst=[3312, 3292, 3614, 3562, 2432, 2964, 1754, 1274] combine_rows_per_src=[9276, 9608, 10770, 10198, 9600, 10552, 8588, 7508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74898 payload_rows_remote=63588 dup_rows=23406 dup_rows_remote=17878 dedup_ratio=0.238098 router_tokens_per_expert=[1333, 293, 845, 979, 270, 921, 766, 1048, 1375, 185, 638, 360, 366, 1210, 866, 1073, 656, 966, 857, 562, 288, 565, 270, 1439, 634, 749, 777, 307, 696, 73, 897, 1239, 846, 1000, 1119, 689, 595, 250, 520, 976, 618, 623, 514, 776, 835, 941, 578, 684, 691, 537, 872, 579, 1042, 1337, 791, 802, 1535, 489, 1452, 775, 175, 734, 1283, 991] payload_rows_per_dst=[7842, 9168, 9036, 8914, 9692, 8636, 10300, 11310] dup_rows_per_dst=[2402, 2602, 2170, 1830, 2674, 2502, 3698, 5528] combine_rows_per_src=[7842, 9168, 9036, 8914, 9692, 8636, 10300, 11310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76176 payload_rows_remote=66520 dup_rows=22128 dup_rows_remote=20118 dedup_ratio=0.225098 router_tokens_per_expert=[1214, 287, 689, 1086, 275, 987, 804, 1061, 1433, 198, 636, 406, 348, 1243, 969, 1226, 643, 949, 826, 631, 311, 599, 310, 1329, 635, 591, 720, 333, 720, 79, 919, 1188, 723, 976, 907, 724, 721, 201, 582, 999, 600, 636, 501, 803, 915, 911, 574, 617, 700, 496, 758, 567, 990, 1403, 757, 833, 1555, 568, 1452, 741, 122, 826, 1263, 1086] payload_rows_per_dst=[9744, 9676, 8984, 9492, 9656, 8592, 9938, 10094] dup_rows_per_dst=[3062, 3242, 2212, 2526, 2010, 2522, 3070, 3484] combine_rows_per_src=[9744, 9676, 8984, 9492, 9656, 8592, 9938, 10094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75410 payload_rows_remote=65150 dup_rows=22894 dup_rows_remote=18648 dedup_ratio=0.232890 router_tokens_per_expert=[1270, 272, 764, 1020, 280, 1111, 827, 1006, 1316, 165, 556, 382, 303, 1094, 947, 1132, 705, 957, 832, 605, 241, 606, 257, 1415, 720, 656, 753, 382, 742, 89, 945, 1062, 832, 880, 1245, 712, 574, 230, 582, 1022, 552, 562, 504, 774, 888, 976, 648, 653, 734, 545, 779, 508, 972, 1412, 771, 806, 1606, 563, 1543, 754, 101, 782, 1242, 988] payload_rows_per_dst=[8916, 9146, 8978, 9004, 9602, 8478, 10260, 11026] dup_rows_per_dst=[2716, 2660, 2258, 1694, 2552, 2636, 4246, 4132] combine_rows_per_src=[8916, 9146, 8978, 9004, 9602, 8478, 10260, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75762 payload_rows_remote=65286 dup_rows=22542 dup_rows_remote=18992 dedup_ratio=0.229309 router_tokens_per_expert=[1299, 261, 734, 1053, 259, 971, 712, 895, 1410, 197, 579, 330, 288, 1344, 827, 1149, 702, 1065, 821, 570, 256, 500, 319, 1454, 629, 626, 747, 335, 764, 85, 920, 1007, 880, 1005, 1152, 776, 833, 218, 517, 1046, 732, 622, 559, 793, 855, 969, 657, 641, 616, 552, 791, 509, 1003, 1405, 712, 916, 1531, 521, 1419, 667, 133, 714, 1239, 1061] payload_rows_per_dst=[9206, 9378, 10476, 9950, 10082, 10584, 8610, 7476] dup_rows_per_dst=[3162, 2870, 3550, 3338, 2772, 3824, 1646, 1380] combine_rows_per_src=[9206, 9378, 10476, 9950, 10082, 10584, 8610, 7476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75390 payload_rows_remote=65208 dup_rows=22914 dup_rows_remote=19174 dedup_ratio=0.233093 router_tokens_per_expert=[1285, 332, 767, 980, 274, 910, 790, 1063, 1291, 229, 644, 381, 351, 1302, 840, 1139, 663, 1057, 769, 563, 261, 492, 276, 1407, 572, 736, 785, 335, 706, 69, 991, 1069, 816, 941, 1296, 718, 690, 207, 538, 906, 694, 708, 572, 795, 938, 933, 625, 566, 632, 571, 747, 491, 985, 1419, 762, 809, 1698, 492, 1441, 686, 102, 777, 1285, 983] payload_rows_per_dst=[9358, 9302, 8888, 10182, 9644, 8892, 9818, 9306] dup_rows_per_dst=[3444, 3052, 2088, 3740, 2580, 2770, 3014, 2226] combine_rows_per_src=[9358, 9302, 8888, 10182, 9644, 8892, 9818, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75960 payload_rows_remote=67344 dup_rows=22344 dup_rows_remote=19796 dedup_ratio=0.227295 router_tokens_per_expert=[1268, 286, 782, 954, 254, 1149, 804, 1066, 1360, 162, 601, 330, 315, 1152, 896, 1114, 680, 1043, 792, 634, 239, 596, 247, 1480, 659, 704, 695, 315, 694, 102, 894, 1026, 818, 874, 1294, 723, 705, 206, 579, 1058, 582, 626, 622, 817, 837, 892, 614, 592, 720, 566, 787, 467, 984, 1476, 840, 749, 1604, 328, 1520, 728, 166, 752, 1305, 1028] payload_rows_per_dst=[9760, 9130, 9204, 8542, 9712, 8616, 9982, 11014] dup_rows_per_dst=[3366, 2730, 2218, 1636, 2802, 2548, 3196, 3848] combine_rows_per_src=[9760, 9130, 9204, 8542, 9712, 8616, 9982, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272351232 dispatch_meta_bytes=341616 combine_payload_bytes=272351232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272692848 ingress_bytes=341557248 link_bytes=341557248 token_bytes=545044080 weight_movement_bytes=69206016 total_nvlink_bytes=614250096 weight_to_token_ratio=0.126973 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285252427e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260456448 dispatch_meta_bytes=325864 combine_payload_bytes=260456448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=260782312 ingress_bytes=364265472 link_bytes=364265472 token_bytes=521238760 weight_movement_bytes=103809024 total_nvlink_bytes=625047784 weight_to_token_ratio=0.199158 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370701645e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272465920 dispatch_meta_bytes=346552 combine_payload_bytes=272465920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272812472 ingress_bytes=307068928 link_bytes=307068928 token_bytes=545278392 weight_movement_bytes=34603008 total_nvlink_bytes=579881400 weight_to_token_ratio=0.063459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155475655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267411456 dispatch_meta_bytes=337112 combine_payload_bytes=267411456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267748568 ingress_bytes=302014464 link_bytes=302014464 token_bytes=535160024 weight_movement_bytes=34603008 total_nvlink_bytes=569763032 weight_to_token_ratio=0.064659 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136456114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=335192 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267189592 ingress_bytes=336060416 link_bytes=336060416 token_bytes=534043992 weight_movement_bytes=69206016 total_nvlink_bytes=603250008 weight_to_token_ratio=0.129589 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264568290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=337528 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267429496 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534521464 weight_movement_bytes=34603008 total_nvlink_bytes=569124472 weight_to_token_ratio=0.064736 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275841024 dispatch_meta_bytes=348560 combine_payload_bytes=275841024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276189584 ingress_bytes=310444032 link_bytes=310444032 token_bytes=552030608 weight_movement_bytes=34603008 total_nvlink_bytes=586633616 weight_to_token_ratio=0.062683 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168175900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75418 payload_rows_remote=66002 dup_rows=22886 dup_rows_remote=19564 dedup_ratio=0.232808 router_tokens_per_expert=[1313, 293, 751, 1004, 319, 900, 790, 999, 1405, 182, 517, 430, 384, 1234, 900, 1172, 647, 1009, 753, 519, 315, 552, 288, 1298, 609, 721, 767, 291, 723, 87, 941, 1041, 811, 891, 1253, 696, 812, 208, 613, 963, 585, 623, 611, 858, 842, 956, 643, 671, 643, 578, 760, 536, 1015, 1305, 771, 789, 1668, 500, 1548, 679, 101, 734, 1208, 1127] payload_rows_per_dst=[9416, 9324, 10298, 10058, 9956, 10454, 8610, 7302] dup_rows_per_dst=[3322, 3124, 3560, 3638, 2538, 3734, 1574, 1396] combine_rows_per_src=[9416, 9324, 10298, 10058, 9956, 10454, 8610, 7302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77336 payload_rows_remote=67250 dup_rows=20968 dup_rows_remote=17530 dedup_ratio=0.213298 router_tokens_per_expert=[665, 782, 1021, 483, 975, 251, 375, 1102, 94, 1106, 1279, 349, 587, 1093, 800, 1454, 1192, 1101, 945, 527, 1381, 171, 1656, 1103, 595, 773, 699, 819, 82, 349, 188, 144, 326, 252, 558, 218, 1186, 1903, 1028, 949, 2097, 1452, 1203, 1339, 426, 866, 46, 182, 670, 1211, 927, 28, 149, 1413, 255, 1338, 947, 359, 788, 580, 916, 636, 377, 386] payload_rows_per_dst=[10448, 10086, 9750, 8698, 10158, 10184, 9718, 8294] dup_rows_per_dst=[3208, 3438, 2256, 2746, 2682, 2690, 2264, 1684] combine_rows_per_src=[10448, 10086, 9750, 8698, 10158, 10184, 9718, 8294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74718 payload_rows_remote=64954 dup_rows=23586 dup_rows_remote=19374 dedup_ratio=0.239929 router_tokens_per_expert=[703, 801, 823, 533, 981, 310, 344, 1132, 100, 996, 1332, 331, 600, 1277, 808, 1487, 1194, 1219, 865, 472, 1468, 94, 1626, 1015, 599, 741, 671, 817, 91, 325, 212, 134, 414, 175, 629, 292, 1100, 1865, 1013, 1078, 2029, 1341, 1295, 1556, 391, 726, 25, 177, 695, 1182, 858, 41, 135, 1490, 218, 1367, 822, 331, 863, 613, 980, 617, 312, 421] payload_rows_per_dst=[8810, 7788, 9994, 9900, 10134, 8502, 9826, 9764] dup_rows_per_dst=[2444, 2360, 2698, 3468, 2760, 3260, 2384, 4212] combine_rows_per_src=[8810, 7788, 9994, 9900, 10134, 8502, 9826, 9764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74710 payload_rows_remote=64540 dup_rows=23594 dup_rows_remote=20716 dedup_ratio=0.240011 router_tokens_per_expert=[720, 805, 906, 556, 984, 241, 347, 1240, 143, 1032, 1337, 369, 571, 1172, 842, 1452, 1371, 1180, 990, 437, 1258, 100, 1503, 1027, 494, 716, 599, 826, 69, 309, 194, 118, 406, 233, 619, 253, 1204, 1854, 987, 968, 1993, 1420, 1223, 1482, 374, 910, 23, 145, 758, 1272, 956, 17, 115, 1452, 256, 1193, 894, 346, 924, 653, 918, 628, 315, 453] payload_rows_per_dst=[8890, 8296, 10146, 9314, 10170, 8344, 9774, 9776] dup_rows_per_dst=[2708, 2636, 2968, 2858, 2878, 3246, 2264, 4036] combine_rows_per_src=[8890, 8296, 10146, 9314, 10170, 8344, 9774, 9776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76192 payload_rows_remote=67658 dup_rows=22112 dup_rows_remote=20644 dedup_ratio=0.224935 router_tokens_per_expert=[586, 949, 765, 546, 1162, 300, 384, 1154, 122, 1091, 1558, 369, 577, 1221, 861, 1549, 1219, 1115, 873, 421, 1220, 174, 1490, 1199, 603, 834, 668, 803, 74, 217, 208, 128, 352, 185, 608, 198, 1202, 1551, 976, 984, 1847, 1345, 1157, 1377, 335, 897, 21, 198, 843, 1356, 1237, 20, 84, 1301, 233, 1334, 865, 322, 900, 605, 897, 639, 365, 478] payload_rows_per_dst=[9020, 10644, 8534, 9436, 9878, 10160, 10250, 8270] dup_rows_per_dst=[2672, 4052, 1468, 3054, 2234, 4194, 2566, 1872] combine_rows_per_src=[9020, 10644, 8534, 9436, 9878, 10160, 10250, 8270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=64978 dup_rows=23440 dup_rows_remote=20902 dedup_ratio=0.238444 router_tokens_per_expert=[721, 704, 935, 694, 1134, 189, 297, 1135, 116, 1059, 1352, 364, 610, 1238, 892, 1483, 1309, 1088, 894, 470, 1227, 131, 1406, 1244, 574, 719, 681, 834, 73, 366, 258, 131, 353, 213, 531, 146, 1091, 1729, 1005, 921, 2114, 1361, 1162, 1342, 328, 942, 12, 190, 828, 1268, 1095, 15, 118, 1394, 220, 1274, 918, 377, 936, 667, 907, 593, 305, 469] payload_rows_per_dst=[8908, 8452, 10382, 9534, 9778, 7852, 9886, 10072] dup_rows_per_dst=[2710, 2810, 2856, 3004, 2200, 2822, 2538, 4500] combine_rows_per_src=[8908, 8452, 10382, 9534, 9778, 7852, 9886, 10072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76476 payload_rows_remote=66938 dup_rows=21828 dup_rows_remote=18720 dedup_ratio=0.222046 router_tokens_per_expert=[708, 806, 762, 582, 1205, 279, 411, 1232, 157, 1042, 1435, 384, 699, 1276, 769, 1584, 1111, 1098, 884, 362, 1330, 114, 1508, 1029, 613, 669, 699, 847, 78, 225, 218, 136, 422, 197, 569, 154, 1179, 1606, 915, 934, 1922, 1368, 1097, 1465, 365, 924, 42, 206, 901, 1322, 1132, 24, 92, 1294, 259, 1249, 972, 349, 920, 698, 885, 630, 339, 469] payload_rows_per_dst=[9152, 9342, 9354, 9538, 9744, 10522, 10178, 8646] dup_rows_per_dst=[2818, 3136, 2056, 3108, 2208, 4256, 2368, 1878] combine_rows_per_src=[9152, 9342, 9354, 9538, 9744, 10522, 10178, 8646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74686 payload_rows_remote=67100 dup_rows=23618 dup_rows_remote=20892 dedup_ratio=0.240255 router_tokens_per_expert=[677, 762, 1069, 547, 1175, 260, 333, 1101, 96, 996, 1427, 371, 623, 1105, 746, 1567, 1132, 1179, 989, 459, 1327, 152, 1622, 1164, 612, 818, 783, 795, 103, 353, 215, 122, 316, 253, 508, 166, 1142, 1579, 847, 910, 2121, 1313, 1092, 1389, 351, 803, 29, 179, 958, 1259, 1091, 15, 115, 1196, 235, 1317, 941, 336, 937, 653, 903, 676, 350, 492] payload_rows_per_dst=[8898, 8252, 10252, 10038, 9448, 7586, 9958, 10254] dup_rows_per_dst=[2950, 2476, 3032, 3462, 1994, 2726, 2414, 4564] combine_rows_per_src=[8898, 8252, 10252, 10038, 9448, 7586, 9958, 10254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270344192 dispatch_meta_bytes=342264 combine_payload_bytes=270344192 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270686456 ingress_bytes=374153216 link_bytes=374153216 token_bytes=541030648 weight_movement_bytes=103809024 total_nvlink_bytes=644839672 weight_to_token_ratio=0.191873 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275456000 dispatch_meta_bytes=339120 combine_payload_bytes=275456000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275795120 ingress_bytes=379265024 link_bytes=379265024 token_bytes=551251120 weight_movement_bytes=103809024 total_nvlink_bytes=655060144 weight_to_token_ratio=0.188315 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427143752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=337312 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266388896 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532440480 weight_movement_bytes=34603008 total_nvlink_bytes=567043488 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264355840 dispatch_meta_bytes=341024 combine_payload_bytes=264355840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264696864 ingress_bytes=333561856 link_bytes=333561856 token_bytes=529052704 weight_movement_bytes=69206016 total_nvlink_bytes=598258720 weight_to_token_ratio=0.130811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255166410e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277127168 dispatch_meta_bytes=353208 combine_payload_bytes=277127168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277480376 ingress_bytes=380936192 link_bytes=380936192 token_bytes=554607544 weight_movement_bytes=103809024 total_nvlink_bytes=658416568 weight_to_token_ratio=0.187176 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433432223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343520 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493408 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643296 weight_movement_bytes=34603008 total_nvlink_bytes=567246304 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274178048 dispatch_meta_bytes=342632 combine_payload_bytes=274178048 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274520680 ingress_bytes=343384064 link_bytes=343384064 token_bytes=548698728 weight_movement_bytes=69206016 total_nvlink_bytes=617904744 weight_to_token_ratio=0.126128 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292126588e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274841600 dispatch_meta_bytes=351968 combine_payload_bytes=274841600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275193568 ingress_bytes=378650624 link_bytes=378650624 token_bytes=550035168 weight_movement_bytes=103809024 total_nvlink_bytes=653844192 weight_to_token_ratio=0.188732 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424831814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73612 payload_rows_remote=63782 dup_rows=24692 dup_rows_remote=20454 dedup_ratio=0.251180 router_tokens_per_expert=[129, 775, 558, 310, 105, 1313, 853, 266, 673, 376, 1064, 1214, 1586, 1078, 231, 1632, 928, 1404, 391, 356, 113, 254, 1433, 1038, 444, 620, 966, 714, 331, 490, 975, 373, 415, 1014, 1173, 738, 2025, 1914, 1756, 566, 401, 202, 329, 294, 1559, 1079, 154, 1219, 870, 197, 1136, 925, 130, 326, 271, 856, 1417, 423, 71, 469, 119, 1457, 366, 2288] payload_rows_per_dst=[9504, 9830, 11702, 9200, 6776, 10006, 9032, 7562] dup_rows_per_dst=[3164, 4238, 3644, 2266, 1642, 5044, 3612, 1082] combine_rows_per_src=[9504, 9830, 11702, 9200, 6776, 10006, 9032, 7562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77504 payload_rows_remote=66506 dup_rows=20800 dup_rows_remote=17130 dedup_ratio=0.211589 router_tokens_per_expert=[752, 767, 918, 519, 1210, 273, 377, 1084, 135, 985, 1330, 370, 619, 1298, 870, 1580, 1127, 1088, 929, 428, 1440, 146, 1613, 1121, 555, 722, 635, 842, 94, 255, 214, 143, 365, 174, 568, 181, 1123, 1803, 858, 1028, 2087, 1302, 1163, 1434, 344, 838, 49, 197, 842, 1214, 841, 20, 127, 1319, 264, 1331, 955, 385, 904, 694, 861, 664, 317, 461] payload_rows_per_dst=[10998, 10718, 10070, 7876, 9666, 9862, 9708, 8606] dup_rows_per_dst=[3670, 3818, 2488, 2270, 2372, 2098, 2208, 1876] combine_rows_per_src=[10998, 10718, 10070, 7876, 9666, 9862, 9708, 8606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=63266 dup_rows=23918 dup_rows_remote=19728 dedup_ratio=0.243306 router_tokens_per_expert=[258, 848, 574, 250, 161, 1126, 837, 277, 754, 323, 859, 1081, 1560, 1235, 585, 1664, 892, 1268, 286, 412, 101, 341, 1307, 759, 369, 659, 925, 544, 330, 500, 792, 567, 457, 1050, 1097, 880, 1833, 1708, 1962, 627, 268, 299, 323, 160, 1761, 1164, 145, 1262, 765, 197, 1262, 851, 265, 401, 284, 934, 1404, 448, 95, 498, 164, 1284, 749, 2111] payload_rows_per_dst=[7166, 8598, 8890, 9866, 11252, 8090, 9404, 11120] dup_rows_per_dst=[1496, 2600, 1842, 2626, 4560, 2674, 3930, 4190] combine_rows_per_src=[7166, 8598, 8890, 9866, 11252, 8090, 9404, 11120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74784 payload_rows_remote=65188 dup_rows=23520 dup_rows_remote=20552 dedup_ratio=0.239258 router_tokens_per_expert=[157, 869, 544, 355, 197, 1244, 829, 262, 664, 252, 871, 1249, 1568, 1150, 459, 1700, 913, 1121, 202, 378, 100, 426, 1383, 724, 419, 651, 955, 688, 276, 498, 741, 403, 465, 1100, 1119, 902, 1740, 1725, 2071, 625, 286, 270, 295, 117, 1844, 1253, 160, 1206, 949, 187, 1015, 797, 218, 433, 294, 1025, 1422, 455, 76, 428, 183, 1331, 715, 2198] payload_rows_per_dst=[9308, 9346, 8760, 9806, 9596, 8292, 9438, 10238] dup_rows_per_dst=[3086, 3344, 1734, 2592, 2968, 2570, 3848, 3378] combine_rows_per_src=[9308, 9346, 8760, 9806, 9596, 8292, 9438, 10238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74694 payload_rows_remote=64570 dup_rows=23610 dup_rows_remote=20824 dedup_ratio=0.240173 router_tokens_per_expert=[114, 921, 600, 325, 87, 1058, 742, 279, 811, 365, 1055, 1096, 1745, 1031, 123, 1167, 943, 1574, 305, 360, 359, 232, 1341, 736, 570, 669, 1111, 628, 257, 447, 731, 333, 427, 1012, 1233, 797, 2462, 2301, 1876, 475, 381, 239, 404, 231, 1705, 1306, 141, 959, 823, 139, 748, 891, 100, 366, 226, 740, 1535, 585, 68, 375, 101, 1592, 374, 2425] payload_rows_per_dst=[9742, 8402, 10124, 10142, 8398, 9246, 9338, 9302] dup_rows_per_dst=[3434, 2894, 2786, 2840, 2032, 3982, 3330, 2312] combine_rows_per_src=[9742, 8402, 10124, 10142, 8398, 9246, 9338, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261251072 dispatch_meta_bytes=336944 combine_payload_bytes=261251072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261588016 ingress_bytes=365060096 link_bytes=365060096 token_bytes=522839088 weight_movement_bytes=103809024 total_nvlink_bytes=626648112 weight_to_token_ratio=0.198549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373691751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65374 dup_rows=23842 dup_rows_remote=20398 dedup_ratio=0.242533 router_tokens_per_expert=[139, 1099, 640, 354, 107, 1182, 816, 263, 779, 314, 1004, 1151, 1804, 1169, 248, 1173, 895, 1388, 307, 333, 88, 329, 1375, 812, 481, 671, 935, 612, 328, 493, 895, 367, 415, 1078, 1081, 820, 2064, 1812, 2037, 614, 420, 277, 325, 182, 1694, 1247, 198, 1301, 910, 205, 939, 794, 151, 331, 328, 796, 1488, 447, 58, 435, 174, 1304, 397, 2279] payload_rows_per_dst=[8218, 8794, 8956, 10302, 10522, 8436, 9088, 10146] dup_rows_per_dst=[2394, 2882, 2098, 2870, 4284, 2852, 3444, 3018] combine_rows_per_src=[8218, 8794, 8956, 10302, 10522, 8436, 9088, 10146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=64636 dup_rows=23688 dup_rows_remote=20624 dedup_ratio=0.240967 router_tokens_per_expert=[104, 1029, 701, 280, 74, 1148, 756, 254, 735, 390, 944, 1154, 1602, 937, 67, 1069, 968, 1568, 353, 350, 122, 174, 1326, 856, 570, 705, 1121, 769, 291, 436, 713, 315, 440, 1089, 1242, 885, 2564, 2312, 1768, 561, 425, 216, 333, 214, 1630, 1220, 138, 1214, 664, 119, 1130, 1023, 117, 248, 168, 743, 1631, 583, 44, 415, 55, 1678, 207, 2195] payload_rows_per_dst=[9954, 7996, 9336, 9980, 9364, 8058, 9358, 10570] dup_rows_per_dst=[3866, 2596, 2098, 3064, 2606, 2722, 3690, 3046] combine_rows_per_src=[9954, 7996, 9336, 9980, 9364, 8058, 9358, 10570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75084 payload_rows_remote=66816 dup_rows=23220 dup_rows_remote=20712 dedup_ratio=0.236206 router_tokens_per_expert=[91, 895, 541, 312, 87, 1329, 745, 239, 680, 380, 986, 1331, 1533, 960, 117, 1199, 992, 1411, 412, 293, 100, 205, 1598, 940, 495, 657, 1056, 827, 301, 421, 869, 285, 432, 1323, 1317, 753, 2699, 1916, 1669, 646, 432, 186, 264, 228, 1561, 1178, 151, 1388, 788, 129, 974, 786, 87, 234, 262, 777, 1446, 507, 80, 486, 127, 1611, 234, 2194] payload_rows_per_dst=[10130, 8534, 9430, 10058, 9564, 8268, 8690, 10410] dup_rows_per_dst=[3746, 2772, 2472, 2830, 2716, 2508, 3216, 2960] combine_rows_per_src=[10130, 8534, 9430, 10058, 9564, 8268, 8690, 10410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272408576 dispatch_meta_bytes=334544 combine_payload_bytes=272408576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272743120 ingress_bytes=307011584 link_bytes=307011584 token_bytes=545151696 weight_movement_bytes=34603008 total_nvlink_bytes=579754704 weight_to_token_ratio=0.063474 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155259874e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259137536 dispatch_meta_bytes=331976 combine_payload_bytes=259137536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259469512 ingress_bytes=328343552 link_bytes=328343552 token_bytes=518607048 weight_movement_bytes=69206016 total_nvlink_bytes=587813064 weight_to_token_ratio=0.133446 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235530352e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=342960 combine_payload_bytes=267010048 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267353008 ingress_bytes=405422080 link_bytes=405422080 token_bytes=534363056 weight_movement_bytes=138412032 total_nvlink_bytes=672775088 weight_to_token_ratio=0.259022 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525570648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264478720 dispatch_meta_bytes=341576 combine_payload_bytes=264478720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264820296 ingress_bytes=299081728 link_bytes=299081728 token_bytes=529299016 weight_movement_bytes=34603008 total_nvlink_bytes=563902024 weight_to_token_ratio=0.065375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125420464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267771904 dispatch_meta_bytes=343088 combine_payload_bytes=267771904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268114992 ingress_bytes=302374912 link_bytes=302374912 token_bytes=535886896 weight_movement_bytes=34603008 total_nvlink_bytes=570489904 weight_to_token_ratio=0.064571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137812451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273678336 dispatch_meta_bytes=350112 combine_payload_bytes=273678336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274028448 ingress_bytes=308281344 link_bytes=308281344 token_bytes=547706784 weight_movement_bytes=34603008 total_nvlink_bytes=582309792 weight_to_token_ratio=0.063178 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160037879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=341040 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265090096 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529839152 weight_movement_bytes=34603008 total_nvlink_bytes=564442160 weight_to_token_ratio=0.065309 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76340 payload_rows_remote=66266 dup_rows=21964 dup_rows_remote=19614 dedup_ratio=0.223429 router_tokens_per_expert=[452, 673, 1354, 321, 489, 560, 802, 730, 1876, 2409, 369, 385, 1426, 365, 1393, 398, 1832, 72, 2444, 523, 570, 311, 23, 723, 1065, 990, 663, 425, 665, 931, 1163, 791, 249, 799, 234, 288, 799, 590, 1347, 415, 429, 2504, 102, 496, 280, 353, 351, 3682, 86, 386, 216, 1011, 411, 1062, 250, 301, 266, 289, 1404, 693, 1017, 844, 459, 346] payload_rows_per_dst=[10544, 10074, 9564, 10306, 12378, 5032, 9606, 8836] dup_rows_per_dst=[3454, 2350, 3432, 3080, 4428, 762, 2658, 1800] combine_rows_per_src=[10544, 10074, 9564, 10306, 12378, 5032, 9606, 8836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73838 payload_rows_remote=64200 dup_rows=24466 dup_rows_remote=21008 dedup_ratio=0.248881 router_tokens_per_expert=[111, 934, 527, 329, 103, 1185, 804, 276, 758, 373, 904, 1210, 1529, 1052, 179, 1359, 939, 1407, 351, 376, 133, 249, 1412, 828, 511, 715, 960, 733, 271, 507, 906, 328, 405, 1239, 1269, 736, 2279, 1793, 1896, 570, 355, 261, 332, 196, 1706, 1220, 165, 1229, 755, 136, 1071, 871, 125, 370, 225, 890, 1516, 554, 53, 519, 123, 1429, 291, 2314] payload_rows_per_dst=[9638, 10608, 11346, 8324, 8772, 10290, 6984, 7876] dup_rows_per_dst=[3458, 4870, 3086, 1538, 3252, 5266, 1902, 1094] combine_rows_per_src=[9638, 10608, 11346, 8324, 8772, 10290, 6984, 7876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74974 payload_rows_remote=63368 dup_rows=23330 dup_rows_remote=18546 dedup_ratio=0.237325 router_tokens_per_expert=[657, 1121, 1316, 316, 409, 473, 735, 726, 1440, 2119, 366, 374, 1264, 408, 1028, 520, 1359, 71, 2199, 576, 487, 213, 38, 870, 981, 733, 663, 519, 533, 776, 1104, 747, 291, 762, 300, 372, 524, 645, 1521, 534, 727, 2411, 84, 538, 335, 603, 526, 3640, 184, 376, 423, 924, 421, 1185, 427, 474, 189, 349, 1510, 797, 1147, 846, 563, 383] payload_rows_per_dst=[8992, 7068, 8240, 9470, 7812, 11960, 9826, 11606] dup_rows_per_dst=[2514, 852, 1868, 2642, 2086, 5344, 3240, 4784] combine_rows_per_src=[8992, 7068, 8240, 9470, 7812, 11960, 9826, 11606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271425536 dispatch_meta_bytes=343520 combine_payload_bytes=271425536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271769056 ingress_bytes=375234560 link_bytes=375234560 token_bytes=543194592 weight_movement_bytes=103809024 total_nvlink_bytes=647003616 weight_to_token_ratio=0.191108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411977440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=67616 dup_rows=22980 dup_rows_remote=20890 dedup_ratio=0.233765 router_tokens_per_expert=[887, 1288, 1162, 375, 374, 465, 757, 712, 1388, 2365, 341, 444, 1142, 401, 1006, 529, 1370, 84, 2302, 626, 453, 281, 54, 812, 885, 621, 625, 605, 540, 761, 1169, 852, 259, 963, 315, 375, 551, 500, 1423, 513, 774, 2225, 122, 498, 332, 624, 577, 3238, 302, 392, 468, 928, 394, 1184, 426, 524, 294, 323, 1431, 810, 952, 890, 547, 322] payload_rows_per_dst=[9494, 8908, 8954, 9516, 7708, 11308, 10264, 9172] dup_rows_per_dst=[2546, 1594, 3010, 2600, 2090, 5472, 3702, 1966] combine_rows_per_src=[9494, 8908, 8954, 9516, 7708, 11308, 10264, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262963200 dispatch_meta_bytes=340832 combine_payload_bytes=262963200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263304032 ingress_bytes=297566208 link_bytes=297566208 token_bytes=526267232 weight_movement_bytes=34603008 total_nvlink_bytes=560870240 weight_to_token_ratio=0.065752 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119717685e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76654 payload_rows_remote=66898 dup_rows=21650 dup_rows_remote=18078 dedup_ratio=0.220235 router_tokens_per_expert=[514, 551, 1593, 200, 543, 731, 742, 850, 1775, 2499, 344, 466, 1618, 362, 1407, 517, 1849, 39, 2479, 497, 705, 316, 8, 771, 913, 1023, 670, 400, 410, 947, 1271, 586, 491, 957, 262, 347, 693, 475, 960, 616, 418, 2915, 120, 508, 265, 501, 319, 3503, 30, 250, 191, 971, 239, 1148, 206, 181, 299, 192, 1583, 588, 1020, 737, 243, 328] payload_rows_per_dst=[8820, 10284, 9756, 9814, 12380, 8152, 9054, 8394] dup_rows_per_dst=[2628, 2694, 3572, 2626, 4228, 1940, 2376, 1586] combine_rows_per_src=[8820, 10284, 9756, 9814, 12380, 8152, 9054, 8394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76042 payload_rows_remote=66370 dup_rows=22262 dup_rows_remote=19424 dedup_ratio=0.226461 router_tokens_per_expert=[474, 892, 1261, 355, 514, 637, 698, 724, 2082, 2444, 419, 332, 1188, 456, 1418, 350, 1697, 52, 2212, 509, 431, 285, 34, 626, 925, 847, 623, 480, 545, 862, 1202, 1006, 274, 803, 364, 337, 704, 514, 1289, 416, 507, 2660, 112, 492, 279, 457, 460, 3505, 61, 405, 294, 920, 353, 1109, 325, 344, 289, 229, 1492, 739, 895, 1057, 434, 453] payload_rows_per_dst=[8884, 9808, 8348, 10294, 7560, 10262, 9672, 11214] dup_rows_per_dst=[2226, 2192, 2582, 2686, 1842, 3216, 2838, 4680] combine_rows_per_src=[8884, 9808, 8348, 10294, 7560, 10262, 9672, 11214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76282 payload_rows_remote=64718 dup_rows=22022 dup_rows_remote=17182 dedup_ratio=0.224019 router_tokens_per_expert=[378, 425, 1494, 180, 582, 675, 741, 799, 2362, 2812, 505, 360, 1412, 441, 1835, 228, 1944, 53, 2253, 512, 663, 390, 15, 694, 552, 1103, 570, 446, 636, 900, 1142, 688, 429, 675, 180, 266, 628, 502, 1149, 419, 368, 3015, 117, 440, 273, 305, 393, 3291, 13, 298, 196, 949, 405, 982, 198, 218, 199, 187, 1525, 604, 1157, 999, 299, 683] payload_rows_per_dst=[8542, 10960, 9486, 10172, 7138, 11564, 9736, 8684] dup_rows_per_dst=[2006, 3326, 3562, 1902, 1358, 4840, 2406, 2622] combine_rows_per_src=[8542, 10960, 9486, 10172, 7138, 11564, 9736, 8684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76734 payload_rows_remote=66730 dup_rows=21570 dup_rows_remote=19212 dedup_ratio=0.219421 router_tokens_per_expert=[452, 406, 1551, 169, 588, 762, 630, 863, 2185, 2763, 450, 384, 1832, 391, 1534, 248, 1858, 56, 2452, 499, 691, 330, 9, 697, 769, 1210, 649, 432, 444, 1138, 1019, 520, 339, 721, 170, 338, 659, 402, 1005, 389, 301, 3169, 107, 473, 251, 313, 313, 3914, 21, 261, 149, 1039, 307, 989, 117, 110, 159, 161, 1364, 608, 1220, 949, 354, 499] payload_rows_per_dst=[8642, 10818, 9590, 10004, 9292, 10406, 9250, 8732] dup_rows_per_dst=[2200, 3230, 3594, 2358, 2364, 3666, 2262, 1896] combine_rows_per_src=[8642, 10818, 9590, 10004, 9292, 10406, 9250, 8732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259555328 dispatch_meta_bytes=327656 combine_payload_bytes=259555328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259882984 ingress_bytes=294158336 link_bytes=294158336 token_bytes=519438312 weight_movement_bytes=34603008 total_nvlink_bytes=554041320 weight_to_token_ratio=0.066616 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106894137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276955136 dispatch_meta_bytes=354024 combine_payload_bytes=276955136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277309160 ingress_bytes=311558144 link_bytes=311558144 token_bytes=554264296 weight_movement_bytes=34603008 total_nvlink_bytes=588867304 weight_to_token_ratio=0.062431 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172368213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274014208 dispatch_meta_bytes=339904 combine_payload_bytes=274014208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274354112 ingress_bytes=343220224 link_bytes=343220224 token_bytes=548368320 weight_movement_bytes=69206016 total_nvlink_bytes=617574336 weight_to_token_ratio=0.126204 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291510071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271851520 dispatch_meta_bytes=343176 combine_payload_bytes=271851520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272194696 ingress_bytes=306454528 link_bytes=306454528 token_bytes=544046216 weight_movement_bytes=34603008 total_nvlink_bytes=578649224 weight_to_token_ratio=0.063603 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153163717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265084928 dispatch_meta_bytes=327600 combine_payload_bytes=265084928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265412528 ingress_bytes=403496960 link_bytes=403496960 token_bytes=530497456 weight_movement_bytes=138412032 total_nvlink_bytes=668909488 weight_to_token_ratio=0.260910 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518326576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71418 payload_rows_remote=60968 dup_rows=26886 dup_rows_remote=21532 dedup_ratio=0.273499 router_tokens_per_expert=[0, 1249, 1507, 837, 1191, 1054, 699, 1387, 297, 1074, 2439, 108, 164, 567, 367, 847, 1759, 34, 86, 134, 94, 454, 353, 164, 802, 375, 2075, 465, 1984, 1355, 615, 67, 299, 3, 27, 1051, 908, 142, 114, 1098, 330, 388, 72, 929, 529, 2, 2435, 1, 142, 853, 237, 1223, 386, 1346, 1002, 1212, 1134, 441, 227, 2039, 1275, 2436, 1155, 1113] payload_rows_per_dst=[10010, 10450, 9396, 10682, 7800, 8006, 8078, 6996] dup_rows_per_dst=[5516, 5354, 3858, 4794, 2498, 1366, 2032, 1468] combine_rows_per_src=[10010, 10450, 9396, 10682, 7800, 8006, 8078, 6996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=273326080 dispatch_meta_bytes=343768 combine_payload_bytes=273326080 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=273669848 ingress_bytes=307929088 link_bytes=307929088 token_bytes=546995928 weight_movement_bytes=34603008 total_nvlink_bytes=581598936 weight_to_token_ratio=0.063260 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158712368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75850 payload_rows_remote=63852 dup_rows=22454 dup_rows_remote=17640 dedup_ratio=0.228414 router_tokens_per_expert=[653, 746, 1276, 318, 525, 538, 749, 737, 1872, 2481, 465, 384, 1232, 451, 1365, 401, 1903, 72, 2110, 540, 609, 341, 31, 767, 616, 1002, 650, 478, 564, 954, 1146, 738, 278, 710, 257, 306, 667, 515, 1377, 381, 449, 2864, 126, 519, 309, 344, 443, 3329, 46, 350, 246, 969, 316, 948, 250, 273, 259, 271, 1579, 735, 1132, 948, 481, 761] payload_rows_per_dst=[11998, 10638, 9290, 9386, 11738, 4138, 9128, 9534] dup_rows_per_dst=[4814, 3508, 3456, 1448, 3902, 242, 2286, 2798] combine_rows_per_src=[11998, 10638, 9290, 9386, 11738, 4138, 9128, 9534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=249724928 dispatch_meta_bytes=330000 combine_payload_bytes=249724928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250054928 ingress_bytes=284327936 link_bytes=284327936 token_bytes=499779856 weight_movement_bytes=34603008 total_nvlink_bytes=534382864 weight_to_token_ratio=0.069237 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069903133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71332 payload_rows_remote=59316 dup_rows=26972 dup_rows_remote=20814 dedup_ratio=0.274373 router_tokens_per_expert=[23, 1404, 1780, 679, 1077, 985, 894, 840, 372, 1019, 1801, 145, 276, 515, 401, 1102, 1761, 436, 180, 92, 110, 487, 232, 293, 810, 423, 1186, 694, 1467, 1113, 743, 126, 507, 48, 157, 640, 1035, 410, 194, 995, 365, 378, 493, 900, 888, 14, 2036, 20, 426, 843, 285, 1940, 453, 1308, 879, 1158, 1403, 384, 335, 1530, 1063, 2167, 1064, 1368] payload_rows_per_dst=[7464, 8422, 8612, 7688, 9530, 9608, 7992, 12016] dup_rows_per_dst=[2660, 2840, 2904, 2502, 3682, 3514, 2712, 6158] combine_rows_per_src=[7464, 8422, 8612, 7688, 9530, 9608, 7992, 12016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71518 payload_rows_remote=62022 dup_rows=26786 dup_rows_remote=23048 dedup_ratio=0.272481 router_tokens_per_expert=[12, 1431, 1669, 849, 1105, 919, 921, 945, 503, 1164, 1847, 210, 296, 492, 400, 969, 1648, 451, 224, 144, 130, 470, 290, 205, 766, 373, 872, 580, 1439, 1052, 682, 168, 556, 82, 108, 673, 1036, 404, 148, 996, 325, 321, 562, 884, 653, 6, 2016, 27, 525, 792, 279, 2247, 475, 1335, 746, 1210, 1511, 379, 370, 1425, 1061, 2449, 908, 1417] payload_rows_per_dst=[7654, 8684, 8920, 7072, 9496, 9406, 10562, 9724] dup_rows_per_dst=[2820, 3078, 3102, 1914, 3738, 3060, 4656, 4418] combine_rows_per_src=[7654, 8684, 8920, 7072, 9496, 9406, 10562, 9724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261537792 dispatch_meta_bytes=325968 combine_payload_bytes=261537792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261863760 ingress_bytes=296140800 link_bytes=296140800 token_bytes=523401552 weight_movement_bytes=34603008 total_nvlink_bytes=558004560 weight_to_token_ratio=0.066112 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114353989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72474 payload_rows_remote=62484 dup_rows=25830 dup_rows_remote=21248 dedup_ratio=0.262756 router_tokens_per_expert=[0, 722, 1546, 1263, 1116, 1052, 675, 1866, 253, 1464, 2055, 80, 143, 459, 317, 568, 2000, 97, 78, 67, 93, 841, 295, 191, 1050, 359, 1592, 361, 2021, 1169, 547, 89, 400, 0, 5, 1234, 855, 35, 96, 972, 242, 389, 15, 879, 567, 0, 2778, 0, 99, 1024, 271, 1139, 395, 1692, 1551, 1273, 968, 407, 142, 1746, 1088, 2270, 837, 1354] payload_rows_per_dst=[10290, 9300, 9990, 10134, 8832, 8538, 8820, 6570] dup_rows_per_dst=[4606, 3880, 4582, 4242, 3330, 1202, 2684, 1304] combine_rows_per_src=[10290, 9300, 9990, 10134, 8832, 8538, 8820, 6570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72168 payload_rows_remote=60896 dup_rows=26136 dup_rows_remote=22116 dedup_ratio=0.265869 router_tokens_per_expert=[0, 785, 1437, 1093, 1060, 1066, 639, 1577, 208, 1158, 2266, 89, 121, 547, 293, 597, 1717, 21, 106, 59, 87, 643, 400, 154, 710, 302, 2633, 336, 2652, 1160, 427, 57, 290, 0, 1, 1158, 659, 35, 55, 987, 305, 289, 15, 906, 470, 0, 3008, 1, 38, 1074, 275, 1249, 438, 1692, 1568, 1235, 962, 399, 143, 1983, 1194, 2122, 903, 1298] payload_rows_per_dst=[7370, 8272, 8162, 8524, 8926, 11272, 10132, 9510] dup_rows_per_dst=[1916, 2286, 2456, 2726, 3472, 4020, 5006, 4254] combine_rows_per_src=[7370, 8272, 8162, 8524, 8926, 11272, 10132, 9510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70660 payload_rows_remote=61368 dup_rows=27644 dup_rows_remote=23606 dedup_ratio=0.281209 router_tokens_per_expert=[0, 1221, 1617, 793, 1067, 1078, 721, 1435, 445, 1022, 1942, 120, 101, 546, 335, 745, 2055, 124, 233, 107, 113, 525, 313, 218, 891, 407, 1603, 473, 1780, 1381, 512, 102, 427, 20, 29, 963, 970, 193, 121, 1126, 315, 634, 358, 988, 634, 6, 2462, 2, 202, 862, 343, 1270, 484, 1463, 1138, 1073, 1333, 290, 273, 1412, 1137, 2166, 975, 1458] payload_rows_per_dst=[7316, 8152, 8542, 7890, 9646, 10182, 9292, 9640] dup_rows_per_dst=[2444, 2360, 3166, 2848, 4156, 4176, 4038, 4456] combine_rows_per_src=[7316, 8152, 8542, 7890, 9646, 10182, 9292, 9640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=65204 dup_rows=24226 dup_rows_remote=21720 dedup_ratio=0.246440 router_tokens_per_expert=[0, 704, 1436, 1163, 1136, 982, 636, 1544, 191, 1298, 2734, 85, 134, 554, 271, 431, 2021, 7, 44, 46, 76, 1139, 483, 235, 973, 41, 2356, 241, 2730, 965, 426, 63, 250, 0, 3, 1481, 694, 17, 51, 848, 269, 485, 11, 872, 418, 0, 2921, 0, 30, 1231, 225, 926, 424, 1879, 1709, 1039, 991, 366, 100, 1843, 1145, 1644, 823, 1312] payload_rows_per_dst=[9276, 8848, 8866, 8874, 9176, 10784, 9028, 9226] dup_rows_per_dst=[3086, 2548, 2608, 2506, 3472, 3378, 2778, 3850] combine_rows_per_src=[9276, 8848, 8866, 8874, 9176, 10784, 9028, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=242958336 dispatch_meta_bytes=320520 combine_payload_bytes=242958336 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=243278856 ingress_bytes=381370368 link_bytes=381370368 token_bytes=486237192 weight_movement_bytes=138412032 total_nvlink_bytes=624649224 weight_to_token_ratio=0.284659 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435065992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254042112 dispatch_meta_bytes=340280 combine_payload_bytes=254042112 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=254382392 ingress_bytes=323248128 link_bytes=323248128 token_bytes=508424504 weight_movement_bytes=69206016 total_nvlink_bytes=577630520 weight_to_token_ratio=0.136119 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216356682e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=255934464 dispatch_meta_bytes=334928 combine_payload_bytes=255934464 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256269392 ingress_bytes=325140480 link_bytes=325140480 token_bytes=512203856 weight_movement_bytes=69206016 total_nvlink_bytes=581409872 weight_to_token_ratio=0.135114 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223477450e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=249430016 dispatch_meta_bytes=332048 combine_payload_bytes=249430016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249762064 ingress_bytes=284033024 link_bytes=284033024 token_bytes=499192080 weight_movement_bytes=34603008 total_nvlink_bytes=533795088 weight_to_token_ratio=0.069318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.068793403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78504 payload_rows_remote=66920 dup_rows=19800 dup_rows_remote=15988 dedup_ratio=0.201416 router_tokens_per_expert=[432, 2020, 839, 459, 109, 383, 2651, 424, 6, 0, 420, 803, 1844, 0, 2, 754, 510, 941, 3942, 30, 169, 528, 361, 1, 150, 3025, 722, 135, 271, 214, 3869, 294, 2021, 510, 152, 370, 568, 271, 1362, 625, 44, 0, 27, 1548, 85, 460, 2328, 565, 846, 1597, 1392, 723, 40, 542, 0, 1117, 1145, 388, 479, 898, 172, 473, 821, 1275] payload_rows_per_dst=[10610, 11584, 10602, 8822, 9378, 8298, 9962, 9248] dup_rows_per_dst=[4024, 3812, 2362, 800, 2380, 1816, 2552, 2054] combine_rows_per_src=[10610, 11584, 10602, 8822, 9378, 8298, 9962, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=251363328 dispatch_meta_bytes=339896 combine_payload_bytes=251363328 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=251703224 ingress_bytes=355172352 link_bytes=355172352 token_bytes=503066552 weight_movement_bytes=103809024 total_nvlink_bytes=606875576 weight_to_token_ratio=0.206352 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336484967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=347696 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267423280 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534498864 weight_movement_bytes=69206016 total_nvlink_bytes=603704880 weight_to_token_ratio=0.129478 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71250 payload_rows_remote=60846 dup_rows=27054 dup_rows_remote=21582 dedup_ratio=0.275208 router_tokens_per_expert=[0, 1055, 1639, 847, 1149, 1206, 719, 1544, 339, 805, 2130, 121, 156, 678, 349, 790, 1692, 61, 86, 87, 95, 504, 413, 171, 741, 429, 1997, 432, 2057, 1228, 559, 78, 358, 2, 24, 1020, 885, 171, 124, 1052, 319, 295, 75, 974, 621, 0, 2611, 1, 141, 1008, 250, 1436, 456, 1418, 1209, 1180, 1080, 477, 232, 1547, 1355, 2036, 1213, 1425] payload_rows_per_dst=[10404, 9322, 9152, 10738, 7840, 8378, 8322, 7094] dup_rows_per_dst=[5472, 4508, 3988, 5170, 2710, 1414, 2172, 1620] combine_rows_per_src=[10404, 9322, 9152, 10738, 7840, 8378, 8322, 7094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=331632 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274435952 ingress_bytes=308707328 link_bytes=308707328 token_bytes=548540272 weight_movement_bytes=34603008 total_nvlink_bytes=583143280 weight_to_token_ratio=0.063082 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161640822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75936 payload_rows_remote=62664 dup_rows=22368 dup_rows_remote=17258 dedup_ratio=0.227539 router_tokens_per_expert=[509, 1588, 1028, 390, 355, 321, 2523, 334, 57, 5, 877, 969, 1665, 0, 28, 1258, 589, 1133, 3153, 52, 284, 288, 318, 7, 205, 1694, 756, 104, 498, 289, 3663, 301, 1828, 578, 204, 842, 706, 330, 1430, 1066, 121, 3, 144, 1691, 90, 411, 1797, 454, 922, 1497, 786, 627, 150, 827, 0, 1119, 977, 494, 510, 1191, 252, 575, 956, 1333] payload_rows_per_dst=[7382, 8284, 5008, 11880, 10280, 10262, 9568, 13272] dup_rows_per_dst=[1668, 1434, 334, 3140, 3688, 4206, 2788, 5110] combine_rows_per_src=[7382, 8284, 5008, 11880, 10280, 10262, 9568, 13272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76798 payload_rows_remote=66438 dup_rows=21506 dup_rows_remote=17948 dedup_ratio=0.218770 router_tokens_per_expert=[594, 1495, 1146, 459, 483, 259, 2453, 430, 56, 0, 812, 896, 1381, 0, 33, 1297, 667, 971, 3397, 48, 314, 340, 253, 10, 238, 1711, 717, 129, 471, 170, 3840, 214, 1857, 726, 212, 923, 704, 263, 1454, 820, 111, 4, 187, 1531, 109, 334, 1779, 498, 829, 1480, 716, 622, 260, 803, 0, 1209, 917, 485, 469, 1187, 288, 597, 1081, 1413] payload_rows_per_dst=[8122, 9454, 9848, 9882, 10360, 9792, 9316, 10024] dup_rows_per_dst=[2140, 2502, 2152, 2092, 3558, 3690, 2522, 2850] combine_rows_per_src=[8122, 9454, 9848, 9882, 10360, 9792, 9316, 10024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=249225216 dispatch_meta_bytes=329712 combine_payload_bytes=249225216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=249554928 ingress_bytes=353034240 link_bytes=353034240 token_bytes=498780144 weight_movement_bytes=103809024 total_nvlink_bytes=602589168 weight_to_token_ratio=0.208126 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328439424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78154 payload_rows_remote=66314 dup_rows=20150 dup_rows_remote=17134 dedup_ratio=0.204976 router_tokens_per_expert=[316, 2244, 718, 291, 116, 385, 2342, 653, 5, 0, 218, 688, 2236, 0, 1, 657, 515, 1063, 4471, 22, 126, 836, 395, 0, 114, 2402, 631, 118, 266, 175, 3954, 252, 3003, 436, 226, 367, 453, 179, 913, 680, 45, 0, 6, 1344, 51, 423, 2399, 694, 857, 1628, 1252, 956, 86, 498, 0, 1084, 1216, 426, 488, 672, 184, 308, 697, 1371] payload_rows_per_dst=[10520, 11312, 11840, 7350, 10116, 8170, 9978, 8868] dup_rows_per_dst=[3610, 4206, 3016, 566, 2398, 1754, 2744, 1856] combine_rows_per_src=[10520, 11312, 11840, 7350, 10116, 8170, 9978, 8868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78588 payload_rows_remote=67488 dup_rows=19716 dup_rows_remote=15720 dedup_ratio=0.200562 router_tokens_per_expert=[389, 2059, 801, 475, 69, 278, 2560, 659, 3, 0, 432, 602, 2069, 0, 0, 596, 476, 864, 4495, 18, 74, 682, 509, 1, 126, 3204, 814, 106, 175, 195, 3703, 242, 2277, 399, 107, 303, 543, 108, 1259, 640, 22, 0, 4, 1281, 56, 505, 2288, 832, 888, 1279, 1536, 1021, 14, 556, 0, 1029, 1230, 317, 415, 685, 134, 295, 641, 1812] payload_rows_per_dst=[7918, 7110, 11716, 12654, 8908, 11100, 10002, 9180] dup_rows_per_dst=[1542, 294, 2522, 4476, 2364, 3996, 2644, 1878] combine_rows_per_src=[7918, 7110, 11716, 12654, 8908, 11100, 10002, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256671744 dispatch_meta_bytes=319688 combine_payload_bytes=256671744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256991432 ingress_bytes=291274752 link_bytes=291274752 token_bytes=513663176 weight_movement_bytes=34603008 total_nvlink_bytes=548266184 weight_to_token_ratio=0.067365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096043442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77468 payload_rows_remote=67884 dup_rows=20836 dup_rows_remote=18204 dedup_ratio=0.211955 router_tokens_per_expert=[628, 1786, 1030, 523, 207, 206, 2511, 443, 28, 1, 675, 727, 1551, 0, 13, 987, 704, 860, 3453, 38, 196, 394, 426, 13, 261, 2483, 721, 95, 466, 208, 3291, 296, 2187, 596, 193, 886, 557, 315, 1733, 799, 67, 1, 72, 1475, 92, 451, 1900, 577, 950, 1293, 1045, 775, 100, 604, 1, 1233, 1109, 501, 512, 944, 182, 374, 920, 1487] payload_rows_per_dst=[7958, 7290, 9880, 11748, 10852, 10422, 9584, 9734] dup_rows_per_dst=[1688, 674, 2074, 3894, 3680, 3870, 2632, 2324] combine_rows_per_src=[7958, 7290, 9880, 11748, 10852, 10422, 9584, 9734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78638 payload_rows_remote=69632 dup_rows=19666 dup_rows_remote=18900 dedup_ratio=0.200053 router_tokens_per_expert=[244, 1945, 654, 561, 81, 379, 2439, 574, 1, 0, 231, 625, 2463, 0, 0, 472, 441, 851, 4406, 25, 95, 761, 483, 0, 58, 3462, 775, 109, 149, 166, 3881, 167, 2766, 399, 136, 186, 550, 31, 1039, 586, 20, 0, 5, 1031, 54, 559, 2792, 754, 889, 1416, 1385, 1355, 15, 474, 0, 1026, 1238, 342, 456, 541, 158, 319, 653, 1479] payload_rows_per_dst=[10274, 11238, 11600, 9006, 9222, 8654, 9974, 8670] dup_rows_per_dst=[3480, 4108, 2524, 766, 2164, 1776, 3146, 1702] combine_rows_per_src=[10274, 11238, 11600, 9006, 9222, 8654, 9974, 8670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272130048 dispatch_meta_bytes=337544 combine_payload_bytes=272130048 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272467592 ingress_bytes=341336064 link_bytes=341336064 token_bytes=544597640 weight_movement_bytes=69206016 total_nvlink_bytes=613803656 weight_to_token_ratio=0.127077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284420129e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271622144 dispatch_meta_bytes=333792 combine_payload_bytes=271622144 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271955936 ingress_bytes=375431168 link_bytes=375431168 token_bytes=543578080 weight_movement_bytes=103809024 total_nvlink_bytes=647387104 weight_to_token_ratio=0.190974 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412717260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84356 payload_rows_remote=71504 dup_rows=13948 dup_rows_remote=9716 dedup_ratio=0.141886 router_tokens_per_expert=[766, 113, 0, 4955, 72, 0, 208, 95, 0, 427, 492, 263, 1011, 701, 433, 922, 0, 0, 542, 0, 4, 416, 376, 1788, 134, 54, 603, 0, 334, 487, 0, 0, 95, 2320, 311, 2, 0, 0, 0, 1888, 1080, 267, 208, 1747, 5486, 90, 915, 4293, 410, 78, 0, 1833, 257, 1784, 6289, 0, 4146, 0, 0, 3, 0, 0, 324, 130] payload_rows_per_dst=[11466, 12852, 13944, 13568, 8290, 7072, 8260, 8904] dup_rows_per_dst=[952, 4232, 4886, 628, 942, 1542, 464, 302] combine_rows_per_src=[11466, 12852, 13944, 13568, 8290, 7072, 8260, 8904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276430848 dispatch_meta_bytes=332832 combine_payload_bytes=276430848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276763680 ingress_bytes=311033856 link_bytes=311033856 token_bytes=553194528 weight_movement_bytes=34603008 total_nvlink_bytes=587797536 weight_to_token_ratio=0.062551 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170395360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285212672 dispatch_meta_bytes=354128 combine_payload_bytes=285212672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285566800 ingress_bytes=354418688 link_bytes=354418688 token_bytes=570779472 weight_movement_bytes=69206016 total_nvlink_bytes=639985488 weight_to_token_ratio=0.121248 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333648990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278052864 dispatch_meta_bytes=344352 combine_payload_bytes=278052864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278397216 ingress_bytes=347258880 link_bytes=347258880 token_bytes=556450080 weight_movement_bytes=69206016 total_nvlink_bytes=625656096 weight_to_token_ratio=0.124371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306707209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77892 payload_rows_remote=66452 dup_rows=20412 dup_rows_remote=14484 dedup_ratio=0.207642 router_tokens_per_expert=[401, 1591, 862, 394, 110, 338, 2761, 654, 8, 0, 621, 751, 1717, 0, 4, 709, 574, 999, 3669, 28, 156, 534, 409, 1, 156, 2902, 880, 121, 244, 213, 3691, 280, 1914, 610, 179, 415, 641, 248, 1333, 754, 50, 0, 57, 1810, 56, 436, 2267, 573, 872, 1308, 1446, 875, 22, 597, 0, 1144, 1147, 396, 508, 758, 179, 434, 772, 1573] payload_rows_per_dst=[11440, 11478, 10448, 9680, 7864, 8644, 8940, 9398] dup_rows_per_dst=[5928, 3524, 2378, 1548, 1092, 1854, 1952, 2136] combine_rows_per_src=[11440, 11478, 10448, 9680, 7864, 8644, 8940, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=292880384 dispatch_meta_bytes=324880 combine_payload_bytes=292880384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293205264 ingress_bytes=327483392 link_bytes=327483392 token_bytes=586085648 weight_movement_bytes=34603008 total_nvlink_bytes=620688656 weight_to_token_ratio=0.059041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232293640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78916 payload_rows_remote=66448 dup_rows=19388 dup_rows_remote=14894 dedup_ratio=0.197225 router_tokens_per_expert=[914, 187, 0, 4213, 84, 12, 315, 88, 0, 398, 655, 338, 1057, 785, 454, 1029, 4, 0, 521, 15, 200, 394, 416, 2613, 104, 84, 953, 0, 617, 251, 0, 3, 88, 2358, 613, 13, 0, 0, 0, 1264, 1011, 256, 248, 1831, 5132, 298, 928, 3266, 429, 145, 0, 1425, 326, 1435, 6167, 0, 4302, 0, 0, 8, 392, 0, 373, 140] payload_rows_per_dst=[10256, 7640, 7386, 3928, 7784, 14176, 15278, 12468] dup_rows_per_dst=[1370, 1792, 940, 96, 888, 5232, 4576, 4494] combine_rows_per_src=[10256, 7640, 7386, 3928, 7784, 14176, 15278, 12468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272187392 dispatch_meta_bytes=323744 combine_payload_bytes=272187392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272511136 ingress_bytes=341393408 link_bytes=341393408 token_bytes=544698528 weight_movement_bytes=69206016 total_nvlink_bytes=613904544 weight_to_token_ratio=0.127054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284635910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79022 payload_rows_remote=65164 dup_rows=19282 dup_rows_remote=12066 dedup_ratio=0.196147 router_tokens_per_expert=[933, 181, 0, 4210, 91, 10, 259, 94, 0, 407, 905, 413, 1026, 720, 437, 924, 0, 3, 475, 15, 298, 387, 411, 2340, 122, 157, 877, 0, 582, 126, 0, 11, 95, 2347, 576, 9, 0, 0, 0, 1335, 1032, 216, 199, 1968, 5006, 246, 1150, 3528, 406, 168, 0, 1563, 365, 1125, 6175, 0, 4301, 1, 0, 7, 366, 0, 399, 155] payload_rows_per_dst=[10202, 7804, 7002, 12224, 13858, 11684, 6898, 9350] dup_rows_per_dst=[1354, 1860, 856, 1538, 7216, 4994, 356, 1108] combine_rows_per_src=[10202, 7804, 7002, 12224, 13858, 11684, 6898, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85062 payload_rows_remote=70802 dup_rows=13242 dup_rows_remote=8844 dedup_ratio=0.134705 router_tokens_per_expert=[739, 85, 1, 5248, 72, 0, 181, 85, 0, 447, 273, 310, 883, 813, 418, 639, 0, 0, 320, 1, 4, 406, 351, 1803, 116, 91, 551, 0, 239, 293, 0, 0, 86, 2408, 330, 1, 0, 0, 0, 1998, 1062, 271, 180, 1588, 5574, 77, 960, 4195, 416, 73, 0, 1815, 82, 2492, 6444, 0, 4221, 0, 0, 0, 0, 0, 414, 96] payload_rows_per_dst=[11914, 12558, 14260, 13324, 8534, 6678, 8708, 9086] dup_rows_per_dst=[908, 3398, 4398, 404, 1112, 1598, 1048, 376] combine_rows_per_src=[11914, 12558, 14260, 13324, 8534, 6678, 8708, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80360 payload_rows_remote=71550 dup_rows=17944 dup_rows_remote=15262 dedup_ratio=0.182536 router_tokens_per_expert=[745, 86, 0, 5101, 68, 0, 201, 65, 0, 356, 317, 282, 893, 709, 370, 674, 0, 0, 282, 0, 0, 435, 438, 1263, 103, 59, 506, 0, 289, 209, 0, 0, 66, 2507, 178, 0, 0, 0, 0, 2543, 1043, 278, 198, 1452, 5576, 79, 755, 4497, 365, 78, 0, 1742, 122, 2745, 6533, 0, 4431, 0, 0, 0, 0, 0, 407, 106] payload_rows_per_dst=[11618, 6222, 4496, 13210, 11684, 8810, 12404, 11916] dup_rows_per_dst=[914, 980, 340, 274, 5184, 2682, 4486, 3084] combine_rows_per_src=[11618, 6222, 4496, 13210, 11684, 8810, 12404, 11916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=272171008 dispatch_meta_bytes=325368 combine_payload_bytes=272171008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272496376 ingress_bytes=306774016 link_bytes=306774016 token_bytes=544667384 weight_movement_bytes=34603008 total_nvlink_bytes=579270392 weight_to_token_ratio=0.063531 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154365925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84788 payload_rows_remote=71742 dup_rows=13516 dup_rows_remote=13338 dedup_ratio=0.137492 router_tokens_per_expert=[769, 77, 0, 5362, 51, 0, 189, 79, 0, 381, 174, 157, 913, 826, 398, 769, 0, 0, 167, 0, 1, 419, 397, 1479, 119, 71, 472, 0, 202, 85, 0, 0, 79, 2316, 298, 0, 0, 0, 0, 2379, 1033, 270, 188, 1498, 5663, 65, 633, 4467, 391, 65, 0, 2027, 52, 2739, 6633, 0, 4417, 0, 0, 0, 0, 0, 311, 71] payload_rows_per_dst=[12106, 10962, 12202, 13046, 10338, 7774, 9032, 9328] dup_rows_per_dst=[948, 2644, 2758, 178, 3038, 2164, 1516, 270] combine_rows_per_src=[12106, 10962, 12202, 13046, 10338, 7774, 9032, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79726 payload_rows_remote=64460 dup_rows=18578 dup_rows_remote=13178 dedup_ratio=0.188985 router_tokens_per_expert=[957, 90, 0, 4170, 108, 5, 226, 52, 0, 496, 491, 497, 1127, 682, 428, 803, 0, 3, 647, 16, 59, 450, 470, 1753, 96, 91, 881, 0, 453, 129, 0, 1, 55, 2296, 395, 9, 0, 1, 0, 1885, 1119, 262, 197, 1954, 5180, 181, 962, 3830, 410, 173, 0, 1299, 399, 1870, 6181, 1, 4320, 0, 0, 6, 399, 0, 445, 142] payload_rows_per_dst=[10094, 7404, 5792, 10786, 8390, 8990, 15266, 13004] dup_rows_per_dst=[1122, 1644, 1004, 828, 892, 2408, 5400, 5280] combine_rows_per_src=[10094, 7404, 5792, 10786, 8390, 8990, 15266, 13004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=308920 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267220664 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534132408 weight_movement_bytes=34603008 total_nvlink_bytes=568735416 weight_to_token_ratio=0.064784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76402 payload_rows_remote=71866 dup_rows=21902 dup_rows_remote=21240 dedup_ratio=0.222799 router_tokens_per_expert=[2336, 6998, 0, 38, 633, 1, 59, 1, 0, 330, 104, 382, 80, 350, 380, 973, 165, 1317, 0, 0, 296, 94, 1566, 1126, 3694, 4862, 1715, 431, 0, 0, 73, 248, 0, 285, 845, 0, 73, 1468, 514, 91, 556, 4, 0, 53, 650, 6252, 73, 0, 5843, 0, 531, 959, 481, 73, 250, 0, 0, 311, 0, 0, 0, 1089, 265, 234] payload_rows_per_dst=[14726, 4536, 10132, 12382, 14410, 4652, 4148, 11416] dup_rows_per_dst=[5406, 662, 3886, 4988, 3828, 586, 440, 2106] combine_rows_per_src=[14726, 4536, 10132, 12382, 14410, 4652, 4148, 11416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290004992 dispatch_meta_bytes=318584 combine_payload_bytes=290004992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290323576 ingress_bytes=324608000 link_bytes=324608000 token_bytes=580328568 weight_movement_bytes=34603008 total_nvlink_bytes=614931576 weight_to_token_ratio=0.059627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221473771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293068800 dispatch_meta_bytes=347248 combine_payload_bytes=293068800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=293416048 ingress_bytes=431480832 link_bytes=431480832 token_bytes=586484848 weight_movement_bytes=138412032 total_nvlink_bytes=724896880 weight_to_token_ratio=0.236003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.623627634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293855232 dispatch_meta_bytes=340320 combine_payload_bytes=293855232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294195552 ingress_bytes=328458240 link_bytes=328458240 token_bytes=588050784 weight_movement_bytes=34603008 total_nvlink_bytes=622653792 weight_to_token_ratio=0.058844 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235961914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83182 payload_rows_remote=71726 dup_rows=15122 dup_rows_remote=13498 dedup_ratio=0.153829 router_tokens_per_expert=[727, 102, 0, 4846, 75, 0, 203, 82, 0, 428, 430, 210, 1031, 677, 453, 987, 0, 0, 590, 0, 5, 386, 432, 1744, 121, 68, 652, 0, 306, 509, 0, 0, 82, 2399, 244, 2, 0, 0, 0, 2350, 1038, 274, 201, 1538, 5480, 86, 968, 3940, 431, 86, 0, 1382, 223, 2329, 6368, 0, 4173, 0, 0, 1, 0, 0, 362, 131] payload_rows_per_dst=[11456, 12486, 13308, 13744, 9082, 9560, 4570, 8976] dup_rows_per_dst=[1624, 3826, 4732, 528, 1072, 2940, 42, 358] combine_rows_per_src=[11456, 12486, 13308, 13744, 9082, 9560, 4570, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=264028160 dispatch_meta_bytes=310552 combine_payload_bytes=264028160 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264338712 ingress_bytes=402440192 link_bytes=402440192 token_bytes=528366872 weight_movement_bytes=138412032 total_nvlink_bytes=666778904 weight_to_token_ratio=0.261962 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514350043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=294363136 dispatch_meta_bytes=372424 combine_payload_bytes=294363136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294735560 ingress_bytes=328966144 link_bytes=328966144 token_bytes=589098696 weight_movement_bytes=34603008 total_nvlink_bytes=623701704 weight_to_token_ratio=0.058739 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237873116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78816 payload_rows_remote=68180 dup_rows=19488 dup_rows_remote=18258 dedup_ratio=0.198242 router_tokens_per_expert=[1741, 5655, 0, 189, 912, 60, 58, 26, 0, 153, 188, 381, 100, 318, 432, 1028, 392, 926, 0, 0, 727, 101, 3076, 1728, 2302, 4458, 1348, 461, 0, 222, 187, 329, 0, 266, 1683, 0, 86, 946, 656, 157, 629, 12, 0, 50, 785, 6104, 215, 0, 5796, 0, 990, 833, 504, 110, 349, 8, 1, 425, 0, 0, 0, 558, 189, 302] payload_rows_per_dst=[5302, 12498, 10906, 4948, 7242, 12994, 14290, 10636] dup_rows_per_dst=[670, 3264, 2994, 146, 346, 3344, 7494, 1230] combine_rows_per_src=[5302, 12498, 10906, 4948, 7242, 12994, 14290, 10636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293789696 dispatch_meta_bytes=340896 combine_payload_bytes=293789696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294130592 ingress_bytes=328392704 link_bytes=328392704 token_bytes=587920288 weight_movement_bytes=34603008 total_nvlink_bytes=622523296 weight_to_token_ratio=0.058857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235715307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81970 payload_rows_remote=74732 dup_rows=16334 dup_rows_remote=15910 dedup_ratio=0.166158 router_tokens_per_expert=[1629, 5669, 1, 189, 820, 106, 66, 27, 0, 134, 195, 458, 132, 328, 417, 920, 431, 800, 0, 0, 660, 89, 3159, 1640, 2305, 4094, 1646, 462, 0, 254, 266, 242, 0, 249, 1633, 0, 92, 943, 752, 162, 654, 9, 0, 54, 765, 6235, 279, 0, 5761, 0, 1171, 887, 483, 121, 350, 4, 3, 382, 0, 0, 0, 566, 143, 315] payload_rows_per_dst=[5018, 13188, 10830, 8950, 7238, 13104, 13710, 9932] dup_rows_per_dst=[658, 3318, 2728, 1400, 424, 2888, 3844, 1074] combine_rows_per_src=[5018, 13188, 10830, 8950, 7238, 13104, 13710, 9932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80150 payload_rows_remote=73278 dup_rows=18154 dup_rows_remote=16330 dedup_ratio=0.184672 router_tokens_per_expert=[2436, 7287, 0, 24, 518, 1, 51, 0, 0, 320, 58, 342, 71, 312, 372, 784, 164, 1324, 0, 0, 266, 82, 1664, 848, 4156, 4914, 2314, 363, 1, 0, 79, 193, 0, 366, 510, 0, 75, 1414, 475, 88, 523, 2, 0, 88, 328, 6324, 16, 0, 5674, 0, 287, 1052, 484, 75, 246, 0, 0, 88, 0, 0, 0, 1597, 248, 248] payload_rows_per_dst=[15572, 3900, 6872, 11274, 14176, 13030, 3944, 11382] dup_rows_per_dst=[5062, 618, 1824, 2938, 3028, 1532, 344, 2808] combine_rows_per_src=[15572, 3900, 6872, 11274, 14176, 13030, 3944, 11382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80882 payload_rows_remote=67986 dup_rows=17422 dup_rows_remote=15646 dedup_ratio=0.177226 router_tokens_per_expert=[2673, 7349, 0, 20, 482, 6, 50, 1, 0, 360, 50, 321, 112, 311, 496, 941, 131, 1704, 0, 0, 262, 74, 724, 756, 5044, 4392, 2122, 371, 0, 0, 58, 231, 0, 393, 444, 0, 80, 1798, 628, 87, 675, 0, 0, 59, 319, 6252, 31, 0, 5375, 0, 322, 940, 441, 77, 56, 0, 0, 17, 0, 0, 0, 1581, 272, 264] payload_rows_per_dst=[5938, 15316, 5796, 11418, 6470, 12896, 12546, 10502] dup_rows_per_dst=[526, 4564, 1506, 2864, 390, 1776, 3246, 2550] combine_rows_per_src=[5938, 15316, 5796, 11418, 6470, 12896, 12546, 10502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=345752 combine_payload_bytes=279265280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279611032 ingress_bytes=313868288 link_bytes=313868288 token_bytes=558876312 weight_movement_bytes=34603008 total_nvlink_bytes=593479320 weight_to_token_ratio=0.061915 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181061099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80250 payload_rows_remote=68080 dup_rows=18054 dup_rows_remote=14864 dedup_ratio=0.183655 router_tokens_per_expert=[2968, 7447, 0, 10, 381, 0, 50, 0, 0, 318, 50, 417, 44, 216, 424, 993, 166, 1697, 0, 0, 265, 65, 879, 782, 4807, 4417, 2357, 264, 0, 0, 71, 181, 0, 422, 388, 0, 66, 1562, 597, 67, 578, 2, 0, 71, 171, 6347, 14, 0, 5459, 0, 224, 957, 453, 61, 123, 0, 0, 8, 0, 0, 0, 1744, 354, 215] payload_rows_per_dst=[11218, 9680, 6124, 12170, 12640, 12908, 4874, 10636] dup_rows_per_dst=[3200, 2538, 1584, 3190, 2692, 1458, 552, 2840] combine_rows_per_src=[11218, 9680, 6124, 12170, 12640, 12908, 4874, 10636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76626 payload_rows_remote=62118 dup_rows=21678 dup_rows_remote=12590 dedup_ratio=0.220520 router_tokens_per_expert=[2000, 6552, 11, 80, 900, 13, 37, 2, 0, 343, 129, 437, 152, 236, 528, 1031, 242, 1385, 0, 4, 404, 61, 2234, 1225, 3372, 3970, 1797, 340, 0, 41, 258, 147, 0, 421, 1009, 0, 107, 1273, 689, 138, 754, 15, 0, 66, 716, 5471, 230, 1, 5392, 1, 1177, 866, 512, 111, 361, 6, 2, 410, 1, 0, 1, 886, 234, 371] payload_rows_per_dst=[5412, 14268, 8698, 5046, 6884, 11868, 14508, 9942] dup_rows_per_dst=[674, 4548, 2412, 120, 390, 2638, 9088, 1808] combine_rows_per_src=[5412, 14268, 8698, 5046, 6884, 11868, 14508, 9942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=306102272 dispatch_meta_bytes=362568 combine_payload_bytes=306102272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306464840 ingress_bytes=340705280 link_bytes=340705280 token_bytes=612567112 weight_movement_bytes=34603008 total_nvlink_bytes=647170120 weight_to_token_ratio=0.056489 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282046540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79152 payload_rows_remote=73066 dup_rows=19152 dup_rows_remote=18082 dedup_ratio=0.194824 router_tokens_per_expert=[533, 122, 5636, 1953, 3185, 0, 6355, 5981, 0, 144, 0, 702, 0, 420, 0, 412, 263, 1, 2813, 0, 981, 58, 397, 153, 59, 72, 569, 0, 0, 131, 0, 133, 5719, 586, 0, 72, 208, 555, 676, 0, 375, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 0, 53, 2, 1390, 376, 0, 3139, 482, 1, 726, 1392, 76, 0, 0] payload_rows_per_dst=[12632, 6086, 14354, 13208, 2454, 13460, 6992, 9966] dup_rows_per_dst=[6426, 1070, 6416, 682, 1740, 0, 1152, 1666] combine_rows_per_src=[12632, 6086, 14354, 13208, 2454, 13460, 6992, 9966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300146688 dispatch_meta_bytes=358432 combine_payload_bytes=300146688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300505120 ingress_bytes=334749696 link_bytes=334749696 token_bytes=600651808 weight_movement_bytes=34603008 total_nvlink_bytes=635254816 weight_to_token_ratio=0.057609 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259636156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278470656 dispatch_meta_bytes=334528 combine_payload_bytes=278470656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278805184 ingress_bytes=382279680 link_bytes=382279680 token_bytes=557275840 weight_movement_bytes=103809024 total_nvlink_bytes=661084864 weight_to_token_ratio=0.186279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.438487660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278855680 dispatch_meta_bytes=331776 combine_payload_bytes=278855680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=279187456 ingress_bytes=382664704 link_bytes=382664704 token_bytes=558043136 weight_movement_bytes=103809024 total_nvlink_bytes=661852160 weight_to_token_ratio=0.186023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439936474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=254435328 dispatch_meta_bytes=298832 combine_payload_bytes=254435328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254734160 ingress_bytes=323641344 link_bytes=323641344 token_bytes=509169488 weight_movement_bytes=69206016 total_nvlink_bytes=578375504 weight_to_token_ratio=0.135919 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217836322e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=60334 dup_rows=23334 dup_rows_remote=19048 dedup_ratio=0.237366 router_tokens_per_expert=[1790, 7105, 0, 39, 465, 3, 59, 0, 0, 315, 93, 263, 86, 323, 504, 1119, 158, 1557, 0, 0, 397, 89, 1505, 1125, 3778, 4370, 1995, 356, 0, 0, 79, 168, 0, 362, 675, 0, 77, 1485, 638, 102, 659, 2, 0, 66, 529, 6175, 59, 0, 5734, 0, 719, 1015, 507, 80, 317, 0, 0, 423, 0, 0, 0, 1318, 232, 237] payload_rows_per_dst=[14636, 4576, 14238, 11650, 14300, 2480, 4844, 8246] dup_rows_per_dst=[4286, 830, 7774, 4820, 3846, 150, 432, 1196] combine_rows_per_src=[14636, 4576, 14238, 11650, 14300, 2480, 4844, 8246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=364592 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299642928 ingress_bytes=333881344 link_bytes=333881344 token_bytes=598921264 weight_movement_bytes=34603008 total_nvlink_bytes=633524272 weight_to_token_ratio=0.057776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83320 payload_rows_remote=71374 dup_rows=14984 dup_rows_remote=11122 dedup_ratio=0.152425 router_tokens_per_expert=[577, 186, 5312, 2202, 3327, 0, 6395, 6110, 0, 169, 0, 717, 0, 562, 0, 288, 258, 86, 2422, 36, 1000, 59, 360, 150, 54, 165, 850, 0, 3, 142, 0, 41, 5455, 651, 0, 85, 195, 443, 635, 0, 330, 0, 3, 0, 0, 0, 208, 48, 2316, 0, 117, 59, 36, 1407, 79, 0, 3775, 506, 12, 470, 725, 100, 26, 0] payload_rows_per_dst=[3834, 11726, 7138, 13574, 10164, 13546, 11392, 11946] dup_rows_per_dst=[64, 2370, 1604, 1156, 2216, 422, 3290, 3862] combine_rows_per_src=[3834, 11726, 7138, 13574, 10164, 13546, 11392, 11946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=247128064 dispatch_meta_bytes=317528 combine_payload_bytes=247128064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=247445592 ingress_bytes=316334080 link_bytes=316334080 token_bytes=494573656 weight_movement_bytes=69206016 total_nvlink_bytes=563779672 weight_to_token_ratio=0.139931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190339676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85800 payload_rows_remote=73412 dup_rows=12504 dup_rows_remote=10360 dedup_ratio=0.127197 router_tokens_per_expert=[589, 297, 5486, 2102, 3244, 0, 6171, 6160, 0, 182, 0, 747, 0, 682, 0, 317, 280, 104, 2709, 19, 984, 51, 320, 166, 55, 175, 807, 0, 1, 157, 0, 20, 5378, 546, 0, 91, 218, 390, 643, 0, 197, 0, 4, 0, 0, 0, 179, 79, 2286, 0, 143, 69, 12, 1382, 81, 0, 3764, 494, 7, 462, 778, 114, 10, 0] payload_rows_per_dst=[5702, 12036, 7662, 13618, 12388, 12900, 11386, 10108] dup_rows_per_dst=[274, 2792, 1604, 1132, 2144, 360, 3048, 1150] combine_rows_per_src=[5702, 12036, 7662, 13618, 12388, 12900, 11386, 10108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85276 payload_rows_remote=74886 dup_rows=13028 dup_rows_remote=9844 dedup_ratio=0.132528 router_tokens_per_expert=[584, 73, 5457, 1250, 3724, 0, 6429, 5468, 0, 146, 0, 707, 0, 478, 0, 382, 240, 0, 3412, 1, 909, 51, 300, 164, 51, 73, 505, 1, 1, 125, 0, 140, 5640, 594, 0, 74, 193, 795, 713, 0, 363, 0, 0, 0, 0, 0, 0, 12, 2293, 0, 2, 90, 16, 1765, 376, 0, 2852, 489, 0, 769, 1425, 20, 0, 0] payload_rows_per_dst=[9710, 11952, 10390, 12192, 10144, 13592, 7806, 9490] dup_rows_per_dst=[1552, 2388, 3184, 536, 2454, 16, 1278, 1620] combine_rows_per_src=[9710, 11952, 10390, 12192, 10144, 13592, 7806, 9490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292347904 dispatch_meta_bytes=329984 combine_payload_bytes=292347904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292677888 ingress_bytes=326950912 link_bytes=326950912 token_bytes=585025792 weight_movement_bytes=34603008 total_nvlink_bytes=619628800 weight_to_token_ratio=0.059148 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230289960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84562 payload_rows_remote=70888 dup_rows=13742 dup_rows_remote=13742 dedup_ratio=0.139791 router_tokens_per_expert=[579, 81, 5404, 1583, 3295, 0, 6505, 5538, 0, 155, 0, 773, 0, 442, 0, 572, 251, 0, 2811, 0, 923, 58, 323, 107, 50, 77, 494, 0, 0, 128, 0, 185, 5822, 650, 0, 78, 196, 843, 733, 0, 332, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 62, 9, 1398, 525, 0, 2841, 441, 8, 1103, 1546, 50, 0, 0] payload_rows_per_dst=[4478, 11988, 7470, 12338, 13494, 13674, 11190, 9930] dup_rows_per_dst=[8, 2704, 1476, 606, 3150, 0, 3750, 2048] combine_rows_per_src=[4478, 11988, 7470, 12338, 13494, 13674, 11190, 9930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85280 payload_rows_remote=74608 dup_rows=13024 dup_rows_remote=10326 dedup_ratio=0.132487 router_tokens_per_expert=[597, 310, 5536, 2002, 2687, 1, 6319, 5842, 0, 214, 0, 769, 1, 549, 1, 302, 252, 35, 2288, 1, 995, 49, 409, 119, 36, 117, 683, 0, 1, 144, 1, 154, 5658, 680, 0, 107, 215, 625, 710, 0, 299, 1, 0, 0, 2, 1, 237, 181, 2258, 0, 123, 73, 58, 1297, 188, 1, 3419, 515, 28, 679, 1294, 18, 71, 0] payload_rows_per_dst=[5584, 12204, 6806, 13048, 13164, 13622, 10672, 10180] dup_rows_per_dst=[236, 2540, 1490, 908, 2826, 458, 2698, 1868] combine_rows_per_src=[5584, 12204, 6806, 13048, 13164, 13622, 10672, 10180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84248 payload_rows_remote=72366 dup_rows=14056 dup_rows_remote=13536 dedup_ratio=0.142985 router_tokens_per_expert=[594, 64, 5244, 1519, 3941, 0, 6499, 5303, 0, 128, 0, 740, 0, 600, 0, 673, 243, 0, 2542, 0, 910, 52, 90, 132, 50, 64, 490, 0, 0, 115, 0, 179, 5921, 631, 0, 65, 199, 980, 734, 0, 153, 0, 0, 0, 0, 0, 0, 11, 1988, 0, 0, 71, 3, 1250, 578, 0, 3254, 455, 1, 1172, 1514, 0, 0, 0] payload_rows_per_dst=[4484, 11846, 6858, 11882, 13602, 13326, 11784, 10466] dup_rows_per_dst=[80, 2924, 1080, 520, 3458, 0, 3668, 2326] combine_rows_per_src=[4484, 11846, 6858, 11882, 13602, 13326, 11784, 10466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72018 payload_rows_remote=65888 dup_rows=26286 dup_rows_remote=26286 dedup_ratio=0.267395 router_tokens_per_expert=[0, 7558, 3082, 0, 7, 59, 305, 4607, 0, 0, 0, 0, 0, 614, 2320, 131, 0, 263, 80, 262, 93, 0, 0, 994, 213, 258, 0, 106, 209, 59, 153, 0, 75, 7888, 72, 0, 1494, 0, 3019, 0, 1585, 0, 0, 0, 2475, 48, 0, 0, 74, 0, 72, 398, 0, 355, 71, 0, 0, 408, 6952, 155, 72, 820, 0, 1746] payload_rows_per_dst=[15724, 6130, 15564, 1274, 16126, 9300, 1796, 6104] dup_rows_per_dst=[15512, 0, 1724, 722, 6532, 1354, 144, 298] combine_rows_per_src=[15724, 6130, 15564, 1274, 16126, 9300, 1796, 6104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=300695552 dispatch_meta_bytes=335088 combine_payload_bytes=300695552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301030640 ingress_bytes=404504576 link_bytes=404504576 token_bytes=601726192 weight_movement_bytes=103809024 total_nvlink_bytes=705535216 weight_to_token_ratio=0.172519 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306733056 dispatch_meta_bytes=338920 combine_payload_bytes=306733056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307071976 ingress_bytes=341336064 link_bytes=341336064 token_bytes=613805032 weight_movement_bytes=34603008 total_nvlink_bytes=648408040 weight_to_token_ratio=0.056375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284420129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=290357248 dispatch_meta_bytes=338520 combine_payload_bytes=290357248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290695768 ingress_bytes=324960256 link_bytes=324960256 token_bytes=581053016 weight_movement_bytes=34603008 total_nvlink_bytes=615656024 weight_to_token_ratio=0.059552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222799282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305594368 dispatch_meta_bytes=339736 combine_payload_bytes=305594368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305934104 ingress_bytes=340197376 link_bytes=340197376 token_bytes=611528472 weight_movement_bytes=34603008 total_nvlink_bytes=646131480 weight_to_token_ratio=0.056584 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280135338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=58538 payload_rows_remote=42860 dup_rows=39766 dup_rows_remote=12204 dedup_ratio=0.404521 router_tokens_per_expert=[568, 144, 5415, 1593, 3039, 0, 6327, 5960, 0, 153, 0, 684, 0, 577, 0, 425, 249, 0, 2882, 0, 995, 53, 297, 141, 59, 77, 655, 0, 0, 136, 0, 180, 5769, 675, 0, 77, 220, 713, 768, 0, 382, 0, 0, 0, 0, 0, 0, 1, 2312, 0, 0, 72, 3, 1244, 468, 0, 3204, 507, 5, 752, 1288, 83, 0, 0] payload_rows_per_dst=[15678, 3318, 14370, 2558, 2848, 2928, 6960, 9878] dup_rows_per_dst=[27562, 360, 6402, 346, 2058, 0, 1238, 1800] combine_rows_per_src=[15678, 3318, 14370, 2558, 2848, 2928, 6960, 9878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=296411136 dispatch_meta_bytes=343608 combine_payload_bytes=296411136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296754744 ingress_bytes=331014144 link_bytes=331014144 token_bytes=593165880 weight_movement_bytes=34603008 total_nvlink_bytes=627768888 weight_to_token_ratio=0.058336 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245579575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=368696 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270245944 ingress_bytes=304480256 link_bytes=304480256 token_bytes=540123192 weight_movement_bytes=34603008 total_nvlink_bytes=574726200 weight_to_token_ratio=0.064065 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145734691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=63304 dup_rows=19818 dup_rows_remote=9292 dedup_ratio=0.201599 router_tokens_per_expert=[0, 7303, 2470, 0, 13, 57, 370, 5103, 0, 0, 0, 0, 0, 596, 2101, 157, 189, 211, 100, 253, 88, 0, 1, 1030, 194, 310, 0, 147, 238, 57, 184, 0, 34, 7721, 85, 0, 404, 0, 3427, 0, 1476, 0, 0, 0, 2154, 7, 0, 0, 85, 0, 85, 562, 0, 524, 83, 0, 0, 420, 6774, 92, 84, 1519, 0, 2444] payload_rows_per_dst=[880, 14176, 3188, 12382, 5786, 11160, 15732, 15182] dup_rows_per_dst=[0, 1738, 556, 1442, 200, 2968, 2388, 10526] combine_rows_per_src=[880, 14176, 3188, 12382, 5786, 11160, 15732, 15182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=175554560 dispatch_meta_bytes=220256 combine_payload_bytes=175554560 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=175774816 ingress_bytes=348569600 link_bytes=348569600 token_bytes=351329376 weight_movement_bytes=173015040 total_nvlink_bytes=524344416 weight_to_token_ratio=0.492458 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311639343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86736 payload_rows_remote=85630 dup_rows=11568 dup_rows_remote=11568 dedup_ratio=0.117676 router_tokens_per_expert=[0, 7242, 2839, 0, 5, 66, 323, 5169, 0, 0, 0, 0, 0, 603, 2132, 178, 202, 236, 133, 268, 93, 0, 11, 961, 247, 282, 0, 135, 249, 67, 225, 0, 19, 7698, 91, 0, 443, 0, 3807, 0, 1415, 0, 0, 0, 1937, 2, 0, 0, 91, 0, 95, 578, 1, 483, 90, 0, 0, 389, 6694, 98, 91, 1216, 0, 2248] payload_rows_per_dst=[6466, 14124, 15236, 15168, 1106, 11358, 15728, 7550] dup_rows_per_dst=[0, 2040, 1960, 1726, 0, 2964, 2344, 534] combine_rows_per_src=[6466, 14124, 15236, 15168, 1106, 11358, 15728, 7550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82708 payload_rows_remote=67142 dup_rows=15596 dup_rows_remote=14160 dedup_ratio=0.158651 router_tokens_per_expert=[0, 7582, 3054, 0, 3, 51, 307, 3994, 0, 0, 0, 0, 0, 630, 2755, 127, 0, 202, 62, 295, 35, 0, 0, 920, 213, 244, 0, 152, 82, 51, 157, 0, 81, 7878, 74, 0, 2174, 0, 3197, 0, 1440, 0, 0, 0, 2271, 157, 0, 0, 74, 0, 76, 340, 0, 357, 69, 0, 0, 404, 6987, 122, 73, 774, 0, 1688] payload_rows_per_dst=[12210, 7964, 15566, 11682, 14062, 11368, 4016, 5840] dup_rows_per_dst=[5710, 250, 1436, 988, 3780, 2762, 388, 282] combine_rows_per_src=[12210, 7964, 15566, 11682, 14062, 11368, 4016, 5840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=259293184 dispatch_meta_bytes=290384 combine_payload_bytes=259293184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259583568 ingress_bytes=328499200 link_bytes=328499200 token_bytes=518876752 weight_movement_bytes=69206016 total_nvlink_bytes=588082768 weight_to_token_ratio=0.133377 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236116043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84804 payload_rows_remote=74652 dup_rows=13500 dup_rows_remote=11238 dedup_ratio=0.137329 router_tokens_per_expert=[0, 7497, 3465, 0, 1, 50, 371, 4119, 0, 0, 0, 0, 0, 678, 2483, 128, 0, 255, 100, 252, 64, 0, 0, 1010, 274, 251, 0, 119, 48, 50, 189, 0, 122, 7908, 78, 0, 2576, 0, 2340, 0, 1339, 0, 0, 0, 2327, 201, 0, 0, 78, 0, 82, 432, 0, 297, 74, 0, 0, 363, 7017, 271, 77, 691, 0, 1475] payload_rows_per_dst=[2292, 12940, 4526, 15650, 9134, 10152, 16072, 14038] dup_rows_per_dst=[2, 1876, 312, 1206, 1898, 2262, 1670, 4274] combine_rows_per_src=[2292, 12940, 4526, 15650, 9134, 10152, 16072, 14038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81484 payload_rows_remote=65610 dup_rows=16820 dup_rows_remote=14302 dedup_ratio=0.171102 router_tokens_per_expert=[0, 7402, 3389, 0, 19, 37, 363, 4200, 0, 1, 0, 0, 0, 660, 2666, 169, 21, 252, 152, 245, 49, 1, 7, 1013, 351, 264, 3, 180, 243, 37, 259, 0, 51, 7786, 109, 1, 1243, 0, 2347, 0, 978, 0, 1, 1, 2201, 210, 0, 1, 107, 1, 108, 606, 1, 479, 107, 1, 0, 405, 6817, 62, 110, 1014, 1, 2421] payload_rows_per_dst=[838, 13036, 3334, 15522, 8236, 9646, 15874, 14998] dup_rows_per_dst=[0, 2356, 146, 1956, 1350, 1832, 2518, 6662] combine_rows_per_src=[838, 13036, 3334, 15522, 8236, 9646, 15874, 14998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89110 payload_rows_remote=73396 dup_rows=9194 dup_rows_remote=8374 dedup_ratio=0.093526 router_tokens_per_expert=[0, 7554, 2962, 0, 8, 50, 382, 3908, 0, 0, 0, 0, 0, 653, 2999, 119, 0, 267, 42, 291, 32, 0, 0, 998, 176, 246, 0, 116, 8, 50, 117, 0, 138, 7911, 65, 0, 2719, 0, 2467, 0, 1570, 0, 0, 0, 2364, 231, 0, 0, 65, 0, 66, 367, 0, 235, 57, 0, 0, 385, 7122, 239, 64, 747, 0, 1362] payload_rows_per_dst=[6786, 13042, 15728, 15714, 5838, 10622, 16052, 5328] dup_rows_per_dst=[18, 2316, 1776, 820, 6, 2642, 1350, 266] combine_rows_per_src=[6786, 13042, 15728, 15714, 5838, 10622, 16052, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=350740480 dispatch_meta_bytes=388792 combine_payload_bytes=350740480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=351129272 ingress_bytes=454549504 link_bytes=454549504 token_bytes=701869752 weight_movement_bytes=103809024 total_nvlink_bytes=805678776 weight_to_token_ratio=0.147904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.710433189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86586 payload_rows_remote=72778 dup_rows=11718 dup_rows_remote=6680 dedup_ratio=0.119202 router_tokens_per_expert=[0, 0, 0, 369, 0, 0, 0, 0, 412, 319, 1554, 0, 663, 328, 0, 71, 53, 0, 1, 0, 0, 95, 0, 0, 211, 6076, 0, 298, 0, 1596, 0, 0, 463, 1475, 0, 0, 0, 3486, 0, 742, 2410, 618, 4483, 0, 608, 811, 1446, 0, 0, 80, 0, 1251, 0, 0, 53, 0, 404, 0, 6811, 6521, 4958, 342, 0, 144] payload_rows_per_dst=[13304, 13808, 13916, 12640, 10400, 10052, 10686, 1780] dup_rows_per_dst=[476, 5038, 4, 1486, 1932, 1734, 1048, 0] combine_rows_per_src=[13304, 13808, 13916, 12640, 10400, 10052, 10686, 1780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=325208 combine_payload_bytes=275013632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275338840 ingress_bytes=309616640 link_bytes=309616640 token_bytes=550352472 weight_movement_bytes=34603008 total_nvlink_bytes=584955480 weight_to_token_ratio=0.062874 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165062490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70680 payload_rows_remote=54940 dup_rows=27624 dup_rows_remote=11498 dedup_ratio=0.281006 router_tokens_per_expert=[0, 7504, 3325, 0, 7, 59, 380, 4658, 0, 0, 0, 0, 0, 676, 2345, 137, 0, 250, 86, 246, 82, 0, 0, 1029, 218, 253, 0, 119, 214, 59, 163, 0, 65, 7810, 77, 0, 1763, 0, 2410, 0, 1494, 0, 0, 0, 2432, 31, 0, 0, 77, 0, 82, 425, 0, 459, 73, 0, 0, 430, 6933, 146, 77, 888, 0, 1670] payload_rows_per_dst=[15740, 6316, 15546, 1284, 16030, 7560, 2078, 6126] dup_rows_per_dst=[16126, 0, 1706, 768, 8220, 354, 154, 296] combine_rows_per_src=[15740, 6316, 15546, 1284, 16030, 7560, 2078, 6126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=305774592 dispatch_meta_bytes=343560 combine_payload_bytes=305774592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306118152 ingress_bytes=340377600 link_bytes=340377600 token_bytes=611892744 weight_movement_bytes=34603008 total_nvlink_bytes=646495752 weight_to_token_ratio=0.056551 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280813506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268738560 dispatch_meta_bytes=319648 combine_payload_bytes=268738560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269058208 ingress_bytes=303341568 link_bytes=303341568 token_bytes=537796768 weight_movement_bytes=34603008 total_nvlink_bytes=572399776 weight_to_token_ratio=0.064342 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141449899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=300630016 dispatch_meta_bytes=327080 combine_payload_bytes=300630016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300957096 ingress_bytes=335233024 link_bytes=335233024 token_bytes=601587112 weight_movement_bytes=34603008 total_nvlink_bytes=636190120 weight_to_token_ratio=0.057520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261454881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298098688 dispatch_meta_bytes=317832 combine_payload_bytes=298098688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298416520 ingress_bytes=332701696 link_bytes=332701696 token_bytes=596515208 weight_movement_bytes=34603008 total_nvlink_bytes=631118216 weight_to_token_ratio=0.058009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251929697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64394 payload_rows_remote=48352 dup_rows=33910 dup_rows_remote=10164 dedup_ratio=0.344950 router_tokens_per_expert=[0, 0, 0, 367, 0, 0, 0, 0, 386, 143, 1300, 0, 1124, 324, 0, 84, 50, 0, 3, 0, 0, 88, 0, 0, 20, 4391, 0, 300, 155, 1562, 0, 0, 955, 883, 0, 0, 0, 3731, 0, 744, 1199, 1351, 5478, 0, 620, 349, 1060, 0, 0, 101, 0, 2381, 0, 0, 50, 0, 262, 0, 6805, 6385, 5308, 1024, 0, 169] payload_rows_per_dst=[2592, 4780, 282, 10846, 7560, 8140, 14152, 16042] dup_rows_per_dst=[68, 1942, 0, 2010, 954, 924, 4266, 23746] combine_rows_per_src=[2592, 4780, 282, 10846, 7560, 8140, 14152, 16042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=225034240 dispatch_meta_bytes=265752 combine_payload_bytes=225034240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=225299992 ingress_bytes=363446272 link_bytes=363446272 token_bytes=450334232 weight_movement_bytes=138412032 total_nvlink_bytes=588746264 weight_to_token_ratio=0.307354 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367619062e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86820 payload_rows_remote=76392 dup_rows=11484 dup_rows_remote=9384 dedup_ratio=0.116821 router_tokens_per_expert=[0, 0, 0, 319, 0, 0, 0, 0, 412, 187, 1166, 0, 1040, 342, 0, 84, 54, 0, 8, 0, 0, 94, 0, 0, 40, 4431, 0, 295, 122, 1408, 0, 0, 776, 1225, 0, 0, 0, 3598, 0, 665, 1211, 1494, 5281, 0, 544, 448, 1183, 0, 0, 135, 0, 2361, 0, 0, 53, 0, 460, 0, 6801, 6357, 5442, 934, 0, 182] payload_rows_per_dst=[12934, 4454, 13894, 10950, 10428, 8418, 13054, 12688] dup_rows_per_dst=[418, 2008, 20, 1642, 2100, 1342, 2606, 1348] combine_rows_per_src=[12934, 4454, 13894, 10950, 10428, 8418, 13054, 12688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87704 payload_rows_remote=73228 dup_rows=10600 dup_rows_remote=10436 dedup_ratio=0.107829 router_tokens_per_expert=[0, 0, 0, 350, 0, 0, 0, 0, 380, 353, 1418, 0, 715, 299, 0, 59, 88, 0, 0, 0, 0, 86, 1, 0, 165, 5619, 0, 299, 0, 1833, 0, 0, 285, 1757, 0, 0, 0, 3815, 0, 608, 2887, 396, 4133, 0, 592, 1200, 1791, 0, 0, 82, 0, 608, 0, 0, 88, 0, 507, 0, 7145, 6538, 4721, 187, 0, 147] payload_rows_per_dst=[13352, 13222, 14476, 7236, 10828, 11404, 9362, 7824] dup_rows_per_dst=[424, 4464, 164, 472, 2102, 2328, 460, 186] combine_rows_per_src=[13352, 13222, 14476, 7236, 10828, 11404, 9362, 7824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=198049792 dispatch_meta_bytes=234064 combine_payload_bytes=198049792 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=198283856 ingress_bytes=336461824 link_bytes=336461824 token_bytes=396333648 weight_movement_bytes=138412032 total_nvlink_bytes=534745680 weight_to_token_ratio=0.349231 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266078756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87728 payload_rows_remote=74828 dup_rows=10576 dup_rows_remote=7668 dedup_ratio=0.107585 router_tokens_per_expert=[0, 0, 0, 378, 0, 0, 0, 0, 367, 314, 1098, 0, 756, 310, 0, 77, 59, 0, 1, 0, 0, 66, 0, 0, 200, 6402, 0, 295, 0, 1604, 0, 0, 299, 1800, 0, 0, 0, 3616, 0, 734, 3488, 457, 3370, 0, 562, 1264, 2133, 0, 0, 112, 0, 658, 0, 0, 53, 0, 489, 0, 7039, 6556, 4264, 176, 0, 155] payload_rows_per_dst=[13406, 4416, 14254, 13820, 10922, 12900, 8040, 9970] dup_rows_per_dst=[462, 1428, 76, 3182, 1976, 2908, 346, 198] combine_rows_per_src=[13406, 4416, 14254, 13820, 10922, 12900, 8040, 9970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86050 payload_rows_remote=73780 dup_rows=12254 dup_rows_remote=10262 dedup_ratio=0.124654 router_tokens_per_expert=[0, 0, 0, 348, 0, 1, 0, 1, 331, 137, 1317, 1, 847, 352, 1, 107, 66, 1, 4, 0, 1, 49, 0, 0, 65, 5685, 0, 277, 130, 1481, 0, 2, 558, 1766, 0, 0, 0, 3556, 1, 769, 2303, 1063, 4708, 1, 596, 594, 1248, 0, 0, 153, 1, 1702, 0, 0, 66, 1, 537, 1, 6526, 6310, 4778, 496, 0, 214] payload_rows_per_dst=[12870, 4384, 11960, 12584, 11022, 9016, 12270, 11944] dup_rows_per_dst=[450, 1802, 10, 2696, 2278, 1594, 1992, 1432] combine_rows_per_src=[12870, 4384, 11960, 12584, 11022, 9016, 12270, 11944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=312901632 dispatch_meta_bytes=343104 combine_payload_bytes=312901632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313244736 ingress_bytes=382107648 link_bytes=382107648 token_bytes=626146368 weight_movement_bytes=69206016 total_nvlink_bytes=695352384 weight_to_token_ratio=0.110527 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437840317e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87922 payload_rows_remote=83532 dup_rows=10382 dup_rows_remote=10362 dedup_ratio=0.105611 router_tokens_per_expert=[1, 0, 0, 342, 0, 0, 0, 0, 373, 475, 836, 0, 804, 237, 0, 62, 71, 0, 0, 0, 0, 79, 0, 0, 120, 6446, 0, 273, 0, 1752, 0, 0, 223, 1807, 0, 0, 0, 3842, 0, 704, 3492, 259, 3773, 0, 505, 1402, 1972, 0, 0, 52, 0, 576, 0, 0, 30, 0, 443, 0, 7356, 6606, 3965, 146, 0, 128] payload_rows_per_dst=[13470, 13908, 14870, 4390, 11080, 12424, 8554, 9226] dup_rows_per_dst=[428, 4438, 142, 20, 2072, 2836, 308, 138] combine_rows_per_src=[13470, 13908, 14870, 4390, 11080, 12424, 8554, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85300 payload_rows_remote=72136 dup_rows=13004 dup_rows_remote=12922 dedup_ratio=0.132284 router_tokens_per_expert=[0, 9, 313, 353, 24, 6980, 0, 375, 0, 0, 42, 0, 72, 0, 0, 0, 2962, 0, 0, 0, 0, 3230, 0, 2979, 0, 13, 49, 2465, 0, 0, 0, 59, 6340, 4, 72, 3434, 0, 0, 167, 588, 226, 0, 0, 2434, 95, 0, 0, 0, 598, 0, 3576, 72, 6509, 41, 1171, 0, 2, 293, 1711, 643, 591, 521, 0, 139] payload_rows_per_dst=[14814, 13164, 10340, 13720, 7992, 9252, 9556, 6462] dup_rows_per_dst=[1294, 82, 2044, 4132, 538, 2216, 1360, 1338] combine_rows_per_src=[14814, 13164, 10340, 13720, 7992, 9252, 9556, 6462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=299941888 dispatch_meta_bytes=334656 combine_payload_bytes=299941888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300276544 ingress_bytes=334544896 link_bytes=334544896 token_bytes=600218432 weight_movement_bytes=34603008 total_nvlink_bytes=634821440 weight_to_token_ratio=0.057651 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258865511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=306495488 dispatch_meta_bytes=329984 combine_payload_bytes=306495488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306825472 ingress_bytes=410304512 link_bytes=410304512 token_bytes=613320960 weight_movement_bytes=103809024 total_nvlink_bytes=717129984 weight_to_token_ratio=0.169257 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543942846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86816 payload_rows_remote=73644 dup_rows=11488 dup_rows_remote=11010 dedup_ratio=0.116862 router_tokens_per_expert=[0, 0, 0, 347, 0, 0, 0, 0, 386, 252, 1292, 0, 845, 315, 0, 77, 66, 0, 0, 0, 0, 82, 0, 0, 197, 5939, 0, 268, 0, 1785, 0, 0, 402, 1659, 0, 0, 0, 3491, 0, 773, 2553, 616, 4062, 0, 633, 898, 1442, 0, 0, 86, 0, 1370, 0, 0, 62, 0, 445, 0, 6970, 6478, 4915, 292, 0, 154] payload_rows_per_dst=[13172, 13714, 14182, 12732, 10644, 10334, 10256, 1782] dup_rows_per_dst=[478, 4498, 54, 1598, 2006, 1950, 904, 0] combine_rows_per_src=[13172, 13714, 14182, 12732, 10644, 10334, 10256, 1782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302202880 dispatch_meta_bytes=336168 combine_payload_bytes=302202880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302539048 ingress_bytes=371408896 link_bytes=371408896 token_bytes=604741928 weight_movement_bytes=69206016 total_nvlink_bytes=673947944 weight_to_token_ratio=0.114439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=35328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397581775e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=342147072 dispatch_meta_bytes=375576 combine_payload_bytes=342147072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342522648 ingress_bytes=411353088 link_bytes=411353088 token_bytes=684669720 weight_movement_bytes=69206016 total_nvlink_bytes=753875736 weight_to_token_ratio=0.101079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.547888554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=295469056 dispatch_meta_bytes=340232 combine_payload_bytes=295469056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295809288 ingress_bytes=330072064 link_bytes=330072064 token_bytes=591278344 weight_movement_bytes=34603008 total_nvlink_bytes=625881352 weight_to_token_ratio=0.058522 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242034604e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73598 payload_rows_remote=57420 dup_rows=24706 dup_rows_remote=15162 dedup_ratio=0.251322 router_tokens_per_expert=[0, 46, 320, 216, 68, 7365, 0, 415, 0, 0, 41, 0, 85, 0, 0, 0, 2677, 0, 0, 0, 0, 2728, 0, 2670, 0, 93, 11, 1145, 0, 0, 0, 57, 6374, 0, 85, 3431, 0, 0, 172, 1079, 306, 0, 0, 2053, 88, 0, 0, 0, 502, 0, 4036, 83, 5790, 57, 1663, 0, 41, 612, 2345, 690, 979, 738, 0, 91] payload_rows_per_dst=[1874, 7822, 9182, 2612, 13562, 8460, 13908, 16178] dup_rows_per_dst=[256, 74, 1430, 0, 1858, 1972, 9572, 9544] combine_rows_per_src=[1874, 7822, 9182, 2612, 13562, 8460, 13908, 16178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=301645824 dispatch_meta_bytes=338616 combine_payload_bytes=301645824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301984440 ingress_bytes=336248832 link_bytes=336248832 token_bytes=603630264 weight_movement_bytes=34603008 total_nvlink_bytes=638233272 weight_to_token_ratio=0.057325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85676 payload_rows_remote=77228 dup_rows=12628 dup_rows_remote=11156 dedup_ratio=0.128459 router_tokens_per_expert=[0, 63, 336, 312, 77, 7284, 0, 408, 0, 0, 23, 0, 92, 0, 0, 0, 2468, 0, 0, 0, 0, 3050, 0, 2804, 0, 126, 17, 1198, 0, 0, 0, 66, 6074, 0, 91, 3004, 0, 0, 185, 910, 339, 0, 0, 2179, 93, 0, 0, 0, 413, 0, 3803, 91, 5832, 65, 1588, 0, 38, 738, 2609, 756, 892, 1029, 0, 99] payload_rows_per_dst=[15504, 11852, 9434, 11660, 8448, 8824, 10252, 9702] dup_rows_per_dst=[1456, 42, 1602, 1762, 1472, 2006, 1668, 2620] combine_rows_per_src=[15504, 11852, 9434, 11660, 8448, 8824, 10252, 9702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84366 payload_rows_remote=73438 dup_rows=13938 dup_rows_remote=11530 dedup_ratio=0.141785 router_tokens_per_expert=[0, 0, 333, 330, 3, 6793, 0, 371, 0, 0, 16, 0, 74, 0, 0, 0, 3194, 0, 0, 0, 0, 3474, 0, 3016, 0, 0, 33, 3115, 0, 0, 0, 51, 6187, 0, 74, 3621, 0, 0, 159, 587, 210, 0, 0, 2825, 86, 0, 0, 0, 713, 1, 2846, 73, 6630, 11, 869, 0, 1, 219, 1323, 654, 554, 550, 0, 156] payload_rows_per_dst=[14392, 13408, 10928, 13856, 8266, 9772, 8078, 5666] dup_rows_per_dst=[1268, 32, 2408, 4916, 616, 2502, 948, 1248] combine_rows_per_src=[14392, 13408, 10928, 13856, 8266, 9772, 8078, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=235192320 dispatch_meta_bytes=290328 combine_payload_bytes=235192320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235482648 ingress_bytes=269795328 link_bytes=269795328 token_bytes=470674968 weight_movement_bytes=34603008 total_nvlink_bytes=505277976 weight_to_token_ratio=0.073518 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.015218099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82068 payload_rows_remote=71276 dup_rows=16236 dup_rows_remote=13198 dedup_ratio=0.165161 router_tokens_per_expert=[0, 8, 306, 317, 5, 6807, 0, 444, 0, 0, 17, 0, 78, 0, 0, 0, 2704, 0, 0, 0, 0, 3370, 0, 3601, 0, 0, 45, 3476, 0, 0, 0, 50, 6219, 1, 78, 3228, 0, 0, 143, 514, 193, 0, 0, 3056, 65, 0, 0, 0, 710, 0, 3143, 77, 6607, 4, 506, 0, 0, 234, 1371, 655, 525, 405, 0, 190] payload_rows_per_dst=[11778, 13370, 10156, 7000, 14226, 10792, 8198, 6548] dup_rows_per_dst=[1184, 34, 1992, 142, 7376, 3038, 682, 1788] combine_rows_per_src=[11778, 13370, 10156, 7000, 14226, 10792, 8198, 6548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78056 payload_rows_remote=65928 dup_rows=20248 dup_rows_remote=17616 dedup_ratio=0.205973 router_tokens_per_expert=[0, 37, 319, 290, 55, 6991, 0, 470, 0, 0, 11, 0, 107, 0, 0, 1, 2166, 0, 0, 0, 1, 2656, 0, 3575, 0, 144, 76, 1619, 1, 0, 0, 37, 6258, 4, 107, 2543, 0, 1, 156, 695, 210, 1, 0, 2884, 49, 0, 1, 0, 596, 2, 4665, 107, 5601, 37, 1556, 1, 18, 480, 2234, 761, 661, 848, 1, 119] payload_rows_per_dst=[1860, 11420, 8548, 3742, 13720, 10758, 12128, 15880] dup_rows_per_dst=[482, 20, 1098, 12, 4976, 2682, 2632, 8346] combine_rows_per_src=[1860, 11420, 8548, 3742, 13720, 10758, 12128, 15880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=316325888 dispatch_meta_bytes=353536 combine_payload_bytes=316325888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=316679424 ingress_bytes=454737920 link_bytes=454737920 token_bytes=633005312 weight_movement_bytes=138412032 total_nvlink_bytes=771417344 weight_to_token_ratio=0.218659 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.711142184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82694 payload_rows_remote=68250 dup_rows=15610 dup_rows_remote=8398 dedup_ratio=0.158793 router_tokens_per_expert=[0, 0, 305, 323, 4, 6782, 0, 399, 0, 0, 27, 0, 65, 0, 0, 0, 3027, 0, 0, 0, 0, 3438, 0, 2944, 0, 0, 84, 4346, 0, 0, 0, 50, 6348, 0, 65, 2832, 0, 0, 143, 534, 193, 0, 0, 3312, 77, 0, 0, 0, 712, 0, 3069, 63, 6708, 0, 554, 0, 0, 155, 1026, 622, 477, 351, 0, 117] payload_rows_per_dst=[14338, 13548, 10688, 14444, 6630, 10388, 8112, 4546] dup_rows_per_dst=[1288, 52, 2242, 7212, 518, 2664, 684, 950] combine_rows_per_src=[14338, 13548, 10688, 14444, 6630, 10388, 8112, 4546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89808 payload_rows_remote=81008 dup_rows=8496 dup_rows_remote=6516 dedup_ratio=0.086426 router_tokens_per_expert=[70, 0, 58, 0, 155, 0, 0, 0, 0, 2933, 0, 127, 72, 54, 463, 1741, 0, 7313, 0, 73, 0, 0, 53, 0, 7530, 0, 135, 258, 0, 1501, 72, 7531, 7104, 752, 0, 0, 0, 0, 0, 0, 72, 12, 1537, 0, 0, 1832, 258, 0, 390, 0, 0, 4150, 0, 0, 0, 0, 0, 2865, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[15220, 8800, 14628, 15338, 14208, 6870, 8932, 5812] dup_rows_per_dst=[406, 1980, 250, 3656, 1504, 552, 148, 0] combine_rows_per_src=[15220, 8800, 14628, 15338, 14208, 6870, 8932, 5812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=300802048 dispatch_meta_bytes=339872 combine_payload_bytes=300802048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301141920 ingress_bytes=404611072 link_bytes=404611072 token_bytes=601943968 weight_movement_bytes=103809024 total_nvlink_bytes=705752992 weight_to_token_ratio=0.172456 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291946496 dispatch_meta_bytes=337896 combine_payload_bytes=291946496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=292284392 ingress_bytes=361152512 link_bytes=361152512 token_bytes=584230888 weight_movement_bytes=69206016 total_nvlink_bytes=653436904 weight_to_token_ratio=0.118457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358987827e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83908 payload_rows_remote=68878 dup_rows=14396 dup_rows_remote=12910 dedup_ratio=0.146444 router_tokens_per_expert=[0, 52, 313, 328, 14, 7110, 0, 441, 0, 0, 50, 0, 77, 0, 0, 0, 2685, 0, 0, 0, 0, 3128, 0, 3574, 0, 0, 36, 2498, 0, 0, 0, 59, 6247, 0, 77, 3191, 0, 0, 159, 626, 218, 0, 0, 2615, 82, 0, 0, 0, 613, 0, 3584, 77, 6565, 49, 940, 0, 0, 346, 1627, 645, 519, 444, 0, 163] payload_rows_per_dst=[15030, 13284, 11562, 10712, 7574, 10316, 9364, 6066] dup_rows_per_dst=[1486, 100, 4650, 2382, 532, 2662, 1162, 1422] combine_rows_per_src=[15030, 13284, 11562, 10712, 7574, 10316, 9364, 6066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270041088 dispatch_meta_bytes=334176 combine_payload_bytes=270041088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270375264 ingress_bytes=339247104 link_bytes=339247104 token_bytes=540416352 weight_movement_bytes=69206016 total_nvlink_bytes=609622368 weight_to_token_ratio=0.128061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276559541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331808768 dispatch_meta_bytes=350096 combine_payload_bytes=331808768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332158864 ingress_bytes=366411776 link_bytes=366411776 token_bytes=663967632 weight_movement_bytes=34603008 total_nvlink_bytes=698570640 weight_to_token_ratio=0.052116 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378778015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=306592 combine_payload_bytes=279552000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279858592 ingress_bytes=314155008 link_bytes=314155008 token_bytes=559410592 weight_movement_bytes=34603008 total_nvlink_bytes=594013600 weight_to_token_ratio=0.061856 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182140004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73846 payload_rows_remote=58368 dup_rows=24458 dup_rows_remote=19316 dedup_ratio=0.248800 router_tokens_per_expert=[7, 0, 57, 0, 132, 0, 0, 0, 0, 2897, 0, 183, 84, 157, 470, 1533, 0, 7290, 0, 85, 0, 0, 50, 0, 7481, 0, 99, 241, 0, 1350, 84, 7080, 7380, 1012, 0, 0, 0, 0, 0, 0, 85, 48, 2829, 0, 0, 1807, 241, 0, 388, 0, 0, 2852, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[278, 8560, 5334, 15796, 5908, 16026, 6466, 15478] dup_rows_per_dst=[114, 2088, 66, 7806, 474, 8754, 14, 5142] combine_rows_per_src=[278, 8560, 5334, 15796, 5908, 16026, 6466, 15478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87174 payload_rows_remote=72470 dup_rows=11130 dup_rows_remote=9096 dedup_ratio=0.113220 router_tokens_per_expert=[7, 0, 66, 1, 110, 0, 0, 0, 0, 2934, 0, 225, 91, 214, 455, 1601, 0, 7257, 0, 91, 0, 0, 54, 0, 7436, 0, 116, 252, 0, 1219, 91, 7278, 7351, 1018, 0, 0, 0, 0, 0, 0, 91, 52, 2871, 0, 0, 1617, 252, 0, 382, 0, 0, 3004, 0, 0, 0, 0, 0, 3015, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[15018, 8918, 14514, 3176, 14704, 9020, 10110, 11714] dup_rows_per_dst=[222, 2122, 290, 180, 2034, 746, 2320, 3216] combine_rows_per_src=[15018, 8918, 14514, 3176, 14704, 9020, 10110, 11714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=282124288 dispatch_meta_bytes=327152 combine_payload_bytes=282124288 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=282451440 ingress_bytes=351330304 link_bytes=351330304 token_bytes=564575728 weight_movement_bytes=69206016 total_nvlink_bytes=633781744 weight_to_token_ratio=0.122581 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322027650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86884 payload_rows_remote=72232 dup_rows=11420 dup_rows_remote=11096 dedup_ratio=0.116170 router_tokens_per_expert=[77, 0, 47, 0, 136, 0, 0, 0, 0, 3228, 0, 83, 74, 22, 504, 1628, 0, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 109, 244, 0, 1792, 73, 7485, 6956, 697, 0, 0, 0, 0, 0, 0, 74, 3, 1364, 0, 0, 1617, 244, 0, 351, 0, 0, 4765, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 0, 0, 85] payload_rows_per_dst=[15164, 9106, 14652, 9580, 13912, 6086, 13230, 5154] dup_rows_per_dst=[384, 1972, 324, 2064, 1394, 518, 4764, 0] combine_rows_per_src=[15164, 9106, 14652, 9580, 13912, 6086, 13230, 5154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=239075328 dispatch_meta_bytes=310736 combine_payload_bytes=239075328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239386064 ingress_bytes=273678336 link_bytes=273678336 token_bytes=478461392 weight_movement_bytes=34603008 total_nvlink_bytes=513064400 weight_to_token_ratio=0.072321 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.029829545e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87448 payload_rows_remote=77612 dup_rows=10856 dup_rows_remote=8768 dedup_ratio=0.110433 router_tokens_per_expert=[304, 0, 45, 0, 115, 0, 0, 0, 0, 3003, 0, 112, 78, 19, 423, 1680, 1, 7337, 0, 79, 0, 0, 59, 0, 7461, 0, 107, 251, 0, 1355, 77, 7507, 6893, 739, 0, 0, 0, 0, 0, 0, 78, 0, 1058, 0, 0, 2509, 251, 0, 392, 0, 0, 4577, 0, 0, 0, 0, 0, 2583, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[15284, 8516, 14676, 3426, 10138, 9836, 9882, 15690] dup_rows_per_dst=[566, 2114, 276, 154, 994, 2088, 56, 4608] combine_rows_per_src=[15284, 8516, 14676, 3426, 10138, 9836, 9882, 15690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80452 payload_rows_remote=72416 dup_rows=17852 dup_rows_remote=17812 dedup_ratio=0.181600 router_tokens_per_expert=[1, 0, 37, 2, 80, 0, 0, 0, 1, 2702, 0, 259, 107, 265, 442, 1613, 0, 7243, 1, 123, 0, 0, 68, 1, 7417, 1, 96, 245, 1, 1312, 107, 7451, 6987, 980, 1, 0, 1, 0, 0, 1, 107, 39, 2101, 1, 0, 2203, 245, 0, 401, 0, 0, 3637, 0, 0, 0, 0, 0, 2813, 0, 0, 0, 0, 0, 60] payload_rows_per_dst=[4130, 8578, 14522, 12102, 1964, 15582, 8036, 15538] dup_rows_per_dst=[100, 2200, 350, 2266, 2, 7784, 40, 5110] combine_rows_per_src=[4130, 8578, 14522, 12102, 1964, 15582, 8036, 15538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=326264 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297163384 ingress_bytes=366043136 link_bytes=366043136 token_bytes=594000504 weight_movement_bytes=69206016 total_nvlink_bytes=663206520 weight_to_token_ratio=0.116508 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377390852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79046 payload_rows_remote=65214 dup_rows=19258 dup_rows_remote=15144 dedup_ratio=0.195903 router_tokens_per_expert=[0, 0, 28, 601, 0, 0, 1547, 0, 0, 4504, 3878, 0, 0, 0, 591, 0, 95, 353, 0, 385, 0, 53, 0, 0, 2828, 0, 0, 72, 0, 0, 0, 0, 53, 330, 0, 0, 4855, 141, 0, 72, 0, 0, 4986, 0, 0, 0, 0, 0, 670, 0, 72, 0, 7554, 0, 95, 63, 7114, 0, 0, 0, 7112, 512, 588, 0] payload_rows_per_dst=[15320, 13832, 14424, 5800, 12146, 13650, 1674, 2200] dup_rows_per_dst=[3256, 4114, 1576, 0, 3188, 6998, 126, 0] combine_rows_per_src=[15320, 13832, 14424, 5800, 12146, 13650, 1674, 2200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83566 payload_rows_remote=80062 dup_rows=14738 dup_rows_remote=14610 dedup_ratio=0.149923 router_tokens_per_expert=[346, 0, 31, 0, 119, 0, 0, 0, 0, 3215, 0, 52, 65, 16, 462, 1783, 0, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 123, 244, 0, 1385, 64, 7534, 6898, 830, 0, 0, 0, 0, 0, 0, 65, 0, 1061, 0, 0, 1951, 244, 0, 289, 0, 0, 4730, 0, 0, 0, 0, 0, 2506, 0, 0, 0, 0, 0, 71] payload_rows_per_dst=[15448, 9180, 14782, 3504, 13798, 6070, 15630, 5154] dup_rows_per_dst=[626, 2006, 272, 128, 1658, 572, 9476, 0] combine_rows_per_src=[15448, 9180, 14782, 3504, 13798, 6070, 15630, 5154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=295862272 dispatch_meta_bytes=333312 combine_payload_bytes=295862272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=296195584 ingress_bytes=399671296 link_bytes=399671296 token_bytes=592057856 weight_movement_bytes=103809024 total_nvlink_bytes=695866880 weight_to_token_ratio=0.175336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.503930911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=317898752 dispatch_meta_bytes=345520 combine_payload_bytes=317898752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318244272 ingress_bytes=352501760 link_bytes=352501760 token_bytes=636143024 weight_movement_bytes=34603008 total_nvlink_bytes=670746032 weight_to_token_ratio=0.054395 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326435744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=296615936 dispatch_meta_bytes=360912 combine_payload_bytes=296615936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296976848 ingress_bytes=331218944 link_bytes=331218944 token_bytes=593592784 weight_movement_bytes=34603008 total_nvlink_bytes=628195792 weight_to_token_ratio=0.058294 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246350221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=321432 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267437976 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534554520 weight_movement_bytes=69206016 total_nvlink_bytes=603760536 weight_to_token_ratio=0.129465 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327933952 dispatch_meta_bytes=378688 combine_payload_bytes=327933952 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=328312640 ingress_bytes=466345984 link_bytes=466345984 token_bytes=656246592 weight_movement_bytes=138412032 total_nvlink_bytes=794658624 weight_to_token_ratio=0.210915 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.754822394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83920 payload_rows_remote=68874 dup_rows=14384 dup_rows_remote=14086 dedup_ratio=0.146322 router_tokens_per_expert=[26, 0, 57, 0, 131, 0, 0, 0, 0, 2935, 0, 140, 77, 58, 496, 1621, 1, 7293, 0, 79, 0, 0, 66, 0, 7458, 0, 120, 246, 0, 1639, 77, 7482, 6933, 873, 0, 0, 0, 0, 0, 0, 77, 0, 1462, 0, 0, 2115, 246, 0, 417, 0, 0, 4183, 0, 0, 0, 0, 0, 2778, 0, 0, 0, 0, 0, 66] payload_rows_per_dst=[15046, 14320, 3330, 15410, 13866, 7162, 9098, 5688] dup_rows_per_dst=[298, 7844, 38, 3718, 1746, 638, 102, 0] combine_rows_per_src=[15046, 14320, 3330, 15410, 13866, 7162, 9098, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75016 payload_rows_remote=58984 dup_rows=23288 dup_rows_remote=8524 dedup_ratio=0.236898 router_tokens_per_expert=[0, 0, 95, 654, 0, 0, 1022, 0, 0, 5852, 2593, 0, 0, 0, 627, 0, 88, 330, 0, 470, 2, 48, 0, 0, 3999, 1, 0, 84, 0, 0, 0, 0, 50, 326, 0, 0, 4438, 162, 0, 84, 4, 0, 4735, 0, 0, 0, 0, 0, 723, 0, 84, 0, 7477, 0, 87, 84, 7001, 0, 0, 0, 6911, 536, 585, 0] payload_rows_per_dst=[3438, 5958, 1522, 14032, 9654, 9470, 14910, 16032] dup_rows_per_dst=[104, 482, 354, 5840, 466, 8, 1270, 14764] combine_rows_per_src=[3438, 5958, 1522, 14032, 9654, 9470, 14910, 16032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86624 payload_rows_remote=77008 dup_rows=11680 dup_rows_remote=11154 dedup_ratio=0.118815 router_tokens_per_expert=[0, 0, 133, 610, 0, 0, 1063, 0, 0, 5898, 2670, 0, 0, 0, 610, 0, 94, 346, 0, 492, 0, 54, 0, 0, 3857, 0, 0, 91, 0, 0, 0, 0, 54, 342, 0, 0, 4404, 180, 0, 91, 0, 0, 4663, 0, 0, 0, 0, 0, 653, 0, 90, 0, 7381, 0, 94, 90, 7093, 0, 0, 0, 7073, 475, 551, 0] payload_rows_per_dst=[8174, 6056, 14420, 13664, 9616, 9326, 15444, 9924] dup_rows_per_dst=[996, 504, 1738, 6028, 526, 0, 1172, 716] combine_rows_per_src=[8174, 6056, 14420, 13664, 9616, 9326, 15444, 9924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=282107904 dispatch_meta_bytes=331840 combine_payload_bytes=282107904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282439744 ingress_bytes=316710912 link_bytes=316710912 token_bytes=564547648 weight_movement_bytes=34603008 total_nvlink_bytes=599150656 weight_to_token_ratio=0.061293 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191757665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=241598464 dispatch_meta_bytes=270032 combine_payload_bytes=241598464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=241868496 ingress_bytes=345407488 link_bytes=345407488 token_bytes=483466960 weight_movement_bytes=103809024 total_nvlink_bytes=587275984 weight_to_token_ratio=0.214718 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299740570e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84294 payload_rows_remote=69886 dup_rows=14010 dup_rows_remote=12566 dedup_ratio=0.142517 router_tokens_per_expert=[0, 0, 4, 578, 0, 0, 1726, 0, 0, 3588, 5072, 0, 0, 0, 479, 0, 87, 330, 0, 316, 1, 84, 0, 0, 2167, 0, 0, 80, 0, 0, 0, 0, 88, 311, 0, 0, 4775, 187, 0, 73, 13, 0, 5552, 0, 0, 0, 0, 0, 674, 0, 73, 0, 7571, 0, 86, 49, 7108, 0, 0, 0, 7040, 480, 560, 0] payload_rows_per_dst=[15252, 11840, 14408, 7256, 10448, 12646, 10364, 2080] dup_rows_per_dst=[3444, 2742, 1444, 934, 420, 4220, 806, 0] combine_rows_per_src=[15252, 11840, 14408, 7256, 10448, 12646, 10364, 2080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90864 payload_rows_remote=80306 dup_rows=7440 dup_rows_remote=7344 dedup_ratio=0.075684 router_tokens_per_expert=[0, 0, 4, 522, 0, 0, 1978, 0, 0, 3348, 5247, 0, 0, 0, 495, 0, 66, 317, 0, 300, 9, 33, 0, 0, 1967, 0, 0, 77, 0, 0, 0, 0, 59, 329, 0, 0, 4983, 140, 0, 77, 48, 0, 5279, 0, 0, 0, 0, 0, 718, 0, 77, 0, 7606, 0, 66, 54, 7242, 0, 0, 0, 6993, 570, 548, 0] payload_rows_per_dst=[4886, 10546, 13782, 9366, 10754, 10558, 15754, 15218] dup_rows_per_dst=[122, 938, 1208, 1418, 422, 96, 1288, 1948] combine_rows_per_src=[4886, 10546, 13782, 9366, 10754, 10558, 15754, 15218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75822 payload_rows_remote=60412 dup_rows=22482 dup_rows_remote=21140 dedup_ratio=0.228699 router_tokens_per_expert=[0, 0, 152, 683, 0, 0, 1453, 0, 0, 5049, 3280, 0, 1, 2, 575, 0, 49, 296, 1, 337, 2, 65, 0, 0, 3831, 2, 0, 107, 1, 0, 0, 1, 67, 352, 1, 2, 3703, 154, 0, 107, 11, 0, 5062, 0, 0, 0, 1, 1, 708, 0, 107, 0, 7406, 1, 47, 107, 7140, 0, 0, 0, 7125, 554, 609, 0] payload_rows_per_dst=[4428, 7206, 1300, 13038, 8262, 10128, 15410, 16050] dup_rows_per_dst=[148, 510, 200, 4944, 510, 22, 1342, 14806] combine_rows_per_src=[4428, 7206, 1300, 13038, 8262, 10128, 15410, 16050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315424768 dispatch_meta_bytes=352648 combine_payload_bytes=315424768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315777416 ingress_bytes=350027776 link_bytes=350027776 token_bytes=631202184 weight_movement_bytes=34603008 total_nvlink_bytes=665805192 weight_to_token_ratio=0.054821 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317126342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89218 payload_rows_remote=80012 dup_rows=9086 dup_rows_remote=7800 dedup_ratio=0.092428 router_tokens_per_expert=[0, 0, 1, 519, 0, 0, 1991, 0, 0, 3217, 5098, 0, 0, 0, 408, 0, 79, 323, 0, 271, 0, 64, 0, 0, 1965, 0, 0, 64, 0, 0, 0, 0, 71, 308, 0, 0, 5124, 129, 0, 64, 23, 0, 5418, 0, 0, 0, 0, 0, 682, 0, 64, 0, 7618, 0, 79, 38, 7233, 0, 0, 0, 7197, 524, 580, 0] payload_rows_per_dst=[15314, 10250, 14632, 9206, 11020, 10836, 15752, 2208] dup_rows_per_dst=[4102, 762, 1308, 1286, 372, 46, 1210, 0] combine_rows_per_src=[15314, 10250, 14632, 9206, 11020, 10836, 15752, 2208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86034 payload_rows_remote=73556 dup_rows=12270 dup_rows_remote=11632 dedup_ratio=0.124817 router_tokens_per_expert=[4364, 0, 0, 72, 432, 4529, 0, 0, 1140, 95, 310, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 116, 0, 0, 0, 0, 72, 0, 0, 0, 0, 1440, 0, 0, 0, 2552, 414, 0, 6847, 4361, 8038, 0, 0, 5013, 11, 0, 0, 0, 72, 0, 0, 303, 6925, 749, 0, 0, 1, 607, 133, 258, 0] payload_rows_per_dst=[12468, 12478, 13838, 16076, 12758, 22, 7230, 11164] dup_rows_per_dst=[4290, 638, 452, 376, 4776, 0, 348, 1390] combine_rows_per_src=[12468, 12478, 13838, 16076, 12758, 22, 7230, 11164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286253056 dispatch_meta_bytes=329808 combine_payload_bytes=286253056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286582864 ingress_bytes=320856064 link_bytes=320856064 token_bytes=572835920 weight_movement_bytes=34603008 total_nvlink_bytes=607438928 weight_to_token_ratio=0.060406 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207355538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=328933376 dispatch_meta_bytes=350600 combine_payload_bytes=328933376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329283976 ingress_bytes=363536384 link_bytes=363536384 token_bytes=658217352 weight_movement_bytes=34603008 total_nvlink_bytes=692820360 weight_to_token_ratio=0.052571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367958146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=247447552 dispatch_meta_bytes=326208 combine_payload_bytes=247447552 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=247773760 ingress_bytes=385859584 link_bytes=385859584 token_bytes=495221312 weight_movement_bytes=138412032 total_nvlink_bytes=633633344 weight_to_token_ratio=0.279495 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451958550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=327729152 dispatch_meta_bytes=351248 combine_payload_bytes=327729152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328080400 ingress_bytes=362332160 link_bytes=362332160 token_bytes=655809552 weight_movement_bytes=34603008 total_nvlink_bytes=690412560 weight_to_token_ratio=0.052764 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363426748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301285376 dispatch_meta_bytes=340752 combine_payload_bytes=301285376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301626128 ingress_bytes=335888384 link_bytes=335888384 token_bytes=602911504 weight_movement_bytes=34603008 total_nvlink_bytes=637514512 weight_to_token_ratio=0.057393 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78326 payload_rows_remote=63042 dup_rows=19978 dup_rows_remote=15942 dedup_ratio=0.203227 router_tokens_per_expert=[0, 0, 80, 610, 0, 0, 1791, 0, 0, 3841, 3819, 0, 0, 0, 587, 0, 82, 328, 0, 346, 2, 60, 0, 0, 2669, 0, 0, 77, 0, 0, 0, 0, 66, 323, 0, 0, 5175, 140, 0, 77, 8, 0, 5142, 0, 0, 0, 0, 0, 657, 0, 77, 0, 7504, 0, 82, 75, 7176, 0, 0, 0, 7179, 567, 612, 0] payload_rows_per_dst=[15284, 14360, 14522, 5492, 14394, 10284, 1632, 2358] dup_rows_per_dst=[4036, 8204, 1466, 0, 6106, 16, 150, 0] combine_rows_per_src=[15284, 14360, 14522, 5492, 14394, 10284, 1632, 2358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65058 payload_rows_remote=55928 dup_rows=33246 dup_rows_remote=31874 dedup_ratio=0.338196 router_tokens_per_expert=[4202, 0, 0, 126, 541, 3496, 0, 0, 1241, 88, 372, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 84, 0, 0, 53, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1912, 0, 0, 0, 2564, 463, 0, 6735, 4300, 8038, 0, 0, 5504, 82, 0, 0, 0, 85, 0, 0, 285, 6911, 739, 0, 0, 0, 623, 151, 241, 1] payload_rows_per_dst=[9110, 6286, 628, 276, 8500, 16302, 14826, 9130] dup_rows_per_dst=[628, 288, 0, 0, 1378, 25934, 3646, 1372] combine_rows_per_src=[9110, 6286, 628, 276, 8500, 16302, 14826, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92672 payload_rows_remote=84094 dup_rows=5632 dup_rows_remote=4034 dedup_ratio=0.057292 router_tokens_per_expert=[4268, 0, 0, 163, 376, 3540, 0, 0, 1117, 94, 399, 0, 0, 0, 0, 0, 0, 91, 160, 0, 0, 91, 0, 0, 58, 0, 0, 0, 0, 91, 0, 0, 0, 0, 2126, 0, 0, 0, 2453, 509, 0, 6581, 4489, 7984, 0, 0, 5418, 87, 0, 0, 0, 91, 0, 0, 289, 6935, 732, 0, 0, 1, 569, 188, 252, 0] payload_rows_per_dst=[8974, 13168, 13488, 15968, 8578, 9152, 14052, 9292] dup_rows_per_dst=[640, 888, 358, 298, 1598, 0, 578, 1272] combine_rows_per_src=[8974, 13168, 13488, 15968, 8578, 9152, 14052, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=229081088 dispatch_meta_bytes=351208 combine_payload_bytes=229081088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=229432296 ingress_bytes=263684096 link_bytes=263684096 token_bytes=458513384 weight_movement_bytes=34603008 total_nvlink_bytes=493116392 weight_to_token_ratio=0.075468 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.922220249e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89994 payload_rows_remote=75998 dup_rows=8310 dup_rows_remote=7890 dedup_ratio=0.084534 router_tokens_per_expert=[4288, 0, 0, 74, 722, 4977, 0, 0, 1014, 85, 356, 0, 0, 0, 0, 0, 0, 74, 137, 0, 0, 73, 0, 0, 165, 0, 0, 0, 0, 74, 0, 0, 0, 0, 1261, 0, 0, 0, 1541, 388, 0, 6924, 5080, 8056, 0, 0, 4881, 0, 0, 0, 0, 74, 1, 0, 8, 7011, 900, 0, 0, 1, 566, 171, 244, 6] payload_rows_per_dst=[8808, 12192, 13996, 16112, 12534, 0, 14170, 12182] dup_rows_per_dst=[1360, 480, 420, 478, 4006, 0, 18, 1548] combine_rows_per_src=[8808, 12192, 13996, 16112, 12534, 0, 14170, 12182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=258220032 dispatch_meta_bytes=315936 combine_payload_bytes=258220032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258535968 ingress_bytes=292823040 link_bytes=292823040 token_bytes=516756000 weight_movement_bytes=34603008 total_nvlink_bytes=551359008 weight_to_token_ratio=0.066962 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101869525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92416 payload_rows_remote=81264 dup_rows=5888 dup_rows_remote=5116 dedup_ratio=0.059896 router_tokens_per_expert=[3491, 0, 0, 78, 666, 4977, 0, 0, 1178, 65, 340, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 142, 0, 0, 0, 1, 78, 0, 0, 0, 0, 1279, 0, 0, 0, 2297, 404, 0, 6967, 5284, 8077, 0, 0, 4629, 0, 0, 0, 0, 78, 0, 0, 61, 6823, 907, 0, 0, 0, 636, 171, 251, 0] payload_rows_per_dst=[7670, 11898, 14092, 14844, 6632, 11152, 13802, 12326] dup_rows_per_dst=[800, 526, 386, 396, 1328, 772, 122, 1558] combine_rows_per_src=[7670, 11898, 14092, 14844, 6632, 11152, 13802, 12326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81648 payload_rows_remote=67830 dup_rows=16656 dup_rows_remote=15972 dedup_ratio=0.169434 router_tokens_per_expert=[3983, 0, 2, 108, 489, 4024, 1, 0, 1424, 49, 374, 0, 0, 0, 0, 0, 0, 107, 87, 1, 0, 107, 1, 0, 71, 0, 0, 1, 2, 107, 0, 1, 1, 0, 1919, 0, 0, 1, 2508, 476, 0, 6818, 4008, 8094, 0, 1, 5083, 107, 0, 0, 0, 107, 0, 0, 342, 6802, 853, 2, 0, 17, 604, 223, 245, 2] payload_rows_per_dst=[8476, 12666, 11348, 362, 8268, 16210, 13818, 10500] dup_rows_per_dst=[690, 1194, 314, 2, 1542, 10790, 684, 1440] combine_rows_per_src=[8476, 12666, 11348, 362, 8268, 16210, 13818, 10500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=344449024 dispatch_meta_bytes=352512 combine_payload_bytes=344449024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=344801536 ingress_bytes=379052032 link_bytes=379052032 token_bytes=689250560 weight_movement_bytes=34603008 total_nvlink_bytes=723853568 weight_to_token_ratio=0.050204 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426342280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72718 payload_rows_remote=57174 dup_rows=25586 dup_rows_remote=18936 dedup_ratio=0.260274 router_tokens_per_expert=[1646, 666, 0, 0, 0, 0, 2712, 0, 0, 4033, 0, 64, 0, 0, 496, 0, 0, 144, 27, 0, 0, 617, 0, 0, 0, 0, 7009, 258, 1675, 0, 212, 2619, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 1264, 154, 6921, 93, 6851, 0, 7041, 0, 0, 187, 0, 72, 0, 0, 94, 0, 0, 0, 729, 72, 0, 194] payload_rows_per_dst=[8634, 15544, 14792, 15548, 11976, 3014, 1420, 1790] dup_rows_per_dst=[2788, 6650, 626, 11316, 3638, 8, 172, 388] combine_rows_per_src=[8634, 15544, 14792, 15548, 11976, 3014, 1420, 1790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92870 payload_rows_remote=76708 dup_rows=5434 dup_rows_remote=5028 dedup_ratio=0.055278 router_tokens_per_expert=[3665, 0, 0, 65, 694, 5383, 0, 0, 1152, 71, 356, 0, 0, 0, 0, 0, 0, 65, 131, 0, 0, 64, 0, 0, 138, 0, 0, 0, 0, 65, 0, 0, 0, 0, 1125, 0, 0, 0, 1983, 335, 0, 7037, 5029, 8081, 0, 0, 4740, 0, 0, 0, 0, 65, 0, 0, 7, 7027, 899, 0, 0, 0, 603, 123, 244, 5] payload_rows_per_dst=[7922, 12086, 14224, 16162, 7358, 7970, 14184, 12964] dup_rows_per_dst=[926, 552, 370, 406, 1616, 0, 14, 1550] combine_rows_per_src=[7922, 12086, 14224, 16162, 7358, 7970, 14184, 12964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=311287808 dispatch_meta_bytes=335552 combine_payload_bytes=311287808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311623360 ingress_bytes=345890816 link_bytes=345890816 token_bytes=622911168 weight_movement_bytes=34603008 total_nvlink_bytes=657514176 weight_to_token_ratio=0.055550 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301559294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=335208 combine_payload_bytes=277831680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278166888 ingress_bytes=347037696 link_bytes=347037696 token_bytes=555998568 weight_movement_bytes=69206016 total_nvlink_bytes=625204584 weight_to_token_ratio=0.124472 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305874911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=332857344 dispatch_meta_bytes=345520 combine_payload_bytes=332857344 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=333202864 ingress_bytes=505872384 link_bytes=505872384 token_bytes=666060208 weight_movement_bytes=173015040 total_nvlink_bytes=839075248 weight_to_token_ratio=0.259759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.903557055e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234184704 dispatch_meta_bytes=304440 combine_payload_bytes=234184704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=234489144 ingress_bytes=268787712 link_bytes=268787712 token_bytes=468673848 weight_movement_bytes=34603008 total_nvlink_bytes=503276856 weight_to_token_ratio=0.073832 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.011426521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76766 payload_rows_remote=62136 dup_rows=21538 dup_rows_remote=19666 dedup_ratio=0.219096 router_tokens_per_expert=[2169, 571, 0, 0, 0, 0, 1998, 0, 0, 4065, 0, 255, 68, 0, 599, 0, 0, 169, 88, 0, 0, 577, 0, 0, 0, 0, 6887, 240, 1381, 0, 153, 1603, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 877, 145, 6765, 192, 7065, 0, 7079, 0, 0, 244, 0, 84, 0, 0, 88, 0, 0, 0, 889, 84, 0, 303] payload_rows_per_dst=[7438, 9354, 1664, 3548, 9028, 15606, 15498, 14630] dup_rows_per_dst=[2038, 620, 4, 0, 0, 14482, 2522, 1872] combine_rows_per_src=[7438, 9354, 1664, 3548, 9028, 15606, 15498, 14630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=314195968 dispatch_meta_bytes=326944 combine_payload_bytes=314195968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314522912 ingress_bytes=348798976 link_bytes=348798976 token_bytes=628718880 weight_movement_bytes=34603008 total_nvlink_bytes=663321888 weight_to_token_ratio=0.055037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312502466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80504 payload_rows_remote=65240 dup_rows=17800 dup_rows_remote=7522 dedup_ratio=0.181071 router_tokens_per_expert=[3803, 0, 0, 77, 537, 4821, 0, 0, 1063, 82, 356, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 146, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1450, 0, 0, 0, 2013, 408, 0, 6900, 4845, 8051, 0, 0, 4981, 0, 0, 0, 0, 77, 0, 0, 329, 6879, 915, 0, 0, 0, 649, 152, 246, 0] payload_rows_per_dst=[15264, 12404, 13954, 16102, 12810, 0, 812, 9158] dup_rows_per_dst=[10278, 560, 436, 446, 4622, 0, 0, 1458] combine_rows_per_src=[15264, 12404, 13954, 16102, 12810, 0, 812, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88570 payload_rows_remote=76852 dup_rows=9734 dup_rows_remote=6804 dedup_ratio=0.099019 router_tokens_per_expert=[2219, 639, 0, 0, 0, 0, 1940, 0, 0, 4609, 0, 133, 43, 0, 542, 0, 0, 182, 53, 0, 0, 527, 0, 0, 0, 0, 6915, 253, 1245, 0, 173, 1575, 0, 0, 0, 0, 0, 0, 0, 4278, 0, 0, 844, 160, 6742, 168, 6966, 0, 7121, 0, 0, 267, 0, 91, 0, 0, 93, 0, 0, 0, 945, 90, 0, 339] payload_rows_per_dst=[7602, 9906, 14444, 6408, 11718, 9526, 14244, 14722] dup_rows_per_dst=[1994, 748, 564, 84, 2930, 658, 714, 2042] combine_rows_per_src=[7602, 9906, 14444, 6408, 11718, 9526, 14244, 14722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=254509056 dispatch_meta_bytes=327208 combine_payload_bytes=254509056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254836264 ingress_bytes=289112064 link_bytes=289112064 token_bytes=509345320 weight_movement_bytes=34603008 total_nvlink_bytes=543948328 weight_to_token_ratio=0.067936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087905421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84514 payload_rows_remote=69906 dup_rows=13790 dup_rows_remote=13206 dedup_ratio=0.140279 router_tokens_per_expert=[1471, 908, 0, 0, 0, 0, 3155, 0, 0, 3488, 0, 38, 0, 0, 551, 0, 0, 147, 2, 0, 0, 558, 0, 0, 0, 0, 7092, 244, 1454, 0, 186, 3095, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 1882, 137, 6889, 79, 6445, 0, 6920, 0, 0, 199, 0, 73, 0, 0, 76, 0, 0, 0, 810, 74, 0, 238] payload_rows_per_dst=[8838, 7522, 14608, 11662, 14284, 4172, 13874, 9554] dup_rows_per_dst=[2230, 632, 584, 4244, 4488, 24, 510, 1078] combine_rows_per_src=[8838, 7522, 14608, 11662, 14284, 4172, 13874, 9554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=291048 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267514088 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534737128 weight_movement_bytes=69206016 total_nvlink_bytes=603943144 weight_to_token_ratio=0.129421 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78364 payload_rows_remote=63036 dup_rows=19940 dup_rows_remote=16308 dedup_ratio=0.202840 router_tokens_per_expert=[1774, 849, 0, 0, 0, 0, 2094, 0, 0, 4241, 0, 123, 18, 0, 537, 0, 1, 214, 33, 0, 1, 555, 0, 1, 1, 0, 6799, 245, 1255, 0, 162, 2134, 2, 0, 0, 0, 2, 0, 0, 4297, 0, 2, 954, 86, 6863, 158, 6847, 1, 6962, 1, 0, 276, 0, 107, 0, 0, 49, 0, 0, 1, 983, 107, 0, 417] payload_rows_per_dst=[7392, 9062, 4624, 3326, 8598, 15482, 15328, 14552] dup_rows_per_dst=[2042, 776, 198, 0, 4, 11128, 3632, 2160] combine_rows_per_src=[7392, 9062, 4624, 3326, 8598, 15482, 15328, 14552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89124 payload_rows_remote=74414 dup_rows=9180 dup_rows_remote=6834 dedup_ratio=0.093384 router_tokens_per_expert=[1611, 771, 0, 0, 0, 0, 3341, 0, 0, 3291, 0, 27, 0, 0, 474, 0, 0, 155, 0, 0, 0, 612, 0, 0, 0, 0, 7079, 251, 1711, 0, 191, 3199, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 1743, 116, 6941, 83, 6586, 0, 6843, 0, 0, 199, 0, 77, 0, 0, 52, 0, 0, 0, 840, 78, 0, 225] payload_rows_per_dst=[9268, 6992, 14730, 9062, 5312, 14710, 14104, 14946] dup_rows_per_dst=[2178, 592, 686, 532, 0, 2346, 1244, 1602] combine_rows_per_src=[9268, 6992, 14730, 9062, 5312, 14710, 14104, 14946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=314785792 dispatch_meta_bytes=334624 combine_payload_bytes=314785792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315120416 ingress_bytes=349388800 link_bytes=349388800 token_bytes=629906208 weight_movement_bytes=34603008 total_nvlink_bytes=664509216 weight_to_token_ratio=0.054934 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314721926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63506 payload_rows_remote=47354 dup_rows=34798 dup_rows_remote=19840 dedup_ratio=0.353984 router_tokens_per_expert=[0, 0, 502, 0, 0, 0, 4868, 0, 0, 0, 131, 95, 7365, 0, 0, 7599, 72, 0, 0, 0, 0, 2868, 0, 0, 0, 0, 0, 8120, 0, 0, 5796, 0, 0, 0, 72, 0, 0, 7731, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 72, 0, 2167, 0, 0, 484, 593, 0, 0] payload_rows_per_dst=[10590, 16152, 5880, 16384, 4596, 718, 2704, 6482] dup_rows_per_dst=[150, 14958, 0, 19666, 0, 0, 18, 6] combine_rows_per_src=[10590, 16152, 5880, 16384, 4596, 718, 2704, 6482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=286334976 dispatch_meta_bytes=332448 combine_payload_bytes=286334976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286667424 ingress_bytes=320937984 link_bytes=320937984 token_bytes=573002400 weight_movement_bytes=34603008 total_nvlink_bytes=607605408 weight_to_token_ratio=0.060389 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207663796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87972 payload_rows_remote=78334 dup_rows=10332 dup_rows_remote=9870 dedup_ratio=0.105103 router_tokens_per_expert=[1524, 761, 0, 0, 0, 0, 3516, 0, 0, 3069, 0, 73, 0, 0, 464, 0, 0, 129, 0, 0, 0, 595, 0, 0, 0, 0, 7060, 244, 1307, 0, 155, 3344, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 1734, 129, 7068, 69, 6660, 0, 7020, 0, 0, 142, 0, 64, 0, 0, 51, 0, 0, 0, 876, 62, 0, 181] payload_rows_per_dst=[9428, 6660, 14928, 9638, 14606, 3800, 14044, 14868] dup_rows_per_dst=[2174, 552, 656, 462, 4424, 64, 408, 1592] combine_rows_per_src=[9428, 6660, 14928, 9638, 14606, 3800, 14044, 14868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=258195456 dispatch_meta_bytes=317376 combine_payload_bytes=258195456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258512832 ingress_bytes=327401472 link_bytes=327401472 token_bytes=516708288 weight_movement_bytes=69206016 total_nvlink_bytes=585914304 weight_to_token_ratio=0.133936 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231985381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304799744 dispatch_meta_bytes=324992 combine_payload_bytes=304799744 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=305124736 ingress_bytes=443211776 link_bytes=443211776 token_bytes=609924480 weight_movement_bytes=138412032 total_nvlink_bytes=748336512 weight_to_token_ratio=0.226933 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.667770232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86348 payload_rows_remote=71878 dup_rows=11956 dup_rows_remote=8392 dedup_ratio=0.121623 router_tokens_per_expert=[0, 0, 386, 0, 0, 0, 4066, 0, 0, 0, 141, 88, 7325, 0, 0, 7572, 85, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 8108, 0, 0, 5051, 0, 0, 0, 84, 0, 0, 7649, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 85, 0, 3024, 0, 0, 803, 139, 0, 0] payload_rows_per_dst=[8780, 458, 15164, 0, 15466, 16384, 15626, 14470] dup_rows_per_dst=[124, 0, 7498, 0, 0, 600, 170, 3564] combine_rows_per_src=[8780, 458, 15164, 0, 15466, 16384, 15626, 14470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=193961984 dispatch_meta_bytes=268776 combine_payload_bytes=193961984 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=194230760 ingress_bytes=297771008 link_bytes=297771008 token_bytes=388192744 weight_movement_bytes=103809024 total_nvlink_bytes=492001768 weight_to_token_ratio=0.267416 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120488331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=320856064 dispatch_meta_bytes=352816 combine_payload_bytes=320856064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=321208880 ingress_bytes=424665088 link_bytes=424665088 token_bytes=642064944 weight_movement_bytes=103809024 total_nvlink_bytes=745873968 weight_to_token_ratio=0.161680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.597980538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=58772 dup_rows=24218 dup_rows_remote=15272 dedup_ratio=0.246358 router_tokens_per_expert=[1753, 704, 0, 0, 0, 0, 2876, 0, 0, 3774, 0, 65, 0, 0, 556, 0, 0, 154, 2, 0, 0, 580, 0, 0, 0, 0, 6916, 255, 1592, 0, 215, 2259, 0, 0, 0, 0, 0, 0, 0, 3649, 0, 0, 1400, 141, 6862, 101, 6797, 0, 6952, 0, 0, 218, 0, 77, 0, 0, 78, 0, 0, 0, 877, 77, 0, 222] payload_rows_per_dst=[15314, 15986, 14610, 15146, 7298, 3268, 436, 2028] dup_rows_per_dst=[8946, 6708, 586, 7328, 0, 16, 154, 480] combine_rows_per_src=[15314, 15986, 14610, 15146, 7298, 3268, 436, 2028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=294412288 dispatch_meta_bytes=321080 combine_payload_bytes=294412288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294733368 ingress_bytes=329015296 link_bytes=329015296 token_bytes=589145656 weight_movement_bytes=34603008 total_nvlink_bytes=623748664 weight_to_token_ratio=0.058734 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238058071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88646 payload_rows_remote=73260 dup_rows=9658 dup_rows_remote=9658 dedup_ratio=0.098246 router_tokens_per_expert=[0, 0, 350, 0, 0, 0, 4079, 0, 0, 0, 157, 94, 7400, 0, 0, 7505, 91, 0, 0, 0, 0, 3741, 0, 0, 0, 0, 0, 8101, 0, 0, 5212, 0, 0, 0, 91, 0, 0, 7602, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 91, 0, 3046, 0, 0, 746, 152, 0, 0] payload_rows_per_dst=[8740, 3812, 13508, 2422, 15386, 16384, 15512, 12882] dup_rows_per_dst=[118, 76, 5570, 0, 0, 704, 182, 3008] combine_rows_per_src=[8740, 3812, 13508, 2422, 15386, 16384, 15512, 12882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93002 payload_rows_remote=89196 dup_rows=5302 dup_rows_remote=5302 dedup_ratio=0.053935 router_tokens_per_expert=[0, 0, 620, 0, 0, 0, 4860, 0, 0, 0, 124, 87, 7343, 0, 0, 7621, 74, 0, 0, 0, 0, 1829, 0, 0, 0, 0, 0, 8116, 0, 0, 6553, 0, 0, 0, 73, 0, 0, 7745, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 74, 0, 2151, 0, 0, 462, 801, 0, 0] payload_rows_per_dst=[10700, 14832, 3806, 13818, 15636, 11666, 15730, 6814] dup_rows_per_dst=[260, 276, 0, 4164, 0, 440, 148, 14] combine_rows_per_src=[10700, 14832, 3806, 13818, 15636, 11666, 15730, 6814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=240730112 dispatch_meta_bytes=296176 combine_payload_bytes=240730112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241026288 ingress_bytes=275333120 link_bytes=275333120 token_bytes=481756400 weight_movement_bytes=34603008 total_nvlink_bytes=516359408 weight_to_token_ratio=0.071827 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.036056364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87520 payload_rows_remote=71960 dup_rows=10784 dup_rows_remote=10570 dedup_ratio=0.109701 router_tokens_per_expert=[0, 0, 506, 0, 0, 0, 4545, 0, 0, 0, 144, 50, 7266, 0, 2, 7535, 107, 0, 0, 1, 0, 2988, 0, 1, 1, 0, 0, 8082, 0, 2, 5859, 0, 0, 0, 107, 0, 0, 7605, 2, 0, 1, 0, 415, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 0, 2725, 0, 0, 543, 310, 1, 0] payload_rows_per_dst=[9958, 392, 14992, 4, 15428, 16384, 15560, 14802] dup_rows_per_dst=[144, 0, 5734, 2, 0, 616, 214, 4074] combine_rows_per_src=[9958, 392, 14992, 4, 15428, 16384, 15560, 14802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90072 payload_rows_remote=73688 dup_rows=8232 dup_rows_remote=7704 dedup_ratio=0.083740 router_tokens_per_expert=[0, 0, 656, 0, 0, 0, 5204, 0, 0, 0, 127, 66, 7284, 0, 0, 7598, 78, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 8105, 0, 0, 6735, 0, 0, 0, 77, 0, 0, 7794, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 0, 1787, 0, 0, 506, 740, 0, 0] payload_rows_per_dst=[11462, 386, 14878, 0, 15742, 16384, 15698, 15522] dup_rows_per_dst=[258, 0, 3276, 0, 0, 528, 156, 4014] combine_rows_per_src=[11462, 386, 14878, 0, 15742, 16384, 15698, 15522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=300072960 dispatch_meta_bytes=331672 combine_payload_bytes=300072960 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300404632 ingress_bytes=403881984 link_bytes=403881984 token_bytes=600477592 weight_movement_bytes=103809024 total_nvlink_bytes=704286616 weight_to_token_ratio=0.172877 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519775391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=64794 dup_rows=18592 dup_rows_remote=17844 dedup_ratio=0.189128 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 59, 181, 0, 1428, 0, 144, 7965, 7915, 0, 0, 0, 0, 0, 677, 0, 7758, 0, 59, 0, 7400, 0, 0, 0, 0, 5941, 7980, 72, 0, 0, 0, 0, 0, 18, 0, 53, 0, 512, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 342, 303] payload_rows_per_dst=[15960, 14918, 16372, 16240, 144, 1734, 12910, 1434] dup_rows_per_dst=[0, 748, 16924, 748, 0, 16, 156, 0] combine_rows_per_src=[15960, 14918, 16372, 16240, 144, 1734, 12910, 1434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=365346816 dispatch_meta_bytes=377992 combine_payload_bytes=365346816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=365724808 ingress_bytes=399949824 link_bytes=399949824 token_bytes=731071624 weight_movement_bytes=34603008 total_nvlink_bytes=765674632 weight_to_token_ratio=0.047332 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504978989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97144 payload_rows_remote=83376 dup_rows=1160 dup_rows_remote=1160 dedup_ratio=0.011800 router_tokens_per_expert=[0, 0, 710, 0, 0, 0, 5420, 0, 0, 0, 114, 79, 7340, 0, 0, 7660, 65, 0, 0, 0, 0, 1290, 0, 0, 0, 0, 0, 8109, 0, 0, 6884, 0, 0, 0, 64, 0, 0, 7884, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 64, 0, 1511, 0, 0, 449, 975, 0, 0] payload_rows_per_dst=[11932, 14808, 2710, 13768, 15896, 16384, 15806, 5840] dup_rows_per_dst=[328, 258, 0, 0, 0, 412, 132, 30] combine_rows_per_src=[11932, 14808, 2710, 13768, 15896, 16384, 15806, 5840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=294748160 dispatch_meta_bytes=330120 combine_payload_bytes=294748160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295078280 ingress_bytes=329351168 link_bytes=329351168 token_bytes=589826440 weight_movement_bytes=34603008 total_nvlink_bytes=624429448 weight_to_token_ratio=0.058666 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.239321930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=301826048 dispatch_meta_bytes=325568 combine_payload_bytes=301826048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302151616 ingress_bytes=336429056 link_bytes=336429056 token_bytes=603977664 weight_movement_bytes=34603008 total_nvlink_bytes=638580672 weight_to_token_ratio=0.057292 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68942 payload_rows_remote=52858 dup_rows=29362 dup_rows_remote=28178 dedup_ratio=0.298686 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 57, 73, 0, 1937, 0, 169, 7871, 7873, 0, 0, 0, 0, 0, 508, 0, 7719, 0, 57, 0, 6045, 0, 1, 0, 0, 6439, 8003, 85, 0, 0, 0, 0, 0, 2, 0, 133, 0, 1157, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 346, 331] payload_rows_per_dst=[0, 592, 4212, 1130, 16270, 15908, 14746, 16084] dup_rows_per_dst=[0, 0, 0, 0, 27436, 104, 638, 1184] combine_rows_per_src=[0, 592, 4212, 1130, 16270, 15908, 14746, 16084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=330552 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265726776 ingress_bytes=299999232 link_bytes=299999232 token_bytes=531123000 weight_movement_bytes=34603008 total_nvlink_bytes=565726008 weight_to_token_ratio=0.065151 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128872958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61870 payload_rows_remote=46304 dup_rows=36434 dup_rows_remote=27240 dedup_ratio=0.370626 router_tokens_per_expert=[0, 0, 546, 0, 0, 0, 4861, 0, 0, 0, 136, 82, 7332, 0, 0, 7634, 76, 0, 0, 0, 0, 2568, 0, 0, 0, 0, 0, 8112, 0, 0, 5926, 0, 0, 0, 77, 0, 0, 7737, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 77, 0, 2261, 0, 0, 463, 667, 0, 0] payload_rows_per_dst=[15566, 16368, 5288, 16378, 154, 700, 646, 6770] dup_rows_per_dst=[9194, 15528, 0, 11698, 0, 0, 2, 12] combine_rows_per_src=[15566, 16368, 5288, 16378, 154, 700, 646, 6770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=341508096 dispatch_meta_bytes=338144 combine_payload_bytes=341508096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=341846240 ingress_bytes=445317120 link_bytes=445317120 token_bytes=683354336 weight_movement_bytes=103809024 total_nvlink_bytes=787163360 weight_to_token_ratio=0.151911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.675692472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95544 payload_rows_remote=95362 dup_rows=2760 dup_rows_remote=2760 dedup_ratio=0.028076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 66, 65, 0, 1904, 0, 182, 7836, 7821, 0, 0, 0, 0, 0, 511, 0, 7718, 0, 66, 0, 5990, 0, 0, 0, 0, 6601, 7979, 91, 0, 0, 0, 0, 0, 0, 0, 136, 0, 1176, 0, 134, 0, 0, 0, 0, 0, 0, 0, 1, 98, 0, 316, 317] payload_rows_per_dst=[15958, 12142, 4172, 16202, 182, 15836, 15044, 16008] dup_rows_per_dst=[0, 388, 0, 388, 0, 108, 778, 1098] combine_rows_per_src=[15958, 12142, 4172, 16202, 182, 15836, 15044, 16008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=216506368 dispatch_meta_bytes=324144 combine_payload_bytes=216506368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216830512 ingress_bytes=251109376 link_bytes=251109376 token_bytes=433336880 weight_movement_bytes=34603008 total_nvlink_bytes=467939888 weight_to_token_ratio=0.079852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.449043659e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90954 payload_rows_remote=83164 dup_rows=7350 dup_rows_remote=6334 dedup_ratio=0.074768 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 51, 331, 0, 1575, 0, 147, 7960, 7875, 0, 0, 0, 0, 0, 1020, 0, 7769, 0, 51, 0, 7623, 0, 1, 0, 0, 4874, 7993, 74, 0, 0, 0, 0, 0, 47, 0, 88, 0, 618, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 308, 298] payload_rows_per_dst=[15986, 15348, 7790, 16236, 150, 15920, 11052, 8472] dup_rows_per_dst=[0, 1230, 1016, 4228, 0, 270, 104, 502] combine_rows_per_src=[15986, 15348, 7790, 16236, 150, 15920, 11052, 8472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=189661184 dispatch_meta_bytes=294176 combine_payload_bytes=189661184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=189955360 ingress_bytes=224264192 link_bytes=224264192 token_bytes=379616544 weight_movement_bytes=34603008 total_nvlink_bytes=414219552 weight_to_token_ratio=0.091153 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.438880997e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73196 payload_rows_remote=60226 dup_rows=25108 dup_rows_remote=24832 dedup_ratio=0.255412 router_tokens_per_expert=[0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 215, 0, 0, 37, 142, 0, 2061, 0, 214, 7813, 7756, 0, 2, 1, 0, 0, 672, 0, 7683, 0, 38, 0, 6843, 1, 0, 0, 11, 5833, 7943, 107, 0, 0, 0, 2, 2, 29, 0, 83, 0, 640, 1, 148, 0, 1, 0, 0, 2, 0, 2, 0, 161, 2, 336, 363] payload_rows_per_dst=[3226, 3272, 4556, 1418, 16142, 15664, 12970, 15948] dup_rows_per_dst=[0, 124, 0, 2, 23216, 194, 276, 1296] combine_rows_per_src=[3226, 3272, 4556, 1418, 16142, 15664, 12970, 15948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94260 payload_rows_remote=78412 dup_rows=4044 dup_rows_remote=3828 dedup_ratio=0.041138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 50, 532, 0, 1897, 0, 155, 7924, 7866, 0, 0, 0, 0, 0, 1163, 0, 7697, 0, 50, 0, 7587, 0, 1, 0, 0, 4389, 7817, 78, 0, 0, 0, 0, 0, 49, 0, 59, 0, 541, 0, 135, 0, 0, 0, 0, 0, 0, 1, 3, 77, 0, 347, 367] payload_rows_per_dst=[15634, 15274, 4104, 5546, 11758, 15848, 10048, 16048] dup_rows_per_dst=[0, 1798, 0, 674, 0, 216, 82, 1274] combine_rows_per_src=[15634, 15274, 4104, 5546, 11758, 15848, 10048, 16048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=390602752 dispatch_meta_bytes=392488 combine_payload_bytes=390602752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=390995240 ingress_bytes=529014784 link_bytes=529014784 token_bytes=781597992 weight_movement_bytes=138412032 total_nvlink_bytes=920010024 weight_to_token_ratio=0.177089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.990640043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86978 payload_rows_remote=70996 dup_rows=11326 dup_rows_remote=8242 dedup_ratio=0.115214 router_tokens_per_expert=[0, 0, 0, 1324, 0, 0, 0, 0, 0, 0, 0, 72, 0, 72, 1756, 0, 0, 0, 1032, 2294, 8014, 815, 0, 72, 7766, 0, 0, 7633, 139, 59, 0, 0, 0, 0, 72, 0, 0, 0, 0, 872, 0, 3876, 0, 0, 0, 0, 72, 0, 0, 0, 0, 3339, 6709, 0, 535, 139, 2090, 72, 0, 0, 0, 315, 13, 0] payload_rows_per_dst=[16028, 15982, 10830, 15650, 1888, 7896, 13890, 4814] dup_rows_per_dst=[2648, 3084, 2806, 278, 0, 0, 2344, 166] combine_rows_per_src=[16028, 15982, 10830, 15650, 1888, 7896, 13890, 4814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=340639744 dispatch_meta_bytes=357992 combine_payload_bytes=340639744 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=340997736 ingress_bytes=444448768 link_bytes=444448768 token_bytes=681637480 weight_movement_bytes=103809024 total_nvlink_bytes=785446504 weight_to_token_ratio=0.152294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.672424933e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93068 payload_rows_remote=76814 dup_rows=5236 dup_rows_remote=3214 dedup_ratio=0.053263 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 50, 381, 0, 1604, 0, 129, 8010, 7940, 0, 0, 0, 0, 0, 1346, 0, 7742, 0, 50, 0, 7670, 0, 1, 0, 0, 4385, 7797, 65, 0, 0, 0, 0, 0, 50, 0, 71, 0, 564, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1, 64, 0, 340, 335] payload_rows_per_dst=[15594, 15440, 3466, 16254, 132, 16020, 10022, 16140] dup_rows_per_dst=[0, 1710, 0, 2022, 0, 242, 42, 1220] combine_rows_per_src=[15594, 15440, 3466, 16254, 132, 16020, 10022, 16140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=246685696 dispatch_meta_bytes=340232 combine_payload_bytes=246685696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247025928 ingress_bytes=281288704 link_bytes=281288704 token_bytes=493711624 weight_movement_bytes=34603008 total_nvlink_bytes=528314632 weight_to_token_ratio=0.070087 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.058466748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85168 payload_rows_remote=70302 dup_rows=13136 dup_rows_remote=10778 dedup_ratio=0.133626 router_tokens_per_expert=[0, 0, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1735, 0, 0, 0, 2122, 2418, 8037, 961, 0, 84, 7965, 0, 0, 7583, 376, 57, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1119, 0, 2564, 0, 0, 0, 0, 83, 0, 0, 0, 0, 3425, 6269, 0, 293, 157, 1763, 84, 0, 0, 0, 492, 4, 0] payload_rows_per_dst=[2618, 3638, 9800, 13498, 16044, 16074, 8630, 14866] dup_rows_per_dst=[0, 168, 1370, 248, 2292, 5294, 1406, 2358] combine_rows_per_src=[2618, 3638, 9800, 13498, 16044, 16074, 8630, 14866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=321175552 dispatch_meta_bytes=328960 combine_payload_bytes=321175552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321504512 ingress_bytes=355778560 link_bytes=355778560 token_bytes=642680064 weight_movement_bytes=34603008 total_nvlink_bytes=677283072 weight_to_token_ratio=0.053842 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338766079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290799616 dispatch_meta_bytes=316952 combine_payload_bytes=290799616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291116568 ingress_bytes=325402624 link_bytes=325402624 token_bytes=581916184 weight_movement_bytes=34603008 total_nvlink_bytes=616519192 weight_to_token_ratio=0.059464 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224463877e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76956 payload_rows_remote=61010 dup_rows=21348 dup_rows_remote=21348 dedup_ratio=0.217163 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 59, 222, 0, 1857, 0, 154, 7949, 7886, 0, 0, 0, 0, 0, 857, 0, 7692, 0, 59, 0, 7166, 0, 1, 0, 0, 5476, 7973, 77, 0, 0, 0, 0, 0, 46, 0, 66, 0, 514, 0, 87, 0, 0, 0, 0, 0, 0, 2, 6, 77, 0, 349, 364] payload_rows_per_dst=[15946, 14450, 16384, 16230, 2826, 224, 9304, 1592] dup_rows_per_dst=[0, 870, 19308, 986, 48, 0, 132, 4] combine_rows_per_src=[15946, 14450, 16384, 16230, 2826, 224, 9304, 1592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=314630144 dispatch_meta_bytes=320112 combine_payload_bytes=314630144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314950256 ingress_bytes=383836160 link_bytes=383836160 token_bytes=629580400 weight_movement_bytes=69206016 total_nvlink_bytes=698786416 weight_to_token_ratio=0.109924 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444344569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=287956992 dispatch_meta_bytes=324320 combine_payload_bytes=287956992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288281312 ingress_bytes=322560000 link_bytes=322560000 token_bytes=576238304 weight_movement_bytes=34603008 total_nvlink_bytes=610841312 weight_to_token_ratio=0.060050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213767312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=70316 dup_rows=11968 dup_rows_remote=9650 dedup_ratio=0.121745 router_tokens_per_expert=[0, 0, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1531, 0, 0, 0, 2256, 2409, 8020, 928, 0, 91, 7944, 0, 0, 7572, 458, 66, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1134, 0, 2402, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3709, 6210, 0, 232, 200, 1751, 91, 0, 0, 0, 438, 2, 0] payload_rows_per_dst=[13700, 8996, 10990, 8772, 16020, 4984, 8156, 14718] dup_rows_per_dst=[2178, 1604, 3232, 244, 2318, 0, 126, 2266] combine_rows_per_src=[13700, 8996, 10990, 8772, 16020, 4984, 8156, 14718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84700 payload_rows_remote=77464 dup_rows=13604 dup_rows_remote=13160 dedup_ratio=0.138387 router_tokens_per_expert=[0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 2192, 0, 0, 0, 653, 2339, 7990, 779, 0, 69, 7728, 0, 0, 7608, 133, 51, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1231, 0, 4351, 0, 0, 0, 0, 70, 0, 0, 0, 0, 2553, 6487, 0, 705, 130, 2250, 73, 0, 0, 0, 282, 3, 0] payload_rows_per_dst=[15980, 15998, 7236, 12084, 5460, 8842, 14050, 5050] dup_rows_per_dst=[2508, 3898, 444, 238, 650, 0, 5700, 166] combine_rows_per_src=[15980, 15998, 7236, 12084, 5460, 8842, 14050, 5050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=249896960 dispatch_meta_bytes=329432 combine_payload_bytes=249896960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250226392 ingress_bytes=284499968 link_bytes=284499968 token_bytes=500123352 weight_movement_bytes=34603008 total_nvlink_bytes=534726360 weight_to_token_ratio=0.069189 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.070550475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84806 payload_rows_remote=77540 dup_rows=13498 dup_rows_remote=13196 dedup_ratio=0.137309 router_tokens_per_expert=[0, 0, 1, 1555, 1, 0, 0, 0, 0, 0, 0, 107, 0, 107, 1240, 0, 0, 0, 1974, 2255, 7945, 863, 1, 107, 7938, 2, 0, 7507, 331, 37, 0, 0, 0, 2, 108, 2, 1, 0, 0, 776, 1, 3329, 0, 2, 0, 0, 108, 0, 0, 0, 0, 3164, 6495, 0, 431, 189, 2020, 108, 0, 0, 2, 389, 54, 0] payload_rows_per_dst=[3114, 2694, 9142, 15356, 15956, 15896, 7266, 15382] dup_rows_per_dst=[0, 214, 1258, 398, 1698, 6874, 302, 2754] combine_rows_per_src=[3114, 2694, 9142, 15356, 15956, 15896, 7266, 15382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288014336 dispatch_meta_bytes=319864 combine_payload_bytes=288014336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288334200 ingress_bytes=322617344 link_bytes=322617344 token_bytes=576348536 weight_movement_bytes=34603008 total_nvlink_bytes=610951544 weight_to_token_ratio=0.060038 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213983093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85366 payload_rows_remote=74098 dup_rows=12938 dup_rows_remote=11330 dedup_ratio=0.131612 router_tokens_per_expert=[0, 0, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1462, 0, 0, 0, 694, 1957, 7969, 804, 0, 76, 7669, 0, 0, 7559, 162, 50, 0, 0, 0, 0, 79, 0, 0, 0, 0, 764, 0, 4997, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2713, 6649, 0, 913, 157, 2307, 77, 0, 0, 0, 279, 60, 0] payload_rows_per_dst=[3050, 3080, 14794, 15222, 15438, 11268, 6958, 15556] dup_rows_per_dst=[0, 156, 5472, 320, 1586, 1608, 608, 3188] combine_rows_per_src=[3050, 3080, 14794, 15222, 15438, 11268, 6958, 15556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70688 payload_rows_remote=67880 dup_rows=27616 dup_rows_remote=27616 dedup_ratio=0.280924 router_tokens_per_expert=[0, 5394, 426, 1077, 8008, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 939, 0, 0, 0, 0, 264, 6630, 72, 0, 0, 0, 0, 0, 1913, 0, 0, 5858, 6799, 59, 0, 0, 580, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, 95, 7851, 72, 1326, 0, 0, 59, 0, 0, 0] payload_rows_per_dst=[14034, 2808, 12024, 3880, 15820, 10252, 1830, 10040] dup_rows_per_dst=[13354, 0, 1218, 122, 11184, 562, 0, 1176] combine_rows_per_src=[14034, 2808, 12024, 3880, 15820, 10252, 1830, 10040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=317292544 dispatch_meta_bytes=362496 combine_payload_bytes=317292544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317655040 ingress_bytes=421101568 link_bytes=421101568 token_bytes=634947584 weight_movement_bytes=103809024 total_nvlink_bytes=738756608 weight_to_token_ratio=0.163492 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584571299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86194 payload_rows_remote=85990 dup_rows=12110 dup_rows_remote=12016 dedup_ratio=0.123189 router_tokens_per_expert=[0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 63, 0, 64, 1972, 0, 0, 0, 489, 2054, 7970, 948, 0, 62, 7688, 0, 0, 7597, 99, 50, 0, 0, 0, 0, 72, 0, 0, 0, 0, 853, 0, 5305, 0, 0, 0, 0, 51, 0, 0, 0, 0, 2580, 6590, 0, 988, 95, 2356, 64, 0, 0, 0, 236, 12, 0] payload_rows_per_dst=[15940, 16076, 6760, 204, 15476, 10710, 10814, 10214] dup_rows_per_dst=[1788, 3316, 346, 94, 1750, 2, 3512, 1302] combine_rows_per_src=[15940, 16076, 6760, 204, 15476, 10710, 10814, 10214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=317603840 dispatch_meta_bytes=362944 combine_payload_bytes=317603840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317966784 ingress_bytes=421412864 link_bytes=421412864 token_bytes=635570624 weight_movement_bytes=103809024 total_nvlink_bytes=739379648 weight_to_token_ratio=0.163332 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585742681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90980 payload_rows_remote=77724 dup_rows=7324 dup_rows_remote=5448 dedup_ratio=0.074504 router_tokens_per_expert=[0, 6037, 439, 1224, 7927, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 895, 0, 0, 0, 0, 283, 6126, 88, 0, 0, 0, 0, 0, 1777, 0, 0, 6188, 6145, 131, 0, 0, 572, 0, 0, 0, 0, 0, 0, 926, 0, 0, 0, 0, 88, 7836, 133, 1163, 0, 0, 82, 0, 0, 0] payload_rows_per_dst=[3762, 15854, 13048, 12560, 3816, 12668, 16016, 13256] dup_rows_per_dst=[1178, 0, 1386, 434, 0, 766, 1684, 1876] combine_rows_per_src=[3762, 15854, 13048, 12560, 3816, 12668, 16016, 13256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303505408 dispatch_meta_bytes=341712 combine_payload_bytes=303505408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303847120 ingress_bytes=338108416 link_bytes=338108416 token_bytes=607352528 weight_movement_bytes=34603008 total_nvlink_bytes=641955536 weight_to_token_ratio=0.056974 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272274749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=278036480 dispatch_meta_bytes=381984 combine_payload_bytes=278036480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278418464 ingress_bytes=312639488 link_bytes=312639488 token_bytes=556454944 weight_movement_bytes=34603008 total_nvlink_bytes=591057952 weight_to_token_ratio=0.062185 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176437224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87834 payload_rows_remote=71846 dup_rows=10470 dup_rows_remote=7706 dedup_ratio=0.106506 router_tokens_per_expert=[0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1997, 0, 0, 0, 834, 2032, 7994, 921, 0, 75, 7645, 0, 0, 7549, 145, 59, 0, 0, 0, 0, 77, 0, 0, 0, 0, 787, 0, 4352, 0, 0, 0, 0, 76, 0, 0, 0, 0, 3050, 6573, 0, 558, 140, 2319, 77, 0, 0, 0, 328, 28, 0] payload_rows_per_dst=[15988, 16038, 10944, 15408, 1728, 8856, 13540, 5332] dup_rows_per_dst=[2764, 3362, 2880, 290, 0, 0, 1002, 172] combine_rows_per_src=[15988, 16038, 10944, 15408, 1728, 8856, 13540, 5332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=352215040 dispatch_meta_bytes=392024 combine_payload_bytes=352215040 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=352607064 ingress_bytes=456024064 link_bytes=456024064 token_bytes=704822104 weight_movement_bytes=103809024 total_nvlink_bytes=808631128 weight_to_token_ratio=0.147284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715981840e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=318357504 dispatch_meta_bytes=332688 combine_payload_bytes=318357504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318690192 ingress_bytes=352960512 link_bytes=352960512 token_bytes=637047696 weight_movement_bytes=34603008 total_nvlink_bytes=671650704 weight_to_token_ratio=0.054318 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328161991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91008 payload_rows_remote=87210 dup_rows=7296 dup_rows_remote=7296 dedup_ratio=0.074219 router_tokens_per_expert=[0, 5938, 484, 1216, 7909, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 993, 0, 0, 0, 0, 308, 6009, 122, 0, 0, 0, 0, 0, 1761, 0, 0, 6207, 6155, 138, 0, 0, 600, 0, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 94, 7785, 143, 1157, 0, 0, 91, 0, 0, 0] payload_rows_per_dst=[3890, 15818, 13084, 12396, 3798, 12766, 15940, 13316] dup_rows_per_dst=[1138, 0, 1436, 482, 0, 744, 1616, 1880] combine_rows_per_src=[3890, 15818, 13084, 12396, 3798, 12766, 15940, 13316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89074 payload_rows_remote=77548 dup_rows=9230 dup_rows_remote=7770 dedup_ratio=0.093892 router_tokens_per_expert=[0, 4918, 403, 1223, 7980, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 1235, 0, 0, 0, 0, 268, 6693, 73, 0, 0, 0, 0, 0, 2560, 0, 0, 5491, 6866, 51, 0, 0, 510, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 0, 86, 7856, 73, 1088, 0, 0, 37, 0, 0, 0] payload_rows_per_dst=[3422, 15960, 11526, 13460, 12358, 14068, 6072, 12208] dup_rows_per_dst=[1124, 0, 1460, 608, 3778, 684, 406, 1170] combine_rows_per_src=[3422, 15960, 11526, 13460, 12358, 14068, 6072, 12208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=294281216 dispatch_meta_bytes=318208 combine_payload_bytes=294281216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294599424 ingress_bytes=328884224 link_bytes=328884224 token_bytes=588880640 weight_movement_bytes=34603008 total_nvlink_bytes=623483648 weight_to_token_ratio=0.058761 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237564858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90790 payload_rows_remote=74940 dup_rows=7514 dup_rows_remote=5660 dedup_ratio=0.076436 router_tokens_per_expert=[0, 5721, 433, 1306, 7916, 832, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 215, 0, 2, 906, 0, 0, 0, 0, 321, 6599, 109, 0, 1, 0, 0, 0, 1564, 0, 0, 5973, 6334, 83, 0, 0, 601, 0, 2, 1, 0, 1, 0, 1033, 1, 0, 0, 2, 49, 7767, 114, 1151, 0, 2, 107, 0, 0, 1] payload_rows_per_dst=[3870, 15836, 12452, 13454, 3294, 13076, 15850, 12958] dup_rows_per_dst=[1272, 2, 1240, 606, 0, 802, 1854, 1738] combine_rows_per_src=[3870, 15836, 12452, 13454, 3294, 13076, 15850, 12958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=357212160 dispatch_meta_bytes=378024 combine_payload_bytes=357212160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357590184 ingress_bytes=461021184 link_bytes=461021184 token_bytes=714802344 weight_movement_bytes=103809024 total_nvlink_bytes=818611368 weight_to_token_ratio=0.145228 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.734785600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91488 payload_rows_remote=76894 dup_rows=6816 dup_rows_remote=6094 dedup_ratio=0.069336 router_tokens_per_expert=[0, 4983, 394, 1188, 8006, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 1098, 0, 0, 0, 0, 280, 6889, 77, 0, 0, 0, 0, 0, 2534, 0, 0, 4668, 7119, 50, 0, 0, 539, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 0, 66, 7807, 77, 1682, 0, 0, 14, 0, 0, 0] payload_rows_per_dst=[3246, 16012, 11210, 13806, 5168, 14594, 15900, 11552] dup_rows_per_dst=[1040, 0, 1398, 686, 0, 722, 1640, 1330] combine_rows_per_src=[3246, 16012, 11210, 13806, 5168, 14594, 15900, 11552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77612 payload_rows_remote=61522 dup_rows=20692 dup_rows_remote=17564 dedup_ratio=0.210490 router_tokens_per_expert=[7813, 0, 0, 0, 0, 0, 0, 1516, 481, 0, 365, 0, 0, 0, 0, 800, 0, 0, 0, 5355, 280, 396, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 7326, 0, 0, 151, 0, 0, 0, 0, 2414, 0, 7963, 0, 0, 0, 0, 0, 1216, 167, 0, 0, 72] payload_rows_per_dst=[16334, 16090, 13010, 10710, 6084, 7646, 4828, 2910] dup_rows_per_dst=[11836, 3128, 5112, 0, 550, 66, 0, 0] combine_rows_per_src=[16334, 16090, 13010, 10710, 6084, 7646, 4828, 2910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317636608 dispatch_meta_bytes=341272 combine_payload_bytes=317636608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317977880 ingress_bytes=352239616 link_bytes=352239616 token_bytes=635614488 weight_movement_bytes=34603008 total_nvlink_bytes=670217496 weight_to_token_ratio=0.054440 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=38528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.325449317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91928 payload_rows_remote=78162 dup_rows=6376 dup_rows_remote=5812 dedup_ratio=0.064860 router_tokens_per_expert=[0, 4414, 387, 1139, 8007, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 1335, 0, 0, 0, 0, 218, 6883, 64, 0, 0, 0, 0, 0, 2882, 0, 0, 4655, 7276, 50, 0, 0, 431, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 79, 7892, 64, 1686, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3128, 16014, 10750, 13766, 5864, 14814, 16070, 11522] dup_rows_per_dst=[1040, 0, 1490, 564, 0, 600, 1394, 1288] combine_rows_per_src=[3128, 16014, 10750, 13766, 5864, 14814, 16070, 11522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=306954240 dispatch_meta_bytes=322400 combine_payload_bytes=306954240 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=307276640 ingress_bytes=376160256 link_bytes=376160256 token_bytes=614230880 weight_movement_bytes=69206016 total_nvlink_bytes=683436896 weight_to_token_ratio=0.112671 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.415460760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75106 payload_rows_remote=59002 dup_rows=23198 dup_rows_remote=20452 dedup_ratio=0.235982 router_tokens_per_expert=[7696, 0, 0, 0, 0, 0, 0, 1821, 505, 0, 167, 0, 0, 0, 0, 943, 0, 0, 0, 6471, 287, 410, 0, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 7437, 0, 0, 60, 0, 0, 0, 0, 1802, 0, 7993, 0, 0, 0, 0, 0, 1472, 172, 0, 0, 85] payload_rows_per_dst=[3642, 2220, 1394, 13568, 7016, 14886, 16276, 16104] dup_rows_per_dst=[0, 1010, 0, 5926, 644, 918, 11954, 2746] combine_rows_per_src=[3642, 2220, 1394, 13568, 7016, 14886, 16276, 16104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=251994112 dispatch_meta_bytes=316344 combine_payload_bytes=251994112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252310456 ingress_bytes=286597120 link_bytes=286597120 token_bytes=504304568 weight_movement_bytes=34603008 total_nvlink_bytes=538907576 weight_to_token_ratio=0.068615 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078441890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314957824 dispatch_meta_bytes=331952 combine_payload_bytes=314957824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315289776 ingress_bytes=349560832 link_bytes=349560832 token_bytes=630247600 weight_movement_bytes=34603008 total_nvlink_bytes=664850608 weight_to_token_ratio=0.054904 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315369269e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=320151552 dispatch_meta_bytes=335896 combine_payload_bytes=320151552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320487448 ingress_bytes=389357568 link_bytes=389357568 token_bytes=640639000 weight_movement_bytes=69206016 total_nvlink_bytes=709845016 weight_to_token_ratio=0.108027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465121183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=53568 payload_rows_remote=37184 dup_rows=44736 dup_rows_remote=18314 dedup_ratio=0.455078 router_tokens_per_expert=[0, 5452, 405, 1358, 7990, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 948, 0, 0, 0, 0, 230, 6627, 77, 0, 0, 0, 0, 0, 1933, 0, 0, 5376, 6835, 59, 0, 0, 593, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 0, 82, 7842, 77, 1354, 0, 0, 66, 0, 0, 0] payload_rows_per_dst=[16384, 0, 4186, 11270, 15648, 1186, 2032, 2862] dup_rows_per_dst=[26422, 0, 396, 210, 17576, 0, 0, 132] combine_rows_per_src=[16384, 0, 4186, 11270, 15648, 1186, 2032, 2862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89852 payload_rows_remote=75238 dup_rows=8452 dup_rows_remote=6168 dedup_ratio=0.085978 router_tokens_per_expert=[7627, 0, 0, 0, 0, 0, 0, 1625, 481, 0, 185, 0, 0, 0, 0, 901, 0, 0, 0, 6471, 246, 463, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 0, 6985, 0, 0, 0, 7458, 0, 0, 55, 0, 0, 0, 0, 1731, 0, 7955, 0, 0, 0, 0, 0, 1431, 185, 0, 0, 91] payload_rows_per_dst=[3250, 10176, 8856, 12942, 14614, 14916, 9024, 16074] dup_rows_per_dst=[0, 2034, 158, 0, 2284, 110, 1272, 2594] combine_rows_per_src=[3250, 10176, 8856, 12942, 14614, 14916, 9024, 16074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=241672192 dispatch_meta_bytes=317816 combine_payload_bytes=241672192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241990008 ingress_bytes=276275200 link_bytes=276275200 token_bytes=483662200 weight_movement_bytes=34603008 total_nvlink_bytes=518265208 weight_to_token_ratio=0.071544 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.039601336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92708 payload_rows_remote=79834 dup_rows=5596 dup_rows_remote=5432 dedup_ratio=0.056925 router_tokens_per_expert=[7824, 0, 0, 0, 0, 0, 0, 881, 489, 0, 349, 0, 0, 0, 0, 809, 0, 0, 0, 4805, 256, 368, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 7317, 0, 0, 109, 0, 0, 0, 0, 2594, 0, 7881, 0, 0, 0, 0, 0, 1447, 159, 0, 0, 74] payload_rows_per_dst=[16216, 15932, 12874, 9610, 14852, 14676, 5188, 3360] dup_rows_per_dst=[1194, 3124, 164, 0, 938, 176, 0, 0] combine_rows_per_src=[16216, 15932, 12874, 9610, 14852, 14676, 5188, 3360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=152305664 dispatch_meta_bytes=221992 combine_payload_bytes=152305664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=152527656 ingress_bytes=256114688 link_bytes=256114688 token_bytes=304833320 weight_movement_bytes=103809024 total_nvlink_bytes=408642344 weight_to_token_ratio=0.340544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.637389520e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=308174848 dispatch_meta_bytes=325624 combine_payload_bytes=308174848 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=308500472 ingress_bytes=411983872 link_bytes=411983872 token_bytes=616675320 weight_movement_bytes=103809024 total_nvlink_bytes=720484344 weight_to_token_ratio=0.168337 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.550262143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86764 payload_rows_remote=70906 dup_rows=11540 dup_rows_remote=8544 dedup_ratio=0.117391 router_tokens_per_expert=[7664, 1, 0, 0, 0, 0, 0, 1380, 513, 2, 222, 0, 1, 0, 1, 954, 0, 2, 0, 6205, 298, 450, 0, 4472, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 2, 0, 0, 0, 0, 0, 7140, 0, 2, 0, 7395, 0, 2, 98, 1, 0, 0, 0, 1497, 1, 7928, 0, 1, 0, 0, 0, 1529, 157, 0, 0, 107] payload_rows_per_dst=[2762, 2360, 6880, 13062, 14832, 14852, 15858, 16158] dup_rows_per_dst=[0, 1026, 38, 2874, 1706, 144, 2996, 2756] combine_rows_per_src=[2762, 2360, 6880, 13062, 14832, 14852, 15858, 16158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83488 payload_rows_remote=67248 dup_rows=14816 dup_rows_remote=2002 dedup_ratio=0.150716 router_tokens_per_expert=[0, 0, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 433, 0, 72, 303, 59, 0, 0, 0, 59, 0, 175, 0, 0, 0, 0, 0, 0, 7540, 7058, 0, 72, 0, 0, 0, 53, 0, 187, 376, 0, 0, 0, 6629, 8061, 0, 0, 0, 0, 0, 258, 0, 72, 0, 0, 0, 1844, 0, 0, 0, 0, 0] payload_rows_per_dst=[15562, 16240, 13402, 4928, 12624, 16384, 660, 3688] dup_rows_per_dst=[0, 12814, 842, 68, 228, 864, 0, 0] combine_rows_per_src=[15562, 16240, 13402, 4928, 12624, 16384, 660, 3688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88274 payload_rows_remote=73576 dup_rows=10030 dup_rows_remote=9912 dedup_ratio=0.102030 router_tokens_per_expert=[7809, 0, 0, 0, 0, 0, 0, 1019, 443, 0, 379, 0, 0, 0, 0, 761, 0, 0, 0, 4961, 268, 403, 0, 5450, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 0, 0, 0, 7406, 0, 0, 0, 7333, 0, 0, 75, 0, 0, 0, 0, 2652, 0, 7793, 0, 0, 0, 0, 0, 1405, 143, 0, 0, 78] payload_rows_per_dst=[2038, 2280, 12120, 9922, 15346, 14698, 15586, 16284] dup_rows_per_dst=[0, 886, 122, 0, 1014, 118, 5304, 2586] combine_rows_per_src=[2038, 2280, 12120, 9922, 15346, 14698, 15586, 16284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=327000064 dispatch_meta_bytes=341064 combine_payload_bytes=327000064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=327341128 ingress_bytes=430809088 link_bytes=430809088 token_bytes=654341192 weight_movement_bytes=103809024 total_nvlink_bytes=758150216 weight_to_token_ratio=0.158647 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621099915e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92080 payload_rows_remote=83106 dup_rows=6224 dup_rows_remote=6224 dedup_ratio=0.063314 router_tokens_per_expert=[7882, 0, 0, 0, 0, 0, 0, 933, 456, 0, 433, 0, 0, 0, 0, 759, 0, 0, 0, 4487, 254, 329, 0, 5852, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 0, 0, 0, 7360, 0, 0, 0, 7317, 0, 0, 84, 0, 0, 0, 0, 2658, 0, 7801, 0, 0, 0, 0, 0, 1705, 143, 0, 0, 65] payload_rows_per_dst=[7366, 15756, 12730, 8974, 15230, 14660, 5316, 12048] dup_rows_per_dst=[504, 3142, 140, 0, 758, 142, 0, 1538] combine_rows_per_src=[7366, 15756, 12730, 8974, 15230, 14660, 5316, 12048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79148 payload_rows_remote=62934 dup_rows=19156 dup_rows_remote=15392 dedup_ratio=0.194865 router_tokens_per_expert=[0, 0, 7806, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8107, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 188, 0, 0, 0, 0, 0, 0, 7581, 7155, 0, 168, 0, 0, 0, 50, 0, 199, 378, 0, 0, 0, 6383, 8051, 0, 0, 1, 0, 0, 241, 0, 85, 0, 0, 0, 1882, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 858, 776, 15162, 14310, 15726, 16102, 16214] dup_rows_per_dst=[0, 6, 114, 376, 436, 13806, 654, 3764] combine_rows_per_src=[0, 858, 776, 15162, 14310, 15726, 16102, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290430976 dispatch_meta_bytes=317800 combine_payload_bytes=290430976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290748776 ingress_bytes=359636992 link_bytes=359636992 token_bytes=581179752 weight_movement_bytes=69206016 total_nvlink_bytes=650385768 weight_to_token_ratio=0.119079 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353285048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=301367296 dispatch_meta_bytes=333952 combine_payload_bytes=301367296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301701248 ingress_bytes=370573312 link_bytes=370573312 token_bytes=603068544 weight_movement_bytes=69206016 total_nvlink_bytes=672274560 weight_to_token_ratio=0.114756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394437539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=275447808 dispatch_meta_bytes=277000 combine_payload_bytes=275447808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275724808 ingress_bytes=344653824 link_bytes=344653824 token_bytes=551172616 weight_movement_bytes=69206016 total_nvlink_bytes=620378632 weight_to_token_ratio=0.125561 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296904593e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=340402176 dispatch_meta_bytes=357320 combine_payload_bytes=340402176 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=340759496 ingress_bytes=409608192 link_bytes=409608192 token_bytes=681161672 weight_movement_bytes=69206016 total_nvlink_bytes=750367688 weight_to_token_ratio=0.101600 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541322650e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69254 payload_rows_remote=52870 dup_rows=29050 dup_rows_remote=12272 dedup_ratio=0.295512 router_tokens_per_expert=[7811, 0, 0, 0, 0, 0, 0, 1704, 508, 0, 333, 0, 0, 0, 0, 830, 0, 0, 0, 5358, 275, 387, 0, 4359, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 0, 0, 0, 0, 0, 0, 7066, 0, 0, 0, 7332, 0, 0, 148, 0, 0, 0, 0, 2351, 0, 7899, 0, 0, 0, 0, 0, 1337, 159, 0, 0, 77] payload_rows_per_dst=[16384, 15978, 15596, 10716, 2436, 296, 4702, 3146] dup_rows_per_dst=[16778, 3162, 9110, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 15978, 15596, 10716, 2436, 296, 4702, 3146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92178 payload_rows_remote=77958 dup_rows=6126 dup_rows_remote=5658 dedup_ratio=0.062317 router_tokens_per_expert=[0, 0, 7780, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8101, 0, 480, 0, 2, 317, 66, 0, 0, 0, 66, 0, 229, 0, 0, 0, 0, 0, 0, 7550, 7110, 0, 180, 0, 0, 0, 54, 0, 204, 397, 0, 0, 0, 6497, 8035, 0, 0, 0, 0, 0, 252, 0, 91, 0, 0, 0, 1740, 0, 0, 0, 0, 0] payload_rows_per_dst=[15560, 960, 12998, 15100, 14220, 1068, 16070, 16202] dup_rows_per_dst=[0, 2, 898, 458, 468, 134, 686, 3480] combine_rows_per_src=[15560, 960, 12998, 15100, 14220, 1068, 16070, 16202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=257777664 dispatch_meta_bytes=313304 combine_payload_bytes=257777664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258090968 ingress_bytes=292380672 link_bytes=292380672 token_bytes=515868632 weight_movement_bytes=34603008 total_nvlink_bytes=550471640 weight_to_token_ratio=0.067077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100204930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95262 payload_rows_remote=82268 dup_rows=3042 dup_rows_remote=2242 dedup_ratio=0.030945 router_tokens_per_expert=[0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 412, 0, 72, 298, 51, 0, 0, 0, 51, 0, 171, 0, 0, 0, 0, 0, 0, 7373, 6946, 0, 74, 0, 0, 0, 88, 0, 170, 338, 0, 0, 0, 6425, 8068, 0, 0, 2, 0, 0, 244, 0, 74, 0, 0, 0, 2363, 0, 0, 0, 0, 0] payload_rows_per_dst=[15628, 16236, 12994, 14746, 13892, 4422, 12618, 4726] dup_rows_per_dst=[0, 824, 800, 342, 324, 260, 492, 0] combine_rows_per_src=[15628, 16236, 12994, 14746, 13892, 4422, 12618, 4726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=216555520 dispatch_meta_bytes=260568 combine_payload_bytes=216555520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216816088 ingress_bytes=285761536 link_bytes=285761536 token_bytes=433371608 weight_movement_bytes=69206016 total_nvlink_bytes=502577624 weight_to_token_ratio=0.159692 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.075297654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=319315968 dispatch_meta_bytes=334464 combine_payload_bytes=319315968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=319650432 ingress_bytes=388521984 link_bytes=388521984 token_bytes=638966400 weight_movement_bytes=69206016 total_nvlink_bytes=708172416 weight_to_token_ratio=0.108309 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.461976947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89850 payload_rows_remote=73754 dup_rows=8454 dup_rows_remote=7748 dedup_ratio=0.085999 router_tokens_per_expert=[0, 0, 7859, 0, 0, 0, 0, 0, 1, 29, 0, 0, 8080, 0, 446, 0, 66, 363, 37, 0, 0, 0, 37, 0, 205, 1, 1, 0, 0, 0, 0, 7453, 7021, 0, 148, 0, 2, 0, 66, 0, 175, 441, 0, 0, 2, 6592, 8046, 0, 2, 0, 0, 0, 245, 0, 107, 1, 0, 2, 1721, 2, 1, 0, 0, 0] payload_rows_per_dst=[0, 894, 11936, 14916, 14046, 15796, 16096, 16166] dup_rows_per_dst=[0, 58, 772, 404, 428, 2640, 706, 3446] combine_rows_per_src=[0, 894, 11936, 14916, 14046, 15796, 16096, 16166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90782 payload_rows_remote=82644 dup_rows=7522 dup_rows_remote=6870 dedup_ratio=0.076518 router_tokens_per_expert=[0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 429, 0, 78, 367, 50, 0, 0, 0, 50, 0, 194, 0, 0, 0, 0, 0, 0, 7244, 6612, 0, 77, 0, 0, 0, 59, 0, 237, 388, 0, 0, 0, 6714, 8065, 0, 0, 0, 0, 0, 251, 0, 78, 0, 0, 0, 2310, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 858, 13584, 14490, 13224, 8138, 16130, 16228] dup_rows_per_dst=[0, 0, 934, 386, 272, 652, 658, 4620] combine_rows_per_src=[8130, 858, 13584, 14490, 13224, 8138, 16130, 16228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76340 payload_rows_remote=64590 dup_rows=21964 dup_rows_remote=21964 dedup_ratio=0.223429 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 398, 0, 0, 7431, 5803, 0, 0, 72, 0, 0, 0, 0, 0, 59, 0, 389, 0, 0, 0, 0, 0, 0, 906, 7675, 0, 0, 0, 0, 0, 0, 1652, 7291, 0, 8120, 72, 0, 0, 816, 72, 0, 72, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 330, 7836, 0, 0, 0, 69] payload_rows_per_dst=[14862, 11750, 14598, 16010, 16384, 1776, 162, 798] dup_rows_per_dst=[812, 0, 880, 16824, 3160, 288, 0, 0] combine_rows_per_src=[14862, 11750, 14598, 16010, 16384, 1776, 162, 798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=336969728 dispatch_meta_bytes=338040 combine_payload_bytes=336969728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337307768 ingress_bytes=371572736 link_bytes=371572736 token_bytes=674277496 weight_movement_bytes=34603008 total_nvlink_bytes=708880504 weight_to_token_ratio=0.051319 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398198292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95218 payload_rows_remote=80802 dup_rows=3086 dup_rows_remote=2792 dedup_ratio=0.031392 router_tokens_per_expert=[0, 0, 7836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 375, 0, 59, 335, 50, 0, 0, 0, 51, 0, 147, 0, 0, 0, 0, 0, 0, 7208, 6637, 0, 64, 0, 0, 0, 71, 0, 202, 365, 0, 0, 0, 6411, 8078, 0, 0, 6, 0, 0, 244, 0, 65, 0, 0, 0, 2821, 0, 0, 0, 0, 0] payload_rows_per_dst=[15672, 15742, 12942, 14416, 13274, 988, 16156, 6028] dup_rows_per_dst=[0, 726, 870, 294, 270, 146, 630, 150] combine_rows_per_src=[15672, 15742, 12942, 14416, 13274, 988, 16156, 6028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81892 payload_rows_remote=66090 dup_rows=16412 dup_rows_remote=8748 dedup_ratio=0.166951 router_tokens_per_expert=[0, 47, 0, 0, 0, 0, 458, 0, 52, 7443, 5087, 0, 0, 84, 0, 0, 0, 0, 0, 26, 0, 432, 0, 0, 0, 0, 0, 0, 860, 7827, 0, 0, 0, 0, 0, 0, 1883, 7244, 0, 8108, 85, 0, 0, 992, 84, 0, 32, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 326, 7816, 0, 0, 0, 81] payload_rows_per_dst=[1010, 14886, 916, 1720, 15670, 15672, 16216, 15802] dup_rows_per_dst=[0, 272, 0, 0, 5738, 2368, 370, 7664] combine_rows_per_src=[1010, 14886, 916, 1720, 15670, 15672, 16216, 15802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302096384 dispatch_meta_bytes=326008 combine_payload_bytes=302096384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302422392 ingress_bytes=336699392 link_bytes=336699392 token_bytes=604518776 weight_movement_bytes=34603008 total_nvlink_bytes=639121784 weight_to_token_ratio=0.057241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266972706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264560640 dispatch_meta_bytes=346216 combine_payload_bytes=264560640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264906856 ingress_bytes=368369664 link_bytes=368369664 token_bytes=529467496 weight_movement_bytes=103809024 total_nvlink_bytes=633276520 weight_to_token_ratio=0.196063 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.386145389e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=338509824 dispatch_meta_bytes=358056 combine_payload_bytes=338509824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=338867880 ingress_bytes=442318848 link_bytes=442318848 token_bytes=677377704 weight_movement_bytes=103809024 total_nvlink_bytes=781186728 weight_to_token_ratio=0.153251 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.664410215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330964992 dispatch_meta_bytes=334376 combine_payload_bytes=330964992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331299368 ingress_bytes=400171008 link_bytes=400171008 token_bytes=662264360 weight_movement_bytes=69206016 total_nvlink_bytes=731470376 weight_to_token_ratio=0.104499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505811287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=299352 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271003992 ingress_bytes=339910656 link_bytes=339910656 token_bytes=541708632 weight_movement_bytes=69206016 total_nvlink_bytes=610914648 weight_to_token_ratio=0.127755 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279056433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92862 payload_rows_remote=89242 dup_rows=5442 dup_rows_remote=5442 dedup_ratio=0.055359 router_tokens_per_expert=[0, 47, 0, 0, 0, 0, 418, 0, 71, 7454, 5042, 0, 0, 91, 0, 0, 0, 0, 0, 57, 0, 367, 0, 0, 0, 0, 0, 0, 875, 7878, 0, 0, 0, 0, 0, 0, 1810, 7185, 0, 8101, 91, 0, 0, 1040, 91, 0, 22, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 343, 7852, 0, 0, 0, 58] payload_rows_per_dst=[14908, 10266, 14446, 1750, 3620, 15784, 16202, 15886] dup_rows_per_dst=[930, 142, 772, 0, 0, 2460, 518, 620] combine_rows_per_src=[14908, 10266, 14446, 1750, 3620, 15784, 16202, 15886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68538 payload_rows_remote=52800 dup_rows=29766 dup_rows_remote=17752 dedup_ratio=0.302795 router_tokens_per_expert=[0, 0, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 414, 0, 76, 364, 59, 0, 0, 0, 59, 0, 176, 0, 0, 0, 0, 0, 0, 7514, 7004, 0, 78, 0, 0, 0, 66, 0, 191, 393, 0, 0, 0, 6663, 8056, 0, 0, 0, 0, 0, 246, 0, 77, 0, 0, 0, 1796, 0, 0, 0, 0, 0] payload_rows_per_dst=[15738, 16230, 13478, 2182, 288, 16384, 646, 3592] dup_rows_per_dst=[12014, 15856, 964, 36, 0, 896, 0, 0] combine_rows_per_src=[15738, 16230, 13478, 2182, 288, 16384, 646, 3592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12625410416 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6316538224 ingress_bytes=7554580480 link_bytes=7554580480 total_nvlink_bytes=13871118704 weight_to_token_ratio=0.098667 prefetch_share_of_ingress=0.100769 dedup_ratio=0.202539 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030761 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.235968615e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85004 payload_rows_remote=70496 dup_rows=13300 dup_rows_remote=12426 dedup_ratio=0.135295 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 349, 0, 0, 7449, 5576, 0, 0, 73, 0, 0, 0, 0, 0, 51, 0, 388, 0, 0, 0, 0, 0, 0, 910, 7867, 0, 0, 0, 0, 0, 0, 1526, 7252, 0, 8119, 74, 0, 0, 926, 74, 0, 74, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 318, 7846, 0, 0, 0, 188] payload_rows_per_dst=[14898, 11298, 14508, 12704, 4018, 5360, 15118, 7100] dup_rows_per_dst=[702, 0, 874, 10324, 168, 718, 166, 348] combine_rows_per_src=[14898, 11298, 14508, 12704, 4018, 5360, 15118, 7100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11872874880 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=5940072832 ingress_bytes=7178510336 link_bytes=7178510336 total_nvlink_bytes=13118583168 weight_to_token_ratio=0.104921 prefetch_share_of_ingress=0.110868 dedup_ratio=0.220677 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=23368 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.174441586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=365535232 dispatch_meta_bytes=378736 combine_payload_bytes=365535232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365913968 ingress_bytes=469344256 link_bytes=469344256 token_bytes=731449200 weight_movement_bytes=103809024 total_nvlink_bytes=835258224 weight_to_token_ratio=0.141922 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.766104650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=216268800 dispatch_meta_bytes=282208 combine_payload_bytes=216268800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216551008 ingress_bytes=285474816 link_bytes=285474816 token_bytes=432819808 weight_movement_bytes=69206016 total_nvlink_bytes=502025824 weight_to_token_ratio=0.159896 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074218750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84760 payload_rows_remote=68594 dup_rows=13544 dup_rows_remote=13152 dedup_ratio=0.137777 router_tokens_per_expert=[0, 25, 2, 2, 3, 1, 449, 0, 17, 7489, 5395, 0, 0, 107, 0, 1, 0, 0, 0, 36, 2, 397, 2, 0, 0, 0, 0, 0, 923, 7737, 1, 0, 0, 2, 0, 0, 1704, 7137, 0, 8083, 107, 0, 0, 923, 110, 0, 93, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 352, 7780, 0, 0, 0, 76] payload_rows_per_dst=[2932, 14196, 4548, 1848, 13770, 15524, 16166, 15776] dup_rows_per_dst=[192, 214, 192, 0, 9498, 2416, 392, 640] combine_rows_per_src=[2932, 14196, 4548, 1848, 13770, 15524, 16166, 15776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=288751616 dispatch_meta_bytes=331688 combine_payload_bytes=288751616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289083304 ingress_bytes=323354624 link_bytes=323354624 token_bytes=577834920 weight_movement_bytes=34603008 total_nvlink_bytes=612437928 weight_to_token_ratio=0.059884 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216757418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93706 payload_rows_remote=79012 dup_rows=4598 dup_rows_remote=2576 dedup_ratio=0.046773 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 382, 0, 0, 7512, 5817, 0, 0, 77, 0, 0, 0, 0, 0, 50, 0, 427, 0, 0, 0, 0, 0, 0, 1107, 7329, 0, 0, 0, 0, 0, 0, 1686, 7293, 0, 8115, 78, 0, 0, 797, 77, 0, 77, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 329, 7818, 0, 0, 0, 65] payload_rows_per_dst=[15024, 11788, 14592, 2214, 3372, 14694, 16230, 15792] dup_rows_per_dst=[768, 0, 948, 0, 0, 2022, 228, 632] combine_rows_per_src=[15024, 11788, 14592, 2214, 3372, 14694, 16230, 15792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14130104576 prefetch_bytes=588251136 reduce_bytes=1141899264 weight_movement_bytes=1730150400 egress_bytes=7069042944 ingress_bytes=8791212032 link_bytes=8791212032 total_nvlink_bytes=15860254976 weight_to_token_ratio=0.122444 prefetch_share_of_ingress=0.066914 dedup_ratio=0.138962 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=33 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.438287962e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93932 payload_rows_remote=91828 dup_rows=4372 dup_rows_remote=4372 dedup_ratio=0.044474 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 348, 0, 0, 7480, 5763, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 406, 0, 0, 0, 0, 0, 0, 1052, 7535, 0, 0, 0, 0, 0, 0, 1682, 7346, 0, 8128, 65, 0, 0, 813, 64, 0, 70, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 309, 7814, 0, 0, 0, 110] payload_rows_per_dst=[14960, 11654, 14704, 2104, 3364, 15132, 16256, 15758] dup_rows_per_dst=[696, 0, 900, 0, 0, 1962, 106, 708] combine_rows_per_src=[14960, 11654, 14704, 2104, 3364, 15132, 16256, 15758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=280961024 dispatch_meta_bytes=326984 combine_payload_bytes=280961024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281288008 ingress_bytes=315564032 link_bytes=315564032 token_bytes=562249032 weight_movement_bytes=34603008 total_nvlink_bytes=596852040 weight_to_token_ratio=0.061544 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187442047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13721049248 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6864443552 ingress_bytes=8206123008 link_bytes=8206123008 total_nvlink_bytes=15070566560 weight_to_token_ratio=0.098354 prefetch_share_of_ingress=0.092768 dedup_ratio=0.143922 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=23749 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.342564358e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=323633152 dispatch_meta_bytes=326352 combine_payload_bytes=323633152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323959504 ingress_bytes=358236160 link_bytes=358236160 token_bytes=647592656 weight_movement_bytes=34603008 total_nvlink_bytes=682195664 weight_to_token_ratio=0.053433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348013830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=376127488 dispatch_meta_bytes=384800 combine_payload_bytes=376127488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=376512288 ingress_bytes=445333504 link_bytes=445333504 token_bytes=752639776 weight_movement_bytes=69206016 total_nvlink_bytes=821845792 weight_to_token_ratio=0.091951 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.675754123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12780964992 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6394309760 ingress_bytes=7839981568 link_bytes=7839981568 total_nvlink_bytes=14234291328 weight_to_token_ratio=0.113710 prefetch_share_of_ingress=0.088273 dedup_ratio=0.181731 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030000 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.282661716e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76430 payload_rows_remote=61564 dup_rows=21874 dup_rows_remote=21130 dedup_ratio=0.222514 router_tokens_per_expert=[0, 21, 0, 0, 0, 0, 351, 0, 0, 7433, 5842, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 401, 0, 0, 0, 0, 0, 0, 1035, 7584, 0, 0, 0, 0, 0, 0, 1656, 7253, 0, 8115, 77, 0, 0, 832, 77, 0, 78, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 323, 7784, 0, 0, 0, 67] payload_rows_per_dst=[14866, 11838, 14548, 16020, 16384, 1820, 174, 780] dup_rows_per_dst=[744, 0, 878, 16786, 3158, 308, 0, 0] combine_rows_per_src=[14866, 11838, 14548, 16020, 16384, 1820, 174, 780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13749556744 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6878672392 ingress_bytes=8462622720 link_bytes=8462622720 total_nvlink_bytes=15341295112 weight_to_token_ratio=0.115767 prefetch_share_of_ingress=0.089956 dedup_ratio=0.138886 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030000 gemm_inner_pad_upper_bound=23368 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.384529043e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14187562448 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=7097763280 ingress_bytes=8577728512 link_bytes=8577728512 total_nvlink_bytes=15675491792 weight_to_token_ratio=0.104876 prefetch_share_of_ingress=0.092783 dedup_ratio=0.129526 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.403360949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=252166144 dispatch_meta_bytes=330776 combine_payload_bytes=252166144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252496920 ingress_bytes=286769152 link_bytes=286769152 token_bytes=504663064 weight_movement_bytes=34603008 total_nvlink_bytes=539266072 weight_to_token_ratio=0.068567 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079089232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11473798856 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=5740439240 ingress_bytes=7186685952 link_bytes=7186685952 total_nvlink_bytes=12927125192 weight_to_token_ratio=0.126665 prefetch_share_of_ingress=0.086668 dedup_ratio=0.238771 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.032065 gemm_inner_pad_upper_bound=21082 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.175779159e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none + [2026-08-29 23:40:36] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 669.5 | throughput per GPU (TFLOP/s/GPU): 190.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.606197E+00 | loss scale: 1.0 | grad norm: 3.129 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75378 payload_rows_remote=64988 dup_rows=22926 dup_rows_remote=19614 dedup_ratio=0.233215 router_tokens_per_expert=[1309, 244, 719, 951, 293, 1032, 800, 1103, 1333, 207, 620, 408, 346, 1187, 843, 1184, 749, 1020, 727, 560, 369, 631, 310, 1482, 462, 654, 817, 341, 682, 61, 903, 1196, 814, 903, 1143, 726, 613, 268, 452, 794, 671, 681, 555, 889, 980, 866, 441, 649, 908, 536, 792, 623, 989, 1299, 829, 875, 1524, 454, 1499, 739, 163, 812, 1233, 889] payload_rows_per_dst=[7864, 9294, 9254, 8676, 9256, 10026, 10390, 10618] dup_rows_per_dst=[2420, 2962, 2442, 1556, 2170, 4056, 3312, 4008] combine_rows_per_src=[7864, 9294, 9254, 8676, 9256, 10026, 10390, 10618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75876 payload_rows_remote=65806 dup_rows=22428 dup_rows_remote=19872 dedup_ratio=0.228149 router_tokens_per_expert=[1319, 260, 722, 971, 302, 924, 724, 1167, 1390, 141, 512, 481, 419, 1051, 855, 1194, 603, 960, 782, 565, 316, 688, 278, 1440, 638, 734, 730, 322, 649, 62, 913, 956, 851, 935, 1218, 688, 880, 236, 501, 1004, 661, 659, 535, 890, 907, 922, 424, 713, 657, 544, 847, 560, 1017, 1320, 891, 790, 1587, 540, 1562, 755, 109, 715, 1169, 997] payload_rows_per_dst=[9424, 9212, 9040, 9556, 10070, 8814, 10088, 9672] dup_rows_per_dst=[3354, 2874, 2224, 2894, 2556, 2608, 3164, 2754] combine_rows_per_src=[9424, 9212, 9040, 9556, 10070, 8814, 10088, 9672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75554 payload_rows_remote=65938 dup_rows=22750 dup_rows_remote=19672 dedup_ratio=0.231425 router_tokens_per_expert=[1260, 219, 760, 1005, 295, 715, 771, 1357, 1212, 151, 500, 381, 462, 941, 945, 1217, 650, 1012, 796, 602, 294, 641, 272, 1595, 790, 697, 696, 344, 745, 61, 857, 1192, 808, 978, 1069, 631, 576, 218, 498, 1088, 674, 617, 489, 994, 877, 853, 626, 772, 744, 515, 871, 443, 1150, 1174, 963, 915, 1515, 532, 1374, 663, 131, 814, 1115, 1030] payload_rows_per_dst=[9304, 9616, 10304, 10386, 9422, 9026, 8428, 9068] dup_rows_per_dst=[3460, 3078, 3768, 3408, 2310, 2778, 1698, 2250] combine_rows_per_src=[9304, 9616, 10304, 10386, 9422, 9026, 8428, 9068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75922 payload_rows_remote=65898 dup_rows=22382 dup_rows_remote=19092 dedup_ratio=0.227681 router_tokens_per_expert=[1266, 282, 777, 975, 354, 1029, 814, 1240, 1318, 190, 575, 359, 388, 1000, 954, 1203, 657, 987, 793, 561, 401, 603, 268, 1506, 652, 655, 744, 341, 706, 74, 848, 1107, 807, 914, 1114, 718, 728, 236, 423, 1074, 710, 713, 469, 882, 839, 879, 483, 676, 894, 517, 827, 517, 1049, 1242, 909, 768, 1530, 331, 1359, 805, 217, 781, 1142, 972] payload_rows_per_dst=[9836, 9262, 9090, 10024, 9548, 8818, 10198, 9146] dup_rows_per_dst=[3638, 2712, 2462, 3290, 2480, 2484, 3248, 2068] combine_rows_per_src=[9836, 9262, 9090, 10024, 9548, 8818, 10198, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75618 payload_rows_remote=66834 dup_rows=22686 dup_rows_remote=20026 dedup_ratio=0.230774 router_tokens_per_expert=[1214, 268, 803, 969, 287, 881, 833, 1277, 1374, 184, 599, 367, 432, 1090, 1030, 1030, 711, 1012, 778, 612, 350, 675, 286, 1522, 644, 594, 728, 333, 754, 65, 937, 1129, 791, 884, 1107, 699, 647, 256, 456, 964, 711, 679, 464, 910, 822, 910, 427, 726, 847, 600, 804, 521, 1026, 1211, 846, 833, 1446, 617, 1521, 661, 148, 781, 1123, 946] payload_rows_per_dst=[9464, 9274, 9376, 8742, 9408, 8784, 10156, 10414] dup_rows_per_dst=[3454, 2938, 2516, 1626, 2200, 2660, 3220, 4072] combine_rows_per_src=[9464, 9274, 9376, 8742, 9408, 8784, 10156, 10414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75382 payload_rows_remote=65804 dup_rows=22922 dup_rows_remote=19918 dedup_ratio=0.233175 router_tokens_per_expert=[1196, 244, 705, 970, 329, 941, 786, 1282, 1247, 155, 595, 445, 435, 947, 943, 1154, 674, 972, 823, 590, 360, 661, 307, 1432, 706, 664, 745, 359, 714, 97, 1006, 1054, 904, 871, 1245, 694, 791, 226, 466, 1021, 686, 657, 506, 909, 893, 904, 561, 772, 674, 520, 784, 516, 1067, 1207, 871, 883, 1507, 350, 1424, 706, 152, 706, 1182, 959] payload_rows_per_dst=[9532, 9070, 9578, 10078, 9828, 8842, 9398, 9056] dup_rows_per_dst=[3374, 2772, 3004, 3626, 2608, 2934, 2702, 1902] combine_rows_per_src=[9532, 9070, 9578, 10078, 9828, 8842, 9398, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=64864 dup_rows=22980 dup_rows_remote=19012 dedup_ratio=0.233765 router_tokens_per_expert=[1280, 281, 752, 1068, 313, 944, 685, 1294, 1315, 200, 532, 450, 459, 950, 807, 1166, 628, 995, 853, 477, 396, 659, 329, 1494, 662, 693, 761, 299, 669, 57, 933, 1222, 884, 875, 1144, 678, 736, 216, 462, 957, 748, 662, 476, 968, 869, 876, 572, 672, 754, 515, 815, 528, 1039, 1184, 892, 882, 1549, 504, 1434, 739, 146, 645, 1111, 997] payload_rows_per_dst=[7830, 9024, 9336, 8742, 9638, 10074, 10220, 10460] dup_rows_per_dst=[2228, 2734, 2326, 1850, 2266, 4172, 3436, 3968] combine_rows_per_src=[7830, 9024, 9336, 8742, 9638, 10074, 10220, 10460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=338408 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266529256 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532720104 weight_movement_bytes=103809024 total_nvlink_bytes=636529128 weight_to_token_ratio=0.194866 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269541376 dispatch_meta_bytes=342712 combine_payload_bytes=269541376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269884088 ingress_bytes=304144384 link_bytes=304144384 token_bytes=539425464 weight_movement_bytes=34603008 total_nvlink_bytes=574028472 weight_to_token_ratio=0.064148 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144470831e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270082048 dispatch_meta_bytes=342440 combine_payload_bytes=270082048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270424488 ingress_bytes=304685056 link_bytes=304685056 token_bytes=540506536 weight_movement_bytes=34603008 total_nvlink_bytes=575109544 weight_to_token_ratio=0.064020 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146505337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269918208 dispatch_meta_bytes=339960 combine_payload_bytes=269918208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270258168 ingress_bytes=304521216 link_bytes=304521216 token_bytes=540176376 weight_movement_bytes=34603008 total_nvlink_bytes=574779384 weight_to_token_ratio=0.064059 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145888820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273752064 dispatch_meta_bytes=347440 combine_payload_bytes=273752064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274099504 ingress_bytes=308355072 link_bytes=308355072 token_bytes=547851568 weight_movement_bytes=34603008 total_nvlink_bytes=582454576 weight_to_token_ratio=0.063161 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160315311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269533184 dispatch_meta_bytes=342888 combine_payload_bytes=269533184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269876072 ingress_bytes=304136192 link_bytes=304136192 token_bytes=539409256 weight_movement_bytes=34603008 total_nvlink_bytes=574012264 weight_to_token_ratio=0.064150 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144440006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=335504 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266018448 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531701392 weight_movement_bytes=103809024 total_nvlink_bytes=635510416 weight_to_token_ratio=0.195239 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73954 payload_rows_remote=64494 dup_rows=24350 dup_rows_remote=20688 dedup_ratio=0.247701 router_tokens_per_expert=[1300, 310, 749, 1034, 317, 730, 760, 1361, 1293, 146, 511, 406, 518, 944, 822, 1013, 684, 1035, 709, 550, 321, 588, 299, 1453, 644, 698, 769, 336, 673, 66, 1020, 1188, 881, 973, 1271, 657, 637, 228, 434, 950, 783, 729, 552, 979, 742, 883, 739, 707, 662, 495, 840, 529, 1088, 1153, 859, 938, 1531, 557, 1378, 724, 99, 675, 1089, 1143] payload_rows_per_dst=[9460, 9918, 10044, 10280, 10130, 9104, 7468, 7550] dup_rows_per_dst=[3662, 3564, 3540, 3570, 4414, 3124, 1178, 1298] combine_rows_per_src=[9460, 9918, 10044, 10280, 10130, 9104, 7468, 7550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76488 payload_rows_remote=65744 dup_rows=21816 dup_rows_remote=18426 dedup_ratio=0.221924 router_tokens_per_expert=[791, 687, 999, 415, 867, 306, 326, 1028, 122, 984, 1209, 428, 577, 1119, 943, 1331, 1186, 1113, 1025, 395, 1418, 114, 1817, 1041, 560, 638, 631, 756, 106, 282, 222, 105, 498, 333, 640, 286, 1062, 2161, 1125, 1064, 1957, 1495, 1388, 1418, 445, 782, 78, 174, 651, 1274, 722, 50, 171, 1493, 289, 1252, 888, 371, 642, 661, 898, 657, 262, 424] payload_rows_per_dst=[8728, 10052, 8892, 8702, 8330, 10662, 10744, 10378] dup_rows_per_dst=[2110, 3374, 1362, 1812, 1686, 5220, 3390, 2862] combine_rows_per_src=[8728, 10052, 8892, 8702, 8330, 10662, 10744, 10378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76776 payload_rows_remote=67278 dup_rows=21528 dup_rows_remote=19156 dedup_ratio=0.218994 router_tokens_per_expert=[833, 691, 981, 558, 895, 300, 334, 1083, 147, 1053, 1137, 425, 539, 1162, 977, 1476, 1326, 1040, 979, 396, 1334, 92, 1482, 1057, 518, 741, 717, 795, 128, 273, 268, 123, 334, 248, 656, 228, 1120, 2161, 1022, 969, 2056, 1410, 1349, 1453, 462, 778, 79, 197, 735, 1231, 851, 36, 145, 1390, 215, 1171, 857, 333, 886, 681, 794, 698, 332, 415] payload_rows_per_dst=[8752, 10374, 9990, 9272, 9498, 9102, 9510, 10278] dup_rows_per_dst=[2598, 3458, 2458, 1966, 2372, 3960, 2038, 2678] combine_rows_per_src=[8752, 10374, 9990, 9272, 9498, 9102, 9510, 10278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77380 payload_rows_remote=69436 dup_rows=20924 dup_rows_remote=18630 dedup_ratio=0.212850 router_tokens_per_expert=[814, 696, 1063, 504, 778, 274, 289, 1061, 169, 992, 913, 308, 502, 1064, 1171, 1229, 1563, 1025, 899, 430, 1436, 79, 1416, 976, 455, 690, 613, 681, 131, 399, 180, 89, 438, 261, 805, 313, 1072, 2530, 1060, 1052, 2094, 1448, 1727, 1740, 534, 575, 48, 109, 570, 1382, 610, 114, 163, 1583, 224, 1149, 576, 330, 847, 688, 903, 619, 295, 404] payload_rows_per_dst=[10536, 7944, 10098, 10692, 11154, 7570, 9420, 9966] dup_rows_per_dst=[2880, 2294, 2718, 3452, 3908, 1312, 2170, 2190] combine_rows_per_src=[10536, 7944, 10098, 10692, 11154, 7570, 9420, 9966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77326 payload_rows_remote=67162 dup_rows=20978 dup_rows_remote=18164 dedup_ratio=0.213399 router_tokens_per_expert=[833, 803, 1043, 507, 838, 316, 359, 1144, 170, 1005, 1090, 371, 511, 1183, 930, 1298, 1343, 1066, 884, 430, 1337, 141, 1316, 1052, 521, 712, 650, 767, 157, 392, 310, 139, 379, 298, 692, 224, 1034, 2333, 1080, 991, 1919, 1496, 1409, 1535, 579, 750, 71, 151, 592, 1292, 711, 60, 178, 1520, 250, 1179, 793, 337, 789, 679, 814, 700, 299, 400] payload_rows_per_dst=[9382, 9438, 9942, 10164, 10948, 8044, 9520, 9888] dup_rows_per_dst=[2836, 3146, 2564, 2814, 3114, 2094, 2044, 2366] combine_rows_per_src=[9382, 9438, 9942, 10164, 10948, 8044, 9520, 9888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76626 payload_rows_remote=65704 dup_rows=21678 dup_rows_remote=15990 dedup_ratio=0.220520 router_tokens_per_expert=[749, 761, 1024, 543, 793, 246, 270, 1052, 123, 1080, 1074, 348, 532, 1081, 930, 1243, 1324, 1021, 964, 511, 1330, 114, 1455, 1023, 509, 722, 600, 733, 151, 476, 225, 124, 452, 388, 731, 261, 948, 2269, 1180, 1094, 2068, 1463, 1469, 1671, 517, 651, 71, 194, 658, 1160, 776, 62, 209, 1506, 235, 1283, 701, 359, 834, 629, 901, 596, 285, 400] payload_rows_per_dst=[8748, 9844, 9940, 9254, 8354, 10922, 9510, 10054] dup_rows_per_dst=[2128, 2978, 2634, 1962, 1754, 5688, 2268, 2266] combine_rows_per_src=[8748, 9844, 9940, 9254, 8354, 10922, 9510, 10054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264167424 dispatch_meta_bytes=340728 combine_payload_bytes=264167424 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264508152 ingress_bytes=367976448 link_bytes=367976448 token_bytes=528675576 weight_movement_bytes=103809024 total_nvlink_bytes=632484600 weight_to_token_ratio=0.196357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384665749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77730 payload_rows_remote=67926 dup_rows=20574 dup_rows_remote=18322 dedup_ratio=0.209290 router_tokens_per_expert=[840, 699, 1020, 523, 895, 245, 348, 1178, 207, 1053, 1117, 401, 526, 1194, 1045, 1345, 1323, 1025, 839, 429, 1221, 99, 1345, 1092, 493, 751, 592, 741, 191, 362, 243, 131, 418, 258, 749, 248, 981, 2237, 1083, 956, 2025, 1305, 1515, 1489, 535, 687, 68, 182, 722, 1284, 838, 51, 157, 1491, 235, 1097, 702, 365, 977, 751, 782, 700, 307, 444] payload_rows_per_dst=[10838, 8408, 9804, 10810, 10736, 7410, 9600, 10124] dup_rows_per_dst=[3348, 2678, 2252, 3226, 3124, 1174, 2150, 2622] combine_rows_per_src=[10838, 8408, 9804, 10810, 10736, 7410, 9600, 10124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269287424 dispatch_meta_bytes=336680 combine_payload_bytes=269287424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269624104 ingress_bytes=303890432 link_bytes=303890432 token_bytes=538911528 weight_movement_bytes=34603008 total_nvlink_bytes=573514536 weight_to_token_ratio=0.064209 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143515230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275570688 dispatch_meta_bytes=345736 combine_payload_bytes=275570688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275916424 ingress_bytes=379379712 link_bytes=379379712 token_bytes=551487112 weight_movement_bytes=103809024 total_nvlink_bytes=655296136 weight_to_token_ratio=0.188235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427575314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=66600 dup_rows=21984 dup_rows_remote=19794 dedup_ratio=0.223633 router_tokens_per_expert=[889, 669, 1046, 488, 774, 348, 429, 1054, 155, 1095, 1143, 368, 564, 1148, 976, 1354, 1343, 1132, 878, 398, 1307, 99, 1452, 946, 480, 646, 616, 694, 117, 283, 291, 131, 460, 314, 735, 312, 1101, 2292, 1076, 1166, 2053, 1451, 1599, 1510, 504, 608, 70, 188, 617, 1291, 590, 90, 199, 1549, 232, 1329, 657, 308, 712, 652, 951, 604, 271, 348] payload_rows_per_dst=[8930, 10076, 8312, 8712, 8760, 10882, 10928, 9720] dup_rows_per_dst=[2464, 3078, 1208, 1910, 2020, 5562, 3552, 2190] combine_rows_per_src=[8930, 10076, 8312, 8712, 8760, 10882, 10928, 9720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=352264 combine_payload_bytes=284409856 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284762120 ingress_bytes=353615872 link_bytes=353615872 token_bytes=569171976 weight_movement_bytes=69206016 total_nvlink_bytes=638377992 weight_to_token_ratio=0.121591 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330628058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275095552 dispatch_meta_bytes=341304 combine_payload_bytes=275095552 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275436856 ingress_bytes=344301568 link_bytes=344301568 token_bytes=550532408 weight_movement_bytes=69206016 total_nvlink_bytes=619738424 weight_to_token_ratio=0.125707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295579082e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269123584 dispatch_meta_bytes=326776 combine_payload_bytes=269123584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269450360 ingress_bytes=372932608 link_bytes=372932608 token_bytes=538573944 weight_movement_bytes=103809024 total_nvlink_bytes=642382968 weight_to_token_ratio=0.192748 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403315380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278224896 dispatch_meta_bytes=344992 combine_payload_bytes=278224896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278569888 ingress_bytes=382033920 link_bytes=382033920 token_bytes=556794784 weight_movement_bytes=103809024 total_nvlink_bytes=660603808 weight_to_token_ratio=0.186440 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437562885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272793600 dispatch_meta_bytes=345576 combine_payload_bytes=272793600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273139176 ingress_bytes=307396608 link_bytes=307396608 token_bytes=545932776 weight_movement_bytes=34603008 total_nvlink_bytes=580535784 weight_to_token_ratio=0.063383 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156708688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73618 payload_rows_remote=64124 dup_rows=24686 dup_rows_remote=21338 dedup_ratio=0.251119 router_tokens_per_expert=[461, 721, 515, 389, 166, 1018, 1009, 265, 715, 336, 839, 957, 1482, 1300, 1113, 1834, 911, 1051, 238, 456, 309, 582, 1177, 843, 210, 743, 958, 578, 482, 568, 702, 416, 396, 932, 975, 627, 1310, 1320, 1932, 652, 368, 410, 385, 249, 1721, 1139, 174, 1617, 1157, 443, 659, 847, 236, 485, 315, 959, 1211, 360, 112, 881, 255, 1215, 799, 1667] payload_rows_per_dst=[7374, 8906, 9970, 9350, 8252, 9852, 9494, 10420] dup_rows_per_dst=[1714, 2682, 4128, 2892, 2468, 4874, 3348, 2580] combine_rows_per_src=[7374, 8906, 9970, 9350, 8252, 9852, 9494, 10420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73140 payload_rows_remote=64364 dup_rows=25164 dup_rows_remote=22258 dedup_ratio=0.255981 router_tokens_per_expert=[315, 735, 663, 310, 128, 1196, 935, 229, 819, 339, 777, 1068, 1511, 1000, 623, 1761, 923, 1343, 305, 375, 195, 361, 1219, 750, 394, 670, 872, 596, 358, 501, 772, 422, 507, 952, 1212, 717, 1848, 1705, 2009, 605, 351, 262, 275, 297, 1562, 1217, 180, 1619, 1051, 201, 985, 1083, 203, 370, 219, 1005, 1334, 555, 75, 557, 169, 1101, 488, 1973] payload_rows_per_dst=[9022, 7604, 9814, 9740, 8776, 8604, 9708, 9872] dup_rows_per_dst=[3522, 1648, 4150, 3448, 2906, 2922, 3936, 2632] combine_rows_per_src=[9022, 7604, 9814, 9740, 8776, 8604, 9708, 9872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77054 payload_rows_remote=66126 dup_rows=21250 dup_rows_remote=18152 dedup_ratio=0.216166 router_tokens_per_expert=[737, 684, 1132, 566, 907, 295, 356, 1038, 225, 1180, 961, 394, 589, 1194, 1038, 1298, 1381, 1130, 892, 331, 1297, 84, 1475, 937, 534, 552, 552, 625, 124, 260, 262, 111, 442, 264, 819, 334, 1008, 2425, 1051, 1092, 2148, 1426, 1581, 1610, 506, 591, 61, 186, 671, 1315, 583, 94, 201, 1470, 208, 1188, 695, 342, 814, 668, 897, 623, 340, 358] payload_rows_per_dst=[10928, 8320, 11712, 10412, 9730, 7440, 9548, 8964] dup_rows_per_dst=[3098, 2842, 4418, 3144, 2816, 1262, 1912, 1758] combine_rows_per_src=[10928, 8320, 11712, 10412, 9730, 7440, 9548, 8964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73666 payload_rows_remote=63846 dup_rows=24638 dup_rows_remote=21404 dedup_ratio=0.250631 router_tokens_per_expert=[908, 535, 588, 324, 362, 1040, 955, 240, 937, 290, 655, 887, 1247, 1237, 1274, 2219, 775, 883, 187, 349, 161, 1220, 923, 606, 124, 611, 695, 281, 391, 527, 618, 702, 582, 937, 1072, 839, 919, 1202, 2336, 679, 243, 522, 288, 126, 1692, 1089, 198, 1570, 1183, 474, 997, 700, 459, 756, 351, 1236, 986, 390, 54, 601, 509, 822, 987, 1602] payload_rows_per_dst=[9700, 9820, 8648, 9282, 8810, 8608, 8792, 10006] dup_rows_per_dst=[4642, 3234, 1560, 3288, 3650, 2848, 3520, 1896] combine_rows_per_src=[9700, 9820, 8648, 9282, 8810, 8608, 8792, 10006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73746 payload_rows_remote=64338 dup_rows=24558 dup_rows_remote=21264 dedup_ratio=0.249817 router_tokens_per_expert=[556, 763, 556, 355, 219, 1329, 962, 221, 978, 309, 838, 1002, 1518, 1176, 1019, 1959, 910, 1135, 290, 396, 153, 625, 1056, 898, 275, 616, 890, 440, 403, 499, 671, 526, 520, 890, 1104, 692, 1375, 1464, 2031, 603, 307, 453, 313, 211, 1529, 1099, 185, 1461, 991, 377, 1185, 877, 254, 530, 315, 982, 1035, 427, 65, 599, 358, 1047, 748, 1582] payload_rows_per_dst=[9788, 9404, 9494, 9408, 9258, 8424, 8308, 9662] dup_rows_per_dst=[4052, 2994, 2714, 3294, 3998, 2692, 2754, 2060] combine_rows_per_src=[9788, 9404, 9494, 9408, 9258, 8424, 8308, 9662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65386 dup_rows=23988 dup_rows_remote=20890 dedup_ratio=0.244019 router_tokens_per_expert=[589, 630, 567, 368, 220, 1078, 997, 194, 981, 306, 765, 951, 1362, 1275, 1069, 1996, 857, 1062, 267, 382, 191, 816, 1163, 867, 227, 646, 885, 403, 446, 515, 660, 524, 478, 862, 1037, 777, 1333, 1152, 2043, 606, 306, 526, 347, 216, 1641, 1076, 231, 1586, 1024, 459, 1043, 840, 235, 468, 445, 1087, 1026, 327, 54, 630, 535, 989, 857, 1657] payload_rows_per_dst=[8692, 9512, 9830, 9528, 7980, 8930, 9786, 10058] dup_rows_per_dst=[3010, 2588, 4104, 3170, 2206, 3098, 3720, 2092] combine_rows_per_src=[8692, 9512, 9830, 9528, 7980, 8930, 9786, 10058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262651904 dispatch_meta_bytes=341848 combine_payload_bytes=262651904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262993752 ingress_bytes=297254912 link_bytes=297254912 token_bytes=525645656 weight_movement_bytes=34603008 total_nvlink_bytes=560248664 weight_to_token_ratio=0.065830 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118546303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263634944 dispatch_meta_bytes=346488 combine_payload_bytes=263634944 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=263981432 ingress_bytes=402046976 link_bytes=402046976 token_bytes=527616376 weight_movement_bytes=138412032 total_nvlink_bytes=666028408 weight_to_token_ratio=0.262335 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.512870403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73862 payload_rows_remote=65356 dup_rows=24442 dup_rows_remote=22626 dedup_ratio=0.248637 router_tokens_per_expert=[510, 757, 699, 342, 218, 1185, 844, 196, 1003, 312, 819, 965, 1595, 988, 808, 1636, 910, 1051, 260, 422, 192, 574, 964, 788, 403, 708, 983, 537, 380, 440, 692, 553, 509, 1017, 1240, 754, 1463, 1599, 2086, 582, 327, 454, 360, 167, 1607, 1276, 171, 1552, 976, 310, 999, 800, 413, 465, 281, 1010, 1265, 566, 86, 503, 287, 985, 634, 1674] payload_rows_per_dst=[9264, 9736, 8506, 9928, 9778, 8836, 7924, 9890] dup_rows_per_dst=[3510, 3244, 1816, 3636, 4550, 2992, 2584, 2110] combine_rows_per_src=[9264, 9736, 8506, 9928, 9778, 8836, 7924, 9890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270852096 dispatch_meta_bytes=337112 combine_payload_bytes=270852096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271189208 ingress_bytes=305455104 link_bytes=305455104 token_bytes=542041304 weight_movement_bytes=34603008 total_nvlink_bytes=576644312 weight_to_token_ratio=0.063838 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149402965e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73726 payload_rows_remote=63186 dup_rows=24578 dup_rows_remote=22010 dedup_ratio=0.250020 router_tokens_per_expert=[845, 676, 526, 276, 287, 1110, 1144, 251, 855, 341, 765, 842, 1373, 1400, 1186, 2185, 828, 912, 273, 423, 247, 873, 964, 791, 154, 646, 692, 331, 412, 534, 683, 799, 540, 785, 996, 788, 1131, 1188, 2086, 676, 263, 506, 292, 180, 1609, 994, 178, 1714, 986, 514, 958, 666, 384, 508, 300, 1106, 1069, 362, 88, 796, 343, 949, 1008, 1565] payload_rows_per_dst=[7934, 9478, 9508, 7218, 9640, 9884, 9524, 10540] dup_rows_per_dst=[2296, 2870, 3860, 1284, 3616, 4388, 3696, 2568] combine_rows_per_src=[7934, 9478, 9508, 7218, 9640, 9884, 9524, 10540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261513216 dispatch_meta_bytes=341000 combine_payload_bytes=261513216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261854216 ingress_bytes=399925248 link_bytes=399925248 token_bytes=523367432 weight_movement_bytes=138412032 total_nvlink_bytes=661779464 weight_to_token_ratio=0.264464 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504886512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263528448 dispatch_meta_bytes=342408 combine_payload_bytes=263528448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263870856 ingress_bytes=298131456 link_bytes=298131456 token_bytes=527399304 weight_movement_bytes=34603008 total_nvlink_bytes=562002312 weight_to_token_ratio=0.065611 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121844667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267821056 dispatch_meta_bytes=345104 combine_payload_bytes=267821056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268166160 ingress_bytes=302424064 link_bytes=302424064 token_bytes=535987216 weight_movement_bytes=34603008 total_nvlink_bytes=570590224 weight_to_token_ratio=0.064559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137997406e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267698176 dispatch_meta_bytes=351928 combine_payload_bytes=267698176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050104 ingress_bytes=302301184 link_bytes=302301184 token_bytes=535748280 weight_movement_bytes=34603008 total_nvlink_bytes=570351288 weight_to_token_ratio=0.064588 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137535018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258809856 dispatch_meta_bytes=340784 combine_payload_bytes=258809856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259150640 ingress_bytes=293412864 link_bytes=293412864 token_bytes=517960496 weight_movement_bytes=34603008 total_nvlink_bytes=552563504 weight_to_token_ratio=0.066806 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104088986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75416 payload_rows_remote=66348 dup_rows=22888 dup_rows_remote=19850 dedup_ratio=0.232829 router_tokens_per_expert=[758, 1644, 1192, 857, 394, 293, 734, 488, 963, 1682, 296, 430, 644, 636, 669, 463, 1390, 106, 1952, 574, 426, 312, 129, 829, 495, 400, 634, 581, 600, 653, 1200, 1154, 259, 803, 341, 410, 447, 500, 2149, 771, 804, 1556, 184, 586, 359, 582, 1113, 2765, 1089, 525, 593, 856, 597, 834, 621, 938, 344, 740, 979, 730, 936, 668, 835, 660] payload_rows_per_dst=[8120, 9480, 8652, 9464, 8526, 10990, 9068, 11116] dup_rows_per_dst=[1312, 2086, 2784, 1970, 2834, 4908, 3038, 3956] combine_rows_per_src=[8120, 9480, 8652, 9464, 8526, 10990, 9068, 11116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75006 payload_rows_remote=67002 dup_rows=23298 dup_rows_remote=21194 dedup_ratio=0.237000 router_tokens_per_expert=[650, 993, 1323, 493, 537, 514, 784, 659, 1700, 2424, 440, 308, 1117, 619, 954, 430, 1570, 57, 2400, 533, 574, 344, 44, 779, 479, 604, 662, 499, 650, 721, 1086, 882, 518, 744, 351, 293, 509, 421, 1530, 688, 663, 2541, 115, 524, 274, 504, 671, 2906, 325, 404, 479, 986, 339, 805, 401, 544, 234, 654, 1274, 712, 804, 826, 694, 590] payload_rows_per_dst=[9376, 11716, 9314, 10546, 8004, 9792, 7042, 9216] dup_rows_per_dst=[2456, 4268, 3288, 3336, 2104, 3888, 1524, 2434] combine_rows_per_src=[9376, 11716, 9314, 10546, 8004, 9792, 7042, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73710 payload_rows_remote=64144 dup_rows=24594 dup_rows_remote=20376 dedup_ratio=0.250183 router_tokens_per_expert=[717, 662, 596, 325, 355, 1126, 875, 224, 944, 280, 682, 857, 1496, 1365, 1160, 2012, 845, 967, 198, 469, 148, 870, 897, 707, 159, 633, 669, 292, 367, 532, 625, 796, 481, 949, 1093, 886, 1177, 1324, 2346, 806, 242, 518, 343, 149, 1704, 1042, 170, 1508, 852, 518, 1030, 851, 340, 568, 261, 1185, 1066, 416, 77, 609, 354, 826, 911, 1700] payload_rows_per_dst=[9566, 10398, 8690, 9310, 8926, 8628, 8262, 9930] dup_rows_per_dst=[4218, 4186, 1512, 3528, 3490, 2724, 2948, 1988] combine_rows_per_src=[9566, 10398, 8690, 9310, 8926, 8628, 8262, 9930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=66990 dup_rows=22786 dup_rows_remote=21328 dedup_ratio=0.231791 router_tokens_per_expert=[1370, 2133, 915, 863, 263, 217, 723, 476, 809, 1435, 275, 342, 487, 644, 526, 475, 886, 49, 1294, 717, 424, 169, 179, 744, 227, 261, 650, 541, 489, 383, 821, 1272, 256, 916, 582, 474, 295, 486, 1661, 717, 1394, 1457, 163, 608, 350, 644, 2188, 2224, 2195, 530, 1037, 595, 434, 871, 1096, 1634, 193, 1015, 829, 917, 802, 573, 716, 241] payload_rows_per_dst=[10926, 8528, 9740, 8128, 10014, 8648, 11012, 8522] dup_rows_per_dst=[2994, 1458, 3520, 1272, 3674, 2046, 5772, 2050] combine_rows_per_src=[10926, 8528, 9740, 8128, 10014, 8648, 11012, 8522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74714 payload_rows_remote=63236 dup_rows=23590 dup_rows_remote=18824 dedup_ratio=0.239970 router_tokens_per_expert=[884, 1285, 1228, 510, 497, 418, 768, 731, 1169, 2108, 387, 376, 848, 635, 722, 368, 1237, 135, 1883, 807, 586, 242, 93, 867, 454, 527, 660, 503, 622, 564, 933, 1053, 686, 811, 355, 408, 460, 458, 1485, 679, 1001, 2112, 142, 515, 386, 589, 969, 2806, 856, 481, 610, 930, 372, 939, 640, 874, 277, 702, 1112, 749, 853, 762, 682, 351] payload_rows_per_dst=[9674, 10230, 8806, 11478, 9464, 7380, 8664, 9018] dup_rows_per_dst=[2968, 2996, 2894, 4766, 3274, 1994, 2740, 1958] combine_rows_per_src=[9674, 10230, 8806, 11478, 9464, 7380, 8664, 9018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75526 payload_rows_remote=64634 dup_rows=22778 dup_rows_remote=17408 dedup_ratio=0.231710 router_tokens_per_expert=[860, 1574, 1086, 871, 353, 338, 806, 615, 1232, 1702, 344, 372, 594, 686, 686, 437, 1027, 75, 1738, 731, 454, 276, 115, 730, 362, 466, 627, 516, 621, 513, 976, 1280, 595, 852, 419, 420, 392, 461, 1801, 652, 752, 1731, 145, 565, 396, 560, 1406, 2576, 1343, 494, 668, 814, 395, 853, 928, 1197, 264, 991, 895, 816, 843, 703, 774, 388] payload_rows_per_dst=[8492, 9894, 7900, 8984, 8608, 10892, 9586, 11170] dup_rows_per_dst=[1366, 2212, 2392, 1738, 2576, 5370, 3798, 3326] combine_rows_per_src=[8492, 9894, 7900, 8984, 8608, 10892, 9586, 11170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271761408 dispatch_meta_bytes=344792 combine_payload_bytes=271761408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272106200 ingress_bytes=340967424 link_bytes=340967424 token_bytes=543867608 weight_movement_bytes=69206016 total_nvlink_bytes=613073624 weight_to_token_ratio=0.127248 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283032966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75034 payload_rows_remote=66148 dup_rows=23270 dup_rows_remote=20454 dedup_ratio=0.236715 router_tokens_per_expert=[1046, 1183, 1488, 464, 450, 640, 846, 759, 1493, 2224, 491, 337, 1035, 578, 929, 270, 1241, 64, 2020, 769, 567, 280, 98, 812, 385, 637, 637, 453, 458, 642, 1047, 861, 571, 773, 382, 344, 387, 448, 1254, 658, 913, 2741, 152, 526, 320, 544, 896, 2694, 563, 453, 616, 906, 344, 846, 526, 649, 204, 736, 1192, 731, 779, 852, 596, 352] payload_rows_per_dst=[10444, 10998, 8886, 11202, 10884, 5944, 7664, 9012] dup_rows_per_dst=[3308, 3716, 2816, 4426, 4232, 758, 2142, 1872] combine_rows_per_src=[10444, 10998, 8886, 11202, 10884, 5944, 7664, 9012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274440192 dispatch_meta_bytes=352784 combine_payload_bytes=274440192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274792976 ingress_bytes=309043200 link_bytes=309043200 token_bytes=549233168 weight_movement_bytes=34603008 total_nvlink_bytes=583836176 weight_to_token_ratio=0.063002 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162904682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262733824 dispatch_meta_bytes=338080 combine_payload_bytes=262733824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263071904 ingress_bytes=297336832 link_bytes=297336832 token_bytes=525805728 weight_movement_bytes=34603008 total_nvlink_bytes=560408736 weight_to_token_ratio=0.065809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118854561e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274391040 dispatch_meta_bytes=353272 combine_payload_bytes=274391040 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=274744312 ingress_bytes=308994048 link_bytes=308994048 token_bytes=549135352 weight_movement_bytes=34603008 total_nvlink_bytes=583738360 weight_to_token_ratio=0.063014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162719727e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=63182 dup_rows=24030 dup_rows_remote=20580 dedup_ratio=0.244446 router_tokens_per_expert=[1056, 1728, 937, 869, 316, 256, 802, 587, 910, 1507, 323, 343, 552, 618, 573, 424, 1112, 89, 1527, 762, 478, 246, 210, 857, 269, 415, 699, 670, 678, 561, 1013, 1131, 538, 751, 431, 404, 315, 572, 1675, 715, 955, 1598, 169, 613, 413, 641, 1660, 2504, 1459, 523, 818, 754, 556, 931, 797, 1299, 173, 893, 1063, 772, 964, 597, 786, 295] payload_rows_per_dst=[8362, 8580, 8132, 8938, 8378, 10682, 10110, 11092] dup_rows_per_dst=[1284, 1726, 2430, 1934, 2424, 5792, 4990, 3450] combine_rows_per_src=[8362, 8580, 8132, 8938, 8378, 10682, 10110, 11092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259014656 dispatch_meta_bytes=328240 combine_payload_bytes=259014656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259342896 ingress_bytes=293617664 link_bytes=293617664 token_bytes=518357552 weight_movement_bytes=34603008 total_nvlink_bytes=552960560 weight_to_token_ratio=0.066755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104859631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=328168 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265069032 ingress_bytes=437755904 link_bytes=437755904 token_bytes=529809896 weight_movement_bytes=173015040 total_nvlink_bytes=702824936 weight_to_token_ratio=0.326561 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.647240225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270942208 dispatch_meta_bytes=346408 combine_payload_bytes=270942208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271288616 ingress_bytes=305545216 link_bytes=305545216 token_bytes=542230824 weight_movement_bytes=34603008 total_nvlink_bytes=576833832 weight_to_token_ratio=0.063816 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149742049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=258793472 dispatch_meta_bytes=335048 combine_payload_bytes=258793472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259128520 ingress_bytes=293396480 link_bytes=293396480 token_bytes=517921992 weight_movement_bytes=34603008 total_nvlink_bytes=552525000 weight_to_token_ratio=0.066811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104027334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73128 payload_rows_remote=63500 dup_rows=25176 dup_rows_remote=21842 dedup_ratio=0.256104 router_tokens_per_expert=[476, 1467, 1577, 170, 891, 1132, 1678, 373, 622, 168, 697, 307, 765, 357, 1098, 1275, 1703, 337, 693, 322, 170, 201, 181, 144, 207, 523, 593, 628, 756, 732, 726, 1123, 922, 1252, 896, 236, 816, 1711, 426, 938, 445, 324, 2137, 735, 831, 550, 1399, 752, 906, 205, 167, 2449, 760, 657, 545, 792, 786, 243, 455, 563, 632, 1743, 839, 948] payload_rows_per_dst=[8924, 8174, 10338, 8230, 8288, 10740, 9628, 8806] dup_rows_per_dst=[3516, 2404, 3674, 2346, 2684, 3606, 3334, 3612] combine_rows_per_src=[8924, 8174, 10338, 8230, 8288, 10740, 9628, 8806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71592 payload_rows_remote=64672 dup_rows=26712 dup_rows_remote=25284 dedup_ratio=0.271729 router_tokens_per_expert=[12, 871, 1682, 819, 1068, 976, 899, 1440, 422, 510, 1834, 151, 297, 480, 473, 1072, 1817, 152, 153, 83, 130, 385, 309, 220, 440, 426, 1548, 422, 1734, 1011, 751, 184, 694, 92, 230, 865, 1105, 642, 179, 1009, 387, 320, 320, 895, 804, 41, 2288, 26, 261, 693, 230, 2375, 590, 1142, 907, 1159, 791, 398, 335, 1244, 1058, 2166, 1511, 1624] payload_rows_per_dst=[8468, 9030, 8900, 9822, 6920, 7168, 10276, 11008] dup_rows_per_dst=[3188, 3074, 2760, 3210, 1428, 1368, 4438, 7246] combine_rows_per_src=[8468, 9030, 8900, 9822, 6920, 7168, 10276, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75516 payload_rows_remote=63184 dup_rows=22788 dup_rows_remote=17898 dedup_ratio=0.231812 router_tokens_per_expert=[1575, 2054, 950, 870, 288, 241, 784, 497, 781, 1453, 257, 424, 695, 604, 487, 507, 1017, 60, 1658, 782, 490, 254, 213, 930, 363, 428, 649, 605, 489, 430, 996, 1101, 201, 872, 385, 439, 369, 402, 1781, 610, 1260, 1595, 178, 583, 362, 533, 1694, 2971, 1641, 469, 718, 735, 447, 962, 809, 1148, 192, 589, 985, 736, 925, 594, 753, 282] payload_rows_per_dst=[12332, 8706, 11004, 8578, 10048, 7824, 8544, 8480] dup_rows_per_dst=[4890, 1710, 5746, 1544, 3260, 1396, 2610, 1632] combine_rows_per_src=[12332, 8706, 11004, 8578, 10048, 7824, 8544, 8480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71482 payload_rows_remote=63440 dup_rows=26822 dup_rows_remote=24166 dedup_ratio=0.272847 router_tokens_per_expert=[965, 1010, 1336, 87, 503, 401, 2086, 224, 354, 108, 291, 167, 834, 214, 1259, 872, 1406, 756, 1396, 169, 141, 204, 150, 66, 130, 697, 207, 437, 340, 326, 418, 2978, 1512, 2316, 1833, 104, 498, 1655, 222, 837, 329, 143, 2164, 635, 897, 1711, 1250, 2641, 1245, 42, 142, 2284, 775, 568, 355, 532, 398, 66, 564, 184, 367, 1136, 274, 941] payload_rows_per_dst=[9830, 8042, 8906, 10250, 11728, 12286, 6054, 4386] dup_rows_per_dst=[4032, 2656, 2342, 3088, 6444, 5794, 1264, 1202] combine_rows_per_src=[9830, 8042, 8906, 10250, 11728, 12286, 6054, 4386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260096000 dispatch_meta_bytes=341368 combine_payload_bytes=260096000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=260437368 ingress_bytes=363905024 link_bytes=363905024 token_bytes=520533368 weight_movement_bytes=103809024 total_nvlink_bytes=624342392 weight_to_token_ratio=0.199428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369345309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71776 payload_rows_remote=62568 dup_rows=26528 dup_rows_remote=23244 dedup_ratio=0.269857 router_tokens_per_expert=[103, 1165, 1530, 435, 1137, 899, 1197, 723, 399, 356, 1441, 205, 537, 488, 864, 1397, 1398, 362, 286, 122, 191, 284, 297, 295, 413, 411, 893, 535, 1321, 1007, 716, 547, 950, 465, 606, 584, 995, 1085, 272, 1214, 347, 254, 896, 797, 1151, 202, 1891, 216, 697, 375, 203, 2378, 591, 917, 604, 1107, 865, 339, 531, 977, 871, 2094, 1468, 1256] payload_rows_per_dst=[8524, 10456, 8880, 9208, 7844, 6556, 9984, 10324] dup_rows_per_dst=[3076, 4700, 2538, 3284, 2328, 1170, 3760, 5672] combine_rows_per_src=[8524, 10456, 8880, 9208, 7844, 6556, 9984, 10324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72616 payload_rows_remote=62100 dup_rows=25688 dup_rows_remote=22240 dedup_ratio=0.261312 router_tokens_per_expert=[329, 1137, 1577, 292, 797, 683, 1652, 440, 384, 268, 742, 227, 765, 395, 1026, 1490, 1436, 400, 758, 243, 205, 263, 188, 213, 335, 520, 719, 595, 594, 632, 562, 1191, 1149, 1075, 1043, 259, 981, 1571, 356, 1012, 410, 166, 1650, 761, 1094, 689, 1501, 711, 1089, 147, 200, 2471, 777, 642, 492, 831, 741, 217, 692, 635, 665, 2029, 966, 1072] payload_rows_per_dst=[7902, 8138, 9904, 8182, 8644, 10516, 9838, 9492] dup_rows_per_dst=[2596, 2456, 3966, 2114, 3106, 3448, 3460, 4542] combine_rows_per_src=[7902, 8138, 9904, 8182, 8644, 10516, 9838, 9492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72526 payload_rows_remote=63820 dup_rows=25778 dup_rows_remote=24158 dedup_ratio=0.262227 router_tokens_per_expert=[38, 705, 1710, 872, 1081, 553, 1021, 1094, 396, 455, 1746, 169, 397, 533, 686, 1157, 1571, 317, 234, 81, 153, 416, 372, 309, 436, 368, 1082, 311, 1557, 861, 597, 393, 1007, 221, 464, 881, 980, 908, 218, 1173, 348, 352, 427, 822, 976, 54, 2129, 89, 499, 701, 217, 2422, 606, 1183, 832, 1165, 915, 309, 448, 1212, 875, 2037, 1445, 1566] payload_rows_per_dst=[9176, 10692, 8706, 10746, 7830, 5614, 10560, 9202] dup_rows_per_dst=[3368, 4644, 1620, 4538, 2058, 842, 4370, 4338] combine_rows_per_src=[9176, 10692, 8706, 10746, 7830, 5614, 10560, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=359824 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265256336 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530152848 weight_movement_bytes=103809024 total_nvlink_bytes=633961872 weight_to_token_ratio=0.195810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=258801664 dispatch_meta_bytes=324328 combine_payload_bytes=258801664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259125992 ingress_bytes=328007680 link_bytes=328007680 token_bytes=517927656 weight_movement_bytes=69206016 total_nvlink_bytes=587133672 weight_to_token_ratio=0.133621 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234266493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259850240 dispatch_meta_bytes=350424 combine_payload_bytes=259850240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260200664 ingress_bytes=294453248 link_bytes=294453248 token_bytes=520050904 weight_movement_bytes=34603008 total_nvlink_bytes=554653912 weight_to_token_ratio=0.066538 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108003867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72924 payload_rows_remote=64822 dup_rows=25380 dup_rows_remote=21980 dedup_ratio=0.258179 router_tokens_per_expert=[481, 1056, 1450, 276, 676, 1021, 2027, 393, 390, 202, 581, 226, 1149, 357, 995, 1442, 1398, 496, 758, 167, 235, 162, 174, 148, 206, 564, 504, 546, 655, 605, 619, 1426, 1275, 1293, 1686, 150, 530, 1734, 334, 752, 391, 163, 1920, 713, 1027, 1089, 1365, 1117, 1055, 153, 178, 2479, 779, 603, 445, 785, 726, 183, 573, 484, 744, 1293, 714, 1034] payload_rows_per_dst=[8514, 8138, 10208, 8024, 8884, 11378, 9676, 8102] dup_rows_per_dst=[2760, 2546, 3822, 2226, 3156, 4192, 3278, 3400] combine_rows_per_src=[8514, 8138, 10208, 8024, 8884, 11378, 9676, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256278528 dispatch_meta_bytes=343248 combine_payload_bytes=256278528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256621776 ingress_bytes=290881536 link_bytes=290881536 token_bytes=512900304 weight_movement_bytes=34603008 total_nvlink_bytes=547503312 weight_to_token_ratio=0.067465 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094563802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261406720 dispatch_meta_bytes=351912 combine_payload_bytes=261406720 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261758632 ingress_bytes=330612736 link_bytes=330612736 token_bytes=523165352 weight_movement_bytes=69206016 total_nvlink_bytes=592371368 weight_to_token_ratio=0.132283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244069109e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254361600 dispatch_meta_bytes=337360 combine_payload_bytes=254361600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254698960 ingress_bytes=323567616 link_bytes=323567616 token_bytes=509060560 weight_movement_bytes=69206016 total_nvlink_bytes=578266576 weight_to_token_ratio=0.135948 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217558890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=347208 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265858120 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531369032 weight_movement_bytes=69206016 total_nvlink_bytes=600575048 weight_to_token_ratio=0.130241 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76898 payload_rows_remote=67056 dup_rows=21406 dup_rows_remote=18574 dedup_ratio=0.217753 router_tokens_per_expert=[848, 962, 1312, 147, 630, 166, 785, 190, 447, 812, 1655, 1070, 792, 12, 188, 1601, 692, 619, 768, 113, 648, 129, 260, 1833, 1020, 602, 788, 636, 823, 653, 1378, 473, 674, 1284, 1051, 2768, 850, 1113, 1431, 572, 407, 796, 254, 1545, 527, 289, 263, 645, 1135, 1035, 525, 228, 562, 713, 609, 1530, 168, 598, 517, 1157, 234, 238, 2079, 303] payload_rows_per_dst=[7988, 10070, 10172, 10366, 7134, 11266, 9842, 10060] dup_rows_per_dst=[2092, 3084, 2178, 2380, 1728, 3722, 2832, 3390] combine_rows_per_src=[7988, 10070, 10172, 10366, 7134, 11266, 9842, 10060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77452 payload_rows_remote=68582 dup_rows=20852 dup_rows_remote=18242 dedup_ratio=0.212118 router_tokens_per_expert=[456, 1412, 1069, 318, 254, 98, 2595, 432, 23, 2, 1104, 824, 1661, 0, 31, 856, 585, 1046, 2751, 37, 372, 612, 265, 15, 202, 2056, 722, 112, 624, 340, 3760, 534, 1293, 593, 278, 972, 1049, 319, 1443, 1084, 104, 0, 172, 1919, 56, 484, 1644, 419, 879, 1219, 1146, 759, 125, 856, 0, 1178, 991, 434, 506, 1166, 162, 428, 1270, 1036] payload_rows_per_dst=[10152, 8156, 9534, 12384, 8870, 9042, 9682, 9632] dup_rows_per_dst=[3116, 920, 2470, 4242, 2610, 2498, 2642, 2354] combine_rows_per_src=[10152, 8156, 9534, 12384, 8870, 9042, 9682, 9632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77732 payload_rows_remote=67140 dup_rows=20572 dup_rows_remote=16696 dedup_ratio=0.209269 router_tokens_per_expert=[779, 244, 938, 50, 223, 73, 123, 211, 352, 2432, 1302, 860, 192, 53, 836, 1207, 469, 460, 199, 17, 526, 65, 111, 2945, 2620, 219, 494, 1171, 609, 683, 483, 281, 265, 1200, 1357, 3482, 625, 2964, 522, 229, 157, 2467, 379, 1019, 360, 197, 133, 641, 636, 654, 276, 174, 503, 664, 2502, 1127, 27, 645, 555, 1203, 60, 190, 2562, 150] payload_rows_per_dst=[5282, 10592, 11920, 10000, 11484, 8812, 10826, 8816] dup_rows_per_dst=[966, 3876, 3592, 2154, 3876, 1894, 2246, 1968] combine_rows_per_src=[5282, 10592, 11920, 10000, 11484, 8812, 10826, 8816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72230 payload_rows_remote=62134 dup_rows=26074 dup_rows_remote=21740 dedup_ratio=0.265238 router_tokens_per_expert=[630, 1222, 1642, 233, 656, 521, 1838, 307, 552, 245, 451, 217, 934, 315, 1097, 1357, 1766, 900, 828, 149, 159, 171, 137, 118, 313, 513, 327, 541, 435, 380, 616, 2000, 1089, 1602, 1628, 120, 719, 1729, 339, 770, 367, 282, 1855, 688, 1103, 1102, 1409, 1416, 1031, 310, 158, 2293, 783, 574, 426, 626, 709, 98, 519, 382, 577, 1393, 567, 918] payload_rows_per_dst=[10096, 9528, 9166, 9586, 11380, 10404, 6514, 5556] dup_rows_per_dst=[4334, 3626, 2574, 3450, 5582, 3222, 1870, 1416] combine_rows_per_src=[10096, 9528, 9166, 9586, 11380, 10404, 6514, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274661376 dispatch_meta_bytes=342520 combine_payload_bytes=274661376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275003896 ingress_bytes=309264384 link_bytes=309264384 token_bytes=549665272 weight_movement_bytes=34603008 total_nvlink_bytes=584268280 weight_to_token_ratio=0.062953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163736979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75364 payload_rows_remote=66892 dup_rows=22940 dup_rows_remote=20846 dedup_ratio=0.233358 router_tokens_per_expert=[510, 1258, 1007, 233, 352, 29, 2652, 405, 94, 48, 1205, 778, 1371, 0, 61, 1008, 688, 1026, 2181, 53, 385, 415, 170, 31, 401, 1705, 705, 169, 725, 351, 3555, 475, 1417, 599, 410, 982, 1309, 334, 1091, 1334, 179, 14, 405, 1858, 70, 609, 1204, 471, 830, 1076, 956, 1040, 247, 945, 3, 1141, 918, 429, 466, 1280, 173, 487, 1725, 1104] payload_rows_per_dst=[12786, 7974, 8472, 8032, 9374, 8780, 9684, 10262] dup_rows_per_dst=[7216, 1156, 2094, 1030, 3432, 2318, 2792, 2902] combine_rows_per_src=[12786, 7974, 8472, 8032, 9374, 8780, 9684, 10262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=67554 dup_rows=22530 dup_rows_remote=21382 dedup_ratio=0.229187 router_tokens_per_expert=[670, 1062, 1143, 210, 492, 151, 2580, 340, 163, 178, 1422, 962, 1096, 1, 113, 1134, 675, 960, 2090, 95, 521, 223, 192, 274, 548, 1340, 708, 301, 828, 404, 2910, 555, 1056, 759, 529, 1292, 1223, 512, 1114, 995, 277, 66, 528, 1998, 151, 458, 957, 443, 900, 1090, 783, 620, 341, 1038, 37, 1270, 775, 457, 457, 1377, 197, 540, 1820, 751] payload_rows_per_dst=[11078, 9876, 8898, 8220, 8748, 9600, 9378, 9976] dup_rows_per_dst=[5106, 3194, 2186, 1148, 2604, 2740, 2780, 2772] combine_rows_per_src=[11078, 9876, 8898, 8220, 8748, 9600, 9378, 9976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77120 payload_rows_remote=66514 dup_rows=21184 dup_rows_remote=18238 dedup_ratio=0.215495 router_tokens_per_expert=[830, 1031, 1363, 175, 486, 128, 2047, 320, 302, 587, 1670, 1003, 760, 14, 205, 1372, 723, 726, 1118, 89, 547, 138, 164, 1140, 1303, 934, 683, 546, 971, 695, 1938, 563, 785, 1053, 859, 2132, 1017, 1020, 975, 772, 356, 446, 491, 1669, 280, 308, 421, 673, 824, 854, 575, 408, 422, 956, 324, 1251, 262, 562, 505, 1331, 132, 335, 2050, 533] payload_rows_per_dst=[9578, 9374, 9372, 11444, 8290, 10606, 9218, 9238] dup_rows_per_dst=[3182, 2452, 1958, 3822, 2450, 2946, 2010, 2364] combine_rows_per_src=[9578, 9374, 9372, 11444, 8290, 10606, 9218, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280911872 dispatch_meta_bytes=347296 combine_payload_bytes=280911872 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=281259168 ingress_bytes=419323904 link_bytes=419323904 token_bytes=562171040 weight_movement_bytes=138412032 total_nvlink_bytes=700583072 weight_to_token_ratio=0.246210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.577882092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=335344 combine_payload_bytes=275005440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275340784 ingress_bytes=309608448 link_bytes=309608448 token_bytes=550346224 weight_movement_bytes=34603008 total_nvlink_bytes=584949232 weight_to_token_ratio=0.062875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165031664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77164 payload_rows_remote=66728 dup_rows=21140 dup_rows_remote=17782 dedup_ratio=0.215047 router_tokens_per_expert=[823, 600, 1211, 130, 499, 176, 1143, 173, 395, 1075, 1781, 1095, 642, 22, 431, 1248, 616, 744, 969, 119, 597, 125, 179, 1628, 1461, 656, 621, 620, 810, 590, 1584, 452, 620, 1141, 1167, 2257, 1237, 1314, 980, 497, 307, 871, 433, 1508, 244, 355, 417, 545, 912, 897, 560, 272, 560, 998, 718, 1210, 224, 661, 495, 1193, 190, 282, 2411, 461] payload_rows_per_dst=[7632, 10084, 10114, 10558, 7904, 10582, 9854, 10436] dup_rows_per_dst=[1878, 3294, 2208, 2460, 1938, 3292, 2712, 3358] combine_rows_per_src=[7632, 10084, 10114, 10558, 7904, 10582, 9854, 10436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254500864 dispatch_meta_bytes=335496 combine_payload_bytes=254500864 prefetch_bytes=69206016 reduce_bytes=69206016 egress_bytes=254836360 ingress_bytes=392912896 link_bytes=392912896 token_bytes=509337224 weight_movement_bytes=138412032 total_nvlink_bytes=647749256 weight_to_token_ratio=0.271749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478499596e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273989632 dispatch_meta_bytes=350952 combine_payload_bytes=273989632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274340584 ingress_bytes=308592640 link_bytes=308592640 token_bytes=548330216 weight_movement_bytes=34603008 total_nvlink_bytes=582933224 weight_to_token_ratio=0.063106 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161209261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276701184 dispatch_meta_bytes=355744 combine_payload_bytes=276701184 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=277056928 ingress_bytes=415113216 link_bytes=415113216 token_bytes=553758112 weight_movement_bytes=138412032 total_nvlink_bytes=692170144 weight_to_token_ratio=0.249950 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.562037612e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272441344 dispatch_meta_bytes=339008 combine_payload_bytes=272441344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272780352 ingress_bytes=307044352 link_bytes=307044352 token_bytes=545221696 weight_movement_bytes=34603008 total_nvlink_bytes=579824704 weight_to_token_ratio=0.063466 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155383177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273317888 dispatch_meta_bytes=338040 combine_payload_bytes=273317888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273655928 ingress_bytes=307920896 link_bytes=307920896 token_bytes=546973816 weight_movement_bytes=34603008 total_nvlink_bytes=581576824 weight_to_token_ratio=0.063263 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158681542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73578 payload_rows_remote=61976 dup_rows=24726 dup_rows_remote=20188 dedup_ratio=0.251526 router_tokens_per_expert=[980, 448, 577, 215, 826, 1119, 488, 787, 797, 368, 307, 733, 559, 43, 112, 710, 16, 417, 891, 168, 2478, 536, 1236, 434, 347, 1334, 128, 506, 1420, 881, 348, 366, 294, 551, 250, 254, 535, 1737, 1544, 1201, 773, 546, 1527, 796, 439, 667, 173, 492, 394, 3452, 681, 103, 1856, 407, 1177, 2531, 424, 1421, 564, 556, 2382, 282, 176, 392] payload_rows_per_dst=[8718, 6252, 9634, 8780, 9098, 8138, 11602, 11356] dup_rows_per_dst=[2162, 1006, 2718, 1880, 3634, 2688, 4538, 6100] combine_rows_per_src=[8718, 6252, 9634, 8780, 9098, 8138, 11602, 11356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79418 payload_rows_remote=70620 dup_rows=18886 dup_rows_remote=17846 dedup_ratio=0.192118 router_tokens_per_expert=[696, 174, 0, 4118, 80, 16, 282, 104, 0, 511, 1131, 828, 1211, 473, 436, 1164, 2, 9, 682, 3, 382, 429, 451, 1818, 106, 184, 637, 0, 401, 505, 0, 6, 98, 1983, 315, 9, 0, 0, 0, 2514, 1240, 268, 230, 1504, 5323, 220, 866, 3812, 433, 187, 0, 1361, 326, 1303, 6043, 0, 3489, 8, 0, 3, 325, 1, 300, 152] payload_rows_per_dst=[9862, 13362, 6318, 13522, 8798, 12664, 6812, 8080] dup_rows_per_dst=[1078, 6206, 1234, 2242, 1040, 6202, 408, 476] combine_rows_per_src=[9862, 13362, 6318, 13522, 8798, 12664, 6812, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73196 payload_rows_remote=69886 dup_rows=25108 dup_rows_remote=24950 dedup_ratio=0.255412 router_tokens_per_expert=[772, 133, 777, 0, 2700, 1424, 597, 699, 556, 182, 53, 338, 484, 0, 8, 109, 54, 421, 149, 3, 3410, 455, 727, 94, 134, 957, 94, 306, 1096, 553, 201, 290, 181, 192, 44, 6, 613, 4527, 2649, 770, 341, 426, 1886, 496, 4, 237, 64, 184, 401, 2306, 692, 52, 1924, 195, 0, 5739, 0, 1685, 681, 405, 3918, 419, 92, 247] payload_rows_per_dst=[10258, 3310, 9532, 6082, 12076, 6152, 14102, 11684] dup_rows_per_dst=[3946, 158, 1462, 1180, 5888, 748, 8516, 3210] combine_rows_per_src=[10258, 3310, 9532, 6082, 12076, 6152, 14102, 11684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=253853696 dispatch_meta_bytes=328656 combine_payload_bytes=253853696 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=254182352 ingress_bytes=426868736 link_bytes=426868736 token_bytes=508036048 weight_movement_bytes=173015040 total_nvlink_bytes=681051088 weight_to_token_ratio=0.340557 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.606272688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75430 payload_rows_remote=64442 dup_rows=22874 dup_rows_remote=17574 dedup_ratio=0.232686 router_tokens_per_expert=[768, 395, 233, 1283, 714, 1003, 462, 346, 379, 401, 1003, 1096, 930, 167, 111, 987, 168, 392, 835, 22, 2925, 515, 878, 1018, 216, 851, 305, 422, 1354, 888, 96, 280, 258, 1066, 291, 120, 136, 1289, 637, 1743, 1095, 311, 975, 1149, 2120, 920, 230, 1344, 497, 2116, 264, 300, 826, 511, 2719, 1463, 1393, 957, 171, 169, 1632, 154, 287, 566] payload_rows_per_dst=[8574, 8392, 10306, 10302, 8536, 10988, 8578, 9754] dup_rows_per_dst=[1834, 1756, 3200, 2632, 2544, 5300, 2290, 3318] combine_rows_per_src=[8574, 8392, 10306, 10302, 8536, 10988, 8578, 9754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78552 payload_rows_remote=66962 dup_rows=19752 dup_rows_remote=15432 dedup_ratio=0.200928 router_tokens_per_expert=[673, 157, 2, 3933, 90, 113, 269, 102, 3, 503, 1353, 923, 1135, 617, 395, 1112, 17, 50, 561, 9, 1082, 447, 414, 2088, 102, 339, 531, 3, 565, 462, 1, 15, 113, 1895, 363, 14, 0, 15, 0, 2370, 1263, 255, 228, 1618, 5065, 401, 655, 3360, 398, 229, 0, 1209, 340, 1146, 5857, 3, 3445, 5, 0, 7, 317, 1, 331, 183] payload_rows_per_dst=[9606, 14310, 11590, 13268, 8426, 7096, 6092, 8164] dup_rows_per_dst=[1072, 7902, 4320, 2482, 1114, 1890, 558, 414] combine_rows_per_src=[9606, 14310, 11590, 13268, 8426, 7096, 6092, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76158 payload_rows_remote=64212 dup_rows=22146 dup_rows_remote=19032 dedup_ratio=0.225281 router_tokens_per_expert=[777, 400, 33, 3124, 123, 326, 322, 127, 36, 620, 1544, 1017, 1155, 496, 428, 1335, 56, 143, 727, 14, 2028, 561, 504, 2139, 166, 516, 500, 109, 846, 526, 11, 76, 159, 1687, 372, 59, 2, 154, 16, 2225, 1390, 258, 343, 1497, 4158, 601, 666, 2515, 473, 763, 7, 910, 411, 721, 4780, 47, 2789, 132, 6, 41, 519, 13, 304, 349] payload_rows_per_dst=[8856, 13718, 9552, 11946, 8080, 10482, 5882, 7642] dup_rows_per_dst=[1608, 7860, 2792, 3114, 1268, 4058, 782, 664] combine_rows_per_src=[8856, 13718, 9552, 11946, 8080, 10482, 5882, 7642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76846 payload_rows_remote=68634 dup_rows=21458 dup_rows_remote=19418 dedup_ratio=0.218282 router_tokens_per_expert=[681, 613, 1078, 127, 373, 192, 518, 139, 395, 1885, 1549, 975, 516, 36, 453, 1292, 694, 598, 398, 78, 667, 87, 151, 2373, 1669, 356, 640, 599, 925, 575, 1405, 268, 470, 1248, 1347, 3212, 963, 1522, 1152, 539, 191, 1482, 473, 1546, 368, 264, 230, 549, 820, 965, 370, 192, 524, 831, 1011, 1190, 85, 598, 497, 1420, 131, 186, 2233, 238] payload_rows_per_dst=[8212, 10680, 10718, 8526, 12126, 8262, 9548, 8774] dup_rows_per_dst=[2040, 3522, 2418, 1538, 5736, 1944, 2258, 2002] combine_rows_per_src=[8212, 10680, 10718, 8526, 12126, 8262, 9548, 8774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289259520 dispatch_meta_bytes=353864 combine_payload_bytes=289259520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289613384 ingress_bytes=323862528 link_bytes=323862528 token_bytes=578872904 weight_movement_bytes=34603008 total_nvlink_bytes=613475912 weight_to_token_ratio=0.059777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218668620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74654 payload_rows_remote=64380 dup_rows=23650 dup_rows_remote=19014 dedup_ratio=0.240580 router_tokens_per_expert=[931, 383, 546, 1203, 1407, 1116, 522, 591, 581, 340, 509, 890, 705, 142, 59, 827, 178, 358, 761, 65, 3259, 560, 865, 663, 271, 950, 137, 535, 1320, 950, 148, 202, 267, 891, 159, 80, 307, 1859, 1125, 1854, 917, 362, 1294, 766, 1484, 482, 283, 868, 560, 2201, 604, 353, 890, 459, 1658, 2217, 930, 736, 332, 391, 1912, 219, 140, 578] payload_rows_per_dst=[10144, 6996, 10582, 7646, 9508, 9346, 10158, 10274] dup_rows_per_dst=[3254, 1110, 2836, 1380, 3576, 3566, 3292, 4636] combine_rows_per_src=[10144, 6996, 10582, 7646, 9508, 9346, 10158, 10274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=286253056 dispatch_meta_bytes=379344 combine_payload_bytes=286253056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286632400 ingress_bytes=320856064 link_bytes=320856064 token_bytes=572885456 weight_movement_bytes=34603008 total_nvlink_bytes=607488464 weight_to_token_ratio=0.060401 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207355538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263954432 dispatch_meta_bytes=328064 combine_payload_bytes=263954432 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264282496 ingress_bytes=367763456 link_bytes=367763456 token_bytes=528236928 weight_movement_bytes=103809024 total_nvlink_bytes=632045952 weight_to_token_ratio=0.196520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383864278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=329576 combine_payload_bytes=274276352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274605928 ingress_bytes=308879360 link_bytes=308879360 token_bytes=548882280 weight_movement_bytes=34603008 total_nvlink_bytes=583485288 weight_to_token_ratio=0.063043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162288165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263012352 dispatch_meta_bytes=332976 combine_payload_bytes=263012352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263345328 ingress_bytes=297615360 link_bytes=297615360 token_bytes=526357680 weight_movement_bytes=34603008 total_nvlink_bytes=560960688 weight_to_token_ratio=0.065740 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119902640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281124864 dispatch_meta_bytes=352208 combine_payload_bytes=281124864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281477072 ingress_bytes=315727872 link_bytes=315727872 token_bytes=562601936 weight_movement_bytes=34603008 total_nvlink_bytes=597204944 weight_to_token_ratio=0.061505 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188058564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263700480 dispatch_meta_bytes=333576 combine_payload_bytes=263700480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264034056 ingress_bytes=298303488 link_bytes=298303488 token_bytes=527734536 weight_movement_bytes=34603008 total_nvlink_bytes=562337544 weight_to_token_ratio=0.065569 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122492010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76076 payload_rows_remote=65888 dup_rows=22228 dup_rows_remote=19140 dedup_ratio=0.226115 router_tokens_per_expert=[349, 469, 2083, 594, 2464, 716, 2356, 479, 87, 546, 1643, 631, 544, 1361, 596, 499, 589, 918, 2755, 1871, 367, 269, 568, 1169, 758, 179, 353, 101, 730, 537, 238, 875, 1249, 431, 811, 366, 75, 305, 724, 210, 791, 242, 418, 668, 705, 309, 39, 816, 502, 2371, 1151, 496, 731, 154, 403, 830, 189, 1149, 2464, 959, 321, 100, 1139, 340] payload_rows_per_dst=[9880, 9386, 11428, 9812, 7554, 6884, 10188, 10944] dup_rows_per_dst=[3274, 2428, 4448, 2658, 1924, 1092, 3088, 3316] combine_rows_per_src=[9880, 9386, 11428, 9812, 7554, 6884, 10188, 10944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80880 payload_rows_remote=69954 dup_rows=17424 dup_rows_remote=14738 dedup_ratio=0.177246 router_tokens_per_expert=[2155, 5053, 0, 212, 827, 91, 72, 18, 0, 325, 117, 154, 81, 343, 587, 594, 309, 1336, 0, 0, 435, 115, 3020, 1380, 2905, 3653, 1868, 369, 1, 241, 124, 228, 0, 325, 1480, 0, 80, 1127, 642, 132, 703, 70, 0, 60, 1298, 5931, 222, 0, 5493, 0, 1738, 867, 500, 80, 333, 2, 2, 433, 0, 0, 0, 526, 203, 292] payload_rows_per_dst=[6018, 13478, 5798, 13156, 10926, 13000, 6534, 11970] dup_rows_per_dst=[732, 1910, 1352, 5622, 2686, 3568, 506, 1048] combine_rows_per_src=[6018, 13478, 5798, 13156, 10926, 13000, 6534, 11970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=340112 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270217360 ingress_bytes=339083264 link_bytes=339083264 token_bytes=540094608 weight_movement_bytes=69206016 total_nvlink_bytes=609300624 weight_to_token_ratio=0.128137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275943024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79136 payload_rows_remote=70486 dup_rows=19168 dup_rows_remote=17710 dedup_ratio=0.194987 router_tokens_per_expert=[256, 130, 4788, 34, 1030, 159, 3944, 37, 27, 308, 530, 6, 1906, 1623, 503, 151, 511, 71, 5602, 1362, 0, 109, 13, 418, 41, 0, 188, 92, 240, 358, 214, 407, 1218, 381, 18, 393, 92, 64, 435, 90, 681, 0, 229, 571, 38, 122, 0, 4035, 0, 5710, 177, 201, 879, 79, 311, 337, 9, 722, 5367, 1182, 449, 0, 73, 231] payload_rows_per_dst=[13006, 8650, 13328, 2946, 4224, 10030, 12952, 14000] dup_rows_per_dst=[7490, 1458, 3088, 134, 1158, 1078, 2436, 2326] combine_rows_per_src=[13006, 8650, 13328, 2946, 4224, 10030, 12952, 14000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=63244 dup_rows=24000 dup_rows_remote=19890 dedup_ratio=0.244141 router_tokens_per_expert=[946, 1284, 153, 830, 2967, 758, 1349, 446, 158, 472, 627, 467, 191, 778, 659, 619, 441, 587, 207, 599, 758, 287, 2970, 986, 1001, 1381, 540, 197, 327, 458, 212, 889, 239, 360, 2462, 116, 100, 549, 847, 403, 873, 1152, 184, 432, 2115, 2301, 427, 101, 3221, 655, 3166, 667, 516, 282, 431, 668, 35, 661, 426, 26, 119, 180, 442, 452] payload_rows_per_dst=[9588, 7908, 6324, 9332, 11488, 11060, 12148, 6456] dup_rows_per_dst=[3606, 1670, 1406, 2382, 4604, 4110, 5428, 794] combine_rows_per_src=[9588, 7908, 6324, 9332, 11488, 11060, 12148, 6456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78592 payload_rows_remote=67606 dup_rows=19712 dup_rows_remote=14492 dedup_ratio=0.200521 router_tokens_per_expert=[1828, 4550, 0, 272, 1000, 168, 79, 59, 0, 208, 140, 184, 136, 339, 549, 608, 344, 1036, 0, 8, 545, 105, 3640, 1297, 2381, 3533, 1764, 288, 0, 214, 177, 185, 1, 373, 1668, 0, 90, 1038, 593, 166, 719, 216, 0, 72, 1317, 5857, 288, 0, 5315, 0, 2525, 846, 466, 126, 358, 13, 2, 379, 0, 0, 0, 526, 186, 375] payload_rows_per_dst=[5950, 13016, 10986, 12486, 10134, 7242, 7952, 10826] dup_rows_per_dst=[862, 1942, 5220, 4598, 2782, 2382, 716, 1210] combine_rows_per_src=[5950, 13016, 10986, 12486, 10134, 7242, 7952, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78086 payload_rows_remote=66926 dup_rows=20218 dup_rows_remote=16964 dedup_ratio=0.205668 router_tokens_per_expert=[1826, 2895, 28, 426, 1503, 287, 165, 171, 8, 295, 231, 272, 117, 347, 575, 508, 369, 799, 3, 38, 977, 146, 3866, 1118, 1630, 3207, 1191, 295, 7, 286, 141, 450, 26, 312, 2682, 0, 76, 1195, 763, 281, 684, 500, 9, 191, 1839, 5083, 458, 5, 5143, 14, 2471, 687, 450, 192, 392, 90, 8, 393, 4, 1, 2, 455, 211, 358] payload_rows_per_dst=[7384, 12866, 5638, 11160, 12754, 12556, 7896, 7832] dup_rows_per_dst=[1428, 2126, 1262, 3254, 5648, 4982, 696, 822] combine_rows_per_src=[7384, 12866, 5638, 11160, 12754, 12556, 7896, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76310 payload_rows_remote=64272 dup_rows=21994 dup_rows_remote=16700 dedup_ratio=0.223735 router_tokens_per_expert=[1035, 442, 744, 46, 1618, 1277, 573, 869, 781, 216, 122, 469, 492, 20, 91, 452, 109, 475, 379, 148, 2873, 469, 1015, 233, 224, 1415, 119, 462, 1827, 718, 184, 224, 217, 522, 161, 64, 436, 2836, 1755, 1346, 685, 427, 1519, 580, 340, 611, 91, 413, 411, 3096, 857, 55, 1315, 365, 751, 3912, 172, 1568, 592, 625, 2693, 261, 99, 256] payload_rows_per_dst=[12038, 5294, 9904, 9426, 10324, 7572, 11660, 10092] dup_rows_per_dst=[5294, 672, 2324, 2422, 4394, 1414, 3034, 2440] combine_rows_per_src=[12038, 5294, 9904, 9426, 10324, 7572, 11660, 10092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286531584 dispatch_meta_bytes=338768 combine_payload_bytes=286531584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286870352 ingress_bytes=321134592 link_bytes=321134592 token_bytes=573401936 weight_movement_bytes=34603008 total_nvlink_bytes=608004944 weight_to_token_ratio=0.060347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208403616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288710656 dispatch_meta_bytes=352784 combine_payload_bytes=288710656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289063440 ingress_bytes=323313664 link_bytes=323313664 token_bytes=577774096 weight_movement_bytes=34603008 total_nvlink_bytes=612377104 weight_to_token_ratio=0.059890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216603289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=259047424 dispatch_meta_bytes=332536 combine_payload_bytes=259047424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259379960 ingress_bytes=328253440 link_bytes=328253440 token_bytes=518427384 weight_movement_bytes=69206016 total_nvlink_bytes=587633400 weight_to_token_ratio=0.133492 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235191268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263258112 dispatch_meta_bytes=323888 combine_payload_bytes=263258112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263582000 ingress_bytes=297861120 link_bytes=297861120 token_bytes=526840112 weight_movement_bytes=34603008 total_nvlink_bytes=561443120 weight_to_token_ratio=0.065680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120827415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276914176 dispatch_meta_bytes=328392 combine_payload_bytes=276914176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277242568 ingress_bytes=346120192 link_bytes=346120192 token_bytes=554156744 weight_movement_bytes=69206016 total_nvlink_bytes=623362760 weight_to_token_ratio=0.124885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302422418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75478 payload_rows_remote=66432 dup_rows=22826 dup_rows_remote=20298 dedup_ratio=0.232198 router_tokens_per_expert=[805, 1165, 732, 1191, 2665, 660, 2202, 458, 154, 486, 902, 370, 246, 1175, 586, 528, 555, 622, 1136, 1103, 688, 340, 2362, 1009, 937, 1160, 617, 170, 487, 435, 136, 967, 446, 463, 2171, 219, 74, 433, 772, 486, 694, 732, 163, 532, 1737, 1456, 209, 431, 2013, 1074, 2101, 542, 578, 166, 395, 793, 34, 735, 1272, 243, 258, 209, 370, 302] payload_rows_per_dst=[7060, 7182, 8316, 11366, 10770, 10214, 11524, 9046] dup_rows_per_dst=[876, 1712, 2590, 3782, 4082, 3148, 4108, 2528] combine_rows_per_src=[7060, 7182, 8316, 11366, 10770, 10214, 11524, 9046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=274128896 dispatch_meta_bytes=335560 combine_payload_bytes=274128896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274464456 ingress_bytes=308731904 link_bytes=308731904 token_bytes=548593352 weight_movement_bytes=34603008 total_nvlink_bytes=583196360 weight_to_token_ratio=0.063076 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161733300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70100 payload_rows_remote=62058 dup_rows=28204 dup_rows_remote=26376 dedup_ratio=0.286906 router_tokens_per_expert=[513, 501, 407, 372, 100, 1538, 1015, 537, 840, 344, 883, 507, 4516, 350, 367, 429, 310, 108, 263, 42, 590, 57, 1991, 488, 203, 563, 735, 1042, 1131, 194, 1882, 768, 408, 462, 646, 292, 416, 1004, 966, 1712, 448, 717, 1074, 3800, 217, 4167, 1937, 400, 483, 479, 232, 125, 1052, 161, 147, 1849, 346, 535, 32, 78, 254, 6, 677, 444] payload_rows_per_dst=[2130, 12404, 7708, 11394, 8994, 13890, 8042, 5538] dup_rows_per_dst=[98, 4068, 1064, 5462, 2818, 11630, 1828, 1236] combine_rows_per_src=[2130, 12404, 7708, 11394, 8994, 13890, 8042, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272105472 dispatch_meta_bytes=346920 combine_payload_bytes=272105472 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272452392 ingress_bytes=341311488 link_bytes=341311488 token_bytes=544557864 weight_movement_bytes=69206016 total_nvlink_bytes=613763880 weight_to_token_ratio=0.127087 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284327652e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=60774 dup_rows=24570 dup_rows_remote=21060 dedup_ratio=0.249939 router_tokens_per_expert=[541, 470, 4548, 2471, 1967, 0, 6086, 5744, 0, 160, 0, 816, 0, 987, 0, 251, 303, 3, 2355, 38, 1010, 45, 250, 161, 68, 132, 1230, 0, 3, 148, 0, 234, 5458, 796, 0, 80, 217, 943, 741, 0, 421, 0, 34, 0, 1, 0, 320, 37, 2912, 0, 91, 301, 4, 1047, 118, 0, 2569, 553, 39, 984, 1419, 27, 19, 0] payload_rows_per_dst=[5436, 4042, 14410, 7378, 12960, 1626, 12776, 15106] dup_rows_per_dst=[520, 386, 5408, 1194, 3510, 0, 5266, 8286] combine_rows_per_src=[5436, 4042, 14410, 7378, 12960, 1626, 12776, 15106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=254189568 dispatch_meta_bytes=353736 combine_payload_bytes=254189568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254543304 ingress_bytes=288792576 link_bytes=288792576 token_bytes=508732872 weight_movement_bytes=34603008 total_nvlink_bytes=543335880 weight_to_token_ratio=0.068018 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086703214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73910 payload_rows_remote=67892 dup_rows=24394 dup_rows_remote=23568 dedup_ratio=0.248149 router_tokens_per_expert=[771, 2102, 2079, 1584, 663, 214, 4161, 4282, 150, 244, 237, 884, 20, 1402, 239, 440, 461, 153, 1928, 191, 1485, 116, 1045, 446, 78, 417, 1816, 79, 1096, 166, 54, 512, 3738, 856, 173, 200, 553, 952, 661, 49, 438, 205, 752, 819, 281, 228, 421, 278, 1510, 65, 326, 611, 650, 536, 92, 26, 2409, 574, 661, 233, 575, 51, 523, 191] payload_rows_per_dst=[6160, 6074, 13970, 9018, 10714, 6018, 9174, 12782] dup_rows_per_dst=[912, 1158, 6244, 3014, 3650, 826, 2616, 5974] combine_rows_per_src=[6160, 6074, 13970, 9018, 10714, 6018, 9174, 12782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70714 payload_rows_remote=54742 dup_rows=27590 dup_rows_remote=16340 dedup_ratio=0.280660 router_tokens_per_expert=[279, 93, 89, 0, 0, 3383, 192, 0, 2393, 186, 3523, 117, 7035, 36, 317, 4, 4, 0, 0, 12, 183, 0, 1025, 5, 66, 547, 0, 683, 516, 143, 3508, 205, 0, 178, 1021, 203, 2, 3, 444, 3967, 346, 256, 222, 4092, 131, 6166, 2188, 196, 313, 509, 0, 0, 158, 0, 5, 2976, 0, 408, 0, 52, 0, 0, 658, 114] payload_rows_per_dst=[5818, 15972, 2448, 10756, 10390, 15338, 7730, 2262] dup_rows_per_dst=[32, 11250, 10, 2624, 1246, 11856, 370, 202] combine_rows_per_src=[5818, 15972, 2448, 10756, 10390, 15338, 7730, 2262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74072 payload_rows_remote=61408 dup_rows=24232 dup_rows_remote=15746 dedup_ratio=0.246501 router_tokens_per_expert=[267, 289, 3192, 341, 2486, 682, 3364, 243, 60, 467, 1320, 221, 1464, 1696, 551, 466, 666, 540, 3856, 1696, 232, 219, 286, 997, 485, 1, 316, 74, 698, 496, 158, 580, 1780, 480, 296, 412, 72, 112, 545, 88, 693, 70, 347, 613, 253, 68, 1, 1502, 41, 2744, 1007, 326, 739, 98, 390, 762, 168, 926, 3993, 1068, 401, 11, 470, 267] payload_rows_per_dst=[12664, 9848, 12278, 4940, 6010, 6468, 9686, 12178] dup_rows_per_dst=[8486, 2724, 4842, 676, 1560, 490, 2446, 3008] combine_rows_per_src=[12664, 9848, 12278, 4940, 6010, 6468, 9686, 12178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=248930304 dispatch_meta_bytes=327336 combine_payload_bytes=248930304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=249257640 ingress_bytes=318136320 link_bytes=318136320 token_bytes=498187944 weight_movement_bytes=69206016 total_nvlink_bytes=567393960 weight_to_token_ratio=0.138915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197121360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60924 payload_rows_remote=53556 dup_rows=37380 dup_rows_remote=36224 dedup_ratio=0.380249 router_tokens_per_expert=[581, 833, 4355, 2213, 2258, 0, 5388, 5756, 0, 178, 0, 955, 0, 1100, 0, 154, 287, 5, 2435, 48, 996, 53, 285, 153, 53, 146, 1509, 0, 32, 143, 0, 473, 5561, 725, 0, 94, 333, 706, 624, 0, 425, 8, 17, 0, 28, 0, 274, 22, 2349, 0, 75, 420, 14, 1206, 261, 0, 3213, 531, 117, 747, 969, 6, 37, 1] payload_rows_per_dst=[14790, 4240, 7368, 7942, 3090, 1548, 12372, 9574] dup_rows_per_dst=[25964, 534, 1156, 1196, 1874, 0, 4988, 1668] combine_rows_per_src=[14790, 4240, 7368, 7942, 3090, 1548, 12372, 9574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68132 payload_rows_remote=59144 dup_rows=30172 dup_rows_remote=28390 dedup_ratio=0.306925 router_tokens_per_expert=[545, 1222, 3526, 2795, 1614, 1, 5488, 6256, 6, 153, 1, 809, 6, 1216, 0, 210, 260, 15, 2087, 124, 1220, 87, 641, 208, 62, 162, 1860, 7, 161, 137, 4, 196, 5134, 749, 0, 82, 316, 635, 596, 1, 407, 1, 120, 4, 116, 4, 306, 42, 2143, 4, 160, 483, 27, 980, 152, 3, 2993, 530, 273, 554, 1019, 18, 207, 14] payload_rows_per_dst=[10504, 4332, 11808, 8988, 7030, 1978, 11150, 12342] dup_rows_per_dst=[11512, 470, 4342, 1782, 2390, 22, 3806, 5848] combine_rows_per_src=[10504, 4332, 11808, 8988, 7030, 1978, 11150, 12342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278085632 dispatch_meta_bytes=365840 combine_payload_bytes=278085632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278451472 ingress_bytes=347291648 link_bytes=347291648 token_bytes=556537104 weight_movement_bytes=69206016 total_nvlink_bytes=625743120 weight_to_token_ratio=0.124351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306830512e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=251527168 dispatch_meta_bytes=308616 combine_payload_bytes=251527168 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=251835784 ingress_bytes=424542208 link_bytes=424542208 token_bytes=503362952 weight_movement_bytes=173015040 total_nvlink_bytes=676377992 weight_to_token_ratio=0.343718 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.597518150e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=224223232 dispatch_meta_bytes=284328 combine_payload_bytes=224223232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=224507560 ingress_bytes=258826240 link_bytes=258826240 token_bytes=448730792 weight_movement_bytes=34603008 total_nvlink_bytes=483333800 weight_to_token_ratio=0.077113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.739423039e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73036 payload_rows_remote=62626 dup_rows=25268 dup_rows_remote=21412 dedup_ratio=0.257039 router_tokens_per_expert=[646, 1869, 1457, 1252, 618, 707, 3297, 3216, 509, 335, 613, 677, 800, 1188, 285, 452, 438, 195, 1655, 158, 1238, 135, 1196, 521, 122, 470, 1363, 335, 1590, 151, 273, 527, 2695, 716, 272, 229, 530, 816, 706, 362, 472, 250, 1170, 1864, 281, 1681, 699, 213, 1263, 265, 293, 587, 783, 402, 115, 364, 1521, 547, 317, 200, 446, 24, 547, 234] payload_rows_per_dst=[5526, 7602, 12500, 9720, 9672, 8998, 8608, 10410] dup_rows_per_dst=[740, 2116, 5004, 3860, 2980, 4262, 2450, 3856] combine_rows_per_src=[5526, 7602, 12500, 9720, 9672, 8998, 8608, 10410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=219365376 dispatch_meta_bytes=359120 combine_payload_bytes=219365376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219724496 ingress_bytes=253968384 link_bytes=253968384 token_bytes=439089872 weight_movement_bytes=34603008 total_nvlink_bytes=473692880 weight_to_token_ratio=0.078806 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.556625829e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=242253824 dispatch_meta_bytes=350136 combine_payload_bytes=242253824 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=242603960 ingress_bytes=311459840 link_bytes=311459840 token_bytes=484857784 weight_movement_bytes=69206016 total_nvlink_bytes=554063800 weight_to_token_ratio=0.142735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171998303e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74696 payload_rows_remote=61126 dup_rows=23608 dup_rows_remote=17798 dedup_ratio=0.240153 router_tokens_per_expert=[551, 872, 338, 459, 51, 141, 306, 355, 53, 2157, 2319, 1061, 265, 736, 467, 223, 913, 67, 1471, 19, 401, 1981, 165, 452, 55, 310, 859, 278, 253, 150, 157, 10, 94, 414, 257, 1141, 111, 2085, 638, 78, 109, 192, 4238, 378, 177, 796, 674, 3026, 164, 2512, 7, 476, 397, 735, 673, 4726, 818, 364, 581, 86, 531, 437, 3563, 779] payload_rows_per_dst=[4094, 11534, 10034, 4332, 7984, 8826, 13570, 14322] dup_rows_per_dst=[308, 3028, 2648, 974, 1652, 1878, 5810, 7310] combine_rows_per_src=[4094, 11534, 10034, 4332, 7984, 8826, 13570, 14322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=256516096 dispatch_meta_bytes=336152 combine_payload_bytes=256516096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256852248 ingress_bytes=291119104 link_bytes=291119104 token_bytes=513368344 weight_movement_bytes=34603008 total_nvlink_bytes=547971352 weight_to_token_ratio=0.067404 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.095457751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81028 payload_rows_remote=65550 dup_rows=17276 dup_rows_remote=12572 dedup_ratio=0.175741 router_tokens_per_expert=[0, 7132, 2530, 0, 16, 69, 569, 4967, 0, 0, 0, 0, 0, 648, 2610, 165, 191, 468, 80, 247, 96, 0, 100, 973, 331, 327, 0, 169, 232, 72, 169, 1, 54, 7476, 84, 0, 1235, 0, 1242, 0, 1403, 0, 0, 0, 2158, 113, 0, 0, 66, 0, 104, 719, 64, 571, 80, 0, 0, 420, 6490, 157, 81, 1783, 14, 2676] payload_rows_per_dst=[12582, 6844, 15606, 14236, 15478, 6842, 3032, 6408] dup_rows_per_dst=[3720, 2, 2968, 4514, 4704, 506, 176, 686] combine_rows_per_src=[12582, 6844, 15606, 14236, 15478, 6842, 3032, 6408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=250372096 dispatch_meta_bytes=315696 combine_payload_bytes=250372096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250687792 ingress_bytes=354181120 link_bytes=354181120 token_bytes=501059888 weight_movement_bytes=103809024 total_nvlink_bytes=604868912 weight_to_token_ratio=0.207179 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332755041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78594 payload_rows_remote=74054 dup_rows=19710 dup_rows_remote=19506 dedup_ratio=0.200500 router_tokens_per_expert=[101, 3365, 1741, 231, 196, 256, 396, 3648, 64, 210, 6, 319, 44, 477, 2606, 600, 365, 318, 856, 147, 73, 59, 500, 1066, 482, 644, 318, 566, 238, 396, 305, 3, 39, 4126, 592, 50, 484, 310, 2067, 2, 450, 15, 139, 118, 1510, 275, 4, 0, 45, 0, 128, 1275, 2404, 1228, 399, 7, 486, 415, 3414, 78, 350, 2478, 1957, 3711] payload_rows_per_dst=[10780, 7828, 11030, 10288, 11906, 4540, 9280, 12942] dup_rows_per_dst=[2358, 824, 2468, 2444, 3434, 204, 1692, 6286] combine_rows_per_src=[10780, 7828, 11030, 10288, 11906, 4540, 9280, 12942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71666 payload_rows_remote=64028 dup_rows=26638 dup_rows_remote=26114 dedup_ratio=0.270976 router_tokens_per_expert=[466, 184, 197, 16, 7, 3011, 339, 0, 1276, 238, 1658, 375, 6529, 55, 398, 242, 104, 23, 17, 37, 235, 51, 1570, 395, 110, 593, 501, 1134, 1567, 122, 2275, 670, 13, 302, 782, 150, 227, 232, 744, 2955, 469, 895, 461, 4714, 150, 5403, 2105, 117, 368, 525, 27, 0, 475, 3, 97, 1933, 3, 629, 0, 47, 5, 0, 648, 278] payload_rows_per_dst=[7638, 15266, 4688, 9910, 11894, 12718, 6514, 3038] dup_rows_per_dst=[524, 8390, 176, 4066, 5092, 7594, 614, 182] combine_rows_per_src=[7638, 15266, 4688, 9910, 11894, 12718, 6514, 3038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268492800 dispatch_meta_bytes=312488 combine_payload_bytes=268492800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268805288 ingress_bytes=337698816 link_bytes=337698816 token_bytes=537298088 weight_movement_bytes=69206016 total_nvlink_bytes=606504104 weight_to_token_ratio=0.128804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270733458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72792 payload_rows_remote=65712 dup_rows=25512 dup_rows_remote=24892 dedup_ratio=0.259521 router_tokens_per_expert=[350, 0, 0, 1, 0, 52, 0, 0, 0, 643, 2096, 629, 11, 318, 153, 0, 691, 0, 140, 0, 442, 5366, 0, 311, 0, 20, 538, 72, 256, 52, 210, 0, 0, 0, 92, 4141, 0, 523, 0, 34, 0, 42, 7375, 190, 0, 591, 538, 7484, 1, 6399, 0, 0, 0, 479, 173, 7303, 52, 393, 0, 0, 92, 0, 807, 92] payload_rows_per_dst=[806, 7080, 11730, 2256, 9072, 15890, 15742, 10216] dup_rows_per_dst=[0, 620, 2170, 224, 508, 7468, 12968, 1554] combine_rows_per_src=[806, 7080, 11730, 2256, 9072, 15890, 15742, 10216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69814 payload_rows_remote=66310 dup_rows=28490 dup_rows_remote=27786 dedup_ratio=0.289815 router_tokens_per_expert=[0, 6949, 1834, 0, 24, 53, 524, 5149, 0, 3, 0, 0, 0, 562, 2788, 159, 271, 367, 130, 265, 73, 0, 99, 899, 418, 352, 19, 199, 247, 54, 236, 2, 39, 7487, 95, 0, 1125, 0, 1510, 0, 1251, 0, 0, 0, 2163, 200, 0, 0, 63, 0, 149, 958, 189, 499, 85, 0, 0, 371, 6375, 117, 104, 1941, 27, 2728] payload_rows_per_dst=[15426, 16010, 3504, 14466, 5168, 6674, 3626, 4940] dup_rows_per_dst=[13640, 5988, 704, 6794, 370, 554, 260, 180] combine_rows_per_src=[15426, 16010, 3504, 14466, 5168, 6674, 3626, 4940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73682 payload_rows_remote=59150 dup_rows=24622 dup_rows_remote=16070 dedup_ratio=0.250468 router_tokens_per_expert=[0, 6142, 1889, 2, 24, 71, 422, 5057, 1, 12, 0, 0, 1, 454, 2542, 161, 325, 431, 127, 239, 88, 4, 233, 960, 473, 324, 52, 223, 237, 76, 205, 0, 83, 7222, 146, 5, 767, 5, 1770, 1, 937, 0, 19, 0, 2134, 224, 1, 7, 28, 6, 82, 1099, 613, 624, 76, 6, 6, 373, 6139, 135, 161, 1808, 87, 3813] payload_rows_per_dst=[13848, 12138, 8036, 14532, 9072, 6236, 4830, 4990] dup_rows_per_dst=[8310, 3200, 1834, 8552, 1930, 370, 238, 188] combine_rows_per_src=[13848, 12138, 8036, 14532, 9072, 6236, 4830, 4990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=303325184 dispatch_meta_bytes=374240 combine_payload_bytes=303325184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303699424 ingress_bytes=372531200 link_bytes=372531200 token_bytes=607024608 weight_movement_bytes=69206016 total_nvlink_bytes=676230624 weight_to_token_ratio=0.114009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401804914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=262258688 dispatch_meta_bytes=360568 combine_payload_bytes=262258688 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=262619256 ingress_bytes=469876736 link_bytes=469876736 token_bytes=524877944 weight_movement_bytes=207618048 total_nvlink_bytes=732495992 weight_to_token_ratio=0.395555 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.768108329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77060 payload_rows_remote=63510 dup_rows=21244 dup_rows_remote=13798 dedup_ratio=0.216105 router_tokens_per_expert=[316, 2505, 1064, 411, 307, 353, 368, 2433, 88, 726, 369, 1015, 177, 449, 1858, 670, 589, 407, 1571, 84, 144, 357, 535, 874, 270, 668, 595, 904, 298, 383, 141, 17, 150, 2652, 711, 337, 534, 1047, 1777, 97, 274, 165, 228, 200, 1030, 313, 230, 162, 107, 51, 107, 873, 1910, 1596, 442, 316, 1002, 405, 2627, 138, 530, 2451, 2554, 3190] payload_rows_per_dst=[8846, 8918, 11296, 9456, 11298, 4440, 9256, 13550] dup_rows_per_dst=[1454, 1786, 2836, 2350, 3312, 308, 1752, 7446] combine_rows_per_src=[8846, 8918, 11296, 9456, 11298, 4440, 9256, 13550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269156352 dispatch_meta_bytes=362416 combine_payload_bytes=269156352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269518768 ingress_bytes=303759360 link_bytes=303759360 token_bytes=538675120 weight_movement_bytes=34603008 total_nvlink_bytes=573278128 weight_to_token_ratio=0.064237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143022017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271605760 dispatch_meta_bytes=376384 combine_payload_bytes=271605760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271982144 ingress_bytes=306208768 link_bytes=306208768 token_bytes=543587904 weight_movement_bytes=34603008 total_nvlink_bytes=578190912 weight_to_token_ratio=0.063657 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152238942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75292 payload_rows_remote=63352 dup_rows=23012 dup_rows_remote=20170 dedup_ratio=0.234090 router_tokens_per_expert=[1935, 1087, 18, 362, 1347, 526, 144, 4630, 417, 83, 74, 3119, 194, 341, 3063, 94, 777, 519, 29, 174, 1098, 190, 18, 1942, 42, 649, 277, 624, 1078, 158, 1697, 1019, 165, 121, 10, 1326, 35, 227, 2564, 709, 168, 1021, 874, 1472, 747, 173, 40, 663, 8, 101, 3916, 829, 659, 871, 507, 500, 28, 1577, 512, 352, 433, 623, 100, 96] payload_rows_per_dst=[13584, 10830, 8346, 9166, 8994, 7094, 11940, 5338] dup_rows_per_dst=[6514, 3940, 2172, 1922, 1320, 3222, 2842, 1080] combine_rows_per_src=[13584, 10830, 8346, 9166, 8994, 7094, 11940, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=242278400 dispatch_meta_bytes=300880 combine_payload_bytes=242278400 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=242579280 ingress_bytes=311484416 link_bytes=311484416 token_bytes=484857680 weight_movement_bytes=69206016 total_nvlink_bytes=554063696 weight_to_token_ratio=0.142735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172090781e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260136960 dispatch_meta_bytes=309232 combine_payload_bytes=260136960 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=260446192 ingress_bytes=363945984 link_bytes=363945984 token_bytes=520583152 weight_movement_bytes=103809024 total_nvlink_bytes=624392176 weight_to_token_ratio=0.199409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369499438e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78442 payload_rows_remote=68834 dup_rows=19862 dup_rows_remote=17496 dedup_ratio=0.202047 router_tokens_per_expert=[0, 0, 0, 337, 0, 0, 0, 0, 412, 314, 1608, 0, 920, 323, 0, 80, 59, 0, 5, 0, 0, 97, 0, 0, 0, 4562, 0, 309, 209, 1395, 0, 0, 947, 1302, 0, 0, 0, 2295, 0, 896, 1809, 1110, 5229, 0, 629, 742, 981, 0, 0, 130, 0, 3247, 0, 0, 59, 0, 535, 0, 6186, 6234, 4842, 1187, 0, 162] payload_rows_per_dst=[2028, 10012, 12684, 12652, 9608, 10462, 6866, 14130] dup_rows_per_dst=[0, 3812, 10, 3784, 2366, 2674, 6, 7210] combine_rows_per_src=[2028, 10012, 12684, 12652, 9608, 10462, 6866, 14130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=259489792 dispatch_meta_bytes=334088 combine_payload_bytes=259489792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259823880 ingress_bytes=328695808 link_bytes=328695808 token_bytes=519313672 weight_movement_bytes=69206016 total_nvlink_bytes=588519688 weight_to_token_ratio=0.133264 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236855863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76734 payload_rows_remote=62974 dup_rows=21570 dup_rows_remote=15222 dedup_ratio=0.219421 router_tokens_per_expert=[1333, 110, 0, 757, 22, 43, 45, 3, 470, 107, 506, 0, 977, 346, 93, 204, 1917, 64, 251, 35, 0, 90, 218, 23, 39, 3689, 58, 171, 331, 1201, 276, 316, 994, 409, 7, 0, 0, 1567, 296, 971, 614, 3535, 4273, 0, 656, 386, 590, 0, 0, 246, 0, 5167, 189, 0, 164, 41, 343, 0, 3775, 3554, 3624, 3841, 41, 174] payload_rows_per_dst=[4192, 4112, 10624, 10734, 7434, 13760, 11366, 14512] dup_rows_per_dst=[434, 1294, 2122, 1428, 1054, 6348, 248, 8642] combine_rows_per_src=[4192, 4112, 10624, 10734, 7434, 13760, 11366, 14512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=281944064 dispatch_meta_bytes=345320 combine_payload_bytes=281944064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282289384 ingress_bytes=316547072 link_bytes=316547072 token_bytes=564233448 weight_movement_bytes=34603008 total_nvlink_bytes=598836456 weight_to_token_ratio=0.061327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191141148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68812 payload_rows_remote=66680 dup_rows=29492 dup_rows_remote=29492 dedup_ratio=0.300008 router_tokens_per_expert=[421, 471, 27, 55, 1, 51, 40, 0, 0, 1834, 2690, 850, 162, 455, 173, 88, 727, 2, 525, 0, 571, 3528, 0, 367, 0, 276, 576, 75, 256, 48, 134, 0, 12, 0, 77, 3565, 0, 1226, 18, 51, 72, 162, 6672, 305, 0, 663, 689, 5173, 33, 4417, 1, 15, 7, 518, 545, 6807, 271, 384, 5, 60, 220, 104, 2604, 73] payload_rows_per_dst=[2132, 10492, 9202, 2188, 9716, 15894, 12280, 6908] dup_rows_per_dst=[0, 2012, 2238, 698, 2700, 17674, 3792, 378] combine_rows_per_src=[2132, 10492, 9202, 2188, 9716, 15894, 12280, 6908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79380 payload_rows_remote=64274 dup_rows=18924 dup_rows_remote=9708 dedup_ratio=0.192505 router_tokens_per_expert=[611, 471, 2, 178, 974, 8, 90, 6963, 371, 28, 0, 5822, 0, 348, 5588, 4, 0, 142, 0, 79, 573, 65, 0, 733, 0, 21, 287, 195, 284, 0, 2235, 683, 0, 92, 0, 965, 0, 0, 2303, 256, 1, 0, 0, 1210, 781, 91, 0, 300, 171, 122, 6972, 2, 370, 1540, 223, 621, 0, 6233, 0, 0, 0, 0, 52, 92] payload_rows_per_dst=[15490, 15106, 3134, 6850, 6738, 3540, 15952, 12570] dup_rows_per_dst=[3104, 9216, 50, 560, 494, 1230, 4086, 184] combine_rows_per_src=[15490, 15106, 3134, 6850, 6738, 3540, 15952, 12570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74044 payload_rows_remote=59630 dup_rows=24260 dup_rows_remote=15420 dedup_ratio=0.246785 router_tokens_per_expert=[5, 0, 0, 332, 0, 0, 0, 0, 392, 243, 1177, 0, 926, 354, 0, 90, 75, 0, 14, 0, 0, 74, 0, 0, 3, 4358, 0, 281, 177, 1370, 0, 0, 1187, 883, 0, 0, 0, 2405, 0, 896, 1548, 1301, 5232, 0, 555, 784, 791, 1, 0, 237, 0, 3622, 0, 0, 72, 0, 475, 0, 6323, 6275, 5141, 1375, 0, 178] payload_rows_per_dst=[11046, 4526, 14414, 10798, 14764, 11058, 4022, 3416] dup_rows_per_dst=[92, 1838, 8840, 1580, 8528, 2726, 16, 640] combine_rows_per_src=[11046, 4526, 14414, 10798, 14764, 11058, 4022, 3416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79380 payload_rows_remote=64904 dup_rows=18924 dup_rows_remote=13192 dedup_ratio=0.192505 router_tokens_per_expert=[52, 0, 0, 453, 1, 2, 0, 7, 396, 226, 676, 5, 996, 316, 3, 86, 194, 0, 27, 0, 6, 91, 4, 3, 11, 4176, 0, 313, 194, 987, 3, 66, 1489, 659, 0, 1, 0, 2141, 4, 875, 1240, 2111, 5479, 1, 556, 642, 605, 1, 0, 223, 9, 4624, 1, 2, 79, 1, 575, 6, 5452, 5801, 5211, 1907, 0, 163] payload_rows_per_dst=[11778, 3934, 11542, 14476, 14372, 9002, 9774, 4502] dup_rows_per_dst=[210, 1474, 1726, 5732, 7568, 1310, 104, 800] combine_rows_per_src=[11778, 3934, 11542, 14476, 14372, 9002, 9774, 4502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=257941504 dispatch_meta_bytes=312784 combine_payload_bytes=257941504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258254288 ingress_bytes=361750528 link_bytes=361750528 token_bytes=516195792 weight_movement_bytes=103809024 total_nvlink_bytes=620004816 weight_to_token_ratio=0.201104 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361238114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=273121280 dispatch_meta_bytes=384688 combine_payload_bytes=273121280 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273505968 ingress_bytes=342327296 link_bytes=342327296 token_bytes=546627248 weight_movement_bytes=69206016 total_nvlink_bytes=615833264 weight_to_token_ratio=0.126605 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288150055e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76970 payload_rows_remote=65310 dup_rows=21334 dup_rows_remote=15652 dedup_ratio=0.217021 router_tokens_per_expert=[3063, 391, 5, 761, 101, 236, 93, 527, 566, 111, 333, 91, 1313, 314, 653, 178, 2780, 248, 106, 97, 56, 225, 188, 496, 75, 2684, 131, 237, 568, 926, 390, 359, 821, 337, 10, 48, 0, 920, 906, 1014, 613, 3106, 4467, 9, 757, 450, 350, 59, 32, 146, 57, 4268, 255, 2, 285, 235, 225, 26, 3032, 2221, 2790, 3181, 75, 153] payload_rows_per_dst=[8198, 5732, 11218, 9276, 7258, 13492, 10136, 11660] dup_rows_per_dst=[2156, 1386, 3238, 1464, 854, 6130, 424, 5682] combine_rows_per_src=[8198, 5732, 11218, 9276, 7258, 13492, 10136, 11660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76372 payload_rows_remote=68900 dup_rows=21932 dup_rows_remote=19896 dedup_ratio=0.223104 router_tokens_per_expert=[1202, 323, 155, 192, 78, 1285, 1195, 651, 26, 804, 4, 209, 503, 2652, 0, 920, 72, 26, 227, 126, 1146, 342, 1115, 911, 506, 695, 755, 367, 121, 155, 0, 184, 1229, 3, 4, 329, 24, 4287, 158, 222, 195, 3055, 4343, 367, 168, 2584, 4959, 368, 384, 721, 713, 88, 176, 347, 276, 2049, 213, 805, 223, 75, 601, 313, 2612, 314] payload_rows_per_dst=[8776, 10686, 6408, 12956, 9754, 11674, 7472, 8646] dup_rows_per_dst=[1386, 2008, 1522, 7922, 300, 5092, 2036, 1666] combine_rows_per_src=[8776, 10686, 6408, 12956, 9754, 11674, 7472, 8646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=244244480 dispatch_meta_bytes=300200 combine_payload_bytes=244244480 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=244544680 ingress_bytes=313450496 link_bytes=313450496 token_bytes=488789160 weight_movement_bytes=69206016 total_nvlink_bytes=557995176 weight_to_token_ratio=0.141587 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179488982e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=263266304 dispatch_meta_bytes=295928 combine_payload_bytes=263266304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263562232 ingress_bytes=297869312 link_bytes=297869312 token_bytes=526828536 weight_movement_bytes=34603008 total_nvlink_bytes=561431544 weight_to_token_ratio=0.065682 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120858241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=312384 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266159168 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532005952 weight_movement_bytes=34603008 total_nvlink_bytes=566608960 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81830 payload_rows_remote=71376 dup_rows=16474 dup_rows_remote=14408 dedup_ratio=0.167582 router_tokens_per_expert=[0, 68, 349, 344, 198, 6964, 0, 460, 0, 0, 55, 0, 155, 0, 0, 0, 1757, 0, 0, 0, 0, 2942, 0, 2063, 0, 45, 48, 980, 0, 0, 0, 68, 6045, 14, 80, 3699, 0, 0, 177, 980, 299, 0, 0, 1582, 96, 0, 0, 73, 406, 6, 3270, 80, 5787, 67, 3401, 0, 32, 1310, 3147, 653, 832, 504, 0, 116] payload_rows_per_dst=[14958, 12412, 12686, 2272, 10454, 9548, 10696, 8804] dup_rows_per_dst=[1808, 98, 5518, 10, 2066, 1092, 4118, 1764] combine_rows_per_src=[14958, 12412, 12686, 2272, 10454, 9548, 10696, 8804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=323848 combine_payload_bytes=267509760 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267833608 ingress_bytes=440524800 link_bytes=440524800 token_bytes=535343368 weight_movement_bytes=173015040 total_nvlink_bytes=708358408 weight_to_token_ratio=0.323185 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657659357e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=282214400 dispatch_meta_bytes=355184 combine_payload_bytes=282214400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282569584 ingress_bytes=386023424 link_bytes=386023424 token_bytes=564783984 weight_movement_bytes=103809024 total_nvlink_bytes=668593008 weight_to_token_ratio=0.183803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.452575067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=292356096 dispatch_meta_bytes=343136 combine_payload_bytes=292356096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292699232 ingress_bytes=396165120 link_bytes=396165120 token_bytes=585055328 weight_movement_bytes=103809024 total_nvlink_bytes=688864352 weight_to_token_ratio=0.177435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490737453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76414 payload_rows_remote=61808 dup_rows=21890 dup_rows_remote=17018 dedup_ratio=0.222677 router_tokens_per_expert=[489, 708, 3, 171, 2179, 58, 118, 6003, 367, 47, 0, 4942, 0, 383, 4735, 72, 0, 536, 0, 78, 1575, 73, 0, 1825, 0, 64, 166, 248, 555, 0, 2094, 814, 72, 72, 0, 1875, 6, 2, 1192, 257, 49, 23, 10, 1551, 786, 82, 0, 637, 139, 62, 6440, 0, 496, 1256, 477, 629, 0, 4620, 5, 0, 0, 0, 38, 73] payload_rows_per_dst=[14606, 14394, 6554, 7094, 6350, 4438, 15168, 7810] dup_rows_per_dst=[4872, 6698, 1630, 788, 602, 1818, 5354, 128] combine_rows_per_src=[14606, 14394, 6554, 7094, 6350, 4438, 15168, 7810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79364 payload_rows_remote=73632 dup_rows=18940 dup_rows_remote=18690 dedup_ratio=0.192668 router_tokens_per_expert=[63, 58, 285, 235, 732, 6944, 68, 488, 39, 85, 11, 0, 400, 0, 0, 0, 472, 0, 0, 106, 20, 2345, 0, 2339, 79, 476, 34, 430, 0, 77, 0, 60, 5595, 54, 20, 3119, 0, 0, 166, 2178, 557, 17, 0, 1651, 91, 1, 7, 427, 249, 1664, 3426, 455, 1444, 68, 2986, 4, 205, 3886, 1662, 588, 1552, 979, 11, 244] payload_rows_per_dst=[15146, 11772, 9444, 2254, 12448, 5732, 13458, 9110] dup_rows_per_dst=[2600, 488, 1120, 72, 5340, 250, 6640, 2430] combine_rows_per_src=[15146, 11772, 9444, 2254, 12448, 5732, 13458, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82520 payload_rows_remote=67516 dup_rows=15784 dup_rows_remote=8142 dedup_ratio=0.160563 router_tokens_per_expert=[0, 53, 349, 252, 311, 6829, 0, 441, 0, 0, 36, 0, 211, 0, 0, 0, 1273, 0, 0, 0, 0, 2844, 0, 1771, 0, 159, 29, 845, 0, 0, 0, 53, 5884, 25, 90, 4149, 0, 0, 163, 1196, 288, 0, 0, 1358, 74, 0, 0, 196, 388, 152, 2997, 90, 5435, 51, 3437, 0, 43, 1841, 3638, 634, 1057, 390, 0, 120] payload_rows_per_dst=[14848, 12202, 15004, 2148, 11846, 8764, 8146, 9562] dup_rows_per_dst=[1622, 60, 7642, 24, 3082, 1062, 90, 2202] combine_rows_per_src=[14848, 12202, 15004, 2148, 11846, 8764, 8146, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76604 payload_rows_remote=72746 dup_rows=21700 dup_rows_remote=20726 dedup_ratio=0.220744 router_tokens_per_expert=[485, 258, 0, 64, 0, 0, 1934, 314, 0, 333, 0, 0, 92, 1170, 0, 821, 0, 0, 0, 0, 788, 0, 1382, 0, 575, 471, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6800, 0, 0, 0, 7477, 7483, 0, 64, 1758, 7735, 311, 178, 0, 0, 0, 0, 52, 0, 3185, 0, 0, 0, 0, 0, 0, 5417, 4] payload_rows_per_dst=[5418, 3858, 4256, 16256, 13600, 15648, 6726, 10842] dup_rows_per_dst=[692, 974, 84, 16274, 0, 3572, 104, 0] combine_rows_per_src=[5418, 3858, 4256, 16256, 13600, 15648, 6726, 10842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82512 payload_rows_remote=80426 dup_rows=15792 dup_rows_remote=15730 dedup_ratio=0.160645 router_tokens_per_expert=[0, 61, 308, 194, 552, 6601, 4, 409, 3, 14, 41, 4, 230, 5, 0, 1, 1035, 2, 0, 0, 2, 2956, 1, 1652, 0, 275, 34, 683, 0, 0, 0, 61, 5990, 110, 67, 4125, 0, 8, 164, 1708, 323, 6, 11, 1111, 88, 1, 10, 330, 319, 360, 3255, 107, 4138, 61, 3702, 3, 51, 2492, 3112, 589, 1303, 396, 4, 80] payload_rows_per_dst=[14698, 12474, 13666, 2086, 13042, 8900, 8992, 8654] dup_rows_per_dst=[1560, 102, 5906, 62, 4306, 1328, 112, 2416] combine_rows_per_src=[14698, 12474, 13666, 2086, 13042, 8900, 8992, 8654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=253165568 dispatch_meta_bytes=315304 combine_payload_bytes=253165568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253480872 ingress_bytes=287768576 link_bytes=287768576 token_bytes=506646440 weight_movement_bytes=34603008 total_nvlink_bytes=541249448 weight_to_token_ratio=0.068298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.082849984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297967616 dispatch_meta_bytes=373888 combine_payload_bytes=297967616 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=298341504 ingress_bytes=367173632 link_bytes=367173632 token_bytes=596309120 weight_movement_bytes=69206016 total_nvlink_bytes=665515136 weight_to_token_ratio=0.116057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381644817e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301596672 dispatch_meta_bytes=369288 combine_payload_bytes=301596672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301965960 ingress_bytes=405405696 link_bytes=405405696 token_bytes=603562632 weight_movement_bytes=103809024 total_nvlink_bytes=707371656 weight_to_token_ratio=0.171994 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525508996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=329424896 dispatch_meta_bytes=384624 combine_payload_bytes=329424896 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=329809520 ingress_bytes=398630912 link_bytes=398630912 token_bytes=659234416 weight_movement_bytes=69206016 total_nvlink_bytes=728440432 weight_to_token_ratio=0.104979 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500016029e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276545536 dispatch_meta_bytes=302632 combine_payload_bytes=276545536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276848168 ingress_bytes=311148544 link_bytes=311148544 token_bytes=553393704 weight_movement_bytes=34603008 total_nvlink_bytes=587996712 weight_to_token_ratio=0.062529 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170826922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71556 payload_rows_remote=58156 dup_rows=26748 dup_rows_remote=16314 dedup_ratio=0.272095 router_tokens_per_expert=[262, 197, 284, 203, 667, 6433, 81, 571, 165, 87, 37, 0, 442, 62, 0, 175, 524, 0, 67, 458, 122, 1779, 28, 2422, 246, 500, 442, 464, 20, 161, 2, 182, 5055, 125, 14, 1996, 34, 209, 158, 1577, 473, 51, 46, 1120, 118, 409, 45, 373, 315, 2908, 3373, 307, 1105, 334, 1804, 731, 982, 3934, 1033, 443, 1066, 1108, 471, 352] payload_rows_per_dst=[7496, 11216, 9434, 3478, 7268, 4936, 14328, 13400] dup_rows_per_dst=[702, 830, 1366, 646, 958, 244, 11568, 10434] combine_rows_per_src=[7496, 11216, 9434, 3478, 7268, 4936, 14328, 13400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75732 payload_rows_remote=68180 dup_rows=22572 dup_rows_remote=21162 dedup_ratio=0.229614 router_tokens_per_expert=[0, 310, 1573, 265, 67, 51, 1276, 2, 344, 147, 49, 288, 23, 322, 593, 391, 0, 1016, 3428, 286, 8, 227, 601, 3914, 1247, 4697, 176, 591, 1078, 291, 43, 1103, 1294, 838, 1470, 53, 2763, 1597, 6, 519, 709, 85, 785, 2144, 133, 245, 258, 3417, 68, 0, 167, 898, 596, 479, 233, 746, 610, 525, 0, 115, 1182, 2114, 100, 596] payload_rows_per_dst=[8256, 3756, 13202, 10528, 11224, 11388, 7552, 9826] dup_rows_per_dst=[1326, 558, 5758, 1838, 4654, 4164, 1410, 2864] combine_rows_per_src=[8256, 3756, 13202, 10528, 11224, 11388, 7552, 9826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82282 payload_rows_remote=75636 dup_rows=16022 dup_rows_remote=16022 dedup_ratio=0.162984 router_tokens_per_expert=[37, 0, 68, 0, 155, 0, 0, 0, 0, 2624, 0, 164, 80, 250, 422, 1356, 0, 7254, 0, 429, 0, 0, 59, 0, 7401, 0, 61, 247, 0, 982, 80, 6491, 7213, 3323, 0, 0, 0, 0, 0, 0, 80, 57, 1883, 0, 0, 1727, 247, 0, 428, 0, 0, 2521, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[14990, 15748, 976, 13732, 6646, 7394, 15770, 7026] dup_rows_per_dst=[332, 8552, 0, 1990, 0, 594, 4554, 0] combine_rows_per_src=[14990, 15748, 976, 13732, 6646, 7394, 15770, 7026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=238206976 dispatch_meta_bytes=297880 combine_payload_bytes=238206976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=238504856 ingress_bytes=307412992 link_bytes=307412992 token_bytes=476711832 weight_movement_bytes=69206016 total_nvlink_bytes=545917848 weight_to_token_ratio=0.145174 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156770340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=309805056 dispatch_meta_bytes=366632 combine_payload_bytes=309805056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=310171688 ingress_bytes=413614080 link_bytes=413614080 token_bytes=619976744 weight_movement_bytes=103809024 total_nvlink_bytes=723785768 weight_to_token_ratio=0.167440 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556396484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=357368 combine_payload_bytes=279265280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279622648 ingress_bytes=313868288 link_bytes=313868288 token_bytes=558887928 weight_movement_bytes=34603008 total_nvlink_bytes=593490936 weight_to_token_ratio=0.061914 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181061099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70020 payload_rows_remote=64362 dup_rows=28284 dup_rows_remote=27348 dedup_ratio=0.287720 router_tokens_per_expert=[477, 204, 11, 64, 0, 0, 2173, 368, 0, 650, 0, 16, 240, 3311, 7, 911, 0, 1, 0, 0, 503, 1, 1563, 56, 529, 389, 40, 54, 32, 1, 6, 47, 0, 5, 0, 0, 0, 6201, 79, 0, 0, 7128, 7652, 0, 64, 2672, 7172, 367, 204, 0, 0, 0, 0, 48, 0, 1657, 0, 62, 0, 0, 367, 0, 3748, 72] payload_rows_per_dst=[5658, 12496, 4128, 16274, 3796, 15494, 3800, 8374] dup_rows_per_dst=[936, 6628, 120, 15570, 44, 4968, 18, 0] combine_rows_per_src=[5658, 12496, 4128, 16274, 3796, 15494, 3800, 8374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=74278 dup_rows=24026 dup_rows_remote=24026 dedup_ratio=0.244405 router_tokens_per_expert=[0, 0, 131, 122, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5928, 19, 0, 0, 256, 3920, 0, 7593, 0, 0, 5182, 0, 0, 0, 0, 0, 738, 0, 5618, 832, 0, 311, 91, 0, 0, 2264, 156, 0, 256, 7074, 0, 0, 0, 0, 178, 1, 248, 816, 510, 0, 0, 0, 922, 5608, 53, 0] payload_rows_per_dst=[1156, 0, 14604, 15510, 13098, 15342, 2484, 12084] dup_rows_per_dst=[0, 0, 5642, 10040, 1900, 4340, 2, 2102] combine_rows_per_src=[1156, 0, 14604, 15510, 13098, 15342, 2484, 12084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87948 payload_rows_remote=76822 dup_rows=10356 dup_rows_remote=9318 dedup_ratio=0.105347 router_tokens_per_expert=[1, 0, 55, 82, 120, 0, 106, 0, 181, 789, 0, 516, 100, 1512, 420, 1076, 0, 6029, 1, 651, 0, 263, 120, 0, 6435, 0, 111, 820, 21, 325, 99, 5080, 7034, 4631, 0, 137, 0, 0, 1, 0, 451, 104, 3833, 0, 0, 1447, 247, 0, 399, 2, 159, 1125, 81, 0, 0, 1, 0, 4424, 0, 24, 6, 0, 0, 133] payload_rows_per_dst=[13394, 8482, 12192, 5642, 9534, 11126, 15352, 12226] dup_rows_per_dst=[204, 1830, 812, 360, 4, 1038, 2250, 3858] combine_rows_per_src=[13394, 8482, 12192, 5642, 9534, 11126, 15352, 12226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84484 payload_rows_remote=73430 dup_rows=13820 dup_rows_remote=12744 dedup_ratio=0.140584 router_tokens_per_expert=[40, 0, 61, 13, 175, 0, 0, 0, 1, 1827, 0, 255, 76, 511, 376, 1332, 1, 7133, 5, 506, 0, 0, 64, 1, 7245, 7, 45, 241, 3, 561, 76, 5643, 7453, 3953, 3, 12, 8, 9, 0, 0, 192, 95, 3112, 5, 0, 1713, 241, 8, 360, 0, 0, 1763, 0, 0, 0, 0, 4, 3939, 8, 0, 2, 9, 0, 65] payload_rows_per_dst=[14734, 15550, 1150, 11054, 7954, 9900, 16100, 8042] dup_rows_per_dst=[1356, 7472, 4, 1076, 16, 832, 3052, 12] combine_rows_per_src=[14734, 15550, 1150, 11054, 7954, 9900, 16100, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71248 payload_rows_remote=70646 dup_rows=27056 dup_rows_remote=27056 dedup_ratio=0.275228 router_tokens_per_expert=[17, 0, 52, 4, 110, 0, 0, 0, 0, 3047, 0, 234, 89, 243, 379, 1399, 0, 7202, 0, 229, 0, 0, 72, 0, 7383, 0, 29, 265, 0, 607, 89, 6304, 7437, 3495, 0, 0, 0, 0, 0, 0, 110, 70, 1964, 0, 0, 1575, 265, 0, 388, 0, 0, 2045, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 72] payload_rows_per_dst=[16070, 15714, 602, 1810, 12812, 7420, 8722, 8098] dup_rows_per_dst=[11670, 9472, 0, 170, 4418, 548, 778, 0] combine_rows_per_src=[16070, 15714, 602, 1810, 12812, 7420, 8722, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=263626752 dispatch_meta_bytes=366840 combine_payload_bytes=263626752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263993592 ingress_bytes=332832768 link_bytes=332832768 token_bytes=527620344 weight_movement_bytes=69206016 total_nvlink_bytes=596826360 weight_to_token_ratio=0.131166 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252422911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304242688 dispatch_meta_bytes=393216 combine_payload_bytes=304242688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304635904 ingress_bytes=338845696 link_bytes=338845696 token_bytes=608878592 weight_movement_bytes=34603008 total_nvlink_bytes=643481600 weight_to_token_ratio=0.056831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275049075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=314662912 dispatch_meta_bytes=344560 combine_payload_bytes=314662912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=315007472 ingress_bytes=383868928 link_bytes=383868928 token_bytes=629670384 weight_movement_bytes=69206016 total_nvlink_bytes=698876400 weight_to_token_ratio=0.109908 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444467872e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=300769280 dispatch_meta_bytes=344696 combine_payload_bytes=300769280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=301113976 ingress_bytes=439181312 link_bytes=439181312 token_bytes=601883256 weight_movement_bytes=138412032 total_nvlink_bytes=740295288 weight_to_token_ratio=0.229965 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.652603920e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=289366016 dispatch_meta_bytes=390808 combine_payload_bytes=289366016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289756824 ingress_bytes=358572032 link_bytes=358572032 token_bytes=579122840 weight_movement_bytes=69206016 total_nvlink_bytes=648328856 weight_to_token_ratio=0.119501 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349277689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83384 payload_rows_remote=74584 dup_rows=14920 dup_rows_remote=14470 dedup_ratio=0.151774 router_tokens_per_expert=[0, 0, 81, 1082, 0, 0, 1344, 0, 0, 4286, 2394, 0, 0, 0, 512, 0, 97, 344, 0, 1011, 0, 55, 0, 0, 4200, 46, 0, 80, 0, 0, 0, 0, 59, 327, 0, 0, 3976, 183, 0, 80, 0, 0, 5417, 0, 0, 0, 0, 0, 659, 0, 80, 0, 7040, 0, 50, 84, 7209, 0, 0, 0, 7118, 633, 705, 0] payload_rows_per_dst=[14072, 5250, 10348, 14870, 8800, 10834, 14644, 4566] dup_rows_per_dst=[3138, 562, 1238, 8018, 450, 0, 1182, 332] combine_rows_per_src=[14072, 5250, 10348, 14870, 8800, 10834, 14644, 4566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76212 payload_rows_remote=63530 dup_rows=22092 dup_rows_remote=15478 dedup_ratio=0.224731 router_tokens_per_expert=[910, 471, 201, 2104, 2189, 0, 343, 8, 0, 2393, 528, 1193, 1290, 4500, 304, 2094, 314, 46, 1041, 549, 167, 59, 340, 1, 538, 182, 58, 153, 110, 456, 20, 178, 211, 34, 1668, 1919, 316, 641, 245, 537, 185, 3075, 289, 5, 1526, 143, 277, 4842, 1508, 1231, 154, 397, 235, 958, 40, 625, 667, 50, 1147, 553, 1808, 676, 404, 46] payload_rows_per_dst=[9556, 9114, 7636, 3256, 9770, 9038, 12682, 15160] dup_rows_per_dst=[2896, 1704, 2184, 134, 1372, 1962, 6614, 5226] combine_rows_per_src=[9556, 9114, 7636, 3256, 9770, 9038, 12682, 15160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83806 payload_rows_remote=70904 dup_rows=14498 dup_rows_remote=10044 dedup_ratio=0.147481 router_tokens_per_expert=[7, 24, 312, 80, 192, 0, 450, 0, 207, 534, 32, 1174, 74, 1366, 650, 1500, 1, 4891, 67, 749, 16, 556, 185, 421, 5390, 12, 231, 1373, 55, 422, 99, 3889, 6475, 4856, 43, 188, 17, 1, 40, 15, 600, 123, 3395, 16, 0, 1192, 242, 14, 264, 6, 191, 1171, 283, 223, 0, 49, 2, 3851, 0, 110, 45, 1, 36, 744] payload_rows_per_dst=[12120, 9122, 12172, 4098, 6504, 12252, 14636, 12902] dup_rows_per_dst=[790, 1952, 1600, 262, 8, 2744, 2688, 4454] combine_rows_per_src=[12120, 9122, 12172, 4098, 6504, 12252, 14636, 12902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=305496064 dispatch_meta_bytes=356216 combine_payload_bytes=305496064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305852280 ingress_bytes=340099072 link_bytes=340099072 token_bytes=611348344 weight_movement_bytes=34603008 total_nvlink_bytes=645951352 weight_to_token_ratio=0.056601 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279765428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260218880 dispatch_meta_bytes=316032 combine_payload_bytes=260218880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260534912 ingress_bytes=294821888 link_bytes=294821888 token_bytes=520753792 weight_movement_bytes=34603008 total_nvlink_bytes=555356800 weight_to_token_ratio=0.066448 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109391029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290422784 dispatch_meta_bytes=323792 combine_payload_bytes=290422784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290746576 ingress_bytes=325025792 link_bytes=325025792 token_bytes=581169360 weight_movement_bytes=34603008 total_nvlink_bytes=615772368 weight_to_token_ratio=0.059540 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223045889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76256 payload_rows_remote=61446 dup_rows=22048 dup_rows_remote=15696 dedup_ratio=0.224284 router_tokens_per_expert=[922, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 6356, 8192, 0, 30, 0, 0, 2057, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 5, 8, 3165, 0, 0, 3126, 4126, 0, 0, 256, 92, 0, 4438, 0, 0, 950, 0, 0, 8192, 270, 4848, 0, 0, 0, 619, 0, 72, 0, 0, 471, 0, 0, 0, 0, 0] payload_rows_per_dst=[2630, 14810, 4428, 6356, 14656, 13288, 11714, 8374] dup_rows_per_dst=[0, 6352, 2, 0, 7728, 6040, 1526, 400] combine_rows_per_src=[2630, 14810, 4428, 6356, 14656, 13288, 11714, 8374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69364 payload_rows_remote=67564 dup_rows=28940 dup_rows_remote=28834 dedup_ratio=0.294393 router_tokens_per_expert=[0, 1, 312, 116, 0, 17, 490, 0, 23, 0, 0, 0, 0, 0, 25, 0, 0, 0, 5092, 72, 0, 0, 365, 2789, 0, 7143, 0, 86, 2868, 0, 0, 17, 0, 0, 2378, 0, 4642, 2123, 0, 623, 225, 4, 0, 3109, 136, 0, 271, 7343, 0, 0, 0, 1, 187, 170, 367, 752, 990, 0, 0, 0, 1056, 5262, 62, 35] payload_rows_per_dst=[1800, 96, 15856, 14928, 13710, 8134, 2614, 12226] dup_rows_per_dst=[106, 0, 13734, 5266, 5822, 1088, 340, 2584] combine_rows_per_src=[1800, 96, 15856, 14928, 13710, 8134, 2614, 12226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71744 payload_rows_remote=64980 dup_rows=26560 dup_rows_remote=26296 dedup_ratio=0.270182 router_tokens_per_expert=[0, 0, 163, 3980, 0, 0, 1013, 0, 0, 7124, 908, 0, 0, 0, 645, 86, 1394, 321, 0, 3303, 243, 52, 0, 0, 3074, 164, 0, 325, 0, 0, 0, 0, 151, 318, 0, 0, 2914, 1293, 179, 99, 12, 0, 2963, 120, 0, 34, 385, 0, 758, 0, 99, 0, 2216, 0, 1, 146, 5546, 0, 0, 336, 6856, 735, 1150, 46] payload_rows_per_dst=[9570, 2810, 15242, 7158, 8714, 6764, 5590, 15896] dup_rows_per_dst=[742, 468, 9632, 254, 1194, 264, 850, 13156] combine_rows_per_src=[9570, 2810, 15242, 7158, 8714, 6764, 5590, 15896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79646 payload_rows_remote=64428 dup_rows=18658 dup_rows_remote=9650 dedup_ratio=0.189799 router_tokens_per_expert=[0, 0, 118, 1749, 0, 1, 1042, 0, 0, 5567, 1493, 0, 7, 12, 600, 10, 103, 327, 7, 1693, 15, 25, 2, 0, 5108, 79, 0, 86, 5, 0, 0, 5, 79, 316, 5, 2, 4300, 349, 16, 76, 16, 4, 4813, 9, 1, 0, 56, 11, 658, 9, 76, 1, 5126, 4, 10, 119, 6993, 0, 0, 0, 6830, 543, 676, 0] payload_rows_per_dst=[15408, 3740, 12490, 15218, 9676, 9730, 10998, 2386] dup_rows_per_dst=[4398, 584, 2884, 9008, 610, 68, 1032, 74] combine_rows_per_src=[15408, 3740, 12490, 15218, 9676, 9730, 10998, 2386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79910 payload_rows_remote=76422 dup_rows=18394 dup_rows_remote=18098 dedup_ratio=0.187113 router_tokens_per_expert=[0, 0, 136, 1230, 0, 0, 1161, 0, 0, 4942, 1924, 0, 0, 0, 409, 0, 75, 339, 0, 1410, 0, 68, 0, 0, 4647, 53, 0, 89, 0, 0, 0, 0, 79, 355, 0, 0, 4007, 241, 0, 89, 4, 0, 5568, 0, 0, 0, 10, 0, 625, 0, 88, 0, 6015, 4, 21, 92, 7266, 0, 0, 0, 7042, 528, 635, 0] payload_rows_per_dst=[15676, 13496, 3488, 15154, 9014, 8182, 12588, 2312] dup_rows_per_dst=[3910, 4016, 296, 8508, 528, 20, 1102, 14] combine_rows_per_src=[15676, 13496, 3488, 15154, 9014, 8182, 12588, 2312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=251682816 dispatch_meta_bytes=308568 combine_payload_bytes=251682816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=251991384 ingress_bytes=390094848 link_bytes=390094848 token_bytes=503674200 weight_movement_bytes=138412032 total_nvlink_bytes=642086232 weight_to_token_ratio=0.274805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.467895508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276742144 dispatch_meta_bytes=385592 combine_payload_bytes=276742144 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277127736 ingress_bytes=345948160 link_bytes=345948160 token_bytes=553869880 weight_movement_bytes=69206016 total_nvlink_bytes=623075896 weight_to_token_ratio=0.124950 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301775075e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=365104 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266523184 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532681264 weight_movement_bytes=69206016 total_nvlink_bytes=601887280 weight_to_token_ratio=0.129920 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=296312 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264193400 ingress_bytes=298500096 link_bytes=298500096 token_bytes=528090488 weight_movement_bytes=34603008 total_nvlink_bytes=562693496 weight_to_token_ratio=0.065525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123231830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89850 payload_rows_remote=83282 dup_rows=8454 dup_rows_remote=7722 dedup_ratio=0.085999 router_tokens_per_expert=[2911, 0, 0, 111, 747, 3425, 0, 0, 1074, 95, 573, 0, 0, 0, 0, 0, 0, 80, 160, 0, 0, 80, 0, 0, 33, 0, 0, 0, 6, 80, 0, 0, 0, 0, 5124, 0, 0, 0, 3150, 500, 0, 7218, 2458, 7718, 0, 0, 5132, 74, 0, 0, 0, 80, 3, 0, 1, 6270, 594, 0, 0, 380, 686, 140, 247, 2] payload_rows_per_dst=[12304, 14616, 10704, 15448, 6568, 5064, 12708, 12438] dup_rows_per_dst=[2084, 3304, 184, 226, 732, 0, 0, 1924] combine_rows_per_src=[12304, 14616, 10704, 15448, 6568, 5064, 12708, 12438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=313024512 dispatch_meta_bytes=378080 combine_payload_bytes=313024512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313402592 ingress_bytes=347627520 link_bytes=347627520 token_bytes=626427104 weight_movement_bytes=34603008 total_nvlink_bytes=661030112 weight_to_token_ratio=0.055239 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308094371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81182 payload_rows_remote=69162 dup_rows=17122 dup_rows_remote=15688 dedup_ratio=0.174174 router_tokens_per_expert=[605, 95, 4213, 48, 586, 78, 3496, 438, 1168, 266, 1872, 227, 371, 1536, 198, 0, 0, 7, 47, 706, 236, 329, 3375, 134, 3, 0, 54, 3687, 38, 75, 0, 6, 1924, 74, 1616, 3, 1, 4361, 1796, 817, 1, 780, 137, 927, 64, 2449, 772, 1659, 252, 87, 9, 38, 385, 220, 495, 1028, 414, 2390, 74, 397, 654, 1270, 164, 0] payload_rows_per_dst=[9456, 8946, 8180, 9462, 14702, 9160, 12020, 9256] dup_rows_per_dst=[1236, 2330, 1488, 118, 6482, 1020, 1434, 3014] combine_rows_per_src=[9456, 8946, 8180, 9462, 14702, 9160, 12020, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72300 payload_rows_remote=56570 dup_rows=26004 dup_rows_remote=14630 dedup_ratio=0.264526 router_tokens_per_expert=[22, 96, 109, 4746, 138, 2, 1959, 1, 0, 6840, 995, 1, 2, 3, 800, 743, 1709, 256, 8, 2579, 434, 67, 22, 0, 2416, 358, 11, 508, 2, 93, 1, 0, 520, 228, 10, 3, 2170, 1606, 196, 96, 72, 0, 1735, 22, 61, 203, 650, 2, 1256, 2, 172, 11, 1394, 27, 1, 244, 3942, 0, 0, 862, 6758, 844, 1034, 110] payload_rows_per_dst=[11190, 4548, 14668, 6632, 8658, 5410, 5464, 15730] dup_rows_per_dst=[2956, 534, 9162, 146, 1000, 76, 756, 11374] combine_rows_per_src=[11190, 4548, 14668, 6632, 8658, 5410, 5464, 15730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=341123072 dispatch_meta_bytes=364016 combine_payload_bytes=341123072 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=341487088 ingress_bytes=444932096 link_bytes=444932096 token_bytes=682610160 weight_movement_bytes=103809024 total_nvlink_bytes=786419184 weight_to_token_ratio=0.152077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674243657e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283287552 dispatch_meta_bytes=339400 combine_payload_bytes=283287552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283626952 ingress_bytes=317890560 link_bytes=317890560 token_bytes=566914504 weight_movement_bytes=34603008 total_nvlink_bytes=601517512 weight_to_token_ratio=0.061037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196196585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=231710720 dispatch_meta_bytes=284800 combine_payload_bytes=231710720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=231995520 ingress_bytes=335519744 link_bytes=335519744 token_bytes=463706240 weight_movement_bytes=103809024 total_nvlink_bytes=567515264 weight_to_token_ratio=0.223868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86486 payload_rows_remote=85670 dup_rows=11818 dup_rows_remote=11790 dedup_ratio=0.120219 router_tokens_per_expert=[0, 0, 8008, 0, 0, 0, 2438, 0, 0, 0, 0, 0, 0, 166, 256, 0, 0, 0, 0, 4418, 221, 92, 6620, 228, 0, 0, 0, 5904, 0, 0, 0, 0, 4965, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 2, 7474, 0, 184, 393, 52, 0, 0, 0, 307, 0, 0, 0, 6441, 345, 0, 84, 0, 0, 0] payload_rows_per_dst=[4876, 816, 14506, 11808, 10124, 14948, 16016, 13392] dup_rows_per_dst=[0, 28, 8652, 0, 914, 372, 1504, 348] combine_rows_per_src=[4876, 816, 14506, 11808, 10124, 14948, 16016, 13392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69948 payload_rows_remote=67812 dup_rows=28356 dup_rows_remote=28330 dedup_ratio=0.288452 router_tokens_per_expert=[551, 415, 62, 32, 21, 0, 0, 0, 0, 0, 41, 456, 6492, 8164, 0, 5, 0, 0, 2196, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 171, 80, 1545, 0, 0, 3690, 4049, 0, 0, 256, 72, 0, 3416, 0, 0, 1793, 0, 0, 8105, 243, 4425, 72, 37, 0, 875, 48, 81, 0, 1, 1288, 0, 0, 363, 0, 0] payload_rows_per_dst=[2136, 16336, 4606, 3580, 13082, 16244, 10660, 3304] dup_rows_per_dst=[26, 13980, 0, 12, 3052, 10384, 902, 0] combine_rows_per_src=[2136, 16336, 4606, 3580, 13082, 16244, 10660, 3304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84682 payload_rows_remote=76918 dup_rows=13622 dup_rows_remote=12972 dedup_ratio=0.138570 router_tokens_per_expert=[2628, 3, 0, 204, 774, 755, 0, 0, 1795, 67, 2595, 0, 0, 0, 0, 0, 0, 64, 38, 0, 0, 99, 0, 0, 6, 0, 0, 0, 61, 100, 0, 0, 0, 1, 6537, 64, 0, 7, 3941, 1334, 0, 6001, 1100, 6640, 20, 0, 4842, 139, 0, 0, 0, 100, 290, 0, 338, 5301, 346, 0, 15, 351, 609, 1739, 247, 1] payload_rows_per_dst=[8112, 11692, 7488, 13402, 11492, 7764, 12030, 12702] dup_rows_per_dst=[616, 3328, 6, 212, 5184, 650, 28, 3598] combine_rows_per_src=[8112, 11692, 7488, 13402, 11492, 7764, 12030, 12702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86744 payload_rows_remote=71430 dup_rows=11560 dup_rows_remote=11392 dedup_ratio=0.117594 router_tokens_per_expert=[2852, 0, 13, 154, 689, 1560, 0, 0, 1577, 93, 935, 0, 0, 2, 1, 1, 0, 76, 96, 9, 3, 76, 6, 1, 8, 0, 0, 5, 62, 76, 0, 0, 3, 0, 5673, 15, 0, 3, 4111, 741, 0, 7099, 1433, 7590, 0, 9, 5594, 74, 3, 2, 5, 76, 8, 1, 329, 6088, 584, 10, 0, 306, 587, 229, 239, 45] payload_rows_per_dst=[10692, 14698, 11812, 15314, 6134, 11360, 12810, 3924] dup_rows_per_dst=[2588, 4718, 68, 168, 842, 1154, 240, 1782] combine_rows_per_src=[10692, 14698, 11812, 15314, 6134, 11360, 12810, 3924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85008 payload_rows_remote=74028 dup_rows=13296 dup_rows_remote=13218 dedup_ratio=0.135254 router_tokens_per_expert=[3128, 0, 0, 140, 732, 2264, 0, 0, 1038, 74, 615, 0, 0, 0, 0, 0, 0, 90, 93, 0, 0, 88, 0, 0, 29, 0, 0, 0, 19, 90, 0, 0, 0, 0, 5258, 0, 0, 0, 3509, 619, 0, 7200, 2254, 7650, 0, 0, 5610, 78, 0, 0, 0, 90, 12, 0, 82, 6387, 649, 0, 0, 278, 595, 205, 265, 11] payload_rows_per_dst=[14076, 14588, 10980, 15338, 1238, 13524, 13004, 2260] dup_rows_per_dst=[5470, 3266, 78, 238, 0, 2360, 138, 1746] combine_rows_per_src=[14076, 14588, 10980, 15338, 1238, 13524, 13004, 2260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=350904320 dispatch_meta_bytes=389840 combine_payload_bytes=350904320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=351294160 ingress_bytes=385507328 link_bytes=385507328 token_bytes=702198480 weight_movement_bytes=34603008 total_nvlink_bytes=736801488 weight_to_token_ratio=0.049278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450633039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277757952 dispatch_meta_bytes=384568 combine_payload_bytes=277757952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278142520 ingress_bytes=312360960 link_bytes=312360960 token_bytes=555900472 weight_movement_bytes=34603008 total_nvlink_bytes=590503480 weight_to_token_ratio=0.062247 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175389145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=315056128 dispatch_meta_bytes=359560 combine_payload_bytes=315056128 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=315415688 ingress_bytes=453468160 link_bytes=453468160 token_bytes=630471816 weight_movement_bytes=138412032 total_nvlink_bytes=768883848 weight_to_token_ratio=0.219537 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.706364179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=81812 dup_rows=16492 dup_rows_remote=16492 dedup_ratio=0.167765 router_tokens_per_expert=[3063, 706, 0, 0, 0, 0, 2353, 0, 0, 4587, 0, 717, 100, 0, 749, 0, 0, 160, 709, 2, 0, 619, 0, 0, 0, 0, 5825, 666, 1990, 0, 309, 744, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 702, 163, 4647, 340, 6362, 0, 6680, 0, 0, 156, 0, 80, 0, 0, 96, 0, 0, 0, 1847, 80, 0, 680] payload_rows_per_dst=[13814, 10754, 14092, 14038, 0, 10878, 13460, 4776] dup_rows_per_dst=[6470, 1552, 1612, 4390, 0, 826, 1012, 630] combine_rows_per_src=[13814, 10754, 14092, 14038, 0, 10878, 13460, 4776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292577280 dispatch_meta_bytes=331288 combine_payload_bytes=292577280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292908568 ingress_bytes=327180288 link_bytes=327180288 token_bytes=585485848 weight_movement_bytes=34603008 total_nvlink_bytes=620088856 weight_to_token_ratio=0.059101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231153084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=303218688 dispatch_meta_bytes=348984 combine_payload_bytes=303218688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303567672 ingress_bytes=337821696 link_bytes=337821696 token_bytes=606786360 weight_movement_bytes=34603008 total_nvlink_bytes=641389368 weight_to_token_ratio=0.057027 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271195845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73166 payload_rows_remote=64946 dup_rows=25138 dup_rows_remote=21324 dedup_ratio=0.255717 router_tokens_per_expert=[201, 29, 132, 301, 0, 292, 320, 0, 1166, 1867, 658, 229, 1946, 361, 336, 173, 492, 59, 830, 123, 385, 358, 714, 2546, 3005, 177, 769, 1745, 50, 1791, 132, 310, 4571, 68, 44, 516, 2860, 48, 792, 1369, 295, 3080, 66, 10, 404, 44, 2392, 1657, 3041, 75, 33, 377, 189, 127, 671, 735, 92, 93, 3, 2433, 586, 155, 43, 786] payload_rows_per_dst=[4410, 9876, 9498, 11186, 10810, 9018, 8220, 10148] dup_rows_per_dst=[878, 3596, 1516, 3234, 6988, 718, 3814, 4394] combine_rows_per_src=[4410, 9876, 9498, 11186, 10810, 9018, 8220, 10148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80170 payload_rows_remote=67254 dup_rows=18134 dup_rows_remote=14082 dedup_ratio=0.184469 router_tokens_per_expert=[2393, 28, 3, 211, 1229, 485, 159, 0, 2636, 96, 4458, 3, 0, 0, 0, 0, 0, 25, 41, 3, 0, 74, 1, 1, 20, 0, 0, 2, 82, 74, 0, 0, 2, 3, 6571, 37, 0, 196, 4478, 1816, 0, 4094, 783, 5017, 40, 3, 3710, 384, 1, 0, 1, 74, 393, 0, 397, 4354, 345, 3, 35, 331, 611, 3189, 240, 20] payload_rows_per_dst=[8336, 10820, 286, 8992, 14874, 13700, 10246, 12916] dup_rows_per_dst=[674, 3566, 4, 134, 5902, 3602, 200, 4052] combine_rows_per_src=[8336, 10820, 286, 8992, 14874, 13700, 10246, 12916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=335101952 dispatch_meta_bytes=393216 combine_payload_bytes=335101952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335495168 ingress_bytes=404307968 link_bytes=404307968 token_bytes=670597120 weight_movement_bytes=69206016 total_nvlink_bytes=739803136 weight_to_token_ratio=0.103201 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=345080 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266363896 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532382712 weight_movement_bytes=34603008 total_nvlink_bytes=566985720 weight_to_token_ratio=0.064996 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=275472384 dispatch_meta_bytes=325344 combine_payload_bytes=275472384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275797728 ingress_bytes=310075392 link_bytes=310075392 token_bytes=551270112 weight_movement_bytes=34603008 total_nvlink_bytes=585873120 weight_to_token_ratio=0.062770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166788737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65156 payload_rows_remote=62138 dup_rows=33148 dup_rows_remote=31936 dedup_ratio=0.337199 router_tokens_per_expert=[0, 0, 270, 5, 0, 93, 0, 0, 1070, 0, 1045, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 64, 3902, 95, 266, 0, 0, 0, 3493, 0, 0, 8192, 0, 85, 0, 8100, 0, 107, 0, 0, 7840, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7851, 0, 92, 0, 0] payload_rows_per_dst=[726, 3018, 8554, 7584, 16384, 12704, 0, 16186] dup_rows_per_dst=[10, 1212, 104, 124, 16584, 720, 0, 14394] combine_rows_per_src=[726, 3018, 8554, 7584, 16384, 12704, 0, 16186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81412 payload_rows_remote=67708 dup_rows=16892 dup_rows_remote=12812 dedup_ratio=0.171834 router_tokens_per_expert=[0, 29, 8048, 0, 0, 0, 2532, 183, 0, 2, 0, 0, 374, 162, 256, 0, 0, 0, 0, 5272, 201, 72, 6086, 303, 0, 0, 3, 3714, 0, 5, 0, 0, 5844, 0, 0, 0, 0, 714, 0, 0, 0, 0, 0, 0, 47, 6627, 0, 139, 602, 100, 0, 0, 0, 317, 44, 0, 0, 6691, 327, 0, 458, 0, 0, 0] payload_rows_per_dst=[13704, 1586, 14086, 7444, 11950, 13354, 5396, 13892] dup_rows_per_dst=[4080, 2, 9782, 0, 1166, 272, 530, 1060] combine_rows_per_src=[13704, 1586, 14086, 7444, 11950, 13354, 5396, 13892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80004 payload_rows_remote=75240 dup_rows=18300 dup_rows_remote=18208 dedup_ratio=0.186157 router_tokens_per_expert=[1482, 301, 0, 0, 0, 0, 2182, 0, 0, 6752, 0, 663, 170, 0, 1181, 155, 0, 173, 1501, 56, 0, 380, 81, 0, 0, 0, 3746, 1134, 576, 0, 274, 390, 0, 0, 0, 649, 0, 20, 0, 5989, 0, 0, 89, 37, 1431, 283, 6658, 0, 6284, 3, 0, 345, 0, 177, 181, 341, 67, 0, 0, 0, 4392, 100, 88, 821] payload_rows_per_dst=[14910, 14444, 13958, 4672, 1338, 4764, 15574, 10344] dup_rows_per_dst=[4998, 3398, 2564, 76, 0, 92, 6580, 592] combine_rows_per_src=[14910, 14444, 13958, 4672, 1338, 4764, 15574, 10344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82014 payload_rows_remote=69356 dup_rows=16290 dup_rows_remote=12970 dedup_ratio=0.165710 router_tokens_per_expert=[2968, 575, 0, 0, 0, 3, 2198, 8, 0, 5792, 3, 676, 155, 0, 858, 0, 3, 152, 1059, 2, 3, 501, 60, 3, 0, 0, 4853, 662, 1473, 8, 324, 669, 12, 13, 0, 3, 10, 0, 0, 4892, 2, 9, 387, 89, 3632, 380, 6707, 7, 6639, 1, 0, 149, 0, 76, 21, 112, 88, 0, 0, 13, 2177, 76, 0, 649] payload_rows_per_dst=[14236, 12812, 14786, 12658, 54, 8526, 13540, 5402] dup_rows_per_dst=[7052, 2156, 2194, 3320, 22, 468, 456, 622] combine_rows_per_src=[14236, 12812, 14786, 12658, 54, 8526, 13540, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82200 payload_rows_remote=68150 dup_rows=16104 dup_rows_remote=14446 dedup_ratio=0.163818 router_tokens_per_expert=[3178, 842, 0, 0, 0, 0, 2333, 1, 0, 4775, 0, 727, 119, 0, 769, 0, 0, 179, 754, 1, 0, 536, 18, 0, 0, 0, 6092, 634, 1770, 0, 336, 708, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 696, 93, 4448, 327, 6366, 0, 6740, 8, 0, 194, 0, 89, 0, 0, 73, 0, 0, 0, 1677, 89, 0, 729] payload_rows_per_dst=[13780, 11192, 14050, 14502, 3936, 6752, 13496, 4492] dup_rows_per_dst=[6630, 1740, 1658, 4578, 0, 288, 566, 644] combine_rows_per_src=[13780, 11192, 14050, 14502, 3936, 6752, 13496, 4492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=254517248 dispatch_meta_bytes=376296 combine_payload_bytes=254517248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254893544 ingress_bytes=323723264 link_bytes=323723264 token_bytes=509410792 weight_movement_bytes=69206016 total_nvlink_bytes=578616808 weight_to_token_ratio=0.135855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218144581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=277331968 dispatch_meta_bytes=322080 combine_payload_bytes=277331968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277654048 ingress_bytes=346537984 link_bytes=346537984 token_bytes=554986016 weight_movement_bytes=69206016 total_nvlink_bytes=624192032 weight_to_token_ratio=0.124699 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303994535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308183040 dispatch_meta_bytes=373792 combine_payload_bytes=308183040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=308556832 ingress_bytes=446595072 link_bytes=446595072 token_bytes=616739872 weight_movement_bytes=138412032 total_nvlink_bytes=755151904 weight_to_token_ratio=0.224425 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.680501302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96734 payload_rows_remote=81990 dup_rows=1570 dup_rows_remote=1570 dedup_ratio=0.015971 router_tokens_per_expert=[0, 29, 786, 0, 0, 0, 4204, 0, 0, 0, 155, 157, 7348, 0, 0, 7549, 41, 0, 0, 0, 0, 3626, 0, 0, 0, 0, 0, 8107, 0, 0, 3241, 0, 0, 0, 80, 0, 0, 7292, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 80, 0, 4821, 0, 0, 694, 292, 0, 10] payload_rows_per_dst=[9650, 15678, 7334, 6482, 14744, 16384, 14856, 11606] dup_rows_per_dst=[388, 44, 0, 0, 0, 616, 494, 28] combine_rows_per_src=[9650, 15678, 7334, 6482, 14744, 16384, 14856, 11606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82336 payload_rows_remote=67808 dup_rows=15968 dup_rows_remote=10578 dedup_ratio=0.162435 router_tokens_per_expert=[84, 330, 141, 589, 1440, 209, 1576, 19, 86, 8, 2691, 249, 3173, 1160, 1937, 1357, 379, 86, 72, 323, 304, 1624, 212, 1563, 120, 0, 122, 3826, 3638, 1581, 702, 69, 585, 0, 69, 0, 167, 401, 3119, 83, 45, 32, 379, 50, 1338, 2380, 625, 74, 2362, 0, 58, 567, 87, 417, 2498, 797, 766, 1154, 60, 337, 571, 14, 388, 59] payload_rows_per_dst=[7880, 7578, 10534, 11028, 10680, 14076, 14528, 6032] dup_rows_per_dst=[896, 810, 2466, 1436, 882, 3422, 5390, 666] combine_rows_per_src=[7880, 7578, 10534, 11028, 10680, 14076, 14528, 6032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=329304 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284411480 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568493656 weight_movement_bytes=69206016 total_nvlink_bytes=637699672 weight_to_token_ratio=0.121736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=279142400 dispatch_meta_bytes=330384 combine_payload_bytes=279142400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279472784 ingress_bytes=313745408 link_bytes=313745408 token_bytes=558615184 weight_movement_bytes=34603008 total_nvlink_bytes=593218192 weight_to_token_ratio=0.061944 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180598712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81378 payload_rows_remote=72332 dup_rows=16926 dup_rows_remote=16458 dedup_ratio=0.172180 router_tokens_per_expert=[1123, 125, 0, 43, 0, 0, 1740, 4, 26, 6744, 0, 562, 186, 43, 1309, 358, 2, 61, 2331, 128, 0, 419, 285, 1, 0, 0, 3054, 2414, 407, 3, 182, 390, 3, 2, 0, 601, 3, 72, 41, 6214, 0, 2, 73, 25, 1137, 208, 6350, 1, 5824, 54, 0, 601, 0, 244, 424, 578, 85, 0, 0, 3, 3743, 88, 101, 735] payload_rows_per_dst=[7262, 13718, 6162, 7116, 9112, 13832, 15130, 9046] dup_rows_per_dst=[2354, 3212, 292, 1202, 1214, 1756, 6428, 468] combine_rows_per_src=[7262, 13718, 6162, 7116, 9112, 13832, 15130, 9046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=335831040 dispatch_meta_bytes=334240 combine_payload_bytes=335831040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336165280 ingress_bytes=370434048 link_bytes=370434048 token_bytes=671996320 weight_movement_bytes=34603008 total_nvlink_bytes=706599328 weight_to_token_ratio=0.051493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393913500e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=277741568 dispatch_meta_bytes=313544 combine_payload_bytes=277741568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278055112 ingress_bytes=312344576 link_bytes=312344576 token_bytes=555796680 weight_movement_bytes=34603008 total_nvlink_bytes=590399688 weight_to_token_ratio=0.062258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175327494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=296271872 dispatch_meta_bytes=355160 combine_payload_bytes=296271872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296627032 ingress_bytes=330874880 link_bytes=330874880 token_bytes=592898904 weight_movement_bytes=34603008 total_nvlink_bytes=627501912 weight_to_token_ratio=0.058362 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245055536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84014 payload_rows_remote=78900 dup_rows=14290 dup_rows_remote=14256 dedup_ratio=0.145365 router_tokens_per_expert=[680, 505, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 86, 7300, 0, 0, 0, 40, 0, 445, 0, 0, 7594, 612, 0, 71, 0, 120, 8192, 0, 0, 0, 0, 0, 22, 90, 0, 5022, 0, 0, 0, 0, 0, 1088, 8062, 188, 0, 0, 0, 0, 0, 0, 0, 7025, 3, 559, 0, 0, 83, 0, 0, 1288, 47] payload_rows_per_dst=[1804, 5114, 16218, 16384, 10266, 16384, 14056, 3788] dup_rows_per_dst=[626, 34, 9564, 1606, 2, 2292, 0, 166] combine_rows_per_src=[1804, 5114, 16218, 16384, 10266, 16384, 14056, 3788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70044 payload_rows_remote=69270 dup_rows=28260 dup_rows_remote=28260 dedup_ratio=0.287476 router_tokens_per_expert=[0, 0, 178, 0, 0, 209, 0, 0, 1379, 0, 1356, 0, 77, 0, 0, 0, 498, 0, 0, 0, 127, 0, 64, 2943, 175, 276, 0, 0, 0, 4361, 0, 0, 8192, 2, 68, 0, 7667, 0, 2, 0, 19, 7742, 0, 0, 0, 0, 0, 5720, 0, 14, 0, 0, 0, 0, 0, 0, 0, 27, 0, 7981, 0, 75, 0, 0] payload_rows_per_dst=[774, 4142, 7116, 9488, 16384, 16096, 28, 16016] dup_rows_per_dst=[0, 1482, 148, 136, 15478, 10866, 0, 150] combine_rows_per_src=[774, 4142, 7116, 9488, 16384, 16096, 28, 16016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94048 payload_rows_remote=77664 dup_rows=4256 dup_rows_remote=3428 dedup_ratio=0.043294 router_tokens_per_expert=[0, 93, 216, 65, 0, 0, 5884, 0, 5, 0, 1698, 290, 7546, 0, 0, 7328, 1, 0, 0, 0, 0, 4800, 0, 0, 0, 0, 0, 8092, 3, 0, 2041, 0, 208, 0, 99, 0, 0, 2141, 0, 0, 0, 0, 424, 0, 0, 0, 0, 90, 0, 0, 1, 0, 0, 346, 0, 100, 1, 6625, 0, 0, 964, 91, 0, 0] payload_rows_per_dst=[12240, 12560, 9602, 4082, 8976, 16384, 15478, 14726] dup_rows_per_dst=[276, 1204, 0, 6, 798, 828, 508, 636] combine_rows_per_src=[12240, 12560, 9602, 4082, 8976, 16384, 15478, 14726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83718 payload_rows_remote=69436 dup_rows=14586 dup_rows_remote=10522 dedup_ratio=0.148376 router_tokens_per_expert=[3, 56, 479, 0, 0, 0, 4784, 0, 0, 0, 228, 141, 7493, 0, 12, 7508, 7, 0, 0, 0, 0, 4822, 4, 7, 3, 0, 0, 8084, 0, 14, 2437, 0, 146, 1, 76, 0, 0, 4945, 4, 7, 2, 0, 463, 0, 1, 14, 13, 17, 1, 7, 4, 0, 0, 255, 11, 76, 0, 5975, 0, 0, 772, 275, 3, 2] payload_rows_per_dst=[15790, 14992, 9694, 14282, 10052, 3508, 1414, 13986] dup_rows_per_dst=[9288, 556, 10, 4064, 306, 242, 52, 68] combine_rows_per_src=[15790, 14992, 9694, 14282, 10052, 3508, 1414, 13986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83160 payload_rows_remote=74822 dup_rows=15144 dup_rows_remote=15144 dedup_ratio=0.154053 router_tokens_per_expert=[0, 67, 831, 0, 0, 0, 3975, 0, 0, 0, 145, 179, 7402, 0, 0, 7485, 9, 0, 0, 0, 0, 4160, 0, 0, 0, 0, 0, 8097, 0, 0, 3124, 0, 27, 0, 89, 0, 0, 6649, 0, 2, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 279, 0, 90, 0, 5274, 0, 0, 674, 212, 0, 0] payload_rows_per_dst=[16182, 15020, 8338, 16372, 13478, 758, 716, 12296] dup_rows_per_dst=[8534, 432, 0, 6070, 56, 0, 28, 24] combine_rows_per_src=[16182, 15020, 8338, 16372, 13478, 758, 716, 12296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=323174400 dispatch_meta_bytes=372624 combine_payload_bytes=323174400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=323547024 ingress_bytes=496189440 link_bytes=496189440 token_bytes=646721424 weight_movement_bytes=173015040 total_nvlink_bytes=819736464 weight_to_token_ratio=0.267526 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.867120916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283729920 dispatch_meta_bytes=390120 combine_payload_bytes=283729920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284120040 ingress_bytes=318332928 link_bytes=318332928 token_bytes=567849960 weight_movement_bytes=34603008 total_nvlink_bytes=602452968 weight_to_token_ratio=0.060937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197861180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=318111744 dispatch_meta_bytes=324368 combine_payload_bytes=318111744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318436112 ingress_bytes=352714752 link_bytes=352714752 token_bytes=636547856 weight_movement_bytes=34603008 total_nvlink_bytes=671150864 weight_to_token_ratio=0.054360 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327237216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87676 payload_rows_remote=87250 dup_rows=10628 dup_rows_remote=10628 dedup_ratio=0.108114 router_tokens_per_expert=[22, 0, 0, 0, 0, 58, 0, 0, 0, 0, 174, 0, 0, 68, 798, 0, 3193, 0, 133, 7645, 7701, 0, 0, 0, 0, 1, 954, 0, 7658, 0, 68, 0, 4728, 0, 8, 0, 125, 5455, 7663, 80, 0, 0, 0, 0, 0, 18, 0, 60, 0, 1262, 0, 145, 0, 0, 0, 0, 0, 0, 0, 7, 388, 0, 360, 380] payload_rows_per_dst=[9616, 16208, 7674, 16224, 426, 15480, 9152, 12896] dup_rows_per_dst=[0, 1274, 618, 7004, 0, 2, 1446, 284] combine_rows_per_src=[9616, 16208, 7674, 16224, 426, 15480, 9152, 12896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=319832 combine_payload_bytes=284409856 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=284729688 ingress_bytes=388218880 link_bytes=388218880 token_bytes=569139544 weight_movement_bytes=103809024 total_nvlink_bytes=672948568 weight_to_token_ratio=0.182396 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460836391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80262 payload_rows_remote=72364 dup_rows=18042 dup_rows_remote=16768 dedup_ratio=0.183533 router_tokens_per_expert=[30, 77, 515, 4, 1656, 232, 3634, 2232, 243, 52, 172, 332, 71, 146, 318, 0, 1972, 263, 126, 2220, 2378, 1, 558, 247, 146, 38, 137, 0, 2756, 627, 53, 3325, 448, 3317, 308, 0, 270, 1352, 3104, 4978, 37, 130, 0, 2567, 55, 34, 100, 521, 47, 801, 263, 18, 594, 302, 81, 260, 126, 3, 50, 21, 217, 3017, 592, 978] payload_rows_per_dst=[11862, 6764, 6052, 13012, 11580, 11944, 7898, 11150] dup_rows_per_dst=[5794, 660, 282, 846, 6472, 1152, 1274, 1562] combine_rows_per_src=[11862, 6764, 6052, 13012, 11580, 11944, 7898, 11150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=306470912 dispatch_meta_bytes=359864 combine_payload_bytes=306470912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306830776 ingress_bytes=341073920 link_bytes=341073920 token_bytes=613301688 weight_movement_bytes=34603008 total_nvlink_bytes=647904696 weight_to_token_ratio=0.056421 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283433702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79924 payload_rows_remote=64080 dup_rows=18380 dup_rows_remote=5108 dedup_ratio=0.186971 router_tokens_per_expert=[0, 68, 207, 246, 8, 1, 5821, 0, 65, 4, 2943, 293, 7488, 0, 4, 6799, 1, 0, 0, 0, 0, 4456, 1, 2, 1, 0, 0, 8100, 45, 3, 1537, 0, 213, 0, 73, 0, 0, 1750, 20, 2, 0, 0, 427, 0, 22, 3, 1, 273, 0, 0, 7, 69, 16, 481, 2, 84, 33, 5582, 0, 137, 1724, 137, 1, 2] payload_rows_per_dst=[12112, 6550, 8924, 3110, 14298, 3664, 15422, 15844] dup_rows_per_dst=[590, 60, 4, 62, 3416, 104, 872, 13272] combine_rows_per_src=[12112, 6550, 8924, 3110, 14298, 3664, 15422, 15844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=357376000 dispatch_meta_bytes=391512 combine_payload_bytes=357376000 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=357767512 ingress_bytes=530391040 link_bytes=530391040 token_bytes=715143512 weight_movement_bytes=173015040 total_nvlink_bytes=888158552 weight_to_token_ratio=0.241931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.995818784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=296402944 dispatch_meta_bytes=356528 combine_payload_bytes=296402944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296759472 ingress_bytes=331005952 link_bytes=331005952 token_bytes=593162416 weight_movement_bytes=34603008 total_nvlink_bytes=627765424 weight_to_token_ratio=0.058336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245548749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95518 payload_rows_remote=95080 dup_rows=2786 dup_rows_remote=2786 dedup_ratio=0.028341 router_tokens_per_expert=[0, 0, 837, 0, 8192, 0, 0, 0, 212, 0, 0, 7, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 5943, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7668, 0, 0, 0, 0, 0, 0, 558, 0, 0, 7945, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 438, 15528, 3562, 11886, 15336, 16000, 16384] dup_rows_per_dst=[1674, 0, 0, 0, 0, 104, 1006, 2] combine_rows_per_src=[16384, 438, 15528, 3562, 11886, 15336, 16000, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=262471680 dispatch_meta_bytes=276752 combine_payload_bytes=262471680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262748432 ingress_bytes=297074688 link_bytes=297074688 token_bytes=525220112 weight_movement_bytes=34603008 total_nvlink_bytes=559823120 weight_to_token_ratio=0.065883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117868134e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89638 payload_rows_remote=88182 dup_rows=8666 dup_rows_remote=8074 dedup_ratio=0.088155 router_tokens_per_expert=[500, 481, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 59, 7002, 0, 0, 0, 6, 0, 465, 0, 4, 7333, 168, 0, 72, 0, 25, 8170, 0, 0, 1, 0, 0, 27, 147, 0, 5000, 0, 12, 0, 0, 0, 782, 8061, 966, 0, 0, 0, 0, 0, 0, 0, 7245, 0, 265, 0, 0, 367, 0, 0, 1894, 57] payload_rows_per_dst=[1456, 14512, 15378, 12874, 10200, 16324, 14490, 4404] dup_rows_per_dst=[592, 3160, 238, 446, 150, 3318, 0, 762] combine_rows_per_src=[1456, 14512, 15378, 12874, 10200, 16324, 14490, 4404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91094 payload_rows_remote=74918 dup_rows=7210 dup_rows_remote=5262 dedup_ratio=0.073344 router_tokens_per_expert=[1, 0, 0, 0, 0, 204, 519, 0, 0, 0, 108, 0, 0, 55, 117, 0, 4786, 0, 105, 7569, 7523, 0, 0, 0, 0, 203, 368, 0, 7650, 0, 55, 0, 1401, 0, 0, 0, 218, 6679, 7973, 101, 0, 0, 0, 0, 0, 24, 0, 1065, 0, 1466, 0, 167, 0, 0, 0, 0, 0, 0, 0, 2, 361, 0, 203, 229] payload_rows_per_dst=[3856, 15360, 9782, 15824, 638, 16176, 15482, 13976] dup_rows_per_dst=[394, 246, 0, 728, 0, 1948, 2922, 972] combine_rows_per_src=[3856, 15360, 9782, 15824, 638, 16176, 15482, 13976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80400 payload_rows_remote=64210 dup_rows=17904 dup_rows_remote=16894 dedup_ratio=0.182129 router_tokens_per_expert=[10, 0, 11, 0, 13, 66, 75, 0, 0, 0, 112, 6, 0, 62, 389, 0, 4254, 11, 97, 7622, 7581, 0, 7, 4, 0, 117, 714, 0, 7703, 1, 65, 0, 3021, 6, 11, 2, 195, 6403, 7566, 76, 0, 4, 0, 12, 1, 4, 0, 138, 0, 1586, 0, 117, 0, 13, 0, 0, 2, 0, 16, 9, 443, 13, 336, 258] payload_rows_per_dst=[6380, 15808, 15620, 16190, 14970, 1108, 3432, 6892] dup_rows_per_dst=[12, 2148, 6714, 1010, 7512, 12, 0, 496] combine_rows_per_src=[6380, 15808, 15620, 16190, 14970, 1108, 3432, 6892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78716 payload_rows_remote=63330 dup_rows=19588 dup_rows_remote=19588 dedup_ratio=0.199259 router_tokens_per_expert=[3, 0, 0, 0, 0, 69, 0, 0, 0, 0, 143, 0, 0, 53, 490, 0, 3284, 0, 118, 7575, 7586, 0, 0, 0, 0, 48, 887, 0, 7701, 0, 53, 0, 4329, 0, 5, 0, 105, 5968, 7781, 90, 0, 0, 0, 0, 0, 10, 0, 151, 0, 1485, 0, 159, 0, 0, 0, 0, 0, 0, 2, 6, 384, 0, 324, 343] payload_rows_per_dst=[12368, 16090, 15386, 16204, 12948, 322, 3288, 2110] dup_rows_per_dst=[6142, 7022, 0, 1174, 5242, 0, 0, 8] combine_rows_per_src=[12368, 16090, 15386, 16204, 12948, 322, 3288, 2110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=389447680 dispatch_meta_bytes=391464 combine_payload_bytes=389447680 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=389839144 ingress_bytes=458653696 link_bytes=458653696 token_bytes=779286824 weight_movement_bytes=69206016 total_nvlink_bytes=848492840 weight_to_token_ratio=0.088807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725876933e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=361193472 dispatch_meta_bytes=385024 combine_payload_bytes=361193472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=361578496 ingress_bytes=395796480 link_bytes=395796480 token_bytes=722771968 weight_movement_bytes=34603008 total_nvlink_bytes=757374976 weight_to_token_ratio=0.047875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489350290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=306864128 dispatch_meta_bytes=320720 combine_payload_bytes=306864128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307184848 ingress_bytes=341467136 link_bytes=341467136 token_bytes=614048976 weight_movement_bytes=34603008 total_nvlink_bytes=648651984 weight_to_token_ratio=0.056352 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284913342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=263004160 dispatch_meta_bytes=324416 combine_payload_bytes=263004160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263328576 ingress_bytes=332210176 link_bytes=332210176 token_bytes=526332736 weight_movement_bytes=69206016 total_nvlink_bytes=595538752 weight_to_token_ratio=0.131487 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250080147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81714 payload_rows_remote=71674 dup_rows=16590 dup_rows_remote=14914 dedup_ratio=0.168762 router_tokens_per_expert=[0, 9, 1940, 703, 291, 0, 117, 100, 0, 1692, 165, 3129, 536, 1011, 111, 0, 170, 0, 1289, 376, 4114, 491, 4693, 83, 2974, 966, 29, 4021, 3874, 8, 79, 27, 0, 256, 153, 921, 240, 73, 0, 99, 929, 122, 268, 1385, 1480, 0, 117, 1727, 602, 435, 337, 119, 226, 65, 44, 9, 642, 132, 133, 351, 303, 4490, 487, 9] payload_rows_per_dst=[5930, 11138, 12248, 12024, 9094, 9800, 10040, 11440] dup_rows_per_dst=[390, 2150, 1498, 3890, 498, 4834, 1676, 1654] combine_rows_per_src=[5930, 11138, 12248, 12024, 9094, 9800, 10040, 11440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81898 payload_rows_remote=77916 dup_rows=16406 dup_rows_remote=16400 dedup_ratio=0.166890 router_tokens_per_expert=[0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 16, 80, 0, 80, 2230, 0, 0, 0, 2810, 1493, 8016, 782, 0, 80, 7584, 0, 0, 7461, 470, 68, 0, 0, 0, 0, 103, 0, 3, 0, 0, 1888, 0, 1935, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1992, 5263, 0, 1234, 146, 1668, 80, 0, 0, 0, 1546, 43, 0] payload_rows_per_dst=[10798, 16222, 11762, 15746, 3982, 4034, 13012, 6342] dup_rows_per_dst=[2460, 3758, 5340, 252, 6, 0, 4258, 332] combine_rows_per_src=[10798, 16222, 11762, 15746, 3982, 4034, 13012, 6342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259399680 dispatch_meta_bytes=331672 combine_payload_bytes=259399680 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=259731352 ingress_bytes=397811712 link_bytes=397811712 token_bytes=519131032 weight_movement_bytes=138412032 total_nvlink_bytes=657543064 weight_to_token_ratio=0.266623 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496933446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89144 payload_rows_remote=75158 dup_rows=9160 dup_rows_remote=8114 dedup_ratio=0.093180 router_tokens_per_expert=[11, 0, 2, 0, 3, 221, 750, 0, 0, 8, 64, 0, 0, 65, 90, 0, 3896, 2, 74, 7565, 7518, 0, 1, 2, 0, 193, 279, 0, 7472, 2, 64, 0, 1357, 1, 4, 0, 215, 6572, 7952, 880, 0, 0, 0, 3, 0, 42, 0, 1531, 0, 1300, 0, 66, 0, 3, 0, 0, 3, 0, 4, 5, 367, 3, 275, 287] payload_rows_per_dst=[4156, 15220, 7948, 1076, 15178, 16252, 15328, 13986] dup_rows_per_dst=[532, 270, 2, 0, 1966, 2804, 2540, 1046] combine_rows_per_src=[4156, 15220, 7948, 1076, 15178, 16252, 15328, 13986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293576704 dispatch_meta_bytes=346352 combine_payload_bytes=293576704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293923056 ingress_bytes=328179712 link_bytes=328179712 token_bytes=587499760 weight_movement_bytes=34603008 total_nvlink_bytes=622102768 weight_to_token_ratio=0.058899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234913836e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=319143936 dispatch_meta_bytes=377264 combine_payload_bytes=319143936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319521200 ingress_bytes=353746944 link_bytes=353746944 token_bytes=638665136 weight_movement_bytes=34603008 total_nvlink_bytes=673268144 weight_to_token_ratio=0.054180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331121271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=72258 dup_rows=24092 dup_rows_remote=24092 dedup_ratio=0.245076 router_tokens_per_expert=[0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 92, 0, 0, 0, 0, 0, 0, 52, 0, 0, 7136, 0, 7591, 0, 0, 0, 0, 4297, 0, 482, 8191, 349, 0, 0, 0, 0, 528, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0] payload_rows_per_dst=[16280, 1954, 5350, 104, 16256, 16384, 2708, 15176] dup_rows_per_dst=[0, 0, 2, 0, 13198, 10254, 638, 0] combine_rows_per_src=[16280, 1954, 5350, 104, 16256, 16384, 2708, 15176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92562 payload_rows_remote=76178 dup_rows=5742 dup_rows_remote=2992 dedup_ratio=0.058411 router_tokens_per_expert=[0, 0, 1375, 0, 8192, 0, 0, 0, 134, 0, 0, 264, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 6093, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4743, 0, 0, 0, 0, 0, 0, 0, 139, 0, 7693, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 7942, 0, 0, 78, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 796, 12698, 5450, 9486, 15386, 15978, 16384] dup_rows_per_dst=[2750, 0, 0, 0, 0, 278, 2558, 156] combine_rows_per_src=[16384, 796, 12698, 5450, 9486, 15386, 15978, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307847168 dispatch_meta_bytes=333088 combine_payload_bytes=307847168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308180256 ingress_bytes=342450176 link_bytes=342450176 token_bytes=616027424 weight_movement_bytes=34603008 total_nvlink_bytes=650630432 weight_to_token_ratio=0.056171 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288612443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78836 payload_rows_remote=71084 dup_rows=19468 dup_rows_remote=19388 dedup_ratio=0.198039 router_tokens_per_expert=[0, 0, 0, 1736, 0, 0, 0, 0, 0, 0, 920, 2429, 0, 100, 601, 0, 0, 0, 4682, 1482, 8042, 820, 0, 100, 7879, 0, 0, 7503, 1068, 51, 0, 0, 0, 0, 14, 0, 50, 0, 0, 546, 0, 770, 0, 0, 0, 0, 160, 0, 0, 0, 159, 989, 1532, 0, 325, 89, 1088, 99, 0, 0, 0, 5140, 733, 45] payload_rows_per_dst=[3472, 8310, 16384, 15948, 1120, 7752, 13700, 12150] dup_rows_per_dst=[0, 1202, 7896, 5574, 100, 80, 2556, 2060] combine_rows_per_src=[3472, 8310, 16384, 15948, 1120, 7752, 13700, 12150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85210 payload_rows_remote=69534 dup_rows=13094 dup_rows_remote=12422 dedup_ratio=0.133199 router_tokens_per_expert=[0, 0, 10, 1816, 0, 14, 0, 0, 0, 0, 226, 93, 8, 75, 1688, 10, 0, 0, 3881, 1858, 8033, 798, 4, 76, 7735, 5, 0, 7478, 627, 64, 0, 0, 0, 12, 89, 12, 54, 0, 0, 1672, 6, 1295, 0, 13, 1, 0, 146, 0, 5, 0, 3, 1946, 3676, 2, 928, 126, 1235, 77, 5, 0, 12, 3209, 124, 5] payload_rows_per_dst=[16134, 16190, 11206, 15676, 3550, 2908, 10756, 8790] dup_rows_per_dst=[3612, 3480, 2028, 672, 128, 14, 2616, 544] combine_rows_per_src=[16134, 16190, 11206, 15676, 3550, 2908, 10756, 8790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=295968768 dispatch_meta_bytes=385400 combine_payload_bytes=295968768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296354168 ingress_bytes=330571776 link_bytes=330571776 token_bytes=592322936 weight_movement_bytes=34603008 total_nvlink_bytes=626925944 weight_to_token_ratio=0.058419 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243914980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81494 payload_rows_remote=67888 dup_rows=16810 dup_rows_remote=11876 dedup_ratio=0.171000 router_tokens_per_expert=[0, 0, 0, 1876, 0, 0, 0, 0, 0, 0, 113, 89, 0, 88, 2123, 0, 0, 0, 3169, 1737, 8021, 766, 0, 88, 7631, 0, 0, 7510, 517, 53, 0, 0, 0, 0, 100, 0, 30, 0, 0, 2017, 0, 1607, 0, 0, 0, 0, 132, 0, 0, 0, 0, 1928, 4728, 0, 1032, 161, 1496, 89, 0, 0, 0, 1974, 70, 7] payload_rows_per_dst=[16048, 16196, 13606, 8774, 4234, 3478, 12278, 6880] dup_rows_per_dst=[3746, 3892, 4934, 366, 60, 0, 3420, 392] combine_rows_per_src=[16048, 16196, 13606, 8774, 4234, 3478, 12278, 6880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=312025088 dispatch_meta_bytes=316680 combine_payload_bytes=312025088 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=312341768 ingress_bytes=381231104 link_bytes=381231104 token_bytes=624366856 weight_movement_bytes=69206016 total_nvlink_bytes=693572872 weight_to_token_ratio=0.110842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434541953e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=291160064 dispatch_meta_bytes=361888 combine_payload_bytes=291160064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291521952 ingress_bytes=325763072 link_bytes=325763072 token_bytes=582682016 weight_movement_bytes=34603008 total_nvlink_bytes=617285024 weight_to_token_ratio=0.059386 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225820214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=284811264 dispatch_meta_bytes=327824 combine_payload_bytes=284811264 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=285139088 ingress_bytes=423223296 link_bytes=423223296 token_bytes=569950352 weight_movement_bytes=138412032 total_nvlink_bytes=708362384 weight_to_token_ratio=0.242849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.592555191e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77852 payload_rows_remote=65108 dup_rows=20452 dup_rows_remote=17064 dedup_ratio=0.208049 router_tokens_per_expert=[1030, 1565, 455, 425, 2073, 1877, 612, 29, 216, 288, 2, 726, 2492, 0, 951, 147, 1534, 177, 187, 4777, 184, 33, 2148, 0, 0, 0, 3281, 550, 86, 535, 343, 59, 0, 0, 164, 454, 0, 2891, 393, 181, 87, 4628, 1117, 0, 143, 150, 0, 0, 43, 3884, 0, 0, 145, 931, 88, 2975, 1404, 203, 148, 24, 1802, 122, 51, 342] payload_rows_per_dst=[4386, 11016, 14532, 8260, 7472, 12296, 12744, 7146] dup_rows_per_dst=[716, 2774, 7302, 1448, 694, 3084, 3388, 1046] combine_rows_per_src=[4386, 11016, 14532, 8260, 7472, 12296, 12744, 7146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83156 payload_rows_remote=67862 dup_rows=15148 dup_rows_remote=7618 dedup_ratio=0.154093 router_tokens_per_expert=[0, 5777, 565, 1181, 7529, 4327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 933, 0, 0, 0, 0, 281, 3209, 80, 0, 0, 0, 0, 0, 580, 0, 0, 5959, 4709, 164, 0, 0, 736, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 97, 7279, 120, 4134, 0, 0, 80, 0, 0, 0] payload_rows_per_dst=[6810, 15058, 5848, 12778, 15294, 12712, 6164, 8492] dup_rows_per_dst=[1660, 0, 170, 4820, 7530, 112, 680, 176] combine_rows_per_src=[6810, 15058, 5848, 12778, 15294, 12712, 6164, 8492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=278069248 dispatch_meta_bytes=319056 combine_payload_bytes=278069248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278388304 ingress_bytes=416481280 link_bytes=416481280 token_bytes=556457552 weight_movement_bytes=138412032 total_nvlink_bytes=694869584 weight_to_token_ratio=0.248738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567185527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85120 payload_rows_remote=72148 dup_rows=13184 dup_rows_remote=9216 dedup_ratio=0.134115 router_tokens_per_expert=[0, 0, 2, 2019, 0, 2, 10, 0, 0, 3, 916, 3835, 1, 310, 481, 1, 0, 0, 4054, 1077, 8023, 704, 0, 74, 7805, 2, 0, 7519, 1265, 33, 0, 3, 0, 3, 15, 4, 61, 2, 0, 310, 1, 552, 1, 3, 0, 0, 156, 3, 1, 1, 183, 699, 1185, 0, 314, 12, 1010, 74, 1, 0, 3, 5544, 839, 36] payload_rows_per_dst=[4062, 9520, 3388, 16054, 16058, 9382, 13684, 12972] dup_rows_per_dst=[4, 1574, 322, 2162, 778, 158, 4218, 3968] combine_rows_per_src=[4062, 9520, 3388, 16054, 16058, 9382, 13684, 12972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=328688 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267011056 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533693424 weight_movement_bytes=103809024 total_nvlink_bytes=637502448 weight_to_token_ratio=0.194511 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=277962752 dispatch_meta_bytes=301920 combine_payload_bytes=277962752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278264672 ingress_bytes=347168768 link_bytes=347168768 token_bytes=556227424 weight_movement_bytes=69206016 total_nvlink_bytes=625433440 weight_to_token_ratio=0.124420 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306368125e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89386 payload_rows_remote=73450 dup_rows=8918 dup_rows_remote=6466 dedup_ratio=0.090719 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 7606, 0, 8192, 92, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6922, 184, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 54, 0, 0, 0, 657, 8192, 0, 0, 0, 0, 0, 5888, 0, 80, 0, 3073] payload_rows_per_dst=[0, 15936, 16384, 14212, 128, 12160, 16384, 14182] dup_rows_per_dst=[0, 2452, 1144, 0, 0, 0, 1422, 3900] combine_rows_per_src=[0, 15936, 16384, 14212, 128, 12160, 16384, 14182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71556 payload_rows_remote=56188 dup_rows=26748 dup_rows_remote=26748 dedup_ratio=0.272095 router_tokens_per_expert=[0, 0, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 0, 0, 0, 72, 0, 0, 0, 0, 0, 1026, 72, 0, 484, 0, 0, 0, 0, 47, 0, 0, 6863, 0, 7758, 0, 0, 0, 0, 5711, 0, 458, 8115, 329, 0, 0, 0, 0, 551, 0, 0, 0, 821, 0, 0, 0, 0, 0, 0, 7802, 0, 0, 0] payload_rows_per_dst=[15368, 2718, 2200, 1062, 16256, 16244, 2104, 15604] dup_rows_per_dst=[0, 0, 140, 0, 12986, 12982, 640, 0] combine_rows_per_src=[15368, 2718, 2200, 1062, 16256, 16244, 2104, 15604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84944 payload_rows_remote=69532 dup_rows=13360 dup_rows_remote=8640 dedup_ratio=0.135905 router_tokens_per_expert=[0, 6456, 705, 1673, 6746, 6558, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 48, 171, 0, 341, 0, 0, 0, 0, 625, 688, 99, 16, 0, 0, 0, 0, 295, 0, 0, 7041, 1841, 133, 0, 1830, 1780, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 95, 6210, 1291, 1747, 0, 0, 104, 0, 0, 0] payload_rows_per_dst=[4066, 13614, 13814, 2488, 15348, 15412, 14244, 5958] dup_rows_per_dst=[690, 0, 422, 368, 3272, 4720, 3562, 326] combine_rows_per_src=[4066, 13614, 13814, 2488, 15348, 15412, 14244, 5958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=295518208 dispatch_meta_bytes=325456 combine_payload_bytes=295518208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295843664 ingress_bytes=330121216 link_bytes=330121216 token_bytes=591361872 weight_movement_bytes=34603008 total_nvlink_bytes=625964880 weight_to_token_ratio=0.058514 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242219559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70996 payload_rows_remote=55178 dup_rows=27308 dup_rows_remote=23692 dedup_ratio=0.277791 router_tokens_per_expert=[0, 6169, 605, 1243, 7309, 5362, 2, 0, 0, 0, 3, 1, 5, 0, 6, 6, 14, 1, 142, 18, 6, 750, 0, 0, 0, 0, 428, 1932, 76, 0, 10, 0, 0, 0, 509, 0, 1, 6597, 3628, 149, 0, 111, 981, 0, 12, 0, 2, 18, 1, 1226, 19, 0, 0, 15, 88, 7271, 284, 4057, 3, 11, 76, 1, 1, 3] payload_rows_per_dst=[16280, 8370, 1766, 15818, 15446, 2048, 2684, 8584] dup_rows_per_dst=[16764, 8, 96, 3616, 6322, 200, 14, 288] combine_rows_per_src=[16280, 8370, 1766, 15818, 15446, 2048, 2684, 8584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300851200 dispatch_meta_bytes=319664 combine_payload_bytes=300851200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301170864 ingress_bytes=335454208 link_bytes=335454208 token_bytes=602022064 weight_movement_bytes=34603008 total_nvlink_bytes=636625072 weight_to_token_ratio=0.057478 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61464 payload_rows_remote=59456 dup_rows=36840 dup_rows_remote=36766 dedup_ratio=0.374756 router_tokens_per_expert=[0, 5909, 659, 1215, 7504, 4640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 916, 0, 0, 0, 0, 339, 2669, 89, 0, 0, 0, 0, 0, 523, 0, 0, 6215, 4462, 127, 0, 0, 796, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 0, 0, 74, 7275, 185, 4096, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16384, 0, 2008, 15124, 15216, 1592, 2638, 8502] dup_rows_per_dst=[25514, 0, 74, 3576, 7438, 0, 0, 238] combine_rows_per_src=[16384, 0, 2008, 15124, 15216, 1592, 2638, 8502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=230146048 dispatch_meta_bytes=331744 combine_payload_bytes=230146048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230477792 ingress_bytes=264749056 link_bytes=264749056 token_bytes=460623840 weight_movement_bytes=34603008 total_nvlink_bytes=495226848 weight_to_token_ratio=0.075122 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.962293837e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=284803072 dispatch_meta_bytes=312688 combine_payload_bytes=284803072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285115760 ingress_bytes=319406080 link_bytes=319406080 token_bytes=569918832 weight_movement_bytes=34603008 total_nvlink_bytes=604521840 weight_to_token_ratio=0.060716 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201899365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=226009088 dispatch_meta_bytes=315480 combine_payload_bytes=226009088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=226324568 ingress_bytes=260612096 link_bytes=260612096 token_bytes=452333656 weight_movement_bytes=34603008 total_nvlink_bytes=486936664 weight_to_token_ratio=0.076499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.806623363e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76936 payload_rows_remote=72164 dup_rows=21368 dup_rows_remote=19756 dedup_ratio=0.217367 router_tokens_per_expert=[427, 282, 321, 49, 0, 5877, 0, 781, 2322, 154, 6, 0, 122, 1537, 17, 439, 319, 41, 44, 1825, 812, 1895, 0, 263, 103, 0, 2598, 0, 246, 113, 2729, 0, 708, 0, 171, 181, 16, 20, 0, 2664, 1388, 58, 214, 5261, 165, 1685, 447, 2754, 20, 0, 0, 337, 1850, 5443, 0, 985, 46, 85, 295, 601, 160, 0, 99, 177] payload_rows_per_dst=[13996, 8588, 7102, 13766, 6990, 9170, 4772, 12552] dup_rows_per_dst=[1478, 606, 3296, 8334, 530, 4252, 1612, 1260] combine_rows_per_src=[13996, 8588, 7102, 13766, 6990, 9170, 4772, 12552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=243531776 dispatch_meta_bytes=384888 combine_payload_bytes=243531776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243916664 ingress_bytes=278134784 link_bytes=278134784 token_bytes=487448440 weight_movement_bytes=34603008 total_nvlink_bytes=522051448 weight_to_token_ratio=0.070988 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.046598801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83546 payload_rows_remote=70120 dup_rows=14758 dup_rows_remote=12558 dedup_ratio=0.150126 router_tokens_per_expert=[7556, 0, 0, 0, 0, 0, 0, 1997, 592, 0, 118, 0, 0, 0, 1, 1600, 0, 0, 0, 5604, 276, 516, 0, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 7308, 0, 0, 194, 0, 0, 0, 0, 2893, 0, 7968, 0, 0, 0, 0, 0, 3478, 177, 0, 0, 80] payload_rows_per_dst=[15940, 5024, 11982, 14616, 13426, 388, 5786, 16384] dup_rows_per_dst=[3166, 1072, 1298, 0, 2200, 0, 0, 7022] combine_rows_per_src=[15940, 5024, 11982, 14616, 13426, 388, 5786, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80878 payload_rows_remote=66470 dup_rows=17426 dup_rows_remote=12716 dedup_ratio=0.177266 router_tokens_per_expert=[99, 6079, 612, 1806, 6578, 6254, 0, 0, 1, 0, 0, 133, 1, 0, 1, 5, 3, 0, 70, 1314, 1, 168, 12, 0, 0, 0, 578, 516, 74, 94, 3, 0, 0, 0, 169, 0, 0, 6915, 1532, 172, 0, 3173, 1392, 0, 3, 0, 0, 5, 1, 2779, 7, 0, 0, 4, 123, 5388, 1685, 1176, 1, 4, 220, 0, 0, 1] payload_rows_per_dst=[4274, 13428, 13604, 2380, 3648, 15532, 13604, 14408] dup_rows_per_dst=[760, 10, 2040, 150, 368, 5772, 3616, 4710] combine_rows_per_src=[4274, 13428, 13604, 2380, 3648, 15532, 13604, 14408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=295583744 dispatch_meta_bytes=367680 combine_payload_bytes=295583744 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=295951424 ingress_bytes=433995776 link_bytes=433995776 token_bytes=591535168 weight_movement_bytes=138412032 total_nvlink_bytes=729947200 weight_to_token_ratio=0.233988 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633091166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64346 payload_rows_remote=48146 dup_rows=33958 dup_rows_remote=33122 dedup_ratio=0.345439 router_tokens_per_expert=[0, 830, 77, 0, 0, 0, 177, 0, 0, 8100, 0, 418, 0, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 540, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 1531, 8192, 0, 0, 0, 0, 0, 0, 7271, 0, 0, 8015, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2168, 16200, 11262, 1426, 516, 16384, 16384, 6] dup_rows_per_dst=[0, 836, 0, 0, 0, 18934, 14188, 0] combine_rows_per_src=[2168, 16200, 11262, 1426, 516, 16384, 16384, 6] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=287211520 dispatch_meta_bytes=330712 combine_payload_bytes=287211520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287542232 ingress_bytes=321814528 link_bytes=321814528 token_bytes=574753752 weight_movement_bytes=34603008 total_nvlink_bytes=609356760 weight_to_token_ratio=0.060205 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210962161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88446 payload_rows_remote=88446 dup_rows=9858 dup_rows_remote=9858 dedup_ratio=0.100281 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 10, 6438, 0, 8192, 117, 0, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6489, 144, 0, 0, 0, 64, 0, 0, 0, 0, 46, 0, 0, 0, 6286, 0, 0, 0, 63, 0, 0, 0, 325, 8192, 0, 0, 0, 0, 0, 6209, 0, 259, 0, 3706] payload_rows_per_dst=[0, 14404, 16384, 13266, 128, 12602, 16384, 15278] dup_rows_per_dst=[0, 1234, 2716, 0, 0, 62, 776, 5070] combine_rows_per_src=[0, 14404, 16384, 13266, 128, 12602, 16384, 15278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76552 payload_rows_remote=75568 dup_rows=21752 dup_rows_remote=21752 dedup_ratio=0.221273 router_tokens_per_expert=[3095, 0, 0, 0, 0, 88, 0, 1588, 1054, 0, 102, 0, 0, 0, 32, 2253, 0, 0, 0, 6599, 473, 4786, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 5530, 0, 0, 0, 7775, 0, 0, 492, 0, 0, 0, 0, 947, 0, 8019, 0, 0, 0, 0, 0, 3245, 166, 0, 0, 123] payload_rows_per_dst=[8204, 5808, 15028, 15550, 12700, 984, 1894, 16384] dup_rows_per_dst=[1338, 1074, 9066, 0, 3552, 0, 0, 6722] combine_rows_per_src=[8204, 5808, 15028, 15550, 12700, 984, 1894, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272261120 dispatch_meta_bytes=316744 combine_payload_bytes=272261120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272577864 ingress_bytes=306864128 link_bytes=306864128 token_bytes=544838984 weight_movement_bytes=34603008 total_nvlink_bytes=579441992 weight_to_token_ratio=0.063511 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154705009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84284 payload_rows_remote=69404 dup_rows=14020 dup_rows_remote=14020 dedup_ratio=0.142619 router_tokens_per_expert=[6191, 11, 0, 1, 0, 0, 1, 1903, 538, 15, 193, 1, 0, 0, 25, 1816, 0, 7, 1, 6088, 257, 1589, 0, 701, 0, 0, 0, 0, 0, 1, 0, 6, 2492, 8, 0, 0, 0, 0, 0, 5939, 0, 13, 0, 7433, 3, 14, 255, 0, 0, 0, 0, 2266, 6, 7906, 0, 12, 0, 7, 0, 3212, 164, 1, 0, 76] payload_rows_per_dst=[13734, 14614, 5108, 14880, 13420, 540, 13624, 8364] dup_rows_per_dst=[2480, 2738, 2, 0, 3458, 30, 4220, 1092] combine_rows_per_src=[13734, 14614, 5108, 14880, 13420, 540, 13624, 8364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84430 payload_rows_remote=80574 dup_rows=13874 dup_rows_remote=13818 dedup_ratio=0.141134 router_tokens_per_expert=[7054, 0, 0, 0, 0, 0, 0, 2030, 539, 0, 158, 0, 0, 0, 41, 1598, 0, 0, 0, 5927, 268, 746, 0, 783, 0, 0, 0, 0, 0, 0, 0, 0, 1686, 0, 0, 0, 0, 0, 0, 5997, 0, 0, 0, 7399, 0, 0, 219, 0, 0, 0, 0, 3310, 0, 7899, 0, 0, 0, 0, 0, 3245, 163, 0, 0, 90] payload_rows_per_dst=[15164, 14300, 3856, 14798, 13266, 438, 15612, 6996] dup_rows_per_dst=[3004, 2226, 56, 0, 2100, 0, 6488, 0] combine_rows_per_src=[15164, 14300, 3856, 14798, 13266, 438, 15612, 6996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=197206016 dispatch_meta_bytes=325072 combine_payload_bytes=197206016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=197531088 ingress_bytes=231809024 link_bytes=231809024 token_bytes=394737104 weight_movement_bytes=34603008 total_nvlink_bytes=429340112 weight_to_token_ratio=0.087661 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.722786952e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=362274816 dispatch_meta_bytes=393216 combine_payload_bytes=362274816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362668032 ingress_bytes=500686848 link_bytes=500686848 token_bytes=724942848 weight_movement_bytes=138412032 total_nvlink_bytes=863354880 weight_to_token_ratio=0.190928 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.884044300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309526528 dispatch_meta_bytes=389280 combine_payload_bytes=309526528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=309915808 ingress_bytes=378732544 link_bytes=378732544 token_bytes=619442336 weight_movement_bytes=69206016 total_nvlink_bytes=688648352 weight_to_token_ratio=0.111723 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425140073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83706 payload_rows_remote=76224 dup_rows=14598 dup_rows_remote=14472 dedup_ratio=0.148499 router_tokens_per_expert=[131, 75, 5338, 0, 177, 2161, 3, 22, 16, 892, 87, 3, 3178, 64, 5643, 0, 63, 364, 120, 0, 26, 2907, 2170, 157, 927, 48, 570, 615, 0, 80, 650, 5554, 4225, 0, 471, 0, 27, 0, 63, 1562, 1394, 110, 901, 16, 81, 1366, 2180, 187, 112, 18, 81, 53, 252, 21, 84, 5, 1146, 15, 799, 319, 112, 205, 1115, 191] payload_rows_per_dst=[12850, 12114, 12408, 14260, 4176, 6964, 7482, 13452] dup_rows_per_dst=[2964, 1296, 3566, 2628, 70, 1146, 126, 2802] combine_rows_per_src=[12850, 12114, 12408, 14260, 4176, 6964, 7482, 13452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284278784 dispatch_meta_bytes=333696 combine_payload_bytes=284278784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284612480 ingress_bytes=318881792 link_bytes=318881792 token_bytes=568891264 weight_movement_bytes=34603008 total_nvlink_bytes=603494272 weight_to_token_ratio=0.060825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199926511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93656 payload_rows_remote=80138 dup_rows=4648 dup_rows_remote=4086 dedup_ratio=0.047282 router_tokens_per_expert=[0, 0, 7731, 0, 0, 0, 0, 0, 0, 246, 0, 0, 7865, 0, 543, 0, 21, 380, 68, 0, 0, 0, 194, 0, 251, 0, 0, 0, 0, 0, 0, 6815, 6759, 0, 142, 0, 0, 0, 139, 0, 879, 786, 0, 0, 0, 6326, 8036, 0, 0, 0, 0, 0, 247, 0, 80, 0, 0, 0, 1644, 0, 0, 0, 0, 0] payload_rows_per_dst=[15462, 1086, 16208, 13632, 13518, 14078, 16384, 3288] dup_rows_per_dst=[0, 492, 1190, 500, 562, 1904, 0, 0] combine_rows_per_src=[15462, 1086, 16208, 13632, 13518, 14078, 16384, 3288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=330031104 dispatch_meta_bytes=377568 combine_payload_bytes=330031104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=330408672 ingress_bytes=433840128 link_bytes=433840128 token_bytes=660439776 weight_movement_bytes=103809024 total_nvlink_bytes=764248800 weight_to_token_ratio=0.157182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.632505475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80020 payload_rows_remote=63642 dup_rows=18284 dup_rows_remote=13440 dedup_ratio=0.185994 router_tokens_per_expert=[2376, 4, 0, 0, 0, 1010, 0, 1582, 2157, 4, 119, 0, 26, 0, 45, 1965, 7, 1, 0, 6582, 667, 5050, 0, 194, 0, 0, 0, 0, 0, 0, 0, 1, 2270, 3, 0, 0, 0, 0, 0, 5025, 71, 4, 5, 7778, 2, 3, 890, 79, 0, 0, 0, 618, 1, 8095, 0, 2, 1, 1, 0, 2253, 142, 0, 0, 119] payload_rows_per_dst=[8736, 7358, 13948, 13518, 11750, 7092, 1240, 16378] dup_rows_per_dst=[1208, 1274, 5228, 0, 2846, 2882, 2, 4844] combine_rows_per_src=[8736, 7358, 13948, 13518, 11750, 7092, 1240, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312213504 dispatch_meta_bytes=362784 combine_payload_bytes=312213504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312576288 ingress_bytes=346816512 link_bytes=346816512 token_bytes=624789792 weight_movement_bytes=34603008 total_nvlink_bytes=659392800 weight_to_token_ratio=0.055383 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305042614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328245248 dispatch_meta_bytes=336896 combine_payload_bytes=328245248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=328582144 ingress_bytes=432054272 link_bytes=432054272 token_bytes=656827392 weight_movement_bytes=103809024 total_nvlink_bytes=760636416 weight_to_token_ratio=0.158046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.625785442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82590 payload_rows_remote=74772 dup_rows=15714 dup_rows_remote=15714 dedup_ratio=0.159851 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 103, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 2352, 8192, 0, 0, 0, 7873, 0, 0, 1319, 0, 0, 0, 0, 0, 8192, 0, 4428, 0, 0, 0, 0] payload_rows_per_dst=[206, 7818, 0, 9416, 16152, 16384, 16230, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4704, 2154, 8856] combine_rows_per_src=[206, 7818, 0, 9416, 16152, 16384, 16230, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71452 payload_rows_remote=64150 dup_rows=26852 dup_rows_remote=26466 dedup_ratio=0.273153 router_tokens_per_expert=[0, 967, 72, 0, 0, 0, 249, 0, 0, 8120, 0, 173, 0, 0, 0, 0, 0, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 582, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 2477, 8192, 0, 0, 0, 0, 0, 0, 6399, 0, 0, 7583, 0, 150, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7302, 16240, 11072, 1416, 10580, 16384, 8158, 300] dup_rows_per_dst=[386, 346, 0, 0, 970, 20826, 4324, 0] combine_rows_per_src=[7302, 16240, 11072, 1416, 10580, 16384, 8158, 300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=260677632 dispatch_meta_bytes=308328 combine_payload_bytes=260677632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260985960 ingress_bytes=295280640 link_bytes=295280640 token_bytes=521663592 weight_movement_bytes=34603008 total_nvlink_bytes=556266600 weight_to_token_ratio=0.066332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111117276e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93664 payload_rows_remote=84192 dup_rows=4640 dup_rows_remote=3910 dedup_ratio=0.047201 router_tokens_per_expert=[0, 0, 7719, 0, 0, 179, 0, 0, 0, 240, 0, 0, 7810, 0, 2723, 0, 2, 363, 66, 0, 0, 0, 74, 0, 197, 0, 0, 0, 0, 45, 0, 7266, 7347, 0, 1398, 0, 0, 0, 154, 0, 402, 806, 0, 0, 0, 3892, 7081, 1, 0, 0, 0, 0, 247, 0, 116, 0, 0, 0, 976, 0, 48, 0, 0, 0] payload_rows_per_dst=[15796, 5446, 14802, 14622, 14542, 9472, 16312, 2672] dup_rows_per_dst=[0, 480, 370, 394, 2594, 730, 34, 38] combine_rows_per_src=[15796, 5446, 14802, 14622, 14542, 9472, 16312, 2672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84964 payload_rows_remote=70564 dup_rows=13340 dup_rows_remote=12726 dedup_ratio=0.135701 router_tokens_per_expert=[0, 0, 7725, 3, 0, 85, 2, 0, 12, 235, 0, 0, 7815, 0, 680, 0, 10, 329, 64, 0, 0, 0, 109, 1, 341, 0, 5, 0, 0, 35, 0, 7126, 6900, 0, 502, 0, 14, 0, 139, 0, 794, 1007, 0, 6, 16, 5571, 7927, 1, 17, 2, 0, 1, 241, 0, 157, 0, 0, 16, 1252, 11, 1, 0, 0, 0] payload_rows_per_dst=[15622, 14266, 16006, 14400, 13938, 4338, 3856, 2538] dup_rows_per_dst=[8, 10198, 874, 614, 1172, 386, 66, 22] combine_rows_per_src=[15622, 14266, 16006, 14400, 13938, 4338, 3856, 2538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78534 payload_rows_remote=62514 dup_rows=19770 dup_rows_remote=18816 dedup_ratio=0.201111 router_tokens_per_expert=[0, 0, 7764, 0, 0, 1, 0, 0, 0, 261, 0, 0, 7837, 0, 662, 0, 17, 343, 53, 0, 0, 0, 117, 0, 282, 0, 0, 0, 0, 0, 0, 7134, 6708, 0, 341, 0, 0, 0, 161, 0, 796, 852, 0, 0, 0, 5884, 7957, 0, 0, 0, 0, 0, 265, 0, 96, 0, 0, 0, 1621, 0, 0, 0, 0, 0] payload_rows_per_dst=[15620, 16204, 16020, 14280, 9400, 3046, 722, 3242] dup_rows_per_dst=[4194, 13084, 954, 552, 736, 250, 0, 0] combine_rows_per_src=[15620, 16204, 16020, 14280, 9400, 3046, 722, 3242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=306266112 dispatch_meta_bytes=361944 combine_payload_bytes=306266112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306628056 ingress_bytes=375472128 link_bytes=375472128 token_bytes=612894168 weight_movement_bytes=69206016 total_nvlink_bytes=682100184 weight_to_token_ratio=0.112917 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412871390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=262758400 dispatch_meta_bytes=362464 combine_payload_bytes=262758400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263120864 ingress_bytes=297361408 link_bytes=297361408 token_bytes=525879264 weight_movement_bytes=34603008 total_nvlink_bytes=560482272 weight_to_token_ratio=0.065800 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118947039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=344850432 dispatch_meta_bytes=352408 combine_payload_bytes=344850432 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=345202840 ingress_bytes=483262464 link_bytes=483262464 token_bytes=690053272 weight_movement_bytes=138412032 total_nvlink_bytes=828465304 weight_to_token_ratio=0.200582 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.818477746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81566 payload_rows_remote=66750 dup_rows=16738 dup_rows_remote=10608 dedup_ratio=0.170268 router_tokens_per_expert=[0, 1187, 0, 0, 222, 61, 5242, 229, 7, 1129, 371, 644, 341, 74, 1795, 153, 61, 0, 0, 540, 22, 159, 68, 0, 482, 162, 2665, 56, 339, 2055, 0, 206, 1964, 34, 0, 119, 211, 7168, 0, 3480, 101, 0, 569, 1424, 65, 0, 0, 61, 0, 125, 363, 0, 0, 6499, 6, 0, 1655, 0, 285, 5519, 262, 902, 0, 70] payload_rows_per_dst=[11344, 7948, 2032, 10974, 4218, 15142, 14816, 15092] dup_rows_per_dst=[1972, 1080, 234, 956, 438, 3634, 6130, 2294] combine_rows_per_src=[11344, 7948, 2032, 10974, 4218, 15142, 14816, 15092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82354 payload_rows_remote=74146 dup_rows=15950 dup_rows_remote=14872 dedup_ratio=0.162252 router_tokens_per_expert=[0, 67, 0, 0, 0, 0, 394, 0, 62, 7428, 2977, 0, 0, 80, 0, 0, 0, 0, 0, 68, 0, 317, 0, 0, 0, 0, 0, 0, 861, 7632, 0, 0, 0, 0, 0, 0, 1121, 7166, 0, 8112, 80, 0, 0, 2804, 82, 0, 20, 0, 0, 0, 0, 0, 0, 1674, 0, 0, 0, 0, 327, 7786, 0, 0, 0, 94] payload_rows_per_dst=[922, 15050, 770, 15344, 8208, 16086, 10242, 15732] dup_rows_per_dst=[0, 6044, 0, 1642, 1078, 4218, 2286, 682] combine_rows_per_src=[922, 15050, 770, 15344, 8208, 16086, 10242, 15732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=289030144 dispatch_meta_bytes=333160 combine_payload_bytes=289030144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=289363304 ingress_bytes=358236160 link_bytes=358236160 token_bytes=578393448 weight_movement_bytes=69206016 total_nvlink_bytes=647599464 weight_to_token_ratio=0.119652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348013830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256057344 dispatch_meta_bytes=325320 combine_payload_bytes=256057344 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256382664 ingress_bytes=325263360 link_bytes=325263360 token_bytes=512440008 weight_movement_bytes=69206016 total_nvlink_bytes=581646024 weight_to_token_ratio=0.135052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223939838e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93156 payload_rows_remote=77506 dup_rows=5148 dup_rows_remote=4358 dedup_ratio=0.052368 router_tokens_per_expert=[0, 0, 7500, 0, 1, 214, 0, 0, 4, 247, 0, 0, 7678, 0, 3911, 1, 2, 346, 73, 0, 0, 0, 76, 0, 153, 0, 0, 1, 0, 71, 0, 7270, 7285, 0, 2197, 0, 3, 0, 148, 0, 211, 563, 0, 1, 3, 3421, 6461, 19, 4, 0, 0, 2, 241, 0, 110, 0, 0, 4, 855, 3, 69, 0, 4, 0] payload_rows_per_dst=[15428, 7850, 13592, 7922, 4834, 11886, 15994, 15650] dup_rows_per_dst=[2, 476, 324, 168, 448, 2864, 76, 790] combine_rows_per_src=[15428, 7850, 13592, 7922, 4834, 11886, 15994, 15650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=273408000 dispatch_meta_bytes=309432 combine_payload_bytes=273408000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273717432 ingress_bytes=308011008 link_bytes=308011008 token_bytes=547125432 weight_movement_bytes=34603008 total_nvlink_bytes=581728440 weight_to_token_ratio=0.063245 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159020626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=303702016 dispatch_meta_bytes=356072 combine_payload_bytes=303702016 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=304058088 ingress_bytes=442114048 link_bytes=442114048 token_bytes=607760104 weight_movement_bytes=138412032 total_nvlink_bytes=746172136 weight_to_token_ratio=0.227741 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.663639570e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50660 payload_rows_remote=50660 dup_rows=47644 dup_rows_remote=47644 dedup_ratio=0.484660 router_tokens_per_expert=[0, 0, 567, 8014, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6634, 0, 0, 483, 0, 0, 0, 0, 0, 0, 414, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[16382, 0, 16384, 966, 16384, 4, 356, 184] dup_rows_per_dst=[17044, 0, 29772, 0, 828, 0, 0, 0] combine_rows_per_src=[16382, 0, 16384, 966, 16384, 4, 356, 184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78596 payload_rows_remote=78444 dup_rows=19708 dup_rows_remote=19708 dedup_ratio=0.200480 router_tokens_per_expert=[35, 0, 0, 0, 0, 0, 0, 41, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 2977, 8094, 0, 0, 0, 7886, 0, 0, 1548, 0, 0, 0, 0, 0, 8157, 0, 6128, 0, 0, 0, 0] payload_rows_per_dst=[152, 7494, 1990, 4916, 16162, 15258, 16240, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4894, 2628, 12186] combine_rows_per_src=[152, 7494, 1990, 4916, 16162, 15258, 16240, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=317464576 dispatch_meta_bytes=327456 combine_payload_bytes=317464576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317792032 ingress_bytes=352067584 link_bytes=352067584 token_bytes=635256608 weight_movement_bytes=34603008 total_nvlink_bytes=669859616 weight_to_token_ratio=0.054471 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324801975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13603041664 prefetch_bytes=692060160 reduce_bytes=1107296256 weight_movement_bytes=1799356416 egress_bytes=6805533056 ingress_bytes=8596865024 link_bytes=8596865024 total_nvlink_bytes=15402398080 weight_to_token_ratio=0.132276 prefetch_share_of_ingress=0.080501 dedup_ratio=0.172256 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=32 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.406491782e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80738 payload_rows_remote=64826 dup_rows=17566 dup_rows_remote=14850 dedup_ratio=0.178691 router_tokens_per_expert=[0, 122, 0, 0, 0, 0, 706, 0, 96, 7184, 989, 0, 0, 99, 2, 18, 0, 0, 0, 55, 0, 195, 0, 0, 0, 0, 0, 0, 693, 7511, 0, 0, 0, 0, 0, 0, 384, 7315, 0, 8062, 100, 0, 0, 1790, 106, 0, 3, 0, 0, 0, 0, 0, 0, 5387, 0, 0, 0, 0, 347, 7790, 0, 138, 0, 60] payload_rows_per_dst=[1522, 14624, 500, 15364, 768, 15912, 16126, 15922] dup_rows_per_dst=[134, 2152, 0, 1044, 0, 2716, 10772, 748] combine_rows_per_src=[1522, 14624, 500, 15364, 768, 15912, 16126, 15922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12565906216 prefetch_bytes=726663168 reduce_bytes=761266176 weight_movement_bytes=1487929344 egress_bytes=6286861096 ingress_bytes=7766974464 link_bytes=7766974464 total_nvlink_bytes=14053835560 weight_to_token_ratio=0.118410 prefetch_share_of_ingress=0.093558 dedup_ratio=0.218173 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=22 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.270717375e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82448 payload_rows_remote=66946 dup_rows=15856 dup_rows_remote=14244 dedup_ratio=0.161296 router_tokens_per_expert=[0, 61, 7, 14, 17, 9, 529, 6, 63, 7470, 2198, 0, 0, 76, 1, 0, 1, 0, 0, 65, 15, 220, 12, 0, 0, 1, 0, 0, 840, 7703, 13, 0, 0, 16, 0, 0, 808, 7173, 3, 8100, 76, 0, 0, 2661, 116, 2, 56, 0, 3, 0, 0, 0, 0, 2653, 0, 0, 0, 0, 315, 7761, 0, 0, 0, 88] payload_rows_per_dst=[1224, 15122, 606, 15502, 16378, 12610, 5312, 15694] dup_rows_per_dst=[10, 4494, 72, 1612, 5962, 3072, 0, 634] combine_rows_per_src=[1224, 15122, 606, 15502, 16378, 12610, 5312, 15694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82426 payload_rows_remote=81810 dup_rows=15878 dup_rows_remote=15878 dedup_ratio=0.161519 router_tokens_per_expert=[0, 52, 0, 0, 0, 0, 378, 0, 73, 7476, 2362, 0, 0, 89, 0, 0, 0, 0, 0, 53, 0, 255, 0, 0, 0, 0, 0, 0, 871, 7724, 0, 0, 0, 0, 0, 0, 953, 7121, 0, 8103, 90, 0, 0, 2787, 117, 0, 21, 0, 0, 0, 0, 0, 0, 2369, 0, 0, 0, 0, 355, 7814, 0, 0, 0, 89] payload_rows_per_dst=[11388, 15238, 616, 15568, 15680, 5818, 4738, 13380] dup_rows_per_dst=[486, 6310, 0, 2638, 5660, 212, 0, 572] combine_rows_per_src=[11388, 15238, 616, 15568, 15680, 5818, 4738, 13380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=207503360 dispatch_meta_bytes=393216 combine_payload_bytes=207503360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=207896576 ingress_bytes=276709376 link_bytes=276709376 token_bytes=415399936 weight_movement_bytes=69206016 total_nvlink_bytes=484605952 weight_to_token_ratio=0.166601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.041235105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=321306624 dispatch_meta_bytes=392608 combine_payload_bytes=321306624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321699232 ingress_bytes=355909632 link_bytes=355909632 token_bytes=643005856 weight_movement_bytes=34603008 total_nvlink_bytes=677608864 weight_to_token_ratio=0.053814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339259292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=318704 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846000 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531373296 weight_movement_bytes=34603008 total_nvlink_bytes=565976304 weight_to_token_ratio=0.065120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=274210816 dispatch_meta_bytes=324760 combine_payload_bytes=274210816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274535576 ingress_bytes=308813824 link_bytes=308813824 token_bytes=548746392 weight_movement_bytes=34603008 total_nvlink_bytes=583349400 weight_to_token_ratio=0.063058 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162041558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=335093760 dispatch_meta_bytes=390752 combine_payload_bytes=335093760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=335484512 ingress_bytes=404299776 link_bytes=404299776 token_bytes=670578272 weight_movement_bytes=69206016 total_nvlink_bytes=739784288 weight_to_token_ratio=0.103203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521347508e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12881328064 prefetch_bytes=795869184 reduce_bytes=553648128 weight_movement_bytes=1349517312 egress_bytes=6444734400 ingress_bytes=7786110976 link_bytes=7786110976 total_nvlink_bytes=14230845376 weight_to_token_ratio=0.104765 prefetch_share_of_ingress=0.102217 dedup_ratio=0.227748 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=16 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.273848207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13190119248 prefetch_bytes=692060160 reduce_bytes=1072693248 weight_movement_bytes=1764753408 egress_bytes=6599024464 ingress_bytes=8355848192 link_bytes=8355848192 total_nvlink_bytes=14954872656 weight_to_token_ratio=0.133794 prefetch_share_of_ingress=0.082823 dedup_ratio=0.188669 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=31 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.367060176e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80456 payload_rows_remote=64350 dup_rows=17848 dup_rows_remote=16728 dedup_ratio=0.181559 router_tokens_per_expert=[0, 204, 2, 4, 4, 2, 1200, 24, 54, 6213, 1015, 87, 0, 73, 110, 70, 0, 0, 0, 65, 4, 205, 3, 0, 0, 0, 0, 0, 580, 7166, 1, 0, 0, 4, 0, 0, 272, 7513, 0, 7570, 74, 0, 0, 1759, 81, 1, 12, 0, 1, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 314, 7910, 0, 311, 0, 78] payload_rows_per_dst=[2644, 8580, 546, 14820, 5554, 16106, 16100, 16106] dup_rows_per_dst=[224, 1662, 20, 674, 0, 2774, 11374, 1120] combine_rows_per_src=[2644, 8580, 546, 14820, 5554, 16106, 16100, 16106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12536771560 prefetch_bytes=761266176 reduce_bytes=657457152 weight_movement_bytes=1418723328 egress_bytes=6272209896 ingress_bytes=7683284992 link_bytes=7683284992 total_nvlink_bytes=13955494888 weight_to_token_ratio=0.113165 prefetch_share_of_ingress=0.099081 dedup_ratio=0.198531 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=19 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.257025342e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12764326632 prefetch_bytes=899678208 reduce_bytes=449839104 weight_movement_bytes=1349517312 egress_bytes=6386174696 ingress_bytes=7727669248 link_bytes=7727669248 total_nvlink_bytes=14113843944 weight_to_token_ratio=0.105726 prefetch_share_of_ingress=0.116423 dedup_ratio=0.220283 prefetch_slots=52 unused_prefetch_slots=316 slot_utilization=0.141304 reduce_slots=13 remote_demand=26 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.264286837e-04 prefetch_bytes_per_gemm_flop=1.471920290e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50548 payload_rows_remote=34164 dup_rows=47756 dup_rows_remote=31132 dedup_ratio=0.485799 router_tokens_per_expert=[0, 0, 659, 8021, 8192, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6992, 0, 0, 600, 0, 0, 0, 0, 0, 0, 382, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 1200, 16384, 0, 52, 144] dup_rows_per_dst=[16624, 0, 30368, 0, 764, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 1200, 16384, 0, 52, 144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263577600 dispatch_meta_bytes=324312 combine_payload_bytes=263577600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263901912 ingress_bytes=332783616 link_bytes=332783616 token_bytes=527479512 weight_movement_bytes=69206016 total_nvlink_bytes=596685528 weight_to_token_ratio=0.131201 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252237956e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=139935744 dispatch_meta_bytes=261184 combine_payload_bytes=139935744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=140196928 ingress_bytes=243744768 link_bytes=243744768 token_bytes=280132672 weight_movement_bytes=103809024 total_nvlink_bytes=383941696 weight_to_token_ratio=0.370571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.171919389e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12461051320 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6234271160 ingress_bytes=7507091456 link_bytes=7507091456 total_nvlink_bytes=13741362616 weight_to_token_ratio=0.102745 prefetch_share_of_ingress=0.096797 dedup_ratio=0.201749 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030000 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228199164e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12536694096 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=6272353616 ingress_bytes=7925284864 link_bytes=7925284864 total_nvlink_bytes=14197638480 weight_to_token_ratio=0.132487 prefetch_share_of_ingress=0.096055 dedup_ratio=0.239197 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.296617778e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none + [2026-08-29 23:40:37] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 674.2 | throughput per GPU (TFLOP/s/GPU): 189.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.653547E+00 | loss scale: 1.0 | grad norm: 203.613 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76020 payload_rows_remote=66968 dup_rows=22284 dup_rows_remote=19740 dedup_ratio=0.226685 router_tokens_per_expert=[1262, 311, 770, 1003, 326, 1113, 802, 1075, 1276, 235, 658, 339, 375, 1280, 827, 947, 697, 983, 809, 592, 361, 664, 304, 1388, 553, 665, 811, 382, 671, 117, 833, 1180, 868, 900, 1263, 783, 596, 208, 510, 1028, 624, 640, 602, 840, 860, 957, 518, 560, 852, 593, 771, 536, 933, 1359, 937, 773, 1423, 341, 1434, 684, 178, 741, 1295, 936] payload_rows_per_dst=[9822, 9356, 9052, 10044, 9578, 8848, 10018, 9302] dup_rows_per_dst=[3502, 2996, 2544, 3226, 2734, 2354, 3012, 1916] combine_rows_per_src=[9822, 9356, 9052, 10044, 9578, 8848, 10018, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75728 payload_rows_remote=65808 dup_rows=22576 dup_rows_remote=19562 dedup_ratio=0.229655 router_tokens_per_expert=[1267, 239, 802, 941, 326, 686, 758, 1383, 1145, 152, 428, 347, 507, 900, 925, 1143, 717, 965, 756, 547, 350, 677, 329, 1493, 780, 658, 742, 315, 650, 53, 1023, 1169, 869, 990, 1132, 670, 610, 251, 434, 961, 780, 690, 498, 1008, 889, 878, 584, 896, 710, 559, 846, 450, 1022, 1120, 872, 888, 1499, 607, 1406, 680, 151, 795, 1126, 1108] payload_rows_per_dst=[8064, 8856, 10470, 8990, 9526, 9280, 9920, 10622] dup_rows_per_dst=[2502, 2238, 3436, 1790, 2308, 3166, 3014, 4122] combine_rows_per_src=[8064, 8856, 10470, 8990, 9526, 9280, 9920, 10622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75426 payload_rows_remote=65514 dup_rows=22878 dup_rows_remote=19384 dedup_ratio=0.232727 router_tokens_per_expert=[1227, 261, 722, 906, 300, 734, 721, 1404, 1264, 146, 498, 458, 500, 927, 920, 1278, 641, 922, 761, 529, 298, 652, 284, 1622, 750, 679, 726, 321, 682, 70, 1004, 1003, 885, 928, 1155, 559, 880, 237, 371, 1180, 716, 658, 505, 1139, 832, 922, 622, 764, 649, 480, 845, 544, 1048, 1220, 878, 835, 1468, 579, 1328, 685, 50, 758, 1085, 1137] payload_rows_per_dst=[9202, 9050, 9112, 9912, 9866, 9288, 9974, 9022] dup_rows_per_dst=[3348, 2932, 2306, 3494, 2524, 3028, 3024, 2222] combine_rows_per_src=[9202, 9050, 9112, 9912, 9866, 9288, 9974, 9022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75942 payload_rows_remote=65532 dup_rows=22362 dup_rows_remote=18758 dedup_ratio=0.227478 router_tokens_per_expert=[1228, 265, 774, 982, 265, 916, 799, 1296, 1242, 167, 568, 369, 478, 1001, 940, 1079, 679, 1041, 752, 542, 335, 697, 294, 1476, 647, 659, 831, 312, 688, 70, 964, 1281, 941, 956, 1120, 680, 494, 214, 561, 920, 696, 659, 541, 1030, 936, 884, 496, 727, 754, 562, 791, 483, 1025, 1163, 936, 903, 1500, 350, 1399, 718, 164, 785, 1150, 977] payload_rows_per_dst=[9478, 10410, 9290, 10438, 9702, 9306, 8390, 8928] dup_rows_per_dst=[3572, 3604, 2342, 3466, 2848, 2832, 1740, 1958] combine_rows_per_src=[9478, 10410, 9290, 10438, 9702, 9306, 8390, 8928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=65002 dup_rows=22972 dup_rows_remote=19140 dedup_ratio=0.233683 router_tokens_per_expert=[1199, 232, 846, 981, 348, 779, 805, 1404, 1295, 165, 492, 396, 494, 936, 975, 1135, 683, 1065, 833, 636, 325, 669, 278, 1432, 776, 677, 686, 329, 672, 76, 944, 1120, 824, 965, 1021, 597, 635, 213, 436, 1078, 783, 648, 509, 871, 838, 933, 575, 812, 781, 479, 950, 395, 1022, 1249, 956, 818, 1541, 531, 1428, 615, 115, 769, 1114, 968] payload_rows_per_dst=[7630, 9062, 10766, 8914, 9320, 9038, 10272, 10330] dup_rows_per_dst=[1854, 2714, 3884, 1646, 2218, 2900, 3924, 3832] combine_rows_per_src=[7630, 9062, 10766, 8914, 9320, 9038, 10272, 10330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=66510 dup_rows=22524 dup_rows_remote=20110 dedup_ratio=0.229126 router_tokens_per_expert=[1281, 298, 824, 997, 360, 766, 809, 1390, 1217, 163, 514, 413, 424, 867, 921, 1214, 672, 938, 838, 610, 270, 674, 296, 1507, 715, 697, 729, 334, 643, 60, 1031, 1254, 811, 957, 1141, 638, 591, 240, 424, 1040, 733, 677, 521, 940, 860, 879, 551, 738, 754, 498, 913, 464, 996, 1221, 793, 859, 1521, 474, 1343, 699, 116, 763, 1160, 1111] payload_rows_per_dst=[9712, 8938, 9296, 9642, 9270, 9060, 9936, 9926] dup_rows_per_dst=[3738, 2528, 2314, 2636, 2414, 2738, 3060, 3096] combine_rows_per_src=[9712, 8938, 9296, 9642, 9270, 9060, 9936, 9926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75794 payload_rows_remote=66656 dup_rows=22510 dup_rows_remote=19650 dedup_ratio=0.228984 router_tokens_per_expert=[1164, 222, 779, 1010, 333, 897, 799, 1306, 1203, 149, 462, 387, 460, 881, 912, 1268, 708, 1061, 777, 490, 364, 603, 263, 1605, 632, 631, 827, 287, 639, 71, 1051, 1341, 851, 931, 1150, 699, 437, 230, 460, 1005, 701, 707, 563, 1033, 846, 810, 589, 750, 666, 489, 769, 491, 1122, 1141, 1013, 886, 1497, 552, 1427, 773, 157, 767, 1083, 975] payload_rows_per_dst=[9592, 8958, 9280, 9172, 9170, 9138, 10042, 10442] dup_rows_per_dst=[3428, 2486, 2462, 1786, 2356, 2860, 3112, 4020] combine_rows_per_src=[9592, 8958, 9280, 9172, 9170, 9138, 10042, 10442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274300928 dispatch_meta_bytes=346832 combine_payload_bytes=274300928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274647760 ingress_bytes=308903936 link_bytes=308903936 token_bytes=548948688 weight_movement_bytes=34603008 total_nvlink_bytes=583551696 weight_to_token_ratio=0.063035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162380642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269549568 dispatch_meta_bytes=341480 combine_payload_bytes=269549568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269891048 ingress_bytes=373358592 link_bytes=373358592 token_bytes=539440616 weight_movement_bytes=103809024 total_nvlink_bytes=643249640 weight_to_token_ratio=0.192438 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404918324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268345344 dispatch_meta_bytes=339592 combine_payload_bytes=268345344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268684936 ingress_bytes=302948352 link_bytes=302948352 token_bytes=537030280 weight_movement_bytes=34603008 total_nvlink_bytes=571633288 weight_to_token_ratio=0.064434 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139970259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268419072 dispatch_meta_bytes=337160 combine_payload_bytes=268419072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268756232 ingress_bytes=303022080 link_bytes=303022080 token_bytes=537175304 weight_movement_bytes=34603008 total_nvlink_bytes=571778312 weight_to_token_ratio=0.064417 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140247692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=336568 combine_payload_bytes=266248192 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584760 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532832952 weight_movement_bytes=103809024 total_nvlink_bytes=636641976 weight_to_token_ratio=0.194825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272424960 dispatch_meta_bytes=346480 combine_payload_bytes=272424960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272771440 ingress_bytes=307027968 link_bytes=307027968 token_bytes=545196400 weight_movement_bytes=34603008 total_nvlink_bytes=579799408 weight_to_token_ratio=0.063469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155321526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273022976 dispatch_meta_bytes=345224 combine_payload_bytes=273022976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273368200 ingress_bytes=307625984 link_bytes=307625984 token_bytes=546391176 weight_movement_bytes=34603008 total_nvlink_bytes=580994184 weight_to_token_ratio=0.063330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157571812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75552 payload_rows_remote=66258 dup_rows=22752 dup_rows_remote=19186 dedup_ratio=0.231445 router_tokens_per_expert=[1216, 244, 764, 952, 298, 692, 847, 1276, 1278, 184, 486, 429, 416, 948, 906, 1156, 678, 1020, 784, 615, 294, 718, 300, 1466, 629, 587, 756, 417, 761, 67, 904, 1000, 878, 885, 1303, 703, 742, 235, 516, 1000, 737, 595, 505, 1005, 932, 927, 490, 728, 765, 551, 777, 539, 973, 1285, 879, 922, 1453, 605, 1456, 658, 129, 811, 1196, 884] payload_rows_per_dst=[9294, 10450, 9368, 9892, 10568, 10894, 7710, 7376] dup_rows_per_dst=[3566, 3726, 2382, 3256, 3902, 3574, 1156, 1190] combine_rows_per_src=[9294, 10450, 9368, 9892, 10568, 10894, 7710, 7376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76484 payload_rows_remote=66936 dup_rows=21820 dup_rows_remote=19380 dedup_ratio=0.221965 router_tokens_per_expert=[774, 690, 1288, 482, 814, 276, 321, 1049, 159, 997, 1086, 419, 551, 1179, 1094, 1285, 1290, 940, 1032, 408, 1397, 80, 1409, 976, 561, 617, 613, 688, 153, 352, 210, 107, 412, 323, 736, 304, 1038, 2444, 1138, 1124, 2008, 1481, 1528, 1461, 506, 702, 61, 131, 640, 1275, 648, 107, 169, 1363, 248, 1153, 851, 396, 813, 646, 828, 644, 290, 387] payload_rows_per_dst=[8794, 9866, 9862, 8762, 8944, 10638, 9548, 10070] dup_rows_per_dst=[2594, 2742, 2384, 1856, 2138, 5208, 2440, 2458] combine_rows_per_src=[8794, 9866, 9862, 8762, 8944, 10638, 9548, 10070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76980 payload_rows_remote=66934 dup_rows=21324 dup_rows_remote=18418 dedup_ratio=0.216919 router_tokens_per_expert=[916, 817, 1061, 463, 864, 264, 288, 1127, 189, 990, 1210, 428, 500, 904, 941, 1360, 1164, 1254, 934, 472, 1361, 194, 1645, 1097, 601, 805, 638, 759, 175, 319, 263, 117, 445, 374, 588, 211, 1129, 1953, 1055, 938, 2053, 1482, 1205, 1343, 579, 701, 100, 166, 730, 1279, 756, 43, 169, 1508, 244, 1226, 814, 333, 739, 606, 955, 621, 341, 346] payload_rows_per_dst=[9466, 9276, 10046, 10780, 10322, 7286, 9662, 10142] dup_rows_per_dst=[3116, 2786, 2906, 3366, 3064, 1180, 2248, 2658] combine_rows_per_src=[9466, 9276, 10046, 10780, 10322, 7286, 9662, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77330 payload_rows_remote=67334 dup_rows=20974 dup_rows_remote=17958 dedup_ratio=0.213359 router_tokens_per_expert=[794, 751, 1258, 472, 836, 305, 348, 975, 241, 981, 1032, 366, 597, 1192, 1055, 1283, 1383, 1058, 941, 407, 1399, 57, 1390, 868, 483, 623, 600, 670, 163, 300, 280, 127, 378, 252, 782, 316, 1120, 2522, 1105, 1169, 1986, 1458, 1743, 1607, 577, 542, 66, 91, 583, 1258, 660, 117, 158, 1396, 260, 1128, 602, 329, 750, 672, 889, 605, 332, 464] payload_rows_per_dst=[8944, 10228, 9754, 9996, 10482, 8802, 9272, 9852] dup_rows_per_dst=[2534, 3266, 2472, 3016, 3072, 2552, 1848, 2214] combine_rows_per_src=[8944, 10228, 9754, 9996, 10482, 8802, 9272, 9852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77326 payload_rows_remote=69082 dup_rows=20978 dup_rows_remote=18350 dedup_ratio=0.213399 router_tokens_per_expert=[884, 710, 1080, 464, 846, 307, 230, 1149, 136, 997, 1258, 403, 585, 1105, 952, 1349, 1285, 1050, 930, 450, 1349, 101, 1600, 955, 565, 644, 632, 769, 124, 283, 298, 130, 432, 334, 661, 271, 1157, 2136, 1066, 1125, 2078, 1486, 1358, 1434, 494, 658, 75, 190, 670, 1303, 758, 38, 180, 1661, 232, 1195, 733, 340, 708, 593, 903, 569, 303, 391] payload_rows_per_dst=[10844, 8244, 9832, 10724, 10654, 7282, 9794, 9952] dup_rows_per_dst=[3194, 2628, 2408, 3190, 3710, 1240, 2280, 2328] combine_rows_per_src=[10844, 8244, 9832, 10724, 10654, 7282, 9794, 9952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76430 payload_rows_remote=66558 dup_rows=21874 dup_rows_remote=19696 dedup_ratio=0.222514 router_tokens_per_expert=[828, 577, 1269, 592, 702, 205, 371, 1186, 182, 1122, 881, 346, 468, 1111, 1157, 1240, 1402, 1018, 887, 430, 1188, 118, 1264, 1136, 479, 748, 587, 684, 221, 334, 280, 102, 395, 303, 871, 294, 1150, 2529, 1085, 963, 2108, 1470, 1616, 1679, 521, 532, 63, 95, 602, 1236, 740, 132, 143, 1483, 256, 1024, 652, 366, 831, 688, 807, 636, 339, 428] payload_rows_per_dst=[8988, 8964, 9954, 9122, 9682, 8498, 11350, 9872] dup_rows_per_dst=[2472, 1828, 2376, 1964, 2662, 3454, 4940, 2178] combine_rows_per_src=[8988, 8964, 9954, 9122, 9682, 8498, 11350, 9872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76484 payload_rows_remote=68024 dup_rows=21820 dup_rows_remote=19882 dedup_ratio=0.221965 router_tokens_per_expert=[760, 662, 1236, 533, 742, 296, 324, 1026, 193, 955, 907, 375, 614, 1199, 1026, 1283, 1422, 1061, 929, 430, 1420, 113, 1360, 961, 518, 635, 650, 693, 146, 341, 278, 88, 418, 303, 781, 310, 1113, 2507, 1184, 1090, 1958, 1432, 1554, 1633, 541, 529, 78, 102, 606, 1217, 667, 121, 204, 1527, 222, 1142, 709, 397, 892, 676, 763, 650, 264, 386] payload_rows_per_dst=[8688, 10008, 10094, 8826, 8460, 11016, 9416, 9976] dup_rows_per_dst=[2470, 3096, 2578, 1788, 1938, 5736, 1996, 2218] combine_rows_per_src=[8688, 10008, 10094, 8826, 8460, 11016, 9416, 9976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76750 payload_rows_remote=65582 dup_rows=21554 dup_rows_remote=15638 dedup_ratio=0.219259 router_tokens_per_expert=[870, 737, 1153, 565, 731, 269, 341, 1109, 153, 1024, 1001, 321, 495, 1122, 1159, 1220, 1342, 1136, 935, 403, 1397, 106, 1570, 941, 454, 647, 551, 626, 152, 340, 261, 122, 483, 343, 853, 312, 942, 2592, 1009, 1057, 2132, 1375, 1692, 1382, 536, 708, 87, 170, 650, 1356, 600, 65, 145, 1538, 258, 1287, 644, 330, 671, 619, 850, 599, 256, 358] payload_rows_per_dst=[9130, 9848, 10234, 8780, 8192, 11168, 9674, 9724] dup_rows_per_dst=[2420, 3142, 2286, 1790, 1806, 5916, 2124, 2070] combine_rows_per_src=[9130, 9848, 10234, 8780, 8192, 11168, 9674, 9724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=345264 combine_payload_bytes=274169856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274515120 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548684976 weight_movement_bytes=34603008 total_nvlink_bytes=583287984 weight_to_token_ratio=0.063065 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274161664 dispatch_meta_bytes=341408 combine_payload_bytes=274161664 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=274503072 ingress_bytes=377970688 link_bytes=377970688 token_bytes=548664736 weight_movement_bytes=103809024 total_nvlink_bytes=652473760 weight_to_token_ratio=0.189203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422273270e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275800064 dispatch_meta_bytes=341168 combine_payload_bytes=275800064 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276141232 ingress_bytes=345006080 link_bytes=345006080 token_bytes=551941296 weight_movement_bytes=69206016 total_nvlink_bytes=621147312 weight_to_token_ratio=0.125387 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298230104e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=282959872 dispatch_meta_bytes=349728 combine_payload_bytes=282959872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=283309600 ingress_bytes=386768896 link_bytes=386768896 token_bytes=566269472 weight_movement_bytes=103809024 total_nvlink_bytes=670078496 weight_to_token_ratio=0.183321 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455380218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271392768 dispatch_meta_bytes=341776 combine_payload_bytes=271392768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271734544 ingress_bytes=375201792 link_bytes=375201792 token_bytes=543127312 weight_movement_bytes=103809024 total_nvlink_bytes=646936336 weight_to_token_ratio=0.191132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411854137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272621568 dispatch_meta_bytes=345016 combine_payload_bytes=272621568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272966584 ingress_bytes=341827584 link_bytes=341827584 token_bytes=545588152 weight_movement_bytes=69206016 total_nvlink_bytes=614794168 weight_to_token_ratio=0.126847 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286269679e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278626304 dispatch_meta_bytes=351624 combine_payload_bytes=278626304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278977928 ingress_bytes=382435328 link_bytes=382435328 token_bytes=557604232 weight_movement_bytes=103809024 total_nvlink_bytes=661413256 weight_to_token_ratio=0.186170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439073351e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268623872 dispatch_meta_bytes=324880 combine_payload_bytes=268623872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268948752 ingress_bytes=372432896 link_bytes=372432896 token_bytes=537572624 weight_movement_bytes=103809024 total_nvlink_bytes=641381648 weight_to_token_ratio=0.193107 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401435004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75538 payload_rows_remote=65644 dup_rows=22766 dup_rows_remote=18836 dedup_ratio=0.231588 router_tokens_per_expert=[690, 577, 580, 309, 257, 945, 897, 195, 875, 311, 770, 888, 1451, 1235, 1110, 2104, 842, 1020, 222, 404, 195, 921, 1032, 880, 177, 624, 751, 329, 403, 550, 797, 656, 423, 866, 1151, 834, 1230, 1378, 2258, 698, 265, 550, 428, 205, 1597, 1035, 162, 1490, 1034, 437, 943, 839, 255, 507, 350, 1096, 1106, 467, 52, 614, 472, 906, 818, 1689] payload_rows_per_dst=[7184, 8336, 9052, 10424, 10530, 10098, 9894, 10020] dup_rows_per_dst=[1716, 2042, 1980, 2358, 4686, 3826, 3930, 2228] combine_rows_per_src=[7184, 8336, 9052, 10424, 10530, 10098, 9894, 10020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75400 payload_rows_remote=65134 dup_rows=22904 dup_rows_remote=19588 dedup_ratio=0.232992 router_tokens_per_expert=[470, 884, 444, 401, 142, 1122, 1165, 216, 758, 284, 954, 1079, 1170, 1121, 995, 1977, 859, 1017, 247, 377, 363, 588, 1177, 917, 270, 615, 978, 492, 485, 556, 865, 409, 450, 916, 1047, 536, 1736, 1246, 1960, 619, 426, 423, 320, 319, 1501, 1181, 172, 1710, 1200, 346, 566, 833, 249, 430, 310, 934, 1166, 318, 136, 734, 298, 1191, 767, 1715] payload_rows_per_dst=[9782, 9680, 10266, 10490, 8360, 8872, 7702, 10248] dup_rows_per_dst=[3826, 3042, 3316, 2804, 2248, 3232, 2034, 2402] combine_rows_per_src=[9782, 9680, 10266, 10490, 8360, 8872, 7702, 10248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75544 payload_rows_remote=65158 dup_rows=22760 dup_rows_remote=20470 dedup_ratio=0.231527 router_tokens_per_expert=[939, 641, 466, 441, 353, 1009, 973, 220, 998, 300, 755, 842, 1421, 1413, 1207, 2113, 815, 894, 181, 530, 90, 1023, 898, 625, 194, 601, 672, 280, 415, 646, 725, 692, 476, 902, 1061, 893, 1041, 1283, 2284, 641, 281, 522, 285, 199, 1607, 1099, 160, 1479, 1046, 521, 966, 760, 336, 661, 325, 1087, 1085, 424, 55, 652, 396, 786, 913, 1554] payload_rows_per_dst=[10258, 10224, 9952, 10386, 7880, 8552, 8372, 9920] dup_rows_per_dst=[4394, 3648, 2726, 2290, 2148, 2712, 3032, 1810] combine_rows_per_src=[10258, 10224, 9952, 10386, 7880, 8552, 8372, 9920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75006 payload_rows_remote=65312 dup_rows=23298 dup_rows_remote=20016 dedup_ratio=0.237000 router_tokens_per_expert=[431, 754, 542, 320, 209, 1130, 988, 176, 899, 304, 900, 949, 1341, 1186, 909, 1870, 849, 1249, 234, 484, 324, 574, 1281, 843, 220, 680, 931, 455, 495, 509, 776, 435, 419, 875, 982, 709, 1629, 1328, 1898, 603, 358, 391, 360, 213, 1559, 1300, 216, 1732, 1081, 379, 771, 871, 246, 356, 307, 981, 1044, 407, 81, 764, 317, 1258, 675, 1795] payload_rows_per_dst=[9428, 9694, 10472, 10322, 8046, 9048, 7822, 10174] dup_rows_per_dst=[3468, 3282, 3860, 2420, 2388, 3210, 2162, 2508] combine_rows_per_src=[9428, 9694, 10472, 10322, 8046, 9048, 7822, 10174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75110 payload_rows_remote=65002 dup_rows=23194 dup_rows_remote=20628 dedup_ratio=0.235942 router_tokens_per_expert=[908, 536, 547, 433, 295, 1062, 959, 214, 1025, 276, 766, 804, 1323, 1261, 1167, 2100, 865, 870, 215, 394, 106, 993, 845, 740, 220, 556, 752, 315, 409, 546, 742, 639, 516, 944, 1126, 859, 963, 1237, 2364, 620, 267, 525, 253, 163, 1578, 1073, 181, 1521, 1315, 481, 1048, 743, 527, 823, 348, 1201, 1020, 368, 73, 436, 521, 666, 918, 1591] payload_rows_per_dst=[7770, 7518, 8566, 10298, 10706, 9588, 10556, 10108] dup_rows_per_dst=[2138, 1592, 1490, 2260, 4626, 3460, 5062, 2566] combine_rows_per_src=[7770, 7518, 8566, 10298, 10706, 9588, 10556, 10108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75626 payload_rows_remote=67382 dup_rows=22678 dup_rows_remote=19990 dedup_ratio=0.230693 router_tokens_per_expert=[924, 669, 430, 408, 263, 925, 871, 218, 982, 338, 775, 979, 1428, 1500, 1195, 1959, 834, 929, 188, 494, 84, 1027, 998, 704, 139, 571, 710, 259, 523, 538, 669, 677, 456, 749, 1009, 851, 1013, 1236, 2166, 628, 290, 583, 307, 191, 1618, 1056, 196, 1494, 1074, 644, 1013, 657, 349, 708, 402, 1192, 1016, 358, 58, 646, 484, 823, 1040, 1637] payload_rows_per_dst=[10016, 9540, 9372, 9994, 8244, 8628, 9750, 10082] dup_rows_per_dst=[3732, 3078, 2096, 2096, 2688, 2842, 4104, 2042] combine_rows_per_src=[10016, 9540, 9372, 9994, 8244, 8628, 9750, 10082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=65864 dup_rows=22936 dup_rows_remote=19850 dedup_ratio=0.233317 router_tokens_per_expert=[838, 586, 433, 394, 291, 876, 1128, 191, 819, 241, 693, 788, 1267, 1585, 1333, 2312, 846, 938, 162, 420, 287, 1140, 894, 715, 119, 642, 747, 284, 408, 520, 569, 623, 505, 866, 938, 795, 963, 1029, 2273, 657, 275, 646, 414, 164, 1570, 992, 196, 1894, 1246, 662, 637, 740, 406, 560, 379, 1074, 921, 339, 96, 859, 529, 802, 1059, 1577] payload_rows_per_dst=[8056, 8734, 8822, 10254, 9866, 9504, 9924, 10208] dup_rows_per_dst=[2608, 2184, 1982, 2194, 4708, 3086, 4018, 2156] combine_rows_per_src=[8056, 8734, 8822, 10254, 9866, 9504, 9924, 10208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77036 payload_rows_remote=65976 dup_rows=21268 dup_rows_remote=17678 dedup_ratio=0.216349 router_tokens_per_expert=[853, 662, 1080, 594, 901, 212, 350, 1326, 163, 1099, 1080, 456, 543, 1265, 951, 1347, 1350, 986, 888, 395, 1149, 139, 1394, 1286, 562, 637, 639, 789, 150, 380, 265, 117, 402, 313, 674, 247, 1095, 1960, 941, 957, 2209, 1458, 1247, 1424, 454, 708, 85, 212, 681, 1359, 857, 35, 150, 1482, 253, 1039, 797, 362, 869, 637, 736, 697, 361, 443] payload_rows_per_dst=[11060, 8708, 9338, 10890, 10176, 7226, 9488, 10150] dup_rows_per_dst=[3590, 3674, 1780, 3454, 3002, 1102, 2224, 2442] combine_rows_per_src=[11060, 8708, 9338, 10890, 10176, 7226, 9488, 10150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268877824 dispatch_meta_bytes=337920 combine_payload_bytes=268877824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269215744 ingress_bytes=303480832 link_bytes=303480832 token_bytes=538093568 weight_movement_bytes=34603008 total_nvlink_bytes=572696576 weight_to_token_ratio=0.064307 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141973939e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=338888 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267127752 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533916616 weight_movement_bytes=34603008 total_nvlink_bytes=568519624 weight_to_token_ratio=0.064810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=342512 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267229680 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534116848 weight_movement_bytes=34603008 total_nvlink_bytes=568719856 weight_to_token_ratio=0.064785 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267517952 dispatch_meta_bytes=341312 combine_payload_bytes=267517952 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267859264 ingress_bytes=405929984 link_bytes=405929984 token_bytes=535377216 weight_movement_bytes=138412032 total_nvlink_bytes=673789248 weight_to_token_ratio=0.258532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.527481850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=342520 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266590712 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532838904 weight_movement_bytes=34603008 total_nvlink_bytes=567441912 weight_to_token_ratio=0.064941 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275996672 dispatch_meta_bytes=349488 combine_payload_bytes=275996672 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=276346160 ingress_bytes=414408704 link_bytes=414408704 token_bytes=552342832 weight_movement_bytes=138412032 total_nvlink_bytes=690754864 weight_to_token_ratio=0.250591 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.559386591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269778944 dispatch_meta_bytes=342856 combine_payload_bytes=269778944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270121800 ingress_bytes=304381952 link_bytes=304381952 token_bytes=539900744 weight_movement_bytes=34603008 total_nvlink_bytes=574503752 weight_to_token_ratio=0.064091 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145364781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270237696 dispatch_meta_bytes=334616 combine_payload_bytes=270237696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270572312 ingress_bytes=304840704 link_bytes=304840704 token_bytes=540810008 weight_movement_bytes=34603008 total_nvlink_bytes=575413016 weight_to_token_ratio=0.063984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147091027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75718 payload_rows_remote=68078 dup_rows=22586 dup_rows_remote=20554 dedup_ratio=0.229757 router_tokens_per_expert=[1426, 1439, 1194, 695, 419, 319, 872, 632, 1306, 1808, 330, 364, 744, 698, 660, 439, 908, 75, 1891, 672, 492, 284, 138, 885, 369, 608, 689, 507, 486, 556, 970, 1160, 250, 728, 393, 402, 604, 425, 1756, 703, 752, 1981, 204, 647, 386, 484, 1325, 2690, 1113, 430, 627, 863, 387, 835, 710, 984, 250, 942, 873, 817, 878, 639, 721, 318] payload_rows_per_dst=[9504, 10160, 8210, 9100, 9474, 11272, 7640, 10358] dup_rows_per_dst=[1610, 2538, 2480, 1590, 3926, 5666, 2032, 2744] combine_rows_per_src=[9504, 10160, 8210, 9100, 9474, 11272, 7640, 10358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75090 payload_rows_remote=64786 dup_rows=23214 dup_rows_remote=18756 dedup_ratio=0.236145 router_tokens_per_expert=[587, 1627, 1222, 887, 385, 290, 803, 469, 1148, 1874, 295, 387, 596, 629, 665, 509, 1321, 111, 1761, 531, 489, 210, 66, 816, 495, 326, 573, 508, 692, 574, 1311, 1287, 273, 935, 484, 395, 452, 365, 1982, 835, 796, 1594, 209, 503, 327, 655, 1320, 2076, 925, 557, 699, 778, 685, 862, 825, 1007, 418, 1030, 1102, 686, 822, 724, 721, 666] payload_rows_per_dst=[9784, 10042, 10304, 11048, 8610, 6608, 9380, 9314] dup_rows_per_dst=[2756, 2164, 4458, 3672, 2832, 1012, 3296, 3024] combine_rows_per_src=[9784, 10042, 10304, 11048, 8610, 6608, 9380, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74816 payload_rows_remote=65604 dup_rows=23488 dup_rows_remote=21428 dedup_ratio=0.238932 router_tokens_per_expert=[1436, 1607, 1046, 799, 339, 270, 790, 517, 913, 1653, 230, 375, 633, 673, 532, 499, 1126, 80, 1960, 646, 456, 229, 160, 824, 263, 436, 716, 505, 510, 505, 912, 1246, 221, 903, 442, 389, 417, 456, 1804, 547, 891, 1655, 167, 700, 348, 563, 1545, 2430, 1680, 468, 757, 700, 507, 910, 882, 1402, 299, 1137, 980, 834, 705, 526, 745, 256] payload_rows_per_dst=[10560, 9210, 9332, 9212, 7936, 9452, 10166, 8948] dup_rows_per_dst=[3048, 1806, 3634, 2060, 2422, 4056, 4446, 2016] combine_rows_per_src=[10560, 9210, 9332, 9212, 7936, 9452, 10166, 8948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73642 payload_rows_remote=62738 dup_rows=24662 dup_rows_remote=20742 dedup_ratio=0.250875 router_tokens_per_expert=[899, 1240, 1351, 605, 436, 391, 797, 705, 1235, 2175, 343, 374, 773, 592, 688, 365, 1173, 94, 2007, 532, 656, 324, 83, 804, 485, 471, 641, 478, 646, 690, 1077, 1039, 331, 823, 323, 400, 447, 577, 2116, 827, 667, 2181, 160, 516, 417, 547, 902, 2678, 607, 467, 509, 965, 521, 864, 471, 807, 226, 746, 1195, 728, 920, 755, 720, 570] payload_rows_per_dst=[9998, 10904, 11034, 11492, 8684, 4378, 8044, 9108] dup_rows_per_dst=[2850, 3920, 5668, 3924, 3004, 306, 2378, 2612] combine_rows_per_src=[9998, 10904, 11034, 11492, 8684, 4378, 8044, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65760 dup_rows=23914 dup_rows_remote=20722 dedup_ratio=0.243266 router_tokens_per_expert=[1277, 1850, 959, 795, 334, 219, 826, 546, 891, 1424, 217, 371, 569, 744, 563, 511, 914, 123, 1641, 709, 455, 215, 173, 863, 277, 344, 690, 500, 456, 544, 931, 1266, 210, 887, 494, 410, 362, 462, 1885, 603, 862, 1645, 189, 647, 371, 521, 1780, 2406, 2091, 564, 780, 715, 590, 883, 965, 1430, 234, 876, 944, 835, 758, 572, 721, 263] payload_rows_per_dst=[10068, 8938, 7876, 8634, 8630, 11102, 10666, 8476] dup_rows_per_dst=[2348, 1642, 2310, 1382, 3192, 5740, 5370, 1930] combine_rows_per_src=[10068, 8938, 7876, 8634, 8630, 11102, 10666, 8476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75176 payload_rows_remote=63816 dup_rows=23128 dup_rows_remote=19948 dedup_ratio=0.235270 router_tokens_per_expert=[1347, 1869, 1007, 983, 339, 263, 795, 508, 950, 1573, 294, 406, 516, 719, 551, 445, 897, 66, 1366, 770, 483, 155, 144, 741, 282, 327, 635, 523, 536, 408, 999, 1313, 251, 836, 660, 427, 296, 537, 1507, 600, 1146, 1763, 226, 571, 293, 559, 1927, 2117, 1811, 563, 1053, 668, 433, 737, 1049, 1453, 261, 1190, 854, 823, 644, 664, 740, 283] payload_rows_per_dst=[8988, 9118, 7320, 8516, 9684, 11030, 9160, 11360] dup_rows_per_dst=[1496, 1790, 1924, 1530, 4282, 5940, 2986, 3180] combine_rows_per_src=[8988, 9118, 7320, 8516, 9684, 11030, 9160, 11360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74626 payload_rows_remote=63328 dup_rows=23678 dup_rows_remote=17874 dedup_ratio=0.240865 router_tokens_per_expert=[1241, 2240, 957, 1177, 333, 243, 732, 453, 749, 1124, 234, 355, 562, 733, 451, 496, 834, 84, 1538, 526, 425, 189, 191, 816, 324, 259, 548, 530, 582, 372, 1033, 1303, 249, 863, 475, 457, 305, 398, 2093, 574, 961, 1231, 170, 562, 396, 707, 2355, 2169, 2044, 512, 908, 677, 470, 865, 895, 1440, 260, 1282, 892, 776, 683, 623, 896, 330] payload_rows_per_dst=[8924, 8148, 7200, 8472, 10328, 11298, 9512, 10744] dup_rows_per_dst=[1348, 1260, 2006, 1430, 4980, 5804, 3434, 3416] combine_rows_per_src=[8924, 8148, 7200, 8472, 10328, 11298, 9512, 10744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278847488 dispatch_meta_bytes=354528 combine_payload_bytes=278847488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279202016 ingress_bytes=348053504 link_bytes=348053504 token_bytes=558049504 weight_movement_bytes=69206016 total_nvlink_bytes=627255520 weight_to_token_ratio=0.124014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309697315e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=64434 dup_rows=23806 dup_rows_remote=19942 dedup_ratio=0.242167 router_tokens_per_expert=[469, 846, 657, 402, 141, 1362, 927, 253, 842, 329, 814, 942, 1629, 902, 723, 1509, 944, 1115, 338, 420, 283, 418, 1058, 945, 402, 713, 950, 604, 453, 499, 767, 414, 478, 987, 1299, 694, 1795, 1655, 1907, 557, 449, 386, 319, 282, 1471, 1195, 217, 1504, 1062, 286, 815, 797, 256, 498, 277, 980, 1273, 445, 82, 501, 279, 1113, 430, 1793] payload_rows_per_dst=[10064, 11070, 10546, 7994, 8454, 8744, 7918, 9708] dup_rows_per_dst=[3864, 4640, 3806, 1720, 2726, 2902, 2024, 2124] combine_rows_per_src=[10064, 11070, 10546, 7994, 8454, 8744, 7918, 9708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=334168 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265697624 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531061080 weight_movement_bytes=34603008 total_nvlink_bytes=565664088 weight_to_token_ratio=0.065158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=268713984 dispatch_meta_bytes=348128 combine_payload_bytes=268713984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269062112 ingress_bytes=303316992 link_bytes=303316992 token_bytes=537776096 weight_movement_bytes=34603008 total_nvlink_bytes=572379104 weight_to_token_ratio=0.064345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141357422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=333920 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257308768 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514283616 weight_movement_bytes=34603008 total_nvlink_bytes=548886624 weight_to_token_ratio=0.067284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269352960 dispatch_meta_bytes=345928 combine_payload_bytes=269352960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269698888 ingress_bytes=303955968 link_bytes=303955968 token_bytes=539051848 weight_movement_bytes=34603008 total_nvlink_bytes=573654856 weight_to_token_ratio=0.064192 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143761837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261390336 dispatch_meta_bytes=335056 combine_payload_bytes=261390336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261725392 ingress_bytes=295993344 link_bytes=295993344 token_bytes=523115728 weight_movement_bytes=34603008 total_nvlink_bytes=557718736 weight_to_token_ratio=0.066148 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113799124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259391488 dispatch_meta_bytes=324808 combine_payload_bytes=259391488 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=259716296 ingress_bytes=432406528 link_bytes=432406528 token_bytes=519107784 weight_movement_bytes=173015040 total_nvlink_bytes=692122824 weight_to_token_ratio=0.333293 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.627110953e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263921664 dispatch_meta_bytes=337504 combine_payload_bytes=263921664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264259168 ingress_bytes=298524672 link_bytes=298524672 token_bytes=528180832 weight_movement_bytes=34603008 total_nvlink_bytes=562783840 weight_to_token_ratio=0.065514 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123324308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70978 payload_rows_remote=61178 dup_rows=27326 dup_rows_remote=23634 dedup_ratio=0.277974 router_tokens_per_expert=[294, 1169, 1669, 324, 935, 777, 1445, 565, 312, 317, 1021, 209, 638, 412, 987, 1394, 1403, 223, 539, 196, 156, 247, 192, 316, 521, 514, 663, 564, 949, 651, 782, 910, 1178, 898, 903, 498, 941, 1150, 247, 1203, 434, 202, 1046, 818, 999, 512, 1735, 615, 1026, 342, 191, 2179, 751, 828, 501, 928, 899, 220, 537, 683, 792, 2229, 1156, 1217] payload_rows_per_dst=[7844, 8988, 7228, 8432, 9964, 8710, 9800, 10012] dup_rows_per_dst=[3622, 3266, 1616, 2676, 4662, 2338, 3692, 5454] combine_rows_per_src=[7844, 8988, 7228, 8432, 9964, 8710, 9800, 10012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71704 payload_rows_remote=61720 dup_rows=26600 dup_rows_remote=22274 dedup_ratio=0.270589 router_tokens_per_expert=[479, 1502, 1518, 154, 738, 1369, 1723, 339, 537, 170, 694, 258, 617, 300, 1096, 995, 1463, 407, 1030, 372, 221, 159, 166, 165, 185, 558, 454, 626, 543, 728, 714, 1522, 1111, 1342, 741, 206, 620, 2061, 427, 1036, 450, 442, 2174, 789, 839, 863, 1328, 1252, 955, 124, 191, 2204, 787, 708, 524, 651, 495, 202, 457, 388, 606, 1614, 730, 1033] payload_rows_per_dst=[9652, 8816, 9984, 8862, 7334, 10202, 8688, 8166] dup_rows_per_dst=[5630, 3174, 4326, 2622, 1772, 3416, 2776, 2884] combine_rows_per_src=[9652, 8816, 9984, 8862, 7334, 10202, 8688, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72914 payload_rows_remote=63160 dup_rows=25390 dup_rows_remote=21380 dedup_ratio=0.258280 router_tokens_per_expert=[644, 1094, 1462, 112, 643, 690, 1904, 250, 333, 152, 418, 203, 810, 264, 1181, 1232, 1322, 547, 1310, 231, 177, 181, 140, 205, 230, 620, 335, 594, 438, 485, 528, 1982, 1484, 1580, 1390, 125, 638, 1715, 290, 955, 430, 307, 1723, 766, 1004, 1482, 1256, 1687, 1398, 119, 145, 2294, 803, 592, 390, 671, 509, 111, 527, 326, 507, 1741, 542, 928] payload_rows_per_dst=[7194, 8714, 10336, 8386, 9754, 11162, 9584, 7784] dup_rows_per_dst=[2596, 2984, 4288, 2038, 4010, 3636, 3240, 2598] combine_rows_per_src=[7194, 8714, 10336, 8386, 9754, 11162, 9584, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72670 payload_rows_remote=64326 dup_rows=25634 dup_rows_remote=23424 dedup_ratio=0.260763 router_tokens_per_expert=[510, 1101, 1559, 195, 690, 596, 1555, 381, 433, 221, 525, 234, 832, 310, 1109, 1497, 1529, 621, 802, 146, 165, 204, 140, 122, 215, 585, 412, 521, 570, 612, 723, 1639, 1223, 1211, 1325, 166, 626, 1858, 369, 952, 375, 348, 1505, 724, 992, 1203, 1374, 1338, 1367, 111, 159, 2373, 800, 636, 520, 783, 511, 141, 438, 394, 647, 1992, 774, 1163] payload_rows_per_dst=[7412, 9520, 10020, 8344, 8858, 10066, 9872, 8578] dup_rows_per_dst=[2652, 3550, 3600, 2210, 3550, 2904, 3626, 3542] combine_rows_per_src=[7412, 9520, 10020, 8344, 8858, 10066, 9872, 8578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69730 payload_rows_remote=59708 dup_rows=28574 dup_rows_remote=24620 dedup_ratio=0.290670 router_tokens_per_expert=[87, 1175, 1646, 554, 1088, 1020, 1215, 868, 490, 228, 1296, 209, 458, 348, 785, 1428, 1526, 131, 273, 149, 190, 315, 245, 292, 377, 415, 1091, 523, 1307, 917, 779, 449, 905, 356, 551, 606, 1113, 1143, 206, 1129, 342, 224, 781, 867, 1034, 159, 1746, 144, 689, 429, 245, 2542, 608, 790, 773, 1187, 869, 355, 444, 1001, 897, 2142, 1730, 1271] payload_rows_per_dst=[9564, 10022, 7920, 11742, 6656, 5482, 7782, 10562] dup_rows_per_dst=[6588, 3954, 2418, 5058, 1266, 774, 2302, 6214] combine_rows_per_src=[9564, 10022, 7920, 11742, 6656, 5482, 7782, 10562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=250585088 dispatch_meta_bytes=339248 combine_payload_bytes=250585088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250924336 ingress_bytes=319791104 link_bytes=319791104 token_bytes=501509424 weight_movement_bytes=69206016 total_nvlink_bytes=570715440 weight_to_token_ratio=0.137995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203348179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72998 payload_rows_remote=64724 dup_rows=25306 dup_rows_remote=22228 dedup_ratio=0.257426 router_tokens_per_expert=[519, 1026, 1595, 132, 569, 435, 1700, 318, 325, 193, 467, 95, 592, 319, 1204, 1004, 1416, 676, 1035, 164, 161, 200, 157, 136, 220, 704, 259, 487, 401, 478, 418, 2512, 1785, 1882, 1345, 130, 544, 2042, 277, 939, 328, 281, 2005, 738, 860, 1302, 1228, 2329, 1151, 35, 153, 2081, 737, 585, 367, 571, 536, 88, 566, 245, 403, 1792, 711, 1199] payload_rows_per_dst=[7030, 6880, 9468, 8826, 11090, 12724, 8706, 8274] dup_rows_per_dst=[2158, 1518, 2506, 2132, 6114, 5146, 2654, 3078] combine_rows_per_src=[7030, 6880, 9468, 8826, 11090, 12724, 8706, 8274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=252805120 dispatch_meta_bytes=335976 combine_payload_bytes=252805120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=253141096 ingress_bytes=322011136 link_bytes=322011136 token_bytes=505946216 weight_movement_bytes=69206016 total_nvlink_bytes=575152232 weight_to_token_ratio=0.136785 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211701981e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72198 payload_rows_remote=60410 dup_rows=26106 dup_rows_remote=21188 dedup_ratio=0.265564 router_tokens_per_expert=[1171, 1105, 1401, 106, 522, 968, 1977, 188, 377, 96, 282, 308, 989, 207, 1273, 1095, 1256, 357, 1707, 301, 126, 172, 160, 109, 127, 578, 213, 497, 286, 380, 442, 2590, 1239, 2127, 1595, 138, 568, 2040, 313, 719, 266, 150, 2228, 632, 895, 1637, 1150, 2545, 1338, 33, 158, 2040, 727, 549, 247, 536, 440, 76, 444, 169, 401, 1275, 312, 799] payload_rows_per_dst=[7758, 8666, 10152, 8502, 10670, 11788, 8520, 6142] dup_rows_per_dst=[3164, 2888, 3972, 1724, 5014, 4918, 2736, 1690] combine_rows_per_src=[7758, 8666, 10152, 8502, 10670, 11788, 8520, 6142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=63002 dup_rows=24066 dup_rows_remote=20280 dedup_ratio=0.244812 router_tokens_per_expert=[905, 1005, 1325, 665, 547, 599, 762, 678, 1677, 2423, 486, 410, 811, 692, 860, 227, 1625, 56, 1838, 701, 566, 304, 58, 747, 449, 461, 666, 475, 647, 662, 1244, 1013, 396, 843, 410, 391, 400, 519, 1477, 724, 750, 2504, 198, 505, 304, 563, 746, 2404, 377, 454, 666, 897, 398, 819, 575, 711, 249, 778, 1136, 696, 728, 853, 632, 465] payload_rows_per_dst=[11236, 11856, 10220, 12204, 7934, 4244, 7688, 8856] dup_rows_per_dst=[3786, 4808, 4328, 4038, 2386, 396, 2106, 2218] combine_rows_per_src=[11236, 11856, 10220, 12204, 7934, 4244, 7688, 8856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258703360 dispatch_meta_bytes=338160 combine_payload_bytes=258703360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259041520 ingress_bytes=362512384 link_bytes=362512384 token_bytes=517744880 weight_movement_bytes=103809024 total_nvlink_bytes=621553904 weight_to_token_ratio=0.200502 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364104916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263479296 dispatch_meta_bytes=351000 combine_payload_bytes=263479296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263830296 ingress_bytes=298082304 link_bytes=298082304 token_bytes=527309592 weight_movement_bytes=34603008 total_nvlink_bytes=561912600 weight_to_token_ratio=0.065622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121659712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=244563968 dispatch_meta_bytes=337312 combine_payload_bytes=244563968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244901280 ingress_bytes=279166976 link_bytes=279166976 token_bytes=489465248 weight_movement_bytes=34603008 total_nvlink_bytes=524068256 weight_to_token_ratio=0.070696 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.050482856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=347808 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265457312 ingress_bytes=334315520 link_bytes=334315520 token_bytes=530566816 weight_movement_bytes=69206016 total_nvlink_bytes=599772832 weight_to_token_ratio=0.130438 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258002387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=247439360 dispatch_meta_bytes=326392 combine_payload_bytes=247439360 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=247765752 ingress_bytes=351248384 link_bytes=351248384 token_bytes=495205112 weight_movement_bytes=103809024 total_nvlink_bytes=599014136 weight_to_token_ratio=0.209628 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321719391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=258056192 dispatch_meta_bytes=333128 combine_payload_bytes=258056192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258389320 ingress_bytes=327262208 link_bytes=327262208 token_bytes=516445512 weight_movement_bytes=69206016 total_nvlink_bytes=585651528 weight_to_token_ratio=0.134004 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231461342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75814 payload_rows_remote=66338 dup_rows=22490 dup_rows_remote=19944 dedup_ratio=0.228780 router_tokens_per_expert=[722, 1218, 1261, 157, 484, 143, 2314, 274, 360, 312, 1510, 992, 675, 3, 156, 1405, 791, 805, 1158, 68, 584, 214, 171, 592, 1079, 1128, 706, 403, 918, 512, 2472, 477, 763, 759, 882, 1788, 1262, 757, 1124, 867, 594, 299, 652, 1682, 179, 404, 646, 435, 699, 1203, 893, 426, 321, 1140, 118, 1211, 686, 447, 507, 1282, 179, 449, 1844, 590] payload_rows_per_dst=[10570, 8676, 7532, 8802, 9266, 10164, 9476, 11328] dup_rows_per_dst=[3172, 2150, 1234, 1644, 2966, 4562, 2546, 4216] combine_rows_per_src=[10570, 8676, 7532, 8802, 9266, 10164, 9476, 11328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75398 payload_rows_remote=63032 dup_rows=22906 dup_rows_remote=17542 dedup_ratio=0.233012 router_tokens_per_expert=[869, 584, 1233, 116, 514, 198, 589, 167, 350, 1011, 1441, 962, 502, 19, 309, 1607, 749, 605, 545, 87, 718, 117, 280, 2279, 947, 621, 683, 1080, 792, 804, 980, 546, 603, 1193, 1491, 3485, 738, 1588, 1240, 497, 381, 1354, 381, 1235, 485, 311, 227, 767, 1045, 696, 509, 229, 655, 650, 1083, 1904, 104, 591, 526, 695, 114, 186, 1659, 226] payload_rows_per_dst=[9084, 10680, 12366, 9056, 7482, 9230, 10528, 6972] dup_rows_per_dst=[2632, 3958, 5364, 1890, 1806, 3012, 3014, 1230] combine_rows_per_src=[9084, 10680, 12366, 9056, 7482, 9230, 10528, 6972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271720448 dispatch_meta_bytes=345128 combine_payload_bytes=271720448 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272065576 ingress_bytes=340926464 link_bytes=340926464 token_bytes=543786024 weight_movement_bytes=69206016 total_nvlink_bytes=612992040 weight_to_token_ratio=0.127267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282878837e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77364 payload_rows_remote=66334 dup_rows=20940 dup_rows_remote=17040 dedup_ratio=0.213013 router_tokens_per_expert=[940, 457, 1290, 63, 277, 184, 659, 204, 466, 1119, 1440, 1073, 413, 42, 557, 1327, 725, 681, 455, 126, 696, 79, 115, 2208, 1909, 372, 599, 1058, 812, 676, 1016, 400, 431, 1089, 1339, 3079, 666, 2200, 895, 375, 547, 1238, 427, 1408, 293, 289, 319, 675, 628, 797, 530, 236, 547, 934, 1272, 1348, 114, 535, 545, 1244, 70, 227, 2213, 204] payload_rows_per_dst=[9804, 9714, 9322, 9438, 11030, 9252, 10292, 8512] dup_rows_per_dst=[2744, 3160, 1666, 2214, 3900, 3172, 2292, 1792] combine_rows_per_src=[9804, 9714, 9322, 9438, 11030, 9252, 10292, 8512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76396 payload_rows_remote=67040 dup_rows=21908 dup_rows_remote=19766 dedup_ratio=0.222860 router_tokens_per_expert=[729, 614, 1316, 101, 440, 166, 1167, 161, 316, 956, 1391, 1075, 612, 36, 384, 1292, 821, 689, 922, 95, 589, 116, 122, 1481, 1620, 649, 596, 707, 914, 657, 1526, 606, 577, 1187, 1228, 2841, 1078, 1388, 984, 510, 416, 901, 490, 1566, 253, 320, 471, 570, 750, 1030, 588, 290, 382, 978, 611, 1205, 218, 516, 553, 1356, 147, 279, 2335, 268] payload_rows_per_dst=[9586, 9406, 11340, 9356, 8664, 9370, 9534, 9140] dup_rows_per_dst=[2578, 2718, 4012, 2142, 2464, 3656, 2134, 2204] combine_rows_per_src=[9586, 9406, 11340, 9356, 8664, 9370, 9534, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76538 payload_rows_remote=67618 dup_rows=21766 dup_rows_remote=19778 dedup_ratio=0.221415 router_tokens_per_expert=[682, 1341, 1254, 210, 455, 119, 2644, 350, 73, 88, 1243, 927, 1165, 0, 70, 1364, 760, 973, 2146, 82, 455, 382, 200, 104, 519, 1349, 763, 275, 760, 414, 2766, 600, 1203, 681, 524, 1437, 1100, 458, 1274, 1104, 358, 37, 287, 2015, 114, 543, 1023, 397, 900, 1070, 958, 683, 238, 900, 5, 1314, 854, 438, 478, 1226, 215, 542, 1547, 696] payload_rows_per_dst=[11144, 8920, 10046, 9952, 8520, 8904, 9390, 9662] dup_rows_per_dst=[3882, 1988, 3032, 2468, 2204, 3116, 2746, 2330] combine_rows_per_src=[11144, 8920, 10046, 9952, 8520, 8904, 9390, 9662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=258179072 dispatch_meta_bytes=322296 combine_payload_bytes=258179072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258501368 ingress_bytes=292782080 link_bytes=292782080 token_bytes=516680440 weight_movement_bytes=34603008 total_nvlink_bytes=551283448 weight_to_token_ratio=0.066972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101715396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76500 payload_rows_remote=63982 dup_rows=21804 dup_rows_remote=16954 dedup_ratio=0.221802 router_tokens_per_expert=[772, 317, 1215, 71, 376, 106, 520, 218, 280, 1380, 1528, 826, 376, 62, 735, 1052, 601, 629, 400, 101, 806, 76, 153, 2020, 1653, 423, 518, 932, 822, 746, 1061, 566, 458, 1563, 1478, 3458, 702, 1932, 731, 429, 193, 1752, 659, 1341, 397, 245, 270, 780, 733, 700, 410, 227, 487, 792, 1418, 1430, 69, 665, 571, 1218, 52, 195, 2245, 211] payload_rows_per_dst=[5996, 9872, 8328, 9210, 10728, 9574, 10274, 12518] dup_rows_per_dst=[1194, 2606, 1244, 2110, 3858, 3822, 2120, 4850] combine_rows_per_src=[5996, 9872, 8328, 9210, 10728, 9574, 10274, 12518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78280 payload_rows_remote=69206 dup_rows=20024 dup_rows_remote=17676 dedup_ratio=0.203695 router_tokens_per_expert=[701, 358, 1240, 82, 356, 121, 359, 192, 317, 2041, 1371, 902, 335, 48, 849, 1085, 672, 383, 384, 53, 566, 84, 123, 2786, 2300, 233, 633, 1400, 705, 639, 517, 581, 362, 1112, 1290, 3141, 763, 2592, 796, 330, 342, 2119, 229, 1193, 290, 174, 177, 670, 530, 659, 317, 191, 451, 765, 2402, 1117, 74, 570, 464, 947, 85, 192, 2235, 157] payload_rows_per_dst=[9462, 10472, 9212, 10242, 9506, 9074, 10630, 9682] dup_rows_per_dst=[2540, 3424, 890, 2740, 3200, 2348, 2234, 2648] combine_rows_per_src=[9462, 10472, 9212, 10242, 9506, 9074, 10630, 9682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70356 payload_rows_remote=61238 dup_rows=27948 dup_rows_remote=21626 dedup_ratio=0.284302 router_tokens_per_expert=[22, 733, 1629, 700, 1069, 949, 1008, 1218, 603, 340, 1860, 158, 283, 447, 602, 1187, 1403, 175, 246, 98, 195, 359, 350, 295, 240, 395, 1227, 461, 1596, 1019, 640, 305, 1045, 134, 261, 867, 1049, 1069, 177, 1307, 325, 303, 392, 840, 891, 74, 2202, 86, 485, 494, 222, 2492, 722, 1015, 848, 1214, 703, 354, 437, 1105, 889, 2163, 1611, 1564] payload_rows_per_dst=[9118, 10772, 7824, 11726, 6506, 4686, 10568, 9156] dup_rows_per_dst=[6322, 4592, 2646, 5024, 1084, 352, 3758, 4170] combine_rows_per_src=[9118, 10772, 7824, 11726, 6506, 4686, 10568, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271704064 dispatch_meta_bytes=333496 combine_payload_bytes=271704064 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=272037560 ingress_bytes=444719104 link_bytes=444719104 token_bytes=543741624 weight_movement_bytes=173015040 total_nvlink_bytes=716756664 weight_to_token_ratio=0.318193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.673442186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274595840 dispatch_meta_bytes=347224 combine_payload_bytes=274595840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274943064 ingress_bytes=343801856 link_bytes=343801856 token_bytes=549538904 weight_movement_bytes=69206016 total_nvlink_bytes=618744920 weight_to_token_ratio=0.125935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293698706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276963328 dispatch_meta_bytes=349584 combine_payload_bytes=276963328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277312912 ingress_bytes=311566336 link_bytes=311566336 token_bytes=554276240 weight_movement_bytes=34603008 total_nvlink_bytes=588879248 weight_to_token_ratio=0.062429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172399039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=262070272 dispatch_meta_bytes=323744 combine_payload_bytes=262070272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262394016 ingress_bytes=296673280 link_bytes=296673280 token_bytes=524464288 weight_movement_bytes=34603008 total_nvlink_bytes=559067296 weight_to_token_ratio=0.065978 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116357668e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283467776 dispatch_meta_bytes=347528 combine_payload_bytes=283467776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283815304 ingress_bytes=318070784 link_bytes=318070784 token_bytes=567283080 weight_movement_bytes=34603008 total_nvlink_bytes=601886088 weight_to_token_ratio=0.060998 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196874753e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=250830848 dispatch_meta_bytes=331456 combine_payload_bytes=250830848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251162304 ingress_bytes=320036864 link_bytes=320036864 token_bytes=501993152 weight_movement_bytes=69206016 total_nvlink_bytes=571199168 weight_to_token_ratio=0.137862 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204272954e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76850 payload_rows_remote=64180 dup_rows=21454 dup_rows_remote=18158 dedup_ratio=0.218241 router_tokens_per_expert=[726, 346, 104, 2135, 285, 501, 427, 269, 193, 601, 1224, 1172, 1086, 245, 177, 1024, 131, 212, 707, 23, 2483, 484, 657, 1600, 178, 718, 346, 348, 1040, 925, 28, 168, 211, 1322, 447, 68, 41, 753, 145, 2425, 1336, 291, 444, 1440, 3428, 900, 401, 1755, 531, 1686, 58, 377, 540, 563, 4308, 447, 2437, 441, 26, 51, 1032, 41, 203, 441] payload_rows_per_dst=[8118, 12254, 9728, 7026, 8756, 9964, 12670, 8334] dup_rows_per_dst=[1468, 5438, 2866, 1370, 2068, 3778, 3296, 1170] combine_rows_per_src=[8118, 12254, 9728, 7026, 8756, 9964, 12670, 8334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75694 payload_rows_remote=63840 dup_rows=22610 dup_rows_remote=18518 dedup_ratio=0.230001 router_tokens_per_expert=[913, 236, 855, 19, 1207, 1185, 521, 672, 679, 382, 162, 546, 435, 7, 3, 355, 4, 355, 831, 137, 2456, 565, 1009, 121, 272, 1072, 95, 397, 933, 697, 335, 236, 255, 420, 75, 179, 609, 2193, 2426, 1148, 642, 471, 1847, 579, 193, 764, 131, 337, 333, 3166, 1035, 49, 2487, 320, 294, 3537, 156, 2112, 835, 741, 3463, 229, 153, 281] payload_rows_per_dst=[11044, 5002, 11854, 11822, 8274, 7562, 8250, 11886] dup_rows_per_dst=[4558, 522, 4092, 3326, 1950, 1980, 2128, 4054] combine_rows_per_src=[11044, 5002, 11854, 11822, 8274, 7562, 8250, 11886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262881280 dispatch_meta_bytes=329352 combine_payload_bytes=262881280 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=263210632 ingress_bytes=435896320 link_bytes=435896320 token_bytes=526091912 weight_movement_bytes=173015040 total_nvlink_bytes=699106952 weight_to_token_ratio=0.328868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.640242760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75486 payload_rows_remote=64840 dup_rows=22818 dup_rows_remote=18458 dedup_ratio=0.232117 router_tokens_per_expert=[965, 283, 823, 260, 1531, 1208, 581, 692, 568, 235, 255, 753, 540, 27, 33, 311, 67, 463, 396, 40, 3275, 498, 854, 337, 215, 1117, 110, 351, 1580, 797, 241, 325, 215, 410, 131, 56, 363, 3161, 1920, 1247, 680, 450, 1300, 676, 540, 601, 99, 398, 462, 2570, 999, 64, 1628, 385, 621, 3485, 462, 1760, 401, 381, 3155, 303, 107, 391] payload_rows_per_dst=[9420, 5654, 10192, 12122, 10646, 6552, 9898, 11002] dup_rows_per_dst=[3266, 870, 1668, 4320, 4360, 1856, 3560, 2918] combine_rows_per_src=[9420, 5654, 10192, 12122, 10646, 6552, 9898, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75890 payload_rows_remote=63976 dup_rows=22414 dup_rows_remote=18292 dedup_ratio=0.228007 router_tokens_per_expert=[792, 285, 648, 687, 1184, 1037, 543, 634, 559, 440, 569, 974, 698, 97, 57, 653, 104, 397, 706, 6, 3237, 541, 829, 467, 257, 1125, 186, 472, 1706, 976, 144, 228, 219, 727, 254, 77, 197, 2154, 968, 1696, 980, 415, 961, 803, 1256, 782, 213, 733, 532, 2293, 430, 195, 905, 394, 1714, 2924, 834, 1481, 276, 307, 2455, 248, 158, 333] payload_rows_per_dst=[10322, 8192, 10230, 11914, 7688, 7526, 10168, 9850] dup_rows_per_dst=[4102, 2414, 2344, 4122, 2092, 2248, 2758, 2334] combine_rows_per_src=[10322, 8192, 10230, 11914, 7688, 7526, 10168, 9850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=261488640 dispatch_meta_bytes=329432 combine_payload_bytes=261488640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261818072 ingress_bytes=296091648 link_bytes=296091648 token_bytes=523306712 weight_movement_bytes=34603008 total_nvlink_bytes=557909720 weight_to_token_ratio=0.066124 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114169034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75664 payload_rows_remote=61418 dup_rows=22640 dup_rows_remote=12096 dedup_ratio=0.230306 router_tokens_per_expert=[695, 230, 15, 2681, 141, 144, 298, 110, 33, 603, 2215, 1219, 1260, 347, 356, 1803, 46, 82, 746, 16, 1189, 483, 515, 2224, 131, 414, 540, 90, 744, 492, 0, 34, 172, 1559, 374, 25, 0, 69, 2, 2457, 1356, 238, 284, 1550, 4559, 685, 643, 2842, 403, 1004, 3, 560, 381, 772, 5158, 12, 3001, 69, 0, 33, 450, 10, 304, 281] payload_rows_per_dst=[7524, 14246, 9784, 4546, 8192, 10744, 12910, 7718] dup_rows_per_dst=[1242, 10544, 2826, 368, 986, 4452, 1644, 578] combine_rows_per_src=[7524, 14246, 9784, 4546, 8192, 10744, 12910, 7718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74660 payload_rows_remote=62508 dup_rows=23644 dup_rows_remote=19410 dedup_ratio=0.240519 router_tokens_per_expert=[784, 165, 808, 16, 2074, 1426, 563, 680, 592, 250, 79, 430, 519, 3, 142, 270, 25, 609, 506, 4, 3469, 479, 845, 114, 214, 1016, 122, 296, 1142, 731, 194, 242, 208, 349, 64, 40, 482, 3569, 2015, 1086, 429, 430, 1493, 602, 98, 586, 86, 298, 443, 2264, 816, 54, 1689, 334, 161, 4715, 73, 2020, 664, 590, 4074, 196, 109, 306] payload_rows_per_dst=[9628, 4062, 10304, 12132, 10838, 6336, 9208, 12152] dup_rows_per_dst=[3404, 508, 1798, 3562, 4788, 1708, 3642, 4234] combine_rows_per_src=[9628, 4062, 10304, 12132, 10838, 6336, 9208, 12152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76268 payload_rows_remote=70184 dup_rows=22036 dup_rows_remote=20628 dedup_ratio=0.224162 router_tokens_per_expert=[724, 142, 1031, 66, 2887, 1116, 532, 814, 654, 184, 124, 470, 446, 44, 19, 292, 89, 530, 230, 6, 3370, 431, 789, 152, 189, 847, 108, 290, 1140, 583, 203, 298, 225, 219, 98, 21, 663, 3893, 2395, 751, 394, 414, 1630, 585, 202, 374, 93, 256, 329, 2601, 717, 94, 1984, 292, 208, 5030, 103, 1308, 520, 447, 3802, 358, 107, 239] payload_rows_per_dst=[10394, 4394, 9810, 13658, 11408, 6084, 9358, 11162] dup_rows_per_dst=[4230, 476, 1384, 3718, 5122, 1408, 3092, 2606] combine_rows_per_src=[10394, 4394, 9810, 13658, 11408, 6084, 9358, 11162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76676 payload_rows_remote=65940 dup_rows=21628 dup_rows_remote=18448 dedup_ratio=0.220011 router_tokens_per_expert=[596, 1131, 1135, 218, 406, 32, 2705, 392, 59, 27, 1342, 833, 1188, 0, 69, 1259, 710, 1004, 2091, 53, 362, 524, 163, 37, 300, 1659, 844, 249, 748, 382, 3362, 886, 1229, 610, 503, 1414, 884, 343, 1253, 1487, 195, 8, 373, 1656, 88, 735, 1054, 415, 857, 1088, 1075, 1122, 214, 805, 2, 1366, 838, 464, 467, 1065, 154, 378, 1444, 800] payload_rows_per_dst=[10736, 8752, 9826, 12208, 9640, 7332, 9054, 9128] dup_rows_per_dst=[3180, 1808, 2890, 4832, 3070, 1716, 2040, 2092] combine_rows_per_src=[10736, 8752, 9826, 12208, 9640, 7332, 9054, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=333192 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265917832 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531502472 weight_movement_bytes=69206016 total_nvlink_bytes=600708488 weight_to_token_ratio=0.130208 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262045696 dispatch_meta_bytes=329072 combine_payload_bytes=262045696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262374768 ingress_bytes=296648704 link_bytes=296648704 token_bytes=524420464 weight_movement_bytes=34603008 total_nvlink_bytes=559023472 weight_to_token_ratio=0.065983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116265191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=251568128 dispatch_meta_bytes=294056 combine_payload_bytes=251568128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251862184 ingress_bytes=286171136 link_bytes=286171136 token_bytes=503430312 weight_movement_bytes=34603008 total_nvlink_bytes=538033320 weight_to_token_ratio=0.068734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076838946e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77292 payload_rows_remote=65172 dup_rows=21012 dup_rows_remote=16172 dedup_ratio=0.213745 router_tokens_per_expert=[1848, 1952, 51, 520, 2173, 496, 472, 341, 178, 446, 352, 356, 102, 506, 570, 553, 381, 587, 2, 146, 757, 185, 3803, 884, 1221, 2304, 771, 212, 136, 365, 71, 592, 82, 332, 2894, 19, 78, 1240, 720, 369, 789, 1177, 25, 380, 2360, 3376, 720, 6, 4150, 271, 3181, 611, 503, 241, 445, 257, 15, 582, 21, 12, 13, 323, 274, 353] payload_rows_per_dst=[8508, 7238, 11188, 9052, 10122, 12016, 12120, 7048] dup_rows_per_dst=[1908, 1246, 2298, 2296, 2290, 5650, 4840, 484] combine_rows_per_src=[8508, 7238, 11188, 9052, 10122, 12016, 12120, 7048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287473664 dispatch_meta_bytes=363248 combine_payload_bytes=287473664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287836912 ingress_bytes=356679680 link_bytes=356679680 token_bytes=575310576 weight_movement_bytes=69206016 total_nvlink_bytes=644516592 weight_to_token_ratio=0.120293 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342156921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=256032768 dispatch_meta_bytes=327672 combine_payload_bytes=256032768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256360440 ingress_bytes=290635776 link_bytes=290635776 token_bytes=512393208 weight_movement_bytes=34603008 total_nvlink_bytes=546996216 weight_to_token_ratio=0.067532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093639027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270090240 dispatch_meta_bytes=337552 combine_payload_bytes=270090240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270427792 ingress_bytes=304693248 link_bytes=304693248 token_bytes=540518032 weight_movement_bytes=34603008 total_nvlink_bytes=575121040 weight_to_token_ratio=0.064018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146536162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76240 payload_rows_remote=62532 dup_rows=22064 dup_rows_remote=14348 dedup_ratio=0.224447 router_tokens_per_expert=[262, 301, 4347, 157, 1408, 239, 2739, 184, 7, 529, 1059, 113, 1285, 1623, 495, 280, 592, 831, 5070, 1816, 198, 118, 40, 567, 449, 1, 229, 95, 792, 514, 199, 538, 1567, 460, 92, 291, 79, 140, 525, 78, 609, 57, 597, 621, 211, 88, 7, 2030, 3, 4227, 482, 357, 1045, 100, 302, 275, 163, 1049, 3861, 1374, 478, 5, 679, 223] payload_rows_per_dst=[9974, 8698, 13708, 5026, 9642, 7464, 8748, 12980] dup_rows_per_dst=[1758, 2090, 7716, 608, 2300, 976, 1868, 4748] combine_rows_per_src=[9974, 8698, 13708, 5026, 9642, 7464, 8748, 12980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=325376 combine_payload_bytes=266944512 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267269888 ingress_bytes=370753536 link_bytes=370753536 token_bytes=534214400 weight_movement_bytes=103809024 total_nvlink_bytes=638023424 weight_to_token_ratio=0.194321 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395115708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78416 payload_rows_remote=70480 dup_rows=19888 dup_rows_remote=18474 dedup_ratio=0.202311 router_tokens_per_expert=[675, 810, 724, 846, 3172, 960, 2176, 560, 354, 459, 1095, 525, 337, 1093, 567, 621, 525, 400, 1205, 936, 550, 340, 2147, 1047, 590, 776, 610, 114, 483, 501, 148, 777, 602, 427, 1957, 141, 63, 464, 671, 312, 816, 833, 242, 573, 1796, 1171, 306, 401, 1835, 1211, 2764, 558, 644, 222, 436, 819, 148, 805, 1296, 281, 296, 152, 429, 358] payload_rows_per_dst=[7840, 11012, 10164, 7936, 10380, 9548, 10522, 11014] dup_rows_per_dst=[1310, 2760, 2784, 1414, 3246, 2728, 2786, 2860] combine_rows_per_src=[7840, 11012, 10164, 7936, 10380, 9548, 10522, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79430 payload_rows_remote=68902 dup_rows=18874 dup_rows_remote=16116 dedup_ratio=0.191996 router_tokens_per_expert=[436, 486, 1995, 515, 2572, 626, 2848, 378, 410, 511, 1016, 302, 788, 1574, 568, 461, 600, 260, 3139, 1556, 354, 361, 943, 791, 517, 333, 323, 111, 534, 550, 157, 746, 852, 435, 1045, 273, 81, 243, 662, 204, 758, 309, 374, 610, 1187, 425, 150, 1562, 482, 2666, 1733, 398, 752, 164, 403, 603, 93, 877, 2972, 659, 772, 56, 288, 303] payload_rows_per_dst=[7762, 9802, 8328, 11090, 10528, 8966, 10626, 12328] dup_rows_per_dst=[1110, 2422, 1402, 1730, 2758, 1784, 2812, 4856] combine_rows_per_src=[7762, 9802, 8328, 11090, 10528, 8966, 10626, 12328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=256131072 dispatch_meta_bytes=307520 combine_payload_bytes=256131072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256438592 ingress_bytes=325337088 link_bytes=325337088 token_bytes=512569664 weight_movement_bytes=69206016 total_nvlink_bytes=581775680 weight_to_token_ratio=0.135018 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224217270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78094 payload_rows_remote=65932 dup_rows=20210 dup_rows_remote=18366 dedup_ratio=0.205587 router_tokens_per_expert=[1963, 2880, 51, 387, 1376, 345, 124, 222, 17, 390, 181, 117, 110, 323, 579, 526, 377, 904, 0, 46, 815, 108, 3585, 1173, 2082, 2627, 1347, 299, 4, 275, 96, 470, 8, 272, 2268, 0, 99, 1415, 715, 305, 719, 510, 3, 199, 2552, 5008, 729, 0, 4760, 0, 2684, 674, 477, 157, 330, 19, 25, 411, 0, 0, 0, 447, 191, 376] payload_rows_per_dst=[10756, 12162, 11266, 10934, 9110, 12878, 8102, 2886] dup_rows_per_dst=[3692, 1844, 2750, 3466, 1302, 6562, 580, 14] combine_rows_per_src=[10756, 12162, 11266, 10934, 9110, 12878, 8102, 2886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83000 payload_rows_remote=72928 dup_rows=15304 dup_rows_remote=14122 dedup_ratio=0.155680 router_tokens_per_expert=[297, 234, 3580, 278, 1647, 513, 3512, 127, 372, 443, 771, 129, 1119, 1701, 578, 396, 533, 48, 4595, 1723, 144, 232, 272, 515, 183, 103, 264, 91, 601, 534, 202, 509, 854, 305, 390, 240, 88, 100, 528, 114, 708, 155, 359, 473, 440, 204, 23, 3265, 264, 4434, 869, 301, 776, 101, 389, 652, 7, 665, 4333, 874, 639, 10, 101, 245] payload_rows_per_dst=[9144, 9166, 6468, 13036, 10224, 10072, 11908, 12982] dup_rows_per_dst=[914, 2380, 466, 1128, 2038, 1182, 3136, 4060] combine_rows_per_src=[9144, 9166, 6468, 13036, 10224, 10072, 11908, 12982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288686080 dispatch_meta_bytes=355816 combine_payload_bytes=288686080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289041896 ingress_bytes=323289088 link_bytes=323289088 token_bytes=577727976 weight_movement_bytes=34603008 total_nvlink_bytes=612330984 weight_to_token_ratio=0.059895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216510811e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80306 payload_rows_remote=66838 dup_rows=17998 dup_rows_remote=12632 dedup_ratio=0.183085 router_tokens_per_expert=[251, 303, 3743, 153, 1990, 493, 3931, 159, 286, 450, 1287, 101, 1661, 1644, 563, 436, 688, 238, 4933, 1265, 61, 208, 125, 661, 169, 0, 237, 90, 554, 565, 247, 353, 1410, 487, 118, 270, 88, 66, 526, 94, 709, 33, 514, 666, 186, 131, 2, 2008, 11, 3546, 963, 295, 949, 92, 361, 355, 72, 825, 4393, 1161, 406, 2, 272, 296] payload_rows_per_dst=[8316, 10116, 5882, 13178, 11302, 7528, 10516, 13468] dup_rows_per_dst=[1154, 2740, 610, 1118, 3412, 970, 2628, 5366] combine_rows_per_src=[8316, 10116, 5882, 13178, 11302, 7528, 10516, 13468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72086 payload_rows_remote=63598 dup_rows=26218 dup_rows_remote=25122 dedup_ratio=0.266703 router_tokens_per_expert=[671, 162, 0, 3403, 97, 90, 272, 93, 1, 961, 1687, 1211, 1187, 394, 403, 947, 5, 45, 793, 1, 897, 548, 453, 1603, 137, 354, 545, 4, 555, 549, 0, 5, 95, 1645, 256, 16, 0, 4, 0, 2745, 1635, 282, 222, 1521, 5073, 364, 645, 3532, 403, 367, 0, 1076, 376, 1001, 5668, 0, 3215, 14, 0, 15, 355, 2, 377, 175] payload_rows_per_dst=[8488, 14370, 7560, 4090, 9152, 15008, 5546, 7872] dup_rows_per_dst=[1096, 9358, 1864, 208, 1984, 11098, 176, 434] combine_rows_per_src=[8488, 14370, 7560, 4090, 9152, 15008, 5546, 7872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=282222592 dispatch_meta_bytes=340072 combine_payload_bytes=282222592 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=282562664 ingress_bytes=351428608 link_bytes=351428608 token_bytes=564785256 weight_movement_bytes=69206016 total_nvlink_bytes=633991272 weight_to_token_ratio=0.122535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322397560e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270057472 dispatch_meta_bytes=337192 combine_payload_bytes=270057472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270394664 ingress_bytes=304660480 link_bytes=304660480 token_bytes=540452136 weight_movement_bytes=34603008 total_nvlink_bytes=575055144 weight_to_token_ratio=0.064026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146412859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72556 payload_rows_remote=66022 dup_rows=25748 dup_rows_remote=24792 dedup_ratio=0.261922 router_tokens_per_expert=[610, 2364, 2877, 3055, 545, 4, 4995, 5550, 23, 162, 60, 780, 0, 1956, 61, 133, 295, 53, 1577, 218, 1242, 64, 755, 274, 59, 132, 2205, 0, 569, 129, 0, 340, 4813, 943, 12, 143, 334, 793, 561, 0, 399, 18, 389, 36, 194, 0, 359, 110, 1473, 0, 188, 1063, 72, 818, 131, 0, 2628, 580, 557, 373, 705, 13, 347, 13] payload_rows_per_dst=[10466, 5840, 14344, 5846, 8164, 6842, 6534, 14520] dup_rows_per_dst=[2690, 510, 7160, 1022, 2746, 456, 956, 10208] combine_rows_per_src=[10466, 5840, 14344, 5846, 8164, 6842, 6534, 14520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298713088 dispatch_meta_bytes=348200 combine_payload_bytes=298713088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299061288 ingress_bytes=367919104 link_bytes=367919104 token_bytes=597774376 weight_movement_bytes=69206016 total_nvlink_bytes=666980392 weight_to_token_ratio=0.115773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384449968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273768448 dispatch_meta_bytes=317880 combine_payload_bytes=273768448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274086328 ingress_bytes=308371456 link_bytes=308371456 token_bytes=547854776 weight_movement_bytes=34603008 total_nvlink_bytes=582457784 weight_to_token_ratio=0.063161 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160376963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71040 payload_rows_remote=67226 dup_rows=27264 dup_rows_remote=26820 dedup_ratio=0.277344 router_tokens_per_expert=[377, 141, 28, 1, 0, 1907, 348, 0, 1623, 225, 1535, 292, 6790, 5, 355, 105, 127, 11, 0, 37, 208, 42, 1259, 97, 98, 500, 135, 1099, 319, 135, 4090, 449, 0, 200, 1092, 407, 259, 476, 685, 2670, 425, 556, 752, 4272, 190, 5685, 2918, 198, 296, 450, 5, 1, 254, 0, 62, 3469, 0, 344, 0, 49, 70, 0, 694, 335] payload_rows_per_dst=[4860, 15102, 3814, 10988, 9264, 15562, 8616, 2834] dup_rows_per_dst=[48, 6758, 444, 2662, 2314, 14430, 458, 150] combine_rows_per_src=[4860, 15102, 3814, 10988, 9264, 15562, 8616, 2834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=260497408 dispatch_meta_bytes=354880 combine_payload_bytes=260497408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260852288 ingress_bytes=295100416 link_bytes=295100416 token_bytes=521349696 weight_movement_bytes=34603008 total_nvlink_bytes=555952704 weight_to_token_ratio=0.066372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110439108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=363256 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270789368 ingress_bytes=305029120 link_bytes=305029120 token_bytes=541215480 weight_movement_bytes=34603008 total_nvlink_bytes=575818488 weight_to_token_ratio=0.063936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147800022e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=66520 dup_rows=23912 dup_rows_remote=21426 dedup_ratio=0.243245 router_tokens_per_expert=[671, 2168, 1488, 1171, 250, 913, 2596, 2468, 413, 339, 565, 632, 1100, 1364, 362, 495, 304, 264, 1405, 237, 1123, 102, 1337, 598, 96, 473, 1427, 348, 1818, 124, 328, 565, 2439, 788, 276, 316, 619, 977, 751, 565, 469, 416, 1184, 1749, 355, 1934, 737, 289, 899, 257, 323, 665, 946, 379, 102, 374, 1284, 632, 325, 229, 363, 12, 666, 288] payload_rows_per_dst=[9830, 8298, 11534, 7872, 9994, 8476, 9026, 9362] dup_rows_per_dst=[3492, 2242, 4398, 2486, 3468, 2374, 2280, 3172] combine_rows_per_src=[9830, 8298, 11534, 7872, 9994, 8476, 9026, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275357696 dispatch_meta_bytes=376184 combine_payload_bytes=275357696 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275733880 ingress_bytes=344563712 link_bytes=344563712 token_bytes=551091576 weight_movement_bytes=69206016 total_nvlink_bytes=620297592 weight_to_token_ratio=0.125580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296565509e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74050 payload_rows_remote=65552 dup_rows=24254 dup_rows_remote=21946 dedup_ratio=0.246724 router_tokens_per_expert=[514, 949, 892, 449, 128, 1811, 1496, 965, 1233, 314, 1847, 453, 4197, 531, 385, 312, 80, 134, 488, 89, 661, 98, 1369, 337, 108, 581, 1008, 785, 1437, 134, 1443, 527, 870, 483, 499, 302, 354, 745, 716, 1434, 486, 359, 1333, 3141, 280, 4450, 1400, 278, 491, 415, 134, 275, 783, 227, 33, 1532, 437, 562, 140, 112, 133, 3, 642, 348] payload_rows_per_dst=[7790, 12270, 7482, 8788, 8498, 11214, 12500, 5508] dup_rows_per_dst=[1696, 6274, 2022, 3258, 2308, 3340, 4180, 1176] combine_rows_per_src=[7790, 12270, 7482, 8788, 8498, 11214, 12500, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73546 payload_rows_remote=68220 dup_rows=24758 dup_rows_remote=24298 dedup_ratio=0.251851 router_tokens_per_expert=[575, 1194, 3139, 3334, 989, 0, 5618, 5966, 0, 195, 0, 799, 0, 1762, 0, 137, 255, 6, 1914, 94, 1029, 105, 573, 168, 63, 135, 2223, 0, 101, 162, 0, 272, 5215, 945, 0, 106, 358, 986, 583, 0, 413, 0, 261, 0, 125, 0, 330, 79, 1680, 0, 125, 779, 52, 1009, 128, 0, 2061, 556, 251, 856, 1186, 2, 258, 0] payload_rows_per_dst=[12898, 5326, 14306, 5172, 12348, 2412, 6622, 14462] dup_rows_per_dst=[5564, 460, 5218, 740, 4038, 4, 924, 7810] combine_rows_per_src=[12898, 5326, 14306, 5172, 12348, 2412, 6622, 14462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75328 payload_rows_remote=63436 dup_rows=22976 dup_rows_remote=19934 dedup_ratio=0.233724 router_tokens_per_expert=[354, 394, 371, 197, 26, 2863, 672, 356, 1924, 235, 2772, 285, 6024, 237, 365, 72, 64, 25, 199, 36, 306, 53, 1157, 102, 126, 484, 261, 822, 1157, 150, 2056, 409, 313, 343, 785, 482, 199, 215, 671, 2847, 462, 342, 830, 3579, 195, 5560, 1913, 146, 459, 451, 39, 75, 496, 94, 18, 2569, 104, 350, 54, 86, 41, 4, 692, 184] payload_rows_per_dst=[7464, 12148, 4686, 11832, 9846, 11892, 14238, 3222] dup_rows_per_dst=[482, 5188, 1006, 5590, 1864, 3042, 5284, 520] combine_rows_per_src=[7464, 12148, 4686, 11832, 9846, 11892, 14238, 3222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=272465920 dispatch_meta_bytes=351784 combine_payload_bytes=272465920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272817704 ingress_bytes=307068928 link_bytes=307068928 token_bytes=545283624 weight_movement_bytes=34603008 total_nvlink_bytes=579886632 weight_to_token_ratio=0.063459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155475655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268500992 dispatch_meta_bytes=349992 combine_payload_bytes=268500992 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=268850984 ingress_bytes=372310016 link_bytes=372310016 token_bytes=537351976 weight_movement_bytes=103809024 total_nvlink_bytes=641161000 weight_to_token_ratio=0.193186 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400972617e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=71090 dup_rows=24280 dup_rows_remote=24016 dedup_ratio=0.246989 router_tokens_per_expert=[316, 146, 140, 0, 1, 2998, 309, 0, 1492, 195, 1792, 398, 6617, 25, 385, 90, 29, 5, 0, 21, 211, 21, 1459, 145, 101, 602, 215, 1197, 899, 140, 3354, 449, 1, 261, 1046, 367, 184, 69, 749, 4076, 421, 690, 425, 4016, 159, 5687, 2204, 224, 345, 466, 18, 2, 166, 0, 38, 2327, 0, 471, 0, 51, 5, 0, 630, 302] payload_rows_per_dst=[6810, 7462, 3964, 15208, 11304, 12408, 13934, 2934] dup_rows_per_dst=[112, 1292, 436, 11940, 2200, 3872, 4164, 264] combine_rows_per_src=[6810, 7462, 3964, 15208, 11304, 12408, 13934, 2934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76402 payload_rows_remote=62076 dup_rows=21902 dup_rows_remote=12070 dedup_ratio=0.222799 router_tokens_per_expert=[2155, 3770, 1, 316, 999, 138, 84, 61, 4, 402, 136, 143, 124, 321, 669, 476, 372, 1168, 0, 4, 463, 93, 3402, 1135, 2525, 3097, 1484, 247, 0, 319, 140, 334, 1, 365, 1746, 1, 90, 1468, 887, 183, 771, 214, 0, 95, 1917, 5623, 523, 0, 5039, 0, 2576, 582, 465, 103, 316, 0, 11, 388, 0, 0, 0, 490, 286, 430] payload_rows_per_dst=[14326, 12756, 10684, 12246, 8530, 7934, 6772, 3154] dup_rows_per_dst=[9832, 1872, 2590, 4046, 1120, 2046, 340, 56] combine_rows_per_src=[14326, 12756, 10684, 12246, 8530, 7934, 6772, 3154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279429120 dispatch_meta_bytes=370072 combine_payload_bytes=279429120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279799192 ingress_bytes=348635136 link_bytes=348635136 token_bytes=559228312 weight_movement_bytes=69206016 total_nvlink_bytes=628434328 weight_to_token_ratio=0.123753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311885949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75542 payload_rows_remote=62692 dup_rows=22762 dup_rows_remote=16920 dedup_ratio=0.231547 router_tokens_per_expert=[0, 4283, 1660, 48, 56, 244, 502, 4618, 17, 9, 0, 1, 0, 492, 2935, 214, 324, 429, 216, 169, 70, 4, 516, 992, 729, 538, 121, 319, 244, 130, 224, 3, 50, 6320, 549, 25, 571, 52, 1200, 0, 732, 0, 52, 27, 1682, 243, 0, 0, 7, 0, 83, 1385, 1912, 759, 202, 0, 55, 422, 5426, 93, 229, 2476, 311, 4182] payload_rows_per_dst=[11826, 7278, 4634, 13042, 6408, 5268, 12850, 14236] dup_rows_per_dst=[2430, 58, 806, 2426, 1130, 204, 5842, 9866] combine_rows_per_src=[11826, 7278, 4634, 13042, 6408, 5268, 12850, 14236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=259833856 dispatch_meta_bytes=333480 combine_payload_bytes=259833856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260167336 ingress_bytes=329039872 link_bytes=329039872 token_bytes=520001192 weight_movement_bytes=69206016 total_nvlink_bytes=589207208 weight_to_token_ratio=0.133088 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238150548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72818 payload_rows_remote=56926 dup_rows=25486 dup_rows_remote=15634 dedup_ratio=0.259257 router_tokens_per_expert=[374, 390, 20, 60, 1, 49, 0, 0, 2, 1654, 2403, 440, 24, 408, 161, 59, 529, 0, 101, 0, 267, 4510, 6, 295, 0, 250, 556, 40, 266, 49, 157, 0, 4, 1, 81, 2251, 1, 1146, 0, 0, 58, 65, 7164, 237, 0, 958, 448, 6923, 19, 6124, 0, 0, 0, 488, 436, 7244, 35, 324, 1, 61, 176, 28, 1730, 78] payload_rows_per_dst=[12358, 15074, 15892, 2142, 9242, 10804, 1886, 5420] dup_rows_per_dst=[898, 9716, 9852, 652, 2702, 1598, 0, 68] combine_rows_per_src=[12358, 15074, 15892, 2142, 9242, 10804, 1886, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291184640 dispatch_meta_bytes=380424 combine_payload_bytes=291184640 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291565064 ingress_bytes=360390656 link_bytes=360390656 token_bytes=582749704 weight_movement_bytes=69206016 total_nvlink_bytes=651955720 weight_to_token_ratio=0.118758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356121025e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=254263296 dispatch_meta_bytes=296584 combine_payload_bytes=254263296 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=254559880 ingress_bytes=392675328 link_bytes=392675328 token_bytes=508823176 weight_movement_bytes=138412032 total_nvlink_bytes=647235208 weight_to_token_ratio=0.272024 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.477605646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75626 payload_rows_remote=63796 dup_rows=22678 dup_rows_remote=16712 dedup_ratio=0.230693 router_tokens_per_expert=[331, 1970, 1338, 639, 226, 256, 385, 1809, 190, 810, 410, 1199, 97, 529, 1986, 636, 686, 370, 1956, 62, 153, 452, 507, 875, 332, 703, 645, 1019, 291, 474, 185, 15, 140, 2069, 733, 408, 492, 757, 1468, 88, 264, 115, 356, 155, 762, 325, 182, 225, 201, 124, 128, 976, 1998, 1486, 529, 594, 954, 475, 2227, 112, 547, 2280, 3439, 3007] payload_rows_per_dst=[8640, 10110, 8534, 11830, 9898, 3818, 9174, 13622] dup_rows_per_dst=[1576, 2792, 2300, 5966, 2412, 238, 1462, 5932] combine_rows_per_src=[8640, 10110, 8534, 11830, 9898, 3818, 9174, 13622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=256786432 dispatch_meta_bytes=318448 combine_payload_bytes=256786432 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257104880 ingress_bytes=360595456 link_bytes=360595456 token_bytes=513891312 weight_movement_bytes=103809024 total_nvlink_bytes=617700336 weight_to_token_ratio=0.202006 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356891671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73456 payload_rows_remote=64834 dup_rows=24848 dup_rows_remote=23278 dedup_ratio=0.252767 router_tokens_per_expert=[433, 1146, 487, 590, 95, 138, 124, 632, 83, 1537, 1345, 1252, 219, 468, 835, 258, 1043, 131, 1327, 22, 370, 2120, 271, 521, 215, 362, 705, 399, 302, 158, 195, 0, 75, 814, 462, 1246, 140, 1616, 548, 195, 111, 135, 3395, 182, 300, 480, 682, 2909, 155, 1650, 59, 564, 774, 1152, 512, 4290, 1000, 414, 809, 69, 394, 947, 3973, 1317] payload_rows_per_dst=[7002, 12846, 12300, 6200, 8622, 8260, 5880, 12346] dup_rows_per_dst=[1232, 7634, 6100, 2724, 1570, 1338, 710, 3540] combine_rows_per_src=[7002, 12846, 12300, 6200, 8622, 8260, 5880, 12346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233168896 dispatch_meta_bytes=290240 combine_payload_bytes=233168896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233459136 ingress_bytes=267771904 link_bytes=267771904 token_bytes=466628032 weight_movement_bytes=34603008 total_nvlink_bytes=501231040 weight_to_token_ratio=0.074155 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.007604117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261308416 dispatch_meta_bytes=322032 combine_payload_bytes=261308416 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261630448 ingress_bytes=330514432 link_bytes=330514432 token_bytes=522938864 weight_movement_bytes=69206016 total_nvlink_bytes=592144880 weight_to_token_ratio=0.132341 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243699199e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=67790 dup_rows=24138 dup_rows_remote=24128 dedup_ratio=0.245544 router_tokens_per_expert=[0, 5925, 1735, 15, 15, 91, 556, 4701, 0, 0, 0, 0, 0, 457, 2553, 183, 303, 726, 125, 231, 82, 0, 323, 934, 780, 384, 43, 197, 229, 87, 227, 1, 69, 7164, 233, 0, 955, 4, 963, 0, 1081, 0, 37, 0, 1982, 222, 0, 0, 20, 0, 100, 1203, 666, 591, 97, 0, 6, 382, 5838, 121, 167, 2133, 78, 4137] payload_rows_per_dst=[13610, 6376, 4490, 14556, 15044, 6202, 5058, 8830] dup_rows_per_dst=[7480, 10, 1032, 9290, 3732, 368, 296, 1930] combine_rows_per_src=[13610, 6376, 4490, 14556, 15044, 6202, 5058, 8830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=352448 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912512 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531472576 weight_movement_bytes=69206016 total_nvlink_bytes=600678592 weight_to_token_ratio=0.130216 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70134 payload_rows_remote=56580 dup_rows=28170 dup_rows_remote=18848 dedup_ratio=0.286560 router_tokens_per_expert=[419, 470, 119, 98, 47, 85, 52, 279, 20, 1227, 1903, 1096, 87, 427, 383, 198, 853, 42, 1155, 14, 396, 3341, 67, 423, 45, 288, 568, 162, 260, 120, 204, 8, 18, 279, 207, 2839, 62, 921, 322, 37, 61, 46, 5417, 205, 109, 504, 862, 5763, 41, 4042, 23, 169, 216, 706, 419, 6228, 557, 327, 370, 21, 309, 336, 2352, 528] payload_rows_per_dst=[2086, 9084, 11358, 3472, 8244, 13554, 14624, 7712] dup_rows_per_dst=[112, 1598, 4282, 1204, 1126, 9322, 9064, 1462] combine_rows_per_src=[2086, 9084, 11358, 3472, 8244, 13554, 14624, 7712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68916 payload_rows_remote=61086 dup_rows=29388 dup_rows_remote=29212 dedup_ratio=0.298950 router_tokens_per_expert=[395, 439, 6, 50, 0, 51, 0, 0, 0, 1416, 1711, 568, 52, 396, 184, 75, 718, 0, 196, 0, 398, 4290, 0, 345, 0, 248, 574, 52, 251, 51, 207, 0, 5, 0, 88, 3330, 0, 1095, 0, 46, 78, 46, 7115, 194, 0, 702, 563, 6392, 19, 5101, 0, 0, 0, 479, 487, 7175, 65, 400, 0, 27, 129, 40, 2815, 88] payload_rows_per_dst=[1004, 7796, 9848, 2270, 8298, 16268, 15602, 7830] dup_rows_per_dst=[0, 1008, 2046, 496, 830, 13912, 10920, 176] combine_rows_per_src=[1004, 7796, 9848, 2270, 8298, 16268, 15602, 7830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70310 payload_rows_remote=56394 dup_rows=27994 dup_rows_remote=17808 dedup_ratio=0.284770 router_tokens_per_expert=[592, 1019, 4147, 2905, 1202, 0, 5204, 5417, 0, 179, 0, 978, 0, 1891, 0, 135, 323, 2, 1650, 46, 796, 71, 458, 195, 61, 110, 1666, 0, 24, 151, 0, 438, 5545, 932, 0, 124, 453, 822, 668, 0, 411, 0, 151, 0, 41, 0, 341, 46, 2141, 0, 229, 790, 18, 974, 142, 0, 2242, 535, 215, 1195, 1396, 0, 80, 1] payload_rows_per_dst=[13916, 7824, 13544, 4232, 7296, 1980, 7478, 14040] dup_rows_per_dst=[10186, 1310, 3946, 668, 2652, 0, 1110, 8122] combine_rows_per_src=[13916, 7824, 13544, 4232, 7296, 1980, 7478, 14040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=277667840 dispatch_meta_bytes=367672 combine_payload_bytes=277667840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278035512 ingress_bytes=312270848 link_bytes=312270848 token_bytes=555703352 weight_movement_bytes=34603008 total_nvlink_bytes=590306360 weight_to_token_ratio=0.062269 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175050061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=68428 dup_rows=24162 dup_rows_remote=23182 dedup_ratio=0.245789 router_tokens_per_expert=[147, 425, 5, 176, 1121, 47, 135, 6672, 389, 37, 1, 4828, 1, 359, 3485, 69, 0, 370, 0, 29, 2386, 92, 0, 470, 0, 70, 180, 409, 487, 0, 2429, 1014, 47, 77, 0, 2113, 30, 0, 1657, 263, 38, 5, 0, 1744, 698, 73, 0, 624, 13, 89, 6655, 3, 505, 3444, 139, 572, 0, 4402, 0, 0, 1, 0, 50, 77] payload_rows_per_dst=[15102, 13294, 5714, 7942, 7502, 4528, 11154, 8906] dup_rows_per_dst=[10550, 5044, 980, 1236, 872, 1836, 3490, 154] combine_rows_per_src=[15102, 13294, 5714, 7942, 7502, 4528, 11154, 8906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231751680 dispatch_meta_bytes=301712 combine_payload_bytes=231751680 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=232053392 ingress_bytes=300957696 link_bytes=300957696 token_bytes=463805072 weight_movement_bytes=69206016 total_nvlink_bytes=533011088 weight_to_token_ratio=0.149214 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74672 payload_rows_remote=63128 dup_rows=23632 dup_rows_remote=23296 dedup_ratio=0.240397 router_tokens_per_expert=[183, 44, 0, 497, 0, 0, 33, 0, 389, 142, 692, 0, 976, 322, 26, 148, 618, 0, 102, 0, 0, 70, 65, 0, 10, 3758, 0, 184, 237, 804, 90, 172, 1683, 630, 0, 0, 0, 1551, 50, 1045, 977, 2718, 5482, 0, 646, 400, 487, 1, 0, 281, 0, 5453, 54, 0, 152, 0, 695, 0, 4316, 4414, 4616, 3771, 0, 168] payload_rows_per_dst=[1488, 4010, 10186, 9752, 8588, 14576, 11544, 14528] dup_rows_per_dst=[26, 1380, 352, 758, 1330, 6846, 336, 12604] combine_rows_per_src=[1488, 4010, 10186, 9752, 8588, 14576, 11544, 14528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=250208256 dispatch_meta_bytes=361192 combine_payload_bytes=250208256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250569448 ingress_bytes=354017280 link_bytes=354017280 token_bytes=500777704 weight_movement_bytes=103809024 total_nvlink_bytes=604586728 weight_to_token_ratio=0.207296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332138524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230989824 dispatch_meta_bytes=296808 combine_payload_bytes=230989824 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=231286632 ingress_bytes=404004864 link_bytes=404004864 token_bytes=462276456 weight_movement_bytes=173015040 total_nvlink_bytes=635291496 weight_to_token_ratio=0.374267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520237778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76506 payload_rows_remote=66094 dup_rows=21798 dup_rows_remote=19824 dedup_ratio=0.221741 router_tokens_per_expert=[2949, 444, 19, 661, 162, 318, 57, 766, 579, 129, 372, 165, 1052, 304, 826, 155, 2866, 300, 125, 137, 49, 215, 268, 506, 50, 3202, 142, 226, 700, 975, 454, 444, 517, 488, 58, 39, 0, 682, 1074, 1115, 584, 3156, 4370, 111, 845, 354, 335, 105, 4, 186, 85, 3975, 308, 0, 384, 248, 273, 56, 2357, 1816, 2420, 3285, 151, 154] payload_rows_per_dst=[8184, 5764, 10954, 10412, 7474, 13086, 9926, 10706] dup_rows_per_dst=[2568, 1400, 2692, 1974, 1090, 6016, 454, 5604] combine_rows_per_src=[8184, 5764, 10954, 10412, 7474, 13086, 9926, 10706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=280281088 dispatch_meta_bytes=366440 combine_payload_bytes=280281088 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=280647528 ingress_bytes=349487104 link_bytes=349487104 token_bytes=560928616 weight_movement_bytes=69206016 total_nvlink_bytes=630134632 weight_to_token_ratio=0.123378 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315091836e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75456 payload_rows_remote=66826 dup_rows=22848 dup_rows_remote=21372 dedup_ratio=0.232422 router_tokens_per_expert=[3474, 776, 43, 385, 593, 375, 123, 3954, 440, 81, 127, 2146, 467, 322, 3125, 116, 1561, 437, 68, 371, 342, 214, 144, 1680, 23, 1400, 258, 283, 883, 389, 1339, 737, 325, 194, 4, 791, 6, 194, 2833, 706, 516, 1552, 1777, 983, 913, 242, 126, 359, 138, 164, 2717, 1453, 504, 94, 437, 446, 51, 1076, 862, 590, 740, 1396, 153, 134] payload_rows_per_dst=[12534, 10186, 8752, 8988, 8630, 8938, 10844, 6584] dup_rows_per_dst=[6912, 3462, 2062, 1636, 1476, 3998, 1062, 2240] combine_rows_per_src=[12534, 10186, 8752, 8988, 8630, 8938, 10844, 6584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=258572288 dispatch_meta_bytes=345696 combine_payload_bytes=258572288 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=258917984 ingress_bytes=362381312 link_bytes=362381312 token_bytes=517490272 weight_movement_bytes=103809024 total_nvlink_bytes=621299296 weight_to_token_ratio=0.200601 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363611703e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=270721024 dispatch_meta_bytes=343672 combine_payload_bytes=270721024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271064696 ingress_bytes=305324032 link_bytes=305324032 token_bytes=541785720 weight_movement_bytes=34603008 total_nvlink_bytes=576388728 weight_to_token_ratio=0.063868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148909752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=70430 dup_rows=23866 dup_rows_remote=22366 dedup_ratio=0.242778 router_tokens_per_expert=[45, 0, 0, 389, 0, 0, 0, 0, 382, 219, 721, 0, 986, 328, 0, 118, 174, 0, 20, 0, 0, 82, 3, 0, 3, 4352, 0, 321, 240, 1011, 0, 17, 1407, 1198, 0, 0, 0, 1692, 0, 970, 1397, 1858, 6136, 0, 555, 621, 644, 0, 0, 259, 0, 4503, 0, 0, 126, 0, 791, 0, 4392, 5747, 4484, 2765, 0, 196] payload_rows_per_dst=[868, 4008, 14532, 13206, 15288, 8604, 9580, 8352] dup_rows_per_dst=[0, 1500, 6304, 4514, 7518, 1546, 196, 2288] combine_rows_per_src=[868, 4008, 14532, 13206, 15288, 8604, 9580, 8352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273719296 dispatch_meta_bytes=352792 combine_payload_bytes=273719296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274072088 ingress_bytes=308322304 link_bytes=308322304 token_bytes=547791384 weight_movement_bytes=34603008 total_nvlink_bytes=582394392 weight_to_token_ratio=0.063168 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160192008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76492 payload_rows_remote=70956 dup_rows=21812 dup_rows_remote=19806 dedup_ratio=0.221883 router_tokens_per_expert=[2558, 760, 41, 222, 853, 126, 104, 5833, 404, 63, 54, 4370, 138, 345, 4766, 96, 561, 517, 1, 183, 409, 149, 18, 1086, 11, 487, 244, 200, 628, 138, 2134, 715, 181, 131, 2, 848, 0, 106, 2905, 391, 108, 608, 633, 1218, 681, 159, 45, 319, 141, 112, 5269, 543, 389, 187, 290, 592, 21, 3466, 263, 290, 293, 523, 124, 100] payload_rows_per_dst=[14270, 13468, 5230, 8196, 7586, 5536, 13540, 8666] dup_rows_per_dst=[6724, 7004, 1198, 918, 1542, 2006, 1506, 914] combine_rows_per_src=[14270, 13468, 5230, 8196, 7586, 5536, 13540, 8666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77580 payload_rows_remote=67996 dup_rows=20724 dup_rows_remote=19886 dedup_ratio=0.210815 router_tokens_per_expert=[403, 529, 4, 186, 1425, 47, 241, 6394, 364, 50, 0, 5553, 0, 350, 4619, 83, 0, 466, 0, 62, 1167, 69, 0, 1824, 0, 63, 434, 198, 470, 0, 1729, 924, 87, 88, 0, 2092, 0, 0, 1050, 263, 22, 10, 0, 1646, 789, 90, 0, 394, 69, 130, 6438, 0, 473, 2086, 352, 647, 0, 4639, 0, 0, 0, 0, 45, 88] payload_rows_per_dst=[14568, 14384, 6014, 6598, 6562, 4304, 15566, 9584] dup_rows_per_dst=[3890, 6776, 1162, 1038, 598, 1598, 4824, 838] combine_rows_per_src=[14568, 14384, 6014, 6598, 6562, 4304, 15566, 9584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72710 payload_rows_remote=57370 dup_rows=25594 dup_rows_remote=12884 dedup_ratio=0.260356 router_tokens_per_expert=[0, 6740, 1778, 22, 10, 62, 596, 4817, 0, 2, 0, 0, 0, 623, 2677, 160, 275, 723, 124, 129, 69, 0, 235, 860, 592, 340, 18, 276, 249, 77, 250, 0, 64, 7224, 118, 0, 1245, 0, 872, 0, 1100, 0, 0, 0, 2223, 252, 0, 0, 39, 0, 94, 1372, 325, 551, 87, 0, 0, 376, 6090, 103, 154, 1968, 66, 3125] payload_rows_per_dst=[15340, 6924, 4306, 14412, 15142, 6694, 4742, 5150] dup_rows_per_dst=[12710, 0, 524, 7622, 3904, 456, 194, 184] combine_rows_per_src=[15340, 6924, 4306, 14412, 15142, 6694, 4742, 5150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288481280 dispatch_meta_bytes=371184 combine_payload_bytes=288481280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=288852464 ingress_bytes=357687296 link_bytes=357687296 token_bytes=577333744 weight_movement_bytes=69206016 total_nvlink_bytes=646539760 weight_to_token_ratio=0.119872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345948499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69184 payload_rows_remote=63592 dup_rows=29120 dup_rows_remote=28942 dedup_ratio=0.296224 router_tokens_per_expert=[487, 152, 1, 85, 0, 1, 3504, 296, 0, 366, 0, 16, 231, 3467, 0, 726, 0, 1, 0, 0, 816, 8, 2002, 58, 483, 429, 18, 55, 22, 0, 0, 38, 2, 0, 0, 0, 0, 6405, 70, 0, 0, 7394, 7692, 9, 85, 1835, 7405, 296, 173, 1, 0, 0, 0, 50, 0, 1714, 2, 83, 1, 1, 224, 0, 2387, 61] payload_rows_per_dst=[8076, 14642, 5592, 15818, 306, 15598, 3804, 5348] dup_rows_per_dst=[976, 7784, 178, 15870, 0, 4236, 72, 4] combine_rows_per_src=[8076, 14642, 5592, 15818, 306, 15598, 3804, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290635776 dispatch_meta_bytes=363048 combine_payload_bytes=290635776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290998824 ingress_bytes=359841792 link_bytes=359841792 token_bytes=581634600 weight_movement_bytes=69206016 total_nvlink_bytes=650840616 weight_to_token_ratio=0.118985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354055694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80282 payload_rows_remote=67920 dup_rows=18022 dup_rows_remote=14370 dedup_ratio=0.183329 router_tokens_per_expert=[0, 51, 328, 165, 966, 6203, 0, 489, 33, 47, 2, 0, 293, 0, 0, 0, 541, 0, 0, 0, 0, 2636, 0, 1355, 5, 337, 14, 434, 0, 0, 0, 52, 5857, 2, 32, 4324, 0, 0, 148, 2469, 419, 0, 0, 925, 112, 0, 0, 397, 290, 895, 2681, 282, 2330, 51, 4664, 0, 65, 3776, 2353, 595, 1992, 366, 0, 176] payload_rows_per_dst=[14156, 12256, 11916, 1684, 11668, 3560, 12362, 12680] dup_rows_per_dst=[2248, 208, 3520, 0, 2282, 146, 3652, 5966] combine_rows_per_src=[14156, 12256, 11916, 1684, 11668, 3560, 12362, 12680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=278511616 dispatch_meta_bytes=351528 combine_payload_bytes=278511616 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278863144 ingress_bytes=416923648 link_bytes=416923648 token_bytes=557374760 weight_movement_bytes=138412032 total_nvlink_bytes=695786792 weight_to_token_ratio=0.248328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.568850122e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234987520 dispatch_meta_bytes=281016 combine_payload_bytes=234987520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235268536 ingress_bytes=304193536 link_bytes=304193536 token_bytes=470256056 weight_movement_bytes=69206016 total_nvlink_bytes=539462072 weight_to_token_ratio=0.147167 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78476 payload_rows_remote=74276 dup_rows=19828 dup_rows_remote=19042 dedup_ratio=0.201701 router_tokens_per_expert=[239, 187, 298, 223, 514, 5976, 108, 699, 236, 98, 32, 0, 511, 62, 0, 183, 270, 0, 107, 512, 183, 1664, 59, 2152, 248, 656, 525, 465, 42, 242, 0, 176, 4897, 50, 11, 1739, 54, 269, 123, 1557, 493, 80, 62, 848, 124, 417, 59, 546, 322, 3073, 3218, 370, 697, 455, 2229, 815, 1144, 4214, 879, 412, 1226, 1207, 548, 347] payload_rows_per_dst=[13862, 11430, 11090, 4200, 10944, 4810, 12712, 9428] dup_rows_per_dst=[2626, 1146, 3262, 786, 3510, 170, 5188, 3140] combine_rows_per_src=[13862, 11430, 11090, 4200, 10944, 4810, 12712, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=260472832 dispatch_meta_bytes=370136 combine_payload_bytes=260472832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260842968 ingress_bytes=295075840 link_bytes=295075840 token_bytes=521315800 weight_movement_bytes=34603008 total_nvlink_bytes=555918808 weight_to_token_ratio=0.066376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110346630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77848 payload_rows_remote=70460 dup_rows=20456 dup_rows_remote=19970 dedup_ratio=0.208089 router_tokens_per_expert=[776, 445, 164, 148, 170, 2359, 461, 819, 76, 364, 18, 8, 475, 596, 0, 843, 164, 15, 225, 372, 1126, 648, 911, 898, 495, 778, 847, 215, 55, 95, 0, 124, 2187, 33, 10, 754, 37, 3731, 115, 601, 225, 2466, 1730, 367, 114, 2473, 3572, 404, 317, 2017, 1222, 145, 229, 583, 842, 2902, 651, 1789, 305, 159, 688, 377, 3139, 278] payload_rows_per_dst=[9146, 10042, 7912, 10548, 7388, 9514, 11476, 11822] dup_rows_per_dst=[1538, 1780, 2490, 6746, 486, 1112, 3354, 2950] combine_rows_per_src=[9146, 10042, 7912, 10548, 7388, 9514, 11476, 11822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=278200320 dispatch_meta_bytes=329160 combine_payload_bytes=278200320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278529480 ingress_bytes=382009344 link_bytes=382009344 token_bytes=556729800 weight_movement_bytes=103809024 total_nvlink_bytes=660538824 weight_to_token_ratio=0.186462 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437470407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=304234496 dispatch_meta_bytes=373272 combine_payload_bytes=304234496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=304607768 ingress_bytes=373440512 link_bytes=373440512 token_bytes=608842264 weight_movement_bytes=69206016 total_nvlink_bytes=678048280 weight_to_token_ratio=0.113668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405226582e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80058 payload_rows_remote=67566 dup_rows=18246 dup_rows_remote=18190 dedup_ratio=0.185608 router_tokens_per_expert=[0, 63, 291, 269, 591, 5601, 0, 450, 16, 0, 14, 0, 247, 0, 0, 0, 754, 0, 0, 0, 0, 2457, 0, 1312, 0, 292, 21, 491, 0, 0, 0, 63, 5997, 23, 98, 4263, 0, 0, 179, 1803, 343, 0, 0, 1133, 82, 0, 0, 346, 251, 430, 2962, 105, 3757, 62, 5448, 0, 58, 3466, 2947, 633, 1297, 329, 0, 208] payload_rows_per_dst=[13024, 12492, 14592, 1722, 13720, 3644, 12190, 8674] dup_rows_per_dst=[1506, 56, 5350, 12, 5944, 164, 2944, 2270] combine_rows_per_src=[13024, 12492, 14592, 1722, 13720, 3644, 12190, 8674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77732 payload_rows_remote=64562 dup_rows=20572 dup_rows_remote=14538 dedup_ratio=0.209269 router_tokens_per_expert=[685, 319, 67, 92, 61, 824, 1093, 481, 3, 293, 1, 0, 310, 779, 0, 792, 24, 0, 83, 114, 986, 185, 1445, 380, 483, 622, 131, 23, 8, 27, 0, 30, 740, 4, 3, 230, 2, 5456, 89, 243, 102, 5388, 4719, 238, 92, 2645, 6011, 386, 198, 647, 603, 42, 35, 155, 285, 4296, 171, 809, 76, 60, 492, 111, 4353, 130] payload_rows_per_dst=[6244, 5030, 6006, 13516, 11776, 13170, 10514, 11476] dup_rows_per_dst=[1000, 806, 998, 9090, 278, 6034, 1438, 928] combine_rows_per_src=[6244, 5030, 6006, 13516, 11776, 13170, 10514, 11476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=288604160 dispatch_meta_bytes=361720 combine_payload_bytes=288604160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288965880 ingress_bytes=392413184 link_bytes=392413184 token_bytes=577570040 weight_movement_bytes=103809024 total_nvlink_bytes=681379064 weight_to_token_ratio=0.179734 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476619220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76288 payload_rows_remote=66436 dup_rows=22016 dup_rows_remote=20734 dedup_ratio=0.223958 router_tokens_per_expert=[449, 286, 0, 63, 0, 0, 2209, 345, 0, 565, 0, 0, 273, 2496, 0, 854, 0, 0, 0, 0, 831, 0, 1734, 42, 577, 470, 21, 0, 18, 0, 0, 36, 0, 0, 0, 0, 0, 6461, 81, 0, 0, 7327, 7540, 0, 63, 2171, 7205, 344, 186, 0, 0, 0, 0, 51, 0, 2005, 0, 105, 0, 0, 338, 0, 3937, 69] payload_rows_per_dst=[5736, 7270, 5042, 15212, 12922, 15658, 4596, 9852] dup_rows_per_dst=[968, 1106, 172, 1442, 162, 16760, 124, 1282] combine_rows_per_src=[5736, 7270, 5042, 15212, 12922, 15658, 4596, 9852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70948 payload_rows_remote=70328 dup_rows=27356 dup_rows_remote=27356 dedup_ratio=0.278280 router_tokens_per_expert=[2, 0, 0, 308, 0, 0, 0, 0, 385, 194, 1136, 0, 1015, 341, 0, 100, 98, 0, 4, 0, 0, 69, 0, 0, 7, 4213, 0, 286, 241, 1140, 0, 1, 1205, 1605, 0, 0, 0, 1687, 0, 968, 1610, 1537, 5659, 0, 556, 754, 765, 0, 0, 244, 0, 4135, 0, 0, 124, 0, 828, 0, 5126, 5789, 4680, 2163, 0, 177] payload_rows_per_dst=[620, 4882, 14800, 14694, 15018, 11446, 4204, 5284] dup_rows_per_dst=[0, 1742, 7372, 6442, 7230, 3440, 78, 1052] combine_rows_per_src=[620, 4882, 14800, 14694, 15018, 11446, 4204, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276750336 dispatch_meta_bytes=343024 combine_payload_bytes=276750336 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277093360 ingress_bytes=345956352 link_bytes=345956352 token_bytes=553843696 weight_movement_bytes=69206016 total_nvlink_bytes=623049712 weight_to_token_ratio=0.124956 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301805901e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75422 payload_rows_remote=59600 dup_rows=22882 dup_rows_remote=9304 dedup_ratio=0.232768 router_tokens_per_expert=[0, 0, 191, 148, 0, 0, 356, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 4939, 76, 0, 0, 250, 3047, 2, 7398, 0, 87, 2852, 0, 0, 2, 2, 0, 1161, 0, 5249, 2105, 0, 463, 121, 0, 0, 3147, 158, 1, 261, 6953, 0, 0, 0, 1, 176, 1, 383, 583, 983, 0, 0, 0, 1253, 6719, 80, 0] payload_rows_per_dst=[15526, 4, 15822, 5882, 13576, 7738, 2288, 14586] dup_rows_per_dst=[664, 4, 13578, 0, 4380, 768, 4, 3484] combine_rows_per_src=[15526, 4, 15822, 5882, 13576, 7738, 2288, 14586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264445952 dispatch_meta_bytes=316400 combine_payload_bytes=264445952 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264762352 ingress_bytes=402857984 link_bytes=402857984 token_bytes=529208304 weight_movement_bytes=138412032 total_nvlink_bytes=667620336 weight_to_token_ratio=0.261545 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515922161e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=272121856 dispatch_meta_bytes=348680 combine_payload_bytes=272121856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272470536 ingress_bytes=341327872 link_bytes=341327872 token_bytes=544592392 weight_movement_bytes=69206016 total_nvlink_bytes=613798408 weight_to_token_ratio=0.127079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284389303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80908 payload_rows_remote=65132 dup_rows=17396 dup_rows_remote=15574 dedup_ratio=0.176961 router_tokens_per_expert=[0, 0, 51, 70, 140, 0, 0, 0, 0, 1180, 0, 404, 78, 1400, 422, 972, 0, 6630, 0, 587, 0, 130, 84, 0, 6708, 0, 86, 321, 0, 299, 78, 4506, 7236, 5319, 0, 345, 0, 0, 0, 0, 361, 123, 3546, 0, 0, 1258, 244, 0, 409, 0, 39, 1115, 0, 0, 0, 0, 0, 4927, 0, 0, 0, 0, 0, 84] payload_rows_per_dst=[13662, 13254, 4782, 3932, 3610, 11540, 15776, 14352] dup_rows_per_dst=[276, 5410, 328, 238, 220, 2794, 1822, 6308] combine_rows_per_src=[13662, 13254, 4782, 3932, 3610, 11540, 15776, 14352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288063488 dispatch_meta_bytes=390736 combine_payload_bytes=288063488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288454224 ingress_bytes=322666496 link_bytes=322666496 token_bytes=576517712 weight_movement_bytes=34603008 total_nvlink_bytes=611120720 weight_to_token_ratio=0.060021 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214168048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=244121600 dispatch_meta_bytes=275616 combine_payload_bytes=244121600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=244397216 ingress_bytes=313327616 link_bytes=313327616 token_bytes=488518816 weight_movement_bytes=69206016 total_nvlink_bytes=557724832 weight_to_token_ratio=0.141665 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84004 payload_rows_remote=74358 dup_rows=14300 dup_rows_remote=12334 dedup_ratio=0.145467 router_tokens_per_expert=[0, 29, 423, 105, 188, 0, 581, 0, 271, 307, 47, 1030, 63, 1849, 778, 1856, 0, 4299, 115, 896, 53, 600, 247, 491, 4692, 47, 248, 1217, 45, 651, 92, 3553, 6097, 5128, 69, 374, 35, 0, 61, 72, 693, 132, 3475, 1, 6, 920, 242, 20, 246, 5, 217, 1335, 327, 220, 6, 61, 1, 3445, 0, 219, 32, 0, 51, 889] payload_rows_per_dst=[11234, 14440, 3812, 9646, 11368, 10446, 14280, 8778] dup_rows_per_dst=[896, 6560, 992, 1966, 110, 532, 2748, 496] combine_rows_per_src=[11234, 14440, 3812, 9646, 11368, 10446, 14280, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75922 payload_rows_remote=65494 dup_rows=22382 dup_rows_remote=21034 dedup_ratio=0.227681 router_tokens_per_expert=[0, 145, 1679, 285, 88, 0, 1650, 5, 229, 38, 63, 764, 45, 699, 618, 947, 0, 1663, 3337, 395, 33, 426, 394, 4032, 1776, 3444, 173, 784, 1166, 390, 67, 1702, 2469, 1824, 472, 121, 2820, 621, 15, 285, 676, 71, 1116, 933, 141, 493, 242, 1605, 88, 3, 213, 1650, 620, 481, 140, 878, 257, 1321, 0, 170, 590, 616, 155, 1029] payload_rows_per_dst=[12914, 7338, 10810, 7520, 10428, 8504, 11080, 7328] dup_rows_per_dst=[5230, 2794, 6424, 1044, 1348, 2050, 2544, 948] combine_rows_per_src=[12914, 7338, 10810, 7520, 10428, 8504, 11080, 7328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=322824 combine_payload_bytes=266780672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267103496 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533884168 weight_movement_bytes=69206016 total_nvlink_bytes=603090184 weight_to_token_ratio=0.129627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86020 payload_rows_remote=70446 dup_rows=12284 dup_rows_remote=5320 dedup_ratio=0.124959 router_tokens_per_expert=[2, 0, 63, 66, 195, 0, 0, 0, 0, 1388, 0, 376, 98, 820, 374, 1167, 0, 7046, 0, 497, 0, 0, 60, 0, 6954, 0, 100, 232, 0, 572, 97, 5108, 7286, 5500, 0, 204, 0, 0, 0, 0, 216, 77, 2543, 0, 0, 1544, 231, 0, 393, 0, 0, 1694, 0, 0, 0, 0, 0, 4189, 0, 0, 0, 0, 0, 60] payload_rows_per_dst=[14170, 15574, 1114, 10942, 11406, 8530, 15786, 8498] dup_rows_per_dst=[390, 6964, 0, 1276, 2, 692, 2960, 0] combine_rows_per_src=[14170, 15574, 1114, 10942, 11406, 8530, 15786, 8498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72950 payload_rows_remote=60710 dup_rows=25354 dup_rows_remote=22160 dedup_ratio=0.257914 router_tokens_per_expert=[1, 4, 1136, 277, 18, 0, 961, 0, 69, 43, 18, 223, 9, 243, 130, 260, 0, 631, 4887, 159, 16, 75, 256, 5673, 771, 6164, 35, 287, 3212, 200, 13, 522, 890, 675, 571, 49, 4541, 817, 5, 479, 260, 20, 475, 1683, 153, 156, 257, 4713, 23, 0, 126, 617, 283, 278, 242, 877, 503, 458, 0, 75, 890, 2194, 110, 439] payload_rows_per_dst=[5874, 2276, 13608, 13688, 11324, 12240, 5842, 8098] dup_rows_per_dst=[1222, 976, 8524, 6418, 1600, 3194, 1748, 1672] combine_rows_per_src=[5874, 2276, 13608, 13688, 11324, 12240, 5842, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304570368 dispatch_meta_bytes=346768 combine_payload_bytes=304570368 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304917136 ingress_bytes=442982400 link_bytes=442982400 token_bytes=609487504 weight_movement_bytes=138412032 total_nvlink_bytes=747899536 weight_to_token_ratio=0.227096 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666907108e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=62124 dup_rows=22998 dup_rows_remote=20114 dedup_ratio=0.233948 router_tokens_per_expert=[0, 0, 176, 193, 0, 0, 415, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5590, 85, 0, 0, 286, 2733, 0, 7340, 0, 38, 3492, 0, 0, 0, 0, 0, 1471, 0, 5256, 1408, 0, 520, 169, 0, 0, 2852, 151, 0, 251, 7193, 0, 0, 0, 0, 186, 121, 391, 801, 830, 0, 0, 0, 1108, 5967, 128, 0] payload_rows_per_dst=[1456, 0, 13624, 15240, 13452, 15602, 2750, 13182] dup_rows_per_dst=[114, 0, 3764, 6500, 3858, 5630, 248, 2884] combine_rows_per_src=[1456, 0, 13624, 15240, 13452, 15602, 2750, 13182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=268263424 dispatch_meta_bytes=346112 combine_payload_bytes=268263424 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268609536 ingress_bytes=406675456 link_bytes=406675456 token_bytes=536872960 weight_movement_bytes=138412032 total_nvlink_bytes=675284992 weight_to_token_ratio=0.257812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530287001e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=66912 dup_rows=17856 dup_rows_remote=14194 dedup_ratio=0.181641 router_tokens_per_expert=[0, 61, 367, 320, 479, 5526, 0, 462, 0, 0, 10, 0, 223, 0, 0, 0, 586, 0, 0, 0, 0, 2429, 0, 1491, 0, 217, 20, 531, 0, 0, 0, 61, 5895, 84, 90, 4333, 0, 0, 158, 1241, 284, 0, 0, 1173, 69, 0, 0, 338, 318, 299, 2848, 89, 4708, 61, 5586, 0, 37, 2553, 3740, 645, 1107, 501, 0, 212] payload_rows_per_dst=[13536, 12238, 14634, 1656, 12734, 3590, 11912, 10148] dup_rows_per_dst=[3662, 18, 5550, 2, 4184, 138, 1966, 2336] combine_rows_per_src=[13536, 12238, 14634, 1656, 12734, 3590, 11912, 10148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288546816 dispatch_meta_bytes=303064 combine_payload_bytes=288546816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288849880 ingress_bytes=323149824 link_bytes=323149824 token_bytes=577396696 weight_movement_bytes=34603008 total_nvlink_bytes=611999704 weight_to_token_ratio=0.059929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215986772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77704 payload_rows_remote=74510 dup_rows=20600 dup_rows_remote=20600 dedup_ratio=0.209554 router_tokens_per_expert=[590, 319, 60, 39, 5, 0, 0, 0, 0, 1, 1, 467, 6527, 8189, 0, 31, 0, 0, 1495, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 134, 12, 2380, 0, 0, 3161, 3378, 0, 1, 262, 79, 0, 4260, 2, 0, 1021, 0, 0, 8148, 255, 5887, 65, 4, 1, 962, 39, 40, 2, 1, 1090, 0, 0, 142, 1, 0] payload_rows_per_dst=[16300, 16382, 3194, 5052, 11764, 9276, 13268, 2468] dup_rows_per_dst=[2022, 14048, 0, 0, 1998, 1290, 1238, 4] combine_rows_per_src=[16300, 16382, 3194, 5052, 11764, 9276, 13268, 2468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=248668160 dispatch_meta_bytes=331480 combine_payload_bytes=248668160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248999640 ingress_bytes=317874176 link_bytes=317874176 token_bytes=497667800 weight_movement_bytes=69206016 total_nvlink_bytes=566873816 weight_to_token_ratio=0.139061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196134933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=254459904 dispatch_meta_bytes=328952 combine_payload_bytes=254459904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254788856 ingress_bytes=289062912 link_bytes=289062912 token_bytes=509248760 weight_movement_bytes=34603008 total_nvlink_bytes=543851768 weight_to_token_ratio=0.067949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087720466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76432 payload_rows_remote=69772 dup_rows=21872 dup_rows_remote=21018 dedup_ratio=0.222493 router_tokens_per_expert=[0, 0, 98, 3447, 0, 0, 955, 0, 0, 5803, 940, 0, 0, 0, 665, 48, 237, 320, 0, 2618, 7, 54, 0, 0, 5259, 83, 0, 107, 0, 0, 0, 0, 103, 333, 0, 0, 3121, 823, 186, 78, 2, 0, 4589, 73, 0, 1, 307, 0, 617, 0, 78, 0, 2903, 1, 4, 154, 6668, 0, 0, 39, 6700, 649, 1082, 0] payload_rows_per_dst=[8644, 2874, 13348, 10878, 8312, 9662, 6660, 16054] dup_rows_per_dst=[356, 432, 4730, 20, 976, 282, 854, 14222] combine_rows_per_src=[8644, 2874, 13348, 10878, 8312, 9662, 6660, 16054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274071552 dispatch_meta_bytes=324424 combine_payload_bytes=274071552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274395976 ingress_bytes=308674560 link_bytes=308674560 token_bytes=548467528 weight_movement_bytes=34603008 total_nvlink_bytes=583070536 weight_to_token_ratio=0.063090 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161517519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=305192960 dispatch_meta_bytes=380440 combine_payload_bytes=305192960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305573400 ingress_bytes=339795968 link_bytes=339795968 token_bytes=610766360 weight_movement_bytes=34603008 total_nvlink_bytes=645369368 weight_to_token_ratio=0.056655 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278624872e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76336 payload_rows_remote=62232 dup_rows=21968 dup_rows_remote=16872 dedup_ratio=0.223470 router_tokens_per_expert=[32, 101, 243, 5489, 149, 0, 1533, 0, 0, 6909, 787, 0, 0, 0, 827, 1198, 1507, 204, 3, 3013, 435, 82, 25, 0, 2456, 330, 28, 574, 1, 153, 0, 8, 648, 214, 13, 0, 1375, 1642, 243, 107, 98, 7, 1551, 47, 83, 164, 976, 18, 1092, 0, 203, 25, 705, 47, 0, 273, 3435, 0, 0, 1576, 6392, 920, 1089, 122] payload_rows_per_dst=[12368, 5176, 14756, 14104, 7602, 5672, 4070, 12588] dup_rows_per_dst=[2762, 448, 9600, 5096, 882, 180, 620, 2380] combine_rows_per_src=[12368, 5176, 14756, 14104, 7602, 5672, 4070, 12588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72844 payload_rows_remote=64540 dup_rows=25460 dup_rows_remote=24664 dedup_ratio=0.258993 router_tokens_per_expert=[671, 558, 246, 4123, 1933, 0, 759, 15, 0, 4130, 513, 840, 205, 1700, 545, 3872, 492, 79, 424, 1129, 270, 61, 305, 0, 875, 340, 103, 265, 97, 419, 0, 63, 485, 74, 752, 714, 522, 713, 244, 1046, 261, 1423, 561, 24, 659, 138, 688, 2780, 2714, 381, 300, 281, 386, 561, 2, 660, 1133, 15, 505, 1446, 3232, 698, 605, 117] payload_rows_per_dst=[13460, 11590, 10046, 4094, 8304, 6380, 8324, 10646] dup_rows_per_dst=[8710, 3760, 3734, 230, 796, 1128, 2246, 4856] combine_rows_per_src=[13460, 11590, 10046, 4094, 8304, 6380, 8324, 10646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=285786112 dispatch_meta_bytes=363160 combine_payload_bytes=285786112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286149272 ingress_bytes=320389120 link_bytes=320389120 token_bytes=571935384 weight_movement_bytes=34603008 total_nvlink_bytes=606538392 weight_to_token_ratio=0.060502 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205598465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81200 payload_rows_remote=77434 dup_rows=17104 dup_rows_remote=16622 dedup_ratio=0.173991 router_tokens_per_expert=[0, 0, 110, 2108, 0, 0, 1162, 0, 0, 4474, 1496, 0, 0, 0, 626, 2, 83, 314, 0, 1959, 0, 13, 0, 0, 5846, 73, 0, 99, 0, 0, 0, 0, 104, 365, 0, 0, 3280, 398, 92, 97, 0, 0, 5219, 2, 0, 0, 244, 0, 661, 0, 97, 0, 4815, 0, 12, 150, 7034, 0, 0, 0, 6655, 694, 868, 0] payload_rows_per_dst=[6542, 3766, 11200, 15332, 7950, 10714, 10478, 15218] dup_rows_per_dst=[218, 482, 2486, 10014, 722, 216, 992, 1974] combine_rows_per_src=[6542, 3766, 11200, 15332, 7950, 10714, 10478, 15218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73044 payload_rows_remote=62340 dup_rows=25260 dup_rows_remote=21134 dedup_ratio=0.256958 router_tokens_per_expert=[787, 469, 137, 1554, 1187, 0, 158, 0, 0, 1667, 212, 957, 2072, 5786, 230, 2985, 159, 27, 1569, 419, 78, 20, 452, 0, 280, 129, 25, 45, 4, 243, 2, 558, 101, 24, 1848, 2608, 152, 348, 255, 515, 78, 4095, 225, 4, 1093, 37, 198, 6190, 1295, 2790, 158, 180, 81, 605, 0, 367, 498, 13, 871, 443, 1181, 456, 199, 33] payload_rows_per_dst=[12524, 13846, 7312, 2530, 11124, 10704, 9386, 5618] dup_rows_per_dst=[5070, 8406, 1470, 42, 2810, 4126, 1566, 1770] combine_rows_per_src=[12524, 13846, 7312, 2530, 11124, 10704, 9386, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=254902272 dispatch_meta_bytes=316416 combine_payload_bytes=254902272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255218688 ingress_bytes=289505280 link_bytes=289505280 token_bytes=510120960 weight_movement_bytes=34603008 total_nvlink_bytes=544723968 weight_to_token_ratio=0.067833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089385062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75102 payload_rows_remote=68618 dup_rows=23202 dup_rows_remote=22684 dedup_ratio=0.236023 router_tokens_per_expert=[807, 410, 106, 31, 2, 0, 0, 0, 0, 0, 0, 481, 6602, 8185, 0, 82, 0, 0, 2330, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 181, 26, 2545, 0, 0, 3538, 2925, 0, 0, 251, 89, 0, 3116, 0, 0, 1245, 0, 0, 8135, 274, 5074, 80, 0, 0, 760, 9, 52, 0, 1, 1537, 0, 0, 196, 0, 0] payload_rows_per_dst=[2648, 12070, 4824, 5504, 15560, 16270, 11742, 6484] dup_rows_per_dst=[64, 136, 0, 0, 10882, 10846, 756, 518] combine_rows_per_src=[2648, 12070, 4824, 5504, 15560, 16270, 11742, 6484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264355840 dispatch_meta_bytes=356816 combine_payload_bytes=264355840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264712656 ingress_bytes=298958848 link_bytes=298958848 token_bytes=529068496 weight_movement_bytes=34603008 total_nvlink_bytes=563671504 weight_to_token_ratio=0.065404 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124958077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85702 payload_rows_remote=71304 dup_rows=12602 dup_rows_remote=12324 dedup_ratio=0.128194 router_tokens_per_expert=[1, 0, 61, 75, 139, 0, 0, 0, 0, 1861, 0, 283, 89, 440, 376, 1326, 0, 6980, 0, 466, 0, 0, 95, 0, 7062, 0, 153, 252, 0, 437, 89, 6075, 7047, 5687, 0, 124, 0, 0, 0, 0, 91, 95, 1852, 0, 0, 1732, 252, 0, 393, 0, 0, 1774, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 95] payload_rows_per_dst=[14398, 15442, 1122, 12684, 11622, 7422, 15322, 7690] dup_rows_per_dst=[278, 7268, 0, 1328, 0, 622, 3106, 0] combine_rows_per_src=[14398, 15442, 1122, 12684, 11622, 7422, 15322, 7690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=317169664 dispatch_meta_bytes=376224 combine_payload_bytes=317169664 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=317545888 ingress_bytes=455581696 link_bytes=455581696 token_bytes=634715552 weight_movement_bytes=138412032 total_nvlink_bytes=773127584 weight_to_token_ratio=0.218069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714317245e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=255344640 dispatch_meta_bytes=333896 combine_payload_bytes=255344640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255678536 ingress_bytes=324550656 link_bytes=324550656 token_bytes=511023176 weight_movement_bytes=69206016 total_nvlink_bytes=580229192 weight_to_token_ratio=0.135426 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221257990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81270 payload_rows_remote=66630 dup_rows=17034 dup_rows_remote=7738 dedup_ratio=0.173279 router_tokens_per_expert=[0, 20, 8032, 0, 0, 0, 1973, 0, 2, 1, 1, 0, 206, 578, 260, 0, 0, 0, 0, 5896, 285, 78, 5568, 140, 1, 0, 0, 5037, 0, 15, 0, 0, 3658, 0, 2, 0, 0, 747, 1, 0, 0, 2, 0, 2, 40, 7102, 0, 148, 972, 77, 0, 0, 0, 321, 0, 0, 0, 7304, 317, 1, 364, 1, 0, 0] payload_rows_per_dst=[16222, 1610, 14640, 10426, 8274, 12356, 2646, 15096] dup_rows_per_dst=[3828, 490, 9296, 1650, 536, 258, 98, 878] combine_rows_per_src=[16222, 1610, 14640, 10426, 8274, 12356, 2646, 15096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281059328 dispatch_meta_bytes=365208 combine_payload_bytes=281059328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281424536 ingress_bytes=384868352 link_bytes=384868352 token_bytes=562483864 weight_movement_bytes=103809024 total_nvlink_bytes=666292888 weight_to_token_ratio=0.184555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448228624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75906 payload_rows_remote=60234 dup_rows=22398 dup_rows_remote=12332 dedup_ratio=0.227844 router_tokens_per_expert=[2848, 0, 0, 181, 793, 1272, 0, 0, 1389, 69, 1444, 0, 0, 0, 0, 0, 0, 78, 42, 0, 0, 78, 0, 0, 16, 0, 0, 0, 60, 78, 0, 0, 0, 0, 7142, 280, 0, 0, 3925, 1026, 0, 7131, 893, 7241, 37, 0, 5345, 78, 0, 0, 0, 78, 217, 0, 307, 5136, 363, 0, 0, 372, 645, 344, 244, 0] payload_rows_per_dst=[9362, 5218, 396, 14602, 15754, 2016, 15672, 12886] dup_rows_per_dst=[826, 586, 0, 188, 8992, 0, 10066, 1740] combine_rows_per_src=[9362, 5218, 396, 14602, 15754, 2016, 15672, 12886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292061184 dispatch_meta_bytes=334512 combine_payload_bytes=292061184 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=292395696 ingress_bytes=361267200 link_bytes=361267200 token_bytes=584456880 weight_movement_bytes=69206016 total_nvlink_bytes=653662896 weight_to_token_ratio=0.118411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359419389e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272916480 dispatch_meta_bytes=297472 combine_payload_bytes=272916480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273213952 ingress_bytes=307519488 link_bytes=307519488 token_bytes=546130432 weight_movement_bytes=34603008 total_nvlink_bytes=580733440 weight_to_token_ratio=0.063360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157171076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75110 payload_rows_remote=66702 dup_rows=23194 dup_rows_remote=23046 dedup_ratio=0.235942 router_tokens_per_expert=[2133, 55, 0, 168, 1612, 384, 215, 0, 2848, 80, 4895, 11, 0, 2, 0, 0, 0, 24, 27, 0, 0, 67, 3, 0, 44, 0, 0, 4, 62, 63, 0, 0, 0, 8, 6987, 157, 0, 267, 4518, 2178, 0, 4144, 421, 4105, 54, 9, 3400, 714, 0, 0, 0, 63, 615, 0, 464, 3828, 292, 0, 35, 356, 604, 2971, 252, 13] payload_rows_per_dst=[8484, 15920, 9278, 8408, 5178, 2378, 13078, 12386] dup_rows_per_dst=[650, 13726, 0, 148, 42, 18, 5150, 3460] combine_rows_per_src=[8484, 15920, 9278, 8408, 5178, 2378, 13078, 12386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=64414 dup_rows=20392 dup_rows_remote=12894 dedup_ratio=0.207438 router_tokens_per_expert=[807, 85, 1242, 95, 1090, 129, 2384, 230, 1760, 485, 3250, 362, 298, 543, 149, 0, 0, 13, 9, 8, 63, 343, 2636, 16, 17, 0, 39, 3186, 34, 82, 0, 1, 991, 146, 3215, 85, 0, 3624, 3447, 1353, 1, 1575, 148, 1541, 52, 534, 1358, 3527, 87, 53, 0, 48, 454, 53, 538, 1784, 393, 906, 198, 355, 623, 2492, 215, 0] payload_rows_per_dst=[9594, 12032, 8080, 9622, 13498, 7754, 7292, 10040] dup_rows_per_dst=[2530, 4166, 318, 178, 7498, 770, 1892, 3040] combine_rows_per_src=[9594, 12032, 8080, 9622, 13498, 7754, 7292, 10040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=246718464 dispatch_meta_bytes=290264 combine_payload_bytes=246718464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247008728 ingress_bytes=281321472 link_bytes=281321472 token_bytes=493727192 weight_movement_bytes=34603008 total_nvlink_bytes=528330200 weight_to_token_ratio=0.070085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.058590051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80822 payload_rows_remote=65788 dup_rows=17482 dup_rows_remote=13482 dedup_ratio=0.177836 router_tokens_per_expert=[2882, 0, 0, 189, 765, 1970, 0, 0, 1347, 81, 1212, 0, 0, 0, 0, 0, 0, 98, 76, 0, 0, 97, 0, 0, 19, 0, 0, 0, 69, 98, 0, 10, 0, 0, 6877, 26, 0, 0, 4065, 911, 0, 7203, 1047, 7489, 63, 0, 5255, 97, 0, 0, 0, 98, 104, 0, 175, 5082, 400, 0, 0, 334, 593, 175, 231, 14] payload_rows_per_dst=[10290, 15034, 8672, 15124, 1874, 6458, 15266, 8104] dup_rows_per_dst=[1322, 4000, 0, 246, 0, 468, 9964, 1482] combine_rows_per_src=[10290, 15034, 8672, 15124, 1874, 6458, 15266, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79680 payload_rows_remote=68808 dup_rows=18624 dup_rows_remote=17342 dedup_ratio=0.189453 router_tokens_per_expert=[321, 66, 5534, 16, 306, 57, 3170, 50, 634, 192, 1151, 198, 80, 614, 214, 0, 0, 6, 2, 776, 157, 287, 5331, 159, 4, 0, 13, 5271, 12, 26, 0, 0, 3458, 69, 1050, 3, 0, 4255, 1021, 514, 0, 589, 59, 544, 61, 4033, 537, 1924, 257, 56, 0, 14, 119, 80, 245, 771, 129, 2818, 331, 85, 568, 839, 73, 3] payload_rows_per_dst=[13678, 4846, 11744, 11696, 14674, 10872, 3346, 8824] dup_rows_per_dst=[5362, 1320, 1692, 44, 6066, 1282, 916, 1942] combine_rows_per_src=[13678, 4846, 11744, 11696, 14674, 10872, 3346, 8824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=273211392 dispatch_meta_bytes=358992 combine_payload_bytes=273211392 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=273570384 ingress_bytes=342417408 link_bytes=342417408 token_bytes=546781776 weight_movement_bytes=69206016 total_nvlink_bytes=615987792 weight_to_token_ratio=0.126570 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288489139e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263839744 dispatch_meta_bytes=309232 combine_payload_bytes=263839744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264148976 ingress_bytes=367648768 link_bytes=367648768 token_bytes=527988720 weight_movement_bytes=103809024 total_nvlink_bytes=631797744 weight_to_token_ratio=0.196612 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383432716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73604 payload_rows_remote=59986 dup_rows=24700 dup_rows_remote=23716 dedup_ratio=0.251261 router_tokens_per_expert=[0, 13, 8016, 0, 0, 0, 2371, 0, 0, 0, 0, 0, 169, 245, 250, 0, 0, 0, 0, 5782, 304, 88, 6129, 298, 0, 0, 0, 5252, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 50, 6939, 0, 176, 679, 83, 0, 0, 0, 302, 0, 0, 0, 6558, 338, 0, 405, 0, 0, 0] payload_rows_per_dst=[4862, 1328, 14510, 10504, 11146, 15610, 2026, 13618] dup_rows_per_dst=[14, 0, 10692, 0, 2992, 9916, 102, 984] combine_rows_per_src=[4862, 1328, 14510, 10504, 11146, 15610, 2026, 13618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=67688 dup_rows=24322 dup_rows_remote=24000 dedup_ratio=0.247416 router_tokens_per_expert=[0, 0, 124, 1787, 0, 0, 1397, 0, 0, 3659, 1787, 0, 0, 0, 635, 0, 71, 326, 0, 1736, 0, 32, 0, 0, 5837, 44, 0, 97, 0, 0, 0, 0, 191, 345, 0, 0, 3225, 400, 61, 89, 1, 0, 5544, 0, 0, 0, 234, 0, 593, 0, 89, 0, 5199, 11, 25, 152, 7216, 0, 0, 0, 6726, 717, 802, 0] payload_rows_per_dst=[6294, 4316, 9926, 15164, 7976, 11384, 15858, 3064] dup_rows_per_dst=[322, 572, 1678, 10244, 646, 174, 10246, 440] combine_rows_per_src=[6294, 4316, 9926, 15164, 7976, 11384, 15858, 3064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=281837568 dispatch_meta_bytes=344600 combine_payload_bytes=281837568 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=282182168 ingress_bytes=454852608 link_bytes=454852608 token_bytes=564019736 weight_movement_bytes=173015040 total_nvlink_bytes=737034776 weight_to_token_ratio=0.306754 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.711573745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=269467648 dispatch_meta_bytes=317080 combine_payload_bytes=269467648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269784728 ingress_bytes=304070656 link_bytes=304070656 token_bytes=539252376 weight_movement_bytes=34603008 total_nvlink_bytes=573855384 weight_to_token_ratio=0.064168 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144193399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84944 payload_rows_remote=68562 dup_rows=13360 dup_rows_remote=7098 dedup_ratio=0.135905 router_tokens_per_expert=[1, 0, 255, 0, 0, 220, 0, 0, 1135, 3, 1204, 0, 4, 0, 0, 0, 379, 0, 3, 0, 122, 0, 85, 2544, 100, 250, 1, 1, 1, 4739, 0, 1, 8189, 0, 51, 0, 7904, 0, 41, 1, 26, 7765, 1, 0, 1, 0, 0, 6058, 1, 0, 0, 0, 0, 1, 1, 0, 0, 73, 0, 7910, 0, 81, 0, 0] payload_rows_per_dst=[952, 12548, 16382, 10002, 15912, 13176, 4, 15968] dup_rows_per_dst=[0, 3338, 6262, 184, 82, 3332, 2, 160] combine_rows_per_src=[952, 12548, 16382, 10002, 15912, 13176, 4, 15968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=245702656 dispatch_meta_bytes=334808 combine_payload_bytes=245702656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246037464 ingress_bytes=280305664 link_bytes=280305664 token_bytes=491740120 weight_movement_bytes=34603008 total_nvlink_bytes=526343128 weight_to_token_ratio=0.070368 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054767647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=59390 dup_rows=23966 dup_rows_remote=13468 dedup_ratio=0.243795 router_tokens_per_expert=[2491, 493, 0, 0, 0, 0, 2193, 18, 0, 6405, 0, 1100, 172, 0, 1179, 0, 0, 156, 2910, 26, 0, 450, 64, 0, 0, 0, 2837, 910, 938, 0, 564, 376, 0, 0, 0, 427, 0, 1, 0, 5658, 0, 0, 126, 54, 1515, 452, 6530, 0, 6289, 65, 0, 204, 0, 99, 132, 276, 70, 0, 0, 0, 3084, 78, 53, 757] payload_rows_per_dst=[8844, 4718, 7026, 9406, 1110, 13466, 14948, 14820] dup_rows_per_dst=[3496, 184, 186, 1568, 22, 1938, 10498, 6074] combine_rows_per_src=[8844, 4718, 7026, 9406, 1110, 13466, 14948, 14820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277250048 dispatch_meta_bytes=366752 combine_payload_bytes=277250048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277616800 ingress_bytes=311853056 link_bytes=311853056 token_bytes=554866848 weight_movement_bytes=34603008 total_nvlink_bytes=589469856 weight_to_token_ratio=0.062363 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173477943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=280829952 dispatch_meta_bytes=302640 combine_payload_bytes=280829952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281132592 ingress_bytes=315432960 link_bytes=315432960 token_bytes=561962544 weight_movement_bytes=34603008 total_nvlink_bytes=596565552 weight_to_token_ratio=0.061575 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186948834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80908 payload_rows_remote=70234 dup_rows=17396 dup_rows_remote=15862 dedup_ratio=0.176961 router_tokens_per_expert=[1089, 188, 0, 116, 0, 0, 1584, 3, 44, 6693, 0, 981, 161, 126, 1289, 445, 4, 52, 3333, 179, 0, 409, 366, 0, 2, 0, 2089, 3193, 203, 0, 291, 326, 0, 0, 0, 1493, 0, 48, 44, 5926, 0, 0, 51, 29, 565, 164, 5580, 0, 5418, 32, 0, 421, 0, 277, 585, 646, 62, 5, 0, 0, 3523, 103, 133, 881] payload_rows_per_dst=[13098, 14902, 8254, 10674, 9662, 1610, 13814, 8894] dup_rows_per_dst=[4022, 4576, 432, 1534, 2082, 8, 4222, 520] combine_rows_per_src=[13098, 14902, 8254, 10674, 9662, 1610, 13814, 8894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=70614 dup_rows=23766 dup_rows_remote=23592 dedup_ratio=0.241760 router_tokens_per_expert=[320, 93, 48, 504, 0, 171, 561, 0, 881, 3946, 278, 309, 2644, 304, 631, 349, 179, 26, 2661, 362, 151, 403, 1020, 2134, 2241, 91, 928, 3558, 207, 366, 226, 317, 2138, 2, 0, 863, 604, 58, 522, 2343, 139, 1000, 98, 7, 433, 200, 3602, 384, 4641, 54, 10, 267, 155, 227, 1556, 1010, 153, 38, 0, 290, 1251, 153, 63, 982] payload_rows_per_dst=[8896, 12310, 12530, 12420, 3924, 3646, 13028, 7784] dup_rows_per_dst=[1702, 3106, 5618, 3448, 174, 876, 7498, 1344] combine_rows_per_src=[8896, 12310, 12530, 12420, 3924, 3646, 13028, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=243261440 dispatch_meta_bytes=291432 combine_payload_bytes=243261440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243552872 ingress_bytes=277864448 link_bytes=277864448 token_bytes=486814312 weight_movement_bytes=34603008 total_nvlink_bytes=521417320 weight_to_token_ratio=0.071081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.045581548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71410 payload_rows_remote=59844 dup_rows=26894 dup_rows_remote=20972 dedup_ratio=0.273580 router_tokens_per_expert=[149, 29, 129, 214, 0, 103, 176, 0, 1219, 1410, 640, 136, 1436, 140, 222, 160, 342, 12, 919, 109, 127, 124, 544, 4424, 2419, 178, 330, 1087, 15, 2380, 52, 91, 6182, 0, 0, 317, 4936, 32, 446, 800, 106, 5211, 38, 1, 187, 45, 1256, 3156, 1814, 19, 3, 83, 34, 65, 419, 365, 118, 47, 23, 3039, 535, 159, 33, 367] payload_rows_per_dst=[3396, 6194, 13386, 10978, 11426, 11566, 4708, 9756] dup_rows_per_dst=[716, 1712, 6436, 2126, 5780, 5922, 2496, 1706] combine_rows_per_src=[3396, 6194, 13386, 10978, 11426, 11566, 4708, 9756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80794 payload_rows_remote=68398 dup_rows=17510 dup_rows_remote=14574 dedup_ratio=0.178121 router_tokens_per_expert=[3694, 654, 0, 0, 0, 0, 2167, 20, 0, 5371, 0, 1237, 160, 0, 898, 0, 0, 195, 2800, 1, 0, 516, 62, 0, 0, 0, 3049, 707, 1698, 0, 523, 420, 0, 0, 0, 5, 0, 0, 0, 5147, 0, 0, 305, 84, 2019, 526, 6321, 0, 6238, 38, 0, 165, 0, 97, 36, 216, 81, 0, 0, 0, 2862, 98, 0, 742] payload_rows_per_dst=[14962, 12396, 6890, 10570, 10298, 5688, 13028, 6962] dup_rows_per_dst=[10750, 2936, 258, 2224, 6, 180, 552, 604] combine_rows_per_src=[14962, 12396, 6890, 10570, 10298, 5688, 13028, 6962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287678464 dispatch_meta_bytes=344384 combine_payload_bytes=287678464 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=288022848 ingress_bytes=322281472 link_bytes=322281472 token_bytes=575701312 weight_movement_bytes=34603008 total_nvlink_bytes=610304320 weight_to_token_ratio=0.060106 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212719233e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=289234944 dispatch_meta_bytes=376824 combine_payload_bytes=289234944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=289611768 ingress_bytes=427646976 link_bytes=427646976 token_bytes=578846712 weight_movement_bytes=138412032 total_nvlink_bytes=717258744 weight_to_token_ratio=0.239117 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609201142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69286 payload_rows_remote=53334 dup_rows=29018 dup_rows_remote=28842 dedup_ratio=0.295186 router_tokens_per_expert=[0, 0, 274, 0, 0, 185, 0, 0, 1263, 0, 1269, 0, 0, 0, 0, 0, 396, 0, 0, 0, 127, 0, 63, 3357, 187, 229, 0, 0, 0, 3922, 0, 0, 8192, 0, 46, 0, 7983, 0, 109, 0, 52, 7774, 0, 0, 0, 0, 0, 5660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 7900, 0, 88, 0, 0] payload_rows_per_dst=[918, 3598, 7784, 8536, 16384, 16114, 0, 15952] dup_rows_per_dst=[0, 1466, 102, 140, 12696, 14438, 0, 176] combine_rows_per_src=[918, 3598, 7784, 8536, 16384, 16114, 0, 15952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79262 payload_rows_remote=69118 dup_rows=19042 dup_rows_remote=17906 dedup_ratio=0.193705 router_tokens_per_expert=[2484, 0, 0, 105, 747, 2304, 0, 0, 1161, 69, 900, 0, 0, 0, 0, 0, 0, 90, 88, 0, 0, 89, 0, 0, 76, 0, 0, 0, 69, 90, 0, 0, 0, 0, 6884, 3, 0, 0, 4226, 887, 0, 7292, 1403, 7499, 61, 0, 5232, 86, 0, 0, 0, 90, 88, 0, 109, 5108, 497, 0, 0, 344, 593, 219, 253, 6] payload_rows_per_dst=[10144, 14998, 8986, 15136, 1780, 15272, 10790, 2156] dup_rows_per_dst=[1136, 3030, 0, 332, 0, 12876, 0, 1668] combine_rows_per_src=[10144, 14998, 8986, 15136, 1780, 15272, 10790, 2156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=245121024 dispatch_meta_bytes=323264 combine_payload_bytes=245121024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=245444288 ingress_bytes=348930048 link_bytes=348930048 token_bytes=490565312 weight_movement_bytes=103809024 total_nvlink_bytes=594374336 weight_to_token_ratio=0.211611 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312995679e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92558 payload_rows_remote=76832 dup_rows=5746 dup_rows_remote=5362 dedup_ratio=0.058451 router_tokens_per_expert=[273, 505, 0, 0, 7, 1, 0, 0, 0, 0, 2, 0, 4, 33, 7114, 2, 0, 0, 113, 25, 451, 0, 0, 7466, 356, 0, 57, 3, 65, 8181, 0, 0, 0, 0, 0, 71, 114, 1, 4690, 0, 0, 0, 0, 0, 642, 7990, 893, 1, 9, 0, 0, 0, 0, 1, 7279, 6, 495, 1, 0, 259, 0, 0, 2004, 38] payload_rows_per_dst=[14754, 68, 15726, 16374, 9678, 16302, 14580, 5076] dup_rows_per_dst=[1046, 2, 384, 950, 74, 2754, 18, 518] combine_rows_per_src=[14754, 68, 15726, 16374, 9678, 16302, 14580, 5076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=280158208 dispatch_meta_bytes=331888 combine_payload_bytes=280158208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280490096 ingress_bytes=349364224 link_bytes=349364224 token_bytes=560648304 weight_movement_bytes=69206016 total_nvlink_bytes=629854320 weight_to_token_ratio=0.123439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314629449e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=218456064 dispatch_meta_bytes=328704 combine_payload_bytes=218456064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=218784768 ingress_bytes=253059072 link_bytes=253059072 token_bytes=437240832 weight_movement_bytes=34603008 total_nvlink_bytes=471843840 weight_to_token_ratio=0.079139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.522409150e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90360 payload_rows_remote=75034 dup_rows=7944 dup_rows_remote=7138 dedup_ratio=0.080811 router_tokens_per_expert=[0, 77, 455, 36, 0, 0, 5352, 0, 0, 61, 1267, 345, 7447, 0, 0, 7457, 0, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 8102, 0, 0, 1744, 0, 190, 0, 87, 0, 0, 2708, 0, 0, 0, 0, 403, 0, 0, 0, 0, 160, 0, 0, 61, 0, 40, 440, 0, 78, 0, 6908, 0, 0, 808, 71, 0, 0] payload_rows_per_dst=[11368, 3194, 9710, 3488, 16384, 15688, 15326, 15202] dup_rows_per_dst=[472, 152, 0, 0, 5790, 352, 806, 372] combine_rows_per_src=[11368, 3194, 9710, 3488, 16384, 15688, 15326, 15202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283107328 dispatch_meta_bytes=348096 combine_payload_bytes=283107328 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283455424 ingress_bytes=352313344 link_bytes=352313344 token_bytes=566562752 weight_movement_bytes=69206016 total_nvlink_bytes=635768768 weight_to_token_ratio=0.122151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.325726750e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=314703872 dispatch_meta_bytes=328776 combine_payload_bytes=314703872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315032648 ingress_bytes=349306880 link_bytes=349306880 token_bytes=629736520 weight_movement_bytes=34603008 total_nvlink_bytes=664339528 weight_to_token_ratio=0.054948 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314413668e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83344 payload_rows_remote=69050 dup_rows=14960 dup_rows_remote=9946 dedup_ratio=0.152181 router_tokens_per_expert=[57, 244, 63, 761, 1618, 37, 2894, 21, 188, 53, 4808, 487, 5708, 810, 305, 2478, 245, 88, 41, 279, 129, 2261, 167, 174, 121, 0, 188, 6950, 2912, 139, 838, 80, 1027, 0, 115, 0, 83, 540, 1396, 65, 0, 30, 565, 68, 553, 199, 205, 389, 1030, 0, 113, 544, 332, 652, 299, 711, 814, 2418, 147, 330, 1233, 17, 87, 46] payload_rows_per_dst=[10090, 11114, 6642, 9180, 14294, 7976, 14746, 9302] dup_rows_per_dst=[1910, 1578, 126, 420, 5014, 998, 4032, 882] combine_rows_per_src=[10090, 11114, 6642, 9180, 14294, 7976, 14746, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90518 payload_rows_remote=74308 dup_rows=7786 dup_rows_remote=5250 dedup_ratio=0.079203 router_tokens_per_expert=[0, 62, 187, 398, 15, 0, 5433, 0, 78, 53, 3906, 693, 7467, 0, 0, 6192, 0, 0, 0, 0, 0, 3900, 3, 0, 19, 0, 17, 8044, 82, 0, 1172, 39, 240, 0, 74, 0, 8, 1164, 38, 0, 0, 0, 360, 0, 45, 0, 0, 644, 0, 0, 31, 116, 39, 666, 0, 75, 28, 6095, 0, 222, 1497, 50, 0, 0] payload_rows_per_dst=[11454, 8966, 7806, 16210, 2592, 13658, 15464, 14368] dup_rows_per_dst=[736, 494, 0, 2536, 456, 824, 1324, 1416] combine_rows_per_src=[11454, 8966, 7806, 16210, 2592, 13658, 15464, 14368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=307339264 dispatch_meta_bytes=328688 combine_payload_bytes=307339264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307667952 ingress_bytes=341942272 link_bytes=341942272 token_bytes=615007216 weight_movement_bytes=34603008 total_nvlink_bytes=649610224 weight_to_token_ratio=0.056264 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286701241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82890 payload_rows_remote=71748 dup_rows=15414 dup_rows_remote=13628 dedup_ratio=0.156799 router_tokens_per_expert=[323, 429, 37, 220, 1431, 158, 1032, 3, 56, 6, 2060, 122, 2035, 2111, 2916, 970, 106, 50, 85, 286, 231, 658, 74, 3942, 334, 0, 81, 2667, 3871, 3915, 352, 22, 419, 0, 32, 0, 128, 189, 3605, 42, 0, 7, 271, 74, 985, 3652, 426, 79, 676, 0, 13, 457, 92, 227, 2378, 1015, 1689, 822, 39, 350, 357, 2, 466, 77] payload_rows_per_dst=[11054, 7482, 10256, 13016, 12134, 11142, 10950, 6856] dup_rows_per_dst=[2044, 1228, 608, 4134, 2030, 1786, 2836, 748] combine_rows_per_src=[11054, 7482, 10256, 13016, 12134, 11142, 10950, 6856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282828800 dispatch_meta_bytes=315984 combine_payload_bytes=282828800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283144784 ingress_bytes=317431808 link_bytes=317431808 token_bytes=565973584 weight_movement_bytes=34603008 total_nvlink_bytes=600576592 weight_to_token_ratio=0.061139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.194470338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83122 payload_rows_remote=68574 dup_rows=15182 dup_rows_remote=4966 dedup_ratio=0.154439 router_tokens_per_expert=[0, 98, 993, 18, 0, 0, 4973, 0, 0, 3, 382, 415, 7424, 0, 0, 7471, 0, 0, 0, 0, 0, 4552, 1, 0, 0, 0, 0, 8074, 0, 0, 1970, 0, 116, 0, 111, 0, 0, 4098, 0, 0, 0, 0, 437, 0, 0, 0, 0, 120, 0, 0, 71, 0, 3, 412, 0, 98, 0, 6471, 0, 0, 746, 95, 0, 0] payload_rows_per_dst=[11414, 14548, 9106, 16342, 8418, 6120, 2556, 14618] dup_rows_per_dst=[750, 10216, 0, 3746, 232, 162, 70, 6] combine_rows_per_src=[11414, 14548, 9106, 16342, 8418, 6120, 2556, 14618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=304365568 dispatch_meta_bytes=318232 combine_payload_bytes=304365568 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304683800 ingress_bytes=442777600 link_bytes=442777600 token_bytes=609049368 weight_movement_bytes=138412032 total_nvlink_bytes=747461400 weight_to_token_ratio=0.227259 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666136462e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90146 payload_rows_remote=78142 dup_rows=8158 dup_rows_remote=4950 dedup_ratio=0.082987 router_tokens_per_expert=[493, 490, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 53, 6979, 0, 0, 0, 53, 0, 478, 0, 0, 7600, 269, 0, 84, 0, 39, 8192, 0, 0, 3, 0, 0, 6, 138, 0, 4938, 0, 0, 0, 0, 0, 842, 8043, 830, 0, 1, 0, 0, 0, 0, 0, 7262, 4, 262, 0, 0, 342, 0, 0, 1663, 82] payload_rows_per_dst=[7788, 7210, 15882, 6342, 10066, 16330, 14524, 12004] dup_rows_per_dst=[976, 68, 380, 312, 104, 3100, 10, 3208] combine_rows_per_src=[7788, 7210, 15882, 6342, 10066, 16330, 14524, 12004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=293879808 dispatch_meta_bytes=341504 combine_payload_bytes=293879808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294221312 ingress_bytes=328482816 link_bytes=328482816 token_bytes=588101120 weight_movement_bytes=34603008 total_nvlink_bytes=622704128 weight_to_token_ratio=0.058839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236054392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=280879104 dispatch_meta_bytes=294160 combine_payload_bytes=280879104 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=281173264 ingress_bytes=419291136 link_bytes=419291136 token_bytes=562052368 weight_movement_bytes=138412032 total_nvlink_bytes=700464400 weight_to_token_ratio=0.246262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.577758789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84584 payload_rows_remote=69266 dup_rows=13720 dup_rows_remote=13720 dedup_ratio=0.139567 router_tokens_per_expert=[0, 0, 1110, 0, 8187, 0, 3, 0, 161, 0, 0, 199, 0, 0, 0, 0, 3, 253, 0, 2, 2, 0, 7403, 0, 0, 1, 1, 0, 2, 2237, 0, 5, 1, 5162, 0, 0, 0, 0, 2, 4, 0, 179, 0, 7671, 0, 0, 1, 0, 0, 0, 452, 0, 0, 7851, 0, 0, 71, 0, 0, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16380, 724, 15318, 4490, 10, 15348, 15936, 16378] dup_rows_per_dst=[12544, 0, 0, 2, 0, 358, 674, 142] combine_rows_per_src=[16380, 724, 15318, 4490, 10, 15348, 15936, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79956 payload_rows_remote=65058 dup_rows=18348 dup_rows_remote=6694 dedup_ratio=0.186646 router_tokens_per_expert=[4068, 828, 0, 0, 0, 0, 2384, 28, 0, 5094, 0, 1095, 149, 0, 822, 0, 0, 179, 2659, 1, 0, 497, 43, 0, 0, 0, 3651, 681, 1932, 0, 559, 424, 0, 0, 0, 0, 0, 0, 0, 4373, 0, 0, 338, 64, 2253, 517, 5968, 0, 6321, 14, 0, 239, 0, 89, 0, 90, 69, 0, 0, 0, 2809, 89, 0, 825] payload_rows_per_dst=[14898, 11962, 6560, 11790, 8746, 6120, 12974, 6906] dup_rows_per_dst=[11654, 2358, 198, 2704, 0, 224, 532, 678] combine_rows_per_src=[14898, 11962, 6560, 11790, 8746, 6120, 12974, 6906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320069632 dispatch_meta_bytes=332368 combine_payload_bytes=320069632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320402000 ingress_bytes=354672640 link_bytes=354672640 token_bytes=640471632 weight_movement_bytes=34603008 total_nvlink_bytes=675074640 weight_to_token_ratio=0.054027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334604591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90928 payload_rows_remote=75688 dup_rows=7376 dup_rows_remote=4654 dedup_ratio=0.075033 router_tokens_per_expert=[5, 0, 0, 0, 0, 77, 427, 0, 0, 8, 100, 0, 0, 51, 434, 0, 5441, 5, 81, 7663, 7450, 0, 0, 0, 0, 388, 630, 0, 7675, 0, 51, 0, 1844, 0, 0, 0, 180, 6202, 7601, 78, 0, 0, 0, 0, 0, 34, 0, 345, 0, 1374, 0, 157, 0, 0, 0, 0, 0, 0, 0, 17, 512, 0, 185, 137] payload_rows_per_dst=[918, 15734, 10894, 6980, 12500, 15440, 15240, 13222] dup_rows_per_dst=[100, 778, 0, 516, 1856, 520, 2722, 884] combine_rows_per_src=[918, 15734, 10894, 6980, 12500, 15440, 15240, 13222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83590 payload_rows_remote=72556 dup_rows=14714 dup_rows_remote=14066 dedup_ratio=0.149679 router_tokens_per_expert=[14, 5, 168, 2, 279, 332, 4358, 1247, 110, 131, 288, 40, 68, 140, 304, 0, 4351, 185, 120, 4383, 3557, 0, 121, 88, 63, 144, 196, 4, 5278, 156, 55, 1975, 568, 1875, 295, 0, 280, 2549, 5522, 4698, 6, 61, 0, 222, 0, 53, 406, 1031, 23, 996, 9, 44, 352, 45, 45, 225, 55, 0, 29, 0, 390, 279, 583, 349] payload_rows_per_dst=[9796, 9790, 9588, 13622, 11034, 12600, 9318, 7842] dup_rows_per_dst=[6764, 1138, 142, 2120, 648, 2002, 1274, 626] combine_rows_per_src=[9796, 9790, 9588, 13622, 11034, 12600, 9318, 7842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=283713536 dispatch_meta_bytes=331944 combine_payload_bytes=283713536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=284045480 ingress_bytes=422125568 link_bytes=422125568 token_bytes=567759016 weight_movement_bytes=138412032 total_nvlink_bytes=706171048 weight_to_token_ratio=0.243787 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.588424528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=287008 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764576 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533242144 weight_movement_bytes=34603008 total_nvlink_bytes=567845152 weight_to_token_ratio=0.064892 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89952 payload_rows_remote=74828 dup_rows=8352 dup_rows_remote=7746 dedup_ratio=0.084961 router_tokens_per_expert=[23, 0, 0, 0, 0, 254, 1150, 2, 0, 23, 153, 0, 0, 54, 138, 0, 4479, 3, 62, 7490, 7069, 0, 0, 0, 0, 227, 317, 0, 7267, 0, 54, 0, 978, 0, 2, 0, 227, 5684, 7921, 1310, 0, 0, 0, 0, 0, 50, 0, 1633, 0, 1381, 0, 112, 0, 0, 0, 0, 0, 0, 0, 6, 505, 0, 276, 302] payload_rows_per_dst=[2734, 15126, 9088, 15124, 4482, 16182, 14576, 12640] dup_rows_per_dst=[124, 590, 0, 606, 552, 3026, 2548, 906] combine_rows_per_src=[2734, 15126, 9088, 15124, 4482, 16182, 14576, 12640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=310018048 dispatch_meta_bytes=321368 combine_payload_bytes=310018048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310339416 ingress_bytes=344621056 link_bytes=344621056 token_bytes=620357464 weight_movement_bytes=34603008 total_nvlink_bytes=654960472 weight_to_token_ratio=0.055779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296781289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82100 payload_rows_remote=68096 dup_rows=16204 dup_rows_remote=15108 dedup_ratio=0.164836 router_tokens_per_expert=[9, 24, 420, 1, 4384, 170, 2873, 616, 236, 31, 63, 228, 19, 33, 52, 0, 1491, 227, 48, 1517, 1154, 0, 1773, 49, 33, 62, 54, 13, 1987, 1300, 11, 3701, 172, 4728, 123, 0, 161, 1006, 1896, 3340, 6, 181, 0, 4861, 0, 19, 202, 385, 0, 368, 174, 5, 200, 2301, 32, 344, 100, 0, 34, 0, 144, 5351, 203, 237] payload_rows_per_dst=[14224, 4090, 7068, 13290, 7438, 14004, 8434, 13552] dup_rows_per_dst=[12226, 268, 108, 1032, 154, 1096, 722, 598] combine_rows_per_src=[14224, 4090, 7068, 13290, 7438, 14004, 8434, 13552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81800 payload_rows_remote=65796 dup_rows=16504 dup_rows_remote=9292 dedup_ratio=0.167887 router_tokens_per_expert=[43, 0, 0, 0, 0, 97, 132, 0, 0, 4, 173, 0, 0, 64, 957, 0, 4786, 37, 98, 7593, 7474, 0, 0, 0, 0, 302, 752, 0, 7671, 0, 64, 0, 2703, 0, 3, 0, 110, 5509, 7504, 98, 0, 0, 0, 0, 0, 23, 0, 150, 0, 1493, 0, 224, 0, 0, 0, 0, 0, 0, 1, 13, 480, 0, 265, 329] payload_rows_per_dst=[426, 16004, 14632, 16172, 15040, 1730, 5602, 12194] dup_rows_per_dst=[118, 7212, 1814, 1406, 5048, 36, 542, 328] combine_rows_per_src=[426, 16004, 14632, 16172, 15040, 1730, 5602, 12194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=297189376 dispatch_meta_bytes=346488 combine_payload_bytes=297189376 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=297535864 ingress_bytes=435601408 link_bytes=435601408 token_bytes=594725240 weight_movement_bytes=138412032 total_nvlink_bytes=733137272 weight_to_token_ratio=0.232733 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.639133030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91508 payload_rows_remote=75124 dup_rows=6796 dup_rows_remote=6624 dedup_ratio=0.069132 router_tokens_per_expert=[0, 0, 1008, 0, 8192, 0, 0, 0, 217, 0, 0, 126, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 7031, 0, 0, 0, 0, 0, 0, 2482, 0, 0, 0, 5216, 0, 0, 0, 0, 0, 0, 0, 89, 0, 7730, 0, 0, 0, 0, 0, 0, 710, 0, 0, 7824, 0, 0, 86, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 686, 14560, 4964, 7164, 15460, 15906, 16384] dup_rows_per_dst=[5284, 0, 0, 0, 0, 178, 1162, 172] combine_rows_per_src=[16384, 686, 14560, 4964, 7164, 15460, 15906, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=306495488 dispatch_meta_bytes=330296 combine_payload_bytes=306495488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306825784 ingress_bytes=375701504 link_bytes=375701504 token_bytes=613321272 weight_movement_bytes=69206016 total_nvlink_bytes=682527288 weight_to_token_ratio=0.112838 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413734513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269500416 dispatch_meta_bytes=300352 combine_payload_bytes=269500416 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269800768 ingress_bytes=338706432 link_bytes=338706432 token_bytes=539301184 weight_movement_bytes=69206016 total_nvlink_bytes=608507200 weight_to_token_ratio=0.128325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274525036e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=332816 combine_payload_bytes=278921216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279254032 ingress_bytes=313524224 link_bytes=313524224 token_bytes=558175248 weight_movement_bytes=34603008 total_nvlink_bytes=592778256 weight_to_token_ratio=0.061993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179766414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70820 payload_rows_remote=68764 dup_rows=27484 dup_rows_remote=27332 dedup_ratio=0.279582 router_tokens_per_expert=[0, 0, 7872, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1100, 0, 0, 0, 74, 0, 1, 0, 4, 0, 951, 79, 2, 343, 0, 3, 3, 0, 60, 0, 0, 7678, 0, 7402, 0, 0, 0, 0, 5761, 0, 402, 7938, 345, 0, 0, 0, 0, 551, 0, 0, 1, 801, 1, 0, 1, 0, 0, 1, 7772, 3, 0, 0] payload_rows_per_dst=[15746, 2208, 2056, 814, 16202, 16224, 2018, 15552] dup_rows_per_dst=[0, 6, 152, 2, 13958, 12668, 696, 2] combine_rows_per_src=[15746, 2208, 2056, 814, 16202, 16224, 2018, 15552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=62956 dup_rows=24118 dup_rows_remote=22900 dedup_ratio=0.245341 router_tokens_per_expert=[0, 89, 1431, 3, 0, 0, 4701, 0, 0, 0, 200, 452, 7285, 0, 0, 7472, 3, 0, 0, 0, 0, 3735, 1, 0, 0, 0, 0, 8063, 0, 0, 2247, 0, 57, 0, 123, 0, 0, 5339, 0, 0, 0, 0, 457, 0, 0, 0, 0, 99, 0, 0, 39, 0, 0, 402, 0, 90, 0, 6211, 0, 0, 580, 73, 0, 0] payload_rows_per_dst=[11230, 16182, 14908, 5048, 10924, 1112, 1058, 13724] dup_rows_per_dst=[1218, 15802, 6770, 206, 114, 0, 4, 4] combine_rows_per_src=[11230, 16182, 14908, 5048, 10924, 1112, 1058, 13724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307707904 dispatch_meta_bytes=326992 combine_payload_bytes=307707904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308034896 ingress_bytes=342310912 link_bytes=342310912 token_bytes=615742800 weight_movement_bytes=34603008 total_nvlink_bytes=650345808 weight_to_token_ratio=0.056197 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288088404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73326 payload_rows_remote=57430 dup_rows=24978 dup_rows_remote=17960 dedup_ratio=0.254089 router_tokens_per_expert=[0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 759, 795, 0, 78, 1393, 0, 0, 0, 4610, 1814, 8023, 746, 0, 78, 7713, 0, 0, 7496, 740, 51, 0, 0, 0, 0, 81, 0, 51, 0, 0, 1529, 0, 597, 0, 0, 0, 0, 133, 0, 0, 0, 59, 848, 1776, 0, 1167, 111, 1103, 78, 0, 0, 0, 4976, 327, 68] payload_rows_per_dst=[4104, 5594, 16384, 1502, 5854, 12170, 15896, 11822] dup_rows_per_dst=[18, 456, 13940, 80, 1478, 706, 7018, 1282] combine_rows_per_src=[4104, 5594, 16384, 1502, 5854, 12170, 15896, 11822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86102 payload_rows_remote=76470 dup_rows=12202 dup_rows_remote=12068 dedup_ratio=0.124125 router_tokens_per_expert=[0, 0, 314, 1508, 99, 0, 96, 10, 0, 251, 371, 5334, 16, 938, 176, 0, 133, 0, 2672, 574, 6662, 573, 2445, 85, 5236, 321, 4, 6518, 3840, 6, 35, 2, 0, 18, 67, 0, 179, 41, 0, 87, 7, 146, 129, 0, 288, 0, 99, 999, 284, 80, 286, 168, 276, 0, 165, 19, 877, 115, 13, 180, 0, 5576, 809, 25] payload_rows_per_dst=[8634, 12476, 16034, 9238, 9632, 3228, 13694, 13166] dup_rows_per_dst=[764, 1696, 4910, 668, 134, 108, 1898, 2024] combine_rows_per_src=[8634, 12476, 16034, 9238, 9632, 3228, 13694, 13166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=257867776 dispatch_meta_bytes=343424 combine_payload_bytes=257867776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258211200 ingress_bytes=292470784 link_bytes=292470784 token_bytes=516078976 weight_movement_bytes=34603008 total_nvlink_bytes=550681984 weight_to_token_ratio=0.067050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100544014e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=281657344 dispatch_meta_bytes=384384 combine_payload_bytes=281657344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282041728 ingress_bytes=385466368 link_bytes=385466368 token_bytes=563699072 weight_movement_bytes=103809024 total_nvlink_bytes=667508096 weight_to_token_ratio=0.184157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450478910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83580 payload_rows_remote=67464 dup_rows=14724 dup_rows_remote=12830 dedup_ratio=0.149780 router_tokens_per_expert=[0, 0, 0, 2096, 0, 0, 17, 0, 0, 14, 797, 4282, 0, 377, 535, 0, 1, 0, 4445, 1040, 8004, 677, 0, 63, 7739, 0, 0, 7539, 1238, 28, 0, 0, 0, 0, 37, 0, 52, 11, 0, 318, 0, 224, 0, 0, 0, 0, 120, 9, 0, 0, 198, 305, 621, 0, 331, 27, 978, 61, 3, 0, 0, 6071, 842, 52] payload_rows_per_dst=[12194, 15030, 9810, 16116, 754, 706, 15424, 13546] dup_rows_per_dst=[922, 5708, 1032, 1894, 82, 0, 2618, 2468] combine_rows_per_src=[12194, 15030, 9810, 16116, 754, 706, 15424, 13546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=235233280 dispatch_meta_bytes=301560 combine_payload_bytes=235233280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235534840 ingress_bytes=269836288 link_bytes=269836288 token_bytes=470768120 weight_movement_bytes=34603008 total_nvlink_bytes=505371128 weight_to_token_ratio=0.073503 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.015372228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76066 payload_rows_remote=59904 dup_rows=22238 dup_rows_remote=17598 dedup_ratio=0.226217 router_tokens_per_expert=[0, 0, 0, 2699, 0, 0, 0, 0, 0, 0, 229, 112, 0, 98, 1979, 0, 0, 0, 3893, 1549, 7983, 747, 0, 98, 7596, 0, 0, 7472, 504, 64, 0, 0, 0, 0, 117, 0, 61, 0, 0, 2003, 0, 735, 0, 0, 0, 0, 156, 0, 0, 0, 0, 1102, 2673, 0, 1681, 142, 1010, 97, 0, 0, 0, 4107, 190, 55] payload_rows_per_dst=[11478, 16162, 4396, 15956, 4240, 1782, 11834, 10218] dup_rows_per_dst=[1706, 4640, 392, 10972, 122, 0, 3706, 700] combine_rows_per_src=[11478, 16162, 4396, 15956, 4240, 1782, 11834, 10218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81670 payload_rows_remote=70644 dup_rows=16634 dup_rows_remote=8418 dedup_ratio=0.169210 router_tokens_per_expert=[0, 0, 4710, 568, 42, 0, 65, 123, 0, 1454, 80, 2183, 1548, 347, 75, 0, 101, 0, 1174, 177, 2373, 186, 5957, 88, 1835, 789, 0, 2276, 2557, 1, 567, 4, 0, 1285, 10, 2595, 109, 6, 0, 71, 1749, 17, 320, 3190, 2869, 0, 33, 1443, 202, 470, 258, 41, 92, 717, 27, 4, 296, 53, 2, 304, 773, 2674, 258, 4] payload_rows_per_dst=[12720, 9986, 15654, 6988, 10350, 11026, 7126, 7820] dup_rows_per_dst=[644, 1388, 2110, 848, 1472, 8216, 1048, 908] combine_rows_per_src=[12720, 9986, 15654, 6988, 10350, 11026, 7126, 7820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=313221120 dispatch_meta_bytes=354152 combine_payload_bytes=313221120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313575272 ingress_bytes=347824128 link_bytes=347824128 token_bytes=626796392 weight_movement_bytes=34603008 total_nvlink_bytes=661399400 weight_to_token_ratio=0.055206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308834192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71120 payload_rows_remote=55634 dup_rows=27184 dup_rows_remote=27184 dedup_ratio=0.276530 router_tokens_per_expert=[0, 0, 7891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 0, 0, 0, 85, 0, 0, 0, 0, 0, 1082, 88, 0, 347, 0, 0, 0, 0, 60, 0, 0, 7086, 0, 7596, 0, 0, 0, 0, 5835, 0, 487, 8085, 339, 0, 0, 0, 0, 527, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 7743, 0, 0, 0] payload_rows_per_dst=[15782, 2128, 2336, 814, 16258, 16214, 2102, 15486] dup_rows_per_dst=[0, 0, 174, 0, 13106, 13278, 626, 0] combine_rows_per_src=[15782, 2128, 2336, 814, 16258, 16214, 2102, 15486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=276332544 dispatch_meta_bytes=321176 combine_payload_bytes=276332544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=276653720 ingress_bytes=414744576 link_bytes=414744576 token_bytes=552986264 weight_movement_bytes=138412032 total_nvlink_bytes=691398296 weight_to_token_ratio=0.250299 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560650450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245366784 dispatch_meta_bytes=310008 combine_payload_bytes=245366784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245676792 ingress_bytes=279969792 link_bytes=279969792 token_bytes=491043576 weight_movement_bytes=34603008 total_nvlink_bytes=525646584 weight_to_token_ratio=0.070468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.053503788e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=289357824 dispatch_meta_bytes=316248 combine_payload_bytes=289357824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289674072 ingress_bytes=323960832 link_bytes=323960832 token_bytes=579031896 weight_movement_bytes=34603008 total_nvlink_bytes=613634904 weight_to_token_ratio=0.059760 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219038530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89500 payload_rows_remote=73144 dup_rows=8804 dup_rows_remote=6388 dedup_ratio=0.089559 router_tokens_per_expert=[0, 1, 0, 1, 2, 1, 3, 0, 0, 0, 0, 0, 1583, 0, 6799, 0, 8174, 163, 0, 3, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6503, 156, 0, 0, 0, 85, 0, 3, 1, 0, 1, 3, 2, 0, 6518, 0, 0, 0, 76, 1, 0, 0, 659, 8187, 0, 4, 3, 0, 0, 6058, 0, 260, 0, 2860] payload_rows_per_dst=[358, 14736, 16356, 13316, 7188, 13042, 9874, 14630] dup_rows_per_dst=[4, 2032, 2416, 4, 102, 6, 508, 3732] combine_rows_per_src=[358, 14736, 16356, 13316, 7188, 13042, 9874, 14630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=227876864 dispatch_meta_bytes=331272 combine_payload_bytes=227876864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=228208136 ingress_bytes=297082880 link_bytes=297082880 token_bytes=456085000 weight_movement_bytes=69206016 total_nvlink_bytes=525291016 weight_to_token_ratio=0.151739 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117898960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70402 payload_rows_remote=70224 dup_rows=27902 dup_rows_remote=27798 dedup_ratio=0.283834 router_tokens_per_expert=[50, 0, 0, 0, 0, 87, 4, 0, 0, 0, 178, 0, 0, 61, 1083, 0, 4214, 27, 101, 7546, 7418, 0, 0, 0, 0, 256, 1024, 0, 7649, 0, 61, 0, 3231, 0, 6, 0, 72, 5288, 7502, 90, 0, 0, 0, 0, 0, 35, 0, 127, 0, 1547, 0, 277, 0, 0, 0, 0, 0, 0, 0, 69, 472, 0, 333, 344] payload_rows_per_dst=[178, 16256, 15092, 16174, 16314, 324, 3648, 2416] dup_rows_per_dst=[104, 9908, 0, 1806, 16064, 0, 0, 20] combine_rows_per_src=[178, 16256, 15092, 16174, 16314, 324, 3648, 2416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86506 payload_rows_remote=74546 dup_rows=11798 dup_rows_remote=9486 dedup_ratio=0.120015 router_tokens_per_expert=[0, 6313, 1075, 1298, 6852, 7397, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 32, 59, 0, 785, 0, 0, 0, 0, 729, 326, 78, 0, 0, 0, 0, 0, 110, 0, 0, 6907, 2064, 142, 0, 969, 1808, 0, 0, 0, 0, 0, 0, 1747, 0, 0, 0, 0, 115, 5274, 609, 4271, 0, 0, 78, 0, 0, 0] payload_rows_per_dst=[4226, 13912, 1750, 15390, 15240, 12438, 11960, 11590] dup_rows_per_dst=[520, 20, 2, 1670, 3206, 1980, 2312, 2088] combine_rows_per_src=[4226, 13912, 1750, 15390, 15240, 12438, 11960, 11590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82420 payload_rows_remote=70954 dup_rows=15884 dup_rows_remote=15308 dedup_ratio=0.161580 router_tokens_per_expert=[913, 2540, 694, 892, 3253, 4622, 0, 0, 132, 158, 0, 472, 348, 0, 57, 3, 76, 19, 70, 4488, 31, 122, 644, 0, 0, 0, 1978, 189, 141, 511, 3, 44, 0, 0, 44, 70, 0, 5483, 259, 165, 0, 6498, 1758, 0, 0, 33, 0, 0, 0, 4177, 0, 0, 42, 10, 122, 4342, 1904, 513, 18, 0, 1295, 0, 17, 2] payload_rows_per_dst=[4326, 8576, 9970, 12724, 11466, 15678, 13164, 6516] dup_rows_per_dst=[672, 270, 930, 2252, 576, 5980, 4222, 982] combine_rows_per_src=[4326, 8576, 9970, 12724, 11466, 15678, 13164, 6516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=299597824 dispatch_meta_bytes=318128 combine_payload_bytes=299597824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299915952 ingress_bytes=334200832 link_bytes=334200832 token_bytes=599513776 weight_movement_bytes=34603008 total_nvlink_bytes=634116784 weight_to_token_ratio=0.057718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=287637504 dispatch_meta_bytes=392088 combine_payload_bytes=287637504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288029592 ingress_bytes=322240512 link_bytes=322240512 token_bytes=575667096 weight_movement_bytes=34603008 total_nvlink_bytes=610270104 weight_to_token_ratio=0.060109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212565104e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84034 payload_rows_remote=68586 dup_rows=14270 dup_rows_remote=12978 dedup_ratio=0.145162 router_tokens_per_expert=[142, 5661, 1035, 1343, 6046, 7338, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 29, 1648, 0, 203, 128, 0, 0, 0, 707, 148, 61, 116, 0, 0, 0, 0, 76, 0, 0, 6932, 742, 200, 0, 5067, 1762, 0, 0, 0, 0, 0, 0, 3191, 0, 0, 0, 0, 238, 3528, 1245, 1177, 0, 0, 176, 0, 0, 0] payload_rows_per_dst=[8224, 12514, 5550, 15448, 14592, 13014, 9868, 4824] dup_rows_per_dst=[1104, 4, 148, 1292, 1308, 7678, 2364, 372] combine_rows_per_src=[8224, 12514, 5550, 15448, 14592, 13014, 9868, 4824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=305340416 dispatch_meta_bytes=336128 combine_payload_bytes=305340416 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=305676544 ingress_bytes=409149440 link_bytes=409149440 token_bytes=611016960 weight_movement_bytes=103809024 total_nvlink_bytes=714825984 weight_to_token_ratio=0.169895 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539596403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=68954 dup_rows=21860 dup_rows_remote=21842 dedup_ratio=0.222371 router_tokens_per_expert=[0, 6068, 948, 1020, 6994, 7206, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 77, 33, 0, 876, 0, 0, 0, 0, 956, 613, 97, 0, 0, 0, 0, 0, 125, 0, 0, 6634, 2902, 151, 0, 316, 1456, 0, 0, 0, 0, 0, 0, 1406, 3, 0, 0, 0, 89, 5337, 232, 5507, 0, 0, 97, 0, 0, 0] payload_rows_per_dst=[16042, 7490, 11802, 8484, 15278, 3052, 2990, 11306] dup_rows_per_dst=[14696, 18, 844, 1092, 4346, 492, 6, 366] combine_rows_per_src=[16042, 7490, 11802, 8484, 15278, 3052, 2990, 11306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75412 payload_rows_remote=66630 dup_rows=22892 dup_rows_remote=21058 dedup_ratio=0.232869 router_tokens_per_expert=[757, 935, 297, 236, 1195, 1530, 644, 0, 74, 262, 0, 454, 4553, 0, 4495, 9, 3451, 204, 5, 3362, 197, 14, 1207, 0, 0, 0, 854, 62, 49, 166, 2119, 142, 0, 0, 25, 205, 0, 2072, 122, 54, 57, 3133, 425, 0, 755, 3, 0, 0, 1, 2063, 0, 0, 1908, 4023, 59, 2050, 1284, 123, 31, 276, 1241, 194, 2, 1773] payload_rows_per_dst=[3476, 13240, 13048, 9088, 4674, 8782, 14400, 8704] dup_rows_per_dst=[392, 8844, 3832, 756, 282, 1834, 5808, 1144] combine_rows_per_src=[3476, 13240, 13048, 9088, 4674, 8782, 14400, 8704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=290627584 dispatch_meta_bytes=345048 combine_payload_bytes=290627584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290972632 ingress_bytes=325230592 link_bytes=325230592 token_bytes=581600216 weight_movement_bytes=34603008 total_nvlink_bytes=616203224 weight_to_token_ratio=0.059496 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223816534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88228 payload_rows_remote=73246 dup_rows=10076 dup_rows_remote=5462 dedup_ratio=0.102498 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 0, 6858, 0, 8192, 154, 0, 0, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6513, 176, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 6024, 0, 0, 0, 117, 0, 0, 0, 541, 8192, 0, 0, 0, 0, 0, 6478, 0, 250, 0, 3070] payload_rows_per_dst=[0, 14926, 16384, 13378, 126, 12048, 16384, 14982] dup_rows_per_dst=[0, 1722, 2424, 0, 0, 0, 1316, 4614] combine_rows_per_src=[0, 14926, 16384, 13378, 126, 12048, 16384, 14982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=326256 combine_payload_bytes=280928256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281254512 ingress_bytes=315531264 link_bytes=315531264 token_bytes=562182768 weight_movement_bytes=34603008 total_nvlink_bytes=596785776 weight_to_token_ratio=0.061551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187318744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282435584 dispatch_meta_bytes=363184 combine_payload_bytes=282435584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282798768 ingress_bytes=317038592 link_bytes=317038592 token_bytes=565234352 weight_movement_bytes=34603008 total_nvlink_bytes=599837360 weight_to_token_ratio=0.061219 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192990698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272916480 dispatch_meta_bytes=350752 combine_payload_bytes=272916480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273267232 ingress_bytes=307519488 link_bytes=307519488 token_bytes=546183712 weight_movement_bytes=34603008 total_nvlink_bytes=580786720 weight_to_token_ratio=0.063354 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157171076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68226 payload_rows_remote=59262 dup_rows=30078 dup_rows_remote=30078 dedup_ratio=0.305969 router_tokens_per_expert=[2, 1011, 74, 0, 0, 5, 198, 0, 0, 8099, 0, 405, 0, 2, 0, 0, 0, 4479, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 139, 453, 13, 0, 2, 258, 0, 0, 0, 0, 0, 1, 1, 7923, 0, 4, 2894, 8188, 0, 0, 35, 0, 0, 0, 6926, 6, 0, 7747, 0, 282, 0, 0, 1, 0, 0, 1] payload_rows_per_dst=[2444, 16200, 8964, 1214, 15714, 16378, 6744, 568] dup_rows_per_dst=[136, 812, 0, 4, 7426, 21634, 66, 0] combine_rows_per_src=[2444, 16200, 8964, 1214, 15714, 16378, 6744, 568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71776 payload_rows_remote=59964 dup_rows=26528 dup_rows_remote=24748 dedup_ratio=0.269857 router_tokens_per_expert=[0, 0, 0, 2859, 0, 0, 0, 0, 0, 0, 111, 90, 0, 89, 2197, 0, 0, 0, 3937, 1513, 7926, 636, 0, 89, 7608, 0, 0, 7408, 496, 61, 0, 0, 0, 0, 164, 0, 61, 0, 0, 2106, 0, 867, 0, 0, 0, 0, 122, 0, 0, 0, 0, 1019, 3081, 0, 2035, 165, 1069, 89, 0, 0, 0, 3191, 86, 77] payload_rows_per_dst=[11812, 16032, 7840, 16056, 4540, 1978, 10406, 3112] dup_rows_per_dst=[1780, 4794, 1084, 16356, 122, 0, 2194, 198] combine_rows_per_src=[11812, 16032, 7840, 16056, 4540, 1978, 10406, 3112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300015616 dispatch_meta_bytes=314832 combine_payload_bytes=300015616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300330448 ingress_bytes=334618624 link_bytes=334618624 token_bytes=600346064 weight_movement_bytes=34603008 total_nvlink_bytes=634949072 weight_to_token_ratio=0.057638 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259142943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76532 payload_rows_remote=73050 dup_rows=21772 dup_rows_remote=21772 dedup_ratio=0.221476 router_tokens_per_expert=[3010, 0, 0, 0, 0, 49, 0, 2271, 579, 0, 90, 0, 0, 0, 409, 4086, 0, 0, 0, 5820, 279, 3150, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 0, 7729, 0, 0, 526, 0, 0, 0, 0, 1741, 0, 8002, 0, 0, 0, 0, 0, 4994, 148, 0, 0, 78] payload_rows_per_dst=[8720, 8266, 13046, 11880, 10346, 4408, 3482, 16384] dup_rows_per_dst=[1940, 1046, 3606, 0, 1894, 3226, 0, 10060] combine_rows_per_src=[8720, 8266, 13046, 11880, 10346, 4408, 3482, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=242737152 dispatch_meta_bytes=357360 combine_payload_bytes=242737152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=243094512 ingress_bytes=346546176 link_bytes=346546176 token_bytes=485831664 weight_movement_bytes=103809024 total_nvlink_bytes=589640688 weight_to_token_ratio=0.213673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304025361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78928 payload_rows_remote=71070 dup_rows=19376 dup_rows_remote=18770 dedup_ratio=0.197103 router_tokens_per_expert=[321, 0, 43, 31, 0, 5599, 0, 993, 4121, 0, 18, 0, 107, 371, 73, 1352, 211, 0, 9, 3009, 962, 3442, 0, 33, 62, 0, 2004, 0, 0, 0, 1261, 0, 1037, 0, 94, 43, 0, 0, 0, 2997, 1204, 0, 144, 7175, 14, 189, 1067, 1406, 0, 0, 0, 275, 186, 7347, 0, 61, 0, 0, 129, 1394, 147, 0, 0, 221] payload_rows_per_dst=[12696, 10808, 10166, 14590, 7858, 6272, 922, 15616] dup_rows_per_dst=[1278, 1276, 5166, 6414, 606, 1776, 0, 2860] combine_rows_per_src=[12696, 10808, 10166, 14590, 7858, 6272, 922, 15616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245612544 dispatch_meta_bytes=338848 combine_payload_bytes=245612544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245951392 ingress_bytes=280215552 link_bytes=280215552 token_bytes=491563936 weight_movement_bytes=34603008 total_nvlink_bytes=526166944 weight_to_token_ratio=0.070394 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054428563e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78438 payload_rows_remote=62734 dup_rows=19866 dup_rows_remote=19864 dedup_ratio=0.202087 router_tokens_per_expert=[926, 0, 0, 0, 0, 1566, 0, 1775, 2855, 0, 46, 0, 32, 0, 177, 3409, 29, 0, 0, 6272, 677, 5645, 0, 33, 0, 0, 1, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 2061, 175, 0, 10, 7852, 0, 0, 1266, 91, 0, 0, 0, 559, 0, 8022, 0, 0, 0, 0, 0, 3659, 93, 0, 0, 137] payload_rows_per_dst=[7600, 10784, 14890, 15704, 6998, 2890, 4494, 15078] dup_rows_per_dst=[934, 2254, 10422, 2, 692, 194, 748, 4620] combine_rows_per_src=[7600, 10784, 14890, 15704, 6998, 2890, 4494, 15078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=299212800 dispatch_meta_bytes=379288 combine_payload_bytes=299212800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=299592088 ingress_bytes=437624832 link_bytes=437624832 token_bytes=598804888 weight_movement_bytes=138412032 total_nvlink_bytes=737216920 weight_to_token_ratio=0.231147 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646747011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82140 payload_rows_remote=73194 dup_rows=16164 dup_rows_remote=15014 dedup_ratio=0.164429 router_tokens_per_expert=[4566, 0, 0, 0, 0, 22, 0, 2679, 558, 0, 117, 0, 0, 0, 286, 4087, 0, 0, 0, 5686, 274, 1352, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 0, 7435, 0, 0, 343, 0, 0, 0, 0, 2700, 0, 7961, 0, 0, 0, 0, 0, 5458, 179, 0, 0, 98] payload_rows_per_dst=[12674, 8946, 8558, 14870, 9014, 686, 15922, 11470] dup_rows_per_dst=[6766, 1150, 1524, 0, 1324, 0, 5400, 0] combine_rows_per_src=[12674, 8946, 8558, 14870, 9014, 686, 15922, 11470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80876 payload_rows_remote=69114 dup_rows=17428 dup_rows_remote=10500 dedup_ratio=0.177287 router_tokens_per_expert=[100, 464, 673, 34, 0, 5033, 3, 405, 1195, 803, 9, 76, 271, 1814, 20, 515, 162, 1279, 30, 1111, 364, 1364, 0, 22, 6, 0, 1161, 0, 238, 230, 4193, 0, 432, 0, 86, 119, 0, 0, 0, 1099, 563, 1278, 194, 3006, 429, 4505, 290, 2086, 143, 0, 0, 146, 4546, 3544, 0, 4061, 0, 118, 219, 564, 111, 0, 2, 36] payload_rows_per_dst=[12274, 8434, 6596, 13184, 11148, 11762, 9384, 8094] dup_rows_per_dst=[1150, 972, 2068, 4484, 446, 6928, 286, 1094] combine_rows_per_src=[12274, 8434, 6596, 13184, 11148, 11762, 9384, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291102720 dispatch_meta_bytes=359360 combine_payload_bytes=291102720 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291462080 ingress_bytes=360308736 link_bytes=360308736 token_bytes=582564800 weight_movement_bytes=69206016 total_nvlink_bytes=651770816 weight_to_token_ratio=0.118795 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355812766e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78496 payload_rows_remote=78100 dup_rows=19808 dup_rows_remote=19808 dedup_ratio=0.201497 router_tokens_per_expert=[0, 1063, 83, 0, 0, 0, 238, 0, 0, 8104, 0, 437, 0, 0, 0, 0, 0, 5473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 404, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 2056, 8192, 0, 0, 3, 0, 0, 0, 6877, 0, 0, 7594, 0, 198, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2606, 16208, 10946, 1164, 15188, 15928, 16060, 396] dup_rows_per_dst=[162, 874, 0, 0, 622, 5454, 12696, 0] combine_rows_per_src=[2606, 16208, 10946, 1164, 15188, 15928, 16060, 396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=256958464 dispatch_meta_bytes=330392 combine_payload_bytes=256958464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257288856 ingress_bytes=291561472 link_bytes=291561472 token_bytes=514247320 weight_movement_bytes=34603008 total_nvlink_bytes=548850328 weight_to_token_ratio=0.067289 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097122347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=299802624 dispatch_meta_bytes=352832 combine_payload_bytes=299802624 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=300155456 ingress_bytes=334405632 link_bytes=334405632 token_bytes=599958080 weight_movement_bytes=34603008 total_nvlink_bytes=634561088 weight_to_token_ratio=0.057676 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258341471e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78700 payload_rows_remote=78684 dup_rows=19604 dup_rows_remote=19602 dedup_ratio=0.199422 router_tokens_per_expert=[0, 1, 3, 0, 0, 4, 0, 67, 4849, 0, 0, 0, 2, 1, 1, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 1848, 24, 0, 0, 0, 2, 4, 0, 0, 0, 8050, 0, 1, 0, 2, 0, 0, 3365, 8035, 1, 1, 0, 7838, 0, 4, 803, 0, 0, 0, 0, 2, 8162, 1, 6069, 0, 2, 3, 0] payload_rows_per_dst=[146, 9702, 16, 3748, 16108, 16374, 16226, 16380] dup_rows_per_dst=[4, 0, 2, 0, 2, 6430, 1068, 12098] combine_rows_per_src=[146, 9702, 16, 3748, 16108, 16374, 16226, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=283090944 dispatch_meta_bytes=318456 combine_payload_bytes=283090944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=283409400 ingress_bytes=386899968 link_bytes=386899968 token_bytes=566500344 weight_movement_bytes=103809024 total_nvlink_bytes=670309368 weight_to_token_ratio=0.183246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455873432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=319897600 dispatch_meta_bytes=391632 combine_payload_bytes=319897600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320289232 ingress_bytes=354500608 link_bytes=354500608 token_bytes=640186832 weight_movement_bytes=34603008 total_nvlink_bytes=674789840 weight_to_token_ratio=0.054051 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333957248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63672 payload_rows_remote=47288 dup_rows=34632 dup_rows_remote=6752 dedup_ratio=0.352295 router_tokens_per_expert=[0, 5818, 922, 1305, 7010, 7077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 17, 0, 1068, 0, 0, 0, 0, 971, 645, 89, 0, 0, 0, 0, 0, 84, 0, 0, 6463, 3280, 133, 0, 64, 1076, 0, 0, 0, 0, 0, 0, 1490, 5, 0, 0, 0, 69, 5268, 212, 5926, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16384, 0, 11768, 3040, 15178, 2152, 3118, 12032] dup_rows_per_dst=[27880, 0, 1080, 370, 4742, 128, 10, 422] combine_rows_per_src=[16384, 0, 11768, 3040, 15178, 2152, 3118, 12032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91312 payload_rows_remote=75240 dup_rows=6992 dup_rows_remote=6918 dedup_ratio=0.071126 router_tokens_per_expert=[0, 0, 7043, 0, 0, 146, 0, 0, 0, 244, 0, 0, 7714, 0, 2626, 0, 2, 372, 61, 0, 0, 0, 223, 0, 328, 0, 0, 0, 0, 40, 0, 6993, 7216, 0, 1172, 0, 0, 0, 168, 0, 931, 3079, 0, 0, 0, 3587, 5782, 73, 0, 0, 0, 0, 244, 0, 115, 0, 0, 0, 962, 0, 31, 0, 0, 0] payload_rows_per_dst=[14378, 5252, 12510, 10344, 5172, 12478, 16072, 15106] dup_rows_per_dst=[0, 488, 370, 498, 1302, 2948, 74, 1312] combine_rows_per_src=[14378, 5252, 12510, 10344, 5172, 12478, 16072, 15106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322289664 dispatch_meta_bytes=393144 combine_payload_bytes=322289664 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=322682808 ingress_bytes=391495680 link_bytes=391495680 token_bytes=644972472 weight_movement_bytes=69206016 total_nvlink_bytes=714178488 weight_to_token_ratio=0.107301 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473166726e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83794 payload_rows_remote=69294 dup_rows=14510 dup_rows_remote=10958 dedup_ratio=0.147603 router_tokens_per_expert=[0, 0, 5966, 0, 5, 3407, 0, 0, 0, 988, 0, 0, 4525, 0, 6394, 0, 19, 355, 425, 0, 0, 1145, 990, 33, 475, 26, 28, 134, 0, 224, 350, 7134, 6624, 0, 1583, 0, 0, 0, 323, 496, 1136, 700, 361, 0, 0, 2115, 1664, 63, 0, 0, 3, 0, 242, 0, 170, 0, 0, 0, 707, 0, 109, 145, 88, 0] payload_rows_per_dst=[13774, 13248, 8342, 15092, 14500, 7104, 9734, 2000] dup_rows_per_dst=[4982, 1516, 1088, 1650, 3552, 1478, 146, 98] combine_rows_per_src=[13774, 13248, 8342, 15092, 14500, 7104, 9734, 2000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=193691648 dispatch_meta_bytes=216160 combine_payload_bytes=193691648 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=193907808 ingress_bytes=366706688 link_bytes=366706688 token_bytes=387599456 weight_movement_bytes=173015040 total_nvlink_bytes=560614496 weight_to_token_ratio=0.446376 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379887745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88772 payload_rows_remote=74128 dup_rows=9532 dup_rows_remote=9016 dedup_ratio=0.096965 router_tokens_per_expert=[0, 0, 6250, 0, 0, 235, 0, 0, 0, 295, 0, 0, 7425, 0, 5780, 0, 1, 356, 137, 0, 0, 0, 194, 0, 254, 0, 0, 0, 0, 58, 0, 7268, 7273, 0, 2787, 0, 0, 0, 465, 14, 551, 2037, 0, 0, 0, 3749, 2673, 86, 0, 0, 0, 0, 242, 0, 93, 0, 0, 0, 871, 0, 47, 0, 11, 0] payload_rows_per_dst=[12952, 11666, 11544, 14644, 15394, 5254, 15460, 1858] dup_rows_per_dst=[18, 484, 2676, 516, 5684, 94, 60, 0] combine_rows_per_src=[12952, 11666, 11544, 14644, 15394, 5254, 15460, 1858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=308183040 dispatch_meta_bytes=328632 combine_payload_bytes=308183040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308511672 ingress_bytes=342786048 link_bytes=342786048 token_bytes=616694712 weight_movement_bytes=34603008 total_nvlink_bytes=651297720 weight_to_token_ratio=0.056110 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289876302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84810 payload_rows_remote=74852 dup_rows=13494 dup_rows_remote=10664 dedup_ratio=0.137268 router_tokens_per_expert=[0, 0, 7381, 0, 0, 48, 0, 0, 0, 231, 0, 0, 7661, 0, 1433, 0, 10, 334, 65, 0, 0, 0, 345, 0, 453, 0, 0, 0, 0, 44, 0, 6747, 7054, 0, 435, 0, 0, 0, 165, 0, 1005, 2872, 0, 0, 0, 4649, 6738, 46, 0, 0, 0, 0, 231, 0, 157, 0, 0, 0, 1048, 0, 0, 0, 0, 0] payload_rows_per_dst=[14858, 9958, 15804, 13620, 7186, 7436, 13852, 2096] dup_rows_per_dst=[0, 2830, 8478, 868, 790, 410, 118, 0] combine_rows_per_src=[14858, 9958, 15804, 13620, 7186, 7436, 13852, 2096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=283828224 dispatch_meta_bytes=321008 combine_payload_bytes=283828224 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=284149232 ingress_bytes=422240256 link_bytes=422240256 token_bytes=567977456 weight_movement_bytes=138412032 total_nvlink_bytes=706389488 weight_to_token_ratio=0.243693 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.588856090e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80352 payload_rows_remote=72316 dup_rows=17952 dup_rows_remote=16306 dedup_ratio=0.182617 router_tokens_per_expert=[0, 105, 2412, 0, 204, 2636, 0, 1389, 160, 788, 141, 68, 1400, 137, 4110, 0, 11, 196, 253, 0, 0, 4165, 210, 115, 370, 25, 3385, 1644, 0, 135, 310, 3035, 2507, 0, 596, 0, 1497, 0, 109, 1181, 1241, 289, 981, 68, 1523, 638, 482, 101, 809, 0, 200, 439, 257, 38, 50, 18, 2890, 673, 334, 886, 44, 975, 2602, 320] payload_rows_per_dst=[10966, 9670, 10482, 14718, 9856, 8036, 6010, 10614] dup_rows_per_dst=[2526, 1138, 382, 3090, 1460, 1646, 412, 7298] combine_rows_per_src=[10966, 9670, 10482, 14718, 9856, 8036, 6010, 10614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79754 payload_rows_remote=63370 dup_rows=18550 dup_rows_remote=7764 dedup_ratio=0.188700 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 100, 4698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 2924, 8072, 0, 0, 0, 7859, 0, 0, 1327, 0, 0, 0, 0, 0, 8186, 0, 5399, 0, 0, 0, 0] payload_rows_per_dst=[200, 9396, 0, 5018, 16156, 16384, 16216, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 5608, 2156, 10786] combine_rows_per_src=[200, 9396, 0, 5018, 16156, 16384, 16216, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=303628288 dispatch_meta_bytes=332576 combine_payload_bytes=303628288 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=303960864 ingress_bytes=372834304 link_bytes=372834304 token_bytes=607589152 weight_movement_bytes=69206016 total_nvlink_bytes=676795168 weight_to_token_ratio=0.113903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.402945470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51406 payload_rows_remote=35048 dup_rows=46898 dup_rows_remote=20352 dedup_ratio=0.477071 router_tokens_per_expert=[0, 1, 565, 8009, 8184, 8062, 3, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 6808, 0, 0, 549, 3, 0, 0, 1, 0, 5, 445, 8188, 0, 0, 0, 5, 0, 2, 0, 0, 24, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4, 0, 0, 3, 0, 0, 3, 79, 0, 0, 0] payload_rows_per_dst=[16378, 16382, 16358, 1110, 900, 48, 60, 170] dup_rows_per_dst=[20340, 2, 26546, 6, 4, 0, 0, 0] combine_rows_per_src=[16378, 16382, 16358, 1110, 900, 48, 60, 170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=306593792 dispatch_meta_bytes=342064 combine_payload_bytes=306593792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306935856 ingress_bytes=375799808 link_bytes=375799808 token_bytes=613529648 weight_movement_bytes=69206016 total_nvlink_bytes=682735664 weight_to_token_ratio=0.112800 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414104423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=296206336 dispatch_meta_bytes=354488 combine_payload_bytes=296206336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296560824 ingress_bytes=400015360 link_bytes=400015360 token_bytes=592767160 weight_movement_bytes=103809024 total_nvlink_bytes=696576184 weight_to_token_ratio=0.175126 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505225596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78618 payload_rows_remote=63240 dup_rows=19686 dup_rows_remote=8140 dedup_ratio=0.200256 router_tokens_per_expert=[5742, 0, 0, 0, 0, 0, 0, 2459, 535, 0, 115, 0, 0, 0, 248, 4239, 0, 0, 0, 5261, 269, 702, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 1548, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 7397, 0, 0, 384, 0, 0, 0, 0, 3149, 0, 7901, 0, 0, 0, 0, 0, 5783, 158, 1, 0, 90] payload_rows_per_dst=[15378, 9234, 2352, 14794, 8228, 768, 15802, 12062] dup_rows_per_dst=[11546, 1040, 0, 0, 800, 0, 6298, 2] combine_rows_per_src=[15378, 9234, 2352, 14794, 8228, 768, 15802, 12062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=259563520 dispatch_meta_bytes=284536 combine_payload_bytes=259563520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259848056 ingress_bytes=294166528 link_bytes=294166528 token_bytes=519411576 weight_movement_bytes=34603008 total_nvlink_bytes=554014584 weight_to_token_ratio=0.066620 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106924963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80376 payload_rows_remote=64350 dup_rows=17928 dup_rows_remote=8608 dedup_ratio=0.182373 router_tokens_per_expert=[0, 158, 0, 0, 0, 0, 343, 0, 77, 7309, 514, 0, 0, 78, 0, 98, 0, 0, 0, 51, 0, 204, 0, 0, 0, 0, 0, 0, 647, 7707, 0, 0, 0, 0, 0, 0, 491, 7195, 0, 8053, 78, 0, 0, 2968, 81, 0, 22, 0, 0, 0, 0, 0, 0, 4805, 0, 0, 0, 0, 322, 7821, 0, 0, 0, 130] payload_rows_per_dst=[788, 14758, 510, 15414, 982, 16086, 16026, 15812] dup_rows_per_dst=[214, 1394, 0, 1294, 0, 4602, 9320, 1104] combine_rows_per_src=[788, 14758, 510, 15414, 982, 16086, 16026, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=143556608 dispatch_meta_bytes=221600 combine_payload_bytes=143556608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=143778208 ingress_bytes=178159616 link_bytes=178159616 token_bytes=287334816 weight_movement_bytes=34603008 total_nvlink_bytes=321937824 weight_to_token_ratio=0.120427 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=6.704002920e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78204 payload_rows_remote=77856 dup_rows=20100 dup_rows_remote=20072 dedup_ratio=0.204468 router_tokens_per_expert=[0, 2475, 0, 0, 0, 0, 4019, 271, 22, 2432, 313, 1000, 233, 80, 754, 136, 41, 0, 0, 375, 0, 198, 0, 0, 77, 31, 755, 24, 515, 3976, 0, 48, 99, 0, 0, 50, 39, 7436, 0, 3378, 188, 0, 4, 3500, 81, 0, 4, 51, 0, 48, 101, 0, 0, 7397, 0, 6, 108, 0, 296, 7500, 80, 946, 0, 65] payload_rows_per_dst=[9404, 8938, 1228, 10406, 348, 16020, 15992, 15868] dup_rows_per_dst=[4126, 1002, 0, 446, 28, 6508, 5868, 2122] combine_rows_per_src=[9404, 8938, 1228, 10406, 348, 16020, 15992, 15868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=259031040 dispatch_meta_bytes=285520 combine_payload_bytes=259031040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259316560 ingress_bytes=293634048 link_bytes=293634048 token_bytes=518347600 weight_movement_bytes=34603008 total_nvlink_bytes=552950608 weight_to_token_ratio=0.066756 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104921283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78738 payload_rows_remote=63960 dup_rows=19566 dup_rows_remote=18750 dedup_ratio=0.199036 router_tokens_per_expert=[0, 423, 0, 0, 0, 0, 1022, 149, 46, 5421, 487, 44, 0, 61, 106, 113, 8, 0, 0, 148, 0, 186, 0, 0, 0, 0, 0, 0, 617, 7180, 0, 0, 0, 0, 0, 0, 124, 7347, 0, 6843, 63, 0, 0, 3141, 64, 0, 3, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 0, 305, 7897, 0, 348, 0, 92] payload_rows_per_dst=[2826, 11608, 684, 14778, 3844, 14474, 14488, 16036] dup_rows_per_dst=[362, 948, 0, 816, 2788, 4210, 9194, 1248] combine_rows_per_src=[2826, 11608, 684, 14778, 3844, 14474, 14488, 16036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12307283664 prefetch_bytes=865075200 reduce_bytes=449839104 weight_movement_bytes=1314914304 egress_bytes=6157467344 ingress_bytes=7464730624 link_bytes=7464730624 total_nvlink_bytes=13622197968 weight_to_token_ratio=0.106840 prefetch_share_of_ingress=0.115888 dedup_ratio=0.228284 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=13 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030109 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.221268712e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263577600 dispatch_meta_bytes=291832 combine_payload_bytes=263577600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263869432 ingress_bytes=298180608 link_bytes=298180608 token_bytes=527447032 weight_movement_bytes=34603008 total_nvlink_bytes=562050040 weight_to_token_ratio=0.065605 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122029622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=318898176 dispatch_meta_bytes=391712 combine_payload_bytes=318898176 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=319289888 ingress_bytes=457310208 link_bytes=457310208 token_bytes=638188064 weight_movement_bytes=138412032 total_nvlink_bytes=776600096 weight_to_token_ratio=0.216883 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.720821496e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78964 payload_rows_remote=63824 dup_rows=19340 dup_rows_remote=17804 dedup_ratio=0.196737 router_tokens_per_expert=[0, 117, 0, 0, 0, 0, 329, 0, 97, 7488, 585, 0, 0, 97, 0, 71, 0, 0, 0, 63, 0, 177, 0, 0, 0, 0, 0, 0, 702, 7598, 0, 0, 0, 0, 0, 0, 498, 7188, 0, 8095, 98, 0, 0, 3721, 98, 0, 39, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 329, 7812, 0, 0, 0, 108] payload_rows_per_dst=[782, 15140, 480, 15196, 16218, 7644, 7684, 15820] dup_rows_per_dst=[110, 1536, 0, 1404, 15344, 268, 0, 678] combine_rows_per_src=[782, 15140, 480, 15196, 16218, 7644, 7684, 15820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81492 payload_rows_remote=67452 dup_rows=16812 dup_rows_remote=10978 dedup_ratio=0.171021 router_tokens_per_expert=[0, 1181, 126, 792, 1877, 519, 3027, 56, 7, 699, 136, 463, 269, 33, 414, 138, 36, 0, 0, 186, 597, 42, 273, 0, 579, 224, 3785, 263, 110, 1314, 0, 1095, 4224, 3148, 0, 40, 51, 4692, 0, 1178, 97, 0, 3571, 1456, 40, 0, 1, 80, 0, 414, 508, 0, 0, 4072, 14, 0, 3269, 0, 198, 3119, 329, 383, 0, 27] payload_rows_per_dst=[9080, 9456, 4450, 12284, 8524, 14040, 10168, 13490] dup_rows_per_dst=[2322, 660, 1572, 2456, 604, 5834, 2204, 1160] combine_rows_per_src=[9080, 9456, 4450, 12284, 8524, 14040, 10168, 13490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=55122 payload_rows_remote=54948 dup_rows=43182 dup_rows_remote=43182 dedup_ratio=0.439270 router_tokens_per_expert=[0, 0, 600, 8006, 8192, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6613, 0, 0, 635, 0, 0, 0, 0, 0, 0, 432, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 5712, 16384, 6, 78, 174] dup_rows_per_dst=[12422, 0, 29610, 286, 864, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 5712, 16384, 6, 78, 174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12565654736 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6286642384 ingress_bytes=7870750720 link_bytes=7870750720 total_nvlink_bytes=14157393104 weight_to_token_ratio=0.126674 prefetch_share_of_ingress=0.083532 dedup_ratio=0.201953 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.287695709e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12876758160 prefetch_bytes=761266176 reduce_bytes=1280311296 weight_movement_bytes=2041577472 egress_bytes=6442368144 ingress_bytes=8475967488 link_bytes=8475967488 total_nvlink_bytes=14918335632 weight_to_token_ratio=0.158547 prefetch_share_of_ingress=0.089815 dedup_ratio=0.210133 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=37 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=26670 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.386712317e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=261980160 dispatch_meta_bytes=330840 combine_payload_bytes=261980160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262311000 ingress_bytes=296583168 link_bytes=296583168 token_bytes=524291160 weight_movement_bytes=34603008 total_nvlink_bytes=558894168 weight_to_token_ratio=0.066000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116018584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=261423104 dispatch_meta_bytes=326512 combine_payload_bytes=261423104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261749616 ingress_bytes=296026112 link_bytes=296026112 token_bytes=523172720 weight_movement_bytes=34603008 total_nvlink_bytes=557775728 weight_to_token_ratio=0.066141 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113922427e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=276283392 dispatch_meta_bytes=313720 combine_payload_bytes=276283392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276597112 ingress_bytes=310886400 link_bytes=310886400 token_bytes=552880504 weight_movement_bytes=34603008 total_nvlink_bytes=587483512 weight_to_token_ratio=0.062587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169840495e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70858 payload_rows_remote=55500 dup_rows=27446 dup_rows_remote=19304 dedup_ratio=0.279195 router_tokens_per_expert=[0, 0, 7453, 0, 0, 0, 0, 0, 0, 252, 0, 0, 7674, 0, 1390, 0, 19, 344, 61, 0, 0, 0, 314, 0, 528, 0, 0, 0, 0, 18, 0, 6703, 6459, 0, 372, 0, 0, 0, 199, 0, 1083, 2184, 0, 0, 0, 5504, 7063, 45, 0, 0, 0, 0, 252, 0, 98, 0, 0, 0, 1137, 0, 0, 0, 0, 0] payload_rows_per_dst=[15358, 15992, 15978, 13530, 774, 6252, 700, 2274] dup_rows_per_dst=[8142, 6964, 10632, 968, 368, 372, 0, 0] combine_rows_per_src=[15358, 15992, 15978, 13530, 774, 6252, 700, 2274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225067008 dispatch_meta_bytes=392520 combine_payload_bytes=225067008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=225459528 ingress_bytes=294273024 link_bytes=294273024 token_bytes=450526536 weight_movement_bytes=69206016 total_nvlink_bytes=519732552 weight_to_token_ratio=0.153611 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107325698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12784470784 prefetch_bytes=795869184 reduce_bytes=553648128 weight_movement_bytes=1349517312 egress_bytes=6396136192 ingress_bytes=7737851904 link_bytes=7737851904 total_nvlink_bytes=14133988096 weight_to_token_ratio=0.105559 prefetch_share_of_ingress=0.102854 dedup_ratio=0.194213 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=16 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.265952772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12693143504 prefetch_bytes=795869184 reduce_bytes=588251136 weight_movement_bytes=1384120320 egress_bytes=6350446544 ingress_bytes=7726817280 link_bytes=7726817280 total_nvlink_bytes=14077263824 weight_to_token_ratio=0.109045 prefetch_share_of_ingress=0.103001 dedup_ratio=0.205512 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=17 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.264147451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12520110648 prefetch_bytes=657457152 reduce_bytes=1107296256 weight_movement_bytes=1764753408 egress_bytes=6263904824 ingress_bytes=8020959232 link_bytes=8020959232 total_nvlink_bytes=14284864056 weight_to_token_ratio=0.140953 prefetch_share_of_ingress=0.081967 dedup_ratio=0.215457 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=32 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=27051 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.312270602e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=227328000 dispatch_meta_bytes=299216 combine_payload_bytes=227328000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=227627216 ingress_bytes=261931008 link_bytes=261931008 token_bytes=454955216 weight_movement_bytes=34603008 total_nvlink_bytes=489558224 weight_to_token_ratio=0.076058 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.856252959e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12358633400 prefetch_bytes=795869184 reduce_bytes=519045120 weight_movement_bytes=1314914304 egress_bytes=6183241656 ingress_bytes=7490306048 link_bytes=7490306048 total_nvlink_bytes=13673547704 weight_to_token_ratio=0.106396 prefetch_share_of_ingress=0.106253 dedup_ratio=0.225565 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=15 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.225452984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79176 payload_rows_remote=76554 dup_rows=19128 dup_rows_remote=18984 dedup_ratio=0.194580 router_tokens_per_expert=[0, 108, 0, 0, 0, 0, 308, 0, 89, 7495, 560, 0, 0, 89, 0, 7, 0, 0, 0, 61, 0, 180, 0, 0, 0, 0, 0, 0, 772, 7576, 0, 0, 0, 0, 0, 0, 514, 7119, 0, 8103, 90, 0, 0, 4053, 92, 0, 47, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 0, 0, 342, 7833, 0, 0, 0, 103] payload_rows_per_dst=[2622, 15128, 482, 15152, 16278, 8292, 7222, 14000] dup_rows_per_dst=[144, 1352, 0, 1544, 15194, 272, 0, 622] combine_rows_per_src=[2622, 15128, 482, 15152, 16278, 8292, 7222, 14000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313565184 dispatch_meta_bytes=382152 combine_payload_bytes=313565184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=313947336 ingress_bytes=417374208 link_bytes=417374208 token_bytes=627512520 weight_movement_bytes=103809024 total_nvlink_bytes=731321544 weight_to_token_ratio=0.165429 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.570545543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12069618896 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6038586576 ingress_bytes=7518961664 link_bytes=7518961664 total_nvlink_bytes=13557548240 weight_to_token_ratio=0.123279 prefetch_share_of_ingress=0.105848 dedup_ratio=0.239781 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.230141192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none + [2026-08-29 23:40:38] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 677.0 | throughput per GPU (TFLOP/s/GPU): 188.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.754238E+00 | loss scale: 1.0 | grad norm: 288.991 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75652 payload_rows_remote=65580 dup_rows=22652 dup_rows_remote=19614 dedup_ratio=0.230428 router_tokens_per_expert=[1279, 310, 785, 1020, 360, 831, 802, 1220, 1319, 115, 585, 421, 446, 776, 910, 1172, 733, 955, 714, 570, 317, 665, 276, 1305, 761, 741, 714, 302, 645, 53, 991, 1285, 829, 914, 1005, 667, 676, 255, 445, 969, 754, 747, 523, 961, 899, 871, 557, 699, 798, 569, 804, 440, 1067, 1196, 856, 825, 1454, 598, 1373, 849, 120, 778, 1205, 1071] payload_rows_per_dst=[9146, 9202, 8924, 9090, 9416, 9148, 10072, 10654] dup_rows_per_dst=[3364, 2990, 2146, 1894, 2104, 2874, 3038, 4242] combine_rows_per_src=[9146, 9202, 8924, 9090, 9416, 9148, 10072, 10654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74748 payload_rows_remote=64206 dup_rows=23556 dup_rows_remote=19620 dedup_ratio=0.239624 router_tokens_per_expert=[1339, 343, 880, 959, 336, 822, 748, 1111, 1321, 202, 605, 415, 476, 1028, 872, 1179, 653, 1003, 785, 528, 339, 669, 324, 1332, 625, 709, 734, 346, 648, 87, 878, 1187, 872, 983, 1174, 765, 671, 223, 455, 924, 663, 594, 529, 899, 866, 906, 544, 727, 743, 525, 802, 592, 981, 1310, 810, 872, 1476, 475, 1384, 728, 173, 800, 1181, 1022] payload_rows_per_dst=[7230, 9880, 9016, 8752, 9466, 9000, 10862, 10542] dup_rows_per_dst=[1798, 4994, 2250, 1676, 2668, 2456, 3778, 3936] combine_rows_per_src=[7230, 9880, 9016, 8752, 9466, 9000, 10862, 10542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75650 payload_rows_remote=66610 dup_rows=22654 dup_rows_remote=19966 dedup_ratio=0.230448 router_tokens_per_expert=[1272, 264, 783, 871, 324, 724, 788, 1343, 1304, 170, 536, 443, 524, 945, 913, 1082, 777, 949, 748, 495, 352, 676, 380, 1487, 667, 682, 784, 316, 698, 65, 996, 1348, 868, 991, 1208, 669, 493, 277, 389, 1088, 709, 668, 433, 952, 861, 852, 547, 842, 765, 545, 891, 544, 1029, 1078, 837, 782, 1433, 598, 1296, 732, 121, 731, 1110, 1107] payload_rows_per_dst=[9254, 9184, 9348, 9216, 9388, 9040, 9920, 10300] dup_rows_per_dst=[3484, 2650, 2380, 1896, 2578, 2688, 3022, 3956] combine_rows_per_src=[9254, 9184, 9348, 9216, 9388, 9040, 9920, 10300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76008 payload_rows_remote=66056 dup_rows=22296 dup_rows_remote=19056 dedup_ratio=0.226807 router_tokens_per_expert=[1277, 286, 791, 975, 340, 767, 832, 1157, 1288, 152, 545, 426, 444, 950, 957, 1127, 688, 1063, 752, 612, 293, 677, 318, 1358, 645, 687, 654, 354, 671, 89, 907, 1167, 814, 866, 1144, 713, 633, 205, 481, 1015, 680, 626, 560, 861, 948, 988, 593, 645, 816, 543, 876, 487, 1047, 1332, 850, 841, 1422, 610, 1404, 695, 176, 750, 1277, 1005] payload_rows_per_dst=[9524, 9156, 9210, 9952, 9446, 9152, 10206, 9362] dup_rows_per_dst=[3326, 2622, 2312, 3240, 2296, 2650, 3378, 2472] combine_rows_per_src=[9524, 9156, 9210, 9952, 9446, 9152, 10206, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76212 payload_rows_remote=66894 dup_rows=22092 dup_rows_remote=20012 dedup_ratio=0.224731 router_tokens_per_expert=[1299, 254, 724, 955, 316, 918, 848, 1139, 1257, 182, 552, 401, 483, 953, 992, 1200, 751, 924, 837, 659, 318, 602, 313, 1429, 686, 624, 727, 339, 734, 68, 825, 1194, 845, 754, 1057, 640, 662, 248, 456, 1037, 721, 607, 476, 949, 895, 929, 623, 788, 904, 592, 800, 465, 1001, 1302, 893, 910, 1402, 451, 1406, 725, 186, 796, 1196, 933] payload_rows_per_dst=[9688, 9292, 9232, 8894, 9318, 9086, 10402, 10300] dup_rows_per_dst=[3218, 2748, 2434, 1688, 2080, 2890, 3332, 3702] combine_rows_per_src=[9688, 9292, 9232, 8894, 9318, 9086, 10402, 10300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75386 payload_rows_remote=65706 dup_rows=22918 dup_rows_remote=19872 dedup_ratio=0.233134 router_tokens_per_expert=[1236, 298, 811, 885, 306, 696, 717, 1419, 1235, 116, 546, 378, 591, 802, 969, 1253, 638, 960, 785, 490, 378, 621, 345, 1556, 744, 682, 788, 276, 635, 59, 1070, 1166, 852, 1150, 1042, 613, 615, 195, 491, 1145, 720, 665, 483, 977, 831, 872, 656, 774, 694, 534, 838, 480, 1018, 1097, 883, 777, 1587, 668, 1353, 732, 85, 744, 1039, 1091] payload_rows_per_dst=[9304, 9142, 9680, 10302, 9616, 9148, 9146, 9048] dup_rows_per_dst=[3432, 2638, 3046, 3712, 2590, 2808, 2316, 2376] combine_rows_per_src=[9304, 9142, 9680, 10302, 9616, 9148, 9146, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75740 payload_rows_remote=66508 dup_rows=22564 dup_rows_remote=19844 dedup_ratio=0.229533 router_tokens_per_expert=[1341, 258, 796, 813, 254, 1098, 855, 839, 1218, 270, 668, 427, 321, 1123, 939, 1010, 664, 1014, 734, 632, 377, 536, 322, 1383, 585, 649, 861, 349, 809, 97, 854, 1339, 936, 873, 1233, 801, 419, 240, 566, 1058, 661, 565, 595, 777, 848, 908, 608, 553, 970, 683, 751, 658, 1028, 1197, 755, 855, 1468, 475, 1400, 711, 181, 758, 1261, 925] payload_rows_per_dst=[9438, 9232, 9876, 10560, 9772, 10454, 8736, 7672] dup_rows_per_dst=[3070, 2720, 3842, 3462, 3444, 2812, 1700, 1514] combine_rows_per_src=[9438, 9232, 9876, 10560, 9772, 10454, 8736, 7672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268615680 dispatch_meta_bytes=340776 combine_payload_bytes=268615680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268956456 ingress_bytes=372424704 link_bytes=372424704 token_bytes=537572136 weight_movement_bytes=103809024 total_nvlink_bytes=641381160 weight_to_token_ratio=0.193107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401404179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273997824 dispatch_meta_bytes=347624 combine_payload_bytes=273997824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274345448 ingress_bytes=308600832 link_bytes=308600832 token_bytes=548343272 weight_movement_bytes=34603008 total_nvlink_bytes=582946280 weight_to_token_ratio=0.063105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161240086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262987776 dispatch_meta_bytes=335304 combine_payload_bytes=262987776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263323080 ingress_bytes=366796800 link_bytes=366796800 token_bytes=526310856 weight_movement_bytes=103809024 total_nvlink_bytes=630119880 weight_to_token_ratio=0.197239 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380226829e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272834560 dispatch_meta_bytes=346304 combine_payload_bytes=272834560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273180864 ingress_bytes=307437568 link_bytes=307437568 token_bytes=546015424 weight_movement_bytes=34603008 total_nvlink_bytes=580618432 weight_to_token_ratio=0.063374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156862818e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270565376 dispatch_meta_bytes=340448 combine_payload_bytes=270565376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270905824 ingress_bytes=305168384 link_bytes=305168384 token_bytes=541471200 weight_movement_bytes=34603008 total_nvlink_bytes=576074208 weight_to_token_ratio=0.063906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148324061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269131776 dispatch_meta_bytes=342312 combine_payload_bytes=269131776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269474088 ingress_bytes=303734784 link_bytes=303734784 token_bytes=538605864 weight_movement_bytes=34603008 total_nvlink_bytes=573208872 weight_to_token_ratio=0.064246 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142929540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75054 payload_rows_remote=65350 dup_rows=23250 dup_rows_remote=19926 dedup_ratio=0.236511 router_tokens_per_expert=[1302, 263, 752, 876, 256, 1143, 826, 915, 1173, 277, 770, 362, 355, 1127, 890, 973, 738, 1046, 944, 619, 306, 627, 352, 1277, 564, 660, 788, 370, 738, 81, 913, 1418, 941, 896, 1181, 765, 466, 246, 558, 978, 604, 577, 628, 745, 876, 896, 600, 641, 823, 577, 709, 549, 1030, 1395, 790, 750, 1453, 410, 1482, 745, 203, 755, 1297, 885] payload_rows_per_dst=[9704, 9112, 10232, 10652, 10042, 11016, 7324, 6972] dup_rows_per_dst=[3324, 2742, 4376, 3318, 4080, 3082, 1072, 1256] combine_rows_per_src=[9704, 9112, 10232, 10652, 10042, 11016, 7324, 6972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272416768 dispatch_meta_bytes=345408 combine_payload_bytes=272416768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272762176 ingress_bytes=307019776 link_bytes=307019776 token_bytes=545178944 weight_movement_bytes=34603008 total_nvlink_bytes=579781952 weight_to_token_ratio=0.063471 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155290700e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76192 payload_rows_remote=65920 dup_rows=22112 dup_rows_remote=19050 dedup_ratio=0.224935 router_tokens_per_expert=[667, 624, 1042, 597, 746, 318, 415, 1065, 187, 1213, 1081, 415, 590, 1171, 1072, 1252, 1308, 1097, 990, 400, 1348, 89, 1391, 844, 478, 677, 630, 691, 108, 348, 283, 109, 389, 238, 900, 347, 1047, 2254, 1049, 993, 2112, 1388, 1569, 1781, 543, 496, 56, 164, 636, 1208, 675, 80, 181, 1402, 270, 1129, 775, 379, 806, 645, 1042, 598, 299, 455] payload_rows_per_dst=[8654, 10346, 8638, 9028, 8204, 10786, 10272, 10264] dup_rows_per_dst=[2294, 3616, 1342, 1844, 1722, 5716, 3062, 2516] combine_rows_per_src=[8654, 10346, 8638, 9028, 8204, 10786, 10272, 10264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267673600 dispatch_meta_bytes=341104 combine_payload_bytes=267673600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268014704 ingress_bytes=371482624 link_bytes=371482624 token_bytes=535688304 weight_movement_bytes=103809024 total_nvlink_bytes=639497328 weight_to_token_ratio=0.193786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397859207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76868 payload_rows_remote=65962 dup_rows=21436 dup_rows_remote=15572 dedup_ratio=0.218058 router_tokens_per_expert=[712, 683, 1053, 605, 724, 369, 512, 927, 180, 938, 1014, 344, 610, 1136, 986, 1279, 1287, 1078, 969, 334, 1331, 97, 1505, 934, 530, 711, 613, 691, 149, 296, 241, 109, 455, 316, 834, 397, 974, 2332, 1088, 1160, 2033, 1416, 1668, 1575, 638, 491, 66, 199, 593, 1304, 633, 100, 222, 1461, 266, 1204, 673, 375, 758, 641, 960, 592, 323, 458] payload_rows_per_dst=[8988, 9846, 9980, 8878, 8584, 10906, 9554, 10132] dup_rows_per_dst=[2182, 3128, 2080, 1868, 1864, 5864, 2012, 2438] combine_rows_per_src=[8988, 9846, 9980, 8878, 8584, 10906, 9554, 10132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76588 payload_rows_remote=66848 dup_rows=21716 dup_rows_remote=18980 dedup_ratio=0.220907 router_tokens_per_expert=[798, 698, 1122, 566, 780, 260, 364, 1145, 131, 1013, 1038, 408, 498, 1024, 1138, 1244, 1434, 1088, 913, 522, 1310, 139, 1389, 1101, 531, 736, 658, 770, 192, 447, 235, 86, 447, 346, 739, 311, 1036, 2053, 1034, 979, 2075, 1459, 1300, 1444, 560, 641, 68, 187, 658, 1265, 760, 50, 237, 1453, 241, 1091, 682, 371, 909, 687, 846, 651, 327, 467] payload_rows_per_dst=[9108, 9792, 10264, 9362, 9740, 8946, 9386, 9990] dup_rows_per_dst=[2358, 3196, 2750, 2098, 2736, 3786, 2124, 2668] combine_rows_per_src=[9108, 9792, 10264, 9362, 9740, 8946, 9386, 9990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76094 payload_rows_remote=65948 dup_rows=22210 dup_rows_remote=19432 dedup_ratio=0.225932 router_tokens_per_expert=[779, 713, 921, 471, 950, 273, 298, 1135, 132, 1069, 1185, 395, 603, 945, 876, 1413, 1192, 1188, 935, 367, 1349, 173, 1512, 1052, 616, 713, 674, 821, 110, 287, 251, 113, 378, 311, 662, 290, 1233, 1777, 1028, 1043, 2270, 1518, 1253, 1381, 488, 704, 91, 275, 733, 1242, 778, 41, 178, 1612, 252, 1153, 791, 354, 780, 634, 965, 649, 321, 456] payload_rows_per_dst=[8640, 10012, 8278, 9448, 8414, 10308, 10848, 10146] dup_rows_per_dst=[2440, 3224, 1050, 2262, 1962, 4666, 3828, 2778] combine_rows_per_src=[8640, 10012, 8278, 9448, 8414, 10308, 10848, 10146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76254 payload_rows_remote=67140 dup_rows=22050 dup_rows_remote=19932 dedup_ratio=0.224304 router_tokens_per_expert=[830, 685, 1101, 625, 739, 281, 290, 1160, 161, 1017, 997, 427, 554, 940, 1032, 1261, 1392, 1140, 922, 449, 1280, 128, 1453, 987, 553, 688, 631, 762, 170, 381, 276, 142, 475, 307, 767, 304, 1094, 2114, 1013, 1134, 2013, 1365, 1497, 1550, 506, 523, 77, 157, 651, 1364, 729, 96, 195, 1612, 235, 1031, 739, 403, 773, 657, 910, 597, 323, 487] payload_rows_per_dst=[8906, 9664, 9990, 9114, 10688, 8230, 9532, 10130] dup_rows_per_dst=[2516, 3114, 2606, 2118, 3728, 3120, 2294, 2554] combine_rows_per_src=[8906, 9664, 9990, 9114, 10688, 8230, 9532, 10130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77732 payload_rows_remote=67866 dup_rows=20572 dup_rows_remote=18302 dedup_ratio=0.209269 router_tokens_per_expert=[738, 630, 1091, 551, 656, 350, 461, 955, 179, 958, 886, 307, 566, 1134, 1130, 1248, 1397, 1091, 967, 353, 1342, 84, 1402, 834, 462, 648, 561, 555, 132, 323, 285, 113, 408, 288, 946, 381, 1122, 2786, 982, 1104, 2135, 1280, 1980, 1686, 561, 470, 64, 152, 639, 1211, 546, 129, 164, 1533, 282, 1101, 638, 312, 754, 810, 996, 576, 290, 437] payload_rows_per_dst=[9682, 8814, 9866, 10298, 11646, 7838, 9426, 10162] dup_rows_per_dst=[2634, 2550, 2270, 3020, 4388, 1658, 1784, 2268] combine_rows_per_src=[9682, 8814, 9866, 10298, 11646, 7838, 9426, 10162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270008320 dispatch_meta_bytes=339880 combine_payload_bytes=270008320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270348200 ingress_bytes=304611328 link_bytes=304611328 token_bytes=540356520 weight_movement_bytes=34603008 total_nvlink_bytes=574959528 weight_to_token_ratio=0.064037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146227904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273809408 dispatch_meta_bytes=343312 combine_payload_bytes=273809408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274152720 ingress_bytes=343015424 link_bytes=343015424 token_bytes=547962128 weight_movement_bytes=69206016 total_nvlink_bytes=617168144 weight_to_token_ratio=0.126297 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290739426e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77192 payload_rows_remote=69374 dup_rows=21112 dup_rows_remote=18726 dedup_ratio=0.214762 router_tokens_per_expert=[837, 953, 1086, 493, 1053, 278, 399, 1024, 125, 893, 1352, 399, 534, 954, 845, 1423, 1027, 1247, 915, 466, 1061, 222, 1621, 1253, 752, 733, 735, 883, 138, 435, 301, 148, 499, 340, 465, 143, 1018, 1627, 1054, 1099, 2114, 1547, 992, 1242, 626, 849, 71, 234, 848, 1077, 1057, 35, 214, 1373, 234, 1196, 787, 330, 646, 575, 935, 630, 336, 374] payload_rows_per_dst=[11170, 7818, 10536, 11124, 9960, 7354, 9784, 9446] dup_rows_per_dst=[3922, 2386, 2718, 3838, 2530, 1284, 2284, 2150] combine_rows_per_src=[11170, 7818, 10536, 11124, 9960, 7354, 9784, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270180352 dispatch_meta_bytes=326136 combine_payload_bytes=270180352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270506488 ingress_bytes=373989376 link_bytes=373989376 token_bytes=540686840 weight_movement_bytes=103809024 total_nvlink_bytes=644495864 weight_to_token_ratio=0.191995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407291913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270123008 dispatch_meta_bytes=341520 combine_payload_bytes=270123008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270464528 ingress_bytes=304726016 link_bytes=304726016 token_bytes=540587536 weight_movement_bytes=34603008 total_nvlink_bytes=575190544 weight_to_token_ratio=0.064010 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146659466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=348288 combine_payload_bytes=275005440 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275353728 ingress_bytes=344211456 link_bytes=344211456 token_bytes=550359168 weight_movement_bytes=69206016 total_nvlink_bytes=619565184 weight_to_token_ratio=0.125747 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295239998e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277979136 dispatch_meta_bytes=344672 combine_payload_bytes=277979136 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=278323808 ingress_bytes=416391168 link_bytes=416391168 token_bytes=556302944 weight_movement_bytes=138412032 total_nvlink_bytes=694714976 weight_to_token_ratio=0.248807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.566846443e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284155904 dispatch_meta_bytes=352400 combine_payload_bytes=284155904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284508304 ingress_bytes=353361920 link_bytes=353361920 token_bytes=568664208 weight_movement_bytes=69206016 total_nvlink_bytes=637870224 weight_to_token_ratio=0.121699 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329672457e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77272 payload_rows_remote=66190 dup_rows=21032 dup_rows_remote=17304 dedup_ratio=0.213949 router_tokens_per_expert=[861, 729, 851, 560, 1079, 248, 336, 1265, 168, 992, 1332, 423, 536, 997, 775, 1476, 1089, 1238, 950, 479, 1249, 240, 1554, 1345, 597, 850, 686, 827, 172, 324, 222, 106, 381, 283, 527, 260, 1166, 1623, 927, 902, 1961, 1321, 1042, 1257, 500, 780, 58, 220, 916, 1243, 1115, 28, 138, 1310, 258, 1049, 928, 377, 971, 649, 975, 639, 342, 450] payload_rows_per_dst=[11082, 8268, 12340, 10694, 9656, 6694, 9786, 8752] dup_rows_per_dst=[3728, 2530, 3596, 3310, 2482, 1148, 2328, 1910] combine_rows_per_src=[11082, 8268, 12340, 10694, 9656, 6694, 9786, 8752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=63790 dup_rows=23952 dup_rows_remote=19578 dedup_ratio=0.243652 router_tokens_per_expert=[896, 573, 466, 329, 374, 823, 903, 232, 854, 318, 741, 934, 1406, 1300, 1108, 1879, 859, 854, 157, 471, 360, 855, 1070, 745, 242, 694, 739, 326, 401, 573, 700, 685, 538, 984, 1086, 963, 995, 1369, 1969, 675, 300, 498, 344, 127, 1820, 1019, 176, 1516, 1037, 585, 1060, 831, 410, 672, 329, 1175, 1022, 417, 78, 530, 371, 755, 1041, 1593] payload_rows_per_dst=[9354, 7766, 9490, 8266, 9228, 9948, 10562, 9738] dup_rows_per_dst=[3540, 1844, 4064, 2242, 2538, 3474, 4374, 1876] combine_rows_per_src=[9354, 7766, 9490, 8266, 9228, 9948, 10562, 9738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64638 dup_rows=24364 dup_rows_remote=20162 dedup_ratio=0.247843 router_tokens_per_expert=[557, 613, 476, 373, 191, 1089, 1016, 296, 730, 336, 933, 924, 1495, 1051, 1039, 1755, 939, 1001, 254, 479, 112, 609, 1156, 937, 290, 633, 953, 602, 466, 515, 700, 562, 464, 1011, 1348, 805, 1187, 1375, 1891, 562, 369, 461, 328, 168, 1549, 1188, 210, 1548, 1216, 437, 1085, 783, 358, 662, 331, 927, 1244, 310, 93, 502, 413, 905, 736, 1604] payload_rows_per_dst=[9314, 9606, 9080, 9768, 9302, 8718, 8564, 9588] dup_rows_per_dst=[3418, 3224, 2080, 3456, 4202, 2924, 3034, 2026] combine_rows_per_src=[9314, 9606, 9080, 9768, 9302, 8718, 8564, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73346 payload_rows_remote=65084 dup_rows=24958 dup_rows_remote=22244 dedup_ratio=0.253886 router_tokens_per_expert=[876, 570, 426, 375, 413, 848, 821, 260, 842, 334, 897, 780, 1351, 1389, 1263, 1921, 867, 941, 205, 487, 141, 1004, 1004, 636, 207, 682, 699, 292, 417, 532, 749, 800, 455, 808, 1121, 1105, 899, 1404, 1964, 647, 263, 555, 351, 164, 1687, 1027, 174, 1267, 1265, 516, 1138, 780, 503, 867, 296, 1097, 1075, 372, 87, 522, 333, 877, 998, 1506] payload_rows_per_dst=[9184, 8778, 9484, 9292, 8116, 8262, 10548, 9682] dup_rows_per_dst=[3836, 2232, 3788, 3392, 2402, 2714, 4736, 1858] combine_rows_per_src=[9184, 8778, 9484, 9292, 8116, 8262, 10548, 9682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271114240 dispatch_meta_bytes=333976 combine_payload_bytes=271114240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271448216 ingress_bytes=305717248 link_bytes=305717248 token_bytes=542562456 weight_movement_bytes=34603008 total_nvlink_bytes=577165464 weight_to_token_ratio=0.063777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150389392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73792 payload_rows_remote=64464 dup_rows=24512 dup_rows_remote=21372 dedup_ratio=0.249349 router_tokens_per_expert=[842, 663, 435, 412, 409, 702, 1006, 264, 761, 286, 861, 1025, 1337, 1513, 1335, 1737, 893, 859, 213, 502, 214, 1011, 1130, 679, 196, 679, 745, 384, 492, 647, 670, 582, 518, 966, 904, 698, 1212, 1168, 1839, 582, 352, 509, 322, 174, 1603, 1081, 183, 1486, 1314, 542, 931, 643, 507, 734, 401, 1028, 1100, 298, 102, 486, 516, 762, 1160, 1547] payload_rows_per_dst=[9308, 10352, 8834, 9328, 8630, 8552, 8932, 9856] dup_rows_per_dst=[3632, 3884, 2168, 3140, 3466, 2868, 3268, 2086] combine_rows_per_src=[9308, 10352, 8834, 9328, 8630, 8552, 8932, 9856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73834 payload_rows_remote=63152 dup_rows=24470 dup_rows_remote=20788 dedup_ratio=0.248922 router_tokens_per_expert=[314, 868, 535, 381, 168, 1092, 924, 244, 855, 335, 950, 969, 1584, 1129, 806, 1789, 951, 1238, 290, 425, 368, 382, 1339, 930, 355, 698, 945, 582, 493, 512, 812, 471, 409, 990, 1197, 684, 1625, 1509, 1594, 542, 486, 302, 332, 267, 1594, 1197, 204, 1466, 1124, 301, 780, 923, 254, 402, 264, 835, 1325, 376, 127, 637, 187, 1185, 526, 1774] payload_rows_per_dst=[7250, 8808, 10256, 8034, 8766, 10480, 9558, 10682] dup_rows_per_dst=[1802, 2600, 4758, 1702, 2234, 4466, 3226, 3682] combine_rows_per_src=[7250, 8808, 10256, 8034, 8766, 10480, 9558, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73966 payload_rows_remote=64996 dup_rows=24338 dup_rows_remote=22548 dedup_ratio=0.247579 router_tokens_per_expert=[1270, 437, 374, 352, 409, 770, 841, 215, 871, 287, 809, 731, 1257, 1719, 1240, 2164, 891, 859, 158, 580, 158, 1311, 863, 560, 138, 667, 634, 245, 437, 553, 614, 799, 494, 643, 1037, 1012, 657, 1202, 2007, 594, 206, 517, 304, 128, 1863, 1011, 177, 1467, 1030, 703, 1092, 645, 675, 916, 316, 1324, 1025, 335, 59, 539, 446, 720, 1229, 1566] payload_rows_per_dst=[9564, 10170, 8970, 8966, 8232, 8722, 9452, 9890] dup_rows_per_dst=[4100, 3658, 1790, 3222, 3046, 2624, 3950, 1948] combine_rows_per_src=[9564, 10170, 8970, 8966, 8232, 8722, 9452, 9890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261283840 dispatch_meta_bytes=333472 combine_payload_bytes=261283840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261617312 ingress_bytes=295886848 link_bytes=295886848 token_bytes=522901152 weight_movement_bytes=34603008 total_nvlink_bytes=557504160 weight_to_token_ratio=0.066175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113398388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=339200 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265096448 ingress_bytes=403169280 link_bytes=403169280 token_bytes=529853696 weight_movement_bytes=138412032 total_nvlink_bytes=668265728 weight_to_token_ratio=0.261227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517093543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64412 dup_rows=24118 dup_rows_remote=20924 dedup_ratio=0.245341 router_tokens_per_expert=[196, 994, 427, 433, 68, 1197, 1042, 204, 662, 339, 1165, 1218, 1350, 1169, 581, 1458, 948, 1239, 302, 474, 331, 288, 1481, 964, 309, 768, 1295, 793, 487, 548, 1013, 298, 329, 885, 937, 444, 2119, 1413, 1552, 559, 699, 313, 372, 468, 1474, 1311, 230, 1572, 1268, 272, 425, 792, 93, 231, 343, 713, 1118, 268, 102, 741, 210, 1529, 417, 1912] payload_rows_per_dst=[8894, 9774, 9492, 10262, 9508, 9416, 6830, 10010] dup_rows_per_dst=[3144, 3194, 2562, 3864, 3864, 3462, 1444, 2584] combine_rows_per_src=[8894, 9774, 9492, 10262, 9508, 9416, 6830, 10010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=349312 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266933376 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533517440 weight_movement_bytes=34603008 total_nvlink_bytes=568120448 weight_to_token_ratio=0.064858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=343344 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264387888 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528432432 weight_movement_bytes=34603008 total_nvlink_bytes=563035440 weight_to_token_ratio=0.065482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258670592 dispatch_meta_bytes=335760 combine_payload_bytes=258670592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259006352 ingress_bytes=293273600 link_bytes=293273600 token_bytes=517676944 weight_movement_bytes=34603008 total_nvlink_bytes=552279952 weight_to_token_ratio=0.066843 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103564946e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=350176 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266573792 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532797408 weight_movement_bytes=34603008 total_nvlink_bytes=567400416 weight_to_token_ratio=0.064946 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263831552 dispatch_meta_bytes=341344 combine_payload_bytes=263831552 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264172896 ingress_bytes=402243584 link_bytes=402243584 token_bytes=528004448 weight_movement_bytes=138412032 total_nvlink_bytes=666416480 weight_to_token_ratio=0.262142 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513610223e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73856 payload_rows_remote=65074 dup_rows=24448 dup_rows_remote=21266 dedup_ratio=0.248698 router_tokens_per_expert=[181, 891, 475, 492, 121, 1162, 938, 321, 735, 316, 981, 1110, 1640, 990, 435, 1401, 980, 1271, 249, 417, 358, 338, 1327, 804, 453, 823, 1043, 739, 427, 471, 795, 314, 450, 1069, 1096, 640, 1956, 1625, 1620, 549, 510, 272, 372, 242, 1573, 1198, 181, 1489, 1282, 254, 579, 975, 269, 431, 291, 832, 1258, 441, 136, 518, 168, 1337, 482, 2059] payload_rows_per_dst=[8782, 9410, 9074, 9932, 10098, 8744, 7890, 9926] dup_rows_per_dst=[3182, 3004, 2414, 3438, 4672, 2930, 1936, 2872] combine_rows_per_src=[8782, 9410, 9074, 9932, 10098, 8744, 7890, 9926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75054 payload_rows_remote=65294 dup_rows=23250 dup_rows_remote=19228 dedup_ratio=0.236511 router_tokens_per_expert=[1374, 1874, 1020, 791, 420, 280, 792, 577, 925, 1455, 358, 469, 888, 526, 538, 551, 1080, 76, 1785, 705, 434, 236, 214, 822, 428, 459, 686, 696, 509, 487, 988, 1096, 332, 985, 492, 416, 440, 535, 1725, 564, 1051, 1667, 216, 613, 453, 587, 1312, 2284, 1259, 511, 902, 772, 476, 973, 831, 1167, 231, 648, 938, 785, 866, 594, 670, 318] payload_rows_per_dst=[9092, 9332, 8074, 11176, 8400, 9236, 9760, 9984] dup_rows_per_dst=[1416, 2088, 2630, 3270, 2578, 3488, 4022, 3758] combine_rows_per_src=[9092, 9332, 8074, 11176, 8400, 9236, 9760, 9984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74826 payload_rows_remote=66614 dup_rows=23478 dup_rows_remote=20948 dedup_ratio=0.238831 router_tokens_per_expert=[1036, 1248, 1413, 677, 625, 436, 729, 799, 1017, 1905, 550, 438, 1088, 619, 730, 280, 1238, 61, 1619, 741, 566, 263, 106, 875, 433, 632, 659, 477, 578, 598, 1101, 933, 361, 822, 451, 376, 363, 492, 1844, 662, 850, 2386, 220, 535, 407, 574, 865, 2417, 636, 562, 747, 808, 397, 832, 657, 895, 151, 638, 1056, 753, 819, 841, 701, 564] payload_rows_per_dst=[10366, 10430, 8326, 9172, 8212, 11134, 8388, 8798] dup_rows_per_dst=[3560, 2824, 2612, 1650, 2530, 5374, 2680, 2248] combine_rows_per_src=[10366, 10430, 8326, 9172, 8212, 11134, 8388, 8798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64500 dup_rows=24306 dup_rows_remote=22610 dedup_ratio=0.247253 router_tokens_per_expert=[739, 1880, 906, 985, 365, 229, 684, 527, 965, 1266, 224, 398, 568, 746, 613, 431, 1105, 116, 1409, 657, 328, 199, 168, 743, 420, 320, 710, 516, 473, 599, 1058, 1501, 160, 824, 549, 494, 459, 499, 1921, 449, 785, 1349, 265, 749, 408, 608, 1925, 1906, 1821, 584, 908, 687, 721, 890, 1031, 1565, 297, 685, 811, 966, 796, 635, 834, 723] payload_rows_per_dst=[9784, 8840, 7264, 9498, 8350, 10760, 10814, 8688] dup_rows_per_dst=[2846, 1582, 2186, 1696, 2360, 5230, 5600, 2806] combine_rows_per_src=[9784, 8840, 7264, 9498, 8350, 10760, 10814, 8688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345360 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888464 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431568 weight_movement_bytes=34603008 total_nvlink_bytes=568034576 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74814 payload_rows_remote=63656 dup_rows=23490 dup_rows_remote=18106 dedup_ratio=0.238953 router_tokens_per_expert=[1341, 1933, 1112, 842, 443, 202, 889, 562, 771, 1063, 287, 424, 811, 614, 499, 616, 1077, 130, 1744, 708, 489, 237, 168, 803, 288, 427, 679, 589, 519, 557, 1010, 1102, 213, 799, 544, 441, 412, 636, 2165, 516, 803, 1477, 190, 758, 408, 620, 1461, 2554, 1516, 563, 799, 747, 500, 1159, 703, 1207, 299, 556, 917, 810, 949, 520, 723, 251] payload_rows_per_dst=[9344, 8402, 8146, 10950, 8564, 11158, 10020, 8230] dup_rows_per_dst=[1688, 1768, 2566, 3008, 2888, 5384, 4368, 1820] combine_rows_per_src=[9344, 8402, 8146, 10950, 8564, 11158, 10020, 8230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267444224 dispatch_meta_bytes=338088 combine_payload_bytes=267444224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267782312 ingress_bytes=302047232 link_bytes=302047232 token_bytes=535226536 weight_movement_bytes=34603008 total_nvlink_bytes=569829544 weight_to_token_ratio=0.064651 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136579417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=66366 dup_rows=23940 dup_rows_remote=21376 dedup_ratio=0.243530 router_tokens_per_expert=[1608, 2111, 1075, 837, 379, 220, 834, 466, 643, 928, 259, 363, 680, 765, 427, 576, 877, 100, 1647, 707, 460, 150, 242, 824, 259, 376, 669, 562, 423, 398, 855, 1167, 200, 947, 480, 430, 374, 600, 1890, 564, 1020, 1504, 155, 760, 391, 614, 2134, 2102, 2105, 585, 878, 669, 473, 888, 996, 1500, 245, 775, 937, 826, 744, 461, 781, 237] payload_rows_per_dst=[11454, 7940, 7998, 8146, 9444, 10310, 10942, 8130] dup_rows_per_dst=[3606, 1342, 2564, 1272, 3310, 4718, 5246, 1882] combine_rows_per_src=[11454, 7940, 7998, 8146, 9444, 10310, 10942, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272850944 dispatch_meta_bytes=350248 combine_payload_bytes=272850944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273201192 ingress_bytes=307453952 link_bytes=307453952 token_bytes=546052136 weight_movement_bytes=34603008 total_nvlink_bytes=580655144 weight_to_token_ratio=0.063369 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156924469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=62754 dup_rows=24218 dup_rows_remote=18516 dedup_ratio=0.246358 router_tokens_per_expert=[621, 1016, 1234, 572, 500, 492, 803, 634, 1387, 2602, 457, 449, 1358, 532, 868, 406, 1587, 84, 2031, 601, 572, 415, 41, 707, 659, 605, 622, 602, 570, 714, 1107, 882, 270, 790, 312, 341, 498, 678, 1845, 664, 495, 2122, 136, 532, 349, 489, 614, 2612, 269, 471, 548, 959, 533, 816, 468, 706, 305, 634, 1223, 639, 966, 777, 630, 731] payload_rows_per_dst=[8092, 10350, 8936, 10622, 8256, 8916, 7582, 11332] dup_rows_per_dst=[1620, 3042, 3140, 2932, 2540, 3284, 1958, 5702] combine_rows_per_src=[8092, 10350, 8936, 10622, 8256, 8916, 7582, 11332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=62502 dup_rows=24106 dup_rows_remote=20414 dedup_ratio=0.245219 router_tokens_per_expert=[413, 717, 1453, 638, 685, 591, 791, 723, 1408, 2588, 356, 409, 918, 636, 982, 397, 1788, 88, 2082, 505, 539, 386, 25, 636, 658, 713, 621, 437, 673, 931, 1449, 1010, 399, 665, 267, 378, 768, 612, 2049, 676, 198, 2024, 166, 470, 374, 403, 476, 2537, 125, 322, 361, 951, 597, 890, 326, 519, 505, 678, 1268, 675, 986, 900, 636, 705] payload_rows_per_dst=[9280, 11696, 11128, 10466, 11164, 3930, 6690, 9844] dup_rows_per_dst=[2742, 3692, 6044, 2518, 4512, 244, 1492, 2862] combine_rows_per_src=[9280, 11696, 11128, 10466, 11164, 3930, 6690, 9844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264192000 dispatch_meta_bytes=348440 combine_payload_bytes=264192000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264540440 ingress_bytes=298795008 link_bytes=298795008 token_bytes=528732440 weight_movement_bytes=34603008 total_nvlink_bytes=563335448 weight_to_token_ratio=0.065445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124341560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=260734976 dispatch_meta_bytes=327048 combine_payload_bytes=260734976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261062024 ingress_bytes=399147008 link_bytes=399147008 token_bytes=521797000 weight_movement_bytes=138412032 total_nvlink_bytes=660209032 weight_to_token_ratio=0.265260 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501958057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271835136 dispatch_meta_bytes=350968 combine_payload_bytes=271835136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272186104 ingress_bytes=306438144 link_bytes=306438144 token_bytes=544021240 weight_movement_bytes=34603008 total_nvlink_bytes=578624248 weight_to_token_ratio=0.063606 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153102066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=257040384 dispatch_meta_bytes=325080 combine_payload_bytes=257040384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257365464 ingress_bytes=291643392 link_bytes=291643392 token_bytes=514405848 weight_movement_bytes=34603008 total_nvlink_bytes=549008856 weight_to_token_ratio=0.067268 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097430605e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=256008192 dispatch_meta_bytes=331664 combine_payload_bytes=256008192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256339856 ingress_bytes=359817216 link_bytes=359817216 token_bytes=512348048 weight_movement_bytes=103809024 total_nvlink_bytes=616157072 weight_to_token_ratio=0.202614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353963216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=64482 dup_rows=23788 dup_rows_remote=20408 dedup_ratio=0.241984 router_tokens_per_expert=[609, 974, 1424, 550, 595, 582, 798, 649, 1539, 2537, 384, 453, 1055, 566, 854, 434, 1555, 98, 2119, 652, 574, 346, 18, 708, 550, 593, 633, 571, 518, 864, 1302, 933, 305, 868, 435, 381, 579, 626, 1693, 598, 460, 2113, 217, 451, 303, 578, 560, 2275, 101, 524, 550, 1001, 546, 842, 470, 529, 436, 515, 1263, 673, 851, 911, 711, 750] payload_rows_per_dst=[10034, 11250, 10924, 9816, 10976, 4794, 7200, 9522] dup_rows_per_dst=[3380, 3726, 5766, 2112, 4220, 344, 1542, 2698] combine_rows_per_src=[10034, 11250, 10924, 9816, 10976, 4794, 7200, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71760 payload_rows_remote=63686 dup_rows=26544 dup_rows_remote=23750 dedup_ratio=0.270020 router_tokens_per_expert=[531, 1420, 1676, 176, 742, 492, 1480, 414, 549, 454, 773, 342, 629, 468, 1204, 1213, 1366, 460, 613, 177, 205, 187, 176, 221, 502, 474, 317, 719, 508, 588, 590, 1486, 863, 1462, 992, 285, 1023, 1439, 466, 917, 329, 426, 1515, 798, 1047, 658, 1347, 1260, 1106, 313, 178, 2143, 741, 675, 452, 706, 977, 295, 477, 543, 762, 2045, 558, 1202] payload_rows_per_dst=[6266, 8386, 9450, 8110, 10020, 10972, 8074, 10482] dup_rows_per_dst=[1544, 2878, 3672, 2258, 4874, 3788, 2794, 4736] combine_rows_per_src=[6266, 8386, 9450, 8110, 10020, 10972, 8074, 10482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72734 payload_rows_remote=63414 dup_rows=25570 dup_rows_remote=21928 dedup_ratio=0.260111 router_tokens_per_expert=[85, 1027, 1739, 670, 1014, 552, 1132, 643, 438, 395, 1671, 193, 452, 562, 841, 1324, 933, 320, 310, 201, 309, 345, 301, 339, 508, 400, 1086, 567, 1207, 919, 543, 452, 978, 423, 477, 861, 1234, 1321, 339, 1473, 304, 209, 656, 827, 1000, 78, 1726, 230, 785, 407, 274, 2252, 660, 915, 591, 1062, 996, 392, 434, 851, 1085, 2255, 1354, 1225] payload_rows_per_dst=[9568, 9240, 8816, 8700, 9320, 8188, 10090, 8812] dup_rows_per_dst=[2924, 3762, 3042, 2664, 3642, 1872, 3802, 3862] combine_rows_per_src=[9568, 9240, 8816, 8700, 9320, 8188, 10090, 8812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264118272 dispatch_meta_bytes=339560 combine_payload_bytes=264118272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264457832 ingress_bytes=333324288 link_bytes=333324288 token_bytes=528576104 weight_movement_bytes=69206016 total_nvlink_bytes=597782120 weight_to_token_ratio=0.130929 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254272461e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260857856 dispatch_meta_bytes=349744 combine_payload_bytes=260857856 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=261207600 ingress_bytes=364666880 link_bytes=364666880 token_bytes=522065456 weight_movement_bytes=103809024 total_nvlink_bytes=625874480 weight_to_token_ratio=0.198843 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372212111e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72448 payload_rows_remote=63750 dup_rows=25856 dup_rows_remote=23772 dedup_ratio=0.263021 router_tokens_per_expert=[1122, 1455, 1581, 52, 557, 981, 1688, 213, 411, 215, 473, 334, 719, 257, 1358, 792, 1062, 378, 1565, 317, 139, 221, 170, 172, 199, 723, 275, 673, 406, 599, 408, 2108, 1032, 2121, 964, 165, 622, 1569, 392, 1001, 387, 395, 2279, 771, 712, 1469, 1066, 2731, 1307, 66, 177, 1643, 765, 651, 478, 526, 508, 151, 381, 246, 458, 1444, 293, 759] payload_rows_per_dst=[9362, 8796, 10318, 8698, 9162, 13174, 8500, 4438] dup_rows_per_dst=[2286, 3460, 4268, 2084, 3432, 6446, 2726, 1154] combine_rows_per_src=[9362, 8796, 10318, 8698, 9162, 13174, 8500, 4438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259743744 dispatch_meta_bytes=341368 combine_payload_bytes=259743744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260085112 ingress_bytes=328949760 link_bytes=328949760 token_bytes=519828856 weight_movement_bytes=69206016 total_nvlink_bytes=589034872 weight_to_token_ratio=0.133132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237811464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72830 payload_rows_remote=62650 dup_rows=25474 dup_rows_remote=21054 dedup_ratio=0.259135 router_tokens_per_expert=[1065, 1218, 1630, 151, 606, 368, 1785, 246, 268, 284, 556, 251, 886, 337, 1367, 1080, 1248, 746, 1046, 205, 164, 154, 113, 209, 410, 564, 227, 696, 361, 359, 450, 2086, 1229, 1914, 1538, 124, 580, 1598, 334, 794, 304, 180, 1634, 654, 971, 1224, 1223, 2465, 1656, 132, 143, 1975, 790, 596, 483, 627, 703, 120, 366, 276, 488, 1637, 359, 899] payload_rows_per_dst=[8628, 9566, 10180, 8380, 9256, 12424, 9444, 4952] dup_rows_per_dst=[1954, 3688, 4420, 1926, 3770, 4886, 3360, 1470] combine_rows_per_src=[8628, 9566, 10180, 8380, 9256, 12424, 9444, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72852 payload_rows_remote=61738 dup_rows=25452 dup_rows_remote=21808 dedup_ratio=0.258911 router_tokens_per_expert=[765, 1376, 1653, 240, 675, 818, 1743, 382, 404, 493, 664, 379, 898, 358, 1120, 1436, 1242, 474, 758, 204, 159, 156, 155, 257, 403, 529, 286, 740, 487, 573, 661, 1447, 887, 1629, 1192, 140, 835, 1401, 494, 893, 375, 222, 1396, 715, 902, 1110, 1173, 1486, 1330, 238, 172, 2112, 674, 696, 595, 686, 787, 238, 369, 433, 658, 2015, 422, 942] payload_rows_per_dst=[9684, 8596, 9924, 8136, 10192, 11114, 9256, 5950] dup_rows_per_dst=[2858, 2908, 3678, 2116, 4750, 3644, 3750, 1748] combine_rows_per_src=[9684, 8596, 9924, 8136, 10192, 11114, 9256, 5950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71152 payload_rows_remote=58002 dup_rows=27152 dup_rows_remote=17240 dedup_ratio=0.276204 router_tokens_per_expert=[20, 1001, 1663, 646, 1102, 1064, 915, 931, 593, 461, 2177, 188, 320, 424, 601, 1255, 1257, 89, 159, 125, 230, 333, 332, 329, 436, 381, 1452, 573, 1681, 1291, 757, 125, 530, 71, 159, 1047, 1317, 830, 180, 1082, 288, 175, 360, 893, 791, 44, 1852, 21, 481, 583, 260, 2026, 562, 984, 830, 1370, 883, 567, 339, 1044, 1100, 2392, 1723, 1457] payload_rows_per_dst=[7042, 9030, 8422, 9812, 8308, 7508, 7880, 13150] dup_rows_per_dst=[2486, 3008, 2442, 3580, 2124, 1340, 2260, 9912] combine_rows_per_src=[7042, 9030, 8422, 9812, 8308, 7508, 7880, 13150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261120000 dispatch_meta_bytes=350088 combine_payload_bytes=261120000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261470088 ingress_bytes=295723008 link_bytes=295723008 token_bytes=522590088 weight_movement_bytes=34603008 total_nvlink_bytes=557193096 weight_to_token_ratio=0.066214 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.112781871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71842 payload_rows_remote=63222 dup_rows=26462 dup_rows_remote=22932 dedup_ratio=0.269185 router_tokens_per_expert=[5, 1177, 1520, 493, 1214, 1487, 786, 1236, 451, 303, 1810, 207, 216, 337, 384, 1481, 1348, 58, 240, 111, 285, 380, 253, 285, 335, 359, 1901, 516, 1969, 1191, 730, 91, 532, 19, 102, 1093, 1205, 886, 160, 1349, 376, 251, 366, 987, 676, 19, 1832, 11, 359, 292, 342, 2143, 469, 1017, 901, 1449, 715, 568, 255, 1160, 868, 2390, 2046, 1155] payload_rows_per_dst=[11372, 8620, 7982, 11034, 7280, 6092, 10146, 9316] dup_rows_per_dst=[5564, 3530, 1618, 5062, 1640, 1032, 3798, 4218] combine_rows_per_src=[11372, 8620, 7982, 11034, 7280, 6092, 10146, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256614400 dispatch_meta_bytes=334816 combine_payload_bytes=256614400 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256949216 ingress_bytes=325820416 link_bytes=325820416 token_bytes=513563616 weight_movement_bytes=69206016 total_nvlink_bytes=582769632 weight_to_token_ratio=0.134756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226035995e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=252878848 dispatch_meta_bytes=334184 combine_payload_bytes=252878848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253213032 ingress_bytes=322084864 link_bytes=322084864 token_bytes=506091880 weight_movement_bytes=69206016 total_nvlink_bytes=575297896 weight_to_token_ratio=0.136746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211979413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=237576192 dispatch_meta_bytes=300968 combine_payload_bytes=237576192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=237877160 ingress_bytes=306782208 link_bytes=306782208 token_bytes=475453352 weight_movement_bytes=69206016 total_nvlink_bytes=544659368 weight_to_token_ratio=0.145558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154396751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71738 payload_rows_remote=60198 dup_rows=26566 dup_rows_remote=21340 dedup_ratio=0.270243 router_tokens_per_expert=[18, 1159, 1761, 512, 1033, 1257, 732, 1217, 790, 591, 1836, 180, 242, 392, 481, 1052, 1476, 119, 214, 112, 204, 437, 370, 343, 530, 347, 1079, 565, 1554, 1132, 657, 126, 644, 72, 53, 1290, 1159, 854, 163, 1295, 305, 339, 375, 926, 751, 7, 1882, 53, 429, 574, 270, 2022, 521, 1219, 785, 1139, 844, 637, 342, 1116, 922, 2455, 1691, 1500] payload_rows_per_dst=[11540, 8838, 8600, 10930, 8054, 5210, 9056, 9510] dup_rows_per_dst=[5226, 3998, 1472, 4814, 2480, 614, 3368, 4594] combine_rows_per_src=[11540, 8838, 8600, 10930, 8054, 5210, 9056, 9510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76774 payload_rows_remote=66870 dup_rows=21530 dup_rows_remote=18694 dedup_ratio=0.219014 router_tokens_per_expert=[723, 744, 1540, 130, 500, 326, 1028, 179, 486, 592, 1092, 1129, 503, 23, 435, 1614, 768, 744, 581, 118, 645, 171, 237, 1728, 987, 729, 690, 424, 916, 579, 2201, 298, 545, 1121, 983, 2625, 1024, 1269, 963, 484, 400, 1176, 510, 1300, 318, 282, 520, 684, 874, 1092, 583, 246, 486, 1155, 509, 1425, 278, 721, 526, 1372, 279, 411, 1843, 288] payload_rows_per_dst=[8358, 8948, 8822, 10732, 8668, 11214, 9904, 10128] dup_rows_per_dst=[1982, 2800, 1162, 2916, 2184, 4416, 2836, 3234] combine_rows_per_src=[8358, 8948, 8822, 10732, 8668, 11214, 9904, 10128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258957312 dispatch_meta_bytes=344616 combine_payload_bytes=258957312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259301928 ingress_bytes=293560320 link_bytes=293560320 token_bytes=518259240 weight_movement_bytes=34603008 total_nvlink_bytes=552862248 weight_to_token_ratio=0.066768 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104643851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75906 payload_rows_remote=67748 dup_rows=22398 dup_rows_remote=20618 dedup_ratio=0.227844 router_tokens_per_expert=[654, 1086, 1243, 243, 734, 95, 2724, 328, 213, 62, 1193, 1018, 793, 1, 154, 1508, 848, 909, 1859, 126, 425, 446, 143, 70, 485, 1425, 721, 267, 625, 499, 2923, 597, 904, 661, 424, 1464, 1107, 429, 1148, 1444, 389, 73, 533, 1626, 177, 672, 1054, 543, 821, 965, 850, 932, 309, 924, 5, 1485, 863, 540, 433, 842, 347, 471, 1539, 761] payload_rows_per_dst=[10362, 8068, 8086, 11698, 8158, 9654, 9698, 10182] dup_rows_per_dst=[3852, 1816, 1566, 3386, 1780, 3408, 2884, 3706] combine_rows_per_src=[10362, 8068, 8086, 11698, 8158, 9654, 9698, 10182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=246571008 dispatch_meta_bytes=326152 combine_payload_bytes=246571008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=246897160 ingress_bytes=350380032 link_bytes=350380032 token_bytes=493468168 weight_movement_bytes=103809024 total_nvlink_bytes=597277192 weight_to_token_ratio=0.210366 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318451853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77974 payload_rows_remote=68760 dup_rows=20330 dup_rows_remote=18586 dedup_ratio=0.206807 router_tokens_per_expert=[772, 323, 1293, 62, 348, 323, 172, 215, 631, 2039, 842, 1022, 268, 72, 973, 1296, 520, 594, 259, 100, 609, 112, 289, 3131, 1373, 296, 589, 1197, 531, 766, 382, 345, 397, 1243, 1099, 2863, 472, 2655, 861, 343, 271, 2453, 234, 833, 557, 187, 294, 743, 740, 657, 365, 223, 544, 661, 2759, 1671, 41, 694, 714, 784, 96, 212, 1565, 177] payload_rows_per_dst=[6678, 10584, 10104, 9214, 11080, 10018, 12078, 8218] dup_rows_per_dst=[1444, 3702, 1124, 1744, 3956, 3128, 3162, 2070] combine_rows_per_src=[6678, 10584, 10104, 9214, 11080, 10018, 12078, 8218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273899520 dispatch_meta_bytes=342256 combine_payload_bytes=273899520 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274241776 ingress_bytes=377708544 link_bytes=377708544 token_bytes=548141296 weight_movement_bytes=103809024 total_nvlink_bytes=651950320 weight_to_token_ratio=0.189384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421286843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277495808 dispatch_meta_bytes=353464 combine_payload_bytes=277495808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=277849272 ingress_bytes=415907840 link_bytes=415907840 token_bytes=555345080 weight_movement_bytes=138412032 total_nvlink_bytes=693757112 weight_to_token_ratio=0.249236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565027719e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78496 payload_rows_remote=69016 dup_rows=19808 dup_rows_remote=18278 dedup_ratio=0.201497 router_tokens_per_expert=[557, 416, 1330, 78, 359, 210, 478, 123, 579, 1361, 1022, 1036, 335, 58, 732, 1092, 797, 616, 446, 183, 788, 108, 135, 2072, 1666, 396, 609, 850, 721, 623, 1372, 328, 426, 981, 1212, 3001, 936, 1961, 717, 372, 567, 2177, 450, 1065, 366, 193, 393, 693, 585, 929, 496, 176, 549, 1198, 1916, 1074, 209, 642, 539, 1282, 149, 241, 2024, 157] payload_rows_per_dst=[9050, 10222, 9480, 10116, 10652, 9338, 10286, 9352] dup_rows_per_dst=[1974, 3588, 1530, 2294, 3204, 2470, 2180, 2568] combine_rows_per_src=[9050, 10222, 9480, 10116, 10652, 9338, 10286, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77374 payload_rows_remote=65336 dup_rows=20930 dup_rows_remote=15986 dedup_ratio=0.212911 router_tokens_per_expert=[680, 515, 1514, 95, 497, 323, 590, 131, 736, 996, 923, 1138, 406, 43, 441, 1432, 646, 639, 488, 164, 649, 104, 298, 2361, 1128, 463, 600, 628, 675, 608, 1382, 306, 691, 1306, 1283, 3026, 891, 1424, 1085, 512, 428, 1617, 324, 1362, 381, 240, 341, 772, 829, 1079, 522, 191, 554, 829, 992, 1493, 279, 714, 585, 1193, 221, 245, 1948, 196] payload_rows_per_dst=[7176, 9212, 9520, 9766, 9368, 12038, 10320, 9974] dup_rows_per_dst=[1514, 3018, 1178, 1814, 2846, 4944, 2658, 2958] combine_rows_per_src=[7176, 9212, 9520, 9766, 9368, 12038, 10320, 9974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76422 payload_rows_remote=65814 dup_rows=21882 dup_rows_remote=18430 dedup_ratio=0.222595 router_tokens_per_expert=[552, 1410, 1163, 251, 302, 206, 2469, 466, 50, 2, 1220, 993, 1507, 0, 49, 1345, 746, 1095, 2451, 64, 408, 654, 160, 10, 193, 1764, 850, 133, 449, 241, 3478, 667, 1079, 552, 265, 1419, 927, 307, 1300, 1120, 188, 4, 231, 1605, 93, 673, 1375, 436, 857, 1170, 1363, 819, 225, 728, 0, 1338, 1110, 459, 469, 947, 302, 558, 1137, 748] payload_rows_per_dst=[10190, 8558, 8898, 11778, 7402, 8656, 10332, 10608] dup_rows_per_dst=[3448, 1774, 2278, 3772, 1098, 2646, 3414, 3452] combine_rows_per_src=[10190, 8558, 8898, 11778, 7402, 8656, 10332, 10608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281640960 dispatch_meta_bytes=349384 combine_payload_bytes=281640960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281990344 ingress_bytes=350846976 link_bytes=350846976 token_bytes=563631304 weight_movement_bytes=69206016 total_nvlink_bytes=632837320 weight_to_token_ratio=0.122786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320208925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76558 payload_rows_remote=66230 dup_rows=21746 dup_rows_remote=18894 dedup_ratio=0.221212 router_tokens_per_expert=[770, 1940, 1231, 186, 264, 240, 2840, 724, 41, 0, 1084, 1052, 1637, 0, 11, 1376, 741, 999, 2875, 87, 324, 692, 246, 5, 244, 1749, 839, 212, 419, 218, 2658, 1104, 1149, 420, 274, 1515, 705, 319, 1324, 1281, 292, 0, 104, 1589, 90, 592, 994, 374, 822, 1163, 1392, 816, 170, 576, 0, 1389, 1304, 463, 442, 579, 260, 309, 1025, 612] payload_rows_per_dst=[12088, 10328, 12510, 8400, 9390, 6606, 8346, 8890] dup_rows_per_dst=[4940, 2852, 4744, 1170, 3048, 1464, 1532, 1996] combine_rows_per_src=[12088, 10328, 12510, 8400, 9390, 6606, 8346, 8890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282689536 dispatch_meta_bytes=349176 combine_payload_bytes=282689536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283038712 ingress_bytes=317292544 link_bytes=317292544 token_bytes=565728248 weight_movement_bytes=34603008 total_nvlink_bytes=600331256 weight_to_token_ratio=0.061165 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193946299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267616256 dispatch_meta_bytes=325288 combine_payload_bytes=267616256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267941544 ingress_bytes=302219264 link_bytes=302219264 token_bytes=535557800 weight_movement_bytes=34603008 total_nvlink_bytes=570160808 weight_to_token_ratio=0.064611 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137226760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269574144 dispatch_meta_bytes=336976 combine_payload_bytes=269574144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269911120 ingress_bytes=304177152 link_bytes=304177152 token_bytes=539485264 weight_movement_bytes=34603008 total_nvlink_bytes=574088272 weight_to_token_ratio=0.064141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144594135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75758 payload_rows_remote=64498 dup_rows=22546 dup_rows_remote=18558 dedup_ratio=0.229350 router_tokens_per_expert=[562, 1473, 1262, 314, 389, 213, 2671, 396, 53, 1, 1059, 968, 1397, 0, 62, 1435, 785, 877, 2375, 68, 401, 662, 287, 3, 222, 1285, 737, 148, 481, 303, 3193, 591, 1332, 612, 257, 1851, 934, 344, 1498, 1251, 105, 4, 176, 1334, 98, 509, 1464, 462, 1079, 1024, 1158, 932, 370, 757, 2, 1442, 1319, 505, 432, 804, 383, 338, 993, 710] payload_rows_per_dst=[11260, 9930, 12404, 8024, 10698, 6904, 7486, 9052] dup_rows_per_dst=[3988, 2904, 4898, 1480, 4772, 1400, 1188, 1916] combine_rows_per_src=[11260, 9930, 12404, 8024, 10698, 6904, 7486, 9052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73806 payload_rows_remote=64960 dup_rows=24498 dup_rows_remote=21656 dedup_ratio=0.249207 router_tokens_per_expert=[1252, 477, 457, 274, 682, 1071, 608, 846, 640, 646, 414, 731, 628, 199, 457, 595, 131, 426, 679, 159, 2772, 581, 969, 611, 210, 1689, 482, 468, 1826, 519, 56, 399, 304, 911, 687, 173, 225, 1536, 580, 1420, 1031, 390, 908, 964, 934, 926, 212, 555, 561, 2190, 637, 276, 1347, 432, 1893, 2603, 925, 1296, 366, 300, 1841, 90, 208, 477] payload_rows_per_dst=[9004, 6784, 9976, 10242, 8834, 9006, 8846, 11114] dup_rows_per_dst=[2330, 1836, 2680, 4040, 2838, 2834, 2842, 5098] combine_rows_per_src=[9004, 6784, 9976, 10242, 8834, 9006, 8846, 11114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=340496 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271618576 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542896656 weight_movement_bytes=34603008 total_nvlink_bytes=577499664 weight_to_token_ratio=0.063738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76354 payload_rows_remote=68680 dup_rows=21950 dup_rows_remote=21050 dedup_ratio=0.223287 router_tokens_per_expert=[757, 223, 18, 2499, 109, 216, 314, 142, 17, 825, 1938, 1111, 1090, 438, 443, 1067, 38, 121, 808, 20, 1287, 566, 442, 1946, 144, 465, 960, 65, 800, 503, 1, 53, 155, 1300, 383, 58, 1, 33, 1, 2356, 1451, 261, 278, 1736, 4480, 592, 548, 2861, 413, 762, 9, 1276, 478, 770, 4861, 14, 3496, 62, 2, 52, 426, 1, 382, 228] payload_rows_per_dst=[7370, 12140, 12276, 12134, 7674, 9698, 6488, 8574] dup_rows_per_dst=[1186, 4894, 5222, 3570, 900, 4498, 956, 724] combine_rows_per_src=[7370, 12140, 12276, 12134, 7674, 9698, 6488, 8574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=332224 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264516032 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528699840 weight_movement_bytes=34603008 total_nvlink_bytes=563302848 weight_to_token_ratio=0.065449 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=346464 combine_payload_bytes=266076160 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266422624 ingress_bytes=404488192 link_bytes=404488192 token_bytes=532498784 weight_movement_bytes=138412032 total_nvlink_bytes=670910816 weight_to_token_ratio=0.259929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522056503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281313280 dispatch_meta_bytes=358920 combine_payload_bytes=281313280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281672200 ingress_bytes=315916288 link_bytes=315916288 token_bytes=562985480 weight_movement_bytes=34603008 total_nvlink_bytes=597588488 weight_to_token_ratio=0.061463 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188767558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71282 payload_rows_remote=64736 dup_rows=27022 dup_rows_remote=25400 dedup_ratio=0.274882 router_tokens_per_expert=[986, 142, 1004, 5, 1804, 1067, 613, 873, 869, 201, 65, 323, 488, 3, 40, 174, 15, 254, 432, 87, 1822, 486, 842, 117, 192, 982, 112, 283, 1184, 486, 354, 474, 220, 338, 55, 105, 880, 3525, 2487, 340, 367, 542, 1543, 608, 23, 476, 118, 240, 390, 2052, 1504, 86, 3441, 272, 17, 5107, 33, 1504, 890, 629, 3791, 420, 124, 246] payload_rows_per_dst=[9608, 4494, 6932, 6546, 12260, 5988, 14140, 11314] dup_rows_per_dst=[3380, 312, 1224, 1622, 7204, 1320, 8000, 3960] combine_rows_per_src=[9608, 4494, 6932, 6546, 12260, 5988, 14140, 11314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=64214 dup_rows=23612 dup_rows_remote=20982 dedup_ratio=0.240194 router_tokens_per_expert=[1068, 164, 1123, 419, 2173, 1183, 636, 929, 675, 204, 154, 614, 546, 106, 247, 334, 186, 468, 342, 26, 3183, 446, 782, 363, 153, 1304, 171, 329, 1946, 712, 120, 327, 188, 392, 173, 65, 334, 3078, 1305, 804, 522, 375, 1115, 701, 758, 399, 152, 459, 463, 1889, 1007, 138, 1601, 330, 944, 4243, 539, 1399, 468, 408, 2866, 207, 94, 303] payload_rows_per_dst=[10634, 5528, 10478, 9660, 11146, 5630, 11650, 9966] dup_rows_per_dst=[4756, 1150, 2630, 2352, 5310, 1580, 3232, 2602] combine_rows_per_src=[10634, 5528, 10478, 9660, 11146, 5630, 11650, 9966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72660 payload_rows_remote=65580 dup_rows=25644 dup_rows_remote=23504 dedup_ratio=0.260864 router_tokens_per_expert=[1329, 278, 974, 52, 1268, 1146, 595, 1180, 738, 442, 112, 491, 504, 28, 305, 515, 73, 436, 488, 239, 2315, 505, 1105, 222, 271, 1571, 184, 514, 2201, 541, 258, 435, 216, 576, 313, 282, 378, 2418, 1527, 702, 643, 558, 1374, 654, 212, 657, 153, 359, 475, 2700, 1371, 136, 1940, 352, 725, 3519, 212, 1132, 764, 608, 2265, 203, 150, 263] payload_rows_per_dst=[10126, 5064, 8838, 10218, 9256, 7080, 10474, 11604] dup_rows_per_dst=[3518, 1206, 1928, 3182, 3568, 2140, 4122, 5980] combine_rows_per_src=[10126, 5064, 8838, 10218, 9256, 7080, 10474, 11604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76218 payload_rows_remote=67454 dup_rows=22086 dup_rows_remote=21760 dedup_ratio=0.224670 router_tokens_per_expert=[686, 129, 0, 3531, 90, 23, 251, 92, 0, 890, 1699, 1032, 1097, 484, 423, 988, 4, 40, 775, 5, 376, 467, 410, 1955, 113, 107, 925, 0, 449, 477, 0, 10, 78, 1579, 206, 9, 0, 0, 0, 2900, 1389, 236, 202, 1666, 5373, 211, 921, 3459, 412, 388, 0, 1246, 313, 764, 5727, 0, 3771, 2, 0, 20, 309, 0, 286, 157] payload_rows_per_dst=[8652, 10024, 6768, 4068, 8744, 14944, 14254, 8764] dup_rows_per_dst=[952, 3202, 1296, 94, 800, 11970, 3446, 326] combine_rows_per_src=[8652, 10024, 6768, 4068, 8744, 14944, 14254, 8764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=360544 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265519200 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530677856 weight_movement_bytes=34603008 total_nvlink_bytes=565280864 weight_to_token_ratio=0.065205 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79034 payload_rows_remote=65080 dup_rows=19270 dup_rows_remote=12206 dedup_ratio=0.196025 router_tokens_per_expert=[650, 126, 0, 3252, 92, 13, 234, 76, 0, 1112, 1837, 1228, 1160, 293, 394, 935, 1, 5, 901, 2, 112, 507, 430, 2079, 136, 76, 722, 1, 401, 318, 0, 3, 81, 1407, 190, 4, 0, 0, 0, 2899, 1613, 270, 223, 1748, 5332, 180, 983, 3550, 363, 778, 1, 951, 363, 876, 5653, 0, 3749, 1, 0, 20, 267, 0, 400, 154] payload_rows_per_dst=[7980, 13954, 13754, 12770, 9612, 7898, 4280, 8786] dup_rows_per_dst=[906, 7064, 4984, 1850, 1106, 2862, 102, 396] combine_rows_per_src=[7980, 13954, 13754, 12770, 9612, 7898, 4280, 8786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263020544 dispatch_meta_bytes=340784 combine_payload_bytes=263020544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263361328 ingress_bytes=297623552 link_bytes=297623552 token_bytes=526381872 weight_movement_bytes=34603008 total_nvlink_bytes=560984880 weight_to_token_ratio=0.065737 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119933466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268615680 dispatch_meta_bytes=356336 combine_payload_bytes=268615680 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268972016 ingress_bytes=407027712 link_bytes=407027712 token_bytes=537587696 weight_movement_bytes=138412032 total_nvlink_bytes=675999728 weight_to_token_ratio=0.257469 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.531612512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76578 payload_rows_remote=64278 dup_rows=21726 dup_rows_remote=16124 dedup_ratio=0.221008 router_tokens_per_expert=[793, 185, 0, 2984, 92, 37, 269, 101, 0, 927, 1662, 1134, 1174, 667, 443, 1007, 0, 42, 851, 6, 284, 597, 443, 1752, 135, 94, 1258, 0, 475, 235, 0, 7, 101, 1589, 215, 9, 0, 1, 1, 1896, 1254, 240, 229, 1818, 5174, 226, 1274, 3312, 346, 714, 1, 1310, 387, 1078, 5774, 1, 3586, 2, 0, 49, 338, 1, 403, 169] payload_rows_per_dst=[12300, 13700, 7440, 12728, 8360, 8060, 5318, 8672] dup_rows_per_dst=[5602, 6952, 1878, 3228, 692, 2796, 154, 424] combine_rows_per_src=[12300, 13700, 7440, 12728, 8360, 8060, 5318, 8672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76410 payload_rows_remote=65064 dup_rows=21894 dup_rows_remote=17414 dedup_ratio=0.222717 router_tokens_per_expert=[743, 657, 474, 1055, 3219, 1109, 1440, 777, 81, 510, 2182, 766, 416, 1269, 592, 695, 762, 590, 898, 968, 712, 367, 1280, 1449, 490, 398, 720, 178, 677, 805, 287, 832, 1213, 500, 1677, 175, 80, 414, 717, 124, 691, 714, 343, 615, 953, 812, 165, 243, 1141, 1386, 1992, 513, 662, 323, 656, 1240, 384, 835, 1143, 347, 322, 164, 747, 463] payload_rows_per_dst=[8722, 10244, 8968, 11352, 7980, 9186, 11346, 8612] dup_rows_per_dst=[2474, 2778, 2524, 3860, 1820, 2446, 4480, 1512] combine_rows_per_src=[8722, 10244, 8968, 11352, 7980, 9186, 11346, 8612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276291584 dispatch_meta_bytes=356856 combine_payload_bytes=276291584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276648440 ingress_bytes=310894592 link_bytes=310894592 token_bytes=552940024 weight_movement_bytes=34603008 total_nvlink_bytes=587543032 weight_to_token_ratio=0.062580 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169871321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78724 payload_rows_remote=67680 dup_rows=19580 dup_rows_remote=17310 dedup_ratio=0.199178 router_tokens_per_expert=[2014, 3212, 19, 627, 1122, 300, 106, 232, 11, 392, 249, 176, 147, 336, 567, 569, 399, 714, 0, 29, 722, 189, 3338, 1220, 1613, 2745, 1238, 250, 9, 279, 248, 472, 9, 298, 2118, 1, 92, 1283, 728, 303, 737, 358, 0, 236, 2509, 5388, 1015, 0, 4851, 0, 2406, 532, 451, 171, 349, 34, 31, 384, 0, 0, 0, 547, 270, 507] payload_rows_per_dst=[7426, 12136, 9666, 10834, 11044, 11172, 7624, 8822] dup_rows_per_dst=[1414, 1964, 2104, 2874, 2270, 7116, 758, 1080] combine_rows_per_src=[7426, 12136, 9666, 10834, 11044, 11172, 7624, 8822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=309144 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266876824 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533444504 weight_movement_bytes=34603008 total_nvlink_bytes=568047512 weight_to_token_ratio=0.064867 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263282688 dispatch_meta_bytes=321608 combine_payload_bytes=263282688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263604296 ingress_bytes=297885696 link_bytes=297885696 token_bytes=526886984 weight_movement_bytes=34603008 total_nvlink_bytes=561489992 weight_to_token_ratio=0.065674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120919892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=329912 combine_payload_bytes=266502144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266832056 ingress_bytes=335708160 link_bytes=335708160 token_bytes=533334200 weight_movement_bytes=69206016 total_nvlink_bytes=602540216 weight_to_token_ratio=0.129761 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263242779e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277217280 dispatch_meta_bytes=339960 combine_payload_bytes=277217280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277557240 ingress_bytes=311820288 link_bytes=311820288 token_bytes=554774520 weight_movement_bytes=34603008 total_nvlink_bytes=589377528 weight_to_token_ratio=0.062373 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173354640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79658 payload_rows_remote=75914 dup_rows=18646 dup_rows_remote=18444 dedup_ratio=0.189677 router_tokens_per_expert=[246, 222, 5206, 49, 624, 36, 1687, 55, 15, 464, 776, 82, 1754, 1539, 502, 173, 754, 221, 5292, 1265, 118, 145, 1, 381, 97, 1, 187, 159, 529, 426, 236, 338, 1330, 358, 16, 552, 95, 95, 404, 82, 708, 4, 449, 600, 36, 145, 1, 3777, 1, 5910, 49, 250, 1802, 72, 346, 336, 50, 729, 4772, 1430, 731, 6, 167, 269] payload_rows_per_dst=[12000, 8828, 12872, 3744, 5258, 9958, 13698, 13300] dup_rows_per_dst=[3806, 1784, 3482, 202, 896, 1192, 3832, 3452] combine_rows_per_src=[12000, 8828, 12872, 3744, 5258, 9958, 13698, 13300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77002 payload_rows_remote=65592 dup_rows=21302 dup_rows_remote=18158 dedup_ratio=0.216695 router_tokens_per_expert=[607, 614, 1737, 941, 2687, 721, 2025, 507, 417, 454, 1867, 414, 739, 1414, 597, 615, 645, 101, 2298, 1273, 392, 476, 1318, 774, 236, 566, 548, 120, 535, 603, 152, 605, 938, 420, 1202, 207, 62, 173, 634, 145, 698, 315, 300, 502, 1122, 797, 107, 1347, 871, 2608, 1406, 420, 760, 134, 520, 1135, 109, 669, 2946, 486, 701, 87, 93, 240] payload_rows_per_dst=[12284, 11386, 11410, 6032, 7398, 7888, 10462, 10142] dup_rows_per_dst=[6634, 3390, 3144, 698, 1758, 894, 3036, 1748] combine_rows_per_src=[12284, 11386, 11410, 6032, 7398, 7888, 10462, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76224 payload_rows_remote=69394 dup_rows=22080 dup_rows_remote=21268 dedup_ratio=0.224609 router_tokens_per_expert=[261, 349, 3542, 478, 2225, 568, 1789, 421, 6, 508, 1994, 346, 1132, 1716, 547, 420, 804, 476, 3477, 1352, 253, 317, 79, 895, 335, 44, 327, 126, 795, 573, 200, 609, 2024, 460, 375, 353, 67, 131, 573, 77, 631, 46, 407, 662, 297, 69, 6, 1624, 21, 3356, 274, 436, 1172, 126, 400, 1163, 364, 899, 3554, 1435, 474, 12, 460, 240] payload_rows_per_dst=[11488, 10260, 11388, 6594, 6676, 6830, 10814, 12174] dup_rows_per_dst=[5410, 3078, 3760, 1094, 1444, 812, 3082, 3400] combine_rows_per_src=[11488, 10260, 11388, 6594, 6676, 6830, 10814, 12174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=62812 dup_rows=23826 dup_rows_remote=22466 dedup_ratio=0.242371 router_tokens_per_expert=[2691, 4751, 0, 280, 775, 198, 74, 87, 0, 416, 136, 158, 74, 328, 589, 543, 350, 1162, 0, 9, 330, 115, 2686, 1275, 2753, 2795, 1786, 285, 1, 229, 175, 332, 0, 292, 1574, 0, 87, 1690, 786, 223, 645, 103, 0, 59, 2154, 5777, 652, 0, 4677, 0, 1891, 512, 469, 102, 314, 0, 17, 401, 0, 0, 0, 792, 225, 327] payload_rows_per_dst=[5510, 3750, 5956, 12478, 8430, 14160, 12528, 11666] dup_rows_per_dst=[186, 738, 1490, 5784, 874, 9992, 3402, 1360] combine_rows_per_src=[5510, 3750, 5956, 12478, 8430, 14160, 12528, 11666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79452 payload_rows_remote=67716 dup_rows=18852 dup_rows_remote=17092 dedup_ratio=0.191772 router_tokens_per_expert=[2673, 4881, 0, 219, 724, 117, 60, 75, 0, 461, 110, 172, 143, 302, 666, 406, 325, 1291, 0, 5, 398, 96, 1859, 718, 3219, 2563, 1402, 255, 1, 273, 193, 412, 0, 264, 1270, 0, 93, 2593, 952, 251, 768, 136, 0, 73, 2677, 5604, 799, 0, 4488, 0, 1602, 528, 435, 95, 319, 0, 9, 373, 0, 0, 0, 1114, 271, 419] payload_rows_per_dst=[6998, 11736, 7786, 12218, 15096, 7828, 5592, 12198] dup_rows_per_dst=[738, 1760, 1598, 4418, 6958, 1078, 366, 1936] combine_rows_per_src=[6998, 11736, 7786, 12218, 15096, 7828, 5592, 12198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310943744 dispatch_meta_bytes=377432 combine_payload_bytes=310943744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311321176 ingress_bytes=345546752 link_bytes=345546752 token_bytes=622264920 weight_movement_bytes=34603008 total_nvlink_bytes=656867928 weight_to_token_ratio=0.055608 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300264609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268664832 dispatch_meta_bytes=335000 combine_payload_bytes=268664832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268999832 ingress_bytes=337870848 link_bytes=337870848 token_bytes=537664664 weight_movement_bytes=69206016 total_nvlink_bytes=606870680 weight_to_token_ratio=0.128716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271380800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79346 payload_rows_remote=69114 dup_rows=18958 dup_rows_remote=15976 dedup_ratio=0.192851 router_tokens_per_expert=[2730, 4145, 10, 221, 858, 175, 86, 72, 0, 463, 195, 159, 119, 347, 704, 584, 436, 1004, 2, 10, 227, 115, 2849, 1435, 2418, 2614, 1876, 252, 0, 347, 257, 297, 0, 422, 1250, 0, 94, 1503, 1107, 208, 723, 78, 0, 53, 1977, 5617, 1045, 0, 4665, 2, 2241, 501, 427, 113, 359, 1, 21, 354, 1, 0, 0, 611, 365, 407] payload_rows_per_dst=[10232, 12200, 9706, 12262, 14726, 6882, 6878, 6460] dup_rows_per_dst=[2982, 2272, 2450, 3860, 5676, 870, 406, 442] combine_rows_per_src=[10232, 12200, 9706, 12262, 14726, 6882, 6878, 6460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64376 dup_rows=23996 dup_rows_remote=20710 dedup_ratio=0.244100 router_tokens_per_expert=[752, 2329, 989, 960, 282, 1283, 1991, 1138, 265, 239, 509, 746, 952, 934, 574, 1005, 252, 358, 858, 193, 698, 75, 2093, 1085, 113, 646, 1367, 518, 1441, 299, 400, 936, 1442, 814, 326, 401, 454, 1040, 722, 534, 527, 692, 1208, 2002, 434, 1249, 905, 518, 685, 134, 630, 792, 1663, 368, 316, 579, 836, 906, 417, 165, 495, 6, 864, 748] payload_rows_per_dst=[8746, 7260, 11482, 9392, 7866, 10720, 9932, 8910] dup_rows_per_dst=[2524, 1284, 3724, 3968, 2620, 4350, 3286, 2240] combine_rows_per_src=[8746, 7260, 11482, 9392, 7866, 10720, 9932, 8910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=284237824 dispatch_meta_bytes=362648 combine_payload_bytes=284237824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284600472 ingress_bytes=353443840 link_bytes=353443840 token_bytes=568838296 weight_movement_bytes=69206016 total_nvlink_bytes=638044312 weight_to_token_ratio=0.121662 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329980715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=257277952 dispatch_meta_bytes=341112 combine_payload_bytes=257277952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257619064 ingress_bytes=291880960 link_bytes=291880960 token_bytes=514897016 weight_movement_bytes=34603008 total_nvlink_bytes=549500024 weight_to_token_ratio=0.067204 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.098324554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74732 payload_rows_remote=68550 dup_rows=23572 dup_rows_remote=21080 dedup_ratio=0.239787 router_tokens_per_expert=[546, 1660, 2704, 3632, 1056, 0, 4335, 5222, 0, 184, 0, 955, 0, 2454, 0, 154, 276, 8, 1490, 210, 838, 108, 656, 175, 69, 137, 2519, 0, 41, 161, 0, 212, 5251, 1057, 0, 174, 367, 607, 538, 0, 433, 2, 131, 0, 126, 0, 304, 208, 1687, 0, 303, 1389, 26, 1268, 222, 0, 2339, 538, 345, 911, 1028, 0, 95, 1] payload_rows_per_dst=[9884, 6850, 12764, 10900, 6182, 2400, 12094, 13658] dup_rows_per_dst=[2048, 644, 3428, 2642, 2492, 8, 5010, 7300] combine_rows_per_src=[9884, 6850, 12764, 10900, 6182, 2400, 12094, 13658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277364736 dispatch_meta_bytes=339232 combine_payload_bytes=277364736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277703968 ingress_bytes=311967744 link_bytes=311967744 token_bytes=555068704 weight_movement_bytes=34603008 total_nvlink_bytes=589671712 weight_to_token_ratio=0.062340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173909505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283090944 dispatch_meta_bytes=340360 combine_payload_bytes=283090944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=283431304 ingress_bytes=421502976 link_bytes=421502976 token_bytes=566522248 weight_movement_bytes=138412032 total_nvlink_bytes=704934280 weight_to_token_ratio=0.244319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.586081765e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=263684096 dispatch_meta_bytes=340344 combine_payload_bytes=263684096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264024440 ingress_bytes=298287104 link_bytes=298287104 token_bytes=527708536 weight_movement_bytes=34603008 total_nvlink_bytes=562311544 weight_to_token_ratio=0.065572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122430358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72660 payload_rows_remote=64358 dup_rows=25644 dup_rows_remote=23530 dedup_ratio=0.260864 router_tokens_per_expert=[502, 1386, 1145, 774, 242, 1813, 1747, 1321, 933, 328, 1569, 606, 3684, 514, 449, 296, 60, 159, 578, 153, 772, 81, 1413, 245, 82, 506, 972, 413, 1402, 103, 948, 461, 1332, 603, 326, 564, 324, 515, 642, 1170, 462, 230, 878, 2491, 395, 3986, 1298, 453, 644, 282, 279, 502, 1302, 390, 48, 1420, 852, 570, 92, 106, 109, 1, 640, 589] payload_rows_per_dst=[8716, 11596, 8302, 8444, 8800, 11538, 8378, 6886] dup_rows_per_dst=[2358, 5162, 2114, 2878, 2152, 7950, 1356, 1674] combine_rows_per_src=[8716, 11596, 8302, 8444, 8800, 11538, 8378, 6886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71220 payload_rows_remote=61454 dup_rows=27084 dup_rows_remote=25570 dedup_ratio=0.275513 router_tokens_per_expert=[232, 96, 47, 0, 2, 2536, 274, 1, 2216, 292, 3121, 157, 6763, 2, 358, 82, 0, 8, 0, 33, 234, 19, 1128, 45, 86, 587, 0, 667, 81, 167, 3830, 222, 0, 112, 882, 497, 77, 18, 575, 2849, 380, 352, 352, 3349, 181, 6158, 2929, 359, 338, 383, 8, 0, 445, 0, 39, 4153, 1, 441, 0, 64, 0, 0, 671, 253] payload_rows_per_dst=[5758, 15284, 3024, 9766, 8880, 15662, 10144, 2702] dup_rows_per_dst=[68, 10698, 458, 1514, 1140, 12458, 588, 160] combine_rows_per_src=[5758, 15284, 3024, 9766, 8880, 15662, 10144, 2702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280780800 dispatch_meta_bytes=358520 combine_payload_bytes=280780800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281139320 ingress_bytes=349986816 link_bytes=349986816 token_bytes=561920120 weight_movement_bytes=69206016 total_nvlink_bytes=631126136 weight_to_token_ratio=0.123160 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316972212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71962 payload_rows_remote=56806 dup_rows=26342 dup_rows_remote=14594 dedup_ratio=0.267965 router_tokens_per_expert=[441, 169, 217, 24, 18, 2131, 421, 7, 956, 396, 1394, 374, 6496, 19, 378, 310, 12, 52, 13, 44, 302, 48, 1751, 408, 107, 630, 494, 1159, 657, 148, 3580, 521, 8, 387, 767, 164, 205, 365, 701, 2790, 441, 383, 586, 3573, 216, 5379, 2460, 414, 350, 368, 130, 4, 967, 10, 110, 2935, 14, 570, 1, 64, 1, 0, 654, 458] payload_rows_per_dst=[5620, 11956, 5378, 10800, 11036, 15156, 8578, 3438] dup_rows_per_dst=[332, 4426, 724, 3840, 3986, 11748, 1186, 100] combine_rows_per_src=[5620, 11956, 5378, 10800, 11036, 15156, 8578, 3438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=263610368 dispatch_meta_bytes=351552 combine_payload_bytes=263610368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263961920 ingress_bytes=298213376 link_bytes=298213376 token_bytes=527572288 weight_movement_bytes=34603008 total_nvlink_bytes=562175296 weight_to_token_ratio=0.065589 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122152926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=251715584 dispatch_meta_bytes=348096 combine_payload_bytes=251715584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252063680 ingress_bytes=286318592 link_bytes=286318592 token_bytes=503779264 weight_movement_bytes=34603008 total_nvlink_bytes=538382272 weight_to_token_ratio=0.068687 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077393811e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74858 payload_rows_remote=61618 dup_rows=23446 dup_rows_remote=17042 dedup_ratio=0.238505 router_tokens_per_expert=[576, 679, 3788, 4001, 1067, 0, 5257, 4754, 0, 173, 0, 940, 0, 2196, 0, 154, 305, 1, 1479, 49, 633, 90, 428, 145, 68, 86, 2077, 0, 2, 155, 0, 392, 5525, 1189, 0, 147, 296, 1192, 534, 0, 415, 3, 160, 0, 49, 0, 325, 67, 1843, 0, 247, 902, 44, 1369, 282, 0, 1444, 530, 141, 1524, 1421, 0, 8, 0] payload_rows_per_dst=[8544, 7844, 13206, 11210, 4826, 2030, 13958, 13240] dup_rows_per_dst=[1330, 1428, 3568, 2352, 1890, 8, 6466, 6404] combine_rows_per_src=[8544, 7844, 13206, 11210, 4826, 2030, 13958, 13240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76606 payload_rows_remote=70766 dup_rows=21698 dup_rows_remote=21154 dedup_ratio=0.220723 router_tokens_per_expert=[589, 586, 3960, 4347, 627, 0, 5437, 4382, 0, 184, 0, 727, 0, 2160, 0, 121, 295, 0, 1496, 43, 729, 112, 581, 147, 58, 92, 1854, 0, 0, 150, 0, 375, 5449, 1528, 0, 168, 326, 1423, 575, 0, 394, 0, 199, 0, 0, 0, 338, 71, 1714, 0, 175, 712, 137, 1248, 134, 0, 914, 452, 355, 2078, 1708, 0, 2, 0] payload_rows_per_dst=[10080, 5840, 13910, 11558, 13260, 2002, 6946, 13010] dup_rows_per_dst=[1444, 544, 3770, 2194, 5678, 2, 1294, 6772] combine_rows_per_src=[10080, 5840, 13910, 11558, 13260, 2002, 6946, 13010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70234 payload_rows_remote=55480 dup_rows=28070 dup_rows_remote=11972 dedup_ratio=0.285543 router_tokens_per_expert=[563, 674, 3763, 3906, 899, 0, 5339, 3917, 1, 183, 0, 917, 2, 2503, 0, 138, 273, 0, 1545, 57, 786, 73, 633, 160, 77, 90, 1840, 0, 1, 169, 0, 346, 5330, 1398, 0, 158, 272, 974, 616, 0, 437, 0, 169, 2, 2, 1, 350, 176, 1640, 1, 251, 699, 171, 1692, 90, 0, 1671, 515, 155, 1660, 1769, 1, 96, 1] payload_rows_per_dst=[14754, 6912, 10048, 5774, 13074, 2268, 7850, 9554] dup_rows_per_dst=[16098, 576, 2404, 1072, 4422, 4, 1238, 2256] combine_rows_per_src=[14754, 6912, 10048, 5774, 13074, 2268, 7850, 9554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=63924 dup_rows=23844 dup_rows_remote=21336 dedup_ratio=0.242554 router_tokens_per_expert=[405, 1316, 1782, 736, 169, 404, 382, 592, 221, 845, 544, 1107, 369, 696, 1937, 509, 786, 179, 2363, 142, 206, 823, 500, 757, 199, 930, 1049, 771, 279, 270, 454, 135, 271, 1214, 693, 549, 343, 1553, 1782, 162, 357, 192, 504, 366, 516, 439, 112, 459, 209, 289, 227, 780, 1654, 1334, 915, 880, 1419, 545, 1454, 86, 547, 2113, 3601, 1730] payload_rows_per_dst=[7658, 10766, 9366, 8184, 10160, 4298, 10536, 13492] dup_rows_per_dst=[1282, 4322, 2146, 2898, 2974, 206, 2508, 7508] combine_rows_per_src=[7658, 10766, 9366, 8184, 10160, 4298, 10536, 13492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232677376 dispatch_meta_bytes=285600 combine_payload_bytes=232677376 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=232962976 ingress_bytes=267280384 link_bytes=267280384 token_bytes=465640352 weight_movement_bytes=34603008 total_nvlink_bytes=500243360 weight_to_token_ratio=0.074313 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005754567e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81796 payload_rows_remote=66818 dup_rows=16508 dup_rows_remote=12844 dedup_ratio=0.167928 router_tokens_per_expert=[0, 5009, 1244, 29, 4, 95, 575, 5042, 0, 8, 0, 0, 0, 472, 2326, 173, 379, 713, 150, 192, 83, 0, 327, 782, 510, 571, 66, 215, 252, 97, 267, 3, 75, 7085, 149, 0, 1016, 1, 995, 0, 1218, 0, 57, 0, 2319, 252, 0, 0, 25, 0, 135, 1905, 1041, 567, 91, 0, 4, 377, 5521, 117, 236, 2556, 82, 3744] payload_rows_per_dst=[11906, 12556, 4322, 13086, 14978, 6992, 6774, 11182] dup_rows_per_dst=[2072, 3420, 930, 1918, 3664, 700, 754, 3050] combine_rows_per_src=[11906, 12556, 4322, 13086, 14978, 6992, 6774, 11182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=252387328 dispatch_meta_bytes=314640 combine_payload_bytes=252387328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252701968 ingress_bytes=286990336 link_bytes=286990336 token_bytes=505089296 weight_movement_bytes=34603008 total_nvlink_bytes=539692304 weight_to_token_ratio=0.068509 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079921530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289857536 dispatch_meta_bytes=367680 combine_payload_bytes=289857536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290225216 ingress_bytes=359063552 link_bytes=359063552 token_bytes=580082752 weight_movement_bytes=69206016 total_nvlink_bytes=649288768 weight_to_token_ratio=0.119304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351127239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=227246080 dispatch_meta_bytes=269808 combine_payload_bytes=227246080 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=227515888 ingress_bytes=400261120 link_bytes=400261120 token_bytes=454761968 weight_movement_bytes=173015040 total_nvlink_bytes=627777008 weight_to_token_ratio=0.380452 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506150371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77158 payload_rows_remote=67420 dup_rows=21146 dup_rows_remote=17838 dedup_ratio=0.215108 router_tokens_per_expert=[504, 1193, 460, 422, 96, 233, 302, 999, 95, 950, 1124, 1466, 335, 487, 1324, 268, 1151, 141, 2280, 61, 259, 1765, 216, 650, 95, 457, 711, 787, 311, 238, 136, 4, 26, 1129, 408, 1416, 179, 1598, 1375, 174, 264, 89, 1789, 222, 343, 463, 623, 2142, 96, 1097, 34, 691, 1366, 1132, 554, 3063, 1328, 402, 1436, 59, 395, 1520, 2794, 1425] payload_rows_per_dst=[10340, 11084, 9738, 6782, 10198, 8606, 8848, 11562] dup_rows_per_dst=[1482, 3400, 3308, 1568, 2748, 3264, 1092, 4284] combine_rows_per_src=[10340, 11084, 9738, 6782, 10198, 8606, 8848, 11562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261832704 dispatch_meta_bytes=341040 combine_payload_bytes=261832704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262173744 ingress_bytes=365641728 link_bytes=365641728 token_bytes=524006448 weight_movement_bytes=103809024 total_nvlink_bytes=627815472 weight_to_token_ratio=0.198106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375880386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65844 payload_rows_remote=63412 dup_rows=32460 dup_rows_remote=32070 dedup_ratio=0.330200 router_tokens_per_expert=[449, 181, 2, 18, 0, 65, 0, 1, 2, 578, 1184, 440, 14, 380, 140, 34, 650, 1, 161, 0, 435, 5313, 26, 325, 0, 195, 629, 65, 244, 65, 212, 0, 0, 0, 95, 4045, 1, 863, 0, 11, 38, 23, 7210, 220, 0, 774, 520, 7566, 7, 6618, 0, 0, 0, 462, 333, 7255, 45, 385, 1, 31, 152, 0, 603, 85] payload_rows_per_dst=[1070, 5298, 11856, 2432, 10858, 16266, 15466, 2598] dup_rows_per_dst=[0, 608, 1966, 390, 3574, 16436, 9482, 4] combine_rows_per_src=[1070, 5298, 11856, 2432, 10858, 16266, 15466, 2598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=273686528 dispatch_meta_bytes=318648 combine_payload_bytes=273686528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274005176 ingress_bytes=342892544 link_bytes=342892544 token_bytes=547691704 weight_movement_bytes=69206016 total_nvlink_bytes=616897720 weight_to_token_ratio=0.126359 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73770 payload_rows_remote=59254 dup_rows=24534 dup_rows_remote=20730 dedup_ratio=0.249573 router_tokens_per_expert=[500, 516, 25, 117, 3, 68, 0, 12, 2, 1145, 1913, 654, 175, 411, 228, 121, 888, 5, 490, 0, 422, 3233, 20, 344, 0, 257, 776, 78, 256, 63, 128, 1, 4, 16, 67, 3398, 7, 1471, 29, 150, 69, 93, 6592, 254, 2, 917, 410, 6441, 35, 5922, 1, 5, 24, 506, 555, 6901, 260, 414, 16, 66, 294, 69, 1236, 77] payload_rows_per_dst=[1442, 8560, 9122, 2622, 9076, 14516, 15470, 12962] dup_rows_per_dst=[8, 1770, 1682, 528, 1208, 3804, 12428, 3106] combine_rows_per_src=[1442, 8560, 9122, 2622, 9076, 14516, 15470, 12962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=276152320 dispatch_meta_bytes=341032 combine_payload_bytes=276152320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276493352 ingress_bytes=310755328 link_bytes=310755328 token_bytes=552645672 weight_movement_bytes=34603008 total_nvlink_bytes=587248680 weight_to_token_ratio=0.062613 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169347281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80936 payload_rows_remote=70138 dup_rows=17368 dup_rows_remote=14752 dedup_ratio=0.176676 router_tokens_per_expert=[0, 6548, 1400, 25, 4, 71, 654, 4649, 0, 5, 0, 0, 0, 519, 2368, 160, 301, 853, 74, 198, 78, 0, 218, 713, 729, 337, 1, 183, 245, 90, 199, 0, 104, 7235, 106, 0, 1660, 0, 549, 0, 1682, 0, 0, 0, 2086, 251, 0, 0, 82, 0, 89, 1105, 391, 537, 86, 0, 0, 387, 5860, 179, 244, 2622, 54, 3221] payload_rows_per_dst=[11530, 14460, 4096, 13228, 15230, 7272, 4322, 10798] dup_rows_per_dst=[2076, 4740, 774, 2060, 4078, 766, 258, 2616] combine_rows_per_src=[11530, 14460, 4096, 13228, 15230, 7272, 4322, 10798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=259735552 dispatch_meta_bytes=381928 combine_payload_bytes=259735552 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=260117480 ingress_bytes=328941568 link_bytes=328941568 token_bytes=519853032 weight_movement_bytes=69206016 total_nvlink_bytes=589059048 weight_to_token_ratio=0.133126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237780638e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76488 payload_rows_remote=66910 dup_rows=21816 dup_rows_remote=20040 dedup_ratio=0.221924 router_tokens_per_expert=[0, 6731, 1636, 24, 1, 64, 711, 3754, 0, 17, 0, 0, 0, 645, 2329, 166, 308, 897, 143, 43, 76, 0, 264, 688, 1266, 319, 1, 183, 224, 85, 245, 9, 229, 7254, 115, 0, 2106, 0, 258, 0, 1197, 0, 0, 0, 2437, 227, 0, 0, 75, 0, 94, 1249, 218, 573, 92, 0, 1, 350, 5791, 192, 259, 1564, 88, 3954] payload_rows_per_dst=[13496, 9578, 15094, 13834, 5344, 7088, 4392, 7662] dup_rows_per_dst=[7306, 1776, 4194, 6390, 130, 634, 210, 1176] combine_rows_per_src=[13496, 9578, 15094, 13834, 5344, 7088, 4392, 7662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72282 payload_rows_remote=57206 dup_rows=26022 dup_rows_remote=14462 dedup_ratio=0.264709 router_tokens_per_expert=[0, 6482, 1584, 27, 11, 78, 732, 4402, 0, 15, 0, 0, 0, 407, 2378, 183, 423, 923, 118, 243, 101, 2, 215, 687, 600, 338, 17, 196, 247, 107, 221, 1, 105, 7143, 104, 1, 1207, 0, 624, 1, 1576, 0, 6, 1, 1896, 247, 0, 2, 90, 2, 141, 1788, 161, 610, 90, 2, 1, 337, 5700, 147, 260, 2320, 87, 3765] payload_rows_per_dst=[15076, 5964, 14968, 14170, 3778, 6790, 5422, 6114] dup_rows_per_dst=[11560, 2, 4742, 8214, 306, 666, 342, 190] combine_rows_per_src=[15076, 5964, 14968, 14170, 3778, 6790, 5422, 6114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=66246 dup_rows=24000 dup_rows_remote=23276 dedup_ratio=0.244141 router_tokens_per_expert=[3335, 1459, 64, 754, 1138, 516, 138, 768, 661, 136, 194, 379, 1440, 464, 1185, 212, 2323, 584, 158, 266, 347, 326, 247, 490, 112, 2302, 287, 295, 1126, 1514, 584, 500, 366, 393, 72, 235, 5, 658, 991, 1278, 796, 3243, 2908, 236, 938, 462, 424, 173, 7, 207, 213, 2332, 347, 148, 769, 368, 299, 196, 1967, 625, 1013, 2670, 216, 293] payload_rows_per_dst=[10962, 7412, 8570, 10458, 7148, 12234, 8058, 9462] dup_rows_per_dst=[5382, 1930, 2162, 2982, 848, 6126, 724, 3846] combine_rows_per_src=[10962, 7412, 8570, 10458, 7148, 12234, 8058, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=242704384 dispatch_meta_bytes=319936 combine_payload_bytes=242704384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=243024320 ingress_bytes=311910400 link_bytes=311910400 token_bytes=485728704 weight_movement_bytes=69206016 total_nvlink_bytes=554934720 weight_to_token_ratio=0.142479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173693724e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72268 payload_rows_remote=56904 dup_rows=26036 dup_rows_remote=17576 dedup_ratio=0.264852 router_tokens_per_expert=[21, 0, 0, 348, 0, 0, 0, 0, 432, 187, 569, 0, 882, 344, 0, 116, 123, 0, 6, 0, 0, 83, 0, 0, 1, 3080, 0, 296, 261, 1272, 0, 0, 1852, 1356, 0, 0, 0, 1294, 0, 1203, 1166, 1841, 6375, 0, 551, 592, 585, 0, 0, 302, 0, 4545, 0, 0, 144, 0, 989, 0, 4585, 5415, 4258, 3894, 0, 184] payload_rows_per_dst=[738, 3476, 11844, 9358, 15364, 8220, 9742, 13526] dup_rows_per_dst=[0, 1584, 1590, 1362, 8460, 1586, 240, 11214] combine_rows_per_src=[738, 3476, 11844, 9358, 15364, 8220, 9742, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=287285248 dispatch_meta_bytes=339560 combine_payload_bytes=287285248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287624808 ingress_bytes=391094272 link_bytes=391094272 token_bytes=574910056 weight_movement_bytes=103809024 total_nvlink_bytes=678719080 weight_to_token_ratio=0.180566 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471656260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=347800 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274411160 ingress_bytes=308666368 link_bytes=308666368 token_bytes=548474520 weight_movement_bytes=34603008 total_nvlink_bytes=583077528 weight_to_token_ratio=0.063090 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161486693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234315776 dispatch_meta_bytes=286672 combine_payload_bytes=234315776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=234602448 ingress_bytes=303521792 link_bytes=303521792 token_bytes=468918224 weight_movement_bytes=69206016 total_nvlink_bytes=538124240 weight_to_token_ratio=0.147587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142128068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271343616 dispatch_meta_bytes=358088 combine_payload_bytes=271343616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271701704 ingress_bytes=305946624 link_bytes=305946624 token_bytes=543045320 weight_movement_bytes=34603008 total_nvlink_bytes=577648328 weight_to_token_ratio=0.063720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151252515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75206 payload_rows_remote=65706 dup_rows=23098 dup_rows_remote=19716 dedup_ratio=0.234965 router_tokens_per_expert=[4179, 1128, 104, 550, 745, 470, 50, 3014, 379, 82, 187, 1702, 602, 318, 2629, 110, 1496, 469, 54, 322, 299, 259, 94, 1128, 41, 1140, 223, 191, 730, 497, 869, 548, 524, 186, 17, 462, 2, 389, 2618, 966, 298, 1358, 2583, 816, 804, 208, 107, 179, 30, 152, 1889, 2252, 309, 17, 502, 470, 126, 537, 1484, 836, 1601, 2407, 226, 188] payload_rows_per_dst=[12450, 9054, 9500, 10328, 11708, 6448, 10552, 5166] dup_rows_per_dst=[8320, 2964, 3382, 2964, 3496, 1092, 690, 190] combine_rows_per_src=[12450, 9054, 9500, 10328, 11708, 6448, 10552, 5166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233078784 dispatch_meta_bytes=297920 combine_payload_bytes=233078784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233376704 ingress_bytes=267681792 link_bytes=267681792 token_bytes=466455488 weight_movement_bytes=34603008 total_nvlink_bytes=501058496 weight_to_token_ratio=0.074183 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.007265033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78632 payload_rows_remote=70940 dup_rows=19672 dup_rows_remote=18698 dedup_ratio=0.200114 router_tokens_per_expert=[232, 464, 34, 188, 902, 32, 178, 6474, 397, 23, 0, 5524, 0, 339, 4271, 74, 0, 262, 0, 100, 1137, 96, 0, 178, 0, 73, 323, 257, 373, 0, 2588, 718, 90, 95, 0, 1310, 10, 2, 2338, 244, 27, 0, 1, 1774, 764, 44, 0, 549, 85, 144, 6660, 0, 368, 2962, 110, 680, 0, 5488, 2, 1, 0, 1, 68, 98] payload_rows_per_dst=[14770, 14250, 3066, 7692, 7436, 4574, 15736, 11108] dup_rows_per_dst=[2238, 7006, 486, 974, 744, 1742, 6282, 200] combine_rows_per_src=[14770, 14250, 3066, 7692, 7436, 4574, 15736, 11108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269131776 dispatch_meta_bytes=341688 combine_payload_bytes=269131776 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269473464 ingress_bytes=338337792 link_bytes=338337792 token_bytes=538605240 weight_movement_bytes=69206016 total_nvlink_bytes=607811256 weight_to_token_ratio=0.128491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273137873e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75938 payload_rows_remote=71064 dup_rows=22366 dup_rows_remote=20230 dedup_ratio=0.227519 router_tokens_per_expert=[276, 647, 10, 223, 1783, 85, 97, 5354, 387, 70, 0, 5240, 6, 382, 4710, 67, 13, 524, 1, 33, 1923, 72, 1, 878, 2, 85, 89, 236, 677, 0, 1710, 946, 72, 80, 0, 2233, 50, 7, 1036, 263, 46, 24, 24, 1904, 850, 73, 2, 582, 32, 61, 6244, 23, 372, 3188, 222, 584, 8, 4440, 15, 9, 18, 26, 68, 69] payload_rows_per_dst=[13646, 14304, 5542, 6414, 6796, 4874, 15270, 9092] dup_rows_per_dst=[3304, 7420, 1366, 1076, 686, 2136, 6182, 196] combine_rows_per_src=[13646, 14304, 5542, 6414, 6796, 4874, 15270, 9092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80334 payload_rows_remote=73952 dup_rows=17970 dup_rows_remote=15636 dedup_ratio=0.182800 router_tokens_per_expert=[1, 0, 0, 326, 0, 0, 0, 0, 374, 224, 1490, 0, 898, 318, 0, 109, 73, 0, 1, 0, 0, 76, 0, 0, 0, 4160, 0, 292, 251, 1133, 1, 0, 1257, 2665, 0, 0, 0, 1238, 0, 1143, 2830, 1094, 5910, 0, 544, 1059, 865, 0, 0, 301, 0, 3630, 0, 0, 130, 0, 871, 0, 3941, 5663, 3267, 2834, 0, 183] payload_rows_per_dst=[12472, 6382, 14288, 12566, 10330, 10632, 8164, 5500] dup_rows_per_dst=[2, 2334, 5220, 4776, 2276, 2722, 368, 272] combine_rows_per_src=[12472, 6382, 14288, 12566, 10330, 10632, 8164, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73492 payload_rows_remote=59182 dup_rows=24812 dup_rows_remote=13042 dedup_ratio=0.252401 router_tokens_per_expert=[0, 0, 0, 281, 0, 0, 0, 0, 400, 327, 1075, 0, 881, 336, 0, 109, 59, 0, 0, 0, 0, 78, 0, 0, 1, 3364, 0, 291, 245, 1206, 0, 0, 1335, 1996, 0, 0, 0, 1173, 0, 1075, 2153, 1197, 6487, 0, 563, 1018, 720, 0, 0, 205, 0, 3760, 0, 0, 112, 0, 939, 0, 4849, 5665, 3880, 3199, 0, 173] payload_rows_per_dst=[562, 4580, 11604, 9340, 9276, 15782, 8038, 14310] dup_rows_per_dst=[0, 1676, 0, 874, 1882, 8494, 116, 11770] combine_rows_per_src=[562, 4580, 11604, 9340, 9276, 15782, 8038, 14310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290570240 dispatch_meta_bytes=358552 combine_payload_bytes=290570240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290928792 ingress_bytes=325173248 link_bytes=325173248 token_bytes=581499032 weight_movement_bytes=34603008 total_nvlink_bytes=616102040 weight_to_token_ratio=0.059507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223600754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77724 payload_rows_remote=65878 dup_rows=20580 dup_rows_remote=15904 dedup_ratio=0.209351 router_tokens_per_expert=[565, 254, 744, 386, 809, 4321, 267, 945, 220, 326, 17, 41, 749, 424, 31, 326, 149, 16, 130, 326, 174, 859, 93, 1551, 377, 1101, 442, 395, 137, 778, 36, 237, 4079, 206, 29, 1086, 95, 286, 199, 1383, 1103, 222, 392, 934, 149, 477, 256, 657, 398, 2302, 2891, 671, 293, 344, 2286, 970, 874, 3011, 880, 541, 1131, 2291, 635, 855] payload_rows_per_dst=[12680, 7802, 10368, 11380, 5954, 6696, 11846, 10998] dup_rows_per_dst=[3902, 1038, 2250, 3784, 614, 456, 4676, 3860] combine_rows_per_src=[12680, 7802, 10368, 11380, 5954, 6696, 11846, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79838 payload_rows_remote=66384 dup_rows=18466 dup_rows_remote=18462 dedup_ratio=0.187846 router_tokens_per_expert=[0, 0, 1, 359, 0, 0, 0, 1, 437, 221, 1241, 1, 921, 337, 0, 124, 53, 0, 8, 0, 4, 100, 0, 1, 1, 3465, 0, 345, 261, 1146, 1, 20, 1015, 2374, 0, 0, 0, 1441, 0, 1097, 1803, 1636, 6368, 0, 521, 884, 976, 1, 0, 235, 3, 3689, 2, 0, 142, 0, 1042, 2, 3930, 5746, 3684, 3331, 0, 182] payload_rows_per_dst=[13454, 10042, 14240, 12416, 9884, 9386, 7964, 2452] dup_rows_per_dst=[4, 3890, 5444, 4724, 1970, 2256, 178, 0] combine_rows_per_src=[13454, 10042, 14240, 12416, 9884, 9386, 7964, 2452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85024 payload_rows_remote=72692 dup_rows=13280 dup_rows_remote=10816 dedup_ratio=0.135091 router_tokens_per_expert=[0, 70, 340, 333, 1023, 4435, 0, 462, 19, 18, 4, 0, 245, 0, 0, 0, 392, 0, 0, 0, 0, 2320, 0, 1067, 0, 363, 25, 392, 0, 0, 0, 69, 5928, 5, 92, 5047, 0, 0, 175, 2079, 358, 0, 0, 714, 83, 0, 0, 281, 266, 488, 2449, 95, 2454, 69, 6243, 0, 41, 3601, 3936, 723, 1565, 641, 0, 242] payload_rows_per_dst=[11244, 12976, 12358, 12464, 12332, 2706, 9826, 11118] dup_rows_per_dst=[2082, 82, 2402, 1090, 2464, 166, 1816, 3178] combine_rows_per_src=[11244, 12976, 12358, 12464, 12332, 2706, 9826, 11118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291078144 dispatch_meta_bytes=365176 combine_payload_bytes=291078144 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291443320 ingress_bytes=394887168 link_bytes=394887168 token_bytes=582521464 weight_movement_bytes=103809024 total_nvlink_bytes=686330488 weight_to_token_ratio=0.178206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485928622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302907392 dispatch_meta_bytes=358352 combine_payload_bytes=302907392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303265744 ingress_bytes=337510400 link_bytes=337510400 token_bytes=606173136 weight_movement_bytes=34603008 total_nvlink_bytes=640776144 weight_to_token_ratio=0.057084 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.270024463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=242409472 dispatch_meta_bytes=288896 combine_payload_bytes=242409472 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=242698368 ingress_bytes=450027520 link_bytes=450027520 token_bytes=485107840 weight_movement_bytes=207618048 total_nvlink_bytes=692725888 weight_to_token_ratio=0.427983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.693417328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=269836288 dispatch_meta_bytes=327128 combine_payload_bytes=269836288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270163416 ingress_bytes=373645312 link_bytes=373645312 token_bytes=539999704 weight_movement_bytes=103809024 total_nvlink_bytes=643808728 weight_to_token_ratio=0.192239 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405997228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77364 payload_rows_remote=67532 dup_rows=20940 dup_rows_remote=18004 dedup_ratio=0.213013 router_tokens_per_expert=[831, 232, 297, 179, 544, 3442, 220, 1013, 103, 407, 11, 0, 555, 309, 0, 788, 194, 67, 238, 416, 857, 576, 570, 1015, 430, 814, 518, 265, 64, 232, 18, 81, 3269, 13, 2, 953, 15, 2396, 114, 776, 505, 1088, 799, 452, 133, 1845, 2111, 466, 353, 2040, 2009, 205, 269, 390, 1010, 3269, 1344, 2451, 371, 199, 1284, 1052, 2512, 171] payload_rows_per_dst=[12374, 6856, 9832, 9144, 8966, 7814, 11420, 10958] dup_rows_per_dst=[3318, 1104, 2936, 2238, 3794, 586, 4056, 2908] combine_rows_per_src=[12374, 6856, 9832, 9144, 8966, 7814, 11420, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271908864 dispatch_meta_bytes=339384 combine_payload_bytes=271908864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272248248 ingress_bytes=306511872 link_bytes=306511872 token_bytes=544157112 weight_movement_bytes=34603008 total_nvlink_bytes=578760120 weight_to_token_ratio=0.063590 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153379498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297746432 dispatch_meta_bytes=334032 combine_payload_bytes=297746432 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=298080464 ingress_bytes=401555456 link_bytes=401555456 token_bytes=595826896 weight_movement_bytes=103809024 total_nvlink_bytes=699635920 weight_to_token_ratio=0.174227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511020853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276611072 dispatch_meta_bytes=342144 combine_payload_bytes=276611072 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276953216 ingress_bytes=345817088 link_bytes=345817088 token_bytes=553564288 weight_movement_bytes=69206016 total_nvlink_bytes=622770304 weight_to_token_ratio=0.125019 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301281862e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67920 payload_rows_remote=65722 dup_rows=30384 dup_rows_remote=30384 dedup_ratio=0.309082 router_tokens_per_expert=[518, 81, 1, 95, 0, 1, 2476, 338, 0, 612, 1, 0, 208, 1596, 0, 830, 1, 48, 0, 0, 902, 0, 1628, 0, 581, 483, 1, 0, 16, 0, 0, 17, 1, 0, 0, 1, 0, 6501, 75, 0, 0, 7457, 7435, 0, 95, 1803, 7396, 305, 187, 1, 0, 0, 0, 66, 1, 3292, 3, 72, 0, 0, 195, 1, 3800, 31] payload_rows_per_dst=[9508, 5454, 5202, 2198, 14916, 15624, 6964, 8054] dup_rows_per_dst=[1454, 1042, 100, 0, 8636, 19018, 128, 6] combine_rows_per_src=[9508, 5454, 5202, 2198, 14916, 15624, 6964, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72030 payload_rows_remote=56026 dup_rows=26274 dup_rows_remote=6238 dedup_ratio=0.267273 router_tokens_per_expert=[496, 135, 5, 72, 2, 33, 2633, 342, 1, 795, 2, 134, 292, 3375, 4, 962, 8, 38, 2, 4, 550, 39, 1947, 222, 546, 386, 24, 156, 136, 2, 1, 57, 21, 3, 0, 15, 0, 5912, 94, 6, 2, 6978, 7539, 64, 82, 1668, 6985, 343, 385, 28, 1, 1, 5, 71, 8, 1872, 16, 79, 2, 2, 340, 3, 3159, 67] payload_rows_per_dst=[6662, 9388, 5298, 2330, 11924, 16004, 12654, 7770] dup_rows_per_dst=[774, 1758, 480, 328, 136, 20036, 2486, 276] combine_rows_per_src=[6662, 9388, 5298, 2330, 11924, 16004, 12654, 7770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80728 payload_rows_remote=66914 dup_rows=17576 dup_rows_remote=12982 dedup_ratio=0.178792 router_tokens_per_expert=[0, 58, 300, 303, 1228, 3017, 0, 450, 0, 0, 15, 0, 184, 0, 0, 0, 893, 0, 0, 0, 0, 2439, 0, 1501, 0, 144, 57, 441, 0, 0, 0, 58, 6499, 81, 92, 5211, 0, 0, 168, 1119, 312, 0, 0, 1037, 76, 0, 0, 38, 266, 54, 2638, 92, 4425, 58, 6367, 0, 42, 3093, 3426, 910, 1233, 592, 0, 235] payload_rows_per_dst=[9054, 13814, 12514, 13662, 9354, 2774, 9604, 9952] dup_rows_per_dst=[1658, 4594, 3338, 3142, 1582, 152, 186, 2924] combine_rows_per_src=[9054, 13814, 12514, 13662, 9354, 2774, 9604, 9952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84774 payload_rows_remote=75180 dup_rows=13530 dup_rows_remote=10868 dedup_ratio=0.137634 router_tokens_per_expert=[0, 68, 315, 329, 626, 3724, 0, 453, 0, 0, 28, 0, 172, 0, 0, 0, 1019, 0, 0, 0, 0, 2411, 0, 1302, 0, 80, 38, 486, 0, 0, 0, 68, 6362, 24, 87, 5396, 0, 0, 164, 1479, 299, 0, 0, 1108, 78, 0, 0, 31, 244, 66, 2469, 86, 4308, 67, 6375, 0, 45, 3262, 3280, 762, 1300, 580, 0, 161] payload_rows_per_dst=[9722, 13100, 12672, 13116, 12122, 2876, 11572, 9594] dup_rows_per_dst=[1308, 50, 3316, 952, 2178, 156, 2908, 2662] combine_rows_per_src=[9722, 13100, 12672, 13116, 12122, 2876, 11572, 9594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=269197312 dispatch_meta_bytes=384424 combine_payload_bytes=269197312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269581736 ingress_bytes=338403328 link_bytes=338403328 token_bytes=538779048 weight_movement_bytes=69206016 total_nvlink_bytes=607985064 weight_to_token_ratio=0.128450 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273384480e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78294 payload_rows_remote=65948 dup_rows=20010 dup_rows_remote=17130 dedup_ratio=0.203552 router_tokens_per_expert=[15, 126, 750, 196, 249, 46, 876, 5, 489, 332, 70, 676, 154, 2835, 1204, 1853, 9, 3673, 143, 846, 96, 980, 426, 966, 2946, 212, 352, 1507, 154, 526, 186, 3271, 4756, 4511, 266, 403, 69, 40, 148, 302, 831, 238, 2716, 127, 36, 1065, 399, 216, 190, 144, 293, 1024, 604, 432, 58, 112, 55, 1897, 4, 512, 135, 142, 154, 1104] payload_rows_per_dst=[9094, 11774, 11252, 9766, 2366, 12320, 12346, 9376] dup_rows_per_dst=[1324, 3452, 3026, 2650, 90, 4454, 2880, 2134] combine_rows_per_src=[9094, 11774, 11252, 9766, 2366, 12320, 12346, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75728 payload_rows_remote=65090 dup_rows=22576 dup_rows_remote=20650 dedup_ratio=0.229655 router_tokens_per_expert=[0, 77, 301, 348, 1201, 3862, 1, 490, 1, 0, 7, 2, 238, 0, 0, 1, 979, 0, 0, 0, 0, 1935, 0, 1417, 0, 161, 39, 416, 0, 0, 1, 78, 6148, 11, 92, 4395, 0, 2, 190, 1369, 366, 2, 3, 1123, 100, 0, 2, 28, 239, 251, 3030, 90, 3424, 77, 6344, 0, 34, 3051, 3960, 728, 1436, 890, 1, 211] payload_rows_per_dst=[10638, 13734, 12072, 14304, 3168, 3040, 8022, 10750] dup_rows_per_dst=[1926, 5512, 3166, 8172, 164, 200, 140, 3296] combine_rows_per_src=[10638, 13734, 12072, 14304, 3168, 3040, 8022, 10750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86446 payload_rows_remote=77974 dup_rows=11858 dup_rows_remote=11700 dedup_ratio=0.120626 router_tokens_per_expert=[7, 0, 69, 92, 186, 0, 0, 0, 0, 1718, 0, 485, 91, 1086, 359, 1159, 0, 6576, 0, 538, 0, 0, 67, 0, 6637, 0, 153, 254, 0, 420, 92, 5041, 7134, 6284, 0, 443, 0, 0, 0, 0, 205, 113, 2294, 0, 0, 1319, 252, 0, 382, 0, 0, 1261, 0, 0, 0, 0, 0, 4367, 0, 0, 0, 0, 0, 68] payload_rows_per_dst=[13610, 11602, 7666, 10770, 8472, 9984, 15472, 8870] dup_rows_per_dst=[372, 4784, 106, 1150, 158, 3206, 2082, 0] combine_rows_per_src=[13610, 11602, 7666, 10770, 8472, 9984, 15472, 8870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229482496 dispatch_meta_bytes=249056 combine_payload_bytes=229482496 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=229731552 ingress_bytes=367894528 link_bytes=367894528 token_bytes=459214048 weight_movement_bytes=138412032 total_nvlink_bytes=597626080 weight_to_token_ratio=0.301411 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384357491e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274079744 dispatch_meta_bytes=319584 combine_payload_bytes=274079744 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274399328 ingress_bytes=343285760 link_bytes=343285760 token_bytes=548479072 weight_movement_bytes=69206016 total_nvlink_bytes=617685088 weight_to_token_ratio=0.126178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291756678e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307937280 dispatch_meta_bytes=344192 combine_payload_bytes=307937280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=308281472 ingress_bytes=411746304 link_bytes=411746304 token_bytes=616218752 weight_movement_bytes=103809024 total_nvlink_bytes=720027776 weight_to_token_ratio=0.168461 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.549368194e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=270123008 dispatch_meta_bytes=332312 combine_payload_bytes=270123008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270455320 ingress_bytes=304726016 link_bytes=304726016 token_bytes=540578328 weight_movement_bytes=34603008 total_nvlink_bytes=575181336 weight_to_token_ratio=0.064011 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146659466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=342960 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266951600 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533560240 weight_movement_bytes=34603008 total_nvlink_bytes=568163248 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79720 payload_rows_remote=70032 dup_rows=18584 dup_rows_remote=13072 dedup_ratio=0.189046 router_tokens_per_expert=[1, 67, 1821, 262, 65, 0, 1847, 7, 286, 309, 72, 1359, 54, 1112, 868, 642, 0, 2268, 2514, 466, 48, 443, 385, 3744, 2402, 2373, 167, 1272, 485, 340, 75, 1972, 3616, 2809, 275, 170, 1524, 251, 57, 262, 688, 95, 2029, 282, 116, 496, 254, 565, 143, 0, 232, 1897, 608, 577, 64, 459, 85, 1617, 0, 225, 428, 344, 161, 1097] payload_rows_per_dst=[10988, 9796, 9688, 10876, 10156, 8024, 12792, 7400] dup_rows_per_dst=[1956, 4144, 5512, 2492, 540, 1026, 2400, 514] combine_rows_per_src=[10988, 9796, 9688, 10876, 10156, 8024, 12792, 7400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=319381504 dispatch_meta_bytes=358696 combine_payload_bytes=319381504 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=319740200 ingress_bytes=457793536 link_bytes=457793536 token_bytes=639121704 weight_movement_bytes=138412032 total_nvlink_bytes=777533736 weight_to_token_ratio=0.216566 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722640221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73808 payload_rows_remote=58340 dup_rows=24496 dup_rows_remote=16790 dedup_ratio=0.249186 router_tokens_per_expert=[0, 0, 89, 170, 0, 0, 391, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 5480, 67, 0, 0, 228, 4695, 1, 7537, 0, 0, 4049, 0, 0, 1, 1, 0, 746, 0, 5560, 786, 1, 364, 79, 0, 1, 2682, 182, 0, 244, 6403, 0, 0, 0, 0, 189, 97, 398, 636, 839, 0, 0, 0, 1101, 6008, 124, 0] payload_rows_per_dst=[1302, 4, 14046, 15468, 13042, 14372, 2440, 13134] dup_rows_per_dst=[0, 2, 6894, 7706, 1872, 4810, 202, 3010] combine_rows_per_src=[1302, 4, 14046, 15468, 13042, 14372, 2440, 13134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=286851072 dispatch_meta_bytes=332416 combine_payload_bytes=286851072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287183488 ingress_bytes=356057088 link_bytes=356057088 token_bytes=574034560 weight_movement_bytes=69206016 total_nvlink_bytes=643240576 weight_to_token_ratio=0.120561 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339814157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81738 payload_rows_remote=66266 dup_rows=16566 dup_rows_remote=10726 dedup_ratio=0.168518 router_tokens_per_expert=[0, 0, 58, 88, 166, 0, 0, 0, 0, 1335, 0, 388, 92, 485, 325, 1127, 0, 6904, 0, 577, 0, 0, 73, 0, 6759, 0, 385, 227, 0, 525, 92, 5553, 6947, 6505, 0, 834, 0, 0, 0, 0, 92, 73, 1134, 0, 0, 1446, 227, 0, 424, 0, 0, 2520, 0, 0, 0, 0, 0, 3717, 0, 0, 0, 0, 0, 74] payload_rows_per_dst=[14274, 15472, 4738, 5460, 13690, 5378, 15144, 7582] dup_rows_per_dst=[3944, 5840, 108, 482, 988, 566, 4638, 0] combine_rows_per_src=[14274, 15472, 4738, 5460, 13690, 5378, 15144, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74840 payload_rows_remote=59674 dup_rows=23464 dup_rows_remote=17754 dedup_ratio=0.238688 router_tokens_per_expert=[0, 0, 264, 186, 4, 25, 414, 0, 5, 1, 2, 35, 1, 2, 247, 11, 0, 15, 5121, 74, 0, 8, 329, 4062, 19, 7059, 1, 256, 1850, 4, 1, 10, 32, 25, 2511, 1, 4315, 1025, 5, 510, 75, 27, 10, 3038, 138, 6, 253, 6866, 2, 0, 2, 21, 259, 225, 379, 762, 1104, 17, 0, 1, 1548, 5910, 67, 12] payload_rows_per_dst=[1582, 586, 13314, 14750, 12858, 15166, 2900, 13684] dup_rows_per_dst=[242, 22, 5904, 3612, 3876, 5710, 464, 3634] combine_rows_per_src=[1582, 586, 13314, 14750, 12858, 15166, 2900, 13684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=238960640 dispatch_meta_bytes=300520 combine_payload_bytes=238960640 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=239261160 ingress_bytes=377372672 link_bytes=377372672 token_bytes=478221800 weight_movement_bytes=138412032 total_nvlink_bytes=616633832 weight_to_token_ratio=0.289431 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420022984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75826 payload_rows_remote=71124 dup_rows=22478 dup_rows_remote=21252 dedup_ratio=0.228658 router_tokens_per_expert=[75, 177, 332, 4212, 354, 17, 1708, 7, 1, 6434, 1190, 1, 115, 196, 917, 767, 2008, 356, 54, 2800, 585, 83, 74, 25, 1510, 705, 35, 710, 40, 261, 12, 116, 828, 252, 165, 62, 1043, 3219, 297, 147, 163, 117, 1032, 95, 168, 164, 1699, 213, 902, 116, 573, 31, 451, 244, 105, 346, 1627, 1, 36, 1340, 5342, 1095, 1207, 195] payload_rows_per_dst=[12140, 11968, 11706, 6158, 10460, 6424, 4702, 12268] dup_rows_per_dst=[4674, 3602, 6564, 620, 1566, 878, 1226, 3348] combine_rows_per_src=[12140, 11968, 11706, 6158, 10460, 6424, 4702, 12268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84488 payload_rows_remote=69642 dup_rows=13816 dup_rows_remote=7530 dedup_ratio=0.140544 router_tokens_per_expert=[1, 0, 68, 72, 203, 0, 0, 0, 0, 1634, 0, 409, 87, 664, 368, 1125, 2, 6986, 0, 480, 0, 0, 59, 0, 7009, 0, 228, 251, 0, 466, 86, 5456, 6961, 6550, 0, 530, 0, 0, 0, 0, 87, 57, 1278, 0, 0, 1312, 251, 0, 404, 0, 0, 2052, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[14300, 7134, 14842, 9864, 2956, 5436, 15110, 14846] dup_rows_per_dst=[406, 1440, 212, 1084, 130, 534, 3724, 6286] combine_rows_per_src=[14300, 7134, 14842, 9864, 2956, 5436, 15110, 14846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69344 payload_rows_remote=53972 dup_rows=28960 dup_rows_remote=19422 dedup_ratio=0.294596 router_tokens_per_expert=[1, 1, 77, 91, 156, 1, 0, 0, 0, 1644, 0, 411, 92, 817, 336, 957, 0, 6869, 1, 444, 0, 0, 53, 0, 6678, 2, 248, 247, 1, 499, 90, 5450, 6924, 6177, 0, 796, 2, 0, 0, 0, 92, 53, 1918, 3, 0, 1344, 248, 1, 417, 0, 0, 2093, 0, 0, 1, 0, 0, 3859, 0, 0, 1, 2, 0, 55] payload_rows_per_dst=[15372, 15350, 996, 2172, 15428, 6682, 5512, 7832] dup_rows_per_dst=[9538, 6902, 0, 2, 11682, 636, 198, 2] combine_rows_per_src=[15372, 15350, 996, 2172, 15428, 6682, 5512, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83328 payload_rows_remote=75288 dup_rows=14976 dup_rows_remote=13716 dedup_ratio=0.152344 router_tokens_per_expert=[0, 0, 150, 2857, 0, 0, 1063, 0, 0, 3858, 1265, 0, 0, 0, 668, 0, 83, 379, 0, 2776, 0, 43, 0, 0, 6219, 82, 0, 102, 0, 0, 0, 0, 255, 480, 0, 0, 2852, 740, 231, 92, 2, 0, 5364, 0, 0, 0, 380, 0, 620, 0, 92, 0, 3138, 5, 1, 147, 6975, 0, 0, 2, 6640, 668, 923, 0] payload_rows_per_dst=[12100, 3334, 13742, 12806, 8040, 11150, 7122, 15034] dup_rows_per_dst=[3756, 532, 6100, 0, 1260, 342, 884, 2102] combine_rows_per_src=[12100, 3334, 13742, 12806, 8040, 11150, 7122, 15034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=271425536 dispatch_meta_bytes=307968 combine_payload_bytes=271425536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271733504 ingress_bytes=306028544 link_bytes=306028544 token_bytes=543159040 weight_movement_bytes=34603008 total_nvlink_bytes=577762048 weight_to_token_ratio=0.063707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151560774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=244424704 dispatch_meta_bytes=309712 combine_payload_bytes=244424704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244734416 ingress_bytes=279027712 link_bytes=279027712 token_bytes=489159120 weight_movement_bytes=34603008 total_nvlink_bytes=523762128 weight_to_token_ratio=0.070740 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.049958817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=291323904 dispatch_meta_bytes=369504 combine_payload_bytes=291323904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291693408 ingress_bytes=325926912 link_bytes=325926912 token_bytes=583017312 weight_movement_bytes=34603008 total_nvlink_bytes=617620320 weight_to_token_ratio=0.059352 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226436731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73386 payload_rows_remote=62942 dup_rows=24918 dup_rows_remote=20814 dedup_ratio=0.253479 router_tokens_per_expert=[391, 325, 326, 5019, 1352, 0, 588, 30, 3, 6208, 627, 325, 33, 336, 745, 3335, 561, 115, 95, 1545, 300, 59, 203, 2, 1637, 676, 83, 339, 22, 368, 68, 35, 519, 140, 503, 308, 651, 1772, 248, 331, 163, 653, 729, 36, 401, 161, 995, 781, 2702, 104, 409, 244, 253, 468, 3, 541, 1845, 15, 166, 2080, 4394, 879, 742, 165] payload_rows_per_dst=[14938, 9560, 10444, 6114, 8124, 6358, 7550, 10298] dup_rows_per_dst=[13540, 1248, 4104, 342, 820, 1480, 1898, 1486] combine_rows_per_src=[14938, 9560, 10444, 6114, 8124, 6358, 7550, 10298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285253632 dispatch_meta_bytes=308688 combine_payload_bytes=285253632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285562320 ingress_bytes=319856640 link_bytes=319856640 token_bytes=570815952 weight_movement_bytes=34603008 total_nvlink_bytes=605418960 weight_to_token_ratio=0.060620 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203594786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=221069312 dispatch_meta_bytes=293576 combine_payload_bytes=221069312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=221362888 ingress_bytes=290275328 link_bytes=290275328 token_bytes=442432200 weight_movement_bytes=69206016 total_nvlink_bytes=511638216 weight_to_token_ratio=0.156422 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092282690e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=308379648 dispatch_meta_bytes=356016 combine_payload_bytes=308379648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308735664 ingress_bytes=342982656 link_bytes=342982656 token_bytes=617115312 weight_movement_bytes=34603008 total_nvlink_bytes=651718320 weight_to_token_ratio=0.056072 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290616122e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72352 payload_rows_remote=65324 dup_rows=25952 dup_rows_remote=25950 dedup_ratio=0.263997 router_tokens_per_expert=[655, 403, 48, 28, 1, 0, 0, 0, 0, 0, 0, 381, 6420, 8190, 0, 48, 0, 0, 1118, 1, 0, 0, 102, 0, 0, 1, 0, 0, 0, 68, 19, 3427, 0, 0, 3092, 3664, 0, 2, 244, 95, 0, 3950, 1, 0, 786, 0, 0, 8171, 285, 6428, 44, 0, 1, 756, 1, 63, 1, 0, 591, 0, 1, 66, 0, 0] payload_rows_per_dst=[2214, 16382, 2442, 7028, 12330, 16344, 14296, 1316] dup_rows_per_dst=[56, 13696, 2, 2, 1864, 9472, 860, 0] combine_rows_per_src=[2214, 16382, 2442, 7028, 12330, 16344, 14296, 1316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=257810432 dispatch_meta_bytes=335024 combine_payload_bytes=257810432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258145456 ingress_bytes=292413440 link_bytes=292413440 token_bytes=515955888 weight_movement_bytes=34603008 total_nvlink_bytes=550558896 weight_to_token_ratio=0.067066 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100328234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78988 payload_rows_remote=74260 dup_rows=19316 dup_rows_remote=18852 dedup_ratio=0.196493 router_tokens_per_expert=[0, 0, 143, 2301, 0, 0, 1585, 0, 0, 2077, 1960, 0, 0, 0, 636, 0, 76, 303, 0, 1778, 0, 7, 0, 0, 6297, 51, 0, 188, 0, 0, 0, 0, 203, 338, 0, 0, 2665, 439, 57, 92, 0, 0, 5807, 0, 0, 0, 318, 0, 533, 0, 92, 0, 5658, 3, 25, 158, 7085, 0, 0, 0, 6560, 698, 1019, 0] payload_rows_per_dst=[10132, 4728, 13516, 14542, 6902, 6258, 11996, 10914] dup_rows_per_dst=[2080, 464, 3932, 9458, 686, 198, 942, 1556] combine_rows_per_src=[10132, 4728, 13516, 14542, 6902, 6258, 11996, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71388 payload_rows_remote=55240 dup_rows=26916 dup_rows_remote=17470 dedup_ratio=0.273804 router_tokens_per_expert=[347, 456, 60, 106, 25, 0, 3, 0, 0, 43, 14, 436, 6273, 8004, 10, 97, 1, 2, 1508, 14, 1, 1, 137, 0, 13, 15, 0, 2, 1, 225, 99, 1679, 2, 1, 4481, 3316, 6, 27, 253, 68, 1, 3089, 10, 1, 1642, 1, 6, 8047, 342, 5556, 69, 34, 6, 742, 55, 73, 14, 0, 1332, 2, 43, 347, 9, 5] payload_rows_per_dst=[1966, 12116, 3380, 4054, 13194, 16148, 14952, 5578] dup_rows_per_dst=[114, 5204, 34, 14, 3114, 9446, 8500, 490] combine_rows_per_src=[1966, 12116, 3380, 4054, 13194, 16148, 14952, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267567104 dispatch_meta_bytes=365096 combine_payload_bytes=267567104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267932200 ingress_bytes=302170112 link_bytes=302170112 token_bytes=535499304 weight_movement_bytes=34603008 total_nvlink_bytes=570102312 weight_to_token_ratio=0.064618 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137041805e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76066 payload_rows_remote=63674 dup_rows=22238 dup_rows_remote=17382 dedup_ratio=0.226217 router_tokens_per_expert=[2022, 70, 228, 366, 2488, 571, 494, 81, 3069, 163, 4459, 39, 33, 13, 22, 13, 2, 24, 99, 181, 77, 105, 72, 101, 95, 11, 24, 61, 82, 79, 9, 66, 56, 10, 6319, 182, 81, 553, 3617, 3458, 32, 3676, 425, 2076, 67, 152, 2487, 779, 14, 65, 50, 86, 1372, 34, 440, 2946, 599, 240, 80, 361, 628, 2655, 315, 78] payload_rows_per_dst=[10278, 12022, 946, 7952, 14986, 9344, 12392, 8146] dup_rows_per_dst=[1906, 3600, 376, 254, 6332, 2692, 4856, 2222] combine_rows_per_src=[10278, 12022, 946, 7952, 14986, 9344, 12392, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79456 payload_rows_remote=63228 dup_rows=18848 dup_rows_remote=4316 dedup_ratio=0.191732 router_tokens_per_expert=[0, 0, 74, 2276, 0, 0, 1423, 0, 0, 2473, 1916, 0, 0, 0, 706, 0, 78, 331, 0, 1987, 1, 4, 0, 0, 6149, 65, 0, 121, 0, 0, 0, 0, 260, 357, 0, 0, 3110, 528, 71, 86, 0, 0, 5687, 0, 0, 0, 374, 0, 527, 0, 86, 0, 4925, 2, 13, 142, 7154, 0, 0, 1, 6662, 670, 893, 0] payload_rows_per_dst=[7202, 8600, 4490, 12670, 7896, 11870, 10500, 16228] dup_rows_per_dst=[344, 1590, 312, 0, 928, 252, 890, 14532] combine_rows_per_src=[7202, 8600, 4490, 12670, 7896, 11870, 10500, 16228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=63620 dup_rows=23802 dup_rows_remote=21636 dedup_ratio=0.242126 router_tokens_per_expert=[0, 0, 118, 1991, 0, 3, 1461, 0, 0, 2951, 1970, 1, 2, 3, 664, 0, 100, 348, 1, 1723, 0, 2, 0, 0, 6346, 83, 0, 108, 1, 1, 0, 1, 402, 375, 1, 0, 2337, 536, 221, 90, 2, 1, 5708, 0, 0, 0, 397, 2, 679, 2, 90, 0, 5049, 9, 20, 144, 7038, 0, 1, 0, 6594, 699, 877, 0] payload_rows_per_dst=[10882, 4778, 13622, 14412, 15296, 1624, 11070, 2818] dup_rows_per_dst=[2166, 502, 3914, 10084, 5824, 60, 916, 336] combine_rows_per_src=[10882, 4778, 13622, 14412, 15296, 1624, 11070, 2818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84340 payload_rows_remote=73028 dup_rows=13964 dup_rows_remote=12666 dedup_ratio=0.142049 router_tokens_per_expert=[2621, 0, 0, 218, 834, 2143, 0, 0, 1177, 83, 1347, 0, 0, 0, 0, 0, 0, 92, 70, 0, 0, 92, 0, 0, 53, 0, 0, 0, 63, 92, 0, 10, 0, 0, 7158, 266, 0, 0, 3929, 1181, 0, 7266, 960, 6985, 69, 0, 5409, 91, 0, 0, 0, 92, 188, 0, 100, 4834, 317, 0, 0, 318, 593, 213, 254, 34] payload_rows_per_dst=[10352, 8658, 10172, 14848, 11312, 13336, 13496, 2166] dup_rows_per_dst=[1280, 1156, 280, 120, 1298, 3748, 4790, 1292] combine_rows_per_src=[10352, 8658, 10172, 14848, 11312, 13336, 13496, 2166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=304168960 dispatch_meta_bytes=372448 combine_payload_bytes=304168960 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=304541408 ingress_bytes=407977984 link_bytes=407977984 token_bytes=608710368 weight_movement_bytes=103809024 total_nvlink_bytes=712519392 weight_to_token_ratio=0.170539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.535188309e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=226263040 dispatch_meta_bytes=290840 combine_payload_bytes=226263040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=226553880 ingress_bytes=295469056 link_bytes=295469056 token_bytes=452816920 weight_movement_bytes=69206016 total_nvlink_bytes=522022936 weight_to_token_ratio=0.152834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111826271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77624 payload_rows_remote=71056 dup_rows=20680 dup_rows_remote=20388 dedup_ratio=0.210368 router_tokens_per_expert=[1264, 68, 76, 156, 1480, 345, 1419, 154, 2617, 349, 4234, 951, 196, 204, 123, 0, 0, 13, 8, 0, 46, 360, 1351, 0, 25, 0, 15, 2056, 41, 62, 0, 3, 206, 74, 4433, 177, 0, 2172, 3450, 2006, 0, 2272, 166, 1652, 37, 65, 1922, 2617, 25, 19, 0, 62, 710, 5, 437, 3626, 327, 438, 145, 423, 658, 3155, 255, 2] payload_rows_per_dst=[8434, 15012, 6568, 8854, 9600, 8196, 12108, 8852] dup_rows_per_dst=[1490, 11202, 292, 94, 642, 446, 4560, 1954] combine_rows_per_src=[8434, 15012, 6568, 8854, 9600, 8196, 12108, 8852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=258981888 dispatch_meta_bytes=270176 combine_payload_bytes=258981888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259252064 ingress_bytes=397393920 link_bytes=397393920 token_bytes=518233952 weight_movement_bytes=138412032 total_nvlink_bytes=656645984 weight_to_token_ratio=0.267084 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.495361328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260808704 dispatch_meta_bytes=324224 combine_payload_bytes=260808704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261132928 ingress_bytes=295411712 link_bytes=295411712 token_bytes=521941632 weight_movement_bytes=34603008 total_nvlink_bytes=556544640 weight_to_token_ratio=0.066297 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111610490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260587520 dispatch_meta_bytes=341024 combine_payload_bytes=260587520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260928544 ingress_bytes=295190528 link_bytes=295190528 token_bytes=521516064 weight_movement_bytes=34603008 total_nvlink_bytes=556119072 weight_to_token_ratio=0.066351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110778192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299122688 dispatch_meta_bytes=342776 combine_payload_bytes=299122688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299465464 ingress_bytes=402931712 link_bytes=402931712 token_bytes=598588152 weight_movement_bytes=103809024 total_nvlink_bytes=702397176 weight_to_token_ratio=0.173423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516199594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71614 payload_rows_remote=60550 dup_rows=26690 dup_rows_remote=26690 dedup_ratio=0.271505 router_tokens_per_expert=[0, 3, 7999, 0, 0, 1, 1910, 0, 0, 0, 0, 1, 186, 253, 243, 0, 0, 0, 0, 5621, 286, 95, 6151, 248, 0, 1, 0, 5524, 0, 6, 0, 0, 3270, 0, 1, 0, 0, 602, 0, 0, 0, 1, 1, 0, 18, 7179, 1, 188, 986, 68, 0, 0, 0, 303, 0, 2, 0, 7345, 335, 0, 323, 1, 0, 0] payload_rows_per_dst=[4280, 1318, 14772, 11064, 7166, 14402, 2588, 16024] dup_rows_per_dst=[180, 50, 10030, 0, 578, 372, 130, 15350] combine_rows_per_src=[4280, 1318, 14772, 11064, 7166, 14402, 2588, 16024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291045376 dispatch_meta_bytes=365776 combine_payload_bytes=291045376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291411152 ingress_bytes=325648384 link_bytes=325648384 token_bytes=582456528 weight_movement_bytes=34603008 total_nvlink_bytes=617059536 weight_to_token_ratio=0.059409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225388652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85322 payload_rows_remote=69794 dup_rows=12982 dup_rows_remote=8428 dedup_ratio=0.132060 router_tokens_per_expert=[2883, 0, 0, 101, 735, 3965, 0, 0, 1467, 75, 1093, 0, 0, 0, 0, 0, 0, 92, 74, 0, 0, 92, 0, 0, 75, 0, 0, 0, 76, 92, 0, 24, 0, 0, 7406, 1, 0, 0, 3870, 1047, 0, 7338, 917, 7549, 58, 0, 4698, 91, 0, 0, 0, 92, 143, 0, 0, 3319, 283, 0, 0, 340, 609, 216, 241, 90] payload_rows_per_dst=[12624, 15528, 15246, 14952, 10636, 2132, 11914, 2290] dup_rows_per_dst=[2744, 4554, 368, 854, 260, 0, 2934, 1268] combine_rows_per_src=[12624, 15528, 15246, 14952, 10636, 2132, 11914, 2290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=248012800 dispatch_meta_bytes=348960 combine_payload_bytes=248012800 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=248361760 ingress_bytes=317218816 link_bytes=317218816 token_bytes=496374560 weight_movement_bytes=69206016 total_nvlink_bytes=565580576 weight_to_token_ratio=0.139423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=35328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193668866e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72456 payload_rows_remote=58668 dup_rows=25848 dup_rows_remote=25626 dedup_ratio=0.262939 router_tokens_per_expert=[14, 68, 7953, 1, 23, 1, 2069, 42, 8, 5, 20, 37, 474, 337, 251, 0, 0, 0, 0, 5550, 314, 70, 5652, 313, 1, 0, 0, 4215, 0, 65, 0, 0, 4101, 6, 34, 4, 0, 874, 23, 17, 0, 13, 5, 13, 63, 6744, 18, 162, 862, 105, 0, 1, 8, 349, 411, 34, 8, 6999, 318, 1, 442, 47, 4, 3] payload_rows_per_dst=[4416, 2022, 14498, 8586, 9532, 13788, 3420, 16194] dup_rows_per_dst=[20, 242, 9300, 2, 540, 222, 166, 15356] combine_rows_per_src=[4416, 2022, 14498, 8586, 9532, 13788, 3420, 16194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78650 payload_rows_remote=76558 dup_rows=19654 dup_rows_remote=18358 dedup_ratio=0.199931 router_tokens_per_expert=[2708, 0, 0, 123, 781, 4086, 0, 0, 1136, 77, 725, 0, 0, 0, 0, 0, 0, 87, 89, 0, 0, 86, 0, 0, 70, 0, 0, 0, 48, 87, 0, 16, 0, 0, 7443, 1, 0, 0, 3626, 936, 0, 7463, 966, 7329, 68, 0, 5248, 85, 0, 0, 0, 87, 72, 0, 0, 4015, 309, 0, 0, 331, 586, 133, 263, 72] payload_rows_per_dst=[12918, 3692, 524, 15170, 15576, 15892, 12786, 2092] dup_rows_per_dst=[2478, 184, 0, 198, 1184, 11500, 2814, 1296] combine_rows_per_src=[12918, 3692, 524, 15170, 15576, 15892, 12786, 2092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80776 payload_rows_remote=67826 dup_rows=17528 dup_rows_remote=14644 dedup_ratio=0.178304 router_tokens_per_expert=[901, 208, 9, 366, 48, 42, 1085, 75, 102, 4351, 97, 1659, 208, 413, 1632, 553, 80, 160, 2297, 765, 22, 515, 616, 15, 2, 40, 1305, 3463, 127, 110, 770, 347, 231, 69, 16, 3434, 219, 190, 93, 4759, 21, 170, 108, 80, 465, 206, 3641, 123, 4894, 82, 22, 652, 0, 518, 779, 800, 91, 69, 3, 257, 3126, 146, 381, 1124] payload_rows_per_dst=[4684, 13254, 7988, 10532, 9198, 9074, 12950, 13096] dup_rows_per_dst=[784, 4776, 952, 1796, 2162, 214, 2884, 3960] combine_rows_per_src=[4684, 13254, 7988, 10532, 9198, 9074, 12950, 13096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=285876224 dispatch_meta_bytes=312888 combine_payload_bytes=285876224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286189112 ingress_bytes=320479232 link_bytes=320479232 token_bytes=572065336 weight_movement_bytes=34603008 total_nvlink_bytes=606668344 weight_to_token_ratio=0.060488 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205937549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80860 payload_rows_remote=65720 dup_rows=17444 dup_rows_remote=7680 dedup_ratio=0.177450 router_tokens_per_expert=[2912, 0, 3, 141, 760, 3863, 0, 0, 1461, 100, 713, 0, 0, 1, 0, 0, 0, 92, 126, 3, 1, 90, 1, 0, 62, 0, 0, 1, 100, 92, 0, 0, 0, 0, 7257, 4, 0, 1, 3484, 963, 0, 7228, 948, 7153, 77, 2, 4968, 90, 1, 1, 0, 92, 122, 0, 1, 4337, 455, 3, 0, 348, 631, 171, 261, 32] payload_rows_per_dst=[15140, 15414, 14564, 14834, 7240, 2234, 9238, 2196] dup_rows_per_dst=[9764, 3650, 368, 522, 1422, 0, 112, 1606] combine_rows_per_src=[15140, 15414, 14564, 14834, 7240, 2234, 9238, 2196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=240304128 dispatch_meta_bytes=337176 combine_payload_bytes=240304128 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=240641304 ingress_bytes=413319168 link_bytes=413319168 token_bytes=480945432 weight_movement_bytes=173015040 total_nvlink_bytes=653960472 weight_to_token_ratio=0.359739 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555286754e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86712 payload_rows_remote=77490 dup_rows=11592 dup_rows_remote=11528 dedup_ratio=0.117920 router_tokens_per_expert=[4789, 941, 0, 0, 0, 0, 1831, 66, 0, 5440, 0, 1628, 178, 0, 1009, 0, 0, 184, 4741, 6, 0, 414, 82, 0, 0, 0, 2087, 759, 1184, 0, 840, 336, 0, 0, 0, 72, 0, 2, 0, 4569, 0, 0, 170, 69, 1460, 575, 5187, 0, 6366, 80, 0, 204, 0, 92, 37, 85, 83, 0, 0, 0, 2584, 92, 2, 908] payload_rows_per_dst=[12102, 12910, 10638, 8976, 9222, 12920, 13122, 6822] dup_rows_per_dst=[3152, 3600, 216, 1436, 64, 2002, 606, 516] combine_rows_per_src=[12102, 12910, 10638, 8976, 9222, 12920, 13122, 6822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77680 payload_rows_remote=66502 dup_rows=20624 dup_rows_remote=18608 dedup_ratio=0.209798 router_tokens_per_expert=[666, 131, 27, 861, 0, 22, 346, 2, 412, 4812, 58, 757, 1285, 305, 1035, 634, 116, 26, 3630, 785, 19, 371, 995, 655, 1134, 65, 1419, 4400, 70, 19, 256, 272, 166, 0, 0, 2612, 10, 77, 378, 3220, 76, 295, 21, 16, 560, 118, 3784, 59, 5762, 43, 9, 391, 70, 228, 1784, 827, 241, 42, 0, 35, 1345, 109, 159, 1130] payload_rows_per_dst=[7372, 13458, 11178, 12314, 10276, 3990, 13626, 5466] dup_rows_per_dst=[1370, 5470, 2016, 2956, 3128, 426, 4602, 656] combine_rows_per_src=[7372, 13458, 11178, 12314, 10276, 3990, 13626, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=313581568 dispatch_meta_bytes=379664 combine_payload_bytes=313581568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313961232 ingress_bytes=348184576 link_bytes=348184576 token_bytes=627542800 weight_movement_bytes=34603008 total_nvlink_bytes=662145808 weight_to_token_ratio=0.055140 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310190528e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277815296 dispatch_meta_bytes=329880 combine_payload_bytes=277815296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278145176 ingress_bytes=312418304 link_bytes=312418304 token_bytes=555960472 weight_movement_bytes=34603008 total_nvlink_bytes=590563480 weight_to_token_ratio=0.062240 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175604926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=269189120 dispatch_meta_bytes=293600 combine_payload_bytes=269189120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269482720 ingress_bytes=338395136 link_bytes=338395136 token_bytes=538671840 weight_movement_bytes=69206016 total_nvlink_bytes=607877856 weight_to_token_ratio=0.128475 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273353654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69236 payload_rows_remote=61888 dup_rows=29068 dup_rows_remote=28872 dedup_ratio=0.295695 router_tokens_per_expert=[0, 0, 279, 1, 0, 100, 1, 0, 1061, 2, 1273, 0, 2, 0, 0, 0, 331, 0, 1, 1, 85, 0, 95, 3450, 60, 305, 1, 0, 1, 3404, 1, 0, 8190, 0, 0, 0, 7973, 0, 134, 0, 88, 7857, 0, 0, 0, 0, 0, 6533, 0, 0, 0, 0, 0, 0, 1, 0, 1, 94, 0, 7729, 1, 97, 0, 0] payload_rows_per_dst=[762, 5264, 7794, 7348, 16346, 16068, 2, 15652] dup_rows_per_dst=[0, 1818, 132, 196, 13842, 12888, 0, 192] combine_rows_per_src=[762, 5264, 7794, 7348, 16346, 16068, 2, 15652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317399040 dispatch_meta_bytes=356072 combine_payload_bytes=317399040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=317755112 ingress_bytes=455811072 link_bytes=455811072 token_bytes=635154152 weight_movement_bytes=138412032 total_nvlink_bytes=773566184 weight_to_token_ratio=0.217919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715180368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272392192 dispatch_meta_bytes=340440 combine_payload_bytes=272392192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272732632 ingress_bytes=306995200 link_bytes=306995200 token_bytes=545124824 weight_movement_bytes=34603008 total_nvlink_bytes=579727832 weight_to_token_ratio=0.063477 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155198222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77080 payload_rows_remote=65984 dup_rows=21224 dup_rows_remote=18986 dedup_ratio=0.215902 router_tokens_per_expert=[4365, 863, 0, 0, 0, 0, 2387, 82, 0, 4361, 0, 1383, 130, 0, 793, 0, 0, 184, 4953, 0, 0, 536, 75, 0, 0, 0, 1735, 722, 2434, 0, 1086, 426, 0, 0, 0, 0, 0, 3, 0, 4446, 0, 0, 261, 61, 1491, 504, 5181, 0, 5866, 72, 0, 185, 0, 92, 0, 0, 76, 0, 0, 0, 3366, 92, 0, 941] payload_rows_per_dst=[14652, 11096, 13458, 10754, 2270, 4506, 11936, 8408] dup_rows_per_dst=[11104, 2238, 4666, 2052, 0, 128, 494, 542] combine_rows_per_src=[14652, 11096, 13458, 10754, 2270, 4506, 11936, 8408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=253493248 dispatch_meta_bytes=363040 combine_payload_bytes=253493248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253856288 ingress_bytes=288096256 link_bytes=288096256 token_bytes=507349536 weight_movement_bytes=34603008 total_nvlink_bytes=541952544 weight_to_token_ratio=0.068203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.084083018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77406 payload_rows_remote=63718 dup_rows=20898 dup_rows_remote=17470 dedup_ratio=0.212585 router_tokens_per_expert=[6, 1, 269, 57, 0, 309, 9, 0, 1241, 65, 1344, 7, 72, 4, 7, 9, 533, 0, 29, 13, 151, 6, 86, 3170, 253, 264, 3, 54, 19, 3251, 9, 6, 8115, 0, 28, 7, 7564, 0, 163, 16, 339, 7578, 4, 0, 4, 8, 19, 5957, 45, 0, 0, 4, 3, 5, 11, 20, 6, 67, 1, 7834, 26, 68, 0, 13] payload_rows_per_dst=[1218, 4264, 7766, 7468, 16270, 13688, 10854, 15878] dup_rows_per_dst=[84, 1234, 210, 250, 15516, 3428, 24, 152] combine_rows_per_src=[1218, 4264, 7766, 7468, 16270, 13688, 10854, 15878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85796 payload_rows_remote=70640 dup_rows=12508 dup_rows_remote=9770 dedup_ratio=0.127238 router_tokens_per_expert=[3, 95, 848, 729, 56, 5, 3792, 29, 132, 169, 3876, 2294, 6858, 2, 241, 4503, 12, 110, 3, 104, 30, 2206, 185, 162, 78, 35, 82, 7312, 166, 262, 931, 278, 284, 7, 260, 1, 81, 704, 131, 37, 5, 2, 386, 0, 136, 310, 96, 1542, 1, 58, 326, 380, 186, 824, 230, 84, 123, 5061, 0, 344, 1470, 269, 185, 41] payload_rows_per_dst=[9726, 10602, 5528, 3482, 15610, 11922, 15156, 13770] dup_rows_per_dst=[1388, 2344, 96, 664, 2024, 2038, 2738, 1216] combine_rows_per_src=[9726, 10602, 5528, 3482, 15610, 11922, 15156, 13770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80502 payload_rows_remote=67538 dup_rows=17802 dup_rows_remote=13446 dedup_ratio=0.181091 router_tokens_per_expert=[4931, 826, 0, 0, 0, 0, 2114, 86, 0, 4244, 0, 1808, 143, 0, 843, 0, 0, 173, 4931, 1, 0, 483, 65, 0, 0, 0, 2136, 752, 1974, 0, 1073, 345, 0, 0, 0, 0, 0, 1, 0, 4156, 0, 0, 224, 74, 1518, 430, 4729, 0, 6268, 54, 0, 179, 0, 86, 0, 1, 77, 0, 0, 0, 3533, 87, 0, 807] payload_rows_per_dst=[12528, 5642, 11034, 13464, 2, 12138, 12730, 12964] dup_rows_per_dst=[3386, 62, 272, 7468, 0, 1812, 446, 4356] combine_rows_per_src=[12528, 5642, 11034, 13464, 2, 12138, 12730, 12964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=270270464 dispatch_meta_bytes=339880 combine_payload_bytes=270270464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270610344 ingress_bytes=339476480 link_bytes=339476480 token_bytes=540880808 weight_movement_bytes=69206016 total_nvlink_bytes=610086824 weight_to_token_ratio=0.127951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277422664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76254 payload_rows_remote=61618 dup_rows=22050 dup_rows_remote=11580 dedup_ratio=0.224304 router_tokens_per_expert=[4637, 1119, 0, 0, 0, 0, 1837, 88, 1, 3820, 0, 1692, 183, 0, 910, 0, 1, 182, 4522, 8, 0, 459, 100, 0, 0, 2, 2314, 710, 1155, 0, 1120, 346, 3, 2, 0, 0, 2, 0, 0, 4720, 0, 2, 165, 100, 1729, 513, 4872, 2, 6617, 100, 1, 186, 0, 90, 16, 4, 100, 0, 0, 3, 3776, 92, 0, 851] payload_rows_per_dst=[14636, 10644, 13860, 9586, 8, 4904, 13454, 9162] dup_rows_per_dst=[10470, 2574, 6124, 1708, 0, 118, 574, 482] combine_rows_per_src=[14636, 10644, 13860, 9586, 8, 4904, 13454, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91098 payload_rows_remote=80632 dup_rows=7206 dup_rows_remote=6130 dedup_ratio=0.073303 router_tokens_per_expert=[0, 92, 1602, 249, 0, 0, 4609, 0, 0, 134, 529, 2069, 7317, 0, 0, 7290, 0, 0, 0, 0, 0, 3678, 3, 0, 0, 0, 0, 7779, 0, 0, 1195, 0, 165, 0, 161, 0, 0, 2676, 0, 4, 0, 0, 504, 0, 0, 0, 0, 757, 0, 0, 120, 0, 55, 490, 0, 92, 0, 6827, 0, 0, 651, 104, 0, 0] payload_rows_per_dst=[11538, 5120, 7362, 11046, 10466, 15556, 15166, 14844] dup_rows_per_dst=[1566, 344, 0, 1372, 1076, 1546, 982, 320] combine_rows_per_src=[11538, 5120, 7362, 11046, 10466, 15556, 15166, 14844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83728 payload_rows_remote=75266 dup_rows=14576 dup_rows_remote=13314 dedup_ratio=0.148275 router_tokens_per_expert=[0, 122, 165, 718, 1066, 12, 3529, 67, 185, 50, 4985, 1263, 6755, 274, 19, 3198, 265, 110, 1, 193, 47, 2583, 156, 1, 45, 0, 146, 7724, 1822, 0, 876, 262, 1345, 0, 101, 0, 55, 611, 330, 45, 0, 8, 1281, 82, 107, 17, 15, 709, 391, 0, 243, 390, 84, 1006, 5, 299, 259, 3508, 63, 328, 1175, 19, 12, 25] payload_rows_per_dst=[10158, 11920, 8462, 15936, 4658, 6910, 15624, 10060] dup_rows_per_dst=[1200, 1632, 1262, 5814, 316, 912, 2722, 718] combine_rows_per_src=[10158, 11920, 8462, 15936, 4658, 6910, 15624, 10060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=260988928 dispatch_meta_bytes=324752 combine_payload_bytes=260988928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=261313680 ingress_bytes=364797952 link_bytes=364797952 token_bytes=522302608 weight_movement_bytes=103809024 total_nvlink_bytes=626111632 weight_to_token_ratio=0.198753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372705325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=289341440 dispatch_meta_bytes=321640 combine_payload_bytes=289341440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289663080 ingress_bytes=323944448 link_bytes=323944448 token_bytes=579004520 weight_movement_bytes=34603008 total_nvlink_bytes=613607528 weight_to_token_ratio=0.059763 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218976878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=276635648 dispatch_meta_bytes=323936 combine_payload_bytes=276635648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276959584 ingress_bytes=311238656 link_bytes=311238656 token_bytes=553595232 weight_movement_bytes=34603008 total_nvlink_bytes=588198240 weight_to_token_ratio=0.062506 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171166006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92482 payload_rows_remote=76098 dup_rows=5822 dup_rows_remote=3952 dedup_ratio=0.059224 router_tokens_per_expert=[507, 548, 1, 0, 3, 145, 0, 0, 0, 0, 1, 0, 2, 80, 6930, 2, 0, 1, 172, 40, 388, 0, 0, 7555, 773, 0, 33, 2, 130, 8189, 0, 0, 90, 0, 0, 83, 136, 1, 4603, 0, 0, 0, 0, 3, 556, 7996, 699, 0, 34, 0, 0, 0, 0, 0, 6760, 45, 676, 1, 0, 224, 0, 0, 1696, 47] payload_rows_per_dst=[2010, 13876, 15822, 16384, 9740, 16268, 13550, 4832] dup_rows_per_dst=[398, 146, 490, 1870, 86, 2244, 132, 456] combine_rows_per_src=[2010, 13876, 15822, 16384, 9740, 16268, 13550, 4832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=252387328 dispatch_meta_bytes=292792 combine_payload_bytes=252387328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252680120 ingress_bytes=286990336 link_bytes=286990336 token_bytes=505067448 weight_movement_bytes=34603008 total_nvlink_bytes=539670456 weight_to_token_ratio=0.068512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079921530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330268672 dispatch_meta_bytes=347048 combine_payload_bytes=330268672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330615720 ingress_bytes=364871680 link_bytes=364871680 token_bytes=660884392 weight_movement_bytes=34603008 total_nvlink_bytes=695487400 weight_to_token_ratio=0.052359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.372982757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=308289536 dispatch_meta_bytes=354320 combine_payload_bytes=308289536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308643856 ingress_bytes=342892544 link_bytes=342892544 token_bytes=616933392 weight_movement_bytes=34603008 total_nvlink_bytes=651536400 weight_to_token_ratio=0.056089 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86804 payload_rows_remote=74086 dup_rows=11500 dup_rows_remote=9902 dedup_ratio=0.116984 router_tokens_per_expert=[0, 92, 2793, 227, 0, 0, 4638, 0, 0, 0, 176, 1099, 7160, 0, 0, 7384, 0, 0, 0, 0, 0, 2226, 0, 0, 0, 0, 0, 7735, 0, 0, 1284, 0, 0, 0, 150, 0, 0, 4764, 0, 0, 0, 0, 484, 0, 0, 0, 0, 1118, 0, 0, 41, 0, 0, 671, 0, 92, 0, 6583, 0, 0, 347, 88, 0, 0] payload_rows_per_dst=[12554, 12718, 15132, 15654, 9828, 3204, 3918, 13796] dup_rows_per_dst=[2946, 1598, 4088, 2384, 0, 0, 244, 240] combine_rows_per_src=[12554, 12718, 15132, 15654, 9828, 3204, 3918, 13796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=311697408 dispatch_meta_bytes=320200 combine_payload_bytes=311697408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312017608 ingress_bytes=415506432 link_bytes=415506432 token_bytes=623715016 weight_movement_bytes=103809024 total_nvlink_bytes=727524040 weight_to_token_ratio=0.166437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.563517253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86122 payload_rows_remote=74254 dup_rows=12182 dup_rows_remote=11144 dedup_ratio=0.123922 router_tokens_per_expert=[46, 12, 242, 1, 289, 396, 2120, 156, 0, 99, 575, 49, 19, 448, 307, 23, 4748, 317, 115, 5827, 5515, 4, 152, 66, 0, 342, 569, 0, 4945, 91, 232, 0, 899, 29, 50, 35, 257, 4450, 6847, 1876, 0, 44, 1, 234, 10, 105, 0, 1652, 11, 1187, 64, 514, 1, 226, 0, 0, 115, 11, 81, 65, 1378, 266, 699, 340] payload_rows_per_dst=[11768, 12484, 3776, 11530, 5998, 14534, 11868, 14164] dup_rows_per_dst=[1752, 2210, 32, 828, 294, 3252, 1038, 2776] combine_rows_per_src=[11768, 12484, 3776, 11530, 5998, 14534, 11868, 14164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92030 payload_rows_remote=75956 dup_rows=6274 dup_rows_remote=3538 dedup_ratio=0.063822 router_tokens_per_expert=[203, 455, 6, 11, 17, 251, 35, 0, 3, 0, 79, 4, 78, 68, 6829, 41, 13, 8, 105, 33, 413, 13, 5, 7234, 251, 0, 71, 98, 229, 7959, 13, 1, 276, 0, 2, 62, 134, 12, 4736, 2, 0, 1, 9, 7, 515, 7817, 1007, 8, 64, 0, 4, 13, 19, 10, 7150, 79, 270, 22, 3, 340, 11, 1, 1986, 66] payload_rows_per_dst=[1790, 13072, 15324, 16110, 10396, 16074, 14556, 4708] dup_rows_per_dst=[166, 98, 324, 1734, 248, 2736, 278, 690] combine_rows_per_src=[1790, 13072, 15324, 16110, 10396, 16074, 14556, 4708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=303456256 dispatch_meta_bytes=335952 combine_payload_bytes=303456256 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=303792208 ingress_bytes=476471296 link_bytes=476471296 token_bytes=607248464 weight_movement_bytes=173015040 total_nvlink_bytes=780263504 weight_to_token_ratio=0.284916 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.792923128e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80568 payload_rows_remote=66058 dup_rows=17736 dup_rows_remote=13908 dedup_ratio=0.180420 router_tokens_per_expert=[0, 87, 2458, 250, 0, 0, 4271, 0, 0, 43, 195, 1890, 7326, 0, 0, 7474, 2, 0, 0, 0, 0, 2395, 1, 0, 0, 0, 0, 7785, 0, 0, 1168, 0, 0, 0, 152, 0, 0, 4900, 0, 0, 0, 0, 394, 0, 0, 0, 0, 606, 0, 0, 91, 0, 0, 513, 0, 87, 0, 6609, 0, 0, 363, 92, 0, 0] payload_rows_per_dst=[11506, 4148, 4796, 2336, 16380, 15722, 11170, 14510] dup_rows_per_dst=[2626, 108, 0, 0, 9294, 1226, 654, 3828] combine_rows_per_src=[11506, 4148, 4796, 2336, 16380, 15722, 11170, 14510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75342 payload_rows_remote=59438 dup_rows=22962 dup_rows_remote=10134 dedup_ratio=0.233582 router_tokens_per_expert=[0, 92, 2529, 248, 0, 0, 4388, 0, 0, 104, 213, 2397, 7349, 1, 3, 7402, 0, 0, 0, 0, 0, 2626, 1, 2, 0, 0, 0, 7766, 0, 3, 1099, 0, 1, 1, 171, 0, 11, 3906, 0, 1, 1, 0, 487, 0, 0, 2, 1, 482, 0, 2, 67, 0, 0, 543, 3, 92, 0, 6550, 0, 0, 500, 106, 2, 0] payload_rows_per_dst=[15904, 15280, 14980, 3356, 8178, 1944, 1414, 14286] dup_rows_per_dst=[12828, 5298, 4638, 162, 4, 2, 0, 30] combine_rows_per_src=[15904, 15280, 14980, 3356, 8178, 1944, 1414, 14286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84276 payload_rows_remote=81528 dup_rows=14028 dup_rows_remote=14028 dedup_ratio=0.142700 router_tokens_per_expert=[21, 0, 18, 0, 3, 341, 4007, 366, 35, 89, 218, 0, 83, 194, 356, 0, 4977, 142, 78, 5459, 4099, 0, 13, 26, 29, 212, 412, 0, 6082, 8, 40, 342, 651, 154, 189, 0, 266, 3831, 7384, 3794, 0, 3, 0, 7, 0, 26, 216, 1464, 0, 1587, 0, 49, 183, 0, 0, 84, 13, 0, 6, 1, 847, 0, 417, 330] payload_rows_per_dst=[13668, 11402, 2748, 13496, 9364, 14914, 10500, 8184] dup_rows_per_dst=[5798, 1466, 0, 754, 1048, 2982, 968, 1012] combine_rows_per_src=[13668, 11402, 2748, 13496, 9364, 14914, 10500, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90778 payload_rows_remote=85060 dup_rows=7526 dup_rows_remote=7334 dedup_ratio=0.076558 router_tokens_per_expert=[4, 0, 0, 0, 0, 89, 399, 0, 0, 54, 185, 0, 0, 79, 1531, 0, 4325, 87, 93, 7264, 6852, 0, 0, 0, 0, 493, 1048, 0, 7376, 0, 69, 0, 2767, 0, 3, 0, 93, 5254, 7519, 92, 0, 0, 0, 0, 0, 69, 0, 86, 0, 1506, 0, 357, 0, 0, 0, 0, 0, 0, 0, 142, 804, 0, 194, 318] payload_rows_per_dst=[9630, 15644, 360, 16010, 5718, 15312, 12982, 15122] dup_rows_per_dst=[4, 2582, 0, 1962, 192, 36, 1252, 1498] combine_rows_per_src=[9630, 15644, 360, 16010, 5718, 15312, 12982, 15122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=304144384 dispatch_meta_bytes=341592 combine_payload_bytes=304144384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304485976 ingress_bytes=338747392 link_bytes=338747392 token_bytes=608630360 weight_movement_bytes=34603008 total_nvlink_bytes=643233368 weight_to_token_ratio=0.056854 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274679165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=311115776 dispatch_meta_bytes=317976 combine_payload_bytes=311115776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311433752 ingress_bytes=345718784 link_bytes=345718784 token_bytes=622549528 weight_movement_bytes=34603008 total_nvlink_bytes=657152536 weight_to_token_ratio=0.055583 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300911952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=270573568 dispatch_meta_bytes=319864 combine_payload_bytes=270573568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270893432 ingress_bytes=305176576 link_bytes=305176576 token_bytes=541467000 weight_movement_bytes=34603008 total_nvlink_bytes=576070008 weight_to_token_ratio=0.063906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148354887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95566 payload_rows_remote=91076 dup_rows=2738 dup_rows_remote=2738 dedup_ratio=0.027852 router_tokens_per_expert=[0, 1, 863, 0, 8189, 0, 3, 0, 236, 0, 0, 42, 0, 0, 1, 0, 0, 200, 0, 2, 1, 0, 7687, 0, 0, 0, 0, 0, 2, 2240, 0, 3, 0, 5217, 0, 0, 0, 2, 2, 1, 0, 101, 0, 7748, 0, 0, 1, 0, 0, 0, 472, 0, 0, 7878, 0, 0, 70, 0, 0, 0, 0, 8190, 0, 0] payload_rows_per_dst=[16382, 560, 15774, 4490, 10436, 15500, 16042, 16382] dup_rows_per_dst=[1730, 2, 0, 0, 0, 204, 662, 140] combine_rows_per_src=[16382, 560, 15774, 4490, 10436, 15500, 16042, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=243458048 dispatch_meta_bytes=278288 combine_payload_bytes=243458048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243736336 ingress_bytes=278061056 link_bytes=278061056 token_bytes=487194384 weight_movement_bytes=34603008 total_nvlink_bytes=521797392 weight_to_token_ratio=0.071025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.046321368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333938688 dispatch_meta_bytes=382224 combine_payload_bytes=333938688 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=334320912 ingress_bytes=472350720 link_bytes=472350720 token_bytes=668259600 weight_movement_bytes=138412032 total_nvlink_bytes=806671632 weight_to_token_ratio=0.207123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.777417732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=348405760 dispatch_meta_bytes=369576 combine_payload_bytes=348405760 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=348775336 ingress_bytes=452214784 link_bytes=452214784 token_bytes=697181096 weight_movement_bytes=103809024 total_nvlink_bytes=800990120 weight_to_token_ratio=0.148898 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.701647826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83100 payload_rows_remote=66740 dup_rows=15204 dup_rows_remote=11418 dedup_ratio=0.154663 router_tokens_per_expert=[92, 0, 0, 0, 0, 92, 213, 0, 0, 0, 185, 0, 0, 58, 2284, 0, 3848, 226, 92, 7546, 6763, 0, 0, 0, 0, 223, 1791, 0, 7417, 0, 58, 0, 3849, 0, 5, 0, 0, 3456, 7541, 92, 0, 0, 0, 0, 0, 58, 0, 73, 0, 1410, 0, 412, 0, 0, 0, 0, 0, 0, 1, 238, 449, 0, 340, 340] payload_rows_per_dst=[8306, 16360, 14286, 15002, 16268, 262, 9894, 2722] dup_rows_per_dst=[184, 3786, 1542, 2310, 6706, 0, 662, 14] combine_rows_per_src=[8306, 16360, 14286, 15002, 16268, 262, 9894, 2722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=373047296 dispatch_meta_bytes=375256 combine_payload_bytes=373047296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=373422552 ingress_bytes=442253312 link_bytes=442253312 token_bytes=746469848 weight_movement_bytes=69206016 total_nvlink_bytes=815675864 weight_to_token_ratio=0.092711 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.664163609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87234 payload_rows_remote=71568 dup_rows=11070 dup_rows_remote=9264 dedup_ratio=0.112610 router_tokens_per_expert=[3, 0, 148, 3072, 155, 71, 51, 0, 16, 44, 1034, 4324, 120, 553, 501, 98, 36, 44, 2983, 674, 6706, 617, 15, 79, 6884, 68, 5, 6678, 1205, 43, 37, 16, 0, 216, 183, 319, 70, 72, 0, 612, 211, 377, 36, 269, 4, 0, 205, 0, 23, 48, 367, 295, 924, 21, 219, 133, 863, 108, 370, 3, 289, 5306, 1261, 68] payload_rows_per_dst=[6922, 11704, 8120, 14762, 2378, 14156, 15666, 13526] dup_rows_per_dst=[78, 1676, 776, 1754, 566, 1404, 1806, 3010] combine_rows_per_src=[6922, 11704, 8120, 14762, 2378, 14156, 15666, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93618 payload_rows_remote=78360 dup_rows=4686 dup_rows_remote=4176 dedup_ratio=0.047668 router_tokens_per_expert=[0, 1, 1310, 0, 8053, 9, 103, 26, 133, 2, 7, 398, 3, 8, 12, 0, 70, 252, 1, 51, 26, 0, 6816, 0, 0, 5, 2, 0, 78, 2404, 0, 170, 20, 4726, 4, 0, 5, 31, 67, 128, 0, 248, 0, 7559, 0, 2, 2, 6, 0, 16, 615, 0, 15, 7557, 2, 13, 82, 0, 3, 0, 8, 8088, 7, 8] payload_rows_per_dst=[16346, 1192, 14138, 5276, 9578, 15258, 15564, 16266] dup_rows_per_dst=[2798, 36, 0, 42, 188, 510, 934, 178] combine_rows_per_src=[16346, 1192, 14138, 5276, 9578, 15258, 15564, 16266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=273367040 dispatch_meta_bytes=312632 combine_payload_bytes=273367040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273679672 ingress_bytes=307970048 link_bytes=307970048 token_bytes=547046712 weight_movement_bytes=34603008 total_nvlink_bytes=581649720 weight_to_token_ratio=0.063254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158866497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87404 payload_rows_remote=72118 dup_rows=10900 dup_rows_remote=9592 dedup_ratio=0.110881 router_tokens_per_expert=[48, 0, 0, 0, 0, 85, 251, 0, 0, 6, 177, 0, 0, 83, 2118, 0, 3772, 193, 88, 7596, 6933, 0, 0, 0, 0, 286, 1563, 0, 7423, 0, 68, 0, 3716, 0, 9, 0, 0, 4000, 7511, 87, 0, 0, 0, 0, 0, 67, 0, 59, 0, 1317, 0, 332, 0, 0, 0, 0, 0, 0, 5, 213, 551, 0, 264, 331] payload_rows_per_dst=[678, 9540, 12438, 16050, 7606, 15274, 10532, 15286] dup_rows_per_dst=[90, 1756, 4332, 2630, 18, 0, 766, 1308] combine_rows_per_src=[678, 9540, 12438, 16050, 7606, 15274, 10532, 15286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70636 payload_rows_remote=61484 dup_rows=27668 dup_rows_remote=27650 dedup_ratio=0.281453 router_tokens_per_expert=[7, 0, 3, 0, 3, 92, 336, 0, 0, 0, 184, 0, 0, 90, 1999, 0, 4144, 248, 90, 7369, 6691, 0, 2, 2, 0, 189, 1743, 0, 7406, 1, 78, 0, 3042, 1, 2, 0, 0, 4182, 7522, 92, 0, 1, 0, 2, 0, 77, 0, 53, 0, 1528, 0, 344, 0, 3, 0, 0, 0, 0, 1, 215, 700, 3, 358, 349] payload_rows_per_dst=[9152, 15986, 15372, 7140, 15496, 266, 4514, 2710] dup_rows_per_dst=[18, 3298, 13506, 784, 9394, 0, 126, 542] combine_rows_per_src=[9152, 15986, 15372, 7140, 15496, 266, 4514, 2710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69450 payload_rows_remote=53176 dup_rows=28854 dup_rows_remote=17962 dedup_ratio=0.293518 router_tokens_per_expert=[0, 0, 0, 1322, 105, 0, 58, 0, 0, 28, 562, 5617, 0, 865, 357, 0, 45, 0, 4136, 716, 7591, 703, 330, 62, 7017, 49, 7, 7454, 2616, 12, 15, 0, 0, 0, 52, 0, 79, 39, 0, 346, 0, 93, 6, 0, 22, 0, 90, 75, 99, 5, 273, 118, 318, 0, 183, 15, 919, 122, 5, 35, 0, 5950, 614, 27] payload_rows_per_dst=[2910, 16032, 16274, 15218, 2866, 572, 1854, 13724] dup_rows_per_dst=[60, 12860, 10892, 3088, 166, 0, 168, 1620] combine_rows_per_src=[2910, 16032, 16274, 15218, 2866, 572, 1854, 13724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83394 payload_rows_remote=71256 dup_rows=14910 dup_rows_remote=11060 dedup_ratio=0.151672 router_tokens_per_expert=[0, 0, 0, 3002, 0, 0, 0, 0, 0, 0, 240, 95, 0, 91, 2363, 0, 0, 0, 3964, 1624, 7877, 660, 0, 91, 7592, 0, 0, 7373, 439, 69, 0, 0, 0, 0, 241, 0, 69, 0, 0, 2434, 0, 574, 0, 0, 0, 0, 150, 0, 0, 0, 4, 682, 1589, 0, 1897, 244, 815, 92, 0, 0, 0, 4605, 184, 92] payload_rows_per_dst=[6004, 5556, 12298, 15512, 12138, 5358, 15550, 10978] dup_rows_per_dst=[0, 414, 2470, 296, 3850, 336, 6946, 598] combine_rows_per_src=[6004, 5556, 12298, 15512, 12138, 5358, 15550, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293142528 dispatch_meta_bytes=323328 combine_payload_bytes=293142528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293465856 ingress_bytes=327745536 link_bytes=327745536 token_bytes=586608384 weight_movement_bytes=34603008 total_nvlink_bytes=621211392 weight_to_token_ratio=0.058988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233280066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320962560 dispatch_meta_bytes=330144 combine_payload_bytes=320962560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321292704 ingress_bytes=355565568 link_bytes=355565568 token_bytes=642255264 weight_movement_bytes=34603008 total_nvlink_bytes=676858272 weight_to_token_ratio=0.053877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337964607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=295395328 dispatch_meta_bytes=326840 combine_payload_bytes=295395328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295722168 ingress_bytes=329998336 link_bytes=329998336 token_bytes=591117496 weight_movement_bytes=34603008 total_nvlink_bytes=625720504 weight_to_token_ratio=0.058538 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241757171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71862 payload_rows_remote=71460 dup_rows=26442 dup_rows_remote=26442 dedup_ratio=0.268982 router_tokens_per_expert=[0, 0, 7960, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1029, 0, 2, 0, 86, 0, 0, 0, 3, 0, 1355, 95, 3, 133, 0, 2, 0, 0, 68, 0, 0, 7773, 0, 7202, 0, 0, 0, 0, 5335, 0, 473, 7934, 339, 0, 0, 3, 0, 451, 1, 0, 0, 1277, 0, 0, 0, 0, 0, 1, 7623, 2, 0, 0] payload_rows_per_dst=[15922, 2064, 2888, 402, 16186, 16208, 2940, 15252] dup_rows_per_dst=[0, 6, 190, 0, 13768, 11960, 518, 0] combine_rows_per_src=[15922, 2064, 2888, 402, 16186, 16208, 2940, 15252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=251838464 dispatch_meta_bytes=356536 combine_payload_bytes=251838464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252195000 ingress_bytes=286441472 link_bytes=286441472 token_bytes=504033464 weight_movement_bytes=34603008 total_nvlink_bytes=538636472 weight_to_token_ratio=0.068652 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077856199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=217808896 dispatch_meta_bytes=284552 combine_payload_bytes=217808896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=218093448 ingress_bytes=321617920 link_bytes=321617920 token_bytes=435902344 weight_movement_bytes=103809024 total_nvlink_bytes=539711368 weight_to_token_ratio=0.238147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210222341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=291864576 dispatch_meta_bytes=329264 combine_payload_bytes=291864576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292193840 ingress_bytes=326467584 link_bytes=326467584 token_bytes=584058416 weight_movement_bytes=34603008 total_nvlink_bytes=618661424 weight_to_token_ratio=0.059246 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228471236e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83096 payload_rows_remote=66910 dup_rows=15208 dup_rows_remote=11936 dedup_ratio=0.154704 router_tokens_per_expert=[0, 0, 0, 4138, 0, 0, 0, 0, 0, 0, 77, 92, 0, 92, 2103, 0, 0, 0, 3042, 928, 7865, 623, 0, 92, 7365, 0, 0, 7277, 422, 58, 0, 0, 0, 0, 235, 0, 58, 0, 0, 2294, 0, 644, 0, 0, 0, 0, 92, 0, 0, 0, 0, 471, 1988, 0, 3532, 268, 938, 92, 0, 0, 0, 3937, 337, 92] payload_rows_per_dst=[14190, 16186, 10306, 15514, 5058, 1472, 10570, 9800] dup_rows_per_dst=[6290, 3272, 2590, 0, 116, 0, 1948, 992] combine_rows_per_src=[14190, 16186, 10306, 15514, 5058, 1472, 10570, 9800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292700160 dispatch_meta_bytes=391608 combine_payload_bytes=292700160 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=293091768 ingress_bytes=431112192 link_bytes=431112192 token_bytes=585791928 weight_movement_bytes=138412032 total_nvlink_bytes=724203960 weight_to_token_ratio=0.236282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.622240471e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=315384 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274378744 ingress_bytes=308666368 link_bytes=308666368 token_bytes=548442104 weight_movement_bytes=34603008 total_nvlink_bytes=583045112 weight_to_token_ratio=0.063093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161486693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80724 payload_rows_remote=70232 dup_rows=17580 dup_rows_remote=15204 dedup_ratio=0.178833 router_tokens_per_expert=[322, 3884, 1934, 898, 4350, 6797, 26, 2, 64, 24, 11, 253, 25, 106, 108, 37, 287, 68, 15, 2224, 209, 268, 327, 7, 16, 0, 995, 111, 158, 260, 189, 11, 18, 7, 147, 17, 4, 6277, 254, 1060, 38, 6105, 1796, 75, 130, 107, 17, 156, 143, 2833, 83, 0, 2, 334, 1316, 2057, 863, 730, 14, 283, 97, 24, 68, 111] payload_rows_per_dst=[5590, 14254, 6270, 11116, 13656, 15614, 10492, 3732] dup_rows_per_dst=[774, 596, 540, 1064, 1912, 9002, 2376, 1316] combine_rows_per_src=[5590, 14254, 6270, 11116, 13656, 15614, 10492, 3732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71154 payload_rows_remote=54948 dup_rows=27150 dup_rows_remote=14018 dedup_ratio=0.276184 router_tokens_per_expert=[0, 0, 7497, 24, 3, 1, 5, 2, 1, 75, 7, 78, 1315, 46, 3, 0, 70, 0, 57, 11, 72, 5, 1210, 67, 61, 542, 0, 91, 48, 0, 95, 1, 0, 6972, 1, 7439, 0, 0, 0, 3, 5851, 3, 476, 7767, 404, 0, 1, 76, 12, 543, 12, 6, 3, 758, 3, 0, 11, 4, 5, 6, 7329, 69, 11, 0] payload_rows_per_dst=[15054, 2934, 2630, 1452, 15884, 16206, 2160, 14834] dup_rows_per_dst=[10, 116, 210, 42, 12946, 13132, 658, 36] combine_rows_per_src=[15054, 2934, 2630, 1452, 15884, 16206, 2160, 14834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83034 payload_rows_remote=73022 dup_rows=15270 dup_rows_remote=13968 dedup_ratio=0.155334 router_tokens_per_expert=[0, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 82, 87, 0, 87, 2335, 0, 0, 0, 3177, 1229, 7873, 613, 0, 87, 7528, 0, 0, 7374, 364, 68, 0, 0, 0, 0, 160, 0, 67, 0, 0, 2778, 0, 617, 0, 0, 0, 0, 92, 0, 0, 0, 0, 534, 2005, 0, 3175, 219, 878, 86, 0, 0, 0, 3688, 251, 85] payload_rows_per_dst=[7226, 4858, 9376, 14438, 5876, 15038, 16210, 10012] dup_rows_per_dst=[0, 324, 836, 144, 134, 1128, 11402, 1302] combine_rows_per_src=[7226, 4858, 9376, 14438, 5876, 15038, 16210, 10012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79620 payload_rows_remote=63602 dup_rows=18684 dup_rows_remote=11064 dedup_ratio=0.190063 router_tokens_per_expert=[0, 0, 2, 3983, 2, 3, 0, 0, 0, 0, 97, 92, 1, 92, 2292, 1, 0, 0, 3516, 1276, 7829, 653, 1, 92, 7576, 1, 0, 7297, 466, 77, 0, 0, 0, 3, 209, 1, 77, 0, 0, 2896, 1, 546, 0, 3, 2, 0, 115, 0, 0, 0, 31, 509, 2106, 0, 2061, 247, 830, 90, 0, 0, 3, 3780, 203, 90] payload_rows_per_dst=[16018, 16194, 13112, 15682, 6216, 1328, 8640, 2430] dup_rows_per_dst=[7620, 4108, 5336, 0, 156, 6, 1268, 190] combine_rows_per_src=[16018, 16194, 13112, 15682, 6216, 1328, 8640, 2430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87044 payload_rows_remote=73474 dup_rows=11260 dup_rows_remote=8908 dedup_ratio=0.114543 router_tokens_per_expert=[611, 3439, 1237, 750, 4140, 6272, 0, 0, 72, 82, 0, 261, 11, 0, 10, 0, 0, 0, 37, 3943, 0, 171, 371, 0, 0, 0, 1275, 63, 85, 473, 0, 22, 0, 0, 58, 16, 0, 6478, 209, 525, 0, 6692, 1723, 0, 0, 8, 0, 0, 0, 3601, 2, 0, 0, 0, 405, 3091, 1870, 676, 0, 0, 460, 0, 13, 0] payload_rows_per_dst=[4570, 13014, 13570, 11172, 13620, 14062, 11490, 5546] dup_rows_per_dst=[626, 402, 2352, 944, 952, 2784, 2708, 492] combine_rows_per_src=[4570, 13014, 13570, 11172, 13620, 14062, 11490, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89132 payload_rows_remote=74316 dup_rows=9172 dup_rows_remote=6080 dedup_ratio=0.093302 router_tokens_per_expert=[0, 5950, 2801, 783, 6862, 7462, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 8, 41, 0, 1135, 0, 0, 0, 0, 1020, 267, 92, 0, 0, 0, 0, 0, 64, 0, 0, 6699, 1985, 206, 0, 601, 1795, 0, 0, 0, 0, 0, 0, 1507, 32, 0, 0, 0, 143, 3062, 233, 6120, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[6766, 14924, 7014, 15668, 14816, 9660, 7776, 12508] dup_rows_per_dst=[402, 384, 848, 814, 3092, 1538, 1712, 382] combine_rows_per_src=[6766, 14924, 7014, 15668, 14816, 9660, 7776, 12508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287670272 dispatch_meta_bytes=341744 combine_payload_bytes=287670272 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=288012016 ingress_bytes=322273280 link_bytes=322273280 token_bytes=575682288 weight_movement_bytes=34603008 total_nvlink_bytes=610285296 weight_to_token_ratio=0.060108 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212688408e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=225067008 dispatch_meta_bytes=275864 combine_payload_bytes=225067008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=225342872 ingress_bytes=259670016 link_bytes=259670016 token_bytes=450409880 weight_movement_bytes=34603008 total_nvlink_bytes=485012888 weight_to_token_ratio=0.076826 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.771173651e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88798 payload_rows_remote=74406 dup_rows=9506 dup_rows_remote=9506 dedup_ratio=0.096700 router_tokens_per_expert=[1, 0, 1, 0, 2, 1, 3, 0, 0, 64, 0, 1, 1226, 0, 6811, 0, 8184, 133, 0, 4, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7004, 190, 0, 0, 1, 95, 0, 2, 0, 0, 0, 1, 1, 0, 5833, 0, 0, 0, 76, 0, 0, 0, 1464, 8189, 0, 5, 0, 0, 0, 6340, 1, 244, 0, 2571] payload_rows_per_dst=[10, 14932, 16376, 14392, 194, 11670, 16384, 14840] dup_rows_per_dst=[0, 1274, 1674, 0, 2, 0, 3084, 3472] combine_rows_per_src=[10, 14932, 16376, 14392, 194, 11670, 16384, 14840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299098112 dispatch_meta_bytes=347960 combine_payload_bytes=299098112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299446072 ingress_bytes=368304128 link_bytes=368304128 token_bytes=598544184 weight_movement_bytes=69206016 total_nvlink_bytes=667750200 weight_to_token_ratio=0.115624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385898783e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=260513792 dispatch_meta_bytes=298664 combine_payload_bytes=260513792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260812456 ingress_bytes=295116800 link_bytes=295116800 token_bytes=521326248 weight_movement_bytes=34603008 total_nvlink_bytes=555929256 weight_to_token_ratio=0.066375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110500760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=304398336 dispatch_meta_bytes=321584 combine_payload_bytes=304398336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304719920 ingress_bytes=373604352 link_bytes=373604352 token_bytes=609118256 weight_movement_bytes=69206016 total_nvlink_bytes=678324272 weight_to_token_ratio=0.113617 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405843099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300949504 dispatch_meta_bytes=329528 combine_payload_bytes=300949504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301279032 ingress_bytes=335552512 link_bytes=335552512 token_bytes=602228536 weight_movement_bytes=34603008 total_nvlink_bytes=636831544 weight_to_token_ratio=0.057458 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262657088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78230 payload_rows_remote=71606 dup_rows=20074 dup_rows_remote=19868 dedup_ratio=0.204203 router_tokens_per_expert=[0, 5852, 2053, 971, 6969, 7364, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1665, 0, 0, 0, 0, 1027, 346, 92, 0, 0, 0, 0, 0, 82, 0, 0, 6163, 3383, 172, 0, 244, 1531, 0, 0, 17, 0, 0, 0, 1512, 23, 0, 0, 0, 78, 2222, 212, 6840, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[11474, 6624, 12926, 9604, 14724, 3064, 5950, 13864] dup_rows_per_dst=[10318, 206, 2224, 634, 4028, 520, 1720, 424] combine_rows_per_src=[11474, 6624, 12926, 9604, 14724, 3064, 5950, 13864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=304766976 dispatch_meta_bytes=335648 combine_payload_bytes=304766976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305102624 ingress_bytes=339369984 link_bytes=339369984 token_bytes=609869600 weight_movement_bytes=34603008 total_nvlink_bytes=644472608 weight_to_token_ratio=0.056738 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277021928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=293298176 dispatch_meta_bytes=365896 combine_payload_bytes=293298176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293664072 ingress_bytes=327901184 link_bytes=327901184 token_bytes=586962248 weight_movement_bytes=34603008 total_nvlink_bytes=621565256 weight_to_token_ratio=0.058953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233865757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77718 payload_rows_remote=76578 dup_rows=20586 dup_rows_remote=20436 dedup_ratio=0.209412 router_tokens_per_expert=[644, 188, 11, 26, 11, 2026, 81, 1964, 3442, 350, 404, 13, 212, 0, 534, 3960, 60, 135, 140, 4957, 916, 4184, 30, 26, 24, 12, 30, 15, 8, 33, 0, 11, 1817, 358, 75, 0, 6, 0, 0, 707, 285, 329, 75, 6904, 148, 279, 1552, 217, 5, 21, 4, 371, 67, 7278, 0, 177, 20, 45, 21, 3399, 132, 51, 38, 324] payload_rows_per_dst=[8674, 9174, 13682, 14730, 7736, 10422, 1140, 12160] dup_rows_per_dst=[1228, 736, 7214, 92, 3316, 4030, 150, 3820] combine_rows_per_src=[8674, 9174, 13682, 14730, 7736, 10422, 1140, 12160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86708 payload_rows_remote=75078 dup_rows=11596 dup_rows_remote=11558 dedup_ratio=0.117961 router_tokens_per_expert=[37, 22, 20, 9, 55, 76, 35, 0, 6, 260, 0, 53, 1976, 0, 5604, 1, 8009, 327, 3, 133, 1075, 2, 33, 0, 0, 0, 19, 0, 1, 7, 6488, 133, 0, 0, 0, 82, 0, 73, 15, 0, 193, 71, 27, 0, 5521, 0, 0, 0, 60, 48, 1, 0, 725, 8037, 4, 90, 29, 10, 0, 5586, 56, 375, 0, 3765] payload_rows_per_dst=[186, 13828, 16266, 13392, 326, 11630, 16124, 14956] dup_rows_per_dst=[16, 2124, 2898, 14, 14, 38, 1624, 4868] combine_rows_per_src=[186, 13828, 16266, 13392, 326, 11630, 16124, 14956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88144 payload_rows_remote=74672 dup_rows=10160 dup_rows_remote=9806 dedup_ratio=0.103353 router_tokens_per_expert=[0, 5706, 2271, 753, 7028, 7447, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1402, 0, 0, 0, 0, 997, 222, 86, 0, 0, 0, 0, 0, 81, 0, 0, 6505, 3429, 177, 0, 316, 1534, 0, 0, 0, 0, 0, 0, 1386, 46, 0, 0, 0, 109, 2484, 177, 6650, 0, 0, 86, 0, 0, 0] payload_rows_per_dst=[5692, 14894, 2960, 15932, 15362, 13422, 6410, 13472] dup_rows_per_dst=[356, 360, 4, 734, 5022, 1690, 1640, 354] combine_rows_per_src=[5692, 14894, 2960, 15932, 15362, 13422, 6410, 13472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=56800 payload_rows_remote=40522 dup_rows=41504 dup_rows_remote=6258 dedup_ratio=0.422201 router_tokens_per_expert=[0, 5928, 2420, 751, 6971, 7485, 1, 0, 0, 0, 1, 240, 0, 0, 2, 1, 3, 0, 93, 6, 1, 1736, 0, 0, 0, 0, 1033, 286, 90, 0, 3, 0, 0, 0, 103, 0, 0, 6558, 2463, 237, 0, 335, 1464, 0, 2, 0, 1, 3, 0, 1572, 1, 0, 0, 3, 149, 2682, 220, 6213, 1, 3, 90, 0, 0, 1] payload_rows_per_dst=[16278, 486, 10452, 2434, 9332, 2938, 7122, 7758] dup_rows_per_dst=[35246, 2, 1930, 390, 1252, 672, 1692, 320] combine_rows_per_src=[16278, 486, 10452, 2434, 9332, 2938, 7122, 7758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83290 payload_rows_remote=67948 dup_rows=15014 dup_rows_remote=10280 dedup_ratio=0.152730 router_tokens_per_expert=[2723, 0, 0, 0, 0, 72, 0, 3005, 534, 0, 113, 0, 0, 0, 666, 6196, 0, 0, 0, 5010, 283, 1456, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 1576, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 7413, 0, 0, 514, 0, 0, 0, 0, 3524, 0, 7811, 0, 0, 0, 0, 0, 6903, 173, 0, 0, 92] payload_rows_per_dst=[9592, 4284, 12126, 15622, 15342, 1028, 10384, 14912] dup_rows_per_dst=[2008, 452, 1450, 0, 4734, 0, 3054, 3316] combine_rows_per_src=[9592, 4284, 12126, 15622, 15342, 1028, 10384, 14912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80198 payload_rows_remote=68174 dup_rows=18106 dup_rows_remote=11800 dedup_ratio=0.184184 router_tokens_per_expert=[231, 0, 8, 14, 0, 4806, 0, 1697, 4316, 0, 18, 0, 49, 39, 182, 2866, 58, 0, 5, 4044, 945, 4101, 0, 12, 0, 0, 1243, 0, 0, 0, 66, 0, 755, 0, 51, 0, 0, 0, 0, 1666, 1059, 0, 81, 7585, 0, 0, 1743, 695, 0, 0, 0, 598, 0, 7885, 0, 0, 0, 0, 10, 2097, 102, 0, 0, 125] payload_rows_per_dst=[12132, 12530, 12024, 15898, 15594, 6156, 1196, 4668] dup_rows_per_dst=[1380, 2410, 6306, 2490, 4520, 1000, 0, 0] combine_rows_per_src=[12132, 12530, 12024, 15898, 15594, 6156, 1196, 4668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=313663488 dispatch_meta_bytes=388056 combine_payload_bytes=313663488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314051544 ingress_bytes=382869504 link_bytes=382869504 token_bytes=627715032 weight_movement_bytes=69206016 total_nvlink_bytes=696921048 weight_to_token_ratio=0.110251 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440707120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=307519488 dispatch_meta_bytes=346544 combine_payload_bytes=307519488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307866032 ingress_bytes=342122496 link_bytes=342122496 token_bytes=615385520 weight_movement_bytes=34603008 total_nvlink_bytes=649988528 weight_to_token_ratio=0.056230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287379409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71182 payload_rows_remote=69816 dup_rows=27122 dup_rows_remote=27122 dedup_ratio=0.275899 router_tokens_per_expert=[0, 1073, 66, 0, 0, 5, 191, 0, 1, 8089, 0, 472, 2, 0, 0, 1, 0, 5447, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 239, 435, 6, 0, 1, 237, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 2, 1597, 8189, 0, 1, 21, 0, 0, 1, 7136, 3, 0, 7912, 0, 78, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[11716, 16220, 10928, 1366, 480, 14074, 16240, 158] dup_rows_per_dst=[1668, 2954, 112, 0, 0, 8488, 13900, 0] combine_rows_per_src=[11716, 16220, 10928, 1366, 480, 14074, 16240, 158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88230 payload_rows_remote=74350 dup_rows=10074 dup_rows_remote=8482 dedup_ratio=0.102478 router_tokens_per_expert=[5469, 0, 0, 0, 0, 0, 0, 2493, 508, 0, 146, 0, 0, 0, 337, 6745, 0, 0, 0, 2369, 285, 482, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 1266, 0, 0, 0, 7440, 0, 0, 459, 0, 0, 0, 0, 3869, 0, 7703, 0, 0, 0, 0, 0, 7393, 160, 0, 0, 92] payload_rows_per_dst=[13362, 13880, 6250, 15406, 15386, 918, 7738, 15290] dup_rows_per_dst=[2562, 1592, 222, 0, 5698, 0, 0, 0] combine_rows_per_src=[13362, 13880, 6250, 15406, 15386, 918, 7738, 15290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=305856512 dispatch_meta_bytes=337912 combine_payload_bytes=305856512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306194424 ingress_bytes=375062528 link_bytes=375062528 token_bytes=612050936 weight_movement_bytes=69206016 total_nvlink_bytes=681256952 weight_to_token_ratio=0.113072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411330098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=165978112 dispatch_meta_bytes=187120 combine_payload_bytes=165978112 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=166165232 ingress_bytes=338993152 link_bytes=338993152 token_bytes=332143344 weight_movement_bytes=173015040 total_nvlink_bytes=505158384 weight_to_token_ratio=0.520905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275603940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=278315008 dispatch_meta_bytes=312912 combine_payload_bytes=278315008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278627920 ingress_bytes=312918016 link_bytes=312918016 token_bytes=556942928 weight_movement_bytes=34603008 total_nvlink_bytes=591545936 weight_to_token_ratio=0.062130 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177485302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=279240704 dispatch_meta_bytes=319896 combine_payload_bytes=279240704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279560600 ingress_bytes=313843712 link_bytes=313843712 token_bytes=558801304 weight_movement_bytes=34603008 total_nvlink_bytes=593404312 weight_to_token_ratio=0.061924 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180968622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=387752 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286354088 ingress_bytes=320569344 link_bytes=320569344 token_bytes=572320424 weight_movement_bytes=34603008 total_nvlink_bytes=606923432 weight_to_token_ratio=0.060461 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206276634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=304537600 dispatch_meta_bytes=331328 combine_payload_bytes=304537600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=304868928 ingress_bytes=408346624 link_bytes=408346624 token_bytes=609406528 weight_movement_bytes=103809024 total_nvlink_bytes=713215552 weight_to_token_ratio=0.170344 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536575472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89158 payload_rows_remote=78646 dup_rows=9146 dup_rows_remote=8500 dedup_ratio=0.093038 router_tokens_per_expert=[32, 30, 2107, 31, 13, 268, 28, 10, 227, 336, 16, 2, 4207, 12, 6428, 20, 93, 371, 627, 7, 1, 0, 627, 27, 324, 94, 22, 3, 236, 86, 2, 6312, 5897, 16, 2456, 0, 301, 4, 2962, 79, 1717, 4167, 27, 221, 360, 4246, 652, 59, 373, 24, 13, 71, 634, 0, 237, 20, 0, 351, 866, 294, 181, 253, 65, 7] payload_rows_per_dst=[7924, 13488, 10500, 13366, 10008, 9138, 10512, 14222] dup_rows_per_dst=[1120, 594, 1340, 792, 1628, 1420, 646, 1606] combine_rows_per_src=[7924, 13488, 10500, 13366, 10008, 9138, 10512, 14222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64762 payload_rows_remote=48552 dup_rows=33542 dup_rows_remote=13776 dedup_ratio=0.341207 router_tokens_per_expert=[11, 1035, 76, 1, 0, 187, 291, 17, 66, 7738, 2, 465, 23, 23, 3, 23, 4, 5140, 1, 32, 14, 30, 0, 0, 3, 0, 47, 0, 139, 586, 217, 0, 29, 350, 4, 17, 0, 0, 0, 8, 20, 7685, 14, 117, 2113, 8054, 9, 93, 72, 0, 0, 10, 6449, 126, 0, 7437, 0, 318, 19, 29, 4, 0, 0, 1] payload_rows_per_dst=[3074, 15806, 10398, 2030, 988, 16210, 15494, 762] dup_rows_per_dst=[162, 834, 44, 206, 62, 19766, 12442, 26] combine_rows_per_src=[3074, 15806, 10398, 2030, 988, 16210, 15494, 762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81616 payload_rows_remote=65456 dup_rows=16688 dup_rows_remote=5010 dedup_ratio=0.169759 router_tokens_per_expert=[4797, 0, 0, 0, 0, 20, 0, 3004, 543, 0, 123, 0, 0, 0, 395, 6384, 0, 0, 0, 3080, 264, 532, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 0, 0, 0, 0, 0, 893, 0, 0, 0, 7365, 0, 0, 358, 0, 0, 0, 0, 4350, 0, 7771, 0, 0, 0, 0, 0, 7290, 158, 0, 0, 87] payload_rows_per_dst=[12724, 1784, 7630, 14998, 4588, 14754, 8978, 16160] dup_rows_per_dst=[2918, 338, 318, 0, 478, 692, 266, 11678] combine_rows_per_src=[12724, 1784, 7630, 14998, 4588, 14754, 8978, 16160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89668 payload_rows_remote=77222 dup_rows=8636 dup_rows_remote=5978 dedup_ratio=0.087850 router_tokens_per_expert=[4130, 2, 0, 0, 0, 12, 0, 3405, 528, 3, 110, 0, 0, 0, 446, 6565, 0, 1, 0, 3775, 275, 590, 0, 159, 1, 0, 0, 0, 0, 1, 0, 0, 1213, 2, 0, 0, 0, 0, 0, 1060, 0, 3, 0, 7416, 0, 3, 383, 0, 0, 0, 0, 3657, 2, 7731, 0, 2, 0, 2, 0, 7396, 187, 0, 0, 92] payload_rows_per_dst=[12446, 13714, 9230, 15466, 15374, 766, 7318, 15354] dup_rows_per_dst=[2658, 1596, 370, 0, 4008, 0, 4, 0] combine_rows_per_src=[12446, 13714, 9230, 15466, 15374, 766, 7318, 15354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88694 payload_rows_remote=85448 dup_rows=9610 dup_rows_remote=9198 dedup_ratio=0.097758 router_tokens_per_expert=[0, 0, 3584, 0, 0, 7, 0, 0, 0, 257, 0, 0, 7200, 0, 4672, 0, 10, 318, 112, 0, 0, 0, 731, 0, 679, 0, 0, 0, 0, 53, 0, 6991, 6020, 0, 670, 0, 0, 0, 487, 0, 2243, 6308, 0, 0, 0, 5414, 1887, 67, 0, 0, 0, 0, 252, 0, 93, 0, 0, 0, 1064, 0, 33, 0, 0, 0] payload_rows_per_dst=[13518, 9354, 13358, 14242, 3246, 7668, 15088, 12220] dup_rows_per_dst=[4492, 504, 1600, 1204, 412, 726, 2, 670] combine_rows_per_src=[13518, 9354, 13358, 14242, 3246, 7668, 15088, 12220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322134016 dispatch_meta_bytes=348584 combine_payload_bytes=322134016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322482600 ingress_bytes=356737024 link_bytes=356737024 token_bytes=644616616 weight_movement_bytes=34603008 total_nvlink_bytes=679219624 weight_to_token_ratio=0.053680 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342372702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=198868992 dispatch_meta_bytes=249312 combine_payload_bytes=198868992 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=199118304 ingress_bytes=337281024 link_bytes=337281024 token_bytes=397987296 weight_movement_bytes=138412032 total_nvlink_bytes=536399328 weight_to_token_ratio=0.347780 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269161340e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=73794 dup_rows=15852 dup_rows_remote=14936 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 3183, 0, 0, 2547, 0, 0, 0, 648, 0, 0, 4434, 0, 7144, 0, 13, 422, 1091, 0, 0, 30, 895, 0, 343, 2, 4, 0, 0, 367, 178, 7302, 7112, 0, 2710, 0, 0, 0, 1482, 178, 1747, 2336, 67, 0, 0, 2531, 854, 59, 0, 0, 0, 0, 255, 0, 166, 0, 0, 0, 809, 0, 77, 137, 29, 0] payload_rows_per_dst=[12396, 14490, 8658, 15028, 15280, 5134, 9532, 1934] dup_rows_per_dst=[4126, 1094, 916, 1364, 7684, 320, 178, 170] combine_rows_per_src=[12396, 14490, 8658, 15028, 15280, 5134, 9532, 1934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87764 payload_rows_remote=80126 dup_rows=10540 dup_rows_remote=10536 dedup_ratio=0.107218 router_tokens_per_expert=[0, 0, 2, 0, 0, 2, 0, 70, 4683, 1, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3604, 213, 0, 0, 1, 3, 1, 0, 0, 0, 8050, 0, 0, 0, 1, 1, 0, 2877, 8084, 0, 3, 2, 7806, 0, 0, 1066, 0, 0, 1, 0, 1, 7987, 0, 4678, 0, 5, 1, 0] payload_rows_per_dst=[148, 9372, 16174, 7638, 16102, 5762, 16190, 16378] dup_rows_per_dst=[0, 0, 6, 4, 0, 4, 1560, 8966] combine_rows_per_src=[148, 9372, 16174, 7638, 16102, 5762, 16190, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82688 payload_rows_remote=72256 dup_rows=15616 dup_rows_remote=13340 dedup_ratio=0.158854 router_tokens_per_expert=[0, 0, 5294, 0, 0, 0, 0, 0, 0, 227, 0, 0, 7166, 0, 4136, 0, 80, 340, 75, 0, 0, 0, 558, 0, 1185, 0, 0, 0, 0, 0, 0, 6722, 4309, 0, 320, 0, 0, 0, 582, 0, 1896, 6118, 0, 0, 0, 6228, 2465, 73, 0, 0, 0, 0, 227, 0, 92, 0, 0, 0, 1003, 0, 56, 0, 0, 0] payload_rows_per_dst=[14714, 10432, 12972, 14100, 9422, 7942, 10988, 2118] dup_rows_per_dst=[8330, 2276, 1370, 1714, 1000, 926, 0, 0] combine_rows_per_src=[14714, 10432, 12972, 14100, 9422, 7942, 10988, 2118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268107776 dispatch_meta_bytes=281864 combine_payload_bytes=268107776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268389640 ingress_bytes=302710784 link_bytes=302710784 token_bytes=536497416 weight_movement_bytes=34603008 total_nvlink_bytes=571100424 weight_to_token_ratio=0.064498 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139076310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=316301312 dispatch_meta_bytes=332800 combine_payload_bytes=316301312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316634112 ingress_bytes=350904320 link_bytes=350904320 token_bytes=632935424 weight_movement_bytes=34603008 total_nvlink_bytes=667538432 weight_to_token_ratio=0.054671 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320424706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=349995008 dispatch_meta_bytes=378584 combine_payload_bytes=349995008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350373592 ingress_bytes=419201024 link_bytes=419201024 token_bytes=700368600 weight_movement_bytes=69206016 total_nvlink_bytes=769574616 weight_to_token_ratio=0.098814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.577419705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302260224 dispatch_meta_bytes=354920 combine_payload_bytes=302260224 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302615144 ingress_bytes=371466240 link_bytes=371466240 token_bytes=604875368 weight_movement_bytes=69206016 total_nvlink_bytes=674081384 weight_to_token_ratio=0.114414 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397797556e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328196096 dispatch_meta_bytes=362648 combine_payload_bytes=328196096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=328558744 ingress_bytes=397402112 link_bytes=397402112 token_bytes=656754840 weight_movement_bytes=69206016 total_nvlink_bytes=725960856 weight_to_token_ratio=0.105376 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.495392154e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=295960576 dispatch_meta_bytes=342384 combine_payload_bytes=295960576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296302960 ingress_bytes=399769600 link_bytes=399769600 token_bytes=592263536 weight_movement_bytes=103809024 total_nvlink_bytes=696072560 weight_to_token_ratio=0.175275 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504300821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75262 payload_rows_remote=61442 dup_rows=23042 dup_rows_remote=12372 dedup_ratio=0.234395 router_tokens_per_expert=[0, 1308, 238, 347, 343, 178, 731, 739, 82, 3497, 307, 88, 0, 429, 275, 181, 61, 83, 0, 415, 331, 183, 213, 0, 1, 83, 0, 4, 550, 5915, 246, 0, 3, 377, 1, 7, 458, 5931, 89, 4677, 76, 26, 2, 5198, 776, 38, 262, 83, 64, 10, 6, 0, 0, 6234, 0, 0, 0, 3, 547, 6752, 11, 584, 24, 85] payload_rows_per_dst=[4870, 9334, 2122, 13530, 1746, 14442, 13820, 15398] dup_rows_per_dst=[1162, 1070, 450, 424, 124, 7834, 10670, 1308] combine_rows_per_src=[4870, 9334, 2122, 13530, 1746, 14442, 13820, 15398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79076 payload_rows_remote=64420 dup_rows=19228 dup_rows_remote=13136 dedup_ratio=0.195597 router_tokens_per_expert=[7, 1, 104, 1, 9, 123, 0, 171, 2968, 28, 5, 4, 30, 252, 136, 0, 2, 5, 12, 0, 0, 202, 7, 2, 11, 3, 3235, 598, 0, 5, 14, 110, 91, 0, 22, 0, 7782, 0, 4, 10, 64, 21, 42, 3243, 7800, 21, 24, 12, 7564, 0, 8, 1428, 10, 3, 2, 0, 85, 7296, 15, 5369, 1, 57, 90, 43] payload_rows_per_dst=[710, 6294, 2010, 7686, 15630, 14656, 15774, 16316] dup_rows_per_dst=[164, 492, 114, 266, 6, 6092, 2316, 9778] combine_rows_per_src=[710, 6294, 2010, 7686, 15630, 14656, 15774, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77818 payload_rows_remote=63494 dup_rows=20486 dup_rows_remote=14424 dedup_ratio=0.208394 router_tokens_per_expert=[0, 1, 3998, 1, 1, 0, 1, 0, 2, 247, 0, 0, 7112, 0, 5256, 0, 77, 351, 110, 0, 0, 0, 468, 0, 945, 0, 1, 0, 0, 20, 0, 6750, 4518, 0, 283, 0, 3, 0, 613, 0, 2210, 6421, 0, 4, 3, 6191, 1930, 53, 4, 0, 0, 0, 250, 0, 92, 0, 0, 3, 1134, 1, 98, 0, 0, 0] payload_rows_per_dst=[14324, 15596, 13432, 13700, 9924, 7680, 692, 2470] dup_rows_per_dst=[6062, 10288, 1428, 1082, 910, 714, 0, 2] combine_rows_per_src=[14324, 15596, 13432, 13700, 9924, 7680, 692, 2470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81138 payload_rows_remote=75898 dup_rows=17166 dup_rows_remote=16750 dedup_ratio=0.174622 router_tokens_per_expert=[0, 252, 0, 0, 0, 0, 318, 21, 92, 7376, 394, 0, 0, 92, 0, 155, 13, 0, 0, 84, 0, 192, 0, 0, 0, 0, 0, 0, 596, 7428, 0, 0, 0, 0, 0, 0, 377, 7041, 0, 8061, 92, 0, 0, 3573, 97, 0, 28, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 363, 7815, 0, 0, 0, 178] payload_rows_per_dst=[816, 14918, 578, 14888, 5240, 13946, 14938, 15814] dup_rows_per_dst=[366, 1300, 0, 1160, 416, 5014, 8012, 898] combine_rows_per_src=[816, 14918, 578, 14888, 5240, 13946, 14938, 15814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80772 payload_rows_remote=69902 dup_rows=17532 dup_rows_remote=16948 dedup_ratio=0.178345 router_tokens_per_expert=[0, 0, 5138, 0, 0, 0, 0, 0, 0, 251, 0, 0, 7075, 0, 4457, 0, 27, 331, 88, 0, 0, 0, 556, 0, 707, 0, 0, 0, 0, 1, 0, 6726, 4701, 0, 287, 0, 0, 0, 546, 0, 2089, 6211, 0, 0, 0, 6024, 2514, 59, 0, 0, 0, 0, 251, 0, 87, 0, 0, 0, 1014, 0, 12, 0, 0, 0] payload_rows_per_dst=[10276, 3816, 12978, 5576, 6390, 16040, 14826, 10870] dup_rows_per_dst=[0, 214, 1448, 542, 662, 14082, 0, 584] combine_rows_per_src=[10276, 3816, 12978, 5576, 6390, 16040, 14826, 10870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=251666432 dispatch_meta_bytes=295256 combine_payload_bytes=251666432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251961688 ingress_bytes=286269440 link_bytes=286269440 token_bytes=503628120 weight_movement_bytes=34603008 total_nvlink_bytes=538231128 weight_to_token_ratio=0.068707 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077208856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263864320 dispatch_meta_bytes=310224 combine_payload_bytes=263864320 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=264174544 ingress_bytes=402276352 link_bytes=402276352 token_bytes=528038864 weight_movement_bytes=138412032 total_nvlink_bytes=666450896 weight_to_token_ratio=0.262125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513733527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90510 payload_rows_remote=82884 dup_rows=7794 dup_rows_remote=7608 dedup_ratio=0.079285 router_tokens_per_expert=[0, 2660, 0, 0, 0, 0, 2966, 507, 30, 2294, 177, 572, 130, 62, 399, 113, 32, 0, 0, 359, 0, 184, 0, 0, 0, 5, 0, 0, 512, 5410, 0, 0, 0, 0, 0, 22, 36, 7518, 0, 2909, 172, 0, 0, 5655, 67, 0, 3, 39, 0, 0, 0, 0, 0, 7420, 0, 0, 0, 0, 337, 7742, 25, 731, 0, 64] payload_rows_per_dst=[8746, 6866, 7626, 11470, 13486, 11444, 14840, 16032] dup_rows_per_dst=[3520, 688, 186, 384, 822, 428, 0, 1766] combine_rows_per_src=[8746, 6866, 7626, 11470, 13486, 11444, 14840, 16032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50800 payload_rows_remote=49580 dup_rows=47504 dup_rows_remote=47502 dedup_ratio=0.483236 router_tokens_per_expert=[0, 2, 569, 7998, 8186, 8187, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 6544, 0, 0, 605, 2, 0, 0, 2, 0, 2, 452, 8189, 0, 0, 0, 4, 0, 1, 0, 0, 60, 2, 0, 0, 0, 0, 0, 59, 0, 0, 0, 2, 0, 0, 2, 0, 0, 3, 95, 0, 0, 0] payload_rows_per_dst=[16382, 2, 16368, 1220, 16384, 122, 122, 200] dup_rows_per_dst=[33504, 0, 13088, 2, 908, 2, 0, 0] combine_rows_per_src=[16382, 2, 16368, 1220, 16384, 122, 122, 200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92940 payload_rows_remote=77926 dup_rows=5364 dup_rows_remote=4166 dedup_ratio=0.054565 router_tokens_per_expert=[0, 241, 0, 0, 0, 0, 310, 64, 92, 7431, 415, 0, 0, 92, 0, 76, 58, 0, 0, 62, 0, 193, 0, 0, 0, 0, 0, 0, 647, 7379, 0, 0, 0, 0, 0, 0, 371, 7103, 0, 8100, 92, 0, 0, 4627, 92, 0, 100, 0, 0, 0, 0, 0, 0, 3187, 0, 0, 0, 0, 323, 7815, 0, 0, 0, 282] payload_rows_per_dst=[864, 15014, 14446, 14774, 16200, 9454, 6374, 15814] dup_rows_per_dst=[366, 1198, 386, 1278, 742, 368, 0, 1026] combine_rows_per_src=[864, 15014, 14446, 14774, 16200, 9454, 6374, 15814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=260071424 dispatch_meta_bytes=311672 combine_payload_bytes=260071424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260383096 ingress_bytes=294674432 link_bytes=294674432 token_bytes=520454520 weight_movement_bytes=34603008 total_nvlink_bytes=555057528 weight_to_token_ratio=0.066486 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108836164e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=310878208 dispatch_meta_bytes=370592 combine_payload_bytes=310878208 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=311248800 ingress_bytes=449290240 link_bytes=449290240 token_bytes=622127008 weight_movement_bytes=138412032 total_nvlink_bytes=760539040 weight_to_token_ratio=0.222482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.690643002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12734239152 prefetch_bytes=622854144 reduce_bytes=692060160 weight_movement_bytes=1314914304 egress_bytes=6371021232 ingress_bytes=7678132224 link_bytes=7678132224 total_nvlink_bytes=14049153456 weight_to_token_ratio=0.103258 prefetch_share_of_ingress=0.081121 dedup_ratio=0.206644 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=20 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256182323e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=286318592 dispatch_meta_bytes=347400 combine_payload_bytes=286318592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286665992 ingress_bytes=320921600 link_bytes=320921600 token_bytes=572984584 weight_movement_bytes=34603008 total_nvlink_bytes=607587592 weight_to_token_ratio=0.060391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207602144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=339492864 dispatch_meta_bytes=361968 combine_payload_bytes=339492864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339854832 ingress_bytes=374095872 link_bytes=374095872 token_bytes=679347696 weight_movement_bytes=34603008 total_nvlink_bytes=713950704 weight_to_token_ratio=0.050936 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407692649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=203079680 dispatch_meta_bytes=388328 combine_payload_bytes=203079680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203468008 ingress_bytes=237682688 link_bytes=237682688 token_bytes=406547688 weight_movement_bytes=34603008 total_nvlink_bytes=441150696 weight_to_token_ratio=0.085114 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.943808199e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=319184896 dispatch_meta_bytes=328368 combine_payload_bytes=319184896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319513264 ingress_bytes=353787904 link_bytes=353787904 token_bytes=638698160 weight_movement_bytes=34603008 total_nvlink_bytes=673301168 weight_to_token_ratio=0.054177 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331275400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13457699392 prefetch_bytes=726663168 reduce_bytes=1003487232 weight_movement_bytes=1730150400 egress_bytes=6732812864 ingress_bytes=8455036928 link_bytes=8455036928 total_nvlink_bytes=15187849792 weight_to_token_ratio=0.128562 prefetch_share_of_ingress=0.085944 dedup_ratio=0.174055 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=29 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.383287968e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12871373384 prefetch_bytes=899678208 reduce_bytes=380633088 weight_movement_bytes=1280311296 egress_bytes=6439629384 ingress_bytes=7712055296 link_bytes=7712055296 total_nvlink_bytes=14151684680 weight_to_token_ratio=0.099470 prefetch_share_of_ingress=0.116659 dedup_ratio=0.207019 prefetch_slots=52 unused_prefetch_slots=316 slot_utilization=0.141304 reduce_slots=11 remote_demand=26 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=29337 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.471920290e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72272 payload_rows_remote=71448 dup_rows=26032 dup_rows_remote=25976 dedup_ratio=0.264811 router_tokens_per_expert=[0, 54, 598, 7592, 7947, 7608, 118, 4, 0, 37, 0, 17, 9, 1, 11, 6, 1, 0, 0, 14, 7703, 5, 6164, 0, 34, 639, 151, 34, 8, 41, 0, 8, 1139, 7994, 0, 1, 2, 169, 0, 29, 4, 0, 308, 66, 3, 0, 2, 28, 0, 69, 27, 0, 0, 153, 0, 1, 131, 0, 7, 112, 95, 7, 0, 1] payload_rows_per_dst=[14406, 15364, 15628, 6910, 16118, 824, 556, 2466] dup_rows_per_dst=[10752, 6, 12146, 604, 2154, 56, 282, 32] combine_rows_per_src=[14406, 15364, 15628, 6910, 16118, 824, 556, 2466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12750984328 prefetch_bytes=830472192 reduce_bytes=484442112 weight_movement_bytes=1314914304 egress_bytes=6379607176 ingress_bytes=7686291456 link_bytes=7686291456 total_nvlink_bytes=14065898632 weight_to_token_ratio=0.103123 prefetch_share_of_ingress=0.108046 dedup_ratio=0.234631 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=14 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.257517215e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12981957536 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=6494860192 ingress_bytes=7940423680 link_bytes=7940423680 total_nvlink_bytes=14435283872 weight_to_token_ratio=0.111950 prefetch_share_of_ingress=0.091514 dedup_ratio=0.185944 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.299094567e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87502 payload_rows_remote=86396 dup_rows=10802 dup_rows_remote=10424 dedup_ratio=0.109884 router_tokens_per_expert=[0, 258, 2, 4, 4, 2, 335, 137, 90, 7407, 430, 0, 0, 90, 0, 150, 21, 0, 0, 78, 3, 183, 3, 0, 0, 0, 0, 0, 566, 7249, 2, 0, 0, 4, 0, 0, 414, 7072, 1, 8097, 92, 0, 0, 3927, 98, 0, 45, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 376, 7819, 0, 0, 0, 202] payload_rows_per_dst=[1106, 15014, 14348, 14576, 16204, 8040, 7982, 10232] dup_rows_per_dst=[378, 1320, 372, 1058, 6820, 284, 0, 570] combine_rows_per_src=[1106, 15014, 14348, 14576, 16204, 8040, 7982, 10232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=292651008 dispatch_meta_bytes=389696 combine_payload_bytes=292651008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293040704 ingress_bytes=327254016 link_bytes=327254016 token_bytes=585691712 weight_movement_bytes=34603008 total_nvlink_bytes=620294720 weight_to_token_ratio=0.059081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231430516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79120 payload_rows_remote=63270 dup_rows=19184 dup_rows_remote=18286 dedup_ratio=0.195150 router_tokens_per_expert=[0, 225, 0, 0, 0, 0, 281, 27, 86, 7424, 401, 0, 0, 86, 0, 108, 30, 0, 0, 72, 0, 178, 0, 0, 0, 0, 0, 0, 600, 7413, 0, 0, 0, 0, 0, 0, 470, 7028, 0, 8106, 87, 0, 0, 4671, 86, 0, 73, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 339, 7838, 0, 0, 0, 197] payload_rows_per_dst=[750, 15004, 560, 14864, 940, 16384, 14768, 15850] dup_rows_per_dst=[316, 1206, 0, 1162, 0, 9662, 5940, 898] combine_rows_per_src=[750, 15004, 560, 14864, 940, 16384, 14768, 15850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=353878016 dispatch_meta_bytes=387280 combine_payload_bytes=353878016 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=354265296 ingress_bytes=457687040 link_bytes=457687040 token_bytes=708143312 weight_movement_bytes=103809024 total_nvlink_bytes=811952336 weight_to_token_ratio=0.146593 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722239485e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12070739088 prefetch_bytes=588251136 reduce_bytes=1211105280 weight_movement_bytes=1799356416 egress_bytes=6039084176 ingress_bytes=7831011328 link_bytes=7831011328 total_nvlink_bytes=13870095504 weight_to_token_ratio=0.149068 prefetch_share_of_ingress=0.075118 dedup_ratio=0.226358 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=35 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.281194138e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=259153920 dispatch_meta_bytes=326224 combine_payload_bytes=259153920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259480144 ingress_bytes=328359936 link_bytes=328359936 token_bytes=518634064 weight_movement_bytes=69206016 total_nvlink_bytes=587840080 weight_to_token_ratio=0.133439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235592004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11971151464 prefetch_bytes=795869184 reduce_bytes=726663168 weight_movement_bytes=1522532352 egress_bytes=5989221992 ingress_bytes=7504461824 link_bytes=7504461824 total_nvlink_bytes=13493683816 weight_to_token_ratio=0.127183 prefetch_share_of_ingress=0.106053 dedup_ratio=0.230185 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=21 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227768943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12604566400 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6306048896 ingress_bytes=7717240832 link_bytes=7717240832 total_nvlink_bytes=14023289728 weight_to_token_ratio=0.112556 prefetch_share_of_ingress=0.094161 dedup_ratio=0.197323 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262580694e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-29 23:40:38] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 676.5 | throughput per GPU (TFLOP/s/GPU): 188.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.575389E+00 | loss scale: 1.0 | grad norm: 70.423 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:40:38 +[rank0]:[W829 23:40:40.274557397 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W829 23:40:41.582491000 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W829 23:40:41.614389073 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W829 23:40:41.622256568 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W829 23:40:41.960810636 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W829 23:40:41.182505415 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W829 23:40:41.197223213 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W829 23:40:42.425736048 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:40:45.688000 225018 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:40:45.689000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225207 closing signal SIGTERM +W0829 23:40:45.689000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225208 closing signal SIGTERM +W0829 23:40:45.690000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225209 closing signal SIGTERM +W0829 23:40:45.690000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225210 closing signal SIGTERM +W0829 23:40:45.691000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225211 closing signal SIGTERM +W0829 23:40:45.691000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225212 closing signal SIGTERM +W0829 23:40:45.691000 225018 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 225213 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 225018 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..8881253865c35814490eebacdc5373f9ee279d18 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -0,0 +1,216 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 4 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 629 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 20231 +1, 17899 +2, 22695 +3, 18031 +4, 17767 +5, 18031 +6, 17767 +7, 17987 +0, 27085 +1, 27085 +2, 27085 +3, 27085 +4, 27085 +5, 27085 +6, 27085 +7, 27085 +0, 30589 +1, 30655 +2, 30547 +3, 30585 +4, 30535 +5, 30539 +6, 30667 +7, 30683 +0, 40621 +1, 40695 +2, 40581 +3, 40565 +4, 40587 +5, 40563 +6, 40689 +7, 40757 +0, 41131 +1, 41215 +2, 41065 +3, 41065 +4, 41065 +5, 41061 +6, 41197 +7, 41215 +0, 41135 +1, 41221 +2, 41069 +3, 41069 +4, 41069 +5, 41067 +6, 41201 +7, 41221 +0, 108921 +1, 109077 +2, 108897 +3, 108871 +4, 108923 +5, 108921 +6, 109057 +7, 109085 +0, 109391 +1, 109477 +2, 109325 +3, 109325 +4, 109325 +5, 109323 +6, 109457 +7, 109477 +0, 109659 +1, 109793 +2, 109647 +3, 109643 +4, 109693 +5, 109669 +6, 109819 +7, 109807 +0, 110407 +1, 110495 +2, 110327 +3, 110353 +4, 110347 +5, 110343 +6, 110473 +7, 110491 +0, 111027 +1, 111139 +2, 110989 +3, 110983 +4, 111023 +5, 111037 +6, 111133 +7, 111157 +0, 125553 +1, 125633 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125633 +0, 125553 +1, 125633 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125633 +0, 125553 +1, 125633 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125633 +0, 125553 +1, 125633 +2, 125013 +3, 125007 +4, 125009 +5, 125003 +6, 125143 +7, 125633 +0, 9575 +1, 9578 +2, 9578 +3, 9898 +4, 125007 +5, 11178 +6, 9578 +7, 9578 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..c27069a6b792ee9c42cff1f12e366de5d91ab2a0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..604110d3aefe150f92f1a570e2295675e7a2cf04 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log @@ -0,0 +1,7034 @@ +[23:40:49] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:41:00.361000 229212 torch/distributed/run.py:766] +W0829 23:41:00.361000 229212 torch/distributed/run.py:766] ***************************************** +W0829 23:41:00.361000 229212 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:41:00.361000 229212 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 72 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 0 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 2 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 6 is connected to 73 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W829 23:41:18.053045186 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:41:18.053044982 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:41:18.053100300 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:41:18.053137048 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:41:18.053152757 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:41:18.053198162 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:41:18.053217715 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:41:18.210507834 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.728 seconds +time to initialize megatron (seconds): 7.302 +[after megatron is initialized] datetime: 2026-08-29 23:41:24 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1567: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:41:28 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:41:28 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3815.20, 3820.38) + train/valid/test-data-iterators-setup ..........: (66.63, 71.14) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-29 23:41:28 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 200.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.01 MiB + total_size : 204.01 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [6272, 6016, 6272, 6272, 6144, 6144, 6144, 6144, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[30, -1, -1, -1, -1, -1, -1, -1], [55, -1, -1, -1, -1, -1, -1, -1], [30, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1], [26, -1, -1, -1, -1, -1, -1, -1], [55, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [40, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [6144, 6272, 6144, 6272, 6144, 6144, 6272, 6272, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[15, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [29, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1], [48, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [6272, 6272, 6016, 6272, 6272, 6016, 6272, 6272, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [6272, 6272, 6016, 6144, 6144, 6144, 6272, 6144, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[18, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [25, -1, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled +Number of parameters in transformer block in billions: 13.53 [2026-08-29 23:41:50] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21828.7 | throughput per GPU (TFLOP/s/GPU): 5.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092157E+01 | loss scale: 1.0 | grad norm: 30.276 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 34475.4970703125 | max allocated: 93364.55908203125 | reserved: 95432.0 | max reserved: 95432.0 + [2026-08-29 23:41:51] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 677.3 | throughput per GPU (TFLOP/s/GPU): 188.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091870E+01 | loss scale: 1.0 | grad norm: 30.188 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64922 dup_rows=24166 dup_rows_remote=21086 dedup_ratio=0.245829 router_tokens_per_expert=[766, 792, 779, 806, 822, 754, 776, 737, 781, 750, 827, 778, 783, 749, 714, 766, 775, 789, 756, 768, 773, 743, 805, 802, 753, 782, 790, 778, 767, 773, 821, 750, 783, 748, 682, 764, 709, 821, 770, 772, 795, 709, 729, 780, 767, 766, 773, 744, 731, 793, 770, 734, 784, 744, 814, 780, 801, 731, 776, 743, 790, 761, 744, 739] payload_rows_per_dst=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] dup_rows_per_dst=[3136, 3080, 3046, 3070, 2880, 2930, 3034, 2990] combine_rows_per_src=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64792 dup_rows=24064 dup_rows_remote=20924 dedup_ratio=0.244792 router_tokens_per_expert=[722, 789, 748, 730, 800, 759, 765, 760, 791, 795, 762, 768, 798, 794, 796, 740, 766, 815, 801, 764, 785, 778, 756, 807, 782, 766, 760, 741, 791, 775, 765, 765, 767, 768, 682, 748, 714, 764, 721, 735, 745, 851, 779, 738, 759, 767, 737, 742, 733, 808, 753, 811, 756, 758, 737, 825, 732, 782, 806, 842, 767, 770, 771, 750] payload_rows_per_dst=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] dup_rows_per_dst=[2642, 2754, 2798, 2944, 3296, 2972, 3518, 3140] combine_rows_per_src=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21016 dedup_ratio=0.244629 router_tokens_per_expert=[745, 717, 776, 802, 751, 792, 740, 758, 800, 804, 754, 791, 764, 792, 800, 766, 751, 746, 807, 712, 804, 738, 780, 781, 752, 778, 806, 740, 760, 782, 730, 736, 740, 741, 757, 808, 798, 780, 786, 780, 741, 819, 782, 725, 767, 751, 779, 776, 743, 766, 740, 724, 787, 790, 779, 740, 737, 815, 775, 784, 783, 751, 781, 772] payload_rows_per_dst=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] dup_rows_per_dst=[3020, 3086, 2954, 2972, 3032, 2978, 2940, 3066] combine_rows_per_src=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65098 dup_rows=23836 dup_rows_remote=20824 dedup_ratio=0.242472 router_tokens_per_expert=[751, 759, 810, 729, 763, 836, 747, 766, 779, 758, 819, 816, 754, 754, 771, 790, 793, 714, 765, 757, 764, 781, 799, 763, 786, 723, 752, 747, 757, 762, 744, 781, 752, 757, 764, 757, 776, 798, 751, 770, 807, 744, 794, 792, 745, 774, 758, 777, 795, 772, 703, 788, 775, 772, 739, 751, 812, 790, 764, 782, 762, 748, 741, 752] payload_rows_per_dst=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] dup_rows_per_dst=[3014, 3028, 2986, 2892, 3004, 3012, 2920, 2980] combine_rows_per_src=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64900 dup_rows=24108 dup_rows_remote=21060 dedup_ratio=0.245239 router_tokens_per_expert=[761, 800, 809, 777, 773, 790, 787, 758, 753, 737, 800, 764, 746, 756, 838, 765, 779, 721, 725, 775, 795, 749, 776, 792, 786, 729, 738, 762, 801, 801, 777, 778, 832, 744, 777, 820, 752, 750, 780, 721, 763, 745, 746, 783, 819, 750, 745, 763, 730, 731, 795, 780, 763, 782, 776, 746, 771, 753, 723, 743, 758, 778, 789, 746] payload_rows_per_dst=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] dup_rows_per_dst=[3110, 3058, 2974, 3048, 3048, 2998, 2930, 2942] combine_rows_per_src=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64892 dup_rows=24128 dup_rows_remote=21062 dedup_ratio=0.245443 router_tokens_per_expert=[782, 754, 773, 735, 755, 868, 802, 770, 801, 774, 745, 748, 721, 751, 730, 809, 776, 726, 786, 784, 812, 744, 798, 749, 779, 751, 787, 793, 787, 760, 770, 786, 748, 800, 812, 713, 742, 732, 777, 731, 805, 790, 774, 797, 776, 755, 761, 731, 748, 778, 758, 758, 778, 758, 794, 706, 725, 786, 753, 801, 767, 764, 755, 773] payload_rows_per_dst=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] dup_rows_per_dst=[3086, 3026, 3066, 3072, 2958, 3112, 2902, 2906] combine_rows_per_src=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64974 dup_rows=24242 dup_rows_remote=21426 dedup_ratio=0.246602 router_tokens_per_expert=[769, 777, 782, 788, 743, 768, 751, 761, 715, 764, 817, 793, 769, 756, 784, 808, 811, 768, 774, 820, 762, 756, 783, 753, 749, 801, 806, 779, 733, 748, 778, 743, 750, 760, 749, 724, 750, 827, 804, 782, 788, 782, 776, 774, 789, 768, 760, 754, 710, 774, 771, 725, 728, 762, 768, 714, 729, 762, 776, 793, 746, 750, 781, 817] payload_rows_per_dst=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] dup_rows_per_dst=[3002, 3030, 3048, 3068, 3040, 3166, 2816, 3072] combine_rows_per_src=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344032 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266264544 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532185056 weight_movement_bytes=69206016 total_nvlink_bytes=601391072 weight_to_token_ratio=0.130041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342864 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730896 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118928 weight_movement_bytes=34603008 total_nvlink_bytes=565721936 weight_to_token_ratio=0.065151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343696 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206864 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070032 weight_movement_bytes=34603008 total_nvlink_bytes=566673040 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343688 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985096 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626504 weight_movement_bytes=34603008 total_nvlink_bytes=568229512 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343840 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266174240 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004640 weight_movement_bytes=69206016 total_nvlink_bytes=601210656 weight_to_token_ratio=0.130085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343816 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266141448 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531939080 weight_movement_bytes=69206016 total_nvlink_bytes=601145096 weight_to_token_ratio=0.130101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=345600 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266479104 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532612608 weight_movement_bytes=34603008 total_nvlink_bytes=567215616 weight_to_token_ratio=0.064968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65034 dup_rows=23906 dup_rows_remote=20782 dedup_ratio=0.243184 router_tokens_per_expert=[749, 829, 788, 739, 749, 734, 775, 806, 784, 766, 757, 750, 697, 777, 750, 756, 794, 746, 744, 815, 827, 742, 726, 775, 789, 715, 750, 749, 773, 729, 845, 732, 800, 770, 793, 776, 772, 780, 760, 778, 712, 754, 798, 796, 780, 779, 778, 720, 788, 714, 776, 795, 772, 752, 852, 796, 775, 783, 733, 712, 809, 776, 765, 751] payload_rows_per_dst=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] dup_rows_per_dst=[2808, 2846, 2984, 2932, 3110, 2864, 3238, 3124] combine_rows_per_src=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65124 dup_rows=24024 dup_rows_remote=20920 dedup_ratio=0.244385 router_tokens_per_expert=[714, 774, 762, 750, 810, 818, 770, 774, 773, 777, 744, 751, 755, 752, 784, 794, 824, 790, 718, 739, 776, 784, 779, 732, 743, 753, 774, 763, 823, 788, 783, 816, 759, 775, 791, 708, 751, 785, 770, 735, 802, 766, 751, 765, 780, 814, 770, 800, 775, 793, 749, 742, 729, 749, 817, 767, 758, 732, 755, 710, 743, 734, 808, 782] payload_rows_per_dst=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] dup_rows_per_dst=[3028, 3104, 2974, 3050, 2918, 3094, 3042, 2814] combine_rows_per_src=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65084 dup_rows=23878 dup_rows_remote=20968 dedup_ratio=0.242900 router_tokens_per_expert=[759, 767, 796, 757, 792, 774, 736, 784, 793, 731, 733, 753, 769, 787, 770, 715, 759, 800, 791, 766, 761, 738, 761, 775, 753, 813, 796, 759, 753, 765, 716, 815, 749, 765, 748, 801, 797, 757, 752, 757, 773, 801, 748, 805, 802, 771, 792, 729, 738, 748, 769, 749, 754, 781, 793, 779, 786, 762, 731, 809, 761, 804, 738, 766] payload_rows_per_dst=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] dup_rows_per_dst=[3046, 2926, 3026, 3008, 2910, 3022, 2978, 2962] combine_rows_per_src=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64978 dup_rows=23860 dup_rows_remote=20740 dedup_ratio=0.242716 router_tokens_per_expert=[763, 802, 813, 769, 785, 774, 791, 783, 798, 769, 769, 785, 726, 766, 783, 788, 799, 774, 744, 785, 792, 785, 770, 763, 758, 721, 835, 750, 783, 810, 742, 779, 803, 725, 774, 723, 788, 729, 758, 720, 735, 815, 751, 781, 755, 719, 789, 771, 804, 744, 723, 757, 761, 809, 716, 763, 757, 770, 722, 727, 769, 772, 817, 751] payload_rows_per_dst=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] dup_rows_per_dst=[3120, 3074, 3048, 3114, 2786, 2892, 2866, 2960] combine_rows_per_src=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65270 dup_rows=23858 dup_rows_remote=20816 dedup_ratio=0.242696 router_tokens_per_expert=[768, 801, 754, 873, 739, 743, 788, 768, 760, 727, 832, 733, 700, 798, 743, 772, 822, 711, 821, 764, 773, 759, 751, 754, 780, 769, 748, 794, 758, 760, 765, 734, 806, 775, 793, 740, 797, 765, 802, 805, 775, 718, 759, 794, 746, 784, 755, 778, 733, 770, 730, 774, 754, 760, 769, 755, 767, 766, 782, 798, 744, 756, 787, 753] payload_rows_per_dst=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] dup_rows_per_dst=[3032, 2956, 2998, 2864, 3064, 3042, 2904, 2998] combine_rows_per_src=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64952 dup_rows=24198 dup_rows_remote=21242 dedup_ratio=0.246155 router_tokens_per_expert=[786, 754, 783, 814, 779, 805, 763, 821, 781, 765, 781, 788, 767, 760, 779, 764, 794, 741, 735, 733, 786, 755, 771, 786, 756, 757, 740, 767, 774, 743, 753, 765, 775, 734, 818, 721, 764, 805, 759, 766, 749, 754, 788, 759, 740, 767, 747, 844, 750, 756, 771, 783, 729, 724, 787, 774, 789, 776, 723, 727, 810, 761, 800, 756] payload_rows_per_dst=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] dup_rows_per_dst=[3118, 3120, 2988, 2956, 2990, 3026, 2962, 3038] combine_rows_per_src=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65036 dup_rows=24140 dup_rows_remote=21236 dedup_ratio=0.245565 router_tokens_per_expert=[726, 756, 715, 751, 777, 748, 836, 775, 772, 763, 737, 761, 751, 796, 746, 776, 769, 777, 733, 785, 749, 730, 722, 751, 766, 783, 726, 769, 773, 759, 770, 768, 788, 745, 759, 755, 739, 774, 799, 805, 806, 801, 793, 760, 740, 744, 807, 759, 806, 799, 784, 764, 797, 777, 810, 753, 789, 745, 763, 766, 813, 791, 761, 744] payload_rows_per_dst=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] dup_rows_per_dst=[2940, 2934, 2904, 3018, 3026, 3062, 3162, 3094] combine_rows_per_src=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343264 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266722528 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533101792 weight_movement_bytes=34603008 total_nvlink_bytes=567704800 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344176 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092080 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839984 weight_movement_bytes=34603008 total_nvlink_bytes=568442992 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65238 dup_rows=23924 dup_rows_remote=20994 dedup_ratio=0.243368 router_tokens_per_expert=[759, 790, 739, 777, 787, 727, 789, 777, 754, 746, 741, 779, 759, 783, 812, 719, 787, 770, 758, 736, 784, 767, 764, 812, 788, 767, 758, 747, 779, 793, 778, 786, 741, 747, 761, 810, 817, 805, 719, 812, 796, 737, 775, 788, 755, 801, 791, 744, 764, 731, 764, 774, 739, 766, 727, 771, 783, 732, 769, 765, 779, 740, 774, 763] payload_rows_per_dst=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] dup_rows_per_dst=[3016, 2978, 3010, 2956, 2996, 3082, 2930, 2956] combine_rows_per_src=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344208 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928272 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512336 weight_movement_bytes=69206016 total_nvlink_bytes=602718352 weight_to_token_ratio=0.129718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344344 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267690264 ingress_bytes=371154944 link_bytes=371154944 token_bytes=535036184 weight_movement_bytes=103809024 total_nvlink_bytes=638845208 weight_to_token_ratio=0.194022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396626174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=345088 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732544 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533120000 weight_movement_bytes=34603008 total_nvlink_bytes=567723008 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344776 combine_payload_bytes=266043392 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266388168 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532431560 weight_movement_bytes=69206016 total_nvlink_bytes=601637576 weight_to_token_ratio=0.129981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=342872 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266492760 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532642648 weight_movement_bytes=138412032 total_nvlink_bytes=671054680 weight_to_token_ratio=0.259859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344928 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559776 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774624 weight_movement_bytes=34603008 total_nvlink_bytes=569377632 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64790 dup_rows=24250 dup_rows_remote=21198 dedup_ratio=0.246684 router_tokens_per_expert=[729, 794, 808, 771, 738, 809, 784, 784, 765, 730, 743, 722, 817, 750, 724, 776, 777, 745, 766, 806, 780, 735, 756, 760, 795, 764, 749, 767, 725, 762, 808, 769, 791, 761, 732, 744, 775, 779, 778, 736, 785, 733, 811, 800, 775, 780, 780, 728, 755, 763, 790, 753, 775, 768, 809, 789, 773, 783, 763, 734, 785, 779, 766, 771] payload_rows_per_dst=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] dup_rows_per_dst=[2850, 2958, 3258, 2996, 2908, 2950, 3278, 3052] combine_rows_per_src=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64956 dup_rows=24154 dup_rows_remote=21096 dedup_ratio=0.245707 router_tokens_per_expert=[820, 810, 763, 763, 796, 757, 753, 772, 726, 760, 803, 751, 762, 759, 765, 800, 713, 768, 756, 753, 730, 718, 741, 778, 805, 779, 786, 763, 758, 748, 786, 759, 774, 768, 749, 796, 756, 745, 760, 824, 846, 771, 739, 770, 767, 791, 763, 720, 783, 776, 760, 773, 759, 768, 771, 765, 761, 749, 795, 794, 757, 752, 788, 761] payload_rows_per_dst=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] dup_rows_per_dst=[3104, 3058, 2816, 3006, 2996, 3000, 3114, 3060] combine_rows_per_src=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65148 dup_rows=23942 dup_rows_remote=21106 dedup_ratio=0.243551 router_tokens_per_expert=[760, 780, 789, 776, 798, 764, 780, 739, 809, 795, 786, 757, 750, 802, 764, 787, 754, 747, 727, 784, 778, 748, 796, 792, 758, 790, 748, 757, 776, 773, 740, 776, 739, 723, 783, 763, 765, 738, 757, 757, 790, 774, 791, 794, 718, 745, 731, 766, 758, 759, 822, 787, 724, 768, 771, 758, 757, 797, 807, 740, 813, 735, 789, 753] payload_rows_per_dst=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] dup_rows_per_dst=[3070, 3130, 3046, 2982, 2836, 2924, 2940, 3014] combine_rows_per_src=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64888 dup_rows=24146 dup_rows_remote=21136 dedup_ratio=0.245626 router_tokens_per_expert=[734, 750, 769, 775, 724, 809, 748, 747, 736, 803, 773, 754, 780, 743, 746, 752, 806, 805, 792, 750, 767, 780, 784, 758, 776, 763, 733, 781, 787, 766, 730, 754, 787, 780, 759, 786, 785, 756, 744, 778, 756, 733, 791, 808, 764, 758, 744, 786, 749, 769, 807, 772, 779, 737, 789, 806, 791, 765, 739, 797, 780, 794, 719, 769] payload_rows_per_dst=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] dup_rows_per_dst=[2900, 3026, 3192, 2968, 2990, 3010, 3056, 3004] combine_rows_per_src=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343952 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723792 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103632 weight_movement_bytes=69206016 total_nvlink_bytes=600309648 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65012 dup_rows=24028 dup_rows_remote=21080 dedup_ratio=0.244425 router_tokens_per_expert=[775, 778, 744, 777, 777, 768, 810, 763, 759, 771, 815, 748, 782, 729, 767, 817, 776, 785, 765, 724, 741, 776, 791, 748, 796, 740, 765, 757, 801, 750, 794, 788, 759, 701, 757, 743, 789, 774, 791, 744, 783, 792, 792, 773, 819, 767, 724, 779, 774, 768, 771, 774, 754, 695, 807, 765, 754, 741, 764, 763, 789, 780, 741, 748] payload_rows_per_dst=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] dup_rows_per_dst=[3136, 3026, 2948, 2982, 2918, 3052, 3042, 2924] combine_rows_per_src=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64812 dup_rows=24104 dup_rows_remote=21102 dedup_ratio=0.245199 router_tokens_per_expert=[736, 756, 776, 783, 774, 793, 794, 774, 781, 699, 783, 749, 776, 743, 745, 788, 733, 799, 750, 707, 792, 699, 755, 777, 796, 774, 793, 778, 750, 779, 765, 760, 782, 733, 805, 755, 807, 778, 770, 771, 749, 791, 862, 786, 769, 745, 775, 741, 760, 756, 742, 738, 745, 748, 758, 781, 819, 768, 766, 740, 800, 790, 790, 775] payload_rows_per_dst=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] dup_rows_per_dst=[3098, 2974, 2928, 3002, 3120, 3018, 2788, 3176] combine_rows_per_src=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=345016 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037432 weight_movement_bytes=69206016 total_nvlink_bytes=603243448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344208 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403984 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463760 weight_movement_bytes=34603008 total_nvlink_bytes=567066768 weight_to_token_ratio=0.064987 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65208 dup_rows=23848 dup_rows_remote=20904 dedup_ratio=0.242594 router_tokens_per_expert=[775, 778, 765, 800, 738, 752, 739, 792, 804, 767, 785, 805, 755, 762, 787, 768, 774, 743, 800, 764, 765, 772, 809, 787, 788, 807, 761, 757, 792, 744, 728, 756, 790, 789, 786, 755, 822, 747, 762, 756, 749, 746, 744, 747, 754, 753, 764, 766, 776, 762, 798, 742, 727, 786, 748, 757, 766, 746, 770, 766, 761, 778, 723, 797] payload_rows_per_dst=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] dup_rows_per_dst=[2996, 3102, 2990, 2938, 3034, 2940, 2944, 2904] combine_rows_per_src=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=344096 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266125344 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906592 weight_movement_bytes=69206016 total_nvlink_bytes=601112608 weight_to_token_ratio=0.130109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65200 dup_rows=23856 dup_rows_remote=20932 dedup_ratio=0.242676 router_tokens_per_expert=[790, 795, 750, 738, 750, 748, 758, 744, 750, 767, 795, 791, 757, 761, 779, 800, 773, 761, 785, 744, 763, 728, 777, 779, 759, 762, 769, 805, 797, 781, 770, 778, 800, 795, 786, 784, 742, 821, 737, 722, 733, 800, 817, 787, 773, 721, 695, 778, 766, 710, 775, 758, 749, 787, 775, 780, 769, 731, 770, 772, 786, 770, 775, 784] payload_rows_per_dst=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] dup_rows_per_dst=[2924, 3296, 3008, 3032, 2828, 2812, 2926, 3030] combine_rows_per_src=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344368 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633520 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922672 weight_movement_bytes=34603008 total_nvlink_bytes=567525680 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343656 combine_payload_bytes=265469952 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265813608 ingress_bytes=265469952 link_bytes=265813608 token_bytes=531283560 weight_movement_bytes=0 total_nvlink_bytes=531283560 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000235207e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64914 dup_rows=24086 dup_rows_remote=21034 dedup_ratio=0.245015 router_tokens_per_expert=[786, 731, 766, 747, 750, 770, 765, 730, 788, 765, 796, 740, 799, 803, 821, 790, 754, 758, 748, 767, 792, 774, 755, 734, 793, 736, 779, 768, 770, 768, 716, 801, 740, 774, 769, 760, 776, 754, 798, 773, 767, 779, 771, 766, 757, 765, 771, 732, 765, 768, 774, 737, 751, 845, 770, 756, 721, 764, 718, 828, 785, 790, 802, 766] payload_rows_per_dst=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] dup_rows_per_dst=[2882, 2768, 3442, 3018, 2974, 2896, 3054, 3052] combine_rows_per_src=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344528 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267403728 ingress_bytes=370868224 link_bytes=370868224 token_bytes=534462928 weight_movement_bytes=103809024 total_nvlink_bytes=638271952 weight_to_token_ratio=0.194231 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395547270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65046 dup_rows=24028 dup_rows_remote=21004 dedup_ratio=0.244425 router_tokens_per_expert=[768, 788, 784, 758, 788, 810, 799, 724, 774, 736, 747, 754, 747, 776, 725, 825, 767, 790, 825, 750, 757, 749, 768, 711, 804, 793, 691, 719, 788, 822, 755, 769, 714, 781, 762, 729, 818, 807, 763, 753, 770, 779, 746, 785, 803, 782, 795, 767, 771, 777, 756, 747, 754, 708, 796, 750, 753, 752, 779, 750, 785, 799, 744, 816] payload_rows_per_dst=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] dup_rows_per_dst=[3112, 3000, 3036, 2954, 3024, 3024, 2870, 3008] combine_rows_per_src=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65110 dup_rows=23780 dup_rows_remote=20764 dedup_ratio=0.241903 router_tokens_per_expert=[749, 785, 759, 751, 761, 747, 775, 762, 769, 721, 757, 767, 773, 746, 752, 811, 762, 737, 769, 758, 708, 748, 782, 775, 797, 816, 790, 828, 761, 771, 760, 759, 791, 779, 746, 795, 760, 733, 787, 743, 782, 789, 822, 796, 741, 771, 769, 745, 785, 797, 757, 795, 750, 780, 740, 764, 779, 744, 765, 758, 749, 778, 786, 770] payload_rows_per_dst=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] dup_rows_per_dst=[2912, 2946, 2852, 3136, 2972, 3016, 3030, 2916] combine_rows_per_src=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64964 dup_rows=24014 dup_rows_remote=20976 dedup_ratio=0.244283 router_tokens_per_expert=[784, 796, 745, 768, 746, 782, 776, 770, 803, 773, 793, 715, 773, 740, 807, 778, 771, 790, 743, 741, 799, 767, 769, 773, 725, 739, 742, 751, 767, 759, 796, 761, 734, 776, 754, 786, 781, 754, 772, 786, 801, 732, 774, 790, 785, 780, 772, 752, 764, 758, 750, 769, 758, 769, 751, 723, 764, 801, 747, 753, 821, 795, 780, 778] payload_rows_per_dst=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] dup_rows_per_dst=[3010, 3038, 2982, 2872, 2942, 3074, 3020, 3076] combine_rows_per_src=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343792 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231536 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119280 weight_movement_bytes=34603008 total_nvlink_bytes=566722288 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=65120 dup_rows=24116 dup_rows_remote=21116 dedup_ratio=0.245321 router_tokens_per_expert=[715, 786, 761, 776, 721, 765, 806, 773, 744, 722, 752, 755, 780, 827, 783, 797, 780, 755, 703, 750, 790, 738, 767, 751, 783, 770, 796, 717, 807, 724, 794, 709, 761, 778, 788, 759, 777, 780, 792, 747, 799, 733, 784, 780, 750, 766, 769, 718, 818, 752, 800, 788, 847, 781, 756, 796, 757, 752, 753, 783, 763, 778, 779, 771] payload_rows_per_dst=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] dup_rows_per_dst=[2968, 3064, 3000, 2910, 3084, 2904, 3294, 2892] combine_rows_per_src=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65038 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[755, 747, 759, 766, 776, 752, 787, 775, 759, 772, 775, 761, 771, 806, 748, 768, 742, 725, 758, 772, 797, 779, 821, 780, 775, 775, 792, 770, 747, 759, 763, 794, 735, 789, 773, 785, 770, 757, 759, 810, 783, 778, 775, 791, 776, 779, 757, 754, 778, 775, 728, 734, 782, 789, 761, 744, 760, 766, 756, 739, 753, 762, 766, 762] payload_rows_per_dst=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] dup_rows_per_dst=[2974, 2988, 3050, 2992, 3038, 3066, 2870, 2930] combine_rows_per_src=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344200 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772616 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201032 weight_movement_bytes=34603008 total_nvlink_bytes=567804040 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343496 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267034056 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724616 weight_movement_bytes=34603008 total_nvlink_bytes=568327624 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64784 dup_rows=24240 dup_rows_remote=21238 dedup_ratio=0.246582 router_tokens_per_expert=[783, 783, 753, 776, 781, 785, 745, 772, 805, 733, 694, 787, 790, 765, 769, 737, 735, 783, 736, 791, 727, 757, 754, 782, 724, 779, 816, 753, 801, 787, 787, 805, 770, 720, 768, 757, 729, 797, 755, 756, 785, 776, 736, 753, 794, 791, 773, 728, 771, 821, 788, 739, 726, 782, 738, 776, 799, 782, 784, 857, 762, 734, 748, 782] payload_rows_per_dst=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] dup_rows_per_dst=[3076, 3046, 2922, 3152, 2926, 3008, 3002, 3108] combine_rows_per_src=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343760 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436304 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528848 weight_movement_bytes=69206016 total_nvlink_bytes=601734864 weight_to_token_ratio=0.129957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64932 dup_rows=23982 dup_rows_remote=20940 dedup_ratio=0.243958 router_tokens_per_expert=[749, 796, 780, 751, 784, 801, 771, 777, 732, 756, 748, 767, 763, 788, 825, 751, 760, 793, 780, 757, 737, 772, 782, 762, 799, 771, 748, 755, 787, 718, 749, 775, 746, 775, 771, 774, 818, 780, 755, 810, 769, 774, 750, 765, 759, 842, 801, 711, 770, 789, 784, 709, 772, 733, 731, 799, 758, 798, 757, 771, 766, 724, 801, 706] payload_rows_per_dst=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] dup_rows_per_dst=[3042, 2978, 3020, 2998, 3110, 2950, 2984, 2900] combine_rows_per_src=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344944 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076464 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807984 weight_movement_bytes=34603008 total_nvlink_bytes=568410992 weight_to_token_ratio=0.064823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343816 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739464 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135112 weight_movement_bytes=69206016 total_nvlink_bytes=602341128 weight_to_token_ratio=0.129810 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344088 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265699352 ingress_bytes=369164288 link_bytes=369164288 token_bytes=531054616 weight_movement_bytes=103809024 total_nvlink_bytes=634863640 weight_to_token_ratio=0.195477 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389135496e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65282 dup_rows=23692 dup_rows_remote=20708 dedup_ratio=0.241007 router_tokens_per_expert=[728, 776, 769, 797, 764, 753, 775, 746, 818, 761, 784, 773, 756, 783, 787, 779, 758, 743, 733, 748, 740, 744, 731, 745, 794, 764, 763, 790, 800, 723, 788, 761, 716, 750, 801, 733, 811, 745, 743, 753, 790, 771, 751, 771, 790, 801, 828, 734, 823, 785, 820, 749, 761, 780, 750, 769, 755, 787, 799, 782, 776, 770, 720, 764] payload_rows_per_dst=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] dup_rows_per_dst=[2950, 2998, 2774, 2968, 2878, 3102, 3038, 2984] combine_rows_per_src=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65074 dup_rows=24024 dup_rows_remote=21110 dedup_ratio=0.244385 router_tokens_per_expert=[763, 739, 740, 775, 752, 745, 777, 783, 759, 822, 737, 794, 789, 813, 766, 749, 763, 714, 776, 759, 771, 752, 783, 814, 754, 748, 738, 782, 789, 766, 803, 779, 795, 762, 730, 754, 774, 755, 728, 762, 758, 813, 744, 767, 745, 807, 776, 769, 818, 788, 762, 759, 750, 815, 758, 776, 734, 736, 778, 744, 757, 805, 778, 761] payload_rows_per_dst=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] dup_rows_per_dst=[2914, 2982, 3034, 3100, 2914, 3042, 3050, 2988] combine_rows_per_src=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343488 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266304960 ingress_bytes=369770496 link_bytes=369770496 token_bytes=532266432 weight_movement_bytes=103809024 total_nvlink_bytes=636075456 weight_to_token_ratio=0.195032 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391416607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64944 dup_rows=24092 dup_rows_remote=21080 dedup_ratio=0.245076 router_tokens_per_expert=[753, 742, 740, 775, 733, 685, 812, 806, 798, 764, 815, 720, 776, 843, 795, 774, 759, 771, 776, 755, 742, 790, 757, 762, 777, 822, 788, 740, 758, 800, 740, 755, 775, 816, 738, 774, 766, 769, 794, 715, 800, 797, 755, 773, 705, 768, 802, 740, 709, 765, 754, 765, 769, 788, 749, 776, 763, 811, 737, 772, 777, 745, 763, 799] payload_rows_per_dst=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] dup_rows_per_dst=[2892, 3128, 3020, 3052, 3052, 3012, 2920, 3016] combine_rows_per_src=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65052 dup_rows=23848 dup_rows_remote=20744 dedup_ratio=0.242594 router_tokens_per_expert=[785, 763, 779, 747, 776, 744, 766, 795, 828, 784, 821, 768, 744, 771, 766, 772, 732, 721, 767, 782, 785, 677, 759, 734, 741, 790, 777, 778, 708, 749, 800, 793, 761, 774, 797, 783, 769, 792, 835, 787, 774, 711, 754, 744, 813, 772, 756, 812, 701, 766, 749, 798, 776, 776, 795, 780, 769, 762, 757, 782, 767, 717, 769, 752] payload_rows_per_dst=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] dup_rows_per_dst=[2990, 3104, 2862, 2976, 2990, 3026, 2962, 2938] combine_rows_per_src=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=343960 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267739032 ingress_bytes=371204096 link_bytes=371204096 token_bytes=535134104 weight_movement_bytes=103809024 total_nvlink_bytes=638943128 weight_to_token_ratio=0.193987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396811129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344736 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887840 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430944 weight_movement_bytes=34603008 total_nvlink_bytes=568033952 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65060 dup_rows=23926 dup_rows_remote=21068 dedup_ratio=0.243388 router_tokens_per_expert=[745, 796, 804, 720, 771, 777, 752, 728, 741, 803, 750, 779, 732, 746, 771, 790, 772, 740, 780, 782, 780, 791, 714, 729, 818, 808, 780, 745, 773, 774, 790, 767, 796, 745, 747, 712, 756, 764, 750, 762, 760, 735, 754, 763, 804, 791, 747, 852, 814, 758, 757, 768, 743, 763, 772, 753, 795, 796, 749, 801, 763, 765, 772, 797] payload_rows_per_dst=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] dup_rows_per_dst=[2996, 3004, 2858, 3134, 2858, 3050, 2974, 3052] combine_rows_per_src=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65070 dup_rows=23914 dup_rows_remote=20842 dedup_ratio=0.243266 router_tokens_per_expert=[753, 727, 763, 752, 778, 769, 760, 743, 744, 852, 785, 736, 772, 780, 782, 727, 761, 788, 742, 775, 751, 776, 773, 765, 775, 784, 759, 739, 740, 795, 821, 783, 754, 771, 778, 817, 799, 784, 758, 776, 763, 730, 769, 774, 762, 758, 759, 764, 758, 764, 750, 768, 782, 712, 746, 747, 765, 823, 833, 754, 785, 782, 771, 746] payload_rows_per_dst=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] dup_rows_per_dst=[2966, 3054, 2962, 3072, 3032, 2842, 2940, 3046] combine_rows_per_src=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64824 dup_rows=24186 dup_rows_remote=21104 dedup_ratio=0.246033 router_tokens_per_expert=[756, 784, 775, 772, 757, 730, 770, 776, 751, 739, 784, 748, 738, 747, 715, 805, 765, 757, 798, 766, 733, 806, 751, 811, 804, 813, 744, 772, 762, 767, 798, 796, 773, 758, 740, 770, 734, 812, 810, 776, 742, 731, 754, 804, 752, 776, 751, 754, 783, 768, 807, 746, 749, 779, 789, 767, 734, 787, 754, 787, 760, 774, 794, 747] payload_rows_per_dst=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] dup_rows_per_dst=[3054, 2874, 3010, 3134, 3096, 2890, 3082, 3046] combine_rows_per_src=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344096 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354720 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365344 weight_movement_bytes=69206016 total_nvlink_bytes=601571360 weight_to_token_ratio=0.129997 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343184 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796176 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249168 weight_movement_bytes=103809024 total_nvlink_bytes=637058192 weight_to_token_ratio=0.194673 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344512 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266830272 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533316032 weight_movement_bytes=34603008 total_nvlink_bytes=567919040 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343648 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870368 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397088 weight_movement_bytes=69206016 total_nvlink_bytes=602603104 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64594 dup_rows=24364 dup_rows_remote=21242 dedup_ratio=0.247843 router_tokens_per_expert=[799, 734, 782, 777, 762, 783, 787, 791, 802, 809, 771, 773, 777, 758, 831, 800, 771, 776, 755, 731, 752, 772, 740, 743, 752, 765, 775, 772, 723, 764, 808, 816, 781, 709, 761, 805, 778, 760, 771, 763, 758, 745, 719, 773, 747, 769, 763, 778, 783, 771, 790, 765, 805, 816, 747, 764, 744, 760, 732, 797, 709, 798, 729, 711] payload_rows_per_dst=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] dup_rows_per_dst=[3122, 3400, 2928, 2970, 3070, 2920, 3090, 2864] combine_rows_per_src=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64700 dup_rows=24208 dup_rows_remote=21012 dedup_ratio=0.246257 router_tokens_per_expert=[788, 709, 765, 801, 765, 743, 734, 768, 770, 764, 778, 778, 800, 812, 769, 774, 820, 741, 765, 744, 717, 781, 738, 787, 768, 792, 735, 764, 739, 784, 777, 727, 786, 787, 799, 749, 769, 758, 767, 773, 760, 774, 771, 740, 777, 807, 777, 726, 775, 773, 778, 806, 760, 776, 783, 757, 775, 768, 767, 746, 756, 787, 777, 751] payload_rows_per_dst=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] dup_rows_per_dst=[2976, 3096, 2966, 2808, 3126, 2784, 3256, 3196] combine_rows_per_src=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65214 dup_rows=23760 dup_rows_remote=20768 dedup_ratio=0.241699 router_tokens_per_expert=[748, 769, 742, 780, 767, 760, 780, 742, 788, 760, 768, 776, 743, 753, 769, 802, 738, 787, 763, 756, 799, 770, 743, 760, 784, 773, 781, 767, 737, 756, 765, 812, 769, 770, 789, 784, 726, 796, 738, 789, 759, 801, 797, 797, 768, 809, 756, 782, 780, 766, 772, 796, 755, 758, 736, 761, 737, 750, 777, 762, 766, 782, 722, 764] payload_rows_per_dst=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] dup_rows_per_dst=[2952, 2938, 3000, 2976, 2992, 3070, 2874, 2958] combine_rows_per_src=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65012 dup_rows=23964 dup_rows_remote=20876 dedup_ratio=0.243774 router_tokens_per_expert=[734, 811, 808, 824, 750, 770, 791, 741, 761, 819, 750, 739, 779, 739, 827, 692, 715, 738, 770, 767, 804, 777, 817, 769, 766, 760, 775, 764, 731, 777, 762, 789, 783, 768, 755, 720, 787, 774, 774, 756, 761, 756, 822, 791, 798, 730, 774, 776, 746, 749, 814, 787, 759, 750, 756, 754, 738, 734, 797, 775, 759, 731, 773, 789] payload_rows_per_dst=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] dup_rows_per_dst=[3144, 2966, 2946, 2938, 2980, 3088, 3012, 2890] combine_rows_per_src=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=343344 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264920368 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529497392 weight_movement_bytes=34603008 total_nvlink_bytes=564100400 weight_to_token_ratio=0.065351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65030 dup_rows=23988 dup_rows_remote=21046 dedup_ratio=0.244019 router_tokens_per_expert=[765, 775, 773, 765, 758, 741, 772, 754, 776, 750, 797, 760, 780, 744, 749, 758, 793, 762, 807, 802, 755, 812, 817, 762, 773, 805, 798, 803, 745, 749, 764, 787, 750, 711, 772, 735, 744, 782, 808, 716, 743, 807, 790, 769, 791, 736, 763, 765, 764, 781, 749, 777, 778, 773, 736, 767, 785, 772, 769, 762, 751, 760, 792, 703] payload_rows_per_dst=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] dup_rows_per_dst=[2924, 2942, 3234, 3002, 2966, 2948, 2958, 3014] combine_rows_per_src=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342848 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265354048 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530365248 weight_movement_bytes=34603008 total_nvlink_bytes=564968256 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=343928 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460472 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577016 weight_movement_bytes=34603008 total_nvlink_bytes=569180024 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65202 dup_rows=24004 dup_rows_remote=21120 dedup_ratio=0.244181 router_tokens_per_expert=[804, 727, 769, 760, 782, 756, 795, 756, 766, 746, 754, 736, 792, 786, 785, 763, 758, 706, 775, 751, 778, 726, 743, 754, 781, 784, 752, 840, 779, 825, 759, 779, 812, 789, 746, 794, 779, 744, 722, 805, 780, 758, 737, 786, 802, 759, 737, 789, 748, 752, 756, 789, 749, 748, 776, 750, 771, 767, 739, 799, 741, 794, 767, 800] payload_rows_per_dst=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] dup_rows_per_dst=[3080, 3046, 2884, 3068, 2984, 2930, 2906, 3106] combine_rows_per_src=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65200 dup_rows=23756 dup_rows_remote=20952 dedup_ratio=0.241659 router_tokens_per_expert=[820, 796, 754, 784, 777, 737, 738, 756, 753, 824, 728, 753, 773, 762, 749, 779, 791, 755, 766, 762, 741, 773, 797, 747, 756, 817, 771, 775, 818, 815, 791, 765, 755, 776, 741, 757, 790, 789, 732, 752, 782, 784, 761, 769, 789, 768, 736, 767, 788, 764, 774, 772, 744, 719, 728, 787, 774, 786, 768, 770, 722, 761, 763, 761] payload_rows_per_dst=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] dup_rows_per_dst=[3012, 3044, 2954, 3110, 2924, 2994, 2804, 2914] combine_rows_per_src=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65324 dup_rows=23660 dup_rows_remote=20758 dedup_ratio=0.240682 router_tokens_per_expert=[796, 750, 759, 757, 779, 793, 755, 752, 704, 827, 725, 763, 790, 774, 770, 774, 704, 762, 747, 770, 752, 772, 806, 763, 766, 766, 781, 780, 734, 765, 757, 762, 774, 805, 758, 767, 742, 736, 771, 794, 764, 787, 807, 758, 747, 804, 786, 770, 761, 809, 767, 769, 754, 769, 764, 789, 790, 789, 774, 769, 760, 777, 739, 747] payload_rows_per_dst=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] dup_rows_per_dst=[2908, 2932, 2894, 2902, 2944, 3118, 3022, 2940] combine_rows_per_src=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343552 combine_payload_bytes=266289152 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266632704 ingress_bytes=404701184 link_bytes=404701184 token_bytes=532921856 weight_movement_bytes=138412032 total_nvlink_bytes=671333888 weight_to_token_ratio=0.259723 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522857974e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344304 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707184 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070064 weight_movement_bytes=34603008 total_nvlink_bytes=567673072 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=345288 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267412680 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534480072 weight_movement_bytes=34603008 total_nvlink_bytes=569083080 weight_to_token_ratio=0.064741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344608 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267403808 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463008 weight_movement_bytes=34603008 total_nvlink_bytes=569066016 weight_to_token_ratio=0.064744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267567104 dispatch_meta_bytes=344328 combine_payload_bytes=267567104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267911432 ingress_bytes=371376128 link_bytes=371376128 token_bytes=535478536 weight_movement_bytes=103809024 total_nvlink_bytes=639287560 weight_to_token_ratio=0.193862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397458471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64638 dup_rows=24056 dup_rows_remote=20644 dedup_ratio=0.244710 router_tokens_per_expert=[768, 717, 737, 781, 778, 711, 841, 797, 809, 783, 762, 768, 788, 758, 756, 785, 789, 750, 750, 739, 758, 723, 781, 747, 739, 814, 739, 790, 781, 777, 778, 771, 731, 809, 788, 761, 721, 738, 803, 742, 766, 783, 776, 754, 784, 781, 755, 751, 789, 758, 753, 751, 791, 786, 757, 747, 772, 773, 742, 789, 794, 762, 804, 776] payload_rows_per_dst=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] dup_rows_per_dst=[3412, 2700, 3432, 2622, 3032, 2970, 2990, 2898] combine_rows_per_src=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64786 dup_rows=23952 dup_rows_remote=20792 dedup_ratio=0.243652 router_tokens_per_expert=[755, 749, 714, 769, 788, 739, 759, 756, 806, 723, 779, 766, 774, 782, 819, 786, 728, 736, 797, 789, 746, 749, 713, 805, 779, 769, 788, 735, 778, 774, 763, 775, 806, 808, 776, 774, 706, 749, 820, 725, 785, 815, 742, 770, 775, 771, 801, 758, 795, 794, 770, 773, 745, 831, 729, 762, 769, 712, 742, 800, 749, 778, 790, 744] payload_rows_per_dst=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] dup_rows_per_dst=[2932, 3008, 2750, 2924, 3066, 3226, 2886, 3160] combine_rows_per_src=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65016 dup_rows=23994 dup_rows_remote=21008 dedup_ratio=0.244080 router_tokens_per_expert=[758, 794, 810, 756, 737, 755, 725, 759, 735, 768, 738, 840, 749, 763, 708, 790, 729, 777, 796, 788, 738, 817, 786, 775, 806, 786, 795, 755, 771, 805, 778, 792, 824, 746, 762, 778, 780, 733, 793, 724, 692, 797, 798, 760, 779, 771, 730, 780, 761, 779, 758, 795, 748, 734, 783, 763, 794, 726, 792, 783, 762, 758, 746, 744] payload_rows_per_dst=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] dup_rows_per_dst=[2914, 2950, 3098, 3178, 2986, 3010, 2910, 2948] combine_rows_per_src=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=341128 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265098376 ingress_bytes=299360256 link_bytes=299360256 token_bytes=529855624 weight_movement_bytes=34603008 total_nvlink_bytes=564458632 weight_to_token_ratio=0.065306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126468543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64960 dup_rows=24024 dup_rows_remote=20988 dedup_ratio=0.244385 router_tokens_per_expert=[753, 732, 746, 792, 753, 748, 767, 797, 831, 778, 799, 767, 762, 738, 754, 746, 799, 754, 781, 820, 807, 765, 742, 719, 766, 749, 771, 781, 768, 745, 791, 760, 762, 774, 679, 733, 744, 763, 803, 775, 803, 801, 750, 734, 759, 754, 781, 796, 812, 782, 823, 781, 768, 791, 751, 756, 722, 739, 746, 792, 809, 753, 786, 749] payload_rows_per_dst=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] dup_rows_per_dst=[2986, 3042, 3038, 3010, 2870, 3036, 3094, 2948] combine_rows_per_src=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64706 dup_rows=24258 dup_rows_remote=21134 dedup_ratio=0.246765 router_tokens_per_expert=[741, 768, 830, 789, 772, 786, 790, 775, 791, 758, 806, 780, 779, 758, 790, 770, 768, 799, 768, 767, 729, 770, 784, 771, 711, 756, 779, 823, 772, 772, 762, 805, 776, 767, 814, 761, 722, 799, 737, 759, 749, 759, 707, 761, 793, 769, 760, 736, 759, 793, 755, 809, 807, 759, 723, 769, 732, 716, 753, 752, 708, 744, 775, 810] payload_rows_per_dst=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] dup_rows_per_dst=[3070, 3124, 2994, 3084, 2972, 2960, 3032, 3022] combine_rows_per_src=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344096 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649632 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955168 weight_movement_bytes=34603008 total_nvlink_bytes=567558176 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342312 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265705768 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531069224 weight_movement_bytes=34603008 total_nvlink_bytes=565672232 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65086 dup_rows=23884 dup_rows_remote=20794 dedup_ratio=0.242961 router_tokens_per_expert=[763, 808, 823, 745, 755, 803, 825, 775, 701, 705, 763, 745, 737, 744, 754, 789, 743, 780, 830, 778, 756, 753, 790, 782, 795, 703, 804, 776, 744, 807, 720, 768, 761, 787, 748, 759, 812, 764, 760, 761, 756, 744, 812, 726, 804, 770, 778, 730, 700, 771, 741, 785, 752, 743, 738, 796, 784, 775, 805, 753, 802, 798, 795, 778] payload_rows_per_dst=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] dup_rows_per_dst=[3044, 2848, 3090, 2858, 3010, 3018, 2962, 3054] combine_rows_per_src=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64918 dup_rows=24106 dup_rows_remote=21180 dedup_ratio=0.245219 router_tokens_per_expert=[795, 780, 804, 713, 853, 732, 702, 731, 766, 826, 780, 780, 732, 722, 755, 782, 773, 742, 784, 749, 800, 757, 747, 765, 769, 779, 804, 768, 738, 810, 722, 745, 779, 765, 792, 772, 785, 749, 782, 737, 743, 776, 729, 744, 778, 784, 776, 790, 787, 788, 735, 730, 801, 787, 762, 713, 756, 805, 828, 740, 801, 821, 727, 785] payload_rows_per_dst=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] dup_rows_per_dst=[2966, 2944, 2978, 3048, 3064, 2990, 2926, 3190] combine_rows_per_src=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64810 dup_rows=24048 dup_rows_remote=21056 dedup_ratio=0.244629 router_tokens_per_expert=[776, 776, 731, 763, 743, 799, 766, 743, 810, 795, 781, 753, 718, 794, 782, 761, 786, 752, 769, 771, 768, 724, 770, 755, 755, 755, 812, 806, 758, 768, 763, 802, 733, 794, 766, 770, 759, 748, 781, 787, 781, 750, 776, 751, 785, 764, 816, 734, 841, 761, 784, 805, 787, 783, 770, 765, 739, 784, 739, 733, 730, 776, 720, 735] payload_rows_per_dst=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] dup_rows_per_dst=[2940, 3042, 3024, 2992, 3024, 2994, 3096, 2936] combine_rows_per_src=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343792 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419952 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496112 weight_movement_bytes=34603008 total_nvlink_bytes=567099120 weight_to_token_ratio=0.064983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=343360 combine_payload_bytes=265035776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265379136 ingress_bytes=334241792 link_bytes=334241792 token_bytes=530414912 weight_movement_bytes=69206016 total_nvlink_bytes=599620928 weight_to_token_ratio=0.130475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257724955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343520 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266935776 ingress_bytes=370401280 link_bytes=370401280 token_bytes=533528032 weight_movement_bytes=103809024 total_nvlink_bytes=637337056 weight_to_token_ratio=0.194571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393790197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344392 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248520 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152648 weight_movement_bytes=69206016 total_nvlink_bytes=601358664 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64726 dup_rows=24204 dup_rows_remote=21164 dedup_ratio=0.246216 router_tokens_per_expert=[774, 770, 813, 735, 795, 746, 789, 785, 750, 753, 760, 775, 771, 762, 750, 748, 784, 745, 739, 764, 751, 802, 762, 767, 784, 754, 773, 769, 745, 744, 810, 757, 751, 744, 777, 738, 747, 760, 776, 765, 810, 780, 827, 766, 761, 746, 817, 794, 813, 756, 760, 761, 712, 767, 761, 759, 768, 750, 774, 776, 791, 777, 774, 768] payload_rows_per_dst=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] dup_rows_per_dst=[3040, 2962, 3160, 2910, 3132, 2908, 3054, 3038] combine_rows_per_src=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343464 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265805224 ingress_bytes=369270784 link_bytes=369270784 token_bytes=531266984 weight_movement_bytes=103809024 total_nvlink_bytes=635076008 weight_to_token_ratio=0.195399 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389536231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64748 dup_rows=24148 dup_rows_remote=21168 dedup_ratio=0.245646 router_tokens_per_expert=[789, 794, 762, 799, 794, 807, 744, 786, 796, 725, 777, 770, 797, 730, 770, 738, 748, 790, 826, 751, 755, 756, 727, 746, 855, 768, 787, 815, 694, 743, 798, 770, 762, 781, 765, 778, 796, 775, 743, 794, 811, 706, 735, 766, 803, 747, 784, 704, 750, 735, 755, 726, 823, 761, 774, 766, 787, 799, 780, 734, 761, 776, 708, 760] payload_rows_per_dst=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] dup_rows_per_dst=[3166, 3002, 2982, 3078, 2980, 2952, 3018, 2970] combine_rows_per_src=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64426 dup_rows=24308 dup_rows_remote=20874 dedup_ratio=0.247274 router_tokens_per_expert=[749, 780, 793, 787, 760, 774, 776, 778, 765, 783, 733, 794, 745, 772, 774, 779, 764, 747, 766, 780, 759, 753, 790, 732, 758, 743, 788, 752, 766, 798, 774, 747, 788, 763, 792, 752, 768, 778, 789, 794, 805, 748, 767, 766, 741, 790, 755, 773, 749, 764, 758, 742, 795, 767, 839, 715, 755, 749, 752, 764, 827, 730, 761, 757] payload_rows_per_dst=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] dup_rows_per_dst=[2562, 3084, 2920, 3042, 2994, 2944, 3328, 3434] combine_rows_per_src=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64918 dup_rows=23896 dup_rows_remote=20820 dedup_ratio=0.243083 router_tokens_per_expert=[761, 818, 778, 794, 796, 737, 777, 781, 732, 811, 747, 781, 746, 725, 755, 772, 772, 795, 762, 754, 806, 757, 769, 784, 788, 760, 798, 779, 729, 761, 756, 779, 751, 785, 830, 762, 740, 798, 740, 749, 785, 795, 805, 805, 773, 772, 756, 792, 760, 714, 791, 756, 758, 734, 695, 793, 727, 777, 789, 762, 732, 727, 778, 761] payload_rows_per_dst=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] dup_rows_per_dst=[3090, 2920, 3070, 2990, 2966, 3076, 2906, 2878] combine_rows_per_src=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343560 combine_payload_bytes=265117696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265461256 ingress_bytes=265117696 link_bytes=265461256 token_bytes=530578952 weight_movement_bytes=0 total_nvlink_bytes=530578952 weight_to_token_ratio=0.000000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.989093349e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343664 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265551472 ingress_bytes=369016832 link_bytes=369016832 token_bytes=530759280 weight_movement_bytes=103809024 total_nvlink_bytes=634568304 weight_to_token_ratio=0.195586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388580631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=263888896 dispatch_meta_bytes=341200 combine_payload_bytes=263888896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264230096 ingress_bytes=298491904 link_bytes=298491904 token_bytes=528118992 weight_movement_bytes=34603008 total_nvlink_bytes=562722000 weight_to_token_ratio=0.065521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123201004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65028 dup_rows=23894 dup_rows_remote=20972 dedup_ratio=0.243062 router_tokens_per_expert=[761, 733, 701, 797, 770, 767, 783, 750, 728, 781, 767, 759, 784, 758, 792, 783, 783, 787, 745, 784, 715, 732, 771, 751, 782, 782, 757, 793, 749, 777, 764, 757, 797, 771, 771, 790, 792, 757, 784, 746, 827, 762, 795, 771, 763, 797, 775, 770, 759, 767, 790, 753, 739, 770, 774, 782, 770, 767, 764, 791, 782, 748, 715, 770] payload_rows_per_dst=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] dup_rows_per_dst=[2920, 2922, 2886, 3048, 3040, 3116, 3000, 2962] combine_rows_per_src=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64788 dup_rows=24136 dup_rows_remote=21162 dedup_ratio=0.245524 router_tokens_per_expert=[760, 781, 796, 832, 809, 786, 753, 799, 765, 809, 704, 753, 795, 777, 750, 744, 772, 746, 751, 780, 782, 777, 785, 784, 783, 777, 743, 800, 771, 779, 780, 757, 786, 746, 766, 762, 775, 756, 823, 772, 760, 731, 773, 735, 793, 745, 723, 804, 747, 755, 767, 744, 756, 776, 748, 777, 681, 721, 761, 787, 776, 769, 777, 780] payload_rows_per_dst=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] dup_rows_per_dst=[3232, 2952, 2974, 3068, 3046, 2900, 2944, 3020] combine_rows_per_src=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65026 dup_rows=24006 dup_rows_remote=21018 dedup_ratio=0.244202 router_tokens_per_expert=[761, 762, 744, 778, 784, 804, 829, 743, 746, 765, 780, 766, 776, 738, 788, 757, 816, 776, 765, 757, 733, 802, 778, 741, 772, 762, 731, 714, 792, 801, 750, 758, 763, 700, 762, 796, 772, 749, 832, 791, 726, 719, 774, 728, 712, 812, 788, 825, 809, 785, 735, 736, 787, 782, 746, 750, 767, 756, 789, 782, 756, 797, 794, 763] payload_rows_per_dst=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] dup_rows_per_dst=[3028, 2984, 3028, 2948, 3058, 2992, 2988, 2980] combine_rows_per_src=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65118 dup_rows=23966 dup_rows_remote=21008 dedup_ratio=0.243795 router_tokens_per_expert=[787, 799, 782, 735, 770, 745, 768, 809, 734, 796, 757, 771, 743, 792, 763, 777, 795, 766, 748, 745, 725, 821, 767, 777, 748, 766, 766, 742, 767, 763, 726, 811, 727, 758, 791, 786, 767, 787, 785, 788, 806, 768, 769, 773, 773, 767, 759, 757, 747, 766, 751, 799, 767, 776, 817, 779, 783, 735, 747, 760, 771, 778, 707, 747] payload_rows_per_dst=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] dup_rows_per_dst=[3146, 2936, 2976, 2958, 3080, 2982, 2978, 2910] combine_rows_per_src=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342952 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266247080 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151208 weight_movement_bytes=69206016 total_nvlink_bytes=601357224 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344000 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698688 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053376 weight_movement_bytes=34603008 total_nvlink_bytes=567656384 weight_to_token_ratio=0.064915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=343800 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265715448 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087096 weight_movement_bytes=34603008 total_nvlink_bytes=565690104 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344176 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690672 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037168 weight_movement_bytes=34603008 total_nvlink_bytes=567640176 weight_to_token_ratio=0.064917 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344504 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067832 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791160 weight_movement_bytes=34603008 total_nvlink_bytes=568394168 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64796 dup_rows=24010 dup_rows_remote=20938 dedup_ratio=0.244242 router_tokens_per_expert=[758, 780, 755, 731, 802, 738, 748, 769, 775, 753, 803, 736, 771, 773, 781, 736, 758, 714, 716, 803, 764, 786, 794, 766, 799, 784, 767, 731, 763, 793, 734, 789, 809, 763, 799, 777, 763, 782, 799, 793, 717, 789, 747, 794, 755, 744, 788, 751, 759, 739, 761, 834, 727, 763, 769, 835, 762, 799, 799, 785, 780, 744, 717, 739] payload_rows_per_dst=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] dup_rows_per_dst=[2944, 2976, 2910, 3030, 3072, 2954, 3112, 3012] combine_rows_per_src=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65116 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[729, 761, 765, 768, 787, 755, 808, 753, 799, 820, 801, 807, 730, 736, 739, 773, 820, 813, 775, 775, 760, 786, 776, 769, 783, 798, 747, 769, 732, 737, 736, 731, 731, 814, 778, 762, 807, 755, 777, 781, 764, 730, 755, 773, 734, 794, 778, 740, 758, 759, 773, 781, 762, 766, 765, 788, 728, 751, 770, 774, 772, 766, 731, 797] payload_rows_per_dst=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] dup_rows_per_dst=[3052, 2984, 3180, 2934, 2964, 2822, 3014, 2918] combine_rows_per_src=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65104 dup_rows=23946 dup_rows_remote=21016 dedup_ratio=0.243591 router_tokens_per_expert=[774, 812, 778, 762, 765, 793, 770, 820, 775, 754, 737, 768, 807, 763, 790, 787, 780, 799, 793, 808, 771, 773, 732, 744, 770, 778, 776, 726, 749, 734, 813, 773, 727, 770, 815, 769, 744, 753, 802, 766, 755, 741, 775, 744, 748, 751, 734, 783, 777, 748, 807, 749, 718, 844, 732, 757, 749, 789, 750, 785, 741, 754, 712, 789] payload_rows_per_dst=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] dup_rows_per_dst=[3026, 3102, 2978, 3038, 3040, 2822, 3010, 2930] combine_rows_per_src=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65258 dup_rows=23796 dup_rows_remote=20846 dedup_ratio=0.242065 router_tokens_per_expert=[810, 768, 752, 753, 716, 814, 772, 763, 793, 785, 768, 792, 759, 790, 753, 802, 779, 721, 748, 757, 731, 808, 779, 754, 727, 736, 783, 755, 783, 766, 773, 749, 789, 728, 803, 751, 777, 778, 767, 777, 782, 756, 809, 761, 761, 762, 703, 766, 775, 772, 737, 789, 778, 782, 778, 754, 791, 748, 770, 808, 777, 753, 776, 755] payload_rows_per_dst=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] dup_rows_per_dst=[2954, 3066, 2944, 2956, 2954, 2950, 3014, 2958] combine_rows_per_src=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=342936 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265747352 ingress_bytes=334610432 link_bytes=334610432 token_bytes=531151768 weight_movement_bytes=69206016 total_nvlink_bytes=600357784 weight_to_token_ratio=0.130294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259112117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65136 dup_rows=23824 dup_rows_remote=20802 dedup_ratio=0.242350 router_tokens_per_expert=[751, 717, 739, 779, 760, 819, 765, 737, 762, 773, 773, 748, 740, 783, 787, 817, 761, 722, 756, 818, 802, 749, 760, 757, 754, 806, 759, 777, 762, 761, 797, 775, 761, 756, 804, 736, 743, 742, 809, 753, 742, 740, 792, 803, 766, 808, 770, 760, 755, 803, 739, 758, 765, 720, 816, 747, 779, 772, 748, 807, 741, 782, 782, 787] payload_rows_per_dst=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] dup_rows_per_dst=[2926, 3022, 2920, 3066, 2884, 2958, 2974, 3074] combine_rows_per_src=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343728 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058864 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774000 weight_movement_bytes=34603008 total_nvlink_bytes=568377008 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010464 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676448 weight_movement_bytes=69206016 total_nvlink_bytes=602882464 weight_to_token_ratio=0.129678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65036 dup_rows=24032 dup_rows_remote=21034 dedup_ratio=0.244466 router_tokens_per_expert=[739, 734, 783, 744, 784, 780, 776, 724, 763, 776, 724, 798, 801, 813, 772, 754, 764, 751, 791, 722, 774, 785, 780, 750, 797, 813, 810, 766, 789, 773, 733, 774, 761, 743, 783, 783, 704, 766, 771, 752, 756, 742, 832, 793, 755, 720, 746, 777, 743, 753, 783, 758, 783, 809, 800, 749, 746, 809, 758, 842, 751, 759, 768, 720] payload_rows_per_dst=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] dup_rows_per_dst=[2924, 3026, 2998, 3160, 2936, 2950, 3084, 2954] combine_rows_per_src=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64974 dup_rows=24110 dup_rows_remote=21048 dedup_ratio=0.245260 router_tokens_per_expert=[746, 786, 742, 711, 731, 735, 743, 762, 799, 786, 802, 743, 754, 749, 760, 741, 780, 734, 776, 768, 825, 749, 729, 759, 755, 808, 752, 796, 773, 782, 786, 813, 770, 767, 768, 793, 769, 737, 733, 759, 800, 761, 790, 762, 807, 814, 745, 819, 758, 762, 801, 823, 746, 713, 759, 779, 753, 786, 783, 784, 741, 761, 764, 770] payload_rows_per_dst=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] dup_rows_per_dst=[2856, 3066, 2978, 3090, 2934, 3136, 3062, 2988] combine_rows_per_src=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21076 dedup_ratio=0.245524 router_tokens_per_expert=[787, 742, 797, 818, 726, 775, 724, 766, 756, 834, 773, 751, 738, 758, 766, 737, 772, 714, 742, 777, 765, 760, 779, 759, 797, 747, 795, 765, 762, 808, 738, 759, 736, 773, 779, 758, 773, 823, 762, 749, 790, 757, 791, 764, 768, 789, 801, 785, 795, 755, 720, 711, 739, 789, 740, 819, 755, 775, 776, 802, 735, 771, 785, 800] payload_rows_per_dst=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] dup_rows_per_dst=[3006, 3020, 2878, 3060, 3092, 3110, 2906, 3064] combine_rows_per_src=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344416 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267641184 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937952 weight_movement_bytes=34603008 total_nvlink_bytes=569540960 weight_to_token_ratio=0.064686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343752 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267140808 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533937864 weight_movement_bytes=69206016 total_nvlink_bytes=603143880 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344280 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731736 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119192 weight_movement_bytes=34603008 total_nvlink_bytes=567722200 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344088 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477592 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611096 weight_movement_bytes=34603008 total_nvlink_bytes=567214104 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343848 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266116904 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889960 weight_movement_bytes=138412032 total_nvlink_bytes=670301992 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65208 dup_rows=23854 dup_rows_remote=20904 dedup_ratio=0.242655 router_tokens_per_expert=[785, 731, 775, 780, 714, 803, 757, 803, 719, 771, 771, 790, 711, 781, 779, 748, 698, 790, 790, 734, 791, 770, 776, 714, 781, 770, 747, 772, 782, 749, 783, 831, 754, 776, 777, 781, 729, 747, 758, 774, 781, 786, 761, 789, 728, 786, 788, 804, 783, 772, 773, 782, 777, 771, 799, 760, 766, 778, 782, 761, 782, 808, 688, 755] payload_rows_per_dst=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] dup_rows_per_dst=[2920, 2872, 2956, 3056, 2950, 3050, 3088, 2962] combine_rows_per_src=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64588 dup_rows=24128 dup_rows_remote=20816 dedup_ratio=0.245443 router_tokens_per_expert=[743, 808, 791, 790, 776, 791, 770, 757, 726, 818, 746, 716, 748, 755, 751, 772, 757, 768, 730, 772, 778, 771, 826, 789, 791, 768, 762, 759, 775, 759, 777, 757, 782, 810, 747, 745, 764, 785, 708, 774, 763, 763, 760, 775, 790, 782, 767, 798, 780, 748, 744, 754, 769, 802, 770, 766, 796, 751, 763, 785, 767, 762, 718, 767] payload_rows_per_dst=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] dup_rows_per_dst=[3090, 2800, 2720, 3048, 3108, 3096, 2954, 3312] combine_rows_per_src=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64840 dup_rows=24192 dup_rows_remote=21124 dedup_ratio=0.246094 router_tokens_per_expert=[811, 732, 726, 732, 786, 800, 761, 812, 745, 787, 771, 823, 781, 749, 764, 714, 809, 779, 749, 757, 753, 766, 740, 740, 717, 779, 815, 777, 788, 704, 783, 717, 775, 787, 782, 753, 788, 774, 802, 795, 803, 748, 782, 791, 728, 791, 766, 802, 756, 742, 808, 773, 729, 752, 763, 794, 751, 760, 797, 804, 734, 761, 741, 753] payload_rows_per_dst=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] dup_rows_per_dst=[3068, 3162, 3130, 2930, 2976, 2962, 2962, 3002] combine_rows_per_src=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64818 dup_rows=24176 dup_rows_remote=21108 dedup_ratio=0.245931 router_tokens_per_expert=[792, 732, 776, 766, 764, 773, 772, 737, 769, 763, 796, 794, 734, 770, 755, 772, 792, 783, 778, 732, 777, 786, 783, 755, 767, 802, 732, 784, 736, 752, 801, 774, 783, 769, 796, 763, 727, 765, 790, 759, 791, 744, 800, 812, 760, 787, 744, 751, 768, 795, 719, 809, 786, 759, 806, 778, 737, 731, 781, 722, 740, 747, 778, 756] payload_rows_per_dst=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] dup_rows_per_dst=[2986, 3000, 3040, 3004, 3118, 3068, 3126, 2834] combine_rows_per_src=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341616 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264894064 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446512 weight_movement_bytes=34603008 total_nvlink_bytes=564049520 weight_to_token_ratio=0.065357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343856 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265928496 ingress_bytes=403996672 link_bytes=403996672 token_bytes=531513136 weight_movement_bytes=138412032 total_nvlink_bytes=669925168 weight_to_token_ratio=0.260411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520206952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65068 dup_rows=24052 dup_rows_remote=20990 dedup_ratio=0.244670 router_tokens_per_expert=[759, 748, 771, 773, 762, 708, 791, 760, 774, 770, 782, 745, 782, 817, 735, 770, 727, 805, 783, 788, 791, 772, 713, 744, 809, 796, 748, 771, 770, 731, 779, 766, 775, 740, 756, 767, 740, 779, 750, 744, 768, 824, 813, 806, 782, 749, 779, 755, 747, 770, 778, 754, 813, 779, 783, 765, 724, 820, 779, 790, 749, 766, 744, 724] payload_rows_per_dst=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] dup_rows_per_dst=[2902, 2984, 3062, 2996, 2942, 3142, 2980, 3044] combine_rows_per_src=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65160 dup_rows=23950 dup_rows_remote=21104 dedup_ratio=0.243632 router_tokens_per_expert=[766, 757, 742, 769, 823, 724, 740, 812, 803, 709, 790, 737, 771, 718, 727, 765, 750, 810, 803, 778, 784, 738, 748, 796, 748, 758, 810, 808, 791, 741, 767, 725, 736, 756, 747, 747, 764, 841, 780, 753, 821, 800, 754, 761, 768, 769, 766, 785, 764, 802, 781, 828, 735, 762, 763, 774, 759, 790, 744, 761, 776, 782, 718, 757] payload_rows_per_dst=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] dup_rows_per_dst=[3028, 2846, 3080, 3020, 2968, 3050, 3008, 2950] combine_rows_per_src=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64918 dup_rows=24008 dup_rows_remote=20996 dedup_ratio=0.244222 router_tokens_per_expert=[749, 756, 746, 736, 799, 757, 743, 767, 774, 842, 743, 757, 755, 770, 799, 781, 763, 798, 744, 803, 758, 844, 787, 784, 816, 736, 751, 747, 776, 717, 727, 763, 781, 766, 764, 765, 741, 746, 721, 785, 774, 796, 778, 785, 765, 777, 766, 776, 804, 745, 795, 774, 789, 740, 781, 767, 747, 773, 747, 749, 751, 769, 751, 796] payload_rows_per_dst=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] dup_rows_per_dst=[2930, 3162, 3130, 2852, 2968, 3050, 3012, 2904] combine_rows_per_src=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64998 dup_rows=24016 dup_rows_remote=21026 dedup_ratio=0.244303 router_tokens_per_expert=[763, 746, 766, 785, 745, 784, 747, 742, 836, 763, 799, 739, 775, 723, 793, 793, 771, 778, 754, 762, 785, 791, 796, 816, 781, 799, 787, 734, 796, 749, 745, 749, 753, 747, 765, 704, 783, 763, 733, 765, 778, 763, 723, 791, 786, 761, 744, 786, 769, 794, 754, 754, 767, 798, 787, 788, 721, 776, 705, 750, 784, 796, 760, 812] payload_rows_per_dst=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] dup_rows_per_dst=[3024, 3060, 3086, 2990, 2956, 2936, 3046, 2918] combine_rows_per_src=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343704 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265838232 ingress_bytes=369303552 link_bytes=369303552 token_bytes=531332760 weight_movement_bytes=103809024 total_nvlink_bytes=635141784 weight_to_token_ratio=0.195375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389659535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344232 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862760 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381288 weight_movement_bytes=69206016 total_nvlink_bytes=602587304 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=345056 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267240416 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135776 weight_movement_bytes=34603008 total_nvlink_bytes=568738784 weight_to_token_ratio=0.064783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343656 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266247784 ingress_bytes=404316160 link_bytes=404316160 token_bytes=532151912 weight_movement_bytes=138412032 total_nvlink_bytes=670563944 weight_to_token_ratio=0.260099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521409160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65120 dup_rows=23928 dup_rows_remote=20924 dedup_ratio=0.243408 router_tokens_per_expert=[840, 766, 754, 699, 777, 794, 825, 794, 783, 790, 786, 725, 748, 793, 750, 804, 734, 754, 717, 793, 769, 769, 766, 734, 760, 760, 771, 763, 784, 765, 771, 792, 762, 743, 707, 830, 737, 792, 791, 768, 763, 716, 787, 734, 774, 831, 794, 765, 766, 770, 798, 762, 743, 792, 762, 766, 796, 761, 731, 742, 726, 748, 793, 772] payload_rows_per_dst=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] dup_rows_per_dst=[3154, 2956, 2866, 3006, 3004, 3012, 2904, 3026] combine_rows_per_src=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344096 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575904 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807712 weight_movement_bytes=34603008 total_nvlink_bytes=567410720 weight_to_token_ratio=0.064945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21122 dedup_ratio=0.246379 router_tokens_per_expert=[751, 719, 754, 716, 792, 731, 778, 799, 821, 762, 752, 785, 769, 785, 787, 784, 776, 784, 756, 781, 815, 728, 816, 732, 769, 739, 760, 757, 734, 827, 809, 736, 799, 749, 754, 804, 785, 813, 713, 746, 790, 788, 719, 794, 775, 765, 731, 767, 755, 775, 757, 736, 769, 786, 778, 756, 777, 740, 781, 778, 758, 759, 820, 731] payload_rows_per_dst=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] dup_rows_per_dst=[2916, 2944, 3040, 2928, 3024, 3148, 3122, 3098] combine_rows_per_src=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64546 dup_rows=24216 dup_rows_remote=20904 dedup_ratio=0.246338 router_tokens_per_expert=[776, 747, 779, 757, 773, 772, 756, 752, 783, 748, 813, 768, 759, 752, 764, 797, 822, 765, 765, 791, 804, 782, 729, 799, 757, 723, 808, 740, 743, 795, 764, 780, 784, 735, 794, 764, 778, 788, 738, 772, 710, 704, 766, 796, 808, 770, 750, 789, 712, 750, 773, 779, 783, 758, 760, 791, 796, 792, 776, 772, 728, 753, 747, 773] payload_rows_per_dst=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] dup_rows_per_dst=[3312, 2804, 3228, 2804, 3018, 3008, 3042, 3000] combine_rows_per_src=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344176 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075696 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807216 weight_movement_bytes=103809024 total_nvlink_bytes=637616240 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65308 dup_rows=23780 dup_rows_remote=20958 dedup_ratio=0.241903 router_tokens_per_expert=[795, 774, 750, 767, 787, 810, 774, 759, 804, 743, 740, 838, 744, 795, 813, 778, 729, 781, 767, 794, 755, 805, 761, 779, 776, 790, 754, 791, 773, 757, 727, 728, 824, 776, 795, 789, 737, 721, 766, 789, 752, 723, 731, 789, 759, 739, 777, 749, 791, 710, 751, 760, 743, 748, 772, 751, 783, 775, 797, 762, 775, 738, 746, 796] payload_rows_per_dst=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] dup_rows_per_dst=[3054, 3150, 2914, 2940, 2976, 2822, 2886, 3038] combine_rows_per_src=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344024 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108888 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873752 weight_movement_bytes=69206016 total_nvlink_bytes=601079768 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=341800 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264722216 ingress_bytes=298983424 link_bytes=298983424 token_bytes=529102632 weight_movement_bytes=34603008 total_nvlink_bytes=563705640 weight_to_token_ratio=0.065399 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125050554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64984 dup_rows=24112 dup_rows_remote=21112 dedup_ratio=0.245280 router_tokens_per_expert=[782, 723, 788, 767, 761, 732, 770, 730, 774, 791, 801, 738, 767, 823, 754, 768, 758, 798, 759, 760, 798, 766, 734, 731, 752, 758, 757, 826, 790, 754, 766, 824, 769, 809, 758, 773, 757, 785, 750, 751, 760, 775, 780, 745, 751, 728, 788, 790, 774, 786, 788, 771, 775, 781, 741, 713, 770, 784, 755, 771, 795, 778, 737, 764] payload_rows_per_dst=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] dup_rows_per_dst=[2928, 3040, 3000, 3068, 2974, 3010, 3016, 3076] combine_rows_per_src=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65044 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[739, 737, 804, 804, 787, 785, 764, 776, 736, 744, 756, 765, 785, 736, 772, 786, 746, 780, 731, 747, 748, 742, 761, 800, 753, 747, 779, 758, 771, 737, 776, 772, 737, 784, 739, 804, 763, 751, 789, 794, 760, 782, 773, 783, 788, 766, 811, 770, 754, 801, 770, 793, 740, 758, 752, 788, 769, 824, 799, 771, 752, 773, 775, 715] payload_rows_per_dst=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] dup_rows_per_dst=[2986, 2972, 2822, 2988, 3002, 3238, 3060, 3004] combine_rows_per_src=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64904 dup_rows=23998 dup_rows_remote=21052 dedup_ratio=0.244120 router_tokens_per_expert=[732, 773, 741, 773, 798, 790, 788, 726, 755, 743, 746, 738, 760, 738, 793, 809, 792, 749, 783, 747, 760, 748, 776, 803, 781, 717, 789, 782, 763, 751, 780, 786, 799, 736, 719, 802, 777, 797, 719, 759, 798, 795, 817, 787, 794, 749, 768, 769, 712, 812, 766, 818, 773, 788, 754, 751, 738, 743, 769, 752, 789, 771, 758, 763] payload_rows_per_dst=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] dup_rows_per_dst=[3008, 2902, 2990, 2940, 3014, 3204, 2946, 2994] combine_rows_per_src=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=64998 dup_rows=23876 dup_rows_remote=20798 dedup_ratio=0.242879 router_tokens_per_expert=[669, 778, 762, 809, 782, 779, 734, 776, 742, 803, 720, 781, 763, 769, 750, 770, 738, 773, 741, 815, 791, 781, 744, 794, 740, 799, 778, 751, 839, 758, 781, 808, 776, 725, 780, 783, 780, 762, 772, 711, 775, 734, 761, 733, 751, 772, 765, 771, 729, 767, 775, 728, 781, 765, 786, 805, 746, 803, 717, 809, 739, 783, 847, 803] payload_rows_per_dst=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] dup_rows_per_dst=[2958, 2902, 2936, 3078, 2986, 2932, 2994, 3090] combine_rows_per_src=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267501568 dispatch_meta_bytes=345064 combine_payload_bytes=267501568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267846632 ingress_bytes=302104576 link_bytes=302104576 token_bytes=535348200 weight_movement_bytes=34603008 total_nvlink_bytes=569951208 weight_to_token_ratio=0.064636 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136795198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344384 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518848 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693312 weight_movement_bytes=34603008 total_nvlink_bytes=567296320 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344576 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266764800 ingress_bytes=335626240 link_bytes=335626240 token_bytes=533185024 weight_movement_bytes=69206016 total_nvlink_bytes=602391040 weight_to_token_ratio=0.129797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262934521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343824 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190608 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037392 weight_movement_bytes=34603008 total_nvlink_bytes=566640400 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65048 dup_rows=23854 dup_rows_remote=20720 dedup_ratio=0.242655 router_tokens_per_expert=[758, 734, 761, 771, 768, 756, 750, 765, 733, 719, 788, 771, 761, 782, 723, 758, 777, 819, 755, 754, 711, 761, 731, 768, 775, 788, 834, 729, 745, 760, 750, 792, 776, 785, 764, 753, 815, 792, 809, 774, 798, 775, 761, 762, 804, 769, 749, 789, 794, 765, 773, 770, 811, 759, 759, 752, 756, 791, 729, 798, 784, 746, 744, 799] payload_rows_per_dst=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] dup_rows_per_dst=[2930, 2814, 2852, 2950, 3134, 3092, 3054, 3028] combine_rows_per_src=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343184 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266574992 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806800 weight_movement_bytes=103809024 total_nvlink_bytes=636615824 weight_to_token_ratio=0.194834 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64594 dup_rows=24144 dup_rows_remote=21050 dedup_ratio=0.245605 router_tokens_per_expert=[837, 766, 797, 780, 750, 748, 794, 778, 723, 763, 799, 774, 788, 779, 786, 751, 737, 755, 791, 765, 766, 813, 792, 716, 804, 745, 758, 769, 789, 732, 775, 742, 760, 745, 849, 775, 778, 751, 776, 757, 716, 804, 761, 759, 784, 748, 745, 751, 735, 737, 783, 728, 770, 781, 728, 749, 743, 787, 808, 776, 787, 758, 788, 773] payload_rows_per_dst=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] dup_rows_per_dst=[3094, 3068, 3110, 3092, 3376, 2644, 2698, 3062] combine_rows_per_src=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64576 dup_rows=23950 dup_rows_remote=20666 dedup_ratio=0.243632 router_tokens_per_expert=[741, 780, 733, 792, 711, 782, 777, 790, 745, 735, 775, 770, 777, 799, 786, 835, 749, 747, 767, 754, 744, 727, 750, 747, 771, 752, 776, 758, 791, 780, 763, 778, 774, 769, 773, 734, 774, 760, 792, 802, 762, 760, 762, 787, 691, 773, 712, 775, 807, 795, 783, 763, 785, 779, 791, 796, 781, 765, 812, 781, 738, 769, 768, 757] payload_rows_per_dst=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] dup_rows_per_dst=[2926, 2726, 2838, 3020, 2882, 3142, 3132, 3284] combine_rows_per_src=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343072 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266779680 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533216288 weight_movement_bytes=103809024 total_nvlink_bytes=637025312 weight_to_token_ratio=0.194685 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=342576 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264919600 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529496624 weight_movement_bytes=34603008 total_nvlink_bytes=564099632 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65096 dup_rows=23970 dup_rows_remote=20950 dedup_ratio=0.243835 router_tokens_per_expert=[803, 742, 764, 773, 766, 761, 741, 788, 767, 740, 742, 730, 830, 831, 821, 764, 747, 778, 794, 806, 794, 795, 714, 772, 748, 775, 738, 784, 720, 732, 819, 799, 769, 804, 782, 807, 717, 749, 784, 780, 749, 767, 759, 786, 783, 752, 784, 749, 773, 780, 753, 730, 764, 743, 770, 792, 745, 772, 719, 772, 747, 754, 766, 773] payload_rows_per_dst=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] dup_rows_per_dst=[2968, 3092, 3050, 2992, 2950, 3020, 2972, 2926] combine_rows_per_src=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64954 dup_rows=23944 dup_rows_remote=20968 dedup_ratio=0.243571 router_tokens_per_expert=[822, 819, 773, 785, 801, 786, 701, 787, 740, 777, 718, 746, 805, 785, 795, 740, 777, 773, 758, 734, 776, 787, 769, 817, 731, 764, 782, 767, 764, 731, 760, 794, 748, 773, 790, 774, 757, 802, 729, 777, 770, 759, 764, 753, 787, 781, 778, 737, 721, 820, 778, 740, 770, 767, 765, 769, 705, 753, 803, 748, 796, 781, 761, 732] payload_rows_per_dst=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] dup_rows_per_dst=[3116, 3036, 2976, 2930, 2872, 3106, 2976, 2932] combine_rows_per_src=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264503296 dispatch_meta_bytes=340968 combine_payload_bytes=264503296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264844264 ingress_bytes=299106304 link_bytes=299106304 token_bytes=529347560 weight_movement_bytes=34603008 total_nvlink_bytes=563950568 weight_to_token_ratio=0.065369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125512942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64888 dup_rows=24068 dup_rows_remote=20982 dedup_ratio=0.244832 router_tokens_per_expert=[763, 773, 738, 759, 806, 791, 818, 767, 772, 784, 752, 745, 808, 791, 806, 759, 763, 768, 783, 790, 776, 735, 809, 786, 762, 743, 763, 779, 741, 781, 792, 761, 717, 793, 757, 766, 814, 757, 764, 787, 779, 766, 727, 771, 723, 781, 725, 731, 775, 749, 796, 775, 768, 750, 753, 744, 714, 803, 747, 760, 794, 790, 766, 746] payload_rows_per_dst=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] dup_rows_per_dst=[3026, 3086, 3122, 3048, 3032, 2848, 2946, 2960] combine_rows_per_src=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73930 payload_rows_remote=64518 dup_rows=24374 dup_rows_remote=21240 dedup_ratio=0.247945 router_tokens_per_expert=[751, 754, 773, 816, 776, 769, 718, 746, 765, 760, 804, 792, 746, 801, 759, 806, 746, 737, 762, 801, 759, 773, 753, 751, 781, 744, 761, 772, 709, 715, 817, 786, 797, 808, 733, 778, 723, 740, 771, 748, 801, 768, 741, 733, 792, 805, 728, 792, 769, 822, 782, 791, 738, 812, 797, 762, 754, 769, 761, 762, 750, 776, 750, 796] payload_rows_per_dst=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] dup_rows_per_dst=[2992, 3118, 2988, 3058, 2980, 3096, 3134, 3008] combine_rows_per_src=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65158 dup_rows=23816 dup_rows_remote=20960 dedup_ratio=0.242269 router_tokens_per_expert=[749, 769, 777, 733, 786, 785, 803, 805, 775, 703, 810, 788, 813, 782, 770, 779, 769, 776, 754, 765, 760, 789, 778, 774, 763, 748, 757, 758, 778, 759, 767, 763, 795, 829, 787, 787, 772, 767, 750, 751, 738, 705, 770, 772, 794, 779, 758, 790, 743, 762, 759, 772, 763, 805, 761, 737, 727, 741, 740, 734, 770, 831, 749, 729] payload_rows_per_dst=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] dup_rows_per_dst=[3064, 3112, 3008, 2856, 3058, 2944, 2910, 2864] combine_rows_per_src=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344184 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977400 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610616 weight_movement_bytes=34603008 total_nvlink_bytes=568213624 weight_to_token_ratio=0.064847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343688 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395272 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446856 weight_movement_bytes=34603008 total_nvlink_bytes=567049864 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343480 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124728 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905976 weight_movement_bytes=69206016 total_nvlink_bytes=601111992 weight_to_token_ratio=0.130109 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264265728 dispatch_meta_bytes=343032 combine_payload_bytes=264265728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264608760 ingress_bytes=368074752 link_bytes=368074752 token_bytes=528874488 weight_movement_bytes=103809024 total_nvlink_bytes=632683512 weight_to_token_ratio=0.196283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385035659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73904 payload_rows_remote=64742 dup_rows=24400 dup_rows_remote=21306 dedup_ratio=0.248210 router_tokens_per_expert=[769, 760, 785, 798, 799, 784, 771, 757, 733, 774, 742, 792, 821, 756, 813, 760, 779, 773, 738, 757, 767, 775, 795, 749, 777, 750, 755, 801, 765, 767, 778, 732, 817, 784, 725, 753, 759, 743, 752, 795, 792, 779, 769, 760, 824, 766, 766, 764, 782, 770, 760, 734, 740, 755, 731, 726, 770, 781, 798, 730, 744, 779, 745, 787] payload_rows_per_dst=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] dup_rows_per_dst=[3104, 3106, 3018, 3026, 3094, 3092, 2982, 2978] combine_rows_per_src=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65018 dup_rows=23948 dup_rows_remote=20910 dedup_ratio=0.243612 router_tokens_per_expert=[779, 804, 721, 839, 775, 698, 747, 752, 762, 741, 799, 760, 744, 746, 785, 773, 810, 772, 770, 766, 812, 753, 768, 762, 710, 753, 756, 753, 808, 805, 762, 802, 748, 762, 773, 784, 800, 808, 773, 788, 734, 749, 714, 778, 798, 757, 766, 736, 793, 794, 784, 740, 798, 757, 778, 729, 774, 794, 768, 739, 781, 768, 750, 750] payload_rows_per_dst=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] dup_rows_per_dst=[3038, 2972, 3066, 2982, 2926, 2922, 3044, 2998] combine_rows_per_src=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344472 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267231640 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534118808 weight_movement_bytes=34603008 total_nvlink_bytes=568721816 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64778 dup_rows=24292 dup_rows_remote=21274 dedup_ratio=0.247111 router_tokens_per_expert=[811, 792, 804, 745, 753, 763, 786, 746, 764, 746, 763, 780, 784, 724, 757, 740, 744, 749, 764, 767, 748, 802, 750, 796, 764, 804, 794, 734, 775, 775, 780, 752, 776, 791, 714, 778, 783, 795, 778, 766, 739, 759, 788, 759, 773, 773, 786, 771, 789, 770, 787, 793, 728, 761, 760, 753, 787, 767, 796, 746, 751, 766, 749, 764] payload_rows_per_dst=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] dup_rows_per_dst=[3042, 3020, 2992, 2834, 3166, 3236, 2984, 3018] combine_rows_per_src=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=344192 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265527424 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530710656 weight_movement_bytes=34603008 total_nvlink_bytes=565313664 weight_to_token_ratio=0.065201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343712 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657440 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971168 weight_movement_bytes=34603008 total_nvlink_bytes=567574176 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64872 dup_rows=24046 dup_rows_remote=21038 dedup_ratio=0.244609 router_tokens_per_expert=[778, 779, 763, 784, 747, 773, 731, 757, 744, 764, 762, 763, 738, 843, 737, 764, 741, 743, 751, 790, 758, 763, 770, 754, 751, 774, 797, 779, 767, 731, 783, 781, 786, 771, 787, 778, 805, 736, 789, 779, 777, 779, 799, 784, 790, 758, 732, 778, 729, 780, 771, 785, 734, 745, 761, 766, 786, 785, 755, 828, 727, 770, 765, 777] payload_rows_per_dst=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] dup_rows_per_dst=[2988, 2942, 2982, 3042, 3082, 3008, 3000, 3002] combine_rows_per_src=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64876 dup_rows=24156 dup_rows_remote=21218 dedup_ratio=0.245728 router_tokens_per_expert=[758, 689, 751, 740, 772, 744, 774, 760, 841, 795, 757, 748, 787, 761, 733, 770, 744, 758, 790, 761, 754, 767, 750, 781, 768, 765, 711, 759, 758, 773, 792, 760, 786, 782, 784, 770, 696, 732, 768, 734, 719, 782, 754, 794, 730, 755, 784, 777, 770, 793, 821, 788, 837, 774, 787, 740, 778, 852, 827, 815, 788, 784, 746, 734] payload_rows_per_dst=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] dup_rows_per_dst=[2924, 3054, 2938, 2938, 2938, 2948, 3232, 3184] combine_rows_per_src=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344208 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265674896 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005584 weight_movement_bytes=103809024 total_nvlink_bytes=634814608 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65112 dup_rows=23832 dup_rows_remote=20804 dedup_ratio=0.242432 router_tokens_per_expert=[769, 779, 798, 771, 746, 758, 791, 751, 773, 808, 732, 733, 776, 809, 791, 772, 807, 763, 731, 706, 703, 748, 764, 824, 818, 760, 745, 771, 795, 756, 793, 800, 784, 746, 763, 770, 735, 770, 758, 745, 763, 779, 777, 742, 754, 763, 745, 767, 760, 777, 770, 751, 767, 754, 759, 755, 769, 798, 764, 752, 793, 796, 795, 790] payload_rows_per_dst=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] dup_rows_per_dst=[3040, 3028, 2854, 3076, 2844, 2966, 2920, 3104] combine_rows_per_src=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65002 dup_rows=23946 dup_rows_remote=20750 dedup_ratio=0.243591 router_tokens_per_expert=[753, 765, 764, 779, 792, 750, 739, 825, 752, 788, 790, 780, 774, 816, 745, 776, 795, 745, 766, 778, 724, 810, 767, 775, 777, 800, 818, 795, 758, 821, 763, 770, 768, 749, 746, 754, 768, 707, 742, 755, 786, 729, 795, 739, 749, 748, 715, 736, 790, 773, 790, 807, 777, 749, 798, 792, 791, 763, 764, 761, 741, 722, 738, 760] payload_rows_per_dst=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] dup_rows_per_dst=[2912, 2980, 3044, 3196, 2834, 2852, 3196, 2932] combine_rows_per_src=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65032 dup_rows=24012 dup_rows_remote=21072 dedup_ratio=0.244263 router_tokens_per_expert=[794, 764, 788, 781, 783, 767, 729, 806, 690, 769, 723, 762, 730, 818, 746, 802, 753, 750, 759, 782, 796, 773, 824, 787, 741, 771, 800, 726, 700, 803, 776, 783, 812, 747, 707, 762, 800, 815, 756, 726, 767, 802, 735, 755, 770, 739, 793, 780, 757, 743, 773, 775, 776, 811, 774, 749, 784, 742, 784, 734, 767, 789, 781, 771] payload_rows_per_dst=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] dup_rows_per_dst=[2992, 2920, 2980, 2940, 3010, 3020, 3010, 3140] combine_rows_per_src=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343640 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059352 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775064 weight_movement_bytes=34603008 total_nvlink_bytes=566378072 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344376 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266076472 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531808568 weight_movement_bytes=34603008 total_nvlink_bytes=566411576 weight_to_token_ratio=0.065067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65082 dup_rows=23878 dup_rows_remote=20924 dedup_ratio=0.242900 router_tokens_per_expert=[793, 750, 811, 837, 825, 740, 756, 768, 710, 788, 760, 749, 755, 745, 790, 783, 771, 742, 778, 792, 748, 757, 796, 809, 803, 764, 750, 780, 773, 789, 750, 769, 757, 794, 784, 794, 729, 724, 801, 766, 771, 773, 743, 755, 741, 793, 791, 735, 717, 736, 764, 749, 779, 748, 784, 789, 781, 789, 777, 801, 721, 702, 752, 781] payload_rows_per_dst=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] dup_rows_per_dst=[3036, 2996, 3028, 3024, 2954, 2970, 2958, 2912] combine_rows_per_src=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343664 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042416 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741168 weight_movement_bytes=34603008 total_nvlink_bytes=568344176 weight_to_token_ratio=0.064831 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343008 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591200 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839392 weight_movement_bytes=34603008 total_nvlink_bytes=567442400 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65054 dup_rows=24074 dup_rows_remote=21060 dedup_ratio=0.244893 router_tokens_per_expert=[789, 780, 745, 765, 719, 762, 787, 736, 791, 772, 789, 801, 726, 781, 781, 805, 778, 728, 763, 762, 758, 726, 773, 786, 789, 799, 748, 749, 782, 742, 752, 813, 740, 788, 824, 769, 760, 751, 740, 791, 779, 779, 737, 739, 757, 765, 835, 794, 822, 780, 762, 788, 740, 744, 777, 726, 767, 779, 782, 786, 746, 714, 729, 785] payload_rows_per_dst=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] dup_rows_per_dst=[3014, 3066, 3168, 3224, 2834, 3070, 2716, 2982] combine_rows_per_src=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64920 dup_rows=24104 dup_rows_remote=21140 dedup_ratio=0.245199 router_tokens_per_expert=[750, 763, 766, 808, 761, 791, 771, 753, 741, 790, 776, 755, 760, 753, 787, 755, 746, 728, 769, 808, 789, 751, 763, 771, 774, 771, 737, 763, 754, 762, 780, 739, 749, 773, 754, 809, 750, 749, 745, 772, 762, 767, 817, 801, 763, 769, 800, 768, 780, 780, 797, 755, 788, 744, 834, 735, 749, 721, 768, 786, 778, 726, 756, 822] payload_rows_per_dst=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] dup_rows_per_dst=[3056, 2972, 3016, 2796, 2856, 3392, 3052, 2964] combine_rows_per_src=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344024 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266919896 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533495768 weight_movement_bytes=69206016 total_nvlink_bytes=602701784 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64854 dup_rows=24002 dup_rows_remote=20992 dedup_ratio=0.244161 router_tokens_per_expert=[777, 740, 789, 743, 756, 748, 754, 742, 758, 761, 763, 755, 736, 749, 731, 768, 813, 778, 746, 772, 760, 764, 755, 740, 809, 825, 777, 785, 794, 714, 783, 778, 752, 792, 779, 719, 750, 799, 757, 779, 802, 815, 788, 721, 781, 829, 732, 761, 744, 757, 767, 773, 760, 731, 793, 769, 781, 769, 798, 767, 782, 781, 777, 784] payload_rows_per_dst=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] dup_rows_per_dst=[2854, 2898, 2990, 3146, 3034, 3010, 2990, 3080] combine_rows_per_src=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65086 dup_rows=23894 dup_rows_remote=20854 dedup_ratio=0.243062 router_tokens_per_expert=[779, 676, 746, 768, 810, 775, 786, 778, 767, 753, 735, 743, 750, 793, 834, 783, 747, 771, 774, 767, 800, 773, 777, 773, 774, 796, 764, 770, 745, 762, 780, 725, 776, 767, 792, 795, 754, 762, 771, 719, 798, 803, 770, 768, 754, 737, 790, 725, 745, 816, 812, 703, 780, 825, 782, 796, 787, 736, 704, 772, 750, 754, 763, 772] payload_rows_per_dst=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] dup_rows_per_dst=[2942, 2980, 3040, 2944, 3044, 3036, 3092, 2816] combine_rows_per_src=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344240 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256560 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532168880 weight_movement_bytes=34603008 total_nvlink_bytes=566771888 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344456 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805640 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266824 weight_movement_bytes=69206016 total_nvlink_bytes=602472840 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65066 dup_rows=23984 dup_rows_remote=21082 dedup_ratio=0.243978 router_tokens_per_expert=[778, 767, 759, 788, 752, 783, 788, 763, 723, 769, 763, 792, 762, 785, 794, 786, 713, 734, 769, 728, 760, 809, 745, 776, 748, 779, 833, 736, 768, 709, 789, 754, 760, 745, 792, 780, 749, 809, 800, 732, 741, 761, 794, 721, 713, 808, 817, 804, 773, 756, 769, 785, 757, 747, 744, 747, 844, 746, 749, 773, 805, 771, 782, 776] payload_rows_per_dst=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] dup_rows_per_dst=[2984, 3068, 2872, 2992, 3070, 3002, 2902, 3094] combine_rows_per_src=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65262 dup_rows=23866 dup_rows_remote=20906 dedup_ratio=0.242778 router_tokens_per_expert=[794, 798, 775, 772, 744, 748, 793, 729, 779, 784, 725, 746, 754, 750, 790, 740, 771, 757, 784, 794, 765, 748, 759, 737, 712, 773, 726, 747, 726, 779, 773, 742, 809, 816, 830, 750, 779, 794, 809, 773, 797, 723, 788, 780, 786, 762, 785, 759, 784, 736, 802, 772, 742, 834, 723, 737, 795, 782, 784, 742, 779, 743, 793, 750] payload_rows_per_dst=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] dup_rows_per_dst=[2978, 2960, 2970, 2852, 3152, 2950, 2990, 3014] combine_rows_per_src=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64970 dup_rows=23924 dup_rows_remote=20830 dedup_ratio=0.243368 router_tokens_per_expert=[746, 765, 713, 813, 766, 736, 796, 768, 783, 770, 740, 742, 747, 785, 763, 734, 766, 796, 737, 760, 821, 749, 785, 778, 774, 738, 760, 754, 794, 793, 863, 776, 775, 737, 758, 780, 787, 732, 764, 792, 784, 758, 751, 760, 799, 770, 787, 780, 750, 747, 768, 784, 758, 763, 760, 768, 749, 767, 793, 787, 702, 742, 812, 777] payload_rows_per_dst=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] dup_rows_per_dst=[2944, 2914, 3074, 3094, 2946, 2994, 2946, 3012] combine_rows_per_src=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343384 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265985368 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531627352 weight_movement_bytes=69206016 total_nvlink_bytes=600833368 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343760 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936016 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528272 weight_movement_bytes=34603008 total_nvlink_bytes=568131280 weight_to_token_ratio=0.064857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65144 dup_rows=23860 dup_rows_remote=20792 dedup_ratio=0.242716 router_tokens_per_expert=[768, 763, 756, 768, 774, 761, 786, 792, 716, 773, 739, 733, 783, 757, 784, 811, 807, 803, 776, 827, 781, 750, 771, 765, 756, 739, 737, 744, 763, 796, 773, 754, 771, 766, 784, 750, 772, 790, 782, 769, 773, 711, 766, 763, 785, 780, 786, 802, 764, 747, 763, 749, 783, 751, 748, 803, 745, 729, 786, 771, 761, 748, 775, 773] payload_rows_per_dst=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] dup_rows_per_dst=[2954, 2924, 3068, 2896, 3068, 3026, 3044, 2880] combine_rows_per_src=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344592 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266854928 ingress_bytes=370319360 link_bytes=370319360 token_bytes=533365264 weight_movement_bytes=103809024 total_nvlink_bytes=637174288 weight_to_token_ratio=0.194630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393481939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344672 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657824 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534970976 weight_movement_bytes=34603008 total_nvlink_bytes=569573984 weight_to_token_ratio=0.064682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343200 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266460320 ingress_bytes=369926144 link_bytes=369926144 token_bytes=532577440 weight_movement_bytes=103809024 total_nvlink_bytes=636386464 weight_to_token_ratio=0.194918 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392002298e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65120 dup_rows=23826 dup_rows_remote=20742 dedup_ratio=0.242371 router_tokens_per_expert=[757, 748, 785, 827, 719, 766, 794, 800, 738, 752, 749, 778, 783, 745, 755, 764, 754, 780, 798, 755, 767, 803, 738, 791, 790, 696, 758, 744, 752, 743, 757, 767, 756, 779, 768, 749, 797, 781, 766, 758, 777, 720, 760, 737, 778, 792, 825, 757, 765, 777, 755, 769, 824, 773, 759, 767, 744, 808, 808, 782, 758, 767, 769, 774] payload_rows_per_dst=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] dup_rows_per_dst=[2992, 2858, 3030, 2878, 2936, 2966, 3082, 3084] combine_rows_per_src=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=65106 dup_rows=23888 dup_rows_remote=20956 dedup_ratio=0.243001 router_tokens_per_expert=[727, 841, 753, 737, 770, 753, 756, 783, 808, 748, 756, 746, 770, 771, 746, 775, 792, 708, 748, 737, 713, 765, 794, 786, 774, 773, 832, 753, 754, 776, 810, 737, 733, 785, 850, 777, 810, 742, 816, 772, 781, 763, 669, 836, 721, 753, 789, 777, 724, 767, 794, 733, 738, 750, 814, 772, 749, 796, 779, 801, 753, 733, 792, 791] payload_rows_per_dst=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] dup_rows_per_dst=[2932, 3010, 3130, 3092, 3058, 2860, 2930, 2876] combine_rows_per_src=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=343744 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267173568 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003392 weight_movement_bytes=34603008 total_nvlink_bytes=568606400 weight_to_token_ratio=0.064799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65228 dup_rows=23926 dup_rows_remote=20896 dedup_ratio=0.243388 router_tokens_per_expert=[743, 738, 787, 778, 814, 804, 768, 791, 777, 800, 814, 781, 753, 737, 731, 753, 771, 756, 776, 713, 770, 758, 767, 763, 774, 784, 782, 776, 739, 753, 763, 743, 730, 803, 768, 771, 795, 726, 810, 740, 798, 745, 777, 757, 777, 733, 755, 748, 807, 784, 812, 755, 751, 835, 788, 759, 795, 725, 769, 788, 767, 730, 743, 754] payload_rows_per_dst=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] dup_rows_per_dst=[3110, 3022, 2902, 2900, 2920, 3030, 3092, 2950] combine_rows_per_src=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343448 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267074968 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533806488 weight_movement_bytes=34603008 total_nvlink_bytes=568409496 weight_to_token_ratio=0.064823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64884 dup_rows=24182 dup_rows_remote=21210 dedup_ratio=0.245992 router_tokens_per_expert=[759, 771, 761, 771, 767, 778, 788, 759, 772, 754, 775, 802, 735, 749, 775, 802, 719, 743, 801, 786, 759, 734, 764, 799, 757, 789, 728, 764, 756, 793, 787, 741, 796, 769, 775, 746, 760, 736, 717, 768, 798, 710, 788, 770, 718, 788, 774, 761, 824, 810, 753, 768, 794, 770, 781, 735, 756, 765, 780, 775, 757, 790, 790, 792] payload_rows_per_dst=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] dup_rows_per_dst=[3106, 3008, 2972, 3008, 2900, 2988, 3082, 3118] combine_rows_per_src=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344248 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267018424 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533692600 weight_movement_bytes=103809024 total_nvlink_bytes=637501624 weight_to_token_ratio=0.194511 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65056 dup_rows=23920 dup_rows_remote=20958 dedup_ratio=0.243327 router_tokens_per_expert=[796, 802, 771, 800, 764, 811, 739, 766, 765, 779, 764, 782, 775, 747, 765, 754, 793, 825, 737, 734, 740, 778, 764, 812, 736, 754, 744, 768, 768, 782, 818, 746, 800, 816, 764, 740, 733, 776, 756, 795, 737, 740, 795, 780, 754, 755, 798, 793, 751, 764, 768, 805, 754, 780, 748, 775, 743, 751, 775, 710, 745, 784, 735, 753] payload_rows_per_dst=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] dup_rows_per_dst=[3028, 3014, 2966, 2930, 3034, 3004, 2962, 2982] combine_rows_per_src=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65180 dup_rows=23900 dup_rows_remote=20860 dedup_ratio=0.243123 router_tokens_per_expert=[763, 758, 735, 778, 734, 762, 753, 742, 766, 743, 753, 745, 786, 749, 806, 784, 796, 774, 757, 771, 775, 810, 733, 731, 745, 785, 779, 761, 739, 819, 810, 797, 771, 705, 786, 758, 780, 817, 763, 750, 753, 771, 749, 771, 765, 747, 734, 769, 820, 780, 786, 785, 798, 743, 781, 770, 769, 776, 767, 786, 757, 764, 762, 780] payload_rows_per_dst=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] dup_rows_per_dst=[2882, 3040, 2866, 3080, 3012, 2868, 3156, 2996] combine_rows_per_src=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344496 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518384 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692272 weight_movement_bytes=34603008 total_nvlink_bytes=569295280 weight_to_token_ratio=0.064716 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65114 dup_rows=23856 dup_rows_remote=20952 dedup_ratio=0.242676 router_tokens_per_expert=[761, 765, 753, 759, 732, 749, 803, 745, 755, 755, 760, 733, 784, 831, 777, 758, 770, 753, 759, 816, 790, 766, 709, 757, 773, 785, 780, 783, 774, 735, 736, 753, 724, 782, 777, 771, 783, 764, 793, 783, 735, 764, 765, 769, 759, 789, 764, 786, 755, 764, 794, 794, 803, 752, 781, 795, 790, 761, 792, 756, 765, 787, 781, 715] payload_rows_per_dst=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] dup_rows_per_dst=[2880, 2970, 2904, 2904, 3082, 3060, 3068, 2988] combine_rows_per_src=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344376 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109240 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874104 weight_movement_bytes=69206016 total_nvlink_bytes=601080120 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64986 dup_rows=24098 dup_rows_remote=21054 dedup_ratio=0.245138 router_tokens_per_expert=[794, 790, 790, 782, 786, 753, 791, 758, 773, 698, 802, 797, 770, 764, 760, 726, 786, 783, 774, 788, 749, 739, 752, 803, 822, 752, 741, 812, 740, 745, 744, 748, 788, 777, 754, 756, 784, 808, 732, 733, 755, 733, 742, 740, 778, 746, 755, 810, 771, 718, 813, 735, 757, 820, 767, 772, 716, 807, 796, 781, 765, 727, 820, 784] payload_rows_per_dst=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] dup_rows_per_dst=[3128, 2836, 3066, 2980, 3044, 2964, 3024, 3056] combine_rows_per_src=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344056 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266813432 ingress_bytes=370278400 link_bytes=370278400 token_bytes=533282808 weight_movement_bytes=103809024 total_nvlink_bytes=637091832 weight_to_token_ratio=0.194660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393327809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298720 weight_movement_bytes=34603008 total_nvlink_bytes=568901728 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344264 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267051208 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533758152 weight_movement_bytes=34603008 total_nvlink_bytes=568361160 weight_to_token_ratio=0.064829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=64816 dup_rows=23868 dup_rows_remote=20564 dedup_ratio=0.242798 router_tokens_per_expert=[761, 797, 728, 771, 730, 736, 784, 775, 704, 766, 766, 768, 783, 756, 767, 772, 740, 804, 743, 744, 774, 809, 774, 798, 718, 788, 747, 752, 784, 768, 775, 735, 780, 748, 769, 733, 796, 767, 753, 792, 723, 780, 765, 743, 771, 751, 816, 766, 770, 783, 759, 780, 777, 810, 771, 801, 770, 738, 814, 741, 760, 805, 816, 787] payload_rows_per_dst=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] dup_rows_per_dst=[2996, 2568, 2974, 2894, 2976, 3000, 3156, 3304] combine_rows_per_src=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64820 dup_rows=24266 dup_rows_remote=21268 dedup_ratio=0.246847 router_tokens_per_expert=[791, 791, 749, 734, 757, 795, 746, 734, 786, 740, 746, 767, 769, 776, 783, 748, 748, 800, 737, 757, 729, 797, 802, 802, 772, 784, 793, 767, 773, 789, 796, 757, 760, 750, 776, 741, 754, 754, 797, 785, 805, 734, 750, 800, 731, 728, 784, 736, 791, 824, 793, 783, 792, 755, 761, 747, 739, 785, 822, 726, 763, 780, 756, 735] payload_rows_per_dst=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] dup_rows_per_dst=[2998, 3330, 3088, 3206, 2948, 2568, 3104, 3024] combine_rows_per_src=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344160 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526816 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709472 weight_movement_bytes=34603008 total_nvlink_bytes=567312480 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64940 dup_rows=24118 dup_rows_remote=21184 dedup_ratio=0.245341 router_tokens_per_expert=[767, 749, 768, 775, 746, 765, 788, 736, 794, 801, 777, 803, 791, 776, 743, 767, 714, 724, 763, 845, 765, 744, 766, 775, 791, 839, 785, 787, 783, 762, 723, 738, 780, 733, 748, 783, 742, 814, 741, 780, 732, 747, 757, 767, 780, 754, 782, 771, 750, 784, 785, 770, 792, 798, 764, 800, 782, 761, 750, 767, 766, 737, 736, 749] payload_rows_per_dst=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] dup_rows_per_dst=[2990, 3094, 3016, 3104, 2932, 2934, 3200, 2848] combine_rows_per_src=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344352 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847072 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349792 weight_movement_bytes=34603008 total_nvlink_bytes=565952800 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=341520 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265827856 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531314192 weight_movement_bytes=34603008 total_nvlink_bytes=565917200 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65056 dup_rows=24032 dup_rows_remote=20966 dedup_ratio=0.244466 router_tokens_per_expert=[708, 757, 780, 753, 813, 755, 736, 802, 793, 756, 791, 815, 754, 767, 811, 829, 803, 741, 804, 791, 722, 761, 731, 788, 730, 788, 828, 788, 812, 775, 772, 785, 719, 755, 779, 785, 791, 763, 724, 779, 770, 747, 763, 813, 789, 752, 699, 760, 750, 764, 742, 799, 756, 792, 710, 745, 784, 751, 739, 792, 756, 726, 803, 716] payload_rows_per_dst=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] dup_rows_per_dst=[2982, 3112, 3066, 3196, 2932, 2940, 2866, 2938] combine_rows_per_src=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=20886 dedup_ratio=0.242676 router_tokens_per_expert=[778, 749, 799, 789, 769, 758, 755, 760, 759, 819, 754, 793, 784, 700, 751, 799, 776, 799, 766, 760, 792, 787, 763, 778, 760, 783, 742, 785, 776, 805, 710, 713, 719, 784, 817, 806, 776, 745, 738, 775, 787, 790, 756, 774, 818, 776, 757, 750, 804, 773, 758, 739, 749, 753, 774, 764, 766, 808, 737, 747, 739, 758, 775, 729] payload_rows_per_dst=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] dup_rows_per_dst=[3024, 3024, 3112, 2792, 2968, 2952, 2970, 3014] combine_rows_per_src=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344496 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338736 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332976 weight_movement_bytes=34603008 total_nvlink_bytes=566935984 weight_to_token_ratio=0.065003 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64956 dup_rows=24016 dup_rows_remote=21020 dedup_ratio=0.244303 router_tokens_per_expert=[747, 840, 782, 749, 758, 761, 765, 748, 791, 695, 773, 777, 781, 799, 796, 752, 761, 779, 768, 760, 748, 786, 762, 738, 788, 810, 772, 741, 779, 749, 803, 791, 748, 753, 736, 779, 760, 784, 776, 773, 777, 774, 772, 778, 786, 763, 778, 722, 805, 750, 756, 779, 799, 754, 774, 730, 726, 713, 749, 798, 782, 792, 790, 747] payload_rows_per_dst=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] dup_rows_per_dst=[2984, 2996, 3016, 3090, 2914, 2936, 3060, 3020] combine_rows_per_src=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344088 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266813464 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533282840 weight_movement_bytes=69206016 total_nvlink_bytes=602488856 weight_to_token_ratio=0.129774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74558 payload_rows_remote=65132 dup_rows=23746 dup_rows_remote=20618 dedup_ratio=0.241557 router_tokens_per_expert=[714, 763, 816, 749, 768, 781, 788, 806, 832, 753, 772, 754, 752, 782, 755, 777, 728, 763, 825, 773, 721, 767, 766, 710, 796, 769, 804, 725, 785, 768, 755, 770, 797, 780, 785, 785, 775, 832, 781, 742, 812, 786, 793, 752, 790, 755, 740, 819, 752, 770, 734, 728, 798, 766, 743, 783, 800, 763, 715, 743, 704, 731, 763, 748] payload_rows_per_dst=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] dup_rows_per_dst=[2974, 2970, 2920, 2962, 3128, 3032, 2894, 2866] combine_rows_per_src=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65084 dup_rows=23976 dup_rows_remote=20926 dedup_ratio=0.243896 router_tokens_per_expert=[805, 790, 781, 726, 693, 726, 781, 720, 759, 760, 807, 804, 755, 775, 780, 808, 740, 784, 738, 799, 818, 805, 737, 741, 782, 723, 709, 798, 762, 793, 810, 770, 788, 812, 778, 727, 794, 773, 770, 746, 771, 775, 804, 766, 797, 783, 751, 756, 735, 722, 771, 743, 794, 769, 774, 801, 820, 741, 742, 728, 769, 750, 755, 768] payload_rows_per_dst=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] dup_rows_per_dst=[2876, 3036, 3066, 3050, 3046, 3052, 2968, 2882] combine_rows_per_src=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344304 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267362544 ingress_bytes=336224256 link_bytes=336224256 token_bytes=534380784 weight_movement_bytes=69206016 total_nvlink_bytes=603586800 weight_to_token_ratio=0.129507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265184807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65048 dup_rows=23890 dup_rows_remote=20912 dedup_ratio=0.243022 router_tokens_per_expert=[823, 777, 750, 768, 783, 795, 748, 743, 744, 745, 823, 773, 807, 748, 748, 812, 790, 802, 786, 782, 766, 778, 799, 785, 740, 772, 746, 809, 732, 768, 731, 750, 781, 764, 755, 817, 783, 799, 732, 775, 732, 794, 804, 779, 760, 737, 762, 698, 717, 745, 808, 754, 759, 742, 761, 764, 727, 770, 778, 765, 801, 771, 780, 715] payload_rows_per_dst=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] dup_rows_per_dst=[2762, 2994, 3214, 3018, 3116, 2782, 3026, 2978] combine_rows_per_src=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65030 dup_rows=23992 dup_rows_remote=20962 dedup_ratio=0.244059 router_tokens_per_expert=[817, 804, 737, 739, 775, 689, 819, 775, 820, 749, 801, 790, 801, 758, 793, 738, 807, 750, 759, 736, 759, 821, 721, 737, 776, 782, 752, 789, 775, 752, 761, 813, 775, 760, 782, 748, 772, 754, 760, 734, 793, 716, 759, 761, 712, 797, 760, 754, 807, 745, 727, 819, 759, 759, 724, 717, 788, 794, 783, 778, 819, 737, 777, 787] payload_rows_per_dst=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] dup_rows_per_dst=[3030, 3042, 2920, 3206, 2988, 3154, 2558, 3094] combine_rows_per_src=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343000 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267123672 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533904344 weight_movement_bytes=34603008 total_nvlink_bytes=568507352 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344040 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928104 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512168 weight_movement_bytes=34603008 total_nvlink_bytes=568115176 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65050 dup_rows=23924 dup_rows_remote=20954 dedup_ratio=0.243368 router_tokens_per_expert=[740, 765, 742, 746, 786, 807, 752, 799, 760, 788, 773, 720, 755, 761, 769, 752, 848, 709, 738, 753, 744, 776, 744, 773, 755, 735, 761, 813, 792, 725, 806, 740, 720, 800, 764, 740, 803, 769, 776, 779, 767, 790, 741, 762, 769, 784, 744, 793, 756, 792, 735, 806, 795, 746, 798, 751, 784, 799, 759, 794, 768, 769, 774, 798] payload_rows_per_dst=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] dup_rows_per_dst=[2954, 2948, 3000, 3004, 2964, 2970, 3078, 3006] combine_rows_per_src=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65002 dup_rows=23972 dup_rows_remote=20978 dedup_ratio=0.243856 router_tokens_per_expert=[781, 768, 740, 816, 757, 790, 802, 754, 720, 759, 728, 767, 802, 778, 749, 745, 771, 796, 780, 766, 766, 732, 775, 776, 737, 775, 808, 775, 762, 749, 776, 734, 748, 731, 754, 742, 767, 731, 735, 780, 775, 804, 768, 802, 773, 761, 782, 810, 765, 758, 756, 799, 804, 745, 717, 794, 791, 754, 814, 775, 764, 734, 802, 783] payload_rows_per_dst=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] dup_rows_per_dst=[3006, 2894, 2994, 2972, 2912, 3124, 3010, 3060] combine_rows_per_src=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343840 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266780448 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217056 weight_movement_bytes=34603008 total_nvlink_bytes=567820064 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65020 dup_rows=24074 dup_rows_remote=20960 dedup_ratio=0.244893 router_tokens_per_expert=[773, 746, 797, 759, 757, 764, 767, 744, 756, 785, 724, 780, 795, 722, 798, 791, 755, 787, 786, 732, 773, 804, 787, 771, 805, 748, 810, 728, 745, 766, 783, 729, 737, 745, 731, 739, 788, 782, 777, 732, 768, 765, 800, 789, 778, 805, 776, 779, 757, 780, 765, 773, 783, 766, 804, 734, 770, 729, 750, 782, 751, 783, 750, 817] payload_rows_per_dst=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] dup_rows_per_dst=[2968, 3040, 3084, 2934, 2896, 3108, 3114, 2930] combine_rows_per_src=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343968 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266706848 ingress_bytes=404774912 link_bytes=404774912 token_bytes=533069728 weight_movement_bytes=138412032 total_nvlink_bytes=671481760 weight_to_token_ratio=0.259651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523135407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344016 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788816 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533233616 weight_movement_bytes=138412032 total_nvlink_bytes=671645648 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64972 dup_rows=24078 dup_rows_remote=21170 dedup_ratio=0.244934 router_tokens_per_expert=[780, 803, 801, 774, 808, 771, 808, 717, 750, 774, 719, 774, 756, 786, 773, 749, 824, 788, 782, 790, 774, 748, 746, 801, 704, 736, 779, 774, 776, 717, 751, 762, 793, 765, 751, 778, 764, 743, 747, 828, 767, 728, 825, 732, 751, 841, 771, 795, 734, 747, 770, 757, 746, 761, 750, 797, 801, 760, 739, 730, 766, 759, 816, 745] payload_rows_per_dst=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] dup_rows_per_dst=[3154, 2908, 3112, 2804, 3086, 3098, 2910, 3006] combine_rows_per_src=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343920 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592112 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840304 weight_movement_bytes=34603008 total_nvlink_bytes=567443312 weight_to_token_ratio=0.064941 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64712 dup_rows=24194 dup_rows_remote=21104 dedup_ratio=0.246114 router_tokens_per_expert=[811, 790, 747, 803, 804, 766, 762, 749, 766, 740, 802, 758, 750, 776, 749, 774, 777, 744, 773, 796, 768, 722, 788, 733, 758, 820, 748, 750, 816, 766, 740, 715, 742, 786, 807, 750, 772, 786, 801, 800, 746, 803, 786, 741, 743, 787, 758, 738, 781, 728, 818, 723, 776, 745, 712, 784, 753, 746, 804, 763, 790, 769, 791, 762] payload_rows_per_dst=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] dup_rows_per_dst=[3052, 3066, 2942, 3010, 3090, 2990, 2992, 3052] combine_rows_per_src=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343920 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266665840 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987760 weight_movement_bytes=34603008 total_nvlink_bytes=567590768 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74052 payload_rows_remote=64554 dup_rows=24252 dup_rows_remote=21100 dedup_ratio=0.246704 router_tokens_per_expert=[749, 816, 765, 798, 789, 750, 780, 771, 768, 769, 749, 691, 738, 789, 795, 805, 756, 731, 713, 750, 741, 725, 800, 747, 818, 744, 851, 784, 794, 772, 795, 767, 772, 715, 777, 745, 797, 768, 759, 781, 758, 812, 795, 742, 785, 751, 765, 775, 787, 792, 771, 780, 752, 772, 761, 764, 772, 814, 746, 775, 785, 752, 718, 704] payload_rows_per_dst=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] dup_rows_per_dst=[3146, 2958, 2934, 3152, 2916, 3070, 3108, 2968] combine_rows_per_src=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344568 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469880 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595192 weight_movement_bytes=34603008 total_nvlink_bytes=567198200 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65030 dup_rows=23890 dup_rows_remote=20844 dedup_ratio=0.243022 router_tokens_per_expert=[729, 732, 731, 715, 761, 766, 744, 740, 745, 755, 754, 789, 761, 772, 781, 742, 797, 749, 765, 727, 717, 772, 757, 735, 798, 748, 815, 783, 807, 767, 799, 796, 720, 777, 741, 774, 762, 816, 831, 765, 732, 796, 764, 775, 741, 824, 810, 737, 812, 797, 797, 752, 783, 747, 794, 773, 789, 796, 763, 780, 771, 756, 750, 778] payload_rows_per_dst=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] dup_rows_per_dst=[2848, 2964, 2862, 3122, 3038, 2876, 3134, 3046] combine_rows_per_src=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343264 combine_payload_bytes=265060352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265403616 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530463968 weight_movement_bytes=138412032 total_nvlink_bytes=668876000 weight_to_token_ratio=0.260926 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64770 dup_rows=24096 dup_rows_remote=20936 dedup_ratio=0.245117 router_tokens_per_expert=[786, 782, 768, 802, 758, 762, 736, 811, 774, 778, 797, 773, 802, 723, 747, 711, 804, 761, 726, 786, 757, 765, 778, 782, 769, 745, 778, 764, 772, 798, 756, 753, 794, 739, 788, 720, 719, 796, 741, 747, 794, 781, 779, 757, 803, 754, 728, 770, 747, 784, 748, 777, 810, 793, 733, 796, 749, 811, 796, 769, 740, 745, 767, 773] payload_rows_per_dst=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] dup_rows_per_dst=[3160, 3026, 2920, 2910, 3060, 3044, 2966, 3010] combine_rows_per_src=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65082 dup_rows=23788 dup_rows_remote=20722 dedup_ratio=0.241984 router_tokens_per_expert=[820, 763, 732, 790, 745, 751, 822, 741, 762, 755, 775, 799, 749, 708, 782, 765, 753, 785, 744, 798, 791, 740, 742, 764, 751, 775, 753, 804, 765, 778, 804, 735, 760, 831, 772, 785, 727, 741, 769, 782, 776, 798, 834, 777, 707, 784, 781, 793, 744, 800, 763, 735, 805, 769, 767, 778, 763, 742, 781, 762, 787, 714, 756, 728] payload_rows_per_dst=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] dup_rows_per_dst=[3044, 2954, 2924, 3018, 2934, 3066, 3022, 2826] combine_rows_per_src=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=342616 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264755800 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529168984 weight_movement_bytes=34603008 total_nvlink_bytes=563771992 weight_to_token_ratio=0.065391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65082 dup_rows=23916 dup_rows_remote=20910 dedup_ratio=0.243286 router_tokens_per_expert=[752, 754, 807, 812, 743, 758, 762, 746, 753, 760, 739, 766, 754, 764, 784, 780, 785, 758, 784, 764, 732, 787, 774, 772, 792, 772, 743, 733, 736, 769, 792, 774, 759, 745, 776, 771, 781, 786, 757, 778, 785, 755, 739, 752, 821, 786, 718, 770, 798, 730, 818, 758, 820, 751, 742, 790, 764, 787, 768, 778, 814, 768, 727, 759] payload_rows_per_dst=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] dup_rows_per_dst=[2884, 2920, 3006, 2978, 3020, 3024, 3018, 3066] combine_rows_per_src=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343496 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706376 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069256 weight_movement_bytes=34603008 total_nvlink_bytes=567672264 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65186 dup_rows=23858 dup_rows_remote=20914 dedup_ratio=0.242696 router_tokens_per_expert=[673, 768, 824, 758, 710, 803, 816, 757, 777, 770, 784, 774, 780, 797, 723, 739, 799, 818, 774, 743, 762, 758, 763, 768, 774, 764, 788, 731, 793, 781, 726, 763, 809, 801, 764, 763, 775, 733, 690, 749, 764, 786, 742, 743, 764, 760, 768, 821, 767, 774, 770, 754, 725, 798, 730, 784, 744, 762, 782, 817, 792, 794, 796, 773] payload_rows_per_dst=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] dup_rows_per_dst=[2920, 3014, 2970, 3014, 2844, 3002, 2944, 3150] combine_rows_per_src=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342824 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640744 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938664 weight_movement_bytes=34603008 total_nvlink_bytes=565541672 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64900 dup_rows=23938 dup_rows_remote=20884 dedup_ratio=0.243510 router_tokens_per_expert=[771, 788, 802, 780, 772, 784, 762, 764, 784, 769, 771, 803, 809, 772, 787, 765, 746, 754, 768, 794, 754, 758, 754, 690, 743, 787, 757, 752, 713, 753, 795, 781, 769, 782, 768, 736, 817, 754, 756, 780, 771, 757, 780, 711, 755, 807, 744, 747, 774, 755, 737, 780, 756, 795, 826, 738, 808, 754, 797, 781, 760, 727, 769, 779] payload_rows_per_dst=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] dup_rows_per_dst=[3012, 3054, 2882, 2976, 2936, 3022, 3058, 2998] combine_rows_per_src=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343216 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266919088 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494960 weight_movement_bytes=69206016 total_nvlink_bytes=602700976 weight_to_token_ratio=0.129722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343968 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919840 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495712 weight_movement_bytes=34603008 total_nvlink_bytes=568098720 weight_to_token_ratio=0.064861 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65050 dup_rows=23986 dup_rows_remote=21018 dedup_ratio=0.243998 router_tokens_per_expert=[770, 747, 756, 805, 766, 745, 793, 777, 759, 798, 758, 766, 770, 792, 807, 777, 825, 782, 783, 765, 726, 776, 757, 780, 779, 776, 727, 725, 781, 752, 777, 736, 760, 771, 807, 801, 777, 753, 748, 701, 798, 744, 798, 721, 758, 806, 725, 757, 732, 796, 756, 784, 757, 798, 791, 787, 772, 739, 769, 750, 744, 742, 763, 814] payload_rows_per_dst=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] dup_rows_per_dst=[3052, 3092, 3020, 2866, 2968, 2952, 3074, 2962] combine_rows_per_src=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344400 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346256 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348112 weight_movement_bytes=69206016 total_nvlink_bytes=603554128 weight_to_token_ratio=0.129515 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65174 dup_rows=23724 dup_rows_remote=20658 dedup_ratio=0.241333 router_tokens_per_expert=[741, 806, 708, 764, 796, 776, 750, 761, 766, 736, 766, 771, 794, 783, 755, 794, 771, 775, 796, 770, 856, 802, 789, 736, 760, 784, 733, 765, 799, 802, 801, 792, 765, 745, 777, 774, 779, 756, 764, 794, 771, 746, 736, 788, 779, 736, 793, 754, 793, 751, 733, 763, 754, 763, 797, 775, 763, 711, 773, 703, 716, 758, 786, 758] payload_rows_per_dst=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] dup_rows_per_dst=[2934, 3006, 3078, 3066, 2980, 2944, 2912, 2804] combine_rows_per_src=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343136 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173536 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532003936 weight_movement_bytes=34603008 total_nvlink_bytes=566606944 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64764 dup_rows=24092 dup_rows_remote=20888 dedup_ratio=0.245076 router_tokens_per_expert=[763, 789, 779, 732, 783, 747, 769, 777, 789, 782, 733, 797, 724, 773, 731, 783, 763, 787, 752, 737, 782, 750, 758, 778, 786, 751, 751, 748, 757, 809, 719, 781, 773, 780, 785, 785, 706, 752, 786, 762, 777, 785, 793, 749, 760, 770, 796, 765, 786, 697, 791, 749, 863, 741, 760, 764, 768, 781, 765, 750, 826, 807, 742, 778] payload_rows_per_dst=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] dup_rows_per_dst=[2848, 2938, 2862, 3222, 2926, 3058, 3034, 3204] combine_rows_per_src=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64916 dup_rows=24108 dup_rows_remote=21178 dedup_ratio=0.245239 router_tokens_per_expert=[724, 794, 769, 720, 744, 748, 794, 780, 791, 738, 756, 741, 754, 761, 804, 755, 714, 777, 759, 781, 747, 730, 773, 790, 832, 763, 724, 729, 781, 785, 750, 793, 773, 825, 716, 797, 760, 752, 735, 760, 781, 783, 768, 792, 788, 769, 762, 766, 785, 728, 775, 781, 749, 782, 769, 818, 764, 777, 771, 821, 725, 822, 758, 799] payload_rows_per_dst=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] dup_rows_per_dst=[2930, 3176, 3016, 3014, 3068, 2718, 3044, 3142] combine_rows_per_src=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344272 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789072 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233872 weight_movement_bytes=34603008 total_nvlink_bytes=567836880 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343328 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267296032 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534248736 weight_movement_bytes=69206016 total_nvlink_bytes=603454752 weight_to_token_ratio=0.129539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65028 dup_rows=24004 dup_rows_remote=21024 dedup_ratio=0.244181 router_tokens_per_expert=[761, 781, 764, 784, 772, 740, 804, 806, 741, 754, 776, 808, 744, 768, 746, 772, 735, 786, 864, 797, 768, 761, 736, 736, 788, 747, 771, 786, 780, 763, 746, 774, 751, 766, 762, 767, 784, 809, 739, 799, 777, 770, 783, 743, 788, 737, 763, 765, 745, 746, 725, 690, 791, 761, 782, 808, 767, 761, 739, 762, 787, 808, 776, 742] payload_rows_per_dst=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] dup_rows_per_dst=[3096, 3044, 3056, 2970, 2948, 2980, 2892, 3018] combine_rows_per_src=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64882 dup_rows=24100 dup_rows_remote=21230 dedup_ratio=0.245158 router_tokens_per_expert=[736, 766, 749, 786, 771, 733, 759, 819, 766, 773, 775, 749, 745, 795, 778, 726, 778, 781, 705, 767, 750, 767, 803, 745, 763, 723, 796, 696, 745, 808, 762, 753, 808, 771, 769, 777, 750, 747, 778, 774, 791, 778, 780, 794, 770, 756, 784, 700, 740, 795, 753, 779, 813, 814, 767, 779, 772, 773, 732, 808, 778, 812, 780, 762] payload_rows_per_dst=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] dup_rows_per_dst=[2872, 3078, 2870, 2866, 2984, 3124, 3116, 3190] combine_rows_per_src=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342608 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265615952 ingress_bytes=334479360 link_bytes=334479360 token_bytes=530889296 weight_movement_bytes=69206016 total_nvlink_bytes=600095312 weight_to_token_ratio=0.130359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258618904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[741, 740, 808, 774, 759, 761, 787, 786, 757, 760, 792, 770, 727, 727, 773, 748, 788, 767, 791, 739, 765, 772, 777, 735, 783, 776, 776, 764, 774, 765, 803, 756, 760, 789, 798, 796, 769, 749, 738, 744, 776, 759, 741, 817, 814, 725, 803, 760, 796, 745, 781, 787, 751, 770, 715, 798, 782, 810, 777, 739, 772, 756, 767, 727] payload_rows_per_dst=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] dup_rows_per_dst=[2972, 2858, 3012, 3090, 2986, 3078, 3024, 3064] combine_rows_per_src=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=344376 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266240312 ingress_bytes=404307968 link_bytes=404307968 token_bytes=532136248 weight_movement_bytes=138412032 total_nvlink_bytes=670548280 weight_to_token_ratio=0.260106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65188 dup_rows=23826 dup_rows_remote=21086 dedup_ratio=0.242371 router_tokens_per_expert=[782, 757, 793, 762, 761, 808, 764, 787, 730, 768, 787, 780, 739, 740, 736, 735, 798, 760, 806, 736, 821, 779, 750, 824, 795, 717, 734, 790, 709, 812, 746, 802, 814, 796, 716, 797, 727, 751, 749, 748, 721, 811, 765, 777, 782, 759, 766, 769, 718, 753, 732, 766, 795, 812, 769, 786, 808, 692, 801, 778, 788, 748, 779, 771] payload_rows_per_dst=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] dup_rows_per_dst=[3066, 2740, 3076, 3042, 2908, 2970, 2964, 3060] combine_rows_per_src=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344448 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266101120 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857792 weight_movement_bytes=69206016 total_nvlink_bytes=601063808 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344208 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266698896 ingress_bytes=370163712 link_bytes=370163712 token_bytes=533053584 weight_movement_bytes=103809024 total_nvlink_bytes=636862608 weight_to_token_ratio=0.194744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392896248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344072 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412040 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480008 weight_movement_bytes=34603008 total_nvlink_bytes=567083016 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65234 dup_rows=23946 dup_rows_remote=21100 dedup_ratio=0.243591 router_tokens_per_expert=[785, 780, 795, 739, 748, 789, 744, 767, 747, 790, 812, 734, 771, 772, 779, 786, 800, 721, 792, 761, 783, 734, 754, 782, 801, 735, 754, 818, 782, 783, 755, 821, 772, 745, 724, 737, 783, 741, 748, 735, 757, 678, 792, 726, 786, 744, 756, 793, 776, 796, 785, 789, 831, 770, 724, 743, 756, 813, 834, 734, 746, 744, 805, 775] payload_rows_per_dst=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] dup_rows_per_dst=[2960, 2986, 3062, 3052, 2846, 2844, 3042, 3154] combine_rows_per_src=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65060 dup_rows=23858 dup_rows_remote=20890 dedup_ratio=0.242696 router_tokens_per_expert=[788, 778, 783, 785, 762, 758, 740, 792, 750, 754, 748, 751, 732, 755, 754, 735, 794, 750, 753, 750, 804, 729, 744, 814, 769, 758, 807, 757, 798, 750, 768, 770, 763, 786, 792, 759, 756, 785, 716, 777, 752, 745, 773, 741, 836, 809, 748, 784, 754, 786, 776, 759, 797, 751, 751, 706, 783, 775, 797, 842, 761, 751, 769, 792] payload_rows_per_dst=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] dup_rows_per_dst=[2994, 2886, 2960, 2968, 2960, 3066, 2864, 3160] combine_rows_per_src=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=345096 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267355144 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534365192 weight_movement_bytes=34603008 total_nvlink_bytes=568968200 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65020 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[779, 780, 780, 754, 733, 773, 739, 761, 797, 790, 779, 741, 780, 732, 802, 770, 722, 748, 766, 807, 774, 779, 759, 748, 742, 742, 795, 800, 767, 772, 776, 734, 822, 758, 749, 764, 775, 751, 799, 771, 792, 803, 744, 747, 749, 752, 737, 793, 805, 773, 750, 784, 789, 749, 774, 780, 747, 791, 794, 778, 762, 781, 730, 738] payload_rows_per_dst=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] dup_rows_per_dst=[2962, 2646, 3438, 2750, 3192, 2966, 3088, 2968] combine_rows_per_src=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64714 dup_rows=24276 dup_rows_remote=21050 dedup_ratio=0.246948 router_tokens_per_expert=[756, 771, 741, 759, 775, 760, 767, 813, 793, 783, 736, 742, 760, 745, 722, 737, 783, 764, 786, 791, 751, 741, 753, 797, 722, 823, 759, 778, 775, 723, 773, 751, 807, 756, 768, 778, 785, 788, 786, 753, 784, 734, 788, 769, 774, 762, 798, 798, 729, 795, 785, 740, 806, 721, 769, 791, 805, 756, 752, 787, 762, 783, 736, 777] payload_rows_per_dst=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] dup_rows_per_dst=[3226, 3080, 3134, 2874, 2928, 3034, 2906, 3094] combine_rows_per_src=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=345336 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267543800 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534742264 weight_movement_bytes=34603008 total_nvlink_bytes=569345272 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65054 dup_rows=23950 dup_rows_remote=21006 dedup_ratio=0.243632 router_tokens_per_expert=[802, 804, 777, 791, 773, 763, 742, 722, 764, 739, 781, 775, 765, 781, 745, 790, 832, 708, 793, 792, 736, 764, 761, 737, 806, 766, 787, 733, 759, 785, 792, 763, 805, 742, 752, 838, 767, 763, 828, 755, 765, 734, 764, 822, 736, 768, 763, 770, 777, 731, 754, 754, 716, 782, 758, 797, 749, 791, 746, 766, 779, 745, 782, 725] payload_rows_per_dst=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] dup_rows_per_dst=[3024, 3004, 3002, 2974, 3132, 2944, 2956, 2914] combine_rows_per_src=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=65046 dup_rows=24066 dup_rows_remote=21098 dedup_ratio=0.244812 router_tokens_per_expert=[769, 780, 780, 767, 769, 721, 713, 773, 719, 805, 787, 777, 784, 795, 769, 734, 783, 757, 769, 768, 758, 734, 748, 763, 720, 807, 765, 809, 700, 774, 813, 781, 764, 797, 711, 787, 771, 771, 810, 780, 814, 746, 789, 719, 740, 773, 741, 772, 778, 741, 807, 749, 717, 778, 819, 722, 740, 816, 817, 768, 741, 790, 818, 775] payload_rows_per_dst=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] dup_rows_per_dst=[2884, 3074, 2968, 3028, 3032, 3016, 2950, 3114] combine_rows_per_src=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=343800 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266829560 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533315320 weight_movement_bytes=34603008 total_nvlink_bytes=567918328 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65078 dup_rows=23900 dup_rows_remote=20936 dedup_ratio=0.243123 router_tokens_per_expert=[752, 767, 726, 826, 731, 780, 773, 750, 790, 737, 774, 754, 739, 747, 784, 767, 757, 796, 780, 733, 761, 790, 736, 738, 744, 754, 798, 794, 826, 772, 779, 766, 768, 750, 801, 736, 763, 784, 735, 761, 768, 761, 760, 763, 767, 771, 825, 743, 768, 782, 784, 769, 764, 787, 767, 724, 815, 775, 762, 768, 770, 810, 756, 774] payload_rows_per_dst=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] dup_rows_per_dst=[2940, 2850, 2982, 3032, 3016, 3024, 2964, 3092] combine_rows_per_src=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344248 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666168 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988088 weight_movement_bytes=69206016 total_nvlink_bytes=602194104 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343056 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265411600 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530480144 weight_movement_bytes=34603008 total_nvlink_bytes=565083152 weight_to_token_ratio=0.065230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64926 dup_rows=24032 dup_rows_remote=20976 dedup_ratio=0.244466 router_tokens_per_expert=[805, 729, 795, 813, 779, 813, 791, 765, 802, 807, 771, 725, 795, 776, 801, 724, 727, 801, 741, 718, 764, 776, 748, 806, 803, 819, 748, 740, 757, 750, 745, 772, 774, 761, 737, 752, 747, 739, 803, 741, 775, 765, 777, 771, 760, 790, 759, 753, 776, 764, 788, 760, 762, 746, 763, 781, 803, 709, 715, 738, 788, 779, 768, 802] payload_rows_per_dst=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] dup_rows_per_dst=[3118, 3056, 2964, 2986, 2856, 2976, 3062, 3014] combine_rows_per_src=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344576 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772992 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201408 weight_movement_bytes=34603008 total_nvlink_bytes=567804416 weight_to_token_ratio=0.064897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344240 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805424 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266608 weight_movement_bytes=34603008 total_nvlink_bytes=567869616 weight_to_token_ratio=0.064889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344056 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266903544 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533463032 weight_movement_bytes=69206016 total_nvlink_bytes=602669048 weight_to_token_ratio=0.129730 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65022 dup_rows=23954 dup_rows_remote=20930 dedup_ratio=0.243673 router_tokens_per_expert=[717, 773, 769, 783, 757, 790, 788, 737, 767, 748, 777, 750, 796, 718, 786, 768, 790, 753, 762, 736, 762, 751, 740, 789, 749, 755, 756, 764, 811, 739, 797, 757, 731, 769, 791, 762, 770, 778, 787, 788, 739, 752, 787, 729, 805, 756, 744, 805, 826, 770, 710, 769, 795, 794, 782, 787, 757, 777, 773, 765, 770, 789, 773, 787] payload_rows_per_dst=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] dup_rows_per_dst=[2884, 2978, 2924, 2998, 3024, 3002, 3082, 3062] combine_rows_per_src=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65110 dup_rows=23928 dup_rows_remote=20954 dedup_ratio=0.243408 router_tokens_per_expert=[799, 736, 720, 824, 751, 793, 758, 809, 772, 798, 769, 790, 787, 774, 781, 774, 779, 756, 775, 716, 773, 742, 753, 749, 726, 797, 776, 774, 752, 757, 792, 746, 767, 772, 765, 726, 752, 772, 811, 739, 740, 808, 797, 761, 788, 777, 805, 756, 771, 753, 757, 784, 782, 754, 773, 812, 746, 769, 734, 738, 752, 763, 751, 779] payload_rows_per_dst=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] dup_rows_per_dst=[3002, 3082, 2882, 2974, 3032, 3078, 2952, 2926] combine_rows_per_src=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343608 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280504 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217400 weight_movement_bytes=69206016 total_nvlink_bytes=601423416 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64960 dup_rows=23982 dup_rows_remote=20978 dedup_ratio=0.243958 router_tokens_per_expert=[793, 746, 768, 729, 707, 767, 777, 760, 767, 722, 738, 766, 747, 822, 798, 780, 793, 798, 823, 739, 778, 772, 786, 778, 727, 734, 781, 764, 787, 737, 707, 778, 749, 740, 786, 778, 777, 793, 773, 749, 705, 776, 813, 750, 776, 788, 770, 791, 805, 747, 833, 743, 771, 749, 770, 768, 770, 799, 778, 767, 776, 772, 758, 763] payload_rows_per_dst=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] dup_rows_per_dst=[2950, 3040, 3172, 2730, 2956, 3078, 3052, 3004] combine_rows_per_src=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64854 dup_rows=24110 dup_rows_remote=21088 dedup_ratio=0.245260 router_tokens_per_expert=[774, 741, 735, 786, 801, 771, 791, 764, 792, 752, 786, 790, 765, 763, 797, 753, 770, 849, 720, 737, 792, 791, 717, 761, 720, 757, 767, 806, 764, 775, 773, 780, 788, 790, 779, 765, 746, 725, 757, 768, 819, 791, 738, 728, 795, 759, 725, 760, 750, 740, 775, 775, 783, 781, 789, 726, 729, 763, 794, 806, 734, 766, 773, 795] payload_rows_per_dst=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] dup_rows_per_dst=[3022, 3424, 2982, 3006, 3048, 2936, 3050, 2642] combine_rows_per_src=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343808 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673920 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004032 weight_movement_bytes=34603008 total_nvlink_bytes=567607040 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344256 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267034816 ingress_bytes=370499584 link_bytes=370499584 token_bytes=533725376 weight_movement_bytes=103809024 total_nvlink_bytes=637534400 weight_to_token_ratio=0.194499 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394160107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64876 dup_rows=24166 dup_rows_remote=21048 dedup_ratio=0.245829 router_tokens_per_expert=[770, 779, 738, 830, 761, 754, 771, 732, 817, 767, 833, 755, 765, 757, 742, 712, 735, 757, 766, 778, 745, 780, 767, 709, 752, 795, 768, 766, 773, 770, 804, 743, 735, 784, 771, 780, 779, 801, 740, 795, 809, 812, 739, 777, 749, 767, 742, 795, 803, 761, 763, 763, 753, 750, 778, 740, 772, 774, 761, 770, 768, 799, 783, 748] payload_rows_per_dst=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] dup_rows_per_dst=[2958, 3068, 2904, 3066, 2976, 3118, 3006, 3070] combine_rows_per_src=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65222 dup_rows=23810 dup_rows_remote=20880 dedup_ratio=0.242208 router_tokens_per_expert=[767, 774, 780, 743, 771, 784, 797, 776, 692, 754, 742, 750, 754, 792, 758, 766, 778, 745, 711, 743, 785, 762, 768, 809, 739, 794, 808, 790, 761, 782, 718, 780, 796, 783, 770, 757, 745, 747, 765, 758, 739, 758, 792, 770, 755, 840, 752, 809, 761, 773, 751, 763, 800, 745, 779, 762, 821, 772, 757, 763, 771, 781, 783, 761] payload_rows_per_dst=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] dup_rows_per_dst=[3064, 2842, 2930, 3030, 2934, 3016, 2974, 3020] combine_rows_per_src=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65052 dup_rows=23918 dup_rows_remote=21044 dedup_ratio=0.243306 router_tokens_per_expert=[746, 743, 725, 754, 776, 784, 806, 720, 763, 778, 768, 801, 755, 765, 767, 786, 791, 724, 775, 753, 736, 798, 833, 748, 768, 761, 786, 756, 826, 755, 791, 745, 770, 769, 741, 748, 766, 726, 786, 776, 829, 765, 778, 773, 748, 766, 784, 794, 756, 784, 741, 838, 740, 736, 757, 752, 790, 798, 726, 815, 769, 831, 712, 705] payload_rows_per_dst=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] dup_rows_per_dst=[2892, 3020, 2990, 3052, 3000, 3084, 2874, 3006] combine_rows_per_src=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343752 combine_payload_bytes=266076160 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266419912 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496072 weight_movement_bytes=69206016 total_nvlink_bytes=601702088 weight_to_token_ratio=0.129965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343768 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265985752 ingress_bytes=369451008 link_bytes=369451008 token_bytes=531627736 weight_movement_bytes=103809024 total_nvlink_bytes=635436760 weight_to_token_ratio=0.195266 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390214400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64878 dup_rows=24116 dup_rows_remote=21126 dedup_ratio=0.245321 router_tokens_per_expert=[759, 758, 790, 774, 749, 796, 782, 702, 779, 772, 782, 762, 755, 799, 736, 765, 696, 739, 770, 790, 783, 769, 764, 811, 812, 769, 783, 811, 724, 747, 766, 769, 734, 804, 742, 770, 785, 770, 774, 782, 734, 807, 773, 742, 749, 760, 808, 758, 780, 854, 771, 785, 729, 804, 744, 755, 799, 732, 743, 754, 755, 755, 786, 751] payload_rows_per_dst=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] dup_rows_per_dst=[3076, 2990, 2902, 3002, 3038, 2982, 3104, 3022] combine_rows_per_src=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344408 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267493720 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643032 weight_movement_bytes=34603008 total_nvlink_bytes=569246040 weight_to_token_ratio=0.064722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266075792 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807888 weight_movement_bytes=69206016 total_nvlink_bytes=601013904 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250368 weight_movement_bytes=69206016 total_nvlink_bytes=602456384 weight_to_token_ratio=0.129781 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65154 dup_rows=23824 dup_rows_remote=20942 dedup_ratio=0.242350 router_tokens_per_expert=[807, 811, 759, 775, 794, 766, 757, 755, 758, 810, 742, 776, 759, 792, 752, 785, 796, 801, 745, 755, 780, 775, 734, 792, 809, 712, 748, 795, 752, 719, 763, 767, 734, 771, 793, 788, 760, 743, 756, 759, 771, 804, 760, 771, 779, 761, 776, 788, 767, 776, 769, 762, 770, 739, 778, 744, 745, 757, 769, 760, 742, 752, 767, 800] payload_rows_per_dst=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] dup_rows_per_dst=[3100, 2988, 2972, 2942, 2882, 3040, 3014, 2886] combine_rows_per_src=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344016 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084304 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824592 weight_movement_bytes=34603008 total_nvlink_bytes=566427600 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64862 dup_rows=24122 dup_rows_remote=21080 dedup_ratio=0.245382 router_tokens_per_expert=[773, 737, 812, 729, 780, 764, 771, 768, 786, 774, 803, 750, 757, 759, 788, 771, 728, 723, 785, 760, 784, 768, 719, 737, 809, 756, 777, 772, 774, 789, 739, 765, 777, 751, 716, 772, 798, 790, 822, 790, 768, 779, 742, 767, 746, 731, 787, 811, 778, 786, 765, 737, 766, 767, 852, 778, 761, 772, 722, 773, 768, 741, 733, 799] payload_rows_per_dst=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] dup_rows_per_dst=[3024, 3076, 2828, 3042, 3130, 2972, 3054, 2996] combine_rows_per_src=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65080 dup_rows=24010 dup_rows_remote=21088 dedup_ratio=0.244242 router_tokens_per_expert=[764, 758, 725, 767, 800, 758, 790, 777, 785, 742, 808, 794, 766, 773, 785, 806, 746, 747, 751, 727, 776, 782, 750, 748, 770, 808, 767, 783, 776, 773, 809, 718, 766, 790, 750, 778, 799, 759, 765, 792, 763, 784, 743, 737, 792, 784, 776, 807, 766, 769, 785, 781, 752, 773, 720, 734, 769, 768, 788, 746, 744, 713, 766, 764] payload_rows_per_dst=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] dup_rows_per_dst=[3028, 3024, 2860, 2952, 3124, 3016, 3084, 2922] combine_rows_per_src=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344384 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215168 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085952 weight_movement_bytes=34603008 total_nvlink_bytes=568688960 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21062 dedup_ratio=0.244324 router_tokens_per_expert=[744, 758, 749, 797, 774, 786, 723, 778, 774, 784, 762, 797, 788, 779, 787, 730, 812, 751, 756, 797, 751, 724, 769, 746, 774, 772, 721, 760, 801, 741, 728, 811, 777, 770, 756, 763, 782, 792, 757, 759, 788, 750, 785, 749, 808, 773, 738, 800, 808, 774, 758, 734, 780, 750, 780, 782, 824, 730, 720, 779, 713, 796, 758, 795] payload_rows_per_dst=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] dup_rows_per_dst=[2956, 3154, 3046, 2938, 2958, 3050, 2918, 2998] combine_rows_per_src=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343768 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266018520 ingress_bytes=404086784 link_bytes=404086784 token_bytes=531693272 weight_movement_bytes=138412032 total_nvlink_bytes=670105304 weight_to_token_ratio=0.260323 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520546037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65074 dup_rows=24032 dup_rows_remote=21216 dedup_ratio=0.244466 router_tokens_per_expert=[716, 770, 771, 761, 774, 745, 755, 809, 809, 740, 787, 754, 764, 821, 735, 765, 772, 762, 745, 762, 745, 764, 719, 738, 797, 776, 733, 749, 810, 773, 758, 769, 824, 775, 736, 782, 768, 756, 736, 759, 728, 773, 759, 782, 823, 795, 809, 763, 769, 765, 739, 788, 760, 743, 807, 804, 750, 805, 771, 797, 760, 713, 768, 797] payload_rows_per_dst=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] dup_rows_per_dst=[3038, 3122, 2816, 3100, 2940, 3128, 2932, 2956] combine_rows_per_src=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65132 dup_rows=23742 dup_rows_remote=20800 dedup_ratio=0.241516 router_tokens_per_expert=[785, 761, 767, 743, 746, 740, 782, 785, 770, 776, 779, 793, 802, 758, 768, 740, 820, 740, 762, 799, 777, 753, 750, 747, 771, 743, 783, 792, 746, 748, 802, 765, 799, 793, 745, 793, 754, 790, 744, 721, 808, 768, 748, 840, 757, 730, 768, 767, 726, 731, 768, 769, 776, 812, 741, 769, 798, 801, 746, 748, 753, 750, 789, 757] payload_rows_per_dst=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] dup_rows_per_dst=[3062, 3032, 3014, 2826, 2954, 2942, 2912, 3000] combine_rows_per_src=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64978 dup_rows=24142 dup_rows_remote=21250 dedup_ratio=0.245585 router_tokens_per_expert=[738, 790, 717, 758, 777, 781, 798, 774, 808, 779, 734, 758, 750, 737, 773, 788, 768, 741, 770, 742, 752, 788, 826, 761, 738, 759, 788, 776, 746, 766, 781, 748, 746, 846, 743, 816, 814, 757, 803, 723, 762, 783, 793, 798, 762, 744, 767, 731, 751, 780, 722, 773, 809, 735, 735, 733, 814, 776, 752, 782, 720, 792, 791, 789] payload_rows_per_dst=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] dup_rows_per_dst=[2998, 3028, 3090, 2968, 3082, 3048, 2892, 3036] combine_rows_per_src=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344672 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912352 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533480032 weight_movement_bytes=103809024 total_nvlink_bytes=637289056 weight_to_token_ratio=0.194588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344328 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266666248 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988168 weight_movement_bytes=69206016 total_nvlink_bytes=602194184 weight_to_token_ratio=0.129845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74740 payload_rows_remote=65304 dup_rows=23564 dup_rows_remote=20558 dedup_ratio=0.239705 router_tokens_per_expert=[766, 764, 747, 784, 717, 795, 748, 743, 795, 749, 797, 742, 777, 782, 793, 786, 755, 820, 765, 763, 710, 742, 766, 777, 763, 803, 789, 790, 783, 758, 747, 758, 718, 728, 772, 762, 754, 794, 730, 759, 760, 771, 761, 781, 733, 778, 728, 780, 776, 761, 810, 773, 798, 831, 784, 749, 772, 773, 804, 745, 801, 749, 766, 777] payload_rows_per_dst=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] dup_rows_per_dst=[2862, 3006, 2944, 3024, 2800, 2890, 3086, 2952] combine_rows_per_src=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345160 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888264 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431368 weight_movement_bytes=34603008 total_nvlink_bytes=568034376 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343728 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124400 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905072 weight_movement_bytes=69206016 total_nvlink_bytes=603111088 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344912 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494800 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644688 weight_movement_bytes=34603008 total_nvlink_bytes=567247696 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74592 payload_rows_remote=65242 dup_rows=23712 dup_rows_remote=20832 dedup_ratio=0.241211 router_tokens_per_expert=[818, 792, 756, 776, 777, 753, 775, 759, 719, 744, 780, 789, 793, 767, 768, 728, 783, 794, 761, 795, 783, 751, 781, 704, 760, 741, 749, 760, 808, 778, 731, 739, 729, 781, 794, 813, 756, 744, 735, 763, 749, 751, 776, 788, 798, 793, 765, 775, 758, 777, 800, 706, 790, 824, 788, 767, 749, 778, 740, 783, 764, 768, 756, 782] payload_rows_per_dst=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] dup_rows_per_dst=[3078, 2944, 2916, 2884, 2880, 3038, 3014, 2958] combine_rows_per_src=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343448 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267828632 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535313816 weight_movement_bytes=103809024 total_nvlink_bytes=639122840 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65070 dup_rows=23852 dup_rows_remote=20866 dedup_ratio=0.242635 router_tokens_per_expert=[787, 751, 734, 761, 775, 776, 746, 722, 735, 806, 717, 747, 778, 818, 797, 760, 752, 714, 801, 765, 802, 764, 742, 805, 779, 747, 752, 796, 815, 742, 784, 769, 767, 796, 753, 774, 763, 722, 774, 757, 780, 762, 781, 729, 773, 790, 771, 754, 788, 733, 805, 748, 753, 707, 812, 766, 795, 768, 770, 764, 817, 748, 769, 824] payload_rows_per_dst=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] dup_rows_per_dst=[2946, 2924, 3012, 2986, 2892, 2958, 3000, 3134] combine_rows_per_src=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=344296 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267575528 ingress_bytes=336437248 link_bytes=336437248 token_bytes=534806760 weight_movement_bytes=69206016 total_nvlink_bytes=604012776 weight_to_token_ratio=0.129404 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265986279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64966 dup_rows=24026 dup_rows_remote=20810 dedup_ratio=0.244405 router_tokens_per_expert=[770, 767, 781, 743, 775, 747, 759, 747, 785, 731, 765, 708, 764, 755, 761, 743, 792, 753, 761, 800, 799, 742, 758, 771, 816, 780, 769, 758, 710, 777, 752, 759, 726, 815, 792, 795, 818, 733, 741, 769, 680, 788, 776, 772, 770, 778, 726, 742, 811, 762, 762, 797, 803, 741, 795, 807, 795, 780, 764, 771, 791, 780, 804, 770] payload_rows_per_dst=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] dup_rows_per_dst=[2826, 2876, 3104, 3006, 2994, 2826, 3178, 3216] combine_rows_per_src=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64834 dup_rows=23992 dup_rows_remote=20738 dedup_ratio=0.244059 router_tokens_per_expert=[749, 789, 806, 766, 722, 727, 757, 768, 749, 781, 751, 797, 771, 787, 799, 726, 751, 756, 772, 779, 775, 762, 783, 734, 791, 782, 783, 775, 784, 780, 785, 792, 725, 768, 791, 723, 739, 770, 767, 775, 805, 743, 782, 763, 756, 720, 803, 791, 787, 778, 819, 745, 732, 750, 770, 800, 765, 781, 765, 790, 735, 762, 805, 718] payload_rows_per_dst=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] dup_rows_per_dst=[3254, 3116, 3122, 3088, 2918, 2812, 2732, 2950] combine_rows_per_src=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343744 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870464 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397184 weight_movement_bytes=69206016 total_nvlink_bytes=602603200 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65066 dup_rows=23930 dup_rows_remote=20842 dedup_ratio=0.243429 router_tokens_per_expert=[754, 735, 742, 764, 765, 738, 766, 772, 819, 705, 782, 776, 764, 787, 759, 756, 751, 748, 793, 764, 779, 773, 762, 786, 788, 779, 743, 764, 752, 801, 723, 796, 732, 827, 760, 776, 718, 800, 759, 817, 738, 773, 805, 751, 784, 786, 760, 801, 795, 755, 739, 705, 813, 748, 759, 742, 772, 763, 810, 806, 751, 757, 807, 757] payload_rows_per_dst=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] dup_rows_per_dst=[2924, 2922, 3072, 2900, 3026, 3088, 2936, 3062] combine_rows_per_src=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64958 dup_rows=23966 dup_rows_remote=20872 dedup_ratio=0.243795 router_tokens_per_expert=[728, 760, 825, 777, 751, 752, 803, 701, 765, 774, 788, 781, 789, 743, 792, 741, 782, 767, 797, 829, 764, 828, 742, 728, 714, 786, 764, 706, 798, 766, 795, 754, 793, 772, 775, 801, 803, 750, 727, 793, 774, 776, 771, 726, 793, 739, 744, 749, 727, 748, 771, 799, 755, 819, 756, 786, 770, 773, 758, 781, 758, 769, 767, 739] payload_rows_per_dst=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] dup_rows_per_dst=[3002, 2948, 3094, 2996, 3032, 2950, 3056, 2888] combine_rows_per_src=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343104 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266443840 ingress_bytes=404512768 link_bytes=404512768 token_bytes=532544576 weight_movement_bytes=138412032 total_nvlink_bytes=670956608 weight_to_token_ratio=0.259907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522148980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64938 dup_rows=23962 dup_rows_remote=20826 dedup_ratio=0.243754 router_tokens_per_expert=[740, 818, 770, 766, 719, 803, 756, 771, 753, 776, 720, 725, 725, 762, 755, 788, 747, 738, 809, 774, 796, 783, 757, 754, 746, 748, 698, 757, 782, 761, 748, 793, 753, 785, 790, 764, 791, 759, 799, 776, 812, 788, 763, 756, 716, 782, 766, 767, 771, 764, 817, 799, 773, 797, 780, 769, 759, 813, 721, 771, 777, 781, 768, 787] payload_rows_per_dst=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] dup_rows_per_dst=[3012, 2800, 2962, 2912, 3066, 3022, 3136, 3052] combine_rows_per_src=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262407376 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6135160016 ingress_bytes=7407558656 link_bytes=7407558656 total_nvlink_bytes=13542718672 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.098098 dedup_ratio=0.243728 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211915081e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342288 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902352 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462416 weight_movement_bytes=34603008 total_nvlink_bytes=566065424 weight_to_token_ratio=0.065109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64770 dup_rows=24112 dup_rows_remote=20910 dedup_ratio=0.245280 router_tokens_per_expert=[777, 772, 730, 766, 776, 765, 739, 787, 801, 807, 788, 773, 777, 783, 802, 781, 738, 722, 772, 785, 761, 803, 785, 749, 754, 744, 771, 766, 718, 767, 821, 777, 755, 731, 775, 795, 782, 781, 806, 714, 733, 784, 799, 781, 817, 719, 762, 742, 786, 764, 779, 713, 760, 755, 772, 744, 768, 788, 792, 778, 726, 774, 746, 774] payload_rows_per_dst=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] dup_rows_per_dst=[2938, 3202, 2962, 3006, 3032, 3006, 2988, 2978] combine_rows_per_src=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266853968 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364304 weight_movement_bytes=34603008 total_nvlink_bytes=567967312 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343320 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411288 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479256 weight_movement_bytes=69206016 total_nvlink_bytes=601685272 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343056 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266329104 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315152 weight_movement_bytes=69206016 total_nvlink_bytes=601521168 weight_to_token_ratio=0.130009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233454992 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120674704 ingress_bytes=7427694592 link_bytes=7427694592 total_nvlink_bytes=13548369296 weight_to_token_ratio=0.107485 prefetch_share_of_ingress=0.093173 dedup_ratio=0.244450 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215209425e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342720 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640640 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938560 weight_movement_bytes=34603008 total_nvlink_bytes=565541568 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64830 dup_rows=24160 dup_rows_remote=21170 dedup_ratio=0.245768 router_tokens_per_expert=[793, 732, 743, 780, 757, 735, 762, 745, 785, 788, 783, 720, 759, 765, 793, 758, 831, 747, 733, 785, 747, 778, 776, 818, 732, 765, 755, 764, 789, 780, 797, 770, 763, 736, 734, 788, 754, 773, 835, 758, 742, 777, 797, 730, 770, 777, 789, 768, 747, 792, 794, 739, 813, 730, 750, 775, 783, 771, 751, 771, 775, 782, 771, 752] payload_rows_per_dst=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] dup_rows_per_dst=[2924, 3062, 3104, 2990, 3010, 3110, 2926, 3034] combine_rows_per_src=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262011640 prefetch_bytes=553648128 reduce_bytes=899678208 weight_movement_bytes=1453326336 egress_bytes=6134960888 ingress_bytes=7580377088 link_bytes=7580377088 total_nvlink_bytes=13715337976 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.073037 dedup_ratio=0.243660 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=26 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240189074e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258284272 prefetch_bytes=692060160 reduce_bytes=380633088 weight_movement_bytes=1072693248 egress_bytes=6133101296 ingress_bytes=7197876224 link_bytes=7197876224 total_nvlink_bytes=13330977520 weight_to_token_ratio=0.087508 prefetch_share_of_ingress=0.096148 dedup_ratio=0.244418 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=11 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.177609946e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255037232 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6131476272 ingress_bytes=7507681280 link_bytes=7507681280 total_nvlink_bytes=13639157552 weight_to_token_ratio=0.112943 prefetch_share_of_ingress=0.078353 dedup_ratio=0.244336 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228295663e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64690 dup_rows=24094 dup_rows_remote=20954 dedup_ratio=0.245097 router_tokens_per_expert=[742, 756, 785, 734, 788, 791, 726, 782, 806, 771, 718, 773, 763, 757, 764, 820, 775, 748, 747, 806, 763, 801, 765, 774, 776, 755, 785, 757, 797, 774, 762, 759, 766, 804, 770, 753, 745, 723, 775, 772, 802, 731, 752, 779, 769, 834, 781, 750, 721, 805, 761, 761, 785, 749, 810, 741, 714, 778, 781, 758, 766, 722, 769, 805] payload_rows_per_dst=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] dup_rows_per_dst=[3140, 2864, 3300, 3026, 2968, 2900, 2978, 2918] combine_rows_per_src=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258702816 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133306848 ingress_bytes=7371104256 link_bytes=7371104256 total_nvlink_bytes=13504411104 weight_to_token_ratio=0.101618 prefetch_share_of_ingress=0.103277 dedup_ratio=0.243937 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205950952e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=344000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265887680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531431360 weight_movement_bytes=34603008 total_nvlink_bytes=566034368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342576 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312816 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283056 weight_movement_bytes=34603008 total_nvlink_bytes=564886064 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254326472 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6131117768 ingress_bytes=7714947072 link_bytes=7715290728 total_nvlink_bytes=13846064840 weight_to_token_ratio=0.129892 prefetch_share_of_ingress=0.085219 dedup_ratio=0.243881 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262261647e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64598 dup_rows=24140 dup_rows_remote=20862 dedup_ratio=0.245565 router_tokens_per_expert=[742, 779, 781, 759, 751, 799, 787, 773, 793, 793, 745, 789, 743, 739, 749, 720, 729, 779, 809, 744, 770, 781, 756, 803, 769, 778, 730, 744, 759, 767, 765, 766, 735, 767, 818, 767, 794, 807, 749, 786, 783, 807, 762, 784, 732, 755, 772, 742, 771, 780, 752, 824, 735, 740, 760, 804, 741, 770, 754, 782, 798, 730, 794, 766] payload_rows_per_dst=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] dup_rows_per_dst=[3278, 3126, 2842, 3292, 2880, 3112, 2810, 2800] combine_rows_per_src=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264593408 dispatch_meta_bytes=341840 combine_payload_bytes=264593408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264935248 ingress_bytes=299196416 link_bytes=299196416 token_bytes=529528656 weight_movement_bytes=34603008 total_nvlink_bytes=564131664 weight_to_token_ratio=0.065347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125852026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12226985568 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6117441120 ingress_bytes=7632076800 link_bytes=7632420360 total_nvlink_bytes=13749517920 weight_to_token_ratio=0.124522 prefetch_share_of_ingress=0.099746 dedup_ratio=0.244912 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.248703624e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none + [2026-08-29 23:41:52] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 678.4 | throughput per GPU (TFLOP/s/GPU): 188.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.944521E+00 | loss scale: 1.0 | grad norm: 41.738 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64880 dup_rows=24276 dup_rows_remote=21192 dedup_ratio=0.246948 router_tokens_per_expert=[744, 774, 806, 762, 817, 777, 743, 803, 743, 797, 749, 766, 739, 784, 789, 749, 793, 799, 732, 743, 714, 748, 790, 762, 741, 803, 751, 768, 760, 735, 748, 830, 794, 762, 768, 769, 764, 776, 719, 815, 743, 781, 805, 738, 728, 788, 742, 762, 750, 795, 726, 744, 775, 789, 778, 784, 769, 787, 781, 749, 746, 800, 784, 782] payload_rows_per_dst=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] dup_rows_per_dst=[3046, 3084, 2952, 3064, 2988, 2998, 3012, 3132] combine_rows_per_src=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64912 dup_rows=24124 dup_rows_remote=21076 dedup_ratio=0.245402 router_tokens_per_expert=[732, 794, 822, 742, 800, 768, 736, 722, 753, 746, 815, 770, 747, 790, 739, 756, 756, 754, 783, 788, 790, 763, 800, 754, 773, 737, 765, 789, 829, 835, 775, 755, 781, 727, 788, 786, 781, 768, 804, 718, 777, 780, 758, 807, 750, 745, 711, 744, 753, 804, 787, 780, 769, 719, 740, 757, 789, 759, 777, 768, 749, 749, 812, 737] payload_rows_per_dst=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] dup_rows_per_dst=[2894, 2938, 3076, 3146, 3142, 2918, 2962, 3048] combine_rows_per_src=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64982 dup_rows=24022 dup_rows_remote=20980 dedup_ratio=0.244364 router_tokens_per_expert=[766, 771, 779, 769, 811, 733, 739, 763, 749, 805, 807, 775, 758, 778, 764, 776, 783, 763, 705, 724, 713, 781, 771, 785, 756, 811, 784, 764, 804, 755, 748, 749, 798, 739, 749, 734, 724, 785, 761, 757, 740, 744, 811, 752, 787, 799, 806, 791, 734, 762, 794, 790, 729, 724, 784, 796, 781, 770, 772, 764, 761, 752, 842, 781] payload_rows_per_dst=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] dup_rows_per_dst=[3024, 3092, 2860, 3042, 2924, 3048, 2922, 3110] combine_rows_per_src=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64944 dup_rows=23980 dup_rows_remote=21000 dedup_ratio=0.243937 router_tokens_per_expert=[778, 771, 764, 755, 767, 785, 818, 736, 800, 776, 747, 771, 729, 757, 764, 782, 771, 755, 726, 724, 745, 780, 762, 734, 758, 761, 817, 789, 757, 774, 765, 750, 730, 764, 806, 798, 783, 819, 778, 702, 776, 773, 804, 756, 780, 757, 808, 789, 712, 755, 814, 794, 769, 781, 774, 742, 754, 767, 745, 741, 794, 762, 802, 755] payload_rows_per_dst=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] dup_rows_per_dst=[3104, 3010, 2890, 3058, 2980, 3032, 2974, 2932] combine_rows_per_src=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64924 dup_rows=24084 dup_rows_remote=20984 dedup_ratio=0.244995 router_tokens_per_expert=[753, 803, 772, 779, 788, 714, 781, 826, 740, 747, 825, 763, 777, 706, 795, 736, 797, 754, 764, 786, 771, 768, 784, 757, 706, 759, 781, 796, 780, 703, 746, 763, 787, 768, 755, 754, 756, 749, 800, 758, 861, 743, 751, 761, 810, 764, 821, 761, 754, 759, 763, 802, 791, 751, 769, 809, 725, 788, 724, 787, 730, 760, 749, 772] payload_rows_per_dst=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] dup_rows_per_dst=[3124, 2952, 2958, 2888, 3000, 3158, 3100, 2904] combine_rows_per_src=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64918 dup_rows=24124 dup_rows_remote=21094 dedup_ratio=0.245402 router_tokens_per_expert=[820, 767, 730, 745, 803, 779, 774, 771, 751, 771, 768, 743, 784, 780, 776, 749, 786, 753, 786, 727, 757, 788, 794, 755, 724, 791, 768, 780, 782, 770, 767, 763, 809, 746, 780, 776, 798, 721, 738, 744, 713, 767, 756, 726, 764, 811, 828, 754, 739, 852, 755, 718, 750, 783, 773, 779, 790, 767, 766, 755, 777, 776, 761, 778] payload_rows_per_dst=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] dup_rows_per_dst=[3146, 2980, 3030, 2984, 2964, 2980, 3030, 3010] combine_rows_per_src=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64896 dup_rows=24234 dup_rows_remote=21328 dedup_ratio=0.246521 router_tokens_per_expert=[736, 768, 738, 786, 748, 768, 741, 770, 705, 785, 780, 787, 707, 751, 772, 744, 767, 789, 769, 748, 776, 756, 754, 769, 760, 739, 842, 801, 770, 731, 788, 735, 782, 828, 756, 801, 781, 781, 764, 790, 758, 736, 781, 730, 783, 762, 722, 768, 804, 792, 758, 804, 777, 803, 759, 793, 762, 748, 787, 729, 785, 775, 802, 771] payload_rows_per_dst=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] dup_rows_per_dst=[2910, 2984, 3060, 3036, 3134, 2906, 3158, 3046] combine_rows_per_src=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=344288 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266092768 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531841248 weight_movement_bytes=34603008 total_nvlink_bytes=566444256 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343952 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266223504 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103056 weight_movement_bytes=34603008 total_nvlink_bytes=566706064 weight_to_token_ratio=0.065031 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343848 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510120 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676392 weight_movement_bytes=34603008 total_nvlink_bytes=567279400 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343776 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354400 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365024 weight_movement_bytes=69206016 total_nvlink_bytes=601571040 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343632 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272336 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201040 weight_movement_bytes=69206016 total_nvlink_bytes=601407056 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344048 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266248176 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532152304 weight_movement_bytes=34603008 total_nvlink_bytes=566755312 weight_to_token_ratio=0.065025 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344896 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266158912 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531972928 weight_movement_bytes=69206016 total_nvlink_bytes=601178944 weight_to_token_ratio=0.130093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64950 dup_rows=23996 dup_rows_remote=21048 dedup_ratio=0.244100 router_tokens_per_expert=[758, 780, 775, 758, 797, 752, 743, 789, 764, 794, 799, 801, 713, 761, 774, 798, 807, 758, 830, 764, 794, 721, 750, 773, 746, 775, 774, 741, 709, 802, 721, 767, 764, 810, 762, 806, 795, 773, 742, 783, 761, 796, 783, 778, 784, 736, 742, 728, 775, 751, 742, 750, 800, 793, 718, 803, 761, 744, 761, 758, 764, 762, 776, 763] payload_rows_per_dst=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] dup_rows_per_dst=[2948, 3228, 3218, 2914, 2924, 2994, 2844, 2926] combine_rows_per_src=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20954 dedup_ratio=0.243184 router_tokens_per_expert=[796, 775, 762, 730, 701, 822, 764, 789, 764, 743, 737, 760, 766, 811, 750, 790, 748, 772, 814, 778, 754, 817, 733, 712, 780, 795, 789, 769, 744, 751, 788, 760, 781, 782, 750, 812, 764, 789, 769, 739, 739, 786, 775, 811, 777, 760, 763, 714, 754, 748, 759, 761, 788, 789, 761, 765, 798, 746, 782, 759, 755, 739, 782, 791] payload_rows_per_dst=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] dup_rows_per_dst=[2972, 2952, 3060, 2940, 3090, 2884, 3000, 3008] combine_rows_per_src=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64826 dup_rows=24118 dup_rows_remote=20980 dedup_ratio=0.245341 router_tokens_per_expert=[824, 740, 768, 768, 796, 749, 706, 758, 745, 776, 800, 758, 749, 772, 751, 735, 780, 773, 780, 758, 757, 767, 805, 758, 700, 755, 791, 772, 754, 804, 733, 701, 790, 842, 796, 770, 748, 770, 785, 788, 814, 729, 798, 749, 782, 731, 757, 758, 749, 771, 754, 705, 783, 782, 816, 755, 793, 767, 802, 760, 817, 795, 773, 740] payload_rows_per_dst=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] dup_rows_per_dst=[2978, 2998, 3034, 2876, 3082, 3080, 2932, 3138] combine_rows_per_src=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65202 dup_rows=23822 dup_rows_remote=20844 dedup_ratio=0.242330 router_tokens_per_expert=[734, 769, 748, 679, 810, 746, 756, 758, 845, 724, 745, 800, 799, 766, 768, 797, 748, 799, 771, 789, 802, 764, 752, 779, 788, 759, 782, 775, 767, 790, 757, 758, 776, 755, 770, 779, 756, 794, 766, 733, 769, 744, 743, 825, 796, 781, 748, 774, 789, 763, 734, 813, 750, 802, 750, 758, 769, 717, 778, 793, 759, 732, 752, 760] payload_rows_per_dst=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] dup_rows_per_dst=[2834, 3140, 2970, 2964, 2978, 2982, 2988, 2966] combine_rows_per_src=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343992 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266379192 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532414392 weight_movement_bytes=138412032 total_nvlink_bytes=670826424 weight_to_token_ratio=0.259970 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65048 dup_rows=23928 dup_rows_remote=20826 dedup_ratio=0.243408 router_tokens_per_expert=[758, 772, 734, 757, 748, 784, 757, 801, 775, 780, 781, 767, 783, 772, 731, 725, 771, 759, 779, 776, 769, 753, 775, 808, 792, 780, 792, 758, 749, 717, 813, 814, 768, 743, 749, 828, 727, 727, 736, 763, 826, 724, 744, 754, 755, 760, 763, 785, 771, 779, 771, 753, 752, 767, 784, 824, 817, 750, 784, 742, 757, 763, 798, 758] payload_rows_per_dst=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] dup_rows_per_dst=[2920, 2974, 3040, 3102, 2864, 2974, 3062, 2992] combine_rows_per_src=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64814 dup_rows=24072 dup_rows_remote=21088 dedup_ratio=0.244873 router_tokens_per_expert=[778, 795, 753, 762, 730, 763, 769, 746, 752, 799, 778, 837, 729, 792, 786, 804, 744, 803, 751, 742, 760, 825, 756, 755, 769, 754, 791, 761, 798, 746, 769, 745, 739, 738, 785, 767, 772, 754, 683, 775, 808, 741, 773, 758, 753, 767, 774, 727, 755, 778, 768, 751, 765, 797, 751, 836, 805, 743, 759, 808, 788, 751, 760, 781] payload_rows_per_dst=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] dup_rows_per_dst=[3002, 3112, 3100, 2910, 2832, 3040, 2984, 3092] combine_rows_per_src=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65018 dup_rows=24074 dup_rows_remote=21082 dedup_ratio=0.244893 router_tokens_per_expert=[747, 734, 782, 835, 799, 818, 771, 744, 770, 781, 772, 779, 721, 736, 791, 804, 736, 767, 754, 788, 757, 775, 764, 761, 736, 815, 803, 736, 798, 806, 742, 761, 743, 737, 714, 807, 761, 784, 778, 791, 770, 768, 820, 782, 760, 755, 753, 752, 764, 758, 764, 724, 772, 778, 800, 747, 785, 766, 761, 714, 780, 770, 749, 762] payload_rows_per_dst=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] dup_rows_per_dst=[3108, 3000, 2992, 3050, 3042, 2998, 2944, 2940] combine_rows_per_src=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64970 dup_rows=24062 dup_rows_remote=21124 dedup_ratio=0.244771 router_tokens_per_expert=[766, 752, 700, 752, 760, 726, 759, 770, 763, 746, 767, 757, 754, 732, 763, 755, 787, 790, 771, 787, 744, 725, 791, 758, 801, 772, 754, 793, 789, 780, 841, 789, 734, 763, 738, 741, 768, 805, 789, 782, 789, 751, 775, 769, 789, 735, 737, 760, 786, 771, 752, 806, 781, 773, 769, 794, 761, 793, 749, 796, 754, 754, 770, 824] payload_rows_per_dst=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] dup_rows_per_dst=[2830, 2952, 3020, 3142, 3040, 2938, 3110, 3030] combine_rows_per_src=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344248 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026616 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708984 weight_movement_bytes=34603008 total_nvlink_bytes=568311992 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343224 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870520 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397816 weight_movement_bytes=103809024 total_nvlink_bytes=635206840 weight_to_token_ratio=0.195351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344184 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411576 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478968 weight_movement_bytes=34603008 total_nvlink_bytes=569081976 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343496 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780104 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216712 weight_movement_bytes=69206016 total_nvlink_bytes=602422728 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343608 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265821752 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299896 weight_movement_bytes=69206016 total_nvlink_bytes=600505912 weight_to_token_ratio=0.130258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344400 combine_payload_bytes=266313728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266658128 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971856 weight_movement_bytes=103809024 total_nvlink_bytes=636780880 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344376 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461496 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578616 weight_movement_bytes=34603008 total_nvlink_bytes=567181624 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64634 dup_rows=24058 dup_rows_remote=20706 dedup_ratio=0.244731 router_tokens_per_expert=[788, 745, 782, 789, 767, 791, 742, 760, 718, 688, 755, 751, 833, 795, 786, 785, 761, 805, 743, 740, 789, 798, 757, 774, 809, 755, 740, 744, 751, 769, 767, 778, 744, 790, 703, 809, 754, 781, 824, 780, 763, 755, 789, 777, 776, 793, 743, 760, 819, 732, 796, 765, 764, 762, 747, 745, 783, 751, 770, 758, 822, 785, 742, 715] payload_rows_per_dst=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] dup_rows_per_dst=[3352, 3006, 3016, 2806, 3090, 2992, 2870, 2926] combine_rows_per_src=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65142 dup_rows=23914 dup_rows_remote=21042 dedup_ratio=0.243266 router_tokens_per_expert=[759, 783, 746, 775, 796, 747, 768, 750, 755, 742, 754, 807, 756, 746, 748, 752, 765, 797, 777, 742, 758, 779, 741, 762, 753, 694, 744, 754, 787, 789, 818, 789, 769, 811, 774, 775, 778, 778, 774, 773, 760, 730, 726, 775, 768, 804, 741, 794, 790, 748, 799, 766, 795, 780, 770, 769, 801, 808, 725, 740, 777, 783, 790, 748] payload_rows_per_dst=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] dup_rows_per_dst=[3016, 2872, 2866, 3000, 3046, 2936, 3160, 3018] combine_rows_per_src=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64898 dup_rows=24076 dup_rows_remote=21030 dedup_ratio=0.244914 router_tokens_per_expert=[796, 795, 717, 747, 817, 747, 774, 773, 783, 773, 767, 755, 766, 780, 809, 805, 790, 749, 739, 739, 766, 752, 715, 807, 768, 784, 812, 773, 746, 775, 785, 815, 755, 774, 767, 753, 776, 746, 775, 784, 753, 753, 782, 776, 737, 784, 789, 784, 749, 782, 766, 761, 809, 744, 784, 740, 740, 726, 765, 746, 762, 762, 776, 733] payload_rows_per_dst=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] dup_rows_per_dst=[2946, 3212, 2870, 3076, 2990, 3080, 2856, 3046] combine_rows_per_src=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64964 dup_rows=24144 dup_rows_remote=21178 dedup_ratio=0.245605 router_tokens_per_expert=[772, 728, 755, 807, 809, 780, 751, 763, 786, 782, 751, 734, 760, 771, 715, 791, 742, 810, 784, 739, 799, 724, 801, 738, 797, 734, 749, 817, 797, 808, 771, 764, 743, 756, 745, 751, 785, 753, 821, 727, 790, 707, 748, 779, 724, 826, 741, 741, 777, 749, 807, 788, 798, 845, 825, 775, 725, 738, 773, 754, 744, 760, 766, 762] payload_rows_per_dst=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] dup_rows_per_dst=[3016, 2974, 2978, 3100, 2966, 2928, 3228, 2954] combine_rows_per_src=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=341360 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265082224 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529823088 weight_movement_bytes=34603008 total_nvlink_bytes=564426096 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64998 dup_rows=24098 dup_rows_remote=21094 dedup_ratio=0.245138 router_tokens_per_expert=[814, 738, 806, 797, 744, 741, 791, 773, 770, 782, 776, 790, 717, 754, 759, 781, 764, 772, 759, 736, 769, 771, 753, 783, 821, 770, 790, 777, 761, 807, 738, 755, 768, 768, 798, 793, 779, 750, 716, 767, 767, 738, 760, 744, 759, 727, 773, 774, 740, 778, 764, 754, 735, 812, 758, 765, 759, 830, 762, 736, 785, 753, 771, 810] payload_rows_per_dst=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] dup_rows_per_dst=[3088, 3044, 3028, 3006, 2938, 2984, 3004, 3006] combine_rows_per_src=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64990 dup_rows=23918 dup_rows_remote=20864 dedup_ratio=0.243306 router_tokens_per_expert=[755, 760, 724, 790, 802, 767, 752, 762, 741, 767, 782, 753, 754, 739, 769, 724, 775, 767, 760, 776, 771, 812, 786, 778, 713, 798, 756, 777, 801, 741, 743, 780, 761, 803, 795, 752, 749, 768, 780, 801, 763, 772, 798, 828, 807, 737, 777, 736, 807, 718, 763, 722, 740, 774, 787, 763, 790, 791, 795, 727, 758, 751, 802, 762] payload_rows_per_dst=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] dup_rows_per_dst=[2992, 2862, 3054, 2942, 3094, 3076, 2874, 3024] combine_rows_per_src=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65296 dup_rows=23766 dup_rows_remote=20912 dedup_ratio=0.241760 router_tokens_per_expert=[823, 727, 784, 785, 831, 743, 783, 798, 769, 755, 754, 771, 777, 775, 767, 763, 719, 799, 756, 736, 749, 784, 724, 769, 788, 806, 733, 749, 739, 708, 762, 763, 792, 780, 796, 740, 779, 736, 777, 747, 786, 760, 791, 752, 791, 760, 793, 787, 736, 736, 795, 745, 790, 763, 803, 808, 709, 774, 772, 803, 793, 745, 775, 749] payload_rows_per_dst=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] dup_rows_per_dst=[3040, 2934, 2890, 2854, 3056, 3068, 3044, 2880] combine_rows_per_src=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64938 dup_rows=24048 dup_rows_remote=21092 dedup_ratio=0.244629 router_tokens_per_expert=[754, 753, 784, 776, 819, 770, 776, 786, 784, 782, 748, 741, 825, 783, 747, 808, 740, 766, 797, 817, 765, 767, 725, 771, 750, 783, 754, 735, 762, 727, 749, 781, 745, 779, 777, 800, 746, 774, 775, 757, 758, 736, 784, 747, 762, 826, 755, 769, 753, 738, 736, 772, 778, 756, 768, 772, 760, 739, 756, 737, 785, 781, 790, 816] payload_rows_per_dst=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] dup_rows_per_dst=[3132, 3094, 3046, 2924, 2962, 2956, 2914, 3020] combine_rows_per_src=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344736 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166368 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533988000 weight_movement_bytes=34603008 total_nvlink_bytes=568591008 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344568 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266437112 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529656 weight_movement_bytes=103809024 total_nvlink_bytes=636338680 weight_to_token_ratio=0.194936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344368 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266576176 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807984 weight_movement_bytes=69206016 total_nvlink_bytes=602014000 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343416 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542456 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741496 weight_movement_bytes=34603008 total_nvlink_bytes=567344504 weight_to_token_ratio=0.064953 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344832 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267797248 ingress_bytes=336658432 link_bytes=336658432 token_bytes=535249664 weight_movement_bytes=69206016 total_nvlink_bytes=604455680 weight_to_token_ratio=0.129297 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266818576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=344120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266330168 ingress_bytes=300589056 link_bytes=300589056 token_bytes=532316216 weight_movement_bytes=34603008 total_nvlink_bytes=566919224 weight_to_token_ratio=0.065005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131092418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65218 dup_rows=23874 dup_rows_remote=21094 dedup_ratio=0.242859 router_tokens_per_expert=[763, 762, 746, 722, 793, 704, 758, 748, 781, 766, 760, 777, 757, 798, 767, 749, 757, 778, 803, 813, 830, 764, 781, 775, 758, 813, 759, 752, 756, 797, 722, 772, 762, 752, 740, 771, 778, 789, 725, 813, 750, 791, 806, 782, 758, 751, 793, 741, 772, 791, 742, 786, 772, 752, 757, 793, 787, 761, 752, 744, 746, 756, 804, 754] payload_rows_per_dst=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] dup_rows_per_dst=[2780, 2994, 3174, 3026, 2936, 3018, 2970, 2976] combine_rows_per_src=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65270 dup_rows=23660 dup_rows_remote=20752 dedup_ratio=0.240682 router_tokens_per_expert=[789, 764, 767, 765, 738, 772, 792, 733, 709, 799, 822, 760, 739, 774, 756, 782, 766, 756, 770, 770, 726, 859, 777, 759, 776, 735, 773, 807, 767, 732, 778, 825, 773, 774, 732, 795, 784, 775, 767, 751, 771, 735, 742, 726, 764, 754, 765, 795, 799, 746, 774, 742, 759, 766, 757, 796, 775, 787, 758, 745, 766, 777, 773, 792] payload_rows_per_dst=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] dup_rows_per_dst=[2990, 2908, 3012, 2958, 2994, 2870, 2986, 2942] combine_rows_per_src=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=64872 dup_rows=24120 dup_rows_remote=21238 dedup_ratio=0.245361 router_tokens_per_expert=[801, 760, 719, 774, 750, 762, 764, 792, 802, 781, 763, 767, 804, 765, 748, 779, 766, 734, 762, 753, 753, 749, 765, 747, 792, 745, 772, 762, 745, 708, 807, 790, 738, 808, 796, 773, 724, 779, 773, 785, 782, 790, 759, 767, 761, 772, 777, 745, 784, 794, 775, 806, 742, 797, 794, 760, 789, 762, 791, 753, 750, 751, 765, 729] payload_rows_per_dst=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] dup_rows_per_dst=[3012, 3050, 2854, 2892, 3068, 3306, 3056, 2882] combine_rows_per_src=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64872 dup_rows=24166 dup_rows_remote=21184 dedup_ratio=0.245829 router_tokens_per_expert=[800, 717, 769, 737, 778, 782, 828, 803, 740, 757, 768, 797, 766, 746, 731, 760, 744, 758, 762, 755, 731, 816, 745, 756, 804, 757, 778, 841, 772, 798, 783, 736, 814, 737, 793, 752, 756, 749, 779, 744, 767, 752, 780, 782, 787, 735, 703, 762, 806, 792, 754, 758, 743, 798, 782, 752, 796, 766, 753, 778, 746, 764, 826, 731] payload_rows_per_dst=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] dup_rows_per_dst=[3026, 2924, 2976, 3154, 2982, 2910, 3108, 3086] combine_rows_per_src=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=345248 combine_payload_bytes=267132928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267478176 ingress_bytes=370941952 link_bytes=370941952 token_bytes=534611104 weight_movement_bytes=103809024 total_nvlink_bytes=638420128 weight_to_token_ratio=0.194177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395824702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65038 dup_rows=23950 dup_rows_remote=20966 dedup_ratio=0.243632 router_tokens_per_expert=[746, 772, 771, 752, 738, 771, 742, 782, 781, 760, 771, 735, 790, 741, 782, 772, 760, 771, 725, 784, 800, 798, 745, 753, 757, 758, 758, 751, 743, 813, 773, 805, 736, 779, 741, 775, 774, 757, 808, 761, 791, 764, 750, 787, 725, 783, 718, 807, 735, 778, 768, 810, 798, 721, 795, 745, 748, 827, 797, 772, 735, 790, 804, 773] payload_rows_per_dst=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] dup_rows_per_dst=[2944, 2988, 2968, 2958, 2996, 3016, 2984, 3096] combine_rows_per_src=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=65020 dup_rows=24180 dup_rows_remote=21278 dedup_ratio=0.245972 router_tokens_per_expert=[769, 741, 777, 782, 776, 748, 777, 775, 800, 771, 802, 815, 807, 795, 769, 765, 707, 803, 769, 774, 753, 682, 764, 751, 760, 827, 820, 778, 746, 740, 757, 790, 804, 734, 804, 755, 740, 757, 778, 780, 713, 746, 757, 763, 811, 765, 770, 773, 728, 711, 768, 751, 787, 794, 743, 732, 726, 774, 777, 770, 808, 798, 785, 760] payload_rows_per_dst=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] dup_rows_per_dst=[3082, 3162, 2902, 3104, 3014, 3058, 2846, 3012] combine_rows_per_src=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65068 dup_rows=23994 dup_rows_remote=20962 dedup_ratio=0.244080 router_tokens_per_expert=[814, 761, 777, 769, 788, 775, 784, 773, 768, 768, 808, 726, 778, 775, 791, 750, 759, 765, 755, 794, 745, 707, 797, 751, 775, 774, 750, 773, 771, 798, 749, 747, 759, 751, 797, 756, 746, 829, 750, 738, 802, 769, 778, 775, 758, 787, 735, 749, 785, 717, 710, 789, 755, 737, 789, 820, 743, 722, 761, 788, 722, 806, 802, 812] payload_rows_per_dst=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] dup_rows_per_dst=[3100, 3004, 2872, 3032, 2908, 3032, 3002, 3044] combine_rows_per_src=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21078 dedup_ratio=0.243958 router_tokens_per_expert=[771, 791, 749, 771, 764, 745, 786, 733, 791, 772, 765, 783, 772, 794, 796, 725, 777, 767, 759, 770, 803, 759, 803, 749, 771, 747, 793, 793, 781, 786, 794, 720, 771, 779, 738, 783, 748, 786, 747, 799, 781, 798, 737, 754, 728, 794, 721, 778, 772, 739, 719, 769, 796, 770, 790, 751, 771, 737, 774, 732, 793, 823, 752, 742] payload_rows_per_dst=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] dup_rows_per_dst=[2916, 2970, 3068, 3176, 3048, 2904, 2892, 3008] combine_rows_per_src=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344088 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690008 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535035928 weight_movement_bytes=69206016 total_nvlink_bytes=604241944 weight_to_token_ratio=0.129348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344440 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266060152 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775864 weight_movement_bytes=103809024 total_nvlink_bytes=635584888 weight_to_token_ratio=0.195212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344224 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059936 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775648 weight_movement_bytes=34603008 total_nvlink_bytes=566378656 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344016 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739664 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135312 weight_movement_bytes=69206016 total_nvlink_bytes=602341328 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=345192 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667112 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532989032 weight_movement_bytes=34603008 total_nvlink_bytes=567592040 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344120 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862648 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533381176 weight_movement_bytes=103809024 total_nvlink_bytes=637190200 weight_to_token_ratio=0.194624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344488 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764712 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184936 weight_movement_bytes=34603008 total_nvlink_bytes=567787944 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=20996 dedup_ratio=0.243754 router_tokens_per_expert=[747, 754, 759, 756, 706, 790, 817, 729, 747, 735, 756, 765, 720, 766, 738, 755, 813, 807, 763, 727, 794, 759, 791, 787, 776, 834, 763, 760, 800, 800, 760, 740, 726, 812, 755, 767, 708, 790, 773, 775, 770, 790, 784, 768, 785, 786, 725, 788, 794, 775, 765, 722, 798, 820, 751, 791, 766, 749, 757, 785, 774, 757, 744, 788] payload_rows_per_dst=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] dup_rows_per_dst=[2966, 2812, 3294, 2942, 2996, 2990, 3040, 2922] combine_rows_per_src=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65016 dup_rows=24120 dup_rows_remote=21102 dedup_ratio=0.245361 router_tokens_per_expert=[808, 778, 822, 780, 772, 761, 766, 770, 796, 764, 764, 773, 739, 765, 745, 747, 802, 780, 747, 788, 750, 750, 777, 733, 744, 741, 793, 789, 726, 775, 740, 722, 770, 747, 761, 770, 803, 772, 795, 780, 761, 780, 768, 748, 794, 767, 794, 792, 760, 764, 748, 805, 778, 796, 793, 739, 795, 733, 746, 763, 726, 732, 795, 770] payload_rows_per_dst=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] dup_rows_per_dst=[3052, 3018, 3048, 2914, 3040, 3062, 3040, 2946] combine_rows_per_src=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64672 dup_rows=24300 dup_rows_remote=21186 dedup_ratio=0.247192 router_tokens_per_expert=[746, 804, 750, 715, 757, 759, 767, 801, 809, 750, 768, 779, 819, 760, 773, 787, 765, 712, 805, 761, 775, 759, 773, 748, 749, 773, 762, 806, 744, 756, 761, 727, 759, 741, 791, 758, 836, 752, 786, 800, 740, 786, 781, 800, 804, 766, 741, 788, 761, 785, 789, 767, 755, 720, 744, 752, 792, 739, 745, 762, 775, 762, 786, 769] payload_rows_per_dst=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] dup_rows_per_dst=[2998, 3230, 3000, 2984, 3114, 3024, 2856, 3094] combine_rows_per_src=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65134 dup_rows=23730 dup_rows_remote=20696 dedup_ratio=0.241394 router_tokens_per_expert=[785, 796, 782, 755, 763, 795, 772, 742, 711, 731, 761, 736, 790, 750, 767, 767, 774, 795, 814, 787, 755, 770, 799, 792, 767, 721, 763, 754, 770, 734, 806, 810, 753, 757, 778, 804, 756, 769, 732, 773, 767, 794, 757, 773, 746, 746, 763, 792, 728, 802, 773, 768, 752, 783, 765, 744, 761, 754, 769, 769, 753, 751, 794, 812] payload_rows_per_dst=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] dup_rows_per_dst=[2946, 2836, 2654, 3142, 3020, 3022, 3076, 3034] combine_rows_per_src=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344208 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813584 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282960 weight_movement_bytes=34603008 total_nvlink_bytes=567885968 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65016 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[765, 791, 734, 711, 755, 780, 742, 712, 749, 797, 771, 798, 759, 777, 767, 781, 766, 829, 768, 779, 743, 799, 810, 788, 805, 780, 828, 781, 760, 779, 761, 714, 768, 792, 749, 804, 711, 755, 758, 772, 787, 802, 726, 755, 731, 743, 781, 804, 768, 780, 835, 745, 770, 781, 741, 747, 783, 732, 754, 771, 744, 700, 780, 804] payload_rows_per_dst=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] dup_rows_per_dst=[2802, 3086, 3184, 3070, 2932, 3004, 3002, 2876] combine_rows_per_src=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64702 dup_rows=24188 dup_rows_remote=21198 dedup_ratio=0.246053 router_tokens_per_expert=[757, 752, 736, 791, 797, 771, 776, 733, 712, 760, 778, 787, 772, 766, 748, 818, 784, 764, 769, 804, 764, 788, 724, 805, 738, 787, 739, 820, 809, 730, 805, 740, 755, 791, 749, 754, 715, 792, 735, 805, 757, 792, 815, 739, 749, 727, 760, 774, 780, 766, 792, 777, 760, 775, 783, 769, 731, 747, 783, 750, 759, 806, 769, 772] payload_rows_per_dst=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] dup_rows_per_dst=[2992, 3030, 2990, 2992, 2968, 3090, 3074, 3052] combine_rows_per_src=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344472 combine_payload_bytes=266305536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266650008 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955544 weight_movement_bytes=69206016 total_nvlink_bytes=602161560 weight_to_token_ratio=0.129853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64932 dup_rows=24078 dup_rows_remote=20960 dedup_ratio=0.244934 router_tokens_per_expert=[758, 779, 788, 809, 732, 787, 817, 779, 791, 728, 756, 777, 784, 730, 762, 805, 744, 817, 799, 783, 806, 767, 758, 765, 786, 794, 831, 759, 777, 782, 751, 726, 793, 744, 747, 725, 739, 769, 758, 743, 799, 779, 735, 774, 746, 740, 770, 755, 766, 719, 768, 735, 757, 779, 712, 788, 758, 754, 732, 756, 805, 785, 774, 821] payload_rows_per_dst=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] dup_rows_per_dst=[3120, 2982, 3088, 3118, 2860, 3046, 2804, 3060] combine_rows_per_src=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343432 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239944 ingress_bytes=299499520 link_bytes=299499520 token_bytes=530136456 weight_movement_bytes=34603008 total_nvlink_bytes=564739464 weight_to_token_ratio=0.065272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126992582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65040 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[752, 777, 817, 735, 760, 791, 739, 788, 760, 733, 795, 776, 788, 803, 778, 756, 774, 766, 750, 817, 768, 773, 779, 769, 761, 782, 720, 768, 782, 737, 783, 746, 783, 783, 751, 727, 734, 801, 780, 773, 781, 789, 764, 726, 749, 757, 787, 770, 744, 736, 768, 739, 785, 753, 775, 738, 758, 826, 764, 812, 810, 761, 740, 765] payload_rows_per_dst=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] dup_rows_per_dst=[2966, 3068, 2950, 3004, 3004, 2994, 2948, 3078] combine_rows_per_src=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343320 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132184 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921048 weight_movement_bytes=34603008 total_nvlink_bytes=568524056 weight_to_token_ratio=0.064809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=343600 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265362992 ingress_bytes=368828416 link_bytes=368828416 token_bytes=530382384 weight_movement_bytes=103809024 total_nvlink_bytes=634191408 weight_to_token_ratio=0.195725 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387871636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343568 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305040 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266512 weight_movement_bytes=34603008 total_nvlink_bytes=566869520 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344232 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748072 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151912 weight_movement_bytes=34603008 total_nvlink_bytes=567754920 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64990 dup_rows=24116 dup_rows_remote=21010 dedup_ratio=0.245321 router_tokens_per_expert=[794, 709, 763, 743, 794, 743, 753, 765, 793, 760, 758, 811, 790, 756, 772, 712, 771, 757, 771, 755, 777, 826, 785, 774, 745, 742, 706, 808, 781, 798, 763, 758, 702, 792, 764, 803, 766, 754, 778, 753, 732, 740, 794, 775, 741, 792, 772, 786, 770, 770, 838, 771, 744, 775, 772, 805, 762, 742, 796, 760, 753, 765, 748, 804] payload_rows_per_dst=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] dup_rows_per_dst=[2942, 3106, 3134, 2984, 2928, 3000, 3090, 2932] combine_rows_per_src=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64888 dup_rows=24040 dup_rows_remote=20990 dedup_ratio=0.244548 router_tokens_per_expert=[792, 736, 790, 777, 797, 779, 785, 757, 729, 749, 799, 749, 759, 764, 802, 807, 791, 787, 717, 742, 802, 721, 767, 795, 704, 748, 771, 738, 765, 756, 765, 769, 790, 805, 781, 802, 809, 713, 735, 818, 773, 762, 760, 767, 779, 790, 756, 710, 758, 801, 812, 728, 790, 775, 729, 781, 744, 739, 735, 809, 782, 746, 765, 799] payload_rows_per_dst=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] dup_rows_per_dst=[3050, 3024, 2978, 2930, 3130, 2924, 3054, 2950] combine_rows_per_src=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64890 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[775, 791, 722, 741, 722, 763, 775, 826, 740, 805, 773, 784, 743, 763, 779, 775, 753, 797, 758, 820, 777, 801, 732, 807, 767, 767, 775, 790, 767, 726, 749, 764, 764, 781, 745, 712, 739, 808, 775, 787, 726, 772, 771, 772, 822, 750, 777, 788, 774, 744, 760, 741, 800, 777, 718, 782, 772, 753, 754, 776, 795, 770, 753, 767] payload_rows_per_dst=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] dup_rows_per_dst=[2996, 3022, 3152, 3024, 2956, 3116, 2912, 2970] combine_rows_per_src=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65104 dup_rows=24018 dup_rows_remote=20974 dedup_ratio=0.244324 router_tokens_per_expert=[763, 776, 752, 788, 713, 789, 794, 760, 748, 737, 782, 795, 754, 755, 767, 793, 749, 802, 777, 801, 777, 785, 793, 785, 749, 768, 772, 753, 768, 760, 737, 766, 733, 774, 766, 799, 741, 740, 764, 789, 768, 785, 785, 795, 698, 765, 789, 779, 786, 774, 802, 771, 752, 760, 755, 761, 775, 752, 775, 758, 761, 758, 747, 787] payload_rows_per_dst=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] dup_rows_per_dst=[2988, 3076, 2874, 2990, 2992, 3062, 2992, 3044] combine_rows_per_src=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344000 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543040 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742080 weight_movement_bytes=34603008 total_nvlink_bytes=567345088 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65232 dup_rows=23760 dup_rows_remote=20876 dedup_ratio=0.241699 router_tokens_per_expert=[770, 766, 757, 787, 764, 728, 764, 786, 724, 713, 756, 750, 810, 752, 772, 769, 743, 785, 796, 752, 810, 774, 783, 729, 767, 739, 789, 745, 722, 841, 763, 742, 794, 801, 782, 706, 777, 771, 751, 773, 798, 727, 780, 758, 760, 805, 757, 792, 726, 760, 777, 760, 761, 760, 791, 763, 816, 793, 780, 770, 790, 778, 757, 790] payload_rows_per_dst=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] dup_rows_per_dst=[2998, 2820, 3090, 2926, 3066, 2956, 2884, 3020] combine_rows_per_src=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65250 dup_rows=23726 dup_rows_remote=20834 dedup_ratio=0.241353 router_tokens_per_expert=[787, 754, 769, 760, 760, 786, 783, 780, 838, 748, 744, 747, 737, 767, 807, 791, 761, 709, 798, 760, 806, 727, 794, 755, 750, 762, 737, 748, 808, 836, 735, 791, 731, 730, 724, 798, 760, 787, 756, 816, 760, 790, 742, 790, 795, 739, 749, 754, 783, 751, 785, 726, 764, 750, 816, 732, 807, 775, 802, 771, 784, 752, 712, 786] payload_rows_per_dst=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] dup_rows_per_dst=[3010, 2960, 2892, 2976, 2932, 3000, 2884, 3072] combine_rows_per_src=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344328 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133768 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531923208 weight_movement_bytes=34603008 total_nvlink_bytes=566526216 weight_to_token_ratio=0.065053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=65002 dup_rows=24190 dup_rows_remote=21258 dedup_ratio=0.246073 router_tokens_per_expert=[770, 719, 773, 798, 763, 766, 762, 747, 759, 793, 761, 736, 728, 720, 756, 790, 819, 789, 762, 778, 813, 812, 808, 734, 770, 679, 801, 784, 701, 745, 733, 809, 811, 812, 785, 773, 796, 739, 796, 771, 788, 814, 790, 777, 802, 775, 761, 791, 805, 731, 761, 754, 782, 767, 767, 751, 742, 755, 750, 762, 775, 737, 733, 721] payload_rows_per_dst=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] dup_rows_per_dst=[2990, 2920, 3198, 2932, 3168, 3198, 2942, 2842] combine_rows_per_src=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343512 combine_payload_bytes=265781248 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266124760 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906008 weight_movement_bytes=69206016 total_nvlink_bytes=601112024 weight_to_token_ratio=0.130109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65014 dup_rows=24096 dup_rows_remote=21134 dedup_ratio=0.245117 router_tokens_per_expert=[762, 777, 687, 758, 778, 757, 782, 781, 784, 772, 765, 745, 760, 776, 779, 790, 786, 789, 821, 774, 748, 760, 790, 781, 768, 778, 771, 765, 756, 756, 783, 767, 761, 805, 748, 790, 766, 740, 756, 771, 776, 743, 775, 777, 729, 768, 786, 724, 757, 758, 732, 730, 770, 767, 752, 763, 825, 796, 766, 771, 776, 777, 744, 807] payload_rows_per_dst=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] dup_rows_per_dst=[2972, 3032, 3116, 2946, 3012, 2962, 2916, 3140] combine_rows_per_src=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344312 combine_payload_bytes=266665984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267010296 ingress_bytes=370475008 link_bytes=370475008 token_bytes=533676280 weight_movement_bytes=103809024 total_nvlink_bytes=637485304 weight_to_token_ratio=0.194517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394067629e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344432 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534704 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724976 weight_movement_bytes=34603008 total_nvlink_bytes=569327984 weight_to_token_ratio=0.064712 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344336 combine_payload_bytes=267264000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267608336 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872336 weight_movement_bytes=69206016 total_nvlink_bytes=604078352 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=345040 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266593232 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841424 weight_movement_bytes=34603008 total_nvlink_bytes=567444432 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=344592 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266641936 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532939280 weight_movement_bytes=69206016 total_nvlink_bytes=602145296 weight_to_token_ratio=0.129857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=21102 dedup_ratio=0.244649 router_tokens_per_expert=[753, 771, 807, 758, 735, 767, 737, 804, 760, 792, 786, 780, 747, 770, 766, 746, 776, 758, 754, 740, 735, 763, 798, 742, 785, 743, 774, 800, 753, 755, 714, 746, 763, 766, 742, 817, 791, 756, 783, 777, 777, 756, 760, 763, 770, 745, 756, 756, 789, 767, 796, 749, 736, 775, 784, 781, 783, 797, 742, 777, 830, 803, 744, 806] payload_rows_per_dst=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] dup_rows_per_dst=[3086, 2948, 2924, 2952, 3098, 2918, 3058, 3066] combine_rows_per_src=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65162 dup_rows=23784 dup_rows_remote=20680 dedup_ratio=0.241943 router_tokens_per_expert=[761, 782, 746, 736, 811, 813, 771, 764, 731, 758, 736, 776, 773, 718, 758, 785, 761, 827, 771, 814, 763, 714, 756, 826, 749, 755, 760, 760, 757, 753, 720, 719, 733, 777, 767, 775, 757, 794, 783, 845, 777, 738, 784, 761, 824, 752, 780, 763, 792, 801, 752, 806, 753, 761, 801, 776, 775, 747, 748, 724, 784, 749, 792, 757] payload_rows_per_dst=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] dup_rows_per_dst=[3028, 2910, 3014, 2812, 3104, 3070, 3014, 2832] combine_rows_per_src=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64586 dup_rows=24272 dup_rows_remote=21004 dedup_ratio=0.246908 router_tokens_per_expert=[728, 728, 799, 717, 835, 714, 758, 726, 782, 793, 815, 759, 776, 791, 779, 768, 780, 801, 798, 740, 765, 742, 723, 735, 747, 776, 759, 755, 801, 771, 764, 736, 729, 773, 758, 779, 730, 752, 737, 793, 755, 768, 784, 773, 808, 765, 756, 764, 764, 837, 749, 793, 795, 800, 729, 823, 773, 751, 789, 754, 799, 753, 748, 810] payload_rows_per_dst=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] dup_rows_per_dst=[3268, 3222, 2570, 3194, 3064, 2826, 3132, 2996] combine_rows_per_src=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344040 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207208 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070376 weight_movement_bytes=69206016 total_nvlink_bytes=601276392 weight_to_token_ratio=0.130069 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21130 dedup_ratio=0.245524 router_tokens_per_expert=[771, 748, 765, 740, 763, 736, 761, 773, 798, 824, 770, 795, 784, 761, 725, 799, 767, 790, 804, 768, 754, 783, 780, 745, 739, 785, 767, 725, 775, 781, 767, 802, 750, 791, 738, 730, 813, 749, 783, 784, 814, 827, 803, 761, 759, 746, 758, 768, 739, 675, 765, 749, 788, 739, 737, 762, 740, 801, 813, 736, 739, 796, 809, 745] payload_rows_per_dst=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] dup_rows_per_dst=[2746, 2906, 3020, 3384, 2972, 3156, 2946, 3006] combine_rows_per_src=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73902 payload_rows_remote=64770 dup_rows=24402 dup_rows_remote=21424 dedup_ratio=0.248230 router_tokens_per_expert=[759, 735, 780, 785, 811, 714, 756, 774, 792, 780, 771, 811, 799, 755, 768, 750, 799, 807, 774, 815, 747, 786, 773, 720, 790, 734, 774, 818, 730, 806, 729, 778, 802, 768, 748, 753, 774, 733, 779, 787, 768, 737, 772, 787, 750, 744, 757, 766, 771, 745, 755, 755, 736, 779, 730, 784, 748, 826, 797, 777, 734, 763, 739, 768] payload_rows_per_dst=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] dup_rows_per_dst=[2986, 3132, 3166, 3064, 3080, 2948, 2978, 3048] combine_rows_per_src=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343368 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267246920 ingress_bytes=405315584 link_bytes=405315584 token_bytes=534150472 weight_movement_bytes=138412032 total_nvlink_bytes=672562504 weight_to_token_ratio=0.259126 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525169912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=65012 dup_rows=24176 dup_rows_remote=21192 dedup_ratio=0.245931 router_tokens_per_expert=[738, 809, 780, 794, 791, 809, 787, 794, 788, 771, 791, 716, 745, 808, 824, 720, 788, 734, 741, 766, 771, 730, 769, 751, 730, 731, 761, 806, 772, 793, 774, 737, 766, 755, 747, 806, 784, 766, 797, 771, 786, 746, 794, 675, 789, 791, 803, 751, 779, 775, 837, 749, 711, 723, 758, 783, 792, 773, 715, 809, 796, 752, 709, 745] payload_rows_per_dst=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] dup_rows_per_dst=[3272, 2978, 2984, 2906, 3106, 3032, 3024, 2874] combine_rows_per_src=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65112 dup_rows=23906 dup_rows_remote=20754 dedup_ratio=0.243184 router_tokens_per_expert=[826, 792, 736, 787, 816, 814, 749, 792, 772, 833, 778, 758, 774, 746, 767, 729, 808, 744, 761, 756, 750, 730, 777, 797, 775, 793, 780, 778, 827, 724, 746, 796, 713, 750, 758, 783, 752, 774, 807, 772, 791, 742, 745, 742, 750, 769, 765, 772, 809, 749, 789, 782, 733, 775, 738, 754, 733, 761, 764, 748, 792, 755, 689, 785] payload_rows_per_dst=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] dup_rows_per_dst=[3036, 3102, 2832, 3152, 2988, 2946, 2978, 2872] combine_rows_per_src=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=342360 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886616 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430872 weight_movement_bytes=34603008 total_nvlink_bytes=564033880 weight_to_token_ratio=0.065359 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64876 dup_rows=24018 dup_rows_remote=20934 dedup_ratio=0.244324 router_tokens_per_expert=[784, 791, 748, 747, 769, 749, 787, 771, 754, 765, 823, 791, 713, 737, 824, 774, 704, 761, 810, 756, 757, 829, 754, 753, 727, 735, 773, 777, 747, 752, 737, 779, 779, 782, 734, 790, 734, 824, 809, 720, 765, 802, 766, 784, 793, 788, 788, 761, 784, 736, 780, 771, 753, 737, 793, 759, 781, 736, 710, 791, 732, 797, 817, 778] payload_rows_per_dst=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] dup_rows_per_dst=[2950, 2996, 2998, 2928, 3050, 3084, 3084, 2928] combine_rows_per_src=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343680 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723520 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103360 weight_movement_bytes=34603008 total_nvlink_bytes=565706368 weight_to_token_ratio=0.065153 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344776 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265642696 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530940616 weight_movement_bytes=34603008 total_nvlink_bytes=565543624 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65150 dup_rows=23842 dup_rows_remote=20886 dedup_ratio=0.242533 router_tokens_per_expert=[757, 795, 802, 717, 759, 783, 710, 749, 727, 745, 748, 786, 737, 805, 792, 794, 800, 797, 711, 723, 765, 779, 763, 778, 779, 808, 781, 739, 758, 745, 843, 781, 811, 786, 772, 762, 777, 748, 804, 748, 771, 726, 767, 799, 769, 776, 776, 764, 765, 775, 756, 757, 738, 727, 776, 772, 760, 788, 747, 754, 763, 811, 777, 774] payload_rows_per_dst=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] dup_rows_per_dst=[2950, 2956, 2932, 3078, 3014, 2974, 2936, 3002] combine_rows_per_src=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343464 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042216 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533740968 weight_movement_bytes=34603008 total_nvlink_bytes=568343976 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344816 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633968 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532923120 weight_movement_bytes=34603008 total_nvlink_bytes=567526128 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343240 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075336 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807432 weight_movement_bytes=69206016 total_nvlink_bytes=601013448 weight_to_token_ratio=0.130134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65220 dup_rows=23794 dup_rows_remote=20860 dedup_ratio=0.242045 router_tokens_per_expert=[742, 769, 783, 795, 817, 795, 777, 793, 746, 765, 757, 744, 782, 790, 791, 754, 790, 755, 806, 740, 767, 762, 720, 727, 775, 744, 787, 729, 767, 781, 710, 793, 745, 777, 776, 762, 820, 729, 736, 767, 761, 781, 757, 811, 771, 784, 766, 778, 769, 767, 801, 771, 769, 811, 783, 782, 763, 774, 724, 783, 762, 750, 757, 712] payload_rows_per_dst=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] dup_rows_per_dst=[3124, 2932, 2950, 2882, 2934, 2996, 3092, 2884] combine_rows_per_src=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344144 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267198544 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534052944 weight_movement_bytes=34603008 total_nvlink_bytes=568655952 weight_to_token_ratio=0.064793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65010 dup_rows=24040 dup_rows_remote=21066 dedup_ratio=0.244548 router_tokens_per_expert=[777, 754, 768, 775, 773, 717, 780, 764, 724, 797, 806, 708, 774, 791, 780, 834, 739, 786, 719, 768, 809, 763, 754, 781, 725, 792, 760, 745, 738, 779, 727, 757, 726, 764, 757, 748, 764, 786, 745, 792, 775, 786, 826, 780, 773, 729, 737, 723, 783, 799, 807, 773, 747, 822, 755, 801, 768, 813, 764, 791, 767, 750, 756, 781] payload_rows_per_dst=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] dup_rows_per_dst=[2974, 3002, 3068, 2928, 2920, 3012, 3114, 3022] combine_rows_per_src=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344320 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267485440 ingress_bytes=336347136 link_bytes=336347136 token_bytes=534626560 weight_movement_bytes=69206016 total_nvlink_bytes=603832576 weight_to_token_ratio=0.129447 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265647195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64750 dup_rows=24058 dup_rows_remote=20918 dedup_ratio=0.244731 router_tokens_per_expert=[748, 734, 819, 781, 732, 763, 773, 771, 744, 817, 735, 760, 768, 774, 769, 772, 750, 784, 737, 721, 781, 760, 771, 740, 795, 771, 765, 772, 748, 748, 775, 754, 771, 761, 762, 794, 746, 814, 770, 763, 785, 782, 763, 729, 774, 752, 758, 826, 801, 788, 769, 743, 743, 780, 759, 758, 824, 785, 748, 831, 781, 732, 761, 767] payload_rows_per_dst=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] dup_rows_per_dst=[2948, 3000, 2892, 2920, 3052, 3110, 2996, 3140] combine_rows_per_src=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65182 dup_rows=23796 dup_rows_remote=20850 dedup_ratio=0.242065 router_tokens_per_expert=[806, 796, 815, 758, 773, 779, 716, 740, 780, 756, 809, 783, 743, 799, 726, 822, 778, 802, 759, 779, 778, 672, 776, 744, 808, 753, 764, 778, 775, 764, 758, 764, 768, 737, 768, 752, 719, 764, 764, 762, 787, 787, 725, 756, 794, 786, 768, 803, 779, 749, 762, 729, 743, 794, 814, 766, 793, 788, 732, 787, 767, 771, 730, 755] payload_rows_per_dst=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] dup_rows_per_dst=[2976, 3094, 2982, 2964, 2838, 3022, 2946, 2974] combine_rows_per_src=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64928 dup_rows=23902 dup_rows_remote=20828 dedup_ratio=0.243144 router_tokens_per_expert=[756, 727, 742, 732, 790, 716, 773, 792, 732, 749, 768, 752, 762, 738, 769, 777, 773, 746, 753, 764, 785, 779, 777, 753, 761, 771, 806, 776, 809, 821, 732, 798, 787, 752, 769, 770, 800, 762, 767, 805, 714, 726, 795, 759, 781, 806, 780, 778, 759, 797, 758, 784, 783, 780, 762, 791, 781, 742, 747, 770, 801, 786, 692, 789] payload_rows_per_dst=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] dup_rows_per_dst=[2820, 2854, 2988, 3074, 3100, 3046, 3068, 2952] combine_rows_per_src=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65078 dup_rows=23976 dup_rows_remote=20960 dedup_ratio=0.243896 router_tokens_per_expert=[734, 759, 762, 753, 762, 787, 752, 767, 759, 758, 748, 763, 816, 775, 735, 739, 787, 817, 757, 748, 770, 739, 742, 773, 800, 774, 758, 801, 808, 779, 743, 787, 749, 730, 777, 764, 753, 764, 803, 760, 807, 790, 729, 782, 740, 774, 780, 788, 772, 719, 775, 789, 771, 839, 757, 744, 781, 776, 774, 757, 739, 760, 787, 770] payload_rows_per_dst=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] dup_rows_per_dst=[2920, 2936, 3016, 3116, 2914, 3030, 3044, 3000] combine_rows_per_src=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64792 dup_rows=24218 dup_rows_remote=21114 dedup_ratio=0.246358 router_tokens_per_expert=[732, 788, 754, 789, 777, 763, 757, 768, 721, 758, 764, 782, 818, 832, 776, 771, 794, 766, 762, 735, 733, 774, 751, 745, 722, 761, 775, 809, 757, 772, 761, 737, 798, 813, 733, 747, 746, 798, 807, 786, 762, 778, 757, 817, 730, 806, 804, 745, 758, 738, 792, 730, 810, 747, 831, 773, 736, 752, 743, 737, 755, 711, 779, 829] payload_rows_per_dst=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] dup_rows_per_dst=[2998, 3048, 2990, 3072, 3108, 3104, 2982, 2916] combine_rows_per_src=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344304 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266625264 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532906224 weight_movement_bytes=69206016 total_nvlink_bytes=602112240 weight_to_token_ratio=0.129865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=342672 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265558672 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530774672 weight_movement_bytes=34603008 total_nvlink_bytes=565377680 weight_to_token_ratio=0.065193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344128 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267329600 ingress_bytes=405397504 link_bytes=405397504 token_bytes=534315072 weight_movement_bytes=138412032 total_nvlink_bytes=672727104 weight_to_token_ratio=0.259046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525478170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21004 dedup_ratio=0.244629 router_tokens_per_expert=[766, 755, 768, 776, 748, 815, 788, 814, 764, 802, 797, 773, 749, 773, 799, 739, 729, 783, 744, 782, 768, 784, 714, 734, 744, 816, 775, 774, 786, 716, 746, 767, 829, 809, 701, 760, 776, 781, 784, 793, 779, 749, 757, 812, 786, 781, 759, 753, 718, 715, 708, 732, 773, 765, 788, 747, 788, 774, 784, 777, 811, 796, 729, 750] payload_rows_per_dst=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] dup_rows_per_dst=[3100, 3044, 2862, 2936, 3128, 3026, 2826, 3126] combine_rows_per_src=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343024 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288112 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233200 weight_movement_bytes=69206016 total_nvlink_bytes=601439216 weight_to_token_ratio=0.130029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344152 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266903640 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463128 weight_movement_bytes=34603008 total_nvlink_bytes=568066136 weight_to_token_ratio=0.064865 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343624 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265731656 ingress_bytes=334594048 link_bytes=334594048 token_bytes=531119688 weight_movement_bytes=69206016 total_nvlink_bytes=600325704 weight_to_token_ratio=0.130302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259050466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64974 dup_rows=24166 dup_rows_remote=21244 dedup_ratio=0.245829 router_tokens_per_expert=[750, 760, 790, 783, 779, 746, 772, 798, 771, 786, 745, 767, 784, 788, 787, 759, 717, 766, 760, 777, 750, 778, 755, 771, 781, 753, 771, 748, 797, 723, 780, 752, 711, 749, 744, 806, 749, 771, 749, 764, 766, 786, 780, 767, 753, 791, 748, 768, 762, 772, 782, 783, 789, 776, 790, 790, 798, 747, 800, 760, 746, 782, 780, 749] payload_rows_per_dst=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] dup_rows_per_dst=[3088, 3046, 2948, 2988, 2922, 3014, 3138, 3022] combine_rows_per_src=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206816 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069984 weight_movement_bytes=34603008 total_nvlink_bytes=566672992 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344872 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478376 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611880 weight_movement_bytes=34603008 total_nvlink_bytes=567214888 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65032 dup_rows=24006 dup_rows_remote=20846 dedup_ratio=0.244202 router_tokens_per_expert=[773, 812, 740, 771, 736, 737, 815, 787, 795, 705, 805, 729, 750, 774, 758, 766, 776, 767, 739, 779, 744, 751, 746, 805, 753, 752, 825, 788, 737, 755, 749, 776, 722, 778, 738, 759, 770, 787, 804, 761, 761, 769, 747, 745, 814, 808, 801, 800, 733, 767, 755, 786, 788, 760, 777, 736, 751, 771, 757, 771, 778, 738, 829, 796] payload_rows_per_dst=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] dup_rows_per_dst=[2686, 2890, 2972, 2690, 2940, 3230, 3438, 3160] combine_rows_per_src=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64810 dup_rows=24110 dup_rows_remote=20948 dedup_ratio=0.245260 router_tokens_per_expert=[779, 787, 739, 780, 743, 748, 771, 710, 745, 722, 769, 804, 740, 772, 809, 780, 787, 758, 789, 783, 753, 810, 800, 802, 765, 758, 797, 751, 763, 760, 760, 760, 803, 788, 751, 768, 764, 775, 745, 766, 775, 789, 775, 793, 753, 742, 765, 713, 827, 721, 775, 722, 723, 786, 755, 766, 799, 787, 795, 788, 748, 748, 756, 797] payload_rows_per_dst=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] dup_rows_per_dst=[3162, 3102, 3310, 2770, 2942, 2826, 2892, 3106] combine_rows_per_src=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20968 dedup_ratio=0.243184 router_tokens_per_expert=[754, 752, 769, 756, 772, 732, 755, 795, 734, 716, 721, 717, 739, 753, 772, 791, 804, 759, 772, 805, 753, 814, 768, 722, 775, 742, 786, 809, 808, 819, 792, 778, 789, 736, 766, 787, 815, 805, 745, 802, 729, 787, 761, 750, 744, 784, 758, 792, 755, 792, 795, 752, 743, 726, 786, 765, 772, 752, 780, 770, 796, 777, 746, 761] payload_rows_per_dst=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] dup_rows_per_dst=[2934, 2798, 3002, 3110, 3178, 2948, 2938, 2998] combine_rows_per_src=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64818 dup_rows=24060 dup_rows_remote=21078 dedup_ratio=0.244751 router_tokens_per_expert=[736, 776, 763, 799, 757, 747, 760, 775, 748, 808, 807, 764, 776, 769, 745, 812, 740, 732, 821, 741, 815, 757, 786, 763, 807, 787, 771, 802, 746, 732, 825, 734, 747, 760, 761, 725, 791, 700, 792, 820, 777, 719, 692, 783, 759, 788, 759, 809, 750, 762, 766, 784, 710, 785, 759, 765, 764, 774, 793, 746, 793, 751, 781, 786] payload_rows_per_dst=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] dup_rows_per_dst=[3046, 3144, 3054, 2982, 2948, 2926, 2944, 3016] combine_rows_per_src=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64766 dup_rows=24218 dup_rows_remote=21112 dedup_ratio=0.246358 router_tokens_per_expert=[774, 763, 774, 762, 779, 782, 747, 799, 786, 762, 769, 818, 791, 778, 724, 770, 780, 766, 791, 762, 804, 796, 761, 753, 754, 783, 774, 767, 773, 807, 779, 722, 792, 775, 726, 756, 723, 824, 778, 777, 766, 778, 741, 792, 789, 728, 734, 746, 738, 778, 762, 821, 768, 750, 757, 769, 763, 745, 796, 748, 782, 711, 706, 783] payload_rows_per_dst=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] dup_rows_per_dst=[3074, 3118, 3106, 2972, 2994, 2962, 3058, 2934] combine_rows_per_src=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65040 dup_rows=24014 dup_rows_remote=21078 dedup_ratio=0.244283 router_tokens_per_expert=[811, 768, 763, 812, 830, 773, 771, 751, 752, 740, 742, 791, 757, 772, 711, 792, 765, 780, 768, 789, 720, 722, 795, 788, 790, 763, 747, 807, 783, 770, 849, 798, 704, 825, 760, 795, 776, 773, 758, 792, 785, 734, 725, 746, 825, 765, 746, 767, 803, 716, 720, 761, 738, 790, 731, 748, 798, 774, 750, 747, 736, 766, 739, 789] payload_rows_per_dst=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] dup_rows_per_dst=[3100, 2930, 2992, 3196, 2990, 2936, 2940, 2930] combine_rows_per_src=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343512 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714584 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085656 weight_movement_bytes=103809024 total_nvlink_bytes=636894680 weight_to_token_ratio=0.194732 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343032 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265804792 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531266552 weight_movement_bytes=34603008 total_nvlink_bytes=565869560 weight_to_token_ratio=0.065133 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64828 dup_rows=24272 dup_rows_remote=21378 dedup_ratio=0.246908 router_tokens_per_expert=[751, 794, 792, 779, 782, 800, 751, 721, 787, 730, 746, 726, 773, 745, 768, 774, 798, 821, 763, 755, 747, 794, 745, 728, 808, 766, 814, 804, 862, 793, 792, 773, 745, 776, 747, 720, 787, 760, 778, 783, 785, 772, 748, 743, 790, 767, 731, 754, 808, 734, 775, 732, 761, 804, 773, 782, 763, 757, 774, 790, 730, 752, 723, 726] payload_rows_per_dst=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] dup_rows_per_dst=[3084, 2894, 3044, 3300, 3048, 3014, 3072, 2816] combine_rows_per_src=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344304 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026672 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533709040 weight_movement_bytes=34603008 total_nvlink_bytes=568312048 weight_to_token_ratio=0.064835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343584 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838112 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332640 weight_movement_bytes=69206016 total_nvlink_bytes=600538656 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343512 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265625048 ingress_bytes=299884544 link_bytes=299884544 token_bytes=530906584 weight_movement_bytes=34603008 total_nvlink_bytes=565509592 weight_to_token_ratio=0.065177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128441396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344472 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748312 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533152152 weight_movement_bytes=34603008 total_nvlink_bytes=567755160 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64946 dup_rows=24118 dup_rows_remote=21190 dedup_ratio=0.245341 router_tokens_per_expert=[782, 778, 753, 814, 750, 789, 789, 765, 753, 776, 745, 762, 767, 774, 760, 801, 772, 761, 750, 776, 774, 755, 785, 724, 779, 764, 772, 742, 758, 762, 740, 739, 771, 750, 785, 793, 712, 724, 780, 769, 809, 808, 765, 785, 756, 816, 696, 722, 784, 780, 767, 806, 792, 783, 781, 790, 718, 729, 735, 787, 776, 783, 771, 818] payload_rows_per_dst=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] dup_rows_per_dst=[3096, 3080, 2904, 2910, 2928, 3126, 3104, 2970] combine_rows_per_src=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344824 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880312 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415800 weight_movement_bytes=34603008 total_nvlink_bytes=566018808 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344544 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266363360 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532382176 weight_movement_bytes=34603008 total_nvlink_bytes=566985184 weight_to_token_ratio=0.064997 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=64664 dup_rows=23850 dup_rows_remote=20458 dedup_ratio=0.242615 router_tokens_per_expert=[755, 736, 770, 806, 778, 700, 747, 811, 784, 748, 822, 784, 707, 758, 753, 763, 756, 823, 780, 783, 760, 733, 738, 740, 773, 748, 799, 796, 778, 719, 799, 794, 771, 757, 732, 744, 790, 745, 746, 754, 819, 790, 821, 786, 771, 746, 765, 744, 795, 756, 730, 781, 792, 782, 792, 751, 751, 776, 754, 777, 784, 780, 761, 768] payload_rows_per_dst=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] dup_rows_per_dst=[2904, 2982, 2986, 2996, 2870, 2678, 3042, 3392] combine_rows_per_src=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64958 dup_rows=24056 dup_rows_remote=20898 dedup_ratio=0.244710 router_tokens_per_expert=[772, 753, 762, 796, 761, 800, 774, 806, 764, 798, 762, 800, 750, 783, 755, 786, 762, 755, 759, 767, 750, 760, 739, 801, 810, 786, 759, 772, 811, 734, 751, 746, 744, 740, 804, 762, 723, 761, 751, 784, 746, 748, 712, 731, 795, 790, 766, 768, 791, 735, 745, 801, 762, 734, 733, 739, 780, 781, 781, 815, 762, 790, 795, 799] payload_rows_per_dst=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] dup_rows_per_dst=[3158, 3114, 3118, 2982, 2988, 2834, 2940, 2922] combine_rows_per_src=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65110 dup_rows=23848 dup_rows_remote=20938 dedup_ratio=0.242594 router_tokens_per_expert=[760, 714, 759, 737, 778, 770, 790, 771, 700, 769, 765, 787, 741, 776, 800, 729, 763, 777, 762, 750, 769, 772, 774, 792, 804, 756, 755, 741, 783, 754, 728, 807, 824, 768, 755, 764, 780, 770, 787, 740, 783, 765, 802, 807, 786, 760, 775, 802, 733, 781, 804, 804, 763, 789, 775, 724, 779, 770, 752, 760, 733, 798, 758, 728] payload_rows_per_dst=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] dup_rows_per_dst=[2876, 2944, 2994, 2910, 3108, 3118, 2966, 2932] combine_rows_per_src=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65082 dup_rows=23894 dup_rows_remote=21030 dedup_ratio=0.243062 router_tokens_per_expert=[759, 771, 743, 757, 749, 769, 772, 764, 766, 781, 768, 753, 768, 771, 746, 771, 800, 771, 731, 720, 767, 748, 733, 753, 770, 763, 730, 791, 786, 810, 782, 774, 771, 780, 784, 776, 806, 782, 743, 756, 817, 747, 768, 733, 755, 724, 817, 764, 762, 763, 736, 777, 767, 788, 743, 760, 764, 777, 728, 841, 789, 791, 786, 820] payload_rows_per_dst=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] dup_rows_per_dst=[2972, 3008, 2924, 2968, 3034, 2954, 2864, 3170] combine_rows_per_src=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65104 dup_rows=23920 dup_rows_remote=20986 dedup_ratio=0.243327 router_tokens_per_expert=[748, 781, 775, 730, 787, 732, 750, 757, 759, 767, 798, 806, 751, 796, 759, 771, 728, 755, 782, 819, 766, 773, 748, 736, 764, 762, 755, 758, 835, 767, 758, 763, 794, 760, 732, 691, 829, 826, 798, 783, 812, 788, 777, 769, 766, 806, 733, 805, 748, 794, 771, 721, 799, 712, 774, 763, 768, 717, 780, 794, 746, 789, 729, 742] payload_rows_per_dst=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] dup_rows_per_dst=[2988, 3074, 2934, 3066, 3070, 3058, 2918, 2812] combine_rows_per_src=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65210 dup_rows=23920 dup_rows_remote=20886 dedup_ratio=0.243327 router_tokens_per_expert=[813, 793, 777, 756, 750, 776, 765, 814, 730, 723, 764, 769, 775, 773, 728, 736, 762, 774, 770, 750, 811, 778, 779, 769, 773, 774, 770, 767, 775, 806, 764, 759, 786, 759, 785, 767, 738, 787, 756, 755, 795, 770, 736, 764, 767, 757, 797, 718, 813, 802, 840, 752, 750, 733, 731, 829, 763, 763, 742, 734, 774, 785, 713, 768] payload_rows_per_dst=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] dup_rows_per_dst=[3064, 2856, 3010, 3016, 2916, 3034, 3090, 2934] combine_rows_per_src=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264863744 dispatch_meta_bytes=340488 combine_payload_bytes=264863744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265204232 ingress_bytes=299466752 link_bytes=299466752 token_bytes=530067976 weight_movement_bytes=34603008 total_nvlink_bytes=564670984 weight_to_token_ratio=0.065280 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126869279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64934 dup_rows=24102 dup_rows_remote=21108 dedup_ratio=0.245178 router_tokens_per_expert=[716, 738, 778, 796, 771, 807, 803, 815, 775, 757, 791, 772, 760, 737, 813, 726, 749, 776, 774, 786, 764, 770, 791, 781, 766, 784, 768, 802, 761, 777, 761, 743, 759, 749, 733, 772, 730, 786, 738, 741, 755, 760, 799, 754, 732, 764, 793, 793, 830, 794, 752, 778, 753, 708, 737, 785, 764, 784, 762, 765, 802, 742, 755, 775] payload_rows_per_dst=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] dup_rows_per_dst=[3102, 2994, 3070, 3012, 2920, 2954, 3010, 3040] combine_rows_per_src=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343424 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266411392 ingress_bytes=404480000 link_bytes=404480000 token_bytes=532479360 weight_movement_bytes=138412032 total_nvlink_bytes=670891392 weight_to_token_ratio=0.259939 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522025677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=65014 dup_rows=24038 dup_rows_remote=20862 dedup_ratio=0.244527 router_tokens_per_expert=[746, 800, 779, 759, 745, 790, 764, 744, 785, 780, 737, 788, 742, 770, 756, 782, 774, 785, 782, 768, 773, 756, 770, 797, 788, 728, 717, 747, 707, 774, 743, 785, 724, 785, 805, 776, 771, 762, 786, 805, 729, 781, 716, 750, 791, 792, 820, 743, 732, 783, 721, 762, 822, 772, 832, 805, 766, 772, 774, 756, 770, 735, 783, 770] payload_rows_per_dst=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] dup_rows_per_dst=[2922, 2994, 3054, 2818, 3176, 2908, 3076, 3090] combine_rows_per_src=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344192 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267034752 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725312 weight_movement_bytes=138412032 total_nvlink_bytes=672137344 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344448 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920320 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496192 weight_movement_bytes=69206016 total_nvlink_bytes=602702208 weight_to_token_ratio=0.129722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344360 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010344 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676328 weight_movement_bytes=34603008 total_nvlink_bytes=568279336 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344384 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444544 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544704 weight_movement_bytes=69206016 total_nvlink_bytes=603750720 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343504 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266640848 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938192 weight_movement_bytes=34603008 total_nvlink_bytes=567541200 weight_to_token_ratio=0.064929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344168 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313832 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283496 weight_movement_bytes=103809024 total_nvlink_bytes=636092520 weight_to_token_ratio=0.195026 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=64952 dup_rows=23976 dup_rows_remote=20970 dedup_ratio=0.243896 router_tokens_per_expert=[826, 676, 763, 799, 759, 757, 784, 754, 764, 754, 787, 776, 747, 751, 782, 768, 744, 751, 733, 793, 746, 800, 782, 731, 747, 760, 734, 737, 781, 778, 764, 767, 754, 823, 740, 740, 779, 780, 785, 812, 799, 713, 804, 759, 809, 762, 787, 799, 762, 763, 773, 757, 759, 798, 773, 737, 750, 813, 754, 795, 786, 790, 749, 753] payload_rows_per_dst=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] dup_rows_per_dst=[3006, 2788, 2994, 2914, 3230, 3028, 3010, 3006] combine_rows_per_src=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65210 dup_rows=23870 dup_rows_remote=20974 dedup_ratio=0.242818 router_tokens_per_expert=[749, 775, 792, 753, 791, 792, 788, 765, 766, 789, 789, 765, 811, 788, 784, 773, 760, 752, 806, 782, 773, 749, 762, 731, 775, 757, 745, 782, 757, 753, 751, 740, 779, 747, 764, 769, 710, 794, 721, 772, 766, 769, 798, 769, 749, 760, 732, 729, 788, 741, 733, 804, 790, 812, 768, 803, 768, 755, 764, 797, 755, 768, 745, 788] payload_rows_per_dst=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] dup_rows_per_dst=[3044, 3094, 2972, 2896, 2980, 2812, 3036, 3036] combine_rows_per_src=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65074 dup_rows=23876 dup_rows_remote=20900 dedup_ratio=0.242879 router_tokens_per_expert=[779, 738, 736, 770, 741, 790, 715, 769, 782, 791, 768, 746, 739, 799, 761, 744, 796, 780, 800, 780, 780, 731, 783, 810, 759, 767, 747, 807, 757, 774, 799, 795, 753, 776, 802, 726, 774, 761, 720, 798, 759, 739, 771, 744, 701, 790, 775, 713, 766, 794, 746, 774, 752, 757, 789, 787, 813, 793, 787, 728, 760, 756, 824, 791] payload_rows_per_dst=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] dup_rows_per_dst=[2950, 2980, 2988, 3018, 2962, 2882, 2976, 3120] combine_rows_per_src=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64950 dup_rows=23990 dup_rows_remote=20946 dedup_ratio=0.244039 router_tokens_per_expert=[781, 749, 792, 775, 786, 732, 793, 796, 782, 763, 773, 834, 731, 771, 739, 748, 775, 785, 781, 782, 743, 732, 769, 789, 761, 787, 710, 800, 783, 710, 746, 761, 813, 719, 787, 763, 747, 730, 770, 807, 753, 788, 808, 763, 785, 786, 829, 776, 795, 793, 750, 790, 800, 735, 790, 768, 700, 709, 762, 720, 759, 762, 798, 738] payload_rows_per_dst=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] dup_rows_per_dst=[3044, 3212, 3200, 2830, 2958, 3092, 2814, 2840] combine_rows_per_src=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65312 dup_rows=23880 dup_rows_remote=20934 dedup_ratio=0.242920 router_tokens_per_expert=[779, 752, 756, 754, 745, 784, 766, 762, 815, 807, 790, 743, 795, 743, 777, 807, 734, 755, 786, 733, 747, 752, 768, 754, 776, 750, 768, 753, 746, 790, 802, 820, 764, 752, 761, 766, 734, 812, 757, 770, 787, 767, 743, 714, 781, 744, 751, 726, 775, 750, 794, 807, 786, 796, 800, 774, 763, 767, 745, 803, 774, 736, 794, 750] payload_rows_per_dst=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] dup_rows_per_dst=[2884, 3092, 2946, 2998, 2998, 2864, 3098, 3000] combine_rows_per_src=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65014 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[777, 751, 767, 743, 717, 747, 727, 758, 740, 756, 743, 774, 818, 780, 743, 771, 757, 777, 798, 741, 787, 754, 740, 741, 766, 779, 766, 761, 744, 780, 736, 807, 790, 741, 763, 780, 763, 757, 756, 812, 802, 783, 748, 778, 751, 780, 788, 740, 793, 814, 775, 743, 799, 784, 798, 771, 738, 778, 828, 721, 763, 788, 780, 801] payload_rows_per_dst=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] dup_rows_per_dst=[2852, 3030, 2972, 2912, 2980, 2994, 3192, 3012] combine_rows_per_src=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343688 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266387080 ingress_bytes=369852416 link_bytes=369852416 token_bytes=532430472 weight_movement_bytes=103809024 total_nvlink_bytes=636239496 weight_to_token_ratio=0.194972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391724866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65052 dup_rows=23926 dup_rows_remote=20838 dedup_ratio=0.243388 router_tokens_per_expert=[824, 752, 754, 768, 754, 721, 784, 784, 782, 749, 796, 771, 792, 802, 779, 736, 735, 816, 816, 769, 739, 736, 730, 773, 768, 802, 754, 737, 800, 762, 752, 760, 785, 778, 800, 780, 826, 738, 758, 759, 749, 749, 748, 787, 800, 745, 761, 765, 747, 801, 755, 744, 806, 796, 763, 733, 769, 764, 782, 771, 741, 759, 749, 747] payload_rows_per_dst=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] dup_rows_per_dst=[3002, 3088, 3000, 2940, 3130, 2864, 2962, 2940] combine_rows_per_src=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64992 dup_rows=23998 dup_rows_remote=20908 dedup_ratio=0.244120 router_tokens_per_expert=[714, 778, 716, 762, 757, 789, 774, 781, 749, 715, 713, 756, 778, 780, 812, 766, 749, 787, 765, 739, 814, 737, 786, 808, 735, 826, 772, 781, 768, 778, 787, 781, 794, 749, 774, 784, 789, 774, 783, 755, 773, 753, 783, 767, 774, 755, 763, 748, 746, 781, 754, 772, 748, 761, 754, 756, 813, 809, 754, 811, 767, 740, 773, 742] payload_rows_per_dst=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] dup_rows_per_dst=[2938, 2898, 3106, 3018, 3090, 2976, 2878, 3094] combine_rows_per_src=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344736 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267444896 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534545056 weight_movement_bytes=34603008 total_nvlink_bytes=569148064 weight_to_token_ratio=0.064734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267517952 dispatch_meta_bytes=344984 combine_payload_bytes=267517952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267862936 ingress_bytes=302120960 link_bytes=302120960 token_bytes=535380888 weight_movement_bytes=34603008 total_nvlink_bytes=569983896 weight_to_token_ratio=0.064633 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136856850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343584 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378784 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413984 weight_movement_bytes=69206016 total_nvlink_bytes=601620000 weight_to_token_ratio=0.129985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343896 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266887000 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430104 weight_movement_bytes=103809024 total_nvlink_bytes=637239128 weight_to_token_ratio=0.194607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343856 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641200 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938544 weight_movement_bytes=34603008 total_nvlink_bytes=567541552 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343560 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796552 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249544 weight_movement_bytes=69206016 total_nvlink_bytes=602455560 weight_to_token_ratio=0.129782 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343600 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550832 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758064 weight_movement_bytes=69206016 total_nvlink_bytes=601964080 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64800 dup_rows=24132 dup_rows_remote=21100 dedup_ratio=0.245483 router_tokens_per_expert=[783, 792, 754, 769, 748, 733, 791, 758, 820, 769, 719, 743, 768, 786, 810, 755, 790, 786, 812, 755, 769, 772, 798, 734, 789, 754, 756, 753, 783, 688, 775, 747, 756, 720, 743, 772, 758, 811, 737, 806, 738, 784, 732, 752, 780, 721, 773, 806, 788, 758, 747, 780, 783, 778, 786, 785, 751, 786, 716, 806, 783, 785, 777, 795] payload_rows_per_dst=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] dup_rows_per_dst=[2972, 2576, 3054, 2912, 2962, 3466, 3158, 3032] combine_rows_per_src=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65214 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[742, 785, 763, 789, 724, 741, 761, 750, 809, 793, 769, 777, 734, 781, 776, 763, 743, 800, 752, 787, 788, 782, 750, 832, 774, 794, 782, 754, 723, 800, 764, 716, 784, 784, 758, 785, 765, 743, 761, 753, 779, 741, 700, 747, 728, 724, 745, 797, 781, 777, 816, 762, 762, 782, 763, 781, 827, 808, 779, 762, 755, 770, 793, 742] payload_rows_per_dst=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] dup_rows_per_dst=[2888, 3084, 3018, 2944, 3050, 2736, 3048, 3052] combine_rows_per_src=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64940 dup_rows=24096 dup_rows_remote=21042 dedup_ratio=0.245117 router_tokens_per_expert=[790, 754, 779, 738, 775, 746, 744, 755, 853, 720, 774, 751, 763, 694, 802, 771, 747, 789, 763, 755, 759, 776, 802, 770, 746, 781, 765, 807, 791, 805, 721, 756, 739, 775, 722, 814, 782, 722, 767, 821, 776, 752, 786, 759, 763, 805, 776, 801, 748, 796, 773, 751, 771, 785, 740, 735, 796, 784, 729, 759, 760, 782, 734, 807] payload_rows_per_dst=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] dup_rows_per_dst=[3042, 2964, 3054, 3028, 2996, 3134, 2928, 2950] combine_rows_per_src=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=64974 dup_rows=23762 dup_rows_remote=20620 dedup_ratio=0.241720 router_tokens_per_expert=[748, 787, 781, 755, 755, 790, 795, 759, 748, 785, 794, 752, 747, 761, 726, 786, 772, 755, 787, 774, 785, 735, 764, 754, 733, 745, 784, 780, 813, 732, 747, 816, 774, 757, 802, 772, 756, 779, 775, 748, 783, 800, 756, 763, 783, 761, 739, 782, 739, 765, 747, 781, 751, 822, 699, 806, 764, 767, 775, 770, 780, 751, 805, 755] payload_rows_per_dst=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] dup_rows_per_dst=[3142, 2890, 2924, 3136, 3028, 3208, 2568, 2866] combine_rows_per_src=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343600 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764400 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185200 weight_movement_bytes=103809024 total_nvlink_bytes=634994224 weight_to_token_ratio=0.195429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64978 dup_rows=23954 dup_rows_remote=20946 dedup_ratio=0.243673 router_tokens_per_expert=[788, 785, 745, 769, 798, 774, 758, 709, 770, 758, 779, 749, 777, 783, 815, 798, 738, 787, 736, 805, 759, 741, 764, 796, 689, 774, 777, 737, 761, 778, 774, 778, 738, 753, 746, 777, 793, 768, 750, 777, 756, 764, 776, 807, 784, 702, 754, 774, 810, 761, 829, 791, 746, 740, 757, 756, 736, 807, 768, 798, 779, 741, 747, 818] payload_rows_per_dst=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] dup_rows_per_dst=[2976, 3126, 2926, 2948, 2952, 3012, 3008, 3006] combine_rows_per_src=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65328 dup_rows=23830 dup_rows_remote=21096 dedup_ratio=0.242411 router_tokens_per_expert=[784, 764, 786, 784, 750, 776, 765, 805, 743, 815, 823, 761, 809, 746, 753, 805, 759, 753, 801, 713, 753, 790, 755, 746, 767, 730, 778, 770, 774, 785, 788, 821, 781, 765, 755, 757, 784, 769, 817, 740, 756, 729, 765, 743, 731, 744, 711, 761, 755, 745, 765, 736, 785, 796, 728, 740, 756, 804, 734, 766, 820, 788, 803, 771] payload_rows_per_dst=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] dup_rows_per_dst=[3070, 3114, 2906, 3044, 3016, 2734, 2936, 3010] combine_rows_per_src=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343928 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338168 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332408 weight_movement_bytes=69206016 total_nvlink_bytes=601538424 weight_to_token_ratio=0.130005 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65106 dup_rows=23924 dup_rows_remote=20992 dedup_ratio=0.243368 router_tokens_per_expert=[740, 771, 767, 767, 789, 787, 771, 768, 764, 771, 756, 754, 775, 770, 762, 751, 798, 844, 757, 814, 793, 760, 729, 761, 736, 776, 784, 750, 767, 763, 735, 787, 744, 733, 762, 753, 768, 763, 751, 751, 748, 747, 807, 786, 753, 795, 747, 766, 783, 746, 744, 806, 749, 771, 770, 776, 818, 735, 809, 780, 778, 774, 772, 750] payload_rows_per_dst=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] dup_rows_per_dst=[2960, 2932, 3042, 2908, 2946, 3036, 3054, 3046] combine_rows_per_src=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344360 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460904 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577448 weight_movement_bytes=34603008 total_nvlink_bytes=569180456 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21118 dedup_ratio=0.244995 router_tokens_per_expert=[778, 741, 814, 736, 768, 746, 793, 787, 771, 738, 789, 770, 758, 748, 760, 766, 733, 793, 803, 809, 770, 757, 802, 748, 770, 813, 755, 766, 788, 775, 806, 758, 791, 796, 810, 728, 750, 764, 732, 743, 744, 752, 762, 804, 763, 769, 763, 747, 801, 766, 771, 799, 710, 811, 726, 745, 782, 738, 750, 795, 757, 742, 723, 809] payload_rows_per_dst=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] dup_rows_per_dst=[3088, 3008, 3050, 3108, 2966, 2944, 2954, 2966] combine_rows_per_src=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=342376 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266475880 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532609384 weight_movement_bytes=34603008 total_nvlink_bytes=567212392 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343696 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266493584 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643472 weight_movement_bytes=69206016 total_nvlink_bytes=601849488 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=345696 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267929184 ingress_bytes=302186496 link_bytes=302186496 token_bytes=535512672 weight_movement_bytes=34603008 total_nvlink_bytes=570115680 weight_to_token_ratio=0.064617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137103456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344392 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267018568 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533692744 weight_movement_bytes=34603008 total_nvlink_bytes=568295752 weight_to_token_ratio=0.064837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20886 dedup_ratio=0.244364 router_tokens_per_expert=[778, 779, 811, 740, 774, 748, 797, 764, 770, 739, 734, 726, 721, 797, 723, 758, 776, 784, 799, 784, 754, 719, 772, 760, 760, 784, 794, 772, 799, 747, 763, 743, 788, 763, 808, 822, 775, 784, 766, 767, 722, 755, 743, 717, 775, 719, 754, 768, 781, 800, 850, 732, 782, 804, 746, 797, 790, 747, 768, 756, 828, 754, 770, 752] payload_rows_per_dst=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] dup_rows_per_dst=[3046, 2852, 2958, 3078, 2980, 2820, 3152, 3136] combine_rows_per_src=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344304 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266412272 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480240 weight_movement_bytes=69206016 total_nvlink_bytes=601686256 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65092 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[803, 752, 735, 760, 772, 796, 752, 763, 751, 780, 811, 792, 707, 787, 790, 735, 733, 764, 791, 783, 753, 773, 755, 758, 762, 752, 776, 756, 771, 739, 769, 761, 777, 774, 763, 757, 756, 808, 811, 760, 721, 775, 790, 732, 800, 812, 800, 763, 759, 775, 798, 746, 801, 738, 785, 792, 740, 719, 780, 763, 778, 760, 793, 744] payload_rows_per_dst=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] dup_rows_per_dst=[2992, 2942, 2954, 2868, 3074, 3090, 3052, 2974] combine_rows_per_src=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64924 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[753, 803, 789, 719, 765, 770, 774, 829, 794, 748, 741, 768, 771, 757, 802, 754, 730, 755, 757, 780, 769, 727, 735, 790, 770, 784, 777, 735, 713, 785, 809, 771, 783, 781, 766, 806, 745, 806, 759, 753, 782, 780, 793, 756, 719, 771, 744, 781, 752, 773, 760, 808, 782, 773, 755, 725, 785, 787, 784, 754, 773, 752, 756, 784] payload_rows_per_dst=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] dup_rows_per_dst=[3100, 3014, 2964, 3046, 3114, 2920, 2940, 3040] combine_rows_per_src=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343200 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266230944 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532118688 weight_movement_bytes=69206016 total_nvlink_bytes=601324704 weight_to_token_ratio=0.130057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21000 dedup_ratio=0.244283 router_tokens_per_expert=[802, 723, 761, 769, 776, 755, 781, 714, 771, 784, 791, 793, 752, 827, 715, 800, 799, 736, 752, 782, 744, 787, 737, 756, 784, 779, 758, 768, 794, 821, 744, 745, 774, 708, 768, 773, 765, 777, 750, 756, 835, 769, 740, 740, 757, 801, 755, 822, 768, 766, 777, 781, 776, 778, 781, 777, 761, 773, 782, 799, 760, 736, 707, 740] payload_rows_per_dst=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] dup_rows_per_dst=[2916, 3062, 2972, 3082, 2894, 3146, 3014, 2928] combine_rows_per_src=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64842 dup_rows=24008 dup_rows_remote=20790 dedup_ratio=0.244222 router_tokens_per_expert=[795, 783, 749, 787, 793, 757, 736, 769, 750, 783, 778, 778, 772, 756, 805, 812, 812, 772, 790, 781, 752, 769, 756, 744, 752, 743, 714, 784, 769, 765, 772, 809, 785, 726, 762, 766, 796, 760, 735, 765, 774, 708, 797, 714, 733, 771, 731, 770, 771, 775, 786, 747, 775, 807, 759, 826, 772, 745, 765, 755, 785, 765, 785, 754] payload_rows_per_dst=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] dup_rows_per_dst=[3218, 3340, 2880, 3054, 2992, 2834, 2990, 2700] combine_rows_per_src=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65174 dup_rows=23820 dup_rows_remote=20798 dedup_ratio=0.242310 router_tokens_per_expert=[751, 779, 752, 814, 739, 788, 838, 752, 749, 753, 800, 807, 756, 766, 733, 761, 769, 816, 738, 725, 804, 749, 784, 785, 767, 805, 784, 764, 763, 741, 764, 747, 778, 747, 767, 768, 791, 783, 743, 767, 796, 793, 759, 782, 752, 754, 774, 756, 730, 767, 721, 726, 744, 750, 719, 790, 751, 771, 766, 766, 807, 790, 774, 827] payload_rows_per_dst=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] dup_rows_per_dst=[3110, 2888, 2956, 2962, 3002, 3022, 2840, 3040] combine_rows_per_src=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64840 dup_rows=24238 dup_rows_remote=21234 dedup_ratio=0.246562 router_tokens_per_expert=[744, 776, 782, 803, 723, 763, 742, 743, 750, 783, 776, 768, 767, 765, 750, 756, 739, 789, 732, 797, 818, 770, 709, 786, 773, 771, 778, 801, 796, 753, 792, 770, 716, 751, 745, 768, 785, 791, 729, 704, 792, 805, 743, 773, 825, 753, 835, 763, 780, 778, 762, 781, 760, 741, 781, 788, 757, 746, 757, 745, 780, 768, 769, 816] payload_rows_per_dst=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] dup_rows_per_dst=[2948, 3004, 3010, 3108, 2854, 3188, 3080, 3046] combine_rows_per_src=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65042 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[747, 769, 780, 794, 734, 786, 789, 768, 746, 829, 752, 751, 780, 749, 752, 754, 713, 798, 777, 725, 766, 800, 781, 771, 756, 811, 824, 780, 753, 744, 774, 804, 744, 760, 775, 749, 755, 780, 789, 743, 780, 744, 780, 798, 798, 809, 766, 766, 767, 758, 750, 766, 716, 757, 715, 806, 763, 751, 745, 748, 811, 753, 793, 760] payload_rows_per_dst=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] dup_rows_per_dst=[2984, 2986, 3030, 3042, 2984, 3100, 2928, 3002] combine_rows_per_src=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344336 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961168 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578000 weight_movement_bytes=69206016 total_nvlink_bytes=602784016 weight_to_token_ratio=0.129702 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344064 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272768 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201472 weight_movement_bytes=69206016 total_nvlink_bytes=601407488 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343584 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266157600 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971616 weight_movement_bytes=69206016 total_nvlink_bytes=601177632 weight_to_token_ratio=0.130093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=342528 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265935360 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528192 weight_movement_bytes=34603008 total_nvlink_bytes=566131200 weight_to_token_ratio=0.065101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343888 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267296592 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534249296 weight_movement_bytes=69206016 total_nvlink_bytes=603455312 weight_to_token_ratio=0.129539 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=344296 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928936 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513576 weight_movement_bytes=34603008 total_nvlink_bytes=566116584 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344456 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756488 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168520 weight_movement_bytes=34603008 total_nvlink_bytes=567771528 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64870 dup_rows=24196 dup_rows_remote=21168 dedup_ratio=0.246134 router_tokens_per_expert=[748, 786, 739, 757, 797, 823, 788, 753, 774, 763, 722, 732, 740, 792, 735, 739, 776, 789, 833, 766, 790, 684, 753, 756, 798, 773, 776, 794, 764, 748, 805, 839, 755, 798, 773, 718, 772, 770, 746, 779, 792, 777, 791, 780, 758, 756, 746, 746, 768, 775, 689, 769, 770, 758, 761, 847, 770, 784, 765, 761, 764, 756, 773, 753] payload_rows_per_dst=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] dup_rows_per_dst=[3118, 2858, 2694, 3182, 3374, 3004, 2938, 3028] combine_rows_per_src=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=344152 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266051672 ingress_bytes=300310528 link_bytes=300310528 token_bytes=531759192 weight_movement_bytes=34603008 total_nvlink_bytes=566362200 weight_to_token_ratio=0.065073 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130044340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64888 dup_rows=23990 dup_rows_remote=20954 dedup_ratio=0.244039 router_tokens_per_expert=[768, 835, 691, 761, 779, 746, 776, 785, 760, 776, 809, 735, 754, 752, 765, 769, 779, 805, 752, 744, 816, 765, 780, 790, 738, 747, 723, 833, 745, 752, 767, 790, 763, 754, 765, 793, 728, 763, 753, 779, 806, 769, 784, 825, 770, 782, 762, 756, 774, 741, 800, 767, 734, 799, 741, 778, 775, 744, 739, 717, 755, 830, 758, 761] payload_rows_per_dst=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] dup_rows_per_dst=[2996, 2852, 3036, 2978, 2940, 3166, 3060, 2962] combine_rows_per_src=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65094 dup_rows=23844 dup_rows_remote=20776 dedup_ratio=0.242554 router_tokens_per_expert=[770, 765, 751, 740, 759, 796, 739, 762, 759, 756, 788, 754, 800, 788, 737, 801, 764, 806, 772, 786, 755, 763, 767, 763, 817, 738, 792, 786, 755, 760, 809, 760, 741, 748, 783, 755, 741, 786, 793, 759, 717, 728, 797, 681, 756, 752, 759, 781, 770, 765, 796, 799, 774, 815, 762, 776, 775, 761, 756, 765, 786, 786, 774, 757] payload_rows_per_dst=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] dup_rows_per_dst=[2906, 2890, 3016, 3068, 2980, 2798, 3112, 3074] combine_rows_per_src=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64880 dup_rows=23942 dup_rows_remote=20900 dedup_ratio=0.243551 router_tokens_per_expert=[720, 798, 786, 788, 708, 764, 777, 746, 815, 817, 737, 775, 746, 776, 757, 780, 751, 778, 814, 748, 746, 820, 792, 763, 709, 812, 704, 762, 736, 757, 765, 755, 768, 719, 739, 818, 774, 760, 775, 730, 759, 794, 820, 780, 732, 738, 745, 730, 791, 787, 737, 782, 798, 786, 797, 784, 761, 780, 802, 774, 796, 762, 772, 760] payload_rows_per_dst=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] dup_rows_per_dst=[3042, 2990, 3076, 2742, 3020, 2986, 3042, 3044] combine_rows_per_src=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65244 dup_rows=23884 dup_rows_remote=21042 dedup_ratio=0.242961 router_tokens_per_expert=[725, 757, 739, 784, 722, 788, 735, 735, 717, 737, 769, 776, 758, 786, 777, 766, 798, 741, 787, 796, 801, 711, 734, 761, 795, 755, 790, 778, 752, 764, 814, 763, 791, 756, 787, 780, 758, 735, 789, 760, 786, 718, 762, 798, 793, 813, 787, 812, 769, 741, 782, 762, 780, 779, 749, 787, 726, 765, 759, 763, 816, 803, 754, 781] payload_rows_per_dst=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] dup_rows_per_dst=[2842, 3198, 2996, 2900, 3014, 3042, 2996, 2896] combine_rows_per_src=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64990 dup_rows=23958 dup_rows_remote=20858 dedup_ratio=0.243713 router_tokens_per_expert=[765, 786, 719, 768, 778, 748, 758, 756, 736, 818, 756, 753, 724, 792, 757, 747, 782, 750, 797, 746, 812, 776, 780, 784, 760, 797, 768, 738, 783, 787, 732, 773, 766, 764, 775, 755, 756, 737, 753, 841, 751, 757, 762, 807, 792, 762, 762, 835, 764, 791, 762, 790, 749, 752, 735, 757, 798, 816, 740, 718, 782, 795, 767, 735] payload_rows_per_dst=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] dup_rows_per_dst=[3048, 2880, 3054, 3012, 2902, 3100, 2982, 2980] combine_rows_per_src=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65258 dup_rows=23902 dup_rows_remote=21004 dedup_ratio=0.243144 router_tokens_per_expert=[762, 779, 795, 799, 796, 758, 741, 795, 754, 734, 771, 715, 755, 750, 791, 751, 789, 722, 814, 772, 750, 776, 795, 768, 819, 800, 780, 777, 754, 766, 759, 767, 796, 731, 808, 766, 795, 732, 795, 758, 774, 740, 775, 735, 809, 764, 761, 734, 705, 770, 741, 746, 768, 768, 785, 755, 781, 808, 758, 770, 768, 800, 713, 789] payload_rows_per_dst=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] dup_rows_per_dst=[3122, 2898, 3024, 2994, 3060, 2942, 2890, 2972] combine_rows_per_src=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343368 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124616 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905864 weight_movement_bytes=69206016 total_nvlink_bytes=601111880 weight_to_token_ratio=0.130110 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65330 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[785, 740, 762, 814, 761, 708, 722, 776, 773, 789, 787, 751, 777, 790, 750, 764, 778, 795, 803, 754, 770, 744, 757, 762, 807, 760, 709, 768, 815, 758, 807, 759, 767, 756, 759, 743, 738, 766, 763, 787, 721, 761, 772, 798, 771, 760, 733, 791, 730, 787, 818, 833, 782, 805, 772, 780, 725, 745, 784, 764, 752, 746, 789, 759] payload_rows_per_dst=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] dup_rows_per_dst=[2900, 2996, 2974, 2998, 3052, 3002, 3082, 2864] combine_rows_per_src=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343480 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968504 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593528 weight_movement_bytes=69206016 total_nvlink_bytes=602799544 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343120 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266091600 ingress_bytes=404160512 link_bytes=404160512 token_bytes=531840080 weight_movement_bytes=138412032 total_nvlink_bytes=670252112 weight_to_token_ratio=0.260251 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520823469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=345144 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584568 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823992 weight_movement_bytes=34603008 total_nvlink_bytes=569427000 weight_to_token_ratio=0.064700 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343392 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542432 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741472 weight_movement_bytes=69206016 total_nvlink_bytes=601947488 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64876 dup_rows=24088 dup_rows_remote=21048 dedup_ratio=0.245036 router_tokens_per_expert=[776, 809, 785, 820, 737, 710, 777, 756, 751, 799, 789, 759, 779, 771, 781, 802, 808, 745, 757, 811, 732, 770, 832, 746, 703, 755, 793, 746, 778, 764, 808, 746, 761, 735, 770, 754, 745, 791, 780, 782, 773, 765, 784, 768, 769, 758, 773, 706, 795, 774, 774, 704, 738, 744, 779, 758, 755, 734, 766, 780, 785, 790, 773, 794] payload_rows_per_dst=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] dup_rows_per_dst=[2742, 3292, 2866, 2932, 3062, 2942, 3212, 3040] combine_rows_per_src=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=345048 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641816 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534938584 weight_movement_bytes=34603008 total_nvlink_bytes=569541592 weight_to_token_ratio=0.064686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=344584 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267936264 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535527944 weight_movement_bytes=34603008 total_nvlink_bytes=570130952 weight_to_token_ratio=0.064615 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266075792 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531807888 weight_movement_bytes=34603008 total_nvlink_bytes=566410896 weight_to_token_ratio=0.065067 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65018 dup_rows=24020 dup_rows_remote=21124 dedup_ratio=0.244344 router_tokens_per_expert=[787, 738, 803, 795, 764, 757, 822, 767, 788, 778, 693, 773, 777, 746, 738, 762, 722, 783, 756, 760, 800, 740, 756, 764, 764, 779, 767, 749, 783, 787, 776, 780, 773, 739, 757, 752, 787, 760, 767, 710, 794, 800, 838, 743, 770, 818, 791, 775, 736, 751, 765, 800, 736, 763, 772, 757, 781, 788, 797, 726, 732, 780, 760, 780] payload_rows_per_dst=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] dup_rows_per_dst=[3046, 2952, 2896, 3018, 2926, 3172, 2970, 3040] combine_rows_per_src=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65036 dup_rows=23904 dup_rows_remote=20940 dedup_ratio=0.243164 router_tokens_per_expert=[781, 742, 797, 790, 782, 765, 743, 726, 757, 760, 768, 730, 784, 727, 760, 826, 748, 732, 809, 795, 737, 800, 822, 765, 765, 830, 716, 746, 808, 739, 794, 766, 779, 739, 818, 765, 785, 785, 794, 769, 708, 739, 778, 739, 799, 752, 786, 754, 768, 777, 778, 734, 740, 727, 776, 786, 758, 766, 785, 722, 764, 792, 802, 778] payload_rows_per_dst=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] dup_rows_per_dst=[2952, 2962, 3020, 2964, 3064, 2934, 2982, 3026] combine_rows_per_src=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65088 dup_rows=24008 dup_rows_remote=20982 dedup_ratio=0.244222 router_tokens_per_expert=[769, 812, 777, 751, 785, 810, 784, 776, 741, 738, 789, 773, 777, 761, 783, 727, 796, 729, 762, 759, 784, 751, 790, 803, 744, 784, 739, 835, 838, 753, 772, 758, 809, 743, 785, 774, 787, 782, 752, 773, 739, 778, 737, 753, 750, 809, 791, 729, 739, 777, 782, 740, 770, 732, 803, 774, 737, 766, 752, 752, 714, 745, 740, 788] payload_rows_per_dst=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] dup_rows_per_dst=[3074, 2988, 2992, 2960, 3050, 3028, 3026, 2890] combine_rows_per_src=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=64996 dup_rows=23796 dup_rows_remote=20648 dedup_ratio=0.242065 router_tokens_per_expert=[737, 763, 767, 769, 768, 722, 767, 799, 788, 740, 803, 770, 786, 749, 725, 808, 778, 756, 802, 804, 735, 774, 767, 749, 796, 772, 766, 754, 736, 764, 766, 732, 718, 775, 803, 785, 784, 793, 769, 773, 745, 807, 781, 743, 769, 756, 807, 728, 807, 756, 784, 745, 769, 744, 751, 788, 777, 812, 755, 790, 801, 741, 741, 743] payload_rows_per_dst=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] dup_rows_per_dst=[3148, 3248, 2980, 2770, 2994, 2810, 2880, 2966] combine_rows_per_src=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65038 dup_rows=24006 dup_rows_remote=21044 dedup_ratio=0.244202 router_tokens_per_expert=[752, 745, 789, 771, 806, 746, 727, 780, 771, 787, 775, 755, 739, 727, 777, 757, 794, 738, 791, 791, 780, 779, 832, 719, 768, 776, 810, 794, 755, 722, 741, 721, 789, 761, 731, 788, 767, 762, 774, 761, 775, 785, 746, 745, 709, 772, 804, 775, 782, 791, 805, 749, 759, 746, 749, 768, 798, 817, 798, 763, 791, 774, 720, 783] payload_rows_per_dst=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] dup_rows_per_dst=[2978, 2890, 3014, 3032, 3070, 2962, 3026, 3034] combine_rows_per_src=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344568 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266658296 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532972024 weight_movement_bytes=69206016 total_nvlink_bytes=602178040 weight_to_token_ratio=0.129849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65136 dup_rows=23870 dup_rows_remote=20900 dedup_ratio=0.242818 router_tokens_per_expert=[748, 750, 762, 709, 760, 753, 769, 772, 783, 836, 756, 705, 787, 734, 762, 771, 771, 772, 736, 751, 766, 749, 761, 834, 749, 771, 767, 720, 796, 786, 762, 822, 750, 706, 762, 810, 759, 776, 786, 787, 770, 796, 760, 765, 744, 788, 754, 761, 795, 749, 810, 823, 796, 736, 809, 768, 751, 771, 806, 785, 721, 756, 768, 764] payload_rows_per_dst=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] dup_rows_per_dst=[2856, 2970, 2978, 2964, 2964, 3020, 3108, 3010] combine_rows_per_src=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64886 dup_rows=24198 dup_rows_remote=21110 dedup_ratio=0.246155 router_tokens_per_expert=[747, 778, 836, 774, 774, 767, 753, 818, 771, 760, 731, 769, 786, 705, 721, 764, 762, 766, 774, 777, 830, 770, 777, 723, 807, 753, 782, 781, 749, 783, 753, 758, 780, 751, 792, 750, 800, 793, 763, 725, 806, 772, 742, 737, 781, 756, 794, 753, 775, 735, 785, 768, 769, 729, 830, 755, 743, 777, 768, 801, 740, 794, 726, 763] payload_rows_per_dst=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] dup_rows_per_dst=[3160, 2794, 3058, 3022, 3088, 3074, 2980, 3022] combine_rows_per_src=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343904 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266731360 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533118816 weight_movement_bytes=69206016 total_nvlink_bytes=602324832 weight_to_token_ratio=0.129813 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344280 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944728 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545176 weight_movement_bytes=34603008 total_nvlink_bytes=568148184 weight_to_token_ratio=0.064855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=342576 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266566192 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532789808 weight_movement_bytes=34603008 total_nvlink_bytes=567392816 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64892 dup_rows=24012 dup_rows_remote=20860 dedup_ratio=0.244263 router_tokens_per_expert=[739, 786, 772, 792, 768, 748, 766, 774, 788, 750, 756, 742, 785, 792, 717, 772, 754, 745, 757, 817, 791, 726, 800, 763, 744, 799, 777, 771, 705, 743, 776, 780, 747, 811, 781, 760, 767, 789, 817, 732, 774, 720, 745, 737, 783, 750, 784, 760, 822, 804, 759, 785, 749, 756, 748, 783, 817, 744, 768, 742, 794, 795, 806, 728] payload_rows_per_dst=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] dup_rows_per_dst=[3020, 2948, 2894, 2874, 3178, 2900, 3046, 3152] combine_rows_per_src=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344328 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739976 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135624 weight_movement_bytes=69206016 total_nvlink_bytes=602341640 weight_to_token_ratio=0.129809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344144 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141200 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938256 weight_movement_bytes=34603008 total_nvlink_bytes=568541264 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343984 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117040 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890096 weight_movement_bytes=34603008 total_nvlink_bytes=566493104 weight_to_token_ratio=0.065057 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65082 dup_rows=23880 dup_rows_remote=20892 dedup_ratio=0.242920 router_tokens_per_expert=[776, 752, 799, 806, 733, 756, 777, 751, 748, 777, 729, 735, 750, 791, 754, 799, 770, 745, 761, 796, 775, 747, 806, 765, 793, 750, 787, 774, 715, 764, 806, 794, 786, 742, 807, 776, 737, 778, 745, 779, 727, 754, 744, 767, 778, 753, 782, 754, 767, 771, 786, 768, 763, 712, 793, 769, 796, 729, 837, 769, 782, 813, 709, 798] payload_rows_per_dst=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] dup_rows_per_dst=[3010, 2934, 2988, 3046, 2990, 2928, 2998, 2986] combine_rows_per_src=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[778, 796, 771, 754, 804, 748, 769, 773, 804, 768, 743, 740, 793, 799, 759, 753, 754, 766, 783, 774, 710, 786, 774, 769, 801, 769, 739, 774, 784, 788, 732, 795, 770, 728, 761, 731, 769, 776, 731, 828, 775, 795, 779, 815, 741, 782, 719, 792, 768, 719, 745, 754, 726, 733, 808, 751, 759, 767, 760, 785, 812, 785, 799, 739] payload_rows_per_dst=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] dup_rows_per_dst=[3140, 3024, 2958, 3002, 2958, 3042, 2836, 2996] combine_rows_per_src=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64898 dup_rows=24102 dup_rows_remote=21000 dedup_ratio=0.245178 router_tokens_per_expert=[746, 714, 792, 762, 798, 776, 819, 794, 786, 728, 809, 782, 765, 782, 768, 771, 734, 781, 727, 740, 779, 767, 759, 754, 742, 761, 727, 746, 784, 765, 780, 763, 740, 769, 759, 746, 769, 747, 776, 758, 822, 803, 793, 766, 799, 767, 773, 775, 745, 794, 770, 776, 770, 770, 761, 741, 785, 806, 783, 740, 785, 760, 748, 755] payload_rows_per_dst=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] dup_rows_per_dst=[3102, 3020, 2930, 2976, 2926, 3180, 2992, 2976] combine_rows_per_src=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64674 dup_rows=24258 dup_rows_remote=21084 dedup_ratio=0.246765 router_tokens_per_expert=[779, 765, 752, 719, 758, 827, 767, 760, 772, 760, 845, 743, 792, 747, 765, 763, 778, 739, 754, 760, 777, 755, 782, 780, 712, 758, 779, 767, 823, 739, 801, 797, 769, 753, 765, 736, 744, 762, 766, 783, 779, 802, 809, 801, 780, 708, 720, 771, 787, 783, 760, 820, 804, 788, 755, 776, 760, 721, 733, 781, 741, 747, 776, 757] payload_rows_per_dst=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] dup_rows_per_dst=[3008, 3134, 2996, 3116, 2950, 3020, 3174, 2860] combine_rows_per_src=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343896 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266919768 ingress_bytes=370384896 link_bytes=370384896 token_bytes=533495640 weight_movement_bytes=103809024 total_nvlink_bytes=637304664 weight_to_token_ratio=0.194583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393728545e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343008 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266140640 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531938272 weight_movement_bytes=34603008 total_nvlink_bytes=566541280 weight_to_token_ratio=0.065051 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266526416 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709072 weight_movement_bytes=103809024 total_nvlink_bytes=636518096 weight_to_token_ratio=0.194870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343592 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266165800 ingress_bytes=404234240 link_bytes=404234240 token_bytes=531988008 weight_movement_bytes=138412032 total_nvlink_bytes=670400040 weight_to_token_ratio=0.260179 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521100902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=343032 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265247736 ingress_bytes=368713728 link_bytes=368713728 token_bytes=530152440 weight_movement_bytes=103809024 total_nvlink_bytes=633961464 weight_to_token_ratio=0.195810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387440075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64686 dup_rows=24182 dup_rows_remote=21154 dedup_ratio=0.245992 router_tokens_per_expert=[788, 726, 761, 794, 767, 781, 772, 787, 775, 789, 792, 753, 847, 813, 740, 723, 743, 797, 749, 733, 760, 790, 759, 766, 779, 748, 754, 753, 741, 714, 862, 754, 762, 762, 780, 791, 813, 800, 816, 742, 728, 751, 781, 723, 739, 774, 772, 770, 766, 777, 770, 762, 801, 707, 775, 734, 791, 787, 765, 751, 771, 755, 776, 750] payload_rows_per_dst=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] dup_rows_per_dst=[2984, 3028, 2992, 3074, 3138, 2970, 2990, 3006] combine_rows_per_src=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20836 dedup_ratio=0.242716 router_tokens_per_expert=[743, 760, 775, 750, 720, 701, 808, 768, 750, 754, 803, 780, 742, 837, 762, 779, 766, 778, 790, 780, 756, 768, 782, 766, 834, 750, 750, 786, 761, 737, 783, 731, 757, 766, 811, 735, 754, 794, 766, 772, 801, 750, 793, 782, 784, 731, 763, 762, 733, 803, 752, 754, 787, 812, 800, 713, 751, 783, 768, 771, 779, 773, 772, 730] payload_rows_per_dst=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] dup_rows_per_dst=[2894, 3050, 3014, 2866, 3028, 3024, 2968, 3016] combine_rows_per_src=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64844 dup_rows=24230 dup_rows_remote=21144 dedup_ratio=0.246480 router_tokens_per_expert=[763, 768, 799, 815, 781, 777, 819, 771, 788, 760, 813, 751, 741, 765, 763, 779, 746, 738, 776, 723, 799, 809, 716, 779, 764, 800, 793, 806, 818, 757, 722, 754, 725, 803, 805, 747, 777, 720, 798, 783, 739, 766, 714, 808, 721, 723, 775, 760, 744, 752, 802, 813, 768, 754, 731, 814, 770, 751, 767, 706, 710, 806, 814, 733] payload_rows_per_dst=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] dup_rows_per_dst=[3158, 3056, 2968, 3102, 3086, 2844, 3072, 2944] combine_rows_per_src=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264953856 dispatch_meta_bytes=343360 combine_payload_bytes=264953856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265297216 ingress_bytes=299556864 link_bytes=299556864 token_bytes=530251072 weight_movement_bytes=34603008 total_nvlink_bytes=564854080 weight_to_token_ratio=0.065258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127208363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343888 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267140944 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938000 weight_movement_bytes=34603008 total_nvlink_bytes=568541008 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64856 dup_rows=24008 dup_rows_remote=20944 dedup_ratio=0.244222 router_tokens_per_expert=[782, 719, 778, 735, 753, 773, 756, 797, 738, 768, 806, 769, 751, 775, 761, 740, 779, 788, 768, 758, 834, 791, 776, 758, 790, 747, 766, 767, 763, 746, 704, 797, 756, 736, 785, 745, 793, 763, 779, 744, 761, 745, 796, 723, 832, 813, 772, 706, 785, 765, 783, 745, 743, 755, 760, 805, 785, 757, 791, 816, 806, 810, 746, 718] payload_rows_per_dst=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] dup_rows_per_dst=[3034, 2938, 3064, 2952, 2972, 3006, 3000, 3042] combine_rows_per_src=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343952 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265944976 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546000 weight_movement_bytes=103809024 total_nvlink_bytes=635355024 weight_to_token_ratio=0.195296 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65054 dup_rows=24026 dup_rows_remote=21034 dedup_ratio=0.244405 router_tokens_per_expert=[769, 780, 768, 734, 787, 783, 758, 776, 763, 761, 722, 777, 751, 777, 824, 744, 745, 775, 761, 755, 779, 739, 754, 772, 800, 750, 742, 789, 727, 784, 728, 788, 762, 757, 771, 790, 775, 785, 798, 760, 762, 771, 771, 762, 795, 794, 832, 720, 756, 776, 792, 786, 770, 771, 768, 736, 789, 809, 773, 779, 743, 729, 749, 759] payload_rows_per_dst=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] dup_rows_per_dst=[2972, 2904, 2988, 2992, 3066, 3036, 3044, 3024] combine_rows_per_src=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65192 dup_rows=23834 dup_rows_remote=20628 dedup_ratio=0.242452 router_tokens_per_expert=[743, 719, 764, 770, 744, 769, 795, 775, 797, 787, 781, 774, 748, 797, 796, 744, 811, 803, 726, 796, 768, 797, 787, 785, 755, 773, 833, 750, 762, 721, 778, 759, 747, 745, 763, 742, 765, 795, 755, 748, 752, 778, 783, 751, 743, 822, 760, 777, 792, 818, 748, 773, 737, 815, 731, 745, 742, 781, 777, 717, 781, 801, 729, 732] payload_rows_per_dst=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] dup_rows_per_dst=[3206, 3014, 3070, 2912, 2932, 3094, 2778, 2828] combine_rows_per_src=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65144 dup_rows=23952 dup_rows_remote=20922 dedup_ratio=0.243652 router_tokens_per_expert=[754, 742, 777, 790, 767, 797, 780, 787, 786, 768, 755, 769, 742, 770, 719, 776, 819, 768, 797, 761, 735, 742, 789, 732, 771, 803, 732, 805, 800, 805, 778, 746, 747, 747, 725, 762, 758, 760, 764, 781, 778, 777, 719, 773, 771, 767, 805, 749, 802, 737, 773, 792, 773, 826, 764, 777, 787, 723, 764, 741, 709, 794, 738, 807] payload_rows_per_dst=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] dup_rows_per_dst=[2920, 2952, 2934, 2840, 2874, 3154, 3248, 3030] combine_rows_per_src=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65396 dup_rows=23908 dup_rows_remote=20960 dedup_ratio=0.243205 router_tokens_per_expert=[784, 746, 754, 774, 718, 785, 755, 744, 769, 760, 791, 746, 746, 771, 807, 711, 834, 760, 795, 742, 794, 806, 745, 789, 797, 828, 780, 754, 795, 768, 770, 800, 763, 714, 752, 717, 779, 795, 787, 801, 825, 761, 853, 725, 801, 743, 779, 738, 706, 717, 738, 775, 753, 733, 775, 777, 756, 713, 764, 775, 779, 791, 785, 764] payload_rows_per_dst=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] dup_rows_per_dst=[2890, 2946, 3080, 3164, 2916, 3040, 2948, 2924] combine_rows_per_src=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343200 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993376 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643552 weight_movement_bytes=103809024 total_nvlink_bytes=635452576 weight_to_token_ratio=0.195261 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344352 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805536 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266720 weight_movement_bytes=69206016 total_nvlink_bytes=602472736 weight_to_token_ratio=0.129777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343280 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267369712 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534396144 weight_movement_bytes=34603008 total_nvlink_bytes=568999152 weight_to_token_ratio=0.064752 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344264 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174088 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003912 weight_movement_bytes=34603008 total_nvlink_bytes=568606920 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64938 dup_rows=24198 dup_rows_remote=21024 dedup_ratio=0.246155 router_tokens_per_expert=[761, 716, 755, 799, 757, 734, 830, 744, 773, 779, 750, 785, 747, 757, 785, 795, 739, 811, 737, 784, 768, 742, 821, 776, 726, 740, 769, 767, 780, 766, 757, 779, 772, 784, 721, 766, 729, 775, 782, 811, 801, 738, 763, 754, 820, 754, 705, 803, 707, 786, 769, 813, 791, 796, 749, 768, 798, 765, 781, 762, 777, 768, 762, 753] payload_rows_per_dst=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] dup_rows_per_dst=[2930, 3174, 3044, 2942, 3052, 3006, 3018, 3032] combine_rows_per_src=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=345424 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268207440 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536069456 weight_movement_bytes=34603008 total_nvlink_bytes=570672464 weight_to_token_ratio=0.064549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65016 dup_rows=23982 dup_rows_remote=21004 dedup_ratio=0.243958 router_tokens_per_expert=[776, 777, 763, 781, 745, 769, 774, 739, 757, 719, 738, 738, 772, 780, 765, 809, 828, 793, 768, 785, 781, 744, 751, 762, 798, 786, 712, 820, 786, 725, 767, 758, 792, 762, 772, 718, 807, 736, 767, 770, 785, 804, 776, 724, 780, 744, 782, 747, 780, 796, 789, 764, 720, 770, 750, 746, 786, 772, 812, 766, 821, 744, 736, 768] payload_rows_per_dst=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] dup_rows_per_dst=[3042, 2932, 2962, 3030, 2988, 2978, 2996, 3054] combine_rows_per_src=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65202 dup_rows=23790 dup_rows_remote=20858 dedup_ratio=0.242004 router_tokens_per_expert=[717, 780, 750, 726, 727, 746, 810, 754, 774, 758, 804, 784, 784, 780, 784, 780, 761, 770, 753, 742, 779, 748, 757, 788, 798, 748, 739, 837, 780, 809, 740, 776, 743, 796, 728, 746, 772, 785, 796, 756, 762, 754, 727, 737, 722, 769, 780, 746, 778, 759, 786, 795, 769, 739, 753, 816, 779, 754, 808, 785, 799, 817, 771, 742] payload_rows_per_dst=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] dup_rows_per_dst=[2844, 3052, 2886, 3038, 2932, 2862, 3078, 3098] combine_rows_per_src=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343848 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=266329896 ingress_bytes=439001088 link_bytes=439001088 token_bytes=532315944 weight_movement_bytes=173015040 total_nvlink_bytes=705330984 weight_to_token_ratio=0.325023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651925752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344240 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411632 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534479024 weight_movement_bytes=34603008 total_nvlink_bytes=569082032 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344080 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649616 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955152 weight_movement_bytes=34603008 total_nvlink_bytes=567558160 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65094 dup_rows=23858 dup_rows_remote=20806 dedup_ratio=0.242696 router_tokens_per_expert=[791, 805, 750, 719, 773, 787, 759, 751, 767, 805, 814, 682, 795, 778, 784, 767, 794, 752, 746, 761, 812, 732, 788, 817, 778, 809, 784, 704, 749, 781, 762, 715, 787, 801, 748, 763, 753, 800, 789, 753, 731, 768, 750, 790, 765, 745, 806, 761, 731, 753, 778, 778, 754, 748, 797, 738, 739, 787, 792, 800, 750, 761, 732, 793] payload_rows_per_dst=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] dup_rows_per_dst=[3008, 3076, 3052, 2942, 3036, 2908, 2850, 2986] combine_rows_per_src=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64992 dup_rows=24034 dup_rows_remote=21098 dedup_ratio=0.244486 router_tokens_per_expert=[695, 754, 769, 761, 745, 740, 726, 794, 819, 806, 751, 809, 736, 803, 774, 791, 746, 770, 767, 764, 749, 733, 784, 749, 754, 777, 756, 781, 753, 780, 822, 684, 750, 774, 744, 757, 778, 764, 786, 808, 753, 774, 782, 760, 786, 751, 797, 772, 768, 818, 780, 828, 716, 752, 745, 758, 785, 780, 794, 772, 777, 770, 775, 756] payload_rows_per_dst=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] dup_rows_per_dst=[2912, 3128, 2880, 2936, 3052, 3086, 3052, 2988] combine_rows_per_src=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64848 dup_rows=24114 dup_rows_remote=20990 dedup_ratio=0.245300 router_tokens_per_expert=[773, 811, 763, 747, 796, 759, 748, 783, 752, 788, 789, 845, 784, 768, 782, 750, 758, 764, 739, 796, 774, 759, 764, 751, 765, 767, 749, 732, 801, 755, 752, 749, 800, 829, 719, 797, 755, 760, 784, 795, 785, 760, 794, 736, 750, 728, 741, 777, 775, 755, 734, 751, 738, 766, 788, 783, 784, 740, 787, 766, 727, 760, 775, 800] payload_rows_per_dst=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] dup_rows_per_dst=[3124, 3070, 2918, 3014, 3000, 2940, 3032, 3016] combine_rows_per_src=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65074 dup_rows=23892 dup_rows_remote=20974 dedup_ratio=0.243042 router_tokens_per_expert=[743, 798, 771, 735, 722, 755, 746, 756, 776, 754, 797, 832, 794, 787, 760, 787, 776, 788, 698, 705, 804, 766, 792, 763, 745, 772, 784, 796, 769, 728, 756, 798, 788, 736, 767, 759, 784, 753, 755, 738, 788, 812, 708, 767, 813, 741, 818, 796, 770, 741, 759, 769, 785, 807, 748, 784, 776, 760, 754, 752, 778, 764, 777, 752] payload_rows_per_dst=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] dup_rows_per_dst=[2868, 2854, 2918, 2960, 2986, 3314, 3074, 2918] combine_rows_per_src=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65192 dup_rows=23798 dup_rows_remote=20790 dedup_ratio=0.242086 router_tokens_per_expert=[807, 751, 795, 806, 708, 756, 740, 779, 751, 783, 783, 754, 692, 757, 719, 782, 771, 749, 763, 742, 800, 764, 769, 775, 792, 806, 753, 761, 757, 764, 772, 764, 753, 741, 792, 754, 801, 767, 752, 750, 732, 756, 765, 800, 755, 802, 807, 759, 784, 764, 749, 755, 767, 780, 780, 782, 749, 785, 778, 763, 856, 750, 787, 772] payload_rows_per_dst=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] dup_rows_per_dst=[2984, 2838, 2988, 2996, 2882, 3014, 3008, 3088] combine_rows_per_src=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343600 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266968624 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533593648 weight_movement_bytes=34603008 total_nvlink_bytes=568196656 weight_to_token_ratio=0.064849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344360 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551592 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758824 weight_movement_bytes=34603008 total_nvlink_bytes=567361832 weight_to_token_ratio=0.064951 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265960760 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578168 weight_movement_bytes=69206016 total_nvlink_bytes=600784184 weight_to_token_ratio=0.130190 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=65306 dup_rows=23720 dup_rows_remote=20804 dedup_ratio=0.241292 router_tokens_per_expert=[786, 757, 731, 790, 757, 780, 788, 736, 768, 785, 754, 758, 803, 736, 759, 734, 749, 758, 810, 755, 708, 755, 753, 729, 738, 781, 806, 785, 789, 804, 769, 763, 799, 747, 760, 778, 786, 805, 754, 789, 753, 778, 740, 780, 800, 759, 771, 768, 801, 748, 770, 759, 749, 806, 773, 721, 787, 778, 831, 754, 780, 781, 734, 739] payload_rows_per_dst=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] dup_rows_per_dst=[2984, 2916, 2932, 2988, 3038, 2920, 2926, 3016] combine_rows_per_src=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344192 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266887296 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430400 weight_movement_bytes=69206016 total_nvlink_bytes=602636416 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343928 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370360 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396792 weight_movement_bytes=69206016 total_nvlink_bytes=603602808 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64848 dup_rows=24158 dup_rows_remote=21114 dedup_ratio=0.245748 router_tokens_per_expert=[755, 775, 771, 792, 707, 749, 793, 758, 781, 784, 757, 780, 778, 744, 746, 786, 727, 771, 838, 757, 754, 777, 720, 770, 738, 796, 775, 761, 743, 798, 785, 750, 770, 788, 801, 766, 750, 747, 768, 781, 762, 749, 770, 754, 785, 777, 722, 748, 795, 810, 774, 749, 755, 803, 738, 781, 784, 780, 787, 744, 807, 785, 753, 753] payload_rows_per_dst=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] dup_rows_per_dst=[2982, 3154, 2944, 3038, 3044, 2876, 3030, 3090] combine_rows_per_src=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65316 dup_rows=23940 dup_rows_remote=21066 dedup_ratio=0.243530 router_tokens_per_expert=[740, 733, 825, 755, 790, 825, 813, 752, 751, 805, 764, 802, 816, 792, 787, 812, 749, 859, 760, 781, 794, 776, 719, 730, 762, 766, 754, 793, 759, 779, 773, 795, 801, 740, 767, 734, 760, 764, 743, 752, 728, 729, 735, 729, 745, 756, 777, 762, 730, 758, 802, 741, 749, 782, 807, 759, 776, 782, 743, 787, 759, 772, 737, 735] payload_rows_per_dst=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] dup_rows_per_dst=[3070, 3132, 2934, 3092, 2936, 2874, 3024, 2878] combine_rows_per_src=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267493376 dispatch_meta_bytes=344440 combine_payload_bytes=267493376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267837816 ingress_bytes=302096384 link_bytes=302096384 token_bytes=535331192 weight_movement_bytes=34603008 total_nvlink_bytes=569934200 weight_to_token_ratio=0.064639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136764372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343848 combine_payload_bytes=265617408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265961256 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578664 weight_movement_bytes=69206016 total_nvlink_bytes=600784680 weight_to_token_ratio=0.130190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345528 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879864 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535414200 weight_movement_bytes=34603008 total_nvlink_bytes=570017208 weight_to_token_ratio=0.064628 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64660 dup_rows=24178 dup_rows_remote=21016 dedup_ratio=0.245951 router_tokens_per_expert=[808, 754, 710, 754, 725, 714, 744, 747, 783, 804, 785, 762, 794, 743, 783, 777, 745, 793, 792, 816, 774, 796, 800, 798, 778, 804, 732, 794, 760, 754, 739, 710, 760, 758, 754, 794, 776, 780, 810, 795, 751, 716, 774, 713, 747, 763, 794, 758, 752, 775, 764, 743, 790, 785, 741, 755, 814, 790, 785, 788, 778, 746, 765, 766] payload_rows_per_dst=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] dup_rows_per_dst=[2860, 3142, 3162, 2974, 3054, 2958, 2964, 3064] combine_rows_per_src=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65204 dup_rows=23924 dup_rows_remote=21012 dedup_ratio=0.243368 router_tokens_per_expert=[829, 763, 729, 754, 771, 801, 689, 781, 770, 761, 777, 781, 748, 804, 820, 738, 761, 788, 771, 757, 750, 801, 778, 780, 736, 755, 777, 756, 727, 764, 782, 747, 720, 789, 816, 786, 758, 798, 758, 793, 780, 812, 803, 761, 746, 808, 740, 764, 744, 743, 778, 748, 748, 777, 750, 721, 784, 786, 760, 758, 747, 797, 732, 801] payload_rows_per_dst=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] dup_rows_per_dst=[2994, 2994, 3100, 2912, 2946, 3166, 2818, 2994] combine_rows_per_src=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64742 dup_rows=23832 dup_rows_remote=20510 dedup_ratio=0.242432 router_tokens_per_expert=[764, 804, 823, 783, 802, 734, 798, 800, 750, 780, 726, 772, 764, 794, 755, 801, 722, 747, 738, 770, 792, 786, 774, 786, 759, 789, 765, 753, 838, 744, 768, 720, 768, 737, 744, 799, 775, 755, 775, 763, 732, 806, 795, 786, 734, 731, 767, 741, 763, 800, 770, 787, 796, 774, 767, 773, 777, 695, 724, 790, 767, 729, 790, 741] payload_rows_per_dst=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] dup_rows_per_dst=[3322, 2792, 3068, 3038, 3100, 2860, 2812, 2840] combine_rows_per_src=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65068 dup_rows=23986 dup_rows_remote=21060 dedup_ratio=0.243998 router_tokens_per_expert=[780, 772, 750, 765, 767, 748, 779, 792, 770, 770, 769, 796, 723, 816, 780, 722, 764, 741, 733, 803, 739, 729, 787, 797, 784, 773, 775, 813, 772, 801, 745, 784, 774, 760, 746, 756, 716, 749, 787, 722, 747, 766, 792, 760, 727, 774, 836, 782, 789, 762, 747, 762, 795, 802, 778, 796, 769, 760, 801, 774, 750, 733, 760, 741] payload_rows_per_dst=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] dup_rows_per_dst=[2992, 3008, 2968, 3078, 2912, 3050, 3052, 2926] combine_rows_per_src=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65114 dup_rows=23920 dup_rows_remote=20858 dedup_ratio=0.243327 router_tokens_per_expert=[770, 735, 772, 759, 755, 752, 753, 743, 754, 808, 750, 769, 793, 756, 757, 722, 746, 729, 814, 797, 801, 768, 741, 811, 779, 758, 741, 818, 759, 759, 772, 818, 804, 714, 752, 773, 753, 810, 778, 740, 776, 800, 763, 790, 768, 718, 775, 763, 790, 783, 785, 775, 741, 760, 789, 743, 749, 800, 759, 827, 718, 812, 759, 726] payload_rows_per_dst=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] dup_rows_per_dst=[2932, 2992, 3084, 3076, 3004, 2906, 3062, 2864] combine_rows_per_src=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=342704 combine_payload_bytes=264847360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265190064 ingress_bytes=334053376 link_bytes=334053376 token_bytes=530037424 weight_movement_bytes=69206016 total_nvlink_bytes=599243440 weight_to_token_ratio=0.130568 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257015960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344864 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267420448 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534496032 weight_movement_bytes=69206016 total_nvlink_bytes=603702048 weight_to_token_ratio=0.129479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65200 dup_rows=24050 dup_rows_remote=21124 dedup_ratio=0.244649 router_tokens_per_expert=[777, 774, 823, 767, 828, 761, 760, 767, 756, 811, 722, 764, 757, 742, 731, 707, 799, 791, 732, 762, 785, 738, 772, 723, 811, 795, 755, 779, 783, 776, 755, 800, 795, 816, 715, 790, 757, 797, 772, 798, 731, 761, 769, 773, 730, 739, 755, 740, 721, 793, 753, 806, 778, 771, 799, 759, 751, 741, 791, 790, 754, 778, 769, 757] payload_rows_per_dst=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] dup_rows_per_dst=[3106, 2926, 2908, 3116, 3062, 2912, 3024, 2996] combine_rows_per_src=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341008 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524240 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530707472 weight_movement_bytes=34603008 total_nvlink_bytes=565310480 weight_to_token_ratio=0.065202 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64816 dup_rows=24178 dup_rows_remote=21304 dedup_ratio=0.245951 router_tokens_per_expert=[794, 783, 772, 737, 776, 773, 809, 715, 758, 787, 815, 785, 809, 782, 810, 738, 786, 772, 754, 753, 751, 773, 765, 784, 749, 765, 772, 780, 761, 822, 724, 770, 752, 769, 797, 784, 733, 723, 742, 792, 778, 742, 755, 803, 741, 750, 775, 768, 712, 748, 707, 772, 772, 783, 740, 793, 718, 774, 749, 838, 775, 781, 817, 745] payload_rows_per_dst=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] dup_rows_per_dst=[3072, 3240, 3026, 2978, 2874, 2990, 2940, 3058] combine_rows_per_src=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344512 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266863040 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381568 weight_movement_bytes=69206016 total_nvlink_bytes=602587584 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20878 dedup_ratio=0.244059 router_tokens_per_expert=[754, 779, 782, 779, 741, 774, 763, 824, 716, 720, 821, 764, 711, 734, 793, 758, 748, 747, 803, 759, 791, 755, 731, 805, 712, 825, 723, 772, 777, 800, 770, 787, 765, 744, 790, 814, 783, 740, 786, 750, 798, 799, 783, 780, 800, 740, 757, 809, 778, 791, 771, 764, 778, 727, 770, 736, 764, 763, 773, 761, 745, 748, 826, 701] payload_rows_per_dst=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] dup_rows_per_dst=[2968, 2818, 3068, 3004, 3056, 3114, 3016, 2948] combine_rows_per_src=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343888 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267050832 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533757776 weight_movement_bytes=34603008 total_nvlink_bytes=568360784 weight_to_token_ratio=0.064829 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=345296 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267404496 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463696 weight_movement_bytes=34603008 total_nvlink_bytes=569066704 weight_to_token_ratio=0.064743 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344480 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265830816 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531317152 weight_movement_bytes=69206016 total_nvlink_bytes=600523168 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343088 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148912 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954736 weight_movement_bytes=34603008 total_nvlink_bytes=566557744 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65030 dup_rows=23970 dup_rows_remote=21016 dedup_ratio=0.243835 router_tokens_per_expert=[762, 729, 829, 774, 783, 713, 729, 757, 786, 780, 722, 786, 758, 750, 735, 761, 790, 730, 758, 765, 776, 757, 798, 755, 745, 823, 786, 796, 735, 788, 818, 762, 793, 757, 767, 752, 746, 832, 752, 762, 770, 797, 731, 784, 715, 757, 774, 766, 777, 803, 776, 764, 741, 814, 754, 784, 784, 755, 782, 751, 796, 748, 769, 763] payload_rows_per_dst=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] dup_rows_per_dst=[2940, 2970, 2954, 3078, 3044, 2926, 3094, 2964] combine_rows_per_src=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64892 dup_rows=24194 dup_rows_remote=21134 dedup_ratio=0.246114 router_tokens_per_expert=[763, 765, 798, 782, 765, 747, 766, 733, 782, 750, 753, 780, 775, 763, 790, 797, 809, 791, 754, 740, 763, 719, 767, 740, 769, 744, 792, 791, 747, 729, 774, 793, 772, 775, 761, 794, 763, 749, 760, 783, 774, 788, 802, 728, 739, 794, 764, 772, 755, 777, 737, 767, 712, 766, 758, 804, 785, 748, 791, 726, 769, 827, 788, 793] payload_rows_per_dst=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] dup_rows_per_dst=[3008, 3040, 2942, 3060, 3050, 3054, 2952, 3088] combine_rows_per_src=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64644 dup_rows=24060 dup_rows_remote=20728 dedup_ratio=0.244751 router_tokens_per_expert=[767, 801, 792, 788, 770, 762, 780, 797, 757, 747, 845, 762, 765, 777, 718, 775, 782, 775, 737, 792, 704, 809, 720, 803, 752, 759, 747, 792, 745, 763, 762, 728, 774, 820, 800, 788, 731, 780, 795, 735, 752, 771, 741, 765, 736, 756, 796, 733, 769, 726, 779, 785, 749, 724, 801, 740, 777, 766, 801, 787, 773, 807, 753, 769] payload_rows_per_dst=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] dup_rows_per_dst=[3332, 3028, 2926, 2948, 2890, 2970, 3092, 2874] combine_rows_per_src=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64632 dup_rows=24278 dup_rows_remote=21216 dedup_ratio=0.246969 router_tokens_per_expert=[794, 822, 772, 749, 729, 807, 722, 731, 729, 810, 735, 769, 789, 768, 782, 723, 740, 776, 805, 776, 766, 783, 795, 745, 818, 756, 771, 767, 784, 742, 750, 782, 782, 805, 800, 747, 790, 775, 768, 764, 780, 749, 748, 764, 783, 758, 774, 724, 767, 783, 766, 761, 788, 755, 780, 751, 751, 766, 741, 759, 811, 772, 782, 721] payload_rows_per_dst=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] dup_rows_per_dst=[2730, 3312, 3114, 2686, 3050, 3284, 3040, 3062] combine_rows_per_src=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65176 dup_rows=23820 dup_rows_remote=20810 dedup_ratio=0.242310 router_tokens_per_expert=[752, 788, 806, 788, 800, 803, 766, 784, 749, 739, 760, 745, 792, 773, 778, 736, 760, 764, 793, 783, 731, 775, 744, 784, 740, 736, 747, 773, 774, 802, 769, 785, 775, 706, 808, 736, 765, 741, 746, 770, 786, 785, 797, 785, 778, 732, 759, 750, 748, 760, 752, 782, 767, 776, 784, 790, 817, 785, 761, 779, 720, 784, 740, 769] payload_rows_per_dst=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] dup_rows_per_dst=[3170, 2866, 2990, 2934, 2844, 3006, 3010, 3000] combine_rows_per_src=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344184 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707064 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069944 weight_movement_bytes=34603008 total_nvlink_bytes=567672952 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65112 dup_rows=23916 dup_rows_remote=20936 dedup_ratio=0.243286 router_tokens_per_expert=[791, 750, 780, 766, 805, 784, 793, 798, 821, 742, 705, 770, 769, 778, 763, 780, 763, 762, 741, 754, 762, 788, 768, 801, 786, 776, 756, 771, 765, 726, 799, 816, 789, 822, 722, 745, 744, 796, 793, 737, 743, 775, 722, 765, 765, 750, 786, 735, 769, 739, 770, 768, 793, 754, 720, 801, 737, 798, 788, 737, 821, 739, 767, 733] payload_rows_per_dst=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] dup_rows_per_dst=[3122, 2980, 2978, 3050, 2990, 2850, 3000, 2946] combine_rows_per_src=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64726 dup_rows=24266 dup_rows_remote=21280 dedup_ratio=0.246847 router_tokens_per_expert=[762, 776, 750, 760, 811, 778, 761, 760, 759, 768, 768, 789, 740, 760, 752, 789, 773, 733, 743, 800, 784, 773, 769, 780, 780, 791, 757, 792, 746, 819, 774, 758, 754, 759, 729, 730, 837, 760, 768, 812, 787, 770, 762, 759, 770, 753, 714, 778, 782, 783, 772, 763, 748, 766, 820, 704, 748, 745, 759, 773, 767, 760, 773, 792] payload_rows_per_dst=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] dup_rows_per_dst=[3028, 3048, 3152, 3064, 2986, 2886, 3112, 2990] combine_rows_per_src=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344104 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266141736 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939368 weight_movement_bytes=103809024 total_nvlink_bytes=635748392 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64912 dup_rows=24220 dup_rows_remote=21258 dedup_ratio=0.246379 router_tokens_per_expert=[781, 769, 761, 787, 738, 767, 747, 782, 805, 762, 761, 750, 733, 793, 719, 767, 710, 706, 799, 769, 774, 735, 775, 774, 764, 759, 778, 792, 785, 823, 794, 775, 728, 756, 792, 803, 747, 737, 746, 802, 755, 771, 774, 743, 747, 754, 775, 748, 738, 736, 845, 760, 825, 811, 776, 799, 773, 766, 701, 804, 770, 796, 781, 759] payload_rows_per_dst=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] dup_rows_per_dst=[3012, 3010, 2932, 3106, 2984, 2962, 3158, 3056] combine_rows_per_src=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341488 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123312 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905136 weight_movement_bytes=34603008 total_nvlink_bytes=564508144 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=343392 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265076064 ingress_bytes=333938688 link_bytes=333938688 token_bytes=529808736 weight_movement_bytes=69206016 total_nvlink_bytes=599014752 weight_to_token_ratio=0.130625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=343944 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267304840 ingress_bytes=336166912 link_bytes=336166912 token_bytes=534265736 weight_movement_bytes=69206016 total_nvlink_bytes=603471752 weight_to_token_ratio=0.129535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264969026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344192 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042944 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741696 weight_movement_bytes=34603008 total_nvlink_bytes=568344704 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=344024 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265461720 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530579416 weight_movement_bytes=69206016 total_nvlink_bytes=599785432 weight_to_token_ratio=0.130435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344680 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266224232 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103784 weight_movement_bytes=34603008 total_nvlink_bytes=566706792 weight_to_token_ratio=0.065031 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65054 dup_rows=23930 dup_rows_remote=20840 dedup_ratio=0.243429 router_tokens_per_expert=[780, 779, 784, 809, 762, 762, 749, 812, 762, 739, 738, 755, 793, 713, 741, 756, 764, 820, 788, 775, 746, 778, 754, 780, 749, 799, 753, 709, 780, 710, 756, 755, 789, 797, 768, 725, 784, 798, 752, 759, 773, 739, 760, 742, 809, 800, 798, 767, 772, 767, 784, 757, 824, 758, 789, 780, 755, 739, 775, 746, 746, 793, 778, 779] payload_rows_per_dst=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] dup_rows_per_dst=[3092, 2856, 3090, 2884, 2960, 3050, 3014, 2984] combine_rows_per_src=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65118 dup_rows=23912 dup_rows_remote=20960 dedup_ratio=0.243245 router_tokens_per_expert=[725, 779, 749, 757, 774, 755, 784, 797, 735, 769, 801, 748, 785, 769, 753, 774, 760, 734, 771, 804, 823, 786, 747, 811, 762, 735, 802, 784, 800, 723, 736, 771, 737, 761, 741, 775, 787, 746, 774, 782, 779, 784, 805, 711, 820, 724, 813, 806, 760, 751, 764, 812, 766, 747, 799, 743, 724, 771, 784, 764, 725, 723, 787, 784] payload_rows_per_dst=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] dup_rows_per_dst=[2884, 3002, 3104, 2952, 2944, 3032, 3050, 2944] combine_rows_per_src=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343576 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266804760 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265944 weight_movement_bytes=69206016 total_nvlink_bytes=602471960 weight_to_token_ratio=0.129778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64942 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[812, 817, 791, 798, 763, 745, 758, 786, 743, 767, 745, 787, 776, 757, 721, 788, 765, 790, 734, 757, 749, 753, 763, 795, 747, 765, 789, 734, 833, 806, 773, 765, 728, 808, 760, 784, 790, 718, 792, 762, 776, 743, 757, 752, 760, 764, 764, 763, 820, 742, 768, 751, 744, 798, 754, 734, 758, 763, 772, 774, 758, 798, 724, 801] payload_rows_per_dst=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] dup_rows_per_dst=[3118, 2856, 3174, 3102, 3014, 2978, 2940, 2758] combine_rows_per_src=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64620 dup_rows=24270 dup_rows_remote=21160 dedup_ratio=0.246887 router_tokens_per_expert=[816, 772, 753, 761, 726, 817, 762, 753, 816, 766, 745, 793, 785, 761, 775, 756, 825, 754, 785, 739, 756, 804, 736, 769, 761, 752, 774, 762, 784, 748, 778, 716, 758, 726, 781, 781, 765, 791, 752, 785, 803, 803, 796, 747, 766, 735, 778, 764, 755, 806, 754, 797, 734, 774, 728, 762, 757, 719, 721, 793, 760, 799, 795, 767] payload_rows_per_dst=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] dup_rows_per_dst=[2856, 3076, 2932, 3210, 2854, 3362, 2870, 3110] combine_rows_per_src=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65024 dup_rows=23968 dup_rows_remote=20892 dedup_ratio=0.243815 router_tokens_per_expert=[826, 717, 748, 740, 786, 768, 758, 796, 805, 806, 780, 764, 713, 751, 784, 785, 794, 776, 814, 749, 698, 790, 763, 732, 763, 746, 767, 835, 807, 748, 792, 732, 757, 754, 784, 818, 736, 782, 773, 729, 773, 791, 745, 783, 713, 746, 734, 779, 773, 780, 768, 814, 777, 749, 784, 749, 791, 785, 751, 749, 785, 742, 764, 761] payload_rows_per_dst=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] dup_rows_per_dst=[2984, 3044, 2952, 3076, 2986, 2862, 3076, 2988] combine_rows_per_src=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65114 dup_rows=23990 dup_rows_remote=21106 dedup_ratio=0.244039 router_tokens_per_expert=[727, 759, 774, 780, 803, 786, 771, 828, 736, 752, 811, 769, 734, 717, 797, 726, 781, 814, 693, 757, 783, 755, 800, 765, 712, 773, 757, 774, 730, 777, 768, 769, 795, 764, 782, 761, 799, 730, 787, 784, 814, 772, 762, 786, 814, 736, 755, 678, 757, 742, 820, 754, 812, 756, 768, 818, 784, 789, 738, 729, 787, 758, 768, 775] payload_rows_per_dst=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] dup_rows_per_dst=[3020, 2884, 3024, 2972, 3022, 2966, 3102, 3000] combine_rows_per_src=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65044 dup_rows=24096 dup_rows_remote=21038 dedup_ratio=0.245117 router_tokens_per_expert=[809, 784, 769, 753, 781, 757, 740, 803, 760, 788, 773, 789, 759, 777, 815, 758, 778, 760, 763, 771, 773, 791, 810, 765, 768, 785, 792, 735, 740, 716, 781, 771, 785, 771, 761, 777, 736, 761, 734, 786, 759, 756, 743, 768, 750, 775, 742, 762, 837, 749, 783, 733, 783, 763, 744, 733, 798, 797, 771, 754, 755, 758, 779, 735] payload_rows_per_dst=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] dup_rows_per_dst=[3082, 3090, 3060, 2864, 3058, 3036, 3014, 2892] combine_rows_per_src=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344312 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067640 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790968 weight_movement_bytes=34603008 total_nvlink_bytes=568393976 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65260 dup_rows=23932 dup_rows_remote=21038 dedup_ratio=0.243449 router_tokens_per_expert=[787, 786, 768, 797, 746, 772, 742, 770, 705, 729, 805, 773, 752, 751, 784, 780, 770, 765, 809, 755, 783, 709, 766, 819, 762, 757, 753, 756, 770, 788, 750, 764, 759, 800, 798, 809, 751, 784, 760, 804, 719, 748, 737, 754, 777, 748, 774, 746, 748, 787, 752, 768, 753, 781, 791, 775, 780, 757, 790, 725, 771, 799, 800, 784] payload_rows_per_dst=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] dup_rows_per_dst=[2956, 2876, 3040, 2950, 3102, 2894, 3056, 3058] combine_rows_per_src=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343056 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266345488 ingress_bytes=335208448 link_bytes=335208448 token_bytes=532347920 weight_movement_bytes=69206016 total_nvlink_bytes=601553936 weight_to_token_ratio=0.130001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261362403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264683520 dispatch_meta_bytes=343120 combine_payload_bytes=264683520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265026640 ingress_bytes=299286528 link_bytes=299286528 token_bytes=529710160 weight_movement_bytes=34603008 total_nvlink_bytes=564313168 weight_to_token_ratio=0.065324 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126191110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343664 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266681968 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020272 weight_movement_bytes=69206016 total_nvlink_bytes=602226288 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344880 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267051824 ingress_bytes=335912960 link_bytes=335912960 token_bytes=533758768 weight_movement_bytes=69206016 total_nvlink_bytes=602964784 weight_to_token_ratio=0.129658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264013425e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344328 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764552 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184776 weight_movement_bytes=34603008 total_nvlink_bytes=567787784 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65120 dup_rows=23884 dup_rows_remote=20980 dedup_ratio=0.242961 router_tokens_per_expert=[715, 792, 744, 782, 790, 809, 751, 755, 731, 782, 755, 778, 741, 750, 771, 803, 778, 764, 781, 790, 783, 739, 731, 736, 765, 781, 783, 782, 769, 792, 742, 750, 756, 758, 752, 763, 783, 766, 765, 765, 747, 825, 754, 741, 751, 746, 788, 777, 758, 760, 791, 807, 700, 841, 754, 757, 761, 811, 785, 803, 789, 747, 769, 767] payload_rows_per_dst=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] dup_rows_per_dst=[3106, 3002, 2904, 3010, 2992, 2940, 2938, 2992] combine_rows_per_src=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345192 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650152 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955112 weight_movement_bytes=34603008 total_nvlink_bytes=569558120 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344400 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075920 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807440 weight_movement_bytes=69206016 total_nvlink_bytes=603013456 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65274 dup_rows=23810 dup_rows_remote=20878 dedup_ratio=0.242208 router_tokens_per_expert=[759, 773, 775, 807, 774, 730, 787, 720, 785, 755, 810, 765, 788, 761, 765, 774, 792, 777, 725, 799, 728, 749, 766, 777, 764, 729, 789, 751, 750, 789, 748, 756, 787, 801, 752, 753, 765, 792, 771, 773, 745, 723, 805, 765, 726, 781, 788, 752, 790, 785, 759, 771, 730, 775, 793, 807, 760, 798, 784, 770, 760, 729, 771, 774] payload_rows_per_dst=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] dup_rows_per_dst=[2924, 3042, 2894, 2932, 3012, 2918, 3092, 2996] combine_rows_per_src=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65056 dup_rows=24026 dup_rows_remote=21154 dedup_ratio=0.244405 router_tokens_per_expert=[742, 740, 778, 757, 742, 745, 770, 773, 806, 791, 829, 767, 783, 760, 779, 756, 753, 782, 797, 745, 737, 759, 770, 764, 787, 754, 746, 773, 761, 724, 747, 746, 804, 820, 776, 813, 772, 760, 771, 779, 751, 767, 768, 720, 782, 764, 713, 768, 743, 776, 798, 804, 795, 774, 780, 745, 775, 743, 781, 768, 811, 767, 772, 729] payload_rows_per_dst=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] dup_rows_per_dst=[2872, 3126, 3040, 2884, 3208, 2862, 3000, 3034] combine_rows_per_src=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64816 dup_rows=23988 dup_rows_remote=20922 dedup_ratio=0.244019 router_tokens_per_expert=[753, 777, 776, 811, 778, 786, 753, 749, 777, 709, 774, 804, 799, 747, 762, 754, 773, 731, 822, 719, 770, 773, 737, 772, 747, 759, 747, 764, 775, 785, 773, 824, 765, 753, 772, 785, 753, 745, 767, 788, 786, 812, 778, 775, 723, 755, 755, 765, 726, 770, 770, 778, 762, 696, 788, 770, 833, 734, 772, 773, 790, 807, 775, 751] payload_rows_per_dst=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] dup_rows_per_dst=[2878, 2846, 2958, 3180, 3068, 3038, 2954, 3066] combine_rows_per_src=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64706 dup_rows=24130 dup_rows_remote=20972 dedup_ratio=0.245463 router_tokens_per_expert=[758, 773, 764, 792, 751, 754, 788, 755, 807, 711, 790, 835, 771, 846, 747, 806, 718, 759, 713, 736, 745, 729, 744, 769, 769, 793, 769, 829, 755, 721, 775, 741, 723, 785, 738, 789, 818, 743, 799, 774, 769, 761, 817, 807, 754, 746, 781, 730, 752, 758, 719, 755, 787, 764, 771, 765, 788, 762, 800, 762, 760, 814, 771, 777] payload_rows_per_dst=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] dup_rows_per_dst=[3056, 3158, 2880, 3038, 2990, 2976, 2892, 3140] combine_rows_per_src=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65086 dup_rows=24032 dup_rows_remote=21112 dedup_ratio=0.244466 router_tokens_per_expert=[753, 761, 756, 816, 754, 759, 779, 745, 747, 771, 785, 781, 750, 739, 788, 796, 751, 778, 784, 750, 753, 774, 848, 748, 775, 843, 767, 711, 781, 748, 759, 767, 792, 742, 822, 750, 758, 786, 781, 760, 753, 751, 744, 741, 808, 764, 712, 760, 755, 772, 777, 749, 749, 708, 770, 773, 781, 780, 808, 752, 835, 757, 760, 785] payload_rows_per_dst=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] dup_rows_per_dst=[3020, 3048, 3096, 3018, 3052, 2904, 2920, 2974] combine_rows_per_src=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65030 dup_rows=24012 dup_rows_remote=21014 dedup_ratio=0.244263 router_tokens_per_expert=[771, 765, 758, 705, 817, 775, 741, 781, 760, 799, 756, 765, 725, 744, 733, 829, 772, 743, 744, 798, 805, 780, 736, 723, 770, 750, 757, 814, 764, 742, 781, 745, 752, 785, 790, 765, 719, 796, 744, 779, 779, 810, 745, 733, 741, 812, 723, 744, 832, 766, 789, 808, 793, 714, 815, 773, 784, 783, 775, 809, 750, 776, 732, 788] payload_rows_per_dst=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] dup_rows_per_dst=[2976, 2954, 2950, 2930, 2998, 2990, 3144, 3070] combine_rows_per_src=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65178 dup_rows=23828 dup_rows_remote=20872 dedup_ratio=0.242391 router_tokens_per_expert=[709, 733, 790, 792, 754, 748, 788, 805, 795, 759, 779, 756, 774, 792, 763, 772, 780, 782, 790, 783, 776, 778, 754, 765, 743, 767, 791, 762, 745, 775, 769, 787, 780, 736, 798, 732, 788, 809, 768, 791, 771, 776, 777, 696, 794, 770, 786, 757, 764, 749, 765, 735, 738, 756, 747, 796, 776, 768, 776, 723, 780, 771, 764, 759] payload_rows_per_dst=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] dup_rows_per_dst=[2976, 3008, 3020, 2958, 2996, 2956, 2932, 2982] combine_rows_per_src=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344608 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267706912 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069216 weight_movement_bytes=34603008 total_nvlink_bytes=569672224 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344840 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266814216 ingress_bytes=404881408 link_bytes=404881408 token_bytes=533283592 weight_movement_bytes=138412032 total_nvlink_bytes=671695624 weight_to_token_ratio=0.259547 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523536143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342952 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829288 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531315624 weight_movement_bytes=69206016 total_nvlink_bytes=600521640 weight_to_token_ratio=0.130254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=342712 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265378488 ingress_bytes=368844800 link_bytes=368844800 token_bytes=530414264 weight_movement_bytes=103809024 total_nvlink_bytes=634223288 weight_to_token_ratio=0.195713 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387933288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344792 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266937048 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529304 weight_movement_bytes=34603008 total_nvlink_bytes=568132312 weight_to_token_ratio=0.064857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64928 dup_rows=23832 dup_rows_remote=20834 dedup_ratio=0.242432 router_tokens_per_expert=[747, 751, 770, 771, 800, 759, 766, 807, 757, 761, 730, 776, 790, 765, 797, 762, 799, 827, 739, 795, 763, 787, 776, 785, 712, 749, 771, 762, 729, 726, 822, 773, 765, 744, 774, 789, 760, 755, 767, 784, 768, 791, 798, 755, 770, 773, 796, 765, 796, 742, 766, 742, 707, 759, 774, 788, 789, 779, 737, 755, 732, 758, 760, 790] payload_rows_per_dst=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] dup_rows_per_dst=[3066, 2948, 2998, 2872, 2992, 2946, 2978, 3032] combine_rows_per_src=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344176 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266707056 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069936 weight_movement_bytes=69206016 total_nvlink_bytes=602275952 weight_to_token_ratio=0.129825 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344200 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313288 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282376 weight_movement_bytes=34603008 total_nvlink_bytes=568885384 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259773720 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133845272 ingress_bytes=7371636736 link_bytes=7371636736 total_nvlink_bytes=13505482008 weight_to_token_ratio=0.101609 prefetch_share_of_ingress=0.098576 dedup_ratio=0.244296 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206038068e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233224 weight_movement_bytes=34603008 total_nvlink_bytes=566836232 weight_to_token_ratio=0.065015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12251498072 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6129706584 ingress_bytes=7436705792 link_bytes=7436705792 total_nvlink_bytes=13566412376 weight_to_token_ratio=0.107327 prefetch_share_of_ingress=0.093060 dedup_ratio=0.244757 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216683704e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65244 dup_rows=23864 dup_rows_remote=20986 dedup_ratio=0.242757 router_tokens_per_expert=[728, 733, 784, 728, 791, 779, 806, 771, 787, 745, 705, 790, 777, 744, 701, 779, 745, 793, 830, 747, 778, 778, 734, 787, 717, 750, 761, 715, 744, 787, 802, 761, 786, 777, 803, 742, 768, 768, 759, 794, 779, 782, 745, 776, 752, 796, 786, 788, 713, 748, 781, 759, 755, 777, 805, 792, 783, 783, 771, 756, 759, 828, 771, 793] payload_rows_per_dst=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] dup_rows_per_dst=[2974, 2848, 2980, 2878, 3092, 3054, 2938, 3100] combine_rows_per_src=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65034 dup_rows=23874 dup_rows_remote=20874 dedup_ratio=0.242859 router_tokens_per_expert=[756, 799, 837, 771, 792, 717, 746, 780, 764, 753, 766, 811, 821, 785, 755, 812, 768, 799, 749, 760, 740, 728, 747, 818, 779, 796, 768, 762, 701, 769, 783, 746, 752, 748, 720, 730, 759, 730, 782, 749, 754, 758, 781, 734, 792, 752, 817, 772, 776, 757, 791, 766, 730, 827, 802, 757, 771, 777, 761, 760, 747, 772, 764, 786] payload_rows_per_dst=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] dup_rows_per_dst=[3000, 3156, 2798, 3000, 2860, 3294, 3018, 2748] combine_rows_per_src=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21146 dedup_ratio=0.244893 router_tokens_per_expert=[781, 749, 795, 770, 742, 751, 782, 755, 795, 797, 797, 785, 812, 776, 786, 772, 811, 740, 791, 747, 757, 793, 755, 765, 783, 800, 757, 800, 764, 795, 767, 779, 761, 742, 775, 759, 786, 765, 742, 797, 778, 725, 782, 789, 749, 740, 772, 765, 745, 760, 817, 749, 717, 780, 737, 718, 762, 743, 756, 753, 799, 740, 726, 774] payload_rows_per_dst=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] dup_rows_per_dst=[2976, 2942, 3130, 3130, 2910, 2914, 3144, 2928] combine_rows_per_src=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74614 payload_rows_remote=65202 dup_rows=23690 dup_rows_remote=20652 dedup_ratio=0.240987 router_tokens_per_expert=[800, 721, 742, 794, 740, 784, 733, 775, 734, 780, 742, 788, 764, 805, 726, 754, 770, 808, 734, 789, 819, 719, 784, 815, 768, 769, 788, 771, 726, 788, 769, 761, 782, 795, 769, 752, 739, 731, 771, 775, 744, 765, 779, 776, 778, 768, 805, 775, 764, 776, 797, 742, 781, 797, 784, 784, 749, 768, 776, 760, 717, 738, 754, 801] payload_rows_per_dst=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] dup_rows_per_dst=[2918, 2938, 3008, 2936, 2930, 3020, 3038, 2902] combine_rows_per_src=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65134 dup_rows=23836 dup_rows_remote=20792 dedup_ratio=0.242472 router_tokens_per_expert=[740, 793, 750, 789, 763, 756, 753, 740, 729, 728, 737, 788, 745, 751, 815, 805, 743, 760, 768, 756, 760, 782, 750, 814, 824, 765, 759, 772, 754, 762, 766, 776, 763, 795, 792, 759, 812, 767, 743, 737, 774, 807, 748, 777, 742, 767, 797, 777, 747, 766, 763, 765, 786, 737, 710, 802, 796, 750, 784, 754, 830, 762, 782, 768] payload_rows_per_dst=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] dup_rows_per_dst=[2910, 2944, 3016, 2996, 2992, 3044, 2882, 3052] combine_rows_per_src=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254394568 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6131153096 ingress_bytes=7472758784 link_bytes=7472758784 total_nvlink_bytes=13603911880 weight_to_token_ratio=0.110125 prefetch_share_of_ingress=0.092611 dedup_ratio=0.244128 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.222582161e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344920 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584344 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823768 weight_movement_bytes=34603008 total_nvlink_bytes=569426776 weight_to_token_ratio=0.064700 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343632 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266722896 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102160 weight_movement_bytes=138412032 total_nvlink_bytes=671514192 weight_to_token_ratio=0.259635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344680 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682984 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021288 weight_movement_bytes=69206016 total_nvlink_bytes=602227304 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343416 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267410808 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478200 weight_movement_bytes=34603008 total_nvlink_bytes=569081208 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343704 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267132568 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921432 weight_movement_bytes=103809024 total_nvlink_bytes=637730456 weight_to_token_ratio=0.194428 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267554288 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6137734640 ingress_bytes=7548542976 link_bytes=7548542976 total_nvlink_bytes=13686277616 weight_to_token_ratio=0.115648 prefetch_share_of_ingress=0.105433 dedup_ratio=0.243694 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234980848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12235805256 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6121853512 ingress_bytes=7532675072 link_bytes=7532675072 total_nvlink_bytes=13654528584 weight_to_token_ratio=0.115949 prefetch_share_of_ingress=0.096468 dedup_ratio=0.244628 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232384777e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262685504 prefetch_bytes=622854144 reduce_bytes=865075200 weight_movement_bytes=1487929344 egress_bytes=6135298880 ingress_bytes=7615315968 link_bytes=7615315968 total_nvlink_bytes=13750614848 weight_to_token_ratio=0.121338 prefetch_share_of_ingress=0.081790 dedup_ratio=0.243745 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=25 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245905257e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264870264 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6136394104 ingress_bytes=7270375424 link_bytes=7270375424 total_nvlink_bytes=13406769528 weight_to_token_ratio=0.093103 prefetch_share_of_ingress=0.090430 dedup_ratio=0.244031 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189471192e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=64832 dup_rows=23934 dup_rows_remote=20842 dedup_ratio=0.243469 router_tokens_per_expert=[753, 801, 768, 740, 741, 834, 766, 724, 803, 829, 817, 758, 783, 770, 775, 758, 799, 786, 704, 802, 783, 737, 747, 751, 727, 774, 748, 787, 732, 776, 770, 805, 782, 801, 789, 778, 734, 741, 748, 752, 785, 770, 793, 741, 785, 750, 740, 757, 759, 718, 772, 746, 754, 769, 768, 763, 796, 787, 781, 768, 757, 750, 807, 763] payload_rows_per_dst=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] dup_rows_per_dst=[3092, 3098, 3034, 3054, 2948, 2814, 2924, 2970] combine_rows_per_src=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=342696 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265894568 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531446440 weight_movement_bytes=34603008 total_nvlink_bytes=566049448 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12241040960 prefetch_bytes=553648128 reduce_bytes=1003487232 weight_movement_bytes=1557135360 egress_bytes=6124467776 ingress_bytes=7673708544 link_bytes=7673708544 total_nvlink_bytes=13798176320 weight_to_token_ratio=0.127206 prefetch_share_of_ingress=0.072149 dedup_ratio=0.243948 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=29 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.255458586e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none + [2026-08-29 23:41:52] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 674.7 | throughput per GPU (TFLOP/s/GPU): 189.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.474883E+00 | loss scale: 1.0 | grad norm: 7.081 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65004 dup_rows=23936 dup_rows_remote=20840 dedup_ratio=0.243490 router_tokens_per_expert=[789, 759, 764, 763, 726, 717, 772, 820, 754, 774, 804, 780, 804, 780, 757, 777, 758, 762, 748, 732, 810, 783, 811, 769, 779, 781, 817, 773, 685, 843, 788, 729, 817, 770, 806, 752, 763, 770, 744, 784, 737, 790, 755, 753, 740, 734, 778, 774, 750, 753, 744, 772, 766, 735, 758, 764, 783, 791, 786, 734, 762, 722, 772, 785] payload_rows_per_dst=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] dup_rows_per_dst=[2984, 3096, 2944, 2976, 3066, 3012, 2894, 2964] combine_rows_per_src=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64972 dup_rows=24112 dup_rows_remote=21216 dedup_ratio=0.245280 router_tokens_per_expert=[755, 777, 795, 757, 769, 777, 754, 737, 787, 795, 813, 728, 791, 759, 735, 776, 745, 775, 725, 754, 728, 791, 770, 770, 746, 777, 728, 769, 774, 748, 789, 725, 756, 771, 788, 774, 823, 782, 761, 778, 798, 771, 757, 721, 806, 779, 750, 799, 788, 783, 719, 764, 729, 783, 813, 753, 761, 780, 797, 774, 784, 775, 736, 780] payload_rows_per_dst=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] dup_rows_per_dst=[2952, 3036, 2896, 2936, 3140, 3060, 3046, 3046] combine_rows_per_src=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65032 dup_rows=24054 dup_rows_remote=21120 dedup_ratio=0.244690 router_tokens_per_expert=[756, 733, 753, 797, 773, 751, 730, 746, 768, 788, 770, 763, 736, 755, 829, 825, 771, 775, 776, 767, 807, 772, 822, 749, 781, 730, 732, 790, 800, 754, 713, 776, 788, 769, 764, 761, 784, 768, 763, 739, 743, 776, 792, 777, 786, 773, 766, 734, 762, 769, 739, 788, 730, 765, 728, 799, 760, 772, 761, 760, 790, 780, 780, 798] payload_rows_per_dst=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] dup_rows_per_dst=[2848, 3082, 3106, 2934, 3044, 2968, 3014, 3058] combine_rows_per_src=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65208 dup_rows=23840 dup_rows_remote=20938 dedup_ratio=0.242513 router_tokens_per_expert=[749, 771, 742, 765, 830, 777, 776, 764, 789, 759, 811, 777, 749, 745, 757, 805, 743, 773, 745, 757, 763, 770, 781, 790, 766, 726, 734, 738, 790, 769, 794, 800, 793, 736, 768, 817, 772, 728, 762, 763, 793, 764, 761, 770, 733, 770, 738, 750, 758, 763, 788, 773, 767, 776, 778, 744, 805, 789, 779, 752, 779, 780, 776, 722] payload_rows_per_dst=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] dup_rows_per_dst=[3012, 3052, 2932, 2894, 3004, 2902, 3024, 3020] combine_rows_per_src=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65080 dup_rows=23890 dup_rows_remote=20998 dedup_ratio=0.243022 router_tokens_per_expert=[770, 754, 792, 776, 725, 732, 732, 777, 834, 829, 772, 744, 764, 800, 810, 813, 781, 746, 774, 726, 772, 769, 781, 766, 748, 710, 766, 807, 801, 751, 771, 793, 772, 779, 806, 747, 768, 807, 797, 783, 738, 794, 785, 760, 788, 737, 797, 736, 708, 764, 748, 758, 743, 759, 735, 788, 750, 734, 791, 778, 729, 779, 690, 818] payload_rows_per_dst=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] dup_rows_per_dst=[2902, 3048, 2904, 3146, 2866, 3180, 2952, 2892] combine_rows_per_src=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64958 dup_rows=24112 dup_rows_remote=21086 dedup_ratio=0.245280 router_tokens_per_expert=[777, 780, 759, 786, 787, 761, 732, 793, 784, 748, 763, 767, 772, 760, 756, 733, 777, 757, 807, 754, 760, 727, 796, 763, 714, 795, 766, 736, 768, 737, 761, 738, 724, 735, 802, 732, 815, 752, 793, 777, 759, 776, 801, 792, 778, 716, 807, 769, 821, 800, 762, 782, 809, 782, 768, 720, 837, 707, 765, 789, 783, 780, 737, 768] payload_rows_per_dst=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] dup_rows_per_dst=[2988, 2946, 3068, 2842, 3026, 3066, 3070, 3106] combine_rows_per_src=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64976 dup_rows=24056 dup_rows_remote=21042 dedup_ratio=0.244710 router_tokens_per_expert=[774, 764, 747, 793, 718, 757, 771, 744, 742, 798, 773, 735, 778, 756, 750, 786, 729, 732, 748, 771, 795, 793, 795, 753, 764, 780, 783, 806, 758, 790, 771, 785, 756, 775, 764, 723, 768, 764, 761, 802, 752, 782, 807, 754, 759, 771, 798, 778, 788, 753, 771, 754, 779, 741, 789, 768, 741, 727, 779, 820, 769, 787, 802, 731] payload_rows_per_dst=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] dup_rows_per_dst=[2930, 2940, 2998, 3100, 3020, 3050, 3014, 3004] combine_rows_per_src=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343376 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266599760 ingress_bytes=404668416 link_bytes=404668416 token_bytes=532856144 weight_movement_bytes=138412032 total_nvlink_bytes=671268176 weight_to_token_ratio=0.259755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522734671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266470064 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532595376 weight_movement_bytes=69206016 total_nvlink_bytes=601801392 weight_to_token_ratio=0.129941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344608 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715680 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086752 weight_movement_bytes=34603008 total_nvlink_bytes=567689760 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344584 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436552 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528520 weight_movement_bytes=34603008 total_nvlink_bytes=569131528 weight_to_token_ratio=0.064736 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344312 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266911992 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479672 weight_movement_bytes=34603008 total_nvlink_bytes=568082680 weight_to_token_ratio=0.064863 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344176 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266412144 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532480112 weight_movement_bytes=103809024 total_nvlink_bytes=636289136 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344072 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485768 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627464 weight_movement_bytes=34603008 total_nvlink_bytes=567230472 weight_to_token_ratio=0.064967 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64848 dup_rows=23888 dup_rows_remote=20642 dedup_ratio=0.243001 router_tokens_per_expert=[757, 792, 725, 770, 748, 770, 811, 767, 804, 798, 744, 789, 782, 794, 739, 789, 756, 752, 786, 779, 816, 736, 816, 829, 798, 704, 760, 780, 755, 784, 785, 789, 787, 782, 780, 769, 784, 801, 766, 757, 778, 711, 699, 763, 735, 788, 714, 730, 732, 767, 792, 780, 790, 830, 765, 736, 740, 752, 740, 783, 686, 761, 760, 790] payload_rows_per_dst=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] dup_rows_per_dst=[3246, 2752, 3206, 3020, 3026, 2744, 3032, 2862] combine_rows_per_src=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65122 dup_rows=23962 dup_rows_remote=21060 dedup_ratio=0.243754 router_tokens_per_expert=[809, 790, 759, 772, 785, 796, 778, 765, 738, 731, 748, 785, 754, 735, 796, 774, 787, 785, 777, 792, 767, 764, 791, 764, 817, 781, 764, 785, 772, 769, 741, 749, 730, 749, 786, 827, 774, 762, 723, 823, 767, 742, 767, 735, 780, 740, 733, 740, 738, 758, 776, 769, 766, 779, 749, 743, 756, 800, 770, 755, 817, 739, 777, 762] payload_rows_per_dst=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] dup_rows_per_dst=[2994, 2902, 3186, 3100, 3016, 2812, 2906, 3046] combine_rows_per_src=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20954 dedup_ratio=0.243490 router_tokens_per_expert=[769, 773, 757, 736, 755, 730, 740, 768, 806, 791, 762, 773, 754, 793, 813, 784, 790, 778, 790, 708, 761, 812, 782, 754, 757, 776, 761, 795, 786, 816, 732, 794, 731, 810, 777, 761, 794, 767, 743, 804, 776, 803, 751, 779, 745, 770, 810, 763, 790, 747, 797, 752, 768, 725, 704, 770, 731, 732, 767, 787, 729, 764, 751, 758] payload_rows_per_dst=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] dup_rows_per_dst=[2908, 3152, 3020, 3022, 3052, 2982, 2932, 2868] combine_rows_per_src=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65018 dup_rows=23976 dup_rows_remote=21064 dedup_ratio=0.243896 router_tokens_per_expert=[750, 785, 788, 787, 727, 766, 804, 793, 742, 771, 788, 778, 766, 744, 788, 762, 744, 775, 763, 813, 775, 780, 683, 778, 765, 732, 759, 737, 785, 790, 789, 809, 755, 803, 743, 801, 753, 793, 760, 786, 772, 753, 728, 755, 830, 783, 770, 766, 760, 721, 778, 767, 741, 774, 777, 766, 797, 712, 747, 747, 774, 771, 756, 797] payload_rows_per_dst=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] dup_rows_per_dst=[3028, 2912, 2912, 3110, 3098, 3004, 2920, 2992] combine_rows_per_src=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65300 dup_rows=23770 dup_rows_remote=20852 dedup_ratio=0.241801 router_tokens_per_expert=[753, 754, 756, 810, 798, 737, 807, 763, 800, 783, 792, 805, 779, 724, 756, 732, 790, 740, 808, 753, 756, 778, 732, 813, 742, 775, 724, 766, 783, 791, 760, 735, 725, 763, 779, 746, 791, 757, 764, 760, 754, 786, 742, 785, 752, 787, 774, 744, 784, 787, 728, 793, 768, 779, 720, 771, 750, 819, 770, 807, 776, 792, 746, 758] payload_rows_per_dst=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] dup_rows_per_dst=[2988, 3078, 2956, 2918, 2932, 2984, 2912, 3002] combine_rows_per_src=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=341960 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265959368 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531576776 weight_movement_bytes=34603008 total_nvlink_bytes=566179784 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64872 dup_rows=24062 dup_rows_remote=21022 dedup_ratio=0.244771 router_tokens_per_expert=[791, 770, 751, 796, 760, 748, 767, 778, 792, 811, 787, 796, 766, 779, 779, 821, 721, 759, 740, 759, 775, 737, 796, 723, 754, 800, 768, 739, 747, 752, 771, 765, 776, 752, 789, 738, 763, 778, 797, 732, 745, 768, 715, 738, 788, 741, 797, 809, 780, 822, 741, 755, 723, 731, 785, 789, 762, 770, 763, 818, 768, 791, 767, 763] payload_rows_per_dst=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] dup_rows_per_dst=[3004, 2946, 3156, 2924, 2734, 2944, 3314, 3040] combine_rows_per_src=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64882 dup_rows=24038 dup_rows_remote=20970 dedup_ratio=0.244527 router_tokens_per_expert=[787, 728, 734, 783, 792, 762, 790, 773, 786, 712, 820, 751, 762, 774, 774, 727, 731, 771, 722, 749, 822, 727, 763, 755, 798, 765, 800, 750, 793, 780, 780, 797, 781, 775, 768, 775, 807, 795, 788, 737, 774, 772, 762, 771, 833, 782, 731, 764, 778, 726, 723, 774, 744, 772, 796, 750, 795, 747, 762, 747, 764, 799, 767, 735] payload_rows_per_dst=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] dup_rows_per_dst=[3094, 2860, 2882, 3086, 3068, 3038, 3038, 2972] combine_rows_per_src=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64980 dup_rows=23898 dup_rows_remote=20848 dedup_ratio=0.243103 router_tokens_per_expert=[765, 738, 755, 813, 736, 782, 786, 771, 798, 810, 751, 762, 785, 819, 726, 753, 736, 707, 775, 729, 721, 746, 824, 791, 737, 768, 787, 776, 737, 775, 798, 728, 836, 812, 754, 763, 753, 730, 755, 758, 770, 790, 758, 803, 767, 748, 799, 728, 813, 800, 758, 777, 776, 811, 743, 760, 768, 777, 760, 697, 801, 749, 786, 767] payload_rows_per_dst=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] dup_rows_per_dst=[3004, 3092, 2880, 2962, 3050, 2978, 3050, 2882] combine_rows_per_src=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344728 combine_payload_bytes=266739712 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267084440 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533824152 weight_movement_bytes=138412032 total_nvlink_bytes=672236184 weight_to_token_ratio=0.259284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343640 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266403416 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532463192 weight_movement_bytes=69206016 total_nvlink_bytes=601669208 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344328 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658056 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971784 weight_movement_bytes=34603008 total_nvlink_bytes=567574792 weight_to_token_ratio=0.064925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267468800 dispatch_meta_bytes=344608 combine_payload_bytes=267468800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267813408 ingress_bytes=302071808 link_bytes=302071808 token_bytes=535282208 weight_movement_bytes=34603008 total_nvlink_bytes=569885216 weight_to_token_ratio=0.064644 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136671895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343576 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059288 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775000 weight_movement_bytes=69206016 total_nvlink_bytes=600981016 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343408 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100080 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856752 weight_movement_bytes=69206016 total_nvlink_bytes=601062768 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343312 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266501392 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532659472 weight_movement_bytes=34603008 total_nvlink_bytes=567262480 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64994 dup_rows=24144 dup_rows_remote=21238 dedup_ratio=0.245605 router_tokens_per_expert=[733, 786, 757, 698, 719, 761, 808, 745, 738, 772, 773, 776, 772, 802, 769, 776, 780, 749, 741, 772, 768, 779, 784, 744, 791, 723, 742, 734, 781, 760, 774, 736, 759, 743, 761, 751, 782, 748, 784, 784, 743, 745, 837, 795, 816, 780, 808, 788, 771, 800, 778, 718, 742, 739, 797, 768, 818, 749, 757, 792, 765, 826, 762, 803] payload_rows_per_dst=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] dup_rows_per_dst=[2906, 3070, 2938, 2888, 3184, 3104, 2990, 3064] combine_rows_per_src=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65018 dup_rows=23908 dup_rows_remote=20986 dedup_ratio=0.243205 router_tokens_per_expert=[774, 772, 788, 770, 742, 780, 781, 764, 829, 722, 771, 813, 744, 757, 748, 766, 786, 782, 788, 760, 725, 753, 724, 751, 748, 791, 775, 810, 828, 788, 762, 821, 750, 731, 850, 753, 767, 724, 770, 818, 738, 815, 743, 786, 799, 718, 739, 739, 726, 720, 766, 764, 784, 736, 770, 790, 776, 727, 737, 816, 778, 769, 768, 772] payload_rows_per_dst=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] dup_rows_per_dst=[2968, 2922, 2894, 3118, 3040, 2926, 2998, 3042] combine_rows_per_src=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74586 payload_rows_remote=65302 dup_rows=23718 dup_rows_remote=20728 dedup_ratio=0.241272 router_tokens_per_expert=[801, 762, 786, 764, 811, 728, 747, 768, 758, 781, 778, 810, 752, 797, 792, 746, 740, 798, 743, 757, 788, 728, 776, 807, 758, 799, 746, 758, 754, 826, 764, 760, 754, 744, 784, 796, 759, 780, 746, 768, 782, 747, 761, 744, 752, 737, 760, 811, 741, 780, 810, 777, 771, 748, 783, 747, 742, 808, 774, 750, 751, 767, 725, 770] payload_rows_per_dst=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] dup_rows_per_dst=[3002, 3062, 2990, 2918, 3006, 2950, 2946, 2844] combine_rows_per_src=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65158 dup_rows=23756 dup_rows_remote=20754 dedup_ratio=0.241659 router_tokens_per_expert=[789, 778, 769, 762, 796, 793, 755, 765, 774, 793, 795, 752, 743, 765, 791, 735, 767, 777, 756, 809, 783, 770, 772, 773, 781, 776, 727, 764, 749, 728, 773, 792, 779, 769, 770, 740, 725, 810, 774, 724, 768, 737, 785, 796, 747, 770, 766, 827, 759, 732, 802, 797, 788, 754, 788, 744, 761, 712, 795, 759, 701, 815, 748, 758] payload_rows_per_dst=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] dup_rows_per_dst=[3004, 2858, 3074, 2930, 2944, 3002, 3030, 2914] combine_rows_per_src=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64884 dup_rows=24028 dup_rows_remote=20862 dedup_ratio=0.244425 router_tokens_per_expert=[748, 803, 787, 780, 811, 775, 799, 762, 791, 760, 754, 771, 780, 755, 779, 788, 733, 755, 762, 737, 797, 738, 743, 722, 745, 788, 807, 833, 759, 754, 774, 819, 749, 764, 766, 740, 802, 773, 721, 745, 761, 767, 771, 757, 767, 779, 780, 750, 782, 737, 765, 753, 768, 794, 776, 763, 747, 722, 835, 804, 708, 782, 769, 746] payload_rows_per_dst=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] dup_rows_per_dst=[3126, 2994, 2854, 3166, 2942, 2934, 2946, 3066] combine_rows_per_src=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64848 dup_rows=24170 dup_rows_remote=21080 dedup_ratio=0.245870 router_tokens_per_expert=[784, 787, 721, 762, 825, 759, 783, 779, 761, 766, 753, 753, 738, 741, 733, 795, 735, 729, 773, 785, 766, 815, 799, 776, 781, 795, 716, 754, 729, 776, 743, 752, 744, 804, 724, 827, 769, 767, 793, 760, 773, 772, 766, 833, 729, 760, 796, 805, 748, 694, 800, 778, 713, 788, 742, 815, 775, 749, 756, 761, 748, 781, 793, 825] payload_rows_per_dst=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] dup_rows_per_dst=[3080, 2914, 3072, 2886, 3090, 3080, 2924, 3124] combine_rows_per_src=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64734 dup_rows=24198 dup_rows_remote=20958 dedup_ratio=0.246155 router_tokens_per_expert=[769, 765, 750, 784, 792, 771, 791, 729, 743, 745, 744, 756, 729, 760, 787, 775, 788, 721, 752, 769, 748, 747, 790, 742, 745, 729, 773, 738, 763, 744, 760, 733, 800, 815, 820, 830, 768, 712, 761, 805, 759, 806, 778, 748, 796, 786, 771, 792, 776, 765, 786, 782, 762, 720, 760, 814, 782, 801, 787, 798, 786, 742, 784, 728] payload_rows_per_dst=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] dup_rows_per_dst=[2802, 2892, 3142, 2898, 2784, 3160, 3280, 3240] combine_rows_per_src=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65282 dup_rows=23846 dup_rows_remote=20862 dedup_ratio=0.242574 router_tokens_per_expert=[718, 737, 718, 744, 792, 802, 821, 783, 760, 748, 795, 782, 751, 783, 811, 753, 784, 749, 744, 798, 778, 777, 746, 781, 750, 752, 779, 843, 769, 768, 793, 793, 749, 775, 755, 780, 801, 796, 773, 805, 745, 734, 743, 732, 744, 786, 738, 730, 734, 758, 744, 727, 760, 770, 824, 763, 741, 786, 783, 764, 769, 783, 749, 809] payload_rows_per_dst=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] dup_rows_per_dst=[2916, 2976, 3060, 3154, 3086, 2756, 2984, 2914] combine_rows_per_src=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344016 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657744 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971472 weight_movement_bytes=34603008 total_nvlink_bytes=567574480 weight_to_token_ratio=0.064925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344928 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560352 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775776 weight_movement_bytes=34603008 total_nvlink_bytes=567378784 weight_to_token_ratio=0.064949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=344120 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267821112 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535298104 weight_movement_bytes=34603008 total_nvlink_bytes=569901112 weight_to_token_ratio=0.064643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343648 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267230816 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534117984 weight_movement_bytes=34603008 total_nvlink_bytes=568720992 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342984 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266107848 ingress_bytes=369573888 link_bytes=369573888 token_bytes=531872712 weight_movement_bytes=103809024 total_nvlink_bytes=635681736 weight_to_token_ratio=0.195176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390676787e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343712 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265961120 ingress_bytes=369426432 link_bytes=369426432 token_bytes=531578528 weight_movement_bytes=103809024 total_nvlink_bytes=635387552 weight_to_token_ratio=0.195284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390121922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342768 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493232 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530643696 weight_movement_bytes=34603008 total_nvlink_bytes=565246704 weight_to_token_ratio=0.065209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344576 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739648 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535134720 weight_movement_bytes=34603008 total_nvlink_bytes=569737728 weight_to_token_ratio=0.064662 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65276 dup_rows=23892 dup_rows_remote=21038 dedup_ratio=0.243042 router_tokens_per_expert=[763, 811, 748, 778, 746, 790, 773, 776, 761, 774, 706, 749, 759, 701, 754, 791, 744, 777, 789, 762, 746, 809, 779, 782, 775, 743, 766, 792, 779, 758, 777, 841, 788, 760, 766, 788, 716, 777, 787, 726, 780, 763, 794, 766, 760, 781, 775, 757, 757, 747, 774, 810, 761, 711, 723, 766, 746, 773, 807, 772, 794, 785, 721, 822] payload_rows_per_dst=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] dup_rows_per_dst=[3034, 2854, 2968, 3064, 2960, 2996, 2920, 3096] combine_rows_per_src=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65202 dup_rows=23764 dup_rows_remote=20774 dedup_ratio=0.241740 router_tokens_per_expert=[803, 738, 755, 756, 795, 796, 768, 732, 774, 749, 705, 782, 798, 799, 753, 817, 762, 764, 787, 818, 799, 772, 726, 723, 782, 752, 783, 719, 810, 743, 755, 726, 786, 804, 820, 756, 793, 842, 774, 754, 744, 805, 786, 756, 734, 749, 783, 795, 728, 774, 790, 764, 733, 759, 759, 741, 794, 767, 689, 760, 720, 747, 811, 794] payload_rows_per_dst=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] dup_rows_per_dst=[2990, 3032, 2944, 2772, 3208, 2920, 2956, 2942] combine_rows_per_src=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64816 dup_rows=24166 dup_rows_remote=21092 dedup_ratio=0.245829 router_tokens_per_expert=[797, 829, 776, 742, 739, 739, 731, 767, 793, 790, 763, 784, 776, 781, 815, 778, 781, 753, 733, 761, 752, 794, 759, 795, 729, 770, 763, 755, 780, 761, 779, 759, 810, 766, 792, 705, 733, 723, 782, 792, 772, 749, 801, 764, 764, 770, 759, 819, 829, 718, 707, 761, 805, 755, 732, 737, 762, 795, 804, 766, 750, 795, 728, 783] payload_rows_per_dst=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] dup_rows_per_dst=[3016, 3092, 3042, 2966, 3004, 3074, 2958, 3014] combine_rows_per_src=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65232 dup_rows=23688 dup_rows_remote=20810 dedup_ratio=0.240967 router_tokens_per_expert=[791, 759, 773, 751, 800, 780, 746, 792, 746, 769, 782, 750, 830, 751, 787, 797, 797, 745, 780, 757, 735, 795, 741, 781, 732, 762, 774, 738, 740, 800, 780, 813, 753, 766, 776, 810, 802, 748, 782, 748, 792, 747, 779, 821, 741, 746, 752, 775, 732, 743, 725, 771, 778, 800, 733, 752, 752, 767, 779, 735, 742, 775, 786, 770] payload_rows_per_dst=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] dup_rows_per_dst=[2980, 3048, 2878, 2936, 3044, 2946, 2994, 2862] combine_rows_per_src=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65144 dup_rows=23920 dup_rows_remote=21084 dedup_ratio=0.243327 router_tokens_per_expert=[786, 749, 778, 809, 744, 774, 801, 753, 785, 774, 826, 795, 855, 730, 778, 705, 737, 760, 764, 760, 795, 754, 813, 744, 755, 723, 784, 771, 746, 747, 765, 747, 750, 760, 769, 742, 752, 781, 771, 770, 752, 789, 796, 825, 764, 736, 793, 745, 773, 781, 802, 724, 765, 754, 746, 753, 788, 749, 737, 742, 754, 860, 758, 764] payload_rows_per_dst=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] dup_rows_per_dst=[3006, 3062, 2982, 2836, 3038, 3042, 2988, 2966] combine_rows_per_src=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65090 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[741, 769, 783, 760, 809, 766, 795, 783, 747, 771, 767, 732, 756, 777, 788, 750, 758, 777, 778, 782, 761, 731, 742, 753, 765, 757, 752, 781, 770, 758, 734, 739, 786, 733, 706, 817, 768, 765, 775, 777, 770, 742, 772, 793, 714, 797, 759, 754, 791, 774, 782, 729, 804, 796, 795, 763, 799, 786, 773, 812, 790, 746, 777, 775] payload_rows_per_dst=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] dup_rows_per_dst=[3088, 2900, 2852, 2946, 3074, 2926, 3118, 3130] combine_rows_per_src=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64980 dup_rows=24062 dup_rows_remote=21220 dedup_ratio=0.244771 router_tokens_per_expert=[782, 771, 777, 718, 746, 770, 775, 755, 796, 718, 731, 788, 759, 751, 804, 750, 749, 829, 747, 762, 755, 794, 756, 759, 762, 825, 750, 780, 735, 807, 814, 803, 803, 761, 762, 755, 827, 726, 766, 744, 781, 801, 733, 762, 784, 717, 774, 819, 764, 767, 809, 759, 762, 759, 781, 770, 757, 691, 784, 768, 778, 747, 771, 752] payload_rows_per_dst=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] dup_rows_per_dst=[2844, 2818, 3018, 3136, 3086, 3028, 3290, 2842] combine_rows_per_src=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65098 dup_rows=23982 dup_rows_remote=20894 dedup_ratio=0.243958 router_tokens_per_expert=[741, 782, 783, 775, 780, 751, 767, 763, 760, 860, 738, 697, 794, 822, 757, 754, 771, 801, 772, 714, 790, 756, 767, 738, 765, 787, 797, 744, 751, 724, 757, 756, 739, 738, 784, 770, 762, 779, 773, 772, 785, 796, 740, 737, 792, 764, 778, 774, 742, 837, 789, 707, 775, 800, 756, 750, 799, 747, 769, 790, 762, 774, 774, 784] payload_rows_per_dst=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] dup_rows_per_dst=[3012, 3028, 2932, 2888, 2962, 3012, 3088, 3060] combine_rows_per_src=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=345256 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267715752 ingress_bytes=336576512 link_bytes=336576512 token_bytes=535086248 weight_movement_bytes=69206016 total_nvlink_bytes=604292264 weight_to_token_ratio=0.129336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266510318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=343632 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829968 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531316304 weight_movement_bytes=69206016 total_nvlink_bytes=600522320 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343904 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411296 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478688 weight_movement_bytes=103809024 total_nvlink_bytes=638287712 weight_to_token_ratio=0.194225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344168 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534440 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724712 weight_movement_bytes=34603008 total_nvlink_bytes=569327720 weight_to_token_ratio=0.064712 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344912 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174736 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534004560 weight_movement_bytes=34603008 total_nvlink_bytes=568607568 weight_to_token_ratio=0.064799 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344200 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952840 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561480 weight_movement_bytes=34603008 total_nvlink_bytes=568164488 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344800 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502880 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660960 weight_movement_bytes=69206016 total_nvlink_bytes=601866976 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985376 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626784 weight_movement_bytes=34603008 total_nvlink_bytes=568229792 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65146 dup_rows=23778 dup_rows_remote=20810 dedup_ratio=0.241882 router_tokens_per_expert=[777, 799, 723, 800, 791, 754, 779, 734, 743, 750, 742, 792, 761, 796, 813, 777, 755, 759, 716, 747, 794, 712, 764, 807, 787, 744, 806, 751, 788, 835, 757, 771, 747, 759, 756, 773, 761, 786, 754, 779, 790, 735, 774, 751, 764, 786, 756, 745, 755, 772, 809, 750, 744, 724, 781, 793, 767, 807, 785, 797, 736, 794, 743, 755] payload_rows_per_dst=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] dup_rows_per_dst=[2956, 2968, 2906, 3082, 2972, 2928, 2948, 3018] combine_rows_per_src=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65102 dup_rows=23968 dup_rows_remote=21042 dedup_ratio=0.243815 router_tokens_per_expert=[764, 821, 777, 768, 807, 701, 779, 760, 764, 764, 765, 739, 756, 746, 756, 780, 787, 777, 792, 786, 824, 780, 752, 781, 789, 794, 743, 771, 743, 789, 809, 785, 734, 755, 739, 795, 753, 774, 785, 769, 815, 766, 747, 720, 730, 752, 789, 761, 767, 754, 802, 736, 812, 794, 745, 753, 787, 746, 728, 741, 765, 730, 795, 764] payload_rows_per_dst=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] dup_rows_per_dst=[3054, 2940, 3170, 3070, 2910, 2926, 2968, 2930] combine_rows_per_src=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65298 dup_rows=23852 dup_rows_remote=21036 dedup_ratio=0.242635 router_tokens_per_expert=[758, 694, 816, 764, 779, 805, 787, 769, 792, 751, 758, 750, 759, 737, 775, 768, 785, 802, 765, 741, 783, 794, 809, 779, 766, 784, 740, 720, 757, 754, 713, 751, 738, 775, 739, 774, 745, 753, 738, 795, 771, 784, 762, 768, 756, 790, 860, 792, 783, 779, 792, 732, 802, 745, 747, 786, 781, 740, 738, 795, 858, 720, 769, 740] payload_rows_per_dst=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] dup_rows_per_dst=[3062, 2940, 3090, 2816, 2898, 2962, 3018, 3066] combine_rows_per_src=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=21020 dedup_ratio=0.243754 router_tokens_per_expert=[782, 790, 748, 767, 783, 741, 718, 801, 744, 751, 765, 785, 771, 746, 757, 769, 759, 726, 771, 776, 798, 761, 808, 715, 771, 769, 768, 750, 798, 808, 741, 757, 775, 799, 803, 770, 736, 780, 716, 827, 749, 786, 758, 719, 759, 728, 768, 792, 781, 767, 765, 770, 786, 775, 829, 774, 717, 750, 718, 787, 789, 796, 751, 838] payload_rows_per_dst=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] dup_rows_per_dst=[2966, 2964, 2942, 3118, 3016, 2938, 3014, 3004] combine_rows_per_src=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64886 dup_rows=24176 dup_rows_remote=21178 dedup_ratio=0.245931 router_tokens_per_expert=[761, 745, 760, 757, 757, 791, 759, 790, 772, 793, 762, 806, 773, 762, 778, 775, 748, 734, 821, 726, 746, 765, 777, 771, 757, 776, 729, 763, 832, 728, 736, 747, 780, 747, 761, 788, 799, 758, 794, 805, 736, 746, 789, 772, 760, 770, 798, 775, 799, 724, 745, 773, 813, 788, 784, 721, 763, 753, 768, 779, 759, 743, 779, 786] payload_rows_per_dst=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] dup_rows_per_dst=[2998, 3134, 3094, 3138, 3116, 2816, 3040, 2840] combine_rows_per_src=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65076 dup_rows=24044 dup_rows_remote=21070 dedup_ratio=0.244588 router_tokens_per_expert=[743, 783, 783, 762, 767, 764, 753, 751, 774, 761, 818, 744, 733, 790, 738, 743, 743, 777, 774, 726, 732, 782, 726, 837, 735, 744, 767, 750, 809, 728, 782, 761, 788, 749, 797, 783, 725, 734, 737, 766, 780, 769, 789, 753, 762, 824, 775, 788, 770, 752, 787, 757, 777, 795, 759, 758, 788, 765, 768, 813, 778, 799, 777, 810] payload_rows_per_dst=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] dup_rows_per_dst=[2960, 2884, 2958, 2988, 2974, 3120, 3046, 3114] combine_rows_per_src=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65052 dup_rows=23952 dup_rows_remote=20968 dedup_ratio=0.243652 router_tokens_per_expert=[751, 810, 808, 737, 831, 781, 749, 788, 782, 732, 756, 768, 780, 714, 773, 752, 744, 740, 791, 782, 740, 779, 788, 744, 751, 747, 794, 720, 788, 785, 750, 769, 772, 738, 760, 745, 777, 788, 758, 789, 744, 772, 730, 780, 774, 759, 783, 772, 803, 772, 744, 734, 772, 802, 773, 742, 835, 791, 783, 786, 735, 757, 783, 775] payload_rows_per_dst=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] dup_rows_per_dst=[3088, 2896, 3026, 3004, 3016, 2882, 2984, 3056] combine_rows_per_src=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64884 dup_rows=23948 dup_rows_remote=20804 dedup_ratio=0.243612 router_tokens_per_expert=[800, 750, 732, 775, 765, 774, 742, 815, 780, 796, 741, 765, 687, 782, 767, 782, 822, 806, 756, 729, 762, 741, 737, 782, 749, 771, 768, 783, 701, 775, 814, 776, 775, 756, 765, 757, 797, 737, 786, 796, 755, 748, 782, 756, 800, 714, 801, 751, 779, 782, 748, 758, 750, 736, 742, 748, 790, 766, 808, 775, 814, 782, 777, 796] payload_rows_per_dst=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] dup_rows_per_dst=[2952, 2904, 2994, 3002, 3154, 2962, 2836, 3144] combine_rows_per_src=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343824 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181840 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019856 weight_movement_bytes=34603008 total_nvlink_bytes=568622864 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344576 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002368 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533660160 weight_movement_bytes=69206016 total_nvlink_bytes=602866176 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=345336 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267805944 ingress_bytes=302063616 link_bytes=302063616 token_bytes=535266552 weight_movement_bytes=34603008 total_nvlink_bytes=569869560 weight_to_token_ratio=0.064646 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136641069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344304 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266813680 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533283056 weight_movement_bytes=69206016 total_nvlink_bytes=602489072 weight_to_token_ratio=0.129774 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344256 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266117312 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531890368 weight_movement_bytes=138412032 total_nvlink_bytes=670302400 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344584 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266895880 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447176 weight_movement_bytes=34603008 total_nvlink_bytes=568050184 weight_to_token_ratio=0.064867 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344080 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266797072 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533250064 weight_movement_bytes=34603008 total_nvlink_bytes=567853072 weight_to_token_ratio=0.064891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266107616 ingress_bytes=404176896 link_bytes=404176896 token_bytes=531872480 weight_movement_bytes=138412032 total_nvlink_bytes=670284512 weight_to_token_ratio=0.260235 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520885121e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64918 dup_rows=24096 dup_rows_remote=21088 dedup_ratio=0.245117 router_tokens_per_expert=[735, 772, 787, 816, 741, 763, 782, 796, 796, 798, 767, 770, 818, 703, 737, 760, 816, 760, 776, 758, 845, 786, 777, 797, 812, 798, 772, 754, 755, 739, 787, 799, 746, 761, 801, 799, 773, 715, 759, 760, 724, 771, 731, 773, 780, 720, 810, 729, 766, 718, 753, 776, 765, 786, 785, 766, 751, 750, 752, 762, 721, 758, 764, 755] payload_rows_per_dst=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] dup_rows_per_dst=[2988, 3008, 3232, 3104, 3034, 2782, 2988, 2960] combine_rows_per_src=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64732 dup_rows=24174 dup_rows_remote=20994 dedup_ratio=0.245911 router_tokens_per_expert=[765, 752, 763, 746, 758, 748, 746, 762, 796, 751, 744, 755, 785, 818, 775, 781, 725, 768, 772, 757, 791, 738, 767, 794, 755, 787, 776, 735, 764, 770, 793, 757, 774, 756, 763, 765, 830, 768, 764, 759, 822, 809, 765, 819, 798, 773, 761, 742, 744, 733, 764, 749, 758, 717, 802, 791, 764, 727, 779, 741, 796, 730, 769, 826] payload_rows_per_dst=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] dup_rows_per_dst=[2880, 3116, 2918, 3008, 3016, 3180, 3056, 3000] combine_rows_per_src=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=65288 dup_rows=23566 dup_rows_remote=20586 dedup_ratio=0.239726 router_tokens_per_expert=[759, 745, 740, 771, 743, 731, 809, 767, 818, 766, 757, 780, 763, 745, 794, 798, 796, 761, 728, 766, 767, 794, 778, 785, 766, 756, 800, 791, 792, 785, 724, 801, 750, 793, 729, 753, 820, 784, 802, 743, 745, 794, 738, 750, 799, 775, 784, 750, 739, 740, 816, 813, 714, 781, 780, 736, 741, 822, 740, 764, 728, 718, 765, 770] payload_rows_per_dst=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] dup_rows_per_dst=[2778, 2986, 3024, 2980, 3024, 2998, 2954, 2822] combine_rows_per_src=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64850 dup_rows=24060 dup_rows_remote=21062 dedup_ratio=0.244751 router_tokens_per_expert=[751, 756, 753, 772, 771, 781, 815, 784, 764, 777, 773, 778, 787, 772, 789, 683, 744, 734, 794, 769, 780, 835, 762, 778, 762, 803, 740, 789, 788, 754, 789, 759, 784, 730, 731, 731, 792, 753, 766, 753, 774, 770, 752, 772, 753, 755, 748, 759, 758, 762, 730, 754, 790, 758, 791, 787, 777, 754, 770, 807, 775, 808, 748, 774] payload_rows_per_dst=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] dup_rows_per_dst=[3050, 3028, 2998, 3010, 2980, 2986, 2958, 3050] combine_rows_per_src=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64900 dup_rows=24186 dup_rows_remote=21154 dedup_ratio=0.246033 router_tokens_per_expert=[812, 748, 772, 754, 776, 774, 776, 760, 769, 793, 794, 731, 755, 776, 765, 776, 756, 759, 779, 737, 773, 744, 763, 775, 775, 796, 782, 787, 763, 751, 722, 759, 793, 775, 808, 744, 750, 768, 737, 750, 758, 769, 773, 757, 781, 742, 768, 750, 805, 754, 767, 792, 787, 765, 794, 765, 759, 769, 772, 776, 745, 763, 777, 787] payload_rows_per_dst=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] dup_rows_per_dst=[3104, 3004, 2920, 3044, 3032, 2974, 3138, 2970] combine_rows_per_src=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64820 dup_rows=24132 dup_rows_remote=21020 dedup_ratio=0.245483 router_tokens_per_expert=[747, 755, 776, 783, 757, 756, 822, 750, 759, 774, 727, 750, 766, 769, 752, 788, 751, 740, 772, 773, 746, 762, 733, 823, 797, 735, 759, 828, 764, 733, 770, 770, 735, 775, 766, 794, 747, 752, 768, 787, 804, 745, 761, 794, 826, 726, 765, 754, 748, 750, 753, 762, 780, 837, 801, 801, 789, 770, 760, 740, 771, 734, 768, 802] payload_rows_per_dst=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] dup_rows_per_dst=[2954, 2912, 2964, 3048, 3012, 3020, 3112, 3110] combine_rows_per_src=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65098 dup_rows=23920 dup_rows_remote=20912 dedup_ratio=0.243327 router_tokens_per_expert=[773, 712, 755, 781, 759, 763, 799, 752, 772, 735, 798, 750, 800, 780, 758, 762, 778, 789, 752, 790, 796, 748, 767, 726, 725, 751, 765, 727, 777, 816, 804, 782, 809, 792, 755, 708, 792, 770, 767, 757, 767, 750, 795, 804, 792, 801, 769, 756, 752, 799, 730, 789, 756, 754, 741, 782, 789, 778, 746, 769, 738, 743, 781, 779] payload_rows_per_dst=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] dup_rows_per_dst=[3008, 3284, 2992, 3140, 2940, 3034, 2926, 2596] combine_rows_per_src=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64658 dup_rows=24112 dup_rows_remote=20844 dedup_ratio=0.245280 router_tokens_per_expert=[738, 781, 735, 811, 775, 732, 750, 813, 790, 720, 779, 744, 792, 782, 811, 754, 753, 708, 785, 736, 724, 792, 777, 774, 819, 774, 764, 783, 751, 764, 797, 775, 743, 728, 782, 746, 795, 757, 778, 758, 800, 758, 815, 694, 742, 784, 796, 735, 780, 758, 783, 760, 710, 809, 787, 746, 748, 724, 792, 768, 793, 804, 802, 794] payload_rows_per_dst=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] dup_rows_per_dst=[2882, 3072, 2934, 2702, 3046, 3112, 3096, 3268] combine_rows_per_src=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343648 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969248 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594848 weight_movement_bytes=34603008 total_nvlink_bytes=566197856 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344024 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248152 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152280 weight_movement_bytes=69206016 total_nvlink_bytes=601358296 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342904 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485176 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627448 weight_movement_bytes=34603008 total_nvlink_bytes=565230456 weight_to_token_ratio=0.065211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=343496 combine_payload_bytes=267419648 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267763144 ingress_bytes=440434688 link_bytes=440434688 token_bytes=535182792 weight_movement_bytes=173015040 total_nvlink_bytes=708197832 weight_to_token_ratio=0.323282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657320273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344040 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266985448 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626856 weight_movement_bytes=69206016 total_nvlink_bytes=602832872 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344216 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174616 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005016 weight_movement_bytes=34603008 total_nvlink_bytes=566608024 weight_to_token_ratio=0.065043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343360 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846080 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348800 weight_movement_bytes=34603008 total_nvlink_bytes=565951808 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342008 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181176 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530020344 weight_movement_bytes=34603008 total_nvlink_bytes=564623352 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20982 dedup_ratio=0.243876 router_tokens_per_expert=[781, 780, 758, 744, 799, 818, 762, 782, 735, 737, 759, 753, 793, 783, 791, 742, 765, 714, 798, 769, 777, 772, 778, 781, 759, 760, 762, 805, 817, 769, 771, 798, 783, 791, 764, 813, 771, 723, 782, 783, 741, 789, 751, 782, 778, 714, 747, 742, 752, 789, 786, 738, 804, 794, 731, 764, 743, 740, 735, 768, 787, 785, 750, 720] payload_rows_per_dst=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] dup_rows_per_dst=[3078, 3022, 2992, 3050, 3106, 2926, 2928, 2872] combine_rows_per_src=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65240 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[754, 725, 771, 794, 787, 764, 784, 775, 789, 694, 788, 706, 763, 775, 768, 778, 710, 727, 774, 734, 765, 792, 735, 791, 773, 799, 759, 803, 776, 730, 775, 779, 807, 812, 801, 784, 744, 785, 767, 806, 744, 718, 745, 724, 780, 744, 794, 773, 763, 758, 789, 753, 769, 760, 778, 795, 761, 787, 745, 803, 788, 788, 801, 749] payload_rows_per_dst=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] dup_rows_per_dst=[3022, 2884, 2794, 3058, 3088, 2868, 3012, 3100] combine_rows_per_src=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64990 dup_rows=23988 dup_rows_remote=20872 dedup_ratio=0.244019 router_tokens_per_expert=[746, 770, 745, 757, 762, 739, 783, 750, 770, 774, 755, 759, 749, 801, 733, 722, 800, 841, 754, 786, 748, 787, 774, 781, 805, 763, 818, 785, 762, 771, 740, 755, 802, 775, 728, 765, 769, 758, 777, 757, 795, 805, 806, 790, 788, 729, 771, 737, 759, 770, 735, 778, 768, 785, 700, 735, 745, 786, 775, 755, 770, 782, 775, 797] payload_rows_per_dst=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] dup_rows_per_dst=[2938, 2958, 3096, 2990, 3008, 3116, 2850, 3032] combine_rows_per_src=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64956 dup_rows=24086 dup_rows_remote=21020 dedup_ratio=0.245015 router_tokens_per_expert=[764, 723, 782, 766, 802, 769, 794, 755, 753, 770, 792, 709, 759, 787, 806, 734, 781, 746, 765, 728, 785, 743, 792, 773, 734, 739, 770, 805, 763, 779, 794, 780, 787, 792, 789, 754, 746, 739, 757, 762, 792, 771, 777, 800, 810, 777, 747, 754, 794, 770, 709, 746, 732, 743, 809, 740, 751, 743, 779, 775, 805, 779, 810, 771] payload_rows_per_dst=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] dup_rows_per_dst=[2996, 2942, 2956, 3066, 2968, 3126, 2992, 3040] combine_rows_per_src=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65234 dup_rows=23844 dup_rows_remote=20936 dedup_ratio=0.242554 router_tokens_per_expert=[778, 784, 744, 761, 783, 788, 770, 757, 785, 808, 758, 816, 785, 747, 791, 749, 746, 772, 766, 753, 748, 772, 694, 803, 790, 780, 760, 752, 747, 736, 761, 769, 724, 771, 730, 787, 749, 742, 776, 788, 798, 788, 767, 761, 718, 725, 776, 832, 764, 734, 811, 783, 806, 815, 779, 762, 715, 777, 813, 789, 743, 730, 772, 774] payload_rows_per_dst=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] dup_rows_per_dst=[3046, 3152, 2850, 2934, 2908, 3008, 3032, 2914] combine_rows_per_src=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64966 dup_rows=23956 dup_rows_remote=20794 dedup_ratio=0.243693 router_tokens_per_expert=[730, 813, 765, 827, 808, 771, 775, 775, 764, 729, 790, 761, 740, 795, 733, 768, 738, 778, 768, 788, 731, 723, 809, 731, 780, 773, 776, 808, 750, 786, 777, 796, 786, 784, 771, 783, 760, 778, 791, 782, 746, 725, 802, 745, 757, 804, 792, 744, 750, 752, 774, 754, 783, 782, 761, 727, 773, 776, 761, 743, 746, 743, 767, 754] payload_rows_per_dst=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] dup_rows_per_dst=[3162, 2904, 2884, 3092, 3112, 3014, 2872, 2916] combine_rows_per_src=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65062 dup_rows=23858 dup_rows_remote=20780 dedup_ratio=0.242696 router_tokens_per_expert=[782, 769, 761, 749, 732, 796, 758, 761, 761, 722, 763, 718, 738, 813, 806, 755, 711, 737, 750, 781, 793, 791, 788, 742, 818, 764, 805, 788, 771, 837, 765, 752, 797, 723, 777, 762, 759, 805, 787, 776, 760, 774, 764, 779, 801, 701, 730, 755, 828, 781, 749, 799, 776, 806, 794, 698, 749, 789, 772, 765, 796, 696, 753, 774] payload_rows_per_dst=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] dup_rows_per_dst=[2938, 2884, 2946, 3148, 3094, 2854, 3078, 2916] combine_rows_per_src=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64932 dup_rows=23962 dup_rows_remote=20846 dedup_ratio=0.243754 router_tokens_per_expert=[778, 741, 816, 736, 763, 806, 777, 748, 778, 799, 797, 754, 794, 786, 731, 791, 762, 811, 805, 747, 765, 754, 784, 794, 731, 787, 765, 721, 756, 730, 752, 785, 768, 739, 778, 800, 778, 806, 799, 763, 761, 757, 752, 819, 742, 794, 777, 805, 761, 772, 722, 749, 713, 760, 742, 754, 804, 773, 734, 777, 705, 770, 773, 761] payload_rows_per_dst=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] dup_rows_per_dst=[2946, 3122, 3028, 2802, 2916, 3054, 2978, 3116] combine_rows_per_src=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343984 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641328 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938672 weight_movement_bytes=34603008 total_nvlink_bytes=567541680 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343448 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542488 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741528 weight_movement_bytes=34603008 total_nvlink_bytes=567344536 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344680 combine_payload_bytes=267198464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267543144 ingress_bytes=405610496 link_bytes=405610496 token_bytes=534741608 weight_movement_bytes=138412032 total_nvlink_bytes=673153640 weight_to_token_ratio=0.258839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526279642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343040 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266443776 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532544512 weight_movement_bytes=69206016 total_nvlink_bytes=601750528 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343368 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266837320 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533331272 weight_movement_bytes=34603008 total_nvlink_bytes=567934280 weight_to_token_ratio=0.064881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343112 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304584 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266056 weight_movement_bytes=34603008 total_nvlink_bytes=566869064 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65000 dup_rows=24036 dup_rows_remote=21062 dedup_ratio=0.244507 router_tokens_per_expert=[769, 791, 776, 795, 782, 766, 771, 767, 791, 734, 792, 799, 744, 753, 776, 756, 763, 735, 796, 779, 766, 753, 743, 786, 798, 794, 698, 800, 811, 751, 769, 759, 789, 736, 744, 778, 761, 716, 766, 750, 783, 794, 787, 753, 763, 787, 706, 780, 807, 773, 749, 766, 773, 789, 756, 791, 744, 787, 752, 780, 741, 771, 756, 761] payload_rows_per_dst=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] dup_rows_per_dst=[3126, 3030, 2974, 2966, 2894, 2924, 3094, 3028] combine_rows_per_src=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65030 dup_rows=23934 dup_rows_remote=20838 dedup_ratio=0.243469 router_tokens_per_expert=[768, 723, 775, 760, 792, 765, 791, 785, 774, 817, 799, 786, 708, 776, 774, 784, 780, 754, 796, 759, 758, 725, 792, 800, 738, 838, 778, 799, 708, 761, 764, 777, 783, 750, 788, 792, 681, 787, 781, 764, 780, 730, 731, 737, 768, 755, 813, 768, 743, 764, 755, 771, 758, 728, 796, 757, 765, 764, 768, 784, 801, 783, 776, 727] payload_rows_per_dst=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] dup_rows_per_dst=[3048, 3096, 3040, 2996, 2954, 2948, 2832, 3020] combine_rows_per_src=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65264 dup_rows=23898 dup_rows_remote=20916 dedup_ratio=0.243103 router_tokens_per_expert=[782, 734, 804, 738, 778, 729, 778, 782, 780, 799, 759, 785, 789, 784, 764, 738, 755, 795, 712, 786, 816, 768, 797, 744, 783, 751, 766, 756, 801, 742, 802, 817, 809, 767, 776, 808, 748, 778, 767, 757, 755, 744, 746, 754, 757, 741, 821, 744, 754, 772, 766, 776, 746, 795, 750, 772, 807, 729, 725, 760, 746, 734, 778, 756] payload_rows_per_dst=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] dup_rows_per_dst=[2996, 2976, 3004, 3114, 2896, 2982, 3002, 2928] combine_rows_per_src=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21068 dedup_ratio=0.244161 router_tokens_per_expert=[822, 760, 774, 783, 739, 770, 740, 758, 777, 759, 761, 751, 802, 756, 771, 760, 744, 761, 778, 751, 715, 776, 749, 801, 780, 773, 720, 758, 745, 811, 764, 769, 754, 846, 809, 780, 810, 749, 760, 788, 722, 748, 800, 775, 765, 753, 762, 771, 693, 793, 775, 792, 782, 732, 761, 767, 781, 732, 733, 761, 794, 798, 794, 794] payload_rows_per_dst=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] dup_rows_per_dst=[3018, 2996, 2914, 2934, 3144, 2962, 3010, 3024] combine_rows_per_src=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65146 dup_rows=23902 dup_rows_remote=20848 dedup_ratio=0.243144 router_tokens_per_expert=[735, 789, 775, 742, 779, 757, 804, 763, 769, 755, 813, 801, 768, 773, 780, 740, 745, 759, 742, 739, 819, 748, 786, 761, 734, 761, 752, 785, 780, 777, 754, 771, 742, 771, 761, 797, 805, 759, 761, 759, 748, 777, 713, 815, 739, 759, 781, 740, 769, 815, 804, 822, 772, 755, 785, 774, 831, 805, 726, 769, 742, 723, 769, 708] payload_rows_per_dst=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] dup_rows_per_dst=[2928, 3034, 2972, 2986, 3054, 2950, 3092, 2886] combine_rows_per_src=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64906 dup_rows=24016 dup_rows_remote=21058 dedup_ratio=0.244303 router_tokens_per_expert=[741, 830, 766, 766, 765, 776, 758, 756, 832, 719, 745, 771, 776, 748, 773, 749, 746, 823, 728, 779, 768, 726, 821, 762, 739, 767, 767, 846, 787, 791, 714, 746, 778, 826, 755, 811, 736, 765, 715, 794, 773, 775, 733, 756, 798, 786, 798, 741, 805, 775, 753, 766, 769, 765, 784, 780, 753, 749, 742, 748, 734, 805, 733, 770] payload_rows_per_dst=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] dup_rows_per_dst=[2958, 3160, 3144, 2982, 2882, 2988, 3020, 2882] combine_rows_per_src=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344248 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584248 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824248 weight_movement_bytes=34603008 total_nvlink_bytes=567427256 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64988 dup_rows=24002 dup_rows_remote=20834 dedup_ratio=0.244161 router_tokens_per_expert=[722, 731, 758, 803, 753, 744, 760, 819, 745, 786, 746, 751, 740, 749, 759, 780, 763, 772, 754, 776, 771, 763, 735, 784, 758, 818, 767, 766, 779, 819, 759, 797, 747, 790, 797, 787, 758, 718, 771, 786, 749, 771, 691, 734, 767, 760, 788, 813, 777, 777, 805, 772, 760, 792, 743, 815, 786, 801, 785, 751, 714, 772, 801, 747] payload_rows_per_dst=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] dup_rows_per_dst=[2976, 2818, 2966, 3128, 3012, 2948, 3168, 2986] combine_rows_per_src=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64726 dup_rows=24112 dup_rows_remote=20872 dedup_ratio=0.245280 router_tokens_per_expert=[753, 745, 724, 778, 769, 733, 766, 808, 746, 776, 776, 758, 747, 770, 771, 790, 778, 774, 764, 751, 717, 779, 749, 776, 798, 784, 755, 736, 799, 763, 810, 757, 732, 747, 817, 749, 760, 792, 825, 802, 746, 740, 742, 804, 835, 789, 804, 750, 741, 725, 787, 762, 779, 756, 774, 781, 724, 780, 784, 760, 806, 709, 767, 783] payload_rows_per_dst=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] dup_rows_per_dst=[2968, 3006, 3030, 2770, 2850, 3350, 2898, 3240] combine_rows_per_src=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343472 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266706352 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069232 weight_movement_bytes=69206016 total_nvlink_bytes=602275248 weight_to_token_ratio=0.129826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65240 dup_rows=23834 dup_rows_remote=20942 dedup_ratio=0.242452 router_tokens_per_expert=[700, 761, 826, 763, 772, 787, 784, 757, 741, 795, 775, 718, 791, 706, 786, 754, 776, 759, 745, 743, 766, 772, 726, 774, 772, 784, 754, 796, 788, 730, 747, 765, 773, 762, 785, 729, 743, 819, 787, 816, 757, 757, 765, 807, 848, 755, 714, 768, 791, 738, 767, 755, 769, 788, 761, 773, 768, 744, 748, 802, 769, 774, 771, 836] payload_rows_per_dst=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] dup_rows_per_dst=[3020, 2900, 2892, 2980, 3004, 2942, 2988, 3108] combine_rows_per_src=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344720 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267666064 ingress_bytes=301924352 link_bytes=301924352 token_bytes=534987408 weight_movement_bytes=34603008 total_nvlink_bytes=569590416 weight_to_token_ratio=0.064680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136117030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344256 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567872 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791488 weight_movement_bytes=103809024 total_nvlink_bytes=636600512 weight_to_token_ratio=0.194840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343976 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181992 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534020008 weight_movement_bytes=69206016 total_nvlink_bytes=603226024 weight_to_token_ratio=0.129594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343856 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266198832 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532053808 weight_movement_bytes=69206016 total_nvlink_bytes=601259824 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343288 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266534136 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532724984 weight_movement_bytes=103809024 total_nvlink_bytes=636534008 weight_to_token_ratio=0.194864 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342392 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265460088 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577784 weight_movement_bytes=34603008 total_nvlink_bytes=565180792 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64862 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[727, 744, 736, 795, 746, 741, 725, 746, 766, 742, 789, 774, 757, 797, 755, 795, 850, 759, 777, 775, 775, 765, 812, 784, 745, 760, 780, 750, 805, 747, 788, 728, 793, 773, 763, 784, 826, 810, 786, 718, 767, 762, 723, 746, 770, 759, 741, 785, 797, 748, 753, 777, 748, 794, 785, 730, 758, 765, 778, 784, 803, 736, 806, 749] payload_rows_per_dst=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] dup_rows_per_dst=[2832, 3046, 3160, 2948, 3208, 2886, 3028, 3030] combine_rows_per_src=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65054 dup_rows=23940 dup_rows_remote=20800 dedup_ratio=0.243530 router_tokens_per_expert=[736, 779, 727, 785, 737, 722, 746, 805, 772, 771, 778, 800, 768, 742, 816, 770, 777, 735, 743, 755, 777, 719, 745, 741, 763, 713, 775, 748, 777, 743, 778, 758, 765, 740, 804, 786, 763, 772, 789, 757, 761, 783, 790, 750, 764, 811, 755, 811, 784, 785, 824, 765, 783, 767, 765, 757, 775, 793, 767, 794, 772, 779, 781, 759] payload_rows_per_dst=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] dup_rows_per_dst=[2866, 2916, 2874, 2878, 3070, 3140, 3100, 3096] combine_rows_per_src=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64968 dup_rows=24020 dup_rows_remote=20906 dedup_ratio=0.244344 router_tokens_per_expert=[785, 807, 795, 749, 780, 803, 749, 762, 793, 727, 801, 817, 763, 766, 750, 785, 763, 746, 782, 743, 776, 792, 762, 797, 752, 828, 806, 757, 737, 752, 796, 787, 758, 749, 752, 745, 733, 766, 767, 837, 745, 859, 773, 719, 777, 746, 807, 768, 750, 755, 795, 731, 743, 745, 795, 757, 685, 775, 738, 766, 734, 791, 750, 733] payload_rows_per_dst=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] dup_rows_per_dst=[3138, 3142, 3000, 3114, 2884, 3064, 2866, 2812] combine_rows_per_src=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65292 dup_rows=23780 dup_rows_remote=20894 dedup_ratio=0.241903 router_tokens_per_expert=[779, 812, 771, 733, 794, 788, 795, 757, 781, 748, 753, 758, 725, 742, 765, 757, 761, 741, 788, 813, 765, 744, 798, 759, 745, 806, 753, 818, 768, 735, 772, 743, 771, 761, 741, 801, 739, 749, 766, 731, 816, 791, 718, 781, 746, 755, 810, 763, 735, 751, 833, 788, 787, 784, 759, 778, 740, 758, 699, 785, 782, 779, 808, 780] payload_rows_per_dst=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] dup_rows_per_dst=[3088, 2858, 2996, 3008, 2886, 2920, 3030, 2994] combine_rows_per_src=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64914 dup_rows=24024 dup_rows_remote=21000 dedup_ratio=0.244385 router_tokens_per_expert=[727, 786, 774, 781, 726, 798, 786, 750, 737, 808, 737, 775, 742, 758, 771, 746, 752, 796, 783, 746, 774, 747, 797, 782, 791, 736, 786, 777, 766, 795, 803, 757, 775, 732, 744, 751, 778, 802, 782, 784, 752, 761, 760, 762, 774, 773, 761, 805, 747, 805, 755, 764, 800, 814, 763, 731, 812, 777, 786, 745, 731, 759, 742, 735] payload_rows_per_dst=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] dup_rows_per_dst=[3024, 2964, 3172, 2986, 3090, 2968, 2876, 2944] combine_rows_per_src=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64980 dup_rows=23860 dup_rows_remote=20862 dedup_ratio=0.242716 router_tokens_per_expert=[769, 746, 731, 774, 822, 790, 751, 766, 750, 733, 772, 750, 749, 789, 736, 817, 823, 739, 754, 787, 830, 762, 792, 768, 755, 765, 760, 738, 790, 747, 783, 754, 761, 767, 752, 735, 768, 734, 755, 775, 748, 804, 794, 748, 820, 744, 758, 752, 788, 791, 791, 784, 804, 802, 744, 727, 759, 768, 744, 732, 795, 797, 750, 769] payload_rows_per_dst=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] dup_rows_per_dst=[2984, 2980, 3064, 2914, 2862, 3058, 2998, 3000] combine_rows_per_src=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20790 dedup_ratio=0.243083 router_tokens_per_expert=[797, 765, 744, 764, 744, 757, 718, 789, 781, 735, 769, 793, 730, 778, 774, 826, 781, 759, 796, 792, 771, 797, 747, 735, 808, 761, 774, 765, 757, 735, 828, 805, 733, 767, 746, 807, 739, 779, 730, 777, 757, 806, 786, 740, 805, 769, 750, 721, 748, 759, 812, 751, 777, 745, 763, 758, 751, 825, 744, 804, 771, 740, 755, 762] payload_rows_per_dst=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] dup_rows_per_dst=[2846, 3100, 3034, 3024, 2904, 2854, 3028, 3106] combine_rows_per_src=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343816 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266018568 ingress_bytes=334880768 link_bytes=334880768 token_bytes=531693320 weight_movement_bytes=69206016 total_nvlink_bytes=600899336 weight_to_token_ratio=0.130162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260129370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65016 dup_rows=24002 dup_rows_remote=20954 dedup_ratio=0.244161 router_tokens_per_expert=[731, 780, 773, 841, 784, 725, 750, 732, 780, 761, 759, 787, 783, 722, 758, 775, 801, 817, 748, 749, 759, 766, 743, 784, 795, 779, 742, 831, 782, 743, 730, 764, 761, 761, 744, 737, 741, 775, 770, 735, 792, 797, 735, 815, 763, 797, 755, 826, 764, 814, 743, 751, 752, 780, 759, 799, 772, 743, 766, 757, 769, 766, 773, 766] payload_rows_per_dst=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] dup_rows_per_dst=[2960, 3054, 3048, 3022, 2844, 3114, 3032, 2928] combine_rows_per_src=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343416 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266804600 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533265784 weight_movement_bytes=103809024 total_nvlink_bytes=637074808 weight_to_token_ratio=0.194667 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343496 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452424 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561352 weight_movement_bytes=34603008 total_nvlink_bytes=567164360 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344744 combine_payload_bytes=267436032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267780776 ingress_bytes=371245056 link_bytes=371245056 token_bytes=535216808 weight_movement_bytes=103809024 total_nvlink_bytes=639025832 weight_to_token_ratio=0.193957 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343656 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231400 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119144 weight_movement_bytes=34603008 total_nvlink_bytes=566722152 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343368 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266501448 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659528 weight_movement_bytes=69206016 total_nvlink_bytes=601865544 weight_to_token_ratio=0.129925 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343232 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266656960 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532970688 weight_movement_bytes=34603008 total_nvlink_bytes=567573696 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65154 dup_rows=23944 dup_rows_remote=20938 dedup_ratio=0.243571 router_tokens_per_expert=[765, 794, 761, 769, 722, 750, 768, 827, 752, 798, 752, 804, 774, 724, 752, 750, 780, 755, 768, 743, 741, 773, 788, 822, 778, 726, 772, 785, 756, 729, 732, 753, 759, 774, 797, 733, 795, 759, 789, 769, 836, 737, 793, 750, 773, 773, 773, 783, 778, 781, 750, 765, 789, 770, 732, 768, 747, 782, 802, 772, 795, 769, 776, 720] payload_rows_per_dst=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] dup_rows_per_dst=[2990, 3006, 2944, 2916, 3004, 3062, 3092, 2930] combine_rows_per_src=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65052 dup_rows=23780 dup_rows_remote=20682 dedup_ratio=0.241903 router_tokens_per_expert=[762, 776, 751, 809, 769, 773, 805, 792, 765, 776, 727, 769, 752, 708, 737, 757, 754, 822, 762, 768, 789, 815, 778, 739, 801, 748, 789, 796, 815, 796, 774, 766, 785, 785, 778, 724, 808, 781, 760, 771, 729, 739, 724, 795, 720, 757, 759, 780, 748, 753, 756, 784, 804, 732, 796, 778, 723, 712, 796, 755, 775, 772, 775, 758] payload_rows_per_dst=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] dup_rows_per_dst=[3106, 2816, 3020, 3098, 3006, 2824, 2978, 2932] combine_rows_per_src=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=65018 dup_rows=24102 dup_rows_remote=21188 dedup_ratio=0.245178 router_tokens_per_expert=[798, 739, 790, 761, 749, 809, 801, 761, 752, 757, 760, 761, 773, 747, 797, 738, 760, 783, 737, 800, 763, 731, 800, 772, 768, 785, 767, 757, 725, 730, 732, 750, 775, 787, 802, 787, 795, 759, 786, 769, 736, 771, 735, 762, 783, 783, 731, 748, 748, 774, 799, 751, 759, 772, 759, 795, 762, 710, 791, 797, 819, 775, 787, 792] payload_rows_per_dst=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] dup_rows_per_dst=[3034, 3002, 3040, 2920, 3098, 2914, 3068, 3026] combine_rows_per_src=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74060 payload_rows_remote=64714 dup_rows=24244 dup_rows_remote=21200 dedup_ratio=0.246623 router_tokens_per_expert=[766, 744, 699, 757, 785, 781, 755, 780, 811, 749, 763, 789, 768, 721, 768, 710, 793, 764, 769, 784, 780, 792, 699, 772, 803, 789, 751, 784, 786, 837, 744, 765, 799, 757, 772, 802, 772, 755, 778, 760, 745, 765, 737, 806, 772, 737, 781, 734, 786, 774, 779, 785, 787, 741, 737, 783, 794, 747, 769, 784, 779, 781, 765, 731] payload_rows_per_dst=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] dup_rows_per_dst=[2966, 2888, 3098, 3166, 3044, 2994, 3060, 3028] combine_rows_per_src=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64846 dup_rows=24034 dup_rows_remote=20820 dedup_ratio=0.244486 router_tokens_per_expert=[775, 750, 749, 741, 761, 779, 783, 806, 801, 775, 749, 775, 828, 790, 738, 768, 767, 745, 780, 708, 676, 791, 727, 759, 774, 729, 764, 784, 760, 790, 766, 795, 794, 778, 750, 791, 810, 781, 782, 832, 772, 801, 791, 713, 755, 733, 782, 781, 812, 804, 713, 736, 807, 800, 740, 735, 724, 760, 767, 799, 781, 735, 752, 758] payload_rows_per_dst=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] dup_rows_per_dst=[3214, 3002, 2950, 3044, 3168, 2884, 2814, 2958] combine_rows_per_src=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64912 dup_rows=24064 dup_rows_remote=21156 dedup_ratio=0.244792 router_tokens_per_expert=[771, 732, 730, 806, 783, 776, 773, 788, 761, 739, 811, 748, 775, 789, 774, 796, 743, 752, 766, 762, 751, 763, 780, 791, 709, 749, 766, 811, 766, 779, 806, 749, 761, 773, 807, 771, 755, 789, 803, 758, 753, 786, 734, 786, 780, 729, 721, 812, 748, 777, 796, 775, 770, 720, 771, 761, 760, 770, 737, 783, 775, 740, 788, 768] payload_rows_per_dst=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] dup_rows_per_dst=[3096, 3096, 2994, 3056, 3030, 2868, 2908, 3016] combine_rows_per_src=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64608 dup_rows=24018 dup_rows_remote=20878 dedup_ratio=0.244324 router_tokens_per_expert=[785, 780, 745, 749, 812, 789, 797, 768, 768, 752, 723, 777, 783, 746, 773, 738, 735, 716, 760, 765, 780, 765, 758, 725, 780, 806, 756, 766, 779, 784, 770, 786, 753, 793, 758, 780, 735, 772, 757, 757, 769, 774, 749, 763, 742, 771, 784, 760, 793, 760, 735, 761, 735, 767, 735, 800, 740, 833, 785, 812, 835, 780, 756, 792] payload_rows_per_dst=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] dup_rows_per_dst=[2968, 3042, 2874, 3006, 2784, 3298, 2906, 3140] combine_rows_per_src=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344368 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215152 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085936 weight_movement_bytes=34603008 total_nvlink_bytes=568688944 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65132 dup_rows=23950 dup_rows_remote=20968 dedup_ratio=0.243632 router_tokens_per_expert=[784, 781, 755, 763, 749, 723, 775, 793, 791, 749, 763, 749, 731, 766, 767, 756, 762, 793, 730, 767, 758, 736, 752, 804, 814, 783, 772, 730, 769, 762, 749, 748, 772, 778, 744, 781, 749, 718, 746, 800, 750, 755, 766, 801, 766, 783, 804, 754, 785, 759, 778, 768, 732, 751, 824, 772, 798, 773, 813, 769, 779, 780, 774, 806] payload_rows_per_dst=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] dup_rows_per_dst=[2958, 2890, 2982, 2954, 2986, 3020, 3036, 3124] combine_rows_per_src=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=342936 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266795928 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533248920 weight_movement_bytes=69206016 total_nvlink_bytes=602454936 weight_to_token_ratio=0.129782 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344824 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658552 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532972280 weight_movement_bytes=34603008 total_nvlink_bytes=567575288 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343656 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265412200 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530480744 weight_movement_bytes=69206016 total_nvlink_bytes=599686760 weight_to_token_ratio=0.130459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=342664 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265951880 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561096 weight_movement_bytes=34603008 total_nvlink_bytes=566164104 weight_to_token_ratio=0.065097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344272 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223824 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103376 weight_movement_bytes=69206016 total_nvlink_bytes=601309392 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=341944 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264976312 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529610680 weight_movement_bytes=69206016 total_nvlink_bytes=598816696 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344400 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267125072 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905744 weight_movement_bytes=34603008 total_nvlink_bytes=568508752 weight_to_token_ratio=0.064811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65146 dup_rows=23872 dup_rows_remote=20784 dedup_ratio=0.242839 router_tokens_per_expert=[782, 800, 756, 784, 781, 777, 758, 744, 760, 786, 804, 774, 743, 735, 790, 795, 714, 712, 761, 726, 757, 809, 810, 774, 761, 774, 751, 802, 761, 798, 803, 761, 817, 762, 753, 726, 760, 764, 775, 793, 781, 754, 764, 808, 765, 767, 767, 795, 752, 726, 720, 724, 759, 735, 761, 800, 783, 767, 764, 749, 732, 822, 737, 827] payload_rows_per_dst=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] dup_rows_per_dst=[3020, 3088, 2924, 3072, 3006, 2954, 2784, 3024] combine_rows_per_src=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343720 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181736 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534019752 weight_movement_bytes=69206016 total_nvlink_bytes=603225768 weight_to_token_ratio=0.129594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=64960 dup_rows=23842 dup_rows_remote=20804 dedup_ratio=0.242533 router_tokens_per_expert=[776, 783, 772, 733, 757, 776, 777, 761, 758, 822, 778, 718, 772, 767, 770, 736, 723, 774, 741, 744, 765, 756, 754, 785, 792, 807, 713, 780, 828, 784, 800, 766, 775, 745, 780, 761, 770, 803, 791, 756, 750, 741, 723, 772, 796, 745, 747, 747, 783, 760, 800, 796, 741, 747, 762, 757, 753, 765, 773, 787, 806, 749, 808, 795] payload_rows_per_dst=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] dup_rows_per_dst=[2904, 2992, 2834, 3038, 3098, 2872, 3028, 3076] combine_rows_per_src=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65126 dup_rows=23938 dup_rows_remote=20942 dedup_ratio=0.243510 router_tokens_per_expert=[744, 822, 799, 751, 767, 707, 749, 743, 758, 764, 751, 776, 777, 777, 770, 758, 764, 726, 784, 794, 816, 751, 764, 743, 720, 761, 753, 799, 761, 726, 773, 733, 729, 790, 799, 784, 755, 728, 807, 765, 778, 786, 763, 722, 765, 756, 771, 777, 791, 817, 807, 802, 770, 795, 796, 738, 758, 770, 782, 791, 783, 756, 801, 739] payload_rows_per_dst=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] dup_rows_per_dst=[2888, 2920, 3028, 2906, 3006, 2996, 3196, 2998] combine_rows_per_src=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64786 dup_rows=24218 dup_rows_remote=21396 dedup_ratio=0.246358 router_tokens_per_expert=[814, 755, 752, 671, 755, 783, 760, 791, 834, 773, 807, 754, 791, 813, 791, 752, 739, 742, 739, 719, 781, 805, 789, 794, 723, 781, 763, 757, 776, 773, 753, 768, 751, 745, 774, 767, 732, 801, 745, 746, 765, 746, 783, 779, 784, 748, 739, 793, 801, 766, 776, 762, 762, 755, 751, 784, 767, 783, 774, 791, 759, 767, 770, 788] payload_rows_per_dst=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] dup_rows_per_dst=[3036, 3154, 2950, 3026, 2822, 3082, 3050, 3098] combine_rows_per_src=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65040 dup_rows=23960 dup_rows_remote=20920 dedup_ratio=0.243734 router_tokens_per_expert=[744, 798, 778, 743, 781, 761, 700, 766, 812, 745, 803, 780, 783, 770, 809, 779, 755, 759, 744, 781, 749, 746, 798, 729, 695, 741, 755, 743, 780, 793, 755, 795, 777, 800, 775, 761, 798, 802, 815, 722, 748, 755, 771, 753, 767, 819, 749, 796, 771, 739, 773, 777, 738, 784, 790, 800, 791, 732, 743, 739, 789, 822, 745, 741] payload_rows_per_dst=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] dup_rows_per_dst=[3032, 3064, 2918, 2796, 3112, 3036, 3040, 2962] combine_rows_per_src=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65078 dup_rows=24084 dup_rows_remote=21138 dedup_ratio=0.244995 router_tokens_per_expert=[722, 733, 783, 781, 751, 756, 754, 764, 774, 787, 817, 821, 771, 756, 728, 803, 754, 709, 760, 791, 746, 808, 749, 752, 751, 767, 761, 748, 800, 778, 748, 749, 804, 724, 776, 806, 802, 770, 759, 796, 758, 764, 765, 761, 752, 774, 730, 735, 768, 774, 750, 735, 836, 775, 807, 805, 772, 746, 782, 753, 782, 770, 776, 773] payload_rows_per_dst=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] dup_rows_per_dst=[2946, 3396, 3076, 2940, 3088, 2906, 2996, 2736] combine_rows_per_src=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64880 dup_rows=24060 dup_rows_remote=20926 dedup_ratio=0.244751 router_tokens_per_expert=[740, 777, 735, 707, 736, 797, 774, 774, 720, 769, 744, 761, 748, 756, 748, 754, 770, 794, 769, 758, 812, 809, 804, 758, 744, 748, 775, 809, 763, 755, 755, 797, 751, 746, 778, 815, 760, 754, 741, 730, 740, 748, 691, 780, 791, 790, 777, 816, 819, 772, 744, 792, 810, 763, 784, 751, 765, 780, 751, 826, 760, 755, 848, 764] payload_rows_per_dst=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] dup_rows_per_dst=[2976, 2600, 3442, 2940, 2934, 2948, 3086, 3134] combine_rows_per_src=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64902 dup_rows=24092 dup_rows_remote=20958 dedup_ratio=0.245076 router_tokens_per_expert=[769, 754, 789, 770, 758, 785, 780, 736, 783, 770, 737, 759, 748, 808, 798, 788, 762, 786, 790, 786, 786, 794, 779, 739, 729, 792, 762, 760, 818, 770, 770, 757, 766, 760, 761, 691, 793, 800, 754, 814, 777, 723, 732, 763, 753, 737, 802, 744, 764, 762, 791, 756, 787, 770, 758, 775, 715, 800, 770, 743, 772, 727, 770, 810] payload_rows_per_dst=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] dup_rows_per_dst=[3038, 3042, 3134, 3064, 3034, 2856, 3004, 2920] combine_rows_per_src=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343056 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419216 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495376 weight_movement_bytes=34603008 total_nvlink_bytes=567098384 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344272 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100368 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856464 weight_movement_bytes=34603008 total_nvlink_bytes=568459472 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344728 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708184 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071640 weight_movement_bytes=34603008 total_nvlink_bytes=565674648 weight_to_token_ratio=0.065157 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343840 combine_payload_bytes=266403840 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266747680 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151520 weight_movement_bytes=103809024 total_nvlink_bytes=636960544 weight_to_token_ratio=0.194708 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343224 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266091704 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840184 weight_movement_bytes=69206016 total_nvlink_bytes=601046200 weight_to_token_ratio=0.130126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344864 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266904352 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533463840 weight_movement_bytes=103809024 total_nvlink_bytes=637272864 weight_to_token_ratio=0.194594 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343440 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182032 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532020624 weight_movement_bytes=69206016 total_nvlink_bytes=601226640 weight_to_token_ratio=0.130081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65196 dup_rows=23892 dup_rows_remote=20910 dedup_ratio=0.243042 router_tokens_per_expert=[761, 812, 790, 791, 798, 818, 741, 745, 714, 812, 742, 769, 760, 813, 761, 728, 801, 780, 787, 788, 773, 809, 753, 786, 731, 772, 773, 739, 738, 781, 803, 765, 831, 786, 749, 760, 786, 740, 734, 716, 754, 721, 767, 755, 779, 802, 752, 746, 733, 760, 756, 770, 759, 790, 781, 731, 759, 783, 776, 731, 816, 791, 767, 737] payload_rows_per_dst=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] dup_rows_per_dst=[3080, 2982, 3088, 3000, 2964, 2880, 2920, 2978] combine_rows_per_src=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344424 combine_payload_bytes=267042816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267387240 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534430056 weight_movement_bytes=69206016 total_nvlink_bytes=603636072 weight_to_token_ratio=0.129495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64950 dup_rows=24036 dup_rows_remote=21046 dedup_ratio=0.244507 router_tokens_per_expert=[815, 813, 746, 762, 765, 794, 831, 807, 740, 795, 783, 771, 746, 750, 749, 772, 734, 801, 809, 740, 776, 767, 802, 769, 790, 810, 784, 746, 771, 770, 711, 772, 716, 735, 742, 763, 798, 752, 790, 778, 794, 808, 758, 702, 792, 755, 759, 757, 785, 723, 765, 755, 737, 795, 804, 756, 720, 746, 832, 728, 752, 775, 757, 732] payload_rows_per_dst=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] dup_rows_per_dst=[3176, 2984, 3068, 2990, 2938, 2988, 2988, 2904] combine_rows_per_src=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64924 dup_rows=23958 dup_rows_remote=20788 dedup_ratio=0.243713 router_tokens_per_expert=[733, 801, 763, 768, 802, 753, 760, 812, 762, 788, 782, 770, 775, 813, 777, 774, 733, 773, 741, 796, 722, 775, 761, 751, 784, 774, 716, 713, 775, 784, 786, 732, 770, 763, 754, 771, 786, 777, 788, 735, 740, 789, 795, 767, 775, 839, 771, 820, 754, 750, 778, 747, 771, 774, 685, 809, 778, 755, 756, 775, 735, 758, 794, 744] payload_rows_per_dst=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] dup_rows_per_dst=[3112, 2976, 2942, 2848, 3024, 3170, 2910, 2976] combine_rows_per_src=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65198 dup_rows=23842 dup_rows_remote=20906 dedup_ratio=0.242533 router_tokens_per_expert=[756, 753, 770, 740, 829, 787, 745, 787, 770, 772, 759, 773, 788, 757, 793, 737, 782, 782, 777, 771, 778, 753, 761, 744, 770, 728, 736, 818, 774, 760, 810, 755, 800, 751, 756, 727, 794, 761, 748, 727, 751, 806, 782, 776, 805, 761, 780, 782, 790, 772, 780, 715, 780, 752, 764, 747, 798, 757, 789, 785, 745, 755, 757, 744] payload_rows_per_dst=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] dup_rows_per_dst=[2940, 3018, 3066, 2986, 2954, 3022, 2936, 2920] combine_rows_per_src=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64676 dup_rows=24020 dup_rows_remote=20548 dedup_ratio=0.244344 router_tokens_per_expert=[769, 834, 777, 758, 778, 703, 753, 759, 758, 789, 770, 781, 758, 768, 748, 745, 746, 757, 792, 707, 795, 775, 772, 774, 744, 743, 823, 740, 784, 730, 748, 788, 799, 744, 745, 774, 751, 760, 800, 747, 784, 787, 792, 789, 802, 774, 771, 777, 736, 747, 771, 782, 730, 742, 791, 772, 780, 802, 798, 790, 759, 765, 748, 777] payload_rows_per_dst=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] dup_rows_per_dst=[2656, 2942, 2866, 3072, 2940, 2948, 3124, 3472] combine_rows_per_src=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64964 dup_rows=23920 dup_rows_remote=20972 dedup_ratio=0.243327 router_tokens_per_expert=[775, 780, 739, 762, 753, 781, 816, 738, 769, 792, 788, 744, 735, 797, 811, 756, 793, 793, 838, 744, 726, 722, 826, 788, 711, 760, 840, 724, 762, 800, 792, 717, 754, 761, 772, 760, 788, 742, 783, 824, 770, 792, 776, 758, 735, 795, 712, 767, 794, 704, 818, 785, 798, 733, 706, 767, 752, 778, 782, 786, 747, 721, 768, 752] payload_rows_per_dst=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] dup_rows_per_dst=[2904, 3056, 3054, 2994, 2948, 2966, 2972, 3026] combine_rows_per_src=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64600 dup_rows=24154 dup_rows_remote=20828 dedup_ratio=0.245707 router_tokens_per_expert=[736, 760, 801, 783, 755, 734, 759, 766, 768, 776, 789, 783, 752, 774, 821, 809, 746, 757, 757, 706, 783, 784, 726, 769, 763, 766, 678, 809, 767, 729, 768, 795, 795, 722, 760, 791, 750, 793, 772, 791, 756, 766, 815, 755, 789, 789, 778, 752, 748, 752, 752, 760, 760, 769, 835, 771, 794, 788, 789, 732, 771, 794, 767, 727] payload_rows_per_dst=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] dup_rows_per_dst=[3326, 3310, 2942, 3126, 3010, 3104, 2856, 2480] combine_rows_per_src=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=21006 dedup_ratio=0.242676 router_tokens_per_expert=[734, 751, 796, 757, 777, 736, 737, 774, 759, 781, 763, 775, 751, 781, 781, 748, 733, 742, 775, 767, 720, 780, 757, 780, 759, 765, 814, 758, 764, 740, 773, 798, 800, 765, 767, 825, 763, 766, 786, 814, 778, 744, 727, 731, 719, 783, 785, 779, 790, 803, 774, 768, 799, 764, 751, 742, 778, 765, 768, 742, 747, 803, 813, 787] payload_rows_per_dst=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] dup_rows_per_dst=[2918, 3016, 2850, 3026, 3112, 2936, 2980, 3018] combine_rows_per_src=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=342848 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266271552 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532200256 weight_movement_bytes=103809024 total_nvlink_bytes=636009280 weight_to_token_ratio=0.195056 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344416 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267395424 ingress_bytes=301654016 link_bytes=301654016 token_bytes=534446432 weight_movement_bytes=34603008 total_nvlink_bytes=569049440 weight_to_token_ratio=0.064746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135099777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343984 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266379184 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414384 weight_movement_bytes=34603008 total_nvlink_bytes=567017392 weight_to_token_ratio=0.064993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=340896 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265253792 ingress_bytes=299515904 link_bytes=299515904 token_bytes=530166688 weight_movement_bytes=34603008 total_nvlink_bytes=564769696 weight_to_token_ratio=0.065268 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127054234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343744 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436288 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528832 weight_movement_bytes=34603008 total_nvlink_bytes=567131840 weight_to_token_ratio=0.064979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65080 dup_rows=23984 dup_rows_remote=21014 dedup_ratio=0.243978 router_tokens_per_expert=[736, 767, 786, 772, 811, 758, 801, 740, 762, 767, 781, 756, 806, 771, 760, 702, 805, 723, 808, 770, 736, 731, 769, 739, 775, 782, 775, 758, 756, 754, 813, 753, 755, 781, 797, 789, 745, 740, 796, 728, 757, 732, 744, 765, 771, 798, 792, 781, 754, 805, 789, 722, 767, 802, 795, 704, 816, 735, 763, 826, 761, 791, 784, 744] payload_rows_per_dst=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] dup_rows_per_dst=[3014, 2970, 2904, 3052, 3012, 3006, 2988, 3038] combine_rows_per_src=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264601600 dispatch_meta_bytes=341712 combine_payload_bytes=264601600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264943312 ingress_bytes=299204608 link_bytes=299204608 token_bytes=529544912 weight_movement_bytes=34603008 total_nvlink_bytes=564147920 weight_to_token_ratio=0.065345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125882852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344784 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267363024 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534381264 weight_movement_bytes=34603008 total_nvlink_bytes=568984272 weight_to_token_ratio=0.064753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344376 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912056 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533479736 weight_movement_bytes=103809024 total_nvlink_bytes=637288760 weight_to_token_ratio=0.194589 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21224 dedup_ratio=0.246379 router_tokens_per_expert=[776, 766, 747, 810, 723, 775, 801, 739, 770, 758, 748, 714, 802, 783, 771, 785, 774, 791, 769, 746, 779, 766, 803, 770, 848, 756, 756, 794, 759, 759, 793, 777, 784, 764, 743, 761, 730, 772, 835, 741, 757, 730, 720, 782, 779, 760, 789, 781, 809, 757, 760, 765, 763, 775, 754, 745, 736, 774, 734, 762, 826, 783, 748, 725] payload_rows_per_dst=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] dup_rows_per_dst=[3050, 3024, 3056, 3164, 2984, 2996, 2940, 3006] combine_rows_per_src=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20810 dedup_ratio=0.243408 router_tokens_per_expert=[764, 768, 771, 757, 736, 774, 800, 818, 773, 754, 788, 736, 788, 785, 734, 785, 739, 818, 802, 813, 771, 744, 771, 754, 749, 751, 726, 735, 772, 801, 766, 736, 756, 781, 765, 739, 733, 746, 777, 767, 760, 793, 764, 741, 761, 768, 741, 771, 731, 753, 772, 769, 845, 796, 759, 778, 762, 783, 791, 760, 772, 754, 814, 771] payload_rows_per_dst=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] dup_rows_per_dst=[3030, 2930, 3024, 2922, 2944, 2950, 3118, 3010] combine_rows_per_src=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65004 dup_rows=23844 dup_rows_remote=20794 dedup_ratio=0.242554 router_tokens_per_expert=[803, 750, 730, 811, 770, 741, 767, 741, 756, 752, 772, 771, 761, 773, 784, 771, 748, 741, 726, 749, 758, 776, 791, 780, 764, 742, 729, 782, 794, 797, 825, 820, 772, 736, 757, 806, 738, 769, 752, 757, 750, 822, 748, 762, 797, 765, 741, 758, 759, 757, 730, 705, 812, 753, 783, 756, 767, 829, 769, 815, 754, 783, 775, 800] payload_rows_per_dst=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] dup_rows_per_dst=[2924, 2924, 2944, 3050, 2868, 3018, 2948, 3168] combine_rows_per_src=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=64714 dup_rows=24332 dup_rows_remote=21360 dedup_ratio=0.247518 router_tokens_per_expert=[762, 734, 791, 739, 701, 809, 754, 784, 790, 755, 802, 766, 792, 759, 773, 826, 751, 785, 780, 773, 775, 789, 764, 785, 759, 754, 758, 804, 774, 750, 799, 787, 755, 770, 769, 785, 734, 760, 746, 790, 767, 754, 786, 769, 764, 761, 766, 741, 766, 769, 769, 736, 767, 785, 722, 782, 751, 758, 752, 758, 748, 808, 780, 760] payload_rows_per_dst=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] dup_rows_per_dst=[2954, 3030, 3200, 3116, 2956, 3022, 3082, 2972] combine_rows_per_src=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64814 dup_rows=24180 dup_rows_remote=21250 dedup_ratio=0.245972 router_tokens_per_expert=[763, 805, 748, 772, 717, 781, 769, 725, 749, 764, 744, 819, 804, 789, 832, 749, 762, 806, 762, 710, 758, 790, 731, 779, 803, 777, 774, 729, 790, 736, 776, 758, 745, 750, 760, 760, 815, 768, 772, 750, 774, 745, 764, 799, 765, 765, 811, 760, 765, 799, 724, 765, 823, 737, 769, 757, 770, 773, 789, 775, 749, 738, 783, 762] payload_rows_per_dst=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] dup_rows_per_dst=[2920, 3110, 3032, 3128, 2930, 3042, 3002, 3016] combine_rows_per_src=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65010 dup_rows=23874 dup_rows_remote=20918 dedup_ratio=0.242859 router_tokens_per_expert=[770, 766, 735, 760, 777, 746, 812, 733, 761, 752, 806, 738, 770, 773, 772, 774, 740, 712, 781, 802, 806, 762, 793, 792, 785, 753, 795, 774, 781, 744, 795, 781, 755, 789, 768, 752, 771, 769, 767, 770, 748, 772, 743, 773, 711, 792, 742, 760, 760, 792, 779, 791, 796, 766, 775, 761, 759, 773, 762, 784, 761, 757, 734, 779] payload_rows_per_dst=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] dup_rows_per_dst=[2954, 2876, 2956, 3130, 2998, 2948, 3030, 2982] combine_rows_per_src=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64816 dup_rows=24236 dup_rows_remote=21212 dedup_ratio=0.246541 router_tokens_per_expert=[786, 780, 739, 718, 788, 771, 749, 802, 765, 808, 762, 725, 762, 773, 761, 752, 771, 773, 754, 763, 746, 711, 756, 777, 767, 779, 783, 745, 775, 785, 721, 772, 769, 774, 796, 770, 813, 746, 778, 778, 772, 801, 769, 811, 737, 786, 778, 817, 786, 770, 757, 784, 791, 756, 831, 771, 720, 736, 764, 744, 795, 706, 770, 757] payload_rows_per_dst=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] dup_rows_per_dst=[3024, 2916, 2956, 3104, 3188, 3002, 3208, 2838] combine_rows_per_src=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344432 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266109296 ingress_bytes=300367872 link_bytes=300367872 token_bytes=531874160 weight_movement_bytes=34603008 total_nvlink_bytes=566477168 weight_to_token_ratio=0.065059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130260121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343592 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944040 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533544488 weight_movement_bytes=34603008 total_nvlink_bytes=568147496 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65304 dup_rows=23760 dup_rows_remote=20782 dedup_ratio=0.241699 router_tokens_per_expert=[799, 759, 738, 774, 761, 743, 782, 726, 728, 717, 825, 750, 752, 793, 743, 801, 790, 740, 786, 779, 778, 744, 728, 773, 790, 736, 755, 731, 791, 773, 738, 824, 785, 762, 771, 775, 768, 776, 779, 774, 761, 752, 794, 821, 747, 756, 771, 751, 795, 769, 801, 814, 739, 786, 733, 725, 734, 795, 817, 729, 789, 788, 756, 792] payload_rows_per_dst=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] dup_rows_per_dst=[2902, 2978, 2840, 3036, 3108, 3016, 2972, 2908] combine_rows_per_src=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343192 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266599576 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855960 weight_movement_bytes=69206016 total_nvlink_bytes=602061976 weight_to_token_ratio=0.129878 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344296 combine_payload_bytes=265068544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265412840 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530481384 weight_movement_bytes=69206016 total_nvlink_bytes=599687400 weight_to_token_ratio=0.130459 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344256 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265822400 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531300544 weight_movement_bytes=34603008 total_nvlink_bytes=565903552 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343712 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266624672 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532905632 weight_movement_bytes=69206016 total_nvlink_bytes=602111648 weight_to_token_ratio=0.129865 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344112 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265830448 ingress_bytes=369295360 link_bytes=369295360 token_bytes=531316784 weight_movement_bytes=103809024 total_nvlink_bytes=635125808 weight_to_token_ratio=0.195381 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389628709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344344 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267829528 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314712 weight_movement_bytes=34603008 total_nvlink_bytes=569917720 weight_to_token_ratio=0.064640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65252 dup_rows=23758 dup_rows_remote=20776 dedup_ratio=0.241679 router_tokens_per_expert=[737, 732, 772, 743, 766, 771, 788, 739, 804, 859, 729, 763, 730, 780, 751, 749, 774, 772, 717, 818, 761, 765, 807, 834, 762, 753, 777, 796, 789, 755, 771, 780, 824, 703, 753, 731, 771, 751, 743, 813, 789, 763, 739, 755, 722, 769, 771, 759, 814, 793, 771, 725, 759, 737, 752, 787, 771, 793, 773, 755, 763, 839, 761, 759] payload_rows_per_dst=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] dup_rows_per_dst=[2902, 3012, 3032, 3026, 2948, 2888, 2982, 2968] combine_rows_per_src=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64964 dup_rows=23996 dup_rows_remote=20960 dedup_ratio=0.244100 router_tokens_per_expert=[790, 727, 743, 794, 775, 757, 803, 750, 783, 737, 772, 778, 777, 774, 774, 769, 802, 760, 746, 804, 734, 800, 764, 769, 779, 794, 757, 753, 771, 756, 756, 788, 795, 754, 753, 789, 748, 829, 721, 699, 762, 823, 714, 770, 787, 765, 806, 763, 766, 710, 782, 729, 778, 776, 767, 787, 752, 770, 788, 740, 781, 788, 763, 761] payload_rows_per_dst=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] dup_rows_per_dst=[3018, 2984, 3054, 3064, 2944, 3036, 2958, 2938] combine_rows_per_src=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65124 dup_rows=24072 dup_rows_remote=21120 dedup_ratio=0.244873 router_tokens_per_expert=[763, 792, 758, 730, 791, 762, 754, 780, 783, 734, 766, 750, 767, 824, 765, 780, 753, 800, 767, 779, 756, 786, 772, 795, 748, 734, 762, 745, 787, 743, 775, 736, 779, 753, 771, 746, 732, 820, 739, 732, 801, 746, 764, 786, 819, 793, 795, 776, 809, 720, 803, 747, 736, 724, 765, 723, 797, 830, 769, 743, 789, 744, 743, 821] payload_rows_per_dst=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] dup_rows_per_dst=[2978, 3070, 3008, 2952, 2982, 3128, 2920, 3034] combine_rows_per_src=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=344112 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267616304 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888496 weight_movement_bytes=34603008 total_nvlink_bytes=569491504 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65226 dup_rows=23944 dup_rows_remote=21088 dedup_ratio=0.243571 router_tokens_per_expert=[802, 775, 758, 789, 804, 816, 754, 770, 733, 771, 771, 738, 780, 809, 758, 771, 753, 763, 758, 794, 769, 749, 795, 759, 794, 791, 737, 740, 764, 792, 771, 763, 708, 769, 794, 842, 767, 757, 753, 732, 762, 800, 770, 754, 789, 782, 778, 770, 727, 758, 746, 788, 799, 763, 802, 758, 781, 742, 777, 740, 716, 743, 754, 740] payload_rows_per_dst=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] dup_rows_per_dst=[3078, 3008, 3018, 2994, 2970, 2830, 3190, 2856] combine_rows_per_src=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266436240 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528784 weight_movement_bytes=103809024 total_nvlink_bytes=636337808 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64972 dup_rows=24044 dup_rows_remote=21032 dedup_ratio=0.244588 router_tokens_per_expert=[717, 752, 736, 768, 769, 755, 723, 747, 761, 740, 752, 794, 768, 739, 770, 782, 784, 737, 758, 807, 802, 777, 789, 762, 751, 764, 738, 753, 765, 726, 802, 765, 745, 807, 749, 738, 796, 787, 768, 760, 750, 797, 782, 788, 754, 746, 807, 780, 726, 789, 750, 746, 778, 764, 778, 797, 822, 782, 789, 785, 756, 770, 795, 818] payload_rows_per_dst=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] dup_rows_per_dst=[2864, 3018, 3088, 2938, 3012, 3006, 2908, 3210] combine_rows_per_src=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344976 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092880 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533840784 weight_movement_bytes=34603008 total_nvlink_bytes=568443792 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64920 dup_rows=24158 dup_rows_remote=21182 dedup_ratio=0.245748 router_tokens_per_expert=[760, 745, 777, 776, 780, 783, 739, 758, 742, 751, 769, 790, 773, 806, 740, 793, 745, 766, 787, 733, 813, 789, 713, 755, 774, 759, 801, 741, 769, 765, 727, 747, 822, 798, 763, 783, 812, 768, 772, 741, 733, 761, 785, 759, 768, 738, 776, 729, 764, 731, 803, 770, 804, 767, 770, 771, 799, 771, 795, 773, 740, 788, 760, 772] payload_rows_per_dst=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] dup_rows_per_dst=[3062, 2984, 2976, 2962, 3116, 2972, 3062, 3024] combine_rows_per_src=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64732 dup_rows=24108 dup_rows_remote=20918 dedup_ratio=0.245239 router_tokens_per_expert=[759, 773, 775, 731, 742, 755, 782, 813, 707, 809, 824, 726, 766, 761, 767, 769, 754, 739, 772, 762, 769, 798, 747, 800, 766, 761, 781, 742, 777, 784, 824, 753, 760, 716, 788, 779, 763, 707, 734, 747, 737, 791, 771, 786, 747, 769, 775, 810, 776, 741, 741, 766, 770, 788, 759, 775, 789, 792, 774, 779, 769, 780, 817, 768] payload_rows_per_dst=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] dup_rows_per_dst=[3190, 2992, 3006, 2900, 3200, 3158, 2932, 2730] combine_rows_per_src=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65008 dup_rows=24042 dup_rows_remote=20996 dedup_ratio=0.244568 router_tokens_per_expert=[763, 816, 783, 762, 732, 741, 737, 796, 742, 795, 785, 754, 729, 808, 754, 783, 770, 737, 795, 783, 749, 759, 797, 714, 751, 778, 745, 802, 763, 730, 787, 728, 747, 760, 743, 782, 809, 764, 726, 772, 825, 786, 749, 792, 722, 740, 782, 761, 750, 791, 760, 793, 749, 775, 780, 797, 782, 762, 815, 763, 813, 759, 748, 787] payload_rows_per_dst=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] dup_rows_per_dst=[2950, 3046, 2958, 3006, 2910, 3012, 3080, 3080] combine_rows_per_src=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=345256 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267510952 ingress_bytes=405577728 link_bytes=405577728 token_bytes=534676648 weight_movement_bytes=138412032 total_nvlink_bytes=673088680 weight_to_token_ratio=0.258871 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526156339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344016 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469328 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594640 weight_movement_bytes=34603008 total_nvlink_bytes=567197648 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344408 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256728 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532169048 weight_movement_bytes=34603008 total_nvlink_bytes=566772056 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342600 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265484872 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627144 weight_movement_bytes=34603008 total_nvlink_bytes=565230152 weight_to_token_ratio=0.065212 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64902 dup_rows=24086 dup_rows_remote=21054 dedup_ratio=0.245015 router_tokens_per_expert=[797, 761, 766, 765, 742, 734, 762, 801, 759, 752, 745, 768, 758, 804, 767, 785, 762, 721, 733, 717, 791, 759, 808, 727, 763, 754, 751, 820, 827, 774, 745, 761, 771, 797, 795, 731, 717, 772, 771, 747, 816, 747, 820, 782, 751, 742, 826, 729, 786, 764, 791, 771, 783, 755, 786, 738, 740, 742, 814, 749, 838, 779, 760, 763] payload_rows_per_dst=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] dup_rows_per_dst=[3006, 2982, 2878, 3048, 2966, 3068, 3032, 3106] combine_rows_per_src=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344016 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616784 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889552 weight_movement_bytes=34603008 total_nvlink_bytes=567492560 weight_to_token_ratio=0.064935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65082 dup_rows=23964 dup_rows_remote=21172 dedup_ratio=0.243774 router_tokens_per_expert=[766, 772, 739, 795, 801, 792, 732, 787, 777, 733, 766, 769, 778, 780, 770, 781, 799, 775, 775, 739, 724, 774, 754, 760, 751, 710, 778, 799, 753, 752, 782, 775, 758, 776, 769, 758, 794, 781, 792, 782, 753, 786, 783, 771, 729, 696, 768, 739, 788, 784, 792, 757, 798, 778, 758, 749, 782, 766, 716, 783, 770, 789, 789, 780] payload_rows_per_dst=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] dup_rows_per_dst=[3022, 3016, 3016, 3004, 3016, 2792, 3032, 3066] combine_rows_per_src=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64978 dup_rows=23958 dup_rows_remote=21066 dedup_ratio=0.243713 router_tokens_per_expert=[786, 737, 776, 778, 793, 771, 793, 791, 816, 774, 763, 735, 802, 768, 741, 778, 746, 748, 746, 764, 788, 775, 767, 731, 746, 709, 773, 786, 787, 790, 756, 783, 720, 782, 780, 787, 756, 775, 773, 768, 783, 754, 797, 749, 669, 774, 724, 780, 759, 759, 750, 811, 780, 780, 776, 781, 783, 800, 760, 731, 777, 764, 774, 799] payload_rows_per_dst=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] dup_rows_per_dst=[2994, 3114, 2910, 2892, 3032, 2910, 3084, 3022] combine_rows_per_src=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343824 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266182416 ingress_bytes=438853632 link_bytes=438853632 token_bytes=532021008 weight_movement_bytes=173015040 total_nvlink_bytes=705036048 weight_to_token_ratio=0.325203 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651370887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=345016 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920888 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496760 weight_movement_bytes=34603008 total_nvlink_bytes=568099768 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344176 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494064 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643952 weight_movement_bytes=34603008 total_nvlink_bytes=567246960 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64990 dup_rows=24134 dup_rows_remote=21234 dedup_ratio=0.245504 router_tokens_per_expert=[771, 785, 793, 746, 741, 744, 767, 799, 763, 803, 788, 754, 737, 737, 756, 716, 782, 755, 709, 799, 789, 742, 773, 798, 736, 777, 760, 772, 769, 755, 781, 729, 805, 807, 731, 776, 753, 806, 746, 763, 754, 794, 768, 785, 767, 749, 812, 833, 784, 773, 797, 788, 771, 743, 823, 758, 741, 712, 769, 778, 802, 728, 769, 741] payload_rows_per_dst=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] dup_rows_per_dst=[2770, 3000, 3014, 3062, 3074, 3178, 3136, 2900] combine_rows_per_src=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65036 dup_rows=23994 dup_rows_remote=21100 dedup_ratio=0.244080 router_tokens_per_expert=[793, 813, 788, 777, 756, 768, 789, 705, 757, 751, 746, 755, 724, 756, 782, 743, 780, 768, 802, 782, 830, 723, 738, 790, 780, 735, 748, 812, 776, 806, 759, 829, 755, 751, 756, 755, 793, 758, 776, 740, 749, 740, 802, 780, 776, 776, 787, 788, 790, 758, 754, 767, 772, 807, 753, 740, 761, 783, 709, 732, 736, 820, 767, 760] payload_rows_per_dst=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] dup_rows_per_dst=[3042, 2906, 3048, 3006, 2894, 3164, 2984, 2950] combine_rows_per_src=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65110 dup_rows=23974 dup_rows_remote=20996 dedup_ratio=0.243876 router_tokens_per_expert=[772, 769, 814, 745, 741, 743, 791, 740, 685, 777, 793, 800, 747, 807, 779, 820, 808, 745, 719, 767, 770, 757, 733, 800, 782, 741, 753, 800, 799, 778, 764, 805, 774, 783, 751, 783, 698, 727, 796, 770, 753, 765, 789, 750, 740, 759, 778, 753, 763, 745, 769, 778, 828, 799, 743, 776, 780, 770, 724, 839, 741, 740, 755, 789] payload_rows_per_dst=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] dup_rows_per_dst=[2954, 3076, 2978, 3102, 2990, 2980, 2936, 2958] combine_rows_per_src=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64636 dup_rows=24110 dup_rows_remote=20830 dedup_ratio=0.245260 router_tokens_per_expert=[759, 758, 784, 738, 756, 772, 758, 813, 757, 743, 797, 740, 809, 767, 731, 794, 774, 749, 756, 789, 780, 763, 774, 747, 799, 723, 782, 794, 791, 782, 766, 769, 776, 777, 784, 773, 736, 800, 777, 777, 790, 811, 809, 768, 709, 788, 784, 746, 717, 761, 790, 733, 734, 812, 795, 712, 772, 735, 789, 694, 765, 769, 804, 751] payload_rows_per_dst=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] dup_rows_per_dst=[3280, 3054, 2810, 3114, 3146, 3012, 2904, 2790] combine_rows_per_src=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65088 dup_rows=23936 dup_rows_remote=21000 dedup_ratio=0.243490 router_tokens_per_expert=[724, 780, 761, 783, 797, 746, 784, 754, 756, 798, 788, 724, 805, 775, 753, 709, 768, 758, 755, 724, 786, 738, 788, 774, 789, 754, 737, 804, 767, 742, 755, 761, 797, 743, 765, 803, 816, 766, 809, 756, 793, 741, 788, 690, 785, 773, 749, 752, 798, 759, 753, 741, 744, 815, 829, 773, 788, 803, 794, 786, 762, 765, 736, 743] payload_rows_per_dst=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] dup_rows_per_dst=[2966, 2936, 2922, 2960, 3082, 2946, 3080, 3044] combine_rows_per_src=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344896 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266543936 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742976 weight_movement_bytes=69206016 total_nvlink_bytes=601948992 weight_to_token_ratio=0.129905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732000 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119456 weight_movement_bytes=34603008 total_nvlink_bytes=567722464 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344424 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267034984 ingress_bytes=335896576 link_bytes=335896576 token_bytes=533725544 weight_movement_bytes=69206016 total_nvlink_bytes=602931560 weight_to_token_ratio=0.129666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64744 dup_rows=24272 dup_rows_remote=21280 dedup_ratio=0.246908 router_tokens_per_expert=[729, 783, 803, 750, 746, 766, 750, 778, 710, 782, 780, 771, 818, 768, 785, 756, 788, 819, 773, 798, 789, 790, 738, 755, 753, 793, 777, 786, 743, 757, 806, 777, 750, 761, 759, 742, 833, 767, 794, 742, 741, 760, 752, 746, 753, 764, 742, 754, 776, 803, 740, 720, 720, 758, 843, 780, 753, 800, 783, 732, 750, 757, 768, 792] payload_rows_per_dst=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] dup_rows_per_dst=[3002, 3126, 3158, 3128, 2966, 2860, 2992, 3040] combine_rows_per_src=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65156 dup_rows=23822 dup_rows_remote=20860 dedup_ratio=0.242330 router_tokens_per_expert=[799, 770, 738, 778, 788, 727, 771, 789, 749, 752, 761, 760, 783, 807, 763, 760, 774, 806, 766, 770, 743, 740, 772, 835, 782, 737, 763, 772, 750, 778, 785, 711, 731, 789, 783, 755, 739, 752, 785, 766, 736, 756, 789, 705, 774, 753, 840, 791, 784, 762, 786, 766, 776, 783, 762, 770, 752, 817, 758, 778, 748, 779, 752, 756] payload_rows_per_dst=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] dup_rows_per_dst=[2946, 2976, 2984, 2972, 2972, 2962, 3030, 2980] combine_rows_per_src=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=341864 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265090920 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529839976 weight_movement_bytes=34603008 total_nvlink_bytes=564442984 weight_to_token_ratio=0.065308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65214 dup_rows=23830 dup_rows_remote=20824 dedup_ratio=0.242411 router_tokens_per_expert=[802, 764, 805, 765, 788, 757, 734, 743, 811, 775, 788, 765, 782, 758, 781, 747, 716, 757, 838, 783, 773, 744, 780, 772, 769, 793, 749, 749, 772, 768, 799, 734, 786, 813, 796, 799, 714, 758, 787, 793, 767, 709, 755, 797, 731, 752, 773, 700, 773, 780, 776, 759, 712, 756, 801, 781, 779, 776, 756, 800, 753, 751, 764, 744] payload_rows_per_dst=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] dup_rows_per_dst=[2936, 3084, 2996, 3006, 3090, 2806, 2984, 2928] combine_rows_per_src=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344352 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944800 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545248 weight_movement_bytes=69206016 total_nvlink_bytes=602751264 weight_to_token_ratio=0.129710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=344096 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535520 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726944 weight_movement_bytes=34603008 total_nvlink_bytes=565329952 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=344064 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223040 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534102016 weight_movement_bytes=34603008 total_nvlink_bytes=568705024 weight_to_token_ratio=0.064787 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344152 combine_payload_bytes=267116544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267460696 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534577240 weight_movement_bytes=69206016 total_nvlink_bytes=603783256 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64964 dup_rows=24152 dup_rows_remote=21220 dedup_ratio=0.245687 router_tokens_per_expert=[747, 732, 765, 783, 808, 724, 775, 768, 774, 733, 800, 789, 794, 770, 782, 775, 746, 744, 798, 817, 803, 771, 795, 752, 799, 772, 764, 719, 798, 759, 747, 821, 732, 805, 739, 770, 782, 753, 748, 731, 779, 750, 743, 726, 751, 793, 734, 768, 734, 776, 742, 746, 778, 780, 790, 747, 703, 763, 806, 796, 806, 793, 793, 771] payload_rows_per_dst=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] dup_rows_per_dst=[3022, 3072, 3114, 2978, 2932, 2880, 2972, 3182] combine_rows_per_src=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64644 dup_rows=24098 dup_rows_remote=20916 dedup_ratio=0.245138 router_tokens_per_expert=[774, 779, 772, 812, 768, 803, 796, 786, 769, 744, 813, 776, 784, 746, 742, 773, 725, 781, 774, 797, 746, 786, 765, 790, 818, 767, 739, 718, 798, 765, 744, 774, 779, 746, 804, 801, 781, 758, 756, 742, 720, 788, 731, 773, 763, 782, 787, 776, 747, 740, 725, 760, 747, 737, 741, 708, 766, 744, 776, 778, 800, 794, 816, 762] payload_rows_per_dst=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] dup_rows_per_dst=[3090, 2882, 2536, 2852, 3090, 3580, 2886, 3182] combine_rows_per_src=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65134 dup_rows=23858 dup_rows_remote=21036 dedup_ratio=0.242696 router_tokens_per_expert=[762, 765, 755, 778, 816, 768, 777, 799, 791, 771, 771, 774, 726, 756, 793, 789, 784, 768, 746, 713, 733, 792, 747, 784, 774, 799, 758, 777, 774, 764, 782, 752, 747, 724, 783, 743, 784, 784, 808, 838, 753, 757, 758, 762, 769, 774, 763, 744, 772, 716, 763, 740, 754, 728, 761, 742, 774, 798, 815, 778, 814, 781, 730, 757] payload_rows_per_dst=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] dup_rows_per_dst=[3118, 2972, 2822, 3056, 3088, 2880, 2810, 3112] combine_rows_per_src=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64978 dup_rows=24056 dup_rows_remote=21006 dedup_ratio=0.244710 router_tokens_per_expert=[782, 761, 753, 754, 819, 761, 781, 744, 777, 753, 760, 750, 744, 765, 794, 792, 789, 792, 766, 747, 781, 696, 747, 785, 756, 770, 759, 745, 754, 808, 770, 746, 813, 771, 701, 712, 766, 767, 740, 761, 767, 781, 759, 795, 722, 758, 750, 812, 738, 838, 801, 794, 771, 795, 733, 769, 783, 778, 808, 769, 774, 790, 758, 777] payload_rows_per_dst=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] dup_rows_per_dst=[3050, 3074, 3236, 2894, 2938, 3158, 2746, 2960] combine_rows_per_src=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64928 dup_rows=24110 dup_rows_remote=21108 dedup_ratio=0.245260 router_tokens_per_expert=[781, 788, 771, 768, 777, 756, 794, 792, 764, 741, 754, 741, 801, 812, 752, 769, 770, 776, 789, 753, 777, 761, 750, 761, 782, 752, 748, 711, 773, 744, 803, 723, 770, 726, 754, 787, 787, 800, 762, 783, 787, 757, 766, 823, 818, 791, 769, 750, 717, 749, 799, 758, 763, 740, 764, 761, 740, 775, 804, 766, 739, 774, 800, 739] payload_rows_per_dst=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] dup_rows_per_dst=[3056, 3002, 3086, 2978, 3054, 3080, 2912, 2942] combine_rows_per_src=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344736 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437280 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532529824 weight_movement_bytes=34603008 total_nvlink_bytes=567132832 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344680 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133544 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922408 weight_movement_bytes=34603008 total_nvlink_bytes=568525416 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=342240 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265124064 ingress_bytes=333987840 link_bytes=333987840 token_bytes=529905888 weight_movement_bytes=69206016 total_nvlink_bytes=599111904 weight_to_token_ratio=0.130601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256769354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65178 dup_rows=23954 dup_rows_remote=20972 dedup_ratio=0.243673 router_tokens_per_expert=[743, 800, 751, 760, 775, 734, 723, 755, 753, 803, 782, 818, 779, 786, 762, 791, 811, 763, 779, 809, 769, 790, 785, 745, 777, 752, 754, 783, 778, 801, 774, 735, 790, 817, 694, 780, 805, 760, 742, 771, 776, 772, 738, 748, 802, 764, 752, 747, 791, 727, 746, 807, 703, 754, 756, 793, 762, 763, 739, 772, 796, 769, 766, 730] payload_rows_per_dst=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] dup_rows_per_dst=[2856, 3172, 3092, 2952, 2980, 3004, 2982, 2916] combine_rows_per_src=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64890 dup_rows=24206 dup_rows_remote=21172 dedup_ratio=0.246236 router_tokens_per_expert=[763, 769, 814, 770, 774, 753, 759, 755, 790, 766, 744, 746, 750, 781, 782, 776, 798, 769, 731, 754, 758, 747, 797, 757, 754, 779, 758, 762, 808, 771, 755, 811, 781, 771, 776, 761, 781, 781, 758, 770, 764, 806, 777, 774, 717, 782, 767, 734, 757, 773, 796, 780, 716, 773, 769, 780, 792, 760, 758, 753, 751, 739, 763, 791] payload_rows_per_dst=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] dup_rows_per_dst=[2986, 2986, 3010, 3098, 3058, 3034, 2932, 3102] combine_rows_per_src=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65048 dup_rows=23872 dup_rows_remote=20868 dedup_ratio=0.242839 router_tokens_per_expert=[815, 774, 789, 725, 747, 748, 755, 747, 725, 763, 777, 746, 780, 786, 736, 740, 781, 756, 717, 833, 797, 728, 765, 800, 770, 782, 799, 770, 789, 773, 749, 762, 769, 743, 770, 793, 734, 782, 764, 762, 821, 764, 783, 779, 759, 751, 719, 735, 778, 790, 806, 740, 790, 764, 758, 758, 767, 735, 770, 794, 780, 780, 775, 815] payload_rows_per_dst=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] dup_rows_per_dst=[2978, 2974, 2952, 3004, 3000, 2996, 3004, 2964] combine_rows_per_src=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343936 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493824 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643712 weight_movement_bytes=69206016 total_nvlink_bytes=601849728 weight_to_token_ratio=0.129929 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344144 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289232 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234320 weight_movement_bytes=69206016 total_nvlink_bytes=601440336 weight_to_token_ratio=0.130029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344600 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313688 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282776 weight_movement_bytes=34603008 total_nvlink_bytes=568885784 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344248 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266133688 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923128 weight_movement_bytes=69206016 total_nvlink_bytes=601129144 weight_to_token_ratio=0.130105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343664 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780272 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216880 weight_movement_bytes=69206016 total_nvlink_bytes=602422896 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65048 dup_rows=23922 dup_rows_remote=21008 dedup_ratio=0.243347 router_tokens_per_expert=[795, 734, 755, 782, 725, 776, 750, 790, 772, 777, 773, 790, 742, 727, 765, 756, 819, 733, 775, 780, 752, 739, 745, 781, 772, 791, 745, 803, 727, 810, 770, 815, 713, 739, 727, 772, 756, 805, 736, 734, 776, 788, 770, 762, 778, 739, 805, 802, 728, 798, 772, 770, 801, 759, 826, 775, 724, 796, 807, 785, 730, 751, 788, 774] payload_rows_per_dst=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] dup_rows_per_dst=[2982, 2878, 2914, 3110, 2866, 3120, 3076, 2976] combine_rows_per_src=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65282 dup_rows=23776 dup_rows_remote=20934 dedup_ratio=0.241862 router_tokens_per_expert=[759, 764, 766, 773, 795, 728, 753, 815, 763, 760, 794, 740, 757, 777, 788, 755, 805, 799, 730, 771, 814, 756, 723, 792, 713, 775, 698, 805, 783, 783, 748, 736, 716, 780, 754, 709, 740, 764, 823, 758, 807, 819, 753, 798, 748, 773, 764, 798, 710, 799, 786, 707, 779, 807, 731, 779, 767, 788, 825, 744, 795, 739, 782, 792] payload_rows_per_dst=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] dup_rows_per_dst=[2924, 3004, 3060, 2878, 2842, 3044, 2970, 3054] combine_rows_per_src=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64964 dup_rows=24012 dup_rows_remote=20950 dedup_ratio=0.244263 router_tokens_per_expert=[745, 773, 794, 765, 701, 765, 784, 741, 769, 737, 781, 773, 785, 754, 779, 786, 773, 746, 783, 729, 764, 768, 752, 750, 762, 781, 730, 797, 781, 752, 790, 758, 794, 745, 788, 797, 724, 763, 743, 798, 771, 811, 757, 792, 757, 784, 781, 727, 762, 806, 755, 764, 808, 763, 739, 788, 769, 801, 740, 789, 807, 801, 739, 741] payload_rows_per_dst=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] dup_rows_per_dst=[2942, 2648, 2972, 3024, 3022, 3000, 3342, 3062] combine_rows_per_src=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344224 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780832 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533217440 weight_movement_bytes=69206016 total_nvlink_bytes=602423456 weight_to_token_ratio=0.129789 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344864 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739936 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535135008 weight_movement_bytes=69206016 total_nvlink_bytes=604341024 weight_to_token_ratio=0.129324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64922 dup_rows=24100 dup_rows_remote=21012 dedup_ratio=0.245158 router_tokens_per_expert=[766, 792, 762, 772, 744, 783, 761, 787, 803, 772, 740, 742, 748, 787, 714, 741, 806, 771, 801, 777, 769, 758, 744, 743, 761, 842, 817, 795, 775, 760, 773, 762, 770, 764, 751, 768, 763, 777, 726, 732, 735, 774, 736, 844, 783, 788, 724, 746, 765, 811, 775, 788, 734, 765, 753, 732, 786, 797, 818, 739, 771, 750, 758, 761] payload_rows_per_dst=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] dup_rows_per_dst=[3088, 2922, 3382, 3136, 2906, 2976, 2980, 2710] combine_rows_per_src=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64866 dup_rows=24174 dup_rows_remote=21144 dedup_ratio=0.245911 router_tokens_per_expert=[796, 763, 753, 729, 766, 733, 776, 746, 740, 798, 805, 764, 772, 782, 754, 732, 774, 775, 759, 719, 762, 741, 811, 748, 833, 759, 747, 752, 782, 767, 773, 777, 808, 753, 790, 768, 736, 794, 792, 752, 738, 763, 800, 729, 764, 788, 763, 749, 842, 733, 773, 777, 801, 792, 771, 757, 787, 761, 792, 729, 764, 756, 784, 758] payload_rows_per_dst=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] dup_rows_per_dst=[2912, 3030, 2950, 3152, 3038, 3002, 3086, 3004] combine_rows_per_src=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343656 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436200 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528744 weight_movement_bytes=103809024 total_nvlink_bytes=636337768 weight_to_token_ratio=0.194936 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65006 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[744, 801, 749, 778, 790, 755, 734, 741, 795, 757, 803, 806, 762, 760, 775, 741, 788, 782, 761, 772, 799, 731, 778, 715, 768, 768, 803, 792, 804, 786, 767, 714, 757, 781, 747, 745, 749, 767, 810, 775, 802, 756, 782, 763, 758, 823, 772, 787, 757, 776, 743, 784, 767, 807, 748, 772, 805, 729, 738, 778, 714, 744, 760, 737] payload_rows_per_dst=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] dup_rows_per_dst=[2936, 3058, 2950, 3066, 2946, 3154, 2974, 2880] combine_rows_per_src=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65134 dup_rows=23864 dup_rows_remote=20856 dedup_ratio=0.242757 router_tokens_per_expert=[789, 722, 780, 773, 743, 767, 750, 754, 801, 736, 750, 747, 766, 726, 768, 793, 807, 769, 744, 749, 717, 804, 761, 771, 786, 724, 759, 772, 742, 793, 739, 781, 792, 750, 779, 793, 778, 773, 787, 800, 816, 752, 779, 783, 747, 782, 759, 739, 792, 746, 767, 711, 800, 783, 781, 787, 785, 779, 784, 756, 793, 782, 761, 753] payload_rows_per_dst=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] dup_rows_per_dst=[2910, 2980, 3054, 2860, 3082, 3008, 2968, 3002] combine_rows_per_src=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64842 dup_rows=24192 dup_rows_remote=21222 dedup_ratio=0.246094 router_tokens_per_expert=[798, 739, 760, 745, 740, 777, 794, 748, 741, 792, 774, 792, 793, 798, 780, 798, 781, 791, 810, 782, 800, 719, 750, 806, 783, 739, 737, 796, 770, 747, 794, 754, 771, 786, 754, 751, 772, 735, 755, 758, 722, 727, 758, 742, 739, 767, 781, 759, 797, 786, 789, 782, 777, 810, 753, 777, 793, 754, 769, 774, 781, 737, 729, 739] payload_rows_per_dst=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] dup_rows_per_dst=[2892, 3128, 3214, 2970, 3008, 2808, 3120, 3052] combine_rows_per_src=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343736 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266264248 ingress_bytes=369729536 link_bytes=369729536 token_bytes=532184760 weight_movement_bytes=103809024 total_nvlink_bytes=635993784 weight_to_token_ratio=0.195062 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391262478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=344040 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266035176 ingress_bytes=369500160 link_bytes=369500160 token_bytes=531726312 weight_movement_bytes=103809024 total_nvlink_bytes=635535336 weight_to_token_ratio=0.195230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390399355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343960 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132824 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921688 weight_movement_bytes=34603008 total_nvlink_bytes=568524696 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343984 combine_payload_bytes=266264576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266608560 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532873136 weight_movement_bytes=69206016 total_nvlink_bytes=602079152 weight_to_token_ratio=0.129873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65046 dup_rows=24086 dup_rows_remote=21272 dedup_ratio=0.245015 router_tokens_per_expert=[786, 804, 754, 756, 742, 775, 786, 786, 754, 740, 818, 782, 745, 791, 773, 748, 752, 713, 799, 696, 769, 795, 699, 770, 812, 796, 754, 752, 820, 779, 805, 799, 740, 739, 755, 749, 773, 720, 828, 734, 788, 784, 726, 730, 786, 769, 781, 776, 770, 815, 712, 818, 771, 801, 789, 763, 740, 818, 762, 756, 754, 774, 727, 754] payload_rows_per_dst=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] dup_rows_per_dst=[3094, 2938, 2814, 3178, 2966, 3004, 3202, 2890] combine_rows_per_src=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265937088 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529920 weight_movement_bytes=34603008 total_nvlink_bytes=566132928 weight_to_token_ratio=0.065101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21056 dedup_ratio=0.244893 router_tokens_per_expert=[774, 767, 789, 755, 779, 765, 710, 764, 724, 824, 777, 716, 753, 760, 784, 754, 796, 786, 765, 800, 776, 846, 786, 821, 755, 737, 731, 728, 795, 775, 736, 773, 744, 755, 803, 776, 752, 757, 767, 758, 741, 772, 751, 752, 797, 780, 797, 725, 753, 767, 784, 753, 747, 831, 775, 770, 788, 795, 772, 784, 740, 750, 769, 746] payload_rows_per_dst=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] dup_rows_per_dst=[2944, 2920, 3206, 2968, 3018, 2924, 3060, 3034] combine_rows_per_src=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64926 dup_rows=24004 dup_rows_remote=20770 dedup_ratio=0.244181 router_tokens_per_expert=[764, 799, 791, 759, 800, 776, 735, 756, 783, 758, 784, 805, 830, 735, 771, 757, 741, 773, 763, 741, 711, 764, 782, 774, 808, 774, 770, 791, 730, 767, 756, 782, 761, 789, 739, 745, 763, 729, 784, 730, 767, 778, 752, 784, 738, 776, 765, 801, 779, 727, 805, 820, 757, 771, 758, 763, 793, 719, 793, 711, 822, 763, 750, 790] payload_rows_per_dst=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] dup_rows_per_dst=[2982, 2980, 2772, 2820, 3058, 3094, 3064, 3234] combine_rows_per_src=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=345272 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773688 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533202104 weight_movement_bytes=34603008 total_nvlink_bytes=567805112 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682624 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020928 weight_movement_bytes=34603008 total_nvlink_bytes=567623936 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65124 dup_rows=23828 dup_rows_remote=20824 dedup_ratio=0.242391 router_tokens_per_expert=[726, 768, 741, 795, 750, 794, 765, 747, 786, 721, 795, 777, 797, 756, 769, 777, 734, 738, 793, 759, 784, 772, 738, 763, 809, 787, 725, 748, 789, 785, 750, 765, 806, 760, 721, 765, 742, 791, 782, 774, 813, 816, 747, 770, 794, 770, 745, 757, 829, 760, 816, 782, 731, 752, 794, 752, 783, 780, 732, 739, 764, 760, 771, 751] payload_rows_per_dst=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] dup_rows_per_dst=[2930, 3004, 2824, 2936, 3092, 3106, 2974, 2962] combine_rows_per_src=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=342784 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266279680 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532216576 weight_movement_bytes=34603008 total_nvlink_bytes=566819584 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64986 dup_rows=24062 dup_rows_remote=20946 dedup_ratio=0.244771 router_tokens_per_expert=[761, 779, 727, 778, 748, 769, 745, 806, 758, 772, 824, 792, 753, 750, 839, 806, 770, 745, 745, 726, 734, 744, 804, 774, 774, 774, 819, 776, 748, 804, 772, 770, 769, 798, 751, 765, 772, 787, 801, 779, 757, 750, 794, 750, 799, 741, 762, 706, 770, 800, 734, 756, 748, 754, 751, 750, 777, 776, 771, 781, 729, 791, 746, 751] payload_rows_per_dst=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] dup_rows_per_dst=[3116, 3366, 3002, 3096, 2940, 2868, 2830, 2844] combine_rows_per_src=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65180 dup_rows=23878 dup_rows_remote=20890 dedup_ratio=0.242900 router_tokens_per_expert=[771, 743, 775, 772, 779, 782, 727, 790, 797, 750, 768, 731, 812, 802, 741, 743, 779, 735, 818, 736, 735, 758, 783, 779, 783, 791, 774, 787, 780, 798, 737, 808, 774, 753, 742, 765, 712, 755, 792, 795, 738, 774, 750, 817, 746, 816, 719, 757, 757, 777, 757, 749, 762, 761, 791, 777, 809, 805, 786, 784, 728, 743, 712, 785] payload_rows_per_dst=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] dup_rows_per_dst=[3002, 3006, 2966, 3056, 2882, 2988, 2892, 3086] combine_rows_per_src=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73978 payload_rows_remote=64976 dup_rows=24326 dup_rows_remote=21368 dedup_ratio=0.247457 router_tokens_per_expert=[755, 758, 752, 758, 735, 756, 783, 801, 780, 757, 762, 786, 748, 743, 792, 795, 778, 816, 774, 805, 727, 776, 774, 783, 741, 741, 751, 829, 745, 749, 793, 763, 761, 777, 801, 759, 795, 774, 824, 765, 797, 763, 758, 743, 817, 769, 804, 767, 720, 755, 754, 774, 730, 761, 764, 722, 751, 754, 806, 772, 727, 761, 761, 760] payload_rows_per_dst=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] dup_rows_per_dst=[2970, 3076, 3142, 3004, 3082, 3092, 2958, 3002] combine_rows_per_src=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=21068 dedup_ratio=0.243530 router_tokens_per_expert=[801, 728, 802, 829, 820, 801, 766, 775, 795, 771, 721, 709, 753, 774, 751, 764, 792, 766, 822, 745, 774, 765, 733, 755, 788, 749, 722, 785, 756, 772, 746, 744, 772, 753, 735, 758, 755, 783, 765, 799, 792, 727, 785, 714, 804, 786, 775, 764, 778, 756, 716, 797, 751, 762, 755, 781, 776, 732, 787, 769, 757, 796, 807, 791] payload_rows_per_dst=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] dup_rows_per_dst=[3102, 2798, 3084, 2872, 2970, 3008, 3010, 3096] combine_rows_per_src=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343792 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267091696 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839600 weight_movement_bytes=34603008 total_nvlink_bytes=568442608 weight_to_token_ratio=0.064819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343728 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526384 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709040 weight_movement_bytes=34603008 total_nvlink_bytes=567312048 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344280 combine_payload_bytes=266977280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267321560 ingress_bytes=405389312 link_bytes=405389312 token_bytes=534298840 weight_movement_bytes=138412032 total_nvlink_bytes=672710872 weight_to_token_ratio=0.259054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525447345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=345376 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266487072 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532628768 weight_movement_bytes=34603008 total_nvlink_bytes=567231776 weight_to_token_ratio=0.064966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64958 dup_rows=24062 dup_rows_remote=21010 dedup_ratio=0.244771 router_tokens_per_expert=[758, 783, 798, 751, 810, 787, 778, 752, 749, 734, 797, 797, 744, 740, 728, 791, 785, 803, 772, 753, 789, 766, 753, 747, 765, 765, 753, 758, 749, 791, 788, 754, 767, 809, 755, 751, 789, 749, 789, 790, 780, 783, 817, 793, 705, 766, 741, 731, 799, 751, 736, 772, 739, 779, 764, 755, 745, 800, 783, 796, 759, 752, 753, 766] payload_rows_per_dst=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] dup_rows_per_dst=[3074, 2920, 3052, 2992, 3084, 2962, 2974, 3004] combine_rows_per_src=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344720 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267043472 ingress_bytes=370507776 link_bytes=370507776 token_bytes=533742224 weight_movement_bytes=103809024 total_nvlink_bytes=637551248 weight_to_token_ratio=0.194493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394190933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64894 dup_rows=24080 dup_rows_remote=21012 dedup_ratio=0.244954 router_tokens_per_expert=[803, 740, 779, 761, 790, 719, 783, 806, 730, 764, 761, 761, 784, 763, 797, 806, 766, 781, 803, 777, 740, 776, 758, 756, 771, 719, 758, 782, 783, 750, 779, 693, 705, 785, 797, 761, 796, 784, 797, 774, 782, 750, 759, 749, 853, 797, 783, 749, 761, 797, 777, 729, 748, 745, 751, 782, 761, 794, 777, 750, 751, 761, 761, 747] payload_rows_per_dst=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] dup_rows_per_dst=[3026, 3108, 3066, 2912, 3068, 3076, 2944, 2880] combine_rows_per_src=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64782 dup_rows=24132 dup_rows_remote=20882 dedup_ratio=0.245483 router_tokens_per_expert=[747, 772, 749, 806, 800, 756, 787, 804, 813, 744, 768, 768, 714, 749, 790, 730, 720, 761, 756, 743, 791, 783, 788, 795, 767, 796, 749, 798, 774, 770, 790, 762, 748, 759, 793, 754, 764, 770, 791, 800, 767, 780, 795, 779, 753, 751, 727, 799, 793, 812, 771, 748, 726, 764, 726, 758, 747, 761, 767, 767, 780, 747, 768, 747] payload_rows_per_dst=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] dup_rows_per_dst=[2792, 2938, 3048, 3084, 3064, 2790, 3166, 3250] combine_rows_per_src=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343872 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411840 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479808 weight_movement_bytes=69206016 total_nvlink_bytes=601685824 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343624 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149448 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955272 weight_movement_bytes=34603008 total_nvlink_bytes=566558280 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65184 dup_rows=23854 dup_rows_remote=20852 dedup_ratio=0.242655 router_tokens_per_expert=[759, 767, 793, 760, 795, 774, 709, 729, 711, 792, 786, 784, 777, 760, 758, 766, 784, 811, 752, 751, 735, 767, 792, 810, 726, 798, 798, 787, 800, 734, 725, 788, 693, 787, 799, 787, 731, 768, 782, 785, 760, 727, 785, 738, 774, 746, 828, 744, 773, 779, 772, 736, 725, 779, 781, 770, 798, 758, 787, 784, 742, 799, 743, 814] payload_rows_per_dst=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] dup_rows_per_dst=[2988, 3002, 2938, 2988, 3010, 2918, 3002, 3008] combine_rows_per_src=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=342656 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265689728 ingress_bytes=299950080 link_bytes=299950080 token_bytes=531036800 weight_movement_bytes=34603008 total_nvlink_bytes=565639808 weight_to_token_ratio=0.065161 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128688003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65424 dup_rows=23688 dup_rows_remote=20794 dedup_ratio=0.240967 router_tokens_per_expert=[811, 760, 728, 785, 795, 791, 763, 733, 796, 761, 758, 779, 739, 809, 739, 780, 718, 785, 744, 809, 697, 785, 783, 778, 752, 752, 783, 779, 785, 781, 797, 779, 752, 789, 724, 798, 777, 763, 743, 742, 798, 768, 744, 766, 811, 753, 799, 740, 751, 726, 754, 799, 763, 790, 707, 753, 772, 749, 780, 804, 767, 774, 799, 763] payload_rows_per_dst=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] dup_rows_per_dst=[3010, 2972, 2972, 2940, 2870, 3068, 2894, 2962] combine_rows_per_src=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20966 dedup_ratio=0.243917 router_tokens_per_expert=[726, 771, 749, 780, 749, 755, 825, 745, 759, 775, 784, 816, 748, 748, 748, 727, 754, 783, 750, 746, 770, 764, 753, 737, 739, 814, 788, 762, 747, 757, 800, 720, 763, 769, 735, 711, 777, 804, 782, 747, 812, 759, 772, 786, 768, 785, 731, 765, 779, 765, 796, 797, 775, 764, 803, 782, 784, 823, 744, 817, 719, 784, 761, 804] payload_rows_per_dst=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] dup_rows_per_dst=[2986, 2970, 2960, 2946, 2930, 3012, 3112, 3062] combine_rows_per_src=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344144 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267337808 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331472 weight_movement_bytes=34603008 total_nvlink_bytes=568934480 weight_to_token_ratio=0.064759 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64994 dup_rows=23966 dup_rows_remote=20860 dedup_ratio=0.243795 router_tokens_per_expert=[736, 770, 803, 779, 797, 751, 735, 775, 776, 797, 763, 746, 764, 752, 822, 769, 754, 763, 748, 800, 774, 756, 757, 787, 790, 800, 738, 804, 844, 763, 720, 766, 755, 772, 763, 725, 759, 754, 762, 761, 747, 771, 770, 775, 764, 818, 777, 766, 785, 764, 727, 732, 790, 797, 768, 739, 717, 741, 726, 770, 807, 805, 758, 788] payload_rows_per_dst=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] dup_rows_per_dst=[2996, 3084, 2992, 3106, 2852, 3024, 2972, 2940] combine_rows_per_src=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344872 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321576 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298280 weight_movement_bytes=69206016 total_nvlink_bytes=605504296 weight_to_token_ratio=0.129044 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343792 combine_payload_bytes=266166272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266510064 ingress_bytes=369975296 link_bytes=369975296 token_bytes=532676336 weight_movement_bytes=103809024 total_nvlink_bytes=636485360 weight_to_token_ratio=0.194882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392187253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64894 dup_rows=24170 dup_rows_remote=21190 dedup_ratio=0.245870 router_tokens_per_expert=[768, 771, 791, 774, 774, 781, 809, 743, 797, 735, 761, 772, 753, 803, 772, 758, 780, 723, 779, 754, 748, 756, 781, 789, 766, 789, 810, 735, 779, 766, 730, 748, 765, 714, 765, 740, 798, 758, 814, 764, 786, 789, 771, 713, 787, 784, 805, 777, 811, 776, 730, 780, 739, 755, 756, 741, 761, 713, 786, 741, 764, 816, 791, 767] payload_rows_per_dst=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] dup_rows_per_dst=[3074, 3066, 2980, 2912, 2980, 3068, 3038, 3052] combine_rows_per_src=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343416 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266558840 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774264 weight_movement_bytes=69206016 total_nvlink_bytes=601980280 weight_to_token_ratio=0.129897 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64932 dup_rows=24104 dup_rows_remote=21048 dedup_ratio=0.245199 router_tokens_per_expert=[726, 778, 764, 774, 799, 797, 778, 741, 790, 729, 777, 748, 819, 773, 762, 686, 753, 755, 733, 778, 800, 820, 740, 754, 786, 776, 766, 775, 756, 749, 763, 773, 754, 759, 777, 775, 788, 794, 768, 747, 698, 739, 737, 800, 767, 765, 794, 771, 753, 770, 795, 786, 795, 795, 803, 777, 766, 761, 761, 723, 789, 770, 802, 755] payload_rows_per_dst=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] dup_rows_per_dst=[2936, 3020, 2940, 3076, 3056, 2946, 3102, 3028] combine_rows_per_src=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65256 dup_rows=23948 dup_rows_remote=21010 dedup_ratio=0.243612 router_tokens_per_expert=[784, 733, 748, 733, 750, 798, 733, 785, 752, 757, 755, 756, 754, 767, 759, 785, 791, 757, 794, 769, 807, 760, 769, 747, 805, 762, 766, 776, 789, 753, 753, 789, 765, 813, 767, 793, 808, 771, 745, 775, 722, 768, 779, 751, 821, 789, 782, 773, 754, 733, 788, 787, 780, 759, 831, 745, 763, 732, 715, 728, 800, 712, 776, 791] payload_rows_per_dst=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] dup_rows_per_dst=[2884, 3006, 3012, 3084, 3018, 2996, 3010, 2938] combine_rows_per_src=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64642 dup_rows=24114 dup_rows_remote=20804 dedup_ratio=0.245300 router_tokens_per_expert=[722, 790, 819, 752, 774, 791, 766, 712, 735, 744, 834, 773, 774, 767, 759, 773, 767, 746, 772, 755, 782, 806, 755, 772, 833, 750, 806, 771, 755, 790, 818, 775, 776, 750, 785, 770, 785, 760, 772, 767, 777, 730, 754, 785, 737, 789, 739, 751, 715, 744, 759, 762, 762, 770, 739, 792, 772, 766, 764, 756, 813, 768, 775, 730] payload_rows_per_dst=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] dup_rows_per_dst=[3310, 3042, 3012, 3070, 3108, 3014, 2572, 2986] combine_rows_per_src=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65042 dup_rows=23768 dup_rows_remote=20682 dedup_ratio=0.241781 router_tokens_per_expert=[760, 698, 773, 766, 779, 782, 728, 781, 784, 838, 771, 783, 748, 842, 734, 790, 738, 775, 784, 792, 796, 797, 772, 772, 749, 759, 755, 814, 740, 773, 722, 748, 783, 702, 768, 728, 765, 772, 758, 780, 796, 754, 810, 774, 758, 756, 743, 706, 782, 812, 780, 780, 765, 741, 770, 769, 763, 759, 775, 790, 774, 768, 767, 761] payload_rows_per_dst=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] dup_rows_per_dst=[2926, 3086, 3120, 2886, 2886, 2940, 2922, 3002] combine_rows_per_src=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344336 combine_payload_bytes=265805824 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266150160 ingress_bytes=404217856 link_bytes=404217856 token_bytes=531955984 weight_movement_bytes=138412032 total_nvlink_bytes=670368016 weight_to_token_ratio=0.260195 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521039250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343920 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305392 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266864 weight_movement_bytes=34603008 total_nvlink_bytes=566869872 weight_to_token_ratio=0.065011 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345064 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267633640 ingress_bytes=336494592 link_bytes=336494592 token_bytes=534922216 weight_movement_bytes=69206016 total_nvlink_bytes=604128232 weight_to_token_ratio=0.129376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266202060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65230 dup_rows=23860 dup_rows_remote=20804 dedup_ratio=0.242716 router_tokens_per_expert=[794, 752, 744, 796, 775, 766, 817, 768, 752, 731, 741, 766, 813, 708, 792, 745, 800, 772, 768, 753, 781, 790, 829, 780, 797, 767, 830, 758, 766, 767, 833, 737, 714, 774, 739, 740, 737, 763, 767, 750, 754, 728, 798, 754, 785, 797, 754, 765, 748, 723, 803, 781, 757, 749, 763, 757, 773, 784, 795, 786, 739, 721, 819, 747] payload_rows_per_dst=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] dup_rows_per_dst=[2996, 2902, 3092, 3098, 2838, 3056, 2880, 2998] combine_rows_per_src=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=65006 dup_rows=24172 dup_rows_remote=21210 dedup_ratio=0.245890 router_tokens_per_expert=[836, 761, 744, 766, 816, 729, 766, 801, 813, 738, 822, 784, 747, 757, 778, 785, 787, 746, 825, 754, 799, 796, 778, 732, 783, 731, 776, 731, 794, 747, 756, 776, 814, 792, 772, 760, 744, 743, 770, 750, 737, 809, 724, 796, 751, 723, 750, 746, 785, 772, 751, 758, 752, 753, 708, 765, 688, 771, 822, 750, 766, 748, 796, 832] payload_rows_per_dst=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] dup_rows_per_dst=[3090, 3088, 3102, 2974, 2994, 2888, 2962, 3074] combine_rows_per_src=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341784 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265115416 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529889048 weight_movement_bytes=34603008 total_nvlink_bytes=564492056 weight_to_token_ratio=0.065302 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342896 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266754928 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533166960 weight_movement_bytes=34603008 total_nvlink_bytes=567769968 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65202 dup_rows=23826 dup_rows_remote=20818 dedup_ratio=0.242371 router_tokens_per_expert=[773, 782, 749, 796, 724, 748, 798, 734, 768, 735, 758, 768, 807, 762, 755, 739, 799, 751, 759, 790, 780, 746, 691, 776, 790, 785, 767, 746, 753, 751, 785, 765, 817, 750, 777, 764, 794, 805, 772, 798, 797, 785, 753, 737, 771, 784, 771, 759, 759, 781, 775, 768, 775, 736, 741, 748, 775, 763, 822, 770, 781, 798, 709, 787] payload_rows_per_dst=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] dup_rows_per_dst=[2920, 2892, 3014, 3008, 3090, 2980, 2880, 3042] combine_rows_per_src=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344136 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526216 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708296 weight_movement_bytes=34603008 total_nvlink_bytes=569311304 weight_to_token_ratio=0.064714 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344864 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266609440 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532874016 weight_movement_bytes=34603008 total_nvlink_bytes=567477024 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344080 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267411472 ingress_bytes=336273408 link_bytes=336273408 token_bytes=534478864 weight_movement_bytes=69206016 total_nvlink_bytes=603684880 weight_to_token_ratio=0.129483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265369762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64948 dup_rows=24104 dup_rows_remote=21120 dedup_ratio=0.245199 router_tokens_per_expert=[774, 823, 776, 746, 768, 812, 767, 771, 760, 769, 746, 743, 778, 716, 789, 714, 786, 702, 773, 783, 742, 787, 811, 734, 745, 764, 755, 795, 784, 743, 782, 775, 816, 792, 790, 734, 766, 786, 774, 769, 730, 768, 778, 813, 737, 766, 744, 741, 789, 786, 758, 760, 735, 772, 783, 809, 767, 740, 784, 735, 773, 787, 780, 777] payload_rows_per_dst=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] dup_rows_per_dst=[3114, 2900, 2984, 2988, 3096, 2974, 3076, 2972] combine_rows_per_src=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65142 dup_rows=23952 dup_rows_remote=20978 dedup_ratio=0.243652 router_tokens_per_expert=[748, 754, 799, 750, 783, 806, 774, 722, 752, 731, 737, 792, 729, 766, 774, 756, 773, 740, 732, 815, 804, 713, 777, 698, 761, 778, 779, 834, 778, 770, 776, 809, 740, 735, 820, 759, 750, 748, 758, 782, 769, 755, 799, 794, 763, 772, 760, 754, 761, 806, 791, 750, 756, 774, 761, 798, 757, 762, 769, 791, 786, 825, 724, 773] payload_rows_per_dst=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] dup_rows_per_dst=[2922, 2862, 2872, 3108, 2974, 3068, 3034, 3112] combine_rows_per_src=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65032 dup_rows=24008 dup_rows_remote=21072 dedup_ratio=0.244222 router_tokens_per_expert=[818, 717, 752, 773, 710, 760, 747, 743, 753, 834, 724, 766, 742, 762, 739, 744, 788, 786, 815, 799, 779, 749, 726, 761, 824, 782, 775, 774, 823, 763, 752, 758, 762, 847, 804, 807, 734, 753, 756, 793, 810, 798, 792, 764, 768, 808, 776, 789, 740, 747, 762, 740, 749, 708, 731, 779, 720, 797, 796, 713, 750, 771, 760, 790] payload_rows_per_dst=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] dup_rows_per_dst=[2908, 2878, 3038, 2912, 3170, 3108, 3058, 2936] combine_rows_per_src=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64906 dup_rows=24090 dup_rows_remote=21120 dedup_ratio=0.245056 router_tokens_per_expert=[764, 790, 776, 741, 795, 728, 740, 755, 776, 803, 730, 795, 791, 747, 811, 760, 772, 759, 732, 755, 793, 764, 774, 787, 760, 776, 732, 766, 742, 807, 788, 787, 786, 777, 738, 761, 759, 784, 751, 742, 782, 837, 803, 761, 789, 758, 753, 790, 792, 721, 756, 744, 768, 757, 777, 764, 768, 786, 788, 697, 794, 759, 749, 765] payload_rows_per_dst=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] dup_rows_per_dst=[2970, 3054, 3190, 2860, 3034, 3012, 2982, 2988] combine_rows_per_src=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344272 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371280 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398288 weight_movement_bytes=34603008 total_nvlink_bytes=567001296 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73986 payload_rows_remote=64660 dup_rows=24318 dup_rows_remote=21150 dedup_ratio=0.247375 router_tokens_per_expert=[786, 773, 777, 771, 751, 762, 840, 747, 773, 775, 742, 819, 785, 776, 769, 808, 746, 754, 776, 743, 778, 768, 788, 737, 785, 722, 790, 748, 729, 843, 732, 751, 760, 814, 760, 795, 782, 789, 718, 756, 817, 714, 772, 797, 732, 713, 772, 743, 779, 810, 784, 766, 743, 790, 703, 755, 767, 770, 773, 759, 717, 776, 789, 793] payload_rows_per_dst=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] dup_rows_per_dst=[3154, 3168, 3024, 2996, 3084, 2928, 3012, 2952] combine_rows_per_src=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344480 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166112 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987744 weight_movement_bytes=34603008 total_nvlink_bytes=568590752 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65046 dup_rows=24090 dup_rows_remote=21186 dedup_ratio=0.245056 router_tokens_per_expert=[810, 763, 770, 751, 736, 849, 782, 722, 796, 695, 755, 772, 749, 797, 775, 799, 769, 783, 748, 791, 747, 734, 827, 780, 809, 766, 790, 769, 754, 778, 754, 727, 764, 739, 723, 749, 771, 775, 727, 775, 738, 782, 739, 752, 691, 768, 761, 805, 778, 820, 768, 819, 750, 775, 774, 750, 740, 828, 750, 800, 760, 799, 762, 773] payload_rows_per_dst=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] dup_rows_per_dst=[3066, 3030, 3110, 3022, 2818, 2904, 3112, 3028] combine_rows_per_src=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65012 dup_rows=24120 dup_rows_remote=21114 dedup_ratio=0.245361 router_tokens_per_expert=[753, 778, 719, 795, 712, 763, 773, 784, 771, 755, 818, 802, 815, 811, 733, 827, 797, 765, 723, 759, 793, 780, 786, 757, 749, 774, 773, 767, 755, 755, 778, 731, 801, 778, 764, 801, 800, 815, 744, 794, 736, 686, 789, 787, 717, 703, 771, 760, 716, 720, 784, 776, 761, 791, 762, 779, 788, 779, 780, 770, 773, 736, 730, 810] payload_rows_per_dst=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] dup_rows_per_dst=[2942, 3098, 2980, 2912, 3220, 2842, 3006, 3120] combine_rows_per_src=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344104 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266199080 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532054056 weight_movement_bytes=69206016 total_nvlink_bytes=601260072 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64974 dup_rows=23992 dup_rows_remote=21038 dedup_ratio=0.244059 router_tokens_per_expert=[743, 720, 750, 789, 800, 766, 794, 784, 800, 730, 743, 812, 740, 812, 736, 740, 773, 774, 799, 755, 750, 749, 759, 766, 770, 782, 789, 765, 752, 763, 746, 779, 783, 746, 787, 765, 744, 789, 735, 770, 746, 758, 783, 781, 775, 789, 747, 811, 768, 769, 762, 802, 800, 758, 828, 768, 752, 799, 749, 738, 779, 748, 771, 722] payload_rows_per_dst=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] dup_rows_per_dst=[2936, 2976, 2972, 2954, 2982, 3146, 3140, 2886] combine_rows_per_src=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=343240 combine_payload_bytes=264847360 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265190600 ingress_bytes=299450368 link_bytes=299450368 token_bytes=530037960 weight_movement_bytes=34603008 total_nvlink_bytes=564640968 weight_to_token_ratio=0.065284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126807627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344928 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773344 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201760 weight_movement_bytes=34603008 total_nvlink_bytes=567804768 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344504 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266633656 ingress_bytes=439304192 link_bytes=439304192 token_bytes=532922808 weight_movement_bytes=173015040 total_nvlink_bytes=705937848 weight_to_token_ratio=0.324653 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653066308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64968 dup_rows=24116 dup_rows_remote=21106 dedup_ratio=0.245321 router_tokens_per_expert=[763, 769, 753, 788, 756, 812, 804, 765, 712, 780, 792, 761, 734, 807, 761, 793, 764, 738, 756, 764, 762, 792, 762, 777, 785, 763, 773, 749, 792, 743, 757, 748, 731, 723, 778, 761, 773, 765, 801, 801, 785, 762, 782, 746, 834, 755, 848, 736, 768, 741, 710, 767, 749, 779, 759, 741, 835, 801, 775, 799, 713, 719, 777, 763] payload_rows_per_dst=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] dup_rows_per_dst=[3104, 3020, 3010, 3006, 2944, 3112, 2868, 3052] combine_rows_per_src=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64770 dup_rows=24268 dup_rows_remote=21306 dedup_ratio=0.246867 router_tokens_per_expert=[836, 778, 761, 776, 764, 806, 737, 752, 782, 750, 781, 745, 772, 800, 744, 741, 769, 796, 735, 789, 800, 755, 733, 781, 794, 784, 737, 813, 724, 779, 768, 779, 764, 813, 754, 776, 762, 741, 761, 743, 791, 754, 774, 758, 751, 720, 802, 798, 784, 770, 759, 722, 780, 738, 809, 796, 751, 737, 791, 779, 727, 772, 739, 775] payload_rows_per_dst=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] dup_rows_per_dst=[3080, 3004, 3126, 3090, 2962, 3000, 3048, 2958] combine_rows_per_src=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344048 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477552 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611056 weight_movement_bytes=34603008 total_nvlink_bytes=567214064 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=65022 dup_rows=24100 dup_rows_remote=21112 dedup_ratio=0.245158 router_tokens_per_expert=[809, 770, 751, 779, 721, 761, 761, 750, 759, 767, 739, 812, 741, 719, 735, 763, 769, 737, 749, 759, 764, 807, 754, 808, 728, 834, 768, 764, 748, 792, 785, 797, 805, 783, 789, 769, 807, 814, 723, 767, 804, 767, 805, 795, 755, 735, 769, 767, 787, 798, 805, 729, 738, 743, 773, 759, 733, 758, 758, 780, 758, 738, 791, 750] payload_rows_per_dst=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] dup_rows_per_dst=[2894, 2986, 3094, 3086, 2758, 3298, 2996, 2988] combine_rows_per_src=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344296 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453224 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562152 weight_movement_bytes=69206016 total_nvlink_bytes=601768168 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344304 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265642224 ingress_bytes=334503936 link_bytes=334503936 token_bytes=530940144 weight_movement_bytes=69206016 total_nvlink_bytes=600146160 weight_to_token_ratio=0.130346 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258711381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=20958 dedup_ratio=0.244975 router_tokens_per_expert=[728, 750, 775, 719, 734, 779, 747, 776, 794, 751, 779, 815, 778, 783, 766, 759, 802, 739, 754, 793, 758, 816, 785, 772, 746, 776, 736, 772, 806, 776, 740, 772, 750, 770, 777, 784, 784, 762, 753, 718, 772, 737, 754, 775, 759, 767, 780, 794, 786, 742, 792, 700, 799, 812, 773, 776, 791, 779, 800, 779, 770, 768, 725, 748] payload_rows_per_dst=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] dup_rows_per_dst=[3124, 3138, 3082, 2710, 3004, 3022, 3020, 2982] combine_rows_per_src=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65068 dup_rows=23966 dup_rows_remote=20890 dedup_ratio=0.243795 router_tokens_per_expert=[752, 745, 768, 736, 780, 777, 766, 769, 750, 767, 781, 774, 762, 767, 757, 815, 757, 747, 797, 743, 738, 787, 746, 767, 779, 831, 734, 786, 784, 762, 779, 725, 745, 789, 803, 754, 765, 798, 765, 752, 731, 782, 779, 739, 747, 779, 753, 776, 797, 800, 769, 764, 774, 749, 775, 792, 774, 778, 750, 786, 760, 791, 738, 770] payload_rows_per_dst=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] dup_rows_per_dst=[2922, 3076, 2962, 3012, 3048, 2966, 3074, 2906] combine_rows_per_src=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64988 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[725, 749, 776, 763, 731, 759, 786, 827, 783, 736, 773, 762, 797, 847, 759, 760, 774, 764, 752, 779, 753, 749, 808, 770, 747, 764, 735, 763, 775, 758, 756, 784, 793, 786, 751, 778, 760, 775, 791, 774, 767, 739, 753, 773, 779, 732, 788, 795, 733, 768, 743, 721, 742, 829, 777, 762, 761, 765, 761, 777, 793, 772, 803, 747] payload_rows_per_dst=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] dup_rows_per_dst=[2950, 3052, 3056, 3010, 3050, 3034, 2874, 3072] combine_rows_per_src=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262822600 prefetch_bytes=761266176 reduce_bytes=415236096 weight_movement_bytes=1176502272 egress_bytes=6135370440 ingress_bytes=7303954432 link_bytes=7303954432 total_nvlink_bytes=13439324872 weight_to_token_ratio=0.095941 prefetch_share_of_ingress=0.104227 dedup_ratio=0.243957 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=12 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.194964892e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64850 dup_rows=24146 dup_rows_remote=21144 dedup_ratio=0.245626 router_tokens_per_expert=[800, 768, 760, 748, 789, 730, 744, 819, 757, 803, 770, 732, 789, 758, 751, 740, 776, 780, 789, 798, 767, 743, 721, 732, 768, 741, 787, 786, 749, 765, 775, 768, 794, 822, 735, 776, 764, 783, 770, 732, 786, 751, 764, 782, 746, 751, 761, 780, 741, 748, 742, 781, 761, 782, 838, 762, 791, 756, 783, 741, 784, 751, 808, 783] payload_rows_per_dst=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] dup_rows_per_dst=[2984, 2948, 3018, 2996, 3100, 3022, 3002, 3076] combine_rows_per_src=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343488 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231232 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118976 weight_movement_bytes=34603008 total_nvlink_bytes=566721984 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344536 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674648 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004760 weight_movement_bytes=34603008 total_nvlink_bytes=567607768 weight_to_token_ratio=0.064921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65204 dup_rows=23762 dup_rows_remote=20894 dedup_ratio=0.241720 router_tokens_per_expert=[810, 784, 705, 780, 799, 803, 778, 754, 785, 747, 750, 734, 759, 758, 734, 784, 731, 784, 796, 745, 754, 795, 784, 757, 825, 769, 764, 743, 752, 738, 766, 746, 785, 818, 775, 781, 784, 760, 764, 783, 812, 797, 806, 775, 789, 724, 777, 741, 783, 772, 721, 721, 829, 701, 732, 824, 758, 781, 729, 771, 748, 774, 764, 760] payload_rows_per_dst=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] dup_rows_per_dst=[3058, 2868, 3026, 2868, 3054, 3084, 2902, 2902] combine_rows_per_src=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343832 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266862360 ingress_bytes=301121536 link_bytes=301121536 token_bytes=533380888 weight_movement_bytes=34603008 total_nvlink_bytes=567983896 weight_to_token_ratio=0.064875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133096098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344208 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266535056 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532725904 weight_movement_bytes=103809024 total_nvlink_bytes=636534928 weight_to_token_ratio=0.194864 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343976 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969576 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595176 weight_movement_bytes=34603008 total_nvlink_bytes=566198184 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64980 dup_rows=24016 dup_rows_remote=20994 dedup_ratio=0.244303 router_tokens_per_expert=[769, 758, 813, 791, 718, 754, 747, 772, 743, 744, 727, 801, 732, 794, 764, 766, 830, 777, 761, 780, 752, 752, 757, 790, 784, 796, 806, 737, 741, 763, 785, 761, 789, 788, 750, 768, 765, 827, 724, 754, 795, 751, 732, 797, 785, 773, 801, 757, 789, 745, 782, 735, 725, 766, 789, 777, 773, 769, 774, 763, 686, 800, 791, 767] payload_rows_per_dst=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] dup_rows_per_dst=[2930, 2916, 3064, 2972, 3022, 3070, 3084, 2958] combine_rows_per_src=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344392 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267419976 ingress_bytes=301678592 link_bytes=301678592 token_bytes=534495560 weight_movement_bytes=34603008 total_nvlink_bytes=569098568 weight_to_token_ratio=0.064740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135192255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266946016 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6137429472 ingress_bytes=7548239872 link_bytes=7548239872 total_nvlink_bytes=13685669344 weight_to_token_ratio=0.115654 prefetch_share_of_ingress=0.087101 dedup_ratio=0.243662 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234931259e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262195000 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135054136 ingress_bytes=7511261184 link_bytes=7511261184 total_nvlink_bytes=13646315320 weight_to_token_ratio=0.112877 prefetch_share_of_ingress=0.082923 dedup_ratio=0.243981 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228881353e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261720312 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6134817016 ingress_bytes=7476420608 link_bytes=7476420608 total_nvlink_bytes=13611237624 weight_to_token_ratio=0.110059 prefetch_share_of_ingress=0.097194 dedup_ratio=0.244081 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223181254e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343896 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501976 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660056 weight_movement_bytes=69206016 total_nvlink_bytes=601866072 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64678 dup_rows=24234 dup_rows_remote=21260 dedup_ratio=0.246521 router_tokens_per_expert=[755, 718, 773, 776, 788, 767, 804, 756, 788, 823, 748, 781, 769, 745, 797, 776, 734, 763, 755, 750, 781, 754, 808, 739, 784, 755, 832, 752, 749, 749, 764, 774, 760, 800, 795, 829, 759, 749, 751, 753, 759, 745, 734, 772, 771, 747, 725, 784, 780, 824, 758, 735, 776, 835, 789, 762, 749, 728, 772, 704, 801, 780, 762, 757] payload_rows_per_dst=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] dup_rows_per_dst=[2974, 3128, 2984, 3048, 3058, 2984, 3124, 2934] combine_rows_per_src=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267437256 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6137674952 ingress_bytes=7548485632 link_bytes=7548485632 total_nvlink_bytes=13686160584 weight_to_token_ratio=0.115650 prefetch_share_of_ingress=0.082514 dedup_ratio=0.243349 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234971467e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64734 dup_rows=24156 dup_rows_remote=20992 dedup_ratio=0.245728 router_tokens_per_expert=[715, 724, 782, 773, 772, 787, 777, 726, 779, 730, 784, 815, 761, 769, 760, 762, 782, 783, 790, 806, 738, 765, 829, 784, 779, 716, 713, 819, 773, 758, 733, 725, 808, 768, 778, 760, 794, 740, 785, 774, 743, 748, 716, 795, 774, 767, 786, 736, 771, 791, 784, 770, 797, 777, 765, 706, 776, 753, 837, 745, 779, 771, 785, 764] payload_rows_per_dst=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] dup_rows_per_dst=[2982, 3030, 3096, 2888, 3086, 2826, 3084, 3164] combine_rows_per_src=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12252499184 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6130207984 ingress_bytes=7437205504 link_bytes=7437205504 total_nvlink_bytes=13567413488 weight_to_token_ratio=0.107318 prefetch_share_of_ingress=0.093054 dedup_ratio=0.244641 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216765459e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343752 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264840 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185928 weight_movement_bytes=34603008 total_nvlink_bytes=564788936 weight_to_token_ratio=0.065266 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342904 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265493368 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530643832 weight_movement_bytes=69206016 total_nvlink_bytes=599849848 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233130616 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6120514168 ingress_bytes=7565942784 link_bytes=7565942784 total_nvlink_bytes=13686456952 weight_to_token_ratio=0.118802 prefetch_share_of_ingress=0.086897 dedup_ratio=0.244675 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237827547e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65094 dup_rows=23944 dup_rows_remote=20982 dedup_ratio=0.243571 router_tokens_per_expert=[743, 765, 766, 783, 747, 771, 785, 748, 784, 750, 757, 759, 803, 812, 732, 761, 746, 726, 770, 775, 792, 816, 703, 723, 728, 747, 732, 779, 712, 764, 808, 733, 748, 777, 770, 783, 731, 760, 815, 816, 778, 734, 768, 799, 765, 753, 800, 809, 773, 759, 783, 807, 766, 736, 746, 783, 755, 779, 800, 785, 797, 791, 786, 780] payload_rows_per_dst=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] dup_rows_per_dst=[2962, 2938, 2976, 2952, 3040, 3060, 2900, 3116] combine_rows_per_src=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344304 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969328 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594352 weight_movement_bytes=69206016 total_nvlink_bytes=602800368 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64720 dup_rows=24246 dup_rows_remote=21156 dedup_ratio=0.246643 router_tokens_per_expert=[756, 816, 727, 788, 770, 761, 802, 781, 734, 767, 756, 756, 747, 781, 753, 723, 794, 769, 733, 807, 748, 728, 787, 733, 786, 743, 786, 682, 754, 744, 787, 766, 772, 750, 789, 704, 765, 767, 800, 809, 752, 762, 796, 756, 767, 796, 769, 799, 823, 762, 776, 808, 757, 768, 781, 786, 785, 699, 804, 774, 771, 808, 752, 780] payload_rows_per_dst=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] dup_rows_per_dst=[3090, 3112, 2910, 3290, 2758, 3144, 3084, 2858] combine_rows_per_src=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343504 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265436624 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530529744 weight_movement_bytes=69206016 total_nvlink_bytes=599735760 weight_to_token_ratio=0.130447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233329296 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6120614544 ingress_bytes=7531438080 link_bytes=7531438080 total_nvlink_bytes=13652052624 weight_to_token_ratio=0.115972 prefetch_share_of_ingress=0.096484 dedup_ratio=0.244823 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232182398e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-29 23:41:53] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 677.3 | throughput per GPU (TFLOP/s/GPU): 188.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.121784E+01 | loss scale: 1.0 | grad norm: 382.140 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65140 dup_rows=23926 dup_rows_remote=21004 dedup_ratio=0.243388 router_tokens_per_expert=[763, 770, 806, 740, 761, 758, 779, 781, 738, 802, 733, 780, 736, 730, 778, 813, 806, 776, 756, 859, 794, 789, 784, 770, 794, 736, 778, 740, 755, 743, 796, 779, 806, 763, 752, 726, 795, 764, 725, 749, 747, 775, 758, 705, 806, 797, 767, 763, 767, 803, 750, 745, 807, 727, 767, 739, 782, 752, 802, 754, 762, 770, 753, 751] payload_rows_per_dst=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] dup_rows_per_dst=[3026, 2948, 3122, 3026, 2922, 3030, 2922, 2930] combine_rows_per_src=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65038 dup_rows=23872 dup_rows_remote=20734 dedup_ratio=0.242839 router_tokens_per_expert=[733, 778, 772, 776, 729, 737, 761, 768, 781, 751, 773, 771, 793, 768, 771, 719, 810, 792, 781, 768, 759, 846, 744, 766, 796, 751, 810, 771, 767, 755, 771, 744, 801, 767, 735, 766, 776, 742, 741, 758, 790, 792, 743, 716, 793, 771, 785, 750, 759, 750, 774, 789, 827, 759, 765, 752, 817, 768, 764, 794, 742, 770, 739, 745] payload_rows_per_dst=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] dup_rows_per_dst=[2914, 3090, 3138, 2946, 2876, 2956, 3022, 2930] combine_rows_per_src=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64644 dup_rows=24128 dup_rows_remote=20738 dedup_ratio=0.245443 router_tokens_per_expert=[814, 758, 812, 743, 732, 739, 779, 759, 745, 712, 752, 790, 812, 792, 832, 777, 706, 727, 769, 730, 770, 722, 760, 727, 779, 751, 754, 741, 725, 807, 760, 758, 760, 739, 796, 761, 723, 753, 789, 777, 784, 790, 832, 801, 775, 812, 767, 751, 749, 768, 785, 733, 799, 770, 788, 769, 823, 785, 771, 788, 758, 778, 772, 772] payload_rows_per_dst=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] dup_rows_per_dst=[3018, 2826, 2798, 2984, 2962, 3222, 2928, 3390] combine_rows_per_src=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64914 dup_rows=24148 dup_rows_remote=21040 dedup_ratio=0.245646 router_tokens_per_expert=[818, 694, 803, 808, 744, 780, 761, 714, 771, 761, 803, 735, 753, 775, 760, 771, 741, 724, 760, 785, 788, 749, 753, 775, 717, 800, 763, 772, 780, 808, 772, 763, 790, 769, 753, 717, 781, 806, 775, 740, 816, 757, 713, 762, 756, 797, 739, 831, 832, 771, 815, 791, 750, 741, 757, 768, 800, 742, 761, 786, 706, 807, 745, 777] payload_rows_per_dst=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] dup_rows_per_dst=[2998, 2996, 2892, 3108, 3032, 3036, 3108, 2978] combine_rows_per_src=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64890 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[772, 747, 781, 773, 733, 763, 744, 735, 744, 742, 786, 805, 806, 768, 844, 805, 757, 754, 751, 791, 778, 725, 759, 757, 697, 730, 785, 796, 803, 723, 783, 787, 784, 806, 793, 735, 792, 767, 740, 801, 770, 791, 795, 761, 770, 741, 760, 748, 779, 812, 753, 762, 813, 781, 779, 771, 718, 748, 768, 746, 763, 771, 734, 776] payload_rows_per_dst=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] dup_rows_per_dst=[2960, 3120, 2964, 3028, 3108, 3008, 3118, 2844] combine_rows_per_src=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65184 dup_rows=23834 dup_rows_remote=20870 dedup_ratio=0.242452 router_tokens_per_expert=[799, 805, 774, 730, 767, 836, 752, 791, 770, 755, 816, 735, 758, 777, 746, 768, 796, 732, 748, 766, 769, 785, 770, 784, 729, 778, 726, 786, 760, 736, 760, 805, 765, 773, 752, 738, 753, 773, 759, 796, 750, 816, 772, 742, 802, 767, 714, 743, 724, 736, 763, 775, 793, 848, 784, 763, 789, 755, 774, 769, 724, 771, 767, 793] payload_rows_per_dst=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] dup_rows_per_dst=[3130, 2964, 2994, 2910, 2968, 2926, 3060, 2882] combine_rows_per_src=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344576 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158016 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971456 weight_movement_bytes=34603008 total_nvlink_bytes=568574464 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343088 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738736 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134384 weight_movement_bytes=69206016 total_nvlink_bytes=602340400 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341528 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123352 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905176 weight_movement_bytes=34603008 total_nvlink_bytes=564508184 weight_to_token_ratio=0.065300 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343816 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231560 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119304 weight_movement_bytes=34603008 total_nvlink_bytes=566722312 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344128 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133568 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923008 weight_movement_bytes=69206016 total_nvlink_bytes=601129024 weight_to_token_ratio=0.130105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64966 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[813, 748, 796, 752, 804, 702, 799, 703, 787, 778, 800, 754, 777, 780, 774, 719, 781, 769, 744, 793, 777, 818, 798, 798, 756, 795, 745, 809, 756, 756, 757, 749, 737, 745, 778, 738, 783, 779, 782, 798, 794, 787, 762, 756, 727, 774, 760, 766, 770, 765, 768, 797, 769, 769, 764, 755, 741, 779, 764, 778, 756, 748, 719, 757] payload_rows_per_dst=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] dup_rows_per_dst=[2864, 3074, 3058, 3076, 2960, 3038, 3022, 2954] combine_rows_per_src=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64990 dup_rows=23978 dup_rows_remote=20788 dedup_ratio=0.243917 router_tokens_per_expert=[774, 782, 791, 792, 725, 798, 724, 724, 784, 795, 738, 776, 763, 775, 823, 758, 821, 765, 778, 790, 758, 720, 771, 803, 790, 776, 734, 750, 751, 765, 745, 726, 769, 783, 784, 780, 760, 767, 746, 758, 772, 790, 793, 808, 736, 745, 745, 778, 735, 786, 839, 781, 796, 710, 765, 767, 763, 774, 775, 794, 743, 725, 755, 765] payload_rows_per_dst=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] dup_rows_per_dst=[3190, 3084, 2964, 3110, 3148, 2834, 2708, 2940] combine_rows_per_src=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344216 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267337880 ingress_bytes=370802688 link_bytes=370802688 token_bytes=534331544 weight_movement_bytes=103809024 total_nvlink_bytes=638140568 weight_to_token_ratio=0.194278 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395300663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65102 dup_rows=24064 dup_rows_remote=21106 dedup_ratio=0.244792 router_tokens_per_expert=[779, 737, 802, 773, 748, 734, 716, 743, 746, 772, 750, 759, 759, 760, 796, 759, 786, 763, 770, 811, 808, 789, 803, 744, 772, 765, 771, 800, 784, 767, 766, 763, 760, 769, 737, 760, 697, 780, 800, 745, 761, 768, 760, 761, 755, 773, 793, 798, 769, 764, 797, 772, 731, 790, 781, 787, 790, 770, 776, 755, 736, 791, 776, 755] payload_rows_per_dst=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] dup_rows_per_dst=[2842, 3008, 3044, 3044, 2958, 3024, 3094, 3050] combine_rows_per_src=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343960 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266444696 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532545432 weight_movement_bytes=103809024 total_nvlink_bytes=636354456 weight_to_token_ratio=0.194930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64938 dup_rows=23992 dup_rows_remote=20916 dedup_ratio=0.244059 router_tokens_per_expert=[817, 792, 791, 759, 801, 822, 740, 793, 830, 754, 764, 755, 741, 760, 761, 765, 797, 741, 794, 790, 769, 791, 796, 747, 769, 759, 758, 756, 781, 758, 751, 757, 769, 768, 763, 767, 780, 772, 743, 763, 763, 781, 726, 770, 777, 789, 783, 768, 750, 808, 711, 731, 725, 801, 756, 763, 713, 753, 789, 746, 801, 742, 755, 767] payload_rows_per_dst=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] dup_rows_per_dst=[3186, 3032, 3076, 2984, 2978, 2922, 2880, 2934] combine_rows_per_src=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343112 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542152 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741192 weight_movement_bytes=34603008 total_nvlink_bytes=567344200 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64802 dup_rows=24136 dup_rows_remote=20994 dedup_ratio=0.245524 router_tokens_per_expert=[712, 735, 777, 734, 742, 744, 770, 782, 741, 756, 763, 783, 815, 802, 779, 766, 789, 740, 747, 810, 754, 735, 782, 733, 775, 782, 765, 763, 752, 792, 769, 831, 747, 833, 783, 785, 730, 780, 763, 741, 735, 796, 754, 792, 785, 765, 776, 778, 779, 806, 772, 782, 733, 787, 724, 796, 729, 739, 789, 773, 779, 767, 758, 776] payload_rows_per_dst=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] dup_rows_per_dst=[2786, 2990, 2692, 3108, 3018, 3292, 3108, 3142] combine_rows_per_src=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65138 dup_rows=23950 dup_rows_remote=21076 dedup_ratio=0.243632 router_tokens_per_expert=[769, 743, 772, 781, 800, 789, 765, 741, 793, 780, 773, 787, 772, 673, 772, 794, 746, 769, 798, 748, 745, 794, 761, 816, 797, 778, 743, 771, 777, 753, 778, 804, 722, 761, 785, 705, 764, 759, 798, 772, 743, 766, 710, 773, 779, 751, 748, 775, 802, 780, 795, 814, 728, 766, 757, 780, 768, 770, 727, 766, 777, 815, 801, 713] payload_rows_per_dst=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] dup_rows_per_dst=[3000, 3008, 3000, 3012, 2896, 2874, 3116, 3044] combine_rows_per_src=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64994 dup_rows=24006 dup_rows_remote=21062 dedup_ratio=0.244202 router_tokens_per_expert=[767, 733, 753, 771, 764, 779, 762, 780, 808, 780, 792, 776, 744, 812, 787, 828, 737, 743, 796, 768, 768, 790, 754, 804, 802, 773, 737, 734, 793, 780, 751, 754, 784, 764, 754, 794, 718, 797, 746, 779, 772, 769, 778, 747, 760, 740, 797, 779, 786, 776, 757, 745, 733, 780, 786, 812, 731, 736, 740, 796, 697, 791, 771, 717] payload_rows_per_dst=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] dup_rows_per_dst=[2908, 3290, 2986, 2944, 2840, 2980, 3120, 2938] combine_rows_per_src=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344832 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002624 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660416 weight_movement_bytes=34603008 total_nvlink_bytes=568263424 weight_to_token_ratio=0.064841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343416 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329464 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315512 weight_movement_bytes=103809024 total_nvlink_bytes=636124536 weight_to_token_ratio=0.195014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65098 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[770, 774, 740, 753, 740, 787, 742, 778, 746, 756, 774, 767, 755, 810, 743, 770, 780, 811, 748, 776, 757, 800, 800, 783, 717, 724, 817, 756, 780, 761, 792, 759, 822, 767, 804, 752, 776, 802, 748, 756, 747, 752, 756, 768, 802, 752, 744, 762, 746, 750, 800, 791, 795, 756, 778, 746, 722, 822, 759, 787, 779, 735, 766, 744] payload_rows_per_dst=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] dup_rows_per_dst=[2892, 3022, 3124, 3012, 3052, 2994, 2938, 2952] combine_rows_per_src=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343184 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265772176 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531201168 weight_movement_bytes=34603008 total_nvlink_bytes=565804176 weight_to_token_ratio=0.065141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344856 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267150104 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955352 weight_movement_bytes=34603008 total_nvlink_bytes=568558360 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344224 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559648 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532775072 weight_movement_bytes=69206016 total_nvlink_bytes=601981088 weight_to_token_ratio=0.129897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65328 dup_rows=23716 dup_rows_remote=20658 dedup_ratio=0.241252 router_tokens_per_expert=[738, 787, 743, 744, 816, 793, 775, 800, 796, 771, 746, 746, 762, 771, 734, 768, 783, 792, 795, 785, 761, 749, 796, 723, 787, 775, 785, 738, 759, 770, 784, 741, 763, 728, 733, 747, 779, 751, 776, 820, 779, 804, 708, 739, 744, 745, 725, 762, 782, 807, 787, 722, 783, 758, 753, 767, 770, 744, 804, 854, 768, 766, 760, 811] payload_rows_per_dst=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] dup_rows_per_dst=[2936, 2904, 3036, 3012, 2916, 2888, 3058, 2966] combine_rows_per_src=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344248 combine_payload_bytes=266641408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266985656 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533627064 weight_movement_bytes=103809024 total_nvlink_bytes=637436088 weight_to_token_ratio=0.194535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64792 dup_rows=24038 dup_rows_remote=20992 dedup_ratio=0.244527 router_tokens_per_expert=[807, 790, 733, 789, 753, 766, 763, 758, 736, 768, 781, 760, 817, 727, 785, 735, 764, 768, 767, 768, 781, 750, 762, 780, 765, 796, 781, 770, 748, 739, 804, 765, 778, 781, 772, 792, 773, 729, 751, 699, 783, 736, 735, 777, 730, 775, 813, 777, 793, 746, 771, 761, 806, 745, 823, 754, 763, 786, 769, 791, 805, 731, 767, 764] payload_rows_per_dst=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] dup_rows_per_dst=[3046, 2982, 2950, 2972, 3154, 3086, 2900, 2948] combine_rows_per_src=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65028 dup_rows=24044 dup_rows_remote=20944 dedup_ratio=0.244588 router_tokens_per_expert=[746, 766, 793, 758, 753, 789, 778, 762, 747, 765, 784, 806, 737, 789, 793, 788, 827, 746, 762, 784, 763, 756, 738, 755, 778, 807, 772, 734, 748, 752, 791, 776, 774, 809, 772, 719, 758, 772, 803, 759, 718, 774, 785, 784, 778, 739, 787, 789, 751, 736, 735, 824, 736, 799, 783, 770, 755, 778, 750, 740, 785, 731, 762, 754] payload_rows_per_dst=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] dup_rows_per_dst=[2934, 3036, 3044, 2968, 3100, 3058, 3014, 2890] combine_rows_per_src=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=64958 dup_rows=23844 dup_rows_remote=20750 dedup_ratio=0.242554 router_tokens_per_expert=[771, 717, 838, 736, 711, 794, 773, 763, 754, 771, 772, 789, 742, 759, 785, 809, 812, 801, 805, 780, 726, 814, 790, 770, 767, 739, 743, 789, 773, 789, 758, 734, 721, 770, 744, 803, 762, 790, 753, 765, 794, 757, 790, 784, 755, 722, 723, 767, 760, 768, 800, 740, 757, 778, 776, 784, 742, 766, 753, 735, 759, 798, 761, 801] payload_rows_per_dst=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] dup_rows_per_dst=[2878, 3008, 3094, 2900, 2978, 3020, 3028, 2938] combine_rows_per_src=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=343944 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267927432 ingress_bytes=336789504 link_bytes=336789504 token_bytes=535510920 weight_movement_bytes=69206016 total_nvlink_bytes=604716936 weight_to_token_ratio=0.129234 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267311790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65034 dup_rows=23958 dup_rows_remote=21016 dedup_ratio=0.243713 router_tokens_per_expert=[737, 796, 795, 817, 773, 725, 805, 788, 790, 784, 730, 805, 766, 801, 748, 759, 725, 755, 791, 756, 796, 759, 777, 747, 761, 713, 808, 762, 781, 759, 805, 775, 743, 764, 764, 778, 772, 787, 751, 730, 764, 800, 801, 731, 731, 778, 730, 792, 722, 775, 771, 746, 749, 779, 778, 776, 717, 813, 760, 791, 735, 799, 765, 771] payload_rows_per_dst=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] dup_rows_per_dst=[3068, 3020, 3014, 2984, 2914, 2942, 2964, 3052] combine_rows_per_src=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=343888 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698576 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053264 weight_movement_bytes=34603008 total_nvlink_bytes=567656272 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64848 dup_rows=24054 dup_rows_remote=20880 dedup_ratio=0.244690 router_tokens_per_expert=[781, 730, 780, 744, 782, 774, 725, 735, 764, 771, 792, 790, 763, 755, 791, 780, 759, 758, 768, 772, 781, 778, 768, 791, 753, 777, 736, 755, 746, 717, 757, 765, 749, 825, 726, 772, 771, 803, 814, 756, 757, 738, 785, 777, 788, 753, 788, 782, 754, 759, 731, 736, 767, 783, 790, 795, 740, 781, 746, 776, 784, 806, 799, 783] payload_rows_per_dst=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] dup_rows_per_dst=[2920, 2968, 2564, 3190, 3394, 2878, 2966, 3174] combine_rows_per_src=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343136 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731168 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119200 weight_movement_bytes=34603008 total_nvlink_bytes=565722208 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342832 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266410800 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532478768 weight_movement_bytes=103809024 total_nvlink_bytes=636287792 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=65028 dup_rows=24104 dup_rows_remote=21146 dedup_ratio=0.245199 router_tokens_per_expert=[776, 760, 750, 775, 786, 778, 729, 785, 764, 771, 780, 777, 806, 779, 735, 756, 815, 741, 851, 755, 840, 854, 784, 795, 769, 803, 753, 710, 758, 759, 742, 771, 795, 722, 752, 740, 710, 754, 725, 732, 753, 784, 819, 691, 744, 814, 773, 725, 760, 784, 800, 739, 763, 782, 742, 788, 725, 761, 758, 751, 755, 765, 834, 805] payload_rows_per_dst=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] dup_rows_per_dst=[2972, 2946, 3322, 2958, 2860, 2970, 3000, 3076] combine_rows_per_src=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65136 dup_rows=23906 dup_rows_remote=21076 dedup_ratio=0.243184 router_tokens_per_expert=[769, 806, 792, 816, 772, 769, 795, 714, 832, 754, 742, 764, 744, 733, 754, 723, 761, 748, 814, 772, 791, 786, 806, 738, 762, 769, 807, 754, 795, 790, 790, 796, 719, 797, 754, 763, 776, 737, 693, 778, 791, 777, 761, 749, 779, 734, 755, 758, 767, 787, 729, 815, 777, 779, 767, 733, 773, 775, 750, 727, 772, 790, 750, 782] payload_rows_per_dst=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] dup_rows_per_dst=[3026, 2830, 3008, 3222, 2806, 2962, 3042, 3010] combine_rows_per_src=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344200 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723464 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102728 weight_movement_bytes=69206016 total_nvlink_bytes=602308744 weight_to_token_ratio=0.129817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=342912 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960320 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531577728 weight_movement_bytes=34603008 total_nvlink_bytes=566180736 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344696 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699384 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533054072 weight_movement_bytes=34603008 total_nvlink_bytes=567657080 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65190 dup_rows=23824 dup_rows_remote=20890 dedup_ratio=0.242350 router_tokens_per_expert=[783, 780, 748, 779, 787, 785, 765, 763, 758, 799, 756, 755, 795, 795, 724, 772, 799, 809, 734, 793, 759, 750, 797, 732, 735, 773, 769, 758, 745, 752, 716, 734, 751, 771, 747, 738, 776, 797, 780, 798, 808, 770, 763, 817, 767, 749, 777, 747, 775, 713, 778, 789, 760, 744, 752, 801, 789, 810, 729, 774, 777, 782, 815, 709] payload_rows_per_dst=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] dup_rows_per_dst=[3072, 3020, 3030, 2720, 3044, 2986, 2934, 3018] combine_rows_per_src=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344848 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141904 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938960 weight_movement_bytes=69206016 total_nvlink_bytes=603144976 weight_to_token_ratio=0.129614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65026 dup_rows=23992 dup_rows_remote=21032 dedup_ratio=0.244059 router_tokens_per_expert=[756, 801, 772, 771, 719, 804, 753, 779, 732, 769, 801, 732, 784, 771, 764, 700, 754, 801, 747, 722, 724, 767, 769, 778, 793, 777, 832, 791, 713, 789, 745, 749, 746, 805, 751, 751, 754, 730, 784, 802, 790, 741, 737, 803, 793, 773, 768, 762, 755, 807, 762, 765, 820, 807, 754, 783, 787, 784, 764, 735, 771, 733, 803, 773] payload_rows_per_dst=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] dup_rows_per_dst=[2972, 2944, 2916, 3120, 2960, 2990, 3064, 3026] combine_rows_per_src=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65088 dup_rows=24034 dup_rows_remote=21118 dedup_ratio=0.244486 router_tokens_per_expert=[813, 817, 768, 806, 772, 757, 773, 749, 732, 793, 757, 787, 727, 771, 777, 787, 764, 694, 787, 723, 764, 761, 784, 772, 761, 733, 755, 742, 776, 753, 755, 782, 764, 794, 732, 822, 710, 756, 776, 781, 770, 775, 747, 779, 786, 806, 758, 768, 744, 785, 792, 788, 747, 815, 847, 745, 761, 811, 764, 714, 801, 745, 730, 747] payload_rows_per_dst=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] dup_rows_per_dst=[2994, 3036, 2916, 2920, 2990, 3108, 3162, 2908] combine_rows_per_src=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65134 dup_rows=23866 dup_rows_remote=20876 dedup_ratio=0.242778 router_tokens_per_expert=[785, 733, 749, 772, 770, 776, 806, 756, 764, 749, 804, 762, 756, 749, 773, 770, 741, 799, 775, 769, 774, 785, 713, 772, 742, 767, 745, 803, 765, 770, 738, 750, 758, 748, 775, 797, 779, 749, 709, 771, 753, 760, 780, 828, 773, 835, 814, 734, 807, 758, 795, 750, 781, 774, 789, 768, 792, 790, 798, 767, 707, 731, 765, 735] payload_rows_per_dst=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] dup_rows_per_dst=[2990, 3008, 2998, 2870, 2910, 3034, 3146, 2910] combine_rows_per_src=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344320 combine_payload_bytes=267018240 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267362560 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380800 weight_movement_bytes=34603008 total_nvlink_bytes=568983808 weight_to_token_ratio=0.064753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344232 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266690728 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037224 weight_movement_bytes=103809024 total_nvlink_bytes=636846248 weight_to_token_ratio=0.194750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65006 dup_rows=23846 dup_rows_remote=20756 dedup_ratio=0.242574 router_tokens_per_expert=[738, 811, 743, 824, 726, 733, 783, 756, 785, 802, 738, 725, 757, 741, 790, 767, 794, 828, 811, 790, 761, 769, 771, 726, 756, 750, 744, 762, 762, 721, 810, 765, 779, 726, 782, 761, 743, 769, 766, 746, 826, 799, 784, 775, 736, 763, 820, 768, 748, 820, 784, 765, 782, 791, 759, 739, 759, 784, 726, 751, 769, 747, 773, 773] payload_rows_per_dst=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] dup_rows_per_dst=[2894, 2954, 3038, 2984, 2908, 3090, 3026, 2952] combine_rows_per_src=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64628 dup_rows=24016 dup_rows_remote=20678 dedup_ratio=0.244303 router_tokens_per_expert=[784, 805, 781, 751, 780, 756, 736, 789, 772, 781, 733, 787, 797, 793, 757, 763, 767, 750, 756, 776, 806, 746, 775, 787, 764, 739, 793, 789, 731, 755, 794, 756, 780, 788, 789, 799, 751, 783, 801, 760, 766, 776, 754, 734, 775, 722, 762, 739, 752, 767, 789, 778, 747, 735, 758, 778, 783, 767, 738, 770, 800, 769, 759, 734] payload_rows_per_dst=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] dup_rows_per_dst=[2974, 3060, 2982, 2990, 3068, 2582, 3022, 3338] combine_rows_per_src=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64960 dup_rows=24086 dup_rows_remote=21102 dedup_ratio=0.245015 router_tokens_per_expert=[786, 732, 753, 766, 763, 745, 808, 751, 766, 793, 754, 758, 777, 760, 794, 754, 781, 720, 726, 761, 783, 742, 750, 747, 745, 730, 751, 769, 804, 811, 748, 763, 831, 761, 801, 776, 727, 803, 732, 768, 786, 753, 766, 783, 759, 781, 820, 785, 790, 756, 763, 779, 783, 744, 767, 788, 823, 785, 786, 717, 769, 721, 746, 812] payload_rows_per_dst=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] dup_rows_per_dst=[3042, 3132, 2844, 2984, 3038, 3038, 3056, 2952] combine_rows_per_src=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343048 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266607624 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532872200 weight_movement_bytes=69206016 total_nvlink_bytes=602078216 weight_to_token_ratio=0.129874 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65046 dup_rows=24122 dup_rows_remote=21084 dedup_ratio=0.245382 router_tokens_per_expert=[761, 785, 745, 743, 781, 826, 763, 742, 782, 756, 783, 732, 709, 785, 788, 752, 756, 770, 781, 748, 797, 790, 756, 800, 799, 749, 760, 740, 778, 788, 750, 785, 777, 751, 769, 776, 757, 756, 754, 802, 829, 752, 773, 772, 758, 771, 763, 777, 730, 771, 779, 789, 734, 769, 740, 791, 782, 766, 754, 765, 732, 745, 784, 804] payload_rows_per_dst=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] dup_rows_per_dst=[3074, 3038, 3016, 3074, 3056, 3030, 2886, 2948] combine_rows_per_src=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264716288 dispatch_meta_bytes=341224 combine_payload_bytes=264716288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265057512 ingress_bytes=299319296 link_bytes=299319296 token_bytes=529773800 weight_movement_bytes=34603008 total_nvlink_bytes=564376808 weight_to_token_ratio=0.065317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126314414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344248 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420408 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496568 weight_movement_bytes=34603008 total_nvlink_bytes=567099576 weight_to_token_ratio=0.064983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64942 dup_rows=24060 dup_rows_remote=20966 dedup_ratio=0.244751 router_tokens_per_expert=[744, 761, 745, 753, 772, 790, 754, 756, 778, 775, 800, 763, 736, 769, 793, 825, 743, 779, 742, 777, 764, 744, 758, 778, 834, 734, 758, 802, 746, 779, 786, 728, 794, 769, 792, 760, 800, 728, 804, 751, 719, 780, 792, 738, 761, 751, 777, 762, 746, 763, 782, 794, 741, 776, 734, 785, 796, 744, 753, 721, 830, 764, 831, 748] payload_rows_per_dst=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] dup_rows_per_dst=[2914, 3030, 2956, 2938, 3094, 3064, 3032, 3032] combine_rows_per_src=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65186 dup_rows=23996 dup_rows_remote=21074 dedup_ratio=0.244100 router_tokens_per_expert=[762, 792, 731, 793, 728, 749, 746, 758, 754, 810, 752, 761, 775, 769, 824, 770, 785, 737, 846, 776, 845, 737, 776, 739, 774, 766, 802, 764, 785, 780, 745, 795, 764, 788, 806, 713, 821, 814, 764, 760, 729, 778, 765, 760, 793, 765, 747, 750, 793, 696, 726, 800, 795, 720, 729, 763, 789, 748, 749, 775, 765, 744, 763, 754] payload_rows_per_dst=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] dup_rows_per_dst=[2900, 3096, 3036, 3040, 3070, 3008, 2922, 2924] combine_rows_per_src=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344520 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772936 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201352 weight_movement_bytes=34603008 total_nvlink_bytes=567804360 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65082 dup_rows=23900 dup_rows_remote=21024 dedup_ratio=0.243123 router_tokens_per_expert=[788, 778, 739, 777, 747, 807, 739, 750, 788, 778, 733, 780, 782, 765, 828, 833, 729, 774, 754, 731, 781, 744, 756, 830, 771, 812, 764, 726, 794, 811, 722, 802, 765, 728, 760, 802, 746, 736, 803, 751, 749, 807, 753, 729, 753, 801, 802, 803, 786, 781, 775, 753, 743, 736, 762, 765, 755, 759, 804, 757, 783, 726, 739, 727] payload_rows_per_dst=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] dup_rows_per_dst=[3022, 3182, 2876, 2976, 2978, 3076, 2918, 2872] combine_rows_per_src=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20892 dedup_ratio=0.243408 router_tokens_per_expert=[806, 782, 757, 740, 759, 761, 804, 744, 790, 754, 774, 721, 764, 798, 791, 746, 726, 746, 762, 798, 752, 739, 765, 810, 737, 784, 757, 784, 749, 760, 779, 792, 761, 784, 751, 790, 779, 746, 822, 752, 796, 770, 771, 776, 802, 736, 739, 839, 776, 770, 718, 776, 764, 818, 795, 820, 762, 711, 753, 737, 707, 759, 788, 753] payload_rows_per_dst=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] dup_rows_per_dst=[3036, 3118, 3028, 2982, 2900, 3138, 2896, 2830] combine_rows_per_src=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65316 dup_rows=23892 dup_rows_remote=20986 dedup_ratio=0.243042 router_tokens_per_expert=[761, 718, 745, 744, 746, 802, 801, 779, 816, 764, 793, 765, 767, 755, 779, 714, 769, 764, 756, 752, 751, 787, 800, 812, 813, 736, 782, 742, 771, 760, 795, 752, 775, 759, 788, 758, 812, 765, 775, 765, 773, 707, 758, 767, 743, 764, 743, 746, 782, 785, 755, 797, 797, 761, 776, 771, 767, 737, 780, 795, 790, 725, 767, 778] payload_rows_per_dst=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] dup_rows_per_dst=[2966, 2964, 3064, 2998, 2934, 2906, 3066, 2994] combine_rows_per_src=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343632 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266346064 ingress_bytes=369811456 link_bytes=369811456 token_bytes=532348496 weight_movement_bytes=103809024 total_nvlink_bytes=636157520 weight_to_token_ratio=0.195002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391570737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64812 dup_rows=23970 dup_rows_remote=20748 dedup_ratio=0.243835 router_tokens_per_expert=[775, 759, 741, 749, 726, 801, 789, 766, 779, 781, 744, 737, 771, 801, 737, 776, 807, 742, 789, 735, 766, 822, 735, 762, 804, 758, 722, 772, 747, 799, 759, 784, 761, 783, 769, 752, 722, 725, 756, 768, 768, 777, 791, 772, 772, 772, 765, 755, 797, 769, 806, 779, 753, 794, 753, 811, 767, 769, 769, 721, 817, 761, 744, 799] payload_rows_per_dst=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] dup_rows_per_dst=[2864, 2748, 2986, 2948, 2890, 3086, 3226, 3222] combine_rows_per_src=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345040 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267346896 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348752 weight_movement_bytes=103809024 total_nvlink_bytes=638157776 weight_to_token_ratio=0.194272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344424 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920296 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496168 weight_movement_bytes=69206016 total_nvlink_bytes=602702184 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65140 dup_rows=23844 dup_rows_remote=21018 dedup_ratio=0.242554 router_tokens_per_expert=[752, 733, 785, 808, 780, 784, 752, 773, 772, 788, 792, 763, 778, 735, 786, 778, 731, 778, 753, 751, 772, 760, 780, 769, 730, 737, 768, 798, 759, 737, 747, 797, 809, 752, 790, 780, 789, 808, 732, 767, 805, 762, 737, 813, 736, 726, 774, 754, 796, 734, 731, 767, 830, 731, 754, 791, 759, 773, 809, 745, 720, 825, 734, 793] payload_rows_per_dst=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] dup_rows_per_dst=[3100, 2974, 2938, 2826, 3052, 3004, 2948, 3002] combine_rows_per_src=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343920 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544816 weight_movement_bytes=69206016 total_nvlink_bytes=602750832 weight_to_token_ratio=0.129710 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345208 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879544 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535413880 weight_movement_bytes=34603008 total_nvlink_bytes=570016888 weight_to_token_ratio=0.064629 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65192 dup_rows=23844 dup_rows_remote=20752 dedup_ratio=0.242554 router_tokens_per_expert=[817, 790, 732, 792, 781, 794, 761, 812, 784, 775, 781, 777, 748, 766, 746, 803, 776, 726, 783, 772, 766, 751, 775, 747, 800, 778, 713, 721, 769, 782, 817, 776, 764, 790, 752, 773, 751, 803, 813, 742, 724, 821, 754, 774, 765, 722, 725, 710, 774, 768, 772, 786, 783, 781, 775, 763, 690, 756, 812, 782, 760, 766, 768, 722] payload_rows_per_dst=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] dup_rows_per_dst=[3090, 3092, 2908, 2934, 3018, 2868, 3078, 2856] combine_rows_per_src=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=342240 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812192 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282144 weight_movement_bytes=34603008 total_nvlink_bytes=565885152 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344632 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158072 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971512 weight_movement_bytes=34603008 total_nvlink_bytes=568574520 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64794 dup_rows=24146 dup_rows_remote=21186 dedup_ratio=0.245626 router_tokens_per_expert=[773, 744, 769, 749, 723, 769, 753, 753, 768, 795, 765, 771, 793, 805, 760, 765, 747, 754, 774, 812, 780, 792, 775, 766, 791, 780, 796, 714, 783, 754, 772, 769, 800, 779, 769, 797, 788, 751, 712, 766, 788, 719, 787, 749, 748, 770, 777, 791, 785, 755, 754, 787, 742, 790, 772, 761, 756, 766, 760, 763, 805, 718, 735, 798] payload_rows_per_dst=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] dup_rows_per_dst=[2952, 3042, 3134, 3118, 2960, 3004, 3030, 2906] combine_rows_per_src=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343776 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370208 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396640 weight_movement_bytes=69206016 total_nvlink_bytes=603602656 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64958 dup_rows=24104 dup_rows_remote=20998 dedup_ratio=0.245199 router_tokens_per_expert=[789, 784, 785, 799, 777, 724, 792, 744, 764, 751, 723, 741, 780, 819, 732, 781, 726, 762, 808, 802, 812, 752, 775, 808, 790, 742, 840, 724, 754, 750, 771, 736, 754, 772, 827, 760, 799, 739, 720, 748, 742, 778, 790, 768, 760, 777, 815, 757, 771, 738, 785, 751, 782, 810, 778, 759, 717, 773, 753, 754, 782, 739, 767, 750] payload_rows_per_dst=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] dup_rows_per_dst=[2988, 2916, 3186, 2964, 2972, 3120, 3106, 2852] combine_rows_per_src=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64970 dup_rows=23948 dup_rows_remote=20838 dedup_ratio=0.243612 router_tokens_per_expert=[809, 764, 721, 756, 752, 749, 727, 766, 768, 792, 766, 778, 751, 794, 770, 745, 791, 784, 794, 785, 782, 767, 758, 787, 805, 727, 750, 828, 774, 756, 831, 704, 780, 804, 765, 763, 762, 774, 771, 759, 814, 758, 762, 763, 756, 765, 751, 782, 786, 759, 736, 761, 700, 749, 749, 802, 741, 786, 797, 743, 781, 778, 720, 804] payload_rows_per_dst=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] dup_rows_per_dst=[2974, 3054, 3110, 2978, 2920, 2976, 2848, 3088] combine_rows_per_src=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65184 dup_rows=23772 dup_rows_remote=20666 dedup_ratio=0.241821 router_tokens_per_expert=[811, 787, 785, 776, 753, 758, 799, 755, 812, 773, 715, 714, 771, 771, 767, 767, 777, 772, 756, 841, 785, 746, 763, 788, 774, 772, 809, 755, 755, 737, 759, 738, 784, 718, 774, 761, 771, 777, 804, 737, 755, 737, 775, 767, 745, 752, 736, 742, 713, 746, 810, 764, 775, 797, 807, 809, 780, 781, 750, 773, 775, 763, 742, 791] payload_rows_per_dst=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] dup_rows_per_dst=[3106, 2992, 3002, 2878, 2908, 3200, 2724, 2962] combine_rows_per_src=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65026 dup_rows=23828 dup_rows_remote=20626 dedup_ratio=0.242391 router_tokens_per_expert=[737, 797, 766, 752, 768, 759, 792, 744, 811, 775, 747, 748, 761, 768, 763, 777, 745, 756, 766, 721, 770, 774, 778, 766, 770, 743, 729, 727, 782, 797, 774, 792, 777, 781, 751, 761, 771, 778, 736, 744, 743, 814, 794, 827, 814, 810, 791, 733, 813, 764, 768, 787, 771, 789, 753, 743, 733, 784, 788, 751, 737, 759, 778, 754] payload_rows_per_dst=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] dup_rows_per_dst=[2978, 3002, 2936, 2910, 2932, 3202, 2982, 2886] combine_rows_per_src=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343920 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265740144 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531136368 weight_movement_bytes=69206016 total_nvlink_bytes=600342384 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64894 dup_rows=23888 dup_rows_remote=20742 dedup_ratio=0.243001 router_tokens_per_expert=[770, 763, 794, 757, 723, 723, 769, 763, 739, 785, 794, 800, 825, 744, 749, 777, 761, 816, 738, 797, 771, 781, 783, 749, 799, 766, 803, 818, 745, 770, 792, 786, 746, 776, 770, 745, 785, 767, 775, 733, 721, 780, 758, 779, 747, 812, 712, 753, 762, 733, 755, 729, 800, 792, 728, 759, 821, 802, 757, 770, 768, 772, 734, 761] payload_rows_per_dst=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] dup_rows_per_dst=[2894, 3022, 2870, 2986, 2864, 2972, 3134, 3146] combine_rows_per_src=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343824 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411792 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479760 weight_movement_bytes=69206016 total_nvlink_bytes=601685776 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343232 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266460352 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577472 weight_movement_bytes=138412032 total_nvlink_bytes=670989504 weight_to_token_ratio=0.259891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64974 dup_rows=24078 dup_rows_remote=21134 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 775, 728, 762, 785, 794, 782, 779, 763, 753, 745, 752, 779, 729, 758, 803, 755, 792, 721, 760, 782, 794, 762, 722, 765, 794, 773, 760, 755, 783, 746, 781, 787, 719, 779, 796, 795, 758, 774, 736, 750, 766, 753, 823, 761, 772, 756, 744, 796, 736, 810, 749, 786, 789, 750, 769, 777, 812, 769, 779, 743, 776, 748] payload_rows_per_dst=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] dup_rows_per_dst=[3028, 2922, 3110, 2944, 3088, 3044, 2952, 2990] combine_rows_per_src=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=343400 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267337064 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534330728 weight_movement_bytes=69206016 total_nvlink_bytes=603536744 weight_to_token_ratio=0.129519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=342608 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689104 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533035600 weight_movement_bytes=34603008 total_nvlink_bytes=567638608 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64940 dup_rows=24112 dup_rows_remote=21072 dedup_ratio=0.245280 router_tokens_per_expert=[767, 735, 770, 792, 762, 758, 784, 746, 770, 777, 752, 776, 789, 749, 742, 791, 807, 762, 715, 763, 840, 799, 814, 784, 757, 739, 759, 747, 772, 745, 764, 819, 777, 755, 762, 749, 773, 733, 773, 728, 742, 731, 797, 769, 804, 774, 746, 725, 750, 742, 771, 797, 793, 826, 748, 787, 792, 761, 729, 718, 797, 777, 809, 771] payload_rows_per_dst=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] dup_rows_per_dst=[3014, 3040, 3146, 2980, 2932, 2966, 3026, 3008] combine_rows_per_src=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=342544 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148368 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954192 weight_movement_bytes=34603008 total_nvlink_bytes=566557200 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344432 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266477936 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532611440 weight_movement_bytes=69206016 total_nvlink_bytes=601817456 weight_to_token_ratio=0.129937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344048 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338288 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332528 weight_movement_bytes=69206016 total_nvlink_bytes=601538544 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64944 dup_rows=23994 dup_rows_remote=21006 dedup_ratio=0.244080 router_tokens_per_expert=[740, 795, 752, 745, 777, 749, 790, 750, 755, 785, 771, 735, 707, 800, 837, 784, 748, 760, 771, 753, 762, 756, 802, 747, 729, 757, 736, 776, 789, 798, 756, 796, 784, 765, 813, 739, 751, 765, 797, 763, 768, 773, 799, 762, 759, 774, 774, 845, 756, 763, 753, 796, 737, 769, 792, 783, 767, 785, 721, 764, 770, 773, 747, 737] payload_rows_per_dst=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] dup_rows_per_dst=[2952, 3026, 2988, 2982, 2988, 3082, 3040, 2936] combine_rows_per_src=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65112 dup_rows=23982 dup_rows_remote=21064 dedup_ratio=0.243958 router_tokens_per_expert=[760, 764, 797, 760, 750, 783, 754, 802, 773, 730, 764, 773, 744, 746, 776, 773, 803, 766, 786, 756, 749, 769, 798, 748, 768, 806, 787, 761, 754, 800, 756, 758, 725, 761, 796, 824, 761, 812, 809, 779, 776, 760, 750, 775, 744, 774, 767, 797, 775, 725, 748, 787, 743, 731, 777, 778, 753, 777, 741, 736, 767, 771, 725, 794] payload_rows_per_dst=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] dup_rows_per_dst=[3020, 2910, 3016, 3002, 3118, 3080, 2918, 2918] combine_rows_per_src=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64842 dup_rows=24054 dup_rows_remote=21008 dedup_ratio=0.244690 router_tokens_per_expert=[713, 743, 774, 783, 785, 764, 729, 757, 715, 748, 761, 794, 788, 776, 782, 757, 774, 734, 762, 763, 794, 809, 768, 823, 787, 761, 778, 782, 786, 795, 754, 830, 812, 757, 765, 761, 803, 782, 787, 725, 756, 732, 815, 789, 777, 757, 750, 767, 758, 768, 787, 770, 742, 793, 762, 716, 768, 764, 770, 753, 754, 720, 772, 751] payload_rows_per_dst=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] dup_rows_per_dst=[2944, 3006, 3046, 3122, 2990, 2998, 3026, 2922] combine_rows_per_src=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64796 dup_rows=23968 dup_rows_remote=20656 dedup_ratio=0.243815 router_tokens_per_expert=[761, 770, 757, 714, 748, 814, 774, 791, 801, 726, 776, 754, 775, 734, 767, 799, 807, 782, 789, 801, 769, 786, 764, 780, 732, 812, 759, 792, 811, 772, 726, 743, 778, 749, 765, 759, 793, 758, 793, 764, 750, 794, 760, 774, 749, 785, 751, 736, 787, 766, 768, 737, 781, 773, 773, 731, 764, 759, 765, 761, 749, 744, 796, 754] payload_rows_per_dst=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] dup_rows_per_dst=[3312, 2974, 2802, 2958, 2976, 2914, 3082, 2950] combine_rows_per_src=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64874 dup_rows=24134 dup_rows_remote=21188 dedup_ratio=0.245504 router_tokens_per_expert=[772, 796, 785, 775, 764, 777, 728, 761, 792, 729, 760, 749, 795, 785, 790, 749, 758, 806, 766, 772, 820, 777, 760, 836, 703, 766, 766, 738, 809, 736, 708, 802, 773, 776, 781, 786, 781, 769, 798, 770, 778, 801, 707, 785, 797, 767, 749, 737, 751, 773, 728, 731, 734, 772, 725, 761, 797, 767, 814, 747, 763, 808, 749, 747] payload_rows_per_dst=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] dup_rows_per_dst=[3020, 3014, 3126, 2896, 3214, 2946, 2834, 3084] combine_rows_per_src=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343800 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354424 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365048 weight_movement_bytes=69206016 total_nvlink_bytes=601571064 weight_to_token_ratio=0.129997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344704 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043456 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742208 weight_movement_bytes=34603008 total_nvlink_bytes=568345216 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64796 dup_rows=23986 dup_rows_remote=20626 dedup_ratio=0.243998 router_tokens_per_expert=[728, 766, 771, 772, 776, 771, 788, 765, 798, 785, 758, 760, 795, 778, 761, 749, 791, 796, 825, 735, 765, 765, 794, 765, 804, 787, 774, 731, 749, 742, 723, 752, 759, 773, 729, 748, 726, 776, 827, 800, 788, 765, 722, 782, 785, 777, 697, 799, 750, 793, 736, 721, 775, 799, 747, 746, 793, 800, 783, 766, 763, 766, 753, 789] payload_rows_per_dst=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] dup_rows_per_dst=[2960, 3044, 2826, 2886, 3088, 2740, 3082, 3360] combine_rows_per_src=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343400 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265936232 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531529064 weight_movement_bytes=103809024 total_nvlink_bytes=635338088 weight_to_token_ratio=0.195303 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65030 dup_rows=23982 dup_rows_remote=21082 dedup_ratio=0.243958 router_tokens_per_expert=[772, 720, 738, 753, 770, 788, 761, 752, 833, 741, 751, 762, 754, 792, 792, 815, 788, 802, 744, 802, 726, 770, 756, 763, 780, 741, 789, 771, 753, 720, 757, 785, 721, 724, 772, 710, 815, 703, 780, 742, 719, 762, 830, 800, 770, 785, 773, 753, 772, 799, 754, 757, 770, 805, 782, 773, 776, 773, 763, 804, 752, 744, 854, 774] payload_rows_per_dst=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] dup_rows_per_dst=[2926, 3042, 2968, 2900, 2892, 3050, 3084, 3120] combine_rows_per_src=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64934 dup_rows=24038 dup_rows_remote=20998 dedup_ratio=0.244527 router_tokens_per_expert=[734, 741, 743, 745, 715, 804, 755, 764, 780, 745, 796, 751, 816, 781, 757, 760, 759, 781, 745, 787, 756, 793, 767, 777, 800, 793, 776, 728, 726, 726, 828, 769, 732, 780, 765, 774, 822, 783, 750, 759, 797, 729, 764, 762, 725, 790, 789, 784, 762, 781, 789, 820, 786, 737, 729, 745, 789, 777, 768, 752, 748, 818, 771, 777] payload_rows_per_dst=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] dup_rows_per_dst=[2870, 3040, 3036, 3012, 3004, 2944, 3072, 3060] combine_rows_per_src=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341808 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746224 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150640 weight_movement_bytes=34603008 total_nvlink_bytes=565753648 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344248 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266068152 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531792056 weight_movement_bytes=69206016 total_nvlink_bytes=600998072 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341688 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746104 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150520 weight_movement_bytes=34603008 total_nvlink_bytes=565753528 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344448 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266707328 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533070208 weight_movement_bytes=69206016 total_nvlink_bytes=602276224 weight_to_token_ratio=0.129825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343728 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313392 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283056 weight_movement_bytes=69206016 total_nvlink_bytes=601489072 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64898 dup_rows=24082 dup_rows_remote=20930 dedup_ratio=0.244975 router_tokens_per_expert=[760, 752, 722, 765, 757, 776, 810, 773, 739, 761, 806, 803, 807, 802, 794, 730, 773, 803, 758, 783, 743, 778, 759, 745, 732, 706, 760, 750, 753, 773, 755, 753, 740, 807, 845, 812, 779, 762, 753, 740, 790, 755, 719, 742, 753, 730, 774, 789, 736, 731, 786, 815, 809, 791, 820, 740, 785, 751, 772, 803, 778, 747, 753, 764] payload_rows_per_dst=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] dup_rows_per_dst=[3012, 3076, 3050, 2838, 3152, 2890, 3086, 2978] combine_rows_per_src=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64968 dup_rows=24024 dup_rows_remote=20996 dedup_ratio=0.244385 router_tokens_per_expert=[750, 762, 799, 763, 699, 802, 778, 751, 764, 757, 805, 790, 750, 759, 767, 752, 730, 753, 736, 811, 770, 788, 757, 754, 813, 806, 761, 780, 760, 803, 749, 704, 787, 776, 761, 786, 784, 773, 784, 790, 767, 780, 769, 775, 794, 765, 764, 786, 724, 770, 815, 858, 727, 756, 775, 745, 769, 734, 803, 781, 760, 743, 722, 706] payload_rows_per_dst=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] dup_rows_per_dst=[2952, 3010, 2962, 3060, 3156, 3022, 3028, 2834] combine_rows_per_src=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65104 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[756, 769, 711, 766, 782, 775, 764, 769, 796, 766, 791, 749, 804, 756, 758, 746, 759, 790, 742, 746, 785, 740, 740, 782, 761, 765, 758, 777, 819, 766, 823, 749, 759, 702, 802, 768, 794, 823, 743, 761, 688, 826, 784, 803, 742, 758, 817, 788, 708, 767, 788, 787, 744, 764, 760, 775, 720, 781, 759, 737, 810, 767, 786, 781] payload_rows_per_dst=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] dup_rows_per_dst=[2836, 3032, 2988, 3122, 2942, 3058, 3000, 3042] combine_rows_per_src=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64728 dup_rows=23930 dup_rows_remote=20472 dedup_ratio=0.243429 router_tokens_per_expert=[793, 777, 769, 761, 761, 797, 739, 768, 782, 759, 771, 787, 816, 783, 749, 756, 808, 769, 794, 732, 762, 711, 788, 748, 797, 793, 746, 754, 820, 779, 808, 775, 756, 744, 799, 741, 781, 781, 765, 755, 809, 744, 808, 737, 801, 785, 744, 794, 772, 786, 730, 741, 768, 798, 754, 762, 714, 730, 760, 748, 719, 786, 785, 703] payload_rows_per_dst=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] dup_rows_per_dst=[3458, 2820, 2922, 3122, 2850, 3044, 2938, 2776] combine_rows_per_src=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65174 dup_rows=23884 dup_rows_remote=21016 dedup_ratio=0.242961 router_tokens_per_expert=[776, 746, 763, 718, 806, 798, 796, 748, 739, 841, 750, 741, 802, 811, 781, 791, 725, 751, 721, 784, 785, 773, 770, 777, 771, 731, 775, 740, 764, 797, 795, 746, 799, 797, 791, 770, 796, 751, 756, 789, 782, 773, 717, 774, 731, 765, 762, 753, 801, 801, 750, 772, 745, 784, 765, 804, 736, 764, 764, 758, 745, 753, 716, 776] payload_rows_per_dst=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] dup_rows_per_dst=[2980, 3096, 2916, 2928, 3080, 2868, 3122, 2894] combine_rows_per_src=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20892 dedup_ratio=0.244059 router_tokens_per_expert=[786, 758, 770, 781, 776, 771, 762, 762, 745, 784, 787, 737, 786, 817, 766, 715, 730, 760, 827, 796, 761, 768, 742, 749, 792, 762, 795, 813, 777, 782, 759, 731, 737, 790, 743, 773, 742, 730, 763, 797, 780, 768, 745, 747, 726, 810, 788, 754, 758, 793, 745, 772, 789, 794, 813, 719, 805, 753, 821, 717, 732, 765, 764, 772] payload_rows_per_dst=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] dup_rows_per_dst=[3034, 2456, 3206, 3152, 2864, 3274, 2906, 3100] combine_rows_per_src=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343312 combine_payload_bytes=265822208 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266165520 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987728 weight_movement_bytes=34603008 total_nvlink_bytes=566590736 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343856 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452784 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561712 weight_movement_bytes=34603008 total_nvlink_bytes=567164720 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344544 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010528 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676512 weight_movement_bytes=34603008 total_nvlink_bytes=568279520 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64902 dup_rows=24060 dup_rows_remote=20982 dedup_ratio=0.244751 router_tokens_per_expert=[825, 783, 751, 721, 777, 782, 776, 771, 767, 780, 801, 779, 776, 789, 765, 761, 799, 741, 729, 722, 765, 756, 763, 739, 773, 790, 794, 756, 763, 799, 783, 752, 740, 754, 772, 765, 799, 746, 776, 762, 768, 763, 793, 744, 756, 800, 733, 790, 760, 758, 727, 775, 763, 789, 764, 768, 759, 739, 758, 798, 773, 804, 753, 775] payload_rows_per_dst=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] dup_rows_per_dst=[3108, 3018, 2884, 3078, 2978, 3046, 2930, 3018] combine_rows_per_src=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64818 dup_rows=24018 dup_rows_remote=20874 dedup_ratio=0.244324 router_tokens_per_expert=[753, 755, 797, 728, 759, 728, 759, 791, 768, 774, 770, 799, 794, 857, 788, 756, 758, 761, 749, 751, 758, 777, 769, 739, 755, 746, 816, 830, 780, 771, 789, 761, 788, 741, 776, 752, 740, 788, 767, 765, 753, 759, 781, 750, 773, 714, 771, 769, 766, 767, 771, 768, 771, 769, 728, 786, 757, 767, 734, 789, 783, 766, 792, 765] payload_rows_per_dst=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] dup_rows_per_dst=[2930, 3144, 2948, 3108, 2934, 2976, 2982, 2996] combine_rows_per_src=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=340800 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265466688 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530592576 weight_movement_bytes=34603008 total_nvlink_bytes=565195584 weight_to_token_ratio=0.065216 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344760 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297464 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250168 weight_movement_bytes=34603008 total_nvlink_bytes=568853176 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343144 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148968 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954792 weight_movement_bytes=34603008 total_nvlink_bytes=566557800 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343536 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182128 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532020720 weight_movement_bytes=34603008 total_nvlink_bytes=566623728 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342768 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265837296 ingress_bytes=438509568 link_bytes=438509568 token_bytes=531331824 weight_movement_bytes=173015040 total_nvlink_bytes=704346864 weight_to_token_ratio=0.325625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650076201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64874 dup_rows=24082 dup_rows_remote=21154 dedup_ratio=0.244975 router_tokens_per_expert=[762, 776, 736, 757, 762, 787, 764, 784, 742, 775, 787, 781, 756, 738, 747, 767, 795, 769, 760, 743, 740, 731, 767, 795, 757, 806, 821, 734, 767, 737, 725, 776, 772, 817, 774, 781, 739, 771, 722, 762, 748, 853, 740, 786, 748, 796, 797, 755, 774, 744, 783, 800, 792, 765, 782, 773, 760, 765, 747, 758, 775, 756, 801, 772] payload_rows_per_dst=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] dup_rows_per_dst=[3016, 2966, 2914, 3036, 2928, 3046, 3156, 3020] combine_rows_per_src=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65164 dup_rows=23910 dup_rows_remote=20916 dedup_ratio=0.243225 router_tokens_per_expert=[751, 731, 794, 753, 758, 748, 761, 758, 783, 731, 791, 807, 760, 775, 797, 740, 756, 777, 777, 777, 749, 747, 754, 775, 786, 722, 825, 694, 714, 820, 706, 753, 770, 772, 745, 752, 797, 761, 771, 754, 745, 806, 759, 744, 818, 775, 772, 785, 773, 773, 781, 786, 762, 800, 806, 774, 757, 805, 750, 822, 736, 785, 759, 787] payload_rows_per_dst=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] dup_rows_per_dst=[2974, 2902, 2994, 2904, 2948, 3030, 3060, 3098] combine_rows_per_src=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64768 dup_rows=24242 dup_rows_remote=21172 dedup_ratio=0.246602 router_tokens_per_expert=[784, 764, 811, 777, 718, 814, 751, 695, 792, 778, 835, 753, 784, 786, 773, 789, 756, 730, 758, 736, 787, 759, 770, 777, 768, 748, 773, 730, 766, 765, 703, 804, 779, 769, 789, 732, 721, 811, 771, 770, 786, 764, 793, 751, 767, 749, 786, 721, 769, 751, 800, 799, 761, 795, 771, 736, 794, 761, 768, 765, 763, 776, 798, 752] payload_rows_per_dst=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] dup_rows_per_dst=[2950, 3130, 2922, 2974, 3148, 2946, 3070, 3102] combine_rows_per_src=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64826 dup_rows=24100 dup_rows_remote=20962 dedup_ratio=0.245158 router_tokens_per_expert=[738, 779, 774, 741, 797, 769, 748, 775, 775, 737, 795, 719, 740, 794, 795, 786, 784, 806, 795, 753, 792, 768, 770, 815, 756, 726, 739, 770, 780, 758, 740, 739, 786, 783, 717, 719, 813, 747, 784, 735, 768, 797, 757, 773, 827, 779, 770, 763, 779, 754, 752, 779, 780, 773, 771, 795, 762, 789, 757, 788, 773, 732, 759, 738] payload_rows_per_dst=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] dup_rows_per_dst=[3138, 2802, 3140, 2948, 2958, 3198, 2988, 2928] combine_rows_per_src=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344112 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266068016 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531791920 weight_movement_bytes=34603008 total_nvlink_bytes=566394928 weight_to_token_ratio=0.065069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65244 dup_rows=23716 dup_rows_remote=20650 dedup_ratio=0.241252 router_tokens_per_expert=[787, 744, 747, 746, 762, 774, 721, 774, 755, 789, 774, 750, 765, 795, 748, 767, 732, 730, 727, 776, 747, 763, 840, 738, 771, 736, 780, 822, 791, 777, 799, 762, 814, 822, 746, 743, 778, 745, 743, 764, 773, 775, 786, 736, 800, 749, 763, 768, 759, 800, 775, 767, 792, 799, 753, 780, 774, 769, 745, 762, 811, 755, 719, 798] payload_rows_per_dst=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] dup_rows_per_dst=[2880, 2964, 2896, 3002, 2988, 2886, 3034, 3066] combine_rows_per_src=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65002 dup_rows=23986 dup_rows_remote=20976 dedup_ratio=0.243998 router_tokens_per_expert=[761, 722, 775, 783, 795, 748, 757, 761, 783, 823, 799, 755, 738, 761, 791, 782, 748, 826, 786, 760, 752, 725, 821, 765, 758, 760, 778, 777, 753, 754, 760, 786, 821, 737, 728, 735, 763, 746, 728, 778, 764, 751, 791, 761, 759, 806, 767, 764, 772, 736, 761, 785, 752, 810, 771, 834, 723, 783, 757, 784, 802, 759, 737, 744] payload_rows_per_dst=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] dup_rows_per_dst=[2920, 3016, 3158, 3044, 2862, 3010, 3054, 2922] combine_rows_per_src=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344320 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256064 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167808 weight_movement_bytes=34603008 total_nvlink_bytes=568770816 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343760 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265633488 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530923216 weight_movement_bytes=103809024 total_nvlink_bytes=634732240 weight_to_token_ratio=0.195525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65256 dup_rows=23858 dup_rows_remote=20994 dedup_ratio=0.242696 router_tokens_per_expert=[762, 738, 747, 770, 779, 766, 708, 762, 790, 799, 750, 714, 844, 765, 774, 769, 748, 761, 789, 757, 729, 782, 779, 777, 729, 773, 758, 739, 752, 737, 748, 791, 781, 782, 750, 740, 778, 728, 743, 773, 780, 808, 783, 794, 733, 793, 791, 782, 778, 768, 815, 709, 770, 743, 755, 751, 810, 789, 753, 830, 730, 836, 838, 752] payload_rows_per_dst=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] dup_rows_per_dst=[2886, 3020, 3010, 2864, 2974, 3058, 2920, 3126] combine_rows_per_src=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21080 dedup_ratio=0.244324 router_tokens_per_expert=[803, 765, 770, 741, 780, 811, 787, 728, 779, 791, 752, 762, 741, 773, 750, 754, 745, 787, 739, 732, 752, 769, 811, 767, 799, 767, 803, 752, 751, 753, 715, 817, 720, 784, 752, 806, 775, 749, 784, 786, 780, 729, 775, 820, 751, 745, 797, 784, 775, 747, 781, 768, 744, 799, 741, 777, 817, 725, 736, 774, 772, 763, 750, 800] payload_rows_per_dst=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] dup_rows_per_dst=[2980, 2938, 2932, 2998, 3090, 3042, 3032, 3006] combine_rows_per_src=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343152 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265870448 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531397744 weight_movement_bytes=34603008 total_nvlink_bytes=566000752 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=343576 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267583000 ingress_bytes=336445440 link_bytes=336445440 token_bytes=534822424 weight_movement_bytes=69206016 total_nvlink_bytes=604028440 weight_to_token_ratio=0.129400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266017105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343912 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592104 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532840296 weight_movement_bytes=103809024 total_nvlink_bytes=636649320 weight_to_token_ratio=0.194822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345000 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267633576 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534922152 weight_movement_bytes=34603008 total_nvlink_bytes=569525160 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344400 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666320 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988240 weight_movement_bytes=69206016 total_nvlink_bytes=602194256 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64920 dup_rows=24166 dup_rows_remote=21022 dedup_ratio=0.245829 router_tokens_per_expert=[742, 788, 762, 771, 780, 762, 821, 744, 739, 730, 750, 798, 797, 777, 778, 795, 755, 748, 771, 805, 781, 723, 729, 744, 780, 735, 758, 749, 767, 779, 741, 799, 761, 764, 769, 777, 803, 757, 786, 764, 787, 754, 742, 758, 807, 773, 733, 767, 818, 735, 772, 736, 773, 771, 789, 797, 765, 785, 744, 799, 776, 752, 772, 768] payload_rows_per_dst=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] dup_rows_per_dst=[3012, 3128, 2854, 3002, 3144, 2920, 3026, 3080] combine_rows_per_src=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64998 dup_rows=23966 dup_rows_remote=20984 dedup_ratio=0.243795 router_tokens_per_expert=[737, 768, 778, 751, 775, 796, 768, 769, 720, 791, 796, 773, 728, 790, 801, 775, 780, 748, 782, 749, 770, 761, 812, 759, 783, 742, 722, 840, 768, 795, 794, 763, 775, 756, 786, 719, 731, 773, 804, 738, 770, 738, 719, 782, 798, 788, 744, 763, 808, 758, 791, 781, 754, 758, 740, 795, 766, 758, 746, 757, 768, 766, 785, 753] payload_rows_per_dst=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] dup_rows_per_dst=[2980, 3106, 2982, 3026, 2924, 2904, 2982, 3062] combine_rows_per_src=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20900 dedup_ratio=0.243510 router_tokens_per_expert=[751, 751, 798, 773, 767, 746, 743, 765, 816, 816, 743, 793, 756, 743, 775, 769, 764, 781, 761, 737, 766, 742, 740, 807, 774, 758, 740, 784, 755, 768, 742, 783, 790, 801, 782, 720, 773, 770, 754, 770, 750, 758, 748, 767, 727, 806, 799, 727, 790, 782, 806, 808, 766, 745, 740, 805, 751, 777, 751, 788, 742, 781, 770, 801] payload_rows_per_dst=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] dup_rows_per_dst=[2978, 3078, 2936, 2954, 3002, 2944, 3038, 3008] combine_rows_per_src=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73710 payload_rows_remote=64134 dup_rows=24594 dup_rows_remote=21166 dedup_ratio=0.250183 router_tokens_per_expert=[806, 814, 756, 799, 789, 785, 775, 793, 748, 762, 772, 779, 770, 728, 757, 746, 761, 805, 734, 757, 744, 771, 757, 794, 778, 708, 751, 763, 751, 748, 776, 801, 763, 716, 744, 800, 786, 745, 737, 803, 787, 762, 757, 790, 756, 830, 761, 746, 813, 756, 755, 820, 774, 798, 708, 764, 736, 735, 790, 782, 773, 781, 769, 737] payload_rows_per_dst=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] dup_rows_per_dst=[3428, 2958, 3414, 3314, 3198, 2780, 2686, 2816] combine_rows_per_src=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343768 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256088 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168408 weight_movement_bytes=69206016 total_nvlink_bytes=601374424 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343928 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575736 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807544 weight_movement_bytes=34603008 total_nvlink_bytes=567410552 weight_to_token_ratio=0.064945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64788 dup_rows=24206 dup_rows_remote=21214 dedup_ratio=0.246236 router_tokens_per_expert=[768, 732, 758, 752, 742, 765, 735, 772, 724, 758, 748, 807, 741, 741, 791, 765, 801, 800, 781, 763, 762, 766, 743, 750, 774, 789, 767, 770, 815, 722, 738, 782, 744, 777, 792, 768, 742, 780, 719, 824, 795, 774, 799, 794, 793, 791, 767, 801, 770, 779, 801, 743, 766, 772, 764, 731, 737, 814, 810, 746, 768, 745, 776, 748] payload_rows_per_dst=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] dup_rows_per_dst=[2962, 2924, 3052, 3006, 2948, 3258, 3064, 2992] combine_rows_per_src=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65140 dup_rows=23790 dup_rows_remote=20666 dedup_ratio=0.242004 router_tokens_per_expert=[762, 773, 751, 752, 767, 732, 763, 742, 739, 788, 764, 788, 765, 769, 767, 773, 788, 785, 789, 736, 751, 736, 747, 801, 760, 774, 781, 805, 767, 760, 741, 786, 719, 815, 741, 717, 813, 742, 744, 759, 779, 801, 798, 715, 817, 793, 749, 797, 717, 761, 784, 774, 779, 787, 727, 780, 778, 783, 744, 774, 746, 792, 830, 795] payload_rows_per_dst=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] dup_rows_per_dst=[2924, 3028, 2944, 2986, 2856, 3124, 2888, 3040] combine_rows_per_src=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343280 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255600 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167920 weight_movement_bytes=103809024 total_nvlink_bytes=635976944 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64894 dup_rows=24046 dup_rows_remote=21002 dedup_ratio=0.244609 router_tokens_per_expert=[784, 791, 772, 765, 773, 727, 754, 742, 790, 793, 745, 759, 790, 812, 755, 772, 744, 804, 776, 766, 750, 755, 790, 799, 783, 765, 774, 791, 780, 763, 787, 761, 781, 724, 776, 799, 797, 772, 765, 731, 759, 755, 730, 750, 744, 755, 777, 761, 759, 763, 762, 760, 817, 822, 785, 746, 813, 803, 728, 712, 742, 730, 767, 755] payload_rows_per_dst=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] dup_rows_per_dst=[2976, 3042, 3068, 3044, 3006, 2924, 3088, 2898] combine_rows_per_src=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64990 dup_rows=23882 dup_rows_remote=20744 dedup_ratio=0.242940 router_tokens_per_expert=[735, 810, 689, 785, 761, 761, 773, 772, 792, 808, 815, 762, 810, 786, 732, 780, 828, 789, 757, 781, 754, 776, 744, 755, 802, 763, 739, 838, 736, 740, 739, 706, 819, 798, 729, 726, 797, 789, 777, 764, 804, 767, 800, 755, 755, 737, 776, 736, 752, 762, 756, 728, 745, 752, 753, 733, 805, 763, 769, 807, 768, 767, 733, 812] payload_rows_per_dst=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] dup_rows_per_dst=[2924, 3138, 3030, 2874, 2984, 3038, 2858, 3036] combine_rows_per_src=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=262692864 dispatch_meta_bytes=341200 combine_payload_bytes=262692864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263034064 ingress_bytes=297295872 link_bytes=297295872 token_bytes=525726928 weight_movement_bytes=34603008 total_nvlink_bytes=560329936 weight_to_token_ratio=0.065819 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118700432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344008 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265715656 ingress_bytes=334577664 link_bytes=334577664 token_bytes=531087304 weight_movement_bytes=69206016 total_nvlink_bytes=600293320 weight_to_token_ratio=0.130310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258988814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343224 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267156664 ingress_bytes=336019456 link_bytes=336019456 token_bytes=533970104 weight_movement_bytes=69206016 total_nvlink_bytes=603176120 weight_to_token_ratio=0.129607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264414161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343584 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149408 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955232 weight_movement_bytes=34603008 total_nvlink_bytes=566558240 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342936 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266541976 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532741016 weight_movement_bytes=138412032 total_nvlink_bytes=671153048 weight_to_token_ratio=0.259811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65080 dup_rows=23954 dup_rows_remote=20902 dedup_ratio=0.243673 router_tokens_per_expert=[764, 784, 724, 773, 793, 791, 746, 735, 799, 805, 780, 781, 779, 706, 777, 789, 752, 805, 740, 766, 778, 762, 806, 752, 743, 825, 758, 762, 796, 799, 740, 739, 789, 736, 778, 784, 764, 757, 787, 731, 773, 764, 768, 811, 750, 749, 752, 786, 796, 795, 757, 803, 744, 754, 776, 762, 746, 727, 731, 743, 736, 769, 808, 777] payload_rows_per_dst=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] dup_rows_per_dst=[3004, 3088, 3052, 3014, 2900, 3074, 3016, 2806] combine_rows_per_src=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64922 dup_rows=24224 dup_rows_remote=21164 dedup_ratio=0.246419 router_tokens_per_expert=[775, 785, 807, 778, 787, 819, 752, 753, 768, 771, 778, 778, 779, 767, 783, 725, 748, 785, 794, 791, 768, 723, 761, 774, 760, 772, 724, 775, 784, 785, 792, 742, 761, 786, 763, 767, 748, 757, 756, 771, 795, 728, 803, 781, 773, 804, 751, 768, 758, 758, 741, 752, 759, 777, 756, 772, 777, 702, 760, 804, 777, 737, 740, 787] payload_rows_per_dst=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] dup_rows_per_dst=[3198, 3114, 3050, 2980, 3060, 2982, 2882, 2958] combine_rows_per_src=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65024 dup_rows=24054 dup_rows_remote=20984 dedup_ratio=0.244690 router_tokens_per_expert=[750, 772, 785, 754, 755, 785, 772, 750, 720, 723, 762, 738, 776, 782, 758, 819, 734, 729, 760, 798, 762, 836, 737, 778, 757, 741, 759, 756, 760, 776, 760, 767, 787, 800, 785, 749, 819, 767, 769, 783, 775, 797, 770, 713, 792, 753, 792, 799, 806, 752, 717, 775, 804, 797, 726, 771, 713, 783, 726, 816, 762, 780, 783, 780] payload_rows_per_dst=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] dup_rows_per_dst=[2940, 2894, 2950, 2990, 3118, 3056, 3070, 3036] combine_rows_per_src=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=343928 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266911608 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479288 weight_movement_bytes=34603008 total_nvlink_bytes=568082296 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64732 dup_rows=23966 dup_rows_remote=20658 dedup_ratio=0.243795 router_tokens_per_expert=[794, 805, 753, 801, 789, 806, 780, 767, 765, 757, 745, 764, 747, 801, 745, 716, 768, 764, 776, 765, 750, 751, 770, 769, 692, 766, 839, 787, 797, 736, 765, 780, 753, 768, 758, 772, 750, 820, 749, 766, 755, 759, 710, 827, 763, 786, 734, 792, 742, 750, 779, 796, 747, 780, 761, 761, 764, 798, 746, 744, 787, 766, 773, 786] payload_rows_per_dst=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] dup_rows_per_dst=[3308, 2900, 2890, 3038, 2938, 2826, 3082, 2984] combine_rows_per_src=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344344 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266264856 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532185368 weight_movement_bytes=34603008 total_nvlink_bytes=566788376 weight_to_token_ratio=0.065021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64654 dup_rows=24284 dup_rows_remote=21076 dedup_ratio=0.247030 router_tokens_per_expert=[786, 842, 747, 786, 772, 829, 750, 792, 762, 764, 776, 821, 763, 718, 744, 753, 747, 779, 789, 776, 742, 794, 810, 762, 784, 780, 772, 732, 762, 760, 740, 806, 722, 756, 743, 801, 759, 748, 742, 744, 791, 747, 749, 821, 757, 820, 783, 746, 748, 771, 795, 726, 805, 722, 775, 761, 758, 774, 756, 763, 740, 753, 743, 793] payload_rows_per_dst=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] dup_rows_per_dst=[2916, 2938, 3078, 2786, 2930, 3198, 3230, 3208] combine_rows_per_src=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64884 dup_rows=24144 dup_rows_remote=21100 dedup_ratio=0.245605 router_tokens_per_expert=[801, 773, 785, 755, 764, 782, 772, 764, 791, 772, 763, 810, 800, 742, 771, 726, 785, 736, 810, 773, 763, 812, 770, 750, 759, 792, 748, 811, 740, 775, 785, 752, 802, 751, 755, 748, 826, 728, 735, 818, 759, 792, 759, 796, 781, 744, 754, 775, 756, 768, 735, 782, 745, 758, 775, 752, 785, 743, 786, 731, 782, 722, 776, 701] payload_rows_per_dst=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] dup_rows_per_dst=[3074, 3042, 3076, 2944, 3100, 3044, 2918, 2946] combine_rows_per_src=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65024 dup_rows=24042 dup_rows_remote=21056 dedup_ratio=0.244568 router_tokens_per_expert=[737, 754, 767, 764, 759, 768, 762, 780, 788, 716, 735, 776, 758, 797, 722, 759, 757, 770, 696, 758, 805, 788, 805, 751, 761, 747, 781, 747, 789, 755, 771, 761, 759, 752, 757, 750, 769, 802, 708, 793, 742, 788, 743, 819, 782, 759, 775, 788, 807, 783, 769, 753, 793, 764, 758, 737, 774, 785, 805, 814, 766, 784, 793, 797] payload_rows_per_dst=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] dup_rows_per_dst=[2960, 2940, 2908, 2986, 2996, 3032, 3040, 3180] combine_rows_per_src=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65152 dup_rows=24036 dup_rows_remote=21176 dedup_ratio=0.244507 router_tokens_per_expert=[719, 783, 733, 788, 781, 821, 735, 701, 741, 709, 794, 720, 781, 734, 765, 744, 761, 752, 786, 759, 772, 754, 816, 792, 753, 754, 786, 782, 824, 769, 759, 788, 793, 771, 821, 807, 768, 775, 798, 781, 737, 727, 704, 837, 811, 762, 783, 779, 751, 735, 756, 764, 749, 742, 780, 823, 784, 789, 699, 745, 779, 735, 806, 805] payload_rows_per_dst=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] dup_rows_per_dst=[2872, 2860, 3158, 3092, 3146, 2936, 3018, 2954] combine_rows_per_src=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344032 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682336 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020640 weight_movement_bytes=34603008 total_nvlink_bytes=567623648 weight_to_token_ratio=0.064919 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342920 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165704 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988488 weight_movement_bytes=34603008 total_nvlink_bytes=564591496 weight_to_token_ratio=0.065290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341560 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483832 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626104 weight_movement_bytes=34603008 total_nvlink_bytes=565229112 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=343936 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108800 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873664 weight_movement_bytes=69206016 total_nvlink_bytes=601079680 weight_to_token_ratio=0.130117 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266682624 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020928 weight_movement_bytes=69206016 total_nvlink_bytes=602226944 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266862592 dispatch_meta_bytes=345312 combine_payload_bytes=266862592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267207904 ingress_bytes=301465600 link_bytes=301465600 token_bytes=534070496 weight_movement_bytes=34603008 total_nvlink_bytes=568673504 weight_to_token_ratio=0.064791 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134390783e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65038 dup_rows=23832 dup_rows_remote=20744 dedup_ratio=0.242432 router_tokens_per_expert=[810, 719, 769, 808, 757, 712, 758, 796, 744, 736, 792, 774, 764, 756, 748, 747, 773, 785, 796, 775, 802, 762, 834, 734, 797, 769, 756, 801, 781, 759, 776, 762, 807, 807, 735, 757, 750, 786, 763, 766, 846, 755, 727, 760, 753, 747, 743, 749, 718, 785, 777, 807, 774, 742, 749, 773, 740, 756, 749, 854, 757, 788, 751, 729] payload_rows_per_dst=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] dup_rows_per_dst=[2966, 2884, 3088, 2934, 3006, 2932, 3024, 2998] combine_rows_per_src=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64986 dup_rows=24134 dup_rows_remote=21096 dedup_ratio=0.245504 router_tokens_per_expert=[770, 752, 749, 788, 768, 724, 780, 781, 741, 770, 732, 777, 713, 748, 776, 797, 791, 747, 784, 813, 765, 839, 797, 726, 739, 767, 749, 750, 771, 739, 800, 787, 738, 777, 770, 743, 768, 768, 755, 792, 818, 805, 787, 815, 791, 783, 733, 748, 776, 774, 749, 735, 793, 753, 750, 716, 778, 751, 808, 754, 777, 790, 776, 751] payload_rows_per_dst=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] dup_rows_per_dst=[3004, 2954, 3030, 2948, 3038, 3172, 2856, 3132] combine_rows_per_src=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65250 dup_rows=23770 dup_rows_remote=20804 dedup_ratio=0.241801 router_tokens_per_expert=[771, 787, 769, 752, 786, 790, 774, 795, 774, 766, 757, 750, 844, 762, 813, 746, 767, 774, 732, 740, 746, 758, 774, 746, 793, 796, 730, 785, 761, 849, 762, 779, 768, 763, 766, 787, 741, 759, 754, 754, 790, 766, 737, 763, 751, 780, 738, 751, 747, 784, 766, 736, 770, 759, 785, 778, 744, 759, 775, 756, 835, 783, 749, 730] payload_rows_per_dst=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] dup_rows_per_dst=[3080, 3040, 2936, 3038, 2882, 2926, 2966, 2902] combine_rows_per_src=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343128 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738776 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134424 weight_movement_bytes=69206016 total_nvlink_bytes=602340440 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64846 dup_rows=24142 dup_rows_remote=21084 dedup_ratio=0.245585 router_tokens_per_expert=[739, 800, 792, 768, 747, 767, 760, 787, 775, 734, 798, 778, 752, 768, 786, 758, 800, 806, 761, 787, 754, 755, 776, 761, 765, 734, 735, 722, 784, 729, 788, 785, 787, 784, 718, 759, 749, 774, 795, 764, 792, 764, 768, 774, 738, 750, 771, 743, 794, 814, 796, 783, 766, 763, 776, 826, 755, 764, 751, 754, 764, 764, 726, 775] payload_rows_per_dst=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] dup_rows_per_dst=[3074, 2978, 3086, 2764, 3032, 2998, 3152, 3058] combine_rows_per_src=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344328 combine_payload_bytes=266182656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266526984 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709640 weight_movement_bytes=103809024 total_nvlink_bytes=636518664 weight_to_token_ratio=0.194870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65046 dup_rows=23948 dup_rows_remote=20852 dedup_ratio=0.243612 router_tokens_per_expert=[780, 788, 803, 753, 735, 780, 779, 758, 757, 725, 764, 801, 783, 730, 698, 802, 725, 741, 755, 785, 794, 755, 714, 845, 770, 727, 777, 767, 816, 807, 802, 794, 804, 807, 742, 785, 756, 751, 774, 738, 756, 769, 766, 780, 753, 799, 810, 753, 719, 760, 790, 742, 761, 749, 770, 781, 791, 781, 719, 769, 788, 742, 801, 736] payload_rows_per_dst=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] dup_rows_per_dst=[3096, 2998, 2976, 3134, 3020, 2866, 2842, 3016] combine_rows_per_src=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=65534 dup_rows=23612 dup_rows_remote=20880 dedup_ratio=0.240194 router_tokens_per_expert=[804, 749, 783, 783, 764, 751, 785, 748, 742, 785, 762, 825, 772, 745, 799, 788, 787, 755, 773, 794, 767, 778, 748, 748, 774, 750, 722, 791, 776, 761, 770, 781, 770, 798, 776, 741, 776, 723, 744, 801, 718, 778, 724, 746, 760, 738, 719, 762, 745, 828, 759, 745, 785, 775, 817, 791, 742, 786, 747, 785, 780, 798, 754, 781] payload_rows_per_dst=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] dup_rows_per_dst=[2972, 3022, 2924, 2944, 3072, 2732, 3002, 2944] combine_rows_per_src=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344216 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608216 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872216 weight_movement_bytes=34603008 total_nvlink_bytes=569475224 weight_to_token_ratio=0.064694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65174 dup_rows=23818 dup_rows_remote=20854 dedup_ratio=0.242289 router_tokens_per_expert=[761, 733, 795, 738, 784, 733, 772, 841, 756, 756, 761, 784, 750, 761, 721, 778, 789, 767, 793, 754, 748, 740, 813, 773, 760, 766, 782, 752, 755, 775, 815, 733, 786, 747, 772, 812, 725, 825, 739, 786, 766, 788, 747, 751, 760, 748, 755, 774, 779, 793, 812, 780, 793, 756, 766, 757, 761, 735, 738, 774, 802, 748, 748, 790] payload_rows_per_dst=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] dup_rows_per_dst=[3044, 2832, 2986, 2964, 3032, 2962, 3058, 2940] combine_rows_per_src=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65038 dup_rows=24098 dup_rows_remote=21204 dedup_ratio=0.245138 router_tokens_per_expert=[786, 763, 815, 819, 797, 737, 747, 791, 743, 753, 767, 731, 757, 800, 756, 724, 796, 771, 727, 810, 721, 751, 775, 752, 772, 801, 797, 770, 757, 800, 750, 800, 834, 722, 776, 755, 750, 767, 764, 778, 729, 733, 747, 767, 757, 774, 785, 767, 761, 796, 802, 765, 804, 784, 785, 731, 766, 715, 770, 748, 785, 763, 780, 756] payload_rows_per_dst=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] dup_rows_per_dst=[3070, 2894, 3044, 3116, 2984, 2844, 3120, 3026] combine_rows_per_src=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343720 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952936 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531562152 weight_movement_bytes=34603008 total_nvlink_bytes=566165160 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343592 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266772008 ingress_bytes=370237440 link_bytes=370237440 token_bytes=533200424 weight_movement_bytes=103809024 total_nvlink_bytes=637009448 weight_to_token_ratio=0.194690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393173680e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=268427264 dispatch_meta_bytes=345656 combine_payload_bytes=268427264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268772920 ingress_bytes=303030272 link_bytes=303030272 token_bytes=537200184 weight_movement_bytes=34603008 total_nvlink_bytes=571803192 weight_to_token_ratio=0.064414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140278518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65164 dup_rows=23730 dup_rows_remote=20822 dedup_ratio=0.241394 router_tokens_per_expert=[800, 799, 766, 751, 780, 742, 728, 798, 771, 794, 757, 767, 723, 751, 773, 727, 777, 757, 759, 730, 744, 779, 796, 817, 792, 799, 799, 747, 746, 745, 769, 735, 754, 760, 722, 772, 771, 785, 710, 829, 756, 749, 831, 740, 776, 765, 818, 787, 772, 775, 802, 729, 787, 763, 787, 783, 780, 793, 731, 768, 696, 775, 770, 798] payload_rows_per_dst=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] dup_rows_per_dst=[2960, 2864, 2908, 2930, 2974, 3062, 3006, 3026] combine_rows_per_src=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267296816 ingress_bytes=370761728 link_bytes=370761728 token_bytes=534249520 weight_movement_bytes=103809024 total_nvlink_bytes=638058544 weight_to_token_ratio=0.194308 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395146534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344968 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740616 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533136264 weight_movement_bytes=34603008 total_nvlink_bytes=567739272 weight_to_token_ratio=0.064905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65412 dup_rows=23726 dup_rows_remote=20744 dedup_ratio=0.241353 router_tokens_per_expert=[807, 816, 746, 739, 752, 751, 806, 776, 750, 784, 808, 815, 776, 799, 778, 761, 764, 737, 743, 828, 790, 720, 817, 752, 804, 759, 785, 751, 803, 752, 732, 764, 741, 759, 735, 775, 768, 752, 762, 782, 701, 768, 788, 752, 817, 775, 750, 782, 751, 783, 754, 745, 749, 821, 754, 754, 725, 747, 771, 745, 786, 769, 786, 740] payload_rows_per_dst=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] dup_rows_per_dst=[3070, 3084, 2986, 2896, 2982, 2924, 2920, 2864] combine_rows_per_src=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65104 dup_rows=23882 dup_rows_remote=20976 dedup_ratio=0.242940 router_tokens_per_expert=[743, 721, 772, 773, 732, 789, 738, 779, 787, 783, 771, 757, 787, 806, 785, 804, 769, 748, 751, 767, 811, 777, 761, 720, 781, 808, 782, 783, 803, 774, 755, 774, 736, 771, 747, 796, 735, 795, 755, 733, 771, 780, 763, 741, 760, 767, 772, 783, 779, 765, 752, 764, 780, 769, 721, 782, 757, 789, 748, 759, 747, 776, 817, 751] payload_rows_per_dst=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] dup_rows_per_dst=[2948, 3108, 2998, 3066, 2886, 2914, 2906, 3056] combine_rows_per_src=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343944 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255688 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167432 weight_movement_bytes=69206016 total_nvlink_bytes=603373448 weight_to_token_ratio=0.129559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=64602 dup_rows=24356 dup_rows_remote=21034 dedup_ratio=0.247762 router_tokens_per_expert=[735, 745, 782, 811, 762, 820, 794, 805, 784, 734, 744, 733, 806, 771, 754, 704, 797, 768, 770, 782, 725, 765, 728, 763, 771, 810, 740, 715, 753, 726, 762, 741, 755, 747, 767, 783, 777, 721, 811, 765, 717, 791, 785, 860, 767, 782, 776, 767, 762, 778, 753, 848, 737, 768, 781, 769, 775, 757, 739, 771, 774, 786, 773, 810] payload_rows_per_dst=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] dup_rows_per_dst=[2924, 2964, 2842, 3128, 2980, 3126, 3070, 3322] combine_rows_per_src=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267927552 dispatch_meta_bytes=344624 combine_payload_bytes=267927552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268272176 ingress_bytes=302530560 link_bytes=302530560 token_bytes=536199728 weight_movement_bytes=34603008 total_nvlink_bytes=570802736 weight_to_token_ratio=0.064534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138398142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65180 dup_rows=23730 dup_rows_remote=20874 dedup_ratio=0.241394 router_tokens_per_expert=[783, 759, 749, 733, 802, 739, 780, 778, 738, 772, 779, 760, 774, 759, 759, 784, 778, 705, 780, 739, 739, 780, 787, 746, 772, 747, 746, 800, 761, 820, 753, 778, 789, 768, 795, 776, 767, 773, 758, 814, 795, 709, 766, 769, 765, 813, 812, 724, 794, 757, 771, 782, 753, 737, 741, 785, 788, 764, 754, 747, 783, 785, 786, 753] payload_rows_per_dst=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] dup_rows_per_dst=[2856, 3096, 2774, 2866, 3028, 3342, 2742, 3026] combine_rows_per_src=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64998 dup_rows=24014 dup_rows_remote=20916 dedup_ratio=0.244283 router_tokens_per_expert=[756, 797, 732, 769, 747, 807, 796, 800, 744, 792, 768, 739, 749, 741, 772, 751, 745, 758, 753, 796, 742, 771, 788, 763, 776, 750, 770, 766, 760, 749, 804, 814, 770, 738, 763, 748, 747, 800, 737, 818, 781, 772, 825, 734, 804, 768, 775, 736, 818, 786, 731, 754, 780, 771, 796, 801, 764, 727, 792, 711, 795, 733, 764, 748] payload_rows_per_dst=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] dup_rows_per_dst=[3040, 2922, 2928, 3054, 3010, 3098, 3008, 2954] combine_rows_per_src=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64854 dup_rows=24092 dup_rows_remote=21208 dedup_ratio=0.245076 router_tokens_per_expert=[748, 747, 743, 783, 783, 747, 798, 737, 706, 768, 778, 794, 827, 777, 740, 731, 779, 781, 758, 809, 746, 813, 771, 790, 736, 781, 756, 790, 717, 770, 761, 738, 773, 798, 773, 757, 767, 773, 749, 769, 774, 781, 746, 819, 739, 769, 729, 762, 795, 791, 741, 772, 771, 761, 735, 793, 720, 834, 722, 821, 764, 751, 784, 816] payload_rows_per_dst=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] dup_rows_per_dst=[3004, 2884, 3208, 2868, 3024, 3078, 3016, 3010] combine_rows_per_src=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344320 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010304 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676288 weight_movement_bytes=69206016 total_nvlink_bytes=602882304 weight_to_token_ratio=0.129678 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65096 dup_rows=23918 dup_rows_remote=20994 dedup_ratio=0.243306 router_tokens_per_expert=[797, 792, 770, 803, 760, 795, 767, 766, 765, 770, 767, 739, 799, 776, 727, 747, 725, 744, 766, 795, 729, 732, 803, 759, 735, 793, 766, 779, 766, 765, 766, 737, 799, 794, 778, 796, 784, 757, 837, 752, 717, 756, 781, 754, 716, 724, 788, 751, 786, 763, 742, 759, 788, 765, 755, 772, 728, 821, 806, 799, 769, 780, 762, 773] payload_rows_per_dst=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] dup_rows_per_dst=[3050, 2978, 2946, 2924, 3142, 2754, 2990, 3134] combine_rows_per_src=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=342544 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264952336 ingress_bytes=299212800 link_bytes=299212800 token_bytes=529562128 weight_movement_bytes=34603008 total_nvlink_bytes=564165136 weight_to_token_ratio=0.065343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125913678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344216 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321496 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298776 weight_movement_bytes=34603008 total_nvlink_bytes=568901784 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343656 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266575464 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807272 weight_movement_bytes=69206016 total_nvlink_bytes=602013288 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344248 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986232 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628216 weight_movement_bytes=34603008 total_nvlink_bytes=566231224 weight_to_token_ratio=0.065089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65130 dup_rows=23988 dup_rows_remote=21058 dedup_ratio=0.244019 router_tokens_per_expert=[785, 753, 761, 776, 759, 783, 752, 789, 740, 825, 795, 820, 780, 761, 781, 794, 751, 674, 782, 803, 746, 768, 774, 760, 805, 741, 745, 722, 733, 801, 769, 776, 789, 781, 784, 775, 741, 782, 776, 809, 777, 781, 746, 777, 737, 760, 750, 794, 743, 747, 784, 770, 767, 732, 736, 752, 778, 782, 788, 773, 765, 796, 722, 754] payload_rows_per_dst=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] dup_rows_per_dst=[2980, 3132, 2930, 3030, 3134, 2890, 2876, 3016] combine_rows_per_src=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64940 dup_rows=24116 dup_rows_remote=21120 dedup_ratio=0.245321 router_tokens_per_expert=[735, 733, 756, 772, 817, 773, 779, 773, 750, 759, 781, 778, 792, 765, 737, 748, 741, 734, 787, 764, 791, 767, 771, 793, 723, 822, 780, 778, 732, 793, 797, 825, 797, 754, 754, 713, 732, 762, 813, 797, 729, 780, 794, 749, 767, 816, 748, 717, 768, 774, 786, 797, 715, 763, 741, 805, 789, 712, 772, 746, 755, 748, 823, 790] payload_rows_per_dst=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] dup_rows_per_dst=[3004, 2876, 3032, 3168, 2996, 3062, 3024, 2954] combine_rows_per_src=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65200 dup_rows=23918 dup_rows_remote=20896 dedup_ratio=0.243306 router_tokens_per_expert=[739, 759, 747, 776, 752, 830, 764, 794, 794, 772, 761, 772, 760, 795, 758, 748, 773, 803, 801, 766, 757, 782, 750, 802, 767, 747, 778, 736, 747, 782, 808, 764, 773, 747, 778, 726, 787, 777, 751, 771, 771, 807, 755, 745, 760, 765, 799, 782, 767, 762, 754, 777, 777, 770, 729, 768, 762, 773, 745, 752, 761, 741, 747, 789] payload_rows_per_dst=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] dup_rows_per_dst=[2996, 3038, 3050, 2948, 2882, 3086, 3022, 2896] combine_rows_per_src=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64896 dup_rows=24026 dup_rows_remote=21030 dedup_ratio=0.244405 router_tokens_per_expert=[792, 802, 740, 778, 748, 776, 762, 730, 740, 806, 763, 787, 787, 768, 747, 792, 774, 785, 779, 798, 768, 781, 758, 753, 751, 719, 725, 773, 727, 776, 746, 820, 740, 811, 761, 759, 757, 726, 838, 740, 781, 767, 751, 765, 762, 762, 757, 745, 776, 725, 809, 747, 833, 790, 738, 774, 763, 794, 764, 731, 801, 737, 793, 804] payload_rows_per_dst=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] dup_rows_per_dst=[2954, 2874, 3168, 2958, 2994, 3054, 3028, 2996] combine_rows_per_src=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344752 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117232 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889712 weight_movement_bytes=34603008 total_nvlink_bytes=568492720 weight_to_token_ratio=0.064813 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344240 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338480 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332720 weight_movement_bytes=103809024 total_nvlink_bytes=636141744 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64996 dup_rows=24010 dup_rows_remote=21078 dedup_ratio=0.244242 router_tokens_per_expert=[783, 786, 757, 747, 765, 735, 778, 764, 785, 760, 762, 775, 758, 770, 785, 774, 752, 770, 760, 810, 766, 760, 770, 811, 769, 764, 779, 774, 723, 836, 738, 781, 789, 744, 773, 687, 792, 747, 808, 737, 754, 802, 777, 780, 768, 758, 824, 795, 725, 757, 776, 773, 767, 776, 738, 774, 757, 781, 730, 722, 788, 755, 790, 761] payload_rows_per_dst=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] dup_rows_per_dst=[2932, 3222, 3054, 2888, 3094, 2974, 2862, 2984] combine_rows_per_src=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74022 payload_rows_remote=64720 dup_rows=24282 dup_rows_remote=21260 dedup_ratio=0.247009 router_tokens_per_expert=[805, 734, 768, 750, 770, 751, 759, 810, 752, 793, 761, 794, 766, 728, 743, 789, 790, 773, 784, 778, 777, 754, 807, 733, 717, 783, 747, 717, 749, 800, 768, 768, 775, 726, 798, 797, 768, 762, 810, 771, 774, 796, 770, 746, 730, 800, 791, 755, 746, 787, 755, 748, 731, 756, 756, 804, 769, 748, 743, 727, 810, 822, 786, 777] payload_rows_per_dst=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] dup_rows_per_dst=[2978, 3008, 3128, 2940, 3168, 3022, 2986, 3052] combine_rows_per_src=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344384 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267403584 ingress_bytes=336265216 link_bytes=336265216 token_bytes=534462784 weight_movement_bytes=69206016 total_nvlink_bytes=603668800 weight_to_token_ratio=0.129487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343704 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157720 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971736 weight_movement_bytes=103809024 total_nvlink_bytes=635780760 weight_to_token_ratio=0.195140 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65414 dup_rows=23754 dup_rows_remote=20830 dedup_ratio=0.241638 router_tokens_per_expert=[739, 783, 773, 779, 766, 788, 720, 790, 784, 732, 767, 718, 777, 741, 752, 759, 763, 747, 804, 821, 757, 722, 797, 728, 765, 794, 753, 778, 724, 774, 757, 765, 751, 736, 799, 724, 769, 788, 764, 784, 767, 768, 747, 793, 763, 803, 777, 824, 791, 776, 730, 755, 818, 761, 788, 768, 817, 742, 742, 782, 743, 741, 822, 802] payload_rows_per_dst=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] dup_rows_per_dst=[2984, 2924, 2850, 2942, 2958, 3088, 2994, 3014] combine_rows_per_src=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65040 dup_rows=23876 dup_rows_remote=20880 dedup_ratio=0.242879 router_tokens_per_expert=[795, 756, 748, 763, 770, 798, 758, 781, 711, 783, 756, 742, 772, 778, 793, 743, 787, 771, 794, 731, 792, 762, 715, 791, 767, 767, 785, 854, 753, 714, 788, 764, 731, 796, 807, 820, 802, 791, 733, 764, 743, 791, 764, 762, 734, 793, 725, 774, 770, 794, 811, 767, 753, 793, 715, 758, 754, 780, 785, 751, 754, 756, 758, 741] payload_rows_per_dst=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] dup_rows_per_dst=[3074, 2886, 2986, 2996, 3088, 2938, 3012, 2896] combine_rows_per_src=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344296 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266567912 ingress_bytes=404635648 link_bytes=404635648 token_bytes=532791528 weight_movement_bytes=138412032 total_nvlink_bytes=671203560 weight_to_token_ratio=0.259786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522611368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343920 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437040 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530160 weight_movement_bytes=34603008 total_nvlink_bytes=565133168 weight_to_token_ratio=0.065223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64926 dup_rows=23982 dup_rows_remote=20916 dedup_ratio=0.243958 router_tokens_per_expert=[747, 785, 766, 778, 754, 812, 802, 768, 755, 773, 773, 773, 788, 778, 763, 762, 764, 785, 800, 803, 777, 776, 713, 813, 773, 727, 710, 740, 766, 813, 787, 786, 765, 751, 751, 748, 780, 747, 750, 778, 735, 770, 826, 755, 738, 753, 749, 707, 706, 788, 813, 755, 785, 785, 790, 781, 759, 773, 804, 752, 797, 736, 740, 775] payload_rows_per_dst=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] dup_rows_per_dst=[3004, 3024, 3066, 2938, 3000, 2904, 3052, 2994] combine_rows_per_src=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65032 dup_rows=24000 dup_rows_remote=21046 dedup_ratio=0.244141 router_tokens_per_expert=[785, 790, 734, 783, 748, 777, 728, 755, 754, 801, 762, 752, 773, 810, 729, 740, 768, 789, 817, 783, 748, 789, 776, 812, 772, 745, 724, 747, 817, 780, 758, 825, 777, 789, 759, 778, 766, 712, 771, 761, 738, 763, 763, 806, 763, 837, 799, 731, 728, 778, 774, 761, 795, 739, 769, 728, 809, 762, 793, 726, 755, 748, 764, 739] payload_rows_per_dst=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] dup_rows_per_dst=[2914, 2948, 3082, 3050, 2954, 3100, 2920, 3032] combine_rows_per_src=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=344976 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268280720 ingress_bytes=371744768 link_bytes=371744768 token_bytes=536216464 weight_movement_bytes=103809024 total_nvlink_bytes=640025488 weight_to_token_ratio=0.193595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398845634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343680 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747520 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151360 weight_movement_bytes=34603008 total_nvlink_bytes=567754368 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65016 dup_rows=24036 dup_rows_remote=21112 dedup_ratio=0.244507 router_tokens_per_expert=[730, 735, 768, 760, 762, 746, 763, 726, 741, 754, 795, 782, 800, 779, 785, 761, 751, 800, 714, 780, 739, 758, 748, 768, 766, 803, 752, 773, 745, 780, 751, 790, 767, 762, 749, 786, 751, 801, 783, 773, 770, 784, 755, 750, 801, 775, 790, 760, 778, 729, 791, 773, 760, 752, 755, 750, 813, 807, 734, 787, 771, 782, 782, 826] payload_rows_per_dst=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] dup_rows_per_dst=[2874, 3080, 2914, 3028, 3022, 2990, 2924, 3204] combine_rows_per_src=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343368 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280264 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217160 weight_movement_bytes=69206016 total_nvlink_bytes=601423176 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64704 dup_rows=24212 dup_rows_remote=21142 dedup_ratio=0.246297 router_tokens_per_expert=[803, 758, 762, 753, 767, 755, 749, 793, 766, 774, 778, 762, 704, 781, 768, 821, 780, 762, 853, 735, 719, 802, 802, 784, 767, 831, 755, 765, 751, 746, 785, 760, 755, 749, 773, 719, 737, 735, 750, 761, 719, 718, 794, 768, 795, 741, 798, 757, 807, 789, 696, 782, 792, 745, 748, 805, 769, 763, 785, 764, 781, 782, 801, 783] payload_rows_per_dst=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] dup_rows_per_dst=[3042, 3066, 2704, 3094, 3234, 3038, 2964, 3070] combine_rows_per_src=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344312 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715384 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086456 weight_movement_bytes=34603008 total_nvlink_bytes=567689464 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64728 dup_rows=24088 dup_rows_remote=21008 dedup_ratio=0.245036 router_tokens_per_expert=[784, 768, 768, 769, 758, 774, 758, 768, 816, 745, 750, 780, 783, 789, 790, 806, 755, 764, 755, 770, 775, 760, 769, 745, 727, 768, 747, 749, 724, 789, 742, 746, 811, 782, 788, 765, 804, 790, 778, 814, 805, 725, 717, 717, 780, 823, 798, 777, 762, 777, 784, 733, 825, 723, 770, 757, 767, 769, 728, 788, 753, 726, 771, 754] payload_rows_per_dst=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] dup_rows_per_dst=[3080, 3088, 2886, 3204, 2814, 3026, 3162, 2828] combine_rows_per_src=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65110 dup_rows=23836 dup_rows_remote=20810 dedup_ratio=0.242472 router_tokens_per_expert=[766, 769, 760, 799, 730, 747, 770, 727, 754, 768, 765, 779, 738, 727, 741, 752, 788, 773, 735, 758, 767, 736, 789, 721, 780, 779, 755, 764, 755, 777, 739, 772, 764, 786, 790, 781, 792, 771, 761, 735, 830, 779, 775, 789, 738, 771, 789, 721, 722, 793, 771, 818, 797, 801, 808, 757, 757, 787, 793, 825, 792, 761, 763, 755] payload_rows_per_dst=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] dup_rows_per_dst=[2898, 2780, 2896, 2962, 3016, 3026, 3168, 3090] combine_rows_per_src=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344512 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266650048 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955584 weight_movement_bytes=34603008 total_nvlink_bytes=567558592 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343384 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265370968 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398552 weight_movement_bytes=69206016 total_nvlink_bytes=599604568 weight_to_token_ratio=0.130479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65096 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[786, 775, 764, 754, 785, 741, 789, 774, 779, 729, 719, 791, 795, 781, 745, 768, 764, 755, 770, 744, 708, 807, 779, 803, 750, 785, 780, 832, 769, 791, 780, 779, 741, 787, 728, 769, 737, 752, 783, 763, 789, 738, 802, 734, 778, 758, 782, 834, 756, 805, 750, 790, 774, 760, 755, 785, 777, 758, 724, 758, 780, 780, 717, 737] payload_rows_per_dst=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] dup_rows_per_dst=[3046, 2914, 3028, 3096, 2926, 3034, 2966, 2898] combine_rows_per_src=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65070 dup_rows=23794 dup_rows_remote=20888 dedup_ratio=0.242045 router_tokens_per_expert=[770, 757, 731, 781, 754, 785, 774, 760, 785, 740, 784, 762, 751, 786, 756, 749, 776, 782, 774, 774, 717, 756, 765, 763, 768, 769, 788, 810, 742, 759, 779, 758, 766, 750, 759, 782, 772, 750, 784, 729, 788, 756, 801, 740, 762, 822, 766, 774, 772, 799, 766, 770, 758, 786, 778, 777, 776, 777, 782, 765, 743, 764, 753, 780] payload_rows_per_dst=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] dup_rows_per_dst=[2932, 2944, 2976, 2906, 3024, 3018, 3034, 2960] combine_rows_per_src=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342944 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468832 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594720 weight_movement_bytes=34603008 total_nvlink_bytes=565197728 weight_to_token_ratio=0.065216 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343680 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267034240 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724800 weight_movement_bytes=34603008 total_nvlink_bytes=568327808 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65104 dup_rows=23784 dup_rows_remote=20716 dedup_ratio=0.241943 router_tokens_per_expert=[800, 731, 774, 837, 798, 763, 792, 736, 740, 733, 753, 738, 718, 775, 786, 742, 767, 772, 757, 743, 805, 769, 825, 804, 713, 779, 789, 786, 769, 808, 752, 764, 797, 815, 744, 745, 771, 778, 764, 723, 797, 717, 775, 775, 796, 721, 747, 791, 795, 778, 777, 778, 749, 746, 740, 791, 768, 769, 730, 777, 768, 783, 779, 750] payload_rows_per_dst=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] dup_rows_per_dst=[3022, 2764, 3068, 3076, 2976, 2966, 2942, 2970] combine_rows_per_src=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20734 dedup_ratio=0.241821 router_tokens_per_expert=[788, 748, 751, 783, 735, 759, 791, 771, 727, 764, 766, 778, 782, 750, 765, 832, 777, 733, 796, 792, 793, 787, 778, 742, 749, 748, 748, 802, 762, 761, 759, 821, 790, 768, 774, 785, 778, 754, 782, 772, 772, 762, 721, 732, 776, 758, 838, 738, 760, 784, 766, 750, 791, 773, 770, 772, 776, 729, 754, 724, 778, 757, 772, 758] payload_rows_per_dst=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] dup_rows_per_dst=[2934, 2934, 3014, 3002, 3038, 2884, 3090, 2876] combine_rows_per_src=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343832 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870552 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397272 weight_movement_bytes=69206016 total_nvlink_bytes=602603288 weight_to_token_ratio=0.129746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65050 dup_rows=23864 dup_rows_remote=20758 dedup_ratio=0.242757 router_tokens_per_expert=[771, 790, 753, 817, 747, 736, 723, 789, 767, 756, 813, 800, 780, 771, 792, 779, 785, 754, 786, 756, 824, 768, 766, 843, 753, 768, 801, 773, 709, 791, 721, 751, 765, 759, 753, 741, 733, 757, 744, 750, 792, 725, 784, 800, 770, 751, 721, 744, 797, 755, 773, 796, 811, 725, 786, 805, 739, 781, 722, 771, 763, 760, 780, 766] payload_rows_per_dst=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] dup_rows_per_dst=[3062, 3036, 3116, 2938, 2784, 2954, 3106, 2868] combine_rows_per_src=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=343280 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267009264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533675248 weight_movement_bytes=34603008 total_nvlink_bytes=568278256 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64622 dup_rows=23902 dup_rows_remote=20404 dedup_ratio=0.243144 router_tokens_per_expert=[793, 752, 750, 787, 744, 723, 776, 739, 780, 778, 758, 801, 745, 747, 762, 750, 777, 793, 776, 776, 705, 789, 797, 774, 737, 788, 771, 792, 739, 768, 792, 735, 815, 735, 698, 793, 795, 783, 778, 778, 756, 789, 767, 820, 754, 799, 760, 743, 742, 786, 782, 779, 796, 748, 788, 755, 786, 799, 737, 791, 729, 766, 790, 721] payload_rows_per_dst=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] dup_rows_per_dst=[2868, 3032, 3008, 2942, 2532, 2970, 3052, 3498] combine_rows_per_src=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343592 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255336 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167080 weight_movement_bytes=69206016 total_nvlink_bytes=603373096 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64940 dup_rows=23930 dup_rows_remote=20912 dedup_ratio=0.243429 router_tokens_per_expert=[757, 810, 765, 792, 772, 765, 771, 755, 752, 783, 783, 769, 793, 718, 780, 759, 745, 799, 755, 776, 735, 799, 740, 789, 784, 761, 795, 739, 772, 763, 813, 775, 785, 817, 766, 795, 748, 767, 770, 775, 745, 728, 726, 700, 769, 774, 745, 794, 758, 737, 739, 743, 780, 766, 743, 774, 758, 775, 737, 840, 822, 792, 779, 741] payload_rows_per_dst=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] dup_rows_per_dst=[3018, 3028, 3068, 2958, 3080, 3012, 2728, 3038] combine_rows_per_src=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20780 dedup_ratio=0.242635 router_tokens_per_expert=[744, 738, 757, 775, 776, 734, 804, 785, 772, 728, 758, 747, 792, 722, 777, 761, 740, 781, 793, 761, 726, 773, 773, 765, 751, 768, 802, 734, 734, 776, 769, 773, 764, 774, 755, 774, 790, 816, 797, 766, 788, 768, 752, 741, 815, 778, 797, 774, 765, 816, 766, 791, 777, 850, 744, 787, 762, 763, 780, 753, 757, 730, 708, 765] payload_rows_per_dst=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] dup_rows_per_dst=[2992, 2964, 2972, 2946, 2954, 3072, 3102, 2850] combine_rows_per_src=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343232 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788032 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533232832 weight_movement_bytes=138412032 total_nvlink_bytes=671644864 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64822 dup_rows=24188 dup_rows_remote=21140 dedup_ratio=0.246053 router_tokens_per_expert=[769, 784, 745, 766, 747, 767, 793, 794, 797, 773, 802, 778, 738, 807, 723, 753, 784, 747, 746, 751, 778, 767, 762, 700, 764, 764, 716, 780, 766, 802, 749, 763, 794, 803, 785, 714, 772, 799, 790, 757, 729, 773, 758, 807, 758, 738, 784, 758, 779, 751, 797, 759, 747, 752, 810, 762, 777, 802, 773, 770, 752, 799, 732, 796] payload_rows_per_dst=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] dup_rows_per_dst=[2986, 3048, 2870, 2992, 3166, 2954, 3092, 3080] combine_rows_per_src=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264691712 dispatch_meta_bytes=340104 combine_payload_bytes=264691712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265031816 ingress_bytes=299294720 link_bytes=299294720 token_bytes=529723528 weight_movement_bytes=34603008 total_nvlink_bytes=564326536 weight_to_token_ratio=0.065323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126221936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64808 dup_rows=24242 dup_rows_remote=21376 dedup_ratio=0.246602 router_tokens_per_expert=[774, 774, 755, 806, 737, 703, 785, 732, 803, 828, 765, 803, 799, 762, 794, 750, 775, 758, 741, 741, 791, 779, 757, 779, 750, 746, 759, 756, 792, 771, 709, 777, 825, 776, 759, 787, 796, 761, 767, 762, 812, 746, 781, 757, 792, 794, 745, 716, 788, 800, 746, 774, 791, 747, 754, 778, 725, 751, 739, 766, 747, 765, 787, 767] payload_rows_per_dst=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] dup_rows_per_dst=[3042, 3094, 2982, 2866, 3158, 3066, 3054, 2980] combine_rows_per_src=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343408 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337648 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331888 weight_movement_bytes=34603008 total_nvlink_bytes=566934896 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65042 dup_rows=24008 dup_rows_remote=21038 dedup_ratio=0.244222 router_tokens_per_expert=[732, 711, 756, 751, 747, 782, 794, 743, 763, 784, 809, 802, 784, 717, 743, 745, 767, 759, 778, 762, 762, 752, 793, 739, 778, 783, 781, 819, 765, 787, 808, 784, 696, 794, 796, 780, 730, 748, 758, 752, 767, 739, 770, 808, 826, 805, 716, 780, 764, 779, 743, 726, 774, 790, 739, 778, 787, 760, 800, 787, 741, 774, 804, 761] payload_rows_per_dst=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] dup_rows_per_dst=[2908, 2932, 2970, 3140, 2930, 3104, 2906, 3118] combine_rows_per_src=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343512 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266984920 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533626328 weight_movement_bytes=103809024 total_nvlink_bytes=637435352 weight_to_token_ratio=0.194535 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64814 dup_rows=24038 dup_rows_remote=20870 dedup_ratio=0.244527 router_tokens_per_expert=[746, 729, 767, 771, 805, 780, 771, 693, 777, 755, 724, 782, 759, 746, 766, 778, 769, 787, 809, 809, 783, 750, 750, 768, 759, 775, 743, 766, 785, 742, 730, 741, 819, 789, 781, 779, 792, 772, 770, 808, 736, 790, 767, 800, 752, 761, 800, 757, 795, 768, 782, 777, 732, 786, 740, 761, 734, 758, 770, 777, 800, 794, 765, 725] payload_rows_per_dst=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] dup_rows_per_dst=[2920, 2996, 2996, 2924, 3168, 3042, 3046, 2946] combine_rows_per_src=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343848 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265854760 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531365672 weight_movement_bytes=34603008 total_nvlink_bytes=565968680 weight_to_token_ratio=0.065121 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=344736 combine_payload_bytes=265453568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265798304 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531251872 weight_movement_bytes=34603008 total_nvlink_bytes=565854880 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65122 dup_rows=23866 dup_rows_remote=20864 dedup_ratio=0.242778 router_tokens_per_expert=[733, 817, 796, 763, 715, 811, 748, 783, 754, 747, 802, 766, 747, 774, 769, 814, 771, 719, 752, 758, 759, 737, 811, 769, 795, 772, 798, 761, 785, 782, 796, 799, 689, 753, 742, 743, 732, 813, 776, 767, 759, 741, 823, 776, 809, 784, 743, 772, 733, 753, 816, 782, 714, 768, 767, 826, 730, 766, 732, 785, 743, 783, 780, 749] payload_rows_per_dst=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] dup_rows_per_dst=[2950, 3006, 2922, 3150, 2888, 3030, 3002, 2918] combine_rows_per_src=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344320 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756352 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168384 weight_movement_bytes=34603008 total_nvlink_bytes=567771392 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65098 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[802, 791, 772, 765, 758, 740, 767, 761, 821, 776, 778, 765, 798, 765, 798, 730, 760, 726, 822, 743, 740, 777, 731, 726, 765, 778, 762, 747, 773, 786, 778, 789, 721, 774, 763, 732, 766, 785, 739, 744, 786, 750, 762, 821, 787, 781, 770, 799, 769, 804, 797, 751, 689, 745, 780, 816, 738, 775, 793, 791, 745, 767, 765, 757] payload_rows_per_dst=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] dup_rows_per_dst=[2966, 3076, 2816, 3058, 2774, 3172, 3004, 3060] combine_rows_per_src=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342736 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265820880 ingress_bytes=369287168 link_bytes=369287168 token_bytes=531299024 weight_movement_bytes=103809024 total_nvlink_bytes=635108048 weight_to_token_ratio=0.195387 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389597883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64824 dup_rows=24064 dup_rows_remote=21038 dedup_ratio=0.244792 router_tokens_per_expert=[808, 771, 789, 761, 776, 747, 690, 749, 763, 768, 794, 704, 790, 798, 728, 791, 761, 711, 793, 758, 807, 832, 731, 736, 753, 789, 800, 758, 758, 782, 779, 751, 782, 798, 795, 793, 752, 821, 760, 789, 780, 785, 833, 742, 752, 822, 754, 761, 795, 787, 737, 778, 757, 742, 741, 763, 749, 702, 744, 755, 761, 750, 759, 787] payload_rows_per_dst=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] dup_rows_per_dst=[3026, 3018, 3044, 3052, 3318, 2972, 2740, 2894] combine_rows_per_src=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64998 dup_rows=23978 dup_rows_remote=20802 dedup_ratio=0.243917 router_tokens_per_expert=[744, 745, 773, 819, 769, 768, 789, 792, 784, 733, 766, 743, 752, 807, 713, 738, 729, 798, 756, 771, 747, 765, 808, 752, 726, 773, 780, 773, 805, 693, 773, 804, 825, 765, 780, 801, 790, 746, 797, 744, 781, 763, 768, 812, 791, 767, 797, 773, 792, 763, 737, 766, 785, 756, 778, 735, 766, 728, 736, 690, 776, 754, 797, 805] payload_rows_per_dst=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] dup_rows_per_dst=[2976, 2954, 2956, 2918, 3098, 3176, 3000, 2900] combine_rows_per_src=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343944 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267083656 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533823368 weight_movement_bytes=138412032 total_nvlink_bytes=672235400 weight_to_token_ratio=0.259284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343856 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985264 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626672 weight_movement_bytes=34603008 total_nvlink_bytes=568229680 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65090 dup_rows=23860 dup_rows_remote=20910 dedup_ratio=0.242716 router_tokens_per_expert=[756, 744, 778, 766, 741, 809, 813, 782, 760, 744, 741, 782, 807, 738, 811, 769, 738, 744, 771, 763, 764, 748, 794, 740, 709, 806, 734, 739, 765, 735, 749, 778, 768, 785, 772, 799, 767, 772, 756, 769, 781, 756, 786, 815, 768, 704, 789, 797, 769, 770, 777, 762, 765, 780, 838, 739, 793, 722, 744, 759, 793, 830, 800, 709] payload_rows_per_dst=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] dup_rows_per_dst=[2998, 2950, 2926, 2854, 3010, 3050, 3058, 3014] combine_rows_per_src=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74094 payload_rows_remote=64808 dup_rows=24210 dup_rows_remote=21110 dedup_ratio=0.246277 router_tokens_per_expert=[771, 791, 732, 780, 789, 786, 765, 771, 764, 743, 769, 784, 758, 718, 772, 790, 772, 772, 767, 751, 743, 731, 715, 762, 792, 798, 797, 819, 765, 742, 776, 704, 759, 800, 766, 755, 781, 824, 814, 757, 763, 789, 747, 789, 812, 763, 731, 768, 770, 781, 828, 755, 733, 798, 764, 769, 784, 748, 731, 790, 772, 743, 712, 767] payload_rows_per_dst=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] dup_rows_per_dst=[3058, 3020, 2916, 3100, 3084, 3020, 3082, 2930] combine_rows_per_src=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343448 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862552 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381656 weight_movement_bytes=34603008 total_nvlink_bytes=565984664 weight_to_token_ratio=0.065119 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343200 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266575008 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806816 weight_movement_bytes=103809024 total_nvlink_bytes=636615840 weight_to_token_ratio=0.194834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64964 dup_rows=24018 dup_rows_remote=20960 dedup_ratio=0.244324 router_tokens_per_expert=[787, 759, 693, 729, 793, 717, 794, 753, 784, 822, 774, 750, 747, 761, 785, 799, 755, 788, 751, 767, 759, 800, 783, 787, 727, 800, 788, 768, 764, 772, 785, 738, 733, 732, 740, 782, 805, 819, 785, 756, 800, 820, 777, 765, 784, 758, 748, 732, 749, 790, 799, 761, 770, 781, 734, 796, 739, 765, 747, 774, 742, 803, 783, 704] payload_rows_per_dst=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] dup_rows_per_dst=[2920, 3060, 3058, 2986, 3028, 3066, 3000, 2900] combine_rows_per_src=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65004 dup_rows=24158 dup_rows_remote=21158 dedup_ratio=0.245748 router_tokens_per_expert=[746, 752, 808, 785, 817, 788, 770, 797, 787, 742, 750, 778, 750, 811, 765, 746, 754, 751, 816, 746, 764, 725, 738, 755, 729, 776, 750, 796, 799, 797, 783, 754, 762, 739, 713, 764, 782, 774, 763, 774, 763, 747, 767, 731, 718, 763, 787, 760, 800, 821, 796, 792, 731, 792, 714, 792, 793, 799, 770, 805, 760, 750, 767, 738] payload_rows_per_dst=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] dup_rows_per_dst=[3088, 3024, 2936, 3070, 3000, 2936, 3084, 3020] combine_rows_per_src=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344000 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952640 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561280 weight_movement_bytes=34603008 total_nvlink_bytes=568164288 weight_to_token_ratio=0.064853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343672 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265797240 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531250808 weight_movement_bytes=34603008 total_nvlink_bytes=565853816 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64896 dup_rows=24070 dup_rows_remote=21084 dedup_ratio=0.244853 router_tokens_per_expert=[798, 792, 789, 764, 760, 730, 761, 789, 805, 803, 746, 779, 791, 782, 725, 769, 754, 812, 738, 760, 773, 749, 761, 789, 747, 736, 761, 743, 755, 743, 751, 787, 778, 743, 772, 759, 745, 779, 764, 757, 785, 776, 802, 738, 727, 725, 819, 796, 788, 785, 786, 818, 719, 723, 758, 785, 809, 808, 781, 744, 775, 763, 724, 779] payload_rows_per_dst=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] dup_rows_per_dst=[3136, 2976, 3060, 2874, 2956, 3022, 2986, 3060] combine_rows_per_src=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65038 dup_rows=23938 dup_rows_remote=20990 dedup_ratio=0.243510 router_tokens_per_expert=[744, 763, 795, 793, 751, 784, 809, 769, 760, 774, 758, 755, 762, 815, 762, 800, 778, 767, 791, 819, 782, 750, 755, 783, 716, 778, 790, 802, 715, 768, 734, 766, 743, 779, 806, 744, 759, 714, 742, 757, 769, 791, 762, 726, 804, 784, 783, 810, 793, 736, 741, 750, 805, 725, 716, 788, 769, 753, 771, 764, 787, 774, 752, 767] payload_rows_per_dst=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] dup_rows_per_dst=[2684, 2732, 3084, 2886, 3314, 3298, 2992, 2948] combine_rows_per_src=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436240 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528784 weight_movement_bytes=34603008 total_nvlink_bytes=567131792 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=344648 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266601032 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532857416 weight_movement_bytes=34603008 total_nvlink_bytes=567460424 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64968 dup_rows=23978 dup_rows_remote=21012 dedup_ratio=0.243917 router_tokens_per_expert=[764, 752, 757, 745, 786, 762, 719, 795, 708, 749, 778, 759, 779, 807, 761, 753, 757, 737, 800, 839, 758, 729, 746, 735, 768, 767, 779, 714, 758, 765, 736, 779, 758, 792, 790, 785, 758, 781, 774, 784, 742, 814, 748, 760, 763, 724, 803, 815, 779, 742, 766, 821, 791, 774, 766, 768, 786, 768, 810, 752, 764, 772, 766, 795] payload_rows_per_dst=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] dup_rows_per_dst=[2966, 3072, 3254, 3110, 2928, 2606, 2950, 3092] combine_rows_per_src=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65168 dup_rows=23932 dup_rows_remote=20984 dedup_ratio=0.243449 router_tokens_per_expert=[783, 780, 768, 824, 789, 786, 770, 778, 795, 755, 767, 775, 778, 730, 798, 765, 713, 783, 794, 724, 776, 749, 782, 732, 771, 799, 784, 796, 745, 788, 776, 760, 816, 775, 762, 716, 753, 792, 829, 741, 765, 762, 787, 742, 795, 754, 733, 738, 733, 777, 762, 782, 732, 721, 766, 755, 808, 790, 807, 724, 779, 765, 750, 728] payload_rows_per_dst=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] dup_rows_per_dst=[3038, 2980, 2928, 2988, 3122, 2948, 2916, 3012] combine_rows_per_src=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343920 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157936 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971952 weight_movement_bytes=103809024 total_nvlink_bytes=635780976 weight_to_token_ratio=0.195140 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344112 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266739760 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135408 weight_movement_bytes=103809024 total_nvlink_bytes=636944432 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=64990 dup_rows=23830 dup_rows_remote=20704 dedup_ratio=0.242411 router_tokens_per_expert=[773, 739, 743, 776, 822, 769, 728, 785, 762, 795, 767, 829, 797, 786, 795, 774, 790, 747, 731, 753, 784, 776, 789, 782, 748, 749, 736, 764, 750, 723, 814, 724, 728, 784, 800, 727, 770, 764, 792, 735, 741, 714, 759, 753, 756, 824, 754, 744, 747, 754, 739, 766, 754, 807, 824, 803, 806, 784, 780, 792, 726, 716, 786, 823] payload_rows_per_dst=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] dup_rows_per_dst=[2998, 3126, 3020, 2900, 2942, 2814, 3028, 3002] combine_rows_per_src=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65100 dup_rows=23924 dup_rows_remote=20876 dedup_ratio=0.243368 router_tokens_per_expert=[815, 773, 781, 734, 772, 745, 769, 803, 765, 731, 736, 797, 760, 800, 754, 784, 803, 749, 760, 781, 797, 738, 733, 751, 757, 738, 736, 780, 795, 781, 766, 811, 767, 742, 754, 738, 751, 740, 802, 766, 764, 794, 781, 754, 772, 796, 733, 766, 780, 745, 806, 770, 740, 775, 763, 780, 803, 792, 754, 761, 755, 750, 779, 784] payload_rows_per_dst=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] dup_rows_per_dst=[3018, 2988, 2928, 3048, 2890, 2992, 3032, 3028] combine_rows_per_src=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343920 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452848 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561776 weight_movement_bytes=34603008 total_nvlink_bytes=567164784 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344608 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272736 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200864 weight_movement_bytes=34603008 total_nvlink_bytes=568803872 weight_to_token_ratio=0.064775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342776 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266541816 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532740856 weight_movement_bytes=138412032 total_nvlink_bytes=671152888 weight_to_token_ratio=0.259811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=65016 dup_rows=24080 dup_rows_remote=21076 dedup_ratio=0.244954 router_tokens_per_expert=[833, 801, 791, 778, 769, 738, 789, 773, 777, 802, 808, 755, 773, 830, 780, 750, 758, 768, 772, 768, 763, 760, 732, 785, 759, 726, 772, 742, 761, 766, 800, 755, 796, 802, 716, 718, 791, 741, 733, 808, 784, 734, 780, 760, 780, 764, 804, 758, 787, 760, 752, 732, 772, 747, 778, 734, 768, 739, 736, 820, 709, 774, 762, 779] payload_rows_per_dst=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] dup_rows_per_dst=[3140, 3080, 3004, 2964, 2958, 3088, 2954, 2892] combine_rows_per_src=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64970 dup_rows=24024 dup_rows_remote=21056 dedup_ratio=0.244385 router_tokens_per_expert=[770, 755, 740, 776, 814, 791, 805, 785, 759, 714, 771, 780, 766, 780, 785, 781, 765, 759, 780, 731, 775, 734, 748, 772, 762, 758, 847, 736, 768, 754, 734, 762, 767, 722, 826, 721, 775, 763, 754, 811, 746, 725, 791, 800, 766, 763, 786, 799, 720, 759, 846, 768, 717, 770, 797, 778, 755, 779, 772, 797, 747, 783, 763, 729] payload_rows_per_dst=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] dup_rows_per_dst=[3114, 2996, 3006, 2958, 2968, 3040, 2994, 2948] combine_rows_per_src=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343904 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993504 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643104 weight_movement_bytes=34603008 total_nvlink_bytes=568246112 weight_to_token_ratio=0.064843 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65168 dup_rows=23802 dup_rows_remote=20934 dedup_ratio=0.242126 router_tokens_per_expert=[779, 801, 786, 814, 781, 768, 790, 786, 770, 748, 775, 790, 775, 771, 769, 762, 738, 741, 720, 780, 733, 709, 832, 796, 777, 803, 723, 770, 783, 742, 775, 755, 776, 786, 728, 785, 767, 738, 762, 750, 808, 770, 782, 727, 793, 770, 771, 725, 766, 747, 787, 707, 768, 781, 776, 769, 788, 740, 801, 777, 785, 785, 737, 758] payload_rows_per_dst=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] dup_rows_per_dst=[3130, 3016, 2860, 2958, 2990, 2980, 2868, 3000] combine_rows_per_src=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64962 dup_rows=23942 dup_rows_remote=20872 dedup_ratio=0.243551 router_tokens_per_expert=[771, 755, 790, 720, 826, 752, 784, 763, 799, 781, 795, 766, 773, 788, 759, 772, 750, 781, 827, 724, 778, 847, 787, 820, 749, 781, 785, 775, 774, 780, 713, 771, 743, 736, 743, 735, 769, 785, 799, 777, 780, 768, 789, 738, 709, 755, 705, 752, 745, 763, 759, 763, 767, 825, 806, 766, 755, 782, 737, 764, 757, 750, 756, 738] payload_rows_per_dst=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] dup_rows_per_dst=[2940, 2916, 3050, 3134, 3052, 2990, 2790, 3070] combine_rows_per_src=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344368 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649904 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955440 weight_movement_bytes=69206016 total_nvlink_bytes=602161456 weight_to_token_ratio=0.129853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64972 dup_rows=24042 dup_rows_remote=21110 dedup_ratio=0.244568 router_tokens_per_expert=[691, 773, 766, 782, 804, 800, 765, 730, 757, 743, 809, 743, 791, 784, 801, 740, 825, 794, 738, 762, 790, 795, 779, 777, 744, 737, 757, 787, 773, 793, 793, 767, 767, 749, 765, 745, 776, 764, 778, 741, 772, 735, 748, 747, 786, 776, 748, 796, 773, 743, 742, 783, 783, 773, 781, 731, 781, 757, 795, 742, 732, 831, 745, 777] payload_rows_per_dst=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] dup_rows_per_dst=[2932, 2994, 3132, 3018, 2992, 3112, 2990, 2872] combine_rows_per_src=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344104 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461224 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578344 weight_movement_bytes=34603008 total_nvlink_bytes=567181352 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344408 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272536 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200664 weight_movement_bytes=34603008 total_nvlink_bytes=568803672 weight_to_token_ratio=0.064775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65016 dup_rows=23974 dup_rows_remote=20872 dedup_ratio=0.243876 router_tokens_per_expert=[770, 750, 736, 768, 755, 766, 765, 750, 758, 770, 734, 832, 732, 819, 757, 740, 748, 747, 733, 795, 768, 743, 773, 748, 742, 769, 797, 800, 731, 784, 734, 770, 745, 790, 757, 802, 765, 765, 775, 797, 783, 762, 739, 806, 781, 779, 788, 770, 755, 771, 823, 762, 794, 775, 733, 808, 754, 741, 782, 776, 747, 811, 793, 739] payload_rows_per_dst=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] dup_rows_per_dst=[2878, 2938, 2858, 2932, 3056, 3102, 3174, 3036] combine_rows_per_src=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343336 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266427688 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512040 weight_movement_bytes=34603008 total_nvlink_bytes=567115048 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65298 dup_rows=23738 dup_rows_remote=20832 dedup_ratio=0.241475 router_tokens_per_expert=[770, 799, 781, 750, 773, 755, 764, 797, 795, 754, 751, 740, 777, 711, 789, 770, 771, 783, 733, 765, 761, 806, 796, 770, 762, 744, 780, 745, 776, 757, 789, 770, 818, 754, 784, 777, 748, 769, 772, 733, 727, 742, 753, 699, 740, 772, 751, 777, 794, 800, 772, 749, 811, 768, 781, 765, 738, 769, 742, 795, 817, 785, 778, 788] payload_rows_per_dst=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] dup_rows_per_dst=[2984, 2906, 3018, 2958, 2878, 2826, 3134, 3034] combine_rows_per_src=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65008 dup_rows=23926 dup_rows_remote=21044 dedup_ratio=0.243388 router_tokens_per_expert=[741, 800, 750, 824, 716, 755, 761, 720, 764, 761, 734, 791, 786, 765, 805, 770, 759, 808, 780, 755, 769, 711, 798, 828, 787, 772, 786, 767, 763, 746, 735, 770, 695, 824, 755, 766, 786, 738, 792, 771, 749, 804, 725, 815, 780, 769, 781, 741, 753, 767, 760, 783, 811, 751, 775, 771, 754, 746, 728, 802, 753, 799, 765, 766] payload_rows_per_dst=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] dup_rows_per_dst=[2992, 3114, 3062, 2882, 3006, 3016, 2956, 2898] combine_rows_per_src=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344328 combine_payload_bytes=266125312 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266469640 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532594952 weight_movement_bytes=69206016 total_nvlink_bytes=601800968 weight_to_token_ratio=0.129941 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343552 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649088 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954624 weight_movement_bytes=34603008 total_nvlink_bytes=567557632 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344520 combine_payload_bytes=267460608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267805128 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535265736 weight_movement_bytes=103809024 total_nvlink_bytes=639074760 weight_to_token_ratio=0.193939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64944 dup_rows=24166 dup_rows_remote=21158 dedup_ratio=0.245829 router_tokens_per_expert=[759, 750, 760, 793, 766, 790, 750, 750, 831, 743, 796, 764, 786, 752, 741, 748, 775, 774, 767, 760, 753, 729, 770, 773, 755, 772, 758, 811, 764, 763, 827, 772, 794, 774, 781, 739, 782, 783, 770, 765, 749, 767, 737, 797, 777, 783, 778, 765, 793, 788, 762, 781, 715, 752, 757, 772, 755, 748, 776, 794, 753, 803, 708, 752] payload_rows_per_dst=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] dup_rows_per_dst=[3046, 3008, 3008, 3034, 3104, 3028, 2970, 2968] combine_rows_per_src=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344208 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616976 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889744 weight_movement_bytes=34603008 total_nvlink_bytes=567492752 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65032 dup_rows=23996 dup_rows_remote=21142 dedup_ratio=0.244100 router_tokens_per_expert=[765, 754, 773, 744, 741, 806, 779, 766, 817, 774, 730, 775, 765, 787, 756, 746, 779, 726, 728, 735, 786, 769, 778, 752, 750, 777, 750, 770, 806, 785, 801, 807, 745, 717, 752, 816, 745, 717, 791, 782, 773, 684, 839, 805, 750, 769, 765, 743, 757, 765, 734, 777, 755, 776, 809, 764, 739, 790, 771, 768, 806, 816, 792, 763] payload_rows_per_dst=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] dup_rows_per_dst=[3002, 3048, 2898, 3128, 2854, 2990, 3010, 3066] combine_rows_per_src=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65054 dup_rows=23932 dup_rows_remote=20958 dedup_ratio=0.243449 router_tokens_per_expert=[787, 752, 776, 802, 752, 765, 754, 795, 772, 766, 810, 746, 770, 762, 756, 826, 728, 778, 776, 731, 764, 765, 818, 789, 799, 779, 792, 768, 754, 767, 735, 780, 728, 785, 796, 715, 747, 755, 767, 789, 769, 817, 741, 741, 763, 792, 743, 768, 785, 805, 726, 793, 789, 795, 689, 764, 786, 751, 760, 750, 770, 734, 775, 750] payload_rows_per_dst=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] dup_rows_per_dst=[3028, 2994, 2998, 3058, 2950, 3000, 2974, 2930] combine_rows_per_src=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64708 dup_rows=24334 dup_rows_remote=21230 dedup_ratio=0.247538 router_tokens_per_expert=[733, 750, 749, 793, 853, 730, 770, 803, 761, 748, 771, 746, 742, 759, 758, 791, 768, 754, 772, 749, 800, 765, 791, 796, 806, 729, 766, 784, 734, 795, 772, 775, 766, 759, 725, 787, 760, 717, 791, 770, 764, 809, 746, 788, 760, 817, 786, 797, 808, 771, 790, 757, 745, 771, 758, 733, 728, 774, 757, 804, 728, 760, 786, 727] payload_rows_per_dst=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] dup_rows_per_dst=[3044, 2894, 3174, 3078, 3010, 3006, 3024, 3104] combine_rows_per_src=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344408 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355032 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365656 weight_movement_bytes=34603008 total_nvlink_bytes=566968664 weight_to_token_ratio=0.064999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344696 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715768 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086840 weight_movement_bytes=34603008 total_nvlink_bytes=567689848 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64890 dup_rows=24104 dup_rows_remote=21162 dedup_ratio=0.245199 router_tokens_per_expert=[739, 786, 747, 729, 751, 779, 799, 760, 728, 777, 820, 760, 760, 771, 745, 740, 716, 770, 729, 792, 732, 792, 815, 794, 759, 771, 750, 739, 797, 756, 776, 758, 784, 791, 778, 740, 747, 761, 777, 793, 735, 776, 790, 818, 764, 771, 777, 776, 772, 764, 821, 770, 740, 767, 738, 804, 774, 765, 771, 767, 796, 785, 735, 768] payload_rows_per_dst=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] dup_rows_per_dst=[2942, 2962, 3054, 2962, 3046, 3098, 3028, 3012] combine_rows_per_src=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344048 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266805232 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266416 weight_movement_bytes=103809024 total_nvlink_bytes=637075440 weight_to_token_ratio=0.194666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65180 dup_rows=23798 dup_rows_remote=20758 dedup_ratio=0.242086 router_tokens_per_expert=[761, 760, 750, 775, 797, 748, 772, 796, 792, 783, 738, 739, 798, 791, 779, 776, 754, 764, 758, 807, 750, 776, 725, 753, 736, 818, 736, 750, 752, 778, 784, 809, 758, 765, 749, 785, 734, 750, 775, 742, 788, 813, 716, 789, 786, 791, 758, 742, 759, 737, 781, 815, 779, 730, 770, 782, 753, 763, 797, 777, 723, 775, 801, 764] payload_rows_per_dst=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] dup_rows_per_dst=[3000, 3000, 2938, 3024, 2828, 3040, 2970, 2998] combine_rows_per_src=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343752 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265387720 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530431688 weight_movement_bytes=34603008 total_nvlink_bytes=565034696 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64808 dup_rows=24270 dup_rows_remote=21076 dedup_ratio=0.246887 router_tokens_per_expert=[758, 794, 742, 762, 729, 774, 817, 796, 799, 755, 797, 786, 782, 728, 792, 771, 795, 757, 752, 740, 778, 804, 803, 780, 815, 766, 716, 743, 703, 755, 796, 771, 774, 794, 785, 749, 768, 769, 785, 739, 746, 783, 799, 761, 778, 785, 732, 730, 751, 790, 729, 760, 724, 756, 759, 803, 771, 818, 782, 736, 741, 765, 800, 734] payload_rows_per_dst=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] dup_rows_per_dst=[3076, 3194, 3044, 2938, 2984, 3114, 2894, 3026] combine_rows_per_src=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74648 payload_rows_remote=65446 dup_rows=23656 dup_rows_remote=20740 dedup_ratio=0.240641 router_tokens_per_expert=[759, 797, 751, 776, 790, 751, 788, 789, 773, 767, 764, 787, 813, 756, 797, 750, 773, 744, 793, 729, 734, 803, 744, 746, 800, 773, 754, 738, 772, 757, 728, 737, 817, 753, 734, 716, 794, 769, 777, 779, 788, 778, 763, 785, 769, 771, 743, 749, 737, 740, 772, 773, 777, 781, 795, 807, 735, 771, 760, 785, 772, 778, 758, 793] payload_rows_per_dst=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] dup_rows_per_dst=[3062, 2948, 2866, 2916, 2936, 3004, 2974, 2950] combine_rows_per_src=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923088 weight_movement_bytes=69206016 total_nvlink_bytes=601129104 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=343752 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267321032 ingress_bytes=336183296 link_bytes=336183296 token_bytes=534298312 weight_movement_bytes=69206016 total_nvlink_bytes=603504328 weight_to_token_ratio=0.129527 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265030678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65272 dup_rows=23786 dup_rows_remote=20840 dedup_ratio=0.241964 router_tokens_per_expert=[774, 760, 792, 759, 825, 771, 788, 797, 758, 768, 767, 796, 832, 778, 736, 786, 768, 743, 791, 727, 763, 790, 752, 762, 772, 759, 765, 842, 695, 736, 784, 756, 726, 774, 731, 770, 764, 760, 771, 798, 739, 781, 805, 727, 723, 732, 768, 761, 784, 800, 770, 776, 752, 799, 754, 776, 806, 728, 790, 803, 790, 766, 683, 753] payload_rows_per_dst=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] dup_rows_per_dst=[3110, 3042, 2946, 2864, 2954, 2926, 3010, 2934] combine_rows_per_src=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343536 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797104 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531250672 weight_movement_bytes=69206016 total_nvlink_bytes=600456688 weight_to_token_ratio=0.130270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65132 dup_rows=23886 dup_rows_remote=20886 dedup_ratio=0.242981 router_tokens_per_expert=[740, 782, 760, 755, 813, 809, 790, 758, 746, 738, 768, 726, 819, 755, 742, 786, 763, 736, 772, 783, 793, 811, 773, 793, 750, 797, 722, 758, 794, 736, 758, 746, 761, 777, 755, 774, 805, 757, 755, 759, 813, 728, 781, 753, 750, 816, 752, 833, 800, 729, 796, 747, 795, 733, 815, 765, 750, 793, 749, 770, 743, 692, 744, 790] payload_rows_per_dst=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] dup_rows_per_dst=[3074, 2878, 3002, 2968, 3000, 3046, 3034, 2884] combine_rows_per_src=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268066816 dispatch_meta_bytes=344744 combine_payload_bytes=268066816 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=268411560 ingress_bytes=302669824 link_bytes=302669824 token_bytes=536478376 weight_movement_bytes=34603008 total_nvlink_bytes=571081384 weight_to_token_ratio=0.064500 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138922181e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65106 dup_rows=23754 dup_rows_remote=20704 dedup_ratio=0.241638 router_tokens_per_expert=[763, 735, 762, 781, 790, 748, 760, 732, 767, 746, 762, 772, 746, 765, 740, 741, 746, 734, 811, 802, 760, 762, 781, 770, 726, 796, 737, 790, 786, 707, 778, 773, 792, 745, 754, 779, 841, 817, 790, 761, 823, 767, 772, 783, 759, 754, 754, 728, 770, 803, 767, 772, 826, 759, 764, 786, 792, 768, 733, 769, 775, 780, 744, 756] payload_rows_per_dst=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] dup_rows_per_dst=[2910, 2910, 2956, 2846, 3118, 2990, 3050, 2974] combine_rows_per_src=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=344448 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267698560 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535052672 weight_movement_bytes=34603008 total_nvlink_bytes=569655680 weight_to_token_ratio=0.064672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64812 dup_rows=24072 dup_rows_remote=20946 dedup_ratio=0.244873 router_tokens_per_expert=[757, 766, 794, 736, 764, 766, 779, 830, 820, 746, 787, 757, 756, 772, 756, 722, 721, 749, 779, 785, 804, 802, 759, 817, 747, 779, 752, 741, 785, 783, 794, 743, 775, 787, 762, 761, 766, 754, 738, 773, 773, 766, 833, 807, 802, 728, 728, 728, 792, 761, 767, 771, 749, 770, 788, 732, 801, 786, 738, 730, 769, 757, 744, 768] payload_rows_per_dst=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] dup_rows_per_dst=[2590, 3140, 3050, 2918, 3174, 3008, 3066, 3126] combine_rows_per_src=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344072 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267124744 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905416 weight_movement_bytes=34603008 total_nvlink_bytes=568508424 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343240 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267017416 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533691592 weight_movement_bytes=103809024 total_nvlink_bytes=637500616 weight_to_token_ratio=0.194511 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65146 dup_rows=23780 dup_rows_remote=20818 dedup_ratio=0.241903 router_tokens_per_expert=[781, 780, 781, 780, 797, 760, 710, 774, 753, 766, 772, 795, 828, 751, 775, 787, 725, 773, 744, 784, 773, 794, 756, 793, 756, 754, 749, 743, 779, 781, 788, 794, 718, 804, 787, 785, 763, 753, 794, 750, 765, 794, 734, 791, 798, 717, 768, 756, 777, 765, 781, 739, 714, 843, 775, 739, 798, 726, 746, 743, 717, 788, 793, 755] payload_rows_per_dst=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] dup_rows_per_dst=[2962, 3068, 2936, 2980, 2986, 2936, 2930, 2982] combine_rows_per_src=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65088 dup_rows=24024 dup_rows_remote=21064 dedup_ratio=0.244385 router_tokens_per_expert=[768, 761, 760, 754, 775, 775, 752, 797, 797, 756, 772, 774, 736, 760, 755, 816, 777, 750, 697, 795, 807, 781, 800, 754, 778, 764, 765, 751, 701, 752, 745, 776, 766, 746, 756, 737, 782, 831, 727, 742, 761, 750, 785, 756, 765, 766, 759, 734, 783, 787, 816, 754, 814, 783, 770, 780, 745, 796, 767, 763, 780, 745, 790, 815] payload_rows_per_dst=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] dup_rows_per_dst=[3052, 2978, 3012, 2920, 2868, 2960, 3150, 3084] combine_rows_per_src=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343032 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812984 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282936 weight_movement_bytes=34603008 total_nvlink_bytes=565885944 weight_to_token_ratio=0.065131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64862 dup_rows=24286 dup_rows_remote=21358 dedup_ratio=0.247050 router_tokens_per_expert=[760, 730, 827, 817, 694, 745, 787, 818, 747, 771, 715, 801, 753, 709, 759, 787, 755, 759, 782, 780, 768, 765, 744, 727, 811, 783, 771, 822, 760, 818, 779, 763, 764, 782, 782, 794, 742, 721, 809, 772, 761, 809, 762, 776, 742, 752, 774, 772, 816, 763, 737, 749, 782, 822, 756, 751, 765, 761, 734, 758, 762, 733, 788, 754] payload_rows_per_dst=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] dup_rows_per_dst=[3034, 2928, 3002, 3156, 3120, 3058, 3044, 2944] combine_rows_per_src=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65094 dup_rows=23924 dup_rows_remote=20990 dedup_ratio=0.243368 router_tokens_per_expert=[759, 793, 793, 770, 803, 801, 746, 755, 749, 801, 737, 777, 740, 763, 794, 758, 795, 743, 742, 751, 760, 752, 747, 750, 735, 770, 742, 800, 776, 778, 753, 756, 772, 800, 797, 797, 737, 757, 790, 742, 769, 753, 817, 781, 787, 772, 771, 780, 778, 737, 763, 761, 795, 733, 724, 764, 735, 790, 812, 798, 755, 734, 759, 803] payload_rows_per_dst=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] dup_rows_per_dst=[3070, 2968, 2868, 2934, 3058, 3120, 2902, 3004] combine_rows_per_src=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343856 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267181872 ingress_bytes=370647040 link_bytes=370647040 token_bytes=534019888 weight_movement_bytes=103809024 total_nvlink_bytes=637828912 weight_to_token_ratio=0.194392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394714972e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64926 dup_rows=23962 dup_rows_remote=20956 dedup_ratio=0.243754 router_tokens_per_expert=[766, 744, 786, 751, 789, 782, 754, 795, 751, 724, 782, 820, 743, 776, 787, 761, 818, 749, 769, 765, 777, 771, 801, 761, 783, 779, 805, 745, 761, 763, 752, 827, 808, 793, 823, 758, 772, 793, 778, 734, 735, 791, 777, 790, 760, 746, 792, 768, 749, 759, 758, 734, 763, 745, 737, 695, 768, 755, 774, 744, 747, 772, 757, 740] payload_rows_per_dst=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] dup_rows_per_dst=[3078, 2946, 3006, 3010, 3186, 3056, 2740, 2940] combine_rows_per_src=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344608 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266945056 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545504 weight_movement_bytes=69206016 total_nvlink_bytes=602751520 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344880 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266019632 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531694384 weight_movement_bytes=34603008 total_nvlink_bytes=566297392 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65148 dup_rows=23930 dup_rows_remote=21066 dedup_ratio=0.243429 router_tokens_per_expert=[813, 752, 746, 723, 749, 814, 755, 758, 753, 801, 729, 788, 771, 725, 790, 793, 777, 779, 796, 775, 780, 782, 775, 745, 756, 815, 799, 708, 831, 759, 732, 764, 784, 810, 723, 748, 734, 759, 735, 752, 736, 758, 791, 742, 757, 792, 771, 764, 768, 751, 755, 743, 816, 771, 802, 791, 751, 743, 750, 771, 769, 793, 822, 767] payload_rows_per_dst=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] dup_rows_per_dst=[2926, 2966, 3088, 2996, 2864, 2996, 3104, 2990] combine_rows_per_src=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344336 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969360 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594384 weight_movement_bytes=34603008 total_nvlink_bytes=568197392 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65030 dup_rows=24056 dup_rows_remote=21114 dedup_ratio=0.244710 router_tokens_per_expert=[771, 744, 798, 740, 794, 764, 780, 798, 814, 783, 753, 777, 750, 750, 772, 753, 739, 773, 779, 724, 786, 778, 784, 781, 772, 761, 789, 802, 824, 744, 780, 749, 771, 723, 748, 748, 759, 745, 750, 739, 747, 796, 747, 797, 787, 792, 747, 796, 761, 762, 779, 782, 786, 747, 709, 754, 757, 772, 794, 785, 812, 730, 769, 755] payload_rows_per_dst=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] dup_rows_per_dst=[3040, 3026, 3006, 3078, 2878, 3092, 2942, 2994] combine_rows_per_src=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343528 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280424 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217320 weight_movement_bytes=69206016 total_nvlink_bytes=601423336 weight_to_token_ratio=0.130033 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64654 dup_rows=24132 dup_rows_remote=21028 dedup_ratio=0.245483 router_tokens_per_expert=[791, 794, 746, 812, 752, 740, 769, 753, 791, 782, 814, 761, 769, 754, 703, 815, 759, 770, 763, 766, 789, 786, 772, 810, 726, 798, 788, 791, 793, 747, 756, 772, 747, 773, 723, 769, 791, 754, 746, 784, 775, 750, 758, 773, 749, 733, 752, 782, 805, 791, 725, 779, 771, 786, 812, 731, 810, 775, 755, 705, 749, 759, 757, 751] payload_rows_per_dst=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] dup_rows_per_dst=[2792, 2726, 2980, 3296, 2958, 3172, 3104, 3104] combine_rows_per_src=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344856 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191064 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037272 weight_movement_bytes=69206016 total_nvlink_bytes=603243288 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344576 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707456 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070336 weight_movement_bytes=103809024 total_nvlink_bytes=636879360 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73862 payload_rows_remote=64468 dup_rows=24442 dup_rows_remote=21236 dedup_ratio=0.248637 router_tokens_per_expert=[814, 801, 796, 805, 776, 776, 774, 752, 767, 775, 758, 700, 767, 793, 753, 784, 820, 735, 739, 770, 759, 754, 793, 725, 761, 766, 796, 732, 752, 811, 745, 729, 766, 768, 796, 726, 791, 811, 755, 755, 780, 767, 739, 789, 760, 753, 770, 784, 781, 696, 785, 764, 805, 769, 774, 777, 773, 756, 788, 742, 735, 802, 727, 790] payload_rows_per_dst=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] dup_rows_per_dst=[3206, 2992, 3168, 3106, 3030, 2926, 2970, 3044] combine_rows_per_src=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64946 dup_rows=24114 dup_rows_remote=21078 dedup_ratio=0.245300 router_tokens_per_expert=[791, 773, 764, 773, 772, 739, 756, 768, 767, 773, 757, 776, 728, 749, 760, 734, 782, 772, 745, 739, 801, 786, 782, 773, 781, 742, 806, 787, 779, 748, 766, 733, 776, 747, 770, 787, 742, 735, 793, 749, 769, 725, 783, 759, 796, 810, 757, 741, 769, 776, 779, 770, 834, 764, 756, 801, 730, 737, 810, 796, 772, 779, 774, 764] payload_rows_per_dst=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] dup_rows_per_dst=[2952, 2968, 3060, 3010, 2996, 3036, 3132, 2960] combine_rows_per_src=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64768 dup_rows=24198 dup_rows_remote=21048 dedup_ratio=0.246155 router_tokens_per_expert=[793, 803, 774, 795, 750, 743, 758, 738, 737, 758, 773, 801, 779, 819, 804, 773, 745, 734, 745, 828, 779, 757, 755, 739, 759, 727, 770, 747, 753, 730, 795, 728, 776, 785, 758, 787, 743, 777, 703, 801, 755, 796, 772, 757, 756, 793, 795, 792, 775, 782, 778, 783, 788, 773, 772, 744, 795, 739, 795, 748, 725, 795, 769, 756] payload_rows_per_dst=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] dup_rows_per_dst=[3024, 3150, 3006, 2826, 3046, 3164, 3004, 2978] combine_rows_per_src=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342728 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165512 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988296 weight_movement_bytes=34603008 total_nvlink_bytes=564591304 weight_to_token_ratio=0.065290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64932 dup_rows=24186 dup_rows_remote=21340 dedup_ratio=0.246033 router_tokens_per_expert=[804, 772, 843, 779, 829, 762, 765, 766, 819, 776, 734, 801, 815, 746, 807, 771, 768, 743, 813, 775, 776, 742, 692, 729, 785, 748, 738, 749, 725, 719, 790, 762, 749, 789, 769, 781, 785, 824, 805, 801, 803, 764, 771, 763, 724, 742, 708, 787, 742, 797, 777, 749, 786, 774, 746, 744, 774, 771, 789, 744, 755, 720, 722, 754] payload_rows_per_dst=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] dup_rows_per_dst=[3176, 3168, 2878, 2846, 3210, 2932, 3056, 2920] combine_rows_per_src=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264060928 dispatch_meta_bytes=342816 combine_payload_bytes=264060928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=264403744 ingress_bytes=402472960 link_bytes=402472960 token_bytes=528464672 weight_movement_bytes=138412032 total_nvlink_bytes=666876704 weight_to_token_ratio=0.261914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514473347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344096 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362912 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381728 weight_movement_bytes=34603008 total_nvlink_bytes=566984736 weight_to_token_ratio=0.064997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65072 dup_rows=23934 dup_rows_remote=20918 dedup_ratio=0.243469 router_tokens_per_expert=[783, 780, 784, 754, 820, 784, 789, 747, 756, 819, 763, 775, 694, 743, 777, 735, 778, 744, 781, 775, 752, 777, 749, 801, 794, 807, 778, 780, 770, 757, 776, 792, 792, 756, 748, 758, 746, 772, 789, 757, 780, 738, 762, 771, 761, 723, 766, 778, 776, 809, 737, 774, 767, 775, 816, 832, 753, 704, 759, 739, 728, 736, 732, 804] payload_rows_per_dst=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] dup_rows_per_dst=[3068, 2938, 3016, 3096, 2914, 2968, 3106, 2828] combine_rows_per_src=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343264 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632992 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922720 weight_movement_bytes=103809024 total_nvlink_bytes=634731744 weight_to_token_ratio=0.195526 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64904 dup_rows=24114 dup_rows_remote=21160 dedup_ratio=0.245300 router_tokens_per_expert=[767, 764, 775, 773, 791, 733, 765, 713, 796, 806, 782, 723, 770, 775, 776, 746, 798, 751, 772, 785, 753, 748, 829, 821, 752, 753, 795, 778, 804, 810, 767, 760, 730, 736, 764, 789, 790, 789, 733, 789, 757, 772, 786, 796, 763, 752, 795, 805, 731, 779, 743, 734, 760, 754, 788, 769, 732, 773, 721, 750, 751, 763, 753, 774] payload_rows_per_dst=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] dup_rows_per_dst=[2950, 2998, 3214, 3132, 2954, 3064, 2958, 2844] combine_rows_per_src=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65104 dup_rows=23972 dup_rows_remote=21016 dedup_ratio=0.243856 router_tokens_per_expert=[792, 765, 738, 794, 750, 732, 760, 802, 775, 754, 838, 800, 768, 774, 789, 754, 775, 732, 721, 772, 763, 796, 800, 771, 730, 769, 761, 726, 738, 767, 785, 781, 812, 790, 757, 761, 759, 796, 750, 745, 771, 779, 794, 753, 752, 774, 757, 751, 768, 759, 752, 755, 744, 779, 758, 777, 798, 782, 752, 747, 773, 766, 782, 787] payload_rows_per_dst=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] dup_rows_per_dst=[2884, 3186, 2976, 2840, 3112, 2984, 2956, 3034] combine_rows_per_src=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=345088 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266306560 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532268032 weight_movement_bytes=34603008 total_nvlink_bytes=566871040 weight_to_token_ratio=0.065010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266878872 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533413784 weight_movement_bytes=34603008 total_nvlink_bytes=568016792 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344256 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191040 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037824 weight_movement_bytes=34603008 total_nvlink_bytes=566640832 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64480 dup_rows=24066 dup_rows_remote=20682 dedup_ratio=0.244812 router_tokens_per_expert=[737, 824, 780, 752, 785, 834, 769, 761, 756, 770, 766, 786, 757, 733, 845, 747, 723, 766, 768, 770, 759, 759, 790, 759, 778, 770, 754, 722, 777, 817, 761, 748, 777, 777, 755, 783, 781, 802, 764, 809, 735, 769, 748, 780, 744, 744, 703, 803, 784, 764, 741, 751, 787, 756, 750, 769, 796, 757, 751, 769, 729, 764, 792, 795] payload_rows_per_dst=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] dup_rows_per_dst=[3202, 2958, 2988, 2906, 3082, 2576, 2970, 3384] combine_rows_per_src=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010464 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676448 weight_movement_bytes=34603008 total_nvlink_bytes=568279456 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64790 dup_rows=24084 dup_rows_remote=20994 dedup_ratio=0.244995 router_tokens_per_expert=[780, 806, 755, 755, 779, 817, 768, 797, 755, 757, 820, 770, 793, 797, 766, 796, 756, 775, 790, 764, 805, 757, 809, 744, 749, 734, 738, 791, 754, 751, 795, 771, 757, 721, 759, 816, 751, 759, 784, 731, 758, 759, 726, 751, 784, 765, 736, 735, 751, 773, 781, 745, 772, 801, 811, 714, 769, 779, 763, 742, 760, 777, 750, 778] payload_rows_per_dst=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] dup_rows_per_dst=[3090, 3152, 3012, 2928, 2952, 2906, 3030, 3014] combine_rows_per_src=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261994256 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6134951696 ingress_bytes=7476559872 link_bytes=7476559872 total_nvlink_bytes=13611511568 weight_to_token_ratio=0.110057 prefetch_share_of_ingress=0.101820 dedup_ratio=0.243594 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223204039e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65094 dup_rows=23828 dup_rows_remote=20862 dedup_ratio=0.242391 router_tokens_per_expert=[751, 797, 740, 743, 805, 752, 745, 739, 752, 747, 733, 796, 760, 752, 782, 751, 810, 807, 768, 766, 778, 758, 824, 746, 800, 751, 757, 781, 800, 713, 723, 749, 737, 799, 759, 759, 796, 763, 771, 752, 785, 752, 743, 804, 770, 731, 793, 796, 761, 700, 765, 810, 727, 807, 764, 798, 771, 777, 762, 817, 786, 747, 769, 805] payload_rows_per_dst=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] dup_rows_per_dst=[2930, 2940, 3058, 2936, 3006, 2966, 2980, 3012] combine_rows_per_src=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64984 dup_rows=24058 dup_rows_remote=20994 dedup_ratio=0.244731 router_tokens_per_expert=[774, 773, 740, 750, 777, 778, 764, 788, 745, 740, 791, 777, 758, 827, 778, 747, 754, 748, 765, 735, 771, 790, 762, 757, 755, 817, 749, 769, 760, 719, 789, 760, 768, 815, 750, 786, 789, 771, 785, 747, 821, 803, 783, 768, 763, 744, 807, 701, 741, 730, 737, 761, 806, 807, 842, 779, 728, 778, 767, 743, 766, 804, 715, 740] payload_rows_per_dst=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] dup_rows_per_dst=[2984, 3064, 3000, 3012, 2968, 2960, 3154, 2916] combine_rows_per_src=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256003456 prefetch_bytes=519045120 reduce_bytes=795869184 weight_movement_bytes=1314914304 egress_bytes=6131959168 ingress_bytes=7438958592 link_bytes=7438958592 total_nvlink_bytes=13570917760 weight_to_token_ratio=0.107287 prefetch_share_of_ingress=0.069774 dedup_ratio=0.244439 prefetch_slots=30 unused_prefetch_slots=338 slot_utilization=0.081522 reduce_slots=23 remote_demand=15 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.217052274e-04 prefetch_bytes_per_gemm_flop=8.491847826e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65150 dup_rows=23898 dup_rows_remote=20794 dedup_ratio=0.243103 router_tokens_per_expert=[743, 760, 781, 709, 758, 719, 783, 767, 730, 803, 751, 717, 780, 779, 783, 812, 729, 786, 754, 758, 746, 778, 722, 760, 750, 777, 759, 795, 796, 764, 753, 786, 784, 765, 774, 776, 788, 762, 790, 755, 786, 808, 787, 739, 800, 790, 827, 808, 748, 742, 725, 774, 780, 820, 771, 770, 745, 776, 771, 744, 798, 773, 755, 733] payload_rows_per_dst=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] dup_rows_per_dst=[2776, 3098, 2818, 3104, 3036, 3180, 2926, 2960] combine_rows_per_src=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264110080 dispatch_meta_bytes=340648 combine_payload_bytes=264110080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264450728 ingress_bytes=298713088 link_bytes=298713088 token_bytes=528560808 weight_movement_bytes=34603008 total_nvlink_bytes=563163816 weight_to_token_ratio=0.065466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124033302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343136 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265722976 ingress_bytes=369188864 link_bytes=369188864 token_bytes=531102816 weight_movement_bytes=103809024 total_nvlink_bytes=634911840 weight_to_token_ratio=0.195459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389227973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343824 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968848 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593872 weight_movement_bytes=69206016 total_nvlink_bytes=602799888 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343912 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518376 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692840 weight_movement_bytes=34603008 total_nvlink_bytes=567295848 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=20934 dedup_ratio=0.243530 router_tokens_per_expert=[755, 789, 740, 802, 745, 733, 774, 744, 727, 730, 774, 759, 757, 790, 755, 791, 763, 805, 802, 739, 766, 774, 770, 790, 731, 790, 725, 741, 805, 795, 751, 790, 781, 741, 760, 766, 777, 723, 781, 757, 794, 791, 796, 783, 782, 734, 781, 749, 755, 759, 776, 820, 716, 775, 766, 762, 740, 769, 807, 757, 774, 769, 780, 829] payload_rows_per_dst=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] dup_rows_per_dst=[2912, 2948, 3064, 2956, 2924, 3076, 3006, 3054] combine_rows_per_src=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343776 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267198176 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534052576 weight_movement_bytes=138412032 total_nvlink_bytes=672464608 weight_to_token_ratio=0.259173 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12217453680 prefetch_bytes=795869184 reduce_bytes=242221056 weight_movement_bytes=1038090240 egress_bytes=6112668784 ingress_bytes=7142875136 link_bytes=7142875136 total_nvlink_bytes=13255543920 weight_to_token_ratio=0.084968 prefetch_share_of_ingress=0.111421 dedup_ratio=0.244770 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=7 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.168611482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266699520 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6137305856 ingress_bytes=7478910976 link_bytes=7478910976 total_nvlink_bytes=13616216832 weight_to_token_ratio=0.110015 prefetch_share_of_ingress=0.101788 dedup_ratio=0.243613 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223588691e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259328616 prefetch_bytes=657457152 reduce_bytes=1038090240 weight_movement_bytes=1695547392 egress_bytes=6133621352 ingress_bytes=7821254656 link_bytes=7821254656 total_nvlink_bytes=13954876008 weight_to_token_ratio=0.138307 prefetch_share_of_ingress=0.084060 dedup_ratio=0.244174 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=30 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.279597896e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344184 combine_payload_bytes=266698752 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267042936 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741688 weight_movement_bytes=34603008 total_nvlink_bytes=568344696 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261855048 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6134886216 ingress_bytes=7268868096 link_bytes=7268868096 total_nvlink_bytes=13403754312 weight_to_token_ratio=0.093126 prefetch_share_of_ingress=0.090448 dedup_ratio=0.243975 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189224585e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64928 dup_rows=23920 dup_rows_remote=20836 dedup_ratio=0.243327 router_tokens_per_expert=[761, 776, 788, 756, 752, 756, 800, 798, 772, 755, 777, 772, 803, 793, 797, 766, 708, 780, 773, 760, 726, 764, 741, 764, 758, 783, 760, 812, 737, 773, 835, 817, 772, 744, 757, 729, 728, 786, 748, 760, 805, 771, 767, 771, 779, 749, 824, 776, 743, 781, 766, 809, 824, 736, 746, 754, 774, 766, 737, 759, 713, 761, 733, 771] payload_rows_per_dst=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] dup_rows_per_dst=[3084, 3072, 2862, 3098, 2800, 3122, 2998, 2884] combine_rows_per_src=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12268323528 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=6138118856 ingress_bytes=7825752064 link_bytes=7825752064 total_nvlink_bytes=13963870920 weight_to_token_ratio=0.138205 prefetch_share_of_ingress=0.079590 dedup_ratio=0.243605 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280333695e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343056 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288144 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233232 weight_movement_bytes=34603008 total_nvlink_bytes=566836240 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12232651112 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6120272232 ingress_bytes=7461896192 link_bytes=7461896192 total_nvlink_bytes=13582168424 weight_to_token_ratio=0.110321 prefetch_share_of_ingress=0.092746 dedup_ratio=0.244256 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220804984e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-29 23:41:54] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 672.4 | throughput per GPU (TFLOP/s/GPU): 189.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.385180E+00 | loss scale: 1.0 | grad norm: 5.190 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65024 dup_rows=23926 dup_rows_remote=20970 dedup_ratio=0.243388 router_tokens_per_expert=[753, 776, 760, 787, 796, 757, 783, 752, 781, 723, 726, 761, 810, 776, 823, 755, 728, 733, 743, 764, 741, 773, 749, 793, 785, 729, 814, 792, 744, 743, 741, 769, 789, 755, 766, 731, 777, 806, 778, 789, 792, 823, 769, 741, 778, 744, 779, 774, 776, 768, 754, 748, 773, 752, 805, 769, 759, 785, 789, 742, 785, 749, 733, 814] payload_rows_per_dst=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] dup_rows_per_dst=[3068, 2956, 2890, 2994, 3058, 2992, 2954, 3014] combine_rows_per_src=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64842 dup_rows=24050 dup_rows_remote=20954 dedup_ratio=0.244649 router_tokens_per_expert=[810, 719, 756, 761, 762, 784, 730, 745, 763, 805, 777, 763, 761, 696, 790, 748, 777, 776, 788, 752, 803, 768, 823, 767, 763, 717, 811, 807, 753, 718, 728, 758, 807, 755, 767, 784, 760, 760, 750, 768, 753, 776, 813, 786, 791, 797, 762, 743, 762, 761, 771, 720, 796, 711, 821, 738, 786, 780, 814, 804, 762, 746, 739, 790] payload_rows_per_dst=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] dup_rows_per_dst=[2922, 2964, 3096, 2950, 2990, 3110, 2942, 3076] combine_rows_per_src=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65206 dup_rows=23798 dup_rows_remote=20718 dedup_ratio=0.242086 router_tokens_per_expert=[801, 758, 781, 746, 782, 708, 760, 818, 754, 746, 755, 765, 760, 766, 804, 778, 759, 732, 759, 749, 804, 738, 784, 747, 740, 754, 764, 780, 762, 774, 754, 802, 796, 741, 755, 737, 777, 799, 766, 762, 760, 749, 763, 849, 768, 758, 795, 755, 741, 756, 761, 781, 782, 793, 801, 775, 802, 786, 796, 763, 766, 792, 715, 728] payload_rows_per_dst=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] dup_rows_per_dst=[2964, 2916, 2876, 3000, 2944, 3000, 3080, 3018] combine_rows_per_src=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64822 dup_rows=24134 dup_rows_remote=20972 dedup_ratio=0.245504 router_tokens_per_expert=[750, 745, 761, 769, 787, 772, 742, 775, 819, 794, 743, 744, 743, 770, 740, 743, 786, 769, 761, 775, 743, 735, 798, 757, 810, 723, 794, 787, 809, 726, 744, 772, 781, 802, 753, 742, 806, 749, 775, 785, 733, 786, 724, 758, 749, 767, 743, 788, 742, 749, 772, 773, 772, 821, 769, 772, 791, 782, 754, 769, 792, 786, 790, 791] payload_rows_per_dst=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] dup_rows_per_dst=[3012, 3000, 3024, 3006, 2964, 2956, 3010, 3162] combine_rows_per_src=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20912 dedup_ratio=0.243083 router_tokens_per_expert=[800, 743, 741, 746, 748, 760, 725, 721, 785, 764, 737, 771, 767, 799, 773, 762, 787, 751, 748, 742, 776, 786, 747, 752, 760, 760, 779, 786, 776, 822, 786, 757, 814, 752, 755, 805, 778, 755, 772, 775, 814, 775, 760, 778, 776, 730, 750, 804, 800, 750, 746, 779, 738, 814, 785, 797, 752, 796, 751, 777, 740, 753, 757, 767] payload_rows_per_dst=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] dup_rows_per_dst=[2834, 3048, 2950, 3092, 2988, 2984, 3022, 2978] combine_rows_per_src=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64804 dup_rows=24168 dup_rows_remote=21056 dedup_ratio=0.245850 router_tokens_per_expert=[786, 762, 843, 766, 781, 763, 773, 743, 785, 780, 755, 767, 769, 811, 761, 776, 760, 756, 779, 751, 783, 777, 818, 762, 761, 744, 811, 750, 748, 763, 800, 729, 737, 814, 714, 812, 776, 800, 817, 752, 751, 774, 767, 753, 773, 808, 749, 775, 738, 764, 768, 756, 745, 773, 705, 788, 771, 774, 769, 736, 744, 747, 766, 723] payload_rows_per_dst=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] dup_rows_per_dst=[3090, 3062, 3018, 2964, 3112, 3050, 2978, 2894] combine_rows_per_src=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64900 dup_rows=24096 dup_rows_remote=21086 dedup_ratio=0.245117 router_tokens_per_expert=[719, 768, 773, 771, 790, 751, 743, 736, 734, 717, 750, 727, 811, 796, 767, 770, 714, 797, 745, 745, 765, 804, 747, 758, 779, 784, 798, 776, 785, 741, 753, 743, 766, 841, 801, 811, 798, 776, 746, 759, 814, 766, 800, 767, 802, 740, 813, 714, 733, 743, 775, 741, 757, 810, 796, 795, 751, 749, 762, 772, 777, 780, 746, 794] payload_rows_per_dst=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] dup_rows_per_dst=[2928, 2864, 2948, 3010, 3196, 3066, 3056, 3028] combine_rows_per_src=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343976 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682280 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020584 weight_movement_bytes=34603008 total_nvlink_bytes=567623592 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343184 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936016 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528848 weight_movement_bytes=34603008 total_nvlink_bytes=566131856 weight_to_token_ratio=0.065101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343696 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427472 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511248 weight_movement_bytes=34603008 total_nvlink_bytes=569114256 weight_to_token_ratio=0.064738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343176 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854088 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365000 weight_movement_bytes=103809024 total_nvlink_bytes=635174024 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343720 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266657448 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971176 weight_movement_bytes=103809024 total_nvlink_bytes=636780200 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64632 dup_rows=24148 dup_rows_remote=20844 dedup_ratio=0.245646 router_tokens_per_expert=[764, 785, 752, 731, 770, 770, 737, 780, 773, 775, 762, 756, 760, 744, 781, 765, 785, 790, 761, 783, 748, 784, 700, 833, 738, 787, 693, 726, 789, 772, 788, 773, 738, 774, 769, 811, 775, 799, 765, 748, 787, 792, 728, 792, 742, 800, 792, 756, 804, 781, 783, 759, 762, 759, 756, 743, 783, 778, 810, 787, 723, 763, 775, 763] payload_rows_per_dst=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] dup_rows_per_dst=[3304, 3092, 3104, 3082, 2622, 2890, 3034, 3020] combine_rows_per_src=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343440 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265780624 ingress_bytes=369246208 link_bytes=369246208 token_bytes=531217808 weight_movement_bytes=103809024 total_nvlink_bytes=635026832 weight_to_token_ratio=0.195417 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343944 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174344 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004744 weight_movement_bytes=34603008 total_nvlink_bytes=566607752 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64958 dup_rows=24238 dup_rows_remote=21330 dedup_ratio=0.246562 router_tokens_per_expert=[772, 736, 753, 761, 712, 770, 775, 706, 787, 754, 740, 757, 724, 741, 734, 771, 764, 792, 769, 781, 776, 804, 786, 773, 719, 744, 819, 769, 765, 758, 771, 772, 849, 800, 763, 804, 756, 737, 783, 764, 764, 739, 713, 815, 754, 780, 786, 762, 831, 765, 751, 751, 790, 812, 784, 768, 779, 764, 756, 754, 810, 756, 785, 772] payload_rows_per_dst=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] dup_rows_per_dst=[2918, 2908, 3082, 3030, 3104, 3030, 3164, 3002] combine_rows_per_src=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65008 dup_rows=24016 dup_rows_remote=21098 dedup_ratio=0.244303 router_tokens_per_expert=[710, 762, 792, 797, 823, 833, 767, 737, 787, 788, 811, 784, 793, 781, 761, 765, 806, 747, 761, 735, 731, 730, 820, 769, 757, 770, 719, 769, 743, 733, 739, 749, 766, 812, 780, 755, 749, 748, 800, 783, 754, 774, 745, 807, 711, 753, 759, 751, 796, 727, 749, 777, 768, 798, 725, 776, 782, 763, 799, 766, 825, 770, 746, 769] payload_rows_per_dst=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] dup_rows_per_dst=[3090, 3114, 2918, 2924, 2996, 2982, 2968, 3024] combine_rows_per_src=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64948 dup_rows=24084 dup_rows_remote=21232 dedup_ratio=0.244995 router_tokens_per_expert=[799, 776, 737, 761, 781, 774, 774, 775, 771, 747, 797, 748, 806, 758, 810, 728, 804, 776, 736, 783, 729, 774, 777, 747, 771, 803, 746, 735, 786, 744, 782, 739, 743, 749, 812, 736, 783, 740, 770, 754, 760, 779, 763, 780, 731, 778, 816, 758, 757, 761, 771, 748, 741, 795, 756, 733, 784, 758, 780, 837, 827, 742, 754, 782] payload_rows_per_dst=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] dup_rows_per_dst=[3064, 3052, 3066, 2868, 3000, 3074, 2852, 3108] combine_rows_per_src=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65072 dup_rows=24010 dup_rows_remote=21056 dedup_ratio=0.244242 router_tokens_per_expert=[784, 782, 790, 714, 746, 784, 788, 752, 734, 810, 774, 744, 731, 788, 808, 761, 776, 770, 806, 776, 769, 816, 746, 760, 750, 773, 750, 771, 751, 762, 774, 726, 763, 757, 761, 770, 728, 789, 777, 759, 764, 772, 771, 752, 758, 738, 764, 769, 756, 779, 781, 785, 785, 809, 782, 768, 775, 755, 757, 765, 807, 795, 737, 758] payload_rows_per_dst=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] dup_rows_per_dst=[2930, 2996, 2994, 2982, 2974, 2954, 3174, 3006] combine_rows_per_src=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=345152 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266413120 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532481088 weight_movement_bytes=34603008 total_nvlink_bytes=567084096 weight_to_token_ratio=0.064984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=341904 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265074576 ingress_bytes=299335680 link_bytes=299335680 token_bytes=529807248 weight_movement_bytes=34603008 total_nvlink_bytes=564410256 weight_to_token_ratio=0.065312 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126376065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65090 dup_rows=23944 dup_rows_remote=20876 dedup_ratio=0.243571 router_tokens_per_expert=[720, 777, 809, 789, 769, 845, 781, 767, 769, 763, 740, 807, 821, 705, 786, 727, 709, 754, 766, 769, 770, 762, 768, 771, 767, 785, 787, 802, 740, 779, 750, 734, 717, 767, 807, 745, 771, 757, 741, 832, 770, 784, 744, 785, 797, 806, 747, 771, 772, 756, 760, 754, 776, 726, 780, 730, 763, 786, 728, 780, 752, 753, 792, 815] payload_rows_per_dst=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] dup_rows_per_dst=[2852, 3028, 2878, 3194, 3008, 2990, 2926, 3068] combine_rows_per_src=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344424 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617192 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532889960 weight_movement_bytes=103809024 total_nvlink_bytes=636698984 weight_to_token_ratio=0.194804 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344720 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371728 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398736 weight_movement_bytes=34603008 total_nvlink_bytes=567001744 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64720 dup_rows=24288 dup_rows_remote=21258 dedup_ratio=0.247070 router_tokens_per_expert=[831, 814, 811, 755, 796, 753, 756, 810, 710, 753, 765, 784, 717, 744, 764, 803, 792, 723, 770, 732, 743, 768, 721, 797, 820, 789, 787, 765, 752, 723, 770, 757, 737, 745, 708, 795, 790, 777, 788, 752, 758, 769, 751, 763, 743, 777, 789, 807, 733, 739, 769, 814, 753, 785, 807, 774, 771, 724, 770, 763, 797, 782, 771, 776] payload_rows_per_dst=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] dup_rows_per_dst=[3118, 2978, 2902, 3030, 2986, 3114, 3052, 3108] combine_rows_per_src=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344512 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266879424 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533414336 weight_movement_bytes=34603008 total_nvlink_bytes=568017344 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266952504 ingress_bytes=405020672 link_bytes=405020672 token_bytes=533561144 weight_movement_bytes=138412032 total_nvlink_bytes=671973176 weight_to_token_ratio=0.259412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524060182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64780 dup_rows=24172 dup_rows_remote=21010 dedup_ratio=0.245890 router_tokens_per_expert=[785, 752, 788, 785, 746, 763, 769, 753, 769, 766, 793, 777, 716, 769, 755, 759, 781, 824, 779, 789, 760, 793, 768, 796, 798, 729, 752, 752, 779, 799, 720, 777, 766, 773, 790, 796, 767, 807, 790, 768, 748, 752, 793, 800, 765, 749, 731, 827, 746, 752, 767, 787, 794, 717, 774, 728, 791, 754, 763, 734, 747, 707, 731, 797] payload_rows_per_dst=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] dup_rows_per_dst=[3010, 2990, 3148, 2998, 3162, 2984, 2970, 2910] combine_rows_per_src=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65012 dup_rows=24016 dup_rows_remote=21048 dedup_ratio=0.244303 router_tokens_per_expert=[749, 778, 791, 804, 833, 788, 759, 794, 762, 754, 746, 781, 776, 775, 774, 754, 799, 730, 743, 766, 769, 776, 742, 760, 768, 835, 723, 742, 778, 750, 747, 774, 757, 780, 780, 774, 817, 805, 752, 774, 786, 790, 756, 719, 807, 780, 727, 783, 748, 756, 763, 753, 760, 738, 765, 759, 783, 747, 741, 759, 757, 808, 747, 761] payload_rows_per_dst=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] dup_rows_per_dst=[3156, 2968, 2912, 2982, 3092, 3048, 2864, 2994] combine_rows_per_src=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343912 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437032 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530152 weight_movement_bytes=34603008 total_nvlink_bytes=565133160 weight_to_token_ratio=0.065223 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[803, 741, 795, 788, 719, 772, 743, 732, 791, 810, 760, 737, 809, 787, 787, 736, 782, 789, 762, 756, 740, 788, 805, 753, 805, 774, 743, 800, 739, 768, 793, 831, 741, 719, 774, 745, 768, 717, 749, 802, 751, 794, 714, 776, 768, 790, 785, 728, 765, 783, 773, 713, 768, 789, 782, 769, 764, 744, 799, 735, 781, 785, 789, 754] payload_rows_per_dst=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] dup_rows_per_dst=[2982, 3110, 2894, 3170, 2970, 3020, 3034, 2956] combine_rows_per_src=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=343160 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265682040 ingress_bytes=299941888 link_bytes=299941888 token_bytes=531020920 weight_movement_bytes=34603008 total_nvlink_bytes=565623928 weight_to_token_ratio=0.065163 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128657177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64854 dup_rows=24278 dup_rows_remote=21228 dedup_ratio=0.246969 router_tokens_per_expert=[744, 727, 765, 751, 737, 769, 767, 743, 742, 738, 763, 721, 794, 792, 736, 808, 775, 767, 751, 782, 730, 771, 743, 792, 801, 789, 774, 727, 720, 768, 760, 807, 815, 765, 797, 805, 750, 749, 787, 747, 787, 770, 716, 776, 780, 775, 837, 759, 773, 758, 784, 749, 751, 794, 780, 740, 725, 770, 800, 772, 823, 792, 789, 783] payload_rows_per_dst=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] dup_rows_per_dst=[2930, 2964, 3050, 2998, 3128, 3110, 2994, 3104] combine_rows_per_src=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344240 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633392 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922544 weight_movement_bytes=34603008 total_nvlink_bytes=567525552 weight_to_token_ratio=0.064931 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65064 dup_rows=24072 dup_rows_remote=21094 dedup_ratio=0.244873 router_tokens_per_expert=[735, 725, 752, 796, 785, 767, 778, 766, 775, 775, 817, 846, 748, 755, 723, 775, 767, 781, 846, 761, 766, 783, 769, 798, 767, 786, 743, 766, 799, 807, 775, 753, 822, 749, 812, 785, 748, 760, 740, 756, 775, 732, 755, 764, 738, 778, 773, 758, 775, 719, 743, 777, 751, 766, 770, 787, 774, 752, 724, 739, 721, 729, 758, 837] payload_rows_per_dst=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] dup_rows_per_dst=[2962, 3090, 3104, 3012, 3014, 2978, 2944, 2968] combine_rows_per_src=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64998 dup_rows=24122 dup_rows_remote=21104 dedup_ratio=0.245382 router_tokens_per_expert=[779, 756, 785, 791, 795, 765, 757, 799, 794, 779, 758, 714, 780, 782, 792, 801, 796, 736, 808, 781, 733, 770, 793, 744, 800, 801, 786, 750, 775, 772, 772, 768, 769, 709, 753, 749, 772, 771, 764, 748, 745, 779, 749, 765, 820, 757, 762, 736, 745, 753, 766, 771, 774, 772, 768, 752, 734, 739, 787, 786, 770, 749, 766, 760] payload_rows_per_dst=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] dup_rows_per_dst=[3104, 3090, 3002, 3020, 2874, 2996, 3018, 3018] combine_rows_per_src=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64956 dup_rows=24114 dup_rows_remote=21056 dedup_ratio=0.245300 router_tokens_per_expert=[762, 812, 745, 812, 793, 783, 757, 749, 805, 789, 708, 762, 801, 719, 801, 772, 756, 752, 773, 781, 708, 747, 839, 760, 811, 766, 779, 775, 806, 782, 717, 755, 793, 800, 732, 796, 751, 819, 698, 798, 792, 764, 787, 740, 808, 786, 745, 755, 743, 756, 786, 766, 746, 764, 707, 741, 752, 740, 783, 776, 742, 778, 771, 760] payload_rows_per_dst=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] dup_rows_per_dst=[2932, 2992, 2872, 2858, 3138, 3154, 3110, 3058] combine_rows_per_src=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344160 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266117216 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531890272 weight_movement_bytes=69206016 total_nvlink_bytes=601096288 weight_to_token_ratio=0.130113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344328 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986312 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628296 weight_movement_bytes=34603008 total_nvlink_bytes=566231304 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344632 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846776 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348920 weight_movement_bytes=34603008 total_nvlink_bytes=567951928 weight_to_token_ratio=0.064879 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344408 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576216 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808024 weight_movement_bytes=34603008 total_nvlink_bytes=567411032 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=64886 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[746, 818, 749, 758, 778, 743, 747, 768, 820, 725, 761, 766, 742, 769, 784, 761, 751, 766, 800, 726, 770, 771, 748, 738, 786, 788, 747, 771, 775, 786, 812, 811, 775, 794, 780, 696, 727, 767, 747, 791, 732, 767, 741, 774, 847, 752, 807, 766, 727, 740, 788, 801, 789, 846, 693, 809, 767, 795, 771, 789, 743, 764, 751, 735] payload_rows_per_dst=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] dup_rows_per_dst=[2968, 3038, 2860, 3134, 2898, 3002, 3054, 3046] combine_rows_per_src=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344048 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403824 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463600 weight_movement_bytes=34603008 total_nvlink_bytes=567066608 weight_to_token_ratio=0.064987 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64972 dup_rows=24154 dup_rows_remote=21298 dedup_ratio=0.245707 router_tokens_per_expert=[792, 786, 811, 778, 749, 768, 733, 783, 761, 774, 754, 711, 747, 811, 796, 766, 774, 785, 740, 801, 714, 760, 816, 810, 778, 743, 780, 735, 777, 775, 783, 766, 768, 737, 765, 746, 734, 752, 758, 757, 730, 747, 750, 784, 782, 762, 769, 768, 772, 762, 747, 830, 817, 820, 763, 749, 774, 793, 757, 838, 738, 741, 757, 728] payload_rows_per_dst=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] dup_rows_per_dst=[3020, 3054, 3072, 3042, 2856, 2920, 3116, 3074] combine_rows_per_src=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64922 dup_rows=24016 dup_rows_remote=20900 dedup_ratio=0.244303 router_tokens_per_expert=[774, 818, 745, 779, 767, 764, 815, 774, 731, 789, 781, 801, 705, 828, 808, 798, 724, 777, 709, 759, 704, 731, 791, 731, 765, 759, 747, 791, 812, 776, 787, 741, 766, 812, 766, 767, 719, 758, 824, 757, 759, 759, 770, 791, 731, 775, 724, 767, 794, 787, 747, 778, 810, 794, 792, 765, 762, 756, 760, 744, 721, 782, 764, 770] payload_rows_per_dst=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] dup_rows_per_dst=[3100, 3116, 2752, 3066, 3054, 2966, 3052, 2910] combine_rows_per_src=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343008 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116064 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889120 weight_movement_bytes=138412032 total_nvlink_bytes=670301152 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=345080 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266470392 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595704 weight_movement_bytes=34603008 total_nvlink_bytes=567198712 weight_to_token_ratio=0.064970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65274 dup_rows=23798 dup_rows_remote=20792 dedup_ratio=0.242086 router_tokens_per_expert=[752, 770, 768, 723, 780, 789, 757, 780, 757, 776, 770, 784, 773, 739, 765, 750, 761, 791, 736, 750, 795, 795, 733, 773, 753, 735, 776, 785, 749, 742, 778, 788, 765, 783, 813, 771, 750, 763, 760, 802, 811, 769, 732, 790, 756, 713, 733, 745, 731, 782, 742, 771, 727, 807, 795, 785, 775, 837, 785, 733, 803, 742, 824, 784] payload_rows_per_dst=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] dup_rows_per_dst=[3006, 2958, 2968, 2962, 2980, 2916, 2878, 3130] combine_rows_per_src=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64926 dup_rows=24086 dup_rows_remote=21140 dedup_ratio=0.245015 router_tokens_per_expert=[765, 782, 827, 775, 780, 721, 778, 771, 749, 745, 801, 773, 781, 752, 737, 829, 766, 772, 731, 732, 755, 804, 790, 769, 762, 769, 773, 765, 827, 808, 749, 833, 780, 765, 787, 730, 767, 785, 720, 762, 782, 786, 717, 763, 791, 803, 743, 772, 797, 790, 763, 763, 794, 697, 746, 720, 786, 752, 794, 754, 723, 729, 746, 774] payload_rows_per_dst=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] dup_rows_per_dst=[3048, 3022, 2946, 3186, 3014, 3024, 2962, 2884] combine_rows_per_src=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65322 dup_rows=23886 dup_rows_remote=21082 dedup_ratio=0.242981 router_tokens_per_expert=[762, 761, 815, 802, 788, 773, 778, 720, 804, 797, 768, 796, 759, 771, 748, 752, 818, 780, 735, 800, 733, 753, 758, 751, 789, 838, 781, 773, 809, 771, 755, 774, 790, 776, 743, 717, 685, 778, 805, 741, 766, 700, 738, 746, 747, 779, 752, 722, 796, 768, 794, 748, 743, 781, 785, 758, 757, 790, 795, 762, 756, 801, 741, 780] payload_rows_per_dst=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] dup_rows_per_dst=[2982, 3032, 2902, 3138, 2878, 2804, 3046, 3104] combine_rows_per_src=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343288 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266263800 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532184312 weight_movement_bytes=34603008 total_nvlink_bytes=566787320 weight_to_token_ratio=0.065021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64992 dup_rows=23972 dup_rows_remote=20958 dedup_ratio=0.243856 router_tokens_per_expert=[743, 737, 717, 801, 774, 760, 739, 773, 750, 754, 772, 793, 795, 783, 742, 764, 787, 744, 806, 739, 778, 758, 723, 761, 766, 755, 724, 759, 792, 787, 814, 764, 772, 753, 753, 725, 792, 796, 801, 754, 785, 765, 820, 789, 771, 806, 744, 802, 786, 794, 769, 720, 754, 781, 790, 783, 791, 736, 761, 741, 751, 816, 745, 752] payload_rows_per_dst=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] dup_rows_per_dst=[2876, 3004, 2972, 2984, 2966, 3168, 3014, 2988] combine_rows_per_src=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64516 dup_rows=24148 dup_rows_remote=20772 dedup_ratio=0.245646 router_tokens_per_expert=[773, 746, 763, 833, 759, 772, 768, 814, 726, 750, 728, 769, 781, 764, 738, 784, 774, 775, 779, 738, 768, 798, 762, 800, 788, 762, 794, 790, 797, 779, 761, 760, 777, 750, 746, 779, 773, 729, 786, 759, 760, 797, 734, 761, 746, 736, 811, 804, 757, 776, 747, 774, 709, 762, 793, 714, 772, 784, 763, 767, 773, 725, 797, 798] payload_rows_per_dst=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] dup_rows_per_dst=[2682, 3008, 3074, 2668, 3184, 3054, 3102, 3376] combine_rows_per_src=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=344264 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266281160 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532218056 weight_movement_bytes=34603008 total_nvlink_bytes=566821064 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344264 combine_payload_bytes=267362304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267706568 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068872 weight_movement_bytes=69206016 total_nvlink_bytes=604274888 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=345616 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267904528 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535463440 weight_movement_bytes=34603008 total_nvlink_bytes=570066448 weight_to_token_ratio=0.064623 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343800 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551032 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758264 weight_movement_bytes=34603008 total_nvlink_bytes=567361272 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65050 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[820, 815, 759, 767, 783, 770, 742, 763, 749, 771, 782, 788, 748, 766, 750, 753, 803, 794, 778, 790, 739, 797, 745, 780, 826, 729, 765, 793, 749, 774, 792, 781, 756, 753, 759, 767, 742, 780, 758, 776, 785, 743, 772, 790, 805, 756, 722, 769, 706, 758, 799, 770, 759, 766, 730, 769, 779, 832, 785, 751, 740, 731, 736, 747] payload_rows_per_dst=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] dup_rows_per_dst=[3044, 3040, 2994, 3058, 2984, 3044, 2944, 2948] combine_rows_per_src=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64964 dup_rows=23944 dup_rows_remote=20770 dedup_ratio=0.243571 router_tokens_per_expert=[757, 829, 786, 741, 815, 775, 773, 769, 760, 771, 811, 747, 748, 744, 730, 764, 811, 681, 747, 783, 836, 770, 817, 753, 739, 764, 770, 793, 758, 830, 833, 798, 738, 741, 711, 794, 814, 747, 726, 819, 798, 756, 787, 763, 749, 787, 808, 782, 746, 729, 746, 701, 786, 709, 827, 777, 759, 745, 731, 716, 761, 771, 764, 761] payload_rows_per_dst=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] dup_rows_per_dst=[3080, 2886, 3058, 3174, 2886, 3090, 2904, 2866] combine_rows_per_src=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=341152 combine_payload_bytes=264257536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264598688 ingress_bytes=298860544 link_bytes=298860544 token_bytes=528856224 weight_movement_bytes=34603008 total_nvlink_bytes=563459232 weight_to_token_ratio=0.065430 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124588167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65172 dup_rows=23968 dup_rows_remote=21128 dedup_ratio=0.243815 router_tokens_per_expert=[758, 741, 789, 792, 742, 834, 750, 778, 775, 765, 753, 752, 738, 723, 772, 724, 755, 819, 774, 738, 799, 735, 767, 762, 820, 825, 750, 782, 769, 806, 741, 741, 747, 788, 781, 759, 765, 792, 750, 712, 765, 797, 762, 778, 774, 767, 727, 765, 758, 749, 763, 790, 806, 799, 802, 769, 726, 785, 755, 775, 735, 828, 722, 792] payload_rows_per_dst=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] dup_rows_per_dst=[3082, 2840, 2994, 3098, 2894, 2910, 3104, 3046] combine_rows_per_src=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344488 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789288 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234088 weight_movement_bytes=34603008 total_nvlink_bytes=567837096 weight_to_token_ratio=0.064893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266435480 ingress_bytes=404504576 link_bytes=404504576 token_bytes=532528024 weight_movement_bytes=138412032 total_nvlink_bytes=670940056 weight_to_token_ratio=0.259915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65020 dup_rows=23962 dup_rows_remote=21076 dedup_ratio=0.243754 router_tokens_per_expert=[746, 829, 805, 754, 778, 784, 770, 746, 779, 756, 766, 774, 762, 727, 769, 756, 753, 761, 779, 721, 788, 759, 764, 779, 786, 760, 750, 800, 779, 792, 788, 782, 780, 779, 760, 772, 780, 758, 741, 723, 731, 752, 808, 753, 761, 736, 828, 761, 765, 728, 804, 761, 756, 757, 801, 759, 791, 777, 781, 743, 721, 754, 807, 782] payload_rows_per_dst=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] dup_rows_per_dst=[3036, 2952, 2886, 3100, 2978, 2958, 3072, 2980] combine_rows_per_src=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=345200 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289712 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534234224 weight_movement_bytes=34603008 total_nvlink_bytes=568837232 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65130 dup_rows=23840 dup_rows_remote=20814 dedup_ratio=0.242513 router_tokens_per_expert=[783, 770, 803, 751, 779, 742, 783, 769, 771, 728, 786, 755, 765, 785, 749, 779, 760, 757, 788, 746, 768, 847, 763, 809, 782, 719, 774, 807, 765, 753, 728, 759, 723, 700, 775, 801, 801, 768, 757, 807, 751, 769, 747, 776, 728, 794, 779, 752, 780, 785, 755, 757, 763, 747, 797, 774, 777, 777, 760, 806, 783, 739, 758, 743] payload_rows_per_dst=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] dup_rows_per_dst=[3026, 3002, 3088, 2942, 3006, 2884, 2932, 2960] combine_rows_per_src=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64712 dup_rows=24188 dup_rows_remote=21164 dedup_ratio=0.246053 router_tokens_per_expert=[758, 789, 776, 729, 760, 751, 747, 740, 793, 767, 770, 783, 797, 765, 701, 815, 759, 769, 776, 736, 752, 749, 736, 774, 773, 776, 783, 779, 757, 748, 782, 763, 783, 795, 772, 781, 734, 760, 727, 746, 738, 805, 738, 780, 796, 792, 792, 773, 765, 765, 746, 799, 758, 769, 749, 795, 805, 718, 788, 781, 797, 766, 808, 778] payload_rows_per_dst=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] dup_rows_per_dst=[2968, 3102, 2946, 3028, 2992, 3024, 2998, 3130] combine_rows_per_src=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64904 dup_rows=24124 dup_rows_remote=21170 dedup_ratio=0.245402 router_tokens_per_expert=[760, 743, 755, 798, 803, 775, 765, 802, 781, 785, 775, 756, 774, 755, 768, 757, 793, 776, 785, 773, 761, 786, 813, 749, 790, 729, 767, 813, 711, 758, 764, 789, 757, 773, 783, 754, 730, 756, 732, 769, 779, 744, 777, 750, 778, 787, 813, 769, 812, 764, 797, 775, 756, 730, 756, 725, 766, 766, 705, 710, 842, 748, 762, 778] payload_rows_per_dst=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] dup_rows_per_dst=[3030, 2966, 3160, 3044, 2982, 3000, 2954, 2988] combine_rows_per_src=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344384 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666304 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532988224 weight_movement_bytes=34603008 total_nvlink_bytes=567591232 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64820 dup_rows=24192 dup_rows_remote=21140 dedup_ratio=0.246094 router_tokens_per_expert=[736, 723, 764, 753, 779, 811, 730, 740, 754, 777, 790, 808, 739, 727, 776, 726, 754, 782, 772, 814, 736, 744, 756, 771, 749, 769, 735, 830, 767, 782, 756, 772, 785, 766, 785, 780, 758, 788, 788, 763, 746, 780, 742, 752, 816, 772, 748, 790, 820, 802, 765, 813, 722, 772, 742, 765, 791, 805, 755, 775, 748, 747, 785, 764] payload_rows_per_dst=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] dup_rows_per_dst=[2842, 3068, 3032, 2852, 3136, 3014, 3196, 3052] combine_rows_per_src=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343776 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267116256 ingress_bytes=405184512 link_bytes=405184512 token_bytes=533888736 weight_movement_bytes=138412032 total_nvlink_bytes=672300768 weight_to_token_ratio=0.259253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524676699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343504 combine_payload_bytes=265060352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265403856 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530464208 weight_movement_bytes=138412032 total_nvlink_bytes=668876240 weight_to_token_ratio=0.260926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344296 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191080 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037864 weight_movement_bytes=34603008 total_nvlink_bytes=566640872 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64734 dup_rows=24168 dup_rows_remote=21074 dedup_ratio=0.245850 router_tokens_per_expert=[762, 767, 804, 773, 773, 814, 750, 720, 747, 801, 816, 776, 755, 745, 750, 790, 754, 745, 794, 765, 719, 772, 766, 750, 774, 784, 806, 746, 806, 772, 770, 767, 799, 802, 758, 822, 754, 792, 773, 748, 783, 777, 747, 792, 746, 715, 783, 773, 744, 777, 733, 739, 724, 751, 820, 734, 709, 745, 790, 795, 767, 772, 755, 800] payload_rows_per_dst=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] dup_rows_per_dst=[3086, 3032, 2912, 3084, 3094, 2950, 3002, 3008] combine_rows_per_src=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=65052 dup_rows=24128 dup_rows_remote=21160 dedup_ratio=0.245443 router_tokens_per_expert=[775, 751, 770, 774, 843, 756, 776, 793, 787, 762, 806, 790, 784, 741, 814, 734, 776, 747, 768, 786, 767, 761, 737, 759, 760, 710, 773, 785, 737, 750, 780, 751, 813, 806, 745, 724, 751, 741, 721, 782, 754, 789, 806, 755, 744, 770, 757, 764, 798, 760, 739, 753, 785, 779, 794, 757, 775, 780, 794, 761, 780, 761, 765, 746] payload_rows_per_dst=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] dup_rows_per_dst=[3182, 3088, 2986, 2968, 2948, 2984, 2996, 2976] combine_rows_per_src=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343840 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846560 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349280 weight_movement_bytes=69206016 total_nvlink_bytes=600555296 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64924 dup_rows=24018 dup_rows_remote=20978 dedup_ratio=0.244324 router_tokens_per_expert=[776, 737, 760, 741, 739, 745, 773, 799, 800, 749, 755, 787, 782, 761, 766, 801, 749, 747, 796, 768, 822, 739, 780, 740, 779, 775, 765, 800, 765, 790, 758, 842, 768, 729, 785, 776, 754, 772, 767, 767, 804, 765, 793, 778, 776, 840, 767, 732, 717, 736, 800, 685, 773, 773, 713, 767, 780, 770, 766, 768, 760, 749, 801, 735] payload_rows_per_dst=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] dup_rows_per_dst=[2996, 3040, 3044, 3050, 2950, 3120, 2844, 2974] combine_rows_per_src=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343232 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493696 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530644160 weight_movement_bytes=34603008 total_nvlink_bytes=565247168 weight_to_token_ratio=0.065209 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64818 dup_rows=24056 dup_rows_remote=20916 dedup_ratio=0.244710 router_tokens_per_expert=[751, 739, 774, 765, 711, 743, 765, 790, 743, 771, 795, 779, 811, 767, 754, 732, 782, 813, 745, 806, 784, 780, 810, 765, 790, 779, 793, 763, 751, 797, 778, 743, 742, 767, 746, 757, 738, 791, 783, 748, 821, 799, 778, 777, 749, 747, 764, 824, 735, 737, 758, 732, 721, 772, 738, 786, 771, 761, 733, 782, 779, 782, 775, 790] payload_rows_per_dst=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] dup_rows_per_dst=[2900, 3046, 3140, 3060, 2990, 3174, 2752, 2994] combine_rows_per_src=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344848 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797840 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250832 weight_movement_bytes=69206016 total_nvlink_bytes=602456848 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64948 dup_rows=23940 dup_rows_remote=20878 dedup_ratio=0.243530 router_tokens_per_expert=[758, 746, 716, 742, 780, 713, 750, 792, 792, 791, 771, 777, 775, 768, 786, 757, 740, 798, 758, 765, 717, 787, 756, 766, 742, 780, 796, 745, 752, 726, 739, 774, 740, 782, 760, 769, 809, 738, 843, 740, 780, 778, 762, 768, 755, 814, 810, 772, 712, 781, 780, 782, 793, 798, 810, 816, 704, 776, 774, 766, 743, 754, 809, 779] payload_rows_per_dst=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] dup_rows_per_dst=[2860, 3082, 2954, 2912, 2980, 3062, 3136, 2954] combine_rows_per_src=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64912 dup_rows=24186 dup_rows_remote=21122 dedup_ratio=0.246033 router_tokens_per_expert=[738, 775, 783, 774, 789, 758, 773, 735, 742, 790, 796, 774, 758, 771, 759, 792, 785, 759, 785, 739, 805, 758, 781, 810, 756, 786, 751, 732, 727, 809, 712, 817, 749, 756, 768, 749, 769, 775, 762, 815, 780, 766, 769, 766, 784, 773, 773, 775, 769, 756, 753, 724, 791, 716, 801, 759, 799, 737, 775, 723, 749, 829, 743, 780] payload_rows_per_dst=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] dup_rows_per_dst=[3064, 3102, 3232, 2982, 2982, 2956, 2900, 2968] combine_rows_per_src=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343608 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272312 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201016 weight_movement_bytes=69206016 total_nvlink_bytes=601407032 weight_to_token_ratio=0.130037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65262 dup_rows=23836 dup_rows_remote=20938 dedup_ratio=0.242472 router_tokens_per_expert=[808, 810, 810, 745, 771, 721, 721, 752, 760, 804, 745, 758, 788, 800, 755, 754, 757, 750, 769, 763, 770, 826, 795, 741, 767, 823, 737, 756, 755, 810, 731, 721, 766, 780, 717, 759, 693, 788, 821, 750, 780, 791, 797, 791, 776, 727, 771, 797, 746, 715, 743, 748, 780, 735, 804, 781, 781, 759, 764, 791, 757, 777, 776, 818] payload_rows_per_dst=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] dup_rows_per_dst=[3032, 3018, 3014, 2870, 2920, 3084, 2898, 3000] combine_rows_per_src=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64618 dup_rows=24284 dup_rows_remote=21240 dedup_ratio=0.247030 router_tokens_per_expert=[800, 726, 758, 704, 769, 743, 754, 716, 824, 754, 745, 756, 785, 796, 772, 777, 756, 716, 800, 803, 753, 811, 741, 736, 790, 706, 755, 729, 786, 820, 834, 769, 739, 766, 795, 806, 772, 755, 799, 780, 732, 794, 726, 801, 768, 789, 785, 767, 750, 786, 781, 783, 774, 782, 712, 763, 775, 734, 772, 787, 754, 783, 792, 766] payload_rows_per_dst=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] dup_rows_per_dst=[2818, 2778, 3050, 2890, 3282, 3080, 3342, 3044] combine_rows_per_src=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342936 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265837464 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531331992 weight_movement_bytes=69206016 total_nvlink_bytes=600538008 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65184 dup_rows=23926 dup_rows_remote=20918 dedup_ratio=0.243388 router_tokens_per_expert=[791, 738, 773, 755, 770, 738, 744, 743, 808, 736, 809, 789, 770, 761, 763, 781, 768, 745, 755, 770, 763, 758, 788, 793, 776, 816, 769, 782, 792, 766, 735, 778, 765, 760, 768, 749, 800, 786, 723, 750, 813, 728, 814, 790, 719, 738, 751, 777, 806, 798, 809, 767, 754, 772, 709, 734, 758, 773, 788, 771, 757, 751, 769, 782] payload_rows_per_dst=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] dup_rows_per_dst=[2888, 3106, 3006, 3078, 3008, 2952, 2928, 2960] combine_rows_per_src=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343304 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266370312 ingress_bytes=335233024 link_bytes=335233024 token_bytes=532397320 weight_movement_bytes=69206016 total_nvlink_bytes=601603336 weight_to_token_ratio=0.129989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261454881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344136 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223688 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103240 weight_movement_bytes=69206016 total_nvlink_bytes=601309256 weight_to_token_ratio=0.130061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344800 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657952 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534971104 weight_movement_bytes=34603008 total_nvlink_bytes=569574112 weight_to_token_ratio=0.064682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64800 dup_rows=24194 dup_rows_remote=21206 dedup_ratio=0.246114 router_tokens_per_expert=[744, 747, 744, 779, 762, 786, 731, 802, 707, 720, 829, 774, 798, 776, 797, 730, 776, 742, 743, 812, 768, 780, 798, 770, 805, 785, 757, 757, 744, 782, 739, 780, 752, 752, 789, 766, 782, 766, 771, 785, 775, 809, 767, 749, 797, 785, 758, 745, 765, 757, 801, 767, 770, 780, 762, 746, 768, 768, 739, 771, 732, 769, 768, 777] payload_rows_per_dst=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] dup_rows_per_dst=[2984, 3064, 3004, 2988, 3090, 3066, 3086, 2912] combine_rows_per_src=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343432 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018760 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694088 weight_movement_bytes=69206016 total_nvlink_bytes=598900104 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65078 dup_rows=24000 dup_rows_remote=21128 dedup_ratio=0.244141 router_tokens_per_expert=[751, 753, 764, 754, 831, 789, 789, 809, 763, 755, 736, 759, 791, 774, 731, 740, 755, 803, 724, 811, 750, 727, 770, 759, 778, 764, 790, 783, 759, 755, 737, 820, 802, 732, 774, 731, 751, 779, 724, 759, 762, 757, 727, 797, 797, 808, 755, 786, 753, 767, 776, 776, 793, 795, 754, 803, 779, 738, 762, 754, 840, 730, 769, 748] payload_rows_per_dst=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] dup_rows_per_dst=[3086, 2872, 2982, 3046, 2882, 3024, 3130, 2978] combine_rows_per_src=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344408 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338072 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331736 weight_movement_bytes=34603008 total_nvlink_bytes=568934744 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65010 dup_rows=23988 dup_rows_remote=20906 dedup_ratio=0.244019 router_tokens_per_expert=[789, 725, 764, 755, 764, 743, 815, 742, 764, 774, 783, 779, 710, 795, 748, 815, 728, 776, 750, 776, 812, 765, 795, 792, 785, 717, 756, 793, 793, 734, 758, 790, 777, 786, 789, 803, 762, 764, 773, 751, 762, 763, 741, 745, 720, 775, 762, 735, 767, 734, 791, 785, 802, 745, 765, 809, 786, 773, 769, 734, 794, 813, 772, 720] payload_rows_per_dst=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] dup_rows_per_dst=[2978, 3094, 3082, 2932, 3040, 2808, 3004, 3050] combine_rows_per_src=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344024 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764824 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185624 weight_movement_bytes=103809024 total_nvlink_bytes=634994648 weight_to_token_ratio=0.195429 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65044 dup_rows=24054 dup_rows_remote=21062 dedup_ratio=0.244690 router_tokens_per_expert=[738, 788, 781, 755, 793, 760, 765, 762, 785, 785, 760, 722, 767, 812, 736, 755, 764, 781, 774, 758, 805, 781, 766, 742, 773, 764, 776, 757, 811, 754, 739, 732, 822, 776, 781, 787, 782, 753, 829, 805, 731, 754, 762, 729, 776, 756, 803, 788, 735, 754, 818, 791, 746, 742, 738, 805, 749, 751, 696, 775, 787, 767, 762, 761] payload_rows_per_dst=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] dup_rows_per_dst=[3018, 3058, 3038, 2968, 3196, 2992, 2918, 2866] combine_rows_per_src=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344824 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266904312 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463800 weight_movement_bytes=34603008 total_nvlink_bytes=568066808 weight_to_token_ratio=0.064865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64216 dup_rows=24148 dup_rows_remote=20504 dedup_ratio=0.245646 router_tokens_per_expert=[800, 796, 774, 775, 806, 767, 724, 798, 772, 739, 733, 766, 750, 743, 747, 838, 745, 806, 689, 742, 774, 801, 742, 739, 724, 748, 818, 796, 792, 748, 788, 767, 756, 754, 721, 761, 784, 774, 752, 785, 781, 766, 790, 788, 766, 797, 743, 799, 729, 763, 775, 789, 712, 747, 798, 789, 773, 792, 802, 734, 737, 743, 802, 803] payload_rows_per_dst=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] dup_rows_per_dst=[3644, 3068, 2862, 3058, 3002, 2612, 2988, 2914] combine_rows_per_src=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343664 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266624624 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532905584 weight_movement_bytes=34603008 total_nvlink_bytes=567508592 weight_to_token_ratio=0.064933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=65086 dup_rows=24094 dup_rows_remote=21080 dedup_ratio=0.245097 router_tokens_per_expert=[797, 787, 808, 790, 792, 755, 793, 751, 766, 747, 739, 775, 795, 750, 814, 750, 747, 769, 782, 748, 729, 717, 753, 767, 758, 783, 806, 763, 763, 763, 757, 787, 804, 819, 784, 784, 807, 753, 725, 781, 737, 776, 764, 771, 699, 772, 767, 820, 750, 820, 719, 742, 769, 759, 749, 761, 777, 808, 781, 745, 728, 772, 753, 755] payload_rows_per_dst=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] dup_rows_per_dst=[3046, 3038, 2850, 3060, 3096, 2980, 3014, 3010] combine_rows_per_src=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64776 dup_rows=24176 dup_rows_remote=21036 dedup_ratio=0.245931 router_tokens_per_expert=[799, 783, 796, 757, 818, 763, 775, 709, 754, 771, 691, 757, 803, 777, 708, 775, 770, 754, 751, 791, 760, 756, 758, 779, 803, 772, 806, 732, 711, 755, 784, 804, 727, 811, 797, 750, 790, 777, 789, 757, 789, 780, 764, 810, 753, 783, 757, 757, 791, 747, 757, 796, 753, 791, 739, 741, 726, 840, 720, 756, 750, 765, 830, 737] payload_rows_per_dst=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] dup_rows_per_dst=[2898, 2896, 2988, 3098, 2860, 3124, 3172, 3140] combine_rows_per_src=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344424 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266764648 ingress_bytes=370229248 link_bytes=370229248 token_bytes=533184872 weight_movement_bytes=103809024 total_nvlink_bytes=636993896 weight_to_token_ratio=0.194696 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393142854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65052 dup_rows=24052 dup_rows_remote=21072 dedup_ratio=0.244670 router_tokens_per_expert=[770, 783, 761, 785, 705, 753, 719, 739, 735, 810, 752, 741, 814, 848, 706, 746, 751, 785, 742, 799, 741, 778, 774, 748, 789, 805, 750, 775, 807, 759, 799, 762, 779, 756, 728, 766, 808, 784, 758, 711, 775, 791, 805, 766, 821, 782, 792, 788, 797, 746, 751, 751, 782, 780, 736, 710, 746, 794, 735, 764, 785, 818, 768, 748] payload_rows_per_dst=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] dup_rows_per_dst=[2954, 2946, 3014, 3052, 2980, 3174, 2892, 3040] combine_rows_per_src=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=338880 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263367616 ingress_bytes=297631744 link_bytes=297631744 token_bytes=526396352 weight_movement_bytes=34603008 total_nvlink_bytes=560999360 weight_to_token_ratio=0.065736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119964291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65072 dup_rows=23968 dup_rows_remote=20988 dedup_ratio=0.243815 router_tokens_per_expert=[751, 809, 704, 765, 781, 768, 788, 746, 860, 775, 762, 750, 767, 780, 779, 832, 790, 747, 748, 782, 770, 795, 801, 756, 788, 790, 759, 748, 742, 766, 783, 746, 775, 755, 774, 768, 715, 730, 780, 732, 822, 778, 784, 771, 722, 795, 771, 798, 723, 778, 800, 749, 756, 751, 735, 748, 769, 765, 730, 758, 792, 789, 767, 744] payload_rows_per_dst=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] dup_rows_per_dst=[3014, 3206, 3070, 2980, 2876, 2996, 2888, 2938] combine_rows_per_src=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529176 weight_movement_bytes=34603008 total_nvlink_bytes=568132184 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343248 combine_payload_bytes=265322496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265665744 ingress_bytes=299925504 link_bytes=299925504 token_bytes=530988240 weight_movement_bytes=34603008 total_nvlink_bytes=565591248 weight_to_token_ratio=0.065167 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128595526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65116 dup_rows=23890 dup_rows_remote=20800 dedup_ratio=0.243022 router_tokens_per_expert=[835, 709, 726, 769, 801, 762, 750, 769, 811, 768, 750, 776, 753, 770, 745, 821, 853, 736, 826, 830, 783, 809, 743, 729, 767, 771, 754, 801, 740, 769, 788, 762, 767, 739, 723, 757, 732, 763, 734, 785, 730, 758, 810, 763, 771, 753, 787, 781, 774, 796, 784, 765, 776, 807, 781, 798, 716, 766, 758, 766, 754, 739, 725, 718] payload_rows_per_dst=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] dup_rows_per_dst=[2912, 3090, 3108, 2990, 2866, 3008, 3110, 2806] combine_rows_per_src=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344496 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797488 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250480 weight_movement_bytes=69206016 total_nvlink_bytes=602456496 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64890 dup_rows=24052 dup_rows_remote=20950 dedup_ratio=0.244670 router_tokens_per_expert=[755, 725, 804, 778, 755, 746, 799, 754, 789, 782, 752, 717, 764, 747, 829, 782, 764, 778, 786, 793, 779, 797, 744, 791, 765, 746, 742, 736, 752, 762, 763, 733, 773, 749, 760, 758, 795, 743, 776, 749, 762, 749, 764, 843, 762, 761, 792, 790, 735, 776, 791, 766, 782, 784, 786, 753, 759, 783, 712, 758, 792, 788, 793, 759] payload_rows_per_dst=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] dup_rows_per_dst=[2982, 2992, 3102, 2874, 2990, 3140, 2930, 3042] combine_rows_per_src=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344240 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879152 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414064 weight_movement_bytes=69206016 total_nvlink_bytes=602620080 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=65432 dup_rows=23574 dup_rows_remote=20612 dedup_ratio=0.239807 router_tokens_per_expert=[782, 731, 772, 839, 754, 728, 791, 739, 762, 773, 773, 746, 731, 766, 747, 739, 783, 795, 759, 777, 773, 787, 810, 777, 767, 730, 775, 797, 771, 735, 765, 814, 729, 715, 751, 789, 764, 757, 789, 769, 757, 843, 760, 710, 761, 748, 781, 770, 784, 777, 719, 792, 774, 777, 804, 770, 791, 766, 779, 776, 747, 818, 746, 751] payload_rows_per_dst=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] dup_rows_per_dst=[2936, 2890, 3030, 2994, 2842, 2962, 3018, 2902] combine_rows_per_src=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343664 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058800 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533773936 weight_movement_bytes=34603008 total_nvlink_bytes=568376944 weight_to_token_ratio=0.064827 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64948 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[755, 782, 785, 766, 792, 786, 788, 757, 782, 743, 768, 708, 770, 771, 793, 768, 719, 773, 756, 752, 760, 752, 763, 810, 760, 773, 761, 781, 761, 736, 792, 802, 802, 772, 809, 724, 759, 782, 763, 777, 793, 735, 768, 776, 811, 753, 732, 741, 764, 777, 770, 765, 734, 761, 772, 761, 744, 736, 784, 749, 800, 797, 800, 776] payload_rows_per_dst=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] dup_rows_per_dst=[2966, 3090, 3102, 2838, 3082, 2858, 3034, 2926] combine_rows_per_src=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343360 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266132800 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922240 weight_movement_bytes=103809024 total_nvlink_bytes=635731264 weight_to_token_ratio=0.195158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=344176 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268353648 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536363120 weight_movement_bytes=34603008 total_nvlink_bytes=570966128 weight_to_token_ratio=0.064514 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64992 dup_rows=24068 dup_rows_remote=21018 dedup_ratio=0.244832 router_tokens_per_expert=[802, 757, 773, 764, 746, 738, 765, 776, 800, 764, 795, 755, 740, 789, 761, 770, 747, 789, 740, 759, 744, 798, 747, 771, 713, 745, 774, 775, 739, 722, 808, 756, 763, 843, 812, 788, 786, 793, 738, 806, 772, 692, 756, 824, 794, 788, 781, 707, 778, 774, 762, 743, 857, 717, 791, 725, 754, 780, 746, 792, 762, 781, 756, 769] payload_rows_per_dst=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] dup_rows_per_dst=[2982, 3030, 2932, 2854, 3196, 3000, 3050, 3024] combine_rows_per_src=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64670 dup_rows=24072 dup_rows_remote=20880 dedup_ratio=0.244873 router_tokens_per_expert=[748, 737, 799, 755, 781, 712, 772, 815, 796, 728, 763, 816, 772, 742, 736, 744, 786, 716, 746, 758, 778, 794, 750, 745, 725, 780, 761, 804, 720, 763, 802, 765, 787, 790, 737, 764, 767, 795, 781, 776, 755, 745, 783, 704, 764, 785, 765, 750, 802, 759, 768, 755, 798, 787, 754, 854, 768, 781, 800, 765, 764, 804, 780, 756] payload_rows_per_dst=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] dup_rows_per_dst=[2830, 3000, 2764, 2946, 3180, 2902, 3258, 3192] combine_rows_per_src=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64918 dup_rows=23952 dup_rows_remote=20970 dedup_ratio=0.243652 router_tokens_per_expert=[768, 782, 770, 794, 772, 768, 730, 710, 761, 770, 757, 763, 776, 786, 794, 752, 760, 753, 766, 765, 738, 786, 745, 761, 797, 733, 802, 798, 764, 820, 782, 781, 775, 835, 768, 728, 760, 782, 746, 814, 805, 763, 784, 790, 734, 729, 752, 747, 781, 789, 774, 757, 746, 719, 749, 746, 783, 803, 724, 796, 788, 774, 745, 762] payload_rows_per_dst=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] dup_rows_per_dst=[2932, 2994, 3000, 3114, 2982, 2934, 2986, 3010] combine_rows_per_src=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343512 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266370520 ingress_bytes=369836032 link_bytes=369836032 token_bytes=532397528 weight_movement_bytes=103809024 total_nvlink_bytes=636206552 weight_to_token_ratio=0.194984 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391663214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65404 dup_rows=23660 dup_rows_remote=20690 dedup_ratio=0.240682 router_tokens_per_expert=[796, 826, 778, 801, 756, 823, 809, 748, 727, 771, 767, 782, 818, 725, 764, 772, 799, 730, 749, 790, 778, 737, 761, 729, 769, 757, 757, 756, 755, 776, 776, 759, 731, 704, 823, 776, 792, 726, 752, 760, 755, 758, 769, 796, 805, 829, 734, 733, 804, 748, 782, 735, 747, 777, 780, 794, 747, 788, 729, 786, 713, 778, 773, 787] payload_rows_per_dst=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] dup_rows_per_dst=[3136, 2902, 2928, 2970, 2906, 3034, 2934, 2850] combine_rows_per_src=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344040 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266551272 ingress_bytes=370016256 link_bytes=370016256 token_bytes=532758504 weight_movement_bytes=103809024 total_nvlink_bytes=636567528 weight_to_token_ratio=0.194852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392341383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65064 dup_rows=24024 dup_rows_remote=21060 dedup_ratio=0.244385 router_tokens_per_expert=[775, 763, 715, 771, 791, 758, 761, 793, 769, 722, 802, 739, 793, 778, 731, 756, 754, 780, 801, 749, 774, 781, 752, 803, 791, 762, 786, 768, 719, 744, 745, 788, 798, 759, 788, 740, 779, 789, 763, 784, 752, 772, 766, 787, 742, 823, 778, 769, 738, 801, 740, 788, 782, 742, 771, 729, 781, 779, 794, 763, 759, 736, 793, 753] payload_rows_per_dst=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] dup_rows_per_dst=[2990, 2964, 3068, 3000, 3010, 3126, 2904, 2962] combine_rows_per_src=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=342200 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265230520 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530118840 weight_movement_bytes=34603008 total_nvlink_bytes=564721848 weight_to_token_ratio=0.065274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74000 payload_rows_remote=64806 dup_rows=24304 dup_rows_remote=21308 dedup_ratio=0.247233 router_tokens_per_expert=[747, 791, 759, 733, 799, 775, 741, 757, 804, 757, 765, 808, 791, 799, 745, 800, 791, 758, 725, 727, 767, 749, 793, 785, 786, 773, 748, 778, 771, 799, 786, 711, 766, 800, 772, 810, 790, 761, 769, 705, 772, 731, 761, 755, 826, 722, 760, 781, 822, 744, 761, 746, 802, 774, 793, 783, 768, 746, 739, 742, 721, 727, 798, 787] payload_rows_per_dst=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] dup_rows_per_dst=[3052, 3094, 2996, 3024, 3006, 3068, 3056, 3008] combine_rows_per_src=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343552 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247680 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151808 weight_movement_bytes=34603008 total_nvlink_bytes=566754816 weight_to_token_ratio=0.065025 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64900 dup_rows=24038 dup_rows_remote=20974 dedup_ratio=0.244527 router_tokens_per_expert=[771, 728, 789, 798, 795, 732, 782, 747, 794, 771, 742, 805, 751, 765, 793, 804, 774, 751, 763, 702, 762, 743, 786, 783, 759, 744, 777, 760, 806, 768, 772, 778, 767, 736, 766, 775, 731, 764, 738, 782, 768, 781, 775, 759, 764, 776, 793, 799, 761, 752, 731, 794, 811, 753, 779, 754, 781, 742, 796, 726, 767, 738, 807, 791] payload_rows_per_dst=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] dup_rows_per_dst=[2990, 3118, 2892, 3100, 2902, 3064, 3000, 2972] combine_rows_per_src=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267894784 dispatch_meta_bytes=344376 combine_payload_bytes=267894784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268239160 ingress_bytes=302497792 link_bytes=302497792 token_bytes=536133944 weight_movement_bytes=34603008 total_nvlink_bytes=570736952 weight_to_token_ratio=0.064542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138274838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344496 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266846640 ingress_bytes=335708160 link_bytes=335708160 token_bytes=533348784 weight_movement_bytes=69206016 total_nvlink_bytes=602554800 weight_to_token_ratio=0.129758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263242779e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=344456 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265789832 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531235208 weight_movement_bytes=34603008 total_nvlink_bytes=565838216 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64966 dup_rows=23966 dup_rows_remote=20812 dedup_ratio=0.243795 router_tokens_per_expert=[787, 743, 789, 791, 806, 754, 798, 779, 770, 773, 787, 786, 786, 769, 743, 755, 765, 775, 772, 766, 760, 819, 832, 746, 731, 764, 713, 761, 773, 788, 750, 738, 749, 767, 764, 758, 711, 762, 790, 766, 798, 748, 771, 755, 769, 770, 776, 775, 711, 784, 767, 775, 802, 759, 798, 759, 736, 725, 774, 814, 747, 796, 761, 746] payload_rows_per_dst=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] dup_rows_per_dst=[3154, 2972, 3014, 3112, 2948, 3040, 2818, 2908] combine_rows_per_src=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343496 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173896 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004296 weight_movement_bytes=69206016 total_nvlink_bytes=601210312 weight_to_token_ratio=0.130085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64950 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[745, 783, 765, 743, 751, 780, 756, 741, 729, 799, 738, 724, 805, 783, 799, 756, 807, 766, 703, 802, 734, 753, 732, 739, 776, 744, 761, 792, 761, 738, 784, 808, 863, 764, 792, 818, 786, 766, 765, 765, 741, 737, 752, 778, 762, 749, 774, 768, 823, 805, 754, 771, 759, 731, 786, 783, 789, 757, 754, 762, 744, 752, 826, 779] payload_rows_per_dst=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] dup_rows_per_dst=[2900, 2974, 2910, 3070, 3218, 2890, 3068, 2968] combine_rows_per_src=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64906 dup_rows=23936 dup_rows_remote=20892 dedup_ratio=0.243490 router_tokens_per_expert=[790, 786, 764, 752, 826, 769, 717, 744, 812, 750, 727, 764, 741, 759, 788, 751, 765, 796, 752, 741, 778, 762, 781, 768, 742, 765, 790, 786, 745, 783, 736, 767, 771, 727, 752, 804, 735, 756, 756, 750, 807, 754, 762, 781, 778, 783, 776, 790, 803, 778, 744, 762, 806, 751, 812, 788, 780, 764, 706, 754, 788, 776, 796, 765] payload_rows_per_dst=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] dup_rows_per_dst=[3032, 2538, 3486, 2906, 2850, 3130, 2950, 3044] combine_rows_per_src=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64964 dup_rows=24138 dup_rows_remote=21270 dedup_ratio=0.245544 router_tokens_per_expert=[766, 815, 741, 750, 801, 814, 721, 769, 753, 761, 776, 786, 792, 793, 815, 779, 779, 774, 782, 801, 789, 690, 746, 742, 779, 797, 747, 771, 765, 783, 770, 744, 715, 786, 741, 691, 782, 727, 803, 790, 754, 721, 786, 792, 745, 789, 775, 824, 810, 806, 751, 814, 745, 733, 733, 764, 757, 759, 767, 733, 792, 795, 734, 747] payload_rows_per_dst=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] dup_rows_per_dst=[3130, 3082, 2958, 3044, 2868, 3054, 3022, 2980] combine_rows_per_src=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343112 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266443848 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532544584 weight_movement_bytes=103809024 total_nvlink_bytes=636353608 weight_to_token_ratio=0.194930 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343520 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266378720 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532413920 weight_movement_bytes=138412032 total_nvlink_bytes=670825952 weight_to_token_ratio=0.259971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64794 dup_rows=24226 dup_rows_remote=21038 dedup_ratio=0.246440 router_tokens_per_expert=[753, 742, 748, 771, 763, 771, 743, 751, 802, 814, 740, 782, 786, 760, 783, 759, 754, 773, 822, 711, 752, 758, 722, 759, 772, 763, 810, 749, 741, 819, 807, 775, 770, 725, 760, 771, 769, 800, 758, 760, 752, 802, 753, 761, 732, 713, 760, 804, 780, 750, 806, 813, 788, 745, 783, 773, 750, 784, 751, 793, 766, 775, 793, 757] payload_rows_per_dst=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] dup_rows_per_dst=[2882, 3106, 2930, 3188, 2906, 2956, 3168, 3090] combine_rows_per_src=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65046 dup_rows=23902 dup_rows_remote=20918 dedup_ratio=0.243144 router_tokens_per_expert=[810, 792, 799, 796, 765, 734, 802, 770, 764, 777, 761, 767, 763, 759, 790, 789, 780, 793, 800, 808, 740, 777, 755, 744, 759, 795, 734, 771, 747, 727, 778, 744, 779, 785, 743, 699, 737, 801, 778, 781, 767, 806, 719, 794, 754, 722, 739, 768, 738, 749, 759, 788, 774, 784, 785, 778, 800, 742, 753, 755, 746, 788, 784, 767] payload_rows_per_dst=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] dup_rows_per_dst=[3132, 2984, 3110, 2824, 2986, 2828, 3010, 3028] combine_rows_per_src=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65118 dup_rows=23872 dup_rows_remote=20786 dedup_ratio=0.242839 router_tokens_per_expert=[816, 759, 780, 736, 806, 748, 699, 765, 725, 760, 789, 783, 738, 767, 765, 793, 778, 737, 771, 756, 768, 808, 822, 760, 810, 780, 776, 793, 777, 753, 766, 785, 784, 769, 761, 757, 733, 734, 794, 750, 711, 767, 811, 751, 775, 723, 825, 706, 784, 763, 791, 747, 714, 745, 793, 755, 769, 768, 779, 777, 770, 798, 789, 790] payload_rows_per_dst=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] dup_rows_per_dst=[2912, 2992, 3086, 3052, 2906, 2930, 2954, 3040] combine_rows_per_src=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437480 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532530024 weight_movement_bytes=34603008 total_nvlink_bytes=567133032 weight_to_token_ratio=0.064979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343192 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198168 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053144 weight_movement_bytes=34603008 total_nvlink_bytes=566656152 weight_to_token_ratio=0.065037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65030 dup_rows=24064 dup_rows_remote=21048 dedup_ratio=0.244792 router_tokens_per_expert=[760, 808, 792, 771, 796, 752, 786, 766, 752, 742, 744, 782, 789, 811, 806, 824, 727, 721, 763, 812, 769, 806, 746, 778, 744, 735, 780, 787, 776, 756, 749, 780, 791, 742, 801, 745, 727, 791, 754, 779, 763, 764, 758, 826, 745, 780, 758, 719, 722, 728, 762, 762, 791, 819, 805, 707, 793, 735, 767, 805, 722, 781, 761, 739] payload_rows_per_dst=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] dup_rows_per_dst=[3096, 3096, 3008, 3046, 2916, 3016, 2960, 2926] combine_rows_per_src=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343328 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265739552 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135776 weight_movement_bytes=69206016 total_nvlink_bytes=600341792 weight_to_token_ratio=0.130298 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343856 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772272 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533200688 weight_movement_bytes=34603008 total_nvlink_bytes=567803696 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64836 dup_rows=23958 dup_rows_remote=20768 dedup_ratio=0.243713 router_tokens_per_expert=[792, 755, 772, 782, 800, 796, 749, 784, 824, 779, 798, 770, 769, 737, 815, 755, 777, 763, 765, 725, 796, 722, 741, 742, 725, 773, 782, 739, 776, 747, 759, 771, 798, 787, 737, 764, 737, 778, 774, 715, 767, 799, 814, 759, 790, 729, 782, 741, 788, 755, 781, 803, 709, 733, 785, 798, 770, 758, 773, 741, 788, 758, 758, 803] payload_rows_per_dst=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] dup_rows_per_dst=[3190, 3092, 2896, 2980, 3074, 2894, 2834, 2998] combine_rows_per_src=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343616 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267066944 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790272 weight_movement_bytes=138412032 total_nvlink_bytes=672202304 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344312 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707192 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070072 weight_movement_bytes=34603008 total_nvlink_bytes=567673080 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65074 dup_rows=24000 dup_rows_remote=20968 dedup_ratio=0.244141 router_tokens_per_expert=[731, 758, 747, 719, 782, 799, 765, 764, 784, 753, 797, 779, 748, 760, 748, 741, 784, 792, 762, 792, 768, 799, 767, 740, 765, 785, 801, 795, 721, 763, 748, 811, 753, 795, 781, 765, 794, 815, 752, 755, 817, 788, 775, 745, 786, 769, 713, 716, 751, 768, 755, 773, 795, 748, 756, 785, 758, 756, 747, 817, 763, 770, 727, 796] payload_rows_per_dst=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] dup_rows_per_dst=[2878, 2970, 3024, 3022, 3098, 2934, 3032, 3042] combine_rows_per_src=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=342416 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265910672 ingress_bytes=300171264 link_bytes=300171264 token_bytes=531478928 weight_movement_bytes=34603008 total_nvlink_bytes=566081936 weight_to_token_ratio=0.065107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129520301e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64900 dup_rows=23998 dup_rows_remote=20886 dedup_ratio=0.244120 router_tokens_per_expert=[781, 767, 730, 770, 817, 753, 760, 768, 795, 767, 776, 807, 767, 766, 746, 759, 750, 762, 759, 743, 722, 768, 755, 755, 787, 750, 724, 777, 744, 768, 754, 800, 822, 769, 778, 800, 731, 778, 792, 789, 776, 787, 790, 744, 711, 782, 779, 761, 763, 761, 762, 747, 800, 779, 771, 762, 764, 843, 808, 757, 766, 750, 740, 743] payload_rows_per_dst=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] dup_rows_per_dst=[3034, 3038, 2878, 2942, 3112, 2958, 3014, 3022] combine_rows_per_src=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21022 dedup_ratio=0.244873 router_tokens_per_expert=[699, 776, 777, 812, 746, 750, 774, 748, 751, 759, 762, 788, 705, 788, 765, 822, 764, 763, 779, 774, 793, 768, 781, 821, 761, 759, 737, 742, 715, 774, 716, 755, 762, 754, 722, 792, 812, 744, 762, 776, 770, 766, 843, 765, 768, 764, 741, 766, 784, 800, 763, 782, 770, 801, 722, 831, 755, 791, 773, 766, 780, 766, 770, 767] payload_rows_per_dst=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] dup_rows_per_dst=[2902, 2974, 2914, 3046, 2814, 3244, 3128, 3050] combine_rows_per_src=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64886 dup_rows=24096 dup_rows_remote=21094 dedup_ratio=0.245117 router_tokens_per_expert=[800, 728, 760, 757, 801, 738, 744, 760, 761, 733, 772, 810, 753, 789, 747, 738, 753, 805, 764, 746, 811, 802, 782, 776, 720, 793, 746, 757, 775, 820, 779, 772, 789, 742, 769, 779, 760, 750, 818, 757, 720, 766, 793, 774, 768, 748, 803, 776, 784, 750, 756, 743, 789, 776, 764, 770, 795, 744, 785, 761, 743, 810, 715, 763] payload_rows_per_dst=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] dup_rows_per_dst=[2922, 3034, 3074, 3002, 3034, 3036, 3004, 2990] combine_rows_per_src=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344168 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266887272 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430376 weight_movement_bytes=69206016 total_nvlink_bytes=602636392 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64858 dup_rows=24194 dup_rows_remote=21128 dedup_ratio=0.246114 router_tokens_per_expert=[754, 771, 774, 738, 778, 800, 791, 735, 784, 764, 693, 792, 731, 778, 827, 790, 781, 773, 778, 765, 735, 722, 770, 781, 812, 777, 791, 745, 755, 785, 786, 760, 790, 795, 752, 752, 761, 820, 772, 792, 772, 794, 770, 770, 771, 727, 761, 753, 732, 728, 734, 756, 745, 759, 772, 739, 722, 786, 831, 771, 764, 813, 762, 770] payload_rows_per_dst=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] dup_rows_per_dst=[3028, 3066, 2962, 3022, 3092, 2990, 2976, 3058] combine_rows_per_src=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343144 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173544 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532003944 weight_movement_bytes=69206016 total_nvlink_bytes=601209960 weight_to_token_ratio=0.130086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64954 dup_rows=23974 dup_rows_remote=20990 dedup_ratio=0.243876 router_tokens_per_expert=[767, 813, 762, 761, 771, 750, 770, 772, 781, 770, 766, 708, 795, 765, 740, 770, 817, 757, 780, 796, 756, 762, 776, 736, 792, 787, 750, 779, 762, 786, 776, 747, 766, 776, 759, 749, 768, 791, 778, 791, 758, 805, 767, 774, 753, 737, 763, 760, 775, 789, 779, 745, 764, 738, 770, 732, 797, 770, 822, 752, 742, 734, 799, 729] payload_rows_per_dst=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] dup_rows_per_dst=[3044, 2856, 2984, 3126, 3064, 2930, 2966, 3004] combine_rows_per_src=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65050 dup_rows=24042 dup_rows_remote=21080 dedup_ratio=0.244568 router_tokens_per_expert=[804, 760, 759, 725, 736, 756, 769, 766, 794, 773, 782, 784, 743, 710, 796, 773, 761, 772, 775, 758, 743, 744, 782, 755, 789, 786, 733, 771, 776, 768, 869, 768, 764, 753, 787, 719, 753, 749, 789, 786, 759, 738, 739, 764, 799, 751, 771, 766, 762, 769, 763, 822, 783, 751, 796, 785, 784, 742, 802, 775, 770, 766, 733, 782] payload_rows_per_dst=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] dup_rows_per_dst=[3014, 2960, 2986, 3102, 2934, 2962, 3088, 2996] combine_rows_per_src=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343760 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247888 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152016 weight_movement_bytes=69206016 total_nvlink_bytes=601358032 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343920 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116976 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890032 weight_movement_bytes=34603008 total_nvlink_bytes=566493040 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64988 dup_rows=24014 dup_rows_remote=20990 dedup_ratio=0.244283 router_tokens_per_expert=[742, 743, 777, 754, 793, 755, 780, 799, 743, 774, 744, 753, 746, 734, 755, 746, 788, 776, 712, 778, 737, 808, 801, 779, 756, 766, 753, 758, 777, 783, 763, 765, 778, 792, 776, 819, 785, 755, 784, 736, 776, 730, 774, 785, 755, 779, 755, 780, 763, 815, 816, 746, 743, 788, 756, 776, 758, 761, 755, 763, 791, 793, 788, 743] payload_rows_per_dst=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] dup_rows_per_dst=[3024, 2982, 3082, 2992, 3150, 2858, 3078, 2848] combine_rows_per_src=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343944 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266002312 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660680 weight_movement_bytes=34603008 total_nvlink_bytes=566263688 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343776 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395360 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446944 weight_movement_bytes=34603008 total_nvlink_bytes=567049952 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344520 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789320 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234120 weight_movement_bytes=34603008 total_nvlink_bytes=567837128 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64912 dup_rows=23898 dup_rows_remote=20732 dedup_ratio=0.243103 router_tokens_per_expert=[728, 793, 767, 740, 779, 763, 715, 774, 769, 757, 752, 768, 741, 791, 769, 761, 805, 762, 784, 813, 746, 759, 777, 716, 776, 738, 752, 780, 788, 718, 731, 684, 793, 751, 748, 803, 766, 736, 774, 756, 790, 748, 788, 764, 756, 744, 740, 737, 820, 808, 774, 794, 737, 760, 840, 797, 810, 822, 788, 799, 764, 815, 779, 755] payload_rows_per_dst=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] dup_rows_per_dst=[2848, 2924, 3042, 2844, 2956, 2948, 3166, 3170] combine_rows_per_src=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343912 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534760 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532725608 weight_movement_bytes=34603008 total_nvlink_bytes=567328616 weight_to_token_ratio=0.064955 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64920 dup_rows=23894 dup_rows_remote=20788 dedup_ratio=0.243062 router_tokens_per_expert=[816, 756, 748, 781, 780, 768, 731, 716, 774, 735, 771, 774, 797, 752, 760, 729, 746, 757, 744, 806, 747, 805, 800, 742, 724, 807, 763, 764, 740, 730, 810, 789, 786, 794, 785, 782, 783, 818, 750, 800, 750, 771, 800, 738, 758, 741, 742, 785, 770, 763, 757, 769, 764, 819, 750, 782, 775, 738, 766, 756, 803, 753, 786, 756] payload_rows_per_dst=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] dup_rows_per_dst=[2964, 2988, 2930, 2998, 3106, 2966, 3020, 2922] combine_rows_per_src=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65148 dup_rows=23900 dup_rows_remote=20856 dedup_ratio=0.243123 router_tokens_per_expert=[823, 790, 784, 718, 817, 778, 753, 748, 772, 816, 772, 739, 738, 740, 757, 770, 747, 763, 783, 768, 773, 771, 786, 790, 754, 773, 779, 724, 763, 808, 783, 850, 756, 747, 754, 736, 709, 745, 742, 754, 768, 804, 764, 766, 811, 740, 793, 788, 746, 782, 802, 725, 757, 774, 763, 756, 814, 815, 728, 782, 770, 709, 747, 775] payload_rows_per_dst=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] dup_rows_per_dst=[3034, 3038, 3026, 3026, 2732, 3046, 2954, 3044] combine_rows_per_src=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65088 dup_rows=23876 dup_rows_remote=20934 dedup_ratio=0.242879 router_tokens_per_expert=[773, 766, 764, 774, 765, 719, 808, 709, 742, 717, 764, 764, 794, 768, 759, 771, 736, 751, 794, 800, 754, 763, 803, 735, 767, 745, 784, 754, 804, 758, 776, 753, 795, 796, 772, 797, 775, 790, 781, 776, 796, 717, 754, 752, 731, 760, 755, 799, 769, 733, 864, 765, 773, 775, 774, 774, 772, 746, 776, 764, 800, 752, 772, 763] payload_rows_per_dst=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] dup_rows_per_dst=[2910, 2876, 2978, 2942, 3170, 2872, 3092, 3036] combine_rows_per_src=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=342576 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266222128 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532101680 weight_movement_bytes=69206016 total_nvlink_bytes=601307696 weight_to_token_ratio=0.130062 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342832 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255152 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167472 weight_movement_bytes=103809024 total_nvlink_bytes=635976496 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=21024 dedup_ratio=0.244975 router_tokens_per_expert=[755, 775, 709, 756, 779, 735, 772, 770, 779, 766, 776, 803, 729, 761, 788, 781, 710, 765, 806, 765, 828, 724, 770, 775, 793, 798, 745, 781, 780, 794, 764, 751, 786, 787, 746, 749, 785, 817, 759, 793, 784, 763, 770, 765, 754, 782, 704, 737, 786, 720, 792, 761, 734, 758, 736, 792, 792, 788, 784, 783, 774, 711, 790, 787] payload_rows_per_dst=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] dup_rows_per_dst=[2944, 3058, 3008, 3104, 3110, 2860, 2920, 3078] combine_rows_per_src=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65042 dup_rows=23944 dup_rows_remote=21008 dedup_ratio=0.243571 router_tokens_per_expert=[781, 718, 777, 793, 745, 767, 779, 806, 776, 797, 714, 757, 735, 735, 785, 779, 741, 772, 768, 729, 738, 826, 779, 774, 782, 765, 796, 776, 750, 791, 739, 804, 776, 741, 753, 791, 766, 798, 724, 764, 774, 778, 763, 800, 727, 790, 761, 751, 789, 781, 798, 729, 742, 772, 725, 818, 777, 802, 816, 757, 756, 752, 748, 759] payload_rows_per_dst=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] dup_rows_per_dst=[3034, 2930, 2936, 3122, 3002, 2970, 2974, 2976] combine_rows_per_src=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64928 dup_rows=24108 dup_rows_remote=21084 dedup_ratio=0.245239 router_tokens_per_expert=[764, 769, 786, 761, 766, 777, 764, 744, 768, 771, 747, 829, 787, 788, 762, 779, 776, 758, 791, 753, 754, 756, 752, 781, 765, 762, 797, 778, 729, 778, 762, 761, 755, 734, 778, 784, 772, 762, 721, 766, 794, 805, 758, 747, 777, 718, 787, 760, 783, 766, 748, 778, 789, 797, 784, 819, 766, 760, 738, 747, 750, 765, 767, 762] payload_rows_per_dst=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] dup_rows_per_dst=[3014, 3060, 2990, 2924, 3032, 3024, 3122, 2942] combine_rows_per_src=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344016 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036432 weight_movement_bytes=69206016 total_nvlink_bytes=603242448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344088 combine_payload_bytes=266600448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266944536 ingress_bytes=370409472 link_bytes=370409472 token_bytes=533544984 weight_movement_bytes=103809024 total_nvlink_bytes=637354008 weight_to_token_ratio=0.194565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393821023e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64880 dup_rows=23938 dup_rows_remote=20772 dedup_ratio=0.243510 router_tokens_per_expert=[774, 819, 762, 763, 787, 795, 810, 741, 798, 733, 741, 751, 771, 757, 750, 734, 747, 745, 747, 719, 786, 733, 790, 785, 771, 759, 761, 755, 792, 763, 725, 804, 761, 730, 763, 770, 768, 774, 788, 776, 745, 785, 752, 767, 740, 807, 771, 746, 800, 756, 810, 798, 791, 743, 758, 786, 760, 775, 807, 777, 797, 792, 731, 760] payload_rows_per_dst=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] dup_rows_per_dst=[3166, 3186, 2982, 2856, 2952, 3208, 2800, 2788] combine_rows_per_src=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343752 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231496 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119240 weight_movement_bytes=34603008 total_nvlink_bytes=566722248 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344200 combine_payload_bytes=266412032 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266756232 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533168264 weight_movement_bytes=69206016 total_nvlink_bytes=602374280 weight_to_token_ratio=0.129801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266289136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532234224 weight_movement_bytes=34603008 total_nvlink_bytes=566837232 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=342608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091088 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531839568 weight_movement_bytes=34603008 total_nvlink_bytes=566442576 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65122 dup_rows=23950 dup_rows_remote=21034 dedup_ratio=0.243632 router_tokens_per_expert=[728, 715, 775, 786, 804, 762, 759, 718, 800, 768, 779, 792, 751, 749, 778, 770, 789, 798, 735, 792, 762, 768, 788, 783, 764, 824, 813, 804, 747, 757, 770, 773, 729, 804, 747, 770, 755, 776, 779, 766, 767, 779, 782, 751, 783, 754, 776, 782, 790, 742, 752, 766, 757, 764, 752, 751, 710, 789, 710, 746, 803, 764, 752, 803] payload_rows_per_dst=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] dup_rows_per_dst=[2854, 3058, 3066, 3082, 2994, 3016, 2916, 2964] combine_rows_per_src=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64850 dup_rows=23998 dup_rows_remote=20842 dedup_ratio=0.244120 router_tokens_per_expert=[766, 781, 813, 758, 792, 759, 744, 733, 770, 748, 738, 770, 818, 734, 740, 783, 733, 759, 873, 746, 788, 776, 751, 746, 739, 772, 765, 734, 755, 756, 748, 827, 783, 809, 799, 819, 780, 776, 790, 750, 731, 788, 765, 728, 766, 768, 775, 745, 730, 747, 719, 808, 766, 793, 724, 766, 790, 772, 780, 804, 760, 768, 744, 794] payload_rows_per_dst=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] dup_rows_per_dst=[2902, 2990, 2978, 2958, 3156, 2978, 2924, 3112] combine_rows_per_src=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65030 dup_rows=23966 dup_rows_remote=20904 dedup_ratio=0.243795 router_tokens_per_expert=[736, 726, 734, 772, 719, 769, 766, 774, 759, 748, 742, 788, 749, 786, 833, 800, 767, 756, 775, 766, 787, 793, 757, 725, 750, 765, 762, 791, 788, 776, 776, 807, 738, 723, 745, 813, 744, 797, 738, 789, 726, 779, 762, 764, 733, 789, 769, 750, 783, 784, 812, 799, 790, 781, 771, 751, 802, 773, 742, 764, 758, 790, 825, 726] payload_rows_per_dst=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] dup_rows_per_dst=[2912, 3036, 2902, 3042, 2782, 3028, 3202, 3062] combine_rows_per_src=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65062 dup_rows=23892 dup_rows_remote=20886 dedup_ratio=0.243042 router_tokens_per_expert=[767, 741, 744, 776, 755, 793, 751, 771, 785, 755, 728, 758, 764, 782, 735, 782, 725, 746, 792, 763, 782, 767, 826, 779, 760, 801, 731, 718, 819, 759, 781, 809, 760, 765, 720, 749, 774, 788, 759, 758, 748, 798, 775, 819, 755, 791, 771, 773, 774, 737, 799, 787, 715, 756, 773, 722, 804, 750, 822, 765, 787, 788, 721, 804] payload_rows_per_dst=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] dup_rows_per_dst=[2932, 2928, 2998, 3006, 2972, 3060, 2932, 3064] combine_rows_per_src=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344624 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267084336 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533824048 weight_movement_bytes=34603008 total_nvlink_bytes=568427056 weight_to_token_ratio=0.064821 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=342768 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265968368 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531593968 weight_movement_bytes=69206016 total_nvlink_bytes=600799984 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64744 dup_rows=24206 dup_rows_remote=21150 dedup_ratio=0.246236 router_tokens_per_expert=[743, 769, 735, 767, 788, 747, 779, 792, 782, 780, 767, 797, 796, 738, 745, 800, 746, 783, 779, 783, 786, 780, 783, 813, 765, 772, 780, 762, 756, 738, 777, 749, 787, 767, 743, 769, 769, 814, 768, 804, 746, 730, 737, 761, 790, 782, 798, 788, 753, 702, 768, 807, 751, 758, 737, 741, 758, 768, 743, 755, 823, 723, 757, 778] payload_rows_per_dst=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] dup_rows_per_dst=[2984, 3056, 3212, 2902, 3076, 3022, 2954, 3000] combine_rows_per_src=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=64844 dup_rows=24274 dup_rows_remote=21350 dedup_ratio=0.246928 router_tokens_per_expert=[793, 782, 716, 806, 755, 775, 786, 793, 759, 769, 732, 767, 801, 756, 803, 723, 703, 747, 790, 713, 782, 760, 781, 779, 718, 738, 749, 767, 751, 760, 816, 764, 775, 770, 789, 814, 776, 802, 783, 761, 792, 750, 774, 786, 770, 739, 721, 821, 719, 775, 732, 792, 758, 778, 805, 738, 773, 738, 759, 815, 801, 704, 809, 799] payload_rows_per_dst=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] dup_rows_per_dst=[3146, 3004, 2924, 2964, 3154, 3066, 2918, 3098] combine_rows_per_src=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64850 dup_rows=24256 dup_rows_remote=21260 dedup_ratio=0.246745 router_tokens_per_expert=[726, 780, 789, 745, 752, 810, 763, 792, 756, 750, 741, 771, 744, 726, 795, 744, 778, 780, 759, 753, 753, 771, 849, 785, 777, 713, 712, 797, 809, 793, 778, 787, 728, 767, 766, 746, 798, 761, 781, 768, 717, 728, 732, 766, 812, 813, 730, 799, 739, 770, 752, 794, 769, 799, 777, 779, 724, 773, 770, 780, 801, 780, 785, 770] payload_rows_per_dst=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] dup_rows_per_dst=[3012, 2946, 3072, 3046, 3038, 2996, 3040, 3106] combine_rows_per_src=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343736 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706616 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069496 weight_movement_bytes=69206016 total_nvlink_bytes=602275512 weight_to_token_ratio=0.129826 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64592 dup_rows=24028 dup_rows_remote=20678 dedup_ratio=0.244425 router_tokens_per_expert=[766, 787, 737, 765, 737, 792, 800, 782, 722, 768, 741, 732, 775, 757, 790, 778, 748, 764, 799, 770, 778, 765, 750, 714, 806, 742, 774, 798, 784, 769, 787, 767, 788, 793, 775, 779, 780, 786, 756, 800, 795, 850, 725, 773, 756, 757, 790, 705, 735, 776, 777, 796, 760, 776, 741, 759, 790, 762, 755, 729, 806, 742, 722, 774] payload_rows_per_dst=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] dup_rows_per_dst=[3350, 2972, 2908, 2784, 3094, 3002, 3064, 2854] combine_rows_per_src=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343792 combine_payload_bytes=266493952 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266837744 ingress_bytes=335699968 link_bytes=335699968 token_bytes=533331696 weight_movement_bytes=69206016 total_nvlink_bytes=602537712 weight_to_token_ratio=0.129762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263211954e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343576 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535000 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726424 weight_movement_bytes=34603008 total_nvlink_bytes=565329432 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=344776 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265945800 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546824 weight_movement_bytes=103809024 total_nvlink_bytes=635355848 weight_to_token_ratio=0.195296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=344440 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265970040 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595640 weight_movement_bytes=34603008 total_nvlink_bytes=566198648 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264568832 dispatch_meta_bytes=341080 combine_payload_bytes=264568832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264909912 ingress_bytes=299171840 link_bytes=299171840 token_bytes=529478744 weight_movement_bytes=34603008 total_nvlink_bytes=564081752 weight_to_token_ratio=0.065353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125759549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65044 dup_rows=23964 dup_rows_remote=20982 dedup_ratio=0.243774 router_tokens_per_expert=[766, 828, 752, 814, 844, 757, 764, 737, 744, 743, 797, 805, 742, 743, 790, 756, 789, 781, 795, 787, 848, 754, 763, 731, 745, 752, 788, 756, 775, 749, 798, 726, 750, 766, 797, 779, 741, 717, 784, 764, 754, 738, 731, 784, 789, 753, 766, 701, 771, 751, 751, 778, 759, 762, 771, 796, 770, 751, 755, 767, 768, 801, 788, 780] payload_rows_per_dst=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] dup_rows_per_dst=[3042, 2960, 3124, 2992, 2968, 2876, 2982, 3020] combine_rows_per_src=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65128 dup_rows=23930 dup_rows_remote=21094 dedup_ratio=0.243429 router_tokens_per_expert=[792, 740, 848, 760, 779, 846, 772, 786, 779, 730, 792, 785, 803, 765, 738, 788, 751, 796, 727, 798, 755, 760, 730, 781, 739, 791, 783, 737, 778, 772, 779, 814, 751, 754, 755, 766, 738, 762, 767, 748, 710, 738, 791, 740, 757, 759, 733, 809, 742, 770, 801, 730, 763, 813, 768, 745, 779, 764, 749, 807, 745, 778, 742, 784] payload_rows_per_dst=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] dup_rows_per_dst=[3260, 3058, 2928, 2934, 2836, 2940, 3042, 2932] combine_rows_per_src=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65120 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[786, 711, 763, 773, 816, 749, 758, 728, 754, 741, 768, 716, 738, 760, 750, 799, 759, 822, 693, 746, 782, 806, 831, 760, 782, 788, 772, 809, 785, 777, 789, 802, 787, 797, 778, 776, 738, 786, 786, 770, 805, 711, 758, 793, 743, 758, 716, 794, 753, 776, 800, 760, 769, 732, 770, 787, 751, 718, 755, 760, 729, 788, 780, 815] payload_rows_per_dst=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] dup_rows_per_dst=[2794, 2902, 2804, 3164, 3020, 3144, 3164, 2954] combine_rows_per_src=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64818 dup_rows=24132 dup_rows_remote=21156 dedup_ratio=0.245483 router_tokens_per_expert=[712, 785, 757, 726, 798, 765, 794, 782, 743, 808, 810, 732, 799, 753, 746, 723, 766, 757, 796, 785, 784, 788, 773, 812, 745, 794, 786, 749, 781, 809, 747, 754, 741, 741, 760, 755, 761, 779, 737, 762, 809, 779, 775, 747, 765, 786, 774, 729, 763, 730, 778, 776, 764, 750, 758, 787, 741, 761, 769, 814, 781, 753, 808, 760] payload_rows_per_dst=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] dup_rows_per_dst=[3010, 3068, 3116, 2976, 2888, 3078, 2992, 3004] combine_rows_per_src=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344104 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764328 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184552 weight_movement_bytes=34603008 total_nvlink_bytes=567787560 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344888 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109176 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873464 weight_movement_bytes=34603008 total_nvlink_bytes=568476472 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65352 dup_rows=23816 dup_rows_remote=21004 dedup_ratio=0.242269 router_tokens_per_expert=[736, 783, 815, 769, 760, 729, 768, 800, 710, 759, 757, 717, 757, 762, 787, 725, 794, 723, 771, 769, 800, 796, 712, 741, 784, 761, 776, 775, 764, 801, 753, 764, 790, 786, 779, 744, 783, 771, 756, 768, 774, 797, 769, 754, 804, 796, 766, 780, 771, 815, 727, 762, 801, 759, 767, 767, 761, 759, 745, 824, 762, 772, 774, 751] payload_rows_per_dst=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] dup_rows_per_dst=[2964, 2812, 3000, 2986, 2976, 3070, 3054, 2954] combine_rows_per_src=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65070 dup_rows=24132 dup_rows_remote=21230 dedup_ratio=0.245483 router_tokens_per_expert=[763, 810, 797, 735, 800, 733, 741, 749, 762, 766, 759, 804, 764, 759, 725, 786, 790, 733, 706, 764, 759, 746, 767, 737, 778, 822, 773, 787, 759, 770, 778, 806, 784, 786, 782, 748, 765, 746, 761, 793, 749, 764, 735, 791, 751, 739, 761, 748, 757, 768, 783, 771, 775, 811, 790, 780, 779, 809, 757, 737, 746, 833, 742, 783] payload_rows_per_dst=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] dup_rows_per_dst=[2986, 2978, 2902, 3136, 3040, 2944, 3036, 3110] combine_rows_per_src=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65072 dup_rows=23898 dup_rows_remote=20918 dedup_ratio=0.243103 router_tokens_per_expert=[746, 770, 762, 800, 752, 765, 806, 692, 809, 779, 763, 763, 724, 732, 781, 788, 801, 800, 813, 813, 771, 792, 767, 775, 777, 746, 769, 712, 763, 797, 804, 800, 803, 749, 746, 738, 776, 723, 785, 787, 738, 745, 800, 799, 793, 748, 774, 760, 753, 782, 793, 785, 801, 718, 775, 695, 759, 778, 762, 759, 737, 753, 738, 768] payload_rows_per_dst=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] dup_rows_per_dst=[3040, 2970, 3122, 2938, 2934, 2980, 2992, 2922] combine_rows_per_src=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344448 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075968 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807488 weight_movement_bytes=103809024 total_nvlink_bytes=637616512 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65212 dup_rows=23880 dup_rows_remote=20966 dedup_ratio=0.242920 router_tokens_per_expert=[743, 774, 753, 750, 750, 737, 736, 725, 734, 783, 780, 781, 716, 778, 767, 755, 788, 763, 776, 804, 843, 782, 788, 774, 810, 777, 733, 786, 747, 760, 768, 764, 768, 770, 753, 748, 800, 725, 809, 755, 782, 791, 747, 756, 792, 784, 798, 773, 805, 793, 725, 776, 766, 770, 745, 773, 729, 768, 732, 798, 780, 780, 746, 790] payload_rows_per_dst=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] dup_rows_per_dst=[2914, 3186, 2890, 2896, 3014, 3102, 3034, 2844] combine_rows_per_src=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343896 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838424 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332952 weight_movement_bytes=69206016 total_nvlink_bytes=600538968 weight_to_token_ratio=0.130250 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267681792 dispatch_meta_bytes=345424 combine_payload_bytes=267681792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268027216 ingress_bytes=302284800 link_bytes=302284800 token_bytes=535709008 weight_movement_bytes=34603008 total_nvlink_bytes=570312016 weight_to_token_ratio=0.064593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137473366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=345200 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871920 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533398640 weight_movement_bytes=34603008 total_nvlink_bytes=568001648 weight_to_token_ratio=0.064873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266878872 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533413784 weight_movement_bytes=69206016 total_nvlink_bytes=602619800 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344712 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267453064 ingress_bytes=336314368 link_bytes=336314368 token_bytes=534561416 weight_movement_bytes=69206016 total_nvlink_bytes=603767432 weight_to_token_ratio=0.129463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265523891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65222 dup_rows=23816 dup_rows_remote=20964 dedup_ratio=0.242269 router_tokens_per_expert=[786, 805, 754, 721, 752, 822, 815, 772, 787, 815, 746, 748, 770, 770, 803, 739, 771, 781, 762, 763, 800, 809, 776, 785, 707, 743, 781, 748, 787, 783, 761, 738, 737, 767, 758, 725, 815, 794, 794, 751, 764, 779, 734, 756, 734, 794, 789, 747, 776, 774, 727, 738, 763, 824, 732, 725, 798, 734, 773, 765, 770, 717, 810, 788] payload_rows_per_dst=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] dup_rows_per_dst=[3066, 2970, 3048, 2940, 3032, 2954, 2852, 2954] combine_rows_per_src=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65182 dup_rows=23944 dup_rows_remote=21006 dedup_ratio=0.243571 router_tokens_per_expert=[767, 748, 801, 752, 759, 782, 751, 731, 763, 730, 750, 728, 736, 776, 775, 777, 765, 788, 735, 764, 757, 761, 776, 756, 753, 758, 787, 748, 801, 769, 787, 760, 768, 772, 733, 767, 728, 791, 743, 756, 802, 771, 852, 783, 779, 791, 785, 789, 692, 756, 789, 805, 769, 786, 727, 816, 797, 767, 765, 767, 838, 780, 766, 731] payload_rows_per_dst=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] dup_rows_per_dst=[2856, 2838, 2924, 3108, 2938, 3204, 3026, 3050] combine_rows_per_src=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64756 dup_rows=24090 dup_rows_remote=21024 dedup_ratio=0.245056 router_tokens_per_expert=[760, 780, 763, 782, 755, 780, 773, 767, 740, 745, 737, 773, 818, 727, 747, 746, 761, 752, 768, 797, 731, 754, 764, 748, 792, 703, 766, 765, 834, 790, 738, 793, 772, 709, 753, 766, 767, 764, 764, 791, 750, 807, 798, 776, 797, 758, 817, 803, 722, 750, 781, 822, 747, 745, 741, 744, 797, 769, 789, 805, 763, 791, 760, 785] payload_rows_per_dst=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] dup_rows_per_dst=[3098, 2852, 2908, 3016, 3008, 3208, 2934, 3066] combine_rows_per_src=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344744 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267494056 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643368 weight_movement_bytes=34603008 total_nvlink_bytes=569246376 weight_to_token_ratio=0.064722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344752 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330224 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315696 weight_movement_bytes=34603008 total_nvlink_bytes=568918704 weight_to_token_ratio=0.064761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64756 dup_rows=24180 dup_rows_remote=21072 dedup_ratio=0.245972 router_tokens_per_expert=[766, 752, 797, 725, 764, 756, 790, 747, 784, 741, 772, 774, 785, 785, 817, 727, 856, 797, 756, 757, 791, 695, 745, 742, 752, 813, 775, 783, 747, 804, 772, 792, 749, 748, 770, 775, 787, 718, 801, 743, 781, 753, 766, 731, 744, 759, 733, 797, 739, 783, 772, 802, 764, 750, 781, 775, 789, 788, 805, 741, 771, 761, 763, 754] payload_rows_per_dst=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] dup_rows_per_dst=[3020, 3062, 3062, 3108, 2918, 2930, 3068, 3012] combine_rows_per_src=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64842 dup_rows=24176 dup_rows_remote=21092 dedup_ratio=0.245931 router_tokens_per_expert=[743, 782, 760, 777, 790, 763, 759, 752, 798, 766, 739, 807, 776, 782, 793, 724, 744, 755, 725, 716, 750, 767, 768, 719, 761, 773, 761, 813, 737, 798, 800, 761, 762, 801, 807, 766, 749, 827, 792, 793, 793, 769, 781, 720, 758, 797, 767, 758, 771, 802, 774, 751, 760, 736, 741, 817, 778, 772, 757, 764, 768, 740, 758, 764] payload_rows_per_dst=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] dup_rows_per_dst=[3054, 3084, 2770, 3072, 3208, 3006, 2986, 2996] combine_rows_per_src=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65166 dup_rows=24010 dup_rows_remote=21090 dedup_ratio=0.244242 router_tokens_per_expert=[826, 772, 816, 784, 784, 782, 782, 777, 803, 783, 793, 768, 787, 725, 736, 742, 763, 723, 750, 785, 757, 754, 743, 749, 744, 738, 774, 767, 748, 738, 769, 741, 837, 773, 779, 765, 767, 776, 799, 754, 758, 783, 721, 761, 756, 782, 803, 740, 750, 788, 808, 748, 774, 723, 787, 766, 770, 789, 717, 844, 720, 742, 770, 799] payload_rows_per_dst=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] dup_rows_per_dst=[3128, 3038, 2920, 2824, 3120, 2958, 2996, 3026] combine_rows_per_src=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64944 dup_rows=24020 dup_rows_remote=20888 dedup_ratio=0.244344 router_tokens_per_expert=[765, 752, 806, 756, 771, 761, 783, 758, 761, 724, 735, 784, 753, 775, 760, 720, 747, 807, 742, 754, 793, 761, 768, 761, 725, 776, 819, 804, 759, 785, 740, 765, 798, 733, 800, 759, 724, 754, 785, 739, 778, 771, 761, 808, 768, 798, 794, 758, 743, 808, 761, 767, 797, 764, 762, 774, 715, 722, 805, 765, 810, 798, 782, 781] payload_rows_per_dst=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] dup_rows_per_dst=[3006, 2924, 2974, 2926, 3010, 3132, 3074, 2974] combine_rows_per_src=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343120 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265583696 ingress_bytes=369049600 link_bytes=369049600 token_bytes=530824272 weight_movement_bytes=103809024 total_nvlink_bytes=634633296 weight_to_token_ratio=0.195562 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388703934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64862 dup_rows=24164 dup_rows_remote=21050 dedup_ratio=0.245809 router_tokens_per_expert=[715, 766, 766, 744, 773, 779, 771, 759, 797, 770, 754, 762, 812, 750, 778, 766, 762, 788, 724, 737, 736, 770, 778, 834, 766, 738, 716, 788, 800, 795, 775, 787, 797, 764, 769, 789, 796, 771, 734, 791, 800, 760, 797, 763, 776, 735, 812, 742, 759, 781, 796, 782, 746, 767, 779, 744, 773, 750, 754, 750, 766, 748, 755, 750] payload_rows_per_dst=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] dup_rows_per_dst=[3114, 3306, 3014, 2892, 3042, 2840, 3184, 2772] combine_rows_per_src=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343312 combine_payload_bytes=265240576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265583888 ingress_bytes=403652608 link_bytes=403652608 token_bytes=530824464 weight_movement_bytes=138412032 total_nvlink_bytes=669236496 weight_to_token_ratio=0.260749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518912267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343736 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936568 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529400 weight_movement_bytes=34603008 total_nvlink_bytes=566132408 weight_to_token_ratio=0.065101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=345024 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264960 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184896 weight_movement_bytes=34603008 total_nvlink_bytes=568787904 weight_to_token_ratio=0.064777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343328 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353952 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364576 weight_movement_bytes=69206016 total_nvlink_bytes=601570592 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343648 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018400 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693152 weight_movement_bytes=34603008 total_nvlink_bytes=566296160 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65176 dup_rows=23878 dup_rows_remote=20922 dedup_ratio=0.242900 router_tokens_per_expert=[841, 724, 787, 750, 775, 768, 760, 739, 784, 751, 699, 783, 799, 782, 765, 749, 766, 746, 802, 787, 763, 773, 758, 757, 773, 763, 751, 784, 770, 744, 741, 784, 787, 782, 742, 795, 781, 822, 736, 770, 783, 768, 747, 788, 756, 790, 800, 768, 771, 743, 789, 771, 757, 733, 813, 726, 753, 734, 755, 769, 805, 733, 768, 799] payload_rows_per_dst=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] dup_rows_per_dst=[3014, 2992, 2982, 2914, 3020, 3056, 2956, 2944] combine_rows_per_src=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65164 dup_rows=24006 dup_rows_remote=21008 dedup_ratio=0.244202 router_tokens_per_expert=[781, 800, 814, 754, 752, 771, 750, 766, 747, 776, 767, 762, 776, 804, 780, 732, 765, 761, 783, 815, 737, 753, 740, 761, 747, 779, 750, 719, 816, 795, 804, 796, 752, 752, 733, 760, 739, 763, 791, 776, 753, 780, 765, 799, 774, 735, 775, 736, 804, 750, 733, 740, 810, 792, 767, 766, 787, 781, 757, 732, 770, 799, 741, 787] payload_rows_per_dst=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] dup_rows_per_dst=[3050, 3040, 2910, 3036, 2998, 2930, 3036, 3006] combine_rows_per_src=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64736 dup_rows=24064 dup_rows_remote=20898 dedup_ratio=0.244792 router_tokens_per_expert=[768, 752, 805, 761, 787, 729, 761, 742, 802, 796, 788, 751, 791, 744, 776, 794, 820, 746, 739, 781, 794, 748, 779, 738, 784, 762, 732, 788, 770, 794, 775, 759, 789, 735, 756, 763, 720, 766, 754, 781, 836, 727, 743, 798, 768, 718, 748, 816, 734, 782, 808, 783, 798, 754, 759, 761, 757, 756, 749, 773, 761, 731, 812, 760] payload_rows_per_dst=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] dup_rows_per_dst=[2630, 3074, 3372, 2800, 3000, 2974, 3048, 3166] combine_rows_per_src=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=344392 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267305288 ingress_bytes=370769920 link_bytes=370769920 token_bytes=534266184 weight_movement_bytes=103809024 total_nvlink_bytes=638075208 weight_to_token_ratio=0.194302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395177360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344688 combine_payload_bytes=266911744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267256432 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534168176 weight_movement_bytes=69206016 total_nvlink_bytes=603374192 weight_to_token_ratio=0.129558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64704 dup_rows=24250 dup_rows_remote=21248 dedup_ratio=0.246684 router_tokens_per_expert=[711, 771, 764, 766, 813, 767, 740, 789, 775, 776, 812, 791, 774, 767, 775, 770, 748, 757, 779, 773, 794, 745, 795, 744, 820, 761, 778, 770, 758, 736, 736, 817, 843, 762, 742, 777, 747, 731, 722, 773, 686, 785, 803, 780, 778, 749, 751, 799, 820, 782, 751, 790, 796, 790, 797, 760, 708, 755, 751, 729, 761, 730, 766, 766] payload_rows_per_dst=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] dup_rows_per_dst=[2960, 3156, 2964, 3002, 3088, 3082, 3138, 2860] combine_rows_per_src=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65000 dup_rows=24020 dup_rows_remote=21000 dedup_ratio=0.244344 router_tokens_per_expert=[790, 774, 758, 777, 787, 814, 747, 737, 748, 764, 736, 730, 784, 803, 805, 782, 765, 754, 753, 753, 746, 748, 775, 780, 790, 735, 768, 711, 779, 773, 792, 775, 793, 775, 758, 805, 778, 817, 765, 773, 752, 796, 744, 766, 725, 802, 780, 719, 771, 769, 763, 781, 789, 743, 804, 775, 766, 749, 760, 730, 737, 773, 789, 772] payload_rows_per_dst=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] dup_rows_per_dst=[3030, 3020, 2948, 3008, 3078, 2994, 3030, 2912] combine_rows_per_src=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65298 dup_rows=23794 dup_rows_remote=20908 dedup_ratio=0.242045 router_tokens_per_expert=[756, 771, 745, 760, 723, 796, 742, 793, 775, 740, 735, 723, 762, 764, 754, 758, 728, 784, 767, 724, 795, 755, 786, 767, 757, 760, 794, 753, 781, 787, 771, 778, 748, 811, 788, 756, 763, 711, 781, 812, 778, 751, 778, 792, 749, 744, 725, 732, 729, 780, 780, 747, 800, 790, 800, 813, 800, 785, 819, 810, 771, 753, 773, 799] payload_rows_per_dst=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] dup_rows_per_dst=[2894, 2876, 2988, 2936, 3008, 2886, 3070, 3136] combine_rows_per_src=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65170 dup_rows=23890 dup_rows_remote=20994 dedup_ratio=0.243022 router_tokens_per_expert=[791, 798, 811, 780, 758, 837, 771, 787, 819, 787, 774, 749, 769, 782, 773, 767, 710, 743, 804, 771, 773, 734, 751, 784, 753, 778, 767, 753, 749, 767, 778, 770, 765, 795, 772, 773, 766, 766, 752, 769, 720, 798, 751, 753, 723, 752, 816, 775, 807, 757, 778, 749, 720, 746, 749, 767, 766, 702, 758, 775, 782, 772, 836, 704] payload_rows_per_dst=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] dup_rows_per_dst=[3114, 2970, 2896, 2998, 3030, 3024, 2918, 2940] combine_rows_per_src=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20714 dedup_ratio=0.243510 router_tokens_per_expert=[809, 769, 778, 753, 778, 781, 749, 772, 757, 738, 778, 815, 737, 698, 748, 775, 758, 746, 754, 738, 772, 797, 747, 850, 775, 774, 788, 819, 787, 812, 764, 747, 796, 725, 770, 746, 755, 790, 771, 804, 786, 798, 763, 748, 790, 742, 767, 762, 703, 795, 778, 703, 764, 749, 764, 769, 773, 785, 797, 769, 785, 724, 774, 744] payload_rows_per_dst=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] dup_rows_per_dst=[3224, 2918, 2992, 3136, 2914, 2946, 2774, 3034] combine_rows_per_src=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=342536 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265501192 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530659848 weight_movement_bytes=34603008 total_nvlink_bytes=565262856 weight_to_token_ratio=0.065208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343808 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265371392 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398976 weight_movement_bytes=69206016 total_nvlink_bytes=599604992 weight_to_token_ratio=0.130479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344000 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584000 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824000 weight_movement_bytes=34603008 total_nvlink_bytes=567427008 weight_to_token_ratio=0.064943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344824 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267805432 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535266040 weight_movement_bytes=103809024 total_nvlink_bytes=639075064 weight_to_token_ratio=0.193939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=344656 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267280976 ingress_bytes=301539328 link_bytes=301539328 token_bytes=534217296 weight_movement_bytes=34603008 total_nvlink_bytes=568820304 weight_to_token_ratio=0.064773 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134668215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342536 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266254856 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532167176 weight_movement_bytes=69206016 total_nvlink_bytes=601373192 weight_to_token_ratio=0.130046 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64996 dup_rows=23926 dup_rows_remote=20884 dedup_ratio=0.243388 router_tokens_per_expert=[744, 789, 762, 760, 820, 774, 767, 759, 788, 782, 707, 726, 819, 778, 705, 764, 768, 808, 844, 796, 783, 764, 747, 807, 736, 793, 790, 740, 790, 757, 717, 755, 754, 732, 784, 741, 750, 756, 721, 799, 776, 725, 797, 765, 781, 768, 741, 778, 785, 783, 794, 792, 824, 711, 753, 770, 753, 707, 723, 781, 789, 793, 757, 830] payload_rows_per_dst=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] dup_rows_per_dst=[2956, 2938, 3114, 2928, 2874, 3012, 3042, 3062] combine_rows_per_src=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64984 dup_rows=23974 dup_rows_remote=20944 dedup_ratio=0.243876 router_tokens_per_expert=[777, 756, 755, 769, 766, 754, 793, 761, 738, 761, 774, 736, 797, 802, 837, 750, 720, 746, 785, 726, 739, 791, 772, 757, 742, 730, 751, 761, 726, 719, 804, 778, 727, 729, 767, 776, 813, 776, 807, 793, 780, 756, 785, 769, 807, 792, 745, 742, 777, 770, 789, 755, 810, 766, 761, 768, 776, 781, 803, 738, 800, 762, 811, 748] payload_rows_per_dst=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] dup_rows_per_dst=[3068, 3094, 2886, 2846, 3030, 2962, 3044, 3044] combine_rows_per_src=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343520 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266567136 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790752 weight_movement_bytes=69206016 total_nvlink_bytes=601996768 weight_to_token_ratio=0.129893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65170 dup_rows=23806 dup_rows_remote=20820 dedup_ratio=0.242167 router_tokens_per_expert=[720, 773, 831, 779, 793, 720, 788, 776, 770, 750, 773, 756, 741, 761, 806, 748, 734, 767, 756, 756, 824, 748, 791, 783, 751, 750, 816, 811, 761, 769, 732, 785, 756, 783, 753, 767, 741, 744, 738, 735, 828, 795, 754, 774, 770, 827, 769, 803, 755, 732, 773, 787, 731, 779, 780, 755, 785, 727, 800, 770, 771, 783, 755, 713] payload_rows_per_dst=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] dup_rows_per_dst=[2982, 2966, 2958, 3020, 2882, 3006, 3006, 2986] combine_rows_per_src=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343712 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518176 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692640 weight_movement_bytes=34603008 total_nvlink_bytes=567295648 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64826 dup_rows=24050 dup_rows_remote=20902 dedup_ratio=0.244649 router_tokens_per_expert=[760, 794, 738, 770, 760, 839, 696, 771, 768, 809, 810, 803, 801, 723, 797, 777, 758, 749, 744, 757, 828, 784, 774, 803, 794, 712, 810, 751, 781, 770, 746, 782, 744, 745, 765, 791, 781, 760, 804, 760, 779, 753, 766, 722, 772, 760, 765, 786, 756, 733, 786, 739, 784, 763, 768, 750, 747, 772, 786, 773, 776, 768, 737, 702] payload_rows_per_dst=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] dup_rows_per_dst=[2980, 3148, 3084, 2992, 2918, 2942, 2974, 3012] combine_rows_per_src=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64914 dup_rows=24174 dup_rows_remote=21198 dedup_ratio=0.245911 router_tokens_per_expert=[770, 796, 745, 768, 794, 743, 742, 761, 732, 778, 750, 754, 797, 766, 819, 802, 764, 794, 770, 718, 753, 768, 754, 758, 747, 750, 721, 727, 777, 789, 839, 746, 741, 725, 763, 787, 799, 758, 755, 769, 750, 797, 810, 819, 828, 744, 757, 736, 791, 790, 750, 717, 743, 771, 778, 780, 748, 774, 763, 765, 836, 786, 732, 798] payload_rows_per_dst=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] dup_rows_per_dst=[2968, 3070, 2974, 2976, 2966, 3192, 2938, 3090] combine_rows_per_src=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65054 dup_rows=23956 dup_rows_remote=20908 dedup_ratio=0.243693 router_tokens_per_expert=[783, 766, 774, 771, 743, 752, 740, 798, 800, 762, 807, 765, 724, 777, 802, 777, 841, 783, 751, 759, 735, 725, 793, 720, 794, 757, 730, 740, 744, 757, 773, 762, 752, 775, 778, 756, 757, 773, 746, 764, 776, 820, 757, 766, 707, 746, 802, 797, 801, 771, 777, 809, 780, 796, 751, 783, 762, 785, 749, 736, 746, 771, 772, 786] payload_rows_per_dst=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] dup_rows_per_dst=[2956, 3072, 2970, 2990, 2966, 3048, 2982, 2972] combine_rows_per_src=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64980 dup_rows=24182 dup_rows_remote=21224 dedup_ratio=0.245992 router_tokens_per_expert=[733, 760, 771, 730, 755, 725, 775, 778, 761, 790, 800, 772, 791, 755, 731, 805, 765, 732, 767, 745, 760, 739, 806, 736, 785, 757, 756, 789, 753, 803, 788, 742, 781, 797, 730, 757, 807, 762, 742, 809, 762, 781, 773, 759, 780, 738, 788, 729, 767, 818, 760, 764, 732, 795, 820, 778, 780, 721, 746, 763, 791, 825, 779, 763] payload_rows_per_dst=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] dup_rows_per_dst=[2858, 3032, 2958, 3066, 3092, 3018, 3054, 3104] combine_rows_per_src=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65048 dup_rows=24090 dup_rows_remote=21164 dedup_ratio=0.245056 router_tokens_per_expert=[748, 714, 775, 760, 727, 773, 779, 770, 778, 737, 802, 747, 740, 779, 775, 803, 742, 758, 757, 784, 732, 758, 806, 738, 747, 797, 778, 808, 729, 769, 708, 786, 787, 760, 756, 750, 768, 757, 820, 779, 751, 717, 767, 790, 795, 794, 772, 803, 817, 787, 730, 795, 766, 775, 758, 743, 742, 773, 804, 797, 767, 762, 804, 762] payload_rows_per_dst=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] dup_rows_per_dst=[2926, 3062, 3138, 3216, 3080, 3024, 2802, 2842] combine_rows_per_src=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343960 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267280280 ingress_bytes=336142336 link_bytes=336142336 token_bytes=534216600 weight_movement_bytes=69206016 total_nvlink_bytes=603422616 weight_to_token_ratio=0.129547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264876549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=342912 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870208 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397504 weight_movement_bytes=103809024 total_nvlink_bytes=635206528 weight_to_token_ratio=0.195351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=344448 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266232192 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119936 weight_movement_bytes=34603008 total_nvlink_bytes=566722944 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343848 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805032 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266216 weight_movement_bytes=34603008 total_nvlink_bytes=567869224 weight_to_token_ratio=0.064889 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344816 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266502896 ingress_bytes=369967104 link_bytes=369967104 token_bytes=532660976 weight_movement_bytes=103809024 total_nvlink_bytes=636470000 weight_to_token_ratio=0.194888 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392156428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344848 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266781456 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533218064 weight_movement_bytes=103809024 total_nvlink_bytes=637027088 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64806 dup_rows=24130 dup_rows_remote=21032 dedup_ratio=0.245463 router_tokens_per_expert=[755, 776, 770, 774, 787, 791, 752, 730, 752, 717, 800, 752, 763, 712, 779, 831, 728, 800, 825, 742, 787, 758, 828, 737, 783, 773, 753, 786, 782, 775, 758, 772, 703, 710, 782, 774, 736, 795, 752, 774, 798, 755, 794, 740, 756, 768, 731, 764, 806, 794, 788, 802, 751, 796, 745, 751, 799, 735, 761, 774, 757, 739, 795, 799] payload_rows_per_dst=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] dup_rows_per_dst=[3000, 3016, 3116, 3022, 2928, 2932, 3098, 3018] combine_rows_per_src=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64862 dup_rows=24204 dup_rows_remote=21140 dedup_ratio=0.246216 router_tokens_per_expert=[720, 772, 791, 771, 781, 759, 767, 783, 747, 755, 759, 723, 799, 738, 776, 780, 773, 773, 725, 801, 739, 791, 810, 728, 780, 775, 704, 746, 739, 791, 787, 764, 765, 765, 758, 780, 784, 758, 765, 776, 745, 827, 760, 826, 784, 761, 778, 774, 749, 717, 746, 775, 815, 767, 768, 763, 790, 800, 793, 773, 776, 731, 761, 775] payload_rows_per_dst=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] dup_rows_per_dst=[3030, 2972, 3054, 2972, 3064, 3082, 2956, 3074] combine_rows_per_src=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343352 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265788728 ingress_bytes=369254400 link_bytes=369254400 token_bytes=531234104 weight_movement_bytes=103809024 total_nvlink_bytes=635043128 weight_to_token_ratio=0.195411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389474580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344008 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018760 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693512 weight_movement_bytes=34603008 total_nvlink_bytes=566296520 weight_to_token_ratio=0.065081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65100 dup_rows=24062 dup_rows_remote=21088 dedup_ratio=0.244771 router_tokens_per_expert=[765, 763, 797, 815, 744, 768, 796, 770, 785, 783, 803, 836, 775, 795, 750, 763, 788, 746, 810, 728, 782, 825, 779, 790, 767, 745, 823, 793, 716, 757, 774, 704, 757, 745, 767, 783, 813, 754, 740, 726, 745, 743, 808, 764, 720, 757, 766, 765, 830, 776, 802, 759, 736, 776, 777, 770, 787, 745, 726, 755, 716, 687, 794, 728] payload_rows_per_dst=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] dup_rows_per_dst=[3082, 2940, 2972, 2862, 3032, 3186, 3014, 2974] combine_rows_per_src=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64990 dup_rows=24166 dup_rows_remote=21152 dedup_ratio=0.245829 router_tokens_per_expert=[786, 788, 698, 763, 734, 782, 752, 792, 788, 751, 756, 762, 794, 743, 740, 747, 750, 750, 751, 822, 819, 790, 749, 779, 757, 796, 695, 757, 764, 764, 746, 777, 799, 763, 740, 752, 780, 793, 746, 759, 784, 765, 794, 782, 804, 767, 785, 749, 762, 795, 773, 758, 725, 760, 760, 757, 744, 793, 826, 795, 822, 784, 782, 712] payload_rows_per_dst=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] dup_rows_per_dst=[2894, 3014, 3102, 2818, 3028, 3150, 2978, 3182] combine_rows_per_src=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65140 dup_rows=23872 dup_rows_remote=20858 dedup_ratio=0.242839 router_tokens_per_expert=[746, 817, 776, 759, 764, 790, 804, 735, 764, 781, 734, 778, 766, 733, 754, 770, 788, 754, 757, 751, 783, 810, 780, 722, 761, 750, 785, 754, 793, 761, 791, 758, 785, 748, 757, 786, 815, 755, 770, 795, 753, 738, 791, 741, 713, 781, 751, 772, 772, 765, 737, 788, 733, 771, 736, 807, 763, 760, 786, 799, 779, 799, 748, 789] payload_rows_per_dst=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] dup_rows_per_dst=[3010, 3016, 2952, 3014, 3078, 2866, 2872, 3064] combine_rows_per_src=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64928 dup_rows=24010 dup_rows_remote=20902 dedup_ratio=0.244242 router_tokens_per_expert=[761, 769, 768, 808, 770, 723, 756, 787, 810, 752, 798, 779, 778, 762, 765, 780, 741, 770, 721, 727, 734, 783, 781, 751, 763, 761, 768, 760, 766, 754, 761, 781, 734, 756, 762, 812, 758, 771, 805, 780, 768, 773, 809, 809, 768, 753, 754, 803, 744, 750, 740, 759, 759, 756, 735, 765, 782, 748, 797, 767, 780, 802, 768, 797] payload_rows_per_dst=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] dup_rows_per_dst=[2992, 3088, 2802, 3008, 3094, 3108, 2902, 3016] combine_rows_per_src=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64808 dup_rows=24162 dup_rows_remote=21158 dedup_ratio=0.245789 router_tokens_per_expert=[725, 734, 763, 769, 731, 763, 768, 773, 779, 780, 724, 735, 804, 774, 776, 827, 763, 789, 767, 785, 705, 804, 774, 782, 754, 734, 754, 787, 798, 783, 809, 775, 741, 773, 782, 733, 778, 745, 787, 732, 759, 780, 739, 821, 731, 787, 737, 823, 771, 814, 757, 732, 808, 786, 808, 748, 783, 742, 789, 763, 733, 748, 749, 785] payload_rows_per_dst=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] dup_rows_per_dst=[2956, 3030, 3004, 3000, 2946, 3070, 3168, 2988] combine_rows_per_src=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65134 dup_rows=23828 dup_rows_remote=20806 dedup_ratio=0.242391 router_tokens_per_expert=[721, 776, 798, 739, 803, 749, 735, 786, 766, 792, 728, 749, 775, 768, 814, 741, 813, 735, 776, 810, 739, 783, 782, 792, 714, 807, 778, 733, 744, 764, 755, 739, 813, 790, 748, 750, 754, 775, 767, 778, 764, 788, 772, 739, 775, 755, 776, 744, 734, 814, 727, 752, 801, 804, 790, 761, 788, 759, 777, 762, 723, 815, 791, 762] payload_rows_per_dst=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] dup_rows_per_dst=[3022, 2892, 3098, 3184, 3006, 2932, 2730, 2964] combine_rows_per_src=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344752 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266994352 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643952 weight_movement_bytes=34603008 total_nvlink_bytes=568246960 weight_to_token_ratio=0.064843 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344568 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266543608 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742648 weight_movement_bytes=103809024 total_nvlink_bytes=636551672 weight_to_token_ratio=0.194858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343992 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157432 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970872 weight_movement_bytes=34603008 total_nvlink_bytes=568573880 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343320 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288408 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233496 weight_movement_bytes=69206016 total_nvlink_bytes=601439512 weight_to_token_ratio=0.130029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343864 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797432 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531251000 weight_movement_bytes=69206016 total_nvlink_bytes=600457016 weight_to_token_ratio=0.130270 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343760 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132624 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921488 weight_movement_bytes=34603008 total_nvlink_bytes=568524496 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65094 dup_rows=23904 dup_rows_remote=20850 dedup_ratio=0.243164 router_tokens_per_expert=[777, 763, 789, 811, 750, 779, 793, 784, 767, 721, 767, 740, 774, 795, 796, 739, 752, 813, 771, 779, 767, 779, 779, 772, 788, 731, 718, 793, 771, 744, 775, 758, 800, 747, 777, 789, 785, 754, 775, 779, 752, 756, 779, 707, 757, 770, 787, 757, 721, 790, 765, 793, 790, 760, 755, 806, 775, 748, 724, 787, 749, 769, 773, 741] payload_rows_per_dst=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] dup_rows_per_dst=[3066, 2922, 3134, 2850, 3072, 2910, 3054, 2896] combine_rows_per_src=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64786 dup_rows=24300 dup_rows_remote=21310 dedup_ratio=0.247192 router_tokens_per_expert=[772, 737, 779, 734, 733, 753, 765, 755, 769, 771, 773, 788, 768, 783, 703, 739, 771, 790, 762, 735, 808, 768, 772, 755, 801, 750, 727, 783, 773, 853, 792, 763, 769, 760, 734, 775, 797, 784, 764, 721, 753, 773, 750, 762, 733, 778, 788, 740, 780, 785, 811, 759, 777, 800, 762, 801, 812, 802, 772, 729, 783, 780, 721, 772] payload_rows_per_dst=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] dup_rows_per_dst=[2976, 2944, 2986, 3118, 2990, 3038, 3202, 3046] combine_rows_per_src=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343776 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968800 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593824 weight_movement_bytes=69206016 total_nvlink_bytes=602799840 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344384 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707840 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071296 weight_movement_bytes=34603008 total_nvlink_bytes=565674304 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65100 dup_rows=23904 dup_rows_remote=20922 dedup_ratio=0.243164 router_tokens_per_expert=[761, 701, 784, 806, 757, 762, 788, 790, 805, 737, 753, 768, 773, 782, 730, 793, 790, 742, 754, 759, 777, 790, 739, 751, 765, 811, 750, 751, 721, 760, 740, 759, 799, 734, 742, 782, 737, 770, 809, 761, 851, 764, 829, 755, 747, 785, 707, 756, 791, 786, 785, 769, 785, 797, 797, 742, 733, 748, 793, 797, 722, 753, 795, 782] payload_rows_per_dst=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] dup_rows_per_dst=[3020, 2982, 3000, 2902, 2938, 3062, 3086, 2914] combine_rows_per_src=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65084 dup_rows=23886 dup_rows_remote=20902 dedup_ratio=0.242981 router_tokens_per_expert=[789, 773, 759, 768, 738, 790, 761, 776, 772, 715, 778, 787, 756, 797, 736, 802, 761, 768, 763, 763, 760, 771, 777, 754, 742, 815, 737, 783, 757, 745, 799, 709, 793, 738, 725, 754, 764, 807, 760, 804, 790, 791, 788, 730, 760, 768, 756, 785, 769, 784, 794, 773, 764, 766, 784, 745, 767, 786, 826, 773, 753, 748, 761, 745] payload_rows_per_dst=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] dup_rows_per_dst=[3014, 2914, 2956, 2932, 2972, 2982, 3132, 2984] combine_rows_per_src=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65134 dup_rows=23928 dup_rows_remote=20876 dedup_ratio=0.243408 router_tokens_per_expert=[782, 726, 725, 743, 792, 793, 801, 756, 776, 795, 795, 752, 735, 807, 778, 716, 713, 794, 750, 809, 787, 794, 739, 789, 831, 763, 763, 781, 726, 738, 765, 766, 746, 755, 790, 737, 749, 785, 747, 775, 791, 763, 747, 754, 785, 774, 779, 754, 782, 783, 786, 793, 791, 781, 788, 788, 733, 810, 757, 787, 753, 739, 741, 729] payload_rows_per_dst=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] dup_rows_per_dst=[3030, 2936, 3032, 2960, 3008, 3052, 3072, 2838] combine_rows_per_src=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64900 dup_rows=24042 dup_rows_remote=20970 dedup_ratio=0.244568 router_tokens_per_expert=[737, 746, 776, 768, 754, 756, 766, 710, 746, 758, 783, 759, 781, 811, 779, 736, 744, 787, 754, 753, 732, 812, 803, 773, 776, 768, 760, 802, 774, 763, 783, 791, 745, 727, 786, 765, 807, 717, 728, 730, 774, 785, 794, 825, 759, 786, 753, 786, 812, 732, 789, 770, 756, 792, 785, 769, 743, 745, 793, 778, 751, 740, 777, 812] payload_rows_per_dst=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] dup_rows_per_dst=[2870, 2952, 3076, 3072, 2852, 3194, 3036, 2990] combine_rows_per_src=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65124 dup_rows=24032 dup_rows_remote=21214 dedup_ratio=0.244466 router_tokens_per_expert=[752, 773, 785, 763, 757, 787, 794, 759, 763, 737, 777, 791, 769, 775, 769, 795, 740, 745, 777, 754, 719, 720, 761, 767, 820, 784, 783, 778, 759, 773, 811, 771, 785, 744, 773, 751, 765, 745, 792, 781, 762, 755, 793, 710, 767, 751, 762, 777, 736, 787, 791, 855, 767, 748, 789, 791, 715, 790, 779, 808, 751, 726, 740, 758] payload_rows_per_dst=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] dup_rows_per_dst=[3060, 3076, 2818, 3158, 2998, 2892, 3062, 2968] combine_rows_per_src=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64732 dup_rows=24012 dup_rows_remote=20676 dedup_ratio=0.244263 router_tokens_per_expert=[751, 753, 710, 733, 802, 774, 779, 798, 748, 776, 786, 750, 759, 823, 794, 788, 760, 813, 746, 758, 760, 785, 779, 795, 774, 769, 786, 793, 738, 732, 770, 755, 787, 728, 788, 760, 715, 753, 736, 746, 759, 772, 753, 804, 783, 731, 792, 748, 714, 787, 741, 737, 784, 790, 840, 784, 740, 735, 806, 761, 784, 744, 816, 797] payload_rows_per_dst=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] dup_rows_per_dst=[3336, 3078, 3038, 2992, 3094, 2840, 2734, 2900] combine_rows_per_src=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344088 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266993688 ingress_bytes=405061632 link_bytes=405061632 token_bytes=533643288 weight_movement_bytes=138412032 total_nvlink_bytes=672055320 weight_to_token_ratio=0.259372 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=343944 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266928008 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512072 weight_movement_bytes=69206016 total_nvlink_bytes=602718088 weight_to_token_ratio=0.129718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343480 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173880 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004280 weight_movement_bytes=34603008 total_nvlink_bytes=566607288 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=345352 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267093256 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533841160 weight_movement_bytes=34603008 total_nvlink_bytes=568444168 weight_to_token_ratio=0.064819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341632 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483904 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626176 weight_movement_bytes=34603008 total_nvlink_bytes=565229184 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64890 dup_rows=24172 dup_rows_remote=21252 dedup_ratio=0.245890 router_tokens_per_expert=[774, 752, 775, 737, 796, 775, 794, 793, 784, 792, 727, 811, 708, 822, 728, 787, 777, 828, 745, 717, 809, 770, 737, 782, 767, 771, 787, 750, 774, 752, 775, 797, 758, 754, 746, 783, 745, 762, 817, 783, 797, 758, 807, 802, 762, 706, 683, 756, 767, 821, 746, 722, 769, 775, 720, 761, 786, 733, 734, 766, 761, 781, 761, 837] payload_rows_per_dst=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] dup_rows_per_dst=[3010, 3062, 2992, 3092, 3028, 2988, 2920, 3080] combine_rows_per_src=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64988 dup_rows=23966 dup_rows_remote=20808 dedup_ratio=0.243795 router_tokens_per_expert=[762, 769, 771, 753, 707, 845, 810, 777, 781, 773, 755, 766, 768, 758, 752, 759, 806, 752, 780, 721, 756, 778, 798, 711, 771, 734, 790, 749, 778, 749, 792, 754, 768, 754, 798, 800, 797, 750, 764, 823, 776, 757, 769, 762, 755, 821, 763, 798, 765, 758, 786, 731, 768, 759, 749, 760, 784, 737, 753, 789, 723, 780, 772, 758] payload_rows_per_dst=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] dup_rows_per_dst=[3004, 2984, 2960, 2958, 3158, 2970, 2942, 2990] combine_rows_per_src=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344568 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266134008 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923448 weight_movement_bytes=69206016 total_nvlink_bytes=601129464 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64974 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[757, 823, 788, 790, 818, 723, 753, 728, 768, 722, 742, 810, 773, 752, 800, 789, 786, 699, 762, 789, 747, 789, 750, 803, 793, 726, 740, 770, 754, 765, 725, 747, 746, 766, 741, 780, 782, 784, 770, 767, 751, 790, 737, 752, 787, 799, 803, 772, 759, 739, 771, 791, 775, 786, 771, 751, 749, 784, 791, 809, 778, 784, 738, 768] payload_rows_per_dst=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] dup_rows_per_dst=[3064, 2994, 2976, 2916, 2988, 3032, 2972, 3070] combine_rows_per_src=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343184 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534032 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532724880 weight_movement_bytes=34603008 total_nvlink_bytes=567327888 weight_to_token_ratio=0.064955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64710 dup_rows=24134 dup_rows_remote=20908 dedup_ratio=0.245504 router_tokens_per_expert=[752, 709, 715, 774, 763, 788, 743, 801, 796, 827, 745, 749, 818, 766, 774, 801, 739, 727, 782, 740, 757, 766, 765, 779, 777, 785, 815, 761, 767, 803, 758, 760, 743, 762, 770, 747, 777, 755, 775, 765, 774, 758, 759, 797, 752, 791, 739, 800, 747, 764, 732, 756, 766, 754, 768, 795, 766, 803, 783, 773, 795, 730, 770, 784] payload_rows_per_dst=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] dup_rows_per_dst=[2944, 2888, 2932, 2894, 3064, 3172, 3014, 3226] combine_rows_per_src=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65026 dup_rows=24022 dup_rows_remote=20938 dedup_ratio=0.244364 router_tokens_per_expert=[803, 745, 771, 776, 800, 755, 738, 752, 746, 757, 744, 721, 759, 737, 780, 794, 763, 739, 773, 770, 813, 800, 744, 759, 782, 761, 744, 757, 775, 786, 768, 748, 772, 788, 734, 760, 790, 797, 782, 798, 768, 744, 781, 786, 810, 756, 769, 756, 755, 757, 756, 775, 793, 798, 722, 784, 747, 731, 744, 788, 768, 803, 797, 783] payload_rows_per_dst=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] dup_rows_per_dst=[2948, 2926, 3066, 2942, 3054, 3084, 2992, 3010] combine_rows_per_src=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64822 dup_rows=24056 dup_rows_remote=20910 dedup_ratio=0.244710 router_tokens_per_expert=[725, 741, 748, 771, 728, 795, 697, 800, 727, 749, 754, 741, 782, 790, 734, 787, 823, 781, 731, 773, 828, 787, 762, 801, 768, 748, 775, 752, 774, 795, 742, 791, 760, 812, 725, 797, 760, 754, 778, 802, 780, 743, 778, 778, 785, 789, 764, 800, 763, 810, 773, 777, 728, 773, 779, 784, 749, 741, 739, 797, 784, 744, 749, 757] payload_rows_per_dst=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] dup_rows_per_dst=[2912, 2884, 3146, 3036, 3016, 3074, 3044, 2944] combine_rows_per_src=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64770 dup_rows=24190 dup_rows_remote=21094 dedup_ratio=0.246073 router_tokens_per_expert=[820, 743, 754, 728, 764, 751, 781, 728, 780, 792, 787, 775, 773, 728, 767, 772, 720, 755, 776, 737, 772, 794, 817, 737, 755, 738, 763, 776, 813, 802, 806, 767, 788, 708, 765, 747, 713, 797, 752, 772, 786, 741, 791, 742, 771, 744, 714, 750, 789, 766, 845, 830, 754, 783, 752, 748, 807, 800, 785, 772, 770, 770, 714, 815] payload_rows_per_dst=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] dup_rows_per_dst=[2962, 2962, 3032, 3096, 2972, 2926, 3170, 3070] combine_rows_per_src=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64858 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[758, 711, 790, 765, 804, 748, 762, 804, 779, 797, 782, 744, 811, 770, 767, 760, 762, 690, 805, 743, 797, 772, 792, 778, 770, 777, 770, 760, 790, 780, 745, 771, 764, 779, 761, 723, 813, 776, 727, 784, 803, 768, 766, 724, 785, 796, 753, 770, 747, 727, 775, 776, 789, 754, 743, 813, 749, 750, 707, 800, 740, 825, 775, 736] payload_rows_per_dst=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] dup_rows_per_dst=[3022, 3108, 3344, 2942, 3036, 2992, 2684, 2918] combine_rows_per_src=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343968 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477472 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532610976 weight_movement_bytes=34603008 total_nvlink_bytes=567213984 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=342472 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265394632 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446792 weight_movement_bytes=34603008 total_nvlink_bytes=565049800 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343856 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266690352 ingress_bytes=335552512 link_bytes=335552512 token_bytes=533036848 weight_movement_bytes=69206016 total_nvlink_bytes=602242864 weight_to_token_ratio=0.129833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262657088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342928 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265853840 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531364752 weight_movement_bytes=69206016 total_nvlink_bytes=600570768 weight_to_token_ratio=0.130242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=343456 combine_payload_bytes=265297920 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265641376 ingress_bytes=369106944 link_bytes=369106944 token_bytes=530939296 weight_movement_bytes=103809024 total_nvlink_bytes=634748320 weight_to_token_ratio=0.195520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343528 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266001896 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660264 weight_movement_bytes=34603008 total_nvlink_bytes=566263272 weight_to_token_ratio=0.065085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65274 dup_rows=23790 dup_rows_remote=20824 dedup_ratio=0.242004 router_tokens_per_expert=[776, 766, 776, 784, 702, 775, 796, 730, 770, 758, 767, 769, 775, 748, 790, 807, 756, 769, 751, 777, 764, 830, 766, 806, 712, 755, 771, 733, 766, 744, 743, 737, 741, 784, 727, 734, 815, 783, 803, 798, 738, 802, 774, 790, 793, 806, 741, 725, 801, 774, 779, 681, 736, 822, 755, 755, 769, 806, 806, 786, 774, 781, 757, 747] payload_rows_per_dst=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] dup_rows_per_dst=[2932, 2996, 3026, 2818, 3070, 2956, 2966, 3026] combine_rows_per_src=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64738 dup_rows=24380 dup_rows_remote=21362 dedup_ratio=0.248006 router_tokens_per_expert=[721, 790, 775, 838, 759, 764, 789, 726, 724, 773, 780, 720, 758, 795, 744, 752, 810, 792, 771, 771, 754, 806, 768, 759, 755, 781, 751, 742, 790, 769, 753, 759, 776, 775, 778, 739, 751, 746, 724, 813, 712, 757, 811, 748, 773, 777, 795, 826, 783, 752, 777, 730, 736, 816, 763, 822, 791, 780, 735, 792, 721, 711, 803, 800] payload_rows_per_dst=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] dup_rows_per_dst=[3084, 2842, 3188, 3054, 3018, 3106, 3040, 3048] combine_rows_per_src=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65118 dup_rows=23854 dup_rows_remote=20932 dedup_ratio=0.242655 router_tokens_per_expert=[796, 731, 774, 759, 754, 771, 770, 753, 765, 779, 785, 759, 756, 750, 772, 761, 788, 750, 780, 772, 759, 742, 731, 772, 798, 794, 750, 824, 705, 753, 756, 775, 768, 773, 772, 780, 758, 766, 757, 764, 742, 792, 810, 738, 756, 790, 786, 797, 775, 737, 766, 832, 784, 729, 778, 811, 746, 767, 773, 789, 731, 752, 787, 762] payload_rows_per_dst=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] dup_rows_per_dst=[2936, 2922, 2944, 2996, 3042, 3060, 3038, 2916] combine_rows_per_src=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344392 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706696 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069000 weight_movement_bytes=69206016 total_nvlink_bytes=604275016 weight_to_token_ratio=0.129340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=344400 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265511248 ingress_bytes=334372864 link_bytes=334372864 token_bytes=530678096 weight_movement_bytes=69206016 total_nvlink_bytes=599884112 weight_to_token_ratio=0.130411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258218168e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65304 dup_rows=23700 dup_rows_remote=20700 dedup_ratio=0.241089 router_tokens_per_expert=[836, 756, 726, 756, 788, 758, 770, 712, 769, 791, 791, 749, 775, 739, 725, 768, 785, 805, 771, 777, 804, 797, 740, 757, 769, 775, 757, 689, 734, 740, 787, 813, 813, 763, 752, 796, 775, 755, 768, 716, 757, 755, 790, 772, 771, 771, 734, 800, 766, 732, 813, 807, 804, 721, 784, 795, 759, 753, 774, 788, 733, 780, 775, 771] payload_rows_per_dst=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] dup_rows_per_dst=[2890, 2916, 3038, 2888, 2944, 3000, 3082, 2942] combine_rows_per_src=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65054 dup_rows=24010 dup_rows_remote=20948 dedup_ratio=0.244242 router_tokens_per_expert=[784, 758, 768, 756, 762, 749, 785, 772, 791, 813, 761, 759, 755, 809, 776, 779, 803, 806, 785, 757, 768, 720, 783, 805, 721, 724, 751, 733, 773, 769, 716, 680, 765, 746, 769, 767, 751, 761, 775, 747, 778, 819, 718, 804, 781, 759, 811, 748, 810, 768, 730, 759, 767, 812, 796, 789, 730, 752, 767, 801, 773, 785, 789, 754] payload_rows_per_dst=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] dup_rows_per_dst=[2998, 3030, 2836, 2758, 2892, 3386, 3048, 3062] combine_rows_per_src=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64652 dup_rows=24202 dup_rows_remote=21020 dedup_ratio=0.246195 router_tokens_per_expert=[759, 739, 765, 754, 775, 751, 744, 782, 723, 729, 743, 745, 775, 835, 743, 757, 807, 789, 759, 815, 776, 801, 779, 790, 806, 766, 774, 782, 732, 784, 730, 791, 764, 764, 739, 789, 761, 764, 759, 776, 712, 776, 754, 767, 784, 786, 796, 746, 762, 783, 759, 791, 779, 736, 792, 752, 772, 747, 814, 783, 779, 743, 784, 739] payload_rows_per_dst=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] dup_rows_per_dst=[2926, 2906, 3182, 3054, 3002, 3056, 2964, 3112] combine_rows_per_src=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64872 dup_rows=24108 dup_rows_remote=21104 dedup_ratio=0.245239 router_tokens_per_expert=[728, 774, 736, 762, 759, 785, 760, 782, 793, 781, 781, 784, 754, 741, 751, 764, 742, 742, 768, 779, 728, 781, 768, 783, 740, 788, 747, 762, 735, 757, 805, 830, 750, 763, 754, 800, 754, 763, 741, 797, 772, 800, 791, 779, 761, 735, 754, 722, 800, 775, 771, 776, 751, 757, 772, 792, 765, 718, 811, 807, 779, 791, 745, 816] payload_rows_per_dst=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] dup_rows_per_dst=[2944, 3062, 2942, 3004, 3024, 2964, 3124, 3044] combine_rows_per_src=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64910 dup_rows=23906 dup_rows_remote=20714 dedup_ratio=0.243184 router_tokens_per_expert=[733, 763, 770, 795, 769, 761, 800, 757, 770, 807, 786, 767, 776, 737, 792, 765, 723, 771, 719, 738, 786, 807, 760, 775, 753, 862, 777, 748, 788, 782, 773, 744, 799, 770, 739, 748, 777, 786, 782, 769, 751, 749, 769, 785, 764, 772, 783, 779, 789, 769, 757, 772, 762, 733, 748, 781, 751, 770, 755, 741, 762, 779, 733, 774] payload_rows_per_dst=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] dup_rows_per_dst=[3192, 3132, 2986, 3572, 2986, 2474, 2674, 2890] combine_rows_per_src=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344200 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067528 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790856 weight_movement_bytes=34603008 total_nvlink_bytes=568393864 weight_to_token_ratio=0.064825 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344016 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267829200 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535314384 weight_movement_bytes=103809024 total_nvlink_bytes=639123408 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344008 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805192 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266376 weight_movement_bytes=103809024 total_nvlink_bytes=637075400 weight_to_token_ratio=0.194666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342688 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265157280 ingress_bytes=299417600 link_bytes=299417600 token_bytes=529971872 weight_movement_bytes=34603008 total_nvlink_bytes=564574880 weight_to_token_ratio=0.065292 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126684324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343904 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059616 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775328 weight_movement_bytes=34603008 total_nvlink_bytes=566378336 weight_to_token_ratio=0.065071 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=342496 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266213856 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532085216 weight_movement_bytes=34603008 total_nvlink_bytes=566688224 weight_to_token_ratio=0.065033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65174 dup_rows=24028 dup_rows_remote=21148 dedup_ratio=0.244425 router_tokens_per_expert=[788, 753, 765, 796, 731, 805, 784, 743, 718, 818, 728, 750, 778, 742, 752, 745, 747, 786, 789, 745, 772, 779, 793, 753, 764, 795, 783, 758, 800, 789, 783, 786, 739, 751, 728, 772, 747, 719, 762, 773, 794, 729, 768, 802, 746, 785, 761, 798, 762, 748, 743, 782, 866, 801, 743, 753, 750, 794, 776, 816, 762, 736, 760, 768] payload_rows_per_dst=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] dup_rows_per_dst=[3020, 2924, 3070, 3100, 2880, 2990, 3050, 2994] combine_rows_per_src=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64872 dup_rows=24158 dup_rows_remote=21150 dedup_ratio=0.245748 router_tokens_per_expert=[768, 764, 749, 727, 774, 737, 752, 776, 810, 816, 760, 783, 741, 775, 725, 792, 725, 786, 766, 792, 738, 792, 738, 783, 795, 747, 781, 752, 783, 782, 796, 769, 780, 751, 776, 770, 767, 742, 775, 757, 781, 792, 775, 821, 673, 745, 770, 744, 797, 755, 742, 777, 770, 768, 778, 754, 816, 750, 726, 750, 788, 787, 790, 811] payload_rows_per_dst=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] dup_rows_per_dst=[2978, 3024, 2994, 3022, 3052, 2976, 3008, 3104] combine_rows_per_src=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=345288 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297992 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250696 weight_movement_bytes=34603008 total_nvlink_bytes=568853704 weight_to_token_ratio=0.064769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344088 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266059800 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775512 weight_movement_bytes=103809024 total_nvlink_bytes=635584536 weight_to_token_ratio=0.195212 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64980 dup_rows=24080 dup_rows_remote=21100 dedup_ratio=0.244954 router_tokens_per_expert=[754, 727, 715, 791, 744, 779, 758, 743, 801, 757, 779, 767, 775, 745, 754, 734, 759, 796, 770, 783, 791, 795, 783, 794, 776, 787, 751, 770, 772, 791, 767, 778, 828, 742, 774, 774, 751, 791, 725, 740, 760, 757, 806, 730, 791, 740, 776, 752, 711, 786, 822, 750, 776, 767, 815, 790, 786, 778, 767, 774, 744, 772, 736, 755] payload_rows_per_dst=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] dup_rows_per_dst=[2928, 2980, 3012, 3064, 3042, 2946, 3106, 3002] combine_rows_per_src=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73818 payload_rows_remote=64530 dup_rows=24486 dup_rows_remote=21518 dedup_ratio=0.249084 router_tokens_per_expert=[741, 727, 792, 779, 754, 777, 777, 737, 728, 820, 786, 792, 723, 753, 782, 786, 764, 783, 754, 815, 771, 745, 777, 793, 709, 751, 775, 785, 741, 794, 754, 790, 794, 794, 751, 803, 740, 751, 753, 752, 783, 730, 783, 755, 756, 776, 799, 746, 779, 779, 781, 790, 771, 778, 785, 785, 729, 740, 740, 755, 826, 757, 755, 781] payload_rows_per_dst=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] dup_rows_per_dst=[3038, 3056, 3200, 3008, 3050, 2968, 3112, 3054] combine_rows_per_src=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65104 dup_rows=24098 dup_rows_remote=21214 dedup_ratio=0.245138 router_tokens_per_expert=[772, 805, 764, 750, 768, 764, 765, 792, 815, 788, 753, 780, 745, 744, 756, 761, 752, 717, 724, 773, 781, 771, 734, 741, 749, 816, 769, 768, 739, 729, 796, 809, 763, 762, 793, 776, 787, 755, 786, 765, 753, 745, 786, 806, 775, 751, 761, 735, 809, 753, 807, 780, 766, 784, 757, 832, 794, 760, 781, 705, 773, 730, 729, 803] payload_rows_per_dst=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] dup_rows_per_dst=[3000, 3090, 2884, 3092, 3082, 2942, 3114, 2894] combine_rows_per_src=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65186 dup_rows=23848 dup_rows_remote=20752 dedup_ratio=0.242594 router_tokens_per_expert=[762, 772, 761, 805, 789, 797, 752, 751, 797, 755, 761, 760, 744, 758, 760, 740, 766, 747, 808, 715, 751, 750, 773, 769, 798, 760, 783, 772, 781, 756, 769, 760, 744, 750, 723, 753, 771, 761, 753, 753, 803, 723, 796, 799, 804, 780, 757, 799, 740, 786, 806, 784, 795, 759, 794, 770, 753, 780, 759, 724, 754, 755, 786, 816] payload_rows_per_dst=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] dup_rows_per_dst=[3044, 2828, 2932, 2938, 2896, 3000, 3114, 3096] combine_rows_per_src=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20782 dedup_ratio=0.243022 router_tokens_per_expert=[830, 732, 798, 730, 797, 779, 791, 713, 784, 797, 769, 802, 783, 740, 759, 771, 751, 745, 731, 738, 746, 770, 773, 757, 765, 768, 808, 812, 797, 757, 713, 779, 798, 761, 774, 719, 780, 768, 754, 764, 755, 808, 799, 787, 774, 773, 785, 772, 792, 769, 746, 757, 740, 769, 770, 737, 742, 785, 805, 765, 784, 756, 734, 745] payload_rows_per_dst=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] dup_rows_per_dst=[3060, 3012, 2836, 3108, 2936, 3106, 2904, 2928] combine_rows_per_src=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64658 dup_rows=24190 dup_rows_remote=21042 dedup_ratio=0.246073 router_tokens_per_expert=[811, 725, 754, 783, 764, 747, 762, 764, 752, 774, 782, 776, 738, 744, 759, 798, 762, 733, 730, 745, 780, 774, 756, 731, 759, 785, 792, 723, 792, 768, 781, 789, 828, 780, 739, 755, 766, 760, 786, 768, 764, 812, 783, 791, 768, 770, 792, 768, 767, 819, 772, 745, 782, 759, 795, 778, 735, 784, 767, 751, 698, 791, 775, 771] payload_rows_per_dst=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] dup_rows_per_dst=[3148, 3230, 3050, 2938, 3352, 2784, 2794, 2894] combine_rows_per_src=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344320 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266502400 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660480 weight_movement_bytes=69206016 total_nvlink_bytes=601866496 weight_to_token_ratio=0.129925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264314880 dispatch_meta_bytes=344192 combine_payload_bytes=264314880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264659072 ingress_bytes=333520896 link_bytes=333520896 token_bytes=528973952 weight_movement_bytes=69206016 total_nvlink_bytes=598179968 weight_to_token_ratio=0.130831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255012281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=345272 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267011256 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533677240 weight_movement_bytes=69206016 total_nvlink_bytes=602883256 weight_to_token_ratio=0.129678 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343624 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267091528 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839432 weight_movement_bytes=103809024 total_nvlink_bytes=637648456 weight_to_token_ratio=0.194457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343752 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345608 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347464 weight_movement_bytes=34603008 total_nvlink_bytes=568950472 weight_to_token_ratio=0.064758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65288 dup_rows=23886 dup_rows_remote=20984 dedup_ratio=0.242981 router_tokens_per_expert=[741, 764, 750, 772, 783, 761, 781, 772, 736, 764, 775, 765, 777, 772, 770, 768, 718, 753, 791, 757, 794, 764, 757, 804, 803, 801, 741, 783, 784, 766, 759, 786, 706, 772, 740, 738, 749, 783, 778, 750, 740, 778, 755, 719, 800, 825, 813, 759, 801, 794, 736, 789, 769, 771, 774, 794, 746, 756, 758, 742, 791, 757, 798, 759] payload_rows_per_dst=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] dup_rows_per_dst=[2926, 2960, 2936, 3060, 2902, 3020, 3066, 3016] combine_rows_per_src=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65132 dup_rows=23962 dup_rows_remote=20920 dedup_ratio=0.243754 router_tokens_per_expert=[764, 773, 783, 826, 724, 751, 791, 777, 750, 761, 775, 756, 784, 742, 772, 777, 739, 792, 776, 756, 756, 760, 790, 786, 772, 771, 756, 780, 812, 767, 751, 767, 816, 775, 776, 763, 736, 755, 748, 742, 789, 759, 740, 762, 828, 740, 783, 751, 784, 732, 770, 747, 786, 796, 739, 772, 758, 765, 758, 766, 774, 796, 744, 765] payload_rows_per_dst=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] dup_rows_per_dst=[3022, 3016, 2946, 3032, 2986, 2944, 3042, 2974] combine_rows_per_src=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342800 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181968 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530021136 weight_movement_bytes=34603008 total_nvlink_bytes=564624144 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=345088 combine_payload_bytes=267419648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267764736 ingress_bytes=302022656 link_bytes=302022656 token_bytes=535184384 weight_movement_bytes=34603008 total_nvlink_bytes=569787392 weight_to_token_ratio=0.064656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136486940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344208 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267124880 ingress_bytes=370589696 link_bytes=370589696 token_bytes=533905552 weight_movement_bytes=103809024 total_nvlink_bytes=637714576 weight_to_token_ratio=0.194433 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394499191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65186 dup_rows=23816 dup_rows_remote=20840 dedup_ratio=0.242269 router_tokens_per_expert=[767, 768, 775, 725, 743, 794, 787, 783, 755, 745, 777, 786, 708, 807, 775, 786, 747, 793, 766, 732, 771, 731, 740, 825, 782, 753, 782, 763, 785, 790, 780, 757, 744, 750, 716, 829, 772, 741, 793, 795, 787, 792, 760, 787, 756, 737, 811, 777, 749, 787, 753, 780, 771, 778, 727, 758, 727, 783, 779, 753, 785, 817, 709, 771] payload_rows_per_dst=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] dup_rows_per_dst=[3014, 2976, 2968, 3010, 2920, 3046, 2936, 2946] combine_rows_per_src=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64886 dup_rows=24160 dup_rows_remote=21078 dedup_ratio=0.245768 router_tokens_per_expert=[777, 768, 809, 775, 738, 728, 810, 769, 728, 761, 755, 745, 764, 762, 751, 761, 742, 789, 730, 750, 790, 754, 796, 756, 798, 753, 784, 780, 839, 744, 731, 724, 747, 778, 729, 797, 773, 750, 758, 809, 772, 749, 770, 768, 798, 759, 774, 780, 809, 751, 780, 810, 748, 808, 808, 786, 761, 784, 785, 728, 772, 746, 737, 767] payload_rows_per_dst=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] dup_rows_per_dst=[3024, 2936, 2954, 3004, 3020, 3082, 3132, 3008] combine_rows_per_src=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64666 dup_rows=24154 dup_rows_remote=21130 dedup_ratio=0.245707 router_tokens_per_expert=[782, 808, 818, 755, 783, 744, 693, 751, 724, 812, 776, 740, 794, 859, 769, 796, 758, 764, 810, 785, 769, 736, 822, 810, 783, 791, 728, 780, 749, 778, 757, 784, 800, 751, 759, 777, 755, 744, 771, 737, 727, 756, 820, 767, 776, 746, 772, 749, 764, 799, 746, 751, 779, 750, 757, 736, 778, 803, 763, 745, 773, 757, 738, 698] payload_rows_per_dst=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] dup_rows_per_dst=[3068, 3092, 3024, 3066, 2984, 3008, 2978, 2934] combine_rows_per_src=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64432 dup_rows=24150 dup_rows_remote=20778 dedup_ratio=0.245667 router_tokens_per_expert=[745, 756, 698, 796, 747, 831, 737, 710, 805, 725, 733, 762, 782, 855, 802, 757, 786, 748, 801, 716, 776, 779, 764, 769, 779, 783, 799, 785, 797, 778, 755, 797, 729, 780, 791, 770, 754, 795, 760, 725, 761, 771, 786, 750, 777, 793, 750, 741, 805, 783, 754, 803, 743, 747, 729, 771, 762, 773, 774, 731, 807, 744, 779, 761] payload_rows_per_dst=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] dup_rows_per_dst=[2926, 3072, 2686, 3186, 3358, 2960, 2590, 3372] combine_rows_per_src=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65128 dup_rows=24002 dup_rows_remote=21126 dedup_ratio=0.244161 router_tokens_per_expert=[801, 773, 799, 729, 796, 792, 818, 759, 803, 774, 746, 750, 761, 791, 772, 816, 735, 792, 773, 761, 794, 795, 770, 780, 778, 771, 772, 723, 765, 720, 748, 748, 761, 748, 754, 754, 779, 769, 726, 784, 736, 812, 805, 825, 777, 776, 748, 761, 713, 749, 795, 742, 763, 752, 812, 754, 758, 723, 805, 751, 730, 758, 793, 734] payload_rows_per_dst=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] dup_rows_per_dst=[3132, 3136, 3020, 2876, 3012, 3086, 2906, 2834] combine_rows_per_src=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344104 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345960 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347816 weight_movement_bytes=34603008 total_nvlink_bytes=568950824 weight_to_token_ratio=0.064757 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=343184 combine_payload_bytes=264871936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265215120 ingress_bytes=403283968 link_bytes=403283968 token_bytes=530087056 weight_movement_bytes=138412032 total_nvlink_bytes=668499088 weight_to_token_ratio=0.261112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517525105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343856 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116912 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889968 weight_movement_bytes=138412032 total_nvlink_bytes=670302000 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64830 dup_rows=24096 dup_rows_remote=20920 dedup_ratio=0.245117 router_tokens_per_expert=[785, 824, 713, 776, 796, 770, 821, 750, 750, 717, 799, 799, 743, 743, 762, 787, 778, 782, 832, 754, 741, 772, 790, 771, 781, 739, 755, 779, 792, 745, 735, 728, 715, 811, 754, 777, 754, 731, 754, 796, 776, 794, 736, 757, 784, 749, 793, 754, 789, 803, 764, 768, 782, 773, 817, 762, 775, 739, 741, 726, 754, 759, 772, 784] payload_rows_per_dst=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] dup_rows_per_dst=[3176, 2970, 3228, 2896, 2964, 2954, 3006, 2902] combine_rows_per_src=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263913472 dispatch_meta_bytes=340840 combine_payload_bytes=263913472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264254312 ingress_bytes=298516480 link_bytes=298516480 token_bytes=528167784 weight_movement_bytes=34603008 total_nvlink_bytes=562770792 weight_to_token_ratio=0.065515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123293482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=345016 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267109304 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533873592 weight_movement_bytes=69206016 total_nvlink_bytes=603079608 weight_to_token_ratio=0.129630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65052 dup_rows=23858 dup_rows_remote=20868 dedup_ratio=0.242696 router_tokens_per_expert=[761, 783, 735, 766, 761, 814, 843, 790, 737, 782, 768, 780, 791, 793, 814, 781, 780, 734, 744, 790, 775, 799, 800, 764, 739, 756, 750, 743, 711, 745, 809, 818, 790, 802, 749, 761, 760, 811, 776, 743, 775, 752, 797, 796, 743, 745, 719, 786, 723, 758, 761, 799, 781, 723, 746, 741, 733, 768, 782, 764, 797, 728, 755, 732] payload_rows_per_dst=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] dup_rows_per_dst=[3130, 3054, 3010, 2920, 2990, 2920, 2876, 2958] combine_rows_per_src=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=20992 dedup_ratio=0.244832 router_tokens_per_expert=[763, 785, 776, 711, 744, 770, 782, 733, 759, 775, 757, 731, 783, 734, 798, 777, 762, 766, 682, 748, 805, 771, 765, 788, 797, 779, 731, 757, 776, 789, 793, 772, 811, 728, 792, 743, 784, 753, 770, 737, 815, 805, 810, 799, 788, 755, 780, 749, 740, 825, 762, 800, 775, 752, 775, 746, 746, 717, 753, 810, 778, 735, 746, 814] payload_rows_per_dst=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] dup_rows_per_dst=[2974, 3014, 2952, 3098, 2854, 3184, 3076, 2916] combine_rows_per_src=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256593296 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6132254096 ingress_bytes=7266238464 link_bytes=7266238464 total_nvlink_bytes=13398492560 weight_to_token_ratio=0.093166 prefetch_share_of_ingress=0.104768 dedup_ratio=0.244306 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.188794363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12244255600 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6126084976 ingress_bytes=7571496960 link_bytes=7571496960 total_nvlink_bytes=13697581936 weight_to_token_ratio=0.118695 prefetch_share_of_ingress=0.091403 dedup_ratio=0.244925 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027065 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259426776 prefetch_bytes=692060160 reduce_bytes=830472192 weight_movement_bytes=1522532352 egress_bytes=6133670360 ingress_bytes=7648288768 link_bytes=7648288768 total_nvlink_bytes=13781959128 weight_to_token_ratio=0.124193 prefetch_share_of_ingress=0.090486 dedup_ratio=0.244186 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=24 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.251299778e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430360 weight_movement_bytes=34603008 total_nvlink_bytes=566033368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343680 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266796672 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533249664 weight_movement_bytes=34603008 total_nvlink_bytes=567852672 weight_to_token_ratio=0.064891 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343816 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428168 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512520 weight_movement_bytes=34603008 total_nvlink_bytes=567115528 weight_to_token_ratio=0.064981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64708 dup_rows=24130 dup_rows_remote=20982 dedup_ratio=0.245463 router_tokens_per_expert=[753, 785, 754, 772, 739, 762, 746, 742, 702, 761, 788, 753, 781, 751, 766, 822, 782, 787, 788, 825, 790, 798, 745, 737, 741, 778, 744, 723, 780, 760, 762, 774, 760, 762, 785, 723, 743, 760, 801, 772, 787, 796, 777, 759, 756, 769, 818, 744, 811, 748, 741, 757, 775, 739, 841, 760, 742, 787, 766, 772, 760, 800, 792, 758] payload_rows_per_dst=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] dup_rows_per_dst=[2894, 3068, 2886, 3000, 3040, 2946, 3148, 3148] combine_rows_per_src=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65172 dup_rows=23812 dup_rows_remote=20842 dedup_ratio=0.242228 router_tokens_per_expert=[779, 770, 782, 793, 790, 740, 768, 759, 718, 800, 761, 752, 784, 751, 703, 817, 755, 749, 773, 748, 768, 761, 765, 804, 760, 782, 769, 786, 810, 758, 734, 746, 761, 839, 738, 752, 742, 774, 763, 768, 743, 771, 770, 787, 781, 737, 780, 780, 806, 713, 738, 751, 814, 775, 784, 740, 746, 752, 761, 790, 768, 832, 798, 763] payload_rows_per_dst=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] dup_rows_per_dst=[3022, 2896, 3042, 2970, 3030, 2914, 2900, 3038] combine_rows_per_src=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12250644616 prefetch_bytes=761266176 reduce_bytes=346030080 weight_movement_bytes=1107296256 egress_bytes=6129279112 ingress_bytes=7228661760 link_bytes=7228661760 total_nvlink_bytes=13357940872 weight_to_token_ratio=0.090387 prefetch_share_of_ingress=0.105312 dedup_ratio=0.244650 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=10 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.182646619e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262441760 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6135178016 ingress_bytes=7580590080 link_bytes=7580590080 total_nvlink_bytes=13715768096 weight_to_token_ratio=0.118519 prefetch_share_of_ingress=0.091294 dedup_ratio=0.244013 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240223921e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=342760 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265386728 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530430696 weight_movement_bytes=34603008 total_nvlink_bytes=565033704 weight_to_token_ratio=0.065236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64956 dup_rows=24072 dup_rows_remote=21046 dedup_ratio=0.244873 router_tokens_per_expert=[773, 768, 780, 733, 752, 775, 777, 767, 720, 751, 769, 822, 751, 790, 759, 738, 784, 767, 793, 717, 838, 770, 810, 791, 791, 786, 775, 738, 791, 744, 769, 704, 767, 742, 751, 807, 749, 721, 791, 765, 752, 778, 773, 782, 740, 740, 779, 838, 782, 781, 725, 778, 743, 737, 731, 789, 739, 779, 823, 757, 756, 805, 758, 801] payload_rows_per_dst=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] dup_rows_per_dst=[3026, 3026, 3104, 2992, 3138, 2836, 2894, 3056] combine_rows_per_src=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344056 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288568 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233080 weight_movement_bytes=34603008 total_nvlink_bytes=568836088 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233030848 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6120463552 ingress_bytes=7531290624 link_bytes=7531290624 total_nvlink_bytes=13651754176 weight_to_token_ratio=0.115975 prefetch_share_of_ingress=0.082702 dedup_ratio=0.244611 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232158274e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344008 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403784 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463560 weight_movement_bytes=34603008 total_nvlink_bytes=567066568 weight_to_token_ratio=0.064987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246151200 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6127030304 ingress_bytes=7745462272 link_bytes=7745462272 total_nvlink_bytes=13872492576 weight_to_token_ratio=0.132804 prefetch_share_of_ingress=0.075948 dedup_ratio=0.244592 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267197868e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233467688 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6120679208 ingress_bytes=7393099776 link_bytes=7393099776 total_nvlink_bytes=13513778984 weight_to_token_ratio=0.104656 prefetch_share_of_ingress=0.093609 dedup_ratio=0.244264 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.209549533e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-29 23:41:54] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 672.1 | throughput per GPU (TFLOP/s/GPU): 189.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.987737E+00 | loss scale: 1.0 | grad norm: 3.603 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65164 dup_rows=23932 dup_rows_remote=20974 dedup_ratio=0.243449 router_tokens_per_expert=[796, 763, 763, 752, 748, 756, 771, 776, 717, 810, 749, 803, 769, 768, 782, 743, 771, 757, 728, 779, 781, 751, 783, 796, 782, 757, 771, 789, 781, 793, 786, 798, 776, 756, 737, 807, 744, 753, 731, 752, 749, 804, 761, 791, 765, 757, 788, 741, 771, 790, 752, 767, 789, 808, 786, 786, 778, 714, 762, 795, 735, 748, 749, 741] payload_rows_per_dst=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] dup_rows_per_dst=[2724, 2916, 2788, 3368, 3118, 3040, 3020, 2958] combine_rows_per_src=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64762 dup_rows=24212 dup_rows_remote=21154 dedup_ratio=0.246297 router_tokens_per_expert=[790, 711, 776, 798, 781, 784, 776, 764, 753, 819, 787, 798, 778, 749, 763, 747, 764, 802, 731, 771, 808, 740, 773, 781, 783, 731, 727, 793, 733, 791, 734, 710, 761, 747, 811, 775, 795, 762, 727, 747, 752, 786, 743, 765, 774, 775, 723, 777, 739, 802, 792, 787, 764, 780, 764, 772, 783, 727, 794, 778, 827, 741, 777, 759] payload_rows_per_dst=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] dup_rows_per_dst=[3070, 3058, 3060, 2932, 2938, 2992, 3120, 3042] combine_rows_per_src=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65066 dup_rows=23916 dup_rows_remote=20908 dedup_ratio=0.243286 router_tokens_per_expert=[759, 786, 762, 775, 787, 748, 755, 757, 790, 819, 819, 715, 765, 742, 774, 753, 751, 808, 836, 787, 763, 755, 827, 760, 790, 757, 717, 791, 773, 787, 767, 735, 799, 809, 784, 749, 751, 765, 737, 771, 738, 742, 740, 801, 771, 787, 770, 731, 731, 763, 768, 760, 755, 788, 758, 747, 783, 728, 780, 788, 763, 781, 768, 736] payload_rows_per_dst=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] dup_rows_per_dst=[2904, 2934, 3174, 3004, 3008, 2970, 2990, 2932] combine_rows_per_src=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21078 dedup_ratio=0.244568 router_tokens_per_expert=[719, 788, 784, 747, 786, 787, 784, 781, 802, 800, 791, 814, 734, 758, 717, 779, 785, 738, 719, 805, 805, 768, 778, 745, 819, 735, 766, 763, 753, 716, 736, 740, 748, 709, 727, 772, 763, 756, 755, 807, 823, 769, 763, 781, 777, 713, 815, 775, 751, 749, 744, 781, 772, 776, 804, 766, 770, 841, 745, 813, 730, 778, 740, 797] payload_rows_per_dst=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] dup_rows_per_dst=[3076, 3074, 2964, 2858, 2884, 3058, 3028, 3100] combine_rows_per_src=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64944 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[782, 754, 721, 754, 801, 810, 763, 797, 761, 754, 732, 773, 747, 747, 747, 815, 784, 793, 795, 764, 774, 801, 805, 766, 781, 776, 760, 770, 765, 752, 756, 743, 737, 768, 777, 782, 804, 751, 750, 780, 720, 772, 761, 776, 752, 771, 789, 730, 751, 779, 818, 778, 753, 800, 746, 767, 759, 740, 743, 755, 750, 829, 770, 751] payload_rows_per_dst=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] dup_rows_per_dst=[3056, 2994, 3134, 2982, 3034, 2990, 3006, 2940] combine_rows_per_src=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65100 dup_rows=23906 dup_rows_remote=20858 dedup_ratio=0.243184 router_tokens_per_expert=[744, 803, 749, 781, 753, 754, 783, 764, 725, 752, 761, 732, 765, 817, 782, 776, 785, 752, 759, 785, 771, 736, 788, 752, 748, 796, 760, 760, 735, 724, 763, 811, 841, 700, 782, 792, 783, 731, 738, 776, 781, 803, 704, 769, 736, 832, 815, 733, 774, 771, 775, 754, 787, 812, 773, 788, 759, 741, 767, 750, 771, 790, 768, 790] payload_rows_per_dst=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] dup_rows_per_dst=[2988, 3068, 2922, 2914, 2934, 3048, 3032, 3000] combine_rows_per_src=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65160 dup_rows=23934 dup_rows_remote=20920 dedup_ratio=0.243469 router_tokens_per_expert=[798, 779, 747, 803, 830, 805, 756, 795, 773, 758, 763, 784, 766, 722, 762, 780, 752, 753, 732, 712, 759, 765, 799, 755, 804, 705, 767, 780, 735, 726, 768, 822, 738, 768, 790, 711, 790, 772, 769, 768, 783, 775, 806, 762, 776, 736, 753, 777, 732, 746, 752, 797, 785, 734, 763, 803, 761, 813, 770, 778, 744, 786, 786, 773] payload_rows_per_dst=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] dup_rows_per_dst=[3158, 2970, 2880, 2926, 2914, 3032, 3014, 3040] combine_rows_per_src=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344552 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256296 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534168040 weight_movement_bytes=34603008 total_nvlink_bytes=568771048 weight_to_token_ratio=0.064779 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265265152 dispatch_meta_bytes=343664 combine_payload_bytes=265265152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265608816 ingress_bytes=334471168 link_bytes=334471168 token_bytes=530873968 weight_movement_bytes=69206016 total_nvlink_bytes=600079984 weight_to_token_ratio=0.130362 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258588078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344072 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338312 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332552 weight_movement_bytes=69206016 total_nvlink_bytes=601538568 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343896 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854232 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364568 weight_movement_bytes=34603008 total_nvlink_bytes=567967576 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344392 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355016 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365640 weight_movement_bytes=34603008 total_nvlink_bytes=566968648 weight_to_token_ratio=0.064999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343832 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993432 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643032 weight_movement_bytes=34603008 total_nvlink_bytes=568246040 weight_to_token_ratio=0.064843 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344320 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267239680 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534135040 weight_movement_bytes=138412032 total_nvlink_bytes=672547072 weight_to_token_ratio=0.259133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64696 dup_rows=24248 dup_rows_remote=21168 dedup_ratio=0.246663 router_tokens_per_expert=[771, 744, 811, 756, 798, 768, 758, 731, 787, 812, 741, 781, 737, 781, 771, 785, 826, 734, 761, 789, 730, 775, 770, 782, 782, 801, 725, 755, 721, 806, 803, 744, 767, 752, 791, 765, 778, 784, 746, 784, 801, 797, 752, 719, 776, 775, 775, 783, 760, 770, 755, 742, 769, 771, 758, 748, 746, 743, 751, 815, 782, 764, 768, 729] payload_rows_per_dst=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] dup_rows_per_dst=[3080, 3138, 3106, 2984, 2996, 3122, 2852, 2970] combine_rows_per_src=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65030 dup_rows=23964 dup_rows_remote=20860 dedup_ratio=0.243774 router_tokens_per_expert=[752, 787, 756, 750, 787, 774, 756, 730, 796, 741, 816, 801, 740, 773, 748, 792, 776, 786, 793, 783, 802, 751, 737, 788, 762, 747, 773, 770, 778, 778, 735, 756, 740, 786, 754, 755, 763, 781, 747, 777, 774, 728, 796, 771, 757, 722, 811, 797, 725, 861, 787, 789, 749, 747, 742, 786, 765, 755, 792, 743, 758, 732, 764, 784] payload_rows_per_dst=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] dup_rows_per_dst=[2916, 3104, 3050, 2972, 2980, 2944, 3060, 2938] combine_rows_per_src=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64970 dup_rows=23942 dup_rows_remote=20832 dedup_ratio=0.243551 router_tokens_per_expert=[778, 750, 785, 759, 791, 772, 766, 721, 797, 808, 786, 761, 783, 761, 756, 806, 747, 785, 731, 781, 749, 732, 767, 801, 732, 795, 763, 745, 742, 759, 779, 799, 722, 742, 752, 701, 793, 785, 729, 845, 776, 735, 806, 828, 765, 761, 783, 763, 762, 763, 798, 779, 737, 768, 790, 750, 735, 766, 746, 784, 743, 798, 811, 749] payload_rows_per_dst=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] dup_rows_per_dst=[2972, 2968, 2960, 2912, 2940, 3016, 3064, 3110] combine_rows_per_src=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65118 dup_rows=23946 dup_rows_remote=20888 dedup_ratio=0.243591 router_tokens_per_expert=[791, 769, 767, 824, 791, 763, 788, 745, 739, 765, 749, 774, 805, 767, 773, 788, 770, 770, 750, 783, 772, 779, 787, 738, 762, 789, 795, 716, 718, 813, 784, 726, 752, 776, 759, 789, 788, 796, 764, 739, 758, 759, 725, 792, 762, 763, 745, 762, 721, 760, 813, 734, 749, 806, 719, 762, 753, 770, 802, 762, 792, 790, 760, 780] payload_rows_per_dst=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] dup_rows_per_dst=[3084, 2946, 3058, 2966, 3018, 2950, 2952, 2972] combine_rows_per_src=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65114 dup_rows=23828 dup_rows_remote=20812 dedup_ratio=0.242391 router_tokens_per_expert=[752, 749, 764, 724, 793, 748, 753, 759, 767, 794, 765, 724, 743, 767, 791, 801, 786, 735, 819, 786, 762, 741, 758, 827, 799, 776, 794, 792, 738, 762, 737, 791, 743, 785, 782, 753, 758, 783, 736, 797, 784, 728, 812, 732, 796, 806, 816, 758, 823, 783, 739, 770, 762, 763, 773, 719, 804, 749, 821, 746, 717, 735, 744, 738] payload_rows_per_dst=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] dup_rows_per_dst=[2852, 2988, 3096, 3016, 2970, 3022, 2966, 2918] combine_rows_per_src=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64960 dup_rows=23998 dup_rows_remote=20934 dedup_ratio=0.244120 router_tokens_per_expert=[797, 777, 761, 703, 803, 713, 783, 777, 789, 775, 772, 765, 718, 752, 781, 791, 758, 763, 792, 791, 753, 773, 776, 732, 804, 801, 779, 767, 758, 739, 815, 742, 770, 823, 804, 746, 774, 740, 779, 769, 772, 756, 809, 788, 778, 774, 772, 747, 792, 737, 769, 749, 837, 766, 702, 752, 810, 754, 731, 774, 698, 761, 766, 753] payload_rows_per_dst=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] dup_rows_per_dst=[3018, 2978, 2992, 3036, 3064, 3034, 2934, 2942] combine_rows_per_src=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65038 dup_rows=24022 dup_rows_remote=21030 dedup_ratio=0.244364 router_tokens_per_expert=[746, 772, 749, 802, 784, 772, 762, 826, 754, 777, 764, 792, 789, 778, 751, 767, 765, 767, 748, 744, 758, 761, 768, 813, 715, 778, 782, 744, 751, 773, 776, 762, 752, 790, 792, 808, 749, 752, 797, 749, 775, 763, 769, 726, 751, 776, 760, 798, 741, 736, 796, 747, 722, 761, 813, 761, 756, 696, 781, 786, 771, 802, 791, 795] payload_rows_per_dst=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] dup_rows_per_dst=[3066, 2962, 3038, 2850, 3044, 2992, 2992, 3078] combine_rows_per_src=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65206 dup_rows=23730 dup_rows_remote=20736 dedup_ratio=0.241394 router_tokens_per_expert=[739, 776, 746, 778, 723, 752, 790, 801, 765, 727, 782, 728, 748, 772, 778, 754, 823, 764, 790, 718, 749, 806, 740, 713, 860, 756, 747, 779, 770, 778, 796, 805, 739, 762, 797, 772, 767, 745, 784, 743, 751, 723, 810, 814, 734, 733, 780, 776, 794, 755, 788, 763, 763, 769, 745, 804, 730, 775, 727, 797, 814, 773, 754, 818] payload_rows_per_dst=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] dup_rows_per_dst=[2902, 2866, 2996, 3130, 2982, 2888, 2994, 2972] combine_rows_per_src=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343560 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266706440 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533069320 weight_movement_bytes=103809024 total_nvlink_bytes=636878344 weight_to_token_ratio=0.194738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343704 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267050648 ingress_bytes=405118976 link_bytes=405118976 token_bytes=533757592 weight_movement_bytes=138412032 total_nvlink_bytes=672169624 weight_to_token_ratio=0.259316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343208 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460328 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532577448 weight_movement_bytes=34603008 total_nvlink_bytes=567180456 weight_to_token_ratio=0.064973 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344024 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067352 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790680 weight_movement_bytes=34603008 total_nvlink_bytes=568393688 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343576 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419736 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495896 weight_movement_bytes=34603008 total_nvlink_bytes=567098904 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344272 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739920 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135568 weight_movement_bytes=69206016 total_nvlink_bytes=602341584 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343768 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427544 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511320 weight_movement_bytes=34603008 total_nvlink_bytes=569114328 weight_to_token_ratio=0.064738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65214 dup_rows=23954 dup_rows_remote=21062 dedup_ratio=0.243673 router_tokens_per_expert=[786, 747, 752, 782, 784, 767, 828, 770, 725, 736, 814, 737, 751, 751, 754, 746, 794, 779, 796, 797, 771, 756, 750, 754, 786, 776, 794, 803, 765, 738, 737, 723, 723, 784, 783, 792, 706, 772, 812, 764, 770, 745, 762, 752, 782, 783, 769, 792, 767, 808, 763, 795, 774, 774, 834, 767, 752, 731, 750, 774, 751, 741, 774, 757] payload_rows_per_dst=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] dup_rows_per_dst=[2952, 2892, 3080, 3020, 2942, 3086, 3102, 2880] combine_rows_per_src=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64938 dup_rows=23946 dup_rows_remote=20842 dedup_ratio=0.243591 router_tokens_per_expert=[774, 707, 772, 773, 752, 764, 786, 788, 765, 730, 801, 749, 745, 778, 815, 801, 790, 801, 796, 744, 803, 771, 788, 769, 829, 754, 768, 743, 797, 793, 741, 761, 781, 757, 769, 794, 751, 750, 799, 781, 766, 762, 736, 737, 775, 786, 744, 775, 756, 788, 741, 752, 792, 775, 747, 806, 768, 733, 767, 745, 758, 770, 721, 722] payload_rows_per_dst=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] dup_rows_per_dst=[3002, 3022, 3104, 3100, 2998, 2918, 2968, 2834] combine_rows_per_src=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64792 dup_rows=24068 dup_rows_remote=21060 dedup_ratio=0.244832 router_tokens_per_expert=[786, 767, 789, 737, 732, 729, 739, 797, 776, 813, 728, 719, 768, 747, 814, 781, 756, 769, 748, 738, 755, 754, 765, 806, 811, 765, 759, 792, 806, 761, 775, 767, 769, 732, 752, 763, 787, 801, 753, 806, 725, 741, 774, 728, 802, 796, 786, 756, 815, 758, 724, 763, 819, 783, 752, 786, 804, 704, 771, 765, 782, 784, 767, 755] payload_rows_per_dst=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] dup_rows_per_dst=[3008, 3060, 2986, 2990, 3040, 2952, 3010, 3022] combine_rows_per_src=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64552 dup_rows=24086 dup_rows_remote=20658 dedup_ratio=0.245015 router_tokens_per_expert=[773, 775, 803, 761, 715, 757, 775, 779, 738, 777, 746, 779, 761, 773, 787, 763, 736, 801, 753, 742, 829, 783, 791, 784, 786, 812, 776, 751, 743, 735, 765, 799, 791, 773, 772, 761, 774, 768, 771, 742, 716, 747, 732, 849, 758, 754, 776, 735, 764, 828, 758, 822, 737, 699, 752, 743, 763, 752, 789, 814, 760, 766, 748, 790] payload_rows_per_dst=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] dup_rows_per_dst=[2676, 2936, 3066, 2898, 3034, 3026, 3022, 3428] combine_rows_per_src=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65038 dup_rows=24052 dup_rows_remote=21064 dedup_ratio=0.244670 router_tokens_per_expert=[856, 814, 775, 795, 780, 754, 759, 785, 729, 762, 783, 790, 719, 796, 733, 780, 738, 792, 755, 744, 746, 777, 777, 751, 806, 773, 783, 728, 782, 730, 777, 722, 772, 751, 774, 733, 737, 796, 768, 760, 736, 769, 756, 815, 794, 756, 754, 786, 736, 764, 812, 732, 782, 781, 715, 764, 844, 736, 754, 793, 772, 764, 768, 787] payload_rows_per_dst=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] dup_rows_per_dst=[3142, 2958, 2868, 2988, 3016, 3026, 2948, 3106] combine_rows_per_src=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64896 dup_rows=24172 dup_rows_remote=21168 dedup_ratio=0.245890 router_tokens_per_expert=[752, 758, 751, 782, 769, 793, 755, 755, 765, 758, 799, 802, 754, 772, 713, 785, 755, 772, 751, 778, 797, 763, 765, 741, 746, 742, 758, 851, 789, 792, 777, 762, 729, 703, 796, 739, 802, 790, 738, 823, 739, 775, 772, 779, 800, 753, 763, 781, 804, 755, 827, 769, 752, 759, 835, 766, 749, 730, 754, 735, 765, 724, 755, 789] payload_rows_per_dst=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] dup_rows_per_dst=[3048, 2982, 2972, 3130, 3004, 3028, 3088, 2920] combine_rows_per_src=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64846 dup_rows=24074 dup_rows_remote=21014 dedup_ratio=0.244893 router_tokens_per_expert=[765, 775, 745, 761, 760, 726, 756, 769, 747, 753, 807, 762, 775, 797, 760, 760, 766, 768, 780, 780, 749, 788, 735, 786, 771, 782, 762, 759, 822, 749, 761, 778, 812, 732, 729, 735, 751, 751, 810, 744, 744, 785, 779, 759, 822, 768, 781, 784, 806, 757, 784, 739, 782, 747, 789, 754, 753, 778, 782, 801, 755, 754, 766, 765] payload_rows_per_dst=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] dup_rows_per_dst=[3012, 2994, 3040, 3116, 2804, 3060, 3042, 3006] combine_rows_per_src=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65030 dup_rows=23928 dup_rows_remote=20972 dedup_ratio=0.243408 router_tokens_per_expert=[819, 733, 730, 782, 847, 798, 819, 713, 790, 799, 765, 734, 757, 765, 804, 813, 736, 795, 769, 728, 736, 778, 728, 827, 778, 765, 745, 797, 752, 765, 759, 756, 723, 800, 757, 767, 757, 791, 745, 710, 801, 767, 757, 756, 762, 741, 812, 771, 765, 772, 777, 765, 771, 743, 777, 781, 744, 724, 746, 776, 723, 820, 793, 776] payload_rows_per_dst=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] dup_rows_per_dst=[3126, 3102, 2896, 2922, 2928, 3018, 2956, 2980] combine_rows_per_src=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=345104 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267461648 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534578192 weight_movement_bytes=69206016 total_nvlink_bytes=603784208 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329168 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315216 weight_movement_bytes=103809024 total_nvlink_bytes=636124240 weight_to_token_ratio=0.195014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264404992 dispatch_meta_bytes=340840 combine_payload_bytes=264404992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264745832 ingress_bytes=299008000 link_bytes=299008000 token_bytes=529150824 weight_movement_bytes=34603008 total_nvlink_bytes=563753832 weight_to_token_ratio=0.065393 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125143032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343408 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731440 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119472 weight_movement_bytes=34603008 total_nvlink_bytes=565722480 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344408 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266740056 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135704 weight_movement_bytes=69206016 total_nvlink_bytes=602341720 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344008 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706888 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069768 weight_movement_bytes=69206016 total_nvlink_bytes=602275784 weight_to_token_ratio=0.129825 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344256 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266158272 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531972288 weight_movement_bytes=34603008 total_nvlink_bytes=566575296 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343440 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952656 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561872 weight_movement_bytes=34603008 total_nvlink_bytes=566164880 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65006 dup_rows=23960 dup_rows_remote=20994 dedup_ratio=0.243734 router_tokens_per_expert=[783, 756, 819, 785, 774, 763, 760, 768, 816, 759, 781, 721, 775, 786, 765, 749, 750, 792, 766, 772, 742, 708, 760, 785, 768, 743, 793, 779, 744, 749, 772, 776, 766, 739, 803, 766, 807, 768, 769, 756, 769, 758, 735, 769, 782, 728, 781, 783, 748, 769, 814, 750, 730, 792, 831, 741, 739, 800, 758, 777, 777, 769, 766, 753] payload_rows_per_dst=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] dup_rows_per_dst=[3042, 2966, 2936, 2942, 3008, 2974, 3084, 3008] combine_rows_per_src=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64974 dup_rows=24134 dup_rows_remote=21224 dedup_ratio=0.245504 router_tokens_per_expert=[750, 737, 735, 752, 744, 741, 755, 784, 761, 722, 811, 772, 802, 766, 772, 778, 768, 740, 707, 765, 753, 758, 774, 788, 768, 770, 790, 731, 778, 758, 776, 803, 782, 740, 802, 750, 756, 760, 768, 779, 753, 753, 799, 773, 737, 799, 737, 751, 776, 773, 837, 818, 802, 770, 815, 751, 817, 758, 802, 804, 763, 743, 722, 753] payload_rows_per_dst=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] dup_rows_per_dst=[2870, 3070, 2910, 3048, 2990, 3024, 3178, 3044] combine_rows_per_src=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64498 dup_rows=24160 dup_rows_remote=20790 dedup_ratio=0.245768 router_tokens_per_expert=[776, 769, 769, 790, 788, 775, 760, 782, 758, 809, 764, 786, 784, 811, 767, 756, 795, 763, 773, 760, 771, 764, 767, 744, 759, 703, 769, 779, 765, 724, 784, 772, 771, 741, 765, 801, 750, 749, 784, 824, 785, 730, 781, 782, 750, 740, 806, 717, 801, 782, 778, 761, 755, 797, 742, 780, 762, 734, 730, 745, 804, 755, 765, 749] payload_rows_per_dst=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] dup_rows_per_dst=[3032, 3068, 3014, 2788, 3106, 3138, 2644, 3370] combine_rows_per_src=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65042 dup_rows=23832 dup_rows_remote=20630 dedup_ratio=0.242432 router_tokens_per_expert=[753, 743, 770, 800, 791, 745, 789, 761, 820, 794, 762, 760, 787, 802, 751, 736, 800, 781, 779, 799, 732, 763, 760, 793, 730, 779, 758, 762, 799, 782, 722, 792, 755, 757, 769, 775, 752, 750, 780, 794, 815, 763, 736, 723, 748, 750, 746, 790, 764, 787, 764, 728, 765, 759, 774, 797, 808, 785, 729, 768, 735, 729, 767, 795] payload_rows_per_dst=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] dup_rows_per_dst=[3202, 3098, 2860, 2898, 3200, 2928, 2708, 2938] combine_rows_per_src=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344000 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608576 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873152 weight_movement_bytes=34603008 total_nvlink_bytes=567476160 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64736 dup_rows=24178 dup_rows_remote=21102 dedup_ratio=0.245951 router_tokens_per_expert=[800, 783, 801, 746, 743, 776, 801, 771, 769, 783, 777, 766, 693, 757, 763, 790, 725, 781, 717, 754, 765, 754, 748, 805, 756, 824, 782, 752, 747, 803, 813, 763, 768, 797, 772, 783, 747, 757, 775, 825, 780, 735, 729, 764, 723, 752, 762, 763, 782, 782, 794, 739, 780, 814, 791, 751, 761, 785, 796, 765, 750, 799, 690, 733] payload_rows_per_dst=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] dup_rows_per_dst=[3074, 2982, 2942, 3112, 3136, 2896, 3076, 2960] combine_rows_per_src=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64980 dup_rows=24058 dup_rows_remote=21058 dedup_ratio=0.244731 router_tokens_per_expert=[766, 810, 804, 728, 728, 751, 760, 803, 758, 768, 734, 752, 774, 795, 772, 808, 776, 747, 778, 737, 704, 782, 764, 808, 810, 785, 744, 759, 743, 774, 740, 778, 767, 735, 769, 753, 768, 785, 762, 749, 783, 798, 775, 801, 741, 791, 749, 781, 768, 758, 737, 797, 769, 761, 794, 825, 770, 767, 752, 768, 783, 776, 707, 773] payload_rows_per_dst=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] dup_rows_per_dst=[3016, 3070, 2990, 3000, 2902, 3096, 3092, 2892] combine_rows_per_src=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64828 dup_rows=24052 dup_rows_remote=20946 dedup_ratio=0.244670 router_tokens_per_expert=[756, 754, 757, 755, 766, 737, 767, 743, 752, 751, 729, 788, 800, 774, 751, 737, 800, 735, 685, 775, 793, 802, 755, 763, 741, 801, 744, 792, 778, 781, 758, 802, 831, 748, 788, 799, 773, 778, 761, 787, 795, 751, 769, 772, 763, 763, 804, 803, 712, 801, 774, 800, 776, 778, 754, 725, 775, 743, 752, 753, 738, 778, 818, 768] payload_rows_per_dst=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] dup_rows_per_dst=[2906, 2952, 2998, 3036, 3106, 3034, 3018, 3002] combine_rows_per_src=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21104 dedup_ratio=0.244568 router_tokens_per_expert=[797, 738, 741, 796, 796, 738, 777, 767, 787, 732, 723, 789, 758, 737, 788, 775, 764, 795, 766, 791, 794, 773, 771, 764, 770, 771, 722, 758, 715, 822, 757, 737, 793, 744, 783, 759, 781, 776, 792, 753, 748, 744, 766, 782, 790, 812, 754, 734, 749, 807, 774, 769, 803, 783, 780, 803, 729, 761, 820, 759, 771, 745, 756, 723] payload_rows_per_dst=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] dup_rows_per_dst=[3076, 2874, 3080, 2922, 3082, 2938, 3146, 2924] combine_rows_per_src=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344792 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478296 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611800 weight_movement_bytes=34603008 total_nvlink_bytes=567214808 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=341152 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264524960 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528708768 weight_movement_bytes=34603008 total_nvlink_bytes=563311776 weight_to_token_ratio=0.065448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342688 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266754720 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533166752 weight_movement_bytes=69206016 total_nvlink_bytes=602372768 weight_to_token_ratio=0.129802 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343352 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265502008 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660664 weight_movement_bytes=103809024 total_nvlink_bytes=634469688 weight_to_token_ratio=0.195622 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344152 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502232 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660312 weight_movement_bytes=69206016 total_nvlink_bytes=601866328 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343096 combine_payload_bytes=265535488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265878584 ingress_bytes=403947520 link_bytes=403947520 token_bytes=531414072 weight_movement_bytes=138412032 total_nvlink_bytes=669826104 weight_to_token_ratio=0.260460 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520021997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344176 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338416 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332656 weight_movement_bytes=34603008 total_nvlink_bytes=566935664 weight_to_token_ratio=0.065003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64902 dup_rows=23984 dup_rows_remote=21158 dedup_ratio=0.243978 router_tokens_per_expert=[778, 758, 749, 710, 770, 756, 754, 803, 758, 757, 728, 779, 797, 758, 804, 741, 745, 735, 790, 798, 736, 780, 814, 809, 786, 771, 743, 779, 750, 772, 806, 732, 746, 763, 785, 771, 757, 754, 787, 727, 773, 775, 792, 782, 801, 771, 751, 805, 812, 772, 785, 760, 769, 811, 813, 821, 735, 713, 777, 752, 722, 730, 717, 777] payload_rows_per_dst=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] dup_rows_per_dst=[2908, 2826, 3096, 3062, 2948, 3162, 3210, 2772] combine_rows_per_src=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64966 dup_rows=23994 dup_rows_remote=20992 dedup_ratio=0.244080 router_tokens_per_expert=[759, 784, 773, 764, 779, 732, 733, 777, 723, 802, 755, 780, 769, 813, 771, 735, 781, 795, 809, 764, 723, 748, 770, 783, 766, 747, 779, 775, 739, 750, 782, 774, 794, 775, 797, 785, 732, 774, 736, 742, 737, 754, 754, 795, 755, 764, 769, 785, 768, 731, 785, 785, 757, 776, 787, 740, 778, 792, 752, 788, 821, 769, 776, 765] payload_rows_per_dst=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] dup_rows_per_dst=[3004, 2956, 3002, 3020, 2954, 2950, 2954, 3154] combine_rows_per_src=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=64958 dup_rows=23878 dup_rows_remote=20776 dedup_ratio=0.242900 router_tokens_per_expert=[767, 766, 753, 763, 788, 785, 735, 815, 762, 762, 784, 803, 772, 731, 745, 788, 747, 743, 768, 761, 788, 764, 760, 747, 794, 761, 769, 770, 786, 778, 826, 768, 769, 726, 782, 808, 778, 778, 795, 762, 780, 755, 798, 815, 760, 759, 759, 736, 777, 732, 749, 762, 732, 747, 711, 768, 730, 790, 751, 755, 781, 798, 782, 778] payload_rows_per_dst=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] dup_rows_per_dst=[3034, 3020, 2960, 3054, 2988, 2884, 2836, 3102] combine_rows_per_src=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344240 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182832 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532021424 weight_movement_bytes=34603008 total_nvlink_bytes=566624432 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64804 dup_rows=24170 dup_rows_remote=21138 dedup_ratio=0.245870 router_tokens_per_expert=[792, 823, 765, 769, 749, 755, 744, 783, 713, 807, 805, 764, 762, 745, 765, 784, 790, 802, 791, 725, 773, 720, 730, 818, 784, 814, 757, 778, 824, 759, 780, 781, 791, 757, 763, 783, 814, 755, 758, 758, 763, 765, 772, 741, 797, 727, 782, 794, 787, 763, 748, 773, 743, 765, 752, 771, 779, 764, 768, 752, 729, 729, 724, 734] payload_rows_per_dst=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] dup_rows_per_dst=[3032, 3044, 3228, 3274, 2832, 3038, 2896, 2826] combine_rows_per_src=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64914 dup_rows=24004 dup_rows_remote=21012 dedup_ratio=0.244181 router_tokens_per_expert=[763, 741, 714, 772, 784, 797, 760, 792, 768, 772, 726, 744, 747, 745, 787, 756, 760, 745, 742, 758, 791, 807, 782, 777, 717, 752, 798, 775, 761, 790, 757, 776, 817, 795, 753, 823, 793, 755, 787, 815, 776, 716, 780, 749, 721, 793, 728, 746, 773, 779, 762, 785, 741, 785, 769, 795, 787, 764, 819, 783, 776, 734, 745, 752] payload_rows_per_dst=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] dup_rows_per_dst=[2906, 2900, 3038, 3004, 3248, 2954, 2992, 2962] combine_rows_per_src=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65004 dup_rows=24016 dup_rows_remote=20986 dedup_ratio=0.244303 router_tokens_per_expert=[725, 727, 809, 809, 735, 763, 743, 762, 709, 734, 740, 754, 743, 813, 758, 751, 731, 773, 786, 791, 790, 819, 826, 756, 765, 793, 729, 791, 741, 797, 816, 755, 766, 739, 785, 804, 778, 752, 790, 743, 758, 772, 757, 788, 753, 765, 781, 731, 797, 761, 735, 814, 774, 745, 769, 787, 786, 777, 808, 753, 763, 744, 768, 775] payload_rows_per_dst=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] dup_rows_per_dst=[2900, 2832, 3160, 3034, 3030, 2906, 3040, 3114] combine_rows_per_src=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343832 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444568 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545304 weight_movement_bytes=69206016 total_nvlink_bytes=601751320 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65068 dup_rows=23942 dup_rows_remote=20906 dedup_ratio=0.243551 router_tokens_per_expert=[747, 742, 755, 779, 775, 792, 782, 794, 746, 793, 789, 746, 781, 722, 746, 770, 767, 806, 778, 777, 726, 768, 748, 749, 786, 798, 744, 794, 720, 780, 798, 807, 773, 782, 777, 757, 755, 744, 755, 790, 793, 829, 722, 755, 781, 795, 738, 752, 751, 743, 772, 788, 769, 765, 739, 780, 778, 756, 773, 737, 795, 733, 825, 745] payload_rows_per_dst=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] dup_rows_per_dst=[3026, 2902, 2954, 3068, 2980, 3036, 2944, 3032] combine_rows_per_src=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74688 payload_rows_remote=65360 dup_rows=23616 dup_rows_remote=20714 dedup_ratio=0.240234 router_tokens_per_expert=[798, 781, 759, 791, 782, 778, 783, 783, 738, 791, 758, 758, 778, 739, 725, 776, 798, 775, 801, 792, 762, 795, 758, 821, 778, 814, 754, 753, 755, 752, 737, 772, 797, 768, 742, 714, 780, 772, 700, 727, 739, 805, 854, 801, 746, 819, 769, 713, 789, 758, 742, 751, 742, 760, 771, 781, 743, 751, 742, 761, 766, 783, 775, 756] payload_rows_per_dst=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] dup_rows_per_dst=[3088, 2880, 3040, 2902, 2842, 3046, 2894, 2924] combine_rows_per_src=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342936 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266410904 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532478872 weight_movement_bytes=34603008 total_nvlink_bytes=567081880 weight_to_token_ratio=0.064985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343704 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266231448 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532119192 weight_movement_bytes=69206016 total_nvlink_bytes=601325208 weight_to_token_ratio=0.130057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343960 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266600344 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532856728 weight_movement_bytes=34603008 total_nvlink_bytes=567459736 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343768 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780952 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531218136 weight_movement_bytes=34603008 total_nvlink_bytes=565821144 weight_to_token_ratio=0.065139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343896 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862424 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533380952 weight_movement_bytes=69206016 total_nvlink_bytes=602586968 weight_to_token_ratio=0.129750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=344296 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268058856 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535773416 weight_movement_bytes=69206016 total_nvlink_bytes=604979432 weight_to_token_ratio=0.129170 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64934 dup_rows=23974 dup_rows_remote=20856 dedup_ratio=0.243876 router_tokens_per_expert=[761, 751, 729, 745, 780, 741, 751, 815, 825, 758, 778, 802, 765, 787, 756, 786, 781, 775, 738, 781, 769, 781, 787, 725, 797, 762, 790, 773, 751, 685, 745, 774, 789, 746, 736, 725, 755, 759, 719, 805, 774, 777, 772, 768, 756, 722, 794, 761, 803, 800, 773, 775, 761, 765, 779, 811, 773, 796, 745, 817, 753, 767, 712, 820] payload_rows_per_dst=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] dup_rows_per_dst=[2964, 3118, 3004, 2966, 2854, 2942, 3054, 3072] combine_rows_per_src=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65000 dup_rows=24024 dup_rows_remote=20932 dedup_ratio=0.244385 router_tokens_per_expert=[744, 791, 762, 797, 773, 816, 835, 752, 806, 745, 793, 792, 781, 754, 759, 753, 781, 780, 718, 777, 807, 795, 741, 787, 774, 727, 769, 800, 762, 750, 790, 740, 748, 778, 803, 800, 717, 775, 795, 772, 728, 775, 761, 779, 768, 773, 775, 778, 776, 795, 751, 732, 787, 807, 763, 708, 727, 740, 763, 759, 786, 740, 733, 709] payload_rows_per_dst=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] dup_rows_per_dst=[3132, 2970, 3092, 3010, 3004, 3066, 2930, 2820] combine_rows_per_src=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64578 dup_rows=24058 dup_rows_remote=20864 dedup_ratio=0.244731 router_tokens_per_expert=[761, 768, 786, 794, 792, 782, 778, 805, 762, 785, 716, 774, 764, 748, 761, 739, 756, 751, 798, 818, 818, 752, 767, 759, 766, 768, 790, 732, 792, 733, 818, 784, 816, 723, 717, 775, 772, 759, 786, 778, 769, 730, 780, 750, 723, 786, 779, 765, 740, 783, 756, 751, 770, 741, 769, 801, 802, 758, 788, 775, 762, 738, 754, 739] payload_rows_per_dst=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] dup_rows_per_dst=[2800, 2972, 2966, 3058, 3066, 3058, 2944, 3194] combine_rows_per_src=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343160 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312824 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282488 weight_movement_bytes=34603008 total_nvlink_bytes=566885496 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65030 dup_rows=24130 dup_rows_remote=21126 dedup_ratio=0.245463 router_tokens_per_expert=[778, 771, 778, 790, 751, 798, 768, 756, 800, 732, 747, 760, 736, 817, 791, 796, 788, 776, 739, 760, 735, 759, 765, 707, 763, 758, 787, 756, 782, 742, 754, 743, 760, 767, 766, 743, 817, 755, 731, 770, 798, 792, 793, 801, 731, 849, 815, 799, 765, 790, 771, 738, 769, 752, 713, 776, 751, 773, 766, 746, 772, 782, 746, 772] payload_rows_per_dst=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] dup_rows_per_dst=[3000, 3086, 2976, 2962, 2892, 3244, 3004, 2966] combine_rows_per_src=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21076 dedup_ratio=0.244548 router_tokens_per_expert=[788, 737, 760, 773, 784, 784, 715, 782, 745, 816, 734, 743, 757, 772, 762, 740, 793, 764, 722, 731, 751, 781, 775, 776, 791, 785, 772, 808, 786, 771, 789, 835, 784, 765, 757, 774, 778, 764, 761, 799, 770, 780, 770, 749, 803, 723, 762, 781, 761, 775, 730, 776, 776, 777, 749, 774, 779, 738, 765, 803, 754, 788, 755, 710] payload_rows_per_dst=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] dup_rows_per_dst=[3050, 2976, 3038, 3100, 2964, 2970, 2988, 2954] combine_rows_per_src=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343728 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583728 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823728 weight_movement_bytes=69206016 total_nvlink_bytes=602029744 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65024 dup_rows=24056 dup_rows_remote=21194 dedup_ratio=0.244710 router_tokens_per_expert=[789, 734, 761, 757, 698, 748, 790, 784, 702, 798, 771, 755, 739, 784, 758, 749, 820, 784, 778, 790, 778, 779, 781, 798, 754, 794, 761, 790, 768, 769, 758, 795, 791, 778, 765, 704, 779, 777, 747, 753, 755, 780, 757, 774, 713, 747, 780, 737, 792, 789, 766, 734, 755, 797, 775, 795, 817, 743, 804, 755, 762, 779, 761, 777] payload_rows_per_dst=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] dup_rows_per_dst=[2984, 2820, 3236, 3050, 2984, 2862, 3044, 3076] combine_rows_per_src=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65186 dup_rows=23818 dup_rows_remote=20750 dedup_ratio=0.242289 router_tokens_per_expert=[781, 755, 747, 797, 791, 774, 743, 734, 733, 810, 770, 712, 818, 762, 735, 749, 717, 748, 748, 774, 801, 781, 759, 776, 785, 771, 762, 790, 748, 775, 786, 756, 743, 817, 752, 735, 789, 773, 794, 760, 790, 780, 797, 767, 782, 755, 761, 764, 768, 780, 746, 759, 781, 822, 755, 744, 775, 778, 822, 747, 732, 729, 792, 775] payload_rows_per_dst=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] dup_rows_per_dst=[3068, 3356, 2672, 2840, 2952, 3058, 2922, 2950] combine_rows_per_src=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64988 dup_rows=24154 dup_rows_remote=21188 dedup_ratio=0.245707 router_tokens_per_expert=[766, 801, 788, 734, 781, 801, 796, 756, 762, 736, 753, 749, 732, 797, 741, 794, 751, 771, 767, 787, 790, 774, 757, 796, 757, 731, 763, 732, 775, 762, 752, 792, 795, 730, 757, 777, 811, 745, 752, 793, 786, 785, 736, 785, 728, 809, 788, 815, 716, 809, 747, 761, 772, 733, 751, 790, 730, 734, 757, 782, 795, 805, 763, 771] payload_rows_per_dst=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] dup_rows_per_dst=[3124, 2950, 3096, 2966, 3062, 3092, 2908, 2956] combine_rows_per_src=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264511488 dispatch_meta_bytes=341768 combine_payload_bytes=264511488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264853256 ingress_bytes=299114496 link_bytes=299114496 token_bytes=529364744 weight_movement_bytes=34603008 total_nvlink_bytes=563967752 weight_to_token_ratio=0.065367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125543768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344624 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707504 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070384 weight_movement_bytes=103809024 total_nvlink_bytes=636879408 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344872 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266683176 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021480 weight_movement_bytes=69206016 total_nvlink_bytes=602227496 weight_to_token_ratio=0.129837 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65172 dup_rows=23990 dup_rows_remote=21026 dedup_ratio=0.244039 router_tokens_per_expert=[795, 739, 793, 752, 784, 791, 781, 768, 730, 731, 774, 752, 769, 777, 732, 788, 786, 763, 773, 778, 782, 763, 772, 725, 824, 779, 745, 800, 759, 743, 755, 812, 786, 752, 751, 769, 768, 761, 735, 761, 818, 795, 756, 730, 744, 726, 729, 822, 824, 800, 772, 795, 758, 761, 752, 750, 782, 757, 809, 761, 747, 749, 752, 765] payload_rows_per_dst=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] dup_rows_per_dst=[2972, 2964, 2994, 3168, 2940, 2952, 2950, 3050] combine_rows_per_src=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343744 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345600 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347456 weight_movement_bytes=34603008 total_nvlink_bytes=568950464 weight_to_token_ratio=0.064758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266026704 ingress_bytes=300285952 link_bytes=300285952 token_bytes=531709648 weight_movement_bytes=34603008 total_nvlink_bytes=566312656 weight_to_token_ratio=0.065079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129951862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344704 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266535552 ingress_bytes=335396864 link_bytes=335396864 token_bytes=532726400 weight_movement_bytes=69206016 total_nvlink_bytes=601932416 weight_to_token_ratio=0.129909 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262071398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64984 dup_rows=24154 dup_rows_remote=21056 dedup_ratio=0.245707 router_tokens_per_expert=[718, 751, 808, 787, 795, 747, 738, 709, 750, 757, 793, 774, 728, 807, 746, 757, 753, 782, 756, 762, 776, 733, 798, 772, 755, 816, 778, 815, 765, 764, 778, 713, 774, 760, 822, 779, 800, 802, 787, 774, 757, 745, 802, 756, 733, 750, 734, 782, 764, 760, 777, 789, 797, 756, 744, 740, 765, 740, 762, 781, 783, 798, 795, 763] payload_rows_per_dst=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] dup_rows_per_dst=[2932, 2992, 3098, 3062, 3132, 2994, 2902, 3042] combine_rows_per_src=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344792 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289304 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233816 weight_movement_bytes=34603008 total_nvlink_bytes=568836824 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64756 dup_rows=24134 dup_rows_remote=21026 dedup_ratio=0.245504 router_tokens_per_expert=[772, 730, 772, 792, 761, 773, 777, 760, 755, 750, 759, 748, 803, 782, 733, 787, 765, 806, 754, 786, 827, 717, 774, 758, 773, 729, 759, 767, 743, 747, 770, 827, 725, 771, 775, 796, 762, 798, 819, 806, 765, 731, 749, 798, 800, 760, 721, 756, 740, 746, 770, 746, 767, 746, 779, 765, 770, 808, 773, 797, 755, 777, 749, 776] payload_rows_per_dst=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] dup_rows_per_dst=[3014, 2982, 2926, 3024, 2800, 3220, 3060, 3108] combine_rows_per_src=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344160 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266518624 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532693088 weight_movement_bytes=103809024 total_nvlink_bytes=636502112 weight_to_token_ratio=0.194876 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65032 dup_rows=24030 dup_rows_remote=21084 dedup_ratio=0.244446 router_tokens_per_expert=[776, 763, 772, 794, 754, 719, 786, 760, 799, 771, 751, 783, 765, 737, 762, 742, 768, 733, 731, 780, 754, 814, 785, 789, 773, 719, 731, 792, 758, 761, 739, 774, 808, 769, 779, 750, 817, 815, 783, 750, 763, 783, 787, 764, 732, 746, 746, 773, 772, 738, 813, 754, 764, 797, 781, 766, 776, 750, 794, 734, 797, 799, 756, 761] payload_rows_per_dst=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] dup_rows_per_dst=[3032, 2938, 3046, 2906, 3150, 2946, 2984, 3028] combine_rows_per_src=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65088 dup_rows=23960 dup_rows_remote=21026 dedup_ratio=0.243734 router_tokens_per_expert=[772, 766, 806, 778, 774, 775, 753, 795, 736, 734, 736, 791, 770, 779, 757, 768, 784, 790, 772, 801, 783, 786, 741, 779, 794, 771, 792, 727, 723, 765, 790, 778, 791, 737, 776, 731, 790, 737, 761, 719, 816, 759, 790, 767, 734, 790, 762, 773, 745, 762, 774, 750, 752, 753, 784, 775, 747, 814, 758, 751, 740, 796, 770, 782] payload_rows_per_dst=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] dup_rows_per_dst=[3026, 2948, 3040, 3052, 2894, 3036, 2934, 3030] combine_rows_per_src=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64996 dup_rows=23894 dup_rows_remote=20786 dedup_ratio=0.243062 router_tokens_per_expert=[759, 783, 742, 731, 769, 763, 780, 796, 814, 776, 786, 795, 758, 792, 736, 763, 788, 740, 762, 752, 773, 770, 764, 711, 781, 773, 736, 772, 777, 765, 760, 803, 757, 755, 806, 753, 769, 792, 776, 740, 769, 789, 781, 742, 772, 743, 787, 768, 743, 775, 751, 807, 780, 823, 788, 770, 768, 797, 756, 708, 751, 711, 772, 783] payload_rows_per_dst=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] dup_rows_per_dst=[3108, 3134, 2868, 2996, 2962, 2960, 2982, 2884] combine_rows_per_src=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64764 dup_rows=24064 dup_rows_remote=20906 dedup_ratio=0.244792 router_tokens_per_expert=[750, 740, 730, 775, 747, 746, 775, 765, 749, 777, 760, 787, 788, 796, 777, 737, 764, 756, 743, 750, 766, 770, 794, 768, 735, 791, 795, 779, 808, 742, 775, 760, 757, 802, 784, 748, 808, 804, 807, 807, 785, 731, 733, 761, 755, 805, 748, 790, 788, 770, 761, 759, 726, 749, 747, 760, 773, 753, 780, 733, 778, 772, 778, 805] payload_rows_per_dst=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] dup_rows_per_dst=[2880, 2966, 3040, 3034, 3158, 2970, 2936, 3080] combine_rows_per_src=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64824 dup_rows=24212 dup_rows_remote=21116 dedup_ratio=0.246297 router_tokens_per_expert=[764, 746, 783, 762, 776, 771, 770, 765, 784, 741, 757, 748, 772, 748, 795, 777, 763, 812, 791, 771, 728, 728, 741, 810, 772, 765, 744, 799, 744, 748, 816, 794, 792, 781, 769, 819, 751, 759, 763, 749, 767, 807, 754, 755, 779, 768, 825, 753, 745, 752, 764, 790, 822, 751, 752, 764, 713, 766, 753, 764, 718, 760, 806, 756] payload_rows_per_dst=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] dup_rows_per_dst=[3018, 2992, 3100, 3096, 2956, 3172, 3012, 2866] combine_rows_per_src=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343128 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265583704 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824280 weight_movement_bytes=34603008 total_nvlink_bytes=565427288 weight_to_token_ratio=0.065187 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65048 dup_rows=23878 dup_rows_remote=20798 dedup_ratio=0.242900 router_tokens_per_expert=[780, 805, 806, 733, 754, 746, 705, 745, 808, 755, 729, 797, 799, 775, 783, 783, 772, 745, 811, 787, 731, 728, 752, 772, 813, 778, 751, 819, 759, 718, 784, 727, 768, 801, 762, 788, 790, 773, 798, 727, 747, 779, 792, 763, 756, 756, 745, 717, 763, 759, 772, 753, 795, 810, 745, 866, 781, 721, 758, 748, 772, 762, 758, 777] payload_rows_per_dst=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] dup_rows_per_dst=[3004, 3080, 2912, 3038, 3022, 2860, 3086, 2876] combine_rows_per_src=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344464 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715536 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086608 weight_movement_bytes=34603008 total_nvlink_bytes=567689616 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344456 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944904 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545352 weight_movement_bytes=34603008 total_nvlink_bytes=568148360 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343128 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266566744 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790360 weight_movement_bytes=69206016 total_nvlink_bytes=601996376 weight_to_token_ratio=0.129894 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342680 combine_payload_bytes=265273344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265616024 ingress_bytes=403685376 link_bytes=403685376 token_bytes=530889368 weight_movement_bytes=138412032 total_nvlink_bytes=669301400 weight_to_token_ratio=0.260717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519035571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343760 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265862864 ingress_bytes=334725120 link_bytes=334725120 token_bytes=531381968 weight_movement_bytes=69206016 total_nvlink_bytes=600587984 weight_to_token_ratio=0.130238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259543679e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65042 dup_rows=24030 dup_rows_remote=21172 dedup_ratio=0.244446 router_tokens_per_expert=[775, 797, 783, 733, 779, 768, 759, 795, 800, 750, 790, 792, 752, 750, 782, 747, 809, 762, 769, 734, 769, 762, 721, 719, 763, 796, 752, 727, 752, 773, 750, 794, 765, 717, 785, 789, 806, 814, 764, 775, 696, 821, 778, 768, 788, 748, 777, 785, 754, 768, 767, 755, 764, 772, 801, 792, 766, 758, 764, 769, 745, 737, 760, 800] payload_rows_per_dst=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] dup_rows_per_dst=[3000, 3100, 2858, 2986, 3068, 3024, 3022, 2972] combine_rows_per_src=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343384 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266779992 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533216600 weight_movement_bytes=138412032 total_nvlink_bytes=671628632 weight_to_token_ratio=0.259579 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64580 dup_rows=24298 dup_rows_remote=21154 dedup_ratio=0.247172 router_tokens_per_expert=[768, 774, 753, 780, 785, 760, 771, 793, 754, 746, 776, 753, 791, 721, 792, 740, 794, 744, 737, 791, 750, 753, 731, 841, 800, 740, 788, 746, 759, 746, 778, 752, 747, 774, 717, 724, 744, 820, 769, 775, 784, 787, 789, 728, 745, 794, 795, 806, 778, 713, 758, 744, 744, 783, 759, 787, 776, 765, 763, 777, 801, 764, 818, 817] payload_rows_per_dst=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] dup_rows_per_dst=[3074, 2952, 3094, 2990, 3004, 3064, 2976, 3144] combine_rows_per_src=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344856 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756888 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168920 weight_movement_bytes=34603008 total_nvlink_bytes=567771928 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64986 dup_rows=23922 dup_rows_remote=21006 dedup_ratio=0.243347 router_tokens_per_expert=[742, 764, 782, 761, 719, 730, 750, 743, 763, 766, 752, 770, 762, 776, 764, 752, 750, 853, 741, 783, 801, 769, 775, 785, 775, 781, 723, 755, 751, 800, 780, 782, 765, 728, 778, 745, 755, 780, 769, 789, 780, 745, 744, 777, 774, 767, 778, 791, 780, 792, 762, 779, 735, 797, 736, 789, 804, 824, 769, 763, 747, 726, 763, 821] payload_rows_per_dst=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] dup_rows_per_dst=[2840, 2944, 3148, 2960, 3000, 2916, 3030, 3084] combine_rows_per_src=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64974 dup_rows=24084 dup_rows_remote=21120 dedup_ratio=0.244995 router_tokens_per_expert=[779, 794, 767, 803, 753, 767, 751, 792, 791, 765, 724, 726, 758, 814, 743, 788, 764, 738, 787, 789, 766, 760, 763, 782, 760, 817, 787, 743, 777, 806, 773, 773, 748, 727, 776, 789, 760, 798, 756, 778, 765, 792, 838, 722, 794, 761, 764, 746, 718, 761, 763, 771, 767, 821, 778, 726, 742, 778, 756, 761, 769, 724, 784, 719] payload_rows_per_dst=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] dup_rows_per_dst=[3066, 2910, 3062, 3102, 3004, 3078, 2964, 2898] combine_rows_per_src=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64678 dup_rows=24152 dup_rows_remote=21124 dedup_ratio=0.245687 router_tokens_per_expert=[808, 761, 778, 794, 724, 791, 751, 762, 752, 802, 715, 772, 828, 788, 789, 748, 786, 727, 781, 789, 791, 780, 809, 752, 715, 751, 777, 803, 778, 757, 758, 768, 777, 756, 787, 757, 780, 828, 767, 758, 756, 785, 719, 734, 756, 773, 787, 767, 789, 790, 755, 779, 752, 787, 727, 752, 719, 731, 795, 785, 792, 733, 745, 749] payload_rows_per_dst=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] dup_rows_per_dst=[3028, 3320, 3098, 3006, 2816, 2974, 2994, 2916] combine_rows_per_src=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65088 dup_rows=24048 dup_rows_remote=21198 dedup_ratio=0.244629 router_tokens_per_expert=[727, 807, 780, 727, 787, 779, 761, 809, 785, 762, 820, 785, 759, 754, 755, 805, 844, 754, 782, 801, 738, 714, 784, 745, 751, 755, 766, 767, 769, 775, 771, 764, 755, 767, 739, 775, 768, 714, 722, 769, 814, 738, 811, 762, 734, 773, 760, 767, 766, 754, 767, 774, 782, 793, 766, 755, 728, 768, 779, 758, 754, 775, 835, 748] payload_rows_per_dst=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] dup_rows_per_dst=[3028, 3196, 2954, 3060, 2850, 2976, 2936, 3048] combine_rows_per_src=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64886 dup_rows=24164 dup_rows_remote=21120 dedup_ratio=0.245809 router_tokens_per_expert=[723, 761, 759, 746, 770, 788, 727, 804, 764, 754, 789, 795, 717, 796, 766, 781, 780, 803, 779, 747, 767, 750, 756, 728, 788, 760, 756, 772, 741, 770, 762, 800, 737, 771, 780, 776, 780, 821, 803, 761, 827, 727, 786, 794, 773, 759, 795, 707, 809, 780, 769, 748, 781, 782, 769, 739, 750, 773, 772, 788, 701, 734, 796, 765] payload_rows_per_dst=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] dup_rows_per_dst=[2990, 3048, 3028, 3044, 3012, 3038, 3020, 2984] combine_rows_per_src=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342936 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264862616 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529382296 weight_movement_bytes=34603008 total_nvlink_bytes=563985304 weight_to_token_ratio=0.065365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343968 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526624 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532709280 weight_movement_bytes=69206016 total_nvlink_bytes=601915296 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65016 dup_rows=23988 dup_rows_remote=21014 dedup_ratio=0.244019 router_tokens_per_expert=[787, 799, 760, 767, 794, 781, 789, 746, 753, 769, 770, 807, 727, 786, 737, 788, 768, 760, 776, 781, 767, 769, 798, 751, 755, 765, 790, 791, 775, 779, 746, 754, 746, 796, 762, 771, 748, 714, 808, 755, 771, 768, 763, 759, 730, 767, 752, 733, 753, 771, 801, 807, 760, 777, 763, 791, 766, 756, 745, 749, 754, 782, 776, 773] payload_rows_per_dst=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] dup_rows_per_dst=[3048, 2974, 2990, 3018, 3000, 2912, 3130, 2916] combine_rows_per_src=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344376 combine_payload_bytes=266133504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266477880 ingress_bytes=369942528 link_bytes=369942528 token_bytes=532611384 weight_movement_bytes=103809024 total_nvlink_bytes=636420408 weight_to_token_ratio=0.194906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392063950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343208 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264296 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185384 weight_movement_bytes=34603008 total_nvlink_bytes=564788392 weight_to_token_ratio=0.065266 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=345144 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945592 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533546040 weight_movement_bytes=34603008 total_nvlink_bytes=568149048 weight_to_token_ratio=0.064855 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344024 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117080 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890136 weight_movement_bytes=34603008 total_nvlink_bytes=566493144 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64976 dup_rows=24032 dup_rows_remote=21100 dedup_ratio=0.244466 router_tokens_per_expert=[768, 753, 758, 790, 755, 792, 783, 833, 741, 821, 744, 737, 769, 719, 747, 787, 762, 776, 765, 765, 741, 745, 788, 772, 813, 738, 770, 733, 770, 773, 779, 821, 769, 759, 780, 761, 724, 763, 746, 750, 698, 788, 746, 813, 779, 794, 801, 783, 761, 735, 754, 775, 760, 705, 793, 780, 765, 758, 819, 760, 783, 773, 759, 810] payload_rows_per_dst=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] dup_rows_per_dst=[3018, 2974, 2932, 3044, 2962, 3082, 2896, 3124] combine_rows_per_src=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344120 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649656 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955192 weight_movement_bytes=34603008 total_nvlink_bytes=567558200 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64884 dup_rows=23944 dup_rows_remote=20578 dedup_ratio=0.243571 router_tokens_per_expert=[746, 767, 791, 741, 744, 771, 767, 777, 712, 775, 764, 734, 803, 762, 798, 752, 781, 782, 741, 743, 802, 759, 761, 777, 777, 814, 728, 751, 798, 729, 784, 761, 777, 762, 759, 755, 787, 748, 773, 787, 792, 748, 775, 768, 787, 817, 750, 786, 764, 764, 758, 792, 749, 748, 710, 755, 811, 747, 810, 752, 744, 787, 807, 791] payload_rows_per_dst=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] dup_rows_per_dst=[3032, 2868, 2776, 2954, 3054, 2832, 3062, 3366] combine_rows_per_src=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344304 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266486000 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627696 weight_movement_bytes=69206016 total_nvlink_bytes=601833712 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65274 dup_rows=23884 dup_rows_remote=20876 dedup_ratio=0.242961 router_tokens_per_expert=[774, 795, 734, 775, 733, 717, 760, 798, 758, 801, 810, 749, 797, 722, 792, 763, 754, 756, 780, 775, 778, 760, 752, 760, 742, 792, 776, 749, 792, 767, 765, 814, 750, 732, 764, 797, 784, 784, 823, 759, 748, 791, 778, 752, 768, 810, 674, 756, 792, 732, 808, 786, 776, 813, 754, 754, 767, 745, 753, 788, 760, 717, 737, 810] payload_rows_per_dst=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] dup_rows_per_dst=[2932, 3002, 3000, 3106, 3008, 3008, 2944, 2884] combine_rows_per_src=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64936 dup_rows=24094 dup_rows_remote=21082 dedup_ratio=0.245097 router_tokens_per_expert=[739, 734, 769, 786, 748, 765, 761, 745, 746, 818, 707, 767, 734, 762, 723, 770, 786, 804, 794, 796, 758, 762, 747, 814, 712, 812, 741, 805, 756, 762, 731, 767, 752, 811, 782, 780, 763, 734, 724, 760, 774, 776, 785, 834, 770, 772, 792, 756, 778, 794, 722, 738, 778, 767, 770, 796, 814, 775, 775, 787, 771, 744, 761, 796] payload_rows_per_dst=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] dup_rows_per_dst=[2922, 2956, 3098, 3016, 2986, 3064, 3012, 3040] combine_rows_per_src=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64802 dup_rows=24196 dup_rows_remote=21214 dedup_ratio=0.246134 router_tokens_per_expert=[782, 774, 724, 801, 758, 744, 827, 775, 779, 744, 771, 725, 754, 744, 780, 740, 790, 762, 745, 794, 734, 721, 772, 781, 808, 824, 762, 809, 722, 746, 761, 763, 790, 754, 751, 763, 731, 793, 823, 739, 785, 763, 797, 788, 761, 795, 759, 747, 751, 773, 765, 744, 748, 746, 797, 750, 793, 777, 754, 826, 769, 770, 788, 746] payload_rows_per_dst=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] dup_rows_per_dst=[2972, 2968, 3016, 3150, 2982, 3042, 2948, 3118] combine_rows_per_src=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64692 dup_rows=24216 dup_rows_remote=21106 dedup_ratio=0.246338 router_tokens_per_expert=[782, 744, 787, 771, 733, 760, 752, 773, 769, 741, 765, 749, 782, 728, 797, 802, 790, 790, 761, 792, 731, 779, 769, 737, 737, 761, 768, 781, 789, 783, 762, 753, 729, 748, 818, 783, 748, 793, 765, 782, 751, 808, 759, 791, 747, 748, 764, 780, 810, 760, 746, 726, 747, 739, 780, 795, 769, 737, 816, 779, 777, 798, 791, 750] payload_rows_per_dst=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] dup_rows_per_dst=[3110, 3010, 3070, 3020, 3064, 2932, 2880, 3130] combine_rows_per_src=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=341848 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266106712 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531871576 weight_movement_bytes=69206016 total_nvlink_bytes=601077592 weight_to_token_ratio=0.130118 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64956 dup_rows=24064 dup_rows_remote=20986 dedup_ratio=0.244792 router_tokens_per_expert=[763, 758, 763, 794, 732, 804, 791, 715, 742, 750, 765, 781, 747, 756, 758, 792, 761, 785, 782, 736, 776, 774, 787, 758, 761, 778, 715, 777, 750, 800, 781, 819, 774, 776, 794, 776, 754, 800, 758, 779, 768, 775, 755, 820, 769, 767, 766, 794, 789, 740, 726, 771, 747, 761, 711, 737, 786, 757, 778, 770, 793, 769, 778, 763] payload_rows_per_dst=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] dup_rows_per_dst=[3000, 2878, 3034, 3078, 3074, 3138, 2810, 3052] combine_rows_per_src=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344600 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706904 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069208 weight_movement_bytes=69206016 total_nvlink_bytes=604275224 weight_to_token_ratio=0.129340 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65140 dup_rows=23846 dup_rows_remote=20806 dedup_ratio=0.242574 router_tokens_per_expert=[778, 737, 759, 786, 794, 736, 764, 751, 783, 766, 816, 734, 776, 804, 730, 770, 785, 765, 737, 782, 785, 783, 769, 787, 745, 775, 759, 754, 706, 822, 798, 771, 755, 777, 788, 820, 766, 722, 751, 787, 819, 747, 824, 743, 806, 746, 769, 761, 797, 750, 760, 731, 781, 754, 738, 757, 743, 763, 813, 740, 785, 747, 687, 818] payload_rows_per_dst=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] dup_rows_per_dst=[2934, 3040, 3004, 3010, 3004, 2998, 2948, 2908] combine_rows_per_src=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=344072 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321928 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299784 weight_movement_bytes=34603008 total_nvlink_bytes=566902792 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344064 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265773056 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531202048 weight_movement_bytes=34603008 total_nvlink_bytes=565805056 weight_to_token_ratio=0.065141 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=343192 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265321624 ingress_bytes=299581440 link_bytes=299581440 token_bytes=530300056 weight_movement_bytes=34603008 total_nvlink_bytes=564903064 weight_to_token_ratio=0.065252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127300840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64994 dup_rows=23980 dup_rows_remote=20954 dedup_ratio=0.243937 router_tokens_per_expert=[810, 772, 737, 799, 780, 796, 770, 702, 739, 745, 776, 723, 799, 786, 784, 778, 803, 793, 765, 759, 773, 734, 764, 787, 778, 812, 794, 751, 745, 779, 801, 784, 739, 707, 728, 777, 795, 768, 764, 767, 821, 822, 763, 733, 775, 761, 791, 766, 771, 726, 769, 730, 761, 724, 791, 776, 765, 766, 786, 781, 803, 759, 741, 708] payload_rows_per_dst=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] dup_rows_per_dst=[3032, 2896, 3026, 3028, 2858, 3108, 2968, 3064] combine_rows_per_src=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343768 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266403544 ingress_bytes=404471808 link_bytes=404471808 token_bytes=532463320 weight_movement_bytes=138412032 total_nvlink_bytes=670875352 weight_to_token_ratio=0.259947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521994851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343784 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157224 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970664 weight_movement_bytes=34603008 total_nvlink_bytes=568573672 weight_to_token_ratio=0.064803 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65096 dup_rows=23724 dup_rows_remote=20720 dedup_ratio=0.241333 router_tokens_per_expert=[747, 787, 740, 754, 785, 744, 724, 755, 776, 755, 750, 757, 749, 773, 745, 783, 752, 764, 761, 741, 775, 816, 738, 783, 781, 788, 752, 776, 773, 743, 701, 816, 756, 806, 780, 749, 772, 751, 758, 790, 801, 810, 785, 744, 761, 744, 751, 807, 788, 758, 759, 747, 742, 790, 786, 797, 784, 787, 762, 771, 850, 747, 795, 740] payload_rows_per_dst=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] dup_rows_per_dst=[2892, 2928, 2874, 3022, 3010, 2768, 3226, 3004] combine_rows_per_src=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343792 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559216 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774640 weight_movement_bytes=69206016 total_nvlink_bytes=601980656 weight_to_token_ratio=0.129897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64962 dup_rows=24020 dup_rows_remote=20938 dedup_ratio=0.244344 router_tokens_per_expert=[772, 736, 770, 781, 739, 738, 789, 815, 784, 754, 741, 758, 766, 795, 754, 804, 791, 755, 760, 832, 763, 709, 769, 743, 746, 763, 739, 742, 754, 790, 812, 754, 758, 792, 740, 828, 710, 751, 820, 761, 771, 755, 800, 809, 789, 759, 744, 775, 790, 747, 748, 799, 717, 790, 768, 780, 778, 750, 762, 754, 747, 789, 785, 768] payload_rows_per_dst=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] dup_rows_per_dst=[2944, 3076, 3004, 2958, 3016, 3082, 2946, 2994] combine_rows_per_src=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65162 dup_rows=23872 dup_rows_remote=20884 dedup_ratio=0.242839 router_tokens_per_expert=[765, 787, 746, 791, 760, 731, 750, 805, 770, 826, 764, 757, 747, 750, 773, 759, 765, 748, 770, 775, 798, 776, 752, 761, 743, 795, 699, 790, 752, 743, 748, 756, 769, 727, 800, 778, 763, 738, 792, 781, 794, 801, 769, 752, 766, 809, 784, 752, 752, 775, 733, 807, 772, 770, 778, 742, 766, 758, 801, 761, 807, 736, 802, 765] payload_rows_per_dst=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] dup_rows_per_dst=[3048, 2946, 2960, 2904, 2984, 3016, 2988, 3026] combine_rows_per_src=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65234 dup_rows=23756 dup_rows_remote=20782 dedup_ratio=0.241659 router_tokens_per_expert=[804, 785, 755, 766, 720, 755, 727, 754, 776, 787, 769, 777, 812, 751, 800, 721, 735, 827, 742, 775, 786, 796, 760, 766, 719, 762, 753, 770, 766, 763, 745, 781, 767, 793, 748, 788, 788, 749, 758, 753, 805, 774, 783, 775, 714, 791, 755, 820, 765, 782, 800, 765, 766, 727, 730, 799, 778, 784, 779, 759, 757, 765, 784, 746] payload_rows_per_dst=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] dup_rows_per_dst=[2932, 3060, 3016, 2944, 2974, 2954, 2904, 2972] combine_rows_per_src=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64786 dup_rows=24246 dup_rows_remote=21208 dedup_ratio=0.246643 router_tokens_per_expert=[785, 739, 789, 788, 796, 718, 757, 747, 766, 774, 733, 762, 756, 781, 788, 762, 727, 811, 776, 793, 787, 778, 768, 803, 782, 791, 767, 729, 750, 748, 779, 796, 801, 763, 732, 824, 744, 776, 767, 764, 771, 726, 785, 767, 819, 747, 770, 725, 822, 735, 787, 783, 771, 751, 752, 777, 764, 791, 772, 708, 744, 737, 803, 748] payload_rows_per_dst=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] dup_rows_per_dst=[3038, 3378, 3202, 2988, 3008, 3006, 3052, 2574] combine_rows_per_src=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343264 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266976480 ingress_bytes=370442240 link_bytes=370442240 token_bytes=533609696 weight_movement_bytes=103809024 total_nvlink_bytes=637418720 weight_to_token_ratio=0.194541 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393944326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343600 combine_payload_bytes=266084352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266427952 ingress_bytes=404496384 link_bytes=404496384 token_bytes=532512304 weight_movement_bytes=138412032 total_nvlink_bytes=670924336 weight_to_token_ratio=0.259923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522087328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65046 dup_rows=23964 dup_rows_remote=21092 dedup_ratio=0.243774 router_tokens_per_expert=[802, 826, 789, 771, 775, 771, 740, 783, 777, 757, 734, 786, 740, 799, 775, 716, 768, 813, 818, 751, 754, 774, 785, 776, 783, 760, 733, 823, 750, 761, 782, 691, 755, 757, 766, 785, 750, 765, 800, 773, 772, 764, 755, 767, 753, 755, 751, 781, 771, 786, 723, 772, 729, 762, 762, 782, 770, 756, 760, 797, 764, 765, 819, 722] payload_rows_per_dst=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] dup_rows_per_dst=[2992, 3006, 3096, 2872, 3008, 2986, 2964, 3040] combine_rows_per_src=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[781, 739, 803, 799, 779, 819, 714, 746, 794, 835, 756, 781, 774, 762, 803, 713, 767, 763, 805, 777, 758, 746, 733, 858, 749, 745, 756, 787, 742, 737, 748, 756, 746, 746, 740, 775, 744, 752, 788, 761, 769, 754, 791, 831, 754, 775, 759, 751, 774, 785, 741, 766, 724, 783, 817, 776, 770, 747, 774, 776, 730, 774, 772, 782] payload_rows_per_dst=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] dup_rows_per_dst=[3022, 3036, 3088, 2926, 2916, 3010, 3066, 2976] combine_rows_per_src=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344184 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247736 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151288 weight_movement_bytes=34603008 total_nvlink_bytes=568754296 weight_to_token_ratio=0.064781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344064 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267542528 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534740992 weight_movement_bytes=34603008 total_nvlink_bytes=569344000 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343976 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707432 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070888 weight_movement_bytes=34603008 total_nvlink_bytes=565673896 weight_to_token_ratio=0.065157 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65204 dup_rows=23628 dup_rows_remote=20530 dedup_ratio=0.240356 router_tokens_per_expert=[736, 726, 751, 748, 749, 763, 734, 739, 732, 763, 785, 745, 764, 782, 787, 802, 756, 776, 729, 792, 873, 790, 771, 798, 809, 785, 781, 703, 764, 782, 755, 832, 745, 798, 814, 759, 757, 771, 781, 756, 797, 731, 770, 778, 814, 760, 717, 802, 712, 721, 757, 777, 806, 791, 771, 707, 786, 742, 719, 806, 779, 737, 789, 800] payload_rows_per_dst=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] dup_rows_per_dst=[2812, 2944, 3098, 3058, 2926, 2970, 2866, 2954] combine_rows_per_src=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344552 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772968 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201384 weight_movement_bytes=34603008 total_nvlink_bytes=567804392 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343472 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266026416 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531709360 weight_movement_bytes=103809024 total_nvlink_bytes=635518384 weight_to_token_ratio=0.195236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=342936 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267418520 ingress_bytes=370884608 link_bytes=370884608 token_bytes=534494104 weight_movement_bytes=103809024 total_nvlink_bytes=638303128 weight_to_token_ratio=0.194219 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395608921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64644 dup_rows=24256 dup_rows_remote=21114 dedup_ratio=0.246745 router_tokens_per_expert=[779, 749, 756, 766, 745, 790, 825, 784, 787, 771, 773, 790, 727, 725, 786, 712, 795, 776, 765, 778, 820, 770, 757, 786, 787, 765, 780, 760, 828, 753, 781, 731, 732, 772, 792, 765, 775, 711, 756, 744, 759, 791, 779, 767, 737, 755, 751, 801, 721, 740, 759, 743, 760, 795, 813, 737, 785, 778, 794, 757, 761, 800, 791, 734] payload_rows_per_dst=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] dup_rows_per_dst=[3094, 2542, 3020, 3282, 3182, 3092, 2902, 3142] combine_rows_per_src=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64858 dup_rows=24048 dup_rows_remote=20996 dedup_ratio=0.244629 router_tokens_per_expert=[720, 740, 752, 796, 756, 725, 783, 777, 770, 772, 817, 769, 798, 787, 765, 764, 707, 747, 773, 770, 804, 778, 785, 752, 764, 759, 758, 760, 781, 759, 744, 702, 731, 781, 757, 749, 749, 783, 785, 748, 796, 761, 769, 777, 788, 764, 736, 834, 809, 822, 718, 814, 767, 801, 809, 727, 762, 735, 746, 758, 769, 781, 770, 822] payload_rows_per_dst=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] dup_rows_per_dst=[3004, 3140, 3006, 2894, 2936, 3052, 3080, 2936] combine_rows_per_src=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20928 dedup_ratio=0.244364 router_tokens_per_expert=[733, 794, 755, 773, 716, 757, 793, 810, 766, 826, 757, 715, 809, 743, 786, 787, 780, 767, 770, 827, 780, 799, 823, 730, 767, 755, 749, 741, 743, 806, 780, 758, 789, 746, 785, 789, 773, 779, 821, 749, 798, 802, 742, 796, 739, 725, 767, 762, 771, 743, 752, 735, 750, 816, 764, 745, 751, 736, 775, 719, 734, 751, 774, 779] payload_rows_per_dst=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] dup_rows_per_dst=[3026, 3140, 3128, 2942, 3094, 2940, 2840, 2912] combine_rows_per_src=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65066 dup_rows=23898 dup_rows_remote=20842 dedup_ratio=0.243103 router_tokens_per_expert=[787, 760, 785, 770, 745, 794, 753, 774, 783, 829, 772, 769, 738, 751, 757, 782, 766, 748, 735, 718, 737, 774, 725, 798, 767, 757, 756, 741, 748, 771, 815, 797, 738, 819, 758, 780, 756, 759, 761, 720, 773, 755, 793, 769, 720, 777, 771, 811, 785, 762, 793, 772, 749, 775, 779, 783, 786, 767, 785, 779, 750, 777, 774, 774] payload_rows_per_dst=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] dup_rows_per_dst=[2988, 3040, 2842, 3026, 2914, 2998, 3056, 3034] combine_rows_per_src=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64872 dup_rows=24078 dup_rows_remote=20938 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 780, 763, 796, 764, 744, 759, 770, 772, 792, 778, 738, 748, 738, 803, 723, 743, 761, 751, 795, 753, 786, 752, 802, 761, 790, 812, 804, 807, 722, 765, 773, 780, 750, 747, 763, 748, 775, 788, 751, 715, 774, 816, 758, 779, 729, 758, 826, 772, 757, 748, 759, 734, 770, 790, 814, 772, 752, 756, 775, 734, 785, 770] payload_rows_per_dst=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] dup_rows_per_dst=[3140, 3106, 2934, 3086, 2996, 2818, 3078, 2920] combine_rows_per_src=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=343032 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265124856 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529906680 weight_movement_bytes=34603008 total_nvlink_bytes=564509688 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343416 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266001784 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531660152 weight_movement_bytes=69206016 total_nvlink_bytes=600866168 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65142 dup_rows=23876 dup_rows_remote=20872 dedup_ratio=0.242879 router_tokens_per_expert=[804, 758, 777, 748, 752, 778, 744, 759, 770, 759, 784, 778, 746, 781, 780, 747, 772, 758, 770, 739, 744, 792, 816, 777, 770, 755, 789, 756, 758, 778, 812, 775, 750, 819, 793, 751, 773, 802, 778, 796, 763, 768, 798, 790, 783, 766, 733, 756, 755, 742, 716, 771, 730, 687, 748, 769, 788, 761, 765, 761, 781, 780, 786, 767] payload_rows_per_dst=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] dup_rows_per_dst=[2990, 3004, 3054, 3058, 3040, 2942, 2768, 3020] combine_rows_per_src=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73942 payload_rows_remote=64714 dup_rows=24362 dup_rows_remote=21292 dedup_ratio=0.247823 router_tokens_per_expert=[764, 794, 758, 746, 757, 732, 745, 813, 774, 768, 803, 762, 805, 791, 763, 763, 731, 809, 772, 770, 756, 762, 766, 747, 764, 769, 785, 786, 777, 746, 744, 778, 737, 796, 800, 763, 757, 764, 735, 773, 795, 839, 757, 741, 746, 756, 732, 731, 797, 789, 760, 785, 808, 756, 750, 775, 740, 741, 790, 796, 760, 768, 776, 739] payload_rows_per_dst=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] dup_rows_per_dst=[3002, 3104, 2984, 3070, 3000, 3028, 3150, 3024] combine_rows_per_src=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343368 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231112 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118856 weight_movement_bytes=34603008 total_nvlink_bytes=566721864 weight_to_token_ratio=0.065029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266853968 ingress_bytes=335716352 link_bytes=335716352 token_bytes=533364304 weight_movement_bytes=69206016 total_nvlink_bytes=602570320 weight_to_token_ratio=0.129754 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263273605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65078 dup_rows=23908 dup_rows_remote=20896 dedup_ratio=0.243205 router_tokens_per_expert=[732, 784, 723, 715, 768, 724, 738, 736, 766, 773, 744, 772, 789, 768, 761, 735, 772, 783, 779, 764, 748, 760, 772, 787, 743, 758, 841, 796, 757, 747, 818, 797, 738, 778, 757, 733, 788, 803, 743, 808, 791, 746, 798, 771, 809, 755, 809, 787, 790, 749, 776, 744, 713, 747, 796, 803, 777, 787, 775, 758, 788, 759, 788, 738] payload_rows_per_dst=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] dup_rows_per_dst=[2738, 2988, 3012, 3080, 2966, 3106, 3036, 2982] combine_rows_per_src=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343240 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266058952 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531774664 weight_movement_bytes=34603008 total_nvlink_bytes=566377672 weight_to_token_ratio=0.065071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344056 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165688 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987320 weight_movement_bytes=34603008 total_nvlink_bytes=568590328 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344024 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265412568 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530481112 weight_movement_bytes=34603008 total_nvlink_bytes=565084120 weight_to_token_ratio=0.065229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343896 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266903384 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462872 weight_movement_bytes=103809024 total_nvlink_bytes=637271896 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65006 dup_rows=24048 dup_rows_remote=21038 dedup_ratio=0.244629 router_tokens_per_expert=[783, 805, 741, 788, 736, 769, 757, 798, 769, 773, 754, 747, 720, 745, 744, 777, 780, 738, 780, 771, 759, 768, 788, 766, 754, 786, 759, 799, 761, 764, 802, 777, 783, 775, 721, 801, 748, 792, 797, 780, 781, 762, 719, 789, 815, 779, 776, 734, 791, 789, 828, 795, 794, 763, 735, 758, 767, 672, 747, 751, 758, 789, 763, 742] payload_rows_per_dst=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] dup_rows_per_dst=[2954, 2912, 2796, 3140, 2988, 2964, 3284, 3010] combine_rows_per_src=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64990 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[767, 776, 746, 764, 731, 773, 781, 791, 752, 745, 775, 786, 796, 740, 776, 781, 778, 741, 788, 760, 750, 780, 761, 791, 806, 795, 796, 691, 804, 703, 757, 729, 789, 764, 734, 757, 791, 772, 753, 826, 772, 787, 775, 784, 750, 760, 782, 783, 820, 782, 751, 756, 767, 752, 812, 783, 759, 704, 788, 772, 746, 754, 756, 761] payload_rows_per_dst=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] dup_rows_per_dst=[3022, 3060, 2916, 2978, 3046, 2996, 3024, 2882] combine_rows_per_src=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65092 dup_rows=24024 dup_rows_remote=21016 dedup_ratio=0.244385 router_tokens_per_expert=[734, 749, 774, 796, 786, 794, 757, 771, 731, 746, 747, 755, 802, 769, 747, 794, 786, 816, 781, 772, 760, 751, 750, 793, 817, 789, 854, 769, 799, 818, 827, 774, 734, 773, 763, 754, 777, 773, 780, 744, 767, 765, 733, 747, 762, 746, 794, 747, 753, 768, 744, 783, 743, 749, 728, 782, 756, 738, 782, 712, 736, 738, 809, 764] payload_rows_per_dst=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] dup_rows_per_dst=[3000, 2998, 3030, 3290, 3008, 2938, 2892, 2868] combine_rows_per_src=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73964 payload_rows_remote=64752 dup_rows=24340 dup_rows_remote=21302 dedup_ratio=0.247599 router_tokens_per_expert=[757, 769, 822, 769, 764, 771, 770, 822, 713, 758, 769, 737, 756, 766, 796, 781, 764, 813, 768, 791, 782, 775, 789, 754, 781, 795, 752, 787, 783, 750, 782, 715, 759, 758, 784, 757, 776, 783, 718, 726, 741, 789, 789, 764, 770, 748, 717, 779, 758, 783, 758, 734, 746, 791, 758, 797, 752, 790, 739, 814, 731, 791, 797, 754] payload_rows_per_dst=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] dup_rows_per_dst=[3132, 3012, 3124, 3048, 3008, 2926, 3038, 3052] combine_rows_per_src=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64726 dup_rows=24140 dup_rows_remote=20828 dedup_ratio=0.245565 router_tokens_per_expert=[788, 807, 782, 768, 783, 752, 749, 761, 776, 787, 812, 746, 791, 768, 761, 774, 761, 753, 755, 752, 744, 761, 702, 787, 712, 781, 747, 819, 807, 754, 783, 729, 723, 758, 787, 763, 798, 748, 795, 757, 751, 746, 798, 802, 764, 806, 793, 719, 754, 750, 764, 786, 778, 811, 748, 769, 737, 753, 796, 769, 783, 754, 756, 784] payload_rows_per_dst=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] dup_rows_per_dst=[3312, 3080, 2902, 2930, 3052, 2842, 3012, 3010] combine_rows_per_src=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344176 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608752 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873328 weight_movement_bytes=34603008 total_nvlink_bytes=567476336 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65092 dup_rows=23980 dup_rows_remote=21074 dedup_ratio=0.243937 router_tokens_per_expert=[716, 761, 734, 765, 802, 772, 778, 816, 759, 771, 746, 746, 756, 763, 798, 730, 776, 753, 768, 746, 784, 770, 772, 751, 790, 749, 718, 803, 784, 775, 775, 810, 780, 788, 721, 825, 795, 756, 747, 761, 816, 759, 791, 760, 750, 765, 767, 816, 783, 759, 810, 795, 784, 771, 720, 751, 729, 776, 739, 756, 762, 760, 757, 766] payload_rows_per_dst=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] dup_rows_per_dst=[3038, 2906, 2952, 3016, 3068, 3128, 3066, 2806] combine_rows_per_src=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343672 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542712 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741752 weight_movement_bytes=69206016 total_nvlink_bytes=601947768 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344432 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961264 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578096 weight_movement_bytes=69206016 total_nvlink_bytes=602784112 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65270 dup_rows=23762 dup_rows_remote=20956 dedup_ratio=0.241720 router_tokens_per_expert=[824, 786, 777, 737, 798, 798, 779, 787, 753, 764, 759, 729, 737, 770, 758, 780, 774, 803, 786, 777, 760, 789, 748, 783, 747, 706, 726, 773, 765, 764, 802, 756, 761, 739, 759, 821, 719, 754, 705, 799, 806, 741, 816, 777, 757, 748, 763, 766, 748, 750, 775, 780, 793, 730, 755, 793, 774, 777, 816, 753, 793, 750, 784, 755] payload_rows_per_dst=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] dup_rows_per_dst=[3120, 3008, 3028, 2806, 2810, 3028, 2958, 3004] combine_rows_per_src=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65316 dup_rows=23782 dup_rows_remote=20704 dedup_ratio=0.241923 router_tokens_per_expert=[733, 731, 782, 807, 756, 752, 752, 768, 748, 764, 772, 782, 762, 782, 779, 737, 762, 764, 779, 757, 776, 783, 793, 728, 794, 788, 728, 734, 748, 792, 756, 769, 774, 790, 807, 747, 805, 749, 764, 739, 782, 741, 786, 758, 796, 791, 755, 771, 804, 744, 741, 774, 763, 723, 788, 777, 781, 785, 770, 749, 760, 803, 764, 813] payload_rows_per_dst=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] dup_rows_per_dst=[2936, 2950, 3078, 2848, 2932, 3052, 2986, 3000] combine_rows_per_src=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265224192 dispatch_meta_bytes=344216 combine_payload_bytes=265224192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265568408 ingress_bytes=299827200 link_bytes=299827200 token_bytes=530792600 weight_movement_bytes=34603008 total_nvlink_bytes=565395608 weight_to_token_ratio=0.065191 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128225616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342216 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265459912 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577608 weight_movement_bytes=34603008 total_nvlink_bytes=565180616 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344664 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266961496 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533578328 weight_movement_bytes=34603008 total_nvlink_bytes=568181336 weight_to_token_ratio=0.064851 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344904 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690824 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535036744 weight_movement_bytes=69206016 total_nvlink_bytes=604242760 weight_to_token_ratio=0.129348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=344080 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267878416 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535412752 weight_movement_bytes=34603008 total_nvlink_bytes=570015760 weight_to_token_ratio=0.064629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64850 dup_rows=24048 dup_rows_remote=21030 dedup_ratio=0.244629 router_tokens_per_expert=[774, 777, 781, 801, 795, 816, 729, 718, 751, 778, 792, 731, 756, 712, 695, 762, 746, 790, 753, 741, 754, 757, 729, 890, 742, 811, 759, 721, 795, 809, 753, 796, 723, 834, 747, 710, 740, 771, 773, 769, 755, 808, 786, 738, 788, 807, 780, 790, 780, 703, 740, 797, 762, 792, 800, 747, 803, 748, 762, 774, 735, 787, 832, 757] payload_rows_per_dst=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] dup_rows_per_dst=[3086, 2844, 3016, 3106, 2972, 3050, 2956, 3018] combine_rows_per_src=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64824 dup_rows=24110 dup_rows_remote=21062 dedup_ratio=0.245260 router_tokens_per_expert=[727, 793, 733, 739, 811, 759, 769, 740, 759, 786, 796, 770, 765, 741, 764, 691, 771, 762, 734, 738, 809, 736, 718, 805, 788, 802, 773, 748, 782, 786, 762, 752, 750, 795, 782, 807, 776, 801, 776, 774, 769, 732, 767, 796, 784, 796, 761, 804, 697, 760, 787, 769, 773, 812, 755, 750, 774, 775, 729, 762, 801, 744, 787, 798] payload_rows_per_dst=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] dup_rows_per_dst=[2868, 3006, 2978, 3026, 3120, 3048, 2982, 3082] combine_rows_per_src=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65066 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[752, 746, 767, 728, 762, 713, 765, 744, 785, 736, 752, 767, 788, 783, 797, 814, 752, 788, 743, 751, 789, 703, 755, 739, 788, 802, 774, 810, 795, 780, 809, 732, 801, 766, 732, 743, 745, 747, 785, 764, 766, 807, 770, 817, 784, 711, 783, 747, 772, 757, 785, 773, 791, 765, 733, 756, 785, 752, 778, 749, 799, 823, 778, 779] payload_rows_per_dst=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] dup_rows_per_dst=[2828, 3030, 2962, 3226, 2864, 2972, 3026, 3028] combine_rows_per_src=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65018 dup_rows=23842 dup_rows_remote=20810 dedup_ratio=0.242533 router_tokens_per_expert=[718, 776, 769, 744, 760, 770, 785, 753, 759, 785, 834, 762, 734, 765, 778, 770, 726, 777, 765, 778, 791, 743, 761, 744, 784, 751, 801, 751, 777, 751, 748, 796, 753, 750, 727, 758, 753, 765, 767, 742, 824, 787, 776, 768, 763, 767, 781, 773, 765, 803, 758, 794, 779, 798, 787, 754, 737, 771, 753, 751, 822, 752, 764, 804] payload_rows_per_dst=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] dup_rows_per_dst=[2894, 3118, 2978, 2970, 2868, 3056, 3032, 2926] combine_rows_per_src=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65222 dup_rows=23778 dup_rows_remote=20852 dedup_ratio=0.241882 router_tokens_per_expert=[808, 743, 749, 736, 786, 779, 736, 778, 764, 763, 815, 721, 795, 759, 759, 766, 801, 768, 755, 789, 769, 714, 768, 752, 746, 758, 769, 743, 751, 746, 763, 806, 789, 783, 817, 760, 787, 721, 767, 761, 765, 803, 761, 792, 771, 770, 707, 767, 756, 805, 764, 792, 743, 733, 772, 793, 770, 773, 773, 811, 802, 802, 744, 743] payload_rows_per_dst=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] dup_rows_per_dst=[2926, 3306, 2874, 2638, 3004, 3002, 2994, 3034] combine_rows_per_src=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343520 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265969120 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594720 weight_movement_bytes=69206016 total_nvlink_bytes=600800736 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343544 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265862648 ingress_bytes=369328128 link_bytes=369328128 token_bytes=531381752 weight_movement_bytes=103809024 total_nvlink_bytes=635190776 weight_to_token_ratio=0.195357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389752012e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65146 dup_rows=23866 dup_rows_remote=20900 dedup_ratio=0.242778 router_tokens_per_expert=[729, 791, 774, 765, 768, 748, 782, 704, 741, 709, 826, 757, 783, 768, 781, 764, 785, 793, 777, 734, 704, 812, 734, 758, 809, 772, 776, 752, 789, 776, 808, 739, 743, 760, 769, 776, 759, 748, 780, 807, 805, 790, 782, 776, 818, 734, 730, 766, 769, 745, 745, 744, 731, 786, 819, 798, 761, 764, 757, 766, 757, 801, 801, 757] payload_rows_per_dst=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] dup_rows_per_dst=[2918, 2966, 2960, 3036, 2956, 3020, 3028, 2982] combine_rows_per_src=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344552 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854888 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533365224 weight_movement_bytes=34603008 total_nvlink_bytes=567968232 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64978 dup_rows=24114 dup_rows_remote=21156 dedup_ratio=0.245300 router_tokens_per_expert=[807, 752, 806, 803, 737, 771, 763, 803, 751, 783, 731, 743, 782, 769, 779, 755, 800, 782, 751, 742, 772, 763, 730, 745, 786, 766, 834, 787, 744, 752, 761, 734, 761, 735, 777, 717, 797, 744, 767, 794, 739, 774, 807, 742, 754, 783, 720, 767, 793, 783, 743, 795, 787, 761, 800, 762, 760, 783, 795, 740, 785, 738, 799, 766] payload_rows_per_dst=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] dup_rows_per_dst=[3122, 3082, 2958, 2986, 2936, 2984, 3112, 2934] combine_rows_per_src=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65202 dup_rows=23844 dup_rows_remote=20976 dedup_ratio=0.242554 router_tokens_per_expert=[814, 767, 746, 758, 775, 807, 775, 755, 786, 763, 778, 763, 769, 805, 748, 820, 767, 785, 756, 784, 743, 766, 770, 774, 748, 758, 780, 756, 737, 772, 765, 747, 728, 752, 777, 810, 789, 733, 803, 824, 770, 802, 750, 770, 760, 715, 766, 769, 773, 757, 789, 816, 766, 748, 733, 752, 751, 779, 756, 730, 790, 737, 727, 793] payload_rows_per_dst=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] dup_rows_per_dst=[3022, 3036, 3014, 2868, 3114, 2888, 2970, 2932] combine_rows_per_src=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343312 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657040 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532970768 weight_movement_bytes=69206016 total_nvlink_bytes=602176784 weight_to_token_ratio=0.129850 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344296 combine_payload_bytes=267149312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267493608 ingress_bytes=267149312 link_bytes=267493608 token_bytes=534642920 weight_movement_bytes=0 total_nvlink_bytes=534642920 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.006556912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344184 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182200 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020216 weight_movement_bytes=34603008 total_nvlink_bytes=568623224 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344536 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494424 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644312 weight_movement_bytes=34603008 total_nvlink_bytes=567247320 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344712 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267412104 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534479496 weight_movement_bytes=103809024 total_nvlink_bytes=638288520 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64742 dup_rows=23984 dup_rows_remote=20656 dedup_ratio=0.243978 router_tokens_per_expert=[780, 800, 796, 756, 779, 704, 729, 758, 770, 733, 787, 729, 738, 752, 817, 765, 773, 750, 741, 792, 778, 781, 773, 786, 763, 758, 781, 782, 720, 740, 801, 745, 768, 768, 793, 780, 745, 733, 770, 803, 762, 771, 728, 819, 766, 794, 811, 721, 810, 790, 746, 814, 794, 781, 758, 767, 816, 767, 710, 769, 756, 797, 748, 740] payload_rows_per_dst=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] dup_rows_per_dst=[2868, 2900, 3012, 3002, 3004, 3034, 2836, 3328] combine_rows_per_src=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65098 dup_rows=23948 dup_rows_remote=21144 dedup_ratio=0.243612 router_tokens_per_expert=[762, 805, 779, 767, 748, 748, 774, 781, 754, 822, 770, 802, 741, 795, 736, 755, 777, 790, 745, 777, 766, 781, 781, 788, 776, 756, 771, 769, 765, 819, 761, 755, 782, 830, 811, 777, 744, 746, 799, 763, 767, 741, 790, 750, 755, 753, 743, 732, 804, 734, 740, 797, 761, 725, 791, 776, 747, 761, 733, 782, 745, 771, 747, 739] payload_rows_per_dst=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] dup_rows_per_dst=[3056, 3030, 3010, 3116, 3062, 2804, 2950, 2920] combine_rows_per_src=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65046 dup_rows=23926 dup_rows_remote=20890 dedup_ratio=0.243388 router_tokens_per_expert=[844, 770, 753, 757, 814, 715, 754, 782, 795, 776, 755, 721, 774, 749, 783, 813, 799, 767, 791, 753, 769, 729, 768, 779, 767, 774, 726, 747, 757, 782, 771, 761, 773, 756, 754, 767, 801, 756, 773, 804, 812, 744, 768, 774, 788, 796, 759, 745, 825, 741, 750, 764, 762, 768, 789, 782, 763, 755, 764, 752, 756, 750, 755, 711] payload_rows_per_dst=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] dup_rows_per_dst=[3010, 3032, 3010, 2906, 3036, 2994, 3100, 2838] combine_rows_per_src=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65092 dup_rows=23910 dup_rows_remote=20870 dedup_ratio=0.243225 router_tokens_per_expert=[749, 759, 807, 781, 760, 746, 766, 736, 748, 800, 779, 826, 803, 724, 725, 796, 749, 782, 819, 780, 748, 764, 794, 717, 749, 758, 777, 778, 816, 754, 747, 802, 720, 767, 750, 739, 734, 735, 771, 811, 833, 788, 790, 789, 773, 777, 712, 746, 759, 766, 810, 744, 791, 794, 771, 736, 789, 783, 770, 725, 793, 743, 764, 740] payload_rows_per_dst=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] dup_rows_per_dst=[2930, 3034, 2968, 3056, 2876, 3026, 3040, 2980] combine_rows_per_src=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341592 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524824 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530708056 weight_movement_bytes=34603008 total_nvlink_bytes=565311064 weight_to_token_ratio=0.065202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266986376 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533627784 weight_movement_bytes=69206016 total_nvlink_bytes=602833800 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64982 dup_rows=24066 dup_rows_remote=21096 dedup_ratio=0.244812 router_tokens_per_expert=[790, 755, 787, 749, 740, 737, 788, 765, 742, 774, 737, 773, 766, 780, 750, 791, 738, 787, 800, 741, 763, 789, 774, 791, 757, 726, 745, 802, 751, 786, 766, 772, 727, 785, 726, 759, 758, 818, 760, 734, 817, 772, 785, 761, 772, 766, 731, 777, 824, 764, 737, 762, 774, 734, 784, 784, 739, 802, 787, 769, 766, 773, 815, 778] payload_rows_per_dst=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] dup_rows_per_dst=[3014, 2970, 3038, 2988, 2902, 3032, 3036, 3086] combine_rows_per_src=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=64690 dup_rows=24030 dup_rows_remote=20782 dedup_ratio=0.244446 router_tokens_per_expert=[814, 757, 743, 688, 784, 746, 810, 770, 755, 766, 771, 778, 822, 726, 765, 750, 822, 777, 766, 797, 787, 731, 725, 782, 772, 778, 800, 761, 774, 733, 801, 761, 725, 766, 792, 813, 793, 741, 791, 733, 742, 758, 792, 756, 770, 752, 778, 727, 751, 755, 826, 746, 768, 846, 763, 771, 741, 770, 739, 754, 757, 755, 799, 770] payload_rows_per_dst=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] dup_rows_per_dst=[3248, 3172, 3230, 2842, 2996, 2674, 2972, 2896] combine_rows_per_src=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343744 combine_payload_bytes=266428416 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266772160 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533200576 weight_movement_bytes=69206016 total_nvlink_bytes=602406592 weight_to_token_ratio=0.129794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64806 dup_rows=24064 dup_rows_remote=21086 dedup_ratio=0.244792 router_tokens_per_expert=[721, 769, 789, 753, 734, 773, 763, 781, 816, 740, 802, 818, 776, 782, 721, 724, 803, 763, 761, 810, 715, 797, 780, 777, 739, 795, 815, 789, 741, 745, 820, 742, 738, 774, 746, 739, 775, 803, 769, 785, 779, 744, 779, 726, 733, 773, 774, 763, 740, 800, 751, 764, 783, 762, 715, 755, 772, 766, 767, 798, 769, 772, 806, 778] payload_rows_per_dst=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] dup_rows_per_dst=[2926, 2956, 2978, 3072, 2984, 3006, 2996, 3146] combine_rows_per_src=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=343848 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266960680 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533577512 weight_movement_bytes=69206016 total_nvlink_bytes=602783528 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64874 dup_rows=24118 dup_rows_remote=21050 dedup_ratio=0.245341 router_tokens_per_expert=[774, 820, 747, 769, 737, 721, 852, 720, 736, 774, 729, 746, 759, 744, 741, 793, 801, 789, 785, 799, 755, 732, 770, 778, 758, 774, 770, 796, 788, 783, 769, 752, 784, 742, 756, 768, 766, 768, 787, 711, 751, 774, 827, 751, 722, 789, 780, 720, 777, 733, 816, 770, 807, 779, 780, 816, 780, 741, 772, 721, 800, 755, 782, 766] payload_rows_per_dst=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] dup_rows_per_dst=[3030, 2880, 3088, 3068, 2966, 2938, 3156, 2992] combine_rows_per_src=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344312 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510584 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676856 weight_movement_bytes=34603008 total_nvlink_bytes=567279864 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=341888 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312128 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530282368 weight_movement_bytes=34603008 total_nvlink_bytes=564885376 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343568 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265788944 ingress_bytes=334651392 link_bytes=334651392 token_bytes=531234320 weight_movement_bytes=69206016 total_nvlink_bytes=600440336 weight_to_token_ratio=0.130274 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259266246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343696 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067600 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531791504 weight_movement_bytes=69206016 total_nvlink_bytes=600997520 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64952 dup_rows=23988 dup_rows_remote=20918 dedup_ratio=0.244019 router_tokens_per_expert=[720, 735, 786, 766, 755, 748, 781, 767, 797, 795, 747, 746, 760, 752, 806, 737, 799, 738, 778, 818, 768, 760, 761, 764, 772, 798, 748, 754, 779, 743, 806, 753, 734, 832, 761, 765, 768, 767, 749, 813, 737, 805, 792, 767, 762, 790, 749, 754, 775, 728, 813, 741, 717, 799, 789, 786, 737, 746, 750, 825, 793, 770, 735, 766] payload_rows_per_dst=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] dup_rows_per_dst=[2898, 3030, 2956, 2968, 3126, 3012, 2928, 3070] combine_rows_per_src=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64820 dup_rows=24150 dup_rows_remote=21104 dedup_ratio=0.245667 router_tokens_per_expert=[782, 770, 778, 803, 773, 799, 740, 727, 691, 804, 812, 763, 806, 778, 770, 707, 762, 740, 777, 747, 722, 749, 760, 738, 745, 785, 783, 817, 803, 720, 740, 786, 746, 767, 749, 794, 734, 761, 778, 808, 818, 760, 757, 821, 756, 742, 753, 783, 776, 768, 747, 783, 811, 756, 834, 783, 726, 770, 792, 739, 734, 745, 798, 786] payload_rows_per_dst=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] dup_rows_per_dst=[3094, 3058, 2926, 3010, 3004, 3046, 3046, 2966] combine_rows_per_src=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64882 dup_rows=24078 dup_rows_remote=20878 dedup_ratio=0.244934 router_tokens_per_expert=[795, 731, 752, 782, 783, 768, 786, 802, 800, 769, 772, 756, 779, 773, 775, 770, 787, 756, 724, 720, 766, 790, 777, 766, 741, 740, 802, 743, 761, 801, 731, 697, 835, 813, 747, 792, 768, 761, 734, 822, 772, 773, 759, 774, 760, 747, 808, 764, 750, 724, 777, 760, 789, 726, 755, 733, 787, 765, 797, 789, 734, 811, 774, 757] payload_rows_per_dst=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] dup_rows_per_dst=[3016, 3048, 2922, 2862, 3200, 3114, 2928, 2988] combine_rows_per_src=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64894 dup_rows=24056 dup_rows_remote=21188 dedup_ratio=0.244710 router_tokens_per_expert=[777, 777, 750, 792, 760, 757, 778, 732, 750, 723, 754, 772, 766, 806, 747, 778, 800, 746, 756, 722, 775, 756, 816, 794, 805, 776, 781, 746, 727, 775, 767, 745, 754, 787, 765, 776, 764, 765, 771, 770, 771, 766, 797, 824, 785, 742, 786, 743, 740, 758, 753, 760, 739, 813, 779, 769, 792, 785, 790, 716, 752, 765, 804, 765] payload_rows_per_dst=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] dup_rows_per_dst=[2998, 2972, 3082, 3000, 3056, 3022, 2868, 3058] combine_rows_per_src=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343480 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266386872 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430264 weight_movement_bytes=69206016 total_nvlink_bytes=601636280 weight_to_token_ratio=0.129981 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65090 dup_rows=23862 dup_rows_remote=20876 dedup_ratio=0.242737 router_tokens_per_expert=[778, 799, 793, 771, 742, 746, 771, 744, 764, 790, 752, 760, 789, 781, 766, 767, 797, 769, 771, 759, 774, 762, 778, 735, 754, 724, 768, 761, 801, 792, 764, 764, 730, 793, 780, 765, 823, 775, 772, 762, 741, 753, 790, 763, 779, 729, 767, 744, 808, 762, 760, 798, 767, 736, 798, 793, 804, 742, 751, 736, 750, 776, 774, 745] payload_rows_per_dst=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] dup_rows_per_dst=[2968, 2986, 2978, 2936, 2972, 2916, 3136, 2970] combine_rows_per_src=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64868 dup_rows=24090 dup_rows_remote=20986 dedup_ratio=0.245056 router_tokens_per_expert=[754, 788, 779, 755, 808, 786, 760, 794, 785, 752, 794, 764, 743, 786, 732, 784, 765, 704, 752, 753, 795, 760, 750, 759, 732, 761, 755, 756, 778, 736, 783, 769, 775, 766, 783, 786, 808, 804, 769, 770, 783, 763, 732, 780, 790, 782, 806, 806, 724, 759, 752, 745, 778, 783, 741, 781, 760, 787, 748, 765, 784, 736, 765, 769] payload_rows_per_dst=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] dup_rows_per_dst=[3104, 3170, 2884, 2836, 3138, 3030, 2914, 3014] combine_rows_per_src=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343040 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266099712 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856384 weight_movement_bytes=69206016 total_nvlink_bytes=601062400 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64894 dup_rows=24112 dup_rows_remote=21108 dedup_ratio=0.245280 router_tokens_per_expert=[828, 730, 769, 802, 749, 752, 774, 745, 764, 730, 781, 786, 745, 785, 749, 735, 751, 760, 780, 796, 779, 790, 777, 718, 774, 788, 774, 768, 786, 781, 726, 814, 731, 722, 777, 752, 775, 778, 792, 741, 747, 802, 788, 759, 764, 780, 787, 811, 752, 776, 796, 797, 765, 754, 761, 805, 741, 731, 781, 763, 738, 815, 736, 749] payload_rows_per_dst=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] dup_rows_per_dst=[3052, 2904, 3004, 3100, 2876, 3182, 3042, 2952] combine_rows_per_src=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64854 dup_rows=24054 dup_rows_remote=21150 dedup_ratio=0.244690 router_tokens_per_expert=[773, 756, 772, 745, 715, 760, 761, 785, 741, 755, 768, 796, 762, 786, 816, 738, 798, 750, 743, 775, 821, 724, 800, 770, 781, 798, 795, 772, 728, 764, 776, 736, 729, 818, 761, 826, 772, 703, 768, 754, 753, 724, 776, 746, 745, 783, 737, 708, 788, 773, 819, 767, 781, 733, 843, 769, 743, 785, 802, 812, 729, 783, 819, 743] payload_rows_per_dst=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] dup_rows_per_dst=[3000, 3002, 3090, 2904, 2964, 2894, 3114, 3086] combine_rows_per_src=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344328 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266150152 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955976 weight_movement_bytes=103809024 total_nvlink_bytes=635765000 weight_to_token_ratio=0.195146 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952504 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561144 weight_movement_bytes=34603008 total_nvlink_bytes=568164152 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343416 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042744 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742072 weight_movement_bytes=103809024 total_nvlink_bytes=635551096 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64492 dup_rows=24134 dup_rows_remote=20808 dedup_ratio=0.245504 router_tokens_per_expert=[765, 720, 761, 801, 764, 769, 751, 794, 773, 784, 717, 729, 760, 774, 771, 726, 766, 766, 786, 776, 763, 742, 773, 832, 777, 761, 781, 750, 771, 747, 797, 737, 743, 803, 756, 749, 811, 766, 780, 778, 776, 789, 732, 735, 757, 750, 804, 772, 743, 767, 759, 835, 764, 769, 782, 771, 770, 749, 779, 801, 794, 718, 798, 768] payload_rows_per_dst=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] dup_rows_per_dst=[2994, 2922, 2928, 2820, 2942, 3122, 3080, 3326] combine_rows_per_src=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344016 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986000 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531627984 weight_movement_bytes=34603008 total_nvlink_bytes=566230992 weight_to_token_ratio=0.065089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344008 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149832 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955656 weight_movement_bytes=103809024 total_nvlink_bytes=635764680 weight_to_token_ratio=0.195146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65058 dup_rows=23998 dup_rows_remote=21138 dedup_ratio=0.244120 router_tokens_per_expert=[765, 778, 788, 777, 778, 752, 748, 762, 771, 778, 780, 773, 801, 766, 747, 788, 781, 803, 779, 761, 786, 735, 756, 748, 809, 811, 775, 732, 749, 757, 825, 760, 742, 783, 731, 772, 828, 748, 730, 792, 768, 767, 743, 707, 742, 794, 779, 754, 771, 795, 746, 751, 792, 771, 764, 762, 736, 769, 751, 793, 800, 735, 747, 770] payload_rows_per_dst=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] dup_rows_per_dst=[2970, 3060, 3008, 3128, 2992, 2860, 3046, 2934] combine_rows_per_src=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264159232 dispatch_meta_bytes=341200 combine_payload_bytes=264159232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264500432 ingress_bytes=298762240 link_bytes=298762240 token_bytes=528659664 weight_movement_bytes=34603008 total_nvlink_bytes=563262672 weight_to_token_ratio=0.065454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124218257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65228 dup_rows=23798 dup_rows_remote=20808 dedup_ratio=0.242086 router_tokens_per_expert=[776, 765, 722, 784, 768, 801, 772, 783, 736, 729, 758, 773, 712, 796, 818, 777, 749, 763, 732, 745, 740, 752, 773, 815, 789, 788, 785, 801, 767, 751, 756, 765, 770, 812, 738, 797, 755, 767, 746, 749, 764, 696, 780, 828, 789, 766, 790, 746, 785, 759, 785, 772, 793, 752, 753, 729, 773, 755, 760, 806, 809, 735, 760, 792] payload_rows_per_dst=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] dup_rows_per_dst=[2996, 3004, 2922, 2982, 2990, 2916, 2964, 3024] combine_rows_per_src=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64956 dup_rows=23898 dup_rows_remote=20802 dedup_ratio=0.243103 router_tokens_per_expert=[766, 748, 763, 787, 733, 753, 742, 745, 718, 741, 744, 756, 789, 762, 798, 770, 758, 773, 788, 760, 784, 774, 750, 780, 749, 768, 790, 789, 742, 765, 779, 743, 758, 789, 736, 775, 779, 810, 765, 769, 791, 803, 770, 797, 755, 761, 723, 725, 735, 808, 791, 809, 790, 762, 771, 807, 731, 764, 773, 751, 792, 794, 797, 764] payload_rows_per_dst=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] dup_rows_per_dst=[2766, 2938, 3000, 3006, 3030, 3034, 3096, 3028] combine_rows_per_src=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65022 dup_rows=24054 dup_rows_remote=21070 dedup_ratio=0.244690 router_tokens_per_expert=[775, 745, 781, 815, 776, 805, 767, 759, 761, 820, 746, 746, 701, 765, 804, 763, 803, 780, 748, 763, 754, 742, 770, 802, 822, 765, 785, 782, 790, 753, 775, 756, 844, 777, 772, 739, 754, 808, 766, 777, 791, 727, 745, 743, 774, 792, 756, 733, 738, 778, 787, 741, 698, 804, 728, 769, 723, 756, 796, 781, 768, 759, 748, 761] payload_rows_per_dst=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] dup_rows_per_dst=[3070, 2984, 2956, 3114, 3074, 2938, 2976, 2942] combine_rows_per_src=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344784 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822352 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299920 weight_movement_bytes=34603008 total_nvlink_bytes=567902928 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64496 dup_rows=24296 dup_rows_remote=21130 dedup_ratio=0.247152 router_tokens_per_expert=[771, 736, 729, 820, 752, 734, 740, 813, 767, 754, 771, 764, 754, 800, 739, 771, 777, 741, 779, 735, 804, 783, 773, 782, 748, 777, 802, 749, 714, 782, 747, 751, 768, 779, 788, 803, 753, 806, 788, 738, 774, 783, 780, 764, 781, 762, 801, 772, 797, 766, 783, 838, 758, 749, 736, 793, 745, 730, 747, 770, 741, 744, 765, 791] payload_rows_per_dst=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] dup_rows_per_dst=[3166, 3034, 3002, 3152, 3020, 3136, 3080, 2706] combine_rows_per_src=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65258 dup_rows=23672 dup_rows_remote=20632 dedup_ratio=0.240804 router_tokens_per_expert=[768, 717, 740, 772, 762, 757, 780, 798, 782, 762, 788, 776, 795, 792, 776, 811, 783, 783, 776, 753, 748, 803, 782, 779, 779, 760, 795, 764, 771, 722, 741, 782, 746, 806, 739, 748, 769, 765, 769, 780, 814, 741, 792, 743, 734, 735, 776, 780, 761, 763, 763, 790, 706, 766, 725, 753, 734, 785, 779, 777, 781, 754, 810, 771] payload_rows_per_dst=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] dup_rows_per_dst=[2878, 3128, 3040, 2934, 2952, 2956, 2794, 2990] combine_rows_per_src=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64876 dup_rows=24052 dup_rows_remote=21126 dedup_ratio=0.244670 router_tokens_per_expert=[745, 757, 795, 824, 766, 762, 783, 805, 750, 769, 793, 753, 693, 774, 813, 748, 783, 780, 773, 740, 811, 768, 770, 798, 739, 753, 783, 766, 730, 762, 809, 809, 754, 732, 800, 834, 753, 772, 750, 756, 795, 748, 751, 768, 779, 835, 766, 757, 759, 743, 756, 762, 742, 758, 782, 753, 771, 729, 722, 788, 795, 712, 771, 755] payload_rows_per_dst=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] dup_rows_per_dst=[3008, 3020, 3120, 2926, 3050, 3066, 2920, 2942] combine_rows_per_src=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344144 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267518032 ingress_bytes=336379904 link_bytes=336379904 token_bytes=534691920 weight_movement_bytes=69206016 total_nvlink_bytes=603897936 weight_to_token_ratio=0.129432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265770498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343032 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266402808 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532462584 weight_movement_bytes=69206016 total_nvlink_bytes=601668600 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344368 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674480 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004592 weight_movement_bytes=34603008 total_nvlink_bytes=567607600 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264175616 dispatch_meta_bytes=342504 combine_payload_bytes=264175616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264518120 ingress_bytes=298778624 link_bytes=298778624 token_bytes=528693736 weight_movement_bytes=34603008 total_nvlink_bytes=563296744 weight_to_token_ratio=0.065450 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124279908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64672 dup_rows=24280 dup_rows_remote=21132 dedup_ratio=0.246989 router_tokens_per_expert=[762, 781, 745, 788, 792, 765, 731, 767, 720, 751, 747, 776, 769, 745, 746, 786, 747, 856, 792, 758, 748, 798, 783, 785, 727, 805, 755, 756, 742, 771, 775, 785, 740, 815, 741, 787, 766, 682, 777, 806, 797, 755, 759, 751, 735, 767, 716, 789, 803, 769, 745, 761, 776, 803, 783, 733, 808, 825, 754, 725, 815, 819, 744, 752] payload_rows_per_dst=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] dup_rows_per_dst=[2880, 2914, 3126, 2966, 3076, 3214, 2956, 3148] combine_rows_per_src=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=343560 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267640328 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937096 weight_movement_bytes=69206016 total_nvlink_bytes=604143112 weight_to_token_ratio=0.129372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344008 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266076104 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531808200 weight_movement_bytes=69206016 total_nvlink_bytes=601014216 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=65008 dup_rows=24082 dup_rows_remote=21184 dedup_ratio=0.244975 router_tokens_per_expert=[797, 767, 760, 788, 809, 783, 746, 763, 791, 741, 771, 742, 786, 777, 770, 719, 786, 834, 760, 795, 755, 762, 759, 771, 776, 750, 806, 752, 752, 783, 786, 729, 738, 758, 731, 734, 750, 736, 765, 729, 771, 724, 748, 778, 758, 759, 733, 785, 824, 731, 780, 741, 799, 804, 800, 782, 776, 826, 800, 748, 790, 759, 825, 704] payload_rows_per_dst=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] dup_rows_per_dst=[3080, 2998, 3032, 3072, 2798, 2898, 3146, 3058] combine_rows_per_src=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343216 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265239728 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530136240 weight_movement_bytes=103809024 total_nvlink_bytes=633945264 weight_to_token_ratio=0.195816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64898 dup_rows=24118 dup_rows_remote=21030 dedup_ratio=0.245341 router_tokens_per_expert=[749, 762, 791, 777, 798, 784, 753, 819, 780, 783, 788, 758, 728, 822, 753, 811, 791, 795, 769, 750, 791, 710, 758, 757, 757, 780, 781, 723, 740, 772, 790, 726, 737, 818, 829, 760, 732, 758, 775, 779, 740, 761, 747, 786, 788, 737, 753, 774, 792, 785, 777, 775, 734, 749, 796, 759, 714, 761, 800, 737, 833, 769, 724, 727] payload_rows_per_dst=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] dup_rows_per_dst=[3072, 3192, 2918, 2940, 3088, 2960, 3020, 2928] combine_rows_per_src=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64924 dup_rows=23924 dup_rows_remote=20896 dedup_ratio=0.243368 router_tokens_per_expert=[771, 727, 780, 766, 752, 756, 762, 749, 759, 783, 739, 806, 763, 793, 828, 771, 791, 797, 754, 759, 786, 760, 751, 752, 764, 791, 757, 744, 781, 764, 804, 719, 742, 739, 767, 815, 751, 750, 775, 777, 791, 785, 752, 772, 728, 773, 767, 780, 820, 722, 756, 761, 774, 783, 816, 798, 779, 758, 774, 749, 794, 747, 757, 721] payload_rows_per_dst=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] dup_rows_per_dst=[2986, 3028, 2926, 3024, 2986, 2978, 3086, 2910] combine_rows_per_src=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64974 dup_rows=24126 dup_rows_remote=21242 dedup_ratio=0.245422 router_tokens_per_expert=[785, 754, 766, 780, 729, 773, 751, 759, 776, 777, 746, 811, 737, 760, 766, 787, 733, 794, 770, 764, 755, 789, 738, 751, 762, 780, 788, 729, 752, 748, 814, 813, 746, 797, 786, 746, 730, 756, 785, 774, 773, 780, 777, 748, 805, 777, 753, 802, 778, 761, 740, 723, 760, 743, 772, 767, 800, 768, 808, 771, 790, 760, 770, 769] payload_rows_per_dst=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] dup_rows_per_dst=[2978, 3064, 2964, 2990, 2988, 3172, 2884, 3086] combine_rows_per_src=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344768 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617536 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532890304 weight_movement_bytes=34603008 total_nvlink_bytes=567493312 weight_to_token_ratio=0.064935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21024 dedup_ratio=0.244670 router_tokens_per_expert=[812, 804, 752, 767, 719, 770, 739, 800, 745, 770, 717, 762, 798, 800, 758, 762, 790, 790, 788, 758, 746, 758, 752, 826, 810, 823, 804, 751, 775, 700, 789, 798, 746, 763, 798, 763, 795, 763, 727, 780, 732, 778, 732, 779, 771, 773, 756, 771, 801, 796, 766, 769, 783, 767, 780, 712, 783, 762, 715, 735, 751, 779, 734, 759] payload_rows_per_dst=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] dup_rows_per_dst=[3028, 3304, 3048, 3098, 2954, 2986, 2726, 2908] combine_rows_per_src=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65040 dup_rows=23930 dup_rows_remote=20960 dedup_ratio=0.243429 router_tokens_per_expert=[782, 827, 764, 839, 775, 765, 769, 783, 784, 727, 751, 791, 767, 770, 719, 761, 773, 711, 727, 796, 781, 765, 812, 787, 724, 750, 802, 733, 743, 797, 798, 792, 749, 723, 752, 825, 738, 732, 775, 728, 786, 763, 738, 813, 758, 752, 768, 791, 749, 754, 777, 792, 762, 742, 794, 766, 772, 783, 776, 794, 759, 761, 732, 783] payload_rows_per_dst=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] dup_rows_per_dst=[3104, 2882, 2970, 3064, 2912, 3034, 2870, 3094] combine_rows_per_src=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65258 dup_rows=23782 dup_rows_remote=20840 dedup_ratio=0.241923 router_tokens_per_expert=[786, 759, 796, 806, 772, 739, 795, 786, 739, 761, 810, 729, 735, 726, 733, 750, 743, 757, 790, 769, 782, 748, 796, 800, 753, 722, 765, 767, 746, 766, 795, 789, 752, 757, 790, 788, 729, 767, 745, 752, 704, 779, 820, 803, 791, 784, 755, 797, 762, 773, 744, 797, 786, 740, 779, 796, 787, 809, 754, 772, 757, 751, 799, 723] payload_rows_per_dst=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] dup_rows_per_dst=[2978, 2896, 3012, 2942, 2856, 3120, 2982, 2996] combine_rows_per_src=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343280 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266271984 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532200688 weight_movement_bytes=69206016 total_nvlink_bytes=601406704 weight_to_token_ratio=0.130037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64802 dup_rows=23944 dup_rows_remote=20638 dedup_ratio=0.243571 router_tokens_per_expert=[742, 753, 758, 761, 739, 758, 784, 762, 737, 821, 800, 739, 803, 777, 747, 791, 770, 785, 753, 763, 719, 740, 771, 725, 763, 748, 732, 795, 766, 765, 720, 772, 725, 750, 792, 772, 748, 763, 775, 716, 724, 760, 771, 760, 785, 800, 770, 781, 763, 798, 785, 769, 798, 784, 799, 787, 774, 841, 801, 792, 777, 782, 795, 756] payload_rows_per_dst=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] dup_rows_per_dst=[2848, 2850, 2882, 2904, 3082, 2896, 3176, 3306] combine_rows_per_src=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343912 combine_payload_bytes=266051584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266395496 ingress_bytes=335257600 link_bytes=335257600 token_bytes=532447080 weight_movement_bytes=69206016 total_nvlink_bytes=601653096 weight_to_token_ratio=0.129977 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344000 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747840 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151680 weight_movement_bytes=34603008 total_nvlink_bytes=567754688 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344392 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267641160 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937928 weight_movement_bytes=69206016 total_nvlink_bytes=604143944 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65010 dup_rows=24032 dup_rows_remote=21054 dedup_ratio=0.244466 router_tokens_per_expert=[796, 769, 792, 731, 798, 747, 780, 806, 809, 791, 755, 731, 746, 735, 798, 743, 821, 782, 718, 781, 750, 814, 740, 778, 772, 792, 740, 756, 759, 762, 766, 746, 767, 833, 784, 783, 742, 760, 748, 779, 812, 747, 789, 703, 741, 761, 835, 732, 722, 713, 756, 788, 837, 784, 746, 811, 829, 743, 742, 746, 760, 766, 748, 741] payload_rows_per_dst=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] dup_rows_per_dst=[3040, 2978, 3034, 2936, 3090, 2978, 3008, 2968] combine_rows_per_src=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=341760 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265770752 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531199744 weight_movement_bytes=69206016 total_nvlink_bytes=600405760 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21050 dedup_ratio=0.244629 router_tokens_per_expert=[751, 737, 718, 816, 789, 727, 785, 748, 750, 788, 788, 730, 778, 779, 827, 789, 776, 748, 732, 767, 790, 788, 773, 753, 738, 727, 772, 790, 770, 750, 788, 733, 741, 782, 813, 801, 785, 738, 776, 737, 798, 758, 768, 756, 791, 749, 751, 806, 778, 789, 788, 757, 762, 768, 802, 740, 768, 760, 743, 765, 792, 754, 735, 806] payload_rows_per_dst=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] dup_rows_per_dst=[2940, 3076, 3040, 2950, 2998, 3050, 2970, 3024] combine_rows_per_src=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64892 dup_rows=24248 dup_rows_remote=21226 dedup_ratio=0.246663 router_tokens_per_expert=[797, 757, 830, 776, 751, 738, 729, 786, 745, 765, 771, 731, 777, 790, 762, 752, 785, 726, 750, 735, 758, 783, 772, 792, 749, 743, 766, 740, 777, 758, 751, 766, 756, 765, 783, 743, 739, 775, 760, 802, 803, 778, 730, 782, 807, 782, 760, 805, 770, 756, 790, 757, 778, 757, 832, 747, 769, 773, 743, 813, 793, 772, 740, 784] payload_rows_per_dst=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] dup_rows_per_dst=[2998, 3022, 3032, 2966, 3034, 3140, 3014, 3042] combine_rows_per_src=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65070 dup_rows=24036 dup_rows_remote=21140 dedup_ratio=0.244507 router_tokens_per_expert=[799, 748, 777, 776, 780, 788, 747, 766, 717, 753, 788, 766, 822, 751, 745, 792, 735, 758, 756, 753, 770, 706, 799, 761, 791, 723, 787, 738, 745, 810, 793, 757, 804, 746, 711, 756, 777, 785, 785, 755, 821, 788, 751, 769, 753, 787, 808, 783, 774, 790, 757, 748, 743, 749, 765, 721, 768, 820, 770, 755, 791, 811, 792, 722] payload_rows_per_dst=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] dup_rows_per_dst=[2950, 3026, 2964, 3036, 2964, 3120, 2896, 3080] combine_rows_per_src=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344256 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266625216 ingress_bytes=370089984 link_bytes=370089984 token_bytes=532906176 weight_movement_bytes=103809024 total_nvlink_bytes=636715200 weight_to_token_ratio=0.194798 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392618815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64994 dup_rows=24190 dup_rows_remote=21128 dedup_ratio=0.246073 router_tokens_per_expert=[722, 777, 745, 792, 779, 782, 792, 752, 749, 774, 769, 791, 806, 760, 745, 767, 753, 748, 719, 815, 777, 772, 742, 765, 761, 759, 807, 764, 806, 743, 724, 763, 758, 757, 757, 779, 732, 760, 735, 782, 794, 744, 734, 754, 775, 745, 767, 741, 783, 806, 778, 784, 785, 738, 794, 781, 804, 742, 789, 792, 802, 772, 797, 771] payload_rows_per_dst=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] dup_rows_per_dst=[3034, 3052, 3062, 2954, 2968, 2858, 3140, 3122] combine_rows_per_src=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64766 dup_rows=24118 dup_rows_remote=21080 dedup_ratio=0.245341 router_tokens_per_expert=[771, 765, 770, 779, 815, 749, 756, 772, 772, 774, 739, 720, 750, 710, 750, 704, 788, 741, 795, 774, 825, 722, 742, 791, 789, 796, 763, 776, 786, 794, 780, 784, 766, 800, 789, 793, 794, 798, 748, 739, 762, 758, 778, 745, 795, 770, 760, 765, 758, 746, 778, 776, 758, 779, 758, 773, 774, 747, 767, 750, 758, 767, 761, 800] payload_rows_per_dst=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] dup_rows_per_dst=[3038, 3152, 2850, 2964, 3122, 3088, 3086, 2818] combine_rows_per_src=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343832 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266207000 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070168 weight_movement_bytes=34603008 total_nvlink_bytes=566673176 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64910 dup_rows=24068 dup_rows_remote=21166 dedup_ratio=0.244832 router_tokens_per_expert=[767, 781, 786, 803, 772, 752, 731, 752, 777, 768, 799, 762, 734, 787, 712, 778, 808, 823, 740, 791, 781, 789, 756, 750, 747, 729, 757, 786, 779, 796, 738, 782, 803, 746, 745, 729, 734, 783, 769, 750, 729, 787, 808, 738, 770, 792, 791, 791, 738, 770, 761, 770, 743, 760, 818, 759, 768, 783, 790, 757, 792, 747, 726, 792] payload_rows_per_dst=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] dup_rows_per_dst=[2904, 2936, 3194, 2902, 3018, 3056, 2994, 3064] combine_rows_per_src=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266142104 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939736 weight_movement_bytes=103809024 total_nvlink_bytes=635748760 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344840 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266871560 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533398280 weight_movement_bytes=69206016 total_nvlink_bytes=602604296 weight_to_token_ratio=0.129745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344488 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559912 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775336 weight_movement_bytes=34603008 total_nvlink_bytes=567378344 weight_to_token_ratio=0.064949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=64964 dup_rows=23928 dup_rows_remote=20820 dedup_ratio=0.243408 router_tokens_per_expert=[748, 748, 789, 750, 757, 786, 768, 752, 789, 782, 813, 784, 766, 771, 748, 792, 801, 767, 781, 717, 817, 747, 711, 752, 771, 778, 813, 774, 729, 690, 752, 751, 757, 774, 795, 737, 735, 805, 789, 751, 777, 773, 736, 764, 735, 786, 745, 757, 771, 719, 783, 808, 792, 769, 800, 773, 800, 737, 801, 745, 815, 807, 777, 745] payload_rows_per_dst=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] dup_rows_per_dst=[2912, 2952, 2988, 2910, 2964, 3060, 3034, 3108] combine_rows_per_src=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343384 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265624920 ingress_bytes=369090560 link_bytes=369090560 token_bytes=530906456 weight_movement_bytes=103809024 total_nvlink_bytes=634715480 weight_to_token_ratio=0.195532 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388858063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344304 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266215664 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532087024 weight_movement_bytes=34603008 total_nvlink_bytes=566690032 weight_to_token_ratio=0.065033 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65012 dup_rows=24026 dup_rows_remote=21082 dedup_ratio=0.244405 router_tokens_per_expert=[749, 708, 734, 812, 796, 756, 781, 757, 841, 803, 781, 719, 792, 787, 757, 750, 772, 786, 773, 742, 763, 756, 758, 752, 786, 749, 728, 767, 769, 778, 748, 754, 747, 835, 810, 745, 785, 787, 786, 742, 809, 713, 764, 724, 770, 764, 815, 746, 759, 821, 752, 782, 793, 808, 769, 770, 741, 774, 763, 797, 758, 754, 727, 738] payload_rows_per_dst=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] dup_rows_per_dst=[2944, 3064, 2984, 2996, 3106, 2944, 3090, 2898] combine_rows_per_src=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343136 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266435680 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528224 weight_movement_bytes=103809024 total_nvlink_bytes=636337248 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21112 dedup_ratio=0.245911 router_tokens_per_expert=[780, 761, 780, 761, 786, 801, 764, 789, 710, 766, 771, 774, 766, 762, 754, 824, 774, 768, 780, 703, 770, 804, 759, 797, 775, 788, 745, 756, 709, 777, 816, 676, 780, 763, 728, 776, 801, 757, 797, 788, 793, 775, 749, 771, 768, 729, 736, 805, 759, 763, 798, 751, 759, 746, 769, 779, 788, 757, 768, 765, 721, 766, 786, 815] payload_rows_per_dst=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] dup_rows_per_dst=[3096, 2984, 3026, 2946, 3062, 3008, 3068, 2984] combine_rows_per_src=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64786 dup_rows=24132 dup_rows_remote=21106 dedup_ratio=0.245483 router_tokens_per_expert=[731, 763, 787, 731, 814, 756, 758, 790, 760, 824, 751, 751, 785, 803, 721, 811, 795, 771, 749, 830, 741, 790, 743, 769, 821, 788, 779, 778, 815, 822, 722, 811, 850, 741, 769, 724, 742, 757, 787, 711, 750, 783, 727, 845, 761, 756, 757, 765, 745, 778, 723, 763, 729, 738, 718, 723, 772, 751, 763, 792, 773, 744, 730, 825] payload_rows_per_dst=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] dup_rows_per_dst=[2962, 3026, 3184, 3246, 2998, 2970, 2772, 2974] combine_rows_per_src=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64968 dup_rows=24098 dup_rows_remote=21068 dedup_ratio=0.245138 router_tokens_per_expert=[761, 769, 753, 742, 806, 773, 731, 774, 774, 762, 758, 829, 744, 722, 743, 759, 775, 744, 786, 760, 782, 797, 739, 774, 785, 747, 749, 784, 761, 752, 783, 793, 753, 813, 750, 774, 735, 762, 750, 755, 744, 777, 796, 758, 799, 764, 745, 798, 767, 762, 808, 756, 768, 757, 790, 726, 786, 761, 811, 758, 779, 779, 785, 775] payload_rows_per_dst=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] dup_rows_per_dst=[2918, 3114, 3000, 2944, 2972, 3080, 3030, 3040] combine_rows_per_src=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344376 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633528 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922680 weight_movement_bytes=34603008 total_nvlink_bytes=567525688 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64884 dup_rows=24280 dup_rows_remote=21304 dedup_ratio=0.246989 router_tokens_per_expert=[744, 759, 766, 804, 753, 775, 799, 709, 787, 847, 753, 739, 816, 780, 763, 769, 727, 733, 734, 773, 783, 793, 782, 733, 765, 757, 796, 809, 755, 762, 772, 798, 757, 748, 778, 760, 801, 800, 752, 752, 745, 750, 766, 772, 808, 784, 775, 808, 785, 703, 750, 755, 766, 780, 788, 727, 776, 746, 724, 789, 752, 790, 803, 727] payload_rows_per_dst=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] dup_rows_per_dst=[2978, 3106, 2976, 3164, 3032, 3100, 2908, 3016] combine_rows_per_src=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343696 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265813648 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531283600 weight_movement_bytes=34603008 total_nvlink_bytes=565886608 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64790 dup_rows=24276 dup_rows_remote=21202 dedup_ratio=0.246948 router_tokens_per_expert=[780, 753, 723, 785, 808, 771, 799, 737, 794, 739, 793, 760, 764, 769, 795, 771, 748, 762, 742, 747, 780, 749, 684, 750, 738, 764, 781, 743, 724, 747, 759, 774, 755, 762, 774, 803, 768, 784, 787, 812, 786, 714, 725, 806, 749, 723, 759, 802, 716, 779, 787, 783, 780, 767, 797, 823, 775, 812, 823, 768, 773, 801, 779, 747] payload_rows_per_dst=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] dup_rows_per_dst=[3074, 3090, 3130, 3244, 3086, 2964, 2892, 2796] combine_rows_per_src=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65002 dup_rows=24062 dup_rows_remote=21152 dedup_ratio=0.244771 router_tokens_per_expert=[752, 759, 753, 805, 785, 783, 787, 748, 769, 779, 797, 762, 808, 819, 754, 765, 778, 734, 774, 776, 807, 805, 794, 816, 746, 738, 712, 753, 782, 760, 765, 819, 752, 746, 763, 801, 753, 787, 769, 752, 809, 714, 764, 742, 780, 739, 763, 772, 791, 765, 743, 759, 787, 778, 742, 793, 777, 770, 762, 752, 767, 727, 759, 690] payload_rows_per_dst=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] dup_rows_per_dst=[3088, 3088, 3122, 2910, 3018, 2902, 3068, 2866] combine_rows_per_src=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343568 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707024 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070480 weight_movement_bytes=34603008 total_nvlink_bytes=565673488 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344144 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453072 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562000 weight_movement_bytes=69206016 total_nvlink_bytes=601768016 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109616 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874480 weight_movement_bytes=69206016 total_nvlink_bytes=601080496 weight_to_token_ratio=0.130117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65122 dup_rows=23974 dup_rows_remote=20814 dedup_ratio=0.243876 router_tokens_per_expert=[787, 778, 745, 807, 765, 756, 746, 753, 799, 721, 744, 752, 743, 709, 779, 712, 756, 800, 776, 791, 798, 851, 761, 808, 748, 781, 771, 813, 806, 721, 779, 771, 797, 803, 773, 765, 745, 785, 746, 768, 747, 769, 763, 773, 744, 755, 773, 773, 783, 790, 770, 744, 781, 728, 756, 756, 794, 749, 813, 750, 764, 758, 747, 763] payload_rows_per_dst=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] dup_rows_per_dst=[2976, 2740, 2970, 2980, 3044, 3008, 3096, 3160] combine_rows_per_src=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343968 combine_payload_bytes=265379840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265723808 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103648 weight_movement_bytes=69206016 total_nvlink_bytes=600309664 weight_to_token_ratio=0.130306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344616 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592808 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841000 weight_movement_bytes=34603008 total_nvlink_bytes=567444008 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64794 dup_rows=24200 dup_rows_remote=21210 dedup_ratio=0.246175 router_tokens_per_expert=[739, 779, 776, 791, 781, 732, 794, 796, 774, 765, 740, 803, 772, 757, 754, 763, 724, 781, 769, 789, 763, 768, 791, 754, 757, 730, 793, 752, 747, 784, 753, 790, 791, 752, 775, 775, 750, 729, 774, 767, 759, 751, 789, 758, 738, 814, 777, 764, 778, 754, 756, 802, 772, 790, 784, 739, 712, 793, 737, 779, 794, 806, 746, 786] payload_rows_per_dst=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] dup_rows_per_dst=[3060, 2994, 3010, 2954, 2976, 2990, 3056, 3160] combine_rows_per_src=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343744 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083456 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823168 weight_movement_bytes=34603008 total_nvlink_bytes=568426176 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64996 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[751, 746, 750, 774, 806, 752, 748, 736, 738, 754, 762, 791, 731, 769, 758, 778, 761, 773, 774, 809, 782, 746, 768, 734, 756, 798, 772, 783, 755, 832, 746, 797, 757, 767, 750, 766, 776, 760, 777, 751, 778, 788, 732, 753, 752, 772, 798, 805, 771, 780, 791, 765, 744, 809, 798, 780, 792, 729, 739, 740, 784, 736, 819, 763] payload_rows_per_dst=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] dup_rows_per_dst=[2926, 3024, 3016, 3098, 2972, 3034, 3042, 2960] combine_rows_per_src=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64994 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[771, 829, 741, 739, 750, 807, 781, 740, 755, 799, 776, 793, 745, 778, 777, 750, 775, 773, 721, 773, 736, 773, 744, 772, 760, 773, 760, 798, 765, 777, 770, 802, 753, 766, 768, 797, 747, 770, 777, 766, 783, 773, 810, 739, 771, 826, 705, 749, 761, 725, 749, 787, 774, 768, 758, 752, 778, 762, 788, 766, 785, 811, 762, 723] payload_rows_per_dst=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] dup_rows_per_dst=[3014, 2972, 2966, 3074, 2944, 2982, 2884, 3100] combine_rows_per_src=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65056 dup_rows=23884 dup_rows_remote=20960 dedup_ratio=0.242961 router_tokens_per_expert=[823, 778, 722, 796, 758, 767, 754, 769, 742, 778, 746, 798, 734, 782, 747, 802, 772, 764, 757, 761, 754, 763, 744, 784, 753, 730, 792, 793, 763, 714, 747, 740, 766, 768, 801, 770, 807, 804, 793, 767, 761, 766, 770, 775, 753, 766, 748, 783, 755, 777, 779, 784, 773, 769, 786, 721, 773, 786, 768, 783, 787, 739, 769, 778] payload_rows_per_dst=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] dup_rows_per_dst=[2942, 3020, 2966, 2950, 3126, 2988, 2924, 2968] combine_rows_per_src=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=65032 dup_rows=24156 dup_rows_remote=21184 dedup_ratio=0.245728 router_tokens_per_expert=[788, 787, 739, 739, 801, 783, 781, 778, 772, 761, 735, 757, 790, 764, 756, 776, 777, 772, 782, 760, 747, 697, 766, 743, 789, 797, 749, 751, 762, 745, 736, 811, 774, 778, 716, 728, 786, 802, 762, 738, 770, 779, 772, 814, 735, 813, 811, 752, 753, 735, 778, 788, 748, 759, 758, 785, 766, 763, 789, 725, 792, 795, 803, 794] payload_rows_per_dst=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] dup_rows_per_dst=[3018, 2990, 2972, 3036, 2942, 3104, 2962, 3132] combine_rows_per_src=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=344016 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265740240 ingress_bytes=299999232 link_bytes=299999232 token_bytes=531136464 weight_movement_bytes=34603008 total_nvlink_bytes=565739472 weight_to_token_ratio=0.065149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128872958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344384 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266568000 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791616 weight_movement_bytes=103809024 total_nvlink_bytes=636600640 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64968 dup_rows=23940 dup_rows_remote=20964 dedup_ratio=0.243530 router_tokens_per_expert=[764, 758, 741, 810, 790, 797, 756, 768, 768, 731, 743, 826, 758, 753, 750, 763, 747, 796, 755, 742, 812, 766, 780, 756, 722, 758, 764, 754, 754, 742, 807, 803, 716, 747, 715, 755, 724, 779, 736, 736, 766, 738, 751, 806, 753, 832, 775, 739, 758, 790, 841, 807, 746, 765, 745, 764, 769, 764, 802, 776, 795, 763, 839, 826] payload_rows_per_dst=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] dup_rows_per_dst=[2976, 2952, 2936, 3354, 3068, 3096, 2772, 2786] combine_rows_per_src=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64940 dup_rows=24174 dup_rows_remote=21324 dedup_ratio=0.245911 router_tokens_per_expert=[771, 789, 776, 776, 795, 754, 734, 781, 768, 738, 787, 748, 768, 766, 773, 837, 750, 767, 732, 774, 786, 744, 833, 825, 786, 742, 796, 761, 788, 763, 720, 664, 758, 801, 773, 807, 775, 774, 741, 761, 766, 783, 758, 739, 751, 733, 739, 772, 771, 757, 760, 782, 778, 758, 775, 833, 794, 785, 750, 727, 735, 779, 772, 773] payload_rows_per_dst=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] dup_rows_per_dst=[3102, 3018, 3112, 2850, 3070, 2888, 3066, 3068] combine_rows_per_src=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343832 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559256 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532774680 weight_movement_bytes=34603008 total_nvlink_bytes=567377688 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344864 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715936 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533087008 weight_movement_bytes=34603008 total_nvlink_bytes=567690016 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64934 dup_rows=24146 dup_rows_remote=21118 dedup_ratio=0.245626 router_tokens_per_expert=[763, 790, 817, 773, 757, 789, 789, 744, 739, 737, 772, 748, 804, 756, 771, 737, 794, 722, 764, 744, 788, 755, 759, 769, 789, 745, 757, 758, 780, 756, 782, 717, 755, 750, 784, 776, 780, 770, 732, 774, 792, 799, 818, 786, 786, 731, 774, 751, 739, 779, 771, 776, 816, 776, 800, 747, 725, 768, 782, 771, 772, 764, 789, 754] payload_rows_per_dst=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] dup_rows_per_dst=[2850, 3082, 2986, 2918, 3036, 3100, 3146, 3028] combine_rows_per_src=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344064 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813440 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282816 weight_movement_bytes=34603008 total_nvlink_bytes=567885824 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343728 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452656 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561584 weight_movement_bytes=69206016 total_nvlink_bytes=601767600 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=345056 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266339296 ingress_bytes=404406272 link_bytes=404406272 token_bytes=532333536 weight_movement_bytes=138412032 total_nvlink_bytes=670745568 weight_to_token_ratio=0.260010 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65000 dup_rows=23970 dup_rows_remote=20854 dedup_ratio=0.243835 router_tokens_per_expert=[799, 743, 771, 740, 794, 757, 770, 796, 764, 751, 756, 732, 712, 789, 743, 789, 780, 776, 789, 774, 761, 749, 743, 730, 726, 790, 757, 787, 822, 755, 757, 780, 772, 773, 722, 760, 754, 767, 823, 775, 772, 778, 782, 763, 766, 799, 780, 785, 763, 791, 737, 768, 796, 798, 744, 735, 775, 763, 709, 765, 755, 845, 747, 808] payload_rows_per_dst=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] dup_rows_per_dst=[2934, 2904, 2934, 2984, 3006, 3116, 3044, 3048] combine_rows_per_src=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344208 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313872 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283536 weight_movement_bytes=103809024 total_nvlink_bytes=636092560 weight_to_token_ratio=0.195026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65032 dup_rows=24064 dup_rows_remote=21114 dedup_ratio=0.244792 router_tokens_per_expert=[773, 735, 758, 751, 765, 778, 756, 762, 743, 774, 798, 770, 734, 765, 768, 762, 757, 776, 783, 751, 739, 792, 712, 771, 732, 738, 785, 818, 793, 782, 803, 783, 765, 776, 738, 774, 736, 757, 769, 764, 763, 763, 777, 777, 770, 759, 768, 761, 769, 754, 755, 782, 799, 728, 747, 766, 796, 790, 802, 771, 792, 795, 765, 817] payload_rows_per_dst=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] dup_rows_per_dst=[2926, 2970, 2958, 3164, 2950, 3002, 2986, 3108] combine_rows_per_src=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343416 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266583416 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823416 weight_movement_bytes=69206016 total_nvlink_bytes=602029432 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74466 payload_rows_remote=65232 dup_rows=23838 dup_rows_remote=20874 dedup_ratio=0.242493 router_tokens_per_expert=[763, 815, 789, 793, 741, 786, 757, 807, 778, 762, 785, 811, 751, 735, 729, 748, 788, 765, 801, 769, 803, 815, 780, 746, 729, 714, 768, 745, 712, 742, 743, 740, 788, 758, 770, 823, 749, 772, 734, 714, 769, 800, 791, 809, 734, 757, 748, 722, 780, 771, 761, 777, 761, 789, 719, 747, 804, 840, 806, 751, 783, 807, 731, 777] payload_rows_per_dst=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] dup_rows_per_dst=[3158, 2964, 3018, 2726, 2890, 2978, 2968, 3136] combine_rows_per_src=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65234 dup_rows=23766 dup_rows_remote=20878 dedup_ratio=0.241760 router_tokens_per_expert=[813, 770, 760, 764, 741, 794, 778, 760, 746, 767, 760, 762, 797, 793, 772, 808, 759, 776, 721, 789, 790, 735, 751, 775, 774, 734, 766, 819, 745, 793, 804, 702, 762, 726, 821, 744, 775, 762, 790, 726, 822, 762, 775, 744, 736, 737, 777, 751, 788, 759, 766, 739, 773, 761, 745, 762, 816, 776, 809, 739, 797, 753, 744, 797] payload_rows_per_dst=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] dup_rows_per_dst=[3016, 3014, 2888, 2928, 2898, 2970, 2956, 3096] combine_rows_per_src=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64756 dup_rows=24214 dup_rows_remote=21190 dedup_ratio=0.246318 router_tokens_per_expert=[715, 731, 745, 725, 754, 782, 790, 780, 765, 774, 742, 797, 745, 822, 742, 791, 794, 720, 798, 791, 774, 732, 735, 775, 762, 766, 759, 742, 776, 786, 768, 782, 785, 787, 779, 745, 851, 733, 783, 739, 763, 769, 782, 773, 791, 773, 738, 742, 764, 791, 787, 739, 799, 790, 761, 748, 784, 827, 773, 725, 720, 778, 806, 767] payload_rows_per_dst=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] dup_rows_per_dst=[2948, 3010, 2986, 2980, 3110, 2958, 3024, 3198] combine_rows_per_src=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344584 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715656 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086728 weight_movement_bytes=34603008 total_nvlink_bytes=567689736 weight_to_token_ratio=0.064911 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64986 dup_rows=23964 dup_rows_remote=20954 dedup_ratio=0.243774 router_tokens_per_expert=[738, 745, 728, 738, 777, 730, 766, 798, 732, 749, 745, 786, 745, 792, 766, 785, 727, 796, 773, 784, 780, 715, 738, 792, 802, 776, 754, 785, 732, 805, 795, 792, 790, 719, 796, 755, 780, 730, 804, 811, 801, 756, 749, 791, 797, 751, 760, 764, 784, 829, 779, 750, 769, 742, 756, 765, 737, 805, 771, 763, 799, 747, 745, 791] payload_rows_per_dst=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] dup_rows_per_dst=[3010, 3088, 2968, 2998, 2928, 2962, 2980, 3030] combine_rows_per_src=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64910 dup_rows=24222 dup_rows_remote=21022 dedup_ratio=0.246399 router_tokens_per_expert=[728, 780, 776, 821, 774, 764, 786, 767, 742, 770, 736, 751, 795, 782, 824, 760, 740, 806, 742, 747, 747, 790, 746, 759, 800, 744, 772, 780, 775, 757, 800, 758, 802, 773, 793, 746, 728, 745, 737, 774, 758, 777, 747, 761, 785, 741, 773, 756, 757, 782, 797, 753, 772, 763, 761, 812, 791, 800, 734, 807, 739, 718, 765, 786] payload_rows_per_dst=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] dup_rows_per_dst=[3118, 3000, 2912, 3200, 2940, 3050, 2976, 3026] combine_rows_per_src=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344424 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267534696 ingress_bytes=336396288 link_bytes=336396288 token_bytes=534724968 weight_movement_bytes=69206016 total_nvlink_bytes=603930984 weight_to_token_ratio=0.129424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265832150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344448 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267542912 ingress_bytes=371007488 link_bytes=371007488 token_bytes=534741376 weight_movement_bytes=103809024 total_nvlink_bytes=638550400 weight_to_token_ratio=0.194129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396071309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64804 dup_rows=24212 dup_rows_remote=21280 dedup_ratio=0.246297 router_tokens_per_expert=[781, 782, 775, 774, 746, 789, 791, 727, 746, 775, 755, 796, 686, 790, 780, 767, 803, 796, 752, 781, 777, 782, 768, 766, 768, 767, 738, 740, 827, 752, 779, 725, 778, 779, 770, 767, 753, 753, 811, 758, 813, 795, 766, 772, 738, 782, 735, 772, 758, 777, 748, 801, 736, 739, 767, 793, 757, 725, 761, 751, 754, 798, 767, 797] payload_rows_per_dst=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] dup_rows_per_dst=[2400, 2994, 3122, 3636, 3014, 3180, 2934, 2932] combine_rows_per_src=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64828 dup_rows=24218 dup_rows_remote=21074 dedup_ratio=0.246358 router_tokens_per_expert=[818, 767, 786, 774, 777, 746, 809, 752, 797, 775, 758, 730, 783, 743, 789, 763, 775, 784, 759, 777, 749, 780, 760, 737, 723, 734, 821, 748, 743, 772, 785, 755, 811, 796, 781, 790, 745, 758, 734, 780, 777, 771, 754, 818, 768, 778, 784, 751, 758, 756, 766, 787, 750, 797, 743, 800, 741, 743, 744, 777, 755, 743, 758, 769] payload_rows_per_dst=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] dup_rows_per_dst=[3012, 3018, 3006, 3062, 3028, 3144, 3030, 2918] combine_rows_per_src=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526416 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709072 weight_movement_bytes=34603008 total_nvlink_bytes=567312080 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=343728 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266215088 ingress_bytes=369680384 link_bytes=369680384 token_bytes=532086448 weight_movement_bytes=103809024 total_nvlink_bytes=635895472 weight_to_token_ratio=0.195098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391077523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=344336 combine_payload_bytes=265437184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265781520 ingress_bytes=334643200 link_bytes=334643200 token_bytes=531218704 weight_movement_bytes=69206016 total_nvlink_bytes=600424720 weight_to_token_ratio=0.130278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259235421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64992 dup_rows=23982 dup_rows_remote=20990 dedup_ratio=0.243958 router_tokens_per_expert=[758, 786, 829, 740, 740, 751, 751, 782, 782, 757, 747, 765, 765, 785, 717, 793, 731, 772, 780, 815, 757, 758, 759, 767, 764, 745, 802, 741, 730, 779, 800, 751, 774, 780, 832, 774, 749, 760, 738, 754, 772, 777, 707, 802, 791, 779, 738, 772, 748, 817, 794, 784, 726, 819, 783, 766, 763, 755, 743, 791, 765, 770, 772, 758] payload_rows_per_dst=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] dup_rows_per_dst=[3018, 3010, 2958, 2928, 2992, 3038, 3050, 2988] combine_rows_per_src=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65148 dup_rows=23842 dup_rows_remote=20874 dedup_ratio=0.242533 router_tokens_per_expert=[767, 741, 747, 789, 750, 800, 723, 762, 751, 753, 754, 742, 782, 739, 786, 834, 759, 733, 745, 736, 760, 779, 812, 748, 765, 752, 746, 787, 798, 775, 797, 754, 724, 715, 750, 820, 781, 749, 807, 800, 804, 732, 775, 767, 767, 752, 783, 740, 776, 777, 763, 759, 780, 786, 778, 736, 774, 803, 746, 767, 809, 813, 768, 785] payload_rows_per_dst=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] dup_rows_per_dst=[3004, 2968, 2922, 2972, 2970, 2962, 2992, 3052] combine_rows_per_src=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343608 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265879096 ingress_bytes=334741504 link_bytes=334741504 token_bytes=531414584 weight_movement_bytes=69206016 total_nvlink_bytes=600620600 weight_to_token_ratio=0.130230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259605331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[776, 795, 782, 765, 788, 806, 707, 774, 786, 750, 791, 764, 792, 719, 735, 741, 780, 769, 800, 764, 738, 754, 739, 754, 749, 772, 747, 746, 763, 775, 801, 796, 754, 841, 764, 762, 728, 768, 736, 800, 786, 788, 757, 783, 769, 759, 758, 798, 794, 737, 779, 748, 766, 759, 735, 785, 767, 791, 744, 788, 770, 796, 746, 778] payload_rows_per_dst=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] dup_rows_per_dst=[3024, 2966, 2974, 2906, 2966, 3086, 3040, 3002] combine_rows_per_src=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65148 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[751, 780, 770, 775, 785, 775, 764, 768, 828, 729, 758, 788, 728, 809, 784, 766, 739, 786, 805, 764, 771, 777, 800, 731, 802, 776, 795, 772, 775, 737, 735, 719, 783, 727, 772, 740, 760, 723, 790, 797, 769, 759, 700, 777, 771, 784, 784, 747, 784, 748, 790, 717, 766, 740, 786, 776, 751, 809, 780, 819, 771, 770, 745, 775] payload_rows_per_dst=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] dup_rows_per_dst=[3038, 3016, 3064, 2924, 2948, 2926, 2884, 3026] combine_rows_per_src=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343928 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551160 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758392 weight_movement_bytes=34603008 total_nvlink_bytes=567361400 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344088 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190296 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036504 weight_movement_bytes=34603008 total_nvlink_bytes=568639512 weight_to_token_ratio=0.064795 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64858 dup_rows=24020 dup_rows_remote=20770 dedup_ratio=0.244344 router_tokens_per_expert=[774, 816, 819, 785, 779, 775, 748, 808, 788, 747, 721, 716, 822, 803, 730, 761, 744, 808, 793, 807, 741, 797, 765, 758, 735, 778, 780, 737, 770, 756, 779, 763, 753, 760, 782, 757, 792, 745, 749, 781, 760, 780, 800, 755, 745, 727, 745, 769, 768, 738, 768, 755, 703, 785, 784, 778, 769, 776, 767, 777, 775, 799, 745, 762] payload_rows_per_dst=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] dup_rows_per_dst=[3250, 2940, 3448, 2864, 3206, 2832, 2896, 2584] combine_rows_per_src=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65128 dup_rows=23994 dup_rows_remote=21062 dedup_ratio=0.244080 router_tokens_per_expert=[768, 787, 745, 766, 818, 763, 731, 756, 758, 796, 756, 755, 807, 750, 800, 805, 745, 763, 786, 781, 799, 757, 755, 786, 764, 756, 758, 826, 789, 748, 690, 726, 786, 769, 746, 758, 777, 801, 748, 721, 760, 786, 776, 749, 762, 813, 812, 753, 797, 808, 776, 713, 741, 802, 777, 747, 763, 754, 761, 784, 742, 769, 747, 764] payload_rows_per_dst=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] dup_rows_per_dst=[2966, 3102, 3012, 2932, 2978, 3042, 3046, 2916] combine_rows_per_src=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344432 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067760 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791088 weight_movement_bytes=34603008 total_nvlink_bytes=568394096 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344360 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190568 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036776 weight_movement_bytes=34603008 total_nvlink_bytes=568639784 weight_to_token_ratio=0.064795 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65002 dup_rows=24044 dup_rows_remote=20992 dedup_ratio=0.244588 router_tokens_per_expert=[830, 758, 813, 757, 803, 782, 774, 769, 770, 758, 745, 768, 785, 775, 787, 785, 799, 735, 725, 779, 754, 767, 773, 756, 743, 783, 757, 785, 759, 719, 763, 728, 804, 772, 790, 774, 745, 766, 765, 778, 737, 760, 784, 759, 783, 738, 735, 787, 807, 748, 728, 813, 776, 758, 754, 774, 775, 743, 788, 731, 781, 760, 750, 805] payload_rows_per_dst=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] dup_rows_per_dst=[2916, 2976, 2944, 3004, 3052, 2876, 3224, 3052] combine_rows_per_src=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259359688 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133636040 ingress_bytes=7371431936 link_bytes=7371431936 total_nvlink_bytes=13505067976 weight_to_token_ratio=0.101613 prefetch_share_of_ingress=0.103272 dedup_ratio=0.243881 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206004562e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65370 dup_rows=23596 dup_rows_remote=20616 dedup_ratio=0.240031 router_tokens_per_expert=[785, 799, 771, 796, 761, 765, 745, 799, 810, 804, 821, 719, 787, 774, 737, 779, 777, 769, 768, 772, 722, 756, 760, 758, 734, 759, 799, 802, 734, 744, 781, 749, 751, 736, 788, 787, 744, 778, 809, 785, 838, 741, 765, 763, 741, 796, 791, 724, 769, 728, 720, 784, 758, 756, 757, 762, 752, 794, 790, 768, 788, 760, 736, 757] payload_rows_per_dst=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] dup_rows_per_dst=[3022, 3008, 2978, 2916, 2918, 2980, 2812, 2962] combine_rows_per_src=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261245704 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6134579976 ingress_bytes=7614595072 link_bytes=7614595072 total_nvlink_bytes=13749175048 weight_to_token_ratio=0.121352 prefetch_share_of_ingress=0.090886 dedup_ratio=0.244073 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245787314e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=342512 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266000880 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531659248 weight_movement_bytes=69206016 total_nvlink_bytes=600865264 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344760 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109048 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873336 weight_movement_bytes=34603008 total_nvlink_bytes=568476344 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343976 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266592168 ingress_bytes=335454208 link_bytes=335454208 token_bytes=532840360 weight_movement_bytes=69206016 total_nvlink_bytes=602046376 weight_to_token_ratio=0.129881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64868 dup_rows=23918 dup_rows_remote=20798 dedup_ratio=0.243306 router_tokens_per_expert=[756, 761, 777, 763, 754, 787, 777, 766, 760, 769, 743, 734, 763, 739, 771, 733, 756, 779, 770, 730, 741, 783, 770, 808, 754, 768, 789, 762, 791, 778, 787, 726, 799, 796, 767, 822, 775, 814, 768, 778, 731, 754, 767, 743, 756, 799, 768, 820, 740, 799, 759, 773, 786, 766, 757, 795, 786, 809, 750, 745, 767, 735, 749, 734] payload_rows_per_dst=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] dup_rows_per_dst=[3028, 2902, 3008, 3050, 3120, 3056, 2968, 2786] combine_rows_per_src=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267755520 dispatch_meta_bytes=343944 combine_payload_bytes=267755520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268099464 ingress_bytes=336961536 link_bytes=336961536 token_bytes=535854984 weight_movement_bytes=69206016 total_nvlink_bytes=605061000 weight_to_token_ratio=0.129151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267959132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=21024 dedup_ratio=0.244832 router_tokens_per_expert=[785, 733, 746, 773, 757, 771, 746, 762, 767, 751, 785, 785, 795, 749, 771, 735, 784, 797, 736, 770, 777, 746, 791, 780, 732, 845, 770, 804, 755, 738, 811, 756, 815, 760, 709, 756, 744, 758, 788, 715, 723, 756, 832, 751, 762, 793, 740, 808, 769, 783, 766, 778, 765, 772, 739, 787, 794, 752, 741, 741, 784, 763, 790, 815] payload_rows_per_dst=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] dup_rows_per_dst=[2906, 3010, 3086, 3066, 2894, 3008, 3044, 3054] combine_rows_per_src=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12222339232 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6115113120 ingress_bytes=7387537408 link_bytes=7387537408 total_nvlink_bytes=13502650528 weight_to_token_ratio=0.104752 prefetch_share_of_ingress=0.103047 dedup_ratio=0.244646 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.208639501e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342664 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266041992 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531741320 weight_movement_bytes=103809024 total_nvlink_bytes=635550344 weight_to_token_ratio=0.195225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254462064 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6131187824 ingress_bytes=7576600576 link_bytes=7576600576 total_nvlink_bytes=13707788400 weight_to_token_ratio=0.118596 prefetch_share_of_ingress=0.082208 dedup_ratio=0.244229 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239571217e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64364 dup_rows=24216 dup_rows_remote=20740 dedup_ratio=0.246338 router_tokens_per_expert=[782, 793, 743, 773, 765, 792, 786, 792, 784, 774, 827, 789, 804, 789, 803, 763, 773, 782, 788, 725, 790, 741, 753, 769, 762, 728, 746, 740, 751, 741, 764, 721, 729, 787, 751, 752, 784, 742, 740, 835, 784, 822, 714, 764, 772, 790, 726, 768, 729, 744, 770, 728, 767, 748, 820, 760, 793, 734, 809, 774, 774, 764, 788, 757] payload_rows_per_dst=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] dup_rows_per_dst=[3476, 2956, 2904, 2800, 3114, 2968, 3108, 2890] combine_rows_per_src=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343944 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428296 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512648 weight_movement_bytes=34603008 total_nvlink_bytes=567115656 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65290 dup_rows=23796 dup_rows_remote=20748 dedup_ratio=0.242065 router_tokens_per_expert=[756, 724, 794, 785, 830, 724, 772, 775, 801, 735, 746, 776, 770, 780, 801, 777, 753, 769, 767, 752, 821, 749, 773, 758, 764, 794, 757, 780, 741, 789, 780, 728, 764, 748, 746, 730, 757, 752, 750, 760, 767, 745, 813, 810, 763, 787, 791, 789, 743, 765, 765, 770, 793, 765, 764, 767, 761, 743, 744, 753, 754, 800, 778, 794] payload_rows_per_dst=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] dup_rows_per_dst=[2854, 2982, 2944, 3048, 2884, 3148, 2964, 2972] combine_rows_per_src=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12248722656 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6128315616 ingress_bytes=7400718336 link_bytes=7400718336 total_nvlink_bytes=13529033952 weight_to_token_ratio=0.104526 prefetch_share_of_ingress=0.088837 dedup_ratio=0.244325 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210795969e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263634944 dispatch_meta_bytes=340416 combine_payload_bytes=263634944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263975360 ingress_bytes=332840960 link_bytes=332840960 token_bytes=527610304 weight_movement_bytes=69206016 total_nvlink_bytes=596816320 weight_to_token_ratio=0.131169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252453737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254527248 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6131220240 ingress_bytes=7576633344 link_bytes=7576633344 total_nvlink_bytes=13707853584 weight_to_token_ratio=0.118595 prefetch_share_of_ingress=0.086774 dedup_ratio=0.244189 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239576578e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267427840 dispatch_meta_bytes=344152 combine_payload_bytes=267427840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267771992 ingress_bytes=302030848 link_bytes=302030848 token_bytes=535199832 weight_movement_bytes=34603008 total_nvlink_bytes=569802840 weight_to_token_ratio=0.064654 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136517766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12257940016 prefetch_bytes=657457152 reduce_bytes=899678208 weight_movement_bytes=1557135360 egress_bytes=6132929072 ingress_bytes=7682146304 link_bytes=7682146304 total_nvlink_bytes=13815075376 weight_to_token_ratio=0.127031 prefetch_share_of_ingress=0.085582 dedup_ratio=0.244379 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=26 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256839047e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64930 dup_rows=24040 dup_rows_remote=21054 dedup_ratio=0.244548 router_tokens_per_expert=[792, 763, 778, 745, 743, 778, 804, 757, 767, 777, 809, 740, 738, 767, 804, 765, 733, 735, 797, 754, 765, 823, 780, 780, 768, 749, 788, 765, 764, 758, 731, 707, 784, 777, 765, 786, 727, 756, 753, 800, 756, 767, 781, 745, 815, 738, 737, 790, 767, 770, 723, 766, 798, 763, 796, 758, 795, 743, 780, 775, 734, 790, 782, 811] payload_rows_per_dst=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] dup_rows_per_dst=[2986, 3158, 3244, 2870, 2986, 2972, 2980, 2844] combine_rows_per_src=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343936 combine_payload_bytes=265953280 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266297216 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250496 weight_movement_bytes=34603008 total_nvlink_bytes=566853504 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12223492848 prefetch_bytes=657457152 reduce_bytes=519045120 weight_movement_bytes=1176502272 egress_bytes=6115693296 ingress_bytes=7284301824 link_bytes=7284646120 total_nvlink_bytes=13399995120 weight_to_token_ratio=0.096249 prefetch_share_of_ingress=0.090257 dedup_ratio=0.244885 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=15 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191805952e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-29 23:41:55] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 682.6 | throughput per GPU (TFLOP/s/GPU): 186.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.789461E+00 | loss scale: 1.0 | grad norm: 5.153 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64790 dup_rows=24200 dup_rows_remote=21312 dedup_ratio=0.246175 router_tokens_per_expert=[810, 806, 755, 824, 777, 767, 745, 843, 770, 766, 799, 789, 815, 705, 756, 778, 775, 754, 767, 768, 740, 761, 802, 734, 759, 801, 724, 810, 759, 777, 766, 778, 762, 794, 761, 784, 790, 750, 759, 724, 752, 804, 771, 775, 778, 728, 711, 824, 794, 737, 774, 746, 821, 731, 766, 727, 755, 754, 759, 718, 784, 750, 766, 723] payload_rows_per_dst=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] dup_rows_per_dst=[3188, 3128, 2888, 3034, 3056, 3026, 2966, 2914] combine_rows_per_src=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64642 dup_rows=24028 dup_rows_remote=20644 dedup_ratio=0.244425 router_tokens_per_expert=[752, 787, 721, 764, 766, 771, 751, 791, 700, 791, 786, 776, 782, 806, 807, 749, 755, 766, 800, 763, 791, 775, 785, 781, 758, 765, 749, 765, 802, 790, 754, 744, 755, 775, 789, 770, 778, 783, 737, 795, 710, 859, 773, 781, 698, 729, 783, 774, 752, 735, 711, 781, 782, 777, 767, 723, 747, 756, 765, 774, 801, 840, 791, 718] payload_rows_per_dst=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] dup_rows_per_dst=[2794, 3010, 3200, 2498, 2972, 3106, 3064, 3384] combine_rows_per_src=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65282 dup_rows=23782 dup_rows_remote=20816 dedup_ratio=0.241923 router_tokens_per_expert=[746, 814, 748, 794, 767, 749, 779, 774, 755, 768, 736, 807, 762, 743, 795, 754, 792, 787, 778, 751, 808, 792, 746, 761, 755, 794, 757, 745, 812, 771, 785, 744, 808, 804, 718, 734, 777, 760, 707, 795, 774, 766, 758, 765, 723, 714, 764, 764, 761, 796, 761, 772, 736, 797, 805, 767, 760, 772, 768, 801, 761, 756, 797, 742] payload_rows_per_dst=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] dup_rows_per_dst=[2956, 2936, 3008, 2952, 2966, 2968, 3008, 2988] combine_rows_per_src=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64990 dup_rows=24140 dup_rows_remote=21106 dedup_ratio=0.245565 router_tokens_per_expert=[797, 755, 767, 753, 767, 729, 753, 788, 770, 792, 733, 776, 791, 744, 725, 773, 772, 799, 733, 708, 789, 736, 778, 749, 763, 758, 741, 816, 775, 747, 789, 733, 783, 778, 751, 810, 782, 763, 769, 791, 767, 760, 749, 796, 750, 775, 769, 771, 797, 775, 798, 781, 750, 783, 773, 841, 779, 725, 789, 736, 763, 748, 714, 837] payload_rows_per_dst=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] dup_rows_per_dst=[2996, 3034, 2870, 3040, 3088, 3032, 3076, 3004] combine_rows_per_src=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64882 dup_rows=24046 dup_rows_remote=20978 dedup_ratio=0.244609 router_tokens_per_expert=[747, 773, 731, 804, 795, 791, 819, 815, 778, 784, 718, 772, 756, 826, 714, 799, 780, 760, 761, 731, 770, 764, 783, 774, 784, 738, 774, 819, 779, 796, 723, 809, 783, 750, 792, 760, 770, 751, 757, 800, 749, 793, 766, 757, 789, 785, 732, 740, 762, 775, 758, 761, 743, 798, 744, 782, 753, 757, 710, 773, 725, 750, 784, 736] payload_rows_per_dst=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] dup_rows_per_dst=[3140, 3060, 2998, 3068, 3062, 2994, 2922, 2802] combine_rows_per_src=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65028 dup_rows=24030 dup_rows_remote=21012 dedup_ratio=0.244446 router_tokens_per_expert=[761, 767, 738, 748, 760, 730, 747, 800, 844, 811, 784, 805, 728, 755, 756, 776, 749, 715, 760, 770, 761, 756, 773, 787, 803, 790, 830, 752, 765, 765, 777, 773, 798, 794, 737, 733, 778, 787, 832, 788, 733, 803, 770, 789, 756, 733, 801, 747, 728, 780, 761, 761, 737, 740, 776, 728, 755, 803, 794, 726, 747, 773, 771, 757] payload_rows_per_dst=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] dup_rows_per_dst=[2852, 3134, 2880, 3086, 3212, 3018, 2888, 2960] combine_rows_per_src=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65050 dup_rows=23902 dup_rows_remote=20976 dedup_ratio=0.243144 router_tokens_per_expert=[680, 768, 785, 754, 783, 788, 765, 710, 758, 756, 804, 726, 772, 811, 742, 753, 802, 726, 746, 754, 767, 749, 760, 770, 756, 785, 842, 815, 821, 747, 762, 805, 741, 732, 785, 771, 772, 775, 749, 781, 783, 755, 803, 803, 740, 741, 816, 736, 799, 750, 761, 723, 797, 802, 761, 746, 775, 792, 744, 808, 779, 711, 749, 810] payload_rows_per_dst=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] dup_rows_per_dst=[2848, 2942, 2874, 3182, 2984, 2996, 2926, 3150] combine_rows_per_src=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=344408 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265724248 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531104088 weight_movement_bytes=34603008 total_nvlink_bytes=565707096 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344384 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266543424 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742464 weight_movement_bytes=103809024 total_nvlink_bytes=636551488 weight_to_token_ratio=0.194858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341144 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114776 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529888408 weight_movement_bytes=34603008 total_nvlink_bytes=564491416 weight_to_token_ratio=0.065302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344392 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739464 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535134536 weight_movement_bytes=69206016 total_nvlink_bytes=604340552 weight_to_token_ratio=0.129325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100112 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531856784 weight_movement_bytes=103809024 total_nvlink_bytes=635665808 weight_to_token_ratio=0.195182 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344160 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698848 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053536 weight_movement_bytes=34603008 total_nvlink_bytes=567656544 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344104 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788904 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233704 weight_movement_bytes=34603008 total_nvlink_bytes=567836712 weight_to_token_ratio=0.064893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65040 dup_rows=23934 dup_rows_remote=20908 dedup_ratio=0.243469 router_tokens_per_expert=[725, 782, 789, 797, 755, 779, 728, 784, 782, 746, 790, 749, 771, 784, 784, 780, 759, 838, 736, 781, 763, 718, 804, 779, 757, 742, 752, 776, 797, 727, 768, 764, 754, 756, 785, 771, 749, 759, 745, 788, 721, 787, 823, 768, 798, 807, 759, 792, 757, 731, 741, 754, 810, 786, 757, 766, 777, 768, 731, 760, 785, 773, 769, 739] payload_rows_per_dst=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] dup_rows_per_dst=[3108, 3012, 3026, 2916, 2934, 3148, 2870, 2920] combine_rows_per_src=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64722 dup_rows=24196 dup_rows_remote=21152 dedup_ratio=0.246134 router_tokens_per_expert=[784, 804, 784, 780, 767, 732, 730, 770, 802, 770, 784, 758, 815, 751, 781, 754, 818, 776, 714, 808, 757, 768, 773, 776, 757, 778, 759, 767, 747, 757, 710, 758, 735, 768, 744, 758, 799, 798, 799, 747, 759, 785, 793, 772, 807, 765, 798, 803, 779, 769, 770, 765, 731, 727, 742, 753, 795, 796, 771, 742, 698, 759, 754, 782] payload_rows_per_dst=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] dup_rows_per_dst=[3026, 3044, 3094, 2982, 2952, 3224, 2934, 2940] combine_rows_per_src=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64928 dup_rows=24134 dup_rows_remote=21102 dedup_ratio=0.245504 router_tokens_per_expert=[764, 801, 768, 762, 840, 756, 794, 773, 731, 750, 745, 800, 764, 742, 765, 766, 770, 727, 764, 787, 822, 774, 734, 781, 813, 753, 788, 742, 775, 745, 745, 813, 745, 809, 735, 794, 782, 742, 773, 757, 829, 786, 752, 753, 764, 777, 720, 770, 747, 744, 721, 745, 788, 781, 794, 755, 772, 748, 761, 795, 796, 763, 746, 754] payload_rows_per_dst=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] dup_rows_per_dst=[3190, 2944, 3000, 2924, 3032, 2972, 3034, 3038] combine_rows_per_src=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65002 dup_rows=23942 dup_rows_remote=20890 dedup_ratio=0.243551 router_tokens_per_expert=[774, 803, 772, 757, 789, 778, 813, 813, 782, 755, 773, 777, 767, 777, 736, 784, 760, 714, 798, 789, 749, 761, 779, 747, 739, 810, 773, 732, 794, 785, 770, 787, 753, 783, 750, 783, 748, 751, 758, 815, 728, 764, 798, 754, 728, 742, 755, 745, 765, 763, 777, 736, 775, 761, 779, 778, 799, 734, 799, 765, 729, 773, 787, 740] payload_rows_per_dst=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] dup_rows_per_dst=[2946, 2888, 2894, 2936, 2960, 2900, 3366, 3052] combine_rows_per_src=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65284 dup_rows=23920 dup_rows_remote=20876 dedup_ratio=0.243327 router_tokens_per_expert=[782, 760, 743, 764, 738, 753, 765, 766, 765, 802, 766, 795, 790, 721, 772, 766, 748, 793, 795, 817, 742, 791, 768, 753, 757, 753, 741, 760, 754, 798, 792, 717, 744, 796, 766, 763, 732, 778, 795, 761, 729, 729, 761, 761, 797, 785, 775, 813, 774, 715, 788, 786, 817, 765, 783, 762, 755, 810, 767, 799, 756, 809, 725, 729] payload_rows_per_dst=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] dup_rows_per_dst=[2952, 2990, 2994, 3044, 2924, 2962, 3028, 3026] combine_rows_per_src=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64950 dup_rows=24240 dup_rows_remote=21324 dedup_ratio=0.246582 router_tokens_per_expert=[790, 784, 785, 728, 812, 685, 766, 776, 774, 789, 810, 742, 781, 735, 757, 780, 795, 723, 782, 818, 821, 756, 786, 792, 751, 716, 783, 804, 731, 747, 790, 819, 753, 811, 774, 755, 783, 746, 772, 798, 772, 755, 737, 751, 734, 787, 756, 723, 758, 739, 739, 773, 803, 770, 774, 781, 755, 761, 782, 742, 768, 760, 782, 750] payload_rows_per_dst=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] dup_rows_per_dst=[3012, 3078, 3182, 3014, 3028, 2916, 3008, 3002] combine_rows_per_src=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65258 dup_rows=23862 dup_rows_remote=20816 dedup_ratio=0.242737 router_tokens_per_expert=[790, 773, 745, 838, 767, 736, 768, 741, 743, 790, 721, 763, 782, 777, 771, 784, 724, 790, 773, 737, 799, 755, 759, 757, 779, 792, 759, 757, 739, 782, 794, 746, 784, 812, 757, 766, 765, 804, 758, 794, 757, 753, 733, 729, 758, 773, 776, 774, 761, 748, 797, 795, 794, 736, 742, 742, 811, 736, 794, 773, 777, 827, 761, 734] payload_rows_per_dst=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] dup_rows_per_dst=[2992, 2930, 3000, 2976, 3042, 2896, 3046, 2980] combine_rows_per_src=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265101312 dispatch_meta_bytes=343496 combine_payload_bytes=265101312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265444808 ingress_bytes=368910336 link_bytes=368910336 token_bytes=530546120 weight_movement_bytes=103809024 total_nvlink_bytes=634355144 weight_to_token_ratio=0.195664 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388179895e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343792 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266747632 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151472 weight_movement_bytes=103809024 total_nvlink_bytes=636960496 weight_to_token_ratio=0.194708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344120 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289208 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234296 weight_movement_bytes=69206016 total_nvlink_bytes=601440312 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343568 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591760 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839952 weight_movement_bytes=34603008 total_nvlink_bytes=567442960 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=344640 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267747904 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535151168 weight_movement_bytes=34603008 total_nvlink_bytes=569754176 weight_to_token_ratio=0.064660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65166 dup_rows=23810 dup_rows_remote=20706 dedup_ratio=0.242208 router_tokens_per_expert=[798, 794, 761, 726, 797, 753, 775, 790, 766, 795, 743, 753, 735, 791, 770, 724, 769, 766, 718, 841, 779, 794, 762, 726, 770, 800, 818, 781, 763, 777, 819, 752, 751, 814, 748, 765, 766, 732, 774, 738, 746, 780, 752, 750, 720, 774, 780, 806, 759, 802, 790, 762, 780, 760, 763, 730, 757, 765, 744, 741, 795, 802, 761, 739] payload_rows_per_dst=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] dup_rows_per_dst=[3104, 2848, 3002, 3096, 3002, 2890, 2960, 2908] combine_rows_per_src=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=345096 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266380296 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532415496 weight_movement_bytes=34603008 total_nvlink_bytes=567018504 weight_to_token_ratio=0.064992 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344296 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641064 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937832 weight_movement_bytes=34603008 total_nvlink_bytes=569540840 weight_to_token_ratio=0.064686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65040 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[766, 815, 767, 774, 766, 764, 772, 773, 790, 760, 801, 765, 713, 822, 727, 772, 722, 798, 715, 765, 775, 773, 766, 826, 733, 756, 786, 754, 817, 766, 747, 774, 773, 775, 773, 776, 728, 849, 799, 766, 772, 783, 767, 798, 758, 741, 754, 777, 774, 806, 801, 744, 727, 768, 763, 736, 785, 780, 710, 764, 703, 747, 771, 764] payload_rows_per_dst=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] dup_rows_per_dst=[3018, 3022, 2966, 3088, 3114, 3012, 2942, 2824] combine_rows_per_src=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65058 dup_rows=23940 dup_rows_remote=21044 dedup_ratio=0.243530 router_tokens_per_expert=[730, 785, 777, 763, 746, 771, 768, 778, 774, 771, 763, 767, 761, 746, 792, 776, 749, 745, 774, 778, 761, 744, 774, 776, 733, 774, 766, 755, 751, 831, 782, 797, 750, 727, 762, 783, 783, 802, 790, 801, 783, 735, 769, 791, 786, 737, 776, 769, 764, 750, 720, 783, 733, 757, 765, 785, 786, 736, 759, 766, 770, 827, 762, 787] payload_rows_per_dst=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] dup_rows_per_dst=[2948, 3038, 2896, 3146, 3048, 2934, 2880, 3050] combine_rows_per_src=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=343488 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267263424 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534183360 weight_movement_bytes=69206016 total_nvlink_bytes=603389376 weight_to_token_ratio=0.129555 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65034 dup_rows=23892 dup_rows_remote=20906 dedup_ratio=0.243042 router_tokens_per_expert=[780, 707, 735, 741, 765, 784, 806, 755, 735, 778, 765, 762, 822, 762, 779, 781, 762, 783, 757, 736, 786, 750, 808, 813, 782, 778, 740, 797, 785, 749, 787, 788, 792, 769, 751, 732, 768, 788, 788, 794, 753, 751, 768, 758, 784, 757, 761, 758, 744, 754, 756, 756, 769, 774, 774, 739, 722, 732, 812, 791, 774, 737, 807, 781] payload_rows_per_dst=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] dup_rows_per_dst=[2886, 3020, 3054, 3116, 2986, 2902, 2966, 2962] combine_rows_per_src=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74040 payload_rows_remote=64662 dup_rows=24264 dup_rows_remote=21060 dedup_ratio=0.246826 router_tokens_per_expert=[765, 806, 739, 745, 819, 746, 755, 765, 753, 775, 752, 773, 707, 780, 802, 788, 741, 777, 769, 815, 800, 744, 772, 769, 805, 758, 754, 779, 799, 762, 767, 774, 746, 755, 813, 754, 769, 774, 806, 729, 748, 759, 752, 738, 764, 772, 756, 773, 746, 754, 785, 756, 764, 762, 796, 799, 766, 730, 715, 789, 744, 788, 813, 782] payload_rows_per_dst=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] dup_rows_per_dst=[2902, 3038, 2914, 2880, 3000, 3028, 3298, 3204] combine_rows_per_src=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344408 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821976 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299544 weight_movement_bytes=69206016 total_nvlink_bytes=602505560 weight_to_token_ratio=0.129770 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344016 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747856 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151696 weight_movement_bytes=34603008 total_nvlink_bytes=567754704 weight_to_token_ratio=0.064903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64820 dup_rows=24100 dup_rows_remote=21162 dedup_ratio=0.245158 router_tokens_per_expert=[752, 748, 790, 829, 758, 791, 751, 820, 780, 778, 739, 789, 740, 746, 766, 754, 810, 753, 725, 762, 763, 814, 774, 819, 755, 759, 759, 777, 759, 798, 783, 771, 737, 810, 757, 762, 760, 722, 762, 764, 770, 798, 758, 800, 751, 778, 815, 724, 796, 745, 737, 789, 819, 746, 760, 752, 724, 741, 755, 788, 725, 756, 801, 738] payload_rows_per_dst=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] dup_rows_per_dst=[3130, 2914, 3084, 2938, 2946, 3094, 3032, 2962] combine_rows_per_src=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64814 dup_rows=24278 dup_rows_remote=21306 dedup_ratio=0.246969 router_tokens_per_expert=[792, 769, 768, 739, 830, 787, 793, 791, 742, 749, 782, 765, 821, 749, 798, 741, 798, 746, 777, 740, 734, 778, 777, 744, 782, 778, 784, 720, 788, 694, 751, 752, 739, 785, 739, 742, 801, 751, 790, 775, 753, 752, 769, 773, 747, 783, 776, 739, 773, 808, 775, 774, 738, 804, 762, 776, 803, 738, 777, 779, 778, 772, 756, 766] payload_rows_per_dst=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] dup_rows_per_dst=[3146, 3066, 2994, 2982, 2970, 2972, 3100, 3048] combine_rows_per_src=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=65080 dup_rows=24092 dup_rows_remote=21234 dedup_ratio=0.245076 router_tokens_per_expert=[734, 750, 805, 784, 787, 783, 815, 748, 764, 751, 793, 767, 789, 774, 707, 742, 792, 731, 765, 768, 768, 825, 728, 747, 749, 777, 749, 789, 778, 775, 803, 776, 736, 771, 779, 790, 757, 758, 756, 785, 774, 821, 760, 772, 759, 735, 764, 798, 727, 753, 703, 799, 728, 788, 777, 720, 831, 769, 743, 785, 732, 755, 823, 791] payload_rows_per_dst=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] dup_rows_per_dst=[3106, 2906, 3040, 2992, 2958, 3092, 2858, 3140] combine_rows_per_src=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343760 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723024 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102288 weight_movement_bytes=103809024 total_nvlink_bytes=636911312 weight_to_token_ratio=0.194726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342888 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198440 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053992 weight_movement_bytes=34603008 total_nvlink_bytes=564657000 weight_to_token_ratio=0.065282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344480 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265822624 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531300768 weight_movement_bytes=69206016 total_nvlink_bytes=600506784 weight_to_token_ratio=0.130258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343928 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846648 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349368 weight_movement_bytes=69206016 total_nvlink_bytes=600555384 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=345256 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266912936 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533480616 weight_movement_bytes=34603008 total_nvlink_bytes=568083624 weight_to_token_ratio=0.064863 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64960 dup_rows=24018 dup_rows_remote=21146 dedup_ratio=0.244324 router_tokens_per_expert=[783, 739, 794, 807, 766, 801, 761, 799, 780, 786, 810, 777, 848, 775, 731, 760, 813, 755, 756, 748, 723, 762, 777, 765, 773, 732, 724, 797, 759, 738, 748, 781, 773, 808, 795, 723, 713, 813, 774, 746, 758, 782, 742, 748, 764, 780, 773, 745, 742, 796, 761, 763, 776, 750, 751, 760, 752, 846, 773, 744, 775, 735, 811, 712] payload_rows_per_dst=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] dup_rows_per_dst=[3112, 3102, 2872, 2928, 3018, 2956, 3030, 3000] combine_rows_per_src=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65034 dup_rows=23916 dup_rows_remote=20948 dedup_ratio=0.243286 router_tokens_per_expert=[718, 732, 759, 787, 787, 783, 759, 778, 791, 762, 785, 779, 758, 723, 784, 779, 775, 737, 750, 804, 741, 786, 732, 722, 704, 815, 765, 753, 757, 771, 818, 788, 783, 792, 858, 778, 769, 789, 716, 767, 773, 739, 786, 743, 745, 773, 782, 718, 758, 780, 787, 747, 753, 757, 778, 733, 786, 785, 792, 758, 777, 803, 783, 782] payload_rows_per_dst=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] dup_rows_per_dst=[2872, 2968, 2964, 3062, 3082, 2962, 2914, 3092] combine_rows_per_src=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64964 dup_rows=23900 dup_rows_remote=20798 dedup_ratio=0.243123 router_tokens_per_expert=[791, 774, 766, 731, 807, 802, 770, 714, 769, 761, 856, 748, 788, 759, 767, 743, 780, 774, 767, 735, 788, 798, 726, 753, 747, 758, 744, 767, 735, 754, 806, 793, 792, 782, 811, 774, 776, 754, 773, 771, 771, 759, 784, 733, 761, 778, 793, 778, 731, 751, 773, 786, 729, 765, 719, 788, 758, 748, 762, 816, 748, 758, 762, 797] payload_rows_per_dst=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] dup_rows_per_dst=[3102, 3140, 2860, 3202, 2884, 2884, 2902, 2926] combine_rows_per_src=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344424 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266420584 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496744 weight_movement_bytes=69206016 total_nvlink_bytes=601702760 weight_to_token_ratio=0.129965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343928 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266723192 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102456 weight_movement_bytes=138412032 total_nvlink_bytes=671514488 weight_to_token_ratio=0.259635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=65046 dup_rows=24108 dup_rows_remote=21164 dedup_ratio=0.245239 router_tokens_per_expert=[766, 708, 750, 776, 775, 792, 770, 806, 733, 765, 706, 776, 780, 775, 729, 759, 804, 791, 812, 752, 785, 774, 791, 757, 756, 742, 781, 832, 769, 752, 765, 800, 755, 793, 759, 750, 724, 757, 751, 758, 791, 746, 747, 758, 761, 766, 763, 780, 756, 796, 778, 760, 746, 786, 757, 802, 768, 781, 777, 778, 758, 795, 766, 760] payload_rows_per_dst=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] dup_rows_per_dst=[3058, 2906, 3098, 2952, 2944, 2940, 3094, 3116] combine_rows_per_src=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64924 dup_rows=24118 dup_rows_remote=21194 dedup_ratio=0.245341 router_tokens_per_expert=[777, 801, 821, 786, 726, 759, 813, 775, 752, 743, 775, 792, 761, 762, 772, 790, 771, 773, 762, 763, 775, 774, 744, 742, 759, 804, 763, 770, 750, 800, 744, 772, 717, 748, 766, 824, 733, 745, 717, 798, 715, 727, 752, 794, 765, 727, 736, 829, 763, 805, 831, 743, 788, 761, 810, 794, 746, 730, 754, 757, 813, 744, 776, 773] payload_rows_per_dst=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] dup_rows_per_dst=[2758, 2822, 2980, 3062, 2990, 3356, 3226, 2924] combine_rows_per_src=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65190 dup_rows=23868 dup_rows_remote=20896 dedup_ratio=0.242798 router_tokens_per_expert=[813, 804, 765, 771, 747, 763, 779, 775, 767, 785, 781, 744, 740, 782, 794, 767, 805, 800, 719, 745, 776, 761, 748, 750, 750, 801, 836, 760, 778, 738, 764, 794, 756, 790, 762, 832, 739, 724, 792, 733, 772, 725, 768, 787, 759, 761, 762, 775, 779, 761, 762, 763, 765, 749, 772, 773, 788, 721, 755, 743, 811, 747, 764, 760] payload_rows_per_dst=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] dup_rows_per_dst=[3080, 3020, 2924, 3042, 2998, 2972, 2934, 2898] combine_rows_per_src=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65154 dup_rows=23906 dup_rows_remote=20904 dedup_ratio=0.243184 router_tokens_per_expert=[786, 771, 773, 764, 782, 797, 768, 804, 778, 774, 736, 809, 766, 729, 748, 783, 747, 774, 754, 783, 798, 756, 716, 768, 729, 778, 780, 765, 803, 734, 762, 772, 732, 782, 747, 795, 799, 776, 741, 756, 742, 769, 794, 760, 766, 780, 753, 813, 780, 737, 785, 787, 781, 751, 767, 761, 759, 772, 804, 779, 733, 761, 718, 785] payload_rows_per_dst=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] dup_rows_per_dst=[3052, 3030, 2946, 3002, 2998, 2980, 2934, 2964] combine_rows_per_src=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343048 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266435592 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528136 weight_movement_bytes=69206016 total_nvlink_bytes=601734152 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65250 dup_rows=23824 dup_rows_remote=20876 dedup_ratio=0.242350 router_tokens_per_expert=[798, 831, 802, 748, 748, 752, 744, 768, 774, 784, 765, 783, 809, 754, 802, 795, 767, 800, 845, 779, 770, 771, 741, 770, 770, 750, 748, 744, 771, 727, 808, 743, 793, 762, 723, 818, 776, 737, 740, 754, 744, 707, 815, 775, 800, 714, 789, 739, 759, 802, 779, 739, 774, 734, 776, 726, 773, 814, 797, 719, 775, 738, 736, 764] payload_rows_per_dst=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] dup_rows_per_dst=[2928, 3096, 3126, 2964, 2922, 2976, 2948, 2864] combine_rows_per_src=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344840 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773256 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201672 weight_movement_bytes=34603008 total_nvlink_bytes=567804680 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344472 combine_payload_bytes=265928704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266273176 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201880 weight_movement_bytes=69206016 total_nvlink_bytes=601407896 weight_to_token_ratio=0.130037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344344 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267362584 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380824 weight_movement_bytes=34603008 total_nvlink_bytes=568983832 weight_to_token_ratio=0.064753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344232 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215016 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085800 weight_movement_bytes=34603008 total_nvlink_bytes=568688808 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65052 dup_rows=23818 dup_rows_remote=20794 dedup_ratio=0.242289 router_tokens_per_expert=[823, 750, 752, 752, 780, 780, 734, 783, 761, 803, 764, 766, 788, 780, 774, 735, 772, 775, 753, 761, 766, 781, 810, 811, 767, 732, 769, 733, 773, 749, 747, 802, 760, 737, 767, 783, 738, 773, 803, 780, 775, 760, 707, 766, 778, 787, 764, 781, 781, 804, 775, 765, 750, 729, 725, 752, 750, 788, 761, 752, 770, 778, 826, 761] payload_rows_per_dst=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] dup_rows_per_dst=[2972, 3006, 3024, 2908, 2910, 2992, 2898, 3108] combine_rows_per_src=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344504 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608504 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872504 weight_movement_bytes=34603008 total_nvlink_bytes=569475512 weight_to_token_ratio=0.064694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65034 dup_rows=24028 dup_rows_remote=21008 dedup_ratio=0.244425 router_tokens_per_expert=[722, 757, 745, 741, 803, 751, 724, 696, 745, 761, 770, 755, 792, 751, 775, 782, 804, 761, 755, 774, 821, 785, 795, 821, 783, 766, 768, 753, 750, 792, 764, 808, 756, 780, 793, 770, 783, 758, 760, 731, 744, 780, 734, 786, 754, 781, 764, 764, 768, 739, 759, 765, 740, 795, 756, 764, 785, 745, 802, 717, 761, 800, 816, 832] payload_rows_per_dst=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] dup_rows_per_dst=[2834, 3020, 3106, 3068, 3014, 2996, 2882, 3108] combine_rows_per_src=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249368 weight_movement_bytes=69206016 total_nvlink_bytes=602455384 weight_to_token_ratio=0.129782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344168 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723432 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102696 weight_movement_bytes=103809024 total_nvlink_bytes=636911720 weight_to_token_ratio=0.194726 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65204 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[753, 778, 763, 750, 747, 797, 764, 733, 742, 791, 726, 772, 740, 787, 746, 762, 773, 797, 793, 790, 788, 737, 757, 750, 783, 755, 783, 743, 772, 804, 752, 811, 786, 853, 743, 732, 765, 737, 731, 793, 764, 780, 753, 811, 838, 823, 794, 776, 737, 762, 722, 758, 737, 801, 743, 812, 714, 791, 781, 768, 736, 739, 750, 783] payload_rows_per_dst=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] dup_rows_per_dst=[2966, 3050, 3036, 2980, 2938, 3016, 3030, 2880] combine_rows_per_src=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65084 dup_rows=24130 dup_rows_remote=21120 dedup_ratio=0.245463 router_tokens_per_expert=[804, 743, 806, 779, 740, 813, 730, 747, 758, 747, 750, 778, 774, 787, 736, 786, 752, 764, 762, 799, 794, 792, 782, 737, 796, 820, 743, 803, 764, 753, 739, 785, 767, 775, 771, 747, 756, 770, 705, 759, 761, 762, 795, 707, 803, 736, 767, 792, 773, 761, 815, 733, 781, 769, 729, 774, 765, 765, 769, 797, 807, 726, 787, 765] payload_rows_per_dst=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] dup_rows_per_dst=[2996, 2934, 3136, 3032, 3010, 2986, 2996, 3040] combine_rows_per_src=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64568 dup_rows=24380 dup_rows_remote=21280 dedup_ratio=0.248006 router_tokens_per_expert=[789, 759, 787, 831, 776, 817, 732, 773, 742, 767, 738, 722, 792, 767, 811, 753, 715, 780, 823, 733, 767, 798, 755, 717, 748, 747, 760, 790, 751, 742, 734, 728, 770, 738, 747, 801, 739, 727, 767, 756, 749, 777, 765, 780, 801, 791, 770, 772, 766, 779, 777, 748, 822, 804, 770, 786, 765, 783, 789, 769, 764, 752, 802, 782] payload_rows_per_dst=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] dup_rows_per_dst=[3150, 2958, 2712, 3248, 2972, 2956, 3284, 3100] combine_rows_per_src=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64852 dup_rows=24240 dup_rows_remote=21278 dedup_ratio=0.246582 router_tokens_per_expert=[796, 762, 785, 738, 796, 787, 804, 811, 770, 769, 749, 732, 797, 750, 783, 808, 754, 784, 789, 741, 777, 766, 812, 737, 769, 755, 780, 751, 728, 728, 774, 791, 782, 742, 811, 769, 739, 784, 759, 757, 735, 775, 735, 782, 765, 781, 758, 756, 752, 711, 742, 824, 757, 741, 743, 820, 781, 769, 788, 761, 761, 746, 794, 759] payload_rows_per_dst=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] dup_rows_per_dst=[3208, 3054, 3074, 2946, 2962, 2962, 2984, 3050] combine_rows_per_src=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65126 dup_rows=23918 dup_rows_remote=20970 dedup_ratio=0.243306 router_tokens_per_expert=[746, 778, 793, 701, 771, 701, 793, 766, 781, 773, 766, 760, 729, 794, 772, 780, 792, 779, 768, 744, 770, 742, 774, 752, 833, 761, 748, 740, 756, 750, 741, 775, 784, 772, 778, 779, 784, 757, 761, 830, 747, 763, 792, 751, 763, 755, 782, 791, 791, 751, 735, 761, 745, 776, 730, 775, 770, 799, 806, 799, 818, 758, 758, 762] payload_rows_per_dst=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] dup_rows_per_dst=[2932, 2994, 3002, 2948, 3048, 3022, 2868, 3104] combine_rows_per_src=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64948 dup_rows=24080 dup_rows_remote=21186 dedup_ratio=0.244954 router_tokens_per_expert=[767, 772, 710, 747, 737, 786, 790, 732, 748, 794, 789, 793, 749, 798, 756, 778, 774, 786, 767, 764, 783, 746, 803, 760, 759, 773, 742, 705, 796, 768, 748, 779, 763, 788, 786, 822, 774, 747, 774, 768, 811, 775, 781, 797, 786, 792, 776, 779, 770, 721, 758, 762, 778, 758, 717, 821, 798, 704, 795, 772, 735, 751, 739, 755] payload_rows_per_dst=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] dup_rows_per_dst=[2902, 3130, 3084, 2910, 3052, 3152, 2894, 2956] combine_rows_per_src=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344536 combine_payload_bytes=267075584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267420120 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534495704 weight_movement_bytes=69206016 total_nvlink_bytes=603701720 weight_to_token_ratio=0.129479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344816 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928880 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512944 weight_movement_bytes=34603008 total_nvlink_bytes=568115952 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264470528 dispatch_meta_bytes=343392 combine_payload_bytes=264470528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264813920 ingress_bytes=368279552 link_bytes=368279552 token_bytes=529284448 weight_movement_bytes=103809024 total_nvlink_bytes=633093472 weight_to_token_ratio=0.196131 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385806305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344520 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265978312 ingress_bytes=334839808 link_bytes=334839808 token_bytes=531612104 weight_movement_bytes=69206016 total_nvlink_bytes=600818120 weight_to_token_ratio=0.130181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259975241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64968 dup_rows=24114 dup_rows_remote=21184 dedup_ratio=0.245300 router_tokens_per_expert=[741, 764, 710, 750, 795, 769, 757, 757, 760, 772, 767, 810, 801, 756, 790, 817, 783, 760, 741, 769, 760, 751, 738, 774, 783, 779, 759, 778, 789, 751, 733, 750, 762, 772, 780, 791, 800, 729, 768, 803, 780, 772, 765, 762, 727, 756, 816, 801, 745, 792, 796, 775, 770, 738, 690, 732, 800, 751, 731, 781, 773, 767, 792, 821] payload_rows_per_dst=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] dup_rows_per_dst=[2996, 3094, 2930, 2972, 3190, 2988, 2904, 3040] combine_rows_per_src=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344384 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100480 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856576 weight_movement_bytes=34603008 total_nvlink_bytes=568459584 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64828 dup_rows=24306 dup_rows_remote=21304 dedup_ratio=0.247253 router_tokens_per_expert=[781, 775, 757, 769, 794, 766, 800, 780, 744, 804, 775, 771, 750, 756, 747, 739, 803, 759, 732, 747, 791, 762, 771, 783, 791, 817, 729, 737, 703, 727, 756, 736, 797, 757, 749, 783, 788, 768, 792, 753, 777, 772, 747, 810, 747, 756, 750, 728, 711, 786, 775, 760, 821, 763, 778, 807, 736, 787, 789, 775, 808, 754, 762, 814] payload_rows_per_dst=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] dup_rows_per_dst=[3070, 3002, 2982, 2876, 3032, 3080, 3118, 3146] combine_rows_per_src=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344536 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371544 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398552 weight_movement_bytes=34603008 total_nvlink_bytes=567001560 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266453536 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532562464 weight_movement_bytes=34603008 total_nvlink_bytes=567165472 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344528 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880016 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415504 weight_movement_bytes=34603008 total_nvlink_bytes=566018512 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65120 dup_rows=23870 dup_rows_remote=20804 dedup_ratio=0.242818 router_tokens_per_expert=[740, 765, 754, 790, 793, 769, 754, 768, 766, 782, 790, 795, 773, 761, 775, 810, 802, 751, 775, 812, 774, 782, 758, 759, 774, 743, 748, 764, 754, 749, 807, 795, 815, 779, 772, 785, 747, 760, 800, 723, 751, 724, 758, 796, 749, 736, 746, 770, 716, 748, 731, 753, 736, 753, 805, 802, 761, 793, 773, 769, 803, 740, 753, 773] payload_rows_per_dst=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] dup_rows_per_dst=[3066, 2978, 2984, 3032, 2972, 2868, 2950, 3020] combine_rows_per_src=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65088 dup_rows=23978 dup_rows_remote=20866 dedup_ratio=0.243917 router_tokens_per_expert=[761, 751, 776, 749, 709, 749, 796, 768, 745, 785, 754, 795, 785, 706, 765, 786, 750, 774, 717, 813, 788, 727, 771, 742, 726, 781, 800, 730, 790, 726, 797, 783, 756, 773, 790, 772, 763, 833, 751, 737, 791, 773, 777, 789, 785, 796, 752, 757, 788, 784, 805, 770, 743, 768, 803, 731, 738, 802, 737, 798, 757, 767, 762, 809] payload_rows_per_dst=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] dup_rows_per_dst=[2846, 2988, 2936, 2908, 3112, 3112, 3110, 2966] combine_rows_per_src=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65084 dup_rows=23992 dup_rows_remote=21046 dedup_ratio=0.244059 router_tokens_per_expert=[807, 774, 757, 776, 713, 783, 756, 793, 781, 791, 767, 789, 752, 748, 769, 765, 784, 750, 788, 768, 803, 760, 790, 804, 751, 715, 730, 791, 733, 774, 780, 777, 813, 749, 802, 771, 737, 801, 797, 772, 773, 798, 773, 772, 746, 731, 753, 792, 778, 757, 764, 783, 752, 740, 769, 729, 770, 789, 727, 765, 797, 779, 738, 716] payload_rows_per_dst=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] dup_rows_per_dst=[3116, 2972, 2992, 2976, 3080, 2934, 2976, 2946] combine_rows_per_src=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73932 payload_rows_remote=64778 dup_rows=24372 dup_rows_remote=21318 dedup_ratio=0.247925 router_tokens_per_expert=[760, 768, 809, 758, 793, 790, 774, 813, 789, 741, 790, 782, 741, 763, 767, 745, 802, 762, 761, 737, 737, 766, 761, 713, 763, 781, 771, 830, 796, 809, 751, 775, 751, 766, 732, 782, 746, 783, 723, 763, 786, 716, 775, 810, 750, 749, 753, 765, 771, 821, 776, 799, 769, 760, 745, 749, 813, 796, 719, 760, 697, 825, 775, 729] payload_rows_per_dst=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] dup_rows_per_dst=[3246, 3104, 2920, 3144, 2824, 3054, 3060, 3020] combine_rows_per_src=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65024 dup_rows=23928 dup_rows_remote=20906 dedup_ratio=0.243408 router_tokens_per_expert=[778, 763, 746, 791, 753, 789, 757, 763, 767, 785, 788, 755, 781, 755, 734, 693, 753, 811, 790, 764, 776, 782, 794, 766, 787, 783, 810, 772, 783, 771, 731, 750, 761, 802, 813, 753, 743, 793, 729, 776, 755, 765, 774, 778, 745, 763, 766, 761, 717, 760, 769, 798, 770, 812, 757, 722, 760, 770, 792, 761, 746, 774, 768, 778] payload_rows_per_dst=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] dup_rows_per_dst=[2978, 2912, 3028, 3022, 3006, 2974, 2974, 3034] combine_rows_per_src=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=65022 dup_rows=24070 dup_rows_remote=21262 dedup_ratio=0.244853 router_tokens_per_expert=[719, 747, 784, 740, 747, 758, 754, 727, 816, 771, 722, 727, 769, 767, 755, 768, 759, 749, 805, 850, 778, 730, 712, 781, 792, 712, 782, 745, 806, 806, 785, 779, 791, 734, 775, 813, 777, 771, 754, 752, 773, 764, 773, 826, 760, 862, 734, 801, 754, 729, 760, 771, 777, 740, 755, 724, 790, 765, 797, 793, 712, 783, 813, 787] payload_rows_per_dst=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] dup_rows_per_dst=[2838, 2954, 3054, 3082, 3040, 3136, 2808, 3158] combine_rows_per_src=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343816 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944264 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544712 weight_movement_bytes=69206016 total_nvlink_bytes=602750728 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344520 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266928584 ingress_bytes=370393088 link_bytes=370393088 token_bytes=533512648 weight_movement_bytes=103809024 total_nvlink_bytes=637321672 weight_to_token_ratio=0.194577 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393759371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65000 dup_rows=24014 dup_rows_remote=20940 dedup_ratio=0.244283 router_tokens_per_expert=[809, 739, 760, 769, 768, 732, 767, 753, 744, 709, 752, 805, 812, 807, 777, 768, 797, 728, 772, 772, 793, 740, 815, 765, 797, 763, 755, 773, 769, 752, 773, 776, 803, 714, 778, 763, 806, 749, 769, 731, 748, 756, 801, 768, 780, 753, 778, 762, 755, 797, 764, 764, 834, 754, 735, 752, 750, 763, 771, 721, 797, 790, 753, 782] payload_rows_per_dst=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] dup_rows_per_dst=[2962, 3084, 3074, 2876, 3046, 2990, 3038, 2944] combine_rows_per_src=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344384 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265675072 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005760 weight_movement_bytes=103809024 total_nvlink_bytes=634814784 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343720 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682024 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020328 weight_movement_bytes=69206016 total_nvlink_bytes=602226344 weight_to_token_ratio=0.129837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=345136 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266675248 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533005360 weight_movement_bytes=34603008 total_nvlink_bytes=567608368 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65230 dup_rows=23904 dup_rows_remote=20912 dedup_ratio=0.243164 router_tokens_per_expert=[779, 791, 797, 744, 759, 737, 741, 813, 767, 728, 732, 808, 743, 720, 808, 775, 733, 771, 800, 760, 750, 794, 809, 814, 778, 726, 778, 775, 739, 789, 758, 734, 813, 740, 741, 757, 779, 776, 765, 745, 750, 779, 802, 757, 709, 826, 820, 752, 736, 777, 786, 762, 774, 781, 817, 739, 743, 779, 747, 728, 770, 811, 751, 790] payload_rows_per_dst=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] dup_rows_per_dst=[2972, 2992, 3070, 2900, 2942, 3010, 3078, 2940] combine_rows_per_src=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343760 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266583760 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532823760 weight_movement_bytes=34603008 total_nvlink_bytes=567426768 weight_to_token_ratio=0.064943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64934 dup_rows=23924 dup_rows_remote=20842 dedup_ratio=0.243368 router_tokens_per_expert=[702, 748, 742, 770, 782, 809, 758, 744, 804, 737, 785, 784, 746, 771, 740, 797, 774, 780, 801, 739, 776, 783, 759, 802, 735, 751, 726, 776, 802, 771, 780, 741, 782, 803, 754, 767, 776, 809, 733, 753, 786, 769, 783, 741, 742, 785, 771, 760, 788, 735, 790, 754, 792, 795, 779, 792, 792, 801, 726, 734, 769, 749, 754, 773] payload_rows_per_dst=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] dup_rows_per_dst=[3082, 3012, 3128, 2880, 3108, 3060, 3018, 2636] combine_rows_per_src=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64876 dup_rows=23930 dup_rows_remote=20868 dedup_ratio=0.243429 router_tokens_per_expert=[779, 765, 746, 740, 720, 743, 764, 725, 761, 787, 712, 763, 801, 709, 760, 754, 790, 719, 752, 799, 750, 779, 771, 783, 764, 809, 777, 765, 835, 757, 716, 793, 746, 774, 815, 811, 782, 748, 796, 808, 732, 774, 763, 788, 728, 753, 764, 800, 725, 769, 780, 780, 787, 748, 768, 757, 779, 760, 800, 797, 795, 809, 745, 783] payload_rows_per_dst=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] dup_rows_per_dst=[2760, 2998, 3024, 3104, 3062, 2936, 2966, 3080] combine_rows_per_src=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344568 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526648 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708728 weight_movement_bytes=34603008 total_nvlink_bytes=569311736 weight_to_token_ratio=0.064714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64786 dup_rows=24018 dup_rows_remote=20926 dedup_ratio=0.244324 router_tokens_per_expert=[766, 759, 736, 797, 760, 786, 736, 729, 727, 753, 767, 783, 751, 787, 755, 742, 718, 786, 761, 740, 774, 782, 786, 756, 737, 771, 762, 754, 786, 758, 756, 803, 772, 768, 805, 798, 795, 800, 729, 749, 800, 772, 747, 756, 781, 734, 774, 765, 800, 790, 755, 779, 774, 798, 753, 726, 801, 781, 779, 771, 819, 764, 760, 793] payload_rows_per_dst=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] dup_rows_per_dst=[2976, 2972, 2954, 3084, 3020, 2926, 2994, 3092] combine_rows_per_src=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65160 dup_rows=23782 dup_rows_remote=20832 dedup_ratio=0.241923 router_tokens_per_expert=[721, 768, 753, 774, 783, 816, 812, 783, 779, 713, 810, 782, 761, 767, 756, 764, 727, 759, 751, 773, 770, 731, 777, 774, 724, 758, 740, 787, 766, 797, 804, 743, 740, 751, 788, 732, 777, 740, 767, 795, 762, 805, 789, 731, 750, 767, 789, 763, 721, 750, 752, 808, 790, 769, 819, 780, 780, 785, 753, 786, 770, 786, 727, 807] payload_rows_per_dst=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] dup_rows_per_dst=[3026, 2970, 2888, 2978, 2882, 2950, 3016, 3072] combine_rows_per_src=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20866 dedup_ratio=0.243205 router_tokens_per_expert=[776, 712, 739, 755, 759, 738, 815, 778, 785, 793, 750, 755, 804, 768, 749, 797, 727, 795, 790, 750, 784, 783, 782, 752, 763, 803, 793, 760, 779, 773, 783, 736, 775, 779, 746, 762, 746, 773, 766, 780, 709, 767, 811, 783, 768, 782, 772, 777, 783, 736, 794, 737, 751, 747, 792, 762, 798, 787, 781, 781, 732, 785, 704, 760] payload_rows_per_dst=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] dup_rows_per_dst=[2924, 3040, 3000, 3042, 2978, 3042, 2920, 2962] combine_rows_per_src=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74750 payload_rows_remote=65304 dup_rows=23554 dup_rows_remote=20528 dedup_ratio=0.239604 router_tokens_per_expert=[765, 784, 767, 767, 807, 761, 786, 717, 770, 728, 768, 786, 717, 772, 761, 764, 739, 794, 762, 725, 740, 757, 772, 776, 749, 765, 756, 751, 730, 758, 788, 801, 774, 734, 813, 771, 747, 746, 761, 794, 749, 831, 761, 766, 756, 758, 783, 759, 788, 771, 816, 757, 767, 806, 802, 729, 757, 793, 769, 786, 794, 752, 812, 767] payload_rows_per_dst=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] dup_rows_per_dst=[2952, 2878, 2918, 2870, 2944, 3006, 3026, 2960] combine_rows_per_src=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312768 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282432 weight_movement_bytes=34603008 total_nvlink_bytes=566885440 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=342976 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075072 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807168 weight_movement_bytes=69206016 total_nvlink_bytes=601013184 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342848 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265706304 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531069760 weight_movement_bytes=138412032 total_nvlink_bytes=669481792 weight_to_token_ratio=0.260629 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343968 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239328 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134688 weight_movement_bytes=69206016 total_nvlink_bytes=603340704 weight_to_token_ratio=0.129567 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64906 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[752, 771, 789, 763, 693, 737, 796, 744, 725, 773, 804, 799, 723, 805, 750, 765, 788, 760, 720, 798, 737, 801, 780, 785, 780, 789, 750, 768, 793, 813, 788, 755, 796, 773, 718, 799, 758, 802, 748, 784, 714, 740, 777, 774, 753, 752, 767, 797, 751, 798, 816, 801, 766, 754, 782, 779, 733, 798, 725, 784, 750, 766, 749, 754] payload_rows_per_dst=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] dup_rows_per_dst=[2884, 2994, 3024, 3182, 3004, 2960, 3130, 2906] combine_rows_per_src=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343696 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821264 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533298832 weight_movement_bytes=34603008 total_nvlink_bytes=567901840 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343328 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267828512 ingress_bytes=336691200 link_bytes=336691200 token_bytes=535313696 weight_movement_bytes=69206016 total_nvlink_bytes=604519712 weight_to_token_ratio=0.129281 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266941880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65120 dup_rows=23952 dup_rows_remote=21002 dedup_ratio=0.243652 router_tokens_per_expert=[772, 762, 750, 741, 848, 755, 738, 797, 772, 799, 749, 800, 765, 762, 723, 721, 780, 772, 790, 797, 734, 728, 789, 762, 791, 777, 797, 810, 739, 764, 845, 765, 699, 771, 755, 711, 785, 739, 769, 775, 761, 787, 741, 724, 773, 799, 765, 779, 835, 764, 812, 790, 769, 778, 749, 774, 751, 800, 754, 777, 769, 730, 676, 797] payload_rows_per_dst=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] dup_rows_per_dst=[3030, 2950, 2962, 3062, 2854, 3006, 3164, 2924] combine_rows_per_src=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343864 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198840 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053816 weight_movement_bytes=34603008 total_nvlink_bytes=566656824 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64880 dup_rows=24068 dup_rows_remote=20948 dedup_ratio=0.244832 router_tokens_per_expert=[765, 825, 756, 748, 747, 764, 787, 753, 791, 750, 771, 764, 733, 778, 748, 746, 796, 745, 764, 746, 778, 798, 743, 717, 782, 752, 793, 794, 777, 752, 774, 724, 740, 784, 807, 736, 811, 765, 789, 806, 828, 746, 780, 749, 741, 775, 751, 759, 703, 794, 748, 794, 721, 746, 805, 744, 812, 786, 785, 797, 770, 772, 756, 791] payload_rows_per_dst=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] dup_rows_per_dst=[3040, 2902, 2988, 2986, 3120, 3028, 2918, 3086] combine_rows_per_src=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64690 dup_rows=24278 dup_rows_remote=21038 dedup_ratio=0.246969 router_tokens_per_expert=[720, 701, 817, 771, 787, 748, 771, 737, 749, 743, 804, 731, 771, 739, 790, 785, 768, 731, 803, 785, 803, 762, 728, 742, 800, 770, 781, 772, 832, 832, 805, 755, 784, 738, 765, 732, 727, 795, 757, 782, 765, 767, 723, 732, 772, 753, 804, 759, 745, 746, 757, 796, 760, 758, 791, 745, 775, 786, 789, 800, 748, 788, 776, 804] payload_rows_per_dst=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] dup_rows_per_dst=[3240, 3140, 3026, 3052, 2868, 2896, 3258, 2798] combine_rows_per_src=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344488 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076008 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807528 weight_movement_bytes=34603008 total_nvlink_bytes=568410536 weight_to_token_ratio=0.064823 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64860 dup_rows=24140 dup_rows_remote=21008 dedup_ratio=0.245565 router_tokens_per_expert=[792, 760, 779, 746, 745, 791, 738, 781, 758, 801, 790, 779, 768, 775, 787, 794, 721, 751, 800, 760, 756, 774, 791, 750, 785, 769, 818, 771, 741, 776, 798, 758, 759, 725, 768, 725, 716, 792, 754, 734, 732, 776, 747, 753, 748, 734, 799, 786, 781, 741, 809, 794, 775, 797, 766, 779, 785, 765, 745, 740, 780, 768, 781, 795] payload_rows_per_dst=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] dup_rows_per_dst=[2950, 3232, 2998, 2772, 2816, 3188, 3052, 3132] combine_rows_per_src=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65044 dup_rows=23942 dup_rows_remote=20930 dedup_ratio=0.243551 router_tokens_per_expert=[782, 721, 799, 784, 744, 759, 814, 739, 758, 764, 762, 776, 770, 832, 756, 777, 821, 831, 756, 721, 759, 746, 782, 739, 799, 798, 798, 761, 730, 794, 769, 763, 748, 743, 738, 695, 736, 816, 743, 773, 758, 843, 740, 775, 741, 741, 773, 794, 749, 788, 726, 740, 782, 748, 801, 721, 735, 773, 744, 783, 805, 823, 746, 827] payload_rows_per_dst=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] dup_rows_per_dst=[3078, 3024, 3060, 3064, 2778, 3012, 2876, 3050] combine_rows_per_src=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64736 dup_rows=24158 dup_rows_remote=21042 dedup_ratio=0.245748 router_tokens_per_expert=[807, 781, 791, 720, 782, 770, 815, 777, 731, 657, 742, 740, 797, 778, 794, 764, 746, 786, 779, 779, 813, 747, 750, 782, 776, 816, 745, 760, 776, 783, 785, 749, 730, 803, 778, 788, 775, 751, 786, 762, 807, 761, 735, 759, 729, 736, 713, 785, 795, 802, 732, 776, 774, 752, 785, 847, 779, 756, 785, 745, 784, 785, 735, 704] payload_rows_per_dst=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] dup_rows_per_dst=[3126, 2930, 3038, 3064, 2994, 2946, 3116, 2944] combine_rows_per_src=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65048 dup_rows=24084 dup_rows_remote=21056 dedup_ratio=0.244995 router_tokens_per_expert=[790, 734, 730, 810, 797, 754, 739, 791, 738, 783, 795, 785, 825, 744, 769, 754, 792, 764, 799, 732, 723, 740, 769, 773, 727, 733, 769, 754, 726, 835, 775, 781, 791, 778, 792, 784, 784, 751, 781, 788, 762, 756, 731, 778, 767, 762, 728, 750, 752, 739, 811, 764, 788, 709, 737, 787, 777, 759, 785, 796, 820, 789, 741, 785] payload_rows_per_dst=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] dup_rows_per_dst=[3022, 2998, 3046, 3028, 3066, 2838, 2960, 3126] combine_rows_per_src=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343312 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091792 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840272 weight_movement_bytes=34603008 total_nvlink_bytes=566443280 weight_to_token_ratio=0.065063 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342912 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313152 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283392 weight_movement_bytes=34603008 total_nvlink_bytes=564886400 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65040 dup_rows=24120 dup_rows_remote=21150 dedup_ratio=0.245361 router_tokens_per_expert=[715, 782, 745, 759, 763, 767, 772, 783, 753, 728, 785, 748, 800, 764, 821, 740, 744, 775, 786, 782, 743, 741, 764, 722, 817, 764, 825, 727, 797, 768, 789, 762, 796, 793, 798, 775, 750, 741, 770, 779, 762, 756, 764, 770, 758, 798, 790, 760, 788, 803, 774, 704, 745, 733, 771, 754, 773, 774, 760, 777, 801, 794, 765, 745] payload_rows_per_dst=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] dup_rows_per_dst=[2960, 2990, 2970, 3074, 3070, 2956, 3026, 3074] combine_rows_per_src=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=343472 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266010032 ingress_bytes=334872576 link_bytes=334872576 token_bytes=531676592 weight_movement_bytes=69206016 total_nvlink_bytes=600882608 weight_to_token_ratio=0.130166 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260098544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343896 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764120 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184344 weight_movement_bytes=34603008 total_nvlink_bytes=567787352 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343112 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265501768 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660424 weight_movement_bytes=103809024 total_nvlink_bytes=634469448 weight_to_token_ratio=0.195622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344416 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266781024 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533217632 weight_movement_bytes=103809024 total_nvlink_bytes=637026656 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65154 dup_rows=23870 dup_rows_remote=20970 dedup_ratio=0.242818 router_tokens_per_expert=[784, 753, 746, 796, 780, 781, 786, 824, 739, 763, 667, 772, 767, 786, 779, 817, 797, 758, 755, 798, 742, 766, 760, 806, 724, 804, 833, 745, 774, 712, 701, 792, 767, 752, 724, 769, 772, 798, 742, 792, 780, 749, 800, 765, 744, 800, 770, 785, 778, 766, 737, 784, 793, 759, 774, 773, 742, 741, 788, 780, 773, 761, 758, 729] payload_rows_per_dst=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] dup_rows_per_dst=[3098, 2900, 3016, 2970, 2902, 3018, 3058, 2908] combine_rows_per_src=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344760 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266748600 ingress_bytes=335609856 link_bytes=335609856 token_bytes=533152440 weight_movement_bytes=69206016 total_nvlink_bytes=602358456 weight_to_token_ratio=0.129805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65160 dup_rows=23924 dup_rows_remote=20916 dedup_ratio=0.243368 router_tokens_per_expert=[746, 748, 738, 748, 747, 774, 803, 782, 775, 780, 776, 747, 784, 820, 776, 794, 800, 780, 779, 722, 776, 768, 794, 809, 744, 775, 758, 770, 723, 751, 740, 743, 802, 694, 753, 771, 771, 745, 783, 795, 781, 727, 795, 738, 757, 757, 838, 735, 803, 750, 792, 756, 739, 835, 777, 758, 754, 756, 735, 776, 758, 796, 781, 774] payload_rows_per_dst=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] dup_rows_per_dst=[2998, 3070, 3104, 2892, 3008, 2892, 3024, 2936] combine_rows_per_src=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64744 dup_rows=23988 dup_rows_remote=20746 dedup_ratio=0.244019 router_tokens_per_expert=[763, 798, 786, 827, 815, 779, 763, 806, 727, 725, 796, 772, 793, 772, 791, 746, 764, 754, 784, 767, 762, 742, 775, 760, 774, 780, 780, 752, 774, 744, 780, 710, 806, 731, 775, 757, 788, 772, 749, 766, 760, 763, 785, 761, 767, 757, 747, 760, 734, 772, 740, 778, 753, 764, 750, 737, 775, 801, 765, 750, 822, 774, 755, 777] payload_rows_per_dst=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] dup_rows_per_dst=[3242, 3226, 3004, 3022, 2900, 3010, 2864, 2720] combine_rows_per_src=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344496 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267215280 ingress_bytes=370679808 link_bytes=370679808 token_bytes=534086064 weight_movement_bytes=103809024 total_nvlink_bytes=637895088 weight_to_token_ratio=0.194368 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394838275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64820 dup_rows=23972 dup_rows_remote=20824 dedup_ratio=0.243856 router_tokens_per_expert=[781, 761, 750, 778, 780, 764, 778, 708, 727, 745, 780, 764, 749, 761, 798, 802, 768, 795, 805, 795, 721, 771, 775, 748, 792, 805, 821, 768, 827, 747, 781, 734, 766, 764, 735, 760, 814, 762, 783, 780, 795, 774, 705, 721, 819, 802, 771, 780, 796, 773, 815, 788, 704, 741, 743, 784, 774, 706, 766, 768, 797, 721, 707, 759] payload_rows_per_dst=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] dup_rows_per_dst=[2922, 2954, 2738, 3026, 2984, 3068, 3132, 3148] combine_rows_per_src=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64882 dup_rows=24116 dup_rows_remote=21102 dedup_ratio=0.245321 router_tokens_per_expert=[814, 746, 791, 746, 772, 791, 765, 794, 749, 757, 767, 808, 758, 739, 754, 794, 801, 772, 784, 733, 777, 787, 760, 765, 761, 704, 785, 786, 713, 808, 759, 788, 748, 798, 758, 692, 785, 796, 770, 802, 795, 736, 763, 758, 746, 764, 816, 782, 757, 788, 758, 775, 780, 767, 774, 727, 756, 755, 730, 790, 741, 796, 734, 787] payload_rows_per_dst=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] dup_rows_per_dst=[3168, 2984, 3054, 3006, 3044, 3014, 2990, 2856] combine_rows_per_src=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64952 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[784, 833, 774, 752, 766, 774, 771, 768, 744, 750, 801, 836, 790, 758, 742, 804, 783, 716, 754, 785, 757, 781, 773, 761, 759, 777, 784, 753, 726, 791, 800, 768, 769, 758, 779, 813, 747, 774, 766, 736, 765, 741, 755, 741, 804, 779, 725, 756, 761, 797, 723, 766, 780, 764, 756, 733, 769, 735, 770, 774, 750, 803, 779, 769] payload_rows_per_dst=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] dup_rows_per_dst=[3036, 3068, 3020, 3044, 3056, 2942, 2956, 3026] combine_rows_per_src=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74654 payload_rows_remote=65222 dup_rows=23650 dup_rows_remote=20610 dedup_ratio=0.240580 router_tokens_per_expert=[785, 782, 725, 741, 751, 753, 751, 782, 794, 764, 725, 742, 799, 731, 786, 754, 788, 779, 790, 779, 750, 808, 778, 734, 775, 805, 795, 758, 801, 763, 789, 750, 802, 793, 759, 770, 788, 755, 779, 752, 778, 740, 758, 809, 765, 799, 787, 770, 765, 780, 749, 756, 779, 733, 756, 716, 770, 735, 780, 751, 736, 749, 811, 775] payload_rows_per_dst=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] dup_rows_per_dst=[2866, 2938, 2970, 3040, 2962, 3012, 2914, 2948] combine_rows_per_src=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344304 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239664 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534135024 weight_movement_bytes=69206016 total_nvlink_bytes=603341040 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=341960 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265533384 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530724808 weight_movement_bytes=34603008 total_nvlink_bytes=565327816 weight_to_token_ratio=0.065200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=65048 dup_rows=24196 dup_rows_remote=21142 dedup_ratio=0.246134 router_tokens_per_expert=[830, 750, 759, 738, 761, 767, 792, 791, 734, 774, 795, 792, 767, 740, 782, 777, 795, 795, 754, 718, 763, 731, 757, 744, 781, 773, 745, 779, 745, 816, 775, 779, 759, 785, 817, 759, 750, 775, 737, 745, 756, 790, 727, 800, 750, 780, 788, 799, 768, 774, 761, 794, 763, 792, 834, 768, 736, 778, 731, 717, 723, 797, 715, 785] payload_rows_per_dst=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] dup_rows_per_dst=[3032, 3014, 3054, 3042, 3020, 3094, 3084, 2856] combine_rows_per_src=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=342576 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265845296 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348016 weight_movement_bytes=34603008 total_nvlink_bytes=565951024 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343936 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100608 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857280 weight_movement_bytes=69206016 total_nvlink_bytes=601063296 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344576 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266387968 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431360 weight_movement_bytes=34603008 total_nvlink_bytes=567034368 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=343328 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267492640 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534641952 weight_movement_bytes=69206016 total_nvlink_bytes=603847968 weight_to_token_ratio=0.129444 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344760 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266781368 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217976 weight_movement_bytes=34603008 total_nvlink_bytes=567820984 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65240 dup_rows=23944 dup_rows_remote=20994 dedup_ratio=0.243571 router_tokens_per_expert=[788, 779, 711, 770, 763, 807, 785, 763, 748, 774, 763, 760, 775, 748, 748, 719, 774, 770, 771, 814, 755, 790, 785, 815, 783, 785, 742, 756, 784, 751, 741, 802, 767, 788, 782, 775, 781, 812, 779, 780, 786, 743, 734, 770, 772, 775, 768, 737, 749, 781, 739, 754, 730, 754, 724, 749, 727, 797, 764, 767, 797, 788, 778, 786] payload_rows_per_dst=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] dup_rows_per_dst=[2990, 2950, 3162, 2950, 3122, 2938, 2818, 3014] combine_rows_per_src=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65118 dup_rows=23994 dup_rows_remote=21072 dedup_ratio=0.244080 router_tokens_per_expert=[746, 771, 772, 802, 740, 776, 809, 789, 762, 738, 756, 798, 813, 732, 776, 764, 717, 730, 723, 767, 789, 789, 741, 814, 762, 795, 755, 761, 725, 770, 783, 744, 779, 730, 732, 766, 693, 768, 825, 764, 764, 711, 749, 739, 792, 768, 792, 797, 783, 746, 827, 769, 739, 755, 773, 803, 776, 782, 802, 803, 782, 769, 778, 787] payload_rows_per_dst=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] dup_rows_per_dst=[2994, 3000, 3038, 2952, 2922, 3008, 3040, 3040] combine_rows_per_src=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344936 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267567976 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534791016 weight_movement_bytes=69206016 total_nvlink_bytes=603997032 weight_to_token_ratio=0.129408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65264 dup_rows=23806 dup_rows_remote=20722 dedup_ratio=0.242167 router_tokens_per_expert=[803, 752, 721, 708, 774, 770, 813, 818, 740, 792, 782, 712, 761, 757, 747, 773, 771, 793, 759, 804, 770, 788, 757, 750, 770, 815, 735, 728, 731, 777, 796, 773, 742, 775, 812, 824, 700, 752, 787, 779, 788, 788, 759, 743, 823, 744, 762, 791, 739, 757, 729, 778, 828, 764, 737, 791, 747, 762, 799, 760, 773, 717, 782, 780] payload_rows_per_dst=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] dup_rows_per_dst=[3084, 2962, 2996, 2966, 3172, 3084, 2718, 2824] combine_rows_per_src=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64856 dup_rows=24104 dup_rows_remote=21042 dedup_ratio=0.245199 router_tokens_per_expert=[732, 758, 741, 790, 722, 769, 819, 780, 785, 726, 759, 813, 719, 769, 782, 786, 788, 803, 816, 759, 768, 723, 789, 724, 794, 753, 781, 787, 758, 787, 742, 794, 778, 763, 678, 780, 798, 784, 763, 763, 847, 731, 746, 795, 745, 764, 738, 757, 753, 781, 744, 740, 789, 805, 766, 725, 747, 791, 772, 785, 816, 770, 790, 732] payload_rows_per_dst=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] dup_rows_per_dst=[2684, 2844, 3104, 2988, 3140, 3148, 3134, 3062] combine_rows_per_src=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64854 dup_rows=24178 dup_rows_remote=21384 dedup_ratio=0.245951 router_tokens_per_expert=[770, 783, 814, 790, 767, 767, 777, 757, 798, 747, 768, 720, 824, 740, 791, 772, 734, 782, 762, 733, 766, 761, 782, 760, 771, 758, 746, 791, 800, 765, 803, 774, 768, 836, 748, 787, 790, 743, 759, 778, 735, 743, 769, 749, 760, 752, 784, 741, 738, 767, 786, 778, 752, 766, 759, 729, 745, 749, 766, 776, 775, 784, 769, 798] payload_rows_per_dst=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] dup_rows_per_dst=[3116, 3086, 2942, 3122, 3120, 2794, 2968, 3030] combine_rows_per_src=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344760 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267068088 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791416 weight_movement_bytes=34603008 total_nvlink_bytes=568394424 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20878 dedup_ratio=0.241821 router_tokens_per_expert=[753, 756, 738, 810, 798, 783, 770, 792, 772, 787, 715, 779, 770, 780, 752, 797, 765, 753, 760, 788, 722, 788, 791, 773, 750, 798, 765, 800, 782, 766, 810, 751, 753, 757, 805, 772, 736, 744, 768, 753, 776, 710, 757, 783, 781, 776, 777, 758, 743, 783, 755, 780, 767, 740, 802, 761, 767, 771, 736, 782, 751, 727, 772, 795] payload_rows_per_dst=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] dup_rows_per_dst=[3120, 3064, 2972, 2998, 2988, 2836, 2894, 2900] combine_rows_per_src=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65042 dup_rows=23904 dup_rows_remote=20894 dedup_ratio=0.243164 router_tokens_per_expert=[749, 800, 760, 725, 771, 747, 787, 757, 822, 794, 810, 745, 789, 723, 771, 705, 747, 769, 745, 798, 705, 774, 842, 804, 775, 767, 800, 761, 750, 742, 808, 777, 777, 742, 824, 800, 758, 767, 776, 783, 805, 792, 775, 735, 736, 765, 711, 775, 728, 784, 762, 729, 780, 797, 729, 755, 807, 752, 790, 723, 777, 739, 760, 800] payload_rows_per_dst=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] dup_rows_per_dst=[2954, 2962, 3010, 2998, 3104, 2968, 2938, 2970] combine_rows_per_src=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65058 dup_rows=24040 dup_rows_remote=21026 dedup_ratio=0.244548 router_tokens_per_expert=[761, 771, 778, 749, 736, 715, 791, 824, 795, 813, 781, 748, 767, 774, 797, 781, 759, 790, 772, 816, 724, 813, 742, 740, 799, 770, 783, 701, 760, 735, 744, 818, 774, 754, 788, 833, 772, 725, 733, 714, 777, 749, 792, 762, 779, 789, 749, 775, 777, 738, 752, 738, 715, 743, 792, 783, 774, 808, 780, 788, 755, 773, 753, 771] payload_rows_per_dst=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] dup_rows_per_dst=[2948, 3142, 2956, 3014, 2950, 3092, 2862, 3076] combine_rows_per_src=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=343944 combine_payload_bytes=267321344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267665288 ingress_bytes=336527360 link_bytes=336527360 token_bytes=534986632 weight_movement_bytes=69206016 total_nvlink_bytes=604192648 weight_to_token_ratio=0.129360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266325363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344952 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986936 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628920 weight_movement_bytes=34603008 total_nvlink_bytes=566231928 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343592 combine_payload_bytes=265650176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265993768 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531643944 weight_movement_bytes=69206016 total_nvlink_bytes=600849960 weight_to_token_ratio=0.130174 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344168 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255912 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167656 weight_movement_bytes=34603008 total_nvlink_bytes=568770664 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343744 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266755776 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533167808 weight_movement_bytes=34603008 total_nvlink_bytes=567770816 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64788 dup_rows=24072 dup_rows_remote=20908 dedup_ratio=0.244873 router_tokens_per_expert=[778, 754, 748, 774, 735, 766, 783, 786, 779, 792, 778, 768, 841, 789, 739, 818, 785, 746, 770, 742, 778, 741, 789, 765, 771, 752, 797, 800, 796, 789, 746, 773, 705, 720, 754, 796, 814, 751, 735, 806, 742, 758, 794, 747, 785, 750, 725, 779, 748, 785, 771, 780, 746, 759, 794, 794, 815, 765, 716, 760, 712, 779, 729, 770] payload_rows_per_dst=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] dup_rows_per_dst=[2930, 3164, 2976, 3100, 2906, 2956, 3066, 2974] combine_rows_per_src=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64786 dup_rows=24192 dup_rows_remote=21142 dedup_ratio=0.246094 router_tokens_per_expert=[787, 721, 749, 813, 768, 759, 773, 811, 770, 767, 733, 746, 772, 761, 738, 784, 736, 748, 725, 797, 753, 744, 799, 769, 745, 761, 778, 811, 765, 753, 767, 763, 769, 780, 785, 785, 770, 780, 740, 779, 743, 756, 766, 808, 790, 746, 792, 791, 772, 767, 764, 800, 797, 740, 764, 763, 758, 791, 738, 730, 771, 765, 807, 779] payload_rows_per_dst=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] dup_rows_per_dst=[3108, 2926, 2964, 3020, 3050, 3094, 3008, 3022] combine_rows_per_src=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=342784 combine_payload_bytes=265371648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265714432 ingress_bytes=369180672 link_bytes=369180672 token_bytes=531086080 weight_movement_bytes=103809024 total_nvlink_bytes=634895104 weight_to_token_ratio=0.195466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389197147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65140 dup_rows=23840 dup_rows_remote=20886 dedup_ratio=0.242513 router_tokens_per_expert=[816, 755, 771, 742, 794, 757, 769, 823, 793, 728, 794, 780, 773, 769, 826, 733, 717, 732, 748, 810, 730, 755, 780, 745, 759, 797, 752, 772, 811, 757, 763, 792, 776, 775, 739, 833, 786, 784, 795, 759, 790, 744, 765, 793, 715, 779, 765, 788, 741, 756, 784, 717, 753, 753, 744, 723, 755, 728, 767, 789, 778, 787, 776, 772] payload_rows_per_dst=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] dup_rows_per_dst=[3000, 3102, 2850, 3026, 3090, 2954, 2850, 2968] combine_rows_per_src=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65074 dup_rows=23936 dup_rows_remote=20992 dedup_ratio=0.243490 router_tokens_per_expert=[717, 752, 783, 786, 747, 750, 768, 773, 734, 774, 759, 791, 781, 744, 782, 760, 745, 754, 775, 783, 747, 804, 834, 751, 749, 761, 840, 792, 773, 759, 772, 723, 767, 781, 773, 753, 776, 778, 801, 776, 771, 775, 811, 765, 726, 698, 747, 813, 717, 787, 793, 801, 767, 762, 809, 756, 745, 727, 775, 760, 753, 742, 832, 752] payload_rows_per_dst=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] dup_rows_per_dst=[2944, 2964, 3256, 3052, 2914, 2904, 2980, 2922] combine_rows_per_src=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64668 dup_rows=24278 dup_rows_remote=21032 dedup_ratio=0.246969 router_tokens_per_expert=[770, 761, 763, 738, 748, 741, 755, 865, 730, 786, 753, 799, 744, 704, 834, 717, 763, 746, 751, 711, 744, 796, 773, 787, 765, 735, 746, 781, 755, 770, 785, 785, 779, 815, 758, 760, 805, 769, 780, 739, 791, 749, 777, 790, 794, 756, 817, 791, 749, 760, 768, 782, 810, 769, 792, 805, 739, 739, 739, 725, 785, 799, 746, 774] payload_rows_per_dst=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] dup_rows_per_dst=[2860, 2910, 3210, 3058, 2766, 3068, 3160, 3246] combine_rows_per_src=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343712 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265707168 ingress_bytes=334569472 link_bytes=334569472 token_bytes=531070624 weight_movement_bytes=69206016 total_nvlink_bytes=600276640 weight_to_token_ratio=0.130314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258957988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65036 dup_rows=23942 dup_rows_remote=20924 dedup_ratio=0.243551 router_tokens_per_expert=[760, 742, 766, 792, 762, 756, 759, 736, 746, 761, 794, 753, 784, 757, 775, 792, 744, 808, 728, 778, 776, 782, 781, 763, 778, 778, 780, 798, 784, 746, 777, 742, 755, 755, 734, 755, 758, 771, 803, 771, 797, 753, 777, 764, 743, 792, 785, 716, 752, 778, 772, 747, 803, 796, 772, 752, 792, 742, 817, 757, 786, 771, 741, 767] payload_rows_per_dst=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] dup_rows_per_dst=[2958, 3042, 2950, 2994, 2944, 3000, 3018, 3036] combine_rows_per_src=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64826 dup_rows=24192 dup_rows_remote=21102 dedup_ratio=0.246094 router_tokens_per_expert=[800, 781, 833, 708, 764, 782, 825, 753, 761, 747, 788, 726, 782, 758, 716, 753, 762, 750, 773, 783, 810, 734, 759, 817, 773, 774, 771, 774, 756, 730, 761, 789, 716, 737, 762, 797, 722, 794, 795, 789, 754, 785, 732, 790, 799, 813, 767, 795, 754, 738, 793, 793, 800, 789, 740, 771, 719, 792, 771, 771, 753, 769, 754, 705] payload_rows_per_dst=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] dup_rows_per_dst=[3098, 2888, 3090, 3068, 3008, 3112, 3038, 2890] combine_rows_per_src=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65424 dup_rows=23744 dup_rows_remote=20838 dedup_ratio=0.241536 router_tokens_per_expert=[802, 784, 786, 792, 754, 769, 784, 749, 774, 788, 762, 766, 747, 793, 777, 732, 762, 773, 779, 803, 765, 794, 799, 758, 780, 730, 753, 766, 765, 751, 728, 748, 752, 804, 762, 731, 781, 759, 807, 787, 721, 805, 781, 752, 829, 757, 806, 774, 746, 783, 784, 754, 767, 748, 705, 759, 802, 762, 808, 742, 724, 772, 747, 728] payload_rows_per_dst=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] dup_rows_per_dst=[3046, 3028, 2998, 2906, 2892, 3006, 2916, 2952] combine_rows_per_src=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344104 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267157544 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533970984 weight_movement_bytes=103809024 total_nvlink_bytes=637780008 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=342800 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265222928 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530103056 weight_movement_bytes=34603008 total_nvlink_bytes=564706064 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344264 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887368 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430472 weight_movement_bytes=34603008 total_nvlink_bytes=568033480 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343840 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731296 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118752 weight_movement_bytes=34603008 total_nvlink_bytes=567721760 weight_to_token_ratio=0.064907 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343712 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871008 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398304 weight_movement_bytes=34603008 total_nvlink_bytes=566001312 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64930 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[773, 752, 744, 734, 691, 734, 751, 747, 737, 794, 791, 814, 780, 797, 780, 783, 827, 791, 719, 747, 766, 778, 794, 770, 762, 809, 773, 775, 751, 768, 729, 740, 755, 765, 757, 743, 746, 783, 784, 784, 781, 793, 766, 731, 779, 799, 827, 765, 743, 771, 787, 769, 771, 747, 822, 776, 749, 725, 759, 801, 757, 737, 807, 772] payload_rows_per_dst=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] dup_rows_per_dst=[2770, 3118, 2998, 2942, 2922, 3130, 3030, 3030] combine_rows_per_src=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=345048 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321752 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536298456 weight_movement_bytes=34603008 total_nvlink_bytes=570901464 weight_to_token_ratio=0.064522 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64948 dup_rows=23910 dup_rows_remote=20960 dedup_ratio=0.243225 router_tokens_per_expert=[773, 734, 783, 801, 752, 786, 777, 805, 790, 783, 763, 756, 778, 790, 815, 777, 735, 782, 769, 735, 733, 829, 774, 809, 774, 729, 781, 772, 795, 749, 768, 781, 783, 749, 733, 780, 809, 786, 764, 794, 745, 777, 777, 766, 709, 721, 755, 737, 729, 757, 756, 741, 761, 749, 751, 766, 758, 753, 749, 806, 780, 766, 787, 780] payload_rows_per_dst=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] dup_rows_per_dst=[3116, 3124, 3064, 3060, 2950, 2746, 2836, 3014] combine_rows_per_src=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343008 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266296288 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532249568 weight_movement_bytes=103809024 total_nvlink_bytes=636058592 weight_to_token_ratio=0.195038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64992 dup_rows=23892 dup_rows_remote=20880 dedup_ratio=0.243042 router_tokens_per_expert=[782, 793, 787, 774, 790, 763, 792, 814, 772, 755, 781, 749, 767, 793, 775, 761, 737, 759, 772, 742, 752, 764, 715, 726, 748, 787, 763, 745, 765, 708, 782, 714, 704, 742, 798, 860, 803, 732, 750, 754, 737, 783, 775, 794, 772, 792, 799, 764, 772, 765, 802, 729, 757, 808, 790, 779, 808, 712, 770, 736, 779, 751, 791, 817] payload_rows_per_dst=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] dup_rows_per_dst=[3056, 3034, 2774, 2938, 2910, 3012, 3110, 3058] combine_rows_per_src=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65108 dup_rows=23824 dup_rows_remote=20786 dedup_ratio=0.242350 router_tokens_per_expert=[746, 762, 777, 738, 759, 768, 748, 773, 783, 768, 738, 767, 760, 716, 755, 800, 773, 755, 762, 786, 812, 722, 750, 753, 731, 740, 790, 811, 783, 763, 799, 801, 733, 747, 791, 736, 799, 787, 761, 795, 756, 823, 762, 803, 759, 759, 785, 761, 800, 742, 780, 741, 746, 803, 763, 726, 774, 776, 831, 810, 762, 759, 759, 734] payload_rows_per_dst=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] dup_rows_per_dst=[2874, 2726, 2972, 3206, 3062, 2964, 2982, 3038] combine_rows_per_src=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343632 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266370640 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532397648 weight_movement_bytes=34603008 total_nvlink_bytes=567000656 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64784 dup_rows=24066 dup_rows_remote=21044 dedup_ratio=0.244812 router_tokens_per_expert=[801, 806, 764, 803, 739, 760, 793, 758, 771, 764, 745, 769, 793, 779, 787, 791, 746, 719, 759, 777, 756, 798, 777, 815, 781, 791, 758, 734, 779, 755, 729, 771, 801, 740, 728, 753, 765, 793, 757, 737, 774, 799, 755, 793, 721, 789, 757, 786, 801, 793, 806, 761, 718, 725, 810, 745, 770, 759, 741, 750, 769, 727, 780, 781] payload_rows_per_dst=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] dup_rows_per_dst=[3022, 3048, 2980, 2984, 2860, 3102, 3122, 2948] combine_rows_per_src=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73960 payload_rows_remote=64618 dup_rows=24344 dup_rows_remote=21280 dedup_ratio=0.247640 router_tokens_per_expert=[748, 764, 737, 756, 769, 795, 767, 778, 763, 747, 784, 762, 766, 788, 770, 793, 763, 812, 771, 765, 723, 774, 726, 807, 747, 773, 750, 758, 774, 804, 716, 742, 745, 794, 750, 731, 766, 751, 784, 801, 734, 798, 784, 743, 765, 790, 760, 725, 737, 753, 788, 757, 803, 764, 814, 787, 757, 792, 782, 769, 797, 781, 806, 752] payload_rows_per_dst=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] dup_rows_per_dst=[2962, 3074, 3106, 2988, 3030, 2972, 3064, 3148] combine_rows_per_src=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65064 dup_rows=23920 dup_rows_remote=20956 dedup_ratio=0.243327 router_tokens_per_expert=[829, 767, 762, 732, 798, 730, 758, 763, 781, 774, 753, 768, 814, 767, 725, 782, 744, 766, 806, 779, 759, 757, 763, 768, 743, 749, 732, 785, 769, 735, 774, 767, 755, 745, 768, 751, 795, 756, 805, 773, 775, 774, 725, 798, 796, 806, 739, 776, 759, 771, 736, 755, 781, 785, 799, 762, 771, 764, 718, 782, 763, 822, 762, 786] payload_rows_per_dst=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] dup_rows_per_dst=[3034, 2978, 2964, 2932, 3010, 3002, 2980, 3020] combine_rows_per_src=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343488 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550720 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532757952 weight_movement_bytes=69206016 total_nvlink_bytes=601963968 weight_to_token_ratio=0.129901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64978 dup_rows=24020 dup_rows_remote=21082 dedup_ratio=0.244344 router_tokens_per_expert=[781, 739, 756, 767, 801, 806, 763, 805, 831, 794, 718, 764, 775, 748, 792, 764, 783, 762, 781, 748, 780, 728, 800, 791, 797, 761, 789, 714, 753, 740, 800, 768, 759, 762, 768, 747, 768, 746, 730, 801, 762, 753, 775, 755, 784, 749, 760, 755, 774, 738, 795, 796, 767, 756, 761, 797, 745, 767, 764, 721, 818, 770, 766, 744] payload_rows_per_dst=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] dup_rows_per_dst=[3048, 3072, 3072, 2938, 2934, 2990, 3012, 2954] combine_rows_per_src=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343576 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267025944 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708312 weight_movement_bytes=103809024 total_nvlink_bytes=637517336 weight_to_token_ratio=0.194505 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343312 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265698576 ingress_bytes=403767296 link_bytes=403767296 token_bytes=531053840 weight_movement_bytes=138412032 total_nvlink_bytes=669465872 weight_to_token_ratio=0.260637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519343829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343592 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018920 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694248 weight_movement_bytes=69206016 total_nvlink_bytes=598900264 weight_to_token_ratio=0.130653 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344080 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846224 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348368 weight_movement_bytes=34603008 total_nvlink_bytes=567951376 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65068 dup_rows=23956 dup_rows_remote=20912 dedup_ratio=0.243693 router_tokens_per_expert=[765, 751, 804, 808, 766, 768, 715, 748, 738, 793, 787, 813, 782, 802, 711, 736, 796, 781, 773, 752, 766, 707, 794, 762, 777, 818, 785, 773, 805, 751, 799, 792, 779, 768, 754, 788, 768, 813, 819, 746, 731, 765, 770, 792, 751, 752, 760, 761, 765, 783, 771, 718, 732, 769, 742, 778, 772, 775, 792, 799, 723, 735, 722, 741] payload_rows_per_dst=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] dup_rows_per_dst=[2946, 3044, 2962, 3136, 3114, 2950, 2858, 2946] combine_rows_per_src=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344240 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494128 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644016 weight_movement_bytes=34603008 total_nvlink_bytes=567247024 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64860 dup_rows=24144 dup_rows_remote=21174 dedup_ratio=0.245605 router_tokens_per_expert=[768, 775, 803, 741, 766, 784, 780, 753, 782, 728, 714, 778, 769, 793, 818, 785, 750, 764, 797, 818, 769, 816, 796, 719, 735, 755, 756, 784, 770, 784, 772, 763, 766, 750, 753, 757, 820, 761, 748, 780, 748, 753, 787, 766, 741, 800, 733, 795, 741, 748, 778, 722, 798, 751, 795, 759, 730, 743, 822, 753, 788, 738, 752, 791] payload_rows_per_dst=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] dup_rows_per_dst=[3114, 3062, 3106, 2968, 2970, 2986, 2940, 2998] combine_rows_per_src=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343920 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862448 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533380976 weight_movement_bytes=103809024 total_nvlink_bytes=637190000 weight_to_token_ratio=0.194625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=65036 dup_rows=24150 dup_rows_remote=21152 dedup_ratio=0.245667 router_tokens_per_expert=[742, 775, 743, 788, 775, 780, 704, 780, 738, 785, 770, 809, 808, 754, 753, 805, 772, 768, 794, 789, 771, 738, 714, 791, 783, 755, 735, 770, 774, 746, 764, 749, 785, 766, 782, 767, 791, 772, 755, 772, 786, 750, 751, 746, 738, 769, 760, 758, 781, 812, 733, 785, 804, 790, 794, 792, 781, 754, 725, 795, 760, 753, 722, 801] payload_rows_per_dst=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] dup_rows_per_dst=[2958, 3172, 3062, 2950, 2954, 2998, 3092, 2964] combine_rows_per_src=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64452 dup_rows=24316 dup_rows_remote=21104 dedup_ratio=0.247355 router_tokens_per_expert=[784, 775, 755, 761, 805, 734, 765, 786, 760, 749, 754, 782, 835, 775, 795, 744, 802, 752, 725, 758, 733, 762, 757, 798, 798, 753, 760, 793, 762, 724, 767, 784, 797, 729, 792, 733, 725, 782, 739, 757, 800, 769, 748, 713, 762, 745, 751, 808, 800, 796, 784, 766, 794, 788, 757, 775, 740, 764, 757, 777, 811, 788, 752, 766] payload_rows_per_dst=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] dup_rows_per_dst=[3076, 2888, 2972, 2982, 2798, 3152, 3236, 3212] combine_rows_per_src=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=344136 combine_payload_bytes=265666560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266010696 ingress_bytes=369475584 link_bytes=369475584 token_bytes=531677256 weight_movement_bytes=103809024 total_nvlink_bytes=635486280 weight_to_token_ratio=0.195248 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390306877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=64886 dup_rows=23870 dup_rows_remote=20734 dedup_ratio=0.242818 router_tokens_per_expert=[815, 772, 776, 805, 778, 778, 765, 809, 803, 789, 791, 744, 749, 761, 718, 789, 701, 758, 780, 747, 776, 733, 772, 781, 768, 756, 736, 749, 714, 738, 816, 754, 794, 749, 809, 769, 753, 777, 748, 735, 728, 798, 779, 785, 772, 720, 782, 780, 764, 807, 742, 803, 775, 772, 813, 770, 746, 750, 730, 759, 765, 793, 748, 816] payload_rows_per_dst=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] dup_rows_per_dst=[3136, 2958, 2990, 2824, 2994, 2936, 3132, 2900] combine_rows_per_src=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65016 dup_rows=23880 dup_rows_remote=20960 dedup_ratio=0.242920 router_tokens_per_expert=[786, 729, 759, 784, 767, 731, 771, 745, 771, 738, 796, 748, 822, 792, 710, 725, 792, 769, 782, 740, 759, 730, 779, 813, 785, 825, 784, 777, 838, 748, 736, 772, 749, 771, 814, 744, 770, 695, 740, 773, 803, 796, 832, 788, 746, 757, 774, 807, 778, 754, 767, 782, 736, 740, 781, 806, 780, 758, 735, 725, 799, 731, 751, 767] payload_rows_per_dst=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] dup_rows_per_dst=[2960, 2992, 2920, 3040, 3010, 3056, 3018, 2884] combine_rows_per_src=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65090 dup_rows=24062 dup_rows_remote=21126 dedup_ratio=0.244771 router_tokens_per_expert=[796, 786, 755, 784, 748, 737, 785, 778, 755, 765, 750, 808, 749, 749, 750, 728, 786, 796, 740, 787, 772, 768, 781, 751, 724, 767, 767, 754, 742, 815, 764, 766, 817, 788, 760, 747, 798, 780, 794, 750, 790, 791, 785, 788, 783, 809, 726, 795, 814, 744, 724, 726, 765, 744, 759, 768, 777, 810, 744, 801, 741, 757, 754, 720] payload_rows_per_dst=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] dup_rows_per_dst=[3050, 2932, 3032, 3056, 3040, 3056, 2936, 2960] combine_rows_per_src=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344752 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732208 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119664 weight_movement_bytes=34603008 total_nvlink_bytes=567722672 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65042 dup_rows=23998 dup_rows_remote=21056 dedup_ratio=0.244120 router_tokens_per_expert=[763, 780, 789, 797, 779, 768, 787, 733, 790, 773, 790, 773, 757, 794, 778, 785, 783, 759, 759, 784, 751, 790, 760, 745, 776, 763, 759, 787, 756, 746, 801, 715, 771, 749, 761, 756, 731, 807, 794, 745, 748, 736, 767, 818, 805, 721, 723, 770, 744, 758, 778, 771, 775, 713, 828, 813, 779, 775, 750, 779, 765, 755, 770, 727] payload_rows_per_dst=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] dup_rows_per_dst=[3006, 3042, 2996, 2942, 2930, 2964, 3112, 3006] combine_rows_per_src=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=342224 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264337616 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528333008 weight_movement_bytes=34603008 total_nvlink_bytes=562936016 weight_to_token_ratio=0.065495 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343904 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649440 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532954976 weight_movement_bytes=69206016 total_nvlink_bytes=602160992 weight_to_token_ratio=0.129853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65098 dup_rows=23968 dup_rows_remote=21028 dedup_ratio=0.243815 router_tokens_per_expert=[743, 737, 728, 784, 824, 795, 833, 787, 752, 761, 754, 758, 706, 772, 798, 788, 757, 737, 781, 811, 766, 773, 783, 749, 804, 767, 790, 787, 780, 755, 761, 782, 743, 745, 748, 775, 761, 769, 748, 756, 765, 770, 762, 744, 773, 769, 763, 734, 769, 768, 772, 779, 792, 761, 761, 756, 735, 785, 798, 791, 759, 734, 817, 747] payload_rows_per_dst=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] dup_rows_per_dst=[3064, 2940, 2992, 3090, 2974, 2952, 3002, 2954] combine_rows_per_src=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266953504 ingress_bytes=370417664 link_bytes=370417664 token_bytes=533562144 weight_movement_bytes=103809024 total_nvlink_bytes=637371168 weight_to_token_ratio=0.194558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393851849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=342480 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266115536 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531888592 weight_movement_bytes=69206016 total_nvlink_bytes=601094608 weight_to_token_ratio=0.130114 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344392 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756424 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168456 weight_movement_bytes=34603008 total_nvlink_bytes=567771464 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64844 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[765, 772, 760, 748, 771, 766, 760, 815, 798, 751, 761, 758, 768, 813, 747, 745, 791, 787, 821, 825, 757, 770, 767, 745, 781, 783, 753, 772, 729, 734, 751, 755, 773, 794, 795, 757, 761, 740, 824, 784, 785, 786, 776, 731, 735, 740, 786, 758, 764, 760, 769, 731, 737, 762, 801, 742, 799, 734, 751, 845, 754, 729, 768, 762] payload_rows_per_dst=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] dup_rows_per_dst=[3078, 2990, 3092, 2890, 3036, 2964, 2974, 3016] combine_rows_per_src=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65180 dup_rows=23876 dup_rows_remote=21110 dedup_ratio=0.242879 router_tokens_per_expert=[743, 822, 762, 783, 752, 723, 764, 763, 814, 792, 769, 772, 760, 770, 805, 812, 783, 728, 777, 730, 796, 740, 796, 781, 713, 758, 792, 760, 760, 768, 750, 733, 769, 765, 788, 793, 749, 750, 808, 785, 702, 768, 793, 733, 752, 754, 755, 750, 768, 792, 747, 773, 765, 844, 759, 801, 769, 774, 779, 754, 747, 769, 749, 777] payload_rows_per_dst=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] dup_rows_per_dst=[2920, 3164, 2924, 2882, 3082, 2766, 3100, 3038] combine_rows_per_src=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344504 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985912 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627320 weight_movement_bytes=34603008 total_nvlink_bytes=568230328 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343392 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265944416 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531545440 weight_movement_bytes=34603008 total_nvlink_bytes=566148448 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64834 dup_rows=23926 dup_rows_remote=20780 dedup_ratio=0.243388 router_tokens_per_expert=[802, 810, 754, 725, 780, 814, 793, 746, 767, 746, 760, 794, 794, 710, 714, 736, 816, 771, 759, 757, 758, 740, 744, 813, 755, 779, 760, 792, 778, 768, 750, 796, 774, 742, 812, 793, 770, 727, 784, 769, 778, 793, 762, 755, 730, 737, 775, 729, 798, 784, 757, 773, 795, 753, 768, 797, 739, 763, 749, 777, 774, 772, 801, 741] payload_rows_per_dst=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] dup_rows_per_dst=[3012, 2840, 2946, 3074, 3022, 3012, 2874, 3146] combine_rows_per_src=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65042 dup_rows=23974 dup_rows_remote=21010 dedup_ratio=0.243876 router_tokens_per_expert=[743, 780, 806, 739, 740, 794, 756, 761, 796, 754, 821, 766, 810, 731, 780, 760, 742, 789, 808, 771, 706, 734, 779, 797, 771, 766, 776, 778, 764, 764, 795, 731, 727, 731, 763, 766, 815, 760, 793, 773, 778, 820, 833, 734, 767, 839, 767, 751, 756, 725, 740, 732, 762, 764, 761, 774, 783, 784, 712, 767, 777, 768, 734, 788] payload_rows_per_dst=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] dup_rows_per_dst=[2920, 3112, 2964, 2978, 3004, 3128, 2910, 2958] combine_rows_per_src=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64880 dup_rows=24074 dup_rows_remote=20946 dedup_ratio=0.244893 router_tokens_per_expert=[762, 765, 735, 805, 733, 781, 744, 764, 765, 715, 765, 771, 769, 781, 724, 753, 792, 781, 762, 785, 790, 758, 758, 799, 771, 771, 738, 772, 767, 753, 749, 757, 775, 756, 758, 775, 761, 780, 801, 787, 739, 788, 746, 821, 770, 794, 757, 739, 777, 782, 792, 782, 782, 758, 785, 781, 779, 802, 750, 779, 745, 792, 749, 735] payload_rows_per_dst=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] dup_rows_per_dst=[2990, 2862, 3022, 2910, 3068, 3032, 3128, 3062] combine_rows_per_src=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=345160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267322440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534299720 weight_movement_bytes=34603008 total_nvlink_bytes=568902728 weight_to_token_ratio=0.064763 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[799, 725, 753, 770, 776, 763, 735, 744, 747, 742, 819, 757, 771, 818, 792, 771, 745, 768, 767, 773, 760, 759, 761, 762, 748, 788, 798, 734, 787, 783, 812, 720, 771, 777, 820, 742, 736, 785, 806, 751, 770, 749, 795, 817, 779, 751, 739, 751, 723, 772, 812, 784, 773, 759, 739, 752, 757, 785, 766, 790, 746, 785, 761, 762] payload_rows_per_dst=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] dup_rows_per_dst=[2988, 3102, 3102, 2990, 3036, 2890, 2944, 2968] combine_rows_per_src=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64904 dup_rows=24046 dup_rows_remote=21108 dedup_ratio=0.244609 router_tokens_per_expert=[809, 787, 755, 768, 762, 741, 746, 732, 738, 772, 748, 811, 765, 789, 785, 742, 753, 689, 779, 749, 804, 775, 808, 739, 752, 753, 781, 711, 786, 787, 814, 762, 803, 715, 778, 784, 760, 777, 744, 778, 800, 753, 760, 832, 768, 744, 814, 762, 742, 796, 741, 784, 755, 717, 820, 789, 799, 758, 772, 792, 821, 734, 718, 750] payload_rows_per_dst=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] dup_rows_per_dst=[2986, 2970, 3000, 2938, 3042, 3116, 2988, 3006] combine_rows_per_src=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342456 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902520 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462584 weight_movement_bytes=34603008 total_nvlink_bytes=566065592 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344208 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756240 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168272 weight_movement_bytes=34603008 total_nvlink_bytes=567771280 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65060 dup_rows=24132 dup_rows_remote=21138 dedup_ratio=0.245483 router_tokens_per_expert=[775, 782, 786, 781, 765, 819, 818, 735, 745, 747, 737, 743, 791, 733, 803, 754, 825, 753, 738, 779, 787, 741, 781, 757, 751, 732, 749, 771, 751, 759, 759, 782, 763, 724, 800, 804, 819, 764, 783, 737, 775, 755, 736, 764, 769, 763, 760, 739, 765, 801, 766, 822, 751, 738, 777, 798, 722, 786, 765, 784, 759, 787, 798, 749] payload_rows_per_dst=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] dup_rows_per_dst=[3042, 2994, 3004, 2944, 3076, 2946, 3138, 2988] combine_rows_per_src=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343304 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266091784 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840264 weight_movement_bytes=103809024 total_nvlink_bytes=635649288 weight_to_token_ratio=0.195188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20874 dedup_ratio=0.242716 router_tokens_per_expert=[799, 767, 759, 759, 799, 809, 802, 780, 789, 771, 759, 751, 730, 755, 730, 821, 768, 724, 789, 749, 735, 779, 792, 787, 781, 744, 781, 748, 756, 764, 766, 762, 757, 736, 809, 747, 776, 789, 751, 782, 775, 799, 794, 741, 748, 723, 751, 786, 745, 742, 779, 835, 759, 764, 782, 787, 750, 736, 777, 763, 797, 758, 768, 741] payload_rows_per_dst=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] dup_rows_per_dst=[3158, 2922, 2996, 2920, 2986, 2936, 3014, 2928] combine_rows_per_src=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344648 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117128 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889608 weight_movement_bytes=34603008 total_nvlink_bytes=568492616 weight_to_token_ratio=0.064813 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344048 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190832 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037616 weight_movement_bytes=34603008 total_nvlink_bytes=566640624 weight_to_token_ratio=0.065039 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344792 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266830552 ingress_bytes=439500800 link_bytes=439500800 token_bytes=533316312 weight_movement_bytes=173015040 total_nvlink_bytes=706331352 weight_to_token_ratio=0.324414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653806128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64996 dup_rows=24092 dup_rows_remote=21086 dedup_ratio=0.245076 router_tokens_per_expert=[789, 739, 741, 781, 774, 749, 779, 758, 750, 768, 759, 782, 762, 765, 764, 768, 766, 742, 803, 763, 759, 747, 811, 747, 787, 774, 762, 757, 805, 739, 772, 757, 787, 726, 807, 778, 741, 787, 750, 730, 748, 746, 790, 777, 746, 764, 794, 746, 728, 814, 801, 785, 701, 810, 814, 822, 727, 779, 778, 785, 803, 790, 720, 759] payload_rows_per_dst=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] dup_rows_per_dst=[2984, 3022, 3094, 3016, 2996, 3006, 3080, 2894] combine_rows_per_src=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344040 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141096 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938152 weight_movement_bytes=34603008 total_nvlink_bytes=568541160 weight_to_token_ratio=0.064807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64428 dup_rows=24288 dup_rows_remote=21000 dedup_ratio=0.247070 router_tokens_per_expert=[786, 731, 739, 793, 739, 762, 734, 766, 796, 783, 777, 778, 780, 803, 741, 786, 711, 762, 759, 768, 765, 774, 756, 741, 744, 735, 776, 791, 819, 755, 725, 723, 743, 748, 810, 769, 786, 743, 750, 778, 714, 808, 785, 759, 789, 811, 760, 777, 769, 798, 754, 776, 786, 784, 738, 815, 812, 740, 796, 792, 827, 734, 753, 750] payload_rows_per_dst=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] dup_rows_per_dst=[2962, 3002, 2866, 3054, 3032, 2894, 3190, 3288] combine_rows_per_src=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65008 dup_rows=23870 dup_rows_remote=20772 dedup_ratio=0.242818 router_tokens_per_expert=[791, 780, 718, 789, 766, 757, 805, 760, 767, 780, 791, 716, 765, 714, 742, 743, 755, 815, 787, 794, 772, 782, 769, 788, 776, 810, 762, 752, 817, 759, 809, 776, 799, 777, 773, 772, 753, 775, 750, 770, 775, 794, 768, 717, 739, 830, 775, 734, 758, 735, 757, 743, 773, 747, 786, 753, 717, 811, 757, 772, 789, 768, 729, 749] payload_rows_per_dst=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] dup_rows_per_dst=[3044, 2918, 3098, 3106, 2924, 3024, 2832, 2924] combine_rows_per_src=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65210 dup_rows=23774 dup_rows_remote=20784 dedup_ratio=0.241842 router_tokens_per_expert=[778, 798, 743, 753, 792, 730, 702, 776, 820, 767, 757, 792, 778, 807, 769, 772, 775, 728, 782, 738, 751, 761, 711, 768, 807, 723, 744, 760, 819, 771, 773, 763, 811, 775, 765, 750, 798, 777, 795, 796, 765, 780, 748, 813, 751, 778, 765, 772, 762, 750, 764, 792, 803, 806, 718, 760, 795, 742, 743, 747, 719, 743, 771, 790] payload_rows_per_dst=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] dup_rows_per_dst=[2914, 3058, 2856, 2986, 3070, 3016, 2990, 2884] combine_rows_per_src=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344328 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266567944 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532791560 weight_movement_bytes=69206016 total_nvlink_bytes=601997576 weight_to_token_ratio=0.129893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=341712 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264238800 ingress_bytes=298500096 link_bytes=298500096 token_bytes=528135888 weight_movement_bytes=34603008 total_nvlink_bytes=562738896 weight_to_token_ratio=0.065519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123231830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=65000 dup_rows=24088 dup_rows_remote=21102 dedup_ratio=0.245036 router_tokens_per_expert=[777, 780, 771, 709, 796, 739, 756, 736, 800, 767, 769, 766, 762, 771, 761, 799, 770, 733, 748, 756, 746, 748, 731, 785, 807, 742, 712, 789, 783, 781, 801, 748, 817, 778, 760, 796, 807, 782, 795, 712, 807, 740, 774, 781, 728, 723, 772, 759, 742, 821, 765, 806, 779, 756, 809, 765, 764, 744, 821, 779, 782, 751, 749, 749] payload_rows_per_dst=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] dup_rows_per_dst=[2986, 3038, 3022, 3110, 3032, 2982, 2930, 2988] combine_rows_per_src=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64950 dup_rows=23962 dup_rows_remote=20954 dedup_ratio=0.243754 router_tokens_per_expert=[710, 785, 765, 691, 755, 747, 753, 760, 752, 829, 750, 783, 767, 768, 752, 765, 783, 763, 788, 731, 778, 751, 747, 795, 748, 795, 757, 816, 812, 759, 762, 751, 742, 750, 779, 732, 766, 774, 781, 737, 790, 803, 775, 779, 845, 781, 794, 815, 761, 810, 826, 736, 765, 758, 757, 733, 792, 756, 758, 731, 802, 758, 754, 744] payload_rows_per_dst=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] dup_rows_per_dst=[2804, 3030, 3086, 3008, 2930, 3216, 2998, 2890] combine_rows_per_src=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343120 combine_payload_bytes=266272768 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266615888 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532888656 weight_movement_bytes=103809024 total_nvlink_bytes=636697680 weight_to_token_ratio=0.194804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65108 dup_rows=23980 dup_rows_remote=20918 dedup_ratio=0.243937 router_tokens_per_expert=[771, 716, 753, 782, 767, 764, 802, 789, 772, 811, 770, 751, 731, 770, 770, 764, 796, 757, 844, 744, 781, 765, 782, 783, 756, 818, 738, 743, 731, 784, 796, 756, 807, 767, 767, 816, 800, 760, 807, 760, 808, 789, 734, 754, 748, 771, 735, 752, 756, 777, 756, 747, 755, 781, 742, 784, 762, 761, 730, 749, 738, 747, 735, 800] payload_rows_per_dst=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] dup_rows_per_dst=[3016, 3062, 3100, 2988, 3084, 2940, 2946, 2844] combine_rows_per_src=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343976 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444136 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544296 weight_movement_bytes=69206016 total_nvlink_bytes=603750312 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64932 dup_rows=23966 dup_rows_remote=20936 dedup_ratio=0.243795 router_tokens_per_expert=[750, 762, 793, 758, 739, 758, 754, 791, 795, 787, 799, 765, 775, 745, 808, 764, 812, 756, 765, 761, 800, 733, 760, 788, 760, 820, 756, 727, 777, 783, 756, 777, 734, 783, 774, 776, 768, 763, 808, 812, 786, 745, 768, 746, 759, 727, 750, 765, 780, 781, 738, 805, 765, 741, 763, 754, 746, 778, 772, 743, 782, 757, 746, 763] payload_rows_per_dst=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] dup_rows_per_dst=[2954, 3106, 3078, 2922, 3030, 2918, 3004, 2954] combine_rows_per_src=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344408 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584408 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824408 weight_movement_bytes=34603008 total_nvlink_bytes=567427416 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343616 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378816 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414016 weight_movement_bytes=34603008 total_nvlink_bytes=567017024 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344104 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267026472 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533708840 weight_movement_bytes=69206016 total_nvlink_bytes=602914856 weight_to_token_ratio=0.129670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343472 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304944 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266416 weight_movement_bytes=34603008 total_nvlink_bytes=566869424 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65044 dup_rows=24008 dup_rows_remote=21090 dedup_ratio=0.244222 router_tokens_per_expert=[823, 763, 813, 837, 767, 789, 739, 767, 739, 789, 776, 798, 746, 792, 803, 828, 746, 799, 762, 785, 751, 794, 740, 751, 784, 747, 777, 726, 766, 775, 750, 770, 701, 782, 775, 732, 743, 700, 754, 735, 734, 758, 796, 734, 774, 744, 750, 795, 758, 716, 783, 782, 772, 769, 759, 753, 785, 770, 755, 808, 742, 795, 800, 806] payload_rows_per_dst=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] dup_rows_per_dst=[3066, 3118, 2982, 2998, 2856, 2918, 2960, 3110] combine_rows_per_src=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64700 dup_rows=23992 dup_rows_remote=20674 dedup_ratio=0.244059 router_tokens_per_expert=[791, 745, 796, 781, 766, 779, 753, 723, 753, 745, 798, 763, 739, 785, 786, 784, 806, 776, 798, 804, 748, 782, 761, 782, 722, 792, 743, 782, 796, 777, 743, 765, 803, 729, 744, 811, 769, 755, 783, 735, 721, 790, 733, 817, 742, 752, 781, 759, 779, 803, 758, 776, 755, 744, 790, 771, 750, 771, 721, 765, 743, 776, 783, 779] payload_rows_per_dst=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] dup_rows_per_dst=[2870, 2866, 2768, 2914, 3020, 3136, 3100, 3318] combine_rows_per_src=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65218 dup_rows=23730 dup_rows_remote=20678 dedup_ratio=0.241394 router_tokens_per_expert=[702, 791, 731, 801, 737, 754, 759, 788, 732, 776, 718, 791, 853, 741, 769, 773, 846, 767, 781, 765, 802, 727, 748, 768, 790, 753, 746, 753, 766, 751, 741, 765, 791, 826, 758, 754, 744, 774, 755, 772, 741, 729, 784, 772, 791, 809, 723, 787, 808, 796, 749, 777, 779, 798, 729, 802, 759, 791, 721, 753, 731, 786, 818, 760] payload_rows_per_dst=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] dup_rows_per_dst=[2892, 2988, 3052, 2886, 2968, 2908, 3070, 2966] combine_rows_per_src=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21164 dedup_ratio=0.245117 router_tokens_per_expert=[717, 756, 748, 785, 782, 816, 803, 707, 791, 743, 756, 762, 754, 762, 758, 754, 786, 812, 773, 827, 771, 798, 752, 737, 775, 764, 781, 783, 809, 781, 780, 731, 745, 756, 746, 746, 754, 796, 785, 745, 763, 773, 786, 782, 716, 774, 769, 827, 770, 799, 764, 795, 756, 771, 770, 732, 728, 804, 755, 714, 784, 776, 746, 771] payload_rows_per_dst=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] dup_rows_per_dst=[3116, 2928, 3102, 3072, 2966, 3050, 2932, 2930] combine_rows_per_src=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64724 dup_rows=24078 dup_rows_remote=20948 dedup_ratio=0.244934 router_tokens_per_expert=[814, 767, 759, 763, 784, 761, 812, 822, 775, 791, 763, 767, 787, 801, 758, 784, 751, 761, 761, 740, 748, 723, 738, 788, 748, 793, 721, 770, 810, 762, 739, 760, 789, 790, 777, 754, 745, 737, 783, 772, 777, 749, 782, 756, 747, 764, 761, 734, 815, 754, 798, 731, 814, 784, 773, 785, 746, 761, 754, 776, 727, 817, 725, 754] payload_rows_per_dst=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] dup_rows_per_dst=[3130, 3122, 2902, 3026, 3022, 2942, 2964, 2970] combine_rows_per_src=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344536 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764760 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184984 weight_movement_bytes=34603008 total_nvlink_bytes=567787992 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=341496 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265352696 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530363896 weight_movement_bytes=34603008 total_nvlink_bytes=564966904 weight_to_token_ratio=0.065244 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=343584 combine_payload_bytes=267132928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476512 ingress_bytes=301735936 link_bytes=301735936 token_bytes=534609440 weight_movement_bytes=34603008 total_nvlink_bytes=569212448 weight_to_token_ratio=0.064726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135408035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64846 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[800, 742, 753, 739, 786, 760, 763, 771, 782, 775, 750, 807, 755, 773, 781, 753, 743, 776, 736, 773, 768, 783, 790, 774, 745, 758, 787, 772, 808, 790, 807, 756, 786, 748, 727, 757, 786, 738, 756, 748, 767, 736, 761, 782, 748, 811, 777, 749, 789, 797, 781, 776, 772, 747, 772, 793, 728, 776, 780, 739, 768, 794, 792, 715] payload_rows_per_dst=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] dup_rows_per_dst=[3026, 3034, 2994, 3028, 2940, 3018, 3078, 2922] combine_rows_per_src=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64978 dup_rows=23892 dup_rows_remote=20866 dedup_ratio=0.243042 router_tokens_per_expert=[790, 760, 753, 756, 825, 807, 737, 762, 732, 795, 753, 759, 843, 792, 791, 765, 800, 719, 765, 750, 771, 746, 795, 753, 820, 751, 803, 755, 719, 757, 780, 738, 753, 735, 759, 749, 782, 789, 763, 773, 784, 759, 781, 784, 742, 754, 767, 756, 765, 699, 776, 767, 797, 787, 771, 774, 797, 777, 786, 716, 716, 744, 825, 783] payload_rows_per_dst=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] dup_rows_per_dst=[3048, 3026, 2912, 2944, 2896, 3012, 3000, 3054] combine_rows_per_src=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64782 dup_rows=24112 dup_rows_remote=21120 dedup_ratio=0.245280 router_tokens_per_expert=[807, 769, 835, 774, 748, 791, 779, 797, 739, 703, 757, 764, 830, 798, 769, 743, 790, 769, 795, 779, 791, 737, 713, 753, 758, 779, 741, 712, 796, 790, 813, 785, 778, 778, 760, 765, 762, 768, 785, 805, 766, 754, 703, 795, 775, 748, 772, 754, 763, 771, 770, 766, 749, 766, 720, 775, 762, 747, 736, 748, 769, 822, 765, 751] payload_rows_per_dst=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] dup_rows_per_dst=[3176, 3040, 3018, 3016, 2992, 2906, 2952, 3012] combine_rows_per_src=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343960 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871256 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398552 weight_movement_bytes=34603008 total_nvlink_bytes=566001560 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=342688 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265452192 ingress_bytes=334315520 link_bytes=334315520 token_bytes=530561696 weight_movement_bytes=69206016 total_nvlink_bytes=599767712 weight_to_token_ratio=0.130439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258002387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343432 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952648 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531561864 weight_movement_bytes=103809024 total_nvlink_bytes=635370888 weight_to_token_ratio=0.195291 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343376 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493264 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643152 weight_movement_bytes=34603008 total_nvlink_bytes=567246160 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343608 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265690680 ingress_bytes=403759104 link_bytes=403759104 token_bytes=531037752 weight_movement_bytes=138412032 total_nvlink_bytes=669449784 weight_to_token_ratio=0.260644 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519313003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64984 dup_rows=24116 dup_rows_remote=21160 dedup_ratio=0.245321 router_tokens_per_expert=[775, 741, 790, 778, 780, 727, 791, 755, 742, 708, 772, 801, 777, 752, 756, 810, 794, 784, 769, 784, 768, 758, 757, 747, 761, 762, 796, 825, 733, 740, 730, 806, 707, 772, 799, 774, 819, 806, 773, 752, 768, 800, 742, 747, 737, 781, 734, 771, 807, 737, 756, 749, 790, 740, 788, 771, 756, 770, 758, 778, 762, 768, 796, 775] payload_rows_per_dst=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] dup_rows_per_dst=[2926, 3024, 3064, 3100, 3062, 2956, 2960, 3024] combine_rows_per_src=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64904 dup_rows=24098 dup_rows_remote=20918 dedup_ratio=0.245138 router_tokens_per_expert=[789, 784, 753, 806, 795, 786, 740, 756, 773, 793, 784, 755, 767, 763, 752, 750, 816, 760, 787, 769, 773, 765, 729, 768, 728, 745, 732, 795, 762, 751, 738, 749, 782, 820, 771, 799, 779, 795, 783, 725, 739, 732, 767, 762, 758, 772, 759, 813, 717, 762, 753, 768, 767, 797, 782, 751, 756, 820, 768, 801, 778, 767, 748, 748] payload_rows_per_dst=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] dup_rows_per_dst=[2872, 3120, 3024, 2784, 3002, 3082, 3034, 3180] combine_rows_per_src=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=65076 dup_rows=24146 dup_rows_remote=21116 dedup_ratio=0.245626 router_tokens_per_expert=[751, 768, 794, 767, 757, 765, 760, 793, 777, 791, 732, 742, 763, 808, 771, 760, 775, 768, 767, 783, 723, 699, 780, 761, 837, 763, 775, 778, 794, 784, 805, 779, 755, 774, 770, 781, 758, 812, 799, 769, 749, 765, 735, 789, 800, 798, 726, 716, 764, 796, 746, 711, 702, 825, 783, 737, 749, 715, 774, 775, 754, 814, 747, 794] payload_rows_per_dst=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] dup_rows_per_dst=[3040, 2914, 3030, 3132, 3126, 2922, 2964, 3018] combine_rows_per_src=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344576 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266519040 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693504 weight_movement_bytes=34603008 total_nvlink_bytes=567296512 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64982 dup_rows=24026 dup_rows_remote=21050 dedup_ratio=0.244405 router_tokens_per_expert=[738, 776, 739, 765, 771, 777, 799, 747, 762, 751, 744, 767, 726, 774, 753, 765, 760, 757, 782, 749, 744, 785, 720, 777, 766, 788, 770, 776, 783, 808, 771, 769, 813, 792, 807, 754, 753, 780, 768, 756, 740, 754, 755, 782, 835, 765, 779, 762, 769, 742, 778, 791, 776, 757, 763, 760, 759, 753, 743, 691, 801, 799, 765, 851] payload_rows_per_dst=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] dup_rows_per_dst=[3004, 2924, 2972, 3100, 3036, 3050, 2976, 2964] combine_rows_per_src=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64662 dup_rows=24098 dup_rows_remote=21042 dedup_ratio=0.245138 router_tokens_per_expert=[762, 745, 778, 785, 770, 826, 784, 817, 752, 788, 769, 753, 787, 769, 789, 801, 769, 796, 737, 754, 760, 725, 745, 771, 757, 847, 794, 757, 801, 692, 736, 808, 764, 769, 755, 801, 785, 752, 747, 743, 757, 788, 784, 704, 754, 800, 719, 746, 782, 767, 767, 792, 753, 755, 829, 740, 762, 769, 740, 788, 778, 731, 741, 766] payload_rows_per_dst=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] dup_rows_per_dst=[3056, 3054, 3240, 3098, 2976, 3022, 2672, 2980] combine_rows_per_src=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343288 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190072 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036856 weight_movement_bytes=34603008 total_nvlink_bytes=566639864 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344768 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266896064 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447360 weight_movement_bytes=34603008 total_nvlink_bytes=568050368 weight_to_token_ratio=0.064867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65056 dup_rows=23938 dup_rows_remote=20976 dedup_ratio=0.243510 router_tokens_per_expert=[781, 770, 757, 770, 728, 763, 852, 805, 790, 746, 756, 725, 740, 733, 813, 738, 739, 824, 798, 740, 736, 742, 745, 825, 756, 749, 791, 808, 785, 772, 750, 725, 741, 785, 777, 755, 747, 797, 724, 758, 789, 737, 779, 754, 786, 766, 776, 773, 744, 764, 772, 788, 765, 750, 761, 793, 794, 747, 770, 785, 748, 773, 787, 815] payload_rows_per_dst=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] dup_rows_per_dst=[3036, 2978, 2988, 2962, 2914, 2964, 2984, 3112] combine_rows_per_src=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64790 dup_rows=24112 dup_rows_remote=21032 dedup_ratio=0.245280 router_tokens_per_expert=[779, 805, 742, 768, 740, 733, 788, 760, 705, 762, 811, 831, 790, 776, 763, 803, 770, 769, 750, 808, 769, 754, 806, 715, 772, 768, 769, 779, 811, 736, 821, 792, 757, 837, 805, 816, 789, 739, 763, 774, 745, 760, 764, 749, 730, 799, 749, 723, 759, 731, 789, 779, 798, 740, 759, 752, 778, 721, 764, 759, 732, 732, 728, 787] payload_rows_per_dst=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] dup_rows_per_dst=[2968, 3080, 2988, 3150, 3142, 2932, 2966, 2886] combine_rows_per_src=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20878 dedup_ratio=0.243693 router_tokens_per_expert=[772, 795, 785, 760, 760, 797, 748, 742, 768, 775, 754, 786, 771, 770, 758, 765, 768, 766, 723, 739, 791, 767, 798, 779, 778, 768, 781, 819, 761, 751, 783, 768, 812, 755, 742, 789, 788, 780, 741, 813, 747, 754, 753, 798, 768, 752, 740, 749, 758, 815, 737, 778, 777, 765, 828, 752, 746, 745, 742, 747, 763, 774, 783, 715] payload_rows_per_dst=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] dup_rows_per_dst=[3014, 2918, 2996, 3122, 3078, 2900, 3020, 2908] combine_rows_per_src=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344128 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510400 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676672 weight_movement_bytes=34603008 total_nvlink_bytes=567279680 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342816 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198368 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053920 weight_movement_bytes=34603008 total_nvlink_bytes=564656928 weight_to_token_ratio=0.065282 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344128 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813504 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282880 weight_movement_bytes=34603008 total_nvlink_bytes=567885888 weight_to_token_ratio=0.064887 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343288 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723128 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531102968 weight_movement_bytes=69206016 total_nvlink_bytes=600308984 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343456 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526112 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532708768 weight_movement_bytes=69206016 total_nvlink_bytes=601914784 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65228 dup_rows=23872 dup_rows_remote=20952 dedup_ratio=0.242839 router_tokens_per_expert=[775, 767, 778, 805, 732, 764, 729, 749, 742, 776, 773, 771, 743, 723, 822, 775, 772, 784, 811, 745, 777, 804, 769, 766, 780, 752, 729, 761, 750, 812, 738, 782, 789, 784, 788, 740, 743, 778, 767, 774, 737, 762, 745, 748, 763, 816, 740, 751, 729, 745, 784, 765, 822, 771, 802, 806, 767, 763, 811, 740, 758, 763, 769, 776] payload_rows_per_dst=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] dup_rows_per_dst=[2992, 2956, 3088, 2946, 2930, 2920, 3000, 3040] combine_rows_per_src=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65012 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[781, 742, 731, 784, 759, 773, 786, 759, 758, 772, 729, 746, 793, 761, 734, 761, 754, 741, 741, 759, 802, 729, 808, 780, 804, 780, 776, 751, 726, 800, 807, 757, 805, 743, 777, 766, 791, 788, 757, 750, 784, 804, 784, 748, 778, 740, 768, 824, 722, 785, 799, 765, 817, 741, 815, 717, 741, 787, 799, 753, 776, 756, 771, 717] payload_rows_per_dst=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] dup_rows_per_dst=[3078, 2810, 3034, 3046, 3056, 3112, 3028, 2934] combine_rows_per_src=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64254 dup_rows=24306 dup_rows_remote=20772 dedup_ratio=0.247253 router_tokens_per_expert=[766, 780, 784, 779, 782, 766, 781, 757, 744, 730, 775, 833, 778, 771, 743, 794, 789, 763, 762, 789, 701, 789, 791, 771, 736, 745, 725, 747, 750, 729, 780, 749, 793, 814, 722, 737, 772, 785, 771, 738, 736, 744, 793, 765, 809, 774, 799, 775, 777, 776, 797, 792, 797, 780, 776, 781, 779, 742, 746, 782, 703, 748, 784, 786] payload_rows_per_dst=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] dup_rows_per_dst=[3010, 2930, 3010, 2610, 2790, 3268, 3154, 3534] combine_rows_per_src=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65154 dup_rows=23834 dup_rows_remote=20944 dedup_ratio=0.242452 router_tokens_per_expert=[790, 786, 766, 777, 813, 786, 749, 788, 774, 740, 762, 779, 753, 811, 781, 783, 722, 760, 783, 737, 789, 720, 797, 765, 777, 787, 766, 768, 766, 789, 789, 795, 755, 801, 801, 743, 720, 747, 784, 787, 783, 765, 710, 751, 763, 769, 809, 790, 715, 800, 766, 750, 762, 777, 789, 744, 753, 763, 758, 762, 761, 758, 745, 723] payload_rows_per_dst=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] dup_rows_per_dst=[3054, 2938, 2930, 3082, 3090, 2954, 2890, 2896] combine_rows_per_src=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344720 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518608 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692496 weight_movement_bytes=34603008 total_nvlink_bytes=569295504 weight_to_token_ratio=0.064716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344304 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266633456 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532922608 weight_movement_bytes=69206016 total_nvlink_bytes=602128624 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64748 dup_rows=24168 dup_rows_remote=20982 dedup_ratio=0.245850 router_tokens_per_expert=[743, 785, 775, 757, 788, 819, 791, 774, 800, 739, 807, 812, 787, 787, 738, 743, 730, 781, 719, 797, 763, 771, 765, 747, 741, 798, 768, 782, 742, 748, 752, 750, 744, 808, 783, 786, 761, 804, 762, 800, 810, 746, 786, 778, 751, 798, 794, 774, 692, 767, 790, 714, 767, 799, 758, 726, 716, 754, 789, 792, 798, 733, 722, 751] payload_rows_per_dst=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] dup_rows_per_dst=[3186, 3122, 2878, 2902, 3160, 3128, 2892, 2900] combine_rows_per_src=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=340104 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263524488 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526708872 weight_movement_bytes=34603008 total_nvlink_bytes=561311880 weight_to_token_ratio=0.065697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65142 dup_rows=23906 dup_rows_remote=20816 dedup_ratio=0.243184 router_tokens_per_expert=[805, 787, 750, 749, 794, 787, 765, 743, 776, 733, 787, 737, 794, 779, 779, 788, 744, 774, 765, 707, 725, 714, 765, 790, 769, 743, 771, 806, 703, 714, 785, 766, 753, 742, 823, 788, 788, 800, 807, 768, 780, 787, 772, 713, 784, 796, 748, 804, 765, 785, 766, 760, 740, 795, 772, 753, 762, 752, 774, 802, 802, 801, 775, 701] payload_rows_per_dst=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] dup_rows_per_dst=[3004, 3090, 2872, 2912, 3142, 2984, 2902, 3000] combine_rows_per_src=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64978 dup_rows=23962 dup_rows_remote=20968 dedup_ratio=0.243754 router_tokens_per_expert=[753, 749, 766, 783, 762, 747, 736, 758, 744, 763, 793, 765, 797, 746, 784, 786, 761, 751, 777, 782, 752, 777, 797, 752, 806, 845, 725, 777, 749, 761, 742, 774, 745, 799, 735, 733, 761, 730, 792, 757, 735, 749, 754, 756, 766, 843, 789, 753, 785, 789, 729, 750, 750, 765, 802, 774, 772, 796, 767, 801, 736, 807, 801, 771] payload_rows_per_dst=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] dup_rows_per_dst=[2870, 3030, 2966, 2994, 2908, 3050, 3046, 3098] combine_rows_per_src=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65026 dup_rows=24030 dup_rows_remote=20966 dedup_ratio=0.244446 router_tokens_per_expert=[776, 772, 764, 742, 743, 744, 815, 842, 735, 753, 764, 764, 778, 716, 768, 782, 776, 744, 796, 784, 764, 742, 823, 770, 773, 735, 705, 784, 743, 719, 767, 791, 790, 749, 743, 803, 754, 752, 774, 791, 790, 754, 797, 787, 735, 753, 801, 709, 799, 791, 778, 787, 764, 799, 773, 743, 747, 744, 797, 785, 796, 748, 754, 791] payload_rows_per_dst=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] dup_rows_per_dst=[3064, 2868, 3028, 2924, 3064, 2994, 3068, 3020] combine_rows_per_src=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344392 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267215176 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085960 weight_movement_bytes=34603008 total_nvlink_bytes=568688968 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=342920 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265550728 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758536 weight_movement_bytes=69206016 total_nvlink_bytes=599964552 weight_to_token_ratio=0.130391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343832 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165464 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987096 weight_movement_bytes=34603008 total_nvlink_bytes=568590104 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343784 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493672 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643560 weight_movement_bytes=69206016 total_nvlink_bytes=601849576 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343968 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266690464 ingress_bytes=404758528 link_bytes=404758528 token_bytes=533036960 weight_movement_bytes=138412032 total_nvlink_bytes=671448992 weight_to_token_ratio=0.259667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523073755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65050 dup_rows=23842 dup_rows_remote=20908 dedup_ratio=0.242533 router_tokens_per_expert=[778, 744, 739, 784, 744, 763, 750, 806, 742, 766, 767, 706, 702, 781, 775, 815, 771, 778, 803, 747, 781, 790, 762, 739, 776, 796, 769, 755, 765, 792, 803, 749, 773, 760, 783, 805, 732, 787, 805, 727, 789, 780, 781, 794, 721, 766, 757, 785, 746, 762, 812, 819, 734, 768, 744, 777, 737, 781, 751, 775, 743, 756, 777, 787] payload_rows_per_dst=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] dup_rows_per_dst=[2948, 2930, 3044, 3020, 3044, 2934, 2998, 2924] combine_rows_per_src=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65076 dup_rows=24072 dup_rows_remote=21168 dedup_ratio=0.244873 router_tokens_per_expert=[798, 732, 752, 778, 808, 778, 760, 727, 742, 842, 782, 782, 745, 781, 830, 767, 788, 762, 760, 732, 730, 735, 781, 742, 779, 743, 778, 775, 815, 770, 796, 817, 731, 724, 773, 732, 788, 732, 759, 806, 726, 762, 767, 806, 808, 733, 746, 784, 788, 768, 747, 754, 754, 781, 726, 722, 748, 824, 763, 773, 807, 735, 824, 754] payload_rows_per_dst=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] dup_rows_per_dst=[2964, 3154, 2904, 3192, 2868, 2968, 2900, 3122] combine_rows_per_src=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65096 dup_rows=23774 dup_rows_remote=20696 dedup_ratio=0.241842 router_tokens_per_expert=[758, 774, 782, 772, 708, 782, 754, 796, 797, 788, 792, 731, 765, 726, 800, 788, 753, 763, 761, 724, 746, 770, 793, 789, 805, 756, 759, 690, 789, 814, 762, 758, 792, 801, 788, 747, 772, 766, 722, 804, 766, 726, 763, 795, 727, 753, 742, 767, 782, 793, 790, 753, 792, 767, 722, 788, 793, 767, 786, 779, 810, 789, 721, 744] payload_rows_per_dst=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] dup_rows_per_dst=[2980, 2978, 2866, 2992, 2838, 3006, 3036, 3078] combine_rows_per_src=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343832 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788632 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233432 weight_movement_bytes=34603008 total_nvlink_bytes=567836440 weight_to_token_ratio=0.064893 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64996 dup_rows=24008 dup_rows_remote=20856 dedup_ratio=0.244222 router_tokens_per_expert=[747, 779, 772, 753, 753, 752, 787, 720, 805, 764, 756, 791, 804, 731, 735, 744, 738, 795, 777, 786, 762, 826, 779, 754, 732, 807, 759, 737, 771, 800, 736, 758, 785, 746, 770, 752, 725, 778, 758, 773, 830, 723, 751, 744, 820, 771, 757, 738, 765, 739, 807, 780, 728, 826, 786, 795, 770, 745, 817, 743, 766, 797, 781, 776] payload_rows_per_dst=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] dup_rows_per_dst=[2936, 2998, 3044, 2950, 2874, 3004, 3152, 3050] combine_rows_per_src=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344976 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266896272 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533447568 weight_movement_bytes=69206016 total_nvlink_bytes=602653584 weight_to_token_ratio=0.129733 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20786 dedup_ratio=0.243022 router_tokens_per_expert=[763, 819, 735, 716, 785, 820, 772, 787, 770, 783, 751, 696, 791, 796, 786, 793, 710, 732, 814, 763, 797, 777, 783, 785, 731, 752, 729, 746, 781, 757, 758, 782, 793, 793, 810, 780, 769, 713, 836, 756, 769, 800, 786, 763, 798, 763, 789, 749, 821, 720, 776, 754, 776, 761, 750, 763, 797, 754, 778, 713, 749, 737, 716, 760] payload_rows_per_dst=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] dup_rows_per_dst=[3104, 3054, 2992, 3078, 2834, 3080, 2892, 2856] combine_rows_per_src=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65068 dup_rows=24032 dup_rows_remote=21042 dedup_ratio=0.244466 router_tokens_per_expert=[756, 719, 780, 741, 780, 836, 760, 745, 805, 729, 742, 778, 780, 754, 778, 731, 772, 782, 740, 757, 787, 756, 769, 766, 751, 809, 808, 747, 779, 762, 772, 747, 745, 716, 765, 823, 786, 735, 790, 811, 731, 786, 837, 804, 740, 804, 796, 735, 731, 758, 743, 801, 794, 814, 783, 733, 717, 755, 743, 747, 764, 797, 776, 774] payload_rows_per_dst=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] dup_rows_per_dst=[3030, 2990, 3084, 3002, 3004, 3074, 2940, 2908] combine_rows_per_src=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343168 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266976384 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533609600 weight_movement_bytes=34603008 total_nvlink_bytes=568212608 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64822 dup_rows=24092 dup_rows_remote=20986 dedup_ratio=0.245076 router_tokens_per_expert=[784, 755, 745, 746, 785, 803, 784, 746, 765, 767, 823, 794, 804, 719, 780, 757, 730, 809, 790, 779, 757, 794, 766, 805, 730, 741, 763, 833, 789, 745, 786, 738, 805, 732, 778, 810, 718, 776, 832, 797, 782, 727, 810, 781, 696, 785, 766, 755, 723, 719, 804, 722, 692, 752, 797, 781, 781, 759, 748, 764, 761, 746, 810, 731] payload_rows_per_dst=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] dup_rows_per_dst=[3062, 3050, 3100, 2994, 3106, 3028, 2804, 2948] combine_rows_per_src=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64824 dup_rows=24226 dup_rows_remote=21260 dedup_ratio=0.246440 router_tokens_per_expert=[755, 820, 750, 762, 775, 813, 766, 753, 729, 734, 777, 760, 802, 817, 770, 763, 797, 748, 729, 765, 787, 795, 737, 843, 786, 795, 748, 737, 728, 813, 762, 741, 771, 747, 813, 745, 770, 776, 723, 799, 746, 766, 740, 799, 724, 798, 763, 773, 769, 816, 787, 730, 787, 748, 759, 763, 747, 787, 750, 762, 775, 737, 782, 743] payload_rows_per_dst=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] dup_rows_per_dst=[3084, 3014, 3076, 2966, 3022, 2958, 3118, 2988] combine_rows_per_src=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343408 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567024 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532790640 weight_movement_bytes=103809024 total_nvlink_bytes=636599664 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343640 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267091544 ingress_bytes=405159936 link_bytes=405159936 token_bytes=533839448 weight_movement_bytes=138412032 total_nvlink_bytes=672251480 weight_to_token_ratio=0.259277 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524584221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344440 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266862968 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381496 weight_movement_bytes=69206016 total_nvlink_bytes=602587512 weight_to_token_ratio=0.129750 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343232 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265854144 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531365056 weight_movement_bytes=69206016 total_nvlink_bytes=600571072 weight_to_token_ratio=0.130242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65094 dup_rows=23966 dup_rows_remote=21108 dedup_ratio=0.243795 router_tokens_per_expert=[802, 795, 735, 760, 778, 782, 765, 793, 764, 787, 731, 768, 767, 794, 785, 790, 773, 759, 757, 790, 725, 743, 785, 789, 727, 742, 795, 785, 770, 760, 803, 771, 748, 742, 826, 750, 766, 783, 738, 759, 784, 728, 751, 797, 715, 806, 742, 728, 742, 748, 799, 745, 783, 764, 770, 794, 786, 780, 778, 778, 780, 762, 777, 733] payload_rows_per_dst=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] dup_rows_per_dst=[2960, 3074, 3014, 2976, 3058, 2858, 3060, 2966] combine_rows_per_src=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=344336 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265863440 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531382544 weight_movement_bytes=34603008 total_nvlink_bytes=565985552 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74608 payload_rows_remote=65496 dup_rows=23696 dup_rows_remote=20906 dedup_ratio=0.241048 router_tokens_per_expert=[746, 778, 748, 776, 774, 812, 805, 757, 775, 791, 719, 811, 757, 758, 790, 718, 782, 771, 716, 757, 714, 767, 700, 744, 778, 767, 726, 758, 776, 785, 788, 757, 806, 786, 784, 775, 755, 764, 743, 792, 768, 794, 782, 765, 772, 776, 721, 773, 825, 797, 765, 757, 800, 773, 817, 758, 810, 727, 776, 743, 787, 752, 752, 756] payload_rows_per_dst=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] dup_rows_per_dst=[3078, 2930, 2790, 2898, 3022, 2964, 3118, 2896] combine_rows_per_src=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64916 dup_rows=23948 dup_rows_remote=20768 dedup_ratio=0.243612 router_tokens_per_expert=[787, 794, 817, 745, 769, 824, 817, 716, 765, 744, 746, 787, 775, 724, 773, 763, 784, 768, 730, 792, 762, 771, 796, 784, 773, 700, 764, 755, 740, 764, 726, 799, 768, 763, 782, 784, 740, 780, 765, 761, 761, 764, 777, 778, 761, 767, 754, 793, 806, 761, 751, 756, 776, 746, 719, 771, 742, 763, 814, 738, 749, 805, 807, 796] payload_rows_per_dst=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] dup_rows_per_dst=[3006, 2880, 3036, 2792, 3030, 2814, 3210, 3180] combine_rows_per_src=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344808 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969832 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594856 weight_movement_bytes=69206016 total_nvlink_bytes=602800872 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65024 dup_rows=24098 dup_rows_remote=21126 dedup_ratio=0.245138 router_tokens_per_expert=[715, 784, 726, 747, 757, 839, 779, 763, 767, 744, 752, 756, 788, 697, 765, 780, 786, 778, 791, 760, 734, 758, 816, 779, 792, 735, 795, 727, 780, 748, 772, 775, 801, 797, 762, 788, 804, 759, 759, 818, 742, 796, 750, 789, 828, 760, 761, 826, 789, 784, 755, 791, 716, 745, 755, 742, 771, 768, 741, 755, 756, 751, 756, 752] payload_rows_per_dst=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] dup_rows_per_dst=[3042, 2874, 3090, 3020, 3136, 3072, 2972, 2892] combine_rows_per_src=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268271616 dispatch_meta_bytes=345608 combine_payload_bytes=268271616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268617224 ingress_bytes=302874624 link_bytes=302874624 token_bytes=536888840 weight_movement_bytes=34603008 total_nvlink_bytes=571491848 weight_to_token_ratio=0.064451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139692827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65110 dup_rows=24016 dup_rows_remote=21128 dedup_ratio=0.244303 router_tokens_per_expert=[767, 731, 753, 777, 750, 756, 749, 796, 748, 760, 777, 770, 803, 807, 782, 778, 737, 795, 786, 753, 776, 751, 758, 713, 783, 756, 773, 815, 746, 754, 786, 812, 729, 771, 733, 788, 758, 741, 772, 741, 745, 772, 772, 772, 766, 786, 742, 778, 770, 772, 779, 788, 763, 777, 747, 767, 780, 811, 774, 786, 766, 758, 762, 788] payload_rows_per_dst=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] dup_rows_per_dst=[2808, 3112, 2938, 3074, 2888, 3050, 3052, 3094] combine_rows_per_src=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65020 dup_rows=24004 dup_rows_remote=20846 dedup_ratio=0.244181 router_tokens_per_expert=[783, 764, 729, 764, 774, 817, 784, 804, 716, 767, 797, 790, 755, 750, 788, 732, 803, 757, 781, 784, 792, 775, 733, 788, 767, 732, 776, 783, 746, 800, 768, 756, 757, 744, 786, 784, 818, 756, 768, 783, 782, 782, 747, 835, 756, 750, 784, 852, 745, 759, 746, 765, 773, 784, 740, 747, 732, 721, 758, 732, 752, 750, 756, 753] payload_rows_per_dst=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] dup_rows_per_dst=[3158, 3018, 3052, 3030, 3014, 3102, 2820, 2810] combine_rows_per_src=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64898 dup_rows=24088 dup_rows_remote=20978 dedup_ratio=0.245036 router_tokens_per_expert=[744, 743, 764, 784, 759, 766, 802, 753, 815, 786, 756, 775, 749, 784, 801, 748, 766, 721, 798, 739, 760, 760, 810, 766, 792, 781, 727, 770, 753, 796, 819, 784, 757, 768, 790, 734, 779, 787, 777, 754, 717, 763, 745, 778, 736, 717, 772, 784, 792, 815, 759, 770, 781, 772, 778, 729, 766, 783, 810, 770, 731, 747, 771, 749] payload_rows_per_dst=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] dup_rows_per_dst=[2988, 3110, 3032, 2988, 3018, 2878, 3070, 3004] combine_rows_per_src=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=342736 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266238672 ingress_bytes=300498944 link_bytes=300498944 token_bytes=532134608 weight_movement_bytes=34603008 total_nvlink_bytes=566737616 weight_to_token_ratio=0.065027 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130753334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344600 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682904 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533021208 weight_movement_bytes=34603008 total_nvlink_bytes=567624216 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65178 dup_rows=23758 dup_rows_remote=20678 dedup_ratio=0.241679 router_tokens_per_expert=[746, 767, 737, 752, 725, 759, 766, 778, 716, 779, 745, 776, 733, 780, 801, 766, 771, 766, 796, 736, 753, 817, 789, 760, 777, 777, 774, 781, 760, 784, 800, 771, 788, 790, 728, 729, 781, 750, 790, 775, 787, 780, 793, 736, 733, 811, 762, 785, 781, 784, 756, 817, 814, 776, 812, 718, 744, 769, 751, 763, 753, 713, 775, 770] payload_rows_per_dst=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] dup_rows_per_dst=[2874, 2912, 2976, 3080, 2966, 2972, 3096, 2882] combine_rows_per_src=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344952 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267035512 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533726072 weight_movement_bytes=34603008 total_nvlink_bytes=568329080 weight_to_token_ratio=0.064833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343464 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266665384 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987304 weight_movement_bytes=69206016 total_nvlink_bytes=602193320 weight_to_token_ratio=0.129846 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65142 dup_rows=23806 dup_rows_remote=20694 dedup_ratio=0.242167 router_tokens_per_expert=[752, 778, 822, 781, 710, 793, 765, 812, 780, 771, 770, 793, 732, 771, 749, 711, 791, 795, 750, 761, 751, 789, 794, 765, 740, 735, 795, 723, 745, 718, 773, 732, 772, 791, 727, 761, 754, 798, 767, 756, 821, 770, 754, 792, 773, 753, 759, 812, 759, 821, 768, 771, 731, 760, 765, 804, 751, 771, 768, 756, 791, 769, 801, 759] payload_rows_per_dst=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] dup_rows_per_dst=[2980, 2896, 3050, 2802, 3004, 3112, 3020, 2942] combine_rows_per_src=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343504 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266165712 ingress_bytes=335028224 link_bytes=335028224 token_bytes=531987920 weight_movement_bytes=69206016 total_nvlink_bytes=601193936 weight_to_token_ratio=0.130089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260684235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65150 dup_rows=23944 dup_rows_remote=21030 dedup_ratio=0.243571 router_tokens_per_expert=[774, 712, 722, 753, 859, 784, 777, 764, 788, 751, 770, 714, 782, 770, 746, 771, 766, 768, 735, 753, 713, 763, 789, 775, 750, 768, 776, 788, 789, 740, 758, 782, 767, 763, 739, 782, 805, 776, 775, 767, 780, 748, 737, 753, 770, 755, 813, 825, 765, 745, 732, 780, 727, 760, 793, 775, 801, 746, 796, 771, 827, 814, 743, 772] payload_rows_per_dst=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] dup_rows_per_dst=[2972, 2970, 2914, 3008, 2998, 2956, 2940, 3186] combine_rows_per_src=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=343424 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267312512 ingress_bytes=336175104 link_bytes=336175104 token_bytes=534281600 weight_movement_bytes=69206016 total_nvlink_bytes=603487616 weight_to_token_ratio=0.129531 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264999852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343344 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267164976 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533986608 weight_movement_bytes=34603008 total_nvlink_bytes=568589616 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64750 dup_rows=24292 dup_rows_remote=21216 dedup_ratio=0.247111 router_tokens_per_expert=[726, 784, 761, 814, 740, 756, 769, 808, 774, 766, 782, 781, 721, 764, 802, 803, 743, 805, 774, 729, 722, 794, 783, 729, 774, 782, 785, 786, 816, 772, 772, 773, 801, 733, 760, 768, 762, 799, 749, 791, 717, 770, 777, 808, 781, 749, 772, 730, 712, 777, 772, 705, 750, 768, 785, 792, 782, 786, 755, 784, 738, 759, 749, 781] payload_rows_per_dst=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] dup_rows_per_dst=[3010, 2832, 3000, 3146, 3126, 3210, 2892, 3076] combine_rows_per_src=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344720 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267199120 ingress_bytes=336060416 link_bytes=336060416 token_bytes=534053520 weight_movement_bytes=69206016 total_nvlink_bytes=603259536 weight_to_token_ratio=0.129586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264568290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65186 dup_rows=23948 dup_rows_remote=21116 dedup_ratio=0.243612 router_tokens_per_expert=[780, 816, 812, 765, 787, 796, 781, 773, 771, 774, 774, 748, 759, 830, 747, 744, 740, 760, 781, 772, 752, 791, 788, 745, 764, 792, 782, 762, 745, 691, 817, 680, 761, 809, 789, 794, 763, 766, 771, 749, 739, 776, 789, 729, 774, 750, 758, 782, 747, 756, 728, 778, 730, 789, 756, 717, 769, 802, 772, 781, 765, 780, 807, 757] payload_rows_per_dst=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] dup_rows_per_dst=[3044, 3098, 2954, 2952, 3044, 2968, 2832, 3056] combine_rows_per_src=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65420 dup_rows=23764 dup_rows_remote=20844 dedup_ratio=0.241740 router_tokens_per_expert=[740, 730, 777, 764, 804, 770, 775, 769, 778, 795, 797, 747, 797, 770, 768, 788, 750, 762, 788, 797, 774, 755, 718, 766, 763, 758, 798, 749, 798, 759, 750, 750, 780, 730, 691, 770, 795, 734, 731, 789, 807, 792, 782, 758, 801, 784, 807, 828, 836, 750, 708, 715, 819, 728, 724, 742, 798, 728, 816, 742, 773, 743, 750, 797] payload_rows_per_dst=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] dup_rows_per_dst=[2944, 3076, 2904, 2982, 2920, 3138, 2880, 2920] combine_rows_per_src=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64946 dup_rows=24070 dup_rows_remote=21022 dedup_ratio=0.244853 router_tokens_per_expert=[771, 774, 770, 750, 796, 763, 749, 715, 777, 746, 741, 757, 779, 780, 768, 714, 793, 755, 792, 784, 798, 821, 767, 727, 766, 793, 795, 761, 777, 755, 783, 789, 765, 818, 770, 753, 800, 719, 800, 812, 745, 780, 756, 756, 762, 756, 761, 728, 781, 777, 797, 715, 796, 798, 774, 747, 744, 744, 768, 780, 760, 784, 774, 726] payload_rows_per_dst=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] dup_rows_per_dst=[3048, 3266, 2972, 3216, 2978, 2848, 2844, 2898] combine_rows_per_src=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343864 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265559864 ingress_bytes=369025024 link_bytes=369025024 token_bytes=530775864 weight_movement_bytes=103809024 total_nvlink_bytes=634584888 weight_to_token_ratio=0.195580 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388611456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64818 dup_rows=24066 dup_rows_remote=21058 dedup_ratio=0.244812 router_tokens_per_expert=[783, 790, 798, 812, 770, 747, 778, 759, 734, 768, 772, 783, 796, 804, 759, 798, 757, 753, 758, 759, 733, 779, 836, 747, 785, 746, 760, 793, 790, 758, 759, 773, 764, 697, 761, 757, 791, 784, 791, 730, 737, 776, 766, 749, 797, 755, 745, 806, 745, 747, 762, 750, 770, 791, 767, 758, 769, 748, 753, 755, 734, 836, 775, 749] payload_rows_per_dst=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] dup_rows_per_dst=[3092, 3008, 2960, 3070, 2974, 2996, 2958, 3008] combine_rows_per_src=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345208 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267347064 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348920 weight_movement_bytes=103809024 total_nvlink_bytes=638157944 weight_to_token_ratio=0.194272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65160 dup_rows=23820 dup_rows_remote=20966 dedup_ratio=0.242310 router_tokens_per_expert=[781, 804, 772, 718, 757, 723, 803, 749, 784, 757, 758, 758, 754, 743, 759, 745, 776, 748, 799, 769, 777, 742, 785, 781, 789, 749, 710, 788, 727, 753, 771, 802, 747, 782, 817, 809, 786, 757, 771, 741, 779, 755, 730, 748, 791, 732, 758, 740, 754, 826, 755, 773, 768, 797, 795, 729, 813, 814, 771, 782, 777, 833, 762, 729] payload_rows_per_dst=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] dup_rows_per_dst=[2990, 2964, 3032, 2854, 3018, 2856, 2984, 3122] combine_rows_per_src=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267960320 dispatch_meta_bytes=345056 combine_payload_bytes=267960320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268305376 ingress_bytes=302563328 link_bytes=302563328 token_bytes=536265696 weight_movement_bytes=34603008 total_nvlink_bytes=570868704 weight_to_token_ratio=0.064526 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138521445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64870 dup_rows=23952 dup_rows_remote=20964 dedup_ratio=0.243652 router_tokens_per_expert=[790, 747, 762, 775, 753, 766, 784, 784, 784, 739, 802, 765, 794, 766, 771, 753, 776, 749, 825, 818, 768, 749, 753, 743, 772, 776, 777, 720, 732, 742, 774, 756, 785, 815, 760, 756, 756, 804, 760, 726, 786, 788, 788, 802, 752, 763, 781, 775, 783, 751, 719, 768, 764, 768, 802, 734, 717, 785, 827, 742, 735, 782, 778, 735] payload_rows_per_dst=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] dup_rows_per_dst=[3016, 3062, 3044, 2932, 3070, 2988, 2914, 2926] combine_rows_per_src=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343872 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362688 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381504 weight_movement_bytes=34603008 total_nvlink_bytes=566984512 weight_to_token_ratio=0.064997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343504 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265838032 ingress_bytes=403906560 link_bytes=403906560 token_bytes=531332560 weight_movement_bytes=138412032 total_nvlink_bytes=669744592 weight_to_token_ratio=0.260500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519867868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65054 dup_rows=24048 dup_rows_remote=21086 dedup_ratio=0.244629 router_tokens_per_expert=[794, 809, 790, 751, 729, 775, 753, 788, 801, 750, 761, 773, 780, 729, 756, 721, 771, 768, 718, 745, 795, 785, 746, 754, 820, 799, 810, 783, 787, 739, 761, 762, 764, 727, 757, 728, 756, 765, 775, 739, 842, 761, 819, 783, 775, 785, 766, 713, 769, 771, 789, 783, 771, 771, 799, 783, 769, 746, 754, 737, 793, 754, 739, 766] payload_rows_per_dst=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] dup_rows_per_dst=[3018, 2914, 2962, 3144, 2882, 3030, 3114, 2984] combine_rows_per_src=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344504 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267239864 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135224 weight_movement_bytes=34603008 total_nvlink_bytes=568738232 weight_to_token_ratio=0.064783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343336 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050856 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758376 weight_movement_bytes=69206016 total_nvlink_bytes=600964392 weight_to_token_ratio=0.130146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64802 dup_rows=24208 dup_rows_remote=21208 dedup_ratio=0.246257 router_tokens_per_expert=[788, 783, 738, 797, 804, 790, 769, 754, 760, 793, 734, 781, 780, 744, 824, 726, 781, 772, 750, 777, 777, 779, 759, 772, 798, 782, 761, 738, 742, 774, 768, 780, 751, 769, 788, 745, 719, 805, 795, 775, 787, 725, 773, 785, 743, 815, 771, 775, 779, 735, 752, 762, 760, 768, 726, 729, 787, 751, 743, 771, 762, 753, 797, 781] payload_rows_per_dst=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] dup_rows_per_dst=[3154, 2824, 2872, 3204, 3098, 2678, 3378, 3000] combine_rows_per_src=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344560 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805744 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266928 weight_movement_bytes=69206016 total_nvlink_bytes=602472944 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65134 dup_rows=24008 dup_rows_remote=21044 dedup_ratio=0.244222 router_tokens_per_expert=[764, 738, 756, 826, 761, 738, 767, 758, 768, 748, 777, 800, 730, 795, 784, 757, 770, 787, 745, 784, 813, 749, 734, 784, 779, 799, 777, 809, 787, 764, 739, 757, 747, 766, 749, 751, 797, 769, 775, 786, 786, 744, 764, 768, 778, 783, 724, 763, 737, 763, 807, 736, 762, 770, 711, 777, 764, 798, 749, 787, 770, 800, 727, 800] payload_rows_per_dst=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] dup_rows_per_dst=[2946, 2990, 3056, 3050, 3008, 2974, 2964, 3020] combine_rows_per_src=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64810 dup_rows=24224 dup_rows_remote=21298 dedup_ratio=0.246419 router_tokens_per_expert=[732, 729, 722, 746, 745, 799, 757, 761, 770, 821, 766, 766, 808, 748, 760, 793, 757, 757, 766, 781, 804, 766, 782, 738, 783, 777, 831, 784, 760, 749, 769, 757, 748, 784, 766, 755, 745, 743, 748, 809, 801, 749, 796, 776, 780, 777, 790, 740, 738, 789, 770, 774, 756, 789, 738, 783, 754, 746, 805, 741, 774, 728, 800, 776] payload_rows_per_dst=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] dup_rows_per_dst=[2866, 3166, 3028, 3134, 2926, 3118, 3032, 2954] combine_rows_per_src=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344040 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265773032 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531202024 weight_movement_bytes=69206016 total_nvlink_bytes=600408040 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262674320 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6135295888 ingress_bytes=7373086720 link_bytes=7373086720 total_nvlink_bytes=13508382608 weight_to_token_ratio=0.101585 prefetch_share_of_ingress=0.103249 dedup_ratio=0.244102 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206275293e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64834 dup_rows=24166 dup_rows_remote=21032 dedup_ratio=0.245829 router_tokens_per_expert=[785, 807, 805, 734, 743, 747, 785, 768, 799, 771, 773, 785, 747, 788, 740, 786, 820, 768, 777, 722, 789, 802, 737, 756, 753, 713, 736, 812, 755, 787, 813, 778, 736, 726, 745, 773, 774, 748, 772, 769, 751, 761, 769, 820, 766, 791, 745, 757, 729, 756, 763, 714, 809, 755, 751, 812, 746, 804, 770, 762, 792, 755, 798, 752] payload_rows_per_dst=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] dup_rows_per_dst=[3134, 3020, 3006, 3040, 2912, 3048, 2964, 3042] combine_rows_per_src=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65058 dup_rows=23944 dup_rows_remote=21062 dedup_ratio=0.243571 router_tokens_per_expert=[744, 763, 769, 788, 776, 787, 729, 783, 734, 777, 746, 782, 741, 743, 783, 786, 763, 744, 799, 787, 755, 718, 760, 778, 745, 793, 729, 834, 722, 796, 800, 791, 803, 756, 765, 760, 753, 756, 719, 781, 809, 814, 710, 783, 786, 764, 729, 810, 744, 747, 786, 781, 796, 757, 766, 735, 742, 770, 747, 779, 815, 761, 769, 814] payload_rows_per_dst=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] dup_rows_per_dst=[3096, 2882, 2974, 3034, 2968, 3072, 2924, 2994] combine_rows_per_src=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344712 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133576 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922440 weight_movement_bytes=34603008 total_nvlink_bytes=568525448 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64878 dup_rows=24166 dup_rows_remote=21138 dedup_ratio=0.245829 router_tokens_per_expert=[773, 763, 765, 688, 778, 760, 742, 794, 733, 782, 811, 775, 800, 767, 767, 840, 791, 791, 775, 791, 789, 742, 811, 799, 805, 755, 758, 775, 757, 770, 742, 782, 775, 781, 787, 764, 769, 759, 740, 723, 773, 814, 721, 787, 779, 731, 736, 778, 806, 743, 754, 747, 766, 732, 777, 765, 700, 823, 751, 795, 764, 774, 756, 711] payload_rows_per_dst=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] dup_rows_per_dst=[2948, 3056, 3154, 3028, 3026, 3042, 2984, 2928] combine_rows_per_src=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=344432 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265806192 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531267952 weight_movement_bytes=34603008 total_nvlink_bytes=565870960 weight_to_token_ratio=0.065133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65084 dup_rows=24056 dup_rows_remote=21126 dedup_ratio=0.244710 router_tokens_per_expert=[814, 741, 761, 747, 769, 766, 713, 746, 750, 788, 772, 785, 777, 746, 782, 796, 766, 801, 774, 763, 788, 739, 737, 792, 769, 752, 785, 852, 774, 762, 746, 836, 732, 808, 757, 765, 785, 769, 710, 760, 738, 798, 741, 754, 810, 784, 725, 697, 798, 759, 745, 776, 734, 721, 756, 805, 802, 753, 750, 771, 782, 802, 770, 806] payload_rows_per_dst=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] dup_rows_per_dst=[2910, 3082, 2986, 3204, 2864, 2930, 3006, 3074] combine_rows_per_src=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=343464 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265903528 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531463592 weight_movement_bytes=69206016 total_nvlink_bytes=600669608 weight_to_token_ratio=0.130218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822048 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299616 weight_movement_bytes=69206016 total_nvlink_bytes=602505632 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344064 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084352 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824640 weight_movement_bytes=34603008 total_nvlink_bytes=566427648 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254608120 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6131260152 ingress_bytes=7542071296 link_bytes=7542071296 total_nvlink_bytes=13673331448 weight_to_token_ratio=0.115771 prefetch_share_of_ingress=0.087172 dedup_ratio=0.244249 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233922048e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344840 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928904 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512968 weight_movement_bytes=69206016 total_nvlink_bytes=602718984 weight_to_token_ratio=0.129718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65108 dup_rows=23708 dup_rows_remote=20658 dedup_ratio=0.241170 router_tokens_per_expert=[786, 845, 769, 811, 769, 792, 750, 740, 786, 745, 810, 731, 797, 771, 807, 831, 715, 787, 738, 751, 790, 770, 776, 819, 729, 781, 763, 751, 822, 727, 784, 734, 730, 782, 746, 742, 788, 791, 770, 747, 773, 804, 758, 750, 732, 713, 800, 725, 729, 805, 802, 726, 756, 767, 822, 764, 792, 747, 795, 782, 733, 744, 737, 723] payload_rows_per_dst=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] dup_rows_per_dst=[3076, 2920, 2906, 2640, 3188, 2934, 2994, 3050] combine_rows_per_src=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65020 dup_rows=23976 dup_rows_remote=21008 dedup_ratio=0.243896 router_tokens_per_expert=[810, 763, 764, 758, 781, 783, 791, 710, 769, 758, 751, 788, 806, 772, 773, 791, 764, 759, 760, 766, 780, 761, 733, 757, 748, 784, 742, 762, 785, 783, 756, 789, 708, 800, 799, 769, 766, 741, 773, 791, 791, 765, 784, 784, 751, 766, 770, 749, 717, 799, 750, 758, 803, 801, 758, 752, 785, 759, 743, 765, 793, 731, 764, 770] payload_rows_per_dst=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] dup_rows_per_dst=[3044, 2982, 2976, 2948, 3002, 3042, 2968, 3014] combine_rows_per_src=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256558176 prefetch_bytes=588251136 reduce_bytes=1245708288 weight_movement_bytes=1833959424 egress_bytes=6132235360 ingress_bytes=7958282240 link_bytes=7958282240 total_nvlink_bytes=14090517600 weight_to_token_ratio=0.149631 prefetch_share_of_ingress=0.073917 dedup_ratio=0.244287 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=36 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.302016321e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258170824 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133045192 ingress_bytes=7370833920 link_bytes=7370833920 total_nvlink_bytes=13503879112 weight_to_token_ratio=0.101623 prefetch_share_of_ingress=0.098586 dedup_ratio=0.244368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205906723e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343064 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267025432 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533707800 weight_movement_bytes=34603008 total_nvlink_bytes=568310808 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64970 dup_rows=24198 dup_rows_remote=21234 dedup_ratio=0.246155 router_tokens_per_expert=[772, 776, 751, 748, 750, 744, 731, 778, 792, 782, 808, 788, 747, 778, 759, 780, 804, 780, 843, 735, 768, 769, 692, 785, 775, 775, 787, 782, 771, 769, 766, 738, 695, 755, 768, 777, 748, 700, 796, 766, 751, 800, 795, 784, 796, 816, 729, 717, 729, 823, 767, 774, 759, 777, 797, 745, 783, 769, 755, 801, 734, 772, 788, 763] payload_rows_per_dst=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] dup_rows_per_dst=[2964, 3108, 3358, 3042, 2852, 3028, 3044, 2802] combine_rows_per_src=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666032 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987952 weight_movement_bytes=69206016 total_nvlink_bytes=602193968 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65150 dup_rows=23922 dup_rows_remote=20948 dedup_ratio=0.243347 router_tokens_per_expert=[779, 730, 788, 752, 772, 729, 810, 804, 735, 782, 715, 762, 717, 744, 793, 754, 824, 809, 711, 798, 833, 771, 740, 778, 806, 722, 771, 773, 729, 760, 752, 790, 783, 790, 790, 759, 816, 760, 757, 746, 763, 750, 773, 752, 753, 754, 796, 790, 742, 753, 772, 758, 818, 775, 797, 762, 780, 755, 783, 790, 738, 727, 775, 762] payload_rows_per_dst=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] dup_rows_per_dst=[2996, 2794, 3104, 2974, 3086, 2984, 3014, 2970] combine_rows_per_src=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12215802008 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6111844504 ingress_bytes=7418871808 link_bytes=7418871808 total_nvlink_bytes=13530716312 weight_to_token_ratio=0.107640 prefetch_share_of_ingress=0.093284 dedup_ratio=0.244988 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.213765972e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344816 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266461936 ingress_bytes=335323136 link_bytes=335323136 token_bytes=532579056 weight_movement_bytes=69206016 total_nvlink_bytes=601785072 weight_to_token_ratio=0.129945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261793965e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262493752 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6135205432 ingress_bytes=7407599616 link_bytes=7407599616 total_nvlink_bytes=13542805048 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.088754 dedup_ratio=0.243987 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211921782e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344392 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267198792 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534053192 weight_movement_bytes=34603008 total_nvlink_bytes=568656200 weight_to_token_ratio=0.064793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263505976 prefetch_bytes=761266176 reduce_bytes=449839104 weight_movement_bytes=1211105280 egress_bytes=6135709752 ingress_bytes=7338901504 link_bytes=7338901504 total_nvlink_bytes=13474611256 weight_to_token_ratio=0.098757 prefetch_share_of_ingress=0.103730 dedup_ratio=0.243743 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=13 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200682414e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65138 dup_rows=23962 dup_rows_remote=20994 dedup_ratio=0.243754 router_tokens_per_expert=[713, 751, 782, 805, 745, 710, 780, 773, 809, 762, 738, 791, 744, 718, 778, 751, 774, 773, 801, 759, 786, 786, 774, 738, 818, 741, 710, 767, 769, 712, 738, 750, 734, 814, 740, 763, 769, 790, 784, 843, 766, 796, 783, 778, 769, 769, 730, 763, 753, 825, 789, 772, 741, 783, 792, 762, 756, 745, 801, 818, 777, 796, 756, 749] payload_rows_per_dst=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] dup_rows_per_dst=[2968, 2904, 3000, 2816, 3154, 3030, 3094, 2996] combine_rows_per_src=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344528 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149776 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955024 weight_movement_bytes=34603008 total_nvlink_bytes=568558032 weight_to_token_ratio=0.064805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21134 dedup_ratio=0.245524 router_tokens_per_expert=[775, 761, 781, 782, 773, 758, 775, 745, 782, 818, 768, 770, 744, 775, 754, 799, 783, 747, 773, 772, 807, 778, 755, 758, 803, 782, 787, 752, 761, 748, 777, 763, 803, 719, 792, 801, 732, 781, 702, 769, 779, 755, 793, 763, 762, 801, 757, 758, 773, 809, 761, 750, 787, 740, 760, 751, 836, 739, 728, 723, 766, 743, 724, 789] payload_rows_per_dst=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] dup_rows_per_dst=[3002, 3144, 3112, 2986, 2964, 2992, 3018, 2918] combine_rows_per_src=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343696 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723536 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103376 weight_movement_bytes=34603008 total_nvlink_bytes=565706384 weight_to_token_ratio=0.065153 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246061576 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6126981640 ingress_bytes=7503200256 link_bytes=7503200256 total_nvlink_bytes=13630181896 weight_to_token_ratio=0.113026 prefetch_share_of_ingress=0.083012 dedup_ratio=0.244239 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227562544e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none + [2026-08-29 23:41:56] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 675.2 | throughput per GPU (TFLOP/s/GPU): 189.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605685E+00 | loss scale: 1.0 | grad norm: 6.650 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64870 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[739, 733, 796, 737, 745, 817, 757, 781, 769, 726, 764, 736, 746, 790, 763, 786, 803, 754, 800, 790, 743, 784, 778, 771, 771, 783, 769, 812, 738, 779, 776, 729, 823, 798, 776, 726, 724, 776, 764, 773, 784, 747, 767, 772, 766, 746, 766, 765, 732, 703, 794, 768, 774, 804, 851, 759, 763, 752, 749, 797, 722, 791, 793, 762] payload_rows_per_dst=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] dup_rows_per_dst=[3000, 2980, 3080, 3062, 3026, 2912, 2988, 3020] combine_rows_per_src=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65128 dup_rows=23864 dup_rows_remote=20864 dedup_ratio=0.242757 router_tokens_per_expert=[763, 781, 809, 747, 784, 777, 761, 767, 755, 793, 712, 784, 802, 782, 775, 753, 738, 790, 804, 800, 767, 765, 799, 753, 792, 810, 779, 739, 690, 772, 761, 751, 752, 760, 731, 754, 797, 736, 746, 746, 745, 760, 741, 773, 830, 769, 728, 840, 756, 764, 799, 748, 777, 767, 717, 751, 763, 728, 781, 769, 775, 794, 804, 796] payload_rows_per_dst=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] dup_rows_per_dst=[3068, 3000, 3060, 2904, 2838, 3016, 2904, 3074] combine_rows_per_src=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74556 payload_rows_remote=65236 dup_rows=23748 dup_rows_remote=20826 dedup_ratio=0.241577 router_tokens_per_expert=[813, 769, 782, 764, 783, 747, 775, 788, 711, 755, 759, 795, 775, 780, 777, 788, 744, 728, 787, 753, 760, 758, 735, 782, 741, 797, 730, 769, 749, 778, 786, 771, 792, 714, 769, 756, 728, 718, 776, 783, 727, 763, 741, 779, 783, 757, 777, 743, 841, 810, 762, 802, 767, 810, 759, 774, 724, 800, 817, 726, 790, 836, 713, 786] payload_rows_per_dst=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] dup_rows_per_dst=[3036, 2958, 2904, 2922, 2796, 2942, 3120, 3070] combine_rows_per_src=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65192 dup_rows=23842 dup_rows_remote=20932 dedup_ratio=0.242533 router_tokens_per_expert=[787, 755, 755, 760, 741, 761, 790, 776, 769, 779, 793, 745, 781, 779, 739, 790, 746, 783, 734, 766, 720, 750, 818, 798, 744, 804, 762, 796, 765, 814, 765, 708, 758, 760, 798, 784, 745, 755, 763, 767, 730, 784, 806, 733, 785, 778, 719, 755, 775, 782, 782, 769, 807, 788, 771, 729, 807, 771, 752, 778, 788, 711, 780, 769] payload_rows_per_dst=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] dup_rows_per_dst=[2908, 3040, 2910, 3054, 2964, 2910, 3038, 3018] combine_rows_per_src=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21146 dedup_ratio=0.245138 router_tokens_per_expert=[763, 751, 772, 740, 765, 743, 770, 749, 752, 754, 765, 816, 773, 815, 772, 795, 793, 784, 794, 789, 749, 789, 717, 794, 765, 763, 774, 784, 749, 780, 809, 784, 780, 725, 766, 768, 827, 768, 761, 756, 760, 773, 748, 781, 747, 700, 763, 809, 785, 782, 800, 778, 752, 719, 770, 757, 771, 793, 732, 764, 754, 761, 748, 742] payload_rows_per_dst=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] dup_rows_per_dst=[2874, 2814, 2830, 3090, 3348, 3192, 2998, 2952] combine_rows_per_src=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65196 dup_rows=23908 dup_rows_remote=20818 dedup_ratio=0.243205 router_tokens_per_expert=[785, 785, 789, 772, 763, 798, 769, 761, 766, 819, 750, 810, 779, 787, 746, 761, 787, 756, 730, 724, 751, 795, 749, 767, 781, 762, 751, 728, 719, 760, 832, 778, 784, 786, 778, 729, 778, 762, 777, 751, 760, 779, 814, 747, 786, 765, 771, 685, 759, 744, 818, 742, 753, 767, 731, 746, 770, 764, 791, 784, 771, 792, 782, 776] payload_rows_per_dst=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] dup_rows_per_dst=[3060, 3154, 2862, 2972, 3090, 2846, 2892, 3032] combine_rows_per_src=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64842 dup_rows=24222 dup_rows_remote=21246 dedup_ratio=0.246399 router_tokens_per_expert=[787, 728, 753, 727, 802, 788, 782, 746, 819, 790, 763, 716, 779, 714, 799, 762, 820, 753, 750, 802, 699, 779, 782, 816, 765, 758, 758, 754, 745, 749, 765, 744, 791, 716, 789, 766, 761, 804, 751, 754, 789, 744, 771, 782, 753, 756, 816, 751, 767, 773, 751, 727, 769, 787, 801, 733, 739, 744, 811, 821, 746, 815, 777, 803] payload_rows_per_dst=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] dup_rows_per_dst=[2980, 3042, 3118, 2918, 3016, 3084, 2976, 3088] combine_rows_per_src=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343432 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050952 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758472 weight_movement_bytes=69206016 total_nvlink_bytes=600964488 weight_to_token_ratio=0.130146 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=343968 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108256 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872544 weight_movement_bytes=69206016 total_nvlink_bytes=603078560 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344248 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550904 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757560 weight_movement_bytes=34603008 total_nvlink_bytes=569360568 weight_to_token_ratio=0.064708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=344496 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267370928 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534397360 weight_movement_bytes=34603008 total_nvlink_bytes=569000368 weight_to_token_ratio=0.064751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344448 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266445184 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545920 weight_movement_bytes=69206016 total_nvlink_bytes=601751936 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344056 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267386872 ingress_bytes=301645824 link_bytes=301645824 token_bytes=534429688 weight_movement_bytes=34603008 total_nvlink_bytes=569032696 weight_to_token_ratio=0.064748 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135068951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21020 dedup_ratio=0.244283 router_tokens_per_expert=[748, 775, 729, 790, 783, 824, 766, 779, 781, 774, 788, 740, 773, 761, 766, 788, 788, 793, 785, 788, 767, 772, 766, 793, 729, 788, 827, 754, 785, 780, 741, 803, 753, 750, 800, 734, 745, 773, 759, 760, 737, 791, 752, 764, 764, 757, 737, 749, 803, 776, 714, 832, 776, 728, 776, 793, 740, 749, 726, 727, 793, 747, 770, 753] payload_rows_per_dst=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] dup_rows_per_dst=[2994, 3242, 3274, 3044, 2936, 2886, 2934, 2704] combine_rows_per_src=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344352 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265937184 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531530016 weight_movement_bytes=103809024 total_nvlink_bytes=635339040 weight_to_token_ratio=0.195302 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64912 dup_rows=24164 dup_rows_remote=21224 dedup_ratio=0.245809 router_tokens_per_expert=[833, 783, 731, 743, 795, 804, 804, 794, 738, 748, 800, 757, 737, 799, 766, 721, 761, 770, 708, 749, 747, 796, 774, 779, 769, 778, 761, 789, 750, 761, 766, 757, 771, 788, 774, 833, 761, 802, 709, 698, 776, 755, 767, 765, 770, 793, 744, 771, 753, 725, 776, 747, 759, 780, 807, 789, 789, 742, 773, 827, 735, 764, 771, 770] payload_rows_per_dst=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] dup_rows_per_dst=[3116, 2948, 2940, 3000, 3048, 3002, 3040, 3070] combine_rows_per_src=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343664 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266157680 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971696 weight_movement_bytes=69206016 total_nvlink_bytes=601177712 weight_to_token_ratio=0.130093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65126 dup_rows=23886 dup_rows_remote=20900 dedup_ratio=0.242981 router_tokens_per_expert=[815, 752, 802, 753, 772, 770, 831, 732, 812, 735, 763, 801, 750, 698, 800, 780, 744, 814, 801, 767, 769, 800, 775, 781, 814, 741, 780, 702, 788, 738, 750, 789, 801, 754, 777, 754, 755, 772, 782, 766, 782, 756, 772, 766, 757, 819, 696, 762, 764, 770, 769, 767, 767, 760, 757, 790, 769, 730, 744, 784, 717, 736, 752, 786] payload_rows_per_dst=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] dup_rows_per_dst=[3094, 2986, 3056, 2920, 3032, 3006, 2980, 2812] combine_rows_per_src=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64952 dup_rows=23964 dup_rows_remote=20938 dedup_ratio=0.243774 router_tokens_per_expert=[763, 797, 741, 751, 768, 797, 816, 773, 789, 812, 791, 742, 776, 759, 745, 760, 733, 767, 770, 741, 736, 765, 729, 815, 807, 780, 784, 770, 778, 787, 736, 765, 754, 732, 757, 736, 768, 768, 807, 718, 791, 765, 787, 798, 740, 756, 771, 753, 773, 761, 777, 792, 734, 767, 782, 783, 761, 762, 764, 797, 755, 777, 774, 749] payload_rows_per_dst=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] dup_rows_per_dst=[3054, 2936, 2960, 3026, 2854, 2982, 3206, 2946] combine_rows_per_src=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65292 dup_rows=23832 dup_rows_remote=20830 dedup_ratio=0.242432 router_tokens_per_expert=[768, 726, 735, 749, 751, 806, 743, 767, 767, 761, 795, 789, 736, 764, 769, 778, 788, 821, 749, 746, 752, 755, 874, 784, 746, 797, 734, 752, 772, 789, 779, 767, 748, 776, 743, 744, 733, 752, 814, 783, 762, 779, 764, 748, 746, 783, 748, 761, 740, 752, 777, 744, 791, 766, 784, 831, 743, 733, 762, 768, 798, 822, 800, 748] payload_rows_per_dst=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] dup_rows_per_dst=[2830, 2946, 3126, 2948, 2952, 3002, 3042, 2986] combine_rows_per_src=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64972 dup_rows=24066 dup_rows_remote=21036 dedup_ratio=0.244812 router_tokens_per_expert=[764, 749, 824, 753, 771, 761, 754, 673, 751, 760, 763, 707, 780, 762, 799, 797, 765, 754, 791, 739, 792, 789, 768, 812, 780, 751, 767, 766, 767, 784, 780, 745, 768, 783, 775, 766, 764, 776, 775, 810, 787, 813, 780, 792, 785, 797, 767, 755, 749, 760, 724, 761, 743, 765, 817, 776, 752, 741, 762, 763, 776, 726, 756, 770] payload_rows_per_dst=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] dup_rows_per_dst=[2840, 2976, 2980, 3090, 3122, 3050, 2978, 3030] combine_rows_per_src=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65044 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[761, 780, 775, 770, 769, 755, 802, 739, 759, 788, 788, 737, 777, 744, 802, 752, 760, 769, 725, 792, 741, 776, 759, 782, 791, 782, 808, 783, 735, 793, 795, 771, 723, 761, 766, 799, 744, 797, 792, 766, 774, 729, 756, 785, 754, 730, 779, 757, 772, 740, 756, 805, 785, 729, 797, 724, 760, 741, 753, 785, 701, 817, 811, 804] payload_rows_per_dst=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] dup_rows_per_dst=[3034, 2928, 3052, 3128, 2972, 2890, 2932, 3000] combine_rows_per_src=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344544 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266224096 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103648 weight_movement_bytes=69206016 total_nvlink_bytes=601309664 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=65032 dup_rows=24068 dup_rows_remote=21190 dedup_ratio=0.244832 router_tokens_per_expert=[806, 786, 783, 741, 746, 729, 749, 763, 807, 802, 830, 745, 722, 758, 767, 779, 801, 811, 804, 744, 778, 772, 735, 763, 718, 744, 742, 741, 741, 790, 793, 759, 764, 792, 762, 750, 756, 747, 798, 809, 795, 782, 730, 792, 798, 786, 764, 759, 758, 729, 774, 781, 757, 725, 773, 744, 794, 781, 778, 770, 769, 742, 787, 757] payload_rows_per_dst=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] dup_rows_per_dst=[2954, 3116, 3020, 2862, 3084, 3108, 2878, 3046] combine_rows_per_src=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344104 combine_payload_bytes=266756096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267100200 ingress_bytes=335962112 link_bytes=335962112 token_bytes=533856296 weight_movement_bytes=69206016 total_nvlink_bytes=603062312 weight_to_token_ratio=0.129634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264198380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344488 combine_payload_bytes=267436032 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267780520 ingress_bytes=302039040 link_bytes=302039040 token_bytes=535216552 weight_movement_bytes=34603008 total_nvlink_bytes=569819560 weight_to_token_ratio=0.064652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136548591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343560 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266386952 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532430344 weight_movement_bytes=34603008 total_nvlink_bytes=567033352 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344032 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469344 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594656 weight_movement_bytes=34603008 total_nvlink_bytes=567197664 weight_to_token_ratio=0.064971 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344032 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764256 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184480 weight_movement_bytes=34603008 total_nvlink_bytes=567787488 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344888 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266715960 ingress_bytes=335577088 link_bytes=335577088 token_bytes=533087032 weight_movement_bytes=69206016 total_nvlink_bytes=602293048 weight_to_token_ratio=0.129821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262749566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64982 dup_rows=23974 dup_rows_remote=20950 dedup_ratio=0.243876 router_tokens_per_expert=[763, 792, 760, 766, 791, 740, 786, 756, 793, 787, 788, 783, 780, 759, 786, 770, 782, 812, 729, 743, 783, 784, 703, 704, 789, 757, 729, 729, 769, 776, 757, 773, 770, 752, 725, 742, 775, 788, 748, 772, 792, 749, 751, 775, 732, 753, 771, 791, 788, 778, 773, 772, 819, 773, 771, 821, 803, 767, 768, 752, 734, 772, 801, 755] payload_rows_per_dst=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] dup_rows_per_dst=[3024, 3022, 2838, 2922, 2970, 3036, 3126, 3036] combine_rows_per_src=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64936 dup_rows=23992 dup_rows_remote=20948 dedup_ratio=0.244059 router_tokens_per_expert=[757, 774, 764, 773, 734, 760, 750, 724, 730, 839, 794, 762, 755, 755, 774, 778, 771, 798, 776, 787, 760, 767, 757, 794, 720, 761, 774, 765, 794, 790, 781, 818, 782, 750, 772, 790, 755, 795, 774, 754, 767, 731, 767, 812, 732, 774, 795, 752, 744, 746, 703, 762, 758, 808, 799, 737, 750, 806, 781, 797, 787, 734, 768, 734] payload_rows_per_dst=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] dup_rows_per_dst=[2866, 3078, 3044, 3094, 2962, 3002, 2970, 2976] combine_rows_per_src=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65156 dup_rows=23900 dup_rows_remote=20836 dedup_ratio=0.243123 router_tokens_per_expert=[787, 801, 785, 727, 747, 806, 744, 741, 784, 754, 740, 764, 820, 759, 768, 767, 779, 797, 740, 767, 795, 763, 741, 724, 732, 718, 801, 762, 754, 828, 735, 769, 744, 749, 809, 755, 748, 712, 796, 767, 745, 746, 764, 800, 766, 736, 798, 762, 754, 749, 752, 806, 787, 776, 806, 735, 750, 789, 806, 763, 795, 816, 782, 790] payload_rows_per_dst=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] dup_rows_per_dst=[2928, 3064, 2944, 2936, 3050, 2922, 2952, 3104] combine_rows_per_src=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64946 dup_rows=24002 dup_rows_remote=20996 dedup_ratio=0.244161 router_tokens_per_expert=[755, 763, 790, 783, 711, 750, 758, 762, 762, 771, 806, 734, 817, 774, 779, 727, 799, 770, 758, 758, 779, 810, 769, 768, 791, 773, 734, 790, 733, 764, 735, 787, 775, 774, 760, 781, 777, 739, 755, 700, 781, 773, 757, 786, 776, 746, 743, 819, 804, 783, 804, 780, 796, 778, 779, 742, 793, 777, 748, 757, 730, 768, 716, 795] payload_rows_per_dst=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] dup_rows_per_dst=[2970, 2978, 3034, 3034, 2966, 3006, 3094, 2920] combine_rows_per_src=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=64894 dup_rows=23960 dup_rows_remote=20904 dedup_ratio=0.243734 router_tokens_per_expert=[780, 791, 777, 730, 773, 816, 765, 816, 790, 788, 723, 742, 732, 769, 771, 754, 781, 747, 767, 781, 767, 718, 825, 770, 807, 740, 819, 805, 734, 792, 756, 800, 755, 730, 772, 776, 737, 762, 726, 742, 802, 764, 732, 761, 804, 777, 804, 795, 726, 835, 796, 799, 749, 743, 728, 786, 732, 744, 787, 722, 737, 755, 777, 771] payload_rows_per_dst=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] dup_rows_per_dst=[3134, 2892, 2988, 3056, 2878, 3032, 2994, 2986] combine_rows_per_src=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343728 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510000 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676272 weight_movement_bytes=69206016 total_nvlink_bytes=601882288 weight_to_token_ratio=0.129921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64804 dup_rows=24126 dup_rows_remote=20950 dedup_ratio=0.245422 router_tokens_per_expert=[769, 773, 756, 727, 800, 780, 822, 731, 731, 736, 781, 827, 781, 782, 742, 814, 753, 773, 780, 775, 749, 802, 754, 772, 757, 779, 760, 769, 829, 799, 740, 739, 764, 758, 726, 802, 760, 743, 795, 738, 767, 774, 756, 750, 738, 778, 806, 786, 776, 794, 735, 809, 729, 766, 779, 773, 753, 778, 763, 667, 765, 805, 796, 741] payload_rows_per_dst=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] dup_rows_per_dst=[3030, 2722, 3050, 2866, 3284, 3046, 2952, 3176] combine_rows_per_src=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343536 combine_payload_bytes=265977856 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266321392 ingress_bytes=404389888 link_bytes=404389888 token_bytes=532299248 weight_movement_bytes=138412032 total_nvlink_bytes=670711280 weight_to_token_ratio=0.260027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521686592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65276 dup_rows=23884 dup_rows_remote=20906 dedup_ratio=0.242961 router_tokens_per_expert=[748, 750, 814, 778, 775, 789, 795, 750, 739, 764, 768, 789, 795, 763, 805, 805, 786, 743, 711, 785, 781, 779, 732, 756, 809, 768, 784, 793, 696, 782, 804, 703, 776, 756, 739, 726, 826, 763, 762, 713, 789, 802, 764, 769, 753, 760, 786, 763, 732, 719, 820, 786, 791, 781, 781, 679, 777, 768, 754, 771, 810, 750, 774, 773] payload_rows_per_dst=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] dup_rows_per_dst=[3058, 2960, 2952, 2974, 2978, 3058, 2910, 2994] combine_rows_per_src=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[747, 804, 759, 761, 711, 777, 753, 762, 817, 735, 786, 759, 813, 764, 763, 769, 745, 742, 797, 778, 766, 842, 760, 761, 749, 839, 809, 746, 779, 780, 772, 753, 771, 829, 754, 744, 774, 701, 782, 774, 781, 755, 757, 716, 786, 798, 742, 768, 762, 729, 760, 786, 755, 762, 748, 824, 794, 783, 762, 751, 767, 749, 765, 725] payload_rows_per_dst=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] dup_rows_per_dst=[2958, 3064, 3050, 3130, 2926, 2924, 2914, 2942] combine_rows_per_src=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343968 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267222944 ingress_bytes=336084992 link_bytes=336084992 token_bytes=534101920 weight_movement_bytes=69206016 total_nvlink_bytes=603307936 weight_to_token_ratio=0.129575 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264660768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343768 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362584 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381400 weight_movement_bytes=69206016 total_nvlink_bytes=601587416 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343192 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266149016 ingress_bytes=335011840 link_bytes=335011840 token_bytes=531954840 weight_movement_bytes=69206016 total_nvlink_bytes=601160856 weight_to_token_ratio=0.130098 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260622583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343016 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780200 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531217384 weight_movement_bytes=34603008 total_nvlink_bytes=565820392 weight_to_token_ratio=0.065139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344728 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267715224 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085720 weight_movement_bytes=34603008 total_nvlink_bytes=569688728 weight_to_token_ratio=0.064668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344208 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821776 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299344 weight_movement_bytes=34603008 total_nvlink_bytes=567902352 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64840 dup_rows=24022 dup_rows_remote=21010 dedup_ratio=0.244364 router_tokens_per_expert=[763, 829, 753, 768, 761, 800, 795, 749, 745, 770, 781, 782, 784, 770, 759, 790, 769, 764, 764, 727, 739, 798, 775, 758, 747, 761, 765, 777, 783, 776, 766, 771, 787, 825, 766, 776, 775, 757, 741, 802, 749, 784, 756, 756, 767, 724, 853, 728, 804, 756, 747, 773, 761, 808, 736, 735, 777, 781, 763, 748, 734, 707, 777, 760] payload_rows_per_dst=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] dup_rows_per_dst=[3012, 3246, 3034, 2992, 3066, 2868, 2922, 2882] combine_rows_per_src=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65074 dup_rows=23930 dup_rows_remote=20886 dedup_ratio=0.243429 router_tokens_per_expert=[800, 796, 781, 797, 749, 758, 755, 746, 801, 759, 713, 758, 724, 773, 797, 798, 761, 772, 818, 769, 744, 783, 731, 794, 759, 744, 784, 780, 760, 808, 779, 771, 746, 760, 757, 812, 779, 744, 744, 799, 761, 824, 775, 772, 789, 746, 762, 803, 736, 731, 750, 787, 739, 751, 800, 730, 724, 764, 732, 770, 750, 804, 804, 745] payload_rows_per_dst=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] dup_rows_per_dst=[3046, 3016, 3044, 2956, 3004, 3048, 2890, 2926] combine_rows_per_src=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65038 dup_rows=24046 dup_rows_remote=21098 dedup_ratio=0.244609 router_tokens_per_expert=[749, 708, 738, 762, 742, 774, 763, 808, 772, 778, 729, 776, 794, 713, 760, 762, 709, 793, 779, 788, 797, 796, 721, 758, 799, 755, 773, 772, 769, 731, 769, 748, 711, 784, 776, 826, 714, 731, 814, 801, 808, 801, 755, 796, 805, 753, 728, 775, 802, 762, 800, 821, 752, 773, 821, 779, 753, 785, 752, 754, 764, 775, 777, 719] payload_rows_per_dst=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] dup_rows_per_dst=[2874, 2948, 2954, 3004, 3104, 3102, 3136, 2924] combine_rows_per_src=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=65144 dup_rows=23680 dup_rows_remote=20586 dedup_ratio=0.240885 router_tokens_per_expert=[797, 727, 824, 746, 724, 756, 771, 740, 711, 827, 756, 754, 771, 741, 745, 748, 768, 784, 751, 763, 753, 792, 741, 758, 766, 784, 731, 761, 793, 765, 793, 793, 766, 744, 757, 803, 784, 771, 779, 761, 779, 808, 769, 770, 790, 791, 764, 816, 738, 732, 768, 789, 797, 829, 735, 748, 781, 778, 793, 772, 776, 739, 785, 706] payload_rows_per_dst=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] dup_rows_per_dst=[2900, 2866, 2854, 3074, 3066, 3094, 2940, 2886] combine_rows_per_src=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21038 dedup_ratio=0.244344 router_tokens_per_expert=[722, 762, 790, 793, 784, 750, 751, 801, 744, 751, 760, 796, 744, 763, 733, 785, 787, 785, 815, 818, 783, 780, 806, 763, 812, 723, 739, 770, 775, 822, 735, 692, 785, 768, 770, 744, 823, 779, 767, 761, 762, 734, 815, 761, 755, 806, 773, 799, 704, 741, 769, 790, 756, 750, 727, 759, 777, 755, 720, 802, 746, 749, 760, 811] payload_rows_per_dst=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] dup_rows_per_dst=[2988, 2992, 3242, 2982, 3012, 3090, 2874, 2840] combine_rows_per_src=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343400 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928040 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531512680 weight_movement_bytes=34603008 total_nvlink_bytes=566115688 weight_to_token_ratio=0.065103 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65224 dup_rows=23866 dup_rows_remote=20810 dedup_ratio=0.242778 router_tokens_per_expert=[787, 792, 805, 793, 778, 773, 777, 790, 730, 742, 778, 763, 754, 801, 796, 723, 817, 731, 778, 773, 794, 765, 767, 782, 763, 744, 789, 753, 808, 741, 823, 730, 826, 743, 774, 777, 739, 728, 773, 775, 754, 778, 765, 756, 811, 770, 760, 746, 781, 767, 737, 748, 782, 743, 727, 721, 730, 795, 740, 797, 738, 822, 733, 776] payload_rows_per_dst=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] dup_rows_per_dst=[3106, 2896, 3042, 2950, 3056, 3008, 2884, 2924] combine_rows_per_src=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343840 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266886944 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430048 weight_movement_bytes=103809024 total_nvlink_bytes=637239072 weight_to_token_ratio=0.194607 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64732 dup_rows=24164 dup_rows_remote=20986 dedup_ratio=0.245809 router_tokens_per_expert=[739, 796, 752, 788, 761, 812, 742, 790, 826, 783, 756, 783, 772, 804, 748, 706, 797, 791, 793, 751, 793, 787, 725, 741, 762, 762, 738, 734, 776, 756, 754, 738, 765, 792, 754, 743, 820, 715, 785, 793, 760, 751, 765, 774, 776, 779, 760, 742, 789, 759, 791, 756, 746, 755, 753, 792, 727, 775, 797, 794, 782, 740, 795, 771] payload_rows_per_dst=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] dup_rows_per_dst=[2888, 2980, 2806, 2924, 3112, 3048, 3228, 3178] combine_rows_per_src=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64972 dup_rows=24006 dup_rows_remote=20966 dedup_ratio=0.244202 router_tokens_per_expert=[794, 751, 769, 777, 779, 726, 791, 814, 812, 783, 760, 718, 742, 753, 742, 763, 758, 786, 759, 756, 789, 743, 796, 775, 765, 777, 785, 827, 793, 791, 766, 743, 751, 779, 733, 744, 725, 748, 733, 793, 794, 735, 805, 731, 757, 783, 709, 767, 773, 759, 756, 775, 816, 756, 796, 752, 738, 803, 753, 832, 764, 770, 781, 758] payload_rows_per_dst=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] dup_rows_per_dst=[3056, 3008, 3050, 3094, 2840, 2862, 3040, 3056] combine_rows_per_src=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344544 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740192 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135840 weight_movement_bytes=34603008 total_nvlink_bytes=567738848 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=342920 combine_payload_bytes=266829824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267172744 ingress_bytes=370638848 link_bytes=370638848 token_bytes=534002568 weight_movement_bytes=103809024 total_nvlink_bytes=637811592 weight_to_token_ratio=0.194398 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394684146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344672 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267117152 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533889632 weight_movement_bytes=69206016 total_nvlink_bytes=603095648 weight_to_token_ratio=0.129626 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344136 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267501640 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659144 weight_movement_bytes=34603008 total_nvlink_bytes=569262152 weight_to_token_ratio=0.064720 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342872 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485144 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627416 weight_movement_bytes=34603008 total_nvlink_bytes=565230424 weight_to_token_ratio=0.065211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469064 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594376 weight_movement_bytes=34603008 total_nvlink_bytes=567197384 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64980 dup_rows=24082 dup_rows_remote=21076 dedup_ratio=0.244975 router_tokens_per_expert=[737, 793, 757, 753, 757, 735, 749, 788, 741, 743, 809, 785, 758, 770, 738, 750, 762, 779, 754, 800, 744, 717, 797, 758, 802, 768, 805, 762, 776, 778, 764, 777, 814, 757, 774, 728, 741, 787, 765, 763, 748, 808, 763, 796, 824, 800, 780, 762, 727, 780, 755, 797, 758, 764, 762, 786, 773, 770, 746, 722, 771, 789, 760, 776] payload_rows_per_dst=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] dup_rows_per_dst=[3006, 3348, 3010, 2984, 2974, 2786, 2990, 2984] combine_rows_per_src=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65226 dup_rows=23776 dup_rows_remote=20872 dedup_ratio=0.241862 router_tokens_per_expert=[801, 731, 792, 783, 791, 756, 743, 802, 738, 793, 702, 746, 763, 815, 781, 810, 776, 780, 748, 791, 776, 755, 733, 744, 744, 809, 769, 783, 742, 768, 756, 787, 730, 784, 795, 755, 791, 764, 722, 777, 763, 745, 762, 786, 724, 783, 770, 782, 752, 853, 783, 772, 759, 800, 773, 740, 757, 741, 747, 774, 794, 747, 764, 755] payload_rows_per_dst=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] dup_rows_per_dst=[3014, 3046, 2904, 3012, 2914, 2934, 3094, 2858] combine_rows_per_src=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65212 dup_rows=23840 dup_rows_remote=20792 dedup_ratio=0.242513 router_tokens_per_expert=[804, 751, 820, 790, 745, 764, 763, 757, 775, 759, 760, 782, 767, 782, 779, 746, 813, 754, 741, 770, 752, 785, 741, 744, 751, 785, 737, 798, 780, 763, 793, 754, 753, 751, 749, 760, 745, 715, 807, 772, 747, 785, 770, 752, 740, 752, 751, 768, 798, 781, 770, 758, 764, 753, 812, 707, 852, 800, 778, 748, 761, 795, 790, 763] payload_rows_per_dst=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] dup_rows_per_dst=[2992, 3048, 2874, 2998, 2922, 2972, 2866, 3168] combine_rows_per_src=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65178 dup_rows=23854 dup_rows_remote=20826 dedup_ratio=0.242655 router_tokens_per_expert=[723, 762, 763, 777, 813, 756, 791, 802, 745, 721, 759, 816, 747, 760, 803, 793, 734, 786, 792, 832, 796, 727, 778, 739, 752, 747, 785, 783, 757, 769, 795, 745, 767, 776, 792, 738, 748, 756, 783, 760, 762, 762, 777, 787, 738, 783, 771, 770, 759, 742, 737, 739, 737, 752, 783, 771, 763, 778, 757, 751, 809, 795, 790, 771] payload_rows_per_dst=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] dup_rows_per_dst=[3044, 3078, 2950, 2958, 2944, 3028, 2850, 3002] combine_rows_per_src=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344224 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502304 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660384 weight_movement_bytes=69206016 total_nvlink_bytes=601866400 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64864 dup_rows=24154 dup_rows_remote=21072 dedup_ratio=0.245707 router_tokens_per_expert=[751, 766, 779, 797, 790, 819, 813, 757, 769, 766, 769, 753, 740, 751, 728, 768, 772, 764, 777, 758, 712, 780, 799, 797, 755, 779, 724, 806, 748, 720, 734, 758, 782, 754, 773, 758, 765, 742, 791, 778, 761, 749, 735, 727, 756, 775, 764, 768, 765, 863, 793, 784, 772, 764, 810, 750, 767, 761, 791, 783, 743, 764, 787, 778] payload_rows_per_dst=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] dup_rows_per_dst=[2968, 3072, 3030, 2878, 2964, 2956, 3204, 3082] combine_rows_per_src=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64756 dup_rows=24268 dup_rows_remote=21190 dedup_ratio=0.246867 router_tokens_per_expert=[748, 753, 737, 779, 782, 728, 809, 757, 747, 773, 779, 761, 780, 790, 765, 775, 748, 752, 741, 765, 770, 768, 741, 741, 780, 747, 758, 772, 778, 774, 786, 784, 799, 789, 796, 770, 748, 776, 772, 788, 775, 707, 802, 767, 776, 789, 745, 743, 809, 764, 781, 770, 705, 769, 790, 793, 773, 781, 775, 731, 797, 769, 757, 778] payload_rows_per_dst=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] dup_rows_per_dst=[2958, 3014, 2968, 3078, 3096, 2986, 3120, 3048] combine_rows_per_src=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65044 dup_rows=23918 dup_rows_remote=20904 dedup_ratio=0.243306 router_tokens_per_expert=[746, 750, 745, 770, 811, 752, 794, 757, 680, 811, 780, 788, 748, 781, 715, 749, 741, 745, 745, 724, 762, 781, 781, 809, 726, 766, 752, 767, 786, 735, 748, 772, 789, 800, 779, 758, 777, 735, 764, 776, 783, 788, 739, 795, 785, 756, 774, 761, 797, 766, 786, 793, 777, 793, 764, 786, 784, 781, 754, 814, 777, 769, 786, 749] payload_rows_per_dst=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] dup_rows_per_dst=[2980, 2852, 2968, 2836, 3014, 3102, 3102, 3064] combine_rows_per_src=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65140 dup_rows=23936 dup_rows_remote=21058 dedup_ratio=0.243490 router_tokens_per_expert=[778, 732, 780, 804, 771, 810, 731, 777, 742, 773, 779, 823, 741, 790, 782, 784, 806, 776, 733, 755, 755, 759, 752, 783, 762, 758, 802, 809, 764, 781, 735, 709, 776, 761, 805, 736, 747, 729, 743, 750, 768, 795, 762, 840, 785, 765, 743, 806, 752, 768, 777, 765, 753, 797, 735, 706, 775, 746, 763, 762, 787, 780, 755, 784] payload_rows_per_dst=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] dup_rows_per_dst=[2990, 3000, 3050, 3034, 2980, 3002, 2878, 3002] combine_rows_per_src=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344392 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267510088 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675784 weight_movement_bytes=34603008 total_nvlink_bytes=569278792 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344016 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267452368 ingress_bytes=301711360 link_bytes=301711360 token_bytes=534560720 weight_movement_bytes=34603008 total_nvlink_bytes=569163728 weight_to_token_ratio=0.064732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135315558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344016 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313104 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282192 weight_movement_bytes=34603008 total_nvlink_bytes=568885200 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343744 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026688 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709632 weight_movement_bytes=69206016 total_nvlink_bytes=600915648 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343792 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764016 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184240 weight_movement_bytes=34603008 total_nvlink_bytes=567787248 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344792 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267158232 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533971672 weight_movement_bytes=103809024 total_nvlink_bytes=637780696 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65110 dup_rows=23976 dup_rows_remote=20946 dedup_ratio=0.243896 router_tokens_per_expert=[700, 779, 768, 761, 770, 788, 786, 772, 755, 751, 763, 753, 777, 725, 762, 777, 755, 781, 764, 791, 783, 735, 719, 728, 809, 715, 752, 802, 753, 758, 870, 738, 802, 692, 763, 785, 742, 787, 785, 788, 798, 766, 753, 734, 849, 801, 766, 755, 753, 784, 719, 743, 755, 813, 833, 768, 788, 774, 764, 757, 769, 747, 778, 801] payload_rows_per_dst=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] dup_rows_per_dst=[3030, 2874, 3236, 3092, 3136, 3118, 2768, 2722] combine_rows_per_src=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65126 dup_rows=24034 dup_rows_remote=21052 dedup_ratio=0.244486 router_tokens_per_expert=[778, 795, 824, 806, 765, 763, 761, 740, 765, 786, 765, 785, 726, 768, 724, 744, 772, 767, 810, 800, 757, 766, 806, 761, 727, 795, 761, 758, 712, 796, 825, 747, 765, 766, 772, 767, 763, 770, 763, 726, 708, 751, 821, 782, 772, 804, 761, 792, 718, 720, 714, 788, 810, 783, 730, 775, 766, 805, 767, 807, 762, 717, 766, 786] payload_rows_per_dst=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] dup_rows_per_dst=[3072, 2982, 3166, 2974, 2944, 3010, 2908, 2978] combine_rows_per_src=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65058 dup_rows=24006 dup_rows_remote=21026 dedup_ratio=0.244202 router_tokens_per_expert=[797, 804, 773, 822, 788, 801, 771, 769, 713, 776, 758, 706, 820, 747, 770, 754, 773, 756, 754, 741, 760, 763, 827, 727, 740, 788, 752, 760, 823, 781, 776, 735, 784, 753, 761, 780, 765, 797, 746, 793, 730, 769, 730, 799, 771, 771, 789, 751, 740, 837, 766, 742, 728, 771, 748, 798, 788, 733, 722, 783, 813, 753, 758, 758] payload_rows_per_dst=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] dup_rows_per_dst=[3160, 2840, 2912, 3010, 3130, 2980, 2978, 2996] combine_rows_per_src=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64986 dup_rows=24162 dup_rows_remote=21220 dedup_ratio=0.245789 router_tokens_per_expert=[801, 785, 789, 779, 756, 784, 777, 750, 772, 762, 743, 799, 715, 767, 755, 756, 707, 766, 748, 724, 798, 768, 740, 798, 749, 781, 764, 786, 787, 744, 738, 776, 767, 762, 772, 798, 806, 794, 741, 772, 813, 743, 772, 788, 776, 745, 799, 783, 748, 760, 687, 798, 798, 742, 794, 707, 751, 782, 800, 821, 781, 740, 794, 754] payload_rows_per_dst=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] dup_rows_per_dst=[3096, 2912, 2942, 2980, 3154, 3112, 2924, 3042] combine_rows_per_src=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344224 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267034784 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725344 weight_movement_bytes=138412032 total_nvlink_bytes=672137376 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64882 dup_rows=24138 dup_rows_remote=21086 dedup_ratio=0.245544 router_tokens_per_expert=[742, 756, 762, 775, 795, 709, 783, 780, 782, 759, 753, 761, 755, 761, 742, 770, 772, 770, 797, 746, 816, 737, 746, 771, 786, 791, 801, 776, 729, 804, 739, 787, 800, 752, 780, 732, 727, 814, 742, 719, 793, 779, 785, 745, 755, 777, 713, 801, 818, 777, 729, 776, 813, 806, 740, 796, 779, 693, 788, 732, 789, 771, 793, 785] payload_rows_per_dst=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] dup_rows_per_dst=[2452, 3478, 2902, 2990, 3070, 3042, 3152, 3052] combine_rows_per_src=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65162 dup_rows=23930 dup_rows_remote=20950 dedup_ratio=0.243429 router_tokens_per_expert=[720, 747, 743, 772, 790, 746, 768, 800, 735, 777, 758, 780, 803, 719, 721, 816, 762, 762, 793, 806, 790, 781, 785, 773, 752, 793, 778, 719, 775, 780, 762, 737, 795, 778, 761, 784, 755, 803, 772, 764, 765, 767, 745, 743, 762, 791, 746, 771, 823, 747, 765, 829, 782, 740, 787, 756, 754, 744, 767, 792, 769, 773, 737, 742] payload_rows_per_dst=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] dup_rows_per_dst=[2914, 2988, 3056, 2980, 3058, 2914, 3092, 2928] combine_rows_per_src=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65030 dup_rows=24014 dup_rows_remote=21036 dedup_ratio=0.244283 router_tokens_per_expert=[717, 746, 784, 770, 768, 781, 757, 763, 793, 742, 735, 717, 712, 746, 786, 772, 746, 764, 826, 794, 714, 758, 767, 731, 793, 772, 745, 786, 791, 807, 772, 810, 766, 721, 780, 816, 802, 771, 746, 717, 799, 781, 799, 738, 775, 772, 767, 792, 760, 763, 796, 789, 805, 783, 807, 697, 773, 749, 741, 784, 781, 794, 767, 756] payload_rows_per_dst=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] dup_rows_per_dst=[2924, 2860, 2928, 3246, 2978, 3038, 3080, 2960] combine_rows_per_src=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64696 dup_rows=24230 dup_rows_remote=21168 dedup_ratio=0.246480 router_tokens_per_expert=[739, 762, 812, 797, 759, 777, 724, 742, 791, 760, 813, 741, 736, 782, 758, 787, 753, 778, 741, 774, 778, 766, 773, 773, 794, 781, 788, 776, 764, 759, 778, 764, 748, 783, 747, 750, 772, 727, 790, 715, 724, 777, 767, 785, 739, 767, 790, 786, 771, 791, 751, 760, 811, 795, 781, 760, 788, 738, 773, 747, 772, 784, 772, 771] payload_rows_per_dst=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] dup_rows_per_dst=[2964, 3090, 3038, 3036, 2928, 3010, 3062, 3102] combine_rows_per_src=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344712 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100808 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856904 weight_movement_bytes=34603008 total_nvlink_bytes=568459912 weight_to_token_ratio=0.064817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344824 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266527480 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532710136 weight_movement_bytes=69206016 total_nvlink_bytes=601916152 weight_to_token_ratio=0.129913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343872 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100544 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857216 weight_movement_bytes=69206016 total_nvlink_bytes=601063232 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344448 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267248000 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534151552 weight_movement_bytes=69206016 total_nvlink_bytes=603357568 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344264 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707144 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070024 weight_movement_bytes=34603008 total_nvlink_bytes=567673032 weight_to_token_ratio=0.064913 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64676 dup_rows=24074 dup_rows_remote=20730 dedup_ratio=0.244893 router_tokens_per_expert=[823, 756, 755, 777, 754, 828, 779, 745, 772, 747, 788, 748, 797, 783, 750, 767, 784, 761, 795, 757, 764, 766, 781, 780, 750, 735, 740, 761, 733, 757, 755, 760, 769, 743, 764, 769, 753, 784, 799, 777, 745, 755, 729, 772, 780, 748, 773, 757, 769, 803, 740, 746, 789, 748, 789, 786, 727, 784, 823, 798, 771, 814, 743, 757] payload_rows_per_dst=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] dup_rows_per_dst=[3344, 2962, 3044, 2924, 3072, 2810, 2918, 3000] combine_rows_per_src=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64938 dup_rows=23902 dup_rows_remote=20844 dedup_ratio=0.243144 router_tokens_per_expert=[761, 790, 717, 751, 762, 779, 745, 812, 781, 776, 816, 808, 720, 809, 775, 776, 746, 715, 740, 757, 772, 768, 770, 782, 766, 792, 808, 820, 711, 788, 779, 763, 762, 720, 773, 763, 717, 776, 751, 773, 791, 838, 763, 762, 734, 742, 784, 780, 742, 793, 818, 717, 799, 781, 766, 744, 779, 768, 744, 740, 760, 785, 786, 746] payload_rows_per_dst=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] dup_rows_per_dst=[3036, 3058, 2950, 3066, 2858, 3038, 2960, 2936] combine_rows_per_src=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65250 dup_rows=23788 dup_rows_remote=20830 dedup_ratio=0.241984 router_tokens_per_expert=[787, 780, 811, 750, 768, 723, 801, 789, 735, 760, 786, 760, 754, 774, 746, 753, 756, 732, 760, 754, 772, 754, 745, 762, 802, 786, 760, 797, 777, 739, 734, 811, 761, 741, 758, 729, 765, 795, 794, 794, 808, 764, 790, 753, 748, 754, 753, 742, 774, 760, 713, 763, 802, 761, 816, 790, 787, 740, 806, 777, 828, 809, 710, 749] payload_rows_per_dst=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] dup_rows_per_dst=[3042, 2906, 2872, 3036, 2950, 2958, 3014, 3010] combine_rows_per_src=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65208 dup_rows=23808 dup_rows_remote=20784 dedup_ratio=0.242188 router_tokens_per_expert=[784, 728, 756, 779, 762, 736, 775, 758, 733, 832, 771, 745, 744, 772, 746, 814, 770, 733, 765, 814, 796, 734, 788, 756, 779, 742, 729, 734, 800, 799, 781, 762, 790, 771, 749, 746, 741, 781, 791, 722, 798, 772, 738, 762, 806, 781, 750, 813, 737, 763, 763, 765, 777, 790, 744, 778, 817, 791, 803, 743, 749, 725, 782, 797] payload_rows_per_dst=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] dup_rows_per_dst=[2934, 2988, 3024, 2936, 2924, 3118, 2902, 2982] combine_rows_per_src=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=341624 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265254520 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530167416 weight_movement_bytes=69206016 total_nvlink_bytes=599373432 weight_to_token_ratio=0.130536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64956 dup_rows=24012 dup_rows_remote=21078 dedup_ratio=0.244263 router_tokens_per_expert=[769, 760, 787, 796, 780, 756, 775, 825, 761, 752, 827, 773, 736, 756, 760, 779, 755, 771, 766, 748, 768, 758, 755, 762, 760, 804, 765, 750, 783, 736, 839, 771, 794, 789, 711, 782, 753, 764, 747, 750, 783, 751, 790, 766, 764, 806, 757, 757, 830, 745, 767, 758, 727, 729, 763, 751, 763, 750, 775, 780, 766, 803, 752, 746] payload_rows_per_dst=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] dup_rows_per_dst=[2914, 2624, 2956, 3036, 3350, 3040, 3158, 2934] combine_rows_per_src=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65148 dup_rows=23822 dup_rows_remote=20936 dedup_ratio=0.242330 router_tokens_per_expert=[740, 771, 745, 787, 747, 793, 790, 819, 782, 804, 756, 733, 757, 787, 757, 785, 718, 776, 769, 768, 745, 791, 777, 757, 797, 772, 781, 762, 772, 724, 728, 774, 772, 747, 787, 775, 745, 758, 751, 793, 752, 726, 797, 777, 773, 746, 800, 760, 765, 753, 854, 766, 740, 758, 783, 772, 777, 789, 784, 783, 773, 725, 765, 742] payload_rows_per_dst=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] dup_rows_per_dst=[3006, 3028, 2968, 2886, 2924, 2984, 3026, 3000] combine_rows_per_src=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343128 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266329176 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315224 weight_movement_bytes=69206016 total_nvlink_bytes=601521240 weight_to_token_ratio=0.130009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65014 dup_rows=23948 dup_rows_remote=20934 dedup_ratio=0.243612 router_tokens_per_expert=[763, 756, 795, 777, 763, 812, 797, 776, 852, 783, 808, 799, 803, 809, 761, 788, 737, 799, 762, 753, 736, 764, 797, 746, 746, 760, 752, 762, 708, 801, 800, 802, 742, 764, 727, 770, 745, 710, 775, 779, 826, 778, 742, 776, 712, 770, 770, 747, 808, 742, 783, 802, 752, 765, 733, 793, 743, 750, 755, 792, 710, 711, 747, 766] payload_rows_per_dst=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] dup_rows_per_dst=[3150, 3212, 2976, 2970, 2768, 2948, 3014, 2910] combine_rows_per_src=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65274 dup_rows=23860 dup_rows_remote=20956 dedup_ratio=0.242716 router_tokens_per_expert=[780, 741, 800, 728, 777, 760, 763, 782, 772, 734, 763, 784, 769, 745, 785, 755, 799, 739, 807, 776, 791, 759, 815, 755, 732, 787, 711, 730, 816, 763, 781, 783, 740, 765, 757, 730, 752, 738, 775, 780, 764, 779, 782, 797, 770, 773, 752, 771, 727, 780, 761, 745, 775, 767, 770, 761, 783, 769, 756, 796, 817, 791, 773, 774] payload_rows_per_dst=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] dup_rows_per_dst=[2990, 2942, 3012, 2856, 2904, 3018, 2948, 3190] combine_rows_per_src=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344320 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267608320 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872320 weight_movement_bytes=69206016 total_nvlink_bytes=604078336 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343968 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267435936 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534527904 weight_movement_bytes=34603008 total_nvlink_bytes=569130912 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344136 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266403912 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463688 weight_movement_bytes=103809024 total_nvlink_bytes=636272712 weight_to_token_ratio=0.194960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344336 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190544 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036752 weight_movement_bytes=69206016 total_nvlink_bytes=603242768 weight_to_token_ratio=0.129590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343792 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641136 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938480 weight_movement_bytes=34603008 total_nvlink_bytes=567541488 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344920 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267707224 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069528 weight_movement_bytes=34603008 total_nvlink_bytes=569672536 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65072 dup_rows=23986 dup_rows_remote=21006 dedup_ratio=0.243998 router_tokens_per_expert=[795, 765, 780, 754, 724, 760, 758, 762, 746, 758, 765, 769, 800, 804, 762, 805, 815, 762, 825, 772, 748, 809, 764, 765, 777, 721, 771, 748, 746, 744, 774, 776, 748, 776, 771, 705, 805, 801, 808, 775, 821, 757, 733, 744, 738, 783, 778, 716, 707, 740, 722, 726, 764, 751, 822, 764, 786, 804, 802, 767, 807, 787, 755, 765] payload_rows_per_dst=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] dup_rows_per_dst=[2980, 3114, 3282, 2870, 3060, 2852, 2902, 2926] combine_rows_per_src=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64896 dup_rows=24042 dup_rows_remote=21024 dedup_ratio=0.244568 router_tokens_per_expert=[761, 745, 789, 730, 771, 754, 782, 756, 801, 769, 775, 784, 774, 711, 782, 796, 751, 777, 785, 771, 735, 772, 766, 769, 758, 783, 743, 789, 781, 751, 771, 796, 748, 738, 718, 787, 771, 767, 742, 776, 769, 762, 732, 794, 768, 776, 770, 840, 809, 808, 799, 709, 732, 743, 779, 745, 806, 774, 748, 805, 701, 799, 806, 753] payload_rows_per_dst=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] dup_rows_per_dst=[2972, 3018, 2996, 3054, 2846, 3094, 2990, 3072] combine_rows_per_src=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65122 dup_rows=23892 dup_rows_remote=20924 dedup_ratio=0.243042 router_tokens_per_expert=[763, 762, 755, 764, 780, 748, 786, 788, 761, 735, 761, 799, 766, 776, 769, 785, 767, 740, 750, 758, 747, 754, 750, 752, 744, 802, 774, 747, 746, 779, 744, 770, 785, 777, 745, 760, 776, 783, 793, 760, 729, 759, 788, 742, 773, 794, 765, 779, 799, 764, 825, 782, 761, 767, 754, 795, 781, 738, 792, 734, 812, 724, 784, 810] payload_rows_per_dst=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] dup_rows_per_dst=[2980, 3016, 2834, 2974, 3092, 2968, 3020, 3008] combine_rows_per_src=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65184 dup_rows=23912 dup_rows_remote=21020 dedup_ratio=0.243245 router_tokens_per_expert=[770, 766, 765, 765, 770, 764, 779, 768, 794, 721, 743, 768, 796, 741, 746, 750, 747, 769, 764, 766, 735, 754, 762, 753, 796, 776, 821, 784, 717, 754, 794, 778, 719, 737, 769, 765, 796, 790, 779, 749, 749, 841, 784, 796, 804, 732, 770, 774, 738, 763, 750, 718, 797, 731, 768, 751, 801, 816, 760, 759, 777, 803, 815, 775] payload_rows_per_dst=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] dup_rows_per_dst=[3036, 2898, 2892, 3056, 2932, 3134, 2872, 3092] combine_rows_per_src=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344312 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879224 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414136 weight_movement_bytes=69206016 total_nvlink_bytes=602620152 weight_to_token_ratio=0.129742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65054 dup_rows=23908 dup_rows_remote=20988 dedup_ratio=0.243205 router_tokens_per_expert=[797, 753, 745, 783, 794, 723, 714, 801, 756, 780, 743, 770, 752, 732, 764, 750, 771, 752, 722, 815, 764, 746, 749, 758, 735, 770, 776, 812, 801, 744, 811, 793, 778, 765, 789, 739, 757, 813, 757, 808, 749, 757, 772, 792, 785, 754, 756, 764, 793, 759, 759, 765, 794, 782, 780, 778, 762, 742, 762, 787, 762, 763, 767, 786] payload_rows_per_dst=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] dup_rows_per_dst=[3000, 2894, 2844, 3006, 2950, 3086, 3208, 2920] combine_rows_per_src=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65318 dup_rows=23790 dup_rows_remote=20940 dedup_ratio=0.242004 router_tokens_per_expert=[790, 768, 765, 746, 759, 809, 752, 765, 745, 777, 804, 757, 797, 752, 784, 738, 748, 771, 752, 803, 748, 731, 767, 818, 743, 741, 770, 750, 768, 760, 745, 746, 799, 798, 804, 771, 799, 737, 773, 827, 801, 763, 773, 780, 771, 800, 732, 786, 740, 725, 759, 749, 781, 777, 762, 770, 806, 754, 757, 739, 757, 718, 787, 788] payload_rows_per_dst=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] dup_rows_per_dst=[2966, 3002, 3000, 2850, 3218, 3000, 2852, 2902] combine_rows_per_src=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64990 dup_rows=24106 dup_rows_remote=21232 dedup_ratio=0.245219 router_tokens_per_expert=[792, 774, 777, 762, 769, 741, 791, 751, 800, 742, 697, 724, 790, 812, 759, 765, 766, 775, 758, 798, 754, 748, 783, 764, 762, 816, 765, 735, 807, 798, 767, 796, 786, 781, 781, 774, 804, 797, 777, 804, 739, 760, 758, 732, 792, 769, 725, 812, 780, 728, 703, 767, 768, 774, 735, 786, 814, 746, 759, 722, 746, 756, 749, 790] payload_rows_per_dst=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] dup_rows_per_dst=[3036, 2986, 3040, 3174, 3080, 2982, 2874, 2934] combine_rows_per_src=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343680 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157696 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971712 weight_movement_bytes=34603008 total_nvlink_bytes=566574720 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64912 dup_rows=24240 dup_rows_remote=21166 dedup_ratio=0.246582 router_tokens_per_expert=[769, 761, 781, 800, 777, 736, 782, 769, 760, 744, 798, 770, 787, 805, 764, 731, 753, 805, 764, 794, 776, 763, 766, 806, 783, 739, 773, 719, 734, 792, 734, 768, 765, 758, 775, 778, 805, 735, 741, 756, 812, 814, 747, 760, 808, 774, 771, 765, 751, 760, 796, 783, 701, 790, 743, 739, 778, 731, 782, 775, 742, 753, 758, 803] payload_rows_per_dst=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] dup_rows_per_dst=[3042, 3116, 3018, 2890, 3074, 3108, 2996, 2996] combine_rows_per_src=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344184 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083896 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823608 weight_movement_bytes=69206016 total_nvlink_bytes=603029624 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344816 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338480 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332144 weight_movement_bytes=34603008 total_nvlink_bytes=568935152 weight_to_token_ratio=0.064759 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344168 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805352 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266536 weight_movement_bytes=103809024 total_nvlink_bytes=637075560 weight_to_token_ratio=0.194666 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267542528 dispatch_meta_bytes=345032 combine_payload_bytes=267542528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267887560 ingress_bytes=336748544 link_bytes=336748544 token_bytes=535430088 weight_movement_bytes=69206016 total_nvlink_bytes=604636104 weight_to_token_ratio=0.129253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267157661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344888 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543928 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742968 weight_movement_bytes=34603008 total_nvlink_bytes=567345976 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344312 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266223864 ingress_bytes=369688576 link_bytes=369688576 token_bytes=532103416 weight_movement_bytes=103809024 total_nvlink_bytes=635912440 weight_to_token_ratio=0.195092 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391108349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74758 payload_rows_remote=65358 dup_rows=23546 dup_rows_remote=20596 dedup_ratio=0.239522 router_tokens_per_expert=[767, 767, 763, 749, 757, 736, 789, 775, 775, 762, 800, 795, 793, 754, 747, 744, 785, 709, 740, 803, 746, 783, 734, 765, 784, 791, 735, 792, 764, 783, 738, 758, 769, 727, 738, 767, 803, 771, 814, 739, 723, 753, 782, 795, 752, 765, 786, 774, 767, 743, 754, 738, 757, 888, 783, 781, 758, 822, 758, 758, 774, 820, 741, 769] payload_rows_per_dst=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] dup_rows_per_dst=[2950, 3078, 3112, 2964, 2948, 2620, 2978, 2896] combine_rows_per_src=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64928 dup_rows=24050 dup_rows_remote=20976 dedup_ratio=0.244649 router_tokens_per_expert=[769, 787, 761, 794, 804, 793, 810, 756, 730, 781, 775, 793, 750, 780, 789, 802, 738, 755, 766, 790, 744, 738, 745, 768, 802, 775, 776, 774, 735, 769, 786, 787, 764, 749, 737, 714, 769, 736, 745, 728, 776, 761, 735, 752, 764, 784, 765, 771, 785, 774, 768, 779, 787, 740, 755, 788, 785, 753, 777, 777, 775, 743, 787, 807] payload_rows_per_dst=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] dup_rows_per_dst=[3116, 3074, 2886, 3158, 2760, 3006, 3050, 3000] combine_rows_per_src=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65124 dup_rows=23692 dup_rows_remote=20688 dedup_ratio=0.241007 router_tokens_per_expert=[762, 768, 777, 790, 754, 785, 762, 782, 802, 792, 798, 742, 763, 722, 794, 720, 773, 757, 744, 779, 769, 765, 783, 732, 787, 809, 799, 701, 760, 759, 792, 773, 751, 758, 774, 780, 737, 784, 789, 766, 798, 747, 782, 811, 803, 800, 758, 747, 780, 790, 752, 751, 781, 752, 764, 747, 745, 787, 733, 749, 779, 747, 783, 732] payload_rows_per_dst=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] dup_rows_per_dst=[3010, 2954, 2902, 2970, 3002, 3004, 3012, 2838] combine_rows_per_src=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=343816 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050184 ingress_bytes=302309376 link_bytes=302309376 token_bytes=535756552 weight_movement_bytes=34603008 total_nvlink_bytes=570359560 weight_to_token_ratio=0.064587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137565844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65142 dup_rows=23852 dup_rows_remote=20926 dedup_ratio=0.242635 router_tokens_per_expert=[791, 800, 731, 781, 754, 820, 752, 735, 774, 825, 753, 795, 771, 746, 780, 771, 741, 776, 784, 719, 782, 789, 801, 726, 734, 778, 813, 725, 746, 769, 783, 745, 734, 781, 770, 756, 777, 761, 768, 756, 756, 771, 712, 802, 798, 793, 810, 760, 760, 749, 788, 742, 797, 806, 763, 770, 737, 707, 754, 836, 753, 735, 813, 747] payload_rows_per_dst=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] dup_rows_per_dst=[2998, 3176, 2926, 2982, 2952, 3010, 2948, 2860] combine_rows_per_src=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64880 dup_rows=24124 dup_rows_remote=21092 dedup_ratio=0.245402 router_tokens_per_expert=[805, 795, 757, 763, 760, 721, 751, 717, 801, 813, 775, 756, 795, 766, 751, 780, 716, 776, 777, 751, 730, 753, 731, 794, 776, 768, 755, 748, 759, 810, 734, 777, 787, 801, 721, 812, 756, 779, 756, 803, 782, 764, 754, 742, 796, 786, 753, 769, 790, 737, 793, 776, 764, 773, 754, 777, 776, 751, 788, 776, 774, 749, 775, 777] payload_rows_per_dst=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] dup_rows_per_dst=[2480, 2662, 3226, 3034, 3606, 3020, 3064, 3032] combine_rows_per_src=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343616 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288704 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233792 weight_movement_bytes=69206016 total_nvlink_bytes=601439808 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65078 dup_rows=23922 dup_rows_remote=20734 dedup_ratio=0.243347 router_tokens_per_expert=[808, 767, 801, 752, 774, 764, 727, 784, 775, 809, 783, 762, 773, 789, 785, 787, 778, 767, 729, 778, 763, 749, 743, 753, 771, 779, 769, 768, 820, 783, 769, 787, 775, 729, 792, 760, 747, 757, 759, 784, 780, 775, 804, 751, 766, 770, 782, 740, 749, 742, 761, 723, 750, 792, 771, 716, 764, 756, 798, 776, 765, 797, 701, 774] payload_rows_per_dst=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] dup_rows_per_dst=[3072, 3048, 2896, 3188, 2874, 3046, 2850, 2948] combine_rows_per_src=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343248 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267091152 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839056 weight_movement_bytes=103809024 total_nvlink_bytes=637648080 weight_to_token_ratio=0.194458 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64820 dup_rows=24174 dup_rows_remote=21158 dedup_ratio=0.245911 router_tokens_per_expert=[714, 769, 788, 750, 768, 776, 807, 797, 725, 822, 759, 702, 825, 733, 762, 797, 774, 767, 783, 775, 767, 727, 752, 764, 780, 809, 751, 729, 734, 776, 772, 779, 749, 771, 817, 769, 781, 723, 765, 727, 776, 749, 767, 790, 791, 762, 769, 773, 772, 752, 757, 775, 819, 749, 779, 760, 754, 787, 803, 760, 775, 760, 793, 745] payload_rows_per_dst=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] dup_rows_per_dst=[3010, 2990, 2972, 3028, 2986, 3102, 3016, 3070] combine_rows_per_src=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65302 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[754, 779, 764, 773, 733, 763, 787, 757, 745, 742, 782, 792, 773, 806, 720, 805, 771, 792, 762, 793, 787, 762, 770, 756, 741, 791, 744, 792, 741, 813, 772, 799, 686, 734, 738, 762, 787, 750, 761, 744, 738, 802, 756, 754, 743, 725, 805, 804, 787, 767, 746, 749, 801, 820, 757, 767, 769, 743, 789, 826, 778, 775, 753, 775] payload_rows_per_dst=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] dup_rows_per_dst=[2944, 2964, 3042, 3020, 2902, 2968, 3032, 3108] combine_rows_per_src=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344272 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165904 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987536 weight_movement_bytes=34603008 total_nvlink_bytes=568590544 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343888 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266092368 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840848 weight_movement_bytes=34603008 total_nvlink_bytes=566443856 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343248 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266902736 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533462224 weight_movement_bytes=69206016 total_nvlink_bytes=602668240 weight_to_token_ratio=0.129730 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343912 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846632 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349352 weight_movement_bytes=69206016 total_nvlink_bytes=600555368 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=345520 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267822512 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535299504 weight_movement_bytes=34603008 total_nvlink_bytes=569902512 weight_to_token_ratio=0.064642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64854 dup_rows=23882 dup_rows_remote=20656 dedup_ratio=0.242940 router_tokens_per_expert=[781, 787, 747, 803, 796, 772, 772, 773, 746, 734, 780, 771, 760, 804, 795, 779, 791, 751, 785, 762, 740, 722, 777, 782, 760, 761, 788, 798, 746, 741, 752, 748, 778, 747, 815, 767, 764, 769, 741, 751, 785, 712, 751, 749, 761, 765, 777, 789, 757, 770, 777, 759, 764, 770, 814, 783, 781, 769, 773, 776, 786, 710, 789, 749] payload_rows_per_dst=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] dup_rows_per_dst=[3226, 3010, 3030, 2898, 2992, 2888, 2968, 2870] combine_rows_per_src=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65208 dup_rows=23780 dup_rows_remote=20760 dedup_ratio=0.241903 router_tokens_per_expert=[755, 760, 763, 747, 813, 778, 717, 734, 748, 799, 807, 819, 704, 726, 788, 777, 766, 789, 772, 753, 819, 726, 783, 779, 752, 760, 729, 729, 802, 782, 760, 754, 762, 793, 776, 790, 781, 721, 759, 798, 766, 744, 747, 772, 745, 773, 813, 817, 818, 678, 805, 785, 768, 791, 715, 754, 794, 773, 754, 831, 725, 740, 772, 802] payload_rows_per_dst=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] dup_rows_per_dst=[2876, 3020, 3012, 2980, 2990, 3030, 2880, 2992] combine_rows_per_src=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64946 dup_rows=24046 dup_rows_remote=21060 dedup_ratio=0.244609 router_tokens_per_expert=[789, 757, 764, 797, 807, 735, 778, 759, 753, 745, 787, 803, 776, 819, 764, 726, 736, 763, 774, 806, 803, 756, 762, 793, 790, 743, 781, 772, 805, 768, 804, 723, 733, 715, 728, 767, 724, 802, 761, 753, 826, 734, 743, 750, 736, 777, 775, 808, 757, 783, 757, 773, 758, 770, 788, 789, 774, 763, 759, 786, 773, 736, 731, 785] payload_rows_per_dst=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] dup_rows_per_dst=[3026, 3072, 3070, 3068, 2790, 2986, 3038, 2996] combine_rows_per_src=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=342040 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265984024 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531626008 weight_movement_bytes=69206016 total_nvlink_bytes=600832024 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65394 dup_rows=23764 dup_rows_remote=20866 dedup_ratio=0.241740 router_tokens_per_expert=[745, 789, 768, 812, 759, 785, 748, 740, 775, 758, 760, 765, 803, 782, 745, 734, 792, 782, 713, 764, 762, 748, 715, 746, 733, 777, 788, 752, 753, 797, 752, 826, 770, 791, 739, 761, 799, 794, 799, 755, 764, 779, 801, 790, 781, 797, 741, 737, 783, 757, 806, 761, 774, 755, 761, 709, 749, 789, 798, 741, 777, 734, 836, 756] payload_rows_per_dst=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] dup_rows_per_dst=[2952, 2964, 2898, 2962, 2942, 3034, 2974, 3038] combine_rows_per_src=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65064 dup_rows=23962 dup_rows_remote=20876 dedup_ratio=0.243754 router_tokens_per_expert=[821, 753, 730, 769, 741, 721, 764, 778, 782, 806, 783, 776, 776, 754, 802, 788, 746, 731, 812, 768, 753, 740, 789, 752, 727, 748, 737, 805, 754, 760, 756, 736, 753, 765, 759, 762, 739, 814, 724, 751, 765, 774, 749, 790, 773, 795, 763, 751, 815, 789, 735, 804, 769, 754, 830, 789, 750, 775, 767, 761, 812, 768, 805, 744] payload_rows_per_dst=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] dup_rows_per_dst=[2934, 3124, 2986, 2882, 2854, 2918, 3178, 3086] combine_rows_per_src=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343872 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435840 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527808 weight_movement_bytes=103809024 total_nvlink_bytes=638336832 weight_to_token_ratio=0.194207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74646 payload_rows_remote=65332 dup_rows=23658 dup_rows_remote=20664 dedup_ratio=0.240662 router_tokens_per_expert=[736, 754, 806, 786, 750, 804, 791, 760, 726, 765, 745, 789, 739, 795, 783, 800, 787, 770, 783, 752, 746, 768, 778, 746, 745, 777, 758, 788, 722, 720, 790, 781, 709, 758, 741, 836, 741, 760, 737, 792, 769, 756, 737, 765, 782, 772, 761, 779, 794, 788, 762, 732, 775, 786, 788, 729, 819, 764, 770, 795, 775, 810, 765, 765] payload_rows_per_dst=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] dup_rows_per_dst=[2972, 2928, 3020, 2856, 2936, 2978, 2994, 2974] combine_rows_per_src=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344024 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362840 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381656 weight_movement_bytes=69206016 total_nvlink_bytes=601587672 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65074 dup_rows=24042 dup_rows_remote=21044 dedup_ratio=0.244568 router_tokens_per_expert=[802, 749, 743, 756, 762, 789, 732, 766, 757, 802, 747, 772, 775, 758, 774, 768, 760, 764, 739, 744, 753, 749, 800, 760, 775, 771, 710, 755, 770, 771, 796, 745, 766, 761, 788, 807, 765, 772, 769, 770, 761, 747, 835, 768, 744, 786, 778, 816, 767, 771, 768, 742, 784, 792, 791, 761, 799, 774, 772, 752, 764, 753, 764, 751] payload_rows_per_dst=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] dup_rows_per_dst=[2994, 3056, 2884, 2998, 3024, 3020, 3040, 3026] combine_rows_per_src=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64880 dup_rows=24034 dup_rows_remote=20988 dedup_ratio=0.244486 router_tokens_per_expert=[709, 730, 782, 791, 741, 794, 786, 772, 711, 775, 754, 758, 773, 751, 782, 772, 775, 775, 810, 791, 757, 748, 755, 795, 727, 752, 754, 763, 773, 771, 770, 782, 794, 784, 733, 777, 792, 803, 783, 752, 768, 779, 742, 811, 742, 761, 771, 789, 819, 765, 782, 722, 763, 785, 799, 763, 803, 735, 729, 770, 804, 767, 760, 726] payload_rows_per_dst=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] dup_rows_per_dst=[2976, 2976, 3130, 2928, 3046, 2996, 3084, 2898] combine_rows_per_src=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267853824 dispatch_meta_bytes=345040 combine_payload_bytes=267853824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268198864 ingress_bytes=302456832 link_bytes=302456832 token_bytes=536052688 weight_movement_bytes=34603008 total_nvlink_bytes=570655696 weight_to_token_ratio=0.064552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138120709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343760 combine_payload_bytes=266502144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266845904 ingress_bytes=370311168 link_bytes=370311168 token_bytes=533348048 weight_movement_bytes=103809024 total_nvlink_bytes=637157072 weight_to_token_ratio=0.194637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393451113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=343984 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267943856 ingress_bytes=336805888 link_bytes=336805888 token_bytes=535543728 weight_movement_bytes=69206016 total_nvlink_bytes=604749744 weight_to_token_ratio=0.129226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267373441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344472 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887576 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430680 weight_movement_bytes=34603008 total_nvlink_bytes=568033688 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64868 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[754, 754, 797, 813, 762, 773, 728, 786, 769, 748, 782, 754, 800, 803, 754, 778, 846, 780, 740, 761, 752, 762, 789, 804, 760, 745, 769, 693, 818, 748, 737, 812, 772, 747, 793, 777, 740, 785, 724, 732, 737, 811, 706, 763, 785, 729, 700, 779, 769, 798, 762, 772, 802, 761, 763, 778, 802, 775, 809, 777, 748, 769, 754, 762] payload_rows_per_dst=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] dup_rows_per_dst=[3060, 2916, 3084, 3292, 3054, 2860, 2732, 2928] combine_rows_per_src=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343472 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091952 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840432 weight_movement_bytes=34603008 total_nvlink_bytes=566443440 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64972 dup_rows=24016 dup_rows_remote=20906 dedup_ratio=0.244303 router_tokens_per_expert=[783, 799, 792, 778, 739, 744, 731, 758, 761, 764, 791, 777, 766, 784, 759, 811, 778, 749, 740, 772, 774, 706, 764, 764, 753, 729, 811, 791, 774, 771, 721, 768, 743, 735, 719, 789, 789, 760, 769, 728, 812, 767, 771, 802, 805, 789, 774, 762, 754, 787, 785, 779, 778, 759, 744, 736, 812, 757, 726, 808, 771, 736, 808, 796] payload_rows_per_dst=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] dup_rows_per_dst=[2972, 3110, 2944, 2884, 2924, 3056, 3016, 3110] combine_rows_per_src=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=20910 dedup_ratio=0.244649 router_tokens_per_expert=[778, 798, 719, 790, 815, 754, 759, 721, 741, 811, 752, 696, 787, 739, 775, 744, 766, 765, 792, 780, 736, 732, 769, 762, 751, 786, 807, 745, 832, 810, 810, 729, 755, 790, 771, 749, 789, 795, 790, 775, 776, 776, 787, 797, 793, 761, 766, 787, 832, 795, 749, 718, 756, 774, 754, 732, 787, 725, 755, 739, 780, 730, 751, 767] payload_rows_per_dst=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] dup_rows_per_dst=[2952, 2938, 2970, 3068, 3130, 3140, 2930, 2922] combine_rows_per_src=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64836 dup_rows=24206 dup_rows_remote=21112 dedup_ratio=0.246236 router_tokens_per_expert=[752, 754, 772, 762, 798, 789, 739, 799, 792, 800, 766, 791, 741, 720, 771, 772, 731, 818, 732, 771, 799, 761, 772, 794, 760, 806, 759, 756, 712, 757, 787, 769, 839, 753, 754, 752, 739, 733, 795, 768, 777, 782, 743, 764, 781, 795, 744, 772, 763, 772, 758, 755, 738, 746, 781, 760, 773, 723, 775, 770, 787, 795, 778, 785] payload_rows_per_dst=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] dup_rows_per_dst=[3010, 2988, 3094, 3000, 3034, 3040, 2938, 3102] combine_rows_per_src=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342936 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042264 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741592 weight_movement_bytes=34603008 total_nvlink_bytes=566344600 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64670 dup_rows=24238 dup_rows_remote=21172 dedup_ratio=0.246562 router_tokens_per_expert=[759, 802, 776, 762, 795, 770, 806, 775, 757, 734, 790, 759, 757, 752, 741, 722, 771, 745, 766, 794, 736, 802, 785, 755, 761, 834, 781, 732, 788, 751, 791, 798, 774, 775, 750, 753, 699, 768, 775, 762, 798, 780, 753, 811, 771, 819, 759, 807, 753, 761, 753, 751, 749, 816, 706, 739, 791, 776, 786, 779, 744, 746, 749, 752] payload_rows_per_dst=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] dup_rows_per_dst=[2928, 2916, 2990, 2814, 3034, 3202, 3288, 3066] combine_rows_per_src=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343512 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266468824 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594136 weight_movement_bytes=34603008 total_nvlink_bytes=567197144 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65144 dup_rows=23866 dup_rows_remote=20940 dedup_ratio=0.242778 router_tokens_per_expert=[813, 745, 742, 764, 801, 788, 776, 757, 742, 789, 796, 750, 782, 798, 830, 759, 723, 766, 759, 747, 782, 747, 766, 742, 753, 783, 771, 780, 785, 778, 776, 759, 776, 798, 787, 741, 755, 777, 748, 731, 829, 743, 798, 746, 725, 735, 772, 752, 789, 763, 757, 747, 754, 763, 751, 786, 795, 745, 753, 784, 755, 795, 766, 787] payload_rows_per_dst=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] dup_rows_per_dst=[3050, 3008, 2894, 3062, 2972, 2954, 2926, 3000] combine_rows_per_src=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343272 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206440 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069608 weight_movement_bytes=69206016 total_nvlink_bytes=601275624 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65130 dup_rows=23874 dup_rows_remote=20796 dedup_ratio=0.242859 router_tokens_per_expert=[737, 792, 805, 786, 772, 776, 775, 756, 769, 772, 767, 782, 760, 763, 772, 707, 769, 773, 769, 790, 740, 794, 779, 752, 720, 738, 812, 734, 826, 761, 810, 788, 789, 742, 744, 733, 757, 798, 793, 791, 760, 760, 770, 765, 727, 790, 749, 735, 757, 745, 787, 777, 775, 760, 758, 786, 728, 755, 757, 778, 793, 797, 806, 744] payload_rows_per_dst=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] dup_rows_per_dst=[3112, 2896, 3018, 3078, 2912, 2878, 2964, 3016] combine_rows_per_src=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64950 dup_rows=24046 dup_rows_remote=20904 dedup_ratio=0.244609 router_tokens_per_expert=[757, 746, 754, 704, 795, 806, 767, 748, 748, 728, 740, 765, 753, 777, 776, 780, 805, 811, 796, 758, 767, 780, 785, 817, 763, 778, 791, 786, 760, 776, 824, 712, 776, 749, 790, 753, 773, 780, 808, 796, 754, 744, 778, 749, 803, 733, 768, 765, 785, 756, 730, 772, 738, 747, 768, 832, 773, 740, 790, 760, 746, 738, 744, 761] payload_rows_per_dst=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] dup_rows_per_dst=[2906, 2898, 3194, 3014, 3142, 2964, 2980, 2948] combine_rows_per_src=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=343792 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912048 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480304 weight_movement_bytes=69206016 total_nvlink_bytes=600686320 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=343368 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265231688 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530120008 weight_movement_bytes=34603008 total_nvlink_bytes=564723016 weight_to_token_ratio=0.065274 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344336 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174160 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003984 weight_movement_bytes=34603008 total_nvlink_bytes=568606992 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343704 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267116184 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533888664 weight_movement_bytes=69206016 total_nvlink_bytes=603094680 weight_to_token_ratio=0.129626 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64882 dup_rows=24114 dup_rows_remote=21104 dedup_ratio=0.245300 router_tokens_per_expert=[813, 792, 765, 794, 806, 711, 755, 711, 823, 752, 772, 789, 781, 724, 707, 804, 751, 767, 763, 807, 775, 751, 806, 744, 751, 819, 755, 757, 736, 711, 743, 786, 786, 796, 803, 805, 755, 800, 772, 808, 757, 810, 757, 795, 725, 735, 742, 743, 763, 733, 788, 799, 771, 761, 732, 798, 795, 743, 780, 718, 761, 778, 774, 748] payload_rows_per_dst=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] dup_rows_per_dst=[3010, 3184, 2994, 3072, 3062, 2782, 3090, 2920] combine_rows_per_src=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343416 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378616 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413816 weight_movement_bytes=69206016 total_nvlink_bytes=601619832 weight_to_token_ratio=0.129985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64922 dup_rows=24032 dup_rows_remote=20896 dedup_ratio=0.244466 router_tokens_per_expert=[725, 752, 734, 749, 817, 767, 766, 777, 783, 794, 760, 805, 824, 780, 738, 759, 767, 794, 798, 762, 742, 737, 775, 795, 784, 766, 766, 793, 842, 786, 781, 730, 798, 744, 764, 798, 765, 749, 767, 755, 726, 790, 809, 746, 747, 747, 712, 742, 785, 745, 743, 742, 720, 772, 736, 771, 793, 790, 794, 743, 736, 754, 777, 844] payload_rows_per_dst=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] dup_rows_per_dst=[2866, 3136, 3096, 3156, 3036, 2894, 2862, 2986] combine_rows_per_src=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65120 dup_rows=23878 dup_rows_remote=20804 dedup_ratio=0.242900 router_tokens_per_expert=[765, 784, 767, 746, 766, 768, 768, 749, 758, 741, 794, 759, 730, 796, 768, 809, 747, 767, 748, 785, 709, 753, 814, 809, 796, 737, 710, 766, 746, 777, 780, 758, 787, 786, 746, 771, 767, 806, 790, 734, 799, 772, 784, 768, 777, 755, 802, 733, 748, 779, 756, 784, 717, 831, 753, 760, 804, 742, 769, 763, 761, 820, 720, 798] payload_rows_per_dst=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] dup_rows_per_dst=[2966, 2932, 2992, 2932, 3042, 3074, 2918, 3022] combine_rows_per_src=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343944 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100616 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531857288 weight_movement_bytes=103809024 total_nvlink_bytes=635666312 weight_to_token_ratio=0.195182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65080 dup_rows=23956 dup_rows_remote=21050 dedup_ratio=0.243693 router_tokens_per_expert=[772, 775, 742, 770, 781, 786, 792, 761, 790, 771, 848, 760, 738, 736, 776, 729, 730, 770, 720, 755, 754, 762, 836, 760, 737, 799, 747, 817, 742, 778, 750, 753, 767, 786, 765, 770, 773, 732, 792, 801, 755, 775, 717, 745, 742, 768, 784, 749, 751, 819, 775, 744, 800, 799, 792, 799, 717, 780, 771, 792, 778, 769, 735, 773] payload_rows_per_dst=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] dup_rows_per_dst=[3006, 3080, 2906, 3020, 3000, 2832, 3150, 2962] combine_rows_per_src=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64834 dup_rows=24074 dup_rows_remote=20912 dedup_ratio=0.244893 router_tokens_per_expert=[732, 741, 785, 796, 776, 774, 778, 788, 772, 761, 783, 762, 779, 750, 773, 767, 768, 792, 755, 741, 781, 769, 792, 713, 715, 760, 731, 777, 732, 769, 754, 801, 764, 796, 726, 740, 788, 789, 804, 778, 795, 799, 776, 793, 776, 773, 756, 745, 792, 779, 763, 785, 743, 751, 728, 850, 754, 787, 725, 763, 753, 732, 810, 772] payload_rows_per_dst=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] dup_rows_per_dst=[3034, 3014, 2936, 2958, 2816, 3150, 3004, 3162] combine_rows_per_src=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343272 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266263784 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532184296 weight_movement_bytes=69206016 total_nvlink_bytes=601390312 weight_to_token_ratio=0.130041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65230 dup_rows=23832 dup_rows_remote=21030 dedup_ratio=0.242432 router_tokens_per_expert=[730, 785, 769, 761, 734, 803, 796, 779, 798, 732, 743, 769, 785, 830, 791, 777, 768, 753, 821, 765, 763, 769, 749, 766, 785, 752, 776, 770, 767, 760, 757, 797, 812, 745, 789, 732, 800, 788, 769, 799, 756, 794, 746, 740, 761, 746, 794, 814, 820, 803, 722, 746, 728, 717, 749, 737, 738, 739, 720, 780, 748, 746, 772, 802] payload_rows_per_dst=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] dup_rows_per_dst=[2910, 3100, 2994, 3050, 3098, 2948, 2802, 2930] combine_rows_per_src=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65098 dup_rows=23958 dup_rows_remote=21000 dedup_ratio=0.243713 router_tokens_per_expert=[732, 774, 809, 766, 766, 778, 765, 756, 762, 761, 752, 786, 720, 810, 766, 738, 762, 809, 769, 746, 768, 772, 725, 755, 757, 795, 772, 757, 741, 770, 732, 779, 756, 727, 764, 742, 790, 801, 822, 740, 795, 765, 769, 767, 831, 826, 796, 774, 767, 760, 745, 780, 817, 769, 770, 774, 760, 765, 741, 743, 742, 785, 754, 765] payload_rows_per_dst=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] dup_rows_per_dst=[3044, 2830, 2942, 2958, 2962, 3142, 3088, 2992] combine_rows_per_src=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65120 dup_rows=23992 dup_rows_remote=20880 dedup_ratio=0.244059 router_tokens_per_expert=[793, 751, 788, 752, 793, 772, 777, 765, 750, 782, 755, 808, 751, 757, 737, 800, 734, 737, 752, 763, 769, 788, 756, 792, 768, 758, 810, 708, 749, 765, 766, 739, 776, 730, 745, 796, 792, 731, 770, 812, 749, 759, 787, 786, 816, 784, 792, 799, 751, 758, 768, 781, 765, 731, 805, 784, 751, 765, 767, 731, 787, 751, 760, 788] payload_rows_per_dst=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] dup_rows_per_dst=[2998, 2938, 2936, 2976, 3112, 3090, 2964, 2978] combine_rows_per_src=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344520 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266912200 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479880 weight_movement_bytes=34603008 total_nvlink_bytes=568082888 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342984 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265903048 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531463112 weight_movement_bytes=34603008 total_nvlink_bytes=566066120 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=345040 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267527120 ingress_bytes=336388096 link_bytes=336388096 token_bytes=534709200 weight_movement_bytes=69206016 total_nvlink_bytes=603915216 weight_to_token_ratio=0.129427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265801324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65028 dup_rows=23974 dup_rows_remote=21058 dedup_ratio=0.243876 router_tokens_per_expert=[731, 725, 732, 772, 767, 729, 752, 748, 740, 792, 719, 773, 748, 787, 767, 768, 756, 759, 797, 809, 777, 807, 777, 827, 757, 783, 736, 753, 808, 748, 775, 784, 757, 755, 772, 754, 775, 758, 754, 760, 730, 791, 781, 784, 782, 800, 731, 806, 756, 750, 781, 789, 712, 773, 835, 813, 791, 748, 792, 757, 756, 753, 792, 761] payload_rows_per_dst=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] dup_rows_per_dst=[2916, 2936, 3156, 3170, 2940, 2832, 3026, 2998] combine_rows_per_src=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344392 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985800 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627208 weight_movement_bytes=34603008 total_nvlink_bytes=568230216 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344000 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075520 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807040 weight_movement_bytes=69206016 total_nvlink_bytes=603013056 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64820 dup_rows=24114 dup_rows_remote=21082 dedup_ratio=0.245300 router_tokens_per_expert=[821, 749, 749, 778, 751, 803, 770, 745, 756, 780, 800, 748, 789, 759, 793, 776, 758, 768, 788, 759, 786, 728, 760, 793, 762, 759, 802, 744, 780, 739, 762, 779, 711, 767, 780, 742, 763, 753, 788, 787, 840, 784, 780, 760, 747, 780, 754, 782, 733, 759, 733, 782, 769, 741, 811, 765, 775, 748, 788, 745, 768, 745, 773, 765] payload_rows_per_dst=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] dup_rows_per_dst=[2958, 3032, 2988, 3050, 2988, 3168, 2910, 3020] combine_rows_per_src=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65186 dup_rows=23896 dup_rows_remote=20936 dedup_ratio=0.243083 router_tokens_per_expert=[756, 755, 786, 739, 787, 779, 762, 744, 758, 766, 768, 801, 773, 784, 743, 772, 766, 766, 725, 730, 806, 809, 816, 811, 778, 752, 755, 795, 788, 786, 796, 740, 807, 784, 791, 769, 770, 721, 759, 751, 754, 755, 738, 784, 749, 777, 769, 765, 739, 783, 808, 769, 794, 749, 776, 734, 745, 783, 743, 787, 749, 755, 752, 751] payload_rows_per_dst=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] dup_rows_per_dst=[2974, 2970, 3054, 3044, 2940, 2960, 3054, 2900] combine_rows_per_src=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344344 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699032 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053720 weight_movement_bytes=34603008 total_nvlink_bytes=567656728 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64754 dup_rows=24184 dup_rows_remote=21076 dedup_ratio=0.246012 router_tokens_per_expert=[778, 758, 721, 786, 721, 775, 755, 790, 768, 788, 768, 787, 751, 773, 754, 770, 751, 802, 827, 792, 765, 755, 763, 782, 806, 752, 703, 776, 796, 762, 775, 798, 785, 738, 835, 812, 767, 750, 726, 771, 731, 782, 797, 765, 714, 830, 768, 794, 769, 735, 768, 724, 756, 758, 768, 725, 748, 775, 742, 778, 770, 763, 792, 768] payload_rows_per_dst=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] dup_rows_per_dst=[2940, 3094, 3108, 3036, 3024, 3052, 2836, 3094] combine_rows_per_src=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=64968 dup_rows=23874 dup_rows_remote=20604 dedup_ratio=0.242859 router_tokens_per_expert=[777, 796, 754, 757, 744, 742, 722, 777, 775, 779, 789, 806, 787, 780, 757, 750, 750, 813, 803, 760, 830, 736, 758, 793, 738, 779, 790, 773, 715, 796, 741, 761, 790, 774, 766, 727, 735, 816, 770, 730, 788, 752, 751, 768, 769, 778, 764, 758, 800, 734, 768, 795, 742, 780, 801, 809, 740, 765, 732, 733, 789, 775, 761, 764] payload_rows_per_dst=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] dup_rows_per_dst=[2818, 2900, 3102, 2652, 3186, 2980, 2966, 3270] combine_rows_per_src=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343608 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846328 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349048 weight_movement_bytes=69206016 total_nvlink_bytes=600555064 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65044 dup_rows=23948 dup_rows_remote=21066 dedup_ratio=0.243612 router_tokens_per_expert=[814, 739, 795, 760, 792, 754, 783, 758, 802, 741, 796, 779, 760, 745, 794, 768, 770, 762, 759, 771, 755, 728, 735, 754, 808, 783, 785, 778, 772, 797, 804, 783, 744, 739, 786, 755, 785, 773, 772, 762, 774, 775, 806, 747, 779, 754, 745, 760, 743, 790, 778, 770, 790, 738, 731, 757, 752, 795, 784, 690, 764, 721, 782, 787] payload_rows_per_dst=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] dup_rows_per_dst=[2972, 3062, 2866, 3200, 3012, 2996, 2882, 2958] combine_rows_per_src=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344488 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346344 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348200 weight_movement_bytes=69206016 total_nvlink_bytes=603554216 weight_to_token_ratio=0.129515 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65060 dup_rows=24050 dup_rows_remote=20986 dedup_ratio=0.244649 router_tokens_per_expert=[787, 766, 778, 770, 798, 743, 768, 767, 718, 761, 770, 805, 762, 762, 759, 809, 754, 740, 768, 741, 763, 783, 779, 712, 772, 725, 774, 786, 732, 768, 790, 782, 771, 765, 745, 763, 747, 795, 712, 784, 781, 739, 766, 752, 777, 728, 813, 809, 787, 780, 755, 778, 753, 804, 795, 725, 794, 805, 778, 767, 749, 775, 784, 784] payload_rows_per_dst=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] dup_rows_per_dst=[2948, 3018, 2884, 3064, 2950, 3072, 3076, 3038] combine_rows_per_src=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65088 dup_rows=24050 dup_rows_remote=21118 dedup_ratio=0.244649 router_tokens_per_expert=[796, 811, 742, 810, 805, 743, 762, 777, 776, 724, 768, 772, 726, 827, 754, 852, 754, 763, 809, 806, 806, 727, 687, 739, 762, 758, 771, 746, 803, 763, 791, 774, 750, 758, 730, 777, 765, 749, 765, 755, 749, 760, 854, 761, 808, 810, 758, 785, 780, 752, 773, 758, 770, 723, 744, 793, 748, 767, 735, 747, 727, 775, 782, 740] payload_rows_per_dst=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] dup_rows_per_dst=[3118, 3076, 2914, 3026, 2932, 3090, 3016, 2878] combine_rows_per_src=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=343320 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265575704 ingress_bytes=334438400 link_bytes=334438400 token_bytes=530808088 weight_movement_bytes=69206016 total_nvlink_bytes=600014104 weight_to_token_ratio=0.130379 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258464775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342288 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266451216 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532560144 weight_movement_bytes=34603008 total_nvlink_bytes=567163152 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344440 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764664 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184888 weight_movement_bytes=34603008 total_nvlink_bytes=567787896 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64852 dup_rows=24114 dup_rows_remote=21112 dedup_ratio=0.245300 router_tokens_per_expert=[765, 807, 769, 763, 750, 727, 830, 755, 725, 745, 803, 758, 745, 735, 761, 725, 811, 756, 779, 747, 821, 749, 792, 752, 783, 745, 762, 784, 759, 783, 735, 762, 764, 742, 815, 770, 770, 750, 804, 744, 733, 771, 730, 750, 821, 748, 768, 731, 792, 827, 791, 761, 760, 806, 750, 825, 782, 751, 731, 777, 780, 780, 760, 785] payload_rows_per_dst=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] dup_rows_per_dst=[3002, 2982, 2990, 3252, 3000, 2762, 3106, 3020] combine_rows_per_src=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344184 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266829944 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315704 weight_movement_bytes=69206016 total_nvlink_bytes=602521720 weight_to_token_ratio=0.129766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=64950 dup_rows=23890 dup_rows_remote=20754 dedup_ratio=0.243022 router_tokens_per_expert=[758, 749, 784, 725, 741, 842, 756, 763, 779, 784, 802, 743, 822, 781, 793, 796, 794, 746, 799, 761, 773, 799, 766, 695, 816, 769, 760, 784, 780, 747, 773, 754, 789, 775, 747, 785, 734, 769, 801, 774, 780, 752, 729, 774, 736, 711, 766, 748, 708, 785, 808, 757, 748, 779, 748, 787, 755, 814, 759, 761, 766, 780, 763, 730] payload_rows_per_dst=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] dup_rows_per_dst=[3056, 3136, 2936, 2976, 3042, 2850, 2950, 2944] combine_rows_per_src=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=343856 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265977648 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611440 weight_movement_bytes=103809024 total_nvlink_bytes=635420464 weight_to_token_ratio=0.195272 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65108 dup_rows=23846 dup_rows_remote=20922 dedup_ratio=0.242574 router_tokens_per_expert=[765, 761, 787, 744, 753, 769, 713, 785, 800, 756, 765, 812, 788, 712, 801, 739, 778, 692, 777, 757, 749, 776, 777, 808, 794, 777, 757, 766, 761, 789, 720, 761, 761, 774, 769, 767, 818, 778, 719, 772, 770, 755, 808, 751, 797, 736, 779, 741, 788, 768, 747, 748, 757, 743, 785, 775, 782, 773, 776, 808, 784, 776, 755, 803] payload_rows_per_dst=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] dup_rows_per_dst=[2948, 2972, 2944, 2980, 3054, 2924, 2900, 3124] combine_rows_per_src=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64998 dup_rows=24096 dup_rows_remote=21130 dedup_ratio=0.245117 router_tokens_per_expert=[711, 775, 771, 727, 763, 837, 760, 780, 753, 779, 744, 754, 746, 811, 757, 767, 743, 732, 693, 776, 770, 836, 777, 761, 746, 774, 743, 776, 734, 734, 739, 749, 781, 749, 804, 789, 770, 753, 744, 728, 767, 788, 795, 793, 785, 758, 789, 779, 794, 769, 757, 747, 774, 804, 809, 780, 773, 761, 803, 794, 753, 757, 793, 794] payload_rows_per_dst=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] dup_rows_per_dst=[2982, 2950, 2966, 2864, 2994, 3188, 3122, 3030] combine_rows_per_src=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=342816 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378016 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532413216 weight_movement_bytes=34603008 total_nvlink_bytes=567016224 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65150 dup_rows=23796 dup_rows_remote=20764 dedup_ratio=0.242065 router_tokens_per_expert=[767, 732, 742, 784, 699, 822, 756, 703, 783, 779, 786, 710, 772, 745, 753, 768, 766, 742, 723, 777, 764, 726, 790, 728, 816, 777, 795, 807, 788, 736, 793, 826, 798, 754, 788, 789, 754, 769, 773, 792, 783, 759, 731, 784, 731, 720, 811, 774, 776, 744, 724, 787, 803, 792, 797, 773, 760, 768, 764, 761, 783, 763, 818, 774] payload_rows_per_dst=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] dup_rows_per_dst=[2850, 2970, 2844, 3178, 3004, 2892, 3026, 3032] combine_rows_per_src=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=65034 dup_rows=24170 dup_rows_remote=21436 dedup_ratio=0.245870 router_tokens_per_expert=[790, 757, 756, 780, 753, 827, 816, 769, 781, 756, 769, 786, 731, 722, 736, 813, 774, 803, 747, 741, 798, 747, 764, 774, 784, 754, 748, 749, 733, 800, 776, 772, 750, 798, 744, 746, 751, 791, 768, 784, 761, 811, 789, 781, 782, 807, 778, 808, 737, 731, 717, 733, 761, 749, 725, 764, 760, 739, 746, 800, 794, 780, 768, 793] payload_rows_per_dst=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] dup_rows_per_dst=[3114, 2964, 3046, 3032, 2974, 3234, 2734, 3072] combine_rows_per_src=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344120 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026488 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708856 weight_movement_bytes=34603008 total_nvlink_bytes=568311864 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64866 dup_rows=24164 dup_rows_remote=21080 dedup_ratio=0.245809 router_tokens_per_expert=[735, 760, 754, 786, 748, 716, 794, 761, 769, 771, 790, 774, 735, 738, 727, 753, 802, 771, 777, 762, 751, 747, 786, 766, 763, 777, 736, 856, 727, 784, 779, 757, 784, 782, 761, 725, 763, 776, 819, 780, 772, 771, 781, 748, 766, 741, 739, 780, 796, 786, 760, 768, 762, 738, 777, 766, 763, 844, 762, 739, 772, 814, 765, 800] payload_rows_per_dst=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] dup_rows_per_dst=[2908, 3010, 3144, 3084, 3076, 2896, 2954, 3092] combine_rows_per_src=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344512 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576320 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808128 weight_movement_bytes=34603008 total_nvlink_bytes=567411136 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65134 dup_rows=23762 dup_rows_remote=20718 dedup_ratio=0.241720 router_tokens_per_expert=[779, 739, 784, 753, 769, 755, 766, 776, 784, 736, 735, 789, 743, 766, 817, 729, 757, 800, 807, 774, 736, 767, 815, 743, 782, 720, 744, 789, 740, 784, 816, 783, 797, 778, 742, 760, 757, 792, 815, 785, 746, 793, 773, 752, 747, 772, 770, 779, 803, 733, 758, 729, 700, 756, 761, 808, 761, 779, 811, 777, 748, 775, 750, 768] payload_rows_per_dst=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] dup_rows_per_dst=[2936, 2964, 3048, 3010, 3044, 2946, 2834, 2980] combine_rows_per_src=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343656 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=267198056 ingress_bytes=439869440 link_bytes=439869440 token_bytes=534052456 weight_movement_bytes=173015040 total_nvlink_bytes=707067496 weight_to_token_ratio=0.323966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.655193290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64586 dup_rows=24138 dup_rows_remote=20908 dedup_ratio=0.245544 router_tokens_per_expert=[751, 793, 756, 796, 774, 723, 791, 758, 778, 782, 763, 780, 784, 784, 756, 750, 744, 785, 752, 773, 799, 743, 785, 770, 738, 771, 777, 734, 767, 754, 746, 739, 774, 766, 747, 739, 778, 819, 738, 753, 780, 816, 702, 757, 777, 762, 712, 792, 797, 811, 801, 797, 769, 790, 781, 741, 760, 767, 775, 802, 749, 744, 785, 775] payload_rows_per_dst=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] dup_rows_per_dst=[3230, 3142, 3148, 2944, 2804, 2954, 3174, 2742] combine_rows_per_src=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=345880 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266725144 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533104408 weight_movement_bytes=34603008 total_nvlink_bytes=567707416 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343784 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034920 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531726056 weight_movement_bytes=34603008 total_nvlink_bytes=566329064 weight_to_token_ratio=0.065077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343408 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267132272 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921136 weight_movement_bytes=103809024 total_nvlink_bytes=637730160 weight_to_token_ratio=0.194428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65002 dup_rows=24016 dup_rows_remote=21004 dedup_ratio=0.244303 router_tokens_per_expert=[768, 763, 764, 760, 737, 792, 713, 756, 769, 854, 761, 728, 781, 749, 741, 766, 772, 811, 764, 759, 789, 800, 733, 754, 747, 751, 764, 756, 831, 780, 730, 782, 810, 749, 759, 710, 779, 723, 808, 767, 778, 775, 793, 769, 823, 728, 780, 747, 785, 743, 776, 803, 753, 752, 750, 765, 744, 765, 791, 751, 818, 777, 783, 773] payload_rows_per_dst=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] dup_rows_per_dst=[2864, 3012, 3006, 2990, 2988, 3128, 3004, 3024] combine_rows_per_src=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65134 dup_rows=23876 dup_rows_remote=20894 dedup_ratio=0.242879 router_tokens_per_expert=[752, 746, 800, 785, 787, 831, 783, 792, 786, 793, 801, 727, 772, 789, 771, 737, 770, 774, 805, 751, 748, 766, 778, 744, 749, 776, 748, 754, 769, 790, 776, 748, 770, 731, 747, 741, 789, 778, 782, 804, 749, 758, 759, 790, 773, 755, 775, 779, 753, 761, 781, 703, 720, 741, 814, 760, 794, 755, 746, 760, 752, 772, 800, 762] payload_rows_per_dst=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] dup_rows_per_dst=[3038, 3050, 2942, 3018, 2988, 2982, 2838, 3020] combine_rows_per_src=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=341976 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886232 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430488 weight_movement_bytes=34603008 total_nvlink_bytes=564033496 weight_to_token_ratio=0.065359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65262 dup_rows=23758 dup_rows_remote=20714 dedup_ratio=0.241679 router_tokens_per_expert=[778, 749, 738, 756, 766, 770, 743, 857, 796, 783, 763, 825, 804, 747, 733, 730, 731, 773, 797, 798, 755, 781, 800, 729, 745, 774, 755, 784, 758, 762, 778, 779, 809, 765, 792, 793, 768, 770, 755, 765, 759, 755, 775, 731, 779, 739, 777, 758, 730, 748, 721, 733, 739, 824, 771, 812, 784, 758, 828, 761, 755, 770, 739, 752] payload_rows_per_dst=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] dup_rows_per_dst=[2976, 3016, 3044, 2944, 2998, 2906, 2826, 3048] combine_rows_per_src=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344024 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592216 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840408 weight_movement_bytes=34603008 total_nvlink_bytes=567443416 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64958 dup_rows=23952 dup_rows_remote=20864 dedup_ratio=0.243652 router_tokens_per_expert=[735, 772, 781, 776, 791, 801, 783, 766, 774, 763, 761, 815, 751, 758, 732, 765, 750, 799, 752, 725, 763, 802, 783, 773, 751, 815, 805, 736, 816, 785, 804, 730, 761, 746, 808, 779, 737, 800, 719, 727, 789, 772, 708, 735, 771, 747, 775, 749, 721, 806, 770, 784, 719, 776, 739, 841, 770, 744, 802, 776, 781, 759, 758, 770] payload_rows_per_dst=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] dup_rows_per_dst=[2920, 3062, 2948, 2978, 2922, 2878, 3156, 3088] combine_rows_per_src=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344112 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132976 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921840 weight_movement_bytes=34603008 total_nvlink_bytes=568524848 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21184 dedup_ratio=0.246195 router_tokens_per_expert=[811, 767, 770, 814, 765, 778, 766, 748, 756, 728, 759, 758, 738, 720, 765, 744, 778, 789, 740, 769, 791, 749, 803, 777, 777, 721, 794, 782, 778, 719, 752, 716, 788, 778, 760, 757, 750, 788, 816, 796, 745, 751, 740, 746, 788, 729, 793, 765, 739, 777, 811, 798, 748, 751, 771, 755, 750, 775, 763, 742, 851, 826, 808, 775] payload_rows_per_dst=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] dup_rows_per_dst=[3152, 2858, 3122, 2938, 2990, 2976, 3018, 3148] combine_rows_per_src=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=343904 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267657056 ingress_bytes=371122176 link_bytes=371122176 token_bytes=534970208 weight_movement_bytes=103809024 total_nvlink_bytes=638779232 weight_to_token_ratio=0.194046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65014 dup_rows=23834 dup_rows_remote=20738 dedup_ratio=0.242452 router_tokens_per_expert=[751, 729, 750, 777, 759, 732, 796, 758, 786, 787, 765, 743, 798, 784, 754, 749, 755, 750, 764, 764, 796, 808, 764, 757, 787, 789, 807, 769, 790, 770, 784, 780, 792, 773, 761, 792, 795, 798, 760, 743, 739, 760, 766, 783, 753, 750, 779, 783, 754, 790, 787, 719, 739, 725, 762, 768, 785, 756, 767, 728, 783, 762, 755, 793] payload_rows_per_dst=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] dup_rows_per_dst=[2898, 3098, 3018, 3096, 3046, 2896, 2838, 2944] combine_rows_per_src=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21002 dedup_ratio=0.244873 router_tokens_per_expert=[770, 758, 761, 732, 770, 794, 746, 704, 799, 769, 696, 753, 738, 769, 775, 770, 748, 788, 793, 808, 734, 797, 788, 702, 789, 776, 781, 760, 770, 744, 758, 761, 764, 790, 813, 777, 740, 814, 745, 749, 697, 763, 779, 736, 788, 743, 781, 804, 772, 776, 818, 784, 780, 752, 762, 765, 769, 791, 823, 741, 762, 817, 764, 792] payload_rows_per_dst=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] dup_rows_per_dst=[2970, 2884, 2998, 3010, 3070, 2918, 3078, 3144] combine_rows_per_src=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343288 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411256 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479224 weight_movement_bytes=69206016 total_nvlink_bytes=601685240 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65082 dup_rows=23786 dup_rows_remote=20492 dedup_ratio=0.241964 router_tokens_per_expert=[759, 779, 760, 802, 808, 804, 769, 792, 766, 784, 811, 796, 734, 801, 806, 763, 743, 777, 749, 755, 764, 780, 802, 759, 738, 739, 771, 798, 752, 804, 746, 804, 697, 769, 772, 782, 772, 769, 682, 765, 774, 717, 782, 779, 758, 771, 763, 741, 731, 802, 796, 764, 806, 717, 745, 784, 765, 768, 785, 758, 760, 801, 725, 737] payload_rows_per_dst=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] dup_rows_per_dst=[3294, 3078, 2842, 2876, 2770, 3326, 3008, 2592] combine_rows_per_src=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344016 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846736 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349456 weight_movement_bytes=34603008 total_nvlink_bytes=565952464 weight_to_token_ratio=0.065123 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343008 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266640352 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532937696 weight_movement_bytes=69206016 total_nvlink_bytes=602143712 weight_to_token_ratio=0.129858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65174 dup_rows=23830 dup_rows_remote=20808 dedup_ratio=0.242411 router_tokens_per_expert=[806, 779, 737, 765, 784, 761, 785, 773, 763, 782, 752, 805, 763, 723, 788, 785, 765, 722, 748, 775, 791, 733, 737, 753, 734, 759, 863, 763, 802, 813, 786, 746, 714, 767, 791, 784, 776, 736, 737, 721, 790, 739, 757, 797, 776, 786, 767, 772, 789, 736, 786, 753, 792, 782, 758, 792, 783, 716, 792, 775, 741, 789, 759, 758] payload_rows_per_dst=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] dup_rows_per_dst=[3014, 3022, 2868, 3118, 2874, 2964, 3058, 2912] combine_rows_per_src=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343680 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247808 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151936 weight_movement_bytes=69206016 total_nvlink_bytes=601357952 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65282 dup_rows=23910 dup_rows_remote=20946 dedup_ratio=0.243225 router_tokens_per_expert=[759, 759, 780, 782, 773, 766, 777, 783, 758, 767, 778, 784, 774, 724, 771, 725, 760, 752, 802, 769, 769, 787, 713, 808, 756, 791, 728, 801, 760, 780, 812, 766, 751, 725, 761, 763, 775, 746, 778, 784, 727, 772, 794, 759, 744, 777, 730, 735, 787, 756, 798, 761, 740, 779, 786, 745, 770, 850, 787, 723, 796, 777, 804, 758] payload_rows_per_dst=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] dup_rows_per_dst=[3006, 2912, 2976, 3012, 2998, 2964, 2982, 3060] combine_rows_per_src=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64918 dup_rows=23930 dup_rows_remote=20874 dedup_ratio=0.243429 router_tokens_per_expert=[750, 773, 790, 810, 782, 760, 776, 789, 778, 817, 775, 738, 732, 756, 736, 716, 735, 837, 764, 781, 803, 783, 795, 758, 776, 747, 787, 761, 766, 739, 791, 793, 772, 730, 766, 735, 760, 695, 747, 742, 769, 798, 694, 724, 802, 761, 789, 770, 801, 783, 735, 724, 749, 786, 740, 770, 776, 757, 816, 835, 833, 769, 744, 786] payload_rows_per_dst=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] dup_rows_per_dst=[3112, 2836, 3056, 3018, 2778, 2998, 2896, 3236] combine_rows_per_src=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=342296 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266918168 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494040 weight_movement_bytes=69206016 total_nvlink_bytes=602700056 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343928 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296632 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249336 weight_movement_bytes=34603008 total_nvlink_bytes=568852344 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65168 dup_rows=23836 dup_rows_remote=20794 dedup_ratio=0.242472 router_tokens_per_expert=[760, 776, 739, 795, 772, 752, 766, 794, 832, 751, 732, 780, 732, 725, 765, 807, 748, 779, 751, 749, 745, 796, 747, 797, 779, 824, 773, 766, 810, 775, 785, 749, 789, 791, 762, 754, 762, 735, 785, 749, 692, 772, 766, 717, 774, 751, 751, 772, 790, 743, 746, 742, 786, 781, 799, 829, 771, 742, 764, 770, 794, 768, 772, 782] payload_rows_per_dst=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] dup_rows_per_dst=[3024, 2932, 2930, 2862, 3122, 2902, 3022, 3042] combine_rows_per_src=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344912 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739984 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535135056 weight_movement_bytes=34603008 total_nvlink_bytes=569738064 weight_to_token_ratio=0.064662 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343168 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247296 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151424 weight_movement_bytes=34603008 total_nvlink_bytes=566754432 weight_to_token_ratio=0.065025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65128 dup_rows=23872 dup_rows_remote=20900 dedup_ratio=0.242839 router_tokens_per_expert=[759, 754, 778, 793, 781, 749, 720, 722, 756, 758, 745, 770, 781, 738, 774, 789, 755, 766, 807, 748, 805, 714, 745, 753, 796, 756, 744, 757, 785, 799, 773, 785, 802, 771, 760, 759, 801, 788, 746, 770, 754, 763, 791, 801, 759, 772, 788, 746, 765, 790, 753, 816, 726, 761, 766, 761, 760, 729, 774, 820, 758, 801, 771, 775] payload_rows_per_dst=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] dup_rows_per_dst=[2886, 2982, 2994, 3048, 2984, 3018, 2972, 2988] combine_rows_per_src=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65146 dup_rows=23786 dup_rows_remote=20788 dedup_ratio=0.241964 router_tokens_per_expert=[792, 766, 796, 785, 753, 756, 794, 719, 796, 750, 692, 790, 747, 779, 780, 746, 758, 753, 790, 780, 780, 765, 770, 763, 750, 791, 753, 772, 745, 725, 790, 786, 772, 761, 820, 752, 769, 784, 764, 763, 751, 736, 826, 801, 791, 804, 761, 739, 762, 755, 741, 747, 742, 805, 779, 808, 785, 728, 804, 745, 767, 791, 741, 746] payload_rows_per_dst=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] dup_rows_per_dst=[2948, 2816, 2990, 3010, 2998, 3076, 3016, 2932] combine_rows_per_src=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64820 dup_rows=24200 dup_rows_remote=21192 dedup_ratio=0.246175 router_tokens_per_expert=[765, 794, 752, 795, 719, 795, 748, 733, 768, 772, 726, 751, 805, 769, 718, 765, 767, 804, 727, 760, 754, 763, 758, 787, 761, 799, 743, 740, 786, 730, 750, 837, 782, 770, 768, 775, 824, 781, 770, 745, 720, 741, 781, 804, 765, 733, 789, 741, 785, 762, 774, 757, 776, 746, 798, 761, 771, 775, 841, 746, 774, 805, 804, 747] payload_rows_per_dst=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] dup_rows_per_dst=[2978, 3004, 2998, 3008, 3104, 3014, 3038, 3056] combine_rows_per_src=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=343848 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267271976 ingress_bytes=405340160 link_bytes=405340160 token_bytes=534200104 weight_movement_bytes=138412032 total_nvlink_bytes=672612136 weight_to_token_ratio=0.259101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525262390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344112 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108400 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872688 weight_movement_bytes=69206016 total_nvlink_bytes=603078704 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64978 dup_rows=24196 dup_rows_remote=21148 dedup_ratio=0.246134 router_tokens_per_expert=[778, 787, 741, 784, 750, 766, 758, 725, 774, 797, 790, 751, 753, 738, 803, 810, 771, 776, 766, 773, 771, 727, 753, 737, 778, 788, 757, 740, 751, 749, 776, 783, 758, 769, 741, 804, 741, 759, 803, 812, 798, 746, 801, 773, 777, 790, 776, 768, 711, 773, 791, 720, 814, 747, 799, 807, 734, 781, 745, 754, 766, 755, 764, 774] payload_rows_per_dst=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] dup_rows_per_dst=[3048, 3364, 3078, 2882, 3108, 3126, 2878, 2712] combine_rows_per_src=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64956 dup_rows=24044 dup_rows_remote=20970 dedup_ratio=0.244588 router_tokens_per_expert=[752, 712, 747, 789, 763, 735, 804, 722, 738, 760, 812, 749, 799, 788, 784, 759, 731, 743, 759, 737, 749, 736, 749, 762, 757, 771, 775, 765, 780, 775, 820, 753, 778, 737, 775, 830, 775, 784, 793, 794, 750, 749, 749, 744, 781, 788, 829, 816, 782, 790, 776, 730, 760, 854, 760, 761, 752, 789, 767, 748, 701, 779, 794, 762] payload_rows_per_dst=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] dup_rows_per_dst=[2806, 3074, 2842, 3118, 3028, 3096, 3092, 2988] combine_rows_per_src=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343736 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181752 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019768 weight_movement_bytes=34603008 total_nvlink_bytes=568622776 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344048 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846768 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349488 weight_movement_bytes=103809024 total_nvlink_bytes=635158512 weight_to_token_ratio=0.195369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65188 dup_rows=23802 dup_rows_remote=20842 dedup_ratio=0.242126 router_tokens_per_expert=[771, 759, 744, 791, 771, 783, 754, 784, 771, 772, 766, 778, 757, 783, 769, 800, 764, 730, 791, 741, 749, 780, 808, 778, 757, 765, 750, 791, 750, 746, 772, 742, 794, 764, 741, 730, 766, 750, 706, 760, 771, 753, 778, 806, 781, 778, 765, 705, 755, 750, 836, 783, 749, 751, 790, 765, 804, 782, 788, 788, 747, 775, 789, 785] payload_rows_per_dst=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] dup_rows_per_dst=[2980, 3060, 2966, 2876, 2926, 2960, 3054, 2980] combine_rows_per_src=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65102 dup_rows=24026 dup_rows_remote=21116 dedup_ratio=0.244405 router_tokens_per_expert=[742, 773, 778, 782, 803, 771, 779, 783, 764, 786, 780, 733, 764, 818, 772, 787, 774, 757, 794, 745, 732, 753, 741, 747, 733, 812, 763, 769, 785, 748, 777, 735, 779, 746, 786, 799, 723, 828, 784, 752, 795, 743, 783, 779, 788, 746, 787, 787, 714, 723, 783, 772, 806, 706, 763, 813, 758, 769, 746, 763, 801, 736, 727, 787] payload_rows_per_dst=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] dup_rows_per_dst=[3086, 3132, 2910, 3006, 3050, 3026, 2884, 2932] combine_rows_per_src=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344504 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266494392 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644280 weight_movement_bytes=34603008 total_nvlink_bytes=567247288 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343704 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403480 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463256 weight_movement_bytes=103809024 total_nvlink_bytes=636272280 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64754 dup_rows=24190 dup_rows_remote=20928 dedup_ratio=0.246073 router_tokens_per_expert=[775, 780, 784, 806, 759, 750, 769, 783, 816, 837, 787, 786, 720, 769, 757, 769, 745, 777, 831, 744, 742, 791, 775, 747, 770, 821, 721, 757, 756, 749, 781, 784, 816, 803, 798, 791, 761, 712, 707, 812, 735, 771, 764, 789, 725, 799, 732, 795, 743, 718, 796, 776, 707, 755, 761, 776, 772, 733, 804, 724, 766, 780, 728, 765] payload_rows_per_dst=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] dup_rows_per_dst=[3052, 3030, 3028, 2858, 3062, 2958, 2940, 3262] combine_rows_per_src=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344120 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354168 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364216 weight_movement_bytes=34603008 total_nvlink_bytes=568967224 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344872 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002664 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660456 weight_movement_bytes=34603008 total_nvlink_bytes=568263464 weight_to_token_ratio=0.064841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64742 dup_rows=24168 dup_rows_remote=21118 dedup_ratio=0.245850 router_tokens_per_expert=[745, 741, 756, 794, 767, 756, 773, 735, 794, 751, 760, 780, 769, 718, 759, 769, 779, 805, 754, 801, 722, 765, 744, 776, 733, 808, 784, 805, 807, 763, 814, 786, 722, 760, 738, 805, 789, 753, 750, 800, 734, 778, 795, 781, 748, 752, 762, 763, 805, 797, 752, 753, 828, 724, 780, 783, 774, 802, 725, 753, 799, 737, 742, 755] payload_rows_per_dst=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] dup_rows_per_dst=[2946, 2964, 3040, 3216, 3006, 3000, 3050, 2946] combine_rows_per_src=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64894 dup_rows=24050 dup_rows_remote=21040 dedup_ratio=0.244649 router_tokens_per_expert=[771, 758, 772, 757, 790, 745, 761, 768, 734, 814, 761, 776, 769, 793, 752, 763, 820, 771, 760, 780, 765, 788, 759, 756, 819, 729, 794, 825, 814, 730, 754, 731, 746, 746, 812, 755, 789, 804, 735, 798, 771, 746, 819, 760, 750, 774, 796, 814, 764, 738, 757, 769, 718, 749, 781, 805, 748, 765, 738, 753, 776, 743, 734, 720] payload_rows_per_dst=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] dup_rows_per_dst=[3008, 3066, 3088, 3030, 3010, 3084, 2904, 2860] combine_rows_per_src=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65060 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[750, 747, 834, 738, 758, 827, 788, 727, 787, 757, 778, 746, 732, 800, 779, 741, 812, 799, 767, 738, 760, 718, 782, 792, 783, 738, 741, 724, 841, 765, 762, 747, 745, 782, 753, 731, 772, 781, 771, 775, 778, 784, 789, 734, 808, 783, 728, 767, 777, 739, 775, 771, 724, 822, 777, 762, 744, 766, 794, 801, 812, 772, 751, 726] payload_rows_per_dst=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] dup_rows_per_dst=[2888, 3026, 3072, 2968, 2954, 3098, 2994, 3010] combine_rows_per_src=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=342728 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265575112 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530807496 weight_movement_bytes=34603008 total_nvlink_bytes=565410504 weight_to_token_ratio=0.065189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=343440 combine_payload_bytes=265183232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265526672 ingress_bytes=334389248 link_bytes=334389248 token_bytes=530709904 weight_movement_bytes=69206016 total_nvlink_bytes=599915920 weight_to_token_ratio=0.130403 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258279820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343736 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149560 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955384 weight_movement_bytes=103809024 total_nvlink_bytes=635764408 weight_to_token_ratio=0.195146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344408 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266830168 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315928 weight_movement_bytes=69206016 total_nvlink_bytes=602521944 weight_to_token_ratio=0.129766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65018 dup_rows=23998 dup_rows_remote=21022 dedup_ratio=0.244120 router_tokens_per_expert=[794, 735, 798, 778, 773, 776, 760, 771, 757, 746, 766, 811, 753, 793, 711, 795, 782, 794, 785, 747, 795, 823, 776, 757, 795, 727, 759, 727, 739, 775, 736, 741, 800, 747, 778, 738, 806, 802, 814, 711, 787, 772, 796, 683, 732, 718, 766, 796, 789, 796, 798, 685, 755, 823, 757, 747, 749, 776, 754, 750, 754, 781, 874, 743] payload_rows_per_dst=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] dup_rows_per_dst=[3038, 2976, 3130, 2906, 3116, 2834, 2956, 3042] combine_rows_per_src=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65296 dup_rows=23700 dup_rows_remote=20716 dedup_ratio=0.241089 router_tokens_per_expert=[733, 743, 759, 768, 781, 811, 848, 736, 818, 745, 807, 790, 765, 739, 779, 721, 779, 772, 766, 766, 743, 731, 799, 735, 733, 786, 726, 757, 793, 704, 762, 801, 789, 775, 748, 774, 750, 773, 754, 761, 804, 772, 782, 785, 762, 732, 761, 748, 802, 787, 778, 798, 777, 783, 775, 820, 769, 791, 706, 721, 764, 746, 795, 774] payload_rows_per_dst=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] dup_rows_per_dst=[2988, 2990, 2918, 2926, 2928, 2984, 3082, 2884] combine_rows_per_src=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64944 dup_rows=24088 dup_rows_remote=21124 dedup_ratio=0.245036 router_tokens_per_expert=[742, 742, 723, 786, 763, 830, 779, 787, 807, 740, 765, 773, 764, 817, 766, 758, 757, 786, 721, 799, 765, 774, 765, 751, 724, 770, 719, 734, 775, 789, 758, 773, 786, 796, 760, 770, 792, 775, 773, 779, 745, 765, 757, 783, 769, 790, 741, 767, 737, 832, 786, 754, 780, 710, 748, 771, 796, 741, 771, 797, 732, 785, 774, 788] payload_rows_per_dst=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] dup_rows_per_dst=[2964, 3052, 2964, 2954, 3152, 2940, 3038, 3024] combine_rows_per_src=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64814 dup_rows=24316 dup_rows_remote=21098 dedup_ratio=0.247355 router_tokens_per_expert=[805, 764, 771, 732, 762, 750, 767, 770, 762, 756, 724, 754, 734, 772, 733, 795, 794, 776, 788, 786, 752, 782, 725, 769, 798, 797, 773, 763, 795, 754, 774, 787, 777, 851, 722, 768, 763, 778, 766, 812, 780, 755, 745, 713, 715, 748, 741, 739, 771, 798, 787, 749, 767, 776, 802, 791, 790, 754, 782, 761, 756, 764, 794, 773] payload_rows_per_dst=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] dup_rows_per_dst=[3218, 2848, 3224, 3096, 2908, 2876, 3032, 3114] combine_rows_per_src=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344160 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657888 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532971616 weight_movement_bytes=69206016 total_nvlink_bytes=602177632 weight_to_token_ratio=0.129849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64594 dup_rows=24108 dup_rows_remote=20782 dedup_ratio=0.245239 router_tokens_per_expert=[807, 817, 775, 796, 743, 766, 775, 771, 772, 771, 776, 749, 800, 740, 765, 716, 819, 774, 755, 749, 774, 740, 748, 763, 743, 783, 744, 807, 763, 781, 765, 758, 754, 744, 796, 771, 733, 766, 790, 734, 811, 768, 787, 725, 740, 753, 775, 703, 743, 784, 748, 773, 739, 776, 768, 787, 763, 807, 810, 781, 776, 768, 811, 763] payload_rows_per_dst=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] dup_rows_per_dst=[2666, 2780, 2864, 3470, 2990, 3106, 2906, 3326] combine_rows_per_src=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344048 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267796464 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535248880 weight_movement_bytes=34603008 total_nvlink_bytes=569851888 weight_to_token_ratio=0.064648 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343648 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265821792 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531299936 weight_movement_bytes=34603008 total_nvlink_bytes=565902944 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64896 dup_rows=24158 dup_rows_remote=21074 dedup_ratio=0.245748 router_tokens_per_expert=[779, 778, 763, 783, 763, 757, 723, 756, 731, 720, 769, 730, 766, 742, 781, 734, 789, 800, 748, 779, 755, 711, 800, 777, 794, 793, 778, 742, 816, 751, 784, 730, 753, 738, 765, 768, 808, 788, 783, 764, 822, 780, 813, 751, 752, 808, 773, 755, 791, 791, 751, 760, 771, 763, 777, 808, 750, 757, 798, 760, 763, 789, 735, 745] payload_rows_per_dst=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] dup_rows_per_dst=[2968, 2876, 3028, 3110, 3084, 3048, 3102, 2942] combine_rows_per_src=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64868 dup_rows=24114 dup_rows_remote=21020 dedup_ratio=0.245300 router_tokens_per_expert=[754, 789, 790, 763, 764, 782, 782, 774, 768, 808, 788, 790, 812, 722, 802, 783, 740, 806, 823, 772, 718, 762, 738, 754, 719, 725, 717, 753, 773, 792, 816, 771, 758, 746, 769, 795, 757, 797, 772, 763, 748, 816, 777, 724, 727, 749, 792, 746, 758, 772, 807, 789, 783, 789, 790, 720, 805, 750, 776, 737, 754, 698, 744, 794] payload_rows_per_dst=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] dup_rows_per_dst=[3052, 3108, 2904, 3000, 3086, 2926, 3094, 2944] combine_rows_per_src=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344272 combine_payload_bytes=266010624 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266354896 ingress_bytes=369819648 link_bytes=369819648 token_bytes=532365520 weight_movement_bytes=103809024 total_nvlink_bytes=636174544 weight_to_token_ratio=0.194996 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391601562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21088 dedup_ratio=0.243958 router_tokens_per_expert=[774, 745, 817, 743, 783, 720, 774, 810, 832, 769, 787, 788, 800, 791, 765, 740, 736, 800, 772, 750, 742, 777, 742, 783, 777, 756, 775, 801, 710, 766, 743, 758, 745, 735, 720, 777, 778, 746, 754, 802, 760, 771, 790, 825, 810, 715, 730, 799, 788, 732, 758, 751, 844, 762, 791, 789, 766, 715, 751, 747, 747, 772, 791, 765] payload_rows_per_dst=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] dup_rows_per_dst=[3098, 3126, 2970, 2894, 2908, 3038, 3050, 2898] combine_rows_per_src=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=341504 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264918528 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529495552 weight_movement_bytes=34603008 total_nvlink_bytes=564098560 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343552 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042880 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742208 weight_movement_bytes=103809024 total_nvlink_bytes=635551232 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343880 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157896 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971912 weight_movement_bytes=34603008 total_nvlink_bytes=566574920 weight_to_token_ratio=0.065047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64788 dup_rows=24238 dup_rows_remote=21346 dedup_ratio=0.246562 router_tokens_per_expert=[787, 769, 797, 766, 747, 809, 819, 726, 771, 714, 774, 783, 733, 783, 742, 785, 749, 752, 782, 706, 743, 761, 774, 747, 772, 765, 779, 760, 770, 733, 760, 764, 739, 784, 846, 760, 792, 786, 814, 744, 739, 789, 789, 764, 773, 753, 764, 747, 781, 786, 758, 748, 772, 761, 728, 750, 758, 778, 793, 732, 805, 785, 825, 787] payload_rows_per_dst=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] dup_rows_per_dst=[3040, 2892, 2956, 3038, 3192, 3032, 2976, 3112] combine_rows_per_src=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344528 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764752 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184976 weight_movement_bytes=34603008 total_nvlink_bytes=567787984 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64718 dup_rows=24076 dup_rows_remote=20848 dedup_ratio=0.244914 router_tokens_per_expert=[817, 760, 763, 797, 751, 858, 787, 784, 784, 778, 784, 755, 803, 745, 771, 757, 805, 753, 807, 758, 772, 795, 810, 794, 750, 755, 797, 763, 778, 740, 786, 747, 767, 775, 776, 751, 713, 734, 769, 764, 726, 758, 758, 788, 793, 762, 803, 737, 772, 743, 729, 749, 756, 809, 702, 766, 753, 767, 805, 704, 757, 733, 746, 783] payload_rows_per_dst=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] dup_rows_per_dst=[3228, 3050, 3096, 2912, 3006, 3010, 2900, 2874] combine_rows_per_src=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64908 dup_rows=24010 dup_rows_remote=20990 dedup_ratio=0.244242 router_tokens_per_expert=[782, 765, 768, 741, 755, 764, 771, 768, 762, 797, 783, 810, 792, 742, 733, 763, 723, 746, 775, 709, 776, 789, 782, 722, 776, 775, 841, 764, 782, 763, 755, 792, 769, 784, 769, 758, 819, 749, 813, 736, 804, 763, 775, 828, 744, 754, 781, 754, 742, 777, 774, 726, 737, 835, 771, 758, 758, 725, 735, 784, 768, 768, 781, 747] payload_rows_per_dst=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] dup_rows_per_dst=[2990, 3094, 2926, 3098, 2976, 3020, 3018, 2888] combine_rows_per_src=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65064 dup_rows=23842 dup_rows_remote=20880 dedup_ratio=0.242533 router_tokens_per_expert=[727, 741, 784, 765, 753, 782, 773, 728, 775, 783, 771, 807, 753, 753, 729, 806, 795, 779, 771, 774, 767, 783, 737, 774, 792, 728, 777, 731, 756, 762, 769, 793, 796, 774, 761, 769, 766, 802, 758, 771, 739, 761, 753, 798, 809, 754, 722, 788, 743, 799, 807, 721, 727, 776, 797, 792, 775, 760, 764, 778, 777, 751, 789, 757] payload_rows_per_dst=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] dup_rows_per_dst=[2886, 3030, 2962, 2958, 3064, 2970, 2960, 3012] combine_rows_per_src=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344536 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265716184 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087832 weight_movement_bytes=34603008 total_nvlink_bytes=565690840 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65174 dup_rows=23886 dup_rows_remote=20854 dedup_ratio=0.242981 router_tokens_per_expert=[785, 767, 800, 762, 789, 798, 791, 745, 778, 809, 790, 803, 734, 750, 749, 753, 771, 720, 759, 743, 814, 755, 802, 801, 788, 773, 761, 741, 776, 816, 723, 749, 746, 768, 755, 760, 750, 714, 799, 776, 800, 799, 720, 773, 767, 760, 797, 736, 764, 740, 795, 769, 755, 758, 752, 766, 783, 761, 739, 768, 818, 786, 708, 775] payload_rows_per_dst=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] dup_rows_per_dst=[3006, 3058, 2890, 2832, 2780, 3134, 3154, 3032] combine_rows_per_src=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343592 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206760 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069928 weight_movement_bytes=34603008 total_nvlink_bytes=566672936 weight_to_token_ratio=0.065035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65220 dup_rows=23798 dup_rows_remote=20822 dedup_ratio=0.242086 router_tokens_per_expert=[799, 735, 769, 807, 745, 707, 733, 783, 726, 741, 732, 755, 726, 771, 787, 738, 729, 754, 757, 820, 785, 755, 735, 771, 734, 819, 758, 751, 810, 777, 768, 832, 776, 750, 780, 783, 772, 756, 783, 797, 810, 761, 799, 733, 741, 822, 784, 749, 750, 731, 843, 778, 766, 756, 745, 762, 766, 739, 733, 767, 795, 818, 806, 792] payload_rows_per_dst=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] dup_rows_per_dst=[2960, 2758, 2982, 3108, 2968, 3046, 2976, 3000] combine_rows_per_src=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265084928 dispatch_meta_bytes=342264 combine_payload_bytes=265084928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265427192 ingress_bytes=334290944 link_bytes=334290944 token_bytes=530512120 weight_movement_bytes=69206016 total_nvlink_bytes=599718136 weight_to_token_ratio=0.130451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257909910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64886 dup_rows=23988 dup_rows_remote=20960 dedup_ratio=0.244019 router_tokens_per_expert=[749, 754, 788, 741, 758, 799, 765, 803, 761, 750, 769, 722, 752, 793, 820, 816, 773, 769, 719, 772, 747, 733, 729, 737, 761, 779, 763, 788, 813, 750, 760, 753, 773, 809, 745, 810, 791, 750, 748, 803, 764, 736, 718, 729, 762, 771, 782, 795, 790, 796, 777, 790, 779, 767, 787, 737, 752, 763, 786, 774, 770, 785, 760, 767] payload_rows_per_dst=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] dup_rows_per_dst=[2998, 3054, 2864, 3034, 3028, 2910, 3166, 2934] combine_rows_per_src=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343776 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266845920 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348064 weight_movement_bytes=34603008 total_nvlink_bytes=567951072 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64790 dup_rows=24194 dup_rows_remote=21194 dedup_ratio=0.246114 router_tokens_per_expert=[721, 755, 743, 804, 762, 759, 766, 733, 754, 805, 795, 730, 748, 743, 798, 808, 783, 746, 763, 834, 768, 804, 807, 797, 769, 762, 774, 792, 784, 792, 737, 750, 722, 750, 791, 778, 728, 771, 782, 771, 780, 742, 766, 742, 766, 718, 718, 756, 775, 717, 737, 799, 775, 756, 786, 783, 803, 763, 765, 762, 787, 767, 812, 798] payload_rows_per_dst=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] dup_rows_per_dst=[2902, 3012, 3158, 3000, 2996, 2882, 2998, 3246] combine_rows_per_src=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267296816 ingress_bytes=405364736 link_bytes=405364736 token_bytes=534249520 weight_movement_bytes=138412032 total_nvlink_bytes=672661552 weight_to_token_ratio=0.259078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525354867e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344168 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267485288 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534626408 weight_movement_bytes=34603008 total_nvlink_bytes=569229416 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343384 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116440 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531889496 weight_movement_bytes=103809024 total_nvlink_bytes=635698520 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65236 dup_rows=23810 dup_rows_remote=20834 dedup_ratio=0.242208 router_tokens_per_expert=[744, 799, 786, 787, 811, 743, 752, 769, 853, 755, 749, 812, 732, 775, 709, 732, 760, 756, 796, 763, 778, 765, 756, 764, 779, 760, 733, 762, 790, 735, 742, 732, 755, 802, 745, 732, 825, 786, 735, 818, 769, 733, 753, 741, 778, 804, 758, 785, 743, 804, 780, 772, 811, 778, 756, 785, 793, 768, 692, 746, 749, 781, 764, 832] payload_rows_per_dst=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] dup_rows_per_dst=[3084, 2976, 2966, 2850, 3024, 2972, 3010, 2928] combine_rows_per_src=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64738 dup_rows=24284 dup_rows_remote=21196 dedup_ratio=0.247030 router_tokens_per_expert=[797, 709, 811, 783, 761, 794, 789, 796, 785, 767, 820, 753, 776, 786, 733, 770, 738, 769, 759, 767, 781, 735, 777, 771, 766, 746, 786, 748, 822, 761, 748, 742, 736, 804, 796, 754, 747, 768, 781, 759, 775, 744, 739, 793, 781, 800, 787, 766, 697, 770, 721, 748, 752, 763, 785, 767, 804, 792, 703, 778, 772, 763, 780, 781] payload_rows_per_dst=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] dup_rows_per_dst=[3108, 3096, 3026, 3028, 3014, 3088, 2886, 3038] combine_rows_per_src=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343936 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723776 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103616 weight_movement_bytes=69206016 total_nvlink_bytes=600309632 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65022 dup_rows=23812 dup_rows_remote=20782 dedup_ratio=0.242228 router_tokens_per_expert=[779, 778, 787, 747, 789, 724, 795, 781, 774, 771, 772, 796, 772, 765, 766, 781, 779, 762, 768, 775, 806, 756, 778, 765, 760, 787, 772, 746, 742, 800, 715, 773, 804, 783, 756, 771, 779, 760, 740, 738, 780, 768, 769, 768, 799, 760, 761, 759, 756, 768, 747, 734, 750, 759, 763, 764, 782, 751, 753, 793, 790, 765, 754, 767] payload_rows_per_dst=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] dup_rows_per_dst=[3030, 2914, 3266, 3020, 2952, 2948, 2940, 2742] combine_rows_per_src=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64992 dup_rows=24028 dup_rows_remote=21012 dedup_ratio=0.244425 router_tokens_per_expert=[765, 724, 777, 809, 740, 780, 801, 714, 740, 774, 758, 735, 756, 766, 771, 797, 816, 749, 751, 788, 770, 746, 752, 778, 774, 810, 762, 793, 769, 803, 784, 785, 742, 742, 777, 774, 696, 766, 779, 796, 792, 757, 755, 742, 772, 804, 739, 736, 805, 752, 690, 742, 751, 758, 751, 789, 785, 793, 787, 780, 816, 821, 757, 769] payload_rows_per_dst=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] dup_rows_per_dst=[2978, 2934, 3016, 3116, 2884, 2996, 2962, 3142] combine_rows_per_src=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344280 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550936 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757592 weight_movement_bytes=34603008 total_nvlink_bytes=569360600 weight_to_token_ratio=0.064708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343736 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265510584 ingress_bytes=403578880 link_bytes=403578880 token_bytes=530677432 weight_movement_bytes=138412032 total_nvlink_bytes=669089464 weight_to_token_ratio=0.260821 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518634835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64958 dup_rows=23994 dup_rows_remote=20976 dedup_ratio=0.244080 router_tokens_per_expert=[732, 752, 771, 752, 746, 767, 797, 747, 763, 725, 761, 732, 804, 749, 785, 776, 767, 769, 762, 714, 822, 754, 758, 746, 761, 773, 808, 799, 775, 798, 757, 803, 795, 770, 740, 761, 782, 720, 766, 772, 786, 724, 783, 784, 764, 789, 799, 795, 781, 776, 813, 786, 763, 779, 744, 747, 720, 797, 849, 765, 763, 743, 781, 690] payload_rows_per_dst=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] dup_rows_per_dst=[2928, 2998, 2886, 3024, 3102, 2880, 3158, 3018] combine_rows_per_src=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343216 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673328 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533003440 weight_movement_bytes=34603008 total_nvlink_bytes=567606448 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74010 payload_rows_remote=64690 dup_rows=24294 dup_rows_remote=21154 dedup_ratio=0.247131 router_tokens_per_expert=[752, 743, 762, 755, 756, 778, 779, 779, 750, 786, 766, 773, 777, 765, 748, 795, 798, 762, 798, 782, 737, 766, 770, 733, 745, 788, 768, 732, 743, 782, 765, 749, 755, 765, 800, 709, 804, 744, 771, 752, 816, 773, 760, 768, 759, 745, 861, 736, 745, 783, 745, 808, 754, 782, 807, 806, 781, 777, 789, 778, 735, 745, 764, 753] payload_rows_per_dst=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] dup_rows_per_dst=[3022, 3016, 3148, 3004, 2954, 3076, 3140, 2934] combine_rows_per_src=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65116 dup_rows=24002 dup_rows_remote=21130 dedup_ratio=0.244161 router_tokens_per_expert=[789, 767, 770, 760, 747, 818, 777, 748, 788, 770, 775, 800, 767, 750, 799, 760, 762, 816, 769, 734, 781, 773, 727, 796, 766, 792, 794, 749, 799, 772, 745, 801, 737, 776, 776, 720, 732, 770, 748, 770, 795, 791, 749, 762, 780, 813, 794, 784, 816, 733, 798, 725, 746, 742, 769, 789, 744, 743, 730, 761, 731, 711, 758, 798] payload_rows_per_dst=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] dup_rows_per_dst=[3046, 3110, 3012, 3104, 2872, 3172, 2880, 2806] combine_rows_per_src=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344016 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551248 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758480 weight_movement_bytes=34603008 total_nvlink_bytes=567361488 weight_to_token_ratio=0.064951 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64874 dup_rows=23970 dup_rows_remote=20846 dedup_ratio=0.243835 router_tokens_per_expert=[765, 774, 829, 782, 722, 751, 763, 742, 778, 734, 756, 749, 785, 736, 795, 817, 756, 743, 744, 734, 774, 765, 773, 744, 794, 806, 752, 818, 738, 818, 764, 802, 766, 743, 741, 784, 809, 801, 805, 777, 771, 707, 729, 775, 791, 757, 788, 779, 764, 787, 761, 732, 765, 767, 788, 745, 749, 765, 738, 798, 773, 736, 802, 756] payload_rows_per_dst=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] dup_rows_per_dst=[2918, 3064, 2868, 3124, 3112, 2960, 2922, 3002] combine_rows_per_src=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343736 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266411704 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532479672 weight_movement_bytes=34603008 total_nvlink_bytes=567082680 weight_to_token_ratio=0.064985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=343376 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313616 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283856 weight_movement_bytes=34603008 total_nvlink_bytes=564886864 weight_to_token_ratio=0.065254 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344984 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267060120 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533775256 weight_movement_bytes=34603008 total_nvlink_bytes=568378264 weight_to_token_ratio=0.064827 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65276 dup_rows=23808 dup_rows_remote=20802 dedup_ratio=0.242188 router_tokens_per_expert=[732, 820, 806, 772, 775, 787, 787, 747, 771, 758, 769, 748, 786, 791, 748, 742, 759, 785, 736, 776, 774, 801, 710, 785, 799, 767, 738, 783, 754, 787, 801, 741, 758, 724, 771, 780, 764, 739, 782, 780, 762, 725, 776, 801, 730, 781, 749, 804, 776, 763, 785, 794, 721, 802, 725, 740, 766, 761, 772, 776, 759, 758, 787, 806] payload_rows_per_dst=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] dup_rows_per_dst=[2946, 3006, 2956, 2998, 3020, 2894, 2958, 3030] combine_rows_per_src=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65058 dup_rows=23946 dup_rows_remote=20948 dedup_ratio=0.243591 router_tokens_per_expert=[739, 773, 764, 763, 764, 730, 772, 757, 772, 733, 768, 749, 777, 760, 769, 776, 760, 754, 775, 774, 782, 803, 744, 802, 785, 732, 767, 769, 767, 786, 724, 763, 819, 764, 773, 734, 787, 756, 768, 725, 773, 798, 740, 730, 759, 781, 762, 806, 764, 809, 769, 808, 805, 787, 759, 775, 748, 754, 820, 788, 743, 764, 742, 789] payload_rows_per_dst=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] dup_rows_per_dst=[2920, 2962, 3060, 2958, 2980, 2998, 3130, 2938] combine_rows_per_src=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64816 dup_rows=24224 dup_rows_remote=21210 dedup_ratio=0.246419 router_tokens_per_expert=[749, 759, 753, 767, 808, 750, 713, 757, 744, 760, 801, 801, 756, 724, 755, 732, 839, 776, 756, 753, 745, 784, 800, 802, 765, 793, 751, 819, 824, 755, 708, 730, 733, 775, 755, 789, 737, 798, 761, 793, 759, 805, 771, 780, 784, 771, 781, 801, 745, 779, 781, 781, 697, 798, 764, 739, 815, 762, 788, 757, 741, 755, 762, 766] payload_rows_per_dst=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] dup_rows_per_dst=[3014, 3040, 3112, 2884, 3124, 3122, 2868, 3060] combine_rows_per_src=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=342880 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266066784 ingress_bytes=404135936 link_bytes=404135936 token_bytes=531790688 weight_movement_bytes=138412032 total_nvlink_bytes=670202720 weight_to_token_ratio=0.260275 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520730992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344312 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267714808 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085304 weight_movement_bytes=34603008 total_nvlink_bytes=569688312 weight_to_token_ratio=0.064668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65022 dup_rows=24030 dup_rows_remote=21124 dedup_ratio=0.244446 router_tokens_per_expert=[737, 722, 793, 758, 787, 836, 745, 746, 743, 737, 798, 734, 749, 813, 791, 804, 765, 770, 754, 760, 783, 782, 765, 700, 754, 746, 807, 786, 811, 783, 770, 745, 790, 761, 731, 746, 764, 752, 739, 780, 802, 747, 815, 782, 789, 823, 764, 712, 791, 796, 783, 779, 760, 748, 799, 730, 746, 799, 796, 734, 761, 728, 785, 746] payload_rows_per_dst=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] dup_rows_per_dst=[2986, 3024, 2906, 3046, 2986, 3112, 3098, 2872] combine_rows_per_src=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344024 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821592 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299160 weight_movement_bytes=69206016 total_nvlink_bytes=602505176 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64968 dup_rows=23946 dup_rows_remote=20648 dedup_ratio=0.243591 router_tokens_per_expert=[766, 717, 771, 784, 779, 757, 750, 798, 821, 731, 791, 757, 751, 765, 718, 787, 729, 711, 769, 758, 743, 736, 797, 788, 787, 774, 776, 808, 778, 772, 714, 784, 835, 765, 734, 730, 767, 775, 772, 786, 740, 749, 760, 765, 756, 771, 783, 815, 751, 751, 758, 789, 729, 802, 731, 783, 796, 802, 788, 736, 811, 797, 758, 800] payload_rows_per_dst=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] dup_rows_per_dst=[2938, 2892, 2916, 3048, 3030, 2888, 2936, 3298] combine_rows_per_src=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344104 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265830440 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531316776 weight_movement_bytes=34603008 total_nvlink_bytes=565919784 weight_to_token_ratio=0.065127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65032 dup_rows=23952 dup_rows_remote=20900 dedup_ratio=0.243652 router_tokens_per_expert=[781, 731, 793, 727, 713, 755, 770, 765, 781, 732, 778, 745, 749, 751, 766, 750, 805, 765, 794, 766, 713, 805, 758, 780, 796, 752, 760, 793, 755, 792, 753, 735, 800, 752, 766, 759, 783, 725, 821, 780, 780, 800, 754, 750, 745, 767, 737, 803, 750, 808, 737, 791, 778, 810, 724, 780, 764, 741, 829, 797, 825, 774, 747, 766] payload_rows_per_dst=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] dup_rows_per_dst=[2888, 2904, 2962, 2914, 3052, 2972, 3062, 3198] combine_rows_per_src=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344584 combine_payload_bytes=266330112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266674696 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533004808 weight_movement_bytes=69206016 total_nvlink_bytes=602210824 weight_to_token_ratio=0.129841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65038 dup_rows=23962 dup_rows_remote=21016 dedup_ratio=0.243754 router_tokens_per_expert=[751, 763, 832, 785, 775, 759, 744, 789, 822, 764, 736, 777, 769, 742, 797, 788, 751, 754, 757, 760, 804, 746, 762, 813, 769, 749, 765, 788, 718, 789, 738, 764, 724, 797, 775, 749, 821, 750, 771, 784, 804, 765, 769, 731, 775, 761, 776, 771, 764, 726, 682, 798, 786, 819, 777, 773, 747, 727, 751, 829, 740, 745, 795, 750] payload_rows_per_dst=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] dup_rows_per_dst=[3028, 3074, 2944, 2982, 3018, 3032, 2946, 2938] combine_rows_per_src=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342464 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266451392 ingress_bytes=369917952 link_bytes=369917952 token_bytes=532560320 weight_movement_bytes=103809024 total_nvlink_bytes=636369344 weight_to_token_ratio=0.194924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391971473e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64976 dup_rows=24116 dup_rows_remote=21006 dedup_ratio=0.245321 router_tokens_per_expert=[749, 730, 777, 754, 774, 758, 780, 790, 769, 760, 783, 778, 850, 765, 727, 761, 758, 740, 785, 850, 801, 809, 755, 766, 764, 742, 823, 806, 747, 742, 805, 732, 757, 745, 716, 784, 755, 757, 730, 735, 780, 780, 730, 769, 767, 722, 731, 818, 730, 799, 739, 808, 787, 735, 786, 712, 763, 821, 793, 768, 768, 777, 774, 786] payload_rows_per_dst=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] dup_rows_per_dst=[2972, 3090, 3088, 3110, 2842, 2934, 2978, 3102] combine_rows_per_src=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343728 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266714800 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533085872 weight_movement_bytes=34603008 total_nvlink_bytes=567688880 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64996 dup_rows=23968 dup_rows_remote=21058 dedup_ratio=0.243815 router_tokens_per_expert=[814, 741, 769, 791, 783, 742, 781, 722, 771, 755, 746, 779, 782, 739, 798, 755, 771, 779, 790, 778, 770, 794, 765, 723, 737, 728, 797, 796, 765, 749, 732, 747, 740, 756, 787, 763, 699, 747, 800, 788, 787, 804, 813, 781, 780, 794, 773, 740, 775, 763, 828, 742, 786, 752, 733, 778, 785, 726, 774, 756, 787, 764, 778, 784] payload_rows_per_dst=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] dup_rows_per_dst=[3008, 2910, 3028, 2940, 2894, 3178, 3014, 2996] combine_rows_per_src=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344216 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739864 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135512 weight_movement_bytes=69206016 total_nvlink_bytes=602341528 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64908 dup_rows=24012 dup_rows_remote=21004 dedup_ratio=0.244263 router_tokens_per_expert=[716, 747, 742, 760, 752, 805, 743, 768, 738, 736, 785, 711, 803, 811, 758, 754, 787, 789, 805, 753, 764, 744, 757, 745, 741, 769, 757, 823, 782, 768, 814, 790, 781, 786, 750, 800, 765, 760, 778, 804, 771, 778, 810, 746, 777, 824, 762, 728, 763, 755, 752, 761, 759, 704, 789, 818, 798, 765, 733, 787, 755, 813, 732, 731] payload_rows_per_dst=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] dup_rows_per_dst=[2936, 2872, 2994, 3082, 3138, 3008, 2956, 3026] combine_rows_per_src=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343928 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266485624 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627320 weight_movement_bytes=69206016 total_nvlink_bytes=601833336 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344216 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567832 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791448 weight_movement_bytes=34603008 total_nvlink_bytes=567394456 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73954 payload_rows_remote=64634 dup_rows=24350 dup_rows_remote=21124 dedup_ratio=0.247701 router_tokens_per_expert=[679, 772, 766, 794, 777, 777, 745, 769, 792, 758, 769, 762, 769, 690, 786, 716, 736, 773, 774, 769, 765, 814, 782, 741, 774, 801, 798, 757, 820, 753, 761, 765, 803, 748, 791, 799, 795, 754, 704, 768, 806, 763, 758, 786, 752, 738, 768, 789, 824, 712, 797, 761, 770, 781, 749, 763, 790, 743, 783, 766, 780, 776, 763, 768] payload_rows_per_dst=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] dup_rows_per_dst=[3226, 3098, 3054, 2990, 3260, 3090, 2916, 2716] combine_rows_per_src=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64856 dup_rows=24170 dup_rows_remote=21152 dedup_ratio=0.245870 router_tokens_per_expert=[794, 751, 755, 744, 715, 774, 741, 770, 758, 697, 788, 797, 823, 763, 744, 726, 794, 835, 725, 761, 799, 758, 756, 720, 780, 807, 774, 789, 786, 745, 764, 789, 792, 753, 813, 801, 769, 781, 758, 760, 732, 796, 748, 784, 792, 822, 815, 752, 779, 771, 751, 748, 782, 736, 761, 792, 725, 770, 746, 801, 756, 751, 752, 741] payload_rows_per_dst=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] dup_rows_per_dst=[2820, 3014, 3018, 3082, 3148, 3114, 3042, 2932] combine_rows_per_src=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206816 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069984 weight_movement_bytes=69206016 total_nvlink_bytes=601276000 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64792 dup_rows=23956 dup_rows_remote=20750 dedup_ratio=0.243693 router_tokens_per_expert=[739, 749, 839, 772, 757, 780, 779, 776, 774, 752, 788, 733, 700, 775, 780, 697, 781, 730, 730, 771, 758, 776, 782, 774, 776, 774, 800, 753, 795, 776, 745, 779, 774, 784, 763, 799, 756, 763, 720, 783, 825, 779, 772, 730, 728, 789, 787, 770, 746, 761, 767, 725, 777, 782, 782, 754, 797, 770, 795, 795, 774, 790, 766, 759] payload_rows_per_dst=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] dup_rows_per_dst=[2996, 2822, 2888, 3100, 2774, 3032, 3138, 3206] combine_rows_per_src=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64794 dup_rows=24190 dup_rows_remote=20966 dedup_ratio=0.246073 router_tokens_per_expert=[749, 768, 736, 744, 753, 775, 760, 803, 765, 773, 726, 715, 771, 802, 745, 740, 758, 746, 758, 756, 807, 760, 824, 752, 784, 772, 749, 780, 737, 735, 838, 765, 771, 781, 803, 810, 785, 752, 785, 785, 761, 760, 756, 757, 793, 742, 821, 813, 786, 809, 805, 751, 813, 706, 798, 722, 754, 746, 787, 736, 772, 765, 705, 776] payload_rows_per_dst=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] dup_rows_per_dst=[2912, 2948, 3040, 3018, 3224, 3132, 3030, 2886] combine_rows_per_src=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=343032 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265083896 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529824760 weight_movement_bytes=34603008 total_nvlink_bytes=564427768 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64942 dup_rows=24122 dup_rows_remote=21042 dedup_ratio=0.245382 router_tokens_per_expert=[792, 786, 803, 760, 736, 744, 748, 759, 763, 766, 747, 754, 766, 787, 739, 818, 735, 753, 756, 790, 743, 737, 742, 775, 752, 769, 780, 801, 751, 711, 766, 740, 760, 781, 802, 797, 739, 777, 769, 769, 778, 766, 811, 762, 774, 745, 784, 742, 770, 767, 812, 754, 756, 761, 798, 742, 807, 805, 815, 762, 768, 754, 794, 762] payload_rows_per_dst=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] dup_rows_per_dst=[3024, 2964, 2872, 2884, 3076, 3062, 3080, 3160] combine_rows_per_src=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=344032 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265994208 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531644384 weight_movement_bytes=69206016 total_nvlink_bytes=600850400 weight_to_token_ratio=0.130174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342168 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730200 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118232 weight_movement_bytes=34603008 total_nvlink_bytes=565721240 weight_to_token_ratio=0.065151 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64980 dup_rows=24028 dup_rows_remote=20794 dedup_ratio=0.244425 router_tokens_per_expert=[756, 793, 745, 731, 764, 788, 742, 762, 771, 772, 766, 777, 732, 722, 799, 812, 730, 795, 803, 828, 764, 735, 779, 770, 812, 798, 768, 746, 781, 775, 784, 801, 766, 778, 730, 800, 776, 800, 739, 792, 752, 753, 738, 836, 777, 746, 808, 756, 749, 752, 730, 766, 731, 761, 796, 780, 752, 749, 752, 772, 722, 752, 764, 776] payload_rows_per_dst=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] dup_rows_per_dst=[2968, 3018, 3028, 3234, 3072, 2916, 2966, 2826] combine_rows_per_src=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65016 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[750, 780, 781, 803, 777, 761, 802, 810, 754, 724, 764, 790, 816, 778, 753, 759, 740, 786, 771, 764, 749, 768, 740, 702, 767, 762, 762, 767, 736, 723, 735, 773, 832, 733, 759, 795, 743, 763, 781, 763, 783, 788, 736, 762, 729, 802, 811, 762, 789, 787, 755, 805, 802, 750, 795, 783, 732, 776, 749, 779, 760, 755, 785, 761] payload_rows_per_dst=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] dup_rows_per_dst=[3162, 3028, 2928, 2890, 3050, 2982, 3068, 2932] combine_rows_per_src=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343040 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265739264 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135488 weight_movement_bytes=69206016 total_nvlink_bytes=600341504 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343936 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266346368 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532348800 weight_movement_bytes=34603008 total_nvlink_bytes=566951808 weight_to_token_ratio=0.065001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65082 dup_rows=23986 dup_rows_remote=21034 dedup_ratio=0.243998 router_tokens_per_expert=[715, 798, 784, 768, 778, 736, 729, 729, 771, 784, 768, 802, 774, 787, 823, 751, 769, 762, 771, 690, 752, 762, 779, 778, 787, 783, 742, 752, 738, 799, 731, 761, 821, 709, 755, 801, 765, 811, 729, 806, 765, 776, 754, 731, 794, 756, 774, 744, 723, 799, 733, 796, 752, 795, 808, 752, 747, 762, 802, 811, 820, 789, 756, 763] payload_rows_per_dst=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] dup_rows_per_dst=[2888, 3152, 2908, 2976, 3026, 2952, 2962, 3122] combine_rows_per_src=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343096 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501176 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659256 weight_movement_bytes=69206016 total_nvlink_bytes=601865272 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344112 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266649648 ingress_bytes=370114560 link_bytes=370114560 token_bytes=532955184 weight_movement_bytes=103809024 total_nvlink_bytes=636764208 weight_to_token_ratio=0.194780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392711293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74076 payload_rows_remote=64464 dup_rows=24228 dup_rows_remote=20952 dedup_ratio=0.246460 router_tokens_per_expert=[785, 776, 737, 788, 812, 794, 737, 791, 779, 745, 747, 774, 769, 777, 775, 766, 741, 776, 809, 851, 761, 768, 761, 811, 731, 788, 771, 781, 802, 741, 782, 783, 768, 763, 709, 770, 759, 724, 738, 774, 810, 763, 731, 794, 710, 772, 731, 759, 787, 809, 748, 754, 790, 749, 782, 735, 773, 752, 774, 755, 741, 767, 787, 765] payload_rows_per_dst=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] dup_rows_per_dst=[3276, 3332, 3186, 2820, 2924, 2668, 3010, 3012] combine_rows_per_src=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64708 dup_rows=24214 dup_rows_remote=21216 dedup_ratio=0.246318 router_tokens_per_expert=[738, 772, 728, 779, 803, 785, 740, 763, 768, 797, 769, 755, 786, 754, 812, 778, 763, 783, 765, 739, 816, 762, 770, 792, 774, 749, 744, 770, 746, 808, 743, 761, 752, 785, 763, 769, 776, 812, 731, 766, 761, 756, 799, 744, 724, 775, 764, 742, 777, 734, 714, 771, 753, 796, 764, 764, 739, 822, 794, 752, 795, 747, 824, 775] payload_rows_per_dst=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] dup_rows_per_dst=[2970, 3074, 2998, 3052, 2996, 2992, 3028, 3104] combine_rows_per_src=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344464 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920336 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496208 weight_movement_bytes=34603008 total_nvlink_bytes=568099216 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64822 dup_rows=24168 dup_rows_remote=21116 dedup_ratio=0.245850 router_tokens_per_expert=[756, 828, 801, 773, 743, 781, 749, 732, 750, 783, 754, 762, 775, 777, 747, 791, 782, 790, 782, 783, 771, 840, 781, 747, 750, 784, 731, 805, 798, 781, 746, 766, 724, 736, 750, 786, 745, 748, 831, 769, 720, 796, 708, 782, 768, 790, 776, 757, 713, 753, 752, 772, 757, 738, 787, 772, 770, 782, 810, 781, 752, 779, 746, 763] payload_rows_per_dst=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] dup_rows_per_dst=[3170, 2768, 3094, 3252, 2946, 2976, 2910, 3052] combine_rows_per_src=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262175432 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6135042760 ingress_bytes=7407443968 link_bytes=7407443968 total_nvlink_bytes=13542486728 weight_to_token_ratio=0.104411 prefetch_share_of_ingress=0.102770 dedup_ratio=0.243764 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211896318e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=341664 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264386208 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528430752 weight_movement_bytes=34603008 total_nvlink_bytes=563033760 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343696 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265387664 ingress_bytes=368852992 link_bytes=368852992 token_bytes=530431632 weight_movement_bytes=103809024 total_nvlink_bytes=634240656 weight_to_token_ratio=0.195707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387964114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12272712288 prefetch_bytes=657457152 reduce_bytes=726663168 weight_movement_bytes=1384120320 egress_bytes=6140312160 ingress_bytes=7516520448 link_bytes=7516520448 total_nvlink_bytes=13656832608 weight_to_token_ratio=0.112780 prefetch_share_of_ingress=0.087468 dedup_ratio=0.243181 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=21 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229741796e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64976 dup_rows=23998 dup_rows_remote=20946 dedup_ratio=0.244120 router_tokens_per_expert=[795, 741, 787, 737, 753, 724, 768, 732, 789, 791, 813, 764, 781, 743, 793, 800, 759, 814, 728, 760, 802, 741, 751, 747, 763, 726, 785, 744, 749, 745, 749, 775, 742, 790, 745, 808, 777, 766, 786, 777, 775, 783, 768, 780, 765, 790, 754, 769, 798, 696, 795, 741, 745, 783, 822, 795, 714, 723, 751, 772, 788, 828, 807, 770] payload_rows_per_dst=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] dup_rows_per_dst=[2890, 3146, 2974, 2864, 3052, 3014, 3068, 2990] combine_rows_per_src=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64970 dup_rows=24090 dup_rows_remote=20982 dedup_ratio=0.245056 router_tokens_per_expert=[755, 806, 790, 755, 760, 762, 756, 779, 745, 788, 788, 760, 779, 829, 738, 769, 755, 772, 777, 729, 792, 751, 793, 771, 764, 764, 795, 771, 756, 792, 742, 794, 734, 779, 713, 825, 754, 735, 732, 751, 674, 763, 741, 792, 760, 794, 792, 787, 776, 789, 731, 828, 780, 736, 744, 792, 827, 753, 746, 777, 745, 784, 742, 799] payload_rows_per_dst=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] dup_rows_per_dst=[3020, 2980, 3000, 3020, 2940, 2972, 3108, 3050] combine_rows_per_src=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64968 dup_rows=23888 dup_rows_remote=20904 dedup_ratio=0.243001 router_tokens_per_expert=[762, 767, 769, 763, 793, 767, 769, 743, 776, 761, 773, 808, 755, 828, 742, 745, 748, 776, 808, 751, 734, 774, 788, 767, 788, 796, 815, 745, 796, 821, 758, 697, 758, 721, 759, 770, 729, 727, 778, 790, 805, 827, 762, 783, 748, 754, 769, 778, 747, 755, 748, 752, 771, 756, 791, 786, 778, 741, 743, 771, 765, 764, 803, 740] payload_rows_per_dst=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] dup_rows_per_dst=[3006, 3018, 3098, 2984, 2826, 3038, 3026, 2892] combine_rows_per_src=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258788984 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6133352056 ingress_bytes=7544160256 link_bytes=7544160256 total_nvlink_bytes=13677512312 weight_to_token_ratio=0.115731 prefetch_share_of_ingress=0.087148 dedup_ratio=0.244121 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234263813e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343752 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854664 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365576 weight_movement_bytes=103809024 total_nvlink_bytes=635174600 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343688 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485384 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627080 weight_movement_bytes=34603008 total_nvlink_bytes=567230088 weight_to_token_ratio=0.064967 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343808 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460928 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578048 weight_movement_bytes=34603008 total_nvlink_bytes=567181056 weight_to_token_ratio=0.064973 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343488 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452416 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561344 weight_movement_bytes=34603008 total_nvlink_bytes=567164352 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64904 dup_rows=23910 dup_rows_remote=20906 dedup_ratio=0.243225 router_tokens_per_expert=[800, 767, 784, 752, 746, 743, 790, 807, 824, 782, 716, 775, 749, 739, 769, 692, 800, 785, 753, 797, 780, 792, 751, 800, 759, 809, 744, 777, 764, 746, 722, 774, 740, 744, 771, 731, 755, 797, 773, 766, 777, 790, 801, 748, 774, 770, 758, 755, 791, 778, 755, 767, 735, 748, 786, 753, 765, 811, 778, 767, 777, 760, 795, 748] payload_rows_per_dst=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] dup_rows_per_dst=[3004, 2942, 3112, 2898, 3114, 3078, 2948, 2814] combine_rows_per_src=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12238422760 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6123160296 ingress_bytes=7741603840 link_bytes=7741603840 total_nvlink_bytes=13864764136 weight_to_token_ratio=0.132888 prefetch_share_of_ingress=0.075986 dedup_ratio=0.244481 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266566609e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264226080 prefetch_bytes=692060160 reduce_bytes=519045120 weight_movement_bytes=1211105280 egress_bytes=6136069408 ingress_bytes=7339261952 link_bytes=7339261952 total_nvlink_bytes=13475331360 weight_to_token_ratio=0.098751 prefetch_share_of_ingress=0.094296 dedup_ratio=0.244002 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=15 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200741385e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12253039416 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6130477880 ingress_bytes=7402872832 link_bytes=7402872832 total_nvlink_bytes=13533350712 weight_to_token_ratio=0.104489 prefetch_share_of_ingress=0.098160 dedup_ratio=0.244448 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211148456e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258456552 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6133183464 ingress_bytes=7543996416 link_bytes=7543996416 total_nvlink_bytes=13677179880 weight_to_token_ratio=0.115734 prefetch_share_of_ingress=0.091737 dedup_ratio=0.244023 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234237007e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343240 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190024 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036808 weight_movement_bytes=34603008 total_nvlink_bytes=566639816 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233767928 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120831992 ingress_bytes=7427850240 link_bytes=7427850240 total_nvlink_bytes=13548682232 weight_to_token_ratio=0.107482 prefetch_share_of_ingress=0.093171 dedup_ratio=0.244459 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215234890e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-29 23:41:56] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 673.4 | throughput per GPU (TFLOP/s/GPU): 189.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.005160E+01 | loss scale: 1.0 | grad norm: 141.371 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65238 dup_rows=23834 dup_rows_remote=20844 dedup_ratio=0.242452 router_tokens_per_expert=[784, 757, 753, 777, 768, 769, 790, 811, 773, 778, 764, 746, 761, 751, 769, 779, 734, 801, 803, 737, 797, 768, 734, 737, 805, 766, 744, 712, 800, 742, 781, 822, 771, 762, 766, 830, 755, 761, 734, 745, 744, 754, 749, 778, 840, 771, 786, 707, 780, 799, 747, 741, 756, 753, 740, 779, 754, 732, 758, 818, 787, 804, 785, 753] payload_rows_per_dst=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] dup_rows_per_dst=[3044, 2936, 2990, 2952, 2928, 2916, 3008, 3060] combine_rows_per_src=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64758 dup_rows=24286 dup_rows_remote=21208 dedup_ratio=0.247050 router_tokens_per_expert=[773, 774, 726, 797, 741, 784, 766, 796, 808, 802, 754, 769, 756, 784, 777, 768, 763, 751, 799, 793, 735, 762, 742, 786, 797, 761, 793, 781, 742, 803, 756, 736, 823, 784, 747, 764, 754, 709, 763, 734, 775, 775, 760, 775, 797, 804, 745, 721, 797, 749, 768, 764, 757, 758, 757, 762, 799, 711, 805, 770, 715, 766, 819, 750] payload_rows_per_dst=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] dup_rows_per_dst=[3046, 3070, 3032, 3078, 2944, 3072, 3008, 3036] combine_rows_per_src=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65202 dup_rows=23816 dup_rows_remote=20730 dedup_ratio=0.242269 router_tokens_per_expert=[783, 740, 755, 838, 722, 731, 780, 754, 802, 720, 766, 737, 801, 769, 790, 709, 804, 768, 782, 797, 709, 791, 761, 767, 794, 771, 764, 792, 780, 751, 759, 832, 805, 765, 802, 727, 775, 758, 754, 741, 799, 791, 750, 782, 771, 746, 737, 768, 748, 763, 788, 739, 814, 760, 739, 741, 774, 776, 777, 755, 738, 788, 787, 775] payload_rows_per_dst=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] dup_rows_per_dst=[2940, 2842, 3048, 2760, 3234, 3002, 2904, 3086] combine_rows_per_src=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65166 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[760, 740, 801, 766, 803, 777, 749, 724, 782, 793, 763, 736, 764, 776, 724, 766, 737, 790, 744, 775, 722, 800, 761, 746, 772, 812, 773, 698, 776, 819, 723, 781, 763, 766, 769, 787, 778, 725, 753, 753, 761, 800, 799, 820, 757, 746, 761, 738, 771, 752, 738, 814, 822, 800, 771, 771, 776, 753, 812, 752, 777, 802, 752, 760] payload_rows_per_dst=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] dup_rows_per_dst=[2920, 3068, 2924, 2960, 2928, 3060, 3076, 3062] combine_rows_per_src=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64932 dup_rows=24156 dup_rows_remote=21136 dedup_ratio=0.245728 router_tokens_per_expert=[773, 701, 738, 745, 795, 748, 783, 820, 809, 729, 727, 772, 811, 782, 796, 766, 775, 774, 717, 724, 762, 772, 770, 734, 790, 777, 751, 832, 719, 792, 791, 766, 788, 759, 773, 736, 774, 771, 761, 756, 776, 805, 760, 756, 754, 801, 780, 766, 800, 809, 756, 758, 740, 771, 821, 743, 803, 776, 745, 787, 771, 741, 768, 706] payload_rows_per_dst=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] dup_rows_per_dst=[2984, 3038, 2910, 3064, 3020, 3134, 3054, 2952] combine_rows_per_src=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65238 dup_rows=23846 dup_rows_remote=20814 dedup_ratio=0.242574 router_tokens_per_expert=[722, 745, 759, 742, 789, 724, 820, 787, 778, 800, 737, 778, 762, 794, 770, 748, 757, 799, 770, 753, 786, 754, 792, 759, 726, 762, 792, 803, 801, 768, 777, 779, 739, 747, 751, 774, 808, 733, 752, 753, 799, 809, 757, 778, 786, 780, 705, 712, 773, 772, 768, 749, 787, 785, 763, 776, 736, 769, 772, 745, 814, 751, 782, 794] payload_rows_per_dst=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] dup_rows_per_dst=[2918, 2962, 3010, 2938, 2946, 3032, 3062, 2978] combine_rows_per_src=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65304 dup_rows=23816 dup_rows_remote=20818 dedup_ratio=0.242269 router_tokens_per_expert=[787, 814, 771, 737, 763, 749, 755, 772, 706, 783, 793, 725, 780, 743, 786, 742, 779, 740, 811, 808, 731, 813, 770, 770, 743, 783, 794, 779, 741, 753, 762, 759, 808, 742, 749, 744, 807, 794, 729, 758, 808, 736, 797, 753, 739, 763, 755, 792, 766, 829, 815, 724, 746, 693, 789, 729, 768, 762, 761, 794, 813, 798, 744, 805] payload_rows_per_dst=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] dup_rows_per_dst=[3054, 2894, 2986, 2928, 2932, 2962, 2998, 3062] combine_rows_per_src=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344328 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559176 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774024 weight_movement_bytes=34603008 total_nvlink_bytes=569377032 weight_to_token_ratio=0.064706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=343864 combine_payload_bytes=265248768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265592632 ingress_bytes=369057792 link_bytes=369057792 token_bytes=530841400 weight_movement_bytes=103809024 total_nvlink_bytes=634650424 weight_to_token_ratio=0.195556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343728 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411120 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478512 weight_movement_bytes=103809024 total_nvlink_bytes=638287536 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344384 combine_payload_bytes=266919936 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267264320 ingress_bytes=266919936 link_bytes=267264320 token_bytes=534184256 weight_movement_bytes=0 total_nvlink_bytes=534184256 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005694120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=344272 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305744 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532267216 weight_movement_bytes=34603008 total_nvlink_bytes=566870224 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344208 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267559056 ingress_bytes=336420864 link_bytes=336420864 token_bytes=534773904 weight_movement_bytes=69206016 total_nvlink_bytes=603979920 weight_to_token_ratio=0.129412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64784 dup_rows=24124 dup_rows_remote=21072 dedup_ratio=0.245402 router_tokens_per_expert=[741, 782, 785, 733, 789, 704, 796, 749, 766, 759, 776, 745, 789, 764, 799, 800, 775, 801, 762, 704, 770, 765, 735, 734, 740, 778, 757, 762, 748, 802, 779, 803, 823, 749, 738, 734, 811, 769, 741, 803, 793, 747, 781, 807, 781, 723, 756, 750, 770, 791, 711, 810, 767, 806, 767, 746, 804, 801, 765, 744, 750, 761, 781, 780] payload_rows_per_dst=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] dup_rows_per_dst=[3052, 3100, 3152, 3030, 3038, 2890, 3004, 2858] combine_rows_per_src=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344488 combine_payload_bytes=267485184 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267829672 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314856 weight_movement_bytes=34603008 total_nvlink_bytes=569917864 weight_to_token_ratio=0.064640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65182 dup_rows=23968 dup_rows_remote=20976 dedup_ratio=0.243815 router_tokens_per_expert=[769, 789, 719, 766, 771, 810, 808, 751, 793, 778, 698, 760, 766, 751, 770, 781, 741, 764, 746, 769, 770, 804, 709, 770, 772, 824, 766, 776, 788, 763, 709, 770, 763, 742, 752, 796, 773, 724, 788, 774, 765, 779, 776, 799, 766, 769, 759, 751, 777, 737, 761, 776, 773, 774, 805, 775, 830, 788, 783, 763, 737, 743, 748, 785] payload_rows_per_dst=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] dup_rows_per_dst=[3002, 2924, 2992, 2990, 2908, 3078, 3004, 3070] combine_rows_per_src=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65132 dup_rows=23966 dup_rows_remote=20930 dedup_ratio=0.243795 router_tokens_per_expert=[796, 758, 801, 762, 735, 754, 790, 759, 765, 784, 763, 759, 790, 771, 793, 758, 788, 820, 791, 764, 755, 763, 786, 820, 797, 752, 788, 766, 758, 736, 758, 766, 774, 812, 777, 750, 709, 781, 782, 808, 754, 705, 723, 751, 801, 767, 795, 745, 760, 802, 766, 732, 757, 773, 745, 759, 798, 767, 725, 751, 750, 752, 769, 766] payload_rows_per_dst=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] dup_rows_per_dst=[3042, 3132, 3078, 3036, 2952, 2902, 2912, 2912] combine_rows_per_src=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=64688 dup_rows=24322 dup_rows_remote=21280 dedup_ratio=0.247416 router_tokens_per_expert=[807, 716, 789, 817, 782, 752, 721, 751, 774, 788, 779, 769, 764, 756, 765, 759, 767, 761, 776, 738, 802, 725, 791, 748, 777, 775, 747, 754, 750, 756, 806, 757, 758, 819, 736, 783, 762, 748, 778, 808, 791, 830, 830, 782, 768, 763, 767, 714, 746, 735, 781, 750, 806, 767, 757, 800, 777, 791, 757, 735, 782, 740, 747, 725] payload_rows_per_dst=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] dup_rows_per_dst=[2948, 2856, 2898, 2854, 3140, 3276, 3308, 3042] combine_rows_per_src=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65066 dup_rows=23830 dup_rows_remote=20782 dedup_ratio=0.242411 router_tokens_per_expert=[771, 743, 772, 765, 743, 828, 731, 797, 781, 781, 750, 738, 829, 785, 788, 776, 787, 772, 767, 744, 789, 726, 784, 822, 803, 800, 763, 731, 719, 742, 722, 793, 786, 769, 790, 828, 748, 772, 767, 781, 792, 768, 789, 753, 787, 749, 751, 763, 760, 740, 749, 767, 796, 782, 704, 764, 766, 761, 775, 724, 763, 747, 814, 705] payload_rows_per_dst=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] dup_rows_per_dst=[3034, 3048, 3042, 2908, 3072, 2966, 2938, 2822] combine_rows_per_src=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343424 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265698688 ingress_bytes=299958272 link_bytes=299958272 token_bytes=531053952 weight_movement_bytes=34603008 total_nvlink_bytes=565656960 weight_to_token_ratio=0.065159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128718829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64904 dup_rows=24088 dup_rows_remote=21146 dedup_ratio=0.245036 router_tokens_per_expert=[763, 824, 746, 810, 800, 767, 799, 772, 811, 802, 753, 736, 759, 772, 734, 743, 759, 764, 796, 807, 784, 739, 762, 778, 789, 762, 736, 746, 806, 760, 766, 781, 756, 743, 782, 773, 761, 758, 775, 779, 778, 754, 790, 792, 771, 712, 776, 755, 749, 724, 783, 763, 765, 748, 750, 784, 756, 745, 717, 775, 782, 770, 778, 782] payload_rows_per_dst=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] dup_rows_per_dst=[3276, 3014, 3012, 2990, 2942, 3012, 2928, 2914] combine_rows_per_src=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65070 dup_rows=23978 dup_rows_remote=21018 dedup_ratio=0.243917 router_tokens_per_expert=[755, 780, 718, 767, 822, 795, 753, 692, 777, 722, 804, 801, 785, 741, 788, 773, 768, 801, 756, 770, 785, 733, 757, 731, 810, 765, 785, 783, 797, 777, 828, 802, 766, 757, 760, 756, 742, 754, 742, 753, 706, 758, 787, 781, 798, 738, 765, 775, 779, 772, 757, 797, 759, 730, 777, 754, 763, 747, 810, 762, 782, 766, 763, 775] payload_rows_per_dst=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] dup_rows_per_dst=[2862, 3010, 3004, 3216, 2906, 2960, 2962, 3058] combine_rows_per_src=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65062 dup_rows=23926 dup_rows_remote=21072 dedup_ratio=0.243388 router_tokens_per_expert=[808, 745, 774, 785, 795, 755, 788, 762, 813, 722, 799, 775, 782, 749, 786, 765, 750, 752, 751, 755, 749, 741, 761, 748, 779, 810, 765, 777, 730, 812, 779, 779, 787, 800, 739, 740, 779, 756, 768, 803, 758, 807, 788, 756, 766, 777, 749, 720, 749, 774, 780, 719, 749, 796, 765, 753, 732, 798, 750, 796, 788, 736, 732, 801] payload_rows_per_dst=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] dup_rows_per_dst=[3066, 3074, 2854, 3126, 3012, 3024, 2854, 2916] combine_rows_per_src=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344632 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267330104 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315576 weight_movement_bytes=34603008 total_nvlink_bytes=568918584 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344248 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267124920 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905592 weight_movement_bytes=69206016 total_nvlink_bytes=603111608 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264962048 dispatch_meta_bytes=343872 combine_payload_bytes=264962048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265305920 ingress_bytes=299565056 link_bytes=299565056 token_bytes=530267968 weight_movement_bytes=34603008 total_nvlink_bytes=564870976 weight_to_token_ratio=0.065256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127239189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343392 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266853728 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364064 weight_movement_bytes=34603008 total_nvlink_bytes=567967072 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344200 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190984 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037768 weight_movement_bytes=69206016 total_nvlink_bytes=601243784 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344352 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871072 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533397792 weight_movement_bytes=34603008 total_nvlink_bytes=568000800 weight_to_token_ratio=0.064873 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=344536 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266838488 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533332440 weight_movement_bytes=34603008 total_nvlink_bytes=567935448 weight_to_token_ratio=0.064881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65222 dup_rows=23920 dup_rows_remote=20980 dedup_ratio=0.243327 router_tokens_per_expert=[766, 750, 781, 760, 747, 768, 772, 739, 758, 782, 794, 744, 758, 750, 767, 799, 767, 754, 748, 775, 772, 747, 741, 747, 867, 772, 803, 751, 775, 759, 756, 803, 769, 753, 752, 824, 756, 783, 789, 757, 756, 759, 781, 756, 721, 833, 753, 716, 758, 774, 770, 759, 793, 808, 781, 726, 778, 781, 789, 751, 785, 749, 742, 778] payload_rows_per_dst=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] dup_rows_per_dst=[2962, 2928, 2940, 3060, 3086, 2996, 3014, 2934] combine_rows_per_src=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=65006 dup_rows=24178 dup_rows_remote=21130 dedup_ratio=0.245951 router_tokens_per_expert=[704, 773, 778, 781, 737, 780, 761, 763, 798, 752, 746, 784, 749, 785, 771, 753, 825, 792, 728, 750, 765, 793, 803, 747, 740, 754, 819, 752, 737, 767, 748, 772, 753, 759, 775, 741, 779, 756, 745, 798, 752, 770, 748, 741, 732, 784, 770, 756, 732, 768, 774, 770, 778, 780, 806, 810, 794, 772, 823, 779, 812, 733, 794, 761] payload_rows_per_dst=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] dup_rows_per_dst=[3048, 2984, 3032, 3070, 2932, 2928, 3048, 3136] combine_rows_per_src=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64966 dup_rows=23950 dup_rows_remote=20988 dedup_ratio=0.243632 router_tokens_per_expert=[786, 752, 760, 747, 830, 749, 728, 767, 746, 821, 811, 803, 776, 757, 732, 729, 804, 714, 779, 748, 804, 772, 798, 784, 782, 761, 775, 751, 786, 788, 777, 789, 791, 754, 776, 745, 731, 793, 797, 737, 809, 751, 755, 827, 784, 745, 817, 718, 743, 755, 775, 741, 736, 738, 703, 826, 768, 734, 780, 730, 780, 747, 791, 769] payload_rows_per_dst=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] dup_rows_per_dst=[2978, 2962, 3086, 3058, 2942, 3056, 2852, 3016] combine_rows_per_src=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64904 dup_rows=24192 dup_rows_remote=21128 dedup_ratio=0.246094 router_tokens_per_expert=[758, 774, 769, 765, 764, 757, 788, 792, 742, 768, 771, 778, 778, 785, 782, 755, 786, 753, 798, 766, 736, 857, 691, 775, 767, 817, 779, 753, 782, 786, 722, 785, 744, 772, 782, 778, 729, 769, 732, 778, 804, 783, 784, 762, 722, 757, 774, 729, 769, 767, 743, 779, 760, 748, 807, 771, 744, 794, 773, 784, 754, 779, 766, 736] payload_rows_per_dst=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] dup_rows_per_dst=[3018, 3006, 2944, 3094, 2996, 2780, 3290, 3064] combine_rows_per_src=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64750 dup_rows=24158 dup_rows_remote=21154 dedup_ratio=0.245748 router_tokens_per_expert=[784, 761, 730, 767, 799, 693, 791, 745, 761, 778, 754, 759, 780, 742, 765, 756, 852, 733, 792, 801, 721, 732, 766, 791, 763, 786, 757, 746, 793, 787, 778, 790, 771, 775, 728, 786, 738, 759, 758, 768, 796, 758, 741, 755, 757, 714, 778, 759, 812, 754, 744, 762, 775, 784, 773, 788, 743, 815, 745, 781, 808, 837, 768, 769] payload_rows_per_dst=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] dup_rows_per_dst=[3006, 2950, 3080, 3004, 2972, 2900, 3136, 3110] combine_rows_per_src=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65244 dup_rows=23882 dup_rows_remote=20996 dedup_ratio=0.242940 router_tokens_per_expert=[782, 744, 750, 831, 773, 754, 742, 753, 775, 739, 774, 788, 735, 797, 800, 773, 756, 799, 734, 791, 804, 745, 763, 780, 762, 746, 760, 710, 784, 795, 731, 755, 784, 783, 738, 744, 776, 715, 731, 761, 767, 802, 813, 787, 743, 765, 776, 730, 802, 774, 758, 745, 808, 770, 783, 778, 734, 786, 790, 775, 817, 726, 794, 772] payload_rows_per_dst=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] dup_rows_per_dst=[2956, 3010, 2956, 2878, 2886, 2984, 3142, 3070] combine_rows_per_src=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64920 dup_rows=24058 dup_rows_remote=21056 dedup_ratio=0.244731 router_tokens_per_expert=[791, 737, 770, 795, 761, 767, 771, 808, 789, 790, 766, 748, 783, 742, 769, 818, 797, 755, 769, 802, 766, 762, 806, 743, 735, 748, 724, 750, 750, 738, 748, 790, 744, 757, 731, 727, 756, 763, 743, 777, 751, 777, 768, 733, 816, 757, 754, 808, 824, 754, 769, 760, 760, 764, 747, 762, 778, 798, 783, 781, 769, 789, 777, 787] payload_rows_per_dst=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] dup_rows_per_dst=[3088, 3050, 3108, 2862, 2890, 3002, 2940, 3118] combine_rows_per_src=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344808 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267494120 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534643432 weight_movement_bytes=69206016 total_nvlink_bytes=603849448 weight_to_token_ratio=0.129443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74014 payload_rows_remote=64658 dup_rows=24290 dup_rows_remote=21308 dedup_ratio=0.247091 router_tokens_per_expert=[795, 748, 770, 727, 787, 741, 825, 774, 782, 805, 718, 764, 816, 787, 775, 756, 721, 781, 808, 767, 753, 755, 777, 727, 745, 740, 769, 758, 726, 772, 798, 749, 738, 754, 741, 796, 756, 785, 748, 760, 774, 829, 821, 759, 783, 747, 773, 796, 785, 719, 759, 796, 743, 765, 791, 811, 740, 786, 742, 776, 769, 792, 751, 751] payload_rows_per_dst=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] dup_rows_per_dst=[3054, 3084, 2916, 3068, 2976, 3146, 2982, 3064] combine_rows_per_src=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344544 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266609120 ingress_bytes=370073600 link_bytes=370073600 token_bytes=532873696 weight_movement_bytes=103809024 total_nvlink_bytes=636682720 weight_to_token_ratio=0.194810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392557163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343816 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444552 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545288 weight_movement_bytes=69206016 total_nvlink_bytes=601751304 weight_to_token_ratio=0.129953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344128 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190912 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037696 weight_movement_bytes=69206016 total_nvlink_bytes=601243712 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343616 combine_payload_bytes=265216000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265559616 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530775616 weight_movement_bytes=34603008 total_nvlink_bytes=565378624 weight_to_token_ratio=0.065193 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344960 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584384 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823808 weight_movement_bytes=34603008 total_nvlink_bytes=569426816 weight_to_token_ratio=0.064700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343904 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256224 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168544 weight_movement_bytes=69206016 total_nvlink_bytes=601374560 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=343864 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265183032 ingress_bytes=334045184 link_bytes=334045184 token_bytes=530022200 weight_movement_bytes=69206016 total_nvlink_bytes=599228216 weight_to_token_ratio=0.130572 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256985135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65074 dup_rows=23918 dup_rows_remote=20782 dedup_ratio=0.243306 router_tokens_per_expert=[743, 768, 821, 747, 762, 744, 734, 737, 798, 770, 769, 785, 770, 760, 788, 797, 745, 799, 786, 760, 787, 757, 757, 833, 744, 787, 774, 735, 820, 805, 811, 819, 738, 779, 739, 744, 735, 771, 761, 763, 773, 736, 683, 767, 822, 773, 827, 716, 709, 788, 794, 742, 804, 788, 774, 736, 769, 722, 753, 743, 738, 761, 762, 830] payload_rows_per_dst=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] dup_rows_per_dst=[2906, 3040, 3136, 3116, 2922, 2938, 2946, 2914] combine_rows_per_src=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65038 dup_rows=23892 dup_rows_remote=20926 dedup_ratio=0.243042 router_tokens_per_expert=[771, 817, 766, 788, 800, 779, 763, 745, 794, 825, 741, 737, 720, 798, 760, 795, 794, 729, 785, 714, 781, 760, 726, 753, 769, 712, 799, 758, 790, 727, 813, 772, 754, 802, 818, 776, 696, 760, 758, 786, 746, 779, 802, 798, 730, 760, 772, 745, 738, 755, 751, 788, 809, 770, 758, 753, 782, 723, 764, 769, 795, 761, 774, 799] payload_rows_per_dst=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] dup_rows_per_dst=[3140, 2966, 2946, 2920, 2954, 3020, 2932, 3014] combine_rows_per_src=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64614 dup_rows=24270 dup_rows_remote=21244 dedup_ratio=0.246887 router_tokens_per_expert=[753, 804, 757, 748, 775, 856, 730, 798, 776, 738, 704, 796, 703, 774, 733, 810, 739, 788, 785, 752, 778, 754, 794, 775, 765, 784, 798, 808, 742, 774, 761, 753, 725, 764, 774, 763, 754, 776, 733, 753, 806, 804, 735, 741, 767, 796, 774, 797, 784, 793, 749, 792, 775, 785, 817, 807, 752, 704, 814, 763, 746, 707, 760, 737] payload_rows_per_dst=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] dup_rows_per_dst=[3026, 3028, 3086, 3060, 3446, 3120, 2660, 2844] combine_rows_per_src=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64874 dup_rows=24052 dup_rows_remote=20898 dedup_ratio=0.244670 router_tokens_per_expert=[744, 796, 761, 793, 751, 754, 807, 766, 708, 744, 750, 736, 810, 755, 763, 769, 772, 757, 808, 775, 726, 746, 777, 773, 784, 763, 826, 788, 736, 768, 769, 774, 755, 736, 772, 798, 798, 714, 726, 828, 785, 748, 803, 739, 788, 749, 720, 760, 768, 803, 789, 768, 775, 776, 746, 788, 773, 754, 780, 746, 802, 790, 759, 767] payload_rows_per_dst=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] dup_rows_per_dst=[3020, 2884, 2994, 2794, 2970, 3150, 3086, 3154] combine_rows_per_src=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65108 dup_rows=23946 dup_rows_remote=20952 dedup_ratio=0.243591 router_tokens_per_expert=[769, 804, 751, 778, 780, 818, 752, 780, 746, 715, 760, 765, 774, 780, 740, 800, 772, 774, 759, 764, 748, 747, 767, 777, 805, 719, 787, 770, 764, 768, 783, 806, 778, 751, 783, 781, 774, 754, 782, 719, 768, 753, 738, 791, 782, 760, 766, 797, 769, 759, 763, 747, 791, 743, 765, 776, 791, 775, 763, 818, 737, 735, 759, 762] payload_rows_per_dst=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] dup_rows_per_dst=[3130, 3008, 2916, 3026, 2994, 2894, 2978, 3000] combine_rows_per_src=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74072 payload_rows_remote=64610 dup_rows=24232 dup_rows_remote=21172 dedup_ratio=0.246501 router_tokens_per_expert=[790, 734, 789, 776, 783, 742, 729, 749, 765, 769, 747, 751, 765, 774, 805, 768, 757, 800, 748, 782, 805, 784, 728, 738, 829, 798, 794, 793, 748, 816, 739, 744, 768, 781, 789, 792, 748, 774, 795, 761, 757, 759, 728, 759, 729, 817, 750, 817, 778, 775, 744, 765, 819, 680, 764, 787, 760, 768, 781, 730, 746, 725, 765, 802] payload_rows_per_dst=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] dup_rows_per_dst=[3006, 3064, 2964, 3060, 3132, 2986, 3004, 3016] combine_rows_per_src=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65032 dup_rows=24010 dup_rows_remote=20968 dedup_ratio=0.244242 router_tokens_per_expert=[776, 718, 729, 761, 769, 760, 758, 733, 735, 792, 750, 763, 794, 752, 744, 739, 778, 799, 771, 773, 747, 797, 776, 796, 783, 719, 876, 791, 749, 752, 752, 765, 777, 782, 740, 775, 835, 787, 734, 769, 749, 775, 782, 754, 738, 754, 792, 808, 780, 793, 742, 768, 747, 763, 810, 736, 744, 757, 762, 751, 761, 820, 776, 794] payload_rows_per_dst=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] dup_rows_per_dst=[2906, 2926, 2988, 2960, 3068, 3042, 3082, 3038] combine_rows_per_src=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343424 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266886528 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533429632 weight_movement_bytes=34603008 total_nvlink_bytes=568032640 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65024 dup_rows=23956 dup_rows_remote=20920 dedup_ratio=0.243693 router_tokens_per_expert=[771, 771, 767, 729, 770, 797, 757, 783, 807, 720, 778, 761, 755, 739, 736, 837, 773, 799, 734, 743, 751, 735, 784, 753, 756, 777, 770, 742, 778, 790, 759, 816, 749, 768, 759, 746, 808, 789, 764, 759, 715, 752, 770, 792, 849, 797, 756, 737, 736, 726, 783, 795, 746, 799, 828, 767, 798, 783, 770, 768, 754, 794, 709, 748] payload_rows_per_dst=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] dup_rows_per_dst=[3022, 2994, 2928, 3068, 2976, 2952, 3036, 2980] combine_rows_per_src=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343856 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266739504 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135152 weight_movement_bytes=34603008 total_nvlink_bytes=567738160 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343088 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266066992 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531790896 weight_movement_bytes=34603008 total_nvlink_bytes=566393904 weight_to_token_ratio=0.065069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=343432 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265002376 ingress_bytes=333864960 link_bytes=333864960 token_bytes=529661320 weight_movement_bytes=69206016 total_nvlink_bytes=598867336 weight_to_token_ratio=0.130661 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256306966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344240 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267026608 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708976 weight_movement_bytes=103809024 total_nvlink_bytes=637518000 weight_to_token_ratio=0.194505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=343128 combine_payload_bytes=264642560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=264985688 ingress_bytes=437657600 link_bytes=437657600 token_bytes=529628248 weight_movement_bytes=173015040 total_nvlink_bytes=702643288 weight_to_token_ratio=0.326673 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646870314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344000 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715072 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086144 weight_movement_bytes=34603008 total_nvlink_bytes=567689152 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343776 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682080 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020384 weight_movement_bytes=34603008 total_nvlink_bytes=567623392 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64850 dup_rows=24046 dup_rows_remote=20958 dedup_ratio=0.244609 router_tokens_per_expert=[805, 781, 754, 796, 768, 769, 754, 764, 767, 792, 768, 738, 752, 784, 773, 781, 787, 787, 828, 775, 788, 756, 737, 790, 817, 711, 794, 784, 785, 819, 766, 791, 734, 738, 750, 828, 709, 767, 710, 746, 797, 766, 773, 746, 718, 771, 735, 766, 791, 773, 765, 744, 779, 764, 742, 758, 802, 780, 772, 744, 740, 765, 753, 765] payload_rows_per_dst=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] dup_rows_per_dst=[3104, 2942, 3088, 3110, 2872, 2956, 2974, 3000] combine_rows_per_src=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64824 dup_rows=24196 dup_rows_remote=21104 dedup_ratio=0.246134 router_tokens_per_expert=[774, 737, 774, 753, 776, 794, 780, 802, 801, 773, 773, 783, 729, 807, 759, 763, 770, 781, 760, 744, 760, 758, 742, 730, 755, 716, 760, 750, 729, 756, 777, 807, 782, 678, 801, 763, 769, 766, 798, 753, 776, 741, 759, 788, 748, 739, 802, 808, 738, 776, 814, 772, 788, 775, 772, 790, 793, 754, 793, 767, 814, 768, 741, 753] payload_rows_per_dst=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] dup_rows_per_dst=[3072, 3092, 2906, 2962, 2902, 3106, 3048, 3108] combine_rows_per_src=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64882 dup_rows=24112 dup_rows_remote=21192 dedup_ratio=0.245280 router_tokens_per_expert=[805, 766, 733, 789, 741, 757, 778, 790, 761, 750, 749, 763, 813, 779, 769, 756, 756, 812, 755, 783, 696, 785, 778, 824, 748, 815, 785, 764, 819, 766, 768, 703, 753, 761, 773, 731, 770, 805, 749, 778, 763, 757, 753, 749, 794, 814, 762, 760, 780, 770, 736, 785, 788, 716, 787, 747, 790, 720, 749, 765, 758, 774, 751, 808] payload_rows_per_dst=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] dup_rows_per_dst=[2930, 2994, 2942, 2858, 3012, 3336, 3120, 2920] combine_rows_per_src=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343232 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265968832 ingress_bytes=369434624 link_bytes=369434624 token_bytes=531594432 weight_movement_bytes=103809024 total_nvlink_bytes=635403456 weight_to_token_ratio=0.195279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390152748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65026 dup_rows=23844 dup_rows_remote=20834 dedup_ratio=0.242554 router_tokens_per_expert=[750, 784, 774, 733, 787, 746, 820, 800, 745, 746, 769, 750, 713, 820, 775, 764, 774, 799, 766, 693, 781, 780, 761, 804, 749, 802, 725, 771, 796, 780, 808, 734, 810, 733, 777, 794, 769, 811, 780, 769, 789, 751, 742, 741, 730, 745, 778, 785, 770, 772, 763, 717, 773, 738, 798, 783, 762, 750, 756, 749, 783, 783, 763, 789] payload_rows_per_dst=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] dup_rows_per_dst=[3010, 3212, 3026, 2788, 2946, 2830, 3054, 2978] combine_rows_per_src=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65090 dup_rows=23902 dup_rows_remote=20920 dedup_ratio=0.243144 router_tokens_per_expert=[786, 821, 739, 763, 787, 815, 724, 715, 741, 794, 712, 685, 768, 734, 795, 765, 741, 801, 741, 738, 758, 776, 784, 770, 779, 810, 797, 805, 810, 745, 782, 785, 753, 788, 740, 777, 761, 768, 785, 775, 768, 759, 773, 763, 760, 763, 787, 797, 769, 775, 764, 795, 780, 786, 752, 743, 791, 781, 770, 751, 746, 720, 806, 740] payload_rows_per_dst=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] dup_rows_per_dst=[2992, 2814, 2972, 3196, 2982, 3010, 2992, 2944] combine_rows_per_src=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65030 dup_rows=23984 dup_rows_remote=21092 dedup_ratio=0.243978 router_tokens_per_expert=[725, 763, 750, 768, 794, 759, 791, 735, 778, 733, 840, 777, 749, 740, 762, 768, 716, 796, 798, 758, 783, 750, 753, 760, 789, 758, 798, 781, 799, 763, 801, 757, 762, 740, 781, 778, 752, 771, 800, 781, 746, 746, 751, 737, 774, 733, 788, 816, 800, 781, 748, 812, 761, 772, 748, 761, 730, 791, 781, 753, 762, 744, 808, 752] payload_rows_per_dst=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] dup_rows_per_dst=[3006, 2974, 3036, 3132, 3072, 2892, 2958, 2914] combine_rows_per_src=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64984 dup_rows=24186 dup_rows_remote=21310 dedup_ratio=0.246033 router_tokens_per_expert=[788, 776, 756, 805, 807, 770, 729, 739, 770, 745, 757, 804, 738, 797, 808, 752, 764, 760, 737, 785, 789, 756, 806, 744, 763, 704, 738, 804, 745, 747, 724, 780, 794, 759, 722, 788, 780, 738, 791, 820, 772, 757, 723, 758, 764, 737, 751, 724, 808, 770, 766, 802, 732, 767, 732, 783, 780, 838, 766, 766, 787, 824, 773, 793] payload_rows_per_dst=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] dup_rows_per_dst=[3130, 2966, 3028, 2876, 3132, 2830, 3030, 3194] combine_rows_per_src=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65262 dup_rows=23736 dup_rows_remote=20790 dedup_ratio=0.241455 router_tokens_per_expert=[774, 809, 759, 751, 755, 773, 724, 729, 789, 708, 773, 773, 827, 742, 769, 791, 757, 791, 752, 778, 769, 771, 803, 776, 769, 734, 721, 786, 777, 791, 780, 790, 775, 770, 744, 741, 751, 779, 769, 771, 777, 792, 759, 745, 776, 811, 752, 768, 726, 787, 745, 739, 807, 761, 762, 799, 724, 773, 753, 778, 797, 768, 741, 821] payload_rows_per_dst=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] dup_rows_per_dst=[2898, 3004, 2942, 2970, 2964, 3050, 2946, 2962] combine_rows_per_src=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344296 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266100968 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857640 weight_movement_bytes=69206016 total_nvlink_bytes=601063656 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343440 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689936 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036432 weight_movement_bytes=34603008 total_nvlink_bytes=567639440 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344040 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952680 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561320 weight_movement_bytes=34603008 total_nvlink_bytes=568164328 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344488 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707368 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070248 weight_movement_bytes=34603008 total_nvlink_bytes=567673256 weight_to_token_ratio=0.064913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=345176 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266519640 ingress_bytes=335380480 link_bytes=335380480 token_bytes=532694104 weight_movement_bytes=69206016 total_nvlink_bytes=601900120 weight_to_token_ratio=0.129917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262009746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344208 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267657360 ingress_bytes=336519168 link_bytes=336519168 token_bytes=534970512 weight_movement_bytes=69206016 total_nvlink_bytes=604176528 weight_to_token_ratio=0.129364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266294537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65424 dup_rows=23660 dup_rows_remote=20672 dedup_ratio=0.240682 router_tokens_per_expert=[762, 754, 754, 770, 751, 787, 799, 759, 733, 765, 782, 754, 744, 785, 735, 735, 765, 743, 782, 750, 784, 746, 774, 760, 712, 806, 832, 751, 799, 763, 762, 776, 680, 757, 727, 758, 710, 720, 770, 758, 798, 770, 806, 828, 799, 797, 768, 747, 781, 742, 787, 769, 777, 807, 845, 768, 769, 795, 762, 767, 743, 770, 819, 784] payload_rows_per_dst=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] dup_rows_per_dst=[2970, 2886, 2988, 3082, 2668, 3122, 3008, 2936] combine_rows_per_src=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65252 dup_rows=23660 dup_rows_remote=20668 dedup_ratio=0.240682 router_tokens_per_expert=[768, 801, 789, 749, 749, 803, 706, 786, 783, 753, 817, 764, 788, 730, 794, 763, 778, 759, 809, 797, 761, 790, 748, 784, 818, 739, 758, 756, 757, 715, 770, 798, 731, 725, 793, 750, 751, 731, 822, 797, 825, 803, 744, 789, 780, 789, 724, 750, 761, 778, 744, 778, 750, 744, 765, 749, 793, 784, 785, 763, 736, 752, 739, 747] payload_rows_per_dst=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] dup_rows_per_dst=[2930, 2992, 3070, 2952, 2936, 2980, 2864, 2936] combine_rows_per_src=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344384 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321088 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536297792 weight_movement_bytes=34603008 total_nvlink_bytes=570900800 weight_to_token_ratio=0.064522 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64748 dup_rows=24090 dup_rows_remote=21064 dedup_ratio=0.245056 router_tokens_per_expert=[768, 777, 773, 820, 741, 757, 752, 738, 780, 771, 769, 741, 739, 763, 752, 744, 757, 755, 783, 740, 762, 751, 740, 806, 699, 767, 760, 798, 811, 808, 799, 813, 748, 762, 761, 781, 724, 772, 768, 797, 796, 767, 744, 797, 779, 748, 749, 779, 754, 784, 759, 787, 778, 759, 782, 749, 801, 772, 759, 770, 779, 773, 748, 792] payload_rows_per_dst=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] dup_rows_per_dst=[2986, 2936, 2954, 3088, 2958, 3056, 3086, 3026] combine_rows_per_src=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64838 dup_rows=24076 dup_rows_remote=20970 dedup_ratio=0.244914 router_tokens_per_expert=[746, 755, 787, 787, 811, 826, 754, 771, 690, 760, 785, 802, 772, 785, 774, 772, 805, 800, 735, 795, 793, 759, 755, 765, 764, 799, 773, 740, 792, 777, 727, 750, 813, 759, 782, 747, 785, 742, 748, 734, 766, 752, 778, 780, 715, 745, 786, 761, 773, 742, 754, 742, 799, 775, 821, 774, 743, 771, 759, 790, 772, 725, 775, 738] payload_rows_per_dst=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] dup_rows_per_dst=[3106, 3048, 3054, 2990, 3166, 2970, 2852, 2890] combine_rows_per_src=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64950 dup_rows=24070 dup_rows_remote=21062 dedup_ratio=0.244853 router_tokens_per_expert=[812, 770, 814, 807, 766, 761, 745, 747, 765, 749, 725, 785, 736, 795, 708, 734, 757, 767, 722, 752, 758, 734, 772, 773, 777, 761, 737, 741, 773, 781, 756, 781, 807, 802, 746, 801, 749, 805, 766, 794, 757, 770, 735, 757, 754, 779, 798, 796, 753, 771, 829, 778, 751, 770, 809, 813, 737, 795, 772, 796, 766, 710, 769, 756] payload_rows_per_dst=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] dup_rows_per_dst=[3106, 2824, 2880, 2876, 3204, 3008, 3162, 3010] combine_rows_per_src=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65030 dup_rows=23972 dup_rows_remote=20928 dedup_ratio=0.243856 router_tokens_per_expert=[845, 784, 755, 778, 790, 764, 745, 747, 729, 824, 718, 730, 809, 803, 717, 795, 810, 760, 751, 764, 759, 818, 804, 682, 766, 788, 776, 728, 790, 746, 748, 714, 789, 843, 751, 736, 755, 764, 749, 786, 772, 766, 754, 776, 775, 763, 755, 789, 800, 769, 769, 744, 791, 755, 805, 773, 776, 754, 707, 735, 760, 809, 774, 771] payload_rows_per_dst=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] dup_rows_per_dst=[3068, 3016, 3044, 2868, 3044, 3012, 3086, 2834] combine_rows_per_src=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64820 dup_rows=24186 dup_rows_remote=21132 dedup_ratio=0.246033 router_tokens_per_expert=[754, 752, 722, 778, 813, 853, 803, 768, 758, 757, 806, 796, 779, 739, 788, 778, 745, 772, 813, 733, 727, 732, 767, 783, 770, 769, 788, 750, 769, 783, 738, 809, 767, 757, 812, 766, 755, 731, 781, 761, 739, 742, 755, 732, 767, 838, 775, 751, 769, 750, 751, 748, 749, 786, 762, 791, 780, 761, 728, 759, 757, 768, 776, 796] payload_rows_per_dst=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] dup_rows_per_dst=[3082, 3120, 2958, 3054, 2994, 2942, 2964, 3072] combine_rows_per_src=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64952 dup_rows=24038 dup_rows_remote=21084 dedup_ratio=0.244527 router_tokens_per_expert=[737, 751, 758, 780, 723, 774, 738, 766, 736, 752, 741, 819, 759, 720, 742, 723, 822, 804, 771, 761, 802, 748, 756, 776, 757, 784, 826, 743, 747, 811, 784, 722, 771, 763, 773, 783, 766, 732, 782, 803, 775, 744, 781, 779, 834, 797, 785, 758, 769, 743, 792, 703, 781, 767, 816, 763, 805, 800, 743, 738, 731, 778, 755, 809] payload_rows_per_dst=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] dup_rows_per_dst=[2968, 2832, 3102, 3056, 3008, 3082, 2954, 3036] combine_rows_per_src=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=343232 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265919680 ingress_bytes=334782464 link_bytes=334782464 token_bytes=531496128 weight_movement_bytes=69206016 total_nvlink_bytes=600702144 weight_to_token_ratio=0.130210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259759460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=343680 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267615872 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888064 weight_movement_bytes=34603008 total_nvlink_bytes=569491072 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343248 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265551056 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758864 weight_movement_bytes=69206016 total_nvlink_bytes=599964880 weight_to_token_ratio=0.130391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=344048 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266379248 ingress_bytes=369844224 link_bytes=369844224 token_bytes=532414448 weight_movement_bytes=103809024 total_nvlink_bytes=636223472 weight_to_token_ratio=0.194978 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391694040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343832 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706712 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069592 weight_movement_bytes=34603008 total_nvlink_bytes=567672600 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65226 dup_rows=23736 dup_rows_remote=20828 dedup_ratio=0.241455 router_tokens_per_expert=[757, 807, 783, 749, 772, 701, 766, 763, 758, 772, 714, 785, 811, 743, 796, 780, 787, 787, 741, 706, 745, 778, 792, 789, 827, 806, 719, 758, 726, 777, 794, 762, 783, 730, 768, 779, 782, 774, 741, 756, 796, 773, 764, 731, 739, 784, 724, 799, 796, 818, 781, 782, 732, 783, 771, 775, 765, 750, 814, 744, 760, 768, 765, 774] payload_rows_per_dst=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] dup_rows_per_dst=[2876, 3026, 2908, 3022, 2924, 2974, 3064, 2942] combine_rows_per_src=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343808 combine_payload_bytes=265502720 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265846528 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349248 weight_movement_bytes=69206016 total_nvlink_bytes=600555264 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344144 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266387536 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430928 weight_movement_bytes=69206016 total_nvlink_bytes=601636944 weight_to_token_ratio=0.129981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344216 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267509912 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675608 weight_movement_bytes=34603008 total_nvlink_bytes=569278616 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64962 dup_rows=23900 dup_rows_remote=20740 dedup_ratio=0.243123 router_tokens_per_expert=[798, 774, 767, 768, 732, 784, 756, 773, 792, 744, 809, 758, 730, 714, 787, 749, 791, 753, 778, 805, 724, 792, 776, 764, 744, 808, 775, 761, 722, 766, 783, 796, 722, 761, 715, 802, 781, 748, 775, 734, 752, 805, 773, 753, 766, 795, 780, 780, 742, 787, 721, 738, 805, 761, 724, 774, 743, 786, 771, 788, 780, 830, 758, 829] payload_rows_per_dst=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] dup_rows_per_dst=[2962, 2972, 2994, 3014, 2806, 3006, 2986, 3160] combine_rows_per_src=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=65058 dup_rows=24112 dup_rows_remote=21064 dedup_ratio=0.245280 router_tokens_per_expert=[727, 759, 761, 738, 793, 736, 786, 784, 799, 774, 723, 797, 786, 774, 759, 795, 740, 824, 742, 745, 748, 718, 757, 762, 775, 782, 739, 752, 769, 747, 767, 802, 785, 802, 693, 771, 768, 736, 742, 752, 805, 730, 825, 803, 787, 776, 741, 783, 724, 754, 732, 793, 791, 783, 780, 788, 755, 795, 773, 795, 781, 821, 807, 721] payload_rows_per_dst=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] dup_rows_per_dst=[3048, 3364, 3048, 2908, 3210, 2758, 2730, 3046] combine_rows_per_src=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64940 dup_rows=24144 dup_rows_remote=21142 dedup_ratio=0.245605 router_tokens_per_expert=[760, 766, 747, 779, 740, 759, 791, 786, 734, 781, 751, 709, 813, 762, 779, 782, 767, 772, 748, 761, 747, 751, 714, 755, 768, 771, 785, 822, 732, 816, 763, 759, 841, 776, 763, 784, 800, 711, 742, 744, 792, 749, 804, 761, 794, 754, 709, 809, 779, 793, 797, 749, 765, 814, 763, 778, 749, 773, 742, 745, 779, 760, 784, 779] payload_rows_per_dst=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] dup_rows_per_dst=[3032, 3002, 2840, 3142, 3058, 2978, 3094, 2998] combine_rows_per_src=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64970 dup_rows=23982 dup_rows_remote=20926 dedup_ratio=0.243958 router_tokens_per_expert=[750, 788, 775, 775, 755, 780, 783, 773, 771, 751, 742, 763, 702, 752, 775, 770, 786, 821, 786, 787, 777, 733, 714, 753, 753, 766, 789, 765, 783, 772, 769, 777, 828, 798, 764, 812, 765, 719, 736, 800, 762, 791, 756, 793, 736, 796, 765, 805, 753, 707, 802, 749, 801, 778, 771, 739, 744, 741, 764, 740, 769, 816, 745, 771] payload_rows_per_dst=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] dup_rows_per_dst=[3052, 2920, 3014, 2956, 3122, 3056, 2928, 2934] combine_rows_per_src=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64836 dup_rows=24218 dup_rows_remote=21274 dedup_ratio=0.246358 router_tokens_per_expert=[779, 753, 767, 771, 820, 756, 719, 783, 758, 785, 743, 790, 768, 741, 785, 767, 804, 805, 807, 734, 767, 807, 796, 787, 713, 752, 769, 733, 736, 802, 745, 763, 738, 798, 780, 772, 757, 759, 725, 768, 773, 757, 784, 787, 761, 790, 750, 761, 757, 792, 792, 759, 727, 781, 707, 780, 816, 741, 758, 773, 782, 794, 763, 765] payload_rows_per_dst=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] dup_rows_per_dst=[3054, 2968, 3186, 2880, 2944, 3106, 3016, 3064] combine_rows_per_src=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65138 dup_rows=23996 dup_rows_remote=21020 dedup_ratio=0.244100 router_tokens_per_expert=[740, 772, 761, 720, 797, 742, 742, 811, 765, 757, 804, 802, 751, 810, 800, 753, 769, 753, 789, 741, 767, 737, 747, 817, 758, 742, 741, 794, 734, 753, 767, 784, 798, 747, 788, 743, 781, 803, 730, 788, 796, 724, 800, 801, 742, 753, 749, 738, 760, 792, 762, 790, 759, 759, 790, 726, 807, 795, 813, 727, 766, 735, 793, 777] payload_rows_per_dst=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] dup_rows_per_dst=[2880, 3098, 3010, 2976, 2990, 2980, 2990, 3072] combine_rows_per_src=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=342808 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266427160 ingress_bytes=369893376 link_bytes=369893376 token_bytes=532511512 weight_movement_bytes=103809024 total_nvlink_bytes=636320536 weight_to_token_ratio=0.194942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391878995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344488 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822056 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299624 weight_movement_bytes=69206016 total_nvlink_bytes=602505640 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65242 dup_rows=23950 dup_rows_remote=21036 dedup_ratio=0.243632 router_tokens_per_expert=[745, 830, 773, 781, 773, 770, 765, 746, 778, 757, 750, 746, 793, 799, 792, 798, 767, 757, 741, 769, 749, 759, 775, 776, 809, 752, 741, 769, 759, 784, 758, 817, 781, 759, 776, 752, 764, 742, 776, 731, 808, 757, 745, 779, 792, 817, 775, 794, 783, 781, 704, 747, 774, 746, 730, 748, 771, 739, 726, 771, 735, 824, 768, 779] payload_rows_per_dst=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] dup_rows_per_dst=[3040, 3058, 2964, 3034, 2928, 3118, 2914, 2894] combine_rows_per_src=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344328 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338568 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332808 weight_movement_bytes=34603008 total_nvlink_bytes=566935816 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64974 dup_rows=24098 dup_rows_remote=21116 dedup_ratio=0.245138 router_tokens_per_expert=[783, 764, 748, 753, 719, 804, 768, 793, 772, 831, 755, 751, 744, 717, 775, 769, 719, 748, 770, 773, 770, 748, 789, 790, 837, 761, 762, 838, 771, 702, 750, 740, 774, 749, 768, 762, 730, 772, 828, 742, 795, 750, 790, 787, 803, 748, 751, 722, 796, 779, 796, 780, 787, 771, 707, 778, 768, 742, 810, 787, 771, 793, 717, 785] payload_rows_per_dst=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] dup_rows_per_dst=[3016, 2908, 2982, 3072, 3052, 3044, 3008, 3016] combine_rows_per_src=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343584 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266460704 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577824 weight_movement_bytes=138412032 total_nvlink_bytes=670989856 weight_to_token_ratio=0.259891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=344440 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912696 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480952 weight_movement_bytes=69206016 total_nvlink_bytes=600686968 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344632 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149880 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955128 weight_movement_bytes=34603008 total_nvlink_bytes=568558136 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=345112 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267576344 ingress_bytes=301834240 link_bytes=301834240 token_bytes=534807576 weight_movement_bytes=34603008 total_nvlink_bytes=569410584 weight_to_token_ratio=0.064702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135777945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344360 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477864 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611368 weight_movement_bytes=34603008 total_nvlink_bytes=567214376 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64964 dup_rows=23950 dup_rows_remote=21014 dedup_ratio=0.243632 router_tokens_per_expert=[723, 759, 751, 788, 757, 792, 778, 737, 746, 763, 723, 758, 767, 800, 733, 782, 760, 734, 775, 758, 765, 739, 791, 773, 782, 735, 787, 765, 779, 761, 750, 745, 751, 796, 806, 781, 782, 808, 764, 748, 790, 760, 740, 762, 833, 749, 819, 776, 742, 766, 784, 795, 811, 721, 789, 786, 757, 786, 763, 779, 728, 808, 762, 754] payload_rows_per_dst=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] dup_rows_per_dst=[2960, 2914, 2946, 2998, 3038, 3074, 3084, 2936] combine_rows_per_src=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64800 dup_rows=24208 dup_rows_remote=21146 dedup_ratio=0.246257 router_tokens_per_expert=[722, 779, 759, 794, 762, 748, 774, 759, 726, 747, 748, 821, 731, 782, 706, 794, 786, 768, 758, 762, 765, 710, 781, 766, 732, 789, 742, 727, 766, 792, 724, 752, 742, 836, 753, 825, 785, 772, 791, 771, 785, 816, 805, 801, 764, 733, 778, 777, 786, 769, 802, 781, 799, 716, 789, 774, 741, 780, 731, 796, 788, 790, 764, 740] payload_rows_per_dst=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] dup_rows_per_dst=[3062, 3078, 2984, 3148, 3126, 2902, 3032, 2876] combine_rows_per_src=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21078 dedup_ratio=0.244670 router_tokens_per_expert=[767, 790, 752, 743, 810, 775, 779, 749, 714, 794, 817, 769, 753, 797, 807, 685, 758, 812, 755, 776, 754, 794, 805, 703, 780, 776, 723, 761, 763, 768, 803, 757, 758, 776, 705, 786, 782, 771, 763, 767, 770, 767, 785, 806, 790, 828, 744, 816, 783, 723, 767, 794, 713, 786, 744, 748, 750, 764, 812, 776, 725, 786, 726, 752] payload_rows_per_dst=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] dup_rows_per_dst=[3080, 2974, 2938, 3014, 3034, 3096, 2918, 2998] combine_rows_per_src=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65074 dup_rows=23970 dup_rows_remote=21054 dedup_ratio=0.243835 router_tokens_per_expert=[753, 710, 767, 801, 787, 781, 787, 758, 816, 734, 778, 760, 742, 756, 736, 784, 783, 726, 784, 821, 743, 796, 733, 750, 771, 757, 793, 777, 781, 785, 804, 763, 800, 772, 766, 767, 811, 758, 707, 741, 754, 779, 736, 739, 758, 750, 789, 783, 787, 785, 762, 781, 756, 751, 763, 764, 778, 775, 793, 783, 800, 753, 757, 737] payload_rows_per_dst=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] dup_rows_per_dst=[2954, 3062, 2990, 3072, 2976, 2916, 2950, 3050] combine_rows_per_src=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64864 dup_rows=24056 dup_rows_remote=21036 dedup_ratio=0.244710 router_tokens_per_expert=[762, 805, 740, 789, 762, 724, 761, 780, 792, 730, 788, 803, 774, 771, 746, 708, 776, 754, 807, 752, 735, 851, 775, 752, 794, 790, 779, 715, 760, 725, 730, 758, 765, 775, 827, 767, 735, 764, 739, 830, 735, 797, 733, 780, 775, 820, 725, 811, 774, 748, 755, 768, 783, 712, 805, 787, 731, 770, 789, 759, 768, 766, 797, 774] payload_rows_per_dst=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] dup_rows_per_dst=[2926, 2962, 3186, 2856, 3020, 3012, 3016, 3078] combine_rows_per_src=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436456 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529000 weight_movement_bytes=103809024 total_nvlink_bytes=636338024 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64986 dup_rows=24078 dup_rows_remote=21136 dedup_ratio=0.244934 router_tokens_per_expert=[783, 759, 725, 741, 775, 775, 793, 735, 775, 739, 772, 749, 771, 781, 735, 788, 754, 768, 751, 744, 775, 808, 803, 797, 777, 749, 735, 778, 749, 786, 736, 781, 767, 824, 743, 770, 773, 773, 780, 739, 789, 707, 762, 778, 792, 805, 725, 807, 737, 735, 750, 781, 733, 773, 759, 771, 788, 782, 786, 807, 767, 794, 755, 813] payload_rows_per_dst=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] dup_rows_per_dst=[2990, 2970, 3096, 2942, 3032, 2992, 2946, 3110] combine_rows_per_src=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343784 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764584 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185384 weight_movement_bytes=34603008 total_nvlink_bytes=565788392 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65050 dup_rows=23928 dup_rows_remote=20878 dedup_ratio=0.243408 router_tokens_per_expert=[750, 780, 788, 755, 766, 745, 766, 743, 776, 806, 753, 775, 766, 730, 782, 774, 784, 775, 748, 771, 719, 763, 764, 749, 770, 761, 748, 776, 730, 763, 774, 746, 783, 777, 794, 777, 704, 722, 780, 774, 782, 774, 766, 750, 771, 774, 784, 810, 759, 745, 733, 829, 777, 753, 790, 802, 782, 798, 775, 777, 813, 770, 749, 782] payload_rows_per_dst=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] dup_rows_per_dst=[2984, 3000, 2906, 2982, 2966, 2978, 3050, 3062] combine_rows_per_src=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344128 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395712 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447296 weight_movement_bytes=34603008 total_nvlink_bytes=567050304 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64776 dup_rows=24184 dup_rows_remote=21108 dedup_ratio=0.246012 router_tokens_per_expert=[745, 765, 778, 753, 783, 747, 726, 773, 752, 797, 783, 769, 787, 753, 735, 759, 777, 755, 779, 799, 812, 763, 759, 766, 742, 805, 754, 741, 781, 786, 756, 806, 732, 741, 753, 744, 767, 781, 774, 822, 789, 780, 753, 764, 735, 795, 728, 751, 787, 701, 803, 789, 779, 791, 750, 743, 801, 820, 772, 758, 769, 776, 764, 754] payload_rows_per_dst=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] dup_rows_per_dst=[2944, 3052, 3076, 3090, 3022, 2964, 2966, 3070] combine_rows_per_src=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344512 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266887616 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430720 weight_movement_bytes=103809024 total_nvlink_bytes=637239744 weight_to_token_ratio=0.194606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343600 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026544 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709488 weight_movement_bytes=69206016 total_nvlink_bytes=600915504 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344488 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266527144 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709800 weight_movement_bytes=34603008 total_nvlink_bytes=567312808 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343712 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788512 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233312 weight_movement_bytes=69206016 total_nvlink_bytes=602439328 weight_to_token_ratio=0.129786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343536 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265666032 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988528 weight_movement_bytes=69206016 total_nvlink_bytes=600194544 weight_to_token_ratio=0.130334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65124 dup_rows=23822 dup_rows_remote=20726 dedup_ratio=0.242330 router_tokens_per_expert=[771, 776, 746, 792, 740, 805, 767, 787, 732, 813, 754, 742, 773, 732, 760, 732, 789, 739, 779, 787, 756, 733, 733, 795, 749, 768, 785, 764, 746, 779, 767, 829, 789, 780, 750, 813, 726, 758, 765, 790, 761, 737, 771, 749, 791, 732, 730, 776, 783, 758, 808, 784, 801, 777, 779, 817, 802, 744, 742, 769, 808, 753, 745, 744] payload_rows_per_dst=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] dup_rows_per_dst=[3020, 2846, 2908, 3060, 2876, 2928, 3088, 3096] combine_rows_per_src=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64816 dup_rows=24112 dup_rows_remote=20840 dedup_ratio=0.245280 router_tokens_per_expert=[760, 744, 735, 804, 763, 742, 787, 772, 828, 758, 761, 776, 728, 782, 779, 740, 836, 762, 815, 737, 746, 740, 747, 754, 741, 741, 764, 775, 756, 783, 766, 788, 727, 758, 788, 740, 793, 810, 744, 735, 765, 801, 767, 776, 802, 751, 794, 778, 773, 800, 753, 799, 791, 746, 742, 806, 756, 785, 737, 770, 735, 795, 797, 728] payload_rows_per_dst=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] dup_rows_per_dst=[3272, 3200, 2966, 3112, 2998, 2756, 3036, 2772] combine_rows_per_src=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65022 dup_rows=23972 dup_rows_remote=20966 dedup_ratio=0.243856 router_tokens_per_expert=[755, 789, 749, 770, 783, 780, 754, 778, 777, 749, 813, 750, 766, 795, 736, 772, 786, 794, 759, 765, 779, 810, 729, 756, 797, 719, 776, 761, 749, 768, 734, 795, 762, 788, 748, 747, 741, 771, 727, 755, 749, 770, 756, 743, 786, 760, 756, 778, 784, 794, 790, 745, 766, 809, 787, 773, 802, 760, 800, 717, 761, 748, 810, 776] payload_rows_per_dst=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] dup_rows_per_dst=[3052, 3006, 3054, 2938, 3014, 2916, 3092, 2900] combine_rows_per_src=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65082 dup_rows=23832 dup_rows_remote=20870 dedup_ratio=0.242432 router_tokens_per_expert=[766, 783, 747, 768, 758, 747, 808, 791, 743, 755, 771, 778, 738, 778, 732, 828, 761, 768, 742, 759, 732, 788, 795, 741, 797, 764, 746, 745, 762, 775, 712, 775, 762, 787, 792, 810, 750, 771, 740, 776, 773, 832, 786, 735, 775, 760, 779, 736, 875, 773, 732, 764, 739, 745, 748, 814, 791, 826, 793, 710, 782, 723, 775, 745] payload_rows_per_dst=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] dup_rows_per_dst=[2944, 2926, 2964, 2928, 3014, 2962, 3072, 3022] combine_rows_per_src=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343400 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267091304 ingress_bytes=335953920 link_bytes=335953920 token_bytes=533839208 weight_movement_bytes=69206016 total_nvlink_bytes=603045224 weight_to_token_ratio=0.129638 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264167554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64872 dup_rows=24138 dup_rows_remote=21054 dedup_ratio=0.245544 router_tokens_per_expert=[776, 760, 779, 804, 694, 785, 746, 820, 786, 745, 765, 764, 802, 736, 751, 791, 736, 795, 758, 795, 786, 810, 787, 827, 722, 734, 826, 736, 752, 766, 756, 733, 800, 795, 749, 770, 730, 801, 780, 764, 773, 749, 730, 784, 734, 763, 787, 724, 730, 789, 804, 722, 731, 795, 809, 754, 766, 788, 755, 775, 747, 768, 788, 775] payload_rows_per_dst=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] dup_rows_per_dst=[3046, 3042, 3096, 2860, 3084, 2844, 3022, 3144] combine_rows_per_src=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65006 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[801, 757, 797, 786, 754, 741, 769, 812, 727, 736, 752, 729, 750, 767, 755, 787, 774, 767, 768, 780, 772, 784, 748, 772, 771, 763, 796, 765, 732, 775, 770, 802, 749, 794, 709, 789, 745, 787, 753, 753, 811, 742, 779, 753, 771, 758, 789, 748, 747, 741, 790, 782, 765, 769, 772, 792, 784, 821, 748, 786, 751, 761, 757, 797] payload_rows_per_dst=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] dup_rows_per_dst=[3152, 2868, 3094, 2994, 2914, 2934, 2956, 3032] combine_rows_per_src=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65050 dup_rows=24060 dup_rows_remote=21114 dedup_ratio=0.244751 router_tokens_per_expert=[781, 748, 782, 763, 772, 783, 804, 810, 797, 716, 742, 713, 811, 740, 758, 759, 744, 774, 739, 715, 792, 773, 740, 757, 778, 764, 758, 719, 760, 767, 788, 789, 800, 790, 767, 789, 811, 799, 759, 743, 748, 769, 760, 794, 787, 739, 762, 771, 777, 748, 754, 769, 762, 744, 772, 744, 817, 766, 745, 766, 794, 819, 784, 767] payload_rows_per_dst=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] dup_rows_per_dst=[3146, 2906, 2862, 2942, 3120, 3032, 2946, 3106] combine_rows_per_src=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342624 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265828960 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531315296 weight_movement_bytes=34603008 total_nvlink_bytes=565918304 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343952 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674064 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004176 weight_movement_bytes=34603008 total_nvlink_bytes=567607184 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64940 dup_rows=23894 dup_rows_remote=20806 dedup_ratio=0.243062 router_tokens_per_expert=[760, 737, 739, 778, 775, 791, 785, 812, 793, 801, 746, 735, 780, 734, 739, 750, 812, 772, 767, 781, 762, 809, 801, 775, 781, 724, 752, 739, 790, 726, 798, 735, 760, 813, 766, 730, 819, 754, 764, 777, 774, 697, 763, 801, 784, 774, 806, 736, 738, 751, 803, 757, 751, 783, 739, 781, 724, 794, 803, 729, 826, 733, 778, 765] payload_rows_per_dst=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] dup_rows_per_dst=[3094, 2994, 3088, 2870, 3042, 2966, 2842, 2998] combine_rows_per_src=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343808 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919680 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495552 weight_movement_bytes=34603008 total_nvlink_bytes=568098560 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343704 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059416 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775128 weight_movement_bytes=69206016 total_nvlink_bytes=600981144 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343824 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608400 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532872976 weight_movement_bytes=34603008 total_nvlink_bytes=567475984 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344656 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266789456 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533234256 weight_movement_bytes=138412032 total_nvlink_bytes=671646288 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=342984 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337224 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331464 weight_movement_bytes=34603008 total_nvlink_bytes=566934472 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65086 dup_rows=23992 dup_rows_remote=20908 dedup_ratio=0.244059 router_tokens_per_expert=[782, 742, 758, 727, 734, 797, 766, 758, 787, 774, 746, 753, 728, 777, 783, 772, 762, 752, 802, 728, 792, 734, 761, 811, 768, 811, 815, 750, 775, 741, 738, 791, 759, 760, 790, 771, 785, 748, 750, 750, 718, 794, 762, 753, 760, 794, 791, 781, 728, 785, 808, 807, 780, 766, 752, 808, 763, 776, 754, 750, 750, 786, 824, 734] payload_rows_per_dst=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] dup_rows_per_dst=[2566, 2920, 3334, 3092, 2840, 2940, 3216, 3084] combine_rows_per_src=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65136 dup_rows=24140 dup_rows_remote=21272 dedup_ratio=0.245565 router_tokens_per_expert=[789, 698, 710, 752, 746, 745, 736, 749, 776, 816, 761, 768, 759, 735, 811, 771, 751, 745, 788, 762, 755, 774, 773, 741, 780, 780, 762, 791, 770, 766, 786, 790, 804, 830, 723, 744, 794, 794, 741, 790, 750, 749, 760, 760, 766, 772, 745, 772, 834, 768, 787, 767, 802, 820, 771, 784, 756, 771, 769, 781, 727, 755, 754, 776] payload_rows_per_dst=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] dup_rows_per_dst=[2868, 3386, 2936, 3130, 3028, 3260, 2886, 2646] combine_rows_per_src=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65116 dup_rows=23934 dup_rows_remote=20916 dedup_ratio=0.243469 router_tokens_per_expert=[777, 758, 728, 736, 788, 774, 810, 796, 740, 746, 747, 808, 812, 773, 743, 767, 796, 762, 778, 759, 739, 734, 711, 752, 807, 785, 762, 777, 824, 796, 727, 766, 763, 793, 775, 752, 773, 780, 781, 808, 776, 774, 758, 769, 727, 786, 780, 774, 734, 748, 776, 801, 795, 745, 749, 751, 705, 776, 770, 806, 795, 750, 790, 714] payload_rows_per_dst=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] dup_rows_per_dst=[2962, 3018, 2858, 3128, 3118, 2938, 2970, 2942] combine_rows_per_src=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65364 dup_rows=23818 dup_rows_remote=20956 dedup_ratio=0.242289 router_tokens_per_expert=[815, 777, 766, 765, 790, 766, 770, 784, 807, 783, 744, 747, 767, 805, 781, 761, 768, 764, 771, 763, 767, 761, 761, 731, 768, 791, 741, 761, 797, 780, 762, 792, 815, 731, 770, 761, 734, 805, 775, 805, 723, 734, 790, 764, 700, 775, 763, 743, 734, 780, 761, 743, 754, 761, 776, 777, 785, 786, 810, 777, 784, 717, 771, 742] payload_rows_per_dst=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] dup_rows_per_dst=[3014, 3088, 2880, 2984, 2990, 2862, 2998, 3002] combine_rows_per_src=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343976 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266936232 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533528488 weight_movement_bytes=69206016 total_nvlink_bytes=602734504 weight_to_token_ratio=0.129714 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21056 dedup_ratio=0.245138 router_tokens_per_expert=[755, 785, 784, 784, 732, 811, 770, 799, 772, 782, 776, 766, 740, 774, 770, 775, 769, 794, 752, 780, 772, 786, 741, 778, 742, 763, 768, 693, 771, 792, 763, 749, 740, 786, 797, 767, 768, 755, 750, 778, 754, 718, 772, 770, 774, 764, 735, 779, 746, 791, 781, 766, 810, 810, 792, 750, 762, 783, 740, 793, 772, 721, 724, 816] payload_rows_per_dst=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] dup_rows_per_dst=[3100, 2970, 2970, 2964, 3042, 2938, 3118, 2996] combine_rows_per_src=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64828 dup_rows=24062 dup_rows_remote=20938 dedup_ratio=0.244771 router_tokens_per_expert=[817, 760, 789, 813, 758, 773, 714, 784, 792, 743, 757, 781, 740, 792, 767, 747, 750, 767, 722, 712, 740, 757, 713, 786, 725, 795, 821, 790, 784, 747, 842, 765, 774, 736, 770, 802, 748, 757, 753, 794, 748, 768, 753, 789, 741, 723, 781, 794, 765, 763, 744, 783, 769, 766, 792, 756, 732, 801, 757, 800, 791, 778, 819, 762] payload_rows_per_dst=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] dup_rows_per_dst=[3124, 3008, 2792, 3124, 2994, 2958, 3026, 3036] combine_rows_per_src=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64826 dup_rows=24068 dup_rows_remote=21042 dedup_ratio=0.244832 router_tokens_per_expert=[821, 712, 726, 800, 716, 789, 772, 807, 757, 739, 790, 719, 734, 724, 796, 751, 776, 755, 743, 769, 764, 767, 733, 782, 839, 764, 756, 773, 751, 761, 784, 770, 753, 777, 771, 786, 746, 765, 786, 771, 763, 728, 823, 774, 768, 715, 800, 816, 807, 759, 779, 766, 745, 747, 790, 825, 728, 766, 794, 795, 772, 770, 770, 757] payload_rows_per_dst=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] dup_rows_per_dst=[2974, 2846, 2960, 3034, 3012, 3168, 3026, 3048] combine_rows_per_src=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=345632 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267142688 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533939744 weight_movement_bytes=34603008 total_nvlink_bytes=568542752 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65142 dup_rows=23868 dup_rows_remote=20966 dedup_ratio=0.242798 router_tokens_per_expert=[766, 791, 752, 796, 766, 771, 733, 768, 752, 733, 785, 772, 751, 787, 832, 760, 834, 716, 779, 741, 717, 797, 767, 747, 761, 782, 737, 762, 744, 797, 762, 750, 764, 756, 785, 754, 742, 799, 761, 818, 777, 801, 772, 769, 815, 740, 769, 734, 790, 756, 791, 790, 754, 762, 769, 761, 729, 787, 785, 753, 724, 771, 745, 821] payload_rows_per_dst=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] dup_rows_per_dst=[2944, 2998, 2902, 2960, 3058, 3054, 3004, 2948] combine_rows_per_src=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344128 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267059264 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774400 weight_movement_bytes=34603008 total_nvlink_bytes=568377408 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=345280 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268076224 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535807168 weight_movement_bytes=34603008 total_nvlink_bytes=570410176 weight_to_token_ratio=0.064581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344088 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266444824 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545560 weight_movement_bytes=69206016 total_nvlink_bytes=601751576 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343064 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265878552 ingress_bytes=369344512 link_bytes=369344512 token_bytes=531414040 weight_movement_bytes=103809024 total_nvlink_bytes=635223064 weight_to_token_ratio=0.195345 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389813664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343472 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265870768 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398064 weight_movement_bytes=69206016 total_nvlink_bytes=600604080 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344432 combine_payload_bytes=266821632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267166064 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533987696 weight_movement_bytes=69206016 total_nvlink_bytes=603193712 weight_to_token_ratio=0.129602 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64700 dup_rows=24122 dup_rows_remote=20840 dedup_ratio=0.245382 router_tokens_per_expert=[787, 786, 777, 760, 741, 779, 801, 715, 763, 777, 798, 767, 757, 772, 794, 768, 761, 758, 727, 847, 786, 765, 761, 780, 772, 751, 774, 783, 763, 815, 754, 795, 750, 767, 746, 723, 789, 836, 767, 788, 753, 739, 731, 810, 744, 746, 735, 782, 730, 745, 775, 774, 730, 796, 785, 723, 758, 777, 764, 794, 773, 790, 742, 756] payload_rows_per_dst=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] dup_rows_per_dst=[3004, 3012, 3048, 3060, 2924, 2882, 2910, 3282] combine_rows_per_src=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65076 dup_rows=23768 dup_rows_remote=20656 dedup_ratio=0.241781 router_tokens_per_expert=[807, 811, 759, 743, 750, 793, 773, 753, 771, 763, 761, 774, 794, 787, 756, 782, 786, 769, 708, 750, 757, 709, 793, 788, 729, 747, 749, 800, 780, 714, 760, 808, 726, 762, 755, 774, 774, 743, 821, 773, 740, 797, 750, 811, 757, 758, 727, 782, 780, 772, 794, 747, 778, 761, 722, 809, 780, 759, 771, 782, 774, 803, 745, 801] payload_rows_per_dst=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] dup_rows_per_dst=[3112, 3236, 2830, 2860, 2978, 3146, 2664, 2942] combine_rows_per_src=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65004 dup_rows=23918 dup_rows_remote=20774 dedup_ratio=0.243306 router_tokens_per_expert=[756, 747, 768, 786, 807, 754, 766, 787, 815, 783, 803, 741, 768, 740, 822, 791, 744, 765, 779, 774, 735, 772, 726, 734, 737, 743, 762, 744, 809, 754, 773, 757, 758, 753, 770, 741, 721, 737, 779, 801, 785, 785, 773, 792, 830, 796, 744, 740, 762, 764, 744, 766, 788, 736, 760, 772, 743, 809, 777, 756, 816, 773, 751, 788] payload_rows_per_dst=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] dup_rows_per_dst=[3070, 3144, 2844, 2932, 2876, 3106, 2952, 2994] combine_rows_per_src=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65260 dup_rows=23648 dup_rows_remote=20748 dedup_ratio=0.240560 router_tokens_per_expert=[781, 760, 746, 750, 762, 792, 761, 776, 765, 802, 733, 760, 764, 762, 707, 758, 749, 745, 740, 789, 779, 791, 756, 769, 800, 757, 795, 804, 786, 749, 758, 718, 800, 791, 761, 832, 817, 764, 778, 776, 759, 790, 766, 764, 736, 765, 772, 796, 785, 737, 747, 763, 771, 799, 795, 771, 767, 744, 770, 746, 791, 747, 769, 719] payload_rows_per_dst=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] dup_rows_per_dst=[2962, 2908, 3006, 2946, 3124, 2900, 3022, 2780] combine_rows_per_src=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342160 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265353360 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530364560 weight_movement_bytes=34603008 total_nvlink_bytes=564967568 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21036 dedup_ratio=0.245911 router_tokens_per_expert=[796, 785, 781, 764, 730, 720, 766, 732, 780, 753, 771, 764, 765, 789, 728, 780, 750, 831, 740, 772, 771, 809, 807, 763, 778, 780, 786, 759, 772, 756, 737, 731, 802, 791, 788, 728, 781, 789, 760, 789, 765, 787, 778, 751, 724, 809, 773, 783, 755, 793, 766, 750, 746, 774, 766, 805, 736, 747, 732, 770, 786, 784, 756, 742] payload_rows_per_dst=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] dup_rows_per_dst=[2964, 3096, 3090, 2980, 3138, 2960, 2978, 2968] combine_rows_per_src=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65036 dup_rows=23792 dup_rows_remote=20600 dedup_ratio=0.242025 router_tokens_per_expert=[769, 760, 764, 766, 803, 760, 768, 743, 773, 753, 788, 771, 763, 747, 740, 747, 813, 798, 770, 726, 767, 769, 763, 786, 815, 772, 745, 759, 798, 793, 764, 807, 720, 738, 762, 790, 743, 773, 736, 752, 767, 772, 734, 772, 761, 751, 743, 722, 791, 798, 790, 821, 807, 786, 792, 749, 784, 796, 787, 790, 735, 702, 760, 768] payload_rows_per_dst=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] dup_rows_per_dst=[2976, 2830, 2972, 3088, 2902, 2836, 3192, 2996] combine_rows_per_src=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65076 dup_rows=24090 dup_rows_remote=21166 dedup_ratio=0.245056 router_tokens_per_expert=[715, 811, 745, 783, 730, 758, 755, 800, 776, 765, 789, 773, 757, 762, 775, 739, 749, 769, 792, 797, 780, 770, 804, 767, 712, 798, 755, 780, 735, 744, 737, 770, 781, 775, 750, 725, 789, 745, 773, 756, 793, 759, 771, 786, 784, 753, 793, 766, 820, 719, 787, 769, 775, 716, 809, 787, 761, 776, 766, 794, 746, 771, 800, 765] payload_rows_per_dst=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] dup_rows_per_dst=[2996, 2988, 3048, 2924, 3006, 3036, 3022, 3070] combine_rows_per_src=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=342928 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266894224 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533445520 weight_movement_bytes=69206016 total_nvlink_bytes=602651536 weight_to_token_ratio=0.129734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65188 dup_rows=23846 dup_rows_remote=20850 dedup_ratio=0.242574 router_tokens_per_expert=[711, 767, 785, 753, 745, 777, 766, 816, 719, 733, 788, 806, 782, 764, 758, 761, 769, 773, 798, 774, 755, 735, 759, 770, 762, 797, 766, 800, 799, 746, 754, 781, 822, 801, 741, 768, 756, 727, 769, 761, 788, 792, 804, 732, 724, 804, 733, 755, 768, 715, 750, 766, 757, 787, 776, 758, 811, 803, 758, 724, 785, 758, 770, 820] payload_rows_per_dst=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] dup_rows_per_dst=[3024, 2932, 2996, 3060, 3024, 2910, 2812, 3088] combine_rows_per_src=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343112 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266599496 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855880 weight_movement_bytes=69206016 total_nvlink_bytes=602061896 weight_to_token_ratio=0.129878 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=344032 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648992 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534953952 weight_movement_bytes=34603008 total_nvlink_bytes=569556960 weight_to_token_ratio=0.064684 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343392 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265813344 ingress_bytes=334675968 link_bytes=334675968 token_bytes=531283296 weight_movement_bytes=69206016 total_nvlink_bytes=600489312 weight_to_token_ratio=0.130262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259358724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266730000 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117456 weight_movement_bytes=69206016 total_nvlink_bytes=602323472 weight_to_token_ratio=0.129814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344968 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266896264 ingress_bytes=370360320 link_bytes=370360320 token_bytes=533447560 weight_movement_bytes=103809024 total_nvlink_bytes=637256584 weight_to_token_ratio=0.194600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393636068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344152 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354200 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364248 weight_movement_bytes=34603008 total_nvlink_bytes=568967256 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65034 dup_rows=23918 dup_rows_remote=21022 dedup_ratio=0.243306 router_tokens_per_expert=[783, 756, 776, 724, 779, 787, 813, 777, 754, 785, 730, 771, 798, 771, 812, 739, 775, 818, 754, 703, 769, 833, 769, 768, 755, 724, 747, 770, 774, 773, 785, 766, 774, 812, 776, 735, 786, 801, 756, 727, 795, 759, 743, 766, 743, 798, 733, 778, 763, 756, 772, 748, 780, 767, 771, 766, 773, 760, 743, 780, 753, 768, 788, 744] payload_rows_per_dst=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] dup_rows_per_dst=[2888, 3176, 3012, 2616, 3076, 3026, 3228, 2896] combine_rows_per_src=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64956 dup_rows=23940 dup_rows_remote=20828 dedup_ratio=0.243530 router_tokens_per_expert=[770, 768, 755, 735, 798, 783, 742, 747, 802, 805, 741, 788, 757, 795, 759, 750, 729, 767, 752, 764, 802, 724, 797, 759, 765, 774, 795, 800, 759, 781, 761, 816, 797, 777, 739, 809, 785, 759, 815, 742, 789, 769, 711, 808, 742, 791, 725, 801, 802, 772, 750, 737, 751, 779, 738, 789, 739, 687, 764, 793, 783, 753, 757, 759] payload_rows_per_dst=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] dup_rows_per_dst=[3112, 3114, 2896, 3124, 3070, 2980, 2804, 2840] combine_rows_per_src=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64800 dup_rows=24226 dup_rows_remote=21174 dedup_ratio=0.246440 router_tokens_per_expert=[820, 790, 735, 807, 764, 753, 692, 766, 805, 776, 755, 767, 768, 791, 764, 739, 719, 785, 772, 783, 780, 743, 762, 745, 791, 824, 737, 739, 833, 760, 760, 770, 777, 738, 727, 770, 739, 815, 705, 757, 793, 809, 791, 731, 768, 806, 834, 769, 713, 728, 733, 766, 814, 769, 761, 734, 831, 748, 700, 764, 799, 771, 783, 814] payload_rows_per_dst=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] dup_rows_per_dst=[3004, 3052, 2912, 3090, 2924, 3230, 2886, 3128] combine_rows_per_src=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344224 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723488 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102752 weight_movement_bytes=69206016 total_nvlink_bytes=602308768 weight_to_token_ratio=0.129817 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64958 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[768, 775, 774, 759, 828, 746, 752, 804, 765, 789, 755, 780, 778, 782, 779, 762, 719, 740, 767, 830, 787, 748, 761, 747, 758, 770, 809, 766, 788, 745, 732, 781, 815, 769, 730, 778, 778, 766, 749, 739, 796, 758, 761, 761, 754, 748, 796, 760, 764, 776, 772, 799, 771, 746, 795, 727, 756, 808, 735, 782, 751, 765, 734, 769] payload_rows_per_dst=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] dup_rows_per_dst=[3042, 3004, 2994, 3132, 2932, 2902, 2990, 2984] combine_rows_per_src=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64968 dup_rows=23926 dup_rows_remote=20768 dedup_ratio=0.243388 router_tokens_per_expert=[723, 821, 749, 757, 756, 752, 779, 790, 744, 728, 754, 780, 784, 774, 757, 732, 750, 743, 738, 777, 752, 783, 757, 771, 780, 709, 766, 726, 785, 798, 790, 797, 754, 841, 775, 788, 814, 766, 771, 775, 798, 774, 765, 760, 734, 742, 786, 799, 726, 776, 779, 791, 764, 746, 779, 752, 809, 781, 802, 752, 784, 748, 795, 724] payload_rows_per_dst=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] dup_rows_per_dst=[3016, 2926, 2898, 3020, 3158, 2886, 2998, 3024] combine_rows_per_src=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64930 dup_rows=24192 dup_rows_remote=21150 dedup_ratio=0.246094 router_tokens_per_expert=[776, 805, 800, 782, 772, 828, 767, 711, 739, 741, 754, 786, 766, 779, 747, 750, 798, 777, 755, 720, 760, 769, 760, 784, 743, 774, 770, 764, 756, 784, 805, 785, 760, 762, 721, 779, 736, 767, 775, 744, 789, 758, 743, 779, 785, 837, 816, 735, 759, 765, 750, 765, 800, 736, 753, 784, 760, 771, 767, 760, 728, 785, 796, 780] payload_rows_per_dst=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] dup_rows_per_dst=[3084, 2962, 3060, 3084, 2876, 3082, 3042, 3002] combine_rows_per_src=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64820 dup_rows=24198 dup_rows_remote=21078 dedup_ratio=0.246155 router_tokens_per_expert=[788, 760, 758, 745, 779, 766, 721, 726, 803, 802, 770, 774, 805, 755, 735, 740, 726, 745, 740, 739, 748, 793, 799, 770, 787, 772, 735, 779, 775, 781, 788, 786, 755, 797, 820, 764, 790, 783, 767, 793, 796, 774, 793, 768, 808, 828, 752, 734, 749, 726, 781, 728, 764, 789, 765, 759, 734, 745, 739, 800, 757, 751, 772, 781] payload_rows_per_dst=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] dup_rows_per_dst=[2932, 3032, 2970, 3120, 3242, 3112, 2830, 2960] combine_rows_per_src=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343136 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402912 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462688 weight_movement_bytes=34603008 total_nvlink_bytes=567065696 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65046 dup_rows=24098 dup_rows_remote=21112 dedup_ratio=0.245138 router_tokens_per_expert=[810, 735, 796, 777, 764, 763, 771, 808, 770, 756, 762, 766, 770, 791, 748, 779, 748, 755, 756, 769, 734, 772, 800, 739, 736, 745, 766, 735, 748, 805, 793, 773, 749, 775, 742, 786, 763, 776, 733, 780, 767, 749, 762, 781, 754, 768, 778, 773, 744, 761, 752, 887, 819, 760, 729, 785, 821, 753, 773, 729, 763, 766, 805, 729] payload_rows_per_dst=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] dup_rows_per_dst=[3074, 2920, 2986, 3020, 2920, 3064, 3056, 3058] combine_rows_per_src=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343896 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764696 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185496 weight_movement_bytes=34603008 total_nvlink_bytes=565788504 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344144 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266412112 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480080 weight_movement_bytes=69206016 total_nvlink_bytes=601686096 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342944 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266451872 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532560800 weight_movement_bytes=69206016 total_nvlink_bytes=601766816 weight_to_token_ratio=0.129950 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=344320 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266297600 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250880 weight_movement_bytes=34603008 total_nvlink_bytes=566853888 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343592 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846312 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349032 weight_movement_bytes=103809024 total_nvlink_bytes=635158056 weight_to_token_ratio=0.195369 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64850 dup_rows=24220 dup_rows_remote=21064 dedup_ratio=0.246379 router_tokens_per_expert=[775, 800, 797, 767, 719, 782, 747, 781, 771, 804, 752, 754, 771, 693, 771, 741, 757, 788, 771, 747, 762, 828, 765, 752, 789, 772, 715, 781, 737, 771, 758, 777, 781, 756, 769, 790, 748, 780, 785, 759, 794, 801, 745, 796, 770, 736, 792, 770, 761, 789, 770, 788, 754, 789, 790, 755, 743, 750, 737, 728, 830, 799, 723, 779] payload_rows_per_dst=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] dup_rows_per_dst=[3076, 2918, 3018, 2964, 3028, 3096, 2964, 3156] combine_rows_per_src=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344632 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266773048 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533201464 weight_movement_bytes=69206016 total_nvlink_bytes=602407480 weight_to_token_ratio=0.129793 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64944 dup_rows=23974 dup_rows_remote=20822 dedup_ratio=0.243876 router_tokens_per_expert=[735, 799, 798, 776, 735, 752, 785, 765, 698, 718, 764, 805, 743, 787, 748, 788, 740, 778, 787, 762, 749, 772, 763, 711, 806, 803, 835, 790, 795, 764, 748, 788, 799, 774, 732, 763, 758, 795, 776, 735, 757, 806, 758, 739, 765, 779, 754, 800, 800, 735, 791, 761, 737, 809, 782, 745, 733, 793, 799, 753, 815, 728, 743, 751] payload_rows_per_dst=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] dup_rows_per_dst=[3152, 2906, 3182, 3242, 2836, 2682, 3044, 2930] combine_rows_per_src=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343656 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969256 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594856 weight_movement_bytes=34603008 total_nvlink_bytes=566197864 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65146 dup_rows=23968 dup_rows_remote=20914 dedup_ratio=0.243815 router_tokens_per_expert=[767, 749, 744, 755, 772, 773, 789, 811, 727, 833, 762, 723, 799, 752, 757, 769, 779, 769, 794, 770, 767, 830, 763, 784, 769, 720, 752, 787, 752, 763, 768, 810, 768, 737, 785, 779, 713, 771, 764, 768, 744, 790, 809, 827, 782, 786, 774, 731, 766, 762, 755, 752, 738, 801, 738, 777, 770, 794, 725, 758, 717, 747, 782, 783] payload_rows_per_dst=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] dup_rows_per_dst=[2986, 3054, 3096, 2948, 2968, 3088, 2944, 2884] combine_rows_per_src=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65078 dup_rows=24016 dup_rows_remote=20998 dedup_ratio=0.244303 router_tokens_per_expert=[811, 720, 756, 768, 743, 778, 779, 791, 736, 822, 795, 795, 743, 802, 787, 798, 798, 745, 810, 734, 765, 758, 783, 795, 747, 728, 764, 744, 751, 773, 732, 799, 771, 811, 767, 777, 746, 745, 799, 763, 760, 774, 760, 759, 775, 776, 773, 737, 717, 765, 816, 793, 764, 741, 790, 715, 729, 763, 744, 749, 799, 779, 778, 767] payload_rows_per_dst=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] dup_rows_per_dst=[3028, 3152, 3020, 2844, 3130, 3018, 2900, 2924] combine_rows_per_src=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64982 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[766, 753, 803, 745, 760, 719, 796, 750, 781, 740, 809, 760, 779, 767, 743, 744, 774, 726, 776, 765, 727, 771, 791, 751, 779, 776, 789, 779, 817, 754, 809, 774, 733, 759, 816, 773, 775, 767, 786, 758, 735, 746, 788, 761, 781, 759, 788, 777, 772, 762, 774, 684, 765, 759, 781, 785, 800, 775, 818, 721, 770, 749, 777, 785] payload_rows_per_dst=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] dup_rows_per_dst=[2948, 2926, 2932, 3088, 3080, 2984, 3042, 3068] combine_rows_per_src=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64652 dup_rows=24182 dup_rows_remote=20956 dedup_ratio=0.245992 router_tokens_per_expert=[779, 784, 821, 765, 753, 746, 743, 717, 768, 775, 791, 779, 754, 750, 756, 751, 764, 783, 800, 742, 765, 749, 799, 757, 772, 747, 781, 743, 804, 806, 734, 735, 771, 730, 769, 719, 736, 756, 789, 783, 748, 747, 751, 788, 795, 768, 792, 761, 808, 754, 795, 809, 798, 793, 790, 801, 763, 785, 755, 779, 732, 772, 754, 748] payload_rows_per_dst=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] dup_rows_per_dst=[3016, 3004, 2978, 2938, 2928, 3090, 3226, 3002] combine_rows_per_src=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343064 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353688 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364312 weight_movement_bytes=69206016 total_nvlink_bytes=601570328 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64944 dup_rows=24130 dup_rows_remote=21050 dedup_ratio=0.245463 router_tokens_per_expert=[766, 756, 717, 761, 722, 762, 810, 792, 711, 729, 780, 773, 783, 752, 810, 770, 740, 807, 825, 764, 720, 746, 775, 796, 795, 730, 774, 775, 777, 765, 759, 780, 726, 775, 777, 754, 734, 769, 802, 770, 726, 777, 785, 776, 793, 798, 770, 753, 789, 764, 796, 764, 730, 767, 752, 770, 776, 785, 766, 820, 733, 776, 770, 787] payload_rows_per_dst=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] dup_rows_per_dst=[3002, 2962, 3126, 3080, 2922, 3004, 2966, 3068] combine_rows_per_src=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344240 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182256 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020272 weight_movement_bytes=34603008 total_nvlink_bytes=568623280 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65206 dup_rows=23708 dup_rows_remote=20704 dedup_ratio=0.241170 router_tokens_per_expert=[749, 734, 806, 743, 812, 771, 795, 776, 776, 761, 738, 736, 765, 731, 847, 709, 754, 782, 761, 767, 796, 773, 807, 757, 766, 781, 774, 786, 773, 752, 749, 791, 777, 751, 789, 749, 765, 747, 761, 763, 784, 786, 783, 794, 748, 745, 801, 792, 774, 742, 743, 816, 784, 755, 773, 744, 694, 761, 738, 742, 806, 818, 780, 729] payload_rows_per_dst=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] dup_rows_per_dst=[3052, 2856, 3004, 3006, 2868, 3062, 2954, 2906] combine_rows_per_src=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344304 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266903792 ingress_bytes=404971520 link_bytes=404971520 token_bytes=533463280 weight_movement_bytes=138412032 total_nvlink_bytes=671875312 weight_to_token_ratio=0.259459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523875227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343880 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510152 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676424 weight_movement_bytes=34603008 total_nvlink_bytes=567279432 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342432 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265157024 ingress_bytes=368623616 link_bytes=368623616 token_bytes=529971616 weight_movement_bytes=103809024 total_nvlink_bytes=633780640 weight_to_token_ratio=0.195877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387100990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343976 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266354600 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365224 weight_movement_bytes=34603008 total_nvlink_bytes=566968232 weight_to_token_ratio=0.064999 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64928 dup_rows=23946 dup_rows_remote=20956 dedup_ratio=0.243591 router_tokens_per_expert=[759, 778, 756, 783, 747, 796, 758, 742, 765, 811, 756, 743, 771, 782, 788, 764, 753, 784, 742, 834, 700, 766, 774, 749, 744, 758, 743, 731, 788, 755, 757, 739, 765, 732, 776, 762, 808, 791, 751, 786, 770, 808, 786, 795, 789, 790, 768, 780, 757, 764, 751, 746, 825, 759, 749, 756, 761, 813, 746, 776, 792, 758, 736, 790] payload_rows_per_dst=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] dup_rows_per_dst=[2764, 3026, 3180, 2968, 2948, 2946, 3124, 2990] combine_rows_per_src=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343640 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427416 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511192 weight_movement_bytes=34603008 total_nvlink_bytes=569114200 weight_to_token_ratio=0.064738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65040 dup_rows=23916 dup_rows_remote=20928 dedup_ratio=0.243286 router_tokens_per_expert=[723, 777, 776, 765, 765, 763, 802, 782, 752, 736, 777, 752, 766, 789, 778, 806, 755, 747, 787, 747, 767, 741, 737, 809, 747, 752, 788, 823, 753, 788, 734, 775, 754, 808, 772, 750, 730, 772, 769, 773, 768, 780, 771, 818, 719, 790, 773, 772, 778, 767, 752, 738, 755, 790, 810, 746, 734, 738, 712, 758, 800, 825, 772, 799] payload_rows_per_dst=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] dup_rows_per_dst=[2988, 3008, 2846, 3084, 3132, 2880, 2960, 3018] combine_rows_per_src=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343536 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288624 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233712 weight_movement_bytes=69206016 total_nvlink_bytes=601439728 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65042 dup_rows=24060 dup_rows_remote=20974 dedup_ratio=0.244751 router_tokens_per_expert=[775, 740, 766, 786, 798, 748, 827, 797, 769, 800, 751, 770, 744, 819, 715, 776, 712, 795, 816, 755, 782, 809, 731, 765, 779, 774, 748, 750, 774, 775, 790, 785, 749, 766, 764, 771, 790, 753, 721, 742, 740, 782, 729, 741, 760, 760, 817, 747, 739, 720, 791, 774, 797, 749, 780, 793, 777, 768, 744, 794, 751, 735, 771, 816] payload_rows_per_dst=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] dup_rows_per_dst=[3122, 3086, 3088, 3000, 2920, 2796, 3050, 2998] combine_rows_per_src=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65042 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[795, 723, 748, 769, 792, 723, 803, 809, 812, 736, 797, 776, 799, 757, 754, 749, 791, 720, 764, 791, 758, 777, 736, 745, 756, 769, 792, 804, 797, 746, 749, 785, 751, 788, 785, 749, 781, 764, 728, 785, 753, 781, 794, 784, 753, 805, 731, 797, 781, 740, 728, 785, 748, 791, 792, 692, 743, 809, 734, 752, 788, 792, 752, 774] payload_rows_per_dst=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] dup_rows_per_dst=[3044, 2964, 2986, 3014, 3004, 3134, 2896, 2958] combine_rows_per_src=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65134 dup_rows=23808 dup_rows_remote=20836 dedup_ratio=0.242188 router_tokens_per_expert=[727, 804, 796, 790, 761, 784, 762, 784, 797, 768, 773, 749, 749, 768, 738, 774, 749, 793, 762, 717, 701, 778, 798, 785, 742, 754, 795, 759, 831, 751, 705, 713, 743, 777, 757, 817, 739, 795, 747, 792, 806, 769, 783, 755, 779, 764, 772, 736, 751, 787, 794, 752, 755, 774, 752, 787, 768, 792, 794, 742, 760, 760, 793, 803] payload_rows_per_dst=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] dup_rows_per_dst=[3036, 2890, 2888, 2892, 2972, 3012, 3052, 3066] combine_rows_per_src=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343872 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747712 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151552 weight_movement_bytes=34603008 total_nvlink_bytes=567754560 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64996 dup_rows=23922 dup_rows_remote=20912 dedup_ratio=0.243347 router_tokens_per_expert=[798, 759, 780, 723, 722, 755, 803, 789, 763, 760, 718, 796, 761, 756, 752, 720, 701, 785, 752, 762, 773, 802, 799, 757, 772, 739, 737, 775, 751, 705, 811, 771, 768, 778, 791, 780, 775, 770, 836, 747, 791, 812, 799, 765, 778, 792, 734, 791, 766, 787, 746, 748, 749, 789, 833, 780, 762, 750, 755, 774, 784, 773, 759, 743] payload_rows_per_dst=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] dup_rows_per_dst=[3056, 2822, 2966, 2902, 3070, 3104, 3010, 2992] combine_rows_per_src=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65184 dup_rows=23968 dup_rows_remote=21210 dedup_ratio=0.243815 router_tokens_per_expert=[733, 802, 793, 752, 789, 779, 748, 790, 773, 770, 755, 804, 778, 757, 767, 801, 791, 732, 807, 745, 780, 731, 738, 738, 743, 764, 741, 737, 742, 754, 733, 741, 759, 731, 772, 781, 784, 775, 802, 718, 772, 822, 777, 743, 774, 779, 754, 757, 797, 757, 748, 764, 767, 752, 818, 718, 781, 784, 810, 768, 783, 813, 785, 799] payload_rows_per_dst=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] dup_rows_per_dst=[3108, 3104, 2872, 2758, 2938, 3044, 2944, 3200] combine_rows_per_src=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64774 dup_rows=24296 dup_rows_remote=21188 dedup_ratio=0.247152 router_tokens_per_expert=[790, 800, 743, 811, 753, 815, 723, 775, 760, 753, 764, 774, 701, 683, 775, 731, 738, 762, 720, 724, 815, 833, 786, 793, 712, 809, 807, 779, 789, 762, 799, 778, 738, 796, 767, 721, 762, 775, 794, 801, 778, 821, 727, 753, 768, 807, 730, 761, 765, 762, 783, 806, 761, 739, 803, 779, 800, 714, 784, 770, 701, 762, 825, 742] payload_rows_per_dst=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] dup_rows_per_dst=[3092, 2832, 3108, 3106, 2998, 3010, 3186, 2964] combine_rows_per_src=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344064 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756096 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168128 weight_movement_bytes=34603008 total_nvlink_bytes=567771136 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343632 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266755664 ingress_bytes=370221056 link_bytes=370221056 token_bytes=533167696 weight_movement_bytes=103809024 total_nvlink_bytes=636976720 weight_to_token_ratio=0.194702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343880 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267132744 ingress_bytes=335994880 link_bytes=335994880 token_bytes=533921608 weight_movement_bytes=69206016 total_nvlink_bytes=603127624 weight_to_token_ratio=0.129618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264321684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343632 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567248 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532790864 weight_movement_bytes=34603008 total_nvlink_bytes=567393872 weight_to_token_ratio=0.064947 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64956 dup_rows=23888 dup_rows_remote=20834 dedup_ratio=0.243001 router_tokens_per_expert=[777, 767, 764, 828, 733, 772, 789, 743, 762, 777, 757, 773, 748, 792, 793, 748, 739, 720, 733, 793, 734, 769, 774, 781, 722, 781, 802, 758, 735, 775, 763, 814, 774, 763, 800, 772, 755, 756, 768, 809, 738, 774, 775, 771, 760, 767, 798, 780, 801, 742, 757, 779, 816, 718, 733, 781, 764, 786, 771, 776, 747, 744, 761, 800] payload_rows_per_dst=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] dup_rows_per_dst=[2968, 3004, 2818, 2954, 3000, 3052, 3038, 3054] combine_rows_per_src=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=345576 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267339240 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332904 weight_movement_bytes=34603008 total_nvlink_bytes=568935912 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343848 combine_payload_bytes=265314304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265658152 ingress_bytes=334520320 link_bytes=334520320 token_bytes=530972456 weight_movement_bytes=69206016 total_nvlink_bytes=600178472 weight_to_token_ratio=0.130338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65024 dup_rows=23958 dup_rows_remote=20998 dedup_ratio=0.243713 router_tokens_per_expert=[726, 716, 786, 793, 808, 813, 758, 741, 779, 792, 756, 721, 739, 741, 764, 776, 768, 758, 766, 764, 779, 726, 791, 780, 757, 798, 819, 792, 741, 747, 779, 802, 786, 776, 749, 771, 778, 745, 797, 784, 735, 752, 764, 746, 760, 790, 784, 820, 785, 781, 788, 763, 730, 792, 754, 737, 778, 780, 752, 767, 747, 791, 743, 751] payload_rows_per_dst=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] dup_rows_per_dst=[2960, 3054, 2952, 3400, 3020, 2774, 3028, 2770] combine_rows_per_src=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343160 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402936 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462712 weight_movement_bytes=34603008 total_nvlink_bytes=567065720 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65012 dup_rows=24008 dup_rows_remote=20956 dedup_ratio=0.244222 router_tokens_per_expert=[765, 737, 752, 724, 753, 794, 762, 776, 762, 768, 735, 731, 770, 785, 816, 801, 780, 771, 793, 778, 754, 797, 744, 879, 775, 740, 761, 759, 749, 774, 731, 795, 797, 779, 730, 774, 794, 832, 771, 750, 783, 772, 793, 697, 728, 752, 774, 757, 761, 757, 776, 784, 792, 771, 813, 803, 743, 767, 765, 764, 732, 715, 715, 800] payload_rows_per_dst=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] dup_rows_per_dst=[2872, 3052, 3166, 2906, 3028, 2970, 3088, 2926] combine_rows_per_src=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21024 dedup_ratio=0.244161 router_tokens_per_expert=[766, 767, 689, 778, 766, 826, 774, 761, 773, 757, 753, 793, 761, 771, 769, 805, 781, 788, 741, 756, 783, 738, 776, 790, 796, 739, 746, 770, 753, 795, 780, 785, 782, 758, 787, 798, 754, 765, 744, 767, 713, 770, 763, 769, 760, 780, 807, 780, 780, 802, 767, 784, 761, 734, 800, 717, 791, 776, 760, 746, 754, 755, 754, 748] payload_rows_per_dst=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] dup_rows_per_dst=[3042, 3124, 2954, 3018, 2942, 2978, 2936, 3008] combine_rows_per_src=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64900 dup_rows=24136 dup_rows_remote=21046 dedup_ratio=0.245524 router_tokens_per_expert=[810, 796, 718, 772, 733, 795, 748, 805, 706, 713, 765, 808, 777, 786, 778, 802, 799, 729, 777, 776, 729, 743, 784, 773, 790, 779, 763, 763, 757, 699, 837, 714, 772, 779, 805, 775, 740, 746, 776, 786, 757, 759, 780, 768, 743, 794, 743, 771, 756, 752, 748, 762, 837, 754, 726, 739, 791, 768, 755, 763, 805, 781, 758, 839] payload_rows_per_dst=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] dup_rows_per_dst=[3094, 3074, 2974, 2964, 3090, 3038, 2878, 3024] combine_rows_per_src=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65224 dup_rows=23914 dup_rows_remote=21058 dedup_ratio=0.243266 router_tokens_per_expert=[762, 759, 772, 698, 783, 741, 807, 755, 800, 742, 783, 771, 743, 787, 774, 758, 740, 808, 740, 810, 791, 768, 814, 747, 762, 733, 726, 743, 791, 794, 805, 779, 793, 790, 750, 780, 773, 793, 743, 742, 786, 792, 783, 761, 773, 765, 789, 775, 748, 804, 732, 764, 723, 749, 752, 739, 755, 826, 749, 780, 790, 719, 751, 797] payload_rows_per_dst=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] dup_rows_per_dst=[2874, 3040, 3070, 3004, 3054, 3062, 2856, 2954] combine_rows_per_src=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21104 dedup_ratio=0.246195 router_tokens_per_expert=[761, 785, 772, 773, 768, 762, 771, 727, 761, 712, 767, 757, 753, 765, 751, 731, 798, 777, 810, 746, 747, 772, 773, 778, 733, 813, 741, 774, 801, 755, 771, 802, 751, 745, 775, 779, 700, 756, 764, 768, 757, 754, 736, 790, 741, 739, 801, 833, 797, 792, 775, 823, 749, 764, 748, 814, 820, 737, 782, 746, 804, 775, 761, 769] payload_rows_per_dst=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] dup_rows_per_dst=[2994, 2860, 3070, 3098, 2948, 2986, 3148, 3098] combine_rows_per_src=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344088 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266682392 ingress_bytes=370147328 link_bytes=370147328 token_bytes=533020696 weight_movement_bytes=103809024 total_nvlink_bytes=636829720 weight_to_token_ratio=0.194756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64902 dup_rows=24136 dup_rows_remote=21196 dedup_ratio=0.245524 router_tokens_per_expert=[765, 774, 768, 744, 756, 779, 771, 780, 756, 764, 764, 796, 780, 809, 762, 742, 789, 762, 730, 771, 729, 773, 778, 771, 718, 784, 762, 799, 784, 724, 801, 815, 786, 830, 791, 757, 748, 779, 742, 793, 748, 773, 729, 757, 797, 790, 735, 729, 752, 792, 741, 769, 770, 785, 735, 710, 754, 767, 776, 768, 769, 781, 802, 797] payload_rows_per_dst=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] dup_rows_per_dst=[3038, 3024, 2940, 3030, 3034, 2970, 3020, 3080] combine_rows_per_src=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343872 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633024 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922176 weight_movement_bytes=34603008 total_nvlink_bytes=567525184 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344080 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567696 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791312 weight_movement_bytes=34603008 total_nvlink_bytes=567394320 weight_to_token_ratio=0.064947 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64768 dup_rows=24038 dup_rows_remote=20898 dedup_ratio=0.244527 router_tokens_per_expert=[731, 755, 815, 774, 769, 715, 789, 806, 758, 748, 757, 762, 756, 822, 763, 732, 763, 795, 757, 782, 782, 745, 753, 794, 809, 764, 777, 810, 749, 799, 768, 756, 751, 800, 779, 749, 756, 772, 778, 760, 730, 779, 747, 743, 741, 767, 736, 747, 805, 790, 727, 822, 738, 730, 795, 730, 763, 779, 728, 815, 794, 764, 804, 778] payload_rows_per_dst=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] dup_rows_per_dst=[2960, 3002, 3058, 2980, 3074, 2918, 2906, 3140] combine_rows_per_src=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343784 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266174184 ingress_bytes=404242432 link_bytes=404242432 token_bytes=532004584 weight_movement_bytes=138412032 total_nvlink_bytes=670416616 weight_to_token_ratio=0.260171 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521131727e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=345128 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267502632 ingress_bytes=336363520 link_bytes=336363520 token_bytes=534660136 weight_movement_bytes=69206016 total_nvlink_bytes=603866152 weight_to_token_ratio=0.129439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265708846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344392 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182984 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532021576 weight_movement_bytes=69206016 total_nvlink_bytes=601227592 weight_to_token_ratio=0.130081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=342664 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632392 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922120 weight_movement_bytes=103809024 total_nvlink_bytes=634731144 weight_to_token_ratio=0.195526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65106 dup_rows=23896 dup_rows_remote=20786 dedup_ratio=0.243083 router_tokens_per_expert=[753, 774, 777, 760, 771, 772, 787, 755, 767, 758, 706, 758, 753, 768, 799, 798, 792, 775, 745, 779, 770, 759, 784, 789, 795, 746, 778, 785, 742, 778, 808, 740, 762, 782, 736, 830, 788, 779, 779, 757, 762, 735, 806, 710, 787, 780, 760, 792, 735, 776, 742, 829, 791, 727, 758, 737, 750, 760, 775, 766, 747, 771, 759, 763] payload_rows_per_dst=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] dup_rows_per_dst=[3110, 3212, 2774, 2876, 2998, 2948, 2998, 2980] combine_rows_per_src=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64844 dup_rows=24278 dup_rows_remote=21154 dedup_ratio=0.246969 router_tokens_per_expert=[810, 805, 781, 783, 769, 751, 758, 742, 788, 728, 746, 770, 789, 770, 791, 771, 788, 808, 767, 777, 751, 762, 771, 754, 778, 800, 730, 741, 737, 777, 739, 776, 761, 765, 688, 797, 759, 767, 818, 781, 784, 780, 769, 766, 793, 798, 769, 750, 807, 795, 769, 730, 725, 753, 787, 762, 756, 771, 765, 743, 782, 754, 744, 756] payload_rows_per_dst=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] dup_rows_per_dst=[3100, 3124, 3102, 2982, 2968, 3064, 2990, 2948] combine_rows_per_src=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64972 dup_rows=24050 dup_rows_remote=20994 dedup_ratio=0.244649 router_tokens_per_expert=[735, 771, 789, 803, 734, 734, 773, 762, 758, 767, 733, 756, 776, 785, 761, 774, 765, 800, 765, 759, 775, 813, 763, 778, 769, 763, 780, 790, 749, 738, 780, 775, 764, 765, 786, 754, 716, 790, 726, 754, 793, 794, 756, 716, 766, 767, 785, 792, 750, 777, 764, 789, 780, 763, 764, 769, 776, 749, 680, 826, 771, 796, 813, 788] payload_rows_per_dst=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] dup_rows_per_dst=[2952, 2998, 3028, 2984, 2896, 3056, 3046, 3090] combine_rows_per_src=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64608 dup_rows=24306 dup_rows_remote=21192 dedup_ratio=0.247253 router_tokens_per_expert=[796, 777, 781, 787, 740, 786, 742, 787, 752, 747, 737, 762, 746, 742, 694, 786, 718, 778, 756, 763, 766, 735, 790, 730, 768, 794, 757, 801, 717, 790, 760, 778, 772, 760, 848, 770, 796, 789, 735, 782, 724, 763, 729, 719, 770, 792, 844, 750, 800, 754, 757, 753, 814, 760, 756, 741, 796, 765, 775, 798, 807, 774, 801, 795] payload_rows_per_dst=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] dup_rows_per_dst=[3048, 2888, 2968, 3054, 3114, 2984, 3022, 3228] combine_rows_per_src=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65258 dup_rows=23880 dup_rows_remote=20918 dedup_ratio=0.242920 router_tokens_per_expert=[753, 729, 801, 821, 746, 722, 729, 714, 745, 792, 769, 785, 700, 817, 784, 792, 787, 740, 781, 779, 759, 786, 804, 783, 757, 767, 773, 770, 778, 777, 733, 709, 797, 805, 796, 807, 730, 732, 768, 790, 773, 722, 784, 793, 757, 743, 746, 772, 737, 743, 744, 757, 795, 774, 804, 824, 811, 770, 752, 760, 785, 758, 746, 795] payload_rows_per_dst=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] dup_rows_per_dst=[2782, 3000, 3066, 2962, 3032, 2972, 3060, 3006] combine_rows_per_src=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=65000 dup_rows=24078 dup_rows_remote=21026 dedup_ratio=0.244934 router_tokens_per_expert=[758, 776, 734, 756, 778, 741, 778, 827, 764, 767, 771, 745, 832, 691, 777, 750, 750, 815, 810, 765, 722, 730, 736, 732, 785, 788, 761, 775, 743, 793, 744, 790, 776, 796, 795, 729, 770, 810, 768, 728, 787, 786, 768, 763, 740, 734, 774, 795, 767, 761, 798, 779, 750, 749, 732, 803, 801, 820, 770, 801, 741, 713, 792, 772] payload_rows_per_dst=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] dup_rows_per_dst=[2958, 3012, 2920, 2980, 3052, 3030, 3052, 3074] combine_rows_per_src=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343568 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267017744 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533691920 weight_movement_bytes=34603008 total_nvlink_bytes=568294928 weight_to_token_ratio=0.064837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343992 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265945016 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531546040 weight_movement_bytes=34603008 total_nvlink_bytes=566149048 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=64984 dup_rows=23866 dup_rows_remote=20736 dedup_ratio=0.242778 router_tokens_per_expert=[776, 718, 763, 759, 744, 772, 814, 749, 781, 731, 808, 770, 768, 725, 790, 718, 856, 782, 748, 811, 738, 770, 805, 782, 789, 785, 724, 751, 755, 804, 772, 744, 752, 747, 808, 745, 763, 736, 816, 738, 760, 761, 756, 803, 766, 781, 724, 813, 798, 751, 779, 767, 705, 783, 797, 740, 772, 780, 740, 763, 754, 788, 800, 764] payload_rows_per_dst=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] dup_rows_per_dst=[2988, 2966, 3130, 2960, 2956, 2862, 2964, 3040] combine_rows_per_src=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343864 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469176 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594488 weight_movement_bytes=34603008 total_nvlink_bytes=567197496 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64912 dup_rows=24060 dup_rows_remote=21082 dedup_ratio=0.244751 router_tokens_per_expert=[760, 786, 755, 761, 776, 746, 783, 804, 824, 765, 788, 708, 800, 728, 842, 726, 709, 733, 776, 780, 801, 769, 775, 798, 779, 757, 737, 765, 782, 753, 776, 745, 770, 729, 784, 728, 739, 753, 762, 743, 716, 726, 798, 824, 777, 772, 781, 767, 821, 742, 800, 751, 777, 810, 781, 762, 768, 737, 784, 786, 776, 782, 755, 764] payload_rows_per_dst=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] dup_rows_per_dst=[3090, 2622, 3042, 3444, 2764, 3032, 3088, 2978] combine_rows_per_src=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=343200 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264977568 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529611936 weight_movement_bytes=69206016 total_nvlink_bytes=598817952 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344704 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267641472 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534938240 weight_movement_bytes=69206016 total_nvlink_bytes=604144256 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344104 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266584104 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532824104 weight_movement_bytes=69206016 total_nvlink_bytes=602030120 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=342880 combine_payload_bytes=266174464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266517344 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532691808 weight_movement_bytes=103809024 total_nvlink_bytes=636500832 weight_to_token_ratio=0.194876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343976 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223528 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103080 weight_movement_bytes=69206016 total_nvlink_bytes=601309096 weight_to_token_ratio=0.130061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65122 dup_rows=23860 dup_rows_remote=20898 dedup_ratio=0.242716 router_tokens_per_expert=[771, 793, 780, 753, 800, 769, 792, 710, 787, 729, 768, 795, 790, 782, 740, 751, 746, 765, 763, 784, 740, 706, 787, 820, 757, 747, 794, 746, 779, 721, 796, 791, 801, 781, 796, 727, 785, 781, 748, 739, 718, 767, 776, 758, 771, 775, 744, 784, 765, 772, 776, 738, 801, 793, 784, 752, 755, 774, 760, 734, 832, 761, 768, 784] payload_rows_per_dst=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] dup_rows_per_dst=[3010, 2962, 2986, 2964, 2970, 2930, 3034, 3004] combine_rows_per_src=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=64424 dup_rows=24570 dup_rows_remote=21476 dedup_ratio=0.249939 router_tokens_per_expert=[754, 748, 737, 754, 792, 743, 764, 799, 788, 762, 815, 801, 743, 752, 776, 744, 745, 767, 772, 749, 788, 813, 763, 789, 782, 773, 738, 748, 780, 794, 790, 768, 779, 805, 761, 732, 742, 744, 737, 732, 706, 746, 801, 768, 788, 720, 786, 793, 784, 736, 810, 758, 788, 768, 773, 833, 759, 749, 798, 761, 795, 731, 729, 809] payload_rows_per_dst=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] dup_rows_per_dst=[3094, 3208, 3136, 3140, 2812, 3370, 3122, 2688] combine_rows_per_src=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65158 dup_rows=23966 dup_rows_remote=20902 dedup_ratio=0.243795 router_tokens_per_expert=[759, 766, 783, 750, 754, 760, 745, 805, 773, 813, 775, 775, 762, 763, 785, 759, 819, 736, 718, 737, 775, 783, 765, 818, 720, 761, 770, 777, 747, 743, 763, 768, 814, 776, 752, 769, 752, 773, 737, 763, 797, 744, 758, 758, 741, 746, 764, 814, 801, 769, 752, 799, 783, 764, 771, 747, 775, 787, 793, 786, 734, 795, 787, 724] payload_rows_per_dst=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] dup_rows_per_dst=[2938, 3076, 3008, 2840, 2984, 3064, 3080, 2976] combine_rows_per_src=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64906 dup_rows=24152 dup_rows_remote=21116 dedup_ratio=0.245687 router_tokens_per_expert=[728, 808, 743, 785, 769, 785, 773, 742, 781, 751, 789, 774, 820, 758, 768, 734, 769, 736, 754, 763, 762, 757, 759, 760, 786, 773, 763, 793, 755, 751, 772, 759, 809, 782, 717, 746, 793, 703, 791, 800, 776, 799, 744, 776, 782, 734, 806, 778, 768, 807, 727, 773, 775, 779, 735, 767, 787, 798, 725, 734, 749, 745, 805, 822] payload_rows_per_dst=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] dup_rows_per_dst=[3018, 3050, 2920, 3014, 3036, 3064, 3044, 3006] combine_rows_per_src=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65362 dup_rows=23768 dup_rows_remote=20900 dedup_ratio=0.241781 router_tokens_per_expert=[764, 820, 771, 727, 778, 733, 786, 834, 695, 759, 792, 790, 779, 803, 759, 769, 770, 763, 789, 748, 766, 764, 772, 776, 752, 752, 773, 747, 770, 755, 756, 716, 786, 723, 781, 720, 772, 777, 816, 786, 770, 805, 779, 800, 819, 739, 747, 749, 748, 770, 792, 757, 801, 759, 737, 796, 771, 748, 814, 736, 793, 756, 744, 733] payload_rows_per_dst=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] dup_rows_per_dst=[3026, 3000, 2914, 2868, 2984, 3004, 3002, 2970] combine_rows_per_src=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65136 dup_rows=23902 dup_rows_remote=20932 dedup_ratio=0.243144 router_tokens_per_expert=[767, 768, 809, 753, 786, 721, 765, 776, 753, 806, 758, 806, 798, 764, 744, 766, 766, 745, 752, 730, 781, 704, 796, 800, 740, 731, 774, 817, 811, 791, 760, 747, 757, 739, 825, 731, 799, 772, 783, 763, 759, 808, 755, 782, 790, 783, 750, 786, 724, 753, 832, 778, 725, 732, 792, 782, 742, 792, 790, 700, 749, 726, 770, 798] payload_rows_per_dst=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] dup_rows_per_dst=[2972, 3026, 2958, 2998, 3002, 3090, 2970, 2886] combine_rows_per_src=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344080 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083792 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823504 weight_movement_bytes=69206016 total_nvlink_bytes=603029520 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=343600 combine_payload_bytes=263880704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264224304 ingress_bytes=298483712 link_bytes=298483712 token_bytes=528105008 weight_movement_bytes=34603008 total_nvlink_bytes=562708016 weight_to_token_ratio=0.065523 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123170178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65094 dup_rows=23880 dup_rows_remote=21010 dedup_ratio=0.242920 router_tokens_per_expert=[751, 817, 815, 752, 737, 747, 790, 738, 752, 764, 793, 798, 782, 824, 763, 791, 750, 785, 768, 793, 752, 757, 759, 736, 780, 727, 746, 774, 796, 783, 773, 742, 740, 791, 781, 775, 773, 800, 759, 769, 755, 777, 714, 773, 813, 732, 804, 765, 780, 776, 744, 777, 781, 781, 723, 762, 719, 743, 810, 746, 815, 734, 757, 748] payload_rows_per_dst=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] dup_rows_per_dst=[2998, 3094, 2870, 2936, 3034, 2958, 3054, 2936] combine_rows_per_src=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344240 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267231408 ingress_bytes=336093184 link_bytes=336093184 token_bytes=534118576 weight_movement_bytes=69206016 total_nvlink_bytes=603324592 weight_to_token_ratio=0.129571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264691594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64830 dup_rows=24092 dup_rows_remote=20944 dedup_ratio=0.245076 router_tokens_per_expert=[756, 750, 752, 757, 810, 722, 756, 767, 772, 793, 775, 741, 778, 742, 767, 763, 781, 744, 747, 799, 744, 744, 788, 781, 760, 714, 799, 790, 746, 796, 759, 795, 750, 771, 787, 759, 764, 757, 765, 762, 817, 750, 814, 780, 736, 788, 786, 724, 789, 794, 767, 782, 797, 752, 786, 747, 785, 782, 778, 762, 811, 714, 768, 740] payload_rows_per_dst=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] dup_rows_per_dst=[2924, 2816, 3016, 3088, 3020, 3054, 3026, 3148] combine_rows_per_src=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344088 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266199064 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532054040 weight_movement_bytes=34603008 total_nvlink_bytes=566657048 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=345048 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268067800 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535790552 weight_movement_bytes=34603008 total_nvlink_bytes=570393560 weight_to_token_ratio=0.064583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344272 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141328 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938384 weight_movement_bytes=69206016 total_nvlink_bytes=603144400 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344416 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969440 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594464 weight_movement_bytes=34603008 total_nvlink_bytes=568197472 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343096 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886776 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430456 weight_movement_bytes=34603008 total_nvlink_bytes=566033464 weight_to_token_ratio=0.065113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65088 dup_rows=23766 dup_rows_remote=20596 dedup_ratio=0.241760 router_tokens_per_expert=[785, 781, 807, 756, 755, 726, 749, 785, 805, 788, 737, 812, 805, 768, 829, 766, 770, 793, 747, 767, 789, 760, 795, 800, 749, 821, 749, 746, 768, 779, 772, 748, 762, 771, 784, 777, 746, 780, 724, 733, 758, 755, 759, 747, 745, 733, 797, 759, 787, 720, 756, 720, 785, 723, 776, 783, 749, 754, 799, 773, 805, 762, 763, 760] payload_rows_per_dst=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] dup_rows_per_dst=[2942, 3170, 3146, 2930, 2868, 2882, 2916, 2912] combine_rows_per_src=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64982 dup_rows=24038 dup_rows_remote=21112 dedup_ratio=0.244527 router_tokens_per_expert=[744, 771, 786, 701, 773, 771, 766, 777, 762, 773, 772, 820, 785, 780, 867, 743, 750, 771, 812, 822, 784, 771, 776, 749, 746, 755, 737, 753, 739, 732, 782, 714, 809, 769, 713, 777, 745, 717, 830, 799, 794, 731, 734, 742, 779, 779, 724, 787, 764, 769, 772, 774, 793, 768, 818, 706, 735, 753, 753, 833, 789, 795, 748, 769] payload_rows_per_dst=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] dup_rows_per_dst=[2926, 3356, 3056, 2856, 3176, 2836, 2836, 2996] combine_rows_per_src=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65050 dup_rows=23908 dup_rows_remote=20884 dedup_ratio=0.243205 router_tokens_per_expert=[794, 746, 750, 728, 770, 760, 750, 748, 769, 768, 803, 759, 830, 730, 779, 708, 792, 780, 766, 747, 739, 807, 778, 808, 805, 749, 769, 764, 796, 741, 767, 758, 802, 722, 786, 786, 784, 734, 762, 756, 781, 725, 834, 793, 752, 771, 754, 775, 741, 768, 799, 751, 843, 771, 774, 781, 740, 776, 742, 735, 799, 781, 727, 749] payload_rows_per_dst=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] dup_rows_per_dst=[2942, 2918, 3082, 2968, 3006, 3024, 3028, 2940] combine_rows_per_src=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64982 dup_rows=24008 dup_rows_remote=21136 dedup_ratio=0.244222 router_tokens_per_expert=[772, 768, 766, 765, 772, 729, 742, 766, 744, 738, 792, 759, 745, 767, 797, 778, 769, 733, 799, 771, 762, 776, 800, 769, 743, 742, 704, 756, 819, 734, 807, 805, 762, 723, 784, 750, 750, 786, 762, 776, 719, 756, 801, 745, 828, 791, 779, 772, 771, 740, 770, 769, 769, 809, 776, 797, 752, 768, 746, 775, 771, 829, 808, 729] payload_rows_per_dst=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] dup_rows_per_dst=[3008, 3006, 2948, 2952, 2872, 3132, 3058, 3032] combine_rows_per_src=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65034 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[752, 796, 729, 775, 778, 767, 797, 767, 765, 751, 805, 789, 738, 776, 750, 761, 794, 764, 754, 714, 753, 737, 701, 759, 799, 753, 786, 793, 786, 745, 714, 795, 793, 755, 772, 797, 774, 801, 740, 811, 787, 765, 752, 744, 795, 777, 743, 740, 761, 765, 739, 804, 741, 807, 808, 801, 723, 778, 788, 717, 813, 775, 775, 768] payload_rows_per_dst=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] dup_rows_per_dst=[3008, 3006, 2818, 2996, 3066, 2964, 3054, 3012] combine_rows_per_src=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64890 dup_rows=24114 dup_rows_remote=21090 dedup_ratio=0.245300 router_tokens_per_expert=[787, 749, 771, 748, 765, 734, 730, 777, 780, 746, 736, 785, 728, 794, 748, 752, 802, 810, 765, 789, 771, 771, 779, 773, 756, 758, 732, 732, 783, 781, 771, 799, 806, 778, 738, 734, 767, 760, 752, 775, 798, 755, 762, 771, 764, 794, 751, 752, 772, 781, 746, 769, 775, 791, 733, 795, 807, 805, 804, 820, 739, 746, 766, 744] payload_rows_per_dst=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] dup_rows_per_dst=[2980, 2996, 3052, 2970, 3006, 2954, 3024, 3132] combine_rows_per_src=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=342736 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266943184 ingress_bytes=405012480 link_bytes=405012480 token_bytes=533543632 weight_movement_bytes=138412032 total_nvlink_bytes=671955664 weight_to_token_ratio=0.259420 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524029356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344376 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510648 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676920 weight_movement_bytes=34603008 total_nvlink_bytes=567279928 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64940 dup_rows=23954 dup_rows_remote=20938 dedup_ratio=0.243673 router_tokens_per_expert=[768, 765, 776, 764, 754, 787, 743, 760, 801, 765, 786, 751, 744, 739, 756, 768, 779, 829, 767, 821, 768, 760, 760, 729, 752, 767, 736, 764, 843, 770, 743, 739, 778, 787, 789, 767, 772, 742, 832, 799, 718, 784, 758, 737, 776, 796, 732, 740, 782, 737, 736, 788, 794, 762, 737, 789, 787, 788, 815, 726, 767, 763, 796, 724] payload_rows_per_dst=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] dup_rows_per_dst=[2960, 2972, 3016, 2906, 3134, 2950, 3034, 2982] combine_rows_per_src=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64636 dup_rows=24148 dup_rows_remote=20926 dedup_ratio=0.245646 router_tokens_per_expert=[769, 765, 741, 814, 757, 740, 799, 750, 742, 808, 746, 796, 741, 806, 743, 763, 795, 759, 777, 773, 752, 759, 785, 775, 786, 737, 791, 784, 736, 795, 842, 777, 760, 736, 731, 776, 756, 739, 750, 740, 793, 822, 781, 744, 787, 783, 753, 796, 731, 746, 785, 762, 740, 755, 769, 794, 800, 739, 803, 722, 774, 756, 768, 758] payload_rows_per_dst=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] dup_rows_per_dst=[2956, 2808, 2832, 3138, 2700, 3306, 3186, 3222] combine_rows_per_src=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343736 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788536 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233336 weight_movement_bytes=69206016 total_nvlink_bytes=602439352 weight_to_token_ratio=0.129786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343848 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266723112 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533102376 weight_movement_bytes=34603008 total_nvlink_bytes=567705384 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344472 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510744 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532677016 weight_movement_bytes=34603008 total_nvlink_bytes=567280024 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343920 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266133360 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922800 weight_movement_bytes=103809024 total_nvlink_bytes=635731824 weight_to_token_ratio=0.195158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343512 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266337752 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532331992 weight_movement_bytes=69206016 total_nvlink_bytes=601538008 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=342248 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265091304 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529840360 weight_movement_bytes=34603008 total_nvlink_bytes=564443368 weight_to_token_ratio=0.065308 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65058 dup_rows=23924 dup_rows_remote=20882 dedup_ratio=0.243368 router_tokens_per_expert=[691, 778, 777, 726, 759, 748, 737, 761, 781, 806, 759, 833, 761, 773, 741, 728, 732, 747, 756, 813, 755, 744, 780, 795, 798, 779, 769, 799, 771, 780, 763, 814, 780, 732, 787, 799, 746, 729, 793, 774, 798, 777, 805, 801, 768, 772, 722, 784, 741, 732, 766, 749, 826, 770, 705, 774, 758, 761, 753, 800, 743, 802, 797, 754] payload_rows_per_dst=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] dup_rows_per_dst=[2750, 3042, 2974, 3110, 3016, 3114, 2916, 3002] combine_rows_per_src=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64668 dup_rows=24118 dup_rows_remote=20750 dedup_ratio=0.245341 router_tokens_per_expert=[732, 813, 814, 755, 792, 745, 783, 787, 792, 815, 753, 765, 787, 742, 795, 760, 786, 766, 748, 744, 736, 771, 739, 746, 752, 740, 782, 783, 754, 766, 782, 786, 812, 814, 739, 763, 744, 770, 765, 724, 789, 740, 768, 764, 745, 802, 769, 788, 799, 781, 787, 778, 765, 755, 757, 768, 750, 772, 699, 725, 777, 779, 767, 786] payload_rows_per_dst=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] dup_rows_per_dst=[3368, 3104, 2960, 2986, 3018, 3022, 2778, 2882] combine_rows_per_src=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64786 dup_rows=24270 dup_rows_remote=21362 dedup_ratio=0.246887 router_tokens_per_expert=[763, 748, 849, 764, 790, 754, 810, 766, 781, 737, 776, 781, 726, 814, 783, 760, 782, 723, 744, 779, 793, 766, 782, 734, 793, 724, 726, 798, 763, 766, 742, 773, 741, 779, 760, 764, 786, 798, 754, 784, 743, 740, 757, 737, 786, 779, 760, 776, 780, 811, 787, 809, 733, 767, 737, 764, 796, 745, 789, 734, 772, 765, 784, 745] payload_rows_per_dst=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] dup_rows_per_dst=[3112, 3090, 2982, 3020, 3084, 2908, 3054, 3020] combine_rows_per_src=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64964 dup_rows=24034 dup_rows_remote=21014 dedup_ratio=0.244486 router_tokens_per_expert=[765, 717, 793, 766, 784, 749, 755, 737, 785, 764, 786, 762, 779, 779, 751, 762, 787, 793, 783, 739, 789, 780, 813, 797, 780, 735, 769, 749, 763, 782, 753, 774, 781, 772, 757, 747, 740, 771, 770, 825, 750, 769, 764, 776, 730, 769, 754, 801, 825, 744, 739, 714, 777, 746, 757, 789, 736, 759, 780, 733, 827, 781, 777, 772] payload_rows_per_dst=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] dup_rows_per_dst=[2940, 2956, 3210, 2926, 3020, 2956, 2974, 3052] combine_rows_per_src=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64856 dup_rows=24118 dup_rows_remote=21000 dedup_ratio=0.245341 router_tokens_per_expert=[748, 759, 761, 770, 749, 785, 777, 817, 767, 738, 763, 797, 714, 792, 783, 798, 753, 756, 774, 774, 791, 730, 783, 738, 783, 763, 721, 796, 792, 758, 804, 807, 777, 701, 769, 806, 747, 780, 763, 811, 732, 789, 764, 743, 824, 756, 783, 759, 715, 811, 718, 721, 791, 803, 787, 785, 761, 778, 754, 741, 774, 759, 765, 744] payload_rows_per_dst=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] dup_rows_per_dst=[3010, 3056, 2922, 3118, 3072, 3012, 3038, 2890] combine_rows_per_src=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343760 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821328 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533298896 weight_movement_bytes=69206016 total_nvlink_bytes=602504912 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65172 dup_rows=23922 dup_rows_remote=21070 dedup_ratio=0.243347 router_tokens_per_expert=[798, 750, 738, 797, 805, 805, 804, 752, 761, 783, 790, 784, 765, 770, 806, 759, 803, 755, 749, 778, 781, 785, 794, 790, 778, 746, 763, 769, 778, 801, 757, 782, 810, 741, 792, 744, 795, 790, 762, 806, 711, 715, 729, 781, 738, 762, 781, 783, 738, 734, 813, 772, 729, 759, 728, 758, 754, 702, 730, 770, 775, 733, 805, 736] payload_rows_per_dst=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] dup_rows_per_dst=[3128, 3108, 3044, 3008, 3040, 2912, 2852, 2830] combine_rows_per_src=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65194 dup_rows=23834 dup_rows_remote=20912 dedup_ratio=0.242452 router_tokens_per_expert=[762, 774, 770, 794, 749, 787, 791, 768, 813, 748, 787, 783, 725, 800, 776, 747, 719, 769, 790, 768, 733, 774, 769, 777, 769, 782, 749, 755, 744, 792, 774, 727, 704, 737, 764, 742, 760, 803, 774, 773, 799, 770, 762, 743, 715, 759, 780, 731, 760, 740, 839, 765, 771, 810, 770, 756, 781, 743, 784, 799, 749, 812, 827, 765] payload_rows_per_dst=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] dup_rows_per_dst=[2998, 2932, 2922, 2934, 2924, 2984, 3036, 3104] combine_rows_per_src=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=341672 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265221800 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530101928 weight_movement_bytes=34603008 total_nvlink_bytes=564704936 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64818 dup_rows=24216 dup_rows_remote=21052 dedup_ratio=0.246338 router_tokens_per_expert=[746, 731, 778, 826, 775, 752, 771, 766, 771, 755, 778, 772, 716, 766, 807, 805, 780, 738, 740, 721, 788, 792, 715, 791, 738, 764, 793, 772, 769, 734, 742, 775, 809, 792, 785, 741, 768, 788, 731, 771, 792, 754, 794, 748, 732, 737, 749, 791, 795, 769, 786, 752, 788, 787, 767, 765, 760, 734, 777, 805, 773, 798, 785, 762] payload_rows_per_dst=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] dup_rows_per_dst=[2990, 2998, 2968, 2970, 3084, 2930, 3112, 3164] combine_rows_per_src=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344592 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708048 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071504 weight_movement_bytes=34603008 total_nvlink_bytes=565674512 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343424 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993600 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643776 weight_movement_bytes=103809024 total_nvlink_bytes=635452800 weight_to_token_ratio=0.195260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436456 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532529000 weight_movement_bytes=69206016 total_nvlink_bytes=601735016 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344968 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289480 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233992 weight_movement_bytes=34603008 total_nvlink_bytes=568837000 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344424 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379048 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413672 weight_movement_bytes=34603008 total_nvlink_bytes=569016680 weight_to_token_ratio=0.064749 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343480 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838008 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332536 weight_movement_bytes=69206016 total_nvlink_bytes=600538552 weight_to_token_ratio=0.130250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64952 dup_rows=24212 dup_rows_remote=21254 dedup_ratio=0.246297 router_tokens_per_expert=[738, 742, 698, 743, 758, 753, 798, 792, 762, 723, 794, 778, 789, 729, 712, 762, 779, 798, 789, 797, 753, 780, 771, 770, 777, 775, 791, 759, 818, 788, 730, 779, 776, 754, 733, 804, 764, 774, 783, 777, 806, 739, 785, 826, 781, 770, 749, 761, 753, 753, 702, 797, 792, 771, 757, 833, 725, 747, 774, 791, 754, 717, 765, 814] payload_rows_per_dst=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] dup_rows_per_dst=[2882, 2958, 3170, 3178, 2970, 3074, 3022, 2958] combine_rows_per_src=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64776 dup_rows=24178 dup_rows_remote=21014 dedup_ratio=0.245951 router_tokens_per_expert=[789, 749, 781, 813, 801, 777, 770, 777, 816, 780, 776, 739, 780, 760, 773, 773, 762, 717, 721, 760, 749, 818, 770, 774, 766, 805, 717, 734, 739, 782, 722, 768, 754, 797, 804, 750, 791, 772, 774, 790, 766, 767, 747, 778, 764, 730, 749, 788, 778, 758, 804, 777, 748, 786, 726, 825, 768, 754, 749, 768, 793, 748, 777, 714] payload_rows_per_dst=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] dup_rows_per_dst=[3164, 3096, 3044, 2808, 3082, 2912, 3072, 3000] combine_rows_per_src=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65086 dup_rows=24006 dup_rows_remote=21032 dedup_ratio=0.244202 router_tokens_per_expert=[775, 762, 782, 777, 807, 782, 788, 726, 782, 745, 797, 742, 791, 783, 747, 792, 762, 760, 754, 738, 780, 800, 751, 724, 766, 786, 751, 801, 764, 787, 735, 768, 779, 784, 742, 772, 788, 748, 767, 755, 781, 777, 730, 791, 747, 764, 771, 732, 819, 732, 721, 767, 753, 800, 768, 809, 784, 829, 724, 793, 794, 727, 745, 754] payload_rows_per_dst=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] dup_rows_per_dst=[3004, 3056, 3022, 3016, 2982, 2974, 3010, 2942] combine_rows_per_src=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64848 dup_rows=24268 dup_rows_remote=21246 dedup_ratio=0.246867 router_tokens_per_expert=[797, 779, 799, 784, 784, 774, 770, 764, 743, 751, 780, 794, 769, 749, 778, 752, 774, 789, 798, 792, 751, 766, 736, 775, 783, 798, 731, 749, 763, 770, 779, 732, 737, 778, 790, 762, 726, 781, 767, 800, 791, 752, 716, 785, 749, 803, 772, 785, 781, 749, 815, 729, 789, 768, 803, 715, 711, 791, 774, 725, 770, 739, 762, 784] payload_rows_per_dst=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] dup_rows_per_dst=[3136, 2996, 3112, 3022, 2946, 3090, 2994, 2972] combine_rows_per_src=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344824 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266388216 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431608 weight_movement_bytes=34603008 total_nvlink_bytes=567034616 weight_to_token_ratio=0.064991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64964 dup_rows=23954 dup_rows_remote=20746 dedup_ratio=0.243673 router_tokens_per_expert=[768, 733, 786, 741, 756, 773, 791, 768, 773, 712, 723, 774, 727, 765, 738, 776, 772, 762, 766, 742, 751, 793, 770, 789, 772, 802, 753, 787, 711, 809, 746, 749, 740, 838, 755, 767, 741, 768, 784, 791, 793, 732, 775, 792, 773, 771, 775, 742, 767, 786, 781, 782, 795, 809, 785, 792, 804, 720, 714, 777, 799, 778, 761, 787] payload_rows_per_dst=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] dup_rows_per_dst=[2960, 2840, 3022, 2952, 3012, 2968, 3208, 2992] combine_rows_per_src=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64970 dup_rows=24002 dup_rows_remote=21046 dedup_ratio=0.244161 router_tokens_per_expert=[755, 790, 751, 683, 759, 789, 787, 755, 769, 800, 769, 758, 749, 782, 790, 750, 774, 809, 736, 768, 761, 754, 758, 755, 778, 769, 800, 769, 804, 786, 806, 794, 802, 725, 766, 765, 763, 800, 737, 786, 805, 694, 793, 723, 729, 750, 750, 776, 735, 816, 786, 783, 755, 802, 708, 781, 791, 799, 766, 730, 774, 750, 784, 771] payload_rows_per_dst=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] dup_rows_per_dst=[2962, 3066, 2924, 3198, 2956, 2860, 3068, 2968] combine_rows_per_src=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65374 dup_rows=23848 dup_rows_remote=20880 dedup_ratio=0.242594 router_tokens_per_expert=[762, 783, 770, 804, 790, 807, 738, 788, 765, 804, 773, 744, 772, 817, 779, 814, 762, 762, 753, 756, 748, 743, 780, 721, 772, 731, 739, 780, 741, 768, 712, 764, 778, 824, 749, 791, 773, 765, 800, 735, 712, 786, 765, 763, 736, 757, 769, 767, 768, 798, 749, 765, 765, 779, 781, 780, 750, 770, 732, 740, 773, 806, 815, 769] payload_rows_per_dst=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] dup_rows_per_dst=[3096, 3126, 2968, 2884, 2914, 2892, 2988, 2980] combine_rows_per_src=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343160 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265665656 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988152 weight_movement_bytes=69206016 total_nvlink_bytes=600194168 weight_to_token_ratio=0.130334 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64882 dup_rows=24004 dup_rows_remote=20978 dedup_ratio=0.244181 router_tokens_per_expert=[757, 775, 773, 719, 745, 801, 766, 771, 768, 780, 696, 784, 792, 785, 756, 802, 736, 789, 741, 796, 769, 772, 792, 776, 771, 757, 760, 762, 778, 716, 767, 799, 764, 715, 732, 750, 794, 742, 804, 771, 768, 741, 802, 765, 770, 775, 788, 821, 735, 776, 762, 803, 768, 760, 781, 764, 781, 767, 775, 764, 756, 803, 772, 732] payload_rows_per_dst=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] dup_rows_per_dst=[2968, 2632, 3002, 2882, 3252, 3240, 3002, 3026] combine_rows_per_src=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344472 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936728 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528984 weight_movement_bytes=34603008 total_nvlink_bytes=568131992 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=344376 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265961784 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531579192 weight_movement_bytes=34603008 total_nvlink_bytes=566182200 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342840 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266435384 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532527928 weight_movement_bytes=103809024 total_nvlink_bytes=636336952 weight_to_token_ratio=0.194936 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266461184 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532578304 weight_movement_bytes=138412032 total_nvlink_bytes=670990336 weight_to_token_ratio=0.259890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267771904 dispatch_meta_bytes=345016 combine_payload_bytes=267771904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268116920 ingress_bytes=302374912 link_bytes=302374912 token_bytes=535888824 weight_movement_bytes=34603008 total_nvlink_bytes=570491832 weight_to_token_ratio=0.064571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137812451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100112 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531856784 weight_movement_bytes=34603008 total_nvlink_bytes=566459792 weight_to_token_ratio=0.065061 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64978 dup_rows=24002 dup_rows_remote=20890 dedup_ratio=0.244161 router_tokens_per_expert=[817, 743, 822, 798, 780, 782, 769, 759, 780, 794, 795, 732, 785, 788, 776, 768, 753, 798, 737, 779, 706, 808, 773, 761, 767, 762, 807, 790, 710, 774, 729, 779, 733, 758, 752, 770, 778, 724, 759, 753, 742, 769, 749, 767, 741, 774, 752, 745, 807, 745, 771, 785, 765, 790, 810, 722, 824, 727, 796, 752, 725, 803, 785, 758] payload_rows_per_dst=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] dup_rows_per_dst=[3028, 3112, 2894, 3042, 2914, 2896, 3062, 3054] combine_rows_per_src=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65064 dup_rows=23898 dup_rows_remote=20930 dedup_ratio=0.243103 router_tokens_per_expert=[776, 751, 752, 782, 741, 771, 774, 782, 769, 781, 815, 721, 783, 742, 795, 770, 732, 792, 708, 816, 742, 740, 728, 809, 773, 731, 776, 790, 727, 744, 765, 731, 779, 780, 765, 761, 752, 785, 844, 735, 754, 759, 759, 795, 753, 735, 785, 792, 803, 779, 770, 745, 764, 795, 774, 729, 762, 794, 757, 814, 781, 786, 785, 772] payload_rows_per_dst=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] dup_rows_per_dst=[2968, 3088, 3118, 2920, 2964, 3220, 2736, 2884] combine_rows_per_src=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343472 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266493360 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532643248 weight_movement_bytes=138412032 total_nvlink_bytes=671055280 weight_to_token_ratio=0.259859 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64806 dup_rows=24194 dup_rows_remote=21074 dedup_ratio=0.246114 router_tokens_per_expert=[776, 787, 812, 726, 733, 759, 739, 793, 743, 769, 817, 762, 773, 784, 752, 799, 734, 810, 765, 768, 781, 753, 768, 791, 778, 784, 776, 772, 826, 757, 754, 770, 783, 693, 769, 774, 759, 781, 753, 756, 762, 710, 793, 757, 791, 810, 805, 784, 748, 779, 775, 714, 756, 747, 756, 786, 738, 814, 752, 741, 787, 775, 727, 766] payload_rows_per_dst=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] dup_rows_per_dst=[3046, 3070, 3014, 3114, 2976, 3120, 2944, 2910] combine_rows_per_src=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65216 dup_rows=23672 dup_rows_remote=20636 dedup_ratio=0.240804 router_tokens_per_expert=[735, 786, 769, 776, 778, 739, 759, 768, 779, 773, 745, 754, 739, 749, 725, 727, 757, 829, 779, 765, 805, 730, 729, 833, 787, 793, 787, 808, 760, 741, 756, 794, 762, 768, 800, 766, 801, 718, 772, 790, 755, 816, 778, 784, 742, 700, 771, 773, 760, 800, 786, 772, 804, 751, 778, 753, 775, 745, 750, 759, 783, 756, 764, 766] payload_rows_per_dst=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] dup_rows_per_dst=[2930, 2780, 3086, 3036, 2938, 2962, 2992, 2948] combine_rows_per_src=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64800 dup_rows=24054 dup_rows_remote=20910 dedup_ratio=0.244690 router_tokens_per_expert=[799, 754, 716, 788, 791, 785, 745, 786, 817, 786, 760, 827, 808, 754, 800, 809, 716, 752, 707, 754, 753, 771, 726, 737, 768, 743, 770, 717, 797, 715, 790, 769, 796, 761, 774, 730, 777, 793, 758, 769, 770, 734, 702, 740, 796, 735, 738, 823, 809, 749, 786, 831, 755, 801, 792, 774, 795, 790, 738, 760, 778, 742, 790, 756] payload_rows_per_dst=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] dup_rows_per_dst=[3054, 3196, 2810, 2958, 3002, 2896, 3144, 2994] combine_rows_per_src=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65032 dup_rows=24084 dup_rows_remote=21110 dedup_ratio=0.244995 router_tokens_per_expert=[760, 746, 787, 750, 748, 719, 719, 749, 749, 792, 739, 750, 733, 778, 791, 778, 791, 827, 738, 765, 784, 751, 760, 748, 806, 807, 729, 771, 751, 793, 764, 770, 725, 781, 765, 750, 758, 768, 732, 802, 734, 756, 780, 780, 795, 795, 801, 762, 805, 778, 768, 762, 788, 760, 764, 767, 769, 784, 808, 732, 782, 803, 765, 790] payload_rows_per_dst=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] dup_rows_per_dst=[2798, 3044, 3022, 3082, 2974, 3032, 3054, 3078] combine_rows_per_src=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=64756 dup_rows=24328 dup_rows_remote=21280 dedup_ratio=0.247477 router_tokens_per_expert=[797, 841, 776, 780, 777, 799, 756, 808, 749, 734, 766, 786, 829, 725, 774, 786, 761, 728, 742, 747, 771, 792, 780, 813, 759, 789, 753, 788, 745, 755, 781, 798, 733, 770, 768, 788, 747, 772, 809, 741, 802, 807, 765, 733, 758, 810, 691, 756, 727, 734, 757, 761, 749, 786, 794, 764, 714, 778, 773, 772, 708, 744, 780, 776] payload_rows_per_dst=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] dup_rows_per_dst=[3224, 3008, 3048, 3026, 2970, 3050, 3026, 2976] combine_rows_per_src=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343976 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846120 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348264 weight_movement_bytes=34603008 total_nvlink_bytes=567951272 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64764 dup_rows=24242 dup_rows_remote=21128 dedup_ratio=0.246602 router_tokens_per_expert=[794, 758, 836, 757, 771, 742, 838, 771, 762, 745, 772, 782, 755, 814, 735, 779, 804, 790, 790, 770, 779, 744, 752, 760, 781, 785, 785, 784, 790, 729, 784, 733, 750, 739, 803, 775, 766, 734, 787, 742, 772, 718, 732, 806, 738, 787, 756, 773, 761, 764, 774, 734, 767, 742, 830, 741, 751, 754, 772, 752, 810, 726, 791, 734] payload_rows_per_dst=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] dup_rows_per_dst=[2856, 3280, 3028, 2974, 3024, 2930, 3036, 3114] combine_rows_per_src=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343520 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265788896 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531234272 weight_movement_bytes=34603008 total_nvlink_bytes=565837280 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=343408 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267468144 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534592880 weight_movement_bytes=34603008 total_nvlink_bytes=569195888 weight_to_token_ratio=0.064728 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=342840 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265763640 ingress_bytes=403832832 link_bytes=403832832 token_bytes=531184440 weight_movement_bytes=138412032 total_nvlink_bytes=669596472 weight_to_token_ratio=0.260572 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519590436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65164 dup_rows=23826 dup_rows_remote=20798 dedup_ratio=0.242371 router_tokens_per_expert=[763, 771, 745, 745, 762, 807, 752, 758, 748, 782, 790, 790, 827, 735, 723, 776, 781, 755, 748, 793, 755, 728, 761, 794, 756, 771, 785, 802, 793, 745, 763, 757, 784, 807, 747, 746, 698, 773, 750, 771, 738, 789, 792, 779, 772, 789, 774, 803, 769, 764, 743, 801, 762, 779, 755, 798, 773, 776, 784, 765, 763, 763, 741, 743] payload_rows_per_dst=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] dup_rows_per_dst=[2900, 3028, 3028, 2938, 2970, 3050, 2974, 2938] combine_rows_per_src=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344568 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715640 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086712 weight_movement_bytes=34603008 total_nvlink_bytes=567689720 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=344144 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584720 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530825296 weight_movement_bytes=34603008 total_nvlink_bytes=565428304 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=343568 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265616912 ingress_bytes=299876352 link_bytes=299876352 token_bytes=530890256 weight_movement_bytes=34603008 total_nvlink_bytes=565493264 weight_to_token_ratio=0.065179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128410571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343848 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255592 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167336 weight_movement_bytes=34603008 total_nvlink_bytes=568770344 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20974 dedup_ratio=0.243876 router_tokens_per_expert=[761, 773, 736, 787, 754, 787, 779, 781, 768, 784, 760, 829, 762, 772, 804, 776, 785, 804, 807, 746, 787, 739, 763, 737, 734, 787, 776, 779, 786, 767, 757, 740, 774, 722, 746, 786, 744, 771, 743, 815, 781, 758, 792, 795, 733, 759, 712, 808, 759, 832, 790, 722, 768, 752, 755, 803, 757, 739, 739, 748, 754, 775, 747, 766] payload_rows_per_dst=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] dup_rows_per_dst=[3000, 3090, 3258, 3048, 2984, 2958, 2724, 2912] combine_rows_per_src=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64936 dup_rows=23970 dup_rows_remote=20914 dedup_ratio=0.243835 router_tokens_per_expert=[750, 772, 746, 759, 842, 746, 763, 789, 785, 769, 775, 775, 725, 770, 804, 745, 772, 763, 762, 778, 720, 792, 761, 729, 744, 771, 731, 775, 776, 754, 744, 770, 780, 764, 743, 791, 730, 762, 809, 724, 774, 754, 811, 773, 793, 777, 774, 771, 784, 773, 774, 755, 772, 775, 753, 803, 783, 769, 783, 743, 746, 758, 786, 808] payload_rows_per_dst=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] dup_rows_per_dst=[3112, 2950, 2954, 2974, 2960, 3056, 2964, 3000] combine_rows_per_src=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65090 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[775, 766, 759, 762, 791, 751, 760, 750, 766, 739, 773, 784, 744, 734, 745, 777, 778, 766, 753, 758, 714, 743, 781, 797, 776, 737, 778, 773, 758, 795, 759, 773, 748, 774, 786, 789, 771, 795, 811, 848, 758, 800, 789, 742, 789, 769, 714, 763, 830, 754, 761, 781, 720, 710, 781, 771, 755, 788, 739, 757, 775, 816, 781, 772] payload_rows_per_dst=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] dup_rows_per_dst=[2858, 2882, 3026, 2992, 3158, 2970, 3018, 3004] combine_rows_per_src=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64892 dup_rows=23986 dup_rows_remote=20940 dedup_ratio=0.243998 router_tokens_per_expert=[726, 827, 760, 780, 766, 737, 812, 763, 774, 705, 794, 733, 815, 769, 739, 782, 750, 784, 730, 777, 753, 789, 764, 798, 772, 785, 742, 799, 749, 784, 743, 801, 759, 774, 772, 811, 805, 779, 739, 767, 739, 739, 742, 732, 754, 750, 791, 743, 794, 812, 766, 756, 796, 754, 792, 766, 754, 812, 767, 757, 767, 741, 752, 768] payload_rows_per_dst=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] dup_rows_per_dst=[2968, 2964, 3034, 3024, 3132, 2856, 3046, 2962] combine_rows_per_src=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343952 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266641296 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938640 weight_movement_bytes=34603008 total_nvlink_bytes=567541648 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65170 dup_rows=23778 dup_rows_remote=20776 dedup_ratio=0.241882 router_tokens_per_expert=[801, 800, 806, 750, 763, 761, 800, 740, 771, 761, 728, 754, 753, 813, 753, 766, 755, 756, 788, 799, 781, 770, 748, 761, 796, 788, 751, 741, 809, 798, 742, 748, 751, 816, 775, 788, 722, 779, 763, 785, 767, 710, 789, 718, 816, 765, 719, 785, 810, 790, 742, 773, 789, 791, 745, 746, 784, 734, 726, 808, 748, 817, 689, 761] payload_rows_per_dst=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] dup_rows_per_dst=[3146, 2960, 3012, 2884, 3002, 2894, 2986, 2894] combine_rows_per_src=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64940 dup_rows=24236 dup_rows_remote=21282 dedup_ratio=0.246541 router_tokens_per_expert=[740, 762, 771, 790, 798, 807, 786, 717, 707, 767, 740, 810, 761, 750, 797, 774, 713, 850, 740, 745, 719, 744, 752, 778, 777, 782, 784, 753, 747, 724, 741, 726, 776, 798, 754, 754, 836, 738, 771, 789, 768, 760, 814, 834, 747, 765, 785, 757, 796, 718, 773, 773, 806, 801, 790, 754, 778, 716, 746, 822, 784, 740, 757, 800] payload_rows_per_dst=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] dup_rows_per_dst=[3028, 3020, 2954, 2952, 3062, 3074, 3022, 3124] combine_rows_per_src=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343400 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321256 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299112 weight_movement_bytes=34603008 total_nvlink_bytes=566902120 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64956 dup_rows=23982 dup_rows_remote=20956 dedup_ratio=0.243958 router_tokens_per_expert=[828, 777, 754, 760, 757, 765, 775, 702, 807, 791, 773, 766, 757, 761, 777, 749, 782, 799, 752, 783, 755, 755, 774, 761, 757, 757, 755, 774, 744, 796, 770, 811, 799, 750, 789, 791, 781, 818, 741, 810, 727, 781, 739, 792, 757, 744, 708, 746, 736, 784, 761, 775, 772, 740, 784, 793, 783, 828, 738, 736, 766, 764, 786, 709] payload_rows_per_dst=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] dup_rows_per_dst=[3012, 3034, 3028, 2982, 3116, 2808, 2976, 3026] combine_rows_per_src=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64924 dup_rows=24172 dup_rows_remote=21192 dedup_ratio=0.245890 router_tokens_per_expert=[775, 791, 755, 766, 782, 783, 741, 804, 797, 744, 735, 750, 767, 741, 786, 774, 756, 711, 792, 791, 781, 768, 729, 790, 775, 790, 751, 744, 766, 734, 775, 794, 788, 750, 779, 749, 763, 830, 794, 714, 758, 790, 747, 728, 775, 779, 792, 768, 781, 711, 768, 782, 798, 768, 815, 785, 753, 757, 774, 776, 770, 765, 781, 726] payload_rows_per_dst=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] dup_rows_per_dst=[3030, 2980, 2934, 3070, 3030, 2958, 3126, 3044] combine_rows_per_src=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344024 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266952664 ingress_bytes=335814656 link_bytes=335814656 token_bytes=533561304 weight_movement_bytes=69206016 total_nvlink_bytes=602767320 weight_to_token_ratio=0.129706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263643515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343328 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266140960 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531938592 weight_movement_bytes=69206016 total_nvlink_bytes=601144608 weight_to_token_ratio=0.130102 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343784 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267280104 ingress_bytes=370745344 link_bytes=370745344 token_bytes=534216424 weight_movement_bytes=103809024 total_nvlink_bytes=638025448 weight_to_token_ratio=0.194320 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395084882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344888 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266339128 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532333368 weight_movement_bytes=69206016 total_nvlink_bytes=601539384 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343648 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403424 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463200 weight_movement_bytes=34603008 total_nvlink_bytes=567066208 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344464 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266273168 ingress_bytes=300531712 link_bytes=300531712 token_bytes=532201872 weight_movement_bytes=34603008 total_nvlink_bytes=566804880 weight_to_token_ratio=0.065019 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130876637e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64852 dup_rows=24298 dup_rows_remote=21220 dedup_ratio=0.247172 router_tokens_per_expert=[781, 791, 746, 753, 752, 761, 784, 741, 786, 732, 734, 770, 783, 706, 762, 770, 754, 779, 785, 750, 794, 770, 758, 754, 775, 748, 818, 745, 752, 786, 733, 745, 762, 735, 775, 777, 769, 783, 759, 787, 762, 770, 834, 693, 766, 790, 805, 809, 779, 771, 691, 806, 791, 761, 778, 752, 772, 793, 765, 788, 782, 761, 795, 793] payload_rows_per_dst=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] dup_rows_per_dst=[3078, 2882, 3086, 3014, 3002, 3250, 2932, 3054] combine_rows_per_src=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266799912 prefetch_bytes=657457152 reduce_bytes=553648128 weight_movement_bytes=1211105280 egress_bytes=6137357096 ingress_bytes=7340548096 link_bytes=7340548096 total_nvlink_bytes=13477905192 weight_to_token_ratio=0.098730 prefetch_share_of_ingress=0.089565 dedup_ratio=0.243767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=16 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200951805e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12234197168 prefetch_bytes=761266176 reduce_bytes=622854144 weight_movement_bytes=1384120320 egress_bytes=6121048240 ingress_bytes=7497269248 link_bytes=7497269248 total_nvlink_bytes=13618317488 weight_to_token_ratio=0.113135 prefetch_share_of_ingress=0.101539 dedup_ratio=0.244638 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=18 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226592200e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65044 dup_rows=23910 dup_rows_remote=20944 dedup_ratio=0.243225 router_tokens_per_expert=[783, 748, 738, 786, 720, 748, 762, 793, 772, 704, 767, 754, 719, 723, 763, 742, 790, 800, 766, 805, 809, 744, 731, 800, 748, 758, 757, 807, 753, 771, 769, 734, 750, 747, 786, 826, 817, 785, 761, 772, 733, 734, 795, 776, 737, 823, 813, 747, 741, 761, 757, 736, 793, 797, 749, 753, 819, 795, 771, 766, 768, 762, 796, 822] payload_rows_per_dst=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] dup_rows_per_dst=[2910, 2814, 3036, 3000, 3074, 2966, 2960, 3150] combine_rows_per_src=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258865512 prefetch_bytes=726663168 reduce_bytes=380633088 weight_movement_bytes=1107296256 egress_bytes=6133387624 ingress_bytes=7232774144 link_bytes=7233118528 total_nvlink_bytes=13366161768 weight_to_token_ratio=0.090326 prefetch_share_of_ingress=0.100468 dedup_ratio=0.244073 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=11 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.183375770e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64976 dup_rows=24112 dup_rows_remote=21134 dedup_ratio=0.245280 router_tokens_per_expert=[755, 791, 805, 848, 804, 762, 811, 759, 776, 744, 771, 741, 757, 768, 747, 827, 812, 769, 784, 766, 698, 763, 744, 752, 750, 782, 774, 785, 740, 779, 816, 779, 770, 746, 717, 759, 760, 783, 777, 758, 796, 754, 759, 775, 806, 759, 720, 784, 739, 780, 757, 764, 787, 754, 761, 755, 775, 723, 800, 792, 779, 733, 721, 750] payload_rows_per_dst=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] dup_rows_per_dst=[3196, 3014, 2916, 3000, 2992, 3062, 2978, 2954] combine_rows_per_src=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64862 dup_rows=24062 dup_rows_remote=21132 dedup_ratio=0.244771 router_tokens_per_expert=[756, 805, 736, 788, 798, 745, 758, 745, 742, 754, 781, 742, 781, 781, 810, 759, 787, 782, 762, 733, 753, 791, 780, 827, 753, 756, 741, 836, 756, 826, 744, 743, 747, 778, 777, 765, 790, 758, 757, 773, 816, 736, 743, 745, 817, 745, 805, 726, 771, 755, 754, 740, 760, 796, 768, 766, 788, 757, 760, 809, 779, 729, 749, 742] payload_rows_per_dst=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] dup_rows_per_dst=[2994, 3074, 3104, 2930, 3016, 2980, 3008, 2956] combine_rows_per_src=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344288 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265978080 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611872 weight_movement_bytes=103809024 total_nvlink_bytes=635420896 weight_to_token_ratio=0.195272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65078 dup_rows=24018 dup_rows_remote=20834 dedup_ratio=0.244324 router_tokens_per_expert=[759, 752, 716, 789, 766, 763, 794, 742, 767, 739, 772, 818, 760, 721, 801, 715, 787, 774, 808, 778, 731, 765, 780, 747, 736, 752, 809, 768, 740, 779, 731, 783, 797, 702, 824, 743, 762, 775, 784, 809, 766, 782, 770, 788, 786, 777, 750, 808, 748, 763, 804, 767, 769, 794, 777, 738, 755, 788, 779, 784, 757, 742, 778, 744] payload_rows_per_dst=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] dup_rows_per_dst=[2912, 2850, 3004, 2948, 3184, 3136, 3016, 2968] combine_rows_per_src=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343952 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764176 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184400 weight_movement_bytes=34603008 total_nvlink_bytes=567787408 weight_to_token_ratio=0.064899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344440 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266486136 ingress_bytes=369950720 link_bytes=369950720 token_bytes=532627832 weight_movement_bytes=103809024 total_nvlink_bytes=636436856 weight_to_token_ratio=0.194900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392094776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343976 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018728 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693480 weight_movement_bytes=34603008 total_nvlink_bytes=566296488 weight_to_token_ratio=0.065081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343648 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903136 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462624 weight_movement_bytes=103809024 total_nvlink_bytes=637271648 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262343336 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135128744 ingress_bytes=7511334912 link_bytes=7511334912 total_nvlink_bytes=13646463656 weight_to_token_ratio=0.112876 prefetch_share_of_ingress=0.082922 dedup_ratio=0.243905 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228893416e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249891808 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6128903136 ingress_bytes=7608918016 link_bytes=7608918016 total_nvlink_bytes=13737821152 weight_to_token_ratio=0.121465 prefetch_share_of_ingress=0.090954 dedup_ratio=0.244708 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244858519e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255753552 prefetch_bytes=657457152 reduce_bytes=968884224 weight_movement_bytes=1626341376 egress_bytes=6131832144 ingress_bytes=7750262784 link_bytes=7750262784 total_nvlink_bytes=13882094928 weight_to_token_ratio=0.132700 prefetch_share_of_ingress=0.084830 dedup_ratio=0.244093 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=28 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267983257e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249642576 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6128776784 ingress_bytes=7678001152 link_bytes=7678001152 total_nvlink_bytes=13806777936 weight_to_token_ratio=0.127117 prefetch_share_of_ingress=0.090135 dedup_ratio=0.244561 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256160878e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64910 dup_rows=24194 dup_rows_remote=21246 dedup_ratio=0.246114 router_tokens_per_expert=[764, 754, 750, 786, 718, 769, 773, 742, 827, 754, 765, 732, 767, 765, 789, 799, 773, 770, 830, 750, 729, 725, 771, 774, 813, 754, 799, 775, 748, 786, 789, 753, 773, 746, 718, 794, 742, 773, 780, 769, 765, 792, 776, 742, 778, 752, 716, 804, 799, 791, 784, 779, 767, 786, 746, 788, 798, 763, 759, 753, 772, 765, 754, 735] payload_rows_per_dst=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] dup_rows_per_dst=[2948, 3296, 3006, 3122, 2800, 2926, 3100, 2996] combine_rows_per_src=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344624 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266215984 ingress_bytes=335077376 link_bytes=335077376 token_bytes=532087344 weight_movement_bytes=69206016 total_nvlink_bytes=601293360 weight_to_token_ratio=0.130065 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260869190e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12237856392 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=6122880648 ingress_bytes=7360684032 link_bytes=7360684032 total_nvlink_bytes=13483564680 weight_to_token_ratio=0.101791 prefetch_share_of_ingress=0.094021 dedup_ratio=0.244930 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.204246149e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-29 23:41:57] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 673.6 | throughput per GPU (TFLOP/s/GPU): 189.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.069310E+01 | loss scale: 1.0 | grad norm: 123.325 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64702 dup_rows=24066 dup_rows_remote=20808 dedup_ratio=0.244812 router_tokens_per_expert=[751, 793, 791, 778, 782, 749, 780, 794, 726, 777, 808, 767, 774, 799, 752, 706, 750, 816, 747, 711, 746, 764, 747, 774, 758, 738, 753, 841, 724, 765, 774, 752, 769, 744, 768, 774, 753, 774, 748, 784, 791, 778, 806, 765, 758, 762, 733, 756, 767, 771, 770, 773, 802, 814, 753, 742, 743, 778, 742, 792, 789, 813, 759, 794] payload_rows_per_dst=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] dup_rows_per_dst=[3062, 2968, 2940, 2946, 2966, 3014, 2912, 3258] combine_rows_per_src=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64876 dup_rows=24150 dup_rows_remote=20984 dedup_ratio=0.245667 router_tokens_per_expert=[747, 774, 758, 802, 774, 825, 727, 747, 706, 796, 791, 807, 742, 743, 761, 804, 765, 766, 797, 775, 754, 724, 799, 743, 774, 776, 706, 760, 800, 763, 740, 749, 774, 754, 738, 759, 790, 849, 770, 788, 734, 770, 773, 719, 790, 766, 735, 695, 729, 756, 779, 809, 774, 804, 785, 826, 766, 784, 762, 788, 759, 763, 780, 789] payload_rows_per_dst=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] dup_rows_per_dst=[3064, 2982, 3048, 2898, 3166, 2742, 3144, 3106] combine_rows_per_src=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65026 dup_rows=23914 dup_rows_remote=20866 dedup_ratio=0.243266 router_tokens_per_expert=[835, 772, 793, 789, 752, 752, 731, 781, 737, 809, 781, 775, 727, 792, 749, 786, 776, 789, 747, 788, 782, 726, 728, 766, 764, 757, 764, 769, 774, 783, 802, 793, 790, 735, 735, 793, 778, 755, 789, 725, 739, 784, 730, 762, 746, 775, 755, 723, 811, 794, 789, 730, 773, 771, 803, 766, 765, 802, 771, 730, 751, 759, 816, 738] payload_rows_per_dst=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] dup_rows_per_dst=[2986, 2966, 3002, 3048, 2922, 2952, 3090, 2948] combine_rows_per_src=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21092 dedup_ratio=0.245117 router_tokens_per_expert=[783, 767, 739, 744, 740, 750, 797, 761, 778, 757, 759, 788, 762, 730, 766, 765, 811, 755, 736, 761, 788, 770, 806, 766, 765, 794, 745, 800, 751, 756, 813, 805, 745, 771, 778, 743, 739, 792, 761, 829, 759, 798, 810, 738, 745, 757, 744, 788, 750, 779, 769, 772, 792, 766, 748, 770, 778, 743, 744, 778, 771, 753, 737, 797] payload_rows_per_dst=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] dup_rows_per_dst=[2974, 2982, 3004, 3086, 2998, 3006, 2998, 3048] combine_rows_per_src=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64832 dup_rows=24148 dup_rows_remote=21102 dedup_ratio=0.245646 router_tokens_per_expert=[773, 795, 777, 772, 737, 759, 738, 780, 736, 777, 829, 744, 744, 775, 777, 803, 758, 758, 784, 753, 804, 781, 768, 817, 776, 711, 756, 755, 786, 752, 782, 760, 784, 773, 747, 776, 807, 764, 747, 788, 756, 740, 754, 757, 764, 791, 737, 800, 763, 766, 720, 772, 762, 763, 726, 784, 769, 766, 809, 752, 783, 803, 747, 765] payload_rows_per_dst=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] dup_rows_per_dst=[2938, 3046, 3048, 3070, 2986, 2978, 3014, 3068] combine_rows_per_src=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65084 dup_rows=24046 dup_rows_remote=21106 dedup_ratio=0.244609 router_tokens_per_expert=[771, 751, 771, 744, 781, 737, 727, 802, 737, 734, 737, 784, 788, 795, 751, 770, 738, 809, 740, 738, 769, 793, 763, 801, 785, 701, 784, 756, 777, 814, 787, 765, 796, 755, 734, 787, 749, 774, 772, 766, 754, 701, 750, 757, 747, 772, 780, 796, 768, 793, 822, 768, 732, 797, 838, 791, 800, 777, 815, 763, 758, 761, 775, 704] payload_rows_per_dst=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] dup_rows_per_dst=[2966, 2932, 3058, 3004, 3026, 2940, 3106, 3014] combine_rows_per_src=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=342040 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265361432 ingress_bytes=334225408 link_bytes=334225408 token_bytes=530380824 weight_movement_bytes=69206016 total_nvlink_bytes=599586840 weight_to_token_ratio=0.130484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343440 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266075536 ingress_bytes=369541120 link_bytes=369541120 token_bytes=531807632 weight_movement_bytes=103809024 total_nvlink_bytes=635616656 weight_to_token_ratio=0.195200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390553484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343568 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690064 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036560 weight_movement_bytes=34603008 total_nvlink_bytes=567639568 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343672 combine_payload_bytes=265527296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265870968 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398264 weight_movement_bytes=69206016 total_nvlink_bytes=600604280 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343736 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895608 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447480 weight_movement_bytes=34603008 total_nvlink_bytes=566050488 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344760 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266928824 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512888 weight_movement_bytes=34603008 total_nvlink_bytes=568115896 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65118 dup_rows=23894 dup_rows_remote=20838 dedup_ratio=0.243062 router_tokens_per_expert=[779, 781, 768, 780, 706, 745, 743, 790, 784, 732, 766, 741, 771, 769, 801, 792, 830, 770, 801, 762, 804, 728, 768, 784, 771, 739, 732, 751, 816, 745, 778, 764, 813, 770, 747, 777, 751, 767, 771, 730, 749, 742, 817, 751, 767, 790, 791, 749, 748, 746, 767, 791, 727, 790, 774, 743, 763, 742, 793, 779, 791, 797, 762, 766] payload_rows_per_dst=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] dup_rows_per_dst=[3056, 3064, 2942, 3004, 2812, 3470, 2848, 2698] combine_rows_per_src=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65002 dup_rows=24122 dup_rows_remote=21154 dedup_ratio=0.245382 router_tokens_per_expert=[781, 759, 725, 816, 751, 712, 798, 763, 774, 797, 761, 723, 788, 817, 768, 726, 782, 790, 791, 766, 746, 756, 765, 749, 794, 769, 756, 761, 765, 735, 758, 770, 828, 800, 790, 768, 742, 782, 770, 739, 723, 806, 811, 735, 776, 776, 729, 801, 744, 782, 784, 765, 739, 728, 774, 758, 768, 773, 819, 761, 770, 774, 784, 741] payload_rows_per_dst=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] dup_rows_per_dst=[3018, 2992, 3018, 2982, 3112, 2992, 2968, 3040] combine_rows_per_src=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64886 dup_rows=24080 dup_rows_remote=20932 dedup_ratio=0.244954 router_tokens_per_expert=[807, 775, 720, 808, 791, 729, 750, 774, 807, 768, 741, 772, 737, 749, 727, 811, 764, 793, 759, 761, 749, 788, 760, 804, 767, 776, 779, 816, 747, 729, 763, 777, 777, 793, 746, 782, 787, 808, 752, 730, 760, 749, 779, 803, 789, 749, 754, 780, 747, 757, 761, 754, 820, 753, 781, 742, 719, 790, 748, 781, 766, 752, 780, 765] payload_rows_per_dst=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] dup_rows_per_dst=[2994, 2962, 3028, 2870, 2976, 2972, 3130, 3148] combine_rows_per_src=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64880 dup_rows=24048 dup_rows_remote=21022 dedup_ratio=0.244629 router_tokens_per_expert=[806, 731, 790, 735, 723, 722, 767, 732, 780, 726, 768, 761, 738, 730, 845, 793, 753, 777, 798, 776, 748, 779, 751, 794, 771, 754, 769, 772, 750, 784, 741, 825, 781, 768, 745, 793, 788, 782, 756, 788, 774, 815, 776, 781, 771, 781, 768, 778, 763, 749, 750, 774, 759, 825, 770, 762, 753, 687, 716, 770, 797, 827, 764, 752] payload_rows_per_dst=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] dup_rows_per_dst=[2858, 2988, 2990, 3092, 3026, 3062, 3042, 2990] combine_rows_per_src=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64726 dup_rows=24250 dup_rows_remote=21066 dedup_ratio=0.246684 router_tokens_per_expert=[736, 750, 708, 773, 790, 771, 738, 783, 804, 749, 739, 764, 798, 764, 741, 784, 795, 808, 757, 814, 753, 768, 784, 777, 825, 785, 709, 798, 752, 773, 760, 785, 790, 773, 765, 754, 718, 771, 811, 756, 778, 802, 767, 813, 757, 737, 760, 721, 786, 775, 776, 766, 751, 733, 772, 751, 750, 758, 744, 760, 817, 801, 741, 763] payload_rows_per_dst=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] dup_rows_per_dst=[2908, 3012, 3184, 3060, 3008, 3070, 3050, 2958] combine_rows_per_src=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65008 dup_rows=23940 dup_rows_remote=20944 dedup_ratio=0.243530 router_tokens_per_expert=[757, 763, 777, 753, 758, 776, 714, 815, 774, 798, 735, 756, 700, 775, 811, 759, 760, 746, 774, 745, 757, 796, 735, 734, 761, 807, 787, 725, 802, 719, 821, 754, 726, 795, 793, 763, 795, 743, 722, 806, 794, 754, 786, 804, 781, 766, 763, 801, 748, 774, 825, 804, 734, 782, 745, 785, 774, 797, 754, 748, 742, 748, 776, 780] payload_rows_per_dst=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] dup_rows_per_dst=[2968, 2998, 2896, 2996, 2956, 3040, 3054, 3032] combine_rows_per_src=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343824 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067152 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790480 weight_movement_bytes=34603008 total_nvlink_bytes=568393488 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65032 dup_rows=23848 dup_rows_remote=20778 dedup_ratio=0.242594 router_tokens_per_expert=[742, 768, 766, 732, 764, 749, 791, 767, 773, 803, 754, 804, 796, 746, 781, 790, 743, 832, 741, 736, 729, 812, 767, 745, 728, 783, 767, 797, 761, 785, 778, 773, 791, 761, 748, 781, 775, 810, 735, 754, 742, 793, 802, 766, 741, 772, 737, 757, 758, 728, 794, 787, 776, 768, 748, 775, 758, 752, 777, 811, 772, 756, 757, 767] payload_rows_per_dst=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] dup_rows_per_dst=[2950, 3070, 2890, 3078, 2944, 2978, 2946, 2992] combine_rows_per_src=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65108 dup_rows=24054 dup_rows_remote=21166 dedup_ratio=0.244690 router_tokens_per_expert=[738, 797, 823, 752, 782, 792, 810, 778, 761, 746, 760, 793, 750, 728, 785, 760, 804, 769, 766, 780, 801, 740, 829, 747, 738, 744, 773, 738, 733, 845, 772, 778, 791, 756, 757, 717, 767, 733, 817, 805, 786, 736, 723, 782, 789, 736, 767, 696, 792, 811, 773, 740, 769, 723, 766, 751, 744, 827, 810, 755, 769, 773, 746, 733] payload_rows_per_dst=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] dup_rows_per_dst=[3082, 2982, 3146, 2976, 2984, 2888, 2986, 3010] combine_rows_per_src=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343272 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116328 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531889384 weight_movement_bytes=34603008 total_nvlink_bytes=566492392 weight_to_token_ratio=0.065057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266092088 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840568 weight_movement_bytes=69206016 total_nvlink_bytes=601046584 weight_to_token_ratio=0.130125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344624 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592816 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532841008 weight_movement_bytes=103809024 total_nvlink_bytes=636650032 weight_to_token_ratio=0.194822 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343168 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265460864 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530578560 weight_movement_bytes=69206016 total_nvlink_bytes=599784576 weight_to_token_ratio=0.130435 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343808 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266616576 ingress_bytes=335478784 link_bytes=335478784 token_bytes=532889344 weight_movement_bytes=69206016 total_nvlink_bytes=602095360 weight_to_token_ratio=0.129869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343240 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714312 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085384 weight_movement_bytes=103809024 total_nvlink_bytes=636894408 weight_to_token_ratio=0.194732 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=345096 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267027464 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533709832 weight_movement_bytes=69206016 total_nvlink_bytes=602915848 weight_to_token_ratio=0.129670 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64710 dup_rows=24006 dup_rows_remote=20718 dedup_ratio=0.244202 router_tokens_per_expert=[764, 785, 767, 759, 783, 814, 727, 736, 765, 816, 770, 757, 789, 797, 775, 775, 756, 754, 825, 782, 729, 762, 771, 787, 789, 740, 743, 771, 741, 786, 796, 777, 730, 803, 786, 787, 800, 734, 782, 739, 760, 751, 801, 756, 756, 791, 782, 711, 761, 746, 701, 779, 800, 756, 764, 765, 773, 743, 745, 752, 795, 769, 797, 749] payload_rows_per_dst=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] dup_rows_per_dst=[3288, 3070, 2832, 2982, 2974, 2928, 2962, 2970] combine_rows_per_src=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65022 dup_rows=23796 dup_rows_remote=20662 dedup_ratio=0.242065 router_tokens_per_expert=[734, 719, 775, 767, 729, 734, 803, 739, 730, 807, 763, 768, 767, 706, 792, 791, 764, 809, 801, 797, 765, 800, 776, 755, 766, 768, 785, 782, 771, 780, 767, 750, 772, 772, 755, 705, 816, 802, 798, 777, 744, 745, 706, 800, 712, 818, 778, 781, 752, 762, 744, 788, 748, 732, 754, 777, 734, 756, 777, 752, 798, 817, 789, 831] payload_rows_per_dst=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] dup_rows_per_dst=[2844, 2956, 3116, 3078, 3016, 2820, 2832, 3134] combine_rows_per_src=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65130 dup_rows=23976 dup_rows_remote=20898 dedup_ratio=0.243896 router_tokens_per_expert=[803, 732, 776, 778, 719, 776, 733, 769, 782, 774, 750, 785, 787, 761, 769, 749, 761, 751, 798, 741, 711, 749, 774, 783, 757, 727, 766, 749, 794, 795, 796, 767, 789, 761, 781, 728, 746, 764, 765, 804, 803, 781, 819, 775, 728, 760, 790, 749, 758, 783, 766, 780, 780, 817, 733, 695, 785, 766, 797, 782, 786, 789, 782, 748] payload_rows_per_dst=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] dup_rows_per_dst=[2972, 3036, 2868, 2898, 3078, 3068, 2998, 3058] combine_rows_per_src=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=65156 dup_rows=23850 dup_rows_remote=20824 dedup_ratio=0.242615 router_tokens_per_expert=[757, 758, 770, 750, 782, 793, 796, 809, 764, 732, 687, 748, 738, 818, 772, 736, 749, 794, 755, 760, 765, 778, 789, 772, 834, 739, 816, 786, 778, 752, 744, 779, 770, 775, 750, 750, 760, 785, 725, 767, 759, 805, 790, 798, 768, 773, 815, 759, 781, 767, 782, 726, 813, 742, 759, 749, 731, 769, 777, 734, 775, 772, 731, 795] payload_rows_per_dst=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] dup_rows_per_dst=[2952, 2876, 3026, 3024, 2922, 3070, 3058, 2922] combine_rows_per_src=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65196 dup_rows=23730 dup_rows_remote=20730 dedup_ratio=0.241394 router_tokens_per_expert=[747, 768, 745, 798, 739, 748, 758, 790, 738, 771, 694, 764, 747, 797, 751, 756, 761, 775, 780, 728, 801, 766, 758, 788, 821, 731, 800, 788, 736, 757, 748, 808, 840, 782, 812, 819, 701, 717, 762, 760, 745, 773, 747, 746, 796, 779, 777, 749, 781, 765, 699, 777, 754, 781, 838, 765, 775, 794, 779, 754, 751, 761, 841, 775] payload_rows_per_dst=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] dup_rows_per_dst=[2914, 2904, 2894, 3000, 3002, 2996, 2972, 3048] combine_rows_per_src=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74606 payload_rows_remote=65210 dup_rows=23698 dup_rows_remote=20722 dedup_ratio=0.241069 router_tokens_per_expert=[802, 775, 763, 768, 719, 748, 783, 795, 782, 751, 763, 768, 818, 778, 779, 747, 783, 716, 785, 761, 739, 747, 763, 753, 742, 746, 793, 789, 741, 784, 836, 777, 765, 788, 811, 769, 749, 772, 787, 724, 749, 804, 746, 738, 759, 755, 753, 786, 768, 773, 808, 760, 754, 795, 733, 768, 751, 769, 760, 807, 753, 741, 794, 769] payload_rows_per_dst=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] dup_rows_per_dst=[2936, 2976, 2928, 3040, 3048, 2928, 2974, 2868] combine_rows_per_src=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64958 dup_rows=24130 dup_rows_remote=21224 dedup_ratio=0.245463 router_tokens_per_expert=[759, 728, 752, 768, 760, 734, 770, 770, 776, 768, 769, 762, 770, 767, 773, 727, 782, 805, 778, 747, 740, 807, 779, 808, 737, 743, 816, 789, 791, 760, 775, 792, 758, 753, 804, 784, 742, 799, 754, 760, 693, 791, 752, 791, 775, 798, 780, 795, 758, 774, 766, 765, 763, 726, 744, 765, 770, 768, 829, 751, 754, 735, 770, 783] payload_rows_per_dst=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] dup_rows_per_dst=[2876, 2950, 3172, 3048, 3106, 3062, 2906, 3010] combine_rows_per_src=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=342736 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266672848 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533002960 weight_movement_bytes=69206016 total_nvlink_bytes=602208976 weight_to_token_ratio=0.129842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65026 dup_rows=23954 dup_rows_remote=21014 dedup_ratio=0.243673 router_tokens_per_expert=[721, 764, 804, 802, 741, 757, 785, 779, 796, 814, 814, 766, 774, 776, 767, 701, 759, 739, 812, 775, 766, 737, 761, 778, 733, 770, 773, 825, 754, 786, 784, 738, 756, 779, 734, 780, 799, 746, 803, 794, 775, 770, 797, 780, 716, 765, 789, 740, 797, 810, 742, 722, 781, 767, 778, 746, 691, 766, 747, 790, 759, 757, 778, 747] payload_rows_per_dst=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] dup_rows_per_dst=[3006, 3112, 2968, 2984, 3042, 2940, 3018, 2884] combine_rows_per_src=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=341712 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265393872 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446032 weight_movement_bytes=34603008 total_nvlink_bytes=565049040 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344112 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267116592 ingress_bytes=370581504 link_bytes=370581504 token_bytes=533889072 weight_movement_bytes=103809024 total_nvlink_bytes=637698096 weight_to_token_ratio=0.194439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394468365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343920 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267222896 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534101872 weight_movement_bytes=34603008 total_nvlink_bytes=568704880 weight_to_token_ratio=0.064787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=343704 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267386520 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534429336 weight_movement_bytes=69206016 total_nvlink_bytes=603635352 weight_to_token_ratio=0.129495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343728 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267443888 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534544048 weight_movement_bytes=34603008 total_nvlink_bytes=569147056 weight_to_token_ratio=0.064734 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344160 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266690656 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037152 weight_movement_bytes=103809024 total_nvlink_bytes=636846176 weight_to_token_ratio=0.194750 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344728 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412696 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480664 weight_movement_bytes=34603008 total_nvlink_bytes=567083672 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64814 dup_rows=24036 dup_rows_remote=20888 dedup_ratio=0.244507 router_tokens_per_expert=[768, 735, 721, 732, 753, 772, 793, 767, 754, 788, 749, 773, 795, 748, 782, 817, 762, 763, 777, 755, 743, 817, 770, 768, 743, 789, 763, 790, 713, 769, 793, 716, 786, 784, 766, 735, 737, 762, 741, 778, 797, 723, 756, 778, 754, 786, 770, 772, 812, 737, 753, 762, 760, 771, 785, 768, 784, 798, 779, 752, 770, 764, 801, 853] payload_rows_per_dst=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] dup_rows_per_dst=[2642, 2950, 2636, 3046, 3444, 3094, 3076, 3148] combine_rows_per_src=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=65238 dup_rows=23570 dup_rows_remote=20552 dedup_ratio=0.239766 router_tokens_per_expert=[763, 749, 805, 712, 796, 728, 759, 795, 753, 771, 742, 733, 768, 727, 778, 793, 766, 716, 754, 775, 765, 761, 794, 742, 771, 769, 779, 769, 775, 749, 772, 773, 785, 811, 720, 794, 798, 773, 780, 796, 746, 786, 782, 800, 781, 761, 791, 807, 726, 787, 771, 790, 732, 776, 735, 743, 793, 791, 778, 803, 723, 777, 809, 705] payload_rows_per_dst=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] dup_rows_per_dst=[2896, 2912, 2896, 2906, 3018, 3066, 2884, 2992] combine_rows_per_src=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64868 dup_rows=24020 dup_rows_remote=20930 dedup_ratio=0.244344 router_tokens_per_expert=[769, 755, 778, 777, 741, 799, 781, 792, 781, 750, 771, 767, 789, 793, 732, 755, 779, 761, 739, 749, 764, 792, 751, 781, 709, 777, 775, 750, 753, 792, 766, 797, 722, 788, 793, 751, 774, 764, 711, 757, 792, 778, 812, 764, 783, 772, 758, 776, 755, 764, 743, 779, 774, 773, 751, 774, 762, 810, 764, 786, 749, 771, 765, 772] payload_rows_per_dst=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] dup_rows_per_dst=[3090, 3182, 2888, 2784, 2946, 3090, 3212, 2828] combine_rows_per_src=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64618 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[779, 779, 800, 766, 773, 777, 854, 785, 769, 755, 771, 769, 744, 802, 763, 735, 766, 757, 807, 791, 820, 789, 780, 786, 761, 735, 813, 732, 769, 764, 772, 746, 735, 745, 729, 785, 786, 746, 748, 682, 798, 759, 788, 730, 742, 758, 706, 752, 749, 742, 801, 778, 728, 790, 763, 784, 799, 735, 793, 808, 757, 762, 817, 748] payload_rows_per_dst=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] dup_rows_per_dst=[3212, 3050, 3042, 2930, 2760, 2972, 3068, 3102] combine_rows_per_src=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342808 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265820952 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299096 weight_movement_bytes=69206016 total_nvlink_bytes=600505112 weight_to_token_ratio=0.130258 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65138 dup_rows=23906 dup_rows_remote=20860 dedup_ratio=0.243184 router_tokens_per_expert=[746, 762, 781, 753, 743, 742, 774, 800, 769, 795, 785, 796, 762, 737, 747, 771, 813, 759, 764, 819, 765, 735, 755, 814, 768, 800, 757, 739, 791, 709, 823, 766, 687, 787, 764, 759, 793, 760, 784, 785, 788, 779, 789, 752, 756, 775, 748, 713, 730, 762, 773, 787, 737, 781, 788, 775, 739, 765, 767, 812, 788, 756, 762, 771] payload_rows_per_dst=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] dup_rows_per_dst=[2950, 2974, 3028, 3046, 3016, 2912, 3004, 2976] combine_rows_per_src=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74570 payload_rows_remote=65208 dup_rows=23734 dup_rows_remote=20772 dedup_ratio=0.241435 router_tokens_per_expert=[777, 798, 812, 785, 735, 810, 778, 743, 746, 799, 804, 751, 722, 750, 816, 774, 745, 756, 743, 756, 740, 777, 786, 784, 742, 753, 768, 792, 773, 761, 788, 771, 778, 739, 729, 779, 806, 763, 727, 740, 760, 754, 777, 778, 737, 783, 708, 806, 809, 756, 779, 743, 759, 781, 807, 766, 812, 764, 794, 734, 796, 753, 743, 757] payload_rows_per_dst=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] dup_rows_per_dst=[3036, 2962, 2946, 2908, 2826, 3048, 3012, 2996] combine_rows_per_src=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64960 dup_rows=24130 dup_rows_remote=21090 dedup_ratio=0.245463 router_tokens_per_expert=[780, 754, 789, 777, 794, 759, 749, 759, 772, 757, 805, 750, 762, 739, 779, 759, 778, 741, 722, 784, 777, 787, 760, 743, 813, 746, 774, 813, 747, 817, 809, 765, 745, 772, 777, 766, 782, 785, 747, 744, 791, 787, 784, 761, 742, 778, 749, 735, 824, 742, 769, 749, 772, 831, 743, 798, 780, 708, 744, 792, 761, 754, 744, 736] payload_rows_per_dst=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] dup_rows_per_dst=[3108, 3050, 2950, 3116, 2962, 3040, 3098, 2806] combine_rows_per_src=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=343160 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267558008 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534772856 weight_movement_bytes=34603008 total_nvlink_bytes=569375864 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343192 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042520 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741848 weight_movement_bytes=34603008 total_nvlink_bytes=566344856 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65280 dup_rows=23814 dup_rows_remote=20950 dedup_ratio=0.242249 router_tokens_per_expert=[800, 757, 785, 764, 804, 766, 754, 769, 735, 769, 758, 762, 816, 763, 781, 777, 766, 727, 742, 720, 811, 777, 824, 830, 780, 741, 764, 738, 742, 788, 778, 753, 731, 754, 764, 772, 796, 786, 820, 777, 713, 743, 748, 816, 756, 801, 729, 781, 750, 742, 796, 776, 753, 778, 718, 724, 780, 769, 768, 775, 779, 764, 776, 776] payload_rows_per_dst=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] dup_rows_per_dst=[3028, 3078, 2954, 2928, 3068, 2890, 2864, 3004] combine_rows_per_src=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=342848 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018176 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529693504 weight_movement_bytes=69206016 total_nvlink_bytes=598899520 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=343992 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149240 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954488 weight_movement_bytes=34603008 total_nvlink_bytes=568557496 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343920 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435888 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527856 weight_movement_bytes=103809024 total_nvlink_bytes=638336880 weight_to_token_ratio=0.194207 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344200 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420360 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496520 weight_movement_bytes=34603008 total_nvlink_bytes=567099528 weight_to_token_ratio=0.064983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64954 dup_rows=23966 dup_rows_remote=20810 dedup_ratio=0.243795 router_tokens_per_expert=[759, 794, 770, 765, 762, 780, 735, 811, 788, 749, 776, 740, 722, 778, 756, 857, 739, 774, 795, 778, 770, 747, 764, 750, 759, 769, 811, 776, 747, 724, 722, 805, 735, 761, 740, 808, 756, 820, 803, 815, 753, 713, 747, 733, 705, 789, 718, 777, 815, 780, 836, 772, 726, 769, 779, 773, 747, 806, 748, 794, 805, 736, 743, 778] payload_rows_per_dst=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] dup_rows_per_dst=[2894, 2818, 3056, 2934, 3230, 2880, 2998, 3156] combine_rows_per_src=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=344920 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267731800 ingress_bytes=301989888 link_bytes=301989888 token_bytes=535118680 weight_movement_bytes=34603008 total_nvlink_bytes=569721688 weight_to_token_ratio=0.064664 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136363636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65018 dup_rows=24064 dup_rows_remote=21072 dedup_ratio=0.244792 router_tokens_per_expert=[749, 797, 725, 761, 748, 785, 770, 818, 772, 755, 796, 775, 767, 786, 736, 759, 783, 765, 753, 777, 732, 784, 822, 760, 792, 780, 732, 820, 749, 697, 794, 754, 781, 750, 782, 777, 752, 763, 726, 776, 785, 771, 756, 763, 753, 784, 780, 821, 756, 741, 753, 764, 757, 798, 742, 786, 754, 751, 753, 745, 800, 790, 797, 752] payload_rows_per_dst=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] dup_rows_per_dst=[2952, 3028, 3118, 2906, 2992, 3058, 3026, 2984] combine_rows_per_src=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64738 dup_rows=24088 dup_rows_remote=21032 dedup_ratio=0.245036 router_tokens_per_expert=[766, 788, 763, 781, 792, 815, 787, 766, 734, 786, 795, 793, 750, 788, 811, 757, 752, 782, 786, 788, 812, 778, 755, 751, 732, 742, 781, 782, 741, 775, 769, 768, 767, 786, 767, 796, 746, 806, 801, 699, 780, 771, 734, 791, 739, 768, 757, 762, 765, 693, 766, 773, 785, 833, 760, 730, 725, 771, 771, 744, 771, 764, 706, 759] payload_rows_per_dst=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] dup_rows_per_dst=[3056, 3126, 3060, 2976, 3084, 2964, 3116, 2706] combine_rows_per_src=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343056 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266394640 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446224 weight_movement_bytes=34603008 total_nvlink_bytes=567049232 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64826 dup_rows=24130 dup_rows_remote=21138 dedup_ratio=0.245463 router_tokens_per_expert=[762, 817, 761, 718, 774, 781, 767, 769, 731, 769, 773, 748, 797, 740, 744, 776, 777, 750, 811, 780, 739, 768, 785, 760, 733, 775, 783, 743, 808, 781, 749, 735, 789, 681, 782, 747, 744, 768, 756, 796, 742, 736, 766, 797, 787, 749, 791, 736, 796, 773, 783, 748, 778, 834, 778, 773, 764, 831, 751, 755, 780, 792, 780, 765] payload_rows_per_dst=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] dup_rows_per_dst=[3086, 2940, 2992, 3040, 2952, 2890, 3068, 3162] combine_rows_per_src=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20986 dedup_ratio=0.243917 router_tokens_per_expert=[726, 728, 779, 795, 701, 774, 798, 776, 745, 826, 792, 828, 763, 792, 757, 762, 803, 735, 790, 747, 754, 749, 729, 724, 756, 730, 793, 799, 790, 756, 761, 783, 792, 770, 763, 773, 766, 752, 757, 762, 801, 740, 756, 833, 784, 760, 751, 780, 761, 764, 758, 787, 776, 799, 766, 754, 767, 763, 770, 769, 751, 749, 790, 747] payload_rows_per_dst=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] dup_rows_per_dst=[2900, 3100, 2938, 2992, 2994, 2994, 3034, 3026] combine_rows_per_src=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65004 dup_rows=23874 dup_rows_remote=20938 dedup_ratio=0.242859 router_tokens_per_expert=[802, 783, 814, 757, 761, 790, 761, 749, 738, 827, 777, 797, 728, 788, 752, 774, 767, 726, 711, 747, 778, 770, 776, 767, 812, 772, 799, 734, 739, 809, 767, 748, 802, 737, 770, 750, 764, 780, 779, 759, 737, 732, 795, 748, 770, 777, 764, 773, 775, 719, 813, 798, 808, 806, 774, 775, 752, 760, 757, 803, 734, 736, 773, 712] payload_rows_per_dst=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] dup_rows_per_dst=[3072, 2936, 2910, 3052, 2954, 2898, 3114, 2938] combine_rows_per_src=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65136 dup_rows=23924 dup_rows_remote=20866 dedup_ratio=0.243368 router_tokens_per_expert=[784, 750, 753, 733, 803, 825, 760, 811, 782, 762, 768, 765, 810, 776, 736, 813, 803, 757, 763, 828, 778, 725, 751, 733, 769, 775, 775, 729, 776, 767, 746, 782, 743, 725, 712, 780, 729, 805, 719, 827, 804, 759, 729, 767, 836, 746, 763, 747, 758, 723, 716, 777, 799, 785, 736, 782, 795, 743, 744, 777, 780, 779, 792, 787] payload_rows_per_dst=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] dup_rows_per_dst=[3024, 3042, 2978, 3038, 2916, 3058, 2916, 2952] combine_rows_per_src=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344360 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658088 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971816 weight_movement_bytes=34603008 total_nvlink_bytes=567574824 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343080 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265509928 ingress_bytes=368975872 link_bytes=368975872 token_bytes=530676776 weight_movement_bytes=103809024 total_nvlink_bytes=634485800 weight_to_token_ratio=0.195616 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388426501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343856 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871152 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398448 weight_movement_bytes=34603008 total_nvlink_bytes=566001456 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64990 dup_rows=24162 dup_rows_remote=21244 dedup_ratio=0.245789 router_tokens_per_expert=[725, 783, 801, 808, 784, 772, 760, 782, 806, 772, 722, 789, 791, 797, 787, 783, 775, 814, 767, 792, 807, 825, 794, 806, 792, 738, 766, 714, 747, 751, 737, 778, 789, 732, 737, 767, 739, 711, 729, 744, 774, 775, 740, 738, 739, 774, 757, 786, 690, 791, 777, 785, 776, 763, 705, 748, 770, 772, 811, 758, 757, 783, 755, 815] payload_rows_per_dst=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] dup_rows_per_dst=[3122, 3100, 3246, 2954, 2860, 2948, 2918, 3014] combine_rows_per_src=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343872 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510144 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676416 weight_movement_bytes=34603008 total_nvlink_bytes=567279424 weight_to_token_ratio=0.064961 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343768 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266600152 ingress_bytes=370065408 link_bytes=370065408 token_bytes=532856536 weight_movement_bytes=103809024 total_nvlink_bytes=636665560 weight_to_token_ratio=0.194816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392526338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344008 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141064 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938120 weight_movement_bytes=34603008 total_nvlink_bytes=568541128 weight_to_token_ratio=0.064807 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64768 dup_rows=24178 dup_rows_remote=21008 dedup_ratio=0.245951 router_tokens_per_expert=[797, 720, 757, 708, 810, 713, 772, 771, 784, 794, 749, 761, 773, 793, 685, 799, 758, 766, 738, 734, 715, 777, 834, 787, 787, 746, 745, 777, 776, 805, 760, 749, 781, 710, 770, 822, 772, 736, 780, 752, 797, 784, 770, 829, 757, 798, 703, 815, 768, 761, 742, 779, 752, 789, 800, 804, 821, 731, 759, 759, 764, 822, 740, 745] payload_rows_per_dst=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] dup_rows_per_dst=[2438, 2608, 2968, 3520, 3044, 3118, 3312, 3170] combine_rows_per_src=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344936 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543976 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532743016 weight_movement_bytes=34603008 total_nvlink_bytes=567346024 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65020 dup_rows=24024 dup_rows_remote=21008 dedup_ratio=0.244385 router_tokens_per_expert=[791, 794, 761, 806, 803, 778, 759, 807, 749, 760, 756, 724, 757, 773, 781, 763, 742, 734, 772, 806, 785, 777, 812, 782, 776, 751, 751, 753, 789, 753, 784, 734, 780, 815, 721, 785, 750, 750, 764, 773, 777, 764, 738, 778, 779, 744, 780, 813, 755, 792, 723, 729, 792, 733, 773, 779, 777, 778, 779, 730, 740, 751, 777, 770] payload_rows_per_dst=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] dup_rows_per_dst=[3162, 2844, 2980, 3002, 3016, 3046, 2950, 3024] combine_rows_per_src=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343104 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265632832 ingress_bytes=299892736 link_bytes=299892736 token_bytes=530922560 weight_movement_bytes=34603008 total_nvlink_bytes=565525568 weight_to_token_ratio=0.065175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128472222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64968 dup_rows=23944 dup_rows_remote=20934 dedup_ratio=0.243571 router_tokens_per_expert=[773, 753, 756, 826, 800, 742, 801, 743, 741, 744, 770, 758, 769, 753, 826, 775, 757, 765, 767, 753, 860, 809, 741, 749, 732, 703, 731, 808, 761, 766, 765, 796, 763, 738, 768, 759, 762, 786, 787, 768, 776, 773, 763, 723, 774, 770, 762, 702, 788, 778, 763, 750, 786, 770, 734, 725, 776, 819, 738, 790, 785, 773, 793, 817] payload_rows_per_dst=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] dup_rows_per_dst=[2958, 3032, 3010, 2920, 3010, 2962, 2972, 3080] combine_rows_per_src=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64784 dup_rows=24220 dup_rows_remote=21252 dedup_ratio=0.246379 router_tokens_per_expert=[796, 753, 779, 736, 777, 733, 795, 765, 748, 733, 762, 734, 772, 780, 738, 771, 767, 778, 782, 801, 807, 798, 786, 729, 731, 744, 779, 721, 820, 723, 749, 787, 768, 741, 775, 781, 744, 774, 820, 764, 772, 765, 756, 807, 778, 802, 764, 746, 756, 779, 748, 801, 810, 793, 767, 789, 755, 754, 776, 810, 724, 757, 737, 765] payload_rows_per_dst=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] dup_rows_per_dst=[2968, 2998, 3126, 3098, 3060, 3074, 2866, 3030] combine_rows_per_src=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65220 dup_rows=23994 dup_rows_remote=21126 dedup_ratio=0.244080 router_tokens_per_expert=[731, 749, 745, 746, 828, 793, 811, 783, 793, 815, 803, 723, 796, 799, 753, 780, 746, 796, 808, 802, 797, 773, 723, 806, 778, 741, 722, 745, 767, 692, 779, 755, 777, 807, 761, 773, 799, 772, 743, 724, 764, 791, 745, 780, 737, 794, 759, 755, 759, 749, 801, 770, 805, 750, 757, 782, 778, 752, 763, 737, 750, 755, 758, 727] payload_rows_per_dst=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] dup_rows_per_dst=[3016, 3058, 3130, 2868, 2906, 3050, 2986, 2980] combine_rows_per_src=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64864 dup_rows=24118 dup_rows_remote=21076 dedup_ratio=0.245341 router_tokens_per_expert=[790, 769, 827, 792, 777, 756, 762, 781, 822, 736, 765, 747, 807, 737, 772, 796, 734, 777, 787, 720, 765, 825, 760, 760, 760, 777, 774, 742, 745, 732, 723, 763, 763, 793, 798, 816, 755, 705, 752, 760, 793, 750, 749, 756, 802, 773, 748, 765, 782, 765, 763, 762, 776, 731, 749, 828, 773, 749, 789, 733, 806, 743, 813, 732] payload_rows_per_dst=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] dup_rows_per_dst=[3028, 3042, 2978, 2912, 2996, 3020, 3076, 3066] combine_rows_per_src=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65146 dup_rows=23928 dup_rows_remote=21078 dedup_ratio=0.243408 router_tokens_per_expert=[783, 763, 765, 819, 752, 810, 808, 779, 750, 755, 815, 773, 784, 795, 763, 779, 723, 744, 798, 727, 747, 749, 762, 780, 770, 747, 775, 769, 761, 780, 770, 785, 741, 750, 773, 722, 762, 778, 775, 778, 755, 790, 714, 752, 745, 757, 781, 746, 833, 783, 785, 759, 780, 776, 781, 757, 790, 796, 743, 785, 747, 724, 737, 777] payload_rows_per_dst=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] dup_rows_per_dst=[3156, 3050, 2880, 2922, 2970, 2850, 3168, 2932] combine_rows_per_src=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266666032 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987952 weight_movement_bytes=34603008 total_nvlink_bytes=567590960 weight_to_token_ratio=0.064923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65118 dup_rows=23864 dup_rows_remote=20904 dedup_ratio=0.242757 router_tokens_per_expert=[789, 770, 771, 756, 750, 756, 752, 798, 744, 784, 749, 830, 783, 790, 777, 780, 725, 811, 759, 805, 716, 810, 784, 740, 755, 802, 706, 723, 793, 765, 755, 772, 743, 746, 771, 795, 753, 798, 793, 757, 728, 779, 796, 759, 748, 787, 780, 764, 794, 763, 758, 773, 775, 782, 746, 750, 807, 757, 761, 738, 804, 819, 717, 711] payload_rows_per_dst=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] dup_rows_per_dst=[3006, 3054, 2938, 2968, 2962, 3000, 2960, 2976] combine_rows_per_src=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452536 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561464 weight_movement_bytes=69206016 total_nvlink_bytes=601767480 weight_to_token_ratio=0.129949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344144 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265699408 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531054672 weight_movement_bytes=69206016 total_nvlink_bytes=600260688 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345384 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486504 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627624 weight_movement_bytes=34603008 total_nvlink_bytes=569230632 weight_to_token_ratio=0.064724 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266026704 ingress_bytes=438697984 link_bytes=438697984 token_bytes=531709648 weight_movement_bytes=173015040 total_nvlink_bytes=704724688 weight_to_token_ratio=0.325394 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650785196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344896 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182912 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020928 weight_movement_bytes=34603008 total_nvlink_bytes=568623936 weight_to_token_ratio=0.064797 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65264 dup_rows=23824 dup_rows_remote=20860 dedup_ratio=0.242350 router_tokens_per_expert=[762, 774, 805, 764, 821, 800, 762, 776, 765, 706, 770, 760, 747, 768, 795, 779, 782, 808, 789, 787, 754, 771, 726, 774, 792, 772, 719, 738, 751, 776, 789, 784, 771, 715, 784, 734, 793, 793, 758, 784, 756, 767, 765, 746, 806, 737, 731, 786, 776, 774, 762, 815, 768, 747, 774, 770, 758, 785, 806, 752, 701, 754, 747, 771] payload_rows_per_dst=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] dup_rows_per_dst=[3084, 2872, 3060, 2960, 2960, 2800, 3124, 2964] combine_rows_per_src=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344088 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267067416 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790744 weight_movement_bytes=138412032 total_nvlink_bytes=672202776 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65090 dup_rows=23958 dup_rows_remote=20934 dedup_ratio=0.243713 router_tokens_per_expert=[771, 809, 783, 739, 711, 740, 760, 731, 756, 782, 758, 737, 781, 752, 748, 782, 817, 845, 784, 756, 747, 804, 766, 768, 779, 743, 768, 770, 745, 760, 762, 779, 763, 773, 769, 742, 774, 803, 764, 752, 778, 849, 774, 731, 736, 775, 805, 767, 714, 757, 832, 748, 740, 787, 740, 762, 796, 783, 763, 788, 788, 786, 707, 773] payload_rows_per_dst=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] dup_rows_per_dst=[2846, 2970, 3080, 2946, 3024, 3112, 2906, 3074] combine_rows_per_src=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344496 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665840 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534987184 weight_movement_bytes=103809024 total_nvlink_bytes=638796208 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64734 dup_rows=24212 dup_rows_remote=21218 dedup_ratio=0.246297 router_tokens_per_expert=[737, 742, 802, 796, 796, 729, 789, 762, 738, 763, 761, 742, 812, 798, 753, 733, 834, 759, 794, 745, 771, 812, 730, 731, 788, 754, 818, 769, 759, 761, 702, 761, 755, 710, 755, 824, 787, 783, 718, 726, 729, 742, 752, 772, 743, 779, 786, 708, 776, 784, 735, 778, 823, 818, 789, 773, 788, 787, 780, 766, 777, 811, 827, 730] payload_rows_per_dst=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] dup_rows_per_dst=[3010, 2952, 2994, 3040, 2936, 2914, 3156, 3210] combine_rows_per_src=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64882 dup_rows=24042 dup_rows_remote=21120 dedup_ratio=0.244568 router_tokens_per_expert=[736, 801, 746, 752, 757, 763, 783, 735, 765, 755, 736, 793, 797, 779, 779, 814, 782, 740, 715, 779, 762, 790, 751, 752, 803, 737, 762, 766, 754, 771, 775, 783, 753, 785, 752, 761, 742, 757, 771, 755, 790, 796, 739, 748, 817, 756, 760, 839, 729, 745, 764, 775, 798, 731, 773, 775, 756, 796, 762, 800, 779, 754, 771, 810] payload_rows_per_dst=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] dup_rows_per_dst=[3004, 3114, 2864, 2922, 2930, 3126, 3004, 3078] combine_rows_per_src=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64974 dup_rows=24006 dup_rows_remote=20812 dedup_ratio=0.244202 router_tokens_per_expert=[786, 727, 718, 771, 819, 784, 779, 766, 738, 785, 772, 777, 775, 807, 798, 782, 769, 778, 761, 742, 780, 717, 809, 785, 749, 788, 782, 745, 775, 772, 768, 760, 800, 845, 738, 812, 731, 799, 737, 768, 746, 767, 794, 773, 770, 814, 772, 740, 768, 755, 768, 754, 749, 742, 737, 731, 744, 743, 793, 706, 833, 703, 777, 779] payload_rows_per_dst=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] dup_rows_per_dst=[3194, 3034, 2946, 2944, 3098, 2992, 2878, 2920] combine_rows_per_src=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64954 dup_rows=24150 dup_rows_remote=21236 dedup_ratio=0.245667 router_tokens_per_expert=[780, 746, 805, 813, 750, 779, 722, 773, 722, 733, 815, 754, 733, 758, 790, 752, 737, 759, 751, 778, 738, 798, 807, 788, 756, 743, 760, 789, 808, 787, 762, 754, 762, 727, 723, 772, 733, 765, 779, 780, 747, 797, 808, 769, 769, 711, 796, 771, 777, 810, 794, 795, 741, 713, 750, 813, 767, 759, 750, 723, 828, 785, 814, 784] payload_rows_per_dst=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] dup_rows_per_dst=[3080, 2914, 3068, 3016, 2922, 3056, 3062, 3032] combine_rows_per_src=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344096 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952736 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561376 weight_movement_bytes=34603008 total_nvlink_bytes=568164384 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64850 dup_rows=24010 dup_rows_remote=21058 dedup_ratio=0.244242 router_tokens_per_expert=[796, 740, 761, 777, 754, 772, 776, 736, 786, 779, 775, 744, 767, 704, 801, 789, 776, 788, 761, 735, 721, 744, 739, 781, 789, 781, 779, 769, 721, 759, 753, 734, 775, 753, 806, 727, 793, 767, 763, 805, 778, 786, 785, 746, 782, 762, 718, 841, 767, 772, 771, 785, 782, 751, 770, 776, 767, 813, 780, 769, 782, 759, 746, 788] payload_rows_per_dst=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] dup_rows_per_dst=[2972, 3050, 2908, 2950, 3112, 2952, 3024, 3042] combine_rows_per_src=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65176 dup_rows=23948 dup_rows_remote=21122 dedup_ratio=0.243612 router_tokens_per_expert=[796, 777, 747, 771, 765, 741, 781, 798, 792, 802, 807, 808, 808, 768, 734, 781, 792, 768, 755, 768, 791, 771, 730, 785, 741, 794, 773, 718, 753, 724, 784, 733, 793, 816, 756, 772, 740, 749, 719, 810, 762, 780, 790, 775, 798, 718, 760, 795, 721, 792, 735, 762, 768, 729, 768, 728, 757, 748, 787, 758, 750, 781, 789, 790] payload_rows_per_dst=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] dup_rows_per_dst=[3010, 3186, 2938, 2954, 2994, 3002, 2826, 3038] combine_rows_per_src=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343808 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265494272 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530644736 weight_movement_bytes=69206016 total_nvlink_bytes=599850752 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344008 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100680 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857352 weight_movement_bytes=34603008 total_nvlink_bytes=566460360 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343144 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266476648 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532610152 weight_movement_bytes=69206016 total_nvlink_bytes=601816168 weight_to_token_ratio=0.129937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=64962 dup_rows=23800 dup_rows_remote=20442 dedup_ratio=0.242106 router_tokens_per_expert=[776, 785, 786, 743, 756, 778, 796, 795, 786, 793, 754, 739, 736, 810, 782, 783, 738, 717, 795, 766, 749, 708, 741, 747, 762, 788, 709, 764, 760, 733, 779, 801, 795, 783, 762, 777, 753, 759, 798, 802, 778, 728, 739, 795, 753, 783, 812, 749, 776, 764, 765, 763, 828, 766, 785, 739, 725, 770, 833, 776, 757, 796, 753, 735] payload_rows_per_dst=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] dup_rows_per_dst=[3042, 3034, 2790, 2928, 2992, 2970, 2686, 3358] combine_rows_per_src=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344760 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266396344 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447928 weight_movement_bytes=34603008 total_nvlink_bytes=567050936 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343632 combine_payload_bytes=265625600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265969232 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594832 weight_movement_bytes=69206016 total_nvlink_bytes=600800848 weight_to_token_ratio=0.130186 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=345192 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267306088 ingress_bytes=301563904 link_bytes=301563904 token_bytes=534266984 weight_movement_bytes=34603008 total_nvlink_bytes=568869992 weight_to_token_ratio=0.064767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134760693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=341616 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266425968 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532510320 weight_movement_bytes=34603008 total_nvlink_bytes=567113328 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65094 dup_rows=23970 dup_rows_remote=20912 dedup_ratio=0.243835 router_tokens_per_expert=[734, 716, 793, 769, 737, 736, 776, 761, 784, 791, 785, 772, 766, 726, 712, 792, 726, 797, 789, 777, 720, 785, 793, 791, 730, 797, 767, 766, 794, 769, 779, 853, 789, 751, 735, 771, 756, 820, 762, 765, 708, 821, 797, 768, 744, 766, 771, 794, 810, 738, 772, 732, 821, 790, 776, 759, 741, 718, 795, 794, 767, 787, 739, 712] payload_rows_per_dst=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] dup_rows_per_dst=[2846, 2928, 3058, 3080, 3058, 3074, 3046, 2880] combine_rows_per_src=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=65174 dup_rows=23800 dup_rows_remote=20884 dedup_ratio=0.242106 router_tokens_per_expert=[788, 770, 748, 759, 783, 721, 769, 758, 740, 781, 776, 804, 779, 758, 774, 805, 798, 734, 760, 757, 738, 762, 756, 818, 741, 734, 757, 779, 796, 784, 780, 795, 770, 729, 706, 760, 794, 772, 782, 761, 783, 754, 732, 754, 759, 752, 808, 738, 825, 773, 787, 739, 777, 735, 780, 811, 777, 785, 707, 798, 796, 759, 768, 779] payload_rows_per_dst=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] dup_rows_per_dst=[2992, 3022, 2916, 2910, 2942, 2960, 3058, 3000] combine_rows_per_src=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64800 dup_rows=24156 dup_rows_remote=21254 dedup_ratio=0.245728 router_tokens_per_expert=[778, 778, 781, 773, 717, 753, 800, 777, 744, 733, 738, 745, 709, 785, 765, 773, 791, 796, 763, 752, 850, 761, 809, 774, 732, 777, 752, 748, 815, 751, 794, 756, 726, 777, 756, 832, 792, 795, 771, 747, 745, 753, 799, 823, 743, 768, 749, 735, 810, 776, 765, 779, 779, 831, 780, 717, 738, 785, 755, 747, 787, 747, 723, 752] payload_rows_per_dst=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] dup_rows_per_dst=[3022, 2824, 3202, 2902, 3120, 3042, 3134, 2910] combine_rows_per_src=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64784 dup_rows=24082 dup_rows_remote=20954 dedup_ratio=0.244975 router_tokens_per_expert=[796, 784, 761, 768, 798, 790, 742, 774, 780, 787, 764, 763, 771, 738, 796, 751, 775, 779, 777, 720, 791, 728, 747, 782, 734, 764, 763, 770, 744, 779, 736, 751, 757, 743, 770, 791, 739, 773, 793, 706, 772, 783, 753, 737, 770, 766, 766, 801, 725, 788, 800, 766, 752, 750, 763, 822, 813, 786, 788, 828, 786, 734, 746, 782] payload_rows_per_dst=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] dup_rows_per_dst=[3128, 3232, 2948, 3140, 3250, 2930, 2824, 2630] combine_rows_per_src=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344024 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969048 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594072 weight_movement_bytes=69206016 total_nvlink_bytes=602800088 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65142 dup_rows=23776 dup_rows_remote=20654 dedup_ratio=0.241862 router_tokens_per_expert=[760, 740, 775, 814, 753, 765, 778, 775, 749, 776, 793, 762, 801, 772, 789, 812, 766, 768, 745, 786, 734, 749, 780, 759, 773, 765, 810, 792, 777, 740, 727, 725, 775, 768, 780, 758, 760, 755, 768, 817, 763, 781, 775, 712, 794, 836, 727, 791, 770, 724, 750, 762, 804, 741, 773, 791, 737, 761, 790, 746, 743, 808, 733, 749] payload_rows_per_dst=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] dup_rows_per_dst=[3000, 3122, 2954, 2956, 2946, 2914, 3010, 2874] combine_rows_per_src=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64832 dup_rows=24096 dup_rows_remote=20934 dedup_ratio=0.245117 router_tokens_per_expert=[759, 774, 790, 753, 720, 773, 730, 740, 743, 781, 768, 791, 750, 799, 774, 777, 759, 759, 770, 775, 746, 750, 792, 799, 740, 755, 722, 787, 800, 759, 820, 762, 718, 734, 763, 738, 796, 793, 768, 777, 771, 789, 764, 785, 810, 800, 763, 787, 798, 746, 741, 766, 760, 791, 786, 788, 770, 728, 770, 787, 752, 778, 737, 781] payload_rows_per_dst=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] dup_rows_per_dst=[2896, 3032, 2916, 3020, 2948, 3162, 3072, 3050] combine_rows_per_src=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344232 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296936 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249640 weight_movement_bytes=34603008 total_nvlink_bytes=568852648 weight_to_token_ratio=0.064769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344216 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265765016 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185816 weight_movement_bytes=34603008 total_nvlink_bytes=565788824 weight_to_token_ratio=0.065143 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64918 dup_rows=24128 dup_rows_remote=21230 dedup_ratio=0.245443 router_tokens_per_expert=[781, 772, 766, 758, 798, 771, 766, 769, 795, 741, 759, 724, 764, 751, 801, 744, 815, 766, 761, 751, 781, 749, 761, 794, 751, 805, 753, 746, 796, 768, 769, 763, 760, 779, 755, 745, 755, 760, 782, 783, 759, 712, 797, 783, 803, 731, 720, 803, 732, 781, 765, 768, 755, 736, 774, 767, 769, 779, 790, 783, 812, 769, 781, 775] payload_rows_per_dst=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] dup_rows_per_dst=[3012, 3026, 2990, 2970, 3022, 3064, 2898, 3146] combine_rows_per_src=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=342952 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265698216 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531053480 weight_movement_bytes=69206016 total_nvlink_bytes=600259496 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64848 dup_rows=24100 dup_rows_remote=20990 dedup_ratio=0.245158 router_tokens_per_expert=[742, 771, 742, 755, 799, 763, 741, 808, 740, 748, 772, 767, 771, 756, 742, 759, 813, 775, 764, 792, 793, 796, 763, 726, 794, 750, 766, 767, 765, 788, 752, 797, 764, 746, 802, 771, 751, 772, 812, 801, 770, 813, 749, 776, 765, 784, 768, 759, 809, 790, 775, 730, 781, 759, 778, 732, 738, 731, 752, 834, 715, 738, 765, 745] payload_rows_per_dst=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] dup_rows_per_dst=[2896, 2932, 3138, 2972, 3116, 3040, 2896, 3110] combine_rows_per_src=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343184 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267164816 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533986448 weight_movement_bytes=69206016 total_nvlink_bytes=603192464 weight_to_token_ratio=0.129603 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343064 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265894936 ingress_bytes=334757888 link_bytes=334757888 token_bytes=531446808 weight_movement_bytes=69206016 total_nvlink_bytes=600652824 weight_to_token_ratio=0.130222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259666982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344592 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248720 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152848 weight_movement_bytes=69206016 total_nvlink_bytes=601358864 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960760 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531578168 weight_movement_bytes=34603008 total_nvlink_bytes=566181176 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65020 dup_rows=23978 dup_rows_remote=20894 dedup_ratio=0.243917 router_tokens_per_expert=[706, 791, 777, 742, 755, 760, 736, 783, 782, 760, 736, 733, 728, 763, 783, 712, 772, 772, 774, 758, 744, 735, 768, 799, 763, 712, 778, 756, 758, 774, 798, 807, 762, 770, 765, 800, 786, 736, 784, 792, 763, 746, 774, 723, 807, 760, 809, 796, 794, 744, 759, 796, 778, 753, 800, 806, 793, 777, 813, 752, 765, 811, 769, 754] payload_rows_per_dst=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] dup_rows_per_dst=[2842, 2840, 2946, 3006, 3084, 3050, 3088, 3122] combine_rows_per_src=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64898 dup_rows=24212 dup_rows_remote=21202 dedup_ratio=0.246297 router_tokens_per_expert=[787, 781, 774, 787, 776, 729, 781, 836, 813, 740, 724, 767, 746, 758, 787, 801, 804, 758, 771, 747, 765, 736, 776, 745, 752, 729, 747, 794, 791, 806, 746, 808, 773, 750, 737, 766, 770, 779, 766, 774, 759, 775, 737, 735, 787, 755, 783, 759, 788, 765, 769, 745, 741, 819, 765, 746, 776, 769, 753, 767, 759, 759, 780, 784] payload_rows_per_dst=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] dup_rows_per_dst=[3046, 3072, 3010, 3048, 3002, 2966, 3032, 3036] combine_rows_per_src=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65264 dup_rows=23788 dup_rows_remote=20796 dedup_ratio=0.241984 router_tokens_per_expert=[803, 686, 781, 770, 777, 763, 816, 770, 802, 747, 769, 744, 752, 768, 784, 709, 785, 807, 795, 712, 762, 761, 842, 731, 768, 771, 786, 717, 823, 779, 760, 718, 789, 785, 787, 848, 773, 774, 761, 752, 754, 740, 743, 748, 734, 813, 772, 737, 757, 764, 751, 764, 791, 756, 763, 777, 794, 773, 781, 847, 762, 691, 748, 765] payload_rows_per_dst=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] dup_rows_per_dst=[3032, 2838, 3106, 2992, 3134, 2870, 2886, 2930] combine_rows_per_src=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64790 dup_rows=24070 dup_rows_remote=20886 dedup_ratio=0.244853 router_tokens_per_expert=[788, 708, 782, 762, 735, 793, 826, 752, 772, 798, 759, 755, 784, 787, 766, 785, 780, 757, 744, 747, 727, 715, 740, 752, 733, 726, 728, 751, 810, 763, 723, 806, 772, 701, 782, 850, 741, 766, 755, 797, 774, 772, 743, 778, 748, 821, 806, 786, 765, 815, 774, 759, 820, 776, 810, 745, 741, 775, 761, 767, 741, 815, 744, 798] payload_rows_per_dst=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] dup_rows_per_dst=[3184, 2932, 3174, 2982, 2846, 2866, 2986, 3100] combine_rows_per_src=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343656 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266665576 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987496 weight_movement_bytes=69206016 total_nvlink_bytes=602193512 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65274 dup_rows=23708 dup_rows_remote=20742 dedup_ratio=0.241170 router_tokens_per_expert=[785, 726, 832, 780, 775, 755, 802, 762, 715, 763, 784, 748, 757, 753, 741, 798, 731, 793, 749, 751, 800, 790, 719, 775, 730, 780, 779, 822, 785, 774, 752, 757, 737, 783, 794, 794, 758, 798, 795, 773, 766, 805, 743, 788, 757, 744, 786, 755, 765, 775, 724, 784, 760, 724, 759, 794, 780, 748, 764, 802, 731, 803, 723, 777] payload_rows_per_dst=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] dup_rows_per_dst=[3080, 2858, 2942, 2954, 3026, 2966, 2910, 2972] combine_rows_per_src=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64872 dup_rows=24114 dup_rows_remote=21094 dedup_ratio=0.245300 router_tokens_per_expert=[813, 773, 733, 774, 747, 732, 736, 753, 712, 807, 745, 774, 731, 792, 801, 807, 764, 748, 745, 805, 811, 769, 724, 758, 714, 794, 756, 773, 795, 805, 771, 800, 764, 764, 767, 766, 784, 775, 782, 729, 790, 765, 790, 790, 795, 808, 772, 753, 739, 781, 785, 731, 828, 743, 746, 726, 735, 785, 773, 756, 800, 742, 745, 781] payload_rows_per_dst=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] dup_rows_per_dst=[3020, 3020, 2996, 3042, 3002, 3052, 2946, 3036] combine_rows_per_src=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=344400 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266166608 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988816 weight_movement_bytes=34603008 total_nvlink_bytes=566591824 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65120 dup_rows=23840 dup_rows_remote=20846 dedup_ratio=0.242513 router_tokens_per_expert=[797, 751, 775, 792, 782, 768, 729, 755, 803, 764, 747, 753, 756, 788, 765, 769, 732, 795, 796, 803, 773, 711, 731, 796, 712, 758, 703, 799, 767, 780, 786, 768, 774, 807, 784, 750, 775, 776, 804, 773, 755, 773, 753, 744, 753, 781, 803, 740, 768, 714, 792, 850, 781, 735, 748, 781, 766, 762, 752, 750, 776, 730, 810, 788] payload_rows_per_dst=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] dup_rows_per_dst=[2990, 2922, 2994, 2940, 3050, 2934, 2994, 3016] combine_rows_per_src=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344240 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665584 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534986928 weight_movement_bytes=103809024 total_nvlink_bytes=638795952 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65034 dup_rows=24026 dup_rows_remote=21112 dedup_ratio=0.244405 router_tokens_per_expert=[786, 748, 767, 747, 751, 740, 796, 783, 755, 807, 754, 765, 797, 808, 722, 806, 765, 748, 790, 776, 753, 809, 779, 786, 765, 770, 773, 801, 760, 816, 756, 732, 755, 781, 815, 793, 768, 750, 743, 710, 745, 774, 754, 777, 746, 817, 736, 745, 787, 772, 745, 789, 764, 807, 734, 755, 751, 770, 757, 772, 733, 767, 783, 746] payload_rows_per_dst=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] dup_rows_per_dst=[2860, 2962, 3018, 3252, 2942, 3096, 2982, 2914] combine_rows_per_src=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=342704 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265722544 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531102384 weight_movement_bytes=34603008 total_nvlink_bytes=565705392 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344064 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706368 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068672 weight_movement_bytes=69206016 total_nvlink_bytes=604274688 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343864 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059576 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775288 weight_movement_bytes=34603008 total_nvlink_bytes=566378296 weight_to_token_ratio=0.065071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343864 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267075384 ingress_bytes=405143552 link_bytes=405143552 token_bytes=533806904 weight_movement_bytes=138412032 total_nvlink_bytes=672218936 weight_to_token_ratio=0.259292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524522569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64832 dup_rows=24128 dup_rows_remote=21080 dedup_ratio=0.245443 router_tokens_per_expert=[741, 769, 797, 818, 786, 766, 775, 762, 742, 731, 725, 780, 783, 779, 786, 754, 752, 788, 808, 794, 758, 757, 728, 763, 769, 749, 738, 744, 758, 744, 792, 750, 738, 767, 746, 824, 780, 795, 750, 796, 799, 732, 771, 819, 714, 793, 818, 780, 791, 717, 761, 805, 741, 802, 780, 846, 737, 717, 766, 761, 749, 726, 743, 802] payload_rows_per_dst=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] dup_rows_per_dst=[3094, 2976, 3094, 2906, 3048, 3060, 3098, 2852] combine_rows_per_src=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344584 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266723848 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533103112 weight_movement_bytes=34603008 total_nvlink_bytes=567706120 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64820 dup_rows=24334 dup_rows_remote=21424 dedup_ratio=0.247538 router_tokens_per_expert=[780, 760, 773, 783, 822, 729, 775, 765, 759, 788, 766, 721, 777, 737, 771, 816, 795, 779, 793, 734, 719, 731, 752, 727, 802, 779, 773, 761, 813, 759, 784, 726, 802, 769, 722, 761, 782, 722, 804, 749, 759, 745, 800, 737, 786, 781, 799, 796, 816, 767, 746, 750, 745, 774, 770, 805, 754, 742, 793, 758, 777, 772, 745, 775] payload_rows_per_dst=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] dup_rows_per_dst=[3132, 3060, 2910, 3156, 2962, 3088, 3006, 3020] combine_rows_per_src=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65182 dup_rows=23954 dup_rows_remote=21016 dedup_ratio=0.243673 router_tokens_per_expert=[756, 762, 775, 788, 762, 715, 792, 837, 760, 752, 806, 767, 789, 769, 807, 786, 778, 760, 784, 758, 808, 758, 750, 758, 772, 724, 766, 747, 735, 770, 745, 794, 704, 810, 730, 778, 793, 748, 759, 767, 806, 738, 710, 749, 804, 753, 770, 727, 811, 773, 821, 758, 786, 752, 817, 762, 794, 776, 779, 790, 725, 766, 711, 755] payload_rows_per_dst=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] dup_rows_per_dst=[3084, 2988, 3012, 2938, 2906, 2962, 3080, 2984] combine_rows_per_src=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343648 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895520 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447392 weight_movement_bytes=34603008 total_nvlink_bytes=566050400 weight_to_token_ratio=0.065111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64668 dup_rows=24268 dup_rows_remote=21120 dedup_ratio=0.246867 router_tokens_per_expert=[797, 793, 762, 770, 785, 802, 749, 800, 790, 724, 757, 782, 761, 781, 755, 760, 717, 747, 773, 777, 756, 763, 758, 768, 728, 814, 792, 759, 730, 776, 728, 750, 771, 738, 753, 769, 774, 755, 753, 801, 794, 744, 771, 833, 734, 761, 794, 772, 758, 796, 807, 805, 804, 728, 754, 789, 763, 797, 752, 756, 774, 749, 742, 757] payload_rows_per_dst=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] dup_rows_per_dst=[3148, 3290, 3086, 2956, 2620, 3044, 3072, 3052] combine_rows_per_src=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65112 dup_rows=23930 dup_rows_remote=21062 dedup_ratio=0.243429 router_tokens_per_expert=[756, 788, 761, 771, 756, 732, 796, 782, 809, 734, 786, 785, 750, 778, 791, 771, 778, 800, 762, 769, 735, 764, 796, 855, 721, 752, 778, 742, 773, 775, 774, 790, 778, 701, 746, 785, 762, 803, 762, 737, 774, 748, 761, 752, 781, 718, 747, 784, 761, 740, 749, 774, 751, 762, 724, 763, 857, 800, 716, 775, 789, 772, 800, 770] payload_rows_per_dst=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] dup_rows_per_dst=[2992, 3090, 3028, 2924, 2964, 2868, 2908, 3156] combine_rows_per_src=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344976 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847696 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531350416 weight_movement_bytes=34603008 total_nvlink_bytes=565953424 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65424 dup_rows=23738 dup_rows_remote=20782 dedup_ratio=0.241475 router_tokens_per_expert=[812, 769, 801, 749, 768, 792, 709, 815, 757, 742, 768, 741, 759, 767, 741, 774, 792, 749, 754, 780, 787, 798, 782, 782, 812, 775, 773, 749, 786, 805, 773, 767, 750, 785, 813, 712, 780, 786, 735, 763, 767, 770, 789, 778, 726, 766, 725, 764, 791, 749, 788, 754, 719, 811, 730, 774, 771, 761, 747, 776, 742, 760, 768, 774] payload_rows_per_dst=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] dup_rows_per_dst=[3030, 2956, 3056, 2996, 2934, 2808, 2992, 2966] combine_rows_per_src=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344792 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330264 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315736 weight_movement_bytes=34603008 total_nvlink_bytes=568918744 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64890 dup_rows=23990 dup_rows_remote=20912 dedup_ratio=0.244039 router_tokens_per_expert=[748, 804, 759, 772, 783, 723, 757, 749, 815, 727, 817, 764, 805, 749, 744, 768, 777, 760, 802, 760, 766, 791, 748, 787, 737, 782, 773, 778, 757, 752, 773, 778, 747, 765, 756, 785, 803, 780, 777, 771, 769, 771, 767, 750, 761, 762, 763, 727, 779, 776, 795, 809, 761, 783, 748, 800, 713, 765, 806, 730, 749, 760, 771, 748] payload_rows_per_dst=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] dup_rows_per_dst=[3036, 3078, 3046, 2916, 3092, 2840, 3078, 2904] combine_rows_per_src=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64936 dup_rows=24134 dup_rows_remote=21056 dedup_ratio=0.245504 router_tokens_per_expert=[802, 767, 798, 799, 751, 726, 801, 787, 775, 745, 735, 761, 690, 757, 741, 812, 822, 791, 702, 720, 763, 771, 783, 783, 801, 787, 775, 755, 770, 796, 747, 834, 778, 747, 769, 734, 769, 728, 708, 836, 754, 812, 789, 730, 739, 772, 733, 771, 790, 780, 793, 811, 789, 724, 757, 774, 746, 783, 779, 804, 747, 735, 764, 760] payload_rows_per_dst=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] dup_rows_per_dst=[2740, 3168, 2768, 3334, 2944, 2906, 3196, 3078] combine_rows_per_src=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=343152 combine_payload_bytes=264880128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265223280 ingress_bytes=334086144 link_bytes=334086144 token_bytes=530103408 weight_movement_bytes=69206016 total_nvlink_bytes=599309424 weight_to_token_ratio=0.130552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257139264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344696 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043448 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742200 weight_movement_bytes=34603008 total_nvlink_bytes=568345208 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344824 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268321528 ingress_bytes=406388736 link_bytes=406388736 token_bytes=536298232 weight_movement_bytes=138412032 total_nvlink_bytes=674710264 weight_to_token_ratio=0.258088 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529208097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266132648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531922088 weight_movement_bytes=69206016 total_nvlink_bytes=601128104 weight_to_token_ratio=0.130106 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64810 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[786, 824, 746, 802, 761, 796, 811, 779, 738, 709, 750, 789, 760, 730, 772, 748, 786, 765, 747, 746, 808, 788, 789, 803, 773, 834, 764, 779, 740, 771, 722, 735, 827, 782, 799, 816, 784, 791, 733, 735, 771, 750, 698, 827, 760, 733, 782, 752, 712, 783, 718, 759, 786, 780, 783, 763, 757, 778, 789, 803, 751, 762, 693, 744] payload_rows_per_dst=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] dup_rows_per_dst=[3128, 2854, 3060, 3012, 3074, 2934, 2980, 2992] combine_rows_per_src=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343968 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321824 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299680 weight_movement_bytes=34603008 total_nvlink_bytes=566902688 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64912 dup_rows=24046 dup_rows_remote=20954 dedup_ratio=0.244609 router_tokens_per_expert=[790, 781, 777, 806, 749, 761, 751, 758, 762, 755, 757, 763, 767, 782, 779, 715, 779, 760, 747, 790, 765, 774, 798, 806, 762, 760, 747, 756, 740, 778, 744, 777, 765, 765, 745, 750, 757, 766, 744, 828, 746, 799, 764, 772, 789, 776, 778, 749, 796, 753, 739, 714, 746, 803, 798, 770, 756, 760, 788, 783, 790, 772, 784, 771] payload_rows_per_dst=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] dup_rows_per_dst=[3058, 2938, 3092, 2964, 3002, 3036, 2958, 2998] combine_rows_per_src=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65220 dup_rows=24010 dup_rows_remote=21116 dedup_ratio=0.244242 router_tokens_per_expert=[801, 766, 786, 723, 793, 739, 763, 730, 768, 769, 785, 775, 781, 774, 827, 767, 785, 781, 749, 768, 718, 751, 798, 772, 756, 712, 734, 761, 747, 787, 716, 771, 755, 772, 778, 750, 773, 775, 804, 784, 808, 770, 779, 757, 782, 738, 767, 782, 803, 737, 731, 763, 767, 804, 752, 760, 728, 778, 747, 796, 829, 750, 802, 778] payload_rows_per_dst=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] dup_rows_per_dst=[2934, 3038, 2994, 2894, 3054, 3076, 2974, 3046] combine_rows_per_src=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343080 combine_payload_bytes=265461760 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265804840 ingress_bytes=334667776 link_bytes=334667776 token_bytes=531266600 weight_movement_bytes=69206016 total_nvlink_bytes=600472616 weight_to_token_ratio=0.130266 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259327898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64990 dup_rows=24054 dup_rows_remote=21058 dedup_ratio=0.244690 router_tokens_per_expert=[746, 795, 745, 783, 775, 742, 740, 786, 823, 818, 753, 750, 783, 726, 777, 760, 711, 751, 786, 788, 741, 717, 730, 754, 739, 744, 751, 749, 775, 788, 776, 738, 764, 809, 770, 756, 771, 745, 737, 815, 772, 770, 781, 803, 777, 759, 740, 789, 710, 765, 736, 789, 813, 811, 746, 767, 774, 777, 750, 792, 788, 823, 841, 772] payload_rows_per_dst=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] dup_rows_per_dst=[2996, 3050, 2844, 2890, 3154, 3104, 2832, 3184] combine_rows_per_src=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65138 dup_rows=23886 dup_rows_remote=20940 dedup_ratio=0.242981 router_tokens_per_expert=[769, 791, 744, 789, 798, 802, 751, 867, 783, 732, 754, 764, 771, 711, 765, 787, 762, 757, 775, 726, 781, 787, 802, 784, 785, 756, 793, 762, 762, 784, 768, 765, 753, 766, 757, 750, 736, 751, 777, 759, 720, 772, 757, 801, 775, 720, 820, 748, 796, 800, 774, 830, 791, 735, 732, 756, 777, 760, 799, 746, 759, 706, 791, 711] payload_rows_per_dst=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] dup_rows_per_dst=[3098, 2952, 2974, 3058, 2828, 2946, 3090, 2940] combine_rows_per_src=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343464 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266223016 ingress_bytes=404291584 link_bytes=404291584 token_bytes=532102568 weight_movement_bytes=138412032 total_nvlink_bytes=670514600 weight_to_token_ratio=0.260123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521316682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345344 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486464 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627584 weight_movement_bytes=34603008 total_nvlink_bytes=569230592 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65108 dup_rows=23948 dup_rows_remote=20858 dedup_ratio=0.243612 router_tokens_per_expert=[786, 748, 757, 768, 800, 761, 744, 773, 731, 829, 761, 767, 739, 767, 809, 766, 805, 795, 801, 739, 768, 754, 751, 750, 784, 769, 779, 777, 754, 766, 726, 777, 854, 749, 785, 811, 755, 754, 769, 777, 804, 767, 765, 735, 762, 743, 771, 766, 800, 756, 752, 766, 779, 808, 754, 704, 757, 756, 741, 732, 790, 772, 771, 746] payload_rows_per_dst=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] dup_rows_per_dst=[2978, 3090, 3006, 3036, 3054, 2956, 2956, 2872] combine_rows_per_src=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65310 dup_rows=23648 dup_rows_remote=20674 dedup_ratio=0.240560 router_tokens_per_expert=[743, 726, 770, 779, 768, 802, 750, 823, 789, 777, 809, 766, 763, 784, 752, 735, 747, 774, 752, 741, 769, 749, 790, 780, 767, 784, 786, 758, 739, 720, 774, 752, 799, 720, 786, 785, 791, 747, 783, 764, 788, 783, 750, 765, 795, 770, 761, 719, 784, 747, 790, 761, 753, 798, 753, 774, 821, 773, 749, 749, 781, 740, 751, 804] payload_rows_per_dst=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] dup_rows_per_dst=[2944, 3008, 2890, 2942, 2948, 2964, 2974, 2978] combine_rows_per_src=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64816 dup_rows=24234 dup_rows_remote=21246 dedup_ratio=0.246521 router_tokens_per_expert=[755, 759, 767, 852, 784, 817, 755, 751, 798, 722, 803, 733, 781, 757, 760, 761, 773, 756, 783, 802, 765, 758, 745, 751, 762, 763, 773, 728, 781, 779, 736, 751, 742, 771, 742, 811, 776, 788, 768, 773, 780, 809, 829, 720, 768, 762, 771, 728, 788, 707, 798, 753, 791, 810, 743, 742, 762, 810, 756, 783, 738, 790, 761, 721] payload_rows_per_dst=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] dup_rows_per_dst=[2790, 3006, 3030, 2984, 3094, 3382, 2960, 2988] combine_rows_per_src=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344192 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266543232 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742272 weight_movement_bytes=69206016 total_nvlink_bytes=601948288 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343864 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026232 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708600 weight_movement_bytes=34603008 total_nvlink_bytes=568311608 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344312 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149560 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954808 weight_movement_bytes=34603008 total_nvlink_bytes=568557816 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=343936 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267853696 ingress_bytes=371318784 link_bytes=371318784 token_bytes=535363456 weight_movement_bytes=103809024 total_nvlink_bytes=639172480 weight_to_token_ratio=0.193904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397242691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65194 dup_rows=23854 dup_rows_remote=20980 dedup_ratio=0.242655 router_tokens_per_expert=[819, 802, 781, 793, 792, 806, 773, 743, 762, 772, 718, 763, 748, 783, 804, 787, 812, 747, 752, 749, 740, 736, 748, 754, 775, 758, 720, 757, 733, 750, 752, 750, 738, 739, 774, 773, 785, 744, 768, 744, 813, 752, 793, 754, 718, 748, 756, 785, 791, 801, 825, 754, 776, 803, 781, 778, 811, 780, 753, 757, 742, 803, 775, 759] payload_rows_per_dst=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] dup_rows_per_dst=[3154, 2948, 2898, 2842, 2874, 2926, 3156, 3056] combine_rows_per_src=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344248 combine_payload_bytes=265486336 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265830584 ingress_bytes=403898368 link_bytes=403898368 token_bytes=531316920 weight_movement_bytes=138412032 total_nvlink_bytes=669728952 weight_to_token_ratio=0.260507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519837042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65178 dup_rows=23922 dup_rows_remote=20934 dedup_ratio=0.243347 router_tokens_per_expert=[794, 799, 779, 763, 734, 780, 795, 783, 732, 792, 760, 765, 771, 815, 756, 809, 772, 753, 757, 753, 773, 780, 768, 740, 798, 761, 794, 770, 763, 757, 739, 743, 747, 778, 730, 740, 779, 747, 802, 753, 744, 737, 762, 749, 784, 755, 803, 769, 715, 773, 837, 834, 807, 779, 730, 749, 769, 763, 773, 767, 724, 786, 769, 750] payload_rows_per_dst=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] dup_rows_per_dst=[3082, 3050, 2988, 2998, 2914, 2964, 3018, 2908] combine_rows_per_src=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64974 dup_rows=24124 dup_rows_remote=21242 dedup_ratio=0.245402 router_tokens_per_expert=[789, 733, 772, 774, 725, 758, 717, 760, 764, 763, 736, 735, 759, 721, 826, 808, 770, 769, 807, 741, 791, 795, 789, 739, 756, 716, 740, 765, 763, 792, 788, 724, 755, 792, 788, 734, 799, 810, 755, 803, 775, 782, 713, 740, 772, 773, 745, 778, 823, 772, 791, 772, 779, 736, 769, 751, 797, 792, 799, 786, 805, 744, 793, 744] payload_rows_per_dst=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] dup_rows_per_dst=[2920, 2974, 3102, 2882, 3106, 3054, 2994, 3092] combine_rows_per_src=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344696 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379320 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413944 weight_movement_bytes=34603008 total_nvlink_bytes=569016952 weight_to_token_ratio=0.064749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344448 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313536 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282624 weight_movement_bytes=34603008 total_nvlink_bytes=568885632 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74576 payload_rows_remote=65236 dup_rows=23728 dup_rows_remote=20804 dedup_ratio=0.241374 router_tokens_per_expert=[760, 753, 729, 764, 770, 771, 788, 777, 771, 757, 739, 748, 818, 775, 730, 792, 790, 770, 749, 716, 773, 832, 813, 778, 785, 740, 772, 766, 774, 751, 817, 753, 759, 761, 758, 768, 736, 790, 728, 778, 754, 768, 796, 744, 813, 767, 793, 772, 716, 735, 758, 757, 736, 731, 776, 834, 776, 799, 763, 807, 745, 783, 753, 777] payload_rows_per_dst=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] dup_rows_per_dst=[2924, 3008, 3072, 2990, 2856, 3084, 2854, 2940] combine_rows_per_src=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64698 dup_rows=24222 dup_rows_remote=21142 dedup_ratio=0.246399 router_tokens_per_expert=[748, 763, 726, 796, 754, 764, 743, 738, 756, 809, 720, 820, 787, 769, 782, 789, 765, 752, 783, 781, 726, 738, 748, 773, 782, 778, 762, 779, 795, 770, 816, 770, 768, 777, 809, 735, 790, 757, 781, 775, 772, 751, 816, 787, 770, 724, 740, 778, 785, 728, 764, 794, 713, 762, 749, 789, 745, 765, 795, 776, 789, 732, 777, 777] payload_rows_per_dst=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] dup_rows_per_dst=[2876, 3080, 2926, 3112, 3110, 2990, 3046, 3082] combine_rows_per_src=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64892 dup_rows=24226 dup_rows_remote=21226 dedup_ratio=0.246440 router_tokens_per_expert=[783, 757, 767, 789, 754, 754, 806, 792, 765, 815, 724, 723, 798, 768, 774, 786, 756, 801, 753, 742, 794, 748, 786, 758, 777, 741, 765, 771, 747, 709, 777, 788, 757, 785, 778, 739, 781, 773, 744, 790, 768, 776, 759, 715, 789, 784, 805, 758, 763, 804, 787, 752, 754, 733, 765, 735, 801, 838, 766, 782, 767, 764, 759, 713] payload_rows_per_dst=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] dup_rows_per_dst=[3086, 3026, 3046, 2962, 3000, 3026, 3000, 3080] combine_rows_per_src=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64994 dup_rows=24090 dup_rows_remote=21262 dedup_ratio=0.245056 router_tokens_per_expert=[811, 786, 806, 821, 801, 791, 741, 768, 763, 772, 732, 781, 781, 714, 778, 772, 779, 764, 765, 752, 777, 772, 771, 814, 729, 755, 780, 770, 766, 745, 754, 775, 756, 795, 776, 803, 740, 762, 721, 771, 746, 731, 736, 771, 766, 717, 784, 773, 793, 756, 808, 779, 746, 724, 745, 799, 756, 809, 765, 772, 766, 773, 759, 768] payload_rows_per_dst=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] dup_rows_per_dst=[3272, 3012, 3066, 2910, 2984, 2828, 3016, 3002] combine_rows_per_src=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=64934 dup_rows=23908 dup_rows_remote=20842 dedup_ratio=0.243205 router_tokens_per_expert=[789, 752, 733, 764, 758, 769, 779, 779, 800, 819, 774, 759, 749, 755, 778, 742, 734, 774, 795, 755, 813, 773, 754, 822, 732, 797, 718, 780, 718, 740, 796, 695, 714, 737, 744, 786, 750, 731, 776, 772, 782, 778, 734, 808, 763, 744, 784, 784, 792, 772, 797, 793, 769, 775, 775, 787, 786, 766, 784, 771, 764, 788, 763, 788] payload_rows_per_dst=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] dup_rows_per_dst=[2926, 2910, 3062, 2640, 3082, 3040, 3182, 3066] combine_rows_per_src=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344160 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267550816 ingress_bytes=371015680 link_bytes=371015680 token_bytes=534757472 weight_movement_bytes=103809024 total_nvlink_bytes=638566496 weight_to_token_ratio=0.194124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396102135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265003008 dispatch_meta_bytes=343360 combine_payload_bytes=265003008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265346368 ingress_bytes=299606016 link_bytes=299606016 token_bytes=530349376 weight_movement_bytes=34603008 total_nvlink_bytes=564952384 weight_to_token_ratio=0.065246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127393318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266312768 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532282432 weight_movement_bytes=69206016 total_nvlink_bytes=601488448 weight_to_token_ratio=0.130017 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266142104 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531939736 weight_movement_bytes=34603008 total_nvlink_bytes=566542744 weight_to_token_ratio=0.065051 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=345024 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560448 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775872 weight_movement_bytes=34603008 total_nvlink_bytes=567378880 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65112 dup_rows=23928 dup_rows_remote=21074 dedup_ratio=0.243408 router_tokens_per_expert=[779, 737, 754, 799, 790, 770, 705, 779, 778, 796, 763, 818, 783, 786, 750, 768, 731, 749, 790, 796, 744, 795, 802, 771, 789, 801, 821, 766, 732, 783, 783, 757, 759, 746, 794, 781, 766, 760, 716, 737, 765, 762, 786, 755, 752, 796, 766, 725, 761, 734, 744, 772, 785, 768, 755, 712, 769, 757, 806, 763, 778, 778, 764, 775] payload_rows_per_dst=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] dup_rows_per_dst=[2994, 3060, 3010, 3086, 2854, 2962, 2920, 3042] combine_rows_per_src=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65026 dup_rows=23920 dup_rows_remote=21038 dedup_ratio=0.243327 router_tokens_per_expert=[807, 753, 764, 764, 751, 802, 751, 753, 735, 830, 778, 738, 764, 732, 742, 787, 756, 769, 754, 785, 747, 760, 753, 796, 769, 777, 761, 780, 769, 826, 737, 758, 780, 767, 760, 780, 757, 727, 787, 744, 794, 783, 759, 755, 760, 738, 802, 752, 755, 746, 758, 760, 804, 813, 770, 814, 745, 746, 770, 799, 742, 744, 792, 801] payload_rows_per_dst=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] dup_rows_per_dst=[2974, 2916, 2882, 2990, 3026, 3070, 3044, 3018] combine_rows_per_src=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64724 dup_rows=24308 dup_rows_remote=21244 dedup_ratio=0.247274 router_tokens_per_expert=[769, 788, 738, 715, 767, 798, 795, 766, 751, 778, 792, 777, 748, 794, 766, 723, 747, 773, 808, 777, 757, 719, 742, 733, 799, 792, 833, 755, 741, 715, 782, 751, 767, 823, 787, 746, 781, 754, 758, 771, 749, 792, 761, 820, 738, 792, 758, 766, 762, 786, 732, 751, 785, 789, 801, 777, 769, 793, 783, 750, 757, 792, 728, 745] payload_rows_per_dst=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] dup_rows_per_dst=[3014, 3036, 2946, 3064, 3110, 3072, 3032, 3034] combine_rows_per_src=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344744 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043496 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742248 weight_movement_bytes=34603008 total_nvlink_bytes=568345256 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344256 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266690752 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037248 weight_movement_bytes=34603008 total_nvlink_bytes=567640256 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=343872 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265453376 ingress_bytes=368918528 link_bytes=368918528 token_bytes=530562880 weight_movement_bytes=103809024 total_nvlink_bytes=634371904 weight_to_token_ratio=0.195658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388210720e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64992 dup_rows=23964 dup_rows_remote=20944 dedup_ratio=0.243774 router_tokens_per_expert=[730, 786, 769, 794, 769, 773, 789, 742, 758, 749, 762, 755, 747, 771, 773, 768, 800, 745, 783, 749, 802, 765, 789, 775, 787, 808, 821, 806, 767, 771, 807, 759, 779, 784, 786, 738, 771, 721, 781, 761, 771, 733, 770, 755, 737, 730, 763, 738, 811, 755, 734, 757, 749, 731, 767, 784, 801, 783, 801, 758, 802, 724, 757, 751] payload_rows_per_dst=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] dup_rows_per_dst=[3020, 2998, 3134, 3328, 3016, 2806, 2866, 2796] combine_rows_per_src=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73990 payload_rows_remote=64758 dup_rows=24314 dup_rows_remote=21276 dedup_ratio=0.247335 router_tokens_per_expert=[787, 748, 804, 767, 763, 798, 743, 793, 744, 756, 767, 747, 715, 767, 771, 766, 770, 794, 812, 743, 744, 765, 709, 736, 776, 775, 732, 747, 802, 852, 712, 775, 740, 809, 765, 789, 768, 733, 787, 743, 773, 739, 778, 776, 756, 793, 805, 801, 765, 783, 804, 757, 759, 783, 788, 743, 797, 742, 707, 814, 784, 737, 773, 781] payload_rows_per_dst=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] dup_rows_per_dst=[3122, 2786, 3220, 2924, 2568, 3388, 3268, 3038] combine_rows_per_src=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=65038 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[761, 721, 805, 699, 809, 758, 764, 732, 722, 759, 762, 796, 718, 792, 787, 750, 733, 783, 741, 759, 763, 736, 754, 826, 781, 794, 780, 790, 750, 728, 779, 798, 752, 805, 777, 740, 789, 751, 783, 785, 746, 747, 760, 780, 715, 764, 774, 848, 761, 715, 768, 755, 807, 775, 821, 783, 790, 793, 785, 739, 806, 760, 793, 755] payload_rows_per_dst=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] dup_rows_per_dst=[2928, 3042, 3006, 2994, 3048, 3008, 3042, 3068] combine_rows_per_src=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65008 dup_rows=24158 dup_rows_remote=21144 dedup_ratio=0.245748 router_tokens_per_expert=[745, 754, 780, 812, 765, 770, 806, 733, 739, 775, 767, 768, 779, 709, 784, 784, 792, 778, 754, 792, 776, 744, 733, 768, 753, 727, 797, 766, 761, 794, 756, 771, 773, 750, 773, 753, 787, 730, 741, 776, 787, 776, 812, 787, 739, 763, 773, 731, 766, 778, 765, 783, 692, 786, 719, 787, 765, 816, 765, 788, 784, 798, 796, 781] payload_rows_per_dst=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] dup_rows_per_dst=[3072, 2982, 2964, 2914, 3036, 3018, 3014, 3158] combine_rows_per_src=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65188 dup_rows=23942 dup_rows_remote=20968 dedup_ratio=0.243551 router_tokens_per_expert=[773, 766, 728, 773, 795, 759, 754, 796, 745, 765, 767, 781, 723, 765, 755, 782, 787, 792, 766, 732, 774, 763, 785, 794, 777, 792, 751, 790, 770, 771, 786, 761, 731, 730, 799, 791, 787, 749, 792, 804, 763, 736, 808, 714, 761, 770, 743, 779, 786, 762, 785, 785, 776, 746, 784, 758, 740, 787, 778, 767, 765, 773, 724, 761] payload_rows_per_dst=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] dup_rows_per_dst=[3012, 2848, 3068, 3090, 3040, 2974, 2944, 2966] combine_rows_per_src=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=344136 combine_payload_bytes=265248768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265592904 ingress_bytes=403660800 link_bytes=403660800 token_bytes=530841672 weight_movement_bytes=138412032 total_nvlink_bytes=669253704 weight_to_token_ratio=0.260741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343744 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550976 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758208 weight_movement_bytes=69206016 total_nvlink_bytes=601964224 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344528 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266740176 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135824 weight_movement_bytes=103809024 total_nvlink_bytes=636944848 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344608 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617376 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532890144 weight_movement_bytes=103809024 total_nvlink_bytes=636699168 weight_to_token_ratio=0.194804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344624 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354672 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364720 weight_movement_bytes=34603008 total_nvlink_bytes=568967728 weight_to_token_ratio=0.064755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64802 dup_rows=24172 dup_rows_remote=21092 dedup_ratio=0.245890 router_tokens_per_expert=[811, 754, 772, 756, 746, 774, 785, 774, 807, 810, 764, 743, 789, 788, 766, 803, 736, 740, 747, 749, 737, 756, 725, 778, 746, 796, 793, 763, 758, 768, 776, 718, 774, 777, 774, 788, 799, 767, 772, 754, 699, 787, 739, 712, 773, 750, 812, 781, 762, 815, 774, 781, 797, 759, 765, 786, 741, 777, 734, 774, 782, 811, 757, 751] payload_rows_per_dst=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] dup_rows_per_dst=[3060, 3116, 2824, 3042, 3080, 2938, 3096, 3016] combine_rows_per_src=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64890 dup_rows=24048 dup_rows_remote=21036 dedup_ratio=0.244629 router_tokens_per_expert=[731, 736, 761, 761, 738, 768, 751, 757, 770, 757, 775, 789, 769, 800, 781, 754, 752, 763, 750, 759, 779, 812, 782, 792, 776, 716, 773, 724, 751, 786, 783, 735, 769, 773, 752, 776, 764, 775, 777, 793, 775, 771, 810, 765, 776, 757, 772, 782, 792, 758, 769, 713, 777, 749, 770, 784, 790, 783, 786, 822, 743, 747, 761, 790] payload_rows_per_dst=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] dup_rows_per_dst=[2858, 3046, 3012, 2964, 3028, 3120, 2924, 3096] combine_rows_per_src=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65198 dup_rows=23814 dup_rows_remote=20924 dedup_ratio=0.242249 router_tokens_per_expert=[753, 760, 794, 768, 801, 779, 725, 748, 757, 787, 773, 751, 715, 837, 792, 763, 760, 782, 776, 752, 790, 805, 792, 734, 767, 801, 735, 750, 753, 767, 751, 767, 749, 802, 760, 812, 793, 753, 744, 718, 814, 766, 792, 721, 768, 771, 758, 703, 770, 722, 782, 784, 760, 781, 752, 763, 758, 752, 781, 797, 786, 780, 771, 804] payload_rows_per_dst=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] dup_rows_per_dst=[2998, 3002, 3036, 2890, 2970, 2896, 3024, 2998] combine_rows_per_src=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343576 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265772568 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531201560 weight_movement_bytes=69206016 total_nvlink_bytes=600407576 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343704 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133144 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531922584 weight_movement_bytes=34603008 total_nvlink_bytes=566525592 weight_to_token_ratio=0.065053 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344488 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267395496 ingress_bytes=336257024 link_bytes=336257024 token_bytes=534446504 weight_movement_bytes=69206016 total_nvlink_bytes=603652520 weight_to_token_ratio=0.129491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265308110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=64822 dup_rows=23886 dup_rows_remote=20678 dedup_ratio=0.242981 router_tokens_per_expert=[738, 768, 797, 757, 749, 805, 805, 740, 759, 778, 761, 754, 804, 779, 799, 749, 745, 811, 772, 781, 758, 733, 742, 781, 811, 789, 768, 756, 793, 797, 774, 792, 786, 720, 775, 717, 747, 751, 778, 764, 723, 763, 791, 807, 737, 784, 755, 759, 777, 759, 769, 789, 776, 786, 779, 751, 738, 768, 791, 728, 711, 798, 768, 762] payload_rows_per_dst=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] dup_rows_per_dst=[3012, 3044, 2978, 2772, 2800, 2942, 3130, 3208] combine_rows_per_src=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65036 dup_rows=23858 dup_rows_remote=20750 dedup_ratio=0.242696 router_tokens_per_expert=[767, 733, 769, 784, 759, 788, 758, 753, 735, 827, 777, 770, 762, 748, 801, 750, 779, 788, 784, 773, 768, 770, 713, 743, 779, 782, 757, 780, 793, 741, 783, 789, 755, 805, 778, 778, 758, 767, 778, 796, 746, 803, 747, 786, 770, 756, 768, 769, 817, 701, 754, 776, 767, 697, 778, 747, 732, 763, 740, 808, 786, 772, 756, 795] payload_rows_per_dst=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] dup_rows_per_dst=[3108, 3032, 2952, 3024, 3006, 2912, 2874, 2950] combine_rows_per_src=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65102 dup_rows=23962 dup_rows_remote=20952 dedup_ratio=0.243754 router_tokens_per_expert=[745, 737, 750, 776, 790, 748, 774, 823, 756, 746, 805, 767, 784, 778, 755, 734, 819, 813, 727, 781, 722, 728, 757, 738, 770, 770, 771, 761, 808, 812, 832, 760, 775, 762, 789, 793, 754, 770, 760, 762, 751, 801, 767, 733, 735, 785, 746, 762, 806, 785, 812, 758, 795, 742, 758, 760, 733, 793, 745, 752, 804, 740, 735, 752] payload_rows_per_dst=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] dup_rows_per_dst=[3008, 3010, 2928, 3158, 3046, 2898, 3074, 2840] combine_rows_per_src=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65144 dup_rows=23982 dup_rows_remote=21034 dedup_ratio=0.243958 router_tokens_per_expert=[738, 800, 752, 764, 794, 784, 761, 790, 790, 775, 805, 770, 781, 762, 813, 732, 760, 738, 773, 775, 765, 744, 788, 831, 791, 776, 820, 780, 768, 750, 768, 755, 742, 774, 759, 781, 764, 727, 771, 707, 746, 760, 775, 791, 770, 777, 771, 759, 753, 768, 744, 814, 727, 764, 749, 744, 719, 747, 795, 758, 779, 773, 808, 743] payload_rows_per_dst=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] dup_rows_per_dst=[3076, 3044, 2970, 3028, 2862, 2996, 2948, 3058] combine_rows_per_src=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65260 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[787, 735, 785, 783, 759, 736, 772, 779, 777, 743, 742, 798, 766, 748, 749, 775, 772, 782, 802, 748, 756, 830, 760, 790, 738, 777, 792, 790, 755, 761, 775, 817, 798, 772, 778, 776, 693, 822, 793, 792, 816, 734, 736, 753, 774, 711, 728, 734, 750, 787, 725, 723, 743, 772, 761, 809, 777, 782, 753, 774, 764, 804, 770, 769] payload_rows_per_dst=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] dup_rows_per_dst=[2964, 2912, 3082, 3100, 3094, 2864, 2902, 3018] combine_rows_per_src=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342000 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265852912 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531363824 weight_movement_bytes=34603008 total_nvlink_bytes=565966832 weight_to_token_ratio=0.065121 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343144 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266730600 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118056 weight_movement_bytes=34603008 total_nvlink_bytes=567721064 weight_to_token_ratio=0.064907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344216 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002008 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533659800 weight_movement_bytes=69206016 total_nvlink_bytes=602865816 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344712 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267174536 ingress_bytes=336035840 link_bytes=336035840 token_bytes=534004360 weight_movement_bytes=69206016 total_nvlink_bytes=603210376 weight_to_token_ratio=0.129598 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264475813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345328 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650288 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955248 weight_movement_bytes=34603008 total_nvlink_bytes=569558256 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65224 dup_rows=23976 dup_rows_remote=20954 dedup_ratio=0.243896 router_tokens_per_expert=[825, 778, 725, 797, 734, 753, 760, 789, 718, 842, 767, 781, 740, 736, 755, 754, 755, 750, 757, 729, 795, 735, 765, 777, 776, 738, 797, 802, 749, 742, 768, 818, 753, 760, 772, 763, 768, 757, 758, 785, 805, 735, 790, 749, 749, 751, 747, 846, 785, 787, 780, 770, 758, 790, 792, 786, 746, 775, 757, 685, 771, 778, 789, 808] payload_rows_per_dst=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] dup_rows_per_dst=[3054, 2896, 3022, 3014, 2954, 2994, 3058, 2984] combine_rows_per_src=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64866 dup_rows=23982 dup_rows_remote=20980 dedup_ratio=0.243958 router_tokens_per_expert=[757, 701, 789, 750, 776, 795, 736, 772, 741, 759, 777, 713, 759, 714, 820, 737, 762, 793, 765, 764, 747, 768, 790, 745, 777, 759, 792, 801, 771, 793, 802, 711, 774, 781, 810, 774, 787, 724, 787, 792, 804, 781, 785, 769, 810, 799, 778, 779, 776, 787, 739, 756, 752, 798, 721, 792, 769, 801, 735, 728, 749, 773, 726, 780] payload_rows_per_dst=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] dup_rows_per_dst=[2882, 2862, 3080, 3056, 3002, 3184, 2978, 2938] combine_rows_per_src=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65232 dup_rows=23724 dup_rows_remote=20740 dedup_ratio=0.241333 router_tokens_per_expert=[769, 767, 762, 705, 764, 755, 758, 728, 752, 795, 747, 766, 795, 732, 763, 771, 813, 763, 735, 807, 783, 753, 718, 765, 785, 783, 736, 792, 751, 804, 771, 744, 723, 804, 762, 774, 768, 766, 743, 816, 804, 742, 786, 763, 745, 802, 765, 769, 778, 802, 742, 810, 822, 790, 785, 750, 800, 790, 751, 797, 745, 720, 762, 744] payload_rows_per_dst=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] dup_rows_per_dst=[2810, 2872, 2928, 2984, 3036, 2952, 3128, 3014] combine_rows_per_src=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344712 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267502216 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659720 weight_movement_bytes=34603008 total_nvlink_bytes=569262728 weight_to_token_ratio=0.064720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343384 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034520 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531725656 weight_movement_bytes=34603008 total_nvlink_bytes=566328664 weight_to_token_ratio=0.065077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=343888 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267534160 ingress_bytes=370999296 link_bytes=370999296 token_bytes=534724432 weight_movement_bytes=103809024 total_nvlink_bytes=638533456 weight_to_token_ratio=0.194136 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396040483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64842 dup_rows=24080 dup_rows_remote=21034 dedup_ratio=0.244954 router_tokens_per_expert=[776, 754, 742, 829, 782, 774, 776, 794, 773, 798, 825, 781, 739, 758, 749, 720, 785, 781, 803, 780, 732, 768, 793, 775, 769, 812, 731, 771, 776, 727, 751, 773, 733, 726, 799, 747, 732, 773, 740, 798, 749, 747, 787, 727, 810, 778, 764, 751, 742, 790, 736, 778, 774, 763, 751, 763, 775, 768, 818, 777, 788, 720, 804, 747] payload_rows_per_dst=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] dup_rows_per_dst=[2986, 3114, 3160, 2756, 2720, 2950, 3348, 3046] combine_rows_per_src=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65010 dup_rows=24044 dup_rows_remote=21108 dedup_ratio=0.244588 router_tokens_per_expert=[704, 761, 803, 723, 771, 746, 802, 773, 794, 810, 773, 827, 778, 774, 798, 763, 785, 697, 794, 773, 741, 731, 808, 750, 733, 793, 759, 835, 817, 781, 770, 741, 855, 815, 749, 793, 758, 805, 775, 764, 770, 744, 766, 771, 751, 749, 751, 753, 767, 771, 771, 716, 718, 738, 771, 715, 766, 794, 751, 801, 733, 762, 722, 779] payload_rows_per_dst=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] dup_rows_per_dst=[2936, 3224, 2970, 3142, 3194, 2894, 2750, 2934] combine_rows_per_src=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65058 dup_rows=23960 dup_rows_remote=21062 dedup_ratio=0.243734 router_tokens_per_expert=[732, 767, 817, 761, 731, 812, 742, 823, 735, 746, 783, 812, 735, 760, 784, 737, 777, 732, 758, 716, 732, 768, 757, 787, 795, 740, 731, 793, 774, 749, 793, 764, 789, 769, 767, 764, 769, 795, 748, 725, 741, 790, 790, 798, 729, 781, 816, 764, 795, 770, 838, 773, 773, 727, 739, 816, 799, 739, 726, 791, 748, 763, 796, 781] payload_rows_per_dst=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] dup_rows_per_dst=[3052, 2898, 2856, 3016, 2998, 2972, 3156, 3012] combine_rows_per_src=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65408 dup_rows=23626 dup_rows_remote=20764 dedup_ratio=0.240336 router_tokens_per_expert=[848, 765, 767, 805, 779, 758, 765, 787, 766, 735, 815, 772, 748, 784, 725, 799, 769, 746, 755, 788, 763, 741, 797, 753, 783, 739, 771, 750, 772, 775, 777, 812, 743, 738, 754, 806, 757, 757, 783, 748, 743, 762, 758, 794, 777, 758, 742, 772, 723, 717, 776, 773, 786, 795, 755, 741, 740, 801, 731, 798, 785, 772, 759, 799] payload_rows_per_dst=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] dup_rows_per_dst=[3074, 2912, 2942, 2906, 2834, 3028, 2862, 3068] combine_rows_per_src=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64996 dup_rows=23972 dup_rows_remote=21066 dedup_ratio=0.243856 router_tokens_per_expert=[740, 760, 750, 756, 714, 768, 749, 770, 761, 753, 725, 764, 778, 814, 763, 716, 756, 768, 741, 798, 788, 826, 753, 748, 781, 777, 825, 758, 761, 719, 786, 796, 786, 765, 753, 788, 790, 816, 715, 758, 726, 803, 770, 733, 759, 791, 757, 782, 788, 809, 781, 813, 778, 746, 794, 728, 776, 746, 817, 786, 767, 747, 767, 755] payload_rows_per_dst=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] dup_rows_per_dst=[2938, 2932, 3056, 3094, 2950, 2906, 3110, 2986] combine_rows_per_src=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343504 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265936336 ingress_bytes=334798848 link_bytes=334798848 token_bytes=531529168 weight_movement_bytes=69206016 total_nvlink_bytes=600735184 weight_to_token_ratio=0.130202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259821112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344472 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266625432 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532906392 weight_movement_bytes=34603008 total_nvlink_bytes=567509400 weight_to_token_ratio=0.064933 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822048 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299616 weight_movement_bytes=34603008 total_nvlink_bytes=567902624 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20928 dedup_ratio=0.243774 router_tokens_per_expert=[778, 778, 755, 778, 764, 706, 768, 733, 780, 749, 772, 753, 725, 771, 802, 753, 794, 797, 784, 757, 750, 727, 781, 739, 826, 758, 781, 735, 810, 743, 819, 766, 784, 769, 765, 788, 778, 755, 784, 811, 796, 804, 772, 750, 765, 770, 758, 731, 814, 759, 720, 758, 792, 781, 750, 760, 759, 781, 760, 780, 738, 796, 728, 764] payload_rows_per_dst=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] dup_rows_per_dst=[2878, 2868, 3036, 3124, 3140, 2988, 3032, 2898] combine_rows_per_src=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344248 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567864 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791480 weight_movement_bytes=103809024 total_nvlink_bytes=636600504 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267911168 dispatch_meta_bytes=344688 combine_payload_bytes=267911168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268255856 ingress_bytes=302514176 link_bytes=302514176 token_bytes=536167024 weight_movement_bytes=34603008 total_nvlink_bytes=570770032 weight_to_token_ratio=0.064538 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138336490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64908 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[829, 797, 762, 770, 754, 778, 752, 747, 791, 743, 758, 767, 811, 751, 759, 776, 766, 751, 721, 754, 785, 773, 748, 716, 783, 789, 807, 783, 765, 747, 790, 767, 777, 786, 773, 768, 763, 747, 733, 780, 778, 769, 729, 795, 768, 771, 802, 788, 782, 726, 739, 738, 796, 767, 762, 799, 744, 738, 765, 780, 787, 765, 767, 780] payload_rows_per_dst=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] dup_rows_per_dst=[3062, 3084, 2876, 3088, 3008, 3090, 2926, 3016] combine_rows_per_src=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65052 dup_rows=23974 dup_rows_remote=20902 dedup_ratio=0.243876 router_tokens_per_expert=[766, 774, 781, 780, 755, 759, 805, 821, 750, 745, 747, 799, 819, 741, 762, 804, 739, 783, 759, 793, 763, 777, 758, 754, 769, 783, 771, 814, 774, 743, 782, 739, 737, 765, 766, 784, 768, 819, 763, 782, 738, 794, 752, 740, 762, 801, 745, 779, 749, 798, 782, 723, 736, 735, 752, 739, 777, 767, 796, 753, 719, 786, 762, 774] payload_rows_per_dst=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] dup_rows_per_dst=[3072, 3000, 2980, 3072, 3060, 2922, 2874, 2994] combine_rows_per_src=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344184 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067512 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790840 weight_movement_bytes=34603008 total_nvlink_bytes=568393848 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64900 dup_rows=24180 dup_rows_remote=21008 dedup_ratio=0.245972 router_tokens_per_expert=[793, 735, 798, 768, 757, 737, 760, 811, 756, 790, 755, 826, 768, 731, 806, 759, 771, 716, 794, 793, 772, 690, 773, 775, 759, 743, 794, 748, 783, 790, 781, 784, 795, 752, 802, 763, 797, 785, 748, 789, 739, 754, 754, 746, 750, 797, 774, 774, 735, 778, 775, 727, 745, 807, 781, 731, 747, 767, 763, 768, 805, 719, 825, 744] payload_rows_per_dst=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] dup_rows_per_dst=[3006, 3042, 2918, 3140, 2978, 2858, 3066, 3172] combine_rows_per_src=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344200 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207368 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070536 weight_movement_bytes=69206016 total_nvlink_bytes=601276552 weight_to_token_ratio=0.130069 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343816 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796808 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249800 weight_movement_bytes=103809024 total_nvlink_bytes=637058824 weight_to_token_ratio=0.194672 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64458 dup_rows=24278 dup_rows_remote=20984 dedup_ratio=0.246969 router_tokens_per_expert=[784, 783, 796, 748, 771, 775, 743, 806, 729, 750, 760, 780, 765, 787, 760, 726, 715, 791, 830, 751, 787, 785, 745, 790, 781, 775, 743, 761, 788, 800, 740, 741, 762, 762, 763, 802, 752, 749, 812, 742, 791, 737, 766, 768, 758, 770, 805, 762, 733, 823, 820, 794, 733, 748, 786, 737, 758, 824, 731, 756, 775, 746, 758, 743] payload_rows_per_dst=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] dup_rows_per_dst=[3294, 3360, 3360, 2874, 3066, 2716, 2556, 3052] combine_rows_per_src=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65166 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[762, 792, 766, 770, 749, 748, 768, 692, 776, 781, 791, 742, 763, 767, 783, 728, 829, 766, 755, 779, 848, 770, 726, 747, 708, 779, 785, 773, 774, 767, 789, 769, 806, 785, 805, 737, 761, 734, 754, 798, 783, 795, 765, 711, 748, 783, 754, 750, 786, 791, 782, 732, 816, 783, 740, 792, 785, 777, 745, 763, 765, 749, 776, 759] payload_rows_per_dst=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] dup_rows_per_dst=[2818, 2944, 2990, 3070, 2980, 2916, 3144, 2958] combine_rows_per_src=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343632 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174032 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004432 weight_movement_bytes=34603008 total_nvlink_bytes=566607440 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65132 dup_rows=23834 dup_rows_remote=20880 dedup_ratio=0.242452 router_tokens_per_expert=[761, 776, 817, 806, 787, 719, 787, 760, 760, 760, 794, 763, 780, 726, 759, 775, 808, 766, 734, 794, 738, 754, 760, 778, 763, 734, 801, 750, 757, 777, 761, 802, 730, 787, 726, 772, 791, 777, 791, 803, 744, 775, 787, 736, 750, 775, 791, 788, 749, 768, 771, 724, 766, 780, 742, 743, 728, 738, 807, 811, 801, 767, 756, 771] payload_rows_per_dst=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] dup_rows_per_dst=[3072, 2894, 2942, 3016, 3022, 2954, 2946, 2988] combine_rows_per_src=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65000 dup_rows=23998 dup_rows_remote=21070 dedup_ratio=0.244120 router_tokens_per_expert=[746, 774, 732, 791, 755, 771, 805, 751, 804, 796, 762, 737, 764, 763, 762, 753, 745, 770, 741, 762, 714, 723, 778, 801, 793, 791, 758, 813, 774, 823, 765, 744, 794, 769, 789, 732, 759, 773, 764, 783, 787, 775, 760, 776, 811, 810, 746, 770, 843, 781, 756, 744, 730, 742, 746, 775, 761, 724, 765, 776, 787, 739, 737, 787] payload_rows_per_dst=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] dup_rows_per_dst=[3052, 3066, 2926, 3118, 2970, 2996, 2928, 2942] combine_rows_per_src=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264019968 dispatch_meta_bytes=341768 combine_payload_bytes=264019968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264361736 ingress_bytes=298622976 link_bytes=298622976 token_bytes=528381704 weight_movement_bytes=34603008 total_nvlink_bytes=562984712 weight_to_token_ratio=0.065489 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123694218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65078 dup_rows=23946 dup_rows_remote=20898 dedup_ratio=0.243591 router_tokens_per_expert=[781, 719, 785, 704, 762, 746, 779, 786, 768, 738, 737, 790, 774, 743, 774, 734, 763, 770, 758, 779, 791, 785, 755, 763, 742, 788, 816, 757, 750, 787, 770, 740, 774, 796, 775, 754, 756, 742, 744, 792, 785, 750, 764, 814, 746, 757, 758, 772, 775, 781, 778, 787, 735, 787, 748, 763, 801, 789, 828, 765, 736, 821, 753, 792] payload_rows_per_dst=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] dup_rows_per_dst=[2932, 2940, 3048, 2964, 2972, 2982, 3018, 3090] combine_rows_per_src=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344168 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264104 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184040 weight_movement_bytes=34603008 total_nvlink_bytes=568787048 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64732 dup_rows=24184 dup_rows_remote=21148 dedup_ratio=0.246012 router_tokens_per_expert=[794, 759, 735, 770, 768, 776, 790, 791, 764, 724, 744, 751, 778, 810, 782, 766, 711, 803, 816, 764, 737, 776, 746, 781, 755, 765, 770, 807, 747, 767, 753, 767, 804, 771, 779, 760, 795, 753, 773, 777, 748, 732, 817, 737, 793, 778, 755, 784, 767, 783, 740, 800, 750, 799, 777, 787, 762, 775, 746, 721, 780, 743, 769, 730] payload_rows_per_dst=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] dup_rows_per_dst=[3078, 2976, 2966, 3132, 3036, 3036, 3036, 2924] combine_rows_per_src=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344048 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124720 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905392 weight_movement_bytes=69206016 total_nvlink_bytes=603111408 weight_to_token_ratio=0.129622 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65164 dup_rows=23782 dup_rows_remote=20802 dedup_ratio=0.241923 router_tokens_per_expert=[808, 782, 785, 720, 731, 751, 754, 792, 809, 834, 753, 742, 783, 746, 739, 721, 775, 788, 806, 792, 776, 746, 760, 722, 801, 741, 774, 746, 782, 743, 782, 800, 809, 751, 749, 727, 781, 755, 770, 779, 774, 738, 816, 757, 749, 797, 806, 761, 730, 799, 768, 728, 748, 802, 718, 746, 832, 768, 797, 782, 762, 759, 751, 759] payload_rows_per_dst=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] dup_rows_per_dst=[2930, 3006, 2984, 2980, 2902, 3000, 2930, 3050] combine_rows_per_src=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344280 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584280 ingress_bytes=370049024 link_bytes=370049024 token_bytes=532824280 weight_movement_bytes=103809024 total_nvlink_bytes=636633304 weight_to_token_ratio=0.194828 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392464686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64572 dup_rows=23910 dup_rows_remote=20526 dedup_ratio=0.243225 router_tokens_per_expert=[757, 802, 728, 801, 777, 780, 738, 765, 759, 772, 783, 806, 752, 823, 773, 730, 771, 740, 794, 760, 766, 774, 787, 737, 770, 821, 713, 807, 780, 805, 795, 719, 788, 780, 791, 781, 748, 747, 764, 763, 732, 751, 782, 777, 772, 773, 788, 712, 785, 750, 784, 701, 788, 759, 829, 729, 789, 731, 768, 745, 754, 780, 759, 767] payload_rows_per_dst=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] dup_rows_per_dst=[2830, 2564, 3064, 3078, 3036, 2898, 3056, 3384] combine_rows_per_src=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343904 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903392 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462880 weight_movement_bytes=103809024 total_nvlink_bytes=637271904 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=343520 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265485792 ingress_bytes=334348288 link_bytes=334348288 token_bytes=530628064 weight_movement_bytes=69206016 total_nvlink_bytes=599834080 weight_to_token_ratio=0.130423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343864 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255608 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167352 weight_movement_bytes=34603008 total_nvlink_bytes=568770360 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65164 dup_rows=23812 dup_rows_remote=20910 dedup_ratio=0.242228 router_tokens_per_expert=[751, 743, 805, 776, 746, 778, 787, 709, 782, 755, 812, 735, 740, 818, 753, 720, 785, 757, 807, 735, 752, 804, 766, 710, 787, 735, 743, 760, 759, 781, 782, 767, 722, 793, 768, 803, 795, 739, 774, 750, 764, 795, 802, 765, 780, 840, 795, 737, 785, 798, 737, 769, 772, 790, 776, 691, 760, 760, 770, 822, 760, 733, 785, 782] payload_rows_per_dst=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] dup_rows_per_dst=[2902, 2950, 3028, 2974, 2992, 3026, 3002, 2938] combine_rows_per_src=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264486912 dispatch_meta_bytes=340392 combine_payload_bytes=264486912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264827304 ingress_bytes=299089920 link_bytes=299089920 token_bytes=529314216 weight_movement_bytes=34603008 total_nvlink_bytes=563917224 weight_to_token_ratio=0.065373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125451290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65178 dup_rows=23934 dup_rows_remote=20990 dedup_ratio=0.243469 router_tokens_per_expert=[755, 805, 754, 771, 785, 779, 731, 776, 780, 714, 725, 768, 789, 760, 739, 793, 780, 776, 837, 768, 768, 727, 786, 749, 780, 766, 756, 746, 702, 777, 738, 797, 722, 775, 830, 778, 751, 760, 768, 786, 815, 760, 752, 782, 767, 736, 774, 738, 755, 784, 780, 755, 791, 763, 772, 806, 756, 804, 773, 788, 767, 769, 744, 774] payload_rows_per_dst=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] dup_rows_per_dst=[3006, 2944, 3052, 2908, 3010, 2932, 3092, 2990] combine_rows_per_src=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64880 dup_rows=24014 dup_rows_remote=20936 dedup_ratio=0.244283 router_tokens_per_expert=[772, 778, 745, 795, 744, 796, 755, 792, 756, 725, 799, 750, 750, 789, 764, 757, 825, 771, 757, 791, 828, 743, 764, 749, 747, 819, 755, 742, 756, 757, 739, 785, 773, 720, 783, 801, 758, 744, 764, 762, 738, 819, 779, 782, 785, 773, 807, 761, 761, 752, 765, 756, 731, 774, 754, 720, 777, 768, 782, 796, 751, 799, 772, 750] payload_rows_per_dst=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] dup_rows_per_dst=[3004, 2960, 3142, 2916, 3018, 3078, 2876, 3020] combine_rows_per_src=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344296 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267256040 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167784 weight_movement_bytes=69206016 total_nvlink_bytes=603373800 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65074 dup_rows=24028 dup_rows_remote=21164 dedup_ratio=0.244425 router_tokens_per_expert=[758, 788, 804, 751, 787, 774, 782, 759, 785, 785, 761, 735, 781, 768, 733, 727, 768, 750, 790, 741, 775, 799, 758, 764, 770, 852, 801, 784, 701, 756, 728, 781, 757, 750, 780, 776, 776, 807, 807, 819, 780, 777, 744, 788, 749, 759, 757, 742, 716, 807, 783, 724, 813, 745, 709, 736, 763, 742, 762, 743, 770, 792, 773, 810] payload_rows_per_dst=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] dup_rows_per_dst=[3162, 2914, 3032, 3040, 3092, 2908, 2864, 3016] combine_rows_per_src=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65024 dup_rows=24012 dup_rows_remote=21050 dedup_ratio=0.244263 router_tokens_per_expert=[756, 745, 750, 816, 775, 777, 785, 787, 763, 759, 735, 783, 818, 796, 812, 750, 754, 759, 783, 746, 790, 761, 773, 749, 795, 783, 794, 748, 755, 767, 764, 783, 773, 753, 769, 755, 771, 769, 734, 730, 749, 759, 742, 781, 803, 736, 761, 763, 773, 763, 807, 749, 773, 761, 812, 716, 704, 744, 727, 753, 812, 794, 780, 825] payload_rows_per_dst=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] dup_rows_per_dst=[2940, 3150, 2962, 3044, 2912, 2994, 3060, 2950] combine_rows_per_src=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344672 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313760 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282848 weight_movement_bytes=34603008 total_nvlink_bytes=568885856 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343264 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266091744 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840224 weight_movement_bytes=103809024 total_nvlink_bytes=635649248 weight_to_token_ratio=0.195188 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64742 dup_rows=24044 dup_rows_remote=20990 dedup_ratio=0.244588 router_tokens_per_expert=[726, 792, 757, 778, 735, 788, 761, 740, 728, 765, 815, 730, 725, 828, 799, 726, 787, 790, 730, 767, 707, 719, 814, 748, 771, 774, 766, 775, 794, 806, 746, 777, 788, 796, 742, 765, 779, 803, 803, 810, 792, 787, 780, 732, 757, 762, 780, 779, 784, 819, 778, 824, 753, 773, 761, 771, 745, 759, 785, 715, 751, 766, 674, 775] payload_rows_per_dst=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] dup_rows_per_dst=[2936, 3016, 2994, 3090, 3054, 2958, 3108, 2888] combine_rows_per_src=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65154 dup_rows=23844 dup_rows_remote=20824 dedup_ratio=0.242554 router_tokens_per_expert=[804, 740, 772, 793, 749, 757, 800, 720, 823, 754, 754, 737, 788, 758, 793, 795, 789, 752, 772, 728, 754, 781, 742, 715, 771, 788, 766, 744, 758, 802, 728, 806, 775, 740, 785, 778, 817, 729, 768, 720, 786, 744, 751, 749, 749, 734, 766, 788, 792, 730, 803, 806, 782, 790, 789, 750, 747, 756, 799, 762, 789, 740, 812, 793] payload_rows_per_dst=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] dup_rows_per_dst=[2940, 3020, 2874, 3020, 2914, 2930, 3108, 3038] combine_rows_per_src=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344952 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888056 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431160 weight_movement_bytes=34603008 total_nvlink_bytes=568034168 weight_to_token_ratio=0.064869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64728 dup_rows=24132 dup_rows_remote=20932 dedup_ratio=0.245483 router_tokens_per_expert=[718, 788, 768, 768, 775, 790, 804, 772, 755, 713, 713, 775, 774, 782, 745, 766, 777, 811, 813, 763, 761, 773, 722, 738, 751, 751, 745, 759, 798, 798, 724, 739, 747, 783, 815, 749, 750, 783, 757, 773, 795, 767, 761, 769, 777, 808, 788, 746, 750, 716, 781, 758, 774, 762, 761, 797, 788, 782, 818, 778, 827, 736, 755, 772] payload_rows_per_dst=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] dup_rows_per_dst=[2702, 3294, 2888, 2986, 2896, 3172, 2994, 3200] combine_rows_per_src=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344296 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682600 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020904 weight_movement_bytes=34603008 total_nvlink_bytes=567623912 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=342928 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265526160 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530709392 weight_movement_bytes=34603008 total_nvlink_bytes=565312400 weight_to_token_ratio=0.065201 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65054 dup_rows=23792 dup_rows_remote=20808 dedup_ratio=0.242025 router_tokens_per_expert=[775, 756, 812, 791, 775, 763, 744, 805, 751, 776, 760, 795, 728, 805, 769, 755, 758, 782, 745, 770, 800, 751, 776, 799, 774, 771, 718, 810, 751, 750, 792, 764, 822, 762, 749, 773, 728, 776, 757, 750, 757, 777, 783, 763, 802, 731, 774, 753, 736, 787, 775, 767, 776, 764, 789, 810, 732, 799, 770, 737, 720, 739, 774, 749] payload_rows_per_dst=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] dup_rows_per_dst=[2984, 3172, 2820, 3046, 2934, 2944, 3092, 2800] combine_rows_per_src=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343912 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267214696 ingress_bytes=336076800 link_bytes=336076800 token_bytes=534085480 weight_movement_bytes=69206016 total_nvlink_bytes=603291496 weight_to_token_ratio=0.129579 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264629942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342640 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468528 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594416 weight_movement_bytes=34603008 total_nvlink_bytes=565197424 weight_to_token_ratio=0.065216 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64840 dup_rows=24060 dup_rows_remote=21118 dedup_ratio=0.244751 router_tokens_per_expert=[768, 786, 772, 775, 796, 784, 784, 768, 762, 813, 818, 748, 734, 735, 811, 752, 821, 761, 750, 770, 800, 785, 777, 778, 791, 766, 743, 766, 790, 715, 753, 772, 794, 768, 751, 753, 785, 803, 777, 703, 725, 762, 795, 748, 761, 768, 755, 773, 732, 773, 705, 771, 751, 769, 776, 796, 743, 829, 726, 761, 713, 773, 790, 779] payload_rows_per_dst=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] dup_rows_per_dst=[3174, 2942, 3118, 3028, 2944, 2950, 2898, 3006] combine_rows_per_src=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64900 dup_rows=24112 dup_rows_remote=21270 dedup_ratio=0.245280 router_tokens_per_expert=[774, 785, 722, 784, 755, 771, 764, 771, 801, 758, 792, 831, 805, 812, 761, 804, 772, 791, 804, 787, 765, 800, 765, 772, 788, 762, 761, 768, 757, 755, 737, 766, 763, 799, 720, 774, 742, 716, 752, 761, 793, 730, 773, 778, 749, 735, 743, 766, 790, 771, 793, 739, 779, 792, 767, 755, 760, 732, 772, 758, 771, 767, 743, 729] payload_rows_per_dst=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] dup_rows_per_dst=[3102, 3214, 3082, 2886, 2916, 2842, 3092, 2978] combine_rows_per_src=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64886 dup_rows=24132 dup_rows_remote=21122 dedup_ratio=0.245483 router_tokens_per_expert=[802, 747, 782, 767, 740, 755, 741, 750, 763, 756, 746, 789, 824, 782, 764, 772, 777, 776, 781, 766, 801, 723, 795, 767, 788, 787, 722, 746, 785, 801, 785, 775, 776, 775, 730, 777, 715, 767, 773, 798, 778, 775, 772, 777, 774, 788, 785, 746, 775, 787, 726, 718, 787, 788, 826, 741, 776, 779, 726, 753, 767, 729, 781, 732] payload_rows_per_dst=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] dup_rows_per_dst=[2990, 3094, 3092, 3084, 2974, 2984, 3010, 2904] combine_rows_per_src=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343448 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266804632 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265816 weight_movement_bytes=69206016 total_nvlink_bytes=602471832 weight_to_token_ratio=0.129778 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74680 payload_rows_remote=65424 dup_rows=23624 dup_rows_remote=20754 dedup_ratio=0.240316 router_tokens_per_expert=[784, 781, 785, 803, 766, 760, 678, 758, 792, 783, 816, 752, 806, 785, 718, 738, 751, 744, 758, 768, 747, 802, 763, 730, 747, 762, 730, 789, 763, 712, 816, 767, 792, 788, 752, 776, 819, 831, 794, 764, 729, 712, 763, 844, 787, 762, 777, 808, 757, 745, 789, 740, 754, 774, 770, 751, 801, 753, 766, 762, 777, 776, 753, 732] payload_rows_per_dst=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] dup_rows_per_dst=[2986, 3010, 2870, 2836, 3106, 3054, 2886, 2876] combine_rows_per_src=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64846 dup_rows=24102 dup_rows_remote=21074 dedup_ratio=0.245178 router_tokens_per_expert=[787, 771, 758, 740, 776, 769, 733, 755, 791, 757, 768, 778, 704, 817, 771, 754, 796, 764, 785, 839, 766, 741, 738, 758, 776, 755, 735, 748, 806, 774, 784, 724, 793, 762, 827, 795, 735, 749, 738, 793, 747, 825, 700, 800, 777, 784, 712, 768, 811, 753, 805, 787, 762, 734, 761, 796, 797, 733, 796, 749, 779, 733, 756, 777] payload_rows_per_dst=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] dup_rows_per_dst=[3092, 2960, 3044, 2986, 3028, 2998, 3004, 2990] combine_rows_per_src=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343832 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265928472 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531513112 weight_movement_bytes=69206016 total_nvlink_bytes=600719128 weight_to_token_ratio=0.130206 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344680 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266175080 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005480 weight_movement_bytes=34603008 total_nvlink_bytes=566608488 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344032 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117088 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890144 weight_movement_bytes=34603008 total_nvlink_bytes=566493152 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64588 dup_rows=24068 dup_rows_remote=20734 dedup_ratio=0.244832 router_tokens_per_expert=[780, 762, 772, 717, 732, 810, 777, 742, 751, 804, 769, 751, 791, 775, 768, 770, 802, 813, 757, 808, 755, 751, 771, 755, 798, 793, 741, 739, 763, 753, 781, 764, 764, 736, 805, 754, 756, 752, 764, 787, 747, 744, 789, 774, 743, 755, 769, 757, 785, 754, 709, 740, 760, 779, 777, 769, 853, 785, 753, 804, 792, 769, 782, 730] payload_rows_per_dst=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] dup_rows_per_dst=[3058, 2800, 3308, 2958, 2868, 2926, 2816, 3334] combine_rows_per_src=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64940 dup_rows=24004 dup_rows_remote=21006 dedup_ratio=0.244181 router_tokens_per_expert=[765, 725, 774, 776, 805, 793, 749, 769, 739, 766, 770, 801, 778, 730, 749, 739, 715, 778, 765, 748, 763, 777, 758, 749, 714, 768, 712, 779, 738, 803, 820, 845, 797, 776, 730, 792, 805, 746, 755, 771, 756, 783, 755, 787, 748, 748, 763, 792, 768, 793, 794, 807, 729, 811, 733, 775, 767, 783, 784, 750, 781, 764, 803, 746] payload_rows_per_dst=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] dup_rows_per_dst=[3002, 2968, 2970, 2998, 2962, 2970, 3130, 3004] combine_rows_per_src=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344712 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321416 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298120 weight_movement_bytes=69206016 total_nvlink_bytes=605504136 weight_to_token_ratio=0.129044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343680 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531562112 weight_movement_bytes=103809024 total_nvlink_bytes=635371136 weight_to_token_ratio=0.195290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65160 dup_rows=23744 dup_rows_remote=20700 dedup_ratio=0.241536 router_tokens_per_expert=[794, 777, 756, 787, 748, 757, 796, 806, 775, 763, 722, 758, 738, 796, 764, 766, 763, 770, 741, 773, 800, 760, 767, 742, 781, 752, 781, 773, 764, 763, 823, 734, 790, 749, 802, 833, 761, 763, 772, 758, 805, 790, 765, 783, 775, 725, 778, 747, 735, 753, 782, 728, 832, 767, 760, 776, 764, 742, 773, 754, 762, 776, 727, 735] payload_rows_per_dst=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] dup_rows_per_dst=[3044, 3082, 2886, 2836, 3092, 3348, 2674, 2782] combine_rows_per_src=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341288 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264893736 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446184 weight_movement_bytes=34603008 total_nvlink_bytes=564049192 weight_to_token_ratio=0.065357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343784 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338024 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332264 weight_movement_bytes=103809024 total_nvlink_bytes=636141288 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65162 dup_rows=23878 dup_rows_remote=20830 dedup_ratio=0.242900 router_tokens_per_expert=[763, 858, 788, 731, 755, 812, 793, 803, 762, 749, 803, 732, 774, 766, 797, 773, 777, 752, 773, 787, 789, 757, 769, 781, 753, 809, 768, 786, 790, 805, 744, 765, 797, 708, 778, 760, 750, 735, 805, 794, 713, 767, 748, 761, 720, 764, 778, 744, 860, 763, 771, 698, 743, 744, 761, 775, 782, 709, 774, 773, 754, 755, 748, 756] payload_rows_per_dst=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] dup_rows_per_dst=[3088, 3048, 3022, 3036, 2952, 2850, 2940, 2942] combine_rows_per_src=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343440 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267238800 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534134160 weight_movement_bytes=138412032 total_nvlink_bytes=672546192 weight_to_token_ratio=0.259133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65122 dup_rows=23768 dup_rows_remote=20858 dedup_ratio=0.241781 router_tokens_per_expert=[739, 751, 755, 834, 738, 776, 760, 801, 729, 789, 796, 748, 753, 760, 767, 712, 767, 783, 796, 814, 791, 734, 748, 750, 789, 755, 775, 762, 793, 766, 773, 750, 774, 776, 765, 753, 722, 782, 772, 764, 747, 738, 824, 809, 788, 751, 745, 760, 727, 842, 761, 788, 783, 765, 776, 797, 778, 730, 756, 751, 752, 795, 766, 761] payload_rows_per_dst=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] dup_rows_per_dst=[3030, 2878, 3018, 2946, 3062, 2910, 3070, 2854] combine_rows_per_src=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20774 dedup_ratio=0.242635 router_tokens_per_expert=[815, 798, 759, 794, 776, 706, 765, 766, 735, 762, 795, 764, 762, 783, 758, 744, 757, 770, 751, 733, 760, 770, 811, 804, 817, 784, 784, 762, 758, 830, 722, 772, 763, 729, 747, 809, 765, 769, 786, 757, 770, 755, 775, 743, 713, 774, 720, 831, 785, 763, 752, 783, 766, 760, 778, 829, 744, 786, 790, 727, 763, 768, 751, 734] payload_rows_per_dst=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] dup_rows_per_dst=[2996, 2952, 2932, 3094, 2996, 2904, 3078, 2900] combine_rows_per_src=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65268 dup_rows=23788 dup_rows_remote=20894 dedup_ratio=0.241984 router_tokens_per_expert=[706, 814, 802, 741, 775, 778, 765, 768, 753, 746, 785, 733, 747, 773, 764, 784, 759, 755, 763, 780, 747, 779, 742, 746, 786, 785, 770, 788, 756, 778, 746, 754, 772, 781, 801, 796, 719, 817, 766, 783, 768, 736, 790, 778, 789, 802, 759, 811, 751, 780, 773, 804, 767, 725, 808, 753, 716, 762, 727, 757, 771, 778, 774, 770] payload_rows_per_dst=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] dup_rows_per_dst=[3062, 2916, 2894, 2924, 3076, 3078, 3002, 2836] combine_rows_per_src=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65160 dup_rows=23872 dup_rows_remote=20754 dedup_ratio=0.242839 router_tokens_per_expert=[742, 814, 736, 751, 704, 724, 799, 792, 778, 773, 775, 753, 815, 754, 767, 772, 771, 744, 777, 753, 713, 740, 752, 750, 781, 778, 788, 757, 736, 791, 796, 767, 759, 734, 798, 762, 760, 821, 779, 782, 717, 803, 804, 784, 770, 754, 739, 784, 781, 793, 830, 761, 735, 794, 748, 767, 756, 784, 751, 759, 767, 752, 819, 762] payload_rows_per_dst=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] dup_rows_per_dst=[2928, 3012, 2768, 2984, 3118, 3052, 3008, 3002] combine_rows_per_src=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343968 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247520 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151072 weight_movement_bytes=34603008 total_nvlink_bytes=568754080 weight_to_token_ratio=0.064781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343920 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083632 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823344 weight_movement_bytes=34603008 total_nvlink_bytes=568426352 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64612 dup_rows=24028 dup_rows_remote=20504 dedup_ratio=0.244425 router_tokens_per_expert=[773, 788, 750, 746, 779, 801, 741, 761, 811, 786, 767, 765, 798, 754, 793, 783, 764, 784, 791, 781, 802, 794, 742, 810, 758, 742, 716, 761, 753, 707, 748, 772, 779, 760, 762, 729, 787, 804, 748, 731, 746, 777, 749, 786, 809, 759, 777, 804, 800, 749, 771, 765, 762, 744, 759, 773, 746, 790, 746, 797, 772, 732, 748, 770] payload_rows_per_dst=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] dup_rows_per_dst=[2698, 3058, 2726, 2834, 3004, 3052, 3132, 3524] combine_rows_per_src=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343488 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266984896 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626304 weight_movement_bytes=69206016 total_nvlink_bytes=602832320 weight_to_token_ratio=0.129690 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=344648 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267682376 ingress_bytes=301940736 link_bytes=301940736 token_bytes=535020104 weight_movement_bytes=34603008 total_nvlink_bytes=569623112 weight_to_token_ratio=0.064676 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65000 dup_rows=23910 dup_rows_remote=20928 dedup_ratio=0.243225 router_tokens_per_expert=[782, 759, 797, 773, 777, 738, 747, 758, 780, 774, 728, 816, 800, 765, 783, 793, 763, 756, 794, 777, 742, 819, 741, 758, 771, 772, 788, 768, 755, 825, 743, 766, 755, 790, 730, 730, 732, 755, 797, 768, 713, 791, 789, 787, 742, 765, 789, 780, 773, 754, 734, 758, 810, 792, 770, 802, 757, 752, 776, 780, 734, 690, 705, 844] payload_rows_per_dst=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] dup_rows_per_dst=[2994, 3054, 3062, 2982, 2912, 2944, 3110, 2852] combine_rows_per_src=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343656 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239016 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134376 weight_movement_bytes=69206016 total_nvlink_bytes=603340392 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64744 dup_rows=24170 dup_rows_remote=21004 dedup_ratio=0.245870 router_tokens_per_expert=[777, 772, 775, 790, 768, 750, 787, 751, 766, 760, 749, 755, 713, 755, 793, 761, 770, 823, 772, 778, 781, 778, 763, 751, 757, 818, 770, 754, 803, 785, 755, 785, 766, 760, 736, 773, 739, 745, 790, 800, 798, 750, 742, 774, 778, 728, 761, 765, 755, 765, 738, 729, 761, 789, 762, 731, 737, 799, 807, 801, 770, 728, 838, 772] payload_rows_per_dst=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] dup_rows_per_dst=[3166, 2962, 3102, 3128, 2902, 2954, 2882, 3074] combine_rows_per_src=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=340464 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264991216 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529641968 weight_movement_bytes=34603008 total_nvlink_bytes=564244976 weight_to_token_ratio=0.065333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343712 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583712 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823712 weight_movement_bytes=69206016 total_nvlink_bytes=602029728 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=342992 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265534416 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530725840 weight_movement_bytes=69206016 total_nvlink_bytes=599931856 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65158 dup_rows=23830 dup_rows_remote=20746 dedup_ratio=0.242411 router_tokens_per_expert=[752, 770, 790, 749, 782, 764, 720, 729, 741, 739, 785, 782, 804, 792, 770, 787, 748, 792, 780, 778, 801, 772, 767, 749, 772, 766, 780, 729, 775, 752, 719, 774, 735, 739, 756, 742, 724, 835, 776, 773, 742, 808, 758, 766, 749, 780, 761, 780, 810, 814, 755, 759, 812, 739, 788, 728, 757, 790, 757, 780, 794, 759, 749, 827] payload_rows_per_dst=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] dup_rows_per_dst=[3004, 3084, 3006, 2884, 2854, 2886, 3016, 3096] combine_rows_per_src=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64982 dup_rows=23946 dup_rows_remote=21008 dedup_ratio=0.243591 router_tokens_per_expert=[836, 793, 743, 786, 756, 790, 770, 796, 769, 767, 715, 793, 773, 812, 781, 795, 737, 756, 796, 769, 778, 749, 789, 729, 756, 737, 765, 785, 762, 747, 791, 769, 802, 850, 722, 795, 774, 774, 786, 767, 755, 748, 756, 792, 797, 792, 745, 772, 771, 702, 769, 783, 753, 778, 759, 723, 807, 742, 717, 764, 771, 713, 729, 754] payload_rows_per_dst=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] dup_rows_per_dst=[3120, 3148, 2874, 3028, 3182, 2938, 2852, 2804] combine_rows_per_src=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64886 dup_rows=24122 dup_rows_remote=21096 dedup_ratio=0.245382 router_tokens_per_expert=[767, 775, 765, 754, 776, 809, 784, 788, 736, 745, 801, 800, 748, 764, 751, 776, 762, 787, 766, 752, 807, 770, 750, 769, 778, 747, 754, 800, 777, 787, 763, 752, 762, 795, 781, 750, 809, 759, 787, 760, 769, 783, 764, 757, 740, 766, 773, 756, 782, 831, 762, 735, 714, 773, 790, 774, 772, 738, 753, 737, 773, 750, 772, 725] payload_rows_per_dst=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] dup_rows_per_dst=[3104, 2910, 3062, 2982, 3074, 2972, 3026, 2992] combine_rows_per_src=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65210 dup_rows=23824 dup_rows_remote=20826 dedup_ratio=0.242350 router_tokens_per_expert=[752, 768, 778, 815, 795, 775, 772, 744, 732, 769, 738, 748, 770, 750, 723, 774, 753, 706, 779, 791, 795, 740, 790, 780, 790, 773, 795, 789, 757, 786, 794, 743, 782, 808, 741, 780, 769, 765, 757, 814, 766, 796, 796, 742, 754, 790, 738, 788, 729, 779, 736, 792, 763, 749, 752, 783, 723, 757, 761, 771, 759, 755, 762, 831] payload_rows_per_dst=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] dup_rows_per_dst=[2946, 2880, 2998, 3012, 3068, 3052, 2918, 2950] combine_rows_per_src=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64676 dup_rows=24190 dup_rows_remote=20990 dedup_ratio=0.246073 router_tokens_per_expert=[807, 797, 787, 764, 775, 775, 784, 779, 737, 835, 746, 776, 733, 768, 768, 752, 787, 755, 825, 763, 762, 747, 752, 770, 786, 795, 759, 766, 745, 785, 788, 704, 776, 795, 806, 772, 802, 820, 768, 780, 773, 777, 819, 737, 755, 772, 743, 749, 731, 752, 774, 739, 710, 763, 771, 776, 709, 755, 757, 775, 756, 737, 775, 756] payload_rows_per_dst=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] dup_rows_per_dst=[3172, 3010, 3090, 3016, 3200, 2982, 2862, 2858] combine_rows_per_src=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343616 combine_payload_bytes=266887168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267230784 ingress_bytes=370696192 link_bytes=370696192 token_bytes=534117952 weight_movement_bytes=103809024 total_nvlink_bytes=637926976 weight_to_token_ratio=0.194356 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394899927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343960 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510232 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676504 weight_movement_bytes=69206016 total_nvlink_bytes=601882520 weight_to_token_ratio=0.129921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65086 dup_rows=23754 dup_rows_remote=20644 dedup_ratio=0.241638 router_tokens_per_expert=[752, 801, 839, 772, 760, 723, 783, 817, 760, 722, 769, 750, 770, 769, 794, 731, 720, 763, 807, 789, 718, 805, 747, 780, 769, 807, 717, 765, 796, 764, 752, 727, 799, 781, 775, 729, 748, 830, 753, 742, 766, 766, 782, 750, 769, 791, 728, 775, 786, 729, 747, 787, 757, 768, 744, 725, 808, 776, 761, 790, 776, 807, 781, 788] payload_rows_per_dst=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] dup_rows_per_dst=[3020, 2868, 3010, 2916, 2918, 2962, 2950, 3110] combine_rows_per_src=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343928 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116984 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890040 weight_movement_bytes=34603008 total_nvlink_bytes=566493048 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65206 dup_rows=23996 dup_rows_remote=21090 dedup_ratio=0.244100 router_tokens_per_expert=[776, 741, 803, 745, 711, 762, 795, 743, 799, 751, 779, 784, 773, 768, 748, 766, 755, 770, 799, 809, 795, 767, 795, 804, 745, 766, 752, 777, 731, 715, 731, 787, 852, 801, 806, 799, 780, 800, 749, 749, 735, 750, 779, 757, 767, 752, 774, 797, 746, 767, 734, 771, 779, 750, 755, 795, 747, 754, 736, 752, 803, 754, 790, 730] payload_rows_per_dst=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] dup_rows_per_dst=[2866, 3074, 3102, 2906, 3200, 2924, 3014, 2910] combine_rows_per_src=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344144 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444304 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544464 weight_movement_bytes=69206016 total_nvlink_bytes=603750480 weight_to_token_ratio=0.129467 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=342664 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265255560 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530168456 weight_movement_bytes=69206016 total_nvlink_bytes=599374472 weight_to_token_ratio=0.130536 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20914 dedup_ratio=0.243490 router_tokens_per_expert=[775, 780, 762, 732, 773, 767, 774, 778, 718, 778, 761, 759, 781, 790, 802, 758, 749, 761, 779, 735, 767, 792, 780, 806, 780, 787, 767, 768, 781, 754, 763, 762, 814, 752, 749, 734, 801, 773, 752, 786, 740, 792, 751, 742, 772, 757, 750, 806, 823, 741, 763, 805, 755, 787, 764, 782, 729, 775, 766, 759, 737, 742, 745, 789] payload_rows_per_dst=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] dup_rows_per_dst=[3022, 2974, 3014, 3036, 2974, 2886, 3062, 2968] combine_rows_per_src=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=342920 combine_payload_bytes=266592256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266935176 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533527432 weight_movement_bytes=69206016 total_nvlink_bytes=602733448 weight_to_token_ratio=0.129714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=345184 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267428960 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534512736 weight_movement_bytes=34603008 total_nvlink_bytes=569115744 weight_to_token_ratio=0.064737 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343480 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403256 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463032 weight_movement_bytes=34603008 total_nvlink_bytes=567066040 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64996 dup_rows=24036 dup_rows_remote=20970 dedup_ratio=0.244507 router_tokens_per_expert=[735, 767, 754, 792, 802, 801, 787, 746, 761, 806, 793, 725, 773, 770, 760, 781, 748, 799, 816, 813, 812, 790, 765, 794, 771, 767, 805, 734, 752, 727, 768, 734, 728, 804, 763, 744, 725, 722, 783, 732, 786, 772, 744, 787, 733, 768, 771, 787, 795, 757, 742, 820, 720, 798, 776, 813, 761, 771, 707, 802, 737, 770, 758, 728] payload_rows_per_dst=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] dup_rows_per_dst=[3004, 3066, 3206, 2950, 2874, 2960, 3070, 2906] combine_rows_per_src=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64940 dup_rows=24176 dup_rows_remote=21174 dedup_ratio=0.245931 router_tokens_per_expert=[775, 745, 749, 771, 774, 757, 800, 746, 790, 755, 786, 783, 776, 795, 822, 750, 798, 785, 751, 745, 817, 775, 779, 752, 745, 777, 778, 813, 792, 794, 718, 792, 732, 749, 778, 786, 763, 769, 782, 771, 750, 763, 754, 745, 801, 789, 732, 761, 786, 752, 764, 781, 740, 796, 752, 720, 757, 716, 762, 803, 781, 731, 761, 740] payload_rows_per_dst=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] dup_rows_per_dst=[3020, 3076, 3018, 3116, 3034, 3002, 2984, 2926] combine_rows_per_src=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64934 dup_rows=24142 dup_rows_remote=21056 dedup_ratio=0.245585 router_tokens_per_expert=[769, 755, 747, 758, 774, 804, 788, 780, 781, 744, 782, 744, 764, 755, 792, 796, 710, 770, 740, 770, 757, 693, 795, 783, 781, 788, 754, 752, 760, 773, 740, 804, 775, 781, 750, 783, 732, 790, 740, 784, 776, 792, 759, 762, 797, 800, 752, 753, 739, 757, 797, 782, 756, 767, 796, 763, 788, 785, 792, 818, 763, 761, 707, 752] payload_rows_per_dst=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] dup_rows_per_dst=[3014, 3074, 2856, 3014, 2964, 3020, 3086, 3114] combine_rows_per_src=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64840 dup_rows=24214 dup_rows_remote=21156 dedup_ratio=0.246318 router_tokens_per_expert=[767, 730, 757, 758, 772, 796, 748, 774, 754, 774, 755, 716, 753, 797, 796, 747, 790, 766, 750, 802, 758, 783, 755, 750, 793, 747, 778, 777, 776, 769, 753, 734, 840, 802, 793, 762, 789, 786, 733, 773, 753, 782, 744, 782, 790, 728, 745, 759, 777, 807, 729, 791, 755, 746, 802, 731, 778, 736, 730, 786, 838, 759, 766, 785] payload_rows_per_dst=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] dup_rows_per_dst=[3054, 3018, 3058, 3012, 3196, 2882, 3004, 2990] combine_rows_per_src=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65134 dup_rows=24044 dup_rows_remote=20982 dedup_ratio=0.244588 router_tokens_per_expert=[773, 772, 758, 713, 845, 791, 782, 763, 763, 796, 761, 777, 759, 759, 749, 791, 744, 771, 800, 785, 728, 790, 747, 798, 810, 732, 775, 732, 802, 748, 797, 736, 776, 755, 779, 782, 752, 714, 763, 773, 713, 796, 764, 815, 821, 734, 756, 760, 785, 742, 755, 777, 774, 747, 739, 768, 799, 693, 781, 827, 780, 775, 721, 789] payload_rows_per_dst=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] dup_rows_per_dst=[3104, 3032, 3016, 2958, 3062, 2990, 2846, 3036] combine_rows_per_src=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343864 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567480 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791096 weight_movement_bytes=103809024 total_nvlink_bytes=636600120 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65048 dup_rows=23742 dup_rows_remote=20616 dedup_ratio=0.241516 router_tokens_per_expert=[779, 735, 774, 769, 751, 727, 740, 773, 783, 783, 728, 743, 769, 762, 776, 760, 801, 781, 781, 767, 781, 743, 801, 716, 777, 749, 818, 763, 746, 793, 794, 780, 773, 774, 749, 789, 785, 740, 751, 742, 764, 750, 766, 810, 792, 752, 748, 817, 714, 727, 763, 787, 726, 766, 740, 764, 785, 782, 822, 828, 809, 770, 760, 764] payload_rows_per_dst=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] dup_rows_per_dst=[2858, 2944, 3012, 2880, 2966, 3016, 2940, 3126] combine_rows_per_src=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344456 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338696 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332936 weight_movement_bytes=69206016 total_nvlink_bytes=601538952 weight_to_token_ratio=0.130005 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343960 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313624 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283288 weight_movement_bytes=69206016 total_nvlink_bytes=601489304 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343984 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928624 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513264 weight_movement_bytes=34603008 total_nvlink_bytes=566116272 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=65008 dup_rows=24058 dup_rows_remote=21102 dedup_ratio=0.244731 router_tokens_per_expert=[734, 780, 798, 753, 784, 757, 775, 750, 747, 835, 774, 769, 746, 784, 776, 791, 738, 785, 752, 796, 720, 758, 816, 774, 783, 743, 785, 752, 690, 790, 778, 776, 786, 736, 757, 748, 785, 833, 734, 751, 799, 801, 730, 774, 799, 745, 758, 750, 747, 758, 768, 825, 777, 756, 690, 790, 760, 761, 752, 758, 774, 792, 796, 773] payload_rows_per_dst=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] dup_rows_per_dst=[2966, 3172, 2996, 2956, 3036, 2974, 2976, 2982] combine_rows_per_src=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344464 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267133328 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533922192 weight_movement_bytes=103809024 total_nvlink_bytes=637731216 weight_to_token_ratio=0.194427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65012 dup_rows=23950 dup_rows_remote=20898 dedup_ratio=0.243632 router_tokens_per_expert=[764, 829, 747, 835, 747, 735, 809, 721, 736, 762, 759, 739, 749, 765, 791, 791, 777, 796, 781, 723, 739, 773, 769, 732, 781, 774, 762, 773, 792, 756, 766, 793, 778, 808, 793, 784, 757, 700, 791, 740, 795, 794, 829, 754, 786, 812, 734, 750, 774, 746, 779, 783, 746, 787, 759, 755, 774, 815, 684, 788, 755, 737, 762, 737] payload_rows_per_dst=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] dup_rows_per_dst=[3052, 2912, 3078, 3206, 2852, 3024, 2934, 2892] combine_rows_per_src=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=342656 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266779264 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533215872 weight_movement_bytes=138412032 total_nvlink_bytes=671627904 weight_to_token_ratio=0.259580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344440 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617208 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889976 weight_movement_bytes=34603008 total_nvlink_bytes=567492984 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343640 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266632792 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532921944 weight_movement_bytes=69206016 total_nvlink_bytes=602127960 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64924 dup_rows=23986 dup_rows_remote=20998 dedup_ratio=0.243998 router_tokens_per_expert=[773, 767, 735, 741, 768, 791, 776, 830, 753, 760, 766, 771, 791, 769, 815, 766, 782, 757, 753, 821, 710, 789, 760, 790, 762, 772, 778, 791, 723, 779, 756, 768, 746, 777, 806, 765, 835, 750, 811, 814, 720, 758, 701, 796, 775, 712, 786, 766, 747, 760, 734, 757, 768, 826, 760, 771, 764, 781, 727, 743, 762, 758, 765, 748] payload_rows_per_dst=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] dup_rows_per_dst=[2904, 2988, 3002, 2974, 3192, 2930, 2974, 3022] combine_rows_per_src=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12229498048 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6118692032 ingress_bytes=7494926336 link_bytes=7494926336 total_nvlink_bytes=13613618368 weight_to_token_ratio=0.113179 prefetch_share_of_ingress=0.078487 dedup_ratio=0.244248 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226208887e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65140 dup_rows=23896 dup_rows_remote=20924 dedup_ratio=0.243083 router_tokens_per_expert=[780, 787, 738, 774, 739, 728, 772, 812, 800, 792, 791, 722, 759, 775, 772, 738, 798, 761, 743, 749, 782, 760, 789, 728, 768, 779, 749, 788, 775, 776, 727, 774, 711, 782, 779, 806, 753, 735, 731, 762, 740, 765, 770, 830, 799, 750, 724, 742, 757, 765, 789, 757, 792, 802, 795, 772, 755, 749, 800, 785, 778, 766, 791, 795] payload_rows_per_dst=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] dup_rows_per_dst=[3022, 2960, 3000, 2972, 2822, 2972, 3062, 3086] combine_rows_per_src=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65180 dup_rows=23862 dup_rows_remote=20832 dedup_ratio=0.242737 router_tokens_per_expert=[796, 743, 790, 800, 768, 776, 801, 744, 760, 763, 795, 766, 752, 805, 800, 714, 731, 754, 719, 739, 740, 772, 796, 721, 814, 773, 825, 783, 780, 760, 753, 761, 748, 733, 730, 752, 790, 721, 760, 803, 728, 767, 808, 742, 781, 772, 786, 775, 784, 789, 781, 768, 728, 770, 756, 770, 784, 799, 772, 750, 790, 731, 828, 762] payload_rows_per_dst=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] dup_rows_per_dst=[3032, 3000, 2862, 3050, 2920, 2974, 3030, 2994] combine_rows_per_src=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65098 dup_rows=23990 dup_rows_remote=21080 dedup_ratio=0.244039 router_tokens_per_expert=[773, 770, 790, 802, 717, 755, 769, 744, 754, 780, 742, 805, 785, 753, 725, 760, 783, 777, 761, 764, 756, 726, 762, 734, 801, 796, 741, 768, 736, 777, 778, 783, 764, 778, 772, 758, 731, 768, 747, 773, 776, 756, 766, 795, 758, 774, 775, 741, 747, 797, 774, 782, 816, 769, 760, 790, 738, 787, 760, 796, 822, 729, 811, 775] payload_rows_per_dst=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] dup_rows_per_dst=[2940, 3018, 2910, 3020, 2990, 2912, 3076, 3124] combine_rows_per_src=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65154 dup_rows=23756 dup_rows_remote=20798 dedup_ratio=0.241659 router_tokens_per_expert=[781, 750, 749, 816, 721, 711, 777, 783, 760, 751, 695, 789, 776, 743, 812, 773, 748, 707, 799, 756, 735, 707, 751, 763, 780, 795, 766, 731, 763, 722, 836, 812, 781, 801, 776, 760, 756, 796, 750, 756, 783, 802, 788, 782, 740, 792, 731, 750, 818, 736, 774, 769, 816, 766, 797, 758, 813, 780, 820, 793, 732, 791, 772, 715] payload_rows_per_dst=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] dup_rows_per_dst=[2870, 2992, 2806, 2996, 2958, 3072, 3044, 3018] combine_rows_per_src=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343688 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266272392 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532201096 weight_movement_bytes=103809024 total_nvlink_bytes=636010120 weight_to_token_ratio=0.195056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344256 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157696 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971136 weight_movement_bytes=34603008 total_nvlink_bytes=568574144 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344712 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266986120 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627528 weight_movement_bytes=34603008 total_nvlink_bytes=568230536 weight_to_token_ratio=0.064845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344048 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321328 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298608 weight_movement_bytes=34603008 total_nvlink_bytes=568901616 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343808 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267214592 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085376 weight_movement_bytes=34603008 total_nvlink_bytes=568688384 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64886 dup_rows=24078 dup_rows_remote=21088 dedup_ratio=0.244934 router_tokens_per_expert=[760, 796, 778, 734, 780, 766, 750, 807, 788, 774, 738, 755, 758, 791, 737, 733, 803, 802, 733, 805, 763, 713, 794, 737, 731, 778, 787, 785, 792, 797, 765, 730, 764, 775, 726, 796, 789, 772, 772, 756, 732, 709, 779, 780, 780, 783, 788, 742, 727, 764, 799, 776, 797, 742, 829, 791, 765, 789, 753, 780, 768, 781, 756, 732] payload_rows_per_dst=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] dup_rows_per_dst=[3000, 2934, 3054, 2990, 3072, 2954, 3092, 2982] combine_rows_per_src=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65166 dup_rows=23860 dup_rows_remote=20934 dedup_ratio=0.242716 router_tokens_per_expert=[709, 769, 758, 763, 760, 780, 755, 802, 780, 762, 755, 782, 767, 759, 765, 783, 773, 770, 705, 815, 766, 792, 797, 770, 714, 817, 745, 731, 776, 787, 762, 719, 762, 782, 808, 752, 735, 809, 774, 735, 747, 784, 817, 770, 802, 788, 793, 741, 794, 761, 775, 718, 762, 793, 758, 766, 810, 759, 741, 751, 777, 772, 779, 749] payload_rows_per_dst=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] dup_rows_per_dst=[2926, 3038, 3036, 2916, 2866, 3334, 2930, 2814] combine_rows_per_src=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255248568 prefetch_bytes=726663168 reduce_bytes=484442112 weight_movement_bytes=1211105280 egress_bytes=6131581112 ingress_bytes=7334772736 link_bytes=7334772736 total_nvlink_bytes=13466353848 weight_to_token_ratio=0.098823 prefetch_share_of_ingress=0.099071 dedup_ratio=0.244246 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=14 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200006926e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249490744 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=6128698680 ingress_bytes=7504912384 link_bytes=7504912384 total_nvlink_bytes=13633611064 weight_to_token_ratio=0.112994 prefetch_share_of_ingress=0.092214 dedup_ratio=0.244219 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227842657e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65064 dup_rows=24086 dup_rows_remote=21080 dedup_ratio=0.245015 router_tokens_per_expert=[727, 734, 819, 751, 770, 730, 775, 778, 793, 769, 827, 723, 751, 735, 709, 773, 771, 791, 771, 752, 771, 800, 759, 827, 798, 760, 761, 709, 775, 794, 756, 774, 725, 798, 785, 780, 783, 752, 792, 771, 755, 794, 784, 737, 749, 719, 777, 783, 754, 771, 793, 737, 811, 781, 739, 777, 806, 725, 827, 774, 738, 792, 738, 772] payload_rows_per_dst=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] dup_rows_per_dst=[2910, 3006, 3076, 2988, 3070, 3006, 3008, 3022] combine_rows_per_src=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343896 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116952 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531890008 weight_movement_bytes=103809024 total_nvlink_bytes=635699032 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344400 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267264336 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534184272 weight_movement_bytes=69206016 total_nvlink_bytes=603390288 weight_to_token_ratio=0.129555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64782 dup_rows=24092 dup_rows_remote=21064 dedup_ratio=0.245076 router_tokens_per_expert=[767, 743, 729, 732, 811, 721, 790, 773, 737, 783, 781, 750, 812, 777, 816, 778, 744, 776, 756, 782, 760, 808, 768, 761, 747, 754, 778, 760, 812, 768, 833, 755, 766, 763, 758, 765, 756, 767, 765, 747, 768, 792, 770, 763, 794, 775, 798, 769, 746, 734, 708, 770, 772, 746, 736, 739, 807, 770, 782, 779, 801, 755, 773, 756] payload_rows_per_dst=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] dup_rows_per_dst=[2926, 3138, 2978, 3078, 2990, 3028, 2790, 3164] combine_rows_per_src=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344576 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846720 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348864 weight_movement_bytes=34603008 total_nvlink_bytes=567951872 weight_to_token_ratio=0.064879 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64882 dup_rows=23996 dup_rows_remote=21070 dedup_ratio=0.244100 router_tokens_per_expert=[754, 795, 750, 738, 765, 740, 779, 787, 747, 724, 820, 747, 827, 776, 761, 755, 746, 762, 785, 786, 730, 765, 744, 760, 838, 763, 767, 784, 789, 800, 758, 776, 751, 747, 763, 801, 759, 759, 814, 759, 755, 754, 775, 778, 729, 755, 755, 738, 780, 775, 783, 772, 746, 763, 770, 787, 747, 786, 727, 765, 821, 781, 796, 743] payload_rows_per_dst=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] dup_rows_per_dst=[2978, 3038, 2960, 3218, 3074, 2834, 2926, 2968] combine_rows_per_src=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266884200 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6137400424 ingress_bytes=7513604096 link_bytes=7513604096 total_nvlink_bytes=13651004520 weight_to_token_ratio=0.112834 prefetch_share_of_ingress=0.096713 dedup_ratio=0.243711 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229264666e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343384 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265690456 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531037528 weight_movement_bytes=69206016 total_nvlink_bytes=600243544 weight_to_token_ratio=0.130322 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263422144 prefetch_bytes=726663168 reduce_bytes=899678208 weight_movement_bytes=1626341376 egress_bytes=6135666880 ingress_bytes=7754096640 link_bytes=7754096640 total_nvlink_bytes=13889763520 weight_to_token_ratio=0.132617 prefetch_share_of_ingress=0.093713 dedup_ratio=0.243723 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=26 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.268610496e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343808 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100480 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857152 weight_movement_bytes=34603008 total_nvlink_bytes=566460160 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64356 dup_rows=24224 dup_rows_remote=20922 dedup_ratio=0.246419 router_tokens_per_expert=[770, 781, 756, 765, 734, 767, 789, 788, 758, 763, 793, 782, 757, 761, 742, 743, 804, 775, 779, 760, 811, 749, 785, 795, 771, 738, 777, 746, 784, 803, 757, 781, 739, 784, 764, 745, 802, 750, 788, 783, 743, 775, 737, 784, 774, 740, 762, 745, 808, 720, 743, 757, 743, 758, 775, 760, 755, 754, 785, 794, 797, 729, 768, 827] payload_rows_per_dst=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] dup_rows_per_dst=[3302, 3110, 3266, 3012, 3024, 2910, 3006, 2594] combine_rows_per_src=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263002624 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6135460352 ingress_bytes=7407853568 link_bytes=7407853568 total_nvlink_bytes=13543313920 weight_to_token_ratio=0.104404 prefetch_share_of_ingress=0.093422 dedup_ratio=0.243895 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211963330e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263602176 dispatch_meta_bytes=341112 combine_payload_bytes=263602176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263943288 ingress_bytes=298205184 link_bytes=298205184 token_bytes=527545464 weight_movement_bytes=34603008 total_nvlink_bytes=562148472 weight_to_token_ratio=0.065592 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122122100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64986 dup_rows=24152 dup_rows_remote=21038 dedup_ratio=0.245687 router_tokens_per_expert=[765, 775, 725, 756, 765, 800, 735, 741, 796, 762, 735, 804, 769, 803, 776, 781, 725, 740, 742, 753, 763, 767, 777, 785, 706, 743, 773, 743, 776, 793, 762, 789, 788, 768, 783, 769, 756, 774, 789, 732, 761, 802, 750, 774, 778, 800, 786, 788, 762, 783, 764, 795, 778, 752, 815, 691, 758, 736, 752, 821, 748, 751, 774, 849] payload_rows_per_dst=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] dup_rows_per_dst=[2936, 3104, 2920, 2946, 3006, 3096, 3114, 3030] combine_rows_per_src=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12236651648 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6122273920 ingress_bytes=7533101056 link_bytes=7533101056 total_nvlink_bytes=13655374976 weight_to_token_ratio=0.115940 prefetch_share_of_ingress=0.091869 dedup_ratio=0.244171 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232454470e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344096 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526752 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709408 weight_movement_bytes=34603008 total_nvlink_bytes=567312416 weight_to_token_ratio=0.064957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264298040 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=6136108600 ingress_bytes=7581515776 link_bytes=7581515776 total_nvlink_bytes=13717624376 weight_to_token_ratio=0.118501 prefetch_share_of_ingress=0.095847 dedup_ratio=0.244106 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240375370e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-29 23:41:58] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 672.8 | throughput per GPU (TFLOP/s/GPU): 189.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.055922E+01 | loss scale: 1.0 | grad norm: 894.654 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:41:58 +[rank0]:[W829 23:41:59.654997632 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:41:59.906000 229212 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:41:59.907000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229394 closing signal SIGTERM +W0829 23:41:59.907000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229395 closing signal SIGTERM +W0829 23:41:59.909000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229396 closing signal SIGTERM +W0829 23:41:59.910000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229397 closing signal SIGTERM +W0829 23:41:59.910000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229398 closing signal SIGTERM +W0829 23:41:59.911000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229399 closing signal SIGTERM +W0829 23:41:59.912000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229400 closing signal SIGTERM +W0829 23:41:59.913000 229212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 229401 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 229212 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1cab0511f160ab241f373fda960baf9c754d9ef9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv @@ -0,0 +1,216 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 3 +4, 4 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 807 +2, 801 +3, 817 +4, 797 +5, 819 +6, 811 +7, 797 +0, 1079 +1, 1079 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1079 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 45191 +1, 45185 +2, 45197 +3, 45191 +4, 45205 +5, 45195 +6, 45191 +7, 45179 +0, 49415 +1, 48859 +2, 48843 +3, 49553 +4, 49593 +5, 49607 +6, 49593 +7, 49571 +0, 49987 +1, 49445 +2, 49463 +3, 50145 +4, 50171 +5, 50169 +6, 50143 +7, 50153 +0, 50687 +1, 50117 +2, 50119 +3, 50819 +4, 50817 +5, 50817 +6, 50817 +7, 50817 +0, 51387 +1, 50817 +2, 50817 +3, 51517 +4, 51517 +5, 51517 +6, 51517 +7, 51517 +0, 72055 +1, 72089 +2, 72083 +3, 72035 +4, 72061 +5, 72079 +6, 72067 +7, 72083 +0, 72595 +1, 72619 +2, 72617 +3, 72581 +4, 72581 +5, 72581 +6, 72581 +7, 72581 +0, 72893 +1, 72923 +2, 72901 +3, 72913 +4, 72911 +5, 72885 +6, 72905 +7, 72949 +0, 82427 +1, 82351 +2, 82373 +3, 82309 +4, 82345 +5, 82331 +6, 82363 +7, 82349 +0, 95855 +1, 95789 +2, 95823 +3, 95787 +4, 95787 +5, 95787 +6, 95787 +7, 95787 +0, 95855 +1, 95789 +2, 95823 +3, 95787 +4, 95787 +5, 95787 +6, 95787 +7, 95787 +0, 95529 +1, 95143 +2, 95177 +3, 95205 +4, 95787 +5, 95461 +6, 95205 +7, 95787 +0, 93609 +1, 93223 +2, 93645 +3, 93609 +4, 93673 +5, 93867 +6, 93605 +7, 93867 +0, 87021 +1, 78619 +2, 92223 +3, 91925 +4, 92187 +5, 91861 +6, 91541 +7, 91541 +0, 4 +1, 4 +2, 77327 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fcf0b050a493a9a3c5cbbdc265102bdaea506e42 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..27638bc9fd20fe87304f87aa478c45a30ad252d8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log @@ -0,0 +1,3537 @@ +[23:42:05] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + QWEN2_57B / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:42:15.474000 231719 torch/distributed/run.py:766] +W0829 23:42:15.474000 231719 torch/distributed/run.py:766] ***************************************** +W0829 23:42:15.474000 231719 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:42:15.474000 231719 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 00 is connected to +0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W829 23:42:33.065017912 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:42:33.067876019 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:42:33.067898480 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:42:33.067936882 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:42:33.068931391 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:42:33.069015143 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:42:33.070118586 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:42:34.226208708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.705 seconds +time to initialize megatron (seconds): 7.667 +[after megatron is initialized] datetime: 2026-08-29 23:42:39 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:42:42 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:42:43 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3050.31, 3057.58) + train/valid/test-data-iterators-setup ..........: (66.78, 75.77) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-29 23:42:43 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 238.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 241.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [6656, 2048, 7936, 256, 2432, 512, 8832, 512, 4224, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[48, -1, -1, -1, -1, -1, -1, -1], [34, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [8, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [53, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [9856, 3456, 3072, 4736, 512, 5632, 1536, 896, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[9, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [1792, 5120, 4608, 6144, 2560, 2816, 6400, 3712, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [34, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [2048, 2176, 768, 8448, 1664, 2048, 9344, 3072, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[30, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [1, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [56, -1, -1, -1, -1, -1, -1, -1], [37, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-29 23:43:03] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 20743.7 | throughput per GPU (TFLOP/s/GPU): 6.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097928E+01 | loss scale: 1.0 | grad norm: 29.863 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 20.11 + +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 60512.9453125 | max allocated: 60512.94580078125 | reserved: 62924.0 | max reserved: 62924.0 + [2026-08-29 23:43:04] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 310.6 | throughput per GPU (TFLOP/s/GPU): 397.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097878E+01 | loss scale: 1.0 | grad norm: 30.635 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46296 payload_rows_remote=39426 dup_rows=19240 dup_rows_remote=15616 dedup_ratio=0.293579 router_tokens_per_expert=[598, 192, 537, 9, 57, 15, 978, 76, 681, 1265, 283, 62, 1328, 219, 200, 514, 84, 16, 553, 244, 322, 125, 236, 24, 203, 888, 402, 890, 580, 407, 205, 50, 462, 454, 1057, 416, 1079, 11, 64, 261, 672, 615, 1050, 112, 828, 201, 57, 205, 2368, 391, 180, 25, 380, 1288, 3289, 1806, 448, 368, 179, 154, 952, 877, 91, 185] payload_rows_per_dst=[6870, 6522, 7254, 5116, 5334, 4790, 5506, 4904] dup_rows_per_dst=[3624, 3524, 2532, 2134, 2274, 1748, 1800, 1604] combine_rows_per_src=[6870, 6522, 7254, 5116, 5334, 4790, 5506, 4904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43884 payload_rows_remote=37152 dup_rows=21652 dup_rows_remote=17196 dedup_ratio=0.330383 router_tokens_per_expert=[350, 143, 603, 10, 122, 18, 899, 102, 1123, 634, 265, 268, 1336, 164, 76, 452, 153, 66, 297, 227, 878, 172, 114, 21, 254, 1148, 359, 709, 485, 610, 321, 28, 1018, 734, 1103, 478, 754, 36, 101, 772, 686, 645, 1175, 80, 811, 215, 195, 232, 1282, 390, 183, 37, 321, 1258, 3343, 1440, 230, 468, 216, 70, 742, 620, 80, 646] payload_rows_per_dst=[5406, 4732, 4160, 4844, 6778, 5558, 6732, 5674] dup_rows_per_dst=[1652, 1752, 1332, 1588, 5366, 2520, 4456, 2986] combine_rows_per_src=[5406, 4732, 4160, 4844, 6778, 5558, 6732, 5674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46250 payload_rows_remote=39778 dup_rows=19286 dup_rows_remote=16780 dedup_ratio=0.294281 router_tokens_per_expert=[331, 117, 993, 23, 82, 35, 368, 119, 642, 1049, 389, 324, 828, 192, 227, 534, 61, 62, 535, 186, 448, 109, 356, 42, 353, 1209, 664, 955, 442, 461, 287, 80, 478, 216, 865, 544, 817, 52, 60, 721, 330, 1125, 1244, 210, 995, 123, 282, 296, 1556, 459, 102, 14, 791, 1611, 2690, 1257, 338, 500, 269, 127, 485, 1391, 91, 226] payload_rows_per_dst=[5538, 5822, 6472, 5910, 5304, 6230, 6022, 4952] dup_rows_per_dst=[1710, 2548, 2506, 2992, 2202, 2980, 2446, 1902] combine_rows_per_src=[5538, 5822, 6472, 5910, 5304, 6230, 6022, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45692 payload_rows_remote=40154 dup_rows=19844 dup_rows_remote=17682 dedup_ratio=0.302795 router_tokens_per_expert=[568, 166, 1177, 12, 105, 1, 723, 81, 891, 876, 190, 182, 1241, 231, 63, 410, 124, 21, 391, 349, 755, 35, 285, 43, 341, 611, 360, 772, 768, 604, 343, 51, 611, 426, 660, 331, 685, 47, 146, 507, 536, 1030, 1187, 214, 1240, 201, 76, 213, 1479, 373, 231, 73, 426, 862, 2547, 813, 666, 831, 441, 119, 577, 1652, 192, 606] payload_rows_per_dst=[6060, 5754, 6378, 5538, 4930, 6402, 3990, 6640] dup_rows_per_dst=[2564, 2414, 2722, 2162, 1896, 2992, 770, 4324] combine_rows_per_src=[6060, 5754, 6378, 5538, 4930, 6402, 3990, 6640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45620 payload_rows_remote=39650 dup_rows=19916 dup_rows_remote=17104 dedup_ratio=0.303894 router_tokens_per_expert=[442, 207, 354, 18, 25, 25, 1099, 25, 1016, 948, 184, 97, 1292, 151, 143, 477, 53, 59, 457, 552, 610, 22, 205, 55, 466, 1091, 379, 825, 1083, 475, 336, 46, 449, 292, 1220, 426, 729, 47, 157, 489, 512, 1281, 777, 156, 981, 253, 60, 371, 1541, 734, 221, 48, 575, 971, 2593, 1090, 414, 644, 270, 58, 491, 1223, 145, 333] payload_rows_per_dst=[5478, 6038, 6478, 6000, 5326, 5970, 4380, 5950] dup_rows_per_dst=[1994, 2578, 2734, 3402, 2292, 2812, 956, 3148] combine_rows_per_src=[5478, 6038, 6478, 6000, 5326, 5970, 4380, 5950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44112 payload_rows_remote=38002 dup_rows=21424 dup_rows_remote=18616 dedup_ratio=0.326904 router_tokens_per_expert=[322, 166, 997, 16, 114, 7, 1051, 192, 864, 1095, 185, 282, 874, 170, 373, 435, 89, 29, 515, 259, 183, 91, 211, 62, 265, 1331, 420, 853, 530, 992, 278, 28, 270, 145, 1334, 279, 1081, 26, 166, 567, 442, 713, 1459, 175, 667, 147, 130, 441, 1287, 679, 137, 29, 1028, 1939, 2553, 1275, 364, 528, 185, 75, 455, 648, 89, 176] payload_rows_per_dst=[6006, 5056, 2580, 4962, 6082, 5828, 7488, 6110] dup_rows_per_dst=[2298, 1752, 298, 1560, 3402, 2730, 6576, 2808] combine_rows_per_src=[6006, 5056, 2580, 4962, 6082, 5828, 7488, 6110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282605568 dispatch_meta_bytes=220168 combine_payload_bytes=282605568 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=282825736 ingress_bytes=502806528 link_bytes=502806528 token_bytes=565431304 weight_movement_bytes=220200960 total_nvlink_bytes=785632264 weight_to_token_ratio=0.389439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.744632497e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285128704 dispatch_meta_bytes=226232 combine_payload_bytes=285128704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285354936 ingress_bytes=395229184 link_bytes=395229184 token_bytes=570483640 weight_movement_bytes=110100480 total_nvlink_bytes=680584120 weight_to_token_ratio=0.192995 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.873685550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=217392 combine_payload_bytes=266305536 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=266522928 ingress_bytes=816807936 link_bytes=816807936 token_bytes=532828464 weight_movement_bytes=550502400 total_nvlink_bytes=1083330864 weight_to_token_ratio=1.033170 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.420563343e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287823872 dispatch_meta_bytes=231344 combine_payload_bytes=287823872 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=288055216 ingress_bytes=508024832 link_bytes=508024832 token_bytes=575879088 weight_movement_bytes=220200960 total_nvlink_bytes=796080048 weight_to_token_ratio=0.382374 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.835387386e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284211200 dispatch_meta_bytes=227016 combine_payload_bytes=284211200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284438216 ingress_bytes=504412160 link_bytes=504412160 token_bytes=568649416 weight_movement_bytes=220200960 total_nvlink_bytes=788850376 weight_to_token_ratio=0.387235 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.772557078e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46462 payload_rows_remote=40998 dup_rows=19074 dup_rows_remote=17330 dedup_ratio=0.291046 router_tokens_per_expert=[325, 136, 575, 35, 19, 18, 1036, 16, 1199, 928, 213, 83, 1172, 221, 286, 319, 47, 38, 670, 489, 400, 114, 302, 21, 215, 1083, 364, 1042, 896, 1103, 217, 49, 481, 110, 1432, 593, 664, 12, 66, 246, 400, 643, 1183, 64, 663, 148, 313, 367, 1279, 718, 124, 56, 686, 930, 3251, 1756, 308, 424, 271, 286, 738, 727, 43, 155] payload_rows_per_dst=[5262, 6170, 7264, 6318, 5464, 5362, 5330, 5292] dup_rows_per_dst=[1616, 2672, 3400, 3620, 1744, 2200, 1350, 2472] combine_rows_per_src=[5262, 6170, 7264, 6318, 5464, 5362, 5330, 5292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46214 payload_rows_remote=40470 dup_rows=19322 dup_rows_remote=16864 dedup_ratio=0.294830 router_tokens_per_expert=[584, 170, 931, 8, 114, 6, 852, 107, 945, 1241, 210, 101, 882, 240, 119, 363, 73, 70, 363, 192, 314, 86, 718, 52, 677, 1414, 331, 776, 356, 801, 219, 121, 812, 266, 528, 267, 658, 193, 77, 770, 402, 1174, 599, 100, 799, 282, 61, 290, 1957, 279, 105, 20, 820, 1618, 2845, 1025, 380, 353, 174, 78, 380, 1553, 131, 336] payload_rows_per_dst=[6424, 5744, 6792, 6106, 5158, 5454, 5704, 4832] dup_rows_per_dst=[3034, 2458, 2634, 3284, 1984, 1960, 2030, 1938] combine_rows_per_src=[6424, 5744, 6792, 6106, 5158, 5454, 5704, 4832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272398336 dispatch_meta_bytes=226472 combine_payload_bytes=272398336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272624808 ingress_bytes=382498816 link_bytes=382498816 token_bytes=545023144 weight_movement_bytes=110100480 total_nvlink_bytes=655123624 weight_to_token_ratio=0.202011 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.652283513e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47236 payload_rows_remote=41802 dup_rows=18300 dup_rows_remote=16898 dedup_ratio=0.279236 router_tokens_per_expert=[5, 0, 571, 47, 7, 27, 944, 2, 902, 1403, 150, 8, 1720, 1022, 4, 1437, 1, 251, 107, 318, 1, 465, 8, 870, 109, 444, 2, 1240, 2539, 139, 32, 15, 2998, 795, 0, 267, 326, 14, 19, 1, 42, 0, 1514, 78, 525, 0, 3, 95, 2057, 2376, 393, 943, 241, 154, 55, 0, 16, 7, 1396, 86, 755, 1815, 437, 570] payload_rows_per_dst=[5434, 6186, 5396, 7568, 6726, 4840, 5942, 5144] dup_rows_per_dst=[1402, 2188, 1520, 6224, 2114, 1718, 1744, 1390] combine_rows_per_src=[5434, 6186, 5396, 7568, 6726, 4840, 5942, 5144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299636736 dispatch_meta_bytes=234800 combine_payload_bytes=299636736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299871536 ingress_bytes=409737216 link_bytes=409737216 token_bytes=599508272 weight_movement_bytes=110100480 total_nvlink_bytes=709608752 weight_to_token_ratio=0.183651 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.126004088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293873664 dispatch_meta_bytes=233312 combine_payload_bytes=293873664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294106976 ingress_bytes=403974144 link_bytes=403974144 token_bytes=587980640 weight_movement_bytes=110100480 total_nvlink_bytes=698081120 weight_to_token_ratio=0.187252 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.025774787e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48296 payload_rows_remote=42834 dup_rows=17240 dup_rows_remote=15466 dedup_ratio=0.263062 router_tokens_per_expert=[116, 49, 1202, 330, 3, 98, 413, 2, 986, 593, 290, 115, 2739, 2070, 9, 2686, 1, 178, 79, 405, 5, 81, 7, 713, 169, 377, 4, 1326, 1634, 86, 6, 16, 2751, 729, 0, 167, 180, 8, 177, 17, 3, 1, 658, 67, 264, 0, 0, 419, 1421, 1853, 984, 39, 436, 118, 17, 7, 43, 37, 2110, 98, 879, 1713, 315, 469] payload_rows_per_dst=[5654, 6746, 6460, 5462, 6254, 5412, 6268, 6040] dup_rows_per_dst=[2198, 2718, 1850, 1774, 1804, 1552, 3482, 1862] combine_rows_per_src=[5654, 6746, 6460, 5462, 6254, 5412, 6268, 6040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47390 payload_rows_remote=41122 dup_rows=18146 dup_rows_remote=16188 dedup_ratio=0.276886 router_tokens_per_expert=[58, 8, 1444, 437, 21, 50, 460, 79, 999, 1285, 285, 16, 1997, 1846, 142, 2432, 1, 377, 69, 392, 5, 539, 12, 286, 221, 123, 12, 355, 1205, 350, 96, 97, 2498, 733, 1, 312, 251, 7, 150, 15, 11, 0, 1796, 112, 503, 1, 10, 233, 1861, 1895, 240, 130, 403, 817, 63, 5, 27, 38, 1591, 127, 391, 2463, 84, 301] payload_rows_per_dst=[6562, 6636, 6268, 5612, 6074, 6108, 5668, 4462] dup_rows_per_dst=[3478, 2812, 1958, 1886, 1860, 2916, 2580, 656] combine_rows_per_src=[6562, 6636, 6268, 5612, 6074, 6108, 5668, 4462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46306 payload_rows_remote=41682 dup_rows=19230 dup_rows_remote=18050 dedup_ratio=0.293427 router_tokens_per_expert=[113, 5, 747, 308, 57, 47, 338, 5, 1511, 1399, 323, 1, 1852, 1851, 16, 1656, 0, 445, 35, 647, 1, 913, 17, 433, 94, 439, 4, 1288, 1415, 217, 4, 154, 3018, 718, 1, 238, 31, 0, 32, 122, 4, 9, 767, 65, 289, 1, 11, 244, 1432, 1530, 163, 649, 393, 85, 68, 14, 61, 53, 1859, 123, 1462, 2328, 171, 492] payload_rows_per_dst=[5834, 5126, 6390, 5366, 6758, 5612, 4624, 6596] dup_rows_per_dst=[2062, 1374, 3796, 1864, 2232, 2012, 1180, 4710] combine_rows_per_src=[5834, 5126, 6390, 5366, 6758, 5612, 4624, 6596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290088960 dispatch_meta_bytes=229336 combine_payload_bytes=290088960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290318296 ingress_bytes=510289920 link_bytes=510289920 token_bytes=580407256 weight_movement_bytes=220200960 total_nvlink_bytes=800608216 weight_to_token_ratio=0.379390 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.874780991e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47990 payload_rows_remote=41082 dup_rows=17546 dup_rows_remote=15504 dedup_ratio=0.267731 router_tokens_per_expert=[110, 45, 922, 140, 22, 27, 632, 5, 1633, 1427, 345, 13, 1875, 3032, 22, 2161, 2, 110, 92, 1012, 1, 160, 0, 512, 145, 777, 11, 687, 1773, 52, 74, 17, 2183, 882, 0, 192, 198, 54, 6, 25, 8, 9, 835, 142, 283, 3, 12, 151, 1230, 1940, 811, 63, 632, 331, 122, 16, 42, 69, 1435, 41, 353, 1854, 347, 663] payload_rows_per_dst=[5622, 6582, 6458, 5280, 5668, 6908, 5660, 5812] dup_rows_per_dst=[1892, 3190, 2500, 1792, 1412, 2042, 2170, 2548] combine_rows_per_src=[5622, 6582, 6458, 5280, 5668, 6908, 5660, 5812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=37530 dup_rows=21232 dup_rows_remote=14710 dedup_ratio=0.323975 router_tokens_per_expert=[20, 9, 1051, 204, 35, 36, 448, 19, 1002, 646, 204, 21, 1199, 1941, 22, 1805, 0, 408, 16, 472, 22, 506, 37, 783, 140, 639, 5, 1283, 1491, 221, 29, 30, 1986, 789, 3, 141, 106, 3, 96, 54, 5, 10, 1675, 168, 315, 1, 2, 152, 1450, 1886, 322, 436, 905, 1409, 65, 45, 54, 115, 765, 177, 1612, 2567, 109, 601] payload_rows_per_dst=[4212, 3442, 6014, 5504, 5314, 6306, 6738, 6774] dup_rows_per_dst=[1036, 348, 2084, 2172, 1042, 3386, 4642, 6522] combine_rows_per_src=[4212, 3442, 6014, 5504, 5314, 6306, 6738, 6774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=307034112 dispatch_meta_bytes=233200 combine_payload_bytes=307034112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307267312 ingress_bytes=417134592 link_bytes=417134592 token_bytes=614301424 weight_movement_bytes=110100480 total_nvlink_bytes=724401904 weight_to_token_ratio=0.179229 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.254656623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294762496 dispatch_meta_bytes=229240 combine_payload_bytes=294762496 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=294991736 ingress_bytes=514963456 link_bytes=514963456 token_bytes=589754232 weight_movement_bytes=220200960 total_nvlink_bytes=809955192 weight_to_token_ratio=0.373377 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.956061469e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298776576 dispatch_meta_bytes=238928 combine_payload_bytes=298776576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=299015504 ingress_bytes=629078016 link_bytes=629078016 token_bytes=597792080 weight_movement_bytes=330301440 total_nvlink_bytes=928093520 weight_to_token_ratio=0.552536 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094070135e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294475776 dispatch_meta_bytes=226344 combine_payload_bytes=294475776 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=294702120 ingress_bytes=514676736 link_bytes=514676736 token_bytes=589177896 weight_movement_bytes=220200960 total_nvlink_bytes=809378856 weight_to_token_ratio=0.373743 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.951074937e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46230 payload_rows_remote=41014 dup_rows=19306 dup_rows_remote=17802 dedup_ratio=0.294586 router_tokens_per_expert=[1106, 759, 7, 43, 35, 55, 499, 0, 0, 3246, 7, 30, 48, 272, 2991, 1, 0, 640, 0, 231, 188, 2247, 0, 2, 1662, 0, 150, 856, 1, 3, 135, 1247, 429, 875, 2, 643, 655, 34, 1980, 7, 306, 1162, 6, 2, 642, 2919, 0, 1, 8, 0, 1900, 2, 477, 1, 221, 0, 652, 1, 96, 22, 1789, 0, 1000, 475] payload_rows_per_dst=[5216, 7820, 5478, 6710, 4332, 7112, 4512, 5050] dup_rows_per_dst=[1504, 7370, 1138, 3646, 958, 2964, 706, 1020] combine_rows_per_src=[5216, 7820, 5478, 6710, 4332, 7112, 4512, 5050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269015040 dispatch_meta_bytes=208960 combine_payload_bytes=269015040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=269224000 ingress_bytes=489216000 link_bytes=489216000 token_bytes=538239040 weight_movement_bytes=220200960 total_nvlink_bytes=758440000 weight_to_token_ratio=0.409114 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.508270862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48136 payload_rows_remote=41008 dup_rows=17400 dup_rows_remote=15766 dedup_ratio=0.265503 router_tokens_per_expert=[12, 17, 1626, 696, 17, 36, 566, 23, 953, 641, 306, 36, 2042, 3142, 14, 1842, 0, 94, 41, 967, 44, 228, 17, 596, 60, 420, 8, 870, 1284, 179, 94, 52, 3383, 711, 1, 37, 184, 9, 34, 22, 2, 27, 1374, 114, 124, 1, 0, 261, 1800, 1220, 113, 360, 335, 293, 31, 63, 43, 155, 597, 61, 2206, 1988, 115, 181] payload_rows_per_dst=[6454, 5940, 5784, 4552, 7128, 7252, 5350, 5676] dup_rows_per_dst=[3508, 2044, 1874, 1382, 1634, 2838, 2090, 2030] combine_rows_per_src=[6454, 5940, 5784, 4552, 7128, 7252, 5350, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=293988352 dispatch_meta_bytes=235264 combine_payload_bytes=293988352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294223616 ingress_bytes=404088832 link_bytes=404088832 token_bytes=588211968 weight_movement_bytes=110100480 total_nvlink_bytes=698312448 weight_to_token_ratio=0.187178 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.027769400e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49038 payload_rows_remote=42132 dup_rows=16498 dup_rows_remote=13200 dedup_ratio=0.251740 router_tokens_per_expert=[43, 8, 1638, 192, 27, 35, 730, 49, 1559, 1437, 309, 14, 1732, 1819, 51, 2044, 2, 90, 116, 429, 26, 639, 6, 240, 282, 112, 58, 582, 1969, 56, 13, 21, 3437, 686, 77, 26, 211, 73, 241, 7, 39, 2, 741, 290, 167, 20, 28, 211, 2025, 1456, 183, 73, 227, 1041, 97, 53, 15, 30, 1998, 66, 284, 2372, 91, 173] payload_rows_per_dst=[6744, 6906, 5842, 6630, 7250, 5132, 5748, 4786] dup_rows_per_dst=[3444, 3298, 1342, 2468, 2266, 1502, 1650, 528] combine_rows_per_src=[6744, 6906, 5842, 6630, 7250, 5132, 5748, 4786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46290 payload_rows_remote=41964 dup_rows=19246 dup_rows_remote=18340 dedup_ratio=0.293671 router_tokens_per_expert=[403, 1213, 74, 6, 19, 502, 762, 1, 0, 2320, 29, 1, 111, 1056, 3041, 0, 6, 500, 8, 947, 13, 1023, 109, 10, 495, 0, 627, 2758, 13, 29, 37, 1220, 1092, 575, 6, 853, 435, 3, 2191, 32, 292, 172, 0, 4, 374, 2266, 0, 0, 1, 0, 2662, 40, 159, 10, 375, 2, 927, 3, 98, 4, 1271, 1, 795, 792] payload_rows_per_dst=[7110, 7686, 4326, 3978, 6854, 5716, 5720, 4900] dup_rows_per_dst=[4366, 6210, 906, 864, 3520, 1310, 778, 1292] combine_rows_per_src=[7110, 7686, 4326, 3978, 6854, 5716, 5720, 4900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47180 payload_rows_remote=41122 dup_rows=18356 dup_rows_remote=16316 dedup_ratio=0.280090 router_tokens_per_expert=[1706, 906, 81, 5, 0, 255, 375, 0, 80, 3191, 0, 28, 107, 707, 2267, 0, 0, 258, 10, 287, 22, 2465, 13, 8, 1615, 1, 464, 989, 0, 14, 48, 1055, 275, 732, 44, 118, 141, 0, 3302, 25, 189, 294, 0, 79, 688, 2364, 43, 0, 2, 0, 1135, 0, 352, 4, 289, 0, 1527, 3, 3, 10, 1252, 1, 2815, 124] payload_rows_per_dst=[5506, 1778, 7682, 6102, 7206, 5882, 6058, 6966] dup_rows_per_dst=[1150, 66, 4826, 2270, 2068, 1432, 2040, 4504] combine_rows_per_src=[5506, 1778, 7682, 6102, 7206, 5882, 6058, 6966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46950 payload_rows_remote=42332 dup_rows=18586 dup_rows_remote=17818 dedup_ratio=0.283600 router_tokens_per_expert=[1433, 1437, 16, 24, 2, 142, 511, 0, 1, 1703, 8, 35, 130, 294, 1836, 0, 0, 778, 0, 814, 177, 1836, 0, 0, 1801, 0, 147, 1296, 0, 7, 83, 655, 241, 1339, 1, 256, 131, 1, 2121, 32, 319, 965, 0, 3, 828, 1840, 1, 0, 0, 0, 2150, 0, 198, 0, 271, 0, 2232, 1, 5, 38, 1628, 0, 2707, 294] payload_rows_per_dst=[6442, 5908, 5528, 4618, 6180, 6932, 4774, 6568] dup_rows_per_dst=[3280, 2106, 1682, 768, 2064, 4108, 464, 4114] combine_rows_per_src=[6442, 5908, 5528, 4618, 6180, 6932, 4774, 6568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46902 payload_rows_remote=40664 dup_rows=18634 dup_rows_remote=17442 dedup_ratio=0.284332 router_tokens_per_expert=[1136, 1038, 33, 22, 28, 221, 1161, 8, 4, 1895, 14, 2, 189, 199, 3075, 1, 0, 311, 0, 610, 260, 1237, 4, 1, 1457, 0, 210, 1917, 32, 13, 147, 716, 1406, 750, 12, 218, 1282, 2, 1593, 34, 338, 147, 0, 9, 456, 2760, 1, 4, 1, 0, 2021, 0, 387, 3, 280, 0, 723, 3, 16, 27, 841, 1, 2832, 680] payload_rows_per_dst=[5880, 788, 6364, 6660, 6714, 6238, 7482, 6776] dup_rows_per_dst=[1414, 30, 2272, 2324, 3880, 1192, 4052, 3470] combine_rows_per_src=[5880, 788, 6364, 6660, 6714, 6238, 7482, 6776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293945344 dispatch_meta_bytes=227096 combine_payload_bytes=293945344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294172440 ingress_bytes=404045824 link_bytes=404045824 token_bytes=588117784 weight_movement_bytes=110100480 total_nvlink_bytes=698218264 weight_to_token_ratio=0.187208 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.027021421e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=302002176 dispatch_meta_bytes=221328 combine_payload_bytes=302002176 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=302223504 ingress_bytes=962605056 link_bytes=962605056 token_bytes=604225680 weight_movement_bytes=660602880 total_nvlink_bytes=1264828560 weight_to_token_ratio=1.093305 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.674128514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47576 payload_rows_remote=42556 dup_rows=17960 dup_rows_remote=16314 dedup_ratio=0.274048 router_tokens_per_expert=[621, 1728, 32, 6, 10, 265, 331, 5, 18, 2448, 2, 21, 818, 1193, 3217, 3, 0, 511, 0, 102, 37, 2015, 5, 12, 1408, 0, 248, 1822, 1, 11, 30, 1437, 1231, 419, 10, 192, 474, 8, 1740, 2, 377, 478, 10, 72, 800, 2490, 0, 0, 3, 0, 1670, 0, 583, 0, 649, 2, 503, 9, 18, 20, 882, 0, 1492, 277] payload_rows_per_dst=[5180, 3754, 7106, 6626, 5802, 6424, 7664, 5020] dup_rows_per_dst=[816, 356, 3154, 3288, 2086, 2030, 4584, 1646] combine_rows_per_src=[5180, 3754, 7106, 6626, 5802, 6424, 7664, 5020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=300797952 dispatch_meta_bytes=241216 combine_payload_bytes=300797952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301039168 ingress_bytes=410898432 link_bytes=410898432 token_bytes=601837120 weight_movement_bytes=110100480 total_nvlink_bytes=711937600 weight_to_token_ratio=0.182941 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.146199544e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=294762496 dispatch_meta_bytes=229752 combine_payload_bytes=294762496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294992248 ingress_bytes=404862976 link_bytes=404862976 token_bytes=589754744 weight_movement_bytes=110100480 total_nvlink_bytes=699855224 weight_to_token_ratio=0.186689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.041233038e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=303435776 dispatch_meta_bytes=240600 combine_payload_bytes=303435776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=303676376 ingress_bytes=523636736 link_bytes=523636736 token_bytes=607112152 weight_movement_bytes=220200960 total_nvlink_bytes=827313112 weight_to_token_ratio=0.362702 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.106904073e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291479552 dispatch_meta_bytes=232424 combine_payload_bytes=291479552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291711976 ingress_bytes=401580032 link_bytes=401580032 token_bytes=583191528 weight_movement_bytes=110100480 total_nvlink_bytes=693292008 weight_to_token_ratio=0.188790 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.984137242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46576 payload_rows_remote=39380 dup_rows=18960 dup_rows_remote=16170 dedup_ratio=0.289307 router_tokens_per_expert=[0, 0, 226, 308, 2830, 254, 1343, 32, 15, 0, 0, 47, 3161, 2601, 26, 16, 229, 0, 2499, 161, 46, 2, 10, 0, 12, 95, 2492, 11, 0, 14, 7, 0, 8, 0, 66, 1628, 6, 3751, 285, 354, 0, 2486, 182, 9, 265, 0, 939, 4, 2233, 2, 137, 3, 156, 0, 191, 23, 1371, 41, 13, 40, 0, 0, 39, 2099] payload_rows_per_dst=[7196, 8190, 5446, 5138, 7050, 2566, 5090, 5900] dup_rows_per_dst=[2790, 11044, 448, 124, 2616, 232, 400, 1306] combine_rows_per_src=[7196, 8190, 5446, 5138, 7050, 2566, 5090, 5900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=305041408 dispatch_meta_bytes=235480 combine_payload_bytes=305041408 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=305276888 ingress_bytes=635342848 link_bytes=635342848 token_bytes=610318296 weight_movement_bytes=330301440 total_nvlink_bytes=940619736 weight_to_token_ratio=0.541195 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.104965709e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282275840 dispatch_meta_bytes=222200 combine_payload_bytes=282275840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=282498040 ingress_bytes=502476800 link_bytes=502476800 token_bytes=564773880 weight_movement_bytes=220200960 total_nvlink_bytes=784974840 weight_to_token_ratio=0.389892 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.738897984e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46192 payload_rows_remote=41750 dup_rows=19344 dup_rows_remote=18328 dedup_ratio=0.295166 router_tokens_per_expert=[409, 2179, 85, 34, 7, 202, 700, 7, 0, 2760, 11, 13, 486, 493, 2746, 0, 0, 230, 0, 1213, 78, 1524, 116, 21, 2088, 3, 1578, 1289, 1, 2, 20, 1233, 425, 343, 4, 134, 401, 1, 1411, 10, 642, 147, 0, 28, 947, 2086, 0, 0, 29, 0, 1328, 1, 795, 0, 252, 0, 762, 0, 17, 5, 720, 0, 2335, 417] payload_rows_per_dst=[6478, 3768, 6424, 6856, 4442, 5808, 6360, 6056] dup_rows_per_dst=[2816, 1968, 3178, 3524, 1016, 1892, 2494, 2456] combine_rows_per_src=[6478, 3768, 6424, 6856, 4442, 5808, 6360, 6056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46088 payload_rows_remote=38226 dup_rows=19448 dup_rows_remote=11382 dedup_ratio=0.296753 router_tokens_per_expert=[1173, 575, 181, 0, 0, 193, 422, 17, 3, 1203, 3, 1, 152, 1667, 3196, 1, 65, 101, 32, 1304, 157, 1038, 1, 38, 1838, 9, 126, 1628, 52, 266, 16, 1127, 485, 1154, 2, 115, 1579, 4, 1060, 25, 599, 53, 0, 7, 151, 2431, 6, 1, 7, 7, 2175, 5, 860, 71, 795, 4, 530, 20, 38, 247, 1210, 0, 1738, 804] payload_rows_per_dst=[6054, 7862, 4542, 5636, 5920, 5456, 5922, 4696] dup_rows_per_dst=[2324, 8066, 930, 1666, 2494, 1040, 1926, 1002] combine_rows_per_src=[6054, 7862, 4542, 5636, 5920, 5456, 5922, 4696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48416 payload_rows_remote=43454 dup_rows=17120 dup_rows_remote=16078 dedup_ratio=0.261230 router_tokens_per_expert=[7, 61, 281, 60, 3006, 4, 2808, 13, 10, 0, 76, 85, 3050, 3032, 125, 195, 860, 5, 2048, 25, 32, 0, 19, 13, 96, 344, 1499, 3, 0, 95, 14, 1, 0, 0, 124, 1789, 1, 3713, 48, 303, 19, 2354, 39, 1, 198, 0, 372, 2, 2776, 11, 115, 0, 51, 0, 233, 129, 499, 47, 0, 0, 37, 0, 6, 2034] payload_rows_per_dst=[7862, 7784, 4962, 3716, 8042, 5390, 5958, 4702] dup_rows_per_dst=[4618, 5362, 1042, 388, 3914, 580, 672, 544] combine_rows_per_src=[7862, 7784, 4962, 3716, 8042, 5390, 5958, 4702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48932 payload_rows_remote=41358 dup_rows=16604 dup_rows_remote=14666 dedup_ratio=0.253357 router_tokens_per_expert=[0, 5, 585, 29, 2091, 11, 3399, 6, 0, 1, 0, 476, 2199, 3578, 62, 2, 156, 11, 2524, 57, 60, 0, 34, 1, 30, 86, 804, 21, 6, 14, 375, 4, 23, 0, 119, 1421, 1, 4064, 1207, 2, 0, 3609, 1, 30, 1632, 0, 742, 0, 1172, 6, 109, 6, 54, 0, 5, 5, 41, 13, 0, 4, 0, 2, 152, 1721] payload_rows_per_dst=[4846, 1082, 6936, 7590, 4808, 7932, 7574, 8164] dup_rows_per_dst=[608, 0, 3148, 2246, 738, 4096, 1938, 3830] combine_rows_per_src=[4846, 1082, 6936, 7590, 4808, 7932, 7574, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47394 payload_rows_remote=40980 dup_rows=18142 dup_rows_remote=16882 dedup_ratio=0.276825 router_tokens_per_expert=[0, 13, 92, 177, 2688, 333, 3184, 51, 45, 0, 48, 94, 3188, 3452, 146, 21, 211, 46, 3351, 110, 172, 0, 52, 9, 5, 322, 803, 195, 0, 51, 381, 0, 23, 1, 18, 1162, 4, 3899, 678, 21, 1, 2759, 18, 68, 180, 0, 810, 1, 1884, 7, 347, 6, 134, 0, 168, 6, 288, 144, 14, 3, 5, 1, 51, 827] payload_rows_per_dst=[5864, 654, 7942, 7488, 3476, 6414, 7590, 7966] dup_rows_per_dst=[844, 54, 6336, 2930, 338, 1260, 3882, 2498] combine_rows_per_src=[5864, 654, 7942, 7488, 3476, 6414, 7590, 7966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52160 payload_rows_remote=45580 dup_rows=13376 dup_rows_remote=11584 dedup_ratio=0.204102 router_tokens_per_expert=[0, 23, 116, 87, 1714, 180, 3697, 2, 4, 0, 1, 21, 3254, 2761, 316, 28, 114, 2, 2395, 56, 42, 0, 72, 14, 5, 123, 2811, 3, 0, 10, 155, 0, 1, 0, 4, 1013, 0, 3528, 648, 70, 0, 3604, 0, 1, 358, 0, 356, 0, 2089, 4, 107, 2, 140, 0, 49, 13, 110, 9, 0, 0, 9, 0, 32, 2615] payload_rows_per_dst=[7960, 7386, 5072, 6580, 7726, 7498, 4564, 5374] dup_rows_per_dst=[3678, 3226, 318, 1792, 2802, 1140, 244, 176] combine_rows_per_src=[7960, 7386, 5072, 6580, 7726, 7498, 4564, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299264000 dispatch_meta_bytes=240312 combine_payload_bytes=299264000 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=299504312 ingress_bytes=519464960 link_bytes=519464960 token_bytes=598768312 weight_movement_bytes=220200960 total_nvlink_bytes=818969272 weight_to_token_ratio=0.367757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.034350027e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46458 payload_rows_remote=38578 dup_rows=19078 dup_rows_remote=16246 dedup_ratio=0.291107 router_tokens_per_expert=[5, 20, 124, 232, 1877, 12, 2875, 5, 8, 1, 5, 92, 3338, 3500, 107, 4, 652, 11, 2682, 27, 60, 0, 123, 11, 94, 388, 691, 27, 2, 50, 10, 0, 0, 1, 158, 2061, 6, 3898, 567, 83, 2, 3371, 3, 11, 198, 7, 1457, 1, 1639, 13, 190, 8, 357, 0, 176, 70, 240, 136, 2, 0, 2, 1, 98, 979] payload_rows_per_dst=[2574, 428, 7866, 7406, 5174, 7738, 7392, 7880] dup_rows_per_dst=[198, 6, 5942, 2118, 578, 4140, 3264, 2832] combine_rows_per_src=[2574, 428, 7866, 7406, 5174, 7738, 7392, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48998 payload_rows_remote=42534 dup_rows=16538 dup_rows_remote=13948 dedup_ratio=0.252350 router_tokens_per_expert=[0, 2329, 4, 2, 28, 19, 26, 410, 0, 3, 0, 85, 5, 140, 0, 20, 891, 52, 1569, 1709, 766, 1489, 44, 1807, 94, 0, 6, 0, 323, 0, 19, 0, 12, 0, 272, 2173, 0, 0, 597, 1035, 3, 1567, 671, 96, 3557, 23, 15, 176, 2, 3784, 22, 477, 512, 1, 41, 67, 14, 8, 0, 1388, 11, 444, 19, 3941] payload_rows_per_dst=[6464, 5690, 5860, 7906, 7158, 4528, 7934, 3458] dup_rows_per_dst=[2590, 1568, 1804, 860, 6520, 1008, 1878, 310] combine_rows_per_src=[6464, 5690, 5860, 7906, 7158, 4528, 7934, 3458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274003968 dispatch_meta_bytes=198432 combine_payload_bytes=274003968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=274202400 ingress_bytes=604305408 link_bytes=604305408 token_bytes=548206368 weight_movement_bytes=330301440 total_nvlink_bytes=878507808 weight_to_token_ratio=0.602513 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.050986496e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=311478272 dispatch_meta_bytes=238128 combine_payload_bytes=311478272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311716400 ingress_bytes=421578752 link_bytes=421578752 token_bytes=623194672 weight_movement_bytes=110100480 total_nvlink_bytes=733295152 weight_to_token_ratio=0.176671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.331947875e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=296454144 dispatch_meta_bytes=224096 combine_payload_bytes=296454144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296678240 ingress_bytes=406554624 link_bytes=406554624 token_bytes=593132384 weight_movement_bytes=110100480 total_nvlink_bytes=703232864 weight_to_token_ratio=0.185625 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.070653579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=293744640 dispatch_meta_bytes=231448 combine_payload_bytes=293744640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293976088 ingress_bytes=513945600 link_bytes=513945600 token_bytes=587720728 weight_movement_bytes=220200960 total_nvlink_bytes=807921688 weight_to_token_ratio=0.374669 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.938359279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=326717440 dispatch_meta_bytes=228656 combine_payload_bytes=326717440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=326946096 ingress_bytes=436817920 link_bytes=436817920 token_bytes=653663536 weight_movement_bytes=110100480 total_nvlink_bytes=763764016 weight_to_token_ratio=0.168436 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.596982071e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276527104 dispatch_meta_bytes=219296 combine_payload_bytes=276527104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276746400 ingress_bytes=386627584 link_bytes=386627584 token_bytes=553273504 weight_movement_bytes=110100480 total_nvlink_bytes=663373984 weight_to_token_ratio=0.198998 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.724089579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=304883712 dispatch_meta_bytes=225928 combine_payload_bytes=304883712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305109640 ingress_bytes=414984192 link_bytes=414984192 token_bytes=609993352 weight_movement_bytes=110100480 total_nvlink_bytes=720093832 weight_to_token_ratio=0.180495 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.217257631e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50966 payload_rows_remote=44568 dup_rows=14570 dup_rows_remote=11842 dedup_ratio=0.222321 router_tokens_per_expert=[2, 188, 133, 43, 1856, 9, 2107, 3, 2, 0, 1, 168, 3574, 3866, 14, 3, 305, 3, 1628, 2, 16, 0, 181, 6, 277, 89, 1129, 4, 0, 21, 196, 0, 0, 0, 1, 2433, 64, 3534, 616, 10, 0, 3939, 28, 1, 175, 0, 1031, 3, 2598, 2, 71, 0, 14, 0, 383, 7, 95, 181, 1, 0, 2, 0, 40, 1713] payload_rows_per_dst=[5444, 5686, 4704, 7940, 6398, 8016, 6448, 6330] dup_rows_per_dst=[932, 262, 1154, 3224, 2728, 2338, 2008, 1924] combine_rows_per_src=[5444, 5686, 4704, 7940, 6398, 8016, 6448, 6330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49282 payload_rows_remote=42486 dup_rows=16254 dup_rows_remote=12776 dedup_ratio=0.248016 router_tokens_per_expert=[0, 1113, 0, 12, 17, 25, 522, 219, 2, 6, 6, 84, 2, 1019, 28, 30, 2226, 229, 1707, 1436, 1203, 1284, 195, 1977, 326, 6, 5, 2, 157, 0, 10, 23, 170, 10, 167, 1697, 0, 0, 1307, 1628, 5, 601, 713, 87, 2153, 37, 0, 1, 147, 2675, 6, 558, 414, 61, 1, 21, 626, 0, 0, 975, 24, 1149, 49, 3615] payload_rows_per_dst=[5096, 6476, 6796, 7342, 6838, 5484, 6374, 4876] dup_rows_per_dst=[1592, 3246, 3478, 946, 3120, 1710, 1392, 770] combine_rows_per_src=[5096, 6476, 6796, 7342, 6838, 5484, 6374, 4876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48814 payload_rows_remote=40988 dup_rows=16722 dup_rows_remote=10176 dedup_ratio=0.255157 router_tokens_per_expert=[19, 187, 26, 66, 1990, 122, 2638, 13, 91, 2, 36, 20, 2923, 2991, 0, 29, 713, 108, 1224, 22, 16, 1, 8, 18, 0, 33, 2468, 67, 0, 571, 256, 33, 83, 1, 0, 477, 56, 3286, 1268, 152, 2, 1894, 0, 14, 50, 5, 1478, 0, 2984, 18, 595, 96, 3, 0, 558, 2, 97, 141, 10, 83, 20, 9, 10, 2685] payload_rows_per_dst=[7128, 7826, 3718, 5652, 6728, 5154, 6784, 5824] dup_rows_per_dst=[2994, 6546, 502, 1204, 2420, 1042, 1728, 286] combine_rows_per_src=[7128, 7826, 3718, 5652, 6728, 5154, 6784, 5824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48084 payload_rows_remote=42554 dup_rows=17452 dup_rows_remote=15994 dedup_ratio=0.266296 router_tokens_per_expert=[4, 1423, 17, 0, 15, 29, 88, 1228, 0, 0, 1, 99, 0, 1015, 0, 121, 308, 397, 1124, 2191, 633, 912, 826, 1666, 184, 2, 11, 1, 896, 1, 82, 0, 706, 1, 351, 2837, 1, 0, 399, 1551, 0, 677, 716, 6, 3243, 0, 0, 14, 8, 1995, 6, 122, 345, 0, 54, 52, 389, 2, 0, 1843, 0, 1358, 65, 2753] payload_rows_per_dst=[5180, 4688, 6576, 6418, 4942, 7168, 5530, 7582] dup_rows_per_dst=[1586, 1116, 3224, 1442, 1076, 2144, 1458, 5406] combine_rows_per_src=[5180, 4688, 6576, 6418, 4942, 7168, 5530, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49218 payload_rows_remote=42718 dup_rows=16318 dup_rows_remote=14052 dedup_ratio=0.248993 router_tokens_per_expert=[1, 1453, 0, 1, 38, 8, 1091, 619, 0, 1, 0, 90, 3, 277, 1, 30, 551, 164, 917, 2235, 1108, 1344, 265, 2142, 140, 0, 1, 0, 501, 1, 28, 0, 333, 2, 218, 2691, 0, 0, 250, 2046, 1, 1787, 321, 22, 2103, 10, 0, 139, 30, 3257, 2, 224, 118, 0, 79, 64, 382, 1, 0, 2382, 21, 714, 66, 2495] payload_rows_per_dst=[7098, 4698, 5672, 5532, 5012, 6500, 7056, 7650] dup_rows_per_dst=[3794, 390, 2098, 800, 686, 2266, 1420, 4864] combine_rows_per_src=[7098, 4698, 5672, 5532, 5012, 6500, 7056, 7650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50410 payload_rows_remote=43370 dup_rows=15126 dup_rows_remote=14148 dedup_ratio=0.230804 router_tokens_per_expert=[0, 1529, 2, 1, 13, 0, 291, 1341, 0, 1, 0, 525, 0, 738, 0, 60, 357, 7, 1864, 921, 341, 1236, 563, 2645, 189, 0, 34, 1, 403, 1, 21, 0, 7, 0, 517, 2026, 0, 0, 365, 1002, 0, 984, 693, 12, 3546, 20, 0, 155, 1, 3510, 1, 174, 288, 2, 32, 587, 55, 1, 0, 1934, 18, 347, 47, 3360] payload_rows_per_dst=[6112, 7260, 4230, 7040, 6068, 7642, 7718, 4340] dup_rows_per_dst=[2084, 4406, 778, 978, 1766, 3178, 1472, 464] combine_rows_per_src=[6112, 7260, 4230, 7040, 6068, 7642, 7718, 4340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47226 payload_rows_remote=40726 dup_rows=18310 dup_rows_remote=16294 dedup_ratio=0.279388 router_tokens_per_expert=[0, 800, 30, 37, 130, 573, 0, 2688, 0, 2299, 0, 37, 808, 423, 2625, 34, 0, 269, 46, 69, 34, 1, 85, 3660, 28, 0, 1, 144, 44, 98, 616, 3405, 1581, 0, 4, 1420, 143, 336, 9, 1959, 0, 96, 0, 2520, 325, 51, 0, 40, 190, 5, 32, 521, 979, 2352, 21, 0, 8, 0, 176, 508, 70, 108, 329, 1] payload_rows_per_dst=[6500, 7348, 7456, 7596, 7458, 5430, 3196, 2242] dup_rows_per_dst=[2016, 5104, 872, 3916, 5310, 634, 300, 158] combine_rows_per_src=[6500, 7348, 7456, 7596, 7458, 5430, 3196, 2242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46878 payload_rows_remote=39510 dup_rows=18658 dup_rows_remote=10912 dedup_ratio=0.284698 router_tokens_per_expert=[0, 1546, 270, 19, 490, 54, 196, 687, 2, 0, 3, 22, 0, 917, 2, 111, 650, 361, 2239, 618, 631, 335, 240, 1999, 70, 0, 8, 0, 788, 13, 323, 6, 49, 0, 691, 1685, 11, 1, 995, 1386, 3, 366, 574, 75, 3634, 6, 1, 43, 9, 2653, 1, 686, 114, 9, 129, 253, 1233, 0, 0, 1543, 3, 1086, 1, 2928] payload_rows_per_dst=[5196, 5164, 6610, 3488, 5048, 7494, 6510, 7368] dup_rows_per_dst=[1328, 948, 2868, 772, 1218, 1910, 1868, 7746] combine_rows_per_src=[5196, 5164, 6610, 3488, 5048, 7494, 6510, 7368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=319463424 dispatch_meta_bytes=225640 combine_payload_bytes=319463424 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=319689064 ingress_bytes=649764864 link_bytes=649764864 token_bytes=639152488 weight_movement_bytes=330301440 total_nvlink_bytes=969453928 weight_to_token_ratio=0.516780 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.130047966e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=304539648 dispatch_meta_bytes=221048 combine_payload_bytes=304539648 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=304760696 ingress_bytes=855042048 link_bytes=855042048 token_bytes=609300344 weight_movement_bytes=550502400 total_nvlink_bytes=1159802744 weight_to_token_ratio=0.903499 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.487058752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=293801984 dispatch_meta_bytes=204656 combine_payload_bytes=293801984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=294006640 ingress_bytes=624103424 link_bytes=624103424 token_bytes=587808624 weight_movement_bytes=330301440 total_nvlink_bytes=918110064 weight_to_token_ratio=0.561920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.085418502e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=305027072 dispatch_meta_bytes=234192 combine_payload_bytes=305027072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305261264 ingress_bytes=415127552 link_bytes=415127552 token_bytes=610288336 weight_movement_bytes=110100480 total_nvlink_bytes=720388816 weight_to_token_ratio=0.180407 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.219750897e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306202624 dispatch_meta_bytes=227080 combine_payload_bytes=306202624 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=306429704 ingress_bytes=526403584 link_bytes=526403584 token_bytes=612632328 weight_movement_bytes=220200960 total_nvlink_bytes=832833288 weight_to_token_ratio=0.359434 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.155024111e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310876160 dispatch_meta_bytes=230072 combine_payload_bytes=310876160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311106232 ingress_bytes=420976640 link_bytes=420976640 token_bytes=621982392 weight_movement_bytes=110100480 total_nvlink_bytes=732082872 weight_to_token_ratio=0.177015 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.321476157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=291923968 dispatch_meta_bytes=228080 combine_payload_bytes=291923968 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=292152048 ingress_bytes=622225408 link_bytes=622225408 token_bytes=584076016 weight_movement_bytes=330301440 total_nvlink_bytes=914377456 weight_to_token_ratio=0.565511 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082152323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283207680 dispatch_meta_bytes=201688 combine_payload_bytes=283207680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283409368 ingress_bytes=503408640 link_bytes=503408640 token_bytes=566617048 weight_movement_bytes=220200960 total_nvlink_bytes=786818008 weight_to_token_ratio=0.388624 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.755104215e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48254 payload_rows_remote=42682 dup_rows=17282 dup_rows_remote=15684 dedup_ratio=0.263702 router_tokens_per_expert=[26, 911, 297, 116, 1056, 353, 0, 3823, 0, 2424, 133, 113, 307, 620, 2658, 134, 0, 12, 157, 1035, 708, 2, 17, 1654, 200, 0, 1, 154, 0, 6, 267, 1989, 1692, 5, 446, 1060, 1, 309, 19, 1165, 0, 269, 16, 1062, 1499, 2, 0, 16, 17, 8, 205, 676, 748, 2630, 0, 0, 0, 97, 334, 173, 265, 69, 812, 0] payload_rows_per_dst=[8000, 5998, 5572, 5618, 5226, 4674, 6580, 6586] dup_rows_per_dst=[5164, 1464, 1598, 1736, 2048, 1054, 1988, 2230] combine_rows_per_src=[8000, 5998, 5572, 5618, 5226, 4674, 6580, 6586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48938 payload_rows_remote=43240 dup_rows=16598 dup_rows_remote=14846 dedup_ratio=0.253265 router_tokens_per_expert=[2, 1573, 0, 11, 16, 16, 742, 565, 7, 0, 5, 65, 45, 201, 45, 228, 441, 137, 1749, 1703, 701, 1719, 1192, 1380, 447, 19, 42, 0, 2069, 6, 87, 25, 351, 19, 173, 2092, 67, 3, 220, 1263, 71, 413, 1154, 0, 1604, 226, 5, 64, 15, 2383, 59, 340, 180, 0, 19, 289, 2208, 40, 3, 911, 103, 59, 197, 2999] payload_rows_per_dst=[6242, 5698, 5984, 7406, 6416, 5516, 5944, 5732] dup_rows_per_dst=[3014, 1752, 2396, 3982, 1960, 1558, 626, 1310] combine_rows_per_src=[6242, 5698, 5984, 7406, 6416, 5516, 5944, 5732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47498 payload_rows_remote=41170 dup_rows=18038 dup_rows_remote=16658 dedup_ratio=0.275238 router_tokens_per_expert=[0, 1458, 0, 0, 3, 6, 1649, 459, 0, 4, 0, 435, 0, 1381, 0, 338, 348, 333, 2557, 505, 530, 1643, 494, 1736, 847, 0, 48, 32, 961, 0, 435, 0, 43, 0, 388, 3034, 0, 0, 220, 605, 0, 366, 86, 65, 2772, 5, 0, 421, 2, 2741, 0, 739, 37, 3, 21, 635, 274, 3, 3, 678, 5, 428, 3, 2989] payload_rows_per_dst=[5858, 6970, 5512, 6946, 6328, 6146, 6672, 3066] dup_rows_per_dst=[2302, 5466, 1650, 3678, 1380, 1284, 1684, 594] combine_rows_per_src=[5858, 6970, 5512, 6946, 6328, 6146, 6672, 3066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50204 payload_rows_remote=42338 dup_rows=15332 dup_rows_remote=12120 dedup_ratio=0.233948 router_tokens_per_expert=[1, 1430, 84, 16, 147, 1092, 2, 3742, 3, 1286, 1, 164, 1556, 1032, 3053, 12, 6, 15, 120, 1356, 262, 0, 17, 2274, 173, 5, 2, 44, 0, 3, 40, 1485, 1051, 0, 81, 1912, 31, 997, 0, 1061, 6, 127, 0, 530, 1130, 0, 1, 3, 3, 55, 19, 136, 824, 3521, 2, 0, 5, 2, 909, 307, 30, 216, 383, 3] payload_rows_per_dst=[4522, 6150, 6270, 5626, 5080, 7866, 7558, 7132] dup_rows_per_dst=[1022, 1958, 1830, 1590, 1474, 3212, 1562, 2684] combine_rows_per_src=[4522, 6150, 6270, 5626, 5080, 7866, 7558, 7132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49492 payload_rows_remote=43518 dup_rows=16044 dup_rows_remote=15576 dedup_ratio=0.244812 router_tokens_per_expert=[39, 234, 20, 321, 316, 1136, 0, 3981, 0, 2492, 2, 931, 194, 1265, 3152, 1, 0, 8, 0, 342, 153, 0, 94, 3091, 725, 2, 0, 32, 0, 3, 338, 1584, 1865, 0, 4, 2346, 0, 649, 0, 915, 0, 8, 0, 161, 1656, 0, 0, 1, 32, 25, 10, 83, 198, 2722, 3, 0, 0, 7, 1173, 302, 23, 67, 62, 0] payload_rows_per_dst=[3566, 6622, 6478, 4610, 7054, 8088, 5974, 7100] dup_rows_per_dst=[566, 2852, 898, 758, 4504, 3526, 468, 2472] combine_rows_per_src=[3566, 6622, 6478, 4610, 7054, 8088, 5974, 7100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48274 payload_rows_remote=41714 dup_rows=17262 dup_rows_remote=14944 dedup_ratio=0.263397 router_tokens_per_expert=[105, 409, 45, 3, 467, 2086, 0, 3910, 0, 765, 1, 97, 874, 1024, 2446, 3, 0, 5, 23, 634, 1, 0, 87, 3699, 35, 1, 5, 1, 0, 0, 561, 2888, 999, 0, 4, 948, 0, 1265, 0, 1649, 0, 10, 0, 1558, 768, 3, 0, 0, 65, 11, 80, 407, 1883, 1632, 21, 0, 27, 14, 857, 98, 150, 42, 99, 3] payload_rows_per_dst=[7624, 4592, 7604, 6560, 5758, 4212, 6094, 5830] dup_rows_per_dst=[4940, 1210, 2506, 2318, 2076, 466, 2104, 1642] combine_rows_per_src=[7624, 4592, 7604, 6560, 5758, 4212, 6094, 5830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=305944576 dispatch_meta_bytes=233464 combine_payload_bytes=305944576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306178040 ingress_bytes=416045056 link_bytes=416045056 token_bytes=612122616 weight_movement_bytes=110100480 total_nvlink_bytes=722223096 weight_to_token_ratio=0.179867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.235707800e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48314 payload_rows_remote=40778 dup_rows=17222 dup_rows_remote=13316 dedup_ratio=0.262787 router_tokens_per_expert=[9, 540, 22, 80, 188, 1037, 7, 3475, 0, 1714, 1, 114, 274, 1046, 3435, 5, 1, 10, 8, 969, 276, 2, 40, 3816, 263, 7, 1, 42, 0, 0, 10, 2794, 1449, 0, 76, 1694, 5, 580, 4, 273, 0, 18, 0, 731, 1172, 1, 0, 2, 6, 0, 3, 135, 1171, 2975, 1, 0, 2, 3, 1264, 21, 783, 41, 171, 1] payload_rows_per_dst=[3372, 2678, 7768, 5892, 6186, 7450, 7432, 7536] dup_rows_per_dst=[394, 202, 2476, 342, 1976, 3348, 4578, 3906] combine_rows_per_src=[3372, 2678, 7768, 5892, 6186, 7450, 7432, 7536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44992 payload_rows_remote=36856 dup_rows=20544 dup_rows_remote=11372 dedup_ratio=0.313477 router_tokens_per_expert=[3242, 1003, 0, 0, 3396, 517, 4, 21, 1, 10, 0, 15, 127, 2629, 0, 27, 4, 0, 174, 86, 4, 3716, 28, 20, 785, 2443, 38, 65, 0, 0, 0, 2856, 2, 0, 0, 289, 0, 175, 1, 1, 42, 704, 225, 3782, 14, 3, 513, 0, 66, 19, 1, 0, 366, 97, 35, 575, 32, 126, 0, 10, 541, 3727, 211, 0] payload_rows_per_dst=[8136, 5530, 7548, 5658, 6088, 8136, 2164, 1732] dup_rows_per_dst=[9172, 88, 1602, 1004, 560, 7856, 154, 108] combine_rows_per_src=[8136, 5530, 7548, 5658, 6088, 8136, 2164, 1732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=232344 combine_payload_bytes=309944320 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310176664 ingress_bytes=530145280 link_bytes=530145280 token_bytes=620120984 weight_movement_bytes=220200960 total_nvlink_bytes=840321944 weight_to_token_ratio=0.355094 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.220098358e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295106560 dispatch_meta_bytes=231312 combine_payload_bytes=295106560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=295337872 ingress_bytes=515307520 link_bytes=515307520 token_bytes=590444432 weight_movement_bytes=220200960 total_nvlink_bytes=810645392 weight_to_token_ratio=0.372941 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.962045308e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=303478784 dispatch_meta_bytes=217832 combine_payload_bytes=303478784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=303696616 ingress_bytes=413579264 link_bytes=413579264 token_bytes=607175400 weight_movement_bytes=110100480 total_nvlink_bytes=717275880 weight_to_token_ratio=0.181332 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.192823622e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=311937024 dispatch_meta_bytes=236376 combine_payload_bytes=311937024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312173400 ingress_bytes=532137984 link_bytes=532137984 token_bytes=624110424 weight_movement_bytes=220200960 total_nvlink_bytes=844311384 weight_to_token_ratio=0.352824 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.254754758e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299005952 dispatch_meta_bytes=226632 combine_payload_bytes=299005952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299232584 ingress_bytes=409106432 link_bytes=409106432 token_bytes=598238536 weight_movement_bytes=110100480 total_nvlink_bytes=708339016 weight_to_token_ratio=0.184041 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.115033717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=292296704 dispatch_meta_bytes=216376 combine_payload_bytes=292296704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292513080 ingress_bytes=402397184 link_bytes=402397184 token_bytes=584809784 weight_movement_bytes=110100480 total_nvlink_bytes=694910264 weight_to_token_ratio=0.188267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.998348859e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=192912 combine_payload_bytes=264183808 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=264376720 ingress_bytes=594485248 link_bytes=594485248 token_bytes=528560528 weight_movement_bytes=330301440 total_nvlink_bytes=858861968 weight_to_token_ratio=0.624908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.033907622e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49260 payload_rows_remote=43380 dup_rows=16276 dup_rows_remote=14972 dedup_ratio=0.248352 router_tokens_per_expert=[2099, 2081, 0, 0, 1942, 1548, 1, 2, 3, 4, 3, 1, 58, 2113, 0, 4, 7, 7, 43, 1955, 0, 1219, 330, 31, 2418, 1354, 72, 195, 28, 0, 0, 2180, 13, 25, 3, 82, 0, 104, 0, 5, 41, 1323, 67, 2883, 1, 0, 1040, 0, 0, 37, 2, 4, 20, 90, 3, 620, 9, 1256, 0, 41, 1512, 3665, 211, 13] payload_rows_per_dst=[7294, 6402, 5880, 6232, 4602, 5162, 6282, 7406] dup_rows_per_dst=[4168, 1854, 1304, 1902, 222, 2188, 786, 3852] combine_rows_per_src=[7294, 6402, 5880, 6232, 4602, 5162, 6282, 7406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47058 payload_rows_remote=40298 dup_rows=18478 dup_rows_remote=15650 dedup_ratio=0.281952 router_tokens_per_expert=[2, 344, 9, 11, 1580, 380, 82, 3982, 1, 941, 61, 402, 98, 490, 3077, 72, 44, 270, 198, 483, 76, 0, 82, 2243, 101, 123, 0, 272, 2, 17, 2231, 2196, 154, 10, 39, 714, 13, 747, 27, 1848, 0, 464, 0, 1038, 781, 8, 1, 0, 555, 60, 1478, 148, 879, 2307, 1, 5, 0, 5, 315, 962, 35, 53, 251, 0] payload_rows_per_dst=[8048, 6760, 5726, 7116, 6318, 3928, 5716, 3446] dup_rows_per_dst=[4732, 2828, 1066, 4196, 2372, 656, 2136, 492] combine_rows_per_src=[8048, 6760, 5726, 7116, 6318, 3928, 5716, 3446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49022 payload_rows_remote=46522 dup_rows=16514 dup_rows_remote=16318 dedup_ratio=0.251984 router_tokens_per_expert=[0, 70, 10, 2, 652, 1801, 0, 3006, 0, 3120, 0, 315, 436, 976, 3816, 38, 0, 63, 41, 85, 110, 0, 205, 3102, 257, 10, 0, 0, 1, 0, 94, 1793, 448, 0, 106, 1676, 0, 681, 0, 113, 0, 5, 0, 2312, 269, 0, 1, 5, 19, 6, 5, 498, 1213, 3184, 0, 0, 0, 0, 877, 270, 929, 51, 60, 37] payload_rows_per_dst=[4584, 7208, 6478, 5802, 2500, 7352, 7112, 7986] dup_rows_per_dst=[486, 2562, 938, 1860, 196, 3640, 2738, 4094] combine_rows_per_src=[4584, 7208, 6478, 5802, 2500, 7352, 7112, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52206 payload_rows_remote=47810 dup_rows=13330 dup_rows_remote=12646 dedup_ratio=0.203400 router_tokens_per_expert=[2993, 596, 0, 0, 1657, 160, 6, 9, 26, 0, 0, 36, 13, 2006, 0, 3, 0, 2, 230, 785, 2, 3211, 67, 54, 1117, 2479, 171, 29, 0, 0, 2, 2834, 3, 1, 0, 187, 0, 1352, 0, 0, 3, 338, 332, 3708, 1, 0, 1866, 0, 1, 39, 5, 0, 78, 456, 239, 549, 0, 276, 3, 16, 831, 3837, 159, 0] payload_rows_per_dst=[5718, 6006, 7120, 6148, 6886, 4396, 7880, 8052] dup_rows_per_dst=[954, 1476, 1582, 2304, 1868, 684, 2270, 2192] combine_rows_per_src=[5718, 6006, 7120, 6148, 6886, 4396, 7880, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48376 payload_rows_remote=40332 dup_rows=17160 dup_rows_remote=13650 dedup_ratio=0.261841 router_tokens_per_expert=[1475, 543, 1, 0, 1281, 901, 6, 0, 0, 0, 0, 0, 0, 3661, 0, 1, 1, 0, 269, 878, 0, 1837, 2, 1, 2475, 3689, 147, 8, 0, 0, 0, 3363, 0, 0, 0, 103, 0, 659, 2, 0, 13, 98, 404, 3793, 0, 0, 754, 0, 4, 2, 11, 3, 551, 8, 103, 1302, 0, 69, 0, 33, 766, 3506, 44, 1] payload_rows_per_dst=[2808, 7570, 5108, 7752, 7108, 2492, 8044, 7494] dup_rows_per_dst=[94, 2316, 868, 6942, 2040, 46, 3510, 1344] combine_rows_per_src=[2808, 7570, 5108, 7752, 7108, 2492, 8044, 7494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52112 payload_rows_remote=44682 dup_rows=13424 dup_rows_remote=9116 dedup_ratio=0.204834 router_tokens_per_expert=[1631, 679, 0, 0, 2144, 203, 0, 0, 2, 0, 0, 3, 9, 2049, 0, 2, 0, 0, 286, 1913, 0, 2051, 36, 0, 2635, 3081, 126, 27, 0, 0, 0, 3630, 0, 0, 0, 97, 0, 308, 0, 0, 3, 456, 378, 3513, 0, 0, 1712, 0, 2, 0, 1, 0, 145, 102, 17, 827, 0, 174, 0, 182, 515, 3743, 86, 0] payload_rows_per_dst=[4414, 6418, 6632, 7430, 7342, 4710, 7328, 7838] dup_rows_per_dst=[612, 2000, 1940, 4308, 728, 388, 1886, 1562] combine_rows_per_src=[4414, 6418, 6632, 7430, 7342, 4710, 7328, 7838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=310947840 dispatch_meta_bytes=233408 combine_payload_bytes=310947840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311181248 ingress_bytes=421048320 link_bytes=421048320 token_bytes=622129088 weight_movement_bytes=110100480 total_nvlink_bytes=732229568 weight_to_token_ratio=0.176974 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.322722790e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44040 payload_rows_remote=38522 dup_rows=21496 dup_rows_remote=19220 dedup_ratio=0.328003 router_tokens_per_expert=[1, 368, 77, 1368, 2, 0, 0, 941, 0, 372, 2, 2, 653, 0, 1140, 0, 111, 0, 463, 1454, 0, 1128, 77, 0, 1729, 0, 4018, 579, 23, 1221, 1212, 2684, 2847, 12, 52, 1, 0, 1, 32, 0, 0, 7, 0, 116, 4, 2542, 145, 11, 742, 0, 5, 3, 704, 0, 2, 3817, 1759, 0, 1, 146, 168, 1, 24, 1] payload_rows_per_dst=[5518, 1996, 8156, 7848, 5800, 8106, 2688, 3928] dup_rows_per_dst=[2276, 62, 6346, 7048, 90, 5178, 224, 272] combine_rows_per_src=[5518, 1996, 8156, 7848, 5800, 8106, 2688, 3928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47598 payload_rows_remote=39532 dup_rows=17938 dup_rows_remote=14420 dedup_ratio=0.273712 router_tokens_per_expert=[485, 490, 2, 3, 1066, 104, 6, 0, 18, 9, 0, 6, 6, 3568, 0, 1, 2, 6, 120, 600, 1, 3670, 285, 0, 1313, 3588, 0, 18, 0, 2, 0, 3565, 4, 2, 0, 225, 0, 82, 0, 1, 1, 967, 13, 3124, 6, 8, 1638, 0, 0, 4, 9, 6, 238, 14, 171, 1529, 4, 313, 0, 19, 1322, 3862, 272, 0] payload_rows_per_dst=[1204, 7432, 7626, 3388, 8040, 4360, 7482, 8066] dup_rows_per_dst=[6, 1916, 1742, 248, 6894, 906, 2708, 3518] combine_rows_per_src=[1204, 7432, 7626, 3388, 8040, 4360, 7482, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=288856064 dispatch_meta_bytes=223792 combine_payload_bytes=288856064 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=289079856 ingress_bytes=619157504 link_bytes=619157504 token_bytes=577935920 weight_movement_bytes=330301440 total_nvlink_bytes=908237360 weight_to_token_ratio=0.571519 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076816733e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=333469696 dispatch_meta_bytes=251360 combine_payload_bytes=333469696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=333721056 ingress_bytes=553670656 link_bytes=553670656 token_bytes=667190752 weight_movement_bytes=220200960 total_nvlink_bytes=887391712 weight_to_token_ratio=0.330042 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.629243340e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=342702080 dispatch_meta_bytes=241824 combine_payload_bytes=342702080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342943904 ingress_bytes=562903040 link_bytes=562903040 token_bytes=685645984 weight_movement_bytes=220200960 total_nvlink_bytes=905846944 weight_to_token_ratio=0.321158 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.789809682e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289099776 dispatch_meta_bytes=215928 combine_payload_bytes=289099776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289315704 ingress_bytes=399200256 link_bytes=399200256 token_bytes=578415480 weight_movement_bytes=110100480 total_nvlink_bytes=688515960 weight_to_token_ratio=0.190348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.942749023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=320280576 dispatch_meta_bytes=215192 combine_payload_bytes=320280576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=320495768 ingress_bytes=650582016 link_bytes=650582016 token_bytes=640776344 weight_movement_bytes=330301440 total_nvlink_bytes=971077784 weight_to_token_ratio=0.515471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.131469128e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276125696 dispatch_meta_bytes=230968 combine_payload_bytes=276125696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276356664 ingress_bytes=386226176 link_bytes=386226176 token_bytes=552482360 weight_movement_bytes=110100480 total_nvlink_bytes=662582840 weight_to_token_ratio=0.199283 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.717108434e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283365376 dispatch_meta_bytes=215808 combine_payload_bytes=283365376 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=283581184 ingress_bytes=613666816 link_bytes=613666816 token_bytes=566946560 weight_movement_bytes=330301440 total_nvlink_bytes=897248000 weight_to_token_ratio=0.582597 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067267524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48290 payload_rows_remote=42542 dup_rows=17246 dup_rows_remote=14922 dedup_ratio=0.263153 router_tokens_per_expert=[62, 614, 1, 992, 3, 13, 2, 353, 3, 2306, 0, 4, 215, 21, 3559, 15, 4, 0, 114, 1229, 0, 1384, 25, 16, 137, 5, 3331, 384, 32, 1281, 207, 2707, 1686, 53, 0, 7, 8, 0, 146, 1, 0, 12, 6, 1150, 13, 1795, 298, 27, 411, 1, 49, 0, 918, 25, 3, 4017, 2017, 1, 9, 529, 128, 326, 37, 76] payload_rows_per_dst=[7556, 4856, 5748, 6892, 4206, 5470, 7700, 5862] dup_rows_per_dst=[3356, 558, 2324, 4520, 416, 1870, 2410, 1792] combine_rows_per_src=[7556, 4856, 5748, 6892, 4206, 5470, 7700, 5862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50264 payload_rows_remote=44760 dup_rows=15272 dup_rows_remote=13994 dedup_ratio=0.233032 router_tokens_per_expert=[155, 483, 3, 1578, 12, 0, 0, 969, 3, 2942, 10, 1, 482, 53, 2242, 0, 0, 2, 145, 1336, 6, 1195, 45, 1, 1089, 3, 3944, 162, 35, 803, 585, 2381, 856, 12, 44, 12, 7, 17, 206, 0, 0, 6, 0, 1773, 8, 1338, 265, 1, 409, 1, 9, 7, 273, 2, 2, 4037, 2513, 0, 48, 138, 70, 9, 0, 40] payload_rows_per_dst=[5102, 7576, 4490, 4532, 7994, 5504, 8122, 6944] dup_rows_per_dst=[1298, 3890, 970, 822, 2202, 1278, 1358, 3454] combine_rows_per_src=[5102, 7576, 4490, 4532, 7994, 5504, 8122, 6944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45004 payload_rows_remote=39008 dup_rows=20532 dup_rows_remote=19864 dedup_ratio=0.313293 router_tokens_per_expert=[981, 2630, 0, 24, 2097, 220, 93, 19, 0, 0, 2, 0, 208, 2711, 0, 89, 6, 2, 21, 1127, 1, 3283, 8, 78, 1683, 1180, 8, 369, 8, 19, 1, 2552, 0, 105, 0, 124, 1, 242, 11, 6, 997, 1210, 4, 2445, 0, 10, 1611, 77, 8, 60, 0, 2, 2, 55, 93, 1541, 1, 166, 32, 162, 724, 3654, 5, 0] payload_rows_per_dst=[7412, 5996, 7322, 5272, 5526, 8050, 3356, 2070] dup_rows_per_dst=[4072, 668, 1730, 1264, 556, 11966, 166, 110] combine_rows_per_src=[7412, 5996, 7322, 5272, 5526, 8050, 3356, 2070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49774 payload_rows_remote=41616 dup_rows=15762 dup_rows_remote=12984 dedup_ratio=0.240509 router_tokens_per_expert=[294, 411, 0, 2558, 4, 0, 32, 1032, 2, 2098, 0, 6, 859, 59, 2899, 0, 29, 0, 136, 1741, 0, 676, 0, 0, 1475, 0, 3874, 305, 226, 1505, 240, 2029, 851, 0, 0, 28, 0, 0, 78, 0, 0, 2, 0, 244, 58, 1763, 537, 0, 10, 0, 9, 0, 15, 0, 16, 4077, 2408, 0, 9, 43, 73, 11, 46, 0] payload_rows_per_dst=[6350, 7708, 4498, 4196, 7848, 4580, 8158, 6436] dup_rows_per_dst=[2312, 4138, 666, 624, 1814, 628, 2778, 2802] combine_rows_per_src=[6350, 7708, 4498, 4196, 7848, 4580, 8158, 6436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50830 payload_rows_remote=44522 dup_rows=14706 dup_rows_remote=13938 dedup_ratio=0.224396 router_tokens_per_expert=[2113, 2528, 0, 0, 1603, 90, 0, 5, 0, 0, 0, 1, 28, 3188, 0, 112, 1, 0, 156, 1531, 0, 1979, 16, 83, 1266, 1537, 36, 2, 0, 1, 0, 2943, 11, 0, 0, 524, 0, 60, 0, 0, 3, 587, 1336, 3645, 1, 0, 171, 0, 0, 0, 3, 0, 329, 8, 44, 1731, 2, 749, 0, 232, 132, 3963, 17, 1] payload_rows_per_dst=[6868, 7114, 5880, 4806, 6308, 3868, 8000, 7986] dup_rows_per_dst=[2604, 2750, 1652, 878, 768, 328, 3520, 2206] combine_rows_per_src=[6868, 7114, 5880, 4806, 6308, 3868, 8000, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304941056 dispatch_meta_bytes=229856 combine_payload_bytes=304941056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305170912 ingress_bytes=415041536 link_bytes=415041536 token_bytes=610111968 weight_movement_bytes=110100480 total_nvlink_bytes=720212448 weight_to_token_ratio=0.180459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.218254937e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49640 payload_rows_remote=45356 dup_rows=15896 dup_rows_remote=14568 dedup_ratio=0.242554 router_tokens_per_expert=[327, 50, 0, 1269, 0, 0, 0, 1572, 1, 1929, 0, 2, 806, 10, 2710, 0, 43, 0, 373, 1206, 0, 766, 0, 0, 287, 0, 4006, 54, 181, 1289, 995, 2190, 1201, 41, 2, 0, 1, 0, 64, 0, 0, 26, 0, 814, 5, 2239, 572, 17, 30, 0, 31, 0, 706, 1, 1, 4030, 2460, 0, 11, 71, 166, 93, 85, 35] payload_rows_per_dst=[5222, 7260, 3954, 4284, 8102, 5900, 8102, 6816] dup_rows_per_dst=[1214, 3656, 822, 1328, 2528, 1446, 1496, 3406] combine_rows_per_src=[5222, 7260, 3954, 4284, 8102, 5900, 8102, 6816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50172 payload_rows_remote=43270 dup_rows=15364 dup_rows_remote=11604 dedup_ratio=0.234436 router_tokens_per_expert=[11, 408, 1120, 119, 0, 2519, 0, 2, 2422, 1005, 1, 0, 0, 151, 0, 1481, 3, 116, 0, 1, 5, 59, 3381, 20, 1, 0, 422, 0, 5, 2324, 1586, 1, 0, 0, 0, 0, 33, 0, 85, 2015, 53, 1240, 0, 268, 1534, 2, 178, 3, 0, 3, 0, 46, 3510, 2138, 72, 962, 66, 14, 2, 371, 553, 2330, 127, 0] payload_rows_per_dst=[6902, 7040, 6834, 5248, 7880, 5274, 5334, 5660] dup_rows_per_dst=[3760, 3080, 336, 1126, 3406, 1282, 1108, 1266] combine_rows_per_src=[6902, 7040, 6834, 5248, 7880, 5274, 5334, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49688 payload_rows_remote=43006 dup_rows=15848 dup_rows_remote=12884 dedup_ratio=0.241821 router_tokens_per_expert=[2, 437, 0, 1523, 0, 0, 2, 312, 0, 1991, 1, 0, 1028, 33, 1546, 0, 1, 1, 982, 1280, 1, 1888, 0, 0, 1633, 1, 3713, 46, 810, 509, 1418, 2373, 1301, 167, 3, 3, 2, 0, 124, 0, 5, 8, 0, 1280, 20, 1088, 671, 0, 80, 0, 8, 0, 229, 7, 6, 3785, 2155, 0, 0, 157, 103, 29, 2, 4] payload_rows_per_dst=[4080, 6370, 5852, 5994, 7820, 5058, 7832, 6682] dup_rows_per_dst=[472, 2310, 2454, 2340, 2806, 1086, 1416, 2964] combine_rows_per_src=[4080, 6370, 5852, 5994, 7820, 5058, 7832, 6682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298303488 dispatch_meta_bytes=218400 combine_payload_bytes=298303488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=298521888 ingress_bytes=518504448 link_bytes=518504448 token_bytes=596825376 weight_movement_bytes=220200960 total_nvlink_bytes=817026336 weight_to_token_ratio=0.368954 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.017645144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=320839680 dispatch_meta_bytes=235016 combine_payload_bytes=320839680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321074696 ingress_bytes=430940160 link_bytes=430940160 token_bytes=641914376 weight_movement_bytes=110100480 total_nvlink_bytes=752014856 weight_to_token_ratio=0.171519 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.494758157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=235488 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279844832 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559454176 weight_movement_bytes=110100480 total_nvlink_bytes=669554656 weight_to_token_ratio=0.196800 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=319133696 dispatch_meta_bytes=233840 combine_payload_bytes=319133696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=319367536 ingress_bytes=539334656 link_bytes=539334656 token_bytes=638501232 weight_movement_bytes=220200960 total_nvlink_bytes=858702192 weight_to_token_ratio=0.344872 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.379916721e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=325111808 dispatch_meta_bytes=239696 combine_payload_bytes=325111808 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=325351504 ingress_bytes=875614208 link_bytes=875614208 token_bytes=650463312 weight_movement_bytes=550502400 total_nvlink_bytes=1200965712 weight_to_token_ratio=0.846324 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.522837122e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310159360 dispatch_meta_bytes=219496 combine_payload_bytes=310159360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=310378856 ingress_bytes=530360320 link_bytes=530360320 token_bytes=620538216 weight_movement_bytes=220200960 total_nvlink_bytes=840739176 weight_to_token_ratio=0.354855 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.223838258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=308267008 dispatch_meta_bytes=223560 combine_payload_bytes=308267008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308490568 ingress_bytes=418367488 link_bytes=418367488 token_bytes=616757576 weight_movement_bytes=110100480 total_nvlink_bytes=726858056 weight_to_token_ratio=0.178515 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.276098713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47022 payload_rows_remote=38922 dup_rows=18514 dup_rows_remote=13282 dedup_ratio=0.282501 router_tokens_per_expert=[21, 502, 2447, 16, 0, 2279, 9, 6, 1844, 347, 66, 4, 0, 0, 1, 2210, 9, 30, 0, 0, 104, 2027, 3767, 729, 0, 5, 869, 10, 43, 2720, 1449, 15, 6, 0, 0, 43, 23, 0, 8, 1593, 10, 52, 1, 1299, 160, 12, 16, 20, 28, 3, 0, 421, 3084, 1351, 73, 437, 163, 1, 47, 882, 760, 495, 251, 0] payload_rows_per_dst=[6916, 6588, 8100, 7022, 7230, 2932, 4054, 4180] dup_rows_per_dst=[3644, 2356, 5232, 3200, 2284, 208, 572, 1018] combine_rows_per_src=[6916, 6588, 8100, 7022, 7230, 2932, 4054, 4180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46704 payload_rows_remote=38622 dup_rows=18832 dup_rows_remote=15258 dedup_ratio=0.287354 router_tokens_per_expert=[0, 1203, 75, 44, 0, 536, 0, 0, 1940, 1666, 0, 0, 0, 159, 0, 1888, 6, 0, 0, 22, 236, 65, 3678, 36, 1, 0, 1314, 21, 0, 1300, 218, 0, 0, 0, 333, 12, 28, 0, 13, 843, 18, 2881, 2, 1772, 420, 0, 14, 0, 123, 100, 0, 39, 3945, 2233, 0, 1621, 80, 26, 25, 582, 991, 2259, 0, 0] payload_rows_per_dst=[2586, 6282, 7454, 4874, 2330, 7724, 8082, 7372] dup_rows_per_dst=[58, 2320, 632, 834, 128, 6266, 3574, 5020] combine_rows_per_src=[2586, 6282, 7454, 4874, 2330, 7724, 8082, 7372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50988 payload_rows_remote=44510 dup_rows=14548 dup_rows_remote=12050 dedup_ratio=0.221985 router_tokens_per_expert=[1, 402, 2220, 47, 7, 2050, 10, 6, 3057, 879, 0, 3, 0, 59, 0, 2201, 6, 13, 1, 0, 34, 229, 3579, 10, 1, 0, 96, 0, 3, 1864, 954, 0, 0, 1, 25, 0, 2, 14, 73, 1547, 4, 133, 1, 748, 1321, 0, 78, 2, 28, 42, 0, 203, 3625, 1291, 2, 2615, 633, 8, 133, 591, 680, 1216, 19, 1] payload_rows_per_dst=[5426, 7526, 7296, 4990, 7338, 6478, 5744, 6190] dup_rows_per_dst=[1260, 3270, 448, 846, 2558, 2498, 714, 2954] combine_rows_per_src=[5426, 7526, 7296, 4990, 7338, 6478, 5744, 6190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48064 payload_rows_remote=42312 dup_rows=17472 dup_rows_remote=17322 dedup_ratio=0.266602 router_tokens_per_expert=[71, 390, 1, 2013, 25, 4, 34, 1148, 1, 2847, 2, 1, 82, 13, 1065, 5, 16, 1, 51, 2600, 4, 972, 45, 76, 7, 4, 3569, 1216, 615, 597, 357, 2407, 2251, 8, 7, 56, 27, 25, 40, 16, 0, 0, 3, 614, 30, 172, 506, 23, 906, 7, 394, 1, 338, 3, 267, 3591, 2426, 0, 13, 641, 11, 43, 40, 70] payload_rows_per_dst=[6416, 5752, 7910, 6884, 4776, 7406, 3216, 5704] dup_rows_per_dst=[3086, 150, 6758, 3522, 84, 2472, 616, 784] combine_rows_per_src=[6416, 5752, 7910, 6884, 4776, 7406, 3216, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52278 payload_rows_remote=44088 dup_rows=13258 dup_rows_remote=12398 dedup_ratio=0.202301 router_tokens_per_expert=[13, 59, 0, 2751, 0, 1, 0, 2815, 0, 1036, 0, 0, 566, 102, 2923, 0, 4, 0, 292, 1451, 0, 537, 10, 0, 671, 0, 4093, 4, 885, 568, 223, 2127, 171, 76, 105, 0, 0, 0, 80, 0, 0, 13, 0, 3481, 3, 886, 394, 0, 108, 0, 248, 0, 399, 0, 2, 3953, 1254, 0, 5, 331, 62, 47, 16, 3] payload_rows_per_dst=[7666, 6924, 3946, 3984, 8190, 7406, 8040, 6122] dup_rows_per_dst=[3612, 2330, 642, 718, 860, 2148, 1380, 1568] combine_rows_per_src=[7666, 6924, 3946, 3984, 8190, 7406, 8040, 6122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=208816 combine_payload_bytes=278992896 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279201712 ingress_bytes=499193856 link_bytes=499193856 token_bytes=558194608 weight_movement_bytes=220200960 total_nvlink_bytes=778395568 weight_to_token_ratio=0.394488 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681802189e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50298 payload_rows_remote=43496 dup_rows=15238 dup_rows_remote=12190 dedup_ratio=0.232513 router_tokens_per_expert=[0, 2240, 382, 65, 0, 753, 0, 0, 2288, 688, 0, 0, 0, 414, 0, 2012, 0, 0, 0, 7, 84, 183, 4054, 167, 7, 0, 1469, 20, 0, 2542, 887, 0, 0, 0, 3, 35, 19, 0, 90, 1539, 0, 198, 10, 875, 1306, 2, 134, 0, 43, 3, 0, 339, 3996, 2572, 0, 1047, 18, 23, 1, 1479, 403, 362, 9, 0] payload_rows_per_dst=[5766, 6462, 8112, 6802, 8086, 5054, 6036, 3980] dup_rows_per_dst=[1114, 2528, 878, 3048, 3278, 1810, 1972, 610] combine_rows_per_src=[5766, 6462, 8112, 6802, 8086, 5054, 6036, 3980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49428 payload_rows_remote=46176 dup_rows=16108 dup_rows_remote=15746 dedup_ratio=0.245789 router_tokens_per_expert=[0, 14, 0, 272, 7, 471, 1, 1, 0, 50, 234, 8, 13, 1347, 30, 9, 0, 0, 350, 3, 181, 47, 50, 33, 897, 101, 1185, 1041, 0, 0, 3, 462, 3337, 10, 443, 1413, 750, 2, 0, 23, 412, 19, 0, 2734, 4044, 0, 9, 1101, 11, 56, 20, 0, 1119, 8, 1, 2, 2491, 0, 0, 3760, 0, 841, 930, 2422] payload_rows_per_dst=[3252, 8114, 7736, 7414, 4608, 6758, 6126, 5420] dup_rows_per_dst=[362, 3356, 2794, 4556, 674, 1792, 1290, 1284] combine_rows_per_src=[3252, 8114, 7736, 7414, 4608, 6758, 6126, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48414 payload_rows_remote=41400 dup_rows=17122 dup_rows_remote=13488 dedup_ratio=0.261261 router_tokens_per_expert=[1, 1149, 2471, 1, 0, 1384, 7, 0, 2197, 843, 2, 1, 0, 3, 2, 2538, 0, 0, 0, 1, 108, 132, 2811, 4, 19, 2, 1647, 0, 1, 3096, 731, 94, 0, 0, 29, 16, 1, 0, 1, 973, 72, 604, 2, 418, 489, 3, 29, 0, 134, 6, 34, 250, 3775, 2348, 5, 1358, 171, 13, 30, 139, 410, 2174, 39, 0] payload_rows_per_dst=[6212, 6438, 4642, 7734, 2022, 6390, 7962, 7014] dup_rows_per_dst=[1046, 2426, 224, 4114, 18, 1920, 3740, 3634] combine_rows_per_src=[6212, 6438, 4642, 7734, 2022, 6390, 7962, 7014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276842496 dispatch_meta_bytes=215520 combine_payload_bytes=276842496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277058016 ingress_bytes=607143936 link_bytes=607143936 token_bytes=553900512 weight_movement_bytes=330301440 total_nvlink_bytes=884201952 weight_to_token_ratio=0.596319 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055923163e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=303292416 dispatch_meta_bytes=238536 combine_payload_bytes=303292416 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=303530952 ingress_bytes=523493376 link_bytes=523493376 token_bytes=606823368 weight_movement_bytes=220200960 total_nvlink_bytes=827024328 weight_to_token_ratio=0.362875 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.104410807e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=319047680 dispatch_meta_bytes=226240 combine_payload_bytes=319047680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319273920 ingress_bytes=429148160 link_bytes=429148160 token_bytes=638321600 weight_movement_bytes=110100480 total_nvlink_bytes=748422080 weight_to_token_ratio=0.172484 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.463592330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=316022784 dispatch_meta_bytes=225944 combine_payload_bytes=316022784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316248728 ingress_bytes=426123264 link_bytes=426123264 token_bytes=632271512 weight_movement_bytes=110100480 total_nvlink_bytes=742371992 weight_to_token_ratio=0.174135 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.410984413e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=222744 combine_payload_bytes=311779328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312002072 ingress_bytes=531980288 link_bytes=531980288 token_bytes=623781400 weight_movement_bytes=220200960 total_nvlink_bytes=843982360 weight_to_token_ratio=0.353010 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.252012166e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=330989568 dispatch_meta_bytes=247688 combine_payload_bytes=330989568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331237256 ingress_bytes=441090048 link_bytes=441090048 token_bytes=662226824 weight_movement_bytes=110100480 total_nvlink_bytes=772327304 weight_to_token_ratio=0.166258 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.671281403e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=296755200 dispatch_meta_bytes=219552 combine_payload_bytes=296755200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296974752 ingress_bytes=406855680 link_bytes=406855680 token_bytes=593729952 weight_movement_bytes=110100480 total_nvlink_bytes=703830432 weight_to_token_ratio=0.185439 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.075889438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46002 payload_rows_remote=38562 dup_rows=19534 dup_rows_remote=16850 dedup_ratio=0.298065 router_tokens_per_expert=[89, 21, 4, 2191, 105, 265, 0, 15, 1, 519, 253, 1, 0, 1792, 49, 26, 1, 2, 26, 2, 190, 7, 2, 16, 280, 164, 21, 3647, 2, 1, 0, 222, 3069, 2, 9, 351, 1296, 58, 0, 64, 2, 35, 5, 248, 4055, 0, 42, 1276, 24, 72, 296, 0, 2284, 1, 438, 3, 2280, 0, 0, 3484, 3, 1332, 569, 1556] payload_rows_per_dst=[7928, 8166, 7440, 1272, 7252, 3104, 7014, 3826] dup_rows_per_dst=[4746, 5226, 2684, 108, 2446, 112, 3782, 430] combine_rows_per_src=[7928, 8166, 7440, 1272, 7252, 3104, 7014, 3826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5875802720 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2939030112 ingress_bytes=4808480768 link_bytes=4808480768 total_nvlink_bytes=7747510880 weight_to_token_ratio=0.318545 prefetch_share_of_ingress=0.206074 dedup_ratio=0.281015 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.362738915e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49308 payload_rows_remote=44524 dup_rows=16228 dup_rows_remote=15524 dedup_ratio=0.247620 router_tokens_per_expert=[0, 1318, 1949, 369, 0, 2983, 3, 0, 631, 1394, 0, 39, 15, 7, 1, 657, 2, 0, 9, 0, 32, 236, 3231, 58, 4, 74, 11, 2, 55, 3621, 1788, 2, 10, 0, 353, 12, 0, 182, 672, 256, 30, 134, 4, 1621, 600, 3, 36, 39, 2, 10, 326, 1753, 1885, 1203, 46, 690, 975, 45, 51, 23, 2137, 1114, 28, 37] payload_rows_per_dst=[7606, 4784, 6824, 7750, 5534, 4270, 6068, 6472] dup_rows_per_dst=[5638, 704, 312, 3364, 1206, 664, 1992, 2348] combine_rows_per_src=[7606, 4784, 6824, 7750, 5534, 4270, 6068, 6472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49766 payload_rows_remote=41706 dup_rows=15770 dup_rows_remote=12028 dedup_ratio=0.240631 router_tokens_per_expert=[21, 70, 0, 1804, 44, 278, 1, 4, 0, 13, 788, 24, 5, 1455, 85, 11, 7, 0, 5, 39, 33, 2, 3, 414, 1930, 39, 503, 3729, 0, 1, 3, 384, 3337, 0, 38, 1339, 123, 62, 0, 7, 122, 9, 0, 647, 4010, 0, 88, 1025, 42, 74, 538, 0, 1606, 1, 34, 1, 1811, 1, 1, 3352, 12, 356, 1389, 1048] payload_rows_per_dst=[7912, 4104, 6922, 4950, 7486, 8060, 5740, 4592] dup_rows_per_dst=[3990, 658, 788, 770, 2326, 3742, 2474, 1022] combine_rows_per_src=[7912, 4104, 6922, 4950, 7486, 8060, 5740, 4592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47660 payload_rows_remote=41860 dup_rows=17876 dup_rows_remote=15804 dedup_ratio=0.272766 router_tokens_per_expert=[0, 167, 0, 1121, 199, 91, 1, 0, 10, 5, 56, 0, 4, 732, 238, 43, 0, 0, 15, 0, 25, 0, 1, 193, 335, 6, 364, 3431, 0, 0, 3, 1985, 3770, 0, 0, 28, 522, 1, 0, 79, 0, 137, 1, 653, 4081, 0, 162, 668, 86, 23, 771, 0, 809, 10, 40, 1, 2196, 0, 0, 3324, 0, 3644, 1148, 1589] payload_rows_per_dst=[4210, 2048, 6740, 6986, 5670, 8178, 5800, 8028] dup_rows_per_dst=[1122, 128, 376, 3088, 848, 3226, 2072, 7016] combine_rows_per_src=[4210, 2048, 6740, 6986, 5670, 8178, 5800, 8028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276412416 dispatch_meta_bytes=221648 combine_payload_bytes=276412416 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=276634064 ingress_bytes=496613376 link_bytes=496613376 token_bytes=553046480 weight_movement_bytes=220200960 total_nvlink_bytes=773247440 weight_to_token_ratio=0.398160 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.636923397e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49392 payload_rows_remote=41470 dup_rows=16144 dup_rows_remote=11828 dedup_ratio=0.246338 router_tokens_per_expert=[0, 1617, 909, 310, 0, 964, 0, 1, 617, 1237, 1, 0, 0, 11, 0, 3493, 26, 2, 0, 3, 8, 4, 4057, 6, 0, 1, 639, 0, 7, 1194, 441, 10, 0, 0, 24, 7, 3, 0, 22, 2563, 2, 505, 0, 502, 1859, 22, 0, 0, 0, 32, 0, 329, 3500, 1781, 52, 2188, 3, 3, 0, 1082, 627, 2049, 55, 0] payload_rows_per_dst=[6010, 3438, 8116, 4040, 7922, 7788, 6184, 5894] dup_rows_per_dst=[1592, 294, 96, 544, 4316, 4978, 2400, 1924] combine_rows_per_src=[6010, 3438, 8116, 4040, 7922, 7788, 6184, 5894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50626 payload_rows_remote=45396 dup_rows=14910 dup_rows_remote=13970 dedup_ratio=0.227509 router_tokens_per_expert=[0, 20, 0, 638, 580, 42, 0, 15, 0, 162, 104, 2, 1, 419, 422, 14, 0, 1, 20, 35, 82, 24, 0, 3, 1422, 2, 382, 3588, 0, 0, 4, 1275, 2957, 2, 13, 1706, 361, 0, 0, 65, 189, 94, 0, 2087, 4084, 0, 112, 1950, 131, 1, 607, 0, 1238, 0, 1, 0, 1302, 0, 0, 3056, 0, 1162, 631, 1762] payload_rows_per_dst=[7482, 8172, 6782, 5230, 7306, 6526, 4910, 4218] dup_rows_per_dst=[2284, 2244, 1984, 940, 2902, 2338, 1650, 568] combine_rows_per_src=[7482, 8172, 6782, 5230, 7306, 6526, 4910, 4218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=319148032 dispatch_meta_bytes=240192 combine_payload_bytes=319148032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=319388224 ingress_bytes=539348992 link_bytes=539348992 token_bytes=638536256 weight_movement_bytes=220200960 total_nvlink_bytes=858737216 weight_to_token_ratio=0.344853 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.380166048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298948608 dispatch_meta_bytes=214936 combine_payload_bytes=298948608 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=299163544 ingress_bytes=519149568 link_bytes=519149568 token_bytes=598112152 weight_movement_bytes=220200960 total_nvlink_bytes=818313112 weight_to_token_ratio=0.368160 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.028864842e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43842 payload_rows_remote=35818 dup_rows=21694 dup_rows_remote=11626 dedup_ratio=0.331024 router_tokens_per_expert=[1, 122, 0, 2671, 40, 402, 0, 1, 0, 1, 184, 4, 0, 441, 115, 5, 11, 0, 24, 5, 54, 4, 0, 138, 327, 170, 2159, 3366, 0, 1, 0, 1138, 3335, 18, 51, 569, 694, 5, 0, 11, 7, 106, 10, 60, 4041, 1, 11, 889, 21, 53, 228, 2, 1299, 4, 111, 1, 1958, 1, 0, 2269, 0, 2631, 1753, 1245] payload_rows_per_dst=[5774, 1376, 4764, 7918, 2538, 8122, 5326, 8024] dup_rows_per_dst=[700, 124, 246, 6404, 158, 2128, 1866, 10068] combine_rows_per_src=[5774, 1376, 4764, 7918, 2538, 8122, 5326, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=300052480 dispatch_meta_bytes=230656 combine_payload_bytes=300052480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300283136 ingress_bytes=410152960 link_bytes=410152960 token_bytes=600335616 weight_movement_bytes=110100480 total_nvlink_bytes=710436096 weight_to_token_ratio=0.183398 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.133234560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5950174768 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=2976228912 ingress_bytes=4845654016 link_bytes=4845654016 total_nvlink_bytes=7821882928 weight_to_token_ratio=0.314564 prefetch_share_of_ingress=0.249936 dedup_ratio=0.272986 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.427389307e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=297256960 dispatch_meta_bytes=213192 combine_payload_bytes=297256960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297470152 ingress_bytes=407357440 link_bytes=407357440 token_bytes=594727112 weight_movement_bytes=110100480 total_nvlink_bytes=704827592 weight_to_token_ratio=0.185128 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.084615869e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=325398528 dispatch_meta_bytes=237464 combine_payload_bytes=325398528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=325635992 ingress_bytes=545599488 link_bytes=545599488 token_bytes=651034520 weight_movement_bytes=220200960 total_nvlink_bytes=871235480 weight_to_token_ratio=0.338232 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.488872453e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6112541408 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3057410784 ingress_bytes=4816738304 link_bytes=4816738304 total_nvlink_bytes=7874149088 weight_to_token_ratio=0.288196 prefetch_share_of_ingress=0.205721 dedup_ratio=0.251477 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.377100128e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=256743424 dispatch_meta_bytes=189776 combine_payload_bytes=256743424 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=256933200 ingress_bytes=697145344 link_bytes=697145344 token_bytes=513676624 weight_movement_bytes=440401920 total_nvlink_bytes=954078544 weight_to_token_ratio=0.857352 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.212450414e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5877383416 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=2939822328 ingress_bytes=5139570688 link_bytes=5139570688 total_nvlink_bytes=8079393016 weight_to_token_ratio=0.374658 prefetch_share_of_ingress=0.214221 dedup_ratio=0.273083 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.938558741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6237232704 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3119769152 ingress_bytes=5319473152 link_bytes=5319473152 total_nvlink_bytes=8439242304 weight_to_token_ratio=0.353043 prefetch_share_of_ingress=0.186279 dedup_ratio=0.245517 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.251438714e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48560 payload_rows_remote=40382 dup_rows=16976 dup_rows_remote=11484 dedup_ratio=0.259033 router_tokens_per_expert=[121, 2, 1, 203, 259, 34, 0, 3, 3, 206, 198, 10, 27, 2224, 5, 99, 3, 14, 605, 0, 75, 272, 169, 11, 322, 60, 294, 1090, 45, 7, 6, 2757, 3476, 648, 29, 332, 1251, 31, 1, 70, 5, 13, 54, 1355, 4063, 9, 171, 1503, 206, 14, 65, 1, 3654, 0, 604, 112, 223, 11, 0, 3242, 29, 293, 1387, 791] payload_rows_per_dst=[3188, 8178, 7286, 6674, 6506, 5200, 7740, 3788] dup_rows_per_dst=[238, 5492, 2082, 2968, 2510, 1020, 2018, 648] combine_rows_per_src=[3188, 8178, 7286, 6674, 6506, 5200, 7740, 3788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5689391528 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2845774248 ingress_bytes=4935526400 link_bytes=4935526400 total_nvlink_bytes=7781300648 weight_to_token_ratio=0.367686 prefetch_share_of_ingress=0.200770 dedup_ratio=0.287134 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.583692164e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49382 payload_rows_remote=41218 dup_rows=16154 dup_rows_remote=12012 dedup_ratio=0.246490 router_tokens_per_expert=[0, 53, 0, 951, 1624, 108, 0, 0, 0, 143, 629, 5, 29, 80, 498, 28, 0, 0, 68, 4, 13, 1, 0, 53, 1348, 95, 1009, 3438, 0, 0, 0, 677, 4057, 163, 40, 547, 638, 6, 0, 0, 3, 992, 14, 17, 4096, 2, 15, 841, 3, 1, 86, 0, 876, 1, 89, 0, 2808, 0, 0, 2995, 0, 805, 2342, 477] payload_rows_per_dst=[7616, 6008, 6282, 5070, 8164, 4496, 6286, 5460] dup_rows_per_dst=[4732, 2142, 630, 1188, 4142, 734, 1442, 1144] combine_rows_per_src=[7616, 6008, 6282, 5070, 8164, 4496, 6286, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289458176 dispatch_meta_bytes=207464 combine_payload_bytes=289458176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289665640 ingress_bytes=399558656 link_bytes=399558656 token_bytes=579123816 weight_movement_bytes=110100480 total_nvlink_bytes=689224296 weight_to_token_ratio=0.190116 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.948982189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=295450624 dispatch_meta_bytes=212920 combine_payload_bytes=295450624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=295663544 ingress_bytes=625752064 link_bytes=625752064 token_bytes=591114168 weight_movement_bytes=330301440 total_nvlink_bytes=921415608 weight_to_token_ratio=0.558778 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088285758e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5902642448 prefetch_bytes=990904320 reduce_bytes=1761607680 weight_movement_bytes=2752512000 egress_bytes=2952437008 ingress_bytes=5702717440 link_bytes=5702717440 total_nvlink_bytes=8655154448 weight_to_token_ratio=0.466319 prefetch_share_of_ingress=0.173760 dedup_ratio=0.272025 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=16 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.917963564e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6128268432 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3065281680 ingress_bytes=5044795392 link_bytes=5044795392 total_nvlink_bytes=8110077072 weight_to_token_ratio=0.323388 prefetch_share_of_ingress=0.218246 dedup_ratio=0.252078 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.773728913e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-29 23:43:04] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 238.7 | throughput per GPU (TFLOP/s/GPU): 517.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.538278E+00 | loss scale: 1.0 | grad norm: 32.381 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45330 payload_rows_remote=39992 dup_rows=20206 dup_rows_remote=18534 dedup_ratio=0.308319 router_tokens_per_expert=[249, 150, 487, 12, 10, 34, 1139, 517, 615, 288, 180, 461, 238, 221, 6, 610, 123, 175, 1256, 125, 129, 237, 161, 130, 785, 676, 322, 1095, 165, 1172, 411, 16, 379, 161, 336, 1057, 124, 21, 106, 812, 503, 1334, 1060, 64, 1443, 807, 319, 551, 531, 1389, 470, 57, 803, 1252, 1606, 1169, 1055, 776, 82, 93, 222, 1444, 408, 139] payload_rows_per_dst=[5322, 5878, 5338, 5102, 5638, 6174, 6080, 5798] dup_rows_per_dst=[2218, 2572, 1672, 1838, 3240, 3102, 2924, 2640] combine_rows_per_src=[5322, 5878, 5338, 5102, 5638, 6174, 6080, 5798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44148 payload_rows_remote=37694 dup_rows=21388 dup_rows_remote=17534 dedup_ratio=0.326355 router_tokens_per_expert=[347, 112, 638, 16, 9, 21, 1235, 307, 536, 430, 124, 242, 236, 202, 3, 367, 260, 41, 1323, 95, 244, 432, 183, 22, 706, 953, 165, 1458, 298, 1236, 304, 10, 419, 140, 611, 1422, 75, 0, 46, 406, 774, 725, 1306, 51, 1398, 361, 437, 602, 578, 981, 328, 65, 473, 1319, 1548, 2300, 1043, 707, 139, 41, 393, 985, 470, 70] payload_rows_per_dst=[4416, 3272, 6408, 5328, 4790, 6762, 6718, 6454] dup_rows_per_dst=[954, 1008, 3392, 2132, 1448, 4734, 3866, 3854] combine_rows_per_src=[4416, 3272, 6408, 5328, 4790, 6762, 6718, 6454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45174 payload_rows_remote=39246 dup_rows=20362 dup_rows_remote=17926 dedup_ratio=0.310699 router_tokens_per_expert=[166, 88, 806, 65, 65, 29, 2029, 454, 231, 535, 205, 238, 152, 117, 17, 449, 174, 300, 1311, 112, 198, 114, 264, 22, 935, 1185, 260, 1152, 194, 1586, 442, 14, 310, 91, 596, 1486, 153, 11, 50, 580, 640, 567, 890, 39, 1538, 387, 411, 292, 773, 1170, 874, 18, 684, 1318, 1946, 964, 1113, 596, 153, 23, 204, 510, 427, 45] payload_rows_per_dst=[6526, 5824, 5326, 5928, 5970, 4452, 6194, 4954] dup_rows_per_dst=[4050, 1956, 1592, 2436, 3660, 1440, 3480, 1748] combine_rows_per_src=[6526, 5824, 5326, 5928, 5970, 4452, 6194, 4954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45246 payload_rows_remote=40052 dup_rows=20290 dup_rows_remote=17938 dedup_ratio=0.309601 router_tokens_per_expert=[214, 102, 493, 12, 68, 47, 1144, 610, 834, 534, 174, 144, 173, 241, 26, 591, 135, 208, 1361, 98, 248, 78, 328, 78, 910, 475, 220, 745, 393, 1599, 554, 45, 324, 426, 214, 1434, 113, 23, 101, 601, 691, 886, 804, 110, 1578, 873, 778, 435, 635, 1316, 343, 90, 481, 713, 1981, 1746, 731, 583, 237, 51, 253, 956, 295, 84] payload_rows_per_dst=[5774, 6348, 6008, 5066, 6010, 5194, 5300, 5546] dup_rows_per_dst=[2804, 3048, 2552, 1618, 3618, 2352, 1856, 2442] combine_rows_per_src=[5774, 6348, 6008, 5066, 6010, 5194, 5300, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45278 payload_rows_remote=39946 dup_rows=20258 dup_rows_remote=17492 dedup_ratio=0.309113 router_tokens_per_expert=[267, 66, 421, 34, 44, 16, 1190, 691, 444, 416, 232, 332, 229, 296, 65, 500, 105, 206, 1332, 85, 279, 108, 222, 25, 1312, 895, 199, 1457, 111, 1706, 388, 57, 393, 91, 548, 1303, 281, 40, 42, 213, 710, 1350, 1003, 228, 1138, 690, 151, 385, 453, 1130, 738, 35, 779, 1351, 1785, 1333, 748, 549, 285, 46, 232, 685, 253, 70] payload_rows_per_dst=[5874, 5908, 5474, 5948, 5332, 5224, 6082, 5436] dup_rows_per_dst=[2996, 2690, 1916, 2890, 2766, 1804, 2890, 2306] combine_rows_per_src=[5874, 5908, 5474, 5948, 5332, 5224, 6082, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45260 payload_rows_remote=39086 dup_rows=20276 dup_rows_remote=16522 dedup_ratio=0.309387 router_tokens_per_expert=[246, 103, 482, 19, 7, 32, 1449, 463, 599, 762, 187, 118, 386, 205, 8, 540, 181, 154, 1075, 116, 171, 138, 102, 61, 832, 689, 177, 1137, 358, 1466, 367, 12, 473, 74, 437, 1754, 94, 4, 59, 225, 867, 720, 1164, 58, 1391, 545, 430, 476, 1305, 902, 560, 30, 572, 1121, 2530, 1137, 883, 842, 226, 50, 265, 556, 323, 53] payload_rows_per_dst=[4836, 6424, 4942, 5988, 5524, 5374, 6174, 5998] dup_rows_per_dst=[1512, 3298, 1328, 3342, 1736, 2580, 3754, 2726] combine_rows_per_src=[4836, 6424, 4942, 5988, 5524, 5374, 6174, 5998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39080 dup_rows=20836 dup_rows_remote=18246 dedup_ratio=0.317932 router_tokens_per_expert=[233, 148, 861, 23, 14, 27, 1175, 478, 217, 825, 344, 351, 159, 233, 19, 603, 101, 242, 1688, 101, 717, 116, 242, 84, 1108, 430, 297, 1723, 347, 1146, 576, 20, 72, 60, 296, 1020, 275, 37, 125, 761, 627, 897, 1110, 95, 1455, 351, 288, 679, 1275, 836, 150, 63, 548, 1287, 1705, 741, 982, 916, 345, 57, 395, 284, 295, 93] payload_rows_per_dst=[5620, 5996, 6646, 6254, 5394, 5576, 4208, 5006] dup_rows_per_dst=[2590, 2916, 3992, 3322, 2808, 2518, 962, 1728] combine_rows_per_src=[5620, 5996, 6646, 6254, 5394, 5576, 4208, 5006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44952 payload_rows_remote=39146 dup_rows=20584 dup_rows_remote=17992 dedup_ratio=0.314087 router_tokens_per_expert=[242, 196, 1175, 8, 16, 7, 1340, 801, 777, 578, 206, 147, 166, 175, 14, 525, 91, 208, 1548, 272, 224, 268, 190, 82, 811, 991, 285, 1226, 434, 1237, 394, 31, 416, 61, 496, 1048, 126, 4, 30, 437, 341, 1363, 907, 165, 1183, 622, 300, 427, 476, 840, 541, 46, 461, 692, 1611, 1060, 1181, 1097, 177, 80, 564, 751, 544, 56] payload_rows_per_dst=[6158, 5806, 6162, 5728, 5120, 5766, 4142, 6070] dup_rows_per_dst=[3886, 2592, 2788, 2616, 2482, 2484, 906, 2830] combine_rows_per_src=[6158, 5806, 6162, 5728, 5120, 5766, 4142, 6070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286662656 dispatch_meta_bytes=234104 combine_payload_bytes=286662656 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286896760 ingress_bytes=506863616 link_bytes=506863616 token_bytes=573559416 weight_movement_bytes=220200960 total_nvlink_bytes=793760376 weight_to_token_ratio=0.383920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.815191930e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270190592 dispatch_meta_bytes=220912 combine_payload_bytes=270190592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=270411504 ingress_bytes=380291072 link_bytes=380291072 token_bytes=540602096 weight_movement_bytes=110100480 total_nvlink_bytes=650702576 weight_to_token_ratio=0.203663 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.613887213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281315328 dispatch_meta_bytes=228688 combine_payload_bytes=281315328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281544016 ingress_bytes=611616768 link_bytes=611616768 token_bytes=562859344 weight_movement_bytes=330301440 total_nvlink_bytes=893160784 weight_to_token_ratio=0.586828 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063702153e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287092736 dispatch_meta_bytes=231960 combine_payload_bytes=287092736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=287324696 ingress_bytes=617394176 link_bytes=617394176 token_bytes=574417432 weight_movement_bytes=330301440 total_nvlink_bytes=904718872 weight_to_token_ratio=0.575020 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073750016e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286332928 dispatch_meta_bytes=229752 combine_payload_bytes=286332928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286562680 ingress_bytes=396433408 link_bytes=396433408 token_bytes=572895608 weight_movement_bytes=110100480 total_nvlink_bytes=682996088 weight_to_token_ratio=0.192182 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.894628986e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=222432 combine_payload_bytes=280168448 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280390880 ingress_bytes=720570368 link_bytes=720570368 token_bytes=560559328 weight_movement_bytes=440401920 total_nvlink_bytes=1000961248 weight_to_token_ratio=0.785647 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253190383e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229304 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280354744 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560480184 weight_movement_bytes=110100480 total_nvlink_bytes=670580664 weight_to_token_ratio=0.196440 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=228552 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280827080 ingress_bytes=390699008 link_bytes=390699008 token_bytes=561425608 weight_movement_bytes=110100480 total_nvlink_bytes=671526088 weight_to_token_ratio=0.196109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794898339e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47802 payload_rows_remote=40104 dup_rows=17734 dup_rows_remote=12148 dedup_ratio=0.270599 router_tokens_per_expert=[0, 248, 4075, 1282, 5, 16, 2289, 108, 201, 1010, 715, 0, 231, 7, 2, 21, 1, 143, 468, 163, 9, 1645, 0, 35, 400, 1264, 1, 15, 2703, 0, 232, 0, 1653, 576, 0, 0, 0, 4, 0, 0, 2, 0, 3765, 5, 184, 1, 0, 0, 616, 1020, 9, 345, 4, 975, 0, 2, 164, 0, 2997, 0, 20, 1275, 969, 893] payload_rows_per_dst=[3040, 3792, 4246, 6286, 8166, 7552, 7022, 7698] dup_rows_per_dst=[278, 582, 682, 2296, 4450, 362, 3498, 5586] combine_rows_per_src=[3040, 3792, 4246, 6286, 8166, 7552, 7022, 7698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48080 payload_rows_remote=39974 dup_rows=17456 dup_rows_remote=12172 dedup_ratio=0.266357 router_tokens_per_expert=[0, 364, 3976, 1049, 7, 10, 2491, 47, 604, 1361, 908, 1, 204, 1, 19, 59, 4, 291, 272, 77, 0, 1983, 2, 90, 998, 427, 1, 20, 2366, 0, 353, 3, 1313, 496, 0, 6, 0, 1, 0, 0, 58, 0, 3441, 1, 84, 0, 0, 0, 571, 833, 0, 645, 3, 1023, 1, 28, 150, 0, 3037, 8, 13, 1713, 1120, 235] payload_rows_per_dst=[6024, 7382, 8106, 6152, 3410, 7048, 4902, 5056] dup_rows_per_dst=[1912, 4396, 5284, 2184, 222, 730, 1306, 1422] combine_rows_per_src=[6024, 7382, 8106, 6152, 3410, 7048, 4902, 5056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46782 payload_rows_remote=40998 dup_rows=18754 dup_rows_remote=16846 dedup_ratio=0.286163 router_tokens_per_expert=[0, 221, 3969, 1617, 8, 22, 1524, 496, 823, 1336, 903, 4, 527, 1, 2, 401, 0, 111, 1245, 50, 21, 1445, 0, 61, 482, 957, 4, 0, 2128, 0, 274, 1, 1249, 705, 0, 0, 20, 3, 2, 3, 55, 0, 3779, 0, 96, 1, 0, 0, 729, 621, 12, 306, 13, 1353, 10, 37, 184, 0, 1960, 14, 20, 1511, 976, 476] payload_rows_per_dst=[5640, 5816, 8060, 5784, 3692, 7902, 4936, 4952] dup_rows_per_dst=[2136, 2178, 5744, 1908, 272, 3880, 1226, 1410] combine_rows_per_src=[5640, 5816, 8060, 5784, 3692, 7902, 4936, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48380 payload_rows_remote=40820 dup_rows=17156 dup_rows_remote=16828 dedup_ratio=0.261780 router_tokens_per_expert=[0, 591, 3986, 730, 2, 42, 2808, 108, 167, 1001, 959, 1, 338, 19, 0, 312, 0, 66, 270, 311, 6, 1081, 0, 19, 183, 729, 19, 3, 2535, 21, 151, 1, 1280, 800, 0, 0, 3, 0, 0, 4, 5, 0, 3766, 0, 171, 2, 0, 0, 289, 900, 4, 233, 103, 2361, 0, 1, 71, 0, 2994, 4, 7, 672, 1952, 687] payload_rows_per_dst=[3978, 4716, 3194, 6014, 8114, 7560, 7184, 7620] dup_rows_per_dst=[946, 878, 312, 1270, 4032, 328, 4236, 5154] combine_rows_per_src=[3978, 4716, 3194, 6014, 8114, 7560, 7184, 7620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50064 payload_rows_remote=43420 dup_rows=15472 dup_rows_remote=13032 dedup_ratio=0.236084 router_tokens_per_expert=[0, 1204, 4058, 1006, 3, 30, 1448, 83, 842, 1148, 1148, 0, 470, 9, 0, 45, 0, 54, 726, 71, 20, 1455, 0, 35, 311, 589, 7, 2, 2772, 28, 175, 10, 2750, 565, 0, 0, 0, 5, 0, 0, 7, 0, 3622, 0, 86, 0, 0, 0, 1396, 613, 5, 242, 2, 1674, 1, 7, 27, 0, 1474, 2, 7, 1477, 534, 523] payload_rows_per_dst=[5480, 5630, 7136, 6280, 6644, 7330, 5852, 5712] dup_rows_per_dst=[2068, 1694, 3258, 1508, 2440, 612, 2028, 1864] combine_rows_per_src=[5480, 5630, 7136, 6280, 6644, 7330, 5852, 5712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48758 payload_rows_remote=41430 dup_rows=16778 dup_rows_remote=13424 dedup_ratio=0.256012 router_tokens_per_expert=[0, 308, 4035, 1152, 0, 6, 2591, 107, 179, 1758, 956, 0, 353, 6, 0, 47, 0, 88, 424, 96, 0, 1738, 0, 16, 385, 711, 1, 3, 2672, 14, 250, 0, 2511, 455, 2, 0, 18, 1, 0, 0, 27, 1, 3677, 2, 62, 0, 0, 0, 474, 537, 6, 561, 41, 1122, 0, 9, 6, 0, 2549, 0, 8, 963, 932, 908] payload_rows_per_dst=[2872, 5410, 4226, 6194, 8152, 7414, 7328, 7162] dup_rows_per_dst=[274, 1188, 498, 1878, 5892, 124, 3354, 3570] combine_rows_per_src=[2872, 5410, 4226, 6194, 8152, 7414, 7328, 7162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=287465472 dispatch_meta_bytes=209008 combine_payload_bytes=287465472 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=287674480 ingress_bytes=727867392 link_bytes=727867392 token_bytes=575139952 weight_movement_bytes=440401920 total_nvlink_bytes=1015541872 weight_to_token_ratio=0.765730 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=9856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.265881108e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45388 payload_rows_remote=37206 dup_rows=20148 dup_rows_remote=10970 dedup_ratio=0.307434 router_tokens_per_expert=[0, 542, 4036, 1128, 1, 24, 1609, 239, 410, 1429, 850, 0, 361, 15, 0, 430, 1, 220, 1128, 25, 0, 1707, 0, 2, 451, 1444, 0, 2, 2292, 4, 150, 2, 1486, 511, 0, 0, 4, 0, 0, 0, 3, 0, 3657, 1, 285, 0, 0, 0, 688, 1198, 5, 247, 61, 1279, 0, 0, 82, 0, 2464, 4, 40, 1101, 854, 296] payload_rows_per_dst=[8182, 7298, 4906, 6266, 3774, 7436, 5164, 2362] dup_rows_per_dst=[9178, 4620, 1260, 2424, 228, 456, 1792, 190] combine_rows_per_src=[8182, 7298, 4906, 6266, 3774, 7436, 5164, 2362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286533632 dispatch_meta_bytes=208584 combine_payload_bytes=286533632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286742216 ingress_bytes=396634112 link_bytes=396634112 token_bytes=573275848 weight_movement_bytes=110100480 total_nvlink_bytes=683376328 weight_to_token_ratio=0.192055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.898119559e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47798 payload_rows_remote=41014 dup_rows=17738 dup_rows_remote=14566 dedup_ratio=0.270660 router_tokens_per_expert=[0, 203, 4049, 809, 0, 180, 2373, 57, 397, 1123, 490, 5, 229, 7, 0, 169, 0, 62, 1208, 86, 2, 972, 5, 13, 336, 1235, 9, 18, 2718, 2, 227, 1, 2023, 993, 0, 0, 6, 2, 0, 0, 10, 0, 3321, 4, 64, 0, 2, 0, 633, 450, 2, 216, 18, 1534, 0, 1, 11, 0, 2558, 2, 37, 1892, 742, 1262] payload_rows_per_dst=[6972, 6784, 6080, 6582, 5170, 6690, 4760, 4760] dup_rows_per_dst=[6556, 3172, 2544, 2510, 878, 112, 948, 1018] combine_rows_per_src=[6972, 6784, 6080, 6582, 5170, 6690, 4760, 4760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293873664 dispatch_meta_bytes=231376 combine_payload_bytes=293873664 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=294105040 ingress_bytes=403974144 link_bytes=403974144 token_bytes=587978704 weight_movement_bytes=110100480 total_nvlink_bytes=698079184 weight_to_token_ratio=0.187252 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.025774787e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=292597760 dispatch_meta_bytes=230592 combine_payload_bytes=292597760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292828352 ingress_bytes=402698240 link_bytes=402698240 token_bytes=585426112 weight_movement_bytes=110100480 total_nvlink_bytes=695526592 weight_to_token_ratio=0.188069 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.003584718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=311234560 dispatch_meta_bytes=225808 combine_payload_bytes=311234560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311460368 ingress_bytes=421335040 link_bytes=421335040 token_bytes=622694928 weight_movement_bytes=110100480 total_nvlink_bytes=732795408 weight_to_token_ratio=0.176813 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.327709323e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296970240 dispatch_meta_bytes=219416 combine_payload_bytes=296970240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297189656 ingress_bytes=407070720 link_bytes=407070720 token_bytes=594159896 weight_movement_bytes=110100480 total_nvlink_bytes=704260376 weight_to_token_ratio=0.185304 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.079629337e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266692608 dispatch_meta_bytes=192704 combine_payload_bytes=266692608 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=266885312 ingress_bytes=707094528 link_bytes=707094528 token_bytes=533577920 weight_movement_bytes=440401920 total_nvlink_bytes=973979840 weight_to_token_ratio=0.825375 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.229753681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293988352 dispatch_meta_bytes=222320 combine_payload_bytes=293988352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294210672 ingress_bytes=404088832 link_bytes=404088832 token_bytes=588199024 weight_movement_bytes=110100480 total_nvlink_bytes=698299504 weight_to_token_ratio=0.187182 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.027769400e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54426 payload_rows_remote=47602 dup_rows=11110 dup_rows_remote=10206 dedup_ratio=0.169525 router_tokens_per_expert=[430, 3896, 642, 28, 46, 0, 9, 0, 12, 18, 5, 0, 64, 39, 3973, 1, 2, 11, 4, 3998, 3, 1, 1, 0, 0, 394, 6, 672, 0, 11, 5, 617, 48, 3317, 4, 1678, 1, 4, 1045, 1962, 364, 6, 0, 3, 4022, 38, 1, 0, 3, 1, 4078, 4, 727, 0, 27, 0, 2, 10, 15, 1, 510, 2, 0, 7] payload_rows_per_dst=[2248, 7714, 8014, 7922, 5438, 8086, 8180, 6824] dup_rows_per_dst=[62, 192, 26, 3280, 1360, 782, 4504, 904] combine_rows_per_src=[2248, 7714, 8014, 7922, 5438, 8086, 8180, 6824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54626 payload_rows_remote=46862 dup_rows=10910 dup_rows_remote=10908 dedup_ratio=0.166473 router_tokens_per_expert=[680, 3840, 639, 44, 329, 0, 6, 0, 0, 37, 1, 9, 91, 26, 3955, 2, 0, 1, 0, 3881, 1, 0, 0, 0, 0, 306, 5, 975, 0, 69, 0, 223, 96, 2969, 0, 926, 0, 13, 1495, 2127, 396, 7, 0, 2, 3950, 7, 0, 0, 0, 0, 4083, 0, 551, 0, 33, 0, 0, 0, 52, 4, 913, 6, 1, 17] payload_rows_per_dst=[7738, 7946, 7764, 3228, 6896, 7952, 6516, 6586] dup_rows_per_dst=[4628, 296, 2, 462, 2418, 772, 994, 1338] combine_rows_per_src=[7738, 7946, 7764, 3228, 6896, 7952, 6516, 6586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55320 payload_rows_remote=47330 dup_rows=10216 dup_rows_remote=5952 dedup_ratio=0.155884 router_tokens_per_expert=[852, 3810, 732, 11, 273, 1, 26, 0, 2, 7, 1, 0, 186, 187, 3759, 0, 0, 0, 0, 3904, 1, 6, 0, 1, 28, 165, 1, 1658, 0, 194, 1, 270, 334, 2396, 0, 2133, 1, 17, 560, 638, 249, 3, 1, 0, 3849, 49, 2, 0, 0, 0, 4081, 1, 700, 1, 12, 0, 0, 0, 343, 6, 1316, 0, 0, 0] payload_rows_per_dst=[3548, 7714, 7822, 7990, 5922, 7778, 8170, 6376] dup_rows_per_dst=[242, 570, 2, 4264, 1444, 528, 1420, 1746] combine_rows_per_src=[3548, 7714, 7822, 7990, 5922, 7778, 8170, 6376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55376 payload_rows_remote=48598 dup_rows=10160 dup_rows_remote=7470 dedup_ratio=0.155029 router_tokens_per_expert=[319, 3817, 962, 149, 289, 0, 87, 0, 0, 4, 0, 0, 31, 62, 3682, 0, 1, 0, 0, 4017, 0, 1, 0, 0, 5, 354, 24, 1140, 0, 28, 1, 165, 81, 2965, 0, 2148, 0, 0, 1286, 1219, 353, 0, 0, 0, 3537, 10, 1, 0, 0, 0, 4091, 0, 952, 0, 42, 0, 0, 0, 350, 1, 594, 0, 0, 0] payload_rows_per_dst=[3302, 7378, 8036, 7872, 6778, 7186, 8182, 6642] dup_rows_per_dst=[310, 180, 2, 3196, 2690, 616, 1988, 1178] combine_rows_per_src=[3302, 7378, 8036, 7872, 6778, 7186, 8182, 6642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55068 payload_rows_remote=47162 dup_rows=10468 dup_rows_remote=9408 dedup_ratio=0.159729 router_tokens_per_expert=[897, 3736, 225, 72, 189, 0, 14, 1, 2, 5, 1, 0, 100, 32, 4043, 1, 0, 3, 0, 3995, 1, 0, 2, 2, 23, 183, 15, 943, 0, 46, 3, 200, 185, 2985, 0, 1178, 0, 0, 636, 1447, 345, 215, 2, 0, 3872, 42, 7, 0, 0, 0, 4085, 0, 1085, 0, 3, 0, 0, 0, 26, 0, 1921, 0, 0, 0] payload_rows_per_dst=[2608, 8090, 7996, 7842, 5406, 7906, 8170, 7050] dup_rows_per_dst=[188, 278, 10, 2456, 1486, 1060, 2176, 2814] combine_rows_per_src=[2608, 8090, 7996, 7842, 5406, 7906, 8170, 7050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55238 payload_rows_remote=47068 dup_rows=10298 dup_rows_remote=7652 dedup_ratio=0.157135 router_tokens_per_expert=[677, 3818, 490, 39, 224, 0, 14, 0, 1, 16, 0, 0, 131, 42, 3987, 0, 0, 2, 1, 3938, 5, 5, 0, 0, 4, 333, 3, 916, 0, 9, 6, 330, 199, 3155, 0, 1316, 0, 0, 1221, 912, 341, 2, 0, 0, 4015, 32, 0, 0, 0, 0, 4083, 0, 1247, 0, 78, 0, 0, 3, 131, 5, 1033, 0, 4, 0] payload_rows_per_dst=[2654, 8014, 7896, 7876, 5658, 8044, 8170, 6926] dup_rows_per_dst=[234, 340, 6, 2962, 1638, 736, 2646, 1736] combine_rows_per_src=[2654, 8014, 7896, 7876, 5658, 8044, 8170, 6926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=341211136 dispatch_meta_bytes=231232 combine_payload_bytes=341211136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341442368 ingress_bytes=451311616 link_bytes=451311616 token_bytes=682653504 weight_movement_bytes=110100480 total_nvlink_bytes=792753984 weight_to_token_ratio=0.161283 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.849051282e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=335906816 dispatch_meta_bytes=231080 combine_payload_bytes=335906816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336137896 ingress_bytes=446007296 link_bytes=446007296 token_bytes=672044712 weight_movement_bytes=110100480 total_nvlink_bytes=782145192 weight_to_token_ratio=0.163829 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.756800433e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=339261440 dispatch_meta_bytes=213128 combine_payload_bytes=339261440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339474568 ingress_bytes=449361920 link_bytes=449361920 token_bytes=678736008 weight_movement_bytes=110100480 total_nvlink_bytes=788836488 weight_to_token_ratio=0.162214 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.815142862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=348350464 dispatch_meta_bytes=224272 combine_payload_bytes=348350464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=348574736 ingress_bytes=678651904 link_bytes=678651904 token_bytes=696925200 weight_movement_bytes=330301440 total_nvlink_bytes=1027226640 weight_to_token_ratio=0.473941 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.180287280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=338057216 dispatch_meta_bytes=226280 combine_payload_bytes=338057216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=338283496 ingress_bytes=558258176 link_bytes=558258176 token_bytes=676340712 weight_movement_bytes=220200960 total_nvlink_bytes=896541672 weight_to_token_ratio=0.325577 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.709027858e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=337383424 dispatch_meta_bytes=218880 combine_payload_bytes=337383424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=337602304 ingress_bytes=557584384 link_bytes=557584384 token_bytes=674985728 weight_movement_bytes=220200960 total_nvlink_bytes=895186688 weight_to_token_ratio=0.326231 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.697309506e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49846 payload_rows_remote=41852 dup_rows=15690 dup_rows_remote=15456 dedup_ratio=0.239410 router_tokens_per_expert=[379, 3856, 753, 86, 408, 2, 67, 1, 10, 7, 0, 3, 67, 50, 3977, 0, 10, 12, 0, 3786, 6, 1, 0, 0, 9, 233, 0, 507, 1, 43, 24, 350, 13, 3230, 0, 1670, 36, 5, 1396, 1370, 215, 0, 0, 1, 3695, 18, 27, 0, 1, 0, 4081, 3, 1481, 0, 23, 1, 0, 0, 25, 4, 786, 0, 0, 39] payload_rows_per_dst=[8174, 7994, 7608, 2168, 6552, 7516, 2990, 6844] dup_rows_per_dst=[11092, 234, 22, 166, 2428, 396, 28, 1324] combine_rows_per_src=[8174, 7994, 7608, 2168, 6552, 7516, 2990, 6844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48110 payload_rows_remote=39918 dup_rows=17426 dup_rows_remote=7110 dedup_ratio=0.265900 router_tokens_per_expert=[514, 3914, 455, 29, 222, 1, 34, 0, 0, 15, 0, 0, 43, 164, 4048, 0, 2, 48, 0, 3920, 0, 4, 0, 0, 0, 552, 0, 1227, 0, 113, 2, 32, 106, 3041, 0, 2058, 0, 0, 1194, 1265, 216, 54, 0, 0, 4016, 22, 0, 0, 0, 3, 4085, 0, 644, 0, 54, 0, 0, 0, 183, 2, 486, 0, 0, 0] payload_rows_per_dst=[8192, 8102, 7974, 3360, 6912, 5774, 1394, 6402] dup_rows_per_dst=[10316, 494, 2356, 492, 2510, 404, 8, 846] combine_rows_per_src=[8192, 8102, 7974, 3360, 6912, 5774, 1394, 6402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299995136 dispatch_meta_bytes=229232 combine_payload_bytes=299995136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=300224368 ingress_bytes=520196096 link_bytes=520196096 token_bytes=600219504 weight_movement_bytes=220200960 total_nvlink_bytes=820420464 weight_to_token_ratio=0.366867 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.047065685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48822 payload_rows_remote=40680 dup_rows=16714 dup_rows_remote=8848 dedup_ratio=0.255035 router_tokens_per_expert=[1, 0, 16, 2587, 968, 3, 10, 0, 1, 0, 1, 0, 8, 4042, 0, 5, 10, 153, 0, 1, 0, 0, 95, 94, 10, 16, 0, 1350, 3, 0, 0, 190, 0, 2, 6, 3, 0, 4095, 1375, 0, 6, 13, 2, 2, 2372, 2, 16, 0, 3812, 7, 2, 12, 2, 0, 1357, 13, 4044, 4088, 905, 1, 2, 1, 1063, 1] payload_rows_per_dst=[6348, 8106, 4410, 3094, 2758, 8190, 7774, 8142] dup_rows_per_dst=[822, 8, 498, 44, 14, 4826, 2636, 7866] combine_rows_per_src=[6348, 8106, 4410, 3094, 2758, 8190, 7774, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=286132224 dispatch_meta_bytes=188112 combine_payload_bytes=286132224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=286320336 ingress_bytes=506333184 link_bytes=506333184 token_bytes=572452560 weight_movement_bytes=220200960 total_nvlink_bytes=792653520 weight_to_token_ratio=0.384662 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.805966845e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52168 payload_rows_remote=44060 dup_rows=13368 dup_rows_remote=12922 dedup_ratio=0.203979 router_tokens_per_expert=[0, 1, 0, 1898, 1395, 0, 10, 2, 16, 0, 0, 0, 0, 4081, 0, 0, 0, 39, 0, 1, 4, 0, 144, 41, 4, 0, 52, 1106, 0, 0, 0, 357, 0, 0, 23, 6, 1, 4087, 1190, 11, 2, 2, 2, 0, 1645, 0, 41, 0, 3367, 0, 0, 0, 0, 0, 2068, 10, 3945, 4048, 2352, 0, 2, 2, 813, 0] payload_rows_per_dst=[5888, 8164, 8108, 8156, 8184, 3352, 4746, 5570] dup_rows_per_dst=[724, 30, 446, 2772, 8156, 32, 440, 768] combine_rows_per_src=[5888, 8164, 8108, 8156, 8184, 3352, 4746, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56510 payload_rows_remote=48368 dup_rows=9026 dup_rows_remote=7220 dedup_ratio=0.137726 router_tokens_per_expert=[0, 1, 20, 1833, 1382, 33, 91, 0, 1, 2, 1, 15, 0, 4088, 0, 0, 5, 40, 2, 10, 0, 0, 156, 42, 2, 19, 9, 746, 0, 0, 8, 208, 1, 0, 2, 1, 0, 4096, 1220, 6, 33, 10, 0, 0, 3076, 0, 17, 0, 3226, 0, 0, 4, 2, 0, 2180, 1, 3982, 4088, 1875, 0, 0, 2, 232, 0] payload_rows_per_dst=[5884, 8178, 8182, 8142, 8192, 6242, 7566, 4124] dup_rows_per_dst=[836, 36, 504, 1806, 2460, 30, 3260, 94] combine_rows_per_src=[5884, 8178, 8182, 8142, 8192, 6242, 7566, 4124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55466 payload_rows_remote=48618 dup_rows=10070 dup_rows_remote=8330 dedup_ratio=0.153656 router_tokens_per_expert=[0, 3, 26, 1272, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 1, 104, 0, 8, 0, 0, 66, 64, 9, 2, 6, 697, 0, 0, 1, 123, 0, 0, 3, 0, 0, 4096, 1090, 7, 4, 12, 2, 0, 2430, 0, 0, 0, 3725, 0, 7, 0, 0, 0, 2087, 5, 4026, 4070, 1683, 0, 0, 1, 1222, 0] payload_rows_per_dst=[5742, 8144, 8174, 8186, 6848, 5524, 7854, 4994] dup_rows_per_dst=[548, 0, 452, 1542, 1740, 1176, 3794, 818] combine_rows_per_src=[5742, 8144, 8174, 8186, 6848, 5524, 7854, 4994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51682 payload_rows_remote=43490 dup_rows=13854 dup_rows_remote=9038 dedup_ratio=0.211395 router_tokens_per_expert=[2, 9, 0, 2454, 846, 1, 13, 0, 1, 0, 0, 0, 1, 4091, 1, 0, 0, 138, 0, 0, 0, 0, 101, 33, 0, 25, 0, 1493, 0, 0, 12, 377, 0, 0, 0, 7, 0, 4093, 983, 7, 2, 1, 0, 0, 2392, 0, 16, 0, 3015, 0, 0, 0, 0, 0, 1319, 1, 4034, 4069, 1396, 0, 0, 0, 1835, 0] payload_rows_per_dst=[5892, 8184, 8150, 5012, 1990, 8192, 6902, 7360] dup_rows_per_dst=[758, 4, 532, 1468, 4, 4816, 1768, 4504] combine_rows_per_src=[5892, 8184, 8150, 5012, 1990, 8192, 6902, 7360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51702 payload_rows_remote=44196 dup_rows=13834 dup_rows_remote=10808 dedup_ratio=0.211090 router_tokens_per_expert=[0, 0, 3, 2271, 1252, 1, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 120, 2, 0, 0, 1, 36, 79, 17, 39, 2, 1065, 0, 0, 0, 179, 0, 0, 0, 0, 0, 4094, 1431, 10, 2, 6, 6, 0, 2324, 0, 0, 0, 3464, 0, 45, 0, 0, 0, 1757, 0, 4034, 4082, 1366, 0, 0, 1, 979, 0] payload_rows_per_dst=[6118, 8192, 8174, 2532, 2872, 8190, 7506, 8118] dup_rows_per_dst=[942, 2, 466, 72, 10, 4674, 3026, 4642] combine_rows_per_src=[6118, 8192, 8174, 2532, 2872, 8190, 7506, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=291594240 dispatch_meta_bytes=198112 combine_payload_bytes=291594240 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=291792352 ingress_bytes=731996160 link_bytes=731996160 token_bytes=583386592 weight_movement_bytes=440401920 total_nvlink_bytes=1023788512 weight_to_token_ratio=0.754906 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.273061715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=315822080 dispatch_meta_bytes=227928 combine_payload_bytes=315822080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316050008 ingress_bytes=425922560 link_bytes=425922560 token_bytes=631872088 weight_movement_bytes=110100480 total_nvlink_bytes=741972568 weight_to_token_ratio=0.174245 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.407493841e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=346701824 dispatch_meta_bytes=222352 combine_payload_bytes=346701824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346924176 ingress_bytes=456802304 link_bytes=456802304 token_bytes=693626000 weight_movement_bytes=110100480 total_nvlink_bytes=803726480 weight_to_token_ratio=0.158732 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.944543377e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=348493824 dispatch_meta_bytes=227792 combine_payload_bytes=348493824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=348721616 ingress_bytes=568694784 link_bytes=568694784 token_bytes=697215440 weight_movement_bytes=220200960 total_nvlink_bytes=917416400 weight_to_token_ratio=0.315829 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.890537636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=311736320 dispatch_meta_bytes=210112 combine_payload_bytes=311736320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311946432 ingress_bytes=421836800 link_bytes=421836800 token_bytes=623682752 weight_movement_bytes=110100480 total_nvlink_bytes=733783232 weight_to_token_ratio=0.176533 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.336435754e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=316796928 dispatch_meta_bytes=220016 combine_payload_bytes=316796928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=317016944 ingress_bytes=647098368 link_bytes=647098368 token_bytes=633813872 weight_movement_bytes=330301440 total_nvlink_bytes=964115312 weight_to_token_ratio=0.521133 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.125410491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51918 payload_rows_remote=43778 dup_rows=13618 dup_rows_remote=13606 dedup_ratio=0.207794 router_tokens_per_expert=[0, 1, 0, 2138, 1304, 0, 2, 0, 2, 0, 12, 1, 3, 4058, 0, 0, 6, 101, 10, 25, 0, 6, 159, 288, 71, 7, 1, 1464, 1, 0, 5, 268, 2, 0, 0, 0, 2, 4089, 724, 3, 1, 0, 3, 0, 2002, 0, 0, 1, 3417, 0, 0, 7, 0, 1, 1277, 0, 3996, 4021, 1449, 1, 0, 0, 1832, 7] payload_rows_per_dst=[7378, 8140, 8176, 8184, 8182, 4014, 2568, 5276] dup_rows_per_dst=[3676, 12, 1056, 3442, 4128, 0, 2, 1302] combine_rows_per_src=[7378, 8140, 8176, 8184, 8182, 4014, 2568, 5276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53800 payload_rows_remote=45810 dup_rows=11736 dup_rows_remote=5746 dedup_ratio=0.179077 router_tokens_per_expert=[0, 0, 3, 2638, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 12, 0, 0, 0, 0, 17, 5, 0, 48, 0, 1155, 0, 0, 1, 245, 0, 0, 0, 3, 0, 4096, 897, 3, 1, 0, 54, 0, 3067, 0, 17, 0, 3021, 0, 0, 0, 0, 0, 1061, 0, 4093, 4070, 1866, 0, 0, 0, 972, 0] payload_rows_per_dst=[7990, 8190, 8140, 8192, 8192, 6164, 4976, 1956] dup_rows_per_dst=[5990, 146, 68, 2892, 1806, 114, 718, 2] combine_rows_per_src=[7990, 8190, 8140, 8192, 8192, 6164, 4976, 1956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=313800704 dispatch_meta_bytes=229536 combine_payload_bytes=313800704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314030240 ingress_bytes=423901184 link_bytes=423901184 token_bytes=627830944 weight_movement_bytes=110100480 total_nvlink_bytes=737931424 weight_to_token_ratio=0.175366 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.372338787e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53684 payload_rows_remote=45782 dup_rows=11852 dup_rows_remote=10734 dedup_ratio=0.180847 router_tokens_per_expert=[0, 251, 1522, 4, 0, 0, 4071, 0, 0, 0, 4, 6, 17, 1296, 0, 0, 0, 0, 4070, 100, 5, 4072, 0, 3969, 2679, 0, 10, 13, 119, 0, 30, 0, 0, 0, 3056, 0, 0, 0, 2793, 3, 17, 7, 627, 9, 1, 1, 2, 0, 0, 2, 0, 17, 0, 0, 0, 0, 13, 0, 0, 14, 0, 92, 0, 3876] payload_rows_per_dst=[3332, 2638, 8172, 8120, 7176, 8168, 8176, 7902] dup_rows_per_dst=[222, 8, 178, 5390, 3498, 1302, 136, 1118] combine_rows_per_src=[3332, 2638, 8172, 8120, 7176, 8168, 8176, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57208 payload_rows_remote=49090 dup_rows=8328 dup_rows_remote=8266 dedup_ratio=0.127075 router_tokens_per_expert=[0, 365, 514, 2, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 1169, 0, 2, 0, 0, 4086, 35, 0, 4089, 0, 4055, 2469, 0, 0, 0, 304, 5, 135, 0, 2, 3, 3204, 0, 0, 0, 1943, 19, 1, 0, 1966, 0, 0, 1, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 216, 4, 4062] payload_rows_per_dst=[8192, 8176, 8118, 5460, 6992, 3944, 8182, 8144] dup_rows_per_dst=[2394, 2342, 62, 366, 2718, 2, 6, 438] combine_rows_per_src=[8192, 8176, 8118, 5460, 6992, 3944, 8182, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56328 payload_rows_remote=50872 dup_rows=9208 dup_rows_remote=8770 dedup_ratio=0.140503 router_tokens_per_expert=[0, 591, 1962, 0, 2, 0, 4078, 1, 0, 0, 0, 1, 0, 1061, 0, 0, 0, 0, 4092, 35, 0, 4052, 1, 4094, 2410, 0, 0, 0, 371, 74, 92, 0, 0, 0, 1804, 3, 0, 0, 1902, 3, 20, 0, 1984, 1, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 1, 39, 0, 4085] payload_rows_per_dst=[6104, 8190, 8190, 5456, 5834, 6278, 8104, 8172] dup_rows_per_dst=[3264, 2118, 70, 438, 1590, 1638, 2, 88] combine_rows_per_src=[6104, 8190, 8190, 5456, 5834, 6278, 8104, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54294 payload_rows_remote=47442 dup_rows=11242 dup_rows_remote=9226 dedup_ratio=0.171539 router_tokens_per_expert=[0, 876, 452, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 1077, 0, 0, 0, 0, 4096, 23, 0, 4096, 0, 4087, 1990, 0, 0, 0, 201, 23, 97, 0, 0, 0, 2550, 0, 0, 0, 1882, 2, 2, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 133, 0, 3985] payload_rows_per_dst=[2388, 8192, 8174, 4326, 6852, 8190, 8192, 7980] dup_rows_per_dst=[268, 2156, 46, 296, 2016, 6196, 0, 264] combine_rows_per_src=[2388, 8192, 8174, 4326, 6852, 8190, 8192, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53226 payload_rows_remote=45036 dup_rows=12310 dup_rows_remote=8344 dedup_ratio=0.187836 router_tokens_per_expert=[0, 1229, 664, 0, 3, 0, 4093, 0, 0, 2, 0, 23, 0, 1290, 0, 0, 1, 0, 4090, 189, 0, 4093, 0, 4083, 1541, 0, 3, 0, 51, 27, 69, 3, 0, 0, 3561, 3, 0, 0, 1594, 1, 10, 0, 1974, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 1, 4070] payload_rows_per_dst=[3408, 6416, 5202, 6038, 7640, 8190, 8188, 8144] dup_rows_per_dst=[384, 2178, 232, 2678, 2678, 3966, 8, 186] combine_rows_per_src=[3408, 6416, 5202, 6038, 7640, 8190, 8188, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53458 payload_rows_remote=45298 dup_rows=12078 dup_rows_remote=12074 dedup_ratio=0.184296 router_tokens_per_expert=[0, 425, 825, 0, 7, 0, 4089, 0, 0, 0, 0, 23, 5, 1481, 0, 5, 0, 0, 4075, 33, 0, 4079, 0, 4074, 2450, 0, 0, 0, 200, 73, 280, 0, 0, 0, 3227, 0, 0, 0, 1686, 7, 3, 3, 1263, 4, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 9, 0, 1, 0, 0, 351, 0, 4084] payload_rows_per_dst=[2316, 3016, 8150, 8190, 7260, 8190, 8160, 8176] dup_rows_per_dst=[198, 12, 66, 5964, 2580, 2540, 4, 714] combine_rows_per_src=[2316, 3016, 8150, 8190, 7260, 8190, 8160, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=328366080 dispatch_meta_bytes=206224 combine_payload_bytes=328366080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328572304 ingress_bytes=438466560 link_bytes=438466560 token_bytes=656938384 weight_movement_bytes=110100480 total_nvlink_bytes=767038864 weight_to_token_ratio=0.167596 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.625654632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=328165376 dispatch_meta_bytes=226064 combine_payload_bytes=328165376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=328391440 ingress_bytes=548366336 link_bytes=548366336 token_bytes=656556816 weight_movement_bytes=220200960 total_nvlink_bytes=876757776 weight_to_token_ratio=0.335388 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.536992491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351877120 dispatch_meta_bytes=229424 combine_payload_bytes=351877120 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=352106544 ingress_bytes=902379520 link_bytes=902379520 token_bytes=703983664 weight_movement_bytes=550502400 total_nvlink_bytes=1254486064 weight_to_token_ratio=0.781982 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.569386401e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364650496 dispatch_meta_bytes=238568 combine_payload_bytes=364650496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364889064 ingress_bytes=474750976 link_bytes=474750976 token_bytes=729539560 weight_movement_bytes=110100480 total_nvlink_bytes=839640040 weight_to_token_ratio=0.150918 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.256700304e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340064256 dispatch_meta_bytes=226672 combine_payload_bytes=340064256 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=340290928 ingress_bytes=560265216 link_bytes=560265216 token_bytes=680355184 weight_movement_bytes=220200960 total_nvlink_bytes=900556144 weight_to_token_ratio=0.323656 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.743933584e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322818048 dispatch_meta_bytes=213520 combine_payload_bytes=322818048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323031568 ingress_bytes=432918528 link_bytes=432918528 token_bytes=645849616 weight_movement_bytes=110100480 total_nvlink_bytes=755950096 weight_to_token_ratio=0.170474 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.529165231e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=324696064 dispatch_meta_bytes=229488 combine_payload_bytes=324696064 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=324925552 ingress_bytes=544897024 link_bytes=544897024 token_bytes=649621616 weight_movement_bytes=220200960 total_nvlink_bytes=869822576 weight_to_token_ratio=0.338968 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.476655449e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53718 payload_rows_remote=45638 dup_rows=11818 dup_rows_remote=8620 dedup_ratio=0.180328 router_tokens_per_expert=[0, 695, 654, 13, 4, 9, 4090, 4, 0, 0, 0, 3, 0, 1599, 0, 0, 0, 0, 4037, 107, 1, 4077, 0, 4068, 2769, 0, 3, 0, 160, 12, 64, 0, 0, 0, 2584, 1, 0, 0, 2015, 5, 4, 0, 1641, 2, 0, 0, 0, 7, 3, 0, 4, 8, 0, 0, 0, 0, 5, 1, 1, 0, 15, 30, 4, 4069] payload_rows_per_dst=[8192, 8080, 8140, 5640, 4034, 3302, 8172, 8158] dup_rows_per_dst=[7914, 3198, 212, 376, 8, 6, 12, 92] combine_rows_per_src=[8192, 8080, 8140, 5640, 4034, 3302, 8172, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=327133184 dispatch_meta_bytes=217032 combine_payload_bytes=327133184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327350216 ingress_bytes=437233664 link_bytes=437233664 token_bytes=654483400 weight_movement_bytes=110100480 total_nvlink_bytes=764583880 weight_to_token_ratio=0.168225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.604212543e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53490 payload_rows_remote=45298 dup_rows=12046 dup_rows_remote=3278 dedup_ratio=0.183807 router_tokens_per_expert=[0, 486, 1053, 0, 4, 0, 4096, 0, 0, 0, 0, 2, 0, 310, 0, 0, 1, 0, 4096, 51, 0, 4094, 0, 4084, 3229, 0, 0, 0, 138, 0, 32, 0, 0, 0, 2841, 0, 0, 0, 2330, 0, 5, 0, 1807, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 37, 0, 4063] payload_rows_per_dst=[8192, 8192, 8190, 6518, 4660, 3620, 8190, 5928] dup_rows_per_dst=[8768, 624, 2334, 280, 0, 4, 4, 32] combine_rows_per_src=[8192, 8192, 8190, 6518, 4660, 3620, 8190, 5928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59990 payload_rows_remote=52304 dup_rows=5546 dup_rows_remote=5494 dedup_ratio=0.084625 router_tokens_per_expert=[0, 4, 0, 0, 4, 3, 0, 4067, 0, 3679, 0, 0, 0, 3764, 539, 21, 0, 0, 24, 3841, 1, 0, 3, 0, 2793, 0, 1, 119, 0, 0, 30, 0, 0, 0, 1920, 0, 0, 4090, 0, 2392, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 1354, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 14] payload_rows_per_dst=[8134, 7472, 7686, 5766, 6798, 8192, 7754, 8188] dup_rows_per_dst=[22, 1006, 52, 120, 1826, 2, 2494, 24] combine_rows_per_src=[8134, 7472, 7686, 5766, 6798, 8192, 7754, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57816 payload_rows_remote=49640 dup_rows=7720 dup_rows_remote=7670 dedup_ratio=0.117798 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 3663, 0, 3747, 0, 1, 0, 3661, 1766, 26, 0, 0, 3, 3779, 0, 0, 0, 1, 1686, 0, 0, 26, 0, 18, 263, 0, 0, 0, 1503, 22, 0, 4075, 0, 3265, 0, 4092, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 1120, 0, 5, 0, 0, 0, 1, 0, 26, 0, 10, 1] payload_rows_per_dst=[7328, 3572, 7560, 7954, 7370, 8192, 7664, 8176] dup_rows_per_dst=[2, 14, 6, 3526, 2180, 32, 1910, 50] combine_rows_per_src=[7328, 3572, 7560, 7954, 7370, 8192, 7664, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58718 payload_rows_remote=53704 dup_rows=6818 dup_rows_remote=6680 dedup_ratio=0.104034 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 4049, 0, 3732, 0, 0, 0, 3459, 1131, 32, 0, 0, 0, 3738, 0, 0, 0, 7, 2456, 0, 0, 88, 0, 2, 28, 2, 0, 0, 2677, 1, 0, 4092, 0, 1860, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1295, 3, 0, 0, 0, 0, 12, 1, 4, 0, 0, 0] payload_rows_per_dst=[8100, 7704, 7480, 5014, 6580, 8190, 7458, 8192] dup_rows_per_dst=[0, 2086, 10, 138, 2496, 0, 2062, 26] combine_rows_per_src=[8100, 7704, 7480, 5014, 6580, 8190, 7458, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58824 payload_rows_remote=51000 dup_rows=6712 dup_rows_remote=2874 dedup_ratio=0.102417 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4086, 0, 2649, 0, 0, 0, 3754, 374, 40, 0, 0, 0, 3702, 0, 0, 0, 0, 2825, 0, 0, 58, 0, 6, 11, 0, 0, 0, 2895, 0, 0, 4003, 0, 2936, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1300, 0, 0, 0, 0, 0, 30, 0, 1, 0, 0, 0] payload_rows_per_dst=[8172, 5716, 7404, 5754, 7824, 8192, 7716, 8046] dup_rows_per_dst=[0, 410, 0, 46, 3838, 0, 2396, 22] combine_rows_per_src=[8172, 5716, 7404, 5754, 7824, 8192, 7716, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58140 payload_rows_remote=49950 dup_rows=7396 dup_rows_remote=7396 dedup_ratio=0.112854 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 4056, 0, 3242, 5, 2, 0, 3829, 2301, 4, 0, 0, 5, 3831, 0, 0, 0, 1, 2152, 0, 0, 5, 1, 3, 44, 0, 0, 0, 2467, 0, 0, 4015, 0, 1490, 0, 4094, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 1201, 0, 0, 3, 0, 0, 3, 0, 1, 0, 0, 1] payload_rows_per_dst=[8114, 5236, 7672, 6968, 6116, 8190, 7810, 8034] dup_rows_per_dst=[0, 754, 2, 2560, 1798, 0, 2276, 6] combine_rows_per_src=[8114, 5236, 7672, 6968, 6116, 8190, 7810, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=324696064 dispatch_meta_bytes=194304 combine_payload_bytes=324696064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=324890368 ingress_bytes=544897024 link_bytes=544897024 token_bytes=649586432 weight_movement_bytes=220200960 total_nvlink_bytes=869787392 weight_to_token_ratio=0.338986 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.476655449e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55408 payload_rows_remote=48492 dup_rows=10128 dup_rows_remote=8852 dedup_ratio=0.154541 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3895, 0, 3724, 0, 3, 0, 3196, 927, 103, 1, 0, 12, 3676, 0, 0, 0, 0, 3556, 0, 2, 8, 0, 2, 131, 1, 0, 0, 1870, 6, 1, 4088, 0, 2559, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 894, 1, 0, 0, 0, 3, 5, 6, 2, 0, 0, 0] payload_rows_per_dst=[7790, 2038, 7364, 8132, 6810, 8182, 6916, 8176] dup_rows_per_dst=[0, 28, 14, 6716, 2062, 0, 1276, 32] combine_rows_per_src=[7790, 2038, 7364, 8132, 6810, 8182, 6916, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374915072 dispatch_meta_bytes=231192 combine_payload_bytes=374915072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375146264 ingress_bytes=485015552 link_bytes=485015552 token_bytes=750061336 weight_movement_bytes=110100480 total_nvlink_bytes=860161816 weight_to_token_ratio=0.146789 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.435218163e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=355819520 dispatch_meta_bytes=229240 combine_payload_bytes=355819520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356048760 ingress_bytes=465920000 link_bytes=465920000 token_bytes=711868280 weight_movement_bytes=110100480 total_nvlink_bytes=821968760 weight_to_token_ratio=0.154664 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.103115107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57220 payload_rows_remote=49440 dup_rows=8316 dup_rows_remote=3150 dedup_ratio=0.126892 router_tokens_per_expert=[0, 5, 0, 0, 1, 0, 2, 4038, 0, 3363, 0, 12, 0, 3119, 822, 9, 0, 13, 3, 3743, 1, 1, 0, 1, 3146, 0, 0, 46, 0, 2, 8, 0, 0, 0, 3018, 0, 5, 4068, 0, 1619, 0, 4081, 1, 1, 0, 1, 0, 2, 11, 5, 2, 1, 1547, 4, 2, 0, 1, 11, 15, 1, 8, 0, 27, 2] payload_rows_per_dst=[8088, 7780, 7510, 6360, 6820, 8170, 4324, 8168] dup_rows_per_dst=[4, 5166, 258, 44, 2220, 2, 524, 98] combine_rows_per_src=[8088, 7780, 7510, 6360, 6820, 8170, 4324, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=384950272 dispatch_meta_bytes=241536 combine_payload_bytes=384950272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385191808 ingress_bytes=495050752 link_bytes=495050752 token_bytes=770142080 weight_movement_bytes=110100480 total_nvlink_bytes=880242560 weight_to_token_ratio=0.142961 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.609746796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=365568000 dispatch_meta_bytes=215496 combine_payload_bytes=365568000 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=365783496 ingress_bytes=916070400 link_bytes=916070400 token_bytes=731351496 weight_movement_bytes=550502400 total_nvlink_bytes=1281853896 weight_to_token_ratio=0.752719 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.593197093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=358041600 dispatch_meta_bytes=229384 combine_payload_bytes=358041600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358270984 ingress_bytes=468142080 link_bytes=468142080 token_bytes=716312584 weight_movement_bytes=110100480 total_nvlink_bytes=826413064 weight_to_token_ratio=0.153705 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.141760733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=347590656 dispatch_meta_bytes=229376 combine_payload_bytes=347590656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347820032 ingress_bytes=457691136 link_bytes=457691136 token_bytes=695410688 weight_movement_bytes=110100480 total_nvlink_bytes=805511168 weight_to_token_ratio=0.158324 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.960001628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354385920 dispatch_meta_bytes=210360 combine_payload_bytes=354385920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=354596280 ingress_bytes=684687360 link_bytes=684687360 token_bytes=708982200 weight_movement_bytes=330301440 total_nvlink_bytes=1039283640 weight_to_token_ratio=0.465881 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.190783931e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50868 payload_rows_remote=42718 dup_rows=14668 dup_rows_remote=13036 dedup_ratio=0.223816 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4070, 0, 3113, 0, 0, 0, 3914, 793, 28, 0, 0, 5, 3929, 0, 0, 0, 0, 3047, 0, 0, 138, 0, 2, 32, 0, 0, 0, 2325, 0, 0, 4092, 0, 1624, 0, 4096, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1460, 0, 0, 0, 0, 0, 56, 0, 0, 0, 25, 1] payload_rows_per_dst=[8150, 8182, 8148, 6272, 798, 8192, 2950, 8176] dup_rows_per_dst=[1632, 8372, 4370, 166, 0, 0, 6, 122] combine_rows_per_src=[8150, 8182, 8148, 6272, 798, 8192, 2950, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58598 payload_rows_remote=50410 dup_rows=6938 dup_rows_remote=6932 dedup_ratio=0.105865 router_tokens_per_expert=[4087, 0, 0, 0, 2031, 3710, 1, 0, 0, 0, 0, 2, 4093, 2, 0, 2020, 0, 1, 0, 4094, 0, 0, 2, 0, 6, 4019, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 3855, 3707, 0, 1118, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8058, 7990, 8188, 8044, 8176, 8192, 2236, 7714] dup_rows_per_dst=[3426, 3472, 6, 20, 0, 8, 0, 6] combine_rows_per_src=[8058, 7990, 8188, 8044, 8176, 8192, 2236, 7714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59716 payload_rows_remote=51690 dup_rows=5820 dup_rows_remote=5808 dedup_ratio=0.088806 router_tokens_per_expert=[4096, 0, 0, 0, 1437, 3909, 0, 0, 0, 17, 0, 0, 4094, 33, 0, 1648, 0, 3, 2, 4092, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 3914, 1, 1563, 0, 0, 0, 0, 0, 0, 1, 2] payload_rows_per_dst=[8100, 8044, 8188, 8026, 8192, 8188, 3128, 7850] dup_rows_per_dst=[2592, 3180, 6, 12, 4, 20, 0, 6] combine_rows_per_src=[8100, 8044, 8188, 8026, 8192, 8188, 3128, 7850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60418 payload_rows_remote=52226 dup_rows=5118 dup_rows_remote=5118 dedup_ratio=0.078094 router_tokens_per_expert=[4096, 0, 0, 0, 1085, 3854, 0, 0, 0, 28, 0, 0, 4096, 11, 0, 2113, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 4010, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8042, 4466, 8190, 8140, 8192, 8192, 7678, 7518] dup_rows_per_dst=[1836, 430, 0, 0, 0, 12, 2840, 0] combine_rows_per_src=[8042, 4466, 8190, 8140, 8192, 8192, 7678, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56280 payload_rows_remote=48530 dup_rows=9256 dup_rows_remote=5960 dedup_ratio=0.141235 router_tokens_per_expert=[4079, 0, 0, 0, 2121, 3900, 0, 8, 2, 0, 0, 4, 4090, 9, 0, 1932, 0, 0, 3, 4078, 0, 0, 0, 0, 0, 3865, 23, 0, 0, 69, 0, 1, 0, 0, 2, 12, 0, 0, 1, 0, 2, 1, 5, 3, 0, 0, 0, 0, 0, 6, 0, 1, 3700, 3889, 6, 954, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4256, 3888, 8162, 7762, 8180, 8186, 8096, 7750] dup_rows_per_dst=[2, 6, 0, 154, 8, 16, 5774, 3296] combine_rows_per_src=[4256, 3888, 8162, 7762, 8180, 8186, 8096, 7750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59584 payload_rows_remote=51392 dup_rows=5952 dup_rows_remote=5942 dedup_ratio=0.090820 router_tokens_per_expert=[4096, 0, 0, 0, 1207, 3384, 0, 0, 0, 4, 0, 1, 4094, 3, 0, 2330, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 3615, 3497, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6960, 4670, 8192, 8116, 8192, 8192, 8032, 7230] dup_rows_per_dst=[1558, 6, 54, 584, 2, 10, 3738, 0] combine_rows_per_src=[6960, 4670, 8192, 8116, 8192, 8192, 8032, 7230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56132 payload_rows_remote=47990 dup_rows=9404 dup_rows_remote=1454 dedup_ratio=0.143494 router_tokens_per_expert=[4092, 1, 0, 0, 1100, 3715, 1, 0, 1, 2, 0, 7, 4095, 3, 0, 2585, 0, 0, 0, 4088, 0, 1, 0, 2, 0, 3977, 6, 0, 3, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4024, 3520, 4, 1499, 0, 4, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[2204, 5190, 8182, 7986, 8188, 8192, 8142, 8048] dup_rows_per_dst=[0, 6, 0, 1394, 14, 28, 7950, 12] combine_rows_per_src=[2204, 5190, 8182, 7986, 8188, 8192, 8142, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306202624 dispatch_meta_bytes=223016 combine_payload_bytes=306202624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306425640 ingress_bytes=416303104 link_bytes=416303104 token_bytes=612628264 weight_movement_bytes=110100480 total_nvlink_bytes=722728744 weight_to_token_ratio=0.179718 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.240195680e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60198 payload_rows_remote=52056 dup_rows=5338 dup_rows_remote=1062 dedup_ratio=0.081451 router_tokens_per_expert=[4079, 0, 0, 1, 654, 3691, 0, 0, 0, 5, 0, 0, 4092, 5, 3, 2234, 0, 6, 1, 4075, 0, 0, 4, 2, 0, 3760, 21, 0, 0, 1, 0, 1, 0, 0, 0, 17, 2, 0, 0, 0, 4, 2, 2, 0, 0, 0, 11, 2, 0, 8, 0, 0, 4020, 3962, 0, 2102, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7724, 8142, 8156, 7544, 8192, 8192, 4206, 8042] dup_rows_per_dst=[968, 4276, 20, 22, 4, 34, 14, 0] combine_rows_per_src=[7724, 8142, 8156, 7544, 8192, 8192, 4206, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=361338880 dispatch_meta_bytes=229368 combine_payload_bytes=361338880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361568248 ingress_bytes=471439360 link_bytes=471439360 token_bytes=722907128 weight_movement_bytes=110100480 total_nvlink_bytes=833007608 weight_to_token_ratio=0.152302 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.199105855e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370513920 dispatch_meta_bytes=229992 combine_payload_bytes=370513920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370743912 ingress_bytes=480614400 link_bytes=480614400 token_bytes=741257832 weight_movement_bytes=110100480 total_nvlink_bytes=851358312 weight_to_token_ratio=0.148532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.358674891e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374355968 dispatch_meta_bytes=229376 combine_payload_bytes=374355968 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=374585344 ingress_bytes=594556928 link_bytes=594556928 token_bytes=748941312 weight_movement_bytes=220200960 total_nvlink_bytes=969142272 weight_to_token_ratio=0.294016 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034032286e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368377856 dispatch_meta_bytes=229336 combine_payload_bytes=368377856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368607192 ingress_bytes=478478336 link_bytes=478478336 token_bytes=736985048 weight_movement_bytes=110100480 total_nvlink_bytes=847085528 weight_to_token_ratio=0.149393 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.321525225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=343992320 dispatch_meta_bytes=197776 combine_payload_bytes=343992320 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=344190096 ingress_bytes=784394240 link_bytes=784394240 token_bytes=688182416 weight_movement_bytes=440401920 total_nvlink_bytes=1128584336 weight_to_token_ratio=0.639949 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.364190594e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=347863040 dispatch_meta_bytes=217960 combine_payload_bytes=347863040 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=348081000 ingress_bytes=568064000 link_bytes=568064000 token_bytes=695944040 weight_movement_bytes=220200960 total_nvlink_bytes=916145000 weight_to_token_ratio=0.316406 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.879567265e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373137408 dispatch_meta_bytes=212472 combine_payload_bytes=373137408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373349880 ingress_bytes=593338368 link_bytes=593338368 token_bytes=746487288 weight_movement_bytes=220200960 total_nvlink_bytes=966688248 weight_to_token_ratio=0.294983 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031913009e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60184 payload_rows_remote=52134 dup_rows=5352 dup_rows_remote=3592 dedup_ratio=0.081665 router_tokens_per_expert=[4096, 0, 0, 0, 1043, 3862, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 1777, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4031, 4032, 0, 1850, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8050, 8182, 8192, 7732, 8192, 8074, 3700, 8062] dup_rows_per_dst=[1760, 3588, 0, 0, 0, 0, 2, 2] combine_rows_per_src=[8050, 8182, 8192, 7732, 8192, 8074, 3700, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63080 payload_rows_remote=54888 dup_rows=2456 dup_rows_remote=854 dedup_ratio=0.037476 router_tokens_per_expert=[0, 3415, 0, 0, 0, 0, 0, 0, 4075, 0, 2, 0, 1, 4091, 0, 0, 0, 0, 801, 0, 0, 4096, 0, 0, 0, 0, 78, 0, 310, 4, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 2, 0, 3749, 0, 2, 0, 0, 0, 0, 1, 0, 0, 121, 0, 0, 4096, 0, 1, 0, 0] payload_rows_per_dst=[6830, 8182, 8192, 7684, 8192, 7652, 8156, 8192] dup_rows_per_dst=[0, 6, 1602, 598, 4, 2, 0, 244] combine_rows_per_src=[6830, 8182, 8192, 7684, 8192, 7652, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62334 payload_rows_remote=54268 dup_rows=3202 dup_rows_remote=1348 dedup_ratio=0.048859 router_tokens_per_expert=[0, 3244, 0, 0, 0, 2, 0, 0, 4071, 4, 0, 0, 0, 4039, 0, 0, 0, 0, 600, 0, 0, 4096, 0, 0, 0, 0, 304, 0, 747, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 1, 2, 3909, 0, 0, 0, 1, 0, 0, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[6490, 8082, 8192, 8066, 8192, 6976, 8144, 8192] dup_rows_per_dst=[2, 4, 1200, 1854, 6, 0, 0, 136] combine_rows_per_src=[6490, 8082, 8192, 8066, 8192, 6976, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60254 payload_rows_remote=52062 dup_rows=5282 dup_rows_remote=5282 dedup_ratio=0.080597 router_tokens_per_expert=[0, 3187, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4041, 0, 0, 0, 0, 1248, 0, 0, 4096, 0, 0, 0, 0, 91, 0, 1395, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[6374, 8082, 8192, 7984, 8192, 5046, 8192, 8192] dup_rows_per_dst=[0, 2, 2496, 2556, 0, 0, 0, 228] combine_rows_per_src=[6374, 8082, 8192, 7984, 8192, 5046, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60868 payload_rows_remote=53276 dup_rows=4668 dup_rows_remote=4668 dedup_ratio=0.071228 router_tokens_per_expert=[0, 2528, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4030, 0, 0, 1, 0, 1608, 0, 0, 4092, 1, 1, 1, 0, 196, 0, 657, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 2, 0, 3519, 0, 0, 2, 0, 4, 0, 0, 0, 0, 46, 0, 0, 4094, 0, 2, 0, 0] payload_rows_per_dst=[5056, 8064, 8186, 7396, 8190, 7592, 8192, 8192] dup_rows_per_dst=[0, 0, 3220, 1350, 0, 0, 6, 92] combine_rows_per_src=[5056, 8064, 8186, 7396, 8190, 7592, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61984 payload_rows_remote=53796 dup_rows=3552 dup_rows_remote=3552 dedup_ratio=0.054199 router_tokens_per_expert=[0, 3084, 1, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4068, 0, 0, 0, 0, 962, 0, 0, 4095, 2, 0, 0, 0, 253, 0, 519, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 5, 0, 3705, 0, 5, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4095, 0, 2, 0, 0] payload_rows_per_dst=[6170, 8138, 8192, 7344, 8186, 7576, 8188, 8190] dup_rows_per_dst=[0, 0, 1926, 1178, 0, 356, 0, 92] combine_rows_per_src=[6170, 8138, 8192, 7344, 8186, 7576, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54272 payload_rows_remote=46086 dup_rows=11264 dup_rows_remote=11130 dedup_ratio=0.171875 router_tokens_per_expert=[0, 3591, 3, 16, 0, 0, 0, 0, 4074, 5, 1, 0, 3, 4086, 0, 5, 0, 0, 1182, 0, 0, 4094, 0, 0, 6, 0, 73, 0, 359, 0, 2, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 3, 3448, 0, 17, 0, 2, 0, 1, 0, 0, 0, 66, 0, 0, 4082, 0, 7, 0, 5] payload_rows_per_dst=[7200, 8096, 5380, 862, 8180, 8188, 8180, 8186] dup_rows_per_dst=[20, 20, 1364, 18, 2, 9614, 92, 134] combine_rows_per_src=[7200, 8096, 5380, 862, 8180, 8188, 8180, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58488 payload_rows_remote=50350 dup_rows=7048 dup_rows_remote=7048 dedup_ratio=0.107544 router_tokens_per_expert=[0, 3397, 0, 0, 2, 0, 0, 1, 4095, 0, 0, 1, 0, 4068, 0, 0, 5, 0, 1395, 0, 0, 4078, 9, 5, 0, 2, 42, 0, 677, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4, 0, 0, 5, 3030, 1, 0, 3678, 0, 0, 0, 1, 0, 0, 1, 0, 0, 72, 0, 9, 4095, 2, 0, 2, 0] payload_rows_per_dst=[8046, 8138, 4080, 7584, 8188, 6070, 8190, 8192] dup_rows_per_dst=[5246, 0, 412, 1214, 2, 2, 4, 168] combine_rows_per_src=[8046, 8138, 4080, 7584, 8188, 6070, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373696512 dispatch_meta_bytes=222904 combine_payload_bytes=373696512 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=373919416 ingress_bytes=924198912 link_bytes=924198912 token_bytes=747615928 weight_movement_bytes=550502400 total_nvlink_bytes=1298118328 weight_to_token_ratio=0.736344 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.607333913e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393437184 dispatch_meta_bytes=222968 combine_payload_bytes=393437184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=393660152 ingress_bytes=723738624 link_bytes=723738624 token_bytes=787097336 weight_movement_bytes=330301440 total_nvlink_bytes=1117398776 weight_to_token_ratio=0.419645 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.258700502e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388993024 dispatch_meta_bytes=222464 combine_payload_bytes=388993024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389215488 ingress_bytes=499093504 link_bytes=499093504 token_bytes=778208512 weight_movement_bytes=110100480 total_nvlink_bytes=888308992 weight_to_token_ratio=0.141479 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=373180416 dispatch_meta_bytes=229376 combine_payload_bytes=373180416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373409792 ingress_bytes=483280896 link_bytes=483280896 token_bytes=746590208 weight_movement_bytes=110100480 total_nvlink_bytes=856690688 weight_to_token_ratio=0.147471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.405049642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=381882368 dispatch_meta_bytes=231776 combine_payload_bytes=381882368 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382114144 ingress_bytes=602083328 link_bytes=602083328 token_bytes=763996512 weight_movement_bytes=220200960 total_nvlink_bytes=984197472 weight_to_token_ratio=0.288222 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047121933e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385609728 dispatch_meta_bytes=229392 combine_payload_bytes=385609728 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=385839120 ingress_bytes=605810688 link_bytes=605810688 token_bytes=771448848 weight_movement_bytes=220200960 total_nvlink_bytes=991649808 weight_to_token_ratio=0.285438 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.053604425e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=330344448 dispatch_meta_bytes=228864 combine_payload_bytes=330344448 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=330573312 ingress_bytes=660645888 link_bytes=660645888 token_bytes=660917760 weight_movement_bytes=330301440 total_nvlink_bytes=991219200 weight_to_token_ratio=0.499762 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.148971857e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=360908800 dispatch_meta_bytes=229592 combine_payload_bytes=360908800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=361138392 ingress_bytes=691210240 link_bytes=691210240 token_bytes=722047192 weight_movement_bytes=330301440 total_nvlink_bytes=1052348632 weight_to_token_ratio=0.457451 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.202128292e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56888 payload_rows_remote=48696 dup_rows=8648 dup_rows_remote=2738 dedup_ratio=0.131958 router_tokens_per_expert=[0, 2955, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1106, 0, 0, 4096, 0, 0, 0, 0, 110, 0, 1004, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 0, 0, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2758, 7900, 8192, 6462, 8192, 7000] dup_rows_per_dst=[5910, 530, 144, 1948, 10, 0, 0, 106] combine_rows_per_src=[8192, 8192, 2758, 7900, 8192, 6462, 8192, 7000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59816 payload_rows_remote=51640 dup_rows=5720 dup_rows_remote=5514 dedup_ratio=0.087280 router_tokens_per_expert=[0, 18, 0, 3876, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 659, 0, 2, 0, 0, 102, 4087, 0, 0, 0, 0, 0, 0, 3, 0, 2521, 3379, 0, 2137, 162, 0, 1, 0, 0, 53, 0, 2, 0, 0, 1, 0, 3948, 0, 0, 0, 0, 1, 0, 0, 3365, 1, 4095, 1, 254, 0, 0, 2, 0, 0] payload_rows_per_dst=[7786, 8192, 8176, 7544, 5298, 7900, 6730, 8190] dup_rows_per_dst=[6, 1318, 206, 3106, 564, 2, 4, 514] combine_rows_per_src=[7786, 8192, 8176, 7544, 5298, 7900, 6730, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57902 payload_rows_remote=50638 dup_rows=7634 dup_rows_remote=5164 dedup_ratio=0.116486 router_tokens_per_expert=[0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 991, 0, 0, 0, 0, 112, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 2726, 0, 2129, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 3111, 0, 4096, 0, 1475, 0, 0, 0, 1, 0] payload_rows_per_dst=[7780, 8192, 8192, 4102, 7264, 7958, 6222, 8192] dup_rows_per_dst=[0, 1982, 224, 0, 2470, 6, 0, 2952] combine_rows_per_src=[7780, 8192, 8192, 4102, 7264, 7958, 6222, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56986 payload_rows_remote=53176 dup_rows=8550 dup_rows_remote=8550 dedup_ratio=0.130463 router_tokens_per_expert=[0, 0, 2, 3893, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 712, 0, 0, 0, 0, 126, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 3329, 0, 1937, 84, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 4095, 1, 1932, 0, 0, 1, 8, 1] payload_rows_per_dst=[7790, 8192, 8188, 3810, 7728, 7760, 5326, 8192] dup_rows_per_dst=[0, 1428, 252, 0, 2976, 10, 0, 3884] combine_rows_per_src=[7790, 8192, 8188, 3810, 7728, 7760, 5326, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59454 payload_rows_remote=51424 dup_rows=6082 dup_rows_remote=6040 dedup_ratio=0.092804 router_tokens_per_expert=[0, 3, 0, 3395, 0, 0, 2, 0, 1, 0, 4096, 0, 0, 0, 0, 786, 0, 3, 0, 0, 137, 4086, 1, 0, 0, 1, 0, 0, 0, 0, 2708, 3900, 0, 1051, 249, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 0, 4013, 1, 0, 0, 0, 0, 0, 0, 3284, 0, 4094, 0, 927, 0, 0, 0, 6, 0] payload_rows_per_dst=[6794, 8192, 8178, 5416, 8088, 8030, 6568, 8188] dup_rows_per_dst=[6, 1574, 276, 2, 2316, 42, 0, 1866] combine_rows_per_src=[6794, 8192, 8178, 5416, 8088, 8030, 6568, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59840 payload_rows_remote=52238 dup_rows=5696 dup_rows_remote=5680 dedup_ratio=0.086914 router_tokens_per_expert=[0, 0, 0, 3605, 2, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 284, 0, 0, 0, 0, 43, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 2253, 3762, 0, 2163, 39, 1, 0, 0, 0, 6, 0, 0, 0, 0, 2, 0, 4059, 1, 0, 0, 0, 8, 0, 0, 3801, 0, 4082, 0, 477, 0, 0, 4, 0, 1] payload_rows_per_dst=[7212, 7740, 8188, 4716, 8074, 8124, 7602, 8184] dup_rows_per_dst=[2, 514, 88, 264, 3868, 0, 16, 944] combine_rows_per_src=[7212, 7740, 8188, 4716, 8074, 8124, 7602, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55726 payload_rows_remote=47538 dup_rows=9810 dup_rows_remote=9010 dedup_ratio=0.149689 router_tokens_per_expert=[0, 1, 3, 3770, 2, 5, 0, 5, 2, 0, 4085, 0, 0, 4, 0, 403, 0, 0, 0, 0, 225, 4089, 0, 15, 0, 0, 0, 1, 0, 3, 2476, 3569, 0, 1858, 278, 0, 0, 3, 1, 12, 2, 0, 0, 10, 0, 0, 3776, 0, 0, 1, 0, 2, 0, 0, 2971, 0, 4069, 1, 1115, 0, 0, 10, 1, 0] payload_rows_per_dst=[7564, 8188, 8190, 7998, 4100, 7574, 7872, 4240] dup_rows_per_dst=[8, 800, 468, 4100, 204, 2, 3854, 374] combine_rows_per_src=[7564, 8188, 8190, 7998, 4100, 7574, 7872, 4240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54474 payload_rows_remote=46290 dup_rows=11062 dup_rows_remote=10476 dedup_ratio=0.168793 router_tokens_per_expert=[0, 0, 20, 3942, 1, 1, 3, 0, 0, 0, 4077, 0, 0, 0, 0, 366, 0, 6, 0, 0, 56, 4088, 0, 0, 0, 0, 0, 1, 5, 2, 2702, 3934, 0, 1368, 139, 0, 0, 0, 0, 4, 1, 0, 0, 0, 2, 0, 3833, 0, 0, 0, 1, 0, 0, 0, 3869, 0, 4091, 0, 252, 0, 0, 4, 0, 0] payload_rows_per_dst=[7926, 732, 5884, 8182, 8126, 7700, 7740, 8184] dup_rows_per_dst=[8, 0, 80, 5392, 2764, 2232, 0, 586] combine_rows_per_src=[7926, 732, 5884, 8182, 8126, 7700, 7740, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=349052928 dispatch_meta_bytes=205736 combine_payload_bytes=349052928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349258664 ingress_bytes=459153408 link_bytes=459153408 token_bytes=698311592 weight_movement_bytes=110100480 total_nvlink_bytes=808412072 weight_to_token_ratio=0.157667 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.985432943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=370155520 dispatch_meta_bytes=228616 combine_payload_bytes=370155520 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=370384136 ingress_bytes=590356480 link_bytes=590356480 token_bytes=740539656 weight_movement_bytes=220200960 total_nvlink_bytes=960740616 weight_to_token_ratio=0.297352 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.026727016e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=362973184 dispatch_meta_bytes=223208 combine_payload_bytes=362973184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363196392 ingress_bytes=473073664 link_bytes=473073664 token_bytes=726169576 weight_movement_bytes=110100480 total_nvlink_bytes=836270056 weight_to_token_ratio=0.151618 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.227529089e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=368607232 dispatch_meta_bytes=229856 combine_payload_bytes=368607232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368837088 ingress_bytes=478707712 link_bytes=478707712 token_bytes=737444320 weight_movement_bytes=110100480 total_nvlink_bytes=847544800 weight_to_token_ratio=0.149300 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.325514451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=381165568 dispatch_meta_bytes=246904 combine_payload_bytes=381165568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381412472 ingress_bytes=601366528 link_bytes=601366528 token_bytes=762578040 weight_movement_bytes=220200960 total_nvlink_bytes=982779000 weight_to_token_ratio=0.288759 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045875300e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=374441984 dispatch_meta_bytes=231672 combine_payload_bytes=374441984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374673656 ingress_bytes=484542464 link_bytes=484542464 token_bytes=749115640 weight_movement_bytes=110100480 total_nvlink_bytes=859216120 weight_to_token_ratio=0.146974 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.426990384e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=340752384 dispatch_meta_bytes=226192 combine_payload_bytes=340752384 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=340978576 ingress_bytes=671053824 link_bytes=671053824 token_bytes=681730960 weight_movement_bytes=330301440 total_nvlink_bytes=1012032400 weight_to_token_ratio=0.484504 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.167072969e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=331806720 dispatch_meta_bytes=227064 combine_payload_bytes=331806720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=332033784 ingress_bytes=662108160 link_bytes=662108160 token_bytes=663840504 weight_movement_bytes=330301440 total_nvlink_bytes=994141944 weight_to_token_ratio=0.497561 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.151514988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49474 payload_rows_remote=41432 dup_rows=16062 dup_rows_remote=11450 dedup_ratio=0.245087 router_tokens_per_expert=[0, 3, 0, 3882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1506, 0, 0, 0, 0, 377, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 3668, 0, 839, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 23, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 4096, 0, 521, 0, 0, 2, 0, 0] payload_rows_per_dst=[8042, 6310, 8192, 7914, 1696, 8150, 8124, 1046] dup_rows_per_dst=[4612, 2182, 754, 2990, 0, 46, 5478, 0] combine_rows_per_src=[8042, 6310, 8192, 7914, 1696, 8150, 8124, 1046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60344 payload_rows_remote=52162 dup_rows=5192 dup_rows_remote=4012 dedup_ratio=0.079224 router_tokens_per_expert=[294, 0, 0, 0, 0, 4066, 0, 0, 1, 4028, 0, 0, 0, 0, 2, 0, 0, 0, 590, 0, 0, 0, 0, 0, 343, 0, 369, 0, 0, 0, 0, 43, 0, 1848, 1462, 4091, 0, 4095, 0, 163, 0, 0, 2628, 0, 1272, 0, 0, 4090, 0, 12, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1, 0, 0, 0, 3285] payload_rows_per_dst=[8132, 8060, 8182, 8192, 6218, 6796, 8192, 6572] dup_rows_per_dst=[588, 2, 1180, 1498, 728, 1004, 192, 0] combine_rows_per_src=[8132, 8060, 8182, 8192, 6218, 6796, 8192, 6572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58586 payload_rows_remote=52412 dup_rows=6950 dup_rows_remote=5362 dedup_ratio=0.106049 router_tokens_per_expert=[696, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 497, 0, 329, 0, 0, 0, 0, 1, 0, 3429, 1548, 4086, 0, 4096, 0, 39, 0, 0, 1907, 0, 292, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 3186] payload_rows_per_dst=[8192, 8190, 8180, 8192, 6174, 4264, 8192, 7202] dup_rows_per_dst=[1392, 0, 626, 1654, 1588, 134, 116, 1440] combine_rows_per_src=[8192, 8190, 8180, 8192, 6174, 4264, 8192, 7202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57362 payload_rows_remote=51770 dup_rows=8174 dup_rows_remote=7822 dedup_ratio=0.124725 router_tokens_per_expert=[253, 0, 0, 2, 3, 4095, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 1, 0, 0, 739, 0, 0, 0, 0, 0, 814, 0, 227, 0, 0, 0, 0, 4, 6, 3140, 1547, 4095, 0, 4096, 0, 34, 0, 0, 2520, 0, 452, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 84, 0, 2, 0, 0, 0, 0, 0, 0, 2464] payload_rows_per_dst=[8190, 8186, 8190, 8192, 3132, 5592, 8192, 7688] dup_rows_per_dst=[516, 2, 1478, 2090, 42, 352, 170, 3524] combine_rows_per_src=[8190, 8186, 8190, 8192, 3132, 5592, 8192, 7688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59546 payload_rows_remote=51354 dup_rows=5990 dup_rows_remote=4954 dedup_ratio=0.091400 router_tokens_per_expert=[539, 0, 0, 2, 0, 4087, 0, 0, 0, 4076, 0, 0, 1, 0, 0, 0, 0, 0, 1193, 0, 0, 0, 1, 1, 282, 0, 228, 0, 0, 0, 0, 19, 0, 2115, 1390, 4031, 2, 4095, 0, 326, 1, 0, 2187, 0, 454, 4, 1, 4085, 0, 19, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 3576] payload_rows_per_dst=[8180, 8154, 8126, 8192, 6470, 5082, 8190, 7152] dup_rows_per_dst=[1076, 0, 2326, 1036, 1196, 212, 144, 0] combine_rows_per_src=[8180, 8154, 8126, 8192, 6470, 5082, 8190, 7152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53798 payload_rows_remote=45608 dup_rows=11738 dup_rows_remote=11732 dedup_ratio=0.179108 router_tokens_per_expert=[264, 1, 0, 0, 0, 4092, 1, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 953, 0, 0, 0, 4, 11, 643, 0, 121, 0, 0, 0, 0, 0, 0, 3089, 1441, 4079, 0, 4095, 0, 88, 0, 0, 2777, 0, 415, 1, 1, 4092, 0, 1, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2499] payload_rows_per_dst=[8190, 8188, 5398, 3232, 5268, 7594, 8190, 7738] dup_rows_per_dst=[526, 4, 1322, 328, 1164, 4946, 6, 3442] combine_rows_per_src=[8190, 8188, 5398, 3232, 5268, 7594, 8190, 7738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58530 payload_rows_remote=50342 dup_rows=7006 dup_rows_remote=6996 dedup_ratio=0.106903 router_tokens_per_expert=[478, 2, 1, 0, 0, 4090, 0, 0, 2, 4091, 3, 0, 1, 0, 1, 1, 0, 0, 494, 0, 0, 0, 0, 0, 1304, 0, 131, 0, 0, 0, 0, 0, 0, 2193, 2039, 4086, 2, 4086, 0, 226, 0, 0, 2053, 1, 233, 5, 0, 4092, 0, 1, 2, 1, 1, 0, 13, 0, 15, 0, 0, 0, 0, 1, 2, 3117] payload_rows_per_dst=[8186, 8188, 8176, 8188, 6888, 4458, 8184, 6262] dup_rows_per_dst=[956, 10, 984, 2866, 2032, 126, 24, 8] combine_rows_per_src=[8186, 8188, 8176, 8188, 6888, 4458, 8184, 6262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=296984576 dispatch_meta_bytes=211528 combine_payload_bytes=296984576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297196104 ingress_bytes=407085056 link_bytes=407085056 token_bytes=594180680 weight_movement_bytes=110100480 total_nvlink_bytes=704281160 weight_to_token_ratio=0.185298 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.079878664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45122 payload_rows_remote=38362 dup_rows=20414 dup_rows_remote=18420 dedup_ratio=0.311493 router_tokens_per_expert=[463, 0, 0, 0, 2, 4049, 0, 0, 0, 4073, 0, 0, 8, 0, 0, 0, 0, 0, 477, 0, 0, 0, 13, 1, 837, 0, 379, 0, 0, 0, 0, 40, 6, 1745, 2210, 4089, 1, 4095, 0, 228, 1, 0, 2007, 0, 1209, 2, 1, 4095, 0, 7, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 2632] payload_rows_per_dst=[2444, 8162, 964, 2218, 8192, 8192, 8190, 6760] dup_rows_per_dst=[136, 0, 18, 294, 4974, 12788, 210, 1994] combine_rows_per_src=[2444, 8162, 964, 2218, 8192, 8192, 8190, 6760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373897216 dispatch_meta_bytes=224696 combine_payload_bytes=373897216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374121912 ingress_bytes=483997696 link_bytes=483997696 token_bytes=748019128 weight_movement_bytes=110100480 total_nvlink_bytes=858119608 weight_to_token_ratio=0.147189 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.417515973e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375689216 dispatch_meta_bytes=231096 combine_payload_bytes=375689216 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=375920312 ingress_bytes=926191616 link_bytes=926191616 token_bytes=751609528 weight_movement_bytes=550502400 total_nvlink_bytes=1302111928 weight_to_token_ratio=0.732431 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.610799553e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371087360 dispatch_meta_bytes=238368 combine_payload_bytes=371087360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=371325728 ingress_bytes=591288320 link_bytes=591288320 token_bytes=742413088 weight_movement_bytes=220200960 total_nvlink_bytes=962614048 weight_to_token_ratio=0.296602 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028347639e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=368105472 dispatch_meta_bytes=225232 combine_payload_bytes=368105472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368330704 ingress_bytes=478205952 link_bytes=478205952 token_bytes=736436176 weight_movement_bytes=110100480 total_nvlink_bytes=846536656 weight_to_token_ratio=0.149504 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.316788019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=326918144 dispatch_meta_bytes=229360 combine_payload_bytes=326918144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327147504 ingress_bytes=437018624 link_bytes=437018624 token_bytes=654065648 weight_movement_bytes=110100480 total_nvlink_bytes=764166128 weight_to_token_ratio=0.168332 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.600472643e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=360851456 dispatch_meta_bytes=229352 combine_payload_bytes=360851456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361080808 ingress_bytes=470951936 link_bytes=470951936 token_bytes=721932264 weight_movement_bytes=110100480 total_nvlink_bytes=832032744 weight_to_token_ratio=0.152508 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.190628750e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274978816 dispatch_meta_bytes=227128 combine_payload_bytes=274978816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275205944 ingress_bytes=385079296 link_bytes=385079296 token_bytes=550184760 weight_movement_bytes=110100480 total_nvlink_bytes=660285240 weight_to_token_ratio=0.200115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.697162304e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6903360312 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3452814136 ingress_bytes=5432354816 link_bytes=5432354816 total_nvlink_bytes=8885168952 weight_to_token_ratio=0.287079 prefetch_share_of_ingress=0.202675 dedup_ratio=0.146667 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.447758494e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6974748480 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3488505664 ingress_bytes=6018553856 link_bytes=6018553856 total_nvlink_bytes=9507059520 weight_to_token_ratio=0.363068 prefetch_share_of_ingress=0.164642 dedup_ratio=0.150906 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.046725504e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6802868176 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3402569680 ingress_bytes=5051805696 link_bytes=5051805696 total_nvlink_bytes=8454375376 weight_to_token_ratio=0.242766 prefetch_share_of_ingress=0.196149 dedup_ratio=0.162277 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=11176 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.785920985e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7041362256 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=3521831248 ingress_bytes=4950837248 link_bytes=4950837248 total_nvlink_bytes=8472668496 weight_to_token_ratio=0.203271 prefetch_share_of_ingress=0.200149 dedup_ratio=0.149454 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.610320247e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6671363680 prefetch_bytes=1211105280 reduce_bytes=1101004800 weight_movement_bytes=2312110080 egress_bytes=3336795744 ingress_bytes=5646678016 link_bytes=5646678016 total_nvlink_bytes=8983473760 weight_to_token_ratio=0.346572 prefetch_share_of_ingress=0.214481 dedup_ratio=0.173618 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=10 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=11176 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.820501789e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6613338384 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3307786512 ingress_bytes=5287360512 link_bytes=5287360512 total_nvlink_bytes=8595147024 weight_to_token_ratio=0.299668 prefetch_share_of_ingress=0.208233 dedup_ratio=0.178476 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.195589552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57102 payload_rows_remote=48910 dup_rows=8434 dup_rows_remote=7794 dedup_ratio=0.128693 router_tokens_per_expert=[320, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 467, 0, 119, 0, 0, 0, 0, 0, 0, 2150, 2869, 4048, 0, 4096, 0, 287, 0, 0, 1648, 0, 350, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2877] payload_rows_per_dst=[8192, 8192, 8134, 8192, 7644, 3862, 7132, 5754] dup_rows_per_dst=[640, 198, 2252, 1172, 3826, 134, 212, 0] combine_rows_per_src=[8192, 8192, 8134, 8192, 7644, 3862, 7132, 5754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6321094304 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3161654944 ingress_bytes=5581649920 link_bytes=5581649920 total_nvlink_bytes=8743304864 weight_to_token_ratio=0.383195 prefetch_share_of_ingress=0.177529 dedup_ratio=0.211356 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=11303 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.707407235e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=350586880 dispatch_meta_bytes=226816 combine_payload_bytes=350586880 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=350813696 ingress_bytes=570787840 link_bytes=570787840 token_bytes=701400576 weight_movement_bytes=220200960 total_nvlink_bytes=921601536 weight_to_token_ratio=0.313945 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.926939322e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6327172520 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=3164636584 ingress_bytes=5364545536 link_bytes=5364545536 total_nvlink_bytes=8529182120 weight_to_token_ratio=0.348024 prefetch_share_of_ingress=0.164190 dedup_ratio=0.206537 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=11303 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.329827003e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-29 23:43:04] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 239.2 | throughput per GPU (TFLOP/s/GPU): 516.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.107359E+01 | loss scale: 1.0 | grad norm: 106.691 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46556 payload_rows_remote=39732 dup_rows=18980 dup_rows_remote=15994 dedup_ratio=0.289612 router_tokens_per_expert=[121, 29, 96, 25, 95, 16, 947, 2274, 93, 127, 85, 732, 10, 114, 0, 360, 72, 735, 1694, 13, 70, 278, 232, 123, 2179, 291, 162, 1628, 128, 2353, 829, 7, 13, 20, 93, 1875, 16, 3, 14, 201, 589, 737, 585, 55, 2361, 988, 1023, 734, 404, 996, 505, 92, 355, 370, 789, 866, 983, 850, 118, 98, 318, 160, 639, 0] payload_rows_per_dst=[7130, 6824, 6136, 7082, 4040, 4276, 6154, 4914] dup_rows_per_dst=[4434, 2986, 2274, 3714, 430, 1124, 2600, 1418] combine_rows_per_src=[7130, 6824, 6136, 7082, 4040, 4276, 6154, 4914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47188 payload_rows_remote=40128 dup_rows=18348 dup_rows_remote=14278 dedup_ratio=0.279968 router_tokens_per_expert=[75, 85, 69, 18, 32, 9, 1481, 1794, 124, 19, 112, 417, 17, 56, 5, 306, 233, 538, 2323, 1, 121, 289, 103, 102, 2002, 218, 231, 1328, 80, 1886, 692, 14, 25, 18, 96, 2004, 12, 0, 7, 166, 745, 637, 784, 82, 2363, 1189, 1250, 830, 235, 999, 311, 214, 525, 274, 323, 712, 1714, 1108, 186, 76, 177, 353, 568, 5] payload_rows_per_dst=[7060, 6696, 6792, 6478, 4300, 4316, 5498, 6048] dup_rows_per_dst=[4070, 2642, 3006, 3082, 356, 1178, 1688, 2326] combine_rows_per_src=[7060, 6696, 6792, 6478, 4300, 4316, 5498, 6048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46752 payload_rows_remote=40046 dup_rows=18784 dup_rows_remote=15428 dedup_ratio=0.286621 router_tokens_per_expert=[132, 73, 97, 17, 10, 22, 1036, 1226, 62, 137, 215, 459, 36, 166, 3, 251, 92, 526, 2384, 19, 334, 314, 52, 44, 2294, 217, 408, 1220, 45, 1983, 590, 12, 20, 27, 99, 1766, 9, 4, 9, 339, 978, 773, 908, 24, 2054, 1266, 1393, 821, 272, 1027, 847, 97, 384, 560, 455, 727, 1223, 776, 144, 166, 182, 117, 825, 0] payload_rows_per_dst=[6370, 6552, 6706, 5878, 4850, 5192, 6008, 5196] dup_rows_per_dst=[2606, 3000, 3356, 2232, 1374, 1816, 2730, 1670] combine_rows_per_src=[6370, 6552, 6706, 5878, 4850, 5192, 6008, 5196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46648 payload_rows_remote=40972 dup_rows=18888 dup_rows_remote=16806 dedup_ratio=0.288208 router_tokens_per_expert=[99, 53, 257, 41, 19, 11, 1270, 1693, 178, 67, 189, 406, 7, 124, 2, 373, 90, 691, 1990, 3, 148, 197, 118, 46, 2441, 248, 330, 1531, 22, 2539, 667, 15, 4, 28, 220, 1907, 23, 0, 13, 353, 554, 813, 949, 31, 2095, 959, 698, 895, 77, 656, 650, 111, 412, 292, 390, 396, 1683, 759, 143, 128, 280, 469, 913, 2] payload_rows_per_dst=[5090, 5706, 5338, 5076, 6748, 5648, 5676, 7366] dup_rows_per_dst=[1796, 1176, 1228, 1360, 3426, 2360, 2082, 5460] combine_rows_per_src=[5090, 5706, 5338, 5076, 6748, 5648, 5676, 7366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47408 payload_rows_remote=41722 dup_rows=18128 dup_rows_remote=16280 dedup_ratio=0.276611 router_tokens_per_expert=[58, 72, 24, 7, 11, 14, 1507, 2094, 137, 46, 134, 563, 5, 39, 0, 396, 67, 420, 2593, 22, 77, 213, 88, 133, 1881, 121, 192, 923, 15, 2422, 621, 14, 26, 27, 83, 2326, 3, 0, 50, 381, 862, 441, 816, 14, 1996, 603, 2549, 1217, 235, 718, 525, 23, 230, 439, 380, 597, 1461, 371, 86, 167, 124, 351, 757, 1] payload_rows_per_dst=[5776, 7036, 6350, 5686, 6940, 5684, 4966, 4970] dup_rows_per_dst=[1798, 3506, 1774, 1848, 3696, 2512, 1328, 1666] combine_rows_per_src=[5776, 7036, 6350, 5686, 6940, 5684, 4966, 4970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47478 payload_rows_remote=40738 dup_rows=18058 dup_rows_remote=14108 dedup_ratio=0.275543 router_tokens_per_expert=[70, 20, 392, 26, 4, 12, 1918, 1145, 143, 99, 127, 240, 9, 246, 0, 294, 277, 180, 3025, 0, 107, 209, 92, 68, 1767, 658, 139, 1143, 123, 2156, 655, 8, 11, 33, 309, 1937, 8, 6, 15, 81, 444, 477, 830, 27, 2069, 1070, 1602, 826, 249, 1274, 464, 78, 539, 628, 328, 533, 1533, 375, 202, 118, 197, 60, 1091, 2] payload_rows_per_dst=[5448, 5450, 6624, 4614, 6344, 5822, 6436, 6740] dup_rows_per_dst=[1726, 1004, 1292, 838, 2768, 3078, 3402, 3950] combine_rows_per_src=[5448, 5450, 6624, 4614, 6344, 5822, 6436, 6740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47814 payload_rows_remote=40482 dup_rows=17722 dup_rows_remote=12228 dedup_ratio=0.270416 router_tokens_per_expert=[111, 40, 51, 17, 2, 34, 1396, 1458, 54, 38, 130, 320, 8, 69, 10, 299, 141, 297, 2710, 3, 56, 376, 98, 82, 1892, 161, 164, 1190, 48, 2272, 523, 1, 22, 28, 63, 2627, 13, 0, 3, 146, 879, 427, 749, 37, 2292, 1332, 2619, 518, 242, 1383, 211, 72, 301, 349, 803, 535, 1222, 728, 162, 71, 228, 98, 556, 1] payload_rows_per_dst=[4898, 5734, 6192, 5974, 7010, 7332, 5882, 4792] dup_rows_per_dst=[1320, 706, 1334, 1984, 3338, 5494, 2206, 1340] combine_rows_per_src=[4898, 5734, 6192, 5974, 7010, 7332, 5882, 4792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284798976 dispatch_meta_bytes=222904 combine_payload_bytes=284798976 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=285021880 ingress_bytes=504999936 link_bytes=504999936 token_bytes=569820856 weight_movement_bytes=220200960 total_nvlink_bytes=790021816 weight_to_token_ratio=0.386439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.782779469e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47630 payload_rows_remote=40842 dup_rows=17906 dup_rows_remote=14778 dedup_ratio=0.273224 router_tokens_per_expert=[77, 94, 52, 6, 1, 7, 1436, 1334, 118, 65, 147, 335, 2, 159, 0, 276, 169, 267, 2480, 11, 190, 238, 78, 134, 2427, 232, 166, 1432, 65, 2401, 517, 6, 14, 8, 239, 2140, 3, 0, 7, 146, 481, 675, 883, 20, 2157, 1175, 888, 730, 126, 975, 691, 45, 596, 514, 438, 1055, 1550, 861, 83, 53, 211, 278, 803, 1] payload_rows_per_dst=[4788, 5690, 5876, 6616, 6788, 6138, 6164, 5570] dup_rows_per_dst=[1226, 828, 1258, 3074, 3128, 3566, 2716, 2110] combine_rows_per_src=[4788, 5690, 5876, 6616, 6788, 6138, 6164, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287637504 dispatch_meta_bytes=217624 combine_payload_bytes=287637504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287855128 ingress_bytes=397737984 link_bytes=397737984 token_bytes=575492632 weight_movement_bytes=110100480 total_nvlink_bytes=685593112 weight_to_token_ratio=0.191315 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.917317708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293687296 dispatch_meta_bytes=231112 combine_payload_bytes=293687296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293918408 ingress_bytes=403787776 link_bytes=403787776 token_bytes=587605704 weight_movement_bytes=110100480 total_nvlink_bytes=697706184 weight_to_token_ratio=0.187371 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.022533541e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287049728 dispatch_meta_bytes=221896 combine_payload_bytes=287049728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287271624 ingress_bytes=397150208 link_bytes=397150208 token_bytes=574321352 weight_movement_bytes=110100480 total_nvlink_bytes=684421832 weight_to_token_ratio=0.191705 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.907095317e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=299063296 dispatch_meta_bytes=232008 combine_payload_bytes=299063296 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=299295304 ingress_bytes=739465216 link_bytes=739465216 token_bytes=598358600 weight_movement_bytes=440401920 total_nvlink_bytes=1038760520 weight_to_token_ratio=0.736017 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.286051632e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292009984 dispatch_meta_bytes=219384 combine_payload_bytes=292009984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292229368 ingress_bytes=402110464 link_bytes=402110464 token_bytes=584239352 weight_movement_bytes=110100480 total_nvlink_bytes=694339832 weight_to_token_ratio=0.188451 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.993362327e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290174976 dispatch_meta_bytes=210840 combine_payload_bytes=290174976 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=290385816 ingress_bytes=840677376 link_bytes=840677376 token_bytes=580560792 weight_movement_bytes=550502400 total_nvlink_bytes=1131063192 weight_to_token_ratio=0.948225 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.462076225e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292755456 dispatch_meta_bytes=222480 combine_payload_bytes=292755456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292977936 ingress_bytes=402855936 link_bytes=402855936 token_bytes=585733392 weight_movement_bytes=110100480 total_nvlink_bytes=695833872 weight_to_token_ratio=0.187970 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.006327311e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52776 payload_rows_remote=45074 dup_rows=12760 dup_rows_remote=11354 dedup_ratio=0.194702 router_tokens_per_expert=[0, 4080, 3783, 425, 3, 412, 1300, 2203, 0, 714, 52, 1, 0, 1, 0, 3, 0, 644, 2151, 6, 0, 105, 9, 11, 78, 3950, 105, 0, 0, 4, 1929, 0, 0, 80, 0, 1, 1, 0, 0, 0, 183, 1, 1360, 4, 3, 0, 0, 0, 5, 0, 2, 93, 20, 110, 0, 0, 2366, 0, 101, 3, 0, 1, 3414, 3051] payload_rows_per_dst=[6286, 7702, 7730, 8054, 8166, 3008, 6978, 4852] dup_rows_per_dst=[2400, 1406, 4224, 4078, 158, 94, 310, 90] combine_rows_per_src=[6286, 7702, 7730, 8054, 8166, 3008, 6978, 4852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45094 payload_rows_remote=37382 dup_rows=20442 dup_rows_remote=10696 dedup_ratio=0.311920 router_tokens_per_expert=[0, 4095, 3724, 234, 3, 505, 1811, 1825, 0, 578, 48, 0, 0, 0, 0, 1, 0, 1205, 1981, 35, 0, 107, 0, 4, 3, 3982, 8, 0, 0, 0, 2434, 0, 0, 185, 0, 0, 0, 0, 0, 0, 244, 0, 1427, 0, 0, 0, 0, 1, 0, 0, 0, 90, 1, 181, 0, 0, 1529, 1, 90, 0, 0, 0, 3414, 3022] payload_rows_per_dst=[7712, 2622, 7698, 8100, 5464, 3268, 7018, 3212] dup_rows_per_dst=[9746, 244, 5010, 4754, 230, 76, 354, 28] combine_rows_per_src=[7712, 2622, 7698, 8100, 5464, 3268, 7018, 3212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52794 payload_rows_remote=45556 dup_rows=12742 dup_rows_remote=12186 dedup_ratio=0.194427 router_tokens_per_expert=[0, 4094, 3787, 369, 0, 327, 1187, 1336, 0, 619, 59, 0, 0, 0, 0, 0, 0, 1677, 1626, 1, 0, 110, 0, 4, 35, 4074, 46, 0, 0, 0, 2807, 0, 0, 285, 0, 0, 0, 0, 0, 0, 165, 0, 2077, 0, 1, 0, 0, 0, 0, 0, 0, 98, 0, 384, 0, 0, 1502, 0, 127, 0, 0, 0, 3415, 2556] payload_rows_per_dst=[3224, 7656, 5290, 5644, 8188, 8164, 7238, 7390] dup_rows_per_dst=[542, 1274, 1546, 132, 570, 4470, 556, 3652] combine_rows_per_src=[3224, 7656, 5290, 5644, 8188, 8164, 7238, 7390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50082 payload_rows_remote=42686 dup_rows=15454 dup_rows_remote=9622 dedup_ratio=0.235809 router_tokens_per_expert=[0, 4080, 3801, 375, 0, 224, 920, 1981, 0, 546, 35, 0, 0, 0, 0, 0, 0, 1371, 2732, 32, 0, 285, 0, 0, 27, 4081, 178, 0, 0, 0, 1940, 0, 0, 245, 0, 0, 0, 0, 0, 0, 93, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 223, 0, 0, 1652, 0, 167, 0, 0, 0, 2726, 3137] payload_rows_per_dst=[5582, 7664, 7396, 8178, 8160, 3134, 6434, 3534] dup_rows_per_dst=[1418, 1100, 5832, 4274, 490, 46, 2190, 104] combine_rows_per_src=[5582, 7664, 7396, 8178, 8160, 3134, 6434, 3534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323090432 dispatch_meta_bytes=225712 combine_payload_bytes=323090432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323316144 ingress_bytes=433190912 link_bytes=433190912 token_bytes=646406576 weight_movement_bytes=110100480 total_nvlink_bytes=756507056 weight_to_token_ratio=0.170327 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.533902436e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51104 payload_rows_remote=45102 dup_rows=14432 dup_rows_remote=12538 dedup_ratio=0.220215 router_tokens_per_expert=[0, 4093, 3464, 289, 0, 611, 1063, 2141, 0, 468, 5, 1, 0, 0, 0, 50, 0, 1115, 1414, 6, 0, 37, 2, 3, 10, 4007, 34, 0, 0, 0, 2803, 0, 8, 156, 0, 4, 0, 0, 0, 0, 115, 0, 1329, 0, 0, 3, 2, 0, 0, 19, 1, 225, 10, 216, 1, 0, 2399, 0, 122, 0, 0, 14, 3465, 3063] payload_rows_per_dst=[6202, 7104, 4370, 6002, 8188, 6260, 8026, 4952] dup_rows_per_dst=[2006, 872, 784, 1894, 334, 2450, 5974, 118] combine_rows_per_src=[6202, 7104, 4370, 6002, 8188, 6260, 8026, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50926 payload_rows_remote=42910 dup_rows=14610 dup_rows_remote=7712 dedup_ratio=0.222931 router_tokens_per_expert=[0, 4094, 4003, 391, 0, 351, 1400, 1553, 0, 610, 26, 0, 3, 0, 0, 0, 0, 770, 1797, 2, 1, 97, 0, 3, 13, 4080, 34, 0, 10, 0, 2070, 1, 1, 142, 0, 0, 0, 0, 0, 0, 235, 0, 1945, 0, 1, 0, 0, 0, 1, 0, 0, 84, 2, 353, 0, 0, 2454, 0, 185, 0, 0, 0, 2791, 3265] payload_rows_per_dst=[3698, 8050, 4564, 4200, 8190, 8180, 6028, 8016] dup_rows_per_dst=[586, 1234, 776, 56, 284, 4342, 434, 6898] combine_rows_per_src=[3698, 8050, 4564, 4200, 8190, 8180, 6028, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52326 payload_rows_remote=44150 dup_rows=13210 dup_rows_remote=9710 dedup_ratio=0.201569 router_tokens_per_expert=[2, 4080, 3722, 503, 0, 231, 1294, 2129, 1, 1261, 58, 0, 0, 0, 7, 1, 0, 869, 1183, 18, 0, 78, 1, 3, 32, 4072, 5, 2, 0, 1, 2153, 0, 1, 226, 0, 0, 0, 0, 0, 0, 157, 0, 1603, 3, 1, 0, 2, 0, 0, 2, 0, 128, 12, 57, 3, 5, 1994, 0, 84, 1, 3, 9, 3213, 3558] payload_rows_per_dst=[5878, 7722, 3590, 4378, 8168, 8176, 6608, 7806] dup_rows_per_dst=[2096, 2378, 714, 56, 446, 3500, 232, 3788] combine_rows_per_src=[5878, 7722, 3590, 4378, 8168, 8176, 6608, 7806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55164 payload_rows_remote=46996 dup_rows=10372 dup_rows_remote=10040 dedup_ratio=0.158264 router_tokens_per_expert=[0, 4079, 3752, 368, 3, 554, 999, 2156, 0, 457, 42, 0, 0, 0, 2, 1, 0, 1445, 1485, 27, 0, 48, 0, 1, 5, 4060, 11, 0, 1, 0, 2851, 0, 0, 165, 0, 2, 0, 0, 0, 4, 46, 0, 1063, 3, 12, 0, 0, 0, 0, 2, 1, 92, 0, 88, 0, 0, 2263, 3, 118, 0, 0, 0, 3295, 3264] payload_rows_per_dst=[6180, 7602, 4994, 5720, 8168, 8160, 6756, 7584] dup_rows_per_dst=[1980, 906, 1018, 16, 332, 2208, 400, 3512] combine_rows_per_src=[6180, 7602, 4994, 5720, 8168, 8160, 6756, 7584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267954176 dispatch_meta_bytes=192312 combine_payload_bytes=267954176 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=268146488 ingress_bytes=708356096 link_bytes=708356096 token_bytes=536100664 weight_movement_bytes=440401920 total_nvlink_bytes=976502584 weight_to_token_ratio=0.821491 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.231947756e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=326545408 dispatch_meta_bytes=230968 combine_payload_bytes=326545408 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=326776376 ingress_bytes=546746368 link_bytes=546746368 token_bytes=653321784 weight_movement_bytes=220200960 total_nvlink_bytes=873522744 weight_to_token_ratio=0.337048 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.508818583e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=305973248 dispatch_meta_bytes=209232 combine_payload_bytes=305973248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306182480 ingress_bytes=416073728 link_bytes=416073728 token_bytes=612155728 weight_movement_bytes=110100480 total_nvlink_bytes=722256208 weight_to_token_ratio=0.179857 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.236206454e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323291136 dispatch_meta_bytes=230560 combine_payload_bytes=323291136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=323521696 ingress_bytes=543492096 link_bytes=543492096 token_bytes=646812832 weight_movement_bytes=220200960 total_nvlink_bytes=867013792 weight_to_token_ratio=0.340440 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.452221440e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=307578880 dispatch_meta_bytes=202488 combine_payload_bytes=307578880 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=307781368 ingress_bytes=747980800 link_bytes=747980800 token_bytes=615360248 weight_movement_bytes=440401920 total_nvlink_bytes=1055762168 weight_to_token_ratio=0.715681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.300861633e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316467200 dispatch_meta_bytes=215440 combine_payload_bytes=316467200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316682640 ingress_bytes=426567680 link_bytes=426567680 token_bytes=633149840 weight_movement_bytes=110100480 total_nvlink_bytes=743250320 weight_to_token_ratio=0.173893 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.418713538e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51700 payload_rows_remote=43516 dup_rows=13836 dup_rows_remote=10656 dedup_ratio=0.211121 router_tokens_per_expert=[3249, 2894, 1755, 3, 2618, 3, 0, 0, 0, 195, 0, 710, 153, 4, 531, 0, 5, 0, 0, 3114, 900, 2, 1, 3, 0, 4089, 0, 3012, 0, 21, 0, 0, 2243, 271, 1, 118, 1, 0, 0, 1491, 1, 0, 0, 0, 4087, 1, 708, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 4, 380, 0, 0, 6, 3, 186] payload_rows_per_dst=[7184, 8184, 6806, 6512, 6948, 3800, 6498, 5768] dup_rows_per_dst=[3386, 3180, 1244, 1480, 3464, 446, 10, 626] combine_rows_per_src=[7184, 8184, 6806, 6512, 6948, 3800, 6498, 5768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=336867328 dispatch_meta_bytes=228144 combine_payload_bytes=336867328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337095472 ingress_bytes=446967808 link_bytes=446967808 token_bytes=673962800 weight_movement_bytes=110100480 total_nvlink_bytes=784063280 weight_to_token_ratio=0.163363 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.773505317e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=311922688 dispatch_meta_bytes=216688 combine_payload_bytes=311922688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312139376 ingress_bytes=422023168 link_bytes=422023168 token_bytes=624062064 weight_movement_bytes=110100480 total_nvlink_bytes=734162544 weight_to_token_ratio=0.176426 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.339677000e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50310 payload_rows_remote=43250 dup_rows=15226 dup_rows_remote=12936 dedup_ratio=0.232330 router_tokens_per_expert=[2656, 2392, 2440, 0, 3434, 0, 0, 0, 0, 135, 0, 554, 4, 0, 619, 0, 0, 0, 0, 2344, 1539, 0, 0, 0, 0, 4096, 0, 2950, 0, 0, 0, 0, 1558, 216, 0, 95, 0, 7, 9, 2278, 5, 0, 0, 0, 4096, 0, 495, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 759, 0, 0, 2, 0, 52] payload_rows_per_dst=[4880, 2422, 5898, 8192, 6222, 8192, 7060, 7444] dup_rows_per_dst=[0, 202, 1868, 5900, 2104, 1000, 2290, 1862] combine_rows_per_src=[4880, 2422, 5898, 8192, 6222, 8192, 7060, 7444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53904 payload_rows_remote=48018 dup_rows=11632 dup_rows_remote=10364 dedup_ratio=0.177490 router_tokens_per_expert=[3263, 3052, 2174, 0, 3205, 0, 0, 0, 0, 150, 0, 376, 51, 2, 592, 0, 0, 0, 0, 2543, 1034, 0, 0, 0, 0, 4096, 0, 2148, 0, 25, 0, 0, 1526, 195, 0, 48, 0, 2, 15, 2259, 0, 0, 0, 0, 4096, 0, 836, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 865, 0, 0, 0, 0, 211] payload_rows_per_dst=[7468, 8192, 5886, 4318, 6154, 8192, 6528, 7166] dup_rows_per_dst=[2984, 2342, 1268, 28, 1936, 1672, 6, 1396] combine_rows_per_src=[7468, 8192, 5886, 4318, 6154, 8192, 6528, 7166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49684 payload_rows_remote=41492 dup_rows=15852 dup_rows_remote=7706 dedup_ratio=0.241882 router_tokens_per_expert=[3397, 2497, 1577, 0, 2989, 0, 0, 0, 0, 123, 0, 325, 15, 0, 688, 0, 0, 0, 0, 2575, 1304, 0, 0, 0, 0, 4095, 0, 2780, 0, 11, 0, 0, 2485, 164, 0, 6, 0, 0, 0, 2563, 2, 0, 1, 0, 4095, 0, 426, 0, 0, 0, 0, 0, 24, 1, 0, 1, 0, 0, 293, 0, 1, 0, 0, 330] payload_rows_per_dst=[8192, 8190, 6268, 5576, 7282, 856, 6802, 6518] dup_rows_per_dst=[8146, 2302, 1490, 6, 3154, 2, 44, 708] combine_rows_per_src=[8192, 8190, 6268, 5576, 7282, 856, 6802, 6518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50008 payload_rows_remote=46998 dup_rows=15528 dup_rows_remote=15528 dedup_ratio=0.236938 router_tokens_per_expert=[3205, 3538, 1181, 13, 2361, 31, 0, 0, 0, 192, 0, 2130, 7, 2, 789, 0, 1, 0, 0, 2656, 1615, 0, 0, 0, 0, 4091, 0, 1504, 0, 1, 0, 0, 1654, 15, 0, 76, 0, 0, 2, 1325, 0, 8, 0, 0, 4078, 2, 1595, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 438, 1, 0, 5, 0, 234] payload_rows_per_dst=[7482, 8182, 6510, 3010, 4954, 8182, 6416, 5272] dup_rows_per_dst=[2044, 6240, 2034, 0, 1190, 3184, 30, 806] combine_rows_per_src=[7482, 8182, 6510, 3010, 4954, 8182, 6416, 5272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51490 payload_rows_remote=43948 dup_rows=14046 dup_rows_remote=9372 dedup_ratio=0.214325 router_tokens_per_expert=[3244, 2212, 2679, 0, 2870, 0, 0, 0, 0, 102, 1, 794, 79, 7, 707, 0, 0, 0, 0, 2875, 1649, 0, 1, 0, 0, 4095, 1, 1602, 0, 18, 0, 1, 1889, 432, 0, 89, 0, 1, 2, 1710, 0, 0, 0, 0, 4095, 0, 578, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 846, 0, 0, 0, 0, 180] payload_rows_per_dst=[5358, 3052, 6520, 8192, 6146, 8190, 6490, 7542] dup_rows_per_dst=[0, 328, 2138, 3242, 2100, 1548, 16, 4674] combine_rows_per_src=[5358, 3052, 6520, 8192, 6146, 8190, 6490, 7542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48420 payload_rows_remote=40228 dup_rows=17116 dup_rows_remote=14370 dedup_ratio=0.261169 router_tokens_per_expert=[3409, 3265, 2129, 0, 2326, 1, 0, 0, 4, 164, 0, 1065, 19, 3, 355, 0, 0, 2, 0, 3112, 1154, 1, 0, 3, 10, 4080, 3, 2674, 0, 4, 11, 0, 1260, 33, 0, 75, 0, 2, 13, 1285, 4, 3, 7, 0, 4079, 11, 1365, 0, 0, 0, 0, 1, 8, 0, 3, 0, 3, 0, 487, 0, 0, 7, 0, 328] payload_rows_per_dst=[4258, 3036, 6788, 8186, 4480, 8192, 7866, 5614] dup_rows_per_dst=[2, 184, 1756, 5378, 856, 2746, 5506, 688] combine_rows_per_src=[4258, 3036, 6788, 8186, 4480, 8192, 7866, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51068 payload_rows_remote=44382 dup_rows=14468 dup_rows_remote=12608 dedup_ratio=0.220764 router_tokens_per_expert=[2815, 2869, 1360, 8, 3162, 0, 0, 0, 0, 232, 0, 629, 28, 1, 796, 0, 0, 0, 0, 2895, 1636, 0, 0, 0, 0, 4095, 0, 1725, 1, 1, 0, 0, 2592, 245, 1, 11, 0, 0, 2, 1422, 2, 0, 1, 0, 4095, 0, 568, 1, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 1343, 2, 0, 0, 1, 218] payload_rows_per_dst=[3830, 6878, 6756, 4488, 6686, 8190, 6942, 7298] dup_rows_per_dst=[322, 2622, 2306, 1028, 1860, 1144, 3032, 2154] combine_rows_per_src=[3830, 6878, 6756, 4488, 6686, 8190, 6942, 7298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=297414656 dispatch_meta_bytes=196792 combine_payload_bytes=297414656 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=297611448 ingress_bytes=847917056 link_bytes=847917056 token_bytes=595026104 weight_movement_bytes=550502400 total_nvlink_bytes=1145528504 weight_to_token_ratio=0.925174 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.474667219e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=310016000 dispatch_meta_bytes=224744 combine_payload_bytes=310016000 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310240744 ingress_bytes=530216960 link_bytes=530216960 token_bytes=620256744 weight_movement_bytes=220200960 total_nvlink_bytes=840457704 weight_to_token_ratio=0.355016 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.221344991e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344193024 dispatch_meta_bytes=233528 combine_payload_bytes=344193024 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=344426552 ingress_bytes=454293504 link_bytes=454293504 token_bytes=688619576 weight_movement_bytes=110100480 total_nvlink_bytes=798720056 weight_to_token_ratio=0.159886 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.900911219e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=336881664 dispatch_meta_bytes=250104 combine_payload_bytes=336881664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=337131768 ingress_bytes=557082624 link_bytes=557082624 token_bytes=674013432 weight_movement_bytes=220200960 total_nvlink_bytes=894214392 weight_to_token_ratio=0.326701 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.688583075e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=315019264 dispatch_meta_bytes=213280 combine_payload_bytes=315019264 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=315232544 ingress_bytes=535220224 link_bytes=535220224 token_bytes=630251808 weight_movement_bytes=220200960 total_nvlink_bytes=850452768 weight_to_token_ratio=0.349386 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.308359981e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54316 payload_rows_remote=46136 dup_rows=11220 dup_rows_remote=7494 dedup_ratio=0.171204 router_tokens_per_expert=[0, 0, 0, 391, 3631, 0, 0, 0, 2, 0, 0, 0, 0, 1865, 0, 0, 0, 2108, 14, 5, 0, 5, 18, 354, 4094, 0, 3, 1, 0, 0, 4, 1927, 0, 120, 9, 0, 1, 4086, 2012, 0, 2, 0, 1, 0, 4082, 153, 0, 0, 2677, 0, 1, 1084, 0, 0, 5, 22, 8, 4080, 1, 0, 0, 2, 0, 0] payload_rows_per_dst=[7366, 8180, 8192, 3864, 4082, 8168, 6294, 8170] dup_rows_per_dst=[678, 3726, 5004, 6, 202, 308, 1284, 12] combine_rows_per_src=[7366, 8180, 8192, 3864, 4082, 8168, 6294, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=288354304 dispatch_meta_bytes=218392 combine_payload_bytes=288354304 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=288572696 ingress_bytes=728756224 link_bytes=728756224 token_bytes=576927000 weight_movement_bytes=440401920 total_nvlink_bytes=1017328920 weight_to_token_ratio=0.763358 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.267426933e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=318130176 dispatch_meta_bytes=227960 combine_payload_bytes=318130176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318358136 ingress_bytes=428230656 link_bytes=428230656 token_bytes=636488312 weight_movement_bytes=110100480 total_nvlink_bytes=746588792 weight_to_token_ratio=0.172981 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.447635426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330702848 dispatch_meta_bytes=214520 combine_payload_bytes=330702848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330917368 ingress_bytes=440803328 link_bytes=440803328 token_bytes=661620216 weight_movement_bytes=110100480 total_nvlink_bytes=771720696 weight_to_token_ratio=0.166410 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.666294871e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57374 payload_rows_remote=49764 dup_rows=8162 dup_rows_remote=4390 dedup_ratio=0.124542 router_tokens_per_expert=[0, 0, 0, 587, 3591, 0, 0, 0, 0, 0, 0, 14, 0, 2511, 0, 0, 0, 1953, 0, 0, 0, 0, 7, 469, 4096, 0, 0, 0, 0, 0, 4, 3185, 0, 189, 0, 0, 0, 4096, 1215, 0, 16, 0, 0, 0, 4096, 137, 0, 0, 1527, 0, 0, 949, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7328, 5048, 4620, 8192, 8192, 8192, 7610, 8192] dup_rows_per_dst=[1028, 2, 238, 8, 2808, 306, 3772, 0] combine_rows_per_src=[7328, 5048, 4620, 8192, 8192, 8192, 7610, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52290 payload_rows_remote=44710 dup_rows=13246 dup_rows_remote=11058 dedup_ratio=0.202118 router_tokens_per_expert=[0, 0, 1, 143, 3581, 0, 1, 0, 0, 1, 0, 0, 0, 2360, 0, 0, 0, 1784, 0, 0, 0, 0, 5, 380, 4094, 0, 0, 0, 0, 0, 0, 2491, 0, 178, 0, 0, 0, 4096, 1163, 0, 0, 0, 0, 0, 4092, 131, 0, 0, 2718, 0, 0, 1373, 0, 0, 1, 80, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7580, 8192, 8192, 4986, 2606, 5950, 6598, 8186] dup_rows_per_dst=[2188, 4722, 4334, 0, 76, 180, 1746, 0] combine_rows_per_src=[7580, 8192, 8192, 4986, 2606, 5950, 6598, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54174 payload_rows_remote=45982 dup_rows=11362 dup_rows_remote=6412 dedup_ratio=0.173370 router_tokens_per_expert=[0, 0, 0, 355, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 2695, 0, 0, 0, 1698, 0, 0, 0, 0, 2, 775, 4096, 0, 0, 2, 0, 0, 15, 3063, 0, 260, 0, 0, 0, 4096, 934, 0, 0, 0, 0, 0, 4095, 125, 0, 0, 1610, 0, 0, 1487, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6748, 7790, 8192, 6136, 3310, 8190, 5616, 8192] dup_rows_per_dst=[582, 4330, 4950, 24, 540, 250, 686, 0] combine_rows_per_src=[6748, 7790, 8192, 6136, 3310, 8190, 5616, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58530 payload_rows_remote=51010 dup_rows=7006 dup_rows_remote=5114 dedup_ratio=0.106903 router_tokens_per_expert=[0, 2, 0, 737, 3588, 0, 0, 0, 32, 0, 0, 0, 0, 3205, 0, 0, 0, 816, 0, 0, 0, 0, 13, 371, 4096, 0, 6, 0, 0, 0, 29, 3586, 0, 187, 0, 0, 0, 4096, 643, 0, 8, 0, 3, 0, 4058, 175, 0, 0, 2021, 0, 0, 713, 0, 0, 0, 285, 10, 4086, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7626, 6468, 6980, 7520, 8192, 8134, 5434, 8176] dup_rows_per_dst=[1028, 6, 1442, 1892, 1660, 354, 604, 20] combine_rows_per_src=[7626, 6468, 6980, 7520, 8192, 8134, 5434, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55736 payload_rows_remote=47544 dup_rows=9800 dup_rows_remote=9796 dedup_ratio=0.149536 router_tokens_per_expert=[0, 1, 0, 248, 3312, 1, 0, 0, 3, 0, 0, 0, 1, 2681, 0, 0, 0, 1998, 0, 0, 0, 0, 1, 618, 4094, 1, 3, 0, 0, 0, 0, 2577, 1, 69, 0, 1, 0, 4094, 1181, 1, 0, 0, 0, 0, 4095, 118, 0, 0, 2279, 0, 0, 1244, 0, 0, 0, 48, 2, 4095, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[6688, 5370, 4906, 6498, 8192, 8190, 7700, 8192] dup_rows_per_dst=[436, 0, 328, 4, 4008, 424, 4596, 4] combine_rows_per_src=[6688, 5370, 4906, 6498, 8192, 8190, 7700, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55782 payload_rows_remote=47600 dup_rows=9754 dup_rows_remote=9508 dedup_ratio=0.148834 router_tokens_per_expert=[0, 0, 1, 355, 3803, 0, 0, 1, 3, 0, 1, 4, 0, 2875, 0, 0, 4, 1090, 0, 0, 0, 2, 5, 646, 4080, 0, 0, 0, 0, 1, 22, 3339, 2, 143, 1, 0, 5, 4096, 1205, 0, 0, 0, 0, 0, 4091, 123, 0, 0, 1763, 0, 0, 806, 0, 0, 0, 192, 10, 4094, 0, 0, 1, 0, 4, 0] payload_rows_per_dst=[7650, 5766, 3406, 8186, 8192, 8182, 6208, 8192] dup_rows_per_dst=[670, 0, 88, 3864, 2712, 246, 2148, 26] combine_rows_per_src=[7650, 5766, 3406, 8186, 8192, 8182, 6208, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54934 payload_rows_remote=46744 dup_rows=10602 dup_rows_remote=8028 dedup_ratio=0.161774 router_tokens_per_expert=[0, 0, 0, 432, 3538, 0, 0, 0, 20, 0, 2, 0, 0, 2946, 0, 0, 0, 1941, 0, 0, 0, 3, 21, 367, 4093, 0, 1, 4, 0, 0, 0, 3080, 0, 123, 0, 0, 0, 4090, 1169, 0, 1, 0, 0, 1, 4094, 135, 0, 0, 1388, 0, 0, 890, 0, 0, 2, 340, 0, 4086, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[7144, 5918, 4490, 8188, 8190, 8190, 4642, 8172] dup_rows_per_dst=[796, 18, 174, 6168, 2574, 272, 598, 2] combine_rows_per_src=[7144, 5918, 4490, 8188, 8190, 8190, 4642, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=356708352 dispatch_meta_bytes=216616 combine_payload_bytes=356708352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356924968 ingress_bytes=466808832 link_bytes=466808832 token_bytes=713633320 weight_movement_bytes=110100480 total_nvlink_bytes=823733800 weight_to_token_ratio=0.154282 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.118573357e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=320481280 dispatch_meta_bytes=223072 combine_payload_bytes=320481280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320704352 ingress_bytes=430581760 link_bytes=430581760 token_bytes=641185632 weight_movement_bytes=110100480 total_nvlink_bytes=751286112 weight_to_token_ratio=0.171714 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.488524992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=329598976 dispatch_meta_bytes=209576 combine_payload_bytes=329598976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329808552 ingress_bytes=439699456 link_bytes=439699456 token_bytes=659407528 weight_movement_bytes=110100480 total_nvlink_bytes=769508008 weight_to_token_ratio=0.166969 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.647096721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=365639680 dispatch_meta_bytes=224496 combine_payload_bytes=365639680 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=365864176 ingress_bytes=916142080 link_bytes=916142080 token_bytes=731503856 weight_movement_bytes=550502400 total_nvlink_bytes=1282006256 weight_to_token_ratio=0.752563 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.593321757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64422 payload_rows_remote=56232 dup_rows=1114 dup_rows_remote=1102 dedup_ratio=0.016998 router_tokens_per_expert=[0, 0, 4092, 90, 0, 0, 4087, 0, 5, 1, 1, 0, 1, 1, 0, 0, 0, 0, 558, 1, 0, 4092, 3971, 3926, 0, 0, 0, 0, 4094, 0, 3960, 0, 0, 3, 3831, 0, 0, 0, 9, 0, 5, 3, 0, 14, 0, 0, 0, 1, 0, 3, 0, 14, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 1] payload_rows_per_dst=[8184, 8190, 8118, 7920, 7668, 8192, 7970, 8180] dup_rows_per_dst=[180, 12, 852, 0, 18, 42, 6, 4] combine_rows_per_src=[8184, 8190, 8118, 7920, 7668, 8192, 7970, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=340795392 dispatch_meta_bytes=229360 combine_payload_bytes=340795392 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=341024752 ingress_bytes=450895872 link_bytes=450895872 token_bytes=681820144 weight_movement_bytes=110100480 total_nvlink_bytes=791920624 weight_to_token_ratio=0.161480 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.841820810e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=341196800 dispatch_meta_bytes=228432 combine_payload_bytes=341196800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=341425232 ingress_bytes=561397760 link_bytes=561397760 token_bytes=682622032 weight_movement_bytes=220200960 total_nvlink_bytes=902822992 weight_to_token_ratio=0.322581 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.763630387e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=335060992 dispatch_meta_bytes=219088 combine_payload_bytes=335060992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=335280080 ingress_bytes=555261952 link_bytes=555261952 token_bytes=670341072 weight_movement_bytes=220200960 total_nvlink_bytes=890542032 weight_to_token_ratio=0.328491 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.656918594e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=403070976 dispatch_meta_bytes=229336 combine_payload_bytes=403070976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403300312 ingress_bytes=513171456 link_bytes=513171456 token_bytes=806371288 weight_movement_bytes=110100480 total_nvlink_bytes=916471768 weight_to_token_ratio=0.136538 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.924895642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64856 payload_rows_remote=56700 dup_rows=680 dup_rows_remote=680 dedup_ratio=0.010376 router_tokens_per_expert=[0, 0, 4096, 131, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 4096, 4078, 3678, 0, 0, 0, 0, 4096, 0, 3998, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8036, 7996, 7900, 8192, 8156, 8192] dup_rows_per_dst=[262, 0, 410, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8036, 7996, 7900, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64888 payload_rows_remote=56830 dup_rows=648 dup_rows_remote=220 dedup_ratio=0.009888 router_tokens_per_expert=[0, 0, 4095, 106, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 0, 4096, 4043, 3782, 0, 0, 0, 0, 4096, 0, 4003, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8058, 8006, 7970, 8192, 8086, 8192] dup_rows_per_dst=[210, 0, 428, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8058, 8006, 7970, 8192, 8086, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64708 payload_rows_remote=56538 dup_rows=828 dup_rows_remote=584 dedup_ratio=0.012634 router_tokens_per_expert=[0, 0, 4081, 120, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 1, 0, 4095, 4027, 3693, 0, 0, 0, 0, 4095, 0, 3888, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8190, 8086, 7858, 8176, 7992, 8056, 8180] dup_rows_per_dst=[244, 0, 448, 132, 0, 2, 2, 0] combine_rows_per_src=[8170, 8190, 8086, 7858, 8176, 7992, 8056, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64600 payload_rows_remote=56614 dup_rows=936 dup_rows_remote=936 dedup_ratio=0.014282 router_tokens_per_expert=[0, 0, 4088, 251, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 4096, 3986, 3137, 0, 0, 0, 0, 4096, 0, 3993, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 16, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[8182, 8192, 7850, 7986, 8034, 8192, 7972, 8192] dup_rows_per_dst=[496, 0, 380, 0, 0, 52, 0, 8] combine_rows_per_src=[8182, 8192, 7850, 7986, 8034, 8192, 7972, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59540 payload_rows_remote=51350 dup_rows=5996 dup_rows_remote=5996 dedup_ratio=0.091492 router_tokens_per_expert=[0, 0, 4095, 153, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 1, 0, 4090, 3844, 3658, 1, 0, 0, 0, 4094, 0, 3824, 0, 0, 0, 4086, 1, 1, 1, 0, 0, 18, 0, 0, 5, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6116, 8160, 5258, 7724, 8174, 8192, 7726, 8190] dup_rows_per_dst=[192, 0, 686, 1858, 1720, 46, 1494, 0] combine_rows_per_src=[6116, 8160, 5258, 7724, 8174, 8192, 7726, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64800 payload_rows_remote=56830 dup_rows=736 dup_rows_remote=734 dedup_ratio=0.011230 router_tokens_per_expert=[0, 4, 4094, 77, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 710, 0, 0, 4093, 4007, 3600, 2, 0, 0, 0, 4093, 0, 3984, 0, 0, 0, 3984, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 3, 10] payload_rows_per_dst=[8188, 8190, 8086, 7972, 7970, 8186, 8016, 8192] dup_rows_per_dst=[162, 4, 534, 0, 2, 4, 2, 28] combine_rows_per_src=[8188, 8190, 8086, 7972, 7970, 8186, 8016, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64626 payload_rows_remote=56440 dup_rows=910 dup_rows_remote=862 dedup_ratio=0.013885 router_tokens_per_expert=[0, 1, 4085, 197, 0, 0, 4084, 0, 0, 0, 0, 7, 1, 0, 1, 4, 0, 0, 548, 0, 0, 4094, 4005, 3605, 0, 2, 1, 0, 4087, 0, 4039, 0, 0, 0, 3946, 2, 0, 0, 4, 0, 22, 0, 0, 7, 1, 0, 0, 0, 5, 0, 1, 14, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 7906, 8080, 7898, 8186, 8012, 8176] dup_rows_per_dst=[390, 22, 400, 4, 6, 48, 38, 2] combine_rows_per_src=[8176, 8192, 7906, 8080, 7898, 8186, 8012, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=406425600 dispatch_meta_bytes=229520 combine_payload_bytes=406425600 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=406655120 ingress_bytes=626626560 link_bytes=626626560 token_bytes=813080720 weight_movement_bytes=220200960 total_nvlink_bytes=1033281680 weight_to_token_ratio=0.270823 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089806650e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57548 payload_rows_remote=49356 dup_rows=7988 dup_rows_remote=7462 dedup_ratio=0.121887 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 4, 0, 3, 0, 208, 0, 55, 0, 0, 0, 1, 3229, 1932, 0, 0, 0, 0, 1104, 0, 0, 0, 0, 4094, 4091, 0, 0, 0, 4092, 0, 0, 342, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 1, 0, 0, 4093, 5, 2, 3411, 0, 2, 9] payload_rows_per_dst=[8192, 8192, 7836, 8190, 6128, 8190, 3978, 6842] dup_rows_per_dst=[12, 526, 2488, 4374, 572, 0, 0, 16] combine_rows_per_src=[8192, 8192, 7836, 8190, 6128, 8190, 3978, 6842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407357440 dispatch_meta_bytes=228200 combine_payload_bytes=407357440 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=407585640 ingress_bytes=957859840 link_bytes=957859840 token_bytes=814943080 weight_movement_bytes=550502400 total_nvlink_bytes=1365445480 weight_to_token_ratio=0.675510 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.665875803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=405809152 dispatch_meta_bytes=230200 combine_payload_bytes=405809152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406039352 ingress_bytes=626010112 link_bytes=626010112 token_bytes=811848504 weight_movement_bytes=220200960 total_nvlink_bytes=1032049464 weight_to_token_ratio=0.271234 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088734546e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=405264384 dispatch_meta_bytes=228488 combine_payload_bytes=405264384 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=405492872 ingress_bytes=735565824 link_bytes=735565824 token_bytes=810757256 weight_movement_bytes=330301440 total_nvlink_bytes=1141058696 weight_to_token_ratio=0.407399 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279269948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=368076800 dispatch_meta_bytes=229384 combine_payload_bytes=368076800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368306184 ingress_bytes=478177280 link_bytes=478177280 token_bytes=736382984 weight_movement_bytes=110100480 total_nvlink_bytes=846483464 weight_to_token_ratio=0.149515 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.316289366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407357440 dispatch_meta_bytes=230256 combine_payload_bytes=407357440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407587696 ingress_bytes=517457920 link_bytes=517457920 token_bytes=814945136 weight_movement_bytes=110100480 total_nvlink_bytes=925045616 weight_to_token_ratio=0.135102 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.999444301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404561920 dispatch_meta_bytes=229208 combine_payload_bytes=404561920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404791128 ingress_bytes=514662400 link_bytes=514662400 token_bytes=809353048 weight_movement_bytes=110100480 total_nvlink_bytes=919453528 weight_to_token_ratio=0.136035 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.950825610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=353783808 dispatch_meta_bytes=227272 combine_payload_bytes=353783808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354011080 ingress_bytes=463884288 link_bytes=463884288 token_bytes=707794888 weight_movement_bytes=110100480 total_nvlink_bytes=817895368 weight_to_token_ratio=0.155554 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.067710727e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58872 payload_rows_remote=50680 dup_rows=6664 dup_rows_remote=4394 dedup_ratio=0.101685 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 32, 0, 0, 0, 0, 3146, 925, 0, 0, 0, 0, 2146, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 425, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3887, 0, 0, 0] payload_rows_per_dst=[8192, 6752, 7160, 4292, 8192, 8192, 8192, 7900] dup_rows_per_dst=[0, 1010, 982, 0, 850, 0, 2270, 1552] combine_rows_per_src=[8192, 6752, 7160, 4292, 8192, 8192, 8192, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60192 payload_rows_remote=53204 dup_rows=5344 dup_rows_remote=4420 dedup_ratio=0.081543 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 162, 0, 0, 0, 0, 3108, 848, 0, 0, 0, 0, 1261, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 562, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3776, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 6988, 8192, 8192, 8192, 4694, 7552] dup_rows_per_dst=[0, 774, 924, 2522, 1124, 0, 0, 0] combine_rows_per_src=[8190, 8192, 6988, 8192, 8192, 8192, 4694, 7552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61362 payload_rows_remote=53178 dup_rows=4174 dup_rows_remote=3002 dedup_ratio=0.063690 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 617, 0, 43, 0, 0, 0, 0, 3362, 1105, 0, 0, 0, 0, 587, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 4096, 0, 0, 146, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 2843, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3581, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 7562, 8184, 8192, 8192, 5688, 7162] dup_rows_per_dst=[18, 1320, 1372, 1172, 292, 0, 0, 0] combine_rows_per_src=[8190, 8192, 7562, 8184, 8192, 8192, 5688, 7162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51788 payload_rows_remote=43616 dup_rows=13748 dup_rows_remote=13746 dedup_ratio=0.209778 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 608, 0, 90, 0, 0, 0, 0, 3587, 924, 0, 0, 0, 0, 1089, 0, 1, 0, 0, 4085, 4095, 0, 0, 0, 4095, 0, 0, 489, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 1878, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3633, 0, 0, 0] payload_rows_per_dst=[8172, 8190, 7616, 8192, 5312, 8190, 3756, 2360] dup_rows_per_dst=[2, 1396, 1408, 10368, 574, 0, 0, 0] combine_rows_per_src=[8172, 8190, 7616, 8192, 5312, 8190, 3756, 2360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51244 payload_rows_remote=43052 dup_rows=14292 dup_rows_remote=5888 dedup_ratio=0.218079 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 802, 0, 91, 0, 0, 0, 0, 3107, 136, 0, 0, 0, 0, 1420, 0, 0, 0, 0, 4090, 4081, 0, 0, 0, 4095, 0, 0, 376, 0, 1, 0, 4094, 0, 0, 1, 2, 0, 1, 0, 1, 2520, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3851, 0, 0, 3] payload_rows_per_dst=[8174, 1782, 5672, 2848, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 114, 2, 752, 4, 5012, 8404] combine_rows_per_src=[8174, 1782, 5672, 2848, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58274 payload_rows_remote=50082 dup_rows=7262 dup_rows_remote=6778 dedup_ratio=0.110809 router_tokens_per_expert=[0, 2, 5, 1, 0, 1, 0, 0, 0, 0, 0, 923, 0, 121, 1, 0, 0, 0, 2545, 928, 0, 0, 0, 2, 1544, 0, 0, 0, 0, 4091, 4093, 0, 0, 1, 4095, 0, 0, 242, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 2114, 4, 0, 0, 0, 0, 0, 4088, 1, 0, 3865, 1, 0, 2] payload_rows_per_dst=[8192, 8188, 6358, 4740, 8192, 8192, 6674, 7738] dup_rows_per_dst=[8, 2078, 592, 794, 484, 0, 3306, 0] combine_rows_per_src=[8192, 8188, 6358, 4740, 8192, 8192, 6674, 7738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363274240 dispatch_meta_bytes=220296 combine_payload_bytes=363274240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363494536 ingress_bytes=473374720 link_bytes=473374720 token_bytes=726768776 weight_movement_bytes=110100480 total_nvlink_bytes=836869256 weight_to_token_ratio=0.151493 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.232764948e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381366272 dispatch_meta_bytes=230496 combine_payload_bytes=381366272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381596768 ingress_bytes=601567232 link_bytes=601567232 token_bytes=762963040 weight_movement_bytes=220200960 total_nvlink_bytes=983164000 weight_to_token_ratio=0.288613 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046224357e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57920 payload_rows_remote=49728 dup_rows=7616 dup_rows_remote=7616 dedup_ratio=0.116211 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 215, 0, 163, 0, 1, 0, 0, 3268, 961, 0, 3, 0, 0, 833, 0, 0, 0, 0, 4089, 4093, 0, 1, 0, 4092, 0, 1, 149, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 2783, 3, 2, 2, 0, 0, 0, 4091, 11, 0, 3897, 0, 1, 0] payload_rows_per_dst=[8188, 8190, 7486, 1666, 8190, 8192, 8190, 7818] dup_rows_per_dst=[6, 750, 978, 0, 298, 0, 5584, 0] combine_rows_per_src=[8188, 8190, 7486, 1666, 8190, 8192, 8190, 7818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62782 payload_rows_remote=54592 dup_rows=2754 dup_rows_remote=2754 dedup_ratio=0.042023 router_tokens_per_expert=[4096, 0, 220, 0, 3705, 1, 0, 3, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 3, 1, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 3, 0, 0, 0, 0, 576, 0, 4, 0, 4094, 4085, 0, 55, 0, 0, 0, 7, 0, 0, 3840, 0] payload_rows_per_dst=[7698, 8190, 8164, 7810, 8170, 8192, 6864, 7694] dup_rows_per_dst=[160, 0, 0, 1860, 34, 140, 560, 0] combine_rows_per_src=[7698, 8190, 8164, 7810, 8170, 8192, 6864, 7694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381179904 dispatch_meta_bytes=224720 combine_payload_bytes=381179904 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=381404624 ingress_bytes=711481344 link_bytes=711481344 token_bytes=762584528 weight_movement_bytes=330301440 total_nvlink_bytes=1092885968 weight_to_token_ratio=0.433134 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.237383076e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312639488 dispatch_meta_bytes=229448 combine_payload_bytes=312639488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312868936 ingress_bytes=422739968 link_bytes=422739968 token_bytes=625508424 weight_movement_bytes=110100480 total_nvlink_bytes=735608904 weight_to_token_ratio=0.176018 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.352143331e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=308596736 dispatch_meta_bytes=195760 combine_payload_bytes=308596736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=308792496 ingress_bytes=638898176 link_bytes=638898176 token_bytes=617389232 weight_movement_bytes=330301440 total_nvlink_bytes=947690672 weight_to_token_ratio=0.534997 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.111149009e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=358987776 dispatch_meta_bytes=227440 combine_payload_bytes=358987776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359215216 ingress_bytes=579188736 link_bytes=579188736 token_bytes=718202992 weight_movement_bytes=220200960 total_nvlink_bytes=938403952 weight_to_token_ratio=0.306600 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007304472e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=356450304 dispatch_meta_bytes=229376 combine_payload_bytes=356450304 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=356679680 ingress_bytes=466550784 link_bytes=466550784 token_bytes=713129984 weight_movement_bytes=110100480 total_nvlink_bytes=823230464 weight_to_token_ratio=0.154390 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.114085478e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=391315456 dispatch_meta_bytes=229384 combine_payload_bytes=391315456 prefetch_bytes=0 reduce_bytes=0 egress_bytes=391544840 ingress_bytes=391315456 link_bytes=391544840 token_bytes=782860296 weight_movement_bytes=0 total_nvlink_bytes=782860296 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.809608748e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63306 payload_rows_remote=55114 dup_rows=2230 dup_rows_remote=344 dedup_ratio=0.034027 router_tokens_per_expert=[4096, 0, 134, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 7, 0, 0, 0, 0, 926, 0, 2, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 3867, 0] payload_rows_per_dst=[7428, 8192, 8192, 7184, 8192, 8192, 8192, 7734] dup_rows_per_dst=[142, 0, 0, 0, 0, 202, 1886, 0] combine_rows_per_src=[7428, 8192, 8192, 7184, 8192, 8192, 8192, 7734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63932 payload_rows_remote=55742 dup_rows=1604 dup_rows_remote=1604 dedup_ratio=0.024475 router_tokens_per_expert=[4096, 0, 122, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 660, 0, 11, 0, 4096, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 4011, 0] payload_rows_per_dst=[7682, 8192, 8190, 7270, 8192, 8192, 8192, 8022] dup_rows_per_dst=[46, 0, 0, 0, 0, 128, 1430, 0] combine_rows_per_src=[7682, 8192, 8190, 7270, 8192, 8192, 8192, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62964 payload_rows_remote=55820 dup_rows=2572 dup_rows_remote=2572 dedup_ratio=0.039246 router_tokens_per_expert=[4096, 0, 343, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 0, 0, 0, 0, 889, 0, 1, 0, 4080, 4096, 0, 171, 0, 0, 0, 0, 0, 0, 3557, 0] payload_rows_per_dst=[7746, 8192, 8192, 7144, 8192, 8192, 8192, 7114] dup_rows_per_dst=[396, 0, 0, 0, 0, 86, 2090, 0] combine_rows_per_src=[7746, 8192, 8192, 7144, 8192, 8192, 8192, 7114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64010 payload_rows_remote=55818 dup_rows=1526 dup_rows_remote=1520 dedup_ratio=0.023285 router_tokens_per_expert=[4091, 0, 201, 0, 3872, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 3, 0, 0, 0, 0, 3690, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 117, 0, 0, 0, 0, 0, 0, 0, 486, 1, 5, 0, 4095, 4096, 0, 12, 0, 0, 0, 10, 0, 1, 3891, 0] payload_rows_per_dst=[7892, 8188, 8192, 7386, 8192, 8184, 8192, 7784] dup_rows_per_dst=[256, 0, 6, 0, 6, 232, 1006, 20] combine_rows_per_src=[7892, 8188, 8192, 7386, 8192, 8184, 8192, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60730 payload_rows_remote=52582 dup_rows=4806 dup_rows_remote=1742 dedup_ratio=0.073334 router_tokens_per_expert=[4095, 0, 428, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 43, 0, 0, 0, 1, 0, 0, 1, 635, 0, 11, 1, 4094, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 4035, 1] payload_rows_per_dst=[7592, 8192, 8190, 6160, 8192, 8192, 6064, 8148] dup_rows_per_dst=[742, 0, 68, 0, 4, 88, 840, 3064] combine_rows_per_src=[7592, 8192, 8190, 6160, 8192, 8192, 6064, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56916 payload_rows_remote=48940 dup_rows=8620 dup_rows_remote=7460 dedup_ratio=0.131531 router_tokens_per_expert=[4096, 0, 169, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 1, 0, 0, 0, 0, 508, 0, 1, 0, 4096, 4095, 0, 10, 0, 1, 0, 0, 0, 0, 4009, 0] payload_rows_per_dst=[7976, 8190, 8190, 8192, 8190, 7122, 1038, 8018] dup_rows_per_dst=[1160, 0, 0, 7322, 0, 136, 0, 2] combine_rows_per_src=[7976, 8190, 8190, 8192, 8190, 7122, 1038, 8018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395057152 dispatch_meta_bytes=221832 combine_payload_bytes=395057152 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=395278984 ingress_bytes=945559552 link_bytes=945559552 token_bytes=790336136 weight_movement_bytes=550502400 total_nvlink_bytes=1340838536 weight_to_token_ratio=0.696542 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.644483579e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=399558656 dispatch_meta_bytes=229384 combine_payload_bytes=399558656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399788040 ingress_bytes=509659136 link_bytes=509659136 token_bytes=799346696 weight_movement_bytes=110100480 total_nvlink_bytes=909447176 weight_to_token_ratio=0.137738 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.863810620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62106 payload_rows_remote=53914 dup_rows=3430 dup_rows_remote=3364 dedup_ratio=0.052338 router_tokens_per_expert=[4095, 0, 284, 1, 4025, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 0, 0, 4075, 0, 0, 0, 1, 4, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1, 1, 0, 1244, 0, 50, 0, 4094, 4086, 4, 103, 0, 1, 0, 13, 0, 0, 3954, 0] payload_rows_per_dst=[8078, 8192, 8152, 5208, 8172, 8192, 8192, 7920] dup_rows_per_dst=[542, 8, 0, 0, 0, 66, 2798, 16] combine_rows_per_src=[8078, 8192, 8152, 5208, 8172, 8192, 8192, 7920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59608 payload_rows_remote=51418 dup_rows=5928 dup_rows_remote=3118 dedup_ratio=0.090454 router_tokens_per_expert=[0, 4096, 4084, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 909, 7, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 765, 0, 3663, 98, 0, 1024, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1, 4095, 0, 0, 0, 1729] payload_rows_per_dst=[8170, 8190, 8188, 8192, 8192, 7024, 8192, 3460] dup_rows_per_dst=[0, 2810, 0, 4, 12, 1050, 2052, 0] combine_rows_per_src=[8170, 8190, 8188, 8192, 8192, 7024, 8192, 3460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=400117760 dispatch_meta_bytes=233568 combine_payload_bytes=400117760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400351328 ingress_bytes=510218240 link_bytes=510218240 token_bytes=800469088 weight_movement_bytes=110100480 total_nvlink_bytes=910569568 weight_to_token_ratio=0.137545 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.873534358e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=400103424 dispatch_meta_bytes=229352 combine_payload_bytes=400103424 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=400332776 ingress_bytes=620304384 link_bytes=620304384 token_bytes=800436200 weight_movement_bytes=220200960 total_nvlink_bytes=1020637160 weight_to_token_ratio=0.275101 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078811346e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376907776 dispatch_meta_bytes=217296 combine_payload_bytes=376907776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377125072 ingress_bytes=487008256 link_bytes=487008256 token_bytes=754032848 weight_movement_bytes=110100480 total_nvlink_bytes=864133328 weight_to_token_ratio=0.146015 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.469874563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=350801920 dispatch_meta_bytes=225600 combine_payload_bytes=350801920 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=351027520 ingress_bytes=681103360 link_bytes=681103360 token_bytes=701829440 weight_movement_bytes=330301440 total_nvlink_bytes=1032130880 weight_to_token_ratio=0.470629 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.184550765e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=386455552 dispatch_meta_bytes=229112 combine_payload_bytes=386455552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386684664 ingress_bytes=496556032 link_bytes=496556032 token_bytes=773140216 weight_movement_bytes=110100480 total_nvlink_bytes=883240696 weight_to_token_ratio=0.142407 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.635926091e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=368564224 dispatch_meta_bytes=218144 combine_payload_bytes=368564224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=368782368 ingress_bytes=588765184 link_bytes=588765184 token_bytes=737346592 weight_movement_bytes=220200960 total_nvlink_bytes=957547552 weight_to_token_ratio=0.298640 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.023959490e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52668 payload_rows_remote=49864 dup_rows=12868 dup_rows_remote=12868 dedup_ratio=0.196350 router_tokens_per_expert=[0, 4096, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 873, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 3083, 111, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1518] payload_rows_per_dst=[8186, 1746, 8192, 8192, 8192, 7164, 2804, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 1640, 0, 11228] combine_rows_per_src=[8186, 1746, 8192, 8192, 8192, 7164, 2804, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58830 payload_rows_remote=50638 dup_rows=6706 dup_rows_remote=6640 dedup_ratio=0.102325 router_tokens_per_expert=[0, 4096, 4085, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 570, 0, 0, 0, 0, 4096, 0, 0, 23, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 3580, 91, 0, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 987] payload_rows_per_dst=[8170, 8192, 8192, 8192, 8192, 7726, 8192, 1974] dup_rows_per_dst=[0, 1140, 66, 0, 0, 1684, 3816, 0] combine_rows_per_src=[8170, 8192, 8192, 8192, 8192, 7726, 8192, 1974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57914 payload_rows_remote=49722 dup_rows=7622 dup_rows_remote=7622 dedup_ratio=0.116302 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 1202, 0, 0, 0, 0, 4096, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 3670, 136, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 486] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7790, 8192, 972] dup_rows_per_dst=[0, 2432, 30, 0, 0, 1646, 3514, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7790, 8192, 972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=357425152 dispatch_meta_bytes=250928 combine_payload_bytes=357425152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357676080 ingress_bytes=467525632 link_bytes=467525632 token_bytes=715101232 weight_movement_bytes=110100480 total_nvlink_bytes=825201712 weight_to_token_ratio=0.153965 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.131039688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60458 payload_rows_remote=52266 dup_rows=5078 dup_rows_remote=5078 dedup_ratio=0.077484 router_tokens_per_expert=[1, 4094, 4089, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 489, 3, 0, 0, 0, 4093, 0, 0, 6, 6, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 3643, 154, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 1, 0, 1750] payload_rows_per_dst=[8180, 8192, 8186, 8190, 8192, 7828, 8188, 3502] dup_rows_per_dst=[2, 982, 24, 4, 0, 1128, 2936, 2] combine_rows_per_src=[8180, 8192, 8186, 8190, 8192, 7828, 8188, 3502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=362973184 dispatch_meta_bytes=229112 combine_payload_bytes=362973184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=363202296 ingress_bytes=693274624 link_bytes=693274624 token_bytes=726175480 weight_movement_bytes=330301440 total_nvlink_bytes=1056476920 weight_to_token_ratio=0.454851 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.205718595e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52132 payload_rows_remote=43942 dup_rows=13404 dup_rows_remote=684 dedup_ratio=0.204529 router_tokens_per_expert=[0, 4095, 4095, 0, 0, 0, 1, 0, 4094, 0, 3, 0, 0, 391, 0, 0, 0, 1, 4096, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 536, 0, 3441, 92, 0, 1457, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2266] payload_rows_per_dst=[8192, 788, 8088, 8190, 8192, 7576, 2916, 8190] dup_rows_per_dst=[0, 0, 8, 2, 8, 666, 0, 12720] combine_rows_per_src=[8192, 788, 8088, 8190, 8192, 7576, 2916, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58328 payload_rows_remote=50552 dup_rows=7208 dup_rows_remote=5418 dedup_ratio=0.109985 router_tokens_per_expert=[0, 4095, 4090, 0, 0, 0, 0, 2, 4092, 0, 14, 0, 0, 355, 6, 2, 0, 0, 4096, 0, 0, 2, 6, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1191, 0, 3454, 138, 0, 2388, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 1, 639] payload_rows_per_dst=[8184, 4858, 8192, 8192, 8192, 7776, 4848, 8086] dup_rows_per_dst=[0, 406, 16, 8, 0, 1790, 36, 4952] combine_rows_per_src=[8184, 4858, 8192, 8192, 8192, 7776, 4848, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54384 payload_rows_remote=46192 dup_rows=11152 dup_rows_remote=11084 dedup_ratio=0.170166 router_tokens_per_expert=[0, 4095, 4093, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 740, 1, 0, 0, 0, 4096, 1, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1402, 0, 3967, 122, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1128] payload_rows_per_dst=[8192, 8192, 8156, 8178, 8170, 3048, 8190, 2258] dup_rows_per_dst=[68, 9416, 6, 0, 2, 0, 1660, 0] combine_rows_per_src=[8192, 8192, 8156, 8178, 8170, 3048, 8190, 2258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64442 payload_rows_remote=56256 dup_rows=1094 dup_rows_remote=956 dedup_ratio=0.016693 router_tokens_per_expert=[0, 4095, 4, 0, 0, 0, 6, 0, 68, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 408, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 6, 0, 4, 0, 2, 0, 0, 0, 0, 613, 3075, 3, 0, 0, 4089, 4085, 4096, 0, 0, 0, 4009, 0] payload_rows_per_dst=[8192, 8186, 8186, 8192, 8192, 8190, 7286, 8018] dup_rows_per_dst=[18, 138, 2, 818, 10, 12, 96, 0] combine_rows_per_src=[8192, 8186, 8186, 8192, 8192, 8190, 7286, 8018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=356407296 dispatch_meta_bytes=229376 combine_payload_bytes=356407296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356636672 ingress_bytes=466507776 link_bytes=466507776 token_bytes=713043968 weight_movement_bytes=110100480 total_nvlink_bytes=823144448 weight_to_token_ratio=0.154409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.113337498e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374642688 dispatch_meta_bytes=229376 combine_payload_bytes=374642688 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=374872064 ingress_bytes=484743168 link_bytes=484743168 token_bytes=749514752 weight_movement_bytes=110100480 total_nvlink_bytes=859615232 weight_to_token_ratio=0.146896 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.430480957e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314976256 dispatch_meta_bytes=178504 combine_payload_bytes=314976256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=315154760 ingress_bytes=535177216 link_bytes=535177216 token_bytes=630131016 weight_movement_bytes=220200960 total_nvlink_bytes=850331976 weight_to_token_ratio=0.349453 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=12288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.307612002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=362356736 dispatch_meta_bytes=223880 combine_payload_bytes=362356736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362580616 ingress_bytes=582557696 link_bytes=582557696 token_bytes=724937352 weight_movement_bytes=220200960 total_nvlink_bytes=945138312 weight_to_token_ratio=0.303752 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.013163648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=331104256 dispatch_meta_bytes=229104 combine_payload_bytes=331104256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331333360 ingress_bytes=551305216 link_bytes=551305216 token_bytes=662437616 weight_movement_bytes=220200960 total_nvlink_bytes=882638576 weight_to_token_ratio=0.332410 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.588104448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403243008 dispatch_meta_bytes=228848 combine_payload_bytes=403243008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403471856 ingress_bytes=513343488 link_bytes=513343488 token_bytes=806714864 weight_movement_bytes=110100480 total_nvlink_bytes=916815344 weight_to_token_ratio=0.136480 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.927887561e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64422 payload_rows_remote=57230 dup_rows=1114 dup_rows_remote=1114 dedup_ratio=0.016998 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 479, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3577, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4060, 0] payload_rows_per_dst=[8192, 8192, 8192, 8150, 8192, 8192, 7192, 8120] dup_rows_per_dst=[0, 160, 0, 954, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8150, 8192, 8192, 7192, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64620 payload_rows_remote=56428 dup_rows=916 dup_rows_remote=916 dedup_ratio=0.013977 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 395, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3679, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3987, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7494, 7974] dup_rows_per_dst=[0, 112, 0, 790, 0, 0, 14, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7494, 7974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410224640 dispatch_meta_bytes=233376 combine_payload_bytes=410224640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410458016 ingress_bytes=520325120 link_bytes=520325120 token_bytes=820682656 weight_movement_bytes=110100480 total_nvlink_bytes=930783136 weight_to_token_ratio=0.134157 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.049309625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64956 payload_rows_remote=56800 dup_rows=580 dup_rows_remote=256 dedup_ratio=0.008850 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 168, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 1, 389, 3612, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4002, 0] payload_rows_per_dst=[8192, 8192, 8192, 8156, 8192, 8192, 7836, 8004] dup_rows_per_dst=[0, 82, 0, 324, 0, 6, 168, 0] combine_rows_per_src=[8192, 8192, 8192, 8156, 8192, 8192, 7836, 8004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=404475904 dispatch_meta_bytes=229376 combine_payload_bytes=404475904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404705280 ingress_bytes=514576384 link_bytes=514576384 token_bytes=809181184 weight_movement_bytes=110100480 total_nvlink_bytes=919281664 weight_to_token_ratio=0.136064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64594 payload_rows_remote=56402 dup_rows=942 dup_rows_remote=936 dedup_ratio=0.014374 router_tokens_per_expert=[0, 4091, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 346, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 1, 0, 15, 0, 0, 0, 12, 54, 3581, 0, 2, 0, 4088, 4094, 4096, 0, 1, 0, 4084, 0] payload_rows_per_dst=[8182, 8190, 8190, 8190, 8192, 8192, 7288, 8170] dup_rows_per_dst=[0, 212, 0, 690, 6, 24, 10, 0] combine_rows_per_src=[8182, 8190, 8190, 8190, 8192, 8192, 7288, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64124 payload_rows_remote=55932 dup_rows=1412 dup_rows_remote=1344 dedup_ratio=0.021545 router_tokens_per_expert=[1, 4095, 1, 0, 0, 0, 0, 0, 208, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 452, 0, 0, 0, 0, 4061, 1, 0, 0, 3, 0, 1, 1, 0, 0, 0, 1, 14, 0, 6, 0, 17, 0, 0, 0, 3, 101, 3387, 0, 11, 0, 4092, 4083, 4093, 0, 1, 0, 4039, 0] payload_rows_per_dst=[8192, 8178, 8188, 8132, 8192, 8192, 6970, 8080] dup_rows_per_dst=[2, 408, 0, 896, 4, 68, 34, 0] combine_rows_per_src=[8192, 8178, 8188, 8132, 8192, 8192, 6970, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58110 payload_rows_remote=49928 dup_rows=7426 dup_rows_remote=5944 dedup_ratio=0.113312 router_tokens_per_expert=[0, 4095, 0, 0, 0, 1, 0, 0, 43, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 423, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 5, 1, 0, 0, 1, 0, 0, 112, 3520, 0, 0, 0, 4094, 4094, 4095, 0, 0, 0, 4091, 0] payload_rows_per_dst=[8184, 6862, 8190, 2570, 8168, 8078, 7876, 8182] dup_rows_per_dst=[0, 76, 8, 302, 2, 8, 5548, 1482] combine_rows_per_src=[8184, 6862, 8190, 2570, 8168, 8078, 7876, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64354 payload_rows_remote=56164 dup_rows=1182 dup_rows_remote=1182 dedup_ratio=0.018036 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 166, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 3530, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4088, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8190, 7702, 7506] dup_rows_per_dst=[0, 164, 4, 990, 2, 0, 22, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8190, 7702, 7506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59722 payload_rows_remote=51532 dup_rows=5814 dup_rows_remote=5808 dedup_ratio=0.088715 router_tokens_per_expert=[0, 0, 108, 0, 4088, 0, 0, 4094, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 8, 2782, 0, 34, 5, 0, 0, 0, 0, 0, 4059, 4095, 0, 9, 1174, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091] payload_rows_per_dst=[8182, 8190, 8192, 8190, 8182, 8190, 2414, 8182] dup_rows_per_dst=[210, 6, 0, 0, 5588, 10, 0, 0] combine_rows_per_src=[8182, 8190, 8192, 8190, 8182, 8190, 2414, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=407142400 dispatch_meta_bytes=228224 combine_payload_bytes=407142400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407370624 ingress_bytes=627343360 link_bytes=627343360 token_bytes=814513024 weight_movement_bytes=220200960 total_nvlink_bytes=1034713984 weight_to_token_ratio=0.270347 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091053283e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=404289536 dispatch_meta_bytes=229352 combine_payload_bytes=404289536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404518888 ingress_bytes=514390016 link_bytes=514390016 token_bytes=808808424 weight_movement_bytes=110100480 total_nvlink_bytes=918908904 weight_to_token_ratio=0.136127 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.946088404e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=400920576 dispatch_meta_bytes=229104 combine_payload_bytes=400920576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401149680 ingress_bytes=511021056 link_bytes=511021056 token_bytes=802070256 weight_movement_bytes=110100480 total_nvlink_bytes=912170736 weight_to_token_ratio=0.137270 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.887496649e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357883904 dispatch_meta_bytes=223488 combine_payload_bytes=357883904 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=358107392 ingress_bytes=908386304 link_bytes=908386304 token_bytes=715991296 weight_movement_bytes=550502400 total_nvlink_bytes=1266493696 weight_to_token_ratio=0.768867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.579833187e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402583552 dispatch_meta_bytes=229384 combine_payload_bytes=402583552 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402812936 ingress_bytes=622784512 link_bytes=622784512 token_bytes=805396488 weight_movement_bytes=220200960 total_nvlink_bytes=1025597448 weight_to_token_ratio=0.273407 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083124697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369381376 dispatch_meta_bytes=229360 combine_payload_bytes=369381376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369610736 ingress_bytes=479481856 link_bytes=479481856 token_bytes=738992112 weight_movement_bytes=110100480 total_nvlink_bytes=849092592 weight_to_token_ratio=0.148987 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.338978088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62176 payload_rows_remote=53984 dup_rows=3360 dup_rows_remote=4 dedup_ratio=0.051270 router_tokens_per_expert=[0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 2539, 0, 32, 0, 0, 0, 0, 0, 0, 3933, 4096, 0, 0, 1623, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5158, 7866, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 3356, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5158, 7866, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61484 payload_rows_remote=53292 dup_rows=4052 dup_rows_remote=4052 dedup_ratio=0.061829 router_tokens_per_expert=[0, 0, 5, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 2688, 0, 12, 3, 0, 0, 0, 0, 0, 4044, 4096, 0, 0, 1351, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6284, 8124, 6118, 8190] dup_rows_per_dst=[10, 0, 0, 0, 2006, 80, 1956, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6284, 8124, 6118, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7081989752 prefetch_bytes=1101004800 reduce_bytes=110100480 weight_movement_bytes=1211105280 egress_bytes=3542115960 ingress_bytes=4750979072 link_bytes=4751208456 total_nvlink_bytes=8293095032 weight_to_token_ratio=0.171012 prefetch_share_of_ingress=0.231743 dedup_ratio=0.124341 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=1 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10160 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.263132944e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386957312 dispatch_meta_bytes=215952 combine_payload_bytes=386957312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387173264 ingress_bytes=497057792 link_bytes=497057792 token_bytes=774130576 weight_movement_bytes=110100480 total_nvlink_bytes=884231056 weight_to_token_ratio=0.142225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.644652522e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62790 payload_rows_remote=54598 dup_rows=2746 dup_rows_remote=2746 dedup_ratio=0.041901 router_tokens_per_expert=[1, 0, 111, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 1, 2744, 0, 38, 16, 0, 0, 0, 0, 0, 4013, 4096, 0, 36, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8192, 8192, 8192, 5584, 8058, 8192, 8192] dup_rows_per_dst=[220, 0, 0, 0, 0, 0, 2526, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 5584, 8058, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63846 payload_rows_remote=57290 dup_rows=1690 dup_rows_remote=1690 dedup_ratio=0.025787 router_tokens_per_expert=[0, 0, 35, 0, 4094, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 18, 0, 0, 0, 3236, 1, 23, 0, 0, 0, 0, 1, 0, 4078, 4096, 0, 13, 731, 0, 60, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4092] payload_rows_per_dst=[8190, 8190, 8192, 8182, 6556, 8158, 8192, 8186] dup_rows_per_dst=[72, 0, 6, 0, 0, 0, 1608, 4] combine_rows_per_src=[8190, 8190, 8192, 8182, 6556, 8158, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381997056 dispatch_meta_bytes=229376 combine_payload_bytes=381997056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382226432 ingress_bytes=602198016 link_bytes=602198016 token_bytes=764223488 weight_movement_bytes=220200960 total_nvlink_bytes=984424448 weight_to_token_ratio=0.288137 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047321394e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63014 payload_rows_remote=54976 dup_rows=2522 dup_rows_remote=2516 dedup_ratio=0.038483 router_tokens_per_expert=[3, 0, 93, 0, 4087, 0, 2, 4094, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 4095, 1, 9, 0, 0, 7, 2886, 0, 22, 3, 1, 0, 0, 1, 0, 4017, 4087, 0, 7, 1137, 0, 22, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4092] payload_rows_per_dst=[8188, 8190, 8188, 8192, 5840, 8038, 8192, 8186] dup_rows_per_dst=[182, 2, 0, 2, 10, 6, 2320, 0] combine_rows_per_src=[8188, 8190, 8188, 8192, 5840, 8038, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62770 payload_rows_remote=54594 dup_rows=2766 dup_rows_remote=2722 dedup_ratio=0.042206 router_tokens_per_expert=[0, 0, 25, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 16, 0, 0, 0, 3114, 1, 30, 8, 0, 0, 0, 0, 0, 4041, 4093, 0, 0, 961, 0, 6, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4087] payload_rows_per_dst=[7426, 8192, 8180, 8190, 6322, 8092, 8192, 8176] dup_rows_per_dst=[48, 0, 8, 12, 2, 722, 1930, 44] combine_rows_per_src=[7426, 8192, 8180, 8190, 6322, 8092, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7214917648 prefetch_bytes=1431306240 reduce_bytes=440401920 weight_movement_bytes=1871708160 egress_bytes=3608596496 ingress_bytes=5478029312 link_bytes=5478029312 total_nvlink_bytes=9086625808 weight_to_token_ratio=0.259422 prefetch_share_of_ingress=0.261281 dedup_ratio=0.125021 prefetch_slots=26 unused_prefetch_slots=134 slot_utilization=0.162500 reduce_slots=4 remote_demand=13 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.527193955e-05 prefetch_bytes_per_gemm_flop=2.489276961e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59682 payload_rows_remote=51490 dup_rows=5854 dup_rows_remote=5814 dedup_ratio=0.089325 router_tokens_per_expert=[0, 0, 19, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 2925, 0, 19, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 0, 1142, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 2348, 8184] dup_rows_per_dst=[40, 0, 2, 0, 5812, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 2348, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=391358464 dispatch_meta_bytes=229376 combine_payload_bytes=391358464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391587840 ingress_bytes=501458944 link_bytes=501458944 token_bytes=782946304 weight_movement_bytes=110100480 total_nvlink_bytes=893046784 weight_to_token_ratio=0.140623 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.721195794e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410654720 dispatch_meta_bytes=235920 combine_payload_bytes=410654720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410890640 ingress_bytes=520755200 link_bytes=520755200 token_bytes=821545360 weight_movement_bytes=110100480 total_nvlink_bytes=931645840 weight_to_token_ratio=0.134016 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.056789423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7211337152 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3606793664 ingress_bytes=5586352128 link_bytes=5586352128 total_nvlink_bytes=9193145792 weight_to_token_ratio=0.274818 prefetch_share_of_ingress=0.177379 dedup_ratio=0.116733 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.715585148e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394067968 dispatch_meta_bytes=229968 combine_payload_bytes=394067968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394297936 ingress_bytes=724369408 link_bytes=724369408 token_bytes=788365904 weight_movement_bytes=330301440 total_nvlink_bytes=1118667344 weight_to_token_ratio=0.418970 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.259797539e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=391329792 dispatch_meta_bytes=229264 combine_payload_bytes=391329792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391559056 ingress_bytes=501430272 link_bytes=501430272 token_bytes=782888848 weight_movement_bytes=110100480 total_nvlink_bytes=892989328 weight_to_token_ratio=0.140634 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.720697141e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7336094136 prefetch_bytes=1101004800 reduce_bytes=1431306240 weight_movement_bytes=2532311040 egress_bytes=3669203384 ingress_bytes=6199201792 link_bytes=6199201792 total_nvlink_bytes=9868405176 weight_to_token_ratio=0.345185 prefetch_share_of_ingress=0.177604 dedup_ratio=0.112494 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=13 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9779 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.078143151e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369080320 dispatch_meta_bytes=229216 combine_payload_bytes=369080320 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=369309536 ingress_bytes=809482240 link_bytes=809482240 token_bytes=738389856 weight_movement_bytes=440401920 total_nvlink_bytes=1178791776 weight_to_token_ratio=0.596435 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.407822752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7279978440 prefetch_bytes=1101004800 reduce_bytes=330301440 weight_movement_bytes=1431306240 egress_bytes=3641128904 ingress_bytes=5070155776 link_bytes=5070155776 total_nvlink_bytes=8711284680 weight_to_token_ratio=0.196609 prefetch_share_of_ingress=0.217154 dedup_ratio=0.107700 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=3 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.817834792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7084256424 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3543250088 ingress_bytes=5853116416 link_bytes=5853116416 total_nvlink_bytes=9396366504 weight_to_token_ratio=0.326373 prefetch_share_of_ingress=0.150485 dedup_ratio=0.123444 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.017953212e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6748487776 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3375312992 ingress_bytes=5685284864 link_bytes=5685284864 total_nvlink_bytes=9060597856 weight_to_token_ratio=0.342612 prefetch_share_of_ingress=0.174293 dedup_ratio=0.160529 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=10668 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.887645447e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6692124112 prefetch_bytes=660602880 reduce_bytes=2202009600 weight_movement_bytes=2862612480 egress_bytes=3347162576 ingress_bytes=6207574016 link_bytes=6207574016 total_nvlink_bytes=9554736592 weight_to_token_ratio=0.427758 prefetch_share_of_ingress=0.106419 dedup_ratio=0.166736 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=20 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9779 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.079599218e-04 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none + [2026-08-29 23:43:04] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 241.7 | throughput per GPU (TFLOP/s/GPU): 510.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000670E+01 | loss scale: 1.0 | grad norm: 26.787 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46680 payload_rows_remote=40020 dup_rows=18856 dup_rows_remote=15724 dedup_ratio=0.287720 router_tokens_per_expert=[48, 40, 127, 36, 8, 3, 1137, 1486, 115, 83, 106, 1048, 1, 183, 0, 327, 160, 788, 2026, 3, 183, 654, 20, 373, 2036, 349, 271, 1171, 29, 2641, 1034, 6, 4, 4, 92, 1576, 5, 4, 29, 479, 405, 457, 520, 19, 2016, 624, 1412, 1391, 15, 1408, 1348, 118, 1206, 152, 84, 259, 617, 486, 141, 188, 245, 108, 864, 0] payload_rows_per_dst=[5814, 6082, 5934, 6660, 6752, 4990, 6144, 4304] dup_rows_per_dst=[2780, 1676, 2480, 3132, 2916, 1842, 3036, 994] combine_rows_per_src=[5814, 6082, 5934, 6660, 6752, 4990, 6144, 4304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47326 payload_rows_remote=41752 dup_rows=18210 dup_rows_remote=16034 dedup_ratio=0.277863 router_tokens_per_expert=[45, 116, 43, 76, 0, 15, 1315, 2213, 94, 20, 150, 728, 0, 76, 1, 341, 209, 623, 1623, 0, 145, 316, 54, 251, 1454, 56, 295, 1278, 36, 2263, 651, 0, 2, 5, 87, 2073, 7, 0, 5, 242, 1323, 240, 597, 6, 2794, 750, 1586, 1033, 39, 655, 1334, 272, 455, 151, 59, 164, 1384, 1216, 260, 311, 130, 55, 1046, 0] payload_rows_per_dst=[6726, 6760, 5574, 5508, 6110, 5532, 4912, 6204] dup_rows_per_dst=[4092, 1648, 2176, 1844, 2138, 2366, 1346, 2600] combine_rows_per_src=[6726, 6760, 5574, 5508, 6110, 5532, 4912, 6204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47508 payload_rows_remote=40868 dup_rows=18028 dup_rows_remote=14632 dedup_ratio=0.275085 router_tokens_per_expert=[45, 18, 19, 41, 4, 9, 1008, 2655, 54, 26, 141, 905, 1, 114, 1, 346, 93, 737, 1349, 0, 89, 440, 86, 130, 1943, 106, 263, 1446, 16, 2590, 1125, 12, 18, 4, 40, 1767, 8, 3, 3, 397, 998, 438, 384, 4, 2403, 713, 821, 1215, 66, 472, 1161, 255, 611, 196, 120, 194, 1002, 1613, 259, 241, 170, 229, 1151, 0] payload_rows_per_dst=[5942, 6354, 4678, 4938, 6738, 5964, 6640, 6254] dup_rows_per_dst=[1656, 1628, 1170, 998, 2922, 3182, 3396, 3076] combine_rows_per_src=[5942, 6354, 4678, 4938, 6738, 5964, 6640, 6254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45276 payload_rows_remote=39314 dup_rows=20260 dup_rows_remote=17776 dedup_ratio=0.309143 router_tokens_per_expert=[40, 35, 45, 47, 1, 25, 1433, 1652, 81, 15, 143, 1156, 1, 97, 0, 379, 217, 490, 1769, 2, 274, 510, 57, 131, 1416, 211, 253, 1624, 15, 1897, 704, 0, 2, 13, 143, 2310, 0, 0, 18, 505, 1180, 223, 421, 2, 2367, 599, 1260, 1243, 94, 987, 923, 97, 1150, 395, 86, 209, 1168, 1069, 117, 101, 272, 60, 1033, 1] payload_rows_per_dst=[5052, 3526, 5334, 4718, 6590, 7250, 6844, 5962] dup_rows_per_dst=[1504, 420, 1566, 896, 3186, 6334, 3870, 2484] combine_rows_per_src=[5052, 3526, 5334, 4718, 6590, 7250, 6844, 5962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47658 payload_rows_remote=41004 dup_rows=17878 dup_rows_remote=15156 dedup_ratio=0.272797 router_tokens_per_expert=[39, 72, 35, 12, 0, 3, 1205, 1942, 79, 104, 173, 618, 0, 127, 0, 703, 467, 436, 1915, 0, 67, 581, 42, 57, 1280, 63, 445, 876, 16, 2547, 848, 3, 9, 13, 49, 1752, 2, 0, 1, 315, 968, 170, 1358, 7, 2656, 631, 1843, 676, 182, 545, 1268, 96, 401, 292, 81, 136, 1318, 1191, 190, 610, 92, 104, 1057, 0] payload_rows_per_dst=[6658, 6812, 5486, 5338, 6654, 5548, 4846, 6316] dup_rows_per_dst=[3644, 2108, 1644, 1724, 2722, 2072, 1156, 2808] combine_rows_per_src=[6658, 6812, 5486, 5338, 6654, 5548, 4846, 6316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46450 payload_rows_remote=40594 dup_rows=19086 dup_rows_remote=15816 dedup_ratio=0.291229 router_tokens_per_expert=[32, 56, 61, 35, 0, 18, 938, 1392, 75, 90, 287, 1574, 0, 83, 0, 458, 224, 462, 2166, 11, 317, 405, 57, 127, 1077, 111, 451, 1179, 17, 2023, 706, 14, 25, 11, 86, 1961, 5, 10, 2, 696, 1085, 437, 740, 18, 2444, 492, 1334, 833, 46, 1025, 1039, 138, 625, 164, 98, 203, 1241, 1281, 279, 132, 119, 125, 1624, 4] payload_rows_per_dst=[4598, 7012, 5696, 4310, 6450, 5856, 6022, 6506] dup_rows_per_dst=[1218, 3010, 1842, 678, 3188, 3270, 2776, 3104] combine_rows_per_src=[4598, 7012, 5696, 4310, 6450, 5856, 6022, 6506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45730 payload_rows_remote=39190 dup_rows=19806 dup_rows_remote=16820 dedup_ratio=0.302216 router_tokens_per_expert=[46, 55, 129, 57, 16, 8, 917, 1737, 567, 205, 170, 1129, 0, 91, 4, 403, 125, 1503, 1769, 2, 209, 424, 39, 502, 2128, 158, 425, 1025, 94, 1988, 593, 1, 29, 23, 142, 1576, 9, 1, 0, 412, 611, 516, 780, 12, 2194, 582, 1091, 1487, 18, 664, 1044, 203, 988, 197, 180, 51, 1159, 594, 115, 253, 249, 89, 979, 1] payload_rows_per_dst=[5774, 6540, 6722, 7000, 3846, 5342, 5270, 5236] dup_rows_per_dst=[2338, 2986, 4474, 4314, 538, 2094, 1420, 1642] combine_rows_per_src=[5774, 6540, 6722, 7000, 3846, 5342, 5270, 5236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44778 payload_rows_remote=39516 dup_rows=20758 dup_rows_remote=18280 dedup_ratio=0.316742 router_tokens_per_expert=[34, 36, 44, 37, 0, 30, 769, 1835, 36, 109, 181, 542, 1, 45, 42, 609, 183, 517, 1566, 56, 230, 828, 48, 225, 1839, 81, 276, 1727, 22, 2468, 969, 4, 41, 22, 60, 1310, 24, 0, 6, 930, 600, 295, 536, 0, 2922, 516, 1085, 1080, 342, 974, 1217, 218, 435, 270, 177, 296, 863, 1196, 279, 507, 203, 69, 854, 52] payload_rows_per_dst=[5262, 6934, 6764, 7448, 3890, 3170, 5562, 5748] dup_rows_per_dst=[2478, 2040, 3996, 6030, 896, 724, 2296, 2298] combine_rows_per_src=[5262, 6934, 6764, 7448, 3890, 3170, 5562, 5748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286863360 dispatch_meta_bytes=222976 combine_payload_bytes=286863360 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=287086336 ingress_bytes=727265280 link_bytes=727265280 token_bytes=573949696 weight_movement_bytes=440401920 total_nvlink_bytes=1014351616 weight_to_token_ratio=0.767318 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.264833937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=231144 combine_payload_bytes=299278336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299509480 ingress_bytes=409378816 link_bytes=409378816 token_bytes=598787816 weight_movement_bytes=110100480 total_nvlink_bytes=708888296 weight_to_token_ratio=0.183872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.119770923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292941824 dispatch_meta_bytes=222000 combine_payload_bytes=292941824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293163824 ingress_bytes=403042304 link_bytes=403042304 token_bytes=586105648 weight_movement_bytes=110100480 total_nvlink_bytes=696206128 weight_to_token_ratio=0.187851 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.009568557e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281802752 dispatch_meta_bytes=228360 combine_payload_bytes=281802752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282031112 ingress_bytes=391903232 link_bytes=391903232 token_bytes=563833864 weight_movement_bytes=110100480 total_nvlink_bytes=673934344 weight_to_token_ratio=0.195271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.815841775e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293916672 dispatch_meta_bytes=224640 combine_payload_bytes=293916672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294141312 ingress_bytes=404017152 link_bytes=404017152 token_bytes=588057984 weight_movement_bytes=110100480 total_nvlink_bytes=698158464 weight_to_token_ratio=0.187227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.026522767e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290977792 dispatch_meta_bytes=225640 combine_payload_bytes=290977792 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=291203432 ingress_bytes=731379712 link_bytes=731379712 token_bytes=582181224 weight_movement_bytes=440401920 total_nvlink_bytes=1022583144 weight_to_token_ratio=0.756469 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.271989610e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280913920 dispatch_meta_bytes=224040 combine_payload_bytes=280913920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281137960 ingress_bytes=391014400 link_bytes=391014400 token_bytes=562051880 weight_movement_bytes=110100480 total_nvlink_bytes=672152360 weight_to_token_ratio=0.195890 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800383524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283250688 dispatch_meta_bytes=231184 combine_payload_bytes=283250688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283481872 ingress_bytes=393351168 link_bytes=393351168 token_bytes=566732560 weight_movement_bytes=110100480 total_nvlink_bytes=676833040 weight_to_token_ratio=0.194272 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.841023763e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58680 payload_rows_remote=50488 dup_rows=6856 dup_rows_remote=5776 dedup_ratio=0.104614 router_tokens_per_expert=[0, 3922, 0, 155, 0, 4089, 29, 1, 0, 84, 46, 0, 0, 0, 0, 0, 0, 3271, 3384, 64, 0, 2197, 0, 3695, 7, 4096, 531, 0, 0, 0, 2, 0, 0, 10, 0, 0, 0, 0, 0, 17, 124, 0, 13, 0, 178, 0, 0, 0, 0, 1, 0, 3157, 723, 2284, 0, 0, 137, 113, 1, 0, 0, 0, 436, 1] payload_rows_per_dst=[7860, 6860, 7852, 8192, 7408, 8184, 5396, 6928] dup_rows_per_dst=[354, 168, 3212, 1080, 36, 624, 620, 762] combine_rows_per_src=[7860, 6860, 7852, 8192, 7408, 8184, 5396, 6928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56758 payload_rows_remote=51750 dup_rows=8778 dup_rows_remote=7278 dedup_ratio=0.133942 router_tokens_per_expert=[0, 3941, 0, 247, 1, 4088, 78, 12, 0, 401, 24, 0, 0, 0, 2, 4, 0, 2785, 3302, 7, 0, 2429, 0, 3744, 6, 4077, 110, 0, 5, 0, 0, 0, 0, 39, 0, 0, 0, 1, 0, 7, 160, 3, 13, 0, 67, 2, 1, 0, 0, 9, 0, 3444, 462, 1773, 6, 0, 418, 21, 0, 0, 0, 1, 1061, 17] payload_rows_per_dst=[7934, 6846, 7350, 6222, 7514, 8186, 5008, 7698] dup_rows_per_dst=[624, 620, 2644, 166, 516, 482, 1500, 2226] combine_rows_per_src=[7934, 6846, 7350, 6222, 7514, 8186, 5008, 7698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59252 payload_rows_remote=51934 dup_rows=6284 dup_rows_remote=3816 dedup_ratio=0.095886 router_tokens_per_expert=[0, 4016, 5, 189, 0, 4092, 36, 21, 1, 76, 5, 0, 0, 0, 0, 0, 0, 2434, 3707, 27, 0, 2427, 5, 3646, 6, 4054, 240, 0, 2, 1, 0, 0, 2, 6, 0, 0, 1, 1, 0, 2, 38, 2, 0, 0, 31, 0, 0, 0, 0, 0, 2, 3595, 545, 2146, 0, 0, 265, 78, 8, 0, 1, 0, 1034, 21] payload_rows_per_dst=[8042, 7426, 7318, 8126, 7308, 8190, 4946, 7896] dup_rows_per_dst=[492, 152, 2468, 480, 8, 136, 440, 2108] combine_rows_per_src=[8042, 7426, 7318, 8126, 7308, 8190, 4946, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56996 payload_rows_remote=49956 dup_rows=8540 dup_rows_remote=8102 dedup_ratio=0.130310 router_tokens_per_expert=[0, 3893, 0, 79, 0, 4096, 61, 3, 0, 260, 5, 0, 0, 0, 0, 0, 0, 2507, 3474, 32, 0, 2284, 0, 3671, 3, 4093, 203, 0, 0, 0, 2, 0, 0, 107, 0, 0, 0, 0, 0, 26, 23, 0, 58, 1, 91, 0, 1, 0, 0, 0, 1, 3575, 809, 2106, 0, 0, 639, 153, 4, 0, 0, 0, 493, 15] payload_rows_per_dst=[7794, 7040, 7222, 8188, 7396, 8192, 7216, 3948] dup_rows_per_dst=[278, 438, 2424, 414, 212, 348, 3696, 730] combine_rows_per_src=[7794, 7040, 7222, 8188, 7396, 8192, 7216, 3948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53130 payload_rows_remote=45290 dup_rows=12406 dup_rows_remote=10288 dedup_ratio=0.189301 router_tokens_per_expert=[0, 3791, 0, 190, 0, 4080, 10, 4, 6, 142, 3, 0, 0, 2, 0, 0, 0, 2801, 3586, 19, 1, 2904, 0, 3675, 3, 4095, 268, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 9, 12, 3, 6, 0, 70, 0, 0, 0, 0, 0, 0, 3491, 379, 1728, 0, 0, 947, 7, 4, 0, 0, 0, 511, 19] payload_rows_per_dst=[2404, 7222, 5818, 5672, 7796, 8188, 8190, 7840] dup_rows_per_dst=[98, 256, 30, 358, 5178, 154, 4214, 2118] combine_rows_per_src=[2404, 7222, 5818, 5672, 7796, 8188, 8190, 7840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59276 payload_rows_remote=51770 dup_rows=6260 dup_rows_remote=6250 dedup_ratio=0.095520 router_tokens_per_expert=[1, 3974, 0, 198, 0, 4092, 73, 5, 0, 398, 9, 0, 0, 0, 0, 1, 0, 3095, 3505, 21, 0, 1222, 0, 3750, 0, 4075, 252, 2, 0, 2, 0, 0, 2, 4, 0, 0, 0, 0, 0, 2, 90, 0, 16, 0, 42, 0, 0, 0, 0, 0, 0, 3510, 638, 2392, 0, 0, 186, 84, 0, 0, 0, 0, 1107, 20] payload_rows_per_dst=[8008, 7136, 7142, 8150, 7506, 8186, 5510, 7638] dup_rows_per_dst=[494, 690, 1534, 512, 10, 294, 550, 2176] combine_rows_per_src=[8008, 7136, 7142, 8150, 7506, 8186, 5510, 7638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58828 payload_rows_remote=50676 dup_rows=6708 dup_rows_remote=6408 dedup_ratio=0.102356 router_tokens_per_expert=[0, 3901, 7, 225, 0, 4048, 33, 3, 0, 244, 4, 0, 23, 0, 0, 0, 0, 2486, 3725, 1, 0, 2063, 0, 3646, 47, 4064, 571, 0, 0, 0, 8, 0, 0, 22, 0, 0, 0, 0, 0, 21, 82, 0, 19, 0, 77, 0, 0, 0, 0, 0, 0, 3443, 558, 2214, 0, 13, 187, 181, 31, 3, 6, 0, 797, 15] payload_rows_per_dst=[7814, 7494, 7092, 8178, 7316, 8152, 5102, 7680] dup_rows_per_dst=[524, 498, 2008, 1202, 62, 300, 468, 1646] combine_rows_per_src=[7814, 7494, 7092, 8178, 7316, 8152, 5102, 7680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=361897984 dispatch_meta_bytes=225056 combine_payload_bytes=361897984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362123040 ingress_bytes=582098944 link_bytes=582098944 token_bytes=724021024 weight_movement_bytes=220200960 total_nvlink_bytes=944221984 weight_to_token_ratio=0.304136 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012365802e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48942 payload_rows_remote=41096 dup_rows=16594 dup_rows_remote=11624 dedup_ratio=0.253204 router_tokens_per_expert=[0, 3363, 0, 187, 11, 3839, 119, 84, 1, 120, 16, 0, 43, 0, 1, 4, 1, 2996, 3209, 60, 34, 1865, 0, 3507, 11, 3936, 326, 3, 6, 0, 16, 0, 0, 90, 0, 1, 0, 2, 0, 35, 78, 5, 6, 4, 365, 1, 0, 12, 2, 6, 2, 2838, 617, 2871, 0, 4, 1010, 179, 1, 1, 0, 12, 655, 213] payload_rows_per_dst=[7846, 7666, 1360, 7958, 7190, 5676, 7814, 3432] dup_rows_per_dst=[4970, 5024, 62, 638, 80, 244, 4866, 710] combine_rows_per_src=[7846, 7666, 1360, 7958, 7190, 5676, 7814, 3432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=370944000 dispatch_meta_bytes=236112 combine_payload_bytes=370944000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=371180112 ingress_bytes=591144960 link_bytes=591144960 token_bytes=742124112 weight_movement_bytes=220200960 total_nvlink_bytes=962325072 weight_to_token_ratio=0.296717 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028098312e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=372262912 dispatch_meta_bytes=223000 combine_payload_bytes=372262912 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=372485912 ingress_bytes=922765312 link_bytes=922765312 token_bytes=744748824 weight_movement_bytes=550502400 total_nvlink_bytes=1295251224 weight_to_token_ratio=0.739179 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.604840646e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=358084608 dispatch_meta_bytes=232232 combine_payload_bytes=358084608 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=358316840 ingress_bytes=578285568 link_bytes=578285568 token_bytes=716401448 weight_movement_bytes=220200960 total_nvlink_bytes=936602408 weight_to_token_ratio=0.307371 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005733714e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324638720 dispatch_meta_bytes=222312 combine_payload_bytes=324638720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324861032 ingress_bytes=434739200 link_bytes=434739200 token_bytes=649499752 weight_movement_bytes=110100480 total_nvlink_bytes=759600232 weight_to_token_ratio=0.169516 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.560829711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=371087360 dispatch_meta_bytes=232080 combine_payload_bytes=371087360 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=371319440 ingress_bytes=591288320 link_bytes=591288320 token_bytes=742406800 weight_movement_bytes=220200960 total_nvlink_bytes=962607760 weight_to_token_ratio=0.296604 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028347639e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363245568 dispatch_meta_bytes=228336 combine_payload_bytes=363245568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363473904 ingress_bytes=473346048 link_bytes=473346048 token_bytes=726719472 weight_movement_bytes=110100480 total_nvlink_bytes=836819952 weight_to_token_ratio=0.151503 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.232266295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294576128 dispatch_meta_bytes=210880 combine_payload_bytes=294576128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=294787008 ingress_bytes=624877568 link_bytes=624877568 token_bytes=589363136 weight_movement_bytes=330301440 total_nvlink_bytes=919664576 weight_to_token_ratio=0.560438 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086764865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55552 payload_rows_remote=47372 dup_rows=9984 dup_rows_remote=8474 dedup_ratio=0.152344 router_tokens_per_expert=[0, 0, 0, 1, 135, 0, 0, 0, 16, 0, 0, 4096, 3838, 1258, 1223, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 715, 0, 0, 0, 2, 44, 0, 0, 2896, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 0, 4084, 0, 0, 0, 0, 0, 2952, 0, 2945, 0, 0, 0, 0, 0, 834, 0, 0, 777] payload_rows_per_dst=[8192, 7246, 8140, 8180, 6462, 4484, 5904, 6944] dup_rows_per_dst=[272, 3480, 970, 1510, 1584, 0, 0, 2168] combine_rows_per_src=[8192, 7246, 8140, 8180, 6462, 4484, 5904, 6944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52122 payload_rows_remote=47638 dup_rows=13414 dup_rows_remote=13414 dedup_ratio=0.204681 router_tokens_per_expert=[1, 0, 0, 1, 90, 0, 0, 0, 105, 4, 0, 4092, 3701, 1504, 491, 0, 0, 0, 0, 0, 4067, 0, 1, 0, 0, 911, 0, 0, 1, 0, 12, 0, 0, 2054, 1, 827, 0, 0, 2, 0, 0, 11, 6, 0, 3392, 0, 4095, 0, 0, 0, 0, 0, 2242, 0, 3441, 0, 2, 0, 0, 0, 955, 3, 0, 756] payload_rows_per_dst=[8188, 3672, 8136, 5130, 7790, 7170, 4484, 7552] dup_rows_per_dst=[180, 536, 0, 1240, 5380, 3316, 0, 2762] combine_rows_per_src=[8188, 3672, 8136, 5130, 7790, 7170, 4484, 7552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57218 payload_rows_remote=49036 dup_rows=8318 dup_rows_remote=8316 dedup_ratio=0.126923 router_tokens_per_expert=[0, 1, 2, 0, 61, 0, 0, 0, 38, 0, 0, 4093, 3486, 1691, 403, 3, 0, 4, 0, 0, 4087, 1, 0, 0, 4, 370, 0, 0, 1, 0, 47, 1, 0, 1693, 0, 347, 0, 1, 2, 0, 1, 5, 0, 1, 3552, 0, 4091, 0, 4, 1, 0, 0, 3305, 0, 3412, 0, 1, 2, 0, 0, 1146, 0, 0, 911] payload_rows_per_dst=[8190, 7796, 8182, 8190, 3882, 7116, 6752, 7110] dup_rows_per_dst=[124, 3446, 2, 838, 204, 2, 886, 2816] combine_rows_per_src=[8190, 7796, 8182, 8190, 3882, 7116, 6752, 7110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54362 payload_rows_remote=46248 dup_rows=11174 dup_rows_remote=7864 dedup_ratio=0.170502 router_tokens_per_expert=[0, 0, 0, 1, 158, 2, 0, 0, 28, 0, 0, 4095, 3843, 1224, 617, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 669, 0, 2, 0, 0, 348, 0, 1, 2021, 0, 661, 0, 0, 1, 0, 0, 0, 0, 0, 2803, 0, 4095, 0, 0, 0, 0, 0, 3186, 0, 3175, 0, 0, 0, 0, 0, 1294, 0, 0, 524] payload_rows_per_dst=[8192, 8114, 8040, 8192, 4848, 5606, 7944, 3426] dup_rows_per_dst=[320, 3310, 0, 2036, 520, 0, 4778, 210] combine_rows_per_src=[8192, 8114, 8040, 8192, 4848, 5606, 7944, 3426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=339562496 dispatch_meta_bytes=223384 combine_payload_bytes=339562496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339785880 ingress_bytes=449662976 link_bytes=449662976 token_bytes=679348376 weight_movement_bytes=110100480 total_nvlink_bytes=789448856 weight_to_token_ratio=0.162068 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.820378721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47972 payload_rows_remote=40684 dup_rows=17564 dup_rows_remote=15520 dedup_ratio=0.268005 router_tokens_per_expert=[0, 0, 0, 1, 31, 7, 0, 0, 16, 3, 1, 4094, 3758, 1495, 381, 0, 0, 0, 0, 0, 4084, 1, 2, 0, 0, 601, 0, 24, 1, 4, 4, 0, 0, 1242, 0, 954, 0, 0, 1, 5, 1, 1, 1, 0, 3868, 0, 4094, 0, 0, 0, 0, 0, 3424, 3, 3378, 0, 0, 0, 0, 0, 755, 0, 0, 533] payload_rows_per_dst=[6038, 4746, 8172, 1230, 4766, 8188, 7544, 7288] dup_rows_per_dst=[70, 1204, 2, 38, 994, 10340, 2872, 2044] combine_rows_per_src=[6038, 4746, 8172, 1230, 4766, 8188, 7544, 7288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56620 payload_rows_remote=49058 dup_rows=8916 dup_rows_remote=5750 dedup_ratio=0.136047 router_tokens_per_expert=[0, 2, 0, 0, 29, 0, 0, 0, 35, 1, 0, 4094, 3423, 2024, 434, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 493, 1, 0, 0, 0, 21, 0, 0, 1636, 0, 301, 0, 2, 0, 2, 0, 1, 0, 1, 3170, 0, 4094, 0, 0, 0, 0, 1, 2998, 0, 3431, 0, 0, 0, 2, 0, 1755, 1, 0, 762] payload_rows_per_dst=[8192, 4470, 8108, 8190, 7562, 6344, 5998, 7756] dup_rows_per_dst=[58, 518, 0, 1028, 3166, 0, 0, 4146] combine_rows_per_src=[8192, 4470, 8108, 8190, 7562, 6344, 5998, 7756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55182 payload_rows_remote=48470 dup_rows=10354 dup_rows_remote=10354 dedup_ratio=0.157990 router_tokens_per_expert=[0, 0, 0, 17, 402, 0, 0, 0, 12, 0, 0, 4096, 3716, 1348, 697, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 740, 0, 0, 0, 1, 44, 0, 0, 2414, 0, 526, 0, 3, 0, 14, 0, 57, 0, 0, 3299, 0, 4075, 0, 0, 0, 0, 0, 2680, 0, 3182, 0, 0, 0, 29, 0, 916, 0, 0, 421] payload_rows_per_dst=[8192, 3646, 8158, 8152, 7806, 6712, 5360, 7156] dup_rows_per_dst=[838, 468, 0, 1568, 5540, 0, 0, 1940] combine_rows_per_src=[8192, 3646, 8158, 8152, 7806, 6712, 5360, 7156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=341469184 dispatch_meta_bytes=244208 combine_payload_bytes=341469184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=341713392 ingress_bytes=561670144 link_bytes=561670144 token_bytes=683182576 weight_movement_bytes=220200960 total_nvlink_bytes=903383536 weight_to_token_ratio=0.322316 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.768367593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351490048 dispatch_meta_bytes=229408 combine_payload_bytes=351490048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351719456 ingress_bytes=461590528 link_bytes=461590528 token_bytes=703209504 weight_movement_bytes=110100480 total_nvlink_bytes=813309984 weight_to_token_ratio=0.156569 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.027818468e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52864 payload_rows_remote=44684 dup_rows=12672 dup_rows_remote=12362 dedup_ratio=0.193359 router_tokens_per_expert=[0, 0, 0, 62, 87, 48, 1, 0, 316, 5, 0, 4047, 3518, 2117, 486, 0, 0, 0, 0, 0, 3808, 1, 2, 2, 0, 795, 5, 4, 0, 0, 117, 0, 0, 1789, 2, 672, 0, 0, 5, 48, 3, 11, 0, 0, 3115, 0, 4086, 0, 0, 0, 17, 0, 2949, 0, 3541, 0, 0, 0, 7, 13, 587, 29, 1, 472] payload_rows_per_dst=[8180, 7986, 7624, 8190, 4526, 6256, 7998, 2104] dup_rows_per_dst=[310, 4898, 2, 1824, 506, 2, 5016, 114] combine_rows_per_src=[8180, 7986, 7624, 8190, 4526, 6256, 7998, 2104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=331505664 dispatch_meta_bytes=216448 combine_payload_bytes=331505664 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=331722112 ingress_bytes=882008064 link_bytes=882008064 token_bytes=663227776 weight_movement_bytes=550502400 total_nvlink_bytes=1213730176 weight_to_token_ratio=0.830035 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.533957089e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291622912 dispatch_meta_bytes=224816 combine_payload_bytes=291622912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291847728 ingress_bytes=401723392 link_bytes=401723392 token_bytes=583470640 weight_movement_bytes=110100480 total_nvlink_bytes=693571120 weight_to_token_ratio=0.188699 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.986630508e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351647744 dispatch_meta_bytes=219232 combine_payload_bytes=351647744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351866976 ingress_bytes=461748224 link_bytes=461748224 token_bytes=703514720 weight_movement_bytes=110100480 total_nvlink_bytes=813615200 weight_to_token_ratio=0.156501 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.030561061e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=347432960 dispatch_meta_bytes=235296 combine_payload_bytes=347432960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=347668256 ingress_bytes=567633920 link_bytes=567633920 token_bytes=695101216 weight_movement_bytes=220200960 total_nvlink_bytes=915302176 weight_to_token_ratio=0.316790 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.872087466e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=320294912 dispatch_meta_bytes=228184 combine_payload_bytes=320294912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320523096 ingress_bytes=430395392 link_bytes=430395392 token_bytes=640818008 weight_movement_bytes=110100480 total_nvlink_bytes=750918488 weight_to_token_ratio=0.171812 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.485283746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64780 payload_rows_remote=56966 dup_rows=756 dup_rows_remote=756 dedup_ratio=0.011536 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 3907, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 2609, 1794, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7858, 7814, 8192, 8148, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 696, 56, 0] combine_rows_per_src=[8192, 8192, 7858, 7814, 8192, 8148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64442 payload_rows_remote=56250 dup_rows=1094 dup_rows_remote=1058 dedup_ratio=0.016693 router_tokens_per_expert=[1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3969, 3688, 0, 4094, 1, 0, 3, 2, 4094, 1, 4094, 0, 0, 0, 0, 2, 2, 22, 0, 0, 0, 2304, 2278, 1, 0, 4095, 0, 0, 16, 2, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 7940, 7378, 8190, 8176, 8192, 8186] dup_rows_per_dst=[6, 0, 0, 10, 8, 1034, 36, 0] combine_rows_per_src=[8192, 8188, 7940, 7378, 8190, 8176, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64952 payload_rows_remote=56778 dup_rows=584 dup_rows_remote=584 dedup_ratio=0.008911 router_tokens_per_expert=[0, 2, 0, 1, 4, 0, 0, 0, 7, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 3869, 0, 4091, 0, 0, 1, 0, 4093, 0, 4090, 2, 2, 3, 1, 0, 1, 11, 0, 1, 0, 2612, 1664, 0, 0, 4091, 0, 0, 39, 0, 1, 2, 0, 4, 4081, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8174, 7740, 8190, 8108, 8186, 8172] dup_rows_per_dst=[6, 22, 0, 0, 8, 468, 80, 0] combine_rows_per_src=[8190, 8192, 8174, 7740, 8190, 8108, 8186, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64850 payload_rows_remote=56658 dup_rows=686 dup_rows_remote=686 dedup_ratio=0.010468 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4074, 3825, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2687, 1614, 0, 0, 4096, 0, 0, 53, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8148, 7652, 8190, 8092, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 572, 106, 0] combine_rows_per_src=[8192, 8192, 8148, 7652, 8190, 8092, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=408332288 dispatch_meta_bytes=230888 combine_payload_bytes=408332288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=408563176 ingress_bytes=738633728 link_bytes=738633728 token_bytes=816895464 weight_movement_bytes=330301440 total_nvlink_bytes=1147196904 weight_to_token_ratio=0.404337 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.284605537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64184 payload_rows_remote=56038 dup_rows=1352 dup_rows_remote=1028 dedup_ratio=0.020630 router_tokens_per_expert=[0, 11, 0, 0, 0, 0, 0, 1, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4006, 3840, 0, 4096, 1, 1, 0, 0, 4093, 0, 4094, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 2583, 1837, 1, 0, 4093, 0, 0, 10, 0, 0, 0, 1, 0, 4071, 2, 0, 1, 0, 0, 0] payload_rows_per_dst=[8148, 7958, 8016, 7698, 8180, 8158, 7880, 8146] dup_rows_per_dst=[24, 40, 0, 260, 0, 686, 18, 324] combine_rows_per_src=[8148, 7958, 8016, 7698, 8180, 8158, 7880, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64946 payload_rows_remote=56754 dup_rows=590 dup_rows_remote=588 dedup_ratio=0.009003 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3899, 0, 4095, 1, 0, 0, 0, 4094, 0, 4095, 1, 0, 0, 1, 0, 0, 4, 0, 0, 0, 2265, 1937, 0, 0, 4095, 0, 0, 95, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8180, 7798, 8192, 8018, 8190, 8188] dup_rows_per_dst=[2, 0, 0, 2, 2, 394, 190, 0] combine_rows_per_src=[8190, 8190, 8180, 7798, 8192, 8018, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=403200000 dispatch_meta_bytes=229232 combine_payload_bytes=403200000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403429232 ingress_bytes=733501440 link_bytes=733501440 token_bytes=806629232 weight_movement_bytes=330301440 total_nvlink_bytes=1136930672 weight_to_token_ratio=0.409484 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275679644e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64412 payload_rows_remote=56244 dup_rows=1124 dup_rows_remote=66 dedup_ratio=0.017151 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 3751, 0, 4096, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 11, 0, 0, 6, 0, 5, 0, 2938, 1664, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7864, 7502, 8188, 8168, 8192, 8114] dup_rows_per_dst=[0, 18, 0, 0, 4, 1058, 44, 0] combine_rows_per_src=[8192, 8192, 7864, 7502, 8188, 8168, 8192, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=406984704 dispatch_meta_bytes=229448 combine_payload_bytes=406984704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407214152 ingress_bytes=517085184 link_bytes=517085184 token_bytes=814198856 weight_movement_bytes=110100480 total_nvlink_bytes=924299336 weight_to_token_ratio=0.135226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.992961809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60356 payload_rows_remote=52164 dup_rows=5180 dup_rows_remote=5174 dedup_ratio=0.079041 router_tokens_per_expert=[0, 1, 0, 2, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4043, 3925, 0, 4096, 11, 0, 1, 36, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 59, 0, 27, 0, 2480, 1668, 0, 0, 4092, 0, 0, 16, 0, 0, 0, 9, 11, 4000, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8100, 8086, 8192, 3612, 7966, 8016] dup_rows_per_dst=[6, 160, 1248, 3596, 0, 148, 16, 6] combine_rows_per_src=[8192, 8192, 8100, 8086, 8192, 3612, 7966, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=406124544 dispatch_meta_bytes=229376 combine_payload_bytes=406124544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406353920 ingress_bytes=516225024 link_bytes=516225024 token_bytes=812478464 weight_movement_bytes=110100480 total_nvlink_bytes=922578944 weight_to_token_ratio=0.135512 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.978002212e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=401680384 dispatch_meta_bytes=228264 combine_payload_bytes=401680384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401908648 ingress_bytes=511780864 link_bytes=511780864 token_bytes=803589032 weight_movement_bytes=110100480 total_nvlink_bytes=913689512 weight_to_token_ratio=0.137011 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.900710960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=406812672 dispatch_meta_bytes=229368 combine_payload_bytes=406812672 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=407042040 ingress_bytes=516913152 link_bytes=516913152 token_bytes=813854712 weight_movement_bytes=110100480 total_nvlink_bytes=923955192 weight_to_token_ratio=0.135283 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.989969889e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=403156992 dispatch_meta_bytes=225240 combine_payload_bytes=403156992 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403382232 ingress_bytes=733458432 link_bytes=733458432 token_bytes=806539224 weight_movement_bytes=330301440 total_nvlink_bytes=1136840664 weight_to_token_ratio=0.409529 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275604846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=373911552 dispatch_meta_bytes=229352 combine_payload_bytes=373911552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374140904 ingress_bytes=484012032 link_bytes=484012032 token_bytes=748052456 weight_movement_bytes=110100480 total_nvlink_bytes=858152936 weight_to_token_ratio=0.147183 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.417765299e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65262 payload_rows_remote=57072 dup_rows=274 dup_rows_remote=274 dedup_ratio=0.004181 router_tokens_per_expert=[1, 1, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 56, 0, 0, 0, 10, 0, 0, 4088, 1, 4, 4095, 0, 3968, 0, 0, 0, 56, 0, 0, 0, 4094, 0, 15, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 7938, 8188, 8190, 8190, 8192] dup_rows_per_dst=[2, 0, 116, 8, 112, 32, 0, 4] combine_rows_per_src=[8192, 8180, 8192, 7938, 8188, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65220 payload_rows_remote=57290 dup_rows=316 dup_rows_remote=316 dedup_ratio=0.004822 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 3965, 0, 0, 0, 120, 0, 0, 0, 4096, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8140, 7930, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 74, 0, 240, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8140, 7930, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65228 payload_rows_remote=57062 dup_rows=308 dup_rows_remote=224 dedup_ratio=0.004700 router_tokens_per_expert=[0, 0, 2, 4094, 5, 0, 4094, 0, 0, 0, 0, 4086, 0, 2, 0, 2, 0, 0, 56, 3, 0, 0, 0, 0, 0, 4066, 0, 2, 4091, 0, 3966, 0, 0, 2, 71, 0, 0, 0, 4091, 0, 34, 0, 2, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8180, 8166, 7934, 8186, 8192, 8188, 8190] dup_rows_per_dst=[10, 0, 84, 2, 142, 64, 2, 4] combine_rows_per_src=[8192, 8180, 8166, 7934, 8186, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65266 payload_rows_remote=57080 dup_rows=270 dup_rows_remote=270 dedup_ratio=0.004120 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 3966, 0, 0, 0, 54, 0, 0, 0, 4095, 0, 62, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8188, 7934, 8190, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 32, 0, 108, 124, 0, 4] combine_rows_per_src=[8192, 8186, 8188, 7934, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64690 payload_rows_remote=56498 dup_rows=846 dup_rows_remote=842 dedup_ratio=0.012909 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 2, 4093, 0, 0, 1, 0, 0, 0, 29, 2, 0, 0, 1, 0, 0, 4090, 0, 8, 4092, 0, 4005, 0, 0, 0, 32, 0, 0, 0, 4094, 0, 29, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 2, 1] payload_rows_per_dst=[8058, 8188, 7662, 8026, 8188, 8188, 8188, 8192] dup_rows_per_dst=[0, 4, 56, 504, 64, 192, 22, 4] combine_rows_per_src=[8058, 8188, 7662, 8026, 8188, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409092096 dispatch_meta_bytes=229384 combine_payload_bytes=409092096 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409321480 ingress_bytes=629293056 link_bytes=629293056 token_bytes=818413576 weight_movement_bytes=220200960 total_nvlink_bytes=1038614536 weight_to_token_ratio=0.269058 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094444125e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=92 dedup_ratio=0.003632 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 23, 1, 0, 1, 2, 0, 0, 4090, 0, 0, 4094, 0, 3984, 0, 0, 0, 73, 0, 0, 0, 4096, 0, 22, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8190, 8186, 7968, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 48, 0, 146, 42, 0, 2] combine_rows_per_src=[8190, 8190, 8186, 7968, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=410654720 dispatch_meta_bytes=230424 combine_payload_bytes=410654720 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410885144 ingress_bytes=851056640 link_bytes=851056640 token_bytes=821539864 weight_movement_bytes=440401920 total_nvlink_bytes=1261941784 weight_to_token_ratio=0.536069 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.480127472e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409020416 dispatch_meta_bytes=229144 combine_payload_bytes=409020416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409249560 ingress_bytes=519120896 link_bytes=519120896 token_bytes=818269976 weight_movement_bytes=110100480 total_nvlink_bytes=928370456 weight_to_token_ratio=0.134553 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.028366189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65432 payload_rows_remote=57240 dup_rows=104 dup_rows_remote=102 dedup_ratio=0.001587 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 26, 0, 0, 5, 0, 0, 0, 4068, 0, 0, 4096, 0, 4055, 0, 0, 0, 43, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8182, 8110, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 16, 0, 86, 2, 0, 0] combine_rows_per_src=[8192, 8180, 8182, 8110, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63746 payload_rows_remote=55554 dup_rows=1790 dup_rows_remote=1668 dedup_ratio=0.027313 router_tokens_per_expert=[0, 32, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 199, 0, 0, 3993, 0, 32, 4091, 0, 3879, 0, 0, 0, 80, 0, 0, 0, 4096, 0, 212, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8142, 8170, 8034, 7232, 7654, 8182, 8140] dup_rows_per_dst=[122, 94, 224, 1148, 78, 88, 0, 36] combine_rows_per_src=[8192, 8142, 8170, 8034, 7232, 7654, 8182, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409149440 dispatch_meta_bytes=229400 combine_payload_bytes=409149440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409378840 ingress_bytes=519249920 link_bytes=519249920 token_bytes=818528280 weight_movement_bytes=110100480 total_nvlink_bytes=928628760 weight_to_token_ratio=0.134510 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.030610128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404977664 dispatch_meta_bytes=229360 combine_payload_bytes=404977664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405207024 ingress_bytes=515078144 link_bytes=515078144 token_bytes=810184688 weight_movement_bytes=110100480 total_nvlink_bytes=920285168 weight_to_token_ratio=0.135896 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.958056082e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409335808 dispatch_meta_bytes=228792 combine_payload_bytes=409335808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=409564600 ingress_bytes=629536768 link_bytes=629536768 token_bytes=818900408 weight_movement_bytes=220200960 total_nvlink_bytes=1039101368 weight_to_token_ratio=0.268898 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094867981e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=410296320 dispatch_meta_bytes=229368 combine_payload_bytes=410296320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410525688 ingress_bytes=740597760 link_bytes=740597760 token_bytes=820822008 weight_movement_bytes=330301440 total_nvlink_bytes=1151123448 weight_to_token_ratio=0.402403 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288021312e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=398211072 dispatch_meta_bytes=228888 combine_payload_bytes=398211072 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=398439960 ingress_bytes=618412032 link_bytes=618412032 token_bytes=796651032 weight_movement_bytes=220200960 total_nvlink_bytes=1016851992 weight_to_token_ratio=0.276408 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.075520235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57408 payload_rows_remote=54790 dup_rows=8128 dup_rows_remote=7994 dedup_ratio=0.124023 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 1, 1, 2950, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 388, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 466, 0, 870, 0, 0, 0, 40, 0, 1, 4079, 3623, 0, 0, 1, 0, 4093] payload_rows_per_dst=[8190, 5904, 8192, 7934, 8190, 8188, 2618, 8192] dup_rows_per_dst=[0, 0, 0, 0, 776, 2, 134, 7216] combine_rows_per_src=[8190, 5904, 8192, 7934, 8190, 8188, 2618, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57252 payload_rows_remote=49064 dup_rows=8284 dup_rows_remote=8278 dedup_ratio=0.126404 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 3, 0, 2582, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4093, 0, 2, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 1058, 1, 0, 0, 13, 2, 0, 4089, 3717, 1, 0, 0, 1, 4096] payload_rows_per_dst=[8188, 7926, 8188, 7814, 8190, 8192, 8188, 566] dup_rows_per_dst=[0, 4110, 6, 10, 774, 4, 3380, 0] combine_rows_per_src=[8188, 7926, 8188, 7814, 8190, 8192, 8188, 566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60960 payload_rows_remote=53058 dup_rows=4576 dup_rows_remote=4576 dedup_ratio=0.069824 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 663, 0, 0, 0, 31, 0, 0, 4096, 3474, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 5150, 8192, 7902, 8192, 8192, 8192, 6948] dup_rows_per_dst=[0, 0, 0, 0, 2378, 0, 2198, 0] combine_rows_per_src=[8192, 5150, 8192, 7902, 8192, 8192, 8192, 6948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56440 payload_rows_remote=48300 dup_rows=9096 dup_rows_remote=4300 dedup_ratio=0.138794 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 2635, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 451, 0, 0, 0, 76, 0, 0, 4095, 3832, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8140, 8192, 7342, 8192, 8192, 8192, 0] dup_rows_per_dst=[0, 4796, 0, 0, 1610, 0, 2690, 0] combine_rows_per_src=[8190, 8140, 8192, 7342, 8192, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392734720 dispatch_meta_bytes=251136 combine_payload_bytes=392734720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392985856 ingress_bytes=502835200 link_bytes=502835200 token_bytes=785720576 weight_movement_bytes=110100480 total_nvlink_bytes=895821056 weight_to_token_ratio=0.140127 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.745131150e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53412 payload_rows_remote=45222 dup_rows=12124 dup_rows_remote=454 dedup_ratio=0.184998 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 3216, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 3912, 0, 0, 1, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 163, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 612, 0, 929, 1, 0, 0, 87, 0, 0, 4058, 3403, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 6434, 8074, 7826, 3432, 8192, 3076, 8190] dup_rows_per_dst=[0, 2, 4, 118, 148, 0, 182, 11670] combine_rows_per_src=[8188, 6434, 8074, 7826, 3432, 8192, 3076, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61724 payload_rows_remote=53532 dup_rows=3812 dup_rows_remote=3558 dedup_ratio=0.058167 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 1143, 0, 0, 0, 3, 0, 0, 4091, 3881, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 5250, 8192, 7758, 8192, 8190, 8190, 7762] dup_rows_per_dst=[0, 0, 8, 0, 254, 0, 3550, 0] combine_rows_per_src=[8190, 5250, 8192, 7758, 8192, 8190, 8190, 7762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57622 payload_rows_remote=49430 dup_rows=7914 dup_rows_remote=7914 dedup_ratio=0.120758 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 909, 0, 0, 0, 4, 0, 0, 4096, 3697, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8182, 5800, 8192, 7998, 8192, 8192, 2904, 8162] dup_rows_per_dst=[0, 0, 0, 0, 1000, 0, 84, 6830] combine_rows_per_src=[8182, 5800, 8192, 7998, 8192, 8192, 2904, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351690752 dispatch_meta_bytes=229368 combine_payload_bytes=351690752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351920120 ingress_bytes=461791232 link_bytes=461791232 token_bytes=703610872 weight_movement_bytes=110100480 total_nvlink_bytes=813711352 weight_to_token_ratio=0.156479 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.031309041e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380319744 dispatch_meta_bytes=230536 combine_payload_bytes=380319744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380550280 ingress_bytes=600520704 link_bytes=600520704 token_bytes=760870024 weight_movement_bytes=220200960 total_nvlink_bytes=981070984 weight_to_token_ratio=0.289407 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044404273e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54302 payload_rows_remote=46110 dup_rows=11234 dup_rows_remote=11210 dedup_ratio=0.171417 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 2449, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 467, 0, 0, 0, 402, 0, 0, 3959, 3562, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8004, 8192, 8030, 5510, 8192, 8182, 0] dup_rows_per_dst=[24, 4020, 104, 2804, 506, 0, 3776, 0] combine_rows_per_src=[8192, 8004, 8192, 8030, 5510, 8192, 8182, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=346214400 dispatch_meta_bytes=210400 combine_payload_bytes=346214400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346424800 ingress_bytes=456314880 link_bytes=456314880 token_bytes=692639200 weight_movement_bytes=110100480 total_nvlink_bytes=802739680 weight_to_token_ratio=0.158958 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.936066272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=324151296 dispatch_meta_bytes=182704 combine_payload_bytes=324151296 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=324334000 ingress_bytes=874653696 link_bytes=874653696 token_bytes=648485296 weight_movement_bytes=550502400 total_nvlink_bytes=1198987696 weight_to_token_ratio=0.848905 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.521166633e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383717376 dispatch_meta_bytes=228360 combine_payload_bytes=383717376 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=383945736 ingress_bytes=714018816 link_bytes=714018816 token_bytes=767663112 weight_movement_bytes=330301440 total_nvlink_bytes=1097964552 weight_to_token_ratio=0.430269 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.241796157e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354314240 dispatch_meta_bytes=229376 combine_payload_bytes=354314240 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=354543616 ingress_bytes=574515200 link_bytes=574515200 token_bytes=708857856 weight_movement_bytes=220200960 total_nvlink_bytes=929058816 weight_to_token_ratio=0.310642 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.991764243e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=330516480 dispatch_meta_bytes=229280 combine_payload_bytes=330516480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330745760 ingress_bytes=440616960 link_bytes=440616960 token_bytes=661262240 weight_movement_bytes=110100480 total_nvlink_bytes=771362720 weight_to_token_ratio=0.166500 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.663053625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63990 payload_rows_remote=55800 dup_rows=1546 dup_rows_remote=582 dedup_ratio=0.023590 router_tokens_per_expert=[4071, 23, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4059, 1, 4095, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 227, 0, 0, 4095, 0, 0, 2287, 0, 1138, 0, 1, 2, 0, 0, 4095, 1, 479, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0] payload_rows_per_dst=[8142, 8178, 8120, 8190, 8190, 6788, 8190, 8192] dup_rows_per_dst=[46, 0, 4, 10, 454, 64, 964, 4] combine_rows_per_src=[8142, 8178, 8120, 8190, 8190, 6788, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64510 payload_rows_remote=56372 dup_rows=1026 dup_rows_remote=1022 dedup_ratio=0.015656 router_tokens_per_expert=[4074, 6, 4091, 0, 0, 4, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 4062, 0, 4094, 0, 0, 0, 0, 2, 8, 3, 0, 0, 0, 0, 0, 94, 0, 0, 4096, 0, 0, 2446, 0, 1196, 1, 0, 5, 4, 0, 4093, 0, 379, 1, 0, 0, 1, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8188, 8138, 8192, 8192, 7264, 8192, 8188] dup_rows_per_dst=[14, 4, 4, 22, 188, 22, 772, 0] combine_rows_per_src=[8156, 8188, 8138, 8192, 8192, 7264, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64278 payload_rows_remote=56086 dup_rows=1258 dup_rows_remote=1252 dedup_ratio=0.019196 router_tokens_per_expert=[4052, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 217, 0, 0, 4096, 0, 0, 2568, 0, 1073, 0, 0, 0, 0, 0, 4096, 0, 348, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8192, 8052, 8192, 8192, 7162, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 6, 434, 120, 696, 0] combine_rows_per_src=[8104, 8192, 8052, 8192, 8192, 7162, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63662 payload_rows_remote=55470 dup_rows=1874 dup_rows_remote=1874 dedup_ratio=0.028595 router_tokens_per_expert=[3981, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 74, 0, 0, 4096, 0, 0, 3077, 0, 434, 0, 0, 0, 0, 0, 4096, 0, 805, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[7962, 8192, 7818, 8192, 8192, 6922, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 2, 148, 100, 1610, 0] combine_rows_per_src=[7962, 8192, 7818, 8192, 8192, 6922, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=399974400 dispatch_meta_bytes=225528 combine_payload_bytes=399974400 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=400199928 ingress_bytes=840376320 link_bytes=840376320 token_bytes=800174328 weight_movement_bytes=440401920 total_nvlink_bytes=1240576248 weight_to_token_ratio=0.550382 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.461552639e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62102 payload_rows_remote=53910 dup_rows=3434 dup_rows_remote=3430 dedup_ratio=0.052399 router_tokens_per_expert=[4071, 9, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4094, 0, 0, 2320, 2, 1210, 1, 0, 12, 0, 0, 4094, 0, 498, 1, 0, 1, 0, 0, 4094, 0, 0, 2, 1] payload_rows_per_dst=[7908, 8170, 8102, 8090, 6000, 7448, 8192, 8192] dup_rows_per_dst=[18, 234, 0, 22, 88, 1948, 1120, 4] combine_rows_per_src=[7908, 8170, 8102, 8090, 6000, 7448, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63288 payload_rows_remote=57086 dup_rows=2248 dup_rows_remote=2054 dedup_ratio=0.034302 router_tokens_per_expert=[4075, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3981, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 109, 0, 0, 4096, 0, 0, 2140, 0, 1058, 0, 0, 4, 0, 0, 4096, 0, 905, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8192, 7976, 8192, 8192, 6202, 8192, 8192] dup_rows_per_dst=[14, 0, 2, 2, 218, 194, 1818, 0] combine_rows_per_src=[8150, 8192, 7976, 8192, 8192, 6202, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404074496 dispatch_meta_bytes=229576 combine_payload_bytes=404074496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404304072 ingress_bytes=514174976 link_bytes=514174976 token_bytes=808378568 weight_movement_bytes=110100480 total_nvlink_bytes=918479048 weight_to_token_ratio=0.136199 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.942348505e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64726 payload_rows_remote=56534 dup_rows=810 dup_rows_remote=316 dedup_ratio=0.012360 router_tokens_per_expert=[4072, 4, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 4096, 0, 0, 2719, 0, 1161, 0, 0, 2, 1, 0, 4096, 0, 58, 0, 2, 0, 0, 0, 4096, 0, 0, 1, 0] payload_rows_per_dst=[8144, 8188, 8046, 8192, 8192, 7580, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 0, 494, 180, 126, 2] combine_rows_per_src=[8144, 8188, 8046, 8192, 8192, 7580, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=402024448 dispatch_meta_bytes=229352 combine_payload_bytes=402024448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402253800 ingress_bytes=622225408 link_bytes=622225408 token_bytes=804278248 weight_movement_bytes=220200960 total_nvlink_bytes=1024479208 weight_to_token_ratio=0.273787 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082152323e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397608960 dispatch_meta_bytes=229376 combine_payload_bytes=397608960 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=397838336 ingress_bytes=617809920 link_bytes=617809920 token_bytes=795447296 weight_movement_bytes=220200960 total_nvlink_bytes=1015648256 weight_to_token_ratio=0.276827 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074473063e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=386426880 dispatch_meta_bytes=229360 combine_payload_bytes=386426880 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=386656240 ingress_bytes=496527360 link_bytes=496527360 token_bytes=773083120 weight_movement_bytes=110100480 total_nvlink_bytes=883183600 weight_to_token_ratio=0.142417 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.635427438e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57452 payload_rows_remote=49302 dup_rows=8084 dup_rows_remote=7380 dedup_ratio=0.123352 router_tokens_per_expert=[3907, 12, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 0, 0, 3008, 0, 1030, 0, 0, 44, 0, 0, 4096, 6, 491, 0, 2, 0, 0, 0, 4096, 0, 0, 13, 0] payload_rows_per_dst=[8150, 7998, 7598, 8192, 8192, 8120, 1030, 8172] dup_rows_per_dst=[704, 0, 1676, 40, 170, 5422, 58, 14] combine_rows_per_src=[8150, 7998, 7598, 8192, 8192, 8120, 1030, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=409192448 dispatch_meta_bytes=236560 combine_payload_bytes=409192448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=409429008 ingress_bytes=629393408 link_bytes=629393408 token_bytes=818621456 weight_movement_bytes=220200960 total_nvlink_bytes=1038822416 weight_to_token_ratio=0.268990 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094618654e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405235712 dispatch_meta_bytes=227400 combine_payload_bytes=405235712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405463112 ingress_bytes=515336192 link_bytes=515336192 token_bytes=810698824 weight_movement_bytes=110100480 total_nvlink_bytes=920799304 weight_to_token_ratio=0.135809 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.962543961e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59814 payload_rows_remote=51654 dup_rows=5722 dup_rows_remote=5692 dedup_ratio=0.087311 router_tokens_per_expert=[0, 0, 4095, 0, 1985, 0, 0, 425, 4095, 0, 2627, 1, 0, 690, 0, 0, 0, 0, 4079, 1, 1, 0, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 13, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[4550, 6148, 8192, 8190, 8192, 8190, 8160, 8192] dup_rows_per_dst=[270, 488, 4902, 0, 30, 0, 30, 2] combine_rows_per_src=[4550, 6148, 8192, 8190, 8192, 8190, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=353396736 dispatch_meta_bytes=226728 combine_payload_bytes=353396736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=353623464 ingress_bytes=683698176 link_bytes=683698176 token_bytes=707020200 weight_movement_bytes=330301440 total_nvlink_bytes=1037321640 weight_to_token_ratio=0.467174 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.189063577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59994 payload_rows_remote=54970 dup_rows=5542 dup_rows_remote=5542 dedup_ratio=0.084564 router_tokens_per_expert=[0, 0, 4096, 0, 2707, 1, 0, 373, 4095, 0, 2146, 0, 0, 409, 2, 0, 2, 0, 4073, 0, 0, 0, 2510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[6942, 7124, 5024, 8188, 8192, 8184, 8148, 8192] dup_rows_per_dst=[2272, 3128, 0, 0, 104, 0, 34, 4] combine_rows_per_src=[6942, 7124, 5024, 8188, 8192, 8184, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60504 payload_rows_remote=54144 dup_rows=5032 dup_rows_remote=4906 dedup_ratio=0.076782 router_tokens_per_expert=[0, 0, 4096, 0, 2313, 0, 0, 118, 4096, 0, 3065, 0, 0, 178, 0, 0, 0, 0, 4092, 0, 0, 0, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6360, 5016, 8192, 8192, 8176, 8184, 8192] dup_rows_per_dst=[4862, 126, 0, 0, 24, 0, 20, 0] combine_rows_per_src=[8192, 6360, 5016, 8192, 8192, 8176, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58806 payload_rows_remote=50614 dup_rows=6730 dup_rows_remote=6730 dedup_ratio=0.102692 router_tokens_per_expert=[0, 0, 4096, 0, 2007, 0, 0, 149, 4096, 0, 2884, 0, 0, 361, 0, 0, 0, 0, 4095, 0, 0, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4210, 8192, 5448, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[102, 6490, 0, 0, 22, 0, 116, 0] combine_rows_per_src=[4210, 8192, 5448, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=370255872 dispatch_meta_bytes=229384 combine_payload_bytes=370255872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370485256 ingress_bytes=480356352 link_bytes=480356352 token_bytes=740741128 weight_movement_bytes=110100480 total_nvlink_bytes=850841608 weight_to_token_ratio=0.148636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.354187012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58914 payload_rows_remote=50722 dup_rows=6622 dup_rows_remote=6608 dedup_ratio=0.101044 router_tokens_per_expert=[0, 0, 4094, 0, 3400, 0, 0, 53, 4095, 0, 800, 0, 0, 929, 0, 0, 0, 0, 4090, 2, 0, 1, 2967, 0, 0, 2, 0, 0, 1, 0, 1, 0, 0, 20, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4067, 0, 1, 4094, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2] payload_rows_per_dst=[6830, 3364, 8192, 8192, 7828, 8136, 8180, 8192] dup_rows_per_dst=[76, 94, 5938, 4, 40, 362, 94, 14] combine_rows_per_src=[6830, 3364, 8192, 8192, 7828, 8136, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59956 payload_rows_remote=51766 dup_rows=5580 dup_rows_remote=5580 dedup_ratio=0.085144 router_tokens_per_expert=[0, 0, 4096, 0, 3198, 0, 0, 177, 4096, 0, 2128, 0, 0, 396, 0, 0, 0, 0, 4096, 0, 0, 0, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6478, 5346, 7174, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[272, 1470, 3788, 0, 24, 0, 26, 0] combine_rows_per_src=[6478, 5346, 7174, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394024960 dispatch_meta_bytes=242048 combine_payload_bytes=394024960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394267008 ingress_bytes=614225920 link_bytes=614225920 token_bytes=788291968 weight_movement_bytes=220200960 total_nvlink_bytes=1008492928 weight_to_token_ratio=0.279339 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068239898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62234 payload_rows_remote=54042 dup_rows=3302 dup_rows_remote=3250 dedup_ratio=0.050385 router_tokens_per_expert=[0, 0, 4096, 0, 2966, 0, 0, 177, 4096, 0, 1207, 0, 0, 324, 0, 0, 0, 0, 4096, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[6206, 8192, 6876, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[80, 3062, 0, 0, 52, 0, 106, 2] combine_rows_per_src=[6206, 8192, 6876, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388104192 dispatch_meta_bytes=236200 combine_payload_bytes=388104192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=388340392 ingress_bytes=718405632 link_bytes=718405632 token_bytes=776444584 weight_movement_bytes=330301440 total_nvlink_bytes=1106746024 weight_to_token_ratio=0.425402 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249425551e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=362801152 dispatch_meta_bytes=229376 combine_payload_bytes=362801152 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=363030528 ingress_bytes=583002112 link_bytes=583002112 token_bytes=725831680 weight_movement_bytes=220200960 total_nvlink_bytes=946032640 weight_to_token_ratio=0.303377 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.013936560e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58392 payload_rows_remote=50200 dup_rows=7144 dup_rows_remote=1142 dedup_ratio=0.109009 router_tokens_per_expert=[0, 0, 4096, 0, 2887, 0, 0, 21, 4096, 0, 2335, 0, 0, 355, 0, 0, 0, 0, 4084, 0, 0, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3959, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5152, 4940, 8192, 8080, 7916, 7770, 8150] dup_rows_per_dst=[6002, 800, 46, 2, 180, 0, 94, 20] combine_rows_per_src=[8192, 5152, 4940, 8192, 8080, 7916, 7770, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=363575296 dispatch_meta_bytes=229320 combine_payload_bytes=363575296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363804616 ingress_bytes=473675776 link_bytes=473675776 token_bytes=727379912 weight_movement_bytes=110100480 total_nvlink_bytes=837480392 weight_to_token_ratio=0.151366 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.238000807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371058688 dispatch_meta_bytes=229384 combine_payload_bytes=371058688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371288072 ingress_bytes=701360128 link_bytes=701360128 token_bytes=742346760 weight_movement_bytes=330301440 total_nvlink_bytes=1072648200 weight_to_token_ratio=0.444942 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.219780617e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=387373056 dispatch_meta_bytes=229168 combine_payload_bytes=387373056 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=387602224 ingress_bytes=607574016 link_bytes=607574016 token_bytes=774975280 weight_movement_bytes=220200960 total_nvlink_bytes=995176240 weight_to_token_ratio=0.284139 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056671143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62922 payload_rows_remote=54732 dup_rows=2614 dup_rows_remote=2612 dedup_ratio=0.039886 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 1, 4095, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 942, 0, 3401, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1028, 0, 1, 3205, 4094, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 7652, 6804, 7512, 8190, 8190, 8192] dup_rows_per_dst=[4, 2, 1490, 0, 1114, 0, 2, 2] combine_rows_per_src=[8192, 8190, 7652, 6804, 7512, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359833600 dispatch_meta_bytes=205368 combine_payload_bytes=359833600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=360038968 ingress_bytes=580034560 link_bytes=580034560 token_bytes=719872568 weight_movement_bytes=220200960 total_nvlink_bytes=940073528 weight_to_token_ratio=0.305889 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008775499e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60370 payload_rows_remote=52908 dup_rows=5166 dup_rows_remote=3632 dedup_ratio=0.078827 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 3447, 0, 0, 1050, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1992, 1, 2, 2944, 4091, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 7462, 7978, 3988, 8186, 8192, 8184] dup_rows_per_dst=[0, 4, 1534, 3614, 4, 0, 10, 0] combine_rows_per_src=[8190, 8190, 7462, 7978, 3988, 8186, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59276 payload_rows_remote=51084 dup_rows=6260 dup_rows_remote=6260 dedup_ratio=0.095520 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3091, 0, 0, 1646, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 0, 0, 2885, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7286, 7720, 3310, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2190, 4070, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7286, 7720, 3310, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61710 payload_rows_remote=55748 dup_rows=3826 dup_rows_remote=3812 dedup_ratio=0.058380 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3131, 0, 0, 1300, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 1, 3002, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 7204, 5962, 7584, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1658, 14, 2152, 0, 0, 2] combine_rows_per_src=[8192, 8192, 7204, 5962, 7584, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392318976 dispatch_meta_bytes=229376 combine_payload_bytes=392318976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392548352 ingress_bytes=502419456 link_bytes=502419456 token_bytes=784867328 weight_movement_bytes=110100480 total_nvlink_bytes=894967808 weight_to_token_ratio=0.140279 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.737900678e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56176 payload_rows_remote=47984 dup_rows=9360 dup_rows_remote=9358 dedup_ratio=0.142822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4074, 4094, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 3433, 0, 2, 495, 0, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1761, 0, 0, 3574, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 2, 0, 0, 4094, 0, 0, 1, 0] payload_rows_per_dst=[8188, 8150, 994, 6082, 8190, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 0, 9328, 20, 6, 2] combine_rows_per_src=[8188, 8150, 994, 6082, 8190, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62022 payload_rows_remote=53830 dup_rows=3514 dup_rows_remote=3514 dedup_ratio=0.053619 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 1374, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 2608, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 7632, 6680, 6760, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2272, 0, 1242, 0, 0, 0] combine_rows_per_src=[8192, 8182, 7632, 6680, 6760, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61870 payload_rows_remote=53828 dup_rows=3666 dup_rows_remote=976 dedup_ratio=0.055939 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3534, 0, 0, 643, 0, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 3126, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 7382, 5490, 8042, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 972, 0, 2690, 0, 4, 0] combine_rows_per_src=[8192, 8188, 7382, 5490, 8042, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=379244544 dispatch_meta_bytes=226160 combine_payload_bytes=379244544 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=379470704 ingress_bytes=819646464 link_bytes=819646464 token_bytes=758715248 weight_movement_bytes=440401920 total_nvlink_bytes=1199117168 weight_to_token_ratio=0.580457 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.425500010e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=366170112 dispatch_meta_bytes=229376 combine_payload_bytes=366170112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=366399488 ingress_bytes=586371072 link_bytes=586371072 token_bytes=732569600 weight_movement_bytes=220200960 total_nvlink_bytes=952770560 weight_to_token_ratio=0.300587 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.019795736e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399601664 dispatch_meta_bytes=238240 combine_payload_bytes=399601664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399839904 ingress_bytes=509702144 link_bytes=509702144 token_bytes=799441568 weight_movement_bytes=110100480 total_nvlink_bytes=909542048 weight_to_token_ratio=0.137722 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.864558600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60142 payload_rows_remote=51950 dup_rows=5394 dup_rows_remote=5392 dedup_ratio=0.082306 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3734, 0, 0, 1220, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 2416, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 7762, 7840, 3620, 8184, 8172, 8186] dup_rows_per_dst=[2, 60, 2042, 3282, 0, 0, 0, 8] combine_rows_per_src=[8192, 8186, 7762, 7840, 3620, 8184, 8172, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=343949312 dispatch_meta_bytes=229368 combine_payload_bytes=343949312 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=344178680 ingress_bytes=454049792 link_bytes=454049792 token_bytes=688127992 weight_movement_bytes=110100480 total_nvlink_bytes=798228472 weight_to_token_ratio=0.160000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.896672666e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385853440 dispatch_meta_bytes=229376 combine_payload_bytes=385853440 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=386082816 ingress_bytes=716154880 link_bytes=716154880 token_bytes=771936256 weight_movement_bytes=330301440 total_nvlink_bytes=1102237696 weight_to_token_ratio=0.427887 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.245511124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385839104 dispatch_meta_bytes=219216 combine_payload_bytes=385839104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386058320 ingress_bytes=606040064 link_bytes=606040064 token_bytes=771897424 weight_movement_bytes=220200960 total_nvlink_bytes=992098384 weight_to_token_ratio=0.285272 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054003348e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59762 payload_rows_remote=51570 dup_rows=5774 dup_rows_remote=2090 dedup_ratio=0.088104 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1609, 1504, 1, 0, 0, 0, 3041, 0, 0, 0, 195, 4095, 0, 0, 0, 0, 1842, 1, 0, 0, 0, 9, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8190, 8174, 8192, 3010, 7622, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 2068, 3684, 20] combine_rows_per_src=[8190, 8190, 8174, 8192, 3010, 7622, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=372377600 dispatch_meta_bytes=229368 combine_payload_bytes=372377600 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372606968 ingress_bytes=592578560 link_bytes=592578560 token_bytes=744984568 weight_movement_bytes=220200960 total_nvlink_bytes=965185528 weight_to_token_ratio=0.295578 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.030591578e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59420 payload_rows_remote=51228 dup_rows=6116 dup_rows_remote=6116 dedup_ratio=0.093323 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1975, 971, 0, 0, 0, 0, 3188, 0, 0, 0, 315, 4096, 0, 0, 0, 0, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8186, 8102, 6694, 3672, 8192] dup_rows_per_dst=[0, 2, 0, 0, 5674, 312, 128, 0] combine_rows_per_src=[8190, 8192, 8192, 8186, 8102, 6694, 3672, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58270 payload_rows_remote=50078 dup_rows=7266 dup_rows_remote=7266 dedup_ratio=0.110870 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2198, 1370, 0, 0, 0, 0, 3404, 0, 0, 0, 136, 4096, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6950, 2168, 8192] dup_rows_per_dst=[0, 0, 0, 0, 7136, 130, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6950, 2168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61384 payload_rows_remote=53192 dup_rows=4152 dup_rows_remote=4152 dedup_ratio=0.063354 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2565, 854, 0, 0, 0, 0, 2902, 0, 0, 0, 225, 4096, 0, 0, 0, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6110, 6122, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 728, 132, 3292, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6110, 6122, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369653760 dispatch_meta_bytes=214640 combine_payload_bytes=369653760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369868400 ingress_bytes=479754240 link_bytes=479754240 token_bytes=739522160 weight_movement_bytes=110100480 total_nvlink_bytes=849622640 weight_to_token_ratio=0.148881 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.343715294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61176 payload_rows_remote=52984 dup_rows=4360 dup_rows_remote=4360 dedup_ratio=0.066528 router_tokens_per_expert=[0, 0, 4096, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 627, 2039, 0, 0, 1, 0, 3771, 0, 0, 1, 224, 4094, 0, 0, 0, 0, 1528, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 8188, 4082, 7956, 8190, 8192] dup_rows_per_dst=[2, 2, 2, 8, 0, 1292, 3054, 0] combine_rows_per_src=[8192, 8192, 8184, 8188, 4082, 7956, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59438 payload_rows_remote=51578 dup_rows=6098 dup_rows_remote=3678 dedup_ratio=0.093048 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1900, 1213, 0, 0, 0, 0, 2980, 0, 0, 0, 260, 4096, 0, 0, 0, 0, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 2426, 7860, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2420, 3678, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 2426, 7860, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61360 payload_rows_remote=56602 dup_rows=4176 dup_rows_remote=3728 dedup_ratio=0.063721 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1946, 1069, 0, 0, 0, 0, 3503, 0, 0, 0, 438, 4096, 0, 0, 0, 0, 1234, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4758, 7450, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 448, 1256, 2470, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 4758, 7450, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367202304 dispatch_meta_bytes=229376 combine_payload_bytes=367202304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=367431680 ingress_bytes=697503744 link_bytes=697503744 token_bytes=734633984 weight_movement_bytes=330301440 total_nvlink_bytes=1064935424 weight_to_token_ratio=0.449614 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.213073730e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=358959104 dispatch_meta_bytes=229376 combine_payload_bytes=358959104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359188480 ingress_bytes=469059584 link_bytes=469059584 token_bytes=718147584 weight_movement_bytes=110100480 total_nvlink_bytes=828248064 weight_to_token_ratio=0.153312 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.157717636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7487480664 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3744895832 ingress_bytes=5724393472 link_bytes=5724393472 total_nvlink_bytes=9469289304 weight_to_token_ratio=0.264683 prefetch_share_of_ingress=0.211569 dedup_ratio=0.099750 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9779 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.955661749e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381280256 dispatch_meta_bytes=229376 combine_payload_bytes=381280256 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=381509632 ingress_bytes=601481216 link_bytes=601481216 token_bytes=762789888 weight_movement_bytes=220200960 total_nvlink_bytes=982990848 weight_to_token_ratio=0.288678 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046074761e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=379789312 dispatch_meta_bytes=229376 combine_payload_bytes=379789312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380018688 ingress_bytes=489889792 link_bytes=489889792 token_bytes=759808000 weight_movement_bytes=110100480 total_nvlink_bytes=869908480 weight_to_token_ratio=0.144906 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.519989213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58664 payload_rows_remote=50472 dup_rows=6872 dup_rows_remote=6870 dedup_ratio=0.104858 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2032, 1330, 0, 0, 0, 0, 3527, 0, 0, 0, 98, 4096, 0, 0, 1, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8172, 8192, 8192, 7124, 2432, 8170] dup_rows_per_dst=[2, 0, 44, 0, 6678, 148, 0, 0] combine_rows_per_src=[8192, 8190, 8172, 8192, 8192, 7124, 2432, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369711104 dispatch_meta_bytes=221024 combine_payload_bytes=369711104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369932128 ingress_bytes=589912064 link_bytes=589912064 token_bytes=739643232 weight_movement_bytes=220200960 total_nvlink_bytes=959844192 weight_to_token_ratio=0.297712 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=26368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025954103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=405723136 dispatch_meta_bytes=241320 combine_payload_bytes=405723136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=405964456 ingress_bytes=736024576 link_bytes=736024576 token_bytes=811687592 weight_movement_bytes=330301440 total_nvlink_bytes=1141989032 weight_to_token_ratio=0.406932 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280067793e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7472845616 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3737572144 ingress_bytes=5937283072 link_bytes=5937283072 total_nvlink_bytes=9674855216 weight_to_token_ratio=0.294668 prefetch_share_of_ingress=0.166895 dedup_ratio=0.091306 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.032591178e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7287936112 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3645101168 ingress_bytes=5734744064 link_bytes=5734744064 total_nvlink_bytes=9379845232 weight_to_token_ratio=0.287037 prefetch_share_of_ingress=0.211187 dedup_ratio=0.106818 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9271 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.973663131e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7468965832 prefetch_bytes=1321205760 reduce_bytes=1211105280 weight_movement_bytes=2532311040 egress_bytes=3735627720 ingress_bytes=6265649152 link_bytes=6265649152 total_nvlink_bytes=10001276872 weight_to_token_ratio=0.339044 prefetch_share_of_ingress=0.210865 dedup_ratio=0.087448 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=11 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9271 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.089699440e-04 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7007462296 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=3504847768 ingress_bytes=5044021248 link_bytes=5044021248 total_nvlink_bytes=8548869016 weight_to_token_ratio=0.219966 prefetch_share_of_ingress=0.174623 dedup_ratio=0.134778 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.772382549e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=361783296 dispatch_meta_bytes=229368 combine_payload_bytes=361783296 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=362012664 ingress_bytes=471883776 link_bytes=471883776 token_bytes=723795960 weight_movement_bytes=110100480 total_nvlink_bytes=833896440 weight_to_token_ratio=0.152115 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.206834980e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7412768704 prefetch_bytes=1101004800 reduce_bytes=1651507200 weight_movement_bytes=2752512000 egress_bytes=3707529152 ingress_bytes=6457751552 link_bytes=6457751552 total_nvlink_bytes=10165280704 weight_to_token_ratio=0.371320 prefetch_share_of_ingress=0.170494 dedup_ratio=0.095718 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=15 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.123109207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7603656856 prefetch_bytes=770703360 reduce_bytes=1211105280 weight_movement_bytes=1981808640 egress_bytes=3802968216 ingress_bytes=5782497280 link_bytes=5782497280 total_nvlink_bytes=9585465496 weight_to_token_ratio=0.260639 prefetch_share_of_ingress=0.133282 dedup_ratio=0.075757 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=11 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.005671383e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6898552728 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3450400664 ingress_bytes=5319860224 link_bytes=5319860224 total_nvlink_bytes=8770260888 weight_to_token_ratio=0.271319 prefetch_share_of_ingress=0.186265 dedup_ratio=0.146060 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.252111896e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-29 23:43:05] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 237.5 | throughput per GPU (TFLOP/s/GPU): 519.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050186E+01 | loss scale: 1.0 | grad norm: 8.731 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45674 payload_rows_remote=39876 dup_rows=19862 dup_rows_remote=17840 dedup_ratio=0.303070 router_tokens_per_expert=[189, 176, 913, 687, 1, 12, 285, 303, 183, 639, 543, 1186, 0, 421, 0, 1143, 994, 524, 392, 0, 493, 1907, 148, 127, 506, 311, 835, 782, 27, 874, 574, 1, 36, 30, 261, 181, 36, 1, 67, 1256, 922, 782, 1137, 96, 349, 1324, 174, 751, 11, 217, 291, 981, 2919, 122, 79, 968, 413, 368, 884, 1479, 621, 669, 167, 0] payload_rows_per_dst=[6970, 5338, 6272, 5798, 5526, 5350, 5800, 4620] dup_rows_per_dst=[4000, 2892, 2898, 2022, 1986, 1944, 2496, 1624] combine_rows_per_src=[6970, 5338, 6272, 5798, 5526, 5350, 5800, 4620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46340 payload_rows_remote=40204 dup_rows=19196 dup_rows_remote=16540 dedup_ratio=0.292908 router_tokens_per_expert=[105, 119, 268, 448, 4, 14, 533, 159, 91, 561, 952, 1746, 0, 424, 0, 1005, 1163, 549, 438, 0, 512, 2378, 103, 275, 322, 755, 934, 835, 135, 705, 476, 1, 12, 18, 238, 236, 19, 0, 83, 824, 704, 477, 1921, 13, 301, 966, 120, 1045, 29, 124, 167, 518, 2638, 553, 54, 763, 327, 457, 798, 2068, 991, 106, 188, 0] payload_rows_per_dst=[6342, 6038, 5894, 5896, 6136, 5316, 6046, 4672] dup_rows_per_dst=[2234, 3520, 2560, 2430, 2656, 2228, 2506, 1062] combine_rows_per_src=[6342, 6038, 5894, 5896, 6136, 5316, 6046, 4672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46030 payload_rows_remote=41752 dup_rows=19506 dup_rows_remote=18376 dedup_ratio=0.297638 router_tokens_per_expert=[181, 102, 292, 532, 3, 17, 361, 136, 115, 1048, 924, 1970, 0, 239, 7, 1029, 823, 1035, 280, 2, 575, 1946, 112, 378, 401, 287, 1014, 742, 97, 620, 611, 0, 27, 10, 194, 188, 21, 0, 48, 1003, 724, 791, 1552, 48, 172, 851, 313, 1238, 2, 229, 343, 667, 2043, 292, 50, 1121, 329, 504, 909, 1369, 1003, 603, 245, 0] payload_rows_per_dst=[5760, 5264, 6326, 5810, 5222, 6962, 4278, 6408] dup_rows_per_dst=[1574, 1936, 3548, 1734, 1498, 4570, 1130, 3516] combine_rows_per_src=[5760, 5264, 6326, 5810, 5222, 6962, 4278, 6408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45108 payload_rows_remote=38586 dup_rows=20428 dup_rows_remote=17304 dedup_ratio=0.311707 router_tokens_per_expert=[117, 290, 902, 436, 16, 67, 181, 376, 229, 910, 568, 1632, 9, 165, 6, 1256, 500, 915, 159, 3, 676, 1775, 58, 270, 329, 314, 1019, 841, 119, 544, 478, 1, 243, 145, 188, 254, 18, 0, 23, 798, 1176, 467, 1186, 29, 415, 715, 246, 783, 84, 83, 530, 1154, 1642, 141, 96, 908, 312, 726, 1308, 2023, 1029, 418, 467, 0] payload_rows_per_dst=[5800, 5982, 4230, 5484, 5324, 6522, 4690, 7076] dup_rows_per_dst=[2254, 3568, 932, 1806, 1952, 3124, 1828, 4964] combine_rows_per_src=[5800, 5982, 4230, 5484, 5324, 6522, 4690, 7076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45606 payload_rows_remote=39052 dup_rows=19930 dup_rows_remote=16852 dedup_ratio=0.304108 router_tokens_per_expert=[101, 195, 412, 311, 2, 27, 532, 300, 330, 1011, 460, 835, 1, 383, 15, 1087, 652, 751, 508, 3, 652, 2060, 63, 127, 497, 420, 848, 579, 230, 655, 502, 0, 22, 43, 410, 226, 18, 0, 160, 983, 1010, 539, 2019, 54, 294, 802, 108, 1081, 40, 229, 312, 1037, 1721, 359, 202, 1013, 602, 377, 757, 1864, 901, 636, 396, 4] payload_rows_per_dst=[5594, 5422, 6554, 5480, 6586, 4376, 6388, 5206] dup_rows_per_dst=[1608, 2822, 3078, 1982, 3338, 1238, 3724, 2140] combine_rows_per_src=[5594, 5422, 6554, 5480, 6586, 4376, 6388, 5206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=37748 dup_rows=21092 dup_rows_remote=15974 dedup_ratio=0.321838 router_tokens_per_expert=[134, 216, 553, 547, 3, 19, 479, 385, 207, 785, 376, 1402, 0, 415, 5, 1210, 457, 766, 243, 2, 648, 2069, 86, 263, 383, 444, 749, 673, 171, 643, 776, 0, 17, 21, 160, 385, 28, 0, 97, 1606, 837, 385, 1718, 65, 718, 635, 115, 942, 22, 247, 724, 1185, 2399, 221, 53, 542, 243, 555, 636, 1460, 583, 741, 318, 1] payload_rows_per_dst=[4910, 3962, 4042, 5972, 5966, 7146, 5750, 6696] dup_rows_per_dst=[1898, 1088, 952, 2652, 2098, 4386, 2900, 5118] combine_rows_per_src=[4910, 3962, 4042, 5972, 5966, 7146, 5750, 6696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46114 payload_rows_remote=39970 dup_rows=19422 dup_rows_remote=17174 dedup_ratio=0.296356 router_tokens_per_expert=[207, 127, 474, 481, 1, 33, 439, 319, 153, 841, 549, 1357, 0, 272, 1, 1298, 934, 724, 503, 9, 650, 1410, 102, 124, 569, 376, 1007, 894, 190, 626, 733, 3, 90, 18, 205, 206, 52, 1, 69, 1236, 693, 602, 1363, 99, 503, 1223, 67, 1099, 42, 135, 616, 610, 2115, 496, 61, 1155, 204, 631, 865, 1435, 654, 666, 151, 0] payload_rows_per_dst=[6144, 5666, 6188, 6202, 6152, 4868, 6076, 4818] dup_rows_per_dst=[2248, 3276, 2724, 2594, 2526, 1506, 3024, 1524] combine_rows_per_src=[6144, 5666, 6188, 6202, 6152, 4868, 6076, 4818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46096 payload_rows_remote=40282 dup_rows=19440 dup_rows_remote=16492 dedup_ratio=0.296631 router_tokens_per_expert=[164, 146, 248, 324, 11, 30, 385, 220, 113, 1047, 214, 1410, 0, 415, 0, 1182, 500, 724, 513, 0, 388, 2073, 149, 322, 828, 120, 920, 480, 98, 1503, 888, 0, 11, 40, 96, 230, 16, 2, 82, 1657, 1070, 372, 1169, 54, 550, 812, 142, 982, 36, 307, 344, 1134, 2572, 352, 28, 623, 264, 387, 871, 1626, 491, 735, 289, 9] payload_rows_per_dst=[6208, 5814, 6464, 6476, 6092, 4590, 5968, 4484] dup_rows_per_dst=[1992, 2948, 2874, 3198, 2478, 1410, 2932, 1608] combine_rows_per_src=[6208, 5814, 6464, 6476, 6092, 4590, 5968, 4484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285831168 dispatch_meta_bytes=230864 combine_payload_bytes=285831168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286062032 ingress_bytes=395931648 link_bytes=395931648 token_bytes=571893200 weight_movement_bytes=110100480 total_nvlink_bytes=681993680 weight_to_token_ratio=0.192519 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.885902554e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=240512 combine_payload_bytes=299278336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=299518848 ingress_bytes=519479296 link_bytes=519479296 token_bytes=598797184 weight_movement_bytes=220200960 total_nvlink_bytes=818998144 weight_to_token_ratio=0.367739 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.034599354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288182272 dispatch_meta_bytes=226976 combine_payload_bytes=288182272 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=288409248 ingress_bytes=508383232 link_bytes=508383232 token_bytes=576591520 weight_movement_bytes=220200960 total_nvlink_bytes=796792480 weight_to_token_ratio=0.381901 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.841620551e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276584448 dispatch_meta_bytes=223560 combine_payload_bytes=276584448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276808008 ingress_bytes=606885888 link_bytes=606885888 token_bytes=553392456 weight_movement_bytes=330301440 total_nvlink_bytes=883693896 weight_to_token_ratio=0.596867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055474375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=223616 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280148352 ingress_bytes=500125696 link_bytes=500125696 token_bytes=560073088 weight_movement_bytes=220200960 total_nvlink_bytes=780274048 weight_to_token_ratio=0.393165 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698008419e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270577664 dispatch_meta_bytes=214888 combine_payload_bytes=270577664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=270792552 ingress_bytes=490778624 link_bytes=490778624 token_bytes=541370216 weight_movement_bytes=220200960 total_nvlink_bytes=761571176 weight_to_token_ratio=0.406747 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.535447463e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286504960 dispatch_meta_bytes=228576 combine_payload_bytes=286504960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286733536 ingress_bytes=396605440 link_bytes=396605440 token_bytes=573238496 weight_movement_bytes=110100480 total_nvlink_bytes=683338976 weight_to_token_ratio=0.192067 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.897620905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288741376 dispatch_meta_bytes=227096 combine_payload_bytes=288741376 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=288968472 ingress_bytes=619042816 link_bytes=619042816 token_bytes=577709848 weight_movement_bytes=330301440 total_nvlink_bytes=908011288 weight_to_token_ratio=0.571743 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076617272e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61264 payload_rows_remote=53090 dup_rows=4272 dup_rows_remote=4272 dedup_ratio=0.065186 router_tokens_per_expert=[0, 0, 0, 20, 0, 0, 0, 147, 0, 8, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 6, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 1, 4086, 0, 2843, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 3813, 732, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 6826, 8186, 8172, 8174, 5692, 8184, 7838] dup_rows_per_dst=[332, 10, 0, 12, 0, 0, 2664, 1254] combine_rows_per_src=[8192, 6826, 8186, 8172, 8174, 5692, 8184, 7838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59530 payload_rows_remote=51394 dup_rows=6006 dup_rows_remote=6002 dedup_ratio=0.091644 router_tokens_per_expert=[0, 0, 0, 6, 0, 1, 16, 95, 1, 6, 1, 0, 3390, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4068, 0, 0, 0, 0, 2, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 4064, 2, 1613, 0, 4095, 0, 0, 0, 0, 1, 1, 0, 2179, 0, 0, 0, 0, 3884, 1157, 0, 0, 0, 0, 16] payload_rows_per_dst=[8192, 6792, 8192, 8136, 8144, 6578, 5470, 8026] dup_rows_per_dst=[234, 6, 2, 4, 0, 2090, 1582, 2088] combine_rows_per_src=[8192, 6792, 8192, 8136, 8144, 6578, 5470, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60064 payload_rows_remote=51874 dup_rows=5472 dup_rows_remote=1576 dedup_ratio=0.083496 router_tokens_per_expert=[0, 0, 0, 62, 2, 0, 0, 53, 2, 9, 1, 0, 3509, 0, 0, 0, 0, 0, 0, 4093, 4, 5, 0, 0, 4086, 0, 0, 0, 0, 7, 0, 0, 0, 4073, 0, 0, 0, 4, 0, 0, 4057, 0, 2174, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 1974, 8, 0, 2, 0, 3720, 823, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 7028, 8192, 8172, 8154, 4348, 8190, 7788] dup_rows_per_dst=[234, 14, 12, 14, 0, 0, 3896, 1302] combine_rows_per_src=[8192, 7028, 8192, 8172, 8154, 4348, 8190, 7788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59326 payload_rows_remote=55024 dup_rows=6210 dup_rows_remote=6210 dedup_ratio=0.094757 router_tokens_per_expert=[0, 0, 0, 80, 2, 0, 0, 50, 0, 56, 27, 0, 3295, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 26, 2, 0, 3, 4082, 1, 0, 0, 0, 0, 0, 4014, 0, 2151, 0, 4093, 0, 0, 0, 0, 1, 0, 2, 1552, 1, 0, 0, 0, 3050, 2128, 0, 0, 0, 2, 7] payload_rows_per_dst=[8192, 6640, 8190, 8116, 8168, 4302, 8090, 7628] dup_rows_per_dst=[258, 116, 0, 36, 4, 0, 3050, 2746] combine_rows_per_src=[8192, 6640, 8190, 8116, 8168, 4302, 8090, 7628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59026 payload_rows_remote=50834 dup_rows=6510 dup_rows_remote=6506 dedup_ratio=0.099335 router_tokens_per_expert=[0, 2, 2, 100, 0, 0, 4, 63, 0, 11, 47, 0, 3120, 0, 0, 0, 3, 0, 0, 4094, 0, 1, 0, 0, 4060, 0, 0, 0, 0, 67, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 4059, 0, 2169, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 1938, 8, 0, 0, 1, 3793, 1035, 0, 1, 0, 3, 4] payload_rows_per_dst=[8192, 6314, 8192, 8156, 8174, 8184, 3892, 7922] dup_rows_per_dst=[336, 42, 4, 98, 2, 4274, 2, 1752] combine_rows_per_src=[8192, 6314, 8192, 8156, 8174, 8184, 3892, 7922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57248 payload_rows_remote=49412 dup_rows=8288 dup_rows_remote=4174 dedup_ratio=0.126465 router_tokens_per_expert=[0, 1, 0, 29, 0, 0, 2, 57, 0, 12, 2, 1, 2801, 0, 0, 0, 0, 0, 0, 4091, 6, 1, 1, 6, 4036, 0, 0, 0, 0, 10, 3, 1, 0, 4079, 1, 0, 0, 4, 0, 3, 4054, 0, 2879, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 2012, 9, 0, 1, 0, 3180, 1381, 0, 1, 0, 6, 1] payload_rows_per_dst=[5706, 5616, 8156, 8080, 8170, 5508, 8176, 7836] dup_rows_per_dst=[102, 16, 18, 20, 4, 38, 3976, 4114] combine_rows_per_src=[5706, 5616, 8156, 8080, 8170, 5508, 8176, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=380549120 dispatch_meta_bytes=229448 combine_payload_bytes=380549120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380778568 ingress_bytes=490649600 link_bytes=490649600 token_bytes=761327688 weight_movement_bytes=110100480 total_nvlink_bytes=871428168 weight_to_token_ratio=0.144616 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.533203524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55768 payload_rows_remote=47576 dup_rows=9768 dup_rows_remote=8994 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 0, 152, 0, 0, 13, 222, 0, 2, 0, 0, 2929, 0, 0, 6, 0, 2, 1, 4095, 0, 0, 1, 0, 4069, 4, 0, 0, 1, 30, 0, 2, 0, 4064, 0, 0, 1, 0, 0, 0, 4010, 0, 1856, 0, 4096, 2, 2, 0, 1, 1, 0, 2, 2276, 3, 0, 0, 2, 3770, 1137, 0, 0, 1, 0, 15] payload_rows_per_dst=[8192, 8158, 8192, 8162, 8132, 8066, 4564, 2302] dup_rows_per_dst=[774, 5256, 6, 148, 274, 3300, 2, 8] combine_rows_per_src=[8192, 8158, 8192, 8162, 8132, 8066, 4564, 2302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=368392192 dispatch_meta_bytes=229584 combine_payload_bytes=368392192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368621776 ingress_bytes=478492672 link_bytes=478492672 token_bytes=737013968 weight_movement_bytes=110100480 total_nvlink_bytes=847114448 weight_to_token_ratio=0.149387 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.321774551e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=371832832 dispatch_meta_bytes=213800 combine_payload_bytes=371832832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372046632 ingress_bytes=481933312 link_bytes=481933312 token_bytes=743879464 weight_movement_bytes=110100480 total_nvlink_bytes=853979944 weight_to_token_ratio=0.148008 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.381612940e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=394412032 dispatch_meta_bytes=244936 combine_payload_bytes=394412032 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=394656968 ingress_bytes=1055014912 link_bytes=1055014912 token_bytes=789069000 weight_movement_bytes=660602880 total_nvlink_bytes=1449671880 weight_to_token_ratio=0.837193 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.834844452e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55118 payload_rows_remote=47276 dup_rows=10418 dup_rows_remote=5558 dedup_ratio=0.158966 router_tokens_per_expert=[0, 0, 0, 59, 0, 0, 0, 49, 0, 0, 5, 0, 3438, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4074, 0, 4, 0, 0, 27, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 2, 4033, 0, 2545, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 0, 3584, 778, 0, 8, 0, 4, 0] payload_rows_per_dst=[8192, 7842, 8192, 8150, 8134, 8168, 3798, 2642] dup_rows_per_dst=[216, 4860, 2, 60, 2, 4988, 0, 290] combine_rows_per_src=[8192, 7842, 8192, 8150, 8134, 8168, 3798, 2642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=364378112 dispatch_meta_bytes=229360 combine_payload_bytes=364378112 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=364607472 ingress_bytes=474478592 link_bytes=474478592 token_bytes=728985584 weight_movement_bytes=110100480 total_nvlink_bytes=839086064 weight_to_token_ratio=0.151032 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.251963098e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=354185216 dispatch_meta_bytes=214344 combine_payload_bytes=354185216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=354399560 ingress_bytes=574386176 link_bytes=574386176 token_bytes=708584776 weight_movement_bytes=220200960 total_nvlink_bytes=928785736 weight_to_token_ratio=0.310762 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.989520304e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=341024768 dispatch_meta_bytes=226280 combine_payload_bytes=341024768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341251048 ingress_bytes=451125248 link_bytes=451125248 token_bytes=682275816 weight_movement_bytes=110100480 total_nvlink_bytes=792376296 weight_to_token_ratio=0.161372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.845810036e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338874368 dispatch_meta_bytes=211336 combine_payload_bytes=338874368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339085704 ingress_bytes=448974848 link_bytes=448974848 token_bytes=677960072 weight_movement_bytes=110100480 total_nvlink_bytes=788060552 weight_to_token_ratio=0.162400 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.808411043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63976 payload_rows_remote=57276 dup_rows=1560 dup_rows_remote=1522 dedup_ratio=0.023804 router_tokens_per_expert=[0, 0, 0, 237, 4071, 9, 0, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 77, 0, 3292, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 24, 0, 0, 545] payload_rows_per_dst=[8156, 8184, 8192, 8192, 6700, 8188, 8186, 8178] dup_rows_per_dst=[478, 0, 0, 4, 38, 0, 0, 1040] combine_rows_per_src=[8156, 8184, 8192, 8192, 6700, 8188, 8186, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63060 payload_rows_remote=54868 dup_rows=2476 dup_rows_remote=2470 dedup_ratio=0.037781 router_tokens_per_expert=[0, 0, 0, 462, 4085, 15, 0, 1, 4095, 0, 0, 0, 0, 0, 4077, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 97, 0, 2885, 0, 1, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 38, 0, 0, 948] payload_rows_per_dst=[8186, 8154, 8180, 8192, 5942, 8192, 8190, 8024] dup_rows_per_dst=[940, 0, 0, 6, 26, 2, 0, 1502] combine_rows_per_src=[8186, 8154, 8180, 8192, 5942, 8192, 8190, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63582 payload_rows_remote=55396 dup_rows=1954 dup_rows_remote=1954 dedup_ratio=0.029816 router_tokens_per_expert=[0, 0, 0, 402, 4072, 7, 0, 0, 4093, 2, 0, 2, 0, 0, 4091, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 6, 4095, 1, 1, 30, 0, 3188, 0, 4, 0, 2, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 42, 0, 0, 629] payload_rows_per_dst=[8176, 8186, 8190, 8192, 6440, 8188, 8186, 8024] dup_rows_per_dst=[786, 4, 4, 20, 10, 0, 0, 1130] combine_rows_per_src=[8176, 8186, 8190, 8192, 6440, 8188, 8186, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62942 payload_rows_remote=54750 dup_rows=2594 dup_rows_remote=2592 dedup_ratio=0.039581 router_tokens_per_expert=[0, 0, 0, 432, 4055, 15, 0, 0, 4090, 0, 0, 0, 0, 0, 4073, 0, 2, 4095, 0, 0, 0, 0, 0, 1, 3, 0, 1, 0, 0, 3, 4095, 0, 0, 71, 0, 2824, 0, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 29, 0, 0, 950] payload_rows_per_dst=[8164, 8146, 8190, 8192, 5746, 8192, 8180, 8132] dup_rows_per_dst=[840, 0, 6, 12, 46, 2, 0, 1688] combine_rows_per_src=[8164, 8146, 8190, 8192, 5746, 8192, 8180, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=410554368 dispatch_meta_bytes=235192 combine_payload_bytes=410554368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410789560 ingress_bytes=520654848 link_bytes=520654848 token_bytes=821343928 weight_movement_bytes=110100480 total_nvlink_bytes=931444408 weight_to_token_ratio=0.134049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055044137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63242 payload_rows_remote=55056 dup_rows=2294 dup_rows_remote=2294 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 0, 435, 4093, 6, 0, 0, 4072, 0, 0, 0, 0, 0, 4067, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 1, 87, 0, 3231, 0, 1, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 2, 37, 0, 0, 453] payload_rows_per_dst=[8192, 8134, 8186, 8192, 6826, 8188, 8144, 7380] dup_rows_per_dst=[876, 0, 0, 0, 616, 0, 0, 802] combine_rows_per_src=[8192, 8134, 8186, 8192, 6826, 8188, 8144, 7380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=393293824 dispatch_meta_bytes=229352 combine_payload_bytes=393293824 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=393523176 ingress_bytes=723595264 link_bytes=723595264 token_bytes=786817000 weight_movement_bytes=330301440 total_nvlink_bytes=1117118440 weight_to_token_ratio=0.419794 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.258451175e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=392448000 dispatch_meta_bytes=229368 combine_payload_bytes=392448000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392677368 ingress_bytes=502548480 link_bytes=502548480 token_bytes=785125368 weight_movement_bytes=110100480 total_nvlink_bytes=895225848 weight_to_token_ratio=0.140233 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.740144617e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=397078528 dispatch_meta_bytes=229400 combine_payload_bytes=397078528 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=397307928 ingress_bytes=617279488 link_bytes=617279488 token_bytes=794386456 weight_movement_bytes=220200960 total_nvlink_bytes=1014587416 weight_to_token_ratio=0.277196 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073550555e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57272 payload_rows_remote=49166 dup_rows=8264 dup_rows_remote=946 dedup_ratio=0.126099 router_tokens_per_expert=[0, 0, 0, 353, 4037, 51, 0, 0, 4090, 1, 0, 0, 0, 0, 4064, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 9, 5, 0, 0, 4092, 0, 4, 253, 1, 2738, 0, 2, 0, 0, 1, 4090, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3780, 2, 35, 0, 0, 1058] payload_rows_per_dst=[2526, 8018, 8188, 8188, 5874, 8190, 8182, 8106] dup_rows_per_dst=[342, 340, 2, 20, 122, 8, 112, 7318] combine_rows_per_src=[2526, 8018, 8188, 8188, 5874, 8190, 8182, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58864 payload_rows_remote=50742 dup_rows=6672 dup_rows_remote=5746 dedup_ratio=0.101807 router_tokens_per_expert=[0, 0, 0, 463, 4050, 3, 0, 0, 4088, 1, 0, 0, 5, 0, 4076, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 172, 0, 2910, 0, 0, 0, 2, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3837, 4, 25, 0, 0, 855] payload_rows_per_dst=[8122, 8158, 8176, 8132, 8148, 8192, 8178, 1758] dup_rows_per_dst=[926, 6, 32, 0, 5694, 2, 2, 10] combine_rows_per_src=[8122, 8158, 8176, 8132, 8148, 8192, 8178, 1758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59532 payload_rows_remote=51350 dup_rows=6004 dup_rows_remote=5980 dedup_ratio=0.091614 router_tokens_per_expert=[0, 0, 0, 325, 4080, 8, 0, 0, 4088, 12, 0, 0, 0, 0, 4091, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 75, 0, 2764, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 36, 0, 0, 1186] payload_rows_per_dst=[8172, 8182, 8190, 8192, 8052, 8192, 8176, 2376] dup_rows_per_dst=[654, 24, 0, 2, 5256, 0, 0, 68] combine_rows_per_src=[8172, 8182, 8190, 8192, 8052, 8192, 8176, 2376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=394641408 dispatch_meta_bytes=229400 combine_payload_bytes=394641408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394870808 ingress_bytes=504741888 link_bytes=504741888 token_bytes=789512216 weight_movement_bytes=110100480 total_nvlink_bytes=899612696 weight_to_token_ratio=0.139454 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.778291590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352421888 dispatch_meta_bytes=200448 combine_payload_bytes=352421888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=352622336 ingress_bytes=572622848 link_bytes=572622848 token_bytes=705044224 weight_movement_bytes=220200960 total_nvlink_bytes=925245184 weight_to_token_ratio=0.312322 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.958853129e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62822 payload_rows_remote=55064 dup_rows=2714 dup_rows_remote=2714 dedup_ratio=0.041412 router_tokens_per_expert=[0, 160, 0, 3965, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 3879, 0, 0, 4096, 0, 929, 0, 0, 0, 0, 0, 499, 0, 0, 5, 0, 0, 0, 3402, 0, 0, 57, 0, 0, 0, 0, 0] payload_rows_per_dst=[7942, 8192, 6980, 8192, 7758, 8192, 7378, 8188] dup_rows_per_dst=[308, 0, 0, 0, 0, 1858, 434, 114] combine_rows_per_src=[7942, 8192, 6980, 8192, 7758, 8192, 7378, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=363718656 dispatch_meta_bytes=225952 combine_payload_bytes=363718656 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=363944608 ingress_bytes=694020096 link_bytes=694020096 token_bytes=727663264 weight_movement_bytes=330301440 total_nvlink_bytes=1057964704 weight_to_token_ratio=0.453921 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207015094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=368076800 dispatch_meta_bytes=229320 combine_payload_bytes=368076800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=368306120 ingress_bytes=698378240 link_bytes=698378240 token_bytes=736382920 weight_movement_bytes=330301440 total_nvlink_bytes=1066684360 weight_to_token_ratio=0.448546 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.214594623e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62740 payload_rows_remote=54548 dup_rows=2796 dup_rows_remote=2794 dedup_ratio=0.042664 router_tokens_per_expert=[0, 152, 0, 3989, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 3940, 0, 0, 4096, 0, 809, 1, 0, 0, 0, 0, 1083, 0, 0, 2, 0, 0, 0, 2792, 0, 0, 118, 0, 0, 0, 1, 0] payload_rows_per_dst=[8002, 8190, 6992, 8192, 7880, 8192, 7100, 8192] dup_rows_per_dst=[282, 0, 0, 2, 0, 1620, 654, 238] combine_rows_per_src=[8002, 8190, 6992, 8192, 7880, 8192, 7100, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61902 payload_rows_remote=53710 dup_rows=3634 dup_rows_remote=1064 dedup_ratio=0.055450 router_tokens_per_expert=[0, 42, 0, 4022, 0, 2, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 3856, 0, 0, 4096, 0, 1284, 0, 0, 1, 0, 0, 812, 0, 0, 17, 0, 0, 0, 2736, 2, 0, 218, 0, 0, 0, 2, 0] payload_rows_per_dst=[8048, 8192, 6788, 8192, 7712, 8192, 6594, 8184] dup_rows_per_dst=[84, 2, 0, 2, 0, 2570, 536, 440] combine_rows_per_src=[8048, 8192, 6788, 8192, 7712, 8192, 6594, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63396 payload_rows_remote=56098 dup_rows=2140 dup_rows_remote=1682 dedup_ratio=0.032654 router_tokens_per_expert=[0, 50, 0, 4022, 0, 4, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 4092, 0, 0, 3955, 0, 0, 4096, 0, 691, 1, 0, 0, 0, 0, 746, 0, 0, 1, 0, 0, 1, 3130, 0, 0, 101, 0, 0, 0, 0, 0] payload_rows_per_dst=[8056, 8188, 7378, 8190, 7910, 8192, 7298, 8184] dup_rows_per_dst=[96, 0, 0, 0, 0, 1384, 458, 202] combine_rows_per_src=[8056, 8188, 7378, 8190, 7910, 8192, 7298, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394698752 dispatch_meta_bytes=231112 combine_payload_bytes=394698752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394929864 ingress_bytes=614899712 link_bytes=614899712 token_bytes=789628616 weight_movement_bytes=220200960 total_nvlink_bytes=1009829576 weight_to_token_ratio=0.278866 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069411733e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=391000064 dispatch_meta_bytes=229368 combine_payload_bytes=391000064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391229432 ingress_bytes=611201024 link_bytes=611201024 token_bytes=782229496 weight_movement_bytes=220200960 total_nvlink_bytes=1002430456 weight_to_token_ratio=0.281504 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062979106e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61964 payload_rows_remote=55298 dup_rows=3572 dup_rows_remote=3572 dedup_ratio=0.054504 router_tokens_per_expert=[0, 134, 0, 3871, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 4024, 0, 1, 4095, 0, 1482, 0, 0, 1, 0, 0, 478, 0, 0, 4, 0, 0, 0, 3016, 1, 1, 44, 0, 0, 0, 0, 0] payload_rows_per_dst=[7764, 8188, 6666, 8188, 8050, 8190, 6726, 8192] dup_rows_per_dst=[246, 0, 0, 2, 0, 2966, 270, 88] combine_rows_per_src=[7764, 8188, 6666, 8188, 8050, 8190, 6726, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=384993280 dispatch_meta_bytes=219096 combine_payload_bytes=384993280 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=385212376 ingress_bytes=825395200 link_bytes=825395200 token_bytes=770205656 weight_movement_bytes=440401920 total_nvlink_bytes=1210607576 weight_to_token_ratio=0.571798 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.435498007e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402110464 dispatch_meta_bytes=231120 combine_payload_bytes=402110464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402341584 ingress_bytes=512210944 link_bytes=512210944 token_bytes=804452048 weight_movement_bytes=110100480 total_nvlink_bytes=914552528 weight_to_token_ratio=0.136864 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.908190758e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56840 payload_rows_remote=48668 dup_rows=8696 dup_rows_remote=8362 dedup_ratio=0.132690 router_tokens_per_expert=[0, 218, 0, 3916, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 1, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 0, 0, 3665, 0, 0, 4096, 0, 1447, 1, 0, 6, 0, 0, 1278, 0, 0, 4, 0, 0, 0, 2431, 6, 0, 167, 0, 0, 0, 0, 0] payload_rows_per_dst=[6888, 8192, 6492, 8192, 7360, 3602, 7942, 8172] dup_rows_per_dst=[382, 0, 0, 8, 990, 806, 6176, 334] combine_rows_per_src=[6888, 8192, 6492, 8192, 7360, 3602, 7942, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55540 payload_rows_remote=47500 dup_rows=9996 dup_rows_remote=9024 dedup_ratio=0.152527 router_tokens_per_expert=[0, 261, 0, 3985, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3320, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 3804, 0, 0, 4096, 0, 1269, 0, 0, 1, 0, 0, 1019, 0, 0, 5, 0, 0, 0, 2424, 4, 0, 294, 0, 0, 1, 0, 0] payload_rows_per_dst=[8040, 8192, 8022, 8182, 8038, 2540, 6264, 6262] dup_rows_per_dst=[972, 8, 6302, 0, 1750, 0, 632, 332] combine_rows_per_src=[8040, 8192, 8022, 8182, 8038, 2540, 6264, 6262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60398 payload_rows_remote=52206 dup_rows=5138 dup_rows_remote=5138 dedup_ratio=0.078400 router_tokens_per_expert=[0, 204, 0, 4009, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4044, 0, 0, 4096, 0, 1171, 0, 0, 4, 0, 0, 840, 0, 0, 6, 0, 0, 0, 2691, 0, 0, 116, 0, 0, 0, 0, 0] payload_rows_per_dst=[8040, 8192, 7326, 8192, 8088, 5800, 6578, 8182] dup_rows_per_dst=[386, 0, 2590, 2, 0, 1432, 496, 232] combine_rows_per_src=[8040, 8192, 7326, 8192, 8088, 5800, 6578, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396376064 dispatch_meta_bytes=235480 combine_payload_bytes=396376064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396611544 ingress_bytes=506476544 link_bytes=506476544 token_bytes=792987608 weight_movement_bytes=110100480 total_nvlink_bytes=903088088 weight_to_token_ratio=0.138843 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.808460111e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63212 payload_rows_remote=55256 dup_rows=2324 dup_rows_remote=2312 dedup_ratio=0.035461 router_tokens_per_expert=[0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3800, 0, 0, 0, 0, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 6, 0, 0, 4096, 0, 335, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 814, 4096, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7600, 6704, 7956, 8192, 8184, 8192] dup_rows_per_dst=[0, 4, 0, 0, 12, 672, 1628, 8] combine_rows_per_src=[8192, 8192, 7600, 6704, 7956, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=348852224 dispatch_meta_bytes=228120 combine_payload_bytes=348852224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349080344 ingress_bytes=458952704 link_bytes=458952704 token_bytes=697932568 weight_movement_bytes=110100480 total_nvlink_bytes=808033048 weight_to_token_ratio=0.157752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.981942370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62954 payload_rows_remote=56428 dup_rows=2582 dup_rows_remote=2582 dedup_ratio=0.039398 router_tokens_per_expert=[0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3750, 0, 0, 0, 0, 3263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 4096, 0, 383, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 905, 4095, 0, 2, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8192, 7500, 6526, 7972, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 766, 1810, 4] combine_rows_per_src=[8190, 8192, 7500, 6526, 7972, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=340480000 dispatch_meta_bytes=226096 combine_payload_bytes=340480000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=340706096 ingress_bytes=560680960 link_bytes=560680960 token_bytes=681186096 weight_movement_bytes=220200960 total_nvlink_bytes=901387056 weight_to_token_ratio=0.323261 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.751164056e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=374212608 dispatch_meta_bytes=229376 combine_payload_bytes=374212608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374441984 ingress_bytes=484313088 link_bytes=484313088 token_bytes=748654592 weight_movement_bytes=110100480 total_nvlink_bytes=858755072 weight_to_token_ratio=0.147064 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.423001158e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62472 payload_rows_remote=54280 dup_rows=3064 dup_rows_remote=1776 dedup_ratio=0.046753 router_tokens_per_expert=[0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 3717, 0, 0, 0, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 4096, 0, 643, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 871, 4096, 0, 10, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7434, 6198, 7886, 8192, 8186, 8192] dup_rows_per_dst=[6, 8, 0, 0, 0, 1288, 1742, 20] combine_rows_per_src=[8192, 8192, 7434, 6198, 7886, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=396075008 dispatch_meta_bytes=230272 combine_payload_bytes=396075008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396305280 ingress_bytes=616275968 link_bytes=616275968 token_bytes=792380288 weight_movement_bytes=220200960 total_nvlink_bytes=1012581248 weight_to_token_ratio=0.277898 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071805268e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62290 payload_rows_remote=54108 dup_rows=3246 dup_rows_remote=972 dedup_ratio=0.049530 router_tokens_per_expert=[0, 4, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 3610, 0, 0, 0, 0, 3092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 4095, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1137, 4096, 0, 8, 0, 0, 0, 1, 1] payload_rows_per_dst=[8190, 8192, 7220, 6190, 7932, 8192, 8182, 8192] dup_rows_per_dst=[4, 6, 0, 0, 0, 942, 2274, 20] combine_rows_per_src=[8190, 8192, 7220, 6190, 7932, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404475904 dispatch_meta_bytes=236040 combine_payload_bytes=404475904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404711944 ingress_bytes=514576384 link_bytes=514576384 token_bytes=809187848 weight_movement_bytes=110100480 total_nvlink_bytes=919288328 weight_to_token_ratio=0.136063 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60766 payload_rows_remote=53574 dup_rows=4770 dup_rows_remote=4770 dedup_ratio=0.072784 router_tokens_per_expert=[0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 3596, 0, 0, 0, 0, 2834, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3989, 4, 0, 0, 4096, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 1137, 4095, 0, 14, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 7192, 6048, 7982, 8190, 6778, 8192] dup_rows_per_dst=[4, 2, 0, 1364, 4, 1420, 1948, 28] combine_rows_per_src=[8192, 8192, 7192, 6048, 7982, 8190, 6778, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=389079040 dispatch_meta_bytes=224224 combine_payload_bytes=389079040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389303264 ingress_bytes=499179520 link_bytes=499179520 token_bytes=778382304 weight_movement_bytes=110100480 total_nvlink_bytes=888482784 weight_to_token_ratio=0.141448 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57208 payload_rows_remote=49016 dup_rows=8328 dup_rows_remote=8262 dedup_ratio=0.127075 router_tokens_per_expert=[0, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4091, 3649, 0, 0, 0, 0, 2399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 4096, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 1175, 4096, 0, 11, 0, 0, 0, 5, 0] payload_rows_per_dst=[8158, 8166, 3464, 4798, 8070, 8192, 8168, 8192] dup_rows_per_dst=[22, 2, 0, 0, 1760, 2078, 4400, 66] combine_rows_per_src=[8158, 8166, 3464, 4798, 8070, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=387846144 dispatch_meta_bytes=220320 combine_payload_bytes=387846144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388066464 ingress_bytes=608047104 link_bytes=608047104 token_bytes=775912608 weight_movement_bytes=220200960 total_nvlink_bytes=996113568 weight_to_token_ratio=0.283796 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057493920e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=384018432 dispatch_meta_bytes=233376 combine_payload_bytes=384018432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=384251808 ingress_bytes=824420352 link_bytes=824420352 token_bytes=768270240 weight_movement_bytes=440401920 total_nvlink_bytes=1208672160 weight_to_token_ratio=0.573238 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.433802586e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65104 payload_rows_remote=56912 dup_rows=432 dup_rows_remote=132 dedup_ratio=0.006592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 4096, 0, 4094, 0, 0, 0, 0, 229, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 7804, 8148, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 132, 0, 300, 0, 0, 0] combine_rows_per_src=[8192, 7804, 8148, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52198 payload_rows_remote=44006 dup_rows=13338 dup_rows_remote=13332 dedup_ratio=0.203522 router_tokens_per_expert=[0, 10, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3667, 0, 0, 0, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 4096, 0, 821, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 861, 4095, 0, 7, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8174, 8054, 1644, 1722, 8028] dup_rows_per_dst=[6, 2, 7510, 5802, 0, 16, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8174, 8054, 1644, 1722, 8028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57198 payload_rows_remote=49006 dup_rows=8338 dup_rows_remote=8336 dedup_ratio=0.127228 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3775, 0, 0, 0, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 1, 0, 0, 4096, 0, 612, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1162, 4096, 0, 8, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 7588, 8172, 7906, 6638, 2324, 8192] dup_rows_per_dst=[0, 2, 1836, 5560, 2, 922, 0, 16] combine_rows_per_src=[8186, 8192, 7588, 8172, 7906, 6638, 2324, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351346688 dispatch_meta_bytes=229112 combine_payload_bytes=351346688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=351575800 ingress_bytes=571547648 link_bytes=571547648 token_bytes=702922488 weight_movement_bytes=220200960 total_nvlink_bytes=923123448 weight_to_token_ratio=0.313265 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.940153633e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65076 payload_rows_remote=56884 dup_rows=460 dup_rows_remote=460 dedup_ratio=0.007019 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 4096, 0, 4080, 0, 0, 0, 0, 263, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 7776, 8180, 8192, 8190, 8192, 8192, 8162] dup_rows_per_dst=[0, 0, 210, 0, 248, 0, 0, 2] combine_rows_per_src=[8192, 7776, 8180, 8192, 8190, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64724 payload_rows_remote=56532 dup_rows=812 dup_rows_remote=812 dedup_ratio=0.012390 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 4096, 0, 4054, 0, 0, 0, 0, 485, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 7470, 8186, 8192, 8186, 8192, 8192, 8114] dup_rows_per_dst=[0, 0, 564, 0, 242, 0, 0, 6] combine_rows_per_src=[8192, 7470, 8186, 8192, 8186, 8192, 8192, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407945216 dispatch_meta_bytes=228176 combine_payload_bytes=407945216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=408173392 ingress_bytes=738246656 link_bytes=738246656 token_bytes=816118608 weight_movement_bytes=330301440 total_nvlink_bytes=1146420048 weight_to_token_ratio=0.404722 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.283932355e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=315435008 dispatch_meta_bytes=229352 combine_payload_bytes=315435008 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=315664360 ingress_bytes=425535488 link_bytes=425535488 token_bytes=631099368 weight_movement_bytes=110100480 total_nvlink_bytes=741199848 weight_to_token_ratio=0.174458 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.400762022e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351275008 dispatch_meta_bytes=229368 combine_payload_bytes=351275008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351504376 ingress_bytes=461375488 link_bytes=461375488 token_bytes=702779384 weight_movement_bytes=110100480 total_nvlink_bytes=812879864 weight_to_token_ratio=0.156664 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.024078569e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65206 payload_rows_remote=57014 dup_rows=330 dup_rows_remote=330 dedup_ratio=0.005035 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3939, 0, 4096, 0, 4092, 0, 0, 0, 0, 241, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 7878, 8182, 8192, 8186, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 112, 0, 210, 2, 0, 4] combine_rows_per_src=[8192, 7878, 8182, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65036 payload_rows_remote=56870 dup_rows=500 dup_rows_remote=282 dedup_ratio=0.007629 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 3863, 0, 4095, 0, 4091, 0, 0, 0, 0, 286, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 7726, 8166, 8190, 8190, 8192, 8192, 8188] dup_rows_per_dst=[4, 0, 218, 0, 278, 0, 0, 0] combine_rows_per_src=[8192, 7726, 8166, 8190, 8190, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407744512 dispatch_meta_bytes=229376 combine_payload_bytes=407744512 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=407973888 ingress_bytes=627945472 link_bytes=627945472 token_bytes=815718400 weight_movement_bytes=220200960 total_nvlink_bytes=1035919360 weight_to_token_ratio=0.269947 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092100455e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405221376 dispatch_meta_bytes=229376 combine_payload_bytes=405221376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405450752 ingress_bytes=515321856 link_bytes=515321856 token_bytes=810672128 weight_movement_bytes=110100480 total_nvlink_bytes=920772608 weight_to_token_ratio=0.135814 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.962294635e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64744 payload_rows_remote=56664 dup_rows=792 dup_rows_remote=498 dedup_ratio=0.012085 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 4096, 0, 4034, 0, 0, 0, 0, 360, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 0, 2, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4] payload_rows_per_dst=[8186, 7540, 8180, 8192, 8182, 8192, 8192, 8080] dup_rows_per_dst=[0, 0, 304, 4, 184, 0, 6, 294] combine_rows_per_src=[8186, 7540, 8180, 8192, 8182, 8192, 8192, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407644160 dispatch_meta_bytes=228608 combine_payload_bytes=407644160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407872768 ingress_bytes=627845120 link_bytes=627845120 token_bytes=815516928 weight_movement_bytes=220200960 total_nvlink_bytes=1035717888 weight_to_token_ratio=0.270014 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091925926e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=408676352 dispatch_meta_bytes=229376 combine_payload_bytes=408676352 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=408905728 ingress_bytes=628877312 link_bytes=628877312 token_bytes=817582080 weight_movement_bytes=220200960 total_nvlink_bytes=1037783040 weight_to_token_ratio=0.269332 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093721078e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60914 payload_rows_remote=54382 dup_rows=4622 dup_rows_remote=3646 dedup_ratio=0.070526 router_tokens_per_expert=[0, 1891, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3656, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 3117, 0, 4096, 0, 0, 181, 21, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7494, 7312, 6808, 8192, 6532, 8192, 8192, 8192] dup_rows_per_dst=[3206, 0, 0, 34, 976, 404, 2, 0] combine_rows_per_src=[7494, 7312, 6808, 8192, 6532, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65022 payload_rows_remote=57268 dup_rows=514 dup_rows_remote=514 dedup_ratio=0.007843 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 4096, 0, 4055, 0, 0, 0, 0, 330, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 7754, 8184, 8192, 8192, 8192, 8192, 8124] dup_rows_per_dst=[0, 0, 326, 0, 188, 0, 0, 0] combine_rows_per_src=[8192, 7754, 8184, 8192, 8192, 8192, 8192, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61344 payload_rows_remote=53152 dup_rows=4192 dup_rows_remote=4192 dedup_ratio=0.063965 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 4095, 0, 4078, 0, 0, 0, 0, 347, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 7958, 6092, 8190, 6366, 8190, 8186, 8170] dup_rows_per_dst=[0, 3630, 352, 0, 202, 0, 0, 8] combine_rows_per_src=[8192, 7958, 6092, 8190, 6366, 8190, 8186, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406167552 dispatch_meta_bytes=228648 combine_payload_bytes=406167552 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=406396200 ingress_bytes=626368512 link_bytes=626368512 token_bytes=812563752 weight_movement_bytes=220200960 total_nvlink_bytes=1032764712 weight_to_token_ratio=0.270995 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089357862e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57974 payload_rows_remote=49782 dup_rows=7562 dup_rows_remote=6328 dedup_ratio=0.115387 router_tokens_per_expert=[0, 1278, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3367, 0, 0, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 42, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 4096, 0, 0, 422, 195, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 2, 4096, 0, 2, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[2556, 6734, 7726, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 80, 6202, 1234, 38, 8] combine_rows_per_src=[2556, 6734, 7726, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389810176 dispatch_meta_bytes=232112 combine_payload_bytes=389810176 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=390042288 ingress_bytes=830212096 link_bytes=830212096 token_bytes=779852464 weight_movement_bytes=440401920 total_nvlink_bytes=1220254384 weight_to_token_ratio=0.564725 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443875381e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61388 payload_rows_remote=53198 dup_rows=4148 dup_rows_remote=4026 dedup_ratio=0.063293 router_tokens_per_expert=[0, 1574, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3325, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 61, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 4096, 0, 0, 332, 103, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6650, 7744, 8190, 6036, 8192, 8192, 8192] dup_rows_per_dst=[3148, 0, 0, 122, 0, 870, 6, 2] combine_rows_per_src=[8192, 6650, 7744, 8190, 6036, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=410497024 dispatch_meta_bytes=231128 combine_payload_bytes=410497024 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=410728152 ingress_bytes=960999424 link_bytes=960999424 token_bytes=821225176 weight_movement_bytes=550502400 total_nvlink_bytes=1371727576 weight_to_token_ratio=0.670343 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.671336056e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380993536 dispatch_meta_bytes=229376 combine_payload_bytes=380993536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381222912 ingress_bytes=491094016 link_bytes=491094016 token_bytes=762216448 weight_movement_bytes=110100480 total_nvlink_bytes=872316928 weight_to_token_ratio=0.144448 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.540932649e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61028 payload_rows_remote=53980 dup_rows=4508 dup_rows_remote=4508 dedup_ratio=0.068787 router_tokens_per_expert=[0, 1607, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3498, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 48, 0, 0, 3, 0, 0, 0, 0, 3012, 0, 4096, 0, 0, 529, 65, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6996, 7048, 8192, 6024, 8192, 8192, 8192] dup_rows_per_dst=[3214, 0, 0, 96, 0, 1188, 10, 0] combine_rows_per_src=[8192, 6996, 7048, 8192, 6024, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59374 payload_rows_remote=51182 dup_rows=6162 dup_rows_remote=6150 dedup_ratio=0.094025 router_tokens_per_expert=[0, 2149, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3273, 0, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 14, 0, 0, 5, 0, 0, 0, 0, 2784, 0, 4096, 0, 0, 259, 18, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4298, 6546, 7570, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 28, 5568, 554, 12, 0] combine_rows_per_src=[4298, 6546, 7570, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57772 payload_rows_remote=49580 dup_rows=7764 dup_rows_remote=7758 dedup_ratio=0.118469 router_tokens_per_expert=[0, 1155, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 3356, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 80, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 4096, 0, 0, 1234, 160, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[2312, 6712, 7790, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 152, 4794, 2788, 24, 6] combine_rows_per_src=[2312, 6712, 7790, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=356837376 dispatch_meta_bytes=224440 combine_payload_bytes=356837376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357061816 ingress_bytes=466937856 link_bytes=466937856 token_bytes=713899192 weight_movement_bytes=110100480 total_nvlink_bytes=823999672 weight_to_token_ratio=0.154224 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.120817297e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381323264 dispatch_meta_bytes=228896 combine_payload_bytes=381323264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381552160 ingress_bytes=601524224 link_bytes=601524224 token_bytes=762875424 weight_movement_bytes=220200960 total_nvlink_bytes=983076384 weight_to_token_ratio=0.288646 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046149559e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=386928640 dispatch_meta_bytes=233952 combine_payload_bytes=386928640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387162592 ingress_bytes=497029120 link_bytes=497029120 token_bytes=774091232 weight_movement_bytes=110100480 total_nvlink_bytes=884191712 weight_to_token_ratio=0.142232 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.644153869e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=355389440 dispatch_meta_bytes=229352 combine_payload_bytes=355389440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355618792 ingress_bytes=465489920 link_bytes=465489920 token_bytes=711008232 weight_movement_bytes=110100480 total_nvlink_bytes=821108712 weight_to_token_ratio=0.154851 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.095635308e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=366872576 dispatch_meta_bytes=229328 combine_payload_bytes=366872576 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=367101904 ingress_bytes=476973056 link_bytes=476973056 token_bytes=733974480 weight_movement_bytes=110100480 total_nvlink_bytes=844074960 weight_to_token_ratio=0.150006 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.295345930e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58756 payload_rows_remote=51648 dup_rows=6780 dup_rows_remote=4936 dedup_ratio=0.103455 router_tokens_per_expert=[0, 1553, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3230, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 4096, 0, 0, 636, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7108, 7716, 8192, 4720, 6444, 8192, 8192] dup_rows_per_dst=[3106, 1844, 634, 334, 0, 862, 0, 0] combine_rows_per_src=[8192, 7108, 7716, 8192, 4720, 6444, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65354 payload_rows_remote=57162 dup_rows=182 dup_rows_remote=182 dedup_ratio=0.002777 router_tokens_per_expert=[0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 3994, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 77, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8020, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 28, 0, 0, 0, 0, 0, 154] combine_rows_per_src=[8182, 8192, 8020, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50204 payload_rows_remote=42012 dup_rows=15332 dup_rows_remote=11898 dedup_ratio=0.233948 router_tokens_per_expert=[0, 1717, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3286, 0, 0, 0, 3283, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 9, 0, 0, 6, 0, 0, 0, 0, 2967, 0, 4096, 0, 0, 843, 177, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8070, 7338, 108, 2028, 8100, 8176] dup_rows_per_dst=[3434, 7442, 4430, 14, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8070, 7338, 108, 2028, 8100, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65176 payload_rows_remote=56984 dup_rows=360 dup_rows_remote=350 dedup_ratio=0.005493 router_tokens_per_expert=[0, 4047, 3, 0, 0, 0, 0, 4096, 0, 0, 96, 0, 0, 4096, 4093, 0, 0, 0, 4093, 0, 0, 3964, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 84, 0, 0, 0, 0] payload_rows_per_dst=[8100, 8192, 7932, 8192, 8190, 8192, 8186, 8192] dup_rows_per_dst=[0, 186, 0, 0, 2, 10, 0, 162] combine_rows_per_src=[8100, 8192, 7932, 8192, 8190, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370212864 dispatch_meta_bytes=226336 combine_payload_bytes=370212864 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370439200 ingress_bytes=700514304 link_bytes=700514304 token_bytes=740652064 weight_movement_bytes=330301440 total_nvlink_bytes=1070953504 weight_to_token_ratio=0.445960 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.218309589e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409737216 dispatch_meta_bytes=229376 combine_payload_bytes=409737216 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409966592 ingress_bytes=629938176 link_bytes=629938176 token_bytes=819703808 weight_movement_bytes=220200960 total_nvlink_bytes=1039904768 weight_to_token_ratio=0.268635 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095566095e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65340 payload_rows_remote=57148 dup_rows=196 dup_rows_remote=196 dedup_ratio=0.002991 router_tokens_per_expert=[0, 4071, 1, 0, 0, 0, 0, 4096, 0, 0, 47, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 51, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8192, 8048, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 92, 0, 0, 2, 2, 0, 100] combine_rows_per_src=[8144, 8192, 8048, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65386 payload_rows_remote=57284 dup_rows=150 dup_rows_remote=150 dedup_ratio=0.002289 router_tokens_per_expert=[0, 4070, 1, 0, 0, 0, 0, 4096, 0, 0, 48, 0, 0, 4096, 4092, 0, 0, 0, 4092, 0, 0, 4049, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0] payload_rows_per_dst=[8142, 8192, 8102, 8192, 8188, 8192, 8186, 8192] dup_rows_per_dst=[0, 88, 0, 0, 2, 4, 0, 56] combine_rows_per_src=[8142, 8192, 8102, 8192, 8188, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=301142016 dispatch_meta_bytes=215640 combine_payload_bytes=301142016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301357656 ingress_bytes=411242496 link_bytes=411242496 token_bytes=602499672 weight_movement_bytes=110100480 total_nvlink_bytes=712600152 weight_to_token_ratio=0.182739 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.152183383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63756 payload_rows_remote=55564 dup_rows=1780 dup_rows_remote=1644 dedup_ratio=0.027161 router_tokens_per_expert=[0, 3920, 2, 0, 0, 0, 0, 4096, 0, 0, 237, 0, 0, 4096, 4087, 0, 0, 0, 4092, 0, 0, 3970, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4095, 69, 0, 0, 0, 0] payload_rows_per_dst=[7844, 7008, 7962, 8192, 8186, 8192, 8180, 8192] dup_rows_per_dst=[0, 454, 1152, 0, 24, 14, 0, 136] combine_rows_per_src=[7844, 7008, 7962, 8192, 8186, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65444 payload_rows_remote=57256 dup_rows=92 dup_rows_remote=92 dedup_ratio=0.001404 router_tokens_per_expert=[0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 4096, 4092, 0, 0, 0, 4089, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 28, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8122, 8192, 8188, 8192, 8188, 8192] dup_rows_per_dst=[0, 32, 0, 0, 0, 6, 0, 54] combine_rows_per_src=[8178, 8192, 8122, 8192, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=408461312 dispatch_meta_bytes=229336 combine_payload_bytes=408461312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408690648 ingress_bytes=518561792 link_bytes=518561792 token_bytes=817151960 weight_movement_bytes=110100480 total_nvlink_bytes=927252440 weight_to_token_ratio=0.134737 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.018642450e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409636864 dispatch_meta_bytes=229376 combine_payload_bytes=409636864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409866240 ingress_bytes=519737344 link_bytes=519737344 token_bytes=819503104 weight_movement_bytes=110100480 total_nvlink_bytes=929603584 weight_to_token_ratio=0.134350 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.039087233e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410611712 dispatch_meta_bytes=229736 combine_payload_bytes=410611712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410841448 ingress_bytes=740913152 link_bytes=740913152 token_bytes=821453160 weight_movement_bytes=330301440 total_nvlink_bytes=1151754600 weight_to_token_ratio=0.402094 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288569831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398282752 dispatch_meta_bytes=228832 combine_payload_bytes=398282752 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=398511584 ingress_bytes=728584192 link_bytes=728584192 token_bytes=796794336 weight_movement_bytes=330301440 total_nvlink_bytes=1127095776 weight_to_token_ratio=0.414538 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.267127741e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410411008 dispatch_meta_bytes=229392 combine_payload_bytes=410411008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410640400 ingress_bytes=520511488 link_bytes=520511488 token_bytes=821051408 weight_movement_bytes=110100480 total_nvlink_bytes=931151888 weight_to_token_ratio=0.134097 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.052550871e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65326 payload_rows_remote=57134 dup_rows=210 dup_rows_remote=58 dedup_ratio=0.003204 router_tokens_per_expert=[0, 4031, 0, 0, 0, 0, 0, 4096, 0, 0, 76, 0, 0, 4096, 4096, 0, 0, 0, 4093, 0, 0, 4055, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 0] payload_rows_per_dst=[8062, 8192, 8118, 8192, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 152, 4, 0, 0, 0, 0, 54] combine_rows_per_src=[8062, 8192, 8118, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63050 payload_rows_remote=55260 dup_rows=2486 dup_rows_remote=496 dedup_ratio=0.037933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1080, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2968, 0, 0, 0, 4093, 0, 4092, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7790, 8192, 6116, 8184] dup_rows_per_dst=[0, 0, 4, 0, 1990, 100, 392, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7790, 8192, 6116, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64598 payload_rows_remote=56408 dup_rows=938 dup_rows_remote=244 dedup_ratio=0.014313 router_tokens_per_expert=[0, 4027, 3, 0, 0, 0, 0, 4096, 0, 0, 72, 0, 0, 4096, 4091, 0, 0, 0, 4085, 0, 0, 4037, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 64, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8076, 8192, 8184, 8156, 8186, 7422] dup_rows_per_dst=[694, 152, 0, 0, 0, 0, 38, 54] combine_rows_per_src=[8190, 8192, 8076, 8192, 8184, 8156, 8186, 7422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60886 payload_rows_remote=52694 dup_rows=4650 dup_rows_remote=4450 dedup_ratio=0.070953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 3, 1784, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2184, 0, 0, 0, 4092, 0, 4089, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8174, 3574, 8192, 8192, 8178] dup_rows_per_dst=[6, 0, 76, 0, 0, 200, 4368, 0] combine_rows_per_src=[8192, 8192, 8192, 8174, 3574, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409536512 dispatch_meta_bytes=228768 combine_payload_bytes=409536512 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=409765280 ingress_bytes=739837952 link_bytes=739837952 token_bytes=819301792 weight_movement_bytes=330301440 total_nvlink_bytes=1149603232 weight_to_token_ratio=0.403150 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.286699881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=396103680 dispatch_meta_bytes=223024 combine_payload_bytes=396103680 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=396326704 ingress_bytes=726405120 link_bytes=726405120 token_bytes=792430384 weight_movement_bytes=330301440 total_nvlink_bytes=1122731824 weight_to_token_ratio=0.416821 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263337977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62408 payload_rows_remote=54222 dup_rows=3128 dup_rows_remote=3128 dedup_ratio=0.047729 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 4096, 1, 1487, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2538, 0, 0, 0, 4094, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8186, 8192, 8192, 5076, 8188] dup_rows_per_dst=[2, 0, 22, 0, 2974, 130, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8186, 8192, 8192, 5076, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61648 payload_rows_remote=53456 dup_rows=3888 dup_rows_remote=3798 dedup_ratio=0.059326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4093, 4096, 3, 1718, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2179, 0, 0, 0, 4089, 0, 4077, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8192, 8192, 4358, 8154] dup_rows_per_dst=[4, 0, 90, 0, 3436, 358, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8176, 8192, 8192, 4358, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59442 payload_rows_remote=51272 dup_rows=6094 dup_rows_remote=5682 dedup_ratio=0.092987 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 4091, 4096, 6, 1967, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2014, 0, 0, 0, 4091, 0, 4085, 0, 0] payload_rows_per_dst=[8156, 8192, 7780, 8182, 3962, 6818, 8182, 8170] dup_rows_per_dst=[12, 0, 104, 0, 20, 118, 5428, 412] combine_rows_per_src=[8156, 8192, 7780, 8182, 3962, 6818, 8182, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60408 payload_rows_remote=52216 dup_rows=5128 dup_rows_remote=120 dedup_ratio=0.078247 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 4096, 3, 1537, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2504, 0, 0, 0, 4093, 0, 4093, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 3080, 8192, 8192, 8186] dup_rows_per_dst=[8, 0, 2, 0, 0, 110, 5008, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 3080, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=377710592 dispatch_meta_bytes=228576 combine_payload_bytes=377710592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377939168 ingress_bytes=487811072 link_bytes=487811072 token_bytes=755649760 weight_movement_bytes=110100480 total_nvlink_bytes=865750240 weight_to_token_ratio=0.145703 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.483836853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404332544 dispatch_meta_bytes=226608 combine_payload_bytes=404332544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=404559152 ingress_bytes=624533504 link_bytes=624533504 token_bytes=808891696 weight_movement_bytes=220200960 total_nvlink_bytes=1029092656 weight_to_token_ratio=0.272226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086166482e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=388663296 dispatch_meta_bytes=229400 combine_payload_bytes=388663296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388892696 ingress_bytes=498763776 link_bytes=498763776 token_bytes=777555992 weight_movement_bytes=110100480 total_nvlink_bytes=887656472 weight_to_token_ratio=0.141598 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674322390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383172608 dispatch_meta_bytes=229016 combine_payload_bytes=383172608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383401624 ingress_bytes=493273088 link_bytes=493273088 token_bytes=766574232 weight_movement_bytes=110100480 total_nvlink_bytes=876674712 weight_to_token_ratio=0.143627 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.578830295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=367517696 dispatch_meta_bytes=227816 combine_payload_bytes=367517696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=367745512 ingress_bytes=697819136 link_bytes=697819136 token_bytes=735263208 weight_movement_bytes=330301440 total_nvlink_bytes=1065564648 weight_to_token_ratio=0.449229 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.213622249e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=374284288 dispatch_meta_bytes=209344 combine_payload_bytes=374284288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=374493632 ingress_bytes=704585728 link_bytes=704585728 token_bytes=748777920 weight_movement_bytes=330301440 total_nvlink_bytes=1079079360 weight_to_token_ratio=0.441121 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.225390465e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62470 payload_rows_remote=54278 dup_rows=3066 dup_rows_remote=3066 dedup_ratio=0.046783 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1448, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2573, 0, 0, 0, 4092, 0, 4086, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 5146, 8172] dup_rows_per_dst=[0, 0, 108, 0, 2896, 62, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 5146, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65452 payload_rows_remote=57260 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.001282 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8108, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 84, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65480 payload_rows_remote=57338 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000854 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 33, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8188, 8192, 8142, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 36, 14, 0, 6, 0] combine_rows_per_src=[8192, 8190, 8192, 8188, 8192, 8142, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=389064704 dispatch_meta_bytes=229376 combine_payload_bytes=389064704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389294080 ingress_bytes=499165184 link_bytes=499165184 token_bytes=778358784 weight_movement_bytes=110100480 total_nvlink_bytes=888459264 weight_to_token_ratio=0.141452 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62104 payload_rows_remote=53916 dup_rows=3432 dup_rows_remote=3424 dedup_ratio=0.052368 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 4094, 4, 1415, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2539, 0, 0, 0, 4084, 0, 4084, 0, 0] payload_rows_per_dst=[8188, 8192, 8184, 8186, 8192, 8192, 5078, 7892] dup_rows_per_dst=[8, 0, 16, 66, 3046, 296, 0, 0] combine_rows_per_src=[8188, 8192, 8184, 8186, 8192, 8192, 5078, 7892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410439680 dispatch_meta_bytes=229376 combine_payload_bytes=410439680 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410669056 ingress_bytes=740741120 link_bytes=740741120 token_bytes=821108736 weight_movement_bytes=330301440 total_nvlink_bytes=1151410176 weight_to_token_ratio=0.402263 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288270639e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65422 payload_rows_remote=57230 dup_rows=114 dup_rows_remote=0 dedup_ratio=0.001740 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8080, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 114, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8080, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65424 payload_rows_remote=57234 dup_rows=112 dup_rows_remote=110 dedup_ratio=0.001709 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4067, 0, 77, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4043, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8188, 8192, 8086, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 100, 6, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8190, 8188, 8192, 8086, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410998784 dispatch_meta_bytes=229576 combine_payload_bytes=410998784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411228360 ingress_bytes=741300224 link_bytes=741300224 token_bytes=822227144 weight_movement_bytes=330301440 total_nvlink_bytes=1152528584 weight_to_token_ratio=0.401716 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289243012e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64652 payload_rows_remote=56460 dup_rows=884 dup_rows_remote=884 dedup_ratio=0.013489 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 77, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7376, 8192, 8124, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 50, 16, 812, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 7376, 8192, 8124, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=130 dedup_ratio=0.002075 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4060, 0, 97, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4030, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8060, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 126, 2, 0, 6, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8060, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386469888 dispatch_meta_bytes=229360 combine_payload_bytes=386469888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386699248 ingress_bytes=496570368 link_bytes=496570368 token_bytes=773169136 weight_movement_bytes=110100480 total_nvlink_bytes=883269616 weight_to_token_ratio=0.142402 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.636175417e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7770486040 prefetch_bytes=1321205760 reduce_bytes=1211105280 weight_movement_bytes=2532311040 egress_bytes=3886390552 ingress_bytes=6416406528 link_bytes=6416406528 total_nvlink_bytes=10302797080 weight_to_token_ratio=0.325888 prefetch_share_of_ingress=0.205911 dedup_ratio=0.057788 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=11 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047059 gemm_inner_pad_upper_bound=7747 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.115918627e-04 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410224640 dispatch_meta_bytes=228920 combine_payload_bytes=410224640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410453560 ingress_bytes=630425600 link_bytes=630425600 token_bytes=820678200 weight_movement_bytes=220200960 total_nvlink_bytes=1040879160 weight_to_token_ratio=0.268316 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.096413806e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410253312 dispatch_meta_bytes=229376 combine_payload_bytes=410253312 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=410482688 ingress_bytes=630454272 link_bytes=630454272 token_bytes=820736000 weight_movement_bytes=220200960 total_nvlink_bytes=1040936960 weight_to_token_ratio=0.268297 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.096463671e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7595774968 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3799028728 ingress_bytes=6218956800 link_bytes=6218956800 total_nvlink_bytes=10017985528 weight_to_token_ratio=0.318889 prefetch_share_of_ingress=0.159336 dedup_ratio=0.075333 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.081578872e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404705280 dispatch_meta_bytes=229376 combine_payload_bytes=404705280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404934656 ingress_bytes=514805760 link_bytes=514805760 token_bytes=809639936 weight_movement_bytes=110100480 total_nvlink_bytes=919740416 weight_to_token_ratio=0.135987 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.953318876e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410066944 dispatch_meta_bytes=229352 combine_payload_bytes=410066944 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=410296296 ingress_bytes=520167424 link_bytes=520167424 token_bytes=820363240 weight_movement_bytes=110100480 total_nvlink_bytes=930463720 weight_to_token_ratio=0.134209 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.046567032e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65452 payload_rows_remote=57260 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.001282 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 79, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8108, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 76, 8, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7683472632 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3842886904 ingress_bytes=5602193408 link_bytes=5602193408 total_nvlink_bytes=9445080312 weight_to_token_ratio=0.229272 prefetch_share_of_ingress=0.196531 dedup_ratio=0.063733 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.743135739e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7638200288 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3820251104 ingress_bytes=5799757824 link_bytes=5799757824 total_nvlink_bytes=9620008928 weight_to_token_ratio=0.259460 prefetch_share_of_ingress=0.189836 dedup_ratio=0.070547 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.008673275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7221123736 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3611677336 ingress_bytes=5701355520 link_bytes=5701355520 total_nvlink_bytes=9313032856 weight_to_token_ratio=0.289693 prefetch_share_of_ingress=0.173802 dedup_ratio=0.109836 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.915594961e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7659176888 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=3830719416 ingress_bytes=5590065152 link_bytes=5590065152 total_nvlink_bytes=9420784568 weight_to_token_ratio=0.230000 prefetch_share_of_ingress=0.157566 dedup_ratio=0.067392 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.722042707e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410439680 dispatch_meta_bytes=229376 combine_payload_bytes=410439680 prefetch_bytes=0 reduce_bytes=0 egress_bytes=410669056 ingress_bytes=410439680 link_bytes=410669056 token_bytes=821108736 weight_movement_bytes=0 total_nvlink_bytes=821108736 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.142210318e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65274 payload_rows_remote=57082 dup_rows=262 dup_rows_remote=262 dedup_ratio=0.003998 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 167, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 4010, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8130, 8016, 8172, 8192] dup_rows_per_dst=[0, 0, 4, 242, 16, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8130, 8016, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7424133368 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3713202424 ingress_bytes=6023041024 link_bytes=6023270400 total_nvlink_bytes=9736243448 weight_to_token_ratio=0.311432 prefetch_share_of_ingress=0.146239 dedup_ratio=0.091541 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.047545788e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409163776 dispatch_meta_bytes=229376 combine_payload_bytes=409163776 prefetch_bytes=0 reduce_bytes=0 egress_bytes=409393152 ingress_bytes=409163776 link_bytes=409393152 token_bytes=818556928 weight_movement_bytes=0 total_nvlink_bytes=818556928 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.120020249e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7060796920 prefetch_bytes=880803840 reduce_bytes=550502400 weight_movement_bytes=1431306240 egress_bytes=3531531768 ingress_bytes=4960571392 link_bytes=4960800768 total_nvlink_bytes=8492103160 weight_to_token_ratio=0.202712 prefetch_share_of_ingress=0.177561 dedup_ratio=0.127185 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=5 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.627648447e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-29 23:43:05] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 236.4 | throughput per GPU (TFLOP/s/GPU): 522.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.452604E+00 | loss scale: 1.0 | grad norm: 5.640 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45126 payload_rows_remote=39150 dup_rows=20410 dup_rows_remote=17708 dedup_ratio=0.311432 router_tokens_per_expert=[433, 729, 1062, 1796, 4, 180, 208, 19, 890, 2001, 1465, 924, 0, 716, 0, 1045, 984, 189, 6, 0, 1449, 535, 487, 458, 207, 792, 661, 153, 624, 123, 385, 0, 176, 169, 666, 6, 88, 9, 599, 287, 583, 869, 556, 202, 10, 502, 28, 593, 250, 427, 67, 1471, 350, 1059, 73, 642, 311, 625, 436, 727, 956, 1324, 180, 2] payload_rows_per_dst=[4128, 5378, 4710, 6474, 6762, 5706, 5976, 5992] dup_rows_per_dst=[1142, 1772, 1538, 3008, 4170, 2948, 2702, 3130] combine_rows_per_src=[4128, 5378, 4710, 6474, 6762, 5706, 5976, 5992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45336 payload_rows_remote=39080 dup_rows=20200 dup_rows_remote=16774 dedup_ratio=0.308228 router_tokens_per_expert=[935, 940, 1362, 1853, 0, 106, 153, 20, 354, 1934, 1331, 454, 1, 755, 3, 1021, 1396, 418, 55, 2, 910, 839, 241, 980, 113, 912, 591, 354, 469, 135, 215, 0, 294, 132, 481, 9, 167, 0, 490, 148, 154, 958, 953, 334, 8, 691, 83, 351, 51, 190, 188, 1012, 228, 799, 180, 871, 436, 533, 685, 610, 1469, 1287, 124, 0] payload_rows_per_dst=[6580, 5748, 6256, 4362, 5506, 5088, 6060, 5736] dup_rows_per_dst=[4158, 2090, 3426, 1216, 1804, 1976, 3552, 1978] combine_rows_per_src=[6580, 5748, 6256, 4362, 5506, 5088, 6060, 5736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45214 payload_rows_remote=39932 dup_rows=20322 dup_rows_remote=18740 dedup_ratio=0.310089 router_tokens_per_expert=[765, 778, 1205, 1462, 0, 110, 243, 42, 900, 1738, 1185, 521, 2, 707, 1, 887, 1921, 737, 26, 0, 1196, 655, 376, 763, 128, 843, 719, 281, 386, 219, 779, 0, 91, 269, 691, 18, 106, 0, 404, 115, 520, 985, 312, 146, 8, 814, 181, 398, 124, 535, 108, 1220, 350, 532, 73, 679, 282, 409, 451, 719, 1037, 1542, 74, 0] payload_rows_per_dst=[4708, 5884, 5254, 6440, 5282, 6436, 6330, 4880] dup_rows_per_dst=[1578, 2522, 2452, 3194, 1582, 3934, 3996, 1064] combine_rows_per_src=[4708, 5884, 5254, 6440, 5282, 6436, 6330, 4880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44870 payload_rows_remote=38406 dup_rows=20666 dup_rows_remote=17136 dedup_ratio=0.315338 router_tokens_per_expert=[1225, 1062, 1915, 1656, 0, 160, 100, 17, 221, 1655, 1345, 659, 0, 908, 28, 890, 1505, 343, 39, 1, 1095, 525, 442, 686, 171, 1036, 525, 304, 412, 220, 455, 0, 206, 94, 311, 22, 88, 0, 505, 298, 342, 963, 736, 261, 1, 674, 99, 390, 50, 219, 516, 719, 349, 748, 56, 524, 413, 418, 635, 581, 1049, 1815, 86, 0] payload_rows_per_dst=[5362, 4238, 5278, 6492, 6226, 6234, 4576, 6464] dup_rows_per_dst=[2056, 1174, 2524, 3066, 2822, 3708, 1786, 3530] combine_rows_per_src=[5362, 4238, 5278, 6492, 6226, 6234, 4576, 6464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45890 payload_rows_remote=39916 dup_rows=19646 dup_rows_remote=17598 dedup_ratio=0.299774 router_tokens_per_expert=[768, 1632, 1250, 1485, 0, 44, 89, 21, 1101, 1548, 1138, 408, 0, 639, 1, 1407, 1008, 277, 32, 0, 1700, 578, 311, 490, 207, 526, 514, 413, 317, 182, 427, 1, 193, 290, 707, 24, 103, 0, 756, 351, 241, 966, 744, 561, 18, 405, 34, 551, 104, 137, 338, 861, 190, 552, 90, 817, 293, 760, 762, 870, 1141, 1202, 193, 0] payload_rows_per_dst=[5812, 6214, 6032, 5974, 5630, 4932, 5578, 5718] dup_rows_per_dst=[1918, 3174, 2760, 2048, 2314, 2108, 3004, 2320] combine_rows_per_src=[5812, 6214, 6032, 5974, 5630, 4932, 5578, 5718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45152 payload_rows_remote=39854 dup_rows=20384 dup_rows_remote=17602 dedup_ratio=0.311035 router_tokens_per_expert=[569, 904, 1624, 1482, 1, 41, 377, 73, 409, 1485, 1025, 469, 0, 1086, 4, 1157, 1037, 431, 9, 44, 1155, 661, 521, 293, 282, 1007, 615, 260, 256, 233, 612, 0, 379, 503, 585, 10, 117, 0, 275, 218, 119, 717, 478, 389, 2, 798, 37, 463, 58, 122, 231, 846, 585, 922, 106, 837, 342, 249, 764, 1439, 1526, 1486, 43, 0] payload_rows_per_dst=[5258, 5556, 4670, 6426, 5458, 5298, 5404, 7082] dup_rows_per_dst=[1920, 2342, 1558, 3068, 2088, 2782, 2182, 4444] combine_rows_per_src=[5258, 5556, 4670, 6426, 5458, 5298, 5404, 7082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45616 payload_rows_remote=39496 dup_rows=19920 dup_rows_remote=16640 dedup_ratio=0.303955 router_tokens_per_expert=[484, 694, 1413, 1602, 1, 111, 161, 24, 467, 1455, 1590, 480, 10, 380, 81, 1082, 1706, 256, 35, 0, 1283, 691, 346, 601, 92, 1073, 688, 197, 711, 256, 605, 1, 317, 353, 874, 15, 297, 1, 329, 117, 322, 1161, 953, 179, 6, 747, 42, 199, 139, 135, 368, 452, 281, 1001, 114, 662, 223, 481, 951, 1101, 1301, 962, 106, 3] payload_rows_per_dst=[6120, 6420, 6342, 5334, 5566, 5288, 5578, 4968] dup_rows_per_dst=[3280, 3618, 3494, 1912, 1950, 1930, 2650, 1086] combine_rows_per_src=[6120, 6420, 6342, 5334, 5566, 5288, 5578, 4968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45186 payload_rows_remote=38934 dup_rows=20350 dup_rows_remote=17196 dedup_ratio=0.310516 router_tokens_per_expert=[934, 1048, 1373, 2059, 0, 149, 188, 7, 634, 2032, 1489, 839, 4, 729, 102, 906, 1130, 323, 48, 1, 1504, 496, 358, 761, 108, 1128, 624, 254, 513, 125, 509, 0, 85, 163, 518, 10, 317, 0, 754, 309, 116, 873, 1116, 113, 9, 848, 83, 426, 52, 139, 96, 451, 242, 821, 68, 734, 235, 523, 503, 883, 1315, 478, 113, 0] payload_rows_per_dst=[6658, 6252, 6136, 4834, 5962, 5176, 4662, 5506] dup_rows_per_dst=[4858, 3154, 3106, 1688, 2414, 1992, 1500, 1638] combine_rows_per_src=[6658, 6252, 6136, 4834, 5962, 5176, 4662, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280627200 dispatch_meta_bytes=227432 combine_payload_bytes=280627200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280854632 ingress_bytes=390727680 link_bytes=390727680 token_bytes=561481832 weight_movement_bytes=110100480 total_nvlink_bytes=671582312 weight_to_token_ratio=0.196089 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795396992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=223416 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280348856 ingress_bytes=500326400 link_bytes=500326400 token_bytes=560474296 weight_movement_bytes=220200960 total_nvlink_bytes=780675256 weight_to_token_ratio=0.392883 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701498992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286232576 dispatch_meta_bytes=234688 combine_payload_bytes=286232576 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286467264 ingress_bytes=506433536 link_bytes=506433536 token_bytes=572699840 weight_movement_bytes=220200960 total_nvlink_bytes=792900800 weight_to_token_ratio=0.384496 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.807712131e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275294208 dispatch_meta_bytes=222168 combine_payload_bytes=275294208 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=275516376 ingress_bytes=605595648 link_bytes=605595648 token_bytes=550810584 weight_movement_bytes=330301440 total_nvlink_bytes=881112024 weight_to_token_ratio=0.599664 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.053230435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286117888 dispatch_meta_bytes=230056 combine_payload_bytes=286117888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286347944 ingress_bytes=396218368 link_bytes=396218368 token_bytes=572465832 weight_movement_bytes=110100480 total_nvlink_bytes=682566312 weight_to_token_ratio=0.192327 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.890889087e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285673472 dispatch_meta_bytes=229824 combine_payload_bytes=285673472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285903296 ingress_bytes=395773952 link_bytes=395773952 token_bytes=571576768 weight_movement_bytes=110100480 total_nvlink_bytes=681677248 weight_to_token_ratio=0.192626 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.883159962e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283107328 dispatch_meta_bytes=224544 combine_payload_bytes=283107328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=283331872 ingress_bytes=613408768 link_bytes=613408768 token_bytes=566439200 weight_movement_bytes=330301440 total_nvlink_bytes=896740640 weight_to_token_ratio=0.583119 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.066818736e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=224520 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279303432 ingress_bytes=609380352 link_bytes=609380352 token_bytes=558382344 weight_movement_bytes=330301440 total_nvlink_bytes=888683784 weight_to_token_ratio=0.591533 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059812658e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62144 payload_rows_remote=53952 dup_rows=3392 dup_rows_remote=3388 dedup_ratio=0.051758 router_tokens_per_expert=[0, 0, 0, 4068, 0, 0, 40, 4096, 0, 2230, 384, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 11, 909, 0, 0, 0, 0, 24, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 799, 0, 4096, 0, 0, 3844] payload_rows_per_dst=[8154, 5150, 8156, 8188, 8192, 8188, 8192, 7924] dup_rows_per_dst=[62, 78, 1834, 50, 0, 2, 4, 1362] combine_rows_per_src=[8154, 5150, 8156, 8188, 8192, 8188, 8192, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59740 payload_rows_remote=56178 dup_rows=5796 dup_rows_remote=5310 dedup_ratio=0.088440 router_tokens_per_expert=[0, 1, 0, 4059, 0, 0, 37, 4094, 0, 2486, 201, 0, 1, 0, 1, 0, 0, 0, 0, 4062, 3, 998, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 783, 0, 4088, 0, 2, 3761] payload_rows_per_dst=[8130, 7510, 3562, 8190, 8178, 8184, 8188, 7798] dup_rows_per_dst=[64, 3946, 486, 2, 0, 2, 0, 1296] combine_rows_per_src=[8130, 7510, 3562, 8190, 8178, 8184, 8188, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61028 payload_rows_remote=52836 dup_rows=4508 dup_rows_remote=4506 dedup_ratio=0.068787 router_tokens_per_expert=[0, 0, 0, 4084, 0, 0, 58, 4096, 0, 1368, 692, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1356, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 4096, 0, 0, 3813] payload_rows_per_dst=[8176, 4042, 8180, 8190, 8192, 8192, 8192, 7864] dup_rows_per_dst=[108, 78, 2694, 4, 2, 0, 0, 1622] combine_rows_per_src=[8176, 4042, 8180, 8190, 8192, 8192, 8192, 7864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61910 payload_rows_remote=54734 dup_rows=3626 dup_rows_remote=2952 dedup_ratio=0.055328 router_tokens_per_expert=[0, 0, 2, 4045, 2, 0, 41, 4095, 0, 2484, 549, 0, 3, 0, 0, 1, 0, 0, 0, 4065, 2, 1262, 0, 0, 0, 0, 5, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 4094, 0, 0, 3364] payload_rows_per_dst=[8098, 5710, 8170, 8190, 8188, 8188, 8190, 7176] dup_rows_per_dst=[82, 364, 2488, 16, 2, 0, 0, 674] combine_rows_per_src=[8098, 5710, 8170, 8190, 8188, 8188, 8190, 7176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62352 payload_rows_remote=54188 dup_rows=3184 dup_rows_remote=3180 dedup_ratio=0.048584 router_tokens_per_expert=[0, 0, 0, 3892, 0, 0, 32, 4096, 3, 2094, 913, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 12, 708, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 905, 0, 4094, 0, 0, 3817] payload_rows_per_dst=[7838, 5756, 8056, 8192, 8188, 8164, 8192, 7966] dup_rows_per_dst=[10, 264, 1352, 2, 4, 4, 70, 1478] combine_rows_per_src=[7838, 5756, 8056, 8192, 8188, 8164, 8192, 7966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61854 payload_rows_remote=53668 dup_rows=3682 dup_rows_remote=3666 dedup_ratio=0.056183 router_tokens_per_expert=[0, 0, 0, 4011, 0, 0, 59, 4096, 0, 2297, 159, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 6, 1050, 0, 0, 0, 0, 8, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 881, 0, 4086, 0, 0, 3927] payload_rows_per_dst=[8088, 4894, 8104, 8186, 8172, 8190, 8192, 8028] dup_rows_per_dst=[52, 18, 2004, 16, 0, 0, 4, 1588] combine_rows_per_src=[8088, 4894, 8104, 8186, 8172, 8190, 8192, 8028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57792 payload_rows_remote=49678 dup_rows=7744 dup_rows_remote=7452 dedup_ratio=0.118164 router_tokens_per_expert=[0, 0, 0, 4035, 0, 1, 35, 4094, 0, 2082, 812, 0, 1, 0, 0, 1, 0, 0, 0, 4041, 4, 1079, 0, 0, 0, 0, 31, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 1, 0, 6, 4, 0, 2, 0, 4092, 2, 4, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 762, 0, 4095, 0, 0, 3487] payload_rows_per_dst=[8114, 8184, 7190, 7988, 8192, 8188, 8124, 1812] dup_rows_per_dst=[292, 5690, 1484, 110, 30, 80, 0, 58] combine_rows_per_src=[8114, 8184, 7190, 7988, 8192, 8188, 8124, 1812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=386727936 dispatch_meta_bytes=229360 combine_payload_bytes=386727936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386957296 ingress_bytes=496828416 link_bytes=496828416 token_bytes=773685232 weight_movement_bytes=110100480 total_nvlink_bytes=883785712 weight_to_token_ratio=0.142307 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.640663297e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=378728448 dispatch_meta_bytes=229368 combine_payload_bytes=378728448 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=378957816 ingress_bytes=598929408 link_bytes=598929408 token_bytes=757686264 weight_movement_bytes=220200960 total_nvlink_bytes=977887224 weight_to_token_ratio=0.290623 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041636747e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58612 payload_rows_remote=50420 dup_rows=6924 dup_rows_remote=1264 dedup_ratio=0.105652 router_tokens_per_expert=[0, 0, 0, 4087, 0, 0, 47, 4093, 1, 2321, 509, 0, 0, 0, 0, 8, 0, 0, 1, 4087, 1, 886, 0, 0, 0, 0, 10, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 4096, 0, 0, 3720] payload_rows_per_dst=[8174, 8192, 1776, 8192, 8192, 8154, 8186, 7746] dup_rows_per_dst=[94, 5660, 0, 20, 4, 0, 2, 1144] combine_rows_per_src=[8174, 8192, 1776, 8192, 8192, 8154, 8186, 7746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=402683904 dispatch_meta_bytes=245952 combine_payload_bytes=402683904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402929856 ingress_bytes=622884864 link_bytes=622884864 token_bytes=805613760 weight_movement_bytes=220200960 total_nvlink_bytes=1025814720 weight_to_token_ratio=0.273333 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083299225e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=392333312 dispatch_meta_bytes=230744 combine_payload_bytes=392333312 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=392564056 ingress_bytes=942835712 link_bytes=942835712 token_bytes=784897368 weight_movement_bytes=550502400 total_nvlink_bytes=1335399768 weight_to_token_ratio=0.701369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.639746373e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=388419584 dispatch_meta_bytes=229472 combine_payload_bytes=388419584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388649056 ingress_bytes=498520064 link_bytes=498520064 token_bytes=777068640 weight_movement_bytes=110100480 total_nvlink_bytes=887169120 weight_to_token_ratio=0.141687 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=384692224 dispatch_meta_bytes=229336 combine_payload_bytes=384692224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=384921560 ingress_bytes=604893184 link_bytes=604893184 token_bytes=769613784 weight_movement_bytes=220200960 total_nvlink_bytes=989814744 weight_to_token_ratio=0.286119 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.052008735e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=356091904 dispatch_meta_bytes=228520 combine_payload_bytes=356091904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=356320424 ingress_bytes=576292864 link_bytes=576292864 token_bytes=712412328 weight_movement_bytes=220200960 total_nvlink_bytes=932613288 weight_to_token_ratio=0.309092 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002268074e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=361410560 dispatch_meta_bytes=206736 combine_payload_bytes=361410560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361617296 ingress_bytes=471511040 link_bytes=471511040 token_bytes=723027856 weight_movement_bytes=110100480 total_nvlink_bytes=833128336 weight_to_token_ratio=0.152277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.200352488e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57250 payload_rows_remote=49058 dup_rows=8286 dup_rows_remote=8286 dedup_ratio=0.126434 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4093, 3706, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 4096, 0, 0, 1766, 0, 0, 3397, 0, 0, 0, 0, 83, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2664, 0, 0, 4096] payload_rows_per_dst=[7412, 8150, 8186, 3778, 8192, 7912, 8192, 5428] dup_rows_per_dst=[0, 0, 0, 8, 6960, 1098, 0, 220] combine_rows_per_src=[7412, 8150, 8186, 3778, 8192, 7912, 8192, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60408 payload_rows_remote=53976 dup_rows=5128 dup_rows_remote=5048 dedup_ratio=0.078247 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4095, 3281, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4096, 0, 0, 1571, 0, 0, 3171, 0, 0, 0, 0, 85, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 4096] payload_rows_per_dst=[6562, 8178, 8190, 8192, 6432, 8192, 8192, 6470] dup_rows_per_dst=[0, 0, 0, 3432, 80, 1616, 0, 0] combine_rows_per_src=[6562, 8178, 8190, 8192, 6432, 8192, 8192, 6470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61088 payload_rows_remote=52906 dup_rows=4448 dup_rows_remote=4448 dedup_ratio=0.067871 router_tokens_per_expert=[0, 0, 0, 4096, 3, 4091, 3293, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 4096, 0, 0, 1618, 0, 0, 3617, 0, 0, 0, 1, 110, 1, 371, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3147, 0, 0, 4094] payload_rows_per_dst=[6586, 8176, 8182, 8192, 7276, 8190, 8192, 6294] dup_rows_per_dst=[6, 0, 0, 3518, 182, 742, 0, 0] combine_rows_per_src=[6586, 8176, 8182, 8192, 7276, 8190, 8192, 6294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59276 payload_rows_remote=51084 dup_rows=6260 dup_rows_remote=5436 dedup_ratio=0.095520 router_tokens_per_expert=[0, 0, 0, 4096, 1, 4096, 2976, 0, 0, 4045, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 131, 4096, 0, 1, 2333, 0, 0, 3371, 0, 0, 0, 0, 37, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 0, 4096] payload_rows_per_dst=[5954, 8090, 8192, 7500, 7010, 8192, 8192, 6146] dup_rows_per_dst=[0, 2, 6, 4280, 1148, 824, 0, 0] combine_rows_per_src=[5954, 8090, 8192, 7500, 7010, 8192, 8192, 6146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50860 payload_rows_remote=42668 dup_rows=14676 dup_rows_remote=714 dedup_ratio=0.223938 router_tokens_per_expert=[0, 0, 0, 4096, 2, 4094, 3663, 0, 0, 4069, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 4096, 0, 0, 1214, 0, 0, 3555, 0, 0, 0, 0, 183, 0, 390, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 3137, 0, 0, 4094] payload_rows_per_dst=[7296, 8140, 8188, 2752, 7316, 784, 8192, 8192] dup_rows_per_dst=[2, 0, 32, 14, 664, 0, 2, 13962] combine_rows_per_src=[7296, 8140, 8188, 2752, 7316, 784, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60122 payload_rows_remote=51930 dup_rows=5414 dup_rows_remote=1082 dedup_ratio=0.082611 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4090, 3212, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 145, 4096, 0, 0, 2021, 0, 0, 3149, 0, 0, 0, 0, 52, 0, 515, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 0, 4096] payload_rows_per_dst=[6424, 8176, 8182, 8192, 6354, 8192, 8192, 6410] dup_rows_per_dst=[0, 0, 0, 4332, 48, 1034, 0, 0] combine_rows_per_src=[6424, 8176, 8182, 8192, 6354, 8192, 8192, 6410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57686 payload_rows_remote=49494 dup_rows=7850 dup_rows_remote=1232 dedup_ratio=0.119781 router_tokens_per_expert=[0, 0, 0, 4095, 6, 4094, 3304, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 4095, 0, 2, 1301, 0, 1, 3554, 0, 0, 0, 0, 65, 1, 547, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 1, 0, 4092] payload_rows_per_dst=[8192, 8160, 8188, 2992, 7150, 8186, 8190, 6628] dup_rows_per_dst=[6618, 0, 0, 38, 92, 1102, 0, 0] combine_rows_per_src=[8192, 8160, 8188, 2992, 7150, 8186, 8190, 6628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351647744 dispatch_meta_bytes=229376 combine_payload_bytes=351647744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351877120 ingress_bytes=461748224 link_bytes=461748224 token_bytes=703524864 weight_movement_bytes=110100480 total_nvlink_bytes=813625344 weight_to_token_ratio=0.156498 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.030561061e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=386899968 dispatch_meta_bytes=236096 combine_payload_bytes=386899968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387136064 ingress_bytes=497000448 link_bytes=497000448 token_bytes=774036032 weight_movement_bytes=110100480 total_nvlink_bytes=884136512 weight_to_token_ratio=0.142242 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643655216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58612 payload_rows_remote=50604 dup_rows=6924 dup_rows_remote=6468 dedup_ratio=0.105652 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4095, 3325, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 4096, 0, 9, 2019, 0, 1, 3167, 0, 0, 0, 0, 147, 1, 641, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 4094] payload_rows_per_dst=[7360, 8008, 8190, 6392, 6494, 8190, 8192, 5786] dup_rows_per_dst=[3124, 456, 0, 1924, 138, 1282, 0, 0] combine_rows_per_src=[7360, 8008, 8190, 6392, 6494, 8190, 8192, 5786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=379230208 dispatch_meta_bytes=229416 combine_payload_bytes=379230208 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=379459624 ingress_bytes=599431168 link_bytes=599431168 token_bytes=758689832 weight_movement_bytes=220200960 total_nvlink_bytes=978890792 weight_to_token_ratio=0.290238 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.042509391e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=366170112 dispatch_meta_bytes=226080 combine_payload_bytes=366170112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366396192 ingress_bytes=476270592 link_bytes=476270592 token_bytes=732566304 weight_movement_bytes=110100480 total_nvlink_bytes=842666784 weight_to_token_ratio=0.150294 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.283128925e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=305844224 dispatch_meta_bytes=173528 combine_payload_bytes=305844224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=306017752 ingress_bytes=526045184 link_bytes=526045184 token_bytes=611861976 weight_movement_bytes=220200960 total_nvlink_bytes=832062936 weight_to_token_ratio=0.359887 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.148790945e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=354772992 dispatch_meta_bytes=202904 combine_payload_bytes=354772992 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=354975896 ingress_bytes=795174912 link_bytes=795174912 token_bytes=709748888 weight_movement_bytes=440401920 total_nvlink_bytes=1150150808 weight_to_token_ratio=0.620504 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.382939956e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=372234240 dispatch_meta_bytes=212048 combine_payload_bytes=372234240 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=372446288 ingress_bytes=812636160 link_bytes=812636160 token_bytes=744680528 weight_movement_bytes=440401920 total_nvlink_bytes=1185082448 weight_to_token_ratio=0.591397 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.413307938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=362729472 dispatch_meta_bytes=228288 combine_payload_bytes=362729472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362957760 ingress_bytes=472829952 link_bytes=472829952 token_bytes=725687232 weight_movement_bytes=110100480 total_nvlink_bytes=835787712 weight_to_token_ratio=0.151719 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.223290537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59370 payload_rows_remote=51178 dup_rows=6166 dup_rows_remote=6164 dedup_ratio=0.094086 router_tokens_per_expert=[0, 1493, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4091, 110, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2435, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 4096, 0] payload_rows_per_dst=[2986, 8174, 8182, 7260, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 220, 1010, 0, 4882, 2, 50] combine_rows_per_src=[2986, 8174, 8182, 7260, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60290 payload_rows_remote=52098 dup_rows=5246 dup_rows_remote=5246 dedup_ratio=0.080048 router_tokens_per_expert=[0, 2188, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 4096, 0] payload_rows_per_dst=[6638, 8188, 8192, 6674, 8192, 6022, 8192, 8192] dup_rows_per_dst=[3184, 316, 24, 182, 0, 1526, 2, 12] combine_rows_per_src=[6638, 8188, 8192, 6674, 8192, 6022, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61626 payload_rows_remote=53448 dup_rows=3910 dup_rows_remote=3760 dedup_ratio=0.059662 router_tokens_per_expert=[0, 1489, 6, 0, 0, 4096, 0, 0, 3, 0, 0, 4084, 0, 0, 0, 0, 0, 4089, 75, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 1, 452, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8174, 8178, 7318, 8192, 5188, 8192, 8192] dup_rows_per_dst=[2990, 0, 150, 720, 0, 0, 12, 38] combine_rows_per_src=[8192, 8174, 8178, 7318, 8192, 5188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59256 payload_rows_remote=51066 dup_rows=6280 dup_rows_remote=1436 dedup_ratio=0.095825 router_tokens_per_expert=[0, 1264, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 4088, 32, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2423, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 4096, 0] payload_rows_per_dst=[2528, 8076, 8176, 7710, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 64, 1288, 0, 4844, 14, 70] combine_rows_per_src=[2528, 8076, 8176, 7710, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366843904 dispatch_meta_bytes=229368 combine_payload_bytes=366843904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367073272 ingress_bytes=476944384 link_bytes=476944384 token_bytes=733917176 weight_movement_bytes=110100480 total_nvlink_bytes=844017656 weight_to_token_ratio=0.150018 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.294847276e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59956 payload_rows_remote=51764 dup_rows=5580 dup_rows_remote=1264 dedup_ratio=0.085144 router_tokens_per_expert=[0, 2124, 6, 0, 0, 4096, 0, 0, 1, 0, 0, 4044, 0, 0, 0, 0, 0, 4076, 69, 0, 0, 0, 0, 0, 0, 3385, 0, 2, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8090, 8154, 7028, 8186, 4444, 8192, 7670] dup_rows_per_dst=[4316, 0, 610, 546, 8, 0, 42, 58] combine_rows_per_src=[8192, 8090, 8154, 7028, 8186, 4444, 8192, 7670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59138 payload_rows_remote=50946 dup_rows=6398 dup_rows_remote=6218 dedup_ratio=0.097626 router_tokens_per_expert=[0, 2004, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 4085, 34, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 43, 0, 4094, 0, 0, 0, 4094, 0] payload_rows_per_dst=[4010, 8192, 6792, 7476, 8190, 8192, 8094, 8192] dup_rows_per_dst=[2, 0, 54, 2164, 0, 3998, 0, 180] combine_rows_per_src=[4010, 8192, 6792, 7476, 8190, 8192, 8094, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62132 payload_rows_remote=54280 dup_rows=3404 dup_rows_remote=2388 dedup_ratio=0.051941 router_tokens_per_expert=[0, 1016, 7, 0, 0, 4094, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4089, 136, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2582, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8170, 8178, 7852, 8192, 5164, 8192, 8192] dup_rows_per_dst=[2042, 0, 272, 1016, 0, 12, 14, 48] combine_rows_per_src=[8192, 8170, 8178, 7852, 8192, 5164, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=373438464 dispatch_meta_bytes=229376 combine_payload_bytes=373438464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373667840 ingress_bytes=483538944 link_bytes=483538944 token_bytes=747106304 weight_movement_bytes=110100480 total_nvlink_bytes=857206784 weight_to_token_ratio=0.147369 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.409537521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60880 payload_rows_remote=52724 dup_rows=4656 dup_rows_remote=4656 dedup_ratio=0.071045 router_tokens_per_expert=[0, 2026, 11, 0, 0, 4094, 0, 0, 3, 0, 0, 4075, 0, 0, 0, 0, 0, 4087, 58, 0, 0, 0, 0, 0, 0, 3365, 0, 1, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8156, 8174, 6900, 8192, 4882, 8192, 8192] dup_rows_per_dst=[4070, 0, 116, 392, 0, 0, 12, 66] combine_rows_per_src=[8192, 8156, 8174, 6900, 8192, 4882, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=383115264 dispatch_meta_bytes=228832 combine_payload_bytes=383115264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383344096 ingress_bytes=603316224 link_bytes=603316224 token_bytes=766459360 weight_movement_bytes=220200960 total_nvlink_bytes=986660320 weight_to_token_ratio=0.287296 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049266142e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366041088 dispatch_meta_bytes=210008 combine_payload_bytes=366041088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366251096 ingress_bytes=476141568 link_bytes=476141568 token_bytes=732292184 weight_movement_bytes=110100480 total_nvlink_bytes=842392664 weight_to_token_ratio=0.150350 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.280884986e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=371044352 dispatch_meta_bytes=212112 combine_payload_bytes=371044352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371256464 ingress_bytes=701345792 link_bytes=701345792 token_bytes=742300816 weight_movement_bytes=330301440 total_nvlink_bytes=1072602256 weight_to_token_ratio=0.444970 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.219755684e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=365180928 dispatch_meta_bytes=228656 combine_payload_bytes=365180928 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=365409584 ingress_bytes=805582848 link_bytes=805582848 token_bytes=730590512 weight_movement_bytes=440401920 total_nvlink_bytes=1170992432 weight_to_token_ratio=0.602803 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.401041068e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=389079040 dispatch_meta_bytes=226672 combine_payload_bytes=389079040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389305712 ingress_bytes=499179520 link_bytes=499179520 token_bytes=778384752 weight_movement_bytes=110100480 total_nvlink_bytes=888485232 weight_to_token_ratio=0.141447 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=377925632 dispatch_meta_bytes=229520 combine_payload_bytes=377925632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378155152 ingress_bytes=488026112 link_bytes=488026112 token_bytes=756080784 weight_movement_bytes=110100480 total_nvlink_bytes=866181264 weight_to_token_ratio=0.145620 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.487576753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58296 payload_rows_remote=56012 dup_rows=7240 dup_rows_remote=7164 dedup_ratio=0.110474 router_tokens_per_expert=[0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 28, 0, 3909, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8088, 8192, 7566, 8192, 8192, 7610, 2284, 8172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 76, 7164] combine_rows_per_src=[8088, 8192, 7566, 8192, 8192, 7610, 2284, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63282 payload_rows_remote=55090 dup_rows=2254 dup_rows_remote=2254 dedup_ratio=0.034393 router_tokens_per_expert=[0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 7, 0, 3872, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8096, 8192, 6976, 8192, 8192, 7694, 8192, 7748] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2244, 10] combine_rows_per_src=[8096, 8192, 6976, 8192, 8192, 7694, 8192, 7748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63396 payload_rows_remote=56478 dup_rows=2140 dup_rows_remote=2140 dedup_ratio=0.032654 router_tokens_per_expert=[0, 5, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3458, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 3868, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1047, 33, 0, 3922, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8104, 8190, 6918, 8192, 8190, 7736, 8192, 7874] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 2102, 36] combine_rows_per_src=[8104, 8190, 6918, 8192, 8190, 7736, 8192, 7874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401494016 dispatch_meta_bytes=252704 combine_payload_bytes=401494016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401746720 ingress_bytes=511594496 link_bytes=511594496 token_bytes=803240736 weight_movement_bytes=110100480 total_nvlink_bytes=913341216 weight_to_token_ratio=0.137070 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.897469714e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63060 payload_rows_remote=55838 dup_rows=2476 dup_rows_remote=2476 dedup_ratio=0.037781 router_tokens_per_expert=[0, 6, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 67, 0, 3937, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8100, 8192, 7058, 8192, 8192, 7222, 8192, 7912] dup_rows_per_dst=[10, 0, 0, 4, 0, 0, 2366, 96] combine_rows_per_src=[8100, 8192, 7058, 8192, 8192, 7222, 8192, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56720 payload_rows_remote=48586 dup_rows=8816 dup_rows_remote=8154 dedup_ratio=0.134521 router_tokens_per_expert=[0, 16, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3422, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 4092, 3835, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1037, 48, 0, 3962, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8134, 8190, 8058, 8192, 8190, 7670, 8190, 96] dup_rows_per_dst=[662, 6, 6056, 2, 0, 6, 2084, 0] combine_rows_per_src=[8134, 8190, 8058, 8192, 8190, 7670, 8190, 96] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57454 payload_rows_remote=49262 dup_rows=8082 dup_rows_remote=10 dedup_ratio=0.123322 router_tokens_per_expert=[0, 5, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3444, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 4094, 3716, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1133, 67, 0, 3969, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8102, 8190, 6892, 8184, 8190, 7434, 2270, 8192] dup_rows_per_dst=[2, 0, 0, 0, 4, 4, 0, 8072] combine_rows_per_src=[8102, 8190, 6892, 8184, 8190, 7434, 2270, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=394885120 dispatch_meta_bytes=229376 combine_payload_bytes=394885120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395114496 ingress_bytes=615086080 link_bytes=615086080 token_bytes=789999616 weight_movement_bytes=220200960 total_nvlink_bytes=1010200576 weight_to_token_ratio=0.278736 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069735857e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63892 payload_rows_remote=55700 dup_rows=1644 dup_rows_remote=1644 dedup_ratio=0.025085 router_tokens_per_expert=[0, 1, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 755, 89, 0, 3924, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8120, 8192, 7338, 8192, 8192, 7766, 8192, 7900] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 1516, 126] combine_rows_per_src=[8120, 8192, 7338, 8192, 8192, 7766, 8192, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58658 payload_rows_remote=50466 dup_rows=6878 dup_rows_remote=6878 dedup_ratio=0.104950 router_tokens_per_expert=[0, 6, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 3836, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 20, 0, 3933, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8116, 8192, 7648, 8192, 8190, 7996, 8192, 2132] dup_rows_per_dst=[0, 0, 1026, 2, 4, 4396, 1444, 6] combine_rows_per_src=[8116, 8192, 7648, 8192, 8190, 7996, 8192, 2132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404834304 dispatch_meta_bytes=234472 combine_payload_bytes=404834304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405068776 ingress_bytes=514934784 link_bytes=514934784 token_bytes=809903080 weight_movement_bytes=110100480 total_nvlink_bytes=920003560 weight_to_token_ratio=0.135943 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.955562816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=400246784 dispatch_meta_bytes=233256 combine_payload_bytes=400246784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400480040 ingress_bytes=510347264 link_bytes=510347264 token_bytes=800726824 weight_movement_bytes=110100480 total_nvlink_bytes=910827304 weight_to_token_ratio=0.137501 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.875778298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353110016 dispatch_meta_bytes=197088 combine_payload_bytes=353110016 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=353307104 ingress_bytes=1013712896 link_bytes=1013712896 token_bytes=706417120 weight_movement_bytes=660602880 total_nvlink_bytes=1367020000 weight_to_token_ratio=0.935146 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.763013453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348264448 dispatch_meta_bytes=226960 combine_payload_bytes=348264448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348491408 ingress_bytes=458364928 link_bytes=458364928 token_bytes=696755856 weight_movement_bytes=110100480 total_nvlink_bytes=806856336 weight_to_token_ratio=0.158019 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.971719979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=399257600 dispatch_meta_bytes=229376 combine_payload_bytes=399257600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399486976 ingress_bytes=509358080 link_bytes=509358080 token_bytes=798744576 weight_movement_bytes=110100480 total_nvlink_bytes=908845056 weight_to_token_ratio=0.137842 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.858574761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=361740288 dispatch_meta_bytes=229376 combine_payload_bytes=361740288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361969664 ingress_bytes=471840768 link_bytes=471840768 token_bytes=723709952 weight_movement_bytes=110100480 total_nvlink_bytes=833810432 weight_to_token_ratio=0.152133 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.206087000e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62270 payload_rows_remote=54078 dup_rows=3266 dup_rows_remote=136 dedup_ratio=0.049835 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 4094, 0, 0, 1572, 0, 0, 0, 0, 68, 0, 4096, 0] payload_rows_per_dst=[4930, 8192, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3130, 136] combine_rows_per_src=[4930, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62886 payload_rows_remote=54694 dup_rows=2650 dup_rows_remote=2650 dedup_ratio=0.040436 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4095, 0, 0, 1320, 0, 0, 0, 0, 6, 0, 4096, 0] payload_rows_per_dst=[5544, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2638, 12] combine_rows_per_src=[5544, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61998 payload_rows_remote=53806 dup_rows=3538 dup_rows_remote=3538 dedup_ratio=0.053986 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4093, 0, 5, 1104, 0, 0, 0, 0, 26, 0, 4096, 0] payload_rows_per_dst=[6476, 8192, 8192, 8192, 8192, 8186, 6376, 8192] dup_rows_per_dst=[1754, 0, 0, 2, 2, 56, 1672, 52] combine_rows_per_src=[6476, 8192, 8192, 8192, 8192, 8186, 6376, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387631104 dispatch_meta_bytes=216856 combine_payload_bytes=387631104 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=387847960 ingress_bytes=938133504 link_bytes=938133504 token_bytes=775479064 weight_movement_bytes=550502400 total_nvlink_bytes=1325981464 weight_to_token_ratio=0.709887 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.631568460e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64198 payload_rows_remote=56008 dup_rows=1338 dup_rows_remote=1338 dedup_ratio=0.020416 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 4095, 0, 0, 549, 0, 0, 0, 0, 123, 0, 4096, 0] payload_rows_per_dst=[6926, 8192, 8192, 8192, 8192, 8190, 8122, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1092, 246] combine_rows_per_src=[6926, 8192, 8192, 8192, 8192, 8190, 8122, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63646 payload_rows_remote=55454 dup_rows=1890 dup_rows_remote=1792 dedup_ratio=0.028839 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4095, 0, 2, 894, 0, 0, 0, 1, 29, 0, 4096, 0] payload_rows_per_dst=[6346, 8192, 8192, 8192, 8154, 8190, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 1788, 98] combine_rows_per_src=[6346, 8192, 8192, 8192, 8154, 8190, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59612 payload_rows_remote=51420 dup_rows=5924 dup_rows_remote=206 dedup_ratio=0.090393 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 0, 4091, 0, 2, 1188, 0, 0, 0, 0, 43, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8182, 3032, 7440] dup_rows_per_dst=[5718, 0, 0, 0, 20, 0, 134, 52] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8182, 3032, 7440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392046592 dispatch_meta_bytes=229376 combine_payload_bytes=392046592 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=392275968 ingress_bytes=502147072 link_bytes=502147072 token_bytes=784322560 weight_movement_bytes=110100480 total_nvlink_bytes=894423040 weight_to_token_ratio=0.140377 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.733163472e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63824 payload_rows_remote=55632 dup_rows=1712 dup_rows_remote=1712 dedup_ratio=0.026123 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4095, 0, 0, 826, 0, 0, 0, 0, 33, 0, 4096, 0] payload_rows_per_dst=[6492, 8192, 8192, 8192, 8192, 8190, 8182, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1646, 66] combine_rows_per_src=[6492, 8192, 8192, 8192, 8192, 8190, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60468 payload_rows_remote=52276 dup_rows=5068 dup_rows_remote=5068 dedup_ratio=0.077332 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4047, 0, 4081, 0, 0, 1576, 0, 0, 0, 0, 45, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8164, 3152, 8192] dup_rows_per_dst=[4966, 0, 0, 0, 12, 0, 0, 90] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8164, 3152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=385681408 dispatch_meta_bytes=229376 combine_payload_bytes=385681408 prefetch_bytes=0 reduce_bytes=0 egress_bytes=385910784 ingress_bytes=385681408 link_bytes=385910784 token_bytes=771592192 weight_movement_bytes=0 total_nvlink_bytes=771592192 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.711623248e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=401465344 dispatch_meta_bytes=229384 combine_payload_bytes=401465344 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401694728 ingress_bytes=621666304 link_bytes=621666304 token_bytes=803160072 weight_movement_bytes=220200960 total_nvlink_bytes=1023361032 weight_to_token_ratio=0.274168 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081179949e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368578560 dispatch_meta_bytes=206504 combine_payload_bytes=368578560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=368785064 ingress_bytes=588779520 link_bytes=588779520 token_bytes=737363624 weight_movement_bytes=220200960 total_nvlink_bytes=957564584 weight_to_token_ratio=0.298633 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.023984423e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=397494272 dispatch_meta_bytes=228984 combine_payload_bytes=397494272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397723256 ingress_bytes=617695232 link_bytes=617695232 token_bytes=795217528 weight_movement_bytes=220200960 total_nvlink_bytes=1015418488 weight_to_token_ratio=0.276907 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074273602e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=398770176 dispatch_meta_bytes=229376 combine_payload_bytes=398770176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398999552 ingress_bytes=508870656 link_bytes=508870656 token_bytes=797769728 weight_movement_bytes=110100480 total_nvlink_bytes=907870208 weight_to_token_ratio=0.138010 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.850097656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374714368 dispatch_meta_bytes=229376 combine_payload_bytes=374714368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374943744 ingress_bytes=484814848 link_bytes=484814848 token_bytes=749658112 weight_movement_bytes=110100480 total_nvlink_bytes=859758592 weight_to_token_ratio=0.146868 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.431727590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63718 payload_rows_remote=56532 dup_rows=1818 dup_rows_remote=1818 dedup_ratio=0.027740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3690, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 106, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3593, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7380, 8192, 8192, 8192, 8192, 7186, 8192] dup_rows_per_dst=[0, 0, 0, 1606, 0, 212, 0, 0] combine_rows_per_src=[8192, 7380, 8192, 8192, 8192, 8192, 7186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63500 payload_rows_remote=55308 dup_rows=2036 dup_rows_remote=2036 dedup_ratio=0.031067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3792, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 48, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3382, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7584, 8192, 8192, 8192, 8192, 6764, 8192] dup_rows_per_dst=[0, 0, 0, 1940, 0, 96, 0, 0] combine_rows_per_src=[8192, 7584, 8192, 8192, 8192, 8192, 6764, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64900 payload_rows_remote=56712 dup_rows=636 dup_rows_remote=636 dedup_ratio=0.009705 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 76, 0, 4096, 4094, 0, 0, 0, 0, 4096, 3915, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7922, 8188, 8192, 8192, 8192, 7830, 8192] dup_rows_per_dst=[0, 0, 0, 492, 0, 140, 0, 4] combine_rows_per_src=[8192, 7922, 8188, 8192, 8192, 8192, 7830, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64606 payload_rows_remote=56414 dup_rows=930 dup_rows_remote=788 dedup_ratio=0.014191 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3834, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 71, 0, 4087, 4096, 0, 0, 0, 0, 4096, 3902, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7668, 8192, 8192, 8192, 8192, 7804, 8174] dup_rows_per_dst=[0, 0, 0, 788, 0, 142, 0, 0] combine_rows_per_src=[8192, 7668, 8192, 8192, 8192, 8192, 7804, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60970 payload_rows_remote=52778 dup_rows=4566 dup_rows_remote=4566 dedup_ratio=0.069672 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3805, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 61, 0, 4095, 4093, 0, 0, 0, 0, 4096, 3819, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7984, 8192, 5470, 8190, 7128, 7624, 8190] dup_rows_per_dst=[0, 3762, 12, 670, 2, 112, 0, 8] combine_rows_per_src=[8192, 7984, 8192, 5470, 8190, 7128, 7624, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59792 payload_rows_remote=51602 dup_rows=5744 dup_rows_remote=5740 dedup_ratio=0.087646 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3918, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 77, 0, 4095, 4094, 0, 0, 0, 0, 4096, 3691, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7838, 8188, 3222, 8190, 8188, 7784, 8190] dup_rows_per_dst=[0, 2, 0, 564, 0, 154, 5020, 4] combine_rows_per_src=[8192, 7838, 8188, 3222, 8190, 8188, 7784, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64918 payload_rows_remote=56726 dup_rows=618 dup_rows_remote=56 dedup_ratio=0.009430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3873, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 27, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4011, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7746, 8192, 8192, 8192, 8190, 8022, 8192] dup_rows_per_dst=[0, 0, 0, 562, 2, 54, 0, 0] combine_rows_per_src=[8192, 7746, 8192, 8192, 8192, 8190, 8022, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64296 payload_rows_remote=56646 dup_rows=1240 dup_rows_remote=1240 dedup_ratio=0.018921 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3824, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 84, 0, 4096, 4094, 0, 0, 0, 0, 4096, 3751, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7650, 8188, 8192, 8192, 8188, 7502, 8192] dup_rows_per_dst=[0, 0, 0, 1070, 0, 164, 0, 6] combine_rows_per_src=[8192, 7650, 8188, 8192, 8192, 8188, 7502, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405221376 dispatch_meta_bytes=233400 combine_payload_bytes=405221376 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=405454776 ingress_bytes=955723776 link_bytes=955723776 token_bytes=810676152 weight_movement_bytes=550502400 total_nvlink_bytes=1361178552 weight_to_token_ratio=0.679066 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.662160836e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=396447744 dispatch_meta_bytes=229376 combine_payload_bytes=396447744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396677120 ingress_bytes=506548224 link_bytes=506548224 token_bytes=793124864 weight_movement_bytes=110100480 total_nvlink_bytes=903225344 weight_to_token_ratio=0.138819 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.809706744e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406511616 dispatch_meta_bytes=229392 combine_payload_bytes=406511616 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=406741008 ingress_bytes=626712576 link_bytes=626712576 token_bytes=813252624 weight_movement_bytes=220200960 total_nvlink_bytes=1033453584 weight_to_token_ratio=0.270766 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089956246e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404375552 dispatch_meta_bytes=228808 combine_payload_bytes=404375552 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=404604360 ingress_bytes=514476032 link_bytes=514476032 token_bytes=808979912 weight_movement_bytes=110100480 total_nvlink_bytes=919080392 weight_to_token_ratio=0.136098 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.947584364e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378312704 dispatch_meta_bytes=229376 combine_payload_bytes=378312704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378542080 ingress_bytes=488413184 link_bytes=488413184 token_bytes=756854784 weight_movement_bytes=110100480 total_nvlink_bytes=866955264 weight_to_token_ratio=0.145471 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.494308571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=369883136 dispatch_meta_bytes=229368 combine_payload_bytes=369883136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370112504 ingress_bytes=479983616 link_bytes=479983616 token_bytes=739995640 weight_movement_bytes=110100480 total_nvlink_bytes=850096120 weight_to_token_ratio=0.148785 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.347704520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406038528 dispatch_meta_bytes=231544 combine_payload_bytes=406038528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406270072 ingress_bytes=626239488 link_bytes=626239488 token_bytes=812308600 weight_movement_bytes=220200960 total_nvlink_bytes=1032509560 weight_to_token_ratio=0.271080 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089133468e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406611968 dispatch_meta_bytes=227128 combine_payload_bytes=406611968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=406839096 ingress_bytes=736913408 link_bytes=736913408 token_bytes=813451064 weight_movement_bytes=330301440 total_nvlink_bytes=1143752504 weight_to_token_ratio=0.406050 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.281613618e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 82, 0, 3914, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 8, 0, 4023, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65532 payload_rows_remote=57342 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 70, 0, 3991, 0, 4096] payload_rows_per_dst=[8190, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 31, 0, 3764, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=2 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 113, 0, 3898, 0, 4096] payload_rows_per_dst=[8192, 8190, 8190, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8190, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=6 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 24, 0, 4010, 0, 4096] payload_rows_per_dst=[8190, 8188, 8192, 8180, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 2, 0, 0, 0, 14] combine_rows_per_src=[8190, 8188, 8192, 8180, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=12 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 66, 0, 3933, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 2, 0, 12, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 28, 0, 3973, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=411027456 dispatch_meta_bytes=229384 combine_payload_bytes=411027456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411256840 ingress_bytes=631228416 link_bytes=631228416 token_bytes=822284296 weight_movement_bytes=220200960 total_nvlink_bytes=1042485256 weight_to_token_ratio=0.267792 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097810035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=411041792 link_bytes=411271168 token_bytes=822312960 weight_movement_bytes=0 total_nvlink_bytes=822312960 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.152682036e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410998784 dispatch_meta_bytes=229360 combine_payload_bytes=410998784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411228144 ingress_bytes=521099264 link_bytes=521099264 token_bytes=822226928 weight_movement_bytes=110100480 total_nvlink_bytes=932327408 weight_to_token_ratio=0.133905 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.062773262e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=229320 combine_payload_bytes=410898432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411127752 ingress_bytes=631099392 link_bytes=631099392 token_bytes=822026184 weight_movement_bytes=220200960 total_nvlink_bytes=1042227144 weight_to_token_ratio=0.267876 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097585641e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410941440 dispatch_meta_bytes=229368 combine_payload_bytes=410941440 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411170808 ingress_bytes=741242880 link_bytes=741242880 token_bytes=822112248 weight_movement_bytes=330301440 total_nvlink_bytes=1152413688 weight_to_token_ratio=0.401772 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289143282e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=411271168 ingress_bytes=631242752 link_bytes=631242752 token_bytes=822312960 weight_movement_bytes=220200960 total_nvlink_bytes=1042513920 weight_to_token_ratio=0.267782 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097834967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62314 payload_rows_remote=55000 dup_rows=3222 dup_rows_remote=1092 dedup_ratio=0.049164 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3515, 4096, 3964, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 12, 0, 0] payload_rows_per_dst=[8192, 8192, 6136, 8192, 7928, 8192, 7314, 8168] dup_rows_per_dst=[0, 0, 1068, 0, 0, 0, 2130, 24] combine_rows_per_src=[8192, 8192, 6136, 8192, 7928, 8192, 7314, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394240000 dispatch_meta_bytes=224368 combine_payload_bytes=394240000 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=394464368 ingress_bytes=834641920 link_bytes=834641920 token_bytes=788704368 weight_movement_bytes=440401920 total_nvlink_bytes=1229106288 weight_to_token_ratio=0.558387 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.451579574e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64426 payload_rows_remote=56806 dup_rows=1110 dup_rows_remote=1110 dedup_ratio=0.016937 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3876, 4096, 3810, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7620, 8192, 7752, 8094] dup_rows_per_dst=[0, 0, 1102, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 7620, 8192, 7752, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64488 payload_rows_remote=56296 dup_rows=1048 dup_rows_remote=58 dedup_ratio=0.015991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 495, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3720, 4096, 3956, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 29, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7912, 8192, 7440, 8176] dup_rows_per_dst=[0, 0, 990, 0, 0, 2, 0, 56] combine_rows_per_src=[8192, 8192, 8192, 8192, 7912, 8192, 7440, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63442 payload_rows_remote=55250 dup_rows=2094 dup_rows_remote=2094 dedup_ratio=0.031952 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1019, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3265, 4096, 3893, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 28, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7786, 8192, 6530, 8166] dup_rows_per_dst=[0, 0, 2038, 0, 0, 0, 0, 56] combine_rows_per_src=[8192, 8192, 8192, 8192, 7786, 8192, 6530, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61600 payload_rows_remote=53408 dup_rows=3936 dup_rows_remote=3936 dedup_ratio=0.060059 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 3483, 4096, 3997, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 19, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8138, 8184, 4368, 8142] dup_rows_per_dst=[0, 0, 1444, 2, 2454, 0, 0, 36] combine_rows_per_src=[8192, 8192, 8192, 8192, 8138, 8184, 4368, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58346 payload_rows_remote=50236 dup_rows=7190 dup_rows_remote=6944 dedup_ratio=0.109711 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 3553, 4096, 3991, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 30, 0, 0] payload_rows_per_dst=[8192, 8192, 1316, 8192, 7982, 8192, 8170, 8110] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 6940, 246] combine_rows_per_src=[8192, 8192, 1316, 8192, 7982, 8192, 8170, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63890 payload_rows_remote=55698 dup_rows=1646 dup_rows_remote=1646 dedup_ratio=0.025116 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3554, 4096, 3855, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 7710, 8192, 7108, 8114] dup_rows_per_dst=[0, 0, 1636, 0, 0, 2, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8190, 7710, 8192, 7108, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64872 payload_rows_remote=56680 dup_rows=664 dup_rows_remote=664 dedup_ratio=0.010132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3892, 4096, 4003, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8006, 8192, 7784, 8122] dup_rows_per_dst=[0, 0, 654, 0, 0, 0, 0, 10] combine_rows_per_src=[8192, 8192, 8192, 8192, 8006, 8192, 7784, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=407185408 dispatch_meta_bytes=231664 combine_payload_bytes=407185408 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=407417072 ingress_bytes=627386368 link_bytes=627386368 token_bytes=814602480 weight_movement_bytes=220200960 total_nvlink_bytes=1034803440 weight_to_token_ratio=0.270317 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091128081e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=396032000 dispatch_meta_bytes=229376 combine_payload_bytes=396032000 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=396261376 ingress_bytes=616232960 link_bytes=616232960 token_bytes=792293376 weight_movement_bytes=220200960 total_nvlink_bytes=1012494336 weight_to_token_ratio=0.277929 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071730470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403529728 dispatch_meta_bytes=225416 combine_payload_bytes=403529728 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403755144 ingress_bytes=733831168 link_bytes=733831168 token_bytes=807284872 weight_movement_bytes=330301440 total_nvlink_bytes=1137586312 weight_to_token_ratio=0.409151 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.276253096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=360091648 dispatch_meta_bytes=228720 combine_payload_bytes=360091648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360320368 ingress_bytes=470192128 link_bytes=470192128 token_bytes=720412016 weight_movement_bytes=110100480 total_nvlink_bytes=830512496 weight_to_token_ratio=0.152830 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.177414439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=382828544 dispatch_meta_bytes=229376 combine_payload_bytes=382828544 prefetch_bytes=0 reduce_bytes=0 egress_bytes=383057920 ingress_bytes=382828544 link_bytes=383057920 token_bytes=765886464 weight_movement_bytes=0 total_nvlink_bytes=765886464 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.662007251e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399243264 dispatch_meta_bytes=229376 combine_payload_bytes=399243264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399472640 ingress_bytes=509343744 link_bytes=509343744 token_bytes=798715904 weight_movement_bytes=110100480 total_nvlink_bytes=908816384 weight_to_token_ratio=0.137847 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.858325435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=406282240 dispatch_meta_bytes=229376 combine_payload_bytes=406282240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406511616 ingress_bytes=516382720 link_bytes=516382720 token_bytes=812793856 weight_movement_bytes=110100480 total_nvlink_bytes=922894336 weight_to_token_ratio=0.135459 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.980744804e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[6, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[12, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410955776 dispatch_meta_bytes=229376 combine_payload_bytes=410955776 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=411185152 ingress_bytes=631156736 link_bytes=631156736 token_bytes=822140928 weight_movement_bytes=220200960 total_nvlink_bytes=1042341888 weight_to_token_ratio=0.267838 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097685371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65478 payload_rows_remote=57286 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000885 router_tokens_per_expert=[29, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8134, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[58, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8134, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[7, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8190, 8192, 8192, 8190] dup_rows_per_dst=[14, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8190, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65262 payload_rows_remote=57074 dup_rows=274 dup_rows_remote=266 dedup_ratio=0.004181 router_tokens_per_expert=[6, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7942, 8174, 8192, 8190, 8192, 8192, 8192, 8188] dup_rows_per_dst=[12, 214, 0, 36, 0, 0, 4, 8] combine_rows_per_src=[7942, 8174, 8192, 8190, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=6 dedup_ratio=0.001221 router_tokens_per_expert=[34, 0, 0, 0, 0, 4095, 0, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8124, 8192, 8192, 8192, 8188, 8184, 8192] dup_rows_per_dst=[74, 0, 0, 0, 4, 2, 0, 0] combine_rows_per_src=[8192, 8124, 8192, 8192, 8192, 8188, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65486 payload_rows_remote=57326 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[23, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 8192, 8178, 8190, 8192, 8192, 8190] dup_rows_per_dst=[46, 0, 0, 0, 0, 2, 2, 0] combine_rows_per_src=[8192, 8160, 8192, 8178, 8190, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65502 payload_rows_remote=57330 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[17, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8172, 8192, 8192, 8192, 8192] dup_rows_per_dst=[34, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8172, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7595163312 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3798732464 ingress_bytes=6218641408 link_bytes=6218641408 total_nvlink_bytes=10017373872 weight_to_token_ratio=0.318915 prefetch_share_of_ingress=0.159344 dedup_ratio=0.082111 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.052206 gemm_inner_pad_upper_bound=5334 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.081524020e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410626048 dispatch_meta_bytes=229376 combine_payload_bytes=410626048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410855424 ingress_bytes=630827008 link_bytes=630827008 token_bytes=821481472 weight_movement_bytes=220200960 total_nvlink_bytes=1041682432 weight_to_token_ratio=0.268053 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097111920e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=229376 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411156480 ingress_bytes=521027584 link_bytes=521027584 token_bytes=822083584 weight_movement_bytes=110100480 total_nvlink_bytes=932184064 weight_to_token_ratio=0.133929 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061526629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409106432 dispatch_meta_bytes=229360 combine_payload_bytes=409106432 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409335792 ingress_bytes=629307392 link_bytes=629307392 token_bytes=818442224 weight_movement_bytes=220200960 total_nvlink_bytes=1038643184 weight_to_token_ratio=0.269049 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094469058e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410468352 dispatch_meta_bytes=229080 combine_payload_bytes=410468352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410697432 ingress_bytes=740769792 link_bytes=740769792 token_bytes=821165784 weight_movement_bytes=330301440 total_nvlink_bytes=1151467224 weight_to_token_ratio=0.402235 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288320504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410912768 dispatch_meta_bytes=229504 combine_payload_bytes=410912768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411142272 ingress_bytes=741214208 link_bytes=741214208 token_bytes=822055040 weight_movement_bytes=330301440 total_nvlink_bytes=1152356480 weight_to_token_ratio=0.401800 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289093417e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410941440 dispatch_meta_bytes=229456 combine_payload_bytes=410941440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411170896 ingress_bytes=631142400 link_bytes=631142400 token_bytes=822112336 weight_movement_bytes=220200960 total_nvlink_bytes=1042313296 weight_to_token_ratio=0.267848 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097660439e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7678203880 prefetch_bytes=1321205760 reduce_bytes=220200960 weight_movement_bytes=1541406720 egress_bytes=3840255976 ingress_bytes=5379354624 link_bytes=5379354624 total_nvlink_bytes=9219610600 weight_to_token_ratio=0.200751 prefetch_share_of_ingress=0.245607 dedup_ratio=0.066000 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=2 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.355582443e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7662458512 prefetch_bytes=880803840 reduce_bytes=440401920 weight_movement_bytes=1321205760 egress_bytes=3832366736 ingress_bytes=5151297536 link_bytes=5151526912 total_nvlink_bytes=8983664272 weight_to_token_ratio=0.172426 prefetch_share_of_ingress=0.170987 dedup_ratio=0.065619 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=4 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.959352581e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7737637896 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3869971464 ingress_bytes=5739374592 link_bytes=5739603968 total_nvlink_bytes=9609346056 weight_to_token_ratio=0.241897 prefetch_share_of_ingress=0.211017 dedup_ratio=0.063684 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.052206 gemm_inner_pad_upper_bound=5334 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.982115303e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7280671152 prefetch_bytes=770703360 reduce_bytes=2312110080 weight_movement_bytes=3082813440 egress_bytes=3641434544 ingress_bytes=6722050048 link_bytes=6722050048 total_nvlink_bytes=10363484592 weight_to_token_ratio=0.423424 prefetch_share_of_ingress=0.114653 dedup_ratio=0.104623 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=21 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.169075062e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7331154672 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3666686704 ingress_bytes=5866477568 link_bytes=5866706944 total_nvlink_bytes=9533164272 weight_to_token_ratio=0.300363 prefetch_share_of_ingress=0.168910 dedup_ratio=0.098300 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.020316828e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7491738072 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3747002840 ingress_bytes=5616443392 link_bytes=5616443392 total_nvlink_bytes=9363446232 weight_to_token_ratio=0.249836 prefetch_share_of_ingress=0.196032 dedup_ratio=0.082016 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.767918805e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7732329416 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3867300808 ingress_bytes=5846837248 link_bytes=5846837248 total_nvlink_bytes=9714138056 weight_to_token_ratio=0.256302 prefetch_share_of_ingress=0.150646 dedup_ratio=0.056180 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.016861161e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-29 23:43:05] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 237.7 | throughput per GPU (TFLOP/s/GPU): 519.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.984210E+00 | loss scale: 1.0 | grad norm: 4.583 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44894 payload_rows_remote=38806 dup_rows=20642 dup_rows_remote=17176 dedup_ratio=0.314972 router_tokens_per_expert=[1534, 965, 1189, 1101, 36, 159, 352, 13, 600, 1289, 775, 151, 0, 1174, 31, 778, 1211, 424, 13, 0, 684, 276, 619, 969, 106, 1175, 371, 280, 930, 113, 658, 2, 353, 301, 1117, 10, 800, 0, 891, 246, 212, 793, 509, 753, 1, 501, 34, 490, 174, 431, 147, 522, 215, 885, 163, 329, 858, 964, 504, 312, 983, 1142, 150, 0] payload_rows_per_dst=[5580, 5270, 5776, 6088, 5138, 5858, 5660, 5524] dup_rows_per_dst=[2380, 1748, 2616, 3466, 2298, 3306, 2810, 2018] combine_rows_per_src=[5580, 5270, 5776, 6088, 5138, 5858, 5660, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38956 dup_rows=21222 dup_rows_remote=19372 dedup_ratio=0.323822 router_tokens_per_expert=[1416, 831, 1079, 957, 5, 626, 411, 69, 531, 1267, 658, 325, 0, 1174, 159, 666, 775, 570, 8, 11, 724, 203, 782, 531, 336, 1062, 322, 278, 527, 200, 826, 1, 448, 548, 691, 17, 810, 4, 893, 223, 223, 947, 338, 942, 4, 639, 25, 404, 138, 227, 573, 562, 296, 927, 245, 269, 475, 615, 1148, 257, 821, 1580, 149, 0] payload_rows_per_dst=[6458, 5334, 5358, 6312, 4984, 5890, 4630, 5348] dup_rows_per_dst=[4330, 1906, 1850, 3952, 2284, 3474, 1844, 1582] combine_rows_per_src=[6458, 5334, 5358, 6312, 4984, 5890, 4630, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44760 payload_rows_remote=39162 dup_rows=20776 dup_rows_remote=17850 dedup_ratio=0.317017 router_tokens_per_expert=[1214, 1251, 800, 1064, 4, 653, 354, 34, 460, 1036, 1168, 293, 1, 1239, 247, 1006, 931, 397, 25, 0, 1015, 228, 981, 765, 235, 743, 436, 498, 869, 153, 484, 0, 283, 548, 744, 8, 475, 0, 1064, 187, 332, 533, 407, 578, 1, 470, 33, 419, 237, 234, 302, 349, 103, 920, 343, 451, 510, 688, 708, 687, 1449, 959, 162, 0] payload_rows_per_dst=[4600, 6350, 5922, 5258, 5598, 5224, 5494, 6314] dup_rows_per_dst=[1814, 2478, 2762, 2154, 2926, 2394, 2812, 3436] combine_rows_per_src=[4600, 6350, 5922, 5258, 5598, 5224, 5494, 6314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44030 payload_rows_remote=37480 dup_rows=21506 dup_rows_remote=16742 dedup_ratio=0.328156 router_tokens_per_expert=[1486, 1318, 935, 781, 2, 374, 284, 24, 394, 1179, 1032, 343, 0, 924, 81, 1099, 933, 195, 11, 8, 877, 366, 874, 568, 108, 1150, 394, 278, 941, 171, 698, 22, 539, 400, 978, 2, 565, 1, 965, 87, 147, 721, 461, 347, 0, 623, 45, 271, 326, 407, 353, 847, 187, 909, 205, 937, 474, 899, 508, 291, 1380, 868, 175, 0] payload_rows_per_dst=[3782, 5606, 5398, 5528, 5938, 5150, 6550, 6078] dup_rows_per_dst=[1018, 2140, 2234, 1996, 3772, 2438, 4764, 3144] combine_rows_per_src=[3782, 5606, 5398, 5528, 5938, 5150, 6550, 6078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39236 dup_rows=20936 dup_rows_remote=18334 dedup_ratio=0.319458 router_tokens_per_expert=[876, 1206, 1237, 1068, 1, 646, 263, 13, 710, 1093, 1145, 308, 0, 1278, 72, 773, 1162, 323, 33, 22, 1329, 287, 568, 602, 145, 712, 579, 417, 1154, 128, 462, 0, 320, 415, 831, 27, 427, 2, 1116, 161, 401, 526, 481, 836, 10, 254, 52, 330, 328, 404, 197, 840, 162, 694, 70, 843, 528, 484, 886, 430, 550, 1271, 272, 8] payload_rows_per_dst=[4642, 5948, 5932, 5302, 5846, 5364, 5658, 5908] dup_rows_per_dst=[1814, 2624, 2720, 1892, 3164, 2602, 3170, 2950] combine_rows_per_src=[4642, 5948, 5932, 5302, 5846, 5364, 5658, 5908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44018 payload_rows_remote=37458 dup_rows=21518 dup_rows_remote=17164 dedup_ratio=0.328339 router_tokens_per_expert=[1212, 1271, 1268, 675, 0, 674, 258, 32, 798, 1605, 835, 322, 0, 685, 74, 1027, 1021, 528, 9, 1, 1401, 191, 1006, 740, 207, 578, 639, 397, 925, 184, 438, 1, 438, 537, 685, 9, 642, 0, 764, 111, 179, 1146, 598, 459, 2, 383, 31, 220, 227, 505, 255, 439, 80, 1006, 287, 327, 462, 702, 1268, 547, 726, 503, 228, 0] payload_rows_per_dst=[4262, 5280, 5574, 4968, 5702, 6008, 5664, 6560] dup_rows_per_dst=[1552, 1824, 2556, 1770, 3212, 3238, 3012, 4354] combine_rows_per_src=[4262, 5280, 5574, 4968, 5702, 6008, 5664, 6560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43912 payload_rows_remote=37092 dup_rows=21624 dup_rows_remote=17104 dedup_ratio=0.329956 router_tokens_per_expert=[1328, 1121, 955, 815, 2, 575, 171, 34, 1072, 1353, 872, 275, 0, 1007, 205, 1324, 1243, 554, 26, 1, 914, 227, 514, 897, 235, 617, 434, 218, 764, 106, 804, 0, 306, 502, 1414, 6, 337, 6, 738, 191, 246, 908, 673, 803, 10, 540, 16, 411, 409, 456, 247, 437, 99, 812, 166, 383, 461, 623, 662, 462, 794, 898, 89, 0] payload_rows_per_dst=[6458, 6820, 5974, 5512, 4862, 5024, 4360, 4902] dup_rows_per_dst=[4420, 4520, 2778, 2640, 2138, 2190, 1658, 1280] combine_rows_per_src=[6458, 6820, 5974, 5512, 4862, 5024, 4360, 4902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=37496 dup_rows=20990 dup_rows_remote=15614 dedup_ratio=0.320282 router_tokens_per_expert=[1328, 1055, 1024, 628, 7, 316, 215, 25, 524, 899, 918, 340, 9, 1615, 69, 1085, 1269, 169, 10, 20, 1031, 219, 804, 521, 274, 847, 305, 352, 732, 203, 599, 3, 458, 489, 935, 23, 912, 0, 660, 360, 465, 1232, 396, 863, 17, 505, 8, 202, 96, 436, 337, 422, 244, 1284, 209, 697, 345, 532, 875, 643, 587, 1034, 87, 0] payload_rows_per_dst=[7050, 5520, 5692, 5618, 5138, 5314, 5316, 4898] dup_rows_per_dst=[5376, 2168, 2394, 3080, 2536, 2062, 2134, 1240] combine_rows_per_src=[7050, 5520, 5692, 5618, 5138, 5314, 5316, 4898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=223928 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278385336 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556546744 weight_movement_bytes=110100480 total_nvlink_bytes=666647224 weight_to_token_ratio=0.197828 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280713216 dispatch_meta_bytes=228048 combine_payload_bytes=280713216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280941264 ingress_bytes=390813696 link_bytes=390813696 token_bytes=561654480 weight_movement_bytes=110100480 total_nvlink_bytes=671754960 weight_to_token_ratio=0.196029 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796892952e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=233312 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279469920 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558706528 weight_movement_bytes=220200960 total_nvlink_bytes=778907488 weight_to_token_ratio=0.394126 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268656640 dispatch_meta_bytes=216888 combine_payload_bytes=268656640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=268873528 ingress_bytes=378757120 link_bytes=378757120 token_bytes=537530168 weight_movement_bytes=110100480 total_nvlink_bytes=647630648 weight_to_token_ratio=0.204827 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.587209265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281243648 dispatch_meta_bytes=230280 combine_payload_bytes=281243648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281473928 ingress_bytes=391344128 link_bytes=391344128 token_bytes=562717576 weight_movement_bytes=110100480 total_nvlink_bytes=672818056 weight_to_token_ratio=0.195659 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806118036e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268498944 dispatch_meta_bytes=218488 combine_payload_bytes=268498944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=268717432 ingress_bytes=488699904 link_bytes=488699904 token_bytes=537216376 weight_movement_bytes=220200960 total_nvlink_bytes=757417336 weight_to_token_ratio=0.409892 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.499295104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=265875456 dispatch_meta_bytes=216784 combine_payload_bytes=265875456 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=266092240 ingress_bytes=596176896 link_bytes=596176896 token_bytes=531967696 weight_movement_bytes=330301440 total_nvlink_bytes=862269136 weight_to_token_ratio=0.620905 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036849676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268771328 dispatch_meta_bytes=212440 combine_payload_bytes=268771328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=268983768 ingress_bytes=599072768 link_bytes=599072768 token_bytes=537755096 weight_movement_bytes=330301440 total_nvlink_bytes=868056536 weight_to_token_ratio=0.614223 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041886074e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52854 payload_rows_remote=47126 dup_rows=12682 dup_rows_remote=11706 dedup_ratio=0.193512 router_tokens_per_expert=[0, 0, 0, 48, 1071, 0, 1320, 3961, 1710, 24, 3915, 0, 0, 0, 0, 1265, 0, 0, 0, 0, 2554, 4017, 0, 0, 0, 0, 3125, 0, 1238, 2114, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1924, 0, 0, 1, 0, 2, 0, 3, 0, 0, 105, 0, 2, 131, 0, 0, 0, 0, 0, 1029, 45, 3137, 0, 11, 12] payload_rows_per_dst=[4474, 8174, 5108, 5728, 8116, 7834, 6518, 6902] dup_rows_per_dst=[404, 5746, 0, 976, 3774, 8, 208, 1566] combine_rows_per_src=[4474, 8174, 5108, 5728, 8116, 7834, 6518, 6902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54986 payload_rows_remote=47174 dup_rows=10550 dup_rows_remote=9388 dedup_ratio=0.160980 router_tokens_per_expert=[0, 0, 0, 50, 1439, 1, 889, 4010, 1618, 6, 4004, 0, 0, 0, 0, 900, 0, 0, 0, 0, 3092, 3859, 1, 0, 0, 0, 3522, 0, 2093, 2182, 0, 0, 0, 0, 0, 1, 0, 0, 0, 627, 0, 1, 0, 0, 0, 0, 5, 0, 0, 105, 0, 0, 22, 0, 0, 0, 2, 0, 1491, 39, 2778, 1, 9, 21] payload_rows_per_dst=[4346, 8176, 6186, 6608, 7812, 8016, 7096, 6746] dup_rows_per_dst=[412, 4892, 0, 1942, 1162, 4, 202, 1936] combine_rows_per_src=[4346, 8176, 6186, 6608, 7812, 8016, 7096, 6746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53860 payload_rows_remote=46318 dup_rows=11676 dup_rows_remote=9062 dedup_ratio=0.178162 router_tokens_per_expert=[0, 0, 0, 29, 1552, 0, 1015, 4027, 1688, 14, 3874, 0, 0, 0, 0, 868, 0, 0, 0, 0, 2837, 4004, 0, 0, 0, 0, 3137, 0, 1582, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 0, 0, 1, 0, 0, 0, 0, 0, 0, 49, 0, 0, 72, 0, 0, 0, 0, 0, 1535, 36, 3329, 0, 8, 51] payload_rows_per_dst=[4776, 4578, 5674, 8100, 8052, 7748, 7542, 7390] dup_rows_per_dst=[416, 562, 0, 3118, 2436, 2, 2614, 2528] combine_rows_per_src=[4776, 4578, 5674, 8100, 8052, 7748, 7542, 7390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52794 payload_rows_remote=46448 dup_rows=12742 dup_rows_remote=10428 dedup_ratio=0.194427 router_tokens_per_expert=[0, 0, 0, 34, 1161, 0, 948, 3846, 2491, 13, 4070, 0, 0, 0, 0, 1017, 2, 0, 0, 5, 2401, 3882, 0, 0, 0, 0, 2853, 0, 848, 2310, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1514, 0, 0, 5, 0, 0, 1, 0, 0, 0, 125, 2, 0, 134, 0, 0, 0, 2, 0, 1695, 50, 3286, 1, 52, 17] payload_rows_per_dst=[7578, 6390, 6346, 5658, 5288, 8140, 5990, 7404] dup_rows_per_dst=[3328, 1724, 2314, 658, 1666, 12, 238, 2802] combine_rows_per_src=[7578, 6390, 6346, 5658, 5288, 8140, 5990, 7404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50828 payload_rows_remote=43592 dup_rows=14708 dup_rows_remote=14700 dedup_ratio=0.224426 router_tokens_per_expert=[0, 0, 0, 42, 1007, 0, 690, 3966, 1187, 5, 3618, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 2181, 3841, 0, 0, 0, 0, 3444, 0, 2210, 2186, 0, 3, 0, 0, 0, 0, 3, 0, 0, 1581, 0, 2, 1, 0, 1, 0, 0, 0, 0, 41, 2, 0, 100, 7, 0, 0, 0, 0, 1961, 47, 3034, 1, 16, 5] payload_rows_per_dst=[3286, 6770, 4362, 6282, 7908, 7236, 7664, 7320] dup_rows_per_dst=[192, 3294, 0, 1824, 2942, 8, 3640, 2808] combine_rows_per_src=[3286, 6770, 4362, 6282, 7908, 7236, 7664, 7320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51184 payload_rows_remote=43630 dup_rows=14352 dup_rows_remote=11150 dedup_ratio=0.218994 router_tokens_per_expert=[0, 0, 0, 222, 844, 0, 912, 3782, 1561, 14, 3982, 0, 0, 0, 0, 1705, 0, 0, 0, 35, 2474, 3875, 0, 0, 0, 0, 2843, 0, 1086, 2340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 0, 0, 64, 0, 5, 0, 0, 0, 0, 62, 0, 0, 102, 0, 0, 0, 0, 0, 1971, 43, 3341, 0, 14, 9] payload_rows_per_dst=[3730, 3800, 4950, 7624, 7928, 8014, 7584, 7554] dup_rows_per_dst=[226, 270, 68, 2112, 2786, 2578, 3110, 3202] combine_rows_per_src=[3730, 3800, 4950, 7624, 7928, 8014, 7584, 7554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=337799168 dispatch_meta_bytes=235328 combine_payload_bytes=337799168 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=338034496 ingress_bytes=668100608 link_bytes=668100608 token_bytes=675833664 weight_movement_bytes=330301440 total_nvlink_bytes=1006135104 weight_to_token_ratio=0.488732 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.161936841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338143232 dispatch_meta_bytes=226248 combine_payload_bytes=338143232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338369480 ingress_bytes=448243712 link_bytes=448243712 token_bytes=676512712 weight_movement_bytes=110100480 total_nvlink_bytes=786613192 weight_to_token_ratio=0.162747 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.795695386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47330 payload_rows_remote=43164 dup_rows=18206 dup_rows_remote=17646 dedup_ratio=0.277802 router_tokens_per_expert=[0, 0, 0, 84, 1597, 0, 1099, 3955, 1009, 8, 3984, 0, 0, 0, 0, 1346, 0, 0, 0, 1, 2823, 3898, 0, 0, 0, 0, 2880, 0, 1233, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 6, 0, 0, 0, 0, 0, 0, 46, 0, 0, 113, 0, 0, 0, 0, 0, 1529, 39, 3006, 0, 19, 13] payload_rows_per_dst=[8144, 4166, 8142, 6234, 3248, 7968, 5978, 3450] dup_rows_per_dst=[6920, 560, 9306, 1144, 0, 12, 100, 164] combine_rows_per_src=[8144, 4166, 8142, 6234, 3248, 7968, 5978, 3450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=332007424 dispatch_meta_bytes=221520 combine_payload_bytes=332007424 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=332228944 ingress_bytes=552208384 link_bytes=552208384 token_bytes=664236368 weight_movement_bytes=220200960 total_nvlink_bytes=884437328 weight_to_token_ratio=0.331510 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.603812024e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48830 payload_rows_remote=40638 dup_rows=16706 dup_rows_remote=7594 dedup_ratio=0.254913 router_tokens_per_expert=[0, 0, 0, 106, 925, 0, 783, 4006, 2076, 42, 3972, 0, 0, 0, 0, 1099, 0, 0, 0, 0, 2852, 3688, 0, 0, 0, 0, 3375, 0, 1659, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 1, 3, 0, 2, 0, 0, 0, 0, 74, 4, 0, 73, 0, 0, 0, 1, 0, 1787, 47, 2832, 0, 12, 35] payload_rows_per_dst=[8192, 5496, 7998, 6178, 2418, 7948, 6908, 3692] dup_rows_per_dst=[9112, 938, 5082, 1350, 0, 8, 144, 72] combine_rows_per_src=[8192, 5496, 7998, 6178, 2418, 7948, 6908, 3692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=332939264 dispatch_meta_bytes=227504 combine_payload_bytes=332939264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=333166768 ingress_bytes=553140224 link_bytes=553140224 token_bytes=666106032 weight_movement_bytes=220200960 total_nvlink_bytes=886306992 weight_to_token_ratio=0.330579 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.620018255e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=312467456 dispatch_meta_bytes=233168 combine_payload_bytes=312467456 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=312700624 ingress_bytes=532668416 link_bytes=532668416 token_bytes=625168080 weight_movement_bytes=220200960 total_nvlink_bytes=845369040 weight_to_token_ratio=0.352227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.263979843e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=312739840 dispatch_meta_bytes=219120 combine_payload_bytes=312739840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=312958960 ingress_bytes=532940800 link_bytes=532940800 token_bytes=625698800 weight_movement_bytes=220200960 total_nvlink_bytes=845899760 weight_to_token_ratio=0.351928 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.268717049e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60862 payload_rows_remote=53966 dup_rows=4674 dup_rows_remote=2900 dedup_ratio=0.071320 router_tokens_per_expert=[0, 0, 0, 2564, 2, 1153, 2999, 0, 0, 552, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 4045, 275, 0, 0, 0, 4045, 4096, 0, 1336, 0, 0, 0, 215, 0, 0, 0, 0, 4092, 2, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 221] payload_rows_per_dst=[6308, 6754, 8160, 6896, 8192, 8190, 8192, 8170] dup_rows_per_dst=[1130, 418, 480, 1774, 426, 4, 0, 442] combine_rows_per_src=[6308, 6754, 8160, 6896, 8192, 8190, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59710 payload_rows_remote=51518 dup_rows=5826 dup_rows_remote=5190 dedup_ratio=0.088898 router_tokens_per_expert=[0, 0, 0, 1915, 0, 942, 2664, 0, 0, 1221, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 0, 4067, 428, 0, 0, 0, 4054, 4091, 0, 1881, 1, 0, 1, 317, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 244] payload_rows_per_dst=[5194, 6868, 8168, 6728, 8192, 8190, 8182, 8188] dup_rows_per_dst=[520, 994, 822, 2364, 636, 0, 0, 490] combine_rows_per_src=[5194, 6868, 8168, 6728, 8192, 8190, 8182, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60262 payload_rows_remote=52070 dup_rows=5274 dup_rows_remote=5274 dedup_ratio=0.080475 router_tokens_per_expert=[0, 0, 0, 1985, 0, 874, 3001, 0, 0, 1003, 0, 0, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 4096, 276, 0, 0, 0, 4043, 4096, 0, 1819, 3, 0, 0, 190, 0, 0, 0, 0, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 223] payload_rows_per_dst=[5232, 6984, 8192, 7118, 8186, 8188, 8192, 8170] dup_rows_per_dst=[486, 882, 552, 2528, 378, 0, 0, 448] combine_rows_per_src=[5232, 6984, 8192, 7118, 8186, 8188, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309399552 dispatch_meta_bytes=243240 combine_payload_bytes=309399552 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=309642792 ingress_bytes=639700992 link_bytes=639700992 token_bytes=619042344 weight_movement_bytes=330301440 total_nvlink_bytes=949343784 weight_to_token_ratio=0.533568 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.112545238e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=291293184 dispatch_meta_bytes=192928 combine_payload_bytes=291293184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=291486112 ingress_bytes=621594624 link_bytes=621594624 token_bytes=582779296 weight_movement_bytes=330301440 total_nvlink_bytes=913080736 weight_to_token_ratio=0.566769 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081055286e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=369281024 dispatch_meta_bytes=226832 combine_payload_bytes=369281024 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=369507856 ingress_bytes=699582464 link_bytes=699582464 token_bytes=738788880 weight_movement_bytes=330301440 total_nvlink_bytes=1069090320 weight_to_token_ratio=0.447085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.216688966e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=373237760 dispatch_meta_bytes=229376 combine_payload_bytes=373237760 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=373467136 ingress_bytes=593438720 link_bytes=593438720 token_bytes=746704896 weight_movement_bytes=220200960 total_nvlink_bytes=966905856 weight_to_token_ratio=0.294897 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032087538e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=386828288 dispatch_meta_bytes=227464 combine_payload_bytes=386828288 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=387055752 ingress_bytes=827230208 link_bytes=827230208 token_bytes=773884040 weight_movement_bytes=440401920 total_nvlink_bytes=1214285960 weight_to_token_ratio=0.569080 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.438689388e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60294 payload_rows_remote=52114 dup_rows=5242 dup_rows_remote=5238 dedup_ratio=0.079987 router_tokens_per_expert=[0, 0, 0, 1868, 8, 1020, 2675, 0, 0, 825, 0, 0, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 4056, 399, 0, 0, 0, 4056, 4096, 0, 2028, 0, 0, 15, 211, 0, 0, 0, 0, 4072, 0, 4087, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 210] payload_rows_per_dst=[5304, 7112, 8166, 6968, 8180, 8180, 8192, 8192] dup_rows_per_dst=[488, 726, 744, 2438, 416, 4, 0, 426] combine_rows_per_src=[5304, 7112, 8166, 6968, 8180, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59098 payload_rows_remote=50950 dup_rows=6438 dup_rows_remote=6012 dedup_ratio=0.098236 router_tokens_per_expert=[0, 0, 0, 1807, 0, 626, 2398, 0, 0, 771, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 4064, 722, 0, 0, 0, 4031, 4096, 0, 2641, 21, 0, 0, 508, 0, 0, 0, 2, 4080, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 213] payload_rows_per_dst=[4720, 6324, 8184, 7170, 8168, 8192, 8192, 8148] dup_rows_per_dst=[146, 516, 1388, 2930, 986, 26, 20, 426] combine_rows_per_src=[4720, 6324, 8184, 7170, 8168, 8192, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60098 payload_rows_remote=51936 dup_rows=5438 dup_rows_remote=5036 dedup_ratio=0.082977 router_tokens_per_expert=[0, 0, 0, 1596, 2, 815, 2898, 0, 0, 1065, 0, 0, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 4062, 220, 0, 0, 0, 4021, 4096, 0, 2547, 13, 0, 1, 365, 0, 0, 0, 0, 4067, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 212] payload_rows_per_dst=[5890, 6640, 8162, 6726, 8166, 8190, 8192, 8132] dup_rows_per_dst=[1358, 780, 402, 1768, 700, 2, 2, 426] combine_rows_per_src=[5890, 6640, 8162, 6726, 8166, 8190, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59554 payload_rows_remote=52368 dup_rows=5982 dup_rows_remote=4622 dedup_ratio=0.091278 router_tokens_per_expert=[0, 0, 0, 2366, 0, 804, 2034, 0, 0, 1172, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 4085, 416, 0, 0, 0, 4056, 4095, 0, 1944, 1, 0, 0, 343, 0, 0, 0, 5, 4072, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 141] payload_rows_per_dst=[7536, 7186, 8188, 3888, 8188, 8188, 8190, 8190] dup_rows_per_dst=[2872, 1360, 814, 2, 652, 0, 0, 282] combine_rows_per_src=[7536, 7186, 8188, 3888, 8188, 8188, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48948 payload_rows_remote=40864 dup_rows=16588 dup_rows_remote=6320 dedup_ratio=0.253113 router_tokens_per_expert=[0, 0, 0, 2774, 0, 736, 2574, 0, 0, 1355, 0, 0, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 4059, 185, 0, 0, 0, 4034, 4091, 0, 1532, 2, 0, 2, 330, 0, 0, 0, 0, 4068, 2, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 201] payload_rows_per_dst=[8084, 7390, 3018, 5222, 3772, 8182, 8182, 5098] dup_rows_per_dst=[10268, 4602, 96, 1016, 298, 0, 0, 308] combine_rows_per_src=[8084, 7390, 3018, 5222, 3772, 8182, 8182, 5098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=373553152 dispatch_meta_bytes=229408 combine_payload_bytes=373553152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373782560 ingress_bytes=483653632 link_bytes=483653632 token_bytes=747335712 weight_movement_bytes=110100480 total_nvlink_bytes=857436192 weight_to_token_ratio=0.147324 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.411532134e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=365209600 dispatch_meta_bytes=227848 combine_payload_bytes=365209600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365437448 ingress_bytes=475310080 link_bytes=475310080 token_bytes=730647048 weight_movement_bytes=110100480 total_nvlink_bytes=840747528 weight_to_token_ratio=0.150689 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.266424042e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60376 payload_rows_remote=52262 dup_rows=5160 dup_rows_remote=5158 dedup_ratio=0.078735 router_tokens_per_expert=[0, 2, 1869, 0, 0, 4088, 0, 0, 1, 0, 0, 241, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 189, 0, 1972, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1, 4057, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4090, 0, 4095, 0, 0, 3970, 4095, 0] payload_rows_per_dst=[4504, 8184, 8096, 7154, 8114, 8192, 8192, 7940] dup_rows_per_dst=[1088, 480, 0, 3538, 2, 10, 42, 0] combine_rows_per_src=[4504, 8184, 8096, 7154, 8114, 8192, 8192, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=372277248 dispatch_meta_bytes=227888 combine_payload_bytes=372277248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=372505136 ingress_bytes=592478208 link_bytes=592478208 token_bytes=744782384 weight_movement_bytes=220200960 total_nvlink_bytes=964983344 weight_to_token_ratio=0.295658 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.030417050e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58986 payload_rows_remote=50794 dup_rows=6550 dup_rows_remote=6428 dedup_ratio=0.099945 router_tokens_per_expert=[0, 0, 1054, 0, 0, 4087, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 269, 0, 2522, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 17, 4047, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 4069, 0, 4085, 0, 0, 3952, 4096, 0] payload_rows_per_dst=[2108, 8188, 8116, 8192, 8094, 8192, 8192, 7904] dup_rows_per_dst=[0, 798, 0, 5592, 34, 4, 122, 0] combine_rows_per_src=[2108, 8188, 8116, 8192, 8094, 8192, 8192, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=375373824 dispatch_meta_bytes=227960 combine_payload_bytes=375373824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375601784 ingress_bytes=485474304 link_bytes=485474304 token_bytes=750975608 weight_movement_bytes=110100480 total_nvlink_bytes=861076088 weight_to_token_ratio=0.146610 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.443196615e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61808 payload_rows_remote=57022 dup_rows=3728 dup_rows_remote=3722 dedup_ratio=0.056885 router_tokens_per_expert=[0, 1, 1336, 0, 0, 4091, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 166, 0, 2228, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 4074, 4, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4088, 0, 4088, 0, 0, 4020, 4094, 0] payload_rows_per_dst=[8192, 8188, 8072, 4786, 8148, 8192, 8190, 8040] dup_rows_per_dst=[2664, 944, 0, 6, 60, 20, 34, 0] combine_rows_per_src=[8192, 8188, 8072, 4786, 8148, 8192, 8190, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=292913152 dispatch_meta_bytes=188736 combine_payload_bytes=292913152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293101888 ingress_bytes=513114112 link_bytes=513114112 token_bytes=586015040 weight_movement_bytes=220200960 total_nvlink_bytes=806216000 weight_to_token_ratio=0.375760 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.923898335e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=374614016 dispatch_meta_bytes=229680 combine_payload_bytes=374614016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374843696 ingress_bytes=484714496 link_bytes=484714496 token_bytes=749457712 weight_movement_bytes=110100480 total_nvlink_bytes=859558192 weight_to_token_ratio=0.146907 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.429982304e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=364091392 dispatch_meta_bytes=228888 combine_payload_bytes=364091392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364320280 ingress_bytes=474191872 link_bytes=474191872 token_bytes=728411672 weight_movement_bytes=110100480 total_nvlink_bytes=838512152 weight_to_token_ratio=0.151151 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.246976566e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60680 payload_rows_remote=52496 dup_rows=4856 dup_rows_remote=4792 dedup_ratio=0.074097 router_tokens_per_expert=[0, 0, 1948, 0, 0, 4088, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 2, 3997, 0, 0, 0, 0, 0, 0, 225, 0, 1640, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 13, 4052, 2, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 4093, 0, 4067, 0, 0, 3965, 4089, 0] payload_rows_per_dst=[3896, 8192, 7994, 8188, 8104, 8184, 8192, 7930] dup_rows_per_dst=[0, 616, 4, 3836, 26, 64, 310, 0] combine_rows_per_src=[3896, 8192, 7994, 8188, 8104, 8184, 8192, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=408733696 dispatch_meta_bytes=242976 combine_payload_bytes=408733696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408976672 ingress_bytes=518834176 link_bytes=518834176 token_bytes=817710368 weight_movement_bytes=110100480 total_nvlink_bytes=927810848 weight_to_token_ratio=0.134645 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.023379656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54322 payload_rows_remote=46924 dup_rows=11214 dup_rows_remote=3854 dedup_ratio=0.171112 router_tokens_per_expert=[0, 4, 2554, 0, 0, 3976, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 160, 0, 1578, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 16, 4015, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 3934, 0, 4064, 0, 0, 3611, 4096, 0] payload_rows_per_dst=[5108, 2818, 7894, 8170, 8030, 7986, 6918, 7398] dup_rows_per_dst=[8, 78, 0, 3282, 32, 44, 410, 7360] combine_rows_per_src=[5108, 2818, 7894, 8170, 8030, 7986, 6918, 7398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61024 payload_rows_remote=53332 dup_rows=4512 dup_rows_remote=4512 dedup_ratio=0.068848 router_tokens_per_expert=[0, 0, 1681, 0, 0, 4093, 0, 0, 0, 0, 0, 465, 1, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 314, 0, 2198, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 4065, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 4085, 0, 4080, 0, 0, 3741, 4093, 0] payload_rows_per_dst=[8192, 8188, 7692, 4964, 8130, 8190, 8186, 7482] dup_rows_per_dst=[3356, 914, 0, 66, 36, 20, 120, 0] combine_rows_per_src=[8192, 8188, 7692, 4964, 8130, 8190, 8186, 7482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60148 payload_rows_remote=51958 dup_rows=5388 dup_rows_remote=4896 dedup_ratio=0.082214 router_tokens_per_expert=[0, 11, 2392, 0, 0, 4050, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 2, 3997, 0, 0, 0, 0, 0, 0, 198, 0, 1464, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 20, 4063, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 4095, 0, 4078, 0, 0, 3972, 4096, 0] payload_rows_per_dst=[8192, 8190, 7994, 3328, 8126, 8192, 8182, 7944] dup_rows_per_dst=[4714, 492, 4, 0, 40, 16, 122, 0] combine_rows_per_src=[8192, 8190, 7994, 3328, 8126, 8192, 8182, 7944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=376291328 dispatch_meta_bytes=229152 combine_payload_bytes=376291328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376520480 ingress_bytes=486391808 link_bytes=486391808 token_bytes=752811808 weight_movement_bytes=110100480 total_nvlink_bytes=862912288 weight_to_token_ratio=0.146252 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.459153518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=336351232 dispatch_meta_bytes=203112 combine_payload_bytes=336351232 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=336554344 ingress_bytes=996954112 link_bytes=996954112 token_bytes=672905576 weight_movement_bytes=660602880 total_nvlink_bytes=1333508456 weight_to_token_ratio=0.981717 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.733867172e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56936 payload_rows_remote=48744 dup_rows=8600 dup_rows_remote=3278 dedup_ratio=0.131226 router_tokens_per_expert=[0, 0, 1565, 0, 0, 4080, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 1, 4040, 0, 0, 0, 0, 0, 0, 308, 0, 1889, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2, 4055, 2, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 3898, 4094, 0] payload_rows_per_dst=[8192, 8192, 8098, 4420, 8112, 8192, 8192, 3538] dup_rows_per_dst=[5322, 1074, 1572, 14, 448, 38, 132, 0] combine_rows_per_src=[8192, 8192, 8098, 4420, 8112, 8192, 8192, 3538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60932 payload_rows_remote=52748 dup_rows=4604 dup_rows_remote=4604 dedup_ratio=0.070251 router_tokens_per_expert=[0, 421, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2714, 2969, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4092, 0, 64, 49, 0, 0, 4055, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 229, 0, 0, 300, 0, 0, 0, 3662, 0] payload_rows_per_dst=[7252, 8192, 8192, 5428, 8184, 8122, 8192, 7370] dup_rows_per_dst=[1432, 1944, 2, 0, 0, 214, 458, 554] combine_rows_per_src=[7252, 8192, 8192, 5428, 8184, 8122, 8192, 7370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57508 payload_rows_remote=49316 dup_rows=8028 dup_rows_remote=7520 dedup_ratio=0.122498 router_tokens_per_expert=[0, 234, 0, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2935, 3045, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4091, 0, 147, 52, 0, 0, 4053, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 254, 0, 0, 256, 0, 0, 0, 3916, 0] payload_rows_per_dst=[7586, 3460, 8192, 5870, 8182, 8114, 8192, 7912] dup_rows_per_dst=[1422, 226, 0, 0, 5050, 390, 508, 432] combine_rows_per_src=[7586, 3460, 8192, 5870, 8182, 8114, 8192, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=382283776 dispatch_meta_bytes=231376 combine_payload_bytes=382283776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382515152 ingress_bytes=492384256 link_bytes=492384256 token_bytes=764798928 weight_movement_bytes=110100480 total_nvlink_bytes=874899408 weight_to_token_ratio=0.143960 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.563372045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=372434944 dispatch_meta_bytes=227416 combine_payload_bytes=372434944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372662360 ingress_bytes=482535424 link_bytes=482535424 token_bytes=745097304 weight_movement_bytes=110100480 total_nvlink_bytes=855197784 weight_to_token_ratio=0.147767 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.392084658e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61198 payload_rows_remote=56032 dup_rows=4338 dup_rows_remote=4338 dedup_ratio=0.066193 router_tokens_per_expert=[0, 230, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2583, 3001, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4083, 2, 119, 18, 0, 0, 4045, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 253, 0, 0, 382, 0, 0, 0, 3891, 0] payload_rows_per_dst=[7328, 8188, 8192, 5166, 8166, 8108, 8192, 7858] dup_rows_per_dst=[1392, 1488, 4, 0, 0, 260, 506, 688] combine_rows_per_src=[7328, 8188, 8192, 5166, 8166, 8108, 8192, 7858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=349396992 dispatch_meta_bytes=208088 combine_payload_bytes=349396992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=349605080 ingress_bytes=569597952 link_bytes=569597952 token_bytes=699002072 weight_movement_bytes=220200960 total_nvlink_bytes=919203032 weight_to_token_ratio=0.315022 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.906245213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=378097664 dispatch_meta_bytes=229408 combine_payload_bytes=378097664 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=378327072 ingress_bytes=818499584 link_bytes=818499584 token_bytes=756424736 weight_movement_bytes=440401920 total_nvlink_bytes=1196826656 weight_to_token_ratio=0.582215 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.423505397e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353497088 dispatch_meta_bytes=227344 combine_payload_bytes=353497088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=353724432 ingress_bytes=573698048 link_bytes=573698048 token_bytes=707221520 weight_movement_bytes=220200960 total_nvlink_bytes=927422480 weight_to_token_ratio=0.311361 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.977552626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60278 payload_rows_remote=52146 dup_rows=5258 dup_rows_remote=4918 dedup_ratio=0.080231 router_tokens_per_expert=[0, 581, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2023, 3388, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4041, 0, 124, 48, 0, 0, 4064, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 243, 0, 0, 566, 0, 0, 0, 3791, 0] payload_rows_per_dst=[7634, 8178, 8192, 4046, 8082, 8132, 8192, 7822] dup_rows_per_dst=[1858, 1668, 14, 0, 0, 340, 486, 892] combine_rows_per_src=[7634, 8178, 8192, 4046, 8082, 8132, 8192, 7822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401637376 dispatch_meta_bytes=241480 combine_payload_bytes=401637376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401878856 ingress_bytes=621838336 link_bytes=621838336 token_bytes=803516232 weight_movement_bytes=220200960 total_nvlink_bytes=1023717192 weight_to_token_ratio=0.274047 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081479141e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50212 payload_rows_remote=42540 dup_rows=15324 dup_rows_remote=11882 dedup_ratio=0.233826 router_tokens_per_expert=[0, 660, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 2772, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 83, 407, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 228, 0, 0, 794, 0, 0, 0, 3564, 0] payload_rows_per_dst=[3426, 2568, 8164, 6128, 8192, 5870, 8192, 7672] dup_rows_per_dst=[342, 0, 0, 2072, 8220, 792, 456, 3442] combine_rows_per_src=[3426, 2568, 8164, 6128, 8192, 5870, 8192, 7672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56630 payload_rows_remote=48438 dup_rows=8906 dup_rows_remote=8902 dedup_ratio=0.135895 router_tokens_per_expert=[0, 409, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2527, 2521, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4078, 0, 77, 97, 0, 0, 4039, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 251, 0, 0, 1005, 0, 0, 0, 3664, 0] payload_rows_per_dst=[6824, 8188, 8192, 7618, 1878, 8118, 8192, 7620] dup_rows_per_dst=[1390, 1270, 4, 3714, 0, 308, 502, 1718] combine_rows_per_src=[6824, 8188, 8192, 7618, 1878, 8118, 8192, 7620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56220 payload_rows_remote=48028 dup_rows=9316 dup_rows_remote=7074 dedup_ratio=0.142151 router_tokens_per_expert=[0, 432, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 3112, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4091, 0, 73, 85, 0, 0, 4055, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 231, 0, 0, 483, 0, 0, 0, 3916, 0] payload_rows_per_dst=[7334, 8192, 8192, 8182, 1390, 8148, 6868, 7914] dup_rows_per_dst=[1486, 2242, 0, 4030, 0, 278, 396, 884] combine_rows_per_src=[7334, 8192, 8192, 8182, 1390, 8148, 6868, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=373782528 dispatch_meta_bytes=228256 combine_payload_bytes=373782528 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=374010784 ingress_bytes=483883008 link_bytes=483883008 token_bytes=747793312 weight_movement_bytes=110100480 total_nvlink_bytes=857893792 weight_to_token_ratio=0.147234 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.415521360e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=304926720 dispatch_meta_bytes=217688 combine_payload_bytes=304926720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=305144408 ingress_bytes=525127680 link_bytes=525127680 token_bytes=610071128 weight_movement_bytes=220200960 total_nvlink_bytes=830272088 weight_to_token_ratio=0.360943 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.132834042e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56926 payload_rows_remote=49394 dup_rows=8610 dup_rows_remote=7312 dedup_ratio=0.131378 router_tokens_per_expert=[0, 374, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2550, 3386, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4091, 0, 70, 50, 0, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 275, 0, 0, 418, 0, 0, 0, 3788, 0] payload_rows_per_dst=[7532, 8188, 8192, 8182, 8192, 8104, 5948, 2588] dup_rows_per_dst=[1298, 1544, 4, 5100, 0, 238, 256, 170] combine_rows_per_src=[7532, 8188, 8192, 8182, 8192, 8104, 5948, 2588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=347203584 dispatch_meta_bytes=229360 combine_payload_bytes=347203584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347432944 ingress_bytes=457304064 link_bytes=457304064 token_bytes=694636528 weight_movement_bytes=110100480 total_nvlink_bytes=804737008 weight_to_token_ratio=0.158501 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.953269809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62556 payload_rows_remote=54368 dup_rows=2980 dup_rows_remote=2980 dedup_ratio=0.045471 router_tokens_per_expert=[0, 0, 0, 3008, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3886, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 1, 335, 0, 3917, 0] payload_rows_per_dst=[6018, 8158, 8192, 8192, 8188, 7774, 8192, 7842] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2316, 664] combine_rows_per_src=[6018, 8158, 8192, 8192, 8188, 7774, 8192, 7842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63186 payload_rows_remote=54994 dup_rows=2350 dup_rows_remote=722 dedup_ratio=0.035858 router_tokens_per_expert=[0, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 364, 0, 4002, 0] payload_rows_per_dst=[6346, 8188, 8192, 8192, 8190, 7876, 8192, 8010] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1628, 722] combine_rows_per_src=[6346, 8188, 8192, 8192, 8190, 7876, 8192, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344264704 dispatch_meta_bytes=220408 combine_payload_bytes=344264704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344485112 ingress_bytes=454365184 link_bytes=454365184 token_bytes=688749816 weight_movement_bytes=110100480 total_nvlink_bytes=798850296 weight_to_token_ratio=0.159856 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.902157852e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63594 payload_rows_remote=55412 dup_rows=1942 dup_rows_remote=1942 dedup_ratio=0.029633 router_tokens_per_expert=[0, 0, 0, 3362, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3996, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 2, 273, 0, 3983, 0] payload_rows_per_dst=[6728, 8128, 8192, 8182, 8192, 7996, 8192, 7984] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1410, 532] combine_rows_per_src=[6728, 8128, 8192, 8182, 8192, 7996, 8192, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=354056192 dispatch_meta_bytes=226824 combine_payload_bytes=354056192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354283016 ingress_bytes=464156672 link_bytes=464156672 token_bytes=708339208 weight_movement_bytes=110100480 total_nvlink_bytes=818439688 weight_to_token_ratio=0.155435 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.072447933e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=389709824 dispatch_meta_bytes=229392 combine_payload_bytes=389709824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389939216 ingress_bytes=499810304 link_bytes=499810304 token_bytes=779649040 weight_movement_bytes=110100480 total_nvlink_bytes=889749520 weight_to_token_ratio=0.141218 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692523233e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=394196992 dispatch_meta_bytes=222864 combine_payload_bytes=394196992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394419856 ingress_bytes=614397952 link_bytes=614397952 token_bytes=788616848 weight_movement_bytes=220200960 total_nvlink_bytes=1008817808 weight_to_token_ratio=0.279224 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068539090e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63054 payload_rows_remote=55284 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.037872 router_tokens_per_expert=[0, 0, 0, 3190, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3865, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 726, 0, 0, 0, 5, 521, 0, 4067, 0] payload_rows_per_dst=[6390, 7978, 8192, 8180, 8192, 7770, 8192, 8160] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 1452, 1026] combine_rows_per_src=[6390, 7978, 8192, 8180, 8192, 7770, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=397193216 dispatch_meta_bytes=229416 combine_payload_bytes=397193216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397422632 ingress_bytes=507293696 link_bytes=507293696 token_bytes=794615848 weight_movement_bytes=110100480 total_nvlink_bytes=904716328 weight_to_token_ratio=0.138558 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.822671728e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62636 payload_rows_remote=54806 dup_rows=2900 dup_rows_remote=1882 dedup_ratio=0.044250 router_tokens_per_expert=[0, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 536, 0, 3888, 0] payload_rows_per_dst=[6470, 8152, 8192, 8064, 8192, 7544, 8192, 7830] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1882, 1018] combine_rows_per_src=[6470, 8152, 8192, 8064, 8192, 7544, 8192, 7830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64146 payload_rows_remote=55954 dup_rows=1390 dup_rows_remote=1390 dedup_ratio=0.021210 router_tokens_per_expert=[0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4036, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 297, 0, 0, 1, 2, 408, 0, 4059, 0] payload_rows_per_dst=[7056, 8114, 8192, 8178, 8192, 8078, 8192, 8144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 594, 796] combine_rows_per_src=[7056, 8114, 8192, 8178, 8192, 8078, 8192, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60190 payload_rows_remote=52048 dup_rows=5346 dup_rows_remote=5346 dedup_ratio=0.081573 router_tokens_per_expert=[0, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3690, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 343, 0, 3904, 0] payload_rows_per_dst=[7754, 8142, 8192, 8128, 8192, 7380, 4562, 7840] dup_rows_per_dst=[4200, 0, 0, 0, 0, 2, 490, 654] combine_rows_per_src=[7754, 8142, 8192, 8128, 8192, 7380, 4562, 7840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=396275712 dispatch_meta_bytes=231064 combine_payload_bytes=396275712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396506776 ingress_bytes=506376192 link_bytes=506376192 token_bytes=792782488 weight_movement_bytes=110100480 total_nvlink_bytes=902882968 weight_to_token_ratio=0.138879 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.806714825e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392849408 dispatch_meta_bytes=226752 combine_payload_bytes=392849408 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=393076160 ingress_bytes=943351808 link_bytes=943351808 token_bytes=785925568 weight_movement_bytes=550502400 total_nvlink_bytes=1336427968 weight_to_token_ratio=0.700451 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.640643949e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54944 payload_rows_remote=46752 dup_rows=10592 dup_rows_remote=4214 dedup_ratio=0.161621 router_tokens_per_expert=[0, 0, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3983, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 710, 0, 0, 0, 2, 383, 0, 4026, 0] payload_rows_per_dst=[8192, 8184, 8192, 8188, 8192, 8040, 1960, 3996] dup_rows_per_dst=[6378, 492, 0, 232, 6, 3076, 90, 318] combine_rows_per_src=[8192, 8184, 8192, 8188, 8192, 8040, 1960, 3996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=401078272 dispatch_meta_bytes=229376 combine_payload_bytes=401078272 prefetch_bytes=0 reduce_bytes=0 egress_bytes=401307648 ingress_bytes=401078272 link_bytes=401307648 token_bytes=802385920 weight_movement_bytes=0 total_nvlink_bytes=802385920 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.979400036e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59360 payload_rows_remote=51168 dup_rows=6176 dup_rows_remote=1938 dedup_ratio=0.094238 router_tokens_per_expert=[0, 1582, 0, 0, 0, 0, 0, 0, 59, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4054, 0, 4096, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 3130, 4087, 0, 0, 0, 0, 4096, 13, 0, 3385, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3164, 8192, 8192, 8164, 8192, 8146, 8192, 7118] dup_rows_per_dst=[0, 182, 66, 8, 0, 0, 4238, 1682] combine_rows_per_src=[3164, 8192, 8192, 8164, 8192, 8146, 8192, 7118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63584 payload_rows_remote=55392 dup_rows=1952 dup_rows_remote=1952 dedup_ratio=0.029785 router_tokens_per_expert=[0, 908, 0, 0, 0, 0, 0, 0, 17, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3584, 4090, 0, 0, 0, 0, 4096, 1, 0, 3634, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8190, 7168, 7268] dup_rows_per_dst=[1806, 100, 26, 18, 0, 0, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8190, 7168, 7268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=373080064 dispatch_meta_bytes=229576 combine_payload_bytes=373080064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373309640 ingress_bytes=483180544 link_bytes=483180544 token_bytes=746389704 weight_movement_bytes=110100480 total_nvlink_bytes=856490184 weight_to_token_ratio=0.147511 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.403304356e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62526 payload_rows_remote=54334 dup_rows=3010 dup_rows_remote=2978 dedup_ratio=0.045929 router_tokens_per_expert=[0, 1380, 0, 0, 0, 0, 0, 0, 9, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4087, 0, 4096, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 3076, 4086, 0, 0, 0, 0, 4096, 12, 0, 3639, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8192, 8192, 8192, 8152, 6152, 7278] dup_rows_per_dst=[2756, 112, 86, 32, 0, 0, 24, 0] combine_rows_per_src=[8176, 8192, 8192, 8192, 8192, 8152, 6152, 7278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=335118336 dispatch_meta_bytes=203864 combine_payload_bytes=335118336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335322200 ingress_bytes=445218816 link_bytes=445218816 token_bytes=670440536 weight_movement_bytes=110100480 total_nvlink_bytes=780541016 weight_to_token_ratio=0.164221 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.743087469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397049856 dispatch_meta_bytes=229376 combine_payload_bytes=397049856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397279232 ingress_bytes=507150336 link_bytes=507150336 token_bytes=794329088 weight_movement_bytes=110100480 total_nvlink_bytes=904429568 weight_to_token_ratio=0.138608 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.820178462e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=366772224 dispatch_meta_bytes=212424 combine_payload_bytes=366772224 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=366984648 ingress_bytes=807174144 link_bytes=807174144 token_bytes=733756872 weight_movement_bytes=440401920 total_nvlink_bytes=1174158792 weight_to_token_ratio=0.600201 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.403808594e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62546 payload_rows_remote=54450 dup_rows=2990 dup_rows_remote=2990 dedup_ratio=0.045624 router_tokens_per_expert=[0, 808, 0, 0, 0, 0, 0, 0, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4095, 0, 4096, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 4096, 0, 3595, 4040, 0, 0, 0, 0, 4096, 46, 0, 3707, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6722, 8192, 8192, 8192, 8192, 8096, 7546, 7414] dup_rows_per_dst=[1036, 66, 68, 146, 0, 0, 1674, 0] combine_rows_per_src=[6722, 8192, 8192, 8192, 8192, 8096, 7546, 7414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389466112 dispatch_meta_bytes=229248 combine_payload_bytes=389466112 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=389695360 ingress_bytes=609667072 link_bytes=609667072 token_bytes=779161472 weight_movement_bytes=220200960 total_nvlink_bytes=999362432 weight_to_token_ratio=0.282613 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060311311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56520 payload_rows_remote=48376 dup_rows=9016 dup_rows_remote=3296 dedup_ratio=0.137573 router_tokens_per_expert=[0, 1106, 0, 0, 0, 0, 0, 0, 10, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4009, 0, 4096, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 3657, 4081, 0, 0, 0, 0, 4096, 0, 0, 3275, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2212, 6792, 7636, 7754, 8192, 7628, 8162, 8144] dup_rows_per_dst=[0, 226, 84, 88, 0, 912, 1986, 5720] combine_rows_per_src=[2212, 6792, 7636, 7754, 8192, 7628, 8162, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62954 payload_rows_remote=54762 dup_rows=2582 dup_rows_remote=2562 dedup_ratio=0.039398 router_tokens_per_expert=[0, 1034, 0, 0, 0, 1, 0, 0, 20, 158, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4073, 0, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4096, 0, 3677, 4085, 0, 0, 0, 0, 4096, 9, 0, 3266, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8192, 8190, 8192, 8126, 7354, 6532] dup_rows_per_dst=[2064, 360, 20, 120, 0, 0, 18, 0] combine_rows_per_src=[8176, 8192, 8192, 8190, 8192, 8126, 7354, 6532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63652 payload_rows_remote=55460 dup_rows=1884 dup_rows_remote=1736 dedup_ratio=0.028748 router_tokens_per_expert=[0, 787, 0, 0, 0, 0, 0, 0, 7, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4094, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 3647, 4085, 0, 0, 0, 0, 4096, 0, 0, 3632, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8192, 8192, 8190, 8192, 8158, 7294, 7264] dup_rows_per_dst=[1574, 148, 126, 36, 0, 0, 0, 0] combine_rows_per_src=[8170, 8192, 8192, 8190, 8192, 8158, 7294, 7264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=390297600 dispatch_meta_bytes=229760 combine_payload_bytes=390297600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=390527360 ingress_bytes=720599040 link_bytes=720599040 token_bytes=780824960 weight_movement_bytes=330301440 total_nvlink_bytes=1111126400 weight_to_token_ratio=0.423016 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253240249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=346759168 dispatch_meta_bytes=206688 combine_payload_bytes=346759168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346965856 ingress_bytes=456859648 link_bytes=456859648 token_bytes=693725024 weight_movement_bytes=110100480 total_nvlink_bytes=803825504 weight_to_token_ratio=0.158709 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.945540684e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=392534016 dispatch_meta_bytes=229296 combine_payload_bytes=392534016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392763312 ingress_bytes=502634496 link_bytes=502634496 token_bytes=785297328 weight_movement_bytes=110100480 total_nvlink_bytes=895397808 weight_to_token_ratio=0.140202 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.741640577e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62722 payload_rows_remote=54530 dup_rows=2814 dup_rows_remote=2804 dedup_ratio=0.042938 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 919, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 4050, 4096, 2700, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8170, 8192, 5400, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1770, 36, 0, 10, 152, 0, 846] combine_rows_per_src=[8192, 8170, 8192, 5400, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61836 payload_rows_remote=53690 dup_rows=3700 dup_rows_remote=356 dedup_ratio=0.056458 router_tokens_per_expert=[0, 1676, 0, 0, 0, 1, 0, 0, 28, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4072, 0, 4096, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 4096, 0, 2841, 4068, 0, 0, 0, 0, 4096, 22, 0, 3558, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8146, 8192, 8192, 8188, 8192, 8128, 5682, 7116] dup_rows_per_dst=[3344, 206, 80, 26, 0, 0, 44, 0] combine_rows_per_src=[8146, 8192, 8192, 8188, 8192, 8128, 5682, 7116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397537280 dispatch_meta_bytes=228784 combine_payload_bytes=397537280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397766064 ingress_bytes=507637760 link_bytes=507637760 token_bytes=795303344 weight_movement_bytes=110100480 total_nvlink_bytes=905403824 weight_to_token_ratio=0.138438 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.828655567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62948 payload_rows_remote=57328 dup_rows=2588 dup_rows_remote=2588 dedup_ratio=0.039490 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 29, 0, 1070, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4096, 0, 0, 0, 3992, 4096, 2810, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 240, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8176, 8192, 5620, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2006, 32, 0, 28, 42, 0, 480] combine_rows_per_src=[8192, 8176, 8192, 5620, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61722 payload_rows_remote=53530 dup_rows=3814 dup_rows_remote=3814 dedup_ratio=0.058197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 0, 2174, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 4050, 4096, 1678, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 4416, 8192, 8154, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 12, 3302, 18, 58, 0, 424] combine_rows_per_src=[8192, 4416, 8192, 8154, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390871040 dispatch_meta_bytes=229336 combine_payload_bytes=390871040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391100376 ingress_bytes=500971520 link_bytes=500971520 token_bytes=781971416 weight_movement_bytes=110100480 total_nvlink_bytes=892071896 weight_to_token_ratio=0.140799 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.712718689e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=384849920 dispatch_meta_bytes=216184 combine_payload_bytes=384849920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385066104 ingress_bytes=494950400 link_bytes=494950400 token_bytes=769916024 weight_movement_bytes=110100480 total_nvlink_bytes=880016504 weight_to_token_ratio=0.143003 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.608001510e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62124 payload_rows_remote=53932 dup_rows=3412 dup_rows_remote=3360 dedup_ratio=0.052063 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 61, 0, 1266, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 0, 4096, 0, 0, 0, 3850, 4096, 2564, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 399, 0, 0, 0, 4094] payload_rows_per_dst=[8154, 7496, 8192, 5514, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1914, 60, 558, 22, 52, 0, 806] combine_rows_per_src=[8154, 7496, 8192, 5514, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=239664 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=411166768 ingress_bytes=851329024 link_bytes=851329024 token_bytes=822093872 weight_movement_bytes=440401920 total_nvlink_bytes=1262495792 weight_to_token_ratio=0.535708 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.480601192e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58022 payload_rows_remote=49830 dup_rows=7514 dup_rows_remote=5238 dedup_ratio=0.114655 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 111, 0, 656, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 3997, 4096, 2641, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 721, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 1534, 8002, 8162, 8192, 7556, 8192, 8192] dup_rows_per_dst=[0, 0, 28, 5114, 84, 12, 0, 2276] combine_rows_per_src=[8192, 1534, 8002, 8162, 8192, 7556, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=383703040 dispatch_meta_bytes=229376 combine_payload_bytes=383703040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383932416 ingress_bytes=493803520 link_bytes=493803520 token_bytes=767635456 weight_movement_bytes=110100480 total_nvlink_bytes=877735936 weight_to_token_ratio=0.143428 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.588055380e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63414 payload_rows_remote=55222 dup_rows=2122 dup_rows_remote=2108 dedup_ratio=0.032379 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 61, 0, 713, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 0, 0, 4043, 4096, 3044, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 4094] payload_rows_per_dst=[8190, 8176, 8192, 6088, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1458, 14, 0, 48, 130, 0, 472] combine_rows_per_src=[8190, 8176, 8192, 6088, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62572 payload_rows_remote=54460 dup_rows=2964 dup_rows_remote=1198 dedup_ratio=0.045227 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 943, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 3972, 4096, 2654, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8112, 8192, 5308, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1766, 16, 0, 34, 132, 0, 1016] combine_rows_per_src=[8192, 8112, 8192, 5308, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386584576 dispatch_meta_bytes=229168 combine_payload_bytes=386584576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386813744 ingress_bytes=606785536 link_bytes=606785536 token_bytes=773398320 weight_movement_bytes=220200960 total_nvlink_bytes=993599280 weight_to_token_ratio=0.284719 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055299846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58412 payload_rows_remote=50220 dup_rows=7124 dup_rows_remote=7084 dedup_ratio=0.108704 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 20, 0, 1221, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4096, 0, 0, 0, 4030, 4096, 2658, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8164, 8152, 7754, 7916, 8192, 8192, 1850] dup_rows_per_dst=[40, 2378, 32, 4562, 32, 58, 0, 22] combine_rows_per_src=[8192, 8164, 8152, 7754, 7916, 8192, 8192, 1850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63872 payload_rows_remote=55682 dup_rows=1664 dup_rows_remote=1664 dedup_ratio=0.025391 router_tokens_per_expert=[0, 0, 3275, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 4020, 4095, 0] payload_rows_per_dst=[6550, 8186, 8192, 8192, 8190, 8192, 8192, 8178] dup_rows_per_dst=[0, 2, 2, 0, 0, 164, 0, 1496] combine_rows_per_src=[6550, 8186, 8192, 8192, 8190, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=357181440 dispatch_meta_bytes=220272 combine_payload_bytes=357181440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=357401712 ingress_bytes=577382400 link_bytes=577382400 token_bytes=714583152 weight_movement_bytes=220200960 total_nvlink_bytes=934784112 weight_to_token_ratio=0.308153 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.004162957e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63436 payload_rows_remote=55244 dup_rows=2100 dup_rows_remote=2098 dedup_ratio=0.032043 router_tokens_per_expert=[0, 0, 3065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 4062, 4093, 0] payload_rows_per_dst=[6130, 8192, 8192, 8192, 8186, 8192, 8192, 8160] dup_rows_per_dst=[0, 8, 0, 2, 0, 1080, 0, 1010] combine_rows_per_src=[6130, 8192, 8192, 8192, 8186, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395831296 dispatch_meta_bytes=229320 combine_payload_bytes=395831296 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=396060616 ingress_bytes=616032256 link_bytes=616032256 token_bytes=791891912 weight_movement_bytes=220200960 total_nvlink_bytes=1012092872 weight_to_token_ratio=0.278069 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071381413e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64376 payload_rows_remote=56184 dup_rows=1160 dup_rows_remote=1160 dedup_ratio=0.017700 router_tokens_per_expert=[0, 0, 3530, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 4021, 4089, 0] payload_rows_per_dst=[7060, 8182, 8192, 8192, 8178, 8192, 8192, 8188] dup_rows_per_dst=[0, 2, 0, 6, 0, 260, 0, 892] combine_rows_per_src=[7060, 8182, 8192, 8192, 8178, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390369280 dispatch_meta_bytes=222632 combine_payload_bytes=390369280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390591912 ingress_bytes=500469760 link_bytes=500469760 token_bytes=780961192 weight_movement_bytes=110100480 total_nvlink_bytes=891061672 weight_to_token_ratio=0.140981 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703992258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359976960 dispatch_meta_bytes=229216 combine_payload_bytes=359976960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360206176 ingress_bytes=470077440 link_bytes=470077440 token_bytes=720183136 weight_movement_bytes=110100480 total_nvlink_bytes=830283616 weight_to_token_ratio=0.152878 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.175419826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399128576 dispatch_meta_bytes=229384 combine_payload_bytes=399128576 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=399357960 ingress_bytes=619329536 link_bytes=619329536 token_bytes=798486536 weight_movement_bytes=220200960 total_nvlink_bytes=1018687496 weight_to_token_ratio=0.275773 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077115925e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395988992 dispatch_meta_bytes=229368 combine_payload_bytes=395988992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396218360 ingress_bytes=506089472 link_bytes=506089472 token_bytes=792207352 weight_movement_bytes=110100480 total_nvlink_bytes=902307832 weight_to_token_ratio=0.138979 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.801728292e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402726912 dispatch_meta_bytes=229376 combine_payload_bytes=402726912 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=402956288 ingress_bytes=512827392 link_bytes=512827392 token_bytes=805683200 weight_movement_bytes=110100480 total_nvlink_bytes=915783680 weight_to_token_ratio=0.136655 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.918911803e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63094 payload_rows_remote=54902 dup_rows=2442 dup_rows_remote=1762 dedup_ratio=0.037262 router_tokens_per_expert=[0, 0, 2931, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1044, 0, 0, 0, 0, 3899, 4077, 0] payload_rows_per_dst=[5862, 8176, 8192, 8192, 8154, 8192, 8192, 8134] dup_rows_per_dst=[0, 10, 0, 0, 0, 680, 0, 1752] combine_rows_per_src=[5862, 8176, 8192, 8192, 8154, 8192, 8192, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57872 payload_rows_remote=49724 dup_rows=7664 dup_rows_remote=626 dedup_ratio=0.116943 router_tokens_per_expert=[0, 0, 2258, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2072, 0, 0, 0, 0, 3516, 4092, 0] payload_rows_per_dst=[4488, 8190, 8164, 8192, 8184, 4314, 8192, 8148] dup_rows_per_dst=[0, 34, 0, 0, 0, 592, 0, 7038] combine_rows_per_src=[4488, 8190, 8164, 8192, 8184, 4314, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63034 payload_rows_remote=54842 dup_rows=2502 dup_rows_remote=2498 dedup_ratio=0.038177 router_tokens_per_expert=[0, 0, 3143, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 920, 0, 0, 0, 0, 3899, 4081, 0] payload_rows_per_dst=[6640, 8190, 8192, 8192, 8162, 8192, 8192, 7274] dup_rows_per_dst=[736, 0, 4, 4, 122, 484, 0, 1152] combine_rows_per_src=[6640, 8190, 8192, 8192, 8162, 8192, 8192, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59658 payload_rows_remote=51486 dup_rows=5878 dup_rows_remote=5878 dedup_ratio=0.089691 router_tokens_per_expert=[0, 0, 3061, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1210, 0, 0, 0, 0, 3778, 4092, 0] payload_rows_per_dst=[8114, 8172, 8192, 8192, 8184, 8192, 8192, 2420] dup_rows_per_dst=[5564, 0, 0, 2, 0, 312, 0, 0] combine_rows_per_src=[8114, 8172, 8192, 8192, 8184, 8192, 8192, 2420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=393537536 dispatch_meta_bytes=226656 combine_payload_bytes=393537536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393764192 ingress_bytes=613738496 link_bytes=613738496 token_bytes=787301728 weight_movement_bytes=220200960 total_nvlink_bytes=1007502688 weight_to_token_ratio=0.279691 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067392187e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58956 payload_rows_remote=50810 dup_rows=6580 dup_rows_remote=334 dedup_ratio=0.100403 router_tokens_per_expert=[0, 0, 3158, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 847, 0, 0, 0, 0, 4038, 4088, 0] payload_rows_per_dst=[8146, 8190, 8192, 8174, 8176, 8192, 8192, 1694] dup_rows_per_dst=[6246, 2, 22, 4, 0, 306, 0, 0] combine_rows_per_src=[8146, 8190, 8192, 8174, 8176, 8192, 8192, 1694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65510 payload_rows_remote=57320 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8190, 8192, 8170, 8192] dup_rows_per_dst=[22, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8190, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=356421632 dispatch_meta_bytes=201400 combine_payload_bytes=356421632 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=356623032 ingress_bytes=796823552 link_bytes=796823552 token_bytes=713044664 weight_movement_bytes=440401920 total_nvlink_bytes=1153446584 weight_to_token_ratio=0.617636 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.385807212e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=72 dedup_ratio=0.001221 router_tokens_per_expert=[37, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8164, 8192, 8192, 8188, 8192, 8144, 8192] dup_rows_per_dst=[70, 0, 0, 8, 0, 2, 0, 0] combine_rows_per_src=[8192, 8164, 8192, 8192, 8188, 8192, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=393107456 dispatch_meta_bytes=229360 combine_payload_bytes=393107456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=393336816 ingress_bytes=723408896 link_bytes=723408896 token_bytes=786444272 weight_movement_bytes=330301440 total_nvlink_bytes=1116745712 weight_to_token_ratio=0.419993 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.258127050e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65464 payload_rows_remote=57316 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[36, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8164, 8192, 8192, 8192, 8192, 8148, 8192] dup_rows_per_dst=[70, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8164, 8192, 8192, 8192, 8192, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369051648 dispatch_meta_bytes=229456 combine_payload_bytes=369051648 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=369281104 ingress_bytes=589252608 link_bytes=589252608 token_bytes=738332752 weight_movement_bytes=220200960 total_nvlink_bytes=958533712 weight_to_token_ratio=0.298241 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024807201e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364206080 dispatch_meta_bytes=204576 combine_payload_bytes=364206080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364410656 ingress_bytes=694507520 link_bytes=694507520 token_bytes=728616736 weight_movement_bytes=330301440 total_nvlink_bytes=1058918176 weight_to_token_ratio=0.453327 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207862804e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229384 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411099144 ingress_bytes=520970240 link_bytes=520970240 token_bytes=821968904 weight_movement_bytes=110100480 total_nvlink_bytes=932069384 weight_to_token_ratio=0.133947 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.060529322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410468352 dispatch_meta_bytes=229344 combine_payload_bytes=410468352 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410697696 ingress_bytes=850870272 link_bytes=850870272 token_bytes=821166048 weight_movement_bytes=440401920 total_nvlink_bytes=1261567968 weight_to_token_ratio=0.536313 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.479803347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410841088 dispatch_meta_bytes=229552 combine_payload_bytes=410841088 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411070640 ingress_bytes=741142528 link_bytes=741142528 token_bytes=821911728 weight_movement_bytes=330301440 total_nvlink_bytes=1152213168 weight_to_token_ratio=0.401870 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288968753e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65258 payload_rows_remote=57066 dup_rows=278 dup_rows_remote=278 dedup_ratio=0.004242 router_tokens_per_expert=[114, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 19, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7996, 8186, 8192, 8186, 8192, 8122, 8192] dup_rows_per_dst=[228, 0, 0, 50, 0, 0, 0, 0] combine_rows_per_src=[8192, 7996, 8186, 8192, 8186, 8192, 8122, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7459243504 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3730765296 ingress_bytes=5490085888 link_bytes=5490085888 total_nvlink_bytes=9220851184 weight_to_token_ratio=0.236164 prefetch_share_of_ingress=0.200544 dedup_ratio=0.085986 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.548162323e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7636735640 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3819531928 ingress_bytes=6349514752 link_bytes=6349514752 total_nvlink_bytes=10169046680 weight_to_token_ratio=0.331596 prefetch_share_of_ingress=0.156060 dedup_ratio=0.085120 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.104285047e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64464 payload_rows_remote=56272 dup_rows=1072 dup_rows_remote=1062 dedup_ratio=0.016357 router_tokens_per_expert=[64, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7268, 8076, 8192, 8172, 8192, 8192, 8180, 8192] dup_rows_per_dst=[128, 550, 0, 0, 20, 0, 364, 10] combine_rows_per_src=[7268, 8076, 8192, 8172, 8192, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65368 payload_rows_remote=57180 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[81, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8104, 8188, 8192, 8188, 8192, 8130, 8182] dup_rows_per_dst=[162, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8104, 8188, 8192, 8188, 8192, 8130, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7301708728 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3651978168 ingress_bytes=5741639680 link_bytes=5741639680 total_nvlink_bytes=9393617848 weight_to_token_ratio=0.286496 prefetch_share_of_ingress=0.210934 dedup_ratio=0.101633 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.985655741e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65446 payload_rows_remote=57324 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.001373 router_tokens_per_expert=[45, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8122, 8192, 8192, 8192, 8192, 8172, 8192] dup_rows_per_dst=[90, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8122, 8192, 8192, 8192, 8192, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409049088 dispatch_meta_bytes=229376 combine_payload_bytes=409049088 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=409278464 ingress_bytes=519149568 link_bytes=519149568 token_bytes=818327552 weight_movement_bytes=110100480 total_nvlink_bytes=928428032 weight_to_token_ratio=0.134543 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.028864842e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65366 payload_rows_remote=57174 dup_rows=170 dup_rows_remote=16 dedup_ratio=0.002594 router_tokens_per_expert=[57, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 8138, 8188, 8190, 8106, 8192] dup_rows_per_dst=[154, 0, 12, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8168, 8192, 8138, 8188, 8190, 8106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=403357696 dispatch_meta_bytes=229336 combine_payload_bytes=403357696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=403587032 ingress_bytes=623558656 link_bytes=623558656 token_bytes=806944728 weight_movement_bytes=220200960 total_nvlink_bytes=1027145688 weight_to_token_ratio=0.272882 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.084471061e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409866240 dispatch_meta_bytes=229392 combine_payload_bytes=409866240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410095632 ingress_bytes=519966720 link_bytes=519966720 token_bytes=819961872 weight_movement_bytes=110100480 total_nvlink_bytes=930062352 weight_to_token_ratio=0.134275 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.043076459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7388461536 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3695378912 ingress_bytes=5344589824 link_bytes=5344589824 total_nvlink_bytes=9039968736 weight_to_token_ratio=0.223525 prefetch_share_of_ingress=0.185403 dedup_ratio=0.095526 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.295120738e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=229656 combine_payload_bytes=410898432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411128088 ingress_bytes=520998912 link_bytes=520998912 token_bytes=822026520 weight_movement_bytes=110100480 total_nvlink_bytes=932127000 weight_to_token_ratio=0.133938 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061027976e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409823232 dispatch_meta_bytes=228760 combine_payload_bytes=409823232 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410051992 ingress_bytes=740124672 link_bytes=740124672 token_bytes=819875224 weight_movement_bytes=330301440 total_nvlink_bytes=1150176664 weight_to_token_ratio=0.402868 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287198534e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6890762064 prefetch_bytes=880803840 reduce_bytes=2091909120 weight_movement_bytes=2972712960 egress_bytes=3446466384 ingress_bytes=6417008640 link_bytes=6417008640 total_nvlink_bytes=9863475024 weight_to_token_ratio=0.431406 prefetch_share_of_ingress=0.137261 dedup_ratio=0.148029 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=19 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.116023344e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7415011704 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3708653944 ingress_bytes=5357864960 link_bytes=5358094336 total_nvlink_bytes=9066518904 weight_to_token_ratio=0.222725 prefetch_share_of_ingress=0.164395 dedup_ratio=0.093970 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5715 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.318607305e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7218846280 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3610561096 ingress_bytes=5259792384 link_bytes=5259792384 total_nvlink_bytes=8870353480 weight_to_token_ratio=0.228777 prefetch_share_of_ingress=0.209325 dedup_ratio=0.117001 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.147644043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6822922368 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3412516992 ingress_bytes=5612414976 link_bytes=5612414976 total_nvlink_bytes=9024931968 weight_to_token_ratio=0.322737 prefetch_share_of_ingress=0.196173 dedup_ratio=0.152069 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.760912727e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-29 23:43:05] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 236.3 | throughput per GPU (TFLOP/s/GPU): 522.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.770842E+00 | loss scale: 1.0 | grad norm: 4.336 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38950 dup_rows=21228 dup_rows_remote=19156 dedup_ratio=0.323914 router_tokens_per_expert=[1341, 1324, 746, 914, 22, 1266, 283, 38, 652, 1216, 271, 320, 12, 862, 312, 587, 799, 361, 25, 13, 943, 223, 677, 1268, 284, 633, 569, 519, 1144, 144, 597, 0, 473, 620, 1085, 5, 591, 2, 1081, 203, 260, 757, 627, 616, 17, 434, 58, 396, 166, 873, 213, 317, 187, 461, 231, 301, 595, 546, 799, 548, 704, 1002, 233, 2] payload_rows_per_dst=[6290, 5358, 5824, 6064, 5462, 5898, 4216, 5196] dup_rows_per_dst=[3930, 2072, 2794, 3720, 2658, 3114, 1282, 1658] combine_rows_per_src=[6290, 5358, 5824, 6064, 5462, 5898, 4216, 5196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45374 payload_rows_remote=39872 dup_rows=20162 dup_rows_remote=18656 dedup_ratio=0.307648 router_tokens_per_expert=[1346, 844, 481, 1150, 0, 839, 301, 63, 467, 1707, 1220, 281, 1, 940, 190, 971, 748, 474, 5, 3, 779, 321, 660, 514, 131, 857, 534, 653, 568, 81, 482, 0, 794, 811, 642, 15, 686, 0, 1167, 161, 310, 344, 456, 424, 22, 426, 66, 553, 437, 389, 342, 623, 95, 690, 137, 647, 392, 711, 662, 626, 1370, 916, 242, 1] payload_rows_per_dst=[5314, 7066, 5502, 5676, 5658, 5532, 4734, 5892] dup_rows_per_dst=[2042, 4488, 1506, 2768, 2894, 2362, 1986, 2116] combine_rows_per_src=[5314, 7066, 5502, 5676, 5658, 5532, 4734, 5892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39038 dup_rows=20938 dup_rows_remote=18326 dedup_ratio=0.319489 router_tokens_per_expert=[1095, 1192, 633, 967, 1, 658, 308, 31, 836, 1107, 878, 429, 0, 1077, 118, 939, 687, 742, 20, 3, 1205, 224, 761, 936, 314, 436, 394, 282, 691, 260, 493, 1, 408, 733, 744, 23, 914, 0, 1006, 212, 449, 671, 370, 745, 25, 300, 12, 261, 423, 456, 215, 474, 218, 827, 232, 447, 426, 980, 759, 525, 771, 1215, 208, 1] payload_rows_per_dst=[5200, 5906, 6082, 5560, 5272, 5526, 5704, 5348] dup_rows_per_dst=[2380, 2742, 3074, 2612, 2808, 2330, 3000, 1992] combine_rows_per_src=[5200, 5906, 6082, 5560, 5272, 5526, 5704, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44796 payload_rows_remote=39842 dup_rows=20740 dup_rows_remote=18464 dedup_ratio=0.316467 router_tokens_per_expert=[1274, 923, 1056, 1071, 32, 1162, 559, 13, 324, 1216, 720, 412, 4, 904, 334, 381, 962, 391, 4, 0, 905, 264, 1343, 652, 178, 909, 590, 339, 679, 126, 903, 0, 375, 320, 597, 8, 1007, 0, 914, 394, 203, 719, 465, 681, 0, 450, 81, 302, 268, 326, 333, 496, 167, 1072, 117, 172, 639, 911, 479, 163, 1179, 1163, 136, 1] payload_rows_per_dst=[6062, 4874, 5918, 6036, 4954, 5742, 5602, 5608] dup_rows_per_dst=[3570, 1284, 3124, 3378, 2276, 2608, 2732, 1768] combine_rows_per_src=[6062, 4874, 5918, 6036, 4954, 5742, 5602, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44298 payload_rows_remote=39404 dup_rows=21238 dup_rows_remote=19338 dedup_ratio=0.324066 router_tokens_per_expert=[594, 1024, 968, 821, 4, 780, 354, 21, 1032, 1258, 904, 257, 0, 1276, 259, 925, 1150, 561, 22, 0, 439, 133, 787, 757, 155, 434, 670, 475, 773, 166, 428, 0, 600, 803, 828, 16, 887, 0, 918, 362, 389, 364, 626, 548, 7, 505, 60, 304, 297, 327, 128, 533, 81, 988, 335, 325, 1167, 705, 928, 344, 980, 770, 216, 0] payload_rows_per_dst=[5370, 6276, 5546, 4750, 5622, 4894, 5704, 6136] dup_rows_per_dst=[2574, 3030, 2152, 1452, 3206, 1900, 2840, 4084] combine_rows_per_src=[5370, 6276, 5546, 4750, 5622, 4894, 5704, 6136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44770 payload_rows_remote=38496 dup_rows=20766 dup_rows_remote=16924 dedup_ratio=0.316864 router_tokens_per_expert=[1040, 1003, 402, 799, 3, 924, 358, 55, 720, 1120, 701, 569, 0, 1212, 10, 787, 788, 642, 2, 3, 614, 300, 843, 547, 240, 533, 745, 360, 657, 280, 1271, 1, 690, 512, 520, 1, 490, 5, 828, 141, 456, 455, 242, 594, 1, 402, 47, 484, 789, 462, 106, 1148, 99, 657, 112, 565, 458, 824, 460, 755, 950, 1780, 206, 0] payload_rows_per_dst=[4982, 5556, 5468, 5948, 4640, 5122, 6274, 6780] dup_rows_per_dst=[2106, 2442, 2010, 2226, 1734, 2320, 3842, 4086] combine_rows_per_src=[4982, 5556, 5468, 5948, 4640, 5122, 6274, 6780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=232424 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279426024 ingress_bytes=719595520 link_bytes=719595520 token_bytes=558619624 weight_movement_bytes=440401920 total_nvlink_bytes=999021544 weight_to_token_ratio=0.788375 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251494962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229456 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280053840 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559878224 weight_movement_bytes=110100480 total_nvlink_bytes=669978704 weight_to_token_ratio=0.196651 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=234112 combine_payload_bytes=285802496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286036608 ingress_bytes=395902976 link_bytes=395902976 token_bytes=571839104 weight_movement_bytes=110100480 total_nvlink_bytes=681939584 weight_to_token_ratio=0.192538 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.885403901e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285587456 dispatch_meta_bytes=233224 combine_payload_bytes=285587456 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=285820680 ingress_bytes=505788416 link_bytes=505788416 token_bytes=571408136 weight_movement_bytes=220200960 total_nvlink_bytes=791609096 weight_to_token_ratio=0.385365 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.796492433e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282447872 dispatch_meta_bytes=234968 combine_payload_bytes=282447872 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=282682840 ingress_bytes=502648832 link_bytes=502648832 token_bytes=565130712 weight_movement_bytes=220200960 total_nvlink_bytes=785331672 weight_to_token_ratio=0.389646 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=9600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.741889904e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275939328 dispatch_meta_bytes=221680 combine_payload_bytes=275939328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276161008 ingress_bytes=386039808 link_bytes=386039808 token_bytes=552100336 weight_movement_bytes=110100480 total_nvlink_bytes=662200816 weight_to_token_ratio=0.199421 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.713867188e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49434 payload_rows_remote=41702 dup_rows=16102 dup_rows_remote=11058 dedup_ratio=0.245697 router_tokens_per_expert=[0, 0, 0, 51, 3299, 0, 2618, 1873, 3070, 72, 3133, 0, 0, 3, 0, 3180, 1, 1, 0, 6, 2399, 3463, 0, 0, 6, 0, 1857, 0, 1802, 417, 0, 7, 0, 0, 0, 0, 5, 0, 0, 1377, 0, 0, 2, 1, 0, 1, 1, 1, 0, 524, 173, 0, 267, 7, 0, 0, 0, 2, 1825, 166, 857, 5, 279, 17] payload_rows_per_dst=[6840, 7732, 7820, 5500, 2764, 6146, 6908, 5724] dup_rows_per_dst=[2244, 5044, 3920, 1630, 0, 6, 1632, 1626] combine_rows_per_src=[6840, 7732, 7820, 5500, 2764, 6146, 6908, 5724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50676 payload_rows_remote=45124 dup_rows=14860 dup_rows_remote=14252 dedup_ratio=0.226746 router_tokens_per_expert=[0, 0, 0, 41, 2886, 0, 2477, 1949, 3124, 16, 3155, 0, 0, 13, 0, 2963, 1, 0, 0, 1, 2062, 3653, 0, 0, 0, 0, 2559, 0, 2413, 666, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1822, 0, 0, 5, 1, 0, 0, 0, 0, 0, 538, 271, 0, 244, 0, 0, 0, 0, 0, 1137, 93, 331, 0, 342, 3] payload_rows_per_dst=[6826, 7702, 6448, 5552, 5042, 6252, 6122, 6732] dup_rows_per_dst=[2108, 4592, 2236, 608, 1354, 8, 1756, 2198] combine_rows_per_src=[6826, 7702, 6448, 5552, 5042, 6252, 6122, 6732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50676 payload_rows_remote=42746 dup_rows=14860 dup_rows_remote=10824 dedup_ratio=0.226746 router_tokens_per_expert=[0, 0, 0, 29, 3216, 0, 2539, 2426, 2907, 27, 2300, 0, 0, 3, 0, 2532, 3, 0, 0, 0, 2188, 3792, 0, 0, 0, 0, 2356, 1, 2310, 181, 1, 0, 0, 0, 1, 0, 0, 0, 0, 2290, 0, 0, 4, 1, 0, 0, 5, 0, 0, 653, 94, 0, 176, 6, 0, 0, 3, 0, 1746, 81, 390, 1, 500, 6] payload_rows_per_dst=[7106, 6648, 7930, 4834, 4582, 5824, 6758, 6994] dup_rows_per_dst=[2882, 3076, 4036, 152, 0, 10, 1532, 3172] combine_rows_per_src=[7106, 6648, 7930, 4834, 4582, 5824, 6758, 6994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50796 payload_rows_remote=42896 dup_rows=14740 dup_rows_remote=10558 dedup_ratio=0.224915 router_tokens_per_expert=[0, 0, 0, 25, 3167, 0, 2289, 2034, 3284, 4, 2796, 0, 0, 0, 0, 2999, 0, 0, 0, 0, 2059, 3807, 0, 0, 0, 0, 2610, 0, 2564, 188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 0, 4, 0, 0, 0, 0, 0, 0, 843, 192, 0, 220, 0, 0, 0, 0, 0, 551, 79, 491, 0, 325, 2] payload_rows_per_dst=[6686, 6754, 4118, 5334, 7900, 6902, 6720, 6382] dup_rows_per_dst=[2010, 3384, 0, 172, 4182, 1134, 2124, 1734] combine_rows_per_src=[6686, 6754, 4118, 5334, 7900, 6902, 6720, 6382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45120 payload_rows_remote=39128 dup_rows=20416 dup_rows_remote=17448 dedup_ratio=0.311523 router_tokens_per_expert=[1582, 400, 393, 1101, 10, 303, 450, 91, 724, 977, 928, 149, 1, 1295, 97, 822, 575, 221, 21, 8, 725, 242, 793, 785, 182, 1160, 365, 614, 672, 163, 324, 1, 386, 324, 981, 5, 963, 24, 1359, 694, 230, 683, 381, 830, 8, 513, 30, 392, 247, 538, 415, 592, 507, 860, 414, 1005, 533, 769, 665, 359, 720, 856, 311, 0] payload_rows_per_dst=[4578, 5376, 5586, 5106, 5724, 6208, 6550, 5992] dup_rows_per_dst=[878, 1692, 2118, 1856, 3214, 3130, 4560, 2968] combine_rows_per_src=[4578, 5376, 5586, 5106, 5724, 6208, 6550, 5992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298919936 dispatch_meta_bytes=211040 combine_payload_bytes=298919936 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=299130976 ingress_bytes=739321856 link_bytes=739321856 token_bytes=598050912 weight_movement_bytes=440401920 total_nvlink_bytes=1038452832 weight_to_token_ratio=0.736395 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.285802305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323448832 dispatch_meta_bytes=237504 combine_payload_bytes=323448832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=323686336 ingress_bytes=543649792 link_bytes=543649792 token_bytes=647135168 weight_movement_bytes=220200960 total_nvlink_bytes=867336128 weight_to_token_ratio=0.340270 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.454964033e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50664 payload_rows_remote=42846 dup_rows=14872 dup_rows_remote=10954 dedup_ratio=0.226929 router_tokens_per_expert=[0, 0, 0, 27, 3348, 0, 2468, 2240, 2838, 28, 2824, 0, 0, 0, 0, 3027, 2, 0, 0, 0, 2113, 3737, 0, 1, 0, 0, 2325, 2, 2586, 502, 0, 0, 0, 0, 0, 0, 4, 1, 0, 2276, 0, 3, 0, 0, 0, 0, 0, 0, 0, 449, 85, 2, 209, 6, 0, 2, 0, 0, 1011, 68, 334, 0, 250, 0] payload_rows_per_dst=[5352, 5668, 4230, 6554, 7918, 7818, 6900, 6224] dup_rows_per_dst=[1116, 36, 2, 2628, 4118, 3918, 1302, 1752] combine_rows_per_src=[5352, 5668, 4230, 6554, 7918, 7818, 6900, 6224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50608 payload_rows_remote=43376 dup_rows=14928 dup_rows_remote=12918 dedup_ratio=0.227783 router_tokens_per_expert=[0, 0, 0, 5, 3442, 0, 2896, 2396, 2994, 11, 2269, 0, 0, 0, 0, 2013, 2, 0, 1, 0, 2448, 3919, 0, 0, 0, 0, 2352, 0, 2218, 134, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2143, 0, 0, 10, 0, 2, 1, 0, 0, 1, 885, 93, 0, 198, 2, 0, 0, 3, 0, 1547, 69, 372, 1, 320, 20] payload_rows_per_dst=[4802, 4544, 4896, 7174, 8036, 7114, 7232, 6810] dup_rows_per_dst=[0, 16, 6, 3322, 4090, 2926, 2010, 2558] combine_rows_per_src=[4802, 4544, 4896, 7174, 8036, 7114, 7232, 6810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=306403328 dispatch_meta_bytes=214280 combine_payload_bytes=306403328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=306617608 ingress_bytes=526604288 link_bytes=526604288 token_bytes=613020936 weight_movement_bytes=220200960 total_nvlink_bytes=833221896 weight_to_token_ratio=0.359206 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.158514684e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=307478528 dispatch_meta_bytes=213816 combine_payload_bytes=307478528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307692344 ingress_bytes=417579008 link_bytes=417579008 token_bytes=615170872 weight_movement_bytes=110100480 total_nvlink_bytes=725271352 weight_to_token_ratio=0.178975 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.262385749e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310919168 dispatch_meta_bytes=225176 combine_payload_bytes=310919168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311144344 ingress_bytes=421019648 link_bytes=421019648 token_bytes=622063512 weight_movement_bytes=110100480 total_nvlink_bytes=732163992 weight_to_token_ratio=0.176992 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.322224137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=307120128 dispatch_meta_bytes=215200 combine_payload_bytes=307120128 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=307335328 ingress_bytes=527321088 link_bytes=527321088 token_bytes=614455456 weight_movement_bytes=220200960 total_nvlink_bytes=834656416 weight_to_token_ratio=0.358368 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.170981014e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280469504 dispatch_meta_bytes=226304 combine_payload_bytes=280469504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280695808 ingress_bytes=500670464 link_bytes=500670464 token_bytes=561165312 weight_movement_bytes=220200960 total_nvlink_bytes=781366272 weight_to_token_ratio=0.392399 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707482830e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58864 payload_rows_remote=52260 dup_rows=6672 dup_rows_remote=6438 dedup_ratio=0.101807 router_tokens_per_expert=[0, 0, 1, 545, 16, 172, 1643, 0, 0, 435, 0, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 4017, 3765, 1, 0, 0, 3905, 3958, 0, 2125, 43, 0, 101, 425, 0, 0, 0, 312, 4010, 13, 4076, 14, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 41, 0, 0, 41] payload_rows_per_dst=[8098, 6604, 7532, 5040, 7382, 8182, 7916, 8110] dup_rows_per_dst=[4466, 234, 0, 354, 1282, 236, 0, 100] combine_rows_per_src=[8098, 6604, 7532, 5040, 7382, 8182, 7916, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59352 payload_rows_remote=54286 dup_rows=6184 dup_rows_remote=4884 dedup_ratio=0.094360 router_tokens_per_expert=[0, 0, 0, 424, 0, 275, 1714, 0, 0, 364, 0, 0, 0, 3336, 0, 0, 0, 0, 0, 0, 0, 4039, 3822, 0, 0, 0, 3768, 3938, 0, 2064, 13, 0, 6, 283, 0, 0, 0, 417, 4087, 0, 4094, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 51, 0, 0, 23] payload_rows_per_dst=[7006, 7202, 7644, 5066, 8188, 8192, 7876, 8178] dup_rows_per_dst=[3144, 198, 0, 1300, 1398, 86, 0, 58] combine_rows_per_src=[7006, 7202, 7644, 5066, 8188, 8192, 7876, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60856 payload_rows_remote=53052 dup_rows=4680 dup_rows_remote=4678 dedup_ratio=0.071411 router_tokens_per_expert=[0, 0, 1, 214, 2, 201, 1504, 0, 0, 224, 0, 0, 0, 3291, 0, 0, 6, 0, 0, 0, 0, 4049, 3897, 0, 0, 0, 3947, 4020, 0, 2687, 1, 0, 7, 255, 0, 2, 1, 193, 4087, 5, 4087, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 38, 0, 0, 26] payload_rows_per_dst=[8140, 6958, 7804, 5374, 8192, 8176, 8040, 8172] dup_rows_per_dst=[3598, 72, 2, 2, 908, 16, 0, 82] combine_rows_per_src=[8140, 6958, 7804, 5374, 8192, 8176, 8040, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59456 payload_rows_remote=51264 dup_rows=6080 dup_rows_remote=5128 dedup_ratio=0.092773 router_tokens_per_expert=[0, 0, 4, 254, 1, 280, 1902, 0, 0, 264, 0, 0, 0, 3280, 0, 0, 0, 0, 0, 0, 0, 3972, 3818, 0, 0, 0, 3788, 3974, 0, 2459, 1, 0, 2, 171, 0, 0, 0, 307, 4092, 0, 4092, 3, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 0, 34] payload_rows_per_dst=[4576, 6978, 7636, 7912, 8192, 8188, 7948, 8026] dup_rows_per_dst=[306, 110, 0, 4584, 952, 56, 0, 72] combine_rows_per_src=[4576, 6978, 7636, 7912, 8192, 8188, 7948, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=374599680 dispatch_meta_bytes=234792 combine_payload_bytes=374599680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374834472 ingress_bytes=484700160 link_bytes=484700160 token_bytes=749434152 weight_movement_bytes=110100480 total_nvlink_bytes=859534632 weight_to_token_ratio=0.146911 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.429732977e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=389122048 dispatch_meta_bytes=236680 combine_payload_bytes=389122048 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=389358728 ingress_bytes=939624448 link_bytes=939624448 token_bytes=778480776 weight_movement_bytes=550502400 total_nvlink_bytes=1328983176 weight_to_token_ratio=0.707150 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.634161457e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=380276736 dispatch_meta_bytes=230920 combine_payload_bytes=380276736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380507656 ingress_bytes=600477696 link_bytes=600477696 token_bytes=760784392 weight_movement_bytes=220200960 total_nvlink_bytes=980985352 weight_to_token_ratio=0.289439 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044329475e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59722 payload_rows_remote=51738 dup_rows=5814 dup_rows_remote=5814 dedup_ratio=0.088715 router_tokens_per_expert=[0, 0, 5, 314, 0, 205, 1742, 0, 0, 491, 0, 0, 1, 3555, 0, 0, 0, 0, 0, 0, 0, 4061, 3788, 0, 0, 0, 3841, 3991, 0, 2140, 13, 0, 1, 127, 0, 0, 0, 254, 4092, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 39, 1, 0, 9] payload_rows_per_dst=[4210, 7542, 7576, 7896, 8190, 8190, 7984, 8134] dup_rows_per_dst=[322, 552, 0, 4092, 760, 2, 0, 86] combine_rows_per_src=[4210, 7542, 7576, 7896, 8190, 8190, 7984, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60012 payload_rows_remote=51828 dup_rows=5524 dup_rows_remote=5418 dedup_ratio=0.084290 router_tokens_per_expert=[0, 2, 11, 346, 0, 279, 1621, 0, 0, 246, 0, 0, 0, 3400, 0, 0, 0, 0, 0, 0, 0, 4042, 4019, 1, 0, 0, 3926, 4011, 0, 1790, 0, 0, 0, 226, 0, 0, 0, 513, 4093, 0, 4087, 4, 53, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 5, 1, 0, 0, 32, 0, 1, 55] payload_rows_per_dst=[4278, 7156, 8040, 7976, 8190, 8184, 8026, 8162] dup_rows_per_dst=[240, 136, 0, 3456, 1474, 106, 2, 110] combine_rows_per_src=[4278, 7156, 8040, 7976, 8190, 8184, 8026, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=367460352 dispatch_meta_bytes=225568 combine_payload_bytes=367460352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=367685920 ingress_bytes=697761792 link_bytes=697761792 token_bytes=735146272 weight_movement_bytes=330301440 total_nvlink_bytes=1065447712 weight_to_token_ratio=0.449300 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.213522518e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=370857984 dispatch_meta_bytes=230208 combine_payload_bytes=370857984 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=371088192 ingress_bytes=591058944 link_bytes=591058944 token_bytes=741946176 weight_movement_bytes=220200960 total_nvlink_bytes=962147136 weight_to_token_ratio=0.296788 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.027948716e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49314 payload_rows_remote=42096 dup_rows=16222 dup_rows_remote=13154 dedup_ratio=0.247528 router_tokens_per_expert=[0, 0, 1, 21, 2568, 1, 2083, 2335, 3181, 8, 2685, 0, 0, 0, 0, 2780, 0, 0, 0, 1, 2495, 3544, 1, 0, 0, 0, 2892, 0, 2702, 338, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1915, 0, 0, 14, 0, 0, 0, 4, 0, 0, 613, 163, 0, 166, 3, 0, 1, 0, 0, 1030, 89, 785, 4, 343, 0] payload_rows_per_dst=[4696, 3572, 5890, 7144, 7514, 7682, 5598, 7218] dup_rows_per_dst=[20, 8, 910, 3102, 3408, 4276, 1430, 3068] combine_rows_per_src=[4696, 3572, 5890, 7144, 7514, 7682, 5598, 7218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=371503104 dispatch_meta_bytes=228984 combine_payload_bytes=371503104 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=371732088 ingress_bytes=481603584 link_bytes=481603584 token_bytes=743235192 weight_movement_bytes=110100480 total_nvlink_bytes=853335672 weight_to_token_ratio=0.148137 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.375878428e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56652 payload_rows_remote=48464 dup_rows=8884 dup_rows_remote=8852 dedup_ratio=0.135559 router_tokens_per_expert=[0, 0, 2713, 0, 0, 3269, 0, 0, 10, 0, 0, 16, 1, 0, 0, 0, 0, 0, 3020, 0, 0, 0, 0, 0, 0, 164, 0, 3099, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 148, 3100, 3, 0, 205, 0, 0, 0, 43, 0, 0, 0, 6, 1951, 0, 0, 0, 0, 3771, 0, 3979, 0, 0, 2891, 4083, 0] payload_rows_per_dst=[7978, 8188, 6040, 6356, 6336, 8008, 7964, 5782] dup_rows_per_dst=[3986, 32, 0, 762, 160, 452, 3492, 0] combine_rows_per_src=[7978, 8188, 6040, 6356, 6336, 8008, 7964, 5782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57934 payload_rows_remote=51256 dup_rows=7602 dup_rows_remote=7026 dedup_ratio=0.115997 router_tokens_per_expert=[0, 0, 2592, 0, 0, 3438, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 0, 0, 3434, 0, 0, 0, 0, 0, 0, 182, 0, 3286, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 231, 2822, 3, 0, 182, 0, 0, 0, 5, 0, 0, 0, 4, 1419, 0, 0, 0, 0, 3592, 0, 3888, 0, 0, 3286, 4095, 0] payload_rows_per_dst=[8066, 8190, 6868, 6678, 5874, 7988, 7698, 6572] dup_rows_per_dst=[3994, 300, 0, 576, 232, 168, 2332, 0] combine_rows_per_src=[8066, 8190, 6868, 6678, 5874, 7988, 7698, 6572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60506 payload_rows_remote=53050 dup_rows=5030 dup_rows_remote=5030 dedup_ratio=0.076752 router_tokens_per_expert=[0, 0, 1634, 0, 0, 3829, 0, 0, 0, 0, 0, 12, 5, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 136, 0, 3042, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 278, 3420, 4, 0, 42, 0, 0, 0, 10, 0, 2, 0, 4, 850, 0, 0, 0, 0, 3621, 0, 4053, 0, 0, 3860, 4091, 0] payload_rows_per_dst=[8098, 8192, 7456, 6230, 6980, 8154, 7676, 7720] dup_rows_per_dst=[2828, 24, 0, 420, 416, 64, 1278, 0] combine_rows_per_src=[8098, 8192, 7456, 6230, 6980, 8154, 7676, 7720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58732 payload_rows_remote=51570 dup_rows=6804 dup_rows_remote=6472 dedup_ratio=0.103821 router_tokens_per_expert=[0, 0, 1517, 0, 0, 3748, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 0, 178, 0, 3395, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 238, 3509, 0, 0, 59, 0, 0, 0, 143, 0, 0, 0, 5, 473, 0, 0, 0, 0, 3589, 0, 4045, 0, 0, 3685, 4096, 0] payload_rows_per_dst=[5880, 8192, 7792, 6910, 7162, 8134, 7292, 7370] dup_rows_per_dst=[1620, 296, 2848, 506, 332, 360, 842, 0] combine_rows_per_src=[5880, 8192, 7792, 6910, 7162, 8134, 7292, 7370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301744128 dispatch_meta_bytes=221000 combine_payload_bytes=301744128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301965128 ingress_bytes=411844608 link_bytes=411844608 token_bytes=603709256 weight_movement_bytes=110100480 total_nvlink_bytes=713809736 weight_to_token_ratio=0.182373 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.162655101e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=347389952 dispatch_meta_bytes=229264 combine_payload_bytes=347389952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347619216 ingress_bytes=457490432 link_bytes=457490432 token_bytes=695009168 weight_movement_bytes=110100480 total_nvlink_bytes=805109648 weight_to_token_ratio=0.158416 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.956511055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=233128 combine_payload_bytes=367403008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367636136 ingress_bytes=477503488 link_bytes=477503488 token_bytes=735039144 weight_movement_bytes=110100480 total_nvlink_bytes=845139624 weight_to_token_ratio=0.149789 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.304571015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=380262400 dispatch_meta_bytes=232320 combine_payload_bytes=380262400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380494720 ingress_bytes=490362880 link_bytes=490362880 token_bytes=760757120 weight_movement_bytes=110100480 total_nvlink_bytes=870857600 weight_to_token_ratio=0.144725 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.528216992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56308 payload_rows_remote=49098 dup_rows=9228 dup_rows_remote=8220 dedup_ratio=0.140808 router_tokens_per_expert=[0, 0, 1196, 0, 0, 3894, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 203, 0, 3382, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 309, 3509, 3, 0, 19, 1, 0, 0, 125, 0, 0, 0, 1, 578, 0, 0, 0, 0, 3530, 0, 4071, 0, 0, 3815, 4095, 0] payload_rows_per_dst=[2392, 8190, 7582, 7742, 7416, 8146, 7210, 7630] dup_rows_per_dst=[0, 162, 0, 5026, 2740, 292, 1008, 0] combine_rows_per_src=[2392, 8190, 7582, 7742, 7416, 8146, 7210, 7630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369653760 dispatch_meta_bytes=232168 combine_payload_bytes=369653760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369885928 ingress_bytes=479754240 link_bytes=479754240 token_bytes=739539688 weight_movement_bytes=110100480 total_nvlink_bytes=849640168 weight_to_token_ratio=0.148877 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.343715294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57404 payload_rows_remote=49286 dup_rows=8132 dup_rows_remote=7868 dedup_ratio=0.124084 router_tokens_per_expert=[0, 0, 2296, 0, 0, 3638, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 0, 0, 3619, 0, 0, 0, 0, 0, 0, 146, 0, 2825, 0, 1, 199, 0, 0, 0, 0, 0, 0, 0, 198, 3320, 0, 0, 23, 0, 0, 0, 112, 0, 0, 0, 1, 697, 1, 0, 0, 0, 3686, 0, 4056, 0, 0, 3810, 4092, 0] payload_rows_per_dst=[4592, 8186, 7766, 6748, 6774, 8118, 7600, 7620] dup_rows_per_dst=[0, 94, 4004, 2338, 262, 264, 1170, 0] combine_rows_per_src=[4592, 8186, 7766, 6748, 6774, 8118, 7600, 7620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=351934464 dispatch_meta_bytes=229272 combine_payload_bytes=351934464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352163736 ingress_bytes=462034944 link_bytes=462034944 token_bytes=704098200 weight_movement_bytes=110100480 total_nvlink_bytes=814198680 weight_to_token_ratio=0.156371 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.035547593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=353282048 dispatch_meta_bytes=228616 combine_payload_bytes=353282048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353510664 ingress_bytes=463382528 link_bytes=463382528 token_bytes=706792712 weight_movement_bytes=110100480 total_nvlink_bytes=816893192 weight_to_token_ratio=0.155775 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.058984295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56330 payload_rows_remote=48650 dup_rows=9206 dup_rows_remote=9206 dedup_ratio=0.140472 router_tokens_per_expert=[0, 3489, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 1021, 321, 0, 0, 0, 0, 815, 4096, 0, 4070, 0, 0, 0, 2783, 1, 3838, 3203, 0, 0, 3904, 0, 0, 0, 4096, 0, 0, 0, 8, 34, 447, 0, 0, 460, 0, 0, 0, 3, 0] payload_rows_per_dst=[8022, 7680, 8192, 8016, 1630, 6406, 8192, 8192] dup_rows_per_dst=[4840, 0, 36, 2476, 0, 2, 978, 874] combine_rows_per_src=[8022, 7680, 8192, 8016, 1630, 6406, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59148 payload_rows_remote=51078 dup_rows=6388 dup_rows_remote=6326 dedup_ratio=0.097473 router_tokens_per_expert=[0, 0, 5, 240, 0, 217, 2156, 0, 0, 294, 0, 0, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 3995, 3743, 0, 0, 0, 3772, 3875, 0, 2521, 6, 0, 4, 232, 0, 1, 0, 315, 4088, 2, 4092, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 40, 0, 0, 27] payload_rows_per_dst=[4918, 6724, 7486, 7736, 8188, 8184, 7842, 8070] dup_rows_per_dst=[318, 108, 0, 2700, 1096, 34, 2070, 62] combine_rows_per_src=[4918, 6724, 7486, 7736, 8188, 8184, 7842, 8070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61096 payload_rows_remote=53068 dup_rows=4440 dup_rows_remote=2934 dedup_ratio=0.067749 router_tokens_per_expert=[0, 3280, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 708, 61, 0, 0, 0, 0, 1772, 4096, 0, 4096, 0, 0, 0, 2439, 0, 4010, 3333, 0, 0, 3998, 0, 0, 0, 4096, 0, 0, 0, 0, 20, 370, 0, 0, 184, 0, 0, 0, 55, 0] payload_rows_per_dst=[6734, 8040, 8192, 8028, 7052, 6666, 8192, 8192] dup_rows_per_dst=[274, 30, 2, 1506, 1370, 0, 780, 478] combine_rows_per_src=[6734, 8040, 8192, 8028, 7052, 6666, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59112 payload_rows_remote=50920 dup_rows=6424 dup_rows_remote=6420 dedup_ratio=0.098022 router_tokens_per_expert=[0, 2900, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 80, 0, 0, 0, 0, 1848, 4096, 0, 4092, 0, 0, 0, 3004, 0, 4010, 2807, 0, 0, 4026, 0, 1, 0, 4095, 0, 0, 0, 2, 20, 447, 0, 0, 162, 0, 0, 0, 60, 0] payload_rows_per_dst=[6870, 8020, 8192, 8064, 5970, 5614, 8190, 8192] dup_rows_per_dst=[2008, 0, 4, 2012, 1024, 2, 938, 436] combine_rows_per_src=[6870, 8020, 8192, 8064, 5970, 5614, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58360 payload_rows_remote=50806 dup_rows=7176 dup_rows_remote=4562 dedup_ratio=0.109497 router_tokens_per_expert=[0, 2734, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 186, 0, 0, 0, 0, 1792, 4096, 0, 4096, 0, 0, 0, 3292, 0, 4010, 2228, 0, 0, 3893, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 477, 0, 0, 108, 0, 0, 0, 26, 0] payload_rows_per_dst=[5768, 8020, 8192, 7986, 7554, 4456, 8192, 8192] dup_rows_per_dst=[170, 0, 0, 3128, 2614, 0, 996, 268] combine_rows_per_src=[5768, 8020, 8192, 7986, 7554, 4456, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348723200 dispatch_meta_bytes=231424 combine_payload_bytes=348723200 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=348954624 ingress_bytes=568924160 link_bytes=568924160 token_bytes=697677824 weight_movement_bytes=220200960 total_nvlink_bytes=917878784 weight_to_token_ratio=0.315620 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.894526862e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=366127104 dispatch_meta_bytes=229616 combine_payload_bytes=366127104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366356720 ingress_bytes=476227584 link_bytes=476227584 token_bytes=732483824 weight_movement_bytes=110100480 total_nvlink_bytes=842584304 weight_to_token_ratio=0.150311 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.282380946e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=380391424 dispatch_meta_bytes=224008 combine_payload_bytes=380391424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380615432 ingress_bytes=490491904 link_bytes=490491904 token_bytes=761006856 weight_movement_bytes=110100480 total_nvlink_bytes=871107336 weight_to_token_ratio=0.144677 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.530460931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364994560 dispatch_meta_bytes=229360 combine_payload_bytes=364994560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365223920 ingress_bytes=475095040 link_bytes=475095040 token_bytes=730218480 weight_movement_bytes=110100480 total_nvlink_bytes=840318960 weight_to_token_ratio=0.150777 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.262684143e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364177408 dispatch_meta_bytes=221472 combine_payload_bytes=364177408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=364398880 ingress_bytes=804579328 link_bytes=804579328 token_bytes=728576288 weight_movement_bytes=440401920 total_nvlink_bytes=1168978208 weight_to_token_ratio=0.604469 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.399295782e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60898 payload_rows_remote=54818 dup_rows=4638 dup_rows_remote=4632 dedup_ratio=0.070770 router_tokens_per_expert=[0, 3355, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 103, 0, 0, 0, 0, 1827, 4096, 0, 4091, 0, 0, 0, 2646, 3, 4026, 3040, 0, 0, 4023, 0, 0, 0, 4095, 0, 0, 0, 3, 24, 376, 0, 0, 93, 0, 0, 0, 57, 0] payload_rows_per_dst=[6812, 8052, 8192, 8068, 7310, 6080, 8192, 8192] dup_rows_per_dst=[260, 0, 0, 1642, 1636, 6, 804, 290] combine_rows_per_src=[6812, 8052, 8192, 8068, 7310, 6080, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56702 payload_rows_remote=48510 dup_rows=8834 dup_rows_remote=7910 dedup_ratio=0.134796 router_tokens_per_expert=[0, 2726, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1023, 31, 0, 0, 0, 0, 1954, 4096, 0, 4095, 0, 0, 0, 3061, 1, 4017, 2841, 0, 0, 4045, 0, 0, 0, 4096, 0, 0, 0, 0, 25, 437, 0, 0, 103, 0, 0, 0, 72, 0] payload_rows_per_dst=[5570, 8078, 8192, 2728, 7580, 8170, 8192, 8192] dup_rows_per_dst=[126, 0, 2, 64, 2450, 4920, 924, 348] combine_rows_per_src=[5570, 8078, 8192, 2728, 7580, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=392935424 dispatch_meta_bytes=237800 combine_payload_bytes=392935424 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=393173224 ingress_bytes=833337344 link_bytes=833337344 token_bytes=786108648 weight_movement_bytes=440401920 total_nvlink_bytes=1226510568 weight_to_token_ratio=0.560230 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.449310702e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=347719680 dispatch_meta_bytes=225680 combine_payload_bytes=347719680 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=347945360 ingress_bytes=567920640 link_bytes=567920640 token_bytes=695665040 weight_movement_bytes=220200960 total_nvlink_bytes=915866000 weight_to_token_ratio=0.316533 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.877073999e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54760 payload_rows_remote=46626 dup_rows=10776 dup_rows_remote=10776 dedup_ratio=0.164429 router_tokens_per_expert=[0, 20, 0, 182, 0, 6, 0, 1598, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 1, 0, 0, 0, 0, 3337, 0, 45, 33, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 58, 3079, 0, 884, 0] payload_rows_per_dst=[8192, 8134, 8192, 8192, 7276, 7196, 616, 6962] dup_rows_per_dst=[3612, 0, 0, 0, 2444, 3640, 0, 1080] combine_rows_per_src=[8192, 8134, 8192, 8192, 7276, 7196, 616, 6962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61400 payload_rows_remote=53208 dup_rows=4136 dup_rows_remote=4136 dedup_ratio=0.063110 router_tokens_per_expert=[0, 16, 0, 119, 0, 1, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 0, 0, 0, 0, 3948, 0, 0, 58, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 0, 14, 3078, 0, 395, 0] payload_rows_per_dst=[8192, 8132, 8190, 8192, 6774, 7942, 7398, 6580] dup_rows_per_dst=[3672, 0, 0, 0, 0, 70, 0, 394] combine_rows_per_src=[8192, 8132, 8190, 8192, 6774, 7942, 7398, 6580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58288 payload_rows_remote=51184 dup_rows=7248 dup_rows_remote=7248 dedup_ratio=0.110596 router_tokens_per_expert=[0, 16, 0, 170, 0, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 3854, 0, 0, 27, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 0, 2, 2974, 0, 420, 0] payload_rows_per_dst=[4496, 8190, 8188, 8192, 7104, 7724, 7992, 6402] dup_rows_per_dst=[614, 0, 0, 0, 0, 38, 6206, 390] combine_rows_per_src=[4496, 8190, 8188, 8192, 7104, 7724, 7992, 6402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60790 payload_rows_remote=52598 dup_rows=4746 dup_rows_remote=4744 dedup_ratio=0.072418 router_tokens_per_expert=[0, 5, 0, 89, 0, 2, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 0, 0, 3820, 0, 9, 36, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3556, 0, 0, 0, 31, 3113, 0, 167, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 6814, 7702, 7112, 6398] dup_rows_per_dst=[4492, 0, 2, 0, 0, 28, 0, 224] combine_rows_per_src=[8192, 8188, 8192, 8192, 6814, 7702, 7112, 6398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49090 payload_rows_remote=41292 dup_rows=16446 dup_rows_remote=7532 dedup_ratio=0.250946 router_tokens_per_expert=[0, 0, 1819, 0, 0, 3619, 0, 0, 0, 0, 0, 180, 1, 0, 0, 0, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 201, 0, 3487, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 168, 3142, 2, 0, 155, 0, 0, 0, 17, 0, 0, 0, 0, 542, 0, 0, 0, 0, 3869, 0, 4032, 0, 0, 3411, 4094, 0] payload_rows_per_dst=[3638, 6834, 7526, 7116, 8048, 6954, 1176, 7798] dup_rows_per_dst=[0, 352, 0, 792, 5810, 330, 248, 8914] combine_rows_per_src=[3638, 6834, 7526, 7116, 8048, 6954, 1176, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=334215168 dispatch_meta_bytes=229608 combine_payload_bytes=334215168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334444776 ingress_bytes=444315648 link_bytes=444315648 token_bytes=668659944 weight_movement_bytes=110100480 total_nvlink_bytes=778760424 weight_to_token_ratio=0.164658 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.727379892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57510 payload_rows_remote=49954 dup_rows=8026 dup_rows_remote=7962 dedup_ratio=0.122467 router_tokens_per_expert=[0, 7, 0, 155, 0, 2, 0, 1510, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 3724, 0, 8, 78, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 2, 7, 3305, 0, 269, 0] payload_rows_per_dst=[3320, 8174, 8192, 8192, 7048, 7556, 8190, 6838] dup_rows_per_dst=[28, 0, 4, 4, 0, 64, 7598, 328] combine_rows_per_src=[3320, 8174, 8192, 8192, 7048, 7556, 8190, 6838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381394944 dispatch_meta_bytes=229376 combine_payload_bytes=381394944 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=381624320 ingress_bytes=491495424 link_bytes=491495424 token_bytes=763019264 weight_movement_bytes=110100480 total_nvlink_bytes=873119744 weight_to_token_ratio=0.144296 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.547913794e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366886912 dispatch_meta_bytes=233728 combine_payload_bytes=366886912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367120640 ingress_bytes=476987392 link_bytes=476987392 token_bytes=734007552 weight_movement_bytes=110100480 total_nvlink_bytes=844108032 weight_to_token_ratio=0.149999 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.295595256e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=377022464 dispatch_meta_bytes=229368 combine_payload_bytes=377022464 prefetch_bytes=0 reduce_bytes=0 egress_bytes=377251832 ingress_bytes=377022464 link_bytes=377251832 token_bytes=754274296 weight_movement_bytes=0 total_nvlink_bytes=754274296 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.561029831e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=295981056 dispatch_meta_bytes=195296 combine_payload_bytes=295981056 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=296176352 ingress_bytes=736382976 link_bytes=736382976 token_bytes=592157408 weight_movement_bytes=440401920 total_nvlink_bytes=1032559328 weight_to_token_ratio=0.743724 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280691109e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58290 payload_rows_remote=50098 dup_rows=7246 dup_rows_remote=3208 dedup_ratio=0.110565 router_tokens_per_expert=[0, 9, 0, 111, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 3926, 0, 2, 37, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 3, 3212, 0, 282, 0] payload_rows_per_dst=[3026, 8188, 8192, 8192, 7416, 7896, 8192, 7188] dup_rows_per_dst=[30, 0, 2, 0, 0, 34, 4038, 3142] combine_rows_per_src=[3026, 8188, 8192, 8192, 7416, 7896, 8192, 7188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=358070272 dispatch_meta_bytes=231664 combine_payload_bytes=358070272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358301936 ingress_bytes=468170752 link_bytes=468170752 token_bytes=716372208 weight_movement_bytes=110100480 total_nvlink_bytes=826472688 weight_to_token_ratio=0.153692 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.142259386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61440 payload_rows_remote=53248 dup_rows=4096 dup_rows_remote=4094 dedup_ratio=0.062500 router_tokens_per_expert=[0, 2434, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 4096, 0, 4081, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 4096, 0, 421, 204, 2768, 0, 0, 0, 4096, 60, 0, 2962, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7828, 8192, 7442, 8162, 8192, 7508, 8192, 5924] dup_rows_per_dst=[2608, 2, 0, 116, 0, 0, 1370, 0] combine_rows_per_src=[7828, 8192, 7442, 8162, 8192, 7508, 8192, 5924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359102464 dispatch_meta_bytes=213224 combine_payload_bytes=359102464 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=359315688 ingress_bytes=1019705344 link_bytes=1019705344 token_bytes=718418152 weight_movement_bytes=660602880 total_nvlink_bytes=1379021032 weight_to_token_ratio=0.919524 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.773435306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60714 payload_rows_remote=52526 dup_rows=4822 dup_rows_remote=4822 dedup_ratio=0.073578 router_tokens_per_expert=[0, 3563, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 4094, 0, 4075, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 113, 195, 2243, 0, 0, 0, 4096, 6, 0, 2422, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7126, 8192, 7608, 8150, 8188, 8110, 6118, 7222] dup_rows_per_dst=[0, 4, 0, 8, 0, 0, 1814, 2996] combine_rows_per_src=[7126, 8192, 7608, 8150, 8188, 8110, 6118, 7222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59092 payload_rows_remote=50908 dup_rows=6444 dup_rows_remote=6412 dedup_ratio=0.098328 router_tokens_per_expert=[0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 4095, 0, 4092, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 4096, 0, 255, 199, 2748, 0, 0, 0, 4096, 4, 0, 2170, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6464, 8192, 7522, 8184, 8190, 8008, 8192, 4340] dup_rows_per_dst=[0, 0, 0, 32, 0, 0, 6412, 0] combine_rows_per_src=[6464, 8192, 7522, 8184, 8190, 8008, 8192, 4340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381681664 dispatch_meta_bytes=229368 combine_payload_bytes=381681664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381911032 ingress_bytes=491782144 link_bytes=491782144 token_bytes=763592696 weight_movement_bytes=110100480 total_nvlink_bytes=873693176 weight_to_token_ratio=0.144187 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.552900327e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60308 payload_rows_remote=52784 dup_rows=5228 dup_rows_remote=5228 dedup_ratio=0.079773 router_tokens_per_expert=[0, 3145, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 4094, 0, 4066, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 121, 248, 2740, 0, 0, 0, 4096, 8, 0, 2349, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7326, 8192, 7524, 8134, 8188, 8054, 8192, 4698] dup_rows_per_dst=[1638, 6, 0, 240, 0, 0, 3344, 0] combine_rows_per_src=[7326, 8192, 7524, 8134, 8188, 8054, 8192, 4698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55752 payload_rows_remote=47560 dup_rows=9784 dup_rows_remote=9214 dedup_ratio=0.149292 router_tokens_per_expert=[0, 2982, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 668, 71, 0, 0, 0, 0, 1629, 4096, 0, 4094, 0, 0, 0, 3072, 0, 3920, 3019, 0, 0, 3987, 0, 0, 0, 4096, 0, 0, 0, 0, 23, 570, 0, 0, 179, 0, 0, 0, 108, 0] payload_rows_per_dst=[6216, 7870, 8144, 1464, 7582, 8092, 8192, 8192] dup_rows_per_dst=[202, 20, 4, 14, 1868, 5920, 1186, 570] combine_rows_per_src=[6216, 7870, 8144, 1464, 7582, 8092, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61338 payload_rows_remote=53216 dup_rows=4198 dup_rows_remote=4198 dedup_ratio=0.064056 router_tokens_per_expert=[0, 3350, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 4096, 0, 4043, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 4096, 0, 133, 256, 2877, 0, 0, 0, 4096, 7, 0, 1964, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6708, 8192, 7560, 8086, 8192, 8122, 6286, 8192] dup_rows_per_dst=[0, 4, 0, 6, 0, 0, 260, 3928] combine_rows_per_src=[6708, 8192, 7560, 8086, 8192, 8122, 6286, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376506368 dispatch_meta_bytes=229392 combine_payload_bytes=376506368 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=376735760 ingress_bytes=596707328 link_bytes=596707328 token_bytes=753242128 weight_movement_bytes=220200960 total_nvlink_bytes=973443088 weight_to_token_ratio=0.292338 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037772185e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=364908544 dispatch_meta_bytes=229280 combine_payload_bytes=364908544 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=365137824 ingress_bytes=695209984 link_bytes=695209984 token_bytes=730046368 weight_movement_bytes=330301440 total_nvlink_bytes=1060347808 weight_to_token_ratio=0.452439 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.209084505e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378355712 dispatch_meta_bytes=232048 combine_payload_bytes=378355712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378587760 ingress_bytes=488456192 link_bytes=488456192 token_bytes=756943472 weight_movement_bytes=110100480 total_nvlink_bytes=867043952 weight_to_token_ratio=0.145454 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.495056551e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340910080 dispatch_meta_bytes=227096 combine_payload_bytes=340910080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341137176 ingress_bytes=451010560 link_bytes=451010560 token_bytes=682047256 weight_movement_bytes=110100480 total_nvlink_bytes=792147736 weight_to_token_ratio=0.161426 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.843815423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62546 payload_rows_remote=56050 dup_rows=2990 dup_rows_remote=2860 dedup_ratio=0.045624 router_tokens_per_expert=[0, 3666, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 4096, 0, 4091, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 4096, 0, 110, 270, 2930, 0, 0, 0, 4096, 3, 0, 1410, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7332, 8192, 7860, 8182, 8192, 8100, 6496, 8192] dup_rows_per_dst=[0, 26, 0, 14, 0, 0, 130, 2820] combine_rows_per_src=[7332, 8192, 7860, 8182, 8192, 8100, 6496, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62740 payload_rows_remote=56470 dup_rows=2796 dup_rows_remote=2468 dedup_ratio=0.042664 router_tokens_per_expert=[0, 0, 0, 0, 0, 15, 0, 4034, 0, 0, 102, 0, 2925, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 20, 0, 4096, 0, 0, 0, 6, 4096, 41, 0, 13, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 4081] payload_rows_per_dst=[8098, 6270, 8036, 8192, 7584, 8192, 8192, 8176] dup_rows_per_dst=[0, 328, 40, 94, 0, 682, 0, 1652] combine_rows_per_src=[8098, 6270, 8036, 8192, 7584, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381452288 dispatch_meta_bytes=229656 combine_payload_bytes=381452288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381681944 ingress_bytes=601653248 link_bytes=601653248 token_bytes=763134232 weight_movement_bytes=220200960 total_nvlink_bytes=983335192 weight_to_token_ratio=0.288548 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046373953e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=401766400 dispatch_meta_bytes=235640 combine_payload_bytes=401766400 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=402002040 ingress_bytes=952268800 link_bytes=952268800 token_bytes=803768440 weight_movement_bytes=550502400 total_nvlink_bytes=1354270840 weight_to_token_ratio=0.684902 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.656152065e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404776960 dispatch_meta_bytes=235752 combine_payload_bytes=404776960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405012712 ingress_bytes=514877440 link_bytes=514877440 token_bytes=809789672 weight_movement_bytes=110100480 total_nvlink_bytes=919890152 weight_to_token_ratio=0.135962 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.954565509e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63874 payload_rows_remote=55932 dup_rows=1662 dup_rows_remote=1662 dedup_ratio=0.025360 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 11, 0, 3677, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4096, 111, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 252, 0, 0, 0, 4096] payload_rows_per_dst=[8184, 7372, 7608, 8192, 7942, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 232, 0, 916, 0, 510] combine_rows_per_src=[8184, 7372, 7608, 8192, 7942, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63004 payload_rows_remote=54812 dup_rows=2532 dup_rows_remote=2406 dedup_ratio=0.038635 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 33, 0, 3398, 0, 0, 0, 0, 0, 0, 3792, 0, 0, 2, 0, 4096, 0, 0, 0, 11, 4096, 52, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 535, 0, 0, 0, 4096] payload_rows_per_dst=[8184, 6856, 7584, 8192, 7612, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 4, 126, 0, 1324, 0, 1072] combine_rows_per_src=[8184, 6856, 7584, 8192, 7612, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63844 payload_rows_remote=55922 dup_rows=1692 dup_rows_remote=1692 dedup_ratio=0.025818 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 32, 0, 3513, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 4096, 66, 0, 1, 2, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 499, 0, 0, 0, 4092] payload_rows_per_dst=[8178, 7078, 7922, 8192, 7898, 8192, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 148, 2, 538, 0, 992] combine_rows_per_src=[8178, 7078, 7922, 8192, 7898, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60406 payload_rows_remote=52242 dup_rows=5130 dup_rows_remote=168 dedup_ratio=0.078278 router_tokens_per_expert=[0, 61, 0, 144, 0, 1, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 0, 0, 0, 0, 0, 3910, 0, 2, 51, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 3, 2526, 0, 340, 0] payload_rows_per_dst=[4830, 8180, 8192, 8190, 6800, 7858, 8192, 8164] dup_rows_per_dst=[98, 0, 0, 0, 0, 68, 2, 4962] combine_rows_per_src=[4830, 8180, 8192, 8190, 6800, 7858, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400920576 dispatch_meta_bytes=230376 combine_payload_bytes=400920576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401150952 ingress_bytes=511021056 link_bytes=511021056 token_bytes=802071528 weight_movement_bytes=110100480 total_nvlink_bytes=912172008 weight_to_token_ratio=0.137270 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.887496649e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63592 payload_rows_remote=55400 dup_rows=1944 dup_rows_remote=896 dedup_ratio=0.029663 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 15, 0, 3614, 0, 0, 0, 0, 0, 0, 3790, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 4096, 42, 0, 5, 1, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 4091] payload_rows_per_dst=[8164, 7232, 7580, 8192, 7848, 8192, 8192, 8192] dup_rows_per_dst=[0, 26, 0, 98, 2, 1048, 0, 770] combine_rows_per_src=[8164, 7232, 7580, 8192, 7848, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392892416 dispatch_meta_bytes=228872 combine_payload_bytes=392892416 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=393121288 ingress_bytes=723193856 link_bytes=723193856 token_bytes=786013704 weight_movement_bytes=330301440 total_nvlink_bytes=1116315144 weight_to_token_ratio=0.420224 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.257753061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400848896 dispatch_meta_bytes=230456 combine_payload_bytes=400848896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401079352 ingress_bytes=510949376 link_bytes=510949376 token_bytes=801928248 weight_movement_bytes=110100480 total_nvlink_bytes=912028728 weight_to_token_ratio=0.137295 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.886250016e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374470656 dispatch_meta_bytes=209640 combine_payload_bytes=374470656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374680296 ingress_bytes=484571136 link_bytes=484571136 token_bytes=749150952 weight_movement_bytes=110100480 total_nvlink_bytes=859251432 weight_to_token_ratio=0.146967 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.427489038e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64242 payload_rows_remote=56050 dup_rows=1294 dup_rows_remote=1294 dedup_ratio=0.019745 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 4, 0, 3853, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4096, 17, 0, 2, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 7710, 7634, 8192, 7942, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 46, 0, 770, 0, 474] combine_rows_per_src=[8188, 7710, 7634, 8192, 7942, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60788 payload_rows_remote=52598 dup_rows=4748 dup_rows_remote=4746 dedup_ratio=0.072449 router_tokens_per_expert=[0, 0, 41, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 1470, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 42, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2631, 0, 0, 0, 0, 1858, 3507, 0] payload_rows_per_dst=[7360, 8190, 8192, 5040, 8192, 8192, 8192, 7430] dup_rows_per_dst=[170, 2, 2942, 0, 2, 84, 0, 1548] combine_rows_per_src=[7360, 8190, 8192, 5040, 8192, 8192, 8192, 7430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397107200 dispatch_meta_bytes=225184 combine_payload_bytes=397107200 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=397332384 ingress_bytes=837509120 link_bytes=837509120 token_bytes=794439584 weight_movement_bytes=440401920 total_nvlink_bytes=1234841504 weight_to_token_ratio=0.554355 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.456566106e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401766400 dispatch_meta_bytes=229376 combine_payload_bytes=401766400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401995776 ingress_bytes=511866880 link_bytes=511866880 token_bytes=803762176 weight_movement_bytes=110100480 total_nvlink_bytes=913862656 weight_to_token_ratio=0.136981 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.902206920e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63098 payload_rows_remote=54906 dup_rows=2438 dup_rows_remote=2438 dedup_ratio=0.037201 router_tokens_per_expert=[0, 0, 40, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 406, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 143, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 3156, 3920, 0] payload_rows_per_dst=[7880, 8186, 8192, 6584, 8192, 8192, 8192, 7680] dup_rows_per_dst=[134, 0, 812, 0, 0, 286, 0, 1206] combine_rows_per_src=[7880, 8186, 8192, 6584, 8192, 8192, 8192, 7680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=377022464 dispatch_meta_bytes=229376 combine_payload_bytes=377022464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377251840 ingress_bytes=487122944 link_bytes=487122944 token_bytes=754274304 weight_movement_bytes=110100480 total_nvlink_bytes=864374784 weight_to_token_ratio=0.145969 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.471869176e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61986 payload_rows_remote=56176 dup_rows=3550 dup_rows_remote=3550 dedup_ratio=0.054169 router_tokens_per_expert=[0, 0, 62, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 682, 0, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 115, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1708, 0, 0, 0, 0, 3011, 3725, 0] payload_rows_per_dst=[7560, 8190, 8192, 5810, 8192, 8192, 8192, 7658] dup_rows_per_dst=[176, 0, 1364, 0, 0, 230, 0, 1780] combine_rows_per_src=[7560, 8190, 8192, 5810, 8192, 8192, 8192, 7658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63292 payload_rows_remote=55100 dup_rows=2244 dup_rows_remote=1576 dedup_ratio=0.034241 router_tokens_per_expert=[0, 0, 21, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 331, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 0, 3400, 3792, 0] payload_rows_per_dst=[7610, 8188, 8192, 6920, 8192, 8192, 8192, 7806] dup_rows_per_dst=[108, 0, 668, 0, 2, 156, 0, 1310] combine_rows_per_src=[7610, 8188, 8192, 6920, 8192, 8192, 8192, 7806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59864 payload_rows_remote=51672 dup_rows=5672 dup_rows_remote=376 dedup_ratio=0.086548 router_tokens_per_expert=[0, 2916, 0, 0, 0, 0, 0, 0, 18, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 4096, 0, 4052, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 4096, 0, 186, 172, 2687, 0, 0, 0, 4096, 1, 0, 2648, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5832, 8110, 7622, 8094, 8192, 7942, 5880, 8192] dup_rows_per_dst=[0, 40, 0, 32, 10, 82, 212, 5296] combine_rows_per_src=[5832, 8110, 7622, 8094, 8192, 7942, 5880, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=393566208 dispatch_meta_bytes=229376 combine_payload_bytes=393566208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393795584 ingress_bytes=503666688 link_bytes=503666688 token_bytes=787361792 weight_movement_bytes=110100480 total_nvlink_bytes=897462272 weight_to_token_ratio=0.139835 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.759592094e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62906 payload_rows_remote=54714 dup_rows=2630 dup_rows_remote=2140 dedup_ratio=0.040131 router_tokens_per_expert=[0, 0, 27, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 557, 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 3088, 3780, 0] payload_rows_per_dst=[7602, 8180, 8192, 7018, 8192, 8192, 8192, 7338] dup_rows_per_dst=[128, 0, 1118, 0, 2, 490, 0, 892] combine_rows_per_src=[7602, 8180, 8192, 7018, 8192, 8192, 8192, 7338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402669568 dispatch_meta_bytes=238904 combine_payload_bytes=402669568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402908472 ingress_bytes=512770048 link_bytes=512770048 token_bytes=805578040 weight_movement_bytes=110100480 total_nvlink_bytes=915678520 weight_to_token_ratio=0.136673 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.917914497e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394956800 dispatch_meta_bytes=226704 combine_payload_bytes=394956800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395183504 ingress_bytes=615157760 link_bytes=615157760 token_bytes=790140304 weight_movement_bytes=220200960 total_nvlink_bytes=1010341264 weight_to_token_ratio=0.278686 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069860521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370384896 dispatch_meta_bytes=208192 combine_payload_bytes=370384896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370593088 ingress_bytes=480485376 link_bytes=480485376 token_bytes=740977984 weight_movement_bytes=110100480 total_nvlink_bytes=851078464 weight_to_token_ratio=0.148588 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.356430951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58266 payload_rows_remote=50074 dup_rows=7270 dup_rows_remote=7270 dedup_ratio=0.110931 router_tokens_per_expert=[0, 0, 27, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 657, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 125, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 876, 0, 0, 0, 0, 3350, 3806, 0] payload_rows_per_dst=[7632, 8192, 2834, 7454, 8192, 8192, 8192, 7578] dup_rows_per_dst=[88, 0, 860, 5198, 0, 250, 0, 874] combine_rows_per_src=[7632, 8192, 2834, 7454, 8192, 8192, 8192, 7578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59592 payload_rows_remote=51400 dup_rows=5944 dup_rows_remote=314 dedup_ratio=0.090698 router_tokens_per_expert=[3892, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 1, 0, 2814, 0, 0, 0, 0, 0, 0, 12, 4080, 0, 0, 134, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 4065, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7784, 8192, 8182, 8192, 8132, 8192, 2766, 8152] dup_rows_per_dst=[0, 5630, 2, 278, 0, 34, 0, 0] combine_rows_per_src=[7784, 8192, 8182, 8192, 8132, 8192, 2766, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392189952 dispatch_meta_bytes=227416 combine_payload_bytes=392189952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392417368 ingress_bytes=722491392 link_bytes=722491392 token_bytes=784607320 weight_movement_bytes=330301440 total_nvlink_bytes=1114908760 weight_to_token_ratio=0.420977 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.256531360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=358930432 dispatch_meta_bytes=229376 combine_payload_bytes=358930432 prefetch_bytes=0 reduce_bytes=0 egress_bytes=359159808 ingress_bytes=358930432 link_bytes=359159808 token_bytes=718090240 weight_movement_bytes=0 total_nvlink_bytes=718090240 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.246379777e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=368435200 dispatch_meta_bytes=206856 combine_payload_bytes=368435200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368642056 ingress_bytes=478535680 link_bytes=478535680 token_bytes=737077256 weight_movement_bytes=110100480 total_nvlink_bytes=847177736 weight_to_token_ratio=0.149374 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.322522531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61952 payload_rows_remote=53760 dup_rows=3584 dup_rows_remote=3584 dedup_ratio=0.054688 router_tokens_per_expert=[3826, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7652, 5150, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 3558, 0] combine_rows_per_src=[7652, 5150, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60992 payload_rows_remote=52800 dup_rows=4544 dup_rows_remote=4504 dedup_ratio=0.069336 router_tokens_per_expert=[3634, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7268, 6764, 8192, 8192, 8192, 8192, 6000, 8192] dup_rows_per_dst=[0, 2766, 0, 40, 0, 0, 1738, 0] combine_rows_per_src=[7268, 6764, 8192, 8192, 8192, 8192, 6000, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60460 payload_rows_remote=52270 dup_rows=5076 dup_rows_remote=5074 dedup_ratio=0.077454 router_tokens_per_expert=[3723, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 3, 0, 2506, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 21, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 1942, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7446, 8192, 8190, 8192, 8180, 8192, 3890, 8178] dup_rows_per_dst=[0, 5018, 2, 56, 0, 0, 0, 0] combine_rows_per_src=[7446, 8192, 8190, 8192, 8180, 8192, 3890, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62676 payload_rows_remote=54484 dup_rows=2860 dup_rows_remote=1046 dedup_ratio=0.043640 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 19, 0, 3585, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 35, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 457, 0, 0, 0, 4094] payload_rows_per_dst=[8184, 7202, 7652, 7268, 7794, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 62, 0, 978, 0, 1814] combine_rows_per_src=[8184, 7202, 7652, 7268, 7794, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7032185552 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3517227728 ingress_bytes=5386665984 link_bytes=5386665984 total_nvlink_bytes=8903893712 weight_to_token_ratio=0.266163 prefetch_share_of_ingress=0.204394 dedup_ratio=0.138019 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.368298100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385351680 dispatch_meta_bytes=229376 combine_payload_bytes=385351680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385581056 ingress_bytes=495452160 link_bytes=495452160 token_bytes=770932736 weight_movement_bytes=110100480 total_nvlink_bytes=881033216 weight_to_token_ratio=0.142815 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.616727941e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62032 payload_rows_remote=53842 dup_rows=3504 dup_rows_remote=3504 dedup_ratio=0.053467 router_tokens_per_expert=[3745, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 2701, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 17, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7490, 5406, 8192, 8192, 8182, 8190, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 36, 0, 0, 3468, 0] combine_rows_per_src=[7490, 5406, 8192, 8192, 8182, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=378470400 dispatch_meta_bytes=229216 combine_payload_bytes=378470400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=378699616 ingress_bytes=708771840 link_bytes=708771840 token_bytes=757170016 weight_movement_bytes=330301440 total_nvlink_bytes=1087471456 weight_to_token_ratio=0.436232 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.232670803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374671360 dispatch_meta_bytes=229376 combine_payload_bytes=374671360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374900736 ingress_bytes=484771840 link_bytes=484771840 token_bytes=749572096 weight_movement_bytes=110100480 total_nvlink_bytes=859672576 weight_to_token_ratio=0.146884 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.430979610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390541312 dispatch_meta_bytes=222120 combine_payload_bytes=390541312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=390763432 ingress_bytes=610742272 link_bytes=610742272 token_bytes=781304744 weight_movement_bytes=220200960 total_nvlink_bytes=1001505704 weight_to_token_ratio=0.281837 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062181261e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61512 payload_rows_remote=53320 dup_rows=4024 dup_rows_remote=12 dedup_ratio=0.061401 router_tokens_per_expert=[3038, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 3145, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 5, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6076, 6292, 8192, 8192, 8190, 8190, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 12, 0, 0, 4012, 0] combine_rows_per_src=[6076, 6292, 8192, 8192, 8190, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7237456992 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3619867744 ingress_bytes=5489297408 link_bytes=5489297408 total_nvlink_bytes=9109165152 weight_to_token_ratio=0.258614 prefetch_share_of_ingress=0.200573 dedup_ratio=0.114890 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.546791027e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7323367560 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3662841992 ingress_bytes=5972635648 link_bytes=5972635648 total_nvlink_bytes=9635477640 weight_to_token_ratio=0.315717 prefetch_share_of_ingress=0.147473 dedup_ratio=0.114792 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.038739572e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385939456 dispatch_meta_bytes=229384 combine_payload_bytes=385939456 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=386168840 ingress_bytes=606140416 link_bytes=606140416 token_bytes=772108296 weight_movement_bytes=220200960 total_nvlink_bytes=992309256 weight_to_token_ratio=0.285194 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054177876e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7289478448 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=3645883696 ingress_bytes=4964800512 link_bytes=4965029880 total_nvlink_bytes=8610684208 weight_to_token_ratio=0.181248 prefetch_share_of_ingress=0.199586 dedup_ratio=0.107028 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.635003568e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382197760 dispatch_meta_bytes=213328 combine_payload_bytes=382197760 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=382411088 ingress_bytes=712499200 link_bytes=712499200 token_bytes=764608848 weight_movement_bytes=330301440 total_nvlink_bytes=1094910288 weight_to_token_ratio=0.431987 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.239153295e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7246384360 prefetch_bytes=1101004800 reduce_bytes=1321205760 weight_movement_bytes=2422210560 egress_bytes=3624336616 ingress_bytes=6044258304 link_bytes=6044258304 total_nvlink_bytes=9668594920 weight_to_token_ratio=0.334265 prefetch_share_of_ingress=0.182157 dedup_ratio=0.113992 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=12 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=7620 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.051195930e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57798 payload_rows_remote=50162 dup_rows=7738 dup_rows_remote=6440 dedup_ratio=0.118073 router_tokens_per_expert=[0, 0, 14, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 754, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 0, 2619, 3844, 0] payload_rows_per_dst=[7700, 8192, 1508, 8192, 8186, 8192, 8192, 7636] dup_rows_per_dst=[88, 0, 4, 6270, 0, 78, 0, 1298] combine_rows_per_src=[7700, 8192, 1508, 8192, 8186, 8192, 8192, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7124521120 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3563387040 ingress_bytes=5983344640 link_bytes=5983574016 total_nvlink_bytes=9546731680 weight_to_token_ratio=0.339982 prefetch_share_of_ingress=0.147209 dedup_ratio=0.125723 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.040641934e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359561216 dispatch_meta_bytes=226408 combine_payload_bytes=359561216 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=359787624 ingress_bytes=799963136 link_bytes=799963136 token_bytes=719348840 weight_movement_bytes=440401920 total_nvlink_bytes=1159750760 weight_to_token_ratio=0.612223 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.391267465e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62134 payload_rows_remote=53946 dup_rows=3402 dup_rows_remote=3334 dedup_ratio=0.051910 router_tokens_per_expert=[3936, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 2844, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7804, 5692, 8192, 7686, 8188, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 36, 88, 0, 3210, 68] combine_rows_per_src=[7804, 5692, 8192, 7686, 8188, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386684928 dispatch_meta_bytes=229120 combine_payload_bytes=386684928 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=386914048 ingress_bytes=606885888 link_bytes=606885888 token_bytes=773598976 weight_movement_bytes=220200960 total_nvlink_bytes=993799936 weight_to_token_ratio=0.284645 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055474375e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6935944552 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3469069672 ingress_bytes=5558784000 link_bytes=5558784000 total_nvlink_bytes=9027853672 weight_to_token_ratio=0.301604 prefetch_share_of_ingress=0.178259 dedup_ratio=0.143521 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.667639639e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45134 payload_rows_remote=38556 dup_rows=20402 dup_rows_remote=16720 dedup_ratio=0.311310 router_tokens_per_expert=[1326, 876, 634, 1236, 6, 909, 305, 27, 248, 617, 1520, 261, 0, 1137, 315, 640, 1245, 221, 6, 0, 930, 353, 1007, 654, 197, 1534, 324, 400, 941, 113, 396, 0, 630, 471, 1174, 6, 1213, 0, 1183, 116, 284, 927, 717, 382, 0, 479, 22, 288, 188, 448, 438, 250, 149, 984, 241, 557, 441, 616, 374, 346, 1047, 287, 131, 1] payload_rows_per_dst=[6578, 5590, 6040, 5708, 5878, 5894, 4672, 4774] dup_rows_per_dst=[3682, 1784, 2792, 2676, 3536, 2956, 1838, 1138] combine_rows_per_src=[6578, 5590, 6040, 5708, 5878, 5894, 4672, 4774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276369408 dispatch_meta_bytes=221104 combine_payload_bytes=276369408 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276590512 ingress_bytes=606670848 link_bytes=606670848 token_bytes=552959920 weight_movement_bytes=330301440 total_nvlink_bytes=883261360 weight_to_token_ratio=0.597333 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055100385e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47996 payload_rows_remote=40394 dup_rows=17540 dup_rows_remote=12294 dedup_ratio=0.267639 router_tokens_per_expert=[0, 0, 0, 31, 3012, 0, 1604, 2600, 2760, 5, 2601, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 2906, 3641, 0, 0, 0, 0, 3014, 0, 2471, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 90, 0, 197, 0, 0, 0, 4, 0, 791, 78, 604, 0, 195, 0] payload_rows_per_dst=[7602, 5916, 7980, 7404, 4628, 5520, 6100, 2846] dup_rows_per_dst=[5246, 844, 5114, 4426, 0, 0, 1412, 498] combine_rows_per_src=[7602, 5916, 7980, 7404, 4628, 5520, 6100, 2846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=289544192 dispatch_meta_bytes=210752 combine_payload_bytes=289544192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=289754944 ingress_bytes=619845632 link_bytes=619845632 token_bytes=579299136 weight_movement_bytes=330301440 total_nvlink_bytes=909600576 weight_to_token_ratio=0.570174 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078013501e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50690 payload_rows_remote=42566 dup_rows=14846 dup_rows_remote=11054 dedup_ratio=0.226532 router_tokens_per_expert=[0, 0, 6, 474, 0, 149, 1492, 0, 0, 468, 0, 0, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 4060, 3689, 0, 0, 0, 3837, 3916, 0, 3014, 0, 0, 0, 146, 0, 0, 0, 181, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 44] payload_rows_per_dst=[8124, 8130, 7954, 1822, 1114, 7704, 7832, 8010] dup_rows_per_dst=[3792, 6832, 3862, 226, 30, 0, 0, 104] combine_rows_per_src=[8124, 8130, 7954, 1822, 1114, 7704, 7832, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=305113088 dispatch_meta_bytes=214480 combine_payload_bytes=305113088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305327568 ingress_bytes=415213568 link_bytes=415213568 token_bytes=610440656 weight_movement_bytes=110100480 total_nvlink_bytes=720541136 weight_to_token_ratio=0.180362 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.221246856e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59318 payload_rows_remote=51244 dup_rows=6218 dup_rows_remote=1828 dedup_ratio=0.094879 router_tokens_per_expert=[0, 0, 2499, 0, 0, 3319, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 146, 0, 3465, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 110, 3387, 0, 0, 20, 0, 0, 0, 79, 0, 0, 0, 1, 693, 0, 0, 0, 0, 3581, 0, 3953, 0, 0, 3479, 4096, 0] payload_rows_per_dst=[8074, 8192, 7654, 7010, 6852, 7914, 7492, 6130] dup_rows_per_dst=[4390, 112, 0, 326, 142, 190, 1058, 0] combine_rows_per_src=[8074, 8192, 7654, 7010, 6852, 7914, 7492, 6130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=367316992 dispatch_meta_bytes=212288 combine_payload_bytes=367316992 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=367529280 ingress_bytes=807718912 link_bytes=807718912 token_bytes=734846272 weight_movement_bytes=440401920 total_nvlink_bytes=1175248192 weight_to_token_ratio=0.599312 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.404756035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50224 payload_rows_remote=42410 dup_rows=15312 dup_rows_remote=11462 dedup_ratio=0.233643 router_tokens_per_expert=[0, 2805, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 150, 0, 0, 0, 0, 1908, 4096, 0, 4096, 0, 0, 0, 2783, 0, 4049, 2935, 0, 0, 3989, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 393, 0, 0, 109, 0, 0, 0, 66, 0] payload_rows_per_dst=[7814, 8152, 8192, 8020, 6026, 5762, 1486, 4772] dup_rows_per_dst=[3850, 1574, 0, 2320, 3870, 3440, 102, 156] combine_rows_per_src=[7814, 8152, 8192, 8020, 6026, 5762, 1486, 4772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=303994880 dispatch_meta_bytes=215488 combine_payload_bytes=303994880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304210368 ingress_bytes=414095360 link_bytes=414095360 token_bytes=608205248 weight_movement_bytes=110100480 total_nvlink_bytes=718305728 weight_to_token_ratio=0.181025 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.201799380e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54682 payload_rows_remote=46490 dup_rows=10854 dup_rows_remote=6304 dedup_ratio=0.165619 router_tokens_per_expert=[0, 10, 0, 119, 0, 0, 0, 2146, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 3847, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 2800, 0, 442, 0] payload_rows_per_dst=[8192, 8168, 8190, 8190, 8154, 7704, 0, 6084] dup_rows_per_dst=[4550, 182, 0, 0, 5706, 16, 0, 400] combine_rows_per_src=[8192, 8168, 8190, 8190, 8154, 7704, 0, 6084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=333240320 dispatch_meta_bytes=211176 combine_payload_bytes=333240320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333451496 ingress_bytes=443340800 link_bytes=443340800 token_bytes=666691816 weight_movement_bytes=110100480 total_nvlink_bytes=776792296 weight_to_token_ratio=0.165144 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.710425682e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54938 payload_rows_remote=46764 dup_rows=10598 dup_rows_remote=6098 dedup_ratio=0.161713 router_tokens_per_expert=[0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 4096, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 4096, 0, 70, 183, 2493, 0, 0, 0, 4096, 0, 0, 1868, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8192, 8064, 8096, 8192, 8046, 2438, 3736] dup_rows_per_dst=[4500, 0, 4334, 0, 0, 1564, 200, 0] combine_rows_per_src=[8174, 8192, 8064, 8096, 8192, 8046, 2438, 3736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=335204352 dispatch_meta_bytes=211448 combine_payload_bytes=335204352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335415800 ingress_bytes=445304832 link_bytes=445304832 token_bytes=670620152 weight_movement_bytes=110100480 total_nvlink_bytes=780720632 weight_to_token_ratio=0.164177 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.744583429e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51266 payload_rows_remote=43074 dup_rows=14270 dup_rows_remote=14106 dedup_ratio=0.217743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3619, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 55, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 7974, 8192, 7774, 1826, 8192, 924] dup_rows_per_dst=[164, 7238, 3502, 118, 3138, 110, 0, 0] combine_rows_per_src=[8192, 8192, 7974, 8192, 7774, 1826, 8192, 924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=308754432 dispatch_meta_bytes=228720 combine_payload_bytes=308754432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308983152 ingress_bytes=418854912 link_bytes=418854912 token_bytes=617737584 weight_movement_bytes=110100480 total_nvlink_bytes=727838064 weight_to_token_ratio=0.178232 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.284575818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56260 payload_rows_remote=48294 dup_rows=9276 dup_rows_remote=5942 dedup_ratio=0.141541 router_tokens_per_expert=[0, 0, 43, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 500, 0, 3525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 100, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 2811, 3602, 0] payload_rows_per_dst=[7966, 8190, 8192, 7050, 8192, 4996, 8192, 3482] dup_rows_per_dst=[3334, 26, 4274, 0, 0, 1622, 0, 20] combine_rows_per_src=[7966, 8190, 8192, 7050, 8192, 4996, 8192, 3482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=346171392 dispatch_meta_bytes=216944 combine_payload_bytes=346171392 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=346388336 ingress_bytes=566372352 link_bytes=566372352 token_bytes=692559728 weight_movement_bytes=220200960 total_nvlink_bytes=912760688 weight_to_token_ratio=0.317952 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.850146724e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58618 payload_rows_remote=50588 dup_rows=6918 dup_rows_remote=932 dedup_ratio=0.105560 router_tokens_per_expert=[3727, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2402, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8030, 5532, 8192, 8192, 8192, 8162, 4126, 8192] dup_rows_per_dst=[5986, 902, 8, 22, 0, 0, 0, 0] combine_rows_per_src=[8030, 5532, 8192, 8192, 8192, 8162, 4126, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=362614784 dispatch_meta_bytes=206080 combine_payload_bytes=362614784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=362820864 ingress_bytes=692916224 link_bytes=692916224 token_bytes=725435648 weight_movement_bytes=330301440 total_nvlink_bytes=1055737088 weight_to_token_ratio=0.455315 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.205095279e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6458796160 prefetch_bytes=1211105280 reduce_bytes=990904320 weight_movement_bytes=2202009600 egress_bytes=3230472320 ingress_bytes=5430333440 link_bytes=5430333440 total_nvlink_bytes=8660805760 weight_to_token_ratio=0.340932 prefetch_share_of_ingress=0.223026 dedup_ratio=0.192618 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=9 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.444242989e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-29 23:43:06] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 239.4 | throughput per GPU (TFLOP/s/GPU): 515.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.711975E+00 | loss scale: 1.0 | grad norm: 4.472 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=39088 dup_rows=21116 dup_rows_remote=18962 dedup_ratio=0.322205 router_tokens_per_expert=[1556, 833, 384, 829, 31, 756, 329, 96, 652, 796, 487, 478, 3, 797, 306, 813, 531, 1090, 7, 38, 1084, 277, 934, 590, 379, 393, 398, 375, 614, 218, 654, 1, 625, 594, 623, 12, 1127, 1, 923, 516, 252, 526, 487, 1044, 11, 338, 14, 331, 347, 391, 710, 666, 291, 736, 567, 340, 412, 850, 632, 485, 727, 1160, 286, 15] payload_rows_per_dst=[5170, 5784, 6106, 5332, 5720, 5700, 5070, 5538] dup_rows_per_dst=[2204, 2880, 2996, 2154, 3122, 2560, 3026, 2174] combine_rows_per_src=[5170, 5784, 6106, 5332, 5720, 5700, 5070, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43748 payload_rows_remote=38010 dup_rows=21788 dup_rows_remote=18996 dedup_ratio=0.332458 router_tokens_per_expert=[990, 727, 806, 700, 23, 618, 422, 58, 911, 633, 330, 366, 6, 1032, 543, 680, 601, 725, 161, 56, 901, 276, 440, 1105, 221, 1493, 665, 444, 1602, 230, 931, 3, 162, 623, 453, 11, 941, 13, 1233, 356, 171, 546, 721, 995, 2, 282, 86, 654, 344, 489, 131, 749, 77, 484, 470, 454, 615, 481, 264, 475, 764, 596, 424, 3] payload_rows_per_dst=[5774, 5912, 5738, 6926, 5208, 4978, 4482, 4730] dup_rows_per_dst=[2914, 3090, 2792, 5444, 2376, 1936, 1914, 1322] combine_rows_per_src=[5774, 5912, 5738, 6926, 5208, 4978, 4482, 4730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44894 payload_rows_remote=38836 dup_rows=20642 dup_rows_remote=16900 dedup_ratio=0.314972 router_tokens_per_expert=[1360, 1234, 879, 421, 3, 883, 409, 66, 869, 947, 243, 466, 3, 903, 405, 590, 383, 709, 1, 22, 652, 344, 900, 630, 254, 232, 641, 305, 740, 281, 791, 4, 798, 1124, 554, 25, 323, 7, 730, 779, 516, 487, 277, 681, 1, 422, 51, 520, 798, 383, 378, 1039, 124, 671, 301, 442, 592, 662, 700, 842, 738, 1061, 168, 4] payload_rows_per_dst=[5260, 5434, 5400, 5122, 5680, 5838, 6058, 6102] dup_rows_per_dst=[2530, 1890, 1882, 1374, 3000, 2792, 3742, 3432] combine_rows_per_src=[5260, 5434, 5400, 5122, 5680, 5838, 6058, 6102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38910 dup_rows=21086 dup_rows_remote=18400 dedup_ratio=0.321747 router_tokens_per_expert=[1789, 469, 652, 353, 18, 445, 439, 60, 475, 962, 433, 651, 4, 484, 238, 738, 623, 988, 41, 4, 648, 138, 1075, 702, 371, 495, 396, 560, 747, 189, 768, 5, 722, 671, 428, 16, 791, 10, 1178, 297, 363, 756, 263, 856, 7, 489, 151, 610, 304, 848, 490, 730, 161, 824, 325, 468, 581, 908, 433, 505, 1060, 1288, 272, 3] payload_rows_per_dst=[3876, 5480, 5888, 5236, 5540, 6644, 5404, 6382] dup_rows_per_dst=[996, 2490, 2550, 1826, 2686, 3924, 2896, 3718] combine_rows_per_src=[3876, 5480, 5888, 5236, 5540, 6644, 5404, 6382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=38182 dup_rows=20942 dup_rows_remote=16852 dedup_ratio=0.319550 router_tokens_per_expert=[1473, 932, 387, 948, 13, 563, 429, 6, 463, 737, 614, 265, 2, 971, 152, 873, 569, 624, 3, 3, 861, 169, 1070, 577, 150, 567, 389, 478, 1191, 170, 826, 0, 409, 618, 478, 2, 986, 0, 993, 468, 544, 586, 355, 1307, 3, 592, 61, 330, 552, 737, 542, 434, 144, 675, 175, 235, 488, 1324, 701, 407, 1062, 807, 275, 3] payload_rows_per_dst=[4954, 5564, 5668, 5460, 5216, 6412, 4834, 6486] dup_rows_per_dst=[1602, 2590, 2084, 2082, 2692, 4090, 2154, 3648] combine_rows_per_src=[4954, 5564, 5668, 5460, 5216, 6412, 4834, 6486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43208 payload_rows_remote=36592 dup_rows=22328 dup_rows_remote=17670 dedup_ratio=0.340698 router_tokens_per_expert=[1347, 331, 561, 522, 15, 1073, 524, 35, 369, 874, 727, 486, 6, 1229, 365, 575, 1012, 415, 11, 0, 285, 319, 837, 1338, 144, 428, 563, 710, 701, 385, 616, 2, 683, 350, 486, 13, 848, 0, 1086, 364, 437, 546, 292, 587, 2, 541, 224, 380, 615, 746, 368, 444, 226, 1425, 202, 440, 539, 845, 656, 279, 648, 1384, 306, 1] payload_rows_per_dst=[6616, 5390, 6156, 6104, 4730, 4346, 5756, 4110] dup_rows_per_dst=[4658, 2690, 3574, 3762, 1654, 1672, 3176, 1142] combine_rows_per_src=[6616, 5390, 6156, 6104, 4730, 4346, 5756, 4110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44820 payload_rows_remote=38846 dup_rows=20716 dup_rows_remote=17222 dedup_ratio=0.316101 router_tokens_per_expert=[1027, 1367, 1230, 611, 15, 1002, 601, 19, 942, 887, 541, 271, 1, 993, 458, 685, 511, 409, 11, 24, 788, 165, 793, 410, 206, 669, 509, 453, 992, 174, 526, 1, 669, 942, 542, 2, 684, 4, 1105, 422, 197, 735, 323, 1070, 2, 275, 64, 358, 318, 769, 533, 458, 112, 603, 177, 480, 955, 515, 481, 386, 1215, 714, 365, 2] payload_rows_per_dst=[6168, 5676, 5008, 5192, 5562, 5586, 5654, 5974] dup_rows_per_dst=[3522, 2106, 1214, 1868, 2976, 2516, 3020, 3494] combine_rows_per_src=[6168, 5676, 5008, 5192, 5562, 5586, 5654, 5974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45102 payload_rows_remote=39140 dup_rows=20434 dup_rows_remote=17836 dedup_ratio=0.311798 router_tokens_per_expert=[1510, 686, 539, 347, 35, 500, 434, 99, 431, 1140, 887, 287, 0, 868, 482, 681, 755, 632, 40, 75, 737, 196, 642, 781, 310, 641, 440, 414, 933, 153, 470, 24, 585, 790, 1213, 14, 870, 6, 931, 278, 354, 905, 327, 698, 0, 542, 37, 408, 477, 443, 395, 632, 133, 985, 401, 709, 459, 1165, 598, 402, 1111, 442, 278, 11] payload_rows_per_dst=[6170, 5962, 5926, 5536, 5824, 4712, 5564, 5408] dup_rows_per_dst=[3122, 2598, 2554, 2118, 3550, 1830, 2786, 1876] combine_rows_per_src=[6170, 5962, 5926, 5536, 5824, 4712, 5564, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=232200 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280414984 ingress_bytes=390283264 link_bytes=390283264 token_bytes=560597768 weight_movement_bytes=110100480 total_nvlink_bytes=670698248 weight_to_token_ratio=0.196398 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787667867e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272455680 dispatch_meta_bytes=228024 combine_payload_bytes=272455680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272683704 ingress_bytes=382556160 link_bytes=382556160 token_bytes=545139384 weight_movement_bytes=110100480 total_nvlink_bytes=655239864 weight_to_token_ratio=0.201968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.653280819e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=222944 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278599392 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556975840 weight_movement_bytes=110100480 total_nvlink_bytes=667076320 weight_to_token_ratio=0.197676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229240 combine_payload_bytes=278906880 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279136120 ingress_bytes=499107840 link_bytes=499107840 token_bytes=558043000 weight_movement_bytes=220200960 total_nvlink_bytes=778243960 weight_to_token_ratio=0.394595 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680306229e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273688576 dispatch_meta_bytes=220136 combine_payload_bytes=273688576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273908712 ingress_bytes=383789056 link_bytes=383789056 token_bytes=547597288 weight_movement_bytes=110100480 total_nvlink_bytes=657697768 weight_to_token_ratio=0.201061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.674722908e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=262291456 dispatch_meta_bytes=217048 combine_payload_bytes=262291456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=262508504 ingress_bytes=482492416 link_bytes=482492416 token_bytes=524799960 weight_movement_bytes=220200960 total_nvlink_bytes=745000920 weight_to_token_ratio=0.419590 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.391336678e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=224272 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278672400 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557120528 weight_movement_bytes=330301440 total_nvlink_bytes=887421968 weight_to_token_ratio=0.592872 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280555520 dispatch_meta_bytes=227904 combine_payload_bytes=280555520 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280783424 ingress_bytes=610856960 link_bytes=610856960 token_bytes=561338944 weight_movement_bytes=330301440 total_nvlink_bytes=891640384 weight_to_token_ratio=0.588417 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=5760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062380722e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49966 payload_rows_remote=42230 dup_rows=15570 dup_rows_remote=10826 dedup_ratio=0.237579 router_tokens_per_expert=[1, 0, 0, 24, 2355, 3, 2287, 72, 3428, 50, 1407, 0, 0, 339, 0, 3373, 2, 0, 0, 0, 824, 2828, 0, 0, 0, 1, 673, 0, 833, 86, 0, 35, 1, 5, 0, 0, 4, 0, 0, 1692, 0, 0, 165, 0, 6, 0, 165, 1, 0, 1855, 2586, 0, 1020, 17, 0, 0, 12, 0, 1504, 1343, 106, 286, 3370, 9] payload_rows_per_dst=[7288, 7486, 7736, 2874, 7478, 6912, 4946, 5246] dup_rows_per_dst=[2196, 2852, 4744, 382, 2666, 618, 838, 1274] combine_rows_per_src=[7288, 7486, 7736, 2874, 7478, 6912, 4946, 5246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50972 payload_rows_remote=43600 dup_rows=14564 dup_rows_remote=11118 dedup_ratio=0.222229 router_tokens_per_expert=[0, 0, 0, 10, 3091, 1, 2094, 171, 3795, 9, 976, 0, 0, 279, 0, 2835, 0, 0, 0, 3, 701, 2966, 0, 0, 1, 0, 758, 0, 1348, 27, 1, 6, 0, 0, 0, 1, 0, 0, 0, 2351, 0, 2, 93, 2, 0, 0, 376, 0, 0, 1968, 2171, 0, 681, 4, 0, 0, 28, 0, 1131, 1554, 28, 236, 3057, 13] payload_rows_per_dst=[4400, 6570, 6350, 7244, 7372, 7626, 6568, 4842] dup_rows_per_dst=[152, 1628, 990, 3220, 3446, 910, 3080, 1138] combine_rows_per_src=[4400, 6570, 6350, 7244, 7372, 7626, 6568, 4842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51190 payload_rows_remote=44392 dup_rows=14346 dup_rows_remote=11744 dedup_ratio=0.218903 router_tokens_per_expert=[0, 0, 0, 58, 3016, 0, 1945, 172, 3634, 41, 1022, 0, 0, 350, 0, 3066, 0, 0, 0, 0, 777, 2602, 0, 0, 0, 0, 425, 0, 1616, 49, 0, 17, 0, 0, 0, 0, 8, 0, 0, 2297, 0, 0, 106, 1, 0, 0, 306, 0, 0, 1605, 2648, 0, 850, 6, 0, 0, 18, 0, 1850, 1047, 55, 341, 2813, 27] payload_rows_per_dst=[4772, 7050, 5986, 6798, 7262, 7310, 6634, 5378] dup_rows_per_dst=[424, 1908, 1338, 2602, 2974, 784, 3018, 1298] combine_rows_per_src=[4772, 7050, 5986, 6798, 7262, 7310, 6634, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50602 payload_rows_remote=44182 dup_rows=14934 dup_rows_remote=14184 dedup_ratio=0.227875 router_tokens_per_expert=[0, 0, 0, 40, 2908, 0, 1960, 154, 3144, 12, 500, 0, 0, 678, 0, 3118, 0, 0, 0, 0, 629, 3275, 0, 0, 0, 0, 541, 0, 1355, 4, 0, 49, 0, 0, 0, 0, 4, 0, 0, 1923, 0, 0, 20, 0, 0, 0, 421, 0, 0, 2042, 2877, 0, 705, 2, 0, 0, 67, 0, 1217, 1716, 64, 236, 3104, 3] payload_rows_per_dst=[4116, 6932, 6792, 6986, 5654, 6420, 7108, 6594] dup_rows_per_dst=[192, 1684, 1016, 2728, 1304, 750, 4144, 3116] combine_rows_per_src=[4116, 6932, 6792, 6986, 5654, 6420, 7108, 6594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51788 payload_rows_remote=44872 dup_rows=13748 dup_rows_remote=10664 dedup_ratio=0.209778 router_tokens_per_expert=[0, 0, 0, 7, 3262, 0, 2899, 129, 3609, 113, 663, 0, 0, 418, 0, 2828, 5, 0, 0, 0, 432, 3008, 0, 0, 0, 0, 400, 0, 1457, 46, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 32, 0, 0, 0, 282, 0, 0, 1968, 2462, 0, 552, 18, 0, 0, 48, 0, 1780, 652, 59, 198, 3504, 11] payload_rows_per_dst=[5850, 6636, 6182, 7312, 3836, 7232, 6916, 7824] dup_rows_per_dst=[220, 1408, 708, 3034, 0, 614, 3084, 4680] combine_rows_per_src=[5850, 6636, 6182, 7312, 3836, 7232, 6916, 7824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49088 payload_rows_remote=41052 dup_rows=16448 dup_rows_remote=8318 dedup_ratio=0.250977 router_tokens_per_expert=[0, 0, 0, 22, 3248, 0, 1886, 172, 3709, 17, 641, 0, 0, 445, 0, 2555, 3, 0, 0, 0, 618, 3227, 0, 0, 0, 0, 1170, 0, 1613, 48, 0, 21, 0, 0, 0, 0, 0, 1, 0, 1548, 0, 0, 259, 0, 0, 0, 409, 0, 0, 1968, 2755, 0, 411, 2, 0, 0, 81, 0, 849, 1572, 30, 251, 3233, 4] payload_rows_per_dst=[8036, 6220, 6686, 4710, 7334, 7464, 4084, 4554] dup_rows_per_dst=[8130, 1474, 1010, 994, 2230, 1290, 300, 1020] combine_rows_per_src=[8036, 6220, 6686, 4710, 7334, 7464, 4084, 4554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48626 payload_rows_remote=40836 dup_rows=16910 dup_rows_remote=10130 dedup_ratio=0.258026 router_tokens_per_expert=[0, 0, 0, 20, 3348, 0, 2525, 150, 3537, 155, 332, 0, 0, 361, 0, 3462, 4, 0, 0, 0, 529, 3015, 0, 0, 0, 0, 714, 0, 1496, 57, 1, 16, 0, 0, 0, 0, 0, 0, 0, 1887, 0, 0, 22, 0, 0, 3, 325, 0, 0, 1967, 2470, 0, 421, 21, 0, 1, 31, 0, 1166, 1182, 31, 114, 3397, 8] payload_rows_per_dst=[5240, 4040, 6302, 7484, 3774, 7262, 6734, 7790] dup_rows_per_dst=[150, 902, 794, 3780, 0, 1478, 3026, 6780] combine_rows_per_src=[5240, 4040, 6302, 7484, 3774, 7262, 6734, 7790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=302704640 dispatch_meta_bytes=212224 combine_payload_bytes=302704640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302916864 ingress_bytes=412805120 link_bytes=412805120 token_bytes=605621504 weight_movement_bytes=110100480 total_nvlink_bytes=715721984 weight_to_token_ratio=0.181798 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.179359985e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=312524800 dispatch_meta_bytes=218872 combine_payload_bytes=312524800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312743672 ingress_bytes=422625280 link_bytes=422625280 token_bytes=625268472 weight_movement_bytes=110100480 total_nvlink_bytes=735368952 weight_to_token_ratio=0.176085 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.350148718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318201856 dispatch_meta_bytes=224544 combine_payload_bytes=318201856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318426400 ingress_bytes=428302336 link_bytes=428302336 token_bytes=636628256 weight_movement_bytes=110100480 total_nvlink_bytes=746728736 weight_to_token_ratio=0.172943 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.448882059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316696576 dispatch_meta_bytes=233464 combine_payload_bytes=316696576 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=316930040 ingress_bytes=536897536 link_bytes=536897536 token_bytes=633626616 weight_movement_bytes=220200960 total_nvlink_bytes=853827576 weight_to_token_ratio=0.347525 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.337531196e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49040 payload_rows_remote=42118 dup_rows=16496 dup_rows_remote=14672 dedup_ratio=0.251709 router_tokens_per_expert=[0, 0, 0, 14, 3255, 0, 2105, 102, 3845, 111, 739, 0, 0, 501, 0, 3022, 1, 4, 0, 0, 796, 2555, 0, 0, 0, 0, 1074, 0, 1494, 108, 0, 18, 0, 0, 1, 0, 3, 0, 0, 2030, 0, 0, 98, 0, 1, 0, 489, 0, 0, 1961, 2804, 0, 551, 14, 0, 2, 22, 1, 902, 791, 26, 214, 3069, 45] payload_rows_per_dst=[8008, 6922, 6448, 4446, 7272, 7712, 4582, 3650] dup_rows_per_dst=[6362, 1824, 2454, 942, 2934, 1154, 474, 352] combine_rows_per_src=[8008, 6922, 6448, 4446, 7272, 7712, 4582, 3650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294260736 dispatch_meta_bytes=197480 combine_payload_bytes=294260736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=294458216 ingress_bytes=514461696 link_bytes=514461696 token_bytes=588718952 weight_movement_bytes=220200960 total_nvlink_bytes=808919912 weight_to_token_ratio=0.374034 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.947335038e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=321642496 dispatch_meta_bytes=222144 combine_payload_bytes=321642496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=321864640 ingress_bytes=651943936 link_bytes=651943936 token_bytes=643507136 weight_movement_bytes=330301440 total_nvlink_bytes=973808576 weight_to_token_ratio=0.513283 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.133837731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=292712448 dispatch_meta_bytes=203864 combine_payload_bytes=292712448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=292916312 ingress_bytes=512913408 link_bytes=512913408 token_bytes=585628760 weight_movement_bytes=220200960 total_nvlink_bytes=805829720 weight_to_token_ratio=0.376008 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.920407763e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301901824 dispatch_meta_bytes=227160 combine_payload_bytes=301901824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=302128984 ingress_bytes=742303744 link_bytes=742303744 token_bytes=604030808 weight_movement_bytes=440401920 total_nvlink_bytes=1044432728 weight_to_token_ratio=0.729105 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.290988299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53428 payload_rows_remote=45576 dup_rows=12108 dup_rows_remote=7806 dedup_ratio=0.184753 router_tokens_per_expert=[0, 0, 2266, 57, 2, 1, 463, 0, 0, 190, 0, 0, 0, 1309, 0, 0, 0, 1, 0, 0, 0, 2342, 3734, 0, 0, 0, 1630, 552, 0, 1040, 20, 0, 74, 538, 0, 0, 4, 3549, 4029, 2, 3979, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 0, 0, 2952, 36, 0, 0, 14] payload_rows_per_dst=[4882, 8154, 7852, 5156, 7342, 5342, 8058, 6642] dup_rows_per_dst=[696, 2802, 4302, 1328, 992, 0, 0, 1988] combine_rows_per_src=[4882, 8154, 7852, 5156, 7342, 5342, 8058, 6642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53956 payload_rows_remote=46754 dup_rows=11580 dup_rows_remote=10912 dedup_ratio=0.176697 router_tokens_per_expert=[0, 0, 3145, 38, 1, 2, 1061, 0, 0, 179, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 0, 2466, 3341, 0, 0, 0, 1567, 916, 0, 573, 3, 0, 37, 357, 0, 0, 28, 3512, 4019, 1, 3961, 1, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 1, 0, 2732, 38, 0, 0, 29] payload_rows_per_dst=[6946, 7674, 5638, 7178, 7202, 4932, 8038, 6348] dup_rows_per_dst=[1548, 2258, 1182, 3734, 668, 388, 0, 1802] combine_rows_per_src=[6946, 7674, 5638, 7178, 7202, 4932, 8038, 6348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52140 payload_rows_remote=44056 dup_rows=13396 dup_rows_remote=7914 dedup_ratio=0.204407 router_tokens_per_expert=[0, 0, 3322, 24, 0, 1, 797, 0, 0, 175, 0, 0, 0, 1252, 0, 0, 0, 0, 0, 0, 0, 2358, 3879, 0, 0, 0, 1433, 718, 0, 263, 12, 0, 18, 404, 0, 0, 45, 3502, 4052, 0, 4041, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 0, 2786, 44, 0, 0, 17] payload_rows_per_dst=[7010, 2806, 4716, 7976, 7226, 8084, 8104, 6218] dup_rows_per_dst=[1278, 48, 0, 4634, 712, 5482, 0, 1242] combine_rows_per_src=[7010, 2806, 4716, 7976, 7226, 8084, 8104, 6218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53168 payload_rows_remote=49716 dup_rows=12368 dup_rows_remote=11644 dedup_ratio=0.188721 router_tokens_per_expert=[0, 0, 3223, 126, 0, 110, 649, 0, 0, 217, 0, 0, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 2491, 3691, 0, 0, 0, 996, 514, 0, 577, 1, 0, 92, 753, 0, 0, 20, 3703, 4013, 0, 3942, 0, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1032, 0, 0, 2649, 39, 0, 0, 39] payload_rows_per_dst=[6790, 8156, 7960, 3452, 7612, 4894, 8028, 6276] dup_rows_per_dst=[1426, 3048, 4404, 724, 1524, 0, 0, 1242] combine_rows_per_src=[6790, 8156, 7960, 3452, 7612, 4894, 8028, 6276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54216 payload_rows_remote=46500 dup_rows=11320 dup_rows_remote=8162 dedup_ratio=0.172729 router_tokens_per_expert=[0, 0, 3649, 14, 0, 1, 767, 0, 0, 175, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 2156, 3289, 0, 0, 0, 1678, 165, 0, 283, 2, 0, 19, 243, 0, 0, 63, 3474, 4041, 0, 3947, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 3496, 33, 0, 0, 7] payload_rows_per_dst=[7716, 8148, 7060, 3888, 7088, 4636, 8082, 7598] dup_rows_per_dst=[3158, 2928, 1818, 368, 510, 0, 0, 2538] combine_rows_per_src=[7716, 8148, 7060, 3888, 7088, 4636, 8082, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=326688768 dispatch_meta_bytes=213528 combine_payload_bytes=326688768 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=326902296 ingress_bytes=877191168 link_bytes=877191168 token_bytes=653591064 weight_movement_bytes=550502400 total_nvlink_bytes=1204093464 weight_to_token_ratio=0.842273 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.525579714e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49596 payload_rows_remote=42872 dup_rows=15940 dup_rows_remote=14254 dedup_ratio=0.243225 router_tokens_per_expert=[0, 0, 3085, 22, 0, 12, 563, 0, 0, 154, 0, 0, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 3277, 3911, 0, 0, 0, 1479, 201, 0, 439, 10, 0, 48, 249, 0, 0, 59, 3500, 4056, 3, 3987, 0, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 2910, 29, 0, 0, 0] payload_rows_per_dst=[6532, 3234, 2724, 7984, 6264, 8020, 8114, 6724] dup_rows_per_dst=[832, 22, 0, 4096, 1586, 6614, 1104, 1686] combine_rows_per_src=[6532, 3234, 2724, 7984, 6264, 8020, 8114, 6724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54748 payload_rows_remote=46706 dup_rows=10788 dup_rows_remote=10788 dedup_ratio=0.164612 router_tokens_per_expert=[0, 0, 3402, 80, 0, 73, 609, 0, 0, 211, 0, 0, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 2856, 3779, 0, 0, 0, 916, 630, 0, 375, 3, 0, 68, 471, 0, 0, 40, 3600, 4021, 0, 4001, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 2676, 35, 0, 0, 4] payload_rows_per_dst=[7150, 4248, 5712, 7830, 7430, 8042, 8042, 6294] dup_rows_per_dst=[1178, 58, 0, 3576, 928, 3664, 0, 1384] combine_rows_per_src=[7150, 4248, 5712, 7830, 7430, 8042, 8042, 6294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52800 payload_rows_remote=44694 dup_rows=12736 dup_rows_remote=8890 dedup_ratio=0.194336 router_tokens_per_expert=[0, 0, 3165, 135, 5, 71, 821, 0, 0, 211, 0, 0, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 2432, 3652, 0, 0, 0, 877, 560, 0, 367, 2, 0, 66, 523, 0, 0, 77, 3555, 3977, 5, 3971, 2, 2717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1210, 0, 0, 2531, 26, 1, 0, 9] payload_rows_per_dst=[6918, 8106, 7826, 3130, 7268, 5436, 7964, 6152] dup_rows_per_dst=[1476, 3846, 4342, 482, 1184, 2, 2, 1402] combine_rows_per_src=[6918, 8106, 7826, 3130, 7268, 5436, 7964, 6152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=315793408 dispatch_meta_bytes=207880 combine_payload_bytes=315793408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=316001288 ingress_bytes=535994368 link_bytes=535994368 token_bytes=631794696 weight_movement_bytes=220200960 total_nvlink_bytes=851995656 weight_to_token_ratio=0.348532 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.321823619e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=335132672 dispatch_meta_bytes=230664 combine_payload_bytes=335132672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=335363336 ingress_bytes=665434112 link_bytes=665434112 token_bytes=670496008 weight_movement_bytes=330301440 total_nvlink_bytes=1000797448 weight_to_token_ratio=0.492623 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.157299366e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356364288 dispatch_meta_bytes=245440 combine_payload_bytes=356364288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356609728 ingress_bytes=466464768 link_bytes=466464768 token_bytes=712974016 weight_movement_bytes=110100480 total_nvlink_bytes=823074496 weight_to_token_ratio=0.154424 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.112589518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334788608 dispatch_meta_bytes=229976 combine_payload_bytes=334788608 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=335018584 ingress_bytes=554989568 link_bytes=554989568 token_bytes=669807192 weight_movement_bytes=220200960 total_nvlink_bytes=890008152 weight_to_token_ratio=0.328753 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.652181389e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=333312000 dispatch_meta_bytes=218648 combine_payload_bytes=333312000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333530648 ingress_bytes=443412480 link_bytes=443412480 token_bytes=666842648 weight_movement_bytes=110100480 total_nvlink_bytes=776943128 weight_to_token_ratio=0.165107 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.711672315e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=307306496 dispatch_meta_bytes=228504 combine_payload_bytes=307306496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307535000 ingress_bytes=417406976 link_bytes=417406976 token_bytes=614841496 weight_movement_bytes=110100480 total_nvlink_bytes=724941976 weight_to_token_ratio=0.179071 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.259393829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53118 payload_rows_remote=45932 dup_rows=12418 dup_rows_remote=11380 dedup_ratio=0.189484 router_tokens_per_expert=[62, 0, 3213, 0, 0, 124, 0, 0, 2, 0, 0, 585, 4, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 255, 0, 3320, 0, 0, 1937, 0, 0, 0, 0, 0, 362, 0, 3754, 110, 5, 0, 1962, 0, 0, 130, 2080, 0, 0, 0, 1991, 3939, 0, 0, 0, 0, 2151, 0, 950, 0, 0, 952, 4088, 0] payload_rows_per_dst=[7616, 8188, 7186, 5244, 7590, 3930, 7976, 5388] dup_rows_per_dst=[3342, 1170, 1038, 1040, 862, 264, 3884, 818] combine_rows_per_src=[7616, 8188, 7186, 5244, 7590, 3930, 7976, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=320366592 dispatch_meta_bytes=214336 combine_payload_bytes=320366592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320580928 ingress_bytes=430467072 link_bytes=430467072 token_bytes=640947520 weight_movement_bytes=110100480 total_nvlink_bytes=751048000 weight_to_token_ratio=0.171778 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.486530379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=329240576 dispatch_meta_bytes=229248 combine_payload_bytes=329240576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329469824 ingress_bytes=439341056 link_bytes=439341056 token_bytes=658710400 weight_movement_bytes=110100480 total_nvlink_bytes=768810880 weight_to_token_ratio=0.167146 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.640863556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53992 payload_rows_remote=47222 dup_rows=11544 dup_rows_remote=8820 dedup_ratio=0.176147 router_tokens_per_expert=[158, 2, 2293, 0, 1, 458, 0, 0, 3, 0, 0, 681, 0, 0, 0, 0, 0, 0, 1007, 0, 0, 0, 0, 0, 0, 399, 0, 3597, 0, 0, 2509, 0, 0, 0, 0, 0, 549, 0, 3819, 86, 2, 0, 2165, 0, 0, 49, 2059, 0, 0, 0, 982, 3414, 0, 0, 0, 0, 1770, 0, 1972, 0, 0, 705, 4088, 0] payload_rows_per_dst=[6164, 8182, 7594, 6770, 7752, 5744, 7036, 4750] dup_rows_per_dst=[1574, 1362, 1614, 2724, 1156, 892, 1756, 466] combine_rows_per_src=[6164, 8182, 7594, 6770, 7752, 5744, 7036, 4750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55960 payload_rows_remote=48188 dup_rows=9576 dup_rows_remote=6406 dedup_ratio=0.146118 router_tokens_per_expert=[194, 1, 1612, 0, 0, 450, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 277, 0, 3223, 0, 0, 3359, 0, 0, 0, 0, 0, 92, 0, 4036, 288, 0, 0, 2195, 0, 0, 19, 3257, 0, 0, 0, 496, 3557, 0, 0, 0, 0, 1705, 0, 1782, 0, 0, 563, 4094, 0] payload_rows_per_dst=[4004, 8188, 7174, 7024, 8118, 7772, 7192, 6488] dup_rows_per_dst=[510, 660, 1748, 248, 714, 3170, 914, 1612] combine_rows_per_src=[4004, 8188, 7174, 7024, 8118, 7772, 7192, 6488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55600 payload_rows_remote=47756 dup_rows=9936 dup_rows_remote=8792 dedup_ratio=0.151611 router_tokens_per_expert=[208, 2, 2564, 0, 0, 898, 0, 0, 5, 0, 0, 435, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 372, 0, 3501, 0, 0, 2888, 0, 0, 0, 0, 0, 497, 0, 3847, 150, 1, 0, 1912, 1, 0, 91, 2716, 0, 0, 0, 1005, 3183, 0, 0, 0, 0, 914, 0, 1895, 0, 0, 949, 4094, 0] payload_rows_per_dst=[6058, 8190, 7236, 6236, 7844, 7254, 6660, 6122] dup_rows_per_dst=[1286, 878, 1046, 284, 1144, 2188, 1716, 1394] combine_rows_per_src=[6058, 8190, 7236, 6236, 7844, 7254, 6660, 6122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55858 payload_rows_remote=49062 dup_rows=9678 dup_rows_remote=8614 dedup_ratio=0.147675 router_tokens_per_expert=[242, 0, 2121, 0, 0, 555, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 455, 0, 2691, 0, 0, 3197, 0, 0, 0, 0, 0, 156, 0, 4009, 276, 0, 0, 1589, 0, 0, 33, 3092, 0, 0, 0, 625, 3305, 0, 0, 0, 0, 978, 0, 2594, 0, 0, 1258, 4096, 0] payload_rows_per_dst=[5150, 8192, 6262, 6876, 8048, 7464, 6796, 7070] dup_rows_per_dst=[686, 1110, 1002, 428, 834, 1964, 1064, 2590] combine_rows_per_src=[5150, 8192, 6262, 6876, 8048, 7464, 6796, 7070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45164 payload_rows_remote=37020 dup_rows=20372 dup_rows_remote=9776 dedup_ratio=0.310852 router_tokens_per_expert=[196, 0, 2537, 0, 0, 502, 0, 0, 4, 0, 0, 341, 3, 0, 0, 0, 0, 0, 1297, 0, 0, 0, 0, 0, 0, 189, 0, 2915, 0, 1, 2774, 0, 0, 0, 0, 0, 131, 0, 3944, 237, 0, 0, 1858, 0, 0, 57, 3394, 0, 0, 0, 718, 3646, 0, 0, 0, 0, 1404, 0, 1735, 0, 0, 794, 4091, 0] payload_rows_per_dst=[5846, 1158, 5468, 6168, 3058, 7706, 7616, 8144] dup_rows_per_dst=[624, 78, 900, 1816, 240, 2912, 3206, 10596] combine_rows_per_src=[5846, 1158, 5468, 6168, 3058, 7706, 7616, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52140 payload_rows_remote=44556 dup_rows=13396 dup_rows_remote=10138 dedup_ratio=0.204407 router_tokens_per_expert=[204, 0, 1827, 0, 0, 181, 0, 0, 1, 0, 0, 463, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 332, 0, 2975, 0, 0, 3231, 0, 0, 0, 0, 0, 84, 0, 3957, 605, 5, 0, 1720, 0, 0, 74, 3209, 0, 0, 0, 757, 3196, 0, 0, 0, 0, 1848, 0, 1786, 0, 0, 952, 4091, 0] payload_rows_per_dst=[7584, 8184, 6868, 7542, 7982, 7392, 1514, 5074] dup_rows_per_dst=[3258, 926, 1622, 3156, 1310, 2598, 0, 526] combine_rows_per_src=[7584, 8184, 6868, 7542, 7982, 7392, 1514, 5074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53036 payload_rows_remote=44858 dup_rows=12500 dup_rows_remote=11608 dedup_ratio=0.190735 router_tokens_per_expert=[148, 0, 2610, 0, 0, 887, 0, 0, 0, 0, 0, 448, 3, 0, 0, 0, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 401, 0, 3385, 0, 4, 2795, 0, 0, 0, 0, 0, 342, 0, 3915, 185, 6, 0, 1608, 0, 0, 7, 2272, 0, 0, 0, 1049, 3576, 0, 0, 0, 0, 1552, 0, 1790, 0, 0, 697, 4084, 0] payload_rows_per_dst=[7452, 8178, 7334, 7094, 7962, 3852, 6964, 4200] dup_rows_per_dst=[4382, 892, 1444, 2886, 922, 434, 1242, 298] combine_rows_per_src=[7452, 8178, 7334, 7094, 7962, 3852, 6964, 4200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342315008 dispatch_meta_bytes=226192 combine_payload_bytes=342315008 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=342541200 ingress_bytes=562515968 link_bytes=562515968 token_bytes=684856208 weight_movement_bytes=220200960 total_nvlink_bytes=905057168 weight_to_token_ratio=0.321529 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.783077863e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=338487296 dispatch_meta_bytes=224168 combine_payload_bytes=338487296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338711464 ingress_bytes=558688256 link_bytes=558688256 token_bytes=677198760 weight_movement_bytes=220200960 total_nvlink_bytes=897399720 weight_to_token_ratio=0.325164 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.716507656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=345411584 dispatch_meta_bytes=218376 combine_payload_bytes=345411584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=345629960 ingress_bytes=565612544 link_bytes=565612544 token_bytes=691041544 weight_movement_bytes=220200960 total_nvlink_bytes=911242504 weight_to_token_ratio=0.318651 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.836932413e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=351676416 dispatch_meta_bytes=230704 combine_payload_bytes=351676416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=351907120 ingress_bytes=571877376 link_bytes=571877376 token_bytes=703583536 weight_movement_bytes=220200960 total_nvlink_bytes=923784496 weight_to_token_ratio=0.312971 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.945888145e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265359360 dispatch_meta_bytes=187184 combine_payload_bytes=265359360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=265546544 ingress_bytes=595660800 link_bytes=595660800 token_bytes=530905904 weight_movement_bytes=330301440 total_nvlink_bytes=861207344 weight_to_token_ratio=0.622147 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035952101e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=319377408 dispatch_meta_bytes=218776 combine_payload_bytes=319377408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319596184 ingress_bytes=429477888 link_bytes=429477888 token_bytes=638973592 weight_movement_bytes=110100480 total_nvlink_bytes=749074072 weight_to_token_ratio=0.172308 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.469326842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=321542144 dispatch_meta_bytes=225864 combine_payload_bytes=321542144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321768008 ingress_bytes=431642624 link_bytes=431642624 token_bytes=643310152 weight_movement_bytes=110100480 total_nvlink_bytes=753410632 weight_to_token_ratio=0.171147 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.506975161e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55412 payload_rows_remote=47220 dup_rows=10124 dup_rows_remote=10086 dedup_ratio=0.154480 router_tokens_per_expert=[0, 1540, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 766, 0, 0, 2, 0, 0, 4089, 4096, 0, 4051, 0, 0, 0, 78, 0, 3993, 3352, 0, 0, 1883, 0, 0, 0, 4096, 0, 0, 0, 81, 2092, 1017, 0, 0, 876, 0, 0, 500, 0, 0] payload_rows_per_dst=[8192, 7986, 8192, 8184, 8140, 6670, 2194, 5854] dup_rows_per_dst=[3554, 0, 38, 1530, 1530, 2388, 2, 1082] combine_rows_per_src=[8192, 7986, 8192, 8184, 8140, 6670, 2194, 5854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338472960 dispatch_meta_bytes=229224 combine_payload_bytes=338472960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338702184 ingress_bytes=448573440 link_bytes=448573440 token_bytes=677175144 weight_movement_bytes=110100480 total_nvlink_bytes=787275624 weight_to_token_ratio=0.162588 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.801429898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55134 payload_rows_remote=48022 dup_rows=10402 dup_rows_remote=8228 dedup_ratio=0.158722 router_tokens_per_expert=[0, 425, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 850, 6, 0, 0, 0, 0, 4094, 4096, 0, 4051, 0, 0, 0, 56, 0, 4079, 2349, 0, 0, 2294, 0, 0, 0, 4096, 0, 0, 0, 9, 3005, 1217, 0, 0, 134, 0, 0, 1745, 0, 0] payload_rows_per_dst=[7852, 8162, 8192, 8192, 112, 7112, 8190, 7322] dup_rows_per_dst=[1282, 0, 12, 1708, 0, 2174, 2780, 2446] combine_rows_per_src=[7852, 8162, 8192, 8192, 112, 7112, 8190, 7322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49908 payload_rows_remote=41716 dup_rows=15628 dup_rows_remote=5078 dedup_ratio=0.238464 router_tokens_per_expert=[0, 656, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 522, 3, 0, 0, 0, 0, 4093, 4096, 0, 4061, 0, 0, 0, 148, 0, 4058, 2909, 0, 0, 1475, 0, 0, 0, 4096, 0, 0, 0, 15, 3295, 1199, 0, 0, 232, 0, 0, 1742, 0, 0] payload_rows_per_dst=[1614, 8128, 8192, 4546, 4452, 7202, 8192, 7582] dup_rows_per_dst=[6, 0, 16, 414, 120, 1566, 10550, 2956] combine_rows_per_src=[1614, 8128, 8192, 4546, 4452, 7202, 8192, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56240 payload_rows_remote=49268 dup_rows=9296 dup_rows_remote=9026 dedup_ratio=0.141846 router_tokens_per_expert=[0, 1075, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1209, 5, 0, 0, 0, 0, 4091, 4096, 0, 4039, 0, 0, 0, 149, 0, 4032, 3087, 0, 0, 1723, 0, 0, 0, 4096, 0, 0, 0, 25, 2280, 1196, 0, 0, 533, 0, 0, 958, 0, 0] payload_rows_per_dst=[8192, 8064, 8192, 8192, 6972, 7358, 3262, 6008] dup_rows_per_dst=[2482, 0, 16, 2418, 270, 2262, 314, 1534] combine_rows_per_src=[8192, 8064, 8192, 8192, 6972, 7358, 3262, 6008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57462 payload_rows_remote=49276 dup_rows=8074 dup_rows_remote=6144 dedup_ratio=0.123199 router_tokens_per_expert=[0, 829, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 963, 7, 0, 1, 0, 0, 4087, 4096, 0, 4050, 0, 0, 0, 51, 0, 4020, 3302, 0, 0, 1735, 0, 0, 0, 4096, 0, 0, 0, 96, 2908, 1093, 0, 0, 370, 0, 0, 822, 0, 0] payload_rows_per_dst=[8192, 8042, 8192, 8186, 8100, 7578, 2366, 6806] dup_rows_per_dst=[2130, 0, 10, 1930, 102, 2496, 12, 1394] combine_rows_per_src=[8192, 8042, 8192, 8186, 8100, 7578, 2366, 6806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50218 payload_rows_remote=42908 dup_rows=15318 dup_rows_remote=13322 dedup_ratio=0.233734 router_tokens_per_expert=[0, 866, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 454, 6, 0, 0, 0, 0, 4082, 4096, 0, 4060, 0, 0, 0, 50, 0, 4068, 2780, 0, 0, 2242, 0, 0, 0, 4096, 0, 0, 0, 18, 3105, 1094, 0, 0, 304, 0, 0, 1244, 0, 0] payload_rows_per_dst=[2116, 8136, 8060, 920, 8164, 7320, 8192, 7310] dup_rows_per_dst=[6, 0, 16, 0, 232, 2724, 10344, 1996] combine_rows_per_src=[2116, 8136, 8060, 920, 8164, 7320, 8192, 7310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55460 payload_rows_remote=47268 dup_rows=10076 dup_rows_remote=8874 dedup_ratio=0.153748 router_tokens_per_expert=[0, 394, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 736, 10, 0, 0, 0, 0, 4090, 4096, 0, 4039, 0, 0, 0, 73, 0, 4016, 3079, 0, 0, 2271, 0, 0, 0, 4096, 0, 0, 0, 75, 3207, 677, 0, 0, 141, 0, 0, 1558, 0, 0] payload_rows_per_dst=[8192, 8176, 8192, 8192, 8138, 4542, 6114, 3914] dup_rows_per_dst=[1202, 742, 6, 1480, 5358, 0, 964, 324] combine_rows_per_src=[8192, 8176, 8192, 8192, 8138, 4542, 6114, 3914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53968 payload_rows_remote=45882 dup_rows=11568 dup_rows_remote=11554 dedup_ratio=0.176514 router_tokens_per_expert=[0, 899, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 789, 7, 0, 0, 0, 0, 4026, 4096, 0, 4055, 0, 0, 0, 56, 1, 3975, 2812, 0, 0, 2071, 0, 0, 0, 4094, 0, 0, 0, 116, 3044, 914, 0, 0, 277, 0, 0, 1183, 0, 0] payload_rows_per_dst=[8192, 8086, 8192, 8066, 8192, 4142, 2052, 7046] dup_rows_per_dst=[2332, 14, 18, 1578, 5654, 2, 8, 1962] combine_rows_per_src=[8192, 8086, 8192, 8066, 8192, 4142, 2052, 7046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344221696 dispatch_meta_bytes=225000 combine_payload_bytes=344221696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=344446696 ingress_bytes=674523136 link_bytes=674523136 token_bytes=688668392 weight_movement_bytes=330301440 total_nvlink_bytes=1018969832 weight_to_token_ratio=0.479623 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.173106673e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299020288 dispatch_meta_bytes=187176 combine_payload_bytes=299020288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=299207464 ingress_bytes=629321728 link_bytes=629321728 token_bytes=598227752 weight_movement_bytes=330301440 total_nvlink_bytes=928529192 weight_to_token_ratio=0.552133 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094493991e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353153024 dispatch_meta_bytes=233176 combine_payload_bytes=353153024 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=353386200 ingress_bytes=793554944 link_bytes=793554944 token_bytes=706539224 weight_movement_bytes=440401920 total_nvlink_bytes=1146941144 weight_to_token_ratio=0.623323 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.380122565e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353210368 dispatch_meta_bytes=221680 combine_payload_bytes=353210368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353432048 ingress_bytes=463310848 link_bytes=463310848 token_bytes=706642416 weight_movement_bytes=110100480 total_nvlink_bytes=816742896 weight_to_token_ratio=0.155808 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.057737662e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=307564544 dispatch_meta_bytes=224920 combine_payload_bytes=307564544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307789464 ingress_bytes=417665024 link_bytes=417665024 token_bytes=615354008 weight_movement_bytes=110100480 total_nvlink_bytes=725454488 weight_to_token_ratio=0.178922 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.263881708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61680 payload_rows_remote=53488 dup_rows=3856 dup_rows_remote=1866 dedup_ratio=0.058838 router_tokens_per_expert=[0, 4094, 0, 0, 0, 4, 4063, 4095, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 0, 997, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 11, 3738, 0, 0, 4096, 0, 0, 385, 0, 0, 0, 490, 0, 0, 0, 3355, 105, 0, 0, 0] payload_rows_per_dst=[8134, 7154, 8192, 7440, 8190, 7506, 8192, 6872] dup_rows_per_dst=[0, 0, 1990, 2, 0, 66, 1750, 48] combine_rows_per_src=[8134, 7154, 8192, 7440, 8190, 7506, 8192, 6872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338817024 dispatch_meta_bytes=224568 combine_payload_bytes=338817024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339041592 ingress_bytes=448917504 link_bytes=448917504 token_bytes=677858616 weight_movement_bytes=110100480 total_nvlink_bytes=787959096 weight_to_token_ratio=0.162424 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.807413737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=328882176 dispatch_meta_bytes=229744 combine_payload_bytes=328882176 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=329111920 ingress_bytes=549083136 link_bytes=549083136 token_bytes=657994096 weight_movement_bytes=220200960 total_nvlink_bytes=878195056 weight_to_token_ratio=0.334655 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.549458822e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383401984 dispatch_meta_bytes=221416 combine_payload_bytes=383401984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383623400 ingress_bytes=493502464 link_bytes=493502464 token_bytes=767025384 weight_movement_bytes=110100480 total_nvlink_bytes=877125864 weight_to_token_ratio=0.143542 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.582819521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62322 payload_rows_remote=54378 dup_rows=3214 dup_rows_remote=3082 dedup_ratio=0.049042 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 1, 3971, 1, 0, 4096, 0, 0, 384, 0, 0, 0, 420, 0, 0, 0, 3036, 160, 0, 0, 0] payload_rows_per_dst=[8182, 7440, 8192, 7862, 8192, 7944, 8192, 6318] dup_rows_per_dst=[0, 0, 1400, 0, 0, 132, 1608, 74] combine_rows_per_src=[8182, 7440, 8192, 7862, 8192, 7944, 8192, 6318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58860 payload_rows_remote=54684 dup_rows=6676 dup_rows_remote=5970 dedup_ratio=0.101868 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 3990, 4096, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3, 3853, 1, 0, 4096, 0, 0, 757, 0, 0, 0, 333, 0, 0, 0, 3186, 108, 0, 0, 0] payload_rows_per_dst=[6940, 7806, 8192, 8038, 8192, 7754, 4176, 7762] dup_rows_per_dst=[0, 980, 636, 0, 0, 38, 706, 4316] combine_rows_per_src=[6940, 7806, 8192, 8038, 8192, 7754, 4176, 7762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62534 payload_rows_remote=54344 dup_rows=3002 dup_rows_remote=3002 dedup_ratio=0.045807 router_tokens_per_expert=[0, 4095, 0, 0, 0, 1, 4036, 4095, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 1, 756, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 14, 3759, 0, 0, 4096, 0, 0, 321, 0, 0, 0, 297, 0, 0, 0, 3446, 110, 0, 0, 0] payload_rows_per_dst=[8074, 7340, 8192, 7918, 8190, 7580, 8192, 7048] dup_rows_per_dst=[0, 0, 1512, 0, 0, 190, 1236, 64] combine_rows_per_src=[8074, 7340, 8192, 7918, 8190, 7580, 8192, 7048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62712 payload_rows_remote=54842 dup_rows=2824 dup_rows_remote=2824 dedup_ratio=0.043091 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 13, 3897, 0, 0, 4096, 0, 0, 404, 0, 0, 0, 460, 0, 0, 0, 3163, 169, 0, 0, 0] payload_rows_per_dst=[8168, 7662, 8192, 7870, 8192, 7856, 8192, 6580] dup_rows_per_dst=[0, 0, 838, 0, 0, 174, 1728, 84] combine_rows_per_src=[8168, 7662, 8192, 7870, 8192, 7856, 8192, 6580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51254 payload_rows_remote=43062 dup_rows=14282 dup_rows_remote=8238 dedup_ratio=0.217926 router_tokens_per_expert=[0, 4095, 0, 0, 0, 1, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 551, 0, 0, 0, 2, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 10, 3903, 0, 0, 4096, 0, 0, 576, 0, 0, 0, 713, 0, 0, 1, 2798, 223, 0, 0, 0] payload_rows_per_dst=[2, 7840, 8192, 7894, 8192, 7830, 3112, 8192] dup_rows_per_dst=[0, 3886, 1104, 2854, 2, 82, 310, 6044] combine_rows_per_src=[2, 7840, 8192, 7894, 8192, 7830, 3112, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53292 payload_rows_remote=45100 dup_rows=12244 dup_rows_remote=4128 dedup_ratio=0.186829 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 23, 3769, 0, 0, 4096, 0, 0, 766, 0, 0, 0, 336, 0, 0, 0, 3181, 83, 0, 0, 0] payload_rows_per_dst=[8192, 7738, 890, 7818, 8192, 7828, 6148, 6486] dup_rows_per_dst=[8116, 0, 0, 0, 0, 2412, 1674, 42] combine_rows_per_src=[8192, 7738, 890, 7818, 8192, 7828, 6148, 6486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60742 payload_rows_remote=53052 dup_rows=4794 dup_rows_remote=4794 dedup_ratio=0.073151 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 106, 0, 51, 3915, 0, 0, 4096, 0, 0, 302, 0, 0, 0, 762, 0, 0, 4, 2744, 55, 0, 0, 0] payload_rows_per_dst=[8180, 7690, 7162, 7840, 8192, 7900, 8192, 5586] dup_rows_per_dst=[1548, 0, 852, 0, 2, 244, 2128, 20] combine_rows_per_src=[8180, 7690, 7162, 7840, 8192, 7900, 8192, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=389781504 dispatch_meta_bytes=229840 combine_payload_bytes=389781504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390011344 ingress_bytes=499881984 link_bytes=499881984 token_bytes=779792848 weight_movement_bytes=110100480 total_nvlink_bytes=889893328 weight_to_token_ratio=0.141192 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=391974912 dispatch_meta_bytes=242616 combine_payload_bytes=391974912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392217528 ingress_bytes=722276352 link_bytes=722276352 token_bytes=784192440 weight_movement_bytes=330301440 total_nvlink_bytes=1114493880 weight_to_token_ratio=0.421199 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.256157370e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=389537792 dispatch_meta_bytes=229384 combine_payload_bytes=389537792 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=389767176 ingress_bytes=609738752 link_bytes=609738752 token_bytes=779304968 weight_movement_bytes=220200960 total_nvlink_bytes=999505928 weight_to_token_ratio=0.282561 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060435975e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=393107456 dispatch_meta_bytes=230664 combine_payload_bytes=393107456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393338120 ingress_bytes=503207936 link_bytes=503207936 token_bytes=786445576 weight_movement_bytes=110100480 total_nvlink_bytes=896546056 weight_to_token_ratio=0.139998 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.751613642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60072 payload_rows_remote=52586 dup_rows=5464 dup_rows_remote=4502 dedup_ratio=0.083374 router_tokens_per_expert=[0, 2541, 0, 0, 0, 227, 0, 1, 1221, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, 0, 0, 0, 539, 3685, 0, 4096, 0, 893, 0, 0, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 4096, 0, 144, 0, 4086, 0, 0, 27, 4096, 34, 0, 3608, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[5544, 7362, 7486, 8192, 8192, 7888, 8192, 7216] dup_rows_per_dst=[304, 1576, 962, 1818, 0, 412, 390, 2] combine_rows_per_src=[5544, 7362, 7486, 8192, 8192, 7888, 8192, 7216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=308668416 dispatch_meta_bytes=205200 combine_payload_bytes=308668416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308873616 ingress_bytes=418768896 link_bytes=418768896 token_bytes=617542032 weight_movement_bytes=110100480 total_nvlink_bytes=727642512 weight_to_token_ratio=0.178288 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.283079858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380276736 dispatch_meta_bytes=231384 combine_payload_bytes=380276736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380508120 ingress_bytes=490377216 link_bytes=490377216 token_bytes=760784856 weight_movement_bytes=110100480 total_nvlink_bytes=870885336 weight_to_token_ratio=0.144720 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.528466318e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=323276800 dispatch_meta_bytes=196912 combine_payload_bytes=323276800 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=323473712 ingress_bytes=983879680 link_bytes=983879680 token_bytes=646750512 weight_movement_bytes=660602880 total_nvlink_bytes=1307353392 weight_to_token_ratio=1.021418 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.711128584e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376936448 dispatch_meta_bytes=228352 combine_payload_bytes=376936448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377164800 ingress_bytes=597137408 link_bytes=597137408 token_bytes=754101248 weight_movement_bytes=220200960 total_nvlink_bytes=974302208 weight_to_token_ratio=0.292005 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.038520165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58478 payload_rows_remote=50286 dup_rows=7058 dup_rows_remote=4538 dedup_ratio=0.107697 router_tokens_per_expert=[0, 2639, 0, 0, 0, 88, 0, 0, 1190, 0, 0, 0, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 579, 3503, 0, 4096, 0, 229, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 0, 0, 0, 4096, 0, 10, 0, 4095, 0, 0, 3, 4096, 8, 0, 3156, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5414, 6320, 7240, 8192, 8192, 8192, 8192, 6736] dup_rows_per_dst=[40, 1414, 924, 460, 0, 2520, 40, 1660] combine_rows_per_src=[5414, 6320, 7240, 8192, 8192, 8192, 8192, 6736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57340 payload_rows_remote=49148 dup_rows=8196 dup_rows_remote=8122 dedup_ratio=0.125061 router_tokens_per_expert=[0, 3293, 0, 0, 0, 97, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 754, 3495, 0, 4096, 0, 446, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 0, 0, 0, 0, 4096, 0, 30, 0, 4096, 0, 0, 0, 4096, 7, 0, 2309, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6708, 2524, 7212, 8192, 8192, 8192, 8192, 8128] dup_rows_per_dst=[72, 0, 1286, 896, 0, 1612, 74, 4256] combine_rows_per_src=[6708, 2524, 7212, 8192, 8192, 8192, 8192, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59180 payload_rows_remote=50988 dup_rows=6356 dup_rows_remote=6356 dedup_ratio=0.096985 router_tokens_per_expert=[0, 2861, 0, 0, 0, 214, 0, 0, 2075, 0, 0, 1, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 876, 3605, 0, 4094, 0, 217, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 4096, 0, 74, 0, 4092, 0, 0, 3, 4096, 27, 0, 2572, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6010, 7812, 7446, 8192, 8192, 8192, 8192, 5144] dup_rows_per_dst=[140, 3356, 1516, 458, 0, 686, 200, 0] combine_rows_per_src=[6010, 7812, 7446, 8192, 8192, 8192, 8192, 5144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60058 payload_rows_remote=51866 dup_rows=5478 dup_rows_remote=4980 dedup_ratio=0.083588 router_tokens_per_expert=[0, 2746, 0, 0, 0, 201, 0, 0, 1323, 0, 0, 0, 0, 0, 0, 3534, 0, 0, 0, 0, 0, 0, 968, 3732, 0, 4095, 0, 217, 0, 0, 3, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 4096, 0, 10, 0, 4091, 0, 0, 19, 4096, 19, 0, 3039, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5828, 7730, 7654, 8192, 8192, 8192, 8192, 6078] dup_rows_per_dst=[66, 1984, 1746, 498, 0, 1098, 86, 0] combine_rows_per_src=[5828, 7730, 7654, 8192, 8192, 8192, 8192, 6078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55406 payload_rows_remote=47694 dup_rows=10130 dup_rows_remote=8400 dedup_ratio=0.154572 router_tokens_per_expert=[0, 2972, 0, 0, 0, 113, 0, 0, 1142, 0, 0, 3, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 887, 3620, 0, 4095, 0, 203, 0, 0, 18, 33, 0, 0, 0, 0, 0, 1, 0, 0, 0, 686, 0, 0, 0, 0, 4096, 0, 47, 0, 4086, 0, 0, 2, 4096, 57, 0, 3035, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7712, 7486, 8192, 8192, 1372, 8190, 6070] dup_rows_per_dst=[6170, 1730, 1528, 506, 2, 0, 194, 0] combine_rows_per_src=[8192, 7712, 7486, 8192, 8192, 1372, 8190, 6070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50482 payload_rows_remote=42336 dup_rows=15054 dup_rows_remote=9566 dedup_ratio=0.229706 router_tokens_per_expert=[0, 3044, 0, 0, 0, 71, 0, 0, 860, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 549, 3721, 0, 4095, 0, 335, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 901, 0, 0, 0, 0, 4096, 0, 48, 0, 4093, 0, 0, 23, 4096, 14, 0, 3082, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6174, 1720, 7556, 2318, 8184, 8192, 8192, 8146] dup_rows_per_dst=[56, 0, 984, 296, 2, 8056, 172, 5488] combine_rows_per_src=[6174, 1720, 7556, 2318, 8184, 8192, 8192, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=360450048 dispatch_meta_bytes=219296 combine_payload_bytes=360450048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=360669344 ingress_bytes=580651008 link_bytes=580651008 token_bytes=721119392 weight_movement_bytes=220200960 total_nvlink_bytes=941320352 weight_to_token_ratio=0.305360 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.009847604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51348 payload_rows_remote=43156 dup_rows=14188 dup_rows_remote=7562 dedup_ratio=0.216492 router_tokens_per_expert=[0, 3209, 0, 0, 0, 104, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 545, 3671, 0, 4096, 0, 179, 0, 0, 15, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 466, 0, 0, 0, 0, 4096, 0, 26, 0, 4078, 0, 0, 0, 4096, 30, 0, 3306, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8092, 4118, 8192, 8192, 932, 7018, 6612] dup_rows_per_dst=[6626, 6204, 392, 942, 4, 0, 20, 0] combine_rows_per_src=[8192, 8092, 4118, 8192, 8192, 932, 7018, 6612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56734 payload_rows_remote=48542 dup_rows=8802 dup_rows_remote=8674 dedup_ratio=0.134308 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 1827, 0, 0, 67, 0, 1324, 0, 0, 0, 0, 0, 0, 25, 0, 0, 21, 18, 4096, 0, 0, 0, 0, 4096, 102, 0, 1347, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 3734, 0, 0, 0, 4077] payload_rows_per_dst=[8174, 8046, 8192, 8192, 8132, 7722, 8192, 84] dup_rows_per_dst=[3654, 2204, 128, 204, 2612, 0, 0, 0] combine_rows_per_src=[8174, 8046, 8192, 8192, 8132, 7722, 8192, 84] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=352292864 dispatch_meta_bytes=229080 combine_payload_bytes=352292864 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=352521944 ingress_bytes=682594304 link_bytes=682594304 token_bytes=704814808 weight_movement_bytes=330301440 total_nvlink_bytes=1035116248 weight_to_token_ratio=0.468636 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.187143762e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=365481984 dispatch_meta_bytes=229376 combine_payload_bytes=365481984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365711360 ingress_bytes=475582464 link_bytes=475582464 token_bytes=731193344 weight_movement_bytes=110100480 total_nvlink_bytes=841293824 weight_to_token_ratio=0.150576 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.271161248e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371775488 dispatch_meta_bytes=227384 combine_payload_bytes=371775488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=372002872 ingress_bytes=591976448 link_bytes=591976448 token_bytes=743778360 weight_movement_bytes=220200960 total_nvlink_bytes=963979320 weight_to_token_ratio=0.296057 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029544406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=341870592 dispatch_meta_bytes=224376 combine_payload_bytes=341870592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342094968 ingress_bytes=562071552 link_bytes=562071552 token_bytes=683965560 weight_movement_bytes=220200960 total_nvlink_bytes=904166520 weight_to_token_ratio=0.321947 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.775348738e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=303464448 dispatch_meta_bytes=207608 combine_payload_bytes=303464448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=303672056 ingress_bytes=413564928 link_bytes=413564928 token_bytes=607136504 weight_movement_bytes=110100480 total_nvlink_bytes=717236984 weight_to_token_ratio=0.181344 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.192574295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=309342208 dispatch_meta_bytes=202872 combine_payload_bytes=309342208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309545080 ingress_bytes=419442688 link_bytes=419442688 token_bytes=618887288 weight_movement_bytes=110100480 total_nvlink_bytes=728987768 weight_to_token_ratio=0.177901 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.294798209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=347949056 dispatch_meta_bytes=228864 combine_payload_bytes=347949056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348177920 ingress_bytes=458049536 link_bytes=458049536 token_bytes=696126976 weight_movement_bytes=110100480 total_nvlink_bytes=806227456 weight_to_token_ratio=0.158161 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.966234793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57024 payload_rows_remote=48922 dup_rows=8512 dup_rows_remote=8512 dedup_ratio=0.129883 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 1573, 0, 0, 40, 0, 2387, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 4096, 74, 0, 1002, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3171, 0, 0, 0, 4093] payload_rows_per_dst=[3148, 4850, 8192, 8192, 8156, 8102, 8192, 8192] dup_rows_per_dst=[0, 4, 8, 148, 1998, 0, 0, 6354] combine_rows_per_src=[3148, 4850, 8192, 8192, 8156, 8102, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52216 payload_rows_remote=44024 dup_rows=13320 dup_rows_remote=13320 dedup_ratio=0.203247 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1939, 0, 0, 15, 0, 2191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 79, 0, 724, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 3258, 0, 0, 0, 4096] payload_rows_per_dst=[3878, 4412, 8192, 8192, 3158, 8000, 8192, 8192] dup_rows_per_dst=[0, 0, 14, 158, 556, 0, 0, 12592] combine_rows_per_src=[3878, 4412, 8192, 8192, 3158, 8000, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56806 payload_rows_remote=48662 dup_rows=8730 dup_rows_remote=5694 dedup_ratio=0.133209 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 1553, 0, 0, 46, 0, 1736, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 107, 0, 1530, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3466, 0, 0, 0, 4088] payload_rows_per_dst=[6246, 6258, 8192, 8192, 8144, 7674, 8192, 3908] dup_rows_per_dst=[1218, 1622, 18, 214, 3036, 0, 0, 2622] combine_rows_per_src=[6246, 6258, 8192, 8192, 8144, 7674, 8192, 3908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56794 payload_rows_remote=48602 dup_rows=8742 dup_rows_remote=8552 dedup_ratio=0.133392 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 1981, 0, 0, 20, 0, 2071, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 1, 4096, 0, 0, 0, 0, 4096, 95, 0, 686, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3556, 0, 0, 0, 4083] payload_rows_per_dst=[8180, 8096, 8192, 8192, 8114, 7716, 8192, 112] dup_rows_per_dst=[3962, 3198, 42, 190, 1350, 0, 0, 0] combine_rows_per_src=[8180, 8096, 8192, 8192, 8114, 7716, 8192, 112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=350672896 dispatch_meta_bytes=229736 combine_payload_bytes=350672896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=350902632 ingress_bytes=570873856 link_bytes=570873856 token_bytes=701575528 weight_movement_bytes=220200960 total_nvlink_bytes=921776488 weight_to_token_ratio=0.313866 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.928435282e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56858 payload_rows_remote=48730 dup_rows=8678 dup_rows_remote=5182 dedup_ratio=0.132416 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 1600, 0, 0, 112, 0, 1921, 0, 0, 0, 0, 0, 0, 38, 0, 0, 1, 2, 4096, 0, 0, 0, 0, 4096, 67, 0, 915, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 3779, 0, 0, 0, 4080] payload_rows_per_dst=[8170, 8128, 8192, 8192, 8158, 7602, 8192, 224] dup_rows_per_dst=[3200, 3496, 82, 134, 1766, 0, 0, 0] combine_rows_per_src=[8170, 8128, 8192, 8192, 8158, 7602, 8192, 224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47586 payload_rows_remote=39394 dup_rows=17950 dup_rows_remote=2964 dedup_ratio=0.273895 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 2132, 0, 0, 36, 0, 1862, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 4096, 68, 0, 855, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3445, 0, 0, 0, 4088] payload_rows_per_dst=[4272, 3784, 7848, 5680, 1710, 7908, 8192, 8192] dup_rows_per_dst=[0, 12, 16, 100, 0, 2836, 0, 14986] combine_rows_per_src=[4272, 3784, 7848, 5680, 1710, 7908, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=315564032 dispatch_meta_bytes=229376 combine_payload_bytes=315564032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315793408 ingress_bytes=425664512 link_bytes=425664512 token_bytes=631357440 weight_movement_bytes=110100480 total_nvlink_bytes=741457920 weight_to_token_ratio=0.174387 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.403005961e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=348809216 dispatch_meta_bytes=217424 combine_payload_bytes=348809216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=349026640 ingress_bytes=569010176 link_bytes=569010176 token_bytes=697835856 weight_movement_bytes=220200960 total_nvlink_bytes=918036816 weight_to_token_ratio=0.315548 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.896022822e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=348379136 dispatch_meta_bytes=228616 combine_payload_bytes=348379136 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=348607752 ingress_bytes=678680576 link_bytes=678680576 token_bytes=696986888 weight_movement_bytes=330301440 total_nvlink_bytes=1027288328 weight_to_token_ratio=0.473899 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.180337145e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63784 payload_rows_remote=55840 dup_rows=1752 dup_rows_remote=1696 dedup_ratio=0.026733 router_tokens_per_expert=[0, 0, 0, 392, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 3971, 0, 0, 0, 0, 0, 28, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3970, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 2056, 145, 0] payload_rows_per_dst=[8192, 8192, 7944, 7888, 8192, 8192, 7940, 7244] dup_rows_per_dst=[824, 2, 56, 0, 202, 0, 0, 668] combine_rows_per_src=[8192, 8192, 7944, 7888, 8192, 8192, 7940, 7244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56088 payload_rows_remote=47898 dup_rows=9448 dup_rows_remote=6204 dedup_ratio=0.144165 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 1622, 0, 0, 53, 0, 2212, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 2, 4096, 0, 0, 0, 1, 4096, 53, 0, 891, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3448, 0, 0, 0, 4091] payload_rows_per_dst=[8190, 8066, 8192, 8192, 7202, 7976, 8192, 78] dup_rows_per_dst=[3244, 3360, 36, 1364, 1444, 0, 0, 0] combine_rows_per_src=[8190, 8066, 8192, 8192, 7202, 7976, 8192, 78] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=349296640 dispatch_meta_bytes=215648 combine_payload_bytes=349296640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349512288 ingress_bytes=459397120 link_bytes=459397120 token_bytes=698808928 weight_movement_bytes=110100480 total_nvlink_bytes=808909408 weight_to_token_ratio=0.157554 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.989671495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282376192 dispatch_meta_bytes=169432 combine_payload_bytes=282376192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=282545624 ingress_bytes=612677632 link_bytes=612677632 token_bytes=564921816 weight_movement_bytes=330301440 total_nvlink_bytes=895223256 weight_to_token_ratio=0.584685 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065547170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=400261120 dispatch_meta_bytes=230144 combine_payload_bytes=400261120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400491264 ingress_bytes=510361600 link_bytes=510361600 token_bytes=800752384 weight_movement_bytes=110100480 total_nvlink_bytes=910852864 weight_to_token_ratio=0.137496 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.876027625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64422 payload_rows_remote=56230 dup_rows=1114 dup_rows_remote=1114 dedup_ratio=0.016998 router_tokens_per_expert=[0, 0, 11, 194, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 8, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 3313, 175, 0] payload_rows_per_dst=[8192, 8192, 7892, 8022, 8192, 8192, 7998, 7742] dup_rows_per_dst=[412, 0, 16, 0, 218, 0, 0, 468] combine_rows_per_src=[8192, 8192, 7892, 8022, 8192, 8192, 7998, 7742] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=343332864 dispatch_meta_bytes=216408 combine_payload_bytes=343332864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343549272 ingress_bytes=453433344 link_bytes=453433344 token_bytes=686882136 weight_movement_bytes=110100480 total_nvlink_bytes=796982616 weight_to_token_ratio=0.160290 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.885951622e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64568 payload_rows_remote=56448 dup_rows=968 dup_rows_remote=968 dedup_ratio=0.014771 router_tokens_per_expert=[0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 24, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 3288, 104, 0] payload_rows_per_dst=[8192, 8192, 7960, 7934, 8192, 8192, 8120, 7786] dup_rows_per_dst=[316, 0, 48, 0, 248, 0, 0, 356] combine_rows_per_src=[8192, 8192, 7960, 7934, 8192, 8192, 8120, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64052 payload_rows_remote=55860 dup_rows=1484 dup_rows_remote=1306 dedup_ratio=0.022644 router_tokens_per_expert=[0, 0, 6, 435, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 3983, 0, 0, 0, 0, 0, 25, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 709, 0, 0, 0, 0, 2945, 162, 0] payload_rows_per_dst=[8192, 8192, 7968, 8010, 8192, 8190, 8032, 7276] dup_rows_per_dst=[898, 2, 50, 0, 178, 0, 0, 356] combine_rows_per_src=[8192, 8192, 7968, 8010, 8192, 8190, 8032, 7276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403056640 dispatch_meta_bytes=229376 combine_payload_bytes=403056640 prefetch_bytes=110100480 reduce_bytes=660602880 egress_bytes=403286016 ingress_bytes=1173760000 link_bytes=1173760000 token_bytes=806342656 weight_movement_bytes=770703360 total_nvlink_bytes=1577046016 weight_to_token_ratio=0.955801 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=2.041361690e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64252 payload_rows_remote=56296 dup_rows=1284 dup_rows_remote=1284 dedup_ratio=0.019592 router_tokens_per_expert=[0, 0, 16, 270, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 31, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, 2999, 158, 0] payload_rows_per_dst=[8192, 8192, 8010, 7956, 8192, 8190, 7986, 7534] dup_rows_per_dst=[580, 0, 62, 0, 222, 0, 0, 420] combine_rows_per_src=[8192, 8192, 8010, 7956, 8192, 8190, 7986, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63566 payload_rows_remote=55374 dup_rows=1970 dup_rows_remote=1964 dedup_ratio=0.030060 router_tokens_per_expert=[0, 0, 12, 561, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 2, 3940, 0, 0, 0, 0, 0, 107, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3964, 0, 0, 3, 0, 0, 427, 0, 0, 0, 0, 3102, 191, 0] payload_rows_per_dst=[8192, 8192, 7882, 7928, 8192, 8192, 7934, 7054] dup_rows_per_dst=[1178, 6, 216, 0, 184, 0, 0, 386] combine_rows_per_src=[8192, 8192, 7882, 7928, 8192, 8192, 7934, 7054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56302 payload_rows_remote=49318 dup_rows=9234 dup_rows_remote=5508 dedup_ratio=0.140900 router_tokens_per_expert=[0, 0, 1, 928, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 23, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 924, 0, 0, 0, 0, 2201, 156, 0] payload_rows_per_dst=[2510, 8180, 8016, 7964, 6382, 8192, 8074, 6984] dup_rows_per_dst=[390, 0, 46, 0, 234, 3350, 1488, 3726] combine_rows_per_src=[2510, 8180, 8016, 7964, 6382, 8192, 8074, 6984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=404619264 dispatch_meta_bytes=229664 combine_payload_bytes=404619264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404848928 ingress_bytes=514719744 link_bytes=514719744 token_bytes=809468192 weight_movement_bytes=110100480 total_nvlink_bytes=919568672 weight_to_token_ratio=0.136016 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.951822917e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=400404480 dispatch_meta_bytes=228664 combine_payload_bytes=400404480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400633144 ingress_bytes=510504960 link_bytes=510504960 token_bytes=801037624 weight_movement_bytes=110100480 total_nvlink_bytes=911138104 weight_to_token_ratio=0.137447 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.878520891e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403529728 dispatch_meta_bytes=230320 combine_payload_bytes=403529728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403760048 ingress_bytes=513630208 link_bytes=513630208 token_bytes=807289776 weight_movement_bytes=110100480 total_nvlink_bytes=917390256 weight_to_token_ratio=0.136383 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.932874094e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61764 payload_rows_remote=53572 dup_rows=3772 dup_rows_remote=3772 dedup_ratio=0.057556 router_tokens_per_expert=[1753, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 0, 0, 2382, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 4084, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4089, 0, 147, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4764, 7908, 8170, 8178, 8168] dup_rows_per_dst=[3468, 8, 0, 0, 0, 0, 296, 0] combine_rows_per_src=[8192, 8192, 8192, 4764, 7908, 8170, 8178, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=396920832 dispatch_meta_bytes=229352 combine_payload_bytes=396920832 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=397150184 ingress_bytes=507021312 link_bytes=507021312 token_bytes=794071016 weight_movement_bytes=110100480 total_nvlink_bytes=904171496 weight_to_token_ratio=0.138653 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.817934522e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60248 payload_rows_remote=52056 dup_rows=5288 dup_rows_remote=4204 dedup_ratio=0.080688 router_tokens_per_expert=[0, 0, 1, 538, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3957, 0, 0, 0, 0, 0, 66, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 860, 0, 0, 0, 0, 2783, 125, 0] payload_rows_per_dst=[8192, 8192, 7928, 7886, 8192, 4846, 7964, 7048] dup_rows_per_dst=[1084, 2, 1442, 1948, 324, 0, 0, 488] combine_rows_per_src=[8192, 8192, 7928, 7886, 8192, 4846, 7964, 7048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=353511424 dispatch_meta_bytes=219304 combine_payload_bytes=353511424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353730728 ingress_bytes=463611904 link_bytes=463611904 token_bytes=707242152 weight_movement_bytes=110100480 total_nvlink_bytes=817342632 weight_to_token_ratio=0.155676 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.062973521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60524 payload_rows_remote=52334 dup_rows=5012 dup_rows_remote=5012 dedup_ratio=0.076477 router_tokens_per_expert=[1725, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2382, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 124, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3450, 8192, 8192, 8192, 7934, 8190, 8186, 8188] dup_rows_per_dst=[0, 0, 0, 4762, 0, 0, 250, 0] combine_rows_per_src=[3450, 8192, 8192, 8192, 7934, 8190, 8186, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384004096 dispatch_meta_bytes=229376 combine_payload_bytes=384004096 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=384233472 ingress_bytes=494104576 link_bytes=494104576 token_bytes=768237568 weight_movement_bytes=110100480 total_nvlink_bytes=878338048 weight_to_token_ratio=0.143316 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.593291239e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=373137408 dispatch_meta_bytes=225040 combine_payload_bytes=373137408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373362448 ingress_bytes=483237888 link_bytes=483237888 token_bytes=746499856 weight_movement_bytes=110100480 total_nvlink_bytes=856600336 weight_to_token_ratio=0.147489 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.404301662e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60560 payload_rows_remote=52386 dup_rows=4976 dup_rows_remote=4832 dedup_ratio=0.075928 router_tokens_per_expert=[1815, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2417, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 65, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3630, 8192, 8192, 8192, 7796, 8192, 8174, 8192] dup_rows_per_dst=[0, 0, 0, 4832, 0, 0, 144, 0] combine_rows_per_src=[3630, 8192, 8192, 8192, 7796, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375130112 dispatch_meta_bytes=229384 combine_payload_bytes=375130112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375359496 ingress_bytes=485230592 link_bytes=485230592 token_bytes=750489608 weight_movement_bytes=110100480 total_nvlink_bytes=860590088 weight_to_token_ratio=0.146705 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.438958062e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61134 payload_rows_remote=53302 dup_rows=4402 dup_rows_remote=4402 dedup_ratio=0.067169 router_tokens_per_expert=[2270, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 1894, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 4086, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4916, 8192, 8192, 7458, 7832, 8188, 8184, 8172] dup_rows_per_dst=[912, 2, 2, 3214, 0, 0, 272, 0] combine_rows_per_src=[4916, 8192, 8192, 7458, 7832, 8188, 8184, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60264 payload_rows_remote=56772 dup_rows=5272 dup_rows_remote=5272 dedup_ratio=0.080444 router_tokens_per_expert=[2521, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4093, 0, 0, 1746, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 0, 0, 0, 4087, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4092, 0, 123, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3492, 7658, 8180, 8184, 8174] dup_rows_per_dst=[5018, 0, 6, 0, 0, 0, 248, 0] combine_rows_per_src=[8192, 8192, 8192, 3492, 7658, 8180, 8184, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6926481056 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3464365728 ingress_bytes=5113622528 link_bytes=5113622528 total_nvlink_bytes=8577988256 weight_to_token_ratio=0.238434 prefetch_share_of_ingress=0.172247 dedup_ratio=0.145956 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.893430623e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60884 payload_rows_remote=52692 dup_rows=4652 dup_rows_remote=4644 dedup_ratio=0.070984 router_tokens_per_expert=[2259, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 1862, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 4065, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 4083, 1, 163, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 3724, 8098, 8182, 8166, 8140] dup_rows_per_dst=[4312, 8, 2, 0, 0, 0, 328, 2] combine_rows_per_src=[8192, 8192, 8190, 3724, 8098, 8182, 8166, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375502848 dispatch_meta_bytes=228872 combine_payload_bytes=375502848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375731720 ingress_bytes=595703808 link_bytes=595703808 token_bytes=751234568 weight_movement_bytes=220200960 total_nvlink_bytes=971435528 weight_to_token_ratio=0.293119 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036026899e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6952048568 prefetch_bytes=1101004800 reduce_bytes=1431306240 weight_movement_bytes=2532311040 egress_bytes=3477145528 ingress_bytes=6007214080 link_bytes=6007214080 total_nvlink_bytes=9484359608 weight_to_token_ratio=0.364254 prefetch_share_of_ingress=0.183280 dedup_ratio=0.143677 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=13 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.044753330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382068736 dispatch_meta_bytes=230816 combine_payload_bytes=382068736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=382299552 ingress_bytes=712370176 link_bytes=712370176 token_bytes=764368288 weight_movement_bytes=330301440 total_nvlink_bytes=1094669728 weight_to_token_ratio=0.432123 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.238928901e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59976 payload_rows_remote=51802 dup_rows=5560 dup_rows_remote=5378 dedup_ratio=0.084839 router_tokens_per_expert=[1708, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 2416, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 4087, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 1, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3416, 8174, 8192, 7904, 7740, 8188, 8188, 8174] dup_rows_per_dst=[0, 0, 0, 4364, 0, 50, 964, 182] combine_rows_per_src=[3416, 8174, 8192, 7904, 7740, 8188, 8188, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406941696 dispatch_meta_bytes=248176 combine_payload_bytes=406941696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407189872 ingress_bytes=627142656 link_bytes=627142656 token_bytes=814131568 weight_movement_bytes=220200960 total_nvlink_bytes=1034332528 weight_to_token_ratio=0.270473 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090704226e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377696256 dispatch_meta_bytes=229344 combine_payload_bytes=377696256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377925600 ingress_bytes=487796736 link_bytes=487796736 token_bytes=755621856 weight_movement_bytes=110100480 total_nvlink_bytes=865722336 weight_to_token_ratio=0.145708 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.483587527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6853168904 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3427710728 ingress_bytes=5737568256 link_bytes=5737568256 total_nvlink_bytes=9165278984 weight_to_token_ratio=0.337378 prefetch_share_of_ingress=0.172705 dedup_ratio=0.159637 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.978574865e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59328 payload_rows_remote=51136 dup_rows=6208 dup_rows_remote=2562 dedup_ratio=0.094727 router_tokens_per_expert=[1837, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 2318, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 3970, 0, 0, 0, 4072, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 4636, 8146, 8192, 5648, 8144] dup_rows_per_dst=[3646, 0, 6, 2, 2424, 4, 126, 0] combine_rows_per_src=[8192, 8192, 8178, 4636, 8146, 8192, 5648, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7018942992 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3510608400 ingress_bytes=5820444672 link_bytes=5820444672 total_nvlink_bytes=9331053072 weight_to_token_ratio=0.329410 prefetch_share_of_ingress=0.151329 dedup_ratio=0.137994 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.012271058e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7142673384 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3572493288 ingress_bytes=5221687296 link_bytes=5221687296 total_nvlink_bytes=8794180584 weight_to_token_ratio=0.231217 prefetch_share_of_ingress=0.210852 dedup_ratio=0.138928 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.081373028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6800873736 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3401564424 ingress_bytes=5271017472 link_bytes=5271017472 total_nvlink_bytes=8672581896 weight_to_token_ratio=0.275216 prefetch_share_of_ingress=0.208879 dedup_ratio=0.158627 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051838 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.167166317e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371316736 dispatch_meta_bytes=228720 combine_payload_bytes=371316736 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=371545456 ingress_bytes=591517696 link_bytes=591517696 token_bytes=742862192 weight_movement_bytes=220200960 total_nvlink_bytes=963063152 weight_to_token_ratio=0.296422 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028746561e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=366542848 dispatch_meta_bytes=214792 combine_payload_bytes=366542848 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=366757640 ingress_bytes=806944768 link_bytes=806944768 token_bytes=733300488 weight_movement_bytes=440401920 total_nvlink_bytes=1173702408 weight_to_token_ratio=0.600575 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.403409671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6143555392 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3072827200 ingress_bytes=5052536832 link_bytes=5052536832 total_nvlink_bytes=8125364032 weight_to_token_ratio=0.322583 prefetch_share_of_ingress=0.239702 dedup_ratio=0.230920 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7747 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.787192550e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6529514048 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3265823296 ingress_bytes=5465700352 link_bytes=5465700352 total_nvlink_bytes=8731523648 weight_to_token_ratio=0.337239 prefetch_share_of_ingress=0.181295 dedup_ratio=0.184546 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.505751865e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-29 23:43:06] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 241.5 | throughput per GPU (TFLOP/s/GPU): 511.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.651175E+00 | loss scale: 1.0 | grad norm: 4.399 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45338 payload_rows_remote=39636 dup_rows=20198 dup_rows_remote=18204 dedup_ratio=0.308197 router_tokens_per_expert=[1472, 1069, 469, 199, 25, 758, 415, 27, 618, 397, 197, 191, 7, 878, 316, 794, 691, 801, 20, 32, 516, 500, 622, 240, 200, 361, 526, 384, 1024, 207, 613, 4, 741, 990, 508, 10, 862, 62, 846, 618, 392, 512, 430, 755, 8, 832, 57, 512, 659, 952, 291, 1082, 206, 579, 373, 593, 1121, 783, 440, 574, 906, 1253, 243, 5] payload_rows_per_dst=[5158, 5054, 5936, 5702, 5494, 6080, 6010, 5904] dup_rows_per_dst=[2230, 1742, 2388, 1994, 2722, 3158, 3460, 2504] combine_rows_per_src=[5158, 5054, 5936, 5702, 5494, 6080, 6010, 5904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44224 payload_rows_remote=38620 dup_rows=21312 dup_rows_remote=18268 dedup_ratio=0.325195 router_tokens_per_expert=[1050, 1147, 351, 618, 11, 979, 807, 32, 514, 492, 457, 250, 1, 934, 459, 532, 485, 581, 25, 27, 866, 110, 600, 405, 355, 494, 593, 647, 929, 158, 569, 4, 643, 619, 781, 16, 833, 0, 1177, 255, 451, 551, 413, 610, 8, 452, 248, 424, 518, 831, 741, 776, 200, 687, 1189, 338, 923, 1060, 666, 280, 615, 554, 422, 5] payload_rows_per_dst=[5544, 5144, 5880, 5382, 5604, 5430, 6072, 5168] dup_rows_per_dst=[2346, 2134, 2418, 2116, 3044, 2598, 4488, 2168] combine_rows_per_src=[5544, 5144, 5880, 5382, 5604, 5430, 6072, 5168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44040 payload_rows_remote=38984 dup_rows=21496 dup_rows_remote=20036 dedup_ratio=0.328003 router_tokens_per_expert=[1392, 875, 1316, 433, 10, 833, 485, 27, 820, 521, 349, 466, 3, 807, 624, 825, 581, 609, 35, 33, 540, 424, 739, 297, 345, 651, 723, 346, 768, 435, 664, 4, 578, 721, 853, 13, 1291, 25, 327, 327, 433, 698, 623, 821, 8, 831, 42, 480, 469, 585, 725, 349, 249, 938, 469, 358, 662, 542, 419, 298, 543, 738, 361, 12] payload_rows_per_dst=[6278, 5878, 5056, 6892, 4294, 5970, 5330, 4342] dup_rows_per_dst=[4464, 2952, 1460, 3562, 1394, 3226, 2954, 1484] combine_rows_per_src=[6278, 5878, 5056, 6892, 4294, 5970, 5330, 4342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=39474 dup_rows=21130 dup_rows_remote=18960 dedup_ratio=0.322418 router_tokens_per_expert=[1481, 672, 479, 805, 34, 807, 282, 30, 299, 357, 690, 370, 5, 733, 449, 699, 1024, 591, 70, 37, 540, 410, 634, 543, 270, 401, 540, 355, 1023, 230, 680, 2, 366, 722, 429, 19, 857, 8, 1134, 402, 449, 473, 414, 701, 14, 634, 69, 797, 818, 884, 687, 565, 217, 836, 551, 458, 764, 671, 681, 472, 790, 934, 408, 2] payload_rows_per_dst=[4730, 5300, 6686, 5334, 5304, 4932, 5972, 6148] dup_rows_per_dst=[1488, 1904, 3974, 1668, 2570, 2170, 4060, 3296] combine_rows_per_src=[4730, 5300, 6686, 5334, 5304, 4932, 5972, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44278 payload_rows_remote=38368 dup_rows=21258 dup_rows_remote=17626 dedup_ratio=0.324371 router_tokens_per_expert=[1152, 964, 246, 688, 119, 669, 676, 107, 470, 436, 688, 185, 0, 1045, 726, 548, 907, 584, 96, 51, 747, 694, 715, 489, 246, 1037, 429, 496, 1080, 206, 535, 4, 637, 740, 1170, 8, 844, 25, 969, 171, 338, 697, 301, 748, 1, 647, 88, 520, 416, 429, 735, 270, 200, 773, 615, 393, 1011, 682, 465, 315, 962, 354, 188, 21] payload_rows_per_dst=[5216, 5676, 6752, 5702, 5706, 4778, 4538, 5910] dup_rows_per_dst=[1722, 2520, 4118, 2364, 3422, 1902, 1578, 3632] combine_rows_per_src=[5216, 5676, 6752, 5702, 5706, 4778, 4538, 5910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44050 payload_rows_remote=39000 dup_rows=21486 dup_rows_remote=19656 dedup_ratio=0.327850 router_tokens_per_expert=[706, 811, 623, 1021, 2, 1201, 565, 32, 892, 677, 877, 531, 1, 1052, 529, 1022, 832, 443, 12, 2, 681, 170, 742, 386, 145, 351, 613, 413, 795, 211, 277, 1, 850, 1396, 644, 33, 1145, 0, 900, 345, 973, 435, 334, 432, 26, 442, 162, 629, 777, 576, 132, 785, 102, 485, 340, 707, 525, 530, 449, 266, 540, 633, 554, 5] payload_rows_per_dst=[5670, 6688, 5790, 4420, 6254, 4866, 5050, 5312] dup_rows_per_dst=[2840, 4474, 2158, 1192, 4372, 2000, 1830, 2620] combine_rows_per_src=[5670, 6688, 5790, 4420, 6254, 4866, 5050, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=38630 dup_rows=20822 dup_rows_remote=17304 dedup_ratio=0.317719 router_tokens_per_expert=[1226, 751, 587, 482, 53, 1061, 485, 25, 489, 465, 352, 254, 4, 668, 458, 570, 576, 393, 13, 26, 685, 425, 765, 531, 275, 959, 546, 740, 847, 176, 672, 5, 1016, 687, 745, 5, 794, 2, 740, 397, 350, 839, 404, 707, 0, 638, 72, 541, 733, 397, 652, 669, 118, 607, 518, 378, 970, 1142, 599, 354, 825, 977, 324, 4] payload_rows_per_dst=[6084, 5326, 5180, 6602, 5330, 6004, 4348, 5840] dup_rows_per_dst=[3518, 2152, 1648, 3426, 2648, 3038, 1782, 2610] combine_rows_per_src=[6084, 5326, 5180, 6602, 5330, 6004, 4348, 5840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44806 payload_rows_remote=39016 dup_rows=20730 dup_rows_remote=17928 dedup_ratio=0.316315 router_tokens_per_expert=[962, 950, 298, 204, 18, 1184, 487, 21, 984, 479, 704, 310, 5, 587, 405, 822, 786, 379, 88, 55, 466, 246, 663, 363, 261, 694, 416, 575, 979, 255, 685, 4, 716, 649, 855, 14, 721, 1, 762, 367, 624, 612, 426, 756, 1, 898, 63, 536, 686, 804, 445, 507, 183, 630, 471, 631, 776, 1312, 301, 733, 805, 584, 543, 21] payload_rows_per_dst=[5606, 5790, 4836, 6284, 5108, 6070, 5266, 5846] dup_rows_per_dst=[2642, 2802, 1256, 2896, 2270, 3314, 2798, 2752] combine_rows_per_src=[5606, 5790, 4836, 6284, 5108, 6070, 5266, 5846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284110848 dispatch_meta_bytes=231360 combine_payload_bytes=284110848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284342208 ingress_bytes=394211328 link_bytes=394211328 token_bytes=568453056 weight_movement_bytes=110100480 total_nvlink_bytes=678553536 weight_to_token_ratio=0.193684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.855983360e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276828160 dispatch_meta_bytes=227552 combine_payload_bytes=276828160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277055712 ingress_bytes=497029120 link_bytes=497029120 token_bytes=553883872 weight_movement_bytes=220200960 total_nvlink_bytes=774084832 weight_to_token_ratio=0.397558 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.644153869e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=236080 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279673392 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559110704 weight_movement_bytes=110100480 total_nvlink_bytes=669211184 weight_to_token_ratio=0.196921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=5888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282949632 dispatch_meta_bytes=233736 combine_payload_bytes=282949632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283183368 ingress_bytes=393050112 link_bytes=393050112 token_bytes=566133000 weight_movement_bytes=110100480 total_nvlink_bytes=676233480 weight_to_token_ratio=0.194478 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=5632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.835787904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275021824 dispatch_meta_bytes=223976 combine_payload_bytes=275021824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=275245800 ingress_bytes=495222784 link_bytes=495222784 token_bytes=550267624 weight_movement_bytes=220200960 total_nvlink_bytes=770468584 weight_to_token_ratio=0.400171 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.612738715e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=234624 combine_payload_bytes=279552000 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279786624 ingress_bytes=719953920 link_bytes=719953920 token_bytes=559338624 weight_movement_bytes=440401920 total_nvlink_bytes=999740544 weight_to_token_ratio=0.787362 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252118279e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276899840 dispatch_meta_bytes=223736 combine_payload_bytes=276899840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277123576 ingress_bytes=497100800 link_bytes=497100800 token_bytes=554023416 weight_movement_bytes=220200960 total_nvlink_bytes=774224376 weight_to_token_ratio=0.397458 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.645400502e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=227776 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279894464 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559561152 weight_movement_bytes=110100480 total_nvlink_bytes=669661632 weight_to_token_ratio=0.196762 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51386 payload_rows_remote=44412 dup_rows=14150 dup_rows_remote=12438 dedup_ratio=0.215912 router_tokens_per_expert=[0, 0, 0, 289, 1370, 0, 327, 0, 2630, 14, 60, 0, 1, 3146, 0, 1049, 125, 1, 0, 0, 252, 490, 0, 0, 0, 0, 158, 0, 41, 16, 0, 870, 0, 0, 0, 0, 178, 6, 0, 909, 0, 72, 315, 0, 0, 0, 2591, 0, 0, 1631, 2814, 0, 1990, 342, 0, 0, 2319, 0, 461, 3250, 4, 2343, 2686, 18] payload_rows_per_dst=[7116, 6172, 6078, 6840, 6974, 7416, 4048, 6742] dup_rows_per_dst=[2484, 1336, 1030, 1622, 1712, 3178, 270, 2518] combine_rows_per_src=[7116, 6172, 6078, 6840, 6974, 7416, 4048, 6742] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53524 payload_rows_remote=46606 dup_rows=12012 dup_rows_remote=10992 dedup_ratio=0.183289 router_tokens_per_expert=[0, 0, 1, 188, 1045, 0, 407, 0, 2990, 12, 149, 0, 18, 3338, 0, 1169, 120, 0, 0, 2, 99, 796, 0, 0, 3, 0, 45, 1, 7, 14, 0, 561, 0, 0, 0, 0, 139, 2, 0, 450, 0, 18, 302, 4, 0, 0, 2243, 0, 0, 1739, 3270, 0, 1771, 173, 0, 0, 2023, 0, 378, 3741, 3, 2262, 3262, 23] payload_rows_per_dst=[7318, 6840, 7026, 6918, 7602, 6962, 5888, 4970] dup_rows_per_dst=[2504, 1836, 1532, 1020, 1062, 2218, 1478, 362] combine_rows_per_src=[7318, 6840, 7026, 6918, 7602, 6962, 5888, 4970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51458 payload_rows_remote=45186 dup_rows=14078 dup_rows_remote=13218 dedup_ratio=0.214813 router_tokens_per_expert=[0, 0, 0, 229, 1252, 0, 376, 0, 3020, 34, 180, 0, 7, 2650, 0, 1438, 151, 0, 0, 0, 275, 313, 0, 0, 0, 0, 57, 0, 0, 16, 0, 1060, 0, 0, 0, 0, 244, 17, 0, 1016, 0, 42, 723, 0, 0, 0, 2117, 0, 0, 1450, 3409, 0, 2072, 508, 0, 2, 1854, 0, 606, 3023, 0, 1790, 2827, 10] payload_rows_per_dst=[7484, 6876, 6272, 6318, 6706, 6994, 6182, 4626] dup_rows_per_dst=[3048, 2482, 860, 1248, 1894, 2478, 1882, 186] combine_rows_per_src=[7484, 6876, 6272, 6318, 6706, 6994, 6182, 4626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51152 payload_rows_remote=45106 dup_rows=14384 dup_rows_remote=12962 dedup_ratio=0.219482 router_tokens_per_expert=[0, 0, 0, 438, 1082, 0, 463, 0, 2773, 9, 300, 0, 19, 2968, 0, 1096, 180, 0, 0, 0, 201, 538, 0, 0, 0, 0, 225, 0, 99, 14, 0, 1327, 0, 0, 0, 0, 51, 26, 0, 485, 0, 52, 316, 0, 0, 0, 2368, 0, 0, 1711, 2838, 0, 1388, 315, 0, 0, 2099, 0, 584, 3485, 3, 2133, 3172, 10] payload_rows_per_dst=[5888, 6474, 6820, 6830, 7232, 6046, 4944, 6918] dup_rows_per_dst=[1804, 1920, 1362, 2436, 862, 1422, 1058, 3520] combine_rows_per_src=[5888, 6474, 6820, 6830, 7232, 6046, 4944, 6918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44872 payload_rows_remote=37070 dup_rows=20664 dup_rows_remote=12998 dedup_ratio=0.315308 router_tokens_per_expert=[0, 0, 0, 356, 945, 0, 432, 15, 2913, 4, 130, 0, 3, 3109, 0, 1019, 205, 0, 0, 0, 401, 236, 0, 0, 0, 0, 301, 1, 118, 56, 0, 1254, 0, 0, 0, 0, 135, 3, 0, 804, 0, 65, 376, 5, 3, 0, 2361, 0, 0, 1492, 2951, 2, 1780, 296, 0, 4, 2205, 0, 420, 3035, 4, 2515, 2788, 26] payload_rows_per_dst=[3154, 2466, 6236, 7024, 5296, 5200, 7694, 7802] dup_rows_per_dst=[342, 100, 1024, 2654, 1090, 420, 7368, 7666] combine_rows_per_src=[3154, 2466, 6236, 7024, 5296, 5200, 7694, 7802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49954 payload_rows_remote=43228 dup_rows=15582 dup_rows_remote=13146 dedup_ratio=0.237762 router_tokens_per_expert=[0, 0, 0, 322, 1552, 0, 417, 4, 2612, 20, 101, 0, 4, 2829, 0, 1237, 143, 0, 0, 0, 189, 740, 0, 0, 0, 0, 155, 0, 13, 7, 0, 1117, 0, 0, 0, 0, 80, 3, 0, 612, 0, 18, 344, 0, 0, 0, 1850, 0, 0, 1565, 2855, 0, 1402, 160, 0, 1, 2464, 0, 878, 3544, 0, 2108, 3393, 29] payload_rows_per_dst=[4006, 6202, 7294, 6550, 7290, 4178, 6726, 7708] dup_rows_per_dst=[584, 1746, 1636, 1692, 1188, 246, 2436, 6054] combine_rows_per_src=[4006, 6202, 7294, 6550, 7290, 4178, 6726, 7708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47562 payload_rows_remote=40112 dup_rows=17974 dup_rows_remote=14558 dedup_ratio=0.274261 router_tokens_per_expert=[0, 0, 0, 231, 1520, 0, 357, 0, 3020, 6, 37, 0, 1, 3129, 0, 1162, 55, 0, 0, 0, 230, 665, 0, 0, 0, 0, 59, 0, 24, 9, 0, 1257, 0, 0, 0, 0, 165, 3, 0, 584, 0, 83, 274, 0, 0, 0, 2369, 0, 0, 1975, 3325, 0, 1575, 274, 0, 0, 1982, 0, 412, 3418, 0, 1957, 2586, 24] payload_rows_per_dst=[7450, 7358, 7084, 3978, 7174, 6916, 6038, 1564] dup_rows_per_dst=[3416, 5390, 3144, 682, 1166, 2500, 1610, 66] combine_rows_per_src=[7450, 7358, 7084, 3978, 7174, 6916, 6038, 1564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323893248 dispatch_meta_bytes=233616 combine_payload_bytes=323893248 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=324126864 ingress_bytes=544094208 link_bytes=544094208 token_bytes=648020112 weight_movement_bytes=220200960 total_nvlink_bytes=868221072 weight_to_token_ratio=0.339806 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.462693158e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318345216 dispatch_meta_bytes=227400 combine_payload_bytes=318345216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318572616 ingress_bytes=428445696 link_bytes=428445696 token_bytes=636917832 weight_movement_bytes=110100480 total_nvlink_bytes=747018312 weight_to_token_ratio=0.172864 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.451375326e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=334071808 dispatch_meta_bytes=230392 combine_payload_bytes=334071808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334302200 ingress_bytes=444172288 link_bytes=444172288 token_bytes=668374008 weight_movement_bytes=110100480 total_nvlink_bytes=778474488 weight_to_token_ratio=0.164729 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.724886626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323319808 dispatch_meta_bytes=232272 combine_payload_bytes=323319808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323552080 ingress_bytes=433420288 link_bytes=433420288 token_bytes=646871888 weight_movement_bytes=110100480 total_nvlink_bytes=756972368 weight_to_token_ratio=0.170204 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.537891662e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52248 payload_rows_remote=45320 dup_rows=13288 dup_rows_remote=11878 dedup_ratio=0.202759 router_tokens_per_expert=[0, 0, 0, 281, 931, 0, 389, 1, 3109, 13, 159, 0, 0, 3285, 0, 888, 147, 1, 0, 5, 579, 397, 0, 0, 2, 0, 271, 0, 11, 85, 0, 1208, 1, 5, 0, 3, 130, 15, 2, 762, 0, 67, 279, 1, 0, 0, 2136, 0, 1, 2014, 2867, 1, 1655, 279, 1, 6, 2198, 4, 543, 3226, 1, 1830, 2940, 39] payload_rows_per_dst=[6906, 6928, 6568, 7182, 6952, 7006, 6114, 4592] dup_rows_per_dst=[2032, 1410, 1570, 2542, 1336, 2356, 1800, 242] combine_rows_per_src=[6906, 6928, 6568, 7182, 6952, 7006, 6114, 4592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265717760 dispatch_meta_bytes=200272 combine_payload_bytes=265717760 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=265918032 ingress_bytes=816220160 link_bytes=816220160 token_bytes=531635792 weight_movement_bytes=550502400 total_nvlink_bytes=1082138192 weight_to_token_ratio=1.035488 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.419541103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309858304 dispatch_meta_bytes=225496 combine_payload_bytes=309858304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=310083800 ingress_bytes=640159744 link_bytes=640159744 token_bytes=619942104 weight_movement_bytes=330301440 total_nvlink_bytes=950243544 weight_to_token_ratio=0.532794 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.113343083e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=287522816 dispatch_meta_bytes=218680 combine_payload_bytes=287522816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287741496 ingress_bytes=397623296 link_bytes=397623296 token_bytes=575264312 weight_movement_bytes=110100480 total_nvlink_bytes=685364792 weight_to_token_ratio=0.191391 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.915323095e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324853760 dispatch_meta_bytes=228792 combine_payload_bytes=324853760 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=325082552 ingress_bytes=545054720 link_bytes=545054720 token_bytes=649936312 weight_movement_bytes=220200960 total_nvlink_bytes=870137272 weight_to_token_ratio=0.338804 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.479398042e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56150 payload_rows_remote=48038 dup_rows=9386 dup_rows_remote=5332 dedup_ratio=0.143219 router_tokens_per_expert=[0, 0, 3688, 40, 0, 17, 497, 0, 0, 142, 0, 0, 0, 401, 0, 0, 371, 0, 0, 0, 0, 89, 1778, 0, 0, 0, 146, 21, 0, 100, 55, 0, 1461, 1403, 0, 0, 3995, 3518, 2206, 0, 886, 1, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 3907, 0, 0, 4031, 48, 0, 0, 15] payload_rows_per_dst=[7528, 8160, 8112, 7432, 7338, 1774, 7992, 7814] dup_rows_per_dst=[956, 988, 4054, 248, 2802, 0, 212, 126] combine_rows_per_src=[7528, 8160, 8112, 7432, 7338, 1774, 7992, 7814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58316 payload_rows_remote=51126 dup_rows=7220 dup_rows_remote=4574 dedup_ratio=0.110168 router_tokens_per_expert=[0, 0, 3748, 12, 5, 0, 504, 0, 0, 154, 0, 0, 0, 543, 0, 0, 502, 0, 0, 0, 0, 46, 1714, 0, 0, 0, 280, 6, 0, 26, 22, 0, 1171, 1235, 0, 0, 4008, 3618, 2512, 0, 838, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 3731, 0, 0, 4045, 85, 1, 0, 0] payload_rows_per_dst=[7576, 8170, 4288, 7798, 7190, 7808, 8022, 7464] dup_rows_per_dst=[962, 1314, 236, 106, 2646, 1544, 242, 170] combine_rows_per_src=[7576, 8170, 4288, 7798, 7190, 7808, 8022, 7464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57620 payload_rows_remote=49910 dup_rows=7916 dup_rows_remote=7644 dedup_ratio=0.120789 router_tokens_per_expert=[0, 0, 3864, 37, 2, 0, 450, 0, 0, 169, 0, 0, 0, 487, 0, 0, 945, 0, 0, 1, 0, 71, 1443, 0, 0, 0, 267, 28, 0, 57, 35, 0, 1543, 1314, 0, 0, 3979, 3604, 1998, 5, 827, 2, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 3687, 0, 0, 4031, 68, 0, 0, 4] payload_rows_per_dst=[7760, 8130, 5048, 7710, 7128, 6496, 7970, 7378] dup_rows_per_dst=[946, 1244, 1308, 272, 2592, 1314, 100, 140] combine_rows_per_src=[7760, 8130, 5048, 7710, 7128, 6496, 7970, 7378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56124 payload_rows_remote=48084 dup_rows=9412 dup_rows_remote=3786 dedup_ratio=0.143616 router_tokens_per_expert=[1, 0, 3491, 55, 0, 0, 505, 0, 0, 205, 0, 0, 0, 445, 0, 0, 574, 0, 0, 0, 0, 147, 1839, 0, 0, 0, 360, 11, 0, 48, 29, 0, 1420, 1230, 0, 0, 4023, 3570, 1872, 0, 1092, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 87, 0, 0, 3759, 0, 0, 4044, 60, 0, 0, 0] payload_rows_per_dst=[7060, 8180, 4638, 7800, 4834, 8040, 8054, 7518] dup_rows_per_dst=[1044, 1208, 482, 236, 528, 5626, 168, 120] combine_rows_per_src=[7060, 8180, 4638, 7800, 4834, 8040, 8054, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56218 payload_rows_remote=48080 dup_rows=9318 dup_rows_remote=9240 dedup_ratio=0.142181 router_tokens_per_expert=[0, 0, 3682, 7, 0, 0, 297, 0, 0, 202, 0, 0, 0, 289, 0, 0, 383, 0, 0, 0, 0, 294, 2703, 0, 0, 0, 326, 12, 0, 23, 34, 0, 1630, 589, 0, 0, 4069, 3260, 2269, 0, 1141, 11, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 3673, 0, 0, 4054, 65, 0, 0, 0] payload_rows_per_dst=[7426, 7726, 6072, 7246, 4200, 8010, 8138, 7400] dup_rows_per_dst=[546, 840, 688, 64, 238, 6264, 78, 600] combine_rows_per_src=[7426, 7726, 6072, 7246, 4200, 8010, 8138, 7400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344336384 dispatch_meta_bytes=213480 combine_payload_bytes=344336384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344549864 ingress_bytes=454436864 link_bytes=454436864 token_bytes=688886248 weight_movement_bytes=110100480 total_nvlink_bytes=798986728 weight_to_token_ratio=0.159824 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.903404485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49086 payload_rows_remote=40906 dup_rows=16450 dup_rows_remote=8108 dedup_ratio=0.251007 router_tokens_per_expert=[0, 0, 3428, 44, 0, 0, 327, 0, 0, 159, 0, 0, 0, 422, 0, 0, 766, 0, 0, 0, 0, 45, 2066, 0, 0, 0, 301, 22, 0, 41, 31, 0, 1430, 1357, 0, 0, 3965, 3419, 2244, 3, 813, 5, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 3783, 0, 0, 4035, 38, 2, 0, 4] payload_rows_per_dst=[6240, 1146, 5154, 7430, 4946, 8018, 7972, 8180] dup_rows_per_dst=[560, 16, 600, 198, 634, 5722, 378, 8342] combine_rows_per_src=[6240, 1146, 5154, 7430, 4946, 8018, 7972, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51910 payload_rows_remote=44156 dup_rows=13626 dup_rows_remote=12890 dedup_ratio=0.207916 router_tokens_per_expert=[0, 0, 3848, 24, 2, 3, 368, 0, 0, 152, 0, 0, 0, 374, 0, 0, 768, 0, 0, 0, 0, 82, 1728, 0, 0, 0, 139, 3, 0, 68, 65, 0, 1311, 1185, 0, 0, 3994, 3412, 2361, 0, 1282, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 3824, 0, 0, 4033, 50, 0, 0, 7] payload_rows_per_dst=[7754, 8162, 7972, 7444, 7372, 2564, 7994, 2648] dup_rows_per_dst=[736, 956, 4362, 2552, 4802, 0, 186, 32] combine_rows_per_src=[7754, 8162, 7972, 7444, 7372, 2564, 7994, 2648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=366471168 dispatch_meta_bytes=222800 combine_payload_bytes=366471168 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=366693968 ingress_bytes=1027074048 link_bytes=1027074048 token_bytes=733165136 weight_movement_bytes=660602880 total_nvlink_bytes=1393768016 weight_to_token_ratio=0.901029 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.786250694e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=357754880 dispatch_meta_bytes=230216 combine_payload_bytes=357754880 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=357985096 ingress_bytes=577955840 link_bytes=577955840 token_bytes=715739976 weight_movement_bytes=220200960 total_nvlink_bytes=935940936 weight_to_token_ratio=0.307655 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005160263e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344666112 dispatch_meta_bytes=207480 combine_payload_bytes=344666112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344873592 ingress_bytes=564867072 link_bytes=564867072 token_bytes=689539704 weight_movement_bytes=220200960 total_nvlink_bytes=909740664 weight_to_token_ratio=0.319345 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.823967429e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55108 payload_rows_remote=47022 dup_rows=10428 dup_rows_remote=9218 dedup_ratio=0.159119 router_tokens_per_expert=[2, 0, 3542, 47, 4, 1, 625, 0, 0, 211, 0, 2, 0, 435, 0, 3, 802, 0, 0, 3, 3, 61, 1632, 3, 1, 0, 278, 16, 5, 92, 32, 0, 1612, 1593, 3, 0, 3925, 3382, 1629, 0, 1171, 0, 3757, 2, 0, 4, 1, 0, 2, 0, 1, 1, 0, 76, 10, 0, 3752, 0, 1, 3997, 39, 0, 0, 10] payload_rows_per_dst=[7244, 8086, 8074, 7386, 6570, 2348, 7894, 7506] dup_rows_per_dst=[1198, 1210, 4448, 1614, 1716, 8, 136, 98] combine_rows_per_src=[7244, 8086, 8074, 7386, 6570, 2348, 7894, 7506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344637440 dispatch_meta_bytes=229280 combine_payload_bytes=344637440 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=344866720 ingress_bytes=564838400 link_bytes=564838400 token_bytes=689504160 weight_movement_bytes=220200960 total_nvlink_bytes=909705120 weight_to_token_ratio=0.319361 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.823468776e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=293214208 dispatch_meta_bytes=196056 combine_payload_bytes=293214208 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=293410264 ingress_bytes=623515648 link_bytes=623515648 token_bytes=586624472 weight_movement_bytes=330301440 total_nvlink_bytes=916925912 weight_to_token_ratio=0.563054 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.084396263e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=316510208 dispatch_meta_bytes=228184 combine_payload_bytes=316510208 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=316738392 ingress_bytes=426610688 link_bytes=426610688 token_bytes=633248600 weight_movement_bytes=110100480 total_nvlink_bytes=743349080 weight_to_token_ratio=0.173866 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.419461518e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=337053696 dispatch_meta_bytes=224960 combine_payload_bytes=337053696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337278656 ingress_bytes=447154176 link_bytes=447154176 token_bytes=674332352 weight_movement_bytes=110100480 total_nvlink_bytes=784432832 weight_to_token_ratio=0.163273 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.776746563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57904 payload_rows_remote=49716 dup_rows=7632 dup_rows_remote=7626 dedup_ratio=0.116455 router_tokens_per_expert=[2178, 0, 490, 0, 240, 0, 0, 0, 1, 0, 0, 1320, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 562, 0, 1042, 0, 2, 2755, 0, 0, 0, 4, 0, 4028, 0, 3551, 1, 0, 0, 1500, 0, 0, 154, 3094, 0, 0, 2953, 4094, 697, 0, 0, 0, 0, 88, 0, 1, 0, 0, 15, 3995, 0] payload_rows_per_dst=[5220, 8068, 8188, 7098, 7106, 7566, 6638, 8020] dup_rows_per_dst=[596, 2630, 6, 1624, 6, 1930, 662, 178] combine_rows_per_src=[5220, 8068, 8188, 7098, 7106, 7566, 6638, 8020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59070 payload_rows_remote=51122 dup_rows=6466 dup_rows_remote=4264 dedup_ratio=0.098663 router_tokens_per_expert=[2679, 0, 648, 0, 189, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 370, 0, 992, 0, 0, 2999, 0, 0, 0, 2, 0, 3902, 0, 3178, 1, 0, 0, 1171, 0, 0, 157, 3747, 0, 0, 2787, 4091, 1321, 0, 0, 0, 0, 56, 0, 0, 0, 0, 14, 4023, 0] payload_rows_per_dst=[6476, 7816, 8182, 7230, 6356, 7948, 7006, 8056] dup_rows_per_dst=[556, 866, 4, 1492, 6, 2202, 1210, 130] combine_rows_per_src=[6476, 7816, 8182, 7230, 6356, 7948, 7006, 8056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59968 payload_rows_remote=53106 dup_rows=5568 dup_rows_remote=5568 dedup_ratio=0.084961 router_tokens_per_expert=[3366, 0, 278, 0, 170, 0, 0, 0, 7, 0, 0, 560, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 403, 0, 1143, 0, 0, 3174, 0, 0, 0, 0, 0, 4020, 0, 3431, 0, 0, 0, 811, 0, 0, 136, 3742, 0, 0, 2904, 4095, 463, 0, 0, 0, 0, 60, 0, 0, 0, 0, 9, 3988, 0] payload_rows_per_dst=[7036, 8060, 8190, 7540, 6862, 7910, 6360, 8010] dup_rows_per_dst=[592, 1124, 6, 1900, 0, 1468, 374, 104] combine_rows_per_src=[7036, 8060, 8190, 7540, 6862, 7910, 6360, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60598 payload_rows_remote=53394 dup_rows=4938 dup_rows_remote=3450 dedup_ratio=0.075348 router_tokens_per_expert=[3256, 2, 200, 0, 129, 2, 0, 0, 8, 0, 0, 800, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 1057, 0, 4, 2923, 0, 0, 0, 29, 0, 4013, 0, 3826, 0, 9, 0, 411, 1, 0, 123, 3575, 0, 0, 3062, 4090, 794, 0, 0, 0, 0, 38, 0, 1, 0, 0, 4, 4046, 0] payload_rows_per_dst=[6764, 8046, 8180, 7204, 7654, 7696, 6954, 8100] dup_rows_per_dst=[414, 1602, 0, 1488, 56, 542, 758, 78] combine_rows_per_src=[6764, 8046, 8180, 7204, 7654, 7696, 6954, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59586 payload_rows_remote=52512 dup_rows=5950 dup_rows_remote=4808 dedup_ratio=0.090790 router_tokens_per_expert=[3278, 2, 391, 0, 151, 0, 0, 0, 5, 0, 0, 376, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 237, 0, 875, 0, 0, 2791, 0, 0, 0, 5, 0, 3981, 0, 3401, 7, 4, 0, 984, 0, 0, 152, 3842, 0, 0, 3001, 4091, 1107, 0, 0, 0, 0, 49, 0, 0, 0, 0, 8, 4029, 0] payload_rows_per_dst=[7090, 7360, 8182, 6826, 6934, 8034, 7074, 8086] dup_rows_per_dst=[554, 692, 2, 980, 564, 1930, 1142, 86] combine_rows_per_src=[7090, 7360, 8182, 6826, 6934, 8034, 7074, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=356364288 dispatch_meta_bytes=229368 combine_payload_bytes=356364288 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=356593656 ingress_bytes=576565248 link_bytes=576565248 token_bytes=712957944 weight_movement_bytes=220200960 total_nvlink_bytes=933158904 weight_to_token_ratio=0.308855 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002741795e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50462 payload_rows_remote=42502 dup_rows=15074 dup_rows_remote=14642 dedup_ratio=0.230011 router_tokens_per_expert=[3019, 0, 421, 0, 188, 0, 0, 0, 4, 0, 0, 445, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 359, 0, 1094, 0, 9, 3489, 0, 0, 0, 25, 0, 3926, 0, 3714, 0, 37, 0, 743, 0, 0, 132, 3605, 0, 0, 2465, 4093, 981, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 3968, 0] payload_rows_per_dst=[6652, 902, 8186, 3198, 8178, 7780, 7606, 7960] dup_rows_per_dst=[604, 0, 8, 252, 7214, 1430, 5134, 432] combine_rows_per_src=[6652, 902, 8186, 3198, 8178, 7780, 7606, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55792 payload_rows_remote=47692 dup_rows=9744 dup_rows_remote=3582 dedup_ratio=0.148682 router_tokens_per_expert=[3260, 0, 230, 0, 161, 1, 0, 0, 7, 0, 0, 832, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 480, 0, 732, 0, 1, 3290, 0, 0, 0, 13, 0, 4003, 0, 3799, 0, 0, 0, 459, 0, 0, 132, 3479, 0, 0, 3306, 4089, 394, 0, 0, 0, 0, 55, 0, 0, 0, 0, 9, 4033, 0] payload_rows_per_dst=[8100, 8018, 8180, 7572, 7598, 1174, 7068, 8082] dup_rows_per_dst=[6162, 1666, 52, 1434, 26, 8, 284, 112] combine_rows_per_src=[8100, 8018, 8180, 7572, 7598, 1174, 7068, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366442496 dispatch_meta_bytes=221544 combine_payload_bytes=366442496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=366664040 ingress_bytes=586643456 link_bytes=586643456 token_bytes=733106536 weight_movement_bytes=220200960 total_nvlink_bytes=953307496 weight_to_token_ratio=0.300367 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.020269456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=380663808 dispatch_meta_bytes=234696 combine_payload_bytes=380663808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380898504 ingress_bytes=600864768 link_bytes=600864768 token_bytes=761562312 weight_movement_bytes=220200960 total_nvlink_bytes=981763272 weight_to_token_ratio=0.289144 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045002657e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=382728192 dispatch_meta_bytes=227376 combine_payload_bytes=382728192 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=382955568 ingress_bytes=823130112 link_bytes=823130112 token_bytes=765683760 weight_movement_bytes=440401920 total_nvlink_bytes=1206085680 weight_to_token_ratio=0.575175 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.431558646e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59552 payload_rows_remote=51716 dup_rows=5984 dup_rows_remote=5134 dedup_ratio=0.091309 router_tokens_per_expert=[3179, 0, 498, 0, 130, 1, 3, 4, 3, 0, 0, 428, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 362, 4, 1251, 0, 3, 3227, 2, 2, 0, 15, 1, 3910, 0, 3470, 0, 3, 1, 796, 0, 0, 141, 3562, 1, 0, 2836, 4084, 792, 0, 5, 0, 0, 50, 0, 1, 3, 3, 6, 3988, 0] payload_rows_per_dst=[7218, 7836, 8170, 7632, 6946, 7062, 6684, 8004] dup_rows_per_dst=[1502, 850, 0, 2066, 30, 856, 582, 98] combine_rows_per_src=[7218, 7836, 8170, 7632, 6946, 7062, 6684, 8004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=376406016 dispatch_meta_bytes=229280 combine_payload_bytes=376406016 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=376635296 ingress_bytes=706707456 link_bytes=706707456 token_bytes=753041312 weight_movement_bytes=330301440 total_nvlink_bytes=1083342752 weight_to_token_ratio=0.438623 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.229080499e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=304654336 dispatch_meta_bytes=228576 combine_payload_bytes=304654336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304882912 ingress_bytes=414754816 link_bytes=414754816 token_bytes=609537248 weight_movement_bytes=110100480 total_nvlink_bytes=719637728 weight_to_token_ratio=0.180630 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.213268405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=341856256 dispatch_meta_bytes=205096 combine_payload_bytes=341856256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342061352 ingress_bytes=451956736 link_bytes=451956736 token_bytes=683917608 weight_movement_bytes=110100480 total_nvlink_bytes=794018088 weight_to_token_ratio=0.160985 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.860270980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=370700288 dispatch_meta_bytes=227400 combine_payload_bytes=370700288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370927688 ingress_bytes=480800768 link_bytes=480800768 token_bytes=741627976 weight_movement_bytes=110100480 total_nvlink_bytes=851728456 weight_to_token_ratio=0.148458 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.361916137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57790 payload_rows_remote=52530 dup_rows=7746 dup_rows_remote=6976 dedup_ratio=0.118195 router_tokens_per_expert=[0, 62, 91, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2359, 0, 0, 182, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 4088, 4060, 0, 19, 0, 0, 0, 0, 0, 2169, 559, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 3905, 3748, 568, 0, 0, 1684, 0, 0, 3904, 0, 0] payload_rows_per_dst=[8120, 8192, 5260, 8176, 8120, 5204, 6806, 7912] dup_rows_per_dst=[964, 2, 770, 1520, 38, 310, 878, 3264] combine_rows_per_src=[8120, 8192, 5260, 8176, 8120, 5204, 6806, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60132 payload_rows_remote=53640 dup_rows=5404 dup_rows_remote=5160 dedup_ratio=0.082458 router_tokens_per_expert=[0, 11, 199, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 0, 249, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 4094, 4087, 0, 1, 0, 0, 0, 0, 0, 2962, 405, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 3842, 3725, 832, 0, 0, 824, 0, 0, 3940, 0, 0] payload_rows_per_dst=[8082, 8192, 5410, 8188, 8174, 6492, 7676, 7918] dup_rows_per_dst=[662, 0, 382, 1066, 2, 244, 1438, 1610] combine_rows_per_src=[8082, 8192, 5410, 8188, 8174, 6492, 7676, 7918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60652 payload_rows_remote=52460 dup_rows=4884 dup_rows_remote=4878 dedup_ratio=0.074524 router_tokens_per_expert=[0, 49, 87, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3056, 0, 0, 245, 0, 0, 0, 0, 0, 0, 830, 0, 0, 1, 0, 0, 4077, 4096, 0, 3, 0, 0, 0, 0, 0, 2870, 113, 0, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 3850, 3937, 642, 0, 0, 540, 0, 0, 3922, 0, 0] payload_rows_per_dst=[8062, 8192, 6216, 8158, 8192, 5938, 7970, 7924] dup_rows_per_dst=[574, 2, 386, 1658, 6, 70, 1188, 1000] combine_rows_per_src=[8062, 8192, 6216, 8158, 8192, 5938, 7970, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59978 payload_rows_remote=51792 dup_rows=5558 dup_rows_remote=4150 dedup_ratio=0.084808 router_tokens_per_expert=[0, 38, 96, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 370, 0, 0, 0, 0, 0, 0, 704, 2, 0, 0, 0, 0, 4091, 4093, 0, 2, 0, 0, 0, 2, 0, 3078, 204, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3798, 3820, 660, 0, 0, 833, 0, 0, 3913, 0, 0] payload_rows_per_dst=[8010, 8192, 5176, 8186, 8186, 6414, 7862, 7952] dup_rows_per_dst=[904, 0, 450, 1408, 8, 150, 1098, 1540] combine_rows_per_src=[8010, 8192, 5176, 8186, 8186, 6414, 7862, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=376535040 dispatch_meta_bytes=238024 combine_payload_bytes=376535040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376773064 ingress_bytes=486635520 link_bytes=486635520 token_bytes=753308104 weight_movement_bytes=110100480 total_nvlink_bytes=863408584 weight_to_token_ratio=0.146156 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.463392071e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=384491520 dispatch_meta_bytes=235200 combine_payload_bytes=384491520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384726720 ingress_bytes=494592000 link_bytes=494592000 token_bytes=769218240 weight_movement_bytes=110100480 total_nvlink_bytes=879318720 weight_to_token_ratio=0.143133 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.601768344e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59310 payload_rows_remote=51252 dup_rows=6226 dup_rows_remote=3128 dedup_ratio=0.095001 router_tokens_per_expert=[0, 25, 83, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 0, 274, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 4090, 4077, 0, 0, 0, 0, 0, 0, 0, 3312, 113, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3807, 3971, 612, 0, 0, 884, 0, 0, 3935, 0, 0] payload_rows_per_dst=[8012, 8192, 5866, 6334, 8166, 6762, 8058, 7920] dup_rows_per_dst=[476, 0, 398, 354, 90, 92, 3098, 1718] combine_rows_per_src=[8012, 8192, 5866, 6334, 8166, 6762, 8058, 7920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49126 payload_rows_remote=41068 dup_rows=16410 dup_rows_remote=10288 dedup_ratio=0.250397 router_tokens_per_expert=[0, 104, 78, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 0, 382, 0, 0, 0, 0, 0, 0, 690, 1, 0, 0, 0, 0, 4082, 4095, 0, 3, 0, 0, 0, 0, 0, 2579, 268, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3855, 3683, 279, 0, 0, 952, 0, 0, 3867, 0, 0] payload_rows_per_dst=[2718, 8182, 6808, 1382, 8164, 5622, 8192, 8058] dup_rows_per_dst=[308, 0, 562, 0, 6, 76, 9336, 6122] combine_rows_per_src=[2718, 8182, 6808, 1382, 8164, 5622, 8192, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=376033280 dispatch_meta_bytes=229352 combine_payload_bytes=376033280 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=376262632 ingress_bytes=706334720 link_bytes=706334720 token_bytes=752295912 weight_movement_bytes=330301440 total_nvlink_bytes=1082597352 weight_to_token_ratio=0.439058 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.228432250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50602 payload_rows_remote=42492 dup_rows=14934 dup_rows_remote=14040 dedup_ratio=0.227875 router_tokens_per_expert=[0, 29, 139, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2625, 0, 0, 218, 0, 0, 0, 0, 0, 0, 972, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 0, 0, 0, 0, 3167, 98, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3879, 3972, 416, 0, 0, 559, 0, 0, 3956, 0, 0] payload_rows_per_dst=[8110, 8192, 8006, 8178, 8190, 7976, 832, 1118] dup_rows_per_dst=[894, 6, 5624, 1944, 0, 6466, 0, 0] combine_rows_per_src=[8110, 8192, 8006, 8178, 8190, 7976, 832, 1118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=371245056 dispatch_meta_bytes=223768 combine_payload_bytes=371245056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371468824 ingress_bytes=481345536 link_bytes=481345536 token_bytes=742713880 weight_movement_bytes=110100480 total_nvlink_bytes=852814360 weight_to_token_ratio=0.148241 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.371390548e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51194 payload_rows_remote=43004 dup_rows=14342 dup_rows_remote=14340 dedup_ratio=0.218842 router_tokens_per_expert=[0, 57, 402, 2, 470, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 2986, 3, 1, 243, 4, 0, 0, 1, 0, 0, 400, 1, 4, 0, 0, 1, 4086, 4092, 0, 4, 0, 1, 0, 0, 0, 2848, 207, 0, 0, 19, 2, 0, 0, 4091, 0, 0, 1, 3889, 3873, 551, 6, 0, 657, 0, 0, 3861, 0, 0] payload_rows_per_dst=[8132, 8190, 8052, 8174, 8186, 7896, 1104, 1460] dup_rows_per_dst=[1508, 2, 6170, 810, 8, 5824, 0, 20] combine_rows_per_src=[8132, 8190, 8052, 8174, 8186, 7896, 1104, 1460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=367374336 dispatch_meta_bytes=217520 combine_payload_bytes=367374336 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=367591856 ingress_bytes=807776256 link_bytes=807776256 token_bytes=734966192 weight_movement_bytes=440401920 total_nvlink_bytes=1175368112 weight_to_token_ratio=0.599214 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.404855766e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=294375424 dispatch_meta_bytes=205424 combine_payload_bytes=294375424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=294580848 ingress_bytes=514576384 link_bytes=514576384 token_bytes=588956272 weight_movement_bytes=220200960 total_nvlink_bytes=809157232 weight_to_token_ratio=0.373883 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=304582656 dispatch_meta_bytes=226128 combine_payload_bytes=304582656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304808784 ingress_bytes=414683136 link_bytes=414683136 token_bytes=609391440 weight_movement_bytes=110100480 total_nvlink_bytes=719491920 weight_to_token_ratio=0.180673 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.212021772e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=308252672 dispatch_meta_bytes=229376 combine_payload_bytes=308252672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308482048 ingress_bytes=418353152 link_bytes=418353152 token_bytes=616734720 weight_movement_bytes=110100480 total_nvlink_bytes=726835200 weight_to_token_ratio=0.178522 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.275849386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64686 payload_rows_remote=57042 dup_rows=850 dup_rows_remote=578 dedup_ratio=0.012970 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 1178, 1448, 0, 4096, 0, 0, 4061, 0, 0, 0, 39, 0, 0, 0, 4027, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8100, 8192, 8192, 7644, 8122, 8054] dup_rows_per_dst=[0, 10, 260, 230, 0, 272, 78, 0] combine_rows_per_src=[8190, 8192, 8100, 8192, 8192, 7644, 8122, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64538 payload_rows_remote=56528 dup_rows=998 dup_rows_remote=682 dedup_ratio=0.015228 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 5, 0, 0, 0, 0, 0, 1, 0, 170, 7, 0, 0, 0, 3986, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 864, 906, 0, 4096, 0, 0, 4063, 0, 0, 0, 35, 0, 0, 0, 4005, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8010, 8192, 8192, 7634, 8126, 8010] dup_rows_per_dst=[0, 12, 316, 246, 2, 352, 70, 0] combine_rows_per_src=[8182, 8192, 8010, 8192, 8192, 7634, 8126, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64622 payload_rows_remote=56430 dup_rows=914 dup_rows_remote=912 dedup_ratio=0.013947 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 6, 0, 0, 0, 0, 0, 2, 0, 166, 12, 0, 0, 0, 4042, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 936, 1188, 0, 4096, 0, 0, 4062, 0, 0, 0, 27, 0, 0, 0, 4032, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8092, 8192, 8192, 7580, 8124, 8064] dup_rows_per_dst=[0, 16, 348, 260, 2, 234, 54, 0] combine_rows_per_src=[8186, 8192, 8092, 8192, 8192, 7580, 8124, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64696 payload_rows_remote=56504 dup_rows=840 dup_rows_remote=604 dedup_ratio=0.012817 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 4090, 4096, 0, 15, 0, 0, 0, 0, 0, 2, 0, 106, 5, 0, 0, 0, 4062, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 965, 1453, 0, 4096, 0, 0, 4066, 0, 0, 0, 15, 0, 0, 0, 3987, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8134, 8192, 8192, 7698, 8132, 7974] dup_rows_per_dst=[0, 34, 212, 236, 4, 324, 30, 0] combine_rows_per_src=[8182, 8192, 8134, 8192, 8192, 7698, 8132, 7974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=408877056 dispatch_meta_bytes=230480 combine_payload_bytes=408877056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409107536 ingress_bytes=518977536 link_bytes=518977536 token_bytes=817984592 weight_movement_bytes=110100480 total_nvlink_bytes=928085072 weight_to_token_ratio=0.134600 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.025872922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405192704 dispatch_meta_bytes=228840 combine_payload_bytes=405192704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405421544 ingress_bytes=625393664 link_bytes=625393664 token_bytes=810614248 weight_movement_bytes=220200960 total_nvlink_bytes=1030815208 weight_to_token_ratio=0.271647 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087662441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64908 payload_rows_remote=56752 dup_rows=628 dup_rows_remote=578 dedup_ratio=0.009583 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 76, 14, 0, 0, 0, 4058, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 1221, 1198, 0, 4096, 0, 0, 4078, 0, 0, 0, 25, 0, 0, 0, 4012, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8120, 8192, 8192, 7840, 8156, 8024] dup_rows_per_dst=[0, 2, 176, 198, 0, 202, 50, 0] combine_rows_per_src=[8192, 8192, 8120, 8192, 8192, 7840, 8156, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60956 payload_rows_remote=52804 dup_rows=4580 dup_rows_remote=3498 dedup_ratio=0.069885 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 280, 30, 0, 0, 0, 4064, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 792, 1247, 0, 4096, 0, 0, 4052, 0, 0, 0, 38, 0, 0, 0, 4062, 0, 0, 0, 0] payload_rows_per_dst=[6354, 8066, 6366, 8192, 8192, 7530, 8104, 8152] dup_rows_per_dst=[0, 4, 544, 240, 0, 2508, 202, 1082] combine_rows_per_src=[6354, 8066, 6366, 8192, 8192, 7530, 8104, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62868 payload_rows_remote=54676 dup_rows=2668 dup_rows_remote=798 dedup_ratio=0.040710 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 125, 9, 0, 0, 0, 4019, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 930, 1789, 0, 4096, 0, 0, 4059, 0, 0, 0, 21, 0, 0, 0, 4014, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8058, 6456, 8182, 7740, 8020, 8028] dup_rows_per_dst=[1870, 40, 248, 150, 0, 318, 42, 0] combine_rows_per_src=[8192, 8192, 8058, 6456, 8182, 7740, 8020, 8028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=404490240 dispatch_meta_bytes=229368 combine_payload_bytes=404490240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404719608 ingress_bytes=514590720 link_bytes=514590720 token_bytes=809209848 weight_movement_bytes=110100480 total_nvlink_bytes=919310328 weight_to_token_ratio=0.136059 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949578977e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405020672 dispatch_meta_bytes=228432 combine_payload_bytes=405020672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405249104 ingress_bytes=515121152 link_bytes=515121152 token_bytes=810269776 weight_movement_bytes=110100480 total_nvlink_bytes=920370256 weight_to_token_ratio=0.135881 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.958804062e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64726 payload_rows_remote=56536 dup_rows=810 dup_rows_remote=800 dedup_ratio=0.012360 router_tokens_per_expert=[1, 4089, 0, 1, 1, 0, 4091, 4091, 0, 4, 0, 0, 0, 4, 0, 0, 1, 212, 8, 1, 3, 1, 4050, 0, 0, 2, 0, 93, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 2121, 710, 998, 0, 4092, 3, 2, 4076, 1, 4, 2, 25, 2, 0, 0, 4066, 4, 0, 2, 0] payload_rows_per_dst=[8188, 8190, 8118, 8182, 8192, 7554, 8164, 8138] dup_rows_per_dst=[0, 10, 434, 186, 2, 106, 62, 10] combine_rows_per_src=[8188, 8190, 8118, 8182, 8192, 7554, 8164, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406798336 dispatch_meta_bytes=229320 combine_payload_bytes=406798336 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=407027656 ingress_bytes=626999296 link_bytes=626999296 token_bytes=813825992 weight_movement_bytes=220200960 total_nvlink_bytes=1034026952 weight_to_token_ratio=0.270575 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090454899e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378499072 dispatch_meta_bytes=225208 combine_payload_bytes=378499072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378724280 ingress_bytes=488599552 link_bytes=488599552 token_bytes=757223352 weight_movement_bytes=110100480 total_nvlink_bytes=867323832 weight_to_token_ratio=0.145400 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.497549817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=391917568 dispatch_meta_bytes=221896 combine_payload_bytes=391917568 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=392139464 ingress_bytes=942419968 link_bytes=942419968 token_bytes=784057032 weight_movement_bytes=550502400 total_nvlink_bytes=1334559432 weight_to_token_ratio=0.702120 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.639023326e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60014 payload_rows_remote=51822 dup_rows=5522 dup_rows_remote=5522 dedup_ratio=0.084259 router_tokens_per_expert=[0, 0, 0, 0, 0, 4042, 0, 0, 79, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 4096, 0, 0, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3473, 4096, 134, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8084, 4168, 8042, 7150, 8192, 8192, 7994, 8192] dup_rows_per_dst=[0, 470, 0, 0, 0, 0, 5052, 0] combine_rows_per_src=[8084, 4168, 8042, 7150, 8192, 8192, 7994, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405250048 dispatch_meta_bytes=229344 combine_payload_bytes=405250048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405479392 ingress_bytes=515350528 link_bytes=515350528 token_bytes=810729440 weight_movement_bytes=110100480 total_nvlink_bytes=920829920 weight_to_token_ratio=0.135804 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.962793288e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62774 payload_rows_remote=54722 dup_rows=2762 dup_rows_remote=2762 dedup_ratio=0.042145 router_tokens_per_expert=[0, 0, 0, 0, 0, 4046, 0, 0, 197, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 1, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4090, 0, 0, 3044, 4096, 47, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 8188, 8052, 7702, 8192, 8192, 6164, 8192] dup_rows_per_dst=[0, 2726, 0, 2, 0, 0, 20, 14] combine_rows_per_src=[8092, 8188, 8052, 7702, 8192, 8192, 6164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371460096 dispatch_meta_bytes=229376 combine_payload_bytes=371460096 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=371689472 ingress_bytes=481560576 link_bytes=481560576 token_bytes=743149568 weight_movement_bytes=110100480 total_nvlink_bytes=853250048 weight_to_token_ratio=0.148154 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.375130448e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62648 payload_rows_remote=55452 dup_rows=2888 dup_rows_remote=2888 dedup_ratio=0.044067 router_tokens_per_expert=[0, 0, 0, 0, 0, 4052, 0, 4, 249, 0, 0, 3, 0, 0, 0, 1118, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 4096, 0, 0, 0, 0, 0, 3598, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4079, 0, 0, 3364, 4096, 68, 0, 26, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8168, 7822, 7196, 8192, 8192, 6782, 8192] dup_rows_per_dst=[8, 2730, 0, 0, 2, 0, 96, 52] combine_rows_per_src=[8104, 8168, 7822, 7196, 8192, 8192, 6782, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62512 payload_rows_remote=54320 dup_rows=3024 dup_rows_remote=3022 dedup_ratio=0.046143 router_tokens_per_expert=[0, 0, 0, 0, 0, 4054, 0, 0, 147, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 4096, 0, 0, 0, 0, 1, 3773, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4081, 0, 0, 3075, 4096, 59, 0, 25, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 8178, 7870, 7548, 8192, 8192, 6232, 8192] dup_rows_per_dst=[0, 2924, 0, 0, 2, 0, 48, 50] combine_rows_per_src=[8108, 8178, 7870, 7548, 8192, 8192, 6232, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=392247296 dispatch_meta_bytes=229936 combine_payload_bytes=392247296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392477232 ingress_bytes=502347776 link_bytes=502347776 token_bytes=784724528 weight_movement_bytes=110100480 total_nvlink_bytes=894825008 weight_to_token_ratio=0.140305 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.736654045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58818 payload_rows_remote=50626 dup_rows=6718 dup_rows_remote=40 dedup_ratio=0.102509 router_tokens_per_expert=[0, 1, 0, 0, 0, 4045, 0, 0, 105, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 4090, 0, 2, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 3292, 4096, 48, 0, 16, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8090, 2448, 7994, 7518, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 10, 0, 0, 8, 0, 6678, 20] combine_rows_per_src=[8090, 2448, 7994, 7518, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59828 payload_rows_remote=51636 dup_rows=5708 dup_rows_remote=5642 dedup_ratio=0.087097 router_tokens_per_expert=[0, 0, 0, 0, 0, 4027, 0, 0, 51, 0, 0, 4, 0, 0, 0, 1738, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 4090, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4080, 0, 0, 2756, 4096, 51, 0, 39, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 3576, 7760, 7712, 8162, 8180, 8192, 8192] dup_rows_per_dst=[0, 10, 0, 0, 4, 0, 5628, 66] combine_rows_per_src=[8054, 3576, 7760, 7712, 8162, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397479936 dispatch_meta_bytes=233360 combine_payload_bytes=397479936 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397713296 ingress_bytes=617680896 link_bytes=617680896 token_bytes=795193232 weight_movement_bytes=220200960 total_nvlink_bytes=1015394192 weight_to_token_ratio=0.276915 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074248669e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56478 payload_rows_remote=48382 dup_rows=9058 dup_rows_remote=8198 dedup_ratio=0.138214 router_tokens_per_expert=[0, 0, 0, 0, 0, 4039, 0, 0, 118, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 4095, 0, 4, 0, 0, 0, 3632, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4087, 0, 0, 3108, 4096, 27, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 8174, 8124, 7732, 8192, 8192, 54, 7914] dup_rows_per_dst=[860, 3028, 1892, 3262, 4, 0, 12, 0] combine_rows_per_src=[8096, 8174, 8124, 7732, 8192, 8192, 54, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389365760 dispatch_meta_bytes=229368 combine_payload_bytes=389365760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389595128 ingress_bytes=499466240 link_bytes=499466240 token_bytes=778960888 weight_movement_bytes=110100480 total_nvlink_bytes=889061368 weight_to_token_ratio=0.141343 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60966 payload_rows_remote=52792 dup_rows=4570 dup_rows_remote=2006 dedup_ratio=0.069733 router_tokens_per_expert=[0, 9, 0, 0, 0, 4010, 0, 0, 109, 0, 3, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 2, 3988, 0, 0, 4090, 0, 0, 0, 0, 5, 3859, 0, 1, 5, 0, 0, 0, 6, 0, 0, 0, 1, 3, 2, 0, 4091, 5, 0, 2, 4076, 0, 0, 3122, 4089, 65, 0, 35, 0, 0, 5, 4, 0, 0] payload_rows_per_dst=[8032, 8174, 7980, 7814, 8192, 8192, 4390, 8192] dup_rows_per_dst=[6, 2564, 0, 1900, 10, 12, 2, 76] combine_rows_per_src=[8032, 8174, 7980, 7814, 8192, 8192, 4390, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=362887168 dispatch_meta_bytes=202664 combine_payload_bytes=362887168 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=363089832 ingress_bytes=1023490048 link_bytes=1023490048 token_bytes=725977000 weight_movement_bytes=660602880 total_nvlink_bytes=1386579880 weight_to_token_ratio=0.909950 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.780017529e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58704 payload_rows_remote=50512 dup_rows=6832 dup_rows_remote=4424 dedup_ratio=0.104248 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1770, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3516, 4096, 0, 0, 0, 0, 4096, 1751, 0, 4096, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 180, 0, 0, 0, 426] payload_rows_per_dst=[8192, 3682, 7036, 8192, 8002, 8192, 8192, 7216] dup_rows_per_dst=[0, 32, 0, 3502, 0, 2408, 0, 890] combine_rows_per_src=[8192, 3682, 7036, 8192, 8002, 8192, 8192, 7216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370126848 dispatch_meta_bytes=229112 combine_payload_bytes=370126848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370355960 ingress_bytes=480227328 link_bytes=480227328 token_bytes=740482808 weight_movement_bytes=110100480 total_nvlink_bytes=850583288 weight_to_token_ratio=0.148687 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.351943072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57688 payload_rows_remote=51860 dup_rows=7848 dup_rows_remote=7848 dedup_ratio=0.119751 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2912, 4096, 0, 0, 0, 0, 4096, 1513, 0, 4096, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 97, 0, 0, 0, 1267] payload_rows_per_dst=[8192, 8192, 5828, 3026, 8050, 8192, 8192, 8016] dup_rows_per_dst=[0, 4430, 0, 0, 0, 934, 0, 2484] combine_rows_per_src=[8192, 8192, 5828, 3026, 8050, 8192, 8192, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=346802176 dispatch_meta_bytes=226320 combine_payload_bytes=346802176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347028496 ingress_bytes=456902656 link_bytes=456902656 token_bytes=693830672 weight_movement_bytes=110100480 total_nvlink_bytes=803931152 weight_to_token_ratio=0.158685 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.946288664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378413056 dispatch_meta_bytes=219192 combine_payload_bytes=378413056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378632248 ingress_bytes=488513536 link_bytes=488513536 token_bytes=757045304 weight_movement_bytes=110100480 total_nvlink_bytes=867145784 weight_to_token_ratio=0.145434 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.496053858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=362070016 dispatch_meta_bytes=219744 combine_payload_bytes=362070016 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=362289760 ingress_bytes=692371456 link_bytes=692371456 token_bytes=724359776 weight_movement_bytes=330301440 total_nvlink_bytes=1054661216 weight_to_token_ratio=0.455991 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.204147838e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57350 payload_rows_remote=54130 dup_rows=8186 dup_rows_remote=8186 dedup_ratio=0.124908 router_tokens_per_expert=[0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 2328, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3004, 4096, 0, 0, 0, 0, 4096, 1610, 0, 4096, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 62, 0, 0, 0, 948] payload_rows_per_dst=[8192, 8192, 6040, 3220, 8042, 8190, 8192, 7282] dup_rows_per_dst=[40, 4792, 4, 0, 0, 1794, 0, 1556] combine_rows_per_src=[8192, 8192, 6040, 3220, 8042, 8190, 8192, 7282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371732480 dispatch_meta_bytes=238832 combine_payload_bytes=371732480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371971312 ingress_bytes=481832960 link_bytes=481832960 token_bytes=743703792 weight_movement_bytes=110100480 total_nvlink_bytes=853804272 weight_to_token_ratio=0.148043 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.379867653e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58694 payload_rows_remote=50742 dup_rows=6842 dup_rows_remote=6842 dedup_ratio=0.104401 router_tokens_per_expert=[0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1978, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3292, 4096, 0, 0, 0, 0, 4096, 2328, 0, 4096, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 638, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3413, 123, 0, 0, 0, 553] payload_rows_per_dst=[8192, 4860, 6584, 7534, 7952, 8192, 8192, 7188] dup_rows_per_dst=[44, 892, 0, 3638, 0, 1276, 2, 990] combine_rows_per_src=[8192, 4860, 6584, 7534, 7952, 8192, 8192, 7188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59504 payload_rows_remote=51312 dup_rows=6032 dup_rows_remote=6032 dedup_ratio=0.092041 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2085, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3613, 4096, 0, 0, 0, 0, 4096, 1926, 0, 4096, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3460, 95, 0, 0, 0, 435] payload_rows_per_dst=[8192, 4252, 7228, 8192, 8012, 8192, 8192, 7244] dup_rows_per_dst=[24, 82, 0, 3852, 0, 1338, 0, 736] combine_rows_per_src=[8192, 4252, 7228, 8192, 8012, 8192, 8192, 7244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388003840 dispatch_meta_bytes=249264 combine_payload_bytes=388003840 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=388253104 ingress_bytes=718305280 link_bytes=718305280 token_bytes=776256944 weight_movement_bytes=330301440 total_nvlink_bytes=1106558384 weight_to_token_ratio=0.425505 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249251023e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57570 payload_rows_remote=50466 dup_rows=7966 dup_rows_remote=7014 dedup_ratio=0.121552 router_tokens_per_expert=[0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1684, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3175, 4096, 0, 0, 0, 0, 4096, 2078, 0, 4096, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 300, 0, 0, 0, 360] payload_rows_per_dst=[7962, 3592, 6408, 8192, 7928, 8192, 8192, 7104] dup_rows_per_dst=[74, 136, 8, 4156, 228, 2412, 0, 952] combine_rows_per_src=[7962, 3592, 6408, 8192, 7928, 8192, 8192, 7104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53608 payload_rows_remote=45416 dup_rows=11928 dup_rows_remote=11910 dedup_ratio=0.182007 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1379, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3049, 4096, 0, 0, 0, 0, 4096, 2289, 0, 4096, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3252, 38, 0, 0, 0, 996] payload_rows_per_dst=[8192, 8192, 8192, 5934, 8086, 2580, 8190, 4242] dup_rows_per_dst=[18, 3158, 6110, 2062, 0, 114, 0, 466] combine_rows_per_src=[8192, 8192, 8192, 5934, 8086, 2580, 8190, 4242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55096 payload_rows_remote=46904 dup_rows=10440 dup_rows_remote=7580 dedup_ratio=0.159302 router_tokens_per_expert=[5, 0, 0, 0, 0, 12, 0, 1, 0, 0, 1311, 4, 114, 1, 4, 1, 1, 0, 0, 2, 5, 0, 20, 3281, 4096, 0, 0, 0, 0, 4091, 2014, 4, 4093, 0, 1, 0, 0, 4008, 0, 2, 0, 0, 0, 1, 4084, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3609, 314, 1, 0, 2, 658] payload_rows_per_dst=[8190, 8192, 7760, 4036, 8028, 3104, 8192, 7594] dup_rows_per_dst=[32, 2860, 4338, 0, 1208, 426, 0, 1576] combine_rows_per_src=[8190, 8192, 7760, 4036, 8028, 3104, 8192, 7594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=363718656 dispatch_meta_bytes=230336 combine_payload_bytes=363718656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=363948992 ingress_bytes=583919616 link_bytes=583919616 token_bytes=727667648 weight_movement_bytes=220200960 total_nvlink_bytes=947868608 weight_to_token_ratio=0.302612 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.015532250e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=367804416 dispatch_meta_bytes=229376 combine_payload_bytes=367804416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368033792 ingress_bytes=477904896 link_bytes=477904896 token_bytes=735838208 weight_movement_bytes=110100480 total_nvlink_bytes=845938688 weight_to_token_ratio=0.149626 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.311552160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52794 payload_rows_remote=44602 dup_rows=12742 dup_rows_remote=4554 dedup_ratio=0.194427 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 0, 10, 0, 0, 0, 0, 2415, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 4031, 0, 2161, 2495, 0, 0, 0, 0, 283, 2, 0] payload_rows_per_dst=[8192, 4802, 8192, 5188, 4830, 8192, 6898, 6500] dup_rows_per_dst=[2, 0, 0, 4, 0, 8188, 2744, 1804] combine_rows_per_src=[8192, 4802, 8192, 5188, 4830, 8192, 6898, 6500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=361740288 dispatch_meta_bytes=229920 combine_payload_bytes=361740288 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=361970208 ingress_bytes=581941248 link_bytes=581941248 token_bytes=723710496 weight_movement_bytes=220200960 total_nvlink_bytes=943911456 weight_to_token_ratio=0.304267 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012091543e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51648 payload_rows_remote=43456 dup_rows=13888 dup_rows_remote=13874 dedup_ratio=0.211914 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2425, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2586, 0, 0, 0, 0, 39, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4034, 0, 1241, 2986, 0, 0, 0, 0, 683, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6294, 8192, 0, 6086, 6500] dup_rows_per_dst=[0, 4860, 14, 1550, 4756, 0, 1870, 838] combine_rows_per_src=[8192, 8192, 8192, 6294, 8192, 0, 6086, 6500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=325541888 dispatch_meta_bytes=229304 combine_payload_bytes=325541888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325771192 ingress_bytes=435642368 link_bytes=435642368 token_bytes=651313080 weight_movement_bytes=110100480 total_nvlink_bytes=761413560 weight_to_token_ratio=0.169044 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.576537288e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=336207872 dispatch_meta_bytes=217936 combine_payload_bytes=336207872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336425808 ingress_bytes=446308352 link_bytes=446308352 token_bytes=672633680 weight_movement_bytes=110100480 total_nvlink_bytes=782734160 weight_to_token_ratio=0.163686 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.762036292e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=319707136 dispatch_meta_bytes=196624 combine_payload_bytes=319707136 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=319903760 ingress_bytes=870209536 link_bytes=870209536 token_bytes=639610896 weight_movement_bytes=550502400 total_nvlink_bytes=1190113296 weight_to_token_ratio=0.860683 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.513437508e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54646 payload_rows_remote=48652 dup_rows=10890 dup_rows_remote=10890 dedup_ratio=0.166168 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2561, 5, 0, 4, 0, 0, 0, 0, 0, 0, 16, 0, 2990, 0, 0, 0, 0, 7, 0, 0, 2, 0, 2032, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 14, 0, 0, 4070, 0, 2480, 1822, 0, 0, 0, 0, 385, 0, 0] payload_rows_per_dst=[8186, 5384, 8192, 5994, 7812, 6704, 8190, 4184] dup_rows_per_dst=[0, 1234, 40, 0, 3726, 722, 4938, 230] combine_rows_per_src=[8186, 5384, 8192, 5994, 7812, 6704, 8190, 4184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311492608 dispatch_meta_bytes=229320 combine_payload_bytes=311492608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311721928 ingress_bytes=421593088 link_bytes=421593088 token_bytes=623214536 weight_movement_bytes=110100480 total_nvlink_bytes=733315016 weight_to_token_ratio=0.176665 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.332197202e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54082 payload_rows_remote=49696 dup_rows=11454 dup_rows_remote=11454 dedup_ratio=0.174774 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3131, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4021, 0, 1370, 2626, 0, 0, 0, 0, 537, 0, 0] payload_rows_per_dst=[8190, 4970, 8192, 6268, 4386, 8192, 8148, 5736] dup_rows_per_dst=[0, 6, 32, 0, 0, 8192, 2634, 590] combine_rows_per_src=[8190, 4970, 8192, 6268, 4386, 8192, 8148, 5736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50408 payload_rows_remote=46944 dup_rows=15128 dup_rows_remote=15128 dedup_ratio=0.230835 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4072, 0, 1732, 2895, 0, 0, 0, 0, 582, 0, 0] payload_rows_per_dst=[8192, 4924, 8192, 5362, 3900, 8192, 3464, 8182] dup_rows_per_dst=[0, 2, 22, 0, 0, 8188, 0, 6916] combine_rows_per_src=[8192, 4924, 8192, 5362, 3900, 8192, 3464, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=348737536 dispatch_meta_bytes=238168 combine_payload_bytes=348737536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348975704 ingress_bytes=458838016 link_bytes=458838016 token_bytes=697713240 weight_movement_bytes=110100480 total_nvlink_bytes=807813720 weight_to_token_ratio=0.157802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.979947757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55428 payload_rows_remote=47290 dup_rows=10108 dup_rows_remote=4390 dedup_ratio=0.154236 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2640, 8, 0, 2, 0, 0, 0, 0, 0, 0, 27, 0, 3112, 0, 0, 0, 0, 4, 0, 0, 1, 0, 1851, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 11, 0, 0, 4002, 0, 1802, 2267, 0, 0, 0, 0, 659, 0, 0] payload_rows_per_dst=[8186, 5282, 7978, 6228, 3704, 8192, 7720, 8138] dup_rows_per_dst=[2, 14, 58, 4, 0, 750, 3562, 5718] combine_rows_per_src=[8186, 5282, 7978, 6228, 3704, 8192, 7720, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=356220928 dispatch_meta_bytes=244600 combine_payload_bytes=356220928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356465528 ingress_bytes=466321408 link_bytes=466321408 token_bytes=712686456 weight_movement_bytes=110100480 total_nvlink_bytes=822786936 weight_to_token_ratio=0.154487 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.110096252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50388 payload_rows_remote=42204 dup_rows=15148 dup_rows_remote=9954 dedup_ratio=0.231140 router_tokens_per_expert=[1, 0, 5, 4085, 0, 0, 1, 0, 2, 0, 0, 0, 1, 2588, 0, 8, 0, 12, 2, 0, 0, 1, 0, 3, 0, 3069, 2, 0, 4, 0, 4, 2, 0, 0, 1, 1594, 0, 0, 0, 0, 2, 0, 4090, 5, 4091, 0, 4091, 5, 0, 0, 0, 0, 0, 4028, 0, 2287, 1992, 0, 0, 0, 2, 790, 0, 0] payload_rows_per_dst=[8174, 8184, 8188, 8158, 8188, 10, 4574, 4912] dup_rows_per_dst=[10, 5194, 30, 6060, 3184, 14, 0, 656] combine_rows_per_src=[8174, 8184, 8188, 8158, 8188, 10, 4574, 4912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49802 payload_rows_remote=41610 dup_rows=15734 dup_rows_remote=15716 dedup_ratio=0.240082 router_tokens_per_expert=[0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 1, 0, 2, 0, 0, 0, 0, 0, 9, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4045, 0, 2354, 1345, 0, 0, 0, 0, 971, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8176, 0, 4708, 4164] dup_rows_per_dst=[18, 4296, 22, 6248, 4682, 0, 0, 468] combine_rows_per_src=[8192, 8192, 8192, 8178, 8176, 0, 4708, 4164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=336494592 dispatch_meta_bytes=248288 combine_payload_bytes=336494592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=336742880 ingress_bytes=666796032 link_bytes=666796032 token_bytes=673237472 weight_movement_bytes=330301440 total_nvlink_bytes=1003538912 weight_to_token_ratio=0.490617 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.159667969e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59688 payload_rows_remote=51496 dup_rows=5848 dup_rows_remote=5846 dedup_ratio=0.089233 router_tokens_per_expert=[2550, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4095, 0, 392, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1581, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3090, 0, 4091, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5100, 8016, 8192, 8158, 7184, 8192, 6654, 8192] dup_rows_per_dst=[0, 852, 70, 784, 2404, 2, 1292, 444] combine_rows_per_src=[5100, 8016, 8192, 8158, 7184, 8192, 6654, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60222 payload_rows_remote=52034 dup_rows=5314 dup_rows_remote=5106 dedup_ratio=0.081085 router_tokens_per_expert=[2989, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4090, 0, 365, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1428, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 4090, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6088, 8000, 8188, 8150, 7898, 8192, 5520, 8186] dup_rows_per_dst=[240, 1072, 208, 722, 2798, 2, 0, 272] combine_rows_per_src=[6088, 8000, 8188, 8150, 7898, 8192, 5520, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=338974720 dispatch_meta_bytes=206720 combine_payload_bytes=338974720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339181440 ingress_bytes=449075200 link_bytes=449075200 token_bytes=678156160 weight_movement_bytes=110100480 total_nvlink_bytes=788256640 weight_to_token_ratio=0.162353 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.810156330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=302518272 dispatch_meta_bytes=208632 combine_payload_bytes=302518272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302726904 ingress_bytes=412618752 link_bytes=412618752 token_bytes=605245176 weight_movement_bytes=110100480 total_nvlink_bytes=715345656 weight_to_token_ratio=0.181911 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.176118739e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298260480 dispatch_meta_bytes=229304 combine_payload_bytes=298260480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298489784 ingress_bytes=408360960 link_bytes=408360960 token_bytes=596750264 weight_movement_bytes=110100480 total_nvlink_bytes=706850744 weight_to_token_ratio=0.184500 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.102068733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369123328 dispatch_meta_bytes=229368 combine_payload_bytes=369123328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369352696 ingress_bytes=479223808 link_bytes=479223808 token_bytes=738476024 weight_movement_bytes=110100480 total_nvlink_bytes=848576504 weight_to_token_ratio=0.149091 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.334490209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60204 payload_rows_remote=52018 dup_rows=5332 dup_rows_remote=4812 dedup_ratio=0.081360 router_tokens_per_expert=[2685, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 364, 5, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4079, 0, 260, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1839, 7, 0, 0, 1, 0, 0, 0, 0, 0, 2904, 0, 4095, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5370, 8084, 8184, 8186, 8192, 8190, 5808, 8190] dup_rows_per_dst=[0, 598, 186, 520, 3678, 12, 0, 338] combine_rows_per_src=[5370, 8084, 8184, 8186, 8192, 8190, 5808, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59334 payload_rows_remote=51142 dup_rows=6202 dup_rows_remote=2212 dedup_ratio=0.094635 router_tokens_per_expert=[2447, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4086, 0, 512, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1995, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 0, 4095, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4894, 8142, 8188, 8176, 8192, 8192, 5360, 8190] dup_rows_per_dst=[0, 922, 52, 1020, 3990, 4, 0, 214] combine_rows_per_src=[4894, 8142, 8188, 8176, 8192, 8192, 5360, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372979712 dispatch_meta_bytes=228560 combine_payload_bytes=372979712 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=373208272 ingress_bytes=483080192 link_bytes=483080192 token_bytes=746187984 weight_movement_bytes=110100480 total_nvlink_bytes=856288464 weight_to_token_ratio=0.147551 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.401559069e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57412 payload_rows_remote=49220 dup_rows=8124 dup_rows_remote=2804 dedup_ratio=0.123962 router_tokens_per_expert=[2101, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4088, 0, 954, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 4096, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4202, 8146, 8192, 8180, 4116, 8192, 8192, 8192] dup_rows_per_dst=[0, 742, 76, 1904, 0, 0, 5320, 82] combine_rows_per_src=[4202, 8146, 8192, 8180, 4116, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7068450224 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3535343024 ingress_bytes=5514915840 link_bytes=5514915840 total_nvlink_bytes=9050258864 weight_to_token_ratio=0.280374 prefetch_share_of_ingress=0.159713 dedup_ratio=0.135178 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045221 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.591345694e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372865024 dispatch_meta_bytes=227320 combine_payload_bytes=372865024 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=373092344 ingress_bytes=703166464 link_bytes=703166464 token_bytes=745957368 weight_movement_bytes=330301440 total_nvlink_bytes=1076258808 weight_to_token_ratio=0.442789 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.222922132e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=366585856 dispatch_meta_bytes=213416 combine_payload_bytes=366585856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=366799272 ingress_bytes=696887296 link_bytes=696887296 token_bytes=733385128 weight_movement_bytes=330301440 total_nvlink_bytes=1063686568 weight_to_token_ratio=0.450379 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.212001626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57402 payload_rows_remote=49304 dup_rows=8134 dup_rows_remote=7414 dedup_ratio=0.124115 router_tokens_per_expert=[2854, 0, 0, 4, 0, 3927, 0, 0, 0, 3, 0, 479, 0, 0, 0, 0, 0, 0, 1, 0, 4, 58, 4035, 0, 467, 4086, 4090, 1, 3, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 1462, 0, 1, 4, 0, 1, 0, 0, 0, 0, 2759, 0, 4091, 0, 2, 0, 4, 0, 331, 2, 0, 0, 5, 2, 0] payload_rows_per_dst=[8190, 8098, 8102, 8182, 2924, 8188, 5526, 8192] dup_rows_per_dst=[5708, 720, 94, 934, 0, 4, 4, 670] combine_rows_per_src=[8190, 8098, 8102, 8182, 2924, 8188, 5526, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55786 payload_rows_remote=47598 dup_rows=9750 dup_rows_remote=9368 dedup_ratio=0.148773 router_tokens_per_expert=[3212, 0, 0, 0, 0, 3882, 0, 0, 0, 0, 0, 270, 5, 0, 0, 0, 0, 0, 0, 0, 0, 107, 4058, 0, 316, 4070, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1533, 7, 0, 0, 2, 0, 0, 0, 0, 0, 2832, 0, 4089, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6424, 7880, 7376, 1678, 7856, 8192, 8192, 8188] dup_rows_per_dst=[0, 434, 138, 110, 3010, 14, 5662, 382] combine_rows_per_src=[6424, 7880, 7376, 1678, 7856, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7070728200 prefetch_bytes=1211105280 reduce_bytes=220200960 weight_movement_bytes=1431306240 egress_bytes=3536517128 ingress_bytes=4965517312 link_bytes=4965517312 total_nvlink_bytes=8502034440 weight_to_token_ratio=0.202427 prefetch_share_of_ingress=0.243903 dedup_ratio=0.139081 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=2 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.635851293e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51304 payload_rows_remote=43112 dup_rows=14232 dup_rows_remote=3104 dedup_ratio=0.217163 router_tokens_per_expert=[2916, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4087, 0, 332, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1647, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 4094, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3610, 8192, 8134, 3294, 8160, 6102, 5620] dup_rows_per_dst=[11128, 220, 98, 698, 0, 0, 1904, 184] combine_rows_per_src=[8192, 3610, 8192, 8134, 3294, 8160, 6102, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=352808960 dispatch_meta_bytes=208096 combine_payload_bytes=352808960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=353017056 ingress_bytes=573009920 link_bytes=573009920 token_bytes=705826016 weight_movement_bytes=220200960 total_nvlink_bytes=926026976 weight_to_token_ratio=0.311976 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.965584948e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7286355240 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3644337448 ingress_bytes=5733926912 link_bytes=5733926912 total_nvlink_bytes=9378264360 weight_to_token_ratio=0.287100 prefetch_share_of_ingress=0.192016 dedup_ratio=0.120175 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.972241969e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7199735032 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3601011960 ingress_bytes=6020933632 link_bytes=6020933632 total_nvlink_bytes=9621945592 weight_to_token_ratio=0.336431 prefetch_share_of_ingress=0.164577 dedup_ratio=0.124466 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046324 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.047139386e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=353411072 dispatch_meta_bytes=226872 combine_payload_bytes=353411072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353637944 ingress_bytes=463511552 link_bytes=463511552 token_bytes=707049016 weight_movement_bytes=110100480 total_nvlink_bytes=817149496 weight_to_token_ratio=0.155718 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.061228235e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=341182464 dispatch_meta_bytes=227864 combine_payload_bytes=341182464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341410328 ingress_bytes=451282944 link_bytes=451282944 token_bytes=682592792 weight_movement_bytes=110100480 total_nvlink_bytes=792693272 weight_to_token_ratio=0.161297 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.848552629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=309026816 dispatch_meta_bytes=184864 combine_payload_bytes=309026816 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=309211680 ingress_bytes=529227776 link_bytes=529227776 token_bytes=618238496 weight_movement_bytes=220200960 total_nvlink_bytes=838439456 weight_to_token_ratio=0.356175 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.204141455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7011497080 prefetch_bytes=990904320 reduce_bytes=2312110080 weight_movement_bytes=3303014400 egress_bytes=3506875512 ingress_bytes=6807635968 link_bytes=6807635968 total_nvlink_bytes=10314511480 weight_to_token_ratio=0.471085 prefetch_share_of_ingress=0.145558 dedup_ratio=0.145251 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=21 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.183959861e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6794895128 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=3398567704 ingress_bytes=4607432704 link_bytes=4607432704 total_nvlink_bytes=8006000408 weight_to_token_ratio=0.178238 prefetch_share_of_ingress=0.215066 dedup_ratio=0.158499 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045956 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.013083265e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6449187016 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3225680072 ingress_bytes=5315416064 link_bytes=5315416064 total_nvlink_bytes=8541096136 weight_to_token_ratio=0.324368 prefetch_share_of_ingress=0.186421 dedup_ratio=0.195264 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.244382771e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6400034920 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3201114216 ingress_bytes=4960528384 link_bytes=4960528384 total_nvlink_bytes=8161642600 weight_to_token_ratio=0.275250 prefetch_share_of_ingress=0.199758 dedup_ratio=0.199463 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.627174726e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-29 23:43:06] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 240.5 | throughput per GPU (TFLOP/s/GPU): 513.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.574734E+00 | loss scale: 1.0 | grad norm: 4.335 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:43:06 +[rank0]:[W829 23:43:07.985973958 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W829 23:43:08.976810349 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W829 23:43:09.543141191 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W829 23:43:09.562418665 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W829 23:43:09.698547915 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W829 23:43:09.900405586 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W829 23:43:09.154706267 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W829 23:43:09.182571317 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:43:15.115000 231719 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:43:15.116000 231719 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 231903 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 231719 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..bce19a73f97c40bc57fd9043e485a924fd500179 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -0,0 +1,96 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 4 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 821 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1919 +1, 1919 +2, 1919 +3, 1919 +4, 1919 +5, 1975 +6, 1975 +7, 1919 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..5fa98a8d773f51ffddedbd36c42f68c64170b5cd --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..cc96ee36121f81dd440a35385df953b0cce283b1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log @@ -0,0 +1,752 @@ +[23:43:17] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + QWEN2_57B / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:43:27.686000 234329 torch/distributed/run.py:766] +W0829 23:43:27.686000 234329 torch/distributed/run.py:766] ***************************************** +W0829 23:43:27.686000 234329 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:43:27.686000 234329 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 01 is connected to is connected to 7[Gloo] Rank 7 peer ranks. peer ranks. Expected number of connected peer ranks is : 2Expected number of connected peer ranks is : 7 is connected to 77 + + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 01324 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + + +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to [Gloo] Rank 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. [Gloo] Rank 0Expected number of connected peer ranks is : peer ranks. 00Expected number of connected peer ranks is : is connected to +00 + peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W829 23:43:46.690436463 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:43:46.690442828 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:43:46.690509297 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:43:46.690565089 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W829 23:43:46.690608406 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:43:46.690613276 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:43:46.692216962 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:43:46.847415590 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.763 seconds +time to initialize megatron (seconds): 7.415 +[after megatron is initialized] datetime: 2026-08-29 23:43:52 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv index 46566f5ddc1f8978815beeba92a2c0253a808999..8f0de9af166e214f8fe973184a57fbccab903bb4 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv @@ -46,3 +46,219 @@ 5, 1 6, 1 7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 45 +2, 35 +3, 629 +4, 629 +5, 35 +6, 629 +7, 629 +0, 939 +1, 949 +2, 941 +3, 949 +4, 977 +5, 951 +6, 959 +7, 959 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2135 +1, 2135 +2, 2135 +3, 1783 +4, 1783 +5, 1783 +6, 1783 +7, 1783 +0, 46689 +1, 46703 +2, 46687 +3, 46675 +4, 46679 +5, 46715 +6, 46693 +7, 46709 +0, 47547 +1, 47549 +2, 47549 +3, 47549 +4, 47549 +5, 47549 +6, 47549 +7, 47549 +0, 48021 +1, 47845 +2, 48027 +3, 47871 +4, 47929 +5, 48093 +6, 47905 +7, 48083 +0, 49085 +1, 48741 +2, 49113 +3, 48743 +4, 48879 +5, 49113 +6, 48741 +7, 49115 +0, 49231 +1, 48889 +2, 49259 +3, 48889 +4, 49025 +5, 49259 +6, 48887 +7, 49261 +0, 49233 +1, 48889 +2, 49261 +3, 48889 +4, 49025 +5, 49261 +6, 48889 +7, 49261 +0, 59143 +1, 59125 +2, 59135 +3, 59147 +4, 59145 +5, 59135 +6, 59147 +7, 59135 +0, 83553 +1, 83539 +2, 83571 +3, 83557 +4, 83563 +5, 83577 +6, 83563 +7, 83577 +0, 83949 +1, 83933 +2, 83967 +3, 83953 +4, 83953 +5, 83967 +6, 83953 +7, 83967 +0, 84199 +1, 84175 +2, 84183 +3, 84225 +4, 84205 +5, 84319 +6, 84241 +7, 84253 +0, 84999 +1, 84983 +2, 85017 +3, 85003 +4, 85003 +5, 85017 +6, 85003 +7, 85017 +0, 95481 +1, 95423 +2, 95525 +3, 95335 +4, 95437 +5, 95557 +6, 95407 +7, 95533 +0, 116419 +1, 116321 +2, 116403 +3, 116243 +4, 116329 +5, 116403 +6, 116291 +7, 116403 +0, 116419 +1, 116321 +2, 116403 +3, 116243 +4, 116329 +5, 116403 +6, 116291 +7, 116403 +0, 116419 +1, 116321 +2, 116403 +3, 116243 +4, 116329 +5, 116403 +6, 116291 +7, 116403 +0, 116419 +1, 116321 +2, 116403 +3, 116243 +4, 116329 +5, 116403 +6, 116291 +7, 116403 +0, 116419 +1, 116321 +2, 116403 +3, 115981 +4, 116009 +5, 116403 +6, 116291 +7, 116403 +0, 114981 +1, 114883 +2, 114965 +3, 114543 +4, 114891 +5, 114965 +6, 114853 +7, 114965 +0, 113297 +1, 113849 +2, 113931 +3, 112795 +4, 112887 +5, 113223 +6, 113819 +7, 113543 +0, 111701 +1, 111747 +2, 112027 +3, 64 +4, 4 +5, 110581 +6, 111991 +7, 112137 +0, 1 +1, 13 +2, 112024 +3, 61 +4, 1 +5, 1 +6, 1 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log index 977ae41c4e74a66cfde0779d33b30cf8ff60f914..d2261b013705679f21f943a0bd5683e10669aab6 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log @@ -12,3 +12,14329 @@ W0829 23:36:47.546000 219093 torch/distributed/run.py:766] W0829 23:36:47.546000 219093 torch/distributed/run.py:766] ***************************************** W0829 23:36:47.546000 219093 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0829 23:36:47.546000 219093 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7[Gloo] Rank +7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2[Gloo] Rank 0 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[rank4]:[W829 23:37:06.275140939 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:37:06.275194312 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:37:06.275215320 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:37:06.275217458 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:37:06.275228269 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:37:06.275327187 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W829 23:37:06.277037976 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:37:06.432536877 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.557 seconds +time to initialize megatron (seconds): 7.763 +[after megatron is initialized] datetime: 2026-08-29 23:37:12 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.45.mlp.router.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.q_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.25.pre_mlp_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.linear_proj.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:37:17 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:37:17 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5189.14, 5190.78) + train/valid/test-data-iterators-setup ..........: (62.65, 69.32) +[before the start of training step] datetime: 2026-08-29 23:37:17 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 144.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 147.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [0, 2048, 1408, 1536, 3968, 2944, 4224, 896, 2816, 1152, 2432, 2176, 896, 2560, 1152, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [110, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [71, 77, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-29 23:37:40] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22957.3 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.362 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66585.4775390625 | max allocated: 66585.47802734375 | reserved: 68254.0 | max reserved: 68254.0 + [2026-08-29 23:37:41] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 969.3 | throughput per GPU (TFLOP/s/GPU): 91.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.508 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43996 payload_rows_remote=37914 dup_rows=21540 dup_rows_remote=17550 dedup_ratio=0.328674 router_tokens_per_expert=[747, 82, 71, 71, 1072, 586, 630, 56, 93, 83, 271, 376, 90, 218, 198, 392, 157, 83, 294, 300, 250, 408, 383, 70, 459, 300, 202, 176, 130, 436, 193, 373, 470, 191, 130, 197, 183, 134, 393, 280, 199, 62, 35, 556, 468, 24, 72, 134, 145, 251, 67, 163, 198, 263, 255, 73, 78, 112, 135, 273, 213, 472, 76, 75, 490, 308, 145, 75, 194, 76, 200, 709, 123, 170, 272, 308, 43, 453, 393, 962, 189, 113, 249, 460, 226, 209, 124, 35, 81, 120, 386, 393, 166, 401, 165, 57, 16, 255, 294, 114, 302, 445, 655, 153, 275, 254, 212, 454, 440, 387, 359, 146, 500, 715, 271, 124, 397, 93, 390, 150, 111, 544, 189, 250, 41, 77, 210, 23] payload_rows_per_dst=[6082, 6048, 5700, 5910, 4860, 4736, 5212, 5448] dup_rows_per_dst=[3990, 3260, 2666, 3130, 1640, 2012, 2120, 2722] combine_rows_per_src=[6082, 6048, 5700, 5910, 4860, 4736, 5212, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38550 dup_rows=21202 dup_rows_remote=18164 dedup_ratio=0.323517 router_tokens_per_expert=[770, 134, 103, 101, 993, 403, 671, 82, 236, 76, 246, 329, 87, 226, 188, 497, 116, 136, 245, 321, 224, 273, 278, 82, 413, 302, 212, 206, 120, 332, 137, 373, 451, 246, 179, 171, 197, 193, 367, 226, 218, 105, 52, 408, 311, 24, 145, 311, 137, 268, 92, 167, 200, 196, 220, 141, 98, 150, 183, 258, 145, 538, 93, 68, 646, 362, 153, 94, 154, 75, 241, 744, 145, 200, 219, 363, 50, 556, 445, 779, 284, 116, 175, 308, 187, 239, 124, 71, 111, 156, 471, 398, 181, 271, 240, 86, 25, 338, 322, 86, 299, 445, 721, 180, 206, 306, 215, 409, 406, 327, 460, 124, 423, 726, 261, 84, 502, 95, 361, 124, 154, 311, 168, 194, 50, 44, 228, 60] payload_rows_per_dst=[5676, 5332, 5086, 5358, 6032, 5784, 5624, 5442] dup_rows_per_dst=[2622, 2208, 2130, 2108, 2862, 3038, 3216, 3018] combine_rows_per_src=[5676, 5332, 5086, 5358, 6032, 5784, 5624, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38640 dup_rows=21000 dup_rows_remote=18386 dedup_ratio=0.320435 router_tokens_per_expert=[736, 148, 94, 101, 704, 354, 634, 63, 201, 109, 280, 424, 162, 283, 146, 493, 158, 77, 338, 339, 257, 233, 310, 109, 346, 400, 163, 222, 82, 306, 171, 404, 532, 223, 115, 177, 216, 159, 230, 342, 258, 101, 60, 357, 380, 38, 118, 288, 220, 230, 86, 201, 106, 272, 139, 71, 80, 116, 185, 226, 247, 362, 72, 88, 510, 307, 148, 123, 173, 94, 242, 644, 182, 204, 279, 343, 54, 639, 355, 983, 275, 134, 154, 278, 245, 151, 156, 34, 121, 161, 447, 575, 92, 344, 182, 79, 17, 260, 275, 125, 316, 589, 661, 222, 250, 232, 221, 398, 464, 348, 451, 156, 504, 663, 281, 153, 424, 114, 346, 159, 129, 455, 202, 183, 46, 51, 173, 50] payload_rows_per_dst=[5580, 5362, 5896, 5818, 5260, 5478, 5828, 5314] dup_rows_per_dst=[2876, 2468, 2614, 2838, 2046, 2786, 2820, 2552] combine_rows_per_src=[5580, 5362, 5896, 5818, 5260, 5478, 5828, 5314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44364 payload_rows_remote=38604 dup_rows=21172 dup_rows_remote=18804 dedup_ratio=0.323059 router_tokens_per_expert=[1012, 228, 77, 74, 960, 390, 521, 79, 83, 69, 241, 244, 179, 276, 247, 456, 129, 85, 319, 413, 260, 262, 384, 115, 426, 424, 154, 180, 101, 225, 184, 350, 450, 308, 127, 156, 154, 100, 359, 316, 162, 99, 66, 424, 654, 57, 90, 356, 105, 290, 59, 144, 134, 273, 210, 78, 72, 134, 167, 272, 198, 470, 99, 85, 679, 325, 176, 84, 174, 101, 221, 639, 177, 155, 368, 309, 31, 368, 328, 927, 211, 93, 166, 354, 261, 262, 181, 32, 101, 138, 445, 374, 74, 323, 157, 27, 18, 283, 244, 129, 293, 346, 620, 107, 305, 295, 204, 341, 376, 309, 486, 109, 634, 564, 256, 99, 323, 92, 426, 143, 166, 696, 187, 266, 44, 62, 233, 36] payload_rows_per_dst=[5582, 5426, 5998, 5452, 5760, 5558, 5080, 5508] dup_rows_per_dst=[2770, 2596, 2998, 2124, 2368, 2760, 2382, 3174] combine_rows_per_src=[5582, 5426, 5998, 5452, 5760, 5558, 5080, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155295744 dispatch_meta_bytes=221856 combine_payload_bytes=155295744 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=155517600 ingress_bytes=211918848 link_bytes=211918848 token_bytes=310813344 weight_movement_bytes=56623104 total_nvlink_bytes=367436448 weight_to_token_ratio=0.182177 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=4864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.030502131e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44252 payload_rows_remote=38500 dup_rows=21284 dup_rows_remote=18336 dedup_ratio=0.324768 router_tokens_per_expert=[722, 157, 116, 93, 918, 364, 735, 71, 91, 85, 280, 369, 185, 218, 130, 435, 144, 71, 398, 358, 301, 276, 216, 114, 359, 339, 184, 187, 89, 295, 202, 354, 416, 281, 117, 303, 205, 198, 423, 228, 169, 114, 54, 411, 331, 32, 121, 293, 144, 256, 57, 191, 133, 311, 183, 82, 56, 93, 220, 282, 190, 459, 50, 99, 637, 404, 153, 98, 144, 88, 215, 668, 112, 210, 366, 371, 41, 671, 241, 876, 158, 107, 99, 373, 244, 211, 115, 40, 92, 147, 493, 485, 130, 328, 156, 84, 21, 294, 315, 87, 291, 518, 655, 161, 226, 257, 166, 283, 446, 443, 383, 147, 570, 742, 242, 131, 392, 107, 449, 144, 149, 513, 200, 127, 54, 59, 246, 35] payload_rows_per_dst=[5474, 5214, 5956, 5752, 5306, 5602, 5436, 5512] dup_rows_per_dst=[2628, 2560, 2746, 2948, 2196, 2758, 2640, 2808] combine_rows_per_src=[5474, 5214, 5956, 5752, 5306, 5602, 5436, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44150 payload_rows_remote=38682 dup_rows=21386 dup_rows_remote=18422 dedup_ratio=0.326324 router_tokens_per_expert=[842, 211, 93, 82, 920, 433, 647, 72, 106, 139, 271, 315, 130, 364, 227, 424, 93, 96, 284, 417, 261, 357, 299, 99, 396, 382, 240, 216, 115, 228, 195, 388, 486, 315, 187, 178, 167, 196, 460, 306, 246, 136, 44, 231, 377, 44, 111, 254, 104, 254, 71, 169, 175, 267, 223, 91, 127, 159, 179, 277, 218, 419, 122, 105, 660, 309, 150, 115, 174, 96, 223, 636, 131, 198, 273, 315, 29, 563, 415, 923, 205, 90, 146, 191, 246, 189, 169, 40, 130, 147, 412, 562, 91, 368, 164, 78, 40, 345, 232, 145, 313, 465, 662, 149, 236, 273, 142, 283, 421, 337, 352, 144, 520, 602, 242, 104, 351, 105, 411, 125, 103, 426, 170, 182, 18, 97, 261, 34] payload_rows_per_dst=[4810, 5534, 5234, 5496, 6246, 5472, 5890, 5468] dup_rows_per_dst=[1582, 2598, 2242, 2270, 4012, 2824, 2894, 2964] combine_rows_per_src=[4810, 5534, 5234, 5496, 6246, 5472, 5890, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44282 payload_rows_remote=38534 dup_rows=21254 dup_rows_remote=18184 dedup_ratio=0.324310 router_tokens_per_expert=[829, 145, 79, 85, 1040, 332, 575, 66, 135, 59, 201, 221, 139, 282, 243, 515, 119, 100, 360, 378, 226, 402, 258, 115, 432, 292, 209, 295, 71, 222, 201, 439, 457, 329, 158, 190, 251, 301, 350, 284, 216, 135, 70, 326, 431, 35, 109, 294, 164, 316, 108, 177, 105, 310, 146, 77, 96, 146, 149, 196, 164, 443, 114, 100, 582, 278, 188, 114, 193, 59, 184, 631, 155, 209, 369, 275, 33, 587, 282, 908, 289, 111, 114, 289, 228, 237, 141, 43, 84, 147, 466, 561, 76, 344, 182, 44, 38, 312, 337, 107, 270, 580, 690, 172, 186, 277, 161, 282, 448, 267, 465, 112, 460, 840, 284, 116, 280, 79, 364, 157, 105, 543, 169, 146, 24, 42, 194, 46] payload_rows_per_dst=[5556, 5748, 6164, 5748, 5080, 5432, 5206, 5348] dup_rows_per_dst=[2886, 3070, 3088, 2952, 1936, 2730, 2242, 2350] combine_rows_per_src=[5556, 5748, 6164, 5748, 5080, 5432, 5206, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157900800 dispatch_meta_bytes=226856 combine_payload_bytes=157900800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158127656 ingress_bytes=176775168 link_bytes=176775168 token_bytes=316028456 weight_movement_bytes=18874368 total_nvlink_bytes=334902824 weight_to_token_ratio=0.059724 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=6144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693772681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158269440 dispatch_meta_bytes=228104 combine_payload_bytes=158269440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158497544 ingress_bytes=177143808 link_bytes=177143808 token_bytes=316766984 weight_movement_bytes=18874368 total_nvlink_bytes=335641352 weight_to_token_ratio=0.059584 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=3712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.697304808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158121984 dispatch_meta_bytes=229632 combine_payload_bytes=158121984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158351616 ingress_bytes=214745088 link_bytes=214745088 token_bytes=316473600 weight_movement_bytes=56623104 total_nvlink_bytes=373096704 weight_to_token_ratio=0.178919 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=4736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.057581772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157696000 dispatch_meta_bytes=227344 combine_payload_bytes=157696000 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=157923344 ingress_bytes=195444736 link_bytes=195444736 token_bytes=315619344 weight_movement_bytes=37748736 total_nvlink_bytes=353368080 weight_to_token_ratio=0.119602 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.076389 gemm_active_groups=19 gemm_peak_group_rows=7552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872655296e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158441472 dispatch_meta_bytes=228416 combine_payload_bytes=158441472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158669888 ingress_bytes=177315840 link_bytes=177315840 token_bytes=317111360 weight_movement_bytes=18874368 total_nvlink_bytes=335985728 weight_to_token_ratio=0.059520 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=5504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.698953134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42098 payload_rows_remote=34694 dup_rows=23438 dup_rows_remote=18302 dedup_ratio=0.357635 router_tokens_per_expert=[0, 65, 137, 0, 2198, 0, 0, 212, 1, 5, 0, 4, 104, 0, 0, 1166, 199, 0, 28, 0, 0, 711, 0, 0, 60, 1, 10, 1659, 3, 20, 47, 53, 0, 0, 918, 109, 3130, 0, 1, 0, 62, 0, 0, 2984, 4, 51, 8, 0, 0, 5, 42, 1748, 17, 204, 16, 15, 35, 0, 0, 205, 36, 13, 46, 209, 9, 98, 16, 106, 0, 120, 2, 0, 55, 24, 0, 8, 48, 8, 93, 0, 257, 1, 0, 5, 34, 179, 4, 319, 51, 0, 39, 2378, 342, 2570, 36, 451, 31, 4, 0, 0, 4, 2, 3, 0, 0, 296, 0, 0, 409, 72, 3020, 0, 2, 3025, 1, 0, 0, 10, 1, 0, 1, 0, 0, 272, 1798, 4, 17, 2] payload_rows_per_dst=[7404, 5086, 8018, 4958, 5674, 2586, 7756, 616] dup_rows_per_dst=[5136, 854, 9246, 1090, 640, 492, 5976, 4] combine_rows_per_src=[7404, 5086, 8018, 4958, 5674, 2586, 7756, 616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157835264 dispatch_meta_bytes=226872 combine_payload_bytes=157835264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158062136 ingress_bytes=176709632 link_bytes=176709632 token_bytes=315897400 weight_movement_bytes=18874368 total_nvlink_bytes=334771768 weight_to_token_ratio=0.059748 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=3328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693144747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=142106624 dispatch_meta_bytes=211984 combine_payload_bytes=142106624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=142318608 ingress_bytes=160980992 link_bytes=160980992 token_bytes=284425232 weight_movement_bytes=18874368 total_nvlink_bytes=303299600 weight_to_token_ratio=0.066360 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=11904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.542440658e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48492 payload_rows_remote=42826 dup_rows=17044 dup_rows_remote=15092 dedup_ratio=0.260071 router_tokens_per_expert=[0, 23, 13, 0, 1208, 0, 0, 389, 9, 7, 0, 0, 226, 0, 3, 1524, 423, 2, 22, 0, 0, 899, 0, 0, 273, 11, 17, 2730, 24, 43, 118, 201, 3, 0, 1014, 193, 1696, 0, 38, 5, 85, 8, 1, 2784, 1, 58, 1, 0, 2, 0, 19, 446, 16, 192, 58, 10, 134, 11, 8, 140, 108, 66, 143, 283, 5, 47, 52, 87, 3, 171, 22, 1, 16, 26, 1, 11, 41, 5, 44, 0, 815, 27, 2, 12, 33, 442, 11, 543, 247, 8, 46, 2834, 1205, 2441, 53, 365, 40, 22, 0, 30, 35, 26, 0, 0, 0, 283, 0, 1, 577, 152, 2145, 0, 5, 1987, 9, 0, 1, 7, 0, 1, 0, 1, 1, 627, 1458, 15, 37, 4] payload_rows_per_dst=[5442, 6770, 5104, 6754, 7136, 5666, 5390, 6230] dup_rows_per_dst=[1362, 2756, 1102, 2086, 4478, 1952, 1232, 2076] combine_rows_per_src=[5442, 6770, 5104, 6754, 7136, 5666, 5390, 6230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49060 payload_rows_remote=41928 dup_rows=16476 dup_rows_remote=14636 dedup_ratio=0.251404 router_tokens_per_expert=[0, 47, 93, 16, 2142, 0, 0, 150, 6, 20, 0, 2, 113, 6, 2, 585, 236, 0, 134, 0, 0, 834, 1, 0, 285, 2, 169, 2455, 3, 11, 68, 607, 0, 0, 944, 235, 3114, 0, 6, 2, 26, 0, 0, 2279, 3, 155, 1, 0, 0, 0, 8, 447, 33, 116, 54, 11, 78, 10, 0, 623, 59, 8, 78, 66, 0, 96, 75, 169, 0, 228, 12, 0, 15, 12, 0, 24, 33, 2, 63, 2, 563, 3, 1, 1, 34, 1612, 4, 346, 164, 4, 47, 1727, 521, 2000, 134, 242, 105, 20, 0, 13, 50, 8, 2, 0, 3, 125, 0, 6, 320, 318, 2522, 3, 2, 2156, 3, 4, 0, 25, 3, 6, 1, 6, 0, 357, 2184, 2, 47, 0] payload_rows_per_dst=[6678, 6634, 7132, 5830, 4762, 5414, 5772, 6838] dup_rows_per_dst=[3140, 2976, 1840, 1910, 700, 1938, 1218, 2754] combine_rows_per_src=[6678, 6634, 7132, 5830, 4762, 5414, 5772, 6838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45510 payload_rows_remote=37594 dup_rows=20026 dup_rows_remote=13022 dedup_ratio=0.305573 router_tokens_per_expert=[0, 42, 0, 0, 1386, 0, 0, 335, 1, 19, 2, 13, 12, 10, 0, 761, 43, 0, 1, 0, 0, 844, 0, 0, 92, 33, 25, 2766, 14, 68, 83, 653, 0, 0, 445, 320, 820, 0, 3, 0, 5, 0, 0, 2652, 0, 95, 1, 3, 0, 1, 13, 302, 33, 241, 87, 0, 44, 28, 1, 542, 28, 2, 180, 69, 1, 148, 3, 444, 0, 157, 0, 0, 0, 107, 0, 27, 27, 1, 66, 2, 606, 0, 0, 0, 1, 1059, 18, 495, 10, 3, 9, 3727, 370, 2750, 21, 36, 1, 25, 0, 0, 43, 13, 0, 0, 0, 579, 0, 1, 157, 50, 2502, 0, 21, 2886, 8, 0, 0, 179, 1, 0, 8, 0, 0, 105, 2969, 4, 10, 0] payload_rows_per_dst=[4376, 6630, 3038, 6246, 7916, 4086, 5432, 7786] dup_rows_per_dst=[786, 2614, 346, 2200, 7004, 1170, 1310, 4596] combine_rows_per_src=[4376, 6630, 3038, 6246, 7916, 4086, 5432, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50090 payload_rows_remote=42934 dup_rows=15446 dup_rows_remote=13170 dedup_ratio=0.235687 router_tokens_per_expert=[0, 70, 5, 0, 627, 0, 0, 328, 0, 7, 0, 16, 11, 1, 0, 1017, 354, 0, 18, 1, 0, 778, 0, 0, 219, 1, 30, 2743, 0, 60, 16, 263, 0, 0, 1345, 43, 1663, 0, 3, 2, 22, 0, 0, 3268, 0, 73, 0, 1, 1, 0, 11, 870, 6, 115, 37, 0, 115, 0, 1, 102, 149, 0, 34, 7, 2, 75, 9, 182, 0, 99, 17, 0, 1, 3, 0, 4, 72, 0, 177, 0, 547, 0, 0, 5, 15, 1065, 5, 596, 94, 0, 275, 2085, 1159, 2543, 176, 304, 3, 252, 0, 2, 7, 1, 0, 0, 0, 400, 0, 0, 366, 81, 3133, 0, 0, 2270, 0, 0, 0, 4, 0, 0, 0, 5, 0, 166, 2120, 0, 15, 0] payload_rows_per_dst=[5976, 6620, 5124, 7156, 5848, 6048, 6570, 6748] dup_rows_per_dst=[1854, 2346, 1180, 2276, 1024, 2434, 1920, 2412] combine_rows_per_src=[5976, 6620, 5124, 7156, 5848, 6048, 6570, 6748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=175415296 dispatch_meta_bytes=231672 combine_payload_bytes=175415296 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=175646968 ingress_bytes=269787136 link_bytes=269787136 token_bytes=351062264 weight_movement_bytes=94371840 total_nvlink_bytes=445434104 weight_to_token_ratio=0.268818 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.584967594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171737088 dispatch_meta_bytes=226256 combine_payload_bytes=171737088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=171963344 ingress_bytes=209485824 link_bytes=209485824 token_bytes=343700432 weight_movement_bytes=37748736 total_nvlink_bytes=381449168 weight_to_token_ratio=0.109830 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.007190092e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48628 payload_rows_remote=41548 dup_rows=16908 dup_rows_remote=13820 dedup_ratio=0.257996 router_tokens_per_expert=[1, 173, 17, 2, 2059, 0, 0, 464, 1, 2, 0, 32, 20, 0, 4, 1131, 219, 0, 36, 0, 0, 490, 0, 0, 254, 0, 128, 1917, 30, 7, 17, 428, 6, 0, 1124, 750, 1314, 3, 1, 0, 75, 1, 15, 2768, 1, 71, 4, 8, 7, 3, 25, 572, 28, 116, 34, 9, 69, 2, 0, 606, 49, 0, 108, 11, 0, 93, 18, 258, 0, 129, 94, 0, 3, 126, 6, 117, 42, 0, 100, 10, 575, 6, 0, 7, 37, 1307, 35, 347, 103, 14, 44, 2524, 317, 2765, 15, 287, 14, 94, 0, 24, 48, 9, 2, 0, 0, 450, 0, 3, 174, 165, 2107, 3, 4, 2737, 3, 0, 0, 82, 18, 1, 1, 20, 1, 69, 1959, 20, 169, 0] payload_rows_per_dst=[5746, 5456, 5276, 6474, 6234, 7384, 4978, 7080] dup_rows_per_dst=[2066, 1596, 1470, 2340, 1288, 3852, 1208, 3088] combine_rows_per_src=[5746, 5456, 5276, 6474, 6234, 7384, 4978, 7080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=40496 payload_rows_remote=32392 dup_rows=25040 dup_rows_remote=18254 dedup_ratio=0.382080 router_tokens_per_expert=[976, 6, 0, 3, 1614, 0, 400, 0, 0, 0, 3772, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 2417, 3, 0, 0, 104, 8, 3235, 0, 0, 3996, 10, 0, 3, 0, 267, 581, 0, 2051, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 26, 2, 0, 0, 633, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 9, 371, 8, 142, 0, 0, 0, 0, 679, 42, 0, 0, 27, 2, 0, 0, 2, 4, 0, 0, 0, 2, 1015, 0, 0, 0, 1, 0, 185, 0, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 468, 1, 0, 0, 3082, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 0, 0] payload_rows_per_dst=[8104, 8136, 7494, 1328, 2372, 2280, 5160, 5622] dup_rows_per_dst=[6786, 12114, 4082, 2, 188, 138, 540, 1190] combine_rows_per_src=[8104, 8136, 7494, 1328, 2372, 2280, 5160, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47548 payload_rows_remote=41708 dup_rows=17988 dup_rows_remote=15806 dedup_ratio=0.274475 router_tokens_per_expert=[16, 54, 47, 2, 1170, 0, 1, 159, 2, 4, 2, 0, 17, 3, 0, 1235, 189, 0, 208, 9, 0, 996, 0, 0, 73, 8, 11, 1989, 4, 4, 49, 109, 1, 0, 999, 171, 2382, 0, 5, 0, 186, 0, 0, 2561, 4, 299, 1, 0, 130, 9, 14, 326, 66, 37, 45, 1, 46, 0, 0, 517, 36, 2, 69, 21, 0, 166, 167, 96, 4, 158, 153, 0, 13, 5, 2, 9, 22, 11, 11, 0, 610, 11, 0, 0, 50, 2007, 4, 706, 160, 0, 53, 1661, 1023, 1798, 33, 1146, 34, 90, 19, 0, 9, 3, 1, 0, 86, 610, 7, 0, 193, 188, 2348, 1, 4, 1388, 25, 1, 0, 35, 37, 0, 0, 1, 0, 31, 3260, 18, 11, 0] payload_rows_per_dst=[6138, 5840, 6778, 6024, 4578, 6598, 6354, 5238] dup_rows_per_dst=[2608, 2182, 2566, 1736, 652, 4574, 3494, 176] combine_rows_per_src=[6138, 5840, 6778, 6024, 4578, 6598, 6354, 5238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=153985024 dispatch_meta_bytes=202464 combine_payload_bytes=153985024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=154187488 ingress_bytes=191733760 link_bytes=191733760 token_bytes=308172512 weight_movement_bytes=37748736 total_nvlink_bytes=345921248 weight_to_token_ratio=0.122492 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.076389 gemm_active_groups=19 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.837098549e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=175857664 dispatch_meta_bytes=224416 combine_payload_bytes=175857664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176082080 ingress_bytes=194732032 link_bytes=194732032 token_bytes=351939744 weight_movement_bytes=18874368 total_nvlink_bytes=370814112 weight_to_token_ratio=0.053630 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=132677632 dispatch_meta_bytes=202584 combine_payload_bytes=132677632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=132880216 ingress_bytes=189300736 link_bytes=189300736 token_bytes=265557848 weight_movement_bytes=56623104 total_nvlink_bytes=322180952 weight_to_token_ratio=0.213223 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=11392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.813786511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=170180608 dispatch_meta_bytes=221472 combine_payload_bytes=170180608 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=170402080 ingress_bytes=226803712 link_bytes=226803712 token_bytes=340582688 weight_movement_bytes=56623104 total_nvlink_bytes=397205792 weight_to_token_ratio=0.166254 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=15360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.173121574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=170835968 dispatch_meta_bytes=230056 combine_payload_bytes=170835968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171066024 ingress_bytes=189710336 link_bytes=189710336 token_bytes=341901992 weight_movement_bytes=18874368 total_nvlink_bytes=360776360 weight_to_token_ratio=0.055204 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.079861 gemm_active_groups=16 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.817711096e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48464 payload_rows_remote=40286 dup_rows=17072 dup_rows_remote=15444 dedup_ratio=0.260498 router_tokens_per_expert=[1679, 4, 0, 0, 2801, 0, 208, 0, 0, 0, 4071, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 3252, 0, 0, 4092, 23, 0, 0, 0, 715, 274, 0, 1826, 0, 0, 0, 0, 29, 2, 0, 0, 0, 5, 0, 0, 0, 4, 2328, 0, 0, 42, 0, 0, 3, 4, 0, 1, 0, 24, 971, 0, 145, 0, 3, 0, 0, 3, 0, 0, 6, 198, 0, 0, 0, 64, 0, 9, 0, 0, 0, 422, 2, 0, 0, 1, 0, 331, 1, 2, 1, 0, 1518, 0, 0, 10, 0, 0, 12, 0, 9, 25, 0, 140, 0, 0, 0, 3558, 1, 407, 0, 0, 0, 0, 17, 0, 0, 1, 0, 2831, 0, 0] payload_rows_per_dst=[4122, 664, 5000, 8190, 7318, 8178, 6956, 8036] dup_rows_per_dst=[382, 0, 738, 4768, 1886, 1628, 2076, 5594] combine_rows_per_src=[4122, 664, 5000, 8190, 7318, 8178, 6956, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=40160 payload_rows_remote=34352 dup_rows=25376 dup_rows_remote=22574 dedup_ratio=0.387207 router_tokens_per_expert=[911, 14, 0, 0, 1426, 0, 269, 0, 0, 0, 4006, 0, 22, 0, 0, 756, 1, 1, 0, 0, 0, 4, 2619, 0, 0, 1, 46, 1, 3763, 0, 0, 4088, 789, 0, 1, 0, 1708, 154, 61, 1284, 1, 0, 0, 0, 0, 35, 0, 0, 0, 1, 6, 0, 0, 0, 1769, 0, 0, 2, 0, 0, 0, 130, 0, 0, 0, 320, 598, 26, 164, 0, 0, 16, 0, 195, 6, 0, 0, 73, 0, 3, 0, 0, 0, 0, 0, 0, 6, 519, 2, 0, 0, 93, 0, 20, 0, 1, 0, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 518, 0, 0, 0, 1365, 15, 1064, 1, 0, 0, 0, 22, 0, 1, 0, 0, 2471, 0, 0] payload_rows_per_dst=[7802, 8192, 5808, 3738, 2390, 2060, 3504, 6666] dup_rows_per_dst=[6022, 12856, 2802, 78, 412, 206, 332, 2668] combine_rows_per_src=[7802, 8192, 5808, 3738, 2390, 2060, 3504, 6666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=165011456 dispatch_meta_bytes=222920 combine_payload_bytes=165011456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=165234376 ingress_bytes=183885824 link_bytes=183885824 token_bytes=330245832 weight_movement_bytes=18874368 total_nvlink_bytes=349120200 weight_to_token_ratio=0.057152 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.761903488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46204 payload_rows_remote=39226 dup_rows=19332 dup_rows_remote=15434 dedup_ratio=0.294983 router_tokens_per_expert=[3, 14, 0, 0, 1769, 0, 9, 0, 0, 0, 4043, 0, 0, 0, 0, 1206, 0, 98, 0, 0, 0, 0, 55, 1, 0, 0, 1, 5, 2095, 0, 0, 4077, 8, 0, 0, 0, 2567, 507, 0, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 71, 0, 0, 299, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1495, 1080, 0, 668, 0, 0, 0, 0, 1, 4, 0, 0, 95, 0, 0, 0, 2, 1, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 2568, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 2, 0, 0, 1306, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2605, 0, 0] payload_rows_per_dst=[2430, 310, 6838, 8154, 6978, 8182, 6360, 6952] dup_rows_per_dst=[34, 10, 3168, 768, 3898, 6368, 2240, 2846] combine_rows_per_src=[2430, 310, 6838, 8154, 6978, 8182, 6360, 6952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51636 payload_rows_remote=44944 dup_rows=13900 dup_rows_remote=13260 dedup_ratio=0.212097 router_tokens_per_expert=[1589, 1, 0, 0, 3455, 0, 428, 0, 0, 0, 3957, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 39, 0, 3480, 0, 0, 4093, 34, 0, 0, 0, 1493, 101, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3280, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 22, 1322, 0, 712, 0, 0, 0, 0, 633, 0, 0, 0, 35, 0, 0, 0, 7, 0, 0, 0, 0, 3, 984, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 2163, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2, 0, 0, 792, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0] payload_rows_per_dst=[7102, 7472, 5264, 6692, 7348, 8074, 4998, 4686] dup_rows_per_dst=[3358, 1696, 1250, 640, 3584, 1984, 712, 676] combine_rows_per_src=[7102, 7472, 5264, 6692, 7348, 8074, 4998, 4686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45742 payload_rows_remote=38042 dup_rows=19794 dup_rows_remote=19378 dedup_ratio=0.302032 router_tokens_per_expert=[0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 212, 0, 0, 0, 0, 0, 282, 92, 0, 5, 0, 0, 0, 0, 0, 735, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 1189, 5, 0, 0, 0, 0, 3982, 3157, 2, 1, 933, 0, 224, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 287, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3637, 0, 0, 9, 0, 0, 7, 6, 0, 0, 0, 40, 0, 0, 4, 0, 0, 0, 2, 0, 1, 965, 0, 3301, 0, 0, 3, 2744, 0, 0, 0, 0, 115, 1, 0, 0, 0, 6, 0, 0, 10, 3840] payload_rows_per_dst=[7700, 8184, 8086, 8172, 3034, 8080, 2018, 468] dup_rows_per_dst=[416, 2016, 7616, 3886, 4, 5842, 6, 8] combine_rows_per_src=[7700, 8184, 8086, 8172, 3034, 8080, 2018, 468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=140705792 dispatch_meta_bytes=227704 combine_payload_bytes=140705792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=140933496 ingress_bytes=159580160 link_bytes=159580160 token_bytes=281639288 weight_movement_bytes=18874368 total_nvlink_bytes=300513656 weight_to_token_ratio=0.067016 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=12800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.529018575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45576 payload_rows_remote=38722 dup_rows=19960 dup_rows_remote=16566 dedup_ratio=0.304565 router_tokens_per_expert=[368, 58, 1, 0, 3242, 0, 240, 0, 0, 0, 4042, 0, 1, 0, 0, 453, 7, 0, 1, 0, 0, 1, 286, 0, 0, 0, 33, 1, 2750, 0, 0, 4074, 1, 0, 1, 0, 977, 825, 0, 1479, 0, 0, 4, 6, 0, 19, 0, 0, 0, 1, 0, 1, 0, 3, 1931, 0, 0, 0, 0, 2, 0, 31, 0, 0, 0, 70, 1379, 0, 645, 35, 0, 0, 0, 8, 5, 0, 0, 270, 0, 0, 0, 0, 285, 0, 0, 0, 0, 374, 0, 16, 0, 5, 0, 1415, 0, 1, 0, 0, 1219, 7, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1067, 0, 0, 0, 1876, 0, 1691, 0, 0, 0, 0, 0, 0, 52, 0, 0, 1504, 0, 1] payload_rows_per_dst=[2122, 658, 5092, 8160, 7108, 8184, 7398, 6854] dup_rows_per_dst=[120, 0, 1532, 3926, 3216, 4092, 3680, 3394] combine_rows_per_src=[2122, 658, 5092, 8160, 7108, 8184, 7398, 6854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160669696 dispatch_meta_bytes=218640 combine_payload_bytes=160669696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=160888336 ingress_bytes=179544064 link_bytes=179544064 token_bytes=321558032 weight_movement_bytes=18874368 total_nvlink_bytes=340432400 weight_to_token_ratio=0.058697 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=14720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.720302880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48116 payload_rows_remote=40000 dup_rows=17420 dup_rows_remote=15510 dedup_ratio=0.265808 router_tokens_per_expert=[2181, 10, 6, 10, 695, 0, 15, 0, 0, 0, 4067, 0, 0, 0, 7, 125, 0, 11, 0, 1, 1, 0, 62, 0, 0, 0, 78, 79, 743, 0, 0, 4038, 12, 0, 0, 0, 1410, 1374, 0, 1811, 0, 0, 0, 31, 0, 65, 0, 0, 1, 3, 2, 14, 0, 0, 3668, 0, 0, 0, 58, 0, 0, 7, 0, 0, 0, 6, 774, 1, 441, 0, 0, 0, 6, 511, 206, 0, 18, 199, 0, 0, 1, 10, 0, 9, 0, 1, 1, 1977, 0, 0, 0, 0, 0, 67, 3, 6, 18, 0, 3082, 0, 114, 0, 0, 0, 5, 0, 87, 0, 0, 1299, 15, 0, 0, 474, 1, 461, 2, 11, 0, 0, 8, 0, 1, 0, 0, 2368, 0, 0] payload_rows_per_dst=[8184, 8116, 7612, 7432, 3750, 4120, 7028, 1874] dup_rows_per_dst=[6048, 1910, 6530, 74, 574, 30, 2212, 42] combine_rows_per_src=[8184, 8116, 7612, 7432, 3750, 4120, 7028, 1874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=229680 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156049712 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311869744 weight_movement_bytes=18874368 total_nvlink_bytes=330744112 weight_to_token_ratio=0.060520 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=184090624 dispatch_meta_bytes=232816 combine_payload_bytes=184090624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184323440 ingress_bytes=202964992 link_bytes=202964992 token_bytes=368414064 weight_movement_bytes=18874368 total_nvlink_bytes=387288432 weight_to_token_ratio=0.051231 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.079861 gemm_active_groups=14 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.944710688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158605312 dispatch_meta_bytes=221152 combine_payload_bytes=158605312 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=158826464 ingress_bytes=215228416 link_bytes=215228416 token_bytes=317431776 weight_movement_bytes=56623104 total_nvlink_bytes=374054880 weight_to_token_ratio=0.178379 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=13568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.062212783e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=163840000 dispatch_meta_bytes=222040 combine_payload_bytes=163840000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=164062040 ingress_bytes=220463104 link_bytes=220463104 token_bytes=327902040 weight_movement_bytes=56623104 total_nvlink_bytes=384525144 weight_to_token_ratio=0.172683 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.079861 gemm_active_groups=15 gemm_peak_group_rows=15872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112368988e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55278 payload_rows_remote=47296 dup_rows=10258 dup_rows_remote=8824 dedup_ratio=0.156525 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 92, 0, 0, 0, 0, 1, 116, 0, 0, 10, 0, 0, 0, 0, 7, 26, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 300, 0, 25, 0, 0, 1, 0, 0, 3960, 3959, 1, 0, 1747, 0, 230, 0, 0, 0, 0, 0, 45, 0, 4065, 0, 0, 1661, 1, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5, 781, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 26, 119, 0, 0, 1, 0, 0, 1, 0, 0, 3539, 2, 3912, 0, 116, 0, 3865, 0, 0, 0, 0, 7, 0, 9, 1, 0, 0, 0, 5, 0, 4009] payload_rows_per_dst=[8044, 7964, 660, 6528, 8170, 7982, 8186, 7744] dup_rows_per_dst=[168, 292, 0, 840, 154, 1434, 7340, 30] combine_rows_per_src=[8044, 7964, 660, 6528, 8170, 7982, 8186, 7744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53898 payload_rows_remote=47042 dup_rows=11638 dup_rows_remote=11612 dedup_ratio=0.177582 router_tokens_per_expert=[0, 12, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 2128, 0, 0, 3, 0, 0, 0, 0, 0, 29, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 3732, 3379, 54, 1, 2873, 0, 94, 0, 0, 0, 0, 0, 16, 0, 4093, 0, 0, 341, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2685, 6, 0, 0, 0, 0, 55, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 292, 1556, 0, 1732, 0, 4, 1, 3377, 0, 0, 0, 0, 7, 29, 0, 0, 20, 0, 0, 0, 0, 3767] payload_rows_per_dst=[7746, 8002, 6856, 6126, 7534, 6940, 3866, 6828] dup_rows_per_dst=[2214, 3794, 26, 1786, 1708, 2018, 52, 40] combine_rows_per_src=[7746, 8002, 6856, 6126, 7534, 6940, 3866, 6828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38042 dup_rows=21818 dup_rows_remote=18548 dedup_ratio=0.332916 router_tokens_per_expert=[785, 183, 126, 97, 861, 472, 600, 76, 247, 84, 279, 415, 198, 251, 202, 441, 179, 109, 290, 317, 206, 309, 298, 84, 462, 331, 194, 287, 128, 258, 233, 371, 444, 211, 160, 160, 170, 161, 371, 265, 179, 68, 84, 389, 387, 43, 120, 297, 119, 241, 110, 120, 116, 235, 149, 132, 166, 134, 213, 211, 179, 378, 89, 65, 483, 346, 247, 120, 193, 83, 250, 673, 103, 293, 263, 328, 50, 460, 329, 1159, 198, 144, 173, 435, 189, 163, 178, 47, 116, 169, 417, 346, 129, 390, 238, 90, 31, 251, 330, 75, 315, 349, 602, 155, 456, 259, 208, 493, 385, 401, 353, 159, 466, 540, 246, 68, 439, 78, 322, 135, 196, 458, 162, 135, 46, 23, 238, 53] payload_rows_per_dst=[5404, 5342, 4840, 5446, 6188, 5622, 5676, 5200] dup_rows_per_dst=[2018, 2770, 2178, 2186, 3744, 2944, 3270, 2708] combine_rows_per_src=[5404, 5342, 4840, 5446, 6188, 5622, 5676, 5200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52776 payload_rows_remote=44726 dup_rows=12760 dup_rows_remote=9808 dedup_ratio=0.194702 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 1704, 1, 0, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 23, 0, 0, 142, 0, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 3625, 3150, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 285, 167, 9, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 2598, 0, 0, 0, 3989] payload_rows_per_dst=[8050, 7766, 8170, 7462, 7084, 7588, 5462, 1194] dup_rows_per_dst=[2952, 1754, 5446, 2208, 0, 100, 296, 4] combine_rows_per_src=[8050, 7766, 8170, 7462, 7084, 7588, 5462, 1194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=193724416 dispatch_meta_bytes=224480 combine_payload_bytes=193724416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193948896 ingress_bytes=212598784 link_bytes=212598784 token_bytes=387673312 weight_movement_bytes=18874368 total_nvlink_bytes=406547680 weight_to_token_ratio=0.048686 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=17920 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037016943e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56800 payload_rows_remote=48608 dup_rows=8736 dup_rows_remote=8670 dedup_ratio=0.133301 router_tokens_per_expert=[0, 8, 19, 5, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 52, 9, 0, 1284, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 379, 34, 0, 0, 0, 0, 4074, 4045, 53, 2, 3405, 0, 121, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 13, 1, 0, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1106, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 10, 0, 0, 0, 0, 8, 0, 0, 692, 0, 3964, 0, 0, 0, 3860, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 1, 4094] payload_rows_per_dst=[8188, 8172, 7712, 6898, 8192, 8106, 1804, 7728] dup_rows_per_dst=[2636, 2688, 982, 180, 66, 2044, 138, 2] combine_rows_per_src=[8188, 8172, 7712, 6898, 8192, 8106, 1804, 7728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58810 payload_rows_remote=51558 dup_rows=6726 dup_rows_remote=4308 dedup_ratio=0.102631 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 15, 0, 0, 6, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 1290, 0, 0, 0, 0, 0, 3637, 4028, 9, 0, 3365, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 113, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2650, 0, 2987, 0, 9, 0, 3899, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4056] payload_rows_per_dst=[8114, 7290, 8124, 7252, 8192, 6724, 5316, 7798] dup_rows_per_dst=[554, 50, 2726, 2418, 46, 906, 18, 8] combine_rows_per_src=[8114, 7290, 8124, 7252, 8192, 6724, 5316, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=192684032 dispatch_meta_bytes=234616 combine_payload_bytes=192684032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=192918648 ingress_bytes=249307136 link_bytes=249307136 token_bytes=385602680 weight_movement_bytes=56623104 total_nvlink_bytes=442225784 weight_to_token_ratio=0.146843 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.083333 gemm_active_groups=14 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388738310e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42850 payload_rows_remote=34750 dup_rows=22686 dup_rows_remote=15032 dedup_ratio=0.346161 router_tokens_per_expert=[0, 0, 3, 0, 0, 1, 0, 3, 0, 0, 0, 1799, 0, 0, 6, 0, 0, 391, 0, 0, 1, 0, 0, 0, 0, 0, 303, 0, 0, 17, 0, 0, 0, 0, 0, 0, 3, 90, 0, 64, 1, 0, 0, 8, 0, 4072, 3302, 11, 2, 1260, 0, 556, 0, 0, 1, 0, 0, 0, 0, 4064, 4, 0, 496, 0, 0, 1, 0, 151, 0, 0, 0, 0, 0, 0, 0, 460, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 557, 0, 1, 1, 0, 0, 10, 0, 0, 2, 0, 91, 0, 0, 0, 0, 0, 0, 400, 0, 35, 3050, 0, 3647, 0, 0, 0, 3924, 0, 0, 0, 0, 28, 0, 0, 0, 14, 51, 4, 13, 0, 3843] payload_rows_per_dst=[3600, 1274, 8164, 4196, 8178, 1168, 8170, 8100] dup_rows_per_dst=[24, 150, 334, 442, 1200, 2, 12880, 7654] combine_rows_per_src=[3600, 1274, 8164, 4196, 8178, 1168, 8170, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53136 payload_rows_remote=45534 dup_rows=12400 dup_rows_remote=11696 dedup_ratio=0.189209 router_tokens_per_expert=[0, 31, 0, 0, 8, 0, 0, 7, 0, 4, 1, 32, 0, 1, 0, 0, 0, 363, 0, 0, 0, 1, 5, 19, 1, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 44, 1120, 0, 0, 0, 0, 0, 3753, 2350, 0, 0, 2094, 1, 400, 0, 0, 0, 0, 0, 2, 0, 3927, 5, 0, 60, 1, 0, 0, 5, 1098, 41, 0, 223, 0, 0, 0, 0, 3957, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3124, 24, 1, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 0, 1790, 0, 0, 94, 3534, 0, 1, 0, 0, 323, 0, 5, 91, 0, 30, 0, 0, 119, 4031] payload_rows_per_dst=[8136, 7602, 5840, 8122, 8048, 7646, 66, 7676] dup_rows_per_dst=[94, 704, 1190, 4858, 2602, 2234, 0, 718] combine_rows_per_src=[8136, 7602, 5840, 8122, 8048, 7646, 66, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=183197696 dispatch_meta_bytes=218136 combine_payload_bytes=183197696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=183415832 ingress_bytes=239820800 link_bytes=239820800 token_bytes=366613528 weight_movement_bytes=56623104 total_nvlink_bytes=423236632 weight_to_token_ratio=0.154449 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=22528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.297844906e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199098368 dispatch_meta_bytes=229112 combine_payload_bytes=199098368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199327480 ingress_bytes=217972736 link_bytes=217972736 token_bytes=398425848 weight_movement_bytes=18874368 total_nvlink_bytes=417300216 weight_to_token_ratio=0.047372 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.088507507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211181568 dispatch_meta_bytes=223464 combine_payload_bytes=211181568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=211405032 ingress_bytes=248930304 link_bytes=248930304 token_bytes=422586600 weight_movement_bytes=37748736 total_nvlink_bytes=460335336 weight_to_token_ratio=0.089328 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.385127692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=142336000 dispatch_meta_bytes=199128 combine_payload_bytes=142336000 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=142535128 ingress_bytes=198959104 link_bytes=198959104 token_bytes=284871128 weight_movement_bytes=56623104 total_nvlink_bytes=341494232 weight_to_token_ratio=0.198767 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.079861 gemm_active_groups=14 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.906328241e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=186507264 dispatch_meta_bytes=228920 combine_payload_bytes=186507264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186736184 ingress_bytes=205381632 link_bytes=205381632 token_bytes=373243448 weight_movement_bytes=18874368 total_nvlink_bytes=392117816 weight_to_token_ratio=0.050569 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.967865744e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=226360 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156046392 ingress_bytes=231317504 link_bytes=231317504 token_bytes=311866424 weight_movement_bytes=75497472 total_nvlink_bytes=387363896 weight_to_token_ratio=0.242083 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=3328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216370508e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56312 payload_rows_remote=48400 dup_rows=9224 dup_rows_remote=9082 dedup_ratio=0.140747 router_tokens_per_expert=[0, 0, 0, 0, 0, 95, 2576, 0, 0, 0, 0, 4088, 1, 0, 1, 0, 0, 0, 0, 1121, 2, 0, 0, 0, 4, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 3928, 3494, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 0, 0, 6, 0, 5, 50, 1, 1, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 7, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 2, 0, 1, 0, 3, 1757, 1, 1, 183, 4, 0, 3, 3401] payload_rows_per_dst=[6750, 5372, 6824, 7684, 8192, 7912, 8174, 5404] dup_rows_per_dst=[3444, 568, 0, 3174, 42, 142, 606, 1248] combine_rows_per_src=[6750, 5372, 6824, 7684, 8192, 7912, 8174, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57396 payload_rows_remote=50992 dup_rows=8140 dup_rows_remote=8080 dedup_ratio=0.124207 router_tokens_per_expert=[2, 0, 0, 16, 0, 1, 223, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 0, 0, 5, 0, 0, 0, 0, 0, 20, 5, 0, 0, 0, 0, 0, 3655, 0, 0, 3621, 2956, 11, 0, 1982, 0, 0, 0, 0, 0, 60, 0, 0, 0, 1, 4001, 0, 0, 0, 0, 7, 5, 0, 7, 1, 0, 0, 0, 0, 0, 0, 453, 10, 0, 1, 35, 0, 20, 0, 0, 0, 3, 0, 0, 0, 114, 1, 3953, 0, 0, 17, 43, 33, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 1935, 0, 0, 0, 3189] payload_rows_per_dst=[8192, 7732, 6404, 7256, 8006, 7682, 8122, 4002] dup_rows_per_dst=[484, 3970, 60, 2762, 38, 604, 202, 20] combine_rows_per_src=[8192, 7732, 6404, 7256, 8006, 7682, 8122, 4002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49184 payload_rows_remote=44652 dup_rows=16352 dup_rows_remote=14606 dedup_ratio=0.249512 router_tokens_per_expert=[0, 0, 1036, 0, 0, 0, 0, 17, 4076, 0, 86, 47, 0, 0, 0, 3, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1708, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 15, 0, 198, 0, 3999, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 63, 0, 0, 0, 0, 0, 40, 232, 4068, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3941, 0, 3752, 0, 0] payload_rows_per_dst=[4532, 8152, 8182, 8112, 6708, 5572, 7924, 2] dup_rows_per_dst=[1746, 2760, 5080, 3418, 2712, 0, 636, 0] combine_rows_per_src=[4532, 8152, 8182, 8112, 6708, 5572, 7924, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=198246400 dispatch_meta_bytes=229928 combine_payload_bytes=198246400 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=198476328 ingress_bytes=235995136 link_bytes=235995136 token_bytes=396722728 weight_movement_bytes=37748736 total_nvlink_bytes=434471464 weight_to_token_ratio=0.095151 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.072917 gemm_active_groups=16 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.261189276e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61268 payload_rows_remote=53098 dup_rows=4268 dup_rows_remote=3102 dedup_ratio=0.065125 router_tokens_per_expert=[14, 0, 0, 220, 0, 0, 129, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 173, 1, 0, 14, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 462, 13, 0, 3981, 3258, 0, 0, 2, 0, 0, 0, 0, 0, 297, 0, 0, 25, 1, 4081, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 6, 20, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 3424, 0, 0, 0, 3438] payload_rows_per_dst=[8192, 7734, 7022, 6704, 8170, 7984, 8190, 7272] dup_rows_per_dst=[726, 834, 256, 436, 1166, 182, 318, 350] combine_rows_per_src=[8192, 7734, 7022, 6704, 8170, 7984, 8190, 7272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=208863232 dispatch_meta_bytes=236288 combine_payload_bytes=208863232 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=209099520 ingress_bytes=246611968 link_bytes=246611968 token_bytes=417962752 weight_movement_bytes=37748736 total_nvlink_bytes=455711488 weight_to_token_ratio=0.090316 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362914537e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60814 payload_rows_remote=53396 dup_rows=4722 dup_rows_remote=4008 dedup_ratio=0.072052 router_tokens_per_expert=[0, 0, 0, 29, 0, 1, 65, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 27, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 3429, 0, 0, 3591, 3630, 3, 0, 31, 0, 0, 26, 0, 0, 376, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 7, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 78, 33, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 0, 0, 1668, 0, 0, 0, 3574] payload_rows_per_dst=[8180, 7310, 7154, 7418, 8048, 7196, 8192, 7316] dup_rows_per_dst=[190, 834, 90, 714, 510, 58, 322, 2004] combine_rows_per_src=[8180, 7310, 7154, 7418, 8048, 7196, 8192, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56482 payload_rows_remote=49150 dup_rows=9054 dup_rows_remote=6408 dedup_ratio=0.138153 router_tokens_per_expert=[223, 0, 0, 1, 0, 0, 655, 0, 55, 0, 36, 4055, 0, 0, 1, 0, 24, 24, 0, 1689, 0, 0, 5, 0, 9, 20, 1, 0, 4, 0, 0, 73, 0, 117, 0, 0, 0, 0, 0, 10, 0, 0, 65, 1, 0, 1, 4, 0, 0, 3140, 1, 0, 3433, 411, 69, 1, 563, 0, 0, 28, 0, 1, 3, 0, 0, 1, 0, 3945, 7, 36, 0, 0, 408, 62, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 55, 0, 0, 248, 0, 43, 4, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3081, 0, 0, 1898, 128, 0, 0, 4042] payload_rows_per_dst=[8184, 7332, 8114, 2020, 8032, 6916, 8162, 7722] dup_rows_per_dst=[1868, 2646, 366, 134, 894, 32, 620, 2494] combine_rows_per_src=[8184, 7332, 8114, 2020, 8032, 6916, 8162, 7722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55384 payload_rows_remote=47260 dup_rows=10152 dup_rows_remote=6406 dedup_ratio=0.154907 router_tokens_per_expert=[71, 0, 0, 2, 0, 1, 1867, 0, 1, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 62, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3605, 0, 0, 3285, 2900, 0, 2, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4062, 0, 0, 0, 0, 20, 18, 0, 0, 7, 0, 0, 0, 0, 5, 0, 1, 0, 0, 7, 11, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 1, 0, 38, 1, 0, 0, 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 648, 1, 0, 0, 3955] payload_rows_per_dst=[3738, 5094, 7916, 7476, 8144, 6738, 8154, 8124] dup_rows_per_dst=[146, 0, 124, 1984, 104, 3954, 94, 3746] combine_rows_per_src=[3738, 5094, 7916, 7476, 8144, 6738, 8154, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=182894592 dispatch_meta_bytes=237032 combine_payload_bytes=182894592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=183131624 ingress_bytes=220643328 link_bytes=220643328 token_bytes=366026216 weight_movement_bytes=37748736 total_nvlink_bytes=403774952 weight_to_token_ratio=0.103131 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.114095806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=217489408 dispatch_meta_bytes=224800 combine_payload_bytes=217489408 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=217714208 ingress_bytes=236363776 link_bytes=236363776 token_bytes=435203616 weight_movement_bytes=18874368 total_nvlink_bytes=454077984 weight_to_token_ratio=0.043369 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264721403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=218710016 dispatch_meta_bytes=229616 combine_payload_bytes=218710016 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=218939632 ingress_bytes=294207488 link_bytes=294207488 token_bytes=437649648 weight_movement_bytes=75497472 total_nvlink_bytes=513147120 weight_to_token_ratio=0.172507 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.818951391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=201318400 dispatch_meta_bytes=222232 combine_payload_bytes=201318400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201540632 ingress_bytes=220192768 link_bytes=220192768 token_bytes=402859032 weight_movement_bytes=18874368 total_nvlink_bytes=421733400 weight_to_token_ratio=0.046851 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109778761e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=193576960 dispatch_meta_bytes=214664 combine_payload_bytes=193576960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=193791624 ingress_bytes=250200064 link_bytes=250200064 token_bytes=387368584 weight_movement_bytes=56623104 total_nvlink_bytes=443991688 weight_to_token_ratio=0.146174 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.069444 gemm_active_groups=15 gemm_peak_group_rows=11776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397293907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56498 payload_rows_remote=48306 dup_rows=9038 dup_rows_remote=9028 dedup_ratio=0.137909 router_tokens_per_expert=[0, 0, 3475, 0, 20, 25, 0, 1, 4091, 0, 0, 481, 0, 0, 0, 0, 0, 20, 0, 3, 0, 1, 0, 6, 0, 2, 0, 0, 420, 0, 1, 0, 0, 0, 0, 0, 0, 27, 0, 17, 0, 0, 5, 0, 0, 0, 0, 10, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3005, 1, 0, 0, 0, 76, 0, 6, 0, 2726, 4095, 0, 0, 29, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 28, 2405, 0, 7, 0, 0, 0, 0, 6, 4026, 9, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 5, 0, 3976, 0, 3564, 0, 0] payload_rows_per_dst=[7176, 8078, 8192, 5682, 6178, 8192, 4884, 8116] dup_rows_per_dst=[828, 880, 116, 128, 56, 10, 26, 6994] combine_rows_per_src=[7176, 8078, 8192, 5682, 6178, 8192, 4884, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49960 payload_rows_remote=44400 dup_rows=15576 dup_rows_remote=14210 dedup_ratio=0.237671 router_tokens_per_expert=[0, 11, 48, 0, 1104, 0, 0, 161, 2, 5, 7, 22, 55, 1, 7, 1989, 90, 0, 6, 3, 0, 663, 0, 0, 216, 0, 3, 1908, 8, 12, 135, 544, 20, 0, 763, 471, 2270, 0, 2, 0, 38, 0, 4, 2848, 0, 76, 1, 13, 3, 0, 3, 313, 24, 384, 168, 1, 142, 2, 0, 68, 72, 2, 182, 16, 14, 63, 41, 107, 0, 285, 2, 0, 3, 48, 0, 6, 60, 7, 58, 0, 586, 11, 0, 0, 82, 241, 70, 542, 333, 1, 105, 2146, 1186, 2939, 78, 458, 36, 31, 0, 0, 51, 17, 0, 0, 0, 496, 0, 4, 391, 120, 2317, 0, 1, 2324, 0, 0, 0, 11, 0, 0, 0, 1, 0, 532, 2060, 3, 15, 0] payload_rows_per_dst=[5604, 5736, 6142, 6416, 6374, 7198, 5560, 6930] dup_rows_per_dst=[1220, 1440, 1174, 2040, 1228, 4144, 1366, 2964] combine_rows_per_src=[5604, 5736, 6142, 6416, 6374, 7198, 5560, 6930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53156 payload_rows_remote=47804 dup_rows=12380 dup_rows_remote=12370 dedup_ratio=0.188904 router_tokens_per_expert=[0, 0, 3773, 0, 0, 0, 0, 0, 4088, 0, 0, 1425, 0, 0, 0, 38, 0, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 102, 0, 0, 612, 14, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 3704, 0, 0, 0, 0, 398, 0, 192, 0, 3868, 2577, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 1, 0, 0, 0, 0, 0, 85, 4095, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 5, 296, 0, 1888, 0, 0] payload_rows_per_dst=[8006, 8190, 5352, 7928, 8094, 8176, 7166, 244] dup_rows_per_dst=[2466, 4322, 10, 1074, 4080, 0, 426, 2] combine_rows_per_src=[8006, 8190, 5352, 7928, 8094, 8176, 7166, 244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56330 payload_rows_remote=48946 dup_rows=9206 dup_rows_remote=8504 dedup_ratio=0.140472 router_tokens_per_expert=[0, 0, 2619, 0, 0, 0, 0, 0, 4094, 0, 5, 20, 0, 0, 0, 2, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 0, 308, 0, 77, 0, 2689, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 3, 1508, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 4094, 0, 0] payload_rows_per_dst=[5256, 8176, 7244, 8048, 7384, 8188, 3842, 8192] dup_rows_per_dst=[36, 300, 0, 4304, 702, 0, 78, 3786] combine_rows_per_src=[5256, 8176, 7244, 8048, 7384, 8188, 3842, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44826 payload_rows_remote=36634 dup_rows=20710 dup_rows_remote=6264 dedup_ratio=0.316010 router_tokens_per_expert=[0, 0, 3150, 0, 0, 4081, 1, 0, 0, 4086, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 677, 0, 0, 1, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1460, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1464, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 4093, 1, 0] payload_rows_per_dst=[8192, 2122, 8156, 7548, 7672, 2942, 8192, 2] dup_rows_per_dst=[14446, 0, 800, 1268, 1106, 4, 3084, 2] combine_rows_per_src=[8192, 2122, 8156, 7548, 7672, 2942, 8192, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=197861376 dispatch_meta_bytes=229336 combine_payload_bytes=197861376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198090712 ingress_bytes=216735744 link_bytes=216735744 token_bytes=395952088 weight_movement_bytes=18874368 total_nvlink_bytes=414826456 weight_to_token_ratio=0.047668 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.076655258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=195805184 dispatch_meta_bytes=240696 combine_payload_bytes=195805184 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=196045880 ingress_bytes=233553920 link_bytes=233553920 token_bytes=391851064 weight_movement_bytes=37748736 total_nvlink_bytes=429599800 weight_to_token_ratio=0.096334 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.237798746e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55338 payload_rows_remote=47194 dup_rows=10198 dup_rows_remote=9924 dedup_ratio=0.155609 router_tokens_per_expert=[0, 95, 3725, 22, 0, 1, 0, 0, 3965, 0, 0, 776, 0, 0, 0, 1, 13, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 1, 31, 0, 0, 0, 0, 0, 11, 0, 0, 30, 0, 0, 0, 3, 0, 0, 47, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 8, 0, 0, 2808, 0, 0, 0, 0, 403, 15, 0, 0, 4038, 3954, 0, 0, 0, 0, 0, 129, 0, 9, 0, 1078, 0, 0, 35, 0, 14, 1, 0, 0, 0, 129, 0, 0, 161, 2758, 0, 13, 0, 5, 0, 115, 263, 4069, 0, 0, 3, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 133, 0, 819, 0, 2193, 0, 0] payload_rows_per_dst=[7834, 8144, 7968, 8168, 7282, 8192, 6766, 984] dup_rows_per_dst=[1406, 274, 92, 790, 3556, 2528, 1508, 44] combine_rows_per_src=[7834, 8144, 7968, 8168, 7282, 8192, 6766, 984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55184 payload_rows_remote=50672 dup_rows=10352 dup_rows_remote=9910 dedup_ratio=0.157959 router_tokens_per_expert=[0, 0, 2759, 0, 0, 0, 0, 0, 4096, 0, 0, 868, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4, 0, 917, 0, 1696, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3358, 0, 0, 0, 0, 0, 0, 127, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 1039, 0, 0] payload_rows_per_dst=[6198, 8150, 7986, 4512, 8178, 8192, 7318, 4650] dup_rows_per_dst=[1056, 2312, 0, 442, 1814, 0, 3436, 1292] combine_rows_per_src=[6198, 8150, 7986, 4512, 8178, 8192, 7318, 4650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52916 payload_rows_remote=44724 dup_rows=12620 dup_rows_remote=5764 dedup_ratio=0.192566 router_tokens_per_expert=[0, 0, 3876, 0, 0, 0, 0, 8, 4082, 0, 0, 945, 0, 0, 0, 0, 12, 1464, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 44, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2268, 10, 0, 0, 0, 1, 0, 4, 0, 3114, 3620, 2, 62, 1, 0, 0, 0, 0, 0, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 914, 0, 10, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 1, 154, 9, 1, 0, 0, 0, 0, 0, 14, 3280, 0, 4065, 0, 0] payload_rows_per_dst=[7858, 3036, 7264, 4746, 5534, 8182, 8104, 8192] dup_rows_per_dst=[1800, 12, 6, 584, 1470, 58, 1834, 6856] combine_rows_per_src=[7858, 3036, 7264, 4746, 5534, 8182, 8104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200482816 dispatch_meta_bytes=229800 combine_payload_bytes=200482816 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=200712616 ingress_bytes=275980288 link_bytes=275980288 token_bytes=401195432 weight_movement_bytes=75497472 total_nvlink_bytes=476692904 weight_to_token_ratio=0.188181 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.644307329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=150052864 dispatch_meta_bytes=171592 combine_payload_bytes=150052864 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=150224456 ingress_bytes=225550336 link_bytes=225550336 token_bytes=300277320 weight_movement_bytes=75497472 total_nvlink_bytes=375774792 weight_to_token_ratio=0.251426 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=14208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.161112342e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=181862400 dispatch_meta_bytes=234440 combine_payload_bytes=181862400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=182096840 ingress_bytes=200736768 link_bytes=200736768 token_bytes=363959240 weight_movement_bytes=18874368 total_nvlink_bytes=382833608 weight_to_token_ratio=0.051858 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.923360942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=193306624 dispatch_meta_bytes=228472 combine_payload_bytes=193306624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193535096 ingress_bytes=212180992 link_bytes=212180992 token_bytes=386841720 weight_movement_bytes=18874368 total_nvlink_bytes=405716088 weight_to_token_ratio=0.048791 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.033013866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=207552512 dispatch_meta_bytes=242328 combine_payload_bytes=207552512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207794840 ingress_bytes=226426880 link_bytes=226426880 token_bytes=415347352 weight_movement_bytes=18874368 total_nvlink_bytes=434221720 weight_to_token_ratio=0.045442 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169510955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=183189504 dispatch_meta_bytes=201952 combine_payload_bytes=183189504 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=183391456 ingress_bytes=239812608 link_bytes=239812608 token_bytes=366580960 weight_movement_bytes=56623104 total_nvlink_bytes=423204064 weight_to_token_ratio=0.154463 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.297766415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49758 payload_rows_remote=42756 dup_rows=15778 dup_rows_remote=14540 dedup_ratio=0.240753 router_tokens_per_expert=[0, 0, 3995, 0, 0, 3105, 0, 0, 0, 3300, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 0, 0, 1, 1894, 38, 1, 0, 0, 2917, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3043, 0, 905, 0, 1, 0, 1, 0, 0, 0, 2294, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 811, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 4, 0, 72, 0, 0, 0, 0, 35, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 4069, 0, 0, 0, 0, 3875, 0, 0] payload_rows_per_dst=[168, 8128, 5736, 8016, 4596, 7002, 7954, 8158] dup_rows_per_dst=[0, 4286, 1712, 5808, 0, 1238, 138, 2596] combine_rows_per_src=[168, 8128, 5736, 8016, 4596, 7002, 7954, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53532 payload_rows_remote=45412 dup_rows=12004 dup_rows_remote=11648 dedup_ratio=0.183167 router_tokens_per_expert=[0, 0, 4096, 0, 0, 735, 0, 0, 2, 3554, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 295, 0, 0, 455, 0, 0, 0, 0, 3982, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 467, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 3931, 1, 50] payload_rows_per_dst=[8192, 6298, 8120, 7180, 8182, 7116, 8190, 254] dup_rows_per_dst=[1476, 0, 356, 708, 7674, 22, 1764, 4] combine_rows_per_src=[8192, 6298, 8120, 7180, 8182, 7116, 8190, 254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=175128576 dispatch_meta_bytes=229184 combine_payload_bytes=175128576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175357760 ingress_bytes=194002944 link_bytes=194002944 token_bytes=350486336 weight_movement_bytes=18874368 total_nvlink_bytes=369360704 weight_to_token_ratio=0.053852 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53668 payload_rows_remote=46342 dup_rows=11868 dup_rows_remote=11868 dedup_ratio=0.181091 router_tokens_per_expert=[0, 0, 2007, 0, 0, 4089, 0, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 19, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 0, 0, 0, 0, 4061, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 0, 810, 0, 0, 2, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 464, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 10, 0, 8, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4078, 1, 199] payload_rows_per_dst=[3830, 4546, 8166, 5258, 7326, 8182, 8168, 8192] dup_rows_per_dst=[2, 250, 9148, 638, 0, 466, 940, 424] combine_rows_per_src=[3830, 4546, 8166, 5258, 7326, 8182, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48728 payload_rows_remote=40890 dup_rows=16808 dup_rows_remote=8508 dedup_ratio=0.256470 router_tokens_per_expert=[0, 0, 1128, 0, 26, 0, 0, 0, 0, 0, 0, 4083, 2, 129, 0, 1653, 0, 0, 428, 0, 0, 0, 0, 0, 0, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 2, 1071, 3927, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 2, 0, 0, 64, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4, 48, 0, 0, 0, 0, 1, 0, 1630, 0, 0, 1, 0, 0, 0, 3065, 0, 0, 0, 0, 0] payload_rows_per_dst=[7838, 8096, 7200, 4820, 2032, 7560, 7920, 3262] dup_rows_per_dst=[8300, 5748, 1862, 0, 10, 826, 60, 2] combine_rows_per_src=[7838, 8096, 7200, 4820, 2032, 7560, 7920, 3262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=186007552 dispatch_meta_bytes=228240 combine_payload_bytes=186007552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=186235792 ingress_bytes=223756288 link_bytes=223756288 token_bytes=372243344 weight_movement_bytes=37748736 total_nvlink_bytes=409992080 weight_to_token_ratio=0.101409 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.143922657e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45842 payload_rows_remote=37814 dup_rows=19694 dup_rows_remote=19694 dedup_ratio=0.300507 router_tokens_per_expert=[55, 7, 3836, 0, 0, 3912, 0, 0, 21, 2973, 0, 238, 0, 147, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 18, 0, 4, 2226, 0, 45, 0, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 227, 0, 1, 0, 6, 51, 0, 13, 2, 0, 2637, 0, 0, 3, 1, 0, 13, 25, 68, 0, 0, 0, 420, 10, 0, 2, 19, 35, 4, 1975, 1, 0, 0, 0, 124, 2, 0, 0, 0, 2, 1, 796, 0, 1663, 0, 0, 0, 0, 73, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 150, 0, 1890, 0, 0, 0, 0, 3903, 1, 88] payload_rows_per_dst=[8192, 8028, 5990, 566, 6924, 7720, 7956, 466] dup_rows_per_dst=[8424, 0, 716, 6, 2380, 3226, 4930, 12] combine_rows_per_src=[8192, 8028, 5990, 566, 6924, 7720, 7956, 466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46820 payload_rows_remote=39060 dup_rows=18716 dup_rows_remote=14136 dedup_ratio=0.285583 router_tokens_per_expert=[703, 1, 0, 0, 3224, 0, 236, 0, 0, 0, 4090, 0, 0, 0, 0, 520, 0, 10, 0, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 3771, 0, 0, 3898, 302, 0, 0, 0, 94, 5, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 26, 967, 2, 2473, 1, 0, 0, 2, 14, 1, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 7, 698, 0, 0, 0, 62, 0, 143, 0, 0, 0, 0, 2786, 0, 6, 2, 0, 0, 0, 33, 0, 0, 0, 119, 0, 0, 0, 1746, 0, 794, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1677, 0, 0] payload_rows_per_dst=[2762, 3396, 2292, 8082, 8042, 8188, 7760, 6298] dup_rows_per_dst=[158, 6, 34, 3504, 6472, 1824, 4580, 2138] combine_rows_per_src=[2762, 3396, 2292, 8082, 8042, 8188, 7760, 6298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57406 payload_rows_remote=50694 dup_rows=8130 dup_rows_remote=8082 dedup_ratio=0.124054 router_tokens_per_expert=[6, 0, 3946, 0, 0, 3266, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 84, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 525, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 3945, 0, 0, 0, 1, 4080, 0, 8] payload_rows_per_dst=[7896, 4240, 8112, 6712, 6186, 8188, 8182, 7890] dup_rows_per_dst=[4672, 486, 918, 48, 0, 918, 1044, 44] combine_rows_per_src=[7896, 4240, 8112, 6712, 6186, 8188, 8182, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=189816832 dispatch_meta_bytes=232840 combine_payload_bytes=189816832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190049672 ingress_bytes=208691200 link_bytes=208691200 token_bytes=379866504 weight_movement_bytes=18874368 total_nvlink_bytes=398740872 weight_to_token_ratio=0.049687 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=22912 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.999576396e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44722 payload_rows_remote=36530 dup_rows=20814 dup_rows_remote=6758 dedup_ratio=0.317596 router_tokens_per_expert=[4, 1, 4078, 0, 0, 3220, 0, 0, 1, 4016, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 2427, 0, 4, 0, 3, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2706, 0, 0, 0, 0, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 211, 0, 0, 387, 0, 0, 0, 0, 3726, 0, 59, 0, 9, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 46, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 21, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 95, 1, 0, 0, 0, 126, 0, 0, 4084, 127, 0, 0, 0, 3140, 0, 0] payload_rows_per_dst=[36, 8192, 5412, 7310, 7458, 8074, 48, 8192] dup_rows_per_dst=[8, 4932, 0, 1550, 130, 138, 0, 14056] combine_rows_per_src=[36, 8192, 5412, 7310, 7458, 8074, 48, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=167485440 dispatch_meta_bytes=197592 combine_payload_bytes=167485440 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=167683032 ingress_bytes=224108544 link_bytes=224108544 token_bytes=335168472 weight_movement_bytes=56623104 total_nvlink_bytes=391791576 weight_to_token_ratio=0.168939 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=13184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.147297800e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=154886144 dispatch_meta_bytes=230032 combine_payload_bytes=154886144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155116176 ingress_bytes=173760512 link_bytes=173760512 token_bytes=310002320 weight_movement_bytes=18874368 total_nvlink_bytes=328876688 weight_to_token_ratio=0.060885 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.664887730e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50300 payload_rows_remote=42110 dup_rows=15236 dup_rows_remote=10882 dedup_ratio=0.232483 router_tokens_per_expert=[8, 1, 267, 0, 37, 5, 0, 0, 8, 0, 0, 3807, 0, 35, 0, 3988, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 1, 0, 0, 10, 0, 0, 10, 800, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 596, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 1919, 1, 18, 2, 4, 0, 0, 4068, 0, 0, 0, 0, 0] payload_rows_per_dst=[698, 5544, 8154, 7900, 8176, 8190, 3446, 8192] dup_rows_per_dst=[24, 0, 5282, 86, 1462, 4354, 192, 3836] combine_rows_per_src=[698, 5544, 8154, 7900, 8176, 8190, 3446, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159989760 dispatch_meta_bytes=212784 combine_payload_bytes=159989760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=160202544 ingress_bytes=178864128 link_bytes=178864128 token_bytes=320192304 weight_movement_bytes=18874368 total_nvlink_bytes=339066672 weight_to_token_ratio=0.058947 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.086806 gemm_active_groups=13 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.713788068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=207642624 dispatch_meta_bytes=235104 combine_payload_bytes=207642624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207877728 ingress_bytes=226516992 link_bytes=226516992 token_bytes=415520352 weight_movement_bytes=18874368 total_nvlink_bytes=434394720 weight_to_token_ratio=0.045423 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.170374364e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=149626880 dispatch_meta_bytes=173152 combine_payload_bytes=149626880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=149800032 ingress_bytes=206249984 link_bytes=206249984 token_bytes=299426912 weight_movement_bytes=56623104 total_nvlink_bytes=356050016 weight_to_token_ratio=0.189105 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.076389 gemm_active_groups=16 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.976185865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54744 payload_rows_remote=47824 dup_rows=10792 dup_rows_remote=8044 dedup_ratio=0.164673 router_tokens_per_expert=[0, 0, 2, 0, 52, 0, 0, 0, 0, 2, 0, 4033, 4, 8, 0, 3991, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2940, 51, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 76, 2109, 2198, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 4, 33, 0, 0, 0, 0, 0, 2, 225, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0] payload_rows_per_dst=[7988, 8192, 6920, 8068, 8172, 7598, 7308, 498] dup_rows_per_dst=[130, 5996, 2748, 2, 6, 0, 1908, 2] combine_rows_per_src=[7988, 8192, 6920, 8068, 8172, 7598, 7308, 498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=172482560 dispatch_meta_bytes=211968 combine_payload_bytes=172482560 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=172694528 ingress_bytes=210231296 link_bytes=210231296 token_bytes=345177088 weight_movement_bytes=37748736 total_nvlink_bytes=382925824 weight_to_token_ratio=0.109360 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.014332838e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50904 payload_rows_remote=42714 dup_rows=14632 dup_rows_remote=9678 dedup_ratio=0.223267 router_tokens_per_expert=[0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 384, 0, 4036, 0, 0, 0, 0, 0, 0, 1, 0, 0, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 2, 38, 2985, 0, 0, 0, 6, 0, 1425, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 0, 0, 10, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 0, 470, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8026, 1788, 1336, 8178, 8190, 8164, 7032, 8190] dup_rows_per_dst=[4272, 0, 134, 2870, 4954, 2, 1446, 954] combine_rows_per_src=[8026, 1788, 1336, 8178, 8190, 8164, 7032, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=195887104 dispatch_meta_bytes=223472 combine_payload_bytes=195887104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=196110576 ingress_bytes=252510208 link_bytes=252510208 token_bytes=391997680 weight_movement_bytes=56623104 total_nvlink_bytes=448620784 weight_to_token_ratio=0.144448 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419428570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48144 payload_rows_remote=39986 dup_rows=17392 dup_rows_remote=16750 dedup_ratio=0.265381 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 320, 1, 0, 1, 0, 0, 0, 0, 0, 2, 5, 0, 10, 0, 0, 0, 0, 0, 0, 3171, 4074, 0, 2, 5, 0, 2892, 0, 0, 0, 3863, 0, 0, 9, 0, 0, 0, 1, 1, 0, 0, 0, 0, 713, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1065, 74, 1, 0, 0, 0, 0, 4096, 0, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 558, 0, 0, 0, 0, 0, 3155, 0, 2901, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 7020, 8192, 358, 8192, 2, 8192, 8030] dup_rows_per_dst=[642, 1500, 6766, 2, 3470, 0, 10, 5002] combine_rows_per_src=[8158, 7020, 8192, 358, 8192, 2, 8192, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55154 payload_rows_remote=47616 dup_rows=10382 dup_rows_remote=5674 dedup_ratio=0.158417 router_tokens_per_expert=[0, 0, 28, 5, 0, 0, 0, 39, 0, 0, 0, 4032, 589, 3893, 0, 157, 0, 3, 142, 0, 0, 0, 0, 0, 0, 2567, 3, 0, 0, 0, 0, 0, 9, 0, 0, 3442, 0, 3, 0, 2, 0, 0, 15, 0, 2, 3557, 12, 0, 0, 180, 0, 8, 0, 0, 0, 0, 0, 0, 96, 0, 16, 0, 7, 18, 0, 0, 0, 49, 5, 0, 0, 0, 0, 3435, 0, 2, 7, 0, 0, 0, 0, 72, 2, 0, 1, 0, 0, 0, 4016, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1189, 0, 0, 999, 0, 658, 0, 54, 0, 0, 0, 0, 0, 0, 3408, 0, 0, 31, 0, 0] payload_rows_per_dst=[8078, 7538, 6946, 8186, 6898, 8062, 7960, 1486] dup_rows_per_dst=[1344, 4708, 24, 528, 98, 144, 3536, 0] combine_rows_per_src=[8078, 7538, 6946, 8186, 6898, 8062, 7960, 1486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55524 payload_rows_remote=47332 dup_rows=10012 dup_rows_remote=9972 dedup_ratio=0.152771 router_tokens_per_expert=[0, 0, 1683, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 1585, 0, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2999, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0] payload_rows_per_dst=[7952, 6992, 1836, 8192, 8192, 8186, 6962, 7212] dup_rows_per_dst=[5412, 1020, 0, 40, 92, 376, 2228, 844] combine_rows_per_src=[7952, 6992, 1836, 8192, 8192, 8186, 6962, 7212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=174956544 dispatch_meta_bytes=209568 combine_payload_bytes=174956544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=175166112 ingress_bytes=212705280 link_bytes=212705280 token_bytes=350122656 weight_movement_bytes=37748736 total_nvlink_bytes=387871392 weight_to_token_ratio=0.107816 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038037335e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47674 payload_rows_remote=39482 dup_rows=17862 dup_rows_remote=6274 dedup_ratio=0.272552 router_tokens_per_expert=[0, 0, 266, 0, 0, 0, 9, 0, 0, 0, 0, 4051, 69, 64, 1, 4089, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1319, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1796, 0, 0, 0, 0, 0, 1, 0, 0, 1162, 2685, 0, 6, 0, 77, 2, 38, 0, 0, 4, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 1, 0, 4077, 0, 0, 0, 0, 1, 0, 0, 2933, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 3, 32, 0, 0, 1, 84, 0, 0, 0, 0, 1366, 4, 222, 0, 0, 3, 0, 0, 0, 4087, 0, 100, 0, 3, 13] payload_rows_per_dst=[754, 2642, 8182, 8132, 8156, 5876, 5740, 8192] dup_rows_per_dst=[64, 2, 4998, 294, 116, 0, 800, 11588] combine_rows_per_src=[754, 2642, 8182, 8132, 8156, 5876, 5740, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49580 payload_rows_remote=42006 dup_rows=15956 dup_rows_remote=7854 dedup_ratio=0.243469 router_tokens_per_expert=[0, 1, 91, 0, 0, 0, 0, 0, 0, 0, 24, 1832, 0, 0, 0, 0, 0, 519, 42, 0, 235, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 74, 0, 0, 0, 0, 0, 3543, 1448, 248, 1, 775, 0, 1004, 2, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 134, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 2, 3, 0, 0, 2, 0, 19, 2820, 0, 3632, 0, 8, 0, 3838, 0, 0, 0, 0, 41, 0, 0, 0, 0, 1, 0, 0, 10, 3480] payload_rows_per_dst=[5592, 6414, 6164, 3620, 8176, 4196, 7574, 7844] dup_rows_per_dst=[2310, 940, 1056, 212, 232, 214, 8102, 2890] combine_rows_per_src=[5592, 6414, 6164, 3620, 8176, 4196, 7574, 7844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=163782656 dispatch_meta_bytes=226944 combine_payload_bytes=163782656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=164009600 ingress_bytes=182657024 link_bytes=182657024 token_bytes=327792256 weight_movement_bytes=18874368 total_nvlink_bytes=346666624 weight_to_token_ratio=0.057580 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.083333 gemm_active_groups=14 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.750129731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=195035136 dispatch_meta_bytes=213160 combine_payload_bytes=195035136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195248296 ingress_bytes=213909504 link_bytes=213909504 token_bytes=390283432 weight_movement_bytes=18874368 total_nvlink_bytes=409157800 weight_to_token_ratio=0.048361 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049575617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48186 payload_rows_remote=40022 dup_rows=17350 dup_rows_remote=13930 dedup_ratio=0.264740 router_tokens_per_expert=[7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 18, 0, 0, 0, 0, 0, 0, 3, 0, 0, 252, 4026, 0, 10, 0, 2, 7, 0, 0, 40, 940, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 570, 0, 0, 12, 1, 0, 3914, 0, 2633, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 10, 0, 1865, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 3700, 0, 4087, 0, 0, 0, 0, 5] payload_rows_per_dst=[8120, 560, 1970, 8186, 5292, 8164, 8188, 7706] dup_rows_per_dst=[12, 10, 10, 9162, 8, 3420, 3158, 1570] combine_rows_per_src=[8120, 560, 1970, 8186, 5292, 8164, 8188, 7706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=193871872 dispatch_meta_bytes=229216 combine_payload_bytes=193871872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194101088 ingress_bytes=212746240 link_bytes=212746240 token_bytes=387972960 weight_movement_bytes=18874368 total_nvlink_bytes=406847328 weight_to_token_ratio=0.048649 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038429794e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=161718272 dispatch_meta_bytes=183024 combine_payload_bytes=161718272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=161901296 ingress_bytes=199467008 link_bytes=199467008 token_bytes=323619568 weight_movement_bytes=37748736 total_nvlink_bytes=361368304 weight_to_token_ratio=0.116645 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.911194727e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53842 payload_rows_remote=46530 dup_rows=11694 dup_rows_remote=9386 dedup_ratio=0.178436 router_tokens_per_expert=[8, 140, 0, 0, 0, 0, 0, 10, 17, 0, 0, 0, 4, 0, 0, 0, 12, 117, 0, 335, 0, 0, 0, 0, 0, 0, 461, 3815, 0, 1, 0, 0, 3032, 0, 0, 0, 1630, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 1609, 17, 0, 29, 0, 0, 4096, 0, 444, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 9, 74, 0, 0, 0, 63, 0, 2952, 0, 4076, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[7876, 7532, 7312, 6426, 7588, 738, 8192, 8178] dup_rows_per_dst=[112, 1184, 2308, 1078, 832, 2, 26, 6152] combine_rows_per_src=[7876, 7532, 7312, 6426, 7588, 738, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=172056576 dispatch_meta_bytes=199440 combine_payload_bytes=172056576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=172256016 ingress_bytes=209805312 link_bytes=209805312 token_bytes=344312592 weight_movement_bytes=37748736 total_nvlink_bytes=382061328 weight_to_token_ratio=0.109635 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.083333 gemm_active_groups=14 gemm_peak_group_rows=15360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.010251269e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=163930112 dispatch_meta_bytes=215808 combine_payload_bytes=163930112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=164145920 ingress_bytes=182804480 link_bytes=182804480 token_bytes=328076032 weight_movement_bytes=18874368 total_nvlink_bytes=346950400 weight_to_token_ratio=0.057530 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.751542582e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47394 payload_rows_remote=47164 dup_rows=18142 dup_rows_remote=18142 dedup_ratio=0.276825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 11, 2555, 0, 0, 2, 0, 0, 0, 0, 0, 106, 3, 0, 2, 0, 0, 0, 0, 0, 1, 2135, 4092, 0, 0, 0, 0, 2511, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 4096, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 0, 0, 0, 17, 4, 0, 1, 0, 0, 0, 0, 0, 58, 0, 883, 20, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 4368, 8168, 8142, 230, 8192, 8192, 1918] dup_rows_per_dst=[5136, 126, 5010, 6674, 0, 1096, 94, 6] combine_rows_per_src=[8184, 4368, 8168, 8142, 230, 8192, 8192, 1918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=190586880 dispatch_meta_bytes=223664 combine_payload_bytes=190586880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190810544 ingress_bytes=209461248 link_bytes=209461248 token_bytes=381397424 weight_movement_bytes=18874368 total_nvlink_bytes=400271792 weight_to_token_ratio=0.049487 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.006954617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42812 payload_rows_remote=34620 dup_rows=22724 dup_rows_remote=19630 dedup_ratio=0.346741 router_tokens_per_expert=[0, 1466, 0, 0, 0, 0, 0, 71, 0, 0, 8, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 122, 0, 0, 0, 0, 34, 3920, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 4093, 0, 444, 0, 0, 0, 0, 0, 496, 0, 0, 228, 0, 0, 43, 0, 2151, 4096, 0, 0, 0, 0, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7976, 8192, 7644, 424, 2106, 86, 8192] dup_rows_per_dst=[3094, 470, 13980, 4298, 0, 316, 0, 566] combine_rows_per_src=[8192, 7976, 8192, 7644, 424, 2106, 86, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49914 payload_rows_remote=41766 dup_rows=15622 dup_rows_remote=14740 dedup_ratio=0.238373 router_tokens_per_expert=[0, 0, 0, 0, 4, 115, 326, 3514, 1, 0, 0, 2, 25, 38, 0, 0, 0, 0, 0, 457, 1, 1, 0, 0, 0, 0, 4042, 3011, 0, 0, 0, 0, 899, 0, 0, 76, 1168, 0, 0, 0, 0, 0, 0, 0, 1777, 0, 32, 0, 1, 5, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4031, 2, 2295, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3150, 0, 0, 0, 14, 7, 0, 0, 0, 1, 28, 1, 0, 0, 0, 0, 0, 953, 0, 0, 0, 0, 9, 28, 0, 0, 0, 0, 1, 0, 1867, 0, 758, 0, 4096, 0, 0, 0, 0, 8] payload_rows_per_dst=[8170, 8148, 6308, 14, 8096, 6332, 8192, 4654] dup_rows_per_dst=[5902, 882, 1606, 4, 4562, 12, 2040, 614] combine_rows_per_src=[8170, 8148, 6308, 14, 8096, 6332, 8192, 4654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54768 payload_rows_remote=51476 dup_rows=10768 dup_rows_remote=10146 dedup_ratio=0.164307 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 3890, 4096, 0, 0, 0, 0, 523, 0, 0, 0, 3774, 0, 0, 22, 0, 0, 0, 0, 133, 0, 0, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 13, 0, 0, 0, 0, 0, 4096, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 1, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 2010, 2, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8018, 7668, 3292, 6484, 8192, 8192, 4730] dup_rows_per_dst=[744, 5510, 1236, 622, 0, 1916, 244, 496] combine_rows_per_src=[8192, 8018, 7668, 3292, 6484, 8192, 8192, 4730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42536 payload_rows_remote=34424 dup_rows=23000 dup_rows_remote=16642 dedup_ratio=0.350952 router_tokens_per_expert=[0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 36, 0, 0, 3, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2222, 4071, 0, 2, 0, 0, 18, 0, 0, 0, 332, 0, 0, 7, 0, 0, 23, 0, 1, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 2, 0, 0, 1, 0, 3381, 22, 0, 0, 0, 0, 0, 4096, 0, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 5, 0, 0, 0, 37, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 90, 3, 4001, 0, 4082, 0, 0, 0, 0, 0] payload_rows_per_dst=[378, 8186, 760, 6816, 8182, 8192, 1910, 8112] dup_rows_per_dst=[36, 4414, 2, 2, 7278, 4788, 122, 6358] combine_rows_per_src=[378, 8186, 760, 6816, 8182, 8192, 1910, 8112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=193183744 dispatch_meta_bytes=261224 combine_payload_bytes=193183744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=193444968 ingress_bytes=230932480 link_bytes=230932480 token_bytes=386628712 weight_movement_bytes=37748736 total_nvlink_bytes=424377448 weight_to_token_ratio=0.097636 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.212681397e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=141803520 dispatch_meta_bytes=217000 combine_payload_bytes=141803520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=142020520 ingress_bytes=160677888 link_bytes=160677888 token_bytes=283824040 weight_movement_bytes=18874368 total_nvlink_bytes=302698408 weight_to_token_ratio=0.066500 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.539536464e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=171073536 dispatch_meta_bytes=226024 combine_payload_bytes=171073536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=171299560 ingress_bytes=208822272 link_bytes=208822272 token_bytes=342373096 weight_movement_bytes=37748736 total_nvlink_bytes=380121832 weight_to_token_ratio=0.110256 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000832263e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55246 payload_rows_remote=47054 dup_rows=10290 dup_rows_remote=9860 dedup_ratio=0.157013 router_tokens_per_expert=[0, 0, 0, 594, 0, 0, 1572, 0, 2, 0, 51, 4096, 0, 0, 1, 0, 0, 0, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 315, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 2071, 0, 0, 3802, 1645, 5, 0, 6, 0, 0, 173, 0, 0, 152, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 413, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 172, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1840, 0, 0, 1379, 0, 0, 0, 3368] payload_rows_per_dst=[4168, 6590, 6898, 6272, 7106, 7828, 8192, 8192] dup_rows_per_dst=[272, 0, 692, 1832, 0, 620, 430, 6444] combine_rows_per_src=[4168, 6590, 6898, 6272, 7106, 7828, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52642 payload_rows_remote=44452 dup_rows=12894 dup_rows_remote=6440 dedup_ratio=0.196747 router_tokens_per_expert=[0, 3900, 4, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 0, 0, 0, 4, 0, 0, 1, 3, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 2, 0, 0, 0, 0, 1, 0, 4057, 0, 36, 1818, 4, 0, 0, 12, 0, 0, 0, 100, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 21, 0, 0, 1353, 0, 0, 0, 4083, 0, 3, 0, 0, 31, 0, 0, 2, 0, 0, 24, 0, 0, 216, 1, 3006, 4039, 0, 0, 0, 0, 36, 28, 2, 668, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 0, 0, 0, 21, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[7828, 566, 6046, 6180, 8182, 8190, 8172, 7478] dup_rows_per_dst=[320, 6, 1868, 38, 2768, 6454, 1420, 20] combine_rows_per_src=[7828, 566, 6046, 6180, 8182, 8190, 8172, 7478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210845696 dispatch_meta_bytes=246488 combine_payload_bytes=210845696 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=211092184 ingress_bytes=286343168 link_bytes=286343168 token_bytes=421937880 weight_movement_bytes=75497472 total_nvlink_bytes=497435352 weight_to_token_ratio=0.178930 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.743599346e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=141000704 dispatch_meta_bytes=204264 combine_payload_bytes=141000704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=141204968 ingress_bytes=178749440 link_bytes=178749440 token_bytes=282205672 weight_movement_bytes=37748736 total_nvlink_bytes=319954408 weight_to_token_ratio=0.133763 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.712689184e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53786 payload_rows_remote=48726 dup_rows=11750 dup_rows_remote=11436 dedup_ratio=0.179291 router_tokens_per_expert=[0, 22, 0, 0, 53, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 0, 0, 165, 4080, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 3800, 0, 247, 0, 0, 0, 0, 0, 8, 0, 0, 12, 0, 0, 0, 0, 979, 4096, 0, 0, 0, 0, 772, 0, 0, 25, 0, 0, 0, 0, 3, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 5060, 7866, 7604, 522, 8168, 8192] dup_rows_per_dst=[168, 6956, 314, 1958, 716, 12, 1626, 0] combine_rows_per_src=[8192, 8182, 5060, 7866, 7604, 522, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=182075392 dispatch_meta_bytes=203568 combine_payload_bytes=182075392 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=182278960 ingress_bytes=238698496 link_bytes=238698496 token_bytes=364354352 weight_movement_bytes=56623104 total_nvlink_bytes=420977456 weight_to_token_ratio=0.155407 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=16128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287091542e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=192733184 dispatch_meta_bytes=227656 combine_payload_bytes=192733184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=192960840 ingress_bytes=230481920 link_bytes=230481920 token_bytes=385694024 weight_movement_bytes=37748736 total_nvlink_bytes=423442760 weight_to_token_ratio=0.097872 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.208364353e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=199581696 dispatch_meta_bytes=240648 combine_payload_bytes=199581696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199822344 ingress_bytes=256204800 link_bytes=256204800 token_bytes=399404040 weight_movement_bytes=56623104 total_nvlink_bytes=456027144 weight_to_token_ratio=0.141769 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=10496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.454828333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48574 payload_rows_remote=41194 dup_rows=16962 dup_rows_remote=15416 dedup_ratio=0.258820 router_tokens_per_expert=[0, 1207, 0, 0, 0, 0, 0, 2, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1981, 0, 0, 0, 0, 0, 0, 0, 3446, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 0, 0, 3354, 0, 2, 0, 0, 0, 0, 0, 1678, 0, 0, 1, 0, 0, 1, 0, 3963, 4096, 0, 0, 0, 0, 22, 0, 0, 375, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2488, 7310, 252, 8074, 7380, 7818, 7060, 8192] dup_rows_per_dst=[428, 2670, 6, 1580, 1546, 9594, 628, 510] combine_rows_per_src=[2488, 7310, 252, 8074, 7380, 7818, 7060, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50074 payload_rows_remote=41926 dup_rows=15462 dup_rows_remote=9988 dedup_ratio=0.235931 router_tokens_per_expert=[4088, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 4010, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 231, 0, 0, 0, 0, 3, 0, 0, 0, 27, 0, 0, 0, 0, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 3590, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 8192, 6218, 8166, 5398, 3602, 7180, 3170] dup_rows_per_dst=[5474, 8082, 424, 1242, 46, 14, 180, 0] combine_rows_per_src=[8148, 8192, 6218, 8166, 5398, 3602, 7180, 3170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50580 payload_rows_remote=42692 dup_rows=14956 dup_rows_remote=14932 dedup_ratio=0.228210 router_tokens_per_expert=[0, 3837, 2, 0, 29, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 3932, 0, 0, 1, 1, 0, 1, 0, 0, 7, 11, 0, 0, 0, 0, 2, 0, 0, 3934, 0, 0, 2135, 121, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 146, 0, 1, 0, 760, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 584, 2, 1, 128, 0, 0, 5, 0, 1655, 4068, 0, 83, 0, 0, 0, 0, 0, 3966, 0, 6, 0, 0, 2, 0, 1, 7, 0, 0, 2, 1, 2524, 1, 0, 0, 0, 17, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8178, 7888, 7612, 4402, 7910, 1404, 8100, 5086] dup_rows_per_dst=[7748, 24, 5214, 266, 440, 36, 1218, 10] combine_rows_per_src=[8178, 7888, 7612, 4402, 7910, 1404, 8100, 5086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50042 payload_rows_remote=41858 dup_rows=15494 dup_rows_remote=7866 dedup_ratio=0.236420 router_tokens_per_expert=[0, 1391, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 920, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 33, 0, 0, 72, 0, 3828, 4095, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8156, 852, 8184, 8192, 216, 8090, 8160] dup_rows_per_dst=[2800, 2886, 0, 7628, 1850, 2, 308, 20] combine_rows_per_src=[8192, 8156, 852, 8184, 8192, 216, 8090, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47578 payload_rows_remote=40760 dup_rows=17958 dup_rows_remote=13808 dedup_ratio=0.274017 router_tokens_per_expert=[0, 3040, 0, 84, 557, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 3309, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3371, 0, 0, 0, 0, 1, 0, 0, 3906, 0, 17, 603, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 36, 0, 0, 93, 0, 0, 0, 0, 1562, 4053, 0, 0, 1, 15, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 36, 0, 0, 44, 0, 2, 0, 3118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6444, 6642, 6944, 1722, 2686, 8138, 8184, 6818] dup_rows_per_dst=[964, 4, 1040, 0, 92, 4276, 7432, 4150] combine_rows_per_src=[6444, 6642, 6944, 1722, 2686, 8138, 8184, 6818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=168730624 dispatch_meta_bytes=226440 combine_payload_bytes=168730624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=168957064 ingress_bytes=225353728 link_bytes=225353728 token_bytes=337687688 weight_movement_bytes=56623104 total_nvlink_bytes=394310792 weight_to_token_ratio=0.167679 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159228541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=171728896 dispatch_meta_bytes=207656 combine_payload_bytes=171728896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=171936552 ingress_bytes=228352000 link_bytes=228352000 token_bytes=343665448 weight_movement_bytes=56623104 total_nvlink_bytes=400288552 weight_to_token_ratio=0.164762 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.187956508e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=174866432 dispatch_meta_bytes=230496 combine_payload_bytes=174866432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175096928 ingress_bytes=193740800 link_bytes=193740800 token_bytes=349963360 weight_movement_bytes=18874368 total_nvlink_bytes=368837728 weight_to_token_ratio=0.053932 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.065972 gemm_active_groups=16 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856329019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52146 payload_rows_remote=44060 dup_rows=13390 dup_rows_remote=7258 dedup_ratio=0.204315 router_tokens_per_expert=[4096, 5, 1, 0, 0, 0, 0, 0, 10, 2557, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 810, 18, 0, 2, 0, 0, 0, 2, 0, 0, 0, 6, 0, 4052, 0, 0, 6, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 62, 0, 3, 2075, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 0, 0, 0, 2, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 13, 0, 1, 0, 2319, 0, 0, 4086, 0, 0, 0, 0, 0, 24, 0, 0, 50, 0, 13, 0, 36, 0, 0, 0, 22, 0] payload_rows_per_dst=[5140, 1838, 8144, 7908, 8192, 8086, 8032, 4806] dup_rows_per_dst=[6, 214, 14, 3472, 2602, 6132, 828, 122] combine_rows_per_src=[5140, 1838, 8144, 7908, 8192, 8086, 8032, 4806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=171450368 dispatch_meta_bytes=198896 combine_payload_bytes=171450368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171649264 ingress_bytes=190324736 link_bytes=190324736 token_bytes=343099632 weight_movement_bytes=18874368 total_nvlink_bytes=361974000 weight_to_token_ratio=0.055011 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.823597975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52102 payload_rows_remote=43934 dup_rows=13434 dup_rows_remote=10014 dedup_ratio=0.204987 router_tokens_per_expert=[0, 0, 3340, 0, 0, 0, 0, 0, 4096, 0, 0, 1329, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 45, 0, 12, 0, 3758, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1144, 0, 26, 0, 0, 0, 0, 564, 4062, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 4038, 0, 0] payload_rows_per_dst=[7506, 2742, 8008, 7528, 1766, 8192, 8168, 8192] dup_rows_per_dst=[1832, 8, 36, 72, 22, 54, 3420, 7990] combine_rows_per_src=[7506, 2742, 8008, 7528, 1766, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55452 payload_rows_remote=47260 dup_rows=10084 dup_rows_remote=9650 dedup_ratio=0.153870 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 0, 2080, 0, 0, 0, 217, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 93, 0, 0, 0, 200, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 1918, 0, 0, 0, 3877, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 2707, 0, 68, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8064, 8186, 8192, 7640, 8192, 1726, 7898, 5554] dup_rows_per_dst=[0, 4174, 434, 1528, 4, 74, 3868, 2] combine_rows_per_src=[8064, 8186, 8192, 7640, 8192, 1726, 7898, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=166952960 dispatch_meta_bytes=218272 combine_payload_bytes=166952960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=167171232 ingress_bytes=185827328 link_bytes=185827328 token_bytes=334124192 weight_movement_bytes=18874368 total_nvlink_bytes=352998560 weight_to_token_ratio=0.056489 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.780506024e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=180469760 dispatch_meta_bytes=205272 combine_payload_bytes=180469760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=180675032 ingress_bytes=218218496 link_bytes=218218496 token_bytes=361144792 weight_movement_bytes=37748736 total_nvlink_bytes=398893528 weight_to_token_ratio=0.104525 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.090862258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=179953664 dispatch_meta_bytes=215792 combine_payload_bytes=179953664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=180169456 ingress_bytes=217702400 link_bytes=217702400 token_bytes=360123120 weight_movement_bytes=37748736 total_nvlink_bytes=397871856 weight_to_token_ratio=0.104822 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.085917281e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=193576960 dispatch_meta_bytes=227640 combine_payload_bytes=193576960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=193804600 ingress_bytes=231325696 link_bytes=231325696 token_bytes=387381560 weight_movement_bytes=37748736 total_nvlink_bytes=425130296 weight_to_token_ratio=0.097446 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216449000e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52978 payload_rows_remote=44786 dup_rows=12558 dup_rows_remote=12432 dedup_ratio=0.191620 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 50, 0, 0, 0, 4, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 56, 0, 0, 0, 0, 54, 0, 0, 0, 2782, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 3933, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 0, 1622, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7352, 8192, 8192, 4680, 8192, 8178, 22, 8170] dup_rows_per_dst=[0, 104, 10, 1438, 126, 5608, 0, 5272] combine_rows_per_src=[7352, 8192, 8192, 4680, 8192, 8178, 22, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53110 payload_rows_remote=44968 dup_rows=12426 dup_rows_remote=4032 dedup_ratio=0.189606 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3825, 0, 0, 2164, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 77, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 392, 55, 4053, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4082, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 2809, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8142, 8188, 6664, 8164, 7908, 8192, 5620, 232] dup_rows_per_dst=[8394, 282, 3554, 22, 0, 174, 0, 0] combine_rows_per_src=[8142, 8188, 6664, 8164, 7908, 8192, 5620, 232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55178 payload_rows_remote=50420 dup_rows=10358 dup_rows_remote=9772 dedup_ratio=0.158051 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 5, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3141, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 2762, 0, 0, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 6, 0, 0, 0, 4070, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7948, 8192, 8192, 4758, 8192, 6418, 5604, 5874] dup_rows_per_dst=[10, 6494, 4, 586, 0, 974, 242, 2048] combine_rows_per_src=[7948, 8192, 8192, 4758, 8192, 6418, 5604, 5874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57414 payload_rows_remote=49224 dup_rows=8122 dup_rows_remote=6258 dedup_ratio=0.123932 router_tokens_per_expert=[3685, 4, 0, 0, 0, 0, 0, 0, 0, 3000, 7, 0, 0, 0, 0, 3181, 2, 1, 0, 0, 0, 0, 0, 218, 0, 245, 85, 6, 389, 0, 0, 0, 0, 0, 684, 0, 0, 7, 3970, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 60, 0, 7, 4032, 0, 6, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 3, 0, 1, 0, 259, 0, 13, 6, 0, 0, 9, 0, 0, 0, 0, 0, 55, 6, 0, 0, 0, 3331, 0, 0, 4081, 0, 0, 9, 10, 0, 0, 0, 4, 1310, 0, 1, 0, 0, 1, 7, 1, 0, 0] payload_rows_per_dst=[8064, 8190, 8040, 8126, 8026, 7520, 6776, 2672] dup_rows_per_dst=[4320, 1864, 1284, 88, 90, 454, 8, 14] combine_rows_per_src=[8064, 8190, 8040, 8126, 8026, 7520, 6776, 2672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=183443456 dispatch_meta_bytes=228872 combine_payload_bytes=183443456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=183672328 ingress_bytes=202317824 link_bytes=202317824 token_bytes=367115784 weight_movement_bytes=18874368 total_nvlink_bytes=385990152 weight_to_token_ratio=0.051413 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.938509843e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51474 payload_rows_remote=43286 dup_rows=14062 dup_rows_remote=7020 dedup_ratio=0.214569 router_tokens_per_expert=[4083, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 17, 1, 1, 4, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 3779, 0, 1, 0, 0, 5, 0, 0, 27, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 973, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4020, 0, 0, 4086, 0, 0, 512, 0, 0, 0, 2, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2942, 52, 8192, 7638, 8188, 8100, 8174, 8188] dup_rows_per_dst=[2, 0, 2, 64, 18, 4986, 1948, 7042] combine_rows_per_src=[2942, 52, 8192, 7638, 8188, 8100, 8174, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=184188928 dispatch_meta_bytes=196000 combine_payload_bytes=184188928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=184384928 ingress_bytes=221937664 link_bytes=221937664 token_bytes=368573856 weight_movement_bytes=37748736 total_nvlink_bytes=406322592 weight_to_token_ratio=0.102418 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.126497496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54460 payload_rows_remote=46278 dup_rows=11076 dup_rows_remote=11060 dedup_ratio=0.169006 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 2692, 0, 1, 0, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 1, 0, 0, 1942, 0, 1, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4086, 0, 0, 0, 3459, 0, 1, 0, 0, 0, 449, 15, 0, 0, 0, 0, 0, 4018, 3, 0, 29, 0, 4020, 0, 0, 739, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 1, 0, 0, 18, 2838, 0, 0, 0, 18, 0, 0, 113, 0, 0, 11, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5392, 4356, 7060, 8186, 8188, 8182, 7316, 5780] dup_rows_per_dst=[2, 1848, 30, 1946, 1498, 16, 5732, 4] combine_rows_per_src=[5392, 4356, 7060, 8186, 8188, 8182, 7316, 5780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=206520320 dispatch_meta_bytes=240768 combine_payload_bytes=206520320 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=206761088 ingress_bytes=225394688 link_bytes=225394688 token_bytes=413281408 weight_movement_bytes=18874368 total_nvlink_bytes=432155776 weight_to_token_ratio=0.045670 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159620999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201621504 dispatch_meta_bytes=221928 combine_payload_bytes=201621504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201843432 ingress_bytes=220495872 link_bytes=220495872 token_bytes=403464936 weight_movement_bytes=18874368 total_nvlink_bytes=422339304 weight_to_token_ratio=0.046781 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112682955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56556 payload_rows_remote=48374 dup_rows=8980 dup_rows_remote=8406 dedup_ratio=0.137024 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 6, 0, 0, 0, 0, 25, 0, 0, 0, 0, 2013, 0, 236, 2386, 0, 47, 0, 90, 0, 0, 0, 0, 0, 203, 0, 4007, 0, 0, 0, 3, 0, 75, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3534, 0, 0, 0, 0, 1051, 0, 0, 3256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7256, 8186, 8182, 2102, 6512, 8192, 8190, 7936] dup_rows_per_dst=[12, 8296, 574, 6, 0, 2, 28, 62] combine_rows_per_src=[7256, 8186, 8182, 2102, 6512, 8192, 8190, 7936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44724 payload_rows_remote=40498 dup_rows=20812 dup_rows_remote=20092 dedup_ratio=0.317566 router_tokens_per_expert=[1, 0, 3984, 0, 0, 3571, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 18, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 0, 82, 0, 0, 0, 0, 0, 0, 0, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 67, 0, 0, 0, 3990, 0, 0, 12, 0, 4092, 0, 298] payload_rows_per_dst=[2, 8188, 2192, 7942, 5902, 8156, 4226, 8116] dup_rows_per_dst=[0, 5840, 2, 4978, 0, 2144, 720, 7128] combine_rows_per_src=[2, 8188, 2192, 7942, 5902, 8156, 4226, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=177299456 dispatch_meta_bytes=201224 combine_payload_bytes=177299456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=177500680 ingress_bytes=196173824 link_bytes=196173824 token_bytes=354800136 weight_movement_bytes=18874368 total_nvlink_bytes=373674504 weight_to_token_ratio=0.053197 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=16128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.879641058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=189554688 dispatch_meta_bytes=229352 combine_payload_bytes=189554688 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=189784040 ingress_bytes=208429056 link_bytes=208429056 token_bytes=379338728 weight_movement_bytes=18874368 total_nvlink_bytes=398213096 weight_to_token_ratio=0.049756 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.997064661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=198139904 dispatch_meta_bytes=227120 combine_payload_bytes=198139904 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=198367024 ingress_bytes=254763008 link_bytes=254763008 token_bytes=396506928 weight_movement_bytes=56623104 total_nvlink_bytes=453130032 weight_to_token_ratio=0.142805 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.441013791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50232 payload_rows_remote=42040 dup_rows=15304 dup_rows_remote=12150 dedup_ratio=0.233521 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 4061, 0, 0, 3, 0, 1596, 0, 0, 3, 117, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 4091, 0, 0, 0, 788, 0, 0, 0, 0, 0, 16, 47, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2288, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 412, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 22, 3911, 0, 0, 0, 0, 0, 85] payload_rows_per_dst=[8122, 8182, 3316, 4666, 8192, 8192, 1376, 8186] dup_rows_per_dst=[376, 3434, 210, 46, 3154, 28, 32, 8024] combine_rows_per_src=[8122, 8182, 3316, 4666, 8192, 8192, 1376, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=165879808 dispatch_meta_bytes=242360 combine_payload_bytes=165879808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=166122168 ingress_bytes=184754176 link_bytes=184754176 token_bytes=332001976 weight_movement_bytes=18874368 total_nvlink_bytes=350876344 weight_to_token_ratio=0.056850 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.770223610e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55818 payload_rows_remote=47636 dup_rows=9718 dup_rows_remote=9678 dedup_ratio=0.148285 router_tokens_per_expert=[0, 4091, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 19, 0, 0, 0, 2, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 115, 2310, 0, 2189, 141, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3987, 0, 0, 4, 1859, 0, 163, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 6144, 8192, 8192, 4962, 8188, 8100, 3858] dup_rows_per_dst=[40, 638, 26, 8394, 8, 416, 0, 196] combine_rows_per_src=[8182, 6144, 8192, 8192, 4962, 8188, 8100, 3858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60708 payload_rows_remote=52726 dup_rows=4828 dup_rows_remote=4802 dedup_ratio=0.073669 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 13, 0, 6, 0, 0, 12, 388, 0, 0, 0, 72, 0, 0, 109, 0, 0, 0, 32, 0, 0, 0, 0, 0, 320, 0, 4096, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3990, 0, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3280, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 526, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4062, 0, 24, 109, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8192, 7598, 7982, 5328, 8192, 7072, 8170] dup_rows_per_dst=[26, 1170, 2486, 26, 338, 8, 554, 220] combine_rows_per_src=[8174, 8192, 7598, 7982, 5328, 8192, 7072, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=172195840 dispatch_meta_bytes=216760 combine_payload_bytes=172195840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172412600 ingress_bytes=191070208 link_bytes=191070208 token_bytes=344608440 weight_movement_bytes=18874368 total_nvlink_bytes=363482808 weight_to_token_ratio=0.054770 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.830740721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51138 payload_rows_remote=44152 dup_rows=14398 dup_rows_remote=14226 dedup_ratio=0.219696 router_tokens_per_expert=[3, 0, 0, 0, 2196, 0, 0, 0, 0, 0, 0, 0, 3919, 82, 0, 1, 1, 0, 0, 0, 0, 85, 0, 0, 6, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 145, 23, 0, 4044, 0, 2, 0, 4054, 0, 0, 0, 0, 3, 1, 0, 0, 4, 10, 0, 0, 3485, 0, 0, 0, 0, 3558, 0, 0, 429, 12, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 3592, 0, 0, 1, 0, 24, 0, 0, 266, 0, 147, 0, 0, 0, 0, 0, 617, 169, 0, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3068, 0, 0, 1, 0, 0, 1, 0, 0, 2] payload_rows_per_dst=[8180, 6986, 8192, 7146, 890, 7818, 5778, 6148] dup_rows_per_dst=[4222, 172, 8344, 6, 0, 242, 1408, 4] combine_rows_per_src=[8180, 6986, 8192, 7146, 890, 7818, 5778, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50396 payload_rows_remote=43530 dup_rows=15140 dup_rows_remote=13504 dedup_ratio=0.231018 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 16, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3958, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 46, 0, 0, 3906, 4, 0, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 874, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2969, 0, 0, 1219, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[482, 2768, 7926, 8192, 8182, 8068, 7912, 6866] dup_rows_per_dst=[0, 0, 6, 7900, 3894, 34, 1670, 1636] combine_rows_per_src=[482, 2768, 7926, 8192, 8182, 8068, 7912, 6866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=195117056 dispatch_meta_bytes=229256 combine_payload_bytes=195117056 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=195346312 ingress_bytes=213991424 link_bytes=213991424 token_bytes=390463368 weight_movement_bytes=18874368 total_nvlink_bytes=409337736 weight_to_token_ratio=0.048338 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050360534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=215965696 dispatch_meta_bytes=230112 combine_payload_bytes=215965696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=216195808 ingress_bytes=272588800 link_bytes=272588800 token_bytes=432161504 weight_movement_bytes=56623104 total_nvlink_bytes=488784608 weight_to_token_ratio=0.131023 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611811760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59322 payload_rows_remote=52206 dup_rows=6214 dup_rows_remote=6184 dedup_ratio=0.094818 router_tokens_per_expert=[0, 4085, 0, 60, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 87, 1, 6, 0, 0, 0, 0, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3954, 0, 0, 0, 0, 185, 0, 0, 0, 2546, 698, 11, 1409, 114, 7, 3768, 47, 0, 1829, 0, 0, 0, 0, 0, 0, 81, 29, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 116, 0, 9, 0, 0, 0, 1, 0, 0, 1, 0, 35, 0, 0, 4061, 4073, 0, 0, 0, 278, 0, 19, 11, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 7682, 8192, 7918, 6594, 7116, 5470, 8168] dup_rows_per_dst=[318, 2742, 276, 666, 1408, 30, 180, 594] combine_rows_per_src=[8182, 7682, 8192, 7918, 6594, 7116, 5470, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=180846592 dispatch_meta_bytes=233512 combine_payload_bytes=180846592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=181080104 ingress_bytes=199720960 link_bytes=199720960 token_bytes=361926696 weight_movement_bytes=18874368 total_nvlink_bytes=380801064 weight_to_token_ratio=0.052150 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.079861 gemm_active_groups=15 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.913627970e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56102 payload_rows_remote=47910 dup_rows=9434 dup_rows_remote=9428 dedup_ratio=0.143951 router_tokens_per_expert=[0, 4089, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 709, 3743, 0, 4046, 0, 1, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4037, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8098, 8192, 5322, 8190, 4964, 8182, 4976] dup_rows_per_dst=[50, 1926, 6, 54, 5016, 22, 2360, 0] combine_rows_per_src=[8178, 8098, 8192, 5322, 8190, 4964, 8182, 4976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57192 payload_rows_remote=49288 dup_rows=8344 dup_rows_remote=7250 dedup_ratio=0.127319 router_tokens_per_expert=[0, 0, 765, 0, 2, 0, 0, 0, 0, 0, 0, 4017, 0, 1561, 0, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 1, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 385, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 99, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0] payload_rows_per_dst=[4410, 7270, 6942, 8090, 8192, 6194, 7904, 8190] dup_rows_per_dst=[246, 0, 2972, 150, 790, 4, 1094, 3088] combine_rows_per_src=[4410, 7270, 6942, 8090, 8192, 6194, 7904, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=178298880 dispatch_meta_bytes=228136 combine_payload_bytes=178298880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=178527016 ingress_bytes=216047616 link_bytes=216047616 token_bytes=356825896 weight_movement_bytes=37748736 total_nvlink_bytes=394574632 weight_to_token_ratio=0.105790 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.070061954e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213835776 dispatch_meta_bytes=233560 combine_payload_bytes=213835776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214069336 ingress_bytes=232710144 link_bytes=232710144 token_bytes=427905112 weight_movement_bytes=18874368 total_nvlink_bytes=446779480 weight_to_token_ratio=0.044109 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229714099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54482 payload_rows_remote=53556 dup_rows=11054 dup_rows_remote=11054 dedup_ratio=0.168671 router_tokens_per_expert=[0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 0, 371, 3481, 0, 3637, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3888, 0, 0, 0, 116, 0, 2374, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 7274, 8192, 5934, 926, 8192, 8180, 7600] dup_rows_per_dst=[0, 0, 26, 12, 0, 3426, 5856, 1734] combine_rows_per_src=[8184, 7274, 8192, 5934, 926, 8192, 8180, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=196239360 dispatch_meta_bytes=229352 combine_payload_bytes=196239360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196468712 ingress_bytes=215113728 link_bytes=215113728 token_bytes=392708072 weight_movement_bytes=18874368 total_nvlink_bytes=411582440 weight_to_token_ratio=0.048062 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.061113899e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=201883648 dispatch_meta_bytes=226152 combine_payload_bytes=201883648 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=202109800 ingress_bytes=239632384 link_bytes=239632384 token_bytes=403993448 weight_movement_bytes=37748736 total_nvlink_bytes=441742184 weight_to_token_ratio=0.093439 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.296039597e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54730 payload_rows_remote=47344 dup_rows=10806 dup_rows_remote=8424 dedup_ratio=0.164886 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 1862, 0, 0, 4065, 926, 0, 2137, 3, 0, 200, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3995, 0, 0, 0, 531, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7812, 8192, 7386, 7448, 8114, 4968, 2618] dup_rows_per_dst=[154, 5978, 0, 2382, 368, 1220, 608, 96] combine_rows_per_src=[8192, 7812, 8192, 7386, 7448, 8114, 4968, 2618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219365376 dispatch_meta_bytes=258440 combine_payload_bytes=219365376 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=219623816 ingress_bytes=313737216 link_bytes=313737216 token_bytes=438989192 weight_movement_bytes=94371840 total_nvlink_bytes=533361032 weight_to_token_ratio=0.214975 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.006075635e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55588 payload_rows_remote=47396 dup_rows=9948 dup_rows_remote=9630 dedup_ratio=0.151794 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 102, 10, 4096, 0, 0, 0, 0, 0, 0, 919, 0, 0, 3897, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 2898, 2, 0, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 635, 4095, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 8138, 8180, 8192, 8190, 3188, 5780, 5728] dup_rows_per_dst=[318, 246, 7326, 1376, 0, 52, 630, 0] combine_rows_per_src=[8192, 8138, 8180, 8192, 8190, 3188, 5780, 5728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60126 payload_rows_remote=54702 dup_rows=5410 dup_rows_remote=5298 dedup_ratio=0.082550 router_tokens_per_expert=[0, 3906, 1, 1138, 0, 13, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 226, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 37, 0, 0, 0, 0, 42, 0, 1, 15, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 47, 0, 88, 4054, 44, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3968, 1991, 0, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0] payload_rows_per_dst=[8178, 5424, 8072, 6724, 8150, 8116, 7768, 7694] dup_rows_per_dst=[1950, 112, 54, 1198, 1836, 54, 18, 188] combine_rows_per_src=[8178, 5424, 8072, 6724, 8150, 8116, 7768, 7694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53588 payload_rows_remote=45396 dup_rows=11948 dup_rows_remote=2432 dedup_ratio=0.182312 router_tokens_per_expert=[0, 3954, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 101, 0, 0, 0, 0, 6, 0, 0, 0, 4009, 6, 0, 4030, 572, 16, 3940, 0, 0, 650, 0, 0, 0, 0, 0, 0, 43, 0, 0, 3678, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 4082, 4096, 0, 1, 0, 3454, 0, 49, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[5736, 7888, 7888, 218, 8172, 7440, 8054, 8192] dup_rows_per_dst=[0, 2, 0, 0, 1232, 16, 1182, 9516] combine_rows_per_src=[5736, 7888, 7888, 218, 8172, 7440, 8054, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57182 payload_rows_remote=48992 dup_rows=8354 dup_rows_remote=2184 dedup_ratio=0.127472 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 5, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 41, 0, 248, 0, 0, 4054, 0, 4, 2, 0, 2, 0, 19, 0, 0, 3154, 0, 0, 43, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 3, 0, 0, 0, 0, 0, 4066, 6, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4088, 79, 0, 2994, 0, 0, 0, 0, 0, 0, 1, 0, 3779, 4053, 4091, 0, 3, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106] payload_rows_per_dst=[8118, 8188, 6434, 8116, 2388, 8190, 7560, 8188] dup_rows_per_dst=[1294, 528, 14, 0, 0, 6170, 26, 322] combine_rows_per_src=[8118, 8188, 6434, 8116, 2388, 8190, 7560, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=193921024 dispatch_meta_bytes=223072 combine_payload_bytes=193921024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194144096 ingress_bytes=212795392 link_bytes=212795392 token_bytes=388065120 weight_movement_bytes=18874368 total_nvlink_bytes=406939488 weight_to_token_ratio=0.048637 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038900744e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=194134016 dispatch_meta_bytes=228104 combine_payload_bytes=194134016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194362120 ingress_bytes=213008384 link_bytes=213008384 token_bytes=388496136 weight_movement_bytes=18874368 total_nvlink_bytes=407370504 weight_to_token_ratio=0.048583 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040941529e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49766 payload_rows_remote=48352 dup_rows=15770 dup_rows_remote=15746 dedup_ratio=0.240631 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 3189, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 0, 653, 0, 0, 38, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 228, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 2554, 1, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 4096, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8040, 1414, 8192, 8192, 8182, 6300, 1254] dup_rows_per_dst=[6388, 1472, 24, 1582, 1536, 4330, 438, 0] combine_rows_per_src=[8192, 8040, 1414, 8192, 8192, 8182, 6300, 1254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224059392 dispatch_meta_bytes=240000 combine_payload_bytes=224059392 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224299392 ingress_bytes=261808128 link_bytes=261808128 token_bytes=448358784 weight_movement_bytes=37748736 total_nvlink_bytes=486107520 weight_to_token_ratio=0.084193 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=23040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.508516665e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49370 payload_rows_remote=41178 dup_rows=16166 dup_rows_remote=15574 dedup_ratio=0.246674 router_tokens_per_expert=[0, 0, 0, 0, 32, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1923, 4012, 0, 0, 0, 0, 12, 0, 0, 0, 4051, 0, 0, 44, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3508, 708, 0, 0, 0, 0, 0, 4096, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[3804, 6360, 8114, 8078, 4620, 8192, 8192, 2010] dup_rows_per_dst=[70, 1718, 106, 6680, 702, 6298, 592, 0] combine_rows_per_src=[3804, 6360, 8114, 8078, 4620, 8192, 8192, 2010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=185942016 dispatch_meta_bytes=191312 combine_payload_bytes=185942016 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=186133328 ingress_bytes=280313856 link_bytes=280313856 token_bytes=372075344 weight_movement_bytes=94371840 total_nvlink_bytes=466447184 weight_to_token_ratio=0.253636 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=14464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.685829445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54132 payload_rows_remote=46006 dup_rows=11404 dup_rows_remote=11404 dedup_ratio=0.174011 router_tokens_per_expert=[0, 0, 0, 0, 0, 223, 0, 8, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 43, 0, 0, 76, 0, 0, 13, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3805, 3, 0, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 2672, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8114, 208, 5346, 8126, 8190, 7872, 8084] dup_rows_per_dst=[466, 22, 56, 34, 0, 6108, 0, 4718] combine_rows_per_src=[8192, 8114, 208, 5346, 8126, 8190, 7872, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=200671232 dispatch_meta_bytes=204704 combine_payload_bytes=200671232 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=200875936 ingress_bytes=257294336 link_bytes=257294336 token_bytes=401547168 weight_movement_bytes=56623104 total_nvlink_bytes=458170272 weight_to_token_ratio=0.141012 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.465267731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=198049792 dispatch_meta_bytes=256392 combine_payload_bytes=198049792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=198306184 ingress_bytes=235798528 link_bytes=235798528 token_bytes=396355976 weight_movement_bytes=37748736 total_nvlink_bytes=434104712 weight_to_token_ratio=0.095239 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=12544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259305475e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=188440576 dispatch_meta_bytes=229640 combine_payload_bytes=188440576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=188670216 ingress_bytes=226189312 link_bytes=226189312 token_bytes=377110792 weight_movement_bytes=37748736 total_nvlink_bytes=414859528 weight_to_token_ratio=0.100100 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.167234695e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=168665088 dispatch_meta_bytes=227008 combine_payload_bytes=168665088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=168892096 ingress_bytes=187539456 link_bytes=187539456 token_bytes=337557184 weight_movement_bytes=18874368 total_nvlink_bytes=356431552 weight_to_token_ratio=0.055915 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.796910792e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51576 payload_rows_remote=43448 dup_rows=13960 dup_rows_remote=6610 dedup_ratio=0.213013 router_tokens_per_expert=[0, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 21, 0, 0, 0, 0, 0, 1466, 0, 0, 1, 0, 62, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2745, 187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4056, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8192, 7938, 8192, 3316, 374, 7954, 7482] dup_rows_per_dst=[7350, 1328, 4664, 342, 274, 0, 2, 0] combine_rows_per_src=[8128, 8192, 7938, 8192, 3316, 374, 7954, 7482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58900 payload_rows_remote=50726 dup_rows=6636 dup_rows_remote=6632 dedup_ratio=0.101257 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 839, 0, 0, 2843, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 6788, 8174, 8154, 8192, 6580, 4652] dup_rows_per_dst=[416, 2, 576, 4, 0, 816, 2912, 1910] combine_rows_per_src=[8168, 8192, 6788, 8174, 8154, 8192, 6580, 4652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48820 payload_rows_remote=40666 dup_rows=16716 dup_rows_remote=13670 dedup_ratio=0.255066 router_tokens_per_expert=[0, 156, 41, 0, 60, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 4, 0, 93, 12, 0, 0, 3877, 0, 0, 0, 0, 32, 0, 3990, 0, 0, 857, 0, 53, 600, 0, 0, 0, 1, 484, 0, 0, 0, 0, 961, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2890, 0, 8, 0, 0, 0, 0, 0, 0, 3837, 0, 2, 0, 0, 0, 6, 48, 0, 12, 0, 4016, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 2815, 4093, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 0, 0, 0, 36, 1, 17, 0, 1] payload_rows_per_dst=[8056, 8154, 8192, 7744, 5800, 136, 8186, 2552] dup_rows_per_dst=[230, 3046, 9636, 942, 1268, 4, 1584, 6] combine_rows_per_src=[8056, 8154, 8192, 7744, 5800, 136, 8186, 2552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46732 payload_rows_remote=38540 dup_rows=18804 dup_rows_remote=10846 dedup_ratio=0.286926 router_tokens_per_expert=[0, 0, 0, 0, 1, 4, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 4093, 0, 0, 0, 6, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 16, 4062, 0, 0, 0, 1, 0, 0, 0, 0, 4030, 0, 0, 3, 0, 0, 0, 0, 0, 0, 10, 3219, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 2537, 4091, 4072, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7] payload_rows_per_dst=[3768, 2660, 8192, 7860, 8160, 598, 7302, 8192] dup_rows_per_dst=[0, 564, 16, 42, 5676, 0, 4548, 7958] combine_rows_per_src=[3768, 2660, 8192, 7860, 8160, 598, 7302, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49836 payload_rows_remote=41650 dup_rows=15700 dup_rows_remote=7736 dedup_ratio=0.239563 router_tokens_per_expert=[0, 0, 0, 0, 0, 4088, 0, 56, 0, 0, 0, 0, 0, 0, 299, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4054, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0, 2958, 0, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 178, 0, 0, 0, 0, 14, 646, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 0, 0, 3477, 6, 34, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4079, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8114, 7872, 464, 1308, 8186, 7508, 8192] dup_rows_per_dst=[698, 0, 2678, 4, 40, 7964, 4224, 92] combine_rows_per_src=[8192, 8114, 7872, 464, 1308, 8186, 7508, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=177963008 dispatch_meta_bytes=200232 combine_payload_bytes=177963008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=178163240 ingress_bytes=215711744 link_bytes=215711744 token_bytes=356126248 weight_movement_bytes=37748736 total_nvlink_bytes=393874984 weight_to_token_ratio=0.105998 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.066843794e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55768 payload_rows_remote=52514 dup_rows=9768 dup_rows_remote=9758 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 2475, 0, 0, 2, 0, 54, 0, 0, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3875, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 3254, 8192, 8192, 5066, 6638, 8056] dup_rows_per_dst=[680, 10, 10, 228, 8838, 2, 0, 0] combine_rows_per_src=[8178, 8192, 3254, 8192, 8192, 5066, 6638, 8056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51372 payload_rows_remote=43340 dup_rows=14164 dup_rows_remote=10256 dedup_ratio=0.216125 router_tokens_per_expert=[0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 975, 73, 0, 0, 2, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 3185, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2127, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4071, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 5616, 8192, 8032, 530, 4478, 8150] dup_rows_per_dst=[2096, 14, 20, 8052, 3908, 0, 74, 0] combine_rows_per_src=[8182, 8192, 5616, 8192, 8032, 530, 4478, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=166567936 dispatch_meta_bytes=217344 combine_payload_bytes=166567936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=166785280 ingress_bytes=185442304 link_bytes=185442304 token_bytes=333353216 weight_movement_bytes=18874368 total_nvlink_bytes=352227584 weight_to_token_ratio=0.056620 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.776816914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49096 payload_rows_remote=44638 dup_rows=16440 dup_rows_remote=15854 dedup_ratio=0.250854 router_tokens_per_expert=[0, 239, 0, 0, 9, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 1684, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3779, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 18, 0, 0, 334, 0, 3701, 4094, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 11, 0, 0, 0, 9, 0, 0, 0, 4019, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[498, 5790, 8148, 5696, 8192, 8192, 4458, 8122] dup_rows_per_dst=[2, 10, 8, 0, 7554, 8260, 586, 20] combine_rows_per_src=[498, 5790, 8148, 5696, 8192, 8192, 4458, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=207773696 dispatch_meta_bytes=229432 combine_payload_bytes=207773696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208003128 ingress_bytes=226648064 link_bytes=226648064 token_bytes=415776824 weight_movement_bytes=18874368 total_nvlink_bytes=434651192 weight_to_token_ratio=0.045395 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.171630231e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=170598400 dispatch_meta_bytes=197544 combine_payload_bytes=170598400 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=170795944 ingress_bytes=227221504 link_bytes=227221504 token_bytes=341394344 weight_movement_bytes=56623104 total_nvlink_bytes=398017448 weight_to_token_ratio=0.165858 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=14848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.177124651e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157859840 dispatch_meta_bytes=197544 combine_payload_bytes=157859840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158057384 ingress_bytes=176734208 link_bytes=176734208 token_bytes=315917224 weight_movement_bytes=18874368 total_nvlink_bytes=334791592 weight_to_token_ratio=0.059745 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693380222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=215097344 dispatch_meta_bytes=249088 combine_payload_bytes=215097344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215346432 ingress_bytes=233971712 link_bytes=233971712 token_bytes=430443776 weight_movement_bytes=18874368 total_nvlink_bytes=449318144 weight_to_token_ratio=0.043849 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.241801823e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=177520640 dispatch_meta_bytes=214384 combine_payload_bytes=177520640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=177735024 ingress_bytes=196395008 link_bytes=196395008 token_bytes=355255664 weight_movement_bytes=18874368 total_nvlink_bytes=374130032 weight_to_token_ratio=0.053129 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=13568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.881760334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=182837248 dispatch_meta_bytes=241968 combine_payload_bytes=182837248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=183079216 ingress_bytes=201711616 link_bytes=201711616 token_bytes=365916464 weight_movement_bytes=18874368 total_nvlink_bytes=384790832 weight_to_token_ratio=0.051581 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.932701456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46856 payload_rows_remote=38664 dup_rows=18680 dup_rows_remote=14096 dedup_ratio=0.285034 router_tokens_per_expert=[0, 0, 0, 2292, 0, 0, 0, 0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 87, 0, 0, 4096, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4094, 0, 36, 0, 0, 0, 0, 1769, 0, 0] payload_rows_per_dst=[8192, 1924, 8192, 8192, 870, 7854, 8088, 3544] dup_rows_per_dst=[4584, 0, 6222, 7794, 0, 0, 0, 80] combine_rows_per_src=[8192, 1924, 8192, 8192, 870, 7854, 8088, 3544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49112 payload_rows_remote=40924 dup_rows=16424 dup_rows_remote=14214 dedup_ratio=0.250610 router_tokens_per_expert=[0, 0, 0, 0, 0, 4066, 21, 0, 0, 0, 0, 0, 0, 0, 0, 510, 31, 1059, 0, 19, 0, 0, 0, 0, 0, 4040, 0, 0, 50, 0, 0, 0, 0, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 2, 0, 72, 1961, 0, 59, 0, 0, 0, 4, 0, 36, 0, 0, 1, 0, 0, 0, 0, 0, 3487, 0, 0, 10, 163, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 1, 0, 232, 0, 0, 0, 0, 4080, 0, 11, 0, 1, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 3441, 0, 0, 75, 0] payload_rows_per_dst=[8170, 8188, 8192, 8190, 8162, 348, 682, 7180] dup_rows_per_dst=[1024, 2210, 58, 4204, 7054, 0, 26, 1848] combine_rows_per_src=[8170, 8188, 8192, 8190, 8162, 348, 682, 7180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55768 payload_rows_remote=48804 dup_rows=9768 dup_rows_remote=9746 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 0, 937, 0, 0, 0, 0, 0, 4051, 0, 4053, 0, 0, 3, 0, 0, 0, 398, 0, 0, 13, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 6, 0, 0, 0, 0, 0, 56, 4011, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 11, 0, 1, 160, 0, 9, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 12, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 3903, 0, 0] payload_rows_per_dst=[6830, 2640, 8174, 6686, 8180, 6964, 8112, 8182] dup_rows_per_dst=[1276, 290, 44, 20, 276, 22, 34, 7806] combine_rows_per_src=[6830, 2640, 8174, 6686, 8180, 6964, 8112, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56630 payload_rows_remote=48466 dup_rows=8906 dup_rows_remote=6326 dedup_ratio=0.135895 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 0, 0, 0, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2370, 0, 0, 0, 3, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1375, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8152, 8164, 6844, 2100, 6844, 8142] dup_rows_per_dst=[116, 74, 38, 2580, 3450, 0, 2648, 0] combine_rows_per_src=[8192, 8192, 8152, 8164, 6844, 2100, 6844, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=41124 payload_rows_remote=33270 dup_rows=24412 dup_rows_remote=24052 dedup_ratio=0.372498 router_tokens_per_expert=[0, 0, 0, 19, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 16, 0, 0, 2, 0, 0, 0, 0, 19, 0, 0, 0, 52, 0, 0, 3333, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 1902, 0, 97, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 945, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 33, 5, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 3751, 10, 0, 0, 0] payload_rows_per_dst=[40, 4282, 6670, 3804, 2094, 8188, 8192, 7854] dup_rows_per_dst=[0, 54, 142, 218, 10, 15676, 7952, 360] combine_rows_per_src=[40, 4282, 6670, 3804, 2094, 8188, 8192, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62338 payload_rows_remote=55136 dup_rows=3198 dup_rows_remote=3070 dedup_ratio=0.048798 router_tokens_per_expert=[0, 0, 0, 17, 0, 0, 183, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1560, 0, 0, 3592, 0, 0, 6, 0, 8, 0, 0, 4096, 0, 0, 1, 53, 5, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7990, 7202, 7568, 8192, 6810, 8192, 8192] dup_rows_per_dst=[400, 2502, 128, 142, 8, 0, 0, 18] combine_rows_per_src=[8192, 7990, 7202, 7568, 8192, 6810, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158367744 dispatch_meta_bytes=211040 combine_payload_bytes=158367744 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158578784 ingress_bytes=233865216 link_bytes=233865216 token_bytes=316946528 weight_movement_bytes=75497472 total_nvlink_bytes=392444000 weight_to_token_ratio=0.238203 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.240781431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62964 payload_rows_remote=54772 dup_rows=2572 dup_rows_remote=2570 dedup_ratio=0.039246 router_tokens_per_expert=[0, 0, 0, 1188, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0] payload_rows_per_dst=[8192, 5664, 8192, 8192, 8192, 8148, 8192, 8192] dup_rows_per_dst=[2376, 0, 34, 142, 2, 0, 0, 18] combine_rows_per_src=[8192, 5664, 8192, 8192, 8192, 8148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=167624704 dispatch_meta_bytes=220552 combine_payload_bytes=167624704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=167845256 ingress_bytes=205373440 link_bytes=205373440 token_bytes=335469960 weight_movement_bytes=37748736 total_nvlink_bytes=373218696 weight_to_token_ratio=0.112525 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.967787252e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52806 payload_rows_remote=44614 dup_rows=12730 dup_rows_remote=6146 dedup_ratio=0.194244 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 40, 738, 0, 0, 96, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 21, 0, 0, 0, 3237, 0, 0, 0, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 17, 0, 0, 0, 2401, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3534, 0, 366, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8066, 1744, 8188, 6758, 8192, 3652, 8192, 8014] dup_rows_per_dst=[0, 18, 6, 1444, 78, 12, 6584, 4588] combine_rows_per_src=[8066, 1744, 8188, 6758, 8192, 3652, 8192, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=199901184 dispatch_meta_bytes=234200 combine_payload_bytes=199901184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200135384 ingress_bytes=218775552 link_bytes=218775552 token_bytes=400036568 weight_movement_bytes=18874368 total_nvlink_bytes=418910936 weight_to_token_ratio=0.047182 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.096199695e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=198516736 dispatch_meta_bytes=219168 combine_payload_bytes=198516736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198735904 ingress_bytes=217391104 link_bytes=217391104 token_bytes=397252640 weight_movement_bytes=18874368 total_nvlink_bytes=416127008 weight_to_token_ratio=0.047512 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.082934595e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=136273920 dispatch_meta_bytes=229288 combine_payload_bytes=136273920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=136503208 ingress_bytes=155148288 link_bytes=155148288 token_bytes=272777128 weight_movement_bytes=18874368 total_nvlink_bytes=291651496 weight_to_token_ratio=0.069193 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.486554558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=232824 combine_payload_bytes=225837056 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226069880 ingress_bytes=282460160 link_bytes=282460160 token_bytes=451906936 weight_movement_bytes=56623104 total_nvlink_bytes=508530040 weight_to_token_ratio=0.125298 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.706394274e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224346112 dispatch_meta_bytes=229368 combine_payload_bytes=224346112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224575480 ingress_bytes=243220480 link_bytes=243220480 token_bytes=448921592 weight_movement_bytes=18874368 total_nvlink_bytes=467795960 weight_to_token_ratio=0.042044 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330418967e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=182738944 dispatch_meta_bytes=203040 combine_payload_bytes=182738944 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=182941984 ingress_bytes=239362048 link_bytes=239362048 token_bytes=365680928 weight_movement_bytes=56623104 total_nvlink_bytes=422304032 weight_to_token_ratio=0.154843 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293449370e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53930 payload_rows_remote=45762 dup_rows=11606 dup_rows_remote=11448 dedup_ratio=0.177094 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3857, 0, 0, 1701, 0, 0, 0, 0, 5, 0, 0, 0, 3191, 0, 0, 0, 2779, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 269, 0, 0, 0, 17, 0, 4037, 0, 0, 212, 0, 0, 0, 0, 1, 1732, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 2, 2442, 0, 0, 0, 0, 31, 0, 0, 0, 0] payload_rows_per_dst=[8168, 7772, 8186, 8086, 7870, 752, 8190, 4906] dup_rows_per_dst=[158, 4168, 3252, 0, 0, 20, 3964, 44] combine_rows_per_src=[8168, 7772, 8186, 8086, 7870, 752, 8190, 4906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48236 payload_rows_remote=40600 dup_rows=17300 dup_rows_remote=15786 dedup_ratio=0.263977 router_tokens_per_expert=[4, 4, 0, 22, 0, 9, 0, 0, 0, 2704, 2, 4057, 6, 0, 0, 0, 0, 0, 15, 0, 0, 87, 3542, 0, 0, 0, 0, 0, 0, 911, 20, 0, 3870, 0, 14, 0, 0, 91, 0, 0, 3889, 0, 0, 671, 1181, 0, 0, 0, 0, 0, 0, 1, 0, 534, 3705, 2, 290, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 6, 3656, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 3, 11, 0, 366, 0, 7, 0, 0, 50, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2870, 0, 6, 0, 0, 0, 0, 38, 0, 0] payload_rows_per_dst=[5444, 7636, 8192, 8044, 8126, 846, 8148, 1800] dup_rows_per_dst=[58, 1514, 3462, 5052, 7166, 30, 2, 16] combine_rows_per_src=[5444, 7636, 8192, 8044, 8126, 846, 8148, 1800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47992 payload_rows_remote=39982 dup_rows=17544 dup_rows_remote=16598 dedup_ratio=0.267700 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 349, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 3494, 0, 0, 3143, 0, 0, 0, 0, 182, 0, 0, 3, 1202, 0, 0, 3, 1532, 0, 0, 0, 0, 0, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 7, 160, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7, 0, 466, 0, 0, 0, 0, 0, 4001, 0, 0, 10, 0, 0, 0, 0, 0, 3380, 0, 0, 7, 5, 0, 0, 0, 0, 1229, 0, 3346, 0, 0, 0, 120, 139, 15, 0, 0, 2, 0, 0, 0, 271, 0] payload_rows_per_dst=[8192, 3538, 8192, 2784, 8190, 8010, 8180, 906] dup_rows_per_dst=[2956, 252, 5084, 0, 344, 946, 7774, 188] combine_rows_per_src=[8192, 3538, 8192, 2784, 8190, 8010, 8180, 906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58150 payload_rows_remote=50124 dup_rows=7386 dup_rows_remote=4678 dedup_ratio=0.112701 router_tokens_per_expert=[0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4000, 0, 0, 0, 0, 2, 0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2957, 0, 0, 2410, 0, 0, 0, 0, 3, 0, 0, 954, 262, 0, 0, 0, 1733, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1342, 0, 0, 0, 0, 0, 3510, 1527, 0, 316, 0, 0, 0, 0, 0, 2635, 0, 0, 0, 4, 18, 0, 0, 0, 0, 0, 690, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8028, 6456, 8026, 7818, 8192, 5250, 6204, 8176] dup_rows_per_dst=[80, 1334, 2708, 1646, 0, 496, 1122, 0] combine_rows_per_src=[8028, 6456, 8026, 7818, 8192, 5250, 6204, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63994 payload_rows_remote=57300 dup_rows=1542 dup_rows_remote=1542 dedup_ratio=0.023529 router_tokens_per_expert=[0, 0, 0, 651, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 48, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 6694, 8188, 8154, 8192, 8192] dup_rows_per_dst=[1302, 132, 12, 0, 0, 0, 0, 96] combine_rows_per_src=[8190, 8192, 8192, 6694, 8188, 8154, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52088 payload_rows_remote=43982 dup_rows=13448 dup_rows_remote=12492 dedup_ratio=0.205200 router_tokens_per_expert=[0, 0, 0, 2055, 1, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1896, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4046, 0, 0, 85, 0, 364, 0, 0, 4079, 0, 0, 0, 67, 306, 0, 0, 0, 0, 0, 1, 0, 24, 877, 0, 25, 0, 0, 0, 2169, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4044, 0, 7, 0, 0, 0, 0, 434, 0, 0] payload_rows_per_dst=[4122, 8120, 1352, 5892, 8182, 8122, 8192, 8106] dup_rows_per_dst=[0, 3654, 292, 300, 6, 8048, 192, 956] combine_rows_per_src=[4122, 8120, 1352, 5892, 8182, 8122, 8192, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=187441152 dispatch_meta_bytes=228840 combine_payload_bytes=187441152 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=187669992 ingress_bytes=225189888 link_bytes=225189888 token_bytes=375111144 weight_movement_bytes=37748736 total_nvlink_bytes=412859880 weight_to_token_ratio=0.100633 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.157658706e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60928 payload_rows_remote=52736 dup_rows=4608 dup_rows_remote=4608 dedup_ratio=0.070312 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 88, 3311, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 639, 0, 0, 0, 19, 0, 4010, 0, 0, 14, 0, 0, 0, 0, 0, 3186, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8150, 8186, 6758, 8192, 8070, 7674, 5714] dup_rows_per_dst=[20, 0, 158, 42, 0, 1288, 3100, 0] combine_rows_per_src=[8184, 8150, 8186, 6758, 8192, 8070, 7674, 5714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=166297600 dispatch_meta_bytes=225544 combine_payload_bytes=166297600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=166523144 ingress_bytes=185171968 link_bytes=185171968 token_bytes=332820744 weight_movement_bytes=18874368 total_nvlink_bytes=351695112 weight_to_token_ratio=0.056710 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.774226687e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51186 payload_rows_remote=43416 dup_rows=14350 dup_rows_remote=9272 dedup_ratio=0.218964 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 174, 0, 0, 0, 0, 3, 1156, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 4096, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4095, 0, 0, 39, 0, 1200, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 0, 36, 0, 4, 0, 0, 0, 0, 1506, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3963, 0, 446, 1700, 0, 0, 0, 0, 0, 25] payload_rows_per_dst=[6592, 2704, 1378, 8190, 8192, 8192, 7770, 8168] dup_rows_per_dst=[164, 772, 16, 956, 3260, 4, 5078, 4100] combine_rows_per_src=[6592, 2704, 1378, 8190, 8192, 8192, 7770, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=163766272 dispatch_meta_bytes=226320 combine_payload_bytes=163766272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=163992592 ingress_bytes=201515008 link_bytes=201515008 token_bytes=327758864 weight_movement_bytes=37748736 total_nvlink_bytes=365507600 weight_to_token_ratio=0.115172 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.930817655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=205307904 dispatch_meta_bytes=219208 combine_payload_bytes=205307904 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=205527112 ingress_bytes=261931008 link_bytes=261931008 token_bytes=410835016 weight_movement_bytes=56623104 total_nvlink_bytes=467458120 weight_to_token_ratio=0.137824 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.509694041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=235368 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234936168 ingress_bytes=253575168 link_bytes=253575168 token_bytes=469636968 weight_movement_bytes=18874368 total_nvlink_bytes=488511336 weight_to_token_ratio=0.040189 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429632493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=180150272 dispatch_meta_bytes=225896 combine_payload_bytes=180150272 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=180376168 ingress_bytes=236773376 link_bytes=236773376 token_bytes=360526440 weight_movement_bytes=56623104 total_nvlink_bytes=417149544 weight_to_token_ratio=0.157057 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.268645989e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=216006656 dispatch_meta_bytes=229376 combine_payload_bytes=216006656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216236032 ingress_bytes=234881024 link_bytes=234881024 token_bytes=432242688 weight_movement_bytes=18874368 total_nvlink_bytes=451117056 weight_to_token_ratio=0.043666 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250514403e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=177831936 dispatch_meta_bytes=210752 combine_payload_bytes=177831936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=178042688 ingress_bytes=196706304 link_bytes=196706304 token_bytes=355874624 weight_movement_bytes=18874368 total_nvlink_bytes=374748992 weight_to_token_ratio=0.053037 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.884743019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48890 payload_rows_remote=40780 dup_rows=16646 dup_rows_remote=14060 dedup_ratio=0.253998 router_tokens_per_expert=[0, 97, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 38, 117, 3988, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 131, 1054, 1389, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0] payload_rows_per_dst=[8110, 8192, 8010, 8192, 8190, 0, 4, 8192] dup_rows_per_dst=[2586, 614, 2560, 8762, 2108, 0, 0, 16] combine_rows_per_src=[8110, 8192, 8010, 8192, 8190, 0, 4, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54696 payload_rows_remote=46756 dup_rows=10840 dup_rows_remote=8802 dedup_ratio=0.165405 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1811, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 102, 3929, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 10, 3949, 0, 0, 0, 3921, 30, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 0, 0, 2348, 0, 0, 0, 84, 0, 0, 0, 0, 28, 0, 0, 0, 44, 678, 2, 0, 0, 0, 36, 3956, 0, 0, 0, 0, 4079, 2, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8192, 7940, 8122, 6060, 7994, 48, 8180, 8160] dup_rows_per_dst=[2372, 2038, 204, 4736, 0, 4, 1404, 82] combine_rows_per_src=[8192, 7940, 8122, 6060, 7994, 48, 8180, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62138 payload_rows_remote=53948 dup_rows=3398 dup_rows_remote=930 dedup_ratio=0.051849 router_tokens_per_expert=[0, 2, 0, 0, 0, 47, 5, 6, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 61, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 93, 0, 0, 0, 0, 4067, 29, 0, 0, 0, 0, 119, 2944, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 786, 0, 99, 69, 0, 368, 8, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4080, 0, 0, 1, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[7488, 8156, 8190, 6266, 7522, 8190, 8152, 8174] dup_rows_per_dst=[102, 28, 198, 598, 0, 2468, 2, 2] combine_rows_per_src=[7488, 8156, 8190, 6266, 7522, 8190, 8152, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54164 payload_rows_remote=49670 dup_rows=11372 dup_rows_remote=11372 dedup_ratio=0.173523 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 241, 2005, 0, 0, 0, 0, 0, 1310, 2655, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 252, 3625, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6226, 8192, 4494, 3100, 7628, 8164, 8192, 8168] dup_rows_per_dst=[4040, 340, 0, 1684, 126, 12, 5170, 0] combine_rows_per_src=[6226, 8192, 4494, 3100, 7628, 8164, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=167034880 dispatch_meta_bytes=219360 combine_payload_bytes=167034880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=167254240 ingress_bytes=185909248 link_bytes=185909248 token_bytes=334289120 weight_movement_bytes=18874368 total_nvlink_bytes=353163488 weight_to_token_ratio=0.056461 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.781290941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58580 payload_rows_remote=50594 dup_rows=6956 dup_rows_remote=3066 dedup_ratio=0.106140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3598, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 7, 0, 0, 13, 4046, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 1254, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 196, 0, 0, 0, 5, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7196, 8086, 8192, 7986, 8192, 8174, 3260, 7494] dup_rows_per_dst=[10, 8, 2, 3890, 0, 2538, 498, 10] combine_rows_per_src=[7196, 8086, 8192, 7986, 8192, 8174, 3260, 7494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42418 payload_rows_remote=34236 dup_rows=23118 dup_rows_remote=8140 dedup_ratio=0.352753 router_tokens_per_expert=[0, 0, 0, 0, 280, 0, 3, 0, 0, 0, 0, 430, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 4050, 0, 6, 3, 0, 0, 0, 0, 21, 0, 0, 182, 4072, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 1575, 0, 0, 0, 0, 0, 2344, 0, 0, 1, 1, 0, 0, 0, 5, 1, 0, 0, 0, 0, 6, 1, 0, 2, 83, 0, 4053, 0, 0, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[2788, 7724, 106, 8172, 8192, 7080, 174, 8182] dup_rows_per_dst=[1278, 0, 0, 378, 146, 6312, 26, 14978] combine_rows_per_src=[2788, 7724, 106, 8172, 8192, 7080, 174, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61328 payload_rows_remote=53158 dup_rows=4208 dup_rows_remote=3794 dedup_ratio=0.064209 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 55, 3775, 0, 0, 0, 0, 0, 0, 0, 1964, 0, 0, 0, 254, 4038, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7556, 8182, 8186, 4660, 8170, 8192, 8192, 8190] dup_rows_per_dst=[3478, 138, 0, 118, 414, 6, 54, 0] combine_rows_per_src=[7556, 8182, 8186, 4660, 8170, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=191512576 dispatch_meta_bytes=222232 combine_payload_bytes=191512576 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=191734808 ingress_bytes=229261312 link_bytes=229261312 token_bytes=383247384 weight_movement_bytes=37748736 total_nvlink_bytes=420996120 weight_to_token_ratio=0.098497 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.196669088e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220971008 dispatch_meta_bytes=219512 combine_payload_bytes=220971008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221190520 ingress_bytes=239845376 link_bytes=239845376 token_bytes=442161528 weight_movement_bytes=18874368 total_nvlink_bytes=461035896 weight_to_token_ratio=0.042687 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.298080382e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=203448320 dispatch_meta_bytes=244168 combine_payload_bytes=203448320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203692488 ingress_bytes=222322688 link_bytes=222322688 token_bytes=407140808 weight_movement_bytes=18874368 total_nvlink_bytes=426015176 weight_to_token_ratio=0.046358 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.130186607e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56234 payload_rows_remote=52064 dup_rows=9302 dup_rows_remote=8862 dedup_ratio=0.141937 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 237, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 945, 8, 0, 4071, 1360, 0, 853, 0, 7, 1527, 0, 0, 0, 0, 0, 0, 8, 8, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 1, 0, 956, 0, 0, 0, 0, 0, 0, 28, 0] payload_rows_per_dst=[8192, 3420, 8192, 7714, 8170, 8190, 4170, 8186] dup_rows_per_dst=[0, 396, 36, 3422, 3056, 32, 440, 1920] combine_rows_per_src=[8192, 3420, 8192, 7714, 8170, 8190, 4170, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=207233024 dispatch_meta_bytes=214640 combine_payload_bytes=207233024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=207447664 ingress_bytes=263856128 link_bytes=263856128 token_bytes=414680688 weight_movement_bytes=56623104 total_nvlink_bytes=471303792 weight_to_token_ratio=0.136546 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528139593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=140230656 dispatch_meta_bytes=169504 combine_payload_bytes=140230656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=140400160 ingress_bytes=177979392 link_bytes=177979392 token_bytes=280630816 weight_movement_bytes=37748736 total_nvlink_bytes=318379552 weight_to_token_ratio=0.134514 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.705310963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=217735168 dispatch_meta_bytes=227808 combine_payload_bytes=217735168 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=217962976 ingress_bytes=293232640 link_bytes=293232640 token_bytes=435698144 weight_movement_bytes=75497472 total_nvlink_bytes=511195616 weight_to_token_ratio=0.173279 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.809610877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213254144 dispatch_meta_bytes=243704 combine_payload_bytes=213254144 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=213497848 ingress_bytes=251002880 link_bytes=251002880 token_bytes=426751992 weight_movement_bytes=37748736 total_nvlink_bytes=464500728 weight_to_token_ratio=0.088456 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404986095e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58596 payload_rows_remote=50614 dup_rows=6940 dup_rows_remote=6496 dedup_ratio=0.105896 router_tokens_per_expert=[0, 0, 1, 0, 0, 13, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 3986, 2617, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 58, 0, 0, 3774, 0, 0, 1096, 318, 0, 3752, 0, 0, 0, 0, 0] payload_rows_per_dst=[7982, 7150, 7552, 8192, 8192, 6236, 5134, 8158] dup_rows_per_dst=[444, 770, 244, 832, 2360, 0, 0, 2290] combine_rows_per_src=[7982, 7150, 7552, 8192, 8192, 6236, 5134, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56390 payload_rows_remote=48346 dup_rows=9146 dup_rows_remote=9024 dedup_ratio=0.139557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 51, 0, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3179, 13, 0, 2, 0, 0, 0, 0, 0, 0, 1, 103, 0, 2, 0, 0, 0, 0, 4019, 0, 0, 0, 1, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 1268, 1, 0, 4004, 0, 3212, 0, 0, 3, 3, 0, 0, 3965, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 1, 2644, 0, 0, 0, 0, 0, 0, 1, 0, 192, 4060, 0, 0, 0, 0, 6, 433, 0, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1690, 8044, 8186, 8164, 7690, 7938, 6892, 7786] dup_rows_per_dst=[22, 122, 1296, 4478, 1088, 1742, 398, 0] combine_rows_per_src=[1690, 8044, 8186, 8164, 7690, 7938, 6892, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58926 payload_rows_remote=50734 dup_rows=6610 dup_rows_remote=5736 dedup_ratio=0.100861 router_tokens_per_expert=[0, 0, 0, 233, 0, 2127, 0, 0, 36, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 3690, 3566, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 3759, 0, 64, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 4005, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 493, 0, 0, 0, 18, 92, 1, 0, 54, 0, 0, 1, 0, 0, 3838, 0, 0, 0, 0, 0, 68, 0, 0, 4026, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7318, 8192, 7840, 8042, 8192, 7870, 3284] dup_rows_per_dst=[4178, 658, 686, 72, 4, 874, 138, 0] combine_rows_per_src=[8188, 7318, 8192, 7840, 8042, 8192, 7870, 3284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56916 payload_rows_remote=48724 dup_rows=8620 dup_rows_remote=8146 dedup_ratio=0.131531 router_tokens_per_expert=[0, 0, 0, 0, 0, 396, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 0, 0, 4042, 13, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 56, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 450, 0, 0, 4096, 0, 0, 3849, 0, 0, 3200, 0, 0, 0, 0, 0] payload_rows_per_dst=[8102, 7040, 8192, 6400, 2996, 8192, 7834, 8160] dup_rows_per_dst=[796, 12, 474, 0, 0, 500, 0, 6838] combine_rows_per_src=[8102, 7040, 8192, 6400, 2996, 8192, 7834, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=207314944 dispatch_meta_bytes=228440 combine_payload_bytes=207314944 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=207543384 ingress_bytes=245063680 link_bytes=245063680 token_bytes=414858328 weight_movement_bytes=37748736 total_nvlink_bytes=452607064 weight_to_token_ratio=0.090992 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348079603e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57010 payload_rows_remote=56922 dup_rows=8526 dup_rows_remote=8526 dedup_ratio=0.130096 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 1, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 88, 7772, 8192, 8192, 8192] dup_rows_per_dst=[7786, 376, 0, 0, 0, 314, 50, 0] combine_rows_per_src=[8192, 8192, 8190, 88, 7772, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52428 payload_rows_remote=44248 dup_rows=13108 dup_rows_remote=12664 dedup_ratio=0.200012 router_tokens_per_expert=[0, 22, 0, 0, 1, 0, 0, 0, 73, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4060, 920, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 1, 305, 3301, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1639, 0, 0, 127, 9, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 310, 8, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[212, 4982, 8032, 8182, 6460, 8192, 8188, 8180] dup_rows_per_dst=[2, 1846, 2, 3070, 342, 3550, 3852, 444] combine_rows_per_src=[212, 4982, 8032, 8182, 6460, 8192, 8188, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63470 payload_rows_remote=55292 dup_rows=2066 dup_rows_remote=2038 dedup_ratio=0.031525 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 3974, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3625, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 40, 0, 0, 4096, 0, 0, 137, 9, 0, 3847, 0, 0, 0, 0, 0] payload_rows_per_dst=[7562, 7948, 8192, 7250, 8178, 8192, 8192, 7956] dup_rows_per_dst=[2, 0, 1920, 6, 28, 0, 0, 110] combine_rows_per_src=[7562, 7948, 8192, 7250, 8178, 8192, 8192, 7956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=198025216 dispatch_meta_bytes=229480 combine_payload_bytes=198025216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=198254696 ingress_bytes=235773952 link_bytes=235773952 token_bytes=396279912 weight_movement_bytes=37748736 total_nvlink_bytes=434028648 weight_to_token_ratio=0.095258 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259070000e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=207806464 dispatch_meta_bytes=225880 combine_payload_bytes=207806464 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208032344 ingress_bytes=245555200 link_bytes=245555200 token_bytes=415838808 weight_movement_bytes=37748736 total_nvlink_bytes=453587544 weight_to_token_ratio=0.090777 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352789106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=199573504 dispatch_meta_bytes=227480 combine_payload_bytes=199573504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199800984 ingress_bytes=218447872 link_bytes=218447872 token_bytes=399374488 weight_movement_bytes=18874368 total_nvlink_bytes=418248856 weight_to_token_ratio=0.047260 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.093060026e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52880 payload_rows_remote=48928 dup_rows=12656 dup_rows_remote=10790 dedup_ratio=0.193115 router_tokens_per_expert=[0, 26, 0, 0, 0, 419, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 398, 0, 0, 999, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 20, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 3928, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[3106, 8074, 6466, 7914, 8006, 7170, 3952, 8192] dup_rows_per_dst=[632, 96, 2014, 18, 40, 4210, 1866, 3780] combine_rows_per_src=[3106, 8074, 6466, 7914, 8006, 7170, 3952, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233152512 dispatch_meta_bytes=261792 combine_payload_bytes=233152512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233414304 ingress_bytes=270901248 link_bytes=270901248 token_bytes=466566816 weight_movement_bytes=37748736 total_nvlink_bytes=504315552 weight_to_token_ratio=0.080907 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595642467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=181239808 dispatch_meta_bytes=227648 combine_payload_bytes=181239808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=181467456 ingress_bytes=200114176 link_bytes=200114176 token_bytes=362707264 weight_movement_bytes=18874368 total_nvlink_bytes=381581632 weight_to_token_ratio=0.052037 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.917395572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=226476032 dispatch_meta_bytes=229320 combine_payload_bytes=226476032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226705352 ingress_bytes=245350400 link_bytes=245350400 token_bytes=453181384 weight_movement_bytes=18874368 total_nvlink_bytes=472055752 weight_to_token_ratio=0.041649 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.350826813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=200409088 dispatch_meta_bytes=238872 combine_payload_bytes=200409088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=200647960 ingress_bytes=257032192 link_bytes=257032192 token_bytes=401057048 weight_movement_bytes=56623104 total_nvlink_bytes=457680152 weight_to_token_ratio=0.141185 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.462755996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47626 payload_rows_remote=39434 dup_rows=17910 dup_rows_remote=13628 dedup_ratio=0.273285 router_tokens_per_expert=[0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 3059, 0, 95, 2941, 0, 2245, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 27, 0, 0, 0, 0, 0, 0, 0, 878, 0, 0, 0, 0, 0, 2993, 0, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 60, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 8192, 7118, 8192, 7882, 7910, 4, 136] dup_rows_per_dst=[4282, 1602, 5300, 1822, 4904, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7118, 8192, 7882, 7910, 4, 136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54464 payload_rows_remote=46304 dup_rows=11072 dup_rows_remote=9320 dedup_ratio=0.168945 router_tokens_per_expert=[0, 18, 98, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 12, 0, 4, 0, 0, 0, 0, 0, 989, 0, 0, 1446, 3951, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 3248, 0, 0, 0, 22, 0, 0, 0, 0, 0, 26, 0, 0, 7, 1847, 0, 0, 0, 0, 0, 3992, 5, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 68, 0, 0, 0, 0, 4, 0, 4053, 1, 0, 234, 0, 0, 0, 0, 16, 25, 0, 0, 0, 715, 0, 3858, 0, 1, 0, 0, 0, 155, 0, 0, 1959, 1, 0, 0, 0, 0, 2579, 0, 0, 0, 1, 0] payload_rows_per_dst=[3120, 8160, 5078, 8140, 8180, 8140, 8184, 5462] dup_rows_per_dst=[32, 1752, 1884, 2222, 3522, 604, 1046, 10] combine_rows_per_src=[3120, 8160, 5078, 8140, 8180, 8140, 8184, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57070 payload_rows_remote=48896 dup_rows=8466 dup_rows_remote=8462 dedup_ratio=0.129181 router_tokens_per_expert=[0, 63, 0, 36, 0, 0, 60, 0, 0, 1, 0, 0, 1, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1589, 3984, 0, 0, 3981, 0, 2443, 3, 0, 4032, 0, 0, 0, 4, 23, 0, 0, 4077, 0, 5, 0, 0, 0, 0, 0, 0, 56, 0, 50, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 2, 0, 24, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 18, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8064, 8056, 222, 8072, 8174, 8104, 8186] dup_rows_per_dst=[268, 2, 68, 0, 48, 4, 4, 8072] combine_rows_per_src=[8192, 8064, 8056, 222, 8072, 8174, 8104, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49058 payload_rows_remote=47238 dup_rows=16478 dup_rows_remote=16468 dedup_ratio=0.251434 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 546, 0, 17, 0, 0, 0, 10, 0, 0, 693, 3649, 0, 0, 3969, 0, 222, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 3111, 0, 27, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 3496, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8190, 1820, 8192, 8180, 178, 7304, 7002] dup_rows_per_dst=[2, 1152, 10, 7918, 6040, 0, 1132, 224] combine_rows_per_src=[8192, 8190, 1820, 8192, 8180, 178, 7304, 7002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=161521664 dispatch_meta_bytes=212248 combine_payload_bytes=161521664 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=161733912 ingress_bytes=199270400 link_bytes=199270400 token_bytes=323255576 weight_movement_bytes=37748736 total_nvlink_bytes=361004312 weight_to_token_ratio=0.116777 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.909310926e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63350 payload_rows_remote=55180 dup_rows=2186 dup_rows_remote=2186 dedup_ratio=0.033356 router_tokens_per_expert=[0, 0, 0, 3, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4057, 3881, 5, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 4063, 0, 0, 0, 0, 0, 440, 0, 0, 4096, 0, 0, 0, 0, 0, 2937, 0, 0, 0, 0, 0] payload_rows_per_dst=[8116, 7798, 8192, 8170, 8188, 8180, 8192, 6514] dup_rows_per_dst=[38, 250, 0, 0, 96, 0, 1562, 240] combine_rows_per_src=[8116, 7798, 8192, 8170, 8188, 8180, 8192, 6514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56446 payload_rows_remote=49096 dup_rows=9090 dup_rows_remote=4444 dedup_ratio=0.138702 router_tokens_per_expert=[0, 0, 2, 0, 0, 197, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 8, 0, 4078, 2244, 2, 0, 49, 0, 34, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 49, 3691, 0, 201, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 150, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2, 1041, 0, 2, 4053, 0, 0, 3, 0, 0, 16, 0, 47, 4094, 0, 0, 0, 2, 0, 3507, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 5398, 6038, 7780, 7980, 7942, 5772, 7350] dup_rows_per_dst=[770, 228, 688, 482, 0, 296, 1980, 4646] combine_rows_per_src=[8186, 5398, 6038, 7780, 7980, 7942, 5772, 7350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61630 payload_rows_remote=53448 dup_rows=3906 dup_rows_remote=3540 dedup_ratio=0.059601 router_tokens_per_expert=[0, 0, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4057, 0, 0, 4089, 0, 4092, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8184, 6440, 8182, 7834, 8116, 6508] dup_rows_per_dst=[1876, 38, 0, 20, 366, 0, 888, 718] combine_rows_per_src=[8192, 8174, 8184, 6440, 8182, 7834, 8116, 6508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=189661184 dispatch_meta_bytes=222496 combine_payload_bytes=189661184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=189883680 ingress_bytes=246284288 link_bytes=246284288 token_bytes=379544864 weight_movement_bytes=56623104 total_nvlink_bytes=436167968 weight_to_token_ratio=0.149187 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.359774868e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200278016 dispatch_meta_bytes=229432 combine_payload_bytes=200278016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200507448 ingress_bytes=219152384 link_bytes=219152384 token_bytes=400785464 weight_movement_bytes=18874368 total_nvlink_bytes=419659832 weight_to_token_ratio=0.047093 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.099810314e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=193486848 dispatch_meta_bytes=254824 combine_payload_bytes=193486848 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=193741672 ingress_bytes=306733056 link_bytes=306733056 token_bytes=387228520 weight_movement_bytes=113246208 total_nvlink_bytes=500474728 weight_to_token_ratio=0.292453 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=18816 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.938965221e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53506 payload_rows_remote=45320 dup_rows=12030 dup_rows_remote=11902 dedup_ratio=0.183563 router_tokens_per_expert=[0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 20, 15, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 933, 0, 0, 0, 28, 0, 0, 0, 52, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2355, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 0, 4, 0, 1, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 65, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0] payload_rows_per_dst=[4420, 8192, 8192, 4710, 4014, 7612, 8186, 8180] dup_rows_per_dst=[70, 302, 2040, 0, 0, 9490, 128, 0] combine_rows_per_src=[4420, 8192, 8192, 4710, 4014, 7612, 8186, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=226017280 dispatch_meta_bytes=229464 combine_payload_bytes=226017280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226246744 ingress_bytes=244891648 link_bytes=244891648 token_bytes=452264024 weight_movement_bytes=18874368 total_nvlink_bytes=471138392 weight_to_token_ratio=0.041733 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.346431277e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=201097216 dispatch_meta_bytes=214160 combine_payload_bytes=201097216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=201311376 ingress_bytes=238845952 link_bytes=238845952 token_bytes=402408592 weight_movement_bytes=37748736 total_nvlink_bytes=440157328 weight_to_token_ratio=0.093807 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.288504393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=218923008 dispatch_meta_bytes=227952 combine_payload_bytes=218923008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219150960 ingress_bytes=237797376 link_bytes=237797376 token_bytes=438073968 weight_movement_bytes=18874368 total_nvlink_bytes=456948336 weight_to_token_ratio=0.043085 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.278457453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=185630720 dispatch_meta_bytes=228888 combine_payload_bytes=185630720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=185859608 ingress_bytes=242253824 link_bytes=242253824 token_bytes=371490328 weight_movement_bytes=56623104 total_nvlink_bytes=428113432 weight_to_token_ratio=0.152421 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=16128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.321156945e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56000 payload_rows_remote=47808 dup_rows=9536 dup_rows_remote=7260 dedup_ratio=0.145508 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 427, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 238, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3644, 0, 71, 0, 0, 4096, 6, 0, 0] payload_rows_per_dst=[8192, 8164, 8192, 7610, 8192, 2, 8192, 7456] dup_rows_per_dst=[2276, 1072, 254, 5884, 0, 0, 2, 48] combine_rows_per_src=[8192, 8164, 8192, 7610, 8192, 2, 8192, 7456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47380 payload_rows_remote=39354 dup_rows=18156 dup_rows_remote=18050 dedup_ratio=0.277039 router_tokens_per_expert=[0, 0, 4074, 12, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 3329, 0, 0, 0, 39, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 4078, 36, 482, 513, 0, 4055, 0, 0, 3985, 0, 0, 0, 6, 0, 0, 0, 3645, 222, 0, 13, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 10, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3384, 0, 110, 0, 0, 0, 0, 0, 111, 0, 0, 0, 9, 0, 21, 149, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 5, 0, 0, 29, 0, 0, 31, 0] payload_rows_per_dst=[8176, 8026, 8160, 7774, 1250, 7196, 6666, 132] dup_rows_per_dst=[6674, 106, 9868, 80, 42, 32, 1350, 4] combine_rows_per_src=[8176, 8026, 8160, 7774, 1250, 7196, 6666, 132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60558 payload_rows_remote=52366 dup_rows=4978 dup_rows_remote=1750 dedup_ratio=0.075958 router_tokens_per_expert=[0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3787, 0, 4093, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8116, 8186, 8192, 7574, 8188, 8192, 3918] dup_rows_per_dst=[728, 0, 0, 3228, 0, 0, 1022, 0] combine_rows_per_src=[8192, 8116, 8186, 8192, 7574, 8188, 8192, 3918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=195821568 dispatch_meta_bytes=220272 combine_payload_bytes=195821568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196041840 ingress_bytes=214695936 link_bytes=214695936 token_bytes=391863408 weight_movement_bytes=18874368 total_nvlink_bytes=410737776 weight_to_token_ratio=0.048166 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.057110822e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49878 payload_rows_remote=41686 dup_rows=15658 dup_rows_remote=15654 dedup_ratio=0.238922 router_tokens_per_expert=[5, 241, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 3959, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 9, 0, 39, 297, 0, 0, 0, 0, 4096, 49, 0, 0] payload_rows_per_dst=[8160, 8168, 8192, 8192, 8192, 0, 8192, 782] dup_rows_per_dst=[28, 7748, 4, 492, 0, 0, 7314, 72] combine_rows_per_src=[8160, 8168, 8192, 8192, 8192, 0, 8192, 782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59878 payload_rows_remote=51710 dup_rows=5658 dup_rows_remote=2374 dedup_ratio=0.086334 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 592, 1, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3590, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3679, 17, 0, 0, 6, 0, 0, 484, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 729, 1450, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 3, 0, 0, 0, 1637, 0, 0, 5, 0, 0, 230, 0, 0, 1, 70, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 103, 0, 0, 0, 0, 4071, 3859, 0, 0] payload_rows_per_dst=[8192, 7188, 7780, 4404, 8192, 8168, 8192, 7762] dup_rows_per_dst=[960, 0, 596, 18, 4, 3284, 610, 186] combine_rows_per_src=[8192, 7188, 7780, 4404, 8192, 8168, 8192, 7762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46608 payload_rows_remote=38416 dup_rows=18928 dup_rows_remote=10750 dedup_ratio=0.288818 router_tokens_per_expert=[0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 451, 218, 2437, 0, 3094, 0, 4, 3885, 0, 0, 0, 0, 4, 0, 0, 4096, 265, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 0, 0, 0, 0, 2183, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4036, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[44, 7778, 1184, 8182, 4874, 8164, 8190, 8192] dup_rows_per_dst=[0, 0, 170, 458, 12, 6194, 3916, 8178] combine_rows_per_src=[44, 7778, 1184, 8182, 4874, 8164, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59054 payload_rows_remote=50862 dup_rows=6482 dup_rows_remote=6482 dedup_ratio=0.098907 router_tokens_per_expert=[0, 20, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 64, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 2218, 0, 0, 0, 0, 4096, 6, 0, 0] payload_rows_per_dst=[8190, 8080, 8184, 2982, 8192, 7878, 8192, 7356] dup_rows_per_dst=[312, 12, 26, 398, 0, 5708, 2, 24] combine_rows_per_src=[8190, 8080, 8184, 2982, 8192, 7878, 8192, 7356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=161193984 dispatch_meta_bytes=229616 combine_payload_bytes=161193984 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=161423600 ingress_bytes=198942720 link_bytes=198942720 token_bytes=322617584 weight_movement_bytes=37748736 total_nvlink_bytes=360366320 weight_to_token_ratio=0.117008 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.906171257e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53958 payload_rows_remote=45766 dup_rows=11578 dup_rows_remote=11578 dedup_ratio=0.176666 router_tokens_per_expert=[0, 0, 0, 3189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3515, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1073, 0, 0] payload_rows_per_dst=[6378, 8192, 5846, 818, 8148, 8192, 8192, 8192] dup_rows_per_dst=[0, 7086, 0, 0, 0, 2346, 0, 2146] combine_rows_per_src=[6378, 8192, 5846, 818, 8148, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=214491136 dispatch_meta_bytes=216464 combine_payload_bytes=214491136 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=214707600 ingress_bytes=252239872 link_bytes=252239872 token_bytes=429198736 weight_movement_bytes=37748736 total_nvlink_bytes=466947472 weight_to_token_ratio=0.087952 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416838344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=170745856 dispatch_meta_bytes=229360 combine_payload_bytes=170745856 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=170975216 ingress_bytes=208494592 link_bytes=208494592 token_bytes=341721072 weight_movement_bytes=37748736 total_nvlink_bytes=379469808 weight_to_token_ratio=0.110467 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.997692595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=211804160 dispatch_meta_bytes=216336 combine_payload_bytes=211804160 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=212020496 ingress_bytes=287301632 link_bytes=287301632 token_bytes=423824656 weight_movement_bytes=75497472 total_nvlink_bytes=499322128 weight_to_token_ratio=0.178134 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.752782877e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=196664 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157548600 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314900536 weight_movement_bytes=18874368 total_nvlink_bytes=333774904 weight_to_token_ratio=0.059938 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=208330752 dispatch_meta_bytes=229376 combine_payload_bytes=208330752 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208560128 ingress_bytes=246079488 link_bytes=246079488 token_bytes=416890880 weight_movement_bytes=37748736 total_nvlink_bytes=454639616 weight_to_token_ratio=0.090548 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357812575e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=187457536 dispatch_meta_bytes=229376 combine_payload_bytes=187457536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=187686912 ingress_bytes=225206272 link_bytes=225206272 token_bytes=375144448 weight_movement_bytes=37748736 total_nvlink_bytes=412893184 weight_to_token_ratio=0.100625 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.157815690e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55296 payload_rows_remote=47104 dup_rows=10240 dup_rows_remote=10234 dedup_ratio=0.156250 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2216, 0, 0, 1, 495, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 104, 4041, 0, 0, 0, 0, 0, 4096, 0, 0, 2916, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 10, 3171, 0, 0, 0, 4096, 0, 0, 0, 109, 0, 0, 0, 0, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0] payload_rows_per_dst=[8192, 8192, 8132, 8186, 6174, 36, 8192, 8192] dup_rows_per_dst=[6, 32, 3130, 6446, 0, 8, 218, 400] combine_rows_per_src=[8192, 8192, 8132, 8186, 6174, 36, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52562 payload_rows_remote=49460 dup_rows=12974 dup_rows_remote=12962 dedup_ratio=0.197968 router_tokens_per_expert=[0, 0, 0, 0, 0, 22, 0, 0, 1448, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 3722, 0, 0, 0, 1508, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 0, 0, 0, 0, 0, 0, 3096, 0, 0, 20, 0, 0, 6, 17, 0, 0, 0, 0, 0, 0, 3263, 0, 0, 0, 1, 0, 191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 48, 0, 2829, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 1, 0, 0, 1, 55, 3690, 0, 0, 0] payload_rows_per_dst=[3666, 3102, 7892, 8192, 7814, 6418, 7396, 8082] dup_rows_per_dst=[60, 12, 3758, 6610, 16, 2444, 0, 74] combine_rows_per_src=[3666, 3102, 7892, 8192, 7814, 6418, 7396, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=192937984 dispatch_meta_bytes=229352 combine_payload_bytes=192937984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193167336 ingress_bytes=211812352 link_bytes=211812352 token_bytes=386105320 weight_movement_bytes=18874368 total_nvlink_bytes=404979688 weight_to_token_ratio=0.048884 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.029481739e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62836 payload_rows_remote=54644 dup_rows=2700 dup_rows_remote=2084 dedup_ratio=0.041199 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 232, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1822, 0, 0, 0, 0, 4096, 2125, 0, 0] payload_rows_per_dst=[8192, 8192, 8038, 8192, 8190, 6200, 8192, 7640] dup_rows_per_dst=[4, 1302, 0, 616, 0, 0, 524, 254] combine_rows_per_src=[8192, 8192, 8038, 8192, 8190, 6200, 8192, 7640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55110 payload_rows_remote=46920 dup_rows=10426 dup_rows_remote=3392 dedup_ratio=0.159088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 4096, 1, 0, 4096, 0, 0, 0, 49, 0, 0, 1, 0, 0, 7, 3450, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 22, 2002, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 0, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 4, 46, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 113, 3942, 0, 0, 21, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 6592, 5916, 4730, 5106, 8192] dup_rows_per_dst=[20, 30, 7034, 660, 2330, 182, 170, 0] combine_rows_per_src=[8192, 8192, 8190, 6592, 5916, 4730, 5106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53384 payload_rows_remote=47892 dup_rows=12152 dup_rows_remote=11888 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1207, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4066, 0, 0, 3977, 0, 0, 19, 3, 19, 0, 510, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 4, 0, 0, 0, 3909, 0, 0, 3443, 4, 3, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 135, 50, 0, 154, 2530, 0, 0, 0, 2160, 2986, 0, 0, 0, 0, 179, 0, 0, 3278, 1, 0, 0, 0, 0, 14, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 0] payload_rows_per_dst=[8164, 7996, 3362, 4414, 7812, 5492, 8182, 7962] dup_rows_per_dst=[2488, 0, 246, 104, 6, 264, 9026, 18] combine_rows_per_src=[8164, 7996, 3362, 4414, 7812, 5492, 8182, 7962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47576 payload_rows_remote=39384 dup_rows=17960 dup_rows_remote=10394 dedup_ratio=0.274048 router_tokens_per_expert=[0, 447, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 0, 0, 2720, 0, 0, 0, 0, 7, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 4, 0, 0, 0, 1, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 2289, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4085, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 49, 0, 0, 0, 6, 0, 0, 365, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 0, 0, 586, 0, 0, 0, 0, 4093, 10, 0, 0] payload_rows_per_dst=[8192, 5802, 3516, 4804, 8136, 8188, 746, 8192] dup_rows_per_dst=[918, 116, 1028, 52, 418, 7860, 2, 7566] combine_rows_per_src=[8192, 5802, 3516, 4804, 8136, 8188, 746, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63216 payload_rows_remote=55374 dup_rows=2320 dup_rows_remote=2298 dedup_ratio=0.035400 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 381, 3485, 0, 0, 0, 0, 0, 4096, 0, 0, 3881, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 5, 0, 610, 4079, 0, 0, 7, 3746, 26, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0] payload_rows_per_dst=[8192, 8192, 7814, 7270, 7842, 8172, 7542, 8192] dup_rows_per_dst=[36, 34, 196, 462, 22, 1370, 170, 30] combine_rows_per_src=[8192, 8192, 7814, 7270, 7842, 8172, 7542, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55038 payload_rows_remote=49546 dup_rows=10498 dup_rows_remote=10204 dedup_ratio=0.160187 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 3898, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 0, 0, 31, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 96, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 41, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2819, 0, 0, 0, 951, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7896, 8010, 5118, 7180, 8192, 8192, 5492, 4958] dup_rows_per_dst=[18, 60, 0, 0, 32, 8356, 294, 1738] combine_rows_per_src=[7896, 8010, 5118, 7180, 8192, 8192, 5492, 4958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=202588160 dispatch_meta_bytes=249688 combine_payload_bytes=202588160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=202837848 ingress_bytes=259211264 link_bytes=259211264 token_bytes=405426008 weight_movement_bytes=56623104 total_nvlink_bytes=462049112 weight_to_token_ratio=0.139663 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.483634792e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=223821824 dispatch_meta_bytes=226912 combine_payload_bytes=223821824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224048736 ingress_bytes=242696192 link_bytes=242696192 token_bytes=447870560 weight_movement_bytes=18874368 total_nvlink_bytes=466744928 weight_to_token_ratio=0.042142 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.325395498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=192184320 dispatch_meta_bytes=201248 combine_payload_bytes=192184320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=192385568 ingress_bytes=211058688 link_bytes=211058688 token_bytes=384569888 weight_movement_bytes=18874368 total_nvlink_bytes=403444256 weight_to_token_ratio=0.049079 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.093750 gemm_active_groups=11 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.022260501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=196165632 dispatch_meta_bytes=239120 combine_payload_bytes=196165632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=196404752 ingress_bytes=233914368 link_bytes=233914368 token_bytes=392570384 weight_movement_bytes=37748736 total_nvlink_bytes=430319120 weight_to_token_ratio=0.096158 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.241252381e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=161316864 dispatch_meta_bytes=199112 combine_payload_bytes=161316864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=161515976 ingress_bytes=199065600 link_bytes=199065600 token_bytes=322832840 weight_movement_bytes=37748736 total_nvlink_bytes=360581576 weight_to_token_ratio=0.116930 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=12672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.907348633e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=226811904 dispatch_meta_bytes=230688 combine_payload_bytes=226811904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227042592 ingress_bytes=245686272 link_bytes=245686272 token_bytes=453854496 weight_movement_bytes=18874368 total_nvlink_bytes=472728864 weight_to_token_ratio=0.041587 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.354044973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=202940416 dispatch_meta_bytes=239000 combine_payload_bytes=202940416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203179416 ingress_bytes=221814784 link_bytes=221814784 token_bytes=406119832 weight_movement_bytes=18874368 total_nvlink_bytes=424994200 weight_to_token_ratio=0.046475 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.125320121e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54562 payload_rows_remote=46370 dup_rows=10974 dup_rows_remote=8196 dedup_ratio=0.167450 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 36, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 1, 0, 0, 3705, 0, 0, 4095, 0, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8000, 8192, 7750, 5000, 8178, 8192, 1058] dup_rows_per_dst=[2778, 5952, 2174, 10, 0, 60, 0, 0] combine_rows_per_src=[8192, 8000, 8192, 7750, 5000, 8178, 8192, 1058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=189931520 dispatch_meta_bytes=218264 combine_payload_bytes=189931520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=190149784 ingress_bytes=246554624 link_bytes=246554624 token_bytes=380081304 weight_movement_bytes=56623104 total_nvlink_bytes=436704408 weight_to_token_ratio=0.148976 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362365095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52776 payload_rows_remote=44650 dup_rows=12760 dup_rows_remote=12760 dedup_ratio=0.194702 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 115, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 268, 0, 0, 37, 0, 0, 3964, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4094, 0, 0, 0, 0, 2371, 1, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8126, 7554, 8192, 8000, 42, 4742, 7930] dup_rows_per_dst=[1922, 0, 24, 3738, 7070, 0, 2, 4] combine_rows_per_src=[8190, 8126, 7554, 8192, 8000, 42, 4742, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56086 payload_rows_remote=55994 dup_rows=9450 dup_rows_remote=9450 dedup_ratio=0.144196 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 165, 0, 0, 0, 0, 0, 261, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 3273, 4, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 3683, 4091, 0, 0, 743, 3501, 156, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 7230, 92, 8182, 8192, 7820, 8192] dup_rows_per_dst=[0, 428, 182, 0, 0, 7846, 986, 8] combine_rows_per_src=[8186, 8192, 7230, 92, 8182, 8192, 7820, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51964 payload_rows_remote=43772 dup_rows=13572 dup_rows_remote=11934 dedup_ratio=0.207092 router_tokens_per_expert=[13, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 4, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8, 0, 84, 3851, 0, 0, 4096, 0, 525, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 3388, 0, 0, 0, 0, 1096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 112, 0, 84, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8080, 7944, 8192, 2666, 8172, 8192, 8176, 542] dup_rows_per_dst=[3326, 6770, 1638, 2, 224, 172, 1434, 6] combine_rows_per_src=[8080, 7944, 8192, 2666, 8172, 8192, 8176, 542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52102 payload_rows_remote=46616 dup_rows=13434 dup_rows_remote=12278 dedup_ratio=0.204987 router_tokens_per_expert=[0, 0, 0, 0, 3969, 0, 0, 0, 0, 5, 439, 0, 4081, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 5, 4076, 0, 0, 1589, 0, 17, 0, 13, 15, 1, 0, 0, 0, 0, 0, 0, 2984, 3988, 0, 0, 0, 0, 7, 3, 0, 140, 0, 0, 0, 0, 0, 0, 26, 57, 0, 0, 0, 0, 0, 0, 104, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 210, 0, 6, 0, 884, 0, 0, 38, 0, 58, 0, 21, 0, 7, 0, 0, 1, 0, 0, 5, 0, 10, 0, 0, 0, 24, 64, 4, 23, 0, 0, 0, 0, 0, 0, 1, 0, 1376, 0, 0, 0, 0] payload_rows_per_dst=[8172, 1536, 5080, 8190, 8124, 5486, 7520, 7994] dup_rows_per_dst=[878, 0, 1310, 6846, 344, 1156, 20, 2880] combine_rows_per_src=[8172, 1536, 5080, 8190, 8124, 5486, 7520, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45010 payload_rows_remote=37008 dup_rows=20526 dup_rows_remote=20322 dedup_ratio=0.313202 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 40, 0, 4058, 0, 0, 4083, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 0, 0, 0, 0, 4001, 0, 0, 3975, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1691, 0, 0, 0, 1, 0, 0, 80, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3600, 3623, 0, 0, 3891, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2726, 5604, 972, 7950, 3386, 8192, 8178, 8002] dup_rows_per_dst=[1044, 0, 0, 5002, 10, 10582, 3684, 204] combine_rows_per_src=[2726, 5604, 972, 7950, 3386, 8192, 8178, 8002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55122 payload_rows_remote=47026 dup_rows=10414 dup_rows_remote=10414 dedup_ratio=0.158905 router_tokens_per_expert=[0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3141, 0, 0, 4096, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 94, 0, 0, 0, 4096, 0, 999, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 0] payload_rows_per_dst=[3416, 6310, 8192, 8192, 8096, 4626, 8192, 8098] dup_rows_per_dst=[0, 0, 586, 1794, 0, 240, 0, 7794] combine_rows_per_src=[3416, 6310, 8192, 8192, 8096, 4626, 8192, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=182886400 dispatch_meta_bytes=229640 combine_payload_bytes=182886400 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=183116040 ingress_bytes=239509504 link_bytes=239509504 token_bytes=366002440 weight_movement_bytes=56623104 total_nvlink_bytes=422625544 weight_to_token_ratio=0.154707 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294862221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65132 payload_rows_remote=56940 dup_rows=404 dup_rows_remote=398 dedup_ratio=0.006165 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 11, 82, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8046, 8118, 8192, 8174, 8026, 8192, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 0, 0, 384, 6, 0] combine_rows_per_src=[8046, 8118, 8192, 8174, 8026, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=229351424 dispatch_meta_bytes=261776 combine_payload_bytes=229351424 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=229613200 ingress_bytes=304848896 link_bytes=304848896 token_bytes=458964624 weight_movement_bytes=75497472 total_nvlink_bytes=534462096 weight_to_token_ratio=0.164495 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=12416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.920912126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=190939136 dispatch_meta_bytes=235576 combine_payload_bytes=190939136 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=191174712 ingress_bytes=247562240 link_bytes=247562240 token_bytes=382113848 weight_movement_bytes=56623104 total_nvlink_bytes=438736952 weight_to_token_ratio=0.148184 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372019575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=179290112 dispatch_meta_bytes=222824 combine_payload_bytes=179290112 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=179512936 ingress_bytes=217038848 link_bytes=217038848 token_bytes=358803048 weight_movement_bytes=37748736 total_nvlink_bytes=396551784 weight_to_token_ratio=0.105207 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.079559452e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=151584768 dispatch_meta_bytes=229320 combine_payload_bytes=151584768 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=151814088 ingress_bytes=189333504 link_bytes=189333504 token_bytes=303398856 weight_movement_bytes=37748736 total_nvlink_bytes=341147592 weight_to_token_ratio=0.124420 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.814100477e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=192618496 dispatch_meta_bytes=229760 combine_payload_bytes=192618496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=192848256 ingress_bytes=211492864 link_bytes=211492864 token_bytes=385466752 weight_movement_bytes=18874368 total_nvlink_bytes=404341120 weight_to_token_ratio=0.048965 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.026420562e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52546 payload_rows_remote=44360 dup_rows=12990 dup_rows_remote=9498 dedup_ratio=0.198212 router_tokens_per_expert=[4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 1, 0, 0, 0, 2, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8176, 5062, 8192, 6198, 8192, 348] dup_rows_per_dst=[3492, 0, 4740, 704, 946, 3090, 18, 0] combine_rows_per_src=[8186, 8192, 8176, 5062, 8192, 6198, 8192, 348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233226240 dispatch_meta_bytes=229352 combine_payload_bytes=233226240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233455592 ingress_bytes=270974976 link_bytes=270974976 token_bytes=466681832 weight_movement_bytes=37748736 total_nvlink_bytes=504430568 weight_to_token_ratio=0.080888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596348892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=181698560 dispatch_meta_bytes=215432 combine_payload_bytes=181698560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=181913992 ingress_bytes=200572928 link_bytes=200572928 token_bytes=363612552 weight_movement_bytes=18874368 total_nvlink_bytes=382486920 weight_to_token_ratio=0.051908 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.921791108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49186 payload_rows_remote=41004 dup_rows=16350 dup_rows_remote=14376 dedup_ratio=0.249481 router_tokens_per_expert=[0, 0, 0, 0, 3823, 0, 0, 0, 0, 5, 0, 0, 2348, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 533, 18, 0, 0, 503, 0, 0, 4049, 0, 3652, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4016, 0, 0, 0, 440, 1671, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1218, 0, 0, 0, 162, 0, 0, 0, 2031, 0, 0, 0, 284, 0, 145, 0, 595, 0, 2930, 0, 3210, 0, 0, 0, 37, 0, 23, 50, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, 0, 0, 3, 0, 2, 11, 0, 1, 0, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0] payload_rows_per_dst=[7658, 8182, 8172, 4118, 6460, 6948, 5914, 1734] dup_rows_per_dst=[4694, 1974, 7290, 264, 930, 1172, 24, 2] combine_rows_per_src=[7658, 8182, 8172, 4118, 6460, 6948, 5914, 1734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55470 payload_rows_remote=49996 dup_rows=10066 dup_rows_remote=9772 dedup_ratio=0.153595 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 3397, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 823, 0, 0, 4096, 0, 264, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4086, 0, 1082, 0, 0, 0, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0] payload_rows_per_dst=[6794, 4248, 8192, 5474, 8192, 6206, 8172, 8192] dup_rows_per_dst=[2, 858, 808, 294, 26, 348, 0, 7730] combine_rows_per_src=[6794, 4248, 8192, 5474, 8192, 6206, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59578 payload_rows_remote=52306 dup_rows=5958 dup_rows_remote=5948 dedup_ratio=0.090912 router_tokens_per_expert=[3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 407, 0, 0, 0, 6, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 74, 0, 0, 0, 0, 996, 0, 0, 4059, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 648, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7832, 8192, 7272, 7368, 8060, 8192, 8140, 4522] dup_rows_per_dst=[0, 0, 10, 2408, 96, 1418, 1980, 46] combine_rows_per_src=[7832, 8192, 7272, 7368, 8060, 8192, 8140, 4522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61178 payload_rows_remote=53068 dup_rows=4358 dup_rows_remote=2956 dedup_ratio=0.066498 router_tokens_per_expert=[4087, 0, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 30, 0, 0, 3411, 0, 1719, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 1, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4092, 0, 56, 18, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8162, 6882, 7090, 8036, 8110, 6518, 8190] dup_rows_per_dst=[24, 108, 12, 1458, 0, 1402, 1178, 176] combine_rows_per_src=[8190, 8162, 6882, 7090, 8036, 8110, 6518, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50320 payload_rows_remote=42338 dup_rows=15216 dup_rows_remote=14988 dedup_ratio=0.232178 router_tokens_per_expert=[0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 2, 112, 3699, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2698, 0, 0, 4096, 0, 25, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 8, 0, 0, 0, 45, 0, 4096, 0, 0, 0, 0, 0, 3904, 5, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 3, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] payload_rows_per_dst=[3930, 5572, 72, 8188, 8192, 8192, 8192, 7982] dup_rows_per_dst=[178, 0, 0, 3334, 3526, 6354, 1596, 228] combine_rows_per_src=[3930, 5572, 72, 8188, 8192, 8192, 8192, 7982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61676 payload_rows_remote=53484 dup_rows=3860 dup_rows_remote=3850 dedup_ratio=0.058899 router_tokens_per_expert=[4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4036, 0, 0, 0, 6, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1865, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 92, 0, 0, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4059, 0, 29, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 8190, 8190, 8132, 8192, 8192, 4506, 8144] dup_rows_per_dst=[0, 0, 0, 30, 10, 3760, 2, 58] combine_rows_per_src=[8130, 8190, 8190, 8132, 8192, 8192, 4506, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=167952384 dispatch_meta_bytes=221520 combine_payload_bytes=167952384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=168173904 ingress_bytes=186826752 link_bytes=186826752 token_bytes=336126288 weight_movement_bytes=18874368 total_nvlink_bytes=355000656 weight_to_token_ratio=0.056153 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.790082013e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=204783616 dispatch_meta_bytes=239072 combine_payload_bytes=204783616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=205022688 ingress_bytes=261406720 link_bytes=261406720 token_bytes=409806304 weight_movement_bytes=56623104 total_nvlink_bytes=466429408 weight_to_token_ratio=0.138170 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.504670571e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=214245376 dispatch_meta_bytes=233016 combine_payload_bytes=214245376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214478392 ingress_bytes=233119744 link_bytes=233119744 token_bytes=428723768 weight_movement_bytes=18874368 total_nvlink_bytes=447598136 weight_to_token_ratio=0.044025 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.233638685e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=224096 combine_payload_bytes=217366528 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=217590624 ingress_bytes=311738368 link_bytes=311738368 token_bytes=434957152 weight_movement_bytes=94371840 total_nvlink_bytes=529328992 weight_to_token_ratio=0.216968 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.986923657e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61198 payload_rows_remote=53006 dup_rows=4338 dup_rows_remote=2898 dedup_ratio=0.066193 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4007, 4030, 1, 0, 1485, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 2254, 0, 0, 0, 0, 0] payload_rows_per_dst=[8020, 8176, 8192, 8150, 7770, 8188, 8192, 4510] dup_rows_per_dst=[0, 2878, 12, 0, 0, 0, 1440, 8] combine_rows_per_src=[8020, 8176, 8192, 8150, 7770, 8188, 8192, 4510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=173416448 dispatch_meta_bytes=229304 combine_payload_bytes=173416448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=173645752 ingress_bytes=192290816 link_bytes=192290816 token_bytes=347062200 weight_movement_bytes=18874368 total_nvlink_bytes=365936568 weight_to_token_ratio=0.054383 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.842435986e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=219070464 dispatch_meta_bytes=229336 combine_payload_bytes=219070464 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=219299800 ingress_bytes=256819200 link_bytes=256819200 token_bytes=438370264 weight_movement_bytes=37748736 total_nvlink_bytes=476119000 weight_to_token_ratio=0.086112 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.460715211e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52758 payload_rows_remote=44566 dup_rows=12778 dup_rows_remote=12706 dedup_ratio=0.194977 router_tokens_per_expert=[0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3036, 0, 0, 3983, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8184, 3538, 8086, 188] dup_rows_per_dst=[72, 4508, 10, 8188, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8184, 3538, 8086, 188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=217112576 dispatch_meta_bytes=223616 combine_payload_bytes=217112576 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=217336192 ingress_bytes=292610048 link_bytes=292610048 token_bytes=434448768 weight_movement_bytes=75497472 total_nvlink_bytes=509946240 weight_to_token_ratio=0.173778 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.079861 gemm_active_groups=14 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.803645507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=182542336 dispatch_meta_bytes=229088 combine_payload_bytes=182542336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=182771424 ingress_bytes=201416704 link_bytes=201416704 token_bytes=365313760 weight_movement_bytes=18874368 total_nvlink_bytes=384188128 weight_to_token_ratio=0.051666 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.929875755e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54054 payload_rows_remote=45952 dup_rows=11482 dup_rows_remote=11456 dedup_ratio=0.175201 router_tokens_per_expert=[4044, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 1, 0, 0, 0, 3442, 0, 199, 12, 0, 0, 493, 0, 1368, 0, 0, 0, 145, 17, 0, 0, 16, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4053, 0, 3681, 0, 0, 0, 0, 8, 0, 470, 0, 4066, 0, 0, 0, 0, 6, 0, 734, 0, 0, 4025, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1477, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 8102, 6898, 7204, 7410, 8192, 8060, 96] dup_rows_per_dst=[0, 26, 12, 250, 0, 9254, 1940, 0] combine_rows_per_src=[8092, 8102, 6898, 7204, 7410, 8192, 8060, 96] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54240 payload_rows_remote=46048 dup_rows=11296 dup_rows_remote=5742 dedup_ratio=0.172363 router_tokens_per_expert=[0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2695, 0, 0, 3, 79, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2914, 1, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4091, 0, 0, 0, 0, 0, 105, 4, 0, 0, 1374, 0, 4096, 0, 0, 0, 0, 36, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 1, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 3046, 0, 16, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8048, 8192, 6940, 8192, 218, 8188, 6272] dup_rows_per_dst=[2600, 1022, 5554, 1936, 134, 0, 26, 24] combine_rows_per_src=[8190, 8048, 8192, 6940, 8192, 218, 8188, 6272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61126 payload_rows_remote=53364 dup_rows=4410 dup_rows_remote=2586 dedup_ratio=0.067291 router_tokens_per_expert=[4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1402, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 0, 63, 0, 0, 0, 0, 7, 0, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 6, 0, 0, 4095, 0, 1246, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8090, 8192, 8192, 7762, 8192, 6772, 5734, 8192] dup_rows_per_dst=[0, 0, 12, 1824, 0, 4, 0, 2570] combine_rows_per_src=[8090, 8192, 8192, 7762, 8192, 6772, 5734, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52270 payload_rows_remote=44416 dup_rows=13266 dup_rows_remote=12294 dedup_ratio=0.202423 router_tokens_per_expert=[0, 177, 0, 0, 24, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 2441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 93, 0, 0, 0, 0, 0, 3, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 3877, 2, 0, 0, 0, 27, 0, 3900, 0, 0, 4082, 0, 4081, 0, 0, 0, 0, 0, 3, 0, 137, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 33, 0, 4087, 0, 125, 4066, 0, 0, 0, 0] payload_rows_per_dst=[7872, 4898, 198, 7220, 8178, 7854, 7858, 8192] dup_rows_per_dst=[310, 2, 0, 3364, 88, 972, 88, 8442] combine_rows_per_src=[7872, 4898, 198, 7220, 8178, 7854, 7858, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55328 payload_rows_remote=47136 dup_rows=10208 dup_rows_remote=1262 dedup_ratio=0.155762 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 3786, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5, 0, 0, 0, 0, 0, 256, 0, 4090, 0, 0, 0, 0, 2, 0, 1, 0, 3551, 0, 0, 0, 0, 0, 0, 2, 0, 0, 362, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 3894, 0, 3809, 2, 0, 0, 0, 0, 29] payload_rows_per_dst=[8192, 8148, 6948, 7742, 8192, 518, 7396, 8192] dup_rows_per_dst=[2, 32, 12, 594, 16, 0, 606, 8946] combine_rows_per_src=[8192, 8148, 6948, 7742, 8192, 518, 7396, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57988 payload_rows_remote=49798 dup_rows=7548 dup_rows_remote=7548 dedup_ratio=0.115173 router_tokens_per_expert=[0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 1, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8136, 1854, 8192, 8190, 7448, 8192, 7784] dup_rows_per_dst=[400, 0, 594, 28, 0, 0, 2, 6524] combine_rows_per_src=[8192, 8136, 1854, 8192, 8190, 7448, 8192, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=188219392 dispatch_meta_bytes=229632 combine_payload_bytes=188219392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=188449024 ingress_bytes=207093760 link_bytes=207093760 token_bytes=376668416 weight_movement_bytes=18874368 total_nvlink_bytes=395542784 weight_to_token_ratio=0.050109 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.984270512e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=188612608 dispatch_meta_bytes=207160 combine_payload_bytes=188612608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=188819768 ingress_bytes=207486976 link_bytes=207486976 token_bytes=377432376 weight_movement_bytes=18874368 total_nvlink_bytes=396306744 weight_to_token_ratio=0.050007 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.988038114e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=218578944 dispatch_meta_bytes=223800 combine_payload_bytes=218578944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218802744 ingress_bytes=237453312 link_bytes=237453312 token_bytes=437381688 weight_movement_bytes=18874368 total_nvlink_bytes=456256056 weight_to_token_ratio=0.043153 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.275160801e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=181927936 dispatch_meta_bytes=226840 combine_payload_bytes=181927936 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=182154776 ingress_bytes=295174144 link_bytes=295174144 token_bytes=364082712 weight_movement_bytes=113246208 total_nvlink_bytes=477328920 weight_to_token_ratio=0.311045 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.093750 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.828213413e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=193069056 dispatch_meta_bytes=193592 combine_payload_bytes=193069056 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=193262648 ingress_bytes=268566528 link_bytes=268566528 token_bytes=386331704 weight_movement_bytes=75497472 total_nvlink_bytes=461829176 weight_to_token_ratio=0.195421 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.573272328e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50936 payload_rows_remote=42744 dup_rows=14600 dup_rows_remote=12144 dedup_ratio=0.222778 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4054, 0, 4093, 0, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2526, 6096, 5216, 4414, 8108, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2196, 402, 2, 3470, 2456, 6074] combine_rows_per_src=[2526, 6096, 5216, 4414, 8108, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=203972608 dispatch_meta_bytes=229384 combine_payload_bytes=203972608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204201992 ingress_bytes=222846976 link_bytes=222846976 token_bytes=408174600 weight_movement_bytes=18874368 total_nvlink_bytes=427048968 weight_to_token_ratio=0.046241 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.135210076e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57730 payload_rows_remote=50642 dup_rows=7806 dup_rows_remote=4556 dedup_ratio=0.119110 router_tokens_per_expert=[1196, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 2085, 3908, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 372, 12, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7088, 8074, 8192, 6374, 6038, 6340, 7534, 8090] dup_rows_per_dst=[3250, 2778, 0, 0, 1714, 64, 0, 0] combine_rows_per_src=[7088, 8074, 8192, 6374, 6038, 6340, 7534, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=175079424 dispatch_meta_bytes=219552 combine_payload_bytes=175079424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175298976 ingress_bytes=193953792 link_bytes=193953792 token_bytes=350378400 weight_movement_bytes=18874368 total_nvlink_bytes=369252768 weight_to_token_ratio=0.053869 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858369804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=207429632 dispatch_meta_bytes=220792 combine_payload_bytes=207429632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=207650424 ingress_bytes=264052736 link_bytes=264052736 token_bytes=415080056 weight_movement_bytes=56623104 total_nvlink_bytes=471703160 weight_to_token_ratio=0.136415 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=15488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.530023394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49456 payload_rows_remote=41490 dup_rows=16080 dup_rows_remote=16078 dedup_ratio=0.245361 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 1, 49, 3007, 0, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 752, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 14, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 3919, 99, 0, 1, 0, 0, 0, 0, 0, 0, 3882, 194, 4017, 241, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 3, 4059, 0, 0, 78, 6] payload_rows_per_dst=[7956, 7966, 8192, 7964, 8174, 632, 8062, 510] dup_rows_per_dst=[6014, 2, 3590, 5992, 38, 438, 0, 6] combine_rows_per_src=[7956, 7966, 8192, 7964, 8174, 632, 8062, 510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54912 payload_rows_remote=46720 dup_rows=10624 dup_rows_remote=10516 dedup_ratio=0.162109 router_tokens_per_expert=[1, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3950, 0, 0, 6, 18, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1935, 3525, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 321, 0, 104, 20, 0, 0, 0, 1, 0, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8002, 8192, 1410, 8112, 7968, 8192, 4848] dup_rows_per_dst=[6020, 464, 108, 24, 3576, 356, 76, 0] combine_rows_per_src=[8188, 8002, 8192, 1410, 8112, 7968, 8192, 4848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56370 payload_rows_remote=48926 dup_rows=9166 dup_rows_remote=9154 dedup_ratio=0.139862 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 32, 0, 0, 0, 0, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3719, 0, 4096, 0, 0, 0, 52, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 67, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8060, 7976, 8192, 7444, 8192, 8180, 8192, 134] dup_rows_per_dst=[126, 536, 8366, 12, 14, 0, 104, 8] combine_rows_per_src=[8060, 7976, 8192, 7444, 8192, 8180, 8192, 134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47082 payload_rows_remote=38896 dup_rows=18454 dup_rows_remote=9994 dedup_ratio=0.281586 router_tokens_per_expert=[0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 3, 0, 115, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 17, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 55, 2249, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 9, 0, 0, 0, 0, 0, 4042, 0, 0, 3848, 0, 4089, 0, 0, 0, 98, 101, 0, 0, 0, 0, 4052, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 913, 0, 0, 4091, 0, 0, 1, 0] payload_rows_per_dst=[1864, 8192, 4500, 64, 7960, 8130, 8186, 8186] dup_rows_per_dst=[0, 444, 120, 0, 0, 9106, 324, 8460] combine_rows_per_src=[1864, 8192, 4500, 64, 7960, 8130, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62596 payload_rows_remote=54466 dup_rows=2940 dup_rows_remote=2808 dedup_ratio=0.044861 router_tokens_per_expert=[0, 0, 0, 4058, 59, 0, 0, 0, 11, 0, 0, 380, 4063, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 28, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 3943, 0, 5, 58, 3667, 0, 0, 0, 0, 44, 0, 4051, 0, 0, 0, 5, 0, 0, 0, 0, 4033, 0, 0, 1, 3469, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 35, 0, 0, 2, 7, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 240, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 61, 14, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7478, 7474, 8120, 8148, 6956, 8130, 8098, 8192] dup_rows_per_dst=[834, 12, 558, 766, 4, 132, 482, 152] combine_rows_per_src=[7478, 7474, 8120, 8148, 6956, 8130, 8098, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57572 payload_rows_remote=49380 dup_rows=7964 dup_rows_remote=6730 dedup_ratio=0.121521 router_tokens_per_expert=[8, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 617, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 244, 8186, 8182] dup_rows_per_dst=[16, 6682, 0, 0, 1234, 0, 0, 32] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 244, 8186, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=169943040 dispatch_meta_bytes=230272 combine_payload_bytes=169943040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=170173312 ingress_bytes=188817408 link_bytes=188817408 token_bytes=340116352 weight_movement_bytes=18874368 total_nvlink_bytes=358990720 weight_to_token_ratio=0.055494 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.809155499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=191365120 dispatch_meta_bytes=228944 combine_payload_bytes=191365120 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=191594064 ingress_bytes=210239488 link_bytes=210239488 token_bytes=382959184 weight_movement_bytes=18874368 total_nvlink_bytes=401833552 weight_to_token_ratio=0.049286 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.014411330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=200400896 dispatch_meta_bytes=232320 combine_payload_bytes=200400896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200633216 ingress_bytes=219275264 link_bytes=219275264 token_bytes=401034112 weight_movement_bytes=18874368 total_nvlink_bytes=419908480 weight_to_token_ratio=0.047064 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.100987689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=195560 combine_payload_bytes=159318016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=159513576 ingress_bytes=197066752 link_bytes=197066752 token_bytes=318831592 weight_movement_bytes=37748736 total_nvlink_bytes=356580328 weight_to_token_ratio=0.118397 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.888196655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=223092736 dispatch_meta_bytes=229096 combine_payload_bytes=223092736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223321832 ingress_bytes=241967104 link_bytes=241967104 token_bytes=446414568 weight_movement_bytes=18874368 total_nvlink_bytes=465288936 weight_to_token_ratio=0.042280 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.318409735e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=202260480 dispatch_meta_bytes=224440 combine_payload_bytes=202260480 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=202484920 ingress_bytes=277757952 link_bytes=277757952 token_bytes=404745400 weight_movement_bytes=75497472 total_nvlink_bytes=480242872 weight_to_token_ratio=0.186531 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.661340031e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61810 payload_rows_remote=53896 dup_rows=3726 dup_rows_remote=3660 dedup_ratio=0.056854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 2490, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 5, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 4096, 910, 0, 0] payload_rows_per_dst=[8192, 6578, 8074, 6538, 8190, 8192, 7914, 8132] dup_rows_per_dst=[2, 0, 2, 552, 0, 1194, 66, 1910] combine_rows_per_src=[8192, 6578, 8074, 6538, 8190, 8192, 7914, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44216 payload_rows_remote=36024 dup_rows=21320 dup_rows_remote=13656 dedup_ratio=0.325317 router_tokens_per_expert=[21, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 2335, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1243, 0, 0, 4085, 0, 0, 0, 0, 0, 3815, 0, 0, 0, 833, 0, 1415, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6988, 8190, 8188, 5310, 2496, 4830, 22] dup_rows_per_dst=[7664, 568, 3910, 8176, 50, 2, 950, 0] combine_rows_per_src=[8192, 6988, 8190, 8188, 5310, 2496, 4830, 22] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=147554304 dispatch_meta_bytes=198720 combine_payload_bytes=147554304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=147753024 ingress_bytes=166428672 link_bytes=166428672 token_bytes=295307328 weight_movement_bytes=18874368 total_nvlink_bytes=314181696 weight_to_token_ratio=0.063914 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.594637647e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=220758016 dispatch_meta_bytes=230224 combine_payload_bytes=220758016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220988240 ingress_bytes=239632384 link_bytes=239632384 token_bytes=441746256 weight_movement_bytes=18874368 total_nvlink_bytes=460620624 weight_to_token_ratio=0.042727 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.296039597e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50926 payload_rows_remote=47284 dup_rows=14610 dup_rows_remote=14412 dedup_ratio=0.222931 router_tokens_per_expert=[4004, 3, 0, 2276, 0, 0, 1, 0, 9, 0, 0, 76, 74, 0, 0, 6, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 0, 3, 0, 0, 3988, 0, 0, 0, 1244, 23, 162, 0, 0, 0, 0, 0, 0, 41, 0, 37, 0, 0, 0, 0, 4078, 0, 0, 7, 2321, 893, 93, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 8, 0, 0, 1520, 0, 835, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 102, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0] payload_rows_per_dst=[8190, 3642, 8006, 324, 6450, 8128, 8190, 7996] dup_rows_per_dst=[4708, 198, 2, 202, 222, 8636, 188, 454] combine_rows_per_src=[8190, 3642, 8006, 324, 6450, 8128, 8190, 7996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50876 payload_rows_remote=42686 dup_rows=14660 dup_rows_remote=14172 dedup_ratio=0.223694 router_tokens_per_expert=[91, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 1, 0, 0, 958, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 1, 0, 0, 2801, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 3344, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 129, 0, 1127, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 6652, 8190, 7472, 8192, 4008, 8190, 58] dup_rows_per_dst=[1992, 22, 488, 2938, 6716, 8, 2496, 0] combine_rows_per_src=[8114, 6652, 8190, 7472, 8192, 4008, 8190, 58] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56532 payload_rows_remote=48910 dup_rows=9004 dup_rows_remote=8946 dedup_ratio=0.137390 router_tokens_per_expert=[136, 0, 0, 0, 0, 0, 4, 6, 3907, 0, 0, 0, 0, 9, 0, 0, 24, 0, 0, 0, 0, 0, 271, 45, 52, 0, 4032, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 7, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 32, 0, 0, 3763, 0, 0, 0, 0, 0, 4087, 0, 0, 72, 45, 0, 0, 0, 0, 3924, 0, 0, 5, 13, 43, 0, 3931, 6, 3, 0, 1, 0, 0, 0, 3, 0, 0, 55, 1, 0, 0] payload_rows_per_dst=[7924, 8180, 308, 8162, 8026, 7622, 8192, 8118] dup_rows_per_dst=[200, 668, 0, 0, 10, 58, 338, 7730] combine_rows_per_src=[7924, 8180, 308, 8162, 8026, 7622, 8192, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50384 payload_rows_remote=42298 dup_rows=15152 dup_rows_remote=11782 dedup_ratio=0.231201 router_tokens_per_expert=[0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 3604, 9, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 76, 4073, 28, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 183, 1823, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 17, 6, 0, 0] payload_rows_per_dst=[0, 4264, 7818, 8192, 5648, 8186, 8190, 8086] dup_rows_per_dst=[0, 70, 0, 7334, 84, 4274, 20, 3370] combine_rows_per_src=[0, 4264, 7818, 8192, 5648, 8186, 8190, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54646 payload_rows_remote=46454 dup_rows=10890 dup_rows_remote=10496 dedup_ratio=0.166168 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1972, 0, 0, 0, 0, 26, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 3934, 1, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1692, 0, 0, 4073, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 81, 0, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3976, 8174, 6808, 7872, 8192, 8146, 3518, 7960] dup_rows_per_dst=[20, 484, 328, 16, 394, 3388, 196, 6064] combine_rows_per_src=[3976, 8174, 6808, 7872, 8192, 8146, 3518, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61260 payload_rows_remote=53070 dup_rows=4276 dup_rows_remote=4276 dedup_ratio=0.065247 router_tokens_per_expert=[7, 0, 0, 4055, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 4096, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[8192, 7412, 8178, 8190, 8192, 4838, 8182, 8076] dup_rows_per_dst=[3352, 2, 4, 0, 878, 28, 8, 4] combine_rows_per_src=[8192, 7412, 8178, 8190, 8192, 4838, 8182, 8076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=193675264 dispatch_meta_bytes=246784 combine_payload_bytes=193675264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193922048 ingress_bytes=212549632 link_bytes=212549632 token_bytes=387597312 weight_movement_bytes=18874368 total_nvlink_bytes=406471680 weight_to_token_ratio=0.048696 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036545993e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=174841856 dispatch_meta_bytes=227432 combine_payload_bytes=174841856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175069288 ingress_bytes=193716224 link_bytes=193716224 token_bytes=349911144 weight_movement_bytes=18874368 total_nvlink_bytes=368785512 weight_to_token_ratio=0.053940 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856093544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=200335360 dispatch_meta_bytes=231424 combine_payload_bytes=200335360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200566784 ingress_bytes=238084096 link_bytes=238084096 token_bytes=400902144 weight_movement_bytes=37748736 total_nvlink_bytes=438650880 weight_to_token_ratio=0.094159 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=21632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.281204663e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=173252608 dispatch_meta_bytes=216320 combine_payload_bytes=173252608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=173468928 ingress_bytes=192126976 link_bytes=192126976 token_bytes=346721536 weight_movement_bytes=18874368 total_nvlink_bytes=365595904 weight_to_token_ratio=0.054437 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.840866152e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=190275584 dispatch_meta_bytes=227800 combine_payload_bytes=190275584 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=190503384 ingress_bytes=246898688 link_bytes=246898688 token_bytes=380778968 weight_movement_bytes=56623104 total_nvlink_bytes=437402072 weight_to_token_ratio=0.148703 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365661747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=217374720 dispatch_meta_bytes=229384 combine_payload_bytes=217374720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217604104 ingress_bytes=255123456 link_bytes=255123456 token_bytes=434978824 weight_movement_bytes=37748736 total_nvlink_bytes=472727560 weight_to_token_ratio=0.086783 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.444467427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47970 payload_rows_remote=40206 dup_rows=17566 dup_rows_remote=13272 dedup_ratio=0.268036 router_tokens_per_expert=[11, 0, 0, 0, 2129, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 263, 0, 0, 0, 0, 23, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 2285, 1163, 0, 0, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3281, 0, 0, 0, 0, 1470, 0, 0, 164, 64, 0, 14, 0, 0, 0, 0, 0, 1, 3025, 0, 0, 0, 1, 2685, 0, 0, 0, 0, 0, 0, 0, 3897, 556, 0, 1, 309, 0, 0, 0, 0, 1, 0, 0, 0, 3789, 0, 0, 84, 0, 0, 0, 0, 2801] payload_rows_per_dst=[7764, 8192, 4578, 2368, 8112, 3164, 8190, 5602] dup_rows_per_dst=[4294, 588, 0, 0, 5544, 262, 6876, 2] combine_rows_per_src=[7764, 8192, 4578, 2368, 8112, 3164, 8190, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55248 payload_rows_remote=53742 dup_rows=10288 dup_rows_remote=10276 dedup_ratio=0.156982 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 3115, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 3206, 2, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 2, 0, 162, 4060, 0, 0, 27, 584, 119, 0, 0, 0, 0, 0, 0, 4, 22, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0] payload_rows_per_dst=[8172, 7524, 7768, 6420, 7532, 8134, 1506, 8192] dup_rows_per_dst=[6, 2994, 3702, 8, 1254, 2294, 12, 18] combine_rows_per_src=[8172, 7524, 7768, 6420, 7532, 8134, 1506, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=164683776 dispatch_meta_bytes=213912 combine_payload_bytes=164683776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=164897688 ingress_bytes=183558144 link_bytes=183558144 token_bytes=329581464 weight_movement_bytes=18874368 total_nvlink_bytes=348455832 weight_to_token_ratio=0.057268 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.758763820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57262 payload_rows_remote=49824 dup_rows=8274 dup_rows_remote=8194 dedup_ratio=0.126251 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 14, 0, 1970, 0, 30, 0, 0, 3363, 0, 14, 3690, 0, 0, 0, 0, 26, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3834, 0, 0, 0, 21, 0, 3964, 0, 0, 0, 0, 0, 3, 0, 14, 4082, 11, 0, 1, 28, 0, 4065, 0, 0, 3, 2851, 0, 0, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 1234, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7652, 7438, 8130, 8188, 6962, 8026, 8144, 2722] dup_rows_per_dst=[3838, 80, 0, 470, 0, 58, 3828, 0] combine_rows_per_src=[7652, 7438, 8130, 8188, 6962, 8026, 8144, 2722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=220127232 dispatch_meta_bytes=256072 combine_payload_bytes=220127232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220383304 ingress_bytes=257875968 link_bytes=257875968 token_bytes=440510536 weight_movement_bytes=37748736 total_nvlink_bytes=478259272 weight_to_token_ratio=0.085693 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.086806 gemm_active_groups=13 gemm_peak_group_rows=14336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.470840642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57778 payload_rows_remote=52426 dup_rows=7758 dup_rows_remote=6698 dedup_ratio=0.118378 router_tokens_per_expert=[242, 0, 0, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 84, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 253, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 2309, 3070, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 45, 0, 0, 0, 532, 110, 0, 0, 375, 0, 0, 3317, 0, 0, 0, 0, 0, 18, 3948, 0, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 1587, 0, 2186, 5, 0, 21, 0, 0, 47, 0, 0, 0, 4064, 0, 0, 3900, 0, 0, 0, 0, 1842] payload_rows_per_dst=[8178, 8192, 5352, 6144, 8124, 7338, 6300, 8150] dup_rows_per_dst=[664, 14, 1060, 0, 978, 302, 2706, 2034] combine_rows_per_src=[8178, 8192, 5352, 6144, 8124, 7338, 6300, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48582 payload_rows_remote=43612 dup_rows=16954 dup_rows_remote=16184 dedup_ratio=0.258698 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 0, 1324, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 43, 689, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 12, 0, 0, 0, 0, 3276, 1086, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 376, 0, 3, 0, 0, 0, 4, 0, 0, 1531, 0, 0, 674, 0, 0, 0, 0, 34, 4, 3251, 0, 0, 0, 0, 381, 17, 0, 0, 10, 0, 0, 3, 0, 0, 1, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 3875, 0, 5, 4068, 0, 0, 0, 0, 10] payload_rows_per_dst=[8072, 8192, 8136, 4210, 5994, 4970, 816, 8192] dup_rows_per_dst=[2338, 1490, 2598, 2046, 22, 770, 8, 7682] combine_rows_per_src=[8072, 8192, 8136, 4210, 5994, 4970, 816, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59156 payload_rows_remote=51698 dup_rows=6380 dup_rows_remote=5486 dedup_ratio=0.097351 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 558, 0, 0, 3, 0, 0, 0, 0, 0, 123, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 3613, 0, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 296, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 122, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7900, 8180, 7240, 7392, 4708, 8086, 8192, 7458] dup_rows_per_dst=[1060, 42, 128, 60, 586, 3364, 246, 894] combine_rows_per_src=[7900, 8180, 7240, 7392, 4708, 8086, 8192, 7458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55716 payload_rows_remote=47604 dup_rows=9820 dup_rows_remote=5140 dedup_ratio=0.149841 router_tokens_per_expert=[9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 66, 0, 20, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 3841, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1089, 0, 0, 62, 0, 0, 2787, 0, 0, 0, 0, 0, 71, 2374, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 3755, 24, 0, 0, 11, 0, 0, 0, 0, 6, 0, 0, 0, 2725, 0, 8, 2068, 0, 0, 0, 0, 0] payload_rows_per_dst=[5488, 8192, 8178, 7682, 8112, 6362, 7538, 4164] dup_rows_per_dst=[932, 134, 2164, 0, 4680, 1656, 254, 0] combine_rows_per_src=[5488, 8192, 8178, 7682, 8112, 6362, 7538, 4164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=204079104 dispatch_meta_bytes=232072 combine_payload_bytes=204079104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204311176 ingress_bytes=222953472 link_bytes=222953472 token_bytes=408390280 weight_movement_bytes=18874368 total_nvlink_bytes=427264648 weight_to_token_ratio=0.046216 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.136230469e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56198 payload_rows_remote=49716 dup_rows=9338 dup_rows_remote=9338 dedup_ratio=0.142487 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 14, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[576, 8186, 8192, 6482, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 2438, 6850, 0, 8, 28, 14, 0] combine_rows_per_src=[576, 8186, 8192, 6482, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=214736896 dispatch_meta_bytes=236496 combine_payload_bytes=214736896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214973392 ingress_bytes=233611264 link_bytes=233611264 token_bytes=429710288 weight_movement_bytes=18874368 total_nvlink_bytes=448584656 weight_to_token_ratio=0.043923 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238348188e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=178634752 dispatch_meta_bytes=239184 combine_payload_bytes=178634752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=178873936 ingress_bytes=216383488 link_bytes=216383488 token_bytes=357508688 weight_movement_bytes=37748736 total_nvlink_bytes=395257424 weight_to_token_ratio=0.105588 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.073280115e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=211755008 dispatch_meta_bytes=228736 combine_payload_bytes=211755008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211983744 ingress_bytes=230629376 link_bytes=230629376 token_bytes=423738752 weight_movement_bytes=18874368 total_nvlink_bytes=442613120 weight_to_token_ratio=0.044542 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.209777204e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=194985984 dispatch_meta_bytes=210976 combine_payload_bytes=194985984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195196960 ingress_bytes=213860352 link_bytes=213860352 token_bytes=390182944 weight_movement_bytes=18874368 total_nvlink_bytes=409057312 weight_to_token_ratio=0.048373 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049104667e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50070 payload_rows_remote=41878 dup_rows=15466 dup_rows_remote=8232 dedup_ratio=0.235992 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3106, 0, 0, 0, 0, 2885, 0, 3924, 0, 0, 5, 0, 0, 0, 483, 0, 407, 0, 0, 0, 0, 3607, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2513, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 8, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8124, 8136, 1776, 5026, 3400, 7224] dup_rows_per_dst=[7234, 0, 2732, 5492, 6, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8124, 8136, 1776, 5026, 3400, 7224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=203636736 dispatch_meta_bytes=236216 combine_payload_bytes=203636736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203872952 ingress_bytes=222511104 link_bytes=222511104 token_bytes=407509688 weight_movement_bytes=18874368 total_nvlink_bytes=426384056 weight_to_token_ratio=0.046316 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.131991916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62054 payload_rows_remote=53952 dup_rows=3482 dup_rows_remote=3480 dedup_ratio=0.053131 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 4096, 0, 0, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4051, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 0] payload_rows_per_dst=[7438, 8190, 5638, 8192, 8110, 8192, 8102, 8192] dup_rows_per_dst=[0, 0, 0, 222, 140, 14, 2, 3104] combine_rows_per_src=[7438, 8190, 5638, 8192, 8110, 8192, 8102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=171532288 dispatch_meta_bytes=200440 combine_payload_bytes=171532288 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=171732728 ingress_bytes=209281024 link_bytes=209281024 token_bytes=343265016 weight_movement_bytes=37748736 total_nvlink_bytes=381013752 weight_to_token_ratio=0.109970 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.005227799e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46088 payload_rows_remote=38260 dup_rows=19448 dup_rows_remote=13996 dedup_ratio=0.296753 router_tokens_per_expert=[369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3361, 0, 88, 0, 0, 0, 0, 0, 0, 233, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3278, 0, 4, 1, 2, 0, 0, 0, 3980, 3184, 0, 0, 109, 0, 1, 1299, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 15, 21, 1077, 0, 0, 1, 0, 0, 2, 716, 43, 0, 0, 1658, 0, 273, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 2, 0, 4089, 0, 38, 3817, 0, 0, 50, 95, 0] payload_rows_per_dst=[8178, 7828, 7988, 6592, 4106, 2266, 8168, 962] dup_rows_per_dst=[1874, 5452, 622, 10, 250, 52, 10988, 200] combine_rows_per_src=[8178, 7828, 7988, 6592, 4106, 2266, 8168, 962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49046 payload_rows_remote=40854 dup_rows=16490 dup_rows_remote=3380 dedup_ratio=0.251617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3821, 0, 0, 3083, 0, 0, 210, 0, 372, 0, 2788, 0, 0, 536, 0, 0, 0, 4011, 0, 589, 52, 0, 0, 0, 186, 0, 0, 3489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 9, 0, 1499, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6978, 8192, 8192, 2602, 8176, 7634, 4272, 3000] dup_rows_per_dst=[0, 0, 13110, 1470, 1500, 148, 262, 0] combine_rows_per_src=[6978, 8192, 8192, 2602, 8176, 7634, 4272, 3000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=220987392 dispatch_meta_bytes=229728 combine_payload_bytes=220987392 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=221217120 ingress_bytes=258736128 link_bytes=258736128 token_bytes=442204512 weight_movement_bytes=37748736 total_nvlink_bytes=479953248 weight_to_token_ratio=0.085365 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.479082272e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53068 payload_rows_remote=45204 dup_rows=12468 dup_rows_remote=9488 dedup_ratio=0.190247 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4077, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1590, 117, 0, 0, 0, 1094, 9, 0, 0, 0, 0, 208, 0, 0, 0, 4052, 0, 0, 52, 0, 3039, 41, 2238, 0, 0, 0, 0, 0, 0, 3989, 0, 12, 0, 0, 0, 0, 0, 0, 125, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2031, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1983, 0, 0, 203, 3909, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7230, 8182, 5414, 4514, 8010, 7864, 8192, 3662] dup_rows_per_dst=[4, 8, 622, 116, 242, 2980, 8496, 0] combine_rows_per_src=[7230, 8182, 5414, 4514, 8010, 7864, 8192, 3662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43326 payload_rows_remote=35260 dup_rows=22210 dup_rows_remote=12864 dedup_ratio=0.338898 router_tokens_per_expert=[1, 0, 0, 0, 153, 0, 0, 0, 74, 333, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 168, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 695, 0, 4, 0, 0, 0, 4091, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 1, 0, 0, 3, 0, 1, 1241, 0, 1, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 3950, 1, 0, 0, 0, 36] payload_rows_per_dst=[1096, 1182, 844, 8184, 8192, 7604, 8158, 8066] dup_rows_per_dst=[40, 12, 0, 7010, 674, 2476, 2652, 9346] combine_rows_per_src=[1096, 1182, 844, 8184, 8192, 7604, 8158, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48794 payload_rows_remote=40650 dup_rows=16742 dup_rows_remote=11496 dedup_ratio=0.255463 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1, 0, 0, 4086, 0, 0, 106, 0, 3955, 0, 376, 0, 0, 2955, 0, 0, 0, 2713, 0, 75, 0, 0, 0, 0, 3907, 0, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 0, 18, 3, 2992, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4878, 8192, 242, 8178, 8144, 7458, 5714, 5988] dup_rows_per_dst=[0, 0, 0, 8152, 5246, 3266, 74, 4] combine_rows_per_src=[4878, 8192, 242, 8178, 8144, 7458, 5714, 5988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54822 payload_rows_remote=46630 dup_rows=10714 dup_rows_remote=10714 dedup_ratio=0.163483 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 574, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 785, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 4096, 0, 0, 2706, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8142, 8192, 8192, 8120, 8146, 8, 5830] dup_rows_per_dst=[56, 26, 7876, 0, 1222, 1530, 0, 4] combine_rows_per_src=[8192, 8142, 8192, 8192, 8120, 8146, 8, 5830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=209024 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156921984 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313634944 weight_movement_bytes=18874368 total_nvlink_bytes=332509312 weight_to_token_ratio=0.060179 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=167337984 dispatch_meta_bytes=176936 combine_payload_bytes=167337984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=167514920 ingress_bytes=186212352 link_bytes=186212352 token_bytes=334852904 weight_movement_bytes=18874368 total_nvlink_bytes=353727272 weight_to_token_ratio=0.056366 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.090278 gemm_active_groups=13 gemm_peak_group_rows=13056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.784195135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=185155584 dispatch_meta_bytes=218768 combine_payload_bytes=185155584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185374352 ingress_bytes=204029952 link_bytes=204029952 token_bytes=370529936 weight_movement_bytes=18874368 total_nvlink_bytes=389404304 weight_to_token_ratio=0.050939 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=22400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.954914611e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=144424960 dispatch_meta_bytes=192496 combine_payload_bytes=144424960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=144617456 ingress_bytes=201048064 link_bytes=201048064 token_bytes=289042416 weight_movement_bytes=56623104 total_nvlink_bytes=345665520 weight_to_token_ratio=0.195899 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.086806 gemm_active_groups=13 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.926343627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=166502400 dispatch_meta_bytes=208584 combine_payload_bytes=166502400 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=166710984 ingress_bytes=241999872 link_bytes=241999872 token_bytes=333213384 weight_movement_bytes=75497472 total_nvlink_bytes=408710856 weight_to_token_ratio=0.226574 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.318723702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=190996480 dispatch_meta_bytes=229376 combine_payload_bytes=190996480 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=191225856 ingress_bytes=266493952 link_bytes=266493952 token_bytes=382222336 weight_movement_bytes=75497472 total_nvlink_bytes=457719808 weight_to_token_ratio=0.197522 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.553413925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56084 payload_rows_remote=48292 dup_rows=9452 dup_rows_remote=9452 dedup_ratio=0.144226 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 2, 376, 0, 0, 0, 0, 0, 0, 0, 2933, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 1, 195, 3925, 0, 0, 0, 0, 0, 52, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1451, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4073, 3660, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7792, 8192, 8192, 7762, 7852, 754, 8174, 7366] dup_rows_per_dst=[0, 6058, 0, 2, 492, 0, 2876, 24] combine_rows_per_src=[7792, 8192, 8192, 7762, 7852, 754, 8174, 7366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54262 payload_rows_remote=46372 dup_rows=11274 dup_rows_remote=7898 dedup_ratio=0.172028 router_tokens_per_expert=[2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3944, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 206, 0, 2647, 0, 1, 0, 0, 0, 0, 1120, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1418, 0, 0, 3983, 0, 2553, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5306, 8192, 8170, 7888, 8192, 432, 7890, 8192] dup_rows_per_dst=[0, 0, 10, 20, 2, 2, 3376, 7864] combine_rows_per_src=[5306, 8192, 8170, 7888, 8192, 432, 7890, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=197804032 dispatch_meta_bytes=230976 combine_payload_bytes=197804032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198035008 ingress_bytes=216678400 link_bytes=216678400 token_bytes=395839040 weight_movement_bytes=18874368 total_nvlink_bytes=414713408 weight_to_token_ratio=0.047682 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.076105816e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54546 payload_rows_remote=46354 dup_rows=10990 dup_rows_remote=10952 dedup_ratio=0.167694 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 28, 0, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 82, 0, 88, 0, 0, 0, 3, 0, 0, 0, 0, 224, 0, 0, 1029, 0, 0, 0, 3956, 0, 0, 791, 0, 4, 0, 3270, 0, 0, 911, 0, 0, 0, 46, 0, 0, 0, 0, 0, 28, 3801, 0, 0, 3614, 0, 0, 6, 3, 186, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 27, 0, 0, 26, 0, 0, 4055, 0, 2857, 0, 16, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7532, 8192, 8192, 7974, 1606, 7082, 8160, 5808] dup_rows_per_dst=[824, 38, 2572, 7278, 122, 82, 74, 0] combine_rows_per_src=[7532, 8192, 8192, 7974, 1606, 7082, 8160, 5808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51294 payload_rows_remote=43102 dup_rows=14242 dup_rows_remote=14242 dedup_ratio=0.217316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 72, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2735, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 1456, 2203, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 60, 8192, 8190, 8192, 8192, 4556, 5720] dup_rows_per_dst=[110, 0, 0, 0, 13014, 18, 8, 1092] combine_rows_per_src=[8192, 60, 8192, 8190, 8192, 8192, 4556, 5720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=189939712 dispatch_meta_bytes=217080 combine_payload_bytes=189939712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190156792 ingress_bytes=208814080 link_bytes=208814080 token_bytes=380096504 weight_movement_bytes=18874368 total_nvlink_bytes=398970872 weight_to_token_ratio=0.049657 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000753772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57532 payload_rows_remote=49524 dup_rows=8004 dup_rows_remote=7938 dedup_ratio=0.122131 router_tokens_per_expert=[0, 0, 25, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 3977, 0, 0, 1, 0, 0, 0, 0, 2129, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4028, 3880, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 42, 0, 0, 0, 0, 0, 461, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 3824, 3813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8142, 4648, 7820, 4884, 8190, 8008, 7648, 8192] dup_rows_per_dst=[44, 38, 0, 0, 1006, 66, 92, 6758] combine_rows_per_src=[8142, 4648, 7820, 4884, 8190, 8008, 7648, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56566 payload_rows_remote=50694 dup_rows=8970 dup_rows_remote=8414 dedup_ratio=0.136871 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 201, 4060, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 0, 0, 262, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 3648, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8114, 8192, 8180, 5872, 8188, 7662, 2170] dup_rows_per_dst=[20, 3286, 0, 0, 556, 4918, 188, 2] combine_rows_per_src=[8188, 8114, 8192, 8180, 5872, 8188, 7662, 2170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49126 payload_rows_remote=40934 dup_rows=16410 dup_rows_remote=9150 dedup_ratio=0.250397 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4091, 6, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 649, 0, 0, 4000, 1, 0, 1, 0, 62, 8, 3857, 1, 0, 0, 0, 0, 0, 1872, 0, 150, 0, 0, 0, 0, 2329, 0, 3, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3555, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4027, 0, 3670, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8026, 1756, 20, 7590, 7196, 8156, 8192] dup_rows_per_dst=[10, 84, 12, 2, 1284, 128, 7630, 7260] combine_rows_per_src=[8190, 8026, 1756, 20, 7590, 7196, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58032 payload_rows_remote=50054 dup_rows=7504 dup_rows_remote=4284 dedup_ratio=0.114502 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2093, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 1416, 0, 0, 0, 4096, 0, 0, 41, 0, 13, 0, 682, 0, 0, 2026, 0, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 736, 9, 4086, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7330, 8192, 6720, 7978, 7970, 8156, 3514, 8172] dup_rows_per_dst=[2, 16, 1466, 3220, 2432, 0, 346, 22] combine_rows_per_src=[7330, 8192, 6720, 7978, 7970, 8156, 3514, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=189865984 dispatch_meta_bytes=229224 combine_payload_bytes=189865984 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=190095208 ingress_bytes=208740352 link_bytes=208740352 token_bytes=379961192 weight_movement_bytes=18874368 total_nvlink_bytes=398835560 weight_to_token_ratio=0.049674 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.072917 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000047346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=176545792 dispatch_meta_bytes=229376 combine_payload_bytes=176545792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176775168 ingress_bytes=195420160 link_bytes=195420160 token_bytes=353320960 weight_movement_bytes=18874368 total_nvlink_bytes=372195328 weight_to_token_ratio=0.053420 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872419820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=202850304 dispatch_meta_bytes=229848 combine_payload_bytes=202850304 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=203080152 ingress_bytes=240599040 link_bytes=240599040 token_bytes=405930456 weight_movement_bytes=37748736 total_nvlink_bytes=443679192 weight_to_token_ratio=0.092993 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.305301619e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=207642624 dispatch_meta_bytes=236432 combine_payload_bytes=207642624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=207879056 ingress_bytes=264265728 link_bytes=264265728 token_bytes=415521680 weight_movement_bytes=56623104 total_nvlink_bytes=472144784 weight_to_token_ratio=0.136270 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.532064179e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=167665664 dispatch_meta_bytes=200336 combine_payload_bytes=167665664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=167866000 ingress_bytes=186540032 link_bytes=186540032 token_bytes=335531664 weight_movement_bytes=18874368 total_nvlink_bytes=354406032 weight_to_token_ratio=0.056252 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.787334803e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=205021184 dispatch_meta_bytes=217352 combine_payload_bytes=205021184 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=205238536 ingress_bytes=299393024 link_bytes=299393024 token_bytes=410259720 weight_movement_bytes=94371840 total_nvlink_bytes=504631560 weight_to_token_ratio=0.230030 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.868636645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59336 payload_rows_remote=51144 dup_rows=6200 dup_rows_remote=3840 dedup_ratio=0.094604 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1073, 0, 0, 0, 99, 0, 0, 0, 0, 0, 4096, 3151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1579, 0, 1110, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5474, 6302, 8192, 8192, 8192, 6600] dup_rows_per_dst=[2360, 0, 28, 0, 3806, 6, 0, 0] combine_rows_per_src=[8192, 8192, 5474, 6302, 8192, 8192, 8192, 6600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51344 payload_rows_remote=43152 dup_rows=14192 dup_rows_remote=14156 dedup_ratio=0.216553 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 73, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 4009, 0, 4095, 0, 0, 0, 17, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1294, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5112, 5572, 7898, 0, 8190, 8188, 8192, 8192] dup_rows_per_dst=[576, 48, 84, 0, 2, 10832, 36, 2614] combine_rows_per_src=[5112, 5572, 7898, 0, 8190, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=209485824 dispatch_meta_bytes=219936 combine_payload_bytes=209485824 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=209705760 ingress_bytes=266108928 link_bytes=266108928 token_bytes=419191584 weight_movement_bytes=56623104 total_nvlink_bytes=475814688 weight_to_token_ratio=0.135077 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.549724814e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54436 payload_rows_remote=46518 dup_rows=11100 dup_rows_remote=6210 dedup_ratio=0.169373 router_tokens_per_expert=[0, 0, 0, 180, 0, 0, 0, 77, 0, 0, 0, 537, 0, 0, 0, 0, 0, 8, 0, 1, 4053, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 3071, 3, 0, 92, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3958, 4080, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4, 0, 0, 3864, 0, 0, 0, 0, 41, 13, 0, 0, 653, 0, 0, 0, 4040, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 3, 84, 0, 0, 246, 0, 2, 0, 680, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1261, 2699, 12, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0] payload_rows_per_dst=[8114, 7918, 8192, 7814, 5018, 8192, 6508, 2680] dup_rows_per_dst=[1390, 4890, 180, 3572, 120, 654, 270, 24] combine_rows_per_src=[8114, 7918, 8192, 7814, 5018, 8192, 6508, 2680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=176750592 dispatch_meta_bytes=229232 combine_payload_bytes=176750592 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=176979824 ingress_bytes=195624960 link_bytes=195624960 token_bytes=353730416 weight_movement_bytes=18874368 total_nvlink_bytes=372604784 weight_to_token_ratio=0.053358 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874382113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60004 payload_rows_remote=52258 dup_rows=5532 dup_rows_remote=1690 dedup_ratio=0.084412 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1495, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 4095, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 1780, 0, 84, 0, 2551, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7746, 6294, 5106, 8190, 8188, 8096] dup_rows_per_dst=[766, 0, 3842, 0, 924, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7746, 6294, 5106, 8190, 8188, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61904 payload_rows_remote=53742 dup_rows=3632 dup_rows_remote=3504 dedup_ratio=0.055420 router_tokens_per_expert=[353, 8, 0, 4000, 0, 0, 0, 13, 1, 53, 0, 24, 155, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 3582, 0, 0, 0, 10, 0, 2, 0, 0, 0, 3936, 4073, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4080, 0, 3617, 0, 0, 0, 0, 1, 0, 2, 0, 222, 0, 0, 160, 0, 15, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 4050, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6518, 7890, 7222, 8148, 7584, 8162, 8190, 8190] dup_rows_per_dst=[1056, 1676, 0, 2, 102, 128, 44, 624] combine_rows_per_src=[6518, 7890, 7222, 8148, 7584, 8162, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61582 payload_rows_remote=55262 dup_rows=3954 dup_rows_remote=2540 dedup_ratio=0.060333 router_tokens_per_expert=[201, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 0, 92, 0, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6188, 8158, 6320, 8192, 8148, 8192] dup_rows_per_dst=[406, 0, 18, 2114, 1414, 2, 0, 0] combine_rows_per_src=[8192, 8192, 6188, 8158, 6320, 8192, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59360 payload_rows_remote=51236 dup_rows=6176 dup_rows_remote=2710 dedup_ratio=0.094238 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 180, 4086, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 4091, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 2360, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7106, 8188, 8096, 7992, 4736, 8188, 6930, 8124] dup_rows_per_dst=[54, 360, 288, 1172, 0, 812, 24, 3466] combine_rows_per_src=[7106, 8188, 8096, 7992, 4736, 8188, 6930, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64198 payload_rows_remote=56042 dup_rows=1338 dup_rows_remote=1338 dedup_ratio=0.020416 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 14, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8156, 8186, 8002, 8190, 7088] dup_rows_per_dst=[0, 0, 0, 0, 32, 22, 1284, 0] combine_rows_per_src=[8192, 8192, 8192, 8156, 8186, 8002, 8190, 7088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=190537728 dispatch_meta_bytes=210912 combine_payload_bytes=190537728 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=190748640 ingress_bytes=247160832 link_bytes=247160832 token_bytes=381286368 weight_movement_bytes=56623104 total_nvlink_bytes=437909472 weight_to_token_ratio=0.148505 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368173482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=214048768 dispatch_meta_bytes=215792 combine_payload_bytes=214048768 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=214264560 ingress_bytes=308420608 link_bytes=308420608 token_bytes=428313328 weight_movement_bytes=94371840 total_nvlink_bytes=522685168 weight_to_token_ratio=0.220334 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.955134513e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=220127232 dispatch_meta_bytes=228984 combine_payload_bytes=220127232 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=220356216 ingress_bytes=257875968 link_bytes=257875968 token_bytes=440483448 weight_movement_bytes=37748736 total_nvlink_bytes=478232184 weight_to_token_ratio=0.085698 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.470840642e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=226353152 dispatch_meta_bytes=231208 combine_payload_bytes=226353152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226584360 ingress_bytes=282976256 link_bytes=282976256 token_bytes=452937512 weight_movement_bytes=56623104 total_nvlink_bytes=509560616 weight_to_token_ratio=0.125013 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=12800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.711339252e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=209862656 dispatch_meta_bytes=215784 combine_payload_bytes=209862656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210078440 ingress_bytes=247611392 link_bytes=247611392 token_bytes=419941096 weight_movement_bytes=37748736 total_nvlink_bytes=457689832 weight_to_token_ratio=0.089891 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=13696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372490526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=229548032 dispatch_meta_bytes=229520 combine_payload_bytes=229548032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229777552 ingress_bytes=286171136 link_bytes=286171136 token_bytes=459325584 weight_movement_bytes=56623104 total_nvlink_bytes=515948688 weight_to_token_ratio=0.123274 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.741951020e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50204 payload_rows_remote=42012 dup_rows=15332 dup_rows_remote=13612 dedup_ratio=0.233948 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 4072, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 34, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8162, 8192, 8192, 1074, 8184, 8192, 16] dup_rows_per_dst=[1720, 766, 11414, 1424, 0, 6, 2, 0] combine_rows_per_src=[8192, 8162, 8192, 8192, 1074, 8184, 8192, 16] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57160 payload_rows_remote=48976 dup_rows=8376 dup_rows_remote=8370 dedup_ratio=0.127808 router_tokens_per_expert=[0, 0, 0, 4077, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 20, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[28, 8172, 8166, 8192, 8192, 8182, 8184, 8044] dup_rows_per_dst=[0, 98, 0, 8188, 44, 30, 6, 10] combine_rows_per_src=[28, 8172, 8166, 8192, 8192, 8182, 8184, 8044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=172081152 dispatch_meta_bytes=222496 combine_payload_bytes=172081152 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=172303648 ingress_bytes=209829888 link_bytes=209829888 token_bytes=344384800 weight_movement_bytes=37748736 total_nvlink_bytes=382133536 weight_to_token_ratio=0.109612 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.010486744e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=200605696 dispatch_meta_bytes=229384 combine_payload_bytes=200605696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200835080 ingress_bytes=219480064 link_bytes=219480064 token_bytes=401440776 weight_movement_bytes=18874368 total_nvlink_bytes=420315144 weight_to_token_ratio=0.047017 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102949982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55490 payload_rows_remote=54194 dup_rows=10046 dup_rows_remote=10046 dedup_ratio=0.153290 router_tokens_per_expert=[4005, 0, 0, 3681, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 630, 0, 0, 0, 16, 2, 2, 0, 0, 0, 0, 0, 139, 0, 0, 3937, 3713, 21, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 237, 0, 52, 6, 9, 209, 0, 0, 0, 0, 0, 0, 1108, 3969, 0, 4061, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 4032, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 5, 611, 0, 0, 58, 0, 0, 0] payload_rows_per_dst=[8188, 1296, 8178, 8052, 5520, 7874, 8190, 8192] dup_rows_per_dst=[7824, 0, 272, 720, 52, 0, 66, 1112] combine_rows_per_src=[8188, 1296, 8178, 8052, 5520, 7874, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63440 payload_rows_remote=56522 dup_rows=2096 dup_rows_remote=2094 dedup_ratio=0.031982 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3421, 0, 0, 0, 0, 4015, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 484, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7968, 8188, 6918, 8192, 7762, 8192, 8184, 8036] dup_rows_per_dst=[0, 0, 2, 212, 0, 968, 910, 4] combine_rows_per_src=[7968, 8188, 6918, 8192, 7762, 8192, 8184, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56520 payload_rows_remote=51916 dup_rows=9016 dup_rows_remote=8708 dedup_ratio=0.137573 router_tokens_per_expert=[0, 0, 8, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 26, 0, 0, 0, 0, 0, 9, 0, 1, 0, 3948, 0, 0, 0, 3864, 0, 0, 0, 20, 4096, 0, 0, 433, 0, 0, 0, 0, 0, 0, 120, 0, 1, 3, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2079, 0, 1, 0, 0, 0, 0, 393, 0, 3346, 0, 426, 0, 4, 0, 0, 0, 0, 0, 1898, 12, 3700, 396, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3710, 0, 0, 53, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6722, 7978, 8166, 7898, 5370, 4604, 7590, 8192] dup_rows_per_dst=[0, 72, 468, 204, 430, 308, 6, 7528] combine_rows_per_src=[6722, 7978, 8166, 7898, 5370, 4604, 7590, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57640 payload_rows_remote=49448 dup_rows=7896 dup_rows_remote=82 dedup_ratio=0.120483 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3830, 0, 0, 0, 0, 1145, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 7668, 8192, 8192, 6740, 8192, 2296] dup_rows_per_dst=[0, 0, 10, 48, 7814, 22, 0, 2] combine_rows_per_src=[8192, 8168, 7668, 8192, 8192, 6740, 8192, 2296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57108 payload_rows_remote=48952 dup_rows=8428 dup_rows_remote=8428 dedup_ratio=0.128601 router_tokens_per_expert=[0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4, 2390, 0, 0, 0, 0, 0, 3777, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4078, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4634, 8192, 4780, 7400, 8172, 7590, 8184, 8156] dup_rows_per_dst=[0, 26, 8, 26, 7658, 4, 706, 0] combine_rows_per_src=[4634, 8192, 4780, 7400, 8172, 7590, 8184, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57256 payload_rows_remote=49064 dup_rows=8280 dup_rows_remote=8280 dedup_ratio=0.126343 router_tokens_per_expert=[14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 2, 8156, 8138, 8192] dup_rows_per_dst=[90, 0, 8190, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 2, 8156, 8138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=221978624 dispatch_meta_bytes=256960 combine_payload_bytes=221978624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222235584 ingress_bytes=240852992 link_bytes=240852992 token_bytes=444214208 weight_movement_bytes=18874368 total_nvlink_bytes=463088576 weight_to_token_ratio=0.042489 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307734862e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=212647936 dispatch_meta_bytes=242496 combine_payload_bytes=212647936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=212890432 ingress_bytes=269271040 link_bytes=269271040 token_bytes=425538368 weight_movement_bytes=56623104 total_nvlink_bytes=482161472 weight_to_token_ratio=0.133062 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580022616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231514112 dispatch_meta_bytes=234464 combine_payload_bytes=231514112 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231748576 ingress_bytes=307011584 link_bytes=307011584 token_bytes=463262688 weight_movement_bytes=75497472 total_nvlink_bytes=538760160 weight_to_token_ratio=0.162969 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.941633939e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=202539008 dispatch_meta_bytes=198120 combine_payload_bytes=202539008 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=202737128 ingress_bytes=278036480 link_bytes=278036480 token_bytes=405276136 weight_movement_bytes=75497472 total_nvlink_bytes=480773608 weight_to_token_ratio=0.186286 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.664008749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49920 payload_rows_remote=42030 dup_rows=15616 dup_rows_remote=12710 dedup_ratio=0.238281 router_tokens_per_expert=[0, 1451, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 512, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 0, 3945, 3047, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7890, 8192, 8174, 8192, 7428, 118, 1766, 8160] dup_rows_per_dst=[2906, 8094, 1410, 176, 3012, 0, 0, 18] combine_rows_per_src=[7890, 8192, 8174, 8192, 7428, 118, 1766, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=200507392 dispatch_meta_bytes=229520 combine_payload_bytes=200507392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200736912 ingress_bytes=219381760 link_bytes=219381760 token_bytes=401244304 weight_movement_bytes=18874368 total_nvlink_bytes=420118672 weight_to_token_ratio=0.047040 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102008082e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=200966144 dispatch_meta_bytes=229376 combine_payload_bytes=200966144 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=201195520 ingress_bytes=238714880 link_bytes=238714880 token_bytes=402161664 weight_movement_bytes=37748736 total_nvlink_bytes=439910400 weight_to_token_ratio=0.093865 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287248525e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53430 payload_rows_remote=45238 dup_rows=12106 dup_rows_remote=12088 dedup_ratio=0.184723 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 3086, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 0, 0, 0, 0, 0, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6354, 7426, 6774, 120, 8190, 8182, 8192, 8192] dup_rows_per_dst=[74, 10, 0, 0, 0, 3878, 18, 8126] combine_rows_per_src=[6354, 7426, 6774, 120, 8190, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=172154880 dispatch_meta_bytes=218960 combine_payload_bytes=172154880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172373840 ingress_bytes=191029248 link_bytes=191029248 token_bytes=344528720 weight_movement_bytes=18874368 total_nvlink_bytes=363403088 weight_to_token_ratio=0.054783 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.830348262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=185294848 dispatch_meta_bytes=229304 combine_payload_bytes=185294848 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=185524152 ingress_bytes=260792320 link_bytes=260792320 token_bytes=370819000 weight_movement_bytes=75497472 total_nvlink_bytes=446316472 weight_to_token_ratio=0.203597 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=13056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.498783692e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53868 payload_rows_remote=45838 dup_rows=11668 dup_rows_remote=11660 dedup_ratio=0.178040 router_tokens_per_expert=[0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4013, 0, 0, 20, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3, 0, 217, 0, 0, 4072, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 3587, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 3568, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 719, 0, 453, 1, 0, 0, 2109, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[7124, 8030, 1568, 8192, 7854, 5526, 7392, 8182] dup_rows_per_dst=[24, 8, 32, 496, 6070, 6, 20, 5012] combine_rows_per_src=[7124, 8030, 1568, 8192, 7854, 5526, 7392, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51068 payload_rows_remote=43746 dup_rows=14468 dup_rows_remote=13100 dedup_ratio=0.220764 router_tokens_per_expert=[0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 2, 0, 0, 0, 0, 0, 4054, 8, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1578, 50, 0, 519, 0, 289, 0, 25, 871, 0, 21, 10, 0, 2610, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 0, 0, 0, 2, 11, 0, 0, 0, 0, 0, 0, 6, 2175, 0, 2822, 719, 0, 0, 0, 0, 0, 0, 4, 3508, 737, 0, 0, 0, 0, 55, 0, 41, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3404, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0] payload_rows_per_dst=[7946, 6, 6948, 7032, 6842, 7322, 6822, 8150] dup_rows_per_dst=[2, 0, 8438, 0, 2638, 1368, 1992, 30] combine_rows_per_src=[7946, 6, 6948, 7032, 6842, 7322, 6822, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46740 payload_rows_remote=38548 dup_rows=18796 dup_rows_remote=5806 dedup_ratio=0.286804 router_tokens_per_expert=[0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 2264, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3895, 438, 0, 0, 0, 0, 0, 0, 0, 1, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2575, 0, 0, 0, 0, 0, 0, 0, 3501, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 7598, 8192, 7584, 5150, 2824, 2054, 5162] dup_rows_per_dst=[906, 4544, 12990, 2, 0, 6, 0, 348] combine_rows_per_src=[8176, 7598, 8192, 7584, 5150, 2824, 2054, 5162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57828 payload_rows_remote=49636 dup_rows=7708 dup_rows_remote=608 dedup_ratio=0.117615 router_tokens_per_expert=[0, 3775, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3106, 4095, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2915, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3434, 0, 4096, 115, 0, 0, 0, 0, 0, 0, 0, 4085, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7550, 6212, 5876, 6342, 8192, 8174, 7292, 8190] dup_rows_per_dst=[4, 0, 58, 0, 7100, 380, 0, 166] combine_rows_per_src=[7550, 6212, 5876, 6342, 8192, 8174, 7292, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56078 payload_rows_remote=48120 dup_rows=9458 dup_rows_remote=9248 dedup_ratio=0.144318 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 1, 0, 2, 3, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 13, 3, 0, 3958, 0, 0, 0, 4083, 0, 0, 0, 0, 3708, 0, 0, 9, 0, 53, 0, 0, 2, 249, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3341, 0, 3949, 0, 98, 0, 0, 0, 0, 0, 0, 2, 4073, 262, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 120, 1, 0, 39, 0, 0, 0, 22, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7950, 8184, 536, 6878, 8188, 8192, 7958] dup_rows_per_dst=[118, 22, 106, 0, 2, 7528, 1472, 210] combine_rows_per_src=[8192, 7950, 8184, 536, 6878, 8188, 8192, 7958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=187752448 dispatch_meta_bytes=229992 combine_payload_bytes=187752448 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=187982440 ingress_bytes=225501184 link_bytes=225501184 token_bytes=375734888 weight_movement_bytes=37748736 total_nvlink_bytes=413483624 weight_to_token_ratio=0.100466 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.160641392e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63128 payload_rows_remote=54936 dup_rows=2408 dup_rows_remote=382 dedup_ratio=0.036743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3810, 0, 0, 0, 2, 3944, 0, 0, 15, 0, 0, 0, 0, 0, 811, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 164, 63, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 2, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 7650, 8192, 8192, 6408, 8188, 8118] dup_rows_per_dst=[50, 0, 4, 2026, 214, 0, 0, 114] combine_rows_per_src=[8192, 8188, 7650, 8192, 8192, 6408, 8188, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=179183616 dispatch_meta_bytes=227384 combine_payload_bytes=179183616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=179411000 ingress_bytes=235806720 link_bytes=235806720 token_bytes=358594616 weight_movement_bytes=56623104 total_nvlink_bytes=415217720 weight_to_token_ratio=0.157903 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=21248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259383967e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157892608 dispatch_meta_bytes=177416 combine_payload_bytes=157892608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158070024 ingress_bytes=176766976 link_bytes=176766976 token_bytes=315962632 weight_movement_bytes=18874368 total_nvlink_bytes=334837000 weight_to_token_ratio=0.059736 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.086806 gemm_active_groups=14 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693694189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=203309056 dispatch_meta_bytes=200976 combine_payload_bytes=203309056 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=203510032 ingress_bytes=278806528 link_bytes=278806528 token_bytes=406819088 weight_movement_bytes=75497472 total_nvlink_bytes=482316560 weight_to_token_ratio=0.185580 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.671386970e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45518 payload_rows_remote=37328 dup_rows=20018 dup_rows_remote=12764 dedup_ratio=0.305450 router_tokens_per_expert=[0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 874, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 22, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 7318, 5362, 686, 44, 7536] dup_rows_per_dst=[7254, 0, 12758, 4, 0, 0, 0, 2] combine_rows_per_src=[8190, 8190, 8192, 7318, 5362, 686, 44, 7536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=197099520 dispatch_meta_bytes=229472 combine_payload_bytes=197099520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197328992 ingress_bytes=215973888 link_bytes=215973888 token_bytes=394428512 weight_movement_bytes=18874368 total_nvlink_bytes=413302880 weight_to_token_ratio=0.047852 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.069355529e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=225017856 dispatch_meta_bytes=221272 combine_payload_bytes=225017856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225239128 ingress_bytes=262766592 link_bytes=262766592 token_bytes=450256984 weight_movement_bytes=37748736 total_nvlink_bytes=488005720 weight_to_token_ratio=0.083838 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.517700195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46536 payload_rows_remote=43662 dup_rows=19000 dup_rows_remote=19000 dedup_ratio=0.289917 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 4096, 3798, 0, 0, 1, 0, 0, 634, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 31, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 1455, 0, 0, 0, 0, 0] payload_rows_per_dst=[6250, 7900, 7916, 8192, 1406, 8192, 2874, 3806] dup_rows_per_dst=[2120, 0, 32, 4906, 0, 10446, 0, 1496] combine_rows_per_src=[6250, 7900, 7916, 8192, 1406, 8192, 2874, 3806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=152895488 dispatch_meta_bytes=200368 combine_payload_bytes=152895488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=153095856 ingress_bytes=171769856 link_bytes=171769856 token_bytes=305991344 weight_movement_bytes=18874368 total_nvlink_bytes=324865712 weight_to_token_ratio=0.061683 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.645814244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57170 payload_rows_remote=49646 dup_rows=8366 dup_rows_remote=7966 dedup_ratio=0.127655 router_tokens_per_expert=[0, 10, 0, 0, 0, 0, 0, 607, 11, 0, 626, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 112, 0, 0, 52, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2444, 0, 0, 0, 21, 0, 58, 1, 0, 0, 2682, 0, 0, 0, 0, 2021, 0, 0, 0, 0, 3643, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 2397, 4039, 0, 0, 0, 0, 0, 0, 0, 4058, 4075, 0, 0, 0, 0, 0, 0, 0, 104, 2, 0, 1, 0, 0, 3561, 1, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6496, 7524, 8050, 8190, 8078, 8192, 7198, 3442] dup_rows_per_dst=[818, 400, 3036, 3248, 548, 172, 94, 50] combine_rows_per_src=[6496, 7524, 8050, 8190, 8078, 8192, 7198, 3442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=178839552 dispatch_meta_bytes=250648 combine_payload_bytes=178839552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179090200 ingress_bytes=197713920 link_bytes=197713920 token_bytes=357929752 weight_movement_bytes=18874368 total_nvlink_bytes=376804120 weight_to_token_ratio=0.052732 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.894397500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56060 payload_rows_remote=47878 dup_rows=9476 dup_rows_remote=9350 dedup_ratio=0.144592 router_tokens_per_expert=[3, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 642, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 1065, 4, 0, 0, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 2, 0, 1, 0, 3972, 0, 56, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 1, 120, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 19, 0, 0, 0, 4061, 0, 0, 1661, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 21, 0, 0, 0] payload_rows_per_dst=[5640, 8076, 3402, 6280, 8134, 8182, 8160, 8186] dup_rows_per_dst=[0, 0, 20, 32, 80, 126, 3336, 5882] combine_rows_per_src=[5640, 8076, 3402, 6280, 8134, 8182, 8160, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48956 payload_rows_remote=40768 dup_rows=16580 dup_rows_remote=12916 dedup_ratio=0.252991 router_tokens_per_expert=[0, 189, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 0, 0, 0, 0, 0, 0, 4, 0, 3881, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[430, 7756, 8188, 6468, 8158, 7768, 1996, 8192] dup_rows_per_dst=[22, 0, 3664, 4226, 6370, 6, 0, 2292] combine_rows_per_src=[430, 7756, 8188, 6468, 8158, 7768, 1996, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53676 payload_rows_remote=45484 dup_rows=11860 dup_rows_remote=11550 dedup_ratio=0.180969 router_tokens_per_expert=[0, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 36, 0, 169, 0, 2698, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 1, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 0, 3466, 0, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5490, 8190, 5396, 7336, 8192, 7862, 8158, 3052] dup_rows_per_dst=[0, 72, 0, 2754, 310, 3480, 5194, 50] combine_rows_per_src=[5490, 8190, 5396, 7336, 8192, 7862, 8158, 3052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=203350016 dispatch_meta_bytes=230448 combine_payload_bytes=203350016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=203580464 ingress_bytes=259973120 link_bytes=259973120 token_bytes=406930480 weight_movement_bytes=56623104 total_nvlink_bytes=463553584 weight_to_token_ratio=0.139147 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=9216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.490934521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47726 payload_rows_remote=39676 dup_rows=17810 dup_rows_remote=13756 dedup_ratio=0.271759 router_tokens_per_expert=[0, 2152, 0, 0, 0, 0, 1, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 3068, 2977, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 251, 0, 4078, 4079, 0, 0, 0, 0, 0, 0, 0, 4095, 2618, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4406, 3868, 8, 8188, 6836, 8192, 8178, 8050] dup_rows_per_dst=[0, 2, 0, 300, 532, 6652, 6270, 4054] combine_rows_per_src=[4406, 3868, 8, 8188, 6836, 8192, 8178, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56436 payload_rows_remote=48250 dup_rows=9100 dup_rows_remote=9100 dedup_ratio=0.138855 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3188, 0, 1393, 3568, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 863, 0, 0, 0, 0, 0, 0, 0, 4087, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7396, 2018, 8186, 6430, 8176, 8082, 7956] dup_rows_per_dst=[1896, 1766, 28, 0, 1704, 92, 0, 3614] combine_rows_per_src=[8192, 7396, 2018, 8186, 6430, 8176, 8082, 7956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=196108288 dispatch_meta_bytes=228912 combine_payload_bytes=196108288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196337200 ingress_bytes=214982656 link_bytes=214982656 token_bytes=392445488 weight_movement_bytes=18874368 total_nvlink_bytes=411319856 weight_to_token_ratio=0.048094 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059858032e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=166985728 dispatch_meta_bytes=214736 combine_payload_bytes=166985728 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=167200464 ingress_bytes=242483200 link_bytes=242483200 token_bytes=334186192 weight_movement_bytes=75497472 total_nvlink_bytes=409683664 weight_to_token_ratio=0.225914 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.323354713e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=186302464 dispatch_meta_bytes=228136 combine_payload_bytes=186302464 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=186530600 ingress_bytes=242925568 link_bytes=242925568 token_bytes=372833064 weight_movement_bytes=56623104 total_nvlink_bytes=429456168 weight_to_token_ratio=0.151873 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.327593266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61536 payload_rows_remote=55482 dup_rows=4000 dup_rows_remote=3518 dedup_ratio=0.061035 router_tokens_per_expert=[0, 348, 0, 0, 0, 326, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 2408, 0, 2896, 0, 0, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 193, 0, 0, 1, 0, 0, 0, 0, 0, 185, 0, 0, 4057, 4096, 648, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6054, 8192, 8192, 6708, 8192, 7892, 8192, 8114] dup_rows_per_dst=[482, 54, 8, 1250, 154, 354, 402, 1296] combine_rows_per_src=[6054, 8192, 8192, 6708, 8192, 7892, 8192, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=162512896 dispatch_meta_bytes=213728 combine_payload_bytes=162512896 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=162726624 ingress_bytes=200261632 link_bytes=200261632 token_bytes=325239520 weight_movement_bytes=37748736 total_nvlink_bytes=362988256 weight_to_token_ratio=0.116064 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.918808423e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53300 payload_rows_remote=46386 dup_rows=12236 dup_rows_remote=11904 dedup_ratio=0.186707 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 1, 12, 0, 0, 0, 0, 3060, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 0, 50, 0, 337, 0, 0, 0, 0, 152, 0, 0, 0, 1694, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 3028, 3, 3979, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7132, 8192, 6190, 344, 8186, 8150, 6914, 8192] dup_rows_per_dst=[0, 30, 192, 60, 3246, 364, 332, 8012] combine_rows_per_src=[7132, 8192, 6190, 344, 8186, 8150, 6914, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=197632000 dispatch_meta_bytes=229400 combine_payload_bytes=197632000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197861400 ingress_bytes=216506368 link_bytes=216506368 token_bytes=395493400 weight_movement_bytes=18874368 total_nvlink_bytes=414367768 weight_to_token_ratio=0.047724 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.074457490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=227254272 dispatch_meta_bytes=236000 combine_payload_bytes=227254272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227490272 ingress_bytes=246128640 link_bytes=246128640 token_bytes=454744544 weight_movement_bytes=18874368 total_nvlink_bytes=473618912 weight_to_token_ratio=0.041505 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358283526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=189997056 dispatch_meta_bytes=233160 combine_payload_bytes=189997056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190230216 ingress_bytes=208871424 link_bytes=208871424 token_bytes=380227272 weight_movement_bytes=18874368 total_nvlink_bytes=399101640 weight_to_token_ratio=0.049640 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.001303214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44936 payload_rows_remote=40670 dup_rows=20600 dup_rows_remote=20380 dedup_ratio=0.314331 router_tokens_per_expert=[0, 4065, 0, 0, 0, 0, 0, 0, 220, 0, 7, 273, 0, 329, 0, 0, 0, 26, 0, 0, 4, 0, 0, 171, 0, 18, 4, 0, 0, 0, 2019, 1, 2884, 0, 2425, 0, 0, 0, 1316, 0, 3783, 16, 0, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3249, 3, 1, 0, 0, 0, 0, 3, 0, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1438, 0, 0, 0, 4, 0, 0, 0, 162, 0, 2, 0, 0, 0, 0, 3993, 1, 0, 2, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 10, 23, 19, 298, 0, 0, 0, 2021] payload_rows_per_dst=[8192, 4266, 8054, 5812, 7000, 2944, 3602, 5066] dup_rows_per_dst=[6312, 220, 11774, 0, 946, 0, 2, 1346] combine_rows_per_src=[8192, 4266, 8054, 5812, 7000, 2944, 3602, 5066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52240 payload_rows_remote=51274 dup_rows=13296 dup_rows_remote=13258 dedup_ratio=0.202881 router_tokens_per_expert=[0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 12, 554, 83, 3963, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 2190, 0, 0, 0, 0, 1083, 0, 3389, 0, 0, 0, 0, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 1, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 3, 11, 0, 0, 0, 0, 2268, 0, 0, 1, 0, 8, 1174, 5, 0, 89, 163, 0, 0, 0, 0, 100, 0, 0, 105, 4040, 0, 4069, 3718, 832, 0, 0, 0, 0, 11, 0, 0] payload_rows_per_dst=[7978, 8190, 8150, 7130, 7510, 966, 6046, 6270] dup_rows_per_dst=[0, 792, 1104, 6196, 32, 38, 4908, 226] combine_rows_per_src=[7978, 8190, 8150, 7130, 7510, 966, 6046, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53986 payload_rows_remote=45794 dup_rows=11550 dup_rows_remote=11384 dedup_ratio=0.176239 router_tokens_per_expert=[0, 4043, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 80, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2302, 0, 0, 0, 18, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 0, 31, 3705, 19, 0, 0, 0, 0, 0, 0, 0, 665, 0, 790, 37, 2798, 119, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8086, 6330, 8192, 5002, 8192, 8148, 8192, 1844] dup_rows_per_dst=[842, 76, 166, 62, 0, 1330, 9018, 56] combine_rows_per_src=[8086, 6330, 8192, 5002, 8192, 8148, 8192, 1844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55556 payload_rows_remote=47364 dup_rows=9980 dup_rows_remote=9980 dedup_ratio=0.152283 router_tokens_per_expert=[0, 3940, 0, 0, 0, 0, 0, 147, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 3094, 0, 356, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 67, 4096, 3, 3656, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7966, 8192, 8192, 7258, 8192, 8168, 7454, 134] dup_rows_per_dst=[1062, 4594, 26, 866, 0, 3386, 40, 6] combine_rows_per_src=[7966, 8192, 8192, 7258, 8192, 8168, 7454, 134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=166584320 dispatch_meta_bytes=244200 combine_payload_bytes=166584320 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=166828520 ingress_bytes=185458688 link_bytes=185458688 token_bytes=333412840 weight_movement_bytes=18874368 total_nvlink_bytes=352287208 weight_to_token_ratio=0.056610 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.776973897e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54250 payload_rows_remote=48056 dup_rows=11286 dup_rows_remote=10604 dedup_ratio=0.172211 router_tokens_per_expert=[0, 4070, 0, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 2391, 0, 162, 0, 1494, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 1, 0, 667, 0, 0, 0, 39, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4080, 0, 0, 2, 0, 0, 1864, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0] payload_rows_per_dst=[8192, 7790, 2988, 8190, 4540, 8192, 8164, 6194] dup_rows_per_dst=[312, 4546, 0, 4, 4, 1598, 4140, 682] combine_rows_per_src=[8192, 7790, 2988, 8190, 4540, 8192, 8164, 6194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=210018304 dispatch_meta_bytes=258128 combine_payload_bytes=210018304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210276432 ingress_bytes=228892672 link_bytes=228892672 token_bytes=420294736 weight_movement_bytes=18874368 total_nvlink_bytes=439169104 weight_to_token_ratio=0.044907 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.193136961e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=187572224 dispatch_meta_bytes=228712 combine_payload_bytes=187572224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=187800936 ingress_bytes=206446592 link_bytes=206446592 token_bytes=375373160 weight_movement_bytes=18874368 total_nvlink_bytes=394247528 weight_to_token_ratio=0.050282 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.978069666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55994 payload_rows_remote=47802 dup_rows=9542 dup_rows_remote=2022 dedup_ratio=0.145599 router_tokens_per_expert=[0, 2879, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 0, 434, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 3, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6052, 8192, 868, 8192, 8192, 8190, 8146, 8162] dup_rows_per_dst=[42, 0, 22, 7520, 2, 14, 0, 1942] combine_rows_per_src=[6052, 8192, 868, 8192, 8192, 8190, 8146, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=194002944 dispatch_meta_bytes=229376 combine_payload_bytes=194002944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194232320 ingress_bytes=212877312 link_bytes=212877312 token_bytes=388235264 weight_movement_bytes=18874368 total_nvlink_bytes=407109632 weight_to_token_ratio=0.048616 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039685661e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54618 payload_rows_remote=46426 dup_rows=10918 dup_rows_remote=7984 dedup_ratio=0.166595 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 118, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 4006, 2754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5356, 8192, 7812, 2222, 6684, 8192, 7968] dup_rows_per_dst=[0, 236, 0, 0, 48, 2838, 2934, 4862] combine_rows_per_src=[8192, 5356, 8192, 7812, 2222, 6684, 8192, 7968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=196837376 dispatch_meta_bytes=234640 combine_payload_bytes=196837376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197072016 ingress_bytes=215711744 link_bytes=215711744 token_bytes=393909392 weight_movement_bytes=18874368 total_nvlink_bytes=412783760 weight_to_token_ratio=0.047916 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.086806 gemm_active_groups=13 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.066843794e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44090 payload_rows_remote=39024 dup_rows=21446 dup_rows_remote=20228 dedup_ratio=0.327240 router_tokens_per_expert=[0, 1, 792, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2201, 0, 1405, 0, 0, 0, 0, 0, 0, 828, 48, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 157, 85, 0, 1524, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 4085, 0, 0, 0, 2, 0, 0, 51, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 1124, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5066, 4180, 8192, 8192, 7808, 8192, 2354, 106] dup_rows_per_dst=[1218, 286, 18, 11424, 238, 8258, 4, 0] combine_rows_per_src=[5066, 4180, 8192, 8192, 7808, 8192, 2354, 106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=195796992 dispatch_meta_bytes=199296 combine_payload_bytes=195796992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195996288 ingress_bytes=214671360 link_bytes=214671360 token_bytes=391793280 weight_movement_bytes=18874368 total_nvlink_bytes=410667648 weight_to_token_ratio=0.048174 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=23936 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.056875347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=190160896 dispatch_meta_bytes=217640 combine_payload_bytes=190160896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190378536 ingress_bytes=209035264 link_bytes=209035264 token_bytes=380539432 weight_movement_bytes=18874368 total_nvlink_bytes=399413800 weight_to_token_ratio=0.049599 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.002873048e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50032 payload_rows_remote=41842 dup_rows=15504 dup_rows_remote=7716 dedup_ratio=0.236572 router_tokens_per_expert=[0, 3317, 0, 1, 0, 8, 0, 0, 0, 2, 20, 0, 0, 0, 0, 0, 0, 69, 0, 3704, 0, 10, 0, 0, 298, 0, 0, 0, 0, 0, 14, 0, 174, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 201, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 17, 0, 0, 0, 0, 40, 0, 133, 0, 0, 16, 0, 0, 7, 122, 0, 3940, 0, 0, 0, 3894, 0, 3824, 4047, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[6752, 8190, 6240, 4814, 8192, 7928, 7912, 4] dup_rows_per_dst=[2506, 7788, 672, 22, 4110, 0, 406, 0] combine_rows_per_src=[6752, 8190, 6240, 4814, 8192, 7928, 7912, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=237008 combine_payload_bytes=159842304 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=160079312 ingress_bytes=216465408 link_bytes=216465408 token_bytes=319921616 weight_movement_bytes=56623104 total_nvlink_bytes=376544720 weight_to_token_ratio=0.176991 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.074065032e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44996 payload_rows_remote=44826 dup_rows=20540 dup_rows_remote=20540 dedup_ratio=0.313416 router_tokens_per_expert=[0, 0, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 6, 0, 4067, 0, 3921, 0, 293, 0, 0, 0, 0, 621, 4065, 0, 0, 1074, 0, 25, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 218, 9, 0, 0, 0, 0, 0, 0, 30, 22, 0, 0, 0, 0, 0, 0, 0, 115, 0, 123, 2928, 0, 0, 24, 0, 0, 6, 0, 0, 0, 0, 78, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3326, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 574, 0] payload_rows_per_dst=[6314, 8192, 8164, 552, 6182, 170, 8146, 7276] dup_rows_per_dst=[166, 11660, 7944, 6, 210, 0, 0, 554] combine_rows_per_src=[6314, 8192, 8164, 552, 6182, 170, 8146, 7276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=38886 payload_rows_remote=30708 dup_rows=26650 dup_rows_remote=26158 dedup_ratio=0.406647 router_tokens_per_expert=[0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3789, 0, 5, 0, 0, 1187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 18, 0, 0, 1752, 0, 0, 4016, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 4, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7730, 2384, 8178, 8192, 3706, 8192, 36, 468] dup_rows_per_dst=[7064, 0, 492, 7570, 0, 11524, 0, 0] combine_rows_per_src=[7730, 2384, 8178, 8192, 3706, 8192, 36, 468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55144 payload_rows_remote=47064 dup_rows=10392 dup_rows_remote=10264 dedup_ratio=0.158569 router_tokens_per_expert=[0, 2, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3616, 0, 8, 0, 0, 0, 0, 3, 0, 2214, 4042, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 930, 0, 0, 0, 5, 0, 0, 573, 0, 0, 82, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1966, 0, 0, 176, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 4450, 8192, 8184, 8080, 2008, 8106, 8010] dup_rows_per_dst=[4938, 0, 26, 12, 128, 1172, 3952, 164] combine_rows_per_src=[8114, 4450, 8192, 8184, 8080, 2008, 8106, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=171384832 dispatch_meta_bytes=198232 combine_payload_bytes=171384832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171583064 ingress_bytes=190259200 link_bytes=190259200 token_bytes=342967896 weight_movement_bytes=18874368 total_nvlink_bytes=361842264 weight_to_token_ratio=0.055032 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.822970041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44204 payload_rows_remote=36012 dup_rows=21332 dup_rows_remote=1174 dedup_ratio=0.325500 router_tokens_per_expert=[0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7, 141, 0, 132, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1106, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 0, 0, 3288, 4081, 0, 1793, 4059, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 2126, 8170, 8160, 4062, 2310, 3070, 8192] dup_rows_per_dst=[0, 0, 2, 274, 0, 92, 806, 20158] combine_rows_per_src=[8114, 2126, 8170, 8160, 4062, 2310, 3070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55620 payload_rows_remote=47432 dup_rows=9916 dup_rows_remote=9210 dedup_ratio=0.151306 router_tokens_per_expert=[0, 3700, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 4083, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 2, 0, 4074, 0, 0, 1, 4096, 0, 3590, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7474, 8192, 8192, 8188, 8192, 8192, 7188, 2] dup_rows_per_dst=[20, 1062, 6, 706, 0, 8120, 2, 0] combine_rows_per_src=[7474, 8192, 8192, 8188, 8192, 8192, 7188, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=125779968 dispatch_meta_bytes=227464 combine_payload_bytes=125779968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=126007432 ingress_bytes=144654336 link_bytes=144654336 token_bytes=251787400 weight_movement_bytes=18874368 total_nvlink_bytes=270661768 weight_to_token_ratio=0.074962 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.386006673e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=183607296 dispatch_meta_bytes=261464 combine_payload_bytes=183607296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=183868760 ingress_bytes=221356032 link_bytes=221356032 token_bytes=367476056 weight_movement_bytes=37748736 total_nvlink_bytes=405224792 weight_to_token_ratio=0.102724 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=12800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.120924585e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=192774144 dispatch_meta_bytes=229312 combine_payload_bytes=192774144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=193003456 ingress_bytes=230522880 link_bytes=230522880 token_bytes=385777600 weight_movement_bytes=37748736 total_nvlink_bytes=423526336 weight_to_token_ratio=0.097851 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=20864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.208756812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54604 payload_rows_remote=47114 dup_rows=10932 dup_rows_remote=10932 dedup_ratio=0.166809 router_tokens_per_expert=[51, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[828, 8192, 5398, 8192, 8192, 8130, 7490, 8182] dup_rows_per_dst=[98, 5496, 0, 1936, 2492, 910, 0, 0] combine_rows_per_src=[828, 8192, 5398, 8192, 8192, 8130, 7490, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51144 payload_rows_remote=42952 dup_rows=14392 dup_rows_remote=14390 dedup_ratio=0.219604 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3008, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 1, 0, 0, 0, 0, 2719, 0, 0, 0, 0, 2697, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8186, 8192, 6822, 4528, 1734, 8096, 5394] dup_rows_per_dst=[2, 5168, 2562, 1258, 0, 44, 5358, 0] combine_rows_per_src=[8192, 8186, 8192, 6822, 4528, 1734, 8096, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=147505152 dispatch_meta_bytes=148744 combine_payload_bytes=147505152 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=147653896 ingress_bytes=241876992 link_bytes=241876992 token_bytes=295159048 weight_movement_bytes=94371840 total_nvlink_bytes=389530888 weight_to_token_ratio=0.319732 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=8192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.317546326e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=194281472 dispatch_meta_bytes=226568 combine_payload_bytes=194281472 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=194508040 ingress_bytes=250904576 link_bytes=250904576 token_bytes=388789512 weight_movement_bytes=56623104 total_nvlink_bytes=445412616 weight_to_token_ratio=0.145639 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404044194e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=192978944 dispatch_meta_bytes=232184 combine_payload_bytes=192978944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193211128 ingress_bytes=211853312 link_bytes=211853312 token_bytes=386190072 weight_movement_bytes=18874368 total_nvlink_bytes=405064440 weight_to_token_ratio=0.048873 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.029874197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54810 payload_rows_remote=46618 dup_rows=10726 dup_rows_remote=10264 dedup_ratio=0.163666 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3856, 0, 4062, 0, 0, 0, 15, 0, 0, 0, 3589, 38, 2, 194, 0, 0, 16, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4061, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 434, 0, 0, 0, 0, 0, 21, 0, 0, 18, 3004, 0, 0, 0, 0, 2, 1704, 0, 0, 0, 3, 12, 0, 0, 3328, 0, 4084, 0, 0, 56, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 34, 0, 0, 17, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18] payload_rows_per_dst=[7732, 8192, 8190, 8122, 6404, 7938, 8152, 80] dup_rows_per_dst=[0, 462, 104, 78, 562, 2160, 7358, 2] combine_rows_per_src=[7732, 8192, 8190, 8122, 6404, 7938, 8152, 80] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=175931392 dispatch_meta_bytes=229368 combine_payload_bytes=175931392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176160760 ingress_bytes=194805760 link_bytes=194805760 token_bytes=352092152 weight_movement_bytes=18874368 total_nvlink_bytes=370966520 weight_to_token_ratio=0.053606 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56308 payload_rows_remote=48116 dup_rows=9228 dup_rows_remote=8632 dedup_ratio=0.140808 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 92, 0, 371, 0, 3964, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7448, 8192, 8192, 8166, 7928, 8190, 0] dup_rows_per_dst=[6, 4, 596, 8496, 0, 2, 124, 0] combine_rows_per_src=[8192, 7448, 8192, 8192, 8166, 7928, 8190, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57272 payload_rows_remote=49258 dup_rows=8264 dup_rows_remote=8008 dedup_ratio=0.126099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3962, 58, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 122, 8014, 8192, 8176, 8192] dup_rows_per_dst=[6, 0, 90, 0, 256, 7910, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 122, 8014, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51802 payload_rows_remote=43950 dup_rows=13734 dup_rows_remote=10712 dedup_ratio=0.209564 router_tokens_per_expert=[87, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4064, 20, 100, 0, 0, 0, 0, 0, 3498, 18, 364, 0, 3745, 0, 0, 3, 0, 3973, 0, 18, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 1687, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 3991] payload_rows_per_dst=[8160, 34, 8192, 7018, 8192, 7852, 4188, 8166] dup_rows_per_dst=[3844, 0, 58, 988, 5820, 3022, 0, 2] combine_rows_per_src=[8160, 34, 8192, 7018, 8192, 7852, 4188, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55528 payload_rows_remote=50276 dup_rows=10008 dup_rows_remote=10008 dedup_ratio=0.152710 router_tokens_per_expert=[0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4015, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 853, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 231, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 4087, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 13, 0] payload_rows_per_dst=[8192, 8032, 8192, 5252, 8192, 1284, 8192, 8192] dup_rows_per_dst=[32, 26, 4, 0, 1698, 0, 130, 8118] combine_rows_per_src=[8192, 8032, 8192, 5252, 8192, 1284, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51200 payload_rows_remote=43008 dup_rows=14336 dup_rows_remote=6336 dedup_ratio=0.218750 router_tokens_per_expert=[0, 0, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 4017, 0, 0, 0, 0, 1193, 0, 0, 0, 1, 3857, 0, 264, 44, 0, 0, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 47, 0, 14, 0, 0, 0, 0, 4, 0, 3059, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 139, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 2948, 2482, 7916, 6122, 7216, 8168, 8192] dup_rows_per_dst=[748, 436, 0, 4, 32, 4522, 594, 8000] combine_rows_per_src=[8156, 2948, 2482, 7916, 6122, 7216, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=190947328 dispatch_meta_bytes=227528 combine_payload_bytes=190947328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=191174856 ingress_bytes=247570432 link_bytes=247570432 token_bytes=382122184 weight_movement_bytes=56623104 total_nvlink_bytes=438745288 weight_to_token_ratio=0.148181 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.079861 gemm_active_groups=14 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372098067e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=197083136 dispatch_meta_bytes=226992 combine_payload_bytes=197083136 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=197310128 ingress_bytes=234831872 link_bytes=234831872 token_bytes=394393264 weight_movement_bytes=37748736 total_nvlink_bytes=432142000 weight_to_token_ratio=0.095713 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250043453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=201760768 dispatch_meta_bytes=229064 combine_payload_bytes=201760768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=201989832 ingress_bytes=239509504 link_bytes=239509504 token_bytes=403750600 weight_movement_bytes=37748736 total_nvlink_bytes=441499336 weight_to_token_ratio=0.093495 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294862221e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57652 payload_rows_remote=50456 dup_rows=7884 dup_rows_remote=7884 dedup_ratio=0.120300 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 2487, 0, 0, 0, 34, 3546, 0, 0, 56, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 2091, 227, 0, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 2, 5, 1086, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8004, 5102, 7730, 5460, 8076, 7196, 7904] dup_rows_per_dst=[862, 0, 52, 4002, 224, 780, 0, 1964] combine_rows_per_src=[8180, 8004, 5102, 7730, 5460, 8076, 7196, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=180019200 dispatch_meta_bytes=218648 combine_payload_bytes=180019200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=180237848 ingress_bytes=198893568 link_bytes=198893568 token_bytes=360257048 weight_movement_bytes=18874368 total_nvlink_bytes=379131416 weight_to_token_ratio=0.052391 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.905700307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=205930496 dispatch_meta_bytes=241136 combine_payload_bytes=205930496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206171632 ingress_bytes=224804864 link_bytes=224804864 token_bytes=412102128 weight_movement_bytes=18874368 total_nvlink_bytes=430976496 weight_to_token_ratio=0.045800 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.153969596e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55694 payload_rows_remote=47502 dup_rows=9842 dup_rows_remote=9820 dedup_ratio=0.150177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 633, 0, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 874, 0, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 4096, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7560, 8172, 7166, 28] dup_rows_per_dst=[22, 68, 6970, 1586, 328, 308, 560, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7560, 8172, 7166, 28] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=176160768 dispatch_meta_bytes=197376 combine_payload_bytes=176160768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=176358144 ingress_bytes=213909504 link_bytes=213909504 token_bytes=352518912 weight_movement_bytes=37748736 total_nvlink_bytes=390267648 weight_to_token_ratio=0.107083 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049575617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=206667776 dispatch_meta_bytes=233360 combine_payload_bytes=206667776 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=206901136 ingress_bytes=244416512 link_bytes=244416512 token_bytes=413568912 weight_movement_bytes=37748736 total_nvlink_bytes=451317648 weight_to_token_ratio=0.091276 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341878758e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=194568192 dispatch_meta_bytes=229288 combine_payload_bytes=194568192 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=194797480 ingress_bytes=232316928 link_bytes=232316928 token_bytes=389365672 weight_movement_bytes=37748736 total_nvlink_bytes=427114408 weight_to_token_ratio=0.096949 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225946497e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56288 payload_rows_remote=48144 dup_rows=9248 dup_rows_remote=1106 dedup_ratio=0.141113 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 36, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 154, 0, 0, 3958, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8144, 8192, 8192, 7196, 8190, 0] dup_rows_per_dst=[0, 276, 8142, 2, 340, 0, 488, 0] combine_rows_per_src=[8192, 8182, 8144, 8192, 8192, 7196, 8190, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60138 payload_rows_remote=51946 dup_rows=5398 dup_rows_remote=5394 dedup_ratio=0.082367 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 318, 0, 0, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 187, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 155, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4096, 43, 4032, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5966, 8190, 8192, 8192, 8154, 5186, 8066] dup_rows_per_dst=[0, 1996, 1312, 976, 4, 2, 1024, 84] combine_rows_per_src=[8192, 5966, 8190, 8192, 8192, 8154, 5186, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51032 payload_rows_remote=42840 dup_rows=14504 dup_rows_remote=10746 dedup_ratio=0.221313 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 19, 0, 3559, 71, 0, 0, 0, 102, 0, 18, 0, 0, 0, 0, 25, 0, 4012, 10, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3807, 0, 47, 0, 0, 0, 0, 0, 2827, 53, 636, 0, 2005, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 5, 8, 0, 57, 3396, 0, 0, 0, 0, 20, 0, 332, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 3414, 3544, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 4019] payload_rows_per_dst=[8076, 8192, 7616, 8152, 6906, 4684, 566, 6840] dup_rows_per_dst=[7122, 3758, 0, 3472, 44, 32, 76, 0] combine_rows_per_src=[8076, 8192, 7616, 8152, 6906, 4684, 566, 6840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54968 payload_rows_remote=47620 dup_rows=10568 dup_rows_remote=10568 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 72, 0, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 9, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 502, 55, 0, 0, 0, 0, 2, 0, 1114, 0, 0, 3, 4042, 0, 3867, 0, 0, 0, 66, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 280, 3, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 10, 81, 102, 4007, 2912, 3662, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 480, 6318, 8192, 8088, 7348, 8192, 8192] dup_rows_per_dst=[68, 0, 188, 1912, 48, 0, 386, 7966] combine_rows_per_src=[8158, 480, 6318, 8192, 8088, 7348, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63350 payload_rows_remote=55332 dup_rows=2186 dup_rows_remote=1948 dedup_ratio=0.033356 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3548, 4004, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 203, 0, 332, 195, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8188, 8188, 8018, 7782, 7266, 8192, 7526] dup_rows_per_dst=[6, 0, 0, 238, 492, 6, 1444, 0] combine_rows_per_src=[8190, 8188, 8188, 8018, 7782, 7266, 8192, 7526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=197197824 dispatch_meta_bytes=197000 combine_payload_bytes=197197824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197394824 ingress_bytes=216072192 link_bytes=216072192 token_bytes=394592648 weight_movement_bytes=18874368 total_nvlink_bytes=413467016 weight_to_token_ratio=0.047833 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.070297430e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48100 payload_rows_remote=39910 dup_rows=17436 dup_rows_remote=6998 dedup_ratio=0.266052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4095, 0, 2, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4, 0, 0, 786, 0, 140, 0, 4062, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 37, 0, 0, 1909, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 0, 0, 134, 0, 2, 0, 0, 0, 1140, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079] payload_rows_per_dst=[0, 8192, 5270, 4064, 7896, 8182, 6306, 8190] dup_rows_per_dst=[0, 10, 200, 618, 0, 3844, 2326, 10438] combine_rows_per_src=[0, 8192, 5270, 4064, 7896, 8182, 6306, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=212770816 dispatch_meta_bytes=229360 combine_payload_bytes=212770816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213000176 ingress_bytes=231645184 link_bytes=231645184 token_bytes=425770992 weight_movement_bytes=18874368 total_nvlink_bytes=444645360 weight_to_token_ratio=0.044330 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219510177e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=175472640 dispatch_meta_bytes=214344 combine_payload_bytes=175472640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175686984 ingress_bytes=194347008 link_bytes=194347008 token_bytes=351159624 weight_movement_bytes=18874368 total_nvlink_bytes=370033992 weight_to_token_ratio=0.053749 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50130 payload_rows_remote=41956 dup_rows=15406 dup_rows_remote=7792 dedup_ratio=0.235077 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 3996, 0, 971, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3846, 510, 0, 0, 0, 0, 0, 0, 0, 4088, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7992, 308, 8096, 1046, 8176, 8174, 8146] dup_rows_per_dst=[156, 1942, 2, 0, 0, 5692, 7614, 0] combine_rows_per_src=[8192, 7992, 308, 8096, 1046, 8176, 8174, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=195051520 dispatch_meta_bytes=232752 combine_payload_bytes=195051520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195284272 ingress_bytes=213925888 link_bytes=213925888 token_bytes=390335792 weight_movement_bytes=18874368 total_nvlink_bytes=409210160 weight_to_token_ratio=0.048354 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049732601e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54596 payload_rows_remote=46404 dup_rows=10940 dup_rows_remote=10934 dedup_ratio=0.166931 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4045, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 3229, 0, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0] payload_rows_per_dst=[8192, 8192, 8184, 8188, 8188, 8192, 3226, 2234] dup_rows_per_dst=[6, 3356, 1576, 46, 5186, 770, 0, 0] combine_rows_per_src=[8192, 8192, 8184, 8188, 8188, 8192, 3226, 2234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=226639872 dispatch_meta_bytes=229120 combine_payload_bytes=226639872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226868992 ingress_bytes=264388608 link_bytes=264388608 token_bytes=453508864 weight_movement_bytes=37748736 total_nvlink_bytes=491257600 weight_to_token_ratio=0.083237 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.533241555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=163471360 dispatch_meta_bytes=187632 combine_payload_bytes=163471360 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=163658992 ingress_bytes=238968832 link_bytes=238968832 token_bytes=327130352 weight_movement_bytes=75497472 total_nvlink_bytes=402627824 weight_to_token_ratio=0.230787 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.289681768e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=171851776 dispatch_meta_bytes=198992 combine_payload_bytes=171851776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172050768 ingress_bytes=190726144 link_bytes=190726144 token_bytes=343902544 weight_movement_bytes=18874368 total_nvlink_bytes=362776912 weight_to_token_ratio=0.054883 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.827444069e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=190070784 dispatch_meta_bytes=229352 combine_payload_bytes=190070784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190300136 ingress_bytes=208945152 link_bytes=208945152 token_bytes=380370920 weight_movement_bytes=18874368 total_nvlink_bytes=399245288 weight_to_token_ratio=0.049621 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.002009639e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57150 payload_rows_remote=48958 dup_rows=8386 dup_rows_remote=8386 dedup_ratio=0.127960 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1474, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 0, 0, 94, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0] payload_rows_per_dst=[8192, 8192, 8192, 3080, 8192, 8192, 8192, 4918] dup_rows_per_dst=[4, 0, 0, 0, 2534, 0, 5848, 0] combine_rows_per_src=[8192, 8192, 8192, 3080, 8192, 8192, 8192, 4918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65316 payload_rows_remote=57124 dup_rows=220 dup_rows_remote=220 dedup_ratio=0.003357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 7976] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 218, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 7976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56444 payload_rows_remote=48256 dup_rows=9092 dup_rows_remote=824 dedup_ratio=0.138733 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 55, 167, 0, 101, 0, 0, 0, 4011, 0, 1, 0, 0, 153, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 6, 4, 82, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 3937, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4030, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4037, 39, 0, 0, 0, 0, 0, 56, 0, 430, 1, 3208, 0, 4044, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8058, 8188, 716, 7874, 8086, 8086, 8094, 7342] dup_rows_per_dst=[174, 8268, 170, 0, 0, 4, 472, 4] combine_rows_per_src=[8058, 8188, 716, 7874, 8086, 8086, 8094, 7342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57070 payload_rows_remote=51268 dup_rows=8466 dup_rows_remote=8466 dedup_ratio=0.129181 router_tokens_per_expert=[0, 0, 0, 18, 0, 0, 5, 8, 0, 1043, 0, 5, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 3998, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3666, 0, 0, 0, 0, 20, 376, 0, 1, 0, 0, 0, 55, 4045, 0, 0, 73, 0, 0, 0, 0, 0, 0, 14, 0, 438, 0, 0, 4080, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 2901, 0, 0, 0, 0, 0, 0, 57, 0, 0, 16, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 3, 0, 0, 3997, 0] payload_rows_per_dst=[8174, 8192, 7338, 8192, 8192, 5802, 3154, 8026] dup_rows_per_dst=[2118, 22, 0, 948, 874, 0, 18, 4486] combine_rows_per_src=[8174, 8192, 7338, 8192, 8192, 5802, 3154, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=200531968 dispatch_meta_bytes=229376 combine_payload_bytes=200531968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200761344 ingress_bytes=238280704 link_bytes=238280704 token_bytes=401293312 weight_movement_bytes=37748736 total_nvlink_bytes=439042048 weight_to_token_ratio=0.094068 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283088464e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64798 payload_rows_remote=56606 dup_rows=738 dup_rows_remote=738 dedup_ratio=0.011261 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 67, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7454, 8192] dup_rows_per_dst=[0, 2, 726, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7454, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=233979904 dispatch_meta_bytes=229376 combine_payload_bytes=233979904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234209280 ingress_bytes=252854272 link_bytes=252854272 token_bytes=468189184 weight_movement_bytes=18874368 total_nvlink_bytes=487063552 weight_to_token_ratio=0.040314 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422725222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53596 payload_rows_remote=45404 dup_rows=11940 dup_rows_remote=9632 dedup_ratio=0.182190 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2378, 0, 1, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 4091, 0, 4069, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 197, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4048, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 8114, 3950, 3910, 8190, 5600, 7544, 8192] dup_rows_per_dst=[0, 4594, 0, 124, 8, 4282, 624, 2308] combine_rows_per_src=[8096, 8114, 3950, 3910, 8190, 5600, 7544, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=197656576 dispatch_meta_bytes=196320 combine_payload_bytes=197656576 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=197852896 ingress_bytes=254279680 link_bytes=254279680 token_bytes=395509472 weight_movement_bytes=56623104 total_nvlink_bytes=452132576 weight_to_token_ratio=0.143165 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.436382780e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=209993728 dispatch_meta_bytes=238936 combine_payload_bytes=209993728 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=210232664 ingress_bytes=247742464 link_bytes=247742464 token_bytes=420226392 weight_movement_bytes=37748736 total_nvlink_bytes=457975128 weight_to_token_ratio=0.089830 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373746393e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49414 payload_rows_remote=41300 dup_rows=16122 dup_rows_remote=9642 dedup_ratio=0.246002 router_tokens_per_expert=[0, 3998, 0, 0, 0, 0, 0, 0, 439, 0, 80, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 105, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 3773, 0, 0, 3, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8162, 8192, 210, 1400, 8192, 8192, 8114, 6952] dup_rows_per_dst=[1246, 46, 0, 0, 54, 8296, 6480, 0] combine_rows_per_src=[8162, 8192, 210, 1400, 8192, 8192, 8114, 6952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42556 payload_rows_remote=34364 dup_rows=22980 dup_rows_remote=19440 dedup_ratio=0.350647 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 113, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 4096, 0, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 286, 4816, 8144, 2, 4732] dup_rows_per_dst=[3540, 904, 12708, 14, 0, 5814, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 286, 4816, 8144, 2, 4732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=231858176 dispatch_meta_bytes=229376 combine_payload_bytes=231858176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232087552 ingress_bytes=288481280 link_bytes=288481280 token_bytes=463945728 weight_movement_bytes=56623104 total_nvlink_bytes=520568832 weight_to_token_ratio=0.122047 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.764085683e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=185974784 dispatch_meta_bytes=220144 combine_payload_bytes=185974784 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=186194928 ingress_bytes=261472256 link_bytes=261472256 token_bytes=372169712 weight_movement_bytes=75497472 total_nvlink_bytes=447667184 weight_to_token_ratio=0.202858 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.505298505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=169164800 dispatch_meta_bytes=203768 combine_payload_bytes=169164800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=169368568 ingress_bytes=206913536 link_bytes=206913536 token_bytes=338533368 weight_movement_bytes=37748736 total_nvlink_bytes=376282104 weight_to_token_ratio=0.111507 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.982543694e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=140754944 dispatch_meta_bytes=215216 combine_payload_bytes=140754944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=140970160 ingress_bytes=178503680 link_bytes=178503680 token_bytes=281725104 weight_movement_bytes=37748736 total_nvlink_bytes=319473840 weight_to_token_ratio=0.133991 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.710334432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52048 payload_rows_remote=45268 dup_rows=13488 dup_rows_remote=10080 dedup_ratio=0.205811 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 0, 0, 4095, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6064, 8192, 3482, 8190, 6780, 5670, 5478, 8192] dup_rows_per_dst=[0, 2, 16, 7800, 3408, 46, 2216, 0] combine_rows_per_src=[6064, 8192, 3482, 8190, 6780, 5670, 5478, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=36362 dup_rows=20982 dup_rows_remote=18952 dedup_ratio=0.320160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 4095, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 199, 1507, 316, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 18, 8192, 8160, 3608, 8192] dup_rows_per_dst=[0, 32, 2030, 0, 8198, 4634, 436, 5652] combine_rows_per_src=[0, 8192, 8192, 18, 8192, 8160, 3608, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51044 payload_rows_remote=43016 dup_rows=14492 dup_rows_remote=13898 dedup_ratio=0.221130 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 2, 0, 0, 0, 0, 33, 0, 0, 72, 0, 0, 3711, 1, 0, 489, 0, 0, 0, 5, 0, 1, 1, 78, 1995, 33, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4058, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 195, 0, 0, 0, 2, 0, 0, 3, 4, 0, 0, 15, 0, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 4001, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 3491, 0, 14, 2885, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 608, 0, 1931, 2] payload_rows_per_dst=[7944, 8028, 8190, 602, 5918, 8002, 8178, 4182] dup_rows_per_dst=[288, 594, 4180, 0, 1774, 0, 6748, 908] combine_rows_per_src=[7944, 8028, 8190, 602, 5918, 8002, 8178, 4182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57160 payload_rows_remote=49878 dup_rows=8376 dup_rows_remote=5760 dedup_ratio=0.127808 router_tokens_per_expert=[0, 0, 0, 13, 0, 50, 0, 3941, 0, 4038, 0, 0, 41, 0, 0, 14, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 884, 0, 0, 0, 27, 0, 0, 578, 0, 0, 37, 0, 0, 0, 2979, 0, 0, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 40, 0, 0, 209, 0, 16, 3717, 894, 1205, 0, 0, 5, 0, 0, 0, 0, 0, 2620, 0, 5, 0, 84, 20, 0, 48, 129, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 375, 2, 0, 0, 0, 43, 0] payload_rows_per_dst=[8100, 7892, 2924, 6716, 7872, 7282, 8192, 8182] dup_rows_per_dst=[212, 0, 146, 4304, 24, 2616, 532, 542] combine_rows_per_src=[8100, 7892, 2924, 6716, 7872, 7282, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=185417728 dispatch_meta_bytes=221392 combine_payload_bytes=185417728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185639120 ingress_bytes=204292096 link_bytes=204292096 token_bytes=371056848 weight_movement_bytes=18874368 total_nvlink_bytes=389931216 weight_to_token_ratio=0.050867 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.957426346e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62044 payload_rows_remote=53852 dup_rows=3492 dup_rows_remote=3490 dedup_ratio=0.053284 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 1550, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0] payload_rows_per_dst=[8192, 7970, 8192, 8192, 4960, 8160, 8192, 8186] dup_rows_per_dst=[0, 0, 264, 2, 0, 0, 3106, 120] combine_rows_per_src=[8192, 7970, 8192, 8192, 4960, 8160, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=148938752 dispatch_meta_bytes=221256 combine_payload_bytes=148938752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=149160008 ingress_bytes=167813120 link_bytes=167813120 token_bytes=298098760 weight_movement_bytes=18874368 total_nvlink_bytes=316973128 weight_to_token_ratio=0.063316 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=13184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.607902747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=176193536 dispatch_meta_bytes=227656 combine_payload_bytes=176193536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176421192 ingress_bytes=195067904 link_bytes=195067904 token_bytes=352614728 weight_movement_bytes=18874368 total_nvlink_bytes=371489096 weight_to_token_ratio=0.053527 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53560 payload_rows_remote=45368 dup_rows=11976 dup_rows_remote=11920 dedup_ratio=0.182739 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3206, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 48, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 4057, 0, 0, 4087, 0, 55, 70, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 27, 0] payload_rows_per_dst=[6036, 6636, 0, 8184, 8162, 8192, 8158, 8192] dup_rows_per_dst=[2, 2, 0, 8066, 0, 1852, 1998, 56] combine_rows_per_src=[6036, 6636, 0, 8184, 8162, 8192, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=204300288 dispatch_meta_bytes=222552 combine_payload_bytes=204300288 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=204522840 ingress_bytes=260923392 link_bytes=260923392 token_bytes=408823128 weight_movement_bytes=56623104 total_nvlink_bytes=465446232 weight_to_token_ratio=0.138503 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.500039560e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52162 payload_rows_remote=50262 dup_rows=13374 dup_rows_remote=13374 dedup_ratio=0.204071 router_tokens_per_expert=[0, 4060, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 3798, 0, 122, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 2010, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 1756, 8192, 7652, 8192, 8192, 1900, 8090] dup_rows_per_dst=[12, 2, 32, 1248, 516, 420, 0, 11144] combine_rows_per_src=[8188, 1756, 8192, 7652, 8192, 8192, 1900, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=220577792 dispatch_meta_bytes=229368 combine_payload_bytes=220577792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220807160 ingress_bytes=239452160 link_bytes=239452160 token_bytes=441384952 weight_movement_bytes=18874368 total_nvlink_bytes=460259320 weight_to_token_ratio=0.042762 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294312779e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46742 payload_rows_remote=38550 dup_rows=18794 dup_rows_remote=17322 dedup_ratio=0.286774 router_tokens_per_expert=[0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 104, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4057, 0, 0, 0, 10, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4037, 210, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5398, 474, 0, 8102] dup_rows_per_dst=[1472, 5838, 8118, 740, 0, 0, 0, 2626] combine_rows_per_src=[8192, 8192, 8192, 8192, 5398, 474, 0, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=185827328 dispatch_meta_bytes=229152 combine_payload_bytes=185827328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186056480 ingress_bytes=204701696 link_bytes=204701696 token_bytes=371883808 weight_movement_bytes=18874368 total_nvlink_bytes=390758176 weight_to_token_ratio=0.050753 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.961350932e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56896 payload_rows_remote=48840 dup_rows=8640 dup_rows_remote=8640 dedup_ratio=0.131836 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7964, 8192, 8192, 8192, 8056, 7578, 8192, 530] dup_rows_per_dst=[1004, 692, 0, 68, 0, 6876, 0, 0] combine_rows_per_src=[7964, 8192, 8192, 8192, 8056, 7578, 8192, 530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=205873152 dispatch_meta_bytes=254544 combine_payload_bytes=205873152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206127696 ingress_bytes=224747520 link_bytes=224747520 token_bytes=412000848 weight_movement_bytes=18874368 total_nvlink_bytes=430875216 weight_to_token_ratio=0.045811 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.086806 gemm_active_groups=13 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.153420154e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157900800 dispatch_meta_bytes=223488 combine_payload_bytes=157900800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158124288 ingress_bytes=176775168 link_bytes=176775168 token_bytes=316025088 weight_movement_bytes=18874368 total_nvlink_bytes=334899456 weight_to_token_ratio=0.059724 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693772681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48340 payload_rows_remote=40148 dup_rows=17196 dup_rows_remote=16924 dedup_ratio=0.262390 router_tokens_per_expert=[0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 39, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8188, 8192, 8192, 7086, 298, 8192, 0] dup_rows_per_dst=[8072, 8848, 272, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 7086, 298, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50786 payload_rows_remote=42594 dup_rows=14750 dup_rows_remote=14488 dedup_ratio=0.225067 router_tokens_per_expert=[0, 0, 0, 0, 0, 3605, 978, 2407, 0, 1520, 5, 0, 2, 2, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 118, 1, 0, 0, 0, 0, 0, 0, 58, 1, 3963, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 7, 0, 196, 34, 0, 0, 4036, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4086, 76, 0, 0, 0, 0, 0, 0, 0, 14, 0, 13, 3185, 0, 504, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0] payload_rows_per_dst=[8054, 8192, 8190, 7752, 6772, 326, 6636, 4864] dup_rows_per_dst=[4964, 262, 8408, 0, 358, 14, 744, 0] combine_rows_per_src=[8054, 8192, 8190, 7752, 6772, 326, 6636, 4864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48852 payload_rows_remote=47174 dup_rows=16684 dup_rows_remote=16606 dedup_ratio=0.254578 router_tokens_per_expert=[0, 0, 0, 65, 0, 266, 0, 0, 0, 0, 19, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 94, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 23, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 13, 0, 0, 3957, 0, 1, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 630, 19, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 134, 16, 0, 0, 0, 0, 0, 0, 1, 0, 0, 391, 2669, 3970, 1967, 4058, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 49, 3445] payload_rows_per_dst=[8192, 5116, 2526, 6936, 8062, 1678, 8192, 8150] dup_rows_per_dst=[640, 1702, 0, 0, 2444, 78, 6078, 5742] combine_rows_per_src=[8192, 5116, 2526, 6936, 8062, 1678, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=200048640 dispatch_meta_bytes=229920 combine_payload_bytes=200048640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200278560 ingress_bytes=218923008 link_bytes=218923008 token_bytes=400327200 weight_movement_bytes=18874368 total_nvlink_bytes=419201568 weight_to_token_ratio=0.047147 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.097612546e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52112 payload_rows_remote=44088 dup_rows=13424 dup_rows_remote=10476 dedup_ratio=0.204834 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 3962, 0, 0, 0, 0, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 665, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 0, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 61, 21, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8034, 8182, 3134, 8024, 8192, 170, 8186, 8190] dup_rows_per_dst=[2324, 4, 16, 2948, 8082, 0, 50, 0] combine_rows_per_src=[8034, 8182, 3134, 8024, 8192, 170, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=164446208 dispatch_meta_bytes=228288 combine_payload_bytes=164446208 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=164674496 ingress_bytes=202194944 link_bytes=202194944 token_bytes=329120704 weight_movement_bytes=37748736 total_nvlink_bytes=366869440 weight_to_token_ratio=0.114696 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.937332467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=174465024 dispatch_meta_bytes=228328 combine_payload_bytes=174465024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=174693352 ingress_bytes=212213760 link_bytes=212213760 token_bytes=349158376 weight_movement_bytes=37748736 total_nvlink_bytes=386907112 weight_to_token_ratio=0.108114 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.033327833e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=193224704 dispatch_meta_bytes=255120 combine_payload_bytes=193224704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=193479824 ingress_bytes=230973440 link_bytes=230973440 token_bytes=386704528 weight_movement_bytes=37748736 total_nvlink_bytes=424453264 weight_to_token_ratio=0.097616 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=15872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.213073856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55356 payload_rows_remote=47164 dup_rows=10180 dup_rows_remote=8330 dedup_ratio=0.155334 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4022, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 0, 70, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 3550, 0, 1433, 0, 12, 0, 0, 0, 0, 0, 0, 1358, 12, 0, 0, 0, 0, 0, 123, 2, 2, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 385, 0, 613, 0, 0, 0, 0] payload_rows_per_dst=[7822, 7228, 7578, 6080, 2076, 8192, 8188, 8192] dup_rows_per_dst=[0, 2, 16, 0, 0, 7536, 776, 1850] combine_rows_per_src=[7822, 7228, 7578, 6080, 2076, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=40968 payload_rows_remote=32782 dup_rows=24568 dup_rows_remote=22410 dedup_ratio=0.374878 router_tokens_per_expert=[0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 7, 4093, 0, 0, 0, 49, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 2, 0, 0, 4078, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[66, 8192, 5350, 32, 8192, 2758, 8186, 8192] dup_rows_per_dst=[0, 6096, 8, 0, 7982, 142, 2158, 8182] combine_rows_per_src=[66, 8192, 5350, 32, 8192, 2758, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47482 payload_rows_remote=39290 dup_rows=18054 dup_rows_remote=12846 dedup_ratio=0.275482 router_tokens_per_expert=[0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 0, 0, 3959, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1993, 2, 0, 0, 0, 0, 1, 0, 1790, 0, 0, 91, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8014, 7918, 2780, 3992, 202] dup_rows_per_dst=[5208, 2132, 5276, 5166, 0, 272, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8014, 7918, 2780, 3992, 202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=180584448 dispatch_meta_bytes=218256 combine_payload_bytes=180584448 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=180802704 ingress_bytes=237207552 link_bytes=237207552 token_bytes=361387152 weight_movement_bytes=56623104 total_nvlink_bytes=418010256 weight_to_token_ratio=0.156683 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272806050e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58058 payload_rows_remote=49868 dup_rows=7478 dup_rows_remote=7478 dedup_ratio=0.114105 router_tokens_per_expert=[0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 4095, 0, 4093, 0, 0, 0, 307, 0, 2177, 0, 4087, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 1474, 5, 0, 4054, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 9] payload_rows_per_dst=[8192, 8192, 8192, 7718, 8190, 1236, 8162, 8176] dup_rows_per_dst=[1726, 12, 320, 2166, 0, 0, 2952, 302] combine_rows_per_src=[8192, 8192, 8192, 7718, 8190, 1236, 8162, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=193183744 dispatch_meta_bytes=221976 combine_payload_bytes=193183744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193405720 ingress_bytes=212058112 link_bytes=212058112 token_bytes=386589464 weight_movement_bytes=18874368 total_nvlink_bytes=405463832 weight_to_token_ratio=0.048823 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.031836490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50032 payload_rows_remote=41840 dup_rows=15504 dup_rows_remote=10296 dedup_ratio=0.236572 router_tokens_per_expert=[0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 0, 0, 4061, 0, 4088, 0, 0, 0, 11, 0, 0, 0, 4067, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 0, 0, 0, 2, 1, 0, 3835, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 172, 0, 0, 0, 0, 2613] payload_rows_per_dst=[8192, 8182, 8192, 2682, 8122, 7702, 44, 6916] dup_rows_per_dst=[744, 0, 5208, 1278, 0, 3638, 2, 4634] combine_rows_per_src=[8192, 8182, 8192, 2682, 8122, 7702, 44, 6916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=134275072 dispatch_meta_bytes=220768 combine_payload_bytes=134275072 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=134495840 ingress_bytes=172023808 link_bytes=172023808 token_bytes=268770912 weight_movement_bytes=37748736 total_nvlink_bytes=306519648 weight_to_token_ratio=0.140449 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.648247487e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=160931840 dispatch_meta_bytes=208544 combine_payload_bytes=160931840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=161140384 ingress_bytes=179806208 link_bytes=179806208 token_bytes=322072224 weight_movement_bytes=18874368 total_nvlink_bytes=340946592 weight_to_token_ratio=0.058603 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.722814615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51094 payload_rows_remote=42972 dup_rows=14442 dup_rows_remote=11988 dedup_ratio=0.220367 router_tokens_per_expert=[0, 77, 0, 0, 0, 0, 72, 0, 0, 21, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 53, 0, 10, 0, 0, 0, 286, 0, 0, 10, 3952, 0, 0, 5, 158, 61, 1, 0, 0, 0, 0, 0, 3541, 144, 12, 0, 784, 0, 0, 0, 0, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 2, 4043, 0, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 18, 0, 0, 4, 0, 0, 0, 0, 0, 4072, 944, 21, 0, 6, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3672] payload_rows_per_dst=[7586, 8122, 7188, 8052, 3802, 8090, 8170, 84] dup_rows_per_dst=[164, 2454, 486, 9574, 0, 0, 1764, 0] combine_rows_per_src=[7586, 8122, 7188, 8052, 3802, 8090, 8170, 84] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=204259328 dispatch_meta_bytes=229384 combine_payload_bytes=204259328 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=204488712 ingress_bytes=242008064 link_bytes=242008064 token_bytes=408748040 weight_movement_bytes=37748736 total_nvlink_bytes=446496776 weight_to_token_ratio=0.092352 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.318802194e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53722 payload_rows_remote=45736 dup_rows=11814 dup_rows_remote=5208 dedup_ratio=0.180267 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3136, 0, 0, 3, 0, 0, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 12, 0, 0, 0, 0, 4070, 0, 4050, 40, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 4086, 3926, 175, 0, 0, 0, 0, 47, 2, 0, 3390, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 35, 0, 0, 0, 46, 227] payload_rows_per_dst=[6492, 8140, 8192, 6570, 8154, 7986, 4066, 4122] dup_rows_per_dst=[3388, 10, 16, 80, 1618, 6606, 0, 96] combine_rows_per_src=[6492, 8140, 8192, 6570, 8154, 7986, 4066, 4122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=171376640 dispatch_meta_bytes=208544 combine_payload_bytes=171376640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171585184 ingress_bytes=190251008 link_bytes=190251008 token_bytes=342961824 weight_movement_bytes=18874368 total_nvlink_bytes=361836192 weight_to_token_ratio=0.055033 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.822891549e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56396 payload_rows_remote=48204 dup_rows=9140 dup_rows_remote=9138 dedup_ratio=0.139465 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 370, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8052, 8192, 8192, 7378, 8192, 6] dup_rows_per_dst=[2586, 0, 0, 2, 5986, 448, 118, 0] combine_rows_per_src=[8192, 8192, 8052, 8192, 8192, 7378, 8192, 6] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=176013312 dispatch_meta_bytes=219840 combine_payload_bytes=176013312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176233152 ingress_bytes=194887680 link_bytes=194887680 token_bytes=352246464 weight_movement_bytes=18874368 total_nvlink_bytes=371120832 weight_to_token_ratio=0.053583 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=187334656 dispatch_meta_bytes=203776 combine_payload_bytes=187334656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=187538432 ingress_bytes=243957760 link_bytes=243957760 token_bytes=374873088 weight_movement_bytes=56623104 total_nvlink_bytes=431496192 weight_to_token_ratio=0.151046 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.093750 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337483222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46268 payload_rows_remote=38076 dup_rows=19268 dup_rows_remote=8220 dedup_ratio=0.294006 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 197, 0, 0, 3677, 29, 0, 0, 0, 0, 0, 1767, 0, 0, 0, 58, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 1211, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[3280, 0, 6742, 3536, 8192, 8180, 8146, 8192] dup_rows_per_dst=[0, 0, 108, 130, 0, 7982, 0, 11048] combine_rows_per_src=[3280, 0, 6742, 3536, 8192, 8180, 8146, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=197443584 dispatch_meta_bytes=229368 combine_payload_bytes=197443584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197672952 ingress_bytes=216317952 link_bytes=216317952 token_bytes=395116536 weight_movement_bytes=18874368 total_nvlink_bytes=413990904 weight_to_token_ratio=0.047769 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.072652181e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56870 payload_rows_remote=48678 dup_rows=8666 dup_rows_remote=1504 dedup_ratio=0.132233 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 514, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 1, 27, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3954, 8190, 8192, 8192, 8112, 3846, 8192] dup_rows_per_dst=[0, 966, 0, 482, 7162, 0, 0, 56] combine_rows_per_src=[8192, 3954, 8190, 8192, 8192, 8112, 3846, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48536 payload_rows_remote=40344 dup_rows=17000 dup_rows_remote=16526 dedup_ratio=0.259399 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2928, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 21, 0, 972, 4096, 0, 0, 0, 0, 0, 0, 1, 927, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 0, 8192, 8078, 5834, 8192, 1856] dup_rows_per_dst=[474, 14048, 0, 22, 4, 0, 2452, 0] combine_rows_per_src=[8192, 8192, 0, 8192, 8078, 5834, 8192, 1856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58548 payload_rows_remote=50370 dup_rows=6988 dup_rows_remote=2672 dedup_ratio=0.106628 router_tokens_per_expert=[8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2369, 4038, 0, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2979, 0, 0, 0, 0, 1728, 0, 820, 720, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4059] payload_rows_per_dst=[5540, 8154, 8188, 8078, 7396, 4822, 8178, 8192] dup_rows_per_dst=[8, 10, 0, 0, 44, 16, 4316, 2594] combine_rows_per_src=[5540, 8154, 8188, 8078, 7396, 4822, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155959296 dispatch_meta_bytes=185184 combine_payload_bytes=155959296 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156144480 ingress_bytes=250331136 link_bytes=250331136 token_bytes=312103776 weight_movement_bytes=94371840 total_nvlink_bytes=406475616 weight_to_token_ratio=0.302373 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=9216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398549775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61708 payload_rows_remote=53520 dup_rows=3828 dup_rows_remote=3828 dedup_ratio=0.058411 router_tokens_per_expert=[0, 0, 152, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 62, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 1676, 4096, 0, 0, 0, 0, 0, 0, 3, 2663, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7346, 8188, 8192, 8088, 7902, 8162, 5638] dup_rows_per_dst=[304, 10, 0, 0, 14, 0, 3470, 30] combine_rows_per_src=[8192, 7346, 8188, 8192, 8088, 7902, 8162, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51182 payload_rows_remote=42990 dup_rows=14354 dup_rows_remote=11902 dedup_ratio=0.219025 router_tokens_per_expert=[0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1138, 82, 0, 0, 0, 0, 2, 0, 478, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3084, 0, 4020, 0, 0, 0, 3999, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 3849, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 112, 0, 0, 29, 0, 0, 0, 0, 0, 988, 0, 455, 0, 0, 0, 3890, 0, 0, 929, 0, 0, 0, 0, 0, 37, 2, 0, 0, 0, 0, 60, 1180, 0, 109, 0, 2, 0] payload_rows_per_dst=[6930, 8192, 8186, 8140, 6954, 2094, 8030, 2656] dup_rows_per_dst=[0, 2452, 1088, 7998, 2, 54, 2636, 124] combine_rows_per_src=[6930, 8192, 8186, 8140, 6954, 2094, 8030, 2656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=199385088 dispatch_meta_bytes=200728 combine_payload_bytes=199385088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199585816 ingress_bytes=256008192 link_bytes=256008192 token_bytes=398970904 weight_movement_bytes=56623104 total_nvlink_bytes=455594008 weight_to_token_ratio=0.141923 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=17280 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.452944532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=165249024 dispatch_meta_bytes=227480 combine_payload_bytes=165249024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=165476504 ingress_bytes=202997760 link_bytes=202997760 token_bytes=330725528 weight_movement_bytes=37748736 total_nvlink_bytes=368474264 weight_to_token_ratio=0.114139 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.945024655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54506 payload_rows_remote=46316 dup_rows=11030 dup_rows_remote=9198 dedup_ratio=0.168304 router_tokens_per_expert=[0, 0, 258, 0, 0, 0, 0, 2, 3973, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 19, 0, 76, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 81, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 453, 3997, 0, 0, 0, 0, 0, 1, 754, 4096, 48, 0, 3127, 0, 0, 0, 40, 0] payload_rows_per_dst=[8192, 462, 7006, 8134, 6488, 8190, 7842, 8192] dup_rows_per_dst=[298, 48, 6, 12, 0, 1832, 894, 7940] combine_rows_per_src=[8192, 462, 7006, 8134, 6488, 8190, 7842, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=206315520 dispatch_meta_bytes=212168 combine_payload_bytes=206315520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206527688 ingress_bytes=225189888 link_bytes=225189888 token_bytes=412843208 weight_movement_bytes=18874368 total_nvlink_bytes=431717576 weight_to_token_ratio=0.045718 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.157658706e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=219217920 dispatch_meta_bytes=229392 combine_payload_bytes=219217920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219447312 ingress_bytes=238092288 link_bytes=238092288 token_bytes=438665232 weight_movement_bytes=18874368 total_nvlink_bytes=457539600 weight_to_token_ratio=0.043027 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.281283155e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56300 payload_rows_remote=49148 dup_rows=9236 dup_rows_remote=9090 dedup_ratio=0.140930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 1, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3043, 0, 0, 0, 3, 0, 3576, 0, 4096, 0, 67, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 1348, 0, 0, 4087, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 6086, 8166, 7152, 8192, 3558, 6806, 8192] dup_rows_per_dst=[984, 0, 0, 146, 0, 14, 584, 7508] combine_rows_per_src=[8148, 6086, 8166, 7152, 8192, 3558, 6806, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=176087040 dispatch_meta_bytes=219568 combine_payload_bytes=176087040 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=176306608 ingress_bytes=213835776 link_bytes=213835776 token_bytes=352393648 weight_movement_bytes=37748736 total_nvlink_bytes=390142384 weight_to_token_ratio=0.107121 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.093750 gemm_active_groups=11 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048869192e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=189710336 dispatch_meta_bytes=222056 combine_payload_bytes=189710336 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=189932392 ingress_bytes=227459072 link_bytes=227459072 token_bytes=379642728 weight_movement_bytes=37748736 total_nvlink_bytes=417391464 weight_to_token_ratio=0.099432 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.179400911e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45502 payload_rows_remote=37394 dup_rows=20034 dup_rows_remote=13986 dedup_ratio=0.305695 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 42, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 1644, 0, 0, 0, 3947, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 4058, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 49, 2901, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 3, 0, 0, 0, 0] payload_rows_per_dst=[1332, 8192, 84, 3288, 8174, 8132, 8192, 8108] dup_rows_per_dst=[2, 74, 2, 0, 10, 7864, 6034, 6048] combine_rows_per_src=[1332, 8192, 84, 3288, 8174, 8132, 8192, 8108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=201310208 dispatch_meta_bytes=232952 combine_payload_bytes=201310208 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=201543160 ingress_bytes=314556416 link_bytes=314556416 token_bytes=402853368 weight_movement_bytes=113246208 total_nvlink_bytes=516099576 weight_to_token_ratio=0.281110 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=9216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.013924807e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61618 payload_rows_remote=53666 dup_rows=3918 dup_rows_remote=3918 dedup_ratio=0.059784 router_tokens_per_expert=[0, 0, 0, 0, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3472, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 37, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8116, 6884, 8078, 7292, 7952, 8160, 6944, 8192] dup_rows_per_dst=[1926, 112, 1726, 0, 0, 72, 0, 82] combine_rows_per_src=[8116, 6884, 8078, 7292, 7952, 8160, 6944, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=39808 payload_rows_remote=38248 dup_rows=25728 dup_rows_remote=25694 dedup_ratio=0.392578 router_tokens_per_expert=[0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 3897, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 5, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1389, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 0, 4095, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1283, 0, 0, 0, 2, 0, 0, 0, 4096, 1, 0] payload_rows_per_dst=[1560, 8192, 8160, 8192, 2936, 8190, 8, 2570] dup_rows_per_dst=[34, 2986, 8002, 8164, 78, 6462, 0, 2] combine_rows_per_src=[1560, 8192, 8160, 8192, 2936, 8190, 8, 2570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=153165824 dispatch_meta_bytes=205520 combine_payload_bytes=153165824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=153371344 ingress_bytes=172040192 link_bytes=172040192 token_bytes=306537168 weight_movement_bytes=18874368 total_nvlink_bytes=325411536 weight_to_token_ratio=0.061573 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.648404471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56440 payload_rows_remote=48248 dup_rows=9096 dup_rows_remote=7132 dedup_ratio=0.138794 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 467, 0, 0, 1042, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 0, 444, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3012, 4276, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 50, 0, 106, 0, 1964, 6970] combine_rows_per_src=[8192, 3012, 4276, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56622 payload_rows_remote=52060 dup_rows=8914 dup_rows_remote=8814 dedup_ratio=0.136017 router_tokens_per_expert=[0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 1737, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 36, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3339, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[5444, 8192, 4562, 8192, 7192, 6678, 8192, 8170] dup_rows_per_dst=[230, 710, 100, 900, 6840, 0, 122, 12] combine_rows_per_src=[5444, 8192, 4562, 8192, 7192, 6678, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=219815936 dispatch_meta_bytes=230336 combine_payload_bytes=219815936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220046272 ingress_bytes=238690304 link_bytes=238690304 token_bytes=439862208 weight_movement_bytes=18874368 total_nvlink_bytes=458736576 weight_to_token_ratio=0.042910 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287013050e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53546 payload_rows_remote=45388 dup_rows=11990 dup_rows_remote=4670 dedup_ratio=0.182953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 17, 0, 0, 1619, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 2, 139, 0, 0, 146, 0, 5, 0, 0, 1799, 4077, 4, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3756, 0, 1, 0, 0, 0, 7, 148, 3983, 30, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8126, 8158, 4156, 7512, 8192, 8186, 1226, 7990] dup_rows_per_dst=[0, 7320, 170, 0, 4152, 0, 0, 348] combine_rows_per_src=[8126, 8158, 4156, 7512, 8192, 8186, 1226, 7990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=255768 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156919576 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313583384 weight_movement_bytes=37748736 total_nvlink_bytes=351332120 weight_to_token_ratio=0.120379 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57080 payload_rows_remote=56262 dup_rows=8456 dup_rows_remote=8426 dedup_ratio=0.129028 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 31, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 39, 0, 0, 0, 0, 0, 0, 0, 0, 331, 4037, 0, 48, 0, 0, 108, 0, 0, 0, 0, 0, 3862, 35, 0, 0, 0, 759, 0, 0, 0, 0, 37, 0, 0, 0, 0, 179, 2, 0, 222, 0, 0, 4049, 0, 21, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 3881, 0, 4, 0, 0, 0, 1, 10, 3344, 0, 0] payload_rows_per_dst=[8188, 7574, 7836, 8192, 8192, 818, 8118, 8162] dup_rows_per_dst=[0, 2, 56, 856, 1194, 30, 0, 6318] combine_rows_per_src=[8188, 7574, 7836, 8192, 8192, 818, 8118, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=197623808 dispatch_meta_bytes=221520 combine_payload_bytes=197623808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197845328 ingress_bytes=216498176 link_bytes=216498176 token_bytes=395469136 weight_movement_bytes=18874368 total_nvlink_bytes=414343504 weight_to_token_ratio=0.047727 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.074378999e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57600 payload_rows_remote=49408 dup_rows=7936 dup_rows_remote=7936 dedup_ratio=0.121094 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 1212, 8192, 7548, 7880] dup_rows_per_dst=[0, 0, 7692, 0, 244, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 1212, 8192, 7548, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=213237760 dispatch_meta_bytes=243496 combine_payload_bytes=213237760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213481256 ingress_bytes=232112128 link_bytes=232112128 token_bytes=426719016 weight_movement_bytes=18874368 total_nvlink_bytes=445593384 weight_to_token_ratio=0.044231 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223984204e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=185909248 dispatch_meta_bytes=200232 combine_payload_bytes=185909248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=186109480 ingress_bytes=223657984 link_bytes=223657984 token_bytes=372018728 weight_movement_bytes=37748736 total_nvlink_bytes=409767464 weight_to_token_ratio=0.101470 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142980756e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=230449152 dispatch_meta_bytes=258752 combine_payload_bytes=230449152 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=230707904 ingress_bytes=287072256 link_bytes=287072256 token_bytes=461157056 weight_movement_bytes=56623104 total_nvlink_bytes=517780160 weight_to_token_ratio=0.122785 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.750585109e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59126 payload_rows_remote=50934 dup_rows=6410 dup_rows_remote=3566 dedup_ratio=0.097809 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3368, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 16, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1326, 4096, 0, 0, 15, 0, 0, 0, 28, 1355, 0, 13, 4096, 0, 0, 11, 0, 0] payload_rows_per_dst=[7182, 6818, 8192, 8192, 5114, 8192, 7244, 8192] dup_rows_per_dst=[0, 22, 14, 222, 338, 898, 2072, 2844] combine_rows_per_src=[7182, 6818, 8192, 8192, 5114, 8192, 7244, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=202375168 dispatch_meta_bytes=229376 combine_payload_bytes=202375168 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=202604544 ingress_bytes=240123904 link_bytes=240123904 token_bytes=404979712 weight_movement_bytes=37748736 total_nvlink_bytes=442728448 weight_to_token_ratio=0.093211 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300749100e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49672 payload_rows_remote=41480 dup_rows=15864 dup_rows_remote=15834 dedup_ratio=0.242065 router_tokens_per_expert=[0, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 240, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2081, 0, 0, 0, 0, 0, 0, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3282] payload_rows_per_dst=[2884, 8134, 8192, 4678, 8192, 5782, 3618, 8192] dup_rows_per_dst=[10, 5388, 2850, 0, 30, 362, 660, 6564] combine_rows_per_src=[2884, 8134, 8192, 4678, 8192, 5782, 3618, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51954 payload_rows_remote=43762 dup_rows=13582 dup_rows_remote=13550 dedup_ratio=0.207245 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 11, 0, 0, 0, 0, 130, 4096, 0, 0, 0, 0, 0, 150, 0, 0, 0, 529, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3777, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 1575, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5386, 8190, 8192, 6590, 7218, 8144, 42] dup_rows_per_dst=[32, 870, 3594, 5644, 664, 0, 2778, 0] combine_rows_per_src=[8192, 5386, 8190, 8192, 6590, 7218, 8144, 42] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=208625664 dispatch_meta_bytes=218000 combine_payload_bytes=208625664 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208843664 ingress_bytes=246374400 link_bytes=246374400 token_bytes=417469328 weight_movement_bytes=37748736 total_nvlink_bytes=455218064 weight_to_token_ratio=0.090423 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.360638277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=169902080 dispatch_meta_bytes=229256 combine_payload_bytes=169902080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=170131336 ingress_bytes=188776448 link_bytes=188776448 token_bytes=340033416 weight_movement_bytes=18874368 total_nvlink_bytes=358907784 weight_to_token_ratio=0.055507 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.808763041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49552 payload_rows_remote=49254 dup_rows=15984 dup_rows_remote=15974 dedup_ratio=0.243896 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0] payload_rows_per_dst=[8192, 6160, 6982, 8182, 3364, 8182, 298, 8192] dup_rows_per_dst=[2, 0, 4246, 3666, 0, 0, 10, 8060] combine_rows_per_src=[8192, 6160, 6982, 8182, 3364, 8182, 298, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52622 payload_rows_remote=44430 dup_rows=12914 dup_rows_remote=11486 dedup_ratio=0.197052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 2, 2259, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 24, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 961, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7608, 8078, 8192, 7572, 8192, 4218, 570, 8192] dup_rows_per_dst=[490, 4564, 1428, 1748, 4640, 0, 44, 0] combine_rows_per_src=[7608, 8078, 8192, 7572, 8192, 4218, 570, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48484 payload_rows_remote=40530 dup_rows=17052 dup_rows_remote=16286 dedup_ratio=0.260193 router_tokens_per_expert=[1, 0, 0, 0, 24, 4025, 0, 0, 0, 0, 0, 0, 0, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 53, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 11, 0, 33, 0, 0, 2593, 0, 0, 3655, 0, 0, 0, 0, 0, 0, 44, 0, 4060, 1, 0, 7, 0, 0, 0, 0, 0, 166, 0, 0, 0, 6, 4, 31, 2, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 6] payload_rows_per_dst=[8166, 7954, 8162, 8186, 138, 7610, 8190, 78] dup_rows_per_dst=[3094, 766, 36, 7794, 2, 2664, 2696, 0] combine_rows_per_src=[8166, 7954, 8162, 8186, 138, 7610, 8190, 78] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=179249152 dispatch_meta_bytes=229248 combine_payload_bytes=179249152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179478400 ingress_bytes=198123520 link_bytes=198123520 token_bytes=358727552 weight_movement_bytes=18874368 total_nvlink_bytes=377601920 weight_to_token_ratio=0.052615 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.898322086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49662 payload_rows_remote=41470 dup_rows=15874 dup_rows_remote=5044 dedup_ratio=0.242218 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 110, 0, 0, 0, 0, 68, 0, 2066, 0, 126, 0, 0, 0, 0, 0, 0, 0, 4020, 1, 0, 3829, 0, 0, 0, 0, 3834, 0, 0, 0, 32, 0, 38, 0, 0, 0, 0, 0, 25, 4077, 0, 3697, 0, 2223, 0, 0, 0, 0, 115, 0, 0, 0, 57, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1698, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 13, 0, 0, 0, 288, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 2053] payload_rows_per_dst=[480, 4522, 7754, 4538, 8160, 8192, 8192, 7824] dup_rows_per_dst=[98, 0, 54, 188, 114, 10830, 480, 4110] combine_rows_per_src=[480, 4522, 7754, 4538, 8160, 8192, 8192, 7824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=201744384 dispatch_meta_bytes=260912 combine_payload_bytes=201744384 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=202005296 ingress_bytes=296116224 link_bytes=296116224 token_bytes=403749680 weight_movement_bytes=94371840 total_nvlink_bytes=498121520 weight_to_token_ratio=0.233738 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.837239960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58260 payload_rows_remote=50594 dup_rows=7276 dup_rows_remote=7276 dedup_ratio=0.111023 router_tokens_per_expert=[0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 94, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0] payload_rows_per_dst=[8192, 8192, 7896, 7666, 8192, 8192, 8184, 1746] dup_rows_per_dst=[654, 192, 0, 0, 6428, 2, 0, 0] combine_rows_per_src=[8192, 8192, 7896, 7666, 8192, 8192, 8184, 1746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=181985280 dispatch_meta_bytes=223664 combine_payload_bytes=181985280 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=182208944 ingress_bytes=238608384 link_bytes=238608384 token_bytes=364194224 weight_movement_bytes=56623104 total_nvlink_bytes=420817328 weight_to_token_ratio=0.155475 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286228133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=166010880 dispatch_meta_bytes=227264 combine_payload_bytes=166010880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=166238144 ingress_bytes=184885248 link_bytes=184885248 token_bytes=332249024 weight_movement_bytes=18874368 total_nvlink_bytes=351123392 weight_to_token_ratio=0.056808 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.771479477e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=169861120 dispatch_meta_bytes=186056 combine_payload_bytes=169861120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=170047176 ingress_bytes=188735488 link_bytes=188735488 token_bytes=339908296 weight_movement_bytes=18874368 total_nvlink_bytes=358782664 weight_to_token_ratio=0.055528 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=15360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.808370582e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47418 payload_rows_remote=39226 dup_rows=18118 dup_rows_remote=7936 dedup_ratio=0.276459 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 542, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 4018, 1, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3838, 0, 0, 0, 202, 0, 0, 0, 881, 0, 0] payload_rows_per_dst=[5932, 6040, 2678, 0, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 212, 756, 0, 0, 274, 6694, 10182] combine_rows_per_src=[5932, 6040, 2678, 0, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55392 payload_rows_remote=47350 dup_rows=10144 dup_rows_remote=7372 dedup_ratio=0.154785 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 25, 4096, 95, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8128, 5844, 8192, 8192, 8042, 8094, 8192, 708] dup_rows_per_dst=[5940, 1354, 0, 0, 2772, 0, 0, 78] combine_rows_per_src=[8128, 5844, 8192, 8192, 8042, 8094, 8192, 708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=207233024 dispatch_meta_bytes=231480 combine_payload_bytes=207233024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207464504 ingress_bytes=226107392 link_bytes=226107392 token_bytes=414697528 weight_movement_bytes=18874368 total_nvlink_bytes=433571896 weight_to_token_ratio=0.045514 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.166449778e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49050 payload_rows_remote=41050 dup_rows=16486 dup_rows_remote=16452 dedup_ratio=0.251556 router_tokens_per_expert=[0, 38, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 4093, 135, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 28, 74, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 72, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 17, 4096, 577, 0, 123, 0, 0, 2, 1] payload_rows_per_dst=[8000, 8192, 8192, 8182, 6942, 144, 8026, 1372] dup_rows_per_dst=[34, 15744, 444, 172, 24, 0, 0, 68] combine_rows_per_src=[8000, 8192, 8192, 8182, 6942, 144, 8026, 1372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=160669696 dispatch_meta_bytes=188648 combine_payload_bytes=160669696 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=160858344 ingress_bytes=236167168 link_bytes=236167168 token_bytes=321528040 weight_movement_bytes=75497472 total_nvlink_bytes=397025512 weight_to_token_ratio=0.234808 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.262837602e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=193945600 dispatch_meta_bytes=218888 combine_payload_bytes=193945600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194164488 ingress_bytes=212819968 link_bytes=212819968 token_bytes=388110088 weight_movement_bytes=18874368 total_nvlink_bytes=406984456 weight_to_token_ratio=0.048631 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039136219e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51300 payload_rows_remote=43108 dup_rows=14236 dup_rows_remote=14228 dedup_ratio=0.217224 router_tokens_per_expert=[0, 74, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3958, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1021, 4096, 58, 0, 0, 0, 0, 40, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8174, 6, 8126, 2226] dup_rows_per_dst=[3248, 7986, 8, 8, 2974, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8192, 8192, 8174, 6, 8126, 2226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46474 payload_rows_remote=38282 dup_rows=19062 dup_rows_remote=6762 dedup_ratio=0.290863 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3474, 0, 2963, 0, 0, 0, 0, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 251, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 1, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 549, 757, 0, 0, 0, 0, 219, 0, 4096, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7402, 8172, 504, 8180, 766, 6310, 8192, 6948] dup_rows_per_dst=[828, 88, 0, 5224, 0, 622, 12300, 0] combine_rows_per_src=[7402, 8172, 504, 8180, 766, 6310, 8192, 6948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48622 payload_rows_remote=40430 dup_rows=16914 dup_rows_remote=16668 dedup_ratio=0.258087 router_tokens_per_expert=[0, 0, 3917, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4057, 0, 0, 3795, 0, 0, 25, 0, 4047, 0, 0, 182, 0, 17, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 773, 0, 0, 2, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1910, 0, 0, 0, 0, 0, 48, 1, 0, 67, 0, 0, 192, 0, 10, 1, 0, 0, 989, 2, 19, 0, 0, 4078, 0, 0, 0, 0, 68, 0, 321, 0, 4, 10, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 31, 0, 0, 3, 0] payload_rows_per_dst=[8092, 8192, 8144, 4998, 6696, 614, 8192, 3694] dup_rows_per_dst=[7736, 246, 4614, 0, 1480, 24, 2770, 44] combine_rows_per_src=[8092, 8192, 8144, 4998, 6696, 614, 8192, 3694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=168140800 dispatch_meta_bytes=230008 combine_payload_bytes=168140800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=168370808 ingress_bytes=187015168 link_bytes=187015168 token_bytes=336511608 weight_movement_bytes=18874368 total_nvlink_bytes=355385976 weight_to_token_ratio=0.056088 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.791887323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46340 payload_rows_remote=38148 dup_rows=19196 dup_rows_remote=12154 dedup_ratio=0.292908 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 6, 3, 0, 0, 804, 3988, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 3, 0, 3890, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 33, 0, 0, 0, 0, 431, 3297, 127, 0, 3729, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 3132, 0, 0, 1, 0, 0, 532, 4091, 3724, 0, 329, 0, 0, 0, 0] payload_rows_per_dst=[86, 86, 8190, 8060, 5724, 8192, 7854, 8148] dup_rows_per_dst=[0, 0, 1402, 70, 124, 7042, 0, 10558] combine_rows_per_src=[86, 86, 8190, 8060, 5724, 8192, 7854, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56564 payload_rows_remote=50926 dup_rows=8972 dup_rows_remote=6596 dedup_ratio=0.136902 router_tokens_per_expert=[0, 0, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 0, 4003, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 0, 0, 22, 0, 0, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 6384, 8170, 5638, 8060, 3992, 8192, 8006] dup_rows_per_dst=[6580, 0, 0, 2376, 0, 0, 16, 0] combine_rows_per_src=[8122, 6384, 8170, 5638, 8060, 3992, 8192, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=176570368 dispatch_meta_bytes=229344 combine_payload_bytes=176570368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=176799712 ingress_bytes=214319104 link_bytes=214319104 token_bytes=353370080 weight_movement_bytes=37748736 total_nvlink_bytes=391118816 weight_to_token_ratio=0.106825 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053500203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=180176 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156983248 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313786320 weight_movement_bytes=56623104 total_nvlink_bytes=370409424 weight_to_token_ratio=0.180451 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=13696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=165601280 dispatch_meta_bytes=228392 combine_payload_bytes=165601280 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=165829672 ingress_bytes=222224384 link_bytes=222224384 token_bytes=331430952 weight_movement_bytes=56623104 total_nvlink_bytes=388054056 weight_to_token_ratio=0.170844 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.129244706e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54434 payload_rows_remote=46242 dup_rows=11102 dup_rows_remote=10766 dedup_ratio=0.169403 router_tokens_per_expert=[0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4046, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2840, 0, 0, 4096, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 4095, 0, 17, 0, 0, 0, 0, 1837, 0, 0, 0, 206, 0, 1416, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 87, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 39, 0, 124] payload_rows_per_dst=[7834, 8114, 0, 8054, 8190, 8192, 5858, 8192] dup_rows_per_dst=[12, 2, 0, 3644, 3290, 3814, 4, 336] combine_rows_per_src=[7834, 8114, 0, 8054, 8190, 8192, 5858, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58608 payload_rows_remote=50688 dup_rows=6928 dup_rows_remote=6928 dedup_ratio=0.105713 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7628, 8176, 2928, 7920, 8192, 8192, 7380] dup_rows_per_dst=[0, 0, 4, 4, 0, 0, 6690, 230] combine_rows_per_src=[8192, 7628, 8176, 2928, 7920, 8192, 8192, 7380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=201208 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156455416 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312709624 weight_movement_bytes=37748736 total_nvlink_bytes=350458360 weight_to_token_ratio=0.120715 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.083333 gemm_active_groups=14 gemm_peak_group_rows=23040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=208592896 dispatch_meta_bytes=230088 combine_payload_bytes=208592896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208822984 ingress_bytes=265216000 link_bytes=265216000 token_bytes=417415880 weight_movement_bytes=56623104 total_nvlink_bytes=474038984 weight_to_token_ratio=0.135652 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.541169218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47156 payload_rows_remote=38964 dup_rows=18380 dup_rows_remote=17082 dedup_ratio=0.280457 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 3718, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 54, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4081, 392, 0, 0, 0] payload_rows_per_dst=[8192, 8148, 8192, 0, 8178, 6138, 116, 8192] dup_rows_per_dst=[1298, 7564, 6588, 0, 2128, 0, 4, 798] combine_rows_per_src=[8192, 8148, 8192, 0, 8178, 6138, 116, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=189407232 dispatch_meta_bytes=228032 combine_payload_bytes=189407232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189635264 ingress_bytes=208281600 link_bytes=208281600 token_bytes=379042496 weight_movement_bytes=18874368 total_nvlink_bytes=397916864 weight_to_token_ratio=0.049795 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.995651810e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=207618048 dispatch_meta_bytes=230464 combine_payload_bytes=207618048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207848512 ingress_bytes=226492416 link_bytes=226492416 token_bytes=415466560 weight_movement_bytes=18874368 total_nvlink_bytes=434340928 weight_to_token_ratio=0.045429 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.170138889e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56326 payload_rows_remote=48134 dup_rows=9210 dup_rows_remote=9070 dedup_ratio=0.140533 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 253, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7516, 8192, 7848, 8192, 8192, 2, 8192] dup_rows_per_dst=[272, 0, 140, 124, 8162, 0, 0, 512] combine_rows_per_src=[8192, 7516, 8192, 7848, 8192, 8192, 2, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43966 payload_rows_remote=35774 dup_rows=21570 dup_rows_remote=21274 dedup_ratio=0.329132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4092, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 4050, 0, 0, 0, 0, 0, 150, 0, 1, 0, 0, 0, 0, 4096] payload_rows_per_dst=[0, 0, 7468, 3730, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 126, 8284, 296, 12864] combine_rows_per_src=[0, 0, 7468, 3730, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48948 payload_rows_remote=46612 dup_rows=16588 dup_rows_remote=16584 dedup_ratio=0.253113 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1393, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 0, 0, 0, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 57, 0, 0, 4049, 0, 0, 3915, 0, 0, 0, 923, 145, 0, 314, 0, 6, 0, 0, 5, 0, 0, 0, 189, 0, 0, 0, 0, 18, 2457, 0, 0, 0, 0, 0, 0, 1108, 2350, 25, 0, 1, 0, 0, 0] payload_rows_per_dst=[8174, 2336, 8098, 8178, 8186, 8192, 998, 4786] dup_rows_per_dst=[7410, 4, 514, 4, 0, 8334, 320, 2] combine_rows_per_src=[8174, 2336, 8098, 8178, 8186, 8192, 998, 4786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=224184 combine_payload_bytes=159596544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=159820728 ingress_bytes=197345280 link_bytes=197345280 token_bytes=319417272 weight_movement_bytes=37748736 total_nvlink_bytes=357166008 weight_to_token_ratio=0.118180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.890865373e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55000 payload_rows_remote=47104 dup_rows=10536 dup_rows_remote=10536 dedup_ratio=0.160767 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 13, 0, 0, 123, 0, 0, 0, 4029, 0, 421, 0, 2, 0, 26, 0, 0, 0, 12, 0, 108, 830, 0, 0, 0, 4096, 0, 107, 0, 0, 30, 0, 0, 0, 7, 0, 0, 0, 0, 1497, 0, 0, 0, 0, 3948, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 2441, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 1862, 0, 0, 1, 0] payload_rows_per_dst=[7144, 8072, 8190, 2716, 8192, 7896, 8190, 4600] dup_rows_per_dst=[0, 0, 238, 82, 3282, 0, 6884, 50] combine_rows_per_src=[7144, 8072, 8190, 2716, 8192, 7896, 8190, 4600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=197156864 dispatch_meta_bytes=228816 combine_payload_bytes=197156864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197385680 ingress_bytes=216031232 link_bytes=216031232 token_bytes=394542544 weight_movement_bytes=18874368 total_nvlink_bytes=413416912 weight_to_token_ratio=0.047839 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.069904971e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57692 payload_rows_remote=49500 dup_rows=7844 dup_rows_remote=7844 dedup_ratio=0.119690 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 4, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4051, 371, 0, 0, 0, 0, 14, 0] payload_rows_per_dst=[8190, 8142, 8192, 8192, 7822, 8192, 8192, 770] dup_rows_per_dst=[7808, 2, 14, 0, 0, 20, 0, 0] combine_rows_per_src=[8190, 8142, 8192, 8192, 7822, 8192, 8192, 770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=146530304 dispatch_meta_bytes=228192 combine_payload_bytes=146530304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=146758496 ingress_bytes=165404672 link_bytes=165404672 token_bytes=293288800 weight_movement_bytes=18874368 total_nvlink_bytes=312163168 weight_to_token_ratio=0.064354 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.584826183e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=190922752 dispatch_meta_bytes=252784 combine_payload_bytes=190922752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191175536 ingress_bytes=209797120 link_bytes=209797120 token_bytes=382098288 weight_movement_bytes=18874368 total_nvlink_bytes=400972656 weight_to_token_ratio=0.049397 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.010172777e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55848 payload_rows_remote=47738 dup_rows=9688 dup_rows_remote=9180 dedup_ratio=0.147827 router_tokens_per_expert=[0, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3538, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4007, 275, 0, 27, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 118, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 8, 0, 0, 3741, 0, 0] payload_rows_per_dst=[7882, 8192, 7608, 16, 8110, 8184, 8192, 7664] dup_rows_per_dst=[90, 7070, 6, 0, 508, 232, 1764, 18] combine_rows_per_src=[7882, 8192, 7608, 16, 8110, 8184, 8192, 7664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50400 payload_rows_remote=42208 dup_rows=15136 dup_rows_remote=2744 dedup_ratio=0.230957 router_tokens_per_expert=[0, 64, 0, 0, 0, 0, 0, 0, 0, 74, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 253, 0, 0, 0, 52, 0, 0, 0, 5, 4062, 30, 1, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 4076, 0, 0, 0, 0, 0, 11, 3724, 149, 0, 392, 0, 0, 0, 0] payload_rows_per_dst=[276, 6846, 4706, 7474, 6540, 8174, 8192, 8192] dup_rows_per_dst=[6, 0, 1200, 626, 20, 892, 0, 12392] combine_rows_per_src=[276, 6846, 4706, 7474, 6540, 8174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=192937984 dispatch_meta_bytes=230560 combine_payload_bytes=192937984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=193168544 ingress_bytes=249561088 link_bytes=249561088 token_bytes=386106528 weight_movement_bytes=56623104 total_nvlink_bytes=442729632 weight_to_token_ratio=0.146652 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.391171553e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=202752000 dispatch_meta_bytes=229376 combine_payload_bytes=202752000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202981376 ingress_bytes=240500736 link_bytes=240500736 token_bytes=405733376 weight_movement_bytes=37748736 total_nvlink_bytes=443482112 weight_to_token_ratio=0.093038 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.304359719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=195534848 dispatch_meta_bytes=227672 combine_payload_bytes=195534848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=195762520 ingress_bytes=233283584 link_bytes=233283584 token_bytes=391297368 weight_movement_bytes=37748736 total_nvlink_bytes=429046104 weight_to_token_ratio=0.096471 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=14976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235208519e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51462 payload_rows_remote=43270 dup_rows=14074 dup_rows_remote=6866 dedup_ratio=0.214752 router_tokens_per_expert=[0, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 1553, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1362, 0, 116, 21, 1346, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 3465, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 0, 0, 18, 0, 0, 3655, 0, 33] payload_rows_per_dst=[8192, 5482, 2692, 8180, 6914, 6982, 5626, 7394] dup_rows_per_dst=[7208, 872, 2, 2194, 1820, 1960, 0, 18] combine_rows_per_src=[8192, 5482, 2692, 8180, 6914, 6982, 5626, 7394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=172883968 dispatch_meta_bytes=179808 combine_payload_bytes=172883968 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=173063776 ingress_bytes=248381440 link_bytes=248381440 token_bytes=345947744 weight_movement_bytes=75497472 total_nvlink_bytes=421445216 weight_to_token_ratio=0.218234 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=11648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379868747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54638 payload_rows_remote=47432 dup_rows=10898 dup_rows_remote=10890 dedup_ratio=0.166290 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1931, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 0, 673, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 771, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 3525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 23, 0, 0, 4036, 0, 16] payload_rows_per_dst=[8036, 8192, 7206, 8190, 6042, 1542, 7304, 8126] dup_rows_per_dst=[10, 3862, 8, 3756, 1344, 0, 866, 1052] combine_rows_per_src=[8036, 8192, 7206, 8190, 6042, 1542, 7304, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19017193688 prefetch_bytes=849346560 reduce_bytes=924844032 weight_movement_bytes=1774190592 egress_bytes=9514014936 ingress_bytes=11277369344 link_bytes=11277369344 total_nvlink_bytes=20791384280 weight_to_token_ratio=0.093294 prefetch_share_of_ingress=0.075314 dedup_ratio=0.146765 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=49 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.089771 gemm_inner_pad_upper_bound=61468 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.251129255e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49960 payload_rows_remote=41820 dup_rows=15576 dup_rows_remote=13512 dedup_ratio=0.237671 router_tokens_per_expert=[0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 481, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 11, 0, 0, 37, 0, 3, 2336, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 11, 151, 0, 0, 131, 342, 3608, 13, 3811, 0, 0, 0, 0, 3, 1, 0, 0, 0, 63, 0, 4047, 0, 230, 0, 0, 0, 0, 33, 0, 0, 4078, 0, 0, 0, 0, 17, 187, 0, 0, 1, 3778, 0, 3857, 2, 103, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7668, 8140, 3050, 6432, 406, 7980, 8186, 8098] dup_rows_per_dst=[0, 2064, 0, 4866, 16, 342, 496, 7792] combine_rows_per_src=[7668, 8140, 3050, 6432, 406, 7980, 8186, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52858 payload_rows_remote=44666 dup_rows=12678 dup_rows_remote=6834 dedup_ratio=0.193451 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 1, 0, 2893, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2836, 0, 0, 0, 351, 0, 2208, 0, 4096, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1574, 0, 0, 0, 0, 149, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 2891, 0, 0, 4084, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0] payload_rows_per_dst=[6006, 5680, 4282, 4416, 8192, 7898, 8192, 8192] dup_rows_per_dst=[0, 0, 140, 722, 10, 5702, 5844, 260] combine_rows_per_src=[6006, 5680, 4282, 4416, 8192, 7898, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=177233920 dispatch_meta_bytes=200544 combine_payload_bytes=177233920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=177434464 ingress_bytes=196108288 link_bytes=196108288 token_bytes=354668384 weight_movement_bytes=18874368 total_nvlink_bytes=373542752 weight_to_token_ratio=0.053217 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.879013124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52256 payload_rows_remote=44066 dup_rows=13280 dup_rows_remote=3758 dedup_ratio=0.202637 router_tokens_per_expert=[0, 3, 10, 3474, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 302, 0, 0, 0, 87, 3050, 4, 0, 0, 0, 4, 0, 0, 0, 212, 0, 0, 0, 0, 9, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3276, 0, 2325, 0, 0, 1213, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 809, 0, 0, 0, 421, 0, 3888, 0, 3738, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 10, 127, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 14, 0, 36, 0, 0, 1, 0, 0] payload_rows_per_dst=[7094, 1106, 7360, 7466, 4658, 8190, 8192, 8190] dup_rows_per_dst=[24, 12, 1780, 1656, 2, 9522, 204, 80] combine_rows_per_src=[7094, 1106, 7360, 7466, 4658, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50194 payload_rows_remote=42002 dup_rows=15342 dup_rows_remote=2066 dedup_ratio=0.234100 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 220, 0, 0, 0, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7794, 8192, 3174, 8192, 4082, 2376] dup_rows_per_dst=[0, 0, 0, 13276, 0, 1898, 0, 168] combine_rows_per_src=[8192, 8192, 7794, 8192, 3174, 8192, 4082, 2376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=194281472 dispatch_meta_bytes=233288 combine_payload_bytes=194281472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194514760 ingress_bytes=213155840 link_bytes=213155840 token_bytes=388796232 weight_movement_bytes=18874368 total_nvlink_bytes=407670600 weight_to_token_ratio=0.048546 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042354380e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=16674250496 prefetch_bytes=868220928 reduce_bytes=736100352 weight_movement_bytes=1604321280 egress_bytes=8342380288 ingress_bytes=9936191488 link_bytes=9936191488 total_nvlink_bytes=18278571776 weight_to_token_ratio=0.096215 prefetch_share_of_ingress=0.087380 dedup_ratio=0.235509 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=39 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.089048 gemm_inner_pad_upper_bound=63627 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.983410373e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56678 payload_rows_remote=48508 dup_rows=8858 dup_rows_remote=3920 dedup_ratio=0.135162 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 27, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 228, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 140, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 3713, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 88, 7990, 8102, 7990, 8192, 7954, 8170] dup_rows_per_dst=[2, 12, 142, 474, 0, 292, 2998, 4938] combine_rows_per_src=[8192, 88, 7990, 8102, 7990, 8192, 7954, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=171294720 dispatch_meta_bytes=221328 combine_payload_bytes=171294720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171516048 ingress_bytes=190169088 link_bytes=190169088 token_bytes=342810768 weight_movement_bytes=18874368 total_nvlink_bytes=361685136 weight_to_token_ratio=0.055058 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.822106632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=180494336 dispatch_meta_bytes=191296 combine_payload_bytes=180494336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=180685632 ingress_bytes=218243072 link_bytes=218243072 token_bytes=361179968 weight_movement_bytes=37748736 total_nvlink_bytes=398928704 weight_to_token_ratio=0.104515 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.091097734e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=182951936 dispatch_meta_bytes=206000 combine_payload_bytes=182951936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=183157936 ingress_bytes=220700672 link_bytes=220700672 token_bytes=366109872 weight_movement_bytes=37748736 total_nvlink_bytes=403858608 weight_to_token_ratio=0.103108 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.114645248e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=172040192 dispatch_meta_bytes=176272 combine_payload_bytes=172040192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172216464 ingress_bytes=190914560 link_bytes=190914560 token_bytes=344256656 weight_movement_bytes=18874368 total_nvlink_bytes=363131024 weight_to_token_ratio=0.054826 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=12032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.829249378e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=18180085640 prefetch_bytes=849346560 reduce_bytes=830472192 weight_movement_bytes=1679818752 egress_bytes=9095477128 ingress_bytes=10764427264 link_bytes=10764427264 total_nvlink_bytes=19859904392 weight_to_token_ratio=0.092399 prefetch_share_of_ingress=0.078903 dedup_ratio=0.185894 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=44 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.090929 gemm_inner_pad_upper_bound=61976 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.148738450e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=198688768 dispatch_meta_bytes=209712 combine_payload_bytes=198688768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198898480 ingress_bytes=217563136 link_bytes=217563136 token_bytes=397587248 weight_movement_bytes=18874368 total_nvlink_bytes=416461616 weight_to_token_ratio=0.047472 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=14976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.084582921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=18429281296 prefetch_bytes=868220928 reduce_bytes=1019215872 weight_movement_bytes=1887436800 egress_bytes=9220056080 ingress_bytes=11096662016 link_bytes=11096662016 total_nvlink_bytes=20316718096 weight_to_token_ratio=0.102415 prefetch_share_of_ingress=0.078242 dedup_ratio=0.178749 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=54 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.088686 gemm_inner_pad_upper_bound=64389 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.215057407e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47446 payload_rows_remote=47302 dup_rows=18090 dup_rows_remote=18018 dedup_ratio=0.276031 router_tokens_per_expert=[1, 38, 0, 0, 26, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1963, 23, 21, 52, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2193, 0, 0, 3, 0, 0, 0, 142, 0, 0, 0, 0, 0, 976, 4082, 4, 4065, 3983, 36, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2058, 0, 4045, 0, 0, 0, 0, 230, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 8, 17, 0, 0, 0, 3, 0, 4054, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 144, 8120, 5506, 8192, 8090, 1056, 8146] dup_rows_per_dst=[4052, 72, 60, 3286, 10142, 460, 0, 18] combine_rows_per_src=[8192, 144, 8120, 5506, 8192, 8090, 1056, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56772 payload_rows_remote=49224 dup_rows=8764 dup_rows_remote=8224 dedup_ratio=0.133728 router_tokens_per_expert=[0, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 426, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3807, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 143, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7548, 8190, 692, 7618, 8148] dup_rows_per_dst=[6424, 10, 1000, 540, 716, 0, 0, 74] combine_rows_per_src=[8192, 8192, 8192, 7548, 8190, 692, 7618, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55500 payload_rows_remote=47308 dup_rows=10036 dup_rows_remote=5640 dedup_ratio=0.153137 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 0, 8192, 8192, 8162, 8192, 8192, 6378] dup_rows_per_dst=[0, 0, 5538, 0, 96, 6, 4396, 0] combine_rows_per_src=[8192, 0, 8192, 8192, 8162, 8192, 8192, 6378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=193748992 dispatch_meta_bytes=261280 combine_payload_bytes=193748992 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=194010272 ingress_bytes=212623360 link_bytes=212623360 token_bytes=387759264 weight_movement_bytes=18874368 total_nvlink_bytes=406633632 weight_to_token_ratio=0.048675 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037252418e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47266 payload_rows_remote=39306 dup_rows=18270 dup_rows_remote=17836 dedup_ratio=0.278778 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3973, 0, 0, 3309, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 78, 2, 2, 0, 0, 0, 0, 14, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 1449, 213, 39, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 4061, 90, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[12, 8000, 5952, 8192, 766, 8192, 8192, 7960] dup_rows_per_dst=[0, 6132, 10, 162, 4, 3256, 8272, 434] combine_rows_per_src=[12, 8000, 5952, 8192, 766, 8192, 8192, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=201621504 dispatch_meta_bytes=229792 combine_payload_bytes=201621504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201851296 ingress_bytes=220495872 link_bytes=220495872 token_bytes=403472800 weight_movement_bytes=18874368 total_nvlink_bytes=422347168 weight_to_token_ratio=0.046780 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112682955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=193773568 dispatch_meta_bytes=211792 combine_payload_bytes=193773568 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=193985360 ingress_bytes=269271040 link_bytes=269271040 token_bytes=387758928 weight_movement_bytes=75497472 total_nvlink_bytes=463256400 weight_to_token_ratio=0.194702 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580022616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=17520393352 prefetch_bytes=887095296 reduce_bytes=547356672 weight_movement_bytes=1434451968 egress_bytes=8765643912 ingress_bytes=10189201408 link_bytes=10189201408 total_nvlink_bytes=18954845320 weight_to_token_ratio=0.081873 prefetch_share_of_ingress=0.087062 dedup_ratio=0.211517 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=29 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.088469 gemm_inner_pad_upper_bound=63500 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.033914885e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=160997376 dispatch_meta_bytes=228568 combine_payload_bytes=160997376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=161225944 ingress_bytes=217620480 link_bytes=217620480 token_bytes=322223320 weight_movement_bytes=56623104 total_nvlink_bytes=378846424 weight_to_token_ratio=0.175726 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.085132363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19757675264 prefetch_bytes=887095296 reduce_bytes=905969664 weight_movement_bytes=1793064960 egress_bytes=9884316416 ingress_bytes=11666423808 link_bytes=11666423808 total_nvlink_bytes=21550740224 weight_to_token_ratio=0.090753 prefetch_share_of_ingress=0.076038 dedup_ratio=0.125606 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=48 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.089699 gemm_inner_pad_upper_bound=62992 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.328790264e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=16563037016 prefetch_bytes=811597824 reduce_bytes=1170210816 weight_movement_bytes=1981808640 egress_bytes=8286520152 ingress_bytes=10258325504 link_bytes=10258325504 total_nvlink_bytes=18544845656 weight_to_token_ratio=0.119652 prefetch_share_of_ingress=0.079116 dedup_ratio=0.236980 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=62 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.087601 gemm_inner_pad_upper_bound=66421 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.047713074e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55204 payload_rows_remote=47016 dup_rows=10332 dup_rows_remote=10324 dedup_ratio=0.157654 router_tokens_per_expert=[0, 0, 0, 0, 0, 2926, 0, 0, 0, 3, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1534, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0] payload_rows_per_dst=[8192, 6744, 7488, 102, 8192, 8192, 8188, 8106] dup_rows_per_dst=[5808, 784, 0, 0, 8, 0, 8, 3724] combine_rows_per_src=[8192, 6744, 7488, 102, 8192, 8192, 8188, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=192577536 dispatch_meta_bytes=229360 combine_payload_bytes=192577536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=192806896 ingress_bytes=211451904 link_bytes=211451904 token_bytes=385384432 weight_movement_bytes=18874368 total_nvlink_bytes=404258800 weight_to_token_ratio=0.048975 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.026028103e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50090 payload_rows_remote=41898 dup_rows=15446 dup_rows_remote=7620 dedup_ratio=0.235687 router_tokens_per_expert=[0, 0, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 4048, 0, 0, 0, 0, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 4096, 0, 0, 25, 0, 0, 0, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122] payload_rows_per_dst=[7366, 5436, 4354, 7216, 1238, 8192, 8192, 8096] dup_rows_per_dst=[0, 220, 0, 1050, 0, 4106, 7826, 2244] combine_rows_per_src=[7366, 5436, 4354, 7216, 1238, 8192, 8192, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=171614208 dispatch_meta_bytes=198072 combine_payload_bytes=171614208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171812280 ingress_bytes=190488576 link_bytes=190488576 token_bytes=343426488 weight_movement_bytes=18874368 total_nvlink_bytes=362300856 weight_to_token_ratio=0.054959 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=12160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.825167809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49976 payload_rows_remote=41784 dup_rows=15560 dup_rows_remote=8116 dedup_ratio=0.237427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3731, 254, 0, 0, 0, 0, 38, 0] payload_rows_per_dst=[44, 800, 8192, 8186, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 8, 0, 0, 0, 62, 7444, 8046] combine_rows_per_src=[44, 800, 8192, 8186, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=171147264 dispatch_meta_bytes=199600 combine_payload_bytes=171147264 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=171346864 ingress_bytes=190021632 link_bytes=190021632 token_bytes=342494128 weight_movement_bytes=18874368 total_nvlink_bytes=361368496 weight_to_token_ratio=0.055109 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.820693781e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59546 payload_rows_remote=51354 dup_rows=5990 dup_rows_remote=2440 dedup_ratio=0.091400 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 0, 0, 0, 45, 0, 0, 0, 1957, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8112, 6840, 8192, 7896, 8192, 8192, 3930] dup_rows_per_dst=[0, 0, 0, 2300, 0, 30, 3550, 110] combine_rows_per_src=[8192, 8112, 6840, 8192, 7896, 8192, 8192, 3930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=210345984 dispatch_meta_bytes=215176 combine_payload_bytes=210345984 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=210561160 ingress_bytes=323592192 link_bytes=323592192 token_bytes=420907144 weight_movement_bytes=113246208 total_nvlink_bytes=534153352 weight_to_token_ratio=0.269053 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=14336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.100501166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56998 payload_rows_remote=48806 dup_rows=8538 dup_rows_remote=8480 dedup_ratio=0.130280 router_tokens_per_expert=[0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 246, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 4063, 1, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2593, 0, 0, 0, 0, 0, 1513, 0, 0] payload_rows_per_dst=[128, 8192, 8190, 8098, 8126, 8192, 8192, 7880] dup_rows_per_dst=[0, 6, 7350, 746, 2, 42, 58, 334] combine_rows_per_src=[128, 8192, 8190, 8098, 8126, 8192, 8192, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=199909376 dispatch_meta_bytes=229144 combine_payload_bytes=199909376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=200138520 ingress_bytes=256532480 link_bytes=256532480 token_bytes=400047896 weight_movement_bytes=56623104 total_nvlink_bytes=456671000 weight_to_token_ratio=0.141541 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.090278 gemm_active_groups=12 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.457968002e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=18056722120 prefetch_bytes=924844032 reduce_bytes=811597824 weight_movement_bytes=1736441856 egress_bytes=9033750216 ingress_bytes=10759413760 link_bytes=10759413760 total_nvlink_bytes=19793163976 weight_to_token_ratio=0.096166 prefetch_share_of_ingress=0.085957 dedup_ratio=0.192389 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=43 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.087818 gemm_inner_pad_upper_bound=68072 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.147737681e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none + [2026-08-29 23:37:42] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 781.2 | throughput per GPU (TFLOP/s/GPU): 114.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008774E+01 | loss scale: 1.0 | grad norm: 180.998 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44986 payload_rows_remote=39002 dup_rows=20550 dup_rows_remote=17104 dedup_ratio=0.313568 router_tokens_per_expert=[729, 10, 13, 64, 1300, 519, 613, 61, 54, 15, 233, 276, 100, 200, 279, 324, 54, 52, 93, 467, 339, 273, 81, 22, 297, 587, 83, 82, 86, 103, 116, 149, 418, 193, 22, 235, 112, 164, 398, 50, 69, 150, 119, 954, 533, 21, 96, 399, 187, 187, 30, 44, 101, 453, 80, 70, 239, 74, 103, 264, 72, 392, 53, 63, 685, 230, 122, 42, 210, 11, 205, 753, 45, 188, 421, 267, 27, 677, 118, 1528, 500, 49, 62, 397, 175, 189, 32, 5, 296, 145, 237, 641, 98, 392, 173, 24, 6, 223, 730, 67, 173, 649, 979, 99, 699, 87, 207, 145, 463, 529, 298, 24, 556, 684, 177, 77, 357, 52, 617, 116, 90, 998, 192, 144, 58, 11, 224, 74] payload_rows_per_dst=[4926, 6094, 5500, 4984, 5826, 5984, 5798, 5874] dup_rows_per_dst=[2054, 2730, 2366, 1798, 2176, 3446, 3000, 2980] combine_rows_per_src=[4926, 6094, 5500, 4984, 5826, 5984, 5798, 5874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44060 payload_rows_remote=37102 dup_rows=21476 dup_rows_remote=16116 dedup_ratio=0.327698 router_tokens_per_expert=[1090, 3, 5, 33, 1222, 460, 629, 27, 22, 37, 245, 424, 160, 293, 221, 363, 69, 49, 47, 313, 374, 192, 84, 8, 292, 592, 104, 110, 52, 62, 205, 212, 560, 261, 35, 171, 63, 124, 461, 88, 146, 143, 67, 831, 609, 26, 136, 389, 87, 115, 62, 43, 97, 434, 70, 42, 250, 53, 147, 362, 129, 495, 57, 64, 569, 271, 119, 34, 172, 16, 158, 925, 33, 224, 554, 310, 29, 596, 190, 1449, 464, 67, 129, 332, 127, 174, 77, 9, 217, 179, 121, 690, 104, 269, 140, 47, 5, 271, 688, 69, 84, 842, 1018, 123, 629, 109, 182, 107, 378, 494, 394, 25, 544, 608, 328, 62, 470, 29, 443, 104, 105, 626, 182, 116, 51, 9, 234, 28] payload_rows_per_dst=[6958, 5806, 5722, 5848, 5006, 4452, 5430, 4838] dup_rows_per_dst=[5360, 2622, 2878, 2886, 1686, 1840, 2674, 1530] combine_rows_per_src=[6958, 5806, 5722, 5848, 5006, 4452, 5430, 4838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45032 payload_rows_remote=38884 dup_rows=20504 dup_rows_remote=17318 dedup_ratio=0.312866 router_tokens_per_expert=[974, 7, 15, 20, 1260, 512, 568, 17, 14, 45, 265, 294, 103, 191, 180, 344, 17, 45, 112, 237, 283, 198, 159, 16, 441, 606, 64, 135, 17, 70, 107, 200, 495, 242, 63, 154, 118, 81, 556, 49, 251, 100, 87, 757, 528, 14, 127, 404, 144, 165, 39, 88, 158, 398, 148, 48, 367, 60, 121, 318, 103, 673, 48, 51, 687, 324, 112, 28, 177, 12, 254, 790, 58, 191, 422, 306, 16, 783, 130, 1477, 513, 36, 84, 353, 97, 231, 63, 14, 241, 142, 149, 639, 45, 405, 193, 36, 3, 235, 748, 41, 140, 912, 1052, 102, 509, 114, 165, 121, 389, 467, 349, 19, 505, 665, 262, 69, 376, 33, 407, 93, 75, 787, 190, 141, 42, 24, 240, 14] payload_rows_per_dst=[5032, 5844, 5620, 6148, 6014, 5850, 5092, 5432] dup_rows_per_dst=[2066, 2524, 2432, 3186, 2566, 3152, 2164, 2414] combine_rows_per_src=[5032, 5844, 5620, 6148, 6014, 5850, 5092, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44962 payload_rows_remote=39120 dup_rows=20574 dup_rows_remote=18044 dedup_ratio=0.313934 router_tokens_per_expert=[1111, 29, 22, 56, 1145, 391, 558, 37, 22, 29, 222, 291, 194, 306, 344, 308, 57, 41, 49, 254, 281, 240, 85, 28, 256, 509, 113, 102, 18, 80, 210, 212, 580, 356, 72, 160, 111, 111, 517, 82, 172, 112, 110, 705, 621, 22, 216, 372, 133, 198, 60, 59, 79, 433, 86, 77, 281, 57, 142, 339, 73, 528, 56, 74, 684, 275, 139, 35, 125, 14, 206, 789, 33, 157, 463, 356, 21, 755, 134, 1359, 404, 100, 65, 367, 133, 233, 50, 16, 206, 168, 162, 663, 63, 343, 222, 64, 8, 302, 592, 70, 118, 965, 991, 130, 502, 127, 192, 179, 381, 362, 374, 37, 497, 659, 392, 72, 398, 44, 481, 99, 68, 656, 170, 141, 41, 32, 259, 31] payload_rows_per_dst=[5410, 5618, 5766, 5306, 5842, 5742, 5866, 5412] dup_rows_per_dst=[2430, 2170, 2872, 2026, 2530, 3066, 2812, 2668] combine_rows_per_src=[5410, 5618, 5766, 5306, 5842, 5742, 5866, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45004 payload_rows_remote=39268 dup_rows=20532 dup_rows_remote=18602 dedup_ratio=0.313293 router_tokens_per_expert=[838, 12, 12, 14, 1345, 450, 573, 30, 18, 22, 197, 395, 189, 236, 301, 317, 65, 41, 96, 217, 289, 157, 100, 19, 294, 414, 172, 70, 46, 80, 122, 176, 450, 208, 30, 137, 154, 140, 557, 145, 101, 130, 143, 785, 592, 10, 96, 472, 67, 209, 66, 65, 210, 430, 69, 32, 208, 54, 140, 347, 66, 533, 43, 61, 773, 259, 115, 56, 117, 8, 195, 916, 33, 165, 498, 212, 12, 716, 127, 1475, 493, 68, 49, 417, 159, 235, 37, 11, 240, 153, 230, 649, 70, 266, 160, 38, 3, 134, 610, 63, 93, 910, 954, 146, 743, 144, 265, 144, 412, 453, 395, 21, 643, 834, 267, 81, 472, 45, 417, 99, 136, 657, 236, 113, 32, 16, 201, 20] payload_rows_per_dst=[6052, 5736, 5698, 6074, 5862, 4580, 5238, 5764] dup_rows_per_dst=[3948, 1930, 2602, 2854, 2440, 1970, 2014, 2774] combine_rows_per_src=[6052, 5736, 5698, 6074, 5862, 4580, 5238, 5764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45012 payload_rows_remote=39418 dup_rows=20524 dup_rows_remote=17986 dedup_ratio=0.313171 router_tokens_per_expert=[833, 10, 6, 27, 1288, 378, 570, 39, 34, 23, 225, 318, 174, 270, 209, 416, 57, 36, 80, 287, 259, 212, 143, 11, 292, 420, 79, 95, 39, 79, 186, 185, 447, 270, 16, 143, 86, 98, 415, 52, 156, 106, 127, 876, 617, 17, 170, 470, 155, 159, 56, 60, 148, 436, 80, 78, 288, 47, 184, 425, 154, 548, 42, 58, 714, 248, 129, 37, 149, 17, 190, 828, 25, 203, 511, 318, 26, 819, 152, 1624, 428, 64, 77, 260, 93, 245, 74, 5, 238, 159, 168, 590, 38, 305, 152, 35, 12, 244, 836, 88, 95, 912, 847, 167, 565, 135, 149, 249, 446, 498, 430, 26, 485, 814, 273, 23, 409, 19, 429, 111, 113, 715, 160, 96, 42, 15, 165, 15] payload_rows_per_dst=[5110, 5834, 5594, 6072, 6034, 5512, 5438, 5418] dup_rows_per_dst=[2186, 2334, 2538, 3282, 2698, 2694, 2442, 2350] combine_rows_per_src=[5110, 5834, 5594, 6072, 6034, 5512, 5438, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45264 payload_rows_remote=39224 dup_rows=20272 dup_rows_remote=17020 dedup_ratio=0.309326 router_tokens_per_expert=[884, 6, 9, 5, 1480, 457, 515, 22, 21, 35, 200, 319, 110, 235, 196, 343, 30, 46, 94, 274, 338, 176, 188, 10, 275, 410, 64, 104, 54, 55, 109, 174, 437, 206, 33, 120, 127, 64, 574, 65, 108, 109, 94, 928, 544, 20, 97, 388, 86, 159, 47, 93, 184, 502, 195, 73, 208, 60, 139, 378, 40, 551, 36, 64, 829, 263, 112, 22, 163, 17, 179, 808, 58, 238, 514, 242, 14, 598, 188, 1529, 442, 41, 103, 436, 125, 223, 76, 11, 229, 117, 205, 420, 91, 302, 136, 38, 6, 205, 712, 39, 270, 1022, 942, 106, 590, 85, 159, 84, 294, 572, 270, 30, 646, 805, 212, 67, 365, 29, 551, 102, 155, 1088, 214, 163, 43, 20, 171, 15] payload_rows_per_dst=[4870, 5284, 5608, 5290, 6134, 5922, 6116, 6040] dup_rows_per_dst=[1844, 1764, 2220, 2224, 3168, 3028, 2772, 3252] combine_rows_per_src=[4870, 5284, 5608, 5290, 6134, 5922, 6116, 6040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=224424 combine_payload_bytes=159752192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159976616 ingress_bytes=178626560 link_bytes=178626560 token_bytes=319728808 weight_movement_bytes=18874368 total_nvlink_bytes=338603176 weight_to_token_ratio=0.059032 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=7552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.711511808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=151969792 dispatch_meta_bytes=212872 combine_payload_bytes=151969792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=152182664 ingress_bytes=189718528 link_bytes=189718528 token_bytes=304152456 weight_movement_bytes=37748736 total_nvlink_bytes=341901192 weight_to_token_ratio=0.124111 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.817789588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=224808 combine_payload_bytes=159268864 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=159493672 ingress_bytes=197017600 link_bytes=197017600 token_bytes=318762536 weight_movement_bytes=37748736 total_nvlink_bytes=356511272 weight_to_token_ratio=0.118423 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.083333 gemm_active_groups=19 gemm_peak_group_rows=7680 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.887725704e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160235520 dispatch_meta_bytes=228656 combine_payload_bytes=160235520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=160464176 ingress_bytes=216858624 link_bytes=216858624 token_bytes=320699696 weight_movement_bytes=56623104 total_nvlink_bytes=377322800 weight_to_token_ratio=0.176561 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=5760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.077832634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160841728 dispatch_meta_bytes=231480 combine_payload_bytes=160841728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=161073208 ingress_bytes=179716096 link_bytes=179716096 token_bytes=321914936 weight_movement_bytes=18874368 total_nvlink_bytes=340789304 weight_to_token_ratio=0.058632 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=11648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.721951206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161456128 dispatch_meta_bytes=229616 combine_payload_bytes=161456128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=161685744 ingress_bytes=180330496 link_bytes=180330496 token_bytes=323141872 weight_movement_bytes=18874368 total_nvlink_bytes=342016240 weight_to_token_ratio=0.058409 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=6784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.727838085e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160661504 dispatch_meta_bytes=224976 combine_payload_bytes=160661504 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=160886480 ingress_bytes=217284608 link_bytes=217284608 token_bytes=321547984 weight_movement_bytes=56623104 total_nvlink_bytes=378171088 weight_to_token_ratio=0.176095 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=5760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.081914203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53738 payload_rows_remote=51192 dup_rows=11798 dup_rows_remote=11614 dedup_ratio=0.180023 router_tokens_per_expert=[0, 0, 2, 0, 599, 0, 0, 187, 1, 0, 0, 0, 1, 0, 0, 26, 214, 0, 0, 1, 0, 83, 0, 0, 0, 0, 0, 3796, 0, 0, 65, 1, 0, 0, 0, 71, 1, 0, 2, 0, 1, 0, 0, 4050, 0, 205, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 329, 6, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 1, 0, 0, 6, 136, 0, 0, 253, 3428, 0, 4069, 26, 3972, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2264, 0, 1877, 0, 1, 1834, 4, 0, 0, 4, 0, 0, 0, 0, 0, 2, 3621, 0, 0, 0] payload_rows_per_dst=[8036, 7736, 8110, 8138, 3678, 2546, 7660, 7834] dup_rows_per_dst=[1540, 584, 550, 6, 238, 184, 5598, 3098] combine_rows_per_src=[8036, 7736, 8110, 8138, 3678, 2546, 7660, 7834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51140 payload_rows_remote=43980 dup_rows=14396 dup_rows_remote=12960 dedup_ratio=0.219666 router_tokens_per_expert=[0, 0, 75, 0, 695, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 64, 375, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 3564, 0, 0, 512, 0, 0, 0, 4, 18, 6, 0, 0, 0, 2, 0, 0, 3992, 0, 183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 973, 0, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 228, 0, 0, 188, 3791, 0, 4086, 0, 3415, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1661, 0, 928, 0, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3238, 0, 0, 0] payload_rows_per_dst=[7160, 7718, 7690, 8174, 7762, 8014, 4592, 30] dup_rows_per_dst=[1436, 1696, 364, 438, 4594, 5280, 588, 0] combine_rows_per_src=[7160, 7718, 7690, 8174, 7762, 8014, 4592, 30] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55050 payload_rows_remote=51000 dup_rows=10486 dup_rows_remote=9894 dedup_ratio=0.160004 router_tokens_per_expert=[0, 0, 24, 0, 1108, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 25, 796, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 3468, 0, 0, 132, 0, 0, 0, 1, 79, 2, 0, 0, 0, 3, 0, 0, 4054, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 1, 1, 295, 0, 0, 516, 3519, 2, 4072, 6, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 2, 1741, 0, 0, 1591, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 3775, 0, 0, 0] payload_rows_per_dst=[7986, 7568, 8136, 8144, 4050, 7402, 4440, 7324] dup_rows_per_dst=[2354, 1518, 1532, 0, 592, 1280, 424, 2786] combine_rows_per_src=[7986, 7568, 8136, 8144, 4050, 7402, 4440, 7324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55644 payload_rows_remote=47526 dup_rows=9892 dup_rows_remote=9872 dedup_ratio=0.150940 router_tokens_per_expert=[0, 29, 138, 0, 980, 0, 0, 94, 0, 0, 0, 0, 2, 0, 0, 74, 377, 0, 0, 3, 0, 39, 0, 0, 0, 0, 0, 3106, 0, 0, 193, 0, 0, 0, 0, 265, 3, 0, 9, 1, 6, 0, 0, 4087, 0, 325, 0, 0, 0, 0, 2, 1, 0, 3, 0, 0, 1, 1, 0, 4, 0, 0, 0, 1, 0, 916, 0, 1959, 0, 0, 5, 0, 0, 2, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 19, 1, 148, 7, 2, 109, 3694, 4, 4056, 23, 3832, 0, 5, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 535, 0, 1231, 0, 0, 2984, 3, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3466, 0, 0, 0] payload_rows_per_dst=[7954, 6748, 8178, 8118, 7792, 7520, 3338, 5996] dup_rows_per_dst=[2344, 688, 1214, 20, 4910, 494, 208, 14] combine_rows_per_src=[7954, 6748, 8178, 8118, 7792, 7520, 3338, 5996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57700 payload_rows_remote=49574 dup_rows=7836 dup_rows_remote=6936 dedup_ratio=0.119568 router_tokens_per_expert=[0, 1, 1, 0, 838, 0, 0, 307, 0, 0, 0, 0, 0, 4, 0, 9, 108, 0, 1, 0, 0, 20, 0, 0, 0, 0, 0, 3691, 0, 0, 38, 0, 0, 0, 6, 227, 1, 0, 1, 3, 25, 0, 0, 4042, 0, 207, 1, 0, 0, 0, 0, 24, 0, 1, 4, 0, 1, 0, 0, 0, 0, 4, 7, 0, 3, 467, 0, 1135, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 1, 0, 4, 1, 1, 0, 113, 0, 0, 127, 3635, 8, 3980, 11, 3930, 0, 1, 0, 2, 1, 0, 0, 0, 0, 0, 1, 0, 1587, 0, 2045, 0, 0, 2601, 11, 0, 0, 0, 2, 0, 1, 0, 1, 5, 3502, 0, 1, 0] payload_rows_per_dst=[7958, 7456, 8126, 7992, 7524, 7392, 6016, 5236] dup_rows_per_dst=[2222, 260, 900, 50, 2728, 410, 1258, 8] combine_rows_per_src=[7958, 7456, 8126, 7992, 7524, 7392, 6016, 5236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52480 payload_rows_remote=45610 dup_rows=13056 dup_rows_remote=12318 dedup_ratio=0.199219 router_tokens_per_expert=[4, 0, 60, 0, 1213, 0, 0, 183, 0, 0, 0, 0, 1, 0, 0, 28, 298, 0, 0, 2, 0, 23, 0, 0, 0, 0, 0, 3202, 3, 0, 276, 0, 0, 0, 0, 54, 0, 0, 0, 0, 14, 0, 0, 3788, 0, 95, 0, 0, 16, 0, 0, 2, 0, 61, 0, 1, 0, 0, 0, 0, 0, 0, 15, 0, 0, 247, 1, 1807, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 1, 300, 0, 0, 175, 3647, 32, 4078, 7, 3460, 0, 2, 0, 3, 0, 1, 0, 2, 0, 2, 1, 0, 1191, 0, 1835, 0, 5, 3095, 0, 0, 1, 3, 0, 0, 0, 3, 0, 8, 3496, 0, 1, 0] payload_rows_per_dst=[7638, 6870, 7620, 8176, 7606, 8024, 5208, 1338] dup_rows_per_dst=[2260, 738, 282, 170, 3500, 5236, 866, 4] combine_rows_per_src=[7638, 6870, 7620, 8176, 7606, 8024, 5208, 1338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49236 payload_rows_remote=41080 dup_rows=16300 dup_rows_remote=6864 dedup_ratio=0.248718 router_tokens_per_expert=[7, 0, 1, 0, 380, 0, 0, 213, 0, 0, 0, 0, 0, 0, 2, 52, 334, 1, 0, 4, 0, 12, 0, 0, 10, 0, 2, 3165, 1, 0, 243, 0, 0, 0, 0, 13, 3, 2, 1, 0, 4, 5, 1, 4074, 0, 30, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 826, 0, 2032, 0, 4, 6, 0, 0, 0, 10, 0, 0, 1, 2, 0, 0, 0, 4, 0, 1, 0, 0, 88, 0, 0, 316, 3662, 0, 4056, 1, 3781, 0, 0, 0, 7, 5, 5, 2, 0, 0, 0, 0, 0, 1022, 0, 1684, 0, 0, 3107, 1, 0, 0, 0, 0, 0, 0, 4, 0, 11, 3559, 0, 0, 0] payload_rows_per_dst=[1258, 6800, 4002, 8114, 5332, 7606, 7968, 8156] dup_rows_per_dst=[52, 744, 38, 18, 430, 776, 4806, 9436] combine_rows_per_src=[1258, 6800, 4002, 8114, 5332, 7606, 7968, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=209682432 dispatch_meta_bytes=251224 combine_payload_bytes=209682432 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=209933656 ingress_bytes=304054272 link_bytes=304054272 token_bytes=419616088 weight_movement_bytes=94371840 total_nvlink_bytes=513987928 weight_to_token_ratio=0.224900 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.913298430e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=180142080 dispatch_meta_bytes=227760 combine_payload_bytes=180142080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=180369840 ingress_bytes=199016448 link_bytes=199016448 token_bytes=360511920 weight_movement_bytes=18874368 total_nvlink_bytes=379386288 weight_to_token_ratio=0.052354 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.906877683e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=208896000 dispatch_meta_bytes=243576 combine_payload_bytes=208896000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209139576 ingress_bytes=227770368 link_bytes=227770368 token_bytes=418035576 weight_movement_bytes=18874368 total_nvlink_bytes=436909944 weight_to_token_ratio=0.045150 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=14080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182383596e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=194666496 dispatch_meta_bytes=229592 combine_payload_bytes=194666496 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=194896088 ingress_bytes=232415232 link_bytes=232415232 token_bytes=389562584 weight_movement_bytes=37748736 total_nvlink_bytes=427311320 weight_to_token_ratio=0.096900 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226888398e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=203055104 dispatch_meta_bytes=226040 combine_payload_bytes=203055104 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=203281144 ingress_bytes=240803840 link_bytes=240803840 token_bytes=406336248 weight_movement_bytes=37748736 total_nvlink_bytes=444084984 weight_to_token_ratio=0.092900 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307263912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=186818560 dispatch_meta_bytes=231712 combine_payload_bytes=186818560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=187050272 ingress_bytes=205692928 link_bytes=205692928 token_bytes=373868832 weight_movement_bytes=18874368 total_nvlink_bytes=392743200 weight_to_token_ratio=0.050484 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.970848429e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=168263680 dispatch_meta_bytes=191776 combine_payload_bytes=168263680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=168455456 ingress_bytes=224886784 link_bytes=224886784 token_bytes=336719136 weight_movement_bytes=56623104 total_nvlink_bytes=393342240 weight_to_token_ratio=0.168161 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.154754513e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62534 payload_rows_remote=54342 dup_rows=3002 dup_rows_remote=2954 dedup_ratio=0.045807 router_tokens_per_expert=[0, 1, 1, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 809, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 308, 0, 20, 0, 0, 4087, 1, 0, 0, 0, 42, 4077, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 4095, 65, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 0, 0, 0, 0, 0, 1, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0] payload_rows_per_dst=[5406, 8182, 8156, 8068, 8190, 8192, 8150, 8190] dup_rows_per_dst=[30, 654, 88, 88, 160, 48, 200, 1734] combine_rows_per_src=[5406, 8182, 8156, 8068, 8190, 8192, 8150, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56528 payload_rows_remote=48340 dup_rows=9008 dup_rows_remote=5424 dedup_ratio=0.137451 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 0, 52, 0, 3, 4032, 0, 0, 0, 0, 1, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 8, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0] payload_rows_per_dst=[8188, 8174, 8086, 8108, 5882, 7944, 7970, 2176] dup_rows_per_dst=[3584, 2486, 2, 2594, 252, 34, 28, 28] combine_rows_per_src=[8188, 8174, 8086, 8108, 5882, 7944, 7970, 2176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58772 payload_rows_remote=50580 dup_rows=6764 dup_rows_remote=6272 dedup_ratio=0.103210 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 1, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 60, 0, 1, 4066, 0, 0, 0, 0, 42, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 4096, 212, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 5, 0, 0, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0] payload_rows_per_dst=[1740, 8146, 7996, 8130, 8192, 8192, 8184, 8192] dup_rows_per_dst=[6, 260, 74, 330, 492, 54, 1088, 4460] combine_rows_per_src=[1740, 8146, 7996, 8130, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59204 payload_rows_remote=51596 dup_rows=6332 dup_rows_remote=6282 dedup_ratio=0.096619 router_tokens_per_expert=[0, 0, 2, 0, 2, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 126, 0, 0, 3996, 0, 0, 0, 0, 6, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 4082, 0, 0, 0, 0, 0, 0, 2, 0, 0, 26, 0, 0, 0, 4096, 170, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 1, 0, 30, 1, 0, 0, 0, 0, 0, 536, 0, 4096, 0, 0, 1, 0, 1, 0, 2, 0, 0, 967, 0, 0] payload_rows_per_dst=[6924, 8116, 7218, 7608, 8192, 8190, 8176, 4780] dup_rows_per_dst=[3012, 1284, 4, 50, 436, 48, 598, 900] combine_rows_per_src=[6924, 8116, 7218, 7608, 8192, 8190, 8176, 4780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=222584832 dispatch_meta_bytes=229184 combine_payload_bytes=222584832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222814016 ingress_bytes=241459200 link_bytes=241459200 token_bytes=445398848 weight_movement_bytes=18874368 total_nvlink_bytes=464273216 weight_to_token_ratio=0.042376 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.313543249e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59310 payload_rows_remote=51172 dup_rows=6226 dup_rows_remote=6178 dedup_ratio=0.095001 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 70, 0, 0, 4042, 0, 0, 0, 0, 26, 4066, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 3802, 0, 0, 6, 0, 0, 0, 0, 0, 11, 19, 0, 0, 0, 4091, 10, 0, 4092, 6, 0, 2, 0, 0, 0, 0, 0, 15, 0, 0, 2, 0, 1, 6, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1414, 0, 4093, 1, 0, 0, 0, 2, 0, 0, 0, 0, 343, 0, 0] payload_rows_per_dst=[8188, 8134, 8138, 7404, 8190, 8192, 7616, 3448] dup_rows_per_dst=[4014, 1844, 48, 38, 58, 32, 120, 72] combine_rows_per_src=[8188, 8134, 8138, 7404, 8190, 8192, 7616, 3448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55660 payload_rows_remote=47472 dup_rows=9876 dup_rows_remote=802 dedup_ratio=0.150696 router_tokens_per_expert=[0, 1, 5, 0, 11, 6, 2143, 0, 0, 6, 0, 1, 0, 0, 0, 757, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1358, 0, 151, 1, 0, 4068, 2, 0, 0, 4, 169, 3945, 0, 0, 2, 0, 22, 0, 1, 7, 0, 3, 0, 3871, 1, 3386, 2, 14, 7, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 4065, 4, 0, 4084, 0, 0, 1, 0, 4, 3, 0, 0, 27, 0, 1, 0, 0, 0, 0, 0, 9, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 7, 11, 0, 0, 0, 0, 0, 1, 0, 0, 9, 8, 3, 372, 0, 4077, 0, 0, 2, 8, 0, 0, 0, 0, 0, 101, 0, 0] payload_rows_per_dst=[5572, 2992, 7986, 7772, 8148, 8178, 6824, 8188] dup_rows_per_dst=[288, 34, 324, 36, 62, 18, 40, 9074] combine_rows_per_src=[5572, 2992, 7986, 7772, 8148, 8178, 6824, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44842 payload_rows_remote=39084 dup_rows=20694 dup_rows_remote=17772 dedup_ratio=0.315765 router_tokens_per_expert=[897, 2, 15, 25, 1177, 537, 518, 29, 42, 30, 218, 308, 187, 136, 316, 385, 72, 48, 63, 329, 305, 238, 178, 14, 315, 889, 169, 77, 27, 80, 125, 184, 459, 270, 32, 197, 97, 74, 506, 88, 98, 80, 87, 887, 638, 15, 114, 418, 129, 215, 45, 104, 104, 411, 157, 30, 293, 103, 149, 396, 126, 570, 63, 56, 709, 328, 91, 25, 131, 8, 183, 775, 41, 200, 437, 286, 13, 460, 90, 1510, 437, 60, 90, 369, 67, 241, 96, 3, 308, 161, 130, 516, 61, 287, 132, 14, 6, 182, 566, 89, 172, 815, 828, 119, 654, 98, 219, 134, 358, 605, 294, 29, 704, 795, 195, 48, 460, 31, 555, 119, 91, 793, 173, 141, 37, 29, 211, 13] payload_rows_per_dst=[5180, 6044, 5658, 5366, 5564, 5560, 5758, 5712] dup_rows_per_dst=[2110, 3202, 2462, 2192, 1990, 2738, 2922, 3078] combine_rows_per_src=[5180, 6044, 5658, 5366, 5564, 5560, 5758, 5712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=198000640 dispatch_meta_bytes=215056 combine_payload_bytes=198000640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198215696 ingress_bytes=216875008 link_bytes=216875008 token_bytes=396216336 weight_movement_bytes=18874368 total_nvlink_bytes=415090704 weight_to_token_ratio=0.047637 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.077989617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58892 payload_rows_remote=50834 dup_rows=6644 dup_rows_remote=5834 dedup_ratio=0.101379 router_tokens_per_expert=[0, 0, 0, 0, 16, 0, 2019, 0, 0, 2, 1, 6, 0, 0, 0, 635, 0, 0, 2, 0, 2, 0, 3, 0, 0, 0, 415, 4, 11, 0, 9, 3988, 0, 1, 0, 3, 6, 3947, 0, 5, 1, 0, 7, 0, 0, 0, 0, 6, 0, 3731, 3, 4031, 0, 0, 0, 1, 3, 1, 0, 0, 0, 18, 0, 1, 0, 4023, 41, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 9, 2, 0, 0, 0, 0, 7, 0, 0, 28, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 5, 878, 0, 0] payload_rows_per_dst=[8186, 8058, 7926, 7478, 8108, 8168, 8078, 2890] dup_rows_per_dst=[5346, 810, 26, 38, 158, 44, 62, 160] combine_rows_per_src=[8186, 8058, 7926, 7478, 8108, 8168, 8078, 2890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=207175680 dispatch_meta_bytes=227408 combine_payload_bytes=207175680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=207403088 ingress_bytes=263798784 link_bytes=263798784 token_bytes=414578768 weight_movement_bytes=56623104 total_nvlink_bytes=471201872 weight_to_token_ratio=0.136580 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.527590151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=211337216 dispatch_meta_bytes=231512 combine_payload_bytes=211337216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=211568728 ingress_bytes=249085952 link_bytes=249085952 token_bytes=422905944 weight_movement_bytes=37748736 total_nvlink_bytes=460654680 weight_to_token_ratio=0.089260 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.386619034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=209600512 dispatch_meta_bytes=229400 combine_payload_bytes=209600512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209829912 ingress_bytes=228474880 link_bytes=228474880 token_bytes=419430424 weight_movement_bytes=18874368 total_nvlink_bytes=438304792 weight_to_token_ratio=0.045000 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.065972 gemm_active_groups=15 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189133884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=194445312 dispatch_meta_bytes=193096 combine_payload_bytes=194445312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=194638408 ingress_bytes=232194048 link_bytes=232194048 token_bytes=389083720 weight_movement_bytes=37748736 total_nvlink_bytes=426832456 weight_to_token_ratio=0.097020 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224769121e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=208216064 dispatch_meta_bytes=226672 combine_payload_bytes=208216064 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=208442736 ingress_bytes=264839168 link_bytes=264839168 token_bytes=416658800 weight_movement_bytes=56623104 total_nvlink_bytes=473281904 weight_to_token_ratio=0.135898 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.537558599e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59704 payload_rows_remote=51524 dup_rows=5832 dup_rows_remote=5830 dedup_ratio=0.088989 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 21, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 3313, 18, 340, 0, 0, 0, 4091, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1746, 0, 0, 0, 0, 1, 0, 0, 3464, 0, 0, 0, 0, 0, 4052, 0, 0, 2, 0, 551, 0, 0, 0, 2, 0, 0, 0, 1, 1] payload_rows_per_dst=[8184, 8186, 7714, 8190, 6626, 8180, 4482, 8142] dup_rows_per_dst=[0, 36, 3706, 528, 0, 2, 484, 1076] combine_rows_per_src=[8184, 8186, 7714, 8190, 6626, 8180, 4482, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=227424 combine_payload_bytes=160088064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=160315488 ingress_bytes=216711168 link_bytes=216711168 token_bytes=320403552 weight_movement_bytes=56623104 total_nvlink_bytes=377026656 weight_to_token_ratio=0.176724 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=5504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.076419783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55858 payload_rows_remote=47666 dup_rows=9678 dup_rows_remote=9674 dedup_ratio=0.147675 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 750, 0, 0, 0, 0, 0, 3208, 45, 73, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8052, 7954, 7580, 6416, 8134, 8192, 1338] dup_rows_per_dst=[4, 1108, 1440, 1718, 0, 2, 5406, 0] combine_rows_per_src=[8192, 8052, 7954, 7580, 6416, 8134, 8192, 1338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59596 payload_rows_remote=52016 dup_rows=5940 dup_rows_remote=5940 dedup_ratio=0.090637 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 3790, 5, 293, 1, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 25, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 10, 0, 0, 0, 4077, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 15] payload_rows_per_dst=[8190, 7866, 8186, 8192, 7580, 8192, 3220, 8170] dup_rows_per_dst=[0, 4, 2498, 2092, 0, 4, 26, 1316] combine_rows_per_src=[8190, 7866, 8186, 8192, 7580, 8192, 3220, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211042304 dispatch_meta_bytes=229416 combine_payload_bytes=211042304 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=211271720 ingress_bytes=229916672 link_bytes=229916672 token_bytes=422314024 weight_movement_bytes=18874368 total_nvlink_bytes=441188392 weight_to_token_ratio=0.044693 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.202948425e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57644 payload_rows_remote=49700 dup_rows=7892 dup_rows_remote=3252 dedup_ratio=0.120422 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 0, 0, 0, 0, 3447, 5, 1, 0, 6, 0, 4093, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 966, 0, 0, 0, 0, 31, 0, 0, 3242, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 2, 0, 0, 6] payload_rows_per_dst=[8186, 8106, 7944, 8192, 6896, 8116, 4808, 5396] dup_rows_per_dst=[0, 4, 4640, 2106, 0, 0, 734, 408] combine_rows_per_src=[8186, 8106, 7944, 8192, 6896, 8116, 4808, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61210 payload_rows_remote=53022 dup_rows=4326 dup_rows_remote=3492 dedup_ratio=0.066010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 0, 1, 3902, 16, 52, 1, 1, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 415, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 103, 0, 0, 0, 1, 0, 0, 46, 0, 2649, 0, 0, 0, 0, 3, 0, 0, 3437, 0, 5, 0, 0, 0, 4055, 0, 1, 3, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 35] payload_rows_per_dst=[8188, 7884, 7452, 8188, 7804, 8188, 5350, 8156] dup_rows_per_dst=[0, 12, 970, 834, 8, 200, 56, 2246] combine_rows_per_src=[8188, 7884, 7452, 8188, 7804, 8188, 5350, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50208 payload_rows_remote=42048 dup_rows=15328 dup_rows_remote=6898 dedup_ratio=0.233887 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 15, 0, 5, 0, 7, 0, 0, 0, 5, 0, 0, 0, 1, 7, 0, 0, 0, 0, 7, 4, 10, 4049, 0, 1, 0, 0, 0, 0, 0, 2250, 11, 2, 0, 0, 3, 3659, 0, 343, 0, 1, 4, 4083, 0, 7, 0, 0, 1, 2, 0, 4070, 2, 0, 325, 30, 0, 0, 0, 29, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 1, 0, 3, 4017, 0, 0, 1, 1, 0, 0, 7, 6, 2, 0, 37, 1, 1470, 0, 0, 2, 0, 0, 0, 0, 4045, 2, 0, 2, 0, 0, 4019, 9, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8136, 708, 718, 8076, 8070, 8148, 8160] dup_rows_per_dst=[36, 30, 12, 26, 3816, 8, 2970, 8430] combine_rows_per_src=[8192, 8136, 708, 718, 8076, 8070, 8148, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=195239936 dispatch_meta_bytes=229360 combine_payload_bytes=195239936 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=195469296 ingress_bytes=232988672 link_bytes=232988672 token_bytes=390709232 weight_movement_bytes=37748736 total_nvlink_bytes=428457968 weight_to_token_ratio=0.096616 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232382818e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59200 payload_rows_remote=51254 dup_rows=6336 dup_rows_remote=6250 dedup_ratio=0.096680 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 110, 0, 0, 4, 0, 1, 0, 0, 2, 0, 0, 0, 0, 3898, 0, 1, 0, 0, 1, 1, 0, 810, 0, 3, 0, 0, 0, 3954, 1, 94, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 4058, 1, 0, 367, 0, 0, 1, 1, 5, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 0, 4075, 0, 0, 8, 8, 0, 0, 14, 0, 2, 0, 0, 0, 1980, 0, 0, 0, 0, 15, 0, 0, 3769, 0, 9, 0, 0, 0, 4081, 1, 7, 0, 0, 1369, 0, 0, 0, 0, 0, 3, 0, 0, 3] payload_rows_per_dst=[8186, 7946, 7920, 8120, 7918, 8184, 8168, 2758] dup_rows_per_dst=[4, 86, 1440, 734, 18, 44, 4002, 8] combine_rows_per_src=[8186, 7946, 7920, 8120, 7918, 8184, 8168, 2758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=213057536 dispatch_meta_bytes=231824 combine_payload_bytes=213057536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=213289360 ingress_bytes=250806272 link_bytes=250806272 token_bytes=426346896 weight_movement_bytes=37748736 total_nvlink_bytes=464095632 weight_to_token_ratio=0.088540 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.076389 gemm_active_groups=12 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403102294e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203571200 dispatch_meta_bytes=211808 combine_payload_bytes=203571200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=203783008 ingress_bytes=260194304 link_bytes=260194304 token_bytes=407354208 weight_movement_bytes=56623104 total_nvlink_bytes=463977312 weight_to_token_ratio=0.139002 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.493053797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=217178112 dispatch_meta_bytes=226056 combine_payload_bytes=217178112 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=217404168 ingress_bytes=311549952 link_bytes=311549952 token_bytes=434582280 weight_movement_bytes=94371840 total_nvlink_bytes=528954120 weight_to_token_ratio=0.217155 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.985118348e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=172228608 dispatch_meta_bytes=195784 combine_payload_bytes=172228608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=172424392 ingress_bytes=209977344 link_bytes=209977344 token_bytes=344653000 weight_movement_bytes=37748736 total_nvlink_bytes=382401736 weight_to_token_ratio=0.109527 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=22528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.011899595e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=209936384 dispatch_meta_bytes=230016 combine_payload_bytes=209936384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210166400 ingress_bytes=228810752 link_bytes=228810752 token_bytes=420102784 weight_movement_bytes=18874368 total_nvlink_bytes=438977152 weight_to_token_ratio=0.044928 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.069444 gemm_active_groups=15 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.192352044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51872 payload_rows_remote=43806 dup_rows=13664 dup_rows_remote=9164 dedup_ratio=0.208496 router_tokens_per_expert=[1, 3, 7, 0, 747, 1, 0, 145, 1, 0, 0, 0, 0, 0, 0, 26, 339, 0, 1, 0, 0, 35, 0, 0, 10, 0, 0, 3289, 0, 0, 532, 0, 0, 0, 0, 161, 0, 0, 1, 0, 51, 0, 0, 4065, 2, 116, 1, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 687, 2, 1679, 0, 1, 6, 0, 0, 0, 0, 1, 0, 0, 23, 1, 0, 0, 0, 2, 0, 1, 0, 118, 8, 0, 78, 3914, 1, 3975, 25, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 964, 0, 1398, 0, 0, 3671, 5, 0, 3, 1, 0, 0, 0, 1, 0, 11, 3442, 0, 4, 0] payload_rows_per_dst=[4600, 7160, 8156, 7970, 4484, 3288, 8066, 8148] dup_rows_per_dst=[738, 1252, 638, 8, 316, 84, 4500, 6128] combine_rows_per_src=[4600, 7160, 8156, 7970, 4484, 3288, 8066, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61100 payload_rows_remote=52908 dup_rows=4436 dup_rows_remote=4420 dedup_ratio=0.067688 router_tokens_per_expert=[0, 95, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4096, 1, 0, 1491, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2494, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 4990, 7338, 8192, 8190, 7818] dup_rows_per_dst=[178, 162, 2984, 0, 0, 16, 1082, 14] combine_rows_per_src=[8192, 8188, 8192, 4990, 7338, 8192, 8190, 7818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55402 payload_rows_remote=47210 dup_rows=10134 dup_rows_remote=6034 dedup_ratio=0.154633 router_tokens_per_expert=[0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 216, 0, 0, 4096, 0, 0, 1216, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 0, 0, 11] payload_rows_per_dst=[8192, 4580, 8164, 7202, 8176, 8074, 3408, 7606] dup_rows_per_dst=[4100, 300, 2432, 3100, 0, 0, 184, 18] combine_rows_per_src=[8192, 4580, 8164, 7202, 8176, 8074, 3408, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63604 payload_rows_remote=55670 dup_rows=1932 dup_rows_remote=1932 dedup_ratio=0.029480 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 924, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7140, 7934, 8192, 8192, 7570] dup_rows_per_dst=[2, 54, 1848, 0, 0, 0, 28, 0] combine_rows_per_src=[8192, 8192, 8192, 7140, 7934, 8192, 8192, 7570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62434 payload_rows_remote=54256 dup_rows=3102 dup_rows_remote=1492 dedup_ratio=0.047333 router_tokens_per_expert=[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4093, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 524, 0, 0, 4083, 0, 0, 805, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2619, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 8] payload_rows_per_dst=[8180, 8190, 8178, 5238, 8100, 8188, 8192, 8168] dup_rows_per_dst=[14, 1116, 1610, 4, 0, 28, 310, 20] combine_rows_per_src=[8180, 8190, 8178, 5238, 8100, 8188, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=179429376 dispatch_meta_bytes=211880 combine_payload_bytes=179429376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179641256 ingress_bytes=198303744 link_bytes=198303744 token_bytes=359070632 weight_movement_bytes=18874368 total_nvlink_bytes=377945000 weight_to_token_ratio=0.052564 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=12800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.900048903e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=216711168 dispatch_meta_bytes=229312 combine_payload_bytes=216711168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216940480 ingress_bytes=235585536 link_bytes=235585536 token_bytes=433651648 weight_movement_bytes=18874368 total_nvlink_bytes=452526016 weight_to_token_ratio=0.043524 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.257264691e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55496 payload_rows_remote=47860 dup_rows=10040 dup_rows_remote=7464 dedup_ratio=0.153198 router_tokens_per_expert=[0, 3, 0, 0, 13, 0, 0, 0, 1, 0, 4037, 4066, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7, 0, 575, 0, 0, 4062, 83, 0, 1541, 4039, 0, 0, 2, 0, 0, 1, 0, 11, 0, 28, 0, 0, 1, 0, 0, 4070, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 0, 4057, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 39, 0, 12, 0, 2, 20, 22, 0, 0, 0, 11, 1134, 0, 0, 0, 0, 0, 0, 1, 11, 5, 2, 0, 20, 3, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 3] payload_rows_per_dst=[3766, 8192, 3288, 8102, 8134, 8190, 8188, 7636] dup_rows_per_dst=[4, 1206, 44, 2090, 1744, 44, 2332, 2576] combine_rows_per_src=[3766, 8192, 3288, 8102, 8134, 8190, 8188, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64310 payload_rows_remote=57118 dup_rows=1226 dup_rows_remote=1218 dedup_ratio=0.018707 router_tokens_per_expert=[0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 3, 0, 520, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 4096, 3594, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 4] payload_rows_per_dst=[8192, 8192, 8192, 7192, 8088, 8188, 8192, 8074] dup_rows_per_dst=[118, 14, 1046, 8, 0, 0, 36, 4] combine_rows_per_src=[8192, 8192, 8192, 7192, 8088, 8188, 8192, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=193372160 dispatch_meta_bytes=212976 combine_payload_bytes=193372160 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=193585136 ingress_bytes=268869632 link_bytes=268869632 token_bytes=386957296 weight_movement_bytes=75497472 total_nvlink_bytes=462454768 weight_to_token_ratio=0.195105 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576176522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63402 payload_rows_remote=55220 dup_rows=2134 dup_rows_remote=2096 dedup_ratio=0.032562 router_tokens_per_expert=[1, 81, 1, 0, 4, 17, 6, 27, 1, 1, 4033, 4074, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 0, 0, 14, 0, 3, 4085, 0, 0, 574, 4069, 0, 0, 0, 0, 0, 3, 4, 0, 3, 0, 0, 0, 0, 0, 0, 4078, 3398, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3937, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 9, 0, 0, 7, 4, 0, 3, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2, 3918, 0, 0, 1] payload_rows_per_dst=[8154, 8182, 8160, 6806, 7886, 8178, 8186, 7850] dup_rows_per_dst=[196, 38, 1152, 0, 4, 0, 738, 6] combine_rows_per_src=[8154, 8182, 8160, 6806, 7886, 8178, 8186, 7850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222232576 dispatch_meta_bytes=222992 combine_payload_bytes=222232576 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=222455568 ingress_bytes=278855680 link_bytes=278855680 token_bytes=444688144 weight_movement_bytes=56623104 total_nvlink_bytes=501311248 weight_to_token_ratio=0.127332 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.671857920e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=228024320 dispatch_meta_bytes=230408 combine_payload_bytes=228024320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228254728 ingress_bytes=246898688 link_bytes=246898688 token_bytes=456279048 weight_movement_bytes=18874368 total_nvlink_bytes=475153416 weight_to_token_ratio=0.041366 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365661747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=196034560 dispatch_meta_bytes=221296 combine_payload_bytes=196034560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196255856 ingress_bytes=214908928 link_bytes=214908928 token_bytes=392290416 weight_movement_bytes=18874368 total_nvlink_bytes=411164784 weight_to_token_ratio=0.048113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059151606e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=233955328 dispatch_meta_bytes=233344 combine_payload_bytes=233955328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234188672 ingress_bytes=271704064 link_bytes=271704064 token_bytes=468144000 weight_movement_bytes=37748736 total_nvlink_bytes=505892736 weight_to_token_ratio=0.080635 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603334655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=226181120 dispatch_meta_bytes=229264 combine_payload_bytes=226181120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226410384 ingress_bytes=245055488 link_bytes=245055488 token_bytes=452591504 weight_movement_bytes=18874368 total_nvlink_bytes=471465872 weight_to_token_ratio=0.041703 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348001111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62024 payload_rows_remote=53856 dup_rows=3512 dup_rows_remote=3512 dedup_ratio=0.053589 router_tokens_per_expert=[0, 0, 48, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 2407, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 792, 0, 0, 0, 0, 0, 4096, 1701, 0, 0, 4084, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8126, 6908, 8168, 7150, 7098] dup_rows_per_dst=[260, 54, 0, 0, 598, 0, 774, 1826] combine_rows_per_src=[8192, 8192, 8190, 8126, 6908, 8168, 7150, 7098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59480 payload_rows_remote=51318 dup_rows=6056 dup_rows_remote=5816 dedup_ratio=0.092407 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 1393, 0, 0, 8, 0, 0, 0, 0, 0, 1122, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 120, 0, 5, 4012, 0, 0, 0, 0, 108, 4071, 0, 0, 0, 1, 1, 0, 0, 3, 2, 0, 3, 3853, 0, 4080, 0, 0, 8, 2, 0, 0, 0, 0, 0, 31, 0, 0, 2, 4093, 5, 2, 4089, 0, 0, 0, 0, 0, 2, 1, 0, 23, 0, 1, 0, 0, 0, 0, 4, 1, 0, 5, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 119, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 798, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 6, 467, 0, 0] payload_rows_per_dst=[4954, 5908, 8148, 7742, 8188, 8190, 8162, 8188] dup_rows_per_dst=[98, 454, 1316, 52, 828, 12, 240, 3056] combine_rows_per_src=[4954, 5908, 8148, 7742, 8188, 8190, 8162, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61524 payload_rows_remote=53332 dup_rows=4012 dup_rows_remote=3892 dedup_ratio=0.061218 router_tokens_per_expert=[0, 0, 50, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 0, 3461, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 0, 0, 0, 0, 4096, 1139, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8058, 7370, 8192, 6922, 6426] dup_rows_per_dst=[120, 16, 0, 0, 2796, 0, 2, 1078] combine_rows_per_src=[8192, 8192, 8172, 8058, 7370, 8192, 6922, 6426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61754 payload_rows_remote=53574 dup_rows=3782 dup_rows_remote=3780 dedup_ratio=0.057709 router_tokens_per_expert=[0, 0, 157, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 6, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 0, 2084, 0, 0, 0, 1, 0, 14, 0, 0, 7, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 4096, 2225, 0, 0, 4091, 0, 0] payload_rows_per_dst=[8192, 8190, 8180, 8106, 8132, 8182, 7850, 4922] dup_rows_per_dst=[336, 86, 2, 0, 194, 0, 2956, 208] combine_rows_per_src=[8192, 8190, 8180, 8106, 8132, 8182, 7850, 4922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60106 payload_rows_remote=54022 dup_rows=5430 dup_rows_remote=4984 dedup_ratio=0.082855 router_tokens_per_expert=[0, 0, 869, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1553, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 0, 0, 0, 0, 4096, 2057, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8188, 8184, 8158, 6084, 8192, 8022, 5086] dup_rows_per_dst=[1814, 154, 0, 0, 446, 0, 2660, 356] combine_rows_per_src=[8192, 8188, 8184, 8158, 6084, 8192, 8022, 5086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=220594176 dispatch_meta_bytes=229472 combine_payload_bytes=220594176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220823648 ingress_bytes=239468544 link_bytes=239468544 token_bytes=441417824 weight_movement_bytes=18874368 total_nvlink_bytes=460292192 weight_to_token_ratio=0.042759 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294469763e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56120 payload_rows_remote=48086 dup_rows=9416 dup_rows_remote=2374 dedup_ratio=0.143677 router_tokens_per_expert=[1, 0, 3, 0, 0, 18, 20, 0, 15, 0, 0, 12, 0, 0, 0, 0, 1, 4012, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 2, 10, 6, 0, 19, 0, 0, 1, 0, 10, 26, 0, 1, 16, 2, 4, 0, 0, 4048, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 3902, 0, 0, 1, 0, 100, 0, 0, 0, 0, 0, 0, 0, 22, 24, 19, 0, 0, 0, 0, 13, 0, 0, 0, 3722, 5, 1738, 0, 0, 0, 0, 67, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 3909, 1683, 59, 0, 4074, 1, 0] payload_rows_per_dst=[1920, 6454, 8028, 8126, 7842, 8112, 7604, 8034] dup_rows_per_dst=[68, 112, 138, 740, 194, 112, 1010, 7042] combine_rows_per_src=[1920, 6454, 8028, 8126, 7842, 8112, 7604, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=210198528 dispatch_meta_bytes=228536 combine_payload_bytes=210198528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210427064 ingress_bytes=229072896 link_bytes=229072896 token_bytes=420625592 weight_movement_bytes=18874368 total_nvlink_bytes=439499960 weight_to_token_ratio=0.044872 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.194863779e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=218447872 dispatch_meta_bytes=228896 combine_payload_bytes=218447872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218676768 ingress_bytes=237322240 link_bytes=237322240 token_bytes=437124640 weight_movement_bytes=18874368 total_nvlink_bytes=455999008 weight_to_token_ratio=0.043178 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.273904934e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=219439104 dispatch_meta_bytes=229416 combine_payload_bytes=219439104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219668520 ingress_bytes=238313472 link_bytes=238313472 token_bytes=439107624 weight_movement_bytes=18874368 total_nvlink_bytes=457981992 weight_to_token_ratio=0.042983 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.065972 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283402431e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60414 payload_rows_remote=52518 dup_rows=5122 dup_rows_remote=5122 dedup_ratio=0.078156 router_tokens_per_expert=[0, 0, 293, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 2467, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 2, 2316, 0, 0, 0, 0, 0, 4089, 997, 1, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 7896, 7136, 8190, 6732, 5890] dup_rows_per_dst=[652, 42, 10, 0, 2028, 0, 1648, 742] combine_rows_per_src=[8192, 8186, 8192, 7896, 7136, 8190, 6732, 5890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=221274112 dispatch_meta_bytes=236024 combine_payload_bytes=221274112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221510136 ingress_bytes=240148480 link_bytes=240148480 token_bytes=442784248 weight_movement_bytes=18874368 total_nvlink_bytes=461658616 weight_to_token_ratio=0.042627 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.065972 gemm_active_groups=16 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300984575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58228 payload_rows_remote=50058 dup_rows=7308 dup_rows_remote=6128 dedup_ratio=0.111511 router_tokens_per_expert=[6, 0, 1171, 0, 0, 0, 57, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 6, 0, 0, 4, 0, 2, 0, 0, 4086, 0, 4, 0, 1, 3, 0, 0, 0, 0, 4080, 1, 3, 0, 0, 2, 5, 0, 7, 0, 2, 0, 1, 0, 2785, 0, 5, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 10, 0, 0, 0, 0, 0, 3, 0, 2222, 0, 2285, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 18, 2274, 0, 5, 0, 0, 0, 4086, 482, 12, 0, 4083, 0, 0] payload_rows_per_dst=[8184, 8170, 8184, 8186, 6684, 8188, 5434, 5198] dup_rows_per_dst=[2460, 1180, 82, 10, 2128, 10, 1054, 384] combine_rows_per_src=[8184, 8170, 8184, 8186, 6684, 8188, 5434, 5198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=196960256 dispatch_meta_bytes=201840 combine_payload_bytes=196960256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=197162096 ingress_bytes=253583360 link_bytes=253583360 token_bytes=394122352 weight_movement_bytes=56623104 total_nvlink_bytes=450745456 weight_to_token_ratio=0.143669 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=215113728 dispatch_meta_bytes=230560 combine_payload_bytes=215113728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215344288 ingress_bytes=252862464 link_bytes=252862464 token_bytes=430458016 weight_movement_bytes=37748736 total_nvlink_bytes=468206752 weight_to_token_ratio=0.087694 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422803714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58336 payload_rows_remote=53194 dup_rows=7200 dup_rows_remote=7168 dedup_ratio=0.109863 router_tokens_per_expert=[0, 0, 1705, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 0, 0, 0, 0, 4096, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 29, 0, 0, 891, 0, 0, 4096, 0, 0, 0, 0, 0, 3407, 0] payload_rows_per_dst=[5262, 8192, 8192, 8192, 8192, 5142, 7886, 7278] dup_rows_per_dst=[1658, 8, 20, 8, 0, 32, 4098, 1376] combine_rows_per_src=[5262, 8192, 8192, 8192, 8192, 5142, 7886, 7278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205037568 dispatch_meta_bytes=224744 combine_payload_bytes=205037568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=205262312 ingress_bytes=242786304 link_bytes=242786304 token_bytes=410299880 weight_movement_bytes=37748736 total_nvlink_bytes=448048616 weight_to_token_ratio=0.092003 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.326258907e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61740 payload_rows_remote=53548 dup_rows=3796 dup_rows_remote=2026 dedup_ratio=0.057922 router_tokens_per_expert=[0, 0, 445, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 0, 0, 402, 0, 0, 3768, 0, 0, 4096, 0, 0, 0, 0, 0, 3563, 0] payload_rows_per_dst=[8192, 8192, 8192, 7316, 8192, 7624, 6448, 7584] dup_rows_per_dst=[1770, 0, 18, 0, 38, 1252, 0, 718] combine_rows_per_src=[8192, 8192, 8192, 7316, 8192, 7624, 6448, 7584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58172 payload_rows_remote=49984 dup_rows=7364 dup_rows_remote=7364 dedup_ratio=0.112366 router_tokens_per_expert=[0, 0, 2541, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 0, 0, 0, 0, 4094, 165, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2955, 0, 0, 0, 0, 576, 0, 0, 1886, 0, 0, 4092, 0, 0, 0, 0, 16, 2561, 0] payload_rows_per_dst=[8156, 8192, 8188, 8192, 8184, 6804, 5910, 4546] dup_rows_per_dst=[5082, 2, 0, 12, 0, 1844, 14, 410] combine_rows_per_src=[8156, 8192, 8188, 8192, 8184, 6804, 5910, 4546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57738 payload_rows_remote=50168 dup_rows=7798 dup_rows_remote=5878 dedup_ratio=0.118988 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 2, 0, 3066, 0, 0, 0, 0, 2, 3593, 0, 0, 0, 15, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 623, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 4, 3, 4, 3, 0, 0, 0, 29, 0, 1612, 6, 0, 0, 0, 0, 0, 0, 3086, 0, 12, 0, 0, 0, 3957, 0, 0, 0, 0, 408, 0, 0, 0, 1, 0, 0, 0, 0, 27] payload_rows_per_dst=[8172, 8080, 2950, 7128, 7586, 8190, 7570, 8062] dup_rows_per_dst=[2, 0, 2, 1190, 2792, 22, 1920, 1870] combine_rows_per_src=[8172, 8080, 2950, 7128, 7586, 8190, 7570, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59228 payload_rows_remote=51036 dup_rows=6308 dup_rows_remote=6308 dedup_ratio=0.096252 router_tokens_per_expert=[1, 0, 1049, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 0, 0, 0, 0, 4096, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 0, 0, 0, 195, 0, 0, 1730, 0, 0, 4096, 0, 0, 0, 0, 5, 3362, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6796, 6176, 5296] dup_rows_per_dst=[2100, 0, 2, 0, 0, 2784, 0, 1422] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6796, 6176, 5296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=217882624 dispatch_meta_bytes=241448 combine_payload_bytes=217882624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218124072 ingress_bytes=255631360 link_bytes=255631360 token_bytes=436006696 weight_movement_bytes=37748736 total_nvlink_bytes=473755432 weight_to_token_ratio=0.086578 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.449333913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=219332608 dispatch_meta_bytes=222296 combine_payload_bytes=219332608 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219554904 ingress_bytes=275955712 link_bytes=275955712 token_bytes=438887512 weight_movement_bytes=56623104 total_nvlink_bytes=495510616 weight_to_token_ratio=0.129015 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.644071854e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54714 payload_rows_remote=46636 dup_rows=10822 dup_rows_remote=4772 dedup_ratio=0.165131 router_tokens_per_expert=[5, 0, 628, 1, 0, 0, 0, 0, 0, 4064, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 12, 8, 0, 4061, 0, 0, 0, 0, 14, 0, 0, 0, 0, 15, 0, 0, 2, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 399, 7, 0, 0, 4041, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2612, 0, 0, 0, 0, 4, 39, 4028, 80, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 29, 17, 9, 12, 0, 0, 0, 2663, 29, 0, 0, 15, 18, 0, 0, 2228, 0, 0, 4070, 0, 0, 10, 0, 0, 3556, 0] payload_rows_per_dst=[1280, 8168, 7848, 8150, 8176, 5540, 7474, 8078] dup_rows_per_dst=[0, 8, 66, 754, 8, 228, 3708, 6050] combine_rows_per_src=[1280, 8168, 7848, 8150, 8176, 5540, 7474, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=204734464 dispatch_meta_bytes=229392 combine_payload_bytes=204734464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204963856 ingress_bytes=223608832 link_bytes=223608832 token_bytes=409698320 weight_movement_bytes=18874368 total_nvlink_bytes=428572688 weight_to_token_ratio=0.046069 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.069444 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142509806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205488128 dispatch_meta_bytes=224184 combine_payload_bytes=205488128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205712312 ingress_bytes=243236864 link_bytes=243236864 token_bytes=411200440 weight_movement_bytes=37748736 total_nvlink_bytes=448949176 weight_to_token_ratio=0.091801 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=14720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330575951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209043456 dispatch_meta_bytes=229376 combine_payload_bytes=209043456 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=209272832 ingress_bytes=246792192 link_bytes=246792192 token_bytes=418316288 weight_movement_bytes=37748736 total_nvlink_bytes=456065024 weight_to_token_ratio=0.090240 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364641354e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60576 payload_rows_remote=52384 dup_rows=4960 dup_rows_remote=4936 dedup_ratio=0.075684 router_tokens_per_expert=[4, 2, 1313, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 0, 0, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 0, 0, 0, 614, 0, 0, 3085, 0, 0, 4091, 0, 0, 0, 0, 1, 2760, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8182, 6566, 6404, 6656] dup_rows_per_dst=[2642, 0, 178, 24, 0, 1372, 0, 744] combine_rows_per_src=[8192, 8192, 8192, 8192, 8182, 6566, 6404, 6656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=191021056 dispatch_meta_bytes=205632 combine_payload_bytes=191021056 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=191226688 ingress_bytes=266518528 link_bytes=266518528 token_bytes=382247744 weight_movement_bytes=75497472 total_nvlink_bytes=457745216 weight_to_token_ratio=0.197509 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.553649400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59212 payload_rows_remote=51030 dup_rows=6324 dup_rows_remote=6314 dedup_ratio=0.096497 router_tokens_per_expert=[0, 0, 915, 0, 0, 0, 5, 0, 0, 4079, 5, 0, 0, 0, 2, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4079, 0, 0, 0, 1, 11, 0, 0, 6, 25, 0, 0, 2, 0, 1, 10, 6, 2, 17, 0, 0, 0, 1, 0, 43, 0, 0, 0, 4081, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 3, 2415, 0, 5, 0, 0, 0, 0, 4066, 15, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 4, 12, 1, 0, 0, 0, 0, 2190, 0, 0, 4, 0, 0, 0, 0, 2885, 0, 0, 4080, 0, 3, 6, 0, 18, 3734, 0] payload_rows_per_dst=[8172, 8182, 8182, 8168, 8172, 8106, 4406, 5824] dup_rows_per_dst=[1840, 10, 78, 120, 0, 4254, 14, 8] combine_rows_per_src=[8172, 8182, 8182, 8168, 8172, 8106, 4406, 5824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=214564864 dispatch_meta_bytes=229280 combine_payload_bytes=214564864 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=214794144 ingress_bytes=233439232 link_bytes=233439232 token_bytes=429359008 weight_movement_bytes=18874368 total_nvlink_bytes=448233376 weight_to_token_ratio=0.043959 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.236699862e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61732 payload_rows_remote=53540 dup_rows=3804 dup_rows_remote=3804 dedup_ratio=0.058044 router_tokens_per_expert=[11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 0, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 3852, 0, 6, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 0, 0, 0] payload_rows_per_dst=[6460, 8148, 6888, 7554, 8188, 8192, 8110, 8192] dup_rows_per_dst=[8, 0, 0, 2, 0, 0, 32, 3762] combine_rows_per_src=[6460, 8148, 6888, 7554, 8188, 8192, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58430 payload_rows_remote=50812 dup_rows=7106 dup_rows_remote=2578 dedup_ratio=0.108429 router_tokens_per_expert=[39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 2, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 0, 0, 0, 0] payload_rows_per_dst=[7618, 8110, 7408, 8036, 8192, 8098, 7964, 3004] dup_rows_per_dst=[4528, 2, 2478, 0, 94, 0, 4, 0] combine_rows_per_src=[7618, 8110, 7408, 8036, 8192, 8098, 7964, 3004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59578 payload_rows_remote=52070 dup_rows=5958 dup_rows_remote=3170 dedup_ratio=0.090912 router_tokens_per_expert=[109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2558, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 5, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 11, 0, 4077, 0, 3, 0, 0, 0, 0, 0, 0, 1789, 0, 0, 0, 0, 0] payload_rows_per_dst=[7650, 8180, 7508, 8162, 8186, 8192, 8122, 3578] dup_rows_per_dst=[170, 0, 2788, 2948, 0, 10, 36, 6] combine_rows_per_src=[7650, 8180, 7508, 8162, 8186, 8192, 8122, 3578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209018880 dispatch_meta_bytes=229376 combine_payload_bytes=209018880 prefetch_bytes=0 reduce_bytes=0 egress_bytes=209248256 ingress_bytes=209018880 link_bytes=209248256 token_bytes=418267136 weight_movement_bytes=0 total_nvlink_bytes=418267136 weight_to_token_ratio=0.000000 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.004913832e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63400 payload_rows_remote=55208 dup_rows=2136 dup_rows_remote=2136 dedup_ratio=0.032593 router_tokens_per_expert=[84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 8, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 0] payload_rows_per_dst=[7914, 8186, 7430, 8146, 8192, 8192, 7148, 8192] dup_rows_per_dst=[60, 4, 0, 30, 0, 0, 10, 2032] combine_rows_per_src=[7914, 8186, 7430, 8146, 8192, 8192, 7148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58860 payload_rows_remote=50674 dup_rows=6676 dup_rows_remote=6584 dedup_ratio=0.101868 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4027, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 0, 8, 0, 0, 0, 0, 1, 348, 0, 0, 4092, 1, 0, 448, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4092, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 1, 2, 0, 0, 1, 0, 0, 0, 0, 4032, 0, 0, 18] payload_rows_per_dst=[8056, 8192, 2046, 7944, 8132, 8192, 8186, 8112] dup_rows_per_dst=[0, 738, 8, 5720, 0, 54, 92, 64] combine_rows_per_src=[8056, 8192, 2046, 7944, 8132, 8192, 8186, 8112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=219299840 dispatch_meta_bytes=229376 combine_payload_bytes=219299840 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=219529216 ingress_bytes=275922944 link_bytes=275922944 token_bytes=438829056 weight_movement_bytes=56623104 total_nvlink_bytes=495452160 weight_to_token_ratio=0.129032 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.643757887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=208125952 dispatch_meta_bytes=213560 combine_payload_bytes=208125952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208339512 ingress_bytes=227000320 link_bytes=227000320 token_bytes=416465464 weight_movement_bytes=18874368 total_nvlink_bytes=435339832 weight_to_token_ratio=0.045320 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.175005375e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61988 payload_rows_remote=54250 dup_rows=3548 dup_rows_remote=1694 dedup_ratio=0.054138 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 5, 310, 0, 189, 0, 0, 39, 0, 0, 11, 6, 5, 9, 96, 18, 0, 35, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2916, 0, 27, 3962, 0, 15, 4, 0, 0, 31, 0, 22, 0, 24, 4049, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 21, 1, 0, 118, 3927, 4059, 22, 0, 21, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 3, 4, 3417, 0, 0, 8, 0, 0, 0, 0, 0, 1316, 20, 0, 0, 6, 0] payload_rows_per_dst=[7914, 8120, 5876, 8030, 8126, 8056, 8128, 7738] dup_rows_per_dst=[952, 352, 0, 212, 2, 168, 8, 1854] combine_rows_per_src=[7914, 8120, 5876, 8030, 8126, 8056, 8128, 7738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213278720 dispatch_meta_bytes=220960 combine_payload_bytes=213278720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213499680 ingress_bytes=232153088 link_bytes=232153088 token_bytes=426778400 weight_movement_bytes=18874368 total_nvlink_bytes=445652768 weight_to_token_ratio=0.044225 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224376663e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=226131968 dispatch_meta_bytes=229376 combine_payload_bytes=226131968 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=226361344 ingress_bytes=263880704 link_bytes=263880704 token_bytes=452493312 weight_movement_bytes=37748736 total_nvlink_bytes=490242048 weight_to_token_ratio=0.083424 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528375068e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=207560704 dispatch_meta_bytes=229032 combine_payload_bytes=207560704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207789736 ingress_bytes=226435072 link_bytes=226435072 token_bytes=415350440 weight_movement_bytes=18874368 total_nvlink_bytes=434224808 weight_to_token_ratio=0.045442 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169589447e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59772 payload_rows_remote=52158 dup_rows=5764 dup_rows_remote=4186 dedup_ratio=0.087952 router_tokens_per_expert=[430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 1, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4084, 0, 103, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 4, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0] payload_rows_per_dst=[8024, 8168, 6800, 7614, 8192, 8192, 7260, 5522] dup_rows_per_dst=[826, 0, 0, 1578, 4, 6, 2230, 1120] combine_rows_per_src=[8024, 8168, 6800, 7614, 8192, 8192, 7260, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=222208000 dispatch_meta_bytes=223776 combine_payload_bytes=222208000 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=222431776 ingress_bytes=316579840 link_bytes=316579840 token_bytes=444639776 weight_movement_bytes=94371840 total_nvlink_bytes=539011616 weight_to_token_ratio=0.212243 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.033312260e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59716 payload_rows_remote=51680 dup_rows=5820 dup_rows_remote=5792 dedup_ratio=0.088806 router_tokens_per_expert=[66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 97, 15, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 3081, 0, 32, 0, 0, 0, 13, 0, 19, 0, 0, 4002, 0, 0, 0, 1, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 4069, 4081, 8, 0, 11, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 2, 0, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 3233, 0, 0, 0, 0, 0] payload_rows_per_dst=[6450, 8036, 8044, 6244, 8186, 8170, 8120, 6466] dup_rows_per_dst=[298, 28, 5378, 46, 14, 48, 8, 0] combine_rows_per_src=[6450, 8036, 8044, 6244, 8186, 8170, 8120, 6466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64972 payload_rows_remote=56780 dup_rows=564 dup_rows_remote=0 dedup_ratio=0.008606 router_tokens_per_expert=[0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3817, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 7634, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 564, 0, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 7634, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213639168 dispatch_meta_bytes=225376 combine_payload_bytes=213639168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213864544 ingress_bytes=251387904 link_bytes=251387904 token_bytes=427503712 weight_movement_bytes=37748736 total_nvlink_bytes=465252448 weight_to_token_ratio=0.088300 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408675206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64420 payload_rows_remote=56228 dup_rows=1116 dup_rows_remote=1116 dedup_ratio=0.017029 router_tokens_per_expert=[0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3572, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8124, 8192, 8192, 7144, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 1110, 0, 0, 0, 0, 0] combine_rows_per_src=[8124, 8192, 8192, 7144, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64412 payload_rows_remote=56220 dup_rows=1124 dup_rows_remote=58 dedup_ratio=0.017151 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7720, 8192, 7762, 8192, 8192, 7970, 8192] dup_rows_per_dst=[1066, 0, 44, 0, 6, 8, 0, 0] combine_rows_per_src=[8192, 7720, 8192, 7762, 8192, 8192, 7970, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=136 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4031, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8062, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 136, 0, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8062, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61588 payload_rows_remote=55878 dup_rows=3948 dup_rows_remote=3940 dedup_ratio=0.060242 router_tokens_per_expert=[0, 2, 46, 0, 0, 0, 219, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4082, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3437, 2, 0, 1, 0, 52, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 2699, 0, 0, 0, 0, 0, 26, 0, 0, 5, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 4082, 810, 5, 0, 4087, 0, 0] payload_rows_per_dst=[8190, 8192, 8174, 8128, 6890, 8184, 5710, 8120] dup_rows_per_dst=[572, 128, 0, 2, 106, 6, 8, 3126] combine_rows_per_src=[8190, 8192, 8174, 8128, 6890, 8184, 5710, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=211681280 dispatch_meta_bytes=229888 combine_payload_bytes=211681280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211911168 ingress_bytes=230555648 link_bytes=230555648 token_bytes=423592448 weight_movement_bytes=18874368 total_nvlink_bytes=442466816 weight_to_token_ratio=0.044558 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.209070779e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59780 payload_rows_remote=51628 dup_rows=5756 dup_rows_remote=5708 dedup_ratio=0.087830 router_tokens_per_expert=[0, 13, 33, 3, 0, 4036, 4039, 0, 0, 0, 0, 0, 0, 8, 31, 0, 8, 0, 0, 10, 0, 0, 34, 0, 8, 2, 6, 133, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 72, 14, 1, 451, 0, 0, 14, 0, 0, 0, 0, 0, 0, 14, 10, 0, 0, 0, 0, 0, 3983, 0, 3465, 0, 0, 0, 4049, 0, 43, 0, 0, 2, 0, 0, 22, 0, 0, 0, 0, 5, 30, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 6, 32, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4049, 0, 0, 0, 0, 20, 0] payload_rows_per_dst=[8152, 8164, 3378, 7544, 8100, 8156, 8134, 8152] dup_rows_per_dst=[96, 316, 370, 4762, 86, 48, 30, 48] combine_rows_per_src=[8152, 8164, 3378, 7544, 8100, 8156, 8134, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=232570880 dispatch_meta_bytes=227120 combine_payload_bytes=232570880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232798000 ingress_bytes=251445248 link_bytes=251445248 token_bytes=465368880 weight_movement_bytes=18874368 total_nvlink_bytes=484243248 weight_to_token_ratio=0.040558 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409224648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=230309888 dispatch_meta_bytes=229376 combine_payload_bytes=230309888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230539264 ingress_bytes=249184256 link_bytes=249184256 token_bytes=460849152 weight_movement_bytes=18874368 total_nvlink_bytes=479723520 weight_to_token_ratio=0.040956 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.387560935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234323968 dispatch_meta_bytes=229376 combine_payload_bytes=234323968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234553344 ingress_bytes=272072704 link_bytes=272072704 token_bytes=468877312 weight_movement_bytes=37748736 total_nvlink_bytes=506626048 weight_to_token_ratio=0.080509 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606866782e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=230277120 dispatch_meta_bytes=225112 combine_payload_bytes=230277120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230502232 ingress_bytes=268025856 link_bytes=268025856 token_bytes=460779352 weight_movement_bytes=37748736 total_nvlink_bytes=498528088 weight_to_token_ratio=0.081924 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568091875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=228876288 dispatch_meta_bytes=239272 combine_payload_bytes=228876288 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229115560 ingress_bytes=285499392 link_bytes=285499392 token_bytes=457991848 weight_movement_bytes=56623104 total_nvlink_bytes=514614952 weight_to_token_ratio=0.123633 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.735514700e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211468288 dispatch_meta_bytes=229344 combine_payload_bytes=211468288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211697632 ingress_bytes=230342656 link_bytes=230342656 token_bytes=423165920 weight_movement_bytes=18874368 total_nvlink_bytes=442040288 weight_to_token_ratio=0.044603 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207029994e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65140 payload_rows_remote=57038 dup_rows=396 dup_rows_remote=396 dedup_ratio=0.006042 router_tokens_per_expert=[0, 0, 0, 0, 2, 3943, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4051, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7890, 8192, 8188, 8102, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 390, 0, 4, 0, 0, 0] combine_rows_per_src=[7890, 8192, 8188, 8102, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=233627648 dispatch_meta_bytes=229736 combine_payload_bytes=233627648 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233857384 ingress_bytes=290250752 link_bytes=290250752 token_bytes=467485032 weight_movement_bytes=56623104 total_nvlink_bytes=524108136 weight_to_token_ratio=0.121123 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.781039893e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64908 payload_rows_remote=56716 dup_rows=628 dup_rows_remote=614 dedup_ratio=0.009583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 7574, 8192, 8188, 8192] dup_rows_per_dst=[0, 8, 14, 96, 0, 20, 0, 490] combine_rows_per_src=[8192, 8186, 8192, 8192, 7574, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64592 payload_rows_remote=56400 dup_rows=944 dup_rows_remote=906 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8192, 8188, 8192, 7286, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 874, 0, 38, 2, 30] combine_rows_per_src=[8158, 8192, 8188, 8192, 7286, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64688 payload_rows_remote=57330 dup_rows=848 dup_rows_remote=848 dedup_ratio=0.012939 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8192, 7358, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 824, 0, 0, 0, 24] combine_rows_per_src=[8178, 8192, 8192, 8192, 7358, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57590 payload_rows_remote=49398 dup_rows=7946 dup_rows_remote=7786 dedup_ratio=0.121246 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8152, 7850, 8192, 628] dup_rows_per_dst=[160, 194, 178, 948, 6466, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8152, 7850, 8192, 628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64564 payload_rows_remote=56378 dup_rows=972 dup_rows_remote=788 dedup_ratio=0.014832 router_tokens_per_expert=[0, 7, 0, 9, 0, 3749, 4073, 0, 0, 0, 0, 0, 0, 13, 11, 2, 0, 0, 0, 4, 0, 0, 8, 0, 0, 0, 4, 93, 0, 3, 0, 0, 1, 3, 0, 1, 0, 0, 0, 0, 0, 16, 6, 6, 245, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 3957, 0, 0, 0, 4083, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 0, 12, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 16, 0] payload_rows_per_dst=[7542, 8186, 8188, 7918, 8184, 8174, 8186, 8186] dup_rows_per_dst=[40, 184, 492, 0, 6, 0, 250, 0] combine_rows_per_src=[7542, 8186, 8188, 7918, 8184, 8174, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57346 payload_rows_remote=49170 dup_rows=8190 dup_rows_remote=3368 dedup_ratio=0.124969 router_tokens_per_expert=[1, 0, 1489, 0, 0, 9, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 4093, 857, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 3, 52, 0, 0, 1497, 0, 1, 4078, 0, 0, 0, 0, 0, 3217, 3] payload_rows_per_dst=[2996, 8184, 8186, 8190, 8188, 6244, 8176, 7182] dup_rows_per_dst=[2, 14, 2, 26, 12, 948, 4822, 2364] combine_rows_per_src=[2996, 8184, 8186, 8190, 8188, 6244, 8176, 7182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56880 payload_rows_remote=48688 dup_rows=8656 dup_rows_remote=3348 dedup_ratio=0.132080 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 4048, 0, 0, 0, 0, 0, 0, 3940, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 32, 0, 39, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 53, 33, 0, 6, 0, 0, 0, 0, 51, 0, 0, 0, 20, 34, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 3992, 0, 0, 0, 1410, 0, 0, 0, 0, 0, 0, 5, 0, 19, 0, 0, 0, 0, 39, 0, 56, 1, 22, 0, 28, 22, 0, 2, 0, 0, 34, 0, 0, 39, 4009, 4033, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4, 0, 2613, 0, 36, 25, 0, 0, 0] payload_rows_per_dst=[8110, 8024, 8078, 214, 7984, 8142, 8136, 8192] dup_rows_per_dst=[0, 18, 0, 182, 2820, 152, 176, 5308] combine_rows_per_src=[8110, 8024, 8078, 214, 7984, 8142, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232308736 dispatch_meta_bytes=229320 combine_payload_bytes=232308736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232538056 ingress_bytes=251183104 link_bytes=251183104 token_bytes=464846792 weight_movement_bytes=18874368 total_nvlink_bytes=483721160 weight_to_token_ratio=0.040603 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406712913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231014400 dispatch_meta_bytes=229224 combine_payload_bytes=231014400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231243624 ingress_bytes=249888768 link_bytes=249888768 token_bytes=462258024 weight_movement_bytes=18874368 total_nvlink_bytes=481132392 weight_to_token_ratio=0.040831 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394311222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=232712 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235056392 ingress_bytes=291446784 link_bytes=291446784 token_bytes=469880072 weight_movement_bytes=56623104 total_nvlink_bytes=526503176 weight_to_token_ratio=0.120505 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792499684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202334208 dispatch_meta_bytes=228736 combine_payload_bytes=202334208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202562944 ingress_bytes=221208576 link_bytes=221208576 token_bytes=404897152 weight_movement_bytes=18874368 total_nvlink_bytes=423771520 weight_to_token_ratio=0.046615 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.119511734e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=230924288 dispatch_meta_bytes=228664 combine_payload_bytes=230924288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231152952 ingress_bytes=249798656 link_bytes=249798656 token_bytes=462077240 weight_movement_bytes=18874368 total_nvlink_bytes=480951608 weight_to_token_ratio=0.040847 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393447813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=201400320 dispatch_meta_bytes=210152 combine_payload_bytes=201400320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201610472 ingress_bytes=220274688 link_bytes=220274688 token_bytes=403010792 weight_movement_bytes=18874368 total_nvlink_bytes=421885160 weight_to_token_ratio=0.046833 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.110563679e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=199426048 dispatch_meta_bytes=208144 combine_payload_bytes=199426048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199634192 ingress_bytes=218300416 link_bytes=218300416 token_bytes=399060240 weight_movement_bytes=18874368 total_nvlink_bytes=417934608 weight_to_token_ratio=0.047297 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.091647176e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62070 payload_rows_remote=53878 dup_rows=3466 dup_rows_remote=78 dedup_ratio=0.052887 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 2369, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8192, 4740, 8192, 8184, 8190] dup_rows_per_dst=[0, 0, 8, 3388, 0, 6, 0, 64] combine_rows_per_src=[8190, 8190, 8192, 8192, 4740, 8192, 8184, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59800 payload_rows_remote=51612 dup_rows=5736 dup_rows_remote=422 dedup_ratio=0.087524 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1248, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 41, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 2496, 8190, 8192, 8158] dup_rows_per_dst=[2, 286, 5314, 4, 0, 46, 82, 2] combine_rows_per_src=[8192, 8192, 8188, 8192, 2496, 8190, 8192, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=220684288 dispatch_meta_bytes=215824 combine_payload_bytes=220684288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220900112 ingress_bytes=239558656 link_bytes=239558656 token_bytes=441584400 weight_movement_bytes=18874368 total_nvlink_bytes=460458768 weight_to_token_ratio=0.042742 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.295333172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60026 payload_rows_remote=51842 dup_rows=5510 dup_rows_remote=5510 dedup_ratio=0.084076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3276, 8116, 7774, 8184, 8192, 8100] dup_rows_per_dst=[0, 2, 398, 1600, 3510, 0, 0, 0] combine_rows_per_src=[8192, 8192, 3276, 8116, 7774, 8184, 8192, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63180 payload_rows_remote=54988 dup_rows=2356 dup_rows_remote=2356 dedup_ratio=0.035950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5958, 8192, 8070, 8192] dup_rows_per_dst=[0, 8, 2348, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5958, 8192, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62538 payload_rows_remote=57320 dup_rows=2998 dup_rows_remote=2998 dedup_ratio=0.045746 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5218, 8192, 8192, 8168] dup_rows_per_dst=[0, 4, 2994, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5218, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59198 payload_rows_remote=51092 dup_rows=6338 dup_rows_remote=6088 dedup_ratio=0.096710 router_tokens_per_expert=[56, 0, 0, 0, 0, 6, 0, 0, 0, 2, 0, 3169, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2469, 0, 0, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 10, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 153, 0, 4094, 0, 91, 0, 0, 0, 0, 0, 0, 2850, 0, 0, 0, 0, 0] payload_rows_per_dst=[6486, 8190, 4938, 6934, 8160, 8192, 8106, 8192] dup_rows_per_dst=[178, 12, 0, 0, 0, 20, 250, 5878] combine_rows_per_src=[6486, 8190, 4938, 6934, 8160, 8192, 8106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62758 payload_rows_remote=54606 dup_rows=2778 dup_rows_remote=2772 dedup_ratio=0.042389 router_tokens_per_expert=[0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 9, 0, 4046, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 14, 0, 17, 0, 4050, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 16, 3, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1100, 18, 0, 2, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4046, 0, 0, 0, 2542, 0, 0, 0, 0, 0, 3, 16, 0, 6, 0, 0, 0, 0, 7, 0, 44, 3, 12, 0, 1, 1, 0, 5, 0, 0, 13, 0, 0, 11, 4061, 4053, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 456, 0, 10, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8152, 8110, 8178, 5934, 8190, 8034, 7972] dup_rows_per_dst=[12, 6, 0, 2236, 26, 116, 34, 348] combine_rows_per_src=[8188, 8152, 8110, 8178, 5934, 8190, 8034, 7972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=211402752 dispatch_meta_bytes=208136 combine_payload_bytes=211402752 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=211610888 ingress_bytes=268025856 link_bytes=268025856 token_bytes=423013640 weight_movement_bytes=56623104 total_nvlink_bytes=479636744 weight_to_token_ratio=0.133856 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=21632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568091875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53142 payload_rows_remote=45012 dup_rows=12394 dup_rows_remote=12006 dedup_ratio=0.189117 router_tokens_per_expert=[14, 0, 19, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 2, 0, 0, 734, 0, 0, 0, 0, 222, 0, 0, 3, 0, 0, 0, 0, 0, 299, 4054, 0, 0, 23, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3471, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 3, 0, 0, 20, 0, 0, 0, 3990, 0, 0, 0, 3443, 68, 0, 0, 0, 0, 0, 42, 15, 0, 35, 0, 40, 2, 4, 0, 0, 0, 0, 4040, 0, 0, 14, 0, 0, 0, 0, 4026, 0, 0, 3937, 0, 0, 4019, 0, 0, 0, 0, 45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7896, 5590, 644, 7002, 8192, 7786, 7902, 8130] dup_rows_per_dst=[64, 1670, 0, 24, 9582, 272, 394, 388] combine_rows_per_src=[7896, 5590, 644, 7002, 8192, 7786, 7902, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=225230848 dispatch_meta_bytes=229376 combine_payload_bytes=225230848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225460224 ingress_bytes=244105216 link_bytes=244105216 token_bytes=450691072 weight_movement_bytes=18874368 total_nvlink_bytes=469565440 weight_to_token_ratio=0.041879 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.338896072e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=212344832 dispatch_meta_bytes=229408 combine_payload_bytes=212344832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212574240 ingress_bytes=231219200 link_bytes=231219200 token_bytes=424919072 weight_movement_bytes=18874368 total_nvlink_bytes=443793440 weight_to_token_ratio=0.044419 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215428607e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=241272 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235023992 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469806712 weight_movement_bytes=37748736 total_nvlink_bytes=507555448 weight_to_token_ratio=0.080350 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=209272832 dispatch_meta_bytes=228720 combine_payload_bytes=209272832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209501552 ingress_bytes=228147200 link_bytes=228147200 token_bytes=418774384 weight_movement_bytes=18874368 total_nvlink_bytes=437648752 weight_to_token_ratio=0.045070 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.185994215e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=223666176 dispatch_meta_bytes=229512 combine_payload_bytes=223666176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223895688 ingress_bytes=242540544 link_bytes=242540544 token_bytes=447561864 weight_movement_bytes=18874368 total_nvlink_bytes=466436232 weight_to_token_ratio=0.042172 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.323904155e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64846 payload_rows_remote=56654 dup_rows=690 dup_rows_remote=680 dedup_ratio=0.010529 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 7510, 8192, 8192, 8188] dup_rows_per_dst=[2, 2, 674, 10, 0, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 7510, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=184369152 dispatch_meta_bytes=228072 combine_payload_bytes=184369152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184597224 ingress_bytes=203243520 link_bytes=203243520 token_bytes=368966376 weight_movement_bytes=18874368 total_nvlink_bytes=387840744 weight_to_token_ratio=0.051155 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.947379407e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=232054784 dispatch_meta_bytes=229336 combine_payload_bytes=232054784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232284120 ingress_bytes=250929152 link_bytes=250929152 token_bytes=464338904 weight_movement_bytes=18874368 total_nvlink_bytes=483213272 weight_to_token_ratio=0.040648 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404279670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63534 payload_rows_remote=55954 dup_rows=2002 dup_rows_remote=1984 dedup_ratio=0.030548 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 949, 0, 16, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 4064, 0, 0, 0, 4096, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7580, 6866, 8192, 8192, 8192, 8128] dup_rows_per_dst=[0, 1984, 18, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7580, 6866, 8192, 8192, 8192, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62504 payload_rows_remote=54312 dup_rows=3032 dup_rows_remote=3032 dedup_ratio=0.046265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 4096, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7856, 5496, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 3010, 22, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7856, 5496, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61370 payload_rows_remote=53178 dup_rows=4166 dup_rows_remote=4166 dedup_ratio=0.063568 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 4096, 0, 0, 0, 4094, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6460, 7514, 6440, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 2182, 1984, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 6460, 7514, 6440, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61390 payload_rows_remote=53198 dup_rows=4146 dup_rows_remote=4146 dedup_ratio=0.063263 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2067, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2742, 0, 4096, 0, 0, 0, 4096, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5610, 6628, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4136, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5610, 6628, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63814 payload_rows_remote=55622 dup_rows=1722 dup_rows_remote=1716 dedup_ratio=0.026276 router_tokens_per_expert=[0, 0, 0, 0, 0, 4030, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3304, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8062, 8190, 8192, 6608, 8190, 8190, 8192, 8190] dup_rows_per_dst=[0, 0, 1716, 0, 0, 0, 6, 0] combine_rows_per_src=[8062, 8190, 8192, 6608, 8190, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=229187584 dispatch_meta_bytes=231752 combine_payload_bytes=229187584 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=229419336 ingress_bytes=323559424 link_bytes=323559424 token_bytes=458606920 weight_movement_bytes=94371840 total_nvlink_bytes=552978760 weight_to_token_ratio=0.205779 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.100187200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56750 payload_rows_remote=48646 dup_rows=8786 dup_rows_remote=8578 dedup_ratio=0.134064 router_tokens_per_expert=[0, 0, 42, 0, 0, 46, 0, 0, 0, 0, 0, 27, 2, 0, 0, 0, 0, 0, 0, 0, 4037, 4031, 0, 32, 0, 0, 0, 0, 335, 0, 0, 59, 0, 0, 0, 20, 0, 0, 3581, 0, 4052, 0, 0, 17, 4022, 0, 154, 58, 0, 0, 0, 21, 6, 0, 0, 0, 30, 4046, 0, 0, 0, 19, 3993, 0, 0, 0, 7, 0, 0, 0, 0, 3, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 40, 0, 46, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7998, 852, 7286, 8156, 8106, 8092, 8156, 8104] dup_rows_per_dst=[134, 0, 118, 8048, 132, 82, 64, 208] combine_rows_per_src=[7998, 852, 7286, 8156, 8106, 8092, 8156, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61414 payload_rows_remote=53238 dup_rows=4122 dup_rows_remote=3480 dedup_ratio=0.062897 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 125, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 47, 0, 0, 12, 275, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 4069, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 12, 0, 0, 0, 4061, 0, 0, 0, 2267, 42, 0, 0, 0, 0, 0, 15, 0, 0, 3, 2, 8, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4, 0, 0, 0, 0, 4074, 0, 0, 4039, 0, 4, 4079, 0, 0, 0, 0, 13, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8176, 8162, 7864, 4560, 8192, 8094, 8192] dup_rows_per_dst=[246, 642, 3098, 0, 14, 96, 0, 26] combine_rows_per_src=[8174, 8176, 8162, 7864, 4560, 8192, 8094, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=222461952 dispatch_meta_bytes=229376 combine_payload_bytes=222461952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222691328 ingress_bytes=241336320 link_bytes=241336320 token_bytes=445153280 weight_movement_bytes=18874368 total_nvlink_bytes=464027648 weight_to_token_ratio=0.042400 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312365873e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=217817088 dispatch_meta_bytes=229376 combine_payload_bytes=217817088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218046464 ingress_bytes=236691456 link_bytes=236691456 token_bytes=435863552 weight_movement_bytes=18874368 total_nvlink_bytes=454737920 weight_to_token_ratio=0.043303 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.267861072e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=217899008 dispatch_meta_bytes=229376 combine_payload_bytes=217899008 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=218128384 ingress_bytes=255647744 link_bytes=255647744 token_bytes=436027392 weight_movement_bytes=37748736 total_nvlink_bytes=473776128 weight_to_token_ratio=0.086574 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.449490896e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62716 payload_rows_remote=57130 dup_rows=2820 dup_rows_remote=2820 dedup_ratio=0.043030 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 4096, 0, 0, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8184, 7988, 5586, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2776, 44, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8184, 7988, 5586, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=227827712 dispatch_meta_bytes=229352 combine_payload_bytes=227827712 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=228057064 ingress_bytes=284450816 link_bytes=284450816 token_bytes=455884776 weight_movement_bytes=56623104 total_nvlink_bytes=512507880 weight_to_token_ratio=0.124205 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.725467760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=199254016 dispatch_meta_bytes=228896 combine_payload_bytes=199254016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=199482912 ingress_bytes=237002752 link_bytes=237002752 token_bytes=398736928 weight_movement_bytes=37748736 total_nvlink_bytes=436485664 weight_to_token_ratio=0.094671 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.270843757e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=218062848 dispatch_meta_bytes=226872 combine_payload_bytes=218062848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218289720 ingress_bytes=236937216 link_bytes=236937216 token_bytes=436352568 weight_movement_bytes=18874368 total_nvlink_bytes=455226936 weight_to_token_ratio=0.043255 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.270215823e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234004480 dispatch_meta_bytes=239800 combine_payload_bytes=234004480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234244280 ingress_bytes=271753216 link_bytes=271753216 token_bytes=468248760 weight_movement_bytes=37748736 total_nvlink_bytes=505997496 weight_to_token_ratio=0.080617 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603805605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62092 payload_rows_remote=53900 dup_rows=3444 dup_rows_remote=3444 dedup_ratio=0.052551 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 4762] dup_rows_per_dst=[264, 194, 0, 0, 2964, 22, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 4762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62382 payload_rows_remote=54190 dup_rows=3154 dup_rows_remote=2626 dedup_ratio=0.048126 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8034, 5196] dup_rows_per_dst=[528, 156, 0, 0, 2450, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8034, 5196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58464 payload_rows_remote=50272 dup_rows=7072 dup_rows_remote=7072 dedup_ratio=0.107910 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 1480, 8192, 8192, 7832] dup_rows_per_dst=[148, 0, 0, 0, 296, 0, 0, 6628] combine_rows_per_src=[8192, 8192, 8192, 8192, 1480, 8192, 8192, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60792 payload_rows_remote=52600 dup_rows=4744 dup_rows_remote=232 dedup_ratio=0.072388 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 3448] dup_rows_per_dst=[188, 44, 0, 0, 4512, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 3448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=220774400 dispatch_meta_bytes=229376 combine_payload_bytes=220774400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221003776 ingress_bytes=239648768 link_bytes=239648768 token_bytes=441778176 weight_movement_bytes=18874368 total_nvlink_bytes=460652544 weight_to_token_ratio=0.042724 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.296196580e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56250 payload_rows_remote=48120 dup_rows=9286 dup_rows_remote=1490 dedup_ratio=0.141693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 48, 0, 10, 97, 0, 0, 1, 63, 0, 0, 2, 0, 1, 0, 0, 0, 4031, 0, 0, 0, 0, 21, 0, 0, 0, 49, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 14, 0, 4003, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 19, 4045, 0, 26, 0, 0, 0, 0, 0, 4002, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 44, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 25, 24, 0, 0, 10, 0, 29, 20, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0] payload_rows_per_dst=[3666, 7202, 8072, 7268, 5630, 8092, 8190, 8130] dup_rows_per_dst=[68, 6, 114, 988, 62, 76, 176, 7796] combine_rows_per_src=[3666, 7202, 8072, 7268, 5630, 8092, 8190, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60730 payload_rows_remote=52538 dup_rows=4806 dup_rows_remote=4792 dedup_ratio=0.073334 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 26, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8182, 8192, 6172, 5452, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1464, 1744, 62, 14, 1522] combine_rows_per_src=[8156, 8182, 8192, 6172, 5452, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=221962240 dispatch_meta_bytes=227264 combine_payload_bytes=221962240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222189504 ingress_bytes=240836608 link_bytes=240836608 token_bytes=444151744 weight_movement_bytes=18874368 total_nvlink_bytes=463026112 weight_to_token_ratio=0.042495 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307577879e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62410 payload_rows_remote=54218 dup_rows=3126 dup_rows_remote=3126 dedup_ratio=0.047699 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8184, 5078] dup_rows_per_dst=[266, 52, 0, 0, 2778, 30, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8184, 5078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=205914112 dispatch_meta_bytes=229376 combine_payload_bytes=205914112 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=206143488 ingress_bytes=262537216 link_bytes=262537216 token_bytes=412057600 weight_movement_bytes=56623104 total_nvlink_bytes=468680704 weight_to_token_ratio=0.137416 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.515502427e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63016 payload_rows_remote=54824 dup_rows=2520 dup_rows_remote=96 dedup_ratio=0.038452 router_tokens_per_expert=[0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4072, 0, 14, 10, 0, 4, 0, 1195, 2, 0, 11, 0, 0, 0, 18, 0, 0, 2492, 0, 4079, 0, 0, 8, 4072, 0, 472, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 11, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 10, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5966, 7990, 8146, 8192, 8180, 8158] dup_rows_per_dst=[62, 2424, 14, 2, 0, 18, 0, 0] combine_rows_per_src=[8192, 8192, 5966, 7990, 8146, 8192, 8180, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215449600 dispatch_meta_bytes=211328 combine_payload_bytes=215449600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215660928 ingress_bytes=253198336 link_bytes=253198336 token_bytes=431110528 weight_movement_bytes=37748736 total_nvlink_bytes=468859264 weight_to_token_ratio=0.087562 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426021874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=197099520 dispatch_meta_bytes=198440 combine_payload_bytes=197099520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197297960 ingress_bytes=215973888 link_bytes=215973888 token_bytes=394397480 weight_movement_bytes=18874368 total_nvlink_bytes=413271848 weight_to_token_ratio=0.047856 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.069355529e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=215195648 dispatch_meta_bytes=229320 combine_payload_bytes=215195648 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=215424968 ingress_bytes=309567488 link_bytes=309567488 token_bytes=430620616 weight_movement_bytes=94371840 total_nvlink_bytes=524992456 weight_to_token_ratio=0.219153 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.966123353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=222076928 dispatch_meta_bytes=229376 combine_payload_bytes=222076928 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222306304 ingress_bytes=278700032 link_bytes=278700032 token_bytes=444383232 weight_movement_bytes=56623104 total_nvlink_bytes=501006336 weight_to_token_ratio=0.127420 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.670366578e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57258 dup_rows=88 dup_rows_remote=86 dedup_ratio=0.001343 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8112, 8190, 8192, 8188, 8192, 8192, 8190] dup_rows_per_dst=[78, 0, 2, 6, 0, 2, 0, 0] combine_rows_per_src=[8192, 8112, 8190, 8192, 8188, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224559104 dispatch_meta_bytes=219680 combine_payload_bytes=224559104 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=224778784 ingress_bytes=318930944 link_bytes=318930944 token_bytes=449337888 weight_movement_bytes=94371840 total_nvlink_bytes=543709728 weight_to_token_ratio=0.210024 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.055839382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58408 payload_rows_remote=50218 dup_rows=7128 dup_rows_remote=7050 dedup_ratio=0.108765 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4087, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0] payload_rows_per_dst=[8190, 8192, 8192, 1668, 8178, 8180, 7630, 8178] dup_rows_per_dst=[78, 10, 7040, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 1668, 8178, 8180, 7630, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229376 combine_payload_bytes=234528768 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234758144 ingress_bytes=328900608 link_bytes=328900608 token_bytes=469286912 weight_movement_bytes=94371840 total_nvlink_bytes=563658752 weight_to_token_ratio=0.201096 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.151363797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57232 dup_rows=112 dup_rows_remote=112 dedup_ratio=0.001709 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4068, 4088, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0] payload_rows_per_dst=[8190, 8174, 8180, 8188, 8186, 8192, 8192, 8122] dup_rows_per_dst=[8, 0, 0, 92, 0, 0, 12, 0] combine_rows_per_src=[8190, 8174, 8180, 8188, 8186, 8192, 8192, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57342 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8156, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 38, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8156, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55878 payload_rows_remote=50400 dup_rows=9658 dup_rows_remote=6824 dedup_ratio=0.147369 router_tokens_per_expert=[0, 4019, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 54, 0, 0, 0, 0, 4017, 3915, 7, 45, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 1, 0, 0, 37, 0, 0, 0, 2606, 3, 50, 16, 0, 33, 0, 0, 0, 0, 0, 0, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 40, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 39, 29, 4019, 0, 0, 0, 193, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1431, 0] payload_rows_per_dst=[7924, 8186, 2072, 8192, 7732, 8122, 8172, 5478] dup_rows_per_dst=[206, 184, 0, 5334, 564, 96, 440, 2834] combine_rows_per_src=[7924, 8186, 2072, 8192, 7732, 8122, 8172, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62946 payload_rows_remote=54754 dup_rows=2590 dup_rows_remote=2558 dedup_ratio=0.039520 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1268, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 4096, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5858, 8148, 8186, 8192, 8192, 7986] dup_rows_per_dst=[0, 6, 8, 0, 2536, 2, 32, 6] combine_rows_per_src=[8192, 8192, 5858, 8148, 8186, 8192, 8192, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=205692928 dispatch_meta_bytes=229072 combine_payload_bytes=205692928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205922000 ingress_bytes=243441664 link_bytes=243441664 token_bytes=411614928 weight_movement_bytes=37748736 total_nvlink_bytes=449363664 weight_to_token_ratio=0.091709 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332538244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57294 dup_rows=52 dup_rows_remote=10 dedup_ratio=0.000793 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 3898, 4091, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0] payload_rows_per_dst=[8192, 8176, 8182, 8190, 8190, 8192, 8192, 8170] dup_rows_per_dst=[0, 2, 0, 42, 0, 2, 6, 0] combine_rows_per_src=[8192, 8176, 8182, 8190, 8190, 8192, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234422272 dispatch_meta_bytes=229376 combine_payload_bytes=234422272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234651648 ingress_bytes=253296640 link_bytes=253296640 token_bytes=469073920 weight_movement_bytes=18874368 total_nvlink_bytes=487948288 weight_to_token_ratio=0.040238 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426963775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229520 combine_payload_bytes=234872832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235102352 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975184 weight_movement_bytes=37748736 total_nvlink_bytes=507723920 weight_to_token_ratio=0.080321 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=224272384 dispatch_meta_bytes=229248 combine_payload_bytes=224272384 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=224501632 ingress_bytes=299769856 link_bytes=299769856 token_bytes=448774016 weight_movement_bytes=75497472 total_nvlink_bytes=524271488 weight_to_token_ratio=0.168230 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.872247264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206438400 dispatch_meta_bytes=228896 combine_payload_bytes=206438400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206667296 ingress_bytes=225312768 link_bytes=225312768 token_bytes=413105696 weight_movement_bytes=18874368 total_nvlink_bytes=431980064 weight_to_token_ratio=0.045689 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.158836082e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62844 payload_rows_remote=54652 dup_rows=2692 dup_rows_remote=2622 dedup_ratio=0.041077 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 15, 0, 0, 1164, 0, 4, 4, 25, 0, 0, 6, 0, 5, 0, 0, 0, 4074, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2, 0, 4062, 0, 0, 0, 10, 15, 0, 26, 0, 0, 0, 0, 58, 4070, 0, 21, 0, 0, 0, 0, 0, 4048, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 20, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 2842, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8164, 8128, 8182, 8136, 8166, 5700] dup_rows_per_dst=[2338, 70, 34, 0, 218, 0, 32, 0] combine_rows_per_src=[8176, 8192, 8164, 8128, 8182, 8136, 8166, 5700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64686 payload_rows_remote=56726 dup_rows=850 dup_rows_remote=850 dedup_ratio=0.012970 router_tokens_per_expert=[0, 0, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3973, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7576, 8192, 7960, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 848, 0, 0, 0, 0] combine_rows_per_src=[7576, 8192, 7960, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229216 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905440 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581664 weight_movement_bytes=18874368 total_nvlink_bytes=488456032 weight_to_token_ratio=0.040194 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=223854592 dispatch_meta_bytes=229096 combine_payload_bytes=223854592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=224083688 ingress_bytes=261603328 link_bytes=261603328 token_bytes=447938280 weight_movement_bytes=37748736 total_nvlink_bytes=485687016 weight_to_token_ratio=0.084272 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.506554372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62050 payload_rows_remote=54072 dup_rows=3486 dup_rows_remote=1424 dedup_ratio=0.053192 router_tokens_per_expert=[0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7978, 8192, 8192, 4920, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2062, 0, 1334, 90, 0, 0, 0, 0] combine_rows_per_src=[7978, 8192, 8192, 4920, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232349696 dispatch_meta_bytes=230304 combine_payload_bytes=232349696 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=232580000 ingress_bytes=345595904 link_bytes=345595904 token_bytes=464929696 weight_movement_bytes=113246208 total_nvlink_bytes=578175904 weight_to_token_ratio=0.243577 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.311329908e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65438 payload_rows_remote=57246 dup_rows=98 dup_rows_remote=98 dedup_ratio=0.001495 router_tokens_per_expert=[0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8094, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 98, 0, 0, 0, 0] combine_rows_per_src=[8094, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65388 payload_rows_remote=57196 dup_rows=148 dup_rows_remote=144 dedup_ratio=0.002258 router_tokens_per_expert=[0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8046, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 142, 0, 4, 0, 0] combine_rows_per_src=[8046, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57666 payload_rows_remote=49486 dup_rows=7870 dup_rows_remote=7870 dedup_ratio=0.120087 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4093, 0, 0, 0, 0, 0, 0, 371, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 4085, 0, 0, 0, 4090, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 750, 8112, 7878, 8192, 8192, 8180, 8170] dup_rows_per_dst=[0, 10, 5414, 2446, 0, 0, 0, 0] combine_rows_per_src=[8192, 750, 8112, 7878, 8192, 8192, 8180, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62976 payload_rows_remote=55602 dup_rows=2560 dup_rows_remote=1652 dedup_ratio=0.039062 router_tokens_per_expert=[0, 0, 0, 0, 0, 4035, 0, 0, 40, 50, 0, 37, 23, 0, 3, 1, 55, 0, 0, 3982, 0, 0, 0, 0, 0, 4013, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3913, 70, 0, 4010, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 357, 60, 0, 0, 0, 48, 0, 0, 24, 40, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 29, 0, 0, 52, 0, 0, 0, 0, 0, 0, 43, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8124, 6970, 8088, 8130, 8094, 8004, 7374] dup_rows_per_dst=[186, 30, 114, 756, 102, 152, 312, 908] combine_rows_per_src=[8192, 8124, 6970, 8088, 8130, 8094, 8004, 7374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=221478912 dispatch_meta_bytes=221984 combine_payload_bytes=221478912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221700896 ingress_bytes=240353280 link_bytes=240353280 token_bytes=443179808 weight_movement_bytes=18874368 total_nvlink_bytes=462054176 weight_to_token_ratio=0.042589 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.302946868e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64942 payload_rows_remote=56750 dup_rows=594 dup_rows_remote=2 dedup_ratio=0.009064 router_tokens_per_expert=[0, 0, 0, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7638, 8192, 8154, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 592, 0, 2, 0, 0] combine_rows_per_src=[7638, 8192, 8154, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234479616 dispatch_meta_bytes=229376 combine_payload_bytes=234479616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234708992 ingress_bytes=253353984 link_bytes=253353984 token_bytes=469188608 weight_movement_bytes=18874368 total_nvlink_bytes=488062976 weight_to_token_ratio=0.040228 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427513217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234274816 dispatch_meta_bytes=229360 combine_payload_bytes=234274816 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234504176 ingress_bytes=272023552 link_bytes=272023552 token_bytes=468778992 weight_movement_bytes=37748736 total_nvlink_bytes=506527728 weight_to_token_ratio=0.080526 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606395831e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=202694656 dispatch_meta_bytes=229424 combine_payload_bytes=202694656 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=202924080 ingress_bytes=240443392 link_bytes=240443392 token_bytes=405618736 weight_movement_bytes=37748736 total_nvlink_bytes=443367472 weight_to_token_ratio=0.093065 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.303810277e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=227745792 dispatch_meta_bytes=229016 combine_payload_bytes=227745792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227974808 ingress_bytes=246620160 link_bytes=246620160 token_bytes=455720600 weight_movement_bytes=18874368 total_nvlink_bytes=474594968 weight_to_token_ratio=0.041417 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362993028e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64406 payload_rows_remote=56220 dup_rows=1130 dup_rows_remote=1122 dedup_ratio=0.017242 router_tokens_per_expert=[0, 3996, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 15, 0, 0, 0, 0, 4053, 3612, 0, 17, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 6, 0, 0, 7, 0, 0, 0, 459, 18, 21, 8, 0, 10, 0, 0, 0, 0, 0, 0, 17, 16, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 4067, 0, 0, 0, 54, 0, 0, 12, 0, 10, 0, 12, 0, 0, 0, 19, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4069, 0] payload_rows_per_dst=[8090, 8186, 7258, 8192, 8140, 8172, 8190, 8178] dup_rows_per_dst=[4, 8, 0, 972, 16, 0, 126, 4] combine_rows_per_src=[8090, 8186, 7258, 8192, 8140, 8172, 8190, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232448000 dispatch_meta_bytes=227008 combine_payload_bytes=232448000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232675008 ingress_bytes=251322368 link_bytes=251322368 token_bytes=465123008 weight_movement_bytes=18874368 total_nvlink_bytes=483997376 weight_to_token_ratio=0.040579 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408047272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64828 payload_rows_remote=56636 dup_rows=708 dup_rows_remote=0 dedup_ratio=0.010803 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8192, 8192, 7492, 8192, 8192] dup_rows_per_dst=[708, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8184, 8192, 8192, 8192, 7492, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230277120 dispatch_meta_bytes=229368 combine_payload_bytes=230277120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230506488 ingress_bytes=268025856 link_bytes=268025856 token_bytes=460783608 weight_movement_bytes=37748736 total_nvlink_bytes=498532344 weight_to_token_ratio=0.081923 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568091875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231981056 dispatch_meta_bytes=226544 combine_payload_bytes=231981056 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232207600 ingress_bytes=288604160 link_bytes=288604160 token_bytes=464188656 weight_movement_bytes=56623104 total_nvlink_bytes=520811760 weight_to_token_ratio=0.121983 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.765263059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61534 payload_rows_remote=53342 dup_rows=4002 dup_rows_remote=4002 dedup_ratio=0.061066 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 4196, 8192, 8192, 8188] dup_rows_per_dst=[748, 468, 0, 0, 0, 8, 0, 2778] combine_rows_per_src=[8190, 8192, 8192, 8192, 4196, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63444 payload_rows_remote=55384 dup_rows=2092 dup_rows_remote=1894 dedup_ratio=0.031921 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 3906, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 61, 3986, 0, 31, 0, 13, 0, 0, 0, 0, 42, 21, 0, 0, 0, 0, 45, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 16, 4030, 0, 0, 0, 0, 65, 76, 30, 34, 4018, 23, 0, 0, 0, 0, 0, 0, 0, 60, 0, 255, 0, 0, 0, 0, 0, 40, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 69, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7602, 8164, 8046, 8022, 7570, 8192, 7788, 8060] dup_rows_per_dst=[0, 0, 120, 218, 428, 574, 554, 198] combine_rows_per_src=[7602, 8164, 8046, 8022, 7570, 8192, 7788, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=218488832 dispatch_meta_bytes=229376 combine_payload_bytes=218488832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218718208 ingress_bytes=237363200 link_bytes=237363200 token_bytes=437207040 weight_movement_bytes=18874368 total_nvlink_bytes=456081408 weight_to_token_ratio=0.043170 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.274297392e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58774 payload_rows_remote=57320 dup_rows=6762 dup_rows_remote=6754 dedup_ratio=0.103180 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2595, 0, 0, 0, 0, 66, 665, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 772, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 1454, 8192, 8192, 8168, 8192, 8192] dup_rows_per_dst=[0, 5210, 8, 0, 0, 0, 0, 1544] combine_rows_per_src=[8192, 8192, 1454, 8192, 8192, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=226852864 dispatch_meta_bytes=229112 combine_payload_bytes=226852864 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=227081976 ingress_bytes=264601600 link_bytes=264601600 token_bytes=453934840 weight_movement_bytes=37748736 total_nvlink_bytes=491683576 weight_to_token_ratio=0.083159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.535282339e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65148 payload_rows_remote=56956 dup_rows=388 dup_rows_remote=334 dedup_ratio=0.005920 router_tokens_per_expert=[0, 0, 0, 0, 0, 4068, 2, 0, 6, 28, 0, 12, 16, 0, 39, 42, 21, 0, 0, 4068, 0, 0, 0, 0, 0, 4061, 0, 0, 34, 0, 0, 0, 0, 0, 20, 0, 0, 3, 0, 0, 0, 4040, 28, 0, 4058, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 23, 0, 0, 0, 14, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8138, 8158, 8126, 8072, 8080] dup_rows_per_dst=[236, 54, 32, 48, 6, 0, 0, 12] combine_rows_per_src=[8190, 8192, 8192, 8138, 8158, 8126, 8072, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60216 payload_rows_remote=52024 dup_rows=5320 dup_rows_remote=5320 dedup_ratio=0.081177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 1448, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 1518, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8022, 3042] dup_rows_per_dst=[0, 2254, 2896, 0, 170, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8022, 3042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=256296 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235039016 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469821736 weight_movement_bytes=37748736 total_nvlink_bytes=507570472 weight_to_token_ratio=0.080347 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62390 payload_rows_remote=54198 dup_rows=3146 dup_rows_remote=3146 dedup_ratio=0.048004 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 954, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 619, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5048, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1908, 0, 0, 0, 0, 1238] combine_rows_per_src=[8192, 5048, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233291776 dispatch_meta_bytes=229160 combine_payload_bytes=233291776 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233520936 ingress_bytes=289914880 link_bytes=289914880 token_bytes=466812712 weight_movement_bytes=56623104 total_nvlink_bytes=523435816 weight_to_token_ratio=0.121297 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.777821733e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=213090304 dispatch_meta_bytes=229376 combine_payload_bytes=213090304 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=213319680 ingress_bytes=231964672 link_bytes=231964672 token_bytes=426409984 weight_movement_bytes=18874368 total_nvlink_bytes=445284352 weight_to_token_ratio=0.044263 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222571353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61268 payload_rows_remote=53076 dup_rows=4268 dup_rows_remote=4268 dedup_ratio=0.065125 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1962, 0, 0, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 443, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3924, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 3382, 0, 0, 0, 0, 886] combine_rows_per_src=[8192, 3924, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.001343 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 8, 0, 4093, 4092, 0, 0, 0, 14, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8114, 8192, 8188, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 36, 50, 0, 2, 0, 0] combine_rows_per_src=[8192, 8114, 8192, 8188, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63596 payload_rows_remote=55416 dup_rows=1940 dup_rows_remote=756 dedup_ratio=0.029602 router_tokens_per_expert=[27, 0, 76, 0, 1, 0, 3999, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 40, 0, 64, 0, 0, 70, 0, 0, 2, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 53, 0, 24, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 51, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4025, 17, 0, 0, 3784, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 64, 0, 0, 0, 0, 29, 35, 0, 75, 0, 0, 9, 47, 72, 0, 0, 0, 0, 3949, 0, 0, 0, 416, 0, 0, 113, 0, 0] payload_rows_per_dst=[8130, 7426, 8136, 8088, 7966, 7610, 8060, 8180] dup_rows_per_dst=[100, 92, 140, 48, 102, 54, 220, 1184] combine_rows_per_src=[8130, 7426, 8136, 8088, 7966, 7610, 8060, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221995008 dispatch_meta_bytes=229376 combine_payload_bytes=221995008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222224384 ingress_bytes=240869376 link_bytes=240869376 token_bytes=444219392 weight_movement_bytes=18874368 total_nvlink_bytes=463093760 weight_to_token_ratio=0.042489 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307891846e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61476 payload_rows_remote=53286 dup_rows=4060 dup_rows_remote=4060 dedup_ratio=0.061951 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1862, 0, 0, 0, 0, 19, 1673, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 542, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4952, 7374, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 700, 2276, 0, 0, 0, 0, 1084] combine_rows_per_src=[8192, 4952, 7374, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=217399296 dispatch_meta_bytes=229376 combine_payload_bytes=217399296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217628672 ingress_bytes=255148032 link_bytes=255148032 token_bytes=435027968 weight_movement_bytes=37748736 total_nvlink_bytes=472776704 weight_to_token_ratio=0.086773 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.444702902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229376 combine_payload_bytes=234520576 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234749952 ingress_bytes=272269312 link_bytes=272269312 token_bytes=469270528 weight_movement_bytes=37748736 total_nvlink_bytes=507019264 weight_to_token_ratio=0.080441 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608750583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=226983936 dispatch_meta_bytes=224688 combine_payload_bytes=226983936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227208624 ingress_bytes=283607040 link_bytes=283607040 token_bytes=454192560 weight_movement_bytes=56623104 total_nvlink_bytes=510815664 weight_to_token_ratio=0.124668 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.717383114e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65112 payload_rows_remote=56920 dup_rows=424 dup_rows_remote=388 dedup_ratio=0.006470 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4053, 0, 16, 0, 3, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4063, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4057, 0, 0, 0, 0, 16, 18, 8, 15, 4065, 8, 0, 0, 0, 3, 0, 0, 0, 23, 0, 40, 0, 0, 0, 0, 0, 12, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 26, 0, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8074, 8192, 8170, 8138, 8192, 8192, 8030, 8124] dup_rows_per_dst=[0, 36, 48, 6, 194, 126, 14, 0] combine_rows_per_src=[8074, 8192, 8170, 8138, 8192, 8192, 8030, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=218259456 dispatch_meta_bytes=229384 combine_payload_bytes=218259456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218488840 ingress_bytes=237133824 link_bytes=237133824 token_bytes=436748296 weight_movement_bytes=18874368 total_nvlink_bytes=455622664 weight_to_token_ratio=0.043216 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272099624e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64908 payload_rows_remote=56716 dup_rows=628 dup_rows_remote=578 dedup_ratio=0.009583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8156, 8192, 8142, 7650, 8192] dup_rows_per_dst=[50, 38, 0, 0, 540, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8156, 8192, 8142, 7650, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233144320 dispatch_meta_bytes=229232 combine_payload_bytes=233144320 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233373552 ingress_bytes=270893056 link_bytes=270893056 token_bytes=466517872 weight_movement_bytes=37748736 total_nvlink_bytes=504266608 weight_to_token_ratio=0.080916 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595563975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232308736 dispatch_meta_bytes=229176 combine_payload_bytes=232308736 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232537912 ingress_bytes=270057472 link_bytes=270057472 token_bytes=464846648 weight_movement_bytes=37748736 total_nvlink_bytes=502595384 weight_to_token_ratio=0.081207 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.587557820e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64650 payload_rows_remote=56458 dup_rows=886 dup_rows_remote=886 dedup_ratio=0.013519 router_tokens_per_expert=[0, 0, 0, 0, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4095, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7336, 8176, 8190, 8192, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 14, 856, 0, 6, 0, 10] combine_rows_per_src=[7336, 8176, 8190, 8192, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64556 payload_rows_remote=56468 dup_rows=980 dup_rows_remote=880 dedup_ratio=0.014954 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 38, 0, 23, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 84, 0, 0, 49, 31, 0, 12, 4027, 0, 0, 0, 0, 3815, 0, 22, 44, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 29, 50, 3960, 0, 0, 3983, 10, 0, 0, 0, 0, 224, 4020, 0, 0, 0, 0, 78, 0, 0, 0, 0, 31, 0, 49, 3995, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0] payload_rows_per_dst=[8062, 8160, 8130, 8176, 7718, 8092, 8130, 8088] dup_rows_per_dst=[36, 2, 56, 94, 60, 118, 514, 100] combine_rows_per_src=[8062, 8160, 8130, 8176, 7718, 8092, 8130, 8088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57342 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 2] combine_rows_per_src=[8192, 8190, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975040 weight_movement_bytes=56623104 total_nvlink_bytes=526598144 weight_to_token_ratio=0.120481 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=231251968 dispatch_meta_bytes=229376 combine_payload_bytes=231251968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231481344 ingress_bytes=250126336 link_bytes=250126336 token_bytes=462733312 weight_movement_bytes=18874368 total_nvlink_bytes=481607680 weight_to_token_ratio=0.040789 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396587482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=90 dedup_ratio=0.001434 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 4096, 321, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0] payload_rows_per_dst=[8192, 8120, 8192, 8188, 8192, 8192, 8192, 8174] dup_rows_per_dst=[24, 0, 4, 0, 0, 0, 6, 60] combine_rows_per_src=[8192, 8120, 8192, 8188, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231292928 dispatch_meta_bytes=229392 combine_payload_bytes=231292928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231522320 ingress_bytes=250167296 link_bytes=250167296 token_bytes=462815248 weight_movement_bytes=18874368 total_nvlink_bytes=481689616 weight_to_token_ratio=0.040782 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396979940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57870 payload_rows_remote=49678 dup_rows=7666 dup_rows_remote=1198 dedup_ratio=0.116974 router_tokens_per_expert=[12, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3218, 21, 0, 20, 0, 0, 657, 0, 0, 11, 0, 0, 4067, 0, 0, 0, 9, 0, 0, 10, 0, 39, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4023, 1, 0, 0, 14, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4047, 34, 0, 0, 0, 0, 7, 6, 0, 18, 7, 0, 18, 26, 0, 0, 8, 0, 0, 4035, 0, 0, 0, 185, 0, 0, 40, 0, 0] payload_rows_per_dst=[8154, 8192, 2758, 8174, 8134, 8088, 8190, 6180] dup_rows_per_dst=[0, 6468, 6, 78, 12, 658, 84, 360] combine_rows_per_src=[8154, 8192, 2758, 8174, 8134, 8088, 8190, 6180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229392 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235102224 ingress_bytes=310370304 link_bytes=310370304 token_bytes=469975056 weight_movement_bytes=75497472 total_nvlink_bytes=545472528 weight_to_token_ratio=0.160641 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973815541e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64786 payload_rows_remote=56594 dup_rows=750 dup_rows_remote=652 dedup_ratio=0.011444 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 3258, 0, 0, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8098, 8166, 8192, 7562] dup_rows_per_dst=[98, 384, 12, 108, 0, 0, 148, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8098, 8166, 8192, 7562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2365, 0, 0, 4096, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8162] dup_rows_per_dst=[30, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229360 combine_payload_bytes=234496000 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234725360 ingress_bytes=272244736 link_bytes=272244736 token_bytes=469221360 weight_movement_bytes=37748736 total_nvlink_bytes=506970096 weight_to_token_ratio=0.080450 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608515108e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=203481088 dispatch_meta_bytes=203504 combine_payload_bytes=203481088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203684592 ingress_bytes=222355456 link_bytes=222355456 token_bytes=407165680 weight_movement_bytes=18874368 total_nvlink_bytes=426040048 weight_to_token_ratio=0.046355 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.130500574e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=231809024 dispatch_meta_bytes=228984 combine_payload_bytes=231809024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232038008 ingress_bytes=250683392 link_bytes=250683392 token_bytes=463847032 weight_movement_bytes=18874368 total_nvlink_bytes=482721400 weight_to_token_ratio=0.040691 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401924918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=142 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 4096, 1828, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8050] dup_rows_per_dst=[42, 0, 2, 0, 0, 2, 98, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229376 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234987520 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469745664 weight_movement_bytes=18874368 total_nvlink_bytes=488620032 weight_to_token_ratio=0.040180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60204 payload_rows_remote=53106 dup_rows=5332 dup_rows_remote=2346 dedup_ratio=0.081360 router_tokens_per_expert=[0, 1, 0, 0, 0, 49, 0, 0, 0, 909, 0, 0, 0, 0, 188, 0, 76, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 3, 3996, 0, 3972, 0, 0, 9, 0, 0, 0, 0, 0, 26, 0, 52, 0, 0, 31, 0, 26, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 4018, 0, 0, 0, 0, 0, 69, 0, 0, 4, 0, 4008, 0, 57, 0, 0, 0, 82, 36, 0, 17, 3591, 0, 0, 71, 0, 0, 0, 0, 3276, 0, 68, 3973, 7, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0] payload_rows_per_dst=[5158, 8062, 8148, 8010, 8122, 8124, 7482, 7098] dup_rows_per_dst=[1796, 6, 48, 10, 166, 94, 226, 2986] combine_rows_per_src=[5158, 8062, 8148, 8010, 8122, 8124, 7482, 7098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229368 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234520568 ingress_bytes=309788672 link_bytes=309788672 token_bytes=468811768 weight_movement_bytes=75497472 total_nvlink_bytes=544309240 weight_to_token_ratio=0.161040 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968242630e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57328 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 4096, 546, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8184, 8192, 8190, 8190, 8184] dup_rows_per_dst=[8, 0, 0, 0, 0, 0, 0, 16] combine_rows_per_src=[8192, 8188, 8192, 8184, 8192, 8190, 8190, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110408 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991432 weight_movement_bytes=18874368 total_nvlink_bytes=488865800 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=217522176 dispatch_meta_bytes=221808 combine_payload_bytes=217522176 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=217743984 ingress_bytes=293019648 link_bytes=293019648 token_bytes=435266160 weight_movement_bytes=75497472 total_nvlink_bytes=510763632 weight_to_token_ratio=0.173451 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.807570092e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57254 dup_rows=112 dup_rows_remote=112 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4051, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8170, 8102, 8192, 8192, 8192, 8192] dup_rows_per_dst=[28, 2, 0, 0, 0, 0, 0, 82] combine_rows_per_src=[8192, 8192, 8170, 8102, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65296 payload_rows_remote=57114 dup_rows=240 dup_rows_remote=238 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 17, 0, 0, 13, 3, 0, 0, 4066, 0, 1, 0, 0, 3996, 0, 4, 17, 0, 0, 0, 4064, 0, 0, 15, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 1, 0, 0, 0, 23, 4030, 0, 0, 4053, 45, 0, 0, 0, 0, 36, 4060, 0, 0, 0, 0, 14, 0, 0, 0, 0, 11, 0, 27, 4062, 0, 6, 9, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8182, 8188, 8154, 8058, 8192, 8188, 8180] dup_rows_per_dst=[82, 2, 10, 6, 8, 32, 54, 46] combine_rows_per_src=[8154, 8182, 8188, 8154, 8058, 8192, 8188, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63624 payload_rows_remote=55432 dup_rows=1912 dup_rows_remote=840 dedup_ratio=0.029175 router_tokens_per_expert=[0, 0, 94, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3978, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8014, 7342, 8192, 8192, 8192, 7308] dup_rows_per_dst=[1072, 148, 610, 12, 0, 0, 70, 0] combine_rows_per_src=[8192, 8192, 8014, 7342, 8192, 8192, 8192, 7308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64652 payload_rows_remote=56460 dup_rows=884 dup_rows_remote=884 dedup_ratio=0.013489 router_tokens_per_expert=[0, 0, 435, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4036, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 7428, 8072, 8192, 8192, 8192, 8192] dup_rows_per_dst=[870, 0, 0, 12, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 7428, 8072, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229408 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235044896 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860384 weight_movement_bytes=37748736 total_nvlink_bytes=507609120 weight_to_token_ratio=0.080340 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229464 combine_payload_bytes=234512384 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234741848 ingress_bytes=310009856 link_bytes=310009856 token_bytes=469254232 weight_movement_bytes=75497472 total_nvlink_bytes=544751704 weight_to_token_ratio=0.160888 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.970361906e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233938944 dispatch_meta_bytes=229408 combine_payload_bytes=233938944 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234168352 ingress_bytes=271687680 link_bytes=271687680 token_bytes=468107296 weight_movement_bytes=37748736 total_nvlink_bytes=505856032 weight_to_token_ratio=0.080641 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603177671e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=227049472 dispatch_meta_bytes=225088 combine_payload_bytes=227049472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227274560 ingress_bytes=264798208 link_bytes=264798208 token_bytes=454324032 weight_movement_bytes=37748736 total_nvlink_bytes=492072768 weight_to_token_ratio=0.083088 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.537166140e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65208 payload_rows_remote=57016 dup_rows=328 dup_rows_remote=328 dedup_ratio=0.005005 router_tokens_per_expert=[0, 0, 77, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 7882, 8176, 8192, 8192, 8190, 8192] dup_rows_per_dst=[154, 2, 0, 142, 0, 0, 10, 20] combine_rows_per_src=[8192, 8192, 7882, 8176, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=229376 combine_payload_bytes=231260160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231489536 ingress_bytes=250134528 link_bytes=250134528 token_bytes=462749696 weight_movement_bytes=18874368 total_nvlink_bytes=481624064 weight_to_token_ratio=0.040787 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396665973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56522 payload_rows_remote=48708 dup_rows=9014 dup_rows_remote=7712 dedup_ratio=0.137543 router_tokens_per_expert=[78, 0, 566, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 40, 0, 15, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 51, 3864, 0, 47, 0, 3985, 0, 0, 2861, 23, 0, 0, 0, 0, 67, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 32, 0, 62, 0, 0, 0, 0, 0, 57, 0, 0, 0, 60, 0, 0, 92, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 22, 0, 0, 341, 0, 0, 3984, 0, 3981, 0, 0, 0, 0, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 3992] payload_rows_per_dst=[1400, 7928, 8156, 8040, 7824, 7228, 8132, 7814] dup_rows_per_dst=[14, 144, 5864, 646, 312, 730, 2, 1302] combine_rows_per_src=[1400, 7928, 8156, 8040, 7824, 7228, 8132, 7814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62228 payload_rows_remote=54036 dup_rows=3308 dup_rows_remote=3308 dedup_ratio=0.050476 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2442, 0, 0, 0, 0, 25, 1190, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 439, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4886, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2430, 0, 0, 0, 0, 878] combine_rows_per_src=[8192, 4886, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233537536 dispatch_meta_bytes=229376 combine_payload_bytes=233537536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233766912 ingress_bytes=252411904 link_bytes=252411904 token_bytes=467304448 weight_movement_bytes=18874368 total_nvlink_bytes=486178816 weight_to_token_ratio=0.040390 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418486670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=199507968 dispatch_meta_bytes=225680 combine_payload_bytes=199507968 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199733648 ingress_bytes=256131072 link_bytes=256131072 token_bytes=399241616 weight_movement_bytes=56623104 total_nvlink_bytes=455864720 weight_to_token_ratio=0.141827 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.454121908e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60340 payload_rows_remote=52148 dup_rows=5196 dup_rows_remote=5196 dedup_ratio=0.079285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 4096, 0, 4096, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 3006, 8192, 8184] dup_rows_per_dst=[0, 0, 0, 5192, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 3006, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61600 payload_rows_remote=55584 dup_rows=3936 dup_rows_remote=3934 dedup_ratio=0.060059 router_tokens_per_expert=[0, 0, 1958, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3007, 0, 0, 3218, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 6436, 6016, 8192, 8188, 8192, 8192] dup_rows_per_dst=[3916, 0, 0, 2, 0, 0, 4, 14] combine_rows_per_src=[8192, 8192, 6436, 6016, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221331456 dispatch_meta_bytes=229376 combine_payload_bytes=221331456 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=221560832 ingress_bytes=277954560 link_bytes=277954560 token_bytes=442892288 weight_movement_bytes=56623104 total_nvlink_bytes=499515392 weight_to_token_ratio=0.127848 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.663223832e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64048 payload_rows_remote=55894 dup_rows=1488 dup_rows_remote=1480 dedup_ratio=0.022705 router_tokens_per_expert=[0, 1, 0, 0, 0, 24, 0, 0, 0, 514, 0, 0, 0, 0, 6, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 37, 4061, 0, 4064, 0, 0, 18, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 18, 0, 0, 0, 0, 0, 21, 0, 4066, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4051, 0, 25, 0, 0, 0, 17, 8, 0, 243, 3906, 0, 0, 14, 0, 0, 0, 0, 3167, 0, 22, 4043, 256, 0, 0, 0, 11, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8152, 8154, 8178, 8180, 8158, 8162, 8162, 6902] dup_rows_per_dst=[1024, 8, 14, 96, 32, 28, 264, 22] combine_rows_per_src=[8152, 8154, 8178, 8180, 8158, 8162, 8162, 6902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62284 payload_rows_remote=54092 dup_rows=3252 dup_rows_remote=3232 dedup_ratio=0.049622 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4042] payload_rows_per_dst=[8192, 8192, 8192, 8180, 8192, 5400, 7836, 8100] dup_rows_per_dst=[20, 274, 250, 2708, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8180, 8192, 5400, 7836, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61154 payload_rows_remote=52962 dup_rows=4382 dup_rows_remote=4382 dedup_ratio=0.066864 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 3824, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 14, 0, 4368, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 3824, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=213598208 dispatch_meta_bytes=229376 combine_payload_bytes=213598208 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=213827584 ingress_bytes=251346944 link_bytes=251346944 token_bytes=427425792 weight_movement_bytes=37748736 total_nvlink_bytes=465174528 weight_to_token_ratio=0.088316 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408282747e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=227672064 dispatch_meta_bytes=238072 combine_payload_bytes=227672064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227910136 ingress_bytes=246546432 link_bytes=246546432 token_bytes=455582200 weight_movement_bytes=18874368 total_nvlink_bytes=474456568 weight_to_token_ratio=0.041429 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362286603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228941824 dispatch_meta_bytes=229496 combine_payload_bytes=228941824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229171320 ingress_bytes=247816192 link_bytes=247816192 token_bytes=458113144 weight_movement_bytes=18874368 total_nvlink_bytes=476987512 weight_to_token_ratio=0.041200 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374452819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=221560832 dispatch_meta_bytes=229296 combine_payload_bytes=221560832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221790128 ingress_bytes=240435200 link_bytes=240435200 token_bytes=443350960 weight_movement_bytes=18874368 total_nvlink_bytes=462225328 weight_to_token_ratio=0.042572 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.303731785e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=216932352 dispatch_meta_bytes=229376 combine_payload_bytes=216932352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217161728 ingress_bytes=235806720 link_bytes=235806720 token_bytes=434094080 weight_movement_bytes=18874368 total_nvlink_bytes=452968448 weight_to_token_ratio=0.043480 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259383967e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59986 payload_rows_remote=51796 dup_rows=5550 dup_rows_remote=2 dedup_ratio=0.084686 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 4096, 0, 4096, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4015] payload_rows_per_dst=[8192, 8192, 8192, 2772, 8192, 8190, 8192, 8064] dup_rows_per_dst=[0, 0, 0, 2, 0, 5548, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 2772, 8192, 8190, 8192, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64066 payload_rows_remote=56110 dup_rows=1470 dup_rows_remote=860 dedup_ratio=0.022430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3996, 80, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 95, 0, 27, 13, 0, 0, 0, 4000, 61, 3996, 3984, 79, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 91, 0, 0, 0, 3958, 35, 0, 58, 6, 3615, 45, 0, 0, 0, 0, 0, 0, 0, 108, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 26, 3936] payload_rows_per_dst=[8162, 7996, 8028, 8190, 8116, 7470, 8148, 7956] dup_rows_per_dst=[4, 0, 0, 366, 62, 230, 198, 610] combine_rows_per_src=[8162, 7996, 8028, 8190, 8116, 7470, 8148, 7956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=4 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8192, 8160] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 34, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=212156416 dispatch_meta_bytes=207192 combine_payload_bytes=212156416 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=212363608 ingress_bytes=306528256 link_bytes=306528256 token_bytes=424520024 weight_movement_bytes=94371840 total_nvlink_bytes=518891864 weight_to_token_ratio=0.222302 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.937002928e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=229826560 dispatch_meta_bytes=227880 combine_payload_bytes=229826560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230054440 ingress_bytes=267575296 link_bytes=267575296 token_bytes=459881000 weight_movement_bytes=37748736 total_nvlink_bytes=497629736 weight_to_token_ratio=0.082084 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.563774831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57312 dup_rows=60 dup_rows_remote=50 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8164, 8192, 8160, 8192, 8192, 8192] dup_rows_per_dst=[20, 0, 10, 0, 0, 0, 30, 0] combine_rows_per_src=[8192, 8192, 8164, 8192, 8160, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229240 combine_payload_bytes=234725376 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234954616 ingress_bytes=291348480 link_bytes=291348480 token_bytes=469679992 weight_movement_bytes=56623104 total_nvlink_bytes=526303096 weight_to_token_ratio=0.120557 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791557783e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64228 payload_rows_remote=56036 dup_rows=1308 dup_rows_remote=2 dedup_ratio=0.019958 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4095, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6924, 8190, 8154] dup_rows_per_dst=[0, 2, 0, 1306, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6924, 8190, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64462 payload_rows_remote=56270 dup_rows=1074 dup_rows_remote=950 dedup_ratio=0.016388 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8192, 7484, 8192, 8190, 7840] dup_rows_per_dst=[124, 600, 0, 122, 0, 228, 0, 0] combine_rows_per_src=[8192, 8192, 8180, 8192, 7484, 8192, 8190, 7840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61826 payload_rows_remote=53726 dup_rows=3710 dup_rows_remote=3650 dedup_ratio=0.056610 router_tokens_per_expert=[8, 0, 267, 0, 0, 0, 4053, 0, 0, 0, 13, 0, 0, 0, 10, 0, 0, 0, 0, 0, 15, 0, 27, 0, 0, 0, 0, 0, 15, 0, 0, 33, 0, 0, 3990, 0, 20, 0, 4037, 0, 0, 2409, 16, 0, 0, 0, 0, 0, 0, 1391, 0, 0, 0, 0, 0, 31, 0, 0, 0, 10, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 19, 0, 0, 0, 35, 0, 0, 0, 0, 28, 0, 0, 0, 0, 16, 0, 0, 0, 15, 0, 16, 0, 0, 91, 0, 0, 4045, 0, 4053, 0, 0, 1, 0, 4, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 4059] payload_rows_per_dst=[8150, 8100, 4888, 8172, 8192, 8186, 8152, 7986] dup_rows_per_dst=[552, 60, 2, 2766, 56, 264, 10, 0] combine_rows_per_src=[8150, 8100, 4888, 8172, 8192, 8186, 8152, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229448 combine_payload_bytes=234749952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234979400 ingress_bytes=253624320 link_bytes=253624320 token_bytes=469729352 weight_movement_bytes=18874368 total_nvlink_bytes=488603720 weight_to_token_ratio=0.040181 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430103443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=229523456 dispatch_meta_bytes=224152 combine_payload_bytes=229523456 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=229747608 ingress_bytes=305020928 link_bytes=305020928 token_bytes=459271064 weight_movement_bytes=75497472 total_nvlink_bytes=534768536 weight_to_token_ratio=0.164385 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.922560452e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230481920 dispatch_meta_bytes=228880 combine_payload_bytes=230481920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230710800 ingress_bytes=268230656 link_bytes=268230656 token_bytes=461192720 weight_movement_bytes=37748736 total_nvlink_bytes=498941456 weight_to_token_ratio=0.081850 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.570054168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229344 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077600 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925856 weight_movement_bytes=18874368 total_nvlink_bytes=488800224 weight_to_token_ratio=0.040165 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=220061696 dispatch_meta_bytes=229504 combine_payload_bytes=220061696 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=220291200 ingress_bytes=257810432 link_bytes=257810432 token_bytes=440352896 weight_movement_bytes=37748736 total_nvlink_bytes=478101632 weight_to_token_ratio=0.085724 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.470212709e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8158, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 44, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 8158, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58866 payload_rows_remote=50674 dup_rows=6670 dup_rows_remote=6412 dedup_ratio=0.101776 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3960, 37, 117, 0, 0, 0, 149, 0, 51, 4008, 0, 72, 0, 0, 0, 0, 0, 3586, 0, 0, 3982, 0, 50, 0, 0, 22, 0, 0, 0, 0, 17, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 24, 0, 0, 0, 0, 0, 37, 0, 3, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 3979, 2, 3995, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 3951, 0, 0, 0, 48, 22, 0, 0, 0, 2661, 0, 0, 0, 0, 0, 0, 87, 55, 0, 0, 0, 0, 0, 60, 0, 44, 0, 0, 0, 0, 0] payload_rows_per_dst=[7736, 7662, 8096, 5382, 8116, 8056, 5626, 8192] dup_rows_per_dst=[76, 474, 178, 2430, 462, 4, 2788, 258] combine_rows_per_src=[7736, 7662, 8096, 5382, 8116, 8056, 5626, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57154 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.002899 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 4096, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8038, 8170, 8186, 8192, 8192, 8192, 8184] dup_rows_per_dst=[52, 0, 0, 0, 0, 2, 0, 136] combine_rows_per_src=[8192, 8038, 8170, 8186, 8192, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229376 combine_payload_bytes=234700800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234930176 ingress_bytes=291323904 link_bytes=291323904 token_bytes=469630976 weight_movement_bytes=56623104 total_nvlink_bytes=526254080 weight_to_token_ratio=0.120569 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791322308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=207560704 dispatch_meta_bytes=228344 combine_payload_bytes=207560704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207789048 ingress_bytes=226435072 link_bytes=226435072 token_bytes=415349752 weight_movement_bytes=18874368 total_nvlink_bytes=434224120 weight_to_token_ratio=0.045442 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169589447e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57312 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 4, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8192, 8192, 8192, 8166, 8186] dup_rows_per_dst=[4, 0, 0, 2, 0, 38, 0, 0] combine_rows_per_src=[8192, 8192, 8180, 8192, 8192, 8192, 8166, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234102784 dispatch_meta_bytes=229376 combine_payload_bytes=234102784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234332160 ingress_bytes=252977152 link_bytes=252977152 token_bytes=468434944 weight_movement_bytes=18874368 total_nvlink_bytes=487309312 weight_to_token_ratio=0.040292 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423902598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=8 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8182, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63920 payload_rows_remote=55728 dup_rows=1616 dup_rows_remote=1542 dedup_ratio=0.024658 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8122, 8192, 8192, 7972, 8134, 6924] dup_rows_per_dst=[74, 0, 0, 220, 1268, 54, 0, 0] combine_rows_per_src=[8192, 8192, 8122, 8192, 8192, 7972, 8134, 6924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8174, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59842 payload_rows_remote=51750 dup_rows=5694 dup_rows_remote=5694 dedup_ratio=0.086884 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 9, 0, 4061, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 34, 0, 11, 4, 0, 0, 0, 4063, 20, 4057, 2638, 33, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1488, 0, 39, 0, 0, 0, 4038, 24, 0, 12, 12, 3978, 28, 0, 0, 0, 4, 0, 0, 0, 109, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 14, 3900] payload_rows_per_dst=[8180, 8092, 8128, 8134, 8182, 3066, 8184, 7876] dup_rows_per_dst=[18, 0, 0, 5296, 24, 140, 210, 6] combine_rows_per_src=[8180, 8092, 8128, 8134, 8182, 3066, 8184, 7876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229424 combine_payload_bytes=234749952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234979376 ingress_bytes=291373056 link_bytes=291373056 token_bytes=469729328 weight_movement_bytes=56623104 total_nvlink_bytes=526352432 weight_to_token_ratio=0.120544 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791793258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229360 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061232 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893104 weight_movement_bytes=18874368 total_nvlink_bytes=488767472 weight_to_token_ratio=0.040167 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228261888 dispatch_meta_bytes=229080 combine_payload_bytes=228261888 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228490968 ingress_bytes=266010624 link_bytes=266010624 token_bytes=456752856 weight_movement_bytes=37748736 total_nvlink_bytes=494501592 weight_to_token_ratio=0.082646 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.548782914e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64756 payload_rows_remote=56564 dup_rows=780 dup_rows_remote=0 dedup_ratio=0.011902 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 0, 10, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7440, 8164] dup_rows_per_dst=[0, 0, 0, 0, 0, 780, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7440, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=211968000 dispatch_meta_bytes=229776 combine_payload_bytes=211968000 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=212197776 ingress_bytes=249716736 link_bytes=249716736 token_bytes=424165776 weight_movement_bytes=37748736 total_nvlink_bytes=461914512 weight_to_token_ratio=0.088995 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392662896e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63134 payload_rows_remote=54942 dup_rows=2402 dup_rows_remote=1314 dedup_ratio=0.036652 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 47, 29, 0, 3958, 0, 0, 0, 0, 0, 0, 54, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 4001, 0, 0, 0, 0, 75, 3998, 0, 0, 0, 0, 16, 15, 0, 0, 30, 0, 0, 0, 79, 0, 41, 0, 0, 0, 3244, 0, 0, 0, 154, 107, 0, 0, 0, 0, 0, 2226, 0, 0, 1, 0, 0, 4015, 0, 0, 0, 0, 0, 55, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 52, 3989, 0, 194, 0, 0, 0, 0, 43, 0, 0, 31, 0, 87, 0] payload_rows_per_dst=[8136, 8104, 8122, 7704, 7060, 8134, 7682, 8192] dup_rows_per_dst=[0, 52, 114, 234, 32, 132, 750, 1088] combine_rows_per_src=[8136, 8104, 8122, 7704, 7060, 8134, 7682, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65254 payload_rows_remote=57062 dup_rows=282 dup_rows_remote=276 dedup_ratio=0.004303 router_tokens_per_expert=[0, 0, 114, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4017, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 4086] payload_rows_per_dst=[8192, 8192, 8086, 8036, 8192, 8172, 8192, 8192] dup_rows_per_dst=[228, 0, 0, 0, 0, 0, 6, 48] combine_rows_per_src=[8192, 8192, 8086, 8036, 8192, 8172, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=231686144 dispatch_meta_bytes=226256 combine_payload_bytes=231686144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231912400 ingress_bytes=269434880 link_bytes=269434880 token_bytes=463598544 weight_movement_bytes=37748736 total_nvlink_bytes=501347280 weight_to_token_ratio=0.081425 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581592450e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57152 dup_rows=192 dup_rows_remote=192 dedup_ratio=0.002930 router_tokens_per_expert=[4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 94, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8038, 8184, 8192, 8192, 8162, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 190, 0] combine_rows_per_src=[8038, 8184, 8192, 8192, 8162, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=225042432 dispatch_meta_bytes=225024 combine_payload_bytes=225042432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225267456 ingress_bytes=262791168 link_bytes=262791168 token_bytes=450309888 weight_movement_bytes=37748736 total_nvlink_bytes=488058624 weight_to_token_ratio=0.083828 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.517935670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233725952 dispatch_meta_bytes=229352 combine_payload_bytes=233725952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233955304 ingress_bytes=271474688 link_bytes=271474688 token_bytes=467681256 weight_movement_bytes=37748736 total_nvlink_bytes=505429992 weight_to_token_ratio=0.080715 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601136887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=4 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8182] dup_rows_per_dst=[0, 0, 0, 10, 0, 4, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63108 payload_rows_remote=54916 dup_rows=2428 dup_rows_remote=2026 dedup_ratio=0.037048 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7944, 8192, 8192, 6014, 8192] dup_rows_per_dst=[402, 248, 948, 52, 778, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 7944, 8192, 8192, 6014, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57344 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64404 payload_rows_remote=56798 dup_rows=1132 dup_rows_remote=1092 dedup_ratio=0.017273 router_tokens_per_expert=[0, 0, 1, 31, 0, 0, 0, 0, 0, 0, 4038, 8, 44, 0, 0, 0, 54, 0, 32, 4061, 0, 26, 0, 0, 0, 0, 0, 3711, 0, 0, 4030, 0, 14, 0, 0, 9, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 3, 0, 7, 1, 0, 13, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4045, 0, 4058, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4029, 0, 0, 0, 48, 2, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 33, 19, 0, 0, 0, 0, 0, 29, 0, 17, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 7606, 8102, 8122, 8170, 8156, 7894, 8190] dup_rows_per_dst=[80, 40, 46, 274, 596, 0, 0, 96] combine_rows_per_src=[8164, 7606, 8102, 8122, 8170, 8156, 7894, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229376 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234323968 ingress_bytes=252968960 link_bytes=252968960 token_bytes=468418560 weight_movement_bytes=18874368 total_nvlink_bytes=487292928 weight_to_token_ratio=0.040294 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423824106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229336 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235053016 ingress_bytes=272572416 link_bytes=272572416 token_bytes=469876696 weight_movement_bytes=37748736 total_nvlink_bytes=507625432 weight_to_token_ratio=0.080338 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611654776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=224935936 dispatch_meta_bytes=227768 combine_payload_bytes=224935936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225163704 ingress_bytes=243810304 link_bytes=243810304 token_bytes=450099640 weight_movement_bytes=18874368 total_nvlink_bytes=468974008 weight_to_token_ratio=0.041934 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.336070371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229392 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=235110416 ingress_bytes=348127232 link_bytes=348127232 token_bytes=469991440 weight_movement_bytes=113246208 total_nvlink_bytes=583237648 weight_to_token_ratio=0.240954 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.335583848e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.001343 router_tokens_per_expert=[4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8110, 8190, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 84, 0] combine_rows_per_src=[8110, 8190, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232644608 dispatch_meta_bytes=231560 combine_payload_bytes=232644608 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232876168 ingress_bytes=289267712 link_bytes=289267712 token_bytes=465520776 weight_movement_bytes=56623104 total_nvlink_bytes=522143880 weight_to_token_ratio=0.121634 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.771620888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62918 payload_rows_remote=54766 dup_rows=2618 dup_rows_remote=2564 dedup_ratio=0.039948 router_tokens_per_expert=[4008, 0, 0, 0, 0, 0, 0, 56, 0, 35, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 11, 0, 3995, 0, 0, 0, 0, 0, 0, 9, 0, 0, 53, 60, 0, 79, 3447, 0, 0, 0, 12, 8, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 34, 0, 0, 0, 62, 0, 3971, 0, 0, 60, 0, 3935, 0, 41, 3888, 100, 0, 0, 5, 63, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 7, 638, 0, 0, 0, 0, 0, 0, 0, 61, 0, 3960, 0, 65, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8030, 7128, 8116, 7372, 7768, 8164, 8152] dup_rows_per_dst=[146, 0, 190, 282, 112, 690, 1144, 54] combine_rows_per_src=[8188, 8030, 7128, 8116, 7372, 7768, 8164, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60402 payload_rows_remote=52210 dup_rows=5134 dup_rows_remote=5132 dedup_ratio=0.078339 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4085, 0, 4096, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2974, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090] payload_rows_per_dst=[8192, 8192, 8170, 3170, 8192, 8114, 8192, 8180] dup_rows_per_dst=[0, 20, 0, 0, 0, 4902, 2, 210] combine_rows_per_src=[8192, 8192, 8170, 3170, 8192, 8114, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229376 combine_payload_bytes=234520576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234749952 ingress_bytes=253394944 link_bytes=253394944 token_bytes=469270528 weight_movement_bytes=18874368 total_nvlink_bytes=488144896 weight_to_token_ratio=0.040221 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427905675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=224321536 dispatch_meta_bytes=229320 combine_payload_bytes=224321536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224550856 ingress_bytes=243195904 link_bytes=243195904 token_bytes=448872392 weight_movement_bytes=18874368 total_nvlink_bytes=467746760 weight_to_token_ratio=0.042048 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330183492e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=213852160 dispatch_meta_bytes=229368 combine_payload_bytes=213852160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214081528 ingress_bytes=232726528 link_bytes=232726528 token_bytes=427933688 weight_movement_bytes=18874368 total_nvlink_bytes=446808056 weight_to_token_ratio=0.044106 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229871083e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58780 payload_rows_remote=50588 dup_rows=6756 dup_rows_remote=3594 dedup_ratio=0.103088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 1436, 8192] dup_rows_per_dst=[3550, 0, 3162, 0, 36, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 1436, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57232 dup_rows=112 dup_rows_remote=112 dedup_ratio=0.001709 router_tokens_per_expert=[4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8098, 8192, 8174, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 106, 0] combine_rows_per_src=[8098, 8192, 8174, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61650 payload_rows_remote=53458 dup_rows=3886 dup_rows_remote=3332 dedup_ratio=0.059296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5946, 8192, 6584, 8160] dup_rows_per_dst=[554, 0, 3108, 194, 2, 28, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5946, 8192, 6584, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57666 payload_rows_remote=49474 dup_rows=7870 dup_rows_remote=7312 dedup_ratio=0.120087 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 1362, 7152, 8192] dup_rows_per_dst=[3442, 0, 392, 0, 558, 4, 3474, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 1362, 7152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64666 payload_rows_remote=56484 dup_rows=870 dup_rows_remote=824 dedup_ratio=0.013275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 41, 23, 0, 4043, 0, 6, 0, 0, 0, 1, 31, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4062, 0, 0, 0, 0, 24, 4057, 0, 0, 0, 0, 0, 4, 0, 0, 15, 0, 0, 0, 7, 0, 14, 0, 0, 0, 3938, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 2221, 0, 20, 2, 0, 0, 2052, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 0, 0, 0, 0, 29, 4034, 0, 100, 0, 0, 0, 0, 16, 0, 0, 10, 0, 34, 0] payload_rows_per_dst=[8172, 8182, 8186, 8156, 8010, 8166, 7602, 8192] dup_rows_per_dst=[0, 46, 64, 58, 14, 432, 2, 254] combine_rows_per_src=[8172, 8182, 8186, 8156, 8010, 8166, 7602, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=207208448 dispatch_meta_bytes=216728 combine_payload_bytes=207208448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=207425176 ingress_bytes=244957184 link_bytes=244957184 token_bytes=414633624 weight_movement_bytes=37748736 total_nvlink_bytes=452382360 weight_to_token_ratio=0.091041 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.347059211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234422272 dispatch_meta_bytes=229376 combine_payload_bytes=234422272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234651648 ingress_bytes=272171008 link_bytes=272171008 token_bytes=469073920 weight_movement_bytes=37748736 total_nvlink_bytes=506822656 weight_to_token_ratio=0.080475 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607808682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=218963968 dispatch_meta_bytes=227160 combine_payload_bytes=218963968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219191128 ingress_bytes=237838336 link_bytes=237838336 token_bytes=438155096 weight_movement_bytes=18874368 total_nvlink_bytes=457029464 weight_to_token_ratio=0.043077 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.278849912e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=202645504 dispatch_meta_bytes=227144 combine_payload_bytes=202645504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202872648 ingress_bytes=221519872 link_bytes=221519872 token_bytes=405518152 weight_movement_bytes=18874368 total_nvlink_bytes=424392520 weight_to_token_ratio=0.046544 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.122494419e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57632 payload_rows_remote=55370 dup_rows=7904 dup_rows_remote=7900 dedup_ratio=0.120605 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6218, 8192, 8192, 8192, 8192, 2262, 8192, 8192] dup_rows_per_dst=[1118, 0, 2034, 0, 1024, 4, 3724, 0] combine_rows_per_src=[6218, 8192, 8192, 8192, 8192, 2262, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49972 payload_rows_remote=41782 dup_rows=15564 dup_rows_remote=15532 dedup_ratio=0.237488 router_tokens_per_expert=[0, 99, 0, 39, 0, 0, 0, 0, 0, 16, 0, 133, 6, 0, 0, 0, 90, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4, 3975, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 305, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 3992, 0, 3993, 0, 0, 22, 0, 101, 0, 0, 43, 0, 0, 0, 0, 62, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 7, 0, 0, 0, 0, 3902, 0, 3958, 0, 0, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 69, 0, 0, 0, 0, 0, 94, 0, 0, 0, 3974, 0, 0, 31, 0, 11, 0, 0, 0] payload_rows_per_dst=[504, 8174, 622, 8058, 8192, 8186, 8046, 8190] dup_rows_per_dst=[82, 12, 2, 14, 266, 7926, 7230, 32] combine_rows_per_src=[504, 8174, 622, 8058, 8192, 8186, 8046, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=231358464 dispatch_meta_bytes=229232 combine_payload_bytes=231358464 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=231587696 ingress_bytes=250232832 link_bytes=250232832 token_bytes=462946160 weight_movement_bytes=18874368 total_nvlink_bytes=481820528 weight_to_token_ratio=0.040770 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397607874e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65294 payload_rows_remote=57102 dup_rows=242 dup_rows_remote=186 dedup_ratio=0.003693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 88, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 32, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8000, 8142, 8192, 8192] dup_rows_per_dst=[176, 2, 2, 6, 0, 0, 56, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8000, 8142, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=226795520 dispatch_meta_bytes=253080 combine_payload_bytes=226795520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227048600 ingress_bytes=283418624 link_bytes=283418624 token_bytes=453844120 weight_movement_bytes=56623104 total_nvlink_bytes=510467224 weight_to_token_ratio=0.124763 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.715577805e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 304, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 2, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=171139072 dispatch_meta_bytes=229256 combine_payload_bytes=171139072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=171368328 ingress_bytes=208887808 link_bytes=208887808 token_bytes=342507400 weight_movement_bytes=37748736 total_nvlink_bytes=380256136 weight_to_token_ratio=0.110213 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.001460197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233889792 dispatch_meta_bytes=229152 combine_payload_bytes=233889792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234118944 ingress_bytes=271638528 link_bytes=271638528 token_bytes=468008736 weight_movement_bytes=37748736 total_nvlink_bytes=505757472 weight_to_token_ratio=0.080658 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602706721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60852 payload_rows_remote=52660 dup_rows=4684 dup_rows_remote=4684 dedup_ratio=0.071472 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 3510, 8192] dup_rows_per_dst=[4366, 0, 62, 0, 172, 84, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 3510, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=60 dedup_ratio=0.001007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 10, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8126] dup_rows_per_dst=[0, 0, 60, 0, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63612 payload_rows_remote=55420 dup_rows=1924 dup_rows_remote=592 dedup_ratio=0.029358 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 43, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 0, 0, 0, 0, 0, 0, 388, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7706, 8192, 6754] dup_rows_per_dst=[1332, 0, 20, 0, 572, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7706, 8192, 6754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235085824 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942272 weight_movement_bytes=56623104 total_nvlink_bytes=526565376 weight_to_token_ratio=0.120489 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.104167 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64922 payload_rows_remote=56806 dup_rows=614 dup_rows_remote=614 dedup_ratio=0.009369 router_tokens_per_expert=[4059, 0, 0, 0, 0, 0, 0, 29, 0, 10, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 28, 4040, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4049, 0, 0, 16, 0, 4023, 0, 16, 3895, 30, 0, 0, 34, 28, 0, 68, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 0, 0, 46, 0, 0, 0, 27, 176, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4046, 0, 25, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8116, 8138, 8140, 8044, 7926, 8192, 8174] dup_rows_per_dst=[48, 0, 14, 42, 102, 0, 404, 4] combine_rows_per_src=[8192, 8116, 8138, 8140, 8044, 7926, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=215695360 dispatch_meta_bytes=229376 combine_payload_bytes=215695360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=215924736 ingress_bytes=272318464 link_bytes=272318464 token_bytes=431620096 weight_movement_bytes=56623104 total_nvlink_bytes=488243200 weight_to_token_ratio=0.131187 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609221533e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229352 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234840040 ingress_bytes=310108160 link_bytes=310108160 token_bytes=469450728 weight_movement_bytes=75497472 total_nvlink_bytes=544948200 weight_to_token_ratio=0.160821 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.971303806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=227000320 dispatch_meta_bytes=224048 combine_payload_bytes=227000320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227224368 ingress_bytes=245874688 link_bytes=245874688 token_bytes=454224688 weight_movement_bytes=18874368 total_nvlink_bytes=473099056 weight_to_token_ratio=0.041553 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.355850283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8024] dup_rows_per_dst=[0, 0, 168, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232677376 dispatch_meta_bytes=229680 combine_payload_bytes=232677376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232907056 ingress_bytes=251551744 link_bytes=251551744 token_bytes=465584432 weight_movement_bytes=18874368 total_nvlink_bytes=484458800 weight_to_token_ratio=0.040539 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410245040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62536 payload_rows_remote=54530 dup_rows=3000 dup_rows_remote=1404 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 14, 0, 27, 0, 72, 37, 0, 0, 3487, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 90, 0, 0, 0, 42, 0, 0, 0, 0, 0, 37, 3863, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 24, 0, 0, 0, 0, 3953, 32, 0, 0, 3833, 68, 3947, 0, 79, 0, 0, 0, 0, 0, 8, 0, 6, 61, 0, 0, 7, 0, 103, 0, 0, 0, 82, 0, 68, 4, 0, 0, 3984, 0, 0, 0, 28, 0, 0, 77, 0, 0, 0, 0, 0, 5, 0, 0, 461, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 51, 0] payload_rows_per_dst=[7192, 8008, 7630, 7994, 7404, 8192, 8110, 8006] dup_rows_per_dst=[82, 190, 272, 250, 164, 364, 82, 1596] combine_rows_per_src=[7192, 8008, 7630, 7994, 7404, 8192, 8110, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64236 payload_rows_remote=56062 dup_rows=1300 dup_rows_remote=78 dedup_ratio=0.019836 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 9, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8192, 8192, 6970, 8174, 8162] dup_rows_per_dst=[0, 0, 12, 0, 0, 66, 1222, 0] combine_rows_per_src=[8192, 8192, 8162, 8192, 8192, 6970, 8174, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63124 payload_rows_remote=54932 dup_rows=2412 dup_rows_remote=20 dedup_ratio=0.036804 router_tokens_per_expert=[0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 5800, 8172] dup_rows_per_dst=[20, 0, 2392, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 5800, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=223354880 dispatch_meta_bytes=223736 combine_payload_bytes=223354880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=223578616 ingress_bytes=261103616 link_bytes=261103616 token_bytes=446933496 weight_movement_bytes=37748736 total_nvlink_bytes=484682232 weight_to_token_ratio=0.084462 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.501766378e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=229629952 dispatch_meta_bytes=224560 combine_payload_bytes=229629952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229854512 ingress_bytes=248504320 link_bytes=248504320 token_bytes=459484464 weight_movement_bytes=18874368 total_nvlink_bytes=478358832 weight_to_token_ratio=0.041077 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381046122e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 87, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59896 payload_rows_remote=51704 dup_rows=5640 dup_rows_remote=5640 dedup_ratio=0.086060 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2896, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8136, 8192, 8192, 8192, 2608, 8192] dup_rows_per_dst=[0, 0, 5584, 0, 0, 0, 56, 0] combine_rows_per_src=[8192, 8192, 8136, 8192, 8192, 8192, 2608, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61428 payload_rows_remote=53236 dup_rows=4108 dup_rows_remote=4108 dedup_ratio=0.062683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7682, 8192, 8192, 7398, 6074, 7506] dup_rows_per_dst=[0, 1824, 1922, 0, 0, 0, 362, 0] combine_rows_per_src=[8192, 8192, 7682, 8192, 8192, 7398, 6074, 7506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225001472 dispatch_meta_bytes=219808 combine_payload_bytes=225001472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225221280 ingress_bytes=281624576 link_bytes=281624576 token_bytes=450222752 weight_movement_bytes=56623104 total_nvlink_bytes=506845856 weight_to_token_ratio=0.125767 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.698388119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64898 payload_rows_remote=56720 dup_rows=638 dup_rows_remote=618 dedup_ratio=0.009735 router_tokens_per_expert=[0, 44, 0, 14, 0, 0, 0, 0, 0, 10, 0, 37, 96, 0, 0, 0, 29, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 105, 23, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4048, 0, 4056, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 23, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 3993, 0, 0, 0, 3874, 0, 0, 0, 0, 3, 0, 0, 0, 0, 10, 0, 18, 0, 0, 0, 0, 23, 36, 0, 0, 0, 4047, 14, 0, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[8140, 8178, 8136, 8120, 8186, 8136, 7810, 8192] dup_rows_per_dst=[248, 20, 230, 0, 78, 0, 0, 62] combine_rows_per_src=[8140, 8178, 8136, 8120, 8186, 8136, 7810, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053056 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876736 weight_movement_bytes=18874368 total_nvlink_bytes=488751104 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=211779584 dispatch_meta_bytes=229376 combine_payload_bytes=211779584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212008960 ingress_bytes=249528320 link_bytes=249528320 token_bytes=423788544 weight_movement_bytes=37748736 total_nvlink_bytes=461537280 weight_to_token_ratio=0.089074 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390857587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=218054656 dispatch_meta_bytes=229376 combine_payload_bytes=218054656 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=218284032 ingress_bytes=255803392 link_bytes=255803392 token_bytes=436338688 weight_movement_bytes=37748736 total_nvlink_bytes=474087424 weight_to_token_ratio=0.086512 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450982239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61814 payload_rows_remote=53622 dup_rows=3722 dup_rows_remote=3722 dedup_ratio=0.056793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2235, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4470, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3722, 0] combine_rows_per_src=[8192, 8192, 4470, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=232325120 dispatch_meta_bytes=229352 combine_payload_bytes=232325120 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=232554472 ingress_bytes=251199488 link_bytes=251199488 token_bytes=464879592 weight_movement_bytes=18874368 total_nvlink_bytes=483753960 weight_to_token_ratio=0.040601 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406869896e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55468 payload_rows_remote=47446 dup_rows=10068 dup_rows_remote=9354 dedup_ratio=0.153625 router_tokens_per_expert=[0, 0, 0, 37, 85, 0, 92, 0, 0, 0, 0, 0, 5, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 75, 8, 0, 0, 0, 0, 66, 0, 0, 0, 3879, 70, 0, 120, 0, 0, 1, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 59, 0, 0, 0, 3949, 6, 76, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 3986, 70, 0, 38, 111, 0, 0, 14, 3942, 0, 59, 0, 0, 0, 54, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 37, 13, 0, 336, 0, 0, 0, 0] payload_rows_per_dst=[7830, 7022, 360, 8072, 7896, 8192, 8074, 8022] dup_rows_per_dst=[180, 0, 158, 142, 354, 636, 7884, 714] combine_rows_per_src=[7830, 7022, 360, 8072, 7896, 8192, 8074, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57068 dup_rows=276 dup_rows_remote=2 dedup_ratio=0.004211 router_tokens_per_expert=[3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7930, 8192, 8178, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 274, 0] combine_rows_per_src=[7930, 8192, 8178, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=219635712 dispatch_meta_bytes=229376 combine_payload_bytes=219635712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219865088 ingress_bytes=257384448 link_bytes=257384448 token_bytes=439500800 weight_movement_bytes=37748736 total_nvlink_bytes=477249536 weight_to_token_ratio=0.085890 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.466131140e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8154, 8192, 8192, 8192, 8192] dup_rows_per_dst=[38, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8154, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233750528 dispatch_meta_bytes=228280 combine_payload_bytes=233750528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233978808 ingress_bytes=252624896 link_bytes=252624896 token_bytes=467729336 weight_movement_bytes=18874368 total_nvlink_bytes=486603704 weight_to_token_ratio=0.040353 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420527454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=194338816 dispatch_meta_bytes=227200 combine_payload_bytes=194338816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194566016 ingress_bytes=213213184 link_bytes=213213184 token_bytes=388904832 weight_movement_bytes=18874368 total_nvlink_bytes=407779200 weight_to_token_ratio=0.048532 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042903822e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60728 payload_rows_remote=52536 dup_rows=4808 dup_rows_remote=4808 dedup_ratio=0.073364 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2402, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 3388, 8188] dup_rows_per_dst=[4, 0, 4804, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 3388, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8172, 8192, 8192, 8192, 8192] dup_rows_per_dst=[20, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8172, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64144 payload_rows_remote=55952 dup_rows=1392 dup_rows_remote=1392 dedup_ratio=0.021240 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7518, 8192, 7474] dup_rows_per_dst=[0, 262, 0, 394, 18, 0, 718, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7518, 8192, 7474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229376 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234954752 ingress_bytes=253599744 link_bytes=253599744 token_bytes=469680128 weight_movement_bytes=18874368 total_nvlink_bytes=488554496 weight_to_token_ratio=0.040186 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429867968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64634 payload_rows_remote=56508 dup_rows=902 dup_rows_remote=844 dedup_ratio=0.013763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 1, 15, 0, 25, 2, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 27, 0, 0, 0, 0, 0, 26, 3993, 0, 38, 0, 0, 0, 0, 0, 0, 0, 29, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 24, 0, 0, 4011, 0, 4035, 0, 45, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 3, 0, 37, 0, 0, 0, 7, 0, 21, 0, 0, 0, 4047, 0, 0, 0, 11, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 8, 0, 0, 28, 0] payload_rows_per_dst=[7740, 8126, 8096, 8108, 8112, 8172, 8146, 8134] dup_rows_per_dst=[38, 58, 138, 46, 48, 142, 16, 416] combine_rows_per_src=[7740, 8126, 8096, 8108, 8112, 8172, 8146, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=215187456 dispatch_meta_bytes=229376 combine_payload_bytes=215187456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215416832 ingress_bytes=234061824 link_bytes=234061824 token_bytes=430604288 weight_movement_bytes=18874368 total_nvlink_bytes=449478656 weight_to_token_ratio=0.043832 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.242665232e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=229179392 dispatch_meta_bytes=229376 combine_payload_bytes=229179392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229408768 ingress_bytes=248053760 link_bytes=248053760 token_bytes=458588160 weight_movement_bytes=18874368 total_nvlink_bytes=477462528 weight_to_token_ratio=0.041158 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376729078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61122 payload_rows_remote=53206 dup_rows=4414 dup_rows_remote=3272 dedup_ratio=0.067352 router_tokens_per_expert=[0, 0, 0, 0, 88, 0, 0, 101, 242, 0, 0, 0, 0, 0, 19, 0, 0, 0, 92, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 109, 32, 0, 0, 0, 45, 0, 0, 1, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 74, 0, 3936, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 3723, 0, 60, 3992, 96, 0, 0, 33, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3660, 0, 108, 71, 0, 0, 0, 2, 0, 3901, 0, 0, 8, 3975, 1, 0, 0, 2, 0, 0, 53, 0] payload_rows_per_dst=[7116, 7968, 7086, 7880, 7976, 7640, 7540, 7916] dup_rows_per_dst=[706, 18, 774, 62, 216, 1288, 208, 1142] combine_rows_per_src=[7116, 7968, 7086, 7880, 7976, 7640, 7540, 7916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60026 payload_rows_remote=51834 dup_rows=5510 dup_rows_remote=5324 dedup_ratio=0.084076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4548, 8192, 6388, 8190, 8192, 8192, 8192, 8132] dup_rows_per_dst=[0, 0, 440, 0, 116, 4768, 186, 0] combine_rows_per_src=[4548, 8192, 6388, 8190, 8192, 8192, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231456768 dispatch_meta_bytes=229408 combine_payload_bytes=231456768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231686176 ingress_bytes=250331136 link_bytes=250331136 token_bytes=463142944 weight_movement_bytes=18874368 total_nvlink_bytes=482017312 weight_to_token_ratio=0.040753 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398549775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61362 payload_rows_remote=53200 dup_rows=4174 dup_rows_remote=4174 dedup_ratio=0.063690 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 2086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 8192, 8162, 8192, 8192, 4186, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 4172, 0, 0, 0, 0] combine_rows_per_src=[8054, 8192, 8162, 8192, 8192, 4186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235094016 ingress_bytes=329236480 link_bytes=329236480 token_bytes=469958656 weight_movement_bytes=94371840 total_nvlink_bytes=564330496 weight_to_token_ratio=0.200809 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154581957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=217931776 dispatch_meta_bytes=225912 combine_payload_bytes=217931776 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218157688 ingress_bytes=274554880 link_bytes=274554880 token_bytes=436089464 weight_movement_bytes=56623104 total_nvlink_bytes=492712568 weight_to_token_ratio=0.129843 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.630649771e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=212312064 dispatch_meta_bytes=228632 combine_payload_bytes=212312064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212540696 ingress_bytes=231186432 link_bytes=231186432 token_bytes=424852760 weight_movement_bytes=18874368 total_nvlink_bytes=443727128 weight_to_token_ratio=0.044426 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215114641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62102 payload_rows_remote=53910 dup_rows=3434 dup_rows_remote=3434 dedup_ratio=0.052399 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8190, 4768, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 3434, 0, 0] combine_rows_per_src=[8184, 8192, 8190, 4768, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 2, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59404 payload_rows_remote=51212 dup_rows=6132 dup_rows_remote=3596 dedup_ratio=0.093567 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8192, 8192, 3648, 7618, 7198] dup_rows_per_dst=[2536, 10, 978, 2034, 574, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8172, 8192, 8192, 3648, 7618, 7198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=217907200 dispatch_meta_bytes=229496 combine_payload_bytes=217907200 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=218136696 ingress_bytes=255655936 link_bytes=255655936 token_bytes=436043896 weight_movement_bytes=37748736 total_nvlink_bytes=473792632 weight_to_token_ratio=0.086571 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.449569388e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63224 payload_rows_remote=55686 dup_rows=2312 dup_rows_remote=2312 dedup_ratio=0.035278 router_tokens_per_expert=[0, 0, 0, 48, 40, 0, 42, 0, 0, 0, 0, 0, 3, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3758, 0, 0, 0, 0, 0, 0, 14, 0, 13, 0, 25, 0, 0, 0, 0, 0, 835, 0, 0, 0, 4000, 3, 0, 12, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 32, 0, 0, 0, 4030, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4051, 34, 0, 0, 44, 0, 0, 4, 4051, 0, 0, 0, 0, 0, 25, 0, 0, 3392, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 10, 0, 0, 109, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7538, 8086, 8170, 8076, 8192, 6832, 8138] dup_rows_per_dst=[154, 0, 1694, 0, 58, 158, 10, 238] combine_rows_per_src=[8192, 7538, 8086, 8170, 8076, 8192, 6832, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=220815360 dispatch_meta_bytes=229376 combine_payload_bytes=220815360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=221044736 ingress_bytes=258564096 link_bytes=258564096 token_bytes=441860096 weight_movement_bytes=37748736 total_nvlink_bytes=479608832 weight_to_token_ratio=0.085431 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.477433946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=209764352 dispatch_meta_bytes=219232 combine_payload_bytes=209764352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209983584 ingress_bytes=228638720 link_bytes=228638720 token_bytes=419747936 weight_movement_bytes=18874368 total_nvlink_bytes=438622304 weight_to_token_ratio=0.044966 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.190703718e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64946 payload_rows_remote=56754 dup_rows=590 dup_rows_remote=16 dedup_ratio=0.009003 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8192, 8188, 7646, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 16, 0, 0, 0, 574, 0, 0] combine_rows_per_src=[8152, 8192, 8188, 7646, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=70 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 34, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 19, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8116] dup_rows_per_dst=[0, 0, 0, 0, 70, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56924 payload_rows_remote=48732 dup_rows=8612 dup_rows_remote=7918 dedup_ratio=0.131409 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 3965, 0, 49, 0, 0, 0, 0, 0, 76, 3981, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 76, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 469, 0, 0, 0, 0, 1, 84, 2, 0, 3943, 0, 0, 10, 0, 0, 0, 49, 0, 0, 0, 0, 0, 3732, 95, 0, 3711, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 36, 0, 22, 0, 3406, 0, 0, 0, 0, 80, 0, 23, 110, 0, 299, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 69, 0, 0, 0, 0, 0, 41, 0, 0, 75, 0] payload_rows_per_dst=[8028, 7788, 7820, 1186, 7668, 8054, 8188, 8192] dup_rows_per_dst=[86, 268, 0, 198, 168, 6920, 278, 694] combine_rows_per_src=[8028, 7788, 7820, 1186, 7668, 8054, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=231992 combine_payload_bytes=228089856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228321848 ingress_bytes=246964224 link_bytes=246964224 token_bytes=456411704 weight_movement_bytes=18874368 total_nvlink_bytes=475286072 weight_to_token_ratio=0.041354 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.366289680e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57324 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.000977 router_tokens_per_expert=[30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8148, 8192, 8172, 8192, 8192, 8192] dup_rows_per_dst=[60, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8148, 8192, 8172, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=232464384 dispatch_meta_bytes=227080 combine_payload_bytes=232464384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232691464 ingress_bytes=251338752 link_bytes=251338752 token_bytes=465155848 weight_movement_bytes=18874368 total_nvlink_bytes=484030216 weight_to_token_ratio=0.040576 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408204255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229296 combine_payload_bytes=234512384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234741680 ingress_bytes=253386752 link_bytes=253386752 token_bytes=469254064 weight_movement_bytes=18874368 total_nvlink_bytes=488128432 weight_to_token_ratio=0.040222 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427827184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=199606272 dispatch_meta_bytes=226600 combine_payload_bytes=199606272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199832872 ingress_bytes=218480640 link_bytes=218480640 token_bytes=399439144 weight_movement_bytes=18874368 total_nvlink_bytes=418313512 weight_to_token_ratio=0.047252 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.093373993e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57344 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8138, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 54, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8138, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60432 payload_rows_remote=52422 dup_rows=5104 dup_rows_remote=198 dedup_ratio=0.077881 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7746, 8192, 7980, 8010, 8192, 3928, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4906, 0, 194, 0, 4] combine_rows_per_src=[7746, 8192, 7980, 8010, 8192, 3928, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229552 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028656 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827760 weight_movement_bytes=18874368 total_nvlink_bytes=488702128 weight_to_token_ratio=0.040173 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58956 payload_rows_remote=50764 dup_rows=6580 dup_rows_remote=726 dedup_ratio=0.100403 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8008, 8192, 1980, 8008] dup_rows_per_dst=[5854, 348, 320, 58, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8008, 8192, 1980, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64700 payload_rows_remote=56600 dup_rows=836 dup_rows_remote=800 dedup_ratio=0.012756 router_tokens_per_expert=[0, 0, 0, 0, 31, 0, 0, 50, 69, 0, 0, 0, 0, 0, 5, 0, 0, 0, 36, 0, 0, 0, 0, 8, 0, 0, 12, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 132, 49, 0, 0, 0, 0, 31, 0, 0, 13, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 9, 0, 4035, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 3991, 0, 13, 4058, 29, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 42, 14, 2, 0, 0, 0, 0, 4006, 0, 0, 14, 4043, 0, 0, 0, 1, 0, 0, 1, 0] payload_rows_per_dst=[8178, 8100, 8192, 8112, 8104, 8048, 7924, 8042] dup_rows_per_dst=[248, 36, 332, 2, 66, 80, 70, 2] combine_rows_per_src=[8178, 8100, 8192, 8112, 8104, 8048, 7924, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229592 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235110616 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991640 weight_movement_bytes=75497472 total_nvlink_bytes=545489112 weight_to_token_ratio=0.160636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=214720512 dispatch_meta_bytes=210480 combine_payload_bytes=214720512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=214930992 ingress_bytes=271343616 link_bytes=271343616 token_bytes=429651504 weight_movement_bytes=56623104 total_nvlink_bytes=486274608 weight_to_token_ratio=0.131788 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599881019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=207929344 dispatch_meta_bytes=205960 combine_payload_bytes=207929344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208135304 ingress_bytes=226803712 link_bytes=226803712 token_bytes=416064648 weight_movement_bytes=18874368 total_nvlink_bytes=434939016 weight_to_token_ratio=0.045364 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.173121574e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8178, 8192, 8192, 8192] dup_rows_per_dst=[14, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8178, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61190 payload_rows_remote=52998 dup_rows=4346 dup_rows_remote=4 dedup_ratio=0.066315 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 10, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3864, 8192, 8192, 8192, 8192, 8174] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 4342, 0] combine_rows_per_src=[8192, 8192, 3864, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59506 payload_rows_remote=51674 dup_rows=6030 dup_rows_remote=5572 dedup_ratio=0.092010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 0, 58, 0, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 42, 0, 0, 3952, 0, 0, 0, 0, 3895, 3925, 0, 0, 0, 0, 0, 0, 0, 12, 26, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 141, 1, 163, 3821, 0, 146, 0, 74, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 2, 3923, 0, 135, 0, 2228, 0, 46, 0, 41, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 114, 0, 0, 0, 0, 91, 0, 65, 91, 0, 0, 0, 0] payload_rows_per_dst=[3532, 7874, 7972, 8070, 8100, 8066, 8060, 7832] dup_rows_per_dst=[0, 82, 108, 76, 500, 94, 4712, 458] combine_rows_per_src=[3532, 7874, 7972, 8070, 8100, 8066, 8060, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=231833600 dispatch_meta_bytes=229600 combine_payload_bytes=231833600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232063200 ingress_bytes=250707968 link_bytes=250707968 token_bytes=463896800 weight_movement_bytes=18874368 total_nvlink_bytes=482771168 weight_to_token_ratio=0.040687 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402160393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4086, 37, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8148] dup_rows_per_dst=[54, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036672 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843968 weight_movement_bytes=18874368 total_nvlink_bytes=488718336 weight_to_token_ratio=0.040172 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=217079808 dispatch_meta_bytes=212008 combine_payload_bytes=217079808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217291816 ingress_bytes=254828544 link_bytes=254828544 token_bytes=434371624 weight_movement_bytes=37748736 total_nvlink_bytes=472120360 weight_to_token_ratio=0.086904 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.441641725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=211656704 dispatch_meta_bytes=228984 combine_payload_bytes=211656704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211885688 ingress_bytes=230531072 link_bytes=230531072 token_bytes=423542392 weight_movement_bytes=18874368 total_nvlink_bytes=442416760 weight_to_token_ratio=0.044563 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.208835304e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229376 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889216 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549056 weight_movement_bytes=18874368 total_nvlink_bytes=488423424 weight_to_token_ratio=0.040197 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65238 payload_rows_remote=57046 dup_rows=298 dup_rows_remote=298 dedup_ratio=0.004547 router_tokens_per_expert=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 7902, 8192, 8192, 8192] dup_rows_per_dst=[20, 10, 0, 0, 0, 0, 268, 0] combine_rows_per_src=[8192, 8192, 8184, 8192, 7902, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64620 payload_rows_remote=56428 dup_rows=916 dup_rows_remote=0 dedup_ratio=0.013977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8076, 8192, 7392] dup_rows_per_dst=[916, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8076, 8192, 7392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62460 payload_rows_remote=54306 dup_rows=3076 dup_rows_remote=2966 dedup_ratio=0.046936 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 4009, 0, 17, 0, 0, 0, 0, 0, 27, 4047, 0, 0, 0, 0, 9, 49, 0, 0, 0, 0, 31, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 1106, 0, 0, 0, 0, 0, 28, 0, 0, 4036, 0, 0, 27, 0, 0, 0, 18, 0, 0, 0, 0, 29, 3966, 28, 0, 3908, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4, 0, 2898, 0, 0, 0, 0, 34, 0, 13, 41, 0, 203, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 25, 0, 0, 0, 0, 0, 17, 0, 0, 44, 0] payload_rows_per_dst=[8052, 8154, 7894, 8160, 7940, 5876, 8192, 8192] dup_rows_per_dst=[56, 110, 0, 2234, 34, 46, 300, 296] combine_rows_per_src=[8052, 8154, 7894, 8160, 7940, 5876, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=233660416 dispatch_meta_bytes=229376 combine_payload_bytes=233660416 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233889792 ingress_bytes=309157888 link_bytes=309157888 token_bytes=467550208 weight_movement_bytes=75497472 total_nvlink_bytes=543047680 weight_to_token_ratio=0.161475 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.962198768e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=231129088 dispatch_meta_bytes=225712 combine_payload_bytes=231129088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231354800 ingress_bytes=287752192 link_bytes=287752192 token_bytes=462483888 weight_movement_bytes=56623104 total_nvlink_bytes=519106992 weight_to_token_ratio=0.122433 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.757099921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8176] dup_rows_per_dst=[12, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57340 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.001190 router_tokens_per_expert=[0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8118, 8192] dup_rows_per_dst=[76, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60648 payload_rows_remote=52464 dup_rows=4888 dup_rows_remote=4184 dedup_ratio=0.074585 router_tokens_per_expert=[2182, 0, 0, 0, 0, 0, 46, 3901, 0, 0, 97, 0, 3922, 0, 107, 0, 48, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 121, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3790, 0, 1630, 3910, 0, 0, 3946, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 1, 62, 0, 0, 60, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 63, 0, 0, 43, 72, 0, 0, 3967, 0, 56, 86, 91, 0, 0, 0] payload_rows_per_dst=[4578, 8036, 7938, 7944, 7902, 8086, 7980, 8184] dup_rows_per_dst=[286, 74, 208, 2590, 442, 242, 342, 704] combine_rows_per_src=[4578, 8036, 7938, 7944, 7902, 8086, 7980, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=222437376 dispatch_meta_bytes=229088 combine_payload_bytes=222437376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222666464 ingress_bytes=279060480 link_bytes=279060480 token_bytes=445103840 weight_movement_bytes=56623104 total_nvlink_bytes=501726944 weight_to_token_ratio=0.127213 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.673820213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57344 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 10] combine_rows_per_src=[8192, 8192, 8182, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229672 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094312 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958952 weight_movement_bytes=18874368 total_nvlink_bytes=488833320 weight_to_token_ratio=0.040162 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=214892544 dispatch_meta_bytes=226592 combine_payload_bytes=214892544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215119136 ingress_bytes=252641280 link_bytes=252641280 token_bytes=430011680 weight_movement_bytes=37748736 total_nvlink_bytes=467760416 weight_to_token_ratio=0.087785 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.104167 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420684438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[8, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8174] dup_rows_per_dst=[18, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8024] dup_rows_per_dst=[0, 160, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229416 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110440 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991464 weight_movement_bytes=37748736 total_nvlink_bytes=507740200 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63730 payload_rows_remote=55702 dup_rows=1806 dup_rows_remote=1768 dedup_ratio=0.027557 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 29, 0, 0, 0, 0, 3934, 0, 21, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 4043, 0, 0, 0, 0, 4019, 4006, 0, 0, 0, 0, 0, 0, 0, 2, 11, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 25, 0, 108, 3951, 0, 62, 0, 27, 5, 0, 0, 0, 0, 0, 33, 13, 25, 0, 0, 0, 0, 5, 4031, 0, 49, 0, 658, 0, 14, 0, 0, 0, 0, 0, 21, 0, 0, 1, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 40, 0, 19, 26, 0, 0, 0, 9] payload_rows_per_dst=[6838, 8028, 8088, 8154, 8172, 8182, 8138, 8130] dup_rows_per_dst=[0, 38, 18, 26, 174, 42, 1360, 148] combine_rows_per_src=[6838, 8028, 8088, 8154, 8172, 8182, 8138, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=235036672 ingress_bytes=348053504 link_bytes=348053504 token_bytes=469843968 weight_movement_bytes=113246208 total_nvlink_bytes=583090176 weight_to_token_ratio=0.241029 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.334877422e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61400 payload_rows_remote=53208 dup_rows=4136 dup_rows_remote=4136 dedup_ratio=0.063110 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6450, 8192, 8152, 5838, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 36, 0, 0, 0, 4100, 0, 0] combine_rows_per_src=[6450, 8192, 8152, 5838, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57728 payload_rows_remote=51910 dup_rows=7808 dup_rows_remote=4434 dedup_ratio=0.119141 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 87, 0, 109, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 155, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 3824, 113, 0, 0, 0, 0, 194, 0, 119, 0, 0, 110, 0, 0, 0, 69, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 322, 3608, 0, 0, 0, 0, 0, 37, 0, 0, 997, 26, 120, 0, 60, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 3601, 2904, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[7710, 7794, 8082, 6726, 7970, 5660, 7968, 5818] dup_rows_per_dst=[350, 116, 128, 608, 890, 2192, 150, 3374] combine_rows_per_src=[7710, 7794, 8082, 6726, 7970, 5660, 7968, 5818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=228155392 dispatch_meta_bytes=229880 combine_payload_bytes=228155392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228385272 ingress_bytes=247029760 link_bytes=247029760 token_bytes=456540664 weight_movement_bytes=18874368 total_nvlink_bytes=475415032 weight_to_token_ratio=0.041342 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.366917614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65338 payload_rows_remote=57146 dup_rows=198 dup_rows_remote=198 dedup_ratio=0.003021 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7994, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 192, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7994, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65256 payload_rows_remote=57064 dup_rows=280 dup_rows_remote=280 dedup_ratio=0.004272 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7912, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 280, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7912, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235044864 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860352 weight_movement_bytes=37748736 total_nvlink_bytes=507609088 weight_to_token_ratio=0.080340 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=217939968 dispatch_meta_bytes=229376 combine_payload_bytes=217939968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218169344 ingress_bytes=236814336 link_bytes=236814336 token_bytes=436109312 weight_movement_bytes=18874368 total_nvlink_bytes=454983680 weight_to_token_ratio=0.043279 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.269038448e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=212623360 dispatch_meta_bytes=225376 combine_payload_bytes=212623360 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=212848736 ingress_bytes=306995200 link_bytes=306995200 token_bytes=425472096 weight_movement_bytes=94371840 total_nvlink_bytes=519843936 weight_to_token_ratio=0.221805 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.941476955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63422 payload_rows_remote=55230 dup_rows=2114 dup_rows_remote=576 dedup_ratio=0.032257 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7158, 8192, 7248, 8126, 8122] dup_rows_per_dst=[1538, 80, 0, 468, 28, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7158, 8192, 7248, 8126, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=4 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8188, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 2, 0, 0, 0, 2] combine_rows_per_src=[8188, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234070016 dispatch_meta_bytes=229376 combine_payload_bytes=234070016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234299392 ingress_bytes=271818752 link_bytes=271818752 token_bytes=468369408 weight_movement_bytes=37748736 total_nvlink_bytes=506118144 weight_to_token_ratio=0.080596 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604433538e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233734144 dispatch_meta_bytes=229376 combine_payload_bytes=233734144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233963520 ingress_bytes=252608512 link_bytes=252608512 token_bytes=467697664 weight_movement_bytes=18874368 total_nvlink_bytes=486572032 weight_to_token_ratio=0.040356 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420370471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=226222080 dispatch_meta_bytes=223224 combine_payload_bytes=226222080 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=226445304 ingress_bytes=339468288 link_bytes=339468288 token_bytes=452667384 weight_movement_bytes=113246208 total_nvlink_bytes=565913592 weight_to_token_ratio=0.250175 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.252618107e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62594 payload_rows_remote=54408 dup_rows=2942 dup_rows_remote=2854 dedup_ratio=0.044891 router_tokens_per_expert=[1259, 0, 0, 0, 0, 0, 18, 3979, 0, 0, 40, 0, 4002, 0, 46, 0, 33, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 48, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 3992, 0, 2789, 4013, 0, 0, 4031, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 20, 27, 0, 0, 2, 0, 0, 0, 0, 23, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 15, 43, 0, 0, 4045, 0, 43, 38, 8, 0, 0, 0] payload_rows_per_dst=[8188, 8186, 8170, 5622, 8026, 8148, 8062, 8192] dup_rows_per_dst=[2496, 88, 68, 0, 0, 34, 0, 256] combine_rows_per_src=[8188, 8186, 8170, 5622, 8026, 8148, 8062, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229368 combine_payload_bytes=234856448 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235085816 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942264 weight_movement_bytes=56623104 total_nvlink_bytes=526565368 weight_to_token_ratio=0.120489 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65276 payload_rows_remote=57344 dup_rows=260 dup_rows_remote=260 dedup_ratio=0.003967 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7932, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 260, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7932, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=66 dedup_ratio=0.001007 router_tokens_per_expert=[33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8126, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[66, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8126, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60854 payload_rows_remote=52798 dup_rows=4682 dup_rows_remote=4532 dedup_ratio=0.071442 router_tokens_per_expert=[0, 3932, 0, 0, 0, 0, 47, 3853, 0, 0, 3848, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 71, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 110, 0, 3964, 0, 0, 102, 0, 45, 0, 0, 83, 0, 1, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 0, 0, 162, 97, 0, 113, 0, 173, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 120, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 3, 0, 0, 0, 0, 0, 0, 72, 134, 0] payload_rows_per_dst=[7258, 7894, 7904, 5636, 7946, 8100, 8060, 8056] dup_rows_per_dst=[0, 264, 402, 534, 320, 2836, 176, 150] combine_rows_per_src=[7258, 7894, 7904, 5636, 7946, 8100, 8060, 8056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=222855168 dispatch_meta_bytes=229048 combine_payload_bytes=222855168 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=223084216 ingress_bytes=241729536 link_bytes=241729536 token_bytes=445939384 weight_movement_bytes=18874368 total_nvlink_bytes=464813752 weight_to_token_ratio=0.042325 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316133476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65246 payload_rows_remote=57054 dup_rows=290 dup_rows_remote=290 dedup_ratio=0.004425 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7902, 8192, 8192] dup_rows_per_dst=[290, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7902, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8182, 8192, 8192] dup_rows_per_dst=[10, 0, 0, 0, 4, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=230416 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235111440 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469992464 weight_movement_bytes=18874368 total_nvlink_bytes=488866832 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61544 payload_rows_remote=53352 dup_rows=3992 dup_rows_remote=0 dedup_ratio=0.060913 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2276, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8058, 8192, 8192, 8080, 4552, 8086, 8192] dup_rows_per_dst=[3992, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8058, 8192, 8192, 8080, 4552, 8086, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229376 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234840064 ingress_bytes=253485056 link_bytes=253485056 token_bytes=469450752 weight_movement_bytes=18874368 total_nvlink_bytes=488325120 weight_to_token_ratio=0.040205 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57302 dup_rows=44 dup_rows_remote=4 dedup_ratio=0.000671 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8150, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 40, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8150, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=216260608 dispatch_meta_bytes=229320 combine_payload_bytes=216260608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216489928 ingress_bytes=235134976 link_bytes=235134976 token_bytes=432750536 weight_movement_bytes=18874368 total_nvlink_bytes=451624904 weight_to_token_ratio=0.043615 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252947646e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233693184 dispatch_meta_bytes=229376 combine_payload_bytes=233693184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233922560 ingress_bytes=252567552 link_bytes=252567552 token_bytes=467615744 weight_movement_bytes=18874368 total_nvlink_bytes=486490112 weight_to_token_ratio=0.040363 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419978012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=218529792 dispatch_meta_bytes=213408 combine_payload_bytes=218529792 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=218743200 ingress_bytes=294027264 link_bytes=294027264 token_bytes=437272992 weight_movement_bytes=75497472 total_nvlink_bytes=512770464 weight_to_token_ratio=0.172655 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.817224573e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229224 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234938216 ingress_bytes=310206464 link_bytes=310206464 token_bytes=469647208 weight_movement_bytes=75497472 total_nvlink_bytes=545144680 weight_to_token_ratio=0.160754 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972245707e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63902 payload_rows_remote=55712 dup_rows=1634 dup_rows_remote=1514 dedup_ratio=0.024933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 47, 0, 1, 0, 35, 0, 39, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 143, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3939, 42, 0, 0, 0, 0, 412, 0, 48, 0, 0, 68, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3761, 0, 0, 0, 0, 0, 43, 0, 0, 132, 9, 21, 0, 25, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 71, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 3929, 3367, 4043, 0, 0, 0, 0, 1, 0, 0, 0, 0, 314, 0, 0] payload_rows_per_dst=[8192, 8190, 8150, 8188, 7524, 8172, 8124, 7362] dup_rows_per_dst=[150, 120, 30, 830, 0, 360, 142, 2] combine_rows_per_src=[8192, 8190, 8150, 8188, 7524, 8172, 8124, 7362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64480 payload_rows_remote=56728 dup_rows=1056 dup_rows_remote=440 dedup_ratio=0.016113 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7576, 8192, 8192, 8192, 8192, 7752, 8192, 8192] dup_rows_per_dst=[440, 0, 0, 0, 0, 616, 0, 0] combine_rows_per_src=[7576, 8192, 8192, 8192, 8192, 7752, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65210 payload_rows_remote=57018 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.004974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7950, 8192, 8192, 8192, 8192, 8182, 8192, 8118] dup_rows_per_dst=[74, 0, 0, 242, 10, 0, 0, 0] combine_rows_per_src=[7950, 8192, 8192, 8192, 8192, 8182, 8192, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=228196352 dispatch_meta_bytes=228904 combine_payload_bytes=228196352 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228425256 ingress_bytes=265945088 link_bytes=265945088 token_bytes=456621608 weight_movement_bytes=37748736 total_nvlink_bytes=494370344 weight_to_token_ratio=0.082670 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.548154980e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62244 payload_rows_remote=54134 dup_rows=3292 dup_rows_remote=3052 dedup_ratio=0.050232 router_tokens_per_expert=[0, 3903, 0, 0, 3924, 0, 0, 0, 0, 0, 3954, 8, 0, 0, 3115, 0, 0, 0, 0, 0, 127, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 71, 0, 81, 20, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 50, 0, 26, 0, 0, 0, 135, 0, 0, 0, 14, 94, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 822, 3853, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3818, 0, 0, 24, 0, 92, 0, 0, 0, 0, 0, 0, 62, 3966, 0, 40, 0, 3744, 157, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[6234, 8192, 7830, 7756, 8180, 8020, 7922, 8110] dup_rows_per_dst=[12, 78, 262, 304, 88, 1916, 392, 240] combine_rows_per_src=[6234, 8192, 7830, 7756, 8180, 8020, 7922, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64220 payload_rows_remote=56028 dup_rows=1316 dup_rows_remote=1316 dedup_ratio=0.020081 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 6876, 8192] dup_rows_per_dst=[0, 682, 0, 634, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 6876, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65058 payload_rows_remote=57006 dup_rows=478 dup_rows_remote=478 dedup_ratio=0.007294 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8052, 8192, 8190, 8092, 7956, 8192] dup_rows_per_dst=[0, 104, 0, 374, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8052, 8192, 8190, 8092, 7956, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=232357888 dispatch_meta_bytes=228672 combine_payload_bytes=232357888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232586560 ingress_bytes=270106624 link_bytes=270106624 token_bytes=464944448 weight_movement_bytes=37748736 total_nvlink_bytes=502693184 weight_to_token_ratio=0.081190 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.588028770e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64254 payload_rows_remote=56062 dup_rows=1282 dup_rows_remote=1282 dedup_ratio=0.019562 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7190, 7912] dup_rows_per_dst=[0, 538, 0, 464, 0, 280, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7190, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65108 payload_rows_remote=56918 dup_rows=428 dup_rows_remote=428 dedup_ratio=0.006531 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 7766, 8192, 8192] dup_rows_per_dst=[428, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 7766, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=233545728 dispatch_meta_bytes=229376 combine_payload_bytes=233545728 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233775104 ingress_bytes=271294464 link_bytes=271294464 token_bytes=467320832 weight_movement_bytes=37748736 total_nvlink_bytes=505069568 weight_to_token_ratio=0.080777 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599410069e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=229490688 dispatch_meta_bytes=229376 combine_payload_bytes=229490688 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=229720064 ingress_bytes=248365056 link_bytes=248365056 token_bytes=459210752 weight_movement_bytes=18874368 total_nvlink_bytes=478085120 weight_to_token_ratio=0.041102 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379711763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=221732864 dispatch_meta_bytes=228744 combine_payload_bytes=221732864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221961608 ingress_bytes=240607232 link_bytes=240607232 token_bytes=443694472 weight_movement_bytes=18874368 total_nvlink_bytes=462568840 weight_to_token_ratio=0.042539 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.305380111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=233496576 dispatch_meta_bytes=229936 combine_payload_bytes=233496576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233726512 ingress_bytes=252370944 link_bytes=252370944 token_bytes=467223088 weight_movement_bytes=18874368 total_nvlink_bytes=486097456 weight_to_token_ratio=0.040397 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418094211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=229629952 dispatch_meta_bytes=229376 combine_payload_bytes=229629952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229859328 ingress_bytes=248504320 link_bytes=248504320 token_bytes=459489280 weight_movement_bytes=18874368 total_nvlink_bytes=478363648 weight_to_token_ratio=0.041077 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381046122e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64254 payload_rows_remote=56154 dup_rows=1282 dup_rows_remote=1238 dedup_ratio=0.019562 router_tokens_per_expert=[0, 4027, 0, 0, 0, 0, 22, 3770, 0, 0, 4027, 0, 0, 2, 0, 4024, 0, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 46, 0, 4031, 0, 0, 51, 0, 299, 0, 0, 39, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 100, 33, 0, 22, 0, 70, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 54, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 49, 0] payload_rows_per_dst=[7584, 8100, 8066, 8192, 8090, 7932, 8134, 8156] dup_rows_per_dst=[4, 44, 6, 772, 0, 312, 80, 64] combine_rows_per_src=[7584, 8100, 8066, 8192, 8090, 7932, 8134, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233136128 dispatch_meta_bytes=229384 combine_payload_bytes=233136128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233365512 ingress_bytes=252010496 link_bytes=252010496 token_bytes=466501640 weight_movement_bytes=18874368 total_nvlink_bytes=485376008 weight_to_token_ratio=0.040459 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414640576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62456 payload_rows_remote=54290 dup_rows=3080 dup_rows_remote=3080 dedup_ratio=0.046997 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 4083, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5442, 8190, 8192, 8190, 8166, 7892, 8192] dup_rows_per_dst=[0, 86, 312, 264, 0, 0, 2412, 6] combine_rows_per_src=[8192, 5442, 8190, 8192, 8190, 8166, 7892, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230006784 dispatch_meta_bytes=229568 combine_payload_bytes=230006784 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230236352 ingress_bytes=267755520 link_bytes=267755520 token_bytes=460243136 weight_movement_bytes=37748736 total_nvlink_bytes=497991872 weight_to_token_ratio=0.082019 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.565501649e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[8178, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62592 payload_rows_remote=54400 dup_rows=2944 dup_rows_remote=2944 dedup_ratio=0.044922 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2629, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5258, 8192, 8192, 8192, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 1108, 0, 1836, 0, 0, 0, 0] combine_rows_per_src=[5258, 8192, 8192, 8192, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57706 payload_rows_remote=49804 dup_rows=7830 dup_rows_remote=7188 dedup_ratio=0.119476 router_tokens_per_expert=[129, 440, 0, 0, 0, 0, 0, 0, 91, 81, 0, 0, 0, 7, 154, 0, 0, 0, 48, 3947, 0, 0, 0, 3685, 116, 0, 0, 34, 0, 0, 30, 0, 92, 0, 0, 11, 0, 69, 0, 3770, 0, 46, 0, 0, 0, 68, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 5, 103, 68, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 334, 0, 0, 0, 121, 0, 0, 13, 0, 0, 0, 0, 0, 3963, 0, 0, 158, 0, 0, 3762, 0, 0, 142, 0, 0] payload_rows_per_dst=[6436, 7706, 7966, 4672, 7024, 8058, 7942, 7902] dup_rows_per_dst=[1212, 120, 146, 80, 1692, 48, 3890, 642] combine_rows_per_src=[6436, 7706, 7966, 4672, 7024, 8058, 7942, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=222371840 dispatch_meta_bytes=229480 combine_payload_bytes=222371840 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=222601320 ingress_bytes=260120576 link_bytes=260120576 token_bytes=444973160 weight_movement_bytes=37748736 total_nvlink_bytes=482721896 weight_to_token_ratio=0.084834 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.492347372e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4021, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8042, 8192, 8192, 8192, 8180, 8192, 8192, 8186] dup_rows_per_dst=[0, 168, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8042, 8192, 8192, 8192, 8180, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60180 payload_rows_remote=52070 dup_rows=5356 dup_rows_remote=86 dedup_ratio=0.081726 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3920, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8110, 5612, 7966, 5756, 8192, 8192, 8192, 8160] dup_rows_per_dst=[5270, 8, 0, 78, 0, 0, 0, 0] combine_rows_per_src=[8110, 5612, 7966, 5756, 8192, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64022 payload_rows_remote=55830 dup_rows=1514 dup_rows_remote=1394 dedup_ratio=0.023102 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8156, 8192, 8192, 8184, 6722, 8192] dup_rows_per_dst=[0, 1382, 0, 120, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8156, 8192, 8192, 8184, 6722, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=222822400 dispatch_meta_bytes=229376 combine_payload_bytes=222822400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223051776 ingress_bytes=241696768 link_bytes=241696768 token_bytes=445874176 weight_movement_bytes=18874368 total_nvlink_bytes=464748544 weight_to_token_ratio=0.042331 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.315819509e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=203997184 dispatch_meta_bytes=227968 combine_payload_bytes=203997184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=204225152 ingress_bytes=260620288 link_bytes=260620288 token_bytes=408222336 weight_movement_bytes=56623104 total_nvlink_bytes=464845440 weight_to_token_ratio=0.138707 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.497135366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59468 payload_rows_remote=51276 dup_rows=6068 dup_rows_remote=6012 dedup_ratio=0.092590 router_tokens_per_expert=[0, 4005, 0, 0, 4028, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 2858, 0, 0, 0, 0, 0, 51, 28, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 41, 0, 32, 0, 0, 0, 49, 0, 0, 0, 0, 38, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1183, 3992, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4009, 0, 0, 22, 0, 34, 0, 0, 0, 0, 0, 0, 1, 4049, 0, 22, 0, 3921, 40, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8082, 8192, 8150, 8192, 8190, 2492, 8138, 8032] dup_rows_per_dst=[5644, 56, 142, 154, 34, 0, 38, 0] combine_rows_per_src=[8082, 8192, 8150, 8192, 8190, 2492, 8138, 8032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=213278720 dispatch_meta_bytes=208624 combine_payload_bytes=213278720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=213487344 ingress_bytes=269901824 link_bytes=269901824 token_bytes=426766064 weight_movement_bytes=56623104 total_nvlink_bytes=483389168 weight_to_token_ratio=0.132679 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.586066478e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=228679680 dispatch_meta_bytes=228896 combine_payload_bytes=228679680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228908576 ingress_bytes=247554048 link_bytes=247554048 token_bytes=457588256 weight_movement_bytes=18874368 total_nvlink_bytes=476462624 weight_to_token_ratio=0.041247 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371941084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64846 payload_rows_remote=56654 dup_rows=690 dup_rows_remote=690 dedup_ratio=0.010529 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3765, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7530, 8192, 8192, 8192, 8164, 8192, 8192, 8192] dup_rows_per_dst=[0, 196, 0, 494, 0, 0, 0, 0] combine_rows_per_src=[7530, 8192, 8192, 8192, 8164, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=210026496 dispatch_meta_bytes=229152 combine_payload_bytes=210026496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210255648 ingress_bytes=228900864 link_bytes=228900864 token_bytes=420282144 weight_movement_bytes=18874368 total_nvlink_bytes=439156512 weight_to_token_ratio=0.044909 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.193215453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64460 payload_rows_remote=56268 dup_rows=1076 dup_rows_remote=1076 dedup_ratio=0.016418 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 7120, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 1046, 0, 0, 0, 26] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 7120, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63872 payload_rows_remote=55770 dup_rows=1664 dup_rows_remote=1486 dedup_ratio=0.025391 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 3962, 99, 31, 0, 3887, 0, 3902, 0, 0, 0, 0, 0, 1, 60, 121, 0, 0, 0, 0, 0, 107, 0, 0, 124, 0, 0, 0, 0, 0, 114, 21, 0, 0, 0, 0, 0, 21, 0, 0, 3, 0, 3, 118, 0, 0, 108, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 25, 0, 160, 0, 0, 63, 0, 3840, 0, 0, 1, 0, 5, 3906, 0, 114, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 11, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 86, 6, 0] payload_rows_per_dst=[8122, 7642, 8102, 7898, 8030, 8048, 7928, 8102] dup_rows_per_dst=[62, 194, 232, 244, 236, 144, 374, 178] combine_rows_per_src=[8122, 7642, 8102, 7898, 8030, 8048, 7928, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=232054784 dispatch_meta_bytes=229376 combine_payload_bytes=232054784 prefetch_bytes=0 reduce_bytes=0 egress_bytes=232284160 ingress_bytes=232054784 link_bytes=232284160 token_bytes=464338944 weight_movement_bytes=0 total_nvlink_bytes=464338944 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225632530e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 4000, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8180, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64394 payload_rows_remote=56202 dup_rows=1142 dup_rows_remote=1142 dedup_ratio=0.017426 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7548, 8192, 8030, 8192, 8192, 7856] dup_rows_per_dst=[0, 644, 0, 162, 0, 336, 0, 0] combine_rows_per_src=[8192, 8192, 7548, 8192, 8030, 8192, 8192, 7856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64218 payload_rows_remote=56026 dup_rows=1318 dup_rows_remote=1232 dedup_ratio=0.020111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3459, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6918, 8192, 8192, 8192, 8178, 8192, 8192, 8162] dup_rows_per_dst=[0, 1232, 0, 86, 0, 0, 0, 0] combine_rows_per_src=[6918, 8192, 8192, 8192, 8178, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230473728 dispatch_meta_bytes=229376 combine_payload_bytes=230473728 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=230703104 ingress_bytes=249348096 link_bytes=249348096 token_bytes=461176832 weight_movement_bytes=18874368 total_nvlink_bytes=480051200 weight_to_token_ratio=0.040927 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.389130769e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=228433920 dispatch_meta_bytes=229024 combine_payload_bytes=228433920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228662944 ingress_bytes=247308288 link_bytes=247308288 token_bytes=457096864 weight_movement_bytes=18874368 total_nvlink_bytes=475971232 weight_to_token_ratio=0.041292 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.369586332e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235061248 ingress_bytes=291454976 link_bytes=291454976 token_bytes=469893120 weight_movement_bytes=56623104 total_nvlink_bytes=526516224 weight_to_token_ratio=0.120502 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792578175e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=230203392 dispatch_meta_bytes=229376 combine_payload_bytes=230203392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230432768 ingress_bytes=249077760 link_bytes=249077760 token_bytes=460636160 weight_movement_bytes=18874368 total_nvlink_bytes=479510528 weight_to_token_ratio=0.040975 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.386540542e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56636 payload_rows_remote=48488 dup_rows=8900 dup_rows_remote=1214 dedup_ratio=0.135803 router_tokens_per_expert=[36, 208, 0, 0, 0, 0, 0, 0, 30, 29, 0, 0, 0, 23, 65, 0, 0, 0, 18, 4039, 0, 0, 0, 3811, 40, 0, 0, 0, 1, 0, 8, 0, 25, 0, 0, 0, 0, 43, 0, 3981, 0, 19, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 14, 52, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 3813, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 113, 0, 0, 0, 26, 0, 0, 2, 0, 0, 0, 0, 0, 4043, 0, 0, 22, 0, 0, 3936, 0, 12, 55, 0, 0] payload_rows_per_dst=[8160, 8148, 8110, 8116, 7672, 8156, 230, 8044] dup_rows_per_dst=[576, 7686, 80, 452, 0, 48, 52, 6] combine_rows_per_src=[8160, 8148, 8110, 8116, 7672, 8156, 230, 8044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=229482496 dispatch_meta_bytes=229032 combine_payload_bytes=229482496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229711528 ingress_bytes=248356864 link_bytes=248356864 token_bytes=459194024 weight_movement_bytes=18874368 total_nvlink_bytes=478068392 weight_to_token_ratio=0.041103 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379633272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3993, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=198606848 dispatch_meta_bytes=198808 combine_payload_bytes=198606848 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=198805656 ingress_bytes=255229952 link_bytes=255229952 token_bytes=397412504 weight_movement_bytes=56623104 total_nvlink_bytes=454035608 weight_to_token_ratio=0.142479 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.445487819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57390 payload_rows_remote=49198 dup_rows=8146 dup_rows_remote=8146 dedup_ratio=0.124298 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[60, 8192, 8192, 8178, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 8146, 0, 0] combine_rows_per_src=[60, 8192, 8192, 8178, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62330 payload_rows_remote=54486 dup_rows=3206 dup_rows_remote=2480 dedup_ratio=0.048920 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 30, 20, 0, 1, 0, 85, 0, 0, 67, 48, 0, 0, 0, 0, 0, 91, 0, 0, 59, 3674, 111, 3960, 3916, 236, 0, 0, 0, 0, 7, 0, 0, 117, 0, 0, 0, 135, 5, 42, 123, 0, 0, 0, 3931, 85, 0, 17, 0, 0, 0, 0, 0, 0, 141, 6, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 100, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 38, 0, 0, 0, 0, 3799, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 3836, 0, 0, 3897, 0, 0, 0, 15, 3, 0, 0, 0, 0] payload_rows_per_dst=[8008, 7874, 7396, 7512, 8074, 7954, 7668, 7844] dup_rows_per_dst=[190, 228, 742, 450, 226, 0, 644, 726] combine_rows_per_src=[8008, 7874, 7396, 7512, 8074, 7954, 7668, 7844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65128 payload_rows_remote=56936 dup_rows=408 dup_rows_remote=408 dedup_ratio=0.006226 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8096, 8192, 8192, 8192, 7880] dup_rows_per_dst=[0, 96, 290, 22, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8096, 8192, 8192, 8192, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=201515008 dispatch_meta_bytes=229376 combine_payload_bytes=201515008 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=201744384 ingress_bytes=258138112 link_bytes=258138112 token_bytes=403259392 weight_movement_bytes=56623104 total_nvlink_bytes=459882496 weight_to_token_ratio=0.140414 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.104167 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.473352377e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=223174656 dispatch_meta_bytes=227864 combine_payload_bytes=223174656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=223402520 ingress_bytes=279797760 link_bytes=279797760 token_bytes=446577176 weight_movement_bytes=56623104 total_nvlink_bytes=503200280 weight_to_token_ratio=0.126794 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.680884467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233209856 dispatch_meta_bytes=229376 combine_payload_bytes=233209856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233439232 ingress_bytes=252084224 link_bytes=252084224 token_bytes=466649088 weight_movement_bytes=18874368 total_nvlink_bytes=485523456 weight_to_token_ratio=0.040447 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415347001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65016 payload_rows_remote=56888 dup_rows=520 dup_rows_remote=430 dedup_ratio=0.007935 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 4035, 43, 7, 0, 4010, 3, 4013, 0, 0, 0, 0, 0, 0, 42, 34, 0, 0, 0, 0, 0, 17, 0, 0, 41, 0, 0, 0, 0, 0, 50, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 53, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 1, 0, 63, 0, 0, 32, 0, 3980, 0, 0, 0, 0, 4, 4022, 0, 49, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 3, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 42, 9, 0] payload_rows_per_dst=[8156, 8128, 8170, 8070, 8112, 8148, 8082, 8150] dup_rows_per_dst=[14, 90, 106, 106, 128, 44, 0, 32] combine_rows_per_src=[8156, 8128, 8170, 8070, 8112, 8148, 8082, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233013248 dispatch_meta_bytes=229272 combine_payload_bytes=233013248 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=233242520 ingress_bytes=308510720 link_bytes=308510720 token_bytes=466255768 weight_movement_bytes=75497472 total_nvlink_bytes=541753240 weight_to_token_ratio=0.161923 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.955997922e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1675, 0, 0, 0, 0, 0, 0, 2433, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 24] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 0, 0, 0, 0, 0, 1972, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58094 payload_rows_remote=49902 dup_rows=7442 dup_rows_remote=1260 dedup_ratio=0.113556 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 0, 4060, 0, 0, 26, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 874, 8140, 8192, 8192, 8120, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1208, 0, 0, 6182, 52] combine_rows_per_src=[8192, 874, 8140, 8192, 8192, 8120, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60552 payload_rows_remote=52522 dup_rows=4984 dup_rows_remote=4584 dedup_ratio=0.076050 router_tokens_per_expert=[0, 0, 94, 0, 1, 2648, 0, 150, 0, 0, 0, 0, 0, 0, 0, 1, 129, 63, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 26, 0, 124, 0, 0, 0, 0, 0, 3, 0, 74, 0, 0, 0, 0, 3918, 23, 0, 0, 0, 0, 0, 41, 16, 0, 0, 0, 3927, 0, 1265, 0, 0, 116, 0, 0, 89, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 0, 0, 0, 36, 0, 133, 102, 0, 0, 29, 0, 0, 0, 3960, 0, 3838, 0, 131, 81, 69, 0, 0, 0, 1] payload_rows_per_dst=[5624, 7538, 7884, 7750, 8058, 7818, 7850, 8030] dup_rows_per_dst=[592, 410, 152, 2452, 94, 530, 354, 400] combine_rows_per_src=[5624, 7538, 7884, 7750, 8058, 7818, 7850, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59924 payload_rows_remote=51732 dup_rows=5612 dup_rows_remote=5182 dedup_ratio=0.085632 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 0, 0, 0, 0, 0, 0, 1980, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8086, 7762, 7036, 4272] dup_rows_per_dst=[430, 0, 106, 1470, 0, 0, 3606, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8086, 7762, 7036, 4272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235012096 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794816 weight_movement_bytes=37748736 total_nvlink_bytes=507543552 weight_to_token_ratio=0.080352 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=204398592 dispatch_meta_bytes=204648 combine_payload_bytes=204398592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204603240 ingress_bytes=242147328 link_bytes=242147328 token_bytes=409001832 weight_movement_bytes=37748736 total_nvlink_bytes=446750568 weight_to_token_ratio=0.092295 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.320136553e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=215130112 dispatch_meta_bytes=228424 combine_payload_bytes=215130112 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=215358536 ingress_bytes=271753216 link_bytes=271753216 token_bytes=430488648 weight_movement_bytes=56623104 total_nvlink_bytes=487111752 weight_to_token_ratio=0.131532 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603805605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=211894272 dispatch_meta_bytes=227656 combine_payload_bytes=211894272 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=212121928 ingress_bytes=268517376 link_bytes=268517376 token_bytes=424016200 weight_movement_bytes=56623104 total_nvlink_bytes=480639304 weight_to_token_ratio=0.133540 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.572801378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64414 payload_rows_remote=56862 dup_rows=1122 dup_rows_remote=1054 dedup_ratio=0.017120 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 12, 0, 0, 0, 24, 0, 0, 21, 11, 0, 0, 0, 0, 0, 58, 0, 0, 3, 3676, 41, 4046, 4031, 346, 0, 0, 0, 0, 0, 4, 0, 19, 0, 0, 0, 46, 0, 0, 32, 0, 0, 0, 4026, 3, 0, 0, 0, 0, 0, 0, 0, 0, 51, 18, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 59, 38, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 4, 0, 0, 0, 0, 3999, 0, 1, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 3985, 0, 0, 4021, 0, 0, 0, 35, 13, 0, 0, 0, 0] payload_rows_per_dst=[8108, 7552, 8184, 8162, 8170, 8088, 8070, 8080] dup_rows_per_dst=[94, 68, 708, 98, 76, 6, 2, 70] combine_rows_per_src=[8108, 7552, 8184, 8162, 8170, 8088, 8070, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110408 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991432 weight_movement_bytes=75497472 total_nvlink_bytes=545488904 weight_to_token_ratio=0.160636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 2834, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 24] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232906752 dispatch_meta_bytes=231664 combine_payload_bytes=232906752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233138416 ingress_bytes=270655488 link_bytes=270655488 token_bytes=466045168 weight_movement_bytes=37748736 total_nvlink_bytes=503793904 weight_to_token_ratio=0.080998 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.593287715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65330 payload_rows_remote=57138 dup_rows=206 dup_rows_remote=206 dedup_ratio=0.003143 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4027, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 8192, 8192, 8192, 8124, 8192, 8192, 8192] dup_rows_per_dst=[0, 198, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8054, 8192, 8192, 8192, 8124, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60316 payload_rows_remote=52438 dup_rows=5220 dup_rows_remote=1534 dedup_ratio=0.079651 router_tokens_per_expert=[0, 0, 0, 0, 3805, 3873, 0, 51, 164, 3856, 0, 2, 28, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 112, 0, 16, 58, 17, 56, 82, 0, 0, 52, 53, 0, 35, 0, 66, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 119, 0, 62, 0, 0, 0, 0, 46, 3933, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 0, 0, 2966, 92, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 0, 0, 0, 0, 3328, 0] payload_rows_per_dst=[7642, 8008, 7934, 6302, 8074, 8192, 6286, 7878] dup_rows_per_dst=[148, 310, 84, 546, 132, 312, 2, 3686] combine_rows_per_src=[7642, 8008, 7934, 6302, 8074, 8192, 6286, 7878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63190 payload_rows_remote=54998 dup_rows=2346 dup_rows_remote=272 dedup_ratio=0.035797 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6118, 7920, 8192] dup_rows_per_dst=[2074, 0, 272, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6118, 7920, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 0, 0, 0, 0, 2822, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 14] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234037248 dispatch_meta_bytes=229376 combine_payload_bytes=234037248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234266624 ingress_bytes=252911616 link_bytes=252911616 token_bytes=468303872 weight_movement_bytes=18874368 total_nvlink_bytes=487178240 weight_to_token_ratio=0.040304 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423274664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=214786048 dispatch_meta_bytes=215888 combine_payload_bytes=214786048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215001936 ingress_bytes=252534784 link_bytes=252534784 token_bytes=429787984 weight_movement_bytes=37748736 total_nvlink_bytes=467536720 weight_to_token_ratio=0.087831 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419664045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=225271808 dispatch_meta_bytes=221080 combine_payload_bytes=225271808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225492888 ingress_bytes=244146176 link_bytes=244146176 token_bytes=450764696 weight_movement_bytes=18874368 total_nvlink_bytes=469639064 weight_to_token_ratio=0.041872 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.339288531e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63044 payload_rows_remote=54890 dup_rows=2492 dup_rows_remote=2306 dedup_ratio=0.038025 router_tokens_per_expert=[1, 0, 32, 0, 0, 966, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 59, 21, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 9, 0, 55, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 27, 8, 0, 0, 0, 4027, 0, 3050, 0, 0, 44, 0, 0, 26, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 1, 0, 49, 53, 0, 0, 22, 0, 0, 0, 4040, 0, 3991, 0, 55, 23, 41, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8154, 8050, 6244, 8184, 8036, 8082, 8124] dup_rows_per_dst=[1998, 186, 8, 14, 48, 0, 98, 140] combine_rows_per_src=[8170, 8154, 8050, 6244, 8184, 8036, 8082, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860352 weight_movement_bytes=37748736 total_nvlink_bytes=507609088 weight_to_token_ratio=0.080340 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.100694 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 8, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8184, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3598, 0, 0, 0, 0, 0, 493, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=224829440 dispatch_meta_bytes=228784 combine_payload_bytes=224829440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225058224 ingress_bytes=262578176 link_bytes=262578176 token_bytes=449887664 weight_movement_bytes=37748736 total_nvlink_bytes=487636400 weight_to_token_ratio=0.083907 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.515894886e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=0 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3080, 0, 0, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8124] dup_rows_per_dst=[0, 0, 68, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235077632 ingress_bytes=291471360 link_bytes=291471360 token_bytes=469925888 weight_movement_bytes=56623104 total_nvlink_bytes=526548992 weight_to_token_ratio=0.120494 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792735159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 3934, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59748 payload_rows_remote=51714 dup_rows=5788 dup_rows_remote=5620 dedup_ratio=0.088318 router_tokens_per_expert=[0, 0, 0, 44, 0, 3338, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 38, 0, 0, 0, 0, 74, 0, 0, 100, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 85, 0, 0, 0, 3911, 0, 0, 0, 115, 0, 0, 294, 0, 0, 0, 3860, 0, 26, 3901, 139, 0, 0, 0, 0, 136, 0, 0, 0, 0, 1, 69, 0, 41, 15, 0, 77, 0, 49, 0, 0, 0, 0, 0, 84, 0, 3947, 0, 0, 140, 0, 0, 0, 3824, 22, 0, 0, 0, 0, 0, 0, 43, 0, 3775, 0, 0, 0, 70, 3947, 0, 0, 64, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 42, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6830, 6642, 6982, 7852, 7602, 7882, 7924, 8034] dup_rows_per_dst=[0, 776, 886, 312, 406, 2474, 766, 168] combine_rows_per_src=[6830, 6642, 6982, 7852, 7602, 7882, 7924, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63776 payload_rows_remote=55584 dup_rows=1760 dup_rows_remote=1760 dedup_ratio=0.026855 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7854, 8192, 8192, 8192, 7248, 7714] dup_rows_per_dst=[0, 340, 0, 0, 1420, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7854, 8192, 8192, 8192, 7248, 7714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229104 combine_payload_bytes=234602496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234831600 ingress_bytes=253476864 link_bytes=253476864 token_bytes=469434096 weight_movement_bytes=18874368 total_nvlink_bytes=488308464 weight_to_token_ratio=0.040207 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428690592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=211820544 dispatch_meta_bytes=229336 combine_payload_bytes=211820544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=212049880 ingress_bytes=249569280 link_bytes=249569280 token_bytes=423870424 weight_movement_bytes=37748736 total_nvlink_bytes=461619160 weight_to_token_ratio=0.089057 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.391250045e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=227672064 dispatch_meta_bytes=229376 combine_payload_bytes=227672064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227901440 ingress_bytes=246546432 link_bytes=246546432 token_bytes=455573504 weight_movement_bytes=18874368 total_nvlink_bytes=474447872 weight_to_token_ratio=0.041430 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362286603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62748 payload_rows_remote=54684 dup_rows=2788 dup_rows_remote=2788 dedup_ratio=0.042542 router_tokens_per_expert=[0, 0, 0, 0, 3959, 3999, 0, 28, 71, 3986, 0, 0, 24, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 57, 0, 7, 0, 13, 43, 28, 0, 0, 38, 30, 0, 36, 0, 25, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 60, 4024, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 0, 3820, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 0, 2912, 0] payload_rows_per_dst=[8082, 8064, 8180, 7780, 8192, 8192, 6856, 7402] dup_rows_per_dst=[82, 0, 80, 286, 92, 118, 880, 1250] combine_rows_per_src=[8082, 8064, 8180, 7780, 8192, 8192, 6856, 7402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235053056 ingress_bytes=291446784 link_bytes=291446784 token_bytes=469876736 weight_movement_bytes=56623104 total_nvlink_bytes=526499840 weight_to_token_ratio=0.120506 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792499684e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4028, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=223985664 dispatch_meta_bytes=229888 combine_payload_bytes=223985664 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224215552 ingress_bytes=261734400 link_bytes=261734400 token_bytes=448201216 weight_movement_bytes=37748736 total_nvlink_bytes=485949952 weight_to_token_ratio=0.084223 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.507810239e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62216 payload_rows_remote=54300 dup_rows=3320 dup_rows_remote=2646 dedup_ratio=0.050659 router_tokens_per_expert=[0, 0, 49, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 122, 0, 109, 0, 0, 0, 0, 0, 0, 123, 0, 0, 3906, 0, 0, 0, 3853, 13, 0, 0, 0, 89, 112, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 3810, 0, 50, 0, 72, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 76, 3883, 0, 0, 0, 0, 0, 0, 1, 0, 0, 30, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 113, 0, 0, 0, 0, 0, 162, 0, 132, 0, 0, 0, 0, 3885, 24, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0] payload_rows_per_dst=[7916, 7740, 8138, 7946, 6870, 7500, 8190, 7916] dup_rows_per_dst=[196, 148, 268, 298, 0, 1050, 686, 674] combine_rows_per_src=[7916, 7740, 8138, 7946, 6870, 7500, 8190, 7916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61546 payload_rows_remote=53354 dup_rows=3990 dup_rows_remote=3688 dedup_ratio=0.060883 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8056, 8108, 8138, 8180, 4488, 8192] dup_rows_per_dst=[302, 3622, 0, 66, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8056, 8108, 8138, 8180, 4488, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3628, 0, 0, 0, 0, 0, 466, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=222412800 dispatch_meta_bytes=227784 combine_payload_bytes=222412800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222640584 ingress_bytes=279035904 link_bytes=279035904 token_bytes=445053384 weight_movement_bytes=56623104 total_nvlink_bytes=501676488 weight_to_token_ratio=0.127228 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.673584738e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=218537984 dispatch_meta_bytes=228168 combine_payload_bytes=218537984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218766152 ingress_bytes=275161088 link_bytes=275161088 token_bytes=437304136 weight_movement_bytes=56623104 total_nvlink_bytes=493927240 weight_to_token_ratio=0.129482 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.636458158e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64228 payload_rows_remote=56074 dup_rows=1308 dup_rows_remote=518 dedup_ratio=0.019958 router_tokens_per_expert=[0, 0, 0, 30, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 61, 0, 0, 0, 0, 37, 0, 0, 48, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4022, 0, 0, 0, 1, 0, 0, 97, 0, 0, 0, 4000, 0, 0, 4023, 56, 0, 0, 0, 0, 54, 0, 0, 0, 0, 21, 20, 0, 19, 7, 0, 40, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4029, 0, 0, 37, 0, 0, 0, 3993, 20, 0, 1, 0, 0, 0, 0, 40, 0, 3870, 0, 0, 0, 0, 4035, 0, 0, 50, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 19, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7492, 8154, 8080, 8112, 8192, 8088, 7966, 8144] dup_rows_per_dst=[10, 790, 164, 108, 184, 14, 0, 38] combine_rows_per_src=[7492, 8154, 8080, 8112, 8192, 8088, 7966, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64674 payload_rows_remote=56482 dup_rows=862 dup_rows_remote=0 dedup_ratio=0.013153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7386, 8192, 8192, 8192, 8136, 8192] dup_rows_per_dst=[0, 0, 0, 0, 862, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7386, 8192, 8192, 8192, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=229679104 dispatch_meta_bytes=226368 combine_payload_bytes=229679104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229905472 ingress_bytes=248553472 link_bytes=248553472 token_bytes=459584576 weight_movement_bytes=18874368 total_nvlink_bytes=478458944 weight_to_token_ratio=0.041068 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381517073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64330 payload_rows_remote=57234 dup_rows=1206 dup_rows_remote=1206 dedup_ratio=0.018402 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7096, 8192, 8192, 8192, 8082, 8192] dup_rows_per_dst=[0, 28, 0, 0, 1136, 0, 0, 42] combine_rows_per_src=[8192, 8192, 7096, 8192, 8192, 8192, 8082, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=231350272 dispatch_meta_bytes=225928 combine_payload_bytes=231350272 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231576200 ingress_bytes=306847744 link_bytes=306847744 token_bytes=462926472 weight_movement_bytes=75497472 total_nvlink_bytes=538423944 weight_to_token_ratio=0.163087 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.940064104e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57344 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 0, 0, 0, 0, 0, 1223, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 24] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60046 payload_rows_remote=52522 dup_rows=5490 dup_rows_remote=4682 dedup_ratio=0.083771 router_tokens_per_expert=[22, 0, 0, 89, 0, 0, 3700, 0, 0, 465, 52, 0, 0, 3884, 0, 0, 44, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 115, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 41, 3897, 0, 299, 0, 3215, 78, 66, 3619, 0, 3938, 138, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 28, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 815, 0, 139, 41, 0, 0, 0, 0, 0, 232, 0, 0, 0, 1, 3403, 0, 0, 0, 0, 169, 0] payload_rows_per_dst=[6920, 5806, 8106, 7248, 8192, 8116, 8134, 7524] dup_rows_per_dst=[1014, 0, 134, 904, 52, 110, 2468, 808] combine_rows_per_src=[6920, 5806, 8106, 7248, 8192, 8116, 8134, 7524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59340 payload_rows_remote=51148 dup_rows=6196 dup_rows_remote=6180 dedup_ratio=0.094543 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8162, 8192, 3170, 8192, 7064, 8192] dup_rows_per_dst=[16, 0, 5022, 1128, 30, 0, 0, 0] combine_rows_per_src=[8192, 8176, 8162, 8192, 3170, 8192, 7064, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234430464 dispatch_meta_bytes=233760 combine_payload_bytes=234430464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234664224 ingress_bytes=253304832 link_bytes=253304832 token_bytes=469094688 weight_movement_bytes=18874368 total_nvlink_bytes=487969056 weight_to_token_ratio=0.040236 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.097222 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427042266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229472 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110496 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991520 weight_movement_bytes=56623104 total_nvlink_bytes=526614624 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=215130112 dispatch_meta_bytes=228816 combine_payload_bytes=215130112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215358928 ingress_bytes=234004480 link_bytes=234004480 token_bytes=430489040 weight_movement_bytes=18874368 total_nvlink_bytes=449363408 weight_to_token_ratio=0.043844 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.242115790e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=209502208 dispatch_meta_bytes=229312 combine_payload_bytes=209502208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209731520 ingress_bytes=247250944 link_bytes=247250944 token_bytes=419233728 weight_movement_bytes=37748736 total_nvlink_bytes=456982464 weight_to_token_ratio=0.090042 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.369036890e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64750 payload_rows_remote=56640 dup_rows=786 dup_rows_remote=740 dedup_ratio=0.011993 router_tokens_per_expert=[0, 0, 25, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 57, 0, 35, 0, 0, 0, 0, 0, 0, 47, 1, 0, 4018, 0, 0, 0, 3988, 7, 0, 0, 0, 21, 52, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 3959, 0, 16, 0, 55, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 55, 0, 0, 0, 0, 0, 64, 0, 56, 0, 0, 0, 0, 4008, 13, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0] payload_rows_per_dst=[8178, 8110, 8172, 8092, 8094, 7832, 8188, 8084] dup_rows_per_dst=[82, 46, 74, 48, 0, 0, 278, 258] combine_rows_per_src=[8178, 8110, 8172, 8092, 8094, 7832, 8188, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235102208 ingress_bytes=329244672 link_bytes=329244672 token_bytes=469975040 weight_movement_bytes=94371840 total_nvlink_bytes=564346880 weight_to_token_ratio=0.200802 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.097222 gemm_active_groups=11 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154660449e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63944 payload_rows_remote=55752 dup_rows=1592 dup_rows_remote=1586 dedup_ratio=0.024292 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6620, 8192, 8192, 8192, 8172, 8192] dup_rows_per_dst=[0, 2, 0, 0, 1584, 6, 0, 0] combine_rows_per_src=[8192, 8192, 6620, 8192, 8192, 8192, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62860 payload_rows_remote=54668 dup_rows=2676 dup_rows_remote=2676 dedup_ratio=0.040833 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5518, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2676, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 5518, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=231997440 dispatch_meta_bytes=229520 combine_payload_bytes=231997440 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232226960 ingress_bytes=288620544 link_bytes=288620544 token_bytes=464224400 weight_movement_bytes=56623104 total_nvlink_bytes=520847504 weight_to_token_ratio=0.121974 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.765420042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65296 payload_rows_remote=57104 dup_rows=240 dup_rows_remote=240 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7952, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[240, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 7952, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=223920128 dispatch_meta_bytes=229376 combine_payload_bytes=223920128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224149504 ingress_bytes=242794496 link_bytes=242794496 token_bytes=448069632 weight_movement_bytes=18874368 total_nvlink_bytes=466944000 weight_to_token_ratio=0.042124 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.326337398e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=228360192 dispatch_meta_bytes=229352 combine_payload_bytes=228360192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228589544 ingress_bytes=247234560 link_bytes=247234560 token_bytes=456949736 weight_movement_bytes=18874368 total_nvlink_bytes=475824104 weight_to_token_ratio=0.041305 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368879907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55874 payload_rows_remote=49878 dup_rows=9662 dup_rows_remote=6752 dedup_ratio=0.147430 router_tokens_per_expert=[92, 0, 158, 0, 0, 0, 0, 0, 3865, 0, 0, 1, 105, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3778, 364, 94, 0, 0, 0, 30, 0, 0, 0, 0, 115, 82, 2956, 1, 0, 134, 174, 0, 3, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 88, 0, 0, 0, 0, 3107, 3890, 0, 125, 0, 33, 0, 7, 0, 0, 0, 0, 0, 0, 50, 0, 0, 109, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3451, 2906, 102, 0, 0, 0, 0, 0, 2, 0, 0, 2895, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7618, 7614, 6322, 7200, 6584, 7800, 6740, 5996] dup_rows_per_dst=[420, 540, 668, 74, 594, 510, 3946, 2910] combine_rows_per_src=[7618, 7614, 6322, 7200, 6584, 7800, 6740, 5996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61960 payload_rows_remote=53768 dup_rows=3576 dup_rows_remote=1018 dedup_ratio=0.054565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7220, 8192, 8192, 8048, 5732, 8192, 8192] dup_rows_per_dst=[2558, 0, 302, 716, 0, 0, 0, 0] combine_rows_per_src=[8192, 7220, 8192, 8192, 8048, 5732, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8192, 8192, 8132, 8192] dup_rows_per_dst=[0, 8, 0, 0, 58, 2, 0, 2] combine_rows_per_src=[8192, 8192, 8182, 8192, 8192, 8192, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233897984 dispatch_meta_bytes=229376 combine_payload_bytes=233897984 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234127360 ingress_bytes=271646720 link_bytes=271646720 token_bytes=468025344 weight_movement_bytes=37748736 total_nvlink_bytes=505774080 weight_to_token_ratio=0.080655 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602785213e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.100694 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=204300288 dispatch_meta_bytes=226520 combine_payload_bytes=204300288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204526808 ingress_bytes=223174656 link_bytes=223174656 token_bytes=408827096 weight_movement_bytes=18874368 total_nvlink_bytes=427701464 weight_to_token_ratio=0.046167 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.138349745e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63564 payload_rows_remote=56608 dup_rows=1972 dup_rows_remote=1972 dedup_ratio=0.030090 router_tokens_per_expert=[4, 0, 0, 38, 0, 0, 3976, 0, 0, 29, 15, 0, 0, 4000, 0, 0, 4, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 53, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 136, 0, 0, 59, 4023, 0, 77, 0, 3778, 7, 0, 3794, 0, 4035, 69, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 537, 0, 42, 3, 0, 0, 0, 0, 0, 125, 0, 0, 0, 1, 3827, 0, 0, 0, 0, 73, 0] payload_rows_per_dst=[8036, 6956, 8182, 7896, 8192, 8162, 8144, 7996] dup_rows_per_dst=[88, 0, 98, 356, 26, 8, 1340, 56] combine_rows_per_src=[8036, 6956, 8182, 7896, 8192, 8162, 8144, 7996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=220233728 dispatch_meta_bytes=219144 combine_payload_bytes=220233728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220452872 ingress_bytes=257982464 link_bytes=257982464 token_bytes=440686600 weight_movement_bytes=37748736 total_nvlink_bytes=478435336 weight_to_token_ratio=0.085659 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.471861035e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229376 combine_payload_bytes=234594304 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234823680 ingress_bytes=272343040 link_bytes=272343040 token_bytes=469417984 weight_movement_bytes=37748736 total_nvlink_bytes=507166720 weight_to_token_ratio=0.080416 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609457008e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8132, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[60, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8132, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=231866368 dispatch_meta_bytes=234320 combine_payload_bytes=231866368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232100688 ingress_bytes=250740736 link_bytes=250740736 token_bytes=463967056 weight_movement_bytes=18874368 total_nvlink_bytes=482841424 weight_to_token_ratio=0.040680 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402474360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229376 combine_payload_bytes=234635264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234864640 ingress_bytes=310132736 link_bytes=310132736 token_bytes=469499904 weight_movement_bytes=75497472 total_nvlink_bytes=544997376 weight_to_token_ratio=0.160804 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.971539282e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57344 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3663, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 8192] dup_rows_per_dst=[10, 0, 2, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58110 payload_rows_remote=50146 dup_rows=7426 dup_rows_remote=7060 dedup_ratio=0.113312 router_tokens_per_expert=[0, 94, 0, 0, 49, 0, 0, 3837, 0, 57, 0, 0, 0, 1875, 0, 0, 0, 95, 0, 0, 30, 0, 0, 0, 3952, 3875, 0, 117, 0, 0, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 77, 3753, 0, 0, 23, 0, 3872, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 123, 0, 0, 0, 0, 181, 48, 3, 0, 3146, 0, 3878, 0, 0, 1145, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 115, 0, 0, 170, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[4004, 8192, 7944, 7660, 8152, 8068, 6126, 7964] dup_rows_per_dst=[146, 2176, 0, 46, 136, 1390, 3166, 366] combine_rows_per_src=[4004, 8192, 7944, 7660, 8152, 8068, 6126, 7964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62788 payload_rows_remote=54596 dup_rows=2748 dup_rows_remote=1416 dedup_ratio=0.041931 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8134, 8192, 6728, 6966] dup_rows_per_dst=[1332, 1226, 0, 58, 132, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8134, 8192, 6728, 6966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61956 payload_rows_remote=53764 dup_rows=3580 dup_rows_remote=3580 dedup_ratio=0.054626 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4612, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[3570, 0, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 4612, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229432 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110456 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991480 weight_movement_bytes=18874368 total_nvlink_bytes=488865848 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=205398016 dispatch_meta_bytes=228824 combine_payload_bytes=205398016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205626840 ingress_bytes=243146752 link_bytes=243146752 token_bytes=411024856 weight_movement_bytes=37748736 total_nvlink_bytes=448773592 weight_to_token_ratio=0.091841 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329712542e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62530 payload_rows_remote=54458 dup_rows=3006 dup_rows_remote=2882 dedup_ratio=0.045868 router_tokens_per_expert=[31, 0, 58, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 41, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 114, 0, 0, 0, 0, 2, 0, 0, 0, 0, 46, 40, 3747, 19, 0, 71, 69, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 38, 0, 0, 0, 0, 1128, 4009, 0, 49, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 27, 0, 0, 77, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 3537, 60, 0, 0, 0, 0, 0, 0, 0, 29, 3625, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8072, 7640, 7964, 8158, 8062, 7074, 7370] dup_rows_per_dst=[140, 124, 348, 18, 2314, 4, 0, 58] combine_rows_per_src=[8190, 8072, 7640, 7964, 8158, 8062, 7074, 7370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=223625216 dispatch_meta_bytes=224048 combine_payload_bytes=223625216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223849264 ingress_bytes=242499584 link_bytes=242499584 token_bytes=447474480 weight_movement_bytes=18874368 total_nvlink_bytes=466348848 weight_to_token_ratio=0.042180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.323511697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57312 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8160] dup_rows_per_dst=[0, 0, 0, 0, 0, 32, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=220217344 dispatch_meta_bytes=229376 combine_payload_bytes=220217344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220446720 ingress_bytes=257966080 link_bytes=257966080 token_bytes=440664064 weight_movement_bytes=37748736 total_nvlink_bytes=478412800 weight_to_token_ratio=0.085663 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.471704051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229376 combine_payload_bytes=234749952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234979328 ingress_bytes=272498688 link_bytes=272498688 token_bytes=469729280 weight_movement_bytes=37748736 total_nvlink_bytes=507478016 weight_to_token_ratio=0.080363 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610948351e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=223059968 dispatch_meta_bytes=229360 combine_payload_bytes=223059968 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=223289328 ingress_bytes=260808704 link_bytes=260808704 token_bytes=446349296 weight_movement_bytes=37748736 total_nvlink_bytes=484098032 weight_to_token_ratio=0.084572 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.498940676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8170, 8192, 8192, 8192] dup_rows_per_dst=[22, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61610 payload_rows_remote=53418 dup_rows=3926 dup_rows_remote=3152 dedup_ratio=0.059906 router_tokens_per_expert=[127, 0, 0, 3368, 0, 0, 0, 19, 105, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 3742, 0, 153, 6, 1, 0, 3476, 0, 0, 79, 0, 0, 0, 213, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 14, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 162, 168, 3888, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 3923, 0, 50, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 170, 0, 0] payload_rows_per_dst=[7126, 7106, 7540, 7906, 7692, 7890, 8158, 8192] dup_rows_per_dst=[274, 2, 832, 228, 324, 420, 1072, 774] combine_rows_per_src=[7126, 7106, 7540, 7906, 7692, 7890, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64566 payload_rows_remote=56374 dup_rows=970 dup_rows_remote=970 dedup_ratio=0.014801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8140, 7276, 8192, 8190] dup_rows_per_dst=[0, 52, 916, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8140, 7276, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=291487744 link_bytes=291487744 token_bytes=469958656 weight_movement_bytes=56623104 total_nvlink_bytes=526581760 weight_to_token_ratio=0.120485 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792892142e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235020288 ingress_bytes=253665280 link_bytes=253665280 token_bytes=469811200 weight_movement_bytes=18874368 total_nvlink_bytes=488685568 weight_to_token_ratio=0.040174 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430495902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=218800128 dispatch_meta_bytes=226280 combine_payload_bytes=218800128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219026408 ingress_bytes=237674496 link_bytes=237674496 token_bytes=437826536 weight_movement_bytes=18874368 total_nvlink_bytes=456700904 weight_to_token_ratio=0.043109 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277280078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63760 payload_rows_remote=55570 dup_rows=1776 dup_rows_remote=1636 dedup_ratio=0.027100 router_tokens_per_expert=[0, 45, 0, 0, 21, 0, 0, 4010, 0, 21, 0, 0, 0, 85, 0, 0, 0, 42, 0, 22, 11, 0, 0, 0, 4033, 4004, 0, 57, 0, 0, 0, 3409, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3953, 0, 0, 1, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 67, 50, 0, 0, 0, 0, 89, 41, 0, 0, 3832, 0, 4006, 0, 0, 562, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 46, 0, 0, 76, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8190, 8102, 7960, 8176, 8144, 6994, 8058] dup_rows_per_dst=[228, 140, 0, 2, 64, 1242, 100, 0] combine_rows_per_src=[8136, 8190, 8102, 7960, 8176, 8144, 6994, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=230907904 dispatch_meta_bytes=229376 combine_payload_bytes=230907904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231137280 ingress_bytes=249782272 link_bytes=249782272 token_bytes=462045184 weight_movement_bytes=18874368 total_nvlink_bytes=480919552 weight_to_token_ratio=0.040850 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393290830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=227614720 dispatch_meta_bytes=228824 combine_payload_bytes=227614720 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=227843544 ingress_bytes=321986560 link_bytes=321986560 token_bytes=455458264 weight_movement_bytes=94371840 total_nvlink_bytes=549830104 weight_to_token_ratio=0.207202 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.085116791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8166, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8166, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235085800 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942248 weight_movement_bytes=56623104 total_nvlink_bytes=526565352 weight_to_token_ratio=0.120489 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65348 payload_rows_remote=57200 dup_rows=188 dup_rows_remote=146 dedup_ratio=0.002869 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8090, 8192, 8150, 8192, 8148, 8192] dup_rows_per_dst=[0, 12, 0, 0, 128, 0, 42, 6] combine_rows_per_src=[8192, 8192, 8090, 8192, 8150, 8192, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57890 payload_rows_remote=49698 dup_rows=7646 dup_rows_remote=4912 dedup_ratio=0.116669 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 119, 177, 0, 66, 0, 0, 0, 3752, 0, 104, 726, 0, 20, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 91, 0, 49, 0, 3844, 0, 34, 0, 0, 24, 0, 0, 0, 0, 3714, 0, 59, 0, 0, 0, 0, 0, 0, 0, 126, 0, 3199, 0, 0, 59, 0, 46, 0, 0, 0, 126, 0, 0, 0, 0, 154, 0, 0, 52, 0, 0, 0, 3843, 0, 90, 0, 0, 0, 0, 35, 0, 3889, 0, 0, 0, 33, 2848, 0, 33, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3874, 0, 7, 163, 0, 0, 0, 79, 1000, 0, 124, 0, 0, 0, 0] payload_rows_per_dst=[7560, 6062, 7314, 6922, 7860, 6066, 7914, 8192] dup_rows_per_dst=[540, 1396, 98, 1964, 356, 146, 412, 2734] combine_rows_per_src=[7560, 6062, 7314, 6922, 7860, 6066, 7914, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59338 payload_rows_remote=51146 dup_rows=6198 dup_rows_remote=5496 dedup_ratio=0.094574 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7798, 8192, 2388] dup_rows_per_dst=[702, 228, 0, 0, 168, 0, 5100, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7798, 8192, 2388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8170] dup_rows_per_dst=[0, 0, 0, 0, 0, 24, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229384 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234520584 ingress_bytes=290914304 link_bytes=290914304 token_bytes=468811784 weight_movement_bytes=56623104 total_nvlink_bytes=525434888 weight_to_token_ratio=0.120780 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787397722e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=203563008 dispatch_meta_bytes=218440 combine_payload_bytes=203563008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203781448 ingress_bytes=241311744 link_bytes=241311744 token_bytes=407344456 weight_movement_bytes=37748736 total_nvlink_bytes=445093192 weight_to_token_ratio=0.092670 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312130398e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=209494016 dispatch_meta_bytes=226568 combine_payload_bytes=209494016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=209720584 ingress_bytes=247242752 link_bytes=247242752 token_bytes=419214600 weight_movement_bytes=37748736 total_nvlink_bytes=456963336 weight_to_token_ratio=0.090046 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368958399e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64174 payload_rows_remote=56124 dup_rows=1362 dup_rows_remote=1362 dedup_ratio=0.020782 router_tokens_per_expert=[39, 0, 0, 3708, 0, 0, 0, 0, 47, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 53, 0, 0, 0, 3838, 0, 0, 22, 0, 0, 0, 66, 0, 50, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 66, 71, 4010, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4025, 0, 15, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 76, 0, 0] payload_rows_per_dst=[7572, 8050, 7920, 8080, 8162, 8074, 8124, 8192] dup_rows_per_dst=[80, 0, 92, 112, 132, 0, 494, 452] combine_rows_per_src=[7572, 8050, 7920, 8080, 8162, 8074, 8124, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=229883904 dispatch_meta_bytes=229944 combine_payload_bytes=229883904 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=230113848 ingress_bytes=286507008 link_bytes=286507008 token_bytes=459997752 weight_movement_bytes=56623104 total_nvlink_bytes=516620856 weight_to_token_ratio=0.123094 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.745169180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 16, 0, 0] combine_rows_per_src=[8192, 8176, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59872 payload_rows_remote=51680 dup_rows=5664 dup_rows_remote=5664 dedup_ratio=0.086426 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 4094, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2756, 7968, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[202, 5436, 26, 0, 0, 0, 0, 0] combine_rows_per_src=[2756, 7968, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21796172952 prefetch_bytes=905969664 reduce_bytes=887095296 weight_movement_bytes=1793064960 egress_bytes=10903598232 ingress_bytes=12685639680 link_bytes=12685639680 total_nvlink_bytes=23589237912 weight_to_token_ratio=0.082265 prefetch_share_of_ingress=0.071417 dedup_ratio=0.033876 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=47 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095920 gemm_inner_pad_upper_bound=46355 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.532240785e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59370 payload_rows_remote=51178 dup_rows=6166 dup_rows_remote=1666 dedup_ratio=0.094086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 158, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 3819, 18, 0, 0, 0, 0, 80, 0, 193, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 33, 0, 70, 0, 0, 0, 0, 270, 0, 0, 3871, 0, 0, 0, 0, 86, 0, 31, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 3893, 0, 0, 0, 0, 3903, 0, 3912, 0, 0, 0, 0, 0, 143, 0, 0, 0, 169, 0, 3725, 0, 0, 2105, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 98, 0, 0, 3603, 0, 67, 0, 0, 185, 0, 0, 0] payload_rows_per_dst=[7766, 7402, 8050, 7408, 7904, 8144, 4504, 8192] dup_rows_per_dst=[102, 422, 124, 534, 160, 324, 0, 4500] combine_rows_per_src=[7766, 7402, 8050, 7408, 7904, 8144, 4504, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60648 payload_rows_remote=52456 dup_rows=4888 dup_rows_remote=4888 dedup_ratio=0.074585 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7812, 4806, 7070] dup_rows_per_dst=[0, 80, 4508, 0, 300, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7812, 4806, 7070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=211681280 dispatch_meta_bytes=229376 combine_payload_bytes=211681280 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=211910656 ingress_bytes=249430016 link_bytes=249430016 token_bytes=423591936 weight_movement_bytes=37748736 total_nvlink_bytes=461340672 weight_to_token_ratio=0.089116 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.389915686e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=209625088 dispatch_meta_bytes=211376 combine_payload_bytes=209625088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209836464 ingress_bytes=247373824 link_bytes=247373824 token_bytes=419461552 weight_movement_bytes=37748736 total_nvlink_bytes=457210288 weight_to_token_ratio=0.089993 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.370214266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21807877472 prefetch_bytes=905969664 reduce_bytes=811597824 weight_movement_bytes=1717567488 egress_bytes=10909420896 ingress_bytes=12616024064 link_bytes=12616024064 total_nvlink_bytes=23525444960 weight_to_token_ratio=0.078759 prefetch_share_of_ingress=0.071811 dedup_ratio=0.033190 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=43 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.094690 gemm_inner_pad_upper_bound=49276 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.518344481e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=214859776 dispatch_meta_bytes=229376 combine_payload_bytes=214859776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215089152 ingress_bytes=233734144 link_bytes=233734144 token_bytes=429948928 weight_movement_bytes=18874368 total_nvlink_bytes=448823296 weight_to_token_ratio=0.043899 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239525563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61318 payload_rows_remote=53218 dup_rows=4218 dup_rows_remote=3444 dedup_ratio=0.064362 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 66, 14, 0, 54, 0, 0, 0, 3978, 0, 44, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 43, 0, 10, 0, 3984, 0, 14, 0, 0, 8, 0, 17, 0, 0, 3936, 0, 31, 0, 0, 0, 0, 0, 0, 0, 51, 0, 3617, 0, 0, 0, 0, 50, 0, 0, 0, 50, 0, 0, 5, 0, 67, 0, 0, 0, 0, 0, 0, 3988, 0, 29, 0, 0, 0, 0, 44, 0, 4009, 0, 0, 0, 24, 1437, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4007, 0, 39, 65, 0, 0, 0, 16, 2571, 0, 38, 0, 0, 0, 0] payload_rows_per_dst=[8102, 8100, 8024, 7448, 8112, 8148, 8058, 5326] dup_rows_per_dst=[182, 774, 32, 50, 108, 2938, 0, 134] combine_rows_per_src=[8102, 8100, 8024, 7448, 8112, 8148, 8058, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20920466016 prefetch_bytes=924844032 reduce_bytes=641728512 weight_movement_bytes=1566572544 egress_bytes=10465630816 ingress_bytes=12021407744 link_bytes=12021407744 total_nvlink_bytes=22487038560 weight_to_token_ratio=0.074882 prefetch_share_of_ingress=0.076933 dedup_ratio=0.064597 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=34 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095631 gemm_inner_pad_upper_bound=48260 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.399650294e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=217980928 dispatch_meta_bytes=226648 combine_payload_bytes=217980928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218207576 ingress_bytes=255729664 link_bytes=255729664 token_bytes=436188504 weight_movement_bytes=37748736 total_nvlink_bytes=473937240 weight_to_token_ratio=0.086542 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450275814e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085800 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942248 weight_movement_bytes=18874368 total_nvlink_bytes=488816616 weight_to_token_ratio=0.040163 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57112 payload_rows_remote=48920 dup_rows=8424 dup_rows_remote=7564 dedup_ratio=0.128540 router_tokens_per_expert=[0, 112, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 1469, 0, 0, 54, 0, 3895, 0, 95, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 3941, 0, 0, 0, 3886, 0, 3857, 167, 0, 0, 0, 0, 0, 0, 0, 0, 98, 16, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 69, 155, 0, 34, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 53, 0, 8, 146, 0, 0, 0, 3903, 19, 0, 184, 26, 93] payload_rows_per_dst=[4000, 7970, 5196, 8092, 7960, 8134, 7568, 8192] dup_rows_per_dst=[2820, 142, 0, 68, 0, 328, 4206, 860] combine_rows_per_src=[4000, 7970, 5196, 8092, 7960, 8134, 7568, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21805720152 prefetch_bytes=754974720 reduce_bytes=943718400 weight_movement_bytes=1698693120 egress_bytes=10908377688 ingress_bytes=12596035584 link_bytes=12596264960 total_nvlink_bytes=23504413272 weight_to_token_ratio=0.077901 prefetch_share_of_ingress=0.059937 dedup_ratio=0.034852 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=50 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095631 gemm_inner_pad_upper_bound=47498 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.514400273e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=200376320 dispatch_meta_bytes=225936 combine_payload_bytes=200376320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200602256 ingress_bytes=238125056 link_bytes=238125056 token_bytes=400978576 weight_movement_bytes=37748736 total_nvlink_bytes=438727312 weight_to_token_ratio=0.094142 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.281597122e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63344 payload_rows_remote=55180 dup_rows=2192 dup_rows_remote=2024 dedup_ratio=0.033447 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 62, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 73, 0, 0, 3984, 0, 0, 0, 0, 0, 28, 0, 79, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 171, 0, 0, 4004, 0, 0, 0, 0, 29, 0, 17, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4017, 0, 0, 0, 0, 4020, 0, 4021, 0, 0, 0, 0, 0, 53, 0, 0, 0, 56, 0, 3816, 0, 0, 3309, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 34, 670, 0, 0, 0, 0, 0, 3929, 0, 40, 0, 0, 75, 0, 0, 0] payload_rows_per_dst=[7756, 8164, 8140, 8106, 8080, 8152, 6780, 8166] dup_rows_per_dst=[62, 168, 70, 336, 120, 106, 0, 1330] combine_rows_per_src=[7756, 8164, 8140, 8106, 8080, 8152, 6780, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19964715264 prefetch_bytes=943718400 reduce_bytes=887095296 weight_movement_bytes=1830813696 egress_bytes=9987694848 ingress_bytes=11807834112 link_bytes=11807834112 total_nvlink_bytes=21795528960 weight_to_token_ratio=0.091702 prefetch_share_of_ingress=0.079923 dedup_ratio=0.106279 prefetch_slots=100 unused_prefetch_slots=1436 slot_utilization=0.065104 reduce_slots=47 remote_demand=50 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.096861 gemm_inner_pad_upper_bound=46736 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.357017847e-04 prefetch_bytes_per_gemm_flop=1.883801119e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21813795232 prefetch_bytes=924844032 reduce_bytes=1038090240 weight_movement_bytes=1962934272 egress_bytes=10912389536 ingress_bytes=12864339968 link_bytes=12864339968 total_nvlink_bytes=23776729504 weight_to_token_ratio=0.089986 prefetch_share_of_ingress=0.071892 dedup_ratio=0.032009 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=55 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095992 gemm_inner_pad_upper_bound=48260 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.567911998e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=226017280 dispatch_meta_bytes=228816 combine_payload_bytes=226017280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226246096 ingress_bytes=263766016 link_bytes=263766016 token_bytes=452263376 weight_movement_bytes=37748736 total_nvlink_bytes=490012112 weight_to_token_ratio=0.083466 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.527276184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65460 payload_rows_remote=57268 dup_rows=76 dup_rows_remote=74 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8122, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 2, 70] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8122, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234569728 dispatch_meta_bytes=229368 combine_payload_bytes=234569728 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234799096 ingress_bytes=272318464 link_bytes=272318464 token_bytes=469368824 weight_movement_bytes=37748736 total_nvlink_bytes=507117560 weight_to_token_ratio=0.080424 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609221533e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63640 payload_rows_remote=55532 dup_rows=1896 dup_rows_remote=1790 dedup_ratio=0.028931 router_tokens_per_expert=[0, 31, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 734, 0, 0, 27, 0, 4002, 0, 44, 3, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4031, 0, 0, 0, 4008, 0, 3992, 70, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 75, 0, 0, 0, 6, 0, 11, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 10, 0, 0, 65, 0, 0, 0, 4012, 16, 0, 85, 0, 40] payload_rows_per_dst=[8174, 8108, 6680, 8192, 8124, 8174, 7996, 8192] dup_rows_per_dst=[1390, 106, 0, 32, 0, 104, 0, 264] combine_rows_per_src=[8174, 8108, 6680, 8192, 8124, 8174, 7996, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=227459072 dispatch_meta_bytes=229288 combine_payload_bytes=227459072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227688360 ingress_bytes=246333440 link_bytes=246333440 token_bytes=455147432 weight_movement_bytes=18874368 total_nvlink_bytes=474021800 weight_to_token_ratio=0.041469 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.360245818e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8170, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21320679304 prefetch_bytes=754974720 reduce_bytes=887095296 weight_movement_bytes=1642070016 egress_bytes=10665811848 ingress_bytes=12296937472 link_bytes=12297166848 total_nvlink_bytes=22962749320 weight_to_token_ratio=0.077018 prefetch_share_of_ingress=0.061395 dedup_ratio=0.050999 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=47 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095269 gemm_inner_pad_upper_bound=47625 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.454695878e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235003904 ingress_bytes=291397632 link_bytes=291397632 token_bytes=469778432 weight_movement_bytes=56623104 total_nvlink_bytes=526401536 weight_to_token_ratio=0.120532 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792028733e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21432562944 prefetch_bytes=830472192 reduce_bytes=849346560 weight_movement_bytes=1679818752 egress_bytes=10721744128 ingress_bytes=12390637568 link_bytes=12390637568 total_nvlink_bytes=23112381696 weight_to_token_ratio=0.078377 prefetch_share_of_ingress=0.067024 dedup_ratio=0.045635 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=45 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.097005 gemm_inner_pad_upper_bound=44704 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.473354013e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none + [2026-08-29 23:37:43] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 784.2 | throughput per GPU (TFLOP/s/GPU): 113.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604645E+00 | loss scale: 1.0 | grad norm: 8.728 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45444 payload_rows_remote=39588 dup_rows=20092 dup_rows_remote=17492 dedup_ratio=0.306580 router_tokens_per_expert=[894, 2, 0, 7, 1452, 686, 500, 3, 4, 14, 217, 368, 111, 141, 377, 156, 7, 4, 13, 160, 357, 126, 59, 0, 241, 730, 83, 48, 40, 19, 60, 151, 551, 240, 5, 77, 21, 56, 709, 16, 72, 157, 85, 1426, 418, 4, 64, 560, 85, 142, 39, 83, 107, 842, 14, 31, 445, 44, 84, 377, 46, 425, 12, 25, 811, 225, 8, 4, 160, 0, 219, 860, 17, 197, 507, 207, 2, 429, 37, 1738, 747, 17, 49, 252, 58, 195, 24, 2, 457, 144, 62, 762, 30, 375, 126, 12, 5, 93, 918, 48, 8, 909, 1427, 60, 557, 38, 252, 45, 255, 543, 194, 5, 665, 782, 182, 27, 493, 37, 649, 110, 70, 975, 192, 37, 35, 13, 119, 0] payload_rows_per_dst=[5430, 5832, 5408, 5856, 5282, 6310, 5674, 5652] dup_rows_per_dst=[2730, 1840, 2314, 2600, 2084, 3218, 2186, 3120] combine_rows_per_src=[5430, 5832, 5408, 5856, 5282, 6310, 5674, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45454 payload_rows_remote=39512 dup_rows=20082 dup_rows_remote=17122 dedup_ratio=0.306427 router_tokens_per_expert=[617, 0, 0, 4, 1325, 554, 488, 11, 0, 2, 180, 308, 147, 202, 326, 168, 8, 35, 5, 146, 352, 139, 7, 0, 267, 574, 46, 73, 31, 9, 95, 140, 509, 234, 7, 83, 40, 38, 581, 23, 67, 152, 176, 1617, 710, 4, 88, 441, 81, 117, 19, 17, 105, 1153, 11, 32, 415, 20, 66, 373, 46, 564, 20, 52, 685, 203, 81, 7, 92, 1, 164, 1028, 10, 166, 570, 228, 9, 318, 48, 2112, 629, 41, 40, 365, 32, 179, 37, 1, 387, 177, 68, 609, 48, 352, 155, 6, 1, 65, 815, 26, 8, 1031, 1537, 53, 564, 26, 229, 75, 378, 533, 182, 2, 798, 669, 190, 40, 340, 14, 511, 83, 113, 1022, 219, 25, 76, 3, 172, 0] payload_rows_per_dst=[4432, 6042, 6004, 6172, 5440, 5942, 5718, 5704] dup_rows_per_dst=[1582, 2036, 3536, 3084, 1780, 2960, 2258, 2846] combine_rows_per_src=[4432, 6042, 6004, 6172, 5440, 5942, 5718, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=38488 dup_rows=20990 dup_rows_remote=18438 dedup_ratio=0.320282 router_tokens_per_expert=[663, 0, 1, 3, 1218, 489, 624, 12, 11, 1, 211, 321, 150, 209, 252, 264, 22, 25, 22, 178, 379, 169, 39, 3, 266, 496, 74, 133, 22, 34, 267, 155, 509, 226, 7, 143, 100, 48, 597, 26, 105, 143, 160, 1115, 872, 5, 126, 491, 60, 122, 75, 41, 106, 772, 6, 27, 549, 41, 115, 334, 99, 467, 41, 58, 742, 255, 60, 14, 130, 1, 155, 939, 20, 251, 613, 272, 19, 607, 43, 1911, 574, 59, 65, 286, 39, 134, 24, 5, 473, 151, 70, 698, 50, 277, 133, 9, 10, 113, 963, 38, 5, 1090, 1333, 74, 580, 39, 214, 97, 373, 430, 260, 8, 590, 681, 220, 17, 403, 11, 451, 90, 75, 732, 206, 24, 51, 3, 205, 4] payload_rows_per_dst=[6394, 6058, 5380, 5920, 6452, 4304, 5298, 4740] dup_rows_per_dst=[4694, 2552, 1736, 2572, 3750, 1790, 2492, 1404] combine_rows_per_src=[6394, 6058, 5380, 5920, 6452, 4304, 5298, 4740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44622 payload_rows_remote=38040 dup_rows=20914 dup_rows_remote=15834 dedup_ratio=0.319122 router_tokens_per_expert=[693, 0, 4, 2, 1400, 635, 517, 7, 7, 7, 189, 374, 97, 169, 276, 265, 20, 20, 9, 246, 324, 217, 53, 5, 279, 538, 34, 88, 15, 18, 166, 131, 470, 316, 6, 101, 16, 36, 585, 23, 52, 111, 127, 1189, 516, 4, 88, 571, 77, 116, 46, 61, 153, 735, 17, 59, 364, 42, 129, 334, 52, 491, 26, 55, 709, 231, 67, 10, 139, 0, 177, 906, 11, 174, 577, 210, 21, 713, 33, 1607, 549, 30, 64, 439, 45, 209, 22, 3, 386, 206, 40, 556, 37, 441, 125, 19, 1, 82, 770, 24, 4, 975, 1511, 65, 666, 52, 206, 46, 361, 552, 288, 6, 661, 741, 210, 21, 403, 17, 797, 100, 100, 1082, 188, 34, 52, 7, 213, 4] payload_rows_per_dst=[6582, 6296, 5432, 6084, 5658, 4466, 5082, 5022] dup_rows_per_dst=[5080, 3056, 1874, 2452, 2436, 1876, 2066, 2074] combine_rows_per_src=[6582, 6296, 5432, 6084, 5658, 4466, 5082, 5022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45600 payload_rows_remote=39826 dup_rows=19936 dup_rows_remote=17834 dedup_ratio=0.304199 router_tokens_per_expert=[710, 2, 0, 1, 1384, 517, 439, 4, 10, 11, 190, 311, 118, 223, 395, 171, 10, 22, 9, 223, 337, 163, 56, 3, 273, 604, 40, 60, 6, 5, 123, 127, 446, 287, 7, 85, 20, 40, 716, 24, 118, 128, 143, 1405, 659, 6, 86, 640, 69, 153, 28, 31, 137, 692, 19, 26, 503, 32, 104, 376, 33, 611, 26, 33, 781, 196, 55, 8, 112, 1, 201, 939, 3, 162, 595, 216, 10, 620, 39, 1656, 649, 33, 55, 378, 43, 194, 16, 3, 412, 134, 16, 649, 54, 353, 121, 3, 1, 126, 806, 30, 10, 1006, 1298, 76, 611, 33, 212, 50, 302, 438, 296, 1, 697, 822, 211, 27, 387, 13, 726, 91, 51, 1043, 190, 42, 75, 5, 151, 4] payload_rows_per_dst=[4466, 5742, 6138, 5772, 5774, 6120, 5674, 5914] dup_rows_per_dst=[1738, 1692, 3482, 2570, 2102, 2874, 2322, 3156] combine_rows_per_src=[4466, 5742, 6138, 5772, 5774, 6120, 5674, 5914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45116 payload_rows_remote=39358 dup_rows=20420 dup_rows_remote=17580 dedup_ratio=0.311584 router_tokens_per_expert=[770, 0, 0, 3, 1288, 586, 479, 5, 3, 3, 160, 330, 98, 128, 306, 131, 13, 21, 6, 197, 376, 241, 71, 0, 256, 530, 53, 84, 14, 16, 115, 126, 552, 283, 6, 105, 14, 43, 690, 52, 63, 174, 204, 1252, 707, 6, 78, 469, 67, 142, 33, 65, 132, 996, 29, 38, 623, 18, 69, 348, 45, 612, 30, 36, 613, 228, 22, 9, 97, 2, 188, 883, 3, 207, 645, 279, 5, 659, 44, 1715, 600, 26, 52, 437, 31, 264, 35, 1, 398, 139, 19, 564, 21, 366, 121, 11, 0, 114, 740, 39, 10, 1004, 1492, 65, 664, 48, 179, 34, 359, 393, 251, 3, 817, 540, 250, 25, 438, 4, 564, 82, 116, 1034, 195, 48, 60, 4, 121, 1] payload_rows_per_dst=[4530, 5640, 6058, 4946, 5752, 5826, 6606, 5758] dup_rows_per_dst=[1474, 2028, 3338, 1620, 2016, 2920, 4184, 2840] combine_rows_per_src=[4530, 5640, 6058, 4946, 5752, 5826, 6606, 5758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45344 payload_rows_remote=40624 dup_rows=20192 dup_rows_remote=18772 dedup_ratio=0.308105 router_tokens_per_expert=[763, 1, 2, 3, 1050, 604, 524, 11, 6, 10, 220, 317, 158, 209, 182, 259, 13, 26, 6, 173, 466, 197, 44, 4, 245, 611, 69, 67, 15, 28, 204, 160, 454, 272, 6, 102, 21, 27, 507, 29, 90, 132, 148, 1332, 660, 11, 129, 482, 67, 151, 55, 36, 95, 799, 19, 19, 400, 33, 140, 498, 54, 565, 12, 24, 819, 270, 36, 6, 142, 0, 211, 1019, 8, 203, 501, 245, 17, 304, 37, 2005, 665, 28, 44, 288, 23, 179, 30, 3, 440, 171, 62, 587, 47, 274, 119, 16, 0, 82, 871, 37, 7, 1239, 1363, 73, 603, 52, 214, 62, 371, 515, 239, 4, 796, 524, 247, 24, 585, 25, 514, 94, 111, 888, 178, 57, 46, 5, 125, 2] payload_rows_per_dst=[5986, 6072, 4720, 5986, 6248, 5476, 5122, 5734] dup_rows_per_dst=[3404, 2594, 1420, 2674, 3124, 2388, 1880, 2708] combine_rows_per_src=[5986, 6072, 4720, 5986, 6248, 5476, 5122, 5734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162152448 dispatch_meta_bytes=228320 combine_payload_bytes=162152448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=162380768 ingress_bytes=181026816 link_bytes=181026816 token_bytes=324533216 weight_movement_bytes=18874368 total_nvlink_bytes=343407584 weight_to_token_ratio=0.058159 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=9344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.734509880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161841152 dispatch_meta_bytes=226536 combine_payload_bytes=161841152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=162067688 ingress_bytes=180715520 link_bytes=180715520 token_bytes=323908840 weight_movement_bytes=18874368 total_nvlink_bytes=342783208 weight_to_token_ratio=0.058271 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=8064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.731527195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157646848 dispatch_meta_bytes=227704 combine_payload_bytes=157646848 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=157874552 ingress_bytes=195395584 link_bytes=195395584 token_bytes=315521400 weight_movement_bytes=37748736 total_nvlink_bytes=353270136 weight_to_token_ratio=0.119639 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.083333 gemm_active_groups=19 gemm_peak_group_rows=14720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872184345e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155811840 dispatch_meta_bytes=215496 combine_payload_bytes=155811840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156027336 ingress_bytes=193560576 link_bytes=193560576 token_bytes=311839176 weight_movement_bytes=37748736 total_nvlink_bytes=349587912 weight_to_token_ratio=0.121052 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=5888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854602201e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163127296 dispatch_meta_bytes=230640 combine_payload_bytes=163127296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=163357936 ingress_bytes=219750400 link_bytes=219750400 token_bytes=326485232 weight_movement_bytes=56623104 total_nvlink_bytes=383108336 weight_to_token_ratio=0.173432 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.105540209e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161210368 dispatch_meta_bytes=227752 combine_payload_bytes=161210368 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=161438120 ingress_bytes=198959104 link_bytes=198959104 token_bytes=322648488 weight_movement_bytes=37748736 total_nvlink_bytes=360397224 weight_to_token_ratio=0.116996 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=6144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.906328241e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166395904 dispatch_meta_bytes=237584 combine_payload_bytes=166395904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=166633488 ingress_bytes=204144640 link_bytes=204144640 token_bytes=333029392 weight_movement_bytes=37748736 total_nvlink_bytes=370778128 weight_to_token_ratio=0.113350 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.956013495e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54400 payload_rows_remote=46392 dup_rows=11136 dup_rows_remote=10732 dedup_ratio=0.169922 router_tokens_per_expert=[0, 2, 2125, 3, 1, 0, 0, 2, 0, 0, 0, 0, 5, 1, 0, 1, 83, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 904, 0, 0, 3843, 0, 1, 0, 0, 281, 0, 0, 8, 0, 273, 0, 0, 1528, 0, 3742, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 2, 0, 187, 0, 0, 200, 0, 1931, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 0, 0, 3997, 300, 6, 3439, 3180, 3224, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 420, 0, 1, 0, 0, 3, 736, 4, 0, 0, 0, 0, 0, 0, 0, 76, 2205, 0, 0, 0] payload_rows_per_dst=[6738, 8042, 5414, 8008, 7402, 7006, 6642, 5148] dup_rows_per_dst=[2490, 1638, 1304, 404, 3242, 498, 660, 900] combine_rows_per_src=[6738, 8042, 5414, 8008, 7402, 7006, 6642, 5148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55416 payload_rows_remote=48964 dup_rows=10120 dup_rows_remote=9478 dedup_ratio=0.154419 router_tokens_per_expert=[0, 5, 2493, 0, 11, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 4, 52, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1187, 0, 4, 3400, 4, 0, 0, 0, 270, 0, 0, 43, 0, 255, 0, 0, 1191, 1, 3813, 0, 0, 16, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 1, 75, 2, 1051, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 1, 5, 3, 0, 3720, 212, 0, 3110, 3625, 3623, 1, 3, 0, 1, 0, 0, 0, 0, 7, 2, 0, 0, 244, 0, 5, 0, 0, 1, 1051, 0, 0, 2, 0, 0, 0, 0, 0, 106, 3115, 0, 0, 0] payload_rows_per_dst=[5018, 6708, 7982, 7486, 7228, 6452, 7314, 7228] dup_rows_per_dst=[22, 1398, 3164, 30, 1874, 642, 458, 2532] combine_rows_per_src=[5018, 6708, 7982, 7486, 7228, 6452, 7314, 7228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49656 payload_rows_remote=41950 dup_rows=15880 dup_rows_remote=12242 dedup_ratio=0.242310 router_tokens_per_expert=[0, 0, 2143, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1297, 0, 0, 3636, 0, 0, 0, 0, 108, 0, 0, 29, 0, 305, 0, 0, 1879, 0, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1, 101, 0, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3945, 185, 7, 2977, 3358, 3968, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2305, 0, 0, 0] payload_rows_per_dst=[7706, 7998, 4410, 8004, 7338, 3110, 5942, 5148] dup_rows_per_dst=[3638, 5112, 620, 2138, 2248, 112, 1372, 640] combine_rows_per_src=[7706, 7998, 4410, 8004, 7338, 3110, 5942, 5148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52964 payload_rows_remote=45758 dup_rows=12572 dup_rows_remote=10692 dedup_ratio=0.191833 router_tokens_per_expert=[0, 0, 2628, 4, 0, 0, 0, 13, 0, 5, 3, 0, 0, 15, 0, 0, 146, 6, 3, 0, 0, 17, 0, 11, 0, 1, 0, 1202, 1, 0, 3155, 1, 16, 1, 0, 179, 1, 0, 73, 0, 504, 0, 0, 1945, 0, 3765, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 1, 681, 0, 927, 1, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 2, 0, 0, 0, 4014, 231, 0, 2622, 3559, 3903, 0, 8, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 688, 0, 10, 0, 0, 9, 626, 3, 8, 0, 0, 6, 0, 0, 0, 13, 1638, 3, 0, 1] payload_rows_per_dst=[7970, 7206, 4776, 8058, 7590, 5450, 7854, 4060] dup_rows_per_dst=[4896, 1880, 662, 188, 2754, 270, 1368, 554] combine_rows_per_src=[7970, 7206, 4776, 8058, 7590, 5450, 7854, 4060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54048 payload_rows_remote=46436 dup_rows=11488 dup_rows_remote=8368 dedup_ratio=0.175293 router_tokens_per_expert=[0, 0, 2017, 3, 0, 0, 0, 82, 2, 0, 0, 0, 1, 0, 0, 0, 25, 0, 0, 0, 1, 6, 0, 3, 0, 0, 0, 1446, 0, 0, 3291, 0, 0, 0, 3, 149, 0, 0, 38, 0, 237, 0, 0, 1607, 0, 3584, 3, 1, 1, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 60, 0, 0, 289, 0, 1577, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 394, 16, 3582, 3486, 3153, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 514, 0, 5, 0, 0, 1, 770, 0, 2, 0, 1, 0, 0, 1, 0, 57, 2616, 0, 0, 0] payload_rows_per_dst=[4130, 7520, 7712, 7474, 7612, 7274, 6542, 5784] dup_rows_per_dst=[80, 2024, 3532, 92, 3120, 710, 818, 1112] combine_rows_per_src=[4130, 7520, 7712, 7474, 7612, 7274, 6542, 5784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49998 payload_rows_remote=42212 dup_rows=15538 dup_rows_remote=10366 dedup_ratio=0.237091 router_tokens_per_expert=[3, 0, 2479, 3, 4, 0, 0, 8, 0, 1, 0, 0, 1, 0, 0, 0, 98, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1165, 0, 0, 3290, 0, 0, 0, 1, 117, 0, 0, 40, 0, 106, 0, 1, 1410, 0, 3840, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 170, 2, 0, 164, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3937, 160, 0, 3352, 3391, 3947, 0, 78, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 4, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2757, 0, 0, 0] payload_rows_per_dst=[4966, 2522, 8020, 7940, 2792, 8020, 7952, 7786] dup_rows_per_dst=[32, 28, 3012, 280, 44, 5790, 1180, 5172] combine_rows_per_src=[4966, 2522, 8020, 7940, 2792, 8020, 7952, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53386 payload_rows_remote=49350 dup_rows=12150 dup_rows_remote=11598 dedup_ratio=0.185394 router_tokens_per_expert=[0, 0, 2561, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3, 94, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1776, 0, 0, 3570, 0, 0, 0, 1, 233, 0, 0, 14, 0, 419, 0, 0, 1627, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 99, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 248, 1, 2984, 3401, 3652, 0, 18, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 466, 0, 4, 0, 0, 2, 392, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2414, 0, 0, 0] payload_rows_per_dst=[8008, 7894, 4036, 7428, 7312, 6102, 7408, 5198] dup_rows_per_dst=[4830, 3024, 552, 36, 1998, 364, 874, 472] combine_rows_per_src=[8008, 7894, 4036, 7428, 7312, 6102, 7408, 5198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=190021632 dispatch_meta_bytes=228496 combine_payload_bytes=190021632 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=190250128 ingress_bytes=227770368 link_bytes=227770368 token_bytes=380271760 weight_movement_bytes=37748736 total_nvlink_bytes=418020496 weight_to_token_ratio=0.099268 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182383596e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=200556544 dispatch_meta_bytes=233768 combine_payload_bytes=200556544 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=200790312 ingress_bytes=313802752 link_bytes=313802752 token_bytes=401346856 weight_movement_bytes=113246208 total_nvlink_bytes=514593064 weight_to_token_ratio=0.282165 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=22528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.006703569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171827200 dispatch_meta_bytes=216768 combine_payload_bytes=171827200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172043968 ingress_bytes=190701568 link_bytes=190701568 token_bytes=343871168 weight_movement_bytes=18874368 total_nvlink_bytes=362745536 weight_to_token_ratio=0.054888 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.065972 gemm_active_groups=15 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.827208594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=187424768 dispatch_meta_bytes=225800 combine_payload_bytes=187424768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=187650568 ingress_bytes=225173504 link_bytes=225173504 token_bytes=375075336 weight_movement_bytes=37748736 total_nvlink_bytes=412824072 weight_to_token_ratio=0.100643 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.157501723e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=190201856 dispatch_meta_bytes=219216 combine_payload_bytes=190201856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190421072 ingress_bytes=209076224 link_bytes=209076224 token_bytes=380622928 weight_movement_bytes=18874368 total_nvlink_bytes=399497296 weight_to_token_ratio=0.049588 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.003265506e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=172900352 dispatch_meta_bytes=210312 combine_payload_bytes=172900352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=173110664 ingress_bytes=191774720 link_bytes=191774720 token_bytes=346011016 weight_movement_bytes=18874368 total_nvlink_bytes=364885384 weight_to_token_ratio=0.054548 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.837491008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45490 payload_rows_remote=39508 dup_rows=20046 dup_rows_remote=17054 dedup_ratio=0.305878 router_tokens_per_expert=[765, 0, 1, 0, 1657, 635, 642, 8, 2, 3, 194, 293, 144, 258, 388, 174, 3, 22, 14, 184, 410, 140, 42, 1, 249, 616, 54, 72, 21, 10, 153, 142, 480, 267, 10, 78, 31, 53, 579, 21, 61, 178, 131, 1255, 589, 9, 96, 510, 56, 125, 38, 35, 160, 793, 13, 33, 420, 31, 107, 451, 34, 444, 24, 25, 682, 252, 28, 5, 86, 0, 165, 968, 19, 186, 666, 256, 15, 470, 21, 1895, 541, 27, 52, 275, 36, 188, 23, 0, 390, 151, 36, 680, 22, 346, 104, 5, 2, 121, 976, 47, 5, 947, 1328, 61, 562, 47, 167, 28, 360, 416, 245, 0, 683, 763, 211, 33, 372, 18, 641, 76, 65, 1017, 222, 23, 55, 7, 240, 6] payload_rows_per_dst=[5078, 5922, 5894, 5240, 5484, 6092, 5982, 5798] dup_rows_per_dst=[1936, 2134, 2802, 1988, 2154, 2974, 2992, 3066] combine_rows_per_src=[5078, 5922, 5894, 5240, 5484, 6092, 5982, 5798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=202137600 dispatch_meta_bytes=243792 combine_payload_bytes=202137600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202381392 ingress_bytes=239886336 link_bytes=239886336 token_bytes=404518992 weight_movement_bytes=37748736 total_nvlink_bytes=442267728 weight_to_token_ratio=0.093318 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.072917 gemm_active_groups=15 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.298472840e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56426 payload_rows_remote=52632 dup_rows=9110 dup_rows_remote=8894 dedup_ratio=0.139008 router_tokens_per_expert=[0, 0, 89, 0, 0, 0, 1, 0, 0, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 3, 0, 3, 88, 7, 0, 0, 0, 42, 4069, 4, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 1199, 0, 533, 0, 0, 0, 0, 0, 0, 273, 0, 0, 4094, 0, 0, 0, 4070, 75, 22, 3965, 1, 0, 4039, 0, 0, 0, 0, 0, 4, 0, 0, 8, 0, 573, 0, 0, 0, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 30, 3, 0, 0, 0, 0, 0, 58, 0, 4066, 0, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7962, 8176, 3794, 8152, 8176, 8128, 3850] dup_rows_per_dst=[2462, 1324, 538, 216, 192, 4220, 112, 46] combine_rows_per_src=[8188, 7962, 8176, 3794, 8152, 8176, 8128, 3850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54882 payload_rows_remote=52576 dup_rows=10654 dup_rows_remote=10562 dedup_ratio=0.162567 router_tokens_per_expert=[0, 0, 518, 0, 2, 0, 11, 0, 0, 0, 8, 1028, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 12, 20, 0, 0, 1, 0, 188, 4080, 0, 0, 3, 0, 1, 3, 0, 0, 300, 0, 2, 1216, 1, 781, 1, 0, 0, 0, 0, 0, 495, 0, 0, 4086, 0, 0, 0, 4045, 292, 0, 4084, 0, 0, 4084, 0, 5, 0, 0, 0, 23, 0, 0, 0, 0, 380, 0, 1, 0, 0, 817, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 113, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 3, 10, 1, 4089, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4962, 8190, 8176, 6802, 8078, 2306, 8178, 8190] dup_rows_per_dst=[1150, 1742, 976, 3384, 600, 92, 232, 2478] combine_rows_per_src=[4962, 8190, 8176, 6802, 8078, 2306, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161824768 dispatch_meta_bytes=226248 combine_payload_bytes=161824768 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=162051016 ingress_bytes=218447872 link_bytes=218447872 token_bytes=323875784 weight_movement_bytes=56623104 total_nvlink_bytes=380498888 weight_to_token_ratio=0.174830 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6912 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.093060026e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54060 payload_rows_remote=45898 dup_rows=11476 dup_rows_remote=8136 dedup_ratio=0.175110 router_tokens_per_expert=[0, 0, 74, 0, 0, 18, 2, 0, 0, 0, 0, 1487, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 1513, 0, 0, 0, 0, 164, 5, 3, 0, 0, 130, 4081, 9, 0, 13, 6, 0, 0, 10, 1, 432, 0, 0, 684, 0, 381, 0, 3, 0, 0, 0, 0, 381, 7, 0, 4072, 0, 0, 0, 3999, 158, 0, 4056, 0, 5, 4067, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 461, 10, 10, 0, 0, 1219, 0, 0, 4, 3, 0, 0, 2, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 14, 0, 5, 0, 3, 6, 0, 51, 0, 4073, 0, 0, 1107, 0, 0, 0, 0, 4, 0, 0, 1, 0] payload_rows_per_dst=[8190, 8162, 8190, 2842, 8010, 8184, 8166, 2316] dup_rows_per_dst=[3126, 3340, 1190, 70, 316, 3380, 44, 10] combine_rows_per_src=[8190, 8162, 8190, 2842, 8010, 8184, 8166, 2316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53066 payload_rows_remote=44876 dup_rows=12470 dup_rows_remote=10360 dedup_ratio=0.190277 router_tokens_per_expert=[0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 1, 0, 0, 143, 1, 0, 1, 0, 43, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 1, 1167, 0, 778, 0, 0, 0, 0, 0, 0, 208, 0, 0, 4093, 0, 0, 0, 4023, 471, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 119, 0, 0, 0, 0, 1719, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 76, 0, 4095, 0, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 4026, 6778, 8192, 6562, 2936] dup_rows_per_dst=[2110, 1818, 3660, 282, 678, 3678, 232, 12] combine_rows_per_src=[8190, 8190, 8192, 4026, 6778, 8192, 6562, 2936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57080 payload_rows_remote=48892 dup_rows=8456 dup_rows_remote=8228 dedup_ratio=0.129028 router_tokens_per_expert=[0, 0, 319, 0, 0, 0, 4, 0, 0, 0, 0, 1240, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 117, 0, 0, 0, 1, 40, 1, 1, 0, 0, 60, 4084, 1, 2, 0, 1, 0, 0, 1, 0, 455, 0, 8, 1755, 0, 1164, 0, 3, 0, 0, 0, 0, 245, 0, 5, 4085, 0, 0, 5, 4087, 104, 2, 4085, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 89, 0, 0, 0, 0, 1402, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2, 21, 0, 4085, 1, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8174, 8178, 8184, 5544, 8188, 7442, 8182, 3188] dup_rows_per_dst=[3124, 316, 1028, 816, 228, 2314, 210, 420] combine_rows_per_src=[8174, 8178, 8184, 5544, 8188, 7442, 8182, 3188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58416 payload_rows_remote=50224 dup_rows=7120 dup_rows_remote=6688 dedup_ratio=0.108643 router_tokens_per_expert=[0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 38, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 1169, 0, 1150, 0, 0, 0, 0, 0, 0, 172, 0, 0, 4096, 0, 0, 0, 4075, 87, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 344, 0, 0, 0, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4604, 8158, 8192, 8192, 4694] dup_rows_per_dst=[1280, 1380, 432, 378, 172, 2908, 562, 8] combine_rows_per_src=[8192, 8192, 8192, 4604, 8158, 8192, 8192, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=41850 payload_rows_remote=33658 dup_rows=23686 dup_rows_remote=12242 dedup_ratio=0.361420 router_tokens_per_expert=[0, 0, 135, 0, 0, 0, 9, 0, 0, 0, 0, 1381, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 0, 0, 0, 0, 72, 0, 0, 0, 0, 47, 4092, 0, 0, 1, 0, 0, 0, 1, 0, 289, 0, 1, 1122, 2, 493, 0, 0, 0, 0, 0, 0, 186, 1, 0, 4094, 0, 0, 1, 4007, 105, 0, 4094, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1041, 1, 0, 0, 0, 1, 0, 2, 0, 0, 0, 69, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 43, 0, 4094, 0, 0, 1670, 0, 0, 0, 1, 1, 0, 0, 1, 1] payload_rows_per_dst=[3012, 3076, 670, 8192, 8190, 2332, 8186, 8192] dup_rows_per_dst=[40, 30, 6, 11790, 210, 26, 140, 11444] combine_rows_per_src=[3012, 3076, 670, 8192, 8190, 2332, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=215580672 dispatch_meta_bytes=246104 combine_payload_bytes=215580672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215826776 ingress_bytes=253329408 link_bytes=253329408 token_bytes=431407448 weight_movement_bytes=37748736 total_nvlink_bytes=469156184 weight_to_token_ratio=0.087501 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=10880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427277742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=215351296 dispatch_meta_bytes=252552 combine_payload_bytes=215351296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215603848 ingress_bytes=234225664 link_bytes=234225664 token_bytes=430955144 weight_movement_bytes=18874368 total_nvlink_bytes=449829512 weight_to_token_ratio=0.043797 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.244235066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=187998208 dispatch_meta_bytes=216136 combine_payload_bytes=187998208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=188214344 ingress_bytes=206872576 link_bytes=206872576 token_bytes=376212552 weight_movement_bytes=18874368 total_nvlink_bytes=395086920 weight_to_token_ratio=0.050169 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.982151236e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183812096 dispatch_meta_bytes=220944 combine_payload_bytes=183812096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184033040 ingress_bytes=202686464 link_bytes=202686464 token_bytes=367845136 weight_movement_bytes=18874368 total_nvlink_bytes=386719504 weight_to_token_ratio=0.051311 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.942041970e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=200261632 dispatch_meta_bytes=228480 combine_payload_bytes=200261632 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=200490112 ingress_bytes=275759104 link_bytes=275759104 token_bytes=400751744 weight_movement_bytes=75497472 total_nvlink_bytes=476249216 weight_to_token_ratio=0.188390 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.642188053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=205717504 dispatch_meta_bytes=227648 combine_payload_bytes=205717504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205945152 ingress_bytes=243466240 link_bytes=243466240 token_bytes=411662656 weight_movement_bytes=37748736 total_nvlink_bytes=449411392 weight_to_token_ratio=0.091698 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332773719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=137863168 dispatch_meta_bytes=183600 combine_payload_bytes=137863168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=138046768 ingress_bytes=175611904 link_bytes=175611904 token_bytes=275909936 weight_movement_bytes=37748736 total_nvlink_bytes=313658672 weight_to_token_ratio=0.136815 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=13056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59194 payload_rows_remote=55768 dup_rows=6342 dup_rows_remote=5356 dedup_ratio=0.096771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3882, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 3024, 0, 0, 0, 8, 49, 0, 7, 0, 0] payload_rows_per_dst=[8192, 7784, 8192, 3426, 7540, 8192, 8192, 7676] dup_rows_per_dst=[10, 1076, 2, 986, 0, 176, 126, 3966] combine_rows_per_src=[8192, 7784, 8192, 3426, 7540, 8192, 8192, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49900 payload_rows_remote=41824 dup_rows=15636 dup_rows_remote=14560 dedup_ratio=0.238586 router_tokens_per_expert=[0, 0, 2500, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1261, 0, 0, 3371, 0, 0, 0, 0, 263, 0, 0, 44, 0, 377, 0, 0, 1046, 0, 3440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 131, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 277, 3, 3264, 3324, 4030, 0, 6, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 536, 0, 1, 0, 0, 1, 639, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2883, 0, 0, 0] payload_rows_per_dst=[5000, 2540, 7588, 7934, 2932, 7934, 8076, 7896] dup_rows_per_dst=[12, 2, 2752, 36, 50, 5802, 1076, 5906] combine_rows_per_src=[5000, 2540, 7588, 7934, 2932, 7934, 8076, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61034 payload_rows_remote=52846 dup_rows=4502 dup_rows_remote=4464 dedup_ratio=0.068695 router_tokens_per_expert=[0, 2, 6, 0, 2, 3, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4079, 0, 2, 4086, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 171, 0, 1, 0, 5, 4077, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 17, 0, 0, 4091, 2, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 2031, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8190, 3770, 8176, 8162, 8164, 8188, 8192, 8192] dup_rows_per_dst=[28, 0, 0, 342, 6, 38, 26, 4062] combine_rows_per_src=[8190, 3770, 8176, 8162, 8164, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59150 payload_rows_remote=51172 dup_rows=6386 dup_rows_remote=2668 dedup_ratio=0.097443 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 21, 0, 0, 3, 0, 0, 0, 1352, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 532, 0, 0, 0, 2, 0, 0, 0, 56, 0, 21, 0, 0, 11, 0, 0, 4074, 3953, 0, 14, 4073, 0, 13, 0, 4075, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 3, 0, 299, 0, 0, 4063, 0, 0, 0, 1, 0, 1, 4075, 0, 2, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2, 3, 15, 0, 0, 1040, 0, 0, 0, 8, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7978, 8192, 8188, 8122, 8154, 8190, 2134] dup_rows_per_dst=[42, 3718, 136, 1866, 8, 578, 30, 8] combine_rows_per_src=[8192, 7978, 8192, 8188, 8122, 8154, 8190, 2134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59318 payload_rows_remote=51286 dup_rows=6218 dup_rows_remote=6214 dedup_ratio=0.094879 router_tokens_per_expert=[0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 0, 0, 4, 0, 0, 4081, 4089, 0, 0, 4090, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 161, 0, 0, 5, 0, 4011, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 0, 0, 0, 1938, 0, 0, 0, 0, 10, 1, 10, 0, 4] payload_rows_per_dst=[8184, 6032, 8178, 4324, 8032, 8190, 8186, 8192] dup_rows_per_dst=[18, 1974, 34, 210, 4, 46, 10, 3922] combine_rows_per_src=[8184, 6032, 8178, 4324, 8032, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60162 payload_rows_remote=51974 dup_rows=5374 dup_rows_remote=5372 dedup_ratio=0.082001 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4092, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 822, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8182, 8190, 8192, 2834] dup_rows_per_dst=[0, 3650, 2, 1642, 0, 72, 8, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8182, 8190, 8192, 2834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58272 payload_rows_remote=50080 dup_rows=7264 dup_rows_remote=7262 dedup_ratio=0.110840 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4095, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 123, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8192, 6990, 7852, 2474] dup_rows_per_dst=[2, 4434, 0, 1746, 904, 176, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8192, 6990, 7852, 2474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=228425728 dispatch_meta_bytes=244496 combine_payload_bytes=228425728 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=228670224 ingress_bytes=303923200 link_bytes=303923200 token_bytes=457095952 weight_movement_bytes=75497472 total_nvlink_bytes=532593424 weight_to_token_ratio=0.165168 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=14464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.912042563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61716 payload_rows_remote=53528 dup_rows=3820 dup_rows_remote=1388 dedup_ratio=0.058289 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4093, 4094, 0, 0, 4094, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 493, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1216, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8088, 4630, 8048, 8190, 8192, 8188, 8192, 8188] dup_rows_per_dst=[8, 0, 4, 1094, 2, 138, 142, 2432] combine_rows_per_src=[8088, 4630, 8048, 8190, 8192, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171311104 dispatch_meta_bytes=225536 combine_payload_bytes=171311104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171536640 ingress_bytes=190185472 link_bytes=190185472 token_bytes=342847744 weight_movement_bytes=18874368 total_nvlink_bytes=361722112 weight_to_token_ratio=0.055052 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.822263616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=216457216 dispatch_meta_bytes=229240 combine_payload_bytes=216457216 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=216686456 ingress_bytes=273080320 link_bytes=273080320 token_bytes=433143672 weight_movement_bytes=56623104 total_nvlink_bytes=489766776 weight_to_token_ratio=0.130726 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.616521262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=209600512 dispatch_meta_bytes=215360 combine_payload_bytes=209600512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209815872 ingress_bytes=228474880 link_bytes=228474880 token_bytes=419416384 weight_movement_bytes=18874368 total_nvlink_bytes=438290752 weight_to_token_ratio=0.045002 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189133884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=210067456 dispatch_meta_bytes=230000 combine_payload_bytes=210067456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210297456 ingress_bytes=228941824 link_bytes=228941824 token_bytes=420364912 weight_movement_bytes=18874368 total_nvlink_bytes=439239280 weight_to_token_ratio=0.044900 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.193607911e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=212885504 dispatch_meta_bytes=229384 combine_payload_bytes=212885504 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=213114888 ingress_bytes=231759872 link_bytes=231759872 token_bytes=426000392 weight_movement_bytes=18874368 total_nvlink_bytes=444874760 weight_to_token_ratio=0.044306 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220609061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219250688 dispatch_meta_bytes=219664 combine_payload_bytes=219250688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219470352 ingress_bytes=238125056 link_bytes=238125056 token_bytes=438721040 weight_movement_bytes=18874368 total_nvlink_bytes=457595408 weight_to_token_ratio=0.043021 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.281597122e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205127680 dispatch_meta_bytes=229368 combine_payload_bytes=205127680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205357048 ingress_bytes=224002048 link_bytes=224002048 token_bytes=410484728 weight_movement_bytes=18874368 total_nvlink_bytes=429359096 weight_to_token_ratio=0.045981 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.146277408e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59262 payload_rows_remote=51730 dup_rows=6274 dup_rows_remote=6272 dedup_ratio=0.095734 router_tokens_per_expert=[0, 3474, 0, 0, 0, 0, 0, 821, 0, 0, 5, 4085, 0, 0, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 106, 0, 77, 4074, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0] payload_rows_per_dst=[7830, 8192, 6626, 7532, 8170, 5428, 8190, 7294] dup_rows_per_dst=[4326, 2, 1480, 2, 0, 0, 420, 44] combine_rows_per_src=[7830, 8192, 6626, 7532, 8170, 5428, 8190, 7294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57156 payload_rows_remote=49490 dup_rows=8380 dup_rows_remote=4834 dedup_ratio=0.127869 router_tokens_per_expert=[0, 2167, 0, 0, 4, 0, 0, 1369, 0, 0, 52, 4091, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 4084, 0, 18, 152, 0, 0, 4083, 2087, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 6, 0, 0, 1, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 1, 0, 4091, 0, 0, 4, 0, 0, 114, 0, 0, 4, 0, 0, 0, 0, 0, 3923, 3, 1, 0, 4, 1, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[2944, 8180, 8188, 5938, 8186, 7666, 8190, 7864] dup_rows_per_dst=[38, 0, 4498, 10, 38, 3546, 236, 14] combine_rows_per_src=[2944, 8180, 8188, 5938, 8186, 7666, 8190, 7864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62658 payload_rows_remote=54470 dup_rows=2878 dup_rows_remote=2878 dedup_ratio=0.043915 router_tokens_per_expert=[0, 3239, 0, 0, 0, 3, 0, 674, 0, 0, 24, 4091, 0, 1, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 277, 0, 0, 4090, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 3, 3960, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4082, 0, 0, 0, 4, 0, 262, 0, 0, 3, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3, 0, 0] payload_rows_per_dst=[7198, 8192, 8192, 7924, 8188, 6890, 8184, 7890] dup_rows_per_dst=[684, 2, 1620, 18, 0, 2, 518, 34] combine_rows_per_src=[7198, 8192, 8192, 7924, 8188, 6890, 8184, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49800 payload_rows_remote=41608 dup_rows=15736 dup_rows_remote=15518 dedup_ratio=0.240112 router_tokens_per_expert=[0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 0, 1, 0, 0, 98, 0, 0, 0, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 777, 0, 675, 0, 0, 0, 0, 0, 0, 498, 0, 0, 4095, 0, 0, 0, 4001, 197, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3298, 3146, 7532, 8190, 7208, 4042, 8192, 8192] dup_rows_per_dst=[38, 490, 394, 4606, 378, 52, 218, 9560] combine_rows_per_src=[3298, 3146, 7532, 8190, 7208, 4042, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57502 payload_rows_remote=49316 dup_rows=8034 dup_rows_remote=8034 dedup_ratio=0.122589 router_tokens_per_expert=[0, 2133, 0, 0, 20, 0, 0, 612, 7, 18, 19, 4072, 0, 0, 0, 3258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4072, 0, 0, 103, 0, 2, 4061, 2409, 0, 0, 11, 0, 21, 19, 22, 7, 0, 0, 0, 0, 0, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 0, 0, 5, 0, 4074, 0, 0, 1, 0, 1, 1092, 0, 0, 0, 0, 4, 0, 0, 0, 3856, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0] payload_rows_per_dst=[8190, 8186, 5026, 5482, 8170, 6560, 8152, 7736] dup_rows_per_dst=[5550, 0, 162, 0, 6, 28, 2192, 96] combine_rows_per_src=[8190, 8186, 5026, 5482, 8170, 6560, 8152, 7736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=211886080 dispatch_meta_bytes=232008 combine_payload_bytes=211886080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212118088 ingress_bytes=230760448 link_bytes=230760448 token_bytes=424004168 weight_movement_bytes=18874368 total_nvlink_bytes=442878536 weight_to_token_ratio=0.044515 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.211033072e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58244 payload_rows_remote=55320 dup_rows=7292 dup_rows_remote=7234 dedup_ratio=0.111267 router_tokens_per_expert=[0, 2012, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 4096, 0, 0, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 197, 0, 0, 4096, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 160, 6, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2924, 7646, 8192, 7000, 8184, 7914] dup_rows_per_dst=[6870, 0, 58, 0, 0, 0, 332, 32] combine_rows_per_src=[8192, 8192, 2924, 7646, 8192, 7000, 8184, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55084 payload_rows_remote=47072 dup_rows=10452 dup_rows_remote=8236 dedup_ratio=0.159485 router_tokens_per_expert=[0, 2457, 0, 0, 2, 0, 0, 246, 0, 0, 27, 4094, 0, 0, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 476, 0, 0, 4094, 1174, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 4089, 2, 0, 0, 0, 0, 782, 2, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[550, 8188, 6518, 7686, 8192, 7754, 8184, 8012] dup_rows_per_dst=[0, 0, 2756, 0, 4, 3910, 1566, 2216] combine_rows_per_src=[550, 8188, 6518, 7686, 8192, 7754, 8184, 8012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53614 payload_rows_remote=45424 dup_rows=11922 dup_rows_remote=7554 dedup_ratio=0.181915 router_tokens_per_expert=[0, 1600, 0, 0, 0, 0, 0, 455, 0, 0, 129, 4095, 0, 0, 0, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 465, 0, 0, 4095, 1854, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 1, 0, 0, 0, 802, 4, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 4504, 8154, 8192, 6484, 1732, 8168] dup_rows_per_dst=[4368, 20, 418, 7074, 2, 0, 32, 8] combine_rows_per_src=[8190, 8190, 4504, 8154, 8192, 6484, 1732, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=217296 combine_payload_bytes=202711040 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=202928336 ingress_bytes=240459776 link_bytes=240459776 token_bytes=405639376 weight_movement_bytes=37748736 total_nvlink_bytes=443388112 weight_to_token_ratio=0.093060 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.303967260e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=223109120 dispatch_meta_bytes=229392 combine_payload_bytes=223109120 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=223338512 ingress_bytes=260857856 link_bytes=260857856 token_bytes=446447632 weight_movement_bytes=37748736 total_nvlink_bytes=484196368 weight_to_token_ratio=0.084554 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.499411626e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=201998336 dispatch_meta_bytes=229400 combine_payload_bytes=201998336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202227736 ingress_bytes=220872704 link_bytes=220872704 token_bytes=404226072 weight_movement_bytes=18874368 total_nvlink_bytes=423100440 weight_to_token_ratio=0.046693 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.116293574e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=170426368 dispatch_meta_bytes=228504 combine_payload_bytes=170426368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=170654872 ingress_bytes=189300736 link_bytes=189300736 token_bytes=341081240 weight_movement_bytes=18874368 total_nvlink_bytes=359955608 weight_to_token_ratio=0.055337 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.813786511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=226590720 dispatch_meta_bytes=250216 combine_payload_bytes=226590720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226840936 ingress_bytes=283213824 link_bytes=283213824 token_bytes=453431656 weight_movement_bytes=56623104 total_nvlink_bytes=510054760 weight_to_token_ratio=0.124877 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.713615512e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=192806912 dispatch_meta_bytes=221232 combine_payload_bytes=192806912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193028144 ingress_bytes=211681280 link_bytes=211681280 token_bytes=385835056 weight_movement_bytes=18874368 total_nvlink_bytes=404709424 weight_to_token_ratio=0.048918 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.028225871e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186056704 dispatch_meta_bytes=211912 combine_payload_bytes=186056704 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=186268616 ingress_bytes=280428544 link_bytes=280428544 token_bytes=372325320 weight_movement_bytes=94371840 total_nvlink_bytes=466697160 weight_to_token_ratio=0.253466 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.686928329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64338 payload_rows_remote=56148 dup_rows=1198 dup_rows_remote=1198 dedup_ratio=0.018280 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 246, 329, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8190, 8192, 6998, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[32, 0, 0, 0, 2, 0, 44, 1120] combine_rows_per_src=[8190, 8192, 6998, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57220 dup_rows=138 dup_rows_remote=132 dedup_ratio=0.002106 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 2, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 3, 1, 4084, 0, 0, 0, 0, 0, 44, 17, 0, 0, 4091, 0, 0] payload_rows_per_dst=[8188, 8188, 8114, 8192, 8172, 8178, 8184, 8182] dup_rows_per_dst=[0, 6, 8, 0, 0, 6, 2, 116] combine_rows_per_src=[8188, 8188, 8114, 8192, 8172, 8178, 8184, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65040 payload_rows_remote=56858 dup_rows=496 dup_rows_remote=496 dedup_ratio=0.007568 router_tokens_per_expert=[0, 0, 6, 6, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 2, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 5, 3890, 0, 0, 5, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 1, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 260, 10, 0, 1, 4091, 0, 0] payload_rows_per_dst=[8192, 8138, 7794, 8182, 8182, 8188, 8184, 8180] dup_rows_per_dst=[30, 0, 14, 10, 0, 0, 2, 440] combine_rows_per_src=[8192, 8138, 7794, 8182, 8182, 8188, 8184, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=229982208 dispatch_meta_bytes=229384 combine_payload_bytes=229982208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230211592 ingress_bytes=248856576 link_bytes=248856576 token_bytes=460193800 weight_movement_bytes=18874368 total_nvlink_bytes=479068168 weight_to_token_ratio=0.041014 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384421266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65210 payload_rows_remote=57328 dup_rows=326 dup_rows_remote=314 dedup_ratio=0.004974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 6, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 3960, 0, 0, 0, 0, 0, 271, 15, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8184, 7882, 8192, 8190, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 314] combine_rows_per_src=[8192, 8184, 7882, 8192, 8190, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65074 payload_rows_remote=56918 dup_rows=462 dup_rows_remote=462 dedup_ratio=0.007050 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4072, 0, 0, 0, 0, 0, 2, 0, 0, 3912, 0, 7, 0, 0, 23, 0, 0, 22, 0, 1, 1, 0, 0, 0, 2, 0, 2, 11, 0, 0, 0, 7, 4068, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 4, 0, 14, 4068, 0, 0, 1, 0, 0, 0, 0, 0, 0, 23, 24, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4069, 12, 0, 0, 0, 0, 0, 6, 0, 4047, 0, 0, 0, 0, 0, 140, 27, 12, 0, 4072, 0, 0] payload_rows_per_dst=[8150, 8156, 7886, 8176, 8188, 8192, 8178, 8148] dup_rows_per_dst=[2, 0, 48, 6, 30, 54, 6, 316] combine_rows_per_src=[8150, 8156, 7886, 8176, 8188, 8192, 8178, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65330 payload_rows_remote=57150 dup_rows=206 dup_rows_remote=26 dedup_ratio=0.003143 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 7, 0, 0, 0, 4017, 0, 0, 0, 0, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4023, 0, 0, 0, 0, 0, 142, 1, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8170, 8184, 8038, 8192, 8188, 8192, 8186, 8180] dup_rows_per_dst=[0, 0, 20, 0, 0, 6, 0, 180] combine_rows_per_src=[8170, 8184, 8038, 8192, 8188, 8192, 8186, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61428 payload_rows_remote=53236 dup_rows=4108 dup_rows_remote=4104 dedup_ratio=0.062683 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4108, 8192, 8178, 8188, 8192, 8192, 8186] dup_rows_per_dst=[6, 0, 0, 800, 0, 28, 4, 3270] combine_rows_per_src=[8192, 4108, 8192, 8178, 8188, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61996 payload_rows_remote=53804 dup_rows=3540 dup_rows_remote=3534 dedup_ratio=0.054016 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 1, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 270, 7, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 7864, 8192, 8192, 8142, 8176, 5046] dup_rows_per_dst=[6, 110, 3090, 12, 14, 2, 0, 306] combine_rows_per_src=[8192, 8192, 7864, 8192, 8192, 8142, 8176, 5046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234373120 dispatch_meta_bytes=229408 combine_payload_bytes=234373120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234602528 ingress_bytes=253247488 link_bytes=253247488 token_bytes=468975648 weight_movement_bytes=18874368 total_nvlink_bytes=487850016 weight_to_token_ratio=0.040246 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426492824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=232890368 dispatch_meta_bytes=229416 combine_payload_bytes=232890368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233119784 ingress_bytes=251764736 link_bytes=251764736 token_bytes=466010152 weight_movement_bytes=18874368 total_nvlink_bytes=484884520 weight_to_token_ratio=0.040502 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412285824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=230568 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235046056 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469861544 weight_movement_bytes=18874368 total_nvlink_bytes=488735912 weight_to_token_ratio=0.040170 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=233136128 dispatch_meta_bytes=229520 combine_payload_bytes=233136128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233365648 ingress_bytes=270884864 link_bytes=270884864 token_bytes=466501776 weight_movement_bytes=37748736 total_nvlink_bytes=504250512 weight_to_token_ratio=0.080919 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595485483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234086400 dispatch_meta_bytes=228704 combine_payload_bytes=234086400 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234315104 ingress_bytes=328458240 link_bytes=328458240 token_bytes=468401504 weight_movement_bytes=94371840 total_nvlink_bytes=562773344 weight_to_token_ratio=0.201476 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.147125244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=218054656 dispatch_meta_bytes=229360 combine_payload_bytes=218054656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218284016 ingress_bytes=255803392 link_bytes=255803392 token_bytes=436338672 weight_movement_bytes=37748736 total_nvlink_bytes=474087408 weight_to_token_ratio=0.086512 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450982239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=220381184 dispatch_meta_bytes=229352 combine_payload_bytes=220381184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220610536 ingress_bytes=239255552 link_bytes=239255552 token_bytes=440991720 weight_movement_bytes=18874368 total_nvlink_bytes=459866088 weight_to_token_ratio=0.042800 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.076389 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.292428978e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62410 payload_rows_remote=54218 dup_rows=3126 dup_rows_remote=3122 dedup_ratio=0.047699 router_tokens_per_expert=[0, 0, 0, 0, 0, 32, 4050, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 4096, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 3998, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8192, 8192, 8192, 8192, 6082, 7270, 8186] dup_rows_per_dst=[596, 0, 0, 4, 0, 0, 1262, 1264] combine_rows_per_src=[8104, 8192, 8192, 8192, 8192, 6082, 7270, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60398 payload_rows_remote=54170 dup_rows=5138 dup_rows_remote=3288 dedup_ratio=0.078400 router_tokens_per_expert=[0, 0, 0, 0, 0, 30, 4054, 0, 0, 0, 0, 1220, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 4085, 0, 0, 0, 0, 2, 0, 0, 0, 4090, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 1, 0, 0, 4089, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3759, 4090, 1880, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 5, 0, 0, 0, 0, 0, 1233, 0, 0, 0, 4089, 86, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 1, 0, 0] payload_rows_per_dst=[5332, 8184, 8192, 8188, 8180, 6228, 7904, 8190] dup_rows_per_dst=[970, 6, 10, 22, 0, 1850, 2104, 176] combine_rows_per_src=[5332, 8184, 8192, 8188, 8180, 6228, 7904, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=222076928 dispatch_meta_bytes=229360 combine_payload_bytes=222076928 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=222306288 ingress_bytes=259825664 link_bytes=259825664 token_bytes=444383216 weight_movement_bytes=37748736 total_nvlink_bytes=482131952 weight_to_token_ratio=0.084946 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.489521670e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60914 payload_rows_remote=52736 dup_rows=4622 dup_rows_remote=4622 dedup_ratio=0.070526 router_tokens_per_expert=[0, 0, 0, 3, 0, 44, 4070, 0, 0, 0, 0, 1704, 0, 15, 0, 0, 3, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3181, 4089, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4085, 246, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0] payload_rows_per_dst=[8180, 8192, 8192, 8180, 8178, 4930, 6878, 8184] dup_rows_per_dst=[3492, 6, 10, 2, 0, 0, 616, 496] combine_rows_per_src=[8180, 8192, 8192, 8180, 8178, 4930, 6878, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61288 payload_rows_remote=53096 dup_rows=4248 dup_rows_remote=4248 dedup_ratio=0.064819 router_tokens_per_expert=[0, 0, 0, 0, 0, 58, 4082, 0, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 4096, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 4096, 125, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8176, 8192, 8192, 8188, 8192, 4840, 7316, 8192] dup_rows_per_dst=[2174, 0, 0, 0, 0, 0, 1818, 256] combine_rows_per_src=[8176, 8192, 8192, 8188, 8192, 4840, 7316, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60394 payload_rows_remote=52268 dup_rows=5142 dup_rows_remote=5100 dedup_ratio=0.078461 router_tokens_per_expert=[0, 0, 0, 0, 1, 243, 4033, 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 22, 1, 13, 4038, 0, 0, 0, 0, 9, 45, 0, 0, 4069, 20, 0, 0, 0, 5, 0, 0, 0, 13, 0, 0, 0, 13, 0, 12, 21, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2702, 4070, 2513, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 853, 16, 0, 0, 4067, 208, 0, 0, 0, 0, 0, 4, 0, 0, 24, 0, 0, 1, 0] payload_rows_per_dst=[8144, 8126, 8192, 8184, 8144, 5066, 6350, 8188] dup_rows_per_dst=[3660, 42, 104, 74, 0, 0, 842, 420] combine_rows_per_src=[8144, 8126, 8192, 8184, 8144, 5066, 6350, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59220 payload_rows_remote=51028 dup_rows=6316 dup_rows_remote=6062 dedup_ratio=0.096375 router_tokens_per_expert=[0, 0, 0, 0, 0, 136, 4007, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4093, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 4094, 2191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1189, 0, 0, 0, 4092, 40, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0] payload_rows_per_dst=[2798, 8190, 8016, 8190, 8188, 8150, 7496, 8192] dup_rows_per_dst=[30, 2, 6, 2, 0, 4250, 1772, 254] combine_rows_per_src=[2798, 8190, 8016, 8190, 8188, 8150, 7496, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57494 payload_rows_remote=49302 dup_rows=8042 dup_rows_remote=3968 dedup_ratio=0.122711 router_tokens_per_expert=[0, 0, 0, 0, 0, 22, 4095, 0, 0, 1, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 4095, 3333, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 4095, 45, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8118, 8192, 7684, 3828, 5098] dup_rows_per_dst=[4074, 20, 56, 0, 78, 3526, 220, 68] combine_rows_per_src=[8192, 8192, 8190, 8118, 8192, 7684, 3828, 5098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56898 payload_rows_remote=48706 dup_rows=8638 dup_rows_remote=7406 dedup_ratio=0.131805 router_tokens_per_expert=[0, 2133, 0, 0, 0, 0, 0, 259, 0, 0, 44, 4096, 0, 0, 0, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 878, 0, 0, 4096, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 603, 13, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0] payload_rows_per_dst=[606, 8192, 8192, 8070, 8192, 7298, 8192, 8156] dup_rows_per_dst=[0, 0, 4374, 0, 4, 2970, 1232, 58] combine_rows_per_src=[606, 8192, 8192, 8070, 8192, 7298, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=221880320 dispatch_meta_bytes=229832 combine_payload_bytes=221880320 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=222110152 ingress_bytes=297377792 link_bytes=297377792 token_bytes=443990472 weight_movement_bytes=75497472 total_nvlink_bytes=519487944 weight_to_token_ratio=0.170043 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.849327684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216006656 dispatch_meta_bytes=229432 combine_payload_bytes=216006656 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=216236088 ingress_bytes=253755392 link_bytes=253755392 token_bytes=432242744 weight_movement_bytes=37748736 total_nvlink_bytes=469991480 weight_to_token_ratio=0.087332 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=217481216 dispatch_meta_bytes=229376 combine_payload_bytes=217481216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217710592 ingress_bytes=255229952 link_bytes=255229952 token_bytes=435191808 weight_movement_bytes=37748736 total_nvlink_bytes=472940544 weight_to_token_ratio=0.086740 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.445487819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=229472 combine_payload_bytes=214089728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214319200 ingress_bytes=232964096 link_bytes=232964096 token_bytes=428408928 weight_movement_bytes=18874368 total_nvlink_bytes=447283296 weight_to_token_ratio=0.044057 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232147342e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209010688 dispatch_meta_bytes=228360 combine_payload_bytes=209010688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209239048 ingress_bytes=246759424 link_bytes=246759424 token_bytes=418249736 weight_movement_bytes=37748736 total_nvlink_bytes=455998472 weight_to_token_ratio=0.090254 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364327388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=201940992 dispatch_meta_bytes=213080 combine_payload_bytes=201940992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202154072 ingress_bytes=239689728 link_bytes=239689728 token_bytes=404095064 weight_movement_bytes=37748736 total_nvlink_bytes=441843800 weight_to_token_ratio=0.093415 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.296589039e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65296 payload_rows_remote=57104 dup_rows=240 dup_rows_remote=240 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3992, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 7984, 8192, 8192] dup_rows_per_dst=[236, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 7984, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=199499776 dispatch_meta_bytes=224448 combine_payload_bytes=199499776 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=199724224 ingress_bytes=256122880 link_bytes=256122880 token_bytes=399224000 weight_movement_bytes=56623104 total_nvlink_bytes=455847104 weight_to_token_ratio=0.141833 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.076389 gemm_active_groups=12 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.454043416e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233897984 dispatch_meta_bytes=229376 combine_payload_bytes=233897984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234127360 ingress_bytes=252772352 link_bytes=252772352 token_bytes=468025344 weight_movement_bytes=18874368 total_nvlink_bytes=486899712 weight_to_token_ratio=0.040328 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421940305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57124 dup_rows=276 dup_rows_remote=258 dedup_ratio=0.004211 router_tokens_per_expert=[0, 2, 0, 0, 113, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 4090, 0, 0, 0, 0, 0, 6, 4088, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 4090, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 4062, 5, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 1, 2, 0, 0, 0] payload_rows_per_dst=[8182, 8188, 8006, 8190, 8192, 8136, 8180, 8186] dup_rows_per_dst=[228, 12, 0, 6, 2, 18, 0, 10] combine_rows_per_src=[8182, 8188, 8006, 8190, 8192, 8136, 8180, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57134 dup_rows=232 dup_rows_remote=232 dedup_ratio=0.003540 router_tokens_per_expert=[0, 0, 0, 0, 86, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 4085, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 1, 0, 5, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 1, 0, 4091, 0, 5, 4051, 3, 5, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0] payload_rows_per_dst=[8182, 8192, 8092, 8190, 8170, 8114, 8180, 8184] dup_rows_per_dst=[170, 12, 2, 8, 0, 34, 0, 6] combine_rows_per_src=[8182, 8192, 8092, 8190, 8170, 8114, 8180, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57292 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 8192, 8192, 8140, 8192, 8192] dup_rows_per_dst=[62, 2, 0, 8, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8170, 8192, 8192, 8140, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65146 payload_rows_remote=56954 dup_rows=390 dup_rows_remote=282 dedup_ratio=0.005951 router_tokens_per_expert=[0, 3, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4069, 0, 2, 0, 0, 0, 19, 4069, 2, 26, 0, 8, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 3, 0, 0, 0, 0, 4070, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 23, 0, 0, 0, 0, 0, 0, 4070, 2, 24, 4024, 23, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 11, 0, 0, 0, 0, 0, 0, 0, 5, 0] payload_rows_per_dst=[8130, 8192, 8096, 8158, 8150, 8100, 8138, 8182] dup_rows_per_dst=[152, 108, 0, 6, 2, 92, 0, 30] combine_rows_per_src=[8130, 8192, 8096, 8158, 8150, 8100, 8138, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63948 payload_rows_remote=55756 dup_rows=1588 dup_rows_remote=1582 dedup_ratio=0.024231 router_tokens_per_expert=[0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4094, 0, 2, 4060, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[6956, 8100, 8016, 8190, 8182, 8124, 8188, 8192] dup_rows_per_dst=[242, 4, 686, 12, 72, 486, 80, 6] combine_rows_per_src=[6956, 8100, 8016, 8190, 8182, 8124, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65300 payload_rows_remote=57110 dup_rows=236 dup_rows_remote=36 dedup_ratio=0.003601 router_tokens_per_expert=[0, 0, 0, 0, 77, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 4045, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1] payload_rows_per_dst=[8190, 8150, 8134, 8192, 8190, 8094, 8190, 8160] dup_rows_per_dst=[200, 0, 2, 0, 0, 32, 0, 2] combine_rows_per_src=[8190, 8150, 8134, 8192, 8190, 8094, 8190, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233979904 dispatch_meta_bytes=229528 combine_payload_bytes=233979904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234209432 ingress_bytes=271728640 link_bytes=271728640 token_bytes=468189336 weight_movement_bytes=37748736 total_nvlink_bytes=505938072 weight_to_token_ratio=0.080627 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603570130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229464 combine_payload_bytes=234020864 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234250328 ingress_bytes=271769600 link_bytes=271769600 token_bytes=468271192 weight_movement_bytes=37748736 total_nvlink_bytes=506019928 weight_to_token_ratio=0.080613 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603962588e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229464 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234897496 ingress_bytes=253542400 link_bytes=253542400 token_bytes=469565528 weight_movement_bytes=18874368 total_nvlink_bytes=488439896 weight_to_token_ratio=0.040195 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429318526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65096 payload_rows_remote=56906 dup_rows=440 dup_rows_remote=440 dedup_ratio=0.006714 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 4049, 0, 0, 0, 0, 0, 239, 25, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8186, 7766, 8192, 8186, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 438] combine_rows_per_src=[8192, 8186, 7766, 8192, 8186, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233283584 dispatch_meta_bytes=228944 combine_payload_bytes=233283584 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233512528 ingress_bytes=289906688 link_bytes=289906688 token_bytes=466796112 weight_movement_bytes=56623104 total_nvlink_bytes=523419216 weight_to_token_ratio=0.121302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.777743241e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=228376576 dispatch_meta_bytes=229352 combine_payload_bytes=228376576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228605928 ingress_bytes=247250944 link_bytes=247250944 token_bytes=456982504 weight_movement_bytes=18874368 total_nvlink_bytes=475856872 weight_to_token_ratio=0.041302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.369036890e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63492 payload_rows_remote=55300 dup_rows=2044 dup_rows_remote=2044 dedup_ratio=0.031189 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1008, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6164, 8192, 8176] dup_rows_per_dst=[0, 28, 0, 0, 0, 0, 0, 2016] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6164, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=228584 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234151144 ingress_bytes=328294400 link_bytes=328294400 token_bytes=468073704 weight_movement_bytes=94371840 total_nvlink_bytes=562445544 weight_to_token_ratio=0.201617 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.145555410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=233086976 dispatch_meta_bytes=229384 combine_payload_bytes=233086976 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233316360 ingress_bytes=270835712 link_bytes=270835712 token_bytes=466403336 weight_movement_bytes=37748736 total_nvlink_bytes=504152072 weight_to_token_ratio=0.080936 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595014533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=226508800 dispatch_meta_bytes=229376 combine_payload_bytes=226508800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226738176 ingress_bytes=264257536 link_bytes=264257536 token_bytes=453246976 weight_movement_bytes=37748736 total_nvlink_bytes=490995712 weight_to_token_ratio=0.083285 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.531985687e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62022 payload_rows_remote=53832 dup_rows=3514 dup_rows_remote=3498 dedup_ratio=0.053619 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4089, 4090, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 1, 0, 4089, 0, 0, 0, 1, 1, 0, 0, 5, 0, 0, 0, 6, 0, 5, 0, 0, 0, 0, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 1, 4, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1747, 0, 0, 0] payload_rows_per_dst=[8190, 8182, 8192, 8180, 8190, 4704, 8192, 8192] dup_rows_per_dst=[6, 0, 2, 2, 16, 0, 4, 3484] combine_rows_per_src=[8190, 8182, 8192, 8180, 8190, 4704, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64074 payload_rows_remote=57276 dup_rows=1462 dup_rows_remote=1460 dedup_ratio=0.022308 router_tokens_per_expert=[0, 4, 0, 0, 0, 1, 4089, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 8, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4088, 4090, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 27, 0, 0, 0, 4090, 0, 6, 0, 0, 1, 0, 0, 5, 0, 0, 0, 1, 1, 5, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 4, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 676, 0, 2, 0] payload_rows_per_dst=[8192, 8184, 8184, 8192, 8192, 6798, 8188, 8144] dup_rows_per_dst=[10, 16, 0, 60, 14, 2, 6, 1354] combine_rows_per_src=[8192, 8184, 8184, 8192, 8192, 6798, 8188, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63090 payload_rows_remote=54898 dup_rows=2446 dup_rows_remote=2446 dedup_ratio=0.037323 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 808, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6742, 8192, 7196] dup_rows_per_dst=[0, 0, 0, 0, 0, 996, 0, 1450] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6742, 8192, 7196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59124 payload_rows_remote=50982 dup_rows=6412 dup_rows_remote=6280 dedup_ratio=0.097839 router_tokens_per_expert=[0, 23, 0, 0, 0, 3, 4066, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 67, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 2, 4068, 4069, 0, 0, 0, 0, 0, 0, 0, 6, 0, 19, 0, 0, 0, 0, 0, 0, 4070, 0, 12, 0, 0, 0, 4070, 0, 0, 0, 0, 22, 0, 0, 24, 0, 0, 0, 5, 1, 25, 0, 0, 0, 0, 0, 0, 3140, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 9, 0, 0, 5, 0, 0, 3, 15, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 938, 0, 4, 0] payload_rows_per_dst=[8188, 8142, 8182, 8186, 8190, 8164, 8156, 1916] dup_rows_per_dst=[44, 132, 2, 28, 52, 6138, 12, 4] combine_rows_per_src=[8188, 8142, 8182, 8186, 8190, 8164, 8156, 1916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61962 payload_rows_remote=53772 dup_rows=3574 dup_rows_remote=148 dedup_ratio=0.054535 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 2522, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1572, 0, 0, 0] payload_rows_per_dst=[8132, 8082, 8012, 8118, 8192, 5048, 8188, 8190] dup_rows_per_dst=[4, 0, 0, 0, 116, 4, 24, 3426] combine_rows_per_src=[8132, 8082, 8012, 8118, 8192, 5048, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=220495872 dispatch_meta_bytes=229320 combine_payload_bytes=220495872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220725192 ingress_bytes=258244608 link_bytes=258244608 token_bytes=441221064 weight_movement_bytes=37748736 total_nvlink_bytes=478969800 weight_to_token_ratio=0.085555 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.474372770e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59304 payload_rows_remote=51112 dup_rows=6232 dup_rows_remote=6230 dedup_ratio=0.095093 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 1078, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 7998, 8192, 8190, 2158] dup_rows_per_dst=[2, 0, 194, 2, 0, 6034, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 7998, 8192, 8190, 2158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=224862208 dispatch_meta_bytes=229376 combine_payload_bytes=224862208 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225091584 ingress_bytes=281485312 link_bytes=281485312 token_bytes=449953792 weight_movement_bytes=56623104 total_nvlink_bytes=506576896 weight_to_token_ratio=0.125842 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.697053760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=234944 combine_payload_bytes=234602496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234837440 ingress_bytes=253476864 link_bytes=253476864 token_bytes=469439936 weight_movement_bytes=18874368 total_nvlink_bytes=488314304 weight_to_token_ratio=0.040206 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428690592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58690 payload_rows_remote=50752 dup_rows=6846 dup_rows_remote=4924 dedup_ratio=0.104462 router_tokens_per_expert=[0, 0, 0, 0, 0, 42, 4092, 0, 0, 0, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 4096, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 0, 0, 4096, 83, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1610, 8192, 8184, 8192, 8192, 8190, 7938, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 4740, 1922, 180] combine_rows_per_src=[1610, 8192, 8184, 8192, 8192, 8190, 7938, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=208822272 dispatch_meta_bytes=229048 combine_payload_bytes=208822272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209051320 ingress_bytes=227696640 link_bytes=227696640 token_bytes=417873592 weight_movement_bytes=18874368 total_nvlink_bytes=436747960 weight_to_token_ratio=0.045168 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.181677171e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64872 payload_rows_remote=56680 dup_rows=664 dup_rows_remote=418 dedup_ratio=0.010132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3764, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7528, 8192] dup_rows_per_dst=[0, 0, 0, 246, 0, 418, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7528, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=220250112 dispatch_meta_bytes=215680 combine_payload_bytes=220250112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220465792 ingress_bytes=257998848 link_bytes=257998848 token_bytes=440715904 weight_movement_bytes=37748736 total_nvlink_bytes=478464640 weight_to_token_ratio=0.085653 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.472018018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=209354752 dispatch_meta_bytes=229368 combine_payload_bytes=209354752 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=209584120 ingress_bytes=247103488 link_bytes=247103488 token_bytes=418938872 weight_movement_bytes=37748736 total_nvlink_bytes=456687608 weight_to_token_ratio=0.090106 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.367624040e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=207880192 dispatch_meta_bytes=222704 combine_payload_bytes=207880192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208102896 ingress_bytes=226754560 link_bytes=226754560 token_bytes=415983088 weight_movement_bytes=18874368 total_nvlink_bytes=434857456 weight_to_token_ratio=0.045373 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.172650624e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232161280 dispatch_meta_bytes=228392 combine_payload_bytes=232161280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232389672 ingress_bytes=251035648 link_bytes=251035648 token_bytes=464550952 weight_movement_bytes=18874368 total_nvlink_bytes=483425320 weight_to_token_ratio=0.040629 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405300062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57302 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.001465 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 5, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 1, 4063, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 20, 0, 0, 2, 0, 6, 1, 0, 0, 5, 0, 0, 4089, 4079, 4090, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 4, 0] payload_rows_per_dst=[8180, 8190, 8186, 8192, 8138, 8182, 8180, 8192] dup_rows_per_dst=[4, 22, 2, 8, 0, 40, 10, 10] combine_rows_per_src=[8180, 8190, 8186, 8192, 8138, 8182, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 8, 0, 2, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65444 payload_rows_remote=57254 dup_rows=92 dup_rows_remote=80 dedup_ratio=0.001404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4089, 0, 0, 0, 4069, 4084, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 4090, 0, 5, 1, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 4081, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8186, 8184, 8192, 8148, 8190, 8174, 8190] dup_rows_per_dst=[26, 26, 0, 6, 2, 12, 20, 0] combine_rows_per_src=[8180, 8186, 8184, 8192, 8148, 8190, 8174, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65322 payload_rows_remote=57142 dup_rows=214 dup_rows_remote=212 dedup_ratio=0.003265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 26, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 19, 12, 0, 0, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 17, 0, 0, 0, 0, 16, 4069, 0, 0, 0, 4068, 0, 0, 0, 4024, 4057, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 4067, 8, 24, 0, 0, 0, 5, 0, 0, 0, 25, 0, 0, 4066, 4069, 4069, 1, 0, 0, 1, 0, 1, 6, 0, 0, 0, 0, 2, 0, 25, 0, 0, 1, 0] payload_rows_per_dst=[8148, 8180, 8156, 8190, 8086, 8180, 8190, 8192] dup_rows_per_dst=[2, 2, 0, 90, 28, 16, 58, 18] combine_rows_per_src=[8148, 8180, 8156, 8190, 8086, 8180, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64932 payload_rows_remote=56740 dup_rows=604 dup_rows_remote=604 dedup_ratio=0.009216 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 2, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8134, 8188, 7836, 8184, 8020, 8190, 8192] dup_rows_per_dst=[0, 4, 2, 4, 58, 2, 534, 0] combine_rows_per_src=[8188, 8134, 8188, 7836, 8184, 8020, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229592 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=234938584 ingress_bytes=347955200 link_bytes=347955200 token_bytes=469647576 weight_movement_bytes=113246208 total_nvlink_bytes=582893784 weight_to_token_ratio=0.241130 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.333935522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65204 payload_rows_remote=57014 dup_rows=332 dup_rows_remote=290 dedup_ratio=0.005066 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 4069, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4095, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 7900, 8190, 8156] dup_rows_per_dst=[42, 16, 50, 2, 154, 2, 66, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 7900, 8190, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893120 weight_movement_bytes=37748736 total_nvlink_bytes=507641856 weight_to_token_ratio=0.080335 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229336 combine_payload_bytes=234512384 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234741720 ingress_bytes=272261120 link_bytes=272261120 token_bytes=469254104 weight_movement_bytes=37748736 total_nvlink_bytes=507002840 weight_to_token_ratio=0.080444 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608672091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60094 payload_rows_remote=51902 dup_rows=5442 dup_rows_remote=5442 dedup_ratio=0.083038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 8192, 8192, 8192, 7998, 4562, 6606] dup_rows_per_dst=[0, 1080, 0, 0, 0, 0, 1734, 2628] combine_rows_per_src=[8192, 8160, 8192, 8192, 8192, 7998, 4562, 6606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234053632 dispatch_meta_bytes=229416 combine_payload_bytes=234053632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234283048 ingress_bytes=252928000 link_bytes=252928000 token_bytes=468336680 weight_movement_bytes=18874368 total_nvlink_bytes=487211048 weight_to_token_ratio=0.040301 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423431648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8192, 8192, 8182, 8192, 8192] dup_rows_per_dst=[30, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8172, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232407040 dispatch_meta_bytes=229376 combine_payload_bytes=232407040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232636416 ingress_bytes=251281408 link_bytes=251281408 token_bytes=465043456 weight_movement_bytes=18874368 total_nvlink_bytes=483917824 weight_to_token_ratio=0.040586 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407654813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233529344 dispatch_meta_bytes=229216 combine_payload_bytes=233529344 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233758560 ingress_bytes=271278080 link_bytes=271278080 token_bytes=467287904 weight_movement_bytes=37748736 total_nvlink_bytes=505036640 weight_to_token_ratio=0.080783 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599253085e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=212590592 dispatch_meta_bytes=229376 combine_payload_bytes=212590592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212819968 ingress_bytes=231464960 link_bytes=231464960 token_bytes=425410560 weight_movement_bytes=18874368 total_nvlink_bytes=444284928 weight_to_token_ratio=0.044367 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217783359e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229376 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234987520 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469745664 weight_movement_bytes=18874368 total_nvlink_bytes=488620032 weight_to_token_ratio=0.040180 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61856 payload_rows_remote=53664 dup_rows=3680 dup_rows_remote=3678 dedup_ratio=0.056152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 393, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 1, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2367, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4053, 4089, 0, 0, 1484, 0, 0, 0, 0, 5, 0, 6, 0, 1, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8012, 8176, 8182, 8192, 8190, 4736, 8180] dup_rows_per_dst=[0, 720, 0, 10, 2, 10, 10, 2928] combine_rows_per_src=[8188, 8012, 8176, 8182, 8192, 8190, 4736, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60952 payload_rows_remote=52760 dup_rows=4584 dup_rows_remote=4582 dedup_ratio=0.069946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 4096, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8130, 8192, 8192, 8192, 8192, 8192, 3670] dup_rows_per_dst=[0, 1838, 2, 0, 0, 0, 2744, 0] combine_rows_per_src=[8192, 8130, 8192, 8192, 8192, 8192, 8192, 3670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62140 payload_rows_remote=53950 dup_rows=3396 dup_rows_remote=3382 dedup_ratio=0.051819 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 3907, 1, 0, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 6, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3942, 4088, 0, 0, 966, 0, 0, 0, 0, 6, 1, 6, 0, 0, 5, 0, 0, 0, 3, 1] payload_rows_per_dst=[8188, 7884, 8172, 8180, 8190, 8190, 5204, 8132] dup_rows_per_dst=[4, 1578, 0, 4, 10, 14, 58, 1728] combine_rows_per_src=[8188, 7884, 8172, 8180, 8190, 8190, 5204, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59848 payload_rows_remote=52794 dup_rows=5688 dup_rows_remote=4546 dedup_ratio=0.086792 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 17, 0, 0, 24, 0, 0, 0, 2, 0, 0, 0, 0, 3364, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 5, 0, 0, 0, 2, 26, 0, 0, 0, 0, 25, 0, 0, 0, 4048, 0, 0, 0, 1, 4, 4069, 0, 0, 0, 0, 25, 0, 0, 1, 0, 26, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3961, 4056, 0, 0, 1940, 0, 0, 0, 0, 26, 2, 26, 0, 0, 15, 0, 0, 0, 0, 18] payload_rows_per_dst=[8174, 7054, 8138, 8138, 8152, 8192, 8068, 3932] dup_rows_per_dst=[26, 1142, 10, 10, 52, 58, 4268, 122] combine_rows_per_src=[8174, 7054, 8138, 8138, 8152, 8192, 8068, 3932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59300 payload_rows_remote=51168 dup_rows=6236 dup_rows_remote=2552 dedup_ratio=0.095154 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4012, 4094, 0, 0, 1891, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8178, 2054, 8186, 8182, 8192, 8192, 8184, 8132] dup_rows_per_dst=[2, 0, 0, 0, 4, 8, 2538, 3684] combine_rows_per_src=[8178, 2054, 8186, 8182, 8192, 8192, 8184, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=219807744 dispatch_meta_bytes=229368 combine_payload_bytes=219807744 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=220037112 ingress_bytes=238682112 link_bytes=238682112 token_bytes=439844856 weight_movement_bytes=18874368 total_nvlink_bytes=458719224 weight_to_token_ratio=0.042911 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286934558e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61664 payload_rows_remote=53472 dup_rows=3872 dup_rows_remote=3864 dedup_ratio=0.059082 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4095, 0, 0, 2205, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 8192, 8164, 8186, 8140, 4412] dup_rows_per_dst=[8, 2842, 0, 2, 2, 2, 1016, 0] combine_rows_per_src=[8192, 8188, 8190, 8192, 8164, 8186, 8140, 4412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=216104960 dispatch_meta_bytes=229368 combine_payload_bytes=216104960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216334328 ingress_bytes=234979328 link_bytes=234979328 token_bytes=432439288 weight_movement_bytes=18874368 total_nvlink_bytes=451313656 weight_to_token_ratio=0.043646 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.251456304e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220979200 dispatch_meta_bytes=229328 combine_payload_bytes=220979200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221208528 ingress_bytes=239853568 link_bytes=239853568 token_bytes=442187728 weight_movement_bytes=18874368 total_nvlink_bytes=461062096 weight_to_token_ratio=0.042684 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.298158873e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57130 dup_rows=218 dup_rows_remote=218 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 7978, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 218] combine_rows_per_src=[8192, 8192, 8192, 8188, 7978, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=216244224 dispatch_meta_bytes=229360 combine_payload_bytes=216244224 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=216473584 ingress_bytes=310616064 link_bytes=310616064 token_bytes=432717808 weight_movement_bytes=94371840 total_nvlink_bytes=527089648 weight_to_token_ratio=0.218091 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.976170293e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=209584128 dispatch_meta_bytes=214880 combine_payload_bytes=209584128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209799008 ingress_bytes=228458496 link_bytes=228458496 token_bytes=419383136 weight_movement_bytes=18874368 total_nvlink_bytes=438257504 weight_to_token_ratio=0.045005 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.188976900e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63894 payload_rows_remote=55702 dup_rows=1642 dup_rows_remote=1642 dedup_ratio=0.025055 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 819, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6550, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 1638] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6550, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=219021312 dispatch_meta_bytes=229344 combine_payload_bytes=219021312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219250656 ingress_bytes=237895680 link_bytes=237895680 token_bytes=438271968 weight_movement_bytes=18874368 total_nvlink_bytes=457146336 weight_to_token_ratio=0.043065 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.279399354e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234004480 dispatch_meta_bytes=229392 combine_payload_bytes=234004480 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234233872 ingress_bytes=290627584 link_bytes=290627584 token_bytes=468238352 weight_movement_bytes=56623104 total_nvlink_bytes=524861456 weight_to_token_ratio=0.120928 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784650512e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57272 dup_rows=78 dup_rows_remote=56 dedup_ratio=0.001190 router_tokens_per_expert=[0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 4089, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 7, 0, 0, 6, 0, 0, 0, 0, 0, 6, 4086, 0, 1, 5, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4078, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8186, 8192, 8180, 8186, 8174, 8168, 8182] dup_rows_per_dst=[12, 4, 16, 14, 22, 0, 0, 10] combine_rows_per_src=[8190, 8186, 8192, 8180, 8186, 8174, 8168, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=228155392 dispatch_meta_bytes=229376 combine_payload_bytes=228155392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228384768 ingress_bytes=247029760 link_bytes=247029760 token_bytes=456540160 weight_movement_bytes=18874368 total_nvlink_bytes=475414528 weight_to_token_ratio=0.041342 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.366917614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8188, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65422 payload_rows_remote=57248 dup_rows=114 dup_rows_remote=114 dedup_ratio=0.001740 router_tokens_per_expert=[0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 0, 0, 1, 4089, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4088, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4055, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8180, 8190, 8174, 8122, 8182] dup_rows_per_dst=[12, 6, 16, 0, 10, 0, 0, 70] combine_rows_per_src=[8190, 8192, 8192, 8180, 8190, 8174, 8122, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65200 payload_rows_remote=57010 dup_rows=336 dup_rows_remote=324 dedup_ratio=0.005127 router_tokens_per_expert=[0, 0, 26, 0, 0, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 4069, 0, 0, 24, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 18, 0, 0, 0, 0, 15, 0, 0, 16, 4068, 0, 4064, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 4069, 0, 0, 0, 1, 0, 0, 0, 0, 0, 26, 3970, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 25, 8, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4057, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8138, 7992, 8190, 8168, 8140] dup_rows_per_dst=[52, 12, 52, 0, 22, 6, 2, 190] combine_rows_per_src=[8190, 8190, 8192, 8138, 7992, 8190, 8168, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229312 combine_payload_bytes=234586112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234815424 ingress_bytes=253460480 link_bytes=253460480 token_bytes=469401536 weight_movement_bytes=18874368 total_nvlink_bytes=488275904 weight_to_token_ratio=0.040209 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428533609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64938 payload_rows_remote=56750 dup_rows=598 dup_rows_remote=584 dedup_ratio=0.009125 router_tokens_per_expert=[0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8168, 8094, 7818, 8184, 8190, 8178, 8188] dup_rows_per_dst=[4, 2, 12, 0, 394, 32, 140, 14] combine_rows_per_src=[8118, 8168, 8094, 7818, 8184, 8190, 8178, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235077632 ingress_bytes=310345728 link_bytes=310345728 token_bytes=469925888 weight_movement_bytes=75497472 total_nvlink_bytes=545423360 weight_to_token_ratio=0.160658 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973580066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65084 payload_rows_remote=56892 dup_rows=452 dup_rows_remote=452 dedup_ratio=0.006897 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8176, 8192, 8192, 7760] dup_rows_per_dst=[0, 2, 2, 444, 2, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8176, 8192, 8192, 7760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63112 payload_rows_remote=54920 dup_rows=2424 dup_rows_remote=2416 dedup_ratio=0.036987 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 5896, 8070, 8192] dup_rows_per_dst=[0, 0, 272, 8, 0, 0, 2144, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 5896, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234487808 dispatch_meta_bytes=229448 combine_payload_bytes=234487808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234717256 ingress_bytes=253362176 link_bytes=253362176 token_bytes=469205064 weight_movement_bytes=18874368 total_nvlink_bytes=488079432 weight_to_token_ratio=0.040226 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427591708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233512960 dispatch_meta_bytes=229336 combine_payload_bytes=233512960 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233742296 ingress_bytes=271261696 link_bytes=271261696 token_bytes=467255256 weight_movement_bytes=37748736 total_nvlink_bytes=505003992 weight_to_token_ratio=0.080788 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599096102e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232448000 dispatch_meta_bytes=229336 combine_payload_bytes=232448000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232677336 ingress_bytes=251322368 link_bytes=251322368 token_bytes=465125336 weight_movement_bytes=18874368 total_nvlink_bytes=483999704 weight_to_token_ratio=0.040579 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408047272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=8 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8174, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 12, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 8174, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233029632 dispatch_meta_bytes=229376 combine_payload_bytes=233029632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233259008 ingress_bytes=251904000 link_bytes=251904000 token_bytes=466288640 weight_movement_bytes=18874368 total_nvlink_bytes=485163008 weight_to_token_ratio=0.040478 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413620183e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224952320 dispatch_meta_bytes=229344 combine_payload_bytes=224952320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225181664 ingress_bytes=281575424 link_bytes=281575424 token_bytes=450133984 weight_movement_bytes=56623104 total_nvlink_bytes=506757088 weight_to_token_ratio=0.125792 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.697917169e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64400 payload_rows_remote=56208 dup_rows=1136 dup_rows_remote=1116 dedup_ratio=0.017334 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 13, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 4090, 0, 5, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 3597, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 580, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 4090, 0, 0] payload_rows_per_dst=[8180, 8192, 8180, 8178, 8192, 7204, 8082, 8192] dup_rows_per_dst=[0, 30, 328, 2, 20, 0, 756, 0] combine_rows_per_src=[8180, 8192, 8180, 8178, 8192, 7204, 8082, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229328 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235028432 ingress_bytes=291422208 link_bytes=291422208 token_bytes=469827536 weight_movement_bytes=56623104 total_nvlink_bytes=526450640 weight_to_token_ratio=0.120519 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792264208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63336 payload_rows_remote=55144 dup_rows=2200 dup_rows_remote=1222 dedup_ratio=0.033569 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6250, 7934, 8192] dup_rows_per_dst=[0, 0, 978, 2, 0, 0, 1220, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6250, 7934, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64472 payload_rows_remote=57066 dup_rows=1064 dup_rows_remote=1064 dedup_ratio=0.016235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 2, 0, 15, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 3, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 3697, 6, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 483, 6, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 0, 0] payload_rows_per_dst=[8184, 8192, 8180, 8168, 8192, 7406, 7960, 8190] dup_rows_per_dst=[0, 32, 398, 0, 12, 0, 616, 6] combine_rows_per_src=[8184, 8192, 8180, 8168, 8192, 7406, 7960, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62318 payload_rows_remote=54126 dup_rows=3218 dup_rows_remote=3104 dedup_ratio=0.049103 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 0, 1, 24, 4, 0, 54, 1, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 3, 0, 0, 0, 4058, 0, 0, 0, 0, 9, 0, 1, 5, 0, 25, 0, 3, 4069, 0, 0, 1, 2, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 3182, 24, 14, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 1627, 25, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 25, 0, 4069, 0, 1] payload_rows_per_dst=[8144, 8192, 8142, 8116, 8192, 6576, 6766, 8190] dup_rows_per_dst=[2, 114, 598, 6, 38, 1100, 1358, 2] combine_rows_per_src=[8144, 8192, 8142, 8116, 8192, 6576, 6766, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=230227968 dispatch_meta_bytes=229296 combine_payload_bytes=230227968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230457264 ingress_bytes=267976704 link_bytes=267976704 token_bytes=460685232 weight_movement_bytes=37748736 total_nvlink_bytes=498433968 weight_to_token_ratio=0.081940 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.567620925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60078 payload_rows_remote=51886 dup_rows=5458 dup_rows_remote=5458 dedup_ratio=0.083282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 3591, 2, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 363, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0] payload_rows_per_dst=[8188, 7928, 4262, 8016, 8192, 7690, 7610, 8192] dup_rows_per_dst=[0, 0, 834, 274, 2, 3932, 416, 0] combine_rows_per_src=[8188, 7928, 4262, 8016, 8192, 7690, 7610, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=225869824 dispatch_meta_bytes=225464 combine_payload_bytes=225869824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226095288 ingress_bytes=244744192 link_bytes=244744192 token_bytes=451965112 weight_movement_bytes=18874368 total_nvlink_bytes=470839480 weight_to_token_ratio=0.041761 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.345018426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60766 payload_rows_remote=52574 dup_rows=4770 dup_rows_remote=4700 dedup_ratio=0.072784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 2689, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8124, 8134, 3554, 8188] dup_rows_per_dst=[70, 86, 194, 0, 2, 4416, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8124, 8134, 3554, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8192, 8058, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[100, 0, 0, 0, 0, 38, 30, 0] combine_rows_per_src=[8158, 8192, 8058, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233742336 dispatch_meta_bytes=232520 combine_payload_bytes=233742336 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233974856 ingress_bytes=309239808 link_bytes=309239808 token_bytes=467717192 weight_movement_bytes=75497472 total_nvlink_bytes=543214664 weight_to_token_ratio=0.161417 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.962983685e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=221700096 dispatch_meta_bytes=228920 combine_payload_bytes=221700096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=221929016 ingress_bytes=259448832 link_bytes=259448832 token_bytes=443629112 weight_movement_bytes=37748736 total_nvlink_bytes=481377848 weight_to_token_ratio=0.085091 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.485911051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59608 payload_rows_remote=54614 dup_rows=5928 dup_rows_remote=4840 dedup_ratio=0.090454 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5486, 8192, 8192, 8192, 8192, 4994, 8168] dup_rows_per_dst=[0, 1368, 86, 60, 0, 112, 1088, 3214] combine_rows_per_src=[8192, 5486, 8192, 8192, 8192, 8192, 4994, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=212525056 dispatch_meta_bytes=229376 combine_payload_bytes=212525056 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=212754432 ingress_bytes=231399424 link_bytes=231399424 token_bytes=425279488 weight_movement_bytes=18874368 total_nvlink_bytes=444153856 weight_to_token_ratio=0.044381 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217155425e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215343104 dispatch_meta_bytes=229096 combine_payload_bytes=215343104 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=215572200 ingress_bytes=253091840 link_bytes=253091840 token_bytes=430915304 weight_movement_bytes=37748736 total_nvlink_bytes=468664040 weight_to_token_ratio=0.087601 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425001482e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234422272 ingress_bytes=271941632 link_bytes=271941632 token_bytes=468615168 weight_movement_bytes=37748736 total_nvlink_bytes=506363904 weight_to_token_ratio=0.080554 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605610914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57258 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.001465 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4087, 4085, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 5, 0, 0, 0, 0, 4090, 4087, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8132, 8186, 8190, 8182, 8192, 8190, 8186] dup_rows_per_dst=[8, 0, 12, 12, 0, 64, 0, 0] combine_rows_per_src=[8182, 8132, 8186, 8190, 8182, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=223698944 dispatch_meta_bytes=237816 combine_payload_bytes=223698944 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=223936760 ingress_bytes=280322048 link_bytes=280322048 token_bytes=447635704 weight_movement_bytes=56623104 total_nvlink_bytes=504258808 weight_to_token_ratio=0.126494 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=13568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.685907937e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57344 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8142, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[44, 0, 0, 4, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8142, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=106 dedup_ratio=0.001770 router_tokens_per_expert=[0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 6, 0, 0, 0, 0, 4090, 4054, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 4090, 0, 1, 3, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 34, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8170, 8122, 8192, 8190, 8192, 8192, 8182] dup_rows_per_dst=[16, 0, 0, 16, 0, 10, 74, 0] combine_rows_per_src=[8180, 8170, 8122, 8192, 8190, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65336 payload_rows_remote=57180 dup_rows=200 dup_rows_remote=194 dedup_ratio=0.003052 router_tokens_per_expert=[0, 4063, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4064, 4054, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 26, 0, 0, 0, 0, 4069, 4065, 2, 27, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4068, 0, 2, 8, 0, 0, 0, 0, 23, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 0, 28, 0, 0, 0, 0, 25, 1, 26, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8132, 8156, 8184, 8192, 8162, 8180, 8192, 8138] dup_rows_per_dst=[0, 6, 16, 48, 0, 24, 106, 0] combine_rows_per_src=[8132, 8156, 8184, 8192, 8162, 8180, 8192, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229416 combine_payload_bytes=234528768 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234758184 ingress_bytes=272277504 link_bytes=272277504 token_bytes=469286952 weight_movement_bytes=37748736 total_nvlink_bytes=507035688 weight_to_token_ratio=0.080438 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608829074e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229576 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110600 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991624 weight_movement_bytes=37748736 total_nvlink_bytes=507740360 weight_to_token_ratio=0.080318 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57128 dup_rows=218 dup_rows_remote=146 dedup_ratio=0.003326 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 4094, 4090, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 4094, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8062, 8178, 8192, 8184, 8192, 8192, 8190] dup_rows_per_dst=[14, 2, 0, 38, 54, 4, 34, 72] combine_rows_per_src=[8128, 8062, 8178, 8192, 8184, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65028 payload_rows_remote=56838 dup_rows=508 dup_rows_remote=508 dedup_ratio=0.007751 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8188, 8072, 8188, 8056, 7950, 8192] dup_rows_per_dst=[0, 240, 258, 4, 0, 6, 0, 0] combine_rows_per_src=[8190, 8192, 8188, 8072, 8188, 8056, 7950, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=229336 combine_payload_bytes=234405888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234635224 ingress_bytes=253280256 link_bytes=253280256 token_bytes=469041112 weight_movement_bytes=18874368 total_nvlink_bytes=487915480 weight_to_token_ratio=0.040240 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426806791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=229496 combine_payload_bytes=234209280 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234438776 ingress_bytes=328581120 link_bytes=328581120 token_bytes=468648056 weight_movement_bytes=94371840 total_nvlink_bytes=563019896 weight_to_token_ratio=0.201370 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.148302620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57344 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229096 combine_payload_bytes=233996288 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234225384 ingress_bytes=271745024 link_bytes=271745024 token_bytes=468221672 weight_movement_bytes=37748736 total_nvlink_bytes=505970408 weight_to_token_ratio=0.080622 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603727113e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232808448 dispatch_meta_bytes=229384 combine_payload_bytes=232808448 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233037832 ingress_bytes=270557184 link_bytes=270557184 token_bytes=465846280 weight_movement_bytes=37748736 total_nvlink_bytes=503595016 weight_to_token_ratio=0.081033 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592345815e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110408 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991432 weight_movement_bytes=56623104 total_nvlink_bytes=526614536 weight_to_token_ratio=0.120477 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229400 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110424 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991448 weight_movement_bytes=18874368 total_nvlink_bytes=488865816 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57292 dup_rows=64 dup_rows_remote=62 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 8, 0, 5, 0, 4087, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 4, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 0, 1, 0, 6, 11, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8184, 8160, 8192, 8180, 8190, 8190, 8184] dup_rows_per_dst=[20, 0, 0, 16, 2, 4, 20, 2] combine_rows_per_src=[8192, 8184, 8160, 8192, 8180, 8190, 8190, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65380 payload_rows_remote=57188 dup_rows=156 dup_rows_remote=148 dedup_ratio=0.002380 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 1, 0, 5, 0, 4084, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 4044, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 1, 0, 4079, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 2, 52, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8102, 8170, 8180, 8192, 8184, 8182] dup_rows_per_dst=[16, 6, 0, 14, 12, 8, 100, 0] combine_rows_per_src=[8192, 8178, 8102, 8170, 8180, 8192, 8184, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229416 combine_payload_bytes=234668032 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234897448 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469565480 weight_movement_bytes=37748736 total_nvlink_bytes=507314216 weight_to_token_ratio=0.080391 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229344 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235077600 ingress_bytes=272596992 link_bytes=272596992 token_bytes=469925856 weight_movement_bytes=37748736 total_nvlink_bytes=507674592 weight_to_token_ratio=0.080329 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611890251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65180 payload_rows_remote=57020 dup_rows=356 dup_rows_remote=312 dedup_ratio=0.005432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 26, 0, 0, 27, 0, 0, 2, 0, 25, 0, 4053, 0, 0, 0, 0, 0, 0, 4068, 0, 22, 2, 0, 0, 0, 0, 0, 25, 4063, 0, 0, 0, 0, 0, 3998, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4067, 0, 0, 4047, 0, 0, 26, 0, 0, 2, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 8, 1, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4, 0, 0, 0, 4, 0, 4, 81, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0] payload_rows_per_dst=[8182, 8160, 8076, 8146, 8140, 8192, 8144, 8140] dup_rows_per_dst=[60, 44, 0, 52, 26, 4, 168, 2] combine_rows_per_src=[8182, 8160, 8076, 8146, 8140, 8192, 8144, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65292 payload_rows_remote=57104 dup_rows=244 dup_rows_remote=176 dedup_ratio=0.003723 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 8120, 8176, 8154, 8188, 8178, 8192, 8188] dup_rows_per_dst=[8, 0, 96, 4, 38, 0, 30, 68] combine_rows_per_src=[8096, 8120, 8176, 8154, 8188, 8178, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65262 payload_rows_remote=57070 dup_rows=274 dup_rows_remote=272 dedup_ratio=0.004181 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4088, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8188, 8186, 8192, 8192, 7930, 8192] dup_rows_per_dst=[2, 78, 146, 44, 2, 0, 0, 2] combine_rows_per_src=[8192, 8190, 8188, 8186, 8192, 8192, 7930, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58312 payload_rows_remote=50636 dup_rows=7224 dup_rows_remote=7224 dedup_ratio=0.110229 router_tokens_per_expert=[0, 0, 0, 0, 3376, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 239, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7676, 8100, 1584, 8192, 8184] dup_rows_per_dst=[6752, 0, 0, 0, 472, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7676, 8100, 1584, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234242048 dispatch_meta_bytes=229344 combine_payload_bytes=234242048 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234471392 ingress_bytes=253116416 link_bytes=253116416 token_bytes=468713440 weight_movement_bytes=18874368 total_nvlink_bytes=487587808 weight_to_token_ratio=0.040268 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425236957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64062 payload_rows_remote=56084 dup_rows=1474 dup_rows_remote=404 dedup_ratio=0.022491 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6932, 7978, 8192] dup_rows_per_dst=[0, 2, 402, 0, 0, 0, 1070, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6932, 7978, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233553920 dispatch_meta_bytes=229328 combine_payload_bytes=233553920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233783248 ingress_bytes=290177024 link_bytes=290177024 token_bytes=467337168 weight_movement_bytes=56623104 total_nvlink_bytes=523960272 weight_to_token_ratio=0.121161 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780333468e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233897984 dispatch_meta_bytes=229120 combine_payload_bytes=233897984 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234127104 ingress_bytes=271646720 link_bytes=271646720 token_bytes=468025088 weight_movement_bytes=37748736 total_nvlink_bytes=505773824 weight_to_token_ratio=0.080655 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602785213e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233758720 dispatch_meta_bytes=229368 combine_payload_bytes=233758720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233988088 ingress_bytes=252633088 link_bytes=252633088 token_bytes=467746808 weight_movement_bytes=18874368 total_nvlink_bytes=486621176 weight_to_token_ratio=0.040352 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420605946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=207405056 dispatch_meta_bytes=231440 combine_payload_bytes=207405056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207636496 ingress_bytes=226279424 link_bytes=226279424 token_bytes=415041552 weight_movement_bytes=18874368 total_nvlink_bytes=433915920 weight_to_token_ratio=0.045476 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.168098104e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62980 payload_rows_remote=54822 dup_rows=2556 dup_rows_remote=2340 dedup_ratio=0.039001 router_tokens_per_expert=[0, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4090, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 34, 0, 0, 1, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1185, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5832, 8180, 8192, 8190, 8158, 8056, 8192, 8180] dup_rows_per_dst=[16, 0, 6, 0, 216, 2314, 4, 0] combine_rows_per_src=[5832, 8180, 8192, 8190, 8158, 8056, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229720064 dispatch_meta_bytes=225952 combine_payload_bytes=229720064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229946016 ingress_bytes=248594432 link_bytes=248594432 token_bytes=459666080 weight_movement_bytes=18874368 total_nvlink_bytes=478540448 weight_to_token_ratio=0.041061 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381909531e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60384 payload_rows_remote=52192 dup_rows=5152 dup_rows_remote=5152 dedup_ratio=0.078613 router_tokens_per_expert=[0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 3044, 8192, 8192] dup_rows_per_dst=[5146, 0, 0, 0, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 3044, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224550912 dispatch_meta_bytes=228648 combine_payload_bytes=224550912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=224779560 ingress_bytes=281174016 link_bytes=281174016 token_bytes=449330472 weight_movement_bytes=56623104 total_nvlink_bytes=505953576 weight_to_token_ratio=0.126017 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.694071075e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=213778432 dispatch_meta_bytes=229376 combine_payload_bytes=213778432 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=214007808 ingress_bytes=251527168 link_bytes=251527168 token_bytes=427786240 weight_movement_bytes=37748736 total_nvlink_bytes=465534976 weight_to_token_ratio=0.088242 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410009565e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63744 payload_rows_remote=55560 dup_rows=1792 dup_rows_remote=1792 dedup_ratio=0.027344 router_tokens_per_expert=[0, 0, 0, 0, 781, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4067, 0, 2, 0, 1, 0, 0, 0, 0, 26, 2, 0, 0, 0, 0, 6, 0, 0, 0, 4001, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 45, 0, 4, 8, 26, 4, 4068, 16, 0, 5, 0, 0, 0, 0, 0, 27, 0, 3366, 0, 0, 0, 0, 0, 25, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 2, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8138, 8184, 8192, 8046, 8178, 6788, 8192, 8026] dup_rows_per_dst=[1564, 0, 16, 0, 160, 52, 0, 0] combine_rows_per_src=[8138, 8184, 8192, 8046, 8178, 6788, 8192, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60092 payload_rows_remote=51900 dup_rows=5444 dup_rows_remote=46 dedup_ratio=0.083069 router_tokens_per_expert=[0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4081, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 16, 0, 1, 1, 5, 4, 4090, 2, 0, 0, 1, 0, 0, 0, 0, 6, 0, 2692, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2842, 8192, 8192, 8168, 8138, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 6, 0, 30, 5398, 10, 0] combine_rows_per_src=[2842, 8192, 8192, 8168, 8138, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61348 payload_rows_remote=53160 dup_rows=4188 dup_rows_remote=3980 dedup_ratio=0.063904 router_tokens_per_expert=[0, 0, 0, 0, 2087, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 1, 2, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2016, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4174, 8192, 8192, 8180, 8184, 8046, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 6, 3974, 0, 208] combine_rows_per_src=[4174, 8192, 8192, 8180, 8184, 8046, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60940 payload_rows_remote=52748 dup_rows=4596 dup_rows_remote=730 dedup_ratio=0.070129 router_tokens_per_expert=[0, 0, 0, 0, 1930, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 2, 0, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8164, 8192, 7504, 4332, 8176, 8190] dup_rows_per_dst=[3866, 22, 0, 700, 6, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8164, 8192, 7504, 4332, 8176, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8156, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 36, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=227573760 dispatch_meta_bytes=229408 combine_payload_bytes=227573760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227803168 ingress_bytes=246448128 link_bytes=246448128 token_bytes=455376928 weight_movement_bytes=18874368 total_nvlink_bytes=474251296 weight_to_token_ratio=0.041448 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.361344702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=212582400 dispatch_meta_bytes=207784 combine_payload_bytes=212582400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212790184 ingress_bytes=231456768 link_bytes=231456768 token_bytes=425372584 weight_movement_bytes=18874368 total_nvlink_bytes=444246952 weight_to_token_ratio=0.044371 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217704867e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=216055808 dispatch_meta_bytes=213912 combine_payload_bytes=216055808 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=216269720 ingress_bytes=291553280 link_bytes=291553280 token_bytes=432325528 weight_movement_bytes=75497472 total_nvlink_bytes=507823000 weight_to_token_ratio=0.174631 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793520076e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=217743360 dispatch_meta_bytes=228560 combine_payload_bytes=217743360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217971920 ingress_bytes=236617728 link_bytes=236617728 token_bytes=435715280 weight_movement_bytes=18874368 total_nvlink_bytes=454589648 weight_to_token_ratio=0.043318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.267154647e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65206 payload_rows_remote=57018 dup_rows=330 dup_rows_remote=330 dedup_ratio=0.005035 router_tokens_per_expert=[6, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 1, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 2, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3941, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8190, 8188, 8180, 7882, 8192] dup_rows_per_dst=[12, 36, 0, 4, 0, 266, 0, 12] combine_rows_per_src=[8192, 8192, 8190, 8190, 8188, 8180, 7882, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8112, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 80, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8112, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=233545728 dispatch_meta_bytes=229392 combine_payload_bytes=233545728 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233775120 ingress_bytes=290168832 link_bytes=290168832 token_bytes=467320848 weight_movement_bytes=56623104 total_nvlink_bytes=523943952 weight_to_token_ratio=0.121165 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780254976e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229376 combine_payload_bytes=234553344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234782720 ingress_bytes=253427712 link_bytes=253427712 token_bytes=469336064 weight_movement_bytes=18874368 total_nvlink_bytes=488210432 weight_to_token_ratio=0.040215 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428219642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65192 payload_rows_remote=57004 dup_rows=344 dup_rows_remote=280 dedup_ratio=0.005249 router_tokens_per_expert=[26, 0, 28, 0, 3, 1, 4068, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 3, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 1, 0, 0, 0, 0, 4, 0, 0, 26, 0, 0, 0, 19, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4066, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 2, 74, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 4067, 0, 0, 0, 0, 0, 20, 0, 0, 6, 0, 0, 1, 27, 2, 0, 5, 0, 0, 4067, 0, 0, 0, 4, 0, 0, 4, 0, 0] payload_rows_per_dst=[8192, 8188, 8142, 8148, 8172, 8152, 8006, 8192] dup_rows_per_dst=[66, 64, 0, 38, 0, 150, 0, 26] combine_rows_per_src=[8192, 8188, 8142, 8148, 8172, 8152, 8006, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57250 dup_rows=96 dup_rows_remote=68 dedup_ratio=0.001465 router_tokens_per_expert=[6, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 4, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 9, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8176, 8186, 8190, 8132, 8192] dup_rows_per_dst=[12, 44, 0, 8, 0, 28, 2, 2] combine_rows_per_src=[8192, 8192, 8180, 8176, 8186, 8190, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=134 dedup_ratio=0.002075 router_tokens_per_expert=[1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8190, 8122, 8130] dup_rows_per_dst=[2, 0, 66, 0, 0, 68, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8190, 8122, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64544 payload_rows_remote=56352 dup_rows=992 dup_rows_remote=970 dedup_ratio=0.015137 router_tokens_per_expert=[2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 8046, 8190, 8192, 8156, 7554, 8118, 8192] dup_rows_per_dst=[4, 8, 0, 12, 68, 64, 814, 22] combine_rows_per_src=[8096, 8046, 8190, 8192, 8156, 7554, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64666 payload_rows_remote=56474 dup_rows=870 dup_rows_remote=870 dedup_ratio=0.013275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 435, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 7332] dup_rows_per_dst=[0, 0, 0, 0, 0, 870, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 7332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=233488384 dispatch_meta_bytes=229136 combine_payload_bytes=233488384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233717520 ingress_bytes=252362752 link_bytes=252362752 token_bytes=467205904 weight_movement_bytes=18874368 total_nvlink_bytes=486080272 weight_to_token_ratio=0.040398 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418015719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229272 combine_payload_bytes=234496000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234725272 ingress_bytes=253370368 link_bytes=253370368 token_bytes=469221272 weight_movement_bytes=18874368 total_nvlink_bytes=488095640 weight_to_token_ratio=0.040225 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427670200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234323968 dispatch_meta_bytes=229368 combine_payload_bytes=234323968 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234553336 ingress_bytes=290947072 link_bytes=290947072 token_bytes=468877304 weight_movement_bytes=56623104 total_nvlink_bytes=525500408 weight_to_token_ratio=0.120763 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787711689e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=230817792 dispatch_meta_bytes=229288 combine_payload_bytes=230817792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231047080 ingress_bytes=268566528 link_bytes=268566528 token_bytes=461864872 weight_movement_bytes=37748736 total_nvlink_bytes=499613608 weight_to_token_ratio=0.081731 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.573272328e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57206 dup_rows=138 dup_rows_remote=128 dedup_ratio=0.002106 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 1, 0, 4090, 0, 2, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 59, 4089, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 4031, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8180, 8192, 8190, 8192, 8192, 8190, 8192, 8070] dup_rows_per_dst=[0, 0, 4, 2, 10, 118, 2, 2] combine_rows_per_src=[8180, 8192, 8190, 8192, 8192, 8190, 8192, 8070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64894 payload_rows_remote=56702 dup_rows=642 dup_rows_remote=642 dedup_ratio=0.009796 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 321, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7550] dup_rows_per_dst=[0, 0, 0, 0, 0, 642, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235077632 ingress_bytes=310345728 link_bytes=310345728 token_bytes=469925888 weight_movement_bytes=75497472 total_nvlink_bytes=545423360 weight_to_token_ratio=0.160658 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973580066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231317504 dispatch_meta_bytes=229376 combine_payload_bytes=231317504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231546880 ingress_bytes=269066240 link_bytes=269066240 token_bytes=462864384 weight_movement_bytes=37748736 total_nvlink_bytes=500613120 weight_to_token_ratio=0.081555 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.578060323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234315776 dispatch_meta_bytes=229336 combine_payload_bytes=234315776 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234545112 ingress_bytes=290938880 link_bytes=290938880 token_bytes=468860888 weight_movement_bytes=56623104 total_nvlink_bytes=525483992 weight_to_token_ratio=0.120767 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787633197e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232251392 dispatch_meta_bytes=229376 combine_payload_bytes=232251392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232480768 ingress_bytes=251125760 link_bytes=251125760 token_bytes=464732160 weight_movement_bytes=18874368 total_nvlink_bytes=483606528 weight_to_token_ratio=0.040613 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406163471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=136 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 2, 0, 0, 4069, 0, 0, 0, 0, 4063, 0, 23, 27, 0, 0, 0, 4068, 0, 0, 2, 0, 0, 0, 16, 0, 0, 27, 0, 0, 0, 1, 0, 0, 0, 3, 4066, 0, 6, 4067, 2, 1, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4061, 0, 4062, 0, 0, 0, 1, 0, 0, 4, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[8140, 8192, 8178, 8192, 8184, 8192, 8190, 8132] dup_rows_per_dst=[12, 0, 12, 6, 78, 18, 0, 10] combine_rows_per_src=[8140, 8192, 8178, 8192, 8184, 8192, 8190, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64738 payload_rows_remote=56548 dup_rows=798 dup_rows_remote=18 dedup_ratio=0.012177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 1, 0, 4089, 0, 4, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 390, 4089, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 0, 3700, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8180, 8192, 8190, 8192, 8190, 8190, 8192, 7412] dup_rows_per_dst=[2, 0, 4, 2, 8, 780, 0, 2] combine_rows_per_src=[8180, 8192, 8190, 8192, 8190, 8190, 8192, 7412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64702 payload_rows_remote=56512 dup_rows=834 dup_rows_remote=766 dedup_ratio=0.012726 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 380, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8190, 8124, 8192, 7430] dup_rows_per_dst=[68, 0, 0, 2, 2, 762, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8190, 8124, 8192, 7430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60422 payload_rows_remote=52874 dup_rows=5114 dup_rows_remote=672 dedup_ratio=0.078033 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 483, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 3608, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8186, 8190, 8180, 8102, 3836, 8192, 7548] dup_rows_per_dst=[0, 6, 0, 0, 4, 650, 12, 4442] combine_rows_per_src=[8188, 8186, 8190, 8180, 8102, 3836, 8192, 7548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234323968 dispatch_meta_bytes=229376 combine_payload_bytes=234323968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234553344 ingress_bytes=253198336 link_bytes=253198336 token_bytes=468877312 weight_movement_bytes=18874368 total_nvlink_bytes=487751680 weight_to_token_ratio=0.040254 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426021874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59966 payload_rows_remote=51774 dup_rows=5570 dup_rows_remote=5570 dedup_ratio=0.084991 router_tokens_per_expert=[0, 0, 0, 0, 1315, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2630, 8192, 8192, 8192, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 6, 5564, 0, 0] combine_rows_per_src=[2630, 8192, 8192, 8192, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57328 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 20, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231620608 dispatch_meta_bytes=226264 combine_payload_bytes=231620608 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231846872 ingress_bytes=307118080 link_bytes=307118080 token_bytes=463467480 weight_movement_bytes=75497472 total_nvlink_bytes=538964952 weight_to_token_ratio=0.162897 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.942654331e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231473152 dispatch_meta_bytes=229112 combine_payload_bytes=231473152 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231702264 ingress_bytes=269221888 link_bytes=269221888 token_bytes=463175416 weight_movement_bytes=37748736 total_nvlink_bytes=500924152 weight_to_token_ratio=0.081500 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.579551665e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57274 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 5, 0, 4061, 12, 0, 0, 6, 0, 0, 0, 0, 4084, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8180, 8192, 8192, 8192, 8182, 8190, 8158, 8170] dup_rows_per_dst=[2, 46, 0, 10, 0, 2, 20, 0] combine_rows_per_src=[8180, 8192, 8192, 8192, 8182, 8190, 8158, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=216571904 dispatch_meta_bytes=214184 combine_payload_bytes=216571904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216786088 ingress_bytes=254320640 link_bytes=254320640 token_bytes=433357992 weight_movement_bytes=37748736 total_nvlink_bytes=471106728 weight_to_token_ratio=0.087108 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.436775239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=212066304 dispatch_meta_bytes=229376 combine_payload_bytes=212066304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212295680 ingress_bytes=230940672 link_bytes=230940672 token_bytes=424361984 weight_movement_bytes=18874368 total_nvlink_bytes=443236352 weight_to_token_ratio=0.044477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.212759889e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229392 combine_payload_bytes=234815488 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235044880 ingress_bytes=291438592 link_bytes=291438592 token_bytes=469860368 weight_movement_bytes=56623104 total_nvlink_bytes=526483472 weight_to_token_ratio=0.120510 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792421192e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229416 combine_payload_bytes=234594304 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234823720 ingress_bytes=272343040 link_bytes=272343040 token_bytes=469418024 weight_movement_bytes=37748736 total_nvlink_bytes=507166760 weight_to_token_ratio=0.080416 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609457008e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65340 payload_rows_remote=57178 dup_rows=196 dup_rows_remote=196 dedup_ratio=0.002991 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 16, 0, 0, 0, 0, 4065, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 4068, 0, 1, 0, 0, 28, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 22, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4068, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4067, 0, 25, 0, 0, 0, 0, 25, 0, 4051, 10, 0, 0, 28, 0, 0, 0, 0, 4066, 0, 1, 4067, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8] payload_rows_per_dst=[8138, 8162, 8188, 8192, 8184, 8142, 8178, 8156] dup_rows_per_dst=[28, 0, 0, 54, 8, 6, 100, 0] combine_rows_per_src=[8138, 8162, 8188, 8192, 8184, 8142, 8178, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57278 dup_rows=68 dup_rows_remote=18 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8152, 8192, 8190, 8192, 8184, 8182, 8186] dup_rows_per_dst=[50, 2, 4, 0, 8, 2, 2, 0] combine_rows_per_src=[8190, 8152, 8192, 8190, 8192, 8184, 8182, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57310 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 3, 0, 4089, 7, 0, 0, 6, 0, 0, 0, 0, 4083, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8180, 8190, 8192, 8192, 8192, 8180, 8192, 8172] dup_rows_per_dst=[2, 0, 0, 14, 2, 0, 28, 0] combine_rows_per_src=[8180, 8190, 8192, 8192, 8192, 8180, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57160 dup_rows=192 dup_rows_remote=84 dedup_ratio=0.002930 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 2, 0, 4093, 3, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 8192, 8116, 8192, 8188, 8094, 8184] dup_rows_per_dst=[2, 0, 0, 4, 0, 66, 12, 108] combine_rows_per_src=[8188, 8190, 8192, 8116, 8192, 8188, 8094, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=229496 combine_payload_bytes=234201088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234430584 ingress_bytes=271949824 link_bytes=271949824 token_bytes=468631672 weight_movement_bytes=37748736 total_nvlink_bytes=506380408 weight_to_token_ratio=0.080551 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605689406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229184 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234839872 ingress_bytes=253485056 link_bytes=253485056 token_bytes=469450560 weight_movement_bytes=18874368 total_nvlink_bytes=488324928 weight_to_token_ratio=0.040205 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229424 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234971184 ingress_bytes=291364864 link_bytes=291364864 token_bytes=469712944 weight_movement_bytes=56623104 total_nvlink_bytes=526336048 weight_to_token_ratio=0.120548 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791714766e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=86 dedup_ratio=0.001434 router_tokens_per_expert=[5, 0, 4090, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4090, 0, 4090, 0, 0, 4082, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1, 4090, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 8190, 8186, 8192, 8190, 8176, 8180] dup_rows_per_dst=[0, 12, 12, 52, 8, 0, 0, 10] combine_rows_per_src=[8136, 8192, 8190, 8186, 8192, 8190, 8176, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65272 payload_rows_remote=57080 dup_rows=264 dup_rows_remote=264 dedup_ratio=0.004028 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 132, 0, 0, 0] payload_rows_per_dst=[7942, 8192, 8192, 8192, 8192, 8192, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 264] combine_rows_per_src=[7942, 8192, 8192, 8192, 8192, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=228976 combine_payload_bytes=234127360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234356336 ingress_bytes=290750464 link_bytes=290750464 token_bytes=468483696 weight_movement_bytes=56623104 total_nvlink_bytes=525106800 weight_to_token_ratio=0.120865 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.785827888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 8, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229344 combine_payload_bytes=234496000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234725344 ingress_bytes=253370368 link_bytes=253370368 token_bytes=469221344 weight_movement_bytes=18874368 total_nvlink_bytes=488095712 weight_to_token_ratio=0.040225 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427670200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233799680 dispatch_meta_bytes=229376 combine_payload_bytes=233799680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234029056 ingress_bytes=252674048 link_bytes=252674048 token_bytes=467828736 weight_movement_bytes=18874368 total_nvlink_bytes=486703104 weight_to_token_ratio=0.040345 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420998405e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65006 payload_rows_remote=56818 dup_rows=530 dup_rows_remote=466 dedup_ratio=0.008087 router_tokens_per_expert=[6, 0, 4065, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 28, 0, 0, 0, 1, 19, 0, 0, 0, 0, 0, 1, 4056, 0, 28, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 0, 0, 1, 0, 4, 0, 0, 21, 20, 0, 27, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4066, 5, 0, 0, 4066, 0, 4068, 0, 0, 3962, 0, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 84, 0, 0, 0] payload_rows_per_dst=[8004, 8188, 8170, 8150, 8192, 8192, 7976, 8134] dup_rows_per_dst=[0, 64, 18, 174, 92, 10, 4, 168] combine_rows_per_src=[8004, 8188, 8170, 8150, 8192, 8192, 7976, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64812 payload_rows_remote=56620 dup_rows=724 dup_rows_remote=126 dedup_ratio=0.011047 router_tokens_per_expert=[1, 0, 4095, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 4095, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7866, 8190, 8192, 8192, 8192, 8178, 7810] dup_rows_per_dst=[598, 0, 0, 22, 0, 0, 102, 2] combine_rows_per_src=[8192, 7866, 8190, 8192, 8192, 8192, 8178, 7810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65438 payload_rows_remote=57246 dup_rows=98 dup_rows_remote=96 dedup_ratio=0.001495 router_tokens_per_expert=[1, 0, 4089, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8, 0, 6, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4085, 0, 6, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 4089, 0, 4090, 0, 0, 4090, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4089, 2, 0, 0, 0] payload_rows_per_dst=[8122, 8192, 8184, 8186, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 20, 10, 42, 6, 2, 10, 8] combine_rows_per_src=[8122, 8192, 8184, 8186, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64928 payload_rows_remote=56740 dup_rows=608 dup_rows_remote=590 dedup_ratio=0.009277 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 4094, 0, 0, 3995, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8192, 8184, 7952, 8078, 8182, 7994, 8188] dup_rows_per_dst=[0, 226, 2, 4, 8, 0, 350, 18] combine_rows_per_src=[8158, 8192, 8184, 7952, 8078, 8182, 7994, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232726528 dispatch_meta_bytes=229136 combine_payload_bytes=232726528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232955664 ingress_bytes=251600896 link_bytes=251600896 token_bytes=465682192 weight_movement_bytes=18874368 total_nvlink_bytes=484556560 weight_to_token_ratio=0.040531 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410715990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61128 payload_rows_remote=52936 dup_rows=4408 dup_rows_remote=4396 dedup_ratio=0.067261 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 6, 0, 0, 4090, 0, 0, 4089, 0, 1, 2174, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 1928, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8192, 3856, 8146, 8180] dup_rows_per_dst=[20, 2, 14, 4360, 12, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8190, 8192, 8192, 3856, 8146, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231915520 dispatch_meta_bytes=226984 combine_payload_bytes=231915520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232142504 ingress_bytes=269664256 link_bytes=269664256 token_bytes=464058024 weight_movement_bytes=37748736 total_nvlink_bytes=501806760 weight_to_token_ratio=0.081345 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.583790218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234479616 dispatch_meta_bytes=229368 combine_payload_bytes=234479616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234708984 ingress_bytes=272228352 link_bytes=272228352 token_bytes=469188600 weight_movement_bytes=37748736 total_nvlink_bytes=506937336 weight_to_token_ratio=0.080455 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608358124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64948 payload_rows_remote=56758 dup_rows=588 dup_rows_remote=588 dedup_ratio=0.008972 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 294, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 7606] dup_rows_per_dst=[0, 0, 0, 0, 0, 588, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232407040 dispatch_meta_bytes=229320 combine_payload_bytes=232407040 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232636360 ingress_bytes=289030144 link_bytes=289030144 token_bytes=465043400 weight_movement_bytes=56623104 total_nvlink_bytes=521666504 weight_to_token_ratio=0.121759 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769344628e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63634 payload_rows_remote=55442 dup_rows=1902 dup_rows_remote=0 dedup_ratio=0.029022 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3194, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6388, 8094, 8192] dup_rows_per_dst=[0, 0, 0, 1902, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6388, 8094, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60746 payload_rows_remote=52554 dup_rows=4790 dup_rows_remote=4790 dedup_ratio=0.073090 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1703, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 3406, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 4790, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 3406, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=216825856 dispatch_meta_bytes=229328 combine_payload_bytes=216825856 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=217055184 ingress_bytes=254574592 link_bytes=254574592 token_bytes=433881040 weight_movement_bytes=37748736 total_nvlink_bytes=471629776 weight_to_token_ratio=0.087003 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.439208482e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232480768 dispatch_meta_bytes=229384 combine_payload_bytes=232480768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232710152 ingress_bytes=251355136 link_bytes=251355136 token_bytes=465190920 weight_movement_bytes=18874368 total_nvlink_bytes=484065288 weight_to_token_ratio=0.040573 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408361239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=227090432 dispatch_meta_bytes=221768 combine_payload_bytes=227090432 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=227312200 ingress_bytes=321462272 link_bytes=321462272 token_bytes=454402632 weight_movement_bytes=94371840 total_nvlink_bytes=548774472 weight_to_token_ratio=0.207683 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.080093321e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=215261184 dispatch_meta_bytes=229376 combine_payload_bytes=215261184 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=215490560 ingress_bytes=234135552 link_bytes=234135552 token_bytes=430751744 weight_movement_bytes=18874368 total_nvlink_bytes=449626112 weight_to_token_ratio=0.043817 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.243371657e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64862 payload_rows_remote=56728 dup_rows=674 dup_rows_remote=624 dedup_ratio=0.010284 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 4068, 27, 0, 0, 0, 10, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 20, 0, 0, 6, 0, 0, 0, 0, 4, 1, 0, 0, 0, 13, 0, 28, 0, 0, 4067, 0, 0, 4060, 0, 5, 237, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 27, 0, 0, 0, 3846, 2, 0, 0, 17, 0, 3, 4029, 0, 0, 0, 0, 0, 0, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 22, 0] payload_rows_per_dst=[8192, 8134, 8178, 8186, 8134, 7746, 8116, 8176] dup_rows_per_dst=[22, 50, 14, 500, 44, 44, 0, 0] combine_rows_per_src=[8192, 8134, 8178, 8186, 8134, 7746, 8116, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62504 payload_rows_remote=54312 dup_rows=3032 dup_rows_remote=3020 dedup_ratio=0.046265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 1497, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8182, 8192, 5192, 8182, 8190] dup_rows_per_dst=[12, 24, 2, 2980, 14, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8182, 8192, 5192, 8182, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59478 payload_rows_remote=51302 dup_rows=6058 dup_rows_remote=62 dedup_ratio=0.092438 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 1, 0, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 4090, 0, 0, 4089, 0, 1, 495, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 3598, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0] payload_rows_per_dst=[8192, 8180, 8188, 2196, 8192, 8176, 8164, 8190] dup_rows_per_dst=[16, 8, 0, 34, 4, 5996, 0, 0] combine_rows_per_src=[8192, 8180, 8188, 2196, 8192, 8176, 8164, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61070 payload_rows_remote=52878 dup_rows=4466 dup_rows_remote=4434 dedup_ratio=0.068146 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4094, 0, 0, 4094, 0, 0, 1986, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 2111, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0] payload_rows_per_dst=[8108, 8188, 8192, 4002, 8174, 8026, 8188, 8192] dup_rows_per_dst=[14, 2, 0, 30, 2, 4118, 268, 32] combine_rows_per_src=[8108, 8188, 8192, 4002, 8174, 8026, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=232357888 dispatch_meta_bytes=229408 combine_payload_bytes=232357888 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232587296 ingress_bytes=270106624 link_bytes=270106624 token_bytes=464945184 weight_movement_bytes=37748736 total_nvlink_bytes=502693920 weight_to_token_ratio=0.081190 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.588028770e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=222461952 dispatch_meta_bytes=229328 combine_payload_bytes=222461952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222691280 ingress_bytes=279085056 link_bytes=279085056 token_bytes=445153232 weight_movement_bytes=56623104 total_nvlink_bytes=501776336 weight_to_token_ratio=0.127199 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.674055688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65120 payload_rows_remote=56928 dup_rows=416 dup_rows_remote=408 dedup_ratio=0.006348 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 168, 0, 0, 4090, 0, 6, 0, 0, 0, 3934, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 2, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4090, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4059, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7880, 8174, 8192, 8192, 8180, 8118, 8192] dup_rows_per_dst=[346, 18, 0, 6, 8, 28, 0, 10] combine_rows_per_src=[8192, 7880, 8174, 8192, 8192, 8180, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=210132992 dispatch_meta_bytes=205456 combine_payload_bytes=210132992 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210338448 ingress_bytes=266756096 link_bytes=266756096 token_bytes=420471440 weight_movement_bytes=56623104 total_nvlink_bytes=477094544 weight_to_token_ratio=0.134666 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.555925660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=216588288 dispatch_meta_bytes=229248 combine_payload_bytes=216588288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216817536 ingress_bytes=235462656 link_bytes=235462656 token_bytes=433405824 weight_movement_bytes=18874368 total_nvlink_bytes=452280192 weight_to_token_ratio=0.043549 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.256087315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57344 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[12, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63882 payload_rows_remote=55814 dup_rows=1654 dup_rows_remote=1654 dedup_ratio=0.025238 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 4096, 0, 0, 0, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6890, 7964, 8068, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1150, 0, 0, 0, 110, 0, 394, 0] combine_rows_per_src=[8192, 6890, 7964, 8068, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233177088 dispatch_meta_bytes=229344 combine_payload_bytes=233177088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233406432 ingress_bytes=252051456 link_bytes=252051456 token_bytes=466583520 weight_movement_bytes=18874368 total_nvlink_bytes=485457888 weight_to_token_ratio=0.040452 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415033034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229400 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110424 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991448 weight_movement_bytes=37748736 total_nvlink_bytes=507740184 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64988 payload_rows_remote=56796 dup_rows=548 dup_rows_remote=500 dedup_ratio=0.008362 router_tokens_per_expert=[0, 0, 0, 8, 0, 0, 0, 70, 0, 12, 4066, 0, 30, 1, 0, 0, 4067, 0, 28, 3, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4057, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 22, 0, 0, 9, 0, 60, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 4067, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 3882, 0, 0, 0, 4066, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 26, 0, 0, 0, 0, 0, 3, 0, 23, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8192, 8154, 8134, 7770, 8192] dup_rows_per_dst=[182, 48, 0, 138, 6, 114, 0, 60] combine_rows_per_src=[8192, 8192, 8162, 8192, 8154, 8134, 7770, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=228614144 dispatch_meta_bytes=229872 combine_payload_bytes=228614144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228844016 ingress_bytes=247488512 link_bytes=247488512 token_bytes=457458160 weight_movement_bytes=18874368 total_nvlink_bytes=476332528 weight_to_token_ratio=0.041259 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371313150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65382 payload_rows_remote=57190 dup_rows=154 dup_rows_remote=128 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 1, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4084, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8146, 8192, 8178, 8190, 8102] dup_rows_per_dst=[26, 54, 0, 0, 0, 4, 68, 2] combine_rows_per_src=[8192, 8190, 8192, 8146, 8192, 8178, 8190, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57168 dup_rows=190 dup_rows_remote=172 dedup_ratio=0.002899 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 56, 0, 0, 4089, 0, 6, 0, 0, 0, 4068, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 3, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4044, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8148, 8166, 8190, 8192, 8178, 8090, 8192] dup_rows_per_dst=[118, 12, 0, 18, 10, 18, 0, 14] combine_rows_per_src=[8190, 8148, 8166, 8190, 8192, 8178, 8090, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232636416 dispatch_meta_bytes=229184 combine_payload_bytes=232636416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232865600 ingress_bytes=270385152 link_bytes=270385152 token_bytes=465502016 weight_movement_bytes=37748736 total_nvlink_bytes=503250752 weight_to_token_ratio=0.081093 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.590697489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63504 payload_rows_remote=55312 dup_rows=2032 dup_rows_remote=2028 dedup_ratio=0.031006 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 2, 0, 0, 0, 4093, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4086, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[6352, 8190, 8190, 8192, 8096, 8120, 8172, 8192] dup_rows_per_dst=[6, 1540, 304, 8, 0, 6, 164, 4] combine_rows_per_src=[6352, 8190, 8190, 8192, 8096, 8120, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234250240 dispatch_meta_bytes=229272 combine_payload_bytes=234250240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234479512 ingress_bytes=290873344 link_bytes=290873344 token_bytes=468729752 weight_movement_bytes=56623104 total_nvlink_bytes=525352856 weight_to_token_ratio=0.120801 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787005264e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57308 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4087, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 18, 0, 4089, 6, 4065] payload_rows_per_dst=[8186, 8192, 8192, 8190, 8180, 8190, 8180, 8178] dup_rows_per_dst=[0, 22, 0, 0, 0, 0, 2, 24] combine_rows_per_src=[8186, 8192, 8192, 8190, 8180, 8190, 8180, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234160128 dispatch_meta_bytes=229360 combine_payload_bytes=234160128 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234389488 ingress_bytes=271908864 link_bytes=271908864 token_bytes=468549616 weight_movement_bytes=37748736 total_nvlink_bytes=506298352 weight_to_token_ratio=0.080565 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605296947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=226557952 dispatch_meta_bytes=229360 combine_payload_bytes=226557952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226787312 ingress_bytes=245432320 link_bytes=245432320 token_bytes=453345264 weight_movement_bytes=18874368 total_nvlink_bytes=472219632 weight_to_token_ratio=0.041634 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351611730e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57068 dup_rows=276 dup_rows_remote=276 dedup_ratio=0.004211 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3958] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7916] dup_rows_per_dst=[0, 0, 0, 0, 0, 272, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229424 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962992 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696560 weight_movement_bytes=18874368 total_nvlink_bytes=488570928 weight_to_token_ratio=0.040184 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65040 payload_rows_remote=56848 dup_rows=496 dup_rows_remote=468 dedup_ratio=0.007568 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4059, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 28, 0, 1, 0, 10, 0, 0, 0, 0, 1, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4068, 4067, 0, 0, 0, 0, 4067, 0, 0, 0, 1, 0, 0, 7, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4066, 0, 0, 0, 0, 0, 130, 0, 2, 18, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 7, 0, 0, 0, 0, 28, 0, 4, 2, 4061, 28, 3897] payload_rows_per_dst=[8150, 8192, 8192, 8192, 8136, 8170, 8136, 7872] dup_rows_per_dst=[0, 28, 2, 14, 14, 260, 64, 114] combine_rows_per_src=[8150, 8192, 8192, 8192, 8136, 8170, 8136, 7872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233750528 dispatch_meta_bytes=229376 combine_payload_bytes=233750528 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233979904 ingress_bytes=328122368 link_bytes=328122368 token_bytes=467730432 weight_movement_bytes=94371840 total_nvlink_bytes=562102272 weight_to_token_ratio=0.201765 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.143907084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=12 dedup_ratio=0.001007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 4092] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8190, 8130] dup_rows_per_dst=[54, 0, 2, 0, 0, 6, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8190, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=232849408 dispatch_meta_bytes=229264 combine_payload_bytes=232849408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233078672 ingress_bytes=251723776 link_bytes=251723776 token_bytes=465928080 weight_movement_bytes=18874368 total_nvlink_bytes=484802448 weight_to_token_ratio=0.040509 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411893366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57286 dup_rows=62 dup_rows_remote=36 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 1, 0, 0, 0, 13, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 4089, 6, 4069] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8180, 8188, 8180, 8160] dup_rows_per_dst=[0, 4, 0, 2, 4, 26, 2, 24] combine_rows_per_src=[8190, 8192, 8192, 8192, 8180, 8188, 8180, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64908 payload_rows_remote=56716 dup_rows=628 dup_rows_remote=86 dedup_ratio=0.009583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 4094, 2, 4077] payload_rows_per_dst=[8188, 8136, 8188, 8172, 8188, 7656, 8188, 8192] dup_rows_per_dst=[0, 2, 0, 6, 68, 0, 10, 542] combine_rows_per_src=[8188, 8136, 8188, 8172, 8188, 7656, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57310 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 1, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4087, 1, 4089, 0, 6, 0, 0, 6, 0, 0, 5, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8186, 8180, 8180, 8190, 8192] dup_rows_per_dst=[0, 0, 16, 0, 0, 0, 22, 8] combine_rows_per_src=[8192, 8178, 8192, 8186, 8180, 8180, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229160 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234839848 ingress_bytes=253485056 link_bytes=253485056 token_bytes=469450536 weight_movement_bytes=18874368 total_nvlink_bytes=488324904 weight_to_token_ratio=0.040205 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61140 payload_rows_remote=52950 dup_rows=4396 dup_rows_remote=4396 dedup_ratio=0.067078 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2198, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3798, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4396, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 3798, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=229288 combine_payload_bytes=234643456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234872744 ingress_bytes=253517824 link_bytes=253517824 token_bytes=469516200 weight_movement_bytes=18874368 total_nvlink_bytes=488390568 weight_to_token_ratio=0.040200 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429083051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=232308736 dispatch_meta_bytes=227208 combine_payload_bytes=232308736 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232535944 ingress_bytes=288931840 link_bytes=288931840 token_bytes=464844680 weight_movement_bytes=56623104 total_nvlink_bytes=521467784 weight_to_token_ratio=0.121811 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.768402728e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229424 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234971184 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469712944 weight_movement_bytes=18874368 total_nvlink_bytes=488587312 weight_to_token_ratio=0.040183 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229360 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235094000 ingress_bytes=310362112 link_bytes=310362112 token_bytes=469958640 weight_movement_bytes=75497472 total_nvlink_bytes=545456112 weight_to_token_ratio=0.160647 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973737049e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65184 payload_rows_remote=57050 dup_rows=352 dup_rows_remote=352 dedup_ratio=0.005371 router_tokens_per_expert=[58, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 29, 2, 28, 0, 0, 0, 4065, 0, 0, 0, 18, 0, 3, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4059, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4055, 1, 4057, 0, 28, 1, 0, 29, 0, 0, 26, 4010, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 7, 13, 0, 27, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8134, 8192, 8132, 8128, 8130, 8086, 8190] dup_rows_per_dst=[94, 0, 98, 4, 0, 0, 114, 42] combine_rows_per_src=[8192, 8134, 8192, 8132, 8128, 8130, 8086, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=216883200 dispatch_meta_bytes=229384 combine_payload_bytes=216883200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217112584 ingress_bytes=235757568 link_bytes=235757568 token_bytes=433995784 weight_movement_bytes=18874368 total_nvlink_bytes=452870152 weight_to_token_ratio=0.043490 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.258913017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57112 dup_rows=232 dup_rows_remote=232 dedup_ratio=0.003540 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 2, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8190, 8190, 8032, 8128] dup_rows_per_dst=[0, 90, 70, 66, 0, 0, 0, 6] combine_rows_per_src=[8192, 8190, 8192, 8190, 8190, 8190, 8032, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233676800 dispatch_meta_bytes=229608 combine_payload_bytes=233676800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233906408 ingress_bytes=252551168 link_bytes=252551168 token_bytes=467583208 weight_movement_bytes=18874368 total_nvlink_bytes=486457576 weight_to_token_ratio=0.040366 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419821029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57296 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 1, 6, 0, 0, 0, 4089, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 1, 4089, 0, 6, 0, 0, 6, 0, 0, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8180, 8174, 8180, 8188, 8190] dup_rows_per_dst=[4, 0, 22, 6, 0, 0, 24, 4] combine_rows_per_src=[8192, 8180, 8192, 8180, 8174, 8180, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65184 payload_rows_remote=56992 dup_rows=352 dup_rows_remote=344 dedup_ratio=0.005371 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 4093, 0, 2, 0, 0, 2, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8100, 8188, 7968, 8188, 8178, 8188, 8182, 8192] dup_rows_per_dst=[4, 0, 8, 8, 110, 92, 122, 8] combine_rows_per_src=[8100, 8188, 7968, 8188, 8178, 8188, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233930752 dispatch_meta_bytes=229376 combine_payload_bytes=233930752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234160128 ingress_bytes=252805120 link_bytes=252805120 token_bytes=468090880 weight_movement_bytes=18874368 total_nvlink_bytes=486965248 weight_to_token_ratio=0.040322 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422254272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57270 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.002899 router_tokens_per_expert=[0, 6, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 4033, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 77, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8190, 8076, 8192, 8132, 8190] dup_rows_per_dst=[20, 12, 0, 0, 0, 158, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8190, 8076, 8192, 8132, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8146, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 46, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8146, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229424 combine_payload_bytes=234684416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234913840 ingress_bytes=253558784 link_bytes=253558784 token_bytes=469598256 weight_movement_bytes=18874368 total_nvlink_bytes=488472624 weight_to_token_ratio=0.040193 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429475510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233439232 dispatch_meta_bytes=229344 combine_payload_bytes=233439232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233668576 ingress_bytes=252313600 link_bytes=252313600 token_bytes=467107808 weight_movement_bytes=18874368 total_nvlink_bytes=485982176 weight_to_token_ratio=0.040407 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417544769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65190 payload_rows_remote=56998 dup_rows=346 dup_rows_remote=346 dedup_ratio=0.005280 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7846, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 346, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7846, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64188 payload_rows_remote=55996 dup_rows=1348 dup_rows_remote=1318 dedup_ratio=0.020569 router_tokens_per_expert=[0, 29, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 10, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 4066, 9, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3, 0, 3491, 0, 0, 0, 0, 4065, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 6, 0, 620, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3, 0, 0, 0, 0, 1, 29, 0, 0, 0, 4, 2, 0, 0, 0, 0, 4063, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8142, 7038, 8190, 8070, 8192] dup_rows_per_dst=[58, 30, 2, 0, 2, 1250, 0, 6] combine_rows_per_src=[8192, 8192, 8172, 8142, 7038, 8190, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229840 combine_payload_bytes=234577920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234807760 ingress_bytes=253452288 link_bytes=253452288 token_bytes=469385680 weight_movement_bytes=18874368 total_nvlink_bytes=488260048 weight_to_token_ratio=0.040211 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428455117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234921984 ingress_bytes=253566976 link_bytes=253566976 token_bytes=469614592 weight_movement_bytes=18874368 total_nvlink_bytes=488488960 weight_to_token_ratio=0.040191 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429554001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=291487744 link_bytes=291487744 token_bytes=469958656 weight_movement_bytes=56623104 total_nvlink_bytes=526581760 weight_to_token_ratio=0.120485 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792892142e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233463808 dispatch_meta_bytes=229376 combine_payload_bytes=233463808 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233693184 ingress_bytes=327835648 link_bytes=327835648 token_bytes=467156992 weight_movement_bytes=94371840 total_nvlink_bytes=561528832 weight_to_token_ratio=0.202013 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.141159874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=229359616 dispatch_meta_bytes=229256 combine_payload_bytes=229359616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229588872 ingress_bytes=248233984 link_bytes=248233984 token_bytes=458948488 weight_movement_bytes=18874368 total_nvlink_bytes=477822856 weight_to_token_ratio=0.041125 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.378455896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63416 payload_rows_remote=55224 dup_rows=2120 dup_rows_remote=2118 dedup_ratio=0.032349 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4076, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8168, 8162, 6136, 8192, 8182] dup_rows_per_dst=[2, 0, 30, 0, 2056, 32, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8168, 8162, 6136, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65312 payload_rows_remote=57120 dup_rows=224 dup_rows_remote=40 dedup_ratio=0.003418 router_tokens_per_expert=[0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 3995, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 91, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 2, 1, 0, 0, 0, 0, 4089, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8176, 8002, 8192, 8176, 8192] dup_rows_per_dst=[16, 12, 0, 0, 6, 184, 0, 6] combine_rows_per_src=[8192, 8192, 8190, 8176, 8002, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65238 payload_rows_remote=57046 dup_rows=298 dup_rows_remote=298 dedup_ratio=0.004547 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4094, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8094, 8138, 8192, 8190, 8188, 8052, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 94, 0, 4, 198, 0] combine_rows_per_src=[8094, 8138, 8192, 8190, 8188, 8052, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65074 payload_rows_remote=57280 dup_rows=462 dup_rows_remote=436 dedup_ratio=0.007050 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 423, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 1, 0, 6, 0, 5, 0, 0, 0, 0, 0, 6, 1, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0] payload_rows_per_dst=[8190, 8192, 8192, 8188, 7794, 8166, 8172, 8180] dup_rows_per_dst=[0, 2, 408, 0, 26, 26, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8188, 7794, 8166, 8172, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=226197504 dispatch_meta_bytes=229368 combine_payload_bytes=226197504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226426872 ingress_bytes=245071872 link_bytes=245071872 token_bytes=452624376 weight_movement_bytes=18874368 total_nvlink_bytes=471498744 weight_to_token_ratio=0.041700 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348158095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=0 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 0, 191, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8132, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 60, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8132, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233963520 dispatch_meta_bytes=228640 combine_payload_bytes=233963520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234192160 ingress_bytes=290586624 link_bytes=290586624 token_bytes=468155680 weight_movement_bytes=56623104 total_nvlink_bytes=524778784 weight_to_token_ratio=0.120949 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784258054e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64956 payload_rows_remote=56764 dup_rows=580 dup_rows_remote=580 dedup_ratio=0.008850 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 175, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7612, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 580, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7612, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233660416 dispatch_meta_bytes=229376 combine_payload_bytes=233660416 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233889792 ingress_bytes=252534784 link_bytes=252534784 token_bytes=467550208 weight_movement_bytes=18874368 total_nvlink_bytes=486424576 weight_to_token_ratio=0.040369 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419664045e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64590 payload_rows_remote=56442 dup_rows=946 dup_rows_remote=934 dedup_ratio=0.014435 router_tokens_per_expert=[0, 20, 0, 0, 0, 0, 16, 0, 23, 0, 0, 1, 0, 0, 4062, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 28, 206, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4065, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 0, 0, 0, 0, 0, 0, 2956, 0, 0, 0, 0, 1089, 4063, 0, 12, 0, 0, 0, 0, 0, 2, 21, 3904, 0, 0, 0, 11, 0, 11, 0, 1, 0, 1, 0, 28, 12, 3, 0, 4067, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 0, 4066, 0] payload_rows_per_dst=[8172, 8148, 8188, 8186, 7780, 7890, 8060, 8166] dup_rows_per_dst=[72, 12, 428, 0, 380, 52, 0, 2] combine_rows_per_src=[8172, 8148, 8188, 8186, 7780, 7890, 8060, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=230864 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234849744 ingress_bytes=291241984 link_bytes=291241984 token_bytes=469468624 weight_movement_bytes=56623104 total_nvlink_bytes=526091728 weight_to_token_ratio=0.120611 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790537391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229136 combine_payload_bytes=234635264 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234864400 ingress_bytes=291258368 link_bytes=291258368 token_bytes=469499664 weight_movement_bytes=56623104 total_nvlink_bytes=526122768 weight_to_token_ratio=0.120603 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790694374e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229360 combine_payload_bytes=234864640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235094000 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958640 weight_movement_bytes=37748736 total_nvlink_bytes=507707376 weight_to_token_ratio=0.080324 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=232505344 dispatch_meta_bytes=229376 combine_payload_bytes=232505344 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232734720 ingress_bytes=270254080 link_bytes=270254080 token_bytes=465240064 weight_movement_bytes=37748736 total_nvlink_bytes=502988800 weight_to_token_ratio=0.081138 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589441621e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231186432 dispatch_meta_bytes=229504 combine_payload_bytes=231186432 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231415936 ingress_bytes=268935168 link_bytes=268935168 token_bytes=462602368 weight_movement_bytes=37748736 total_nvlink_bytes=500351104 weight_to_token_ratio=0.081601 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576804455e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65404 payload_rows_remote=57212 dup_rows=132 dup_rows_remote=132 dedup_ratio=0.002014 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 42, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 198, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8098, 8188, 8160, 8192] dup_rows_per_dst=[0, 28, 88, 12, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8098, 8188, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65248 payload_rows_remote=57092 dup_rows=288 dup_rows_remote=282 dedup_ratio=0.004395 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 5, 0, 5, 0, 0, 1, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 124, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 3527, 0, 0, 0, 0, 467, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 1, 4072, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0] payload_rows_per_dst=[8170, 8192, 8192, 8192, 7988, 8156, 8178, 8180] dup_rows_per_dst=[18, 0, 252, 0, 12, 6, 0, 0] combine_rows_per_src=[8170, 8192, 8192, 8192, 7988, 8156, 8178, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63322 payload_rows_remote=55132 dup_rows=2214 dup_rows_remote=2164 dedup_ratio=0.033783 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 267, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0] payload_rows_per_dst=[8140, 8192, 6156, 8192, 8074, 8190, 8188, 8190] dup_rows_per_dst=[6, 2, 116, 0, 1556, 282, 202, 50] combine_rows_per_src=[8140, 8192, 6156, 8192, 8074, 8190, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=56 dedup_ratio=0.001343 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 3, 6, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 2, 6, 0, 4089, 0, 0, 0, 12, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8136, 8188, 8192, 8192, 8192, 8178, 8178] dup_rows_per_dst=[12, 0, 0, 4, 32, 16, 24, 0] combine_rows_per_src=[8192, 8136, 8188, 8192, 8192, 8192, 8178, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233848832 dispatch_meta_bytes=229496 combine_payload_bytes=233848832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234078328 ingress_bytes=252723200 link_bytes=252723200 token_bytes=467927160 weight_movement_bytes=18874368 total_nvlink_bytes=486801528 weight_to_token_ratio=0.040336 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421469355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=225820672 dispatch_meta_bytes=229184 combine_payload_bytes=225820672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226049856 ingress_bytes=244695040 link_bytes=244695040 token_bytes=451870528 weight_movement_bytes=18874368 total_nvlink_bytes=470744896 weight_to_token_ratio=0.041769 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.344547476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234340352 dispatch_meta_bytes=229376 combine_payload_bytes=234340352 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234569728 ingress_bytes=253214720 link_bytes=253214720 token_bytes=468910080 weight_movement_bytes=18874368 total_nvlink_bytes=487784448 weight_to_token_ratio=0.040252 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426178858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65234 payload_rows_remote=57242 dup_rows=302 dup_rows_remote=302 dedup_ratio=0.004608 router_tokens_per_expert=[0, 0, 0, 2, 1, 1, 28, 0, 0, 24, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 2, 28, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 18, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 22, 0, 0, 0, 29, 0, 0, 0, 4066, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4067, 3, 0, 6, 28, 0, 4061, 0, 0, 0, 36, 4062, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 1, 4055, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7992, 8186, 8188, 8192, 8192, 8134, 8158] dup_rows_per_dst=[50, 0, 6, 32, 106, 36, 72, 0] combine_rows_per_src=[8192, 7992, 8186, 8188, 8192, 8192, 8134, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229248 combine_payload_bytes=234520576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234749824 ingress_bytes=272269312 link_bytes=272269312 token_bytes=469270400 weight_movement_bytes=37748736 total_nvlink_bytes=507019136 weight_to_token_ratio=0.080441 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608750583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235085824 ingress_bytes=329228288 link_bytes=329228288 token_bytes=469942272 weight_movement_bytes=94371840 total_nvlink_bytes=564314112 weight_to_token_ratio=0.200816 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154503465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234463232 dispatch_meta_bytes=230176 combine_payload_bytes=234463232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234693408 ingress_bytes=253337600 link_bytes=253337600 token_bytes=469156640 weight_movement_bytes=18874368 total_nvlink_bytes=488031008 weight_to_token_ratio=0.040230 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427356233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=52 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 6, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 1, 6, 0, 4089, 0, 0, 0, 11, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8156, 8186, 8192, 8192, 8192, 8180, 8186] dup_rows_per_dst=[10, 0, 0, 4, 18, 8, 20, 0] combine_rows_per_src=[8192, 8156, 8186, 8192, 8192, 8192, 8180, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65246 payload_rows_remote=57054 dup_rows=290 dup_rows_remote=232 dedup_ratio=0.004425 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8042, 8166, 8192, 8144, 8192, 8130, 8188, 8192] dup_rows_per_dst=[4, 0, 0, 10, 162, 0, 56, 58] combine_rows_per_src=[8042, 8166, 8192, 8144, 8192, 8130, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65124 payload_rows_remote=56932 dup_rows=412 dup_rows_remote=338 dedup_ratio=0.006287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 7870, 8192, 8108, 8188] dup_rows_per_dst=[74, 322, 10, 2, 2, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 7870, 8192, 8108, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57312 dup_rows=44 dup_rows_remote=28 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4090, 0, 4, 0, 4090, 0, 0, 0, 6, 0, 1, 0, 0] payload_rows_per_dst=[8164, 8190, 8192, 8192, 8180, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 2, 0, 16, 2, 14, 10] combine_rows_per_src=[8164, 8190, 8192, 8192, 8180, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229344 combine_payload_bytes=234635264 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234864608 ingress_bytes=272384000 link_bytes=272384000 token_bytes=469499872 weight_movement_bytes=37748736 total_nvlink_bytes=507248608 weight_to_token_ratio=0.080402 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609849467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233693184 dispatch_meta_bytes=229144 combine_payload_bytes=233693184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233922328 ingress_bytes=252567552 link_bytes=252567552 token_bytes=467615512 weight_movement_bytes=18874368 total_nvlink_bytes=486489880 weight_to_token_ratio=0.040363 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419978012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233193472 dispatch_meta_bytes=229080 combine_payload_bytes=233193472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233422552 ingress_bytes=270942208 link_bytes=270942208 token_bytes=466616024 weight_movement_bytes=37748736 total_nvlink_bytes=504364760 weight_to_token_ratio=0.080899 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596034925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65358 payload_rows_remote=57166 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8014, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 178, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8014, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8180] dup_rows_per_dst=[2, 10, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57188 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 3, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 8, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 29, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 1, 0, 4066, 0, 0, 0, 6, 0, 0, 0, 4064, 0, 0, 0, 24, 0, 29, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 29, 0, 0, 2, 0, 4063, 0, 3, 0, 4066, 20, 0, 0, 28, 0, 6, 1, 0] payload_rows_per_dst=[8118, 8132, 8190, 8176, 8134, 8192, 8190, 8188] dup_rows_per_dst=[2, 0, 8, 0, 34, 58, 56, 58] combine_rows_per_src=[8118, 8132, 8190, 8176, 8134, 8192, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229360 combine_payload_bytes=234749952 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234979312 ingress_bytes=310247424 link_bytes=310247424 token_bytes=469729264 weight_movement_bytes=75497472 total_nvlink_bytes=545226736 weight_to_token_ratio=0.160726 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972638166e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085800 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942248 weight_movement_bytes=18874368 total_nvlink_bytes=488816616 weight_to_token_ratio=0.040163 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234151936 dispatch_meta_bytes=229376 combine_payload_bytes=234151936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234381312 ingress_bytes=253026304 link_bytes=253026304 token_bytes=468533248 weight_movement_bytes=18874368 total_nvlink_bytes=487407616 weight_to_token_ratio=0.040284 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424373548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234242048 dispatch_meta_bytes=229616 combine_payload_bytes=234242048 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234471664 ingress_bytes=271990784 link_bytes=271990784 token_bytes=468713712 weight_movement_bytes=37748736 total_nvlink_bytes=506462448 weight_to_token_ratio=0.080537 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606081864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=36 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4090, 0, 1, 0, 4090, 4, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8180, 8192, 8192, 8180, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 2, 0, 12, 14, 12, 10] combine_rows_per_src=[8168, 8180, 8192, 8192, 8180, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65454 payload_rows_remote=57262 dup_rows=82 dup_rows_remote=14 dedup_ratio=0.001251 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 1, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8184, 8170, 8190, 8148, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 2, 2, 4, 4, 68] combine_rows_per_src=[8186, 8184, 8170, 8190, 8148, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65238 payload_rows_remote=57048 dup_rows=298 dup_rows_remote=168 dedup_ratio=0.004547 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8192, 8190, 8114, 8112, 8058] dup_rows_per_dst=[130, 80, 4, 16, 64, 0, 0, 4] combine_rows_per_src=[8190, 8190, 8192, 8192, 8190, 8114, 8112, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57302 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 2, 0, 6, 6, 0, 4088, 0, 0, 6, 4090, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4088, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8186, 8182, 8186, 8180] dup_rows_per_dst=[28, 12, 2, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8186, 8182, 8186, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229320 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905544 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581768 weight_movement_bytes=18874368 total_nvlink_bytes=488456136 weight_to_token_ratio=0.040194 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234545152 dispatch_meta_bytes=229104 combine_payload_bytes=234545152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234774256 ingress_bytes=291168256 link_bytes=291168256 token_bytes=469319408 weight_movement_bytes=56623104 total_nvlink_bytes=525942512 weight_to_token_ratio=0.120649 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.789830965e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65428 payload_rows_remote=57236 dup_rows=108 dup_rows_remote=108 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 298, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8084, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 108, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8084, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233668608 dispatch_meta_bytes=228864 combine_payload_bytes=233668608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233897472 ingress_bytes=252542976 link_bytes=252542976 token_bytes=467566080 weight_movement_bytes=18874368 total_nvlink_bytes=486440448 weight_to_token_ratio=0.040367 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419742537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57162 dup_rows=190 dup_rows_remote=162 dedup_ratio=0.002899 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 24, 0, 18, 0, 0, 0, 0, 0, 29, 23, 0, 4061, 0, 0, 15, 4067, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 28, 0, 0, 0, 7, 0, 2, 0, 3, 0, 0, 0, 0, 4065, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4065, 0, 0, 0, 0, 0, 4, 25, 1, 0, 0, 0, 2, 4058, 0, 0, 4065, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6, 0, 21, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4059, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0] payload_rows_per_dst=[8188, 8184, 8188, 8136, 8178, 8142, 8178, 8152] dup_rows_per_dst=[84, 28, 20, 0, 4, 6, 44, 4] combine_rows_per_src=[8188, 8184, 8188, 8136, 8178, 8142, 8178, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229400 combine_payload_bytes=234708992 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234938392 ingress_bytes=310206464 link_bytes=310206464 token_bytes=469647384 weight_movement_bytes=75497472 total_nvlink_bytes=545144856 weight_to_token_ratio=0.160754 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972245707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234438656 dispatch_meta_bytes=229376 combine_payload_bytes=234438656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234668032 ingress_bytes=291061760 link_bytes=291061760 token_bytes=469106688 weight_movement_bytes=56623104 total_nvlink_bytes=525729792 weight_to_token_ratio=0.120704 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.788810573e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234135552 dispatch_meta_bytes=229296 combine_payload_bytes=234135552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234364848 ingress_bytes=290758656 link_bytes=290758656 token_bytes=468500400 weight_movement_bytes=56623104 total_nvlink_bytes=525123504 weight_to_token_ratio=0.120860 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.785906380e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57308 dup_rows=48 dup_rows_remote=46 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 6, 5, 0, 4089, 0, 0, 6, 4089, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 6, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4088, 0, 0, 4089, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 8192, 8192, 8178, 8190, 8180, 8186, 8180] dup_rows_per_dst=[18, 16, 6, 0, 2, 2, 4, 0] combine_rows_per_src=[8190, 8192, 8192, 8178, 8190, 8180, 8186, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57072 dup_rows=276 dup_rows_remote=210 dedup_ratio=0.004211 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8060, 8126, 8130, 8188, 8190, 8188, 8190, 8188] dup_rows_per_dst=[4, 4, 2, 0, 120, 64, 16, 66] combine_rows_per_src=[8060, 8126, 8130, 8188, 8190, 8188, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=32 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 490, 0, 0, 0, 1, 0, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 6] payload_rows_per_dst=[8192, 8180, 8180, 8192, 8192, 8188, 8176, 8192] dup_rows_per_dst=[4, 0, 0, 4, 12, 0, 0, 24] combine_rows_per_src=[8192, 8180, 8180, 8192, 8192, 8188, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=140 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8090, 8192, 8154, 8190] dup_rows_per_dst=[4, 12, 36, 90, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8090, 8192, 8154, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229416 combine_payload_bytes=234733568 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234962984 ingress_bytes=272482304 link_bytes=272482304 token_bytes=469696552 weight_movement_bytes=37748736 total_nvlink_bytes=507445288 weight_to_token_ratio=0.080368 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610791367e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=233766912 dispatch_meta_bytes=229128 combine_payload_bytes=233766912 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233996040 ingress_bytes=271515648 link_bytes=271515648 token_bytes=467762952 weight_movement_bytes=37748736 total_nvlink_bytes=505511688 weight_to_token_ratio=0.080701 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601529345e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65332 payload_rows_remote=57200 dup_rows=204 dup_rows_remote=204 dedup_ratio=0.003113 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 4064, 11, 0, 0, 0, 0, 0, 0, 0, 14, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 8, 4, 4063, 0, 29, 4, 0, 3, 0, 0, 0, 0, 0, 4, 4054, 5, 0, 0, 0, 0, 24, 4065, 0, 2, 0, 1898, 0, 0, 0, 0, 0, 1, 2167, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 0, 2, 29, 0, 0, 0, 26] payload_rows_per_dst=[8184, 8132, 8132, 8190, 8192, 8168, 8142, 8192] dup_rows_per_dst=[0, 0, 0, 48, 32, 12, 2, 110] combine_rows_per_src=[8184, 8132, 8132, 8190, 8192, 8168, 8142, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229360 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234520560 ingress_bytes=253165568 link_bytes=253165568 token_bytes=468811760 weight_movement_bytes=18874368 total_nvlink_bytes=487686128 weight_to_token_ratio=0.040260 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425707907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229328 combine_payload_bytes=234700800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234930128 ingress_bytes=291323904 link_bytes=291323904 token_bytes=469630928 weight_movement_bytes=56623104 total_nvlink_bytes=526254032 weight_to_token_ratio=0.120569 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791322308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229616 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234520816 ingress_bytes=290914304 link_bytes=290914304 token_bytes=468812016 weight_movement_bytes=56623104 total_nvlink_bytes=525435120 weight_to_token_ratio=0.120780 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787397722e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57306 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 1, 4089, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 371, 0, 0, 0, 1, 0, 0, 3713, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 6] payload_rows_per_dst=[8192, 8178, 8180, 8192, 8192, 8188, 8180, 8192] dup_rows_per_dst=[0, 0, 0, 12, 16, 0, 0, 14] combine_rows_per_src=[8192, 8178, 8180, 8192, 8192, 8188, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65326 payload_rows_remote=57134 dup_rows=210 dup_rows_remote=202 dedup_ratio=0.003204 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 2] payload_rows_per_dst=[8190, 8098, 8188, 8106, 8174, 8192, 8186, 8192] dup_rows_per_dst=[2, 0, 90, 2, 2, 20, 86, 8] combine_rows_per_src=[8190, 8098, 8188, 8106, 8174, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65422 payload_rows_remote=57242 dup_rows=114 dup_rows_remote=112 dedup_ratio=0.001740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 35, 0, 0, 0, 5, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8110, 8190, 8180, 8192, 8174, 8192] dup_rows_per_dst=[16, 0, 0, 0, 2, 16, 0, 80] combine_rows_per_src=[8192, 8192, 8110, 8190, 8180, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65326 payload_rows_remote=57134 dup_rows=210 dup_rows_remote=208 dedup_ratio=0.003204 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8146, 8192, 8190, 8034] dup_rows_per_dst=[2, 182, 0, 22, 2, 0, 0, 2] combine_rows_per_src=[8192, 8190, 8190, 8192, 8146, 8192, 8190, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229392 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234954768 ingress_bytes=272474112 link_bytes=272474112 token_bytes=469680144 weight_movement_bytes=37748736 total_nvlink_bytes=507428880 weight_to_token_ratio=0.080371 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610712876e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229344 combine_payload_bytes=234020864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234250208 ingress_bytes=252895232 link_bytes=252895232 token_bytes=468271072 weight_movement_bytes=18874368 total_nvlink_bytes=487145440 weight_to_token_ratio=0.040307 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423117681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64866 payload_rows_remote=56674 dup_rows=670 dup_rows_remote=670 dedup_ratio=0.010223 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7522, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 670] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7522, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8156, 8192] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 28] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65032 payload_rows_remote=56854 dup_rows=504 dup_rows_remote=498 dedup_ratio=0.007690 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 9, 4049, 0, 28, 0, 0, 0, 29, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 4058, 0, 4065, 0, 0, 0, 28, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3897, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 4066, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 19, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 25, 1, 161, 0, 2, 0, 10, 1, 15, 0, 0, 0] payload_rows_per_dst=[8174, 8178, 7814, 8184, 8132, 8192, 8168, 8190] dup_rows_per_dst=[58, 6, 0, 0, 0, 32, 38, 370] combine_rows_per_src=[8174, 8178, 7814, 8184, 8132, 8192, 8168, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234463232 dispatch_meta_bytes=229416 combine_payload_bytes=234463232 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234692648 ingress_bytes=272211968 link_bytes=272211968 token_bytes=469155880 weight_movement_bytes=37748736 total_nvlink_bytes=506904616 weight_to_token_ratio=0.080461 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608201141e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229368 combine_payload_bytes=234020864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234250232 ingress_bytes=252895232 link_bytes=252895232 token_bytes=468271096 weight_movement_bytes=18874368 total_nvlink_bytes=487145464 weight_to_token_ratio=0.040306 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423117681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=232136704 dispatch_meta_bytes=229376 combine_payload_bytes=232136704 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232366080 ingress_bytes=288759808 link_bytes=288759808 token_bytes=464502784 weight_movement_bytes=56623104 total_nvlink_bytes=521125888 weight_to_token_ratio=0.121900 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.766754402e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=232873984 dispatch_meta_bytes=229408 combine_payload_bytes=232873984 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233103392 ingress_bytes=308371456 link_bytes=308371456 token_bytes=465977376 weight_movement_bytes=75497472 total_nvlink_bytes=541474848 weight_to_token_ratio=0.162020 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.954663563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=42 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 8, 0, 1, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8192, 8180, 8192, 8188, 8192] dup_rows_per_dst=[12, 0, 0, 0, 0, 4, 4, 26] combine_rows_per_src=[8192, 8192, 8162, 8192, 8180, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64208 payload_rows_remote=56016 dup_rows=1328 dup_rows_remote=1322 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0] payload_rows_per_dst=[8102, 6966, 8184, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 504, 2, 88, 0, 724, 6] combine_rows_per_src=[8102, 6966, 8184, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64594 payload_rows_remote=56402 dup_rows=942 dup_rows_remote=930 dedup_ratio=0.014374 router_tokens_per_expert=[0, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 131, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 4088, 0, 0, 4089, 0, 1, 0, 0, 0, 6, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 327, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4090, 0, 0] payload_rows_per_dst=[7282, 8190, 8190, 8190, 8192, 8192, 8178, 8180] dup_rows_per_dst=[12, 262, 0, 0, 12, 6, 0, 650] combine_rows_per_src=[7282, 8190, 8190, 8190, 8192, 8192, 8178, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64278 payload_rows_remote=56086 dup_rows=1258 dup_rows_remote=1240 dedup_ratio=0.019196 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8190, 8136, 8186, 7004] dup_rows_per_dst=[18, 1234, 0, 0, 0, 4, 0, 2] combine_rows_per_src=[8192, 8192, 8186, 8192, 8190, 8136, 8186, 7004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229360 combine_payload_bytes=234692608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234921968 ingress_bytes=272441344 link_bytes=272441344 token_bytes=469614576 weight_movement_bytes=37748736 total_nvlink_bytes=507363312 weight_to_token_ratio=0.080382 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610398909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=229441536 dispatch_meta_bytes=229352 combine_payload_bytes=229441536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229670888 ingress_bytes=248315904 link_bytes=248315904 token_bytes=459112424 weight_movement_bytes=18874368 total_nvlink_bytes=477986792 weight_to_token_ratio=0.041111 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379240813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64442 payload_rows_remote=56250 dup_rows=1094 dup_rows_remote=1094 dedup_ratio=0.016693 router_tokens_per_expert=[0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[7098, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 544, 0, 0, 0, 0, 0, 550] combine_rows_per_src=[7098, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64432 payload_rows_remote=56240 dup_rows=1104 dup_rows_remote=1104 dedup_ratio=0.016846 router_tokens_per_expert=[0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[7088, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 192, 0, 0, 0, 0, 0, 912] combine_rows_per_src=[7088, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231022592 dispatch_meta_bytes=229328 combine_payload_bytes=231022592 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231251920 ingress_bytes=268771328 link_bytes=268771328 token_bytes=462274512 weight_movement_bytes=37748736 total_nvlink_bytes=500023248 weight_to_token_ratio=0.081659 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.575234621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61166 payload_rows_remote=52984 dup_rows=4370 dup_rows_remote=4226 dedup_ratio=0.066681 router_tokens_per_expert=[0, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 30, 3, 0, 0, 70, 29, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 27, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4061, 28, 0, 16, 0, 0, 4055, 0, 0, 4059, 0, 6, 0, 0, 0, 7, 0, 6, 0, 0, 29, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 6, 26, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 964, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 4062, 0, 0] payload_rows_per_dst=[7498, 8182, 8174, 8178, 8192, 8192, 8132, 4618] dup_rows_per_dst=[3528, 144, 0, 56, 20, 38, 2, 582] combine_rows_per_src=[7498, 8182, 8174, 8178, 8192, 8192, 8132, 4618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=229728256 dispatch_meta_bytes=229304 combine_payload_bytes=229728256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229957560 ingress_bytes=267476992 link_bytes=267476992 token_bytes=459685816 weight_movement_bytes=37748736 total_nvlink_bytes=497434552 weight_to_token_ratio=0.082119 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.562832930e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=230400000 dispatch_meta_bytes=229376 combine_payload_bytes=230400000 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=230629376 ingress_bytes=324771840 link_bytes=324771840 token_bytes=461029376 weight_movement_bytes=94371840 total_nvlink_bytes=555401216 weight_to_token_ratio=0.204698 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.111803973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=230359040 dispatch_meta_bytes=229376 combine_payload_bytes=230359040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230588416 ingress_bytes=249233408 link_bytes=249233408 token_bytes=460947456 weight_movement_bytes=18874368 total_nvlink_bytes=479821824 weight_to_token_ratio=0.040947 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388031885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=217022464 dispatch_meta_bytes=228840 combine_payload_bytes=217022464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217251304 ingress_bytes=235896832 link_bytes=235896832 token_bytes=434273768 weight_movement_bytes=18874368 total_nvlink_bytes=453148136 weight_to_token_ratio=0.043462 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260247376e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63462 payload_rows_remote=55270 dup_rows=2074 dup_rows_remote=2068 dedup_ratio=0.031647 router_tokens_per_expert=[0, 0, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 35, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4089, 6, 0, 1, 0, 0, 4089, 1, 0, 4088, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4089, 0, 0] payload_rows_per_dst=[6152, 8188, 8188, 8192, 8192, 8192, 8178, 8180] dup_rows_per_dst=[10, 70, 0, 14, 2, 6, 0, 1972] combine_rows_per_src=[6152, 8188, 8188, 8192, 8192, 8192, 8178, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61294 payload_rows_remote=53106 dup_rows=4242 dup_rows_remote=106 dedup_ratio=0.064728 router_tokens_per_expert=[0, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 2, 0, 1, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[5320, 6952, 8192, 8122, 8192, 8140, 8188, 8188] dup_rows_per_dst=[4, 4, 0, 4, 0, 2, 92, 4136] combine_rows_per_src=[5320, 6952, 8192, 8122, 8192, 8140, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57312 dup_rows=44 dup_rows_remote=40 dedup_ratio=0.000671 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 4089, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 6, 0, 6, 0, 0, 0, 4084, 0, 0, 2, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8178, 8186, 8192, 8192, 8180, 8192, 8180, 8192] dup_rows_per_dst=[0, 0, 0, 10, 4, 14, 16, 0] combine_rows_per_src=[8178, 8186, 8192, 8192, 8180, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59624 payload_rows_remote=51432 dup_rows=5912 dup_rows_remote=62 dedup_ratio=0.090210 router_tokens_per_expert=[0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8158, 8192, 8190, 2316] dup_rows_per_dst=[5850, 26, 24, 12, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8158, 8192, 8190, 2316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=226385920 dispatch_meta_bytes=229352 combine_payload_bytes=226385920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226615272 ingress_bytes=245260288 link_bytes=245260288 token_bytes=453001192 weight_movement_bytes=18874368 total_nvlink_bytes=471875560 weight_to_token_ratio=0.041665 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.349963404e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=217522176 dispatch_meta_bytes=212848 combine_payload_bytes=217522176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=217735024 ingress_bytes=274145280 link_bytes=274145280 token_bytes=435257200 weight_movement_bytes=56623104 total_nvlink_bytes=491880304 weight_to_token_ratio=0.130091 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.626725185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229408 combine_payload_bytes=234749952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234979360 ingress_bytes=253624320 link_bytes=253624320 token_bytes=469729312 weight_movement_bytes=18874368 total_nvlink_bytes=488603680 weight_to_token_ratio=0.040181 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430103443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65370 payload_rows_remote=57240 dup_rows=166 dup_rows_remote=166 dedup_ratio=0.002533 router_tokens_per_expert=[0, 4065, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4065, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 30, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 2, 6, 0, 11, 0, 31, 0, 0, 0, 4056, 0, 0, 18, 0, 0, 31, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4, 0] payload_rows_per_dst=[8176, 8130, 8184, 8190, 8134, 8192, 8174, 8190] dup_rows_per_dst=[0, 0, 0, 8, 18, 38, 94, 8] combine_rows_per_src=[8176, 8130, 8184, 8190, 8134, 8192, 8174, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=210665472 dispatch_meta_bytes=205976 combine_payload_bytes=210665472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210871448 ingress_bytes=229539840 link_bytes=229539840 token_bytes=421536920 weight_movement_bytes=18874368 total_nvlink_bytes=440411288 weight_to_token_ratio=0.044775 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.199337806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102200 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975032 weight_movement_bytes=37748736 total_nvlink_bytes=507723768 weight_to_token_ratio=0.080321 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229624 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234684664 ingress_bytes=272203776 link_bytes=272203776 token_bytes=469139704 weight_movement_bytes=37748736 total_nvlink_bytes=506888440 weight_to_token_ratio=0.080464 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608122649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=36 dedup_ratio=0.000763 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 1, 0, 6, 0, 6, 0, 0, 0, 4085, 0, 0, 6, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8180, 8178, 8192, 8192, 8180, 8192, 8182, 8190] dup_rows_per_dst=[2, 0, 0, 10, 0, 14, 24, 0] combine_rows_per_src=[8180, 8178, 8192, 8192, 8180, 8192, 8182, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8180, 8188, 8192, 8192, 8188, 8188, 8180, 8192] dup_rows_per_dst=[0, 0, 8, 12, 0, 4, 12, 0] combine_rows_per_src=[8180, 8188, 8192, 8192, 8188, 8188, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=52 dedup_ratio=0.001343 router_tokens_per_expert=[0, 4089, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 6, 0, 0, 0, 4065, 6, 4090, 0, 0, 0, 0, 1] payload_rows_per_dst=[8180, 8190, 8192, 8192, 8192, 8180, 8178, 8144] dup_rows_per_dst=[2, 12, 14, 0, 36, 0, 10, 14] combine_rows_per_src=[8180, 8190, 8192, 8192, 8192, 8180, 8178, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65310 payload_rows_remote=57120 dup_rows=226 dup_rows_remote=172 dedup_ratio=0.003448 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8138, 8190, 8124, 8096, 8188] dup_rows_per_dst=[54, 86, 0, 14, 68, 2, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8138, 8190, 8124, 8096, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229320 combine_payload_bytes=234676224 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234905544 ingress_bytes=272424960 link_bytes=272424960 token_bytes=469581768 weight_movement_bytes=37748736 total_nvlink_bytes=507330504 weight_to_token_ratio=0.080388 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610241925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229232 combine_payload_bytes=234520576 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234749808 ingress_bytes=253394944 link_bytes=253394944 token_bytes=469270384 weight_movement_bytes=18874368 total_nvlink_bytes=488144752 weight_to_token_ratio=0.040221 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427905675e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65306 payload_rows_remote=57114 dup_rows=230 dup_rows_remote=230 dedup_ratio=0.003510 router_tokens_per_expert=[0, 4096, 0, 0, 3981, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[7962, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 230, 0] combine_rows_per_src=[7962, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65158 payload_rows_remote=56978 dup_rows=378 dup_rows_remote=324 dedup_ratio=0.005768 router_tokens_per_expert=[0, 4060, 0, 0, 4066, 0, 0, 0, 0, 0, 4065, 1, 1, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 11, 0, 19, 0, 0, 0, 2, 0, 28, 0, 11, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 7, 0, 29, 0, 0, 0, 0, 0, 0, 0, 45, 4, 1, 0, 28, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4063, 41, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 4054, 0, 0, 24, 0, 0, 0, 3964, 30, 4064, 0, 0, 0, 0, 5] payload_rows_per_dst=[8134, 8180, 8192, 8192, 8192, 8136, 8134, 7998] dup_rows_per_dst=[2, 54, 58, 14, 66, 0, 136, 48] combine_rows_per_src=[8134, 8180, 8192, 8192, 8192, 8136, 8134, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57344 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233963520 dispatch_meta_bytes=229168 combine_payload_bytes=233963520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234192688 ingress_bytes=290586624 link_bytes=290586624 token_bytes=468156208 weight_movement_bytes=56623104 total_nvlink_bytes=524779312 weight_to_token_ratio=0.120949 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784258054e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233938944 dispatch_meta_bytes=229376 combine_payload_bytes=233938944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234168320 ingress_bytes=252813312 link_bytes=252813312 token_bytes=468107264 weight_movement_bytes=18874368 total_nvlink_bytes=486981632 weight_to_token_ratio=0.040321 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422332764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233381888 dispatch_meta_bytes=229208 combine_payload_bytes=233381888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233611096 ingress_bytes=252256256 link_bytes=252256256 token_bytes=466992984 weight_movement_bytes=18874368 total_nvlink_bytes=485867352 weight_to_token_ratio=0.040417 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416995327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229408 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110432 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991456 weight_movement_bytes=18874368 total_nvlink_bytes=488865824 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65438 payload_rows_remote=57256 dup_rows=98 dup_rows_remote=96 dedup_ratio=0.001495 router_tokens_per_expert=[0, 4088, 0, 0, 4089, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 0, 4089, 1, 0, 6, 1, 0, 0, 4059, 6, 4089, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8190, 8190, 8192, 8192, 8182, 8182, 8132] dup_rows_per_dst=[0, 12, 8, 2, 0, 2, 60, 14] combine_rows_per_src=[8178, 8190, 8190, 8192, 8192, 8182, 8182, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57302 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[6, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8180, 8180, 8190, 8180, 8178] dup_rows_per_dst=[24, 2, 10, 2, 0, 14, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 8180, 8180, 8190, 8180, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65180 payload_rows_remote=57004 dup_rows=356 dup_rows_remote=24 dedup_ratio=0.005432 router_tokens_per_expert=[0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 4086, 2, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8106, 8192, 8076, 8188, 8062, 8176] dup_rows_per_dst=[0, 4, 4, 0, 12, 0, 4, 332] combine_rows_per_src=[8188, 8192, 8106, 8192, 8076, 8188, 8062, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65124 payload_rows_remote=56934 dup_rows=412 dup_rows_remote=90 dedup_ratio=0.006287 router_tokens_per_expert=[0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4091, 1, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8124, 8192, 8176, 8192, 8190, 7958, 8102] dup_rows_per_dst=[322, 2, 4, 0, 2, 82, 0, 0] combine_rows_per_src=[8190, 8124, 8192, 8176, 8192, 8190, 7958, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229408 combine_payload_bytes=234520576 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234749984 ingress_bytes=272269312 link_bytes=272269312 token_bytes=469270560 weight_movement_bytes=37748736 total_nvlink_bytes=507019296 weight_to_token_ratio=0.080441 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608750583e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229424 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234938416 ingress_bytes=272457728 link_bytes=272457728 token_bytes=469647408 weight_movement_bytes=37748736 total_nvlink_bytes=507396144 weight_to_token_ratio=0.080377 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610555892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233488384 dispatch_meta_bytes=228112 combine_payload_bytes=233488384 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233716496 ingress_bytes=308985856 link_bytes=308985856 token_bytes=467204880 weight_movement_bytes=75497472 total_nvlink_bytes=542702352 weight_to_token_ratio=0.161594 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.960550442e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65302 payload_rows_remote=57110 dup_rows=234 dup_rows_remote=0 dedup_ratio=0.003571 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 117, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7958] dup_rows_per_dst=[0, 0, 0, 234, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57068 dup_rows=276 dup_rows_remote=262 dedup_ratio=0.004211 router_tokens_per_expert=[29, 4065, 0, 0, 0, 2, 0, 0, 0, 30, 0, 0, 0, 0, 31, 0, 0, 0, 29, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 6, 0, 0, 0, 13, 0, 0, 0, 25, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4062, 0, 0, 0, 0, 21, 0, 0, 0, 4, 0, 13, 0, 0, 0, 0, 0, 0, 5, 4063, 0, 4065, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 8136, 8088, 8178, 8160, 8140] dup_rows_per_dst=[122, 14, 76, 2, 0, 8, 44, 10] combine_rows_per_src=[8192, 8192, 8174, 8136, 8088, 8178, 8160, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233201664 dispatch_meta_bytes=228096 combine_payload_bytes=233201664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233429760 ingress_bytes=289824768 link_bytes=289824768 token_bytes=466631424 weight_movement_bytes=56623104 total_nvlink_bytes=523254528 weight_to_token_ratio=0.121344 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.776958324e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64846 payload_rows_remote=56654 dup_rows=690 dup_rows_remote=690 dedup_ratio=0.010529 router_tokens_per_expert=[0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[7502, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 0, 0, 0, 0, 676] combine_rows_per_src=[7502, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=228440 combine_payload_bytes=233922560 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234151000 ingress_bytes=271671296 link_bytes=271671296 token_bytes=468073560 weight_movement_bytes=37748736 total_nvlink_bytes=505822296 weight_to_token_ratio=0.080647 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603020688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=233750528 dispatch_meta_bytes=229320 combine_payload_bytes=233750528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233979848 ingress_bytes=252624896 link_bytes=252624896 token_bytes=467730376 weight_movement_bytes=18874368 total_nvlink_bytes=486604744 weight_to_token_ratio=0.040353 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420527454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57290 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000854 router_tokens_per_expert=[6, 4090, 0, 0, 0, 1, 0, 0, 1, 6, 0, 1, 0, 0, 6, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8178, 8170, 8190, 8188, 8180] dup_rows_per_dst=[30, 0, 12, 0, 0, 0, 12, 2] combine_rows_per_src=[8192, 8192, 8190, 8178, 8170, 8190, 8188, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=232054784 dispatch_meta_bytes=229376 combine_payload_bytes=232054784 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232284160 ingress_bytes=269803520 link_bytes=269803520 token_bytes=464338944 weight_movement_bytes=37748736 total_nvlink_bytes=502087680 weight_to_token_ratio=0.081296 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.585124577e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=58 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4082, 4090, 0, 4090, 6, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4073, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8182, 8188, 8192, 8192, 8158, 8184] dup_rows_per_dst=[0, 46, 0, 2, 12, 10, 0, 0] combine_rows_per_src=[8178, 8192, 8182, 8188, 8192, 8192, 8158, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65052 payload_rows_remote=56864 dup_rows=484 dup_rows_remote=180 dedup_ratio=0.007385 router_tokens_per_expert=[2, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7904, 8176, 8192, 8034, 8178, 8192, 8188, 8188] dup_rows_per_dst=[10, 0, 6, 2, 150, 0, 12, 304] combine_rows_per_src=[7904, 8176, 8192, 8034, 8178, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229384 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889224 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549064 weight_movement_bytes=18874368 total_nvlink_bytes=488423432 weight_to_token_ratio=0.040197 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65432 payload_rows_remote=57240 dup_rows=104 dup_rows_remote=2 dedup_ratio=0.001587 router_tokens_per_expert=[1, 4095, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8192, 8110, 8190, 8192, 8188, 8176, 8192] dup_rows_per_dst=[102, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8110, 8190, 8192, 8188, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229328 combine_payload_bytes=234594304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234823632 ingress_bytes=253468672 link_bytes=253468672 token_bytes=469417936 weight_movement_bytes=18874368 total_nvlink_bytes=488292304 weight_to_token_ratio=0.040208 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428612101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=232914944 dispatch_meta_bytes=228176 combine_payload_bytes=232914944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233143120 ingress_bytes=270663680 link_bytes=270663680 token_bytes=466058064 weight_movement_bytes=37748736 total_nvlink_bytes=503806800 weight_to_token_ratio=0.080996 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.593366207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65406 payload_rows_remote=57214 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8062, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 130, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8062, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57082 dup_rows=276 dup_rows_remote=136 dedup_ratio=0.004211 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 29, 0, 4053, 4064, 0, 4064, 6, 0, 0, 0, 0, 0, 7, 2, 0, 4063, 0, 0, 63, 18, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 2, 30, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 2, 3995, 0, 27, 4066, 0, 0, 0, 0, 0, 0, 4064, 0, 4, 4062, 0, 0, 0, 3, 3, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8190, 8178, 8180, 8132, 8192, 8188, 8054, 8146] dup_rows_per_dst=[0, 140, 0, 2, 66, 52, 2, 14] combine_rows_per_src=[8190, 8178, 8180, 8132, 8192, 8188, 8054, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=228968 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234684008 ingress_bytes=272203776 link_bytes=272203776 token_bytes=469139048 weight_movement_bytes=37748736 total_nvlink_bytes=506887784 weight_to_token_ratio=0.080464 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608122649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234348544 dispatch_meta_bytes=229376 combine_payload_bytes=234348544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234577920 ingress_bytes=272097280 link_bytes=272097280 token_bytes=468926464 weight_movement_bytes=37748736 total_nvlink_bytes=506675200 weight_to_token_ratio=0.080500 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607102257e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233807872 dispatch_meta_bytes=228872 combine_payload_bytes=233807872 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234036744 ingress_bytes=328179712 link_bytes=328179712 token_bytes=467844616 weight_movement_bytes=94371840 total_nvlink_bytes=562216456 weight_to_token_ratio=0.201716 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.144456526e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=76 dedup_ratio=0.001373 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4071, 4090, 1, 4089, 6, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4077, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8192, 8178, 8190, 8192, 8192, 8166, 8178] dup_rows_per_dst=[0, 56, 0, 8, 12, 14, 0, 0] combine_rows_per_src=[8158, 8192, 8178, 8190, 8192, 8192, 8166, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65006 payload_rows_remote=56814 dup_rows=530 dup_rows_remote=36 dedup_ratio=0.008087 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 4084, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 1, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4090, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 3855] payload_rows_per_dst=[8172, 8192, 8192, 8178, 8192, 8178, 8180, 7722] dup_rows_per_dst=[0, 12, 24, 0, 494, 0, 0, 0] combine_rows_per_src=[8172, 8192, 8192, 8178, 8192, 8178, 8180, 7722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65132 payload_rows_remote=56944 dup_rows=404 dup_rows_remote=348 dedup_ratio=0.006165 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4085, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7912, 8192, 8188, 8096, 8192, 8174, 8188] dup_rows_per_dst=[0, 20, 38, 0, 4, 4, 282, 56] combine_rows_per_src=[8190, 7912, 8192, 8188, 8096, 8192, 8174, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229320 combine_payload_bytes=234512384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234741704 ingress_bytes=253386752 link_bytes=253386752 token_bytes=469254088 weight_movement_bytes=18874368 total_nvlink_bytes=488128456 weight_to_token_ratio=0.040222 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427827184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232710144 dispatch_meta_bytes=227400 combine_payload_bytes=232710144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232937544 ingress_bytes=251584512 link_bytes=251584512 token_bytes=465647688 weight_movement_bytes=18874368 total_nvlink_bytes=484522056 weight_to_token_ratio=0.040534 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410559007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65290 payload_rows_remote=57098 dup_rows=246 dup_rows_remote=64 dedup_ratio=0.003754 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8094, 8190, 8190, 8192, 8116, 8124, 8192] dup_rows_per_dst=[182, 0, 0, 60, 2, 0, 0, 2] combine_rows_per_src=[8192, 8094, 8190, 8190, 8192, 8116, 8124, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233242624 dispatch_meta_bytes=229168 combine_payload_bytes=233242624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233471792 ingress_bytes=252116992 link_bytes=252116992 token_bytes=466714416 weight_movement_bytes=18874368 total_nvlink_bytes=485588784 weight_to_token_ratio=0.040441 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415660968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65292 payload_rows_remote=57100 dup_rows=244 dup_rows_remote=244 dedup_ratio=0.003723 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3974] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7948] dup_rows_per_dst=[0, 0, 0, 0, 244, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65050 payload_rows_remote=56858 dup_rows=486 dup_rows_remote=416 dedup_ratio=0.007416 router_tokens_per_expert=[0, 0, 0, 30, 0, 0, 3995, 0, 0, 4062, 3, 20, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 30, 3, 1, 5, 0, 0, 0, 0, 0, 30, 0, 31, 0, 0, 0, 17, 0, 2, 0, 0, 0, 0, 4065, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 11, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4057, 4062, 0, 0, 0, 29, 0, 0, 0, 0, 3975] payload_rows_per_dst=[8092, 8192, 8192, 8116, 8188, 8132, 8130, 8008] dup_rows_per_dst=[4, 70, 104, 0, 296, 0, 12, 0] combine_rows_per_src=[8092, 8192, 8192, 8116, 8188, 8132, 8130, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4069] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8138] dup_rows_per_dst=[0, 0, 0, 0, 54, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233873408 dispatch_meta_bytes=228648 combine_payload_bytes=233873408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234102056 ingress_bytes=271622144 link_bytes=271622144 token_bytes=467975464 weight_movement_bytes=37748736 total_nvlink_bytes=505724200 weight_to_token_ratio=0.080664 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602549737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 6, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233881600 dispatch_meta_bytes=229376 combine_payload_bytes=233881600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234110976 ingress_bytes=271630336 link_bytes=271630336 token_bytes=467992576 weight_movement_bytes=37748736 total_nvlink_bytes=505741312 weight_to_token_ratio=0.080661 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602628229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232890368 dispatch_meta_bytes=229096 combine_payload_bytes=232890368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233119464 ingress_bytes=251764736 link_bytes=251764736 token_bytes=466009832 weight_movement_bytes=18874368 total_nvlink_bytes=484884200 weight_to_token_ratio=0.040502 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412285824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229376 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889216 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549056 weight_movement_bytes=18874368 total_nvlink_bytes=488423424 weight_to_token_ratio=0.040197 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57252 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 4085, 0, 0, 4089, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 4089, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 4083, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 4057] payload_rows_per_dst=[8188, 8192, 8192, 8180, 8190, 8178, 8180, 8130] dup_rows_per_dst=[2, 14, 14, 0, 76, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8180, 8190, 8178, 8180, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=48 dedup_ratio=0.001221 router_tokens_per_expert=[2, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 4090, 4090, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8180, 8180, 8152, 8192, 8180, 8190, 8192] dup_rows_per_dst=[16, 0, 28, 0, 32, 0, 2, 2] combine_rows_per_src=[8190, 8180, 8180, 8152, 8192, 8180, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235085800 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942248 weight_movement_bytes=56623104 total_nvlink_bytes=526565352 weight_to_token_ratio=0.120489 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64184 payload_rows_remote=56008 dup_rows=1352 dup_rows_remote=140 dedup_ratio=0.020630 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 4091, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4094, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 4083] payload_rows_per_dst=[8160, 8090, 8192, 8188, 7002, 8188, 8188, 8176] dup_rows_per_dst=[0, 4, 6, 108, 22, 0, 0, 1212] combine_rows_per_src=[8160, 8090, 8192, 8188, 7002, 8188, 8188, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234504192 dispatch_meta_bytes=229432 combine_payload_bytes=234504192 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234733624 ingress_bytes=272252928 link_bytes=272252928 token_bytes=469237816 weight_movement_bytes=37748736 total_nvlink_bytes=506986552 weight_to_token_ratio=0.080447 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608593599e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229248 combine_payload_bytes=234553344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234782592 ingress_bytes=253427712 link_bytes=253427712 token_bytes=469335936 weight_movement_bytes=18874368 total_nvlink_bytes=488210304 weight_to_token_ratio=0.040215 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428219642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65202 payload_rows_remote=57010 dup_rows=334 dup_rows_remote=184 dedup_ratio=0.005096 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8192, 8192, 8040, 8190, 8192, 8190, 8190, 8016] dup_rows_per_dst=[150, 2, 4, 0, 4, 94, 80, 0] combine_rows_per_src=[8192, 8192, 8040, 8190, 8192, 8190, 8190, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=229408768 dispatch_meta_bytes=224592 combine_payload_bytes=229408768 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=229633360 ingress_bytes=342654976 link_bytes=342654976 token_bytes=459042128 weight_movement_bytes=113246208 total_nvlink_bytes=572288336 weight_to_token_ratio=0.246701 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.283151383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65380 payload_rows_remote=57188 dup_rows=156 dup_rows_remote=156 dedup_ratio=0.002380 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8036] dup_rows_per_dst=[0, 0, 152, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65274 payload_rows_remote=57144 dup_rows=262 dup_rows_remote=258 dedup_ratio=0.003998 router_tokens_per_expert=[9, 0, 1, 0, 0, 4062, 0, 30, 0, 0, 0, 0, 0, 0, 0, 28, 2, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4064, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4065, 0, 4041, 0, 0, 10, 0, 0, 0, 0, 22, 0, 0, 2, 0, 0, 0, 7, 31, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4066, 0, 4064, 4064, 2, 0, 30, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8130, 8140, 8132, 8192, 8128, 8176, 8192] dup_rows_per_dst=[76, 4, 48, 16, 96, 0, 4, 18] combine_rows_per_src=[8184, 8130, 8140, 8132, 8192, 8128, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233512960 dispatch_meta_bytes=228776 combine_payload_bytes=233512960 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233741736 ingress_bytes=290136064 link_bytes=290136064 token_bytes=467254696 weight_movement_bytes=56623104 total_nvlink_bytes=523877800 weight_to_token_ratio=0.121183 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.779941009e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234242048 dispatch_meta_bytes=229376 combine_payload_bytes=234242048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234471424 ingress_bytes=290865152 link_bytes=290865152 token_bytes=468713472 weight_movement_bytes=56623104 total_nvlink_bytes=525336576 weight_to_token_ratio=0.120805 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.786926772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234061824 dispatch_meta_bytes=229608 combine_payload_bytes=234061824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234291432 ingress_bytes=290684928 link_bytes=290684928 token_bytes=468353256 weight_movement_bytes=56623104 total_nvlink_bytes=524976360 weight_to_token_ratio=0.120898 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.785199954e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57296 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[5, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 3, 0, 1, 0, 0, 0, 4090, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 4090, 4090, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8180, 8170, 8192, 8180, 8192, 8192] dup_rows_per_dst=[22, 0, 16, 2, 18, 0, 0, 2] combine_rows_per_src=[8192, 8178, 8180, 8170, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57284 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 1, 0, 1, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 6, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4089, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8180, 8190, 8190, 8190, 8192, 8178, 8178] dup_rows_per_dst=[6, 2, 2, 28, 0, 24, 0, 0] combine_rows_per_src=[8176, 8180, 8190, 8190, 8190, 8192, 8178, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65170 payload_rows_remote=56978 dup_rows=366 dup_rows_remote=140 dedup_ratio=0.005585 router_tokens_per_expert=[0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 4094, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8084, 8182, 7992, 8148, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 96, 18, 0, 8, 0, 14, 226] combine_rows_per_src=[8084, 8182, 7992, 8148, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229424 combine_payload_bytes=234684416 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234913840 ingress_bytes=272433152 link_bytes=272433152 token_bytes=469598256 weight_movement_bytes=37748736 total_nvlink_bytes=507346992 weight_to_token_ratio=0.080385 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610320417e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229384 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234864648 ingress_bytes=253509632 link_bytes=253509632 token_bytes=469499912 weight_movement_bytes=18874368 total_nvlink_bytes=488374280 weight_to_token_ratio=0.040201 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429004559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233381888 dispatch_meta_bytes=228472 combine_payload_bytes=233381888 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233610360 ingress_bytes=308879360 link_bytes=308879360 token_bytes=466992248 weight_movement_bytes=75497472 total_nvlink_bytes=542489720 weight_to_token_ratio=0.161668 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.959530049e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64836 payload_rows_remote=56644 dup_rows=700 dup_rows_remote=0 dedup_ratio=0.010681 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7492, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 700, 0, 0, 0, 0] combine_rows_per_src=[7492, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57078 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 4095, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8028, 8190, 8190, 8098] dup_rows_per_dst=[0, 0, 14, 150, 6, 94, 0, 2] combine_rows_per_src=[8192, 8190, 8192, 8190, 8028, 8190, 8190, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65286 payload_rows_remote=57160 dup_rows=250 dup_rows_remote=250 dedup_ratio=0.003815 router_tokens_per_expert=[0, 0, 0, 0, 13, 5, 0, 2, 6, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 2, 2, 4, 0, 0, 29, 5, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 3, 4062, 0, 0, 30, 0, 0, 31, 4062, 1, 0, 0, 31, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4060, 0, 1, 0, 0, 5, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0] payload_rows_per_dst=[8132, 8126, 8182, 8184, 8186, 8192, 8122, 8162] dup_rows_per_dst=[48, 0, 2, 76, 0, 124, 0, 0] combine_rows_per_src=[8132, 8126, 8182, 8184, 8186, 8192, 8122, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232013824 dispatch_meta_bytes=226576 combine_payload_bytes=232013824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232240400 ingress_bytes=250888192 link_bytes=250888192 token_bytes=464254224 weight_movement_bytes=18874368 total_nvlink_bytes=483128592 weight_to_token_ratio=0.040655 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403887211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233791488 dispatch_meta_bytes=229376 combine_payload_bytes=233791488 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234020864 ingress_bytes=271540224 link_bytes=271540224 token_bytes=467812352 weight_movement_bytes=37748736 total_nvlink_bytes=505561088 weight_to_token_ratio=0.080692 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601764820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=229640 combine_payload_bytes=234127360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234357000 ingress_bytes=253001728 link_bytes=253001728 token_bytes=468484360 weight_movement_bytes=18874368 total_nvlink_bytes=487358728 weight_to_token_ratio=0.040288 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424138073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57296 dup_rows=48 dup_rows_remote=24 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 1, 3, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 6, 0, 0, 6, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0] payload_rows_per_dst=[8180, 8180, 8192, 8192, 8192, 8192, 8176, 8184] dup_rows_per_dst=[8, 0, 0, 16, 0, 24, 0, 0] combine_rows_per_src=[8180, 8180, 8192, 8192, 8192, 8192, 8176, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65038 payload_rows_remote=57314 dup_rows=498 dup_rows_remote=496 dedup_ratio=0.007599 router_tokens_per_expert=[0, 0, 0, 6, 0, 26, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 6, 0, 1, 0, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 208, 0, 4089, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 7724, 8180, 8190, 8180] dup_rows_per_dst=[52, 2, 0, 12, 2, 2, 428, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 7724, 8180, 8190, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64618 payload_rows_remote=56428 dup_rows=918 dup_rows_remote=862 dedup_ratio=0.014008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8188, 8188, 8192, 7372, 8192, 8108, 8188, 8190] dup_rows_per_dst=[0, 2, 0, 4, 0, 8, 848, 56] combine_rows_per_src=[8188, 8188, 8192, 7372, 8192, 8108, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229280 combine_payload_bytes=234684416 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234913696 ingress_bytes=310181888 link_bytes=310181888 token_bytes=469598112 weight_movement_bytes=75497472 total_nvlink_bytes=545095584 weight_to_token_ratio=0.160770 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972010232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=231240 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234989384 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469747528 weight_movement_bytes=18874368 total_nvlink_bytes=488621896 weight_to_token_ratio=0.040180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=231129088 dispatch_meta_bytes=229160 combine_payload_bytes=231129088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231358248 ingress_bytes=250003456 link_bytes=250003456 token_bytes=462487336 weight_movement_bytes=18874368 total_nvlink_bytes=481361704 weight_to_token_ratio=0.040811 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.395410106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.001343 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8104, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 88, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8104, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64684 payload_rows_remote=56494 dup_rows=852 dup_rows_remote=120 dedup_ratio=0.013000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8190, 8192, 8190, 8092, 7448, 8190] dup_rows_per_dst=[732, 90, 10, 6, 10, 4, 0, 0] combine_rows_per_src=[8190, 8192, 8190, 8192, 8190, 8092, 7448, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63552 payload_rows_remote=55412 dup_rows=1984 dup_rows_remote=1984 dedup_ratio=0.030273 router_tokens_per_expert=[0, 0, 0, 31, 0, 41, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4064, 0, 0, 0, 0, 24, 0, 5, 0, 0, 0, 4063, 0, 0, 4, 29, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 20, 0, 0, 0, 0, 3167, 30, 0, 11, 0, 0, 11, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 864, 0, 4054, 0, 0, 0, 12, 4065, 0, 0, 3, 0, 0, 0, 28, 1, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 11, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8140, 8180, 8190, 6396, 8114, 8172, 8170] dup_rows_per_dst=[74, 0, 10, 62, 50, 22, 1752, 14] combine_rows_per_src=[8190, 8140, 8180, 8190, 6396, 8114, 8172, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8088, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 106, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8088, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229376 combine_payload_bytes=234520576 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234749952 ingress_bytes=291143680 link_bytes=291143680 token_bytes=469270528 weight_movement_bytes=56623104 total_nvlink_bytes=525893632 weight_to_token_ratio=0.120662 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.789595490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=231399424 dispatch_meta_bytes=226456 combine_payload_bytes=231399424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231625880 ingress_bytes=250273792 link_bytes=250273792 token_bytes=463025304 weight_movement_bytes=18874368 total_nvlink_bytes=481899672 weight_to_token_ratio=0.040763 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398000333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=226967552 dispatch_meta_bytes=229584 combine_payload_bytes=226967552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227197136 ingress_bytes=245841920 link_bytes=245841920 token_bytes=454164688 weight_movement_bytes=18874368 total_nvlink_bytes=473039056 weight_to_token_ratio=0.041558 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.355536316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64976 payload_rows_remote=56798 dup_rows=560 dup_rows_remote=556 dedup_ratio=0.008545 router_tokens_per_expert=[0, 1, 0, 6, 0, 21, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3826, 6, 0, 2, 0, 0, 2, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 244, 0, 4088, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 7664, 8178, 8188, 8190] dup_rows_per_dst=[44, 0, 0, 12, 2, 4, 498, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 7664, 8178, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57302 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 4090, 0, 0, 0, 4090, 0, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8176, 8180, 8180, 8192, 8180] dup_rows_per_dst=[12, 2, 12, 0, 0, 0, 24, 4] combine_rows_per_src=[8192, 8192, 8190, 8176, 8180, 8180, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229376 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234676224 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469123072 weight_movement_bytes=18874368 total_nvlink_bytes=487997440 weight_to_token_ratio=0.040233 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65022 payload_rows_remote=56834 dup_rows=514 dup_rows_remote=486 dedup_ratio=0.007843 router_tokens_per_expert=[0, 0, 0, 2, 0, 4, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3980, 2, 0, 1, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 110, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7998, 8192, 8192, 8110, 7964, 8188, 8190, 8188] dup_rows_per_dst=[8, 62, 0, 4, 4, 84, 324, 28] combine_rows_per_src=[7998, 8192, 8192, 8110, 7964, 8188, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942272 weight_movement_bytes=37748736 total_nvlink_bytes=507691008 weight_to_token_ratio=0.080326 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=232644608 dispatch_meta_bytes=229416 combine_payload_bytes=232644608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232874024 ingress_bytes=251518976 link_bytes=251518976 token_bytes=465518632 weight_movement_bytes=18874368 total_nvlink_bytes=484393000 weight_to_token_ratio=0.040545 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409931073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229424 combine_payload_bytes=234708992 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234938416 ingress_bytes=291332096 link_bytes=291332096 token_bytes=469647408 weight_movement_bytes=56623104 total_nvlink_bytes=526270512 weight_to_token_ratio=0.120565 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791400800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=232792064 dispatch_meta_bytes=229280 combine_payload_bytes=232792064 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233021344 ingress_bytes=270540800 link_bytes=270540800 token_bytes=465813408 weight_movement_bytes=37748736 total_nvlink_bytes=503562144 weight_to_token_ratio=0.081038 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592188831e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65396 payload_rows_remote=57338 dup_rows=140 dup_rows_remote=140 dedup_ratio=0.002136 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8058, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 140] combine_rows_per_src=[8192, 8192, 8186, 8058, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62318 payload_rows_remote=54126 dup_rows=3218 dup_rows_remote=3198 dedup_ratio=0.049103 router_tokens_per_expert=[0, 0, 0, 1, 0, 9, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4021, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 65, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8184, 8188, 4994, 8190] dup_rows_per_dst=[20, 2, 4, 0, 3182, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8184, 8188, 4994, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65106 payload_rows_remote=56920 dup_rows=430 dup_rows_remote=426 dedup_ratio=0.006561 router_tokens_per_expert=[0, 0, 26, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4064, 0, 0, 0, 4062, 0, 4063, 0, 0, 4, 28, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 3953, 3, 0, 0, 0, 0, 0, 0, 0, 0, 23, 14, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 7, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 4063, 2, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 94, 2, 0, 0] payload_rows_per_dst=[8190, 8186, 8190, 7912, 8180, 8128, 8188, 8132] dup_rows_per_dst=[18, 4, 60, 0, 28, 0, 120, 200] combine_rows_per_src=[8190, 8186, 8190, 7912, 8180, 8128, 8188, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8180, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8192, 8180, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229912 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235086360 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942808 weight_movement_bytes=37748736 total_nvlink_bytes=507691544 weight_to_token_ratio=0.080326 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=221700096 dispatch_meta_bytes=229296 combine_payload_bytes=221700096 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=221929392 ingress_bytes=278323200 link_bytes=278323200 token_bytes=443629488 weight_movement_bytes=56623104 total_nvlink_bytes=500252592 weight_to_token_ratio=0.127636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.666755959e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233144320 dispatch_meta_bytes=229384 combine_payload_bytes=233144320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233373704 ingress_bytes=289767424 link_bytes=289767424 token_bytes=466518024 weight_movement_bytes=56623104 total_nvlink_bytes=523141128 weight_to_token_ratio=0.121374 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.776408882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57218 dup_rows=138 dup_rows_remote=138 dedup_ratio=0.002106 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8082, 8190, 8180, 8190, 8180] dup_rows_per_dst=[2, 0, 12, 0, 0, 0, 24, 100] combine_rows_per_src=[8192, 8192, 8192, 8082, 8190, 8180, 8190, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57292 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.001190 router_tokens_per_expert=[0, 14, 0, 6, 0, 0, 4088, 0, 0, 2, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4077, 2, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8166, 8154, 8192, 8192] dup_rows_per_dst=[40, 0, 12, 24, 0, 0, 0, 2] combine_rows_per_src=[8192, 8180, 8190, 8192, 8166, 8154, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235061248 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893120 weight_movement_bytes=37748736 total_nvlink_bytes=507641856 weight_to_token_ratio=0.080335 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65384 payload_rows_remote=57196 dup_rows=152 dup_rows_remote=118 dedup_ratio=0.002319 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[8160, 8098, 8192, 8174, 8192, 8188, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 12, 94, 8, 34] combine_rows_per_src=[8160, 8098, 8192, 8174, 8192, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234364928 dispatch_meta_bytes=229424 combine_payload_bytes=234364928 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234594352 ingress_bytes=272113664 link_bytes=272113664 token_bytes=468959280 weight_movement_bytes=37748736 total_nvlink_bytes=506708016 weight_to_token_ratio=0.080495 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607259240e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229480 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234897512 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469565544 weight_movement_bytes=37748736 total_nvlink_bytes=507314280 weight_to_token_ratio=0.080391 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57344 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234274816 dispatch_meta_bytes=229256 combine_payload_bytes=234274816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234504072 ingress_bytes=272023552 link_bytes=272023552 token_bytes=468778888 weight_movement_bytes=37748736 total_nvlink_bytes=506527624 weight_to_token_ratio=0.080526 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606395831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64838 payload_rows_remote=56646 dup_rows=698 dup_rows_remote=696 dedup_ratio=0.010651 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8112, 8190, 8178, 8190, 8020, 7764] dup_rows_per_dst=[2, 94, 2, 598, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8112, 8190, 8178, 8190, 8020, 7764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65066 payload_rows_remote=56938 dup_rows=470 dup_rows_remote=470 dedup_ratio=0.007172 router_tokens_per_expert=[1, 14, 0, 28, 3, 0, 4056, 0, 0, 106, 0, 0, 31, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4065, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 3965, 0, 0, 0, 0, 2, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 3, 0, 47, 0, 0, 4063, 3, 29, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0] payload_rows_per_dst=[8192, 8128, 8178, 8192, 7994, 7998, 8192, 8192] dup_rows_per_dst=[286, 0, 60, 8, 0, 0, 98, 18] combine_rows_per_src=[8192, 8128, 8178, 8192, 7994, 7998, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229416 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110440 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991464 weight_movement_bytes=18874368 total_nvlink_bytes=488865832 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=233218048 dispatch_meta_bytes=229632 combine_payload_bytes=233218048 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233447680 ingress_bytes=270966784 link_bytes=270966784 token_bytes=466665728 weight_movement_bytes=37748736 total_nvlink_bytes=504414464 weight_to_token_ratio=0.080890 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596270400e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=232022016 dispatch_meta_bytes=229368 combine_payload_bytes=232022016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232251384 ingress_bytes=250896384 link_bytes=250896384 token_bytes=464273400 weight_movement_bytes=18874368 total_nvlink_bytes=483147768 weight_to_token_ratio=0.040654 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403965703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65452 payload_rows_remote=57292 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.001282 router_tokens_per_expert=[0, 12, 0, 6, 2, 0, 4088, 0, 0, 16, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4071, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8154, 8160, 8192, 8192] dup_rows_per_dst=[68, 0, 12, 0, 0, 0, 2, 2] combine_rows_per_src=[8192, 8180, 8190, 8192, 8154, 8160, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65218 payload_rows_remote=57026 dup_rows=318 dup_rows_remote=306 dedup_ratio=0.004852 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 4090, 1, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 4090, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4087, 0, 0, 0, 147, 0, 4082, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7906, 8190, 8190, 8192, 8180, 8178, 8192] dup_rows_per_dst=[2, 0, 2, 12, 12, 2, 0, 288] combine_rows_per_src=[8190, 7906, 8190, 8190, 8192, 8180, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65002 payload_rows_remote=56810 dup_rows=534 dup_rows_remote=534 dedup_ratio=0.008148 router_tokens_per_expert=[0, 4, 0, 2, 0, 0, 4094, 0, 0, 2, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[7750, 8188, 8122, 8192, 8188, 8178, 8192, 8192] dup_rows_per_dst=[16, 94, 4, 0, 248, 70, 102, 0] combine_rows_per_src=[7750, 8188, 8122, 8192, 8188, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235110400 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991424 weight_movement_bytes=94371840 total_nvlink_bytes=564363264 weight_to_token_ratio=0.200795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229504 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234897536 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469565568 weight_movement_bytes=37748736 total_nvlink_bytes=507314304 weight_to_token_ratio=0.080391 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8166, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 26] combine_rows_per_src=[8192, 8166, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233578496 dispatch_meta_bytes=229328 combine_payload_bytes=233578496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233807824 ingress_bytes=271327232 link_bytes=271327232 token_bytes=467386320 weight_movement_bytes=37748736 total_nvlink_bytes=505135056 weight_to_token_ratio=0.080766 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599724036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=232693760 dispatch_meta_bytes=229376 combine_payload_bytes=232693760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232923136 ingress_bytes=251568128 link_bytes=251568128 token_bytes=465616896 weight_movement_bytes=18874368 total_nvlink_bytes=484491264 weight_to_token_ratio=0.040536 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410402023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64408 payload_rows_remote=57144 dup_rows=1128 dup_rows_remote=1128 dedup_ratio=0.017212 router_tokens_per_expert=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4057, 0, 7, 0, 0, 0, 4033, 0, 0, 0, 0, 4063, 8, 3617, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 7, 0, 0, 5, 29, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 29, 31, 0, 1, 0, 0, 0, 4063, 0, 29, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4059, 0, 0, 0, 446, 0, 4050, 30, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 7264, 8192, 8130, 8184, 8140, 8128, 8190] dup_rows_per_dst=[6, 0, 22, 82, 52, 40, 0, 926] combine_rows_per_src=[8180, 7264, 8192, 8130, 8184, 8140, 8128, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65274 payload_rows_remote=57082 dup_rows=262 dup_rows_remote=230 dedup_ratio=0.003998 router_tokens_per_expert=[0, 2, 0, 1, 0, 0, 4095, 0, 0, 2, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8190, 8192, 8170, 8192, 8190, 7978, 8170] dup_rows_per_dst=[32, 0, 0, 0, 18, 210, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8170, 8192, 8190, 7978, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8172, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8172, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234061824 dispatch_meta_bytes=233088 combine_payload_bytes=234061824 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=234294912 ingress_bytes=347308032 link_bytes=347308032 token_bytes=468356736 weight_movement_bytes=113246208 total_nvlink_bytes=581602944 weight_to_token_ratio=0.241795 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.327734676e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=233807872 dispatch_meta_bytes=229248 combine_payload_bytes=233807872 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234037120 ingress_bytes=328179712 link_bytes=328179712 token_bytes=467844992 weight_movement_bytes=94371840 total_nvlink_bytes=562216832 weight_to_token_ratio=0.201716 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.144456526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65232 payload_rows_remote=57052 dup_rows=304 dup_rows_remote=304 dedup_ratio=0.004639 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4088, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 4090, 1, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 4089, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 4089, 0, 0, 0, 136, 0, 4084, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7920, 8192, 8192, 8190, 8180, 8178, 8192] dup_rows_per_dst=[2, 0, 4, 12, 12, 0, 0, 274] combine_rows_per_src=[8188, 7920, 8192, 8192, 8190, 8180, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57300 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4, 4087, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4, 4081, 0, 9, 1, 0, 0, 3, 0, 6, 5, 0, 4086, 6, 0, 4089, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8182, 8180, 8190, 8192, 8192, 8186] dup_rows_per_dst=[0, 2, 0, 4, 0, 2, 26, 12] combine_rows_per_src=[8176, 8192, 8182, 8180, 8190, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57128 dup_rows=216 dup_rows_remote=148 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 4072, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 22, 0, 4088, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8108, 8166, 8190, 8098, 8188, 8188, 8192] dup_rows_per_dst=[0, 0, 2, 4, 4, 40, 98, 68] combine_rows_per_src=[8190, 8108, 8166, 8190, 8098, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229296 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235028400 ingress_bytes=310296576 link_bytes=310296576 token_bytes=469827504 weight_movement_bytes=75497472 total_nvlink_bytes=545324976 weight_to_token_ratio=0.160692 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973109116e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233684992 dispatch_meta_bytes=229424 combine_payload_bytes=233684992 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233914416 ingress_bytes=271433728 link_bytes=271433728 token_bytes=467599408 weight_movement_bytes=37748736 total_nvlink_bytes=505348144 weight_to_token_ratio=0.080729 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600744428e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229384 combine_payload_bytes=234700800 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234930184 ingress_bytes=272449536 link_bytes=272449536 token_bytes=469630984 weight_movement_bytes=37748736 total_nvlink_bytes=507379720 weight_to_token_ratio=0.080380 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610477400e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229104 combine_payload_bytes=233996288 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234225392 ingress_bytes=290619392 link_bytes=290619392 token_bytes=468221680 weight_movement_bytes=56623104 total_nvlink_bytes=524844784 weight_to_token_ratio=0.120932 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784572020e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65328 payload_rows_remote=57136 dup_rows=208 dup_rows_remote=180 dedup_ratio=0.003174 router_tokens_per_expert=[0, 0, 4061, 0, 3, 0, 0, 5, 0, 21, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 5, 4062, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4060, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 11, 4042, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 24, 4035, 1, 10, 8, 0, 2, 27, 0, 32, 25, 0, 4054, 28, 0, 4056, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8192, 8122, 8128, 8188, 8170, 8192, 8164] dup_rows_per_dst=[8, 28, 0, 20, 0, 2, 136, 14] combine_rows_per_src=[8172, 8192, 8122, 8128, 8188, 8170, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63900 payload_rows_remote=55708 dup_rows=1636 dup_rows_remote=1626 dedup_ratio=0.024963 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 15, 0, 4095, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8146, 8192, 8190, 8190, 6608] dup_rows_per_dst=[10, 1598, 0, 2, 24, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 8146, 8192, 8190, 8190, 6608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234029056 dispatch_meta_bytes=229264 combine_payload_bytes=234029056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234258320 ingress_bytes=271777792 link_bytes=271777792 token_bytes=468287376 weight_movement_bytes=37748736 total_nvlink_bytes=506036112 weight_to_token_ratio=0.080610 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604041080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=228179968 dispatch_meta_bytes=229336 combine_payload_bytes=228179968 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228409304 ingress_bytes=265928704 link_bytes=265928704 token_bytes=456589272 weight_movement_bytes=37748736 total_nvlink_bytes=494338008 weight_to_token_ratio=0.082675 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.547997997e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8166, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 26] combine_rows_per_src=[8192, 8192, 8192, 8166, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57304 dup_rows=44 dup_rows_remote=42 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 4087, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 4086, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 5, 4082, 0, 5, 1, 0, 0, 8, 0, 6, 5, 0, 4087, 6, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8182, 8178, 8190, 8188, 8192, 8186] dup_rows_per_dst=[0, 2, 2, 2, 0, 2, 32, 4] combine_rows_per_src=[8184, 8192, 8182, 8178, 8190, 8188, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64812 payload_rows_remote=56642 dup_rows=724 dup_rows_remote=88 dedup_ratio=0.011047 router_tokens_per_expert=[6, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 58, 0, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 318, 0, 0, 0, 4079, 0, 4090, 0, 4089, 9, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8190, 7528, 8192, 8180, 8170, 8180, 8180, 8192] dup_rows_per_dst=[12, 2, 14, 0, 636, 48, 2, 10] combine_rows_per_src=[8190, 7528, 8192, 8180, 8170, 8180, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65382 payload_rows_remote=57190 dup_rows=154 dup_rows_remote=50 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 1, 0, 0, 0, 3, 0, 2, 2, 0, 4094, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8156, 8188, 8188, 8192, 8188, 8088, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 36, 14, 104] combine_rows_per_src=[8190, 8156, 8188, 8188, 8192, 8188, 8088, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942272 weight_movement_bytes=37748736 total_nvlink_bytes=507691008 weight_to_token_ratio=0.080326 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64716 payload_rows_remote=56524 dup_rows=820 dup_rows_remote=820 dedup_ratio=0.012512 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 359, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8192, 8192, 8192, 8192, 7392, 8192] dup_rows_per_dst=[0, 2, 0, 0, 718, 100, 0, 0] combine_rows_per_src=[8192, 8172, 8192, 8192, 8192, 8192, 7392, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229384 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234946568 ingress_bytes=291340288 link_bytes=291340288 token_bytes=469663752 weight_movement_bytes=56623104 total_nvlink_bytes=526286856 weight_to_token_ratio=0.120561 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791479291e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=232005632 dispatch_meta_bytes=226920 combine_payload_bytes=232005632 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=232232552 ingress_bytes=345251840 link_bytes=345251840 token_bytes=464238184 weight_movement_bytes=113246208 total_nvlink_bytes=577484392 weight_to_token_ratio=0.243940 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.308033256e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234250240 dispatch_meta_bytes=228960 combine_payload_bytes=234250240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234479200 ingress_bytes=290873344 link_bytes=290873344 token_bytes=468729440 weight_movement_bytes=56623104 total_nvlink_bytes=525352544 weight_to_token_ratio=0.120801 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787005264e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64406 payload_rows_remote=57106 dup_rows=1130 dup_rows_remote=1048 dedup_ratio=0.017242 router_tokens_per_expert=[28, 0, 0, 4061, 0, 25, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3149, 0, 0, 0, 0, 434, 0, 14, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 31, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4, 4056, 416, 0, 0, 0, 4022, 0, 4063, 0, 4059, 7, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4063, 0, 2, 26, 0, 0, 3, 0, 1, 0, 0, 0, 2, 32, 0, 0] payload_rows_per_dst=[8182, 7300, 8192, 8128, 8114, 8124, 8174, 8192] dup_rows_per_dst=[50, 82, 62, 0, 832, 32, 6, 66] combine_rows_per_src=[8182, 7300, 8192, 8128, 8114, 8124, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57128 dup_rows=218 dup_rows_remote=172 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4093, 1, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8180, 8086, 8096] dup_rows_per_dst=[46, 4, 92, 64, 8, 0, 2, 2] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8180, 8086, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=231522304 dispatch_meta_bytes=229376 combine_payload_bytes=231522304 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231751680 ingress_bytes=269271040 link_bytes=269271040 token_bytes=463273984 weight_movement_bytes=37748736 total_nvlink_bytes=501022720 weight_to_token_ratio=0.081483 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580022616e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=232616 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234138792 ingress_bytes=252780544 link_bytes=252780544 token_bytes=468044968 weight_movement_bytes=18874368 total_nvlink_bytes=486919336 weight_to_token_ratio=0.040326 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422018797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229200 combine_payload_bytes=233996288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234225488 ingress_bytes=252870656 link_bytes=252870656 token_bytes=468221776 weight_movement_bytes=18874368 total_nvlink_bytes=487096144 weight_to_token_ratio=0.040311 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422882206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65008 payload_rows_remote=56816 dup_rows=528 dup_rows_remote=528 dedup_ratio=0.008057 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 264, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7664, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 528, 0, 0, 0] combine_rows_per_src=[8192, 7664, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65010 payload_rows_remote=56832 dup_rows=526 dup_rows_remote=516 dedup_ratio=0.008026 router_tokens_per_expert=[6, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 3773, 0, 0, 0, 0, 1, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 238, 0, 0, 0, 4084, 0, 4090, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8190, 7710, 8192, 8180, 8180, 8178, 8190, 8190] dup_rows_per_dst=[8, 2, 14, 4, 476, 10, 0, 12] combine_rows_per_src=[8190, 7710, 8192, 8180, 8180, 8178, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57288 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 1, 2, 0, 1, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 4, 0, 0, 0, 13, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 4088, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8192, 8192, 8180, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 8, 34, 0, 0, 14, 0, 0] combine_rows_per_src=[8150, 8192, 8192, 8180, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64292 payload_rows_remote=56100 dup_rows=1244 dup_rows_remote=1178 dedup_ratio=0.018982 router_tokens_per_expert=[2, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3871, 0, 0, 0, 0, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 177, 0, 0, 0, 4093, 0, 4094, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8130, 7834, 8094, 8188, 7476, 8188, 8190, 8192] dup_rows_per_dst=[4, 0, 4, 0, 338, 4, 828, 66] combine_rows_per_src=[8130, 7834, 8094, 8188, 7476, 8188, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=232718336 dispatch_meta_bytes=229376 combine_payload_bytes=232718336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232947712 ingress_bytes=251592704 link_bytes=251592704 token_bytes=465666048 weight_movement_bytes=18874368 total_nvlink_bytes=484540416 weight_to_token_ratio=0.040532 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410637498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65214 payload_rows_remote=57022 dup_rows=322 dup_rows_remote=322 dedup_ratio=0.004913 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 4096, 131, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8102, 8192, 8192, 8192, 7960, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 322, 0, 0] combine_rows_per_src=[8102, 8192, 8192, 8192, 7960, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=232783872 dispatch_meta_bytes=229392 combine_payload_bytes=232783872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233013264 ingress_bytes=270532608 link_bytes=270532608 token_bytes=465797136 weight_movement_bytes=37748736 total_nvlink_bytes=503545872 weight_to_token_ratio=0.081041 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592110340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234651648 dispatch_meta_bytes=229376 combine_payload_bytes=234651648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234881024 ingress_bytes=253526016 link_bytes=253526016 token_bytes=469532672 weight_movement_bytes=18874368 total_nvlink_bytes=488407040 weight_to_token_ratio=0.040198 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429161543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57198 dup_rows=146 dup_rows_remote=88 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 17, 3, 4057, 0, 0, 0, 0, 0, 0, 18, 4, 6, 31, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4064, 0, 30, 4057, 6, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 1, 32, 6, 2, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 6, 20, 0, 0, 0, 0, 4059, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8192, 8184, 8130, 8188, 8192, 8186, 8168] dup_rows_per_dst=[4, 58, 0, 0, 0, 72, 4, 8] combine_rows_per_src=[8150, 8192, 8184, 8130, 8188, 8192, 8186, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62560 payload_rows_remote=54368 dup_rows=2976 dup_rows_remote=2974 dedup_ratio=0.045410 router_tokens_per_expert=[1, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 24, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 4095, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 5480, 8086, 8190, 8040] dup_rows_per_dst=[2, 2868, 4, 90, 2, 10, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 5480, 8086, 8190, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=229785600 dispatch_meta_bytes=229112 combine_payload_bytes=229785600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230014712 ingress_bytes=267534336 link_bytes=267534336 token_bytes=459800312 weight_movement_bytes=37748736 total_nvlink_bytes=497549048 weight_to_token_ratio=0.082098 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.563382372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=233562112 dispatch_meta_bytes=229376 combine_payload_bytes=233562112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233791488 ingress_bytes=271310848 link_bytes=271310848 token_bytes=467353600 weight_movement_bytes=37748736 total_nvlink_bytes=505102336 weight_to_token_ratio=0.080771 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599567052e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229144 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512152 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795160 weight_movement_bytes=18874368 total_nvlink_bytes=487669528 weight_to_token_ratio=0.040261 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=222691328 dispatch_meta_bytes=229368 combine_payload_bytes=222691328 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=222920696 ingress_bytes=241565696 link_bytes=241565696 token_bytes=445612024 weight_movement_bytes=18874368 total_nvlink_bytes=464486392 weight_to_token_ratio=0.042356 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.314563641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=14 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4085, 0, 0, 0, 0, 0, 0, 2, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 4089, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 1, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 6, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8190, 8180, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 6, 0, 0, 12, 2, 2] combine_rows_per_src=[8180, 8192, 8190, 8180, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57294 dup_rows=62 dup_rows_remote=40 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4089, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 4089, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8182, 8178, 8180, 8190, 8170, 8192] dup_rows_per_dst=[10, 12, 0, 0, 22, 12, 0, 6] combine_rows_per_src=[8190, 8192, 8182, 8178, 8180, 8190, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57230 dup_rows=116 dup_rows_remote=12 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8118, 8160, 8188, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 4, 0, 0, 2, 4, 2, 104] combine_rows_per_src=[8188, 8118, 8160, 8188, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229328 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235003856 ingress_bytes=329146368 link_bytes=329146368 token_bytes=469778384 weight_movement_bytes=94371840 total_nvlink_bytes=564150224 weight_to_token_ratio=0.200886 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153718548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65232 payload_rows_remote=57040 dup_rows=304 dup_rows_remote=180 dedup_ratio=0.004639 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 31, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 32, 0, 0, 4063, 0, 0, 0, 0, 0, 5, 0, 32, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7, 0, 0, 4063, 0, 0, 0, 0, 0, 4060, 4065, 0, 0, 1, 0, 0, 31, 0, 0, 0, 0, 4052, 0, 2, 0, 4022, 27, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 32, 0, 0, 0] payload_rows_per_dst=[8166, 8192, 8132, 8128, 8138, 8182, 8102, 8192] dup_rows_per_dst=[54, 124, 0, 2, 82, 2, 0, 40] combine_rows_per_src=[8166, 8192, 8132, 8128, 8138, 8182, 8102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229336 combine_payload_bytes=234676224 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234905560 ingress_bytes=272424960 link_bytes=272424960 token_bytes=469581784 weight_movement_bytes=37748736 total_nvlink_bytes=507330520 weight_to_token_ratio=0.080388 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610241925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65016 payload_rows_remote=56826 dup_rows=520 dup_rows_remote=332 dedup_ratio=0.007935 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 7766, 8192, 8102] dup_rows_per_dst=[188, 0, 2, 92, 234, 2, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 7766, 8192, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=228968 combine_payload_bytes=234414080 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234643048 ingress_bytes=291037184 link_bytes=291037184 token_bytes=469057128 weight_movement_bytes=56623104 total_nvlink_bytes=525680232 weight_to_token_ratio=0.120717 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.788575098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233635840 dispatch_meta_bytes=228880 combine_payload_bytes=233635840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233864720 ingress_bytes=271384576 link_bytes=271384576 token_bytes=467500560 weight_movement_bytes=37748736 total_nvlink_bytes=505249296 weight_to_token_ratio=0.080746 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600273478e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=232759296 dispatch_meta_bytes=228632 combine_payload_bytes=232759296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232987928 ingress_bytes=251633664 link_bytes=251633664 token_bytes=465747224 weight_movement_bytes=18874368 total_nvlink_bytes=484621592 weight_to_token_ratio=0.040525 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411029957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57284 dup_rows=62 dup_rows_remote=58 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 4088, 0, 0, 0, 4081, 5, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8178, 8180, 8180, 8190, 8174, 8192] dup_rows_per_dst=[12, 22, 0, 2, 16, 4, 0, 6] combine_rows_per_src=[8188, 8192, 8178, 8180, 8180, 8190, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63936 payload_rows_remote=57290 dup_rows=1600 dup_rows_remote=1590 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 3315, 0, 6, 0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4088, 0, 0, 6, 0, 4090, 763, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 4089, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 8180, 8180, 8180, 6646, 8188, 8180, 8190] dup_rows_per_dst=[2, 18, 0, 0, 10, 16, 1530, 24] combine_rows_per_src=[8192, 8180, 8180, 8180, 6646, 8188, 8180, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65256 payload_rows_remote=57064 dup_rows=280 dup_rows_remote=280 dedup_ratio=0.004272 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4094, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8144, 8014, 8188, 8188, 8154, 8192, 8184, 8192] dup_rows_per_dst=[4, 8, 0, 0, 6, 36, 226, 0] combine_rows_per_src=[8144, 8014, 8188, 8188, 8154, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 16, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62206 payload_rows_remote=54080 dup_rows=3330 dup_rows_remote=3330 dedup_ratio=0.050812 router_tokens_per_expert=[0, 11, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 2, 23, 2, 1, 0, 0, 0, 4063, 0, 0, 2504, 0, 31, 0, 27, 0, 0, 12, 0, 4, 0, 0, 0, 0, 0, 18, 0, 1, 4057, 0, 0, 7, 0, 4062, 1521, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 28, 30, 0, 0, 0, 4063, 1, 0, 18, 0, 21] payload_rows_per_dst=[8190, 8126, 8180, 8122, 5114, 8150, 8132, 8192] dup_rows_per_dst=[20, 0, 0, 0, 62, 48, 3044, 156] combine_rows_per_src=[8190, 8126, 8180, 8122, 5114, 8150, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229368 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234864632 ingress_bytes=291258368 link_bytes=291258368 token_bytes=469499896 weight_movement_bytes=56623104 total_nvlink_bytes=526123000 weight_to_token_ratio=0.120603 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790694374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=235520 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234895360 ingress_bytes=272408576 link_bytes=272408576 token_bytes=469555200 weight_movement_bytes=37748736 total_nvlink_bytes=507303936 weight_to_token_ratio=0.080393 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610084942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57152 dup_rows=192 dup_rows_remote=192 dedup_ratio=0.002930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8182, 8152, 8104, 8140] dup_rows_per_dst=[0, 12, 88, 92, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8182, 8152, 8104, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233734144 dispatch_meta_bytes=229376 combine_payload_bytes=233734144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233963520 ingress_bytes=252608512 link_bytes=252608512 token_bytes=467697664 weight_movement_bytes=18874368 total_nvlink_bytes=486572032 weight_to_token_ratio=0.040356 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420370471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=221511680 dispatch_meta_bytes=229640 combine_payload_bytes=221511680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221741320 ingress_bytes=240386048 link_bytes=240386048 token_bytes=443253000 weight_movement_bytes=18874368 total_nvlink_bytes=462127368 weight_to_token_ratio=0.042581 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.303260835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21974871744 prefetch_bytes=905969664 reduce_bytes=962592768 weight_movement_bytes=1868562432 egress_bytes=10992938688 ingress_bytes=12850495488 link_bytes=12850495488 total_nvlink_bytes=23843434176 weight_to_token_ratio=0.085032 prefetch_share_of_ingress=0.070501 dedup_ratio=0.024605 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=51 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.094184 gemm_inner_pad_upper_bound=45212 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.565148436e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229376 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234323968 ingress_bytes=252968960 link_bytes=252968960 token_bytes=468418560 weight_movement_bytes=18874368 total_nvlink_bytes=487292928 weight_to_token_ratio=0.040294 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423824106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21958236976 prefetch_bytes=924844032 reduce_bytes=905969664 weight_movement_bytes=1830813696 egress_bytes=10984635184 ingress_bytes=12804415488 link_bytes=12804415488 total_nvlink_bytes=23789050672 weight_to_token_ratio=0.083377 prefetch_share_of_ingress=0.072229 dedup_ratio=0.027514 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=48 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.094039 gemm_inner_pad_upper_bound=44323 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.555950188e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 6, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21750814864 prefetch_bytes=905969664 reduce_bytes=868220928 weight_movement_bytes=1774190592 egress_bytes=10880899216 ingress_bytes=12644106240 link_bytes=12644106240 total_nvlink_bytes=23525005456 weight_to_token_ratio=0.081569 prefetch_share_of_ingress=0.071652 dedup_ratio=0.033775 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=46 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.095197 gemm_inner_pad_upper_bound=42545 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.523950098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64350 payload_rows_remote=56160 dup_rows=1186 dup_rows_remote=1168 dedup_ratio=0.018097 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4090, 0, 0, 3518, 0, 6, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 2, 0, 4089, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 4089, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8180, 7050, 8190, 8178, 8190] dup_rows_per_dst=[4, 6, 2, 0, 12, 18, 1120, 24] combine_rows_per_src=[8192, 8178, 8192, 8180, 7050, 8190, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65198 payload_rows_remote=57006 dup_rows=338 dup_rows_remote=310 dedup_ratio=0.005157 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3955, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 0, 1, 0, 4094, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 0, 0, 4094, 0, 0, 1, 0, 1] payload_rows_per_dst=[8164, 8188, 8192, 8188, 7914, 8172, 8188, 8192] dup_rows_per_dst=[2, 4, 0, 0, 30, 8, 266, 28] combine_rows_per_src=[8164, 8188, 8192, 8188, 7914, 8172, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=230031360 dispatch_meta_bytes=229312 combine_payload_bytes=230031360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=230260672 ingress_bytes=267780096 link_bytes=267780096 token_bytes=460292032 weight_movement_bytes=37748736 total_nvlink_bytes=498040768 weight_to_token_ratio=0.082010 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.565737124e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59362 payload_rows_remote=51170 dup_rows=6174 dup_rows_remote=6172 dedup_ratio=0.094208 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4017, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4095, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8190, 8192, 8168, 2280, 7960] dup_rows_per_dst=[2, 70, 24, 100, 5960, 2, 12, 4] combine_rows_per_src=[8192, 8190, 8190, 8190, 8192, 8168, 2280, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=233496576 dispatch_meta_bytes=229264 combine_payload_bytes=233496576 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233725840 ingress_bytes=308994048 link_bytes=308994048 token_bytes=467222416 weight_movement_bytes=75497472 total_nvlink_bytes=542719888 weight_to_token_ratio=0.161588 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.960628933e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=22023395408 prefetch_bytes=905969664 reduce_bytes=622854144 weight_movement_bytes=1528823808 egress_bytes=11017222224 ingress_bytes=12534996992 link_bytes=12534996992 total_nvlink_bytes=23552219216 weight_to_token_ratio=0.069418 prefetch_share_of_ingress=0.072275 dedup_ratio=0.025031 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=33 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.093605 gemm_inner_pad_upper_bound=45847 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.502170283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=209592320 dispatch_meta_bytes=229368 combine_payload_bytes=209592320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209821688 ingress_bytes=228466688 link_bytes=228466688 token_bytes=419414008 weight_movement_bytes=18874368 total_nvlink_bytes=438288376 weight_to_token_ratio=0.045002 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189055392e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21986727008 prefetch_bytes=868220928 reduce_bytes=1000341504 weight_movement_bytes=1868562432 egress_bytes=10998854752 ingress_bytes=12856434688 link_bytes=12856434688 total_nvlink_bytes=23855289440 weight_to_token_ratio=0.084986 prefetch_share_of_ingress=0.067532 dedup_ratio=0.026961 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=53 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.093967 gemm_inner_pad_upper_bound=44831 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.566333988e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21493614896 prefetch_bytes=887095296 reduce_bytes=905969664 weight_movement_bytes=1793064960 egress_bytes=10752231728 ingress_bytes=12534448128 link_bytes=12534448128 total_nvlink_bytes=23286679856 weight_to_token_ratio=0.083423 prefetch_share_of_ingress=0.070773 dedup_ratio=0.042576 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=48 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.092303 gemm_inner_pad_upper_bound=48133 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.502060721e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21467900416 prefetch_bytes=849346560 reduce_bytes=868220928 weight_movement_bytes=1717567488 egress_bytes=10739395072 ingress_bytes=12446072832 link_bytes=12446072832 total_nvlink_bytes=23185467904 weight_to_token_ratio=0.080006 prefetch_share_of_ingress=0.068242 dedup_ratio=0.043033 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=46 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.094618 gemm_inner_pad_upper_bound=44069 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.484419709e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229344 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235077600 ingress_bytes=272596992 link_bytes=272596992 token_bytes=469925856 weight_movement_bytes=37748736 total_nvlink_bytes=507674592 weight_to_token_ratio=0.080329 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611890251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21853603640 prefetch_bytes=943718400 reduce_bytes=1056964608 weight_movement_bytes=2000683008 egress_bytes=10932299576 ingress_bytes=12921987072 link_bytes=12921987072 total_nvlink_bytes=23854286648 weight_to_token_ratio=0.091549 prefetch_share_of_ingress=0.073032 dedup_ratio=0.029308 prefetch_slots=100 unused_prefetch_slots=1436 slot_utilization=0.065104 reduce_slots=56 remote_demand=50 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.094401 gemm_inner_pad_upper_bound=45466 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.579419211e-04 prefetch_bytes_per_gemm_flop=1.883801119e-05 anomalies=none + [2026-08-29 23:37:44] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 753.6 | throughput per GPU (TFLOP/s/GPU): 118.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307933E+00 | loss scale: 1.0 | grad norm: 5.322 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38218 dup_rows=21122 dup_rows_remote=17304 dedup_ratio=0.322296 router_tokens_per_expert=[521, 0, 0, 3, 1137, 565, 373, 8, 6, 2, 184, 273, 102, 190, 419, 291, 6, 23, 9, 205, 365, 207, 6, 0, 284, 715, 75, 76, 11, 2, 247, 114, 364, 404, 8, 89, 11, 38, 742, 20, 127, 243, 406, 1004, 649, 5, 114, 785, 57, 128, 95, 58, 184, 959, 4, 29, 835, 32, 115, 577, 59, 247, 24, 66, 647, 204, 0, 2, 105, 0, 232, 724, 4, 275, 651, 262, 18, 357, 15, 1538, 644, 38, 20, 189, 20, 219, 27, 0, 995, 224, 20, 574, 22, 332, 123, 15, 2, 103, 909, 40, 3, 1036, 1500, 94, 228, 16, 179, 29, 341, 397, 281, 0, 599, 651, 299, 10, 374, 8, 939, 72, 172, 576, 207, 16, 81, 6, 204, 3] payload_rows_per_dst=[5404, 5524, 5476, 6196, 5212, 5776, 5898, 4928] dup_rows_per_dst=[2744, 2166, 2534, 3818, 1780, 3156, 3296, 1628] combine_rows_per_src=[5404, 5524, 5476, 6196, 5212, 5776, 5898, 4928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44798 payload_rows_remote=38920 dup_rows=20738 dup_rows_remote=17338 dedup_ratio=0.316437 router_tokens_per_expert=[476, 2, 0, 2, 1199, 681, 411, 3, 4, 1, 150, 287, 124, 205, 455, 124, 1, 7, 5, 121, 581, 189, 4, 0, 210, 837, 31, 51, 18, 3, 199, 116, 462, 301, 2, 84, 4, 30, 793, 14, 106, 268, 330, 1347, 683, 5, 104, 537, 42, 117, 74, 45, 103, 842, 4, 27, 786, 20, 92, 560, 31, 215, 9, 21, 690, 229, 0, 1, 132, 0, 193, 780, 0, 209, 732, 230, 12, 175, 10, 1775, 594, 38, 25, 331, 24, 216, 0, 0, 799, 181, 14, 581, 48, 326, 108, 7, 1, 48, 797, 36, 5, 1207, 1915, 66, 379, 24, 260, 27, 179, 407, 238, 0, 902, 324, 356, 15, 327, 7, 767, 97, 144, 811, 227, 22, 96, 1, 65, 3] payload_rows_per_dst=[5608, 5984, 5306, 4666, 6490, 5878, 5512, 5354] dup_rows_per_dst=[2640, 2592, 2140, 1310, 3846, 3400, 2288, 2522] combine_rows_per_src=[5608, 5984, 5306, 4666, 6490, 5878, 5512, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38530 dup_rows=21098 dup_rows_remote=18198 dedup_ratio=0.321930 router_tokens_per_expert=[696, 0, 0, 1, 1188, 672, 458, 4, 0, 0, 193, 259, 113, 164, 465, 134, 6, 15, 4, 189, 659, 195, 5, 1, 241, 681, 30, 50, 20, 6, 207, 135, 362, 273, 5, 66, 8, 40, 578, 11, 111, 328, 347, 1243, 448, 9, 78, 660, 103, 119, 99, 14, 189, 723, 3, 19, 795, 16, 150, 512, 34, 243, 8, 32, 769, 254, 0, 10, 143, 1, 205, 788, 0, 248, 673, 300, 11, 231, 13, 1771, 537, 17, 26, 357, 3, 156, 9, 1, 688, 199, 24, 621, 42, 352, 100, 8, 0, 83, 573, 20, 1, 1186, 1693, 31, 497, 33, 253, 11, 240, 481, 188, 0, 810, 527, 313, 22, 401, 8, 852, 81, 125, 804, 290, 38, 95, 0, 138, 0] payload_rows_per_dst=[5616, 5908, 5930, 6034, 5394, 4516, 5754, 5286] dup_rows_per_dst=[3078, 2900, 3204, 3626, 1898, 1764, 2610, 2018] combine_rows_per_src=[5616, 5908, 5930, 6034, 5394, 4516, 5754, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44778 payload_rows_remote=39972 dup_rows=20758 dup_rows_remote=19268 dedup_ratio=0.316742 router_tokens_per_expert=[318, 0, 0, 0, 1234, 666, 329, 3, 1, 5, 161, 343, 115, 165, 471, 218, 8, 13, 4, 124, 448, 163, 14, 0, 233, 749, 15, 67, 39, 18, 234, 12, 367, 283, 4, 101, 3, 24, 738, 11, 64, 204, 369, 1720, 524, 2, 82, 637, 106, 104, 68, 1, 167, 983, 4, 22, 831, 46, 115, 446, 21, 237, 8, 22, 551, 165, 0, 2, 121, 0, 199, 781, 1, 209, 649, 234, 6, 106, 10, 1736, 646, 22, 35, 375, 8, 163, 4, 0, 745, 181, 26, 522, 66, 486, 122, 0, 0, 79, 916, 43, 2, 1008, 1799, 39, 379, 15, 369, 35, 215, 466, 194, 0, 829, 474, 269, 21, 331, 3, 869, 111, 146, 881, 336, 18, 121, 3, 140, 2] payload_rows_per_dst=[5588, 5780, 4970, 6332, 4806, 6084, 5932, 5286] dup_rows_per_dst=[2470, 2100, 1856, 3274, 1490, 4158, 3326, 2084] combine_rows_per_src=[5588, 5780, 4970, 6332, 4806, 6084, 5932, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=37834 dup_rows=21828 dup_rows_remote=18426 dedup_ratio=0.333069 router_tokens_per_expert=[326, 6, 2, 1, 1196, 629, 337, 16, 24, 1, 207, 282, 172, 201, 599, 334, 19, 28, 26, 181, 494, 203, 19, 3, 244, 616, 88, 83, 23, 5, 193, 161, 324, 346, 9, 83, 38, 45, 580, 26, 90, 269, 401, 1745, 520, 6, 93, 463, 117, 225, 122, 47, 143, 776, 10, 28, 642, 53, 159, 520, 53, 235, 7, 39, 573, 197, 13, 4, 166, 13, 188, 885, 10, 211, 510, 206, 33, 214, 19, 1504, 737, 40, 69, 360, 13, 161, 2, 3, 664, 198, 67, 414, 148, 434, 177, 5, 0, 87, 840, 55, 10, 1190, 1250, 62, 289, 40, 206, 57, 259, 502, 164, 1, 792, 571, 324, 40, 324, 31, 827, 103, 236, 579, 237, 34, 228, 26, 227, 6] payload_rows_per_dst=[5874, 6146, 6488, 5150, 4966, 4902, 4804, 5378] dup_rows_per_dst=[3402, 3506, 4844, 2344, 1518, 2082, 1994, 2138] combine_rows_per_src=[5874, 6146, 6488, 5150, 4966, 4902, 4804, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44640 payload_rows_remote=39026 dup_rows=20896 dup_rows_remote=17828 dedup_ratio=0.318848 router_tokens_per_expert=[386, 1, 0, 1, 1135, 600, 402, 0, 0, 4, 191, 307, 103, 150, 492, 103, 4, 18, 5, 163, 541, 188, 4, 0, 239, 719, 13, 49, 25, 2, 208, 11, 327, 369, 1, 76, 5, 29, 739, 5, 60, 217, 328, 1505, 450, 8, 63, 634, 65, 88, 58, 15, 171, 1158, 4, 21, 723, 19, 126, 480, 24, 279, 4, 22, 658, 236, 2, 3, 120, 0, 195, 721, 0, 267, 603, 246, 3, 225, 15, 1713, 694, 16, 29, 289, 17, 204, 0, 1, 858, 220, 9, 479, 32, 405, 127, 5, 0, 77, 814, 36, 3, 1140, 1772, 37, 367, 20, 217, 34, 224, 599, 155, 0, 867, 570, 230, 19, 467, 10, 898, 96, 95, 1053, 237, 19, 60, 1, 122, 0] payload_rows_per_dst=[5364, 5694, 5614, 6328, 4948, 5140, 6046, 5506] dup_rows_per_dst=[2386, 2228, 3068, 3612, 1640, 2580, 3196, 2186] combine_rows_per_src=[5364, 5694, 5614, 6328, 4948, 5140, 6046, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=37654 dup_rows=21050 dup_rows_remote=15776 dedup_ratio=0.321198 router_tokens_per_expert=[320, 0, 0, 0, 1168, 593, 424, 4, 0, 8, 161, 245, 156, 157, 586, 151, 3, 12, 0, 157, 546, 156, 8, 0, 228, 778, 28, 46, 17, 6, 148, 19, 361, 239, 8, 77, 4, 25, 813, 11, 106, 252, 321, 1849, 550, 4, 29, 639, 50, 114, 39, 0, 207, 812, 3, 13, 787, 26, 119, 644, 41, 261, 5, 23, 599, 231, 0, 3, 121, 0, 140, 667, 1, 263, 707, 214, 4, 92, 18, 2073, 786, 27, 11, 440, 4, 173, 7, 1, 689, 129, 14, 540, 66, 420, 84, 6, 0, 60, 643, 29, 11, 1284, 1747, 33, 522, 13, 213, 23, 184, 590, 125, 0, 956, 455, 195, 12, 355, 4, 672, 74, 102, 845, 244, 13, 180, 3, 93, 1] payload_rows_per_dst=[5428, 5604, 4990, 4708, 6598, 4796, 5530, 6832] dup_rows_per_dst=[2518, 2194, 1888, 1580, 3668, 1998, 1930, 5274] combine_rows_per_src=[5428, 5604, 4990, 4708, 6598, 4796, 5530, 6832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=222088 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156763016 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313303944 weight_movement_bytes=18874368 total_nvlink_bytes=332178312 weight_to_token_ratio=0.060243 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=225032 combine_payload_bytes=159416320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159641352 ingress_bytes=178290688 link_bytes=178290688 token_bytes=319057672 weight_movement_bytes=18874368 total_nvlink_bytes=337932040 weight_to_token_ratio=0.059157 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.708293648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=226912 combine_payload_bytes=157818880 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=158045792 ingress_bytes=195567616 link_bytes=195567616 token_bytes=315864672 weight_movement_bytes=37748736 total_nvlink_bytes=353613408 weight_to_token_ratio=0.119509 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.076389 gemm_active_groups=19 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873832671e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163725312 dispatch_meta_bytes=236960 combine_payload_bytes=163725312 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=163962272 ingress_bytes=220348416 link_bytes=220348416 token_bytes=327687584 weight_movement_bytes=56623104 total_nvlink_bytes=384310688 weight_to_token_ratio=0.172796 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=6144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.111270104e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=154968064 dispatch_meta_bytes=225040 combine_payload_bytes=154968064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155193104 ingress_bytes=173842432 link_bytes=173842432 token_bytes=310161168 weight_movement_bytes=18874368 total_nvlink_bytes=329035536 weight_to_token_ratio=0.060853 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=9472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.665672648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159850496 dispatch_meta_bytes=227416 combine_payload_bytes=159850496 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=160077912 ingress_bytes=216473600 link_bytes=216473600 token_bytes=319928408 weight_movement_bytes=56623104 total_nvlink_bytes=376551512 weight_to_token_ratio=0.176987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=5632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.074143524e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=154230784 dispatch_meta_bytes=213720 combine_payload_bytes=154230784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=154444504 ingress_bytes=191979520 link_bytes=191979520 token_bytes=308675288 weight_movement_bytes=37748736 total_nvlink_bytes=346424024 weight_to_token_ratio=0.122293 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.839453301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60826 payload_rows_remote=53124 dup_rows=4710 dup_rows_remote=4708 dedup_ratio=0.071869 router_tokens_per_expert=[0, 2, 1678, 3064, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 8, 94, 0, 0, 0, 0, 4, 23, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 292, 0, 0, 3262, 0, 3473, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 151, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 17, 2, 0, 3649, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4018, 27, 0, 0, 84, 0, 3, 0, 0, 4094, 2, 0, 0, 0] payload_rows_per_dst=[7450, 8192, 6612, 7702, 7326, 8190, 7308, 8046] dup_rows_per_dst=[2410, 352, 506, 2, 1150, 56, 12, 222] combine_rows_per_src=[7450, 8192, 6612, 7702, 7326, 8190, 7308, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60136 payload_rows_remote=52042 dup_rows=5400 dup_rows_remote=5232 dedup_ratio=0.082397 router_tokens_per_expert=[15, 0, 3781, 1411, 0, 0, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 2, 2, 10, 68, 6, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 1, 155, 0, 1, 2873, 0, 3323, 0, 0, 1, 0, 26, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 6, 1, 0, 0, 4019, 0, 0, 130, 3, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 806, 0, 0, 1, 0, 7, 0, 0, 5, 0, 0, 0, 0, 0, 0, 157, 0, 25, 0, 1, 3754, 11, 2, 0, 0, 8, 0, 4055, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3858, 1, 0, 0, 50, 0, 22, 0, 0, 3936, 1, 6, 0, 0] payload_rows_per_dst=[6714, 8150, 6512, 8056, 7280, 8094, 7546, 7784] dup_rows_per_dst=[1902, 514, 1432, 2, 1270, 168, 20, 92] combine_rows_per_src=[6714, 8150, 6512, 8056, 7280, 8094, 7546, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60552 payload_rows_remote=52368 dup_rows=4984 dup_rows_remote=3594 dedup_ratio=0.076050 router_tokens_per_expert=[0, 0, 2670, 1527, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 261, 0, 0, 0, 0, 5, 16, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 650, 0, 0, 2998, 0, 3396, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 56, 0, 3, 0, 0, 0, 0, 0, 22, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 28, 0, 8, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 3, 0, 0, 36, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[6912, 8184, 6376, 8042, 7196, 8192, 7754, 7896] dup_rows_per_dst=[1494, 1390, 926, 0, 1024, 76, 0, 74] combine_rows_per_src=[6912, 8184, 6376, 8042, 7196, 8192, 7754, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59570 payload_rows_remote=53194 dup_rows=5966 dup_rows_remote=5236 dedup_ratio=0.091034 router_tokens_per_expert=[0, 0, 2826, 2083, 0, 1, 1, 12, 0, 0, 0, 0, 0, 0, 1, 4, 988, 1, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 0, 245, 0, 0, 0, 0, 98, 0, 0, 3019, 0, 2940, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4056, 2, 0, 25, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 9, 0, 2, 0, 0, 3597, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 3, 1, 0, 7, 0, 7, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[7330, 8192, 6080, 8118, 6376, 8190, 7194, 8090] dup_rows_per_dst=[2526, 2474, 170, 2, 730, 30, 0, 34] combine_rows_per_src=[7330, 8192, 6080, 8118, 6376, 8190, 7194, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55562 payload_rows_remote=47740 dup_rows=9974 dup_rows_remote=5974 dedup_ratio=0.152191 router_tokens_per_expert=[0, 4, 3506, 1714, 5, 2, 0, 11, 0, 0, 0, 0, 45, 18, 0, 1, 617, 0, 2, 3, 0, 6, 17, 0, 15, 2, 0, 0, 5, 12, 597, 0, 0, 0, 11, 688, 6, 5, 2308, 0, 3034, 0, 0, 0, 2, 127, 0, 0, 11, 19, 0, 0, 1, 23, 1, 0, 0, 0, 0, 0, 3, 0, 3894, 0, 7, 197, 0, 28, 14, 6, 0, 0, 2, 36, 0, 0, 0, 422, 0, 0, 0, 3, 0, 0, 0, 29, 3, 0, 37, 10, 1, 5, 6, 4, 157, 0, 0, 3154, 0, 0, 0, 12, 30, 0, 3787, 0, 0, 9, 6, 35, 0, 2, 0, 0, 3893, 12, 0, 0, 17, 2, 1, 1, 3, 4094, 27, 0, 0, 1] payload_rows_per_dst=[7822, 8052, 5508, 7888, 6606, 8192, 3644, 7850] dup_rows_per_dst=[4000, 2074, 900, 1634, 772, 506, 24, 64] combine_rows_per_src=[7822, 8052, 5508, 7888, 6606, 8192, 3644, 7850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=217595904 dispatch_meta_bytes=231328 combine_payload_bytes=217595904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217827232 ingress_bytes=236470272 link_bytes=236470272 token_bytes=435423136 weight_movement_bytes=18874368 total_nvlink_bytes=454297504 weight_to_token_ratio=0.043347 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.065972 gemm_active_groups=15 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265741796e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=213164032 dispatch_meta_bytes=229096 combine_payload_bytes=213164032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213393128 ingress_bytes=232038400 link_bytes=232038400 token_bytes=426557160 weight_movement_bytes=18874368 total_nvlink_bytes=445431528 weight_to_token_ratio=0.044248 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.069444 gemm_active_groups=16 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223277779e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=214499328 dispatch_meta_bytes=223848 combine_payload_bytes=214499328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214723176 ingress_bytes=233373696 link_bytes=233373696 token_bytes=429222504 weight_movement_bytes=18874368 total_nvlink_bytes=448096872 weight_to_token_ratio=0.043973 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.236071928e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=217882624 dispatch_meta_bytes=233720 combine_payload_bytes=217882624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218116344 ingress_bytes=236756992 link_bytes=236756992 token_bytes=435998968 weight_movement_bytes=18874368 total_nvlink_bytes=454873336 weight_to_token_ratio=0.043290 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.268489006e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=195543040 dispatch_meta_bytes=214856 combine_payload_bytes=195543040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=195757896 ingress_bytes=233291776 link_bytes=233291776 token_bytes=391300936 weight_movement_bytes=37748736 total_nvlink_bytes=429049672 weight_to_token_ratio=0.096470 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235287011e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60424 payload_rows_remote=53892 dup_rows=5112 dup_rows_remote=4870 dedup_ratio=0.078003 router_tokens_per_expert=[0, 0, 2587, 2336, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 10, 741, 0, 0, 0, 0, 15, 20, 0, 0, 0, 0, 0, 0, 1, 152, 0, 0, 0, 7, 133, 0, 0, 3239, 0, 3305, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 58, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 16, 0, 1, 4, 0, 3592, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4018, 7, 0, 0, 3, 0, 4, 0, 0, 4092, 0, 0, 0, 0] payload_rows_per_dst=[7470, 8190, 6532, 7964, 6854, 8192, 7186, 8036] dup_rows_per_dst=[2416, 1852, 242, 0, 520, 52, 2, 28] combine_rows_per_src=[7470, 8190, 6532, 7964, 6854, 8192, 7186, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51754 payload_rows_remote=43734 dup_rows=13782 dup_rows_remote=12646 dedup_ratio=0.210297 router_tokens_per_expert=[0, 1, 3449, 1503, 1, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 486, 0, 0, 0, 0, 47, 14, 0, 1, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 188, 0, 0, 1754, 0, 3334, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4077, 0, 0, 86, 0, 18, 0, 0, 1, 0, 1, 2, 0, 0, 0, 1544, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7, 2, 1, 2, 6, 0, 0, 3648, 0, 0, 0, 0, 0, 0, 4033, 1, 0, 0, 1, 2, 3, 0, 0, 0, 4003, 0, 0, 0, 10, 0, 2, 0, 0, 4093, 3, 0, 0, 0] payload_rows_per_dst=[3018, 1808, 7854, 8158, 7580, 7194, 8122, 8020] dup_rows_per_dst=[4, 138, 2934, 12, 2392, 32, 7134, 1136] combine_rows_per_src=[3018, 1808, 7854, 8158, 7580, 7194, 8122, 8020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59500 payload_rows_remote=51412 dup_rows=6036 dup_rows_remote=6036 dedup_ratio=0.092102 router_tokens_per_expert=[0, 0, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 2, 0, 0, 0, 3669, 8, 3545, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4044, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4196, 7472, 7004, 8088, 8192, 8192, 8182, 8174] dup_rows_per_dst=[58, 2126, 2476, 0, 38, 0, 0, 1338] combine_rows_per_src=[4196, 7472, 7004, 8088, 8192, 8192, 8182, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=220741632 dispatch_meta_bytes=235048 combine_payload_bytes=220741632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220976680 ingress_bytes=258490368 link_bytes=258490368 token_bytes=441718312 weight_movement_bytes=37748736 total_nvlink_bytes=479467048 weight_to_token_ratio=0.085459 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=22400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.476727521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=179134464 dispatch_meta_bytes=225520 combine_payload_bytes=179134464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=179359984 ingress_bytes=216883200 link_bytes=216883200 token_bytes=358494448 weight_movement_bytes=37748736 total_nvlink_bytes=396243184 weight_to_token_ratio=0.105298 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.078068109e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59822 payload_rows_remote=51666 dup_rows=5714 dup_rows_remote=5698 dedup_ratio=0.087189 router_tokens_per_expert=[0, 0, 2901, 0, 0, 23, 0, 0, 0, 0, 0, 273, 0, 0, 3, 0, 5, 0, 1, 0, 0, 0, 0, 0, 22, 0, 2, 0, 0, 0, 158, 0, 703, 2, 0, 0, 3160, 0, 2941, 0, 22, 1, 0, 0, 7, 2, 1480, 17, 0, 0, 0, 0, 7, 0, 46, 0, 0, 0, 4073, 16, 0, 3439, 1, 0, 0, 0, 0, 97, 0, 0, 0, 4055, 0, 13, 7, 0, 0, 0, 0, 0, 0, 23, 4055, 0, 0, 0, 0, 0, 3, 0, 3, 1, 0, 0, 1, 0, 0, 0, 4074, 0, 16, 7, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[6112, 6410, 7658, 7004, 8156, 8156, 8180, 8146] dup_rows_per_dst=[288, 286, 2692, 14, 188, 16, 34, 2196] combine_rows_per_src=[6112, 6410, 7658, 7004, 8156, 8156, 8180, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61744 payload_rows_remote=53554 dup_rows=3792 dup_rows_remote=3790 dedup_ratio=0.057861 router_tokens_per_expert=[0, 0, 3324, 0, 0, 2, 0, 0, 0, 0, 0, 284, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 378, 1, 2, 0, 0, 0, 3225, 6, 3391, 0, 2, 0, 0, 0, 2, 0, 1494, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 3953, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6874, 6622, 7586, 7910, 8190, 8190, 8184, 8188] dup_rows_per_dst=[348, 590, 2210, 0, 2, 0, 0, 642] combine_rows_per_src=[6874, 6622, 7586, 7910, 8190, 8190, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56686 payload_rows_remote=49906 dup_rows=8850 dup_rows_remote=8028 dedup_ratio=0.135040 router_tokens_per_expert=[0, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 541, 0, 107, 0, 0, 0, 3255, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3578, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7908, 6780, 2854, 7434, 8190, 8192, 8190, 7138] dup_rows_per_dst=[4200, 822, 16, 2044, 8, 0, 4, 1756] combine_rows_per_src=[7908, 6780, 2854, 7434, 8190, 8192, 8190, 7138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54396 payload_rows_remote=47080 dup_rows=11140 dup_rows_remote=8354 dedup_ratio=0.169983 router_tokens_per_expert=[0, 0, 1852, 0, 0, 0, 1, 0, 0, 0, 54, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 10, 0, 0, 0, 685, 10, 39, 0, 0, 0, 3695, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 3983, 0, 0, 0, 0, 0, 153, 0, 0, 0, 4008, 0, 2, 0, 0, 0, 94, 0, 0, 0, 0, 4087, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 2, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7316, 7580, 7146, 8000, 8180, 8096, 7480, 598] dup_rows_per_dst=[2786, 1652, 3676, 2504, 406, 114, 0, 2] combine_rows_per_src=[7316, 7580, 7146, 8000, 8180, 8096, 7480, 598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=210583552 dispatch_meta_bytes=229792 combine_payload_bytes=210583552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210813344 ingress_bytes=267206656 link_bytes=267206656 token_bytes=421396896 weight_movement_bytes=56623104 total_nvlink_bytes=478020000 weight_to_token_ratio=0.134370 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.560242704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=211623936 dispatch_meta_bytes=229456 combine_payload_bytes=211623936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=211853392 ingress_bytes=249372672 link_bytes=249372672 token_bytes=423477328 weight_movement_bytes=37748736 total_nvlink_bytes=461226064 weight_to_token_ratio=0.089140 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.389366244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44198 payload_rows_remote=38898 dup_rows=21338 dup_rows_remote=19218 dedup_ratio=0.325592 router_tokens_per_expert=[439, 1, 0, 3, 1140, 521, 349, 1, 1, 0, 167, 276, 103, 165, 419, 123, 5, 20, 2, 154, 415, 184, 5, 2, 235, 675, 22, 47, 6, 3, 226, 10, 267, 314, 5, 94, 1, 46, 611, 15, 63, 284, 455, 1369, 660, 12, 73, 714, 51, 85, 91, 11, 158, 1094, 6, 22, 796, 21, 77, 449, 30, 327, 7, 54, 695, 176, 2, 4, 90, 0, 168, 782, 7, 221, 813, 310, 21, 212, 5, 2135, 640, 31, 26, 221, 19, 220, 5, 0, 908, 205, 19, 616, 33, 405, 123, 4, 0, 60, 882, 34, 0, 1103, 1594, 78, 363, 17, 199, 30, 297, 439, 208, 0, 648, 566, 258, 12, 416, 8, 903, 99, 181, 819, 236, 8, 87, 6, 120, 0] payload_rows_per_dst=[5158, 5340, 5166, 4890, 6968, 5678, 5300, 5698] dup_rows_per_dst=[2258, 1870, 2062, 1668, 4314, 3628, 2120, 3418] combine_rows_per_src=[5158, 5340, 5166, 4890, 6968, 5678, 5300, 5698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=219357184 dispatch_meta_bytes=229376 combine_payload_bytes=219357184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219586560 ingress_bytes=257105920 link_bytes=257105920 token_bytes=438943744 weight_movement_bytes=37748736 total_nvlink_bytes=476692480 weight_to_token_ratio=0.085999 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.463462421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=204414976 dispatch_meta_bytes=231736 combine_payload_bytes=204414976 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=204646712 ingress_bytes=242163712 link_bytes=242163712 token_bytes=409061688 weight_movement_bytes=37748736 total_nvlink_bytes=446810424 weight_to_token_ratio=0.092281 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.320293536e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=192839680 dispatch_meta_bytes=221736 combine_payload_bytes=192839680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193061416 ingress_bytes=211714048 link_bytes=211714048 token_bytes=385901096 weight_movement_bytes=18874368 total_nvlink_bytes=404775464 weight_to_token_ratio=0.048910 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.028539838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60534 payload_rows_remote=52358 dup_rows=5002 dup_rows_remote=3654 dedup_ratio=0.076324 router_tokens_per_expert=[0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 350, 0, 681, 0, 0, 0, 3335, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3365, 0, 0, 0, 1, 5, 22, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7446, 6742, 7678, 6730, 7424, 8192, 8146, 8176] dup_rows_per_dst=[608, 644, 1576, 0, 56, 4, 766, 1348] combine_rows_per_src=[7446, 6742, 7678, 6730, 7424, 8192, 8146, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58490 payload_rows_remote=55500 dup_rows=7046 dup_rows_remote=7046 dedup_ratio=0.107513 router_tokens_per_expert=[0, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 11, 0, 0, 0, 3151, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7606, 7286, 2990, 7844, 8192, 8192, 8188, 8192] dup_rows_per_dst=[3912, 2664, 0, 0, 100, 0, 0, 370] combine_rows_per_src=[7606, 7286, 2990, 7844, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61884 payload_rows_remote=54350 dup_rows=3652 dup_rows_remote=3458 dedup_ratio=0.055725 router_tokens_per_expert=[0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 124, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8098, 8192, 8190, 7534, 8192, 5296, 8190, 8192] dup_rows_per_dst=[1452, 1466, 8, 194, 132, 0, 0, 400] combine_rows_per_src=[8098, 8192, 8190, 7534, 8192, 5296, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=214458368 dispatch_meta_bytes=224048 combine_payload_bytes=214458368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214682416 ingress_bytes=233332736 link_bytes=233332736 token_bytes=429140784 weight_movement_bytes=18874368 total_nvlink_bytes=448015152 weight_to_token_ratio=0.043982 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235679469e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=227328000 dispatch_meta_bytes=250184 combine_payload_bytes=227328000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=227578184 ingress_bytes=302825472 link_bytes=302825472 token_bytes=454906184 weight_movement_bytes=75497472 total_nvlink_bytes=530403656 weight_to_token_ratio=0.165963 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.901524673e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58282 payload_rows_remote=50132 dup_rows=7254 dup_rows_remote=5040 dedup_ratio=0.110687 router_tokens_per_expert=[0, 1123, 0, 2, 0, 0, 0, 0, 0, 0, 25, 3981, 0, 0, 5, 0, 0, 0, 363, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 4064, 7, 0, 29, 0, 0, 7, 0, 0, 22, 0, 0, 35, 0, 0, 5, 27, 0, 0, 0, 3447, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 565, 0, 0, 2, 2, 3, 0, 0, 0, 0, 0, 0, 24, 0, 17, 7, 0, 0, 7, 0, 4070, 0, 26, 1090, 0, 0, 53, 1, 0, 0, 4066, 0, 0, 0, 2, 0, 21, 0, 11, 4067, 6, 0, 0, 1, 0, 19, 0, 0, 24, 3, 1, 0, 0, 0, 0, 1422, 4041, 26, 0, 0, 0] payload_rows_per_dst=[2300, 8156, 8188, 6926, 8186, 8150, 8186, 8190] dup_rows_per_dst=[10, 802, 142, 36, 1138, 2214, 30, 2882] combine_rows_per_src=[2300, 8156, 8188, 6926, 8186, 8150, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=232464 combine_payload_bytes=159326208 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=159558672 ingress_bytes=215949312 link_bytes=215949312 token_bytes=318884880 weight_movement_bytes=56623104 total_nvlink_bytes=375507984 weight_to_token_ratio=0.177566 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=7808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.069120054e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59278 payload_rows_remote=51088 dup_rows=6258 dup_rows_remote=4712 dedup_ratio=0.095490 router_tokens_per_expert=[0, 782, 0, 0, 0, 0, 2, 0, 0, 0, 2, 4071, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 3, 17, 0, 0, 435, 0, 2, 0, 2591, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 0, 4094, 0, 0, 2432, 0, 0, 5, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 17, 0, 0, 2, 1, 0, 0, 0, 0, 0, 248, 4094, 0, 0, 0, 0] payload_rows_per_dst=[6218, 8188, 8112, 5836, 8190, 6352, 8190, 8192] dup_rows_per_dst=[1036, 1888, 10, 258, 1546, 988, 0, 532] combine_rows_per_src=[6218, 8188, 8112, 5836, 8190, 6352, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57370 payload_rows_remote=49178 dup_rows=8166 dup_rows_remote=6814 dedup_ratio=0.124603 router_tokens_per_expert=[0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 674, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 79, 0, 0, 0, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1894, 0, 0, 248, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7084, 8190, 4044, 8192, 5284] dup_rows_per_dst=[4304, 1352, 30, 128, 1060, 240, 2, 1050] combine_rows_per_src=[8192, 8192, 8192, 7084, 8190, 4044, 8192, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45502 payload_rows_remote=37310 dup_rows=20034 dup_rows_remote=7404 dedup_ratio=0.305695 router_tokens_per_expert=[0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 19, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 3797, 0, 0, 2137, 0, 0, 114, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4037, 0, 0, 29, 0, 2, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 1, 0, 0, 0] payload_rows_per_dst=[8192, 7696, 8112, 8160, 8192, 4286, 324, 540] dup_rows_per_dst=[12630, 66, 210, 6626, 168, 334, 0, 0] combine_rows_per_src=[8192, 7696, 8112, 8160, 8192, 4286, 324, 540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=222617600 dispatch_meta_bytes=231232 combine_payload_bytes=222617600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222848832 ingress_bytes=241491968 link_bytes=241491968 token_bytes=445466432 weight_movement_bytes=18874368 total_nvlink_bytes=464340800 weight_to_token_ratio=0.042370 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.313857216e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205340672 dispatch_meta_bytes=220688 combine_payload_bytes=205340672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205561360 ingress_bytes=243089408 link_bytes=243089408 token_bytes=410902032 weight_movement_bytes=37748736 total_nvlink_bytes=448650768 weight_to_token_ratio=0.091868 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329163100e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=209256448 dispatch_meta_bytes=223200 combine_payload_bytes=209256448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209479648 ingress_bytes=228130816 link_bytes=228130816 token_bytes=418736096 weight_movement_bytes=18874368 total_nvlink_bytes=437610464 weight_to_token_ratio=0.045075 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.185837232e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=201433088 dispatch_meta_bytes=223968 combine_payload_bytes=201433088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201657056 ingress_bytes=220307456 link_bytes=220307456 token_bytes=403090144 weight_movement_bytes=18874368 total_nvlink_bytes=421964512 weight_to_token_ratio=0.046824 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.110877645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=152821760 dispatch_meta_bytes=178856 combine_payload_bytes=152821760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=153000616 ingress_bytes=190570496 link_bytes=190570496 token_bytes=305822376 weight_movement_bytes=37748736 total_nvlink_bytes=343571112 weight_to_token_ratio=0.123434 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.825952726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=35460 dup_rows=21884 dup_rows_remote=17526 dedup_ratio=0.333923 router_tokens_per_expert=[0, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 58, 0, 0, 0, 3334, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 1164, 0, 0, 121, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 4096, 0, 0, 0, 0] payload_rows_per_dst=[1908, 240, 8186, 6728, 2014, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 58, 0, 10648, 6818, 4358] combine_rows_per_src=[1908, 240, 8186, 6728, 2014, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58198 payload_rows_remote=50242 dup_rows=7338 dup_rows_remote=7328 dedup_ratio=0.111969 router_tokens_per_expert=[0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 716, 0, 0, 0, 3471, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4091, 0, 0, 995, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 405, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8182, 7956, 7320, 8192, 1994, 8186, 8192] dup_rows_per_dst=[1288, 2318, 10, 1058, 1730, 24, 0, 910] combine_rows_per_src=[8176, 8182, 7956, 7320, 8192, 1994, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62144 payload_rows_remote=54086 dup_rows=3392 dup_rows_remote=3392 dedup_ratio=0.051758 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 2662, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7798, 8172, 8192, 8058, 8062, 8192, 5478, 8192] dup_rows_per_dst=[0, 2, 0, 0, 3248, 0, 142, 0] combine_rows_per_src=[7798, 8172, 8192, 8058, 8062, 8192, 5478, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=145244160 dispatch_meta_bytes=211944 combine_payload_bytes=145244160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=145456104 ingress_bytes=182992896 link_bytes=182992896 token_bytes=290700264 weight_movement_bytes=37748736 total_nvlink_bytes=328449000 weight_to_token_ratio=0.129854 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.753347891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64340 payload_rows_remote=56150 dup_rows=1196 dup_rows_remote=1070 dedup_ratio=0.018250 router_tokens_per_expert=[0, 0, 0, 0, 26, 0, 0, 4045, 1, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 26, 15, 19, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 2, 0, 0, 3754, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 25, 20, 0, 0, 7, 0, 4066, 19, 2, 0, 0, 27, 4067, 436, 0, 4065, 0, 0, 0, 0, 0, 0, 1, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0] payload_rows_per_dst=[7838, 8130, 8152, 8126, 7528, 8190, 8190, 8186] dup_rows_per_dst=[0, 50, 62, 18, 8, 126, 910, 22] combine_rows_per_src=[7838, 8130, 8152, 8126, 7528, 8190, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56096 payload_rows_remote=49090 dup_rows=9440 dup_rows_remote=6856 dedup_ratio=0.144043 router_tokens_per_expert=[0, 10, 3491, 1935, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 28, 28, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 112, 0, 0, 2873, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 94, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 9, 14, 44, 0, 25, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 3, 0, 1, 0, 0, 3, 4049, 7, 0, 0, 19, 0, 3, 0, 0, 4089, 1, 0, 0, 0] payload_rows_per_dst=[4050, 5430, 8062, 7956, 7304, 8188, 7006, 8100] dup_rows_per_dst=[18, 544, 4890, 0, 1156, 184, 2584, 64] combine_rows_per_src=[4050, 5430, 8062, 7956, 7304, 8188, 7006, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205791232 dispatch_meta_bytes=230280 combine_payload_bytes=205791232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206021512 ingress_bytes=224665600 link_bytes=224665600 token_bytes=411812744 weight_movement_bytes=18874368 total_nvlink_bytes=430687112 weight_to_token_ratio=0.045832 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.152635237e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60868 payload_rows_remote=56584 dup_rows=4668 dup_rows_remote=4668 dedup_ratio=0.071228 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3898, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2, 4094, 2331, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8036, 7800, 8186, 7990, 4284, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 4656, 4] combine_rows_per_src=[8036, 7800, 8186, 7990, 4284, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61110 payload_rows_remote=53084 dup_rows=4426 dup_rows_remote=4426 dedup_ratio=0.067535 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4033, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 2084, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8068, 8026, 8192, 8082, 8192, 8190, 4168, 8192] dup_rows_per_dst=[0, 0, 8, 0, 4410, 8, 0, 0] combine_rows_per_src=[8068, 8026, 8192, 8082, 8192, 8190, 4168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=221536256 dispatch_meta_bytes=229912 combine_payload_bytes=221536256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221766168 ingress_bytes=240410624 link_bytes=240410624 token_bytes=443302424 weight_movement_bytes=18874368 total_nvlink_bytes=462176792 weight_to_token_ratio=0.042577 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.303496310e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59906 payload_rows_remote=51762 dup_rows=5630 dup_rows_remote=3896 dedup_ratio=0.085907 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 208, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 4096, 2988, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8188, 8106, 8098, 5044, 8184, 5976, 8166] dup_rows_per_dst=[1734, 830, 18, 892, 1986, 144, 26, 0] combine_rows_per_src=[8144, 8188, 8106, 8098, 5044, 8184, 5976, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=229990400 dispatch_meta_bytes=228880 combine_payload_bytes=229990400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230219280 ingress_bytes=248864768 link_bytes=248864768 token_bytes=460209680 weight_movement_bytes=18874368 total_nvlink_bytes=479084048 weight_to_token_ratio=0.041013 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384499758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=231768064 dispatch_meta_bytes=245008 combine_payload_bytes=231768064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232013072 ingress_bytes=250642432 link_bytes=250642432 token_bytes=463781136 weight_movement_bytes=18874368 total_nvlink_bytes=482655504 weight_to_token_ratio=0.040697 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401532460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=217432064 dispatch_meta_bytes=230040 combine_payload_bytes=217432064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217662104 ingress_bytes=236306432 link_bytes=236306432 token_bytes=435094168 weight_movement_bytes=18874368 total_nvlink_bytes=453968536 weight_to_token_ratio=0.043380 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264171961e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=201072640 dispatch_meta_bytes=223784 combine_payload_bytes=201072640 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=201296424 ingress_bytes=276570112 link_bytes=276570112 token_bytes=402369064 weight_movement_bytes=75497472 total_nvlink_bytes=477866536 weight_to_token_ratio=0.187632 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.072917 gemm_active_groups=14 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.649958732e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=212017152 dispatch_meta_bytes=222632 combine_payload_bytes=212017152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212239784 ingress_bytes=249765888 link_bytes=249765888 token_bytes=424256936 weight_movement_bytes=37748736 total_nvlink_bytes=462005672 weight_to_token_ratio=0.088976 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393133846e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62530 payload_rows_remote=54338 dup_rows=3006 dup_rows_remote=3006 dedup_ratio=0.045868 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1268, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8018, 8144, 8192, 8192, 5788, 7812, 8192, 8192] dup_rows_per_dst=[0, 0, 10, 2, 0, 90, 2904, 0] combine_rows_per_src=[8018, 8144, 8192, 8192, 5788, 7812, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62594 payload_rows_remote=54402 dup_rows=2942 dup_rows_remote=2936 dedup_ratio=0.044891 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 2715, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8172, 8158, 8192, 8074, 8184, 8192, 5430, 8192] dup_rows_per_dst=[0, 0, 6, 0, 2922, 8, 0, 6] combine_rows_per_src=[8172, 8158, 8192, 8074, 8184, 8192, 5430, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58338 payload_rows_remote=50146 dup_rows=7198 dup_rows_remote=6964 dedup_ratio=0.109833 router_tokens_per_expert=[0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 0, 0, 0, 890, 0, 0, 0, 1, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 97, 1, 0, 0, 0, 0, 4096, 0, 270, 4095, 2, 0, 4096, 0, 0] payload_rows_per_dst=[7916, 8192, 6456, 8192, 8188, 8190, 3012, 8192] dup_rows_per_dst=[0, 3682, 2484, 234, 58, 0, 0, 740] combine_rows_per_src=[7916, 8192, 6456, 8192, 8188, 8190, 3012, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58480 payload_rows_remote=50288 dup_rows=7056 dup_rows_remote=6990 dedup_ratio=0.107666 router_tokens_per_expert=[0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 22, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 0, 37, 7, 0, 0, 0, 0, 2166, 0, 5, 30, 0, 0, 29, 0, 0, 0, 0, 0, 0, 9, 0, 0, 24, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 2, 3, 0, 30, 4065, 0, 0, 5, 0, 0, 0, 0, 0, 0, 28, 30, 0, 0, 0, 0, 2, 0, 0, 13, 4062, 0, 0, 2, 0, 0, 0, 0, 3, 0, 5, 0, 4062, 0, 0, 0, 7, 0, 0, 0, 1, 0, 4062, 0, 10, 4064, 0, 0, 4030, 0, 0] payload_rows_per_dst=[8136, 5086, 4462, 8172, 8190, 8192, 8162, 8080] dup_rows_per_dst=[42, 2088, 86, 20, 42, 66, 4696, 16] combine_rows_per_src=[8136, 5086, 4462, 8172, 8190, 8192, 8162, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222568448 dispatch_meta_bytes=229376 combine_payload_bytes=222568448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222797824 ingress_bytes=241442816 link_bytes=241442816 token_bytes=445366272 weight_movement_bytes=18874368 total_nvlink_bytes=464240640 weight_to_token_ratio=0.042379 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.313386266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57642 payload_rows_remote=49450 dup_rows=7894 dup_rows_remote=5778 dedup_ratio=0.120453 router_tokens_per_expert=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 457, 0, 0, 0, 0, 0, 4095, 1, 45, 4096, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8190, 8192, 8192, 3466, 5032] dup_rows_per_dst=[0, 2116, 5010, 0, 68, 0, 0, 700] combine_rows_per_src=[8186, 8192, 8192, 8190, 8192, 8192, 3466, 5032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62634 payload_rows_remote=54442 dup_rows=2902 dup_rows_remote=2896 dedup_ratio=0.044281 router_tokens_per_expert=[0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 2614, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8124, 8188, 5832, 8192, 8192, 8192, 7726, 8188] dup_rows_per_dst=[0, 346, 190, 8, 6, 4, 0, 2348] combine_rows_per_src=[8124, 8188, 5832, 8192, 8192, 8192, 7726, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222830592 dispatch_meta_bytes=229352 combine_payload_bytes=222830592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223059944 ingress_bytes=241704960 link_bytes=241704960 token_bytes=445890536 weight_movement_bytes=18874368 total_nvlink_bytes=464764904 weight_to_token_ratio=0.042330 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.315898000e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205398016 dispatch_meta_bytes=228440 combine_payload_bytes=205398016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205626456 ingress_bytes=224272384 link_bytes=224272384 token_bytes=411024472 weight_movement_bytes=18874368 total_nvlink_bytes=429898840 weight_to_token_ratio=0.045920 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.148867635e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61300 payload_rows_remote=53118 dup_rows=4236 dup_rows_remote=4236 dedup_ratio=0.064636 router_tokens_per_expert=[0, 0, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 252, 0, 34, 0, 0, 0, 3672, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3982, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5376, 7358, 7844, 7964, 8192, 8192, 8182, 8192] dup_rows_per_dst=[502, 502, 2130, 0, 22, 0, 0, 1080] combine_rows_per_src=[5376, 7358, 7844, 7964, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55058 payload_rows_remote=47360 dup_rows=10478 dup_rows_remote=9356 dedup_ratio=0.159882 router_tokens_per_expert=[0, 0, 0, 0, 0, 3538, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 44, 0, 0, 0, 19, 0, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4095, 0, 248, 4094, 1, 0, 3846, 0, 0] payload_rows_per_dst=[7698, 8190, 8132, 8192, 8192, 8188, 5958, 508] dup_rows_per_dst=[1122, 3940, 3944, 1284, 186, 0, 0, 2] combine_rows_per_src=[7698, 8190, 8132, 8192, 8192, 8188, 5958, 508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205979648 dispatch_meta_bytes=229112 combine_payload_bytes=205979648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206208760 ingress_bytes=224854016 link_bytes=224854016 token_bytes=412188408 weight_movement_bytes=18874368 total_nvlink_bytes=431062776 weight_to_token_ratio=0.045791 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.154440546e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=202547200 dispatch_meta_bytes=220912 combine_payload_bytes=202547200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202768112 ingress_bytes=221421568 link_bytes=221421568 token_bytes=405315312 weight_movement_bytes=18874368 total_nvlink_bytes=424189680 weight_to_token_ratio=0.046567 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.121552518e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=222994432 dispatch_meta_bytes=229352 combine_payload_bytes=222994432 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=223223784 ingress_bytes=241868800 link_bytes=241868800 token_bytes=446218216 weight_movement_bytes=18874368 total_nvlink_bytes=465092584 weight_to_token_ratio=0.042299 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.317467835e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=217571328 dispatch_meta_bytes=229416 combine_payload_bytes=217571328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217800744 ingress_bytes=236445696 link_bytes=236445696 token_bytes=435372072 weight_movement_bytes=18874368 total_nvlink_bytes=454246440 weight_to_token_ratio=0.043352 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265506321e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=193986560 dispatch_meta_bytes=226864 combine_payload_bytes=193986560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194213424 ingress_bytes=212860928 link_bytes=212860928 token_bytes=388199984 weight_movement_bytes=18874368 total_nvlink_bytes=407074352 weight_to_token_ratio=0.048620 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039528678e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54854 payload_rows_remote=46662 dup_rows=10682 dup_rows_remote=8228 dedup_ratio=0.162994 router_tokens_per_expert=[0, 0, 0, 0, 0, 4074, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2947, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4095, 0, 8, 4095, 0, 0, 4092, 0, 0] payload_rows_per_dst=[8152, 5894, 2204, 6368, 7724, 8130, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 74, 36, 0, 8118, 2454] combine_rows_per_src=[8152, 5894, 2204, 6368, 7724, 8130, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57164 payload_rows_remote=49012 dup_rows=8372 dup_rows_remote=3524 dedup_ratio=0.127747 router_tokens_per_expert=[0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 0, 0, 0, 1490, 0, 0, 0, 0, 0, 4096, 0, 5, 4095, 0, 0, 4091, 0, 0] payload_rows_per_dst=[8054, 8192, 8152, 8192, 8192, 8190, 8, 8184] dup_rows_per_dst=[0, 368, 4848, 138, 30, 0, 0, 2988] combine_rows_per_src=[8054, 8192, 8152, 8192, 8192, 8190, 8, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64792 payload_rows_remote=56602 dup_rows=744 dup_rows_remote=744 dedup_ratio=0.011353 router_tokens_per_expert=[0, 0, 0, 2542, 0, 4096, 4096, 0, 0, 0, 0, 1238, 0, 23, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 199, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7540, 8192, 8170, 8190, 8192, 8192, 8192, 8124] dup_rows_per_dst=[66, 0, 240, 0, 0, 438, 0, 0] combine_rows_per_src=[7540, 8192, 8170, 8190, 8192, 8192, 8192, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=191127552 dispatch_meta_bytes=219560 combine_payload_bytes=191127552 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=191347112 ingress_bytes=285499392 link_bytes=285499392 token_bytes=382474664 weight_movement_bytes=94371840 total_nvlink_bytes=476846504 weight_to_token_ratio=0.246740 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.735514700e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64018 payload_rows_remote=55830 dup_rows=1518 dup_rows_remote=476 dedup_ratio=0.023163 router_tokens_per_expert=[0, 0, 0, 287, 0, 4066, 4053, 0, 0, 1, 0, 3706, 0, 37, 3949, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 29, 0, 16, 0, 0, 0, 0, 0, 2, 0, 0, 0, 32, 26, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 51, 3868, 12, 0, 0, 0, 0, 0, 1, 0, 0, 68, 0, 0, 4066, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 28, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 4, 0, 0, 0, 0, 0, 30, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 5, 0] payload_rows_per_dst=[7910, 8166, 7994, 7862, 8134, 8188, 8192, 7572] dup_rows_per_dst=[152, 80, 26, 2, 138, 1042, 70, 8] combine_rows_per_src=[7910, 8166, 7994, 7862, 8134, 8188, 8192, 7572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64330 payload_rows_remote=56140 dup_rows=1206 dup_rows_remote=1206 dedup_ratio=0.018402 router_tokens_per_expert=[0, 0, 0, 2821, 0, 4095, 4095, 0, 0, 0, 0, 888, 0, 24, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 195, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7296, 8190, 8186, 8188, 8192, 8192, 8190, 7896] dup_rows_per_dst=[170, 0, 152, 0, 4, 880, 0, 0] combine_rows_per_src=[7296, 8190, 8186, 8188, 8192, 8192, 8190, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64618 payload_rows_remote=56430 dup_rows=918 dup_rows_remote=918 dedup_ratio=0.014008 router_tokens_per_expert=[0, 0, 0, 2724, 0, 4094, 4094, 0, 0, 0, 0, 674, 0, 534, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 303, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[7580, 8192, 8192, 8186, 8188, 8192, 8192, 7896] dup_rows_per_dst=[284, 4, 14, 0, 0, 616, 0, 0] combine_rows_per_src=[7580, 8192, 8192, 8186, 8188, 8192, 8192, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200753152 dispatch_meta_bytes=210144 combine_payload_bytes=200753152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200963296 ingress_bytes=219627520 link_bytes=219627520 token_bytes=401716448 weight_movement_bytes=18874368 total_nvlink_bytes=420590816 weight_to_token_ratio=0.046984 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.104362833e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=231841792 dispatch_meta_bytes=229384 combine_payload_bytes=231841792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232071176 ingress_bytes=250716160 link_bytes=250716160 token_bytes=463912968 weight_movement_bytes=18874368 total_nvlink_bytes=482787336 weight_to_token_ratio=0.040685 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402238885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60696 payload_rows_remote=52848 dup_rows=4840 dup_rows_remote=1528 dedup_ratio=0.073853 router_tokens_per_expert=[0, 0, 0, 523, 0, 4095, 4084, 0, 0, 0, 0, 1600, 0, 31, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1007, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7848, 8142, 7002, 8192, 8042, 5162, 8130, 8178] dup_rows_per_dst=[3312, 0, 824, 254, 110, 340, 0, 0] combine_rows_per_src=[7848, 8142, 7002, 8192, 8042, 5162, 8130, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56136 payload_rows_remote=47972 dup_rows=9400 dup_rows_remote=8338 dedup_ratio=0.143433 router_tokens_per_expert=[0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 297, 0, 0, 0, 3498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4080, 0, 0, 1638, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 4096, 0, 0, 0, 0] payload_rows_per_dst=[1732, 7178, 8184, 7126, 7368, 8192, 8164, 8192] dup_rows_per_dst=[0, 1620, 4, 464, 1238, 4582, 1062, 430] combine_rows_per_src=[1732, 7178, 8184, 7126, 7368, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=228679680 dispatch_meta_bytes=225224 combine_payload_bytes=228679680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228904904 ingress_bytes=285302784 link_bytes=285302784 token_bytes=457584584 weight_movement_bytes=56623104 total_nvlink_bytes=514207688 weight_to_token_ratio=0.123743 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.733630899e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=229949440 dispatch_meta_bytes=229384 combine_payload_bytes=229949440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230178824 ingress_bytes=248823808 link_bytes=248823808 token_bytes=460128264 weight_movement_bytes=18874368 total_nvlink_bytes=479002632 weight_to_token_ratio=0.041020 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384107299e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=231137280 dispatch_meta_bytes=229392 combine_payload_bytes=231137280 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231366672 ingress_bytes=268886016 link_bytes=268886016 token_bytes=462503952 weight_movement_bytes=37748736 total_nvlink_bytes=500252688 weight_to_token_ratio=0.081618 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576333505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216465408 dispatch_meta_bytes=217504 combine_payload_bytes=216465408 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=216682912 ingress_bytes=291962880 link_bytes=291962880 token_bytes=433148320 weight_movement_bytes=75497472 total_nvlink_bytes=508645792 weight_to_token_ratio=0.174299 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.797444661e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=196493312 dispatch_meta_bytes=225240 combine_payload_bytes=196493312 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=196718552 ingress_bytes=271990784 link_bytes=271990784 token_bytes=393211864 weight_movement_bytes=75497472 total_nvlink_bytes=468709336 weight_to_token_ratio=0.192002 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606081864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61930 payload_rows_remote=53800 dup_rows=3606 dup_rows_remote=1274 dedup_ratio=0.055023 router_tokens_per_expert=[0, 0, 0, 1133, 0, 4093, 4095, 0, 0, 0, 0, 2194, 0, 110, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 82, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6838, 8190, 8170, 8192, 8144, 6080, 8186, 8130] dup_rows_per_dst=[36, 0, 98, 62, 2, 1076, 0, 2332] combine_rows_per_src=[6838, 8190, 8170, 8192, 8144, 6080, 8186, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64312 payload_rows_remote=56120 dup_rows=1224 dup_rows_remote=1224 dedup_ratio=0.018677 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 0] payload_rows_per_dst=[8192, 7998, 8192, 8192, 8192, 8192, 8184, 7170] dup_rows_per_dst=[0, 8, 0, 0, 2, 1214, 0, 0] combine_rows_per_src=[8192, 7998, 8192, 8192, 8192, 8192, 8184, 7170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64656 payload_rows_remote=56464 dup_rows=880 dup_rows_remote=816 dedup_ratio=0.013428 router_tokens_per_expert=[0, 0, 0, 3608, 0, 4096, 4096, 0, 0, 0, 0, 416, 0, 40, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 302, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7938, 8192, 8192, 8190, 8192, 8192, 8192, 7568] dup_rows_per_dst=[190, 0, 64, 16, 0, 610, 0, 0] combine_rows_per_src=[7938, 8192, 8192, 8190, 8192, 8192, 8192, 7568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=220364800 dispatch_meta_bytes=220296 combine_payload_bytes=220364800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220585096 ingress_bytes=239239168 link_bytes=239239168 token_bytes=440949896 weight_movement_bytes=18874368 total_nvlink_bytes=459824264 weight_to_token_ratio=0.042804 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.292271995e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65286 payload_rows_remote=57338 dup_rows=250 dup_rows_remote=250 dedup_ratio=0.003815 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[8192, 7948, 8192, 8192, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 4, 244, 2, 0] combine_rows_per_src=[8192, 7948, 8192, 8192, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64042 payload_rows_remote=55852 dup_rows=1494 dup_rows_remote=230 dedup_ratio=0.022797 router_tokens_per_expert=[0, 1, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 3997, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4066, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 29, 0, 0, 23, 0, 550, 0, 4065, 0, 30, 0, 30, 0, 0, 0, 0, 4065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 74, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 0] payload_rows_per_dst=[8134, 8054, 8188, 8146, 8074, 8190, 8132, 7124] dup_rows_per_dst=[0, 76, 0, 2, 0, 1264, 152, 0] combine_rows_per_src=[8134, 8054, 8188, 8146, 8074, 8190, 8132, 7124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57212 dup_rows=136 dup_rows_remote=132 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 52, 0, 0, 0, 4041, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 58, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0] payload_rows_per_dst=[8184, 8190, 8192, 8188, 8188, 8192, 8188, 8078] dup_rows_per_dst=[0, 4, 0, 0, 4, 128, 0, 0] combine_rows_per_src=[8184, 8190, 8192, 8188, 8188, 8192, 8188, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=229867520 dispatch_meta_bytes=229376 combine_payload_bytes=229867520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230096896 ingress_bytes=248741888 link_bytes=248741888 token_bytes=459964416 weight_movement_bytes=18874368 total_nvlink_bytes=478838784 weight_to_token_ratio=0.041034 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.383322382e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64552 payload_rows_remote=56360 dup_rows=984 dup_rows_remote=906 dedup_ratio=0.015015 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0] payload_rows_per_dst=[8192, 8136, 8192, 8192, 8192, 7694, 8064, 7890] dup_rows_per_dst=[78, 462, 0, 46, 102, 260, 36, 0] combine_rows_per_src=[8192, 8136, 8192, 8192, 8192, 7694, 8064, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=231276544 dispatch_meta_bytes=229120 combine_payload_bytes=231276544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231505664 ingress_bytes=250150912 link_bytes=250150912 token_bytes=462782208 weight_movement_bytes=18874368 total_nvlink_bytes=481656576 weight_to_token_ratio=0.040785 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396822957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=230352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235086800 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469943248 weight_movement_bytes=18874368 total_nvlink_bytes=488817616 weight_to_token_ratio=0.040163 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62374 payload_rows_remote=54182 dup_rows=3162 dup_rows_remote=82 dedup_ratio=0.048248 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 1559, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7728, 7996, 8192, 8004, 5880, 8190, 8192, 8192] dup_rows_per_dst=[0, 40, 20, 0, 0, 22, 3080, 0] combine_rows_per_src=[7728, 7996, 8192, 8004, 5880, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=228769792 dispatch_meta_bytes=224328 combine_payload_bytes=228769792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228994120 ingress_bytes=285392896 link_bytes=285392896 token_bytes=457763912 weight_movement_bytes=56623104 total_nvlink_bytes=514387016 weight_to_token_ratio=0.123695 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.734494307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234340352 dispatch_meta_bytes=229376 combine_payload_bytes=234340352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234569728 ingress_bytes=253214720 link_bytes=253214720 token_bytes=468910080 weight_movement_bytes=18874368 total_nvlink_bytes=487784448 weight_to_token_ratio=0.040252 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426178858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=230850560 dispatch_meta_bytes=229064 combine_payload_bytes=230850560 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=231079624 ingress_bytes=287473664 link_bytes=287473664 token_bytes=461930184 weight_movement_bytes=56623104 total_nvlink_bytes=518553288 weight_to_token_ratio=0.122579 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.754431203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62570 payload_rows_remote=54390 dup_rows=2966 dup_rows_remote=126 dedup_ratio=0.045258 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0] payload_rows_per_dst=[8180, 8090, 8192, 8192, 8192, 5352, 8192, 8180] dup_rows_per_dst=[0, 0, 4, 2, 2, 110, 8, 2840] combine_rows_per_src=[8180, 8090, 8192, 8192, 8192, 5352, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=221929472 dispatch_meta_bytes=217056 combine_payload_bytes=221929472 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=222146528 ingress_bytes=335175680 link_bytes=335175680 token_bytes=444076000 weight_movement_bytes=113246208 total_nvlink_bytes=557322208 weight_to_token_ratio=0.255015 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.211488449e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=222781440 dispatch_meta_bytes=218064 combine_payload_bytes=222781440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222999504 ingress_bytes=241655808 link_bytes=241655808 token_bytes=445780944 weight_movement_bytes=18874368 total_nvlink_bytes=464655312 weight_to_token_ratio=0.042340 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.315427050e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65256 payload_rows_remote=57064 dup_rows=280 dup_rows_remote=280 dedup_ratio=0.004272 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0] payload_rows_per_dst=[8192, 8182, 8192, 8192, 8192, 8192, 8190, 7924] dup_rows_per_dst=[0, 254, 0, 0, 8, 14, 4, 0] combine_rows_per_src=[8192, 8182, 8192, 8192, 8192, 8192, 8190, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 8188, 8192] dup_rows_per_dst=[8, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65134 payload_rows_remote=56944 dup_rows=402 dup_rows_remote=350 dedup_ratio=0.006134 router_tokens_per_expert=[0, 28, 0, 0, 0, 1, 0, 0, 0, 0, 0, 80, 126, 0, 30, 0, 0, 4065, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 4063, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4062, 4066, 0, 1, 0, 4065, 0, 0, 22, 0, 0, 30, 0, 0, 0, 4, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3977, 3952, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065] payload_rows_per_dst=[8192, 8136, 8134, 8192, 8186, 8190, 7914, 8190] dup_rows_per_dst=[292, 0, 2, 2, 52, 52, 2, 0] combine_rows_per_src=[8192, 8136, 8134, 8192, 8186, 8190, 7914, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57330 dup_rows=46 dup_rows_remote=42 dedup_ratio=0.000702 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4094, 0, 1, 0, 4094, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8190, 8188, 8188, 8192, 8160, 8192, 8188, 8192] dup_rows_per_dst=[8, 0, 30, 0, 4, 4, 0, 0] combine_rows_per_src=[8190, 8188, 8188, 8192, 8160, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233734144 dispatch_meta_bytes=229376 combine_payload_bytes=233734144 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233963520 ingress_bytes=271482880 link_bytes=271482880 token_bytes=467697664 weight_movement_bytes=37748736 total_nvlink_bytes=505446400 weight_to_token_ratio=0.080712 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601215378e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64886 payload_rows_remote=56696 dup_rows=650 dup_rows_remote=90 dedup_ratio=0.009918 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 279, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8182, 8138, 7608, 8192] dup_rows_per_dst=[560, 60, 10, 0, 20, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8182, 8138, 7608, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55652 payload_rows_remote=47460 dup_rows=9884 dup_rows_remote=1876 dedup_ratio=0.150818 router_tokens_per_expert=[0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 528, 0, 0, 0, 1, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 807, 3, 0, 0, 0, 0, 4095, 0, 54, 4096, 0, 0, 4092, 0, 0] payload_rows_per_dst=[8106, 1062, 5524, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 20, 88, 48, 0, 8008, 1720] combine_rows_per_src=[8106, 1062, 5524, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=233242624 dispatch_meta_bytes=229176 combine_payload_bytes=233242624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233471800 ingress_bytes=252116992 link_bytes=252116992 token_bytes=466714424 weight_movement_bytes=18874368 total_nvlink_bytes=485588792 weight_to_token_ratio=0.040441 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415660968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229488 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235053168 ingress_bytes=310321152 link_bytes=310321152 token_bytes=469876848 weight_movement_bytes=75497472 total_nvlink_bytes=545374320 weight_to_token_ratio=0.160675 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973344591e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64656 payload_rows_remote=56464 dup_rows=880 dup_rows_remote=878 dedup_ratio=0.013428 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7318, 8192, 8190, 8192, 8190, 8192, 8190, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 874, 2] combine_rows_per_src=[7318, 8192, 8190, 8192, 8190, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=232226816 dispatch_meta_bytes=227144 combine_payload_bytes=232226816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232453960 ingress_bytes=251101184 link_bytes=251101184 token_bytes=464680776 weight_movement_bytes=18874368 total_nvlink_bytes=483555144 weight_to_token_ratio=0.040618 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405927996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=194396160 dispatch_meta_bytes=197344 combine_payload_bytes=194396160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=194593504 ingress_bytes=251019264 link_bytes=251019264 token_bytes=388989664 weight_movement_bytes=56623104 total_nvlink_bytes=445612768 weight_to_token_ratio=0.145565 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405143078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64628 payload_rows_remote=56436 dup_rows=908 dup_rows_remote=816 dedup_ratio=0.013855 router_tokens_per_expert=[3926, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1595, 0, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 15, 0, 245, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7858, 8192, 8192, 8192, 7644, 8166, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 92, 304, 0, 0, 512] combine_rows_per_src=[7858, 8192, 8192, 8192, 7644, 8166, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=231276544 dispatch_meta_bytes=229368 combine_payload_bytes=231276544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231505912 ingress_bytes=250150912 link_bytes=250150912 token_bytes=462782456 weight_movement_bytes=18874368 total_nvlink_bytes=481656824 weight_to_token_ratio=0.040785 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396822957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=4 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8178, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 8178, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65032 payload_rows_remote=56840 dup_rows=504 dup_rows_remote=504 dedup_ratio=0.007690 router_tokens_per_expert=[3989, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1895, 0, 0, 0, 0, 0, 0, 0, 2172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4092, 0, 7, 0, 23, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7980, 8192, 8192, 8192, 7918, 8178, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 230, 216, 0, 0, 58] combine_rows_per_src=[7980, 8192, 8192, 8192, 7918, 8178, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64558 payload_rows_remote=56388 dup_rows=978 dup_rows_remote=936 dedup_ratio=0.014923 router_tokens_per_expert=[3638, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 3, 0, 9, 20, 0, 185, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 4066, 3893, 0, 0, 0, 0, 0, 0, 0, 411, 3, 0, 0, 0, 16, 5, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4053, 14, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 1, 4065, 3, 0, 14, 0, 0, 0, 4066, 0, 1, 1, 0, 0, 0, 30, 0, 0, 0, 0] payload_rows_per_dst=[7278, 8190, 8168, 8188, 8180, 8170, 8192, 8192] dup_rows_per_dst=[2, 0, 6, 380, 482, 42, 34, 32] combine_rows_per_src=[7278, 8190, 8168, 8188, 8180, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64972 payload_rows_remote=57046 dup_rows=564 dup_rows_remote=316 dedup_ratio=0.008606 router_tokens_per_expert=[3955, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 89, 0, 0, 2, 0, 0, 0, 0, 1, 4094, 414, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 35, 0, 20, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[7912, 8192, 8188, 8192, 7926, 8178, 8192, 8192] dup_rows_per_dst=[22, 0, 0, 180, 248, 4, 0, 110] combine_rows_per_src=[7912, 8192, 8188, 8192, 7926, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231161856 dispatch_meta_bytes=229008 combine_payload_bytes=231161856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231390864 ingress_bytes=250036224 link_bytes=250036224 token_bytes=462552720 weight_movement_bytes=18874368 total_nvlink_bytes=481427088 weight_to_token_ratio=0.040805 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.395724073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229320 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235036616 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843912 weight_movement_bytes=18874368 total_nvlink_bytes=488718280 weight_to_token_ratio=0.040172 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60162 payload_rows_remote=52224 dup_rows=5374 dup_rows_remote=640 dedup_ratio=0.082001 router_tokens_per_expert=[3709, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 82, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7938, 8192, 8192, 6506, 6252, 8192, 8158, 6732] dup_rows_per_dst=[4734, 2, 22, 286, 162, 164, 0, 4] combine_rows_per_src=[7938, 8192, 8192, 6506, 6252, 8192, 8158, 6732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65014 payload_rows_remote=56822 dup_rows=522 dup_rows_remote=522 dedup_ratio=0.007965 router_tokens_per_expert=[0, 0, 0, 1552, 0, 4096, 4094, 0, 0, 0, 0, 2567, 0, 69, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 66, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8042, 8188, 8184, 8192, 8192, 8192, 8192, 7832] dup_rows_per_dst=[334, 0, 28, 10, 6, 144, 0, 0] combine_rows_per_src=[8042, 8188, 8184, 8192, 8192, 8192, 8192, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232816640 dispatch_meta_bytes=229376 combine_payload_bytes=232816640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233046016 ingress_bytes=251691008 link_bytes=251691008 token_bytes=465862656 weight_movement_bytes=18874368 total_nvlink_bytes=484737024 weight_to_token_ratio=0.040515 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411579399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230965248 dispatch_meta_bytes=229296 combine_payload_bytes=230965248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231194544 ingress_bytes=249839616 link_bytes=249839616 token_bytes=462159792 weight_movement_bytes=18874368 total_nvlink_bytes=481034160 weight_to_token_ratio=0.040839 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393840272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233660416 dispatch_meta_bytes=229448 combine_payload_bytes=233660416 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233889864 ingress_bytes=328032256 link_bytes=328032256 token_bytes=467550280 weight_movement_bytes=94371840 total_nvlink_bytes=561922120 weight_to_token_ratio=0.201843 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.143043675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63408 payload_rows_remote=55216 dup_rows=2128 dup_rows_remote=1606 dedup_ratio=0.032471 router_tokens_per_expert=[3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2440, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6872, 8192, 8192, 8192, 7488, 8088, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 252, 1180, 174, 0, 522] combine_rows_per_src=[6872, 8192, 8192, 8192, 7488, 8088, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=213909504 dispatch_meta_bytes=211456 combine_payload_bytes=213909504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=214120960 ingress_bytes=251658240 link_bytes=251658240 token_bytes=428030464 weight_movement_bytes=37748736 total_nvlink_bytes=465779200 weight_to_token_ratio=0.088192 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411265432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=232742912 dispatch_meta_bytes=229376 combine_payload_bytes=232742912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232972288 ingress_bytes=251617280 link_bytes=251617280 token_bytes=465715200 weight_movement_bytes=18874368 total_nvlink_bytes=484589568 weight_to_token_ratio=0.040528 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410872974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=226164736 dispatch_meta_bytes=227288 combine_payload_bytes=226164736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226392024 ingress_bytes=263913472 link_bytes=263913472 token_bytes=452556760 weight_movement_bytes=37748736 total_nvlink_bytes=490305496 weight_to_token_ratio=0.083412 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528689035e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64784 payload_rows_remote=56592 dup_rows=752 dup_rows_remote=752 dedup_ratio=0.011475 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 7442, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 752, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 7442, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64438 payload_rows_remote=56246 dup_rows=1098 dup_rows_remote=1098 dedup_ratio=0.016754 router_tokens_per_expert=[3720, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1104, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 20, 0, 76, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7468, 8192, 8192, 8192, 7834, 8176, 8192, 8192] dup_rows_per_dst=[60, 0, 0, 200, 644, 0, 0, 194] combine_rows_per_src=[7468, 8192, 8192, 8192, 7834, 8176, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64568 payload_rows_remote=56376 dup_rows=968 dup_rows_remote=968 dedup_ratio=0.014771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 7226, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 70, 0, 898, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 7226, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62872 payload_rows_remote=57042 dup_rows=2664 dup_rows_remote=2608 dedup_ratio=0.040649 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 4, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4065, 0, 0, 0, 0, 3, 0, 4065, 0, 0, 0, 0, 4065, 0, 0, 0, 1, 31, 0, 0, 0, 0, 30, 0, 46, 0, 0, 0, 0, 0, 0, 1, 2884, 0, 0, 0, 0, 30, 0, 0, 4066, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 30, 0, 30, 0, 0, 30, 0, 0, 0, 4063, 26] payload_rows_per_dst=[8142, 8130, 8132, 8130, 8188, 5830, 8132, 8188] dup_rows_per_dst=[0, 0, 6, 6, 154, 56, 2270, 172] combine_rows_per_src=[8142, 8130, 8132, 8130, 8188, 5830, 8132, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64734 payload_rows_remote=56542 dup_rows=802 dup_rows_remote=798 dedup_ratio=0.012238 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 0, 0, 0, 4094, 2] payload_rows_per_dst=[8188, 8188, 8188, 8188, 8192, 7410, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 4, 782, 12] combine_rows_per_src=[8188, 8188, 8188, 8188, 8192, 7410, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230383616 dispatch_meta_bytes=229376 combine_payload_bytes=230383616 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230612992 ingress_bytes=268132352 link_bytes=268132352 token_bytes=460996608 weight_movement_bytes=37748736 total_nvlink_bytes=498745344 weight_to_token_ratio=0.081885 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.569112268e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231800832 dispatch_meta_bytes=229376 combine_payload_bytes=231800832 prefetch_bytes=0 reduce_bytes=132120576 egress_bytes=232030208 ingress_bytes=363921408 link_bytes=363921408 token_bytes=463831040 weight_movement_bytes=132120576 total_nvlink_bytes=595951616 weight_to_token_ratio=0.284846 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.486915871e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59196 payload_rows_remote=51004 dup_rows=6340 dup_rows_remote=6282 dedup_ratio=0.096741 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7960, 8182, 2298, 7988] dup_rows_per_dst=[58, 66, 58, 84, 0, 6074, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7960, 8182, 2298, 7988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230916096 dispatch_meta_bytes=229376 combine_payload_bytes=230916096 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231145472 ingress_bytes=268664832 link_bytes=268664832 token_bytes=462061568 weight_movement_bytes=37748736 total_nvlink_bytes=499810304 weight_to_token_ratio=0.081696 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.574214229e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233644032 dispatch_meta_bytes=238600 combine_payload_bytes=233644032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233882632 ingress_bytes=271392768 link_bytes=271392768 token_bytes=467526664 weight_movement_bytes=37748736 total_nvlink_bytes=505275400 weight_to_token_ratio=0.080741 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600351969e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65406 payload_rows_remote=57214 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0] payload_rows_per_dst=[8192, 8110, 8192, 8192, 8192, 8192, 8192, 8144] dup_rows_per_dst=[0, 24, 0, 0, 0, 106, 0, 0] combine_rows_per_src=[8192, 8110, 8192, 8192, 8192, 8192, 8192, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65082 payload_rows_remote=56890 dup_rows=454 dup_rows_remote=430 dedup_ratio=0.006927 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8168, 8192, 7762, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 422, 24] combine_rows_per_src=[8192, 8192, 8192, 8168, 8192, 7762, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231596032 dispatch_meta_bytes=229360 combine_payload_bytes=231596032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231825392 ingress_bytes=250470400 link_bytes=250470400 token_bytes=463421424 weight_movement_bytes=18874368 total_nvlink_bytes=482295792 weight_to_token_ratio=0.040728 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399884134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=208912384 dispatch_meta_bytes=229144 combine_payload_bytes=208912384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209141528 ingress_bytes=227786752 link_bytes=227786752 token_bytes=418053912 weight_movement_bytes=18874368 total_nvlink_bytes=436928280 weight_to_token_ratio=0.045148 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182540580e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234348544 dispatch_meta_bytes=229376 combine_payload_bytes=234348544 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234577920 ingress_bytes=309846016 link_bytes=309846016 token_bytes=468926464 weight_movement_bytes=75497472 total_nvlink_bytes=544423936 weight_to_token_ratio=0.161001 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968792072e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233021440 dispatch_meta_bytes=229280 combine_payload_bytes=233021440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233250720 ingress_bytes=270770176 link_bytes=270770176 token_bytes=466272160 weight_movement_bytes=37748736 total_nvlink_bytes=504020896 weight_to_token_ratio=0.080959 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594386599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64946 payload_rows_remote=56754 dup_rows=590 dup_rows_remote=590 dedup_ratio=0.009003 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7602, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 590, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7602, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57340 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8176, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 20, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8176, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57248 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.001465 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8098, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 2, 92, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8098, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65334 payload_rows_remote=57152 dup_rows=202 dup_rows_remote=146 dedup_ratio=0.003082 router_tokens_per_expert=[0, 0, 30, 0, 0, 30, 0, 0, 4065, 0, 0, 0, 0, 4059, 0, 0, 0, 4064, 0, 2, 4065, 0, 0, 30, 0, 0, 0, 2, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 14, 4033, 0, 0, 30, 0, 0, 0, 0, 0, 13, 0, 0, 0, 15, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 30, 3, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 3, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8132, 8078, 8190, 8182, 8192, 8190] dup_rows_per_dst=[60, 4, 2, 16, 56, 56, 8, 0] combine_rows_per_src=[8178, 8192, 8132, 8078, 8190, 8182, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=232464384 dispatch_meta_bytes=229376 combine_payload_bytes=232464384 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232693760 ingress_bytes=270213120 link_bytes=270213120 token_bytes=465158144 weight_movement_bytes=37748736 total_nvlink_bytes=502906880 weight_to_token_ratio=0.081152 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589049163e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=50 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 2, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4071, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8142, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 4, 46, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8142, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229440 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094080 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958720 weight_movement_bytes=37748736 total_nvlink_bytes=507707456 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64582 payload_rows_remote=56390 dup_rows=954 dup_rows_remote=870 dedup_ratio=0.014557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8132, 7380, 8122, 8180] dup_rows_per_dst=[84, 56, 0, 746, 0, 2, 60, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8132, 7380, 8122, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234487808 dispatch_meta_bytes=229376 combine_payload_bytes=234487808 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234717184 ingress_bytes=328859648 link_bytes=328859648 token_bytes=469204992 weight_movement_bytes=94371840 total_nvlink_bytes=563576832 weight_to_token_ratio=0.201131 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.150971338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229192 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234323784 ingress_bytes=252968960 link_bytes=252968960 token_bytes=468418376 weight_movement_bytes=18874368 total_nvlink_bytes=487292744 weight_to_token_ratio=0.040294 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423824106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57340 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8188, 8192] dup_rows_per_dst=[4, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65334 payload_rows_remote=57142 dup_rows=202 dup_rows_remote=200 dedup_ratio=0.003082 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8066, 8132, 8192, 8176, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 60, 126, 0, 14, 0, 2] combine_rows_per_src=[8066, 8132, 8192, 8176, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229360 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889200 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549040 weight_movement_bytes=18874368 total_nvlink_bytes=488423408 weight_to_token_ratio=0.040197 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=230973440 dispatch_meta_bytes=229040 combine_payload_bytes=230973440 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231202480 ingress_bytes=287596544 link_bytes=287596544 token_bytes=462175920 weight_movement_bytes=56623104 total_nvlink_bytes=518799024 weight_to_token_ratio=0.122514 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.755608578e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229392 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235094032 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958672 weight_movement_bytes=37748736 total_nvlink_bytes=507707408 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234053632 dispatch_meta_bytes=229368 combine_payload_bytes=234053632 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234283000 ingress_bytes=252928000 link_bytes=252928000 token_bytes=468336632 weight_movement_bytes=18874368 total_nvlink_bytes=487211000 weight_to_token_ratio=0.040301 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423431648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 7826, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 366, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 7826, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57344 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65388 payload_rows_remote=57196 dup_rows=148 dup_rows_remote=148 dedup_ratio=0.002258 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0] payload_rows_per_dst=[8192, 8192, 8192, 8046, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 2, 50, 94] combine_rows_per_src=[8192, 8192, 8192, 8046, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65182 payload_rows_remote=57016 dup_rows=354 dup_rows_remote=330 dedup_ratio=0.005402 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 0, 0, 29, 1, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4065, 25, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 77, 31, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 4065, 12, 0] payload_rows_per_dst=[8192, 8188, 8132, 7928, 8192, 8166, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 26, 24, 276, 28] combine_rows_per_src=[8192, 8188, 8132, 7928, 8192, 8166, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 3, 0] payload_rows_per_dst=[8192, 8192, 8188, 8182, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 6] combine_rows_per_src=[8192, 8192, 8188, 8182, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229376 combine_payload_bytes=233373696 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233603072 ingress_bytes=271122432 link_bytes=271122432 token_bytes=466976768 weight_movement_bytes=37748736 total_nvlink_bytes=504725504 weight_to_token_ratio=0.080836 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597761743e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229392 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110416 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991440 weight_movement_bytes=37748736 total_nvlink_bytes=507740176 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64560 payload_rows_remote=56368 dup_rows=976 dup_rows_remote=976 dedup_ratio=0.014893 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 736, 0] payload_rows_per_dst=[8192, 8192, 8192, 8094, 8184, 8192, 7830, 7684] dup_rows_per_dst=[0, 4, 62, 794, 0, 0, 18, 98] combine_rows_per_src=[8192, 8192, 8192, 8094, 8184, 8192, 7830, 7684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234274816 dispatch_meta_bytes=229376 combine_payload_bytes=234274816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234504192 ingress_bytes=272023552 link_bytes=272023552 token_bytes=468779008 weight_movement_bytes=37748736 total_nvlink_bytes=506527744 weight_to_token_ratio=0.080526 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606395831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233537536 dispatch_meta_bytes=229384 combine_payload_bytes=233537536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233766920 ingress_bytes=252411904 link_bytes=252411904 token_bytes=467304456 weight_movement_bytes=18874368 total_nvlink_bytes=486178824 weight_to_token_ratio=0.040390 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418486670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57126 dup_rows=218 dup_rows_remote=0 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 101, 0] payload_rows_per_dst=[8192, 8192, 8192, 7990, 8176, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 218] combine_rows_per_src=[8192, 8192, 8192, 7990, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64472 payload_rows_remote=56280 dup_rows=1064 dup_rows_remote=1064 dedup_ratio=0.016235 router_tokens_per_expert=[3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1494, 0, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7412, 8192, 8192, 8192, 8096, 8004, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 70, 962, 0, 0, 32] combine_rows_per_src=[7412, 8192, 8192, 8192, 8096, 8004, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235053056 ingress_bytes=291446784 link_bytes=291446784 token_bytes=469876736 weight_movement_bytes=56623104 total_nvlink_bytes=526499840 weight_to_token_ratio=0.120506 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792499684e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=230883328 dispatch_meta_bytes=229376 combine_payload_bytes=230883328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231112704 ingress_bytes=249757696 link_bytes=249757696 token_bytes=461996032 weight_movement_bytes=18874368 total_nvlink_bytes=480870400 weight_to_token_ratio=0.040854 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393055355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233988096 dispatch_meta_bytes=228504 combine_payload_bytes=233988096 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234216600 ingress_bytes=290611200 link_bytes=290611200 token_bytes=468204696 weight_movement_bytes=56623104 total_nvlink_bytes=524827800 weight_to_token_ratio=0.120937 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784493529e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230522880 dispatch_meta_bytes=229376 combine_payload_bytes=230522880 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=230752256 ingress_bytes=268271616 link_bytes=268271616 token_bytes=461275136 weight_movement_bytes=37748736 total_nvlink_bytes=499023872 weight_to_token_ratio=0.081836 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.570446627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65362 payload_rows_remote=57170 dup_rows=174 dup_rows_remote=8 dedup_ratio=0.002655 router_tokens_per_expert=[0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8024, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 166, 0, 0, 0, 0] combine_rows_per_src=[8024, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64796 payload_rows_remote=56664 dup_rows=740 dup_rows_remote=740 dedup_ratio=0.011292 router_tokens_per_expert=[0, 0, 3778, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4061, 31, 0, 0, 31, 0, 0, 76, 0, 2, 0, 3991, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 4065, 0, 1, 1, 0, 0, 0, 28, 31, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4065, 0, 0, 0, 4062, 0, 0, 0, 0, 30, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7560, 8184, 8186, 8192, 8192, 8132, 8186, 8164] dup_rows_per_dst=[0, 88, 12, 638, 0, 0, 2, 0] combine_rows_per_src=[7560, 8184, 8186, 8192, 8192, 8132, 8186, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 12, 0, 0, 6, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8192, 8192, 8192, 8192, 8188, 8192, 8190] dup_rows_per_dst=[0, 6, 0, 40, 0, 0, 0, 0] combine_rows_per_src=[8152, 8192, 8192, 8192, 8192, 8188, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234168320 dispatch_meta_bytes=228712 combine_payload_bytes=234168320 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234397032 ingress_bytes=309665792 link_bytes=309665792 token_bytes=468565352 weight_movement_bytes=75497472 total_nvlink_bytes=544062824 weight_to_token_ratio=0.161125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.967065254e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63344 payload_rows_remote=55152 dup_rows=2192 dup_rows_remote=1280 dedup_ratio=0.033447 router_tokens_per_expert=[0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 4096, 6, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 7440, 6756, 8192, 8192, 8192, 8192] dup_rows_per_dst=[912, 0, 830, 450, 0, 0, 0, 0] combine_rows_per_src=[8192, 8188, 7440, 6756, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232095744 dispatch_meta_bytes=229616 combine_payload_bytes=232095744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232325360 ingress_bytes=269844480 link_bytes=269844480 token_bytes=464421104 weight_movement_bytes=37748736 total_nvlink_bytes=502169840 weight_to_token_ratio=0.081281 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.585517036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234921984 ingress_bytes=253566976 link_bytes=253566976 token_bytes=469614592 weight_movement_bytes=18874368 total_nvlink_bytes=488488960 weight_to_token_ratio=0.040191 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429554001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65292 payload_rows_remote=57100 dup_rows=244 dup_rows_remote=0 dedup_ratio=0.003723 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7948, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 244, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7948, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65322 payload_rows_remote=57130 dup_rows=214 dup_rows_remote=184 dedup_ratio=0.003265 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4096, 5, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8116, 8126, 8158, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 114, 0, 66, 4, 30] combine_rows_per_src=[8154, 8116, 8126, 8158, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=225902592 dispatch_meta_bytes=225728 combine_payload_bytes=225902592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226128320 ingress_bytes=244776960 link_bytes=244776960 token_bytes=452030912 weight_movement_bytes=18874368 total_nvlink_bytes=470905280 weight_to_token_ratio=0.041755 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.345332393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233881600 dispatch_meta_bytes=228400 combine_payload_bytes=233881600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234110000 ingress_bytes=252755968 link_bytes=252755968 token_bytes=467991600 weight_movement_bytes=18874368 total_nvlink_bytes=486865968 weight_to_token_ratio=0.040331 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421783322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234004480 dispatch_meta_bytes=229256 combine_payload_bytes=234004480 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234233736 ingress_bytes=271753216 link_bytes=271753216 token_bytes=468238216 weight_movement_bytes=37748736 total_nvlink_bytes=505986952 weight_to_token_ratio=0.080619 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603805605e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57338 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8172, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8172, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62504 payload_rows_remote=54312 dup_rows=3032 dup_rows_remote=3032 dedup_ratio=0.046265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 4096, 0, 778, 0, 2133, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 99, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7670, 5682, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 140, 0, 0, 0, 2652, 188, 52] combine_rows_per_src=[7670, 5682, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58642 payload_rows_remote=50450 dup_rows=6894 dup_rows_remote=376 dedup_ratio=0.105194 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 4096, 0, 506, 0, 2753, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8004, 8192, 8192, 8192, 8192, 1490, 8188, 8192] dup_rows_per_dst=[0, 6518, 0, 0, 0, 0, 10, 366] combine_rows_per_src=[8004, 8192, 8192, 8192, 8192, 1490, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61454 payload_rows_remote=53264 dup_rows=4082 dup_rows_remote=3630 dedup_ratio=0.062286 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 31, 0, 0, 31, 0, 0, 2139, 0, 30, 0, 4065, 0, 2236, 0, 3496, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4065, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4065, 0, 0, 0, 4065, 0, 0, 52, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4, 3, 0, 0, 1, 0, 0] payload_rows_per_dst=[4340, 8086, 8190, 8190, 8140, 8190, 8128, 8190] dup_rows_per_dst=[66, 3438, 0, 54, 2, 452, 60, 10] combine_rows_per_src=[4340, 8086, 8190, 8190, 8140, 8190, 8128, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59232 payload_rows_remote=51044 dup_rows=6304 dup_rows_remote=6304 dedup_ratio=0.096191 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4081, 0, 2, 0, 4094, 0, 71, 0, 888, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 15, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8166, 1922, 8192, 8192, 8188, 8192, 8188, 8192] dup_rows_per_dst=[4, 0, 0, 4, 0, 6116, 8, 172] combine_rows_per_src=[8166, 1922, 8192, 8192, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229456 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085904 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942352 weight_movement_bytes=18874368 total_nvlink_bytes=488816720 weight_to_token_ratio=0.040163 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57102 payload_rows_remote=48930 dup_rows=8434 dup_rows_remote=1256 dedup_ratio=0.128693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 4096, 0, 478, 0, 1686, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 573, 3645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 5680, 8192, 8192, 8192, 3562, 7742, 7370] dup_rows_per_dst=[7178, 872, 4, 0, 0, 224, 142, 14] combine_rows_per_src=[8172, 5680, 8192, 8192, 8192, 3562, 7742, 7370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=222461952 dispatch_meta_bytes=229376 combine_payload_bytes=222461952 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=222691328 ingress_bytes=241336320 link_bytes=241336320 token_bytes=445153280 weight_movement_bytes=18874368 total_nvlink_bytes=464027648 weight_to_token_ratio=0.042400 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312365873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=206643200 dispatch_meta_bytes=203304 combine_payload_bytes=206643200 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=206846504 ingress_bytes=282140672 link_bytes=282140672 token_bytes=413489704 weight_movement_bytes=75497472 total_nvlink_bytes=488987176 weight_to_token_ratio=0.182586 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.703333097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=218169344 dispatch_meta_bytes=227576 combine_payload_bytes=218169344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218396920 ingress_bytes=274792448 link_bytes=274792448 token_bytes=436566264 weight_movement_bytes=56623104 total_nvlink_bytes=493189368 weight_to_token_ratio=0.129701 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.632926030e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65302 payload_rows_remote=57110 dup_rows=234 dup_rows_remote=234 dedup_ratio=0.003571 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 7960, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 228, 0, 6] combine_rows_per_src=[8190, 8192, 8192, 7960, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=209076224 dispatch_meta_bytes=229392 combine_payload_bytes=209076224 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=209305616 ingress_bytes=246824960 link_bytes=246824960 token_bytes=418381840 weight_movement_bytes=37748736 total_nvlink_bytes=456130576 weight_to_token_ratio=0.090226 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364955321e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62870 payload_rows_remote=54678 dup_rows=2666 dup_rows_remote=2654 dedup_ratio=0.040680 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 0, 0, 0, 4096, 0, 1709, 0, 3627, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 10, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5668, 8116, 8190, 8132, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 2502, 0, 2, 54, 30, 66, 12] combine_rows_per_src=[5668, 8116, 8190, 8132, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=200417280 dispatch_meta_bytes=200744 combine_payload_bytes=200417280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200618024 ingress_bytes=219291648 link_bytes=219291648 token_bytes=401035304 weight_movement_bytes=18874368 total_nvlink_bytes=419909672 weight_to_token_ratio=0.047064 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.101144673e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=229376 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234151936 ingress_bytes=252796928 link_bytes=252796928 token_bytes=468074496 weight_movement_bytes=18874368 total_nvlink_bytes=486948864 weight_to_token_ratio=0.040323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422175780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=223961088 dispatch_meta_bytes=229328 combine_payload_bytes=223961088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224190416 ingress_bytes=242835456 link_bytes=242835456 token_bytes=448151504 weight_movement_bytes=18874368 total_nvlink_bytes=467025872 weight_to_token_ratio=0.042116 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.326729857e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62464 payload_rows_remote=54272 dup_rows=3072 dup_rows_remote=3072 dedup_ratio=0.046875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 177, 0, 1340, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 25, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8116, 8008, 8192, 8192, 8192, 5392, 8180, 8192] dup_rows_per_dst=[0, 2730, 0, 2, 0, 188, 12, 140] combine_rows_per_src=[8116, 8008, 8192, 8192, 8192, 5392, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65406 payload_rows_remote=57214 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8062, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 128, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8062, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61562 payload_rows_remote=53372 dup_rows=3974 dup_rows_remote=3792 dedup_ratio=0.060638 router_tokens_per_expert=[0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4065, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 1863, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 4065, 0, 1, 0, 12, 2, 2204, 0, 0, 0, 4064, 0, 0, 0, 0, 31, 31, 0, 0, 0, 0, 30, 0, 30, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 1, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8158, 8140, 8188, 4436, 8190, 8192, 8130] dup_rows_per_dst=[0, 0, 4, 3722, 2, 182, 64, 0] combine_rows_per_src=[8128, 8158, 8140, 8188, 4436, 8190, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57328 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4083, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8188, 8188, 8192, 8170, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 22, 0, 12, 4, 0] combine_rows_per_src=[8188, 8188, 8188, 8192, 8170, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63286 payload_rows_remote=55094 dup_rows=2250 dup_rows_remote=2182 dedup_ratio=0.034332 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6140, 7998, 8188, 8192] dup_rows_per_dst=[68, 38, 52, 2012, 78, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6140, 7998, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=222298112 dispatch_meta_bytes=229376 combine_payload_bytes=222298112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222527488 ingress_bytes=241172480 link_bytes=241172480 token_bytes=444825600 weight_movement_bytes=18874368 total_nvlink_bytes=463699968 weight_to_token_ratio=0.042431 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310796039e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085800 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942248 weight_movement_bytes=18874368 total_nvlink_bytes=488816616 weight_to_token_ratio=0.040163 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234348544 dispatch_meta_bytes=229376 combine_payload_bytes=234348544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234577920 ingress_bytes=272097280 link_bytes=272097280 token_bytes=468926464 weight_movement_bytes=37748736 total_nvlink_bytes=506675200 weight_to_token_ratio=0.080500 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607102257e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218611712 dispatch_meta_bytes=228656 combine_payload_bytes=218611712 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=218840368 ingress_bytes=294109184 link_bytes=294109184 token_bytes=437452080 weight_movement_bytes=75497472 total_nvlink_bytes=512949552 weight_to_token_ratio=0.172585 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.818009490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0] payload_rows_per_dst=[8192, 8192, 8192, 8132, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 60] combine_rows_per_src=[8192, 8192, 8192, 8132, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229464 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235044952 ingress_bytes=310312960 link_bytes=310312960 token_bytes=469860440 weight_movement_bytes=75497472 total_nvlink_bytes=545357912 weight_to_token_ratio=0.160681 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973266099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58124 payload_rows_remote=49932 dup_rows=7412 dup_rows_remote=7350 dedup_ratio=0.113098 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3523, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 1552, 7482, 8192, 8130, 8192] dup_rows_per_dst=[0, 0, 0, 710, 6640, 0, 0, 62] combine_rows_per_src=[8192, 8192, 8192, 1552, 7482, 8192, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=225665024 dispatch_meta_bytes=229104 combine_payload_bytes=225665024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225894128 ingress_bytes=244539392 link_bytes=244539392 token_bytes=451559152 weight_movement_bytes=18874368 total_nvlink_bytes=470433520 weight_to_token_ratio=0.041798 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.343056133e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229376 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234864640 ingress_bytes=253509632 link_bytes=253509632 token_bytes=469499904 weight_movement_bytes=18874368 total_nvlink_bytes=488374272 weight_to_token_ratio=0.040201 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429004559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=204521472 dispatch_meta_bytes=229128 combine_payload_bytes=204521472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204750600 ingress_bytes=223395840 link_bytes=223395840 token_bytes=409272072 weight_movement_bytes=18874368 total_nvlink_bytes=428146440 weight_to_token_ratio=0.046117 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.140469021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64890 payload_rows_remote=56698 dup_rows=646 dup_rows_remote=646 dedup_ratio=0.009857 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7548, 8192, 8192, 8192] dup_rows_per_dst=[4, 572, 0, 0, 0, 70, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 7548, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59428 payload_rows_remote=51236 dup_rows=6108 dup_rows_remote=3944 dedup_ratio=0.093201 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2718, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7268, 3008, 8192, 8192] dup_rows_per_dst=[0, 2164, 0, 0, 3612, 332, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7268, 3008, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60362 payload_rows_remote=52232 dup_rows=5174 dup_rows_remote=4042 dedup_ratio=0.078949 router_tokens_per_expert=[31, 0, 0, 0, 4051, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 0, 0, 7, 0, 31, 0, 4064, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 2, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 4064, 0, 4065, 1, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 566, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8166, 8192, 8190, 8188, 3112, 8130, 8192, 8192] dup_rows_per_dst=[2, 3988, 0, 0, 0, 1132, 0, 52] combine_rows_per_src=[8166, 8192, 8190, 8188, 3112, 8130, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60128 payload_rows_remote=57336 dup_rows=5408 dup_rows_remote=5408 dedup_ratio=0.082520 router_tokens_per_expert=[2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1394, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2304, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 2792, 8188, 8192, 8192] dup_rows_per_dst=[0, 796, 0, 0, 0, 4608, 0, 4] combine_rows_per_src=[8192, 8192, 8188, 8192, 2792, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57424 payload_rows_remote=49232 dup_rows=8112 dup_rows_remote=8090 dedup_ratio=0.123779 router_tokens_per_expert=[0, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 136, 8192, 8136] dup_rows_per_dst=[22, 1856, 8, 4, 6222, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 136, 8192, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=232235008 dispatch_meta_bytes=229376 combine_payload_bytes=232235008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232464384 ingress_bytes=251109376 link_bytes=251109376 token_bytes=464699392 weight_movement_bytes=18874368 total_nvlink_bytes=483573760 weight_to_token_ratio=0.040616 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406006487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 8186, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 34, 0, 0, 0, 0] combine_rows_per_src=[8164, 8192, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=209862656 dispatch_meta_bytes=220720 combine_payload_bytes=209862656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210083376 ingress_bytes=228737024 link_bytes=228737024 token_bytes=419946032 weight_movement_bytes=18874368 total_nvlink_bytes=438820400 weight_to_token_ratio=0.044945 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.191645618e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61842 payload_rows_remote=53650 dup_rows=3694 dup_rows_remote=3694 dedup_ratio=0.056366 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1227, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 617, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 4500, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2460, 1234, 0, 0] combine_rows_per_src=[8190, 4500, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=213942272 dispatch_meta_bytes=225096 combine_payload_bytes=213942272 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=214167368 ingress_bytes=270565376 link_bytes=270565376 token_bytes=428109640 weight_movement_bytes=56623104 total_nvlink_bytes=484732744 weight_to_token_ratio=0.132263 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592424306e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=250976 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235099232 ingress_bytes=291471360 link_bytes=291471360 token_bytes=469947488 weight_movement_bytes=56623104 total_nvlink_bytes=526570592 weight_to_token_ratio=0.120488 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792735159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=201654272 dispatch_meta_bytes=229288 combine_payload_bytes=201654272 prefetch_bytes=0 reduce_bytes=0 egress_bytes=201883560 ingress_bytes=201654272 link_bytes=201883560 token_bytes=403537832 weight_movement_bytes=0 total_nvlink_bytes=403537832 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.934348939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229376 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234971136 ingress_bytes=291364864 link_bytes=291364864 token_bytes=469712896 weight_movement_bytes=56623104 total_nvlink_bytes=526336000 weight_to_token_ratio=0.120548 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791714766e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=219750400 dispatch_meta_bytes=229376 combine_payload_bytes=219750400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219979776 ingress_bytes=238624768 link_bytes=238624768 token_bytes=439730176 weight_movement_bytes=18874368 total_nvlink_bytes=458604544 weight_to_token_ratio=0.042923 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286385116e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65018 payload_rows_remote=57302 dup_rows=518 dup_rows_remote=518 dedup_ratio=0.007904 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7716, 8192, 8192, 8192, 8150] dup_rows_per_dst=[0, 0, 0, 0, 0, 464, 54, 0] combine_rows_per_src=[8192, 8192, 8192, 7716, 8192, 8192, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62620 payload_rows_remote=54428 dup_rows=2916 dup_rows_remote=2916 dedup_ratio=0.044495 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5276, 8192, 8192, 8192] dup_rows_per_dst=[0, 366, 0, 0, 0, 2550, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5276, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58878 payload_rows_remote=50686 dup_rows=6658 dup_rows_remote=6656 dedup_ratio=0.101593 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8110, 8190, 1710, 8192, 8100] dup_rows_per_dst=[0, 2, 0, 5250, 0, 48, 1358, 0] combine_rows_per_src=[8192, 8192, 8192, 8110, 8190, 1710, 8192, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63206 payload_rows_remote=55014 dup_rows=2330 dup_rows_remote=646 dedup_ratio=0.035553 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 31, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3050, 1, 17, 0, 0, 0, 0, 30, 27, 0, 0, 252, 4065, 0, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 1, 0, 4065, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4064, 0, 4065, 0, 0] payload_rows_per_dst=[8132, 8190, 8192, 6140, 8192, 8192, 8158, 8010] dup_rows_per_dst=[0, 0, 60, 2, 558, 1684, 26, 0] combine_rows_per_src=[8132, 8190, 8192, 6140, 8192, 8192, 8158, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64804 payload_rows_remote=56612 dup_rows=732 dup_rows_remote=728 dedup_ratio=0.011169 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 2, 0, 0, 0, 0, 2, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 4094, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8152, 8192, 8192, 8188, 7508] dup_rows_per_dst=[0, 0, 4, 0, 4, 558, 166, 0] combine_rows_per_src=[8188, 8192, 8192, 8152, 8192, 8192, 8188, 7508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=231280 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234940272 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469649264 weight_movement_bytes=18874368 total_nvlink_bytes=488523632 weight_to_token_ratio=0.040188 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=222937088 dispatch_meta_bytes=229376 combine_payload_bytes=222937088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223166464 ingress_bytes=241811456 link_bytes=241811456 token_bytes=446103552 weight_movement_bytes=18874368 total_nvlink_bytes=464977920 weight_to_token_ratio=0.042309 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316918393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58198 payload_rows_remote=50006 dup_rows=7338 dup_rows_remote=7274 dedup_ratio=0.111969 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1352, 1, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8128, 8192, 8192, 5466, 8192, 3644] dup_rows_per_dst=[64, 0, 0, 2898, 124, 1780, 2472, 0] combine_rows_per_src=[8192, 8192, 8128, 8192, 8192, 5466, 8192, 3644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60950 payload_rows_remote=52760 dup_rows=4586 dup_rows_remote=4532 dedup_ratio=0.069977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 391, 0, 1434, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 52, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 3648, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 4472, 54, 58] combine_rows_per_src=[8152, 3648, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55260 payload_rows_remote=47072 dup_rows=10276 dup_rows_remote=4362 dedup_ratio=0.156799 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3384, 7482, 8192, 3438, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 3968, 394, 5914] combine_rows_per_src=[8192, 8192, 8192, 3384, 7482, 8192, 3438, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=207609856 dispatch_meta_bytes=229368 combine_payload_bytes=207609856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207839224 ingress_bytes=226484224 link_bytes=226484224 token_bytes=415449080 weight_movement_bytes=18874368 total_nvlink_bytes=434323448 weight_to_token_ratio=0.045431 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.170060397e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=225337344 dispatch_meta_bytes=222640 combine_payload_bytes=225337344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225559984 ingress_bytes=263086080 link_bytes=263086080 token_bytes=450897328 weight_movement_bytes=37748736 total_nvlink_bytes=488646064 weight_to_token_ratio=0.083719 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.520761372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231882752 dispatch_meta_bytes=229360 combine_payload_bytes=231882752 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=232112112 ingress_bytes=250757120 link_bytes=250757120 token_bytes=463994864 weight_movement_bytes=18874368 total_nvlink_bytes=482869232 weight_to_token_ratio=0.040678 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402631344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=204824576 dispatch_meta_bytes=229120 combine_payload_bytes=204824576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205053696 ingress_bytes=223698944 link_bytes=223698944 token_bytes=409878272 weight_movement_bytes=18874368 total_nvlink_bytes=428752640 weight_to_token_ratio=0.046049 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.143373215e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=216104960 dispatch_meta_bytes=229168 combine_payload_bytes=216104960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=216334128 ingress_bytes=272728064 link_bytes=272728064 token_bytes=432439088 weight_movement_bytes=56623104 total_nvlink_bytes=489062192 weight_to_token_ratio=0.130939 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.613146119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=192806912 dispatch_meta_bytes=205736 combine_payload_bytes=192806912 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=193012648 ingress_bytes=268304384 link_bytes=268304384 token_bytes=385819560 weight_movement_bytes=75497472 total_nvlink_bytes=461317032 weight_to_token_ratio=0.195681 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.570760593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65160 payload_rows_remote=56968 dup_rows=376 dup_rows_remote=376 dedup_ratio=0.005737 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7816, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 320, 0, 0, 0, 56, 0] combine_rows_per_src=[7816, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8048, 8192, 8192, 8192, 7968] dup_rows_per_dst=[0, 0, 0, 0, 26, 300, 42, 0] combine_rows_per_src=[8192, 8192, 8192, 8048, 8192, 8192, 8192, 7968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64914 payload_rows_remote=56722 dup_rows=622 dup_rows_remote=622 dedup_ratio=0.009491 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7582, 8192, 8192, 8190, 8192, 8182, 8192, 8192] dup_rows_per_dst=[0, 0, 468, 0, 4, 0, 150, 0] combine_rows_per_src=[7582, 8192, 8192, 8190, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64116 payload_rows_remote=55986 dup_rows=1420 dup_rows_remote=1420 dedup_ratio=0.021667 router_tokens_per_expert=[0, 0, 4065, 0, 0, 0, 0, 0, 3471, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4063, 0, 0, 0, 0, 4059, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 27, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4065, 0, 30, 0, 0, 0, 0, 30, 0, 0, 179, 24, 0, 30, 0, 0, 0, 0, 4065, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11] payload_rows_per_dst=[6946, 8130, 8188, 8192, 8186, 8130, 8192, 8152] dup_rows_per_dst=[0, 0, 832, 62, 2, 0, 524, 0] combine_rows_per_src=[6946, 8130, 8188, 8192, 8186, 8130, 8192, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62452 payload_rows_remote=54264 dup_rows=3084 dup_rows_remote=3084 dedup_ratio=0.047058 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 2, 0, 0, 1494, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[5120, 8188, 8192, 8192, 8188, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 80, 4, 0, 0, 3000, 0] combine_rows_per_src=[5120, 8188, 8192, 8192, 8188, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=233340928 dispatch_meta_bytes=229376 combine_payload_bytes=233340928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233570304 ingress_bytes=252215296 link_bytes=252215296 token_bytes=466911232 weight_movement_bytes=18874368 total_nvlink_bytes=485785600 weight_to_token_ratio=0.040424 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416602869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229376 combine_payload_bytes=233373696 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233603072 ingress_bytes=252248064 link_bytes=252248064 token_bytes=466976768 weight_movement_bytes=18874368 total_nvlink_bytes=485851136 weight_to_token_ratio=0.040418 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416916835e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58212 payload_rows_remote=50034 dup_rows=7324 dup_rows_remote=176 dedup_ratio=0.111755 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 5450, 8192, 8192, 8192, 3628, 8188] dup_rows_per_dst=[7148, 124, 48, 0, 4, 0, 0, 0] combine_rows_per_src=[8178, 8192, 5450, 8192, 8192, 8192, 3628, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8156, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 36, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8156, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64462 payload_rows_remote=56270 dup_rows=1074 dup_rows_remote=1030 dedup_ratio=0.016388 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 3619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7238, 8136, 8192, 8128, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 866, 0, 0, 76, 88, 44] combine_rows_per_src=[7238, 8136, 8192, 8128, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=232333312 dispatch_meta_bytes=229376 combine_payload_bytes=232333312 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=232562688 ingress_bytes=307830784 link_bytes=307830784 token_bytes=464896000 weight_movement_bytes=75497472 total_nvlink_bytes=540393472 weight_to_token_ratio=0.162396 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.949483110e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=229318656 dispatch_meta_bytes=229624 combine_payload_bytes=229318656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229548280 ingress_bytes=267067392 link_bytes=267067392 token_bytes=458866936 weight_movement_bytes=37748736 total_nvlink_bytes=496615672 weight_to_token_ratio=0.082265 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.558908345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=222265344 dispatch_meta_bytes=229392 combine_payload_bytes=222265344 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=222494736 ingress_bytes=260014080 link_bytes=260014080 token_bytes=444760080 weight_movement_bytes=37748736 total_nvlink_bytes=482508816 weight_to_token_ratio=0.084874 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.491326980e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=204939264 dispatch_meta_bytes=200840 combine_payload_bytes=204939264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205140104 ingress_bytes=242688000 link_bytes=242688000 token_bytes=410079368 weight_movement_bytes=37748736 total_nvlink_bytes=447828104 weight_to_token_ratio=0.092052 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.325317006e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234962944 ingress_bytes=272482304 link_bytes=272482304 token_bytes=469696512 weight_movement_bytes=37748736 total_nvlink_bytes=507445248 weight_to_token_ratio=0.080368 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610791367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=230481920 dispatch_meta_bytes=229200 combine_payload_bytes=230481920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=230711120 ingress_bytes=287105024 link_bytes=287105024 token_bytes=461193040 weight_movement_bytes=56623104 total_nvlink_bytes=517816144 weight_to_token_ratio=0.122775 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.750899075e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61596 payload_rows_remote=53404 dup_rows=3940 dup_rows_remote=3940 dedup_ratio=0.060120 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0] payload_rows_per_dst=[6424, 8192, 6020, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2146, 0, 1768, 0, 0, 0, 2, 24] combine_rows_per_src=[6424, 8192, 6020, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64900 payload_rows_remote=56708 dup_rows=636 dup_rows_remote=604 dedup_ratio=0.009705 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7558, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 32, 0, 0, 0, 604, 0] combine_rows_per_src=[7558, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62750 payload_rows_remote=54560 dup_rows=2786 dup_rows_remote=2786 dedup_ratio=0.042511 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 5414, 8192, 8186, 8192, 8192, 8192] dup_rows_per_dst=[2752, 0, 0, 0, 0, 0, 2, 32] combine_rows_per_src=[8192, 8190, 5414, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62874 payload_rows_remote=54688 dup_rows=2662 dup_rows_remote=2662 dedup_ratio=0.040619 router_tokens_per_expert=[0, 31, 0, 27, 0, 0, 0, 0, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 4064, 0, 31, 0, 0, 0, 7, 6, 0, 0, 0, 0, 4065, 0, 947, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 4065, 0, 1, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 16, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 4065, 0, 0, 0] payload_rows_per_dst=[6418, 7462, 8132, 8130, 8190, 8186, 8190, 8166] dup_rows_per_dst=[54, 26, 1894, 4, 70, 0, 34, 580] combine_rows_per_src=[6418, 7462, 8132, 8130, 8190, 8186, 8190, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60892 payload_rows_remote=52700 dup_rows=4644 dup_rows_remote=4640 dedup_ratio=0.070862 router_tokens_per_expert=[0, 2, 0, 2, 0, 0, 0, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] payload_rows_per_dst=[3556, 8192, 8188, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 4636, 0, 4, 0, 0, 0] combine_rows_per_src=[3556, 8192, 8188, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=218742784 dispatch_meta_bytes=229376 combine_payload_bytes=218742784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218972160 ingress_bytes=237617152 link_bytes=237617152 token_bytes=437714944 weight_movement_bytes=18874368 total_nvlink_bytes=456589312 weight_to_token_ratio=0.043120 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276730636e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=232275968 dispatch_meta_bytes=229248 combine_payload_bytes=232275968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232505216 ingress_bytes=251150336 link_bytes=251150336 token_bytes=464781184 weight_movement_bytes=18874368 total_nvlink_bytes=483655552 weight_to_token_ratio=0.040609 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406398946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58790 payload_rows_remote=50604 dup_rows=6746 dup_rows_remote=892 dedup_ratio=0.102936 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 4096, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 2248, 8192, 8128, 8192, 8136, 7516] dup_rows_per_dst=[5854, 710, 100, 62, 0, 6, 14, 0] combine_rows_per_src=[8186, 8192, 2248, 8192, 8128, 8192, 8136, 7516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=223477760 dispatch_meta_bytes=229384 combine_payload_bytes=223477760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=223707144 ingress_bytes=261226496 link_bytes=261226496 token_bytes=447184904 weight_movement_bytes=37748736 total_nvlink_bytes=484933640 weight_to_token_ratio=0.084414 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.502943753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60900 payload_rows_remote=52708 dup_rows=4636 dup_rows_remote=4612 dedup_ratio=0.070740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0] payload_rows_per_dst=[3560, 8192, 8188, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4612, 0, 0, 0, 0, 24] combine_rows_per_src=[3560, 8192, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59150 payload_rows_remote=50958 dup_rows=6386 dup_rows_remote=6386 dedup_ratio=0.097443 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1723, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1690, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1986, 8192, 8192, 8012, 8192, 8192, 8192] dup_rows_per_dst=[0, 156, 0, 0, 2850, 3380, 0, 0] combine_rows_per_src=[8192, 1986, 8192, 8192, 8012, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=224002048 dispatch_meta_bytes=229400 combine_payload_bytes=224002048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224231448 ingress_bytes=242876416 link_bytes=242876416 token_bytes=448233496 weight_movement_bytes=18874368 total_nvlink_bytes=467107864 weight_to_token_ratio=0.042108 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.327122315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=215859200 dispatch_meta_bytes=229360 combine_payload_bytes=215859200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216088560 ingress_bytes=234733568 link_bytes=234733568 token_bytes=431947760 weight_movement_bytes=18874368 total_nvlink_bytes=450822128 weight_to_token_ratio=0.043696 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249101552e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=207273984 dispatch_meta_bytes=205984 combine_payload_bytes=207273984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207479968 ingress_bytes=226148352 link_bytes=226148352 token_bytes=414753952 weight_movement_bytes=18874368 total_nvlink_bytes=433628320 weight_to_token_ratio=0.045507 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.166842237e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=215891968 dispatch_meta_bytes=229280 combine_payload_bytes=215891968 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=216121248 ingress_bytes=291389440 link_bytes=291389440 token_bytes=432013216 weight_movement_bytes=75497472 total_nvlink_bytes=507510688 weight_to_token_ratio=0.174757 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791950242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=208723968 dispatch_meta_bytes=229376 combine_payload_bytes=208723968 prefetch_bytes=0 reduce_bytes=0 egress_bytes=208953344 ingress_bytes=208723968 link_bytes=208953344 token_bytes=417677312 weight_movement_bytes=0 total_nvlink_bytes=417677312 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.002088131e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60866 payload_rows_remote=57338 dup_rows=4670 dup_rows_remote=4670 dedup_ratio=0.071259 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 3528, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[4664, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8188, 3528, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235085824 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942272 weight_movement_bytes=56623104 total_nvlink_bytes=526565376 weight_to_token_ratio=0.120489 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65406 payload_rows_remote=57276 dup_rows=130 dup_rows_remote=124 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 1, 26, 0, 0, 0, 4062, 0, 0, 0, 4065, 0, 0, 0, 0, 23, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 7, 0, 20, 0, 0, 11, 0, 0, 4058, 0, 4065, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 1, 0, 30, 0] payload_rows_per_dst=[8180, 8176, 8178, 8170, 8190, 8130, 8190, 8192] dup_rows_per_dst=[12, 8, 0, 22, 62, 6, 0, 20] combine_rows_per_src=[8180, 8176, 8178, 8170, 8190, 8130, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=42 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4092, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8188, 8154, 8192] dup_rows_per_dst=[0, 0, 0, 24, 4, 0, 0, 18] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8188, 8154, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=248032 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235104480 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469960928 weight_movement_bytes=37748736 total_nvlink_bytes=507709664 weight_to_token_ratio=0.080323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=116 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8128, 8188, 8188, 8148] dup_rows_per_dst=[0, 8, 64, 44, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8128, 8188, 8188, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65402 payload_rows_remote=57210 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.002045 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8142, 8112, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 80, 50, 0] combine_rows_per_src=[8192, 8192, 8142, 8112, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229360 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234921968 ingress_bytes=253566976 link_bytes=253566976 token_bytes=469614576 weight_movement_bytes=18874368 total_nvlink_bytes=488488944 weight_to_token_ratio=0.040191 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429554001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229600 combine_payload_bytes=234602496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234832096 ingress_bytes=253476864 link_bytes=253476864 token_bytes=469434592 weight_movement_bytes=18874368 total_nvlink_bytes=488308960 weight_to_token_ratio=0.040207 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428690592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64850 payload_rows_remote=56658 dup_rows=686 dup_rows_remote=406 dedup_ratio=0.010468 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 198, 5, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7644, 8192, 8192, 8192, 8054] dup_rows_per_dst=[0, 0, 0, 0, 0, 406, 280, 0] combine_rows_per_src=[8192, 8192, 8192, 7644, 8192, 8192, 8192, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62392 payload_rows_remote=54760 dup_rows=3144 dup_rows_remote=3144 dedup_ratio=0.047974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5608, 8192, 7632, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 3144, 0, 0] combine_rows_per_src=[8192, 5608, 8192, 7632, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234332160 dispatch_meta_bytes=229376 combine_payload_bytes=234332160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234561536 ingress_bytes=253206528 link_bytes=253206528 token_bytes=468893696 weight_movement_bytes=18874368 total_nvlink_bytes=487768064 weight_to_token_ratio=0.040253 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426100366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=229376 combine_payload_bytes=234405888 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234635264 ingress_bytes=234405888 link_bytes=234635264 token_bytes=469041152 weight_movement_bytes=0 total_nvlink_bytes=469041152 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.248159652e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232071168 dispatch_meta_bytes=228256 combine_payload_bytes=232071168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232299424 ingress_bytes=250945536 link_bytes=250945536 token_bytes=464370592 weight_movement_bytes=18874368 total_nvlink_bytes=483244960 weight_to_token_ratio=0.040645 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404436653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=224296960 dispatch_meta_bytes=231616 combine_payload_bytes=224296960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224528576 ingress_bytes=243171328 link_bytes=243171328 token_bytes=448825536 weight_movement_bytes=18874368 total_nvlink_bytes=467699904 weight_to_token_ratio=0.042053 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329948017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58666 payload_rows_remote=50474 dup_rows=6870 dup_rows_remote=0 dedup_ratio=0.104828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3434, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8140, 8192, 8192, 5548, 8192, 4018, 8192, 8192] dup_rows_per_dst=[0, 6870, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8140, 8192, 8192, 5548, 8192, 4018, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61578 payload_rows_remote=53386 dup_rows=3958 dup_rows_remote=3958 dedup_ratio=0.060394 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4094, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3711, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5012, 8188, 7426, 8192, 8188, 8188, 8192] dup_rows_per_dst=[4, 0, 0, 4, 0, 3946, 0, 4] combine_rows_per_src=[8192, 5012, 8188, 7426, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63704 payload_rows_remote=55574 dup_rows=1832 dup_rows_remote=196 dedup_ratio=0.027954 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 4065, 0, 17, 3626, 0, 31, 0, 0, 0, 0, 0, 31, 1, 4065, 0, 0, 1, 0, 3785, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 26, 0, 0, 3, 0, 3903, 0, 31, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 1, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7314, 7744, 8120, 7868, 8192, 8130, 8144, 8192] dup_rows_per_dst=[46, 26, 0, 58, 10, 1636, 0, 56] combine_rows_per_src=[7314, 7744, 8120, 7868, 8192, 8130, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=206741504 dispatch_meta_bytes=201896 combine_payload_bytes=206741504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=206943400 ingress_bytes=244490240 link_bytes=244490240 token_bytes=413684904 weight_movement_bytes=37748736 total_nvlink_bytes=451433640 weight_to_token_ratio=0.091250 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.342585183e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=218669056 dispatch_meta_bytes=229376 combine_payload_bytes=218669056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218898432 ingress_bytes=237543424 link_bytes=237543424 token_bytes=437567488 weight_movement_bytes=18874368 total_nvlink_bytes=456441856 weight_to_token_ratio=0.043135 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276024210e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63968 payload_rows_remote=55776 dup_rows=1568 dup_rows_remote=1568 dedup_ratio=0.023926 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 7658, 8192, 7204, 8192, 8166, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 1516, 52, 0] combine_rows_per_src=[8172, 7658, 8192, 7204, 8192, 8166, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=227631104 dispatch_meta_bytes=223080 combine_payload_bytes=227631104 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=227854184 ingress_bytes=322002944 link_bytes=322002944 token_bytes=455485288 weight_movement_bytes=94371840 total_nvlink_bytes=549857128 weight_to_token_ratio=0.207190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.085273774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58616 payload_rows_remote=50708 dup_rows=6920 dup_rows_remote=5734 dedup_ratio=0.105591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7908, 7956, 8192, 7660, 8182, 2394, 8192, 8132] dup_rows_per_dst=[1186, 5734, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7908, 7956, 8192, 7660, 8182, 2394, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65294 payload_rows_remote=57102 dup_rows=242 dup_rows_remote=100 dedup_ratio=0.003693 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7950, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 100, 0, 0, 0, 142, 0] combine_rows_per_src=[7950, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=228458496 dispatch_meta_bytes=229376 combine_payload_bytes=228458496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228687872 ingress_bytes=247332864 link_bytes=247332864 token_bytes=457146368 weight_movement_bytes=18874368 total_nvlink_bytes=476020736 weight_to_token_ratio=0.041287 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.369821807e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=207699968 dispatch_meta_bytes=225768 combine_payload_bytes=207699968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=207925736 ingress_bytes=245448704 link_bytes=245448704 token_bytes=415625704 weight_movement_bytes=37748736 total_nvlink_bytes=453374440 weight_to_token_ratio=0.090824 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351768713e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58612 payload_rows_remote=50420 dup_rows=6924 dup_rows_remote=6848 dedup_ratio=0.105652 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 5680, 8192, 8140, 8192, 3834, 8192, 8192] dup_rows_per_dst=[0, 546, 76, 6206, 0, 96, 0, 0] combine_rows_per_src=[8190, 5680, 8192, 8140, 8192, 3834, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=233889792 dispatch_meta_bytes=228808 combine_payload_bytes=233889792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234118600 ingress_bytes=252764160 link_bytes=252764160 token_bytes=468008392 weight_movement_bytes=18874368 total_nvlink_bytes=486882760 weight_to_token_ratio=0.040329 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421861813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57158 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.002899 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 87, 2, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8188, 8014, 8188, 8192] dup_rows_per_dst=[0, 0, 4, 4, 0, 0, 0, 182] combine_rows_per_src=[8188, 8192, 8192, 8192, 8188, 8014, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8168, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 26] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8168, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235061248 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893120 weight_movement_bytes=37748736 total_nvlink_bytes=507641856 weight_to_token_ratio=0.080335 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=206520320 dispatch_meta_bytes=229072 combine_payload_bytes=206520320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206749392 ingress_bytes=225394688 link_bytes=225394688 token_bytes=413269712 weight_movement_bytes=18874368 total_nvlink_bytes=432144080 weight_to_token_ratio=0.045671 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159620999e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65198 payload_rows_remote=57050 dup_rows=338 dup_rows_remote=338 dedup_ratio=0.005157 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 4064, 0, 0, 0, 0, 6, 0, 31, 4064, 31, 0, 4065, 0, 0, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4065, 0, 52, 31, 0] payload_rows_per_dst=[8130, 8190, 8192, 8192, 8130, 8148, 8024, 8192] dup_rows_per_dst=[0, 2, 74, 66, 2, 0, 0, 194] combine_rows_per_src=[8130, 8190, 8192, 8192, 8130, 8148, 8024, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234119168 dispatch_meta_bytes=229392 combine_payload_bytes=234119168 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234348560 ingress_bytes=271867904 link_bytes=271867904 token_bytes=468467728 weight_movement_bytes=37748736 total_nvlink_bytes=506216464 weight_to_token_ratio=0.080579 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604904489e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65224 payload_rows_remote=57032 dup_rows=312 dup_rows_remote=64 dedup_ratio=0.004761 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 115, 0, 0] payload_rows_per_dst=[8192, 8192, 8128, 8174, 8192, 7962, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 64, 0, 0, 248] combine_rows_per_src=[8192, 8192, 8128, 8174, 8192, 7962, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=234774528 link_bytes=235003904 token_bytes=469778432 weight_movement_bytes=0 total_nvlink_bytes=469778432 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.251691779e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233676800 dispatch_meta_bytes=229552 combine_payload_bytes=233676800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233906352 ingress_bytes=252551168 link_bytes=252551168 token_bytes=467583152 weight_movement_bytes=18874368 total_nvlink_bytes=486457520 weight_to_token_ratio=0.040366 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419821029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64366 payload_rows_remote=56174 dup_rows=1170 dup_rows_remote=1104 dedup_ratio=0.017853 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 499, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8140, 8192, 8078, 8192, 7202] dup_rows_per_dst=[66, 0, 0, 0, 0, 848, 176, 80] combine_rows_per_src=[8192, 8192, 8178, 8140, 8192, 8078, 8192, 7202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62878 payload_rows_remote=54686 dup_rows=2658 dup_rows_remote=2652 dedup_ratio=0.040558 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0] payload_rows_per_dst=[5534, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2616, 0, 0, 0, 6, 36] combine_rows_per_src=[5534, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233603072 dispatch_meta_bytes=228384 combine_payload_bytes=233603072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233831456 ingress_bytes=290226176 link_bytes=290226176 token_bytes=467434528 weight_movement_bytes=56623104 total_nvlink_bytes=524057632 weight_to_token_ratio=0.121136 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780804418e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65358 payload_rows_remote=57166 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 89, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8016, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 178] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8016, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=230088704 dispatch_meta_bytes=229112 combine_payload_bytes=230088704 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230317816 ingress_bytes=267837440 link_bytes=267837440 token_bytes=460406520 weight_movement_bytes=37748736 total_nvlink_bytes=498155256 weight_to_token_ratio=0.081990 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.566286566e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=223993856 dispatch_meta_bytes=229352 combine_payload_bytes=223993856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=224223208 ingress_bytes=261742592 link_bytes=261742592 token_bytes=448217064 weight_movement_bytes=37748736 total_nvlink_bytes=485965800 weight_to_token_ratio=0.084220 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.507888731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4077, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8188, 8192, 8188, 8158, 8192] dup_rows_per_dst=[0, 0, 4, 34, 0, 0, 8, 0] combine_rows_per_src=[8192, 8188, 8192, 8188, 8192, 8188, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65328 payload_rows_remote=57198 dup_rows=208 dup_rows_remote=208 dedup_ratio=0.003174 router_tokens_per_expert=[0, 0, 0, 4065, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4065, 0, 0, 4065, 0, 31, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 127, 0, 0, 0, 31, 0, 3, 31, 3937, 0, 31, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 31, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8130, 8172, 8132, 8190, 8130, 8192, 8192] dup_rows_per_dst=[0, 0, 60, 0, 14, 0, 130, 4] combine_rows_per_src=[8190, 8130, 8172, 8132, 8190, 8130, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234151936 dispatch_meta_bytes=229376 combine_payload_bytes=234151936 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234381312 ingress_bytes=328523776 link_bytes=328523776 token_bytes=468533248 weight_movement_bytes=94371840 total_nvlink_bytes=562905088 weight_to_token_ratio=0.201420 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.147753178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234921984 ingress_bytes=272441344 link_bytes=272441344 token_bytes=469614592 weight_movement_bytes=37748736 total_nvlink_bytes=507363328 weight_to_token_ratio=0.080382 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610398909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8138, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 30, 0, 0, 24, 0] combine_rows_per_src=[8192, 8192, 8138, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110408 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991432 weight_movement_bytes=37748736 total_nvlink_bytes=507740168 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229624 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512632 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795640 weight_movement_bytes=18874368 total_nvlink_bytes=487670008 weight_to_token_ratio=0.040261 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8182, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57198 dup_rows=146 dup_rows_remote=144 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8180, 8192, 8062, 8188] dup_rows_per_dst=[2, 68, 10, 0, 0, 66, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8180, 8192, 8062, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229376 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889216 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549056 weight_movement_bytes=18874368 total_nvlink_bytes=488423424 weight_to_token_ratio=0.040197 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229368 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512376 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795384 weight_movement_bytes=18874368 total_nvlink_bytes=487669752 weight_to_token_ratio=0.040261 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57334 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8190, 8192, 8188, 8188, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 4, 4] combine_rows_per_src=[8190, 8192, 8188, 8188, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229360 combine_payload_bytes=234864640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235094000 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958640 weight_movement_bytes=37748736 total_nvlink_bytes=507707376 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57156 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.002899 router_tokens_per_expert=[0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 31, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4064, 4065, 1, 1, 0, 4063, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 31, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4061, 0, 0] payload_rows_per_dst=[8192, 8192, 8134, 8130, 8134, 8190, 8190, 8184] dup_rows_per_dst=[0, 62, 4, 12, 0, 0, 50, 62] combine_rows_per_src=[8192, 8192, 8134, 8130, 8134, 8190, 8190, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229384 combine_payload_bytes=234840064 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235069448 ingress_bytes=272588800 link_bytes=272588800 token_bytes=469909512 weight_movement_bytes=37748736 total_nvlink_bytes=507658248 weight_to_token_ratio=0.080332 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611811760e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110408 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991432 weight_movement_bytes=56623104 total_nvlink_bytes=526614536 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234110976 dispatch_meta_bytes=229384 combine_payload_bytes=234110976 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234340360 ingress_bytes=290734080 link_bytes=290734080 token_bytes=468451336 weight_movement_bytes=56623104 total_nvlink_bytes=525074440 weight_to_token_ratio=0.120873 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.785670904e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65432 payload_rows_remote=57240 dup_rows=104 dup_rows_remote=98 dedup_ratio=0.001587 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8146, 8134, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 58, 40, 0, 6] combine_rows_per_src=[8192, 8146, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61452 payload_rows_remote=53260 dup_rows=4084 dup_rows_remote=4084 dedup_ratio=0.062317 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4510, 8192, 7790, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4084, 0, 0] combine_rows_per_src=[8192, 4510, 8192, 7790, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=166 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8192, 8192, 8154, 8140, 8128] dup_rows_per_dst=[2, 0, 116, 50, 0, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 8192, 8154, 8140, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229352 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234684392 ingress_bytes=272203776 link_bytes=272203776 token_bytes=469139432 weight_movement_bytes=37748736 total_nvlink_bytes=506888168 weight_to_token_ratio=0.080464 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608122649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=218152960 dispatch_meta_bytes=229376 combine_payload_bytes=218152960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218382336 ingress_bytes=237027328 link_bytes=237027328 token_bytes=436535296 weight_movement_bytes=18874368 total_nvlink_bytes=455409664 weight_to_token_ratio=0.043237 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.271079232e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57328 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8188, 8192, 8188, 8188] dup_rows_per_dst=[8, 0, 4, 0, 0, 8, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8188, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229368 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422264 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615160 weight_movement_bytes=18874368 total_nvlink_bytes=487489528 weight_to_token_ratio=0.040277 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65214 payload_rows_remote=57022 dup_rows=322 dup_rows_remote=180 dedup_ratio=0.004913 router_tokens_per_expert=[0, 25, 0, 0, 0, 0, 30, 0, 31, 0, 0, 0, 0, 0, 1, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 31, 4064, 0, 0, 0, 0, 0, 4063, 0, 0, 31, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4065, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 0, 12, 0, 0, 0, 25, 33, 4065, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8126, 8190, 8188, 8130, 8128, 8192, 8130, 8130] dup_rows_per_dst=[110, 0, 62, 0, 0, 142, 4, 4] combine_rows_per_src=[8126, 8190, 8188, 8130, 8128, 8192, 8130, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229392 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235044880 ingress_bytes=329187328 link_bytes=329187328 token_bytes=469860368 weight_movement_bytes=94371840 total_nvlink_bytes=564232208 weight_to_token_ratio=0.200851 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154111007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64718 payload_rows_remote=56526 dup_rows=818 dup_rows_remote=756 dedup_ratio=0.012482 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8146, 8192, 8132, 8192, 7494, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 66, 0, 668, 22, 62] combine_rows_per_src=[8146, 8192, 8132, 8192, 7494, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233562112 dispatch_meta_bytes=228808 combine_payload_bytes=233562112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233790920 ingress_bytes=252436480 link_bytes=252436480 token_bytes=467353032 weight_movement_bytes=18874368 total_nvlink_bytes=486227400 weight_to_token_ratio=0.040386 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418722145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8122, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 70] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8122, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65358 payload_rows_remote=57166 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8018, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 174, 4, 0] combine_rows_per_src=[8192, 8188, 8018, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231530496 dispatch_meta_bytes=229128 combine_payload_bytes=231530496 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231759624 ingress_bytes=269279232 link_bytes=269279232 token_bytes=463290120 weight_movement_bytes=37748736 total_nvlink_bytes=501038856 weight_to_token_ratio=0.081480 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580101107e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64750 payload_rows_remote=56558 dup_rows=786 dup_rows_remote=786 dedup_ratio=0.011993 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 7440, 8160, 8192] dup_rows_per_dst=[0, 2, 0, 0, 766, 0, 18, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 7440, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229376 combine_payload_bytes=234594304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234823680 ingress_bytes=253468672 link_bytes=253468672 token_bytes=469417984 weight_movement_bytes=18874368 total_nvlink_bytes=488292352 weight_to_token_ratio=0.040208 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428612101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234151936 dispatch_meta_bytes=229376 combine_payload_bytes=234151936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234381312 ingress_bytes=253026304 link_bytes=253026304 token_bytes=468533248 weight_movement_bytes=18874368 total_nvlink_bytes=487407616 weight_to_token_ratio=0.040284 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424373548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65172 payload_rows_remote=56980 dup_rows=364 dup_rows_remote=364 dedup_ratio=0.005554 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 176, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8156, 7870, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 360, 0, 0] combine_rows_per_src=[8192, 8156, 7870, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231661568 dispatch_meta_bytes=229376 combine_payload_bytes=231661568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231890944 ingress_bytes=269410304 link_bytes=269410304 token_bytes=463552512 weight_movement_bytes=37748736 total_nvlink_bytes=501301248 weight_to_token_ratio=0.081434 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581356975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57126 dup_rows=218 dup_rows_remote=64 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 31, 0, 0, 31, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 31, 0, 0, 0, 0, 0, 29, 0, 0, 31, 0, 0, 0, 14, 2, 0, 1, 31, 0, 4054, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8130, 8142, 8192, 8192, 8192, 8108, 8170] dup_rows_per_dst=[62, 0, 0, 0, 2, 154, 0, 0] combine_rows_per_src=[8192, 8130, 8142, 8192, 8192, 8192, 8108, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235077632 ingress_bytes=329220096 link_bytes=329220096 token_bytes=469925888 weight_movement_bytes=94371840 total_nvlink_bytes=564297728 weight_to_token_ratio=0.200823 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154424973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=229376 combine_payload_bytes=233390080 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233619456 ingress_bytes=271138816 link_bytes=271138816 token_bytes=467009536 weight_movement_bytes=37748736 total_nvlink_bytes=504758272 weight_to_token_ratio=0.080831 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597918726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63876 payload_rows_remote=55684 dup_rows=1660 dup_rows_remote=1644 dedup_ratio=0.025330 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 7988, 8174, 8192, 8192, 6828, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 1638, 16] combine_rows_per_src=[8118, 7988, 8174, 8192, 8192, 6828, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233988096 dispatch_meta_bytes=228760 combine_payload_bytes=233988096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234216856 ingress_bytes=252862464 link_bytes=252862464 token_bytes=468204952 weight_movement_bytes=18874368 total_nvlink_bytes=487079320 weight_to_token_ratio=0.040312 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422803714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57128 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7976, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 216, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7976, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=228081664 dispatch_meta_bytes=229312 combine_payload_bytes=228081664 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228310976 ingress_bytes=265830400 link_bytes=265830400 token_bytes=456392640 weight_movement_bytes=37748736 total_nvlink_bytes=494141376 weight_to_token_ratio=0.082711 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.547056096e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63958 payload_rows_remote=55766 dup_rows=1578 dup_rows_remote=1570 dedup_ratio=0.024078 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8176, 8192, 8190, 8190, 6640, 8192] dup_rows_per_dst=[8, 302, 1244, 0, 0, 24, 0, 0] combine_rows_per_src=[8192, 8186, 8176, 8192, 8190, 8190, 6640, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64932 payload_rows_remote=57344 dup_rows=604 dup_rows_remote=604 dedup_ratio=0.009216 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7588, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 600, 0, 0] combine_rows_per_src=[8192, 8192, 7588, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229376 combine_payload_bytes=233996288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234225664 ingress_bytes=252870656 link_bytes=252870656 token_bytes=468221952 weight_movement_bytes=18874368 total_nvlink_bytes=487096320 weight_to_token_ratio=0.040311 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422882206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57294 dup_rows=54 dup_rows_remote=12 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8182, 8192, 8192, 8188, 8160, 8188, 8192] dup_rows_per_dst=[0, 4, 4, 0, 42, 4, 0, 0] combine_rows_per_src=[8188, 8182, 8192, 8192, 8188, 8160, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 26, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=228417536 dispatch_meta_bytes=229344 combine_payload_bytes=228417536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228646880 ingress_bytes=266166272 link_bytes=266166272 token_bytes=457064416 weight_movement_bytes=37748736 total_nvlink_bytes=494813152 weight_to_token_ratio=0.082590 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.550274256e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64534 payload_rows_remote=57046 dup_rows=1002 dup_rows_remote=924 dedup_ratio=0.015289 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 31, 0, 0, 22, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4065, 30, 0, 0, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 0, 30, 0, 31, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4065, 0, 1, 0, 4065, 0, 0, 0, 24, 0, 0, 0, 0] payload_rows_per_dst=[8132, 8090, 8192, 8190, 8130, 7488, 8132, 8180] dup_rows_per_dst=[2, 44, 62, 12, 804, 78, 0, 0] combine_rows_per_src=[8132, 8090, 8192, 8190, 8130, 7488, 8132, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=231792 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235112816 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469993840 weight_movement_bytes=18874368 total_nvlink_bytes=488868208 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229224 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905448 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581672 weight_movement_bytes=18874368 total_nvlink_bytes=488456040 weight_to_token_ratio=0.040194 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63996 payload_rows_remote=55804 dup_rows=1540 dup_rows_remote=1526 dedup_ratio=0.023499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7976, 8116, 8182, 6978, 8170, 8190, 8192] dup_rows_per_dst=[0, 2, 10, 0, 22, 1416, 76, 14] combine_rows_per_src=[8192, 7976, 8116, 8182, 6978, 8170, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235003904 ingress_bytes=329146368 link_bytes=329146368 token_bytes=469778432 weight_movement_bytes=94371840 total_nvlink_bytes=564150272 weight_to_token_ratio=0.200886 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153718548e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233660416 dispatch_meta_bytes=231880 combine_payload_bytes=233660416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233892296 ingress_bytes=271409152 link_bytes=271409152 token_bytes=467552712 weight_movement_bytes=37748736 total_nvlink_bytes=505301448 weight_to_token_ratio=0.080737 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600508953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8154, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 36] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8154, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=228573184 dispatch_meta_bytes=229320 combine_payload_bytes=228573184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228802504 ingress_bytes=266321920 link_bytes=266321920 token_bytes=457375688 weight_movement_bytes=37748736 total_nvlink_bytes=495124424 weight_to_token_ratio=0.082533 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.551765599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65204 payload_rows_remote=57012 dup_rows=332 dup_rows_remote=270 dedup_ratio=0.005066 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8192, 8178, 7892, 8182, 8192] dup_rows_per_dst=[62, 270, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8184, 8192, 8192, 8178, 7892, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57312 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8160, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 32, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8160, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229376 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234954752 ingress_bytes=253599744 link_bytes=253599744 token_bytes=469680128 weight_movement_bytes=18874368 total_nvlink_bytes=488554496 weight_to_token_ratio=0.040186 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429867968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 4094, 0, 0, 4081, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9] payload_rows_per_dst=[8192, 8192, 8166, 8192, 8192, 8188, 8180, 8184] dup_rows_per_dst=[4, 0, 4, 0, 0, 36, 0, 6] combine_rows_per_src=[8192, 8192, 8166, 8192, 8192, 8188, 8180, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65362 payload_rows_remote=57170 dup_rows=174 dup_rows_remote=174 dedup_ratio=0.002655 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4009, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 80] payload_rows_per_dst=[8192, 8192, 8018, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 20, 0, 154] combine_rows_per_src=[8192, 8192, 8018, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233521152 dispatch_meta_bytes=229128 combine_payload_bytes=233521152 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233750280 ingress_bytes=271269888 link_bytes=271269888 token_bytes=467271432 weight_movement_bytes=37748736 total_nvlink_bytes=505020168 weight_to_token_ratio=0.080785 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599174594e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64752 payload_rows_remote=56622 dup_rows=784 dup_rows_remote=268 dedup_ratio=0.011963 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 1, 0, 0, 31, 25, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4065, 0, 25, 0, 4065, 0, 0, 3810, 3953, 31, 0, 0, 0, 0, 0, 0, 0, 6, 0, 31, 0, 0, 0, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 4064, 0, 4065, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 76] payload_rows_per_dst=[8192, 8180, 7682, 8192, 8190, 8130, 8192, 7994] dup_rows_per_dst=[64, 6, 50, 74, 0, 516, 0, 74] combine_rows_per_src=[8192, 8180, 7682, 8192, 8190, 8130, 8192, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229376 combine_payload_bytes=234749952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234979328 ingress_bytes=272498688 link_bytes=272498688 token_bytes=469729280 weight_movement_bytes=37748736 total_nvlink_bytes=507478016 weight_to_token_ratio=0.080363 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610948351e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229376 combine_payload_bytes=234676224 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234905600 ingress_bytes=272424960 link_bytes=272424960 token_bytes=469581824 weight_movement_bytes=37748736 total_nvlink_bytes=507330560 weight_to_token_ratio=0.080388 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610241925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64964 payload_rows_remote=56772 dup_rows=572 dup_rows_remote=310 dedup_ratio=0.008728 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3957, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131] payload_rows_per_dst=[8124, 8192, 7762, 8118, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 156, 152, 262] combine_rows_per_src=[8124, 8192, 7762, 8118, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234168320 dispatch_meta_bytes=229376 combine_payload_bytes=234168320 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234397696 ingress_bytes=271917056 link_bytes=271917056 token_bytes=468566016 weight_movement_bytes=37748736 total_nvlink_bytes=506314752 weight_to_token_ratio=0.080562 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605375439e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=231923712 dispatch_meta_bytes=227560 combine_payload_bytes=231923712 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232151272 ingress_bytes=288546816 link_bytes=288546816 token_bytes=464074984 weight_movement_bytes=56623104 total_nvlink_bytes=520698088 weight_to_token_ratio=0.122013 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.764713617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=0 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8150, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 44, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8150, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=232538112 dispatch_meta_bytes=228328 combine_payload_bytes=232538112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232766440 ingress_bytes=251412480 link_bytes=251412480 token_bytes=465304552 weight_movement_bytes=18874368 total_nvlink_bytes=484178920 weight_to_token_ratio=0.040563 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408910681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64208 payload_rows_remote=56016 dup_rows=1328 dup_rows_remote=1328 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 3616, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852] payload_rows_per_dst=[8192, 8192, 8084, 8192, 8192, 7520, 8144, 7692] dup_rows_per_dst=[0, 6, 1214, 0, 0, 0, 0, 108] combine_rows_per_src=[8192, 8192, 8084, 8192, 8192, 7520, 8144, 7692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57344 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4088, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10] payload_rows_per_dst=[8192, 8192, 8176, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 12] combine_rows_per_src=[8192, 8192, 8176, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229200 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234930000 ingress_bytes=253575168 link_bytes=253575168 token_bytes=469630800 weight_movement_bytes=18874368 total_nvlink_bytes=488505168 weight_to_token_ratio=0.040190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429632493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 3227, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 2] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57256 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.001343 router_tokens_per_expert=[4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 0, 4054, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 792, 6, 0, 0, 0, 0, 0, 0, 3277, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4065, 31] payload_rows_per_dst=[8190, 8108, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 20, 2, 0, 2, 62] combine_rows_per_src=[8190, 8108, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229440 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=235110464 ingress_bytes=348127232 link_bytes=348127232 token_bytes=469991488 weight_movement_bytes=113246208 total_nvlink_bytes=583237696 weight_to_token_ratio=0.240954 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.335583848e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=229441536 dispatch_meta_bytes=229376 combine_payload_bytes=229441536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229670912 ingress_bytes=248315904 link_bytes=248315904 token_bytes=459112448 weight_movement_bytes=18874368 total_nvlink_bytes=477986816 weight_to_token_ratio=0.041111 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379240813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8172, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 20] combine_rows_per_src=[8192, 8192, 8192, 8172, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229376 combine_payload_bytes=234520576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234749952 ingress_bytes=253394944 link_bytes=253394944 token_bytes=469270528 weight_movement_bytes=18874368 total_nvlink_bytes=488144896 weight_to_token_ratio=0.040221 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427905675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8132, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 54, 0, 6] combine_rows_per_src=[8192, 8192, 8132, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64074 payload_rows_remote=55882 dup_rows=1462 dup_rows_remote=0 dedup_ratio=0.022308 router_tokens_per_expert=[0, 0, 0, 3397, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6794, 8192, 8192, 8192, 8192, 8128, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1462, 0, 0, 0, 0] combine_rows_per_src=[6794, 8192, 8192, 8192, 8192, 8128, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229296 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235028400 ingress_bytes=272547840 link_bytes=272547840 token_bytes=469827504 weight_movement_bytes=37748736 total_nvlink_bytes=507576240 weight_to_token_ratio=0.080346 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611419301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 0, 0, 0, 0, 0, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229376 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234864640 ingress_bytes=291258368 link_bytes=291258368 token_bytes=469499904 weight_movement_bytes=56623104 total_nvlink_bytes=526123008 weight_to_token_ratio=0.120603 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790694374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.001373 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8190, 8106] dup_rows_per_dst=[0, 88, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8190, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=228892672 dispatch_meta_bytes=223528 combine_payload_bytes=228892672 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=229116200 ingress_bytes=323264512 link_bytes=323264512 token_bytes=458008872 weight_movement_bytes=94371840 total_nvlink_bytes=552380712 weight_to_token_ratio=0.206048 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.097361498e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64324 payload_rows_remote=56136 dup_rows=1212 dup_rows_remote=1212 dedup_ratio=0.018494 router_tokens_per_expert=[0, 0, 0, 3494, 0, 2, 0, 4094, 4094, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 600, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6992, 8192, 8188, 8192, 8188, 8188, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 1204, 0, 0, 0, 0] combine_rows_per_src=[6992, 8192, 8188, 8192, 8188, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60978 payload_rows_remote=52786 dup_rows=4558 dup_rows_remote=4558 dedup_ratio=0.069550 router_tokens_per_expert=[0, 0, 0, 3452, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7030, 8192, 8192, 4862, 8192, 8126, 8192, 8192] dup_rows_per_dst=[3342, 0, 0, 1212, 4, 0, 0, 0] combine_rows_per_src=[7030, 8192, 8192, 4862, 8192, 8126, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61542 payload_rows_remote=53442 dup_rows=3994 dup_rows_remote=3994 dedup_ratio=0.060944 router_tokens_per_expert=[0, 0, 0, 2180, 0, 31, 0, 4065, 4065, 0, 31, 0, 0, 0, 31, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1901, 0, 0, 0, 31, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 30, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4422, 8192, 8130, 8192, 8128, 8100, 8188, 8190] dup_rows_per_dst=[124, 6, 0, 3864, 0, 0, 0, 0] combine_rows_per_src=[4422, 8192, 8130, 8192, 8128, 8100, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229376 combine_payload_bytes=234512384 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234741760 ingress_bytes=234512384 link_bytes=234741760 token_bytes=469254144 weight_movement_bytes=0 total_nvlink_bytes=469254144 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249180044e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=229933056 dispatch_meta_bytes=229392 combine_payload_bytes=229933056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230162448 ingress_bytes=248807424 link_bytes=248807424 token_bytes=460095504 weight_movement_bytes=18874368 total_nvlink_bytes=478969872 weight_to_token_ratio=0.041023 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.383950316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64456 payload_rows_remote=56264 dup_rows=1080 dup_rows_remote=1078 dedup_ratio=0.016479 router_tokens_per_expert=[0, 0, 0, 4055, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8044, 8192, 8192, 7344, 8192, 8108, 8192, 8192] dup_rows_per_dst=[0, 0, 66, 100, 34, 874, 4, 2] combine_rows_per_src=[8044, 8192, 8192, 7344, 8192, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=216211456 dispatch_meta_bytes=229376 combine_payload_bytes=216211456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216440832 ingress_bytes=235085824 link_bytes=235085824 token_bytes=432652288 weight_movement_bytes=18874368 total_nvlink_bytes=451526656 weight_to_token_ratio=0.043625 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252476696e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=218898432 dispatch_meta_bytes=229744 combine_payload_bytes=218898432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219128176 ingress_bytes=237772800 link_bytes=237772800 token_bytes=438026608 weight_movement_bytes=18874368 total_nvlink_bytes=456900976 weight_to_token_ratio=0.043090 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.278221978e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8024, 8192, 8192] dup_rows_per_dst=[0, 16, 0, 0, 152, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8024, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63696 payload_rows_remote=55504 dup_rows=1840 dup_rows_remote=1840 dedup_ratio=0.028076 router_tokens_per_expert=[0, 4096, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6354, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1796, 44, 0, 0] combine_rows_per_src=[6354, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=230457344 dispatch_meta_bytes=229368 combine_payload_bytes=230457344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230686712 ingress_bytes=249331712 link_bytes=249331712 token_bytes=461144056 weight_movement_bytes=18874368 total_nvlink_bytes=480018424 weight_to_token_ratio=0.040929 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388973786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64730 payload_rows_remote=56538 dup_rows=806 dup_rows_remote=806 dedup_ratio=0.012299 router_tokens_per_expert=[0, 0, 0, 3705, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7410, 8192, 8192, 8192, 8192, 8168, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 800, 6, 0, 0, 0] combine_rows_per_src=[7410, 8192, 8192, 8192, 8192, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=227344384 dispatch_meta_bytes=229376 combine_payload_bytes=227344384 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227573760 ingress_bytes=283967488 link_bytes=283967488 token_bytes=454918144 weight_movement_bytes=56623104 total_nvlink_bytes=511541248 weight_to_token_ratio=0.124469 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.720836749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59302 payload_rows_remote=51110 dup_rows=6234 dup_rows_remote=24 dedup_ratio=0.095123 router_tokens_per_expert=[0, 0, 0, 3102, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 2470, 8192, 7686, 8192, 8192] dup_rows_per_dst=[6210, 0, 0, 0, 24, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 2470, 8192, 7686, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63220 payload_rows_remote=55028 dup_rows=2316 dup_rows_remote=154 dedup_ratio=0.035339 router_tokens_per_expert=[0, 4094, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4068, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5936, 8192, 8140, 8188, 8192, 8192, 8192, 8188] dup_rows_per_dst=[8, 0, 0, 0, 2162, 146, 0, 0] combine_rows_per_src=[5936, 8192, 8140, 8188, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59262 payload_rows_remote=51070 dup_rows=6274 dup_rows_remote=6274 dedup_ratio=0.095734 router_tokens_per_expert=[0, 4096, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 1920, 8192, 8192, 8190] dup_rows_per_dst=[6232, 0, 2, 0, 0, 40, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 1920, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61180 payload_rows_remote=52990 dup_rows=4356 dup_rows_remote=4198 dedup_ratio=0.066467 router_tokens_per_expert=[0, 4065, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 31, 31, 0, 0, 0, 31, 0, 0, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4005, 0, 31, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 6, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 30, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4082, 8192, 8072, 8142, 8192, 8190, 8180, 8130] dup_rows_per_dst=[110, 2, 0, 0, 4086, 158, 0, 0] combine_rows_per_src=[4082, 8192, 8072, 8142, 8192, 8190, 8180, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=231579648 dispatch_meta_bytes=229376 combine_payload_bytes=231579648 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231809024 ingress_bytes=269328384 link_bytes=269328384 token_bytes=463388672 weight_movement_bytes=37748736 total_nvlink_bytes=501137408 weight_to_token_ratio=0.081462 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580572058e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=209346560 dispatch_meta_bytes=204536 combine_payload_bytes=209346560 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=209551096 ingress_bytes=284844032 link_bytes=284844032 token_bytes=418897656 weight_movement_bytes=75497472 total_nvlink_bytes=494395128 weight_to_token_ratio=0.180229 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.729235363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=225394688 dispatch_meta_bytes=220728 combine_payload_bytes=225394688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225615416 ingress_bytes=244269056 link_bytes=244269056 token_bytes=451010104 weight_movement_bytes=18874368 total_nvlink_bytes=469884472 weight_to_token_ratio=0.041849 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.340465907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61552 payload_rows_remote=53360 dup_rows=3984 dup_rows_remote=3916 dedup_ratio=0.060791 router_tokens_per_expert=[0, 4096, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4354, 8190, 8060, 8180, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 66, 1794, 2044, 12, 68] combine_rows_per_src=[4354, 8190, 8060, 8180, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=209182720 dispatch_meta_bytes=229376 combine_payload_bytes=209182720 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=209412096 ingress_bytes=228057088 link_bytes=228057088 token_bytes=418594816 weight_movement_bytes=18874368 total_nvlink_bytes=437469184 weight_to_token_ratio=0.045090 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.185130806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=217047040 dispatch_meta_bytes=228752 combine_payload_bytes=217047040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217275792 ingress_bytes=235921408 link_bytes=235921408 token_bytes=434322832 weight_movement_bytes=18874368 total_nvlink_bytes=453197200 weight_to_token_ratio=0.043457 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260482851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65478 payload_rows_remote=57340 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000885 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 8192, 8138, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 54] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 8192, 8138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65026 payload_rows_remote=56834 dup_rows=510 dup_rows_remote=4 dedup_ratio=0.007782 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7682, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 506, 0, 0, 4, 0] combine_rows_per_src=[7682, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=218562560 dispatch_meta_bytes=229104 combine_payload_bytes=218562560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218791664 ingress_bytes=237436928 link_bytes=237436928 token_bytes=437354224 weight_movement_bytes=18874368 total_nvlink_bytes=456228592 weight_to_token_ratio=0.043156 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.275003818e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229592 combine_payload_bytes=234864640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235094232 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958872 weight_movement_bytes=37748736 total_nvlink_bytes=507707608 weight_to_token_ratio=0.080323 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232792064 dispatch_meta_bytes=227352 combine_payload_bytes=232792064 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=233019416 ingress_bytes=308289536 link_bytes=308289536 token_bytes=465811480 weight_movement_bytes=75497472 total_nvlink_bytes=541308952 weight_to_token_ratio=0.162077 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.953878646e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62476 payload_rows_remote=54284 dup_rows=3060 dup_rows_remote=3060 dedup_ratio=0.046692 router_tokens_per_expert=[0, 4096, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6048, 8192, 8192, 8192, 7276, 8192, 8192, 8192] dup_rows_per_dst=[856, 0, 0, 0, 1682, 522, 0, 0] combine_rows_per_src=[6048, 8192, 8192, 8192, 7276, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55930 payload_rows_remote=47738 dup_rows=9606 dup_rows_remote=994 dedup_ratio=0.146576 router_tokens_per_expert=[0, 4096, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 2290, 4488, 8192, 8192] dup_rows_per_dst=[8612, 0, 304, 0, 0, 690, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 2290, 4488, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64182 payload_rows_remote=55990 dup_rows=1354 dup_rows_remote=1354 dedup_ratio=0.020660 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 671, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[6848, 8192, 8186, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1346, 0, 0, 8, 0] combine_rows_per_src=[6848, 8192, 8186, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64880 payload_rows_remote=56688 dup_rows=656 dup_rows_remote=656 dedup_ratio=0.010010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7536, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 652, 0, 0, 4, 0] combine_rows_per_src=[7536, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63594 payload_rows_remote=55458 dup_rows=1942 dup_rows_remote=1942 dedup_ratio=0.029633 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 31, 4065, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 30, 31, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 874, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4063, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 31, 0, 0, 31, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0] payload_rows_per_dst=[6396, 8192, 8126, 8192, 8182, 8136, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 1810, 8, 0, 124, 0] combine_rows_per_src=[6396, 8192, 8126, 8192, 8182, 8136, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=222347264 dispatch_meta_bytes=229376 combine_payload_bytes=222347264 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=222576640 ingress_bytes=316719104 link_bytes=316719104 token_bytes=444923904 weight_movement_bytes=94371840 total_nvlink_bytes=539295744 weight_to_token_ratio=0.212108 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.034646619e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=195534848 dispatch_meta_bytes=194928 combine_payload_bytes=195534848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=195729776 ingress_bytes=233283584 link_bytes=233283584 token_bytes=391264624 weight_movement_bytes=37748736 total_nvlink_bytes=429013360 weight_to_token_ratio=0.096479 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235208519e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57152 dup_rows=192 dup_rows_remote=192 dedup_ratio=0.002930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8064, 8190, 8190, 8132, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 130, 2, 60, 0, 0] combine_rows_per_src=[8064, 8190, 8190, 8132, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=229335040 dispatch_meta_bytes=229376 combine_payload_bytes=229335040 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229564416 ingress_bytes=267083776 link_bytes=267083776 token_bytes=458899456 weight_movement_bytes=37748736 total_nvlink_bytes=496648192 weight_to_token_ratio=0.082259 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.559065328e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232194048 dispatch_meta_bytes=229376 combine_payload_bytes=232194048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232423424 ingress_bytes=288817152 link_bytes=288817152 token_bytes=464617472 weight_movement_bytes=56623104 total_nvlink_bytes=521240576 weight_to_token_ratio=0.121870 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.767303844e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=227155968 dispatch_meta_bytes=229600 combine_payload_bytes=227155968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227385568 ingress_bytes=264904704 link_bytes=264904704 token_bytes=454541536 weight_movement_bytes=37748736 total_nvlink_bytes=492290272 weight_to_token_ratio=0.083048 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.538186532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 24] payload_rows_per_dst=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 14, 0, 0, 56] combine_rows_per_src=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229376 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234323968 ingress_bytes=252968960 link_bytes=252968960 token_bytes=468418560 weight_movement_bytes=18874368 total_nvlink_bytes=487292928 weight_to_token_ratio=0.040294 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423824106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234815488 ingress_bytes=253460480 link_bytes=253460480 token_bytes=469401600 weight_movement_bytes=18874368 total_nvlink_bytes=488275968 weight_to_token_ratio=0.040209 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428533609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64556 payload_rows_remote=56364 dup_rows=980 dup_rows_remote=980 dedup_ratio=0.014954 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7212, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 958, 0, 0, 22, 0] combine_rows_per_src=[7212, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65220 payload_rows_remote=57028 dup_rows=316 dup_rows_remote=310 dedup_ratio=0.004822 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 4094, 2, 32, 0, 0, 0, 0, 119] payload_rows_per_dst=[8188, 7888, 8192, 8192, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 4, 0, 0, 6, 0, 0, 306] combine_rows_per_src=[8188, 7888, 8192, 8192, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59938 payload_rows_remote=51746 dup_rows=5598 dup_rows_remote=76 dedup_ratio=0.085419 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2856, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 2742, 8192, 8192, 8044, 8192] dup_rows_per_dst=[5522, 2, 74, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 2742, 8192, 8192, 8044, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62246 payload_rows_remote=54528 dup_rows=3290 dup_rows_remote=684 dedup_ratio=0.050201 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 191] payload_rows_per_dst=[8192, 7718, 8192, 8192, 8192, 8192, 8192, 5376] dup_rows_per_dst=[0, 2606, 0, 0, 288, 68, 0, 328] combine_rows_per_src=[8192, 7718, 8192, 8192, 8192, 8192, 8192, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58658 payload_rows_remote=50530 dup_rows=6878 dup_rows_remote=6878 dedup_ratio=0.104950 router_tokens_per_expert=[0, 4065, 0, 0, 2, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 31, 0, 0, 0, 726, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 7, 0, 30, 0, 0, 4065, 0, 0, 447, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 30, 0, 0, 0, 4037, 31, 11, 0, 0, 0, 0, 2907] payload_rows_per_dst=[8130, 1514, 8180, 8192, 8192, 8128, 8130, 8192] dup_rows_per_dst=[4, 62, 2, 2, 968, 0, 0, 5840] combine_rows_per_src=[8130, 1514, 8180, 8192, 8192, 8128, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230866944 dispatch_meta_bytes=229376 combine_payload_bytes=230866944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231096320 ingress_bytes=249741312 link_bytes=249741312 token_bytes=461963264 weight_movement_bytes=18874368 total_nvlink_bytes=480837632 weight_to_token_ratio=0.040857 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392898371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60976 payload_rows_remote=52784 dup_rows=4560 dup_rows_remote=1004 dedup_ratio=0.069580 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 561] payload_rows_per_dst=[8192, 5916, 8192, 8192, 5908, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 938, 0, 66, 3556] combine_rows_per_src=[8192, 5916, 8192, 8192, 5908, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233586688 dispatch_meta_bytes=229352 combine_payload_bytes=233586688 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=233816040 ingress_bytes=327958528 link_bytes=327958528 token_bytes=467402728 weight_movement_bytes=94371840 total_nvlink_bytes=561774568 weight_to_token_ratio=0.201907 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.142337250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=211951616 dispatch_meta_bytes=207288 combine_payload_bytes=211951616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212158904 ingress_bytes=249700352 link_bytes=249700352 token_bytes=424110520 weight_movement_bytes=37748736 total_nvlink_bytes=461859256 weight_to_token_ratio=0.089007 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392505913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64278 payload_rows_remote=56086 dup_rows=1258 dup_rows_remote=1258 dedup_ratio=0.019196 router_tokens_per_expert=[0, 0, 0, 3484, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6968, 8192, 8192, 8192, 8192, 8158, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1258, 0, 0, 0, 0] combine_rows_per_src=[6968, 8192, 8192, 8192, 8192, 8158, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60334 payload_rows_remote=52142 dup_rows=5202 dup_rows_remote=5202 dedup_ratio=0.079376 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8160, 7892, 3322, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4894, 308, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8160, 7892, 3322, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=223346688 dispatch_meta_bytes=220848 combine_payload_bytes=223346688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223567536 ingress_bytes=242221056 link_bytes=242221056 token_bytes=446914224 weight_movement_bytes=18874368 total_nvlink_bytes=465788592 weight_to_token_ratio=0.042233 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.320842978e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=206970880 dispatch_meta_bytes=229632 combine_payload_bytes=206970880 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=207200512 ingress_bytes=244719616 link_bytes=244719616 token_bytes=414171392 weight_movement_bytes=37748736 total_nvlink_bytes=451920128 weight_to_token_ratio=0.091143 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.344782951e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=216203264 dispatch_meta_bytes=215152 combine_payload_bytes=216203264 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=216418416 ingress_bytes=291700736 link_bytes=291700736 token_bytes=432621680 weight_movement_bytes=75497472 total_nvlink_bytes=508119152 weight_to_token_ratio=0.174512 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.794932927e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=229728256 dispatch_meta_bytes=229376 combine_payload_bytes=229728256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229957632 ingress_bytes=248602624 link_bytes=248602624 token_bytes=459685888 weight_movement_bytes=18874368 total_nvlink_bytes=478560256 weight_to_token_ratio=0.041059 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381988023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=213573632 dispatch_meta_bytes=229376 combine_payload_bytes=213573632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213803008 ingress_bytes=232448000 link_bytes=232448000 token_bytes=427376640 weight_movement_bytes=18874368 total_nvlink_bytes=446251008 weight_to_token_ratio=0.044163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227202364e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65300 payload_rows_remote=57108 dup_rows=236 dup_rows_remote=236 dedup_ratio=0.003601 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 105] payload_rows_per_dst=[8192, 7956, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 232] combine_rows_per_src=[8192, 7956, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58036 payload_rows_remote=49926 dup_rows=7500 dup_rows_remote=7500 dedup_ratio=0.114441 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8110, 782, 8192, 8188] dup_rows_per_dst=[4, 0, 10, 0, 0, 0, 7486, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8110, 782, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57474 payload_rows_remote=49282 dup_rows=8062 dup_rows_remote=8002 dedup_ratio=0.123016 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 100] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8124, 8192, 8192, 200] dup_rows_per_dst=[60, 7890, 10, 0, 102, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8124, 8192, 8192, 200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233914368 dispatch_meta_bytes=229376 combine_payload_bytes=233914368 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234143744 ingress_bytes=271663104 link_bytes=271663104 token_bytes=468058112 weight_movement_bytes=37748736 total_nvlink_bytes=505806848 weight_to_token_ratio=0.080650 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602942196e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57344 payload_rows_remote=49152 dup_rows=8192 dup_rows_remote=8192 dedup_ratio=0.125000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2131, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 2, 8192] dup_rows_per_dst=[0, 0, 0, 0, 3214, 4978, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 2, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57752 payload_rows_remote=54980 dup_rows=7784 dup_rows_remote=7782 dedup_ratio=0.118774 router_tokens_per_expert=[0, 4065, 0, 0, 1, 0, 0, 0, 0, 31, 0, 0, 0, 0, 31, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4064, 0, 31, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 1355, 0, 1, 0, 0, 0, 0, 1, 4064, 0, 0, 31, 1, 0, 0, 0, 3783, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4064, 4065, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8130, 5952, 2772, 8192, 8130] dup_rows_per_dst=[64, 0, 144, 0, 0, 2, 7572, 2] combine_rows_per_src=[8192, 8192, 8192, 8130, 5952, 2772, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59022 payload_rows_remote=50830 dup_rows=6514 dup_rows_remote=6450 dedup_ratio=0.099396 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8124, 8192, 8038, 8192, 4164, 5928, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 66, 0, 0, 6384, 64] combine_rows_per_src=[8124, 8192, 8038, 8192, 4164, 5928, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=204496896 dispatch_meta_bytes=229704 combine_payload_bytes=204496896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204726600 ingress_bytes=223371264 link_bytes=223371264 token_bytes=409223496 weight_movement_bytes=18874368 total_nvlink_bytes=428097864 weight_to_token_ratio=0.046122 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.140233546e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63904 payload_rows_remote=55712 dup_rows=1632 dup_rows_remote=1632 dedup_ratio=0.024902 router_tokens_per_expert=[0, 4096, 3280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6560, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1570, 62, 0, 0] combine_rows_per_src=[6560, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=201859072 dispatch_meta_bytes=229136 combine_payload_bytes=201859072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202088208 ingress_bytes=239607808 link_bytes=239607808 token_bytes=403947280 weight_movement_bytes=37748736 total_nvlink_bytes=441696016 weight_to_token_ratio=0.093450 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.295804122e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=201326592 dispatch_meta_bytes=229376 combine_payload_bytes=201326592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201555968 ingress_bytes=220200960 link_bytes=220200960 token_bytes=402882560 weight_movement_bytes=18874368 total_nvlink_bytes=421756928 weight_to_token_ratio=0.046848 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109857253e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=225198080 dispatch_meta_bytes=251048 combine_payload_bytes=225198080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225449128 ingress_bytes=244072448 link_bytes=244072448 token_bytes=450647208 weight_movement_bytes=18874368 total_nvlink_bytes=469521576 weight_to_token_ratio=0.041883 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.338582106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=208199680 dispatch_meta_bytes=229120 combine_payload_bytes=208199680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=208428800 ingress_bytes=245948416 link_bytes=245948416 token_bytes=416628480 weight_movement_bytes=37748736 total_nvlink_bytes=454377216 weight_to_token_ratio=0.090605 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.356556708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=228196352 dispatch_meta_bytes=229376 combine_payload_bytes=228196352 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228425728 ingress_bytes=265945088 link_bytes=265945088 token_bytes=456622080 weight_movement_bytes=37748736 total_nvlink_bytes=494370816 weight_to_token_ratio=0.082670 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.548154980e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59542 payload_rows_remote=51350 dup_rows=5994 dup_rows_remote=5994 dedup_ratio=0.091461 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7842, 8192, 2548, 8192] dup_rows_per_dst=[0, 0, 0, 0, 3634, 2360, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7842, 8192, 2548, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 2, 4094, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 6, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=210329600 dispatch_meta_bytes=229376 combine_payload_bytes=210329600 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210558976 ingress_bytes=266952704 link_bytes=266952704 token_bytes=420888576 weight_movement_bytes=56623104 total_nvlink_bytes=477511680 weight_to_token_ratio=0.134532 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.557809461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57344 payload_rows_remote=49152 dup_rows=8192 dup_rows_remote=8192 dedup_ratio=0.125000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 0, 8192] dup_rows_per_dst=[0, 0, 0, 0, 110, 8082, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65428 payload_rows_remote=57256 dup_rows=108 dup_rows_remote=86 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4065, 0, 0, 4062, 0, 0, 31, 4065, 0, 0, 4065, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 31, 4054, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 30, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0] payload_rows_per_dst=[8192, 8186, 8172, 8192, 8192, 8172, 8192, 8130] dup_rows_per_dst=[0, 0, 0, 10, 0, 22, 74, 2] combine_rows_per_src=[8192, 8186, 8172, 8192, 8192, 8172, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57258 dup_rows=86 dup_rows_remote=22 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8108, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 20, 0, 0, 2, 0, 64] combine_rows_per_src=[8192, 8108, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65428 payload_rows_remote=57236 dup_rows=108 dup_rows_remote=108 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8084, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 108, 0, 0, 0, 0] combine_rows_per_src=[8084, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=201326592 dispatch_meta_bytes=229376 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=201555968 ingress_bytes=239075328 link_bytes=239075328 token_bytes=402882560 weight_movement_bytes=37748736 total_nvlink_bytes=440631296 weight_to_token_ratio=0.093697 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.290702160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=113246208 egress_bytes=235102200 ingress_bytes=366993408 link_bytes=366993408 token_bytes=469975032 weight_movement_bytes=132120576 total_nvlink_bytes=602095608 weight_to_token_ratio=0.281123 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.516350263e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229120 combine_payload_bytes=234528768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234757888 ingress_bytes=253403136 link_bytes=253403136 token_bytes=469286656 weight_movement_bytes=18874368 total_nvlink_bytes=488161024 weight_to_token_ratio=0.040219 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427984167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=229368 combine_payload_bytes=234520576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234749944 ingress_bytes=253394944 link_bytes=253394944 token_bytes=469270520 weight_movement_bytes=18874368 total_nvlink_bytes=488144888 weight_to_token_ratio=0.040221 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427905675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234438656 dispatch_meta_bytes=229376 combine_payload_bytes=234438656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234668032 ingress_bytes=253313024 link_bytes=253313024 token_bytes=469106688 weight_movement_bytes=18874368 total_nvlink_bytes=487981056 weight_to_token_ratio=0.040235 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427120758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=8 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 4, 4, 0, 4, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235102200 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975032 weight_movement_bytes=37748736 total_nvlink_bytes=507723768 weight_to_token_ratio=0.080321 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65444 payload_rows_remote=57252 dup_rows=92 dup_rows_remote=92 dedup_ratio=0.001404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8190, 8110, 8192] dup_rows_per_dst=[0, 92, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8190, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229360 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235061232 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893104 weight_movement_bytes=37748736 total_nvlink_bytes=507641840 weight_to_token_ratio=0.080335 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65402 payload_rows_remote=57210 dup_rows=134 dup_rows_remote=42 dedup_ratio=0.002045 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8126, 8126, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 38, 2, 0, 2, 92] combine_rows_per_src=[8192, 8126, 8126, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65322 payload_rows_remote=57218 dup_rows=214 dup_rows_remote=214 dedup_ratio=0.003265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 0, 0, 31, 0, 4065, 0, 4065, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 31, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 4064, 0, 0, 0, 31, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 8192, 8192, 8192, 8190, 8104, 8130, 8192] dup_rows_per_dst=[0, 62, 62, 0, 62, 0, 28, 0] combine_rows_per_src=[8130, 8192, 8192, 8192, 8190, 8104, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65014 payload_rows_remote=56822 dup_rows=522 dup_rows_remote=522 dedup_ratio=0.007965 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 84, 0, 0, 0, 0, 161] payload_rows_per_dst=[8192, 7670, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 32, 0, 0, 490] combine_rows_per_src=[8192, 7670, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234504192 dispatch_meta_bytes=229376 combine_payload_bytes=234504192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234733568 ingress_bytes=253378560 link_bytes=253378560 token_bytes=469237760 weight_movement_bytes=18874368 total_nvlink_bytes=488112128 weight_to_token_ratio=0.040223 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427748692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234332160 dispatch_meta_bytes=229008 combine_payload_bytes=234332160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234561168 ingress_bytes=290955264 link_bytes=290955264 token_bytes=468893328 weight_movement_bytes=56623104 total_nvlink_bytes=525516432 weight_to_token_ratio=0.120759 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787790181e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234364928 dispatch_meta_bytes=229728 combine_payload_bytes=234364928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234594656 ingress_bytes=253239296 link_bytes=253239296 token_bytes=468959584 weight_movement_bytes=18874368 total_nvlink_bytes=487833952 weight_to_token_ratio=0.040247 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426414333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=232742912 dispatch_meta_bytes=229376 combine_payload_bytes=232742912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232972288 ingress_bytes=251617280 link_bytes=251617280 token_bytes=465715200 weight_movement_bytes=18874368 total_nvlink_bytes=484589568 weight_to_token_ratio=0.040528 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410872974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975040 weight_movement_bytes=56623104 total_nvlink_bytes=526598144 weight_to_token_ratio=0.120481 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=20 dedup_ratio=0.000366 router_tokens_per_expert=[0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4086, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8176] dup_rows_per_dst=[0, 0, 16, 0, 4, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229360 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012080 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794800 weight_movement_bytes=18874368 total_nvlink_bytes=488669168 weight_to_token_ratio=0.040176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=172 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8136, 8126, 8192, 8168, 8100] dup_rows_per_dst=[66, 0, 0, 24, 0, 92, 56, 0] combine_rows_per_src=[8192, 8192, 8192, 8136, 8126, 8192, 8168, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57206 dup_rows=138 dup_rows_remote=66 dedup_ratio=0.002106 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8126, 8192, 8192, 8192, 8192, 8120, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 66, 0, 72] combine_rows_per_src=[8192, 8126, 8192, 8192, 8192, 8192, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65402 payload_rows_remote=57272 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.002045 router_tokens_per_expert=[0, 4063, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4065, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4065, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 31, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 2, 0, 4065, 4063, 1, 0, 31, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8190, 8130, 8190, 8192, 8130, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 66, 0, 66, 2] combine_rows_per_src=[8186, 8190, 8130, 8190, 8192, 8130, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60390 payload_rows_remote=52198 dup_rows=5146 dup_rows_remote=0 dedup_ratio=0.078522 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7968, 3270, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 5146, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7968, 3270, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57344 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8102, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 90, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8102, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975040 weight_movement_bytes=56623104 total_nvlink_bytes=526598144 weight_to_token_ratio=0.120481 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229112 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234135288 ingress_bytes=252780544 link_bytes=252780544 token_bytes=468041464 weight_movement_bytes=18874368 total_nvlink_bytes=486915832 weight_to_token_ratio=0.040326 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422018797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234315776 dispatch_meta_bytes=229088 combine_payload_bytes=234315776 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234544864 ingress_bytes=309813248 link_bytes=309813248 token_bytes=468860640 weight_movement_bytes=75497472 total_nvlink_bytes=544358112 weight_to_token_ratio=0.161023 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968478105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229624 combine_payload_bytes=234586112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234815736 ingress_bytes=272334848 link_bytes=272334848 token_bytes=469401848 weight_movement_bytes=37748736 total_nvlink_bytes=507150584 weight_to_token_ratio=0.080419 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609378516e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=213803008 dispatch_meta_bytes=208792 combine_payload_bytes=213803008 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=214011800 ingress_bytes=270426112 link_bytes=270426112 token_bytes=427814808 weight_movement_bytes=56623104 total_nvlink_bytes=484437912 weight_to_token_ratio=0.132354 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591089947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229736 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110760 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991784 weight_movement_bytes=18874368 total_nvlink_bytes=488866152 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65438 payload_rows_remote=57246 dup_rows=98 dup_rows_remote=94 dedup_ratio=0.001495 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4094, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8188, 8116, 8182, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 2, 0, 0, 4, 92, 0, 0] combine_rows_per_src=[8188, 8188, 8116, 8182, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234479616 dispatch_meta_bytes=229360 combine_payload_bytes=234479616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234708976 ingress_bytes=253353984 link_bytes=253353984 token_bytes=469188592 weight_movement_bytes=18874368 total_nvlink_bytes=488062960 weight_to_token_ratio=0.040228 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427513217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65248 payload_rows_remote=57058 dup_rows=288 dup_rows_remote=288 dedup_ratio=0.004395 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8176, 7924, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 286, 0, 0] combine_rows_per_src=[8192, 8190, 8176, 7924, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65410 payload_rows_remote=57218 dup_rows=126 dup_rows_remote=120 dedup_ratio=0.001923 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8122, 8192, 8192, 8136] dup_rows_per_dst=[6, 68, 48, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8122, 8192, 8192, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65444 payload_rows_remote=57252 dup_rows=92 dup_rows_remote=26 dedup_ratio=0.001404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8126, 8192, 8166, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 0, 66] combine_rows_per_src=[8192, 8126, 8192, 8166, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63974 payload_rows_remote=55782 dup_rows=1562 dup_rows_remote=76 dedup_ratio=0.023834 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 6, 0, 4065, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 30, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 3589, 0, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 31, 4059, 0, 0, 0, 31, 0, 103, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 8130, 7744, 7264, 8192, 8192, 8192, 8130] dup_rows_per_dst=[0, 10, 0, 2, 62, 1486, 2, 0] combine_rows_per_src=[8130, 8130, 7744, 7264, 8192, 8192, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64114 payload_rows_remote=55922 dup_rows=1422 dup_rows_remote=1422 dedup_ratio=0.021698 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6770, 8192, 8192] dup_rows_per_dst=[0, 1422, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6770, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=233709568 dispatch_meta_bytes=229384 combine_payload_bytes=233709568 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233938952 ingress_bytes=309207040 link_bytes=309207040 token_bytes=467648520 weight_movement_bytes=75497472 total_nvlink_bytes=543145992 weight_to_token_ratio=0.161441 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.962669718e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65374 payload_rows_remote=57198 dup_rows=162 dup_rows_remote=162 dedup_ratio=0.002472 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8176, 8048, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 162, 0, 0] combine_rows_per_src=[8192, 8190, 8176, 8048, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234364928 dispatch_meta_bytes=229352 combine_payload_bytes=234364928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234594280 ingress_bytes=253239296 link_bytes=253239296 token_bytes=468959208 weight_movement_bytes=18874368 total_nvlink_bytes=487833576 weight_to_token_ratio=0.040247 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426414333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234504192 dispatch_meta_bytes=229112 combine_payload_bytes=234504192 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234733304 ingress_bytes=272252928 link_bytes=272252928 token_bytes=469237496 weight_movement_bytes=37748736 total_nvlink_bytes=506986232 weight_to_token_ratio=0.080447 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608593599e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=228483072 dispatch_meta_bytes=223432 combine_payload_bytes=228483072 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=228706504 ingress_bytes=303980544 link_bytes=303980544 token_bytes=457189576 weight_movement_bytes=75497472 total_nvlink_bytes=532687048 weight_to_token_ratio=0.165134 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.912592005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64674 payload_rows_remote=56482 dup_rows=862 dup_rows_remote=862 dedup_ratio=0.013153 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 7334, 8192, 8192] dup_rows_per_dst=[0, 854, 0, 4, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 7334, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=229056512 dispatch_meta_bytes=229376 combine_payload_bytes=229056512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229285888 ingress_bytes=285679616 link_bytes=285679616 token_bytes=458342400 weight_movement_bytes=56623104 total_nvlink_bytes=514965504 weight_to_token_ratio=0.123539 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.737241517e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229440 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512448 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795456 weight_movement_bytes=18874368 total_nvlink_bytes=487669824 weight_to_token_ratio=0.040261 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=231350272 dispatch_meta_bytes=229376 combine_payload_bytes=231350272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231579648 ingress_bytes=250224640 link_bytes=250224640 token_bytes=462929920 weight_movement_bytes=18874368 total_nvlink_bytes=481804288 weight_to_token_ratio=0.040772 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397529382e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64806 payload_rows_remote=56614 dup_rows=730 dup_rows_remote=0 dedup_ratio=0.011139 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7462, 8192, 8192] dup_rows_per_dst=[0, 730, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7462, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63092 payload_rows_remote=54900 dup_rows=2444 dup_rows_remote=2378 dedup_ratio=0.037292 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8076, 8132, 8126, 5992, 8192, 8192] dup_rows_per_dst=[66, 126, 676, 1570, 0, 6, 0, 0] combine_rows_per_src=[8192, 8190, 8076, 8132, 8126, 5992, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59118 payload_rows_remote=50926 dup_rows=6418 dup_rows_remote=6418 dedup_ratio=0.097931 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8086, 1880, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 106, 0, 6312, 0, 0] combine_rows_per_src=[8086, 1880, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64540 payload_rows_remote=57216 dup_rows=996 dup_rows_remote=996 dedup_ratio=0.015198 router_tokens_per_expert=[0, 0, 0, 31, 0, 4, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4064, 0, 0, 4065, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 31, 0, 0, 0, 0, 31, 4061, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4062, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 31, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8076, 8184, 8192, 7324, 8188, 8192] dup_rows_per_dst=[106, 766, 0, 62, 12, 0, 50, 0] combine_rows_per_src=[8192, 8192, 8076, 8184, 8192, 7324, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=231890944 dispatch_meta_bytes=226456 combine_payload_bytes=231890944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232117400 ingress_bytes=250765312 link_bytes=250765312 token_bytes=464008344 weight_movement_bytes=18874368 total_nvlink_bytes=482882712 weight_to_token_ratio=0.040677 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402709835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63202 payload_rows_remote=55010 dup_rows=2334 dup_rows_remote=2334 dedup_ratio=0.035614 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 5858, 8192, 8192] dup_rows_per_dst=[0, 2334, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 5858, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=224870400 dispatch_meta_bytes=229112 combine_payload_bytes=224870400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225099512 ingress_bytes=243744768 link_bytes=243744768 token_bytes=449969912 weight_movement_bytes=18874368 total_nvlink_bytes=468844280 weight_to_token_ratio=0.041946 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.335442437e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 4094, 0, 2, 0, 1, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8188, 8192, 8188, 8192, 8188] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 12, 0] combine_rows_per_src=[8186, 8192, 8192, 8188, 8192, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=208592896 dispatch_meta_bytes=229376 combine_payload_bytes=208592896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208822272 ingress_bytes=227467264 link_bytes=227467264 token_bytes=417415168 weight_movement_bytes=18874368 total_nvlink_bytes=436289536 weight_to_token_ratio=0.045217 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.179479403e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229360 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094000 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958640 weight_movement_bytes=18874368 total_nvlink_bytes=488833008 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234356736 dispatch_meta_bytes=232848 combine_payload_bytes=234356736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234589584 ingress_bytes=253231104 link_bytes=253231104 token_bytes=468946320 weight_movement_bytes=18874368 total_nvlink_bytes=487820688 weight_to_token_ratio=0.040248 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426335841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=225320960 dispatch_meta_bytes=229376 combine_payload_bytes=225320960 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=225550336 ingress_bytes=319692800 link_bytes=319692800 token_bytes=450871296 weight_movement_bytes=94371840 total_nvlink_bytes=545243136 weight_to_token_ratio=0.209310 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.063139111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036672 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843968 weight_movement_bytes=18874368 total_nvlink_bytes=488718336 weight_to_token_ratio=0.040172 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64542 payload_rows_remote=56350 dup_rows=994 dup_rows_remote=994 dedup_ratio=0.015167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8028, 8180, 7424, 8142, 8192] dup_rows_per_dst=[0, 768, 226, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8028, 8180, 7424, 8142, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57198 dup_rows=146 dup_rows_remote=90 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8048, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 28, 2, 60, 0, 56] combine_rows_per_src=[8192, 8190, 8048, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65220 payload_rows_remote=57090 dup_rows=316 dup_rows_remote=310 dedup_ratio=0.004822 router_tokens_per_expert=[0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4018, 1, 0, 0, 0, 0, 0, 0, 0, 4065, 31, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4062, 0, 4065, 0, 31, 0, 40, 0, 0, 0, 4065, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 14, 31, 0, 31, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[8072, 8192, 8186, 8128, 8190, 8130, 8192, 8130] dup_rows_per_dst=[2, 2, 140, 0, 0, 6, 152, 14] combine_rows_per_src=[8072, 8192, 8186, 8128, 8190, 8130, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 4, 4, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=230809600 dispatch_meta_bytes=229376 combine_payload_bytes=230809600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231038976 ingress_bytes=249683968 link_bytes=249683968 token_bytes=461848576 weight_movement_bytes=18874368 total_nvlink_bytes=480722944 weight_to_token_ratio=0.040867 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392348929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229152 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512160 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795168 weight_movement_bytes=18874368 total_nvlink_bytes=487669536 weight_to_token_ratio=0.040261 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233840640 dispatch_meta_bytes=229600 combine_payload_bytes=233840640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234070240 ingress_bytes=271589376 link_bytes=271589376 token_bytes=467910880 weight_movement_bytes=37748736 total_nvlink_bytes=505659616 weight_to_token_ratio=0.080675 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602235771e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235102200 ingress_bytes=310370304 link_bytes=310370304 token_bytes=469975032 weight_movement_bytes=75497472 total_nvlink_bytes=545472504 weight_to_token_ratio=0.160641 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973815541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229360 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235077616 ingress_bytes=310345728 link_bytes=310345728 token_bytes=469925872 weight_movement_bytes=75497472 total_nvlink_bytes=545423344 weight_to_token_ratio=0.160658 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973580066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65372 payload_rows_remote=57180 dup_rows=164 dup_rows_remote=40 dedup_ratio=0.002502 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8028, 8192] dup_rows_per_dst=[124, 0, 0, 40, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8028, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57292 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8144, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 52, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8144, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=56 dedup_ratio=0.001770 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8128, 8142, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 56, 60] combine_rows_per_src=[8192, 8192, 8192, 8190, 8128, 8142, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65396 payload_rows_remote=57204 dup_rows=140 dup_rows_remote=92 dedup_ratio=0.002136 router_tokens_per_expert=[0, 4065, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 31, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4056, 4065, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 8, 0] payload_rows_per_dst=[8130, 8190, 8190, 8192, 8192, 8192, 8174, 8136] dup_rows_per_dst=[28, 0, 0, 2, 62, 48, 0, 0] combine_rows_per_src=[8130, 8190, 8190, 8192, 8192, 8192, 8174, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8172, 8192] dup_rows_per_dst=[0, 24, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57326 dup_rows=20 dup_rows_remote=16 dedup_ratio=0.000305 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4093, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4094, 2, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8190, 8188, 8182, 8192] dup_rows_per_dst=[0, 8, 0, 4, 4, 0, 0, 4] combine_rows_per_src=[8192, 8188, 8192, 8192, 8190, 8188, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=228880 combine_payload_bytes=234209280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234438160 ingress_bytes=253083648 link_bytes=253083648 token_bytes=468647440 weight_movement_bytes=18874368 total_nvlink_bytes=487521808 weight_to_token_ratio=0.040274 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424922990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229376 combine_payload_bytes=234668032 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234897408 ingress_bytes=234668032 link_bytes=234897408 token_bytes=469565440 weight_movement_bytes=0 total_nvlink_bytes=469565440 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250671387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=229136 combine_payload_bytes=234405888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234635024 ingress_bytes=253280256 link_bytes=253280256 token_bytes=469040912 weight_movement_bytes=18874368 total_nvlink_bytes=487915280 weight_to_token_ratio=0.040240 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426806791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234307584 dispatch_meta_bytes=229184 combine_payload_bytes=234307584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234536768 ingress_bytes=253181952 link_bytes=253181952 token_bytes=468844352 weight_movement_bytes=18874368 total_nvlink_bytes=487718720 weight_to_token_ratio=0.040257 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425864891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=0 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] dup_rows_per_dst=[0, 16, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229368 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036664 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843960 weight_movement_bytes=18874368 total_nvlink_bytes=488718328 weight_to_token_ratio=0.040172 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229312 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235044800 ingress_bytes=329187328 link_bytes=329187328 token_bytes=469860288 weight_movement_bytes=94371840 total_nvlink_bytes=564232128 weight_to_token_ratio=0.200851 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154111007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57302 dup_rows=42 dup_rows_remote=4 dedup_ratio=0.000641 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8154, 8192] dup_rows_per_dst=[38, 2, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8154, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62750 payload_rows_remote=54558 dup_rows=2786 dup_rows_remote=2786 dedup_ratio=0.042511 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6570, 8086, 8192, 8192, 7134, 8192, 8192] dup_rows_per_dst=[0, 1060, 0, 0, 0, 1726, 0, 0] combine_rows_per_src=[8192, 6570, 8086, 8192, 8192, 7134, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65100 payload_rows_remote=56908 dup_rows=436 dup_rows_remote=370 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7844, 8192, 8126, 8182, 8192, 8180, 8192] dup_rows_per_dst=[0, 22, 0, 0, 0, 56, 292, 66] combine_rows_per_src=[8192, 7844, 8192, 8126, 8182, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65294 payload_rows_remote=57160 dup_rows=242 dup_rows_remote=236 dedup_ratio=0.003693 router_tokens_per_expert=[31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4059, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 31, 0, 1, 0, 0, 0, 4046, 0, 31, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 31, 4064, 1, 0, 0, 0, 0, 4065, 29, 4064, 8, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8120, 8192, 8190, 8154, 8134, 8120, 8192] dup_rows_per_dst=[0, 38, 0, 64, 60, 6, 0, 74] combine_rows_per_src=[8192, 8120, 8192, 8190, 8154, 8134, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8170, 8192, 8144, 8192] dup_rows_per_dst=[0, 70, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8170, 8192, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229224 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234938216 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469647208 weight_movement_bytes=18874368 total_nvlink_bytes=488521576 weight_to_token_ratio=0.040188 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=223469568 dispatch_meta_bytes=229376 combine_payload_bytes=223469568 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=223698944 ingress_bytes=242343936 link_bytes=242343936 token_bytes=447168512 weight_movement_bytes=18874368 total_nvlink_bytes=466042880 weight_to_token_ratio=0.042209 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.322020354e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233095168 dispatch_meta_bytes=229112 combine_payload_bytes=233095168 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=233324280 ingress_bytes=308592640 link_bytes=308592640 token_bytes=466419448 weight_movement_bytes=75497472 total_nvlink_bytes=541916920 weight_to_token_ratio=0.161866 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.956782839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=229584 combine_payload_bytes=234127360 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234356944 ingress_bytes=271876096 link_bytes=271876096 token_bytes=468484304 weight_movement_bytes=37748736 total_nvlink_bytes=506233040 weight_to_token_ratio=0.080576 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604982980e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229376 combine_payload_bytes=234594304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234823680 ingress_bytes=253468672 link_bytes=253468672 token_bytes=469417984 weight_movement_bytes=18874368 total_nvlink_bytes=488292352 weight_to_token_ratio=0.040208 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428612101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65218 payload_rows_remote=57026 dup_rows=318 dup_rows_remote=318 dedup_ratio=0.004852 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8124, 8192, 8086, 8048] dup_rows_per_dst=[0, 222, 0, 0, 96, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8124, 8192, 8086, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 72, 0] combine_rows_per_src=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65338 payload_rows_remote=57192 dup_rows=198 dup_rows_remote=186 dedup_ratio=0.003021 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 3, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 2, 0, 0, 0, 4065, 0, 0, 31, 0, 0, 31, 0, 4060, 0, 0, 31, 0, 0, 0, 0, 25, 0, 4065, 0, 0] payload_rows_per_dst=[8116, 8192, 8130, 8190, 8190, 8146, 8192, 8182] dup_rows_per_dst=[0, 68, 2, 0, 0, 12, 4, 112] combine_rows_per_src=[8116, 8192, 8130, 8190, 8190, 8146, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64718 payload_rows_remote=57344 dup_rows=818 dup_rows_remote=818 dedup_ratio=0.012482 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7374, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 818, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7374, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64958 payload_rows_remote=56766 dup_rows=578 dup_rows_remote=578 dedup_ratio=0.008820 router_tokens_per_expert=[2, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 7622, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 566, 0, 0, 0, 0, 4, 4] combine_rows_per_src=[8192, 8192, 8188, 7622, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233578496 dispatch_meta_bytes=229376 combine_payload_bytes=233578496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233807872 ingress_bytes=252452864 link_bytes=252452864 token_bytes=467386368 weight_movement_bytes=18874368 total_nvlink_bytes=486260736 weight_to_token_ratio=0.040383 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418879128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234815488 ingress_bytes=328957952 link_bytes=328957952 token_bytes=469401600 weight_movement_bytes=94371840 total_nvlink_bytes=563773440 weight_to_token_ratio=0.201047 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.151913239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234258432 dispatch_meta_bytes=229512 combine_payload_bytes=234258432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234487944 ingress_bytes=253132800 link_bytes=253132800 token_bytes=468746376 weight_movement_bytes=18874368 total_nvlink_bytes=487620744 weight_to_token_ratio=0.040266 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425393940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=232648 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235113672 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469994696 weight_movement_bytes=18874368 total_nvlink_bytes=488869064 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63790 payload_rows_remote=55598 dup_rows=1746 dup_rows_remote=0 dedup_ratio=0.026642 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6446, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1746, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 6446, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=232513536 dispatch_meta_bytes=229376 combine_payload_bytes=232513536 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232742912 ingress_bytes=270262272 link_bytes=270262272 token_bytes=465256448 weight_movement_bytes=37748736 total_nvlink_bytes=503005184 weight_to_token_ratio=0.081135 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589520113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=227729408 dispatch_meta_bytes=222392 combine_payload_bytes=227729408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227951800 ingress_bytes=246603776 link_bytes=246603776 token_bytes=455681208 weight_movement_bytes=18874368 total_nvlink_bytes=474555576 weight_to_token_ratio=0.041420 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362836045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65350 payload_rows_remote=57158 dup_rows=186 dup_rows_remote=106 dedup_ratio=0.002838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8116, 8082] dup_rows_per_dst=[80, 0, 64, 2, 2, 38, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8116, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59184 payload_rows_remote=50992 dup_rows=6352 dup_rows_remote=6250 dedup_ratio=0.096924 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8090, 2502, 8192, 7632, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 286, 0, 0, 0, 5964, 0, 102] combine_rows_per_src=[8090, 2502, 8192, 7632, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65070 payload_rows_remote=56942 dup_rows=466 dup_rows_remote=466 dedup_ratio=0.007111 router_tokens_per_expert=[31, 0, 0, 4065, 0, 5, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4059, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4064, 4064, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 0, 4065, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0] payload_rows_per_dst=[8192, 8192, 8130, 7862, 8182, 8128, 8192, 8192] dup_rows_per_dst=[50, 298, 2, 0, 0, 0, 54, 62] combine_rows_per_src=[8192, 8192, 8130, 7862, 8182, 8128, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65258 payload_rows_remote=57066 dup_rows=278 dup_rows_remote=278 dedup_ratio=0.004242 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7914, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 278, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7914, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=8 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 4, 4, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234119168 dispatch_meta_bytes=229056 combine_payload_bytes=234119168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234348224 ingress_bytes=252993536 link_bytes=252993536 token_bytes=468467392 weight_movement_bytes=18874368 total_nvlink_bytes=487341760 weight_to_token_ratio=0.040290 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424059581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233742336 dispatch_meta_bytes=229376 combine_payload_bytes=233742336 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233971712 ingress_bytes=271491072 link_bytes=271491072 token_bytes=467714048 weight_movement_bytes=37748736 total_nvlink_bytes=505462784 weight_to_token_ratio=0.080709 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601293870e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=208863232 dispatch_meta_bytes=228968 combine_payload_bytes=208863232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209092200 ingress_bytes=246611968 link_bytes=246611968 token_bytes=417955432 weight_movement_bytes=37748736 total_nvlink_bytes=455704168 weight_to_token_ratio=0.090318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.362914537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233234432 dispatch_meta_bytes=229632 combine_payload_bytes=233234432 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233464064 ingress_bytes=327606272 link_bytes=327606272 token_bytes=466698496 weight_movement_bytes=94371840 total_nvlink_bytes=561070336 weight_to_token_ratio=0.202212 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.138962106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229360 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061232 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893104 weight_movement_bytes=18874368 total_nvlink_bytes=488767472 weight_to_token_ratio=0.040167 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57344 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] dup_rows_per_dst=[0, 24, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59254 payload_rows_remote=51062 dup_rows=6282 dup_rows_remote=6234 dedup_ratio=0.095856 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8032, 8192, 2296, 8134, 8024] dup_rows_per_dst=[48, 1454, 64, 4706, 10, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8032, 8192, 2296, 8134, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65418 payload_rows_remote=57226 dup_rows=118 dup_rows_remote=52 dedup_ratio=0.001801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8144, 8188, 8192, 8126, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 52, 66] combine_rows_per_src=[8192, 8144, 8188, 8192, 8126, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57154 dup_rows=190 dup_rows_remote=176 dedup_ratio=0.002899 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 31, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 31, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 1, 0, 31, 0, 4064, 0, 0, 0, 0, 4065, 0, 6, 0, 0, 0, 1, 1, 4065, 0, 0, 4063, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4064] payload_rows_per_dst=[8130, 8192, 8192, 8130, 8192, 8192, 8188, 8130] dup_rows_per_dst=[4, 48, 62, 0, 62, 14, 0, 0] combine_rows_per_src=[8130, 8192, 8192, 8130, 8192, 8192, 8188, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57332 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8188, 8188, 8192, 8188, 8192] dup_rows_per_dst=[4, 8, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8188, 8188, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229472 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110496 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991520 weight_movement_bytes=18874368 total_nvlink_bytes=488865888 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229360 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094000 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958640 weight_movement_bytes=18874368 total_nvlink_bytes=488833008 weight_to_token_ratio=0.040162 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=209149952 dispatch_meta_bytes=229184 combine_payload_bytes=209149952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209379136 ingress_bytes=246898688 link_bytes=246898688 token_bytes=418529088 weight_movement_bytes=37748736 total_nvlink_bytes=456277824 weight_to_token_ratio=0.090194 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365661747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234397696 dispatch_meta_bytes=229112 combine_payload_bytes=234397696 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234626808 ingress_bytes=253272064 link_bytes=253272064 token_bytes=469024504 weight_movement_bytes=18874368 total_nvlink_bytes=487898872 weight_to_token_ratio=0.040242 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426728300e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234102784 dispatch_meta_bytes=229320 combine_payload_bytes=234102784 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234332104 ingress_bytes=290725888 link_bytes=290725888 token_bytes=468434888 weight_movement_bytes=56623104 total_nvlink_bytes=525057992 weight_to_token_ratio=0.120877 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.785592413e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229392 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061264 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893136 weight_movement_bytes=18874368 total_nvlink_bytes=488767504 weight_to_token_ratio=0.040167 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65408 payload_rows_remote=57216 dup_rows=128 dup_rows_remote=66 dedup_ratio=0.001953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8130, 8192, 8192, 8174, 8144, 8192] dup_rows_per_dst=[62, 0, 0, 66, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8130, 8192, 8192, 8174, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57316 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8164, 8192] dup_rows_per_dst=[0, 0, 0, 28, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65086 payload_rows_remote=56894 dup_rows=450 dup_rows_remote=450 dedup_ratio=0.006866 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 7992, 8012, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 70, 380, 0] combine_rows_per_src=[8122, 7992, 8012, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65202 payload_rows_remote=57012 dup_rows=334 dup_rows_remote=334 dedup_ratio=0.005096 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 31, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4065, 0, 0, 29, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 31, 4065, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8126, 8130, 8132, 8190, 8048, 8192] dup_rows_per_dst=[66, 120, 0, 84, 0, 0, 0, 64] combine_rows_per_src=[8192, 8192, 8126, 8130, 8132, 8190, 8048, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234356736 dispatch_meta_bytes=229128 combine_payload_bytes=234356736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234585864 ingress_bytes=253231104 link_bytes=253231104 token_bytes=468942600 weight_movement_bytes=18874368 total_nvlink_bytes=487816968 weight_to_token_ratio=0.040249 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426335841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229376 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234995712 ingress_bytes=291389440 link_bytes=291389440 token_bytes=469762048 weight_movement_bytes=56623104 total_nvlink_bytes=526385152 weight_to_token_ratio=0.120536 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791950242e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229376 combine_payload_bytes=233037824 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233267200 ingress_bytes=270786560 link_bytes=270786560 token_bytes=466305024 weight_movement_bytes=37748736 total_nvlink_bytes=504053760 weight_to_token_ratio=0.080953 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594543583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233521152 dispatch_meta_bytes=229384 combine_payload_bytes=233521152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233750536 ingress_bytes=290144256 link_bytes=290144256 token_bytes=467271688 weight_movement_bytes=56623104 total_nvlink_bytes=523894792 weight_to_token_ratio=0.121178 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780019501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235102208 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975040 weight_movement_bytes=56623104 total_nvlink_bytes=526598144 weight_to_token_ratio=0.120481 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=22001266224 prefetch_bytes=792723456 reduce_bytes=849346560 weight_movement_bytes=1642070016 egress_bytes=11006135856 ingress_bytes=12637200384 link_bytes=12637200384 total_nvlink_bytes=23643336240 weight_to_token_ratio=0.074635 prefetch_share_of_ingress=0.062729 dedup_ratio=0.023366 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=45 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100116 gemm_inner_pad_upper_bound=33782 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.522571587e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21975598976 prefetch_bytes=773849088 reduce_bytes=868220928 weight_movement_bytes=1642070016 egress_bytes=10993321856 ingress_bytes=12624347136 link_bytes=12624347136 total_nvlink_bytes=23617668992 weight_to_token_ratio=0.074722 prefetch_share_of_ingress=0.061298 dedup_ratio=0.027418 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=46 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.099682 gemm_inner_pad_upper_bound=34798 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.520005889e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21657812752 prefetch_bytes=868220928 reduce_bytes=1000341504 weight_movement_bytes=1868562432 egress_bytes=10834362128 ingress_bytes=12692013056 link_bytes=12692242432 total_nvlink_bytes=23526375184 weight_to_token_ratio=0.086277 prefetch_share_of_ingress=0.068407 dedup_ratio=0.036451 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=53 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100333 gemm_inner_pad_upper_bound=33909 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.533558792e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65292 payload_rows_remote=57100 dup_rows=244 dup_rows_remote=244 dedup_ratio=0.003723 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7948, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 244, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7948, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64952 payload_rows_remote=56760 dup_rows=584 dup_rows_remote=584 dedup_ratio=0.008911 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7944, 8192, 8124, 7992, 8124] dup_rows_per_dst=[0, 72, 250, 198, 64, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7944, 8192, 8124, 7992, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57270 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8118, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 66, 0] combine_rows_per_src=[8192, 8118, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=144 dedup_ratio=0.002197 router_tokens_per_expert=[0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4061, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4065, 0, 4065, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 31, 0, 0, 31, 0, 1, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 25, 31, 0, 0, 0, 4, 0, 0, 31, 0, 0] payload_rows_per_dst=[8192, 8130, 8184, 8180, 8192, 8192, 8130, 8192] dup_rows_per_dst=[0, 0, 0, 12, 0, 0, 0, 132] combine_rows_per_src=[8192, 8130, 8184, 8180, 8192, 8192, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233881600 dispatch_meta_bytes=229376 combine_payload_bytes=233881600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234110976 ingress_bytes=252755968 link_bytes=252755968 token_bytes=467992576 weight_movement_bytes=18874368 total_nvlink_bytes=486866944 weight_to_token_ratio=0.040330 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421783322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=232488960 dispatch_meta_bytes=229376 combine_payload_bytes=232488960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232718336 ingress_bytes=251363328 link_bytes=251363328 token_bytes=465207296 weight_movement_bytes=18874368 total_nvlink_bytes=484081664 weight_to_token_ratio=0.040572 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408439730e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229376 combine_payload_bytes=234577920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234807296 ingress_bytes=253452288 link_bytes=253452288 token_bytes=469385216 weight_movement_bytes=18874368 total_nvlink_bytes=488259584 weight_to_token_ratio=0.040211 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428455117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229376 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234520576 ingress_bytes=253165568 link_bytes=253165568 token_bytes=468811776 weight_movement_bytes=18874368 total_nvlink_bytes=487686144 weight_to_token_ratio=0.040260 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425707907e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21958912064 prefetch_bytes=981467136 reduce_bytes=924844032 weight_movement_bytes=1906311168 egress_bytes=10984974400 ingress_bytes=12880248832 link_bytes=12880248832 total_nvlink_bytes=23865223232 weight_to_token_ratio=0.086813 prefetch_share_of_ingress=0.076199 dedup_ratio=0.028090 prefetch_slots=104 unused_prefetch_slots=1432 slot_utilization=0.067708 reduce_slots=49 remote_demand=52 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.099320 gemm_inner_pad_upper_bound=35179 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.571087642e-04 prefetch_bytes_per_gemm_flop=1.959153164e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21314694224 prefetch_bytes=849346560 reduce_bytes=924844032 weight_movement_bytes=1774190592 egress_bytes=10662792272 ingress_bytes=12426092544 link_bytes=12426092544 total_nvlink_bytes=23088884816 weight_to_token_ratio=0.083238 prefetch_share_of_ingress=0.068352 dedup_ratio=0.048820 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=49 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.099537 gemm_inner_pad_upper_bound=35306 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.480431349e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21761297736 prefetch_bytes=887095296 reduce_bytes=811597824 weight_movement_bytes=1698693120 egress_bytes=10886147400 ingress_bytes=12573843456 link_bytes=12573843456 total_nvlink_bytes=23459990856 weight_to_token_ratio=0.078060 prefetch_share_of_ingress=0.070551 dedup_ratio=0.035115 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=43 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.098958 gemm_inner_pad_upper_bound=36830 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.509924610e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65394 payload_rows_remote=57202 dup_rows=142 dup_rows_remote=142 dedup_ratio=0.002167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8052] dup_rows_per_dst=[0, 140, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234299392 dispatch_meta_bytes=229376 combine_payload_bytes=234299392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234528768 ingress_bytes=253173760 link_bytes=253173760 token_bytes=468828160 weight_movement_bytes=18874368 total_nvlink_bytes=487702528 weight_to_token_ratio=0.040259 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425786399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20932086880 prefetch_bytes=830472192 reduce_bytes=490733568 weight_movement_bytes=1321205760 egress_bytes=10471402592 ingress_bytes=11781890048 link_bytes=11782578088 total_nvlink_bytes=22253292640 weight_to_token_ratio=0.063119 prefetch_share_of_ingress=0.070487 dedup_ratio=0.064743 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=26 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100116 gemm_inner_pad_upper_bound=34544 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.351976372e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21851561416 prefetch_bytes=754974720 reduce_bytes=868220928 weight_movement_bytes=1623195648 egress_bytes=10931248584 ingress_bytes=12543508480 link_bytes=12543967232 total_nvlink_bytes=23474757064 weight_to_token_ratio=0.074283 prefetch_share_of_ingress=0.060188 dedup_ratio=0.028807 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=46 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.098814 gemm_inner_pad_upper_bound=35306 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.503960875e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none + [2026-08-29 23:37:45] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 1019.0 | throughput per GPU (TFLOP/s/GPU): 87.5 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.199980E+00 | loss scale: 1.0 | grad norm: 4.285 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45218 payload_rows_remote=39232 dup_rows=20318 dup_rows_remote=17242 dedup_ratio=0.310028 router_tokens_per_expert=[195, 0, 1, 1, 612, 479, 110, 2, 0, 2, 163, 302, 350, 241, 347, 167, 6, 35, 0, 329, 399, 240, 6, 2, 272, 444, 50, 95, 107, 1, 309, 6, 393, 542, 8, 92, 3, 27, 514, 11, 135, 421, 612, 1273, 586, 11, 95, 703, 163, 284, 283, 2, 562, 878, 15, 20, 962, 64, 207, 672, 33, 388, 14, 71, 465, 231, 2, 5, 104, 0, 281, 599, 2, 417, 543, 433, 14, 76, 7, 1559, 420, 16, 50, 293, 15, 249, 2, 0, 1218, 247, 33, 267, 102, 447, 119, 30, 1, 77, 365, 44, 0, 895, 1507, 68, 130, 15, 673, 24, 347, 370, 297, 0, 590, 411, 220, 18, 309, 7, 910, 98, 347, 635, 360, 9, 230, 6, 242, 0] payload_rows_per_dst=[5986, 5886, 6384, 6112, 4772, 6008, 4944, 5126] dup_rows_per_dst=[3076, 2668, 3530, 3124, 1586, 2828, 1668, 1838] combine_rows_per_src=[5986, 5886, 6384, 6112, 4772, 6008, 4944, 5126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44938 payload_rows_remote=39374 dup_rows=20598 dup_rows_remote=18522 dedup_ratio=0.314301 router_tokens_per_expert=[225, 0, 0, 0, 739, 445, 143, 1, 4, 6, 172, 379, 404, 308, 447, 155, 0, 44, 8, 308, 370, 199, 2, 0, 246, 320, 42, 44, 139, 0, 316, 7, 326, 467, 4, 95, 10, 27, 515, 14, 86, 672, 662, 1172, 347, 16, 92, 603, 138, 172, 170, 1, 612, 861, 9, 22, 968, 36, 148, 942, 34, 289, 6, 57, 397, 243, 0, 4, 149, 0, 286, 576, 1, 382, 650, 410, 13, 92, 9, 1605, 393, 17, 75, 160, 9, 166, 5, 1, 1120, 462, 35, 374, 86, 532, 153, 15, 0, 73, 274, 41, 1, 807, 1345, 45, 101, 19, 538, 59, 470, 570, 163, 0, 570, 496, 194, 27, 375, 4, 1036, 98, 397, 739, 345, 44, 266, 2, 201, 2] payload_rows_per_dst=[6362, 5564, 6182, 5998, 4660, 6076, 4796, 5300] dup_rows_per_dst=[3704, 2076, 3174, 2932, 1764, 3202, 1526, 2220] combine_rows_per_src=[6362, 5564, 6182, 5998, 4660, 6076, 4796, 5300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45138 payload_rows_remote=38862 dup_rows=20398 dup_rows_remote=17066 dedup_ratio=0.311249 router_tokens_per_expert=[287, 1, 0, 0, 814, 558, 111, 0, 0, 6, 132, 325, 428, 296, 455, 132, 1, 23, 0, 194, 619, 179, 6, 0, 233, 688, 47, 46, 133, 2, 275, 4, 409, 495, 2, 71, 1, 17, 719, 4, 112, 417, 619, 1279, 616, 21, 97, 588, 89, 259, 196, 0, 342, 778, 3, 16, 946, 32, 214, 745, 27, 226, 6, 68, 508, 212, 0, 2, 107, 0, 413, 562, 2, 297, 630, 330, 7, 23, 9, 1702, 387, 14, 31, 280, 14, 248, 1, 0, 1316, 267, 3, 332, 130, 400, 93, 13, 0, 71, 249, 39, 2, 835, 1758, 42, 126, 23, 676, 23, 254, 254, 231, 0, 812, 227, 363, 22, 359, 5, 880, 94, 328, 590, 342, 8, 264, 2, 147, 0] payload_rows_per_dst=[4918, 5940, 5158, 5542, 6276, 5514, 6266, 5524] dup_rows_per_dst=[2172, 2694, 2042, 2352, 3332, 2326, 2900, 2580] combine_rows_per_src=[4918, 5940, 5158, 5542, 6276, 5514, 6266, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45128 payload_rows_remote=38634 dup_rows=20408 dup_rows_remote=16532 dedup_ratio=0.311401 router_tokens_per_expert=[337, 0, 1, 0, 655, 648, 84, 3, 0, 2, 153, 277, 435, 234, 421, 95, 6, 31, 3, 239, 503, 226, 0, 0, 263, 425, 64, 59, 205, 3, 273, 11, 438, 651, 8, 51, 3, 17, 656, 31, 43, 571, 571, 1060, 525, 12, 101, 744, 106, 267, 114, 0, 632, 770, 10, 13, 1035, 31, 168, 812, 26, 364, 21, 38, 525, 205, 1, 1, 123, 2, 277, 554, 1, 362, 595, 392, 8, 48, 5, 1657, 386, 21, 30, 298, 8, 155, 5, 0, 1449, 199, 20, 405, 124, 483, 117, 14, 1, 54, 335, 66, 0, 776, 1507, 44, 173, 12, 664, 29, 326, 324, 214, 0, 627, 239, 229, 40, 260, 3, 883, 83, 330, 707, 330, 9, 281, 2, 205, 0] payload_rows_per_dst=[6484, 5770, 6494, 5928, 4700, 6130, 4666, 4956] dup_rows_per_dst=[3520, 2460, 3876, 2886, 1498, 3064, 1370, 1734] combine_rows_per_src=[6484, 5770, 6494, 5928, 4700, 6130, 4666, 4956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=37294 dup_rows=21598 dup_rows_remote=17090 dedup_ratio=0.329559 router_tokens_per_expert=[274, 1, 0, 0, 876, 418, 147, 1, 1, 5, 173, 309, 290, 222, 485, 218, 17, 47, 0, 301, 473, 301, 1, 0, 255, 382, 21, 107, 183, 6, 303, 9, 345, 452, 4, 93, 1, 27, 719, 16, 140, 481, 558, 1182, 422, 7, 68, 763, 114, 222, 238, 1, 589, 926, 5, 31, 801, 35, 187, 788, 23, 240, 6, 56, 486, 281, 1, 10, 145, 0, 238, 592, 1, 510, 662, 459, 30, 49, 5, 1588, 421, 20, 75, 255, 22, 258, 4, 0, 896, 364, 57, 341, 124, 517, 115, 40, 0, 66, 244, 28, 0, 715, 1290, 50, 139, 26, 443, 35, 458, 499, 193, 0, 564, 417, 235, 21, 345, 5, 921, 84, 416, 665, 373, 17, 306, 6, 274, 1] payload_rows_per_dst=[4852, 5904, 4902, 4178, 6358, 4922, 6178, 6644] dup_rows_per_dst=[1988, 2682, 1880, 892, 3756, 2096, 3796, 4508] combine_rows_per_src=[4852, 5904, 4902, 4178, 6358, 4922, 6178, 6644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45002 payload_rows_remote=38760 dup_rows=20534 dup_rows_remote=16878 dedup_ratio=0.313324 router_tokens_per_expert=[365, 0, 0, 0, 729, 446, 110, 3, 0, 1, 142, 264, 368, 335, 398, 166, 0, 26, 1, 291, 459, 215, 2, 0, 281, 577, 61, 41, 91, 3, 341, 6, 348, 511, 7, 53, 2, 40, 569, 16, 85, 540, 541, 1003, 529, 9, 49, 676, 121, 275, 142, 0, 582, 905, 3, 17, 1180, 25, 220, 937, 40, 280, 8, 71, 525, 212, 0, 0, 115, 0, 289, 604, 1, 469, 502, 497, 11, 62, 8, 1556, 403, 11, 39, 273, 14, 218, 0, 0, 1235, 272, 48, 383, 95, 543, 96, 22, 0, 48, 389, 32, 0, 938, 1496, 29, 180, 6, 571, 19, 324, 362, 273, 1, 640, 264, 262, 12, 461, 1, 837, 73, 340, 493, 321, 16, 211, 0, 160, 0] payload_rows_per_dst=[5956, 5716, 6072, 6242, 5030, 5974, 4974, 5038] dup_rows_per_dst=[3078, 2432, 3320, 3656, 2006, 3004, 1568, 1470] combine_rows_per_src=[5956, 5716, 6072, 6242, 5030, 5974, 4974, 5038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44048 payload_rows_remote=39080 dup_rows=21488 dup_rows_remote=19538 dedup_ratio=0.327881 router_tokens_per_expert=[324, 1, 0, 0, 793, 445, 135, 0, 0, 0, 159, 469, 385, 244, 211, 186, 7, 54, 1, 242, 442, 266, 0, 2, 263, 371, 43, 74, 239, 3, 267, 2, 339, 379, 1, 67, 6, 27, 446, 24, 100, 681, 609, 930, 530, 11, 99, 654, 205, 221, 244, 0, 530, 836, 6, 18, 1058, 26, 160, 691, 34, 318, 4, 42, 349, 244, 0, 11, 175, 1, 417, 646, 3, 398, 394, 453, 13, 98, 15, 1610, 402, 14, 49, 167, 14, 160, 7, 0, 994, 338, 71, 371, 176, 538, 114, 44, 0, 54, 452, 27, 5, 862, 1613, 29, 164, 12, 612, 59, 419, 560, 161, 1, 599, 416, 321, 27, 376, 2, 1029, 83, 285, 537, 394, 3, 216, 0, 240, 0] payload_rows_per_dst=[4856, 5436, 4698, 5856, 6194, 4968, 6288, 5752] dup_rows_per_dst=[1848, 2284, 1940, 2930, 3460, 1950, 3772, 3304] combine_rows_per_src=[4856, 5436, 4698, 5856, 6194, 4968, 6288, 5752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161275904 dispatch_meta_bytes=231584 combine_payload_bytes=161275904 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=161507488 ingress_bytes=199024640 link_bytes=199024640 token_bytes=322783392 weight_movement_bytes=37748736 total_nvlink_bytes=360532128 weight_to_token_ratio=0.116948 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.076389 gemm_active_groups=19 gemm_peak_group_rows=9216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.906956174e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159178752 dispatch_meta_bytes=223712 combine_payload_bytes=159178752 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=159402464 ingress_bytes=196927488 link_bytes=196927488 token_bytes=318581216 weight_movement_bytes=37748736 total_nvlink_bytes=356329952 weight_to_token_ratio=0.118490 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=7040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.886862296e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160694272 dispatch_meta_bytes=225896 combine_payload_bytes=160694272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=160920168 ingress_bytes=179568640 link_bytes=179568640 token_bytes=321614440 weight_movement_bytes=18874368 total_nvlink_bytes=340488808 weight_to_token_ratio=0.058686 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.720538355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=152756224 dispatch_meta_bytes=217536 combine_payload_bytes=152756224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=152973760 ingress_bytes=190504960 link_bytes=190504960 token_bytes=305729984 weight_movement_bytes=37748736 total_nvlink_bytes=343478720 weight_to_token_ratio=0.123471 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=5248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.825324792e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158244864 dispatch_meta_bytes=220664 combine_payload_bytes=158244864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158465528 ingress_bytes=214867968 link_bytes=214867968 token_bytes=316710392 weight_movement_bytes=56623104 total_nvlink_bytes=373333496 weight_to_token_ratio=0.178785 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=5760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.058759148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=222552 combine_payload_bytes=158760960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158983512 ingress_bytes=215384064 link_bytes=215384064 token_bytes=317744472 weight_movement_bytes=56623104 total_nvlink_bytes=374367576 weight_to_token_ratio=0.178203 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=6656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.063704126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=234472 combine_payload_bytes=160071680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=160306152 ingress_bytes=178946048 link_bytes=178946048 token_bytes=320377832 weight_movement_bytes=18874368 total_nvlink_bytes=339252200 weight_to_token_ratio=0.058913 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=9728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.714572985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58534 payload_rows_remote=50342 dup_rows=7002 dup_rows_remote=6546 dedup_ratio=0.106842 router_tokens_per_expert=[0, 3, 0, 0, 5, 0, 0, 44, 0, 0, 0, 0, 1229, 0, 0, 3, 4096, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 97, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 760, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 2, 3279, 0, 1, 0, 2269, 6, 0, 0, 0, 1, 1390, 0, 0, 1579, 0, 1660, 0, 0, 27, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7356, 8170, 7600, 8192, 7038, 4960, 7026] dup_rows_per_dst=[2562, 0, 0, 220, 456, 1052, 424, 2288] combine_rows_per_src=[8192, 7356, 8170, 7600, 8192, 7038, 4960, 7026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55768 payload_rows_remote=47688 dup_rows=9768 dup_rows_remote=6830 dedup_ratio=0.149048 router_tokens_per_expert=[0, 39, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1256, 7, 0, 16, 4082, 0, 1, 0, 0, 4035, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4060, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 2, 0, 2, 110, 0, 155, 0, 0, 16, 0, 0, 25, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 3, 0, 56, 3718, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 1472, 0, 0, 1600, 0, 1144, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8080, 8124, 6816, 8192, 7470, 4528, 4366] dup_rows_per_dst=[2780, 2938, 6, 1756, 678, 432, 50, 1128] combine_rows_per_src=[8192, 8080, 8124, 6816, 8192, 7470, 4528, 4366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51548 payload_rows_remote=43356 dup_rows=13988 dup_rows_remote=11890 dedup_ratio=0.213440 router_tokens_per_expert=[0, 1, 0, 1, 6, 0, 0, 94, 3, 0, 0, 0, 906, 0, 0, 43, 4088, 0, 0, 0, 0, 4089, 4091, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 3, 0, 0, 0, 5, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 3, 81, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 401, 1, 1, 0, 0, 0, 0, 0, 3065, 1, 0, 0, 25, 3680, 0, 0, 0, 2727, 1, 0, 0, 0, 6, 1633, 0, 1, 1554, 0, 1246, 0, 0, 26, 5, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8084, 6892, 3864, 7460, 5472, 3398] dup_rows_per_dst=[2098, 7798, 1696, 1052, 108, 720, 36, 480] combine_rows_per_src=[8192, 8186, 8084, 6892, 3864, 7460, 5472, 3398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56878 payload_rows_remote=48838 dup_rows=8658 dup_rows_remote=8650 dedup_ratio=0.132111 router_tokens_per_expert=[0, 19, 0, 0, 0, 0, 3, 58, 0, 0, 1, 0, 980, 3, 2, 79, 4092, 0, 3, 0, 0, 4059, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4016, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1912, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 41, 0, 391, 0, 0, 0, 0, 0, 8, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 1, 0, 0, 2776, 0, 0, 0, 39, 3828, 3, 0, 6, 2065, 0, 0, 0, 0, 2, 1498, 0, 0, 720, 0, 1051, 0, 0, 21, 0, 1, 0, 0] payload_rows_per_dst=[8184, 8132, 8040, 7110, 8186, 7768, 4200, 5258] dup_rows_per_dst=[2276, 168, 8, 2272, 890, 1858, 28, 1158] combine_rows_per_src=[8184, 8132, 8040, 7110, 8186, 7768, 4200, 5258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51392 payload_rows_remote=43896 dup_rows=14144 dup_rows_remote=10564 dedup_ratio=0.215820 router_tokens_per_expert=[0, 43, 0, 0, 0, 0, 5, 86, 0, 0, 1, 0, 666, 4, 2, 24, 4078, 0, 6, 0, 0, 4029, 4081, 5, 0, 0, 5, 0, 0, 0, 3, 0, 3, 0, 0, 4014, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 629, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 0, 0, 0, 4, 0, 2, 220, 0, 162, 1, 0, 2, 0, 0, 42, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 1, 0, 0, 1, 3505, 0, 0, 0, 53, 3512, 5, 0, 3, 1665, 0, 0, 0, 0, 0, 2218, 2, 0, 1851, 0, 1463, 0, 0, 3, 0, 1, 0, 0] payload_rows_per_dst=[1588, 8178, 8034, 3194, 8184, 6624, 8094, 7496] dup_rows_per_dst=[74, 22, 10, 72, 900, 496, 8990, 3580] combine_rows_per_src=[1588, 8178, 8034, 3194, 8184, 6624, 8094, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58290 payload_rows_remote=50666 dup_rows=7246 dup_rows_remote=5414 dedup_ratio=0.110565 router_tokens_per_expert=[0, 0, 0, 0, 27, 0, 0, 80, 1, 0, 0, 0, 596, 0, 0, 32, 4096, 0, 0, 0, 0, 4038, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3792, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 97, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 642, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 131, 3819, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 2015, 0, 0, 1582, 0, 1128, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8076, 7590, 7624, 8192, 7722, 3882, 7012] dup_rows_per_dst=[1468, 0, 0, 1832, 256, 1212, 26, 2452] combine_rows_per_src=[8192, 8076, 7590, 7624, 8192, 7722, 3882, 7012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=206200832 dispatch_meta_bytes=227552 combine_payload_bytes=206200832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206428384 ingress_bytes=225075200 link_bytes=225075200 token_bytes=412629216 weight_movement_bytes=18874368 total_nvlink_bytes=431503584 weight_to_token_ratio=0.045742 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.156559822e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=195330048 dispatch_meta_bytes=218072 combine_payload_bytes=195330048 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=195548120 ingress_bytes=270827520 link_bytes=270827520 token_bytes=390878168 weight_movement_bytes=75497472 total_nvlink_bytes=466375640 weight_to_token_ratio=0.193148 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594936041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54362 payload_rows_remote=46762 dup_rows=11174 dup_rows_remote=10860 dedup_ratio=0.170502 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 1, 52, 0, 0, 3, 0, 94, 2, 3, 151, 4093, 0, 0, 0, 0, 4086, 4075, 3, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 3797, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 2, 1, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 29, 0, 147, 0, 0, 0, 1, 0, 17, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 194, 0, 0, 0, 0, 1, 0, 0, 2826, 0, 0, 0, 167, 3761, 2, 0, 15, 2168, 0, 3, 0, 0, 2, 2619, 0, 0, 588, 0, 890, 0, 0, 30, 0, 0, 0, 0] payload_rows_per_dst=[8168, 12, 7600, 7866, 8188, 7600, 8192, 6736] dup_rows_per_dst=[596, 2, 6, 3648, 394, 314, 4692, 1522] combine_rows_per_src=[8168, 12, 7600, 7866, 8188, 7600, 8192, 6736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=177586176 dispatch_meta_bytes=220984 combine_payload_bytes=177586176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=177807160 ingress_bytes=196460544 link_bytes=196460544 token_bytes=355393336 weight_movement_bytes=18874368 total_nvlink_bytes=374267704 weight_to_token_ratio=0.053108 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.083333 gemm_active_groups=13 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.882388268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=200040448 dispatch_meta_bytes=229952 combine_payload_bytes=200040448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200270400 ingress_bytes=218914816 link_bytes=218914816 token_bytes=400310848 weight_movement_bytes=18874368 total_nvlink_bytes=419185216 weight_to_token_ratio=0.047149 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.097534054e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=179798016 dispatch_meta_bytes=217840 combine_payload_bytes=179798016 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=180015856 ingress_bytes=217546752 link_bytes=217546752 token_bytes=359813872 weight_movement_bytes=37748736 total_nvlink_bytes=397562608 weight_to_token_ratio=0.104912 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.084425938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=37810 dup_rows=21110 dup_rows_remote=16902 dedup_ratio=0.322113 router_tokens_per_expert=[291, 0, 0, 2, 937, 507, 122, 0, 0, 1, 135, 306, 306, 251, 404, 106, 0, 18, 1, 216, 493, 194, 1, 0, 280, 462, 46, 42, 144, 0, 326, 1, 396, 421, 1, 34, 0, 31, 410, 8, 103, 495, 589, 1264, 375, 10, 69, 524, 120, 238, 188, 0, 523, 779, 6, 13, 1059, 24, 218, 936, 40, 199, 4, 50, 554, 230, 0, 4, 116, 1, 402, 493, 1, 410, 553, 378, 9, 46, 14, 1861, 444, 15, 17, 229, 3, 249, 0, 0, 1399, 389, 7, 332, 129, 413, 96, 15, 0, 57, 373, 32, 2, 915, 1656, 25, 161, 6, 603, 23, 321, 379, 204, 0, 787, 273, 304, 21, 348, 0, 880, 85, 356, 603, 447, 7, 211, 6, 155, 0] payload_rows_per_dst=[4924, 5686, 4420, 5078, 6380, 5310, 6616, 6012] dup_rows_per_dst=[1812, 2338, 1464, 1830, 3764, 2164, 4208, 3530] combine_rows_per_src=[4924, 5686, 4420, 5078, 6380, 5310, 6616, 6012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=207527936 dispatch_meta_bytes=224320 combine_payload_bytes=207527936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207752256 ingress_bytes=226402304 link_bytes=226402304 token_bytes=415280192 weight_movement_bytes=18874368 total_nvlink_bytes=434154560 weight_to_token_ratio=0.045450 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169275480e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191537152 dispatch_meta_bytes=230488 combine_payload_bytes=191537152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191767640 ingress_bytes=210411520 link_bytes=210411520 token_bytes=383304792 weight_movement_bytes=18874368 total_nvlink_bytes=402179160 weight_to_token_ratio=0.049241 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.016059656e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60324 payload_rows_remote=55260 dup_rows=5212 dup_rows_remote=5184 dedup_ratio=0.079529 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 297, 0, 4092, 0, 1823, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3795, 0, 0, 4092, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 3501, 2603, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8192, 8034, 8192, 5064, 8192, 6266, 8192] dup_rows_per_dst=[10, 678, 2774, 4, 28, 10, 1704, 4] combine_rows_per_src=[8192, 8192, 8034, 8192, 5064, 8192, 6266, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59786 payload_rows_remote=51620 dup_rows=5750 dup_rows_remote=4840 dedup_ratio=0.087738 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 65, 0, 0, 21, 0, 386, 0, 4061, 0, 1149, 0, 0, 0, 14, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 9, 0, 4020, 0, 0, 0, 1691, 0, 0, 0, 4084, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 1776, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8166, 8010, 8186, 7458, 8040, 3552, 8192] dup_rows_per_dst=[4, 910, 1916, 34, 2766, 66, 2, 52] combine_rows_per_src=[8182, 8166, 8010, 8186, 7458, 8040, 3552, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60920 payload_rows_remote=52730 dup_rows=4616 dup_rows_remote=4424 dedup_ratio=0.070435 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 22, 0, 310, 0, 4095, 0, 758, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 3685, 0, 0, 4095, 0, 0, 0, 1016, 0, 0, 0, 4087, 0, 0, 0, 0, 26, 0, 0, 0, 6, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3625, 2492, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 10, 0, 2, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8192, 7696, 8188, 7642, 8010, 4994, 8008] dup_rows_per_dst=[192, 1184, 1230, 4, 1952, 52, 0, 2] combine_rows_per_src=[8190, 8192, 7696, 8188, 7642, 8010, 4994, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=154869760 dispatch_meta_bytes=218848 combine_payload_bytes=154869760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155088608 ingress_bytes=192618496 link_bytes=192618496 token_bytes=309958368 weight_movement_bytes=37748736 total_nvlink_bytes=347707104 weight_to_token_ratio=0.121786 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=6784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.845575654e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60182 payload_rows_remote=52654 dup_rows=5354 dup_rows_remote=4358 dedup_ratio=0.081696 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 48, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 614, 3, 0, 3, 0, 548, 3, 4091, 0, 406, 0, 0, 0, 6, 0, 131, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3742, 2, 0, 4065, 0, 0, 0, 1058, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 3714, 1958, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7528, 8186, 7714, 8132, 4050, 8188] dup_rows_per_dst=[98, 2332, 996, 10, 1890, 8, 20, 0] combine_rows_per_src=[8192, 8192, 7528, 8186, 7714, 8132, 4050, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=226344960 dispatch_meta_bytes=241776 combine_payload_bytes=226344960 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=226586736 ingress_bytes=320716800 link_bytes=320716800 token_bytes=452931696 weight_movement_bytes=94371840 total_nvlink_bytes=547303536 weight_to_token_ratio=0.208358 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.072950575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51978 payload_rows_remote=43804 dup_rows=13558 dup_rows_remote=13504 dedup_ratio=0.206879 router_tokens_per_expert=[0, 2, 0, 0, 0, 5, 0, 0, 2, 0, 4088, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 272, 5, 0, 11, 0, 341, 4, 4020, 8, 1164, 0, 0, 0, 3, 0, 58, 0, 5, 0, 0, 0, 3, 5, 0, 0, 0, 0, 2, 0, 0, 3, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3702, 10, 0, 4025, 0, 0, 0, 927, 1, 0, 0, 4071, 0, 0, 0, 0, 20, 2, 0, 0, 6, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 3413, 2460, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5924, 3042, 2446, 8158, 8064, 8072, 8098, 8174] dup_rows_per_dst=[10, 506, 30, 6, 1852, 42, 11058, 54] combine_rows_per_src=[5924, 3042, 2446, 8158, 8064, 8072, 8098, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=211435520 dispatch_meta_bytes=225840 combine_payload_bytes=211435520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=211661360 ingress_bytes=249184256 link_bytes=249184256 token_bytes=423096880 weight_movement_bytes=37748736 total_nvlink_bytes=460845616 weight_to_token_ratio=0.089220 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.387560935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58972 payload_rows_remote=50780 dup_rows=6564 dup_rows_remote=6564 dedup_ratio=0.100159 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 3, 0, 756, 0, 4096, 0, 723, 0, 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 0, 0, 4096, 0, 0, 0, 1196, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 3822, 1184, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7888, 8192, 7692, 8192, 2432, 8192] dup_rows_per_dst=[46, 2204, 2124, 0, 2034, 56, 0, 100] combine_rows_per_src=[8192, 8192, 7888, 8192, 7692, 8192, 2432, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=215982080 dispatch_meta_bytes=228616 combine_payload_bytes=215982080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216210696 ingress_bytes=253730816 link_bytes=253730816 token_bytes=432192776 weight_movement_bytes=37748736 total_nvlink_bytes=469941512 weight_to_token_ratio=0.087342 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62118 payload_rows_remote=54010 dup_rows=3418 dup_rows_remote=3048 dedup_ratio=0.052155 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 964, 0, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3, 0, 11, 0, 110, 0, 4089, 7, 50, 0, 0, 0, 0, 0, 12, 0, 3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 3244, 1, 0, 4048, 0, 0, 0, 3520, 0, 0, 0, 4091, 0, 0, 0, 1, 182, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3685, 375, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 7408, 8188, 7042, 8108, 6806, 8188] dup_rows_per_dst=[1932, 548, 100, 8, 0, 370, 460, 0] combine_rows_per_src=[8188, 8190, 7408, 8188, 7042, 8108, 6806, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=215670784 dispatch_meta_bytes=228048 combine_payload_bytes=215670784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215898832 ingress_bytes=234545152 link_bytes=234545152 token_bytes=431569616 weight_movement_bytes=18874368 total_nvlink_bytes=450443984 weight_to_token_ratio=0.043734 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247296243e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=179421184 dispatch_meta_bytes=229232 combine_payload_bytes=179421184 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=179650416 ingress_bytes=198295552 link_bytes=198295552 token_bytes=359071600 weight_movement_bytes=18874368 total_nvlink_bytes=377945968 weight_to_token_ratio=0.052564 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.899970412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=207994880 dispatch_meta_bytes=229376 combine_payload_bytes=207994880 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208224256 ingress_bytes=245743616 link_bytes=245743616 token_bytes=416219136 weight_movement_bytes=37748736 total_nvlink_bytes=453967872 weight_to_token_ratio=0.090694 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.076389 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.354594415e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=221224960 dispatch_meta_bytes=228232 combine_payload_bytes=221224960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221453192 ingress_bytes=240099328 link_bytes=240099328 token_bytes=442678152 weight_movement_bytes=18874368 total_nvlink_bytes=461552520 weight_to_token_ratio=0.042637 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300513625e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64250 payload_rows_remote=56058 dup_rows=1286 dup_rows_remote=1286 dedup_ratio=0.019623 router_tokens_per_expert=[0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 14, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7904, 7230, 8192, 8192, 8192, 8192, 8156] dup_rows_per_dst=[70, 0, 196, 2, 0, 0, 990, 28] combine_rows_per_src=[8192, 7904, 7230, 8192, 8192, 8192, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57510 payload_rows_remote=50856 dup_rows=8026 dup_rows_remote=5248 dedup_ratio=0.122467 router_tokens_per_expert=[0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 0, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 4077, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 2, 0, 0, 816, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 1, 2, 5, 0, 0] payload_rows_per_dst=[8188, 6654, 6842, 8192, 8142, 8192, 3700, 7600] dup_rows_per_dst=[260, 2778, 2368, 2326, 0, 70, 218, 6] combine_rows_per_src=[8188, 6654, 6842, 8192, 8142, 8192, 3700, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54092 payload_rows_remote=45900 dup_rows=11444 dup_rows_remote=11434 dedup_ratio=0.174622 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 0, 0, 0, 0, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4071, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 2, 0, 2] payload_rows_per_dst=[8192, 8192, 7976, 4418, 8192, 8128, 2308, 6686] dup_rows_per_dst=[10, 6140, 5194, 0, 96, 2, 0, 2] combine_rows_per_src=[8192, 8192, 7976, 4418, 8192, 8128, 2308, 6686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55908 payload_rows_remote=47766 dup_rows=9628 dup_rows_remote=5808 dedup_ratio=0.146912 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 5, 34, 0, 0, 1, 0, 631, 4, 4, 5, 4084, 0, 4, 0, 0, 4036, 4084, 5, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4076, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 274, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 2, 0, 1, 208, 0, 116, 0, 0, 6, 0, 0, 12, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 1, 0, 3510, 0, 0, 2, 27, 3521, 5, 0, 0, 1907, 0, 3, 0, 0, 0, 1484, 0, 0, 2074, 0, 2242, 0, 0, 13, 0, 5, 0, 0] payload_rows_per_dst=[2730, 6584, 8162, 7332, 8190, 7178, 8142, 7590] dup_rows_per_dst=[244, 4, 6, 250, 666, 592, 3820, 4046] combine_rows_per_src=[2730, 6584, 8162, 7332, 8190, 7178, 8142, 7590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=229613568 dispatch_meta_bytes=229376 combine_payload_bytes=229613568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229842944 ingress_bytes=248487936 link_bytes=248487936 token_bytes=459456512 weight_movement_bytes=18874368 total_nvlink_bytes=478330880 weight_to_token_ratio=0.041080 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.380889139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60614 payload_rows_remote=52802 dup_rows=4922 dup_rows_remote=1986 dedup_ratio=0.075104 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2993, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2027, 3, 0, 0, 3, 0, 3337, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 81, 4082, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 873, 0, 4093, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6004, 7812, 8188, 8182, 8186, 8192, 5858] dup_rows_per_dst=[16, 6, 2936, 140, 0, 76, 1748, 0] combine_rows_per_src=[8192, 6004, 7812, 8188, 8182, 8186, 8192, 5858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57834 payload_rows_remote=49932 dup_rows=7702 dup_rows_remote=3928 dedup_ratio=0.117523 router_tokens_per_expert=[0, 14, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 13, 1, 3569, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 1291, 5, 0, 0, 3, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 514, 4038, 0, 4082, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 2, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 16, 0, 0, 2, 0, 1394, 1, 4090, 4084, 0, 0, 5, 1, 0, 0, 0, 0, 0, 3832, 1, 2, 0, 0, 0] payload_rows_per_dst=[7548, 7162, 2594, 8184, 8178, 8074, 8192, 7902] dup_rows_per_dst=[26, 8, 16, 926, 6, 16, 2930, 3774] combine_rows_per_src=[7548, 7162, 2594, 8184, 8178, 8074, 8192, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=208306176 dispatch_meta_bytes=224416 combine_payload_bytes=208306176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208530592 ingress_bytes=227180544 link_bytes=227180544 token_bytes=416836768 weight_movement_bytes=18874368 total_nvlink_bytes=435711136 weight_to_token_ratio=0.045280 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.176732193e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63810 payload_rows_remote=55618 dup_rows=1726 dup_rows_remote=1660 dedup_ratio=0.026337 router_tokens_per_expert=[0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7730, 7684, 8192, 8192, 8188, 8192, 7440] dup_rows_per_dst=[36, 0, 662, 66, 0, 22, 940, 0] combine_rows_per_src=[8192, 7730, 7684, 8192, 8192, 8188, 8192, 7440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=188006400 dispatch_meta_bytes=229336 combine_payload_bytes=188006400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=188235736 ingress_bytes=206880768 link_bytes=206880768 token_bytes=376242136 weight_movement_bytes=18874368 total_nvlink_bytes=395116504 weight_to_token_ratio=0.050165 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.982229727e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=195649536 dispatch_meta_bytes=214296 combine_payload_bytes=195649536 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=195863832 ingress_bytes=252272640 link_bytes=252272640 token_bytes=391513368 weight_movement_bytes=56623104 total_nvlink_bytes=448136472 weight_to_token_ratio=0.144626 gemm_rows_defined=34560 gemm_rows_charged=36864 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.062500 gemm_active_groups=18 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417152311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58382 payload_rows_remote=50192 dup_rows=7154 dup_rows_remote=7142 dedup_ratio=0.109161 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3325, 2, 0, 0, 2, 0, 3548, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 76, 4082, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 929, 2, 4093, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5322, 7226, 8192, 8178, 8190, 8192, 4890] dup_rows_per_dst=[8, 0, 3962, 132, 0, 12, 1872, 1168] combine_rows_per_src=[8192, 5322, 7226, 8192, 8178, 8190, 8192, 4890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=204521472 dispatch_meta_bytes=215440 combine_payload_bytes=204521472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204736912 ingress_bytes=242270208 link_bytes=242270208 token_bytes=409258384 weight_movement_bytes=37748736 total_nvlink_bytes=447007120 weight_to_token_ratio=0.092237 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.321313929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=216276992 dispatch_meta_bytes=219152 combine_payload_bytes=216276992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216496144 ingress_bytes=254025728 link_bytes=254025728 token_bytes=432773136 weight_movement_bytes=37748736 total_nvlink_bytes=470521872 weight_to_token_ratio=0.087225 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.433949537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=227811328 dispatch_meta_bytes=229112 combine_payload_bytes=227811328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228040440 ingress_bytes=265560064 link_bytes=265560064 token_bytes=455851768 weight_movement_bytes=37748736 total_nvlink_bytes=493600504 weight_to_token_ratio=0.082809 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.544465870e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63162 payload_rows_remote=54978 dup_rows=2374 dup_rows_remote=2374 dedup_ratio=0.036224 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 1281, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 8192, 8184, 8192, 5832, 8192] dup_rows_per_dst=[0, 2318, 14, 6, 0, 36, 0, 0] combine_rows_per_src=[8192, 8188, 8190, 8192, 8184, 8192, 5832, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64410 payload_rows_remote=56242 dup_rows=1126 dup_rows_remote=600 dedup_ratio=0.017181 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 282, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 18, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 7754, 8192, 8192, 7532, 8188] dup_rows_per_dst=[0, 526, 544, 0, 0, 32, 0, 24] combine_rows_per_src=[8192, 8168, 8192, 7754, 8192, 8192, 7532, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205586432 dispatch_meta_bytes=229336 combine_payload_bytes=205586432 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=205815768 ingress_bytes=243335168 link_bytes=243335168 token_bytes=411402200 weight_movement_bytes=37748736 total_nvlink_bytes=449150936 weight_to_token_ratio=0.091756 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331517851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63794 payload_rows_remote=55602 dup_rows=1742 dup_rows_remote=1726 dedup_ratio=0.026581 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4073, 0, 0, 0, 257, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 5, 4026, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8064, 8184, 7254, 7532, 8188] dup_rows_per_dst=[16, 1428, 292, 2, 2, 2, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8064, 8184, 7254, 7532, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=225189888 dispatch_meta_bytes=229408 combine_payload_bytes=225189888 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=225419296 ingress_bytes=244064256 link_bytes=244064256 token_bytes=450609184 weight_movement_bytes=18874368 total_nvlink_bytes=469483552 weight_to_token_ratio=0.041886 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.338503614e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230367232 dispatch_meta_bytes=227368 combine_payload_bytes=230367232 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230594600 ingress_bytes=305864704 link_bytes=305864704 token_bytes=460961832 weight_movement_bytes=75497472 total_nvlink_bytes=536459304 weight_to_token_ratio=0.163782 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.930645099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56234 payload_rows_remote=48058 dup_rows=9302 dup_rows_remote=9296 dedup_ratio=0.141937 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3929, 0, 0, 0, 410, 0, 4082, 0, 0, 0, 6, 0, 0, 0, 4089, 0, 0, 267, 0, 0, 0, 0, 0, 0, 6, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 6, 3826, 0, 0, 2, 5, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 19, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 3820, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8176, 1796, 6298, 7674, 8186, 8190, 7738, 8176] dup_rows_per_dst=[2, 380, 556, 1744, 0, 178, 6436, 6] combine_rows_per_src=[8176, 1796, 6298, 7674, 8186, 8190, 7738, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64172 payload_rows_remote=55980 dup_rows=1364 dup_rows_remote=1122 dedup_ratio=0.020813 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 478, 0, 4092, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 115, 0, 0, 0, 0, 0, 0, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 3983, 0, 0, 0, 2, 7, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 4093, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8182, 8192, 7988, 8186, 8188, 7058, 8188] dup_rows_per_dst=[0, 688, 242, 0, 16, 418, 0, 0] combine_rows_per_src=[8190, 8182, 8192, 7988, 8186, 8188, 7058, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55396 payload_rows_remote=47272 dup_rows=10140 dup_rows_remote=2170 dedup_ratio=0.154724 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 9, 0, 367, 3, 4070, 2, 2360, 0, 0, 0, 0, 0, 79, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3521, 2, 0, 4086, 0, 0, 0, 209, 1, 0, 0, 4077, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 4, 2, 0, 3805, 1939, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8182, 4556, 6004, 8168, 3992, 8186, 8124, 8184] dup_rows_per_dst=[2, 452, 1352, 12, 346, 0, 7970, 6] combine_rows_per_src=[8182, 4556, 6004, 8168, 3992, 8186, 8124, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=227745792 dispatch_meta_bytes=229312 combine_payload_bytes=227745792 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=227975104 ingress_bytes=265494528 link_bytes=265494528 token_bytes=455720896 weight_movement_bytes=37748736 total_nvlink_bytes=493469632 weight_to_token_ratio=0.082833 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.543837936e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65014 payload_rows_remote=57014 dup_rows=522 dup_rows_remote=522 dedup_ratio=0.007965 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 165, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4000, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8000, 8188, 8192, 7866, 8192] dup_rows_per_dst=[0, 246, 30, 0, 22, 224, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8000, 8188, 8192, 7866, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=196845568 dispatch_meta_bytes=229416 combine_payload_bytes=196845568 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=197074984 ingress_bytes=234594304 link_bytes=234594304 token_bytes=393920552 weight_movement_bytes=37748736 total_nvlink_bytes=431669288 weight_to_token_ratio=0.095828 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247767193e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65080 payload_rows_remote=56892 dup_rows=456 dup_rows_remote=410 dedup_ratio=0.006958 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 131, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 77, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 37, 3952, 0, 0, 0, 1, 118, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4091, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 7978, 8146, 8188, 8008, 8192] dup_rows_per_dst=[0, 198, 162, 0, 48, 46, 0, 2] combine_rows_per_src=[8192, 8184, 8192, 7978, 8146, 8188, 8008, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=229294080 dispatch_meta_bytes=228408 combine_payload_bytes=229294080 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229522488 ingress_bytes=285917184 link_bytes=285917184 token_bytes=458816568 weight_movement_bytes=56623104 total_nvlink_bytes=515439672 weight_to_token_ratio=0.123411 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.739517777e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193626112 dispatch_meta_bytes=197768 combine_payload_bytes=193626112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=193823880 ingress_bytes=231374848 link_bytes=231374848 token_bytes=387449992 weight_movement_bytes=37748736 total_nvlink_bytes=425198728 weight_to_token_ratio=0.097429 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216919950e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65376 payload_rows_remote=57184 dup_rows=160 dup_rows_remote=160 dedup_ratio=0.002441 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8192, 8192, 8192, 8192, 8192, 8042] dup_rows_per_dst=[0, 2, 158, 0, 0, 0, 0, 0] combine_rows_per_src=[8182, 8192, 8192, 8192, 8192, 8192, 8192, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=233529344 dispatch_meta_bytes=230144 combine_payload_bytes=233529344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233759488 ingress_bytes=290152448 link_bytes=290152448 token_bytes=467288832 weight_movement_bytes=56623104 total_nvlink_bytes=523911936 weight_to_token_ratio=0.121174 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780097993e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62456 payload_rows_remote=54272 dup_rows=3080 dup_rows_remote=3078 dedup_ratio=0.046997 router_tokens_per_expert=[0, 0, 4055, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4096, 0, 0, 0, 2642, 0, 0, 0, 0, 4092, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8184, 8042, 8190, 8192, 8192, 8192, 5284] dup_rows_per_dst=[138, 2, 2898, 0, 36, 0, 0, 6] combine_rows_per_src=[8180, 8184, 8042, 8190, 8192, 8192, 8192, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=233029632 dispatch_meta_bytes=229208 combine_payload_bytes=233029632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233258840 ingress_bytes=270778368 link_bytes=270778368 token_bytes=466288472 weight_movement_bytes=37748736 total_nvlink_bytes=504037208 weight_to_token_ratio=0.080956 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594465091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234225664 dispatch_meta_bytes=229376 combine_payload_bytes=234225664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234455040 ingress_bytes=271974400 link_bytes=271974400 token_bytes=468680704 weight_movement_bytes=37748736 total_nvlink_bytes=506429440 weight_to_token_ratio=0.080543 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605924881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61966 payload_rows_remote=53784 dup_rows=3570 dup_rows_remote=3402 dedup_ratio=0.054474 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4, 0, 1596, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4084, 0, 0, 0, 2399, 0, 2, 2, 0, 4088, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8176, 8190, 8188, 8094, 8154, 8182, 4800] dup_rows_per_dst=[168, 4, 3388, 0, 4, 0, 0, 6] combine_rows_per_src=[8182, 8176, 8190, 8188, 8094, 8154, 8182, 4800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=222298112 dispatch_meta_bytes=229400 combine_payload_bytes=222298112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222527512 ingress_bytes=241172480 link_bytes=241172480 token_bytes=444825624 weight_movement_bytes=18874368 total_nvlink_bytes=463699992 weight_to_token_ratio=0.042431 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310796039e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58362 payload_rows_remote=50206 dup_rows=7174 dup_rows_remote=328 dedup_ratio=0.109467 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 100, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 3996, 1, 0, 0, 0, 2, 0, 606, 0, 6, 0, 0, 6, 0, 60, 0, 0, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 6, 0, 4090, 9, 4089, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 5, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4087, 4086, 0, 0, 0, 3412, 0, 0, 0, 0, 4082, 0, 0, 0, 0] payload_rows_per_dst=[7876, 7986, 1584, 8188, 8192, 8190, 8190, 8156] dup_rows_per_dst=[190, 0, 72, 8, 38, 12, 8, 6846] combine_rows_per_src=[7876, 7986, 1584, 8188, 8192, 8190, 8190, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61702 payload_rows_remote=53544 dup_rows=3834 dup_rows_remote=118 dedup_ratio=0.058502 router_tokens_per_expert=[0, 0, 4086, 0, 0, 0, 47, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 1736, 0, 3, 0, 0, 3, 0, 119, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 1, 4093, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 0, 2204, 0, 0, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8186, 8158, 8192, 8192, 8192, 8190, 4408] dup_rows_per_dst=[94, 10, 3716, 0, 8, 6, 0, 0] combine_rows_per_src=[8184, 8186, 8158, 8192, 8192, 8192, 8190, 4408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=220299264 dispatch_meta_bytes=228744 combine_payload_bytes=220299264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220528008 ingress_bytes=258048000 link_bytes=258048000 token_bytes=440827272 weight_movement_bytes=37748736 total_nvlink_bytes=478576008 weight_to_token_ratio=0.085632 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.472488968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59526 payload_rows_remote=51334 dup_rows=6010 dup_rows_remote=4332 dedup_ratio=0.091705 router_tokens_per_expert=[0, 109, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1398, 7, 0, 0, 7, 0, 1953, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 101, 4088, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 834, 5, 4089, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7636, 2810, 8186, 8178, 8178, 8192, 8154] dup_rows_per_dst=[222, 0, 28, 200, 0, 10, 1678, 3872] combine_rows_per_src=[8192, 7636, 2810, 8186, 8178, 8178, 8192, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62044 payload_rows_remote=53852 dup_rows=3492 dup_rows_remote=3492 dedup_ratio=0.053284 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 2353, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8190, 4706] dup_rows_per_dst=[2, 12, 3476, 0, 0, 2, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8190, 4706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205643776 dispatch_meta_bytes=202136 combine_payload_bytes=205643776 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=205845912 ingress_bytes=281141248 link_bytes=281141248 token_bytes=411489688 weight_movement_bytes=75497472 total_nvlink_bytes=486987160 weight_to_token_ratio=0.183474 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.693757108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64604 payload_rows_remote=56412 dup_rows=932 dup_rows_remote=932 dedup_ratio=0.014221 router_tokens_per_expert=[4075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8192, 7428, 8192, 8192, 8192, 8064, 8192] dup_rows_per_dst=[0, 200, 0, 732, 0, 0, 0, 0] combine_rows_per_src=[8152, 8192, 7428, 8192, 8192, 8192, 8064, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63006 payload_rows_remote=54814 dup_rows=2530 dup_rows_remote=2524 dedup_ratio=0.038605 router_tokens_per_expert=[0, 0, 3928, 0, 0, 0, 53, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 3552, 0, 3, 0, 0, 3, 0, 55, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 1, 4093, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 499, 0, 0, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[7930, 8186, 7478, 8192, 8192, 8192, 8192, 6644] dup_rows_per_dst=[40, 220, 1586, 0, 8, 6, 0, 670] combine_rows_per_src=[7930, 8186, 7478, 8192, 8192, 8192, 8192, 6644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=219316224 dispatch_meta_bytes=214648 combine_payload_bytes=219316224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219530872 ingress_bytes=257064960 link_bytes=257064960 token_bytes=438847096 weight_movement_bytes=37748736 total_nvlink_bytes=476595832 weight_to_token_ratio=0.086018 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.463069963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63678 payload_rows_remote=55486 dup_rows=1858 dup_rows_remote=1246 dedup_ratio=0.028351 router_tokens_per_expert=[4072, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8192, 6756, 8192, 8192, 8188, 7822, 8192] dup_rows_per_dst=[6, 612, 0, 1228, 12, 0, 0, 0] combine_rows_per_src=[8144, 8192, 6756, 8192, 8192, 8188, 7822, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=210264064 dispatch_meta_bytes=222664 combine_payload_bytes=210264064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210486728 ingress_bytes=266887168 link_bytes=266887168 token_bytes=420750792 weight_movement_bytes=56623104 total_nvlink_bytes=477373896 weight_to_token_ratio=0.134576 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.557181527e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=220577792 dispatch_meta_bytes=229376 combine_payload_bytes=220577792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220807168 ingress_bytes=239452160 link_bytes=239452160 token_bytes=441384960 weight_movement_bytes=18874368 total_nvlink_bytes=460259328 weight_to_token_ratio=0.042762 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294312779e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=231063552 dispatch_meta_bytes=229376 combine_payload_bytes=231063552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231292928 ingress_bytes=249937920 link_bytes=249937920 token_bytes=462356480 weight_movement_bytes=18874368 total_nvlink_bytes=481230848 weight_to_token_ratio=0.040822 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394782172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57908 payload_rows_remote=49742 dup_rows=7628 dup_rows_remote=416 dedup_ratio=0.116394 router_tokens_per_expert=[4014, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 753, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8166, 8192, 6604, 2520, 8192, 8086, 7962, 8186] dup_rows_per_dst=[7212, 32, 0, 348, 32, 0, 0, 4] combine_rows_per_src=[8166, 8192, 6604, 2520, 8192, 8086, 7962, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=224518144 dispatch_meta_bytes=229352 combine_payload_bytes=224518144 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=224747496 ingress_bytes=243392512 link_bytes=243392512 token_bytes=449265640 weight_movement_bytes=18874368 total_nvlink_bytes=468140008 weight_to_token_ratio=0.042012 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332067293e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=227270656 dispatch_meta_bytes=226928 combine_payload_bytes=227270656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227497584 ingress_bytes=246145024 link_bytes=246145024 token_bytes=454768240 weight_movement_bytes=18874368 total_nvlink_bytes=473642608 weight_to_token_ratio=0.041503 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358440509e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57022 payload_rows_remote=48830 dup_rows=8514 dup_rows_remote=8514 dedup_ratio=0.129913 router_tokens_per_expert=[4062, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 5, 0, 0, 0, 14, 0, 156, 0, 3, 0, 1, 0, 0, 0, 0, 0, 1, 3919, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 136, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 6, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3881, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[1856, 6720, 7918, 8192, 8180, 8190, 7774, 8192] dup_rows_per_dst=[140, 344, 6206, 346, 0, 116, 1362, 0] combine_rows_per_src=[1856, 6720, 7918, 8192, 8180, 8190, 7774, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=203743232 dispatch_meta_bytes=200632 combine_payload_bytes=203743232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=203943864 ingress_bytes=260366336 link_bytes=260366336 token_bytes=407687096 weight_movement_bytes=56623104 total_nvlink_bytes=464310200 weight_to_token_ratio=0.138889 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.494702123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64424 payload_rows_remote=57026 dup_rows=1112 dup_rows_remote=1112 dedup_ratio=0.016968 router_tokens_per_expert=[3972, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[7944, 8192, 7398, 8192, 8192, 8192, 8122, 8192] dup_rows_per_dst=[8, 70, 0, 1034, 0, 0, 0, 0] combine_rows_per_src=[7944, 8192, 7398, 8192, 8192, 8192, 8122, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64792 payload_rows_remote=57258 dup_rows=744 dup_rows_remote=744 dedup_ratio=0.011353 router_tokens_per_expert=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 354, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 15, 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4066, 0, 0, 1, 6, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8134, 8182, 8192, 7534, 8186] dup_rows_per_dst=[0, 674, 56, 0, 0, 12, 0, 2] combine_rows_per_src=[8192, 8180, 8192, 8134, 8182, 8192, 7534, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200007680 dispatch_meta_bytes=229376 combine_payload_bytes=200007680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200237056 ingress_bytes=218882048 link_bytes=218882048 token_bytes=400244736 weight_movement_bytes=18874368 total_nvlink_bytes=419119104 weight_to_token_ratio=0.047157 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.097220087e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64796 payload_rows_remote=56604 dup_rows=740 dup_rows_remote=16 dedup_ratio=0.011292 router_tokens_per_expert=[3996, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8008, 8192, 7636, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[6, 8, 0, 724, 0, 2, 0, 0] combine_rows_per_src=[8008, 8192, 7636, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57344 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64494 payload_rows_remote=56308 dup_rows=1042 dup_rows_remote=1042 dedup_ratio=0.015900 router_tokens_per_expert=[3822, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 386, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4038, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0] payload_rows_per_dst=[7644, 8192, 7832, 8192, 8174, 8186, 8082, 8192] dup_rows_per_dst=[6, 62, 0, 970, 0, 0, 2, 2] combine_rows_per_src=[7644, 8192, 7832, 8192, 8174, 8186, 8082, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=233578496 dispatch_meta_bytes=232552 combine_payload_bytes=233578496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233811048 ingress_bytes=252452864 link_bytes=252452864 token_bytes=467389544 weight_movement_bytes=18874368 total_nvlink_bytes=486263912 weight_to_token_ratio=0.040383 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418879128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57270 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8150, 8160, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 34, 0, 0, 0, 34, 2] combine_rows_per_src=[8192, 8192, 8192, 8150, 8160, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=232008 combine_payload_bytes=234528768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234760776 ingress_bytes=253403136 link_bytes=253403136 token_bytes=469289544 weight_movement_bytes=18874368 total_nvlink_bytes=488163912 weight_to_token_ratio=0.040219 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427984167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=231849984 dispatch_meta_bytes=226480 combine_payload_bytes=231849984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232076464 ingress_bytes=250724352 link_bytes=250724352 token_bytes=463926448 weight_movement_bytes=18874368 total_nvlink_bytes=482800816 weight_to_token_ratio=0.040684 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402317377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229424 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235110448 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991472 weight_movement_bytes=94371840 total_nvlink_bytes=564363312 weight_to_token_ratio=0.200795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65180 payload_rows_remote=56988 dup_rows=356 dup_rows_remote=318 dedup_ratio=0.005432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4092, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8184, 7972, 8086, 8174] dup_rows_per_dst=[38, 0, 20, 192, 102, 2, 2, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8184, 7972, 8086, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=230637568 dispatch_meta_bytes=229400 combine_payload_bytes=230637568 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=230866968 ingress_bytes=325009408 link_bytes=325009408 token_bytes=461504536 weight_movement_bytes=94371840 total_nvlink_bytes=555876376 weight_to_token_ratio=0.204487 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.114080233e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229376 combine_payload_bytes=234577920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234807296 ingress_bytes=253452288 link_bytes=253452288 token_bytes=469385216 weight_movement_bytes=18874368 total_nvlink_bytes=488259584 weight_to_token_ratio=0.040211 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428455117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65202 payload_rows_remote=57014 dup_rows=334 dup_rows_remote=316 dedup_ratio=0.005096 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4088, 0, 4089, 0, 0, 0, 0, 4068, 0, 6, 0, 0, 4, 0, 4090, 0, 4090, 0, 6, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 3, 4, 0, 4088, 5, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8118, 8092, 8122, 8136, 8192, 8180, 8188] dup_rows_per_dst=[0, 20, 40, 0, 70, 132, 54, 18] combine_rows_per_src=[8174, 8118, 8092, 8122, 8136, 8192, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233422848 dispatch_meta_bytes=229224 combine_payload_bytes=233422848 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233652072 ingress_bytes=271171584 link_bytes=271171584 token_bytes=467074920 weight_movement_bytes=37748736 total_nvlink_bytes=504823656 weight_to_token_ratio=0.080819 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.598232693e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=62 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 4093, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 3, 0, 4093, 0, 4093, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8148, 8184, 8192, 8186, 8192] dup_rows_per_dst=[0, 10, 2, 0, 0, 18, 34, 0] combine_rows_per_src=[8186, 8192, 8192, 8148, 8184, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=233529344 dispatch_meta_bytes=229320 combine_payload_bytes=233529344 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233758664 ingress_bytes=252403712 link_bytes=252403712 token_bytes=467288008 weight_movement_bytes=18874368 total_nvlink_bytes=486162376 weight_to_token_ratio=0.040391 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418408178e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57662 payload_rows_remote=49472 dup_rows=7874 dup_rows_remote=7864 dedup_ratio=0.120148 router_tokens_per_expert=[0, 0, 4076, 0, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 0, 0, 0, 0, 158, 0, 7, 0, 0, 7, 0, 30, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 2, 4089, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4088, 0, 0, 0, 3902, 0, 0, 0, 0, 4089, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8178, 390, 8192, 8192, 8192, 8190, 8174] dup_rows_per_dst=[28, 6, 16, 2, 4, 14, 10, 7794] combine_rows_per_src=[8154, 8178, 390, 8192, 8192, 8192, 8190, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57330 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8168, 8192, 8192, 8192] dup_rows_per_dst=[0, 10, 28, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8168, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=16 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 4093, 0, 0, 0, 0, 4085, 0, 3, 0, 2, 3, 0, 4093, 0, 4093, 0, 3, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8180, 8170, 8192, 8186, 8192] dup_rows_per_dst=[2, 4, 0, 0, 0, 30, 10, 0] combine_rows_per_src=[8186, 8192, 8192, 8180, 8170, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229368 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234848248 ingress_bytes=253493248 link_bytes=253493248 token_bytes=469467128 weight_movement_bytes=18874368 total_nvlink_bytes=488341496 weight_to_token_ratio=0.040204 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428847576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=202637312 dispatch_meta_bytes=229344 combine_payload_bytes=202637312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202866656 ingress_bytes=221511680 link_bytes=221511680 token_bytes=405503968 weight_movement_bytes=18874368 total_nvlink_bytes=424378336 weight_to_token_ratio=0.046545 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.122415927e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229472 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235053152 ingress_bytes=272572416 link_bytes=272572416 token_bytes=469876832 weight_movement_bytes=37748736 total_nvlink_bytes=507625568 weight_to_token_ratio=0.080338 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611654776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=32 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[8192, 8184, 7994, 8192, 8192, 8192, 8192, 8160] dup_rows_per_dst=[206, 0, 0, 0, 0, 0, 32, 0] combine_rows_per_src=[8192, 8184, 7994, 8192, 8192, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229256 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234921864 ingress_bytes=310190080 link_bytes=310190080 token_bytes=469614472 weight_movement_bytes=75497472 total_nvlink_bytes=545111944 weight_to_token_ratio=0.160765 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972088724e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65414 payload_rows_remote=57236 dup_rows=122 dup_rows_remote=78 dedup_ratio=0.001862 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4090, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 6, 4089, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4083, 0, 0, 4090] payload_rows_per_dst=[8170, 8144, 8190, 8182, 8178, 8192, 8180, 8178] dup_rows_per_dst=[12, 10, 2, 36, 16, 0, 2, 44] combine_rows_per_src=[8170, 8144, 8190, 8182, 8178, 8192, 8180, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=228552 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234134728 ingress_bytes=290529280 link_bytes=290529280 token_bytes=468040904 weight_movement_bytes=56623104 total_nvlink_bytes=524664008 weight_to_token_ratio=0.120979 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.783708612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57288 dup_rows=56 dup_rows_remote=24 dedup_ratio=0.000854 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4093, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 4093] payload_rows_per_dst=[8152, 8186, 8192, 8190, 8192, 8192, 8186, 8190] dup_rows_per_dst=[4, 10, 32, 0, 6, 0, 0, 4] combine_rows_per_src=[8152, 8186, 8192, 8190, 8192, 8192, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65024 payload_rows_remote=56872 dup_rows=512 dup_rows_remote=510 dedup_ratio=0.007812 router_tokens_per_expert=[4089, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 69, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 46, 0, 0, 0, 6, 0, 0, 7, 0, 0, 0, 0, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 4089, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4069, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 7748, 8192, 8178, 8192, 8152, 8192] dup_rows_per_dst=[66, 144, 10, 290, 0, 0, 2, 0] combine_rows_per_src=[8178, 8192, 7748, 8192, 8178, 8192, 8152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59120 payload_rows_remote=50936 dup_rows=6416 dup_rows_remote=6416 dedup_ratio=0.097900 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4090, 3419, 0, 0, 0, 0, 0, 2520, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 5142, 8184, 7858, 5180, 8192] dup_rows_per_dst=[0, 26, 0, 672, 0, 5508, 208, 2] combine_rows_per_src=[8192, 8192, 8180, 5142, 8184, 7858, 5180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234438656 dispatch_meta_bytes=229256 combine_payload_bytes=234438656 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234667912 ingress_bytes=309936128 link_bytes=309936128 token_bytes=469106568 weight_movement_bytes=75497472 total_nvlink_bytes=544604040 weight_to_token_ratio=0.160939 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.969655480e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57152 dup_rows=192 dup_rows_remote=192 dedup_ratio=0.002930 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 4093, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 69, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 4093] payload_rows_per_dst=[8016, 8186, 8192, 8188, 8192, 8192, 8186, 8192] dup_rows_per_dst=[6, 42, 138, 0, 6, 0, 0, 0] combine_rows_per_src=[8016, 8186, 8192, 8188, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62426 payload_rows_remote=54234 dup_rows=3110 dup_rows_remote=3102 dedup_ratio=0.047455 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 4085, 3387, 0, 0, 0, 0, 0, 3414, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 8192, 7886, 6774, 6828, 8192] dup_rows_per_dst=[200, 8, 0, 2900, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8170, 8192, 7886, 6774, 6828, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234651648 dispatch_meta_bytes=229248 combine_payload_bytes=234651648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234880896 ingress_bytes=253526016 link_bytes=253526016 token_bytes=469532544 weight_movement_bytes=18874368 total_nvlink_bytes=488406912 weight_to_token_ratio=0.040198 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429161543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=232947712 dispatch_meta_bytes=229528 combine_payload_bytes=232947712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233177240 ingress_bytes=251822080 link_bytes=251822080 token_bytes=466124952 weight_movement_bytes=18874368 total_nvlink_bytes=484999320 weight_to_token_ratio=0.040492 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412835266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=208633856 dispatch_meta_bytes=229408 combine_payload_bytes=208633856 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208863264 ingress_bytes=246382592 link_bytes=246382592 token_bytes=417497120 weight_movement_bytes=37748736 total_nvlink_bytes=455245856 weight_to_token_ratio=0.090417 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.360716769e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57660 payload_rows_remote=49468 dup_rows=7876 dup_rows_remote=7820 dedup_ratio=0.120178 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4085, 3608, 0, 0, 0, 0, 0, 2046, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8186, 5936, 4092, 6682] dup_rows_per_dst=[56, 68, 102, 6486, 1164, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8186, 5936, 4092, 6682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229376 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234323968 ingress_bytes=252968960 link_bytes=252968960 token_bytes=468418560 weight_movement_bytes=18874368 total_nvlink_bytes=487292928 weight_to_token_ratio=0.040294 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423824106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=222142464 dispatch_meta_bytes=229344 combine_payload_bytes=222142464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222371808 ingress_bytes=241016832 link_bytes=241016832 token_bytes=444514272 weight_movement_bytes=18874368 total_nvlink_bytes=463388640 weight_to_token_ratio=0.042461 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.309304697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59050 payload_rows_remote=50858 dup_rows=6486 dup_rows_remote=5840 dedup_ratio=0.098969 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1875, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 3907, 4089, 3845, 0, 0, 4, 0, 0, 2462, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[7536, 7660, 8188, 3754, 7826, 7702, 8192, 8192] dup_rows_per_dst=[302, 60, 0, 0, 12, 2, 5464, 646] combine_rows_per_src=[7536, 7660, 8188, 3754, 7826, 7702, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202620928 dispatch_meta_bytes=229152 combine_payload_bytes=202620928 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=202850080 ingress_bytes=221495296 link_bytes=221495296 token_bytes=405471008 weight_movement_bytes=18874368 total_nvlink_bytes=424345376 weight_to_token_ratio=0.046549 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.122258944e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62088 payload_rows_remote=53920 dup_rows=3448 dup_rows_remote=3448 dedup_ratio=0.052612 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4043, 4081, 3962, 0, 0, 3, 0, 0, 2568, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8168, 8186, 8092, 7930, 5142, 8192] dup_rows_per_dst=[58, 270, 0, 3070, 6, 0, 6, 38] combine_rows_per_src=[8186, 8192, 8168, 8186, 8092, 7930, 5142, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61714 payload_rows_remote=53522 dup_rows=3822 dup_rows_remote=10 dedup_ratio=0.058319 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3834, 0, 0, 0, 0, 0, 2447, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7668, 4894, 8192] dup_rows_per_dst=[0, 10, 0, 3812, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7668, 4894, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57288 dup_rows=70 dup_rows_remote=60 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4089, 0, 4089, 0, 0, 0, 0, 4087, 0, 7, 0, 0, 4, 0, 4089, 0, 4089, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8168, 8174, 8192, 8178, 8192] dup_rows_per_dst=[0, 18, 4, 0, 0, 36, 10, 2] combine_rows_per_src=[8178, 8192, 8192, 8168, 8174, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64522 payload_rows_remote=57182 dup_rows=1014 dup_rows_remote=1014 dedup_ratio=0.015472 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 10, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3670, 0, 4096, 0, 0, 0, 4076, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8120, 8184, 7340, 8154, 8192, 8148] dup_rows_per_dst=[0, 0, 130, 872, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8120, 8184, 7340, 8154, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=208314368 dispatch_meta_bytes=226792 combine_payload_bytes=208314368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=208541160 ingress_bytes=246063104 link_bytes=246063104 token_bytes=416855528 weight_movement_bytes=37748736 total_nvlink_bytes=454604264 weight_to_token_ratio=0.090556 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357655592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64400 payload_rows_remote=56208 dup_rows=1136 dup_rows_remote=1136 dedup_ratio=0.017334 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 298, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 3, 0, 0, 0, 11, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 3760, 0, 3893, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 34, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 7522, 8164, 7786, 8170] dup_rows_per_dst=[0, 0, 666, 466, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8182, 8192, 7522, 8164, 7786, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=220856320 dispatch_meta_bytes=229472 combine_payload_bytes=220856320 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=221085792 ingress_bytes=258605056 link_bytes=258605056 token_bytes=441942112 weight_movement_bytes=37748736 total_nvlink_bytes=479690848 weight_to_token_ratio=0.085416 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.477826405e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61298 payload_rows_remote=53190 dup_rows=4238 dup_rows_remote=4236 dedup_ratio=0.064667 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 4011, 4075, 4051, 0, 0, 3, 0, 0, 1658, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8156, 4244, 8028, 8108, 8192, 8192] dup_rows_per_dst=[0, 88, 0, 0, 4, 2, 3322, 822] combine_rows_per_src=[8186, 8192, 8156, 4244, 8028, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=219226112 dispatch_meta_bytes=214128 combine_payload_bytes=219226112 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219440240 ingress_bytes=275849216 link_bytes=275849216 token_bytes=438666352 weight_movement_bytes=56623104 total_nvlink_bytes=495289456 weight_to_token_ratio=0.129080 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.643051461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234651648 dispatch_meta_bytes=229392 combine_payload_bytes=234651648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234881040 ingress_bytes=253526016 link_bytes=253526016 token_bytes=469532688 weight_movement_bytes=18874368 total_nvlink_bytes=488407056 weight_to_token_ratio=0.040198 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429161543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234217472 dispatch_meta_bytes=232784 combine_payload_bytes=234217472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234450256 ingress_bytes=271966208 link_bytes=271966208 token_bytes=468667728 weight_movement_bytes=37748736 total_nvlink_bytes=506416464 weight_to_token_ratio=0.080545 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605846389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64786 payload_rows_remote=56594 dup_rows=750 dup_rows_remote=750 dedup_ratio=0.011444 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 111, 0, 0, 91, 0, 0, 0, 0, 0, 0, 6, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3925, 0, 4020, 0, 0, 0, 4043, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 7850, 8094, 8044, 8032] dup_rows_per_dst=[0, 34, 532, 178, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 7850, 8094, 8044, 8032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230227968 dispatch_meta_bytes=229376 combine_payload_bytes=230227968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230457344 ingress_bytes=249102336 link_bytes=249102336 token_bytes=460685312 weight_movement_bytes=18874368 total_nvlink_bytes=479559680 weight_to_token_ratio=0.040970 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.386776018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=217866240 dispatch_meta_bytes=229704 combine_payload_bytes=217866240 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=218095944 ingress_bytes=312238080 link_bytes=312238080 token_bytes=435962184 weight_movement_bytes=94371840 total_nvlink_bytes=530334024 weight_to_token_ratio=0.216468 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.991711652e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231809024 dispatch_meta_bytes=229376 combine_payload_bytes=231809024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232038400 ingress_bytes=250683392 link_bytes=250683392 token_bytes=463847424 weight_movement_bytes=18874368 total_nvlink_bytes=482721792 weight_to_token_ratio=0.040691 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401924918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58672 payload_rows_remote=50508 dup_rows=6864 dup_rows_remote=6764 dedup_ratio=0.104736 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 218, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 20, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 6, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 4090, 4056, 0, 3988, 0, 0, 0, 3955, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4067, 6, 0, 0, 6, 0, 0, 0, 41, 6] payload_rows_per_dst=[8192, 8178, 4674, 5416, 8146, 7926, 7976, 8164] dup_rows_per_dst=[0, 2, 434, 594, 2962, 1830, 942, 100] combine_rows_per_src=[8192, 8178, 4674, 5416, 8146, 7926, 7976, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64756 payload_rows_remote=56652 dup_rows=780 dup_rows_remote=376 dedup_ratio=0.011902 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 166, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 4001, 0, 4066, 0, 0, 0, 3877, 0, 3, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 3, 0, 0, 3, 0, 0, 0, 5, 3] payload_rows_per_dst=[8192, 8186, 8104, 8180, 8008, 7770, 8132, 8184] dup_rows_per_dst=[0, 0, 404, 334, 6, 10, 0, 26] combine_rows_per_src=[8192, 8186, 8104, 8180, 8008, 7770, 8132, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63094 payload_rows_remote=54902 dup_rows=2442 dup_rows_remote=2442 dedup_ratio=0.037262 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5778, 8192, 8192, 8192, 8192, 8164, 8192] dup_rows_per_dst=[2324, 0, 92, 0, 0, 0, 0, 26] combine_rows_per_src=[8192, 5778, 8192, 8192, 8192, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65108 payload_rows_remote=56990 dup_rows=428 dup_rows_remote=316 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 72, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3980, 0, 4095, 0, 0, 0, 4034, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8118, 7960, 8080, 8190, 8184] dup_rows_per_dst=[0, 2, 284, 112, 0, 30, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8118, 7960, 8080, 8190, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57320 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4089, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4088, 0, 0, 4089] payload_rows_per_dst=[8190, 8178, 8192, 8186, 8192, 8192, 8178, 8190] dup_rows_per_dst=[14, 4, 0, 0, 14, 0, 0, 6] combine_rows_per_src=[8190, 8178, 8192, 8186, 8192, 8192, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=206880768 dispatch_meta_bytes=229088 combine_payload_bytes=206880768 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=207109856 ingress_bytes=244629504 link_bytes=244629504 token_bytes=413990624 weight_movement_bytes=37748736 total_nvlink_bytes=451739360 weight_to_token_ratio=0.091183 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.343919542e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=232046592 dispatch_meta_bytes=228112 combine_payload_bytes=232046592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232274704 ingress_bytes=288669696 link_bytes=288669696 token_bytes=464321296 weight_movement_bytes=56623104 total_nvlink_bytes=520944400 weight_to_token_ratio=0.121948 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.765890993e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62500 payload_rows_remote=57096 dup_rows=3036 dup_rows_remote=3036 dedup_ratio=0.046326 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 358, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5404, 8192, 8192, 8192, 8192, 7944, 8192] dup_rows_per_dst=[1970, 0, 332, 2, 16, 0, 0, 716] combine_rows_per_src=[8192, 5404, 8192, 8192, 8192, 8192, 7944, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63918 payload_rows_remote=56250 dup_rows=1618 dup_rows_remote=1614 dedup_ratio=0.024689 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3796, 0, 0, 0, 0, 0, 92, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 3969, 0, 4062, 0, 0, 0, 3831, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4082, 3, 0, 0, 3, 0, 0, 0, 9, 3] payload_rows_per_dst=[8192, 8184, 8060, 7572, 7944, 7668, 8124, 8174] dup_rows_per_dst=[0, 4, 1304, 268, 6, 4, 0, 32] combine_rows_per_src=[8192, 8184, 8060, 7572, 7944, 7668, 8124, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224878592 dispatch_meta_bytes=229376 combine_payload_bytes=224878592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225107968 ingress_bytes=262627328 link_bytes=262627328 token_bytes=449986560 weight_movement_bytes=37748736 total_nvlink_bytes=487735296 weight_to_token_ratio=0.083889 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.516365836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233431040 dispatch_meta_bytes=229224 combine_payload_bytes=233431040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233660264 ingress_bytes=271179776 link_bytes=271179776 token_bytes=467091304 weight_movement_bytes=37748736 total_nvlink_bytes=504840040 weight_to_token_ratio=0.080817 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.598311185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229432 combine_payload_bytes=234782720 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235012152 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794872 weight_movement_bytes=37748736 total_nvlink_bytes=507543608 weight_to_token_ratio=0.080352 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58548 payload_rows_remote=50356 dup_rows=6988 dup_rows_remote=636 dedup_ratio=0.106628 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 50, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4826, 8192, 8192, 8192, 8188, 8098, 4668] dup_rows_per_dst=[6352, 0, 594, 40, 2, 0, 0, 0] combine_rows_per_src=[8192, 4826, 8192, 8192, 8192, 8188, 8098, 4668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233865216 dispatch_meta_bytes=240528 combine_payload_bytes=233865216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234105744 ingress_bytes=252739584 link_bytes=252739584 token_bytes=467970960 weight_movement_bytes=18874368 total_nvlink_bytes=486845328 weight_to_token_ratio=0.040332 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421626338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230400000 dispatch_meta_bytes=231456 combine_payload_bytes=230400000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=230631456 ingress_bytes=287023104 link_bytes=287023104 token_bytes=461031456 weight_movement_bytes=56623104 total_nvlink_bytes=517654560 weight_to_token_ratio=0.122818 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.750114158e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=206258176 dispatch_meta_bytes=203968 combine_payload_bytes=206258176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=206462144 ingress_bytes=244006912 link_bytes=244006912 token_bytes=412720320 weight_movement_bytes=37748736 total_nvlink_bytes=450469056 weight_to_token_ratio=0.091463 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337954172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62168 payload_rows_remote=53982 dup_rows=3368 dup_rows_remote=3222 dedup_ratio=0.051392 router_tokens_per_expert=[0, 0, 3, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2486, 1, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4978, 8186, 8186, 8192, 8192, 8050, 8192] dup_rows_per_dst=[3126, 2, 146, 0, 4, 0, 0, 90] combine_rows_per_src=[8192, 4978, 8186, 8186, 8192, 8192, 8050, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56972 payload_rows_remote=48780 dup_rows=8564 dup_rows_remote=6532 dedup_ratio=0.130676 router_tokens_per_expert=[0, 0, 6, 0, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 10, 0, 4090, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4078, 0, 0, 0, 696, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[1626, 8192, 6424, 8176, 8126, 8192, 8044, 8192] dup_rows_per_dst=[12, 5158, 156, 0, 32, 4, 1170, 2032] combine_rows_per_src=[1626, 8192, 6424, 8176, 8126, 8192, 8044, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64176 payload_rows_remote=55984 dup_rows=1360 dup_rows_remote=1360 dedup_ratio=0.020752 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8184, 8192, 6846] dup_rows_per_dst=[0, 1330, 0, 0, 0, 0, 26, 4] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8184, 8192, 6846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62232 payload_rows_remote=54040 dup_rows=3304 dup_rows_remote=3304 dedup_ratio=0.050415 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5008, 8192, 8192, 8192, 8192, 8072, 8192] dup_rows_per_dst=[2858, 0, 376, 0, 0, 0, 0, 70] combine_rows_per_src=[8192, 5008, 8192, 8192, 8192, 8192, 8072, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59632 payload_rows_remote=51440 dup_rows=5904 dup_rows_remote=80 dedup_ratio=0.090088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1346, 0, 0, 7, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3965, 4089, 4017, 0, 0, 7, 0, 0, 2905, 4089, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 2694, 7944, 8048, 8192, 8192] dup_rows_per_dst=[66, 2, 0, 0, 12, 0, 5824, 0] combine_rows_per_src=[8178, 8192, 8192, 2694, 7944, 8048, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=221110272 dispatch_meta_bytes=228816 combine_payload_bytes=221110272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221339088 ingress_bytes=239984640 link_bytes=239984640 token_bytes=442449360 weight_movement_bytes=18874368 total_nvlink_bytes=461323728 weight_to_token_ratio=0.042659 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.299414741e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64214 payload_rows_remote=56022 dup_rows=1322 dup_rows_remote=220 dedup_ratio=0.020172 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 7974, 8192, 7090] dup_rows_per_dst=[0, 1102, 28, 0, 0, 0, 192, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 7974, 8192, 7090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=199802880 dispatch_meta_bytes=221248 combine_payload_bytes=199802880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200024128 ingress_bytes=237551616 link_bytes=237551616 token_bytes=399827008 weight_movement_bytes=37748736 total_nvlink_bytes=437575744 weight_to_token_ratio=0.094413 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276102702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229310464 dispatch_meta_bytes=229376 combine_payload_bytes=229310464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229539840 ingress_bytes=248184832 link_bytes=248184832 token_bytes=458850304 weight_movement_bytes=18874368 total_nvlink_bytes=477724672 weight_to_token_ratio=0.041134 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.377984946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59662 payload_rows_remote=51470 dup_rows=5874 dup_rows_remote=5874 dedup_ratio=0.089630 router_tokens_per_expert=[0, 0, 3, 0, 0, 3, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3830, 3, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 70, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[2880, 7826, 8186, 8186, 8192, 8192, 8008, 8192] dup_rows_per_dst=[200, 5454, 80, 0, 0, 0, 0, 140] combine_rows_per_src=[2880, 7826, 8186, 8186, 8192, 8192, 8008, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=221347840 dispatch_meta_bytes=229376 combine_payload_bytes=221347840 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=221577216 ingress_bytes=259096576 link_bytes=259096576 token_bytes=442925056 weight_movement_bytes=37748736 total_nvlink_bytes=480673792 weight_to_token_ratio=0.085226 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.482535908e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64610 payload_rows_remote=56418 dup_rows=926 dup_rows_remote=838 dedup_ratio=0.014130 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0] payload_rows_per_dst=[8192, 8134, 8190, 8192, 8156, 8162, 8192, 7392] dup_rows_per_dst=[88, 796, 0, 0, 0, 0, 38, 4] combine_rows_per_src=[8192, 8134, 8190, 8192, 8156, 8162, 8192, 7392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=210698240 dispatch_meta_bytes=206080 combine_payload_bytes=210698240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210904320 ingress_bytes=248446976 link_bytes=248446976 token_bytes=421602560 weight_movement_bytes=37748736 total_nvlink_bytes=459351296 weight_to_token_ratio=0.089536 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.380496680e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229466112 dispatch_meta_bytes=224968 combine_payload_bytes=229466112 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=229691080 ingress_bytes=304963584 link_bytes=304963584 token_bytes=459157192 weight_movement_bytes=75497472 total_nvlink_bytes=534654664 weight_to_token_ratio=0.164426 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.922011010e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=210821120 dispatch_meta_bytes=229376 combine_payload_bytes=210821120 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=211050496 ingress_bytes=229695488 link_bytes=229695488 token_bytes=421871616 weight_movement_bytes=18874368 total_nvlink_bytes=440745984 weight_to_token_ratio=0.044740 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.200829149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231088128 dispatch_meta_bytes=229024 combine_payload_bytes=231088128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231317152 ingress_bytes=268836864 link_bytes=268836864 token_bytes=462405280 weight_movement_bytes=37748736 total_nvlink_bytes=500154016 weight_to_token_ratio=0.081636 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.575862555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64000 payload_rows_remote=55814 dup_rows=1536 dup_rows_remote=1528 dedup_ratio=0.023438 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 4090, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 35, 3, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 103, 0] payload_rows_per_dst=[8186, 8192, 8186, 8186, 8192, 7884, 8192, 6982] dup_rows_per_dst=[0, 1248, 8, 0, 6, 0, 76, 198] combine_rows_per_src=[8186, 8192, 8186, 8186, 8192, 7884, 8192, 6982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65330 payload_rows_remote=57318 dup_rows=206 dup_rows_remote=206 dedup_ratio=0.003143 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 8186, 8012, 8190, 8192, 8192] dup_rows_per_dst=[206, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8174, 8186, 8012, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56714 payload_rows_remote=48522 dup_rows=8822 dup_rows_remote=390 dedup_ratio=0.134613 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 4090, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 153, 6, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4090, 29, 0] payload_rows_per_dst=[8192, 662, 8148, 8180, 8190, 7728, 7422, 8192] dup_rows_per_dst=[0, 0, 28, 32, 12, 0, 318, 8432] combine_rows_per_src=[8192, 662, 8148, 8180, 8190, 7728, 7422, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64646 payload_rows_remote=56454 dup_rows=890 dup_rows_remote=890 dedup_ratio=0.013580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8154, 8192, 7340] dup_rows_per_dst=[0, 848, 0, 0, 0, 0, 14, 28] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8154, 8192, 7340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64600 payload_rows_remote=56450 dup_rows=936 dup_rows_remote=936 dedup_ratio=0.014282 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 54, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 3882, 0, 4075, 0, 0, 0, 3887, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4088, 7, 0, 0, 7, 0, 0, 0, 43, 7] payload_rows_per_dst=[8192, 8178, 8146, 8178, 7778, 7788, 8150, 8190] dup_rows_per_dst=[0, 0, 112, 682, 14, 0, 0, 128] combine_rows_per_src=[8192, 8178, 8146, 8178, 7778, 7788, 8150, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65296 payload_rows_remote=57104 dup_rows=240 dup_rows_remote=240 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7952, 8192, 8192, 8192] dup_rows_per_dst=[240, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7952, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=228614144 dispatch_meta_bytes=229368 combine_payload_bytes=228614144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=228843512 ingress_bytes=266362880 link_bytes=266362880 token_bytes=457457656 weight_movement_bytes=37748736 total_nvlink_bytes=495206392 weight_to_token_ratio=0.082519 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.552158057e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=230096 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235004624 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469779152 weight_movement_bytes=18874368 total_nvlink_bytes=488653520 weight_to_token_ratio=0.040177 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=198746112 dispatch_meta_bytes=195648 combine_payload_bytes=198746112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=198941760 ingress_bytes=236494848 link_bytes=236494848 token_bytes=397687872 weight_movement_bytes=37748736 total_nvlink_bytes=435436608 weight_to_token_ratio=0.094921 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265977271e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63180 payload_rows_remote=55482 dup_rows=2356 dup_rows_remote=2356 dedup_ratio=0.035950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 4056, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 38, 0] payload_rows_per_dst=[8118, 8192, 8186, 8186, 8192, 7698, 8192, 6416] dup_rows_per_dst=[0, 2244, 8, 4, 6, 0, 18, 76] combine_rows_per_src=[8118, 8192, 8186, 8186, 8192, 7698, 8192, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231235584 dispatch_meta_bytes=229376 combine_payload_bytes=231235584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231464960 ingress_bytes=250109952 link_bytes=250109952 token_bytes=462700544 weight_movement_bytes=18874368 total_nvlink_bytes=481574912 weight_to_token_ratio=0.040792 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396430498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65414 payload_rows_remote=57222 dup_rows=122 dup_rows_remote=16 dedup_ratio=0.001862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8088, 8174, 8192, 8192] dup_rows_per_dst=[106, 0, 0, 8, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 8088, 8174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231219200 dispatch_meta_bytes=229544 combine_payload_bytes=231219200 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231448744 ingress_bytes=268967936 link_bytes=268967936 token_bytes=462667944 weight_movement_bytes=37748736 total_nvlink_bytes=500416680 weight_to_token_ratio=0.081589 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.577118422e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233897984 dispatch_meta_bytes=229376 combine_payload_bytes=233897984 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234127360 ingress_bytes=271646720 link_bytes=271646720 token_bytes=468025344 weight_movement_bytes=37748736 total_nvlink_bytes=505774080 weight_to_token_ratio=0.080655 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602785213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227254272 dispatch_meta_bytes=231352 combine_payload_bytes=227254272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227485624 ingress_bytes=246128640 link_bytes=246128640 token_bytes=454739896 weight_movement_bytes=18874368 total_nvlink_bytes=473614264 weight_to_token_ratio=0.041506 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358283526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234381312 dispatch_meta_bytes=228952 combine_payload_bytes=234381312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234610264 ingress_bytes=253255680 link_bytes=253255680 token_bytes=468991576 weight_movement_bytes=18874368 total_nvlink_bytes=487865944 weight_to_token_ratio=0.040245 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426571316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61394 payload_rows_remote=53202 dup_rows=4142 dup_rows_remote=4142 dedup_ratio=0.063202 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 4050, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4142, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 4050, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65324 payload_rows_remote=57134 dup_rows=212 dup_rows_remote=212 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8190, 8192, 8002, 8180, 8192, 8190] dup_rows_per_dst=[198, 8, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8190, 8192, 8002, 8180, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63112 payload_rows_remote=54920 dup_rows=2424 dup_rows_remote=2422 dedup_ratio=0.036987 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 4090, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 6, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5924, 8146, 8192, 8144, 8146, 8176, 8192, 8192] dup_rows_per_dst=[46, 12, 38, 20, 2218, 88, 0, 2] combine_rows_per_src=[5924, 8146, 8192, 8144, 8146, 8176, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57198 dup_rows=146 dup_rows_remote=146 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 8192, 8082, 8174, 8192, 8192] dup_rows_per_dst=[146, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8174, 8192, 8082, 8174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60324 payload_rows_remote=52338 dup_rows=5212 dup_rows_remote=5212 dedup_ratio=0.079529 router_tokens_per_expert=[0, 0, 7, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2357, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4088, 0, 0, 0, 225, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[3214, 8192, 8178, 8178, 8192, 8192, 7986, 8192] dup_rows_per_dst=[14, 4718, 18, 0, 12, 0, 0, 450] combine_rows_per_src=[3214, 8192, 8178, 8178, 8192, 8192, 7986, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61904 payload_rows_remote=53712 dup_rows=3632 dup_rows_remote=3628 dedup_ratio=0.055420 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8192, 8192, 8192, 8026, 4742, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 3448, 180, 0] combine_rows_per_src=[8176, 8192, 8192, 8192, 8192, 8026, 4742, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=217915392 dispatch_meta_bytes=229376 combine_payload_bytes=217915392 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=218144768 ingress_bytes=255664128 link_bytes=255664128 token_bytes=436060160 weight_movement_bytes=37748736 total_nvlink_bytes=473808896 weight_to_token_ratio=0.086568 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.449647880e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229384 combine_payload_bytes=234020864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234250248 ingress_bytes=252895232 link_bytes=252895232 token_bytes=468271112 weight_movement_bytes=18874368 total_nvlink_bytes=487145480 weight_to_token_ratio=0.040306 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423117681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=224952320 dispatch_meta_bytes=229368 combine_payload_bytes=224952320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225181688 ingress_bytes=243826688 link_bytes=243826688 token_bytes=450134008 weight_movement_bytes=18874368 total_nvlink_bytes=469008376 weight_to_token_ratio=0.041931 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.336227354e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64202 payload_rows_remote=56018 dup_rows=1334 dup_rows_remote=1334 dedup_ratio=0.020355 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 4092, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3437, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8190, 6880, 8184, 8192, 8186] dup_rows_per_dst=[1316, 10, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8190, 6880, 8184, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229376 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234512384 ingress_bytes=328654848 link_bytes=328654848 token_bytes=468795392 weight_movement_bytes=94371840 total_nvlink_bytes=563167232 weight_to_token_ratio=0.201307 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.149009045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58476 payload_rows_remote=50284 dup_rows=7060 dup_rows_remote=7060 dedup_ratio=0.107727 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8138, 8192, 8192, 1186, 8192] dup_rows_per_dst=[0, 108, 0, 0, 0, 6952, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8138, 8192, 8192, 1186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=214376448 dispatch_meta_bytes=230200 combine_payload_bytes=214376448 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=214606648 ingress_bytes=308748288 link_bytes=308748288 token_bytes=428983096 weight_movement_bytes=94371840 total_nvlink_bytes=523354936 weight_to_token_ratio=0.219990 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.958274182e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=220004352 dispatch_meta_bytes=229360 combine_payload_bytes=220004352 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=220233712 ingress_bytes=295501824 link_bytes=295501824 token_bytes=440238064 weight_movement_bytes=75497472 total_nvlink_bytes=515735536 weight_to_token_ratio=0.171492 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.831353082e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229449728 dispatch_meta_bytes=229408 combine_payload_bytes=229449728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229679136 ingress_bytes=248324096 link_bytes=248324096 token_bytes=459128864 weight_movement_bytes=18874368 total_nvlink_bytes=478003232 weight_to_token_ratio=0.041109 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379319305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=205963264 dispatch_meta_bytes=229376 combine_payload_bytes=205963264 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=206192640 ingress_bytes=243712000 link_bytes=243712000 token_bytes=412155904 weight_movement_bytes=37748736 total_nvlink_bytes=449904640 weight_to_token_ratio=0.091588 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.335128470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62872 payload_rows_remote=54680 dup_rows=2664 dup_rows_remote=2664 dedup_ratio=0.040649 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2764, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 5528, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1460, 0, 0, 0, 0, 0, 1204] combine_rows_per_src=[8192, 8192, 8192, 5528, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65402 payload_rows_remote=57210 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.002045 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 34, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8070, 8186, 8192] dup_rows_per_dst=[6, 0, 0, 6, 0, 0, 122, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8070, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63036 payload_rows_remote=54844 dup_rows=2500 dup_rows_remote=2500 dedup_ratio=0.038147 router_tokens_per_expert=[0, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 4089, 0, 0, 3, 4090, 0, 3, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 2930, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1150, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8054, 8192, 8192, 8180, 5872, 8180, 8192] dup_rows_per_dst=[12, 12, 0, 28, 38, 4, 2406, 0] combine_rows_per_src=[8174, 8054, 8192, 8192, 8180, 5872, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65012 payload_rows_remote=56820 dup_rows=524 dup_rows_remote=524 dedup_ratio=0.007996 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7668, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 520, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7668, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61612 payload_rows_remote=53420 dup_rows=3924 dup_rows_remote=3836 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 4089, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 1539, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 39, 7, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 1, 0] payload_rows_per_dst=[8192, 6698, 8178, 8178, 8192, 7968, 8192, 6014] dup_rows_per_dst=[0, 1514, 0, 2, 14, 1784, 88, 522] combine_rows_per_src=[8192, 6698, 8178, 8178, 8192, 7968, 8192, 6014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=223969280 dispatch_meta_bytes=229376 combine_payload_bytes=223969280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224198656 ingress_bytes=242843648 link_bytes=242843648 token_bytes=448167936 weight_movement_bytes=18874368 total_nvlink_bytes=467042304 weight_to_token_ratio=0.042114 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.326808349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58646 payload_rows_remote=50454 dup_rows=6890 dup_rows_remote=5020 dedup_ratio=0.105133 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 7572, 7538, 8190, 8192, 2578] dup_rows_per_dst=[0, 1870, 0, 4372, 630, 0, 18, 0] combine_rows_per_src=[8192, 8192, 8192, 7572, 7538, 8190, 8192, 2578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234332160 dispatch_meta_bytes=229376 combine_payload_bytes=234332160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234561536 ingress_bytes=253206528 link_bytes=253206528 token_bytes=468893696 weight_movement_bytes=18874368 total_nvlink_bytes=487768064 weight_to_token_ratio=0.040253 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426100366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65386 payload_rows_remote=57332 dup_rows=150 dup_rows_remote=146 dedup_ratio=0.002289 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 67, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8054, 8186, 8192] dup_rows_per_dst=[2, 0, 0, 6, 0, 4, 138, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8054, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=224641024 dispatch_meta_bytes=229376 combine_payload_bytes=224641024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=224870400 ingress_bytes=224641024 link_bytes=224870400 token_bytes=449511424 weight_movement_bytes=0 total_nvlink_bytes=449511424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.154597530e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232734720 dispatch_meta_bytes=229376 combine_payload_bytes=232734720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232964096 ingress_bytes=270483456 link_bytes=270483456 token_bytes=465698816 weight_movement_bytes=37748736 total_nvlink_bytes=503447552 weight_to_token_ratio=0.081058 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591639389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61618 payload_rows_remote=53426 dup_rows=3918 dup_rows_remote=3880 dedup_ratio=0.059784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7778, 8156, 8134, 4782] dup_rows_per_dst=[38, 1950, 38, 1874, 18, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7778, 8156, 8134, 4782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=218808320 dispatch_meta_bytes=229024 combine_payload_bytes=218808320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219037344 ingress_bytes=237682688 link_bytes=237682688 token_bytes=437845664 weight_movement_bytes=18874368 total_nvlink_bytes=456720032 weight_to_token_ratio=0.043107 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277358569e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=206659584 dispatch_meta_bytes=221896 combine_payload_bytes=206659584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206881480 ingress_bytes=225533952 link_bytes=225533952 token_bytes=413541064 weight_movement_bytes=18874368 total_nvlink_bytes=432415432 weight_to_token_ratio=0.045641 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=20864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.160955358e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229912 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061784 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893656 weight_movement_bytes=18874368 total_nvlink_bytes=488768024 weight_to_token_ratio=0.040167 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218832896 dispatch_meta_bytes=229224 combine_payload_bytes=218832896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219062120 ingress_bytes=237707264 link_bytes=237707264 token_bytes=437895016 weight_movement_bytes=18874368 total_nvlink_bytes=456769384 weight_to_token_ratio=0.043102 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277594044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54884 payload_rows_remote=46698 dup_rows=10652 dup_rows_remote=10652 dedup_ratio=0.162537 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 3, 1, 4093, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8186, 8186, 8186, 2344, 8192, 8192, 8192, 3406] dup_rows_per_dst=[0, 3144, 0, 0, 7488, 14, 6, 0] combine_rows_per_src=[8186, 8186, 8186, 2344, 8192, 8192, 8192, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59088 payload_rows_remote=50908 dup_rows=6448 dup_rows_remote=6176 dedup_ratio=0.098389 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 4090, 0, 0, 6, 3, 4089, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 1, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090] payload_rows_per_dst=[8180, 2072, 8180, 8192, 7954, 8138, 8192, 8180] dup_rows_per_dst=[0, 0, 0, 6126, 6, 30, 14, 272] combine_rows_per_src=[8180, 2072, 8180, 8192, 7954, 8138, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57694 payload_rows_remote=57016 dup_rows=7842 dup_rows_remote=7842 dedup_ratio=0.119659 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 678, 7906, 8192, 8192, 8150] dup_rows_per_dst=[0, 2488, 0, 0, 860, 0, 0, 4494] combine_rows_per_src=[8192, 8192, 8192, 678, 7906, 8192, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=191275008 dispatch_meta_bytes=229400 combine_payload_bytes=191275008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191504408 ingress_bytes=210149376 link_bytes=210149376 token_bytes=382779416 weight_movement_bytes=18874368 total_nvlink_bytes=401653784 weight_to_token_ratio=0.049309 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.013547921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=66 dedup_ratio=0.001007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 7, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8192, 8156, 8178, 8192, 8190] dup_rows_per_dst=[36, 16, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8192, 8156, 8178, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8148, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 44, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8148, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57222 payload_rows_remote=49030 dup_rows=8314 dup_rows_remote=8302 dedup_ratio=0.126862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 4093, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8186, 7162, 8186, 8016, 1102, 8192, 8192, 8186] dup_rows_per_dst=[0, 146, 0, 1008, 6, 12, 6, 7136] combine_rows_per_src=[8186, 7162, 8186, 8016, 1102, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64740 payload_rows_remote=56550 dup_rows=796 dup_rows_remote=794 dedup_ratio=0.012146 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8156, 8192, 7434, 8192, 8192] dup_rows_per_dst=[2, 0, 8, 724, 28, 34, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8156, 8192, 7434, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=208519168 dispatch_meta_bytes=228336 combine_payload_bytes=208519168 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208747504 ingress_bytes=265142272 link_bytes=265142272 token_bytes=417266672 weight_movement_bytes=56623104 total_nvlink_bytes=473889776 weight_to_token_ratio=0.135700 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.540462792e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233537536 dispatch_meta_bytes=259432 combine_payload_bytes=233537536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233796968 ingress_bytes=271286272 link_bytes=271286272 token_bytes=467334504 weight_movement_bytes=37748736 total_nvlink_bytes=505083240 weight_to_token_ratio=0.080775 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=13056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599331577e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229376 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234930176 ingress_bytes=253575168 link_bytes=253575168 token_bytes=469630976 weight_movement_bytes=18874368 total_nvlink_bytes=488505344 weight_to_token_ratio=0.040190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429632493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229376 combine_payload_bytes=234610688 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234840064 ingress_bytes=234610688 link_bytes=234840064 token_bytes=469450752 weight_movement_bytes=0 total_nvlink_bytes=469450752 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250121945e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200826880 dispatch_meta_bytes=229328 combine_payload_bytes=200826880 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=201056208 ingress_bytes=276324352 link_bytes=276324352 token_bytes=401883088 weight_movement_bytes=75497472 total_nvlink_bytes=477380560 weight_to_token_ratio=0.187859 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.647603981e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231628800 dispatch_meta_bytes=229376 combine_payload_bytes=231628800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231858176 ingress_bytes=269377536 link_bytes=269377536 token_bytes=463486976 weight_movement_bytes=37748736 total_nvlink_bytes=501235712 weight_to_token_ratio=0.081445 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581043008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63332 payload_rows_remote=55140 dup_rows=2204 dup_rows_remote=2204 dedup_ratio=0.033630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5994, 8192, 8186, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 2202, 0] combine_rows_per_src=[8192, 8192, 8192, 5994, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4059, 4093, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8124, 8186, 8186, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 6, 0, 68, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8124, 8186, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=225853440 dispatch_meta_bytes=229376 combine_payload_bytes=225853440 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=226082816 ingress_bytes=263602176 link_bytes=263602176 token_bytes=451936256 weight_movement_bytes=37748736 total_nvlink_bytes=489684992 weight_to_token_ratio=0.083527 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.525706350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229376 combine_payload_bytes=234553344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234782720 ingress_bytes=253427712 link_bytes=253427712 token_bytes=469336064 weight_movement_bytes=18874368 total_nvlink_bytes=488210432 weight_to_token_ratio=0.040215 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428219642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57344 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8164, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 28, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8164, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.000977 router_tokens_per_expert=[6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 4090, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 8184, 8192, 8182, 8184, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 44, 8, 0, 0, 0] combine_rows_per_src=[8192, 8154, 8184, 8192, 8182, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64138 payload_rows_remote=55946 dup_rows=1398 dup_rows_remote=1398 dedup_ratio=0.021332 router_tokens_per_expert=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6796, 8190, 8192, 8192, 8192] dup_rows_per_dst=[42, 0, 0, 0, 0, 0, 1356, 0] combine_rows_per_src=[8192, 8192, 8192, 6796, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62504 payload_rows_remote=54326 dup_rows=3032 dup_rows_remote=46 dedup_ratio=0.046265 router_tokens_per_expert=[0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 4089, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 2589, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1489, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8178, 5192, 8178, 8192] dup_rows_per_dst=[14, 14, 0, 14, 0, 4, 2986, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8178, 5192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60288 payload_rows_remote=52096 dup_rows=5248 dup_rows_remote=5224 dedup_ratio=0.080078 router_tokens_per_expert=[0, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8164, 7980, 8190, 8192, 3192, 8192] dup_rows_per_dst=[24, 12, 0, 5000, 0, 0, 212, 0] combine_rows_per_src=[8192, 8186, 8164, 7980, 8190, 8192, 3192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64926 payload_rows_remote=56734 dup_rows=610 dup_rows_remote=6 dedup_ratio=0.009308 router_tokens_per_expert=[3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3791, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7588, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 604, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7588, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229488 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110512 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991536 weight_movement_bytes=56623104 total_nvlink_bytes=526614640 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=229154816 dispatch_meta_bytes=229376 combine_payload_bytes=229154816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229384192 ingress_bytes=248029184 link_bytes=248029184 token_bytes=458539008 weight_movement_bytes=18874368 total_nvlink_bytes=477413376 weight_to_token_ratio=0.041162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376493603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229376 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234848256 ingress_bytes=253493248 link_bytes=253493248 token_bytes=469467136 weight_movement_bytes=18874368 total_nvlink_bytes=488341504 weight_to_token_ratio=0.040204 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428847576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=222519296 dispatch_meta_bytes=217488 combine_payload_bytes=222519296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=222736784 ingress_bytes=260268032 link_bytes=260268032 token_bytes=445256080 weight_movement_bytes=37748736 total_nvlink_bytes=483004816 weight_to_token_ratio=0.084780 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.493760223e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=213385216 dispatch_meta_bytes=229280 combine_payload_bytes=213385216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213614496 ingress_bytes=232259584 link_bytes=232259584 token_bytes=426999712 weight_movement_bytes=18874368 total_nvlink_bytes=445874080 weight_to_token_ratio=0.044202 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225397055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=232382464 dispatch_meta_bytes=226960 combine_payload_bytes=232382464 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232609424 ingress_bytes=289005568 link_bytes=289005568 token_bytes=464991888 weight_movement_bytes=56623104 total_nvlink_bytes=521614992 weight_to_token_ratio=0.121772 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769109153e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=44 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 4, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 3, 4093, 0, 0, 0, 3, 3, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8158, 8190, 8190, 8186, 8192] dup_rows_per_dst=[18, 0, 6, 0, 6, 0, 20, 0] combine_rows_per_src=[8186, 8192, 8192, 8158, 8190, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975040 weight_movement_bytes=56623104 total_nvlink_bytes=526598144 weight_to_token_ratio=0.120481 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229352 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234905576 ingress_bytes=291299328 link_bytes=291299328 token_bytes=469581800 weight_movement_bytes=56623104 total_nvlink_bytes=526204904 weight_to_token_ratio=0.120582 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791086833e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65328 payload_rows_remote=57342 dup_rows=208 dup_rows_remote=208 dedup_ratio=0.003174 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7986, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 206, 0] combine_rows_per_src=[8192, 8190, 8192, 7986, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65224 payload_rows_remote=57032 dup_rows=312 dup_rows_remote=312 dedup_ratio=0.004761 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7880, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 312, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7880, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61260 payload_rows_remote=53068 dup_rows=4276 dup_rows_remote=4262 dedup_ratio=0.065247 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 0, 4089, 0, 0, 7, 5, 4089, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089] payload_rows_per_dst=[8178, 4290, 8178, 8192, 7860, 8192, 8192, 8178] dup_rows_per_dst=[0, 0, 0, 3442, 6, 36, 14, 778] combine_rows_per_src=[8178, 4290, 8178, 8192, 7860, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64900 payload_rows_remote=56708 dup_rows=636 dup_rows_remote=636 dedup_ratio=0.009705 router_tokens_per_expert=[0, 0, 0, 0, 31, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 0, 0, 5, 4090, 0, 0, 0, 6, 6, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8036, 8192, 8184, 7766, 8160, 8186, 8184, 8192] dup_rows_per_dst=[82, 0, 10, 144, 12, 24, 364, 0] combine_rows_per_src=[8036, 8192, 8184, 7766, 8160, 8186, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64670 payload_rows_remote=56478 dup_rows=866 dup_rows_remote=838 dedup_ratio=0.013214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8158, 8166, 7386, 8192] dup_rows_per_dst=[28, 38, 0, 0, 0, 800, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8158, 8166, 7386, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=230200 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235103032 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975864 weight_movement_bytes=37748736 total_nvlink_bytes=507724600 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65324 payload_rows_remote=57134 dup_rows=212 dup_rows_remote=212 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 3, 4093, 0, 0, 0, 3, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8188, 8192, 8000, 8192, 8190, 8184, 8192] dup_rows_per_dst=[4, 0, 6, 0, 6, 0, 196, 0] combine_rows_per_src=[8186, 8188, 8192, 8000, 8192, 8190, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=233603072 dispatch_meta_bytes=229376 combine_payload_bytes=233603072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233832448 ingress_bytes=252477440 link_bytes=252477440 token_bytes=467435520 weight_movement_bytes=18874368 total_nvlink_bytes=486309888 weight_to_token_ratio=0.040379 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419114603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=229320 combine_payload_bytes=217366528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217595848 ingress_bytes=236240896 link_bytes=236240896 token_bytes=434962376 weight_movement_bytes=18874368 total_nvlink_bytes=453836744 weight_to_token_ratio=0.043393 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.263544028e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232275968 dispatch_meta_bytes=229376 combine_payload_bytes=232275968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232505344 ingress_bytes=251150336 link_bytes=251150336 token_bytes=464781312 weight_movement_bytes=18874368 total_nvlink_bytes=483655680 weight_to_token_ratio=0.040609 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406398946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63854 payload_rows_remote=55662 dup_rows=1682 dup_rows_remote=1682 dedup_ratio=0.025665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3258, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6516, 8192, 8186] dup_rows_per_dst=[0, 1656, 0, 0, 0, 0, 26, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6516, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=231333888 dispatch_meta_bytes=229264 combine_payload_bytes=231333888 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231563152 ingress_bytes=269082624 link_bytes=269082624 token_bytes=462897040 weight_movement_bytes=37748736 total_nvlink_bytes=500645776 weight_to_token_ratio=0.081549 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.578217306e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57270 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 30, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8190, 8186, 8192, 8192, 8192, 8126, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 68, 0] combine_rows_per_src=[8190, 8186, 8192, 8192, 8192, 8126, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229384 combine_payload_bytes=234020864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234250248 ingress_bytes=271769600 link_bytes=271769600 token_bytes=468271112 weight_movement_bytes=37748736 total_nvlink_bytes=506019848 weight_to_token_ratio=0.080613 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603962588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=227991552 dispatch_meta_bytes=229376 combine_payload_bytes=227991552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228220928 ingress_bytes=246865920 link_bytes=246865920 token_bytes=456212480 weight_movement_bytes=18874368 total_nvlink_bytes=475086848 weight_to_token_ratio=0.041372 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365347780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229376 combine_payload_bytes=234577920 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234807296 ingress_bytes=253452288 link_bytes=253452288 token_bytes=469385216 weight_movement_bytes=18874368 total_nvlink_bytes=488259584 weight_to_token_ratio=0.040211 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428455117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64758 payload_rows_remote=56566 dup_rows=778 dup_rows_remote=658 dedup_ratio=0.011871 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7490, 8192, 8116] dup_rows_per_dst=[0, 120, 0, 0, 0, 0, 658, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7490, 8192, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 4089, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 8184, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65238 payload_rows_remote=57052 dup_rows=298 dup_rows_remote=286 dedup_ratio=0.004547 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 107, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8184, 8180, 8190, 8154, 8192, 7966, 8186, 8186] dup_rows_per_dst=[0, 0, 2, 12, 34, 0, 238, 12] combine_rows_per_src=[8184, 8180, 8190, 8154, 8192, 7966, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60742 payload_rows_remote=52552 dup_rows=4794 dup_rows_remote=4794 dedup_ratio=0.073151 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 2, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8164, 3530, 8090] dup_rows_per_dst=[0, 260, 32, 0, 0, 4502, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8164, 3530, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231694336 dispatch_meta_bytes=228896 combine_payload_bytes=231694336 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231923232 ingress_bytes=269443072 link_bytes=269443072 token_bytes=463617568 weight_movement_bytes=37748736 total_nvlink_bytes=501366304 weight_to_token_ratio=0.081422 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581670942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65426 payload_rows_remote=57328 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.001678 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 3, 0, 0, 0, 3, 0, 2, 0, 44, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8186, 8186, 8192, 8192, 8192, 8098, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 104, 0] combine_rows_per_src=[8186, 8186, 8192, 8192, 8192, 8098, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235020288 ingress_bytes=253665280 link_bytes=253665280 token_bytes=469811200 weight_movement_bytes=18874368 total_nvlink_bytes=488685568 weight_to_token_ratio=0.040174 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430495902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=233684992 dispatch_meta_bytes=229352 combine_payload_bytes=233684992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233914344 ingress_bytes=252559360 link_bytes=252559360 token_bytes=467599336 weight_movement_bytes=18874368 total_nvlink_bytes=486473704 weight_to_token_ratio=0.040364 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419899521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=215252992 dispatch_meta_bytes=229384 combine_payload_bytes=215252992 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=215482376 ingress_bytes=234127360 link_bytes=234127360 token_bytes=430735368 weight_movement_bytes=18874368 total_nvlink_bytes=449609736 weight_to_token_ratio=0.043819 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.243293166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65344 payload_rows_remote=57158 dup_rows=192 dup_rows_remote=192 dedup_ratio=0.002930 router_tokens_per_expert=[0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 11, 3, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8186, 8192, 8020, 8192, 8184] dup_rows_per_dst=[6, 34, 0, 0, 6, 0, 146, 0] combine_rows_per_src=[8192, 8192, 8186, 8186, 8192, 8020, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229752 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235045240 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860728 weight_movement_bytes=18874368 total_nvlink_bytes=488735096 weight_to_token_ratio=0.040170 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234119168 dispatch_meta_bytes=229400 combine_payload_bytes=234119168 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234348568 ingress_bytes=271867904 link_bytes=271867904 token_bytes=468467736 weight_movement_bytes=37748736 total_nvlink_bytes=506216472 weight_to_token_ratio=0.080579 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604904489e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65064 payload_rows_remote=57312 dup_rows=472 dup_rows_remote=472 dedup_ratio=0.007202 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7752, 8192, 8192, 8160, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 440, 32, 0, 0, 0, 0] combine_rows_per_src=[8192, 7752, 8192, 8192, 8160, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65308 payload_rows_remote=57116 dup_rows=228 dup_rows_remote=228 dedup_ratio=0.003479 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7972, 8192, 8184] dup_rows_per_dst=[0, 98, 0, 0, 0, 0, 130, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7972, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62232 payload_rows_remote=54058 dup_rows=3304 dup_rows_remote=3066 dedup_ratio=0.050415 router_tokens_per_expert=[0, 6, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4087, 0, 0, 22, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4090, 0, 0, 0] payload_rows_per_dst=[8154, 5846, 8180, 8180, 8154, 7352, 8192, 8174] dup_rows_per_dst=[14, 54, 6, 38, 12, 0, 2942, 238] combine_rows_per_src=[8154, 5846, 8180, 8180, 8154, 7352, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65022 payload_rows_remote=56842 dup_rows=514 dup_rows_remote=36 dedup_ratio=0.007843 router_tokens_per_expert=[0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 6, 4089, 0, 0, 0, 7, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 7712, 8192, 8184, 8180, 8192] dup_rows_per_dst=[8, 0, 14, 0, 14, 0, 478, 0] combine_rows_per_src=[8178, 8192, 8192, 7712, 8192, 8184, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64624 payload_rows_remote=56432 dup_rows=912 dup_rows_remote=900 dedup_ratio=0.013916 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7318, 8192, 8180, 8192, 8192, 8166] dup_rows_per_dst=[12, 860, 4, 18, 0, 18, 0, 0] combine_rows_per_src=[8192, 8192, 7318, 8192, 8180, 8192, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=231136 combine_payload_bytes=234749952 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234981088 ingress_bytes=310247424 link_bytes=310247424 token_bytes=469731040 weight_movement_bytes=75497472 total_nvlink_bytes=545228512 weight_to_token_ratio=0.160725 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972638166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233947136 dispatch_meta_bytes=229376 combine_payload_bytes=233947136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234176512 ingress_bytes=252821504 link_bytes=252821504 token_bytes=468123648 weight_movement_bytes=18874368 total_nvlink_bytes=486998016 weight_to_token_ratio=0.040319 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422411255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65338 payload_rows_remote=57326 dup_rows=198 dup_rows_remote=198 dedup_ratio=0.003021 router_tokens_per_expert=[0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8186, 8192, 8012, 8192, 8186] dup_rows_per_dst=[6, 6, 0, 0, 6, 0, 180, 0] combine_rows_per_src=[8192, 8192, 8186, 8186, 8192, 8012, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=221421568 dispatch_meta_bytes=228496 combine_payload_bytes=221421568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=221650064 ingress_bytes=259170304 link_bytes=259170304 token_bytes=443071632 weight_movement_bytes=37748736 total_nvlink_bytes=480820368 weight_to_token_ratio=0.085198 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.483242333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8190] dup_rows_per_dst=[6, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=232824832 dispatch_meta_bytes=227512 combine_payload_bytes=232824832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233052344 ingress_bytes=270573568 link_bytes=270573568 token_bytes=465877176 weight_movement_bytes=37748736 total_nvlink_bytes=503625912 weight_to_token_ratio=0.081027 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592502798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=10 dedup_ratio=0.000519 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[8186, 8174, 8192, 8192, 8188, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 24, 2, 6, 2, 0, 0] combine_rows_per_src=[8186, 8174, 8192, 8192, 8188, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=231145472 dispatch_meta_bytes=229328 combine_payload_bytes=231145472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231374800 ingress_bytes=250019840 link_bytes=250019840 token_bytes=462520272 weight_movement_bytes=18874368 total_nvlink_bytes=481394640 weight_to_token_ratio=0.040808 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.395567089e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=230096 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235037392 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469844688 weight_movement_bytes=18874368 total_nvlink_bytes=488719056 weight_to_token_ratio=0.040172 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=234856448 link_bytes=235085824 token_bytes=469942272 weight_movement_bytes=0 total_nvlink_bytes=469942272 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252476696e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229280 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234971040 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469712800 weight_movement_bytes=18874368 total_nvlink_bytes=488587168 weight_to_token_ratio=0.040183 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65302 payload_rows_remote=57110 dup_rows=234 dup_rows_remote=234 dedup_ratio=0.003571 router_tokens_per_expert=[4046, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4068, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0] payload_rows_per_dst=[8092, 8192, 8172, 8136, 8192, 8192, 8192, 8134] dup_rows_per_dst=[58, 0, 0, 0, 0, 0, 0, 176] combine_rows_per_src=[8092, 8192, 8172, 8136, 8192, 8192, 8192, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65090 payload_rows_remote=56898 dup_rows=446 dup_rows_remote=420 dedup_ratio=0.006805 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 5, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8186, 7848, 8192, 8136, 8164, 8180, 8192, 8192] dup_rows_per_dst=[0, 0, 336, 2, 68, 14, 0, 26] combine_rows_per_src=[8186, 7848, 8192, 8136, 8164, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=14 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 2, 0, 0, 0, 7, 0, 6, 0, 31, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[8180, 8178, 8192, 8192, 8192, 8116, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 14, 0, 0, 92, 0] combine_rows_per_src=[8180, 8178, 8192, 8192, 8192, 8116, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=229376 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234151936 ingress_bytes=252796928 link_bytes=252796928 token_bytes=468074496 weight_movement_bytes=18874368 total_nvlink_bytes=486948864 weight_to_token_ratio=0.040323 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422175780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65340 payload_rows_remote=57148 dup_rows=196 dup_rows_remote=74 dedup_ratio=0.002991 router_tokens_per_expert=[4087, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8114, 8192, 8192, 8192, 8074] dup_rows_per_dst=[122, 0, 74, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8114, 8192, 8192, 8192, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235085824 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942272 weight_movement_bytes=37748736 total_nvlink_bytes=507691008 weight_to_token_ratio=0.080326 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57312 dup_rows=38 dup_rows_remote=34 dedup_ratio=0.000580 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[8192, 8160, 8192, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 28, 0, 6, 4, 0, 0] combine_rows_per_src=[8192, 8160, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233054208 dispatch_meta_bytes=229272 combine_payload_bytes=233054208 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233283480 ingress_bytes=289677312 link_bytes=289677312 token_bytes=466337688 weight_movement_bytes=56623104 total_nvlink_bytes=522960792 weight_to_token_ratio=0.121421 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.775545473e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65296 payload_rows_remote=57104 dup_rows=240 dup_rows_remote=240 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3976, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7952] dup_rows_per_dst=[0, 2, 0, 0, 0, 238, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229008 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234675856 ingress_bytes=309944320 link_bytes=309944320 token_bytes=469122704 weight_movement_bytes=75497472 total_nvlink_bytes=544620176 weight_to_token_ratio=0.160933 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.969733972e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65454 payload_rows_remote=57268 dup_rows=82 dup_rows_remote=82 dedup_ratio=0.001251 router_tokens_per_expert=[4063, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 3, 4093, 0, 0, 4091, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0] payload_rows_per_dst=[8134, 8192, 8186, 8188, 8192, 8186, 8186, 8190] dup_rows_per_dst=[2, 0, 0, 12, 0, 0, 0, 68] combine_rows_per_src=[8134, 8192, 8186, 8188, 8192, 8186, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=228888 combine_payload_bytes=234078208 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234307096 ingress_bytes=309575680 link_bytes=309575680 token_bytes=468385304 weight_movement_bytes=75497472 total_nvlink_bytes=543882776 weight_to_token_ratio=0.161187 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.966201845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229384 combine_payload_bytes=234749952 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234979336 ingress_bytes=272498688 link_bytes=272498688 token_bytes=469729288 weight_movement_bytes=37748736 total_nvlink_bytes=507478024 weight_to_token_ratio=0.080363 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610948351e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233897984 dispatch_meta_bytes=229376 combine_payload_bytes=233897984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234127360 ingress_bytes=271646720 link_bytes=271646720 token_bytes=468025344 weight_movement_bytes=37748736 total_nvlink_bytes=505774080 weight_to_token_ratio=0.080655 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602785213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234569728 dispatch_meta_bytes=229400 combine_payload_bytes=234569728 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234799128 ingress_bytes=272318464 link_bytes=272318464 token_bytes=469368856 weight_movement_bytes=37748736 total_nvlink_bytes=507117592 weight_to_token_ratio=0.080424 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609221533e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64008 payload_rows_remote=55816 dup_rows=1528 dup_rows_remote=1526 dedup_ratio=0.023315 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3363, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8130, 6726] dup_rows_per_dst=[0, 2, 0, 0, 0, 1526, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8130, 6726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65138 payload_rows_remote=56946 dup_rows=398 dup_rows_remote=398 dedup_ratio=0.006073 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7794] dup_rows_per_dst=[398, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64754 payload_rows_remote=56660 dup_rows=782 dup_rows_remote=282 dedup_ratio=0.011932 router_tokens_per_expert=[4052, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 6, 4081, 0, 0, 4087, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 55, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0] payload_rows_per_dst=[7564, 8192, 8164, 8186, 8192, 8180, 8182, 8094] dup_rows_per_dst=[108, 0, 0, 98, 0, 42, 34, 500] combine_rows_per_src=[7564, 8192, 8164, 8186, 8192, 8180, 8182, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64810 payload_rows_remote=56618 dup_rows=726 dup_rows_remote=146 dedup_ratio=0.011078 router_tokens_per_expert=[0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4089, 0, 0, 51, 7, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8178, 8192, 7508, 8192, 8178] dup_rows_per_dst=[14, 118, 0, 0, 14, 0, 580, 0] combine_rows_per_src=[8192, 8192, 8178, 8178, 8192, 7508, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228622336 dispatch_meta_bytes=229368 combine_payload_bytes=228622336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228851704 ingress_bytes=285245440 link_bytes=285245440 token_bytes=457474040 weight_movement_bytes=56623104 total_nvlink_bytes=514097144 weight_to_token_ratio=0.123773 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.733081457e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65196 payload_rows_remote=57004 dup_rows=340 dup_rows_remote=302 dedup_ratio=0.005188 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3972, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 8192, 8170, 7906] dup_rows_per_dst=[38, 32, 0, 0, 0, 270, 0, 0] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 8192, 8170, 7906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233250816 dispatch_meta_bytes=229376 combine_payload_bytes=233250816 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233480192 ingress_bytes=289873920 link_bytes=289873920 token_bytes=466731008 weight_movement_bytes=56623104 total_nvlink_bytes=523354112 weight_to_token_ratio=0.121318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.777429275e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65266 payload_rows_remote=57074 dup_rows=270 dup_rows_remote=270 dedup_ratio=0.004120 router_tokens_per_expert=[0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7922, 8192] dup_rows_per_dst=[268, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7922, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65338 payload_rows_remote=57152 dup_rows=198 dup_rows_remote=198 dedup_ratio=0.003021 router_tokens_per_expert=[4063, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 3, 4093, 0, 0, 4093, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0] payload_rows_per_dst=[8132, 8192, 8186, 8192, 8192, 8186, 8186, 8072] dup_rows_per_dst=[120, 0, 0, 12, 0, 0, 0, 66] combine_rows_per_src=[8132, 8192, 8186, 8192, 8192, 8186, 8186, 8072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232079360 dispatch_meta_bytes=227768 combine_payload_bytes=232079360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232307128 ingress_bytes=250953728 link_bytes=250953728 token_bytes=464386488 weight_movement_bytes=18874368 total_nvlink_bytes=483260856 weight_to_token_ratio=0.040644 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404515145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231907328 dispatch_meta_bytes=227056 combine_payload_bytes=231907328 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=232134384 ingress_bytes=307404800 link_bytes=307404800 token_bytes=464041712 weight_movement_bytes=75497472 total_nvlink_bytes=539539184 weight_to_token_ratio=0.162695 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.945401541e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64712 payload_rows_remote=56520 dup_rows=824 dup_rows_remote=818 dedup_ratio=0.012573 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 74, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4093, 0, 3694, 3, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8186, 8186, 8184, 7394] dup_rows_per_dst=[0, 148, 6, 6, 0, 652, 0, 12] combine_rows_per_src=[8186, 8192, 8192, 8192, 8186, 8186, 8184, 7394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233488384 dispatch_meta_bytes=229224 combine_payload_bytes=233488384 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233717608 ingress_bytes=271237120 link_bytes=271237120 token_bytes=467205992 weight_movement_bytes=37748736 total_nvlink_bytes=504954728 weight_to_token_ratio=0.080797 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.598860627e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233775104 dispatch_meta_bytes=229376 combine_payload_bytes=233775104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234004480 ingress_bytes=252649472 link_bytes=252649472 token_bytes=467779584 weight_movement_bytes=18874368 total_nvlink_bytes=486653952 weight_to_token_ratio=0.040349 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420762929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234094592 dispatch_meta_bytes=229400 combine_payload_bytes=234094592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234323992 ingress_bytes=271843328 link_bytes=271843328 token_bytes=468418584 weight_movement_bytes=37748736 total_nvlink_bytes=506167320 weight_to_token_ratio=0.080588 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604669014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=231505920 dispatch_meta_bytes=229352 combine_payload_bytes=231505920 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231735272 ingress_bytes=269254656 link_bytes=269254656 token_bytes=463241192 weight_movement_bytes=37748736 total_nvlink_bytes=500989928 weight_to_token_ratio=0.081488 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.579865632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65394 payload_rows_remote=57234 dup_rows=142 dup_rows_remote=142 dedup_ratio=0.002167 router_tokens_per_expert=[0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 8192, 8192, 8192, 8192, 8082, 8192] dup_rows_per_dst=[74, 0, 68, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8160, 8192, 8192, 8192, 8192, 8082, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64464 payload_rows_remote=56272 dup_rows=1072 dup_rows_remote=1072 dedup_ratio=0.016357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3560, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7120] dup_rows_per_dst=[0, 2, 0, 0, 0, 1070, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58616 payload_rows_remote=50906 dup_rows=6920 dup_rows_remote=848 dedup_ratio=0.105591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 4090, 0, 3678, 6, 0] payload_rows_per_dst=[8180, 7872, 8186, 8154, 8180, 2228, 8106, 7710] dup_rows_per_dst=[0, 2, 12, 12, 44, 556, 222, 6072] combine_rows_per_src=[8180, 7872, 8186, 8154, 8180, 2228, 8106, 7710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[8192, 8174, 8192, 8192, 8174, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 36, 0, 14, 0, 0, 0] combine_rows_per_src=[8192, 8174, 8192, 8192, 8174, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234430464 dispatch_meta_bytes=229504 combine_payload_bytes=234430464 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234659968 ingress_bytes=328802304 link_bytes=328802304 token_bytes=469090432 weight_movement_bytes=94371840 total_nvlink_bytes=563462272 weight_to_token_ratio=0.201180 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.150421896e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62812 payload_rows_remote=54620 dup_rows=2724 dup_rows_remote=2724 dedup_ratio=0.041565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1361, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5470, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2724, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 5470, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=230490112 dispatch_meta_bytes=229376 combine_payload_bytes=230490112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230719488 ingress_bytes=249364480 link_bytes=249364480 token_bytes=461209600 weight_movement_bytes=18874368 total_nvlink_bytes=480083968 weight_to_token_ratio=0.040924 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.389287752e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65056 payload_rows_remote=56864 dup_rows=480 dup_rows_remote=400 dedup_ratio=0.007324 router_tokens_per_expert=[0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8192, 7826, 8192, 8192, 8192, 8084, 8186] dup_rows_per_dst=[80, 390, 0, 6, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 7826, 8192, 8192, 8192, 8084, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=208510976 dispatch_meta_bytes=207016 combine_payload_bytes=208510976 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=208717992 ingress_bytes=284008448 link_bytes=284008448 token_bytes=417228968 weight_movement_bytes=75497472 total_nvlink_bytes=492726440 weight_to_token_ratio=0.180950 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.721229208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229376 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905600 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581824 weight_movement_bytes=18874368 total_nvlink_bytes=488456192 weight_to_token_ratio=0.040194 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64944 payload_rows_remote=56758 dup_rows=592 dup_rows_remote=112 dedup_ratio=0.009033 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 44, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4090, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4093, 0, 3812, 3, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8180, 8186, 8186, 7630] dup_rows_per_dst=[0, 88, 6, 6, 0, 480, 0, 12] combine_rows_per_src=[8186, 8192, 8192, 8192, 8180, 8186, 8186, 7630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65324 payload_rows_remote=57132 dup_rows=212 dup_rows_remote=150 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 69, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 27, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 3997, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8186, 8190, 8186, 8000, 8192] dup_rows_per_dst=[138, 0, 62, 0, 0, 0, 12, 0] combine_rows_per_src=[8192, 8186, 8192, 8186, 8190, 8186, 8000, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=223723520 dispatch_meta_bytes=229376 combine_payload_bytes=223723520 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=223952896 ingress_bytes=261472256 link_bytes=261472256 token_bytes=447676416 weight_movement_bytes=37748736 total_nvlink_bytes=485425152 weight_to_token_ratio=0.084321 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.505298505e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232914944 dispatch_meta_bytes=229056 combine_payload_bytes=232914944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233144000 ingress_bytes=251789312 link_bytes=251789312 token_bytes=466058944 weight_movement_bytes=18874368 total_nvlink_bytes=484933312 weight_to_token_ratio=0.040498 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412521300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=232480768 dispatch_meta_bytes=227480 combine_payload_bytes=232480768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232708248 ingress_bytes=251355136 link_bytes=251355136 token_bytes=465189016 weight_movement_bytes=18874368 total_nvlink_bytes=484063384 weight_to_token_ratio=0.040574 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408361239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234012672 dispatch_meta_bytes=229128 combine_payload_bytes=234012672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234241800 ingress_bytes=252887040 link_bytes=252887040 token_bytes=468254472 weight_movement_bytes=18874368 total_nvlink_bytes=487128840 weight_to_token_ratio=0.040308 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423039189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64346 payload_rows_remote=56154 dup_rows=1190 dup_rows_remote=1190 dedup_ratio=0.018158 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 595, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7002, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1190, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7002, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] dup_rows_per_dst=[16, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63924 payload_rows_remote=55732 dup_rows=1612 dup_rows_remote=1610 dedup_ratio=0.024597 router_tokens_per_expert=[0, 0, 0, 8, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4090, 0, 0, 46, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4090, 6, 4041, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1] payload_rows_per_dst=[7042, 7860, 8192, 8180, 8182, 8180, 8096, 8192] dup_rows_per_dst=[14, 0, 114, 32, 20, 38, 1392, 2] combine_rows_per_src=[7042, 7860, 8192, 8180, 8182, 8180, 8096, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57282 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[4072, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 7, 4089, 0, 0, 4089, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8192, 8192, 8174, 8180, 8162] dup_rows_per_dst=[32, 0, 0, 26, 0, 0, 0, 16] combine_rows_per_src=[8192, 8192, 8178, 8192, 8192, 8174, 8180, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=230006784 dispatch_meta_bytes=229376 combine_payload_bytes=230006784 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=230236160 ingress_bytes=286629888 link_bytes=286629888 token_bytes=460242944 weight_movement_bytes=56623104 total_nvlink_bytes=516866048 weight_to_token_ratio=0.123029 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.746346556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235044864 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860352 weight_movement_bytes=37748736 total_nvlink_bytes=507609088 weight_to_token_ratio=0.080340 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64706 payload_rows_remote=56514 dup_rows=830 dup_rows_remote=812 dedup_ratio=0.012665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 368, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8192, 7456, 8192, 8164, 8154, 8182] dup_rows_per_dst=[18, 0, 802, 10, 0, 0, 0, 0] combine_rows_per_src=[8192, 8174, 8192, 7456, 8192, 8164, 8154, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=228278272 dispatch_meta_bytes=229368 combine_payload_bytes=228278272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228507640 ingress_bytes=247152640 link_bytes=247152640 token_bytes=456785912 weight_movement_bytes=18874368 total_nvlink_bytes=475660280 weight_to_token_ratio=0.041320 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368094990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65416 payload_rows_remote=57230 dup_rows=120 dup_rows_remote=120 dedup_ratio=0.001831 router_tokens_per_expert=[0, 0, 0, 32, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 19, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 4042, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8186, 8192, 8186, 8090, 8192] dup_rows_per_dst=[64, 0, 44, 0, 0, 0, 12, 0] combine_rows_per_src=[8192, 8186, 8192, 8186, 8192, 8186, 8090, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234627072 dispatch_meta_bytes=229424 combine_payload_bytes=234627072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234856496 ingress_bytes=253501440 link_bytes=253501440 token_bytes=469483568 weight_movement_bytes=18874368 total_nvlink_bytes=488357936 weight_to_token_ratio=0.040202 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428926068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64410 payload_rows_remote=56224 dup_rows=1126 dup_rows_remote=20 dedup_ratio=0.017181 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 553, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3541, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8186, 7086, 8186, 8192, 8192, 8188] dup_rows_per_dst=[0, 8, 1106, 0, 0, 0, 6, 6] combine_rows_per_src=[8188, 8192, 8186, 7086, 8186, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=231481344 dispatch_meta_bytes=229304 combine_payload_bytes=231481344 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231710648 ingress_bytes=269230080 link_bytes=269230080 token_bytes=463191992 weight_movement_bytes=37748736 total_nvlink_bytes=500940728 weight_to_token_ratio=0.081497 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.579630157e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=229400 combine_payload_bytes=234414080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234643480 ingress_bytes=253288448 link_bytes=253288448 token_bytes=469057560 weight_movement_bytes=18874368 total_nvlink_bytes=487931928 weight_to_token_ratio=0.040239 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426885283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8132, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 60, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8132, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=230293504 dispatch_meta_bytes=224976 combine_payload_bytes=230293504 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230518480 ingress_bytes=305790976 link_bytes=305790976 token_bytes=460811984 weight_movement_bytes=75497472 total_nvlink_bytes=536309456 weight_to_token_ratio=0.163836 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.929938673e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64532 payload_rows_remote=57344 dup_rows=1004 dup_rows_remote=1004 dedup_ratio=0.015320 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 502, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1004, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57344 payload_rows_remote=49152 dup_rows=8192 dup_rows_remote=8180 dedup_ratio=0.125000 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 27, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4063, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8164, 54, 8186, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 14, 0, 8088, 38, 28, 12, 12] combine_rows_per_src=[8184, 8164, 54, 8186, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=229376 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234864640 ingress_bytes=253509632 link_bytes=253509632 token_bytes=469499904 weight_movement_bytes=18874368 total_nvlink_bytes=488374272 weight_to_token_ratio=0.040201 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429004559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64238 payload_rows_remote=56084 dup_rows=1298 dup_rows_remote=1298 dedup_ratio=0.019806 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 22, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 4089, 0, 3480, 7, 0] payload_rows_per_dst=[8178, 8192, 8192, 8192, 8178, 8178, 8154, 6974] dup_rows_per_dst=[0, 44, 14, 14, 0, 1198, 0, 28] combine_rows_per_src=[8178, 8192, 8192, 8192, 8178, 8178, 8154, 6974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62974 payload_rows_remote=56946 dup_rows=2562 dup_rows_remote=2562 dedup_ratio=0.039093 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6028, 8144, 8192, 7842] dup_rows_per_dst=[0, 0, 0, 348, 0, 1804, 50, 360] combine_rows_per_src=[8192, 8192, 8192, 8192, 6028, 8144, 8192, 7842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=233392 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235114416 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469995440 weight_movement_bytes=37748736 total_nvlink_bytes=507744176 weight_to_token_ratio=0.080317 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57326 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8174, 8192, 8192] dup_rows_per_dst=[0, 0, 20, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=201326592 dispatch_meta_bytes=229328 combine_payload_bytes=201326592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201555920 ingress_bytes=220200960 link_bytes=220200960 token_bytes=402882512 weight_movement_bytes=18874368 total_nvlink_bytes=421756880 weight_to_token_ratio=0.046848 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109857253e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=229720064 dispatch_meta_bytes=229528 combine_payload_bytes=229720064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229949592 ingress_bytes=248594432 link_bytes=248594432 token_bytes=459669656 weight_movement_bytes=18874368 total_nvlink_bytes=478544024 weight_to_token_ratio=0.041061 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381909531e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64790 payload_rows_remote=56598 dup_rows=746 dup_rows_remote=746 dedup_ratio=0.011383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 364, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3733, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8186, 7472, 8186, 8192, 8192, 8184] dup_rows_per_dst=[0, 6, 728, 0, 0, 0, 6, 6] combine_rows_per_src=[8186, 8192, 8186, 7472, 8186, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233250816 dispatch_meta_bytes=238032 combine_payload_bytes=233250816 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=233488848 ingress_bytes=346497024 link_bytes=346497024 token_bytes=466739664 weight_movement_bytes=113246208 total_nvlink_bytes=579985872 weight_to_token_ratio=0.242632 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.319963997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4091, 0, 3, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8188, 8186, 8186] dup_rows_per_dst=[12, 0, 0, 4, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8188, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229384 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036680 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843976 weight_movement_bytes=18874368 total_nvlink_bytes=488718344 weight_to_token_ratio=0.040172 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63156 payload_rows_remote=54964 dup_rows=2380 dup_rows_remote=2380 dedup_ratio=0.036316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 7720, 8192, 6288] dup_rows_per_dst=[0, 0, 0, 1860, 0, 0, 516, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 7720, 8192, 6288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=231825408 dispatch_meta_bytes=229376 combine_payload_bytes=231825408 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232054784 ingress_bytes=288448512 link_bytes=288448512 token_bytes=463880192 weight_movement_bytes=56623104 total_nvlink_bytes=520503296 weight_to_token_ratio=0.122064 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.763771716e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235020288 ingress_bytes=291414016 link_bytes=291414016 token_bytes=469811200 weight_movement_bytes=56623104 total_nvlink_bytes=526434304 weight_to_token_ratio=0.120523 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792185717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65226 payload_rows_remote=57046 dup_rows=310 dup_rows_remote=290 dedup_ratio=0.004730 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 35, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 4054, 0, 6, 4090, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8188, 8172, 8082, 8190, 8120, 8176, 8180] dup_rows_per_dst=[24, 4, 0, 70, 20, 128, 44, 20] combine_rows_per_src=[8118, 8188, 8172, 8082, 8190, 8120, 8176, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225132544 dispatch_meta_bytes=229376 combine_payload_bytes=225132544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225361920 ingress_bytes=244006912 link_bytes=244006912 token_bytes=450494464 weight_movement_bytes=18874368 total_nvlink_bytes=469368832 weight_to_token_ratio=0.041897 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337954172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64936 payload_rows_remote=57294 dup_rows=600 dup_rows_remote=572 dedup_ratio=0.009155 router_tokens_per_expert=[0, 0, 0, 248, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 27, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 7, 3814, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8178, 8184, 8178, 7642, 8192] dup_rows_per_dst=[496, 0, 76, 0, 0, 0, 28, 0] combine_rows_per_src=[8192, 8178, 8192, 8178, 8184, 8178, 7642, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57310 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8158, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 38, 0, 0] combine_rows_per_src=[8192, 8158, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61778 payload_rows_remote=53586 dup_rows=3758 dup_rows_remote=3748 dedup_ratio=0.057343 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7746, 6248, 6824] dup_rows_per_dst=[10, 0, 10, 1494, 2196, 0, 48, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7746, 6248, 6824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235102200 ingress_bytes=310370304 link_bytes=310370304 token_bytes=469975032 weight_movement_bytes=75497472 total_nvlink_bytes=545472504 weight_to_token_ratio=0.160641 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973815541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233660416 dispatch_meta_bytes=229344 combine_payload_bytes=233660416 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233889760 ingress_bytes=271409152 link_bytes=271409152 token_bytes=467550176 weight_movement_bytes=37748736 total_nvlink_bytes=505298912 weight_to_token_ratio=0.080737 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600508953e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=231464 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234907688 ingress_bytes=272424960 link_bytes=272424960 token_bytes=469583912 weight_movement_bytes=37748736 total_nvlink_bytes=507332648 weight_to_token_ratio=0.080388 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610241925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=12 dedup_ratio=0.000275 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 3, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8186, 8192, 8186, 8192] dup_rows_per_dst=[12, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8186, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61502 payload_rows_remote=53314 dup_rows=4034 dup_rows_remote=4032 dedup_ratio=0.061554 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 4093, 4093, 3, 0, 4062, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8188, 8186, 8192, 8130, 8186, 4244] dup_rows_per_dst=[6, 0, 2, 3666, 6, 10, 344, 0] combine_rows_per_src=[8192, 8184, 8188, 8186, 8192, 8130, 8186, 4244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229392 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234971152 ingress_bytes=310239232 link_bytes=310239232 token_bytes=469712912 weight_movement_bytes=75497472 total_nvlink_bytes=545210384 weight_to_token_ratio=0.160731 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972559674e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=219488256 dispatch_meta_bytes=229336 combine_payload_bytes=219488256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219717592 ingress_bytes=257236992 link_bytes=257236992 token_bytes=439205848 weight_movement_bytes=37748736 total_nvlink_bytes=476954584 weight_to_token_ratio=0.085948 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.464718289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65132 payload_rows_remote=57316 dup_rows=404 dup_rows_remote=404 dedup_ratio=0.006165 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7816, 8192, 8192, 8192, 8192, 8192, 8164] dup_rows_per_dst=[0, 0, 304, 0, 0, 100, 0, 0] combine_rows_per_src=[8192, 7816, 8192, 8192, 8192, 8192, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229352 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235036648 ingress_bytes=272556032 link_bytes=272556032 token_bytes=469843944 weight_movement_bytes=37748736 total_nvlink_bytes=507592680 weight_to_token_ratio=0.080343 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611497793e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=218374144 dispatch_meta_bytes=229384 combine_payload_bytes=218374144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218603528 ingress_bytes=237248512 link_bytes=237248512 token_bytes=436977672 weight_movement_bytes=18874368 total_nvlink_bytes=455852040 weight_to_token_ratio=0.043193 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.273198508e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63040 payload_rows_remote=54848 dup_rows=2496 dup_rows_remote=624 dedup_ratio=0.038086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7578, 8192, 6310] dup_rows_per_dst=[0, 0, 0, 1872, 0, 0, 624, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7578, 8192, 6310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=230880 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234997216 ingress_bytes=272515072 link_bytes=272515072 token_bytes=469763552 weight_movement_bytes=37748736 total_nvlink_bytes=507512288 weight_to_token_ratio=0.080357 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611105334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63242 payload_rows_remote=55050 dup_rows=2294 dup_rows_remote=136 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4, 4090, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 4089, 4090, 6, 0, 3994, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 4, 0, 0] payload_rows_per_dst=[8144, 8180, 8186, 6168, 8192, 8000, 8180, 8192] dup_rows_per_dst=[12, 48, 10, 0, 12, 16, 38, 2158] combine_rows_per_src=[8144, 8180, 8186, 6168, 8192, 8000, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64856 payload_rows_remote=56664 dup_rows=680 dup_rows_remote=666 dedup_ratio=0.010376 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 311, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8166, 7566, 8178, 8192, 8192, 8178] dup_rows_per_dst=[0, 18, 622, 12, 0, 0, 14, 14] combine_rows_per_src=[8192, 8192, 8166, 7566, 8178, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60256 payload_rows_remote=57344 dup_rows=5280 dup_rows_remote=5280 dedup_ratio=0.080566 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 2912, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 5280, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 2912, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=224657408 dispatch_meta_bytes=221888 combine_payload_bytes=224657408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224879296 ingress_bytes=243531776 link_bytes=243531776 token_bytes=449536704 weight_movement_bytes=18874368 total_nvlink_bytes=468411072 weight_to_token_ratio=0.041986 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.333401653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64208 payload_rows_remote=56016 dup_rows=1328 dup_rows_remote=1308 dedup_ratio=0.020264 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7312, 8192, 8192, 7756, 8192, 8180] dup_rows_per_dst=[20, 976, 0, 0, 330, 2, 0, 0] combine_rows_per_src=[8192, 8192, 7312, 8192, 8192, 7756, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225484800 dispatch_meta_bytes=220744 combine_payload_bytes=225484800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225705544 ingress_bytes=244359168 link_bytes=244359168 token_bytes=451190344 weight_movement_bytes=18874368 total_nvlink_bytes=470064712 weight_to_token_ratio=0.041832 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341329316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=232095744 dispatch_meta_bytes=229320 combine_payload_bytes=232095744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232325064 ingress_bytes=250970112 link_bytes=250970112 token_bytes=464420808 weight_movement_bytes=18874368 total_nvlink_bytes=483295176 weight_to_token_ratio=0.040641 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404672128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63340 payload_rows_remote=55170 dup_rows=2196 dup_rows_remote=2188 dedup_ratio=0.033508 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 4093, 4093, 3, 0, 4082, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8186, 6104, 8126, 8170, 8186, 8192] dup_rows_per_dst=[6, 0, 2, 66, 6, 8, 206, 1902] combine_rows_per_src=[8192, 8184, 8186, 6104, 8126, 8170, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65438 payload_rows_remote=57246 dup_rows=98 dup_rows_remote=14 dedup_ratio=0.001495 router_tokens_per_expert=[4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4051, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8108, 8192, 8192, 8184, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 84, 0, 0, 8, 0, 6] combine_rows_per_src=[8192, 8108, 8192, 8192, 8184, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=250496 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235131520 ingress_bytes=272629760 link_bytes=272629760 token_bytes=470012544 weight_movement_bytes=37748736 total_nvlink_bytes=507761280 weight_to_token_ratio=0.080314 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58950 payload_rows_remote=50758 dup_rows=6586 dup_rows_remote=6568 dedup_ratio=0.100494 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3264, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 1606, 8192, 8192] dup_rows_per_dst=[0, 18, 0, 0, 6566, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 1606, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=229441536 dispatch_meta_bytes=229296 combine_payload_bytes=229441536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229670832 ingress_bytes=248315904 link_bytes=248315904 token_bytes=459112368 weight_movement_bytes=18874368 total_nvlink_bytes=477986736 weight_to_token_ratio=0.041111 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379240813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225976320 dispatch_meta_bytes=229432 combine_payload_bytes=225976320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226205752 ingress_bytes=244850688 link_bytes=244850688 token_bytes=452182072 weight_movement_bytes=18874368 total_nvlink_bytes=471056440 weight_to_token_ratio=0.041741 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.346038818e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234479616 dispatch_meta_bytes=229040 combine_payload_bytes=234479616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234708656 ingress_bytes=253353984 link_bytes=253353984 token_bytes=469188272 weight_movement_bytes=18874368 total_nvlink_bytes=488062640 weight_to_token_ratio=0.040228 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427513217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=207904768 dispatch_meta_bytes=229304 combine_payload_bytes=207904768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208134072 ingress_bytes=226779136 link_bytes=226779136 token_bytes=416038840 weight_movement_bytes=18874368 total_nvlink_bytes=434913208 weight_to_token_ratio=0.045367 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.172886099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65104 payload_rows_remote=56914 dup_rows=432 dup_rows_remote=416 dedup_ratio=0.006592 router_tokens_per_expert=[4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3914, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4085, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8180, 7842, 8192, 8180, 8152, 8188, 8180, 8190] dup_rows_per_dst=[0, 0, 344, 12, 0, 30, 30, 16] combine_rows_per_src=[8180, 7842, 8192, 8180, 8152, 8188, 8180, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57316 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000641 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 7, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8180, 8192, 8178, 8190] dup_rows_per_dst=[28, 0, 0, 0, 0, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8180, 8192, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233119744 dispatch_meta_bytes=229320 combine_payload_bytes=233119744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233349064 ingress_bytes=251994112 link_bytes=251994112 token_bytes=466468808 weight_movement_bytes=18874368 total_nvlink_bytes=485343176 weight_to_token_ratio=0.040462 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414483592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229432 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234995768 ingress_bytes=253640704 link_bytes=253640704 token_bytes=469762104 weight_movement_bytes=18874368 total_nvlink_bytes=488636472 weight_to_token_ratio=0.040179 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430260427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61610 payload_rows_remote=53418 dup_rows=3926 dup_rows_remote=3926 dedup_ratio=0.059906 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 4280, 8178, 8192] dup_rows_per_dst=[0, 20, 12, 0, 3890, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 4280, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=98 dedup_ratio=0.002197 router_tokens_per_expert=[4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4047, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4073, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8100, 8192, 8192, 8146, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 92, 0, 0, 46, 0, 6] combine_rows_per_src=[8192, 8100, 8192, 8192, 8146, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=234848256 link_bytes=235077632 token_bytes=469925888 weight_movement_bytes=0 total_nvlink_bytes=469925888 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252398204e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61244 payload_rows_remote=53052 dup_rows=4292 dup_rows_remote=4286 dedup_ratio=0.065491 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2140, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 1953, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 3906, 8192, 8192] dup_rows_per_dst=[6, 0, 6, 0, 4280, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 3906, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65182 payload_rows_remote=57342 dup_rows=354 dup_rows_remote=354 dedup_ratio=0.005402 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 7840, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 354, 0, 0, 0, 0] combine_rows_per_src=[8192, 7840, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=218800128 dispatch_meta_bytes=229376 combine_payload_bytes=218800128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219029504 ingress_bytes=237674496 link_bytes=237674496 token_bytes=437829632 weight_movement_bytes=18874368 total_nvlink_bytes=456704000 weight_to_token_ratio=0.043109 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277280078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229192 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234520392 ingress_bytes=309788672 link_bytes=309788672 token_bytes=468811592 weight_movement_bytes=75497472 total_nvlink_bytes=544309064 weight_to_token_ratio=0.161040 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968242630e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60356 payload_rows_remote=52164 dup_rows=5180 dup_rows_remote=5180 dedup_ratio=0.079041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6744, 4460, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 3732, 1448, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6744, 4460, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=217300992 dispatch_meta_bytes=229352 combine_payload_bytes=217300992 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=217530344 ingress_bytes=292798464 link_bytes=292798464 token_bytes=434831336 weight_movement_bytes=75497472 total_nvlink_bytes=510328808 weight_to_token_ratio=0.173625 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.805450816e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=230784 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235103616 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469976448 weight_movement_bytes=37748736 total_nvlink_bytes=507725184 weight_to_token_ratio=0.080320 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63492 payload_rows_remote=55302 dup_rows=2044 dup_rows_remote=2040 dedup_ratio=0.031189 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3119, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090] payload_rows_per_dst=[8154, 8180, 8142, 8192, 6262, 8180, 8192, 8190] dup_rows_per_dst=[12, 0, 12, 0, 26, 1984, 6, 4] combine_rows_per_src=[8154, 8180, 8142, 8192, 6262, 8180, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60288 payload_rows_remote=52110 dup_rows=5248 dup_rows_remote=5092 dedup_ratio=0.080078 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 7, 0, 0, 0, 4089, 4089, 7, 0, 4085, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2508, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 8172, 8178, 3000, 8192, 8184, 8178, 8192] dup_rows_per_dst=[14, 0, 6, 4, 14, 26, 156, 5028] combine_rows_per_src=[8192, 8172, 8178, 3000, 8192, 8184, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=213663744 dispatch_meta_bytes=229376 combine_payload_bytes=213663744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213893120 ingress_bytes=232538112 link_bytes=232538112 token_bytes=427556864 weight_movement_bytes=18874368 total_nvlink_bytes=446431232 weight_to_token_ratio=0.044145 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228065773e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 101, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7824, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 166, 0, 202] combine_rows_per_src=[8192, 8192, 8192, 7824, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=226516992 dispatch_meta_bytes=229368 combine_payload_bytes=226516992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226746360 ingress_bytes=245391360 link_bytes=245391360 token_bytes=453263352 weight_movement_bytes=18874368 total_nvlink_bytes=472137720 weight_to_token_ratio=0.041641 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351219271e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=213442560 dispatch_meta_bytes=228808 combine_payload_bytes=213442560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213671368 ingress_bytes=232316928 link_bytes=232316928 token_bytes=427113928 weight_movement_bytes=18874368 total_nvlink_bytes=445988296 weight_to_token_ratio=0.044190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225946497e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64918 payload_rows_remote=56726 dup_rows=618 dup_rows_remote=618 dedup_ratio=0.009430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7616, 8192, 8192, 8188, 8154] dup_rows_per_dst=[0, 594, 0, 4, 0, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8192, 7616, 8192, 8192, 8188, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59124 payload_rows_remote=50938 dup_rows=6412 dup_rows_remote=12 dedup_ratio=0.097839 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 893, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8192, 8186, 8192, 8192, 1792, 8186, 8192, 8192] dup_rows_per_dst=[6, 0, 6, 0, 0, 6400, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 1792, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61034 payload_rows_remote=52842 dup_rows=4502 dup_rows_remote=4496 dedup_ratio=0.068695 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1846, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 3692] dup_rows_per_dst=[0, 6, 0, 4190, 0, 306, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 3692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 3] payload_rows_per_dst=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 66, 0, 0, 0, 6] combine_rows_per_src=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229376 combine_payload_bytes=233373696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233603072 ingress_bytes=289996800 link_bytes=289996800 token_bytes=466976768 weight_movement_bytes=56623104 total_nvlink_bytes=523599872 weight_to_token_ratio=0.121255 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.778606650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=232349696 dispatch_meta_bytes=229376 combine_payload_bytes=232349696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=232579072 ingress_bytes=232349696 link_bytes=232579072 token_bytes=464928768 weight_movement_bytes=0 total_nvlink_bytes=464928768 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228458232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=208642048 dispatch_meta_bytes=203800 combine_payload_bytes=208642048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208845848 ingress_bytes=265265152 link_bytes=265265152 token_bytes=417487896 weight_movement_bytes=56623104 total_nvlink_bytes=474111000 weight_to_token_ratio=0.135628 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.541640168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=216440832 dispatch_meta_bytes=229352 combine_payload_bytes=216440832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216670184 ingress_bytes=235315200 link_bytes=235315200 token_bytes=433111016 weight_movement_bytes=18874368 total_nvlink_bytes=451985384 weight_to_token_ratio=0.043579 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.254674464e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234815488 ingress_bytes=234586112 link_bytes=234815488 token_bytes=469401600 weight_movement_bytes=0 total_nvlink_bytes=469401600 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249886470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=124 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 6] payload_rows_per_dst=[8192, 8060, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 124, 0, 0, 0, 12] combine_rows_per_src=[8192, 8060, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57074 dup_rows=276 dup_rows_remote=276 dedup_ratio=0.004211 router_tokens_per_expert=[4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4071, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3983, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8156, 8192, 8192, 7966, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 52, 0, 0, 210, 0, 14] combine_rows_per_src=[8184, 8156, 8192, 8192, 7966, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234323968 dispatch_meta_bytes=229328 combine_payload_bytes=234323968 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234553296 ingress_bytes=272072704 link_bytes=272072704 token_bytes=468877264 weight_movement_bytes=37748736 total_nvlink_bytes=506626000 weight_to_token_ratio=0.080509 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606866782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64758 payload_rows_remote=56566 dup_rows=778 dup_rows_remote=778 dedup_ratio=0.011871 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7414, 8192, 8192] dup_rows_per_dst=[0, 0, 778, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7414, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233775104 dispatch_meta_bytes=229400 combine_payload_bytes=233775104 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234004504 ingress_bytes=271523840 link_bytes=271523840 token_bytes=467779608 weight_movement_bytes=37748736 total_nvlink_bytes=505528344 weight_to_token_ratio=0.080698 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601607837e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57372 payload_rows_remote=49180 dup_rows=8164 dup_rows_remote=8164 dedup_ratio=0.124573 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1490, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 8192, 5248, 8192, 2980] dup_rows_per_dst=[0, 18, 0, 7870, 0, 276, 0, 0] combine_rows_per_src=[8192, 8192, 8184, 8192, 8192, 5248, 8192, 2980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 3] payload_rows_per_dst=[8192, 8170, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 0, 6] combine_rows_per_src=[8192, 8170, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64642 payload_rows_remote=56450 dup_rows=894 dup_rows_remote=894 dedup_ratio=0.013641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 240, 0, 0, 0, 3409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7298, 8192, 8192] dup_rows_per_dst=[0, 0, 894, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7298, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62982 payload_rows_remote=54790 dup_rows=2554 dup_rows_remote=2554 dedup_ratio=0.038971 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2302, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 1708, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6710, 8186, 8186, 8192, 7132] dup_rows_per_dst=[12, 0, 0, 894, 0, 166, 0, 1482] combine_rows_per_src=[8192, 8192, 8192, 6710, 8186, 8186, 8192, 7132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231694336 dispatch_meta_bytes=229376 combine_payload_bytes=231694336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231923712 ingress_bytes=288317440 link_bytes=288317440 token_bytes=463618048 weight_movement_bytes=56623104 total_nvlink_bytes=520241152 weight_to_token_ratio=0.122133 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.762515849e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=201441280 dispatch_meta_bytes=229376 combine_payload_bytes=201441280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201670656 ingress_bytes=220315648 link_bytes=220315648 token_bytes=403111936 weight_movement_bytes=18874368 total_nvlink_bytes=421986304 weight_to_token_ratio=0.046822 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.110956137e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231219200 dispatch_meta_bytes=229376 combine_payload_bytes=231219200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231448576 ingress_bytes=250093568 link_bytes=250093568 token_bytes=462667776 weight_movement_bytes=18874368 total_nvlink_bytes=481542144 weight_to_token_ratio=0.040795 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396273515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235020288 ingress_bytes=253665280 link_bytes=253665280 token_bytes=469811200 weight_movement_bytes=18874368 total_nvlink_bytes=488685568 weight_to_token_ratio=0.040174 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430495902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63494 payload_rows_remote=57344 dup_rows=2042 dup_rows_remote=2042 dedup_ratio=0.031158 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 850, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6150, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 342, 0, 1700] combine_rows_per_src=[8192, 8192, 8192, 6150, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62762 payload_rows_remote=54570 dup_rows=2774 dup_rows_remote=874 dedup_ratio=0.042328 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 2773, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 4090, 0, 0, 0, 0, 0, 927, 0, 0, 0] payload_rows_per_dst=[8116, 8160, 8184, 5558, 8180, 8180, 8192, 8192] dup_rows_per_dst=[24, 8, 18, 2, 44, 778, 0, 1900] combine_rows_per_src=[8116, 8160, 8184, 5558, 8180, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=224419840 dispatch_meta_bytes=229376 combine_payload_bytes=224419840 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=224649216 ingress_bytes=281042944 link_bytes=281042944 token_bytes=449069056 weight_movement_bytes=56623104 total_nvlink_bytes=505692160 weight_to_token_ratio=0.126090 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.692815208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62070 payload_rows_remote=53878 dup_rows=3466 dup_rows_remote=3464 dedup_ratio=0.052887 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2371, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 1718, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089] payload_rows_per_dst=[8192, 8178, 8192, 8192, 4756, 8178, 8192, 8190] dup_rows_per_dst=[14, 0, 14, 0, 0, 3436, 2, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 4756, 8178, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=223518720 dispatch_meta_bytes=221776 combine_payload_bytes=223518720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=223740496 ingress_bytes=261267456 link_bytes=261267456 token_bytes=447259216 weight_movement_bytes=37748736 total_nvlink_bytes=485007952 weight_to_token_ratio=0.084400 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.503336212e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=237544 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235118568 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469999592 weight_movement_bytes=37748736 total_nvlink_bytes=507748328 weight_to_token_ratio=0.080317 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=220684288 dispatch_meta_bytes=229368 combine_payload_bytes=220684288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220913656 ingress_bytes=239558656 link_bytes=239558656 token_bytes=441597944 weight_movement_bytes=18874368 total_nvlink_bytes=460472312 weight_to_token_ratio=0.042741 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.295333172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57316 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000427 router_tokens_per_expert=[0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 4096, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8172, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8172, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65314 payload_rows_remote=57122 dup_rows=222 dup_rows_remote=222 dedup_ratio=0.003387 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 58, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7970, 8192, 8192] dup_rows_per_dst=[0, 0, 222, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7970, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65378 payload_rows_remote=57192 dup_rows=158 dup_rows_remote=158 dedup_ratio=0.002411 router_tokens_per_expert=[0, 4093, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 4096, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8186, 8192, 8192, 8192, 8060, 8178, 8192] dup_rows_per_dst=[2, 0, 0, 16, 0, 0, 140, 0] combine_rows_per_src=[8186, 8186, 8192, 8192, 8192, 8060, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61288 payload_rows_remote=53102 dup_rows=4248 dup_rows_remote=3930 dedup_ratio=0.064819 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1977, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 1959, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3960, 8182, 8186, 8192, 8192] dup_rows_per_dst=[12, 0, 0, 0, 0, 318, 0, 3918] combine_rows_per_src=[8192, 8192, 8192, 3960, 8182, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229376 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234995712 ingress_bytes=253640704 link_bytes=253640704 token_bytes=469762048 weight_movement_bytes=18874368 total_nvlink_bytes=488636416 weight_to_token_ratio=0.040179 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430260427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64744 payload_rows_remote=56552 dup_rows=792 dup_rows_remote=18 dedup_ratio=0.012085 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 0, 71, 0, 0, 0, 3635, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 7418, 8186, 8186] dup_rows_per_dst=[12, 0, 774, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 7418, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=233971712 dispatch_meta_bytes=229376 combine_payload_bytes=233971712 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234201088 ingress_bytes=271720448 link_bytes=271720448 token_bytes=468172800 weight_movement_bytes=37748736 total_nvlink_bytes=505921536 weight_to_token_ratio=0.080630 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603491638e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234258432 dispatch_meta_bytes=229400 combine_payload_bytes=234258432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234487832 ingress_bytes=253132800 link_bytes=253132800 token_bytes=468746264 weight_movement_bytes=18874368 total_nvlink_bytes=487620632 weight_to_token_ratio=0.040266 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425393940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58076 payload_rows_remote=49896 dup_rows=7460 dup_rows_remote=7422 dedup_ratio=0.113831 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 6, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 271, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 0, 0, 137, 0, 0, 0, 3682, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 8192, 844, 8184, 8192, 8182, 8180, 8180] dup_rows_per_dst=[20, 0, 2, 32, 0, 7330, 38, 38] combine_rows_per_src=[8122, 8192, 844, 8184, 8192, 8182, 8180, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=217505792 dispatch_meta_bytes=228128 combine_payload_bytes=217505792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217733920 ingress_bytes=236380160 link_bytes=236380160 token_bytes=435239712 weight_movement_bytes=18874368 total_nvlink_bytes=454114080 weight_to_token_ratio=0.043365 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264878387e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64576 payload_rows_remote=56384 dup_rows=960 dup_rows_remote=960 dedup_ratio=0.014648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 268, 0, 0, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7232, 8192, 8192] dup_rows_per_dst=[0, 0, 960, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7232, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 7] payload_rows_per_dst=[8192, 8174, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 14] combine_rows_per_src=[8192, 8174, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231636992 dispatch_meta_bytes=226280 combine_payload_bytes=231636992 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231863272 ingress_bytes=269385728 link_bytes=269385728 token_bytes=463500264 weight_movement_bytes=37748736 total_nvlink_bytes=501249000 weight_to_token_ratio=0.081443 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581121500e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=204374016 dispatch_meta_bytes=229272 combine_payload_bytes=204374016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=204603288 ingress_bytes=242122752 link_bytes=242122752 token_bytes=408977304 weight_movement_bytes=37748736 total_nvlink_bytes=446726040 weight_to_token_ratio=0.092300 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.319901078e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59986 payload_rows_remote=51794 dup_rows=5550 dup_rows_remote=5550 dedup_ratio=0.084686 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 2692, 4096, 0, 0, 0, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[5384, 8192, 8192, 8192, 8192, 8192, 5450, 8192] dup_rows_per_dst=[0, 5550, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5384, 8192, 8192, 8192, 8192, 8192, 5450, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=230948864 dispatch_meta_bytes=229376 combine_payload_bytes=230948864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231178240 ingress_bytes=268697600 link_bytes=268697600 token_bytes=462127104 weight_movement_bytes=37748736 total_nvlink_bytes=499875840 weight_to_token_ratio=0.081685 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.574528196e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036672 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843968 weight_movement_bytes=18874368 total_nvlink_bytes=488718336 weight_to_token_ratio=0.040172 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63304 payload_rows_remote=55112 dup_rows=2232 dup_rows_remote=2206 dedup_ratio=0.034058 router_tokens_per_expert=[0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 0, 0, 4096, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 8192, 8192, 8142, 6034, 8192] dup_rows_per_dst=[26, 0, 0, 8, 38, 2152, 8, 0] combine_rows_per_src=[8192, 8168, 8192, 8192, 8192, 8142, 6034, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61236 payload_rows_remote=53264 dup_rows=4300 dup_rows_remote=646 dedup_ratio=0.065613 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3258, 4096, 0, 0, 0, 0, 0, 0, 2196, 0, 0, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[6516, 7972, 8192, 8192, 8192, 8192, 5788, 8192] dup_rows_per_dst=[0, 3654, 0, 0, 646, 0, 0, 0] combine_rows_per_src=[6516, 7972, 8192, 8192, 8192, 8192, 5788, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=212148224 dispatch_meta_bytes=229376 combine_payload_bytes=212148224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212377600 ingress_bytes=231022592 link_bytes=231022592 token_bytes=424525824 weight_movement_bytes=18874368 total_nvlink_bytes=443400192 weight_to_token_ratio=0.044460 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.213544806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64032 payload_rows_remote=57326 dup_rows=1504 dup_rows_remote=1498 dedup_ratio=0.022949 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 0, 804, 0, 0, 0, 2546, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 6706, 8186, 8186] dup_rows_per_dst=[12, 0, 1486, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 6706, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65134 payload_rows_remote=56948 dup_rows=402 dup_rows_remote=402 dedup_ratio=0.006134 router_tokens_per_expert=[0, 4093, 4093, 0, 4093, 0, 0, 0, 3, 2, 0, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 1348, 0, 3, 0, 4093, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8186, 7814, 8184, 8188] dup_rows_per_dst=[6, 2, 0, 24, 0, 0, 370, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8186, 7814, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=225738752 dispatch_meta_bytes=229272 combine_payload_bytes=225738752 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=225968024 ingress_bytes=301236224 link_bytes=301236224 token_bytes=451706776 weight_movement_bytes=75497472 total_nvlink_bytes=527204248 weight_to_token_ratio=0.167138 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.886297281e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=218169344 dispatch_meta_bytes=215640 combine_payload_bytes=218169344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218384984 ingress_bytes=274792448 link_bytes=274792448 token_bytes=436554328 weight_movement_bytes=56623104 total_nvlink_bytes=493177432 weight_to_token_ratio=0.129705 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.632926030e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64576 payload_rows_remote=56396 dup_rows=960 dup_rows_remote=942 dedup_ratio=0.014648 router_tokens_per_expert=[0, 4090, 4084, 0, 4090, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 2929, 0, 6, 0, 4090, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8142, 8192, 8180, 8012, 8180, 7516, 8174, 8180] dup_rows_per_dst=[10, 10, 38, 36, 0, 168, 680, 18] combine_rows_per_src=[8142, 8192, 8180, 8012, 8180, 7516, 8174, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=235296 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235042592 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469849888 weight_movement_bytes=18874368 total_nvlink_bytes=488724256 weight_to_token_ratio=0.040171 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65078 payload_rows_remote=56886 dup_rows=458 dup_rows_remote=446 dedup_ratio=0.006989 router_tokens_per_expert=[0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 0, 0, 0, 4096, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7746, 8180, 8192] dup_rows_per_dst=[0, 0, 0, 12, 0, 0, 446, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7746, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63308 payload_rows_remote=55116 dup_rows=2228 dup_rows_remote=2228 dedup_ratio=0.033997 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 2989, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 0, 785, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5994, 8176, 8178, 8192, 8192] dup_rows_per_dst=[28, 0, 0, 0, 0, 630, 0, 1570] combine_rows_per_src=[8192, 8192, 8192, 5994, 8176, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233259008 dispatch_meta_bytes=229400 combine_payload_bytes=233259008 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233488408 ingress_bytes=271007744 link_bytes=271007744 token_bytes=466747416 weight_movement_bytes=37748736 total_nvlink_bytes=504496152 weight_to_token_ratio=0.080876 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596662859e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61516 payload_rows_remote=53324 dup_rows=4020 dup_rows_remote=4020 dedup_ratio=0.061340 router_tokens_per_expert=[0, 2086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4172, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4020, 0, 0] combine_rows_per_src=[4172, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230998016 dispatch_meta_bytes=229352 combine_payload_bytes=230998016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231227368 ingress_bytes=268746752 link_bytes=268746752 token_bytes=462225384 weight_movement_bytes=37748736 total_nvlink_bytes=499974120 weight_to_token_ratio=0.081667 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.574999146e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233005056 dispatch_meta_bytes=229328 combine_payload_bytes=233005056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233234384 ingress_bytes=251879424 link_bytes=251879424 token_bytes=466239440 weight_movement_bytes=18874368 total_nvlink_bytes=485113808 weight_to_token_ratio=0.040482 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413384708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=225755136 dispatch_meta_bytes=229376 combine_payload_bytes=225755136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225984512 ingress_bytes=244629504 link_bytes=244629504 token_bytes=451739648 weight_movement_bytes=18874368 total_nvlink_bytes=470614016 weight_to_token_ratio=0.041782 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.343919542e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63434 payload_rows_remote=55242 dup_rows=2102 dup_rows_remote=2102 dedup_ratio=0.032074 router_tokens_per_expert=[0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6090, 8192, 8192] dup_rows_per_dst=[2102, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6090, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57500 payload_rows_remote=49434 dup_rows=8036 dup_rows_remote=616 dedup_ratio=0.122620 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3664, 4094, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8066, 4122, 8192, 8192, 7162, 8192, 5456, 8118] dup_rows_per_dst=[7420, 616, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8066, 4122, 8192, 8192, 7162, 8192, 5456, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=218415104 dispatch_meta_bytes=229376 combine_payload_bytes=218415104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218644480 ingress_bytes=237289472 link_bytes=237289472 token_bytes=437059584 weight_movement_bytes=18874368 total_nvlink_bytes=455933952 weight_to_token_ratio=0.043185 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.273590967e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65184 payload_rows_remote=57292 dup_rows=352 dup_rows_remote=352 dedup_ratio=0.005371 router_tokens_per_expert=[0, 4093, 4093, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 427, 0, 3, 0, 4093, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8186, 7892, 8152, 8192] dup_rows_per_dst=[10, 6, 0, 48, 0, 0, 288, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8186, 7892, 8152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61634 payload_rows_remote=53442 dup_rows=3902 dup_rows_remote=3902 dedup_ratio=0.059540 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 3753, 4093, 0, 0, 0, 0, 0, 0, 1915, 0, 0, 3, 0, 0, 0, 0, 4049, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0] payload_rows_per_dst=[7506, 8128, 8192, 8192, 8192, 8186, 5046, 8192] dup_rows_per_dst=[0, 3812, 0, 0, 80, 0, 0, 10] combine_rows_per_src=[7506, 8128, 8192, 8192, 8192, 8186, 5046, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=226271232 dispatch_meta_bytes=229376 combine_payload_bytes=226271232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226500608 ingress_bytes=245145600 link_bytes=245145600 token_bytes=452771840 weight_movement_bytes=18874368 total_nvlink_bytes=471646208 weight_to_token_ratio=0.041686 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348864520e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=202481664 dispatch_meta_bytes=200200 combine_payload_bytes=202481664 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=202681864 ingress_bytes=277979136 link_bytes=277979136 token_bytes=405163528 weight_movement_bytes=75497472 total_nvlink_bytes=480661000 weight_to_token_ratio=0.186338 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.663459307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=218898432 dispatch_meta_bytes=229376 combine_payload_bytes=218898432 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=219127808 ingress_bytes=237772800 link_bytes=237772800 token_bytes=438026240 weight_movement_bytes=18874368 total_nvlink_bytes=456900608 weight_to_token_ratio=0.043090 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.278221978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58916 payload_rows_remote=50724 dup_rows=6620 dup_rows_remote=6618 dedup_ratio=0.101013 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 3823, 4090, 0, 0, 0, 0, 0, 0, 1084, 0, 0, 6, 0, 0, 0, 0, 3983, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 95, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0] payload_rows_per_dst=[7608, 2180, 8192, 8192, 8192, 8180, 8180, 8192] dup_rows_per_dst=[0, 12, 0, 0, 220, 38, 6348, 2] combine_rows_per_src=[7608, 2180, 8192, 8192, 8192, 8180, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59138 payload_rows_remote=50946 dup_rows=6398 dup_rows_remote=6398 dedup_ratio=0.097626 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3664, 4096, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[7328, 8192, 8192, 8192, 8192, 8192, 2658, 8192] dup_rows_per_dst=[0, 6398, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7328, 8192, 8192, 8192, 8192, 8192, 2658, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=230576 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234898608 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469566640 weight_movement_bytes=37748736 total_nvlink_bytes=507315376 weight_to_token_ratio=0.080391 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63264 payload_rows_remote=55086 dup_rows=2272 dup_rows_remote=2272 dedup_ratio=0.034668 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 7, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1116, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 0, 0, 282, 0, 0, 0, 2691, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8180, 8192, 5960, 8178, 8178] dup_rows_per_dst=[26, 0, 2232, 0, 0, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8180, 8192, 5960, 8178, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 3345, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 14, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=207765504 dispatch_meta_bytes=229368 combine_payload_bytes=207765504 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=207994872 ingress_bytes=226639872 link_bytes=226639872 token_bytes=415760376 weight_movement_bytes=18874368 total_nvlink_bytes=434634744 weight_to_token_ratio=0.045397 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.171551740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=208674816 dispatch_meta_bytes=229376 combine_payload_bytes=208674816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208904192 ingress_bytes=227549184 link_bytes=227549184 token_bytes=417579008 weight_movement_bytes=18874368 total_nvlink_bytes=436453376 weight_to_token_ratio=0.045200 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.180264320e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65452 payload_rows_remote=57260 dup_rows=84 dup_rows_remote=0 dedup_ratio=0.001282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 3416, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8108] dup_rows_per_dst=[0, 84, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=225632256 dispatch_meta_bytes=229432 combine_payload_bytes=225632256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225861688 ingress_bytes=244506624 link_bytes=244506624 token_bytes=451493944 weight_movement_bytes=18874368 total_nvlink_bytes=470368312 weight_to_token_ratio=0.041804 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.342742166e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64168 payload_rows_remote=55976 dup_rows=1368 dup_rows_remote=4 dedup_ratio=0.020874 router_tokens_per_expert=[0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 6828, 8192, 8192] dup_rows_per_dst=[1364, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 6828, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053056 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876736 weight_movement_bytes=18874368 total_nvlink_bytes=488751104 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57992 payload_rows_remote=49800 dup_rows=7544 dup_rows_remote=7532 dedup_ratio=0.115112 router_tokens_per_expert=[0, 3798, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4058, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 666, 8186, 8186] dup_rows_per_dst=[7532, 0, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 666, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234536960 dispatch_meta_bytes=229040 combine_payload_bytes=234536960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234766000 ingress_bytes=291160064 link_bytes=291160064 token_bytes=469302960 weight_movement_bytes=56623104 total_nvlink_bytes=525926064 weight_to_token_ratio=0.120654 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.789752474e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61530 payload_rows_remote=53344 dup_rows=4006 dup_rows_remote=4006 dedup_ratio=0.061127 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 3938, 4093, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 3, 0, 0, 0, 0, 4068, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0] payload_rows_per_dst=[7876, 4514, 8192, 8192, 8192, 8186, 8186, 8192] dup_rows_per_dst=[0, 76, 0, 0, 38, 0, 3892, 0] combine_rows_per_src=[7876, 4514, 8192, 8192, 8192, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=229277696 dispatch_meta_bytes=223920 combine_payload_bytes=229277696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229501616 ingress_bytes=248152064 link_bytes=248152064 token_bytes=458779312 weight_movement_bytes=18874368 total_nvlink_bytes=477653680 weight_to_token_ratio=0.041140 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.377670979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=203980800 dispatch_meta_bytes=229328 combine_payload_bytes=203980800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=204210128 ingress_bytes=260603904 link_bytes=260603904 token_bytes=408190928 weight_movement_bytes=56623104 total_nvlink_bytes=464814032 weight_to_token_ratio=0.138717 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.496978383e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62326 payload_rows_remote=54146 dup_rows=3210 dup_rows_remote=3172 dedup_ratio=0.048981 router_tokens_per_expert=[0, 2614, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4036, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5240, 8192, 8116, 8046, 8180, 8192, 8180, 8180] dup_rows_per_dst=[12, 0, 24, 4, 42, 3060, 30, 38] combine_rows_per_src=[5240, 8192, 8116, 8046, 8180, 8192, 8180, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=218497024 dispatch_meta_bytes=229400 combine_payload_bytes=218497024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=218726424 ingress_bytes=256245760 link_bytes=256245760 token_bytes=437223448 weight_movement_bytes=37748736 total_nvlink_bytes=474972184 weight_to_token_ratio=0.086337 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.455220792e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62006 payload_rows_remote=53814 dup_rows=3530 dup_rows_remote=3342 dedup_ratio=0.053864 router_tokens_per_expert=[0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4934, 8192, 8192, 8192, 8192, 7920, 8192, 8192] dup_rows_per_dst=[224, 0, 0, 188, 0, 3118, 0, 0] combine_rows_per_src=[4934, 8192, 8192, 8192, 8192, 7920, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65172 payload_rows_remote=56996 dup_rows=364 dup_rows_remote=42 dedup_ratio=0.005554 router_tokens_per_expert=[0, 4089, 4089, 0, 4089, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 7, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 1489, 0, 7, 0, 4089, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8192, 8178, 7874, 8176, 8190] dup_rows_per_dst=[2, 12, 0, 28, 0, 0, 322, 0] combine_rows_per_src=[8192, 8192, 8178, 8192, 8178, 7874, 8176, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61784 payload_rows_remote=53592 dup_rows=3752 dup_rows_remote=3752 dedup_ratio=0.057251 router_tokens_per_expert=[0, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[4440, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 3752, 0, 0, 0, 0, 0] combine_rows_per_src=[4440, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=221782016 dispatch_meta_bytes=229272 combine_payload_bytes=221782016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=222011288 ingress_bytes=259530752 link_bytes=259530752 token_bytes=443793304 weight_movement_bytes=37748736 total_nvlink_bytes=481542040 weight_to_token_ratio=0.085059 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.486695969e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58388 payload_rows_remote=50196 dup_rows=7148 dup_rows_remote=7148 dedup_ratio=0.109070 router_tokens_per_expert=[0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[4932, 8192, 8192, 8192, 8192, 8192, 8192, 4304] dup_rows_per_dst=[2818, 0, 4292, 0, 0, 38, 0, 0] combine_rows_per_src=[4932, 8192, 8192, 8192, 8192, 8192, 8192, 4304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=220422144 dispatch_meta_bytes=228624 combine_payload_bytes=220422144 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=220650768 ingress_bytes=333668352 link_bytes=333668352 token_bytes=441072912 weight_movement_bytes=113246208 total_nvlink_bytes=554319120 weight_to_token_ratio=0.256752 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.197045974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=228152 combine_payload_bytes=233455616 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=233683768 ingress_bytes=327827456 link_bytes=327827456 token_bytes=467139384 weight_movement_bytes=94371840 total_nvlink_bytes=561511224 weight_to_token_ratio=0.202021 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.141081382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65328 payload_rows_remote=57136 dup_rows=208 dup_rows_remote=208 dedup_ratio=0.003174 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 60, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 3751, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8110, 8192, 8066] dup_rows_per_dst=[0, 152, 38, 18, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8110, 8192, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=219512832 dispatch_meta_bytes=229376 combine_payload_bytes=219512832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219742208 ingress_bytes=238387200 link_bytes=238387200 token_bytes=439255040 weight_movement_bytes=18874368 total_nvlink_bytes=458129408 weight_to_token_ratio=0.042969 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284108857e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65382 payload_rows_remote=57196 dup_rows=154 dup_rows_remote=154 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 70, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 3579, 0] payload_rows_per_dst=[8192, 8192, 8186, 8190, 8192, 8192, 8192, 8046] dup_rows_per_dst=[0, 140, 0, 0, 2, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8190, 8192, 8192, 8192, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=205602816 dispatch_meta_bytes=229376 combine_payload_bytes=205602816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205832192 ingress_bytes=243351552 link_bytes=243351552 token_bytes=411435008 weight_movement_bytes=37748736 total_nvlink_bytes=449183744 weight_to_token_ratio=0.091749 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331674835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65262 payload_rows_remote=57072 dup_rows=274 dup_rows_remote=18 dedup_ratio=0.004181 router_tokens_per_expert=[0, 3978, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4080, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7962, 8192, 8192, 8166, 8186, 8190, 8188, 8186] dup_rows_per_dst=[6, 0, 12, 0, 0, 256, 0, 0] combine_rows_per_src=[7962, 8192, 8192, 8166, 8186, 8190, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234029056 dispatch_meta_bytes=229376 combine_payload_bytes=234029056 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234258432 ingress_bytes=234029056 link_bytes=234258432 token_bytes=468287488 weight_movement_bytes=0 total_nvlink_bytes=468287488 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.244549033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234274816 dispatch_meta_bytes=229400 combine_payload_bytes=234274816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234504216 ingress_bytes=253149184 link_bytes=253149184 token_bytes=468779032 weight_movement_bytes=18874368 total_nvlink_bytes=487653400 weight_to_token_ratio=0.040263 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425550924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64484 payload_rows_remote=56496 dup_rows=1052 dup_rows_remote=232 dedup_ratio=0.016052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 18, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 82, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4087, 4089, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 3767, 0] payload_rows_per_dst=[8192, 7372, 8180, 8192, 8178, 8190, 8192, 7988] dup_rows_per_dst=[0, 200, 0, 0, 0, 32, 0, 820] combine_rows_per_src=[8192, 7372, 8180, 8192, 8178, 8190, 8192, 7988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=233766912 dispatch_meta_bytes=228360 combine_payload_bytes=233766912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233995272 ingress_bytes=252641280 link_bytes=252641280 token_bytes=467762184 weight_movement_bytes=18874368 total_nvlink_bytes=486636552 weight_to_token_ratio=0.040350 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420684438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57288 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 0, 0, 2706, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8136] dup_rows_per_dst=[0, 56, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59060 payload_rows_remote=50882 dup_rows=6476 dup_rows_remote=122 dedup_ratio=0.098816 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3916, 4089, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 7, 0, 0, 0, 0, 3964, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0] payload_rows_per_dst=[7832, 2104, 8192, 8192, 8192, 8178, 8178, 8192] dup_rows_per_dst=[0, 14, 0, 0, 46, 0, 6354, 62] combine_rows_per_src=[7832, 2104, 8192, 8192, 8192, 8178, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=231407616 dispatch_meta_bytes=226912 combine_payload_bytes=231407616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231634528 ingress_bytes=250281984 link_bytes=250281984 token_bytes=463042144 weight_movement_bytes=18874368 total_nvlink_bytes=481916512 weight_to_token_ratio=0.040762 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398078824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234651648 dispatch_meta_bytes=229376 combine_payload_bytes=234651648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234881024 ingress_bytes=253526016 link_bytes=253526016 token_bytes=469532672 weight_movement_bytes=18874368 total_nvlink_bytes=488407040 weight_to_token_ratio=0.040198 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429161543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=208412672 dispatch_meta_bytes=204016 combine_payload_bytes=208412672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208616688 ingress_bytes=227287040 link_bytes=227287040 token_bytes=417029360 weight_movement_bytes=18874368 total_nvlink_bytes=435903728 weight_to_token_ratio=0.045259 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.177752585e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57336 payload_rows_remote=49144 dup_rows=8200 dup_rows_remote=2130 dedup_ratio=0.125122 router_tokens_per_expert=[0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 0] dup_rows_per_dst=[6070, 8, 2122, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65276 payload_rows_remote=57084 dup_rows=260 dup_rows_remote=12 dedup_ratio=0.003967 router_tokens_per_expert=[0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 118, 0, 0, 0, 3, 0, 3, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] payload_rows_per_dst=[7950, 8192, 8192, 8192, 8192, 8186, 8186, 8186] dup_rows_per_dst=[0, 6, 248, 0, 6, 0, 0, 0] combine_rows_per_src=[7950, 8192, 8192, 8192, 8192, 8186, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65414 payload_rows_remote=57222 dup_rows=122 dup_rows_remote=110 dedup_ratio=0.001862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 54, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4009, 0] payload_rows_per_dst=[8192, 8192, 8186, 8190, 8192, 8192, 8192, 8078] dup_rows_per_dst=[0, 110, 0, 0, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8190, 8192, 8192, 8192, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=201293824 dispatch_meta_bytes=205096 combine_payload_bytes=201293824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201498920 ingress_bytes=220168192 link_bytes=220168192 token_bytes=402792744 weight_movement_bytes=18874368 total_nvlink_bytes=421667112 weight_to_token_ratio=0.046859 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109543286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233816064 dispatch_meta_bytes=228384 combine_payload_bytes=233816064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234044448 ingress_bytes=252690432 link_bytes=252690432 token_bytes=467860512 weight_movement_bytes=18874368 total_nvlink_bytes=486734880 weight_to_token_ratio=0.040342 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421155388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234381312 dispatch_meta_bytes=229328 combine_payload_bytes=234381312 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234610640 ingress_bytes=309878784 link_bytes=309878784 token_bytes=468991952 weight_movement_bytes=75497472 total_nvlink_bytes=544489424 weight_to_token_ratio=0.160978 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.969106038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65290 payload_rows_remote=57098 dup_rows=246 dup_rows_remote=246 dedup_ratio=0.003754 router_tokens_per_expert=[0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[7946, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 246, 0, 0, 0, 0, 0] combine_rows_per_src=[7946, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57224 payload_rows_remote=49044 dup_rows=8312 dup_rows_remote=86 dedup_ratio=0.126831 router_tokens_per_expert=[0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 35, 0, 0, 0, 2, 0, 6, 0, 0, 0, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] payload_rows_per_dst=[8110, 8146, 82, 8192, 8154, 8180, 8180, 8180] dup_rows_per_dst=[0, 12, 16, 8, 12, 0, 38, 8226] combine_rows_per_src=[8110, 8146, 82, 8192, 8154, 8180, 8180, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64680 payload_rows_remote=56502 dup_rows=856 dup_rows_remote=854 dedup_ratio=0.013062 router_tokens_per_expert=[0, 3692, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4079, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7398, 8192, 8192, 8172, 8178, 8192, 8178, 8178] dup_rows_per_dst=[14, 0, 28, 0, 0, 812, 2, 0] combine_rows_per_src=[7398, 8192, 8192, 8172, 8178, 8192, 8178, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57304 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8152, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 40, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8152, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233873408 dispatch_meta_bytes=229376 combine_payload_bytes=233873408 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234102784 ingress_bytes=252747776 link_bytes=252747776 token_bytes=467976192 weight_movement_bytes=18874368 total_nvlink_bytes=486850560 weight_to_token_ratio=0.040332 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421704830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=200884224 dispatch_meta_bytes=196520 combine_payload_bytes=200884224 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=201080744 ingress_bytes=295256064 link_bytes=295256064 token_bytes=401964968 weight_movement_bytes=94371840 total_nvlink_bytes=496336808 weight_to_token_ratio=0.234776 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.828998330e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64356 payload_rows_remote=56164 dup_rows=1180 dup_rows_remote=1176 dedup_ratio=0.018005 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7052, 8158, 8186] dup_rows_per_dst=[4, 0, 36, 0, 1140, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7052, 8158, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=231432192 dispatch_meta_bytes=229424 combine_payload_bytes=231432192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231661616 ingress_bytes=250306560 link_bytes=250306560 token_bytes=463093808 weight_movement_bytes=18874368 total_nvlink_bytes=481968176 weight_to_token_ratio=0.040757 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398314299e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65426 payload_rows_remote=57240 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.001678 router_tokens_per_expert=[3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4093, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8186, 8192, 8100, 8186, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 12, 88, 6, 0] combine_rows_per_src=[8192, 8186, 8186, 8192, 8100, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229376 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234946560 ingress_bytes=272465920 link_bytes=272465920 token_bytes=469663744 weight_movement_bytes=37748736 total_nvlink_bytes=507412480 weight_to_token_ratio=0.080374 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610634384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64618 payload_rows_remote=56432 dup_rows=918 dup_rows_remote=918 dedup_ratio=0.014008 router_tokens_per_expert=[0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 447, 0, 0, 0, 3, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] payload_rows_per_dst=[7292, 8192, 8192, 8192, 8192, 8186, 8186, 8186] dup_rows_per_dst=[0, 6, 906, 0, 6, 0, 0, 0] combine_rows_per_src=[7292, 8192, 8192, 8192, 8192, 8186, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=230047744 dispatch_meta_bytes=229360 combine_payload_bytes=230047744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230277104 ingress_bytes=248922112 link_bytes=248922112 token_bytes=460324848 weight_movement_bytes=18874368 total_nvlink_bytes=479199216 weight_to_token_ratio=0.041002 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.385049200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229400 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234684440 ingress_bytes=328826880 link_bytes=328826880 token_bytes=469139480 weight_movement_bytes=94371840 total_nvlink_bytes=563511320 weight_to_token_ratio=0.201159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.150657371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=231145472 dispatch_meta_bytes=229400 combine_payload_bytes=231145472 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231374872 ingress_bytes=268894208 link_bytes=268894208 token_bytes=462520344 weight_movement_bytes=37748736 total_nvlink_bytes=500269080 weight_to_token_ratio=0.081615 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576411997e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8160, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 32, 0, 0] combine_rows_per_src=[8192, 8188, 8160, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8174, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 18, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8174, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65172 payload_rows_remote=56980 dup_rows=364 dup_rows_remote=360 dedup_ratio=0.005554 router_tokens_per_expert=[6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 4090, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 4090, 6, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8156, 8180, 8192, 7890, 8180, 8192, 8192] dup_rows_per_dst=[0, 34, 2, 0, 48, 266, 10, 4] combine_rows_per_src=[8190, 8156, 8180, 8192, 7890, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 32, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4088, 4089, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 3154, 0] payload_rows_per_dst=[8192, 8192, 8178, 8178, 8190, 8192, 8192, 8116] dup_rows_per_dst=[0, 64, 0, 0, 14, 28, 0, 0] combine_rows_per_src=[8192, 8192, 8178, 8178, 8190, 8192, 8192, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65406 payload_rows_remote=57214 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 65, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8062, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 130, 0, 0] combine_rows_per_src=[8192, 8192, 8062, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036672 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843968 weight_movement_bytes=18874368 total_nvlink_bytes=488718336 weight_to_token_ratio=0.040172 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=229360 combine_payload_bytes=233390080 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233619440 ingress_bytes=271138816 link_bytes=271138816 token_bytes=467009520 weight_movement_bytes=37748736 total_nvlink_bytes=504758256 weight_to_token_ratio=0.080831 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597918726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8188, 8152, 8192, 8192] dup_rows_per_dst=[0, 38, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8188, 8152, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229376 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234676224 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469123072 weight_movement_bytes=18874368 total_nvlink_bytes=487997440 weight_to_token_ratio=0.040233 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 4085, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234348544 dispatch_meta_bytes=229376 combine_payload_bytes=234348544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234577920 ingress_bytes=272097280 link_bytes=272097280 token_bytes=468926464 weight_movement_bytes=37748736 total_nvlink_bytes=506675200 weight_to_token_ratio=0.080500 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607102257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65418 payload_rows_remote=57232 dup_rows=118 dup_rows_remote=24 dedup_ratio=0.001801 router_tokens_per_expert=[3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 4093, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8186, 8192, 8092, 8186, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 12, 94, 6, 0] combine_rows_per_src=[8192, 8186, 8186, 8192, 8092, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234921984 ingress_bytes=234692608 link_bytes=234921984 token_bytes=469614592 weight_movement_bytes=0 total_nvlink_bytes=469614592 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250906862e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234422272 dispatch_meta_bytes=229024 combine_payload_bytes=234422272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234651296 ingress_bytes=253296640 link_bytes=253296640 token_bytes=469073568 weight_movement_bytes=18874368 total_nvlink_bytes=487947936 weight_to_token_ratio=0.040238 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426963775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57324 dup_rows=22 dup_rows_remote=14 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 26, 4064, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8184, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 8, 4, 2, 0, 8] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8184, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61444 payload_rows_remote=53266 dup_rows=4092 dup_rows_remote=4092 dedup_ratio=0.062439 router_tokens_per_expert=[0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 0, 0, 0, 7, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0] payload_rows_per_dst=[7786, 8192, 4548, 8192, 8192, 8178, 8178, 8178] dup_rows_per_dst=[0, 14, 172, 0, 14, 0, 0, 3892] combine_rows_per_src=[7786, 8192, 4548, 8192, 8192, 8178, 8178, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63128 payload_rows_remote=54936 dup_rows=2408 dup_rows_remote=2408 dedup_ratio=0.036743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8192, 8164, 5890, 8144, 8192] dup_rows_per_dst=[0, 580, 1798, 0, 0, 30, 0, 0] combine_rows_per_src=[8192, 8192, 8162, 8192, 8164, 5890, 8144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229352 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028456 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827560 weight_movement_bytes=18874368 total_nvlink_bytes=488701928 weight_to_token_ratio=0.040173 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=218177536 dispatch_meta_bytes=229432 combine_payload_bytes=218177536 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=218406968 ingress_bytes=237051904 link_bytes=237051904 token_bytes=436584504 weight_movement_bytes=18874368 total_nvlink_bytes=455458872 weight_to_token_ratio=0.043232 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.271314707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64618 payload_rows_remote=56984 dup_rows=918 dup_rows_remote=918 dedup_ratio=0.014008 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 86, 3, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 367, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7838, 7634, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 172, 0, 0, 6, 734, 6, 0] combine_rows_per_src=[8192, 7838, 7634, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=6 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 6, 0, 2, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=225017856 dispatch_meta_bytes=229376 combine_payload_bytes=225017856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225247232 ingress_bytes=243892224 link_bytes=243892224 token_bytes=450265088 weight_movement_bytes=18874368 total_nvlink_bytes=469139456 weight_to_token_ratio=0.041918 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.336855288e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=233406464 dispatch_meta_bytes=231608 combine_payload_bytes=233406464 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233638072 ingress_bytes=290029568 link_bytes=290029568 token_bytes=467044536 weight_movement_bytes=56623104 total_nvlink_bytes=523667640 weight_to_token_ratio=0.121237 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.778920617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229368 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235077624 ingress_bytes=310345728 link_bytes=310345728 token_bytes=469925880 weight_movement_bytes=75497472 total_nvlink_bytes=545423352 weight_to_token_ratio=0.160658 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973580066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57312 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 30, 0, 0] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64766 payload_rows_remote=56574 dup_rows=770 dup_rows_remote=758 dedup_ratio=0.011749 router_tokens_per_expert=[7, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 4089, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 4089, 7, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8180, 8178, 8192, 7464, 8178, 8192, 8192] dup_rows_per_dst=[0, 16, 0, 0, 28, 712, 12, 2] combine_rows_per_src=[8190, 8180, 8178, 8192, 7464, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65340 payload_rows_remote=57148 dup_rows=196 dup_rows_remote=196 dedup_ratio=0.002991 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 20, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 61, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8018, 8182, 8192, 8180, 8192, 8192] dup_rows_per_dst=[0, 40, 0, 2, 10, 122, 22, 0] combine_rows_per_src=[8192, 8192, 8018, 8182, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229376 combine_payload_bytes=234749952 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234979328 ingress_bytes=253624320 link_bytes=253624320 token_bytes=469729280 weight_movement_bytes=18874368 total_nvlink_bytes=488603648 weight_to_token_ratio=0.040181 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430103443e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65422 payload_rows_remote=57230 dup_rows=114 dup_rows_remote=112 dedup_ratio=0.001740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8080, 8192] dup_rows_per_dst=[2, 8, 30, 74, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8080, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=231727104 dispatch_meta_bytes=229328 combine_payload_bytes=231727104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231956432 ingress_bytes=250601472 link_bytes=250601472 token_bytes=463683536 weight_movement_bytes=18874368 total_nvlink_bytes=482557904 weight_to_token_ratio=0.040705 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401140001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8190, 8192, 8186, 8192, 8186, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 18, 2] combine_rows_per_src=[8192, 8190, 8192, 8186, 8192, 8186, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=229376 combine_payload_bytes=234078208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234307584 ingress_bytes=252952576 link_bytes=252952576 token_bytes=468385792 weight_movement_bytes=18874368 total_nvlink_bytes=487260160 weight_to_token_ratio=0.040297 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423667123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64252 payload_rows_remote=56066 dup_rows=1284 dup_rows_remote=58 dedup_ratio=0.019592 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 613, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7700, 7406, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 46, 0, 4, 2, 1226, 6, 0] combine_rows_per_src=[8192, 7700, 7406, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=229368 combine_payload_bytes=234414080 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234643448 ingress_bytes=272162816 link_bytes=272162816 token_bytes=469057528 weight_movement_bytes=37748736 total_nvlink_bytes=506806264 weight_to_token_ratio=0.080478 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607730190e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235028480 ingress_bytes=291422208 link_bytes=291422208 token_bytes=469827584 weight_movement_bytes=56623104 total_nvlink_bytes=526450688 weight_to_token_ratio=0.120519 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792264208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=229646336 dispatch_meta_bytes=224496 combine_payload_bytes=229646336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229870832 ingress_bytes=286269440 link_bytes=286269440 token_bytes=459517168 weight_movement_bytes=56623104 total_nvlink_bytes=516140272 weight_to_token_ratio=0.123223 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.742892921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 55, 4034, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 14, 0, 4, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57250 dup_rows=106 dup_rows_remote=76 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 4090, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 6, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8190, 8162, 8144, 8192, 8180, 8192, 8180] dup_rows_per_dst=[2, 0, 0, 0, 8, 32, 34, 30] combine_rows_per_src=[8190, 8190, 8162, 8144, 8192, 8180, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8166, 8192, 8192, 8182, 8192] dup_rows_per_dst=[0, 26, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8166, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235110400 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991424 weight_movement_bytes=94371840 total_nvlink_bytes=564363264 weight_to_token_ratio=0.200795 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8190, 8192, 8192, 8192, 8186] dup_rows_per_dst=[2, 0, 0, 6, 6, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8190, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235036672 ingress_bytes=234807296 link_bytes=235036672 token_bytes=469843968 weight_movement_bytes=0 total_nvlink_bytes=469843968 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252005746e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57332 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8186, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 18, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8186, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229304 combine_payload_bytes=234496000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234725304 ingress_bytes=253370368 link_bytes=253370368 token_bytes=469221304 weight_movement_bytes=18874368 total_nvlink_bytes=488095672 weight_to_token_ratio=0.040225 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427670200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053056 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876736 weight_movement_bytes=18874368 total_nvlink_bytes=488751104 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229400 combine_payload_bytes=234831872 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235061272 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893144 weight_movement_bytes=37748736 total_nvlink_bytes=507641880 weight_to_token_ratio=0.080335 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=58 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4, 7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 18, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8136, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 8, 0, 4, 10, 36, 14, 0] combine_rows_per_src=[8192, 8190, 8136, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64880 payload_rows_remote=56688 dup_rows=656 dup_rows_remote=656 dedup_ratio=0.010010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 323, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[7536, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 656, 0] combine_rows_per_src=[7536, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57270 dup_rows=86 dup_rows_remote=48 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 4090, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 6, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8180, 8182, 8190, 8154, 8192, 8192, 8180] dup_rows_per_dst=[2, 12, 0, 12, 12, 0, 10, 38] combine_rows_per_src=[8180, 8180, 8182, 8190, 8154, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=38 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8154, 8192, 8192, 8192, 8186] dup_rows_per_dst=[6, 38, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8154, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229320 combine_payload_bytes=234586112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234815432 ingress_bytes=272334848 link_bytes=272334848 token_bytes=469401544 weight_movement_bytes=37748736 total_nvlink_bytes=507150280 weight_to_token_ratio=0.080419 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609378516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 6, 6, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8190, 8192, 8192, 8192, 8186] dup_rows_per_dst=[2, 0, 0, 6, 6, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8190, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=232194048 dispatch_meta_bytes=229376 combine_payload_bytes=232194048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232423424 ingress_bytes=251068416 link_bytes=251068416 token_bytes=464617472 weight_movement_bytes=18874368 total_nvlink_bytes=483491840 weight_to_token_ratio=0.040623 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405614029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229272 combine_payload_bytes=234577920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234807192 ingress_bytes=253452288 link_bytes=253452288 token_bytes=469385112 weight_movement_bytes=18874368 total_nvlink_bytes=488259480 weight_to_token_ratio=0.040211 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428455117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229352 combine_payload_bytes=234700800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234930152 ingress_bytes=291323904 link_bytes=291323904 token_bytes=469630952 weight_movement_bytes=56623104 total_nvlink_bytes=526254056 weight_to_token_ratio=0.120569 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791322308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65260 payload_rows_remote=57336 dup_rows=276 dup_rows_remote=276 dedup_ratio=0.004211 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 7924, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 276, 0, 0] combine_rows_per_src=[8192, 8192, 8184, 8192, 7924, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235053056 ingress_bytes=234823680 link_bytes=235053056 token_bytes=469876736 weight_movement_bytes=0 total_nvlink_bytes=469876736 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252162729e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=230448 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235078704 ingress_bytes=310345728 link_bytes=310345728 token_bytes=469926960 weight_movement_bytes=75497472 total_nvlink_bytes=545424432 weight_to_token_ratio=0.160658 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973580066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=12 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8190, 8188, 8192, 8178, 8186, 8178, 8192, 8178] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 42, 6] combine_rows_per_src=[8190, 8188, 8192, 8178, 8186, 8178, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8140, 8192, 8156, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 88, 0, 0] combine_rows_per_src=[8192, 8192, 8140, 8192, 8156, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=66 dedup_ratio=0.001038 router_tokens_per_expert=[6, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8180, 8192, 8192, 8154, 8190, 8180, 8192] dup_rows_per_dst=[0, 2, 0, 12, 12, 2, 38, 2] combine_rows_per_src=[8188, 8180, 8192, 8192, 8154, 8190, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64036 payload_rows_remote=55844 dup_rows=1500 dup_rows_remote=78 dedup_ratio=0.022888 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 163, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8152, 6770, 8154] dup_rows_per_dst=[1422, 0, 38, 2, 0, 0, 38, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8152, 6770, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229208 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889048 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469548888 weight_movement_bytes=18874368 total_nvlink_bytes=488423256 weight_to_token_ratio=0.040197 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65408 payload_rows_remote=57222 dup_rows=128 dup_rows_remote=128 dedup_ratio=0.001953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 55, 4093, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8076, 8192, 8186, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 110, 6] combine_rows_per_src=[8076, 8192, 8186, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229376 combine_payload_bytes=234512384 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234741760 ingress_bytes=234512384 link_bytes=234741760 token_bytes=469254144 weight_movement_bytes=0 total_nvlink_bytes=469254144 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249180044e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 6, 6, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229368 combine_payload_bytes=234602496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234831864 ingress_bytes=253476864 link_bytes=253476864 token_bytes=469434360 weight_movement_bytes=18874368 total_nvlink_bytes=488308728 weight_to_token_ratio=0.040207 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428690592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=228737024 dispatch_meta_bytes=223688 combine_payload_bytes=228737024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228960712 ingress_bytes=247611392 link_bytes=247611392 token_bytes=457697736 weight_movement_bytes=18874368 total_nvlink_bytes=476572104 weight_to_token_ratio=0.041238 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372490526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234381312 dispatch_meta_bytes=229400 combine_payload_bytes=234381312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234610712 ingress_bytes=253255680 link_bytes=253255680 token_bytes=468992024 weight_movement_bytes=18874368 total_nvlink_bytes=487866392 weight_to_token_ratio=0.040245 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426571316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235061248 ingress_bytes=310329344 link_bytes=310329344 token_bytes=469893120 weight_movement_bytes=75497472 total_nvlink_bytes=545390592 weight_to_token_ratio=0.160669 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973423083e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 4089, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 7, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8186, 8192, 8192, 8192, 8178] dup_rows_per_dst=[6, 0, 0, 14, 14, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8186, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64936 payload_rows_remote=56744 dup_rows=600 dup_rows_remote=588 dedup_ratio=0.009155 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 0, 0, 242, 4090, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4090, 0, 0, 0, 0] payload_rows_per_dst=[7692, 8116, 8180, 8190, 8192, 8182, 8192, 8192] dup_rows_per_dst=[0, 24, 0, 0, 0, 76, 488, 12] combine_rows_per_src=[7692, 8116, 8180, 8190, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229376 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234971136 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469712896 weight_movement_bytes=18874368 total_nvlink_bytes=488587264 weight_to_token_ratio=0.040183 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57894 payload_rows_remote=49702 dup_rows=7642 dup_rows_remote=7642 dedup_ratio=0.116608 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8184, 598, 8192, 8154] dup_rows_per_dst=[0, 0, 2368, 38, 5234, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8184, 598, 8192, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64780 payload_rows_remote=57010 dup_rows=756 dup_rows_remote=756 dedup_ratio=0.011536 router_tokens_per_expert=[3, 0, 0, 4093, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 4093, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 0, 4093, 4093, 0, 2, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8192, 7770, 8186, 7866, 8190, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 0, 2, 740, 0, 6] combine_rows_per_src=[8192, 8192, 7770, 8186, 7866, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65436 payload_rows_remote=57250 dup_rows=100 dup_rows_remote=100 dedup_ratio=0.001526 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 41, 4093, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8192, 8186, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 82, 6] combine_rows_per_src=[8104, 8192, 8186, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57304 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8152, 8192] dup_rows_per_dst=[0, 0, 0, 40, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=232423424 dispatch_meta_bytes=229328 combine_payload_bytes=232423424 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=232652752 ingress_bytes=251297792 link_bytes=251297792 token_bytes=465076176 weight_movement_bytes=18874368 total_nvlink_bytes=483950544 weight_to_token_ratio=0.040583 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407811797e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=203579392 dispatch_meta_bytes=229376 combine_payload_bytes=203579392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=203808768 ingress_bytes=222453760 link_bytes=222453760 token_bytes=407388160 weight_movement_bytes=18874368 total_nvlink_bytes=426262528 weight_to_token_ratio=0.046330 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.131442474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233512960 dispatch_meta_bytes=231064 combine_payload_bytes=233512960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233744024 ingress_bytes=252387328 link_bytes=252387328 token_bytes=467256984 weight_movement_bytes=18874368 total_nvlink_bytes=486131352 weight_to_token_ratio=0.040394 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418251195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229400 combine_payload_bytes=234496000 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234725400 ingress_bytes=291119104 link_bytes=291119104 token_bytes=469221400 weight_movement_bytes=56623104 total_nvlink_bytes=525844504 weight_to_token_ratio=0.120675 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.789360015e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64632 payload_rows_remote=56440 dup_rows=904 dup_rows_remote=904 dedup_ratio=0.013794 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7288, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 904, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7288, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229376 combine_payload_bytes=234717184 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234946560 ingress_bytes=234717184 link_bytes=234946560 token_bytes=469663744 weight_movement_bytes=0 total_nvlink_bytes=469663744 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.251142337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57326 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4088, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8176, 8192, 8192, 8192, 8192, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 14, 14, 0, 0, 4] combine_rows_per_src=[8190, 8176, 8192, 8192, 8192, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.000977 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 4000, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8128, 8192] dup_rows_per_dst=[0, 0, 0, 64, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8128, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=231178240 dispatch_meta_bytes=229376 combine_payload_bytes=231178240 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231407616 ingress_bytes=268926976 link_bytes=268926976 token_bytes=462585856 weight_movement_bytes=37748736 total_nvlink_bytes=500334592 weight_to_token_ratio=0.081604 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576725964e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64382 payload_rows_remote=56190 dup_rows=1154 dup_rows_remote=1142 dedup_ratio=0.017609 router_tokens_per_expert=[6, 0, 0, 4090, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 4090, 0, 5, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8150, 8192, 7786, 8180, 7500, 8190, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 1128, 0, 12] combine_rows_per_src=[8150, 8192, 7786, 8180, 7500, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229432 combine_payload_bytes=234807296 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235036728 ingress_bytes=272556032 link_bytes=272556032 token_bytes=469844024 weight_movement_bytes=37748736 total_nvlink_bytes=507592760 weight_to_token_ratio=0.080343 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611497793e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229376 combine_payload_bytes=234618880 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234848256 ingress_bytes=253493248 link_bytes=253493248 token_bytes=469467136 weight_movement_bytes=18874368 total_nvlink_bytes=488341504 weight_to_token_ratio=0.040204 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428847576e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=52 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8170, 8126, 8192] dup_rows_per_dst=[38, 0, 0, 38, 0, 10, 0, 4] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8170, 8126, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=12 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4093, 0, 3, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8186, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 6, 6, 0, 0, 6, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8186, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62766 payload_rows_remote=54574 dup_rows=2770 dup_rows_remote=12 dedup_ratio=0.042267 router_tokens_per_expert=[3, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 4093, 0, 3, 0, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8192, 7406, 8186, 6214, 8192, 8192, 8192] dup_rows_per_dst=[6, 0, 0, 0, 0, 2758, 0, 6] combine_rows_per_src=[8192, 8192, 7406, 8186, 6214, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=230154240 dispatch_meta_bytes=229328 combine_payload_bytes=230154240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230383568 ingress_bytes=249028608 link_bytes=249028608 token_bytes=460537808 weight_movement_bytes=18874368 total_nvlink_bytes=479412176 weight_to_token_ratio=0.040983 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.386069592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229224 combine_payload_bytes=234512384 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234741608 ingress_bytes=272261120 link_bytes=272261120 token_bytes=469253992 weight_movement_bytes=37748736 total_nvlink_bytes=507002728 weight_to_token_ratio=0.080444 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608672091e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229352 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235036648 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843944 weight_movement_bytes=18874368 total_nvlink_bytes=488718312 weight_to_token_ratio=0.040172 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=223535104 dispatch_meta_bytes=218344 combine_payload_bytes=223535104 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=223753448 ingress_bytes=299032576 link_bytes=299032576 token_bytes=447288552 weight_movement_bytes=75497472 total_nvlink_bytes=522786024 weight_to_token_ratio=0.168789 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.865183010e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57258 dup_rows=86 dup_rows_remote=42 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 20, 4089, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0] payload_rows_per_dst=[8134, 8192, 8178, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 28, 0, 0, 0, 0, 44, 14] combine_rows_per_src=[8134, 8192, 8178, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64738 payload_rows_remote=56546 dup_rows=798 dup_rows_remote=798 dedup_ratio=0.012177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3697, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7394, 8192] dup_rows_per_dst=[0, 0, 0, 756, 0, 0, 0, 42] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7394, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21972163760 prefetch_bytes=811597824 reduce_bytes=830472192 weight_movement_bytes=1642070016 egress_bytes=10991606960 ingress_bytes=12622626816 link_bytes=12623314944 total_nvlink_bytes=23614233776 weight_to_token_ratio=0.074734 prefetch_share_of_ingress=0.064297 dedup_ratio=0.027198 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=44 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101128 gemm_inner_pad_upper_bound=31369 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.519799849e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229200 combine_payload_bytes=234528768 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234757968 ingress_bytes=291151872 link_bytes=291151872 token_bytes=469286736 weight_movement_bytes=56623104 total_nvlink_bytes=525909840 weight_to_token_ratio=0.120658 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.789673982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=231612416 dispatch_meta_bytes=229376 combine_payload_bytes=231612416 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=231841792 ingress_bytes=307109888 link_bytes=307109888 token_bytes=463454208 weight_movement_bytes=75497472 total_nvlink_bytes=538951680 weight_to_token_ratio=0.162902 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.942575839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57212 dup_rows=144 dup_rows_remote=112 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 4090, 0, 6, 0, 4090, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8154, 8156, 8180, 8192, 8160, 8190, 8180] dup_rows_per_dst=[0, 12, 12, 0, 0, 14, 74, 32] combine_rows_per_src=[8180, 8154, 8156, 8180, 8192, 8160, 8190, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57296 dup_rows=48 dup_rows_remote=44 dedup_ratio=0.000732 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8186, 8154] dup_rows_per_dst=[4, 0, 0, 6, 0, 38, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8186, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[4093, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8186, 8182, 8192] dup_rows_per_dst=[6, 0, 0, 4, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8186, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=12 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4093, 0, 3, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8186, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 6, 6, 0, 0, 6, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8186, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21757116544 prefetch_bytes=754974720 reduce_bytes=943718400 weight_movement_bytes=1698693120 egress_bytes=10884046976 ingress_bytes=12571762688 link_bytes=12571992064 total_nvlink_bytes=23455809664 weight_to_token_ratio=0.078075 prefetch_share_of_ingress=0.060053 dedup_ratio=0.033857 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=50 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101490 gemm_inner_pad_upper_bound=31115 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.509555045e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234340352 dispatch_meta_bytes=229296 combine_payload_bytes=234340352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234569648 ingress_bytes=253214720 link_bytes=253214720 token_bytes=468910000 weight_movement_bytes=18874368 total_nvlink_bytes=487784368 weight_to_token_ratio=0.040252 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426178858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229360 combine_payload_bytes=234684416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234913776 ingress_bytes=272433152 link_bytes=272433152 token_bytes=469598192 weight_movement_bytes=37748736 total_nvlink_bytes=507346928 weight_to_token_ratio=0.080385 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610320417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229352 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235036648 ingress_bytes=291430400 link_bytes=291430400 token_bytes=469843944 weight_movement_bytes=56623104 total_nvlink_bytes=526467048 weight_to_token_ratio=0.120515 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792342700e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63804 payload_rows_remote=55612 dup_rows=1732 dup_rows_remote=1732 dedup_ratio=0.026428 router_tokens_per_expert=[7, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 4089, 0, 7, 0, 0, 0, 0, 852, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[8192, 8192, 7500, 8178, 7166, 8192, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 1704, 0, 14] combine_rows_per_src=[8192, 8192, 7500, 8178, 7166, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=227786752 dispatch_meta_bytes=229376 combine_payload_bytes=227786752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228016128 ingress_bytes=246661120 link_bytes=246661120 token_bytes=455802880 weight_movement_bytes=18874368 total_nvlink_bytes=474677248 weight_to_token_ratio=0.041409 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.363385487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65264 payload_rows_remote=57072 dup_rows=272 dup_rows_remote=260 dedup_ratio=0.004150 router_tokens_per_expert=[4090, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4044, 5, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8154, 8192, 8192, 8004, 8192, 8180, 8158, 8192] dup_rows_per_dst=[10, 2, 0, 24, 0, 0, 224, 12] combine_rows_per_src=[8154, 8192, 8192, 8004, 8192, 8180, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64180 payload_rows_remote=55988 dup_rows=1356 dup_rows_remote=1356 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3661, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7322, 7706] dup_rows_per_dst=[0, 504, 0, 852, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7322, 7706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64330 payload_rows_remote=56140 dup_rows=1206 dup_rows_remote=1206 dedup_ratio=0.018402 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 3500, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3, 3, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8186, 8190, 8186, 8192, 8186, 7006, 8192] dup_rows_per_dst=[0, 0, 0, 1160, 0, 0, 0, 46] combine_rows_per_src=[8192, 8186, 8190, 8186, 8192, 8186, 7006, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57334 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[4093, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8186, 8182, 8192] dup_rows_per_dst=[6, 0, 0, 4, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8186, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=229326848 dispatch_meta_bytes=229376 combine_payload_bytes=229326848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229556224 ingress_bytes=248201216 link_bytes=248201216 token_bytes=458883072 weight_movement_bytes=18874368 total_nvlink_bytes=477757440 weight_to_token_ratio=0.041131 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.378141929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233766912 dispatch_meta_bytes=229328 combine_payload_bytes=233766912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233996240 ingress_bytes=252641280 link_bytes=252641280 token_bytes=467763152 weight_movement_bytes=18874368 total_nvlink_bytes=486637520 weight_to_token_ratio=0.040350 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420684438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=229949440 dispatch_meta_bytes=229384 combine_payload_bytes=229949440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230178824 ingress_bytes=267698176 link_bytes=267698176 token_bytes=460128264 weight_movement_bytes=37748736 total_nvlink_bytes=497877000 weight_to_token_ratio=0.082040 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.564952207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229400 combine_payload_bytes=234840064 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235069464 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909528 weight_movement_bytes=18874368 total_nvlink_bytes=488783896 weight_to_token_ratio=0.040166 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65314 payload_rows_remote=57122 dup_rows=222 dup_rows_remote=4 dedup_ratio=0.003387 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3985, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7970, 8192] dup_rows_per_dst=[0, 0, 0, 218, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7970, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57302 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 4088, 0, 7, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8178, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 14, 14, 0, 0, 14, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8178, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21280107752 prefetch_bytes=849346560 reduce_bytes=585105408 weight_movement_bytes=1434451968 egress_bytes=10645490920 ingress_bytes=12069068800 link_bytes=12069756928 total_nvlink_bytes=22714559720 weight_to_token_ratio=0.067408 prefetch_share_of_ingress=0.070374 dedup_ratio=0.050400 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=31 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101635 gemm_inner_pad_upper_bound=30480 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.409301504e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21813880464 prefetch_bytes=811597824 reduce_bytes=792723456 weight_movement_bytes=1604321280 egress_bytes=10912425616 ingress_bytes=12505776128 link_bytes=12506005504 total_nvlink_bytes=23418201744 weight_to_token_ratio=0.073546 prefetch_share_of_ingress=0.064898 dedup_ratio=0.030488 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=42 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101562 gemm_inner_pad_upper_bound=30861 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.496383154e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=233971712 dispatch_meta_bytes=228504 combine_payload_bytes=233971712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234200216 ingress_bytes=252846080 link_bytes=252846080 token_bytes=468171928 weight_movement_bytes=18874368 total_nvlink_bytes=487046296 weight_to_token_ratio=0.040315 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422646731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229376 combine_payload_bytes=234708992 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=234938368 ingress_bytes=347955200 link_bytes=347955200 token_bytes=469647360 weight_movement_bytes=113246208 total_nvlink_bytes=582893568 weight_to_token_ratio=0.241130 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.333935522e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59338 payload_rows_remote=51146 dup_rows=6198 dup_rows_remote=6150 dedup_ratio=0.094574 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 0, 3709, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 5, 6, 3, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 7714, 8192, 2766, 8192, 8182, 7908, 8192] dup_rows_per_dst=[0, 0, 8, 276, 0, 32, 5834, 48] combine_rows_per_src=[8192, 7714, 8192, 2766, 8192, 8182, 7908, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21988249840 prefetch_bytes=868220928 reduce_bytes=905969664 weight_movement_bytes=1774190592 egress_bytes=10999632112 ingress_bytes=12762808320 link_bytes=12762808320 total_nvlink_bytes=23762440432 weight_to_token_ratio=0.080688 prefetch_share_of_ingress=0.068027 dedup_ratio=0.026487 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=48 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100622 gemm_inner_pad_upper_bound=32385 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.547644784e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64112 payload_rows_remote=55924 dup_rows=1424 dup_rows_remote=1424 dedup_ratio=0.021729 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 3390, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 2, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8186, 8190, 8186, 8192, 8188, 6786, 8192] dup_rows_per_dst=[0, 0, 0, 1386, 0, 0, 0, 38] combine_rows_per_src=[8192, 8186, 8190, 8186, 8192, 8188, 6786, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=209494016 dispatch_meta_bytes=229184 combine_payload_bytes=209494016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209723200 ingress_bytes=228368384 link_bytes=228368384 token_bytes=419217216 weight_movement_bytes=18874368 total_nvlink_bytes=438091584 weight_to_token_ratio=0.045023 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.188113491e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=229064704 dispatch_meta_bytes=229392 combine_payload_bytes=229064704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229294096 ingress_bytes=266813440 link_bytes=266813440 token_bytes=458358800 weight_movement_bytes=37748736 total_nvlink_bytes=496107536 weight_to_token_ratio=0.082356 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.556475102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57330 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[4089, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4085, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8178, 8132, 8192] dup_rows_per_dst=[12, 2, 0, 46, 0, 0, 0, 14] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8178, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21018878152 prefetch_bytes=830472192 reduce_bytes=679477248 weight_movement_bytes=1509949440 egress_bytes=10514849992 ingress_bytes=12013977600 link_bytes=12014206976 total_nvlink_bytes=22528827592 weight_to_token_ratio=0.071838 prefetch_share_of_ingress=0.069125 dedup_ratio=0.060916 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=36 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101345 gemm_inner_pad_upper_bound=31369 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.398212914e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21798701184 prefetch_bytes=754974720 reduce_bytes=1151336448 weight_movement_bytes=1906311168 egress_bytes=10904840320 ingress_bytes=12800172032 link_bytes=12800401408 total_nvlink_bytes=23705012352 weight_to_token_ratio=0.087451 prefetch_share_of_ingress=0.058982 dedup_ratio=0.031855 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=61 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=31877 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.555148919e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229616 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053296 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876976 weight_movement_bytes=18874368 total_nvlink_bytes=488751344 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64440 payload_rows_remote=57300 dup_rows=1096 dup_rows_remote=1096 dedup_ratio=0.016724 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4087, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 0, 3563, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 134, 0, 7, 6, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[8192, 8178, 8188, 8178, 8192, 8180, 7140, 8192] dup_rows_per_dst=[0, 0, 0, 788, 0, 0, 0, 308] combine_rows_per_src=[8192, 8178, 8188, 8178, 8192, 8180, 7140, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=233584 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234934384 ingress_bytes=272449536 link_bytes=272449536 token_bytes=469635184 weight_movement_bytes=37748736 total_nvlink_bytes=507383920 weight_to_token_ratio=0.080379 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610477400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21670864840 prefetch_bytes=887095296 reduce_bytes=679477248 weight_movement_bytes=1566572544 egress_bytes=10840877000 ingress_bytes=12396560384 link_bytes=12397019136 total_nvlink_bytes=23237437384 weight_to_token_ratio=0.072289 prefetch_share_of_ingress=0.071560 dedup_ratio=0.035922 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=36 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100622 gemm_inner_pad_upper_bound=33020 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.474627868e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none + [2026-08-29 23:37:46] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 988.9 | throughput per GPU (TFLOP/s/GPU): 90.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.044850E+00 | loss scale: 1.0 | grad norm: 3.410 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45226 payload_rows_remote=40070 dup_rows=20310 dup_rows_remote=18520 dedup_ratio=0.309906 router_tokens_per_expert=[297, 0, 0, 0, 293, 260, 56, 2, 1, 1, 150, 463, 720, 402, 152, 377, 5, 53, 0, 452, 227, 273, 0, 0, 421, 121, 96, 125, 329, 3, 448, 3, 317, 410, 5, 56, 4, 28, 376, 43, 207, 833, 647, 332, 453, 11, 182, 838, 430, 270, 428, 0, 810, 605, 18, 13, 1082, 120, 243, 806, 97, 298, 29, 70, 18, 232, 0, 8, 159, 0, 580, 542, 0, 808, 468, 1429, 25, 79, 17, 1524, 238, 8, 60, 179, 9, 196, 0, 0, 803, 373, 86, 190, 115, 534, 106, 51, 0, 40, 223, 34, 0, 466, 970, 56, 85, 10, 520, 96, 614, 311, 458, 0, 398, 285, 248, 36, 262, 2, 1211, 73, 677, 131, 426, 5, 163, 0, 340, 0] payload_rows_per_dst=[5816, 6032, 5538, 5978, 5830, 5156, 5496, 5380] dup_rows_per_dst=[2696, 2128, 3086, 3034, 2900, 1790, 2592, 2084] combine_rows_per_src=[5816, 6032, 5538, 5978, 5830, 5156, 5496, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45156 payload_rows_remote=39436 dup_rows=20380 dup_rows_remote=17578 dedup_ratio=0.310974 router_tokens_per_expert=[249, 0, 0, 0, 384, 315, 57, 0, 1, 0, 202, 473, 545, 483, 251, 261, 8, 81, 2, 399, 400, 267, 0, 0, 409, 257, 81, 62, 414, 4, 518, 0, 345, 342, 3, 69, 1, 34, 258, 67, 106, 689, 500, 722, 267, 15, 135, 506, 321, 308, 410, 0, 520, 620, 10, 18, 849, 47, 214, 735, 102, 230, 6, 123, 313, 295, 1, 7, 211, 0, 522, 467, 0, 913, 429, 978, 64, 58, 3, 1546, 168, 21, 47, 262, 11, 228, 3, 0, 1355, 313, 50, 281, 221, 548, 116, 42, 0, 69, 225, 74, 0, 392, 1138, 34, 164, 28, 658, 56, 373, 425, 223, 0, 427, 366, 221, 39, 279, 0, 935, 88, 842, 329, 572, 22, 335, 6, 279, 1] payload_rows_per_dst=[5496, 6278, 5430, 5240, 5720, 6228, 5374, 5390] dup_rows_per_dst=[2644, 2618, 2688, 2088, 2802, 2974, 2344, 2222] combine_rows_per_src=[5496, 6278, 5430, 5240, 5720, 6228, 5374, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44728 payload_rows_remote=39022 dup_rows=20808 dup_rows_remote=17888 dedup_ratio=0.317505 router_tokens_per_expert=[198, 0, 0, 1, 308, 249, 77, 1, 4, 3, 168, 600, 565, 387, 176, 355, 15, 93, 2, 477, 333, 316, 0, 1, 358, 226, 88, 159, 371, 4, 463, 3, 376, 365, 6, 123, 3, 30, 344, 35, 227, 591, 603, 841, 294, 33, 209, 601, 539, 303, 432, 1, 430, 569, 27, 30, 623, 98, 285, 598, 148, 334, 14, 156, 113, 294, 0, 16, 189, 0, 654, 421, 3, 930, 390, 799, 49, 59, 20, 1096, 328, 31, 75, 368, 35, 239, 7, 0, 1078, 467, 95, 224, 258, 553, 124, 111, 0, 88, 150, 96, 4, 431, 1135, 64, 133, 21, 516, 156, 469, 371, 225, 0, 518, 233, 260, 50, 285, 3, 823, 114, 767, 354, 347, 24, 507, 2, 327, 0] payload_rows_per_dst=[5706, 5860, 5890, 4996, 5358, 6348, 5194, 5376] dup_rows_per_dst=[2920, 2150, 3636, 1736, 2352, 3284, 2524, 2206] combine_rows_per_src=[5706, 5860, 5890, 4996, 5358, 6348, 5194, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44954 payload_rows_remote=38672 dup_rows=20582 dup_rows_remote=18070 dedup_ratio=0.314056 router_tokens_per_expert=[287, 0, 0, 0, 381, 312, 50, 2, 0, 10, 135, 550, 506, 558, 244, 261, 2, 121, 0, 402, 569, 212, 0, 0, 401, 280, 98, 69, 312, 6, 449, 8, 510, 458, 3, 40, 4, 41, 355, 52, 278, 546, 503, 789, 330, 27, 134, 616, 434, 301, 445, 0, 406, 451, 8, 15, 918, 32, 276, 818, 105, 275, 10, 155, 307, 289, 0, 6, 208, 0, 620, 388, 0, 989, 369, 777, 35, 35, 16, 1468, 196, 23, 63, 361, 15, 314, 8, 0, 1062, 496, 46, 279, 195, 259, 94, 93, 0, 91, 144, 46, 0, 511, 1261, 57, 93, 14, 499, 47, 346, 338, 280, 1, 567, 254, 231, 67, 285, 1, 769, 95, 833, 298, 390, 29, 408, 0, 241, 1] payload_rows_per_dst=[5834, 6282, 5976, 4732, 5740, 5880, 5166, 5344] dup_rows_per_dst=[3586, 2512, 3396, 1738, 2338, 2666, 2290, 2056] combine_rows_per_src=[5834, 6282, 5976, 4732, 5740, 5880, 5166, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38162 dup_rows=21044 dup_rows_remote=17398 dedup_ratio=0.321106 router_tokens_per_expert=[229, 0, 0, 3, 428, 291, 42, 2, 0, 2, 189, 441, 545, 456, 202, 247, 7, 107, 7, 433, 287, 224, 0, 0, 441, 167, 64, 80, 330, 2, 644, 3, 392, 423, 3, 60, 0, 17, 405, 32, 160, 581, 658, 680, 287, 18, 104, 631, 480, 216, 493, 0, 637, 551, 11, 11, 953, 66, 266, 871, 185, 307, 14, 141, 224, 281, 0, 7, 204, 1, 635, 416, 1, 1005, 434, 842, 21, 47, 10, 1370, 269, 16, 66, 248, 28, 316, 2, 0, 744, 411, 65, 262, 194, 520, 101, 70, 0, 66, 176, 42, 0, 477, 1075, 62, 125, 23, 491, 44, 520, 401, 268, 0, 492, 259, 218, 90, 184, 3, 865, 98, 903, 274, 561, 29, 318, 0, 368, 0] payload_rows_per_dst=[5536, 5644, 4220, 6128, 5908, 4876, 5850, 6330] dup_rows_per_dst=[2524, 2100, 1508, 3532, 2936, 1748, 3050, 3646] combine_rows_per_src=[5536, 5644, 4220, 6128, 5908, 4876, 5850, 6330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45304 payload_rows_remote=39566 dup_rows=20232 dup_rows_remote=17010 dedup_ratio=0.308716 router_tokens_per_expert=[240, 0, 0, 0, 367, 313, 42, 1, 1, 2, 169, 365, 481, 421, 230, 229, 2, 78, 1, 403, 342, 345, 1, 0, 389, 295, 71, 89, 391, 2, 343, 0, 373, 494, 6, 70, 5, 16, 376, 26, 143, 586, 586, 633, 393, 19, 190, 564, 529, 311, 472, 1, 602, 581, 10, 22, 825, 97, 247, 831, 57, 248, 15, 108, 125, 240, 0, 2, 210, 0, 541, 386, 0, 893, 463, 966, 40, 34, 15, 1367, 207, 10, 60, 246, 16, 425, 3, 0, 1192, 429, 50, 229, 263, 421, 111, 77, 0, 60, 189, 57, 0, 422, 1108, 47, 103, 12, 611, 80, 412, 307, 370, 0, 481, 297, 328, 58, 210, 1, 943, 106, 907, 274, 629, 16, 369, 1, 300, 0] payload_rows_per_dst=[5286, 5994, 5738, 5616, 5514, 6304, 5196, 5656] dup_rows_per_dst=[2086, 2244, 3222, 2646, 2316, 3060, 2360, 2298] combine_rows_per_src=[5286, 5994, 5738, 5616, 5514, 6304, 5196, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45538 payload_rows_remote=39666 dup_rows=19998 dup_rows_remote=17426 dedup_ratio=0.305145 router_tokens_per_expert=[183, 0, 0, 0, 236, 274, 53, 1, 2, 4, 213, 363, 538, 324, 189, 238, 4, 80, 0, 465, 279, 243, 0, 0, 288, 213, 71, 88, 456, 2, 345, 6, 319, 516, 1, 71, 1, 40, 346, 19, 191, 833, 659, 552, 228, 12, 132, 686, 502, 344, 405, 0, 728, 567, 9, 12, 1018, 47, 191, 878, 111, 347, 7, 74, 71, 260, 0, 5, 271, 0, 693, 464, 0, 1045, 495, 859, 21, 38, 10, 1298, 166, 9, 49, 196, 7, 244, 2, 0, 1136, 405, 71, 289, 238, 563, 129, 90, 0, 100, 190, 35, 2, 382, 1243, 44, 128, 5, 575, 67, 439, 437, 314, 0, 455, 472, 238, 34, 335, 1, 963, 80, 806, 241, 558, 1, 264, 0, 231, 0] payload_rows_per_dst=[5192, 5784, 5886, 5872, 5626, 6214, 5470, 5494] dup_rows_per_dst=[2080, 1892, 3326, 2572, 2838, 2900, 2452, 1938] combine_rows_per_src=[5192, 5784, 5886, 5872, 5626, 6214, 5470, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164126720 dispatch_meta_bytes=234360 combine_payload_bytes=164126720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=164361080 ingress_bytes=183001088 link_bytes=183001088 token_bytes=328487800 weight_movement_bytes=18874368 total_nvlink_bytes=347362168 weight_to_token_ratio=0.057458 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=8448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.753426383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=227640 combine_payload_bytes=159834112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=160061752 ingress_bytes=197582848 link_bytes=197582848 token_bytes=319895864 weight_movement_bytes=37748736 total_nvlink_bytes=357644600 weight_to_token_ratio=0.118003 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=7040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.893141633e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161529856 dispatch_meta_bytes=228056 combine_payload_bytes=161529856 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=161757912 ingress_bytes=199278592 link_bytes=199278592 token_bytes=323287768 weight_movement_bytes=37748736 total_nvlink_bytes=361036504 weight_to_token_ratio=0.116765 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.909389417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158400512 dispatch_meta_bytes=226968 combine_payload_bytes=158400512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158627480 ingress_bytes=177274880 link_bytes=177274880 token_bytes=317027992 weight_movement_bytes=18874368 total_nvlink_bytes=335902360 weight_to_token_ratio=0.059535 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=10880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.698560675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=222240 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156533792 ingress_bytes=194060288 link_bytes=194060288 token_bytes=312845344 weight_movement_bytes=37748736 total_nvlink_bytes=350594080 weight_to_token_ratio=0.120663 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859390196e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162062336 dispatch_meta_bytes=226304 combine_payload_bytes=162062336 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=162288640 ingress_bytes=237559808 link_bytes=237559808 token_bytes=324350976 weight_movement_bytes=75497472 total_nvlink_bytes=399848448 weight_to_token_ratio=0.232765 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=4736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276181194e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162471936 dispatch_meta_bytes=228368 combine_payload_bytes=162471936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=162700304 ingress_bytes=219095040 link_bytes=219095040 token_bytes=325172240 weight_movement_bytes=56623104 total_nvlink_bytes=381795344 weight_to_token_ratio=0.174133 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=5376 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.099260872e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56236 payload_rows_remote=48080 dup_rows=9300 dup_rows_remote=7930 dedup_ratio=0.141907 router_tokens_per_expert=[0, 2889, 0, 0, 2751, 0, 0, 1392, 1, 0, 0, 0, 3957, 2, 0, 4088, 4091, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2121, 8, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 703, 0, 19, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 667, 22, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0] payload_rows_per_dst=[5506, 2786, 8042, 7186, 8186, 8156, 8188, 8186] dup_rows_per_dst=[2, 2, 2146, 2892, 1490, 1370, 342, 1056] combine_rows_per_src=[5506, 2786, 8042, 7186, 8186, 8156, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57046 payload_rows_remote=48982 dup_rows=8490 dup_rows_remote=6484 dedup_ratio=0.129547 router_tokens_per_expert=[0, 3863, 0, 0, 1155, 0, 0, 3279, 58, 0, 0, 0, 3879, 0, 0, 4096, 3990, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 204, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 279, 0, 1, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1982, 0] payload_rows_per_dst=[2404, 6410, 7794, 7872, 8064, 8120, 8192, 8190] dup_rows_per_dst=[170, 0, 700, 848, 2006, 572, 304, 3890] combine_rows_per_src=[2404, 6410, 7794, 7872, 8064, 8120, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55344 payload_rows_remote=47162 dup_rows=10192 dup_rows_remote=8242 dedup_ratio=0.155518 router_tokens_per_expert=[0, 3962, 0, 0, 240, 0, 1, 2891, 163, 0, 1, 0, 4046, 0, 1, 4093, 4016, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 157, 10, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 0, 935, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 527, 0, 4, 4068, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 45, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 0, 975, 0] payload_rows_per_dst=[800, 5786, 8122, 7930, 8176, 8158, 8190, 8182] dup_rows_per_dst=[12, 0, 698, 416, 5978, 1046, 92, 1950] combine_rows_per_src=[800, 5786, 8122, 7930, 8176, 8158, 8190, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=34438 payload_rows_remote=26246 dup_rows=31098 dup_rows_remote=8900 dedup_ratio=0.474518 router_tokens_per_expert=[0, 3792, 0, 0, 533, 0, 0, 3169, 184, 0, 0, 0, 3998, 0, 0, 4094, 4084, 0, 0, 0, 0, 2, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 91, 0, 0, 0, 7, 55, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 0, 734, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 153, 519, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 61, 0, 0, 2, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0] payload_rows_per_dst=[8192, 8176, 1036, 3320, 3934, 1348, 6570, 1862] dup_rows_per_dst=[22198, 8160, 0, 330, 304, 0, 106, 0] combine_rows_per_src=[8192, 8176, 1036, 3320, 3934, 1348, 6570, 1862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52328 payload_rows_remote=44160 dup_rows=13208 dup_rows_remote=5176 dedup_ratio=0.201538 router_tokens_per_expert=[0, 3886, 0, 0, 108, 0, 0, 2644, 70, 0, 0, 0, 4037, 1, 0, 4051, 4006, 0, 0, 1, 0, 9, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 81, 0, 0, 0, 1, 51, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 1522, 0, 874, 0, 0, 12, 0, 0, 74, 18, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1377, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 1, 4, 0, 8, 0, 0, 154, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 1, 8, 0, 22, 0, 0, 1, 1, 0, 1160, 0] payload_rows_per_dst=[6174, 8168, 7698, 7796, 4454, 7540, 8136, 2362] dup_rows_per_dst=[1308, 8032, 658, 354, 552, 1962, 316, 26] combine_rows_per_src=[6174, 8168, 7698, 7796, 4454, 7540, 8136, 2362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58148 payload_rows_remote=50020 dup_rows=7388 dup_rows_remote=6494 dedup_ratio=0.112732 router_tokens_per_expert=[0, 4001, 0, 0, 335, 0, 0, 3682, 42, 0, 0, 0, 4043, 0, 0, 4096, 4085, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 29, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 0, 556, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 17, 733, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0] payload_rows_per_dst=[7458, 8176, 8128, 8034, 2716, 8180, 8192, 7264] dup_rows_per_dst=[660, 1066, 894, 1386, 92, 1514, 274, 1502] combine_rows_per_src=[7458, 8176, 8128, 8034, 2716, 8180, 8192, 7264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56324 payload_rows_remote=48512 dup_rows=9212 dup_rows_remote=8294 dedup_ratio=0.140564 router_tokens_per_expert=[0, 3857, 0, 0, 938, 0, 0, 3306, 37, 0, 0, 0, 3966, 0, 0, 4092, 4042, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 27, 0, 0, 0, 0, 56, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 352, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 6, 10, 311, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 116, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1387, 0] payload_rows_per_dst=[7130, 1892, 8094, 7812, 6912, 8106, 8186, 8192] dup_rows_per_dst=[1432, 0, 1498, 918, 1724, 642, 234, 2764] combine_rows_per_src=[7130, 1892, 8094, 7812, 6912, 8106, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=196935680 dispatch_meta_bytes=224040 combine_payload_bytes=196935680 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=197159720 ingress_bytes=215810048 link_bytes=215810048 token_bytes=394095400 weight_movement_bytes=18874368 total_nvlink_bytes=412969768 weight_to_token_ratio=0.047893 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.067785695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=200630272 dispatch_meta_bytes=221864 combine_payload_bytes=200630272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200852136 ingress_bytes=219504640 link_bytes=219504640 token_bytes=401482408 weight_movement_bytes=18874368 total_nvlink_bytes=420356776 weight_to_token_ratio=0.047012 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.083333 gemm_active_groups=12 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.103185457e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=193175552 dispatch_meta_bytes=221616 combine_payload_bytes=193175552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193397168 ingress_bytes=212049920 link_bytes=212049920 token_bytes=386572720 weight_movement_bytes=18874368 total_nvlink_bytes=405447088 weight_to_token_ratio=0.048825 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.031757998e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=107503616 dispatch_meta_bytes=140584 combine_payload_bytes=107503616 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=107644200 ingress_bytes=201875456 link_bytes=201875456 token_bytes=215147816 weight_movement_bytes=94371840 total_nvlink_bytes=309519656 weight_to_token_ratio=0.438637 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.086806 gemm_active_groups=12 gemm_peak_group_rows=12928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.934271290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=180879360 dispatch_meta_bytes=197344 combine_payload_bytes=180879360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=181076704 ingress_bytes=237502464 link_bytes=237502464 token_bytes=361956064 weight_movement_bytes=56623104 total_nvlink_bytes=418579168 weight_to_token_ratio=0.156436 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=13184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.275631752e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=204881920 dispatch_meta_bytes=226056 combine_payload_bytes=204881920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205107976 ingress_bytes=223756288 link_bytes=223756288 token_bytes=409989896 weight_movement_bytes=18874368 total_nvlink_bytes=428864264 weight_to_token_ratio=0.046036 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.143922657e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=198705152 dispatch_meta_bytes=227224 combine_payload_bytes=198705152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198932376 ingress_bytes=217579520 link_bytes=217579520 token_bytes=397637528 weight_movement_bytes=18874368 total_nvlink_bytes=416511896 weight_to_token_ratio=0.047466 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.084739905e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44960 payload_rows_remote=38546 dup_rows=20576 dup_rows_remote=16812 dedup_ratio=0.313965 router_tokens_per_expert=[252, 0, 0, 1, 313, 339, 33, 1, 0, 5, 223, 674, 608, 479, 179, 252, 6, 99, 1, 472, 222, 238, 0, 0, 381, 205, 49, 51, 444, 2, 407, 4, 302, 435, 4, 69, 3, 35, 345, 53, 247, 805, 568, 609, 271, 11, 137, 518, 433, 234, 424, 0, 644, 526, 16, 21, 708, 57, 312, 840, 124, 270, 21, 87, 93, 297, 1, 11, 224, 0, 511, 428, 0, 746, 375, 832, 54, 100, 7, 1428, 144, 14, 98, 140, 15, 194, 0, 0, 963, 385, 54, 202, 178, 419, 125, 123, 0, 38, 221, 72, 0, 472, 1062, 50, 117, 8, 965, 161, 463, 503, 348, 0, 412, 473, 198, 68, 445, 0, 1049, 108, 837, 246, 665, 4, 233, 0, 320, 0] payload_rows_per_dst=[5740, 5926, 4522, 6120, 5156, 4750, 6414, 6332] dup_rows_per_dst=[2394, 2092, 1474, 3508, 2202, 1358, 3764, 3784] combine_rows_per_src=[5740, 5926, 4522, 6120, 5156, 4750, 6414, 6332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56664 payload_rows_remote=48538 dup_rows=8872 dup_rows_remote=8840 dedup_ratio=0.135376 router_tokens_per_expert=[0, 0, 0, 1948, 0, 0, 4096, 0, 0, 0, 3411, 2, 0, 0, 0, 1953, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 3730, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 5, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 917, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6378, 7704, 8192, 4126, 8126, 7232, 6996, 7910] dup_rows_per_dst=[1428, 0, 0, 0, 32, 1596, 1350, 4466] combine_rows_per_src=[6378, 7704, 8192, 4126, 8126, 7232, 6996, 7910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55332 payload_rows_remote=47244 dup_rows=10204 dup_rows_remote=6488 dedup_ratio=0.155701 router_tokens_per_expert=[0, 0, 0, 737, 0, 0, 4091, 0, 0, 0, 3988, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 4, 3281, 0, 0, 1, 0, 2735, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 1, 0, 0, 0, 0, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 26, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7924, 850, 8190, 8028, 8174, 8088, 8176, 5902] dup_rows_per_dst=[1360, 10, 16, 4584, 0, 3716, 56, 462] combine_rows_per_src=[7924, 850, 8190, 8028, 8174, 8088, 8176, 5902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54400 payload_rows_remote=46818 dup_rows=11136 dup_rows_remote=8076 dedup_ratio=0.169922 router_tokens_per_expert=[0, 0, 0, 512, 0, 1, 4089, 0, 0, 0, 3477, 0, 0, 0, 0, 3044, 0, 0, 0, 0, 0, 0, 0, 3357, 1, 0, 0, 0, 3337, 0, 0, 7, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 2896, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 14, 0, 0, 0, 0, 896, 0, 0, 0, 0, 1, 0, 0, 0, 3742, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[1026, 5812, 8182, 8088, 8178, 7844, 7688, 7582] dup_rows_per_dst=[0, 12, 286, 4424, 2, 2370, 982, 3060] combine_rows_per_src=[1026, 5812, 8182, 8088, 8178, 7844, 7688, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50376 payload_rows_remote=42304 dup_rows=15160 dup_rows_remote=7950 dedup_ratio=0.231323 router_tokens_per_expert=[0, 0, 0, 497, 0, 0, 4096, 0, 0, 0, 3339, 0, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 16, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 0, 15, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 1092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2823, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8072, 8190, 8192, 5158, 8186, 3056, 3876, 5646] dup_rows_per_dst=[7210, 7176, 4, 0, 326, 38, 404, 2] combine_rows_per_src=[8072, 8190, 8192, 5158, 8186, 3056, 3876, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55938 payload_rows_remote=47746 dup_rows=9598 dup_rows_remote=9598 dedup_ratio=0.146454 router_tokens_per_expert=[0, 0, 0, 439, 0, 0, 4096, 0, 0, 0, 3649, 0, 0, 0, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 3, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 11, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7384, 7374, 8192, 7076, 8158, 4504, 8018, 5232] dup_rows_per_dst=[4032, 2942, 0, 0, 0, 710, 884, 1030] combine_rows_per_src=[7384, 7374, 8192, 7076, 8158, 4504, 8018, 5232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54070 payload_rows_remote=45972 dup_rows=11466 dup_rows_remote=5902 dedup_ratio=0.174957 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 3956, 0, 0, 0, 3266, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 74, 3186, 0, 0, 32, 0, 3538, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 1, 0, 1, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 1210, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[7484, 8098, 7962, 7028, 8000, 4244, 8160, 3094] dup_rows_per_dst=[3332, 5564, 0, 0, 6, 440, 2122, 2] combine_rows_per_src=[7484, 8098, 7962, 7028, 8000, 4244, 8160, 3094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58454 payload_rows_remote=52992 dup_rows=7082 dup_rows_remote=7082 dedup_ratio=0.108063 router_tokens_per_expert=[0, 0, 0, 736, 0, 0, 4094, 0, 0, 0, 3563, 0, 0, 0, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 8, 0, 3732, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 8, 0, 0, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 448, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6646, 6678, 8188, 5462, 8180, 7600, 7752, 7948] dup_rows_per_dst=[784, 16, 0, 0, 10, 2056, 626, 3590] combine_rows_per_src=[6646, 6678, 8188, 5462, 8180, 7600, 7752, 7948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=198811648 dispatch_meta_bytes=229512 combine_payload_bytes=198811648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199041160 ingress_bytes=217686016 link_bytes=217686016 token_bytes=397852808 weight_movement_bytes=18874368 total_nvlink_bytes=416727176 weight_to_token_ratio=0.047441 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.085760297e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193511424 dispatch_meta_bytes=214928 combine_payload_bytes=193511424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193726352 ingress_bytes=212385792 link_bytes=212385792 token_bytes=387237776 weight_movement_bytes=18874368 total_nvlink_bytes=406112144 weight_to_token_ratio=0.048741 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034976159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=191766528 dispatch_meta_bytes=219576 combine_payload_bytes=191766528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191986104 ingress_bytes=210640896 link_bytes=210640896 token_bytes=383752632 weight_movement_bytes=18874368 total_nvlink_bytes=402627000 weight_to_token_ratio=0.049184 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.018257424e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=173277184 dispatch_meta_bytes=201016 combine_payload_bytes=173277184 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=173478200 ingress_bytes=267649024 link_bytes=267649024 token_bytes=346755384 weight_movement_bytes=94371840 total_nvlink_bytes=441127224 weight_to_token_ratio=0.272157 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.564481256e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=195567616 dispatch_meta_bytes=229376 combine_payload_bytes=195567616 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=195796992 ingress_bytes=233316352 link_bytes=233316352 token_bytes=391364608 weight_movement_bytes=37748736 total_nvlink_bytes=429113344 weight_to_token_ratio=0.096454 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235522486e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=188301312 dispatch_meta_bytes=207496 combine_payload_bytes=188301312 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=188508808 ingress_bytes=244924416 link_bytes=244924416 token_bytes=376810120 weight_movement_bytes=56623104 total_nvlink_bytes=433433224 weight_to_token_ratio=0.150270 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.346745244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=217055232 dispatch_meta_bytes=240296 combine_payload_bytes=217055232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217295528 ingress_bytes=235929600 link_bytes=235929600 token_bytes=434350760 weight_movement_bytes=18874368 total_nvlink_bytes=453225128 weight_to_token_ratio=0.043454 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260561343e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=221432 combine_payload_bytes=157884416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158105848 ingress_bytes=176758784 link_bytes=176758784 token_bytes=315990264 weight_movement_bytes=18874368 total_nvlink_bytes=334864632 weight_to_token_ratio=0.059731 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=9088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693615698e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59104 payload_rows_remote=50916 dup_rows=6432 dup_rows_remote=6432 dedup_ratio=0.098145 router_tokens_per_expert=[0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 56, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 142, 9, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8166, 8192, 1808, 8190, 8188, 8176, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 112, 0, 0, 2162, 4158] combine_rows_per_src=[8166, 8192, 1808, 8190, 8188, 8176, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62938 payload_rows_remote=54752 dup_rows=2598 dup_rows_remote=2598 dedup_ratio=0.039642 router_tokens_per_expert=[0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 141, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7394, 8192, 6446, 8168, 8168, 8186, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 282, 0, 0, 976, 1334] combine_rows_per_src=[7394, 8192, 6446, 8168, 8168, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59196 payload_rows_remote=51004 dup_rows=6340 dup_rows_remote=2194 dedup_ratio=0.096741 router_tokens_per_expert=[0, 4006, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 1, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 33, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 1, 284, 2, 0, 0, 1, 2070, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2] payload_rows_per_dst=[8016, 8108, 3080, 7252, 8182, 8176, 8190, 8192] dup_rows_per_dst=[2, 6, 2, 64, 86, 430, 1604, 4146] combine_rows_per_src=[8016, 8108, 3080, 7252, 8182, 8176, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61324 payload_rows_remote=53134 dup_rows=4212 dup_rows_remote=588 dedup_ratio=0.064270 router_tokens_per_expert=[0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 195, 5, 0, 0, 0, 2049, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0] payload_rows_per_dst=[8052, 8192, 8190, 8192, 8188, 8190, 8192, 4128] dup_rows_per_dst=[0, 0, 3624, 66, 0, 0, 520, 2] combine_rows_per_src=[8052, 8192, 8190, 8192, 8188, 8190, 8192, 4128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55292 payload_rows_remote=47282 dup_rows=10244 dup_rows_remote=7860 dedup_ratio=0.156311 router_tokens_per_expert=[0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 284, 14, 0, 0, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 1] payload_rows_per_dst=[8010, 8192, 8190, 8192, 8192, 8082, 1092, 5342] dup_rows_per_dst=[2384, 0, 7842, 12, 4, 0, 0, 2] combine_rows_per_src=[8010, 8192, 8190, 8192, 8192, 8082, 1092, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60544 payload_rows_remote=52352 dup_rows=4992 dup_rows_remote=4920 dedup_ratio=0.076172 router_tokens_per_expert=[0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 4096, 0, 773, 38, 0, 0, 0, 2279, 0, 0, 0, 0, 0, 0, 0, 4, 56, 0, 1] payload_rows_per_dst=[7844, 8192, 8192, 8192, 8142, 8002, 7318, 4662] dup_rows_per_dst=[0, 72, 3384, 136, 0, 14, 1292, 94] combine_rows_per_src=[7844, 8192, 8192, 8192, 8142, 8002, 7318, 4662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61334 payload_rows_remote=53142 dup_rows=4202 dup_rows_remote=4012 dedup_ratio=0.064117 router_tokens_per_expert=[0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 95, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 97, 9, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[7686, 8192, 7400, 8192, 8192, 8140, 8192, 5340] dup_rows_per_dst=[0, 0, 2704, 190, 4, 0, 448, 856] combine_rows_per_src=[7686, 8192, 7400, 8192, 8192, 8140, 8192, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=208551936 dispatch_meta_bytes=229392 combine_payload_bytes=208551936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208781328 ingress_bytes=227426304 link_bytes=227426304 token_bytes=417333264 weight_movement_bytes=18874368 total_nvlink_bytes=436207632 weight_to_token_ratio=0.045226 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.179086944e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=208912384 dispatch_meta_bytes=212792 combine_payload_bytes=208912384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209125176 ingress_bytes=227786752 link_bytes=227786752 token_bytes=418037560 weight_movement_bytes=18874368 total_nvlink_bytes=436911928 weight_to_token_ratio=0.045150 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182540580e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=224264192 dispatch_meta_bytes=229400 combine_payload_bytes=224264192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=224493592 ingress_bytes=262012928 link_bytes=262012928 token_bytes=448757784 weight_movement_bytes=37748736 total_nvlink_bytes=486506520 weight_to_token_ratio=0.084118 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.510478958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57132 payload_rows_remote=48970 dup_rows=8404 dup_rows_remote=8352 dedup_ratio=0.128235 router_tokens_per_expert=[0, 3667, 0, 0, 1020, 0, 0, 3435, 25, 0, 0, 0, 4025, 0, 0, 4076, 3982, 0, 0, 0, 0, 6, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 6, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 348, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 358, 0, 1, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 965, 0] payload_rows_per_dst=[2086, 6876, 8122, 7462, 8122, 8118, 8162, 8184] dup_rows_per_dst=[4, 6, 1906, 980, 2826, 704, 52, 1926] combine_rows_per_src=[2086, 6876, 8122, 7462, 8122, 8118, 8162, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=217636864 dispatch_meta_bytes=214888 combine_payload_bytes=217636864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217851752 ingress_bytes=255385600 link_bytes=255385600 token_bytes=435488616 weight_movement_bytes=37748736 total_nvlink_bytes=473237352 weight_to_token_ratio=0.086681 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=15488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.446979162e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=193667072 dispatch_meta_bytes=220568 combine_payload_bytes=193667072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193887640 ingress_bytes=212541440 link_bytes=212541440 token_bytes=387554712 weight_movement_bytes=18874368 total_nvlink_bytes=406429080 weight_to_token_ratio=0.048701 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036467501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=214433792 dispatch_meta_bytes=229088 combine_payload_bytes=214433792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214662880 ingress_bytes=233308160 link_bytes=233308160 token_bytes=429096672 weight_movement_bytes=18874368 total_nvlink_bytes=447971040 weight_to_token_ratio=0.043986 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235443994e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=217669632 dispatch_meta_bytes=228616 combine_payload_bytes=217669632 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=217898248 ingress_bytes=274292736 link_bytes=274292736 token_bytes=435567880 weight_movement_bytes=56623104 total_nvlink_bytes=492190984 weight_to_token_ratio=0.129998 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.628138036e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=200581120 dispatch_meta_bytes=229288 combine_payload_bytes=200581120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200810408 ingress_bytes=219455488 link_bytes=219455488 token_bytes=401391528 weight_movement_bytes=18874368 total_nvlink_bytes=420265896 weight_to_token_ratio=0.047022 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102714507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64498 payload_rows_remote=56306 dup_rows=1038 dup_rows_remote=1038 dedup_ratio=0.015839 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 3669, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0] payload_rows_per_dst=[8192, 7970, 7690, 7880, 8192, 8190, 8192, 8192] dup_rows_per_dst=[4, 760, 2, 0, 0, 0, 0, 272] combine_rows_per_src=[8192, 7970, 7690, 7880, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58164 payload_rows_remote=49972 dup_rows=7372 dup_rows_remote=7308 dedup_ratio=0.112488 router_tokens_per_expert=[4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3826, 0, 0, 4077, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0] payload_rows_per_dst=[8174, 1124, 7950, 8156, 8188, 8192, 8188, 8192] dup_rows_per_dst=[0, 48, 6, 7232, 10, 0, 12, 64] combine_rows_per_src=[8174, 1124, 7950, 8156, 8188, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63918 payload_rows_remote=55726 dup_rows=1618 dup_rows_remote=1608 dedup_ratio=0.024689 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3994, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0] payload_rows_per_dst=[8188, 8120, 6678, 8168, 8190, 8192, 8192, 8190] dup_rows_per_dst=[144, 1280, 0, 0, 2, 10, 0, 182] combine_rows_per_src=[8188, 8120, 6678, 8168, 8190, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64634 payload_rows_remote=56940 dup_rows=902 dup_rows_remote=902 dedup_ratio=0.013763 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 3998, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0] payload_rows_per_dst=[8192, 8166, 7694, 7814, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 868, 0, 0, 0, 0, 0, 34] combine_rows_per_src=[8192, 8166, 7694, 7814, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59936 payload_rows_remote=51744 dup_rows=5600 dup_rows_remote=5570 dedup_ratio=0.085449 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3482, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0] payload_rows_per_dst=[8192, 8180, 8182, 4126, 8066, 7992, 8190, 7008] dup_rows_per_dst=[30, 1200, 4154, 4, 172, 0, 0, 40] combine_rows_per_src=[8192, 8180, 8182, 4126, 8066, 7992, 8190, 7008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63242 payload_rows_remote=55054 dup_rows=2294 dup_rows_remote=250 dedup_ratio=0.035004 router_tokens_per_expert=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3075, 0, 0, 4001, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0] payload_rows_per_dst=[8050, 8188, 8150, 6154, 8138, 8192, 8184, 8186] dup_rows_per_dst=[0, 2044, 0, 0, 0, 160, 2, 88] combine_rows_per_src=[8050, 8188, 8150, 6154, 8138, 8192, 8184, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230629376 dispatch_meta_bytes=229376 combine_payload_bytes=230629376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=230858752 ingress_bytes=287252480 link_bytes=287252480 token_bytes=461488128 weight_movement_bytes=56623104 total_nvlink_bytes=518111232 weight_to_token_ratio=0.122697 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.752311926e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=204685312 dispatch_meta_bytes=229120 combine_payload_bytes=204685312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204914432 ingress_bytes=223559680 link_bytes=223559680 token_bytes=409599744 weight_movement_bytes=18874368 total_nvlink_bytes=428474112 weight_to_token_ratio=0.046080 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142038856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64656 payload_rows_remote=56550 dup_rows=880 dup_rows_remote=880 dedup_ratio=0.013428 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 3973, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0] payload_rows_per_dst=[8192, 8078, 7514, 8106, 8190, 8192, 8192, 8192] dup_rows_per_dst=[2, 664, 0, 0, 0, 4, 0, 210] combine_rows_per_src=[8192, 8078, 7514, 8106, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=228253696 dispatch_meta_bytes=229336 combine_payload_bytes=228253696 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=228483032 ingress_bytes=266002432 link_bytes=266002432 token_bytes=456736728 weight_movement_bytes=37748736 total_nvlink_bytes=494485464 weight_to_token_ratio=0.082649 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.548704422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=233226240 dispatch_meta_bytes=231368 combine_payload_bytes=233226240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233457608 ingress_bytes=252100608 link_bytes=252100608 token_bytes=466683848 weight_movement_bytes=18874368 total_nvlink_bytes=485558216 weight_to_token_ratio=0.040444 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415503985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=211943424 dispatch_meta_bytes=229256 combine_payload_bytes=211943424 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=212172680 ingress_bytes=268566528 link_bytes=268566528 token_bytes=424116104 weight_movement_bytes=56623104 total_nvlink_bytes=480739208 weight_to_token_ratio=0.133508 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.573272328e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55342 payload_rows_remote=47424 dup_rows=10194 dup_rows_remote=9666 dedup_ratio=0.155548 router_tokens_per_expert=[0, 0, 0, 673, 0, 0, 4057, 0, 0, 0, 3784, 12, 0, 0, 0, 3104, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 2, 0, 3607, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 3, 0, 0, 0, 0, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 427, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2408, 5110, 8132, 7932, 8158, 8076, 7918, 7608] dup_rows_per_dst=[68, 0, 0, 3630, 0, 3966, 528, 2002] combine_rows_per_src=[2408, 5110, 8132, 7932, 8158, 8076, 7918, 7608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=225501184 dispatch_meta_bytes=221216 combine_payload_bytes=225501184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225722400 ingress_bytes=244375552 link_bytes=244375552 token_bytes=451223584 weight_movement_bytes=18874368 total_nvlink_bytes=470097952 weight_to_token_ratio=0.041829 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341486299e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=231628800 dispatch_meta_bytes=229720 combine_payload_bytes=231628800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231858520 ingress_bytes=288251904 link_bytes=288251904 token_bytes=463487320 weight_movement_bytes=56623104 total_nvlink_bytes=520110424 weight_to_token_ratio=0.122168 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.761887915e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=194248704 dispatch_meta_bytes=228360 combine_payload_bytes=194248704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=194477064 ingress_bytes=231997440 link_bytes=231997440 token_bytes=388725768 weight_movement_bytes=37748736 total_nvlink_bytes=426474504 weight_to_token_ratio=0.097109 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222885320e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=238 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3745, 0, 4096, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8162, 7988, 8190, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[148, 0, 90, 0, 0, 0, 0, 0] combine_rows_per_src=[8162, 7988, 8190, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64524 payload_rows_remote=56334 dup_rows=1012 dup_rows_remote=758 dedup_ratio=0.015442 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4065, 0, 4087, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[7498, 8164, 8190, 7934, 8192, 8178, 8178, 8190] dup_rows_per_dst=[48, 0, 12, 0, 12, 540, 146, 254] combine_rows_per_src=[7498, 8164, 8190, 7934, 8192, 8178, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=0 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3798, 0, 4095, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 30, 0, 0, 0, 0, 0] combine_rows_per_src=[8164, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57332 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3470, 0, 4096, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8186, 8192, 8192, 8192, 8176, 8192, 8190] dup_rows_per_dst=[0, 0, 6, 12, 10, 0, 0, 0] combine_rows_per_src=[8188, 8186, 8192, 8192, 8192, 8176, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64752 payload_rows_remote=56560 dup_rows=784 dup_rows_remote=564 dedup_ratio=0.011963 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4001, 0, 4094, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 7856, 8192, 7986, 8190, 8188, 7958] dup_rows_per_dst=[220, 552, 2, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8190, 7856, 8192, 7986, 8190, 8188, 7958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229376 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234135552 ingress_bytes=271654912 link_bytes=271654912 token_bytes=468041728 weight_movement_bytes=37748736 total_nvlink_bytes=505790464 weight_to_token_ratio=0.080653 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602863704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=228368 combine_payload_bytes=230744064 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=230972432 ingress_bytes=287367168 link_bytes=287367168 token_bytes=461716496 weight_movement_bytes=56623104 total_nvlink_bytes=518339600 weight_to_token_ratio=0.122636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.753410810e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64924 payload_rows_remote=56886 dup_rows=612 dup_rows_remote=612 dedup_ratio=0.009338 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4034, 0, 4045, 0, 13, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8038, 8192, 8190, 8192, 7868, 8090, 8162] dup_rows_per_dst=[240, 0, 14, 214, 138, 0, 0, 6] combine_rows_per_src=[8192, 8038, 8192, 8190, 8192, 7868, 8090, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65326 payload_rows_remote=57134 dup_rows=210 dup_rows_remote=206 dedup_ratio=0.003204 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3712, 0, 4087, 0, 1, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8122, 8098, 8166, 8192, 8192, 8190, 8174, 8192] dup_rows_per_dst=[116, 0, 84, 4, 4, 0, 0, 2] combine_rows_per_src=[8122, 8098, 8166, 8192, 8192, 8190, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229256 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234987400 ingress_bytes=272506880 link_bytes=272506880 token_bytes=469745544 weight_movement_bytes=37748736 total_nvlink_bytes=507494280 weight_to_token_ratio=0.080360 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611026842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229440 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061312 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893184 weight_movement_bytes=18874368 total_nvlink_bytes=488767552 weight_to_token_ratio=0.040167 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=231669760 dispatch_meta_bytes=228496 combine_payload_bytes=231669760 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231898256 ingress_bytes=307167232 link_bytes=307167232 token_bytes=463568016 weight_movement_bytes=75497472 total_nvlink_bytes=539065488 weight_to_token_ratio=0.162862 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.943125281e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61748 payload_rows_remote=53556 dup_rows=3788 dup_rows_remote=2814 dedup_ratio=0.057800 router_tokens_per_expert=[0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2337, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 52, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 4096, 0, 83, 2, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[7982, 8192, 4674, 8192, 8190, 8140, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 104, 0, 0, 974, 2710] combine_rows_per_src=[7982, 8192, 4674, 8192, 8190, 8140, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=233005056 dispatch_meta_bytes=229992 combine_payload_bytes=233005056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233235048 ingress_bytes=251879424 link_bytes=251879424 token_bytes=466240104 weight_movement_bytes=18874368 total_nvlink_bytes=485114472 weight_to_token_ratio=0.040482 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413384708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234020864 dispatch_meta_bytes=229360 combine_payload_bytes=234020864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234250224 ingress_bytes=252895232 link_bytes=252895232 token_bytes=468271088 weight_movement_bytes=18874368 total_nvlink_bytes=487145456 weight_to_token_ratio=0.040306 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423117681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60200 payload_rows_remote=52008 dup_rows=5336 dup_rows_remote=4498 dedup_ratio=0.081421 router_tokens_per_expert=[0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 527, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575] payload_rows_per_dst=[7656, 8192, 7926, 5276, 8192, 8192, 8192, 6574] dup_rows_per_dst=[0, 0, 2, 732, 838, 1054, 0, 2710] combine_rows_per_src=[7656, 8192, 7926, 5276, 8192, 8192, 8192, 6574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219365376 dispatch_meta_bytes=225480 combine_payload_bytes=219365376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219590856 ingress_bytes=275988480 link_bytes=275988480 token_bytes=438956232 weight_movement_bytes=56623104 total_nvlink_bytes=495579336 weight_to_token_ratio=0.128995 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.104167 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.644385821e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53762 payload_rows_remote=45572 dup_rows=11774 dup_rows_remote=7344 dedup_ratio=0.179657 router_tokens_per_expert=[0, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 4094, 4095, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 350, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2215] payload_rows_per_dst=[7438, 2326, 8086, 3146, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 4, 1518, 2, 772, 5046, 2, 4430] combine_rows_per_src=[7438, 2326, 8086, 3146, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60962 payload_rows_remote=52780 dup_rows=4574 dup_rows_remote=4574 dedup_ratio=0.069794 router_tokens_per_expert=[0, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 233, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315] payload_rows_per_dst=[7190, 8192, 8182, 8192, 8192, 8192, 8192, 4630] dup_rows_per_dst=[0, 0, 0, 4036, 72, 466, 0, 0] combine_rows_per_src=[7190, 8192, 8182, 8192, 8192, 8192, 8192, 4630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60122 payload_rows_remote=51930 dup_rows=5414 dup_rows_remote=5384 dedup_ratio=0.082611 router_tokens_per_expert=[0, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2677] payload_rows_per_dst=[5498, 8192, 8182, 5482, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 6, 30, 0, 5374] combine_rows_per_src=[5498, 8192, 8182, 5482, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56412 payload_rows_remote=48220 dup_rows=9124 dup_rows_remote=9016 dedup_ratio=0.139221 router_tokens_per_expert=[0, 0, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4096, 4096, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1346, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1786] payload_rows_per_dst=[6056, 8192, 7662, 8184, 8192, 6362, 8192, 3572] dup_rows_per_dst=[1896, 108, 0, 4656, 1040, 1424, 0, 0] combine_rows_per_src=[6056, 8192, 7662, 8184, 8192, 6362, 8192, 3572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213024768 dispatch_meta_bytes=226024 combine_payload_bytes=213024768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213250792 ingress_bytes=231899136 link_bytes=231899136 token_bytes=426275560 weight_movement_bytes=18874368 total_nvlink_bytes=445149928 weight_to_token_ratio=0.044277 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221943420e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47178 payload_rows_remote=38988 dup_rows=18358 dup_rows_remote=10422 dedup_ratio=0.280121 router_tokens_per_expert=[0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4096, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 0, 0, 0, 0, 729, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 316, 4094, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980] payload_rows_per_dst=[8190, 8192, 7920, 8192, 3886, 652, 8186, 1960] dup_rows_per_dst=[7936, 5650, 0, 4388, 380, 0, 4, 0] combine_rows_per_src=[8190, 8192, 7920, 8192, 3886, 652, 8186, 1960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=186662912 dispatch_meta_bytes=211664 combine_payload_bytes=186662912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186874576 ingress_bytes=205537280 link_bytes=205537280 token_bytes=373537488 weight_movement_bytes=18874368 total_nvlink_bytes=392411856 weight_to_token_ratio=0.050529 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.969357086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216186880 dispatch_meta_bytes=229416 combine_payload_bytes=216186880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216416296 ingress_bytes=235061248 link_bytes=235061248 token_bytes=432603176 weight_movement_bytes=18874368 total_nvlink_bytes=451477544 weight_to_token_ratio=0.043630 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252241221e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=212705280 dispatch_meta_bytes=229256 combine_payload_bytes=212705280 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=212934536 ingress_bytes=269328384 link_bytes=269328384 token_bytes=425639816 weight_movement_bytes=56623104 total_nvlink_bytes=482262920 weight_to_token_ratio=0.133031 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.580572058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61162 payload_rows_remote=52970 dup_rows=4374 dup_rows_remote=1434 dedup_ratio=0.066742 router_tokens_per_expert=[0, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 330, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966] payload_rows_per_dst=[6198, 8192, 8074, 8192, 8192, 8190, 8192, 5932] dup_rows_per_dst=[0, 14, 0, 2940, 760, 660, 0, 0] combine_rows_per_src=[6198, 8192, 8074, 8192, 8192, 8190, 8192, 5932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=197509120 dispatch_meta_bytes=228944 combine_payload_bytes=197509120 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=197738064 ingress_bytes=273006592 link_bytes=273006592 token_bytes=395247184 weight_movement_bytes=75497472 total_nvlink_bytes=470744656 weight_to_token_ratio=0.191013 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.615814837e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64360 payload_rows_remote=56168 dup_rows=1176 dup_rows_remote=1176 dedup_ratio=0.017944 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 3882, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0] payload_rows_per_dst=[8172, 8016, 7516, 7896, 8188, 8188, 8192, 8192] dup_rows_per_dst=[4, 830, 0, 0, 8, 32, 0, 302] combine_rows_per_src=[8172, 8016, 7516, 7896, 8188, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=197640 combine_payload_bytes=159694848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159892488 ingress_bytes=178569216 link_bytes=178569216 token_bytes=319587336 weight_movement_bytes=18874368 total_nvlink_bytes=338461704 weight_to_token_ratio=0.059059 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.710962366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216965120 dispatch_meta_bytes=217616 combine_payload_bytes=216965120 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=217182736 ingress_bytes=254713856 link_bytes=254713856 token_bytes=434147856 weight_movement_bytes=37748736 total_nvlink_bytes=471896592 weight_to_token_ratio=0.086949 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.440542841e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230064128 dispatch_meta_bytes=229376 combine_payload_bytes=230064128 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230293504 ingress_bytes=267812864 link_bytes=267812864 token_bytes=460357632 weight_movement_bytes=37748736 total_nvlink_bytes=498106368 weight_to_token_ratio=0.081999 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.566051091e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61586 payload_rows_remote=53394 dup_rows=3950 dup_rows_remote=3950 dedup_ratio=0.060272 router_tokens_per_expert=[2123, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 142, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1663, 0, 0, 0, 24, 0, 0, 0, 0, 0] payload_rows_per_dst=[4246, 8192, 8190, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 310, 0, 0, 276, 0, 3364] combine_rows_per_src=[4246, 8192, 8190, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59674 payload_rows_remote=51482 dup_rows=5862 dup_rows_remote=72 dedup_ratio=0.089447 router_tokens_per_expert=[1172, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 45, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 2, 0, 1, 0, 1, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 2886, 0, 0, 0, 10, 0, 0, 0, 0, 0] payload_rows_per_dst=[2344, 8190, 8192, 8186, 8188, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 56, 0, 0, 14, 2, 5790] combine_rows_per_src=[2344, 8190, 8192, 8186, 8188, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60406 payload_rows_remote=52220 dup_rows=5130 dup_rows_remote=4924 dedup_ratio=0.078278 router_tokens_per_expert=[2464, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8186, 8192, 8192, 8192, 8192, 3072] dup_rows_per_dst=[4914, 0, 206, 0, 0, 8, 0, 2] combine_rows_per_src=[8188, 8192, 8186, 8192, 8192, 8192, 8192, 3072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60228 payload_rows_remote=52036 dup_rows=5308 dup_rows_remote=5268 dedup_ratio=0.080994 router_tokens_per_expert=[1448, 0, 0, 0, 0, 1, 4095, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 13, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 2112, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[2898, 8192, 8190, 8186, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 1060, 0, 0, 40, 0, 4208] combine_rows_per_src=[2898, 8192, 8190, 8186, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=218701824 dispatch_meta_bytes=229376 combine_payload_bytes=218701824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218931200 ingress_bytes=237576192 link_bytes=237576192 token_bytes=437633024 weight_movement_bytes=18874368 total_nvlink_bytes=456507392 weight_to_token_ratio=0.043128 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276338177e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60552 payload_rows_remote=52362 dup_rows=4984 dup_rows_remote=4984 dedup_ratio=0.076050 router_tokens_per_expert=[1955, 0, 0, 0, 0, 53, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2, 0, 0, 1694, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8180, 8026, 8188, 8192, 8192, 3392] dup_rows_per_dst=[4004, 0, 930, 0, 0, 0, 48, 2] combine_rows_per_src=[8192, 8190, 8180, 8026, 8188, 8192, 8192, 3392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=210870272 dispatch_meta_bytes=206216 combine_payload_bytes=210870272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211076488 ingress_bytes=229744640 link_bytes=229744640 token_bytes=421946760 weight_movement_bytes=18874368 total_nvlink_bytes=440821128 weight_to_token_ratio=0.044732 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.201300099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213893120 dispatch_meta_bytes=228576 combine_payload_bytes=213893120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=214121696 ingress_bytes=251641856 link_bytes=251641856 token_bytes=428014816 weight_movement_bytes=37748736 total_nvlink_bytes=465763552 weight_to_token_ratio=0.088195 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411108449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213139456 dispatch_meta_bytes=229216 combine_payload_bytes=213139456 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=213368672 ingress_bytes=288636928 link_bytes=288636928 token_bytes=426508128 weight_movement_bytes=75497472 total_nvlink_bytes=502005600 weight_to_token_ratio=0.177013 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.765577026e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56594 payload_rows_remote=48402 dup_rows=8942 dup_rows_remote=590 dedup_ratio=0.136444 router_tokens_per_expert=[1807, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 288, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 39, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4310, 8188, 8190, 7640, 8144, 3738] dup_rows_per_dst=[8352, 4, 378, 192, 10, 6, 0, 0] combine_rows_per_src=[8192, 8192, 4310, 8188, 8190, 7640, 8144, 3738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61806 payload_rows_remote=53620 dup_rows=3730 dup_rows_remote=3730 dedup_ratio=0.056915 router_tokens_per_expert=[1434, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 272, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 82, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 0, 0, 0, 21, 0, 0, 0, 0, 0] payload_rows_per_dst=[7708, 8192, 8192, 8186, 8192, 8192, 8192, 4952] dup_rows_per_dst=[2546, 0, 556, 0, 0, 166, 0, 462] combine_rows_per_src=[7708, 8192, 8192, 8186, 8192, 8192, 8192, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=214474752 dispatch_meta_bytes=229384 combine_payload_bytes=214474752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214704136 ingress_bytes=233349120 link_bytes=233349120 token_bytes=429178888 weight_movement_bytes=18874368 total_nvlink_bytes=448053256 weight_to_token_ratio=0.043978 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.235836453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65284 payload_rows_remote=57098 dup_rows=252 dup_rows_remote=252 dedup_ratio=0.003845 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3738, 0, 4093, 0, 1, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8128, 8192, 8192, 8192, 8012, 8186, 8192] dup_rows_per_dst=[56, 0, 128, 30, 32, 0, 0, 6] combine_rows_per_src=[8190, 8128, 8192, 8192, 8192, 8012, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=198254592 dispatch_meta_bytes=195968 combine_payload_bytes=198254592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=198450560 ingress_bytes=254877696 link_bytes=254877696 token_bytes=396705152 weight_movement_bytes=56623104 total_nvlink_bytes=453328256 weight_to_token_ratio=0.142733 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.442112675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=219627520 dispatch_meta_bytes=229400 combine_payload_bytes=219627520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219856920 ingress_bytes=238501888 link_bytes=238501888 token_bytes=439484440 weight_movement_bytes=18874368 total_nvlink_bytes=458358808 weight_to_token_ratio=0.042947 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.285207741e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60912 payload_rows_remote=52720 dup_rows=4624 dup_rows_remote=4624 dedup_ratio=0.070557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4091, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1683, 0, 0, 0] payload_rows_per_dst=[8170, 8192, 8086, 8192, 8192, 8182, 7128, 4770] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3542, 1082] combine_rows_per_src=[8170, 8192, 8086, 8192, 8192, 8182, 7128, 4770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59418 payload_rows_remote=51226 dup_rows=6118 dup_rows_remote=28 dedup_ratio=0.093353 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 4094, 4090, 0, 1, 1, 4095, 0, 1, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3040, 0, 0, 1] payload_rows_per_dst=[8164, 8182, 8188, 8192, 8192, 8190, 2118, 8192] dup_rows_per_dst=[4, 0, 0, 0, 4, 20, 0, 6090] combine_rows_per_src=[8164, 8182, 8188, 8192, 8192, 8190, 2118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63316 payload_rows_remote=55200 dup_rows=2220 dup_rows_remote=2220 dedup_ratio=0.033875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3025, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8116, 8192, 8192, 8190, 8192, 6050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2220, 0] combine_rows_per_src=[8192, 8192, 8116, 8192, 8192, 8190, 8192, 6050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=233873408 dispatch_meta_bytes=229400 combine_payload_bytes=233873408 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234102808 ingress_bytes=271622144 link_bytes=271622144 token_bytes=467976216 weight_movement_bytes=37748736 total_nvlink_bytes=505724952 weight_to_token_ratio=0.080664 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602549737e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57432 payload_rows_remote=49242 dup_rows=8104 dup_rows_remote=8104 dedup_ratio=0.123657 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4052, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 90, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8104] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 90, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=215941120 dispatch_meta_bytes=229376 combine_payload_bytes=215941120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216170496 ingress_bytes=234815488 link_bytes=234815488 token_bytes=432111616 weight_movement_bytes=18874368 total_nvlink_bytes=450985984 weight_to_token_ratio=0.043679 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249886470e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=209821696 dispatch_meta_bytes=205016 combine_payload_bytes=209821696 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=210026712 ingress_bytes=285319168 link_bytes=285319168 token_bytes=419848408 weight_movement_bytes=75497472 total_nvlink_bytes=495345880 weight_to_token_ratio=0.179821 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.733787882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=226099200 dispatch_meta_bytes=229680 combine_payload_bytes=226099200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226328880 ingress_bytes=244973568 link_bytes=244973568 token_bytes=452428080 weight_movement_bytes=18874368 total_nvlink_bytes=471302448 weight_to_token_ratio=0.041718 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.347216194e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59970 payload_rows_remote=51778 dup_rows=5566 dup_rows_remote=5560 dedup_ratio=0.084930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1373, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8076, 8192, 8192, 8188, 8192, 2746] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 5560, 0] combine_rows_per_src=[8192, 8192, 8076, 8192, 8192, 8188, 8192, 2746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=201695232 dispatch_meta_bytes=229384 combine_payload_bytes=201695232 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=201924616 ingress_bytes=239443968 link_bytes=239443968 token_bytes=403619848 weight_movement_bytes=37748736 total_nvlink_bytes=441368584 weight_to_token_ratio=0.093525 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294234288e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60672 payload_rows_remote=52488 dup_rows=4864 dup_rows_remote=4682 dedup_ratio=0.074219 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4063, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2031, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8162, 8192, 8188, 8186, 8192, 3376] dup_rows_per_dst=[182, 4, 500, 0, 0, 0, 4178, 0] combine_rows_per_src=[8184, 8192, 8162, 8192, 8188, 8186, 8192, 3376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61132 payload_rows_remote=52940 dup_rows=4404 dup_rows_remote=4404 dedup_ratio=0.067200 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1985, 0, 0, 0] payload_rows_per_dst=[8128, 8192, 8074, 8192, 8192, 8192, 8192, 3970] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 4398, 0] combine_rows_per_src=[8128, 8192, 8074, 8192, 8192, 8192, 8192, 3970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=212082688 dispatch_meta_bytes=229352 combine_payload_bytes=212082688 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=212312040 ingress_bytes=249831424 link_bytes=249831424 token_bytes=424394728 weight_movement_bytes=37748736 total_nvlink_bytes=462143464 weight_to_token_ratio=0.088947 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393761780e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58538 payload_rows_remote=50346 dup_rows=6998 dup_rows_remote=6998 dedup_ratio=0.106781 router_tokens_per_expert=[0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4096, 4096, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 346, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2890] payload_rows_per_dst=[7478, 8192, 7982, 2118, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 86, 2, 0, 438, 692, 0, 5780] combine_rows_per_src=[7478, 8192, 7982, 2118, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=214990848 dispatch_meta_bytes=228680 combine_payload_bytes=214990848 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=215219528 ingress_bytes=290488320 link_bytes=290488320 token_bytes=430210376 weight_movement_bytes=75497472 total_nvlink_bytes=505707848 weight_to_token_ratio=0.175490 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.783316153e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65294 payload_rows_remote=57102 dup_rows=242 dup_rows_remote=242 dedup_ratio=0.003693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 207, 0, 0, 0, 0, 3909, 0, 0] payload_rows_per_dst=[8192, 8192, 8122, 8192, 8192, 8192, 8108, 8104] dup_rows_per_dst=[0, 0, 0, 112, 0, 0, 2, 128] combine_rows_per_src=[8192, 8192, 8122, 8192, 8192, 8192, 8108, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=216842240 dispatch_meta_bytes=229376 combine_payload_bytes=216842240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217071616 ingress_bytes=235716608 link_bytes=235716608 token_bytes=433913856 weight_movement_bytes=18874368 total_nvlink_bytes=452788224 weight_to_token_ratio=0.043498 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.258520558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64276 payload_rows_remote=56154 dup_rows=1260 dup_rows_remote=132 dedup_ratio=0.019226 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 4084, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 26, 0, 0, 1, 0, 4032, 0, 0] payload_rows_per_dst=[8158, 8188, 8170, 7068, 8192, 8192, 8186, 8122] dup_rows_per_dst=[4, 0, 32, 92, 2, 2, 0, 1128] combine_rows_per_src=[8158, 8188, 8170, 7068, 8192, 8192, 8186, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57202 dup_rows=144 dup_rows_remote=144 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 63, 0, 0, 0, 0, 3963, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8190, 8052] dup_rows_per_dst=[0, 0, 0, 142, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8190, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65412 payload_rows_remote=57220 dup_rows=124 dup_rows_remote=124 dedup_ratio=0.001892 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 4082, 0, 0] payload_rows_per_dst=[8192, 8192, 8084, 8190, 8192, 8192, 8186, 8184] dup_rows_per_dst=[0, 0, 0, 52, 0, 0, 6, 66] combine_rows_per_src=[8192, 8192, 8084, 8190, 8192, 8192, 8186, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233889792 dispatch_meta_bytes=229376 combine_payload_bytes=233889792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234119168 ingress_bytes=252764160 link_bytes=252764160 token_bytes=468008960 weight_movement_bytes=18874368 total_nvlink_bytes=486883328 weight_to_token_ratio=0.040329 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421861813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=206217216 dispatch_meta_bytes=229376 combine_payload_bytes=206217216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206446592 ingress_bytes=225091584 link_bytes=225091584 token_bytes=412663808 weight_movement_bytes=18874368 total_nvlink_bytes=431538176 weight_to_token_ratio=0.045738 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.156716806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230006784 dispatch_meta_bytes=225144 combine_payload_bytes=230006784 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=230231928 ingress_bytes=324378624 link_bytes=324378624 token_bytes=460238712 weight_movement_bytes=94371840 total_nvlink_bytes=554610552 weight_to_token_ratio=0.205050 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.108036371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65408 payload_rows_remote=57216 dup_rows=128 dup_rows_remote=128 dedup_ratio=0.001953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4095, 0, 61, 0, 0, 0, 0, 3977, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8190, 8192, 8190, 8076] dup_rows_per_dst=[28, 0, 0, 90, 0, 2, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8190, 8192, 8190, 8076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234299392 dispatch_meta_bytes=229384 combine_payload_bytes=234299392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234528776 ingress_bytes=272048128 link_bytes=272048128 token_bytes=468828168 weight_movement_bytes=37748736 total_nvlink_bytes=506576904 weight_to_token_ratio=0.080517 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606631306e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234373120 dispatch_meta_bytes=229376 combine_payload_bytes=234373120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234602496 ingress_bytes=253247488 link_bytes=253247488 token_bytes=468975616 weight_movement_bytes=18874368 total_nvlink_bytes=487849984 weight_to_token_ratio=0.040246 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426492824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64704 payload_rows_remote=56512 dup_rows=832 dup_rows_remote=832 dedup_ratio=0.012695 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 3883, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8188, 8178, 7378] dup_rows_per_dst=[0, 4, 232, 482, 0, 0, 114, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8188, 8178, 7378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=56 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 146, 0, 0, 0, 0, 3946, 0, 0] payload_rows_per_dst=[8192, 8192, 8142, 8192, 8192, 8192, 8160, 8130] dup_rows_per_dst=[2, 0, 0, 88, 0, 0, 0, 54] combine_rows_per_src=[8192, 8192, 8142, 8192, 8192, 8192, 8160, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234356736 dispatch_meta_bytes=229376 combine_payload_bytes=234356736 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234586112 ingress_bytes=272105472 link_bytes=272105472 token_bytes=468942848 weight_movement_bytes=37748736 total_nvlink_bytes=506691584 weight_to_token_ratio=0.080498 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607180748e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231473152 dispatch_meta_bytes=229376 combine_payload_bytes=231473152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231702528 ingress_bytes=250347520 link_bytes=250347520 token_bytes=463175680 weight_movement_bytes=18874368 total_nvlink_bytes=482050048 weight_to_token_ratio=0.040750 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398706758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58924 payload_rows_remote=50732 dup_rows=6612 dup_rows_remote=6592 dedup_ratio=0.100891 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 2310, 8192, 7470, 8192, 8192] dup_rows_per_dst=[0, 592, 0, 0, 20, 4050, 1950, 0] combine_rows_per_src=[8184, 8192, 8192, 2310, 8192, 7470, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229024 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234520224 ingress_bytes=253165568 link_bytes=253165568 token_bytes=468811424 weight_movement_bytes=18874368 total_nvlink_bytes=487685792 weight_to_token_ratio=0.040260 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425707907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57294 payload_rows_remote=49102 dup_rows=8242 dup_rows_remote=8238 dedup_ratio=0.125763 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4001, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 658, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8180, 4506, 8188, 3684, 8192, 8162, 8190, 8192] dup_rows_per_dst=[2, 78, 0, 0, 14, 1160, 6984, 4] combine_rows_per_src=[8180, 4506, 8188, 3684, 8192, 8162, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61290 payload_rows_remote=53098 dup_rows=4246 dup_rows_remote=4246 dedup_ratio=0.064789 router_tokens_per_expert=[1976, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 196, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 0, 0, 0, 8, 0, 0, 0, 0, 0] payload_rows_per_dst=[3952, 8192, 8190, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 410, 0, 0, 42, 0, 3794] combine_rows_per_src=[3952, 8192, 8190, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57236 payload_rows_remote=49044 dup_rows=8300 dup_rows_remote=8300 dedup_ratio=0.126648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 6304, 1782, 8192] dup_rows_per_dst=[0, 164, 0, 8106, 30, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 6304, 1782, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57332 payload_rows_remote=49144 dup_rows=8204 dup_rows_remote=6754 dedup_ratio=0.125183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 0, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 6, 1450, 6744, 0] combine_rows_per_src=[8184, 8192, 8192, 0, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=207798272 dispatch_meta_bytes=229296 combine_payload_bytes=207798272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208027568 ingress_bytes=226672640 link_bytes=226672640 token_bytes=415825840 weight_movement_bytes=18874368 total_nvlink_bytes=434700208 weight_to_token_ratio=0.045390 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.171865707e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201121792 dispatch_meta_bytes=229360 combine_payload_bytes=201121792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201351152 ingress_bytes=219996160 link_bytes=219996160 token_bytes=402472944 weight_movement_bytes=18874368 total_nvlink_bytes=421347312 weight_to_token_ratio=0.046896 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.107894960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57044 payload_rows_remote=48852 dup_rows=8492 dup_rows_remote=8262 dedup_ratio=0.129578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 8192, 8192, 8192, 5996, 1952, 8192] dup_rows_per_dst=[0, 230, 0, 8132, 130, 0, 0, 0] combine_rows_per_src=[8136, 8192, 8192, 8192, 8192, 5996, 1952, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=217489408 dispatch_meta_bytes=229376 combine_payload_bytes=217489408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217718784 ingress_bytes=236363776 link_bytes=236363776 token_bytes=435208192 weight_movement_bytes=18874368 total_nvlink_bytes=454082560 weight_to_token_ratio=0.043369 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264721403e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=200884224 dispatch_meta_bytes=229376 combine_payload_bytes=200884224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=201113600 ingress_bytes=257507328 link_bytes=257507328 token_bytes=401997824 weight_movement_bytes=56623104 total_nvlink_bytes=458620928 weight_to_token_ratio=0.140854 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.467308515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201293824 dispatch_meta_bytes=223592 combine_payload_bytes=201293824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201517416 ingress_bytes=220168192 link_bytes=220168192 token_bytes=402811240 weight_movement_bytes=18874368 total_nvlink_bytes=421685608 weight_to_token_ratio=0.046857 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109543286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55172 payload_rows_remote=46980 dup_rows=10364 dup_rows_remote=10278 dedup_ratio=0.158142 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7902, 5122, 1192, 8188] dup_rows_per_dst=[86, 2016, 4, 8186, 72, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7902, 5122, 1192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59618 payload_rows_remote=51488 dup_rows=5918 dup_rows_remote=3506 dedup_ratio=0.090302 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8130, 8192, 4346, 6182, 8192] dup_rows_per_dst=[0, 444, 0, 2412, 6, 0, 3056, 0] combine_rows_per_src=[8192, 8192, 8192, 8130, 8192, 4346, 6182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=200097792 dispatch_meta_bytes=228456 combine_payload_bytes=200097792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200326248 ingress_bytes=237846528 link_bytes=237846528 token_bytes=400424040 weight_movement_bytes=37748736 total_nvlink_bytes=438172776 weight_to_token_ratio=0.094272 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.278928404e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=192430080 dispatch_meta_bytes=229032 combine_payload_bytes=192430080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=192659112 ingress_bytes=211304448 link_bytes=211304448 token_bytes=385089192 weight_movement_bytes=18874368 total_nvlink_bytes=403963560 weight_to_token_ratio=0.049013 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.024615252e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57324 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 51, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8178, 8174, 8192, 8192] dup_rows_per_dst=[26, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8178, 8174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=210894848 dispatch_meta_bytes=219976 combine_payload_bytes=210894848 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=211114824 ingress_bytes=286392320 link_bytes=286392320 token_bytes=422009672 weight_movement_bytes=75497472 total_nvlink_bytes=497507144 weight_to_token_ratio=0.178900 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.744070296e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60368 payload_rows_remote=52176 dup_rows=5168 dup_rows_remote=5168 dedup_ratio=0.078857 router_tokens_per_expert=[4071, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 4069, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 214, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8088, 3336, 8192, 8006, 8172, 8190, 8192, 8192] dup_rows_per_dst=[26, 12, 0, 0, 1368, 3758, 4, 0] combine_rows_per_src=[8088, 3336, 8192, 8006, 8172, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65308 payload_rows_remote=57116 dup_rows=228 dup_rows_remote=228 dedup_ratio=0.003479 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8040, 8188, 8120, 8192, 8192] dup_rows_per_dst=[24, 70, 0, 0, 0, 126, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8040, 8188, 8120, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60794 payload_rows_remote=56932 dup_rows=4742 dup_rows_remote=4706 dedup_ratio=0.072357 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3905, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1467, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8164, 7810, 3862, 8192, 8192] dup_rows_per_dst=[48, 4650, 0, 0, 0, 36, 8, 0] combine_rows_per_src=[8190, 8192, 8192, 8164, 7810, 3862, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60764 payload_rows_remote=57230 dup_rows=4772 dup_rows_remote=4772 dedup_ratio=0.072815 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2386, 0, 0, 0] payload_rows_per_dst=[8128, 8192, 8142, 8192, 8192, 8192, 3534, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4772] combine_rows_per_src=[8128, 8192, 8142, 8192, 8192, 8192, 3534, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229432 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235028536 ingress_bytes=329170944 link_bytes=329170944 token_bytes=469827640 weight_movement_bytes=94371840 total_nvlink_bytes=564199480 weight_to_token_ratio=0.200865 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153954023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64748 payload_rows_remote=56556 dup_rows=788 dup_rows_remote=786 dedup_ratio=0.012024 router_tokens_per_expert=[4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8192, 8192, 7460, 8184, 8186, 8192, 8192] dup_rows_per_dst=[0, 2, 6, 0, 0, 426, 354, 0] combine_rows_per_src=[8150, 8192, 8192, 7460, 8184, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=213712896 dispatch_meta_bytes=229376 combine_payload_bytes=213712896 prefetch_bytes=0 reduce_bytes=0 egress_bytes=213942272 ingress_bytes=213712896 link_bytes=213942272 token_bytes=427655168 weight_movement_bytes=0 total_nvlink_bytes=427655168 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049889584e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233193472 dispatch_meta_bytes=246552 combine_payload_bytes=233193472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233440024 ingress_bytes=252067840 link_bytes=252067840 token_bytes=466633496 weight_movement_bytes=18874368 total_nvlink_bytes=485507864 weight_to_token_ratio=0.040448 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415190018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233947136 dispatch_meta_bytes=229376 combine_payload_bytes=233947136 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234176512 ingress_bytes=252821504 link_bytes=252821504 token_bytes=468123648 weight_movement_bytes=18874368 total_nvlink_bytes=486998016 weight_to_token_ratio=0.040319 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422411255e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=248008 combine_payload_bytes=234414080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234662088 ingress_bytes=253288448 link_bytes=253288448 token_bytes=469076168 weight_movement_bytes=18874368 total_nvlink_bytes=487950536 weight_to_token_ratio=0.040237 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426885283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64040 payload_rows_remote=55852 dup_rows=1496 dup_rows_remote=1490 dedup_ratio=0.022827 router_tokens_per_expert=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8186, 8118, 7346, 8148, 7670, 8192] dup_rows_per_dst=[6, 52, 0, 1360, 0, 78, 0, 0] combine_rows_per_src=[8188, 8192, 8186, 8118, 7346, 8148, 7670, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=231653376 dispatch_meta_bytes=229368 combine_payload_bytes=231653376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231882744 ingress_bytes=250527744 link_bytes=250527744 token_bytes=463536120 weight_movement_bytes=18874368 total_nvlink_bytes=482410488 weight_to_token_ratio=0.040718 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.400433576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65350 payload_rows_remote=57184 dup_rows=186 dup_rows_remote=186 dedup_ratio=0.002838 router_tokens_per_expert=[4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8192, 8192, 8166, 8108, 8154, 8192, 8192] dup_rows_per_dst=[10, 28, 0, 0, 0, 146, 2, 0] combine_rows_per_src=[8154, 8192, 8192, 8166, 8108, 8154, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=228769792 dispatch_meta_bytes=229368 combine_payload_bytes=228769792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228999160 ingress_bytes=266518528 link_bytes=266518528 token_bytes=457768952 weight_movement_bytes=37748736 total_nvlink_bytes=495517688 weight_to_token_ratio=0.082462 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.553649400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61250 payload_rows_remote=53068 dup_rows=4286 dup_rows_remote=4286 dedup_ratio=0.065399 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1960, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8190, 8192, 3920, 8192, 8182, 8192, 8192, 8190] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 4282] combine_rows_per_src=[8190, 8192, 3920, 8192, 8182, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234225664 dispatch_meta_bytes=229480 combine_payload_bytes=234225664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234455144 ingress_bytes=253100032 link_bytes=253100032 token_bytes=468680808 weight_movement_bytes=18874368 total_nvlink_bytes=487555176 weight_to_token_ratio=0.040271 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425079974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64236 payload_rows_remote=56046 dup_rows=1300 dup_rows_remote=12 dedup_ratio=0.019836 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 3452, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0] payload_rows_per_dst=[8190, 8192, 6906, 8184, 8190, 8192, 8192, 8190] dup_rows_per_dst=[0, 4, 0, 0, 2, 4, 2, 1288] combine_rows_per_src=[8190, 8192, 6906, 8184, 8190, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65188 payload_rows_remote=56996 dup_rows=348 dup_rows_remote=348 dedup_ratio=0.005310 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 3920, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 7844, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 0, 0, 0, 0, 340] combine_rows_per_src=[8192, 8192, 7844, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60116 payload_rows_remote=51960 dup_rows=5420 dup_rows_remote=54 dedup_ratio=0.082703 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2720, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8156, 8192, 8192, 8192, 8192, 2808] dup_rows_per_dst=[0, 0, 5366, 0, 36, 0, 0, 18] combine_rows_per_src=[8192, 8192, 8156, 8192, 8192, 8192, 8192, 2808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=229416 combine_payload_bytes=217366528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217595944 ingress_bytes=236240896 link_bytes=236240896 token_bytes=434962472 weight_movement_bytes=18874368 total_nvlink_bytes=453836840 weight_to_token_ratio=0.043393 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.263544028e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57244 dup_rows=106 dup_rows_remote=86 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 59, 0, 0, 0, 0, 3998, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8192, 8192, 8192, 8186, 8104] dup_rows_per_dst=[0, 0, 0, 76, 0, 0, 20, 10] combine_rows_per_src=[8192, 8192, 8180, 8192, 8192, 8192, 8186, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63034 payload_rows_remote=54842 dup_rows=2502 dup_rows_remote=2502 dedup_ratio=0.038177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 1206, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8162, 8192, 8192, 8192, 8192, 8192, 8192, 5720] dup_rows_per_dst=[38, 0, 2462, 0, 0, 2, 0, 0] combine_rows_per_src=[8162, 8192, 8192, 8192, 8192, 8192, 8192, 5720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=229564416 dispatch_meta_bytes=224232 combine_payload_bytes=229564416 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=229788648 ingress_bytes=286187520 link_bytes=286187520 token_bytes=459353064 weight_movement_bytes=56623104 total_nvlink_bytes=515976168 weight_to_token_ratio=0.123267 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.742108004e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=229376 combine_payload_bytes=233455616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233684992 ingress_bytes=252329984 link_bytes=252329984 token_bytes=467140608 weight_movement_bytes=18874368 total_nvlink_bytes=486014976 weight_to_token_ratio=0.040404 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417701753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=212828160 dispatch_meta_bytes=208056 combine_payload_bytes=212828160 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=213036216 ingress_bytes=307200000 link_bytes=307200000 token_bytes=425864376 weight_movement_bytes=94371840 total_nvlink_bytes=520236216 weight_to_token_ratio=0.221601 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.943439248e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61044 payload_rows_remote=52854 dup_rows=4492 dup_rows_remote=4444 dedup_ratio=0.068542 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4087, 2241, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8190, 8174, 8192, 8192, 8168, 8186, 8190, 3752] dup_rows_per_dst=[48, 14, 4422, 8, 0, 0, 0, 0] combine_rows_per_src=[8190, 8174, 8192, 8192, 8168, 8186, 8190, 3752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=234471424 dispatch_meta_bytes=229320 combine_payload_bytes=234471424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234700744 ingress_bytes=253345792 link_bytes=253345792 token_bytes=469172168 weight_movement_bytes=18874368 total_nvlink_bytes=488046536 weight_to_token_ratio=0.040229 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427434725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62168 payload_rows_remote=53976 dup_rows=3368 dup_rows_remote=3368 dedup_ratio=0.051392 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2439, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8140, 8192, 4878, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 0, 0, 0, 0, 3360] combine_rows_per_src=[8140, 8192, 4878, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224632832 dispatch_meta_bytes=229376 combine_payload_bytes=224632832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224862208 ingress_bytes=262381568 link_bytes=262381568 token_bytes=449495040 weight_movement_bytes=37748736 total_nvlink_bytes=487243776 weight_to_token_ratio=0.083980 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.514011085e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57200 dup_rows=146 dup_rows_remote=146 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 73, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8048, 8190, 8192, 8192, 8192] dup_rows_per_dst=[146, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8048, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=216489984 dispatch_meta_bytes=229192 combine_payload_bytes=216489984 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=216719176 ingress_bytes=254238720 link_bytes=254238720 token_bytes=433209160 weight_movement_bytes=37748736 total_nvlink_bytes=470957896 weight_to_token_ratio=0.087137 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.435990322e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=221085696 dispatch_meta_bytes=229376 combine_payload_bytes=221085696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=221315072 ingress_bytes=277708800 link_bytes=277708800 token_bytes=442400768 weight_movement_bytes=56623104 total_nvlink_bytes=499023872 weight_to_token_ratio=0.127991 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.660869080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64304 payload_rows_remote=56114 dup_rows=1232 dup_rows_remote=1228 dedup_ratio=0.018799 router_tokens_per_expert=[0, 0, 0, 68, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4035, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7110, 8180, 8186, 8074, 8182, 8190, 8192, 8190] dup_rows_per_dst=[128, 310, 4, 362, 422, 2, 0, 4] combine_rows_per_src=[7110, 8180, 8186, 8074, 8182, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65208 payload_rows_remote=57016 dup_rows=328 dup_rows_remote=328 dedup_ratio=0.005005 router_tokens_per_expert=[0, 0, 0, 163, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8098, 8192, 8006, 8144, 8192, 8192, 8192] dup_rows_per_dst=[326, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8098, 8192, 8006, 8144, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229384 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234520584 ingress_bytes=272039936 link_bytes=272039936 token_bytes=468811784 weight_movement_bytes=37748736 total_nvlink_bytes=506560520 weight_to_token_ratio=0.080520 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606552815e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8192, 8172, 8192, 8192, 8192, 8192] dup_rows_per_dst=[34, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8176, 8192, 8172, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57336 payload_rows_remote=49144 dup_rows=8200 dup_rows_remote=5272 dedup_ratio=0.125122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 0, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 158, 0, 0, 12, 5102, 2928, 0] combine_rows_per_src=[8190, 8192, 8192, 0, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65308 payload_rows_remote=57118 dup_rows=228 dup_rows_remote=228 dedup_ratio=0.003479 router_tokens_per_expert=[0, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8178, 7980, 8192, 8192, 8192] dup_rows_per_dst=[52, 0, 2, 0, 0, 0, 0, 174] combine_rows_per_src=[8192, 8190, 8192, 8178, 7980, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229842944 dispatch_meta_bytes=229368 combine_payload_bytes=229842944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230072312 ingress_bytes=248717312 link_bytes=248717312 token_bytes=459915256 weight_movement_bytes=18874368 total_nvlink_bytes=478789624 weight_to_token_ratio=0.041039 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.383086907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233537536 dispatch_meta_bytes=229376 combine_payload_bytes=233537536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233766912 ingress_bytes=252411904 link_bytes=252411904 token_bytes=467304448 weight_movement_bytes=18874368 total_nvlink_bytes=486178816 weight_to_token_ratio=0.040390 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418486670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65122 payload_rows_remote=56930 dup_rows=414 dup_rows_remote=198 dedup_ratio=0.006317 router_tokens_per_expert=[0, 0, 0, 108, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8030, 8192, 8188, 8142, 8192, 8192, 7994] dup_rows_per_dst=[216, 0, 0, 198, 0, 0, 0, 0] combine_rows_per_src=[8192, 8030, 8192, 8188, 8142, 8192, 8192, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57326 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8162, 8176, 8192, 8192, 8192] dup_rows_per_dst=[34, 0, 0, 0, 0, 0, 0, 14] combine_rows_per_src=[8192, 8190, 8192, 8162, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201293824 dispatch_meta_bytes=217664 combine_payload_bytes=201293824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201511488 ingress_bytes=220168192 link_bytes=220168192 token_bytes=402805312 weight_movement_bytes=18874368 total_nvlink_bytes=421679680 weight_to_token_ratio=0.046857 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109543286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233955328 dispatch_meta_bytes=229384 combine_payload_bytes=233955328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234184712 ingress_bytes=252829696 link_bytes=252829696 token_bytes=468140040 weight_movement_bytes=18874368 total_nvlink_bytes=487014408 weight_to_token_ratio=0.040318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422489747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59784 payload_rows_remote=51592 dup_rows=5752 dup_rows_remote=5752 dedup_ratio=0.087769 router_tokens_per_expert=[0, 0, 0, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2860, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2454, 8192, 8192, 8192, 8192, 8192, 8178, 8192] dup_rows_per_dst=[0, 5720, 0, 32, 0, 0, 0, 0] combine_rows_per_src=[2454, 8192, 8192, 8192, 8192, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233185280 dispatch_meta_bytes=228512 combine_payload_bytes=233185280 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=233413792 ingress_bytes=327557120 link_bytes=327557120 token_bytes=466599072 weight_movement_bytes=94371840 total_nvlink_bytes=560970912 weight_to_token_ratio=0.202255 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.138491156e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229496 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235036792 ingress_bytes=272556032 link_bytes=272556032 token_bytes=469844088 weight_movement_bytes=37748736 total_nvlink_bytes=507592824 weight_to_token_ratio=0.080343 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611497793e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63012 payload_rows_remote=54820 dup_rows=2524 dup_rows_remote=2524 dedup_ratio=0.038513 router_tokens_per_expert=[0, 0, 0, 3629, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 461, 0, 0, 0, 0, 4095, 0, 1, 4092, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7260, 8188, 8190, 6612, 8188, 8192, 8190, 8192] dup_rows_per_dst=[2, 920, 0, 18, 0, 4, 1580, 0] combine_rows_per_src=[7260, 8188, 8190, 6612, 8188, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64024 payload_rows_remote=55832 dup_rows=1512 dup_rows_remote=1512 dedup_ratio=0.023071 router_tokens_per_expert=[0, 0, 0, 3434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 155, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6868, 8192, 8192, 8192, 8192, 8192, 8004, 8192] dup_rows_per_dst=[0, 312, 0, 1200, 0, 0, 0, 0] combine_rows_per_src=[6868, 8192, 8192, 8192, 8192, 8192, 8004, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=211320832 dispatch_meta_bytes=229376 combine_payload_bytes=211320832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211550208 ingress_bytes=230195200 link_bytes=230195200 token_bytes=422871040 weight_movement_bytes=18874368 total_nvlink_bytes=441745408 weight_to_token_ratio=0.044634 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.205617143e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63492 payload_rows_remote=55300 dup_rows=2044 dup_rows_remote=2044 dedup_ratio=0.031189 router_tokens_per_expert=[0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 425, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7748, 6598, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[1580, 442, 0, 22, 0, 0, 0, 0] combine_rows_per_src=[7748, 6598, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57078 dup_rows=266 dup_rows_remote=120 dedup_ratio=0.004059 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8032, 8182, 8098, 8192, 8192] dup_rows_per_dst=[52, 42, 0, 0, 0, 26, 146, 0] combine_rows_per_src=[8190, 8192, 8192, 8032, 8182, 8098, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=224542720 dispatch_meta_bytes=229376 combine_payload_bytes=224542720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224772096 ingress_bytes=243417088 link_bytes=243417088 token_bytes=449314816 weight_movement_bytes=18874368 total_nvlink_bytes=468189184 weight_to_token_ratio=0.042007 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332302769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59982 payload_rows_remote=57290 dup_rows=5554 dup_rows_remote=5520 dedup_ratio=0.084747 router_tokens_per_expert=[0, 0, 0, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 1346, 0, 0, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 2692, 8182, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[5500, 34, 0, 0, 0, 20, 0, 0] combine_rows_per_src=[8148, 2692, 8182, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=228687872 dispatch_meta_bytes=229376 combine_payload_bytes=228687872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228917248 ingress_bytes=247562240 link_bytes=247562240 token_bytes=457605120 weight_movement_bytes=18874368 total_nvlink_bytes=476479488 weight_to_token_ratio=0.041246 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372019575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=226508800 dispatch_meta_bytes=229376 combine_payload_bytes=226508800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226738176 ingress_bytes=245383168 link_bytes=245383168 token_bytes=453246976 weight_movement_bytes=18874368 total_nvlink_bytes=472121344 weight_to_token_ratio=0.041643 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351140780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233791488 dispatch_meta_bytes=228792 combine_payload_bytes=233791488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234020280 ingress_bytes=252665856 link_bytes=252665856 token_bytes=467811768 weight_movement_bytes=18874368 total_nvlink_bytes=486686136 weight_to_token_ratio=0.040346 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420919913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60334 payload_rows_remote=52146 dup_rows=5202 dup_rows_remote=560 dedup_ratio=0.079376 router_tokens_per_expert=[0, 0, 0, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1686, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 4392, 8192, 8192, 8192, 8162, 6828, 8188] dup_rows_per_dst=[4642, 362, 12, 178, 4, 0, 0, 4] combine_rows_per_src=[8188, 4392, 8192, 8192, 8192, 8162, 6828, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63462 payload_rows_remote=55270 dup_rows=2074 dup_rows_remote=1958 dedup_ratio=0.031647 router_tokens_per_expert=[0, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 996, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6124, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 1952, 0, 116, 0, 6, 0, 0] combine_rows_per_src=[6124, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=251240 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234911080 ingress_bytes=310157312 link_bytes=310157312 token_bytes=469570920 weight_movement_bytes=75497472 total_nvlink_bytes=545068392 weight_to_token_ratio=0.160780 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.971774757e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8174] dup_rows_per_dst=[0, 0, 0, 0, 20, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=213590016 dispatch_meta_bytes=210824 combine_payload_bytes=213590016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213800840 ingress_bytes=232464384 link_bytes=232464384 token_bytes=427390856 weight_movement_bytes=18874368 total_nvlink_bytes=446265224 weight_to_token_ratio=0.044162 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23680 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227359348e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=226385920 dispatch_meta_bytes=228912 combine_payload_bytes=226385920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226614832 ingress_bytes=245260288 link_bytes=245260288 token_bytes=453000752 weight_movement_bytes=18874368 total_nvlink_bytes=471875120 weight_to_token_ratio=0.041665 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.349963404e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65384 payload_rows_remote=57204 dup_rows=152 dup_rows_remote=18 dedup_ratio=0.002319 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 4089, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8190, 8092, 8192, 8156, 8192, 8192, 8180] dup_rows_per_dst=[0, 0, 0, 0, 4, 12, 2, 134] combine_rows_per_src=[8190, 8190, 8092, 8192, 8156, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8190] dup_rows_per_dst=[2, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229296 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028400 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827504 weight_movement_bytes=18874368 total_nvlink_bytes=488701872 weight_to_token_ratio=0.040173 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62518 payload_rows_remote=54326 dup_rows=3018 dup_rows_remote=3018 dedup_ratio=0.046051 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2599, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8170, 8192, 5198, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 3014] combine_rows_per_src=[8170, 8192, 5198, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8122] dup_rows_per_dst=[0, 0, 72, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234307584 dispatch_meta_bytes=228888 combine_payload_bytes=234307584 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234536472 ingress_bytes=309805056 link_bytes=309805056 token_bytes=468844056 weight_movement_bytes=75497472 total_nvlink_bytes=544341528 weight_to_token_ratio=0.161029 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968399613e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235085824 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942272 weight_movement_bytes=37748736 total_nvlink_bytes=507691008 weight_to_token_ratio=0.080326 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=222519296 dispatch_meta_bytes=229376 combine_payload_bytes=222519296 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=222748672 ingress_bytes=241393664 link_bytes=241393664 token_bytes=445267968 weight_movement_bytes=18874368 total_nvlink_bytes=464142336 weight_to_token_ratio=0.042389 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312915315e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8170, 8190, 8182] dup_rows_per_dst=[0, 2, 24, 4, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8170, 8190, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234815488 ingress_bytes=272334848 link_bytes=272334848 token_bytes=469401600 weight_movement_bytes=37748736 total_nvlink_bytes=507150336 weight_to_token_ratio=0.080419 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609378516e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8184] dup_rows_per_dst=[2, 0, 0, 0, 2, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57322 dup_rows=116 dup_rows_remote=116 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8192, 8098, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 74, 0, 42, 0, 0] combine_rows_per_src=[8192, 8184, 8192, 8192, 8098, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229376 combine_payload_bytes=234741760 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234971136 ingress_bytes=234741760 link_bytes=234971136 token_bytes=469712896 weight_movement_bytes=0 total_nvlink_bytes=469712896 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.251377812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229752 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235020664 ingress_bytes=291414016 link_bytes=291414016 token_bytes=469811576 weight_movement_bytes=56623104 total_nvlink_bytes=526434680 weight_to_token_ratio=0.120523 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792185717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235077632 ingress_bytes=272596992 link_bytes=272596992 token_bytes=469925888 weight_movement_bytes=37748736 total_nvlink_bytes=507674624 weight_to_token_ratio=0.080329 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611890251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64614 payload_rows_remote=56424 dup_rows=922 dup_rows_remote=920 dedup_ratio=0.014069 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 1, 0, 0, 0, 0, 1, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8190, 8192, 7290, 8180, 8192, 8188, 8190] dup_rows_per_dst=[0, 4, 0, 8, 590, 314, 4, 2] combine_rows_per_src=[8192, 8190, 8192, 7290, 8180, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57336 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 1, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 8180, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 20, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57302 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 1, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8166, 8176, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 22, 0, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8166, 8176, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65386 payload_rows_remote=57194 dup_rows=150 dup_rows_remote=150 dedup_ratio=0.002289 router_tokens_per_expert=[0, 0, 0, 73, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8170, 8192, 8148, 8108, 8192, 8192, 8192] dup_rows_per_dst=[146, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8170, 8192, 8148, 8108, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65108 payload_rows_remote=56918 dup_rows=428 dup_rows_remote=428 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 187, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 36, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 8192, 8192, 7918, 8044, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 384, 0, 44, 0, 0] combine_rows_per_src=[8188, 8190, 8192, 8192, 7918, 8044, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231112704 dispatch_meta_bytes=229376 combine_payload_bytes=231112704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231342080 ingress_bytes=249987072 link_bytes=249987072 token_bytes=462454784 weight_movement_bytes=18874368 total_nvlink_bytes=481329152 weight_to_token_ratio=0.040813 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.395253123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229424 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077680 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925936 weight_movement_bytes=18874368 total_nvlink_bytes=488800304 weight_to_token_ratio=0.040165 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=17280 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229376 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234938368 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469647360 weight_movement_bytes=18874368 total_nvlink_bytes=488521728 weight_to_token_ratio=0.040188 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234266624 dispatch_meta_bytes=229376 combine_payload_bytes=234266624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234496000 ingress_bytes=253140992 link_bytes=253140992 token_bytes=468762624 weight_movement_bytes=18874368 total_nvlink_bytes=487636992 weight_to_token_ratio=0.040264 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425472432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57286 dup_rows=62 dup_rows_remote=54 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 3, 3628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8152, 8178, 8192, 8192] dup_rows_per_dst=[8, 10, 0, 14, 0, 30, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8152, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=233136128 dispatch_meta_bytes=229384 combine_payload_bytes=233136128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233365512 ingress_bytes=252010496 link_bytes=252010496 token_bytes=466501640 weight_movement_bytes=18874368 total_nvlink_bytes=485376008 weight_to_token_ratio=0.040459 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414640576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65316 payload_rows_remote=57124 dup_rows=220 dup_rows_remote=14 dedup_ratio=0.003357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 0, 0, 0, 0, 0, 0, 0, 3, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8148, 8016, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 206, 0, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8148, 8016, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=229360 combine_payload_bytes=234643456 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234872816 ingress_bytes=272392192 link_bytes=272392192 token_bytes=469516272 weight_movement_bytes=37748736 total_nvlink_bytes=507265008 weight_to_token_ratio=0.080399 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609927958e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=233979904 dispatch_meta_bytes=228552 combine_payload_bytes=233979904 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=234208456 ingress_bytes=347226112 link_bytes=347226112 token_bytes=468188360 weight_movement_bytes=113246208 total_nvlink_bytes=581434568 weight_to_token_ratio=0.241882 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.326949759e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=14 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8188, 8190, 8190, 8188, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 2, 2, 2, 4, 2, 2, 2] combine_rows_per_src=[8188, 8190, 8190, 8188, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64366 payload_rows_remote=56174 dup_rows=1170 dup_rows_remote=1170 dedup_ratio=0.017853 router_tokens_per_expert=[0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 578, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7022, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1156, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[7022, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57342 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229368 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044856 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860344 weight_movement_bytes=18874368 total_nvlink_bytes=488734712 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61810 payload_rows_remote=53618 dup_rows=3726 dup_rows_remote=622 dedup_ratio=0.056854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3822, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2505, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7648, 8192, 8192, 8192, 8192, 5010] dup_rows_per_dst=[0, 0, 0, 0, 3104, 0, 622, 0] combine_rows_per_src=[8192, 8192, 7648, 8192, 8192, 8192, 8192, 5010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=230088704 dispatch_meta_bytes=229376 combine_payload_bytes=230088704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=230318080 ingress_bytes=305586176 link_bytes=305586176 token_bytes=460406784 weight_movement_bytes=75497472 total_nvlink_bytes=535904256 weight_to_token_ratio=0.163980 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.927976381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=14 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8188, 8188, 8190, 8188] dup_rows_per_dst=[4, 2, 0, 8, 0, 0, 2, 2] combine_rows_per_src=[8192, 8188, 8192, 8192, 8188, 8188, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229384 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102216 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975048 weight_movement_bytes=18874368 total_nvlink_bytes=488849416 weight_to_token_ratio=0.040160 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219619328 dispatch_meta_bytes=216960 combine_payload_bytes=219619328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219836288 ingress_bytes=238493696 link_bytes=238493696 token_bytes=439455616 weight_movement_bytes=18874368 total_nvlink_bytes=458329984 weight_to_token_ratio=0.042949 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.285129249e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229360 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235036656 ingress_bytes=272556032 link_bytes=272556032 token_bytes=469843952 weight_movement_bytes=37748736 total_nvlink_bytes=507592688 weight_to_token_ratio=0.080343 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611497793e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59082 payload_rows_remote=50890 dup_rows=6454 dup_rows_remote=154 dedup_ratio=0.098480 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4095, 0, 0, 2, 0, 0, 0, 1831, 1, 0, 0, 0] payload_rows_per_dst=[8190, 8188, 5686, 8192, 6884, 8190, 5560, 8192] dup_rows_per_dst=[0, 0, 2, 2, 0, 0, 150, 6300] combine_rows_per_src=[8190, 8188, 5686, 8192, 6884, 8190, 5560, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61964 payload_rows_remote=53772 dup_rows=3572 dup_rows_remote=726 dedup_ratio=0.054504 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1421, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 3059, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6692, 8192, 8192, 6120] dup_rows_per_dst=[0, 0, 2846, 0, 0, 0, 724, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 6692, 8192, 8192, 6120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57422 payload_rows_remote=49230 dup_rows=8114 dup_rows_remote=8114 dedup_ratio=0.123810 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 218, 8192, 8076, 8192, 8192, 8170] dup_rows_per_dst=[0, 0, 0, 0, 1328, 0, 6, 6780] combine_rows_per_src=[8192, 8190, 218, 8192, 8076, 8192, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 0, 0, 0, 6, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57852 payload_rows_remote=49660 dup_rows=7684 dup_rows_remote=7684 dedup_ratio=0.117249 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 7466, 8192, 8192, 1236] dup_rows_per_dst=[0, 0, 7486, 0, 0, 0, 198, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 7466, 8192, 8192, 1236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=208445440 dispatch_meta_bytes=204176 combine_payload_bytes=208445440 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=208649616 ingress_bytes=283942912 link_bytes=283942912 token_bytes=417095056 weight_movement_bytes=75497472 total_nvlink_bytes=492592528 weight_to_token_ratio=0.181008 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.720601274e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=220250112 dispatch_meta_bytes=217992 combine_payload_bytes=220250112 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=220468104 ingress_bytes=295747584 link_bytes=295747584 token_bytes=440718216 weight_movement_bytes=75497472 total_nvlink_bytes=516215688 weight_to_token_ratio=0.171306 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.833707833e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=201646080 dispatch_meta_bytes=229376 combine_payload_bytes=201646080 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=201875456 ingress_bytes=239394816 link_bytes=239394816 token_bytes=403521536 weight_movement_bytes=37748736 total_nvlink_bytes=441270272 weight_to_token_ratio=0.093548 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293763337e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57226 payload_rows_remote=49034 dup_rows=8310 dup_rows_remote=8306 dedup_ratio=0.126801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3941, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4096, 0, 0, 3, 0, 0, 0, 1780, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 4516, 8192, 8190, 3562] dup_rows_per_dst=[4, 6, 7876, 0, 0, 2, 420, 2] combine_rows_per_src=[8192, 8192, 8192, 8190, 4516, 8192, 8190, 3562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58978 payload_rows_remote=50786 dup_rows=6558 dup_rows_remote=6558 dedup_ratio=0.100067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2921, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4504, 8192, 7672, 8192, 8192, 5842] dup_rows_per_dst=[0, 0, 1254, 0, 4946, 0, 358, 0] combine_rows_per_src=[8192, 8192, 4504, 8192, 7672, 8192, 8192, 5842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=203407360 dispatch_meta_bytes=229376 combine_payload_bytes=203407360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203636736 ingress_bytes=241156096 link_bytes=241156096 token_bytes=407044096 weight_movement_bytes=37748736 total_nvlink_bytes=444792832 weight_to_token_ratio=0.092739 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310639056e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=200843264 dispatch_meta_bytes=229360 combine_payload_bytes=200843264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201072624 ingress_bytes=219717632 link_bytes=219717632 token_bytes=401915888 weight_movement_bytes=18874368 total_nvlink_bytes=420790256 weight_to_token_ratio=0.046961 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.105226242e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=208019456 dispatch_meta_bytes=229376 combine_payload_bytes=208019456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208248832 ingress_bytes=226893824 link_bytes=226893824 token_bytes=416268288 weight_movement_bytes=18874368 total_nvlink_bytes=435142656 weight_to_token_ratio=0.045342 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.173984983e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57304 dup_rows=46 dup_rows_remote=16 dedup_ratio=0.000702 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 4095, 1, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 4091, 0, 0] payload_rows_per_dst=[8190, 8190, 8170, 8180, 8192, 8192, 8190, 8186] dup_rows_per_dst=[2, 6, 4, 0, 2, 0, 2, 30] combine_rows_per_src=[8190, 8190, 8170, 8180, 8192, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65340 payload_rows_remote=57148 dup_rows=196 dup_rows_remote=196 dedup_ratio=0.002991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 19, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8100, 8090, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 178, 0, 18, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8100, 8090, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229280 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234946464 ingress_bytes=272465920 link_bytes=272465920 token_bytes=469663648 weight_movement_bytes=37748736 total_nvlink_bytes=507412384 weight_to_token_ratio=0.080374 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610634384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=20 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 3, 0, 4096, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4083, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8172] dup_rows_per_dst=[0, 2, 8, 12, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61806 payload_rows_remote=54928 dup_rows=3730 dup_rows_remote=3730 dedup_ratio=0.056915 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1907, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5776, 6878, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 3722] combine_rows_per_src=[8192, 8192, 8192, 5776, 6878, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4094, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8190, 8192, 8192, 8190] dup_rows_per_dst=[0, 2, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8190, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=229376 combine_payload_bytes=234078208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234307584 ingress_bytes=252952576 link_bytes=252952576 token_bytes=468385792 weight_movement_bytes=18874368 total_nvlink_bytes=487260160 weight_to_token_ratio=0.040297 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423667123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=6 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 6, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229368 combine_payload_bytes=234790912 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235020280 ingress_bytes=272539648 link_bytes=272539648 token_bytes=469811192 weight_movement_bytes=37748736 total_nvlink_bytes=507559928 weight_to_token_ratio=0.080349 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611340809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=224985088 dispatch_meta_bytes=234632 combine_payload_bytes=224985088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225219720 ingress_bytes=281608192 link_bytes=281608192 token_bytes=450204808 weight_movement_bytes=56623104 total_nvlink_bytes=506827912 weight_to_token_ratio=0.125772 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.698231136e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229368 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235077624 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925880 weight_movement_bytes=18874368 total_nvlink_bytes=488800248 weight_to_token_ratio=0.040165 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59824 payload_rows_remote=51634 dup_rows=5712 dup_rows_remote=20 dedup_ratio=0.087158 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3044, 0, 0, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2849, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 8186, 8192, 4594, 6090, 8190, 8192, 8190] dup_rows_per_dst=[0, 2, 0, 0, 2, 4, 12, 5692] combine_rows_per_src=[8190, 8186, 8192, 4594, 6090, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60714 payload_rows_remote=52522 dup_rows=4822 dup_rows_remote=4822 dedup_ratio=0.073578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 746, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7232, 8092, 8192, 8192, 4430] dup_rows_per_dst=[0, 0, 0, 0, 4402, 0, 2, 418] combine_rows_per_src=[8192, 8192, 8192, 7232, 8092, 8192, 8192, 4430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57078 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 419, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7944, 8176, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 10, 254] combine_rows_per_src=[8192, 8190, 8192, 7944, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=211492864 dispatch_meta_bytes=206616 combine_payload_bytes=211492864 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=211699480 ingress_bytes=268115968 link_bytes=268115968 token_bytes=423192344 weight_movement_bytes=56623104 total_nvlink_bytes=479815448 weight_to_token_ratio=0.133800 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568955284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57802 payload_rows_remote=49620 dup_rows=7734 dup_rows_remote=7734 dedup_ratio=0.118011 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 667, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8192, 7858, 7660, 8192, 8192, 1334] dup_rows_per_dst=[0, 0, 0, 4868, 2760, 10, 96, 0] combine_rows_per_src=[8192, 8182, 8192, 7858, 7660, 8192, 8192, 1334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=215130112 dispatch_meta_bytes=229376 combine_payload_bytes=215130112 prefetch_bytes=0 reduce_bytes=0 egress_bytes=215359488 ingress_bytes=215130112 link_bytes=215359488 token_bytes=430489600 weight_movement_bytes=0 total_nvlink_bytes=430489600 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.063468651e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233791488 dispatch_meta_bytes=229376 combine_payload_bytes=233791488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234020864 ingress_bytes=252665856 link_bytes=252665856 token_bytes=467812352 weight_movement_bytes=18874368 total_nvlink_bytes=486686720 weight_to_token_ratio=0.040346 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420919913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58340 payload_rows_remote=50148 dup_rows=7196 dup_rows_remote=7196 dedup_ratio=0.109802 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 1312, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 6722, 8180, 8050, 2624] dup_rows_per_dst=[0, 20, 0, 7102, 72, 0, 2, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 6722, 8180, 8050, 2624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62884 payload_rows_remote=56308 dup_rows=2652 dup_rows_remote=2652 dedup_ratio=0.040466 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1332, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6576, 7156, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 6, 2646] combine_rows_per_src=[8192, 8192, 8192, 6576, 7156, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=203243520 dispatch_meta_bytes=229416 combine_payload_bytes=203243520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203472936 ingress_bytes=240992256 link_bytes=240992256 token_bytes=406716456 weight_movement_bytes=37748736 total_nvlink_bytes=444465192 weight_to_token_ratio=0.092813 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.309069221e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235110400 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991424 weight_movement_bytes=94371840 total_nvlink_bytes=564363264 weight_to_token_ratio=0.200795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=205406208 dispatch_meta_bytes=229376 combine_payload_bytes=205406208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205635584 ingress_bytes=224280576 link_bytes=224280576 token_bytes=411041792 weight_movement_bytes=18874368 total_nvlink_bytes=429916160 weight_to_token_ratio=0.045918 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.148946126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=230637568 dispatch_meta_bytes=235840 combine_payload_bytes=230637568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230873408 ingress_bytes=249511936 link_bytes=249511936 token_bytes=461510976 weight_movement_bytes=18874368 total_nvlink_bytes=480385344 weight_to_token_ratio=0.040897 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390700603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=6 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8190, 8184, 8190, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 2, 2, 10] combine_rows_per_src=[8192, 8192, 8190, 8190, 8184, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229336 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044824 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860312 weight_movement_bytes=18874368 total_nvlink_bytes=488734680 weight_to_token_ratio=0.040170 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59252 payload_rows_remote=51060 dup_rows=6284 dup_rows_remote=6128 dedup_ratio=0.095886 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1719, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3440, 8192, 6660, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 156, 6128] combine_rows_per_src=[8192, 8192, 3440, 8192, 6660, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4044, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8156, 8122, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 106] combine_rows_per_src=[8192, 8192, 8192, 8156, 8122, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65364 payload_rows_remote=57172 dup_rows=172 dup_rows_remote=172 dedup_ratio=0.002625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 86, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8020, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 172] combine_rows_per_src=[8192, 8020, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65404 payload_rows_remote=57212 dup_rows=132 dup_rows_remote=132 dedup_ratio=0.002014 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8062] dup_rows_per_dst=[0, 0, 2, 42, 88, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=209141760 dispatch_meta_bytes=228752 combine_payload_bytes=209141760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209370512 ingress_bytes=228016128 link_bytes=228016128 token_bytes=418512272 weight_movement_bytes=18874368 total_nvlink_bytes=437386640 weight_to_token_ratio=0.045099 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.184738348e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57334 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8192, 8190, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229376 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234676224 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469123072 weight_movement_bytes=18874368 total_nvlink_bytes=487997440 weight_to_token_ratio=0.040233 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234176512 dispatch_meta_bytes=229376 combine_payload_bytes=234176512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234405888 ingress_bytes=253050880 link_bytes=253050880 token_bytes=468582400 weight_movement_bytes=18874368 total_nvlink_bytes=487456768 weight_to_token_ratio=0.040280 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424609023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=6 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 28, 1, 0, 4095, 0, 1, 1, 0] payload_rows_per_dst=[8190, 8136, 8190, 8192, 8190, 8190, 8186, 8192] dup_rows_per_dst=[0, 0, 2, 0, 2, 0, 2, 64] combine_rows_per_src=[8190, 8136, 8190, 8192, 8190, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234340352 dispatch_meta_bytes=229376 combine_payload_bytes=234340352 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234569728 ingress_bytes=253214720 link_bytes=253214720 token_bytes=468910080 weight_movement_bytes=18874368 total_nvlink_bytes=487784448 weight_to_token_ratio=0.040252 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426178858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229384 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069448 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909512 weight_movement_bytes=18874368 total_nvlink_bytes=488783880 weight_to_token_ratio=0.040166 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229120 combine_payload_bytes=234594304 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234823424 ingress_bytes=328966144 link_bytes=328966144 token_bytes=469417728 weight_movement_bytes=94371840 total_nvlink_bytes=563789568 weight_to_token_ratio=0.201040 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.151991730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8142, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 50] combine_rows_per_src=[8192, 8142, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62524 payload_rows_remote=54332 dup_rows=3012 dup_rows_remote=2086 dedup_ratio=0.045959 router_tokens_per_expert=[2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 4096] payload_rows_per_dst=[5180, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 926, 1804, 0, 282] combine_rows_per_src=[5180, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64250 payload_rows_remote=57286 dup_rows=1286 dup_rows_remote=1286 dedup_ratio=0.019623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 638, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8188, 6964, 8140, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 4, 1274] combine_rows_per_src=[8188, 6964, 8140, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63492 payload_rows_remote=55300 dup_rows=2044 dup_rows_remote=2038 dedup_ratio=0.031189 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 114, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8052, 8190, 8192, 8182, 8192, 8192, 6300] dup_rows_per_dst=[6, 1844, 54, 0, 0, 2, 0, 138] combine_rows_per_src=[8192, 8052, 8190, 8192, 8182, 8192, 8192, 6300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229376 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905600 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581824 weight_movement_bytes=18874368 total_nvlink_bytes=488456192 weight_to_token_ratio=0.040194 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=222543872 dispatch_meta_bytes=225672 combine_payload_bytes=222543872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222769544 ingress_bytes=241418240 link_bytes=241418240 token_bytes=445313416 weight_movement_bytes=18874368 total_nvlink_bytes=464187784 weight_to_token_ratio=0.042384 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.313150790e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 22] combine_rows_per_src=[8192, 8168, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=234288 combine_payload_bytes=234643456 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234877744 ingress_bytes=291266560 link_bytes=291266560 token_bytes=469521200 weight_movement_bytes=56623104 total_nvlink_bytes=526144304 weight_to_token_ratio=0.120598 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790772866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56656 payload_rows_remote=48464 dup_rows=8880 dup_rows_remote=8792 dedup_ratio=0.135498 router_tokens_per_expert=[2606, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 0, 0, 4095] payload_rows_per_dst=[5214, 8172, 8190, 8190, 2318, 8190, 8190, 8192] dup_rows_per_dst=[0, 0, 2, 2, 10, 2890, 5888, 88] combine_rows_per_src=[5214, 8172, 8190, 8190, 2318, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=226508800 dispatch_meta_bytes=229352 combine_payload_bytes=226508800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226738152 ingress_bytes=245383168 link_bytes=245383168 token_bytes=453246952 weight_movement_bytes=18874368 total_nvlink_bytes=472121320 weight_to_token_ratio=0.041643 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351140780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=198508544 dispatch_meta_bytes=229024 combine_payload_bytes=198508544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=198737568 ingress_bytes=236257280 link_bytes=236257280 token_bytes=397246112 weight_movement_bytes=37748736 total_nvlink_bytes=434994848 weight_to_token_ratio=0.095026 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.263701011e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64986 payload_rows_remote=56794 dup_rows=550 dup_rows_remote=22 dedup_ratio=0.008392 router_tokens_per_expert=[3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7642, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 528, 0, 20] combine_rows_per_src=[7642, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62636 payload_rows_remote=54444 dup_rows=2900 dup_rows_remote=2900 dedup_ratio=0.044250 router_tokens_per_expert=[1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 5292, 8192, 8192] dup_rows_per_dst=[2576, 0, 0, 0, 210, 0, 0, 114] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 5292, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60112 payload_rows_remote=55946 dup_rows=5424 dup_rows_remote=4082 dedup_ratio=0.082764 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6794, 8192, 4166, 8192, 8192, 8192] dup_rows_per_dst=[0, 510, 3572, 0, 1342, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6794, 8192, 4166, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62072 payload_rows_remote=53880 dup_rows=3464 dup_rows_remote=3458 dedup_ratio=0.052856 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1748, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5394, 7526, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 6, 3458] combine_rows_per_src=[8192, 8192, 8192, 5394, 7526, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61912 payload_rows_remote=53720 dup_rows=3624 dup_rows_remote=3624 dedup_ratio=0.055298 router_tokens_per_expert=[887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 2, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 4982, 7778, 8192] dup_rows_per_dst=[1774, 0, 0, 0, 780, 0, 0, 1070] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 4982, 7778, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57402 payload_rows_remote=49210 dup_rows=8134 dup_rows_remote=5010 dedup_ratio=0.124115 router_tokens_per_expert=[1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1596, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 2720, 5530] dup_rows_per_dst=[3124, 16, 0, 0, 3192, 946, 0, 856] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 2720, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232628224 dispatch_meta_bytes=227264 combine_payload_bytes=232628224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232855488 ingress_bytes=270376960 link_bytes=270376960 token_bytes=465483712 weight_movement_bytes=37748736 total_nvlink_bytes=503232448 weight_to_token_ratio=0.081096 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.590618997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=223002624 dispatch_meta_bytes=229376 combine_payload_bytes=223002624 prefetch_bytes=0 reduce_bytes=0 egress_bytes=223232000 ingress_bytes=223002624 link_bytes=223232000 token_bytes=446234624 weight_movement_bytes=0 total_nvlink_bytes=446234624 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.138899187e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=229154816 dispatch_meta_bytes=240112 combine_payload_bytes=229154816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229394928 ingress_bytes=266903552 link_bytes=266903552 token_bytes=458549744 weight_movement_bytes=37748736 total_nvlink_bytes=496298480 weight_to_token_ratio=0.082322 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.557338511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60906 payload_rows_remote=52714 dup_rows=4630 dup_rows_remote=4630 dedup_ratio=0.070648 router_tokens_per_expert=[2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7258, 8192, 8192, 8192, 8192, 4496, 8192, 8192] dup_rows_per_dst=[3182, 4, 0, 0, 406, 924, 0, 114] combine_rows_per_src=[7258, 8192, 8192, 8192, 8192, 4496, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=220692480 dispatch_meta_bytes=229352 combine_payload_bytes=220692480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220921832 ingress_bytes=239566848 link_bytes=239566848 token_bytes=441614312 weight_movement_bytes=18874368 total_nvlink_bytes=460488680 weight_to_token_ratio=0.042739 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.295411663e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55962 payload_rows_remote=47770 dup_rows=9574 dup_rows_remote=9572 dedup_ratio=0.146088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1, 0, 0, 0, 4095, 0, 4094, 4095, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 3328, 3492, 8188, 8190, 8192, 8190, 8192] dup_rows_per_dst=[0, 146, 0, 2, 9422, 0, 2, 2] combine_rows_per_src=[8190, 3328, 3492, 8188, 8190, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=220037120 dispatch_meta_bytes=229376 combine_payload_bytes=220037120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220266496 ingress_bytes=238911488 link_bytes=238911488 token_bytes=440303616 weight_movement_bytes=18874368 total_nvlink_bytes=459177984 weight_to_token_ratio=0.042867 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.289132326e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=201564160 dispatch_meta_bytes=216880 combine_payload_bytes=201564160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201781040 ingress_bytes=220438528 link_bytes=220438528 token_bytes=403345200 weight_movement_bytes=18874368 total_nvlink_bytes=422219568 weight_to_token_ratio=0.046795 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112133513e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=215916544 dispatch_meta_bytes=229376 combine_payload_bytes=215916544 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=216145920 ingress_bytes=253665280 link_bytes=253665280 token_bytes=432062464 weight_movement_bytes=37748736 total_nvlink_bytes=469811200 weight_to_token_ratio=0.087369 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430495902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=195665920 dispatch_meta_bytes=229368 combine_payload_bytes=195665920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195895288 ingress_bytes=214540288 link_bytes=214540288 token_bytes=391561208 weight_movement_bytes=18874368 total_nvlink_bytes=410435576 weight_to_token_ratio=0.048203 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.055619479e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57534 payload_rows_remote=49342 dup_rows=8002 dup_rows_remote=8002 dedup_ratio=0.122101 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 634, 0, 0, 7368, 0, 0, 0] combine_rows_per_src=[8192, 8192, 190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62486 payload_rows_remote=54294 dup_rows=3050 dup_rows_remote=686 dedup_ratio=0.046539 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5142, 8192, 8192, 8192] dup_rows_per_dst=[0, 686, 2364, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5142, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61336 payload_rows_remote=53144 dup_rows=4200 dup_rows_remote=3594 dedup_ratio=0.064087 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 3992, 8192, 8192, 8192] dup_rows_per_dst=[0, 606, 3590, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 3992, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57794 payload_rows_remote=49602 dup_rows=7742 dup_rows_remote=7740 dedup_ratio=0.118134 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 3451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 454, 8192, 8192, 8190] dup_rows_per_dst=[2, 840, 6900, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 454, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=202104832 dispatch_meta_bytes=229376 combine_payload_bytes=202104832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202334208 ingress_bytes=220979200 link_bytes=220979200 token_bytes=404439040 weight_movement_bytes=18874368 total_nvlink_bytes=423313408 weight_to_token_ratio=0.046668 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.117313966e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62272 payload_rows_remote=54080 dup_rows=3264 dup_rows_remote=3264 dedup_ratio=0.049805 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4928, 8192, 8192, 8192] dup_rows_per_dst=[0, 264, 3000, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 4928, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=222388224 dispatch_meta_bytes=219920 combine_payload_bytes=222388224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=222608144 ingress_bytes=279011328 link_bytes=279011328 token_bytes=444996368 weight_movement_bytes=56623104 total_nvlink_bytes=501619472 weight_to_token_ratio=0.127244 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.673349263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57334 dup_rows=12 dup_rows_remote=10 dedup_ratio=0.000183 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 8192, 8190, 8192, 8192, 8190, 8190] dup_rows_per_dst=[4, 0, 0, 2, 0, 2, 2, 2] combine_rows_per_src=[8188, 8190, 8192, 8190, 8192, 8192, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=217677824 dispatch_meta_bytes=226952 combine_payload_bytes=217677824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217904776 ingress_bytes=236552192 link_bytes=236552192 token_bytes=435582600 weight_movement_bytes=18874368 total_nvlink_bytes=454456968 weight_to_token_ratio=0.043331 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.266526713e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=203169792 dispatch_meta_bytes=229368 combine_payload_bytes=203169792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203399160 ingress_bytes=222044160 link_bytes=222044160 token_bytes=406568952 weight_movement_bytes=18874368 total_nvlink_bytes=425443320 weight_to_token_ratio=0.046424 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.127517889e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=221511680 dispatch_meta_bytes=229376 combine_payload_bytes=221511680 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=221741056 ingress_bytes=259260416 link_bytes=259260416 token_bytes=443252736 weight_movement_bytes=37748736 total_nvlink_bytes=481001472 weight_to_token_ratio=0.085163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.484105742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=272588800 link_bytes=272588800 token_bytes=469909504 weight_movement_bytes=37748736 total_nvlink_bytes=507658240 weight_to_token_ratio=0.080332 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611811760e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63400 payload_rows_remote=55332 dup_rows=2136 dup_rows_remote=438 dedup_ratio=0.032593 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 9, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8068, 8192, 6180, 8192] dup_rows_per_dst=[334, 0, 0, 0, 1698, 0, 104, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8068, 8192, 6180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 159, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8024, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 168] combine_rows_per_src=[8192, 8024, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58402 payload_rows_remote=50210 dup_rows=7134 dup_rows_remote=7134 dedup_ratio=0.108856 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 4, 1, 0, 0, 0, 0, 1449, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6362, 8190, 8188, 8190, 2908, 8184, 8188, 8192] dup_rows_per_dst=[8, 0, 2, 4, 2, 0, 7118, 0] combine_rows_per_src=[6362, 8190, 8188, 8190, 2908, 8184, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=226639872 dispatch_meta_bytes=223080 combine_payload_bytes=226639872 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=226862952 ingress_bytes=245514240 link_bytes=245514240 token_bytes=453502824 weight_movement_bytes=18874368 total_nvlink_bytes=472377192 weight_to_token_ratio=0.041619 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352396647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=205660160 dispatch_meta_bytes=229376 combine_payload_bytes=205660160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205889536 ingress_bytes=224534528 link_bytes=224534528 token_bytes=411549696 weight_movement_bytes=18874368 total_nvlink_bytes=430424064 weight_to_token_ratio=0.045862 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.151379369e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64892 payload_rows_remote=56700 dup_rows=644 dup_rows_remote=638 dedup_ratio=0.009827 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 178, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7708, 8192, 8032, 8192] dup_rows_per_dst=[6, 0, 0, 0, 160, 6, 472, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7708, 8192, 8032, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63202 payload_rows_remote=55010 dup_rows=2334 dup_rows_remote=2334 dedup_ratio=0.035614 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 3, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 5858, 8192] dup_rows_per_dst=[4, 0, 0, 0, 2330, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 5858, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62294 payload_rows_remote=54102 dup_rows=3242 dup_rows_remote=3242 dedup_ratio=0.049469 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 4952, 8192] dup_rows_per_dst=[544, 0, 2, 0, 2694, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 4952, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62340 payload_rows_remote=54148 dup_rows=3196 dup_rows_remote=3196 dedup_ratio=0.048767 router_tokens_per_expert=[3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 0] payload_rows_per_dst=[6850, 8192, 8192, 8192, 8192, 8192, 8192, 6338] dup_rows_per_dst=[0, 0, 0, 3196, 0, 0, 0, 0] combine_rows_per_src=[6850, 8192, 8192, 8192, 8192, 8192, 8192, 6338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=232243200 dispatch_meta_bytes=229352 combine_payload_bytes=232243200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232472552 ingress_bytes=251117568 link_bytes=251117568 token_bytes=464715752 weight_movement_bytes=18874368 total_nvlink_bytes=483590120 weight_to_token_ratio=0.040615 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406084979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62354 payload_rows_remote=54162 dup_rows=3182 dup_rows_remote=3182 dedup_ratio=0.048553 router_tokens_per_expert=[2505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 4096] payload_rows_per_dst=[5010, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 348, 2716, 0, 118] combine_rows_per_src=[5010, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61282 payload_rows_remote=53090 dup_rows=4254 dup_rows_remote=3704 dedup_ratio=0.064911 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 33, 0, 0, 0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8172, 8192, 3960, 8192] dup_rows_per_dst=[550, 2, 2, 0, 3700, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8172, 8192, 3960, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60308 payload_rows_remote=52116 dup_rows=5228 dup_rows_remote=6 dedup_ratio=0.079773 router_tokens_per_expert=[3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2610, 0, 0, 0] payload_rows_per_dst=[7016, 8192, 8190, 4144, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 5222] combine_rows_per_src=[7016, 8192, 8190, 4144, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225320960 dispatch_meta_bytes=229376 combine_payload_bytes=225320960 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=225550336 ingress_bytes=263069696 link_bytes=263069696 token_bytes=450871296 weight_movement_bytes=37748736 total_nvlink_bytes=488620032 weight_to_token_ratio=0.083724 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.520604389e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63526 payload_rows_remote=55334 dup_rows=2010 dup_rows_remote=2010 dedup_ratio=0.030670 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 32, 0, 0, 0, 0, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 6184, 8192] dup_rows_per_dst=[140, 0, 0, 0, 1870, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 6184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=221601792 dispatch_meta_bytes=229376 combine_payload_bytes=221601792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221831168 ingress_bytes=240476160 link_bytes=240476160 token_bytes=443432960 weight_movement_bytes=18874368 total_nvlink_bytes=462307328 weight_to_token_ratio=0.042564 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.304124243e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=221790208 dispatch_meta_bytes=229376 combine_payload_bytes=221790208 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=222019584 ingress_bytes=259538944 link_bytes=259538944 token_bytes=443809792 weight_movement_bytes=37748736 total_nvlink_bytes=481558528 weight_to_token_ratio=0.085056 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.486774460e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=221847552 dispatch_meta_bytes=229376 combine_payload_bytes=221847552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222076928 ingress_bytes=278470656 link_bytes=278470656 token_bytes=443924480 weight_movement_bytes=56623104 total_nvlink_bytes=500547584 weight_to_token_ratio=0.127551 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.668168810e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=217456640 dispatch_meta_bytes=227176 combine_payload_bytes=217456640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217683816 ingress_bytes=236331008 link_bytes=236331008 token_bytes=435140456 weight_movement_bytes=18874368 total_nvlink_bytes=454014824 weight_to_token_ratio=0.043375 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264407437e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=213467136 dispatch_meta_bytes=208488 combine_payload_bytes=213467136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213675624 ingress_bytes=232341504 link_bytes=232341504 token_bytes=427142760 weight_movement_bytes=18874368 total_nvlink_bytes=446017128 weight_to_token_ratio=0.044187 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226181972e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=226648064 dispatch_meta_bytes=229376 combine_payload_bytes=226648064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226877440 ingress_bytes=245522432 link_bytes=245522432 token_bytes=453525504 weight_movement_bytes=18874368 total_nvlink_bytes=472399872 weight_to_token_ratio=0.041617 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352475139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60936 payload_rows_remote=52744 dup_rows=4600 dup_rows_remote=4600 dedup_ratio=0.070190 router_tokens_per_expert=[3483, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2267, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 0] payload_rows_per_dst=[6974, 8192, 8190, 6108, 8192, 8192, 8192, 6896] dup_rows_per_dst=[0, 0, 0, 2008, 0, 0, 0, 2592] combine_rows_per_src=[6974, 8192, 8190, 6108, 8192, 8192, 8192, 6896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59144 payload_rows_remote=50952 dup_rows=6392 dup_rows_remote=6392 dedup_ratio=0.097534 router_tokens_per_expert=[1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3131, 0, 0, 0] payload_rows_per_dst=[3730, 8192, 8192, 8192, 8192, 8192, 8192, 6262] dup_rows_per_dst=[0, 0, 0, 6392, 0, 0, 0, 0] combine_rows_per_src=[3730, 8192, 8192, 8192, 8192, 8192, 8192, 6262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=216039424 dispatch_meta_bytes=229376 combine_payload_bytes=216039424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216268800 ingress_bytes=234913792 link_bytes=234913792 token_bytes=432308224 weight_movement_bytes=18874368 total_nvlink_bytes=451182592 weight_to_token_ratio=0.043660 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250828370e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60896 payload_rows_remote=52704 dup_rows=4640 dup_rows_remote=4640 dedup_ratio=0.070801 router_tokens_per_expert=[2302, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3257, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0] payload_rows_per_dst=[8108, 8192, 8110, 6654, 8192, 8192, 8192, 5256] dup_rows_per_dst=[4414, 0, 0, 144, 80, 2, 0, 0] combine_rows_per_src=[8108, 8192, 8110, 6654, 8192, 8192, 8192, 5256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60210 payload_rows_remote=52018 dup_rows=5326 dup_rows_remote=5326 dedup_ratio=0.081268 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2866, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1178, 0, 0, 4148, 0, 0, 0] combine_rows_per_src=[8192, 8192, 2866, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65324 payload_rows_remote=57132 dup_rows=212 dup_rows_remote=210 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 7998, 8186, 8190, 8190, 8188, 8190, 8192] dup_rows_per_dst=[2, 0, 4, 10, 2, 6, 186, 2] combine_rows_per_src=[8190, 7998, 8186, 8190, 8190, 8188, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57532 payload_rows_remote=49340 dup_rows=8004 dup_rows_remote=90 dedup_ratio=0.122131 router_tokens_per_expert=[3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5236, 8110, 8184, 8192, 3234] dup_rows_per_dst=[7914, 0, 90, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 5236, 8110, 8184, 8192, 3234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=208699392 dispatch_meta_bytes=229376 combine_payload_bytes=208699392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=208928768 ingress_bytes=246448128 link_bytes=246448128 token_bytes=417628160 weight_movement_bytes=37748736 total_nvlink_bytes=455376896 weight_to_token_ratio=0.090388 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.361344702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61416 payload_rows_remote=53224 dup_rows=4120 dup_rows_remote=6 dedup_ratio=0.062866 router_tokens_per_expert=[3064, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 0, 0] payload_rows_per_dst=[6128, 8192, 8188, 8192, 8192, 8192, 8192, 6140] dup_rows_per_dst=[2, 0, 0, 4114, 4, 0, 0, 0] combine_rows_per_src=[6128, 8192, 8188, 8192, 8192, 8192, 8192, 6140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=215875584 dispatch_meta_bytes=229376 combine_payload_bytes=215875584 prefetch_bytes=0 reduce_bytes=0 egress_bytes=216104960 ingress_bytes=215875584 link_bytes=216104960 token_bytes=431980544 weight_movement_bytes=0 total_nvlink_bytes=431980544 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.070611396e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=202096640 dispatch_meta_bytes=197720 combine_payload_bytes=202096640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202294360 ingress_bytes=220971008 link_bytes=220971008 token_bytes=404391000 weight_movement_bytes=18874368 total_nvlink_bytes=423265368 weight_to_token_ratio=0.046674 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.117235474e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=213065728 dispatch_meta_bytes=229376 combine_payload_bytes=213065728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213295104 ingress_bytes=231940096 link_bytes=231940096 token_bytes=426360832 weight_movement_bytes=18874368 total_nvlink_bytes=445235200 weight_to_token_ratio=0.044269 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222335878e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234012672 dispatch_meta_bytes=229368 combine_payload_bytes=234012672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234242040 ingress_bytes=271761408 link_bytes=271761408 token_bytes=468254712 weight_movement_bytes=37748736 total_nvlink_bytes=506003448 weight_to_token_ratio=0.080616 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603884096e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8170, 8192, 8192, 8192] dup_rows_per_dst=[0, 20, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=218005504 dispatch_meta_bytes=212920 combine_payload_bytes=218005504 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=218218424 ingress_bytes=255754240 link_bytes=255754240 token_bytes=436223928 weight_movement_bytes=37748736 total_nvlink_bytes=473972664 weight_to_token_ratio=0.086535 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450511289e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235020288 ingress_bytes=253665280 link_bytes=253665280 token_bytes=469811200 weight_movement_bytes=18874368 total_nvlink_bytes=488685568 weight_to_token_ratio=0.040174 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430495902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8172] dup_rows_per_dst=[0, 0, 0, 0, 20, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57206 dup_rows=138 dup_rows_remote=36 dedup_ratio=0.002106 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8182, 8190, 8070, 8192] dup_rows_per_dst=[0, 102, 2, 0, 34, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8182, 8190, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8186] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64658 payload_rows_remote=56544 dup_rows=878 dup_rows_remote=96 dedup_ratio=0.013397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 2, 0, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4036, 0, 1] payload_rows_per_dst=[8188, 8134, 8190, 8192, 7460, 8192, 8188, 8114] dup_rows_per_dst=[2, 20, 0, 4, 66, 0, 4, 782] combine_rows_per_src=[8188, 8134, 8190, 8192, 7460, 8192, 8188, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229296 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235028400 ingress_bytes=272547840 link_bytes=272547840 token_bytes=469827504 weight_movement_bytes=37748736 total_nvlink_bytes=507576240 weight_to_token_ratio=0.080346 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611419301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 57, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234315776 dispatch_meta_bytes=228968 combine_payload_bytes=234315776 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234544744 ingress_bytes=309813248 link_bytes=309813248 token_bytes=468860520 weight_movement_bytes=75497472 total_nvlink_bytes=544357992 weight_to_token_ratio=0.161023 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968478105e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=231604224 dispatch_meta_bytes=226560 combine_payload_bytes=231604224 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231830784 ingress_bytes=288227328 link_bytes=288227328 token_bytes=463435008 weight_movement_bytes=56623104 total_nvlink_bytes=520058112 weight_to_token_ratio=0.122181 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.761652440e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 8192, 8192, 8150] dup_rows_per_dst=[0, 0, 0, 4, 42, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 8192, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57292 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8140] dup_rows_per_dst=[0, 0, 0, 0, 52, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234921984 ingress_bytes=234692608 link_bytes=234921984 token_bytes=469614592 weight_movement_bytes=0 total_nvlink_bytes=469614592 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250906862e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57344 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 23, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8146, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 46] combine_rows_per_src=[8192, 8192, 8192, 8192, 8146, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=8 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8192, 8190, 8192, 8192, 8188] dup_rows_per_dst=[0, 12, 0, 6, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8178, 8192, 8190, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229376 combine_payload_bytes=234668032 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234897408 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469565440 weight_movement_bytes=37748736 total_nvlink_bytes=507314176 weight_to_token_ratio=0.080391 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=44 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8160] dup_rows_per_dst=[2, 26, 18, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64558 payload_rows_remote=56368 dup_rows=978 dup_rows_remote=18 dedup_ratio=0.014923 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 467, 0, 0, 4094] payload_rows_per_dst=[8188, 8158, 8190, 8192, 7258, 8190, 8192, 8190] dup_rows_per_dst=[4, 0, 8, 0, 2, 2, 2, 960] combine_rows_per_src=[8188, 8158, 8190, 8192, 7258, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229560 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110584 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991608 weight_movement_bytes=18874368 total_nvlink_bytes=488865976 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58980 payload_rows_remote=50796 dup_rows=6556 dup_rows_remote=252 dedup_ratio=0.100037 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 5, 0, 0, 0, 0, 0, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3281, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 1644, 8192, 8184, 8192] dup_rows_per_dst=[244, 0, 0, 0, 8, 0, 6304, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 1644, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=116 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8076] dup_rows_per_dst=[0, 2, 0, 0, 114, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229328 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235028432 ingress_bytes=272547840 link_bytes=272547840 token_bytes=469827536 weight_movement_bytes=37748736 total_nvlink_bytes=507576272 weight_to_token_ratio=0.080346 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611419301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229368 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234921976 ingress_bytes=272441344 link_bytes=272441344 token_bytes=469614584 weight_movement_bytes=37748736 total_nvlink_bytes=507363320 weight_to_token_ratio=0.080382 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610398909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=230883328 dispatch_meta_bytes=225544 combine_payload_bytes=230883328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231108872 ingress_bytes=287506432 link_bytes=287506432 token_bytes=461992200 weight_movement_bytes=56623104 total_nvlink_bytes=518615304 weight_to_token_ratio=0.122563 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.754745169e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63038 payload_rows_remote=54846 dup_rows=2498 dup_rows_remote=2498 dedup_ratio=0.038116 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1249, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5694, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2498] combine_rows_per_src=[8192, 8192, 8192, 8192, 5694, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=208060416 dispatch_meta_bytes=204192 combine_payload_bytes=208060416 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=208264608 ingress_bytes=283557888 link_bytes=283557888 token_bytes=416325024 weight_movement_bytes=75497472 total_nvlink_bytes=491822496 weight_to_token_ratio=0.181343 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.716912164e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=229376 combine_payload_bytes=234405888 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234635264 ingress_bytes=272154624 link_bytes=272154624 token_bytes=469041152 weight_movement_bytes=37748736 total_nvlink_bytes=506789888 weight_to_token_ratio=0.080481 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607651699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64580 payload_rows_remote=56388 dup_rows=956 dup_rows_remote=956 dedup_ratio=0.014587 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 479, 0, 0, 4095] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7236, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 956] combine_rows_per_src=[8192, 8192, 8192, 8192, 7236, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=224649216 dispatch_meta_bytes=229376 combine_payload_bytes=224649216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224878592 ingress_bytes=243523584 link_bytes=243523584 token_bytes=449527808 weight_movement_bytes=18874368 total_nvlink_bytes=468402176 weight_to_token_ratio=0.041987 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.333323161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=230965248 dispatch_meta_bytes=229376 combine_payload_bytes=230965248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231194624 ingress_bytes=249839616 link_bytes=249839616 token_bytes=462159872 weight_movement_bytes=18874368 total_nvlink_bytes=481034240 weight_to_token_ratio=0.040839 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393840272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64694 payload_rows_remote=56502 dup_rows=842 dup_rows_remote=842 dedup_ratio=0.012848 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 410, 0, 0, 4078] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7380, 8192, 8192, 8164] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 28, 812] combine_rows_per_src=[8192, 8192, 8190, 8192, 7380, 8192, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57656 payload_rows_remote=49464 dup_rows=7880 dup_rows_remote=7880 dedup_ratio=0.120239 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 20, 0, 0, 4089] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8188, 8192, 8160, 350] dup_rows_per_dst=[0, 34, 0, 0, 7842, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8190, 8192, 8188, 8192, 8160, 350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57294 dup_rows=56 dup_rows_remote=14 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8190, 8148, 8192, 8192, 8190, 8186] dup_rows_per_dst=[2, 0, 0, 0, 0, 10, 2, 42] combine_rows_per_src=[8190, 8192, 8190, 8148, 8192, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=234872832 link_bytes=235102208 token_bytes=469975040 weight_movement_bytes=0 total_nvlink_bytes=469975040 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252633680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=231432192 dispatch_meta_bytes=229376 combine_payload_bytes=231432192 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=231661568 ingress_bytes=325804032 link_bytes=325804032 token_bytes=463093760 weight_movement_bytes=94371840 total_nvlink_bytes=557465600 weight_to_token_ratio=0.203786 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.121693929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58322 payload_rows_remote=50130 dup_rows=7214 dup_rows_remote=7214 dedup_ratio=0.110077 router_tokens_per_expert=[3103, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0] payload_rows_per_dst=[6206, 8192, 8192, 2964, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 7212] combine_rows_per_src=[6206, 8192, 8192, 2964, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64986 payload_rows_remote=56794 dup_rows=550 dup_rows_remote=550 dedup_ratio=0.008392 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 274, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7644, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 548] combine_rows_per_src=[8192, 8192, 8190, 8192, 7644, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=202604544 dispatch_meta_bytes=229376 combine_payload_bytes=202604544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202833920 ingress_bytes=221478912 link_bytes=221478912 token_bytes=405438464 weight_movement_bytes=18874368 total_nvlink_bytes=424312832 weight_to_token_ratio=0.046553 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.122101960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229232 combine_payload_bytes=234676224 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234905456 ingress_bytes=272424960 link_bytes=272424960 token_bytes=469581680 weight_movement_bytes=37748736 total_nvlink_bytes=507330416 weight_to_token_ratio=0.080388 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610241925e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=205332480 dispatch_meta_bytes=229376 combine_payload_bytes=205332480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205561856 ingress_bytes=224206848 link_bytes=224206848 token_bytes=410894336 weight_movement_bytes=18874368 total_nvlink_bytes=429768704 weight_to_token_ratio=0.045935 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.148239701e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=232628224 dispatch_meta_bytes=229376 combine_payload_bytes=232628224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232857600 ingress_bytes=251502592 link_bytes=251502592 token_bytes=465485824 weight_movement_bytes=18874368 total_nvlink_bytes=484360192 weight_to_token_ratio=0.040548 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409774090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235102200 ingress_bytes=291495936 link_bytes=291495936 token_bytes=469975032 weight_movement_bytes=56623104 total_nvlink_bytes=526598136 weight_to_token_ratio=0.120481 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792970634e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64854 payload_rows_remote=56662 dup_rows=682 dup_rows_remote=682 dedup_ratio=0.010406 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 7512] dup_rows_per_dst=[0, 0, 2, 0, 0, 678, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 7512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65426 payload_rows_remote=57234 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.001678 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 8190, 8176, 8192, 8192, 8192, 8156] dup_rows_per_dst=[0, 0, 0, 0, 0, 108, 0, 2] combine_rows_per_src=[8136, 8192, 8190, 8176, 8192, 8192, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235053056 ingress_bytes=272572416 link_bytes=272572416 token_bytes=469876736 weight_movement_bytes=37748736 total_nvlink_bytes=507625472 weight_to_token_ratio=0.080338 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611654776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65372 payload_rows_remote=57180 dup_rows=164 dup_rows_remote=102 dedup_ratio=0.002502 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8192, 8038, 8192, 8192] dup_rows_per_dst=[62, 0, 4, 88, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8192, 8038, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232087552 dispatch_meta_bytes=229376 combine_payload_bytes=232087552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232316928 ingress_bytes=250961920 link_bytes=250961920 token_bytes=464404480 weight_movement_bytes=18874368 total_nvlink_bytes=483278848 weight_to_token_ratio=0.040642 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404593636e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60020 payload_rows_remote=51978 dup_rows=5516 dup_rows_remote=218 dedup_ratio=0.084167 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8170, 8156, 8110, 8190, 8190, 2970, 8192, 8042] dup_rows_per_dst=[0, 20, 0, 4, 2, 186, 6, 5298] combine_rows_per_src=[8170, 8156, 8110, 8190, 8190, 2970, 8192, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234430464 dispatch_meta_bytes=229376 combine_payload_bytes=234430464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234659840 ingress_bytes=253304832 link_bytes=253304832 token_bytes=469090304 weight_movement_bytes=18874368 total_nvlink_bytes=487964672 weight_to_token_ratio=0.040236 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427042266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65478 payload_rows_remote=57342 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000885 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8136, 8192] dup_rows_per_dst=[0, 58, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=2 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8178, 8192, 8190, 8192, 8192, 8192, 8102] dup_rows_per_dst=[0, 2, 0, 0, 0, 104, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8190, 8192, 8192, 8192, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=229128 combine_payload_bytes=234209280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234438408 ingress_bytes=253083648 link_bytes=253083648 token_bytes=468647688 weight_movement_bytes=18874368 total_nvlink_bytes=487522056 weight_to_token_ratio=0.040274 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424922990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=212901888 dispatch_meta_bytes=208784 combine_payload_bytes=212901888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213110672 ingress_bytes=250650624 link_bytes=250650624 token_bytes=426012560 weight_movement_bytes=37748736 total_nvlink_bytes=463761296 weight_to_token_ratio=0.088609 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401610951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64938 payload_rows_remote=56746 dup_rows=598 dup_rows_remote=598 dedup_ratio=0.009125 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7594] dup_rows_per_dst=[0, 0, 0, 0, 0, 598, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229600 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102432 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975264 weight_movement_bytes=18874368 total_nvlink_bytes=488849632 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235085824 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942272 weight_movement_bytes=56623104 total_nvlink_bytes=526565376 weight_to_token_ratio=0.120489 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=228960 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234675808 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469122656 weight_movement_bytes=18874368 total_nvlink_bytes=487997024 weight_to_token_ratio=0.040233 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64554 payload_rows_remote=56362 dup_rows=982 dup_rows_remote=956 dedup_ratio=0.014984 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7210] dup_rows_per_dst=[0, 26, 60, 0, 0, 896, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232431616 dispatch_meta_bytes=229376 combine_payload_bytes=232431616 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232660992 ingress_bytes=270180352 link_bytes=270180352 token_bytes=465092608 weight_movement_bytes=37748736 total_nvlink_bytes=502841344 weight_to_token_ratio=0.081164 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.588735196e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63856 payload_rows_remote=55664 dup_rows=1680 dup_rows_remote=1658 dedup_ratio=0.025635 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 6522] dup_rows_per_dst=[22, 4, 0, 0, 0, 1654, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 6522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230858752 dispatch_meta_bytes=229272 combine_payload_bytes=230858752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231088024 ingress_bytes=249733120 link_bytes=249733120 token_bytes=461946776 weight_movement_bytes=18874368 total_nvlink_bytes=480821144 weight_to_token_ratio=0.040858 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392819879e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57330 dup_rows=16 dup_rows_remote=10 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 4094] payload_rows_per_dst=[8190, 8190, 8192, 8192, 8188, 8188, 8190, 8190] dup_rows_per_dst=[0, 2, 0, 0, 6, 2, 0, 6] combine_rows_per_src=[8190, 8190, 8192, 8192, 8188, 8188, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65098 payload_rows_remote=56906 dup_rows=438 dup_rows_remote=438 dedup_ratio=0.006683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7754] dup_rows_per_dst=[0, 0, 0, 0, 438, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64944 payload_rows_remote=56752 dup_rows=592 dup_rows_remote=592 dedup_ratio=0.009033 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7600] dup_rows_per_dst=[0, 4, 10, 0, 0, 578, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=227999744 dispatch_meta_bytes=229288 combine_payload_bytes=227999744 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=228229032 ingress_bytes=284622848 link_bytes=284622848 token_bytes=456228776 weight_movement_bytes=56623104 total_nvlink_bytes=512851880 weight_to_token_ratio=0.124111 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.727116086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229360 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235053040 ingress_bytes=329195520 link_bytes=329195520 token_bytes=469876720 weight_movement_bytes=94371840 total_nvlink_bytes=564248560 weight_to_token_ratio=0.200844 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154189498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=233086976 dispatch_meta_bytes=229376 combine_payload_bytes=233086976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233316352 ingress_bytes=251961344 link_bytes=251961344 token_bytes=466403328 weight_movement_bytes=18874368 total_nvlink_bytes=485277696 weight_to_token_ratio=0.040468 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414169625e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232456192 dispatch_meta_bytes=229376 combine_payload_bytes=232456192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232685568 ingress_bytes=251330560 link_bytes=251330560 token_bytes=465141760 weight_movement_bytes=18874368 total_nvlink_bytes=484016128 weight_to_token_ratio=0.040578 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408125764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57342 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 4, 2, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229392 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102224 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975056 weight_movement_bytes=18874368 total_nvlink_bytes=488849424 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=6 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8190] dup_rows_per_dst=[2, 4, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57328 dup_rows=18 dup_rows_remote=16 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8186, 8190, 8190, 8192, 8188, 8190] dup_rows_per_dst=[2, 0, 4, 0, 2, 2, 6, 2] combine_rows_per_src=[8190, 8192, 8186, 8190, 8190, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63488 payload_rows_remote=55296 dup_rows=2048 dup_rows_remote=2048 dedup_ratio=0.031250 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1024, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6144, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2048] combine_rows_per_src=[8192, 8192, 8192, 8192, 6144, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229368 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077624 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925880 weight_movement_bytes=18874368 total_nvlink_bytes=488800248 weight_to_token_ratio=0.040165 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860352 weight_movement_bytes=37748736 total_nvlink_bytes=507609088 weight_to_token_ratio=0.080340 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=226492416 dispatch_meta_bytes=229376 combine_payload_bytes=226492416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226721792 ingress_bytes=245366784 link_bytes=245366784 token_bytes=453214208 weight_movement_bytes=18874368 total_nvlink_bytes=472088576 weight_to_token_ratio=0.041646 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.350983796e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57338 dup_rows=8 dup_rows_remote=6 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8190, 8188, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 2, 0, 0, 0, 4, 0, 2] combine_rows_per_src=[8192, 8190, 8188, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64626 payload_rows_remote=57344 dup_rows=910 dup_rows_remote=910 dedup_ratio=0.013885 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 454, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7282, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 908] combine_rows_per_src=[8192, 8192, 8192, 8192, 7282, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=234872832 link_bytes=235102208 token_bytes=469975040 weight_movement_bytes=0 total_nvlink_bytes=469975040 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252633680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8174, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 18, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8174, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235110400 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991424 weight_movement_bytes=94371840 total_nvlink_bytes=564363264 weight_to_token_ratio=0.200795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=233016 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235114040 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469995064 weight_movement_bytes=18874368 total_nvlink_bytes=488869432 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=25728 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60908 payload_rows_remote=52716 dup_rows=4628 dup_rows_remote=12 dedup_ratio=0.070618 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 2306, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] payload_rows_per_dst=[8190, 8188, 8186, 8190, 3582, 8190, 8190, 8192] dup_rows_per_dst=[0, 2, 2, 0, 2, 0, 6, 4616] combine_rows_per_src=[8190, 8188, 8186, 8190, 3582, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57312 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8164, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 32, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8164, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235036672 ingress_bytes=272556032 link_bytes=272556032 token_bytes=469843968 weight_movement_bytes=37748736 total_nvlink_bytes=507592704 weight_to_token_ratio=0.080343 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611497793e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=215924736 dispatch_meta_bytes=210912 combine_payload_bytes=215924736 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=216135648 ingress_bytes=310296576 link_bytes=310296576 token_bytes=432060384 weight_movement_bytes=94371840 total_nvlink_bytes=526432224 weight_to_token_ratio=0.218423 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973109116e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64996 payload_rows_remote=56804 dup_rows=540 dup_rows_remote=538 dedup_ratio=0.008240 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 7656] dup_rows_per_dst=[0, 0, 10, 0, 0, 528, 2, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 7656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=238 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 119, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7954, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 238] combine_rows_per_src=[8192, 8192, 8192, 8192, 7954, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229376 combine_payload_bytes=234749952 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234979328 ingress_bytes=253624320 link_bytes=253624320 token_bytes=469729280 weight_movement_bytes=18874368 total_nvlink_bytes=488603648 weight_to_token_ratio=0.040181 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430103443e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64652 payload_rows_remote=56460 dup_rows=884 dup_rows_remote=884 dedup_ratio=0.013489 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 442, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7308, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 884] combine_rows_per_src=[8192, 8192, 8192, 8192, 7308, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=0 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62770 payload_rows_remote=54578 dup_rows=2766 dup_rows_remote=2766 dedup_ratio=0.042206 router_tokens_per_expert=[0, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2713, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5426, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2766, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5426, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232669184 dispatch_meta_bytes=229368 combine_payload_bytes=232669184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232898552 ingress_bytes=251543552 link_bytes=251543552 token_bytes=465567736 weight_movement_bytes=18874368 total_nvlink_bytes=484442104 weight_to_token_ratio=0.040541 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410166548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229376 combine_payload_bytes=233906176 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234135552 ingress_bytes=271654912 link_bytes=271654912 token_bytes=468041728 weight_movement_bytes=37748736 total_nvlink_bytes=505790464 weight_to_token_ratio=0.080653 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602863704e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60920 payload_rows_remote=52728 dup_rows=4616 dup_rows_remote=4616 dedup_ratio=0.070435 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1287, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8158, 7686, 8192, 8192, 4116] dup_rows_per_dst=[0, 0, 0, 0, 4108, 2, 0, 506] combine_rows_per_src=[8192, 8192, 8192, 8158, 7686, 8192, 8192, 4116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=229376 combine_payload_bytes=231260160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231489536 ingress_bytes=250134528 link_bytes=250134528 token_bytes=462749696 weight_movement_bytes=18874368 total_nvlink_bytes=481624064 weight_to_token_ratio=0.040787 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396665973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229336 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235069400 ingress_bytes=272588800 link_bytes=272588800 token_bytes=469909464 weight_movement_bytes=37748736 total_nvlink_bytes=507658200 weight_to_token_ratio=0.080332 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611811760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=223551488 dispatch_meta_bytes=229376 combine_payload_bytes=223551488 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=223780864 ingress_bytes=242425856 link_bytes=242425856 token_bytes=447332352 weight_movement_bytes=18874368 total_nvlink_bytes=466206720 weight_to_token_ratio=0.042193 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.322805271e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59322 payload_rows_remote=51130 dup_rows=6214 dup_rows_remote=6214 dedup_ratio=0.094818 router_tokens_per_expert=[1, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3103, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[1984, 8190, 8190, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 6208, 0, 4, 0, 2, 0] combine_rows_per_src=[1984, 8190, 8190, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59236 payload_rows_remote=51044 dup_rows=6300 dup_rows_remote=6298 dedup_ratio=0.096130 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 948, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 1896] dup_rows_per_dst=[2, 2, 0, 38, 6258, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 1896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=215973888 dispatch_meta_bytes=229376 combine_payload_bytes=215973888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216203264 ingress_bytes=253722624 link_bytes=253722624 token_bytes=432177152 weight_movement_bytes=37748736 total_nvlink_bytes=469925888 weight_to_token_ratio=0.087346 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=209428480 dispatch_meta_bytes=229376 combine_payload_bytes=209428480 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=209657856 ingress_bytes=228302848 link_bytes=228302848 token_bytes=419086336 weight_movement_bytes=18874368 total_nvlink_bytes=437960704 weight_to_token_ratio=0.045037 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.187485558e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61928 payload_rows_remote=53736 dup_rows=3608 dup_rows_remote=3608 dedup_ratio=0.055054 router_tokens_per_expert=[0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2280, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8090, 8192, 4686, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[3506, 0, 102, 0, 0, 0, 0, 0] combine_rows_per_src=[8090, 8192, 4686, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=209076224 dispatch_meta_bytes=229368 combine_payload_bytes=209076224 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=209305592 ingress_bytes=246824960 link_bytes=246824960 token_bytes=418381816 weight_movement_bytes=37748736 total_nvlink_bytes=456130552 weight_to_token_ratio=0.090226 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364955321e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64248 payload_rows_remote=57344 dup_rows=1288 dup_rows_remote=1288 dedup_ratio=0.019653 router_tokens_per_expert=[0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3452, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6904, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1286, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6904, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64706 payload_rows_remote=56516 dup_rows=830 dup_rows_remote=830 dedup_ratio=0.012665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 415, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 7364, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 830] combine_rows_per_src=[8192, 8192, 8192, 8190, 7364, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64096 payload_rows_remote=55904 dup_rows=1440 dup_rows_remote=1440 dedup_ratio=0.021973 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 6752, 8192] dup_rows_per_dst=[0, 0, 0, 58, 0, 0, 0, 1382] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 6752, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=220102656 dispatch_meta_bytes=229376 combine_payload_bytes=220102656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220332032 ingress_bytes=238977024 link_bytes=238977024 token_bytes=440434688 weight_movement_bytes=18874368 total_nvlink_bytes=459309056 weight_to_token_ratio=0.042854 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.289760260e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=234528 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235115552 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469996576 weight_movement_bytes=75497472 total_nvlink_bytes=545494048 weight_to_token_ratio=0.160634 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24576 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63616 payload_rows_remote=55424 dup_rows=1920 dup_rows_remote=1920 dedup_ratio=0.029297 router_tokens_per_expert=[0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3136, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6272, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1914, 0, 0, 0, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6272, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=231489536 dispatch_meta_bytes=229384 combine_payload_bytes=231489536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231718920 ingress_bytes=250363904 link_bytes=250363904 token_bytes=463208456 weight_movement_bytes=18874368 total_nvlink_bytes=482082824 weight_to_token_ratio=0.040747 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398863741e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57350 payload_rows_remote=49158 dup_rows=8186 dup_rows_remote=6950 dedup_ratio=0.124908 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 4095, 0, 3386, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8188, 8192, 7132, 8192, 1072, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 178, 0, 0, 6772, 1236] combine_rows_per_src=[8190, 8188, 8192, 7132, 8192, 1072, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=228982784 dispatch_meta_bytes=229376 combine_payload_bytes=228982784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229212160 ingress_bytes=247857152 link_bytes=247857152 token_bytes=458194944 weight_movement_bytes=18874368 total_nvlink_bytes=477069312 weight_to_token_ratio=0.041193 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374845277e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64826 payload_rows_remote=56634 dup_rows=710 dup_rows_remote=4 dedup_ratio=0.010834 router_tokens_per_expert=[0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3743, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7502, 8192, 8174, 8192, 8190, 8192] dup_rows_per_dst=[706, 2, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 7502, 8192, 8174, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=227016704 dispatch_meta_bytes=229376 combine_payload_bytes=227016704 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=227246080 ingress_bytes=264765440 link_bytes=264765440 token_bytes=454262784 weight_movement_bytes=37748736 total_nvlink_bytes=492011520 weight_to_token_ratio=0.083099 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.536852173e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=201351168 dispatch_meta_bytes=224432 combine_payload_bytes=201351168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201575600 ingress_bytes=220225536 link_bytes=220225536 token_bytes=402926768 weight_movement_bytes=18874368 total_nvlink_bytes=421801136 weight_to_token_ratio=0.046843 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.110092728e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=231972864 dispatch_meta_bytes=226552 combine_payload_bytes=231972864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232199416 ingress_bytes=250847232 link_bytes=250847232 token_bytes=464172280 weight_movement_bytes=18874368 total_nvlink_bytes=483046648 weight_to_token_ratio=0.040662 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403494752e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63034 payload_rows_remote=54850 dup_rows=2502 dup_rows_remote=2502 dedup_ratio=0.038177 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8184, 5698, 8192] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 2492] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8184, 5698, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63742 payload_rows_remote=55550 dup_rows=1794 dup_rows_remote=1794 dedup_ratio=0.027374 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 6398, 8192] dup_rows_per_dst=[0, 0, 0, 398, 0, 0, 0, 1396] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 6398, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63902 payload_rows_remote=55710 dup_rows=1634 dup_rows_remote=1634 dedup_ratio=0.024933 router_tokens_per_expert=[0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3279, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6558, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1634, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6558, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=224665600 dispatch_meta_bytes=229408 combine_payload_bytes=224665600 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=224895008 ingress_bytes=337911808 link_bytes=337911808 token_bytes=449560608 weight_movement_bytes=113246208 total_nvlink_bytes=562806816 weight_to_token_ratio=0.251904 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.237704681e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=227532800 dispatch_meta_bytes=229376 combine_payload_bytes=227532800 prefetch_bytes=0 reduce_bytes=0 egress_bytes=227762176 ingress_bytes=227532800 link_bytes=227762176 token_bytes=455294976 weight_movement_bytes=0 total_nvlink_bytes=455294976 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182305104e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58152 payload_rows_remote=49962 dup_rows=7384 dup_rows_remote=7384 dedup_ratio=0.112671 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 6384, 2618] dup_rows_per_dst=[0, 0, 0, 6, 0, 6878, 12, 488] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 6384, 2618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=228188160 dispatch_meta_bytes=229376 combine_payload_bytes=228188160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228417536 ingress_bytes=247062528 link_bytes=247062528 token_bytes=456605696 weight_movement_bytes=18874368 total_nvlink_bytes=475480064 weight_to_token_ratio=0.041336 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.367231581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 8190, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 4, 2, 2, 0] combine_rows_per_src=[8192, 8188, 8190, 8190, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57346 payload_rows_remote=49154 dup_rows=8190 dup_rows_remote=8188 dedup_ratio=0.124969 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8082, 112] dup_rows_per_dst=[2, 6, 0, 2, 0, 8180, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8082, 112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=204644352 dispatch_meta_bytes=229384 combine_payload_bytes=204644352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204873736 ingress_bytes=223518720 link_bytes=223518720 token_bytes=409518088 weight_movement_bytes=18874368 total_nvlink_bytes=428392456 weight_to_token_ratio=0.046089 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.141646397e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=253714432 link_bytes=253714432 token_bytes=469909504 weight_movement_bytes=18874368 total_nvlink_bytes=488783872 weight_to_token_ratio=0.040166 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430966852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=201334784 dispatch_meta_bytes=229368 combine_payload_bytes=201334784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201564152 ingress_bytes=220209152 link_bytes=220209152 token_bytes=402898936 weight_movement_bytes=18874368 total_nvlink_bytes=421773304 weight_to_token_ratio=0.046846 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109935745e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62238 payload_rows_remote=54046 dup_rows=3298 dup_rows_remote=3072 dedup_ratio=0.050323 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 4896, 8192] dup_rows_per_dst=[0, 0, 0, 226, 0, 0, 0, 3072] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 4896, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64418 payload_rows_remote=56226 dup_rows=1118 dup_rows_remote=1118 dedup_ratio=0.017059 router_tokens_per_expert=[0, 0, 0, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[7074, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1116, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[7074, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65460 payload_rows_remote=57344 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8116, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 76, 0, 0, 0] combine_rows_per_src=[8192, 8116, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57260 payload_rows_remote=49070 dup_rows=8276 dup_rows_remote=8198 dedup_ratio=0.126282 router_tokens_per_expert=[0, 0, 1, 0, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1414, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 4094, 0, 0, 4095] payload_rows_per_dst=[5298, 2830, 8190, 8190, 8188, 8188, 8186, 8190] dup_rows_per_dst=[0, 0, 0, 0, 2, 8190, 6, 78] combine_rows_per_src=[5298, 2830, 8190, 8190, 8188, 8188, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=221372416 dispatch_meta_bytes=228472 combine_payload_bytes=221372416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221600888 ingress_bytes=240246784 link_bytes=240246784 token_bytes=442973304 weight_movement_bytes=18874368 total_nvlink_bytes=461847672 weight_to_token_ratio=0.042608 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.301926475e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=230301696 dispatch_meta_bytes=229376 combine_payload_bytes=230301696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230531072 ingress_bytes=249176064 link_bytes=249176064 token_bytes=460832768 weight_movement_bytes=18874368 total_nvlink_bytes=479707136 weight_to_token_ratio=0.040957 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.387482443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57258 dup_rows=86 dup_rows_remote=86 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8108, 8192, 8190, 8192] dup_rows_per_dst=[0, 84, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8108, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229680 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110704 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991728 weight_movement_bytes=37748736 total_nvlink_bytes=507740464 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=200990720 dispatch_meta_bytes=229072 combine_payload_bytes=200990720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=201219792 ingress_bytes=257613824 link_bytes=257613824 token_bytes=402210512 weight_movement_bytes=56623104 total_nvlink_bytes=458833616 weight_to_token_ratio=0.140780 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.468328908e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229376 combine_payload_bytes=234528768 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234758144 ingress_bytes=234528768 link_bytes=234758144 token_bytes=469286912 weight_movement_bytes=0 total_nvlink_bytes=469286912 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249337028e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63144 payload_rows_remote=54952 dup_rows=2392 dup_rows_remote=2392 dedup_ratio=0.036499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1195, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5800, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 2390] combine_rows_per_src=[8192, 8192, 8192, 8192, 5800, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65240 payload_rows_remote=57048 dup_rows=296 dup_rows_remote=296 dedup_ratio=0.004517 router_tokens_per_expert=[0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[7896, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 296, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7896, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64626 payload_rows_remote=56434 dup_rows=910 dup_rows_remote=910 dedup_ratio=0.013885 router_tokens_per_expert=[0, 0, 0, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[7282, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 910, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7282, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=225083392 dispatch_meta_bytes=229376 combine_payload_bytes=225083392 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=225312768 ingress_bytes=300580864 link_bytes=300580864 token_bytes=450396160 weight_movement_bytes=75497472 total_nvlink_bytes=525893632 weight_to_token_ratio=0.167625 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.880017944e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233668608 dispatch_meta_bytes=229376 combine_payload_bytes=233668608 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233897984 ingress_bytes=252542976 link_bytes=252542976 token_bytes=467566592 weight_movement_bytes=18874368 total_nvlink_bytes=486440960 weight_to_token_ratio=0.040367 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419742537e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=231153664 dispatch_meta_bytes=229376 combine_payload_bytes=231153664 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=231383040 ingress_bytes=344399872 link_bytes=344399872 token_bytes=462536704 weight_movement_bytes=113246208 total_nvlink_bytes=575782912 weight_to_token_ratio=0.244837 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.299870118e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61760 payload_rows_remote=53568 dup_rows=3776 dup_rows_remote=1060 dedup_ratio=0.057617 router_tokens_per_expert=[0, 0, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096] payload_rows_per_dst=[5480, 8192, 8192, 8192, 8192, 7130, 8192, 8190] dup_rows_per_dst=[1056, 2716, 0, 2, 2, 0, 0, 0] combine_rows_per_src=[5480, 8192, 8192, 8192, 8192, 7130, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57324 dup_rows=24 dup_rows_remote=20 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8188, 8186, 8186, 8192, 8188] dup_rows_per_dst=[2, 2, 0, 4, 4, 8, 0, 4] combine_rows_per_src=[8190, 8190, 8192, 8188, 8186, 8186, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57338 payload_rows_remote=49146 dup_rows=8198 dup_rows_remote=430 dedup_ratio=0.125092 router_tokens_per_expert=[0, 0, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 68, 8192, 8118] dup_rows_per_dst=[7768, 430, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 68, 8192, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=219414528 dispatch_meta_bytes=218512 combine_payload_bytes=219414528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219633040 ingress_bytes=238288896 link_bytes=238288896 token_bytes=439047568 weight_movement_bytes=18874368 total_nvlink_bytes=457921936 weight_to_token_ratio=0.042989 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283166956e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=201302016 dispatch_meta_bytes=198304 combine_payload_bytes=201302016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201500320 ingress_bytes=220176384 link_bytes=220176384 token_bytes=402802336 weight_movement_bytes=18874368 total_nvlink_bytes=421676704 weight_to_token_ratio=0.046858 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109621778e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60034 payload_rows_remote=51842 dup_rows=5502 dup_rows_remote=5502 dedup_ratio=0.083954 router_tokens_per_expert=[0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2748, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2690, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 5496, 0, 6, 0, 0, 0] combine_rows_per_src=[2690, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64912 payload_rows_remote=56720 dup_rows=624 dup_rows_remote=624 dedup_ratio=0.009521 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 4096, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7568, 8192, 8192] dup_rows_per_dst=[0, 624, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7568, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64966 payload_rows_remote=56774 dup_rows=570 dup_rows_remote=570 dedup_ratio=0.008698 router_tokens_per_expert=[0, 0, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096] payload_rows_per_dst=[7624, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 568, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[7624, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=212344832 dispatch_meta_bytes=229376 combine_payload_bytes=212344832 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=212574208 ingress_bytes=231219200 link_bytes=231219200 token_bytes=424919040 weight_movement_bytes=18874368 total_nvlink_bytes=443793408 weight_to_token_ratio=0.044419 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215428607e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62572 payload_rows_remote=54382 dup_rows=2964 dup_rows_remote=2962 dedup_ratio=0.045227 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 4095, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5326, 8190, 8190, 8192, 8102, 8190, 8190] dup_rows_per_dst=[0, 90, 2, 2, 4, 2862, 2, 2] combine_rows_per_src=[8192, 5326, 8190, 8190, 8192, 8102, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=232325120 dispatch_meta_bytes=229376 combine_payload_bytes=232325120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232554496 ingress_bytes=270073856 link_bytes=270073856 token_bytes=464879616 weight_movement_bytes=37748736 total_nvlink_bytes=502628352 weight_to_token_ratio=0.081201 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.587714804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=232546304 dispatch_meta_bytes=229376 combine_payload_bytes=232546304 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232775680 ingress_bytes=270295040 link_bytes=270295040 token_bytes=465321984 weight_movement_bytes=37748736 total_nvlink_bytes=503070720 weight_to_token_ratio=0.081124 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589834080e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=234872832 link_bytes=235102208 token_bytes=469975040 weight_movement_bytes=0 total_nvlink_bytes=469975040 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252633680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=222748672 dispatch_meta_bytes=229376 combine_payload_bytes=222748672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222978048 ingress_bytes=241623040 link_bytes=241623040 token_bytes=445726720 weight_movement_bytes=18874368 total_nvlink_bytes=464601088 weight_to_token_ratio=0.042345 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.315113083e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 4096, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8158, 8192, 8192] dup_rows_per_dst=[0, 34, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8158, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57602 payload_rows_remote=49416 dup_rows=7934 dup_rows_remote=3770 dedup_ratio=0.121063 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 264, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 182, 0, 0, 4164, 3588] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 264, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57344 dup_rows=232 dup_rows_remote=232 dedup_ratio=0.003540 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7960, 8192, 8192] dup_rows_per_dst=[0, 232, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7960, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62596 payload_rows_remote=57140 dup_rows=2940 dup_rows_remote=2940 dedup_ratio=0.044861 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7988, 5456, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2840, 0, 0, 0, 100, 0] combine_rows_per_src=[8192, 8192, 8192, 7988, 5456, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229376 combine_payload_bytes=234741760 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234971136 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469712896 weight_movement_bytes=18874368 total_nvlink_bytes=488587264 weight_to_token_ratio=0.040183 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=202407936 dispatch_meta_bytes=212744 combine_payload_bytes=202407936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202620680 ingress_bytes=221282304 link_bytes=221282304 token_bytes=405028616 weight_movement_bytes=18874368 total_nvlink_bytes=423902984 weight_to_token_ratio=0.046600 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.120218159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65418 payload_rows_remote=57226 dup_rows=118 dup_rows_remote=0 dedup_ratio=0.001801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1558, 0, 0, 0, 0, 0, 4096, 2479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8074, 8192, 8192] dup_rows_per_dst=[0, 118, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8074, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=230304 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235111328 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469992352 weight_movement_bytes=75497472 total_nvlink_bytes=545489824 weight_to_token_ratio=0.160636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57300 payload_rows_remote=49108 dup_rows=8236 dup_rows_remote=8234 dedup_ratio=0.125671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8188, 30, 8128, 8192, 8188, 8192, 8192] dup_rows_per_dst=[2, 4, 0, 0, 8170, 0, 58, 2] combine_rows_per_src=[8190, 8188, 30, 8128, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234045440 dispatch_meta_bytes=240320 combine_payload_bytes=234045440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234285760 ingress_bytes=271794176 link_bytes=271794176 token_bytes=468331200 weight_movement_bytes=37748736 total_nvlink_bytes=506079936 weight_to_token_ratio=0.080603 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604198063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64080 payload_rows_remote=55888 dup_rows=1456 dup_rows_remote=1456 dedup_ratio=0.022217 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 4096, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 6742, 8192, 8192] dup_rows_per_dst=[0, 1454, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 6742, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234397696 dispatch_meta_bytes=228904 combine_payload_bytes=234397696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234626600 ingress_bytes=272146432 link_bytes=272146432 token_bytes=469024296 weight_movement_bytes=37748736 total_nvlink_bytes=506773032 weight_to_token_ratio=0.080484 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607573207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=201146368 dispatch_meta_bytes=229368 combine_payload_bytes=201146368 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=201375736 ingress_bytes=220020736 link_bytes=220020736 token_bytes=402522104 weight_movement_bytes=18874368 total_nvlink_bytes=421396472 weight_to_token_ratio=0.046890 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.108130435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65204 payload_rows_remote=57012 dup_rows=332 dup_rows_remote=48 dedup_ratio=0.005066 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8070, 7982, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 284, 0, 0, 0, 48, 0] combine_rows_per_src=[8192, 8192, 8192, 8070, 7982, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=228917248 dispatch_meta_bytes=229376 combine_payload_bytes=228917248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229146624 ingress_bytes=247791616 link_bytes=247791616 token_bytes=458063872 weight_movement_bytes=18874368 total_nvlink_bytes=476938240 weight_to_token_ratio=0.041205 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374217343e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65086 payload_rows_remote=56894 dup_rows=450 dup_rows_remote=450 dedup_ratio=0.006866 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7752, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 16, 0, 0, 0, 434, 0] combine_rows_per_src=[8192, 8192, 8192, 7752, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233521152 dispatch_meta_bytes=228240 combine_payload_bytes=233521152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233749392 ingress_bytes=290144256 link_bytes=290144256 token_bytes=467270544 weight_movement_bytes=56623104 total_nvlink_bytes=523893648 weight_to_token_ratio=0.121178 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780019501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65234 payload_rows_remote=57042 dup_rows=302 dup_rows_remote=302 dedup_ratio=0.004608 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 928, 0, 0, 0, 0, 0, 4096, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7890, 8192, 8192] dup_rows_per_dst=[0, 302, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7890, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63110 payload_rows_remote=54918 dup_rows=2426 dup_rows_remote=2426 dedup_ratio=0.037018 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7678, 6280, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2384, 0, 0, 0, 42, 0] combine_rows_per_src=[8192, 8192, 8192, 7678, 6280, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229376 combine_payload_bytes=233037824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233267200 ingress_bytes=270786560 link_bytes=270786560 token_bytes=466305024 weight_movement_bytes=37748736 total_nvlink_bytes=504053760 weight_to_token_ratio=0.080953 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594543583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=16 dedup_ratio=0.000275 router_tokens_per_expert=[1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8186, 8192, 8190, 8192, 8190, 8188, 8188, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 4, 8, 2] combine_rows_per_src=[8186, 8192, 8190, 8192, 8190, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233644032 dispatch_meta_bytes=229376 combine_payload_bytes=233644032 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233873408 ingress_bytes=252518400 link_bytes=252518400 token_bytes=467517440 weight_movement_bytes=18874368 total_nvlink_bytes=486391808 weight_to_token_ratio=0.040371 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419507062e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61178 payload_rows_remote=52986 dup_rows=4358 dup_rows_remote=4358 dedup_ratio=0.066498 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7404, 5350, 8192, 7466, 8190] dup_rows_per_dst=[0, 0, 4348, 0, 0, 4, 6, 0] combine_rows_per_src=[8192, 8192, 8192, 7404, 5350, 8192, 7466, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=224944128 dispatch_meta_bytes=229376 combine_payload_bytes=224944128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225173504 ingress_bytes=243818496 link_bytes=243818496 token_bytes=450117632 weight_movement_bytes=18874368 total_nvlink_bytes=468992000 weight_to_token_ratio=0.041932 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.336148862e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229368 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235036664 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843960 weight_movement_bytes=18874368 total_nvlink_bytes=488718328 weight_to_token_ratio=0.040172 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=217030656 dispatch_meta_bytes=229376 combine_payload_bytes=217030656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217260032 ingress_bytes=235905024 link_bytes=235905024 token_bytes=434290688 weight_movement_bytes=18874368 total_nvlink_bytes=453165056 weight_to_token_ratio=0.043460 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260325867e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64238 payload_rows_remote=56046 dup_rows=1298 dup_rows_remote=1298 dedup_ratio=0.019806 router_tokens_per_expert=[0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] payload_rows_per_dst=[7866, 7220, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 298, 0, 0, 0, 1000, 0, 0] combine_rows_per_src=[7866, 7220, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64028 payload_rows_remote=56204 dup_rows=1508 dup_rows_remote=1508 dedup_ratio=0.023010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7824, 7052, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1472, 0, 0, 0, 36, 0] combine_rows_per_src=[8192, 8192, 8192, 7824, 7052, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=229564416 dispatch_meta_bytes=229376 combine_payload_bytes=229564416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229793792 ingress_bytes=248438784 link_bytes=248438784 token_bytes=459358208 weight_movement_bytes=18874368 total_nvlink_bytes=478232576 weight_to_token_ratio=0.041089 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.380418189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[2, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57336 dup_rows=10 dup_rows_remote=8 dedup_ratio=0.000153 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8190, 8192, 8190, 8192, 8190] dup_rows_per_dst=[0, 2, 2, 0, 0, 2, 2, 2] combine_rows_per_src=[8190, 8190, 8192, 8190, 8192, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=230211584 dispatch_meta_bytes=230848 combine_payload_bytes=230211584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230442432 ingress_bytes=249085952 link_bytes=249085952 token_bytes=460654016 weight_movement_bytes=18874368 total_nvlink_bytes=479528384 weight_to_token_ratio=0.040973 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.386619034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=272596992 link_bytes=272596992 token_bytes=469925888 weight_movement_bytes=37748736 total_nvlink_bytes=507674624 weight_to_token_ratio=0.080329 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611890251e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64832 payload_rows_remote=56640 dup_rows=704 dup_rows_remote=704 dedup_ratio=0.010742 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7488, 8192] dup_rows_per_dst=[0, 0, 0, 78, 0, 626, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7488, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57504 payload_rows_remote=49314 dup_rows=8032 dup_rows_remote=8028 dedup_ratio=0.122559 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 83, 4095, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 145, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 168, 8192, 8192, 8190, 8190] dup_rows_per_dst=[2, 0, 0, 0, 0, 292, 7734, 4] combine_rows_per_src=[8188, 8192, 8192, 168, 8192, 8192, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=231997440 dispatch_meta_bytes=229376 combine_payload_bytes=231997440 prefetch_bytes=0 reduce_bytes=0 egress_bytes=232226816 ingress_bytes=231997440 link_bytes=232226816 token_bytes=464224256 weight_movement_bytes=0 total_nvlink_bytes=464224256 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225083088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64378 payload_rows_remote=56186 dup_rows=1158 dup_rows_remote=1158 dedup_ratio=0.017670 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7034, 8192] dup_rows_per_dst=[0, 0, 0, 184, 0, 974, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7034, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=201990144 dispatch_meta_bytes=229368 combine_payload_bytes=201990144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202219512 ingress_bytes=220864512 link_bytes=220864512 token_bytes=404209656 weight_movement_bytes=18874368 total_nvlink_bytes=423084024 weight_to_token_ratio=0.046695 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.116215082e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229368 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094008 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958648 weight_movement_bytes=37748736 total_nvlink_bytes=507707384 weight_to_token_ratio=0.080324 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 4096, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=230137856 dispatch_meta_bytes=229376 combine_payload_bytes=230137856 prefetch_bytes=0 reduce_bytes=0 egress_bytes=230367232 ingress_bytes=230137856 link_bytes=230367232 token_bytes=460505088 weight_movement_bytes=0 total_nvlink_bytes=460505088 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207265469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64888 payload_rows_remote=56696 dup_rows=648 dup_rows_remote=628 dedup_ratio=0.009888 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 314, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7544, 8192] dup_rows_per_dst=[0, 0, 0, 628, 0, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7544, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64008 payload_rows_remote=55816 dup_rows=1528 dup_rows_remote=1528 dedup_ratio=0.023315 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8178, 6680, 8190] dup_rows_per_dst=[0, 0, 0, 486, 0, 1042, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8178, 6680, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893120 weight_movement_bytes=37748736 total_nvlink_bytes=507641856 weight_to_token_ratio=0.080335 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=232226816 dispatch_meta_bytes=229296 combine_payload_bytes=232226816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232456112 ingress_bytes=251101184 link_bytes=251101184 token_bytes=464682928 weight_movement_bytes=18874368 total_nvlink_bytes=483557296 weight_to_token_ratio=0.040618 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405927996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=2 dedup_ratio=0.000153 router_tokens_per_expert=[0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8188, 8188, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8188, 8188, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57344 payload_rows_remote=49152 dup_rows=8192 dup_rows_remote=8190 dedup_ratio=0.125000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 4092, 4100, 8192] dup_rows_per_dst=[2, 0, 0, 8190, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 4092, 4100, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=228622336 dispatch_meta_bytes=229376 combine_payload_bytes=228622336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228851712 ingress_bytes=247496704 link_bytes=247496704 token_bytes=457474048 weight_movement_bytes=18874368 total_nvlink_bytes=476348416 weight_to_token_ratio=0.041258 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371391642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21762716296 prefetch_bytes=698351616 reduce_bytes=641728512 weight_movement_bytes=1340080128 egress_bytes=10886861448 ingress_bytes=12215934976 link_bytes=12216623104 total_nvlink_bytes=23102796424 weight_to_token_ratio=0.061577 prefetch_share_of_ingress=0.057167 dedup_ratio=0.034720 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=34 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101490 gemm_inner_pad_upper_bound=30480 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.438618159e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229344 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235069408 ingress_bytes=291463168 link_bytes=291463168 token_bytes=469909472 weight_movement_bytes=56623104 total_nvlink_bytes=526532576 weight_to_token_ratio=0.120498 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792656667e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=201326592 dispatch_meta_bytes=229368 combine_payload_bytes=201326592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=201555960 ingress_bytes=239075328 link_bytes=239075328 token_bytes=402882552 weight_movement_bytes=37748736 total_nvlink_bytes=440631288 weight_to_token_ratio=0.093697 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.290702160e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61520 payload_rows_remote=53328 dup_rows=4016 dup_rows_remote=4014 dedup_ratio=0.061279 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4594, 8152, 7814, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 416, 2540, 1058, 0, 2, 0] combine_rows_per_src=[8192, 8192, 4594, 8152, 7814, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20951719056 prefetch_bytes=849346560 reduce_bytes=943718400 weight_movement_bytes=1793064960 egress_bytes=10481245328 ingress_bytes=12263538688 link_bytes=12263768064 total_nvlink_bytes=22744784016 weight_to_token_ratio=0.085581 prefetch_share_of_ingress=0.069258 dedup_ratio=0.063286 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=50 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101418 gemm_inner_pad_upper_bound=30861 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.448028988e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64846 payload_rows_remote=56654 dup_rows=690 dup_rows_remote=588 dedup_ratio=0.010529 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7502, 8192] dup_rows_per_dst=[0, 0, 0, 102, 0, 588, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7502, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21768232704 prefetch_bytes=660602880 reduce_bytes=736100352 weight_movement_bytes=1396703232 egress_bytes=10889592576 ingress_bytes=12275343360 link_bytes=12276948992 total_nvlink_bytes=23164935936 weight_to_token_ratio=0.064162 prefetch_share_of_ingress=0.053815 dedup_ratio=0.032117 prefetch_slots=70 unused_prefetch_slots=1466 slot_utilization=0.045573 reduce_slots=39 remote_demand=35 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102069 gemm_inner_pad_upper_bound=29718 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.450660096e-04 prefetch_bytes_per_gemm_flop=1.318660783e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=218431488 dispatch_meta_bytes=229368 combine_payload_bytes=218431488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218660856 ingress_bytes=275054592 link_bytes=275054592 token_bytes=437092344 weight_movement_bytes=56623104 total_nvlink_bytes=493715448 weight_to_token_ratio=0.129545 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.635437765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110408 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991432 weight_movement_bytes=18874368 total_nvlink_bytes=488865800 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8184] dup_rows_per_dst=[0, 6, 2, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=232054784 dispatch_meta_bytes=228968 combine_payload_bytes=232054784 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=232283752 ingress_bytes=307552256 link_bytes=307552256 token_bytes=464338536 weight_movement_bytes=75497472 total_nvlink_bytes=539836008 weight_to_token_ratio=0.162591 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.946814392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21708991344 prefetch_bytes=754974720 reduce_bytes=792723456 weight_movement_bytes=1547698176 egress_bytes=10859998064 ingress_bytes=12396691456 link_bytes=12396920832 total_nvlink_bytes=23256689520 weight_to_token_ratio=0.071293 prefetch_share_of_ingress=0.060901 dedup_ratio=0.035482 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=42 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102286 gemm_inner_pad_upper_bound=29210 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.474608245e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21424796216 prefetch_bytes=792723456 reduce_bytes=717225984 weight_movement_bytes=1509949440 egress_bytes=10717876792 ingress_bytes=12216868864 link_bytes=12217327616 total_nvlink_bytes=22934745656 weight_to_token_ratio=0.070477 prefetch_share_of_ingress=0.064888 dedup_ratio=0.046900 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=38 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101635 gemm_inner_pad_upper_bound=30353 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.438758790e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235069440 ingress_bytes=234840064 link_bytes=235069440 token_bytes=469909504 weight_movement_bytes=0 total_nvlink_bytes=469909504 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252319713e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20682407648 prefetch_bytes=792723456 reduce_bytes=698351616 weight_movement_bytes=1491075072 egress_bytes=10346561248 ingress_bytes=11826921472 link_bytes=11827609600 total_nvlink_bytes=22173482720 weight_to_token_ratio=0.072094 prefetch_share_of_ingress=0.067027 dedup_ratio=0.073727 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=37 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102648 gemm_inner_pad_upper_bound=28321 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.360965326e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21807373824 prefetch_bytes=698351616 reduce_bytes=905969664 weight_movement_bytes=1604321280 egress_bytes=10909179392 ingress_bytes=12502515712 link_bytes=12502515712 total_nvlink_bytes=23411695104 weight_to_token_ratio=0.073568 prefetch_share_of_ingress=0.055857 dedup_ratio=0.031614 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=48 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.103009 gemm_inner_pad_upper_bound=27940 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.495686540e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62402 payload_rows_remote=55184 dup_rows=3134 dup_rows_remote=1088 dedup_ratio=0.047821 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 463, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6032, 8192, 8192, 7218, 8192] dup_rows_per_dst=[0, 0, 0, 516, 0, 572, 2046, 0] combine_rows_per_src=[8192, 8192, 8192, 6032, 8192, 8192, 7218, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=226033664 dispatch_meta_bytes=225088 combine_payload_bytes=226033664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226258752 ingress_bytes=244908032 link_bytes=244908032 token_bytes=452292416 weight_movement_bytes=18874368 total_nvlink_bytes=471166784 weight_to_token_ratio=0.041730 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.346588260e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21533093952 prefetch_bytes=773849088 reduce_bytes=585105408 weight_movement_bytes=1358954496 egress_bytes=10772025408 ingress_bytes=12120023040 link_bytes=12120711168 total_nvlink_bytes=22892048448 weight_to_token_ratio=0.063110 prefetch_share_of_ingress=0.063849 dedup_ratio=0.042309 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=31 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102937 gemm_inner_pad_upper_bound=27940 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.419472722e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none + [2026-08-29 23:37:46] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 747.6 | throughput per GPU (TFLOP/s/GPU): 119.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.863722E+00 | loss scale: 1.0 | grad norm: 3.292 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44860 payload_rows_remote=38820 dup_rows=20676 dup_rows_remote=17860 dedup_ratio=0.315491 router_tokens_per_expert=[243, 0, 0, 1, 254, 256, 49, 1, 0, 3, 286, 462, 610, 392, 222, 385, 7, 121, 2, 540, 328, 373, 3, 0, 391, 251, 111, 143, 524, 2, 563, 6, 343, 457, 7, 107, 5, 38, 292, 70, 296, 509, 428, 538, 415, 29, 190, 503, 549, 522, 491, 0, 472, 521, 20, 11, 672, 135, 330, 880, 177, 326, 15, 176, 26, 258, 0, 7, 208, 0, 783, 366, 0, 835, 331, 1064, 41, 57, 14, 963, 162, 7, 54, 296, 27, 392, 0, 1, 932, 381, 80, 241, 321, 325, 152, 207, 0, 101, 123, 71, 1, 417, 900, 54, 106, 30, 377, 189, 334, 263, 337, 0, 518, 239, 375, 93, 322, 5, 850, 115, 857, 274, 481, 48, 412, 9, 282, 1] payload_rows_per_dst=[5486, 6106, 5082, 5856, 5526, 6040, 5160, 5604] dup_rows_per_dst=[2602, 2550, 2354, 2978, 2454, 2816, 2464, 2458] combine_rows_per_src=[5486, 6106, 5082, 5856, 5526, 6040, 5160, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45074 payload_rows_remote=39248 dup_rows=20462 dup_rows_remote=17840 dedup_ratio=0.312225 router_tokens_per_expert=[225, 0, 0, 3, 126, 251, 50, 2, 1, 3, 221, 383, 625, 403, 223, 306, 13, 173, 4, 463, 308, 415, 1, 0, 413, 150, 122, 126, 366, 4, 588, 4, 368, 471, 4, 97, 3, 34, 297, 78, 355, 617, 495, 491, 295, 31, 243, 637, 617, 463, 532, 0, 610, 560, 11, 21, 683, 95, 385, 848, 172, 418, 14, 153, 98, 313, 0, 15, 255, 0, 491, 393, 0, 859, 417, 1177, 32, 68, 12, 1083, 178, 18, 55, 281, 11, 321, 0, 1, 685, 389, 74, 209, 255, 426, 127, 206, 0, 173, 134, 73, 2, 330, 887, 55, 146, 13, 288, 150, 450, 260, 473, 0, 402, 410, 310, 82, 295, 0, 827, 111, 817, 147, 628, 31, 413, 2, 340, 0] payload_rows_per_dst=[5696, 5984, 5244, 5826, 5652, 5628, 5222, 5822] dup_rows_per_dst=[2664, 2482, 2554, 2622, 2608, 2498, 2880, 2154] combine_rows_per_src=[5696, 5984, 5244, 5826, 5652, 5628, 5222, 5822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44538 payload_rows_remote=38430 dup_rows=20998 dup_rows_remote=17464 dedup_ratio=0.320404 router_tokens_per_expert=[246, 0, 0, 0, 92, 225, 46, 1, 2, 0, 265, 574, 625, 556, 131, 316, 25, 130, 3, 577, 175, 395, 0, 0, 316, 156, 130, 165, 376, 1, 457, 1, 319, 526, 9, 88, 7, 30, 301, 119, 288, 324, 597, 331, 485, 18, 295, 588, 603, 568, 545, 0, 418, 465, 36, 23, 751, 142, 348, 991, 166, 277, 26, 136, 33, 215, 2, 6, 202, 0, 700, 430, 6, 909, 392, 822, 26, 48, 26, 1192, 278, 15, 81, 270, 29, 462, 1, 0, 851, 315, 85, 163, 340, 470, 178, 279, 1, 71, 155, 58, 0, 424, 735, 69, 100, 26, 468, 329, 484, 198, 489, 2, 455, 182, 355, 68, 368, 4, 852, 112, 827, 136, 395, 18, 363, 2, 390, 0] payload_rows_per_dst=[6108, 5808, 6028, 5464, 4636, 6182, 4824, 5488] dup_rows_per_dst=[3534, 2390, 4266, 2042, 1354, 3156, 2394, 1862] combine_rows_per_src=[6108, 5808, 6028, 5464, 4636, 6182, 4824, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44900 payload_rows_remote=39402 dup_rows=20636 dup_rows_remote=18084 dedup_ratio=0.314880 router_tokens_per_expert=[238, 0, 0, 0, 137, 232, 55, 5, 0, 5, 234, 567, 641, 433, 169, 313, 8, 123, 1, 452, 265, 344, 0, 0, 342, 240, 93, 115, 463, 0, 590, 5, 348, 422, 4, 92, 2, 30, 280, 147, 479, 582, 500, 566, 359, 21, 281, 469, 578, 522, 488, 0, 319, 412, 20, 26, 722, 132, 341, 801, 166, 307, 15, 148, 28, 292, 0, 5, 288, 0, 727, 382, 0, 918, 348, 920, 48, 39, 30, 1108, 247, 22, 87, 344, 27, 327, 2, 2, 806, 303, 72, 165, 350, 416, 118, 170, 0, 104, 128, 85, 3, 434, 1069, 53, 118, 33, 473, 283, 472, 274, 432, 0, 481, 199, 372, 96, 327, 2, 779, 97, 895, 156, 346, 26, 462, 4, 325, 0] payload_rows_per_dst=[5492, 5934, 5236, 5812, 5498, 5850, 5570, 5508] dup_rows_per_dst=[2168, 2364, 2764, 2580, 2552, 2624, 3516, 2068] combine_rows_per_src=[5492, 5934, 5236, 5812, 5498, 5850, 5570, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44958 payload_rows_remote=38846 dup_rows=20578 dup_rows_remote=18014 dedup_ratio=0.313995 router_tokens_per_expert=[223, 1, 0, 0, 155, 257, 59, 4, 0, 3, 207, 402, 663, 466, 210, 321, 16, 124, 0, 524, 290, 443, 0, 2, 328, 211, 98, 152, 411, 4, 548, 15, 426, 420, 7, 77, 4, 44, 345, 181, 255, 618, 449, 432, 302, 40, 239, 420, 706, 379, 477, 0, 603, 464, 21, 13, 851, 141, 301, 900, 177, 281, 13, 145, 94, 236, 0, 9, 285, 1, 728, 342, 1, 838, 367, 1074, 67, 81, 18, 1172, 92, 20, 91, 281, 28, 241, 8, 1, 767, 429, 64, 198, 348, 410, 111, 110, 0, 96, 162, 62, 0, 371, 925, 49, 130, 23, 439, 273, 337, 243, 425, 0, 420, 344, 269, 130, 314, 0, 990, 113, 855, 244, 467, 19, 373, 2, 313, 0] payload_rows_per_dst=[6178, 6112, 5894, 5388, 4880, 5810, 5162, 5534] dup_rows_per_dst=[3266, 2564, 3768, 2054, 1430, 2568, 2736, 2192] combine_rows_per_src=[6178, 6112, 5894, 5388, 4880, 5810, 5162, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44000 payload_rows_remote=37494 dup_rows=21536 dup_rows_remote=17314 dedup_ratio=0.328613 router_tokens_per_expert=[305, 0, 0, 0, 226, 302, 112, 7, 2, 6, 226, 474, 548, 464, 118, 290, 26, 108, 8, 496, 346, 397, 0, 0, 528, 170, 97, 167, 444, 3, 394, 8, 394, 456, 9, 82, 9, 56, 243, 192, 360, 521, 424, 357, 379, 41, 161, 459, 740, 362, 553, 0, 570, 510, 33, 20, 520, 131, 315, 542, 222, 262, 39, 163, 48, 333, 0, 16, 379, 0, 740, 431, 1, 896, 427, 1007, 75, 150, 26, 908, 137, 34, 56, 287, 19, 290, 14, 2, 771, 389, 90, 197, 218, 376, 124, 239, 1, 82, 173, 122, 1, 365, 712, 56, 237, 33, 660, 242, 499, 340, 490, 0, 514, 156, 246, 125, 325, 6, 865, 128, 858, 260, 463, 19, 350, 3, 360, 0] payload_rows_per_dst=[5174, 4842, 4876, 5292, 6654, 4918, 5738, 6506] dup_rows_per_dst=[2070, 1542, 2368, 2108, 4328, 1568, 3330, 4222] combine_rows_per_src=[5174, 4842, 4876, 5292, 6654, 4918, 5738, 6506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44970 payload_rows_remote=39690 dup_rows=20566 dup_rows_remote=17780 dedup_ratio=0.313812 router_tokens_per_expert=[266, 0, 0, 2, 129, 211, 49, 3, 1, 3, 244, 338, 723, 358, 149, 339, 30, 181, 0, 472, 161, 459, 1, 0, 549, 114, 128, 179, 366, 4, 542, 25, 319, 485, 17, 92, 3, 58, 294, 175, 485, 546, 539, 295, 411, 32, 211, 617, 630, 557, 536, 0, 530, 570, 30, 56, 670, 123, 370, 710, 201, 404, 36, 164, 38, 274, 2, 10, 308, 0, 634, 402, 5, 934, 414, 1082, 82, 84, 13, 994, 210, 13, 77, 197, 29, 230, 11, 0, 761, 437, 104, 238, 176, 397, 166, 246, 1, 71, 234, 73, 0, 362, 471, 90, 117, 54, 344, 315, 698, 231, 508, 0, 373, 333, 178, 109, 290, 2, 1109, 98, 734, 107, 328, 12, 327, 3, 436, 0] payload_rows_per_dst=[5784, 5906, 5280, 5874, 5662, 5888, 5336, 5240] dup_rows_per_dst=[2606, 2504, 2786, 2540, 2902, 2914, 2894, 1420] combine_rows_per_src=[5784, 5906, 5280, 5874, 5662, 5888, 5336, 5240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=226720 combine_payload_bytes=159006720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159233440 ingress_bytes=177881088 link_bytes=177881088 token_bytes=318240160 weight_movement_bytes=18874368 total_nvlink_bytes=337114528 weight_to_token_ratio=0.059309 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.704369062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160759808 dispatch_meta_bytes=228352 combine_payload_bytes=160759808 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160988160 ingress_bytes=236257280 link_bytes=236257280 token_bytes=321747968 weight_movement_bytes=75497472 total_nvlink_bytes=397245440 weight_to_token_ratio=0.234648 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.083333 gemm_active_groups=15 gemm_peak_group_rows=4224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.263701011e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=223576 combine_payload_bytes=157409280 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=157632856 ingress_bytes=195158016 link_bytes=195158016 token_bytes=315042136 weight_movement_bytes=37748736 total_nvlink_bytes=352790872 weight_to_token_ratio=0.119821 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869908086e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161390592 dispatch_meta_bytes=229944 combine_payload_bytes=161390592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=161620536 ingress_bytes=218013696 link_bytes=218013696 token_bytes=323011128 weight_movement_bytes=56623104 total_nvlink_bytes=379634232 weight_to_token_ratio=0.175298 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.088899966e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=227440 combine_payload_bytes=159113216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159340656 ingress_bytes=177987584 link_bytes=177987584 token_bytes=318453872 weight_movement_bytes=18874368 total_nvlink_bytes=337328240 weight_to_token_ratio=0.059269 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=7552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.705389455e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=153575424 dispatch_meta_bytes=219232 combine_payload_bytes=153575424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=153794656 ingress_bytes=191324160 link_bytes=191324160 token_bytes=307370080 weight_movement_bytes=37748736 total_nvlink_bytes=345118816 weight_to_token_ratio=0.122812 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.833173964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162570240 dispatch_meta_bytes=229880 combine_payload_bytes=162570240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=162800120 ingress_bytes=200318976 link_bytes=200318976 token_bytes=325370360 weight_movement_bytes=37748736 total_nvlink_bytes=363119096 weight_to_token_ratio=0.116018 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.919357865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53672 payload_rows_remote=45998 dup_rows=11864 dup_rows_remote=7990 dedup_ratio=0.181030 router_tokens_per_expert=[0, 1353, 0, 0, 603, 0, 0, 55, 3620, 0, 0, 0, 3637, 0, 0, 4042, 0, 0, 0, 0, 0, 1, 347, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 3922, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 1087, 0, 132, 0, 0, 0, 43, 281, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 134, 0, 388, 1075, 3987, 1, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 12, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 0] payload_rows_per_dst=[2772, 8030, 7828, 7970, 3754, 7674, 8114, 7530] dup_rows_per_dst=[44, 642, 2406, 1734, 880, 3874, 48, 2236] combine_rows_per_src=[2772, 8030, 7828, 7970, 3754, 7674, 8114, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55196 payload_rows_remote=47188 dup_rows=10340 dup_rows_remote=9036 dedup_ratio=0.157776 router_tokens_per_expert=[0, 836, 0, 0, 1542, 0, 0, 113, 3472, 0, 0, 0, 3615, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 456, 0, 585, 0, 0, 0, 47, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 454, 1371, 3992, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 6, 2, 0, 0, 0, 0, 1, 0, 1, 2, 1, 1, 0, 0, 0, 0, 0, 1721, 0] payload_rows_per_dst=[4550, 8004, 7718, 8008, 7666, 3584, 8162, 7504] dup_rows_per_dst=[432, 498, 2332, 1304, 2686, 168, 26, 2894] combine_rows_per_src=[4550, 8004, 7718, 8008, 7666, 3584, 8162, 7504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55410 payload_rows_remote=48110 dup_rows=10126 dup_rows_remote=8212 dedup_ratio=0.154510 router_tokens_per_expert=[0, 923, 0, 0, 1673, 0, 1, 165, 3687, 0, 0, 0, 3554, 0, 1, 4059, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2048, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 780, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 371, 0, 414, 0, 1, 0, 1, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 272, 1688, 3995, 0, 7, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1124, 0] payload_rows_per_dst=[4284, 8002, 7838, 7936, 7300, 4310, 8132, 7608] dup_rows_per_dst=[492, 302, 3372, 1486, 1914, 530, 12, 2018] combine_rows_per_src=[4284, 8002, 7838, 7936, 7300, 4310, 8132, 7608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55206 payload_rows_remote=48208 dup_rows=10330 dup_rows_remote=8674 dedup_ratio=0.157623 router_tokens_per_expert=[0, 101, 0, 0, 3409, 0, 0, 37, 3075, 0, 0, 0, 2444, 0, 0, 4089, 16, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 355, 0, 0, 0, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 141, 0, 3601, 0, 0, 0, 4, 58, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 107, 0, 49, 3720, 1569, 1, 0, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1097, 0] payload_rows_per_dst=[7364, 3382, 6998, 7158, 7980, 7474, 8182, 6668] dup_rows_per_dst=[2342, 68, 1656, 830, 3388, 290, 72, 1684] combine_rows_per_src=[7364, 3382, 6998, 7158, 7980, 7474, 8182, 6668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=188407808 dispatch_meta_bytes=215952 combine_payload_bytes=188407808 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=188623760 ingress_bytes=263905280 link_bytes=263905280 token_bytes=377031568 weight_movement_bytes=75497472 total_nvlink_bytes=452529040 weight_to_token_ratio=0.200242 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528610543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=41900 payload_rows_remote=33722 dup_rows=23636 dup_rows_remote=8444 dedup_ratio=0.360657 router_tokens_per_expert=[0, 686, 0, 0, 1932, 0, 2, 245, 3329, 0, 2, 0, 2677, 0, 2, 4088, 1, 0, 2, 0, 0, 0, 110, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 449, 0, 0, 0, 0, 3890, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 141, 0, 480, 0, 876, 0, 0, 0, 2, 380, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 52, 0, 203, 3173, 3537, 0, 1, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0] payload_rows_per_dst=[8178, 8126, 4218, 2876, 7308, 512, 8182, 2500] dup_rows_per_dst=[15192, 5524, 0, 230, 2678, 0, 12, 0] combine_rows_per_src=[8178, 8126, 4218, 2876, 7308, 512, 8182, 2500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47956 payload_rows_remote=40278 dup_rows=17580 dup_rows_remote=14982 dedup_ratio=0.268250 router_tokens_per_expert=[0, 775, 0, 0, 1653, 0, 0, 153, 3447, 0, 0, 0, 3215, 10, 0, 3840, 0, 0, 1, 0, 0, 6, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 556, 0, 0, 0, 0, 3860, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 453, 0, 731, 5, 0, 0, 386, 60, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 45, 0, 416, 2480, 3612, 0, 0, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1537, 0] payload_rows_per_dst=[8088, 7678, 2278, 7798, 7630, 3084, 7840, 3560] dup_rows_per_dst=[9770, 2598, 2, 1038, 3762, 166, 80, 164] combine_rows_per_src=[8088, 7678, 2278, 7798, 7630, 3084, 7840, 3560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49716 payload_rows_remote=41848 dup_rows=15820 dup_rows_remote=11714 dedup_ratio=0.241394 router_tokens_per_expert=[0, 376, 0, 0, 2851, 0, 0, 128, 3378, 0, 0, 0, 1373, 0, 0, 4080, 1, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 335, 0, 0, 0, 0, 3863, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 137, 0, 2025, 0, 0, 0, 5, 37, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 107, 0, 535, 2710, 2795, 0, 1, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 0] payload_rows_per_dst=[994, 5674, 6814, 7794, 4354, 8054, 8164, 7868] dup_rows_per_dst=[14, 176, 2032, 666, 190, 8324, 312, 4106] combine_rows_per_src=[994, 5674, 6814, 7794, 4354, 8054, 8164, 7868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=193282048 dispatch_meta_bytes=224896 combine_payload_bytes=193282048 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=193506944 ingress_bytes=212156416 link_bytes=212156416 token_bytes=386788992 weight_movement_bytes=18874368 total_nvlink_bytes=405663360 weight_to_token_ratio=0.048798 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.032778391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44260 payload_rows_remote=38964 dup_rows=21276 dup_rows_remote=18262 dedup_ratio=0.324646 router_tokens_per_expert=[190, 0, 0, 0, 135, 218, 36, 1, 1, 1, 141, 426, 664, 338, 154, 331, 10, 178, 2, 509, 199, 470, 0, 0, 468, 130, 105, 154, 460, 2, 561, 4, 293, 455, 12, 49, 0, 26, 299, 98, 289, 611, 475, 428, 280, 23, 194, 597, 692, 495, 560, 0, 405, 470, 17, 11, 1095, 120, 321, 718, 157, 295, 9, 120, 20, 259, 0, 3, 293, 0, 626, 374, 2, 1183, 346, 1071, 44, 30, 11, 1193, 162, 12, 65, 334, 8, 283, 0, 0, 714, 389, 63, 156, 353, 502, 137, 159, 0, 96, 158, 88, 0, 432, 764, 60, 183, 16, 465, 211, 455, 235, 381, 0, 440, 334, 375, 78, 305, 4, 775, 84, 856, 222, 603, 15, 484, 7, 348, 0] payload_rows_per_dst=[4958, 5870, 4850, 5948, 6024, 5148, 5296, 6166] dup_rows_per_dst=[1750, 2578, 2186, 3152, 3376, 1928, 3014, 3292] combine_rows_per_src=[4958, 5870, 4850, 5948, 6024, 5148, 5296, 6166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=197058560 dispatch_meta_bytes=225288 combine_payload_bytes=197058560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197283848 ingress_bytes=215932928 link_bytes=215932928 token_bytes=394342408 weight_movement_bytes=18874368 total_nvlink_bytes=413216776 weight_to_token_ratio=0.047863 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.068963070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=197459968 dispatch_meta_bytes=227528 combine_payload_bytes=197459968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197687496 ingress_bytes=216334336 link_bytes=216334336 token_bytes=395147464 weight_movement_bytes=18874368 total_nvlink_bytes=414021832 weight_to_token_ratio=0.047765 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.072809164e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=138125312 dispatch_meta_bytes=168664 combine_payload_bytes=138125312 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=138293976 ingress_bytes=251371520 link_bytes=251371520 token_bytes=276419288 weight_movement_bytes=113246208 total_nvlink_bytes=389665496 weight_to_token_ratio=0.409690 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=9856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408518222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=164978688 dispatch_meta_bytes=221040 combine_payload_bytes=164978688 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=165199728 ingress_bytes=202727424 link_bytes=202727424 token_bytes=330178416 weight_movement_bytes=37748736 total_nvlink_bytes=367927152 weight_to_token_ratio=0.114328 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.942434429e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171409408 dispatch_meta_bytes=214248 combine_payload_bytes=171409408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171623656 ingress_bytes=190283776 link_bytes=190283776 token_bytes=343033064 weight_movement_bytes=18874368 total_nvlink_bytes=361907432 weight_to_token_ratio=0.055022 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.823205516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57254 payload_rows_remote=49356 dup_rows=8282 dup_rows_remote=5692 dedup_ratio=0.126373 router_tokens_per_expert=[0, 0, 0, 1326, 0, 0, 3892, 0, 0, 0, 127, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 278, 473, 0, 0, 3919, 0, 5, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 3525, 1, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 20, 0, 0, 0, 0, 0, 3911, 0, 0, 1313, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 3900, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[3222, 8062, 7050, 7804, 7730, 7898, 7970, 7518] dup_rows_per_dst=[182, 1392, 2, 636, 4, 2590, 2270, 1206] combine_rows_per_src=[3222, 8062, 7050, 7804, 7730, 7898, 7970, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=228904 combine_payload_bytes=159596544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159825448 ingress_bytes=178470912 link_bytes=178470912 token_bytes=319421992 weight_movement_bytes=18874368 total_nvlink_bytes=338296360 weight_to_token_ratio=0.059089 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.710020466e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=202162176 dispatch_meta_bytes=220192 combine_payload_bytes=202162176 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=202382368 ingress_bytes=277659648 link_bytes=277659648 token_bytes=404544544 weight_movement_bytes=75497472 total_nvlink_bytes=480042016 weight_to_token_ratio=0.186623 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.660398130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55364 payload_rows_remote=47266 dup_rows=10172 dup_rows_remote=9780 dedup_ratio=0.155212 router_tokens_per_expert=[0, 0, 0, 2239, 0, 0, 4047, 0, 0, 0, 112, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 227, 45, 0, 0, 3972, 0, 6, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 0, 0, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 1479, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 3496, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 139, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[5006, 8046, 7980, 8098, 8166, 2958, 7816, 7294] dup_rows_per_dst=[392, 566, 4638, 392, 16, 0, 4002, 166] combine_rows_per_src=[5006, 8046, 7980, 8098, 8166, 2958, 7816, 7294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56982 payload_rows_remote=48800 dup_rows=8554 dup_rows_remote=7482 dedup_ratio=0.130524 router_tokens_per_expert=[0, 0, 0, 2213, 0, 1, 4092, 0, 0, 0, 101, 3, 0, 0, 0, 574, 0, 0, 0, 1, 0, 0, 87, 1, 1, 0, 3956, 0, 9, 0, 0, 102, 0, 1, 0, 0, 0, 0, 0, 3494, 1, 0, 0, 0, 1, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 3504, 0, 0, 1147, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 3484, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[5264, 8004, 5856, 8186, 8182, 6480, 7886, 7124] dup_rows_per_dst=[520, 310, 278, 410, 1072, 1428, 4366, 170] combine_rows_per_src=[5264, 8004, 5856, 8186, 8182, 6480, 7886, 7124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55758 payload_rows_remote=47574 dup_rows=9778 dup_rows_remote=6856 dedup_ratio=0.149200 router_tokens_per_expert=[0, 0, 0, 3893, 0, 0, 4059, 0, 0, 0, 761, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 1, 637, 0, 0, 3238, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 0, 0, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 78, 1, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 2989, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 2345, 0, 0, 1, 0, 0, 185, 0, 0, 2, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 184, 0, 0] payload_rows_per_dst=[8068, 6758, 8184, 8118, 7936, 5980, 6034, 4680] dup_rows_per_dst=[4284, 1034, 2922, 130, 160, 0, 1042, 206] combine_rows_per_src=[8068, 6758, 8184, 8118, 7936, 5980, 6034, 4680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51420 payload_rows_remote=43258 dup_rows=14116 dup_rows_remote=8336 dedup_ratio=0.215393 router_tokens_per_expert=[0, 0, 0, 1019, 0, 0, 3646, 0, 0, 0, 346, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 722, 359, 0, 0, 4006, 0, 27, 0, 0, 127, 2, 0, 0, 0, 0, 0, 0, 3131, 0, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 1536, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 3565, 0, 0, 22, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[6042, 8162, 8170, 7706, 8142, 3072, 3550, 6576] dup_rows_per_dst=[844, 5780, 6118, 376, 40, 0, 694, 264] combine_rows_per_src=[6042, 8162, 8170, 7706, 8142, 3072, 3550, 6576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52916 payload_rows_remote=44856 dup_rows=12620 dup_rows_remote=5308 dedup_ratio=0.192566 router_tokens_per_expert=[0, 0, 0, 2687, 0, 2, 4083, 0, 0, 0, 253, 0, 0, 0, 0, 842, 0, 0, 0, 0, 0, 0, 73, 243, 2, 0, 3748, 0, 8, 1, 0, 48, 0, 0, 0, 1, 0, 0, 0, 3236, 2, 0, 0, 0, 2, 2426, 0, 1, 0, 0, 0, 0, 1, 0, 135, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1, 0, 0, 0, 0, 3753, 0, 0, 1891, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 3274, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 311, 0, 0, 0, 0, 20, 0, 0] payload_rows_per_dst=[8060, 7776, 8084, 6914, 7988, 3784, 3428, 6882] dup_rows_per_dst=[7312, 470, 4286, 272, 26, 0, 0, 254] combine_rows_per_src=[8060, 7776, 8084, 6914, 7988, 3784, 3428, 6882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47886 payload_rows_remote=39960 dup_rows=17650 dup_rows_remote=12640 dedup_ratio=0.269318 router_tokens_per_expert=[0, 0, 0, 3925, 0, 0, 4062, 0, 0, 0, 98, 5, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 21, 993, 0, 0, 3319, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 2611, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 191, 2808, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[7916, 2030, 0, 8126, 7780, 8190, 5918, 7926] dup_rows_per_dst=[624, 0, 0, 58, 82, 11670, 206, 5010] combine_rows_per_src=[7916, 2030, 0, 8126, 7780, 8190, 5918, 7926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193601536 dispatch_meta_bytes=228184 combine_payload_bytes=193601536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193829720 ingress_bytes=212475904 link_bytes=212475904 token_bytes=387431256 weight_movement_bytes=18874368 total_nvlink_bytes=406305624 weight_to_token_ratio=0.048717 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035839567e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=199884800 dispatch_meta_bytes=225128 combine_payload_bytes=199884800 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=200109928 ingress_bytes=237633536 link_bytes=237633536 token_bytes=399994728 weight_movement_bytes=37748736 total_nvlink_bytes=437743464 weight_to_token_ratio=0.094373 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276887619e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=194863104 dispatch_meta_bytes=217720 combine_payload_bytes=194863104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=195080824 ingress_bytes=251486208 link_bytes=251486208 token_bytes=389943928 weight_movement_bytes=56623104 total_nvlink_bytes=446567032 weight_to_token_ratio=0.145208 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409617106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=177184768 dispatch_meta_bytes=206376 combine_payload_bytes=177184768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=177391144 ingress_bytes=214933504 link_bytes=214933504 token_bytes=354575912 weight_movement_bytes=37748736 total_nvlink_bytes=392324648 weight_to_token_ratio=0.106462 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059387081e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183730176 dispatch_meta_bytes=200656 combine_payload_bytes=183730176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=183930832 ingress_bytes=221478912 link_bytes=221478912 token_bytes=367661008 weight_movement_bytes=37748736 total_nvlink_bytes=405409744 weight_to_token_ratio=0.102673 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.122101960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53546 payload_rows_remote=49192 dup_rows=11990 dup_rows_remote=11990 dedup_ratio=0.182953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 882, 0, 0, 3329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 3941, 0, 0, 0, 0, 4095, 0, 0, 27, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2399, 4093, 0, 26] payload_rows_per_dst=[8190, 8184, 1772, 6500, 8192, 4354, 8162, 8192] dup_rows_per_dst=[0, 642, 0, 0, 64, 0, 6380, 4904] combine_rows_per_src=[8190, 8184, 1772, 6500, 8192, 4354, 8162, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=163676160 dispatch_meta_bytes=210400 combine_payload_bytes=163676160 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=163886560 ingress_bytes=201424896 link_bytes=201424896 token_bytes=327562720 weight_movement_bytes=37748736 total_nvlink_bytes=365311456 weight_to_token_ratio=0.115241 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.929954246e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50138 payload_rows_remote=41964 dup_rows=15398 dup_rows_remote=15374 dedup_ratio=0.234955 router_tokens_per_expert=[0, 403, 0, 0, 2161, 0, 0, 107, 3304, 0, 0, 0, 3192, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 351, 0, 0, 0, 1, 3993, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 478, 0, 1409, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 201, 2820, 3209, 0, 2, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1293, 0] payload_rows_per_dst=[1016, 6506, 7334, 8016, 3724, 8164, 8174, 7204] dup_rows_per_dst=[4, 326, 2710, 692, 322, 9328, 24, 1992] combine_rows_per_src=[1016, 6506, 7334, 8016, 3724, 8164, 8174, 7204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=201490432 dispatch_meta_bytes=244728 combine_payload_bytes=201490432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=201735160 ingress_bytes=239239168 link_bytes=239239168 token_bytes=403225592 weight_movement_bytes=37748736 total_nvlink_bytes=440974328 weight_to_token_ratio=0.093617 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.292271995e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59532 payload_rows_remote=51438 dup_rows=6004 dup_rows_remote=1956 dedup_ratio=0.091614 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2444, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 863, 4095, 0, 2] payload_rows_per_dst=[8192, 8188, 6278, 8094, 8192, 7516, 4888, 8184] dup_rows_per_dst=[0, 268, 4, 4048, 6, 0, 0, 1678] combine_rows_per_src=[8192, 8188, 6278, 8094, 8192, 7516, 4888, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171884544 dispatch_meta_bytes=229352 combine_payload_bytes=171884544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=172113896 ingress_bytes=209633280 link_bytes=209633280 token_bytes=343998440 weight_movement_bytes=37748736 total_nvlink_bytes=381747176 weight_to_token_ratio=0.109735 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.008602943e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59822 payload_rows_remote=51630 dup_rows=5714 dup_rows_remote=5714 dedup_ratio=0.087189 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 1, 0, 3206, 0, 0, 3906, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 971, 4086, 0, 0] payload_rows_per_dst=[8192, 8190, 6422, 5910, 8192, 7608, 7126, 8182] dup_rows_per_dst=[10, 14, 10, 1654, 0, 0, 2076, 1950] combine_rows_per_src=[8192, 8190, 6422, 5910, 8192, 7608, 7126, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52610 payload_rows_remote=44426 dup_rows=12926 dup_rows_remote=7686 dedup_ratio=0.197235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 17, 0, 0, 0, 0, 0, 3865, 0, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 0, 29, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 1578, 4096, 0, 15] payload_rows_per_dst=[8190, 8192, 8184, 8146, 8192, 7718, 824, 3164] dup_rows_per_dst=[0, 40, 5240, 7524, 80, 0, 0, 42] combine_rows_per_src=[8190, 8192, 8184, 8146, 8192, 7718, 824, 3164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55996 payload_rows_remote=47804 dup_rows=9540 dup_rows_remote=8254 dedup_ratio=0.145569 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3090, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 2583, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1841, 4096, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8088, 8192, 6272, 5166, 3702] dup_rows_per_dst=[0, 1286, 4614, 3582, 0, 0, 50, 8] combine_rows_per_src=[8192, 8192, 8192, 8088, 8192, 6272, 5166, 3702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56064 payload_rows_remote=47872 dup_rows=9472 dup_rows_remote=9472 dedup_ratio=0.144531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 1, 2, 0, 0, 1, 0, 2828, 0, 0, 3611, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 0, 12, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 2386, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1522, 4094, 0, 38] payload_rows_per_dst=[8192, 8182, 8192, 8046, 8192, 7432, 4776, 3052] dup_rows_per_dst=[0, 2, 5712, 3598, 82, 0, 4, 74] combine_rows_per_src=[8192, 8182, 8192, 8046, 8192, 7432, 4776, 3052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49518 payload_rows_remote=41328 dup_rows=16018 dup_rows_remote=11412 dedup_ratio=0.244415 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 0, 0, 0, 0, 0, 1515, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 0, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2302, 4095, 0, 0] payload_rows_per_dst=[8176, 5912, 3054, 0, 7904, 8170, 8112, 8190] dup_rows_per_dst=[6, 8, 18, 0, 54, 5156, 6170, 4606] combine_rows_per_src=[8176, 5912, 3054, 0, 7904, 8170, 8112, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=210690048 dispatch_meta_bytes=213576 combine_payload_bytes=210690048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210903624 ingress_bytes=267313152 link_bytes=267313152 token_bytes=421593672 weight_movement_bytes=56623104 total_nvlink_bytes=478216776 weight_to_token_ratio=0.134307 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=15744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.561263096e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211476480 dispatch_meta_bytes=229376 combine_payload_bytes=211476480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211705856 ingress_bytes=230350848 link_bytes=230350848 token_bytes=423182336 weight_movement_bytes=18874368 total_nvlink_bytes=442056704 weight_to_token_ratio=0.044601 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207108486e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=181968896 dispatch_meta_bytes=208448 combine_payload_bytes=181968896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=182177344 ingress_bytes=200843264 link_bytes=200843264 token_bytes=364146240 weight_movement_bytes=18874368 total_nvlink_bytes=383020608 weight_to_token_ratio=0.051832 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.924381335e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=195805184 dispatch_meta_bytes=224232 combine_payload_bytes=195805184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196029416 ingress_bytes=214679552 link_bytes=214679552 token_bytes=391834600 weight_movement_bytes=18874368 total_nvlink_bytes=410708968 weight_to_token_ratio=0.048169 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.056953838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61184 payload_rows_remote=53146 dup_rows=4352 dup_rows_remote=2108 dedup_ratio=0.066406 router_tokens_per_expert=[4046, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 108, 0, 0, 848, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4075, 0, 0, 0, 0, 0, 688, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 4090, 0] payload_rows_per_dst=[4724, 8092, 8192, 8148, 8122, 8038, 8150, 7718] dup_rows_per_dst=[0, 2, 426, 1680, 0, 2244, 0, 0] combine_rows_per_src=[4724, 8092, 8192, 8148, 8122, 8038, 8150, 7718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=196083712 dispatch_meta_bytes=229376 combine_payload_bytes=196083712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196313088 ingress_bytes=214958080 link_bytes=214958080 token_bytes=392396800 weight_movement_bytes=18874368 total_nvlink_bytes=411271168 weight_to_token_ratio=0.048100 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059622557e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=169279488 dispatch_meta_bytes=210960 combine_payload_bytes=169279488 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=169490448 ingress_bytes=207028224 link_bytes=207028224 token_bytes=338769936 weight_movement_bytes=37748736 total_nvlink_bytes=376518672 weight_to_token_ratio=0.111429 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.983642578e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54366 payload_rows_remote=47006 dup_rows=11170 dup_rows_remote=8176 dedup_ratio=0.170441 router_tokens_per_expert=[0, 0, 0, 2036, 0, 0, 4096, 0, 0, 0, 160, 0, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 59, 82, 0, 0, 3748, 0, 1, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 0, 0, 0, 0, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 1509, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 2219, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[6434, 6822, 1968, 8192, 8180, 7842, 7360, 7568] dup_rows_per_dst=[1366, 324, 0, 118, 32, 3644, 2994, 2692] combine_rows_per_src=[6434, 6822, 1968, 8192, 8180, 7842, 7360, 7568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=217686016 dispatch_meta_bytes=221016 combine_payload_bytes=217686016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217907032 ingress_bytes=236560384 link_bytes=236560384 token_bytes=435593048 weight_movement_bytes=18874368 total_nvlink_bytes=454467416 weight_to_token_ratio=0.043330 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.266605205e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=192536576 dispatch_meta_bytes=220728 combine_payload_bytes=192536576 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=192757304 ingress_bytes=230285312 link_bytes=230285312 token_bytes=385293880 weight_movement_bytes=37748736 total_nvlink_bytes=423042616 weight_to_token_ratio=0.097974 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.206480552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60202 payload_rows_remote=52096 dup_rows=5334 dup_rows_remote=5334 dedup_ratio=0.081390 router_tokens_per_expert=[4095, 0, 0, 0, 1, 0, 0, 0, 4043, 0, 0, 0, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 4095, 0, 0, 0, 0, 0, 641, 0, 0, 0, 228, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4087, 0] payload_rows_per_dst=[3096, 8192, 8180, 8190, 8106, 8146, 8192, 8100] dup_rows_per_dst=[0, 22, 4, 3628, 0, 1680, 0, 0] combine_rows_per_src=[3096, 8192, 8180, 8190, 8106, 8146, 8192, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61552 payload_rows_remote=53378 dup_rows=3984 dup_rows_remote=1824 dedup_ratio=0.060791 router_tokens_per_expert=[4093, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 0, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1088, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4094, 0, 0, 0, 0, 0, 597, 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4087, 0] payload_rows_per_dst=[4632, 8190, 8176, 8174, 8044, 8052, 8188, 8096] dup_rows_per_dst=[0, 44, 38, 2160, 0, 1742, 0, 0] combine_rows_per_src=[4632, 8190, 8176, 8174, 8044, 8052, 8188, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59616 payload_rows_remote=51428 dup_rows=5920 dup_rows_remote=5652 dedup_ratio=0.090332 router_tokens_per_expert=[4085, 0, 0, 0, 0, 0, 0, 0, 3635, 0, 0, 0, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 27, 0, 0, 182, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 0, 4093, 0, 0, 0, 0, 0, 3012, 0, 2, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 4093, 0] payload_rows_per_dst=[7710, 8176, 8188, 8118, 5532, 6278, 8188, 7426] dup_rows_per_dst=[4584, 2, 268, 368, 0, 696, 2, 0] combine_rows_per_src=[7710, 8176, 8188, 8118, 5532, 6278, 8188, 7426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58412 payload_rows_remote=50226 dup_rows=7124 dup_rows_remote=6598 dedup_ratio=0.108704 router_tokens_per_expert=[3997, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 0, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 217, 0, 0, 217, 0, 0, 3640, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 4095, 0, 0, 0, 0, 0, 420, 0, 3, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3871, 4096, 0] payload_rows_per_dst=[7414, 8186, 8192, 7726, 8010, 2932, 8190, 7762] dup_rows_per_dst=[5312, 526, 796, 426, 0, 62, 2, 0] combine_rows_per_src=[7414, 8186, 8192, 7726, 8010, 2932, 8190, 7762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52720 payload_rows_remote=44528 dup_rows=12816 dup_rows_remote=3388 dedup_ratio=0.195557 router_tokens_per_expert=[4091, 0, 0, 0, 2, 0, 0, 0, 3652, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 1, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 1319, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 4094, 0, 0, 0, 0, 0, 1231, 0, 0, 0, 510, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3884, 4094, 0] payload_rows_per_dst=[8192, 7706, 8192, 8158, 1610, 2898, 8192, 7772] dup_rows_per_dst=[9428, 0, 162, 2638, 0, 588, 0, 0] combine_rows_per_src=[8192, 7706, 8192, 8158, 1610, 2898, 8192, 7772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51094 payload_rows_remote=43664 dup_rows=14442 dup_rows_remote=11242 dedup_ratio=0.220367 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 332, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 4096, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 4096, 0] payload_rows_per_dst=[3806, 8192, 6444, 666, 8192, 8172, 8192, 7430] dup_rows_per_dst=[0, 2, 84, 0, 5980, 5128, 48, 3200] combine_rows_per_src=[3806, 8192, 6444, 666, 8192, 8172, 8192, 7430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=213385216 dispatch_meta_bytes=229720 combine_payload_bytes=213385216 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=213614936 ingress_bytes=307757056 link_bytes=307757056 token_bytes=427000152 weight_movement_bytes=94371840 total_nvlink_bytes=521371992 weight_to_token_ratio=0.221011 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.948776685e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=218636288 dispatch_meta_bytes=220808 combine_payload_bytes=218636288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218857096 ingress_bytes=237510656 link_bytes=237510656 token_bytes=437493384 weight_movement_bytes=18874368 total_nvlink_bytes=456367752 weight_to_token_ratio=0.043142 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.275710243e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=210649088 dispatch_meta_bytes=228320 combine_payload_bytes=210649088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210877408 ingress_bytes=229523456 link_bytes=229523456 token_bytes=421526496 weight_movement_bytes=18874368 total_nvlink_bytes=440400864 weight_to_token_ratio=0.044776 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.199180823e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57014 payload_rows_remote=48822 dup_rows=8522 dup_rows_remote=8500 dedup_ratio=0.130035 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 2416, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 0] payload_rows_per_dst=[390, 8190, 8190, 8192, 8192, 8192, 7600, 8068] dup_rows_per_dst=[0, 34, 1310, 0, 5582, 22, 0, 1574] combine_rows_per_src=[390, 8190, 8190, 8192, 8192, 8192, 7600, 8068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=205725696 dispatch_meta_bytes=227296 combine_payload_bytes=205725696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205952992 ingress_bytes=224600064 link_bytes=224600064 token_bytes=411678688 weight_movement_bytes=18874368 total_nvlink_bytes=430553056 weight_to_token_ratio=0.045847 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.152007303e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=182386688 dispatch_meta_bytes=191664 combine_payload_bytes=182386688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=182578352 ingress_bytes=239009792 link_bytes=239009792 token_bytes=364965040 weight_movement_bytes=56623104 total_nvlink_bytes=421588144 weight_to_token_ratio=0.155147 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=18816 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.290074227e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=178847744 dispatch_meta_bytes=219624 combine_payload_bytes=178847744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=179067368 ingress_bytes=216596480 link_bytes=216596480 token_bytes=357915112 weight_movement_bytes=37748736 total_nvlink_bytes=395663848 weight_to_token_ratio=0.105468 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.075320899e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56798 payload_rows_remote=48630 dup_rows=8738 dup_rows_remote=5104 dedup_ratio=0.133331 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 26, 0, 0, 0, 0, 0, 3798, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 628, 4096, 0, 0] payload_rows_per_dst=[8192, 8190, 7624, 346, 8192, 7894, 8168, 8192] dup_rows_per_dst=[4, 46, 220, 0, 2, 3416, 3634, 1416] combine_rows_per_src=[8192, 8190, 7624, 346, 8192, 7894, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=199974912 dispatch_meta_bytes=229288 combine_payload_bytes=199974912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200204200 ingress_bytes=218849280 link_bytes=218849280 token_bytes=400179112 weight_movement_bytes=18874368 total_nvlink_bytes=419053480 weight_to_token_ratio=0.047165 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.096906120e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60292 payload_rows_remote=53526 dup_rows=5244 dup_rows_remote=2770 dedup_ratio=0.080017 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 1, 0, 373, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 597, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[4396, 8192, 8182, 8192, 6766, 8192, 8182, 8190] dup_rows_per_dst=[32, 72, 748, 0, 2474, 1196, 0, 722] combine_rows_per_src=[4396, 8192, 8182, 8192, 6766, 8192, 8182, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62606 payload_rows_remote=54414 dup_rows=2930 dup_rows_remote=2930 dedup_ratio=0.044708 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 27, 0, 1, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 2722, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8190, 8186, 8192, 5542, 8192, 7966, 8182] dup_rows_per_dst=[54, 40, 708, 0, 48, 1228, 0, 852] combine_rows_per_src=[8156, 8190, 8186, 8192, 5542, 8192, 7966, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199188480 dispatch_meta_bytes=214936 combine_payload_bytes=199188480 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199403416 ingress_bytes=255811584 link_bytes=255811584 token_bytes=398591896 weight_movement_bytes=56623104 total_nvlink_bytes=455215000 weight_to_token_ratio=0.142058 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.451060731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58218 payload_rows_remote=50026 dup_rows=7318 dup_rows_remote=6806 dedup_ratio=0.111664 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 3795, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 983, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3008, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0] payload_rows_per_dst=[7758, 8186, 8192, 8192, 2292, 7542, 8058, 7998] dup_rows_per_dst=[1694, 22, 512, 0, 12, 5078, 0, 0] combine_rows_per_src=[7758, 8186, 8192, 8192, 2292, 7542, 8058, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52608 payload_rows_remote=44436 dup_rows=12928 dup_rows_remote=12822 dedup_ratio=0.197266 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 305, 0, 4, 0, 4078, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 1614, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 6994, 8192, 4364, 1330, 7220, 8144] dup_rows_per_dst=[9970, 106, 1718, 0, 80, 0, 0, 1054] combine_rows_per_src=[8192, 8172, 6994, 8192, 4364, 1330, 7220, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46790 payload_rows_remote=38602 dup_rows=18746 dup_rows_remote=2272 dedup_ratio=0.286041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 79, 0, 1, 0, 4065, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 2, 0, 278, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 165, 2, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 4072, 2, 0, 0, 0, 0, 0, 0, 0, 0, 388, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8188, 3608, 8192, 4646, 2252, 8154, 3562] dup_rows_per_dst=[16474, 24, 92, 0, 12, 4, 2140, 0] combine_rows_per_src=[8188, 8188, 3608, 8192, 4646, 2252, 8154, 3562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57130 payload_rows_remote=48940 dup_rows=8406 dup_rows_remote=7654 dedup_ratio=0.128265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[188, 7878, 8192, 8192, 8192, 8192, 8106, 8190] dup_rows_per_dst=[0, 20, 342, 0, 6406, 886, 0, 752] combine_rows_per_src=[188, 7878, 8192, 8192, 8192, 8192, 8106, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=219242496 dispatch_meta_bytes=225184 combine_payload_bytes=219242496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219467680 ingress_bytes=238116864 link_bytes=238116864 token_bytes=438710176 weight_movement_bytes=18874368 total_nvlink_bytes=457584544 weight_to_token_ratio=0.043022 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.281518630e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=222879744 dispatch_meta_bytes=229376 combine_payload_bytes=222879744 prefetch_bytes=0 reduce_bytes=0 egress_bytes=223109120 ingress_bytes=222879744 link_bytes=223109120 token_bytes=445988864 weight_movement_bytes=0 total_nvlink_bytes=445988864 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.137721811e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=204906496 dispatch_meta_bytes=227328 combine_payload_bytes=204906496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205133824 ingress_bytes=242655232 link_bytes=242655232 token_bytes=410040320 weight_movement_bytes=37748736 total_nvlink_bytes=447789056 weight_to_token_ratio=0.092061 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.325003039e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=182009856 dispatch_meta_bytes=229032 combine_payload_bytes=182009856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=182238888 ingress_bytes=200884224 link_bytes=200884224 token_bytes=364248744 weight_movement_bytes=18874368 total_nvlink_bytes=383123112 weight_to_token_ratio=0.051817 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.924773793e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158113792 dispatch_meta_bytes=163496 combine_payload_bytes=158113792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158277288 ingress_bytes=214736896 link_bytes=214736896 token_bytes=316391080 weight_movement_bytes=56623104 total_nvlink_bytes=373014184 weight_to_token_ratio=0.178966 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.057503280e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64986 payload_rows_remote=56822 dup_rows=550 dup_rows_remote=550 dedup_ratio=0.008392 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 4072, 0, 11, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1] payload_rows_per_dst=[8192, 8188, 8192, 7856, 8186, 8164, 8192, 8016] dup_rows_per_dst=[0, 0, 0, 0, 166, 0, 382, 2] combine_rows_per_src=[8192, 8188, 8192, 7856, 8186, 8164, 8192, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200458240 dispatch_meta_bytes=226376 combine_payload_bytes=200458240 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=200684616 ingress_bytes=257081344 link_bytes=257081344 token_bytes=401142856 weight_movement_bytes=56623104 total_nvlink_bytes=457765960 weight_to_token_ratio=0.141154 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.463226946e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60066 payload_rows_remote=51874 dup_rows=5470 dup_rows_remote=5470 dedup_ratio=0.083466 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 659, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 4096, 0, 0, 0, 0, 0, 415, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4088, 0] payload_rows_per_dst=[6638, 8192, 8178, 4722, 7972, 8140, 8192, 8032] dup_rows_per_dst=[0, 12, 16, 734, 3968, 740, 0, 0] combine_rows_per_src=[6638, 8192, 8178, 4722, 7972, 8140, 8192, 8032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=232742912 dispatch_meta_bytes=229488 combine_payload_bytes=232742912 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232972400 ingress_bytes=270491648 link_bytes=270491648 token_bytes=465715312 weight_movement_bytes=37748736 total_nvlink_bytes=503464048 weight_to_token_ratio=0.081055 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591717881e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65170 payload_rows_remote=57036 dup_rows=366 dup_rows_remote=366 dedup_ratio=0.005585 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4041, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4094, 0, 0, 0, 0, 179, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2] payload_rows_per_dst=[8190, 8084, 8192, 8036, 8134, 8190, 8192, 8152] dup_rows_per_dst=[0, 2, 0, 2, 0, 0, 358, 4] combine_rows_per_src=[8190, 8084, 8192, 8036, 8134, 8190, 8192, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=212475904 dispatch_meta_bytes=229376 combine_payload_bytes=212475904 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=212705280 ingress_bytes=250224640 link_bytes=250224640 token_bytes=425181184 weight_movement_bytes=37748736 total_nvlink_bytes=462929920 weight_to_token_ratio=0.088783 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397529382e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64796 payload_rows_remote=57150 dup_rows=740 dup_rows_remote=740 dedup_ratio=0.011292 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4094, 0, 3, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8104, 8190, 7646, 8160, 8192, 8192, 8120] dup_rows_per_dst=[0, 0, 0, 0, 20, 6, 710, 4] combine_rows_per_src=[8192, 8104, 8190, 7646, 8160, 8192, 8192, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59478 payload_rows_remote=51340 dup_rows=6058 dup_rows_remote=1080 dedup_ratio=0.092438 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 4085, 0, 15, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4] payload_rows_per_dst=[8192, 8138, 8192, 6510, 8184, 8192, 4384, 7686] dup_rows_per_dst=[0, 4978, 0, 0, 362, 22, 688, 8] combine_rows_per_src=[8192, 8138, 8192, 6510, 8184, 8192, 4384, 7686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62012 payload_rows_remote=53826 dup_rows=3524 dup_rows_remote=3524 dedup_ratio=0.053772 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 4087, 0, 4, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 10] payload_rows_per_dst=[8192, 5924, 8186, 7358, 8142, 8186, 8192, 7832] dup_rows_per_dst=[0, 0, 0, 0, 36, 4, 3464, 20] combine_rows_per_src=[8192, 5924, 8186, 7358, 8142, 8186, 8192, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58098 payload_rows_remote=49910 dup_rows=7438 dup_rows_remote=7414 dedup_ratio=0.113495 router_tokens_per_expert=[0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3688, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 3543, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3964, 0, 0, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4091, 0, 2, 0, 0, 1230, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8192, 8192, 7092, 7932, 8176, 2464, 7862] dup_rows_per_dst=[24, 7384, 0, 4, 20, 0, 4, 2] combine_rows_per_src=[8188, 8192, 8192, 7092, 7932, 8176, 2464, 7862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56252 payload_rows_remote=48976 dup_rows=9284 dup_rows_remote=6940 dedup_ratio=0.141663 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21] payload_rows_per_dst=[8170, 4, 8192, 8036, 8192, 8190, 8192, 7276] dup_rows_per_dst=[4, 0, 8, 5060, 130, 22, 1716, 2344] combine_rows_per_src=[8170, 4, 8192, 8036, 8192, 8190, 8192, 7276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=233619456 dispatch_meta_bytes=229608 combine_payload_bytes=233619456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233849064 ingress_bytes=252493824 link_bytes=252493824 token_bytes=467468520 weight_movement_bytes=18874368 total_nvlink_bytes=486342888 weight_to_token_ratio=0.040376 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419271587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=234086400 dispatch_meta_bytes=231560 combine_payload_bytes=234086400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234317960 ingress_bytes=252960768 link_bytes=252960768 token_bytes=468404360 weight_movement_bytes=18874368 total_nvlink_bytes=487278728 weight_to_token_ratio=0.040295 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423745615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=210288640 dispatch_meta_bytes=209680 combine_payload_bytes=210288640 prefetch_bytes=18874368 reduce_bytes=113246208 egress_bytes=210498320 ingress_bytes=342409216 link_bytes=342409216 token_bytes=420786960 weight_movement_bytes=132120576 total_nvlink_bytes=552907536 weight_to_token_ratio=0.313984 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.280796632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=220471296 dispatch_meta_bytes=229400 combine_payload_bytes=220471296 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=220700696 ingress_bytes=258220032 link_bytes=258220032 token_bytes=441171992 weight_movement_bytes=37748736 total_nvlink_bytes=478920728 weight_to_token_ratio=0.085565 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.474137294e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=204431360 dispatch_meta_bytes=229296 combine_payload_bytes=204431360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204660656 ingress_bytes=223305728 link_bytes=223305728 token_bytes=409092016 weight_movement_bytes=18874368 total_nvlink_bytes=427966384 weight_to_token_ratio=0.046137 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.139605612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55202 payload_rows_remote=47010 dup_rows=10334 dup_rows_remote=8040 dedup_ratio=0.157684 router_tokens_per_expert=[6, 0, 0, 0, 0, 129, 3047, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4073, 1513, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 0, 0, 0, 17, 0, 0, 0, 0, 19, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0] payload_rows_per_dst=[6222, 8192, 28, 8192, 8008, 8192, 8176, 8192] dup_rows_per_dst=[144, 0, 0, 1894, 2924, 2294, 3006, 72] combine_rows_per_src=[6222, 8192, 28, 8192, 8008, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200605696 dispatch_meta_bytes=223664 combine_payload_bytes=200605696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200829360 ingress_bytes=219480064 link_bytes=219480064 token_bytes=401435056 weight_movement_bytes=18874368 total_nvlink_bytes=420309424 weight_to_token_ratio=0.047017 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102949982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57294 payload_rows_remote=49184 dup_rows=8242 dup_rows_remote=8242 dedup_ratio=0.125763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3963, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[44, 8192, 8192, 8192, 8180, 8192, 8110, 8192] dup_rows_per_dst=[0, 30, 370, 0, 2748, 5038, 0, 56] combine_rows_per_src=[44, 8192, 8192, 8192, 8180, 8192, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=192552960 dispatch_meta_bytes=220200 combine_payload_bytes=192552960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=192773160 ingress_bytes=230301696 link_bytes=230301696 token_bytes=385326120 weight_movement_bytes=37748736 total_nvlink_bytes=423074856 weight_to_token_ratio=0.097966 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.206637536e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58286 payload_rows_remote=51430 dup_rows=7250 dup_rows_remote=5354 dedup_ratio=0.110626 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 3385, 1, 0, 4059, 0, 0, 0, 0, 0, 0, 368, 0, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 296, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0] payload_rows_per_dst=[6408, 8192, 4078, 8178, 6856, 8192, 8190, 8192] dup_rows_per_dst=[0, 6, 0, 3670, 1896, 598, 1006, 74] combine_rows_per_src=[6408, 8192, 4078, 8178, 6856, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58428 payload_rows_remote=50384 dup_rows=7108 dup_rows_remote=2438 dedup_ratio=0.108459 router_tokens_per_expert=[7, 0, 0, 0, 0, 19, 2654, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4085, 2542, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 71, 0, 0, 0, 0, 0] payload_rows_per_dst=[5354, 8192, 5240, 8044, 7038, 8192, 8176, 8192] dup_rows_per_dst=[18, 0, 4, 4670, 0, 202, 2058, 156] combine_rows_per_src=[5354, 8192, 5240, 8044, 7038, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51708 payload_rows_remote=43516 dup_rows=13828 dup_rows_remote=13828 dedup_ratio=0.210999 router_tokens_per_expert=[10, 0, 0, 0, 0, 417, 2736, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4009, 2557, 0, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 84, 0, 0, 0, 0, 375, 0, 0, 0, 12, 0, 0, 53, 0, 0, 0, 0, 0] payload_rows_per_dst=[7672, 8192, 8192, 2858, 7422, 2480, 6700, 8192] dup_rows_per_dst=[5278, 0, 7204, 0, 0, 34, 456, 856] combine_rows_per_src=[7672, 8192, 8192, 2858, 7422, 2480, 6700, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=201457664 dispatch_meta_bytes=229704 combine_payload_bytes=201457664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201687368 ingress_bytes=220332032 link_bytes=220332032 token_bytes=403145032 weight_movement_bytes=18874368 total_nvlink_bytes=422019400 weight_to_token_ratio=0.046818 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.111113120e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57586 payload_rows_remote=49408 dup_rows=7950 dup_rows_remote=1140 dedup_ratio=0.121307 router_tokens_per_expert=[0, 0, 0, 0, 0, 41, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4083, 3521, 0, 0, 3972, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 449, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 10, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 79, 0, 0, 0, 0, 0] payload_rows_per_dst=[6866, 8190, 8178, 6746, 3044, 8192, 8182, 8188] dup_rows_per_dst=[28, 0, 6810, 0, 0, 900, 50, 162] combine_rows_per_src=[6866, 8190, 8178, 6746, 3044, 8192, 8182, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=40200 payload_rows_remote=32012 dup_rows=25336 dup_rows_remote=17610 dedup_ratio=0.386597 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4063, 3549, 2, 0, 4062, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2, 0, 0, 536, 0, 0, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 3002, 5120, 1076, 82, 6348] dup_rows_per_dst=[7726, 2, 17554, 12, 0, 14, 0, 28] combine_rows_per_src=[8188, 8192, 8192, 3002, 5120, 1076, 82, 6348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50716 payload_rows_remote=42524 dup_rows=14820 dup_rows_remote=14352 dedup_ratio=0.226135 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2418, 0, 0, 4062, 24, 0, 0, 0, 0, 0, 2, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0] payload_rows_per_dst=[6636, 8184, 56, 3126, 8138, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 36, 4242, 8684, 1388, 468] combine_rows_per_src=[6636, 8184, 56, 3126, 8138, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=210657280 dispatch_meta_bytes=227136 combine_payload_bytes=210657280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210884416 ingress_bytes=229531648 link_bytes=229531648 token_bytes=421541696 weight_movement_bytes=18874368 total_nvlink_bytes=440416064 weight_to_token_ratio=0.044775 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.199259315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=206372864 dispatch_meta_bytes=211288 combine_payload_bytes=206372864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206584152 ingress_bytes=225247232 link_bytes=225247232 token_bytes=412957016 weight_movement_bytes=18874368 total_nvlink_bytes=431831384 weight_to_token_ratio=0.045705 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.158208148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=178241536 dispatch_meta_bytes=229376 combine_payload_bytes=178241536 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=178470912 ingress_bytes=197115904 link_bytes=197115904 token_bytes=356712448 weight_movement_bytes=18874368 total_nvlink_bytes=375586816 weight_to_token_ratio=0.052912 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.888667605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=202375168 dispatch_meta_bytes=202192 combine_payload_bytes=202375168 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=202577360 ingress_bytes=315621376 link_bytes=315621376 token_bytes=404952528 weight_movement_bytes=113246208 total_nvlink_bytes=518198736 weight_to_token_ratio=0.279653 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.024128729e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=131121152 dispatch_meta_bytes=198488 combine_payload_bytes=131121152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=131319640 ingress_bytes=149995520 link_bytes=149995520 token_bytes=262440792 weight_movement_bytes=18874368 total_nvlink_bytes=281315160 weight_to_token_ratio=0.071919 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.437183270e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62616 payload_rows_remote=54426 dup_rows=2920 dup_rows_remote=2920 dedup_ratio=0.044556 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2972, 3609, 0, 166, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6182, 8192, 7466, 8010, 8192, 8190, 8192, 8192] dup_rows_per_dst=[94, 20, 1170, 1476, 0, 0, 160, 0] combine_rows_per_src=[6182, 8192, 7466, 8010, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=174178304 dispatch_meta_bytes=227504 combine_payload_bytes=174178304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=174405808 ingress_bytes=193052672 link_bytes=193052672 token_bytes=348584112 weight_movement_bytes=18874368 total_nvlink_bytes=367458480 weight_to_token_ratio=0.054146 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.849735715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63292 payload_rows_remote=55100 dup_rows=2244 dup_rows_remote=30 dedup_ratio=0.034241 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12] payload_rows_per_dst=[8192, 6538, 8192, 7930, 7872, 8192, 8192, 8184] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 2214, 24] combine_rows_per_src=[8192, 6538, 8192, 7930, 7872, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=222928896 dispatch_meta_bytes=229384 combine_payload_bytes=222928896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223158280 ingress_bytes=241803264 link_bytes=241803264 token_bytes=446087176 weight_movement_bytes=18874368 total_nvlink_bytes=464961544 weight_to_token_ratio=0.042311 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316839901e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60768 payload_rows_remote=52576 dup_rows=4768 dup_rows_remote=4768 dedup_ratio=0.072754 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 2825, 4092, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 1, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5652, 8190, 8192, 5978, 8192, 8182, 8190, 8192] dup_rows_per_dst=[2, 0, 4382, 378, 0, 0, 4, 2] combine_rows_per_src=[5652, 8190, 8192, 5978, 8192, 8182, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60386 payload_rows_remote=52412 dup_rows=5150 dup_rows_remote=4372 dedup_ratio=0.078583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3288, 4039, 0, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7664, 8192, 3790, 7974, 8192, 8192, 8192, 8190] dup_rows_per_dst=[3856, 12, 358, 778, 0, 0, 146, 0] combine_rows_per_src=[7664, 8192, 3790, 7974, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59864 payload_rows_remote=51672 dup_rows=5672 dup_rows_remote=5568 dedup_ratio=0.086548 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2367, 3382, 0, 88, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 8192, 3092, 7782, 8192, 8136, 8192, 8164] dup_rows_per_dst=[3560, 104, 0, 1810, 0, 0, 198, 0] combine_rows_per_src=[8114, 8192, 3092, 7782, 8192, 8136, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63918 payload_rows_remote=56378 dup_rows=1618 dup_rows_remote=1618 dedup_ratio=0.024689 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 806, 3946, 0, 84, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 7540, 7262, 8192, 8188, 8192, 8188] dup_rows_per_dst=[1508, 12, 0, 34, 0, 0, 64, 0] combine_rows_per_src=[8164, 8192, 7540, 7262, 8192, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=225689600 dispatch_meta_bytes=220520 combine_payload_bytes=225689600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225910120 ingress_bytes=244563968 link_bytes=244563968 token_bytes=451599720 weight_movement_bytes=18874368 total_nvlink_bytes=470474088 weight_to_token_ratio=0.041794 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.343291608e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60726 payload_rows_remote=52606 dup_rows=4810 dup_rows_remote=1328 dedup_ratio=0.073395 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 1882, 3911, 0, 2, 0, 4094, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 736, 0, 0, 1, 0, 0, 0, 0, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8120, 8188, 6008, 6408, 8188, 8192, 7430, 8192] dup_rows_per_dst=[3482, 758, 0, 560, 4, 0, 4, 2] combine_rows_per_src=[8120, 8188, 6008, 6408, 8188, 8192, 7430, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=215351296 dispatch_meta_bytes=229376 combine_payload_bytes=215351296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215580672 ingress_bytes=234225664 link_bytes=234225664 token_bytes=430931968 weight_movement_bytes=18874368 total_nvlink_bytes=449806336 weight_to_token_ratio=0.043799 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.244235066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59812 payload_rows_remote=51620 dup_rows=5724 dup_rows_remote=5694 dedup_ratio=0.087341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1717, 4051, 0, 150, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3496, 7286, 8178, 8084, 8192, 8192, 8192, 8192] dup_rows_per_dst=[10, 6, 3710, 1832, 0, 72, 64, 30] combine_rows_per_src=[3496, 7286, 8178, 8084, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=214679552 dispatch_meta_bytes=227136 combine_payload_bytes=214679552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214906688 ingress_bytes=233553920 link_bytes=233553920 token_bytes=429586240 weight_movement_bytes=18874368 total_nvlink_bytes=448460608 weight_to_token_ratio=0.043936 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.237798746e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211648512 dispatch_meta_bytes=228960 combine_payload_bytes=211648512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=211877472 ingress_bytes=268271616 link_bytes=268271616 token_bytes=423525984 weight_movement_bytes=56623104 total_nvlink_bytes=480149088 weight_to_token_ratio=0.133695 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.570446627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=230924288 dispatch_meta_bytes=231984 combine_payload_bytes=230924288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231156272 ingress_bytes=249798656 link_bytes=249798656 token_bytes=462080560 weight_movement_bytes=18874368 total_nvlink_bytes=480954928 weight_to_token_ratio=0.040846 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393447813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63400 payload_rows_remote=55208 dup_rows=2136 dup_rows_remote=2136 dedup_ratio=0.032593 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7052, 8192, 8192, 7278, 8192, 8192, 8110] dup_rows_per_dst=[42, 0, 2094, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 7052, 8192, 8192, 7278, 8192, 8192, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=215474176 dispatch_meta_bytes=215736 combine_payload_bytes=215474176 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=215689912 ingress_bytes=309846016 link_bytes=309846016 token_bytes=431164088 weight_movement_bytes=94371840 total_nvlink_bytes=525535928 weight_to_token_ratio=0.218877 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968792072e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211435520 dispatch_meta_bytes=229256 combine_payload_bytes=211435520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211664776 ingress_bytes=230309888 link_bytes=230309888 token_bytes=423100296 weight_movement_bytes=18874368 total_nvlink_bytes=441974664 weight_to_token_ratio=0.044610 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.206716027e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=226131968 dispatch_meta_bytes=229376 combine_payload_bytes=226131968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226361344 ingress_bytes=245006336 link_bytes=245006336 token_bytes=452493312 weight_movement_bytes=18874368 total_nvlink_bytes=471367680 weight_to_token_ratio=0.041712 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.347530161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53768 payload_rows_remote=45578 dup_rows=11768 dup_rows_remote=10084 dedup_ratio=0.179565 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 3241, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0] payload_rows_per_dst=[4992, 8192, 0, 8184, 7826, 8192, 8190, 8192] dup_rows_per_dst=[2, 2, 0, 6924, 3022, 102, 1684, 32] combine_rows_per_src=[4992, 8192, 0, 8184, 7826, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64992 payload_rows_remote=56808 dup_rows=544 dup_rows_remote=542 dedup_ratio=0.008301 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3829, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 153, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 21, 0, 1, 0, 97, 0, 0] payload_rows_per_dst=[8190, 7660, 8192, 8192, 8184, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 306, 0, 2, 0, 0, 236] combine_rows_per_src=[8190, 7660, 8192, 8192, 8184, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64664 payload_rows_remote=56472 dup_rows=872 dup_rows_remote=872 dedup_ratio=0.013306 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 3692, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7384, 8192, 8192, 8152, 8192, 8192, 8168] dup_rows_per_dst=[4, 0, 858, 0, 6, 0, 0, 4] combine_rows_per_src=[8192, 7384, 8192, 8192, 8152, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63610 payload_rows_remote=56206 dup_rows=1926 dup_rows_remote=1926 dedup_ratio=0.029388 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7404, 8192, 8190, 7104, 8192, 8192, 8144] dup_rows_per_dst=[4, 0, 1922, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 7404, 8192, 8190, 7104, 8192, 8192, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65198 payload_rows_remote=57006 dup_rows=338 dup_rows_remote=20 dedup_ratio=0.005157 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8138, 8192, 8192, 7928, 8192, 8192, 8172] dup_rows_per_dst=[18, 0, 318, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8138, 8192, 8192, 7928, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=186687488 dispatch_meta_bytes=222648 combine_payload_bytes=186687488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186910136 ingress_bytes=205561856 link_bytes=205561856 token_bytes=373597624 weight_movement_bytes=18874368 total_nvlink_bytes=392471992 weight_to_token_ratio=0.050521 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.969592561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232685568 dispatch_meta_bytes=229400 combine_payload_bytes=232685568 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232914968 ingress_bytes=270434304 link_bytes=270434304 token_bytes=465600536 weight_movement_bytes=37748736 total_nvlink_bytes=503349272 weight_to_token_ratio=0.081075 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591168439e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57834 payload_rows_remote=49646 dup_rows=7702 dup_rows_remote=7698 dedup_ratio=0.117523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 3997, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4094, 28, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 2, 0, 162, 0, 0] payload_rows_per_dst=[8188, 8190, 1328, 8190, 8002, 8190, 8192, 7554] dup_rows_per_dst=[4, 7562, 18, 4, 4, 0, 30, 80] combine_rows_per_src=[8188, 8190, 1328, 8190, 8002, 8190, 8192, 7554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60988 payload_rows_remote=52796 dup_rows=4548 dup_rows_remote=4428 dedup_ratio=0.069397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 25, 0, 0] payload_rows_per_dst=[8126, 4500, 8192, 8192, 7430, 8192, 8164, 8192] dup_rows_per_dst=[0, 0, 1164, 0, 3262, 2, 0, 120] combine_rows_per_src=[8126, 4500, 8192, 8192, 7430, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231309312 dispatch_meta_bytes=229376 combine_payload_bytes=231309312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231538688 ingress_bytes=250183680 link_bytes=250183680 token_bytes=462848000 weight_movement_bytes=18874368 total_nvlink_bytes=481722368 weight_to_token_ratio=0.040779 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397136924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230219776 dispatch_meta_bytes=232528 combine_payload_bytes=230219776 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230452304 ingress_bytes=305717248 link_bytes=305717248 token_bytes=460672080 weight_movement_bytes=75497472 total_nvlink_bytes=536169552 weight_to_token_ratio=0.163885 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.929232248e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233496576 dispatch_meta_bytes=228104 combine_payload_bytes=233496576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233724680 ingress_bytes=271245312 link_bytes=271245312 token_bytes=467221256 weight_movement_bytes=37748736 total_nvlink_bytes=504969992 weight_to_token_ratio=0.080794 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.598939119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65418 payload_rows_remote=57226 dup_rows=118 dup_rows_remote=68 dedup_ratio=0.001801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8190, 8192, 8192, 8192, 8192, 8110, 8192] dup_rows_per_dst=[0, 0, 12, 44, 10, 50, 2, 0] combine_rows_per_src=[8158, 8190, 8192, 8192, 8192, 8192, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=203350016 dispatch_meta_bytes=229376 combine_payload_bytes=203350016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203579392 ingress_bytes=222224384 link_bytes=222224384 token_bytes=406929408 weight_movement_bytes=18874368 total_nvlink_bytes=425803776 weight_to_token_ratio=0.046382 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.129244706e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=216252416 dispatch_meta_bytes=228896 combine_payload_bytes=216252416 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=216481312 ingress_bytes=291749888 link_bytes=291749888 token_bytes=432733728 weight_movement_bytes=75497472 total_nvlink_bytes=508231200 weight_to_token_ratio=0.174466 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.795403877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234397696 dispatch_meta_bytes=229176 combine_payload_bytes=234397696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234626872 ingress_bytes=253272064 link_bytes=253272064 token_bytes=469024568 weight_movement_bytes=18874368 total_nvlink_bytes=487898936 weight_to_token_ratio=0.040242 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426728300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61694 payload_rows_remote=53502 dup_rows=3842 dup_rows_remote=3718 dedup_ratio=0.058624 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2816, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5632, 8192, 8184, 6918, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 3466, 252, 0, 0, 124, 0] combine_rows_per_src=[5632, 8192, 8184, 6918, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65410 payload_rows_remote=57240 dup_rows=126 dup_rows_remote=124 dedup_ratio=0.001923 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4050, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8100, 8190, 8190, 8170, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 4, 0, 2, 4, 112, 4] combine_rows_per_src=[8188, 8100, 8190, 8190, 8170, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65428 payload_rows_remote=57236 dup_rows=108 dup_rows_remote=106 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4044, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8088, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 38, 2, 2, 2, 64, 0] combine_rows_per_src=[8190, 8088, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65052 payload_rows_remote=56894 dup_rows=484 dup_rows_remote=484 dedup_ratio=0.007385 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4079, 0, 0, 210, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7798, 8158, 8192, 8192, 8188, 8192, 8140, 8192] dup_rows_per_dst=[0, 0, 418, 16, 6, 2, 42, 0] combine_rows_per_src=[7798, 8158, 8192, 8192, 8188, 8192, 8140, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63230 payload_rows_remote=55038 dup_rows=2306 dup_rows_remote=2276 dedup_ratio=0.035187 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 3604, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3469, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 7208, 8192, 8188, 6938, 8192, 8156, 8192] dup_rows_per_dst=[0, 0, 30, 0, 8, 10, 2258, 0] combine_rows_per_src=[8164, 7208, 8192, 8188, 6938, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=219144192 dispatch_meta_bytes=228880 combine_payload_bytes=219144192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219373072 ingress_bytes=238018560 link_bytes=238018560 token_bytes=438517264 weight_movement_bytes=18874368 total_nvlink_bytes=457391632 weight_to_token_ratio=0.043041 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.280576729e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61930 payload_rows_remote=53738 dup_rows=3606 dup_rows_remote=3008 dedup_ratio=0.055023 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4052, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8104, 7688, 8122, 8180, 8092, 5372, 8180] dup_rows_per_dst=[598, 2880, 0, 14, 4, 12, 90, 8] combine_rows_per_src=[8192, 8104, 7688, 8122, 8180, 8092, 5372, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229456 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234684496 ingress_bytes=253329408 link_bytes=253329408 token_bytes=469139536 weight_movement_bytes=18874368 total_nvlink_bytes=488013904 weight_to_token_ratio=0.040232 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427277742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234438656 dispatch_meta_bytes=229368 combine_payload_bytes=234438656 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234668024 ingress_bytes=291061760 link_bytes=291061760 token_bytes=469106680 weight_movement_bytes=56623104 total_nvlink_bytes=525729784 weight_to_token_ratio=0.120704 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.788810573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64038 payload_rows_remote=55846 dup_rows=1498 dup_rows_remote=1498 dedup_ratio=0.022858 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8056, 6858, 8192, 8192, 8166, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 0, 1320, 12, 162, 0] combine_rows_per_src=[8056, 6858, 8192, 8192, 8166, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229512 combine_payload_bytes=233037824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233267336 ingress_bytes=289660928 link_bytes=289660928 token_bytes=466305160 weight_movement_bytes=56623104 total_nvlink_bytes=522928264 weight_to_token_ratio=0.121429 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.775388490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65366 payload_rows_remote=57188 dup_rows=170 dup_rows_remote=170 dedup_ratio=0.002594 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8040, 8192, 8188, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 14, 0, 42, 0, 0, 114] combine_rows_per_src=[8040, 8192, 8188, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=225435648 dispatch_meta_bytes=229256 combine_payload_bytes=225435648 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225664904 ingress_bytes=282058752 link_bytes=282058752 token_bytes=451100552 weight_movement_bytes=56623104 total_nvlink_bytes=507723656 weight_to_token_ratio=0.125522 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.702548180e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220110848 dispatch_meta_bytes=226984 combine_payload_bytes=220110848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220337832 ingress_bytes=238985216 link_bytes=238985216 token_bytes=440448680 weight_movement_bytes=18874368 total_nvlink_bytes=459323048 weight_to_token_ratio=0.042853 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.289838752e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=228745216 dispatch_meta_bytes=229376 combine_payload_bytes=228745216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228974592 ingress_bytes=247619584 link_bytes=247619584 token_bytes=457719808 weight_movement_bytes=18874368 total_nvlink_bytes=476594176 weight_to_token_ratio=0.041236 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372569017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234242048 dispatch_meta_bytes=229432 combine_payload_bytes=234242048 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234471480 ingress_bytes=253116416 link_bytes=253116416 token_bytes=468713528 weight_movement_bytes=18874368 total_nvlink_bytes=487587896 weight_to_token_ratio=0.040268 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425236957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64982 payload_rows_remote=56790 dup_rows=554 dup_rows_remote=554 dedup_ratio=0.008453 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 185, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7646, 8192, 8192, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 184, 0, 0, 0, 0, 370] combine_rows_per_src=[8192, 7646, 8192, 8192, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65434 payload_rows_remote=57244 dup_rows=102 dup_rows_remote=40 dedup_ratio=0.001556 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8094, 8190, 8190, 8192, 8192, 8192] dup_rows_per_dst=[2, 2, 0, 0, 62, 4, 0, 32] combine_rows_per_src=[8192, 8192, 8094, 8190, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57282 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.000946 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8166, 8188, 8160, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 10, 0, 42, 2, 0, 8] combine_rows_per_src=[8166, 8188, 8160, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65388 payload_rows_remote=57210 dup_rows=148 dup_rows_remote=148 dedup_ratio=0.002258 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 30, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8058, 8178, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 12, 0, 62, 6, 0, 68] combine_rows_per_src=[8058, 8178, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57316 dup_rows=32 dup_rows_remote=18 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8186, 8188, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 14, 2, 16, 0, 0, 0] combine_rows_per_src=[8172, 8186, 8188, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64592 payload_rows_remote=56400 dup_rows=944 dup_rows_remote=592 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8002, 8188, 7696, 8192, 8192, 7938] dup_rows_per_dst=[352, 22, 380, 2, 160, 4, 0, 24] combine_rows_per_src=[8192, 8192, 8002, 8188, 7696, 8192, 8192, 7938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232611840 dispatch_meta_bytes=229376 combine_payload_bytes=232611840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232841216 ingress_bytes=251486208 link_bytes=251486208 token_bytes=465453056 weight_movement_bytes=18874368 total_nvlink_bytes=484327424 weight_to_token_ratio=0.040551 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409617106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234471424 dispatch_meta_bytes=229136 combine_payload_bytes=234471424 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=234700560 ingress_bytes=347717632 link_bytes=347717632 token_bytes=469171984 weight_movement_bytes=113246208 total_nvlink_bytes=582418192 weight_to_token_ratio=0.241375 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.331659262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234627072 dispatch_meta_bytes=229376 combine_payload_bytes=234627072 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234856448 ingress_bytes=272375808 link_bytes=272375808 token_bytes=469483520 weight_movement_bytes=37748736 total_nvlink_bytes=507232256 weight_to_token_ratio=0.080405 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609770975e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62002 payload_rows_remote=53810 dup_rows=3534 dup_rows_remote=3524 dedup_ratio=0.053925 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 723, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 275, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 2371, 0] payload_rows_per_dst=[7436, 8192, 8182, 8192, 8190, 8192, 8192, 5426] dup_rows_per_dst=[1548, 1458, 436, 0, 0, 10, 0, 82] combine_rows_per_src=[7436, 8192, 8182, 8192, 8190, 8192, 8192, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8150, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 42, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8150, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234332160 dispatch_meta_bytes=229432 combine_payload_bytes=234332160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234561592 ingress_bytes=253206528 link_bytes=253206528 token_bytes=468893752 weight_movement_bytes=18874368 total_nvlink_bytes=487768120 weight_to_token_ratio=0.040253 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426100366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=231014400 dispatch_meta_bytes=227968 combine_payload_bytes=231014400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231242368 ingress_bytes=249888768 link_bytes=249888768 token_bytes=462256768 weight_movement_bytes=18874368 total_nvlink_bytes=481131136 weight_to_token_ratio=0.040831 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394311222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229336 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234995672 ingress_bytes=272515072 link_bytes=272515072 token_bytes=469762008 weight_movement_bytes=37748736 total_nvlink_bytes=507510744 weight_to_token_ratio=0.080357 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611105334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=220405760 dispatch_meta_bytes=229336 combine_payload_bytes=220405760 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=220635096 ingress_bytes=277028864 link_bytes=277028864 token_bytes=441040856 weight_movement_bytes=56623104 total_nvlink_bytes=497663960 weight_to_token_ratio=0.128385 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.654354268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229376 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234930176 ingress_bytes=253575168 link_bytes=253575168 token_bytes=469630976 weight_movement_bytes=18874368 total_nvlink_bytes=488505344 weight_to_token_ratio=0.040190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429632493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65252 payload_rows_remote=57068 dup_rows=284 dup_rows_remote=8 dedup_ratio=0.004333 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7950, 8192, 8192, 8158, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 8, 276, 0] combine_rows_per_src=[8192, 7950, 8192, 8192, 8158, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64866 payload_rows_remote=56674 dup_rows=670 dup_rows_remote=670 dedup_ratio=0.010223 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 156, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 65, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 1, 0, 0, 3898, 0] payload_rows_per_dst=[7866, 8192, 8190, 8190, 8192, 8192, 8192, 7852] dup_rows_per_dst=[116, 390, 130, 0, 0, 2, 2, 30] combine_rows_per_src=[7866, 8192, 8190, 8190, 8192, 8192, 8192, 7852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62022 payload_rows_remote=53830 dup_rows=3514 dup_rows_remote=3514 dedup_ratio=0.053619 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 40, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 0, 0, 0, 2535, 0] payload_rows_per_dst=[7532, 8192, 8192, 8192, 8192, 8192, 8192, 5338] dup_rows_per_dst=[1452, 1898, 54, 0, 0, 0, 24, 86] combine_rows_per_src=[7532, 8192, 8192, 8192, 8192, 8192, 8192, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63864 payload_rows_remote=56630 dup_rows=1672 dup_rows_remote=1642 dedup_ratio=0.025513 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 97, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 376, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 2, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 2, 0, 0, 3771, 0] payload_rows_per_dst=[7234, 8192, 8188, 8188, 8192, 8182, 8112, 7576] dup_rows_per_dst=[30, 788, 804, 6, 4, 4, 4, 32] combine_rows_per_src=[7234, 8192, 8188, 8188, 8192, 8182, 8112, 7576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61210 payload_rows_remote=53018 dup_rows=4326 dup_rows_remote=3236 dedup_ratio=0.066010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 579, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 0, 0, 2156, 0] payload_rows_per_dst=[7442, 8192, 8192, 8192, 8192, 8192, 8192, 4616] dup_rows_per_dst=[2326, 1090, 874, 0, 0, 0, 0, 36] combine_rows_per_src=[7442, 8192, 8192, 8192, 8192, 8192, 8192, 4616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63702 payload_rows_remote=55510 dup_rows=1834 dup_rows_remote=1688 dedup_ratio=0.027985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 4064, 1, 104, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 2314, 0] payload_rows_per_dst=[7840, 8192, 8192, 8192, 8190, 8192, 8192, 6712] dup_rows_per_dst=[234, 1350, 146, 0, 0, 0, 20, 84] combine_rows_per_src=[7840, 8192, 8192, 8192, 8190, 8192, 8192, 6712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=232136704 dispatch_meta_bytes=229376 combine_payload_bytes=232136704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232366080 ingress_bytes=251011072 link_bytes=251011072 token_bytes=464502784 weight_movement_bytes=18874368 total_nvlink_bytes=483377152 weight_to_token_ratio=0.040633 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405064587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233750528 dispatch_meta_bytes=228304 combine_payload_bytes=233750528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233978832 ingress_bytes=252624896 link_bytes=252624896 token_bytes=467729360 weight_movement_bytes=18874368 total_nvlink_bytes=486603728 weight_to_token_ratio=0.040353 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420527454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=220487680 dispatch_meta_bytes=229376 combine_payload_bytes=220487680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220717056 ingress_bytes=239362048 link_bytes=239362048 token_bytes=441204736 weight_movement_bytes=18874368 total_nvlink_bytes=460079104 weight_to_token_ratio=0.042779 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293449370e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58890 payload_rows_remote=50784 dup_rows=6646 dup_rows_remote=6646 dedup_ratio=0.101410 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2624] payload_rows_per_dst=[8192, 8042, 8158, 8192, 2272, 8106, 8192, 7736] dup_rows_per_dst=[0, 0, 12, 104, 160, 0, 1780, 4590] combine_rows_per_src=[8192, 8042, 8158, 8192, 2272, 8106, 8192, 7736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231956480 dispatch_meta_bytes=233088 combine_payload_bytes=231956480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232189568 ingress_bytes=250830848 link_bytes=250830848 token_bytes=464146048 weight_movement_bytes=18874368 total_nvlink_bytes=483020416 weight_to_token_ratio=0.040665 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403337769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217161728 dispatch_meta_bytes=225016 combine_payload_bytes=217161728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217386744 ingress_bytes=236036096 link_bytes=236036096 token_bytes=434548472 weight_movement_bytes=18874368 total_nvlink_bytes=453422840 weight_to_token_ratio=0.043434 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.261581735e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56024 payload_rows_remote=47832 dup_rows=9512 dup_rows_remote=1928 dedup_ratio=0.145142 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 505, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 0, 0, 0, 1395, 0] payload_rows_per_dst=[4660, 2220, 8190, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[882, 0, 1016, 0, 0, 0, 30, 7584] combine_rows_per_src=[4660, 2220, 8190, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227368960 dispatch_meta_bytes=228792 combine_payload_bytes=227368960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227597752 ingress_bytes=283992064 link_bytes=283992064 token_bytes=454966712 weight_movement_bytes=56623104 total_nvlink_bytes=511589816 weight_to_token_ratio=0.124455 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.721072224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=208011264 dispatch_meta_bytes=229720 combine_payload_bytes=208011264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208240984 ingress_bytes=226885632 link_bytes=226885632 token_bytes=416252248 weight_movement_bytes=18874368 total_nvlink_bytes=435126616 weight_to_token_ratio=0.045344 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.173906491e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=195919872 dispatch_meta_bytes=199040 combine_payload_bytes=195919872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=196118912 ingress_bytes=233668608 link_bytes=233668608 token_bytes=392038784 weight_movement_bytes=37748736 total_nvlink_bytes=429787520 weight_to_token_ratio=0.096288 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238897630e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60832 payload_rows_remote=52640 dup_rows=4704 dup_rows_remote=1368 dedup_ratio=0.071777 router_tokens_per_expert=[0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1663, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1746] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8190, 8192, 3492] dup_rows_per_dst=[0, 2, 6, 2, 3336, 0, 1358, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8190, 8192, 3492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61050 payload_rows_remote=52858 dup_rows=4486 dup_rows_remote=4448 dedup_ratio=0.068451 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1883] payload_rows_per_dst=[8192, 8168, 8186, 8192, 8192, 8162, 8192, 3766] dup_rows_per_dst=[0, 0, 18, 38, 2510, 0, 1920, 0] combine_rows_per_src=[8192, 8168, 8186, 8192, 8192, 8162, 8192, 3766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8192, 8132, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 72, 0, 0, 8] combine_rows_per_src=[8172, 8192, 8132, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53164 payload_rows_remote=44976 dup_rows=12372 dup_rows_remote=3700 dedup_ratio=0.188782 router_tokens_per_expert=[0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4053, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1831, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1765] payload_rows_per_dst=[8188, 8110, 8192, 7820, 8192, 8130, 1002, 3530] dup_rows_per_dst=[8672, 4, 2, 8, 3686, 0, 0, 0] combine_rows_per_src=[8188, 8110, 8192, 7820, 8192, 8130, 1002, 3530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=215613440 dispatch_meta_bytes=216032 combine_payload_bytes=215613440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215829472 ingress_bytes=234487808 link_bytes=234487808 token_bytes=431442912 weight_movement_bytes=18874368 total_nvlink_bytes=450317280 weight_to_token_ratio=0.043747 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.246746801e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61768 payload_rows_remote=53846 dup_rows=3768 dup_rows_remote=3768 dedup_ratio=0.057495 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2649] payload_rows_per_dst=[8192, 7922, 8148, 8192, 8192, 8096, 7728, 5298] dup_rows_per_dst=[638, 0, 30, 126, 2076, 0, 898, 0] combine_rows_per_src=[8192, 7922, 8148, 8192, 8192, 8096, 7728, 5298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59818 payload_rows_remote=51632 dup_rows=5718 dup_rows_remote=5716 dedup_ratio=0.087250 router_tokens_per_expert=[0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1402] payload_rows_per_dst=[8188, 7928, 8186, 8192, 8192, 8138, 8190, 2804] dup_rows_per_dst=[0, 0, 2, 90, 4570, 0, 1056, 0] combine_rows_per_src=[8188, 7928, 8186, 8192, 8192, 8138, 8190, 2804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=216506368 dispatch_meta_bytes=229224 combine_payload_bytes=216506368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216735592 ingress_bytes=235380736 link_bytes=235380736 token_bytes=433241960 weight_movement_bytes=18874368 total_nvlink_bytes=452116328 weight_to_token_ratio=0.043565 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.255302398e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229376 combine_payload_bytes=234553344 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234782720 ingress_bytes=234553344 link_bytes=234782720 token_bytes=469336064 weight_movement_bytes=0 total_nvlink_bytes=469336064 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249572503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57892 payload_rows_remote=49700 dup_rows=7644 dup_rows_remote=1158 dedup_ratio=0.116638 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 4092, 0, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 0, 4091, 0, 0, 0, 0, 3862, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 3790, 8190, 8182, 4970, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 1154, 6486, 2, 0] combine_rows_per_src=[8184, 3790, 8190, 8182, 4970, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=184221696 dispatch_meta_bytes=194704 combine_payload_bytes=184221696 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=184416400 ingress_bytes=297467904 link_bytes=297467904 token_bytes=368638096 weight_movement_bytes=113246208 total_nvlink_bytes=481884304 weight_to_token_ratio=0.307202 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.850191093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52384 payload_rows_remote=44192 dup_rows=13152 dup_rows_remote=208 dedup_ratio=0.200684 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2403] payload_rows_per_dst=[0, 8166, 8192, 8192, 3258, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 54, 154, 12944] combine_rows_per_src=[0, 8166, 8192, 8192, 3258, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=220553216 dispatch_meta_bytes=230456 combine_payload_bytes=220553216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220783672 ingress_bytes=239427584 link_bytes=239427584 token_bytes=441336888 weight_movement_bytes=18874368 total_nvlink_bytes=460211256 weight_to_token_ratio=0.042766 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294077304e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=211484672 dispatch_meta_bytes=229392 combine_payload_bytes=211484672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211714064 ingress_bytes=230359040 link_bytes=230359040 token_bytes=423198736 weight_movement_bytes=18874368 total_nvlink_bytes=442073104 weight_to_token_ratio=0.044599 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207186978e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=203571200 dispatch_meta_bytes=203432 combine_payload_bytes=203571200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=203774632 ingress_bytes=260194304 link_bytes=260194304 token_bytes=407345832 weight_movement_bytes=56623104 total_nvlink_bytes=463968936 weight_to_token_ratio=0.139005 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.493053797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59562 payload_rows_remote=51372 dup_rows=5974 dup_rows_remote=4 dedup_ratio=0.091156 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3413, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 0, 0, 0, 4095, 0, 0, 0, 0, 1792, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6828, 8192, 8192, 8190, 3586, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 2, 5970, 0, 0, 0] combine_rows_per_src=[8192, 6828, 8192, 8192, 8190, 3586, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=181010432 dispatch_meta_bytes=177600 combine_payload_bytes=181010432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=181188032 ingress_bytes=218759168 link_bytes=218759168 token_bytes=362198464 weight_movement_bytes=37748736 total_nvlink_bytes=399947200 weight_to_token_ratio=0.104221 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.096042712e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60838 payload_rows_remote=52646 dup_rows=4698 dup_rows_remote=4696 dedup_ratio=0.071686 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 4095, 0, 0, 3263, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2348, 0, 0, 0, 4096, 0, 0, 0, 0, 2580, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 6532, 8188, 8192, 8192, 5160, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 4696, 0, 0, 0] combine_rows_per_src=[8190, 6532, 8188, 8192, 8192, 5160, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59934 payload_rows_remote=51742 dup_rows=5602 dup_rows_remote=5602 dedup_ratio=0.085480 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 1, 0, 3265, 0] payload_rows_per_dst=[8106, 6610, 4730, 8192, 8192, 8192, 8192, 7720] dup_rows_per_dst=[832, 580, 20, 0, 0, 0, 0, 4170] combine_rows_per_src=[8106, 6610, 4730, 8192, 8192, 8192, 8192, 7720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60916 payload_rows_remote=52724 dup_rows=4620 dup_rows_remote=4604 dedup_ratio=0.070496 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 2, 2252, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 0, 0, 4094, 0, 0, 0, 0, 2939, 4094, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5934, 5882, 8142, 8190] dup_rows_per_dst=[16, 4494, 46, 4, 0, 58, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5934, 5882, 8142, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=210419712 dispatch_meta_bytes=205504 combine_payload_bytes=210419712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210625216 ingress_bytes=248168448 link_bytes=248168448 token_bytes=421044928 weight_movement_bytes=37748736 total_nvlink_bytes=458793664 weight_to_token_ratio=0.089655 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=21248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.377827962e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60348 payload_rows_remote=52156 dup_rows=5188 dup_rows_remote=36 dedup_ratio=0.079163 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 0, 0, 0, 4096, 0, 0, 0, 0, 2646, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8158, 8192, 5940, 5292, 8192, 8192] dup_rows_per_dst=[0, 5152, 0, 0, 0, 0, 36, 0] combine_rows_per_src=[8190, 8192, 8158, 8192, 5940, 5292, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60298 payload_rows_remote=52112 dup_rows=5238 dup_rows_remote=5238 dedup_ratio=0.079926 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 0, 4096, 0, 0, 0, 0, 2440, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 7944, 8186, 8192, 6526, 4880, 8192, 8192] dup_rows_per_dst=[0, 2172, 0, 0, 3054, 0, 12, 0] combine_rows_per_src=[8186, 7944, 8186, 8192, 6526, 4880, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=215638016 dispatch_meta_bytes=229368 combine_payload_bytes=215638016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215867384 ingress_bytes=234512384 link_bytes=234512384 token_bytes=431505400 weight_movement_bytes=18874368 total_nvlink_bytes=450379768 weight_to_token_ratio=0.043741 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.246982276e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=211935232 dispatch_meta_bytes=229376 combine_payload_bytes=211935232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212164608 ingress_bytes=249683968 link_bytes=249683968 token_bytes=424099840 weight_movement_bytes=37748736 total_nvlink_bytes=461848576 weight_to_token_ratio=0.089009 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392348929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=215957504 dispatch_meta_bytes=229312 combine_payload_bytes=215957504 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=216186816 ingress_bytes=253706240 link_bytes=253706240 token_bytes=432144320 weight_movement_bytes=37748736 total_nvlink_bytes=469893056 weight_to_token_ratio=0.087352 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64866 payload_rows_remote=56674 dup_rows=670 dup_rows_remote=646 dedup_ratio=0.010223 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 47, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3914, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7906, 8192, 8192, 8192, 8170, 8192, 8192, 7830] dup_rows_per_dst=[88, 0, 526, 0, 0, 24, 32, 0] combine_rows_per_src=[7906, 8192, 8192, 8192, 8170, 8192, 8192, 7830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=213630976 dispatch_meta_bytes=208768 combine_payload_bytes=213630976 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=213839744 ingress_bytes=308002816 link_bytes=308002816 token_bytes=427470720 weight_movement_bytes=94371840 total_nvlink_bytes=521842560 weight_to_token_ratio=0.220768 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.951131436e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58352 payload_rows_remote=50160 dup_rows=7184 dup_rows_remote=7182 dedup_ratio=0.109619 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 0, 0, 0, 4096, 0, 0, 0, 0, 3591, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4776, 8192, 8192, 4424, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 7182, 0, 2] combine_rows_per_src=[8192, 4776, 8192, 8192, 4424, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=213450752 dispatch_meta_bytes=229400 combine_payload_bytes=213450752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213680152 ingress_bytes=232325120 link_bytes=232325120 token_bytes=427130904 weight_movement_bytes=18874368 total_nvlink_bytes=446005272 weight_to_token_ratio=0.044189 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226024989e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=232136704 dispatch_meta_bytes=229280 combine_payload_bytes=232136704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232365984 ingress_bytes=269885440 link_bytes=269885440 token_bytes=464502688 weight_movement_bytes=37748736 total_nvlink_bytes=502251424 weight_to_token_ratio=0.081267 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.585909494e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64584 payload_rows_remote=56392 dup_rows=952 dup_rows_remote=950 dedup_ratio=0.014526 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7246, 8190, 8190, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 494, 452, 0, 2, 2, 2, 0] combine_rows_per_src=[7246, 8190, 8190, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=205455360 dispatch_meta_bytes=229368 combine_payload_bytes=205455360 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205684728 ingress_bytes=243204096 link_bytes=243204096 token_bytes=411140088 weight_movement_bytes=37748736 total_nvlink_bytes=448888824 weight_to_token_ratio=0.091815 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330261984e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64792 payload_rows_remote=56600 dup_rows=744 dup_rows_remote=744 dedup_ratio=0.011353 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7500, 8192, 8192, 8192, 8188, 8192, 8192, 8144] dup_rows_per_dst=[4, 96, 606, 0, 0, 4, 34, 0] combine_rows_per_src=[7500, 8192, 8192, 8192, 8188, 8192, 8192, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58932 payload_rows_remote=50740 dup_rows=6604 dup_rows_remote=6114 dedup_ratio=0.100769 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2496] payload_rows_per_dst=[7084, 8182, 8190, 8192, 2708, 8192, 8192, 8192] dup_rows_per_dst=[0, 820, 22, 4, 0, 276, 490, 4992] combine_rows_per_src=[7084, 8182, 8190, 8192, 2708, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58920 payload_rows_remote=50806 dup_rows=6616 dup_rows_remote=878 dedup_ratio=0.100952 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3945, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 8188, 2410, 8174, 8192, 8116, 7836, 7890] dup_rows_per_dst=[5738, 472, 318, 20, 60, 8, 0, 0] combine_rows_per_src=[8114, 8188, 2410, 8174, 8192, 8116, 7836, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230981632 dispatch_meta_bytes=229368 combine_payload_bytes=230981632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231211000 ingress_bytes=249856000 link_bytes=249856000 token_bytes=462192632 weight_movement_bytes=18874368 total_nvlink_bytes=481067000 weight_to_token_ratio=0.040837 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393997255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64762 payload_rows_remote=56580 dup_rows=774 dup_rows_remote=764 dedup_ratio=0.011810 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 75, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 3817, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8006, 8182, 8192, 8192, 8162, 8192, 8192, 7644] dup_rows_per_dst=[150, 10, 328, 0, 0, 30, 256, 0] combine_rows_per_src=[8006, 8182, 8192, 8192, 8162, 8192, 8192, 7644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63102 payload_rows_remote=54910 dup_rows=2434 dup_rows_remote=36 dedup_ratio=0.037140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 3902, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6150, 8188, 8192, 8192, 8186, 8192, 8192, 7810] dup_rows_per_dst=[0, 0, 2398, 0, 0, 6, 30, 0] combine_rows_per_src=[6150, 8188, 8192, 8192, 8186, 8192, 8192, 7810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231833600 dispatch_meta_bytes=229376 combine_payload_bytes=231833600 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232062976 ingress_bytes=288456704 link_bytes=288456704 token_bytes=463896576 weight_movement_bytes=56623104 total_nvlink_bytes=520519680 weight_to_token_ratio=0.122060 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.763850208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=207831040 dispatch_meta_bytes=227416 combine_payload_bytes=207831040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208058456 ingress_bytes=226705408 link_bytes=226705408 token_bytes=415889496 weight_movement_bytes=18874368 total_nvlink_bytes=434763864 weight_to_token_ratio=0.045383 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.172179673e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64876 payload_rows_remote=57306 dup_rows=660 dup_rows_remote=660 dedup_ratio=0.010071 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 3653, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8174, 8192, 8188, 8192, 7570, 8176, 8192] dup_rows_per_dst=[0, 622, 0, 2, 24, 0, 0, 12] combine_rows_per_src=[8192, 8174, 8192, 8188, 8192, 7570, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=208101376 dispatch_meta_bytes=206736 combine_payload_bytes=208101376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208308112 ingress_bytes=226975744 link_bytes=226975744 token_bytes=416409488 weight_movement_bytes=18874368 total_nvlink_bytes=435283856 weight_to_token_ratio=0.045326 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.174769900e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231751680 dispatch_meta_bytes=229376 combine_payload_bytes=231751680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231981056 ingress_bytes=250626048 link_bytes=250626048 token_bytes=463732736 weight_movement_bytes=18874368 total_nvlink_bytes=482607104 weight_to_token_ratio=0.040701 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401375476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224911360 dispatch_meta_bytes=219784 combine_payload_bytes=224911360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225131144 ingress_bytes=281534464 link_bytes=281534464 token_bytes=450042504 weight_movement_bytes=56623104 total_nvlink_bytes=506665608 weight_to_token_ratio=0.125817 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.697524710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59598 payload_rows_remote=53246 dup_rows=5938 dup_rows_remote=2426 dedup_ratio=0.090607 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3167, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7350, 7122, 6008, 8190, 8192, 8192, 8192, 6352] dup_rows_per_dst=[2, 4, 2418, 0, 0, 0, 2, 3512] combine_rows_per_src=[7350, 7122, 6008, 8190, 8192, 8192, 8192, 6352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=231864 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234957240 ingress_bytes=272474112 link_bytes=272474112 token_bytes=469682616 weight_movement_bytes=37748736 total_nvlink_bytes=507431352 weight_to_token_ratio=0.080371 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610712876e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62932 payload_rows_remote=54740 dup_rows=2604 dup_rows_remote=2604 dedup_ratio=0.039734 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 1386, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2795, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 5592, 8190, 8192] dup_rows_per_dst=[0, 2602, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 5592, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218095616 dispatch_meta_bytes=222688 combine_payload_bytes=218095616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218318304 ingress_bytes=255844352 link_bytes=255844352 token_bytes=436413920 weight_movement_bytes=37748736 total_nvlink_bytes=474162656 weight_to_token_ratio=0.086498 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.451374698e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64658 payload_rows_remote=56466 dup_rows=878 dup_rows_remote=876 dedup_ratio=0.013397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 3769, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7320, 8186, 8192] dup_rows_per_dst=[0, 874, 0, 2, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7320, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63102 payload_rows_remote=54910 dup_rows=2434 dup_rows_remote=2434 dedup_ratio=0.037140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 0, 4096, 0, 0, 0, 0, 1218, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7464, 8192, 8192, 6486, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2434, 0, 0] combine_rows_per_src=[8192, 7464, 8192, 8192, 6486, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=224215040 dispatch_meta_bytes=229376 combine_payload_bytes=224215040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224444416 ingress_bytes=243089408 link_bytes=243089408 token_bytes=448659456 weight_movement_bytes=18874368 total_nvlink_bytes=467533824 weight_to_token_ratio=0.042068 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329163100e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63988 payload_rows_remote=55796 dup_rows=1548 dup_rows_remote=1548 dedup_ratio=0.023621 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 781, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3334, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 6672, 8182, 8178] dup_rows_per_dst=[0, 1534, 6, 4, 4, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 6672, 8182, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64720 payload_rows_remote=56530 dup_rows=816 dup_rows_remote=14 dedup_ratio=0.012451 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 3663, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8060, 8192, 7526, 8176, 8192] dup_rows_per_dst=[0, 802, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8060, 8192, 7526, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64958 payload_rows_remote=56766 dup_rows=578 dup_rows_remote=578 dedup_ratio=0.008820 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4040, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 7626, 8186, 8192] dup_rows_per_dst=[0, 570, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 7626, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231284736 dispatch_meta_bytes=229368 combine_payload_bytes=231284736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231514104 ingress_bytes=269033472 link_bytes=269033472 token_bytes=462798840 weight_movement_bytes=37748736 total_nvlink_bytes=500547576 weight_to_token_ratio=0.081566 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.577746356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65250 payload_rows_remote=57110 dup_rows=286 dup_rows_remote=104 dedup_ratio=0.004364 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4073, 0, 0, 122, 0, 0, 4000, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0] payload_rows_per_dst=[8192, 8192, 8000, 8192, 8150, 8140, 8192, 8192] dup_rows_per_dst=[18, 20, 32, 0, 0, 182, 0, 34] combine_rows_per_src=[8192, 8192, 8000, 8192, 8150, 8140, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=224911360 dispatch_meta_bytes=229376 combine_payload_bytes=224911360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=225140736 ingress_bytes=262660096 link_bytes=262660096 token_bytes=450052096 weight_movement_bytes=37748736 total_nvlink_bytes=487800832 weight_to_token_ratio=0.083876 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.516679803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=228540416 dispatch_meta_bytes=229376 combine_payload_bytes=228540416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228769792 ingress_bytes=247414784 link_bytes=247414784 token_bytes=457310208 weight_movement_bytes=18874368 total_nvlink_bytes=476184576 weight_to_token_ratio=0.041273 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.370606725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231546880 dispatch_meta_bytes=226176 combine_payload_bytes=231546880 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231773056 ingress_bytes=307044352 link_bytes=307044352 token_bytes=463319936 weight_movement_bytes=75497472 total_nvlink_bytes=538817408 weight_to_token_ratio=0.162949 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.941947906e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=232513536 dispatch_meta_bytes=229376 combine_payload_bytes=232513536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232742912 ingress_bytes=270262272 link_bytes=270262272 token_bytes=465256448 weight_movement_bytes=37748736 total_nvlink_bytes=503005184 weight_to_token_ratio=0.081135 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589520113e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58304 payload_rows_remote=50112 dup_rows=7232 dup_rows_remote=7232 dedup_ratio=0.110352 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4058, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1000, 8192, 8192, 8156, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 104, 8, 7064, 56, 0] combine_rows_per_src=[8192, 1000, 8192, 8192, 8156, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=228856 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234151416 ingress_bytes=309420032 link_bytes=309420032 token_bytes=468073976 weight_movement_bytes=75497472 total_nvlink_bytes=543571448 weight_to_token_ratio=0.161294 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.964710502e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61898 payload_rows_remote=53706 dup_rows=3638 dup_rows_remote=3634 dedup_ratio=0.055511 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1776, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1978, 4095, 0, 0, 303, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 4564, 8186, 8192] dup_rows_per_dst=[0, 3552, 66, 0, 4, 0, 0, 16] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 4564, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65226 payload_rows_remote=57034 dup_rows=310 dup_rows_remote=310 dedup_ratio=0.004730 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 4085, 0, 0, 66, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8164, 8192, 8170, 7932, 8192, 8192] dup_rows_per_dst=[6, 122, 138, 0, 0, 44, 0, 0] combine_rows_per_src=[8192, 8192, 8164, 8192, 8170, 7932, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=205258752 dispatch_meta_bytes=229376 combine_payload_bytes=205258752 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=205488128 ingress_bytes=224133120 link_bytes=224133120 token_bytes=410746880 weight_movement_bytes=18874368 total_nvlink_bytes=429621248 weight_to_token_ratio=0.045951 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.147533275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=219979776 dispatch_meta_bytes=229360 combine_payload_bytes=219979776 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=220209136 ingress_bytes=257728512 link_bytes=257728512 token_bytes=440188912 weight_movement_bytes=37748736 total_nvlink_bytes=477937648 weight_to_token_ratio=0.085756 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.469427792e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64598 payload_rows_remote=56406 dup_rows=938 dup_rows_remote=938 dedup_ratio=0.014313 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7282, 8192, 8192, 8192, 8192, 8192, 8192, 8164] dup_rows_per_dst=[0, 14, 922, 0, 0, 2, 0, 0] combine_rows_per_src=[7282, 8192, 8192, 8192, 8192, 8192, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61962 payload_rows_remote=53774 dup_rows=3574 dup_rows_remote=3574 dedup_ratio=0.054535 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2390, 4091, 0, 0, 284, 0, 0, 4089, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 41, 0] payload_rows_per_dst=[8188, 8192, 8182, 8188, 8192, 4848, 8142, 8030] dup_rows_per_dst=[0, 2582, 390, 0, 524, 0, 0, 78] combine_rows_per_src=[8188, 8192, 8182, 8188, 8192, 4848, 8142, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64746 payload_rows_remote=56554 dup_rows=790 dup_rows_remote=720 dedup_ratio=0.012054 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 3866, 0, 0, 329, 0, 0, 3969, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8192, 7938, 8192, 7732, 8116, 8192, 8192] dup_rows_per_dst=[36, 70, 28, 0, 0, 652, 0, 4] combine_rows_per_src=[8192, 8192, 7938, 8192, 7732, 8116, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233611264 dispatch_meta_bytes=229376 combine_payload_bytes=233611264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233840640 ingress_bytes=252485632 link_bytes=252485632 token_bytes=467451904 weight_movement_bytes=18874368 total_nvlink_bytes=486326272 weight_to_token_ratio=0.040377 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419193095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64294 payload_rows_remote=56174 dup_rows=1242 dup_rows_remote=118 dedup_ratio=0.018951 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 4092, 0, 0, 30, 0, 0, 4060, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8170, 8120, 8192, 8184, 7052, 8192, 8192] dup_rows_per_dst=[8, 38, 1124, 0, 0, 60, 0, 12] combine_rows_per_src=[8192, 8170, 8120, 8192, 8184, 7052, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59870 payload_rows_remote=51678 dup_rows=5666 dup_rows_remote=5666 dedup_ratio=0.086456 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1325, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 199, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 64, 4096, 18, 0, 4096, 2539, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 2658, 8136, 8122, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 398, 0, 0, 0, 26, 5242] combine_rows_per_src=[8186, 2658, 8136, 8122, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231038976 dispatch_meta_bytes=229376 combine_payload_bytes=231038976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231268352 ingress_bytes=249913344 link_bytes=249913344 token_bytes=462307328 weight_movement_bytes=18874368 total_nvlink_bytes=481181696 weight_to_token_ratio=0.040826 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394546697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220258304 dispatch_meta_bytes=229392 combine_payload_bytes=220258304 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=220487696 ingress_bytes=258007040 link_bytes=258007040 token_bytes=440746000 weight_movement_bytes=37748736 total_nvlink_bytes=478494736 weight_to_token_ratio=0.085647 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.472096510e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231645184 dispatch_meta_bytes=229096 combine_payload_bytes=231645184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231874280 ingress_bytes=269393920 link_bytes=269393920 token_bytes=463519464 weight_movement_bytes=37748736 total_nvlink_bytes=501268200 weight_to_token_ratio=0.081439 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581199991e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=230088704 dispatch_meta_bytes=225168 combine_payload_bytes=230088704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230313872 ingress_bytes=248963072 link_bytes=248963072 token_bytes=460402576 weight_movement_bytes=18874368 total_nvlink_bytes=479276944 weight_to_token_ratio=0.040995 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.385441658e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57112 payload_rows_remote=48920 dup_rows=8424 dup_rows_remote=8410 dedup_ratio=0.128540 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[8130, 1718, 6318, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 6, 0, 8346, 56, 14] combine_rows_per_src=[8130, 1718, 6318, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=211673088 dispatch_meta_bytes=229376 combine_payload_bytes=211673088 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=211902464 ingress_bytes=249421824 link_bytes=249421824 token_bytes=423575552 weight_movement_bytes=37748736 total_nvlink_bytes=461324288 weight_to_token_ratio=0.089119 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.389837194e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61588 payload_rows_remote=53396 dup_rows=3948 dup_rows_remote=3948 dedup_ratio=0.060242 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4048, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 17, 4095, 1, 0, 4095, 1026, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 6842, 8002, 8098, 8192, 8190, 8192, 5880] dup_rows_per_dst=[0, 0, 2458, 2, 0, 0, 2, 1486] combine_rows_per_src=[8192, 6842, 8002, 8098, 8192, 8190, 8192, 5880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62440 payload_rows_remote=54268 dup_rows=3096 dup_rows_remote=3096 dedup_ratio=0.047241 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 904, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 4096, 9, 0, 4096, 3190, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 6936, 8140, 8172, 8192, 8192, 8192, 6426] dup_rows_per_dst=[0, 1226, 1864, 0, 0, 0, 6, 0] combine_rows_per_src=[8190, 6936, 8140, 8172, 8192, 8192, 8192, 6426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=200376320 dispatch_meta_bytes=229320 combine_payload_bytes=200376320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200605640 ingress_bytes=219250688 link_bytes=219250688 token_bytes=400981960 weight_movement_bytes=18874368 total_nvlink_bytes=419856328 weight_to_token_ratio=0.047070 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.100752214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=218710016 dispatch_meta_bytes=229376 combine_payload_bytes=218710016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218939392 ingress_bytes=237584384 link_bytes=237584384 token_bytes=437649408 weight_movement_bytes=18874368 total_nvlink_bytes=456523776 weight_to_token_ratio=0.043127 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276416669e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65322 payload_rows_remote=57130 dup_rows=214 dup_rows_remote=214 dedup_ratio=0.003265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7978, 8192, 8192] dup_rows_per_dst=[0, 214, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7978, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62226 payload_rows_remote=54034 dup_rows=3310 dup_rows_remote=1162 dedup_ratio=0.050507 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1072, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 499, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 42, 0, 4092, 2476, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8012, 8192, 8192, 8192, 5092] dup_rows_per_dst=[0, 2148, 998, 0, 14, 0, 150, 0] combine_rows_per_src=[8192, 8192, 8162, 8012, 8192, 8192, 8192, 5092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58766 payload_rows_remote=50574 dup_rows=6770 dup_rows_remote=6758 dedup_ratio=0.103302 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 65, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4046, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7, 4094, 44, 0, 4094, 1141, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 7356, 8096, 8174, 8188, 8192, 2380] dup_rows_per_dst=[12, 6592, 148, 4, 0, 6, 6, 2] combine_rows_per_src=[8192, 8188, 7356, 8096, 8174, 8188, 8192, 2380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=222281728 dispatch_meta_bytes=229456 combine_payload_bytes=222281728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222511184 ingress_bytes=241156096 link_bytes=241156096 token_bytes=444792912 weight_movement_bytes=18874368 total_nvlink_bytes=463667280 weight_to_token_ratio=0.042434 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310639056e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59680 payload_rows_remote=53344 dup_rows=5856 dup_rows_remote=5536 dedup_ratio=0.089355 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2748, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3168, 160, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 13, 0, 4096, 2128, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 6336, 8100, 8192, 8192, 8192, 4288] dup_rows_per_dst=[0, 5504, 320, 0, 4, 0, 28, 0] combine_rows_per_src=[8188, 8192, 6336, 8100, 8192, 8192, 8192, 4288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65104 payload_rows_remote=56912 dup_rows=432 dup_rows_remote=432 dedup_ratio=0.006592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3636, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 255, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8186, 8192, 8190, 7776] dup_rows_per_dst=[0, 18, 408, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8184, 8186, 8192, 8190, 7776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234004480 dispatch_meta_bytes=229376 combine_payload_bytes=234004480 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234233856 ingress_bytes=290627584 link_bytes=290627584 token_bytes=468238336 weight_movement_bytes=56623104 total_nvlink_bytes=524861440 weight_to_token_ratio=0.120928 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.784650512e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=221323264 dispatch_meta_bytes=220784 combine_payload_bytes=221323264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=221544048 ingress_bytes=296820736 link_bytes=296820736 token_bytes=442867312 weight_movement_bytes=75497472 total_nvlink_bytes=518364784 weight_to_token_ratio=0.170474 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=17280 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.843990247e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=207151104 dispatch_meta_bytes=229328 combine_payload_bytes=207151104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207380432 ingress_bytes=226025472 link_bytes=226025472 token_bytes=414531536 weight_movement_bytes=18874368 total_nvlink_bytes=433405904 weight_to_token_ratio=0.045532 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.165664861e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=218497024 dispatch_meta_bytes=235520 combine_payload_bytes=218497024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218732544 ingress_bytes=256245760 link_bytes=256245760 token_bytes=437229568 weight_movement_bytes=37748736 total_nvlink_bytes=474978304 weight_to_token_ratio=0.086336 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.455220792e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233111552 dispatch_meta_bytes=229376 combine_payload_bytes=233111552 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233340928 ingress_bytes=270860288 link_bytes=270860288 token_bytes=466452480 weight_movement_bytes=37748736 total_nvlink_bytes=504201216 weight_to_token_ratio=0.080927 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595250008e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60602 payload_rows_remote=52410 dup_rows=4934 dup_rows_remote=1190 dedup_ratio=0.075287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2063, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 290, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 60, 0, 4096, 1703, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3546, 8124, 8190, 8192, 8192, 7974, 8192] dup_rows_per_dst=[0, 0, 580, 610, 0, 0, 0, 3744] combine_rows_per_src=[8192, 3546, 8124, 8190, 8192, 8192, 7974, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57336 dup_rows=18 dup_rows_remote=16 dedup_ratio=0.000275 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3891, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 208, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8190, 8182, 8192, 8192, 8188] dup_rows_per_dst=[2, 2, 2, 0, 2, 0, 0, 10] combine_rows_per_src=[8192, 8192, 8190, 8190, 8182, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 18, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 8192, 8190, 8138] dup_rows_per_dst=[0, 4, 52, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 8192, 8190, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=214671360 dispatch_meta_bytes=214400 combine_payload_bytes=214671360 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=214885760 ingress_bytes=290168832 link_bytes=290168832 token_bytes=429557120 weight_movement_bytes=75497472 total_nvlink_bytes=505054592 weight_to_token_ratio=0.175757 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780254976e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229408 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077664 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925920 weight_movement_bytes=18874368 total_nvlink_bytes=488800288 weight_to_token_ratio=0.040165 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64764 payload_rows_remote=56572 dup_rows=772 dup_rows_remote=772 dedup_ratio=0.011780 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 7422, 8192, 8192] dup_rows_per_dst=[0, 114, 650, 0, 4, 0, 0, 4] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 7422, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64780 payload_rows_remote=56588 dup_rows=756 dup_rows_remote=746 dedup_ratio=0.011536 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3676, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 89, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8150, 8192, 8190, 7488] dup_rows_per_dst=[0, 10, 704, 0, 0, 0, 0, 42] combine_rows_per_src=[8192, 8192, 8192, 8184, 8150, 8192, 8190, 7488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65156 payload_rows_remote=56964 dup_rows=380 dup_rows_remote=376 dedup_ratio=0.005798 router_tokens_per_expert=[0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 4094, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 19, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8188, 8180, 8188, 8192, 7836] dup_rows_per_dst=[4, 14, 242, 14, 106, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8188, 8180, 8188, 8192, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229376 combine_payload_bytes=234618880 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234848256 ingress_bytes=272367616 link_bytes=272367616 token_bytes=469467136 weight_movement_bytes=37748736 total_nvlink_bytes=507215872 weight_to_token_ratio=0.080408 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609692483e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65230 payload_rows_remote=57038 dup_rows=306 dup_rows_remote=10 dedup_ratio=0.004669 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3875, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 73, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8190, 8192, 8188, 7894] dup_rows_per_dst=[0, 6, 296, 2, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8190, 8190, 8192, 8188, 7894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59930 payload_rows_remote=54742 dup_rows=5606 dup_rows_remote=4028 dedup_ratio=0.085541 router_tokens_per_expert=[0, 0, 0, 2226, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 1580, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5902, 8092, 7984, 8192, 8188, 5188, 8192, 8192] dup_rows_per_dst=[2022, 0, 0, 0, 2006, 1578, 0, 0] combine_rows_per_src=[5902, 8092, 7984, 8192, 8188, 5188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231718912 dispatch_meta_bytes=229376 combine_payload_bytes=231718912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231948288 ingress_bytes=288342016 link_bytes=288342016 token_bytes=463667200 weight_movement_bytes=56623104 total_nvlink_bytes=520290304 weight_to_token_ratio=0.122120 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.762751324e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231784448 dispatch_meta_bytes=229336 combine_payload_bytes=231784448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=232013784 ingress_bytes=307281920 link_bytes=307281920 token_bytes=463798232 weight_movement_bytes=75497472 total_nvlink_bytes=539295704 weight_to_token_ratio=0.162781 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.944224165e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233324544 dispatch_meta_bytes=229360 combine_payload_bytes=233324544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233553904 ingress_bytes=252198912 link_bytes=252198912 token_bytes=466878448 weight_movement_bytes=18874368 total_nvlink_bytes=485752816 weight_to_token_ratio=0.040427 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416445885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233627648 dispatch_meta_bytes=228192 combine_payload_bytes=233627648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233855840 ingress_bytes=252502016 link_bytes=252502016 token_bytes=467483488 weight_movement_bytes=18874368 total_nvlink_bytes=486357856 weight_to_token_ratio=0.040374 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419350079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=224223232 dispatch_meta_bytes=235080 combine_payload_bytes=224223232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224458312 ingress_bytes=243097600 link_bytes=243097600 token_bytes=448681544 weight_movement_bytes=18874368 total_nvlink_bytes=467555912 weight_to_token_ratio=0.042066 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=22528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329241592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63238 payload_rows_remote=55048 dup_rows=2298 dup_rows_remote=2296 dedup_ratio=0.035065 router_tokens_per_expert=[0, 0, 0, 1123, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8178, 7956, 8192, 8190, 6146, 8192, 8192] dup_rows_per_dst=[2290, 0, 0, 0, 2, 0, 0, 6] combine_rows_per_src=[8192, 8178, 7956, 8192, 8190, 6146, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61326 payload_rows_remote=54356 dup_rows=4210 dup_rows_remote=1236 dedup_ratio=0.064240 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3472, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0] payload_rows_per_dst=[8186, 8138, 5268, 8188, 8192, 8192, 8192, 6970] dup_rows_per_dst=[0, 0, 1216, 6, 0, 4, 10, 2974] combine_rows_per_src=[8186, 8138, 5268, 8188, 8192, 8192, 8192, 6970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62356 payload_rows_remote=54164 dup_rows=3180 dup_rows_remote=3180 dedup_ratio=0.048523 router_tokens_per_expert=[0, 0, 0, 1288, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2497, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8186, 8192, 8186, 5038, 8192, 8192] dup_rows_per_dst=[3180, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8186, 8192, 8186, 5038, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=222642176 dispatch_meta_bytes=222368 combine_payload_bytes=222642176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222864544 ingress_bytes=279265280 link_bytes=279265280 token_bytes=445506720 weight_movement_bytes=56623104 total_nvlink_bytes=502129824 weight_to_token_ratio=0.127098 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.675782506e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=225476608 dispatch_meta_bytes=229376 combine_payload_bytes=225476608 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=225705984 ingress_bytes=338722816 link_bytes=338722816 token_bytes=451182592 weight_movement_bytes=113246208 total_nvlink_bytes=564428800 weight_to_token_ratio=0.250999 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.245475361e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59116 payload_rows_remote=50924 dup_rows=6420 dup_rows_remote=6420 dedup_ratio=0.097961 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 3207, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5512, 4514, 8130, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6420] combine_rows_per_src=[8192, 5512, 4514, 8130, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62006 payload_rows_remote=54000 dup_rows=3530 dup_rows_remote=3530 dedup_ratio=0.053864 router_tokens_per_expert=[0, 0, 0, 1143, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 2623, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8006, 7412, 8192, 8150, 5670, 8192, 8192] dup_rows_per_dst=[3518, 0, 0, 0, 0, 8, 0, 4] combine_rows_per_src=[8192, 8006, 7412, 8192, 8150, 5670, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=221855744 dispatch_meta_bytes=229376 combine_payload_bytes=221855744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222085120 ingress_bytes=240730112 link_bytes=240730112 token_bytes=443940864 weight_movement_bytes=18874368 total_nvlink_bytes=462815232 weight_to_token_ratio=0.042516 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.306557487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60462 payload_rows_remote=52272 dup_rows=5074 dup_rows_remote=1792 dedup_ratio=0.077423 router_tokens_per_expert=[0, 0, 0, 1634, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 4094, 0, 0, 0, 4039, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2614, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8190, 8192, 8040, 8192, 6246, 5228, 8192, 8182] dup_rows_per_dst=[3282, 322, 1444, 0, 4, 22, 0, 0] combine_rows_per_src=[8190, 8192, 8040, 8192, 6246, 5228, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64222 payload_rows_remote=56090 dup_rows=1314 dup_rows_remote=1314 dedup_ratio=0.020050 router_tokens_per_expert=[0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 3457, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8132, 8190, 8180, 6960, 8192, 8192] dup_rows_per_dst=[1310, 0, 0, 2, 0, 2, 0, 0] combine_rows_per_src=[8192, 8184, 8132, 8190, 8180, 6960, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65010 payload_rows_remote=56820 dup_rows=526 dup_rows_remote=274 dedup_ratio=0.008026 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8020, 8192, 8120, 8192, 8190, 7912, 8192] dup_rows_per_dst=[0, 0, 0, 0, 234, 252, 0, 40] combine_rows_per_src=[8192, 8020, 8192, 8120, 8192, 8190, 7912, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=208584704 dispatch_meta_bytes=229376 combine_payload_bytes=208584704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208814080 ingress_bytes=227459072 link_bytes=227459072 token_bytes=417398784 weight_movement_bytes=18874368 total_nvlink_bytes=436273152 weight_to_token_ratio=0.045219 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.179400911e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=221184000 dispatch_meta_bytes=230120 combine_payload_bytes=221184000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221414120 ingress_bytes=240058368 link_bytes=240058368 token_bytes=442598120 weight_movement_bytes=18874368 total_nvlink_bytes=461472488 weight_to_token_ratio=0.042644 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300121166e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=214106112 dispatch_meta_bytes=216256 combine_payload_bytes=214106112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214322368 ingress_bytes=232980480 link_bytes=232980480 token_bytes=428428480 weight_movement_bytes=18874368 total_nvlink_bytes=447302848 weight_to_token_ratio=0.044055 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232304326e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=229744640 dispatch_meta_bytes=229616 combine_payload_bytes=229744640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229974256 ingress_bytes=248619008 link_bytes=248619008 token_bytes=459718896 weight_movement_bytes=18874368 total_nvlink_bytes=478593264 weight_to_token_ratio=0.041056 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.382145006e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232734720 dispatch_meta_bytes=228376 combine_payload_bytes=232734720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232963096 ingress_bytes=270483456 link_bytes=270483456 token_bytes=465697816 weight_movement_bytes=37748736 total_nvlink_bytes=503446552 weight_to_token_ratio=0.081058 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591639389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 12, 0, 4] combine_rows_per_src=[8192, 8180, 8190, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60028 payload_rows_remote=51836 dup_rows=5508 dup_rows_remote=5508 dedup_ratio=0.084045 router_tokens_per_expert=[0, 0, 0, 1326, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2686, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 5508, 0, 0] combine_rows_per_src=[2686, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65330 payload_rows_remote=57168 dup_rows=206 dup_rows_remote=206 dedup_ratio=0.003143 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8162, 8192, 8162, 8192, 8192, 8046, 8192] dup_rows_per_dst=[0, 0, 0, 0, 36, 164, 0, 6] combine_rows_per_src=[8192, 8162, 8192, 8162, 8192, 8192, 8046, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235036672 ingress_bytes=291430400 link_bytes=291430400 token_bytes=469843968 weight_movement_bytes=56623104 total_nvlink_bytes=526467072 weight_to_token_ratio=0.120515 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792342700e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 129, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8164, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 10, 6, 0, 0, 0, 28] combine_rows_per_src=[8192, 8192, 8192, 8192, 8164, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=212320256 dispatch_meta_bytes=229376 combine_payload_bytes=212320256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212549632 ingress_bytes=231194624 link_bytes=231194624 token_bytes=424869888 weight_movement_bytes=18874368 total_nvlink_bytes=443744256 weight_to_token_ratio=0.044424 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215193132e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64402 payload_rows_remote=56310 dup_rows=1134 dup_rows_remote=1134 dedup_ratio=0.017303 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8092, 8192, 7928, 8192, 8192, 7430, 8192] dup_rows_per_dst=[0, 0, 0, 0, 382, 626, 0, 126] combine_rows_per_src=[8184, 8092, 8192, 7928, 8192, 8192, 7430, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234160128 dispatch_meta_bytes=229496 combine_payload_bytes=234160128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234389624 ingress_bytes=253034496 link_bytes=253034496 token_bytes=468549752 weight_movement_bytes=18874368 total_nvlink_bytes=487424120 weight_to_token_ratio=0.040283 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424452040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64488 payload_rows_remote=56296 dup_rows=1048 dup_rows_remote=838 dedup_ratio=0.015991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 7384, 8182, 8172, 7986] dup_rows_per_dst=[210, 438, 0, 392, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8188, 8192, 7384, 8182, 8172, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65362 payload_rows_remote=57170 dup_rows=174 dup_rows_remote=174 dedup_ratio=0.002655 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8164, 8192, 8192, 8060, 8192] dup_rows_per_dst=[0, 0, 0, 0, 128, 24, 0, 22] combine_rows_per_src=[8192, 8178, 8192, 8164, 8192, 8192, 8060, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8192, 8192, 8188, 8192, 8192, 8188] dup_rows_per_dst=[0, 12, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8180, 8192, 8192, 8192, 8188, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229376 combine_payload_bytes=234700800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234930176 ingress_bytes=272449536 link_bytes=272449536 token_bytes=469630976 weight_movement_bytes=37748736 total_nvlink_bytes=507379712 weight_to_token_ratio=0.080380 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610477400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230645760 dispatch_meta_bytes=229776 combine_payload_bytes=230645760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230875536 ingress_bytes=249520128 link_bytes=249520128 token_bytes=461521296 weight_movement_bytes=18874368 total_nvlink_bytes=480395664 weight_to_token_ratio=0.040896 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390779095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230588416 dispatch_meta_bytes=228536 combine_payload_bytes=230588416 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=230816952 ingress_bytes=306085888 link_bytes=306085888 token_bytes=461405368 weight_movement_bytes=75497472 total_nvlink_bytes=536902840 weight_to_token_ratio=0.163625 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.932764375e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234168320 dispatch_meta_bytes=229376 combine_payload_bytes=234168320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234397696 ingress_bytes=253042688 link_bytes=253042688 token_bytes=468566016 weight_movement_bytes=18874368 total_nvlink_bytes=487440384 weight_to_token_ratio=0.040281 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424530532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65340 payload_rows_remote=57340 dup_rows=196 dup_rows_remote=196 dedup_ratio=0.002991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8000, 8190, 8190, 8192] dup_rows_per_dst=[0, 184, 8, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8000, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63936 payload_rows_remote=55744 dup_rows=1600 dup_rows_remote=1600 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7990, 8162, 8192, 8192, 8192, 6928, 8088, 8192] dup_rows_per_dst=[0, 0, 6, 0, 28, 120, 1446, 0] combine_rows_per_src=[7990, 8162, 8192, 8192, 8192, 6928, 8088, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=50 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8192, 8152, 8192, 8192, 8192] dup_rows_per_dst=[0, 50, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8192, 8152, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=230144 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094784 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469959424 weight_movement_bytes=18874368 total_nvlink_bytes=488833792 weight_to_token_ratio=0.040162 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57732 payload_rows_remote=49540 dup_rows=7804 dup_rows_remote=7804 dedup_ratio=0.119080 router_tokens_per_expert=[0, 0, 0, 184, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[388, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 7804, 0, 0] combine_rows_per_src=[388, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=228327424 dispatch_meta_bytes=229376 combine_payload_bytes=228327424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228556800 ingress_bytes=247201792 link_bytes=247201792 token_bytes=456884224 weight_movement_bytes=18874368 total_nvlink_bytes=475758592 weight_to_token_ratio=0.041311 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368565940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229360 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889200 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549040 weight_movement_bytes=18874368 total_nvlink_bytes=488423408 weight_to_token_ratio=0.040197 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65394 payload_rows_remote=57202 dup_rows=142 dup_rows_remote=58 dedup_ratio=0.002167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8192, 8192, 8192, 8112, 8192, 8192, 8168] dup_rows_per_dst=[0, 84, 0, 58, 0, 0, 0, 0] combine_rows_per_src=[8154, 8192, 8192, 8192, 8112, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64512 payload_rows_remote=56320 dup_rows=1024 dup_rows_remote=1024 dedup_ratio=0.015625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 7174, 8192, 8192, 8192] dup_rows_per_dst=[0, 1022, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 7174, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65366 payload_rows_remote=57178 dup_rows=170 dup_rows_remote=96 dedup_ratio=0.002594 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 40, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8188, 8190, 8188, 8120, 8108, 8192, 8188, 8192] dup_rows_per_dst=[74, 84, 4, 0, 0, 0, 0, 8] combine_rows_per_src=[8188, 8190, 8188, 8120, 8108, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58298 payload_rows_remote=50106 dup_rows=7238 dup_rows_remote=7224 dedup_ratio=0.110443 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3242, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 2856, 8000, 6484, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 20, 14, 0, 7204] combine_rows_per_src=[8192, 2856, 8000, 6484, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=202915840 dispatch_meta_bytes=229376 combine_payload_bytes=202915840 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=203145216 ingress_bytes=240664576 link_bytes=240664576 token_bytes=406061056 weight_movement_bytes=37748736 total_nvlink_bytes=443809792 weight_to_token_ratio=0.092963 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.305929553e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234299392 dispatch_meta_bytes=229040 combine_payload_bytes=234299392 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234528432 ingress_bytes=328671232 link_bytes=328671232 token_bytes=468827824 weight_movement_bytes=94371840 total_nvlink_bytes=563199664 weight_to_token_ratio=0.201293 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.149166029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=230686720 dispatch_meta_bytes=229376 combine_payload_bytes=230686720 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=230916096 ingress_bytes=287309824 link_bytes=287309824 token_bytes=461602816 weight_movement_bytes=56623104 total_nvlink_bytes=518225920 weight_to_token_ratio=0.122666 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.752861368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=229096 combine_payload_bytes=234201088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234430184 ingress_bytes=253075456 link_bytes=253075456 token_bytes=468631272 weight_movement_bytes=18874368 total_nvlink_bytes=487505640 weight_to_token_ratio=0.040276 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424844498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60738 payload_rows_remote=52546 dup_rows=4798 dup_rows_remote=4798 dedup_ratio=0.073212 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 1, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 5424, 8186, 7446, 8192, 8190, 8192, 6916] dup_rows_per_dst=[0, 0, 2, 1430, 0, 0, 4, 3362] combine_rows_per_src=[8192, 5424, 8186, 7446, 8192, 8190, 8192, 6916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=205234176 dispatch_meta_bytes=229320 combine_payload_bytes=205234176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205463496 ingress_bytes=242982912 link_bytes=242982912 token_bytes=410697672 weight_movement_bytes=37748736 total_nvlink_bytes=448446408 weight_to_token_ratio=0.091914 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.328142708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59398 payload_rows_remote=51450 dup_rows=6138 dup_rows_remote=436 dedup_ratio=0.093658 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 3484, 8126, 7948, 8192, 8192, 8192, 7072] dup_rows_per_dst=[0, 430, 0, 5702, 4, 2, 0, 0] combine_rows_per_src=[8192, 3484, 8126, 7948, 8192, 8192, 8192, 7072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62404 payload_rows_remote=54212 dup_rows=3132 dup_rows_remote=3116 dedup_ratio=0.047791 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5072, 8184, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 3104, 2, 6, 16] combine_rows_per_src=[8192, 5072, 8184, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=215228416 dispatch_meta_bytes=229376 combine_payload_bytes=215228416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215457792 ingress_bytes=252977152 link_bytes=252977152 token_bytes=430686208 weight_movement_bytes=37748736 total_nvlink_bytes=468434944 weight_to_token_ratio=0.087648 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423902598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8112, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 80, 0, 0] combine_rows_per_src=[8192, 8112, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=210739200 dispatch_meta_bytes=207544 combine_payload_bytes=210739200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210946744 ingress_bytes=248487936 link_bytes=248487936 token_bytes=421685944 weight_movement_bytes=37748736 total_nvlink_bytes=459434680 weight_to_token_ratio=0.089519 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.380889139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=222052352 dispatch_meta_bytes=229312 combine_payload_bytes=222052352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222281664 ingress_bytes=240926720 link_bytes=240926720 token_bytes=444334016 weight_movement_bytes=18874368 total_nvlink_bytes=463208384 weight_to_token_ratio=0.042478 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.308441288e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61760 payload_rows_remote=53568 dup_rows=3776 dup_rows_remote=122 dedup_ratio=0.057617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 2, 4096] payload_rows_per_dst=[8192, 8192, 8044, 6268, 8192, 8192, 8188, 6492] dup_rows_per_dst=[0, 3654, 0, 0, 0, 118, 0, 4] combine_rows_per_src=[8192, 8192, 8044, 6268, 8192, 8192, 8188, 6492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59678 payload_rows_remote=51732 dup_rows=5858 dup_rows_remote=5858 dedup_ratio=0.089386 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 1, 4096] payload_rows_per_dst=[8192, 8192, 7946, 8078, 8192, 8192, 8188, 2698] dup_rows_per_dst=[0, 5834, 0, 0, 6, 16, 0, 2] combine_rows_per_src=[8192, 8192, 7946, 8078, 8192, 8192, 8188, 2698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58292 payload_rows_remote=50100 dup_rows=7244 dup_rows_remote=7244 dedup_ratio=0.110535 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 2, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8188, 8188, 6088, 8192, 8188, 8190, 3066] dup_rows_per_dst=[0, 6254, 980, 0, 2, 0, 8, 0] combine_rows_per_src=[8192, 8188, 8188, 6088, 8192, 8188, 8190, 3066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229376 combine_payload_bytes=234553344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234782720 ingress_bytes=253427712 link_bytes=253427712 token_bytes=469336064 weight_movement_bytes=18874368 total_nvlink_bytes=488210432 weight_to_token_ratio=0.040215 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428219642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 4096, 62, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8170] dup_rows_per_dst=[0, 0, 26, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=219414528 dispatch_meta_bytes=214760 combine_payload_bytes=219414528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219629288 ingress_bytes=238288896 link_bytes=238288896 token_bytes=439043816 weight_movement_bytes=18874368 total_nvlink_bytes=457918184 weight_to_token_ratio=0.042990 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283166956e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=211894272 dispatch_meta_bytes=230360 combine_payload_bytes=211894272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212124632 ingress_bytes=230768640 link_bytes=230768640 token_bytes=424018904 weight_movement_bytes=18874368 total_nvlink_bytes=442893272 weight_to_token_ratio=0.044513 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.211111563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=205209600 dispatch_meta_bytes=229376 combine_payload_bytes=205209600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205438976 ingress_bytes=224083968 link_bytes=224083968 token_bytes=410648576 weight_movement_bytes=18874368 total_nvlink_bytes=429522944 weight_to_token_ratio=0.045962 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.147062325e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 20, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62814 payload_rows_remote=54622 dup_rows=2722 dup_rows_remote=0 dedup_ratio=0.041534 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 6144, 7872, 8014, 8160, 8048, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2722] combine_rows_per_src=[8192, 6144, 7872, 8014, 8160, 8048, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63854 payload_rows_remote=55662 dup_rows=1682 dup_rows_remote=1682 dedup_ratio=0.025665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6510, 8192, 8192, 8192] dup_rows_per_dst=[0, 1682, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6510, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 4096, 71, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8164, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 30, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8164, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=223731712 dispatch_meta_bytes=218488 combine_payload_bytes=223731712 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=223950200 ingress_bytes=299229184 link_bytes=299229184 token_bytes=447681912 weight_movement_bytes=75497472 total_nvlink_bytes=523179384 weight_to_token_ratio=0.168641 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.867066811e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=0 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3822, 4096, 30, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 8192, 8190, 8186] dup_rows_per_dst=[0, 0, 22, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57332 dup_rows=16 dup_rows_remote=10 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094] payload_rows_per_dst=[8188, 8192, 8186, 8188, 8190, 8192, 8192, 8192] dup_rows_per_dst=[6, 4, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8188, 8192, 8186, 8188, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=227991552 dispatch_meta_bytes=229376 combine_payload_bytes=227991552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228220928 ingress_bytes=246865920 link_bytes=246865920 token_bytes=456212480 weight_movement_bytes=18874368 total_nvlink_bytes=475086848 weight_to_token_ratio=0.041372 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365347780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59364 payload_rows_remote=51172 dup_rows=6172 dup_rows_remote=6172 dedup_ratio=0.094177 router_tokens_per_expert=[0, 0, 0, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7788, 8192, 8192, 2442, 8192, 8192, 8174, 8192] dup_rows_per_dst=[5768, 0, 0, 404, 0, 0, 0, 0] combine_rows_per_src=[7788, 8192, 8192, 2442, 8192, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229376 combine_payload_bytes=234758144 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234987520 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469745664 weight_movement_bytes=18874368 total_nvlink_bytes=488620032 weight_to_token_ratio=0.040180 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229288 combine_payload_bytes=234790912 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235020200 ingress_bytes=329162752 link_bytes=329162752 token_bytes=469811112 weight_movement_bytes=94371840 total_nvlink_bytes=564182952 weight_to_token_ratio=0.200872 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153875531e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229368 combine_payload_bytes=234831872 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235061240 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893112 weight_movement_bytes=37748736 total_nvlink_bytes=507641848 weight_to_token_ratio=0.080335 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=209600512 dispatch_meta_bytes=229376 combine_payload_bytes=209600512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209829888 ingress_bytes=228474880 link_bytes=228474880 token_bytes=419430400 weight_movement_bytes=18874368 total_nvlink_bytes=438304768 weight_to_token_ratio=0.045000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189133884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63832 payload_rows_remote=55640 dup_rows=1704 dup_rows_remote=1704 dedup_ratio=0.026001 router_tokens_per_expert=[0, 1, 0, 849, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6492, 8192, 8192, 8190, 8190] dup_rows_per_dst=[1702, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 6492, 8192, 8192, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63412 payload_rows_remote=57336 dup_rows=2124 dup_rows_remote=2124 dedup_ratio=0.032410 router_tokens_per_expert=[0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6076, 8192, 8192, 8184, 8192] dup_rows_per_dst=[2124, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 6076, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57258 dup_rows=86 dup_rows_remote=62 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3069, 4096, 21, 0, 0, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8186, 8114, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 6, 50, 0, 4, 24] combine_rows_per_src=[8192, 8186, 8114, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64114 payload_rows_remote=55922 dup_rows=1422 dup_rows_remote=1422 dedup_ratio=0.021698 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 6884, 8192, 8078, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1422] combine_rows_per_src=[8192, 6884, 8192, 8078, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=227901440 dispatch_meta_bytes=229376 combine_payload_bytes=227901440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228130816 ingress_bytes=246775808 link_bytes=246775808 token_bytes=456032256 weight_movement_bytes=18874368 total_nvlink_bytes=474906624 weight_to_token_ratio=0.041388 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364484371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=237840 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235086096 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469934352 weight_movement_bytes=18874368 total_nvlink_bytes=488808720 weight_to_token_ratio=0.040164 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62802 payload_rows_remote=54610 dup_rows=2734 dup_rows_remote=2734 dedup_ratio=0.041718 router_tokens_per_expert=[0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 6, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5488, 8192, 8192, 8176, 8178] dup_rows_per_dst=[2720, 0, 0, 2, 12, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 5488, 8192, 8192, 8176, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7836, 8188, 8192, 8186, 8190] dup_rows_per_dst=[362, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7836, 8188, 8192, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229280 combine_payload_bytes=234528768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234758048 ingress_bytes=253403136 link_bytes=253403136 token_bytes=469286816 weight_movement_bytes=18874368 total_nvlink_bytes=488161184 weight_to_token_ratio=0.040219 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427984167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63350 payload_rows_remote=55158 dup_rows=2186 dup_rows_remote=4 dedup_ratio=0.033356 router_tokens_per_expert=[0, 2, 0, 1084, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6020, 8192, 8186, 8188, 8188] dup_rows_per_dst=[2182, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 6020, 8192, 8186, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=229056512 dispatch_meta_bytes=229376 combine_payload_bytes=229056512 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=229285888 ingress_bytes=247930880 link_bytes=247930880 token_bytes=458342400 weight_movement_bytes=18874368 total_nvlink_bytes=477216768 weight_to_token_ratio=0.041180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.375551703e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65316 payload_rows_remote=57342 dup_rows=220 dup_rows_remote=220 dedup_ratio=0.003357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 4096, 106, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 7974, 8192, 8192] dup_rows_per_dst=[0, 60, 0, 0, 0, 0, 160, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 7974, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=223682560 dispatch_meta_bytes=229376 combine_payload_bytes=223682560 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=223911936 ingress_bytes=261431296 link_bytes=261431296 token_bytes=447594496 weight_movement_bytes=37748736 total_nvlink_bytes=485343232 weight_to_token_ratio=0.084337 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.504906046e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229376 combine_payload_bytes=233373696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=233603072 ingress_bytes=233373696 link_bytes=233603072 token_bytes=466976768 weight_movement_bytes=0 total_nvlink_bytes=466976768 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238269696e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=225927168 dispatch_meta_bytes=220648 combine_payload_bytes=225927168 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=226147816 ingress_bytes=301424640 link_bytes=301424640 token_bytes=452074984 weight_movement_bytes=75497472 total_nvlink_bytes=527572456 weight_to_token_ratio=0.167002 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.888102590e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=230248 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235103080 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975912 weight_movement_bytes=18874368 total_nvlink_bytes=488850280 weight_to_token_ratio=0.040160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57296 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8150, 8192, 8186] dup_rows_per_dst=[0, 24, 0, 0, 0, 0, 26, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8150, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59016 payload_rows_remote=50824 dup_rows=6520 dup_rows_remote=6498 dedup_ratio=0.099487 router_tokens_per_expert=[0, 0, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1684, 8192, 8192, 8188, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6444, 0, 0, 54, 22] combine_rows_per_src=[1684, 8192, 8192, 8188, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229368 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235094008 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958648 weight_movement_bytes=37748736 total_nvlink_bytes=507707384 weight_to_token_ratio=0.080324 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229384 combine_payload_bytes=234684416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234913800 ingress_bytes=253558784 link_bytes=253558784 token_bytes=469598216 weight_movement_bytes=18874368 total_nvlink_bytes=488472584 weight_to_token_ratio=0.040193 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429475510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65200 payload_rows_remote=57010 dup_rows=336 dup_rows_remote=304 dedup_ratio=0.005127 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4082, 139, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 7902, 8192, 8148] dup_rows_per_dst=[0, 32, 0, 0, 26, 0, 278, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 7902, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=234 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 54, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 7962, 8192, 8188] dup_rows_per_dst=[0, 126, 4, 0, 0, 0, 108, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 7962, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64334 payload_rows_remote=56142 dup_rows=1202 dup_rows_remote=1196 dedup_ratio=0.018341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 4086, 1, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8190, 8192, 8192, 8184, 7008, 8192] dup_rows_per_dst=[6, 232, 0, 0, 26, 938, 0, 0] combine_rows_per_src=[8192, 8184, 8190, 8192, 8192, 8184, 7008, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=208175104 dispatch_meta_bytes=229288 combine_payload_bytes=208175104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208404392 ingress_bytes=227049472 link_bytes=227049472 token_bytes=416579496 weight_movement_bytes=18874368 total_nvlink_bytes=435453864 weight_to_token_ratio=0.045308 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.175476325e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64790 payload_rows_remote=56598 dup_rows=746 dup_rows_remote=746 dedup_ratio=0.011383 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 4096, 0, 0, 25, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3697, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7446, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 656, 0, 30, 60, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7446, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233512960 dispatch_meta_bytes=229256 combine_payload_bytes=233512960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233742216 ingress_bytes=252387328 link_bytes=252387328 token_bytes=467255176 weight_movement_bytes=18874368 total_nvlink_bytes=486129544 weight_to_token_ratio=0.040394 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418251195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229360 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234135536 ingress_bytes=252780544 link_bytes=252780544 token_bytes=468041712 weight_movement_bytes=18874368 total_nvlink_bytes=486916080 weight_to_token_ratio=0.040326 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422018797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=229957632 dispatch_meta_bytes=229352 combine_payload_bytes=229957632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230186984 ingress_bytes=267706368 link_bytes=267706368 token_bytes=460144616 weight_movement_bytes=37748736 total_nvlink_bytes=497893352 weight_to_token_ratio=0.082037 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.565030698e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64926 payload_rows_remote=56736 dup_rows=610 dup_rows_remote=604 dedup_ratio=0.009308 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 299, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1, 0, 0, 0, 3764, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 7588, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 598, 2, 0, 6, 2, 0, 2] combine_rows_per_src=[8188, 8192, 7588, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=231825408 dispatch_meta_bytes=229376 combine_payload_bytes=231825408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232054784 ingress_bytes=269574144 link_bytes=269574144 token_bytes=463880192 weight_movement_bytes=37748736 total_nvlink_bytes=501628928 weight_to_token_ratio=0.081376 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.582926809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64942 payload_rows_remote=56750 dup_rows=594 dup_rows_remote=590 dedup_ratio=0.009064 router_tokens_per_expert=[1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3770, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 7600, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 586, 0, 4, 0, 2, 0, 0] combine_rows_per_src=[8190, 8192, 7600, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=232390656 dispatch_meta_bytes=229360 combine_payload_bytes=232390656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232620016 ingress_bytes=270139392 link_bytes=270139392 token_bytes=465010672 weight_movement_bytes=37748736 total_nvlink_bytes=502759408 weight_to_token_ratio=0.081178 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.588342737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64706 payload_rows_remote=56514 dup_rows=830 dup_rows_remote=776 dedup_ratio=0.012665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 7576, 8190, 8192, 8138, 8042, 8192, 8192] dup_rows_per_dst=[0, 152, 0, 8, 0, 0, 616, 54] combine_rows_per_src=[8184, 7576, 8190, 8192, 8138, 8042, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57590 payload_rows_remote=49398 dup_rows=7946 dup_rows_remote=7946 dedup_ratio=0.121246 router_tokens_per_expert=[0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[246, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 7946, 0, 0, 0, 0] combine_rows_per_src=[246, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=232448000 dispatch_meta_bytes=229360 combine_payload_bytes=232448000 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232677360 ingress_bytes=270196736 link_bytes=270196736 token_bytes=465125360 weight_movement_bytes=37748736 total_nvlink_bytes=502874096 weight_to_token_ratio=0.081158 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.588892179e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65100 payload_rows_remote=56908 dup_rows=436 dup_rows_remote=46 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 4096, 0, 0, 83, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3791, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7756, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 390, 0, 46, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7756, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57344 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4034, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8144, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 4, 28, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8144, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64842 payload_rows_remote=56652 dup_rows=694 dup_rows_remote=680 dedup_ratio=0.010590 router_tokens_per_expert=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 184, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 154, 2, 0, 0, 0, 3729, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 7772, 8192, 7944, 8176, 8192, 8186] dup_rows_per_dst=[14, 616, 8, 0, 48, 4, 0, 4] combine_rows_per_src=[8190, 8190, 7772, 8192, 7944, 8176, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58194 payload_rows_remote=53598 dup_rows=7342 dup_rows_remote=7342 dedup_ratio=0.112030 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7660, 4596, 4978, 8192] dup_rows_per_dst=[0, 0, 522, 0, 0, 0, 0, 6820] combine_rows_per_src=[8192, 8192, 8192, 8192, 7660, 4596, 4978, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=231481344 dispatch_meta_bytes=229160 combine_payload_bytes=231481344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231710504 ingress_bytes=250355712 link_bytes=250355712 token_bytes=463191848 weight_movement_bytes=18874368 total_nvlink_bytes=482066216 weight_to_token_ratio=0.040748 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.398785250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=202334208 dispatch_meta_bytes=229376 combine_payload_bytes=202334208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202563584 ingress_bytes=240082944 link_bytes=240082944 token_bytes=404897792 weight_movement_bytes=37748736 total_nvlink_bytes=442646528 weight_to_token_ratio=0.093230 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300356641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=233095168 dispatch_meta_bytes=227816 combine_payload_bytes=233095168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233322984 ingress_bytes=270843904 link_bytes=270843904 token_bytes=466418152 weight_movement_bytes=37748736 total_nvlink_bytes=504166888 weight_to_token_ratio=0.080933 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.595093025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229568 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110592 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991616 weight_movement_bytes=37748736 total_nvlink_bytes=507740352 weight_to_token_ratio=0.080318 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=232046592 dispatch_meta_bytes=229328 combine_payload_bytes=232046592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232275920 ingress_bytes=269795328 link_bytes=269795328 token_bytes=464322512 weight_movement_bytes=37748736 total_nvlink_bytes=502071248 weight_to_token_ratio=0.081299 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.585046085e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57124 payload_rows_remote=54894 dup_rows=8412 dup_rows_remote=8410 dedup_ratio=0.128357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 4095] payload_rows_per_dst=[8192, 8190, 8192, 8190, 2230, 7266, 6678, 8186] dup_rows_per_dst=[2, 1092, 10, 0, 2, 0, 2, 7304] combine_rows_per_src=[8192, 8190, 8192, 8190, 2230, 7266, 6678, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=219537408 dispatch_meta_bytes=243760 combine_payload_bytes=219537408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219781168 ingress_bytes=257286144 link_bytes=257286144 token_bytes=439318576 weight_movement_bytes=37748736 total_nvlink_bytes=477067312 weight_to_token_ratio=0.085926 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.465189239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58210 payload_rows_remote=50018 dup_rows=7326 dup_rows_remote=7326 dedup_ratio=0.111786 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6960, 6844, 3446] dup_rows_per_dst=[0, 20, 86, 0, 4984, 0, 2236, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6960, 6844, 3446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=224845824 dispatch_meta_bytes=253216 combine_payload_bytes=224845824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225099040 ingress_bytes=243720192 link_bytes=243720192 token_bytes=449944864 weight_movement_bytes=18874368 total_nvlink_bytes=468819232 weight_to_token_ratio=0.041948 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.335206962e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61806 payload_rows_remote=53614 dup_rows=3730 dup_rows_remote=3730 dedup_ratio=0.056915 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 4898, 7846, 8108, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 332, 3294, 0, 100, 4, 0, 0] combine_rows_per_src=[8186, 4898, 7846, 8108, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=204873728 dispatch_meta_bytes=229376 combine_payload_bytes=204873728 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205103104 ingress_bytes=242622464 link_bytes=242622464 token_bytes=409976832 weight_movement_bytes=37748736 total_nvlink_bytes=447725568 weight_to_token_ratio=0.092075 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.324689072e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55058 payload_rows_remote=46866 dup_rows=10478 dup_rows_remote=10474 dedup_ratio=0.159882 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2767, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7768, 6330, 0] dup_rows_per_dst=[0, 4, 1090, 0, 5550, 886, 2948, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7768, 6330, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56628 payload_rows_remote=48436 dup_rows=8908 dup_rows_remote=8844 dedup_ratio=0.135925 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7520, 8148, 0] dup_rows_per_dst=[0, 258, 64, 0, 1200, 0, 7386, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7520, 8148, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56800 payload_rows_remote=48608 dup_rows=8736 dup_rows_remote=8732 dedup_ratio=0.133301 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 4094] payload_rows_per_dst=[8192, 8188, 8192, 8188, 8188, 8186, 7662, 4] dup_rows_per_dst=[4, 1138, 8, 6, 1070, 6506, 4, 0] combine_rows_per_src=[8192, 8188, 8192, 8188, 8188, 8186, 7662, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57342 payload_rows_remote=49150 dup_rows=8194 dup_rows_remote=8194 dedup_ratio=0.125031 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4095, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 0, 8190, 8192, 8192, 8192] dup_rows_per_dst=[42, 4, 0, 0, 0, 0, 5964, 2184] combine_rows_per_src=[8192, 8192, 8192, 0, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=219602944 dispatch_meta_bytes=229376 combine_payload_bytes=219602944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219832320 ingress_bytes=238477312 link_bytes=238477312 token_bytes=439435264 weight_movement_bytes=18874368 total_nvlink_bytes=458309632 weight_to_token_ratio=0.042951 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284972265e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235110408 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991432 weight_movement_bytes=94371840 total_nvlink_bytes=564363272 weight_to_token_ratio=0.200795 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=191963136 dispatch_meta_bytes=229360 combine_payload_bytes=191963136 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=192192496 ingress_bytes=229711872 link_bytes=229711872 token_bytes=384155632 weight_movement_bytes=37748736 total_nvlink_bytes=421904368 weight_to_token_ratio=0.098264 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.200986132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=198393856 dispatch_meta_bytes=229120 combine_payload_bytes=198393856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198622976 ingress_bytes=217268224 link_bytes=217268224 token_bytes=397016832 weight_movement_bytes=18874368 total_nvlink_bytes=415891200 weight_to_token_ratio=0.047540 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.081757220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59448 payload_rows_remote=51258 dup_rows=6088 dup_rows_remote=6088 dedup_ratio=0.092896 router_tokens_per_expert=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 1831, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8186, 8190, 8192, 5404, 8190, 8192, 4902, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 1552, 4534] combine_rows_per_src=[8186, 8190, 8192, 5404, 8190, 8192, 4902, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=199098368 dispatch_meta_bytes=229360 combine_payload_bytes=199098368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199327728 ingress_bytes=217972736 link_bytes=217972736 token_bytes=398426096 weight_movement_bytes=18874368 total_nvlink_bytes=417300464 weight_to_token_ratio=0.047372 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.088507507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=201318400 dispatch_meta_bytes=229376 combine_payload_bytes=201318400 prefetch_bytes=0 reduce_bytes=0 egress_bytes=201547776 ingress_bytes=201318400 link_bytes=201547776 token_bytes=402866176 weight_movement_bytes=0 total_nvlink_bytes=402866176 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.931131622e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62588 payload_rows_remote=54396 dup_rows=2948 dup_rows_remote=736 dedup_ratio=0.044983 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 9, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 6894, 6544] dup_rows_per_dst=[10, 18, 0, 2212, 0, 0, 0, 708] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 6894, 6544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=209952768 dispatch_meta_bytes=229384 combine_payload_bytes=209952768 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=210182152 ingress_bytes=247701504 link_bytes=247701504 token_bytes=420134920 weight_movement_bytes=37748736 total_nvlink_bytes=457883656 weight_to_token_ratio=0.089849 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373353935e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65422 payload_rows_remote=57230 dup_rows=114 dup_rows_remote=114 dedup_ratio=0.001740 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8080, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 106, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8080, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=222806016 dispatch_meta_bytes=220528 combine_payload_bytes=222806016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=223026544 ingress_bytes=279429120 link_bytes=279429120 token_bytes=445832560 weight_movement_bytes=56623104 total_nvlink_bytes=502455664 weight_to_token_ratio=0.127005 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.677352340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56818 payload_rows_remote=48626 dup_rows=8718 dup_rows_remote=8192 dedup_ratio=0.133026 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 263, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 3692, 3976] dup_rows_per_dst=[0, 526, 0, 8192, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 3692, 3976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55210 payload_rows_remote=47018 dup_rows=10326 dup_rows_remote=1702 dedup_ratio=0.157562 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096] payload_rows_per_dst=[8186, 5438, 6154, 8192, 6456, 7154, 5438, 8192] dup_rows_per_dst=[0, 6, 256, 0, 0, 1440, 0, 8624] combine_rows_per_src=[8186, 5438, 6154, 8192, 6456, 7154, 5438, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57334 payload_rows_remote=49142 dup_rows=8202 dup_rows_remote=8202 dedup_ratio=0.125153 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 5094, 3092] dup_rows_per_dst=[4, 6, 0, 8192, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 5094, 3092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57336 payload_rows_remote=49144 dup_rows=8200 dup_rows_remote=8200 dedup_ratio=0.125122 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 1505, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8188, 8192, 3026, 5166] dup_rows_per_dst=[0, 0, 0, 8192, 0, 0, 4, 4] combine_rows_per_src=[8192, 8188, 8192, 8192, 8188, 8192, 3026, 5166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65414 payload_rows_remote=57222 dup_rows=122 dup_rows_remote=122 dedup_ratio=0.001862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 60, 4096, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 3683, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8072, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 62, 0, 0, 60, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8072, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=229376 combine_payload_bytes=234414080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234643456 ingress_bytes=253288448 link_bytes=253288448 token_bytes=469057536 weight_movement_bytes=18874368 total_nvlink_bytes=487931904 weight_to_token_ratio=0.040239 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426885283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=199172096 dispatch_meta_bytes=227272 combine_payload_bytes=199172096 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199399368 ingress_bytes=255795200 link_bytes=255795200 token_bytes=398571464 weight_movement_bytes=56623104 total_nvlink_bytes=455194568 weight_to_token_ratio=0.142065 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450903747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=192585728 dispatch_meta_bytes=194880 combine_payload_bytes=192585728 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=192780608 ingress_bytes=305831936 link_bytes=305831936 token_bytes=385366336 weight_movement_bytes=113246208 total_nvlink_bytes=498612544 weight_to_token_ratio=0.293866 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.930331132e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=201285632 dispatch_meta_bytes=229376 combine_payload_bytes=201285632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201515008 ingress_bytes=220160000 link_bytes=220160000 token_bytes=402800640 weight_movement_bytes=18874368 total_nvlink_bytes=421675008 weight_to_token_ratio=0.046858 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109464795e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=201293824 dispatch_meta_bytes=229376 combine_payload_bytes=201293824 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=201523200 ingress_bytes=257916928 link_bytes=257916928 token_bytes=402817024 weight_movement_bytes=56623104 total_nvlink_bytes=459440128 weight_to_token_ratio=0.140568 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.471233101e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57318 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4082, 0, 0, 1] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8190, 8192, 8168] dup_rows_per_dst=[4, 24, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8190, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234381312 dispatch_meta_bytes=229376 combine_payload_bytes=234381312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234610688 ingress_bytes=253255680 link_bytes=253255680 token_bytes=468992000 weight_movement_bytes=18874368 total_nvlink_bytes=487866368 weight_to_token_ratio=0.040245 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426571316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 3927, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8158, 8192, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 18, 0, 0, 28, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8158, 8192, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229400 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235003928 ingress_bytes=272523264 link_bytes=272523264 token_bytes=469778456 weight_movement_bytes=37748736 total_nvlink_bytes=507527192 weight_to_token_ratio=0.080354 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611183826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58716 payload_rows_remote=51834 dup_rows=6820 dup_rows_remote=6820 dedup_ratio=0.104065 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 3620, 7254, 6882, 8192] dup_rows_per_dst=[0, 242, 2, 0, 0, 0, 0, 6576] combine_rows_per_src=[8192, 8192, 8192, 8192, 3620, 7254, 6882, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229376 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234921984 ingress_bytes=253566976 link_bytes=253566976 token_bytes=469614592 weight_movement_bytes=18874368 total_nvlink_bytes=488488960 weight_to_token_ratio=0.040191 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429554001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57316 dup_rows=28 dup_rows_remote=24 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 3420, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8164, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 24, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8164, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57330 payload_rows_remote=49138 dup_rows=8206 dup_rows_remote=5880 dedup_ratio=0.125214 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 7644, 8192, 574, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 5866, 2326] combine_rows_per_src=[8152, 7644, 8192, 574, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57298 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.001190 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 3943, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 8192, 8192, 8146] dup_rows_per_dst=[0, 50, 0, 0, 28, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 8192, 8192, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64728 payload_rows_remote=56536 dup_rows=808 dup_rows_remote=800 dedup_ratio=0.012329 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 280, 4094, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 31, 0, 0, 0, 0, 0, 3766, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7994, 8188, 8180, 7598] dup_rows_per_dst=[8, 566, 214, 0, 14, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7994, 8188, 8180, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=212312064 dispatch_meta_bytes=234616 combine_payload_bytes=212312064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212546680 ingress_bytes=231186432 link_bytes=231186432 token_bytes=424858744 weight_movement_bytes=18874368 total_nvlink_bytes=443733112 weight_to_token_ratio=0.044425 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215114641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65404 payload_rows_remote=57344 dup_rows=132 dup_rows_remote=132 dedup_ratio=0.002014 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8060, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 132, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8060, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229360 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234995696 ingress_bytes=310263808 link_bytes=310263808 token_bytes=469762032 weight_movement_bytes=75497472 total_nvlink_bytes=545259504 weight_to_token_ratio=0.160714 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972795149e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=201269248 dispatch_meta_bytes=220072 combine_payload_bytes=201269248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201489320 ingress_bytes=220143616 link_bytes=220143616 token_bytes=402758568 weight_movement_bytes=18874368 total_nvlink_bytes=421632936 weight_to_token_ratio=0.046863 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109307811e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229504 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234922112 ingress_bytes=253566976 link_bytes=253566976 token_bytes=469614720 weight_movement_bytes=18874368 total_nvlink_bytes=488489088 weight_to_token_ratio=0.040191 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429554001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=231571456 dispatch_meta_bytes=229344 combine_payload_bytes=231571456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231800800 ingress_bytes=250445824 link_bytes=250445824 token_bytes=463372256 weight_movement_bytes=18874368 total_nvlink_bytes=482246624 weight_to_token_ratio=0.040733 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399648659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=36 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8180, 8192, 8192, 8172, 8190] dup_rows_per_dst=[0, 0, 34, 0, 2, 2, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8180, 8192, 8192, 8172, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229904 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110928 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991952 weight_movement_bytes=56623104 total_nvlink_bytes=526615056 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=0 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8156, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 36, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8156, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229368 combine_payload_bytes=234725376 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234954744 ingress_bytes=253599744 link_bytes=253599744 token_bytes=469680120 weight_movement_bytes=18874368 total_nvlink_bytes=488554488 weight_to_token_ratio=0.040186 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429867968e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57336 payload_rows_remote=49144 dup_rows=8200 dup_rows_remote=4234 dedup_ratio=0.125122 router_tokens_per_expert=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1973, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 0, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3966, 4234] combine_rows_per_src=[8184, 8192, 8192, 0, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229232 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234962800 ingress_bytes=272482304 link_bytes=272482304 token_bytes=469696368 weight_movement_bytes=37748736 total_nvlink_bytes=507445104 weight_to_token_ratio=0.080368 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610791367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65306 payload_rows_remote=57114 dup_rows=230 dup_rows_remote=230 dedup_ratio=0.003510 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7962, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 202, 0, 0, 28, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7962, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64202 payload_rows_remote=56312 dup_rows=1334 dup_rows_remote=360 dedup_ratio=0.020355 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 149, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 3805, 0, 0, 1] payload_rows_per_dst=[8180, 7172, 8192, 8192, 8192, 8192, 8192, 7890] dup_rows_per_dst=[0, 298, 0, 0, 50, 0, 12, 974] combine_rows_per_src=[8180, 7172, 8192, 8192, 8192, 8192, 8192, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=20 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 2, 0, 0, 0, 2, 0, 0, 28, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8190, 8192, 8180, 8188] dup_rows_per_dst=[4, 0, 8, 8, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8190, 8192, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57304 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8152, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 36, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8152, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=201293824 dispatch_meta_bytes=213512 combine_payload_bytes=201293824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201507336 ingress_bytes=220168192 link_bytes=220168192 token_bytes=402801160 weight_movement_bytes=18874368 total_nvlink_bytes=421675528 weight_to_token_ratio=0.046858 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109543286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=233938944 dispatch_meta_bytes=229376 combine_payload_bytes=233938944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234168320 ingress_bytes=252813312 link_bytes=252813312 token_bytes=468107264 weight_movement_bytes=18874368 total_nvlink_bytes=486981632 weight_to_token_ratio=0.040321 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422332764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=230653952 dispatch_meta_bytes=226688 combine_payload_bytes=230653952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230880640 ingress_bytes=249528320 link_bytes=249528320 token_bytes=461534592 weight_movement_bytes=18874368 total_nvlink_bytes=480408960 weight_to_token_ratio=0.040895 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390857587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64450 payload_rows_remote=56260 dup_rows=1086 dup_rows_remote=1074 dedup_ratio=0.016571 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095] payload_rows_per_dst=[8192, 8182, 8190, 7122, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 12, 0, 4, 1068] combine_rows_per_src=[8192, 8182, 8190, 7122, 8190, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229360 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012080 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794800 weight_movement_bytes=18874368 total_nvlink_bytes=488669168 weight_to_token_ratio=0.040176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229376 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234946560 ingress_bytes=310214656 link_bytes=310214656 token_bytes=469663744 weight_movement_bytes=75497472 total_nvlink_bytes=545161216 weight_to_token_ratio=0.160748 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972324199e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57344 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8144, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 48] combine_rows_per_src=[8192, 8192, 8192, 8144, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230440960 dispatch_meta_bytes=229336 combine_payload_bytes=230440960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230670296 ingress_bytes=268189696 link_bytes=268189696 token_bytes=461111256 weight_movement_bytes=37748736 total_nvlink_bytes=498859992 weight_to_token_ratio=0.081865 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.569661710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229568 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110592 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991616 weight_movement_bytes=18874368 total_nvlink_bytes=488865984 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65086 payload_rows_remote=56894 dup_rows=450 dup_rows_remote=444 dedup_ratio=0.006866 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8150, 7790, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 432, 6, 6] combine_rows_per_src=[8192, 8192, 8150, 7790, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63428 payload_rows_remote=55236 dup_rows=2108 dup_rows_remote=2104 dedup_ratio=0.032166 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2, 4094, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094] payload_rows_per_dst=[8192, 8192, 8188, 7770, 8180, 8188, 8178, 6540] dup_rows_per_dst=[4, 12, 12, 1652, 0, 0, 8, 420] combine_rows_per_src=[8192, 8192, 8188, 7770, 8180, 8188, 8178, 6540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64654 payload_rows_remote=56462 dup_rows=882 dup_rows_remote=798 dedup_ratio=0.013458 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 83, 0, 0, 0, 4096] payload_rows_per_dst=[7448, 8160, 8086, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 632, 84, 0, 166] combine_rows_per_src=[7448, 8160, 8086, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229352 combine_payload_bytes=233037824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233267176 ingress_bytes=251912192 link_bytes=251912192 token_bytes=466305000 weight_movement_bytes=18874368 total_nvlink_bytes=485179368 weight_to_token_ratio=0.040476 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413698675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59944 payload_rows_remote=51752 dup_rows=5592 dup_rows_remote=10 dedup_ratio=0.085327 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2789, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 1, 0, 4, 0, 0, 0, 4095] payload_rows_per_dst=[2610, 8192, 8186, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 0, 5582, 0, 0, 8] combine_rows_per_src=[2610, 8192, 8186, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=226246656 dispatch_meta_bytes=229360 combine_payload_bytes=226246656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226476016 ingress_bytes=263995392 link_bytes=263995392 token_bytes=452722672 weight_movement_bytes=37748736 total_nvlink_bytes=490471408 weight_to_token_ratio=0.083382 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.529473952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235085824 ingress_bytes=291479552 link_bytes=291479552 token_bytes=469942272 weight_movement_bytes=56623104 total_nvlink_bytes=526565376 weight_to_token_ratio=0.120489 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792813650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64368 payload_rows_remote=56176 dup_rows=1168 dup_rows_remote=1168 dedup_ratio=0.017822 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4096] payload_rows_per_dst=[7102, 8182, 8124, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1046, 6, 0, 116] combine_rows_per_src=[7102, 8182, 8124, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=231268352 dispatch_meta_bytes=229040 combine_payload_bytes=231268352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231497392 ingress_bytes=250142720 link_bytes=250142720 token_bytes=462765744 weight_movement_bytes=18874368 total_nvlink_bytes=481640112 weight_to_token_ratio=0.040786 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396744465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=211976192 dispatch_meta_bytes=207048 combine_payload_bytes=211976192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212183240 ingress_bytes=230850560 link_bytes=230850560 token_bytes=424159432 weight_movement_bytes=18874368 total_nvlink_bytes=443033800 weight_to_token_ratio=0.044498 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.211896480e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=230096896 dispatch_meta_bytes=229376 combine_payload_bytes=230096896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230326272 ingress_bytes=248971264 link_bytes=248971264 token_bytes=460423168 weight_movement_bytes=18874368 total_nvlink_bytes=479297536 weight_to_token_ratio=0.040994 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.385520150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58546 payload_rows_remote=50462 dup_rows=6990 dup_rows_remote=6990 dedup_ratio=0.106659 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 52, 0, 0, 0, 4096] payload_rows_per_dst=[8120, 8084, 7424, 8192, 2150, 8192, 8192, 8192] dup_rows_per_dst=[6636, 0, 0, 0, 20, 230, 0, 104] combine_rows_per_src=[8120, 8084, 7424, 8192, 2150, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=10 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8186, 8190, 8186, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 6, 2, 6] combine_rows_per_src=[8186, 8190, 8186, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62872 payload_rows_remote=54684 dup_rows=2664 dup_rows_remote=1036 dedup_ratio=0.040649 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 1, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3268, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 2, 0, 19, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8192, 8174, 8192, 6540, 7834, 8188, 7564] dup_rows_per_dst=[1628, 4, 980, 0, 8, 0, 6, 38] combine_rows_per_src=[8188, 8192, 8174, 8192, 6540, 7834, 8188, 7564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=206692352 dispatch_meta_bytes=229808 combine_payload_bytes=206692352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206922160 ingress_bytes=225566720 link_bytes=225566720 token_bytes=413614512 weight_movement_bytes=18874368 total_nvlink_bytes=432488880 weight_to_token_ratio=0.045633 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.161269325e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57262 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096] payload_rows_per_dst=[8142, 8160, 8184, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 44, 40, 0, 6] combine_rows_per_src=[8142, 8160, 8184, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57332 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8190, 8186, 8190, 8190] dup_rows_per_dst=[2, 6, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8190, 8186, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229352 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235044840 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860328 weight_movement_bytes=37748736 total_nvlink_bytes=507609064 weight_to_token_ratio=0.080340 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=223985664 dispatch_meta_bytes=222880 combine_payload_bytes=223985664 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=224208544 ingress_bytes=280608768 link_bytes=280608768 token_bytes=448194208 weight_movement_bytes=56623104 total_nvlink_bytes=504817312 weight_to_token_ratio=0.126336 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.688655147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234545152 dispatch_meta_bytes=229408 combine_payload_bytes=234545152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234774560 ingress_bytes=272293888 link_bytes=272293888 token_bytes=469319712 weight_movement_bytes=37748736 total_nvlink_bytes=507068448 weight_to_token_ratio=0.080433 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608986058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229384 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061256 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893128 weight_movement_bytes=18874368 total_nvlink_bytes=488767496 weight_to_token_ratio=0.040167 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65434 payload_rows_remote=57242 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.001556 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8090, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 100] combine_rows_per_src=[8192, 8192, 8192, 8090, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8176, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65098 payload_rows_remote=56906 dup_rows=438 dup_rows_remote=264 dedup_ratio=0.006683 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096] payload_rows_per_dst=[7756, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 176, 0, 0, 88, 0, 0, 174] combine_rows_per_src=[7756, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234463232 dispatch_meta_bytes=229376 combine_payload_bytes=234463232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234692608 ingress_bytes=253337600 link_bytes=253337600 token_bytes=469155840 weight_movement_bytes=18874368 total_nvlink_bytes=488030208 weight_to_token_ratio=0.040230 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427356233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=26 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8188, 8178, 8188, 8188] dup_rows_per_dst=[4, 4, 12, 4, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8188, 8178, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8168] dup_rows_per_dst=[22, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57332 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233086976 dispatch_meta_bytes=228680 combine_payload_bytes=233086976 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233315656 ingress_bytes=327458816 link_bytes=327458816 token_bytes=466402632 weight_movement_bytes=94371840 total_nvlink_bytes=560774472 weight_to_token_ratio=0.202340 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.137549255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229360 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234987504 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469745648 weight_movement_bytes=18874368 total_nvlink_bytes=488620016 weight_to_token_ratio=0.040180 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=272523264 link_bytes=272523264 token_bytes=469778432 weight_movement_bytes=37748736 total_nvlink_bytes=507527168 weight_to_token_ratio=0.080354 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611183826e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229384 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061256 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893128 weight_movement_bytes=18874368 total_nvlink_bytes=488767496 weight_to_token_ratio=0.040167 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110408 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991432 weight_movement_bytes=18874368 total_nvlink_bytes=488865800 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65246 payload_rows_remote=57054 dup_rows=290 dup_rows_remote=290 dedup_ratio=0.004425 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096] payload_rows_per_dst=[7906, 8192, 8188, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 274, 0, 0, 16] combine_rows_per_src=[7906, 8192, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=291487744 link_bytes=291487744 token_bytes=469958656 weight_movement_bytes=56623104 total_nvlink_bytes=526581760 weight_to_token_ratio=0.120485 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792892142e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57312 dup_rows=32 dup_rows_remote=26 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8182, 8176, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 10, 6] combine_rows_per_src=[8186, 8182, 8176, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233693184 dispatch_meta_bytes=229376 combine_payload_bytes=233693184 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233922560 ingress_bytes=271441920 link_bytes=271441920 token_bytes=467615744 weight_movement_bytes=37748736 total_nvlink_bytes=505364480 weight_to_token_ratio=0.080726 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600822920e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229376 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235053056 ingress_bytes=291446784 link_bytes=291446784 token_bytes=469876736 weight_movement_bytes=56623104 total_nvlink_bytes=526499840 weight_to_token_ratio=0.120506 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792499684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64932 payload_rows_remote=56740 dup_rows=604 dup_rows_remote=4 dedup_ratio=0.009216 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3796, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 7594, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[0, 2, 2, 0, 600, 0, 0, 0] combine_rows_per_src=[8190, 8192, 7594, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=14 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 65, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087] payload_rows_per_dst=[8192, 8076, 8188, 8186, 8192, 8192, 8192, 8174] dup_rows_per_dst=[130, 0, 6, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 8076, 8188, 8186, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64284 payload_rows_remote=56092 dup_rows=1252 dup_rows_remote=1252 dedup_ratio=0.019104 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6940, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1252, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6940, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8164, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[30, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8164, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234749952 dispatch_meta_bytes=229352 combine_payload_bytes=234749952 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234979304 ingress_bytes=272498688 link_bytes=272498688 token_bytes=469729256 weight_movement_bytes=37748736 total_nvlink_bytes=507477992 weight_to_token_ratio=0.080363 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610948351e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232407040 dispatch_meta_bytes=226976 combine_payload_bytes=232407040 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232634016 ingress_bytes=289030144 link_bytes=289030144 token_bytes=465041056 weight_movement_bytes=56623104 total_nvlink_bytes=521664160 weight_to_token_ratio=0.121759 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769344628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=228856 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234520056 ingress_bytes=309788672 link_bytes=309788672 token_bytes=468811256 weight_movement_bytes=75497472 total_nvlink_bytes=544308728 weight_to_token_ratio=0.161040 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.968242630e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65358 payload_rows_remote=57166 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8014, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 178, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8014, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=229752832 dispatch_meta_bytes=229376 combine_payload_bytes=229752832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229982208 ingress_bytes=267501568 link_bytes=267501568 token_bytes=459735040 weight_movement_bytes=37748736 total_nvlink_bytes=497483776 weight_to_token_ratio=0.082110 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.563068406e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229376 combine_payload_bytes=234758144 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234987520 ingress_bytes=272506880 link_bytes=272506880 token_bytes=469745664 weight_movement_bytes=37748736 total_nvlink_bytes=507494400 weight_to_token_ratio=0.080360 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611026842e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64258 payload_rows_remote=56066 dup_rows=1278 dup_rows_remote=1278 dedup_ratio=0.019501 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6914, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1278, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6914, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234151936 dispatch_meta_bytes=229376 combine_payload_bytes=234151936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234381312 ingress_bytes=253026304 link_bytes=253026304 token_bytes=468533248 weight_movement_bytes=18874368 total_nvlink_bytes=487407616 weight_to_token_ratio=0.040284 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424373548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=229646336 dispatch_meta_bytes=229376 combine_payload_bytes=229646336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229875712 ingress_bytes=248520704 link_bytes=248520704 token_bytes=459522048 weight_movement_bytes=18874368 total_nvlink_bytes=478396416 weight_to_token_ratio=0.041074 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381203106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65312 payload_rows_remote=57120 dup_rows=224 dup_rows_remote=182 dedup_ratio=0.003418 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7974, 8188, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[4, 160, 0, 6, 2, 10, 0, 42] combine_rows_per_src=[7974, 8188, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991424 weight_movement_bytes=56623104 total_nvlink_bytes=526614528 weight_to_token_ratio=0.120477 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=2 dedup_ratio=0.000092 router_tokens_per_expert=[1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61156 payload_rows_remote=52968 dup_rows=4380 dup_rows_remote=4374 dedup_ratio=0.066833 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3076, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4094, 4094, 4094, 0, 0, 0, 10, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4064, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8186, 8192, 3896, 8188, 8174, 8140] dup_rows_per_dst=[6, 4, 4342, 0, 10, 6, 0, 12] combine_rows_per_src=[8188, 8192, 8186, 8192, 3896, 8188, 8174, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 16, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8176, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233963520 dispatch_meta_bytes=229208 combine_payload_bytes=233963520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234192728 ingress_bytes=252837888 link_bytes=252837888 token_bytes=468156248 weight_movement_bytes=18874368 total_nvlink_bytes=487030616 weight_to_token_ratio=0.040316 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422568239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64888 payload_rows_remote=57344 dup_rows=648 dup_rows_remote=648 dedup_ratio=0.009888 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7544, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 648, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7544, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229360 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085808 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942256 weight_movement_bytes=18874368 total_nvlink_bytes=488816624 weight_to_token_ratio=0.040163 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=216956928 dispatch_meta_bytes=229368 combine_payload_bytes=216956928 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=217186296 ingress_bytes=254705664 link_bytes=254705664 token_bytes=434143224 weight_movement_bytes=37748736 total_nvlink_bytes=471891960 weight_to_token_ratio=0.086950 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.440464349e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235044864 ingress_bytes=291438592 link_bytes=291438592 token_bytes=469860352 weight_movement_bytes=56623104 total_nvlink_bytes=526483456 weight_to_token_ratio=0.120510 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792421192e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57344 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8166, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 24, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8166, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=231968 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235112992 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469994016 weight_movement_bytes=18874368 total_nvlink_bytes=488868384 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229384 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110408 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991432 weight_movement_bytes=18874368 total_nvlink_bytes=488865800 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=272523264 link_bytes=272523264 token_bytes=469778432 weight_movement_bytes=37748736 total_nvlink_bytes=507527168 weight_to_token_ratio=0.080354 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611183826e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=234881024 link_bytes=235110400 token_bytes=469991424 weight_movement_bytes=0 total_nvlink_bytes=469991424 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.252712171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65166 payload_rows_remote=56974 dup_rows=370 dup_rows_remote=330 dedup_ratio=0.005646 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 7882, 8192, 8138, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 310, 0, 20, 40] combine_rows_per_src=[8192, 8186, 7882, 8192, 8138, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=12 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 4095, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8190, 8190, 8186, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 4, 2, 2, 6, 0] combine_rows_per_src=[8190, 8190, 8190, 8186, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 8192, 8192, 8136, 8192, 8186] dup_rows_per_dst=[0, 6, 0, 52, 8, 0, 2, 0] combine_rows_per_src=[8192, 8188, 8190, 8192, 8192, 8136, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64986 payload_rows_remote=56794 dup_rows=550 dup_rows_remote=550 dedup_ratio=0.008392 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7642, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 550, 0] combine_rows_per_src=[8192, 8192, 8192, 7642, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233365504 dispatch_meta_bytes=229216 combine_payload_bytes=233365504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233594720 ingress_bytes=271114240 link_bytes=271114240 token_bytes=466960224 weight_movement_bytes=37748736 total_nvlink_bytes=504708960 weight_to_token_ratio=0.080839 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597683251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229368 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053048 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876728 weight_movement_bytes=18874368 total_nvlink_bytes=488751096 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229376 combine_payload_bytes=234602496 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234831872 ingress_bytes=234602496 link_bytes=234831872 token_bytes=469434368 weight_movement_bytes=0 total_nvlink_bytes=469434368 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250043453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=2 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232628224 dispatch_meta_bytes=229376 combine_payload_bytes=232628224 prefetch_bytes=0 reduce_bytes=132120576 egress_bytes=232857600 ingress_bytes=364748800 link_bytes=364748800 token_bytes=465485824 weight_movement_bytes=132120576 total_nvlink_bytes=597606400 weight_to_token_ratio=0.283834 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.494843534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65170 payload_rows_remote=57344 dup_rows=366 dup_rows_remote=366 dedup_ratio=0.005585 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7826, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 364, 0] combine_rows_per_src=[8192, 8192, 8192, 7826, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64070 payload_rows_remote=55878 dup_rows=1466 dup_rows_remote=1466 dedup_ratio=0.022369 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6726, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1466, 0] combine_rows_per_src=[8192, 8192, 8192, 6726, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65462 payload_rows_remote=57270 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8118, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 74, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8118, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229352 combine_payload_bytes=234848256 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235077608 ingress_bytes=272596992 link_bytes=272596992 token_bytes=469925864 weight_movement_bytes=37748736 total_nvlink_bytes=507674600 weight_to_token_ratio=0.080329 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611890251e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=230840 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235111864 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469992888 weight_movement_bytes=18874368 total_nvlink_bytes=488867256 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=228876288 dispatch_meta_bytes=229376 combine_payload_bytes=228876288 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229105664 ingress_bytes=266625024 link_bytes=266625024 token_bytes=457981952 weight_movement_bytes=37748736 total_nvlink_bytes=495730688 weight_to_token_ratio=0.082424 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.554669792e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234577920 dispatch_meta_bytes=229376 combine_payload_bytes=234577920 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=234807296 ingress_bytes=347824128 link_bytes=347824128 token_bytes=469385216 weight_movement_bytes=113246208 total_nvlink_bytes=582631424 weight_to_token_ratio=0.241265 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.332679654e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 4095, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61770 payload_rows_remote=53582 dup_rows=3766 dup_rows_remote=3760 dedup_ratio=0.057465 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4005, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 4094, 0, 89, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8188, 8188, 8188, 8192, 8192, 8176, 4454, 8192] dup_rows_per_dst=[6, 6, 4, 3746, 4, 0, 0, 0] combine_rows_per_src=[8188, 8188, 8188, 8192, 8192, 8176, 4454, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=58 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 8192, 8154, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 58, 0, 6] combine_rows_per_src=[8192, 8192, 8170, 8192, 8154, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57344 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8192] dup_rows_per_dst=[6, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229368 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235094008 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958648 weight_movement_bytes=37748736 total_nvlink_bytes=507707384 weight_to_token_ratio=0.080324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=219471872 dispatch_meta_bytes=229368 combine_payload_bytes=219471872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219701240 ingress_bytes=238346240 link_bytes=238346240 token_bytes=439173112 weight_movement_bytes=18874368 total_nvlink_bytes=458047480 weight_to_token_ratio=0.042977 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283716398e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229352 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234848232 ingress_bytes=291241984 link_bytes=291241984 token_bytes=469467112 weight_movement_bytes=56623104 total_nvlink_bytes=526090216 weight_to_token_ratio=0.120611 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790537391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229408 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110432 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991456 weight_movement_bytes=18874368 total_nvlink_bytes=488865824 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64882 payload_rows_remote=56690 dup_rows=654 dup_rows_remote=652 dedup_ratio=0.009979 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7538, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 6, 646, 0] combine_rows_per_src=[8192, 8192, 8192, 7538, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=4 dedup_ratio=0.000305 router_tokens_per_expert=[0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8190, 8176, 8192] dup_rows_per_dst=[4, 16, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8190, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232202240 dispatch_meta_bytes=229368 combine_payload_bytes=232202240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232431608 ingress_bytes=251076608 link_bytes=251076608 token_bytes=464633848 weight_movement_bytes=18874368 total_nvlink_bytes=483508216 weight_to_token_ratio=0.040622 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405692520e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229312 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235028416 ingress_bytes=291422208 link_bytes=291422208 token_bytes=469827520 weight_movement_bytes=56623104 total_nvlink_bytes=526450624 weight_to_token_ratio=0.120519 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792264208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65432 payload_rows_remote=57278 dup_rows=104 dup_rows_remote=104 dedup_ratio=0.001587 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4095] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8154, 8130, 8192, 8190] dup_rows_per_dst=[2, 0, 2, 0, 0, 0, 4, 96] combine_rows_per_src=[8192, 8192, 8192, 8190, 8154, 8130, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57324 dup_rows=24 dup_rows_remote=16 dedup_ratio=0.000366 router_tokens_per_expert=[0, 4070, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8188, 8186, 8186, 8188, 8192, 8192] dup_rows_per_dst=[8, 8, 0, 4, 4, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8188, 8186, 8186, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64872 payload_rows_remote=56680 dup_rows=664 dup_rows_remote=662 dedup_ratio=0.010132 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7536, 8186, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 658, 2] combine_rows_per_src=[8192, 8192, 8192, 7536, 8186, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 5, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8122, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 72] combine_rows_per_src=[8192, 8192, 8190, 8192, 8122, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229528 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234840216 ingress_bytes=253485056 link_bytes=253485056 token_bytes=469450904 weight_movement_bytes=18874368 total_nvlink_bytes=488325272 weight_to_token_ratio=0.040205 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229360 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235028464 ingress_bytes=291422208 link_bytes=291422208 token_bytes=469827568 weight_movement_bytes=56623104 total_nvlink_bytes=526450672 weight_to_token_ratio=0.120519 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792264208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232161280 dispatch_meta_bytes=229368 combine_payload_bytes=232161280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232390648 ingress_bytes=251035648 link_bytes=251035648 token_bytes=464551928 weight_movement_bytes=18874368 total_nvlink_bytes=483426296 weight_to_token_ratio=0.040629 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405300062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234815488 ingress_bytes=272334848 link_bytes=272334848 token_bytes=469401600 weight_movement_bytes=37748736 total_nvlink_bytes=507150336 weight_to_token_ratio=0.080419 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609378516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57338 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8190, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8182, 8190, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 24] combine_rows_per_src=[8192, 8192, 8182, 8190, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57292 dup_rows=52 dup_rows_remote=4 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8140, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 48, 0] combine_rows_per_src=[8192, 8192, 8192, 8140, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=272605184 link_bytes=272605184 token_bytes=469942272 weight_movement_bytes=37748736 total_nvlink_bytes=507691008 weight_to_token_ratio=0.080326 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611968743e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229384 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085832 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942280 weight_movement_bytes=18874368 total_nvlink_bytes=488816648 weight_to_token_ratio=0.040163 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64798 payload_rows_remote=56606 dup_rows=738 dup_rows_remote=736 dedup_ratio=0.011261 router_tokens_per_expert=[0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 7458, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 2, 2, 0, 732, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 7458, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229184 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234897216 ingress_bytes=272416768 link_bytes=272416768 token_bytes=469565248 weight_movement_bytes=37748736 total_nvlink_bytes=507313984 weight_to_token_ratio=0.080391 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610163434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65228 payload_rows_remote=57036 dup_rows=308 dup_rows_remote=308 dedup_ratio=0.004700 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8168, 8188, 8180, 7928] dup_rows_per_dst=[0, 0, 14, 12, 208, 56, 12, 6] combine_rows_per_src=[8192, 8192, 8192, 8188, 8168, 8188, 8180, 7928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8162, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 0, 14, 16, 0] combine_rows_per_src=[8186, 8162, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57344 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8156, 8192, 8192] dup_rows_per_dst=[12, 0, 0, 2, 0, 0, 22, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8156, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=231858176 dispatch_meta_bytes=229368 combine_payload_bytes=231858176 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232087544 ingress_bytes=288481280 link_bytes=288481280 token_bytes=463945720 weight_movement_bytes=56623104 total_nvlink_bytes=520568824 weight_to_token_ratio=0.122047 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.764085683e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233619456 dispatch_meta_bytes=229376 combine_payload_bytes=233619456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233848832 ingress_bytes=252493824 link_bytes=252493824 token_bytes=467468288 weight_movement_bytes=18874368 total_nvlink_bytes=486342656 weight_to_token_ratio=0.040376 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419271587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229376 combine_payload_bytes=234733568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234962944 ingress_bytes=253607936 link_bytes=253607936 token_bytes=469696512 weight_movement_bytes=18874368 total_nvlink_bytes=488570880 weight_to_token_ratio=0.040184 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429946460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229520 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110544 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991568 weight_movement_bytes=37748736 total_nvlink_bytes=507740304 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57314 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8188, 8190, 8164, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 30] combine_rows_per_src=[8192, 8192, 8188, 8190, 8164, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4089, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8192] dup_rows_per_dst=[16, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8192] dup_rows_per_dst=[20, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229392 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234987536 ingress_bytes=272506880 link_bytes=272506880 token_bytes=469745680 weight_movement_bytes=37748736 total_nvlink_bytes=507494416 weight_to_token_ratio=0.080360 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611026842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794816 weight_movement_bytes=37748736 total_nvlink_bytes=507543552 weight_to_token_ratio=0.080352 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64156 payload_rows_remote=56712 dup_rows=1380 dup_rows_remote=1378 dedup_ratio=0.021057 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 581, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4083, 0, 3721, 0, 0, 169, 1, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 4095, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8180, 7444, 7578, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 1152, 2, 190, 36, 0] combine_rows_per_src=[8192, 8188, 8192, 8180, 7444, 7578, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235012096 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794816 weight_movement_bytes=18874368 total_nvlink_bytes=488669184 weight_to_token_ratio=0.040176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64352 payload_rows_remote=56160 dup_rows=1184 dup_rows_remote=1168 dedup_ratio=0.018066 router_tokens_per_expert=[0, 0, 0, 2, 4094, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8180, 8190, 7992, 7230, 8188] dup_rows_per_dst=[16, 0, 2, 4, 4, 962, 196, 0] combine_rows_per_src=[8192, 8188, 8192, 8180, 8190, 7992, 7230, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=8 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8164, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 8, 0, 30] combine_rows_per_src=[8186, 8192, 8192, 8192, 8164, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59802 payload_rows_remote=51634 dup_rows=5734 dup_rows_remote=3900 dedup_ratio=0.087494 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 1248, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 2496, 8168, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 3894, 0, 1834, 6, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 2496, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=232292352 dispatch_meta_bytes=232360 combine_payload_bytes=232292352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232524712 ingress_bytes=270041088 link_bytes=270041088 token_bytes=464817064 weight_movement_bytes=37748736 total_nvlink_bytes=502565800 weight_to_token_ratio=0.081212 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=23040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.587400837e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=230031360 dispatch_meta_bytes=229312 combine_payload_bytes=230031360 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230260672 ingress_bytes=305528832 link_bytes=305528832 token_bytes=460292032 weight_movement_bytes=75497472 total_nvlink_bytes=535789504 weight_to_token_ratio=0.164021 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.927426939e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229256 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234954632 ingress_bytes=310222848 link_bytes=310222848 token_bytes=469680008 weight_movement_bytes=75497472 total_nvlink_bytes=545177480 weight_to_token_ratio=0.160742 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972402690e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=211492864 dispatch_meta_bytes=222136 combine_payload_bytes=211492864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=211715000 ingress_bytes=268115968 link_bytes=268115968 token_bytes=423207864 weight_movement_bytes=56623104 total_nvlink_bytes=479830968 weight_to_token_ratio=0.133795 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568955284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61754 payload_rows_remote=53564 dup_rows=3782 dup_rows_remote=3782 dedup_ratio=0.057709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 2255, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8156, 4510, 8130, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1828, 0, 1952, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 8156, 4510, 8130, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62244 payload_rows_remote=54052 dup_rows=3292 dup_rows_remote=620 dedup_ratio=0.050232 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2468, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4936, 8156, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2672, 0, 620, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 4936, 8156, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=219398144 dispatch_meta_bytes=229384 combine_payload_bytes=219398144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219627528 ingress_bytes=238272512 link_bytes=238272512 token_bytes=439025672 weight_movement_bytes=18874368 total_nvlink_bytes=457900040 weight_to_token_ratio=0.042991 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283009973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57304 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 4078, 0, 0, 15, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8156, 8190, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 30, 0, 4, 0, 2] combine_rows_per_src=[8192, 8192, 8156, 8190, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=221396992 dispatch_meta_bytes=218688 combine_payload_bytes=221396992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221615680 ingress_bytes=240271360 link_bytes=240271360 token_bytes=443012672 weight_movement_bytes=18874368 total_nvlink_bytes=461887040 weight_to_token_ratio=0.042605 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.302161951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60846 payload_rows_remote=52654 dup_rows=4690 dup_rows_remote=4684 dedup_ratio=0.071564 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1014, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 4033, 0, 3225, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 0, 0, 4094, 0, 149, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8140, 8192, 8120, 8052, 3862, 8096, 8192] dup_rows_per_dst=[6, 60, 0, 1982, 2568, 42, 32, 0] combine_rows_per_src=[8192, 8140, 8192, 8120, 8052, 3862, 8096, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65268 payload_rows_remote=57076 dup_rows=268 dup_rows_remote=182 dedup_ratio=0.004089 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3962, 0, 0, 91, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7924, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 182, 0, 86, 0, 0] combine_rows_per_src=[8192, 8192, 7924, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57266 dup_rows=78 dup_rows_remote=74 dedup_ratio=0.001190 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8192, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 68, 0, 4] combine_rows_per_src=[8118, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229376 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234946560 ingress_bytes=253591552 link_bytes=253591552 token_bytes=469663744 weight_movement_bytes=18874368 total_nvlink_bytes=488538112 weight_to_token_ratio=0.040187 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429789476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=215670784 dispatch_meta_bytes=229352 combine_payload_bytes=215670784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215900136 ingress_bytes=234545152 link_bytes=234545152 token_bytes=431570920 weight_movement_bytes=18874368 total_nvlink_bytes=450445288 weight_to_token_ratio=0.043734 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247296243e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233783296 dispatch_meta_bytes=229032 combine_payload_bytes=233783296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234012328 ingress_bytes=252657664 link_bytes=252657664 token_bytes=467795624 weight_movement_bytes=18874368 total_nvlink_bytes=486669992 weight_to_token_ratio=0.040347 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420841421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234561536 dispatch_meta_bytes=229360 combine_payload_bytes=234561536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234790896 ingress_bytes=272310272 link_bytes=272310272 token_bytes=469352432 weight_movement_bytes=37748736 total_nvlink_bytes=507101168 weight_to_token_ratio=0.080427 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609143041e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64980 payload_rows_remote=56788 dup_rows=556 dup_rows_remote=556 dedup_ratio=0.008484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 0, 3819, 0, 0, 204, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7638, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 408, 0, 148, 0, 0] combine_rows_per_src=[8192, 8192, 7638, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64122 payload_rows_remote=55930 dup_rows=1414 dup_rows_remote=1412 dedup_ratio=0.021576 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 3508, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8162, 7016, 7986, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 1250, 0, 162, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8162, 7016, 7986, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=28 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 4065, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8130, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 36, 0, 26, 0, 0] combine_rows_per_src=[8192, 8192, 8130, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=232603648 dispatch_meta_bytes=229376 combine_payload_bytes=232603648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232833024 ingress_bytes=251478016 link_bytes=251478016 token_bytes=465436672 weight_movement_bytes=18874368 total_nvlink_bytes=484311040 weight_to_token_ratio=0.040552 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409538614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65490 payload_rows_remote=57298 dup_rows=46 dup_rows_remote=44 dedup_ratio=0.000702 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8190, 8150, 8192] dup_rows_per_dst=[2, 0, 0, 0, 2, 40, 0, 2] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8190, 8150, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=229089280 dispatch_meta_bytes=229368 combine_payload_bytes=229089280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229318648 ingress_bytes=247963648 link_bytes=247963648 token_bytes=458407928 weight_movement_bytes=18874368 total_nvlink_bytes=477282296 weight_to_token_ratio=0.041174 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.375865669e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64442 payload_rows_remote=56250 dup_rows=1094 dup_rows_remote=1094 dedup_ratio=0.016693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 4040, 0, 0, 43, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 7410, 8192, 7900, 8182, 8186] dup_rows_per_dst=[0, 8, 1054, 14, 4, 0, 10, 4] combine_rows_per_src=[8192, 8192, 8188, 7410, 8192, 7900, 8182, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229232 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234848112 ingress_bytes=272367616 link_bytes=272367616 token_bytes=469466992 weight_movement_bytes=37748736 total_nvlink_bytes=507215728 weight_to_token_ratio=0.080408 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609692483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63502 payload_rows_remote=55310 dup_rows=2034 dup_rows_remote=0 dedup_ratio=0.031036 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 6160, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2034, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 6160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234692608 dispatch_meta_bytes=229368 combine_payload_bytes=234692608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234921976 ingress_bytes=272441344 link_bytes=272441344 token_bytes=469614584 weight_movement_bytes=37748736 total_nvlink_bytes=507363320 weight_to_token_ratio=0.080382 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610398909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59196 payload_rows_remote=51004 dup_rows=6340 dup_rows_remote=6340 dedup_ratio=0.096741 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 2737, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8190, 1870, 8186, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 5428, 892, 20, 0] combine_rows_per_src=[8186, 8192, 8190, 1870, 8186, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=230400000 dispatch_meta_bytes=229376 combine_payload_bytes=230400000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230629376 ingress_bytes=249274368 link_bytes=249274368 token_bytes=461029376 weight_movement_bytes=18874368 total_nvlink_bytes=479903744 weight_to_token_ratio=0.040940 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388424344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=226549760 dispatch_meta_bytes=221240 combine_payload_bytes=226549760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226771000 ingress_bytes=245424128 link_bytes=245424128 token_bytes=453320760 weight_movement_bytes=18874368 total_nvlink_bytes=472195128 weight_to_token_ratio=0.041636 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351533238e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63326 payload_rows_remote=55134 dup_rows=2210 dup_rows_remote=2210 dedup_ratio=0.033722 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 5982, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2210, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 5982, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=208912384 dispatch_meta_bytes=229376 combine_payload_bytes=208912384 prefetch_bytes=0 reduce_bytes=0 egress_bytes=209141760 ingress_bytes=208912384 link_bytes=209141760 token_bytes=418054144 weight_movement_bytes=0 total_nvlink_bytes=418054144 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.003893440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58096 payload_rows_remote=49904 dup_rows=7440 dup_rows_remote=7438 dedup_ratio=0.113525 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3785, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 1584, 7870, 7682, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 316, 7028, 94, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 1584, 7870, 7682, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57340 dup_rows=146 dup_rows_remote=146 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4025, 0, 0, 54, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8050, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 108, 0, 34, 0, 0] combine_rows_per_src=[8192, 8192, 8050, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=225828864 dispatch_meta_bytes=229376 combine_payload_bytes=225828864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=226058240 ingress_bytes=263577600 link_bytes=263577600 token_bytes=451887104 weight_movement_bytes=37748736 total_nvlink_bytes=489635840 weight_to_token_ratio=0.083536 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.525470875e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64834 payload_rows_remote=56642 dup_rows=702 dup_rows_remote=702 dedup_ratio=0.010712 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 7492, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 702, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 7492, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64826 payload_rows_remote=56634 dup_rows=710 dup_rows_remote=710 dedup_ratio=0.010834 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 292, 0] payload_rows_per_dst=[8190, 8192, 7490, 8190, 8192, 8190, 8192, 8190] dup_rows_per_dst=[0, 2, 2, 0, 0, 2, 120, 584] combine_rows_per_src=[8190, 8192, 7490, 8190, 8192, 8190, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=204406784 dispatch_meta_bytes=229368 combine_payload_bytes=204406784 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=204636152 ingress_bytes=261029888 link_bytes=261029888 token_bytes=409042936 weight_movement_bytes=56623104 total_nvlink_bytes=465666040 weight_to_token_ratio=0.138428 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.501059952e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58346 payload_rows_remote=50154 dup_rows=7190 dup_rows_remote=7184 dedup_ratio=0.109711 router_tokens_per_expert=[2, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8186, 1090, 8122, 8184] dup_rows_per_dst=[6, 0, 7176, 0, 4, 0, 0, 4] combine_rows_per_src=[8192, 8188, 8192, 8192, 8186, 1090, 8122, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229944 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235094584 ingress_bytes=310362112 link_bytes=310362112 token_bytes=469959224 weight_movement_bytes=75497472 total_nvlink_bytes=545456696 weight_to_token_ratio=0.160647 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973737049e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=232005632 dispatch_meta_bytes=229376 combine_payload_bytes=232005632 prefetch_bytes=0 reduce_bytes=0 egress_bytes=232235008 ingress_bytes=232005632 link_bytes=232235008 token_bytes=464240640 weight_movement_bytes=0 total_nvlink_bytes=464240640 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225161580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60468 payload_rows_remote=52276 dup_rows=5068 dup_rows_remote=5068 dedup_ratio=0.077332 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1291, 0] payload_rows_per_dst=[8192, 8192, 3124, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2486, 2582] combine_rows_per_src=[8192, 8192, 3124, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=231972864 dispatch_meta_bytes=229376 combine_payload_bytes=231972864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232202240 ingress_bytes=250847232 link_bytes=250847232 token_bytes=464175104 weight_movement_bytes=18874368 total_nvlink_bytes=483049472 weight_to_token_ratio=0.040662 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403494752e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8180, 8186, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 18, 6, 0, 0, 0, 0] combine_rows_per_src=[8186, 8180, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=205430784 dispatch_meta_bytes=229352 combine_payload_bytes=205430784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205660136 ingress_bytes=224305152 link_bytes=224305152 token_bytes=411090920 weight_movement_bytes=18874368 total_nvlink_bytes=429965288 weight_to_token_ratio=0.045913 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.149181601e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60646 payload_rows_remote=52454 dup_rows=4890 dup_rows_remote=4890 dedup_ratio=0.074615 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1652, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 3304] dup_rows_per_dst=[0, 0, 3330, 0, 0, 0, 1560, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 3304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=214122496 dispatch_meta_bytes=229376 combine_payload_bytes=214122496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214351872 ingress_bytes=232996864 link_bytes=232996864 token_bytes=428474368 weight_movement_bytes=18874368 total_nvlink_bytes=447348736 weight_to_token_ratio=0.044050 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232461309e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235012096 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794816 weight_movement_bytes=37748736 total_nvlink_bytes=507543552 weight_to_token_ratio=0.080352 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=214851584 dispatch_meta_bytes=229376 combine_payload_bytes=214851584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215080960 ingress_bytes=233725952 link_bytes=233725952 token_bytes=429932544 weight_movement_bytes=18874368 total_nvlink_bytes=448806912 weight_to_token_ratio=0.043901 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239447072e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65080 payload_rows_remote=56894 dup_rows=456 dup_rows_remote=456 dedup_ratio=0.006958 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8192, 8192, 7742, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 456, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8192, 8192, 7742, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[4094, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64318 payload_rows_remote=56126 dup_rows=1218 dup_rows_remote=1218 dedup_ratio=0.018585 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 323, 0] payload_rows_per_dst=[8192, 8192, 6974, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 572, 646] combine_rows_per_src=[8192, 8192, 6974, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59496 payload_rows_remote=51308 dup_rows=6040 dup_rows_remote=6030 dedup_ratio=0.092163 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1087, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8192, 8192, 8178, 2174] dup_rows_per_dst=[10, 4, 5226, 2, 4, 6, 788, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8192, 8192, 8178, 2174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229400 combine_payload_bytes=233037824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233267224 ingress_bytes=289660928 link_bytes=289660928 token_bytes=466305048 weight_movement_bytes=56623104 total_nvlink_bytes=522928152 weight_to_token_ratio=0.121429 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.775388490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235085824 ingress_bytes=253730816 link_bytes=253730816 token_bytes=469942272 weight_movement_bytes=18874368 total_nvlink_bytes=488816640 weight_to_token_ratio=0.040163 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431123836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65394 payload_rows_remote=57202 dup_rows=142 dup_rows_remote=0 dedup_ratio=0.002167 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4030, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8092, 8192, 8192, 8192, 8150] dup_rows_per_dst=[0, 0, 0, 0, 0, 142, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8092, 8192, 8192, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=229892096 dispatch_meta_bytes=229376 combine_payload_bytes=229892096 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230121472 ingress_bytes=267640832 link_bytes=267640832 token_bytes=460013568 weight_movement_bytes=37748736 total_nvlink_bytes=497762304 weight_to_token_ratio=0.082060 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.564402765e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=210157568 dispatch_meta_bytes=229352 combine_payload_bytes=210157568 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210386920 ingress_bytes=266780672 link_bytes=266780672 token_bytes=420544488 weight_movement_bytes=56623104 total_nvlink_bytes=477167592 weight_to_token_ratio=0.134642 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.556161135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65302 payload_rows_remote=57110 dup_rows=234 dup_rows_remote=234 dedup_ratio=0.003571 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3981, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7998, 8192, 8192, 8192, 8152] dup_rows_per_dst=[0, 0, 0, 0, 0, 234, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7998, 8192, 8192, 8192, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57690 payload_rows_remote=49498 dup_rows=7846 dup_rows_remote=7844 dedup_ratio=0.119720 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 1026, 8192, 8192, 8192, 7520, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 1652, 6186, 2] combine_rows_per_src=[8184, 8192, 1026, 8192, 8192, 8192, 7520, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234299392 dispatch_meta_bytes=228808 combine_payload_bytes=234299392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234528200 ingress_bytes=253173760 link_bytes=253173760 token_bytes=468827592 weight_movement_bytes=18874368 total_nvlink_bytes=487701960 weight_to_token_ratio=0.040259 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425786399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57344 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=229376 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234151936 ingress_bytes=252796928 link_bytes=252796928 token_bytes=468074496 weight_movement_bytes=18874368 total_nvlink_bytes=486948864 weight_to_token_ratio=0.040323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422175780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=202743808 dispatch_meta_bytes=229368 combine_payload_bytes=202743808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202973176 ingress_bytes=240492544 link_bytes=240492544 token_bytes=405716984 weight_movement_bytes=37748736 total_nvlink_bytes=443465720 weight_to_token_ratio=0.093042 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.304281227e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64018 payload_rows_remote=55826 dup_rows=1518 dup_rows_remote=1518 dedup_ratio=0.023163 router_tokens_per_expert=[0, 1, 0, 756, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 7384, 7488, 8192, 8190, 8190, 8192] dup_rows_per_dst=[1514, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8190, 7384, 7488, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61242 payload_rows_remote=53876 dup_rows=4294 dup_rows_remote=1678 dedup_ratio=0.065521 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 190, 0] payload_rows_per_dst=[8192, 8192, 7366, 8192, 8188, 8192, 8192, 4728] dup_rows_per_dst=[0, 0, 2616, 0, 0, 4, 1434, 240] combine_rows_per_src=[8192, 8192, 7366, 8192, 8188, 8192, 8192, 4728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57334 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 0, 0, 0, 0, 18, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229416 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110440 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991464 weight_movement_bytes=18874368 total_nvlink_bytes=488865832 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65082 payload_rows_remote=56894 dup_rows=454 dup_rows_remote=450 dedup_ratio=0.006927 router_tokens_per_expert=[4083, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8188, 7752, 8190, 8192] dup_rows_per_dst=[4, 4, 0, 444, 0, 2, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8188, 7752, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=220676096 dispatch_meta_bytes=222216 combine_payload_bytes=220676096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220898312 ingress_bytes=258424832 link_bytes=258424832 token_bytes=441574408 weight_movement_bytes=37748736 total_nvlink_bytes=479323144 weight_to_token_ratio=0.085487 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.476099587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64274 payload_rows_remote=56082 dup_rows=1262 dup_rows_remote=1262 dedup_ratio=0.019257 router_tokens_per_expert=[0, 0, 0, 570, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 7094, 8046, 8192, 8192, 8190, 8192] dup_rows_per_dst=[1140, 0, 0, 0, 0, 0, 0, 122] combine_rows_per_src=[8192, 8176, 7094, 8046, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=228663296 dispatch_meta_bytes=229376 combine_payload_bytes=228663296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228892672 ingress_bytes=247537664 link_bytes=247537664 token_bytes=457555968 weight_movement_bytes=18874368 total_nvlink_bytes=476430336 weight_to_token_ratio=0.041250 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371784100e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229408 combine_payload_bytes=234840064 prefetch_bytes=0 reduce_bytes=132120576 egress_bytes=235069472 ingress_bytes=366960640 link_bytes=366960640 token_bytes=469909536 weight_movement_bytes=132120576 total_nvlink_bytes=602030112 weight_to_token_ratio=0.281162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.516036297e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63352 payload_rows_remote=55160 dup_rows=2184 dup_rows_remote=2184 dedup_ratio=0.033325 router_tokens_per_expert=[0, 0, 0, 1045, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6112, 8088, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2090, 0, 0, 0, 0, 0, 0, 94] combine_rows_per_src=[8192, 8192, 6112, 8088, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233037824 dispatch_meta_bytes=229376 combine_payload_bytes=233037824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233267200 ingress_bytes=251912192 link_bytes=251912192 token_bytes=466305024 weight_movement_bytes=18874368 total_nvlink_bytes=485179392 weight_to_token_ratio=0.040476 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413698675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57002 payload_rows_remote=48810 dup_rows=8534 dup_rows_remote=8 dedup_ratio=0.130219 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 167, 0] payload_rows_per_dst=[8188, 8186, 7158, 8192, 8192, 8192, 702, 8192] dup_rows_per_dst=[0, 0, 4, 4, 0, 0, 0, 8526] combine_rows_per_src=[8188, 8186, 7158, 8192, 8192, 8192, 702, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21626799200 prefetch_bytes=905969664 reduce_bytes=698351616 weight_movement_bytes=1604321280 egress_bytes=10818880608 ingress_bytes=12412239872 link_bytes=12412239872 total_nvlink_bytes=23231120480 weight_to_token_ratio=0.074182 prefetch_share_of_ingress=0.072990 dedup_ratio=0.040487 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=37 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101128 gemm_inner_pad_upper_bound=31496 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.477666151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=229711872 dispatch_meta_bytes=229376 combine_payload_bytes=229711872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229941248 ingress_bytes=248586240 link_bytes=248586240 token_bytes=459653120 weight_movement_bytes=18874368 total_nvlink_bytes=478527488 weight_to_token_ratio=0.041062 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381831040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62586 payload_rows_remote=55006 dup_rows=2950 dup_rows_remote=1794 dedup_ratio=0.045013 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 242, 0] payload_rows_per_dst=[8192, 8192, 5862, 8192, 8192, 8184, 7580, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1156, 1794] combine_rows_per_src=[8192, 8192, 5862, 8192, 8192, 8184, 7580, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=225935360 dispatch_meta_bytes=229376 combine_payload_bytes=225935360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226164736 ingress_bytes=244809728 link_bytes=244809728 token_bytes=452100096 weight_movement_bytes=18874368 total_nvlink_bytes=470974464 weight_to_token_ratio=0.041748 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.345646360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=199925760 dispatch_meta_bytes=195272 combine_payload_bytes=199925760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=200121032 ingress_bytes=237674496 link_bytes=237674496 token_bytes=400046792 weight_movement_bytes=37748736 total_nvlink_bytes=437795528 weight_to_token_ratio=0.094361 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277280078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8158, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 0, 6, 0, 44, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8158, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21360592664 prefetch_bytes=849346560 reduce_bytes=811597824 weight_movement_bytes=1660944384 egress_bytes=10685777688 ingress_bytes=12335759360 link_bytes=12335988736 total_nvlink_bytes=23021537048 weight_to_token_ratio=0.077757 prefetch_share_of_ingress=0.068852 dedup_ratio=0.051224 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=43 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101562 gemm_inner_pad_upper_bound=31242 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.462445300e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64820 payload_rows_remote=56646 dup_rows=716 dup_rows_remote=716 dedup_ratio=0.010925 router_tokens_per_expert=[0, 0, 0, 351, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7498, 8174, 8192, 8192, 8188, 8192] dup_rows_per_dst=[702, 0, 0, 0, 0, 0, 0, 14] combine_rows_per_src=[8192, 8192, 7498, 8174, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=225304576 dispatch_meta_bytes=227200 combine_payload_bytes=225304576 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=225531776 ingress_bytes=300802048 link_bytes=300802048 token_bytes=450836352 weight_movement_bytes=75497472 total_nvlink_bytes=526333824 weight_to_token_ratio=0.167461 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.882137220e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21078527736 prefetch_bytes=849346560 reduce_bytes=1019215872 weight_movement_bytes=1868562432 egress_bytes=10544713464 ingress_bytes=12402376704 link_bytes=12402376704 total_nvlink_bytes=22947090168 weight_to_token_ratio=0.088648 prefetch_share_of_ingress=0.068483 dedup_ratio=0.058810 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=54 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101997 gemm_inner_pad_upper_bound=29972 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.475697317e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63312 payload_rows_remote=55120 dup_rows=2224 dup_rows_remote=66 dedup_ratio=0.033936 router_tokens_per_expert=[0, 2, 0, 964, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8188, 7604, 6850, 8188, 8188, 8188, 7914] dup_rows_per_dst=[2158, 32, 0, 0, 4, 2, 20, 8] combine_rows_per_src=[8192, 8188, 7604, 6850, 8188, 8188, 8188, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229376 combine_payload_bytes=234676224 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234905600 ingress_bytes=234676224 link_bytes=234905600 token_bytes=469581824 weight_movement_bytes=0 total_nvlink_bytes=469581824 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.250749878e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=232022016 dispatch_meta_bytes=229448 combine_payload_bytes=232022016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232251464 ingress_bytes=250896384 link_bytes=250896384 token_bytes=464273480 weight_movement_bytes=18874368 total_nvlink_bytes=483147848 weight_to_token_ratio=0.040654 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403965703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=225771520 dispatch_meta_bytes=220744 combine_payload_bytes=225771520 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=225992264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=451763784 weight_movement_bytes=75497472 total_nvlink_bytes=527261256 weight_to_token_ratio=0.167117 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.886611248e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=8 dedup_ratio=0.001770 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8084, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 4, 108] combine_rows_per_src=[8192, 8184, 8192, 8084, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21660779792 prefetch_bytes=792723456 reduce_bytes=660602880 weight_movement_bytes=1453326336 egress_bytes=10835854608 ingress_bytes=12278251520 link_bytes=12278710272 total_nvlink_bytes=23114106128 weight_to_token_ratio=0.067095 prefetch_share_of_ingress=0.064563 dedup_ratio=0.036950 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=35 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101924 gemm_inner_pad_upper_bound=29464 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.451011673e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20478087232 prefetch_bytes=830472192 reduce_bytes=1000341504 weight_movement_bytes=1830813696 egress_bytes=10244378688 ingress_bytes=12064522240 link_bytes=12064751616 total_nvlink_bytes=22308900928 weight_to_token_ratio=0.089404 prefetch_share_of_ingress=0.068836 dedup_ratio=0.081840 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=53 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102069 gemm_inner_pad_upper_bound=30607 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.408302370e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=228944 combine_payload_bytes=234405888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234634832 ingress_bytes=253280256 link_bytes=253280256 token_bytes=469040720 weight_movement_bytes=18874368 total_nvlink_bytes=487915088 weight_to_token_ratio=0.040240 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426806791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65076 payload_rows_remote=57242 dup_rows=460 dup_rows_remote=460 dedup_ratio=0.007019 router_tokens_per_expert=[0, 0, 0, 211, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 7834, 8106, 8192, 8192, 8184, 8192] dup_rows_per_dst=[422, 0, 0, 0, 0, 4, 0, 34] combine_rows_per_src=[8192, 8184, 7834, 8106, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234463232 dispatch_meta_bytes=230808 combine_payload_bytes=234463232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234694040 ingress_bytes=253337600 link_bytes=253337600 token_bytes=469157272 weight_movement_bytes=18874368 total_nvlink_bytes=488031640 weight_to_token_ratio=0.040230 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427356233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21484118000 prefetch_bytes=887095296 reduce_bytes=792723456 weight_movement_bytes=1679818752 egress_bytes=10747527152 ingress_bytes=12416409600 link_bytes=12416868352 total_nvlink_bytes=23163936752 weight_to_token_ratio=0.078189 prefetch_share_of_ingress=0.071445 dedup_ratio=0.044631 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=42 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100477 gemm_inner_pad_upper_bound=33274 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.478590064e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57550 payload_rows_remote=49358 dup_rows=7986 dup_rows_remote=7986 dedup_ratio=0.121857 router_tokens_per_expert=[0, 0, 0, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[242, 8192, 8158, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 5584, 2402, 0, 0, 0, 0] combine_rows_per_src=[242, 8192, 8158, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64514 payload_rows_remote=56322 dup_rows=1022 dup_rows_remote=1022 dedup_ratio=0.015594 router_tokens_per_expert=[0, 0, 0, 240, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7644, 8190, 7720, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[472, 0, 548, 0, 0, 0, 0, 2] combine_rows_per_src=[7644, 8190, 7720, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=202170368 dispatch_meta_bytes=229376 combine_payload_bytes=202170368 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=202399744 ingress_bytes=277667840 link_bytes=277667840 token_bytes=404570112 weight_movement_bytes=75497472 total_nvlink_bytes=480067584 weight_to_token_ratio=0.186612 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.660476622e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=230694912 dispatch_meta_bytes=229376 combine_payload_bytes=230694912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230924288 ingress_bytes=249569280 link_bytes=249569280 token_bytes=461619200 weight_movement_bytes=18874368 total_nvlink_bytes=480493568 weight_to_token_ratio=0.040887 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.391250045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20426889888 prefetch_bytes=849346560 reduce_bytes=887095296 weight_movement_bytes=1736441856 egress_bytes=10218805920 ingress_bytes=11944525824 link_bytes=11944755200 total_nvlink_bytes=22163331744 weight_to_token_ratio=0.085008 prefetch_share_of_ingress=0.071108 dedup_ratio=0.085076 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=47 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=32639 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.384349315e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21311809512 prefetch_bytes=754974720 reduce_bytes=849346560 weight_movement_bytes=1604321280 egress_bytes=10661382120 ingress_bytes=12254748672 link_bytes=12255207424 total_nvlink_bytes=22916130792 weight_to_token_ratio=0.075279 prefetch_share_of_ingress=0.061607 dedup_ratio=0.050261 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=45 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.102358 gemm_inner_pad_upper_bound=29337 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.446320158e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none + [2026-08-29 23:37:47] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 764.3 | throughput per GPU (TFLOP/s/GPU): 116.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.804422E+00 | loss scale: 1.0 | grad norm: 3.053 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38544 dup_rows=21062 dup_rows_remote=18306 dedup_ratio=0.321381 router_tokens_per_expert=[272, 0, 0, 0, 146, 259, 43, 0, 0, 1, 357, 326, 701, 410, 162, 304, 3, 137, 3, 519, 245, 407, 0, 0, 479, 207, 99, 114, 489, 4, 487, 2, 334, 522, 15, 106, 3, 46, 303, 152, 259, 714, 434, 320, 344, 30, 248, 427, 743, 472, 513, 0, 533, 545, 36, 19, 740, 157, 349, 799, 270, 225, 22, 175, 15, 283, 0, 14, 345, 0, 632, 373, 2, 659, 352, 1106, 67, 70, 19, 1198, 92, 17, 65, 313, 27, 230, 5, 0, 762, 446, 92, 174, 235, 397, 126, 164, 0, 90, 150, 81, 0, 377, 755, 44, 110, 11, 583, 321, 430, 302, 451, 0, 411, 439, 328, 150, 300, 1, 811, 118, 723, 164, 643, 19, 309, 2, 334, 0] payload_rows_per_dst=[5356, 5550, 4820, 5646, 5452, 5930, 5622, 6098] dup_rows_per_dst=[2086, 2462, 2266, 2472, 2422, 2756, 3216, 3382] combine_rows_per_src=[5356, 5550, 4820, 5646, 5452, 5930, 5622, 6098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38828 dup_rows=21028 dup_rows_remote=17636 dedup_ratio=0.320862 router_tokens_per_expert=[191, 0, 0, 0, 154, 204, 63, 2, 1, 0, 335, 427, 620, 430, 180, 380, 11, 143, 2, 562, 187, 461, 0, 0, 373, 141, 145, 148, 546, 4, 505, 2, 375, 438, 10, 61, 1, 19, 287, 175, 294, 702, 705, 574, 290, 25, 291, 467, 523, 436, 524, 1, 386, 507, 19, 12, 573, 135, 301, 593, 193, 355, 25, 155, 14, 275, 0, 6, 328, 0, 764, 403, 3, 1032, 366, 1045, 95, 54, 21, 955, 132, 22, 66, 291, 20, 274, 2, 0, 574, 384, 68, 187, 375, 413, 129, 97, 0, 104, 211, 70, 2, 295, 733, 67, 152, 10, 648, 324, 542, 309, 448, 0, 453, 274, 334, 102, 218, 1, 751, 95, 1010, 243, 558, 9, 391, 3, 347, 0] payload_rows_per_dst=[5566, 5646, 5680, 5214, 5708, 5592, 5398, 5704] dup_rows_per_dst=[2600, 2316, 3392, 2070, 3104, 2386, 2788, 2372] combine_rows_per_src=[5566, 5646, 5680, 5214, 5708, 5592, 5398, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38894 dup_rows=21022 dup_rows_remote=18476 dedup_ratio=0.320770 router_tokens_per_expert=[211, 0, 0, 2, 91, 209, 38, 4, 1, 2, 281, 350, 564, 514, 204, 314, 7, 164, 5, 496, 238, 371, 0, 1, 406, 147, 71, 129, 473, 11, 458, 9, 394, 431, 11, 126, 6, 40, 285, 206, 313, 784, 591, 361, 389, 27, 242, 575, 680, 409, 524, 0, 485, 503, 29, 17, 952, 162, 383, 752, 248, 291, 20, 178, 19, 260, 0, 16, 269, 1, 573, 338, 0, 889, 372, 1153, 97, 69, 14, 1018, 141, 25, 61, 229, 17, 234, 3, 0, 820, 482, 63, 186, 210, 317, 156, 179, 0, 44, 145, 138, 1, 485, 585, 71, 124, 23, 578, 256, 362, 221, 552, 0, 361, 398, 262, 109, 337, 2, 970, 115, 801, 242, 440, 6, 402, 4, 338, 0] payload_rows_per_dst=[5804, 5596, 5176, 5620, 5608, 5786, 5398, 5526] dup_rows_per_dst=[2866, 2316, 2818, 2546, 2532, 2496, 3340, 2108] combine_rows_per_src=[5804, 5596, 5176, 5620, 5608, 5786, 5398, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44174 payload_rows_remote=38078 dup_rows=21362 dup_rows_remote=17990 dedup_ratio=0.325958 router_tokens_per_expert=[394, 0, 0, 0, 104, 248, 31, 2, 0, 7, 247, 474, 797, 430, 163, 317, 15, 128, 1, 416, 306, 512, 1, 0, 422, 238, 140, 146, 486, 3, 525, 33, 405, 544, 8, 89, 9, 35, 346, 190, 396, 552, 423, 460, 384, 34, 210, 488, 710, 619, 424, 0, 504, 391, 21, 11, 810, 120, 351, 766, 190, 321, 20, 213, 25, 254, 0, 4, 336, 0, 857, 348, 0, 861, 321, 706, 31, 30, 22, 846, 177, 28, 82, 486, 40, 271, 5, 1, 726, 394, 66, 181, 448, 308, 181, 199, 0, 157, 119, 65, 0, 437, 959, 52, 156, 18, 442, 200, 374, 126, 424, 0, 527, 165, 276, 131, 238, 2, 771, 101, 816, 143, 430, 44, 488, 0, 243, 0] payload_rows_per_dst=[6096, 5636, 5632, 5884, 5684, 5224, 4786, 5232] dup_rows_per_dst=[3372, 2650, 3514, 3160, 2346, 2072, 2272, 1976] combine_rows_per_src=[6096, 5636, 5632, 5884, 5684, 5224, 4786, 5232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45138 payload_rows_remote=39770 dup_rows=20398 dup_rows_remote=17794 dedup_ratio=0.311249 router_tokens_per_expert=[152, 0, 0, 0, 101, 252, 41, 2, 1, 0, 299, 393, 626, 361, 183, 361, 10, 270, 0, 565, 161, 272, 0, 0, 476, 108, 109, 170, 497, 6, 680, 4, 318, 484, 8, 80, 3, 28, 322, 174, 275, 686, 678, 340, 306, 19, 252, 712, 752, 567, 517, 3, 429, 535, 20, 22, 703, 91, 317, 827, 149, 265, 36, 168, 17, 242, 0, 12, 294, 0, 671, 383, 2, 782, 358, 1073, 64, 76, 12, 1157, 120, 12, 56, 226, 27, 252, 0, 1, 737, 343, 80, 186, 223, 468, 142, 134, 0, 85, 177, 49, 0, 307, 568, 74, 92, 18, 604, 354, 564, 293, 398, 1, 446, 404, 237, 163, 319, 4, 1016, 87, 837, 190, 428, 25, 354, 1, 337, 0] payload_rows_per_dst=[5178, 6026, 5282, 6138, 5368, 5882, 5524, 5740] dup_rows_per_dst=[1772, 2662, 2716, 2900, 2604, 2446, 3016, 2282] combine_rows_per_src=[5178, 6026, 5282, 6138, 5368, 5882, 5524, 5740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44310 payload_rows_remote=38584 dup_rows=21226 dup_rows_remote=18290 dedup_ratio=0.323883 router_tokens_per_expert=[342, 0, 0, 1, 53, 244, 31, 3, 0, 6, 237, 355, 689, 608, 206, 317, 10, 104, 2, 451, 206, 407, 3, 0, 472, 127, 172, 174, 294, 6, 467, 13, 411, 636, 18, 95, 8, 53, 363, 189, 395, 625, 554, 393, 454, 34, 399, 526, 528, 540, 460, 0, 564, 458, 17, 20, 892, 188, 307, 909, 140, 436, 38, 246, 81, 234, 1, 6, 341, 0, 649, 407, 1, 756, 381, 887, 57, 53, 13, 913, 188, 37, 75, 387, 31, 337, 1, 0, 777, 369, 60, 242, 200, 456, 175, 169, 0, 170, 122, 81, 0, 301, 686, 75, 99, 26, 445, 299, 509, 141, 594, 0, 391, 156, 219, 103, 215, 2, 835, 125, 730, 50, 392, 22, 454, 3, 343, 0] payload_rows_per_dst=[5578, 5040, 5468, 5622, 5630, 5988, 5258, 5726] dup_rows_per_dst=[2390, 2012, 3302, 2262, 2390, 2846, 3088, 2936] combine_rows_per_src=[5578, 5040, 5468, 5622, 5630, 5988, 5258, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44300 payload_rows_remote=38534 dup_rows=21236 dup_rows_remote=18494 dedup_ratio=0.324036 router_tokens_per_expert=[307, 0, 0, 2, 133, 227, 50, 12, 3, 4, 282, 684, 565, 255, 164, 413, 27, 224, 5, 646, 192, 367, 2, 1, 341, 101, 91, 147, 605, 18, 400, 45, 282, 462, 10, 119, 13, 38, 316, 182, 343, 565, 344, 344, 458, 20, 222, 584, 706, 365, 408, 0, 480, 430, 39, 23, 1000, 220, 429, 870, 267, 364, 33, 117, 51, 294, 2, 42, 267, 0, 599, 400, 26, 796, 340, 950, 53, 99, 37, 1013, 197, 12, 113, 229, 41, 201, 6, 2, 641, 505, 101, 189, 287, 295, 126, 286, 0, 78, 149, 75, 1, 340, 722, 52, 97, 26, 372, 327, 552, 329, 374, 0, 375, 505, 172, 140, 253, 3, 1042, 130, 697, 257, 469, 19, 284, 8, 354, 0] payload_rows_per_dst=[6004, 5766, 5442, 5634, 5422, 5774, 4926, 5332] dup_rows_per_dst=[3610, 2742, 3162, 2456, 2490, 2714, 2062, 2000] combine_rows_per_src=[6004, 5766, 5442, 5634, 5422, 5774, 4926, 5332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157876224 dispatch_meta_bytes=227400 combine_payload_bytes=157876224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158103624 ingress_bytes=176750592 link_bytes=176750592 token_bytes=315979848 weight_movement_bytes=18874368 total_nvlink_bytes=334854216 weight_to_token_ratio=0.059733 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=7552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693537206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=225856 combine_payload_bytes=159039488 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=159265344 ingress_bytes=196788224 link_bytes=196788224 token_bytes=318304832 weight_movement_bytes=37748736 total_nvlink_bytes=356053568 weight_to_token_ratio=0.118593 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.885527937e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=229480 combine_payload_bytes=159309824 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=159539304 ingress_bytes=234807296 link_bytes=234807296 token_bytes=318849128 weight_movement_bytes=75497472 total_nvlink_bytes=394346600 weight_to_token_ratio=0.236781 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=4096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249807978e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=224272 combine_payload_bytes=155967488 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=156191760 ingress_bytes=193716224 link_bytes=193716224 token_bytes=312159248 weight_movement_bytes=37748736 total_nvlink_bytes=349907984 weight_to_token_ratio=0.120928 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856093544e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162897920 dispatch_meta_bytes=230256 combine_payload_bytes=162897920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=163128176 ingress_bytes=219521024 link_bytes=219521024 token_bytes=326026096 weight_movement_bytes=56623104 total_nvlink_bytes=382649200 weight_to_token_ratio=0.173677 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=7552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.103342441e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158040064 dispatch_meta_bytes=227496 combine_payload_bytes=158040064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158267560 ingress_bytes=195788800 link_bytes=195788800 token_bytes=316307624 weight_movement_bytes=37748736 total_nvlink_bytes=354056360 weight_to_token_ratio=0.119342 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.875951947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157835264 dispatch_meta_bytes=228112 combine_payload_bytes=157835264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158063376 ingress_bytes=176709632 link_bytes=176709632 token_bytes=315898640 weight_movement_bytes=18874368 total_nvlink_bytes=334773008 weight_to_token_ratio=0.059748 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=7296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693144747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53548 payload_rows_remote=46770 dup_rows=11988 dup_rows_remote=8904 dedup_ratio=0.182922 router_tokens_per_expert=[0, 79, 0, 0, 2748, 0, 0, 42, 2313, 0, 0, 0, 1347, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 388, 0, 0, 0, 2, 3761, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1206, 0, 2286, 0, 0, 0, 42, 1295, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 845, 0, 630, 3095, 2909, 0, 6, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 157, 12, 0, 0, 0, 0, 0, 5, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0] payload_rows_per_dst=[6848, 8092, 6858, 7612, 6778, 4802, 6528, 6030] dup_rows_per_dst=[1584, 42, 1836, 720, 3084, 1152, 1484, 2086] combine_rows_per_src=[6848, 8092, 6858, 7612, 6778, 4802, 6528, 6030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52274 payload_rows_remote=46722 dup_rows=13262 dup_rows_remote=11558 dedup_ratio=0.202362 router_tokens_per_expert=[0, 268, 0, 0, 2151, 0, 0, 50, 2436, 0, 0, 0, 1911, 0, 0, 3944, 1, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1848, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 2, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 61, 0, 937, 0, 2251, 0, 0, 0, 105, 1005, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 1142, 0, 916, 2925, 3055, 0, 1, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 66, 67, 0, 0, 3, 0, 0, 36, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0] payload_rows_per_dst=[6814, 7894, 7234, 7652, 4212, 5552, 7286, 5630] dup_rows_per_dst=[1946, 10, 2606, 1318, 1378, 1704, 2762, 1538] combine_rows_per_src=[6814, 7894, 7234, 7652, 4212, 5552, 7286, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54082 payload_rows_remote=46536 dup_rows=11454 dup_rows_remote=8264 dedup_ratio=0.174774 router_tokens_per_expert=[0, 94, 0, 0, 2932, 0, 0, 49, 2615, 0, 0, 0, 1760, 0, 0, 4056, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2043, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 659, 0, 0, 0, 36, 3832, 69, 0, 0, 0, 0, 0, 0, 0, 2, 0, 53, 0, 465, 0, 2518, 0, 0, 0, 0, 727, 0, 0, 0, 14, 0, 0, 2, 0, 0, 0, 600, 0, 670, 2928, 3254, 0, 1, 1910, 2, 0, 2, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 8, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 0] payload_rows_per_dst=[7314, 8116, 7546, 7854, 5950, 5356, 5928, 6018] dup_rows_per_dst=[2356, 8, 3190, 1372, 1604, 1018, 306, 1600] combine_rows_per_src=[7314, 8116, 7546, 7854, 5950, 5356, 5928, 6018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53920 payload_rows_remote=46050 dup_rows=11616 dup_rows_remote=9518 dedup_ratio=0.177246 router_tokens_per_expert=[0, 95, 0, 0, 3072, 0, 0, 27, 2410, 0, 0, 0, 1324, 0, 0, 4032, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1098, 0, 0, 0, 13, 3841, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 490, 0, 2645, 0, 0, 0, 0, 1481, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 720, 0, 689, 3285, 2198, 0, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 14, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0] payload_rows_per_dst=[7192, 8074, 6326, 7870, 6722, 5290, 6632, 5814] dup_rows_per_dst=[1844, 30, 1844, 2098, 2656, 1086, 552, 1506] combine_rows_per_src=[7192, 8074, 6326, 7870, 6722, 5290, 6632, 5814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50746 payload_rows_remote=43034 dup_rows=14790 dup_rows_remote=8966 dedup_ratio=0.225677 router_tokens_per_expert=[0, 537, 0, 0, 484, 0, 0, 102, 3126, 0, 0, 0, 3491, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 0, 0, 0, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 1108, 0, 394, 0, 0, 0, 138, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 0, 1082, 785, 4057, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 15, 19, 0, 0, 0, 0, 0, 55, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1201, 0] payload_rows_per_dst=[7712, 7870, 8136, 5006, 6488, 7074, 1740, 6720] dup_rows_per_dst=[5824, 40, 892, 946, 2468, 2672, 10, 1938] combine_rows_per_src=[7712, 7870, 8136, 5006, 6488, 7074, 1740, 6720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48066 payload_rows_remote=42534 dup_rows=17470 dup_rows_remote=15770 dedup_ratio=0.266571 router_tokens_per_expert=[0, 334, 0, 0, 693, 0, 0, 167, 2390, 0, 0, 0, 2574, 0, 0, 3932, 3, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 13, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 23, 1192, 0, 0, 0, 0, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 2369, 0, 1006, 0, 0, 0, 66, 1294, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1561, 0, 1080, 2628, 3867, 0, 12, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 388, 5, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 529, 0] payload_rows_per_dst=[5608, 7864, 2260, 4468, 7434, 7702, 7198, 5532] dup_rows_per_dst=[542, 42, 188, 888, 4414, 5932, 3764, 1700] combine_rows_per_src=[5608, 7864, 2260, 4468, 7434, 7702, 7198, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55442 payload_rows_remote=47288 dup_rows=10094 dup_rows_remote=10078 dedup_ratio=0.154022 router_tokens_per_expert=[0, 5, 0, 0, 3807, 0, 0, 24, 2089, 0, 0, 0, 447, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 683, 0, 1, 0, 190, 2858, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 4042, 0, 0, 0, 0, 904, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 1136, 0, 602, 4030, 591, 0, 1, 755, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 0, 0, 0, 8, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0] payload_rows_per_dst=[7848, 8154, 6806, 7250, 8152, 4116, 8068, 5048] dup_rows_per_dst=[718, 16, 872, 2218, 2370, 872, 1978, 1050] combine_rows_per_src=[7848, 8154, 6806, 7250, 8152, 4116, 8068, 5048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191569920 dispatch_meta_bytes=222696 combine_payload_bytes=191569920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=191792616 ingress_bytes=229318656 link_bytes=229318656 token_bytes=383362536 weight_movement_bytes=37748736 total_nvlink_bytes=421111272 weight_to_token_ratio=0.098467 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.197218530e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191373312 dispatch_meta_bytes=233120 combine_payload_bytes=191373312 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=191606432 ingress_bytes=266870784 link_bytes=266870784 token_bytes=382979744 weight_movement_bytes=75497472 total_nvlink_bytes=458477216 weight_to_token_ratio=0.197132 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.557024544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=190611456 dispatch_meta_bytes=219200 combine_payload_bytes=190611456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190830656 ingress_bytes=209485824 link_bytes=209485824 token_bytes=381442112 weight_movement_bytes=18874368 total_nvlink_bytes=400316480 weight_to_token_ratio=0.049482 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.007190092e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=188620800 dispatch_meta_bytes=222272 combine_payload_bytes=188620800 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=188843072 ingress_bytes=226369536 link_bytes=226369536 token_bytes=377463872 weight_movement_bytes=37748736 total_nvlink_bytes=415212608 weight_to_token_ratio=0.100006 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.168961513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=176267264 dispatch_meta_bytes=208000 combine_payload_bytes=176267264 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=176475264 ingress_bytes=270639104 link_bytes=270639104 token_bytes=352742528 weight_movement_bytes=94371840 total_nvlink_bytes=447114368 weight_to_token_ratio=0.267537 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=15360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.593130732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174219264 dispatch_meta_bytes=233216 combine_payload_bytes=174219264 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=174452480 ingress_bytes=211968000 link_bytes=211968000 token_bytes=348671744 weight_movement_bytes=37748736 total_nvlink_bytes=386420480 weight_to_token_ratio=0.108264 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.030973081e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=193691648 dispatch_meta_bytes=229464 combine_payload_bytes=193691648 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=193921112 ingress_bytes=231440384 link_bytes=231440384 token_bytes=387612760 weight_movement_bytes=37748736 total_nvlink_bytes=425361496 weight_to_token_ratio=0.097388 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217547884e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38846 dup_rows=21088 dup_rows_remote=17848 dedup_ratio=0.321777 router_tokens_per_expert=[198, 1, 0, 0, 193, 194, 41, 6, 3, 2, 258, 330, 623, 392, 175, 394, 15, 184, 0, 668, 198, 353, 1, 0, 342, 93, 96, 176, 679, 3, 497, 7, 268, 418, 8, 103, 4, 30, 235, 216, 206, 701, 519, 359, 354, 7, 232, 474, 540, 409, 596, 0, 692, 510, 19, 19, 819, 144, 315, 742, 264, 264, 18, 166, 9, 346, 0, 19, 338, 1, 535, 453, 0, 986, 392, 1191, 72, 35, 11, 1147, 100, 6, 90, 128, 18, 200, 0, 0, 618, 306, 99, 200, 269, 347, 142, 152, 0, 91, 259, 62, 3, 298, 495, 54, 123, 22, 645, 338, 559, 289, 481, 1, 382, 550, 204, 115, 262, 0, 1246, 95, 885, 261, 504, 6, 205, 0, 350, 0] payload_rows_per_dst=[5300, 6170, 4542, 5596, 5654, 5492, 5602, 6092] dup_rows_per_dst=[1958, 2946, 2324, 2316, 3122, 2152, 3240, 3030] combine_rows_per_src=[5300, 6170, 4542, 5596, 5654, 5492, 5602, 6092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55144 payload_rows_remote=47016 dup_rows=10392 dup_rows_remote=10392 dedup_ratio=0.158569 router_tokens_per_expert=[0, 0, 0, 2277, 0, 0, 4038, 0, 0, 0, 148, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 811, 34, 0, 0, 2560, 0, 2, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3628, 0, 0, 1795, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 841, 370, 0, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 2499, 0, 0] payload_rows_per_dst=[6566, 6604, 6732, 8076, 8128, 7636, 6124, 5278] dup_rows_per_dst=[1308, 2290, 2010, 38, 0, 3212, 1370, 164] combine_rows_per_src=[6566, 6604, 6732, 8076, 8128, 7636, 6124, 5278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54872 payload_rows_remote=47874 dup_rows=10664 dup_rows_remote=9204 dedup_ratio=0.162720 router_tokens_per_expert=[0, 0, 0, 2473, 0, 0, 4094, 0, 0, 0, 211, 0, 0, 0, 0, 980, 0, 0, 0, 2, 0, 0, 1122, 132, 0, 0, 3199, 0, 1, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 925, 2, 0, 2, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 113, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 1739, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 646, 966, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 1914, 0, 0] payload_rows_per_dst=[6172, 7074, 6998, 8192, 8140, 7368, 6796, 4132] dup_rows_per_dst=[1444, 2246, 1460, 52, 88, 3074, 2024, 276] combine_rows_per_src=[6172, 7074, 6998, 8192, 8140, 7368, 6796, 4132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52504 payload_rows_remote=48234 dup_rows=13032 dup_rows_remote=12148 dedup_ratio=0.198853 router_tokens_per_expert=[0, 0, 0, 1062, 4, 0, 3967, 0, 0, 0, 281, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 1472, 18, 0, 0, 3504, 0, 4, 0, 0, 561, 1, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3698, 0, 0, 1509, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 882, 0, 0, 1, 0, 0, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 2563, 0, 0] payload_rows_per_dst=[3820, 7684, 6702, 7976, 8128, 4270, 6936, 6988] dup_rows_per_dst=[538, 3434, 2156, 278, 2, 884, 1900, 3840] combine_rows_per_src=[3820, 7684, 6702, 7976, 8128, 4270, 6936, 6988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56106 payload_rows_remote=47914 dup_rows=9430 dup_rows_remote=9324 dedup_ratio=0.143890 router_tokens_per_expert=[0, 0, 0, 2638, 0, 0, 4096, 0, 0, 0, 91, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 558, 160, 0, 0, 2758, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 0, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 3598, 0, 0, 1830, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 645, 836, 0, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 2896, 0, 0] payload_rows_per_dst=[6324, 6292, 6274, 8192, 8104, 7574, 7426, 5920] dup_rows_per_dst=[802, 1256, 1434, 106, 16, 3284, 2286, 246] combine_rows_per_src=[6324, 6292, 6274, 8192, 8104, 7574, 7426, 5920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51442 payload_rows_remote=43794 dup_rows=14094 dup_rows_remote=12180 dedup_ratio=0.215057 router_tokens_per_expert=[0, 0, 0, 90, 1, 0, 4027, 0, 0, 0, 151, 2, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 1962, 53, 0, 0, 3753, 0, 2, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 3239, 0, 0, 0, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 845, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 2360, 2898, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 950, 0, 0] payload_rows_per_dst=[7648, 6582, 7632, 7132, 8138, 3966, 7600, 2744] dup_rows_per_dst=[1914, 3724, 3930, 436, 320, 744, 2916, 110] combine_rows_per_src=[7648, 6582, 7632, 7132, 8138, 3966, 7600, 2744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50002 payload_rows_remote=41856 dup_rows=15534 dup_rows_remote=9574 dedup_ratio=0.237030 router_tokens_per_expert=[0, 0, 0, 285, 0, 0, 3995, 0, 0, 0, 50, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 1469, 73, 0, 0, 2919, 0, 0, 0, 0, 1638, 0, 0, 0, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 1840, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 1515, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 1, 0, 155, 0, 0, 0, 0, 2959, 0, 0] payload_rows_per_dst=[1024, 7670, 7164, 7990, 7790, 3680, 6538, 8146] dup_rows_per_dst=[44, 4528, 2440, 696, 0, 0, 1866, 5960] combine_rows_per_src=[1024, 7670, 7164, 7990, 7790, 3680, 6538, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49430 payload_rows_remote=48548 dup_rows=16106 dup_rows_remote=16086 dedup_ratio=0.245758 router_tokens_per_expert=[0, 0, 0, 3823, 0, 0, 4093, 0, 0, 0, 463, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 148, 90, 0, 0, 1920, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 369, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 2879, 0, 0, 0, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 171, 324, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 3, 0, 237, 0, 0, 0, 0, 3696, 0, 0] payload_rows_per_dst=[8162, 882, 6328, 8186, 7692, 8148, 2584, 7448] dup_rows_per_dst=[7216, 20, 932, 24, 242, 6852, 396, 424] combine_rows_per_src=[8162, 882, 6328, 8186, 7692, 8148, 2584, 7448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=226776 combine_payload_bytes=159113216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159339992 ingress_bytes=177987584 link_bytes=177987584 token_bytes=318453208 weight_movement_bytes=18874368 total_nvlink_bytes=337327576 weight_to_token_ratio=0.059269 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=5504 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.705389455e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=192577536 dispatch_meta_bytes=229632 combine_payload_bytes=192577536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=192807168 ingress_bytes=230326272 link_bytes=230326272 token_bytes=385384704 weight_movement_bytes=37748736 total_nvlink_bytes=423133440 weight_to_token_ratio=0.097951 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.206873011e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=196091904 dispatch_meta_bytes=228312 combine_payload_bytes=196091904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196320216 ingress_bytes=214966272 link_bytes=214966272 token_bytes=392412120 weight_movement_bytes=18874368 total_nvlink_bytes=411286488 weight_to_token_ratio=0.048098 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059701048e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=197566464 dispatch_meta_bytes=241528 combine_payload_bytes=197566464 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=197807992 ingress_bytes=291938304 link_bytes=291938304 token_bytes=395374456 weight_movement_bytes=94371840 total_nvlink_bytes=489746296 weight_to_token_ratio=0.238690 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17280 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.797209186e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=196255744 dispatch_meta_bytes=228952 combine_payload_bytes=196255744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196484696 ingress_bytes=215130112 link_bytes=215130112 token_bytes=392740440 weight_movement_bytes=18874368 total_nvlink_bytes=411614808 weight_to_token_ratio=0.048058 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.061270883e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=179380224 dispatch_meta_bytes=223896 combine_payload_bytes=179380224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=179604120 ingress_bytes=217128960 link_bytes=217128960 token_bytes=358984344 weight_movement_bytes=37748736 total_nvlink_bytes=396733080 weight_to_token_ratio=0.105154 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.080422861e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=171442176 dispatch_meta_bytes=205720 combine_payload_bytes=171442176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=171647896 ingress_bytes=190316544 link_bytes=190316544 token_bytes=343090072 weight_movement_bytes=18874368 total_nvlink_bytes=361964440 weight_to_token_ratio=0.055013 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.823519483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=198852608 dispatch_meta_bytes=258536 combine_payload_bytes=198852608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=199111144 ingress_bytes=236601344 link_bytes=236601344 token_bytes=397963752 weight_movement_bytes=37748736 total_nvlink_bytes=435712488 weight_to_token_ratio=0.094855 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.266997663e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58924 payload_rows_remote=50732 dup_rows=6612 dup_rows_remote=6612 dedup_ratio=0.100891 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 16, 0, 0, 0, 0, 0, 2241, 0, 0, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3569, 0, 0, 0, 3, 0, 0, 0, 0, 104, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 2786, 4096, 0, 9] payload_rows_per_dst=[8192, 8192, 6806, 6586, 8192, 7148, 8190, 5618] dup_rows_per_dst=[16, 1012, 2066, 92, 0, 204, 3152, 70] combine_rows_per_src=[8192, 8192, 6806, 6586, 8192, 7148, 8190, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48572 payload_rows_remote=40578 dup_rows=16964 dup_rows_remote=13658 dedup_ratio=0.258850 router_tokens_per_expert=[0, 37, 1, 0, 3713, 0, 2, 53, 2277, 0, 1, 0, 444, 1, 1, 4065, 0, 0, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 150, 499, 0, 0, 0, 0, 3324, 156, 2, 0, 1, 0, 0, 0, 0, 0, 0, 16, 0, 227, 0, 3928, 2, 0, 0, 2, 665, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1181, 0, 495, 3777, 1384, 0, 0, 1310, 0, 0, 0, 0, 1, 0, 1, 0, 517, 5, 0, 0, 1, 1, 0, 0, 2, 34, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 1395, 0] payload_rows_per_dst=[1324, 8134, 7036, 7134, 1582, 7998, 7994, 7370] dup_rows_per_dst=[62, 1150, 1906, 1130, 244, 3210, 3306, 5956] combine_rows_per_src=[1324, 8134, 7036, 7134, 1582, 7998, 7994, 7370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55998 payload_rows_remote=50404 dup_rows=9538 dup_rows_remote=8380 dedup_ratio=0.145538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 91, 0, 2, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 87, 0, 0, 0, 0, 0, 1195, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 25, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3023, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2824, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 2683, 4094, 0, 2] payload_rows_per_dst=[8192, 8192, 5594, 7184, 8192, 7566, 5648, 5430] dup_rows_per_dst=[188, 1128, 1158, 1058, 80, 5866, 0, 60] combine_rows_per_src=[8192, 8192, 5594, 7184, 8192, 7566, 5648, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55994 payload_rows_remote=49672 dup_rows=9542 dup_rows_remote=9292 dedup_ratio=0.145599 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2838, 26, 0, 0, 0, 0, 0, 1738, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 37, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 2666, 4096, 0, 29] payload_rows_per_dst=[8192, 8192, 7086, 6322, 8192, 7874, 4698, 5438] dup_rows_per_dst=[82, 792, 2118, 250, 100, 4610, 1438, 152] combine_rows_per_src=[8192, 8192, 7086, 6322, 8192, 7874, 4698, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57222 payload_rows_remote=50200 dup_rows=8314 dup_rows_remote=8142 dedup_ratio=0.126862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 19, 0, 0, 0, 0, 0, 1239, 0, 0, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3502, 0, 0, 0, 8, 0, 0, 0, 0, 87, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 2495, 4096, 0, 1] payload_rows_per_dst=[8192, 8192, 6130, 6218, 8192, 7022, 8192, 5084] dup_rows_per_dst=[8, 1478, 1336, 36, 28, 172, 5200, 56] combine_rows_per_src=[8192, 8192, 6130, 6218, 8192, 7022, 8192, 5084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45182 payload_rows_remote=36990 dup_rows=20354 dup_rows_remote=6000 dedup_ratio=0.310577 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 3, 0, 0, 0, 0, 0, 1522, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2000, 0, 0, 0, 6, 0, 0, 0, 0, 45, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 3676, 4095, 0, 4] payload_rows_per_dst=[5204, 3188, 3178, 4588, 7888, 5152, 7792, 8192] dup_rows_per_dst=[18, 0, 60, 74, 46, 1936, 3866, 14354] combine_rows_per_src=[5204, 3188, 3178, 4588, 7888, 5152, 7792, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=207798272 dispatch_meta_bytes=229376 combine_payload_bytes=207798272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208027648 ingress_bytes=226672640 link_bytes=226672640 token_bytes=415825920 weight_movement_bytes=18874368 total_nvlink_bytes=434700288 weight_to_token_ratio=0.045390 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.171865707e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53706 payload_rows_remote=45514 dup_rows=11830 dup_rows_remote=11822 dedup_ratio=0.180511 router_tokens_per_expert=[0, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 1, 0, 0, 0, 0, 0, 315, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1588, 0, 0, 0, 17, 0, 0, 0, 0, 93, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 3812, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 2204, 8190, 8190, 3294, 7808, 7636] dup_rows_per_dst=[8, 5708, 172, 5784, 0, 102, 0, 56] combine_rows_per_src=[8192, 8192, 2204, 8190, 8190, 3294, 7808, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53356 payload_rows_remote=45164 dup_rows=12180 dup_rows_remote=12100 dedup_ratio=0.185852 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 792, 0, 0, 0, 0, 0, 2177, 0, 0, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 2521, 4096, 0, 7] payload_rows_per_dst=[8192, 8192, 7922, 8192, 8192, 6682, 914, 5070] dup_rows_per_dst=[1976, 80, 5318, 4740, 52, 0, 0, 14] combine_rows_per_src=[8192, 8192, 7922, 8192, 8192, 6682, 914, 5070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=166207488 dispatch_meta_bytes=216944 combine_payload_bytes=166207488 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=166424432 ingress_bytes=203956224 link_bytes=203956224 token_bytes=332631920 weight_movement_bytes=37748736 total_nvlink_bytes=370380656 weight_to_token_ratio=0.113485 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.079861 gemm_active_groups=14 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.954208186e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=206454784 dispatch_meta_bytes=235136 combine_payload_bytes=206454784 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=206689920 ingress_bytes=225329152 link_bytes=225329152 token_bytes=413144704 weight_movement_bytes=18874368 total_nvlink_bytes=432019072 weight_to_token_ratio=0.045685 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.158993066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203456512 dispatch_meta_bytes=235856 combine_payload_bytes=203456512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203692368 ingress_bytes=222330880 link_bytes=222330880 token_bytes=407148880 weight_movement_bytes=18874368 total_nvlink_bytes=426023248 weight_to_token_ratio=0.046357 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.130265099e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205619200 dispatch_meta_bytes=233368 combine_payload_bytes=205619200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205852568 ingress_bytes=243367936 link_bytes=243367936 token_bytes=411471768 weight_movement_bytes=37748736 total_nvlink_bytes=449220504 weight_to_token_ratio=0.091741 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331831818e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=151511040 dispatch_meta_bytes=171960 combine_payload_bytes=151511040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=151683000 ingress_bytes=189259776 link_bytes=189259776 token_bytes=303194040 weight_movement_bytes=37748736 total_nvlink_bytes=340942776 weight_to_token_ratio=0.124504 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.813394052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=186425344 dispatch_meta_bytes=229344 combine_payload_bytes=186425344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186654688 ingress_bytes=205299712 link_bytes=205299712 token_bytes=373080032 weight_movement_bytes=18874368 total_nvlink_bytes=391954400 weight_to_token_ratio=0.050591 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.967080827e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=184991744 dispatch_meta_bytes=229056 combine_payload_bytes=184991744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185220800 ingress_bytes=203866112 link_bytes=203866112 token_bytes=370212544 weight_movement_bytes=18874368 total_nvlink_bytes=389086912 weight_to_token_ratio=0.050983 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.953344777e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57066 payload_rows_remote=48878 dup_rows=8470 dup_rows_remote=5674 dedup_ratio=0.129242 router_tokens_per_expert=[4089, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 31, 0, 0, 0, 0, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 18, 0, 0, 3978, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 4095, 0, 0, 0, 0, 0, 36, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1807, 4065, 0] payload_rows_per_dst=[7264, 8188, 6174, 7976, 8188, 6238, 8190, 4848] dup_rows_per_dst=[722, 50, 1596, 36, 2796, 2356, 0, 914] combine_rows_per_src=[7264, 8188, 6174, 7976, 8188, 6238, 8190, 4848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57142 payload_rows_remote=51242 dup_rows=8394 dup_rows_remote=7160 dedup_ratio=0.128082 router_tokens_per_expert=[4094, 0, 0, 0, 0, 2, 0, 0, 2746, 0, 0, 0, 527, 2, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1468, 0, 0, 0, 0, 0, 2098, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 4093, 0, 0, 0, 1, 0, 181, 0, 0, 0, 2136, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 3997, 0] payload_rows_per_dst=[6100, 8192, 5900, 7990, 8188, 8172, 8186, 4414] dup_rows_per_dst=[454, 6, 1234, 42, 2196, 4462, 0, 0] combine_rows_per_src=[6100, 8192, 5900, 7990, 8188, 8172, 8186, 4414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53448 payload_rows_remote=45366 dup_rows=12088 dup_rows_remote=11396 dedup_ratio=0.184448 router_tokens_per_expert=[0, 0, 0, 3618, 3, 2, 4070, 0, 0, 0, 225, 2, 0, 0, 2, 2425, 0, 0, 0, 0, 0, 0, 91, 5, 2, 0, 2231, 0, 0, 0, 0, 296, 0, 0, 0, 1, 0, 0, 0, 1394, 2, 0, 0, 0, 2, 273, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 290, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 2422, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 253, 151, 0, 0, 1, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 3207, 0, 0] payload_rows_per_dst=[8044, 4904, 3198, 8144, 8180, 4844, 8082, 8052] dup_rows_per_dst=[4510, 346, 146, 18, 568, 0, 692, 5808] combine_rows_per_src=[8044, 4904, 3198, 8144, 8180, 4844, 8082, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53068 payload_rows_remote=44882 dup_rows=12468 dup_rows_remote=3182 dedup_ratio=0.190247 router_tokens_per_expert=[4091, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 17, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 0, 0, 3989, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 4094, 0, 0, 0, 0, 0, 90, 0, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 4008, 0] payload_rows_per_dst=[7740, 8188, 3498, 7984, 6014, 3270, 8188, 8186] dup_rows_per_dst=[436, 182, 164, 10, 2258, 86, 46, 9286] combine_rows_per_src=[7740, 8188, 3498, 7984, 6014, 3270, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57462 payload_rows_remote=49494 dup_rows=8074 dup_rows_remote=8060 dedup_ratio=0.123199 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 13, 0, 0, 0, 0, 2016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 0, 0, 4, 0, 0, 3971, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 4084, 0, 0, 0, 0, 0, 460, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 4044, 0] payload_rows_per_dst=[6980, 8192, 6150, 7968, 8192, 8124, 8168, 3688] dup_rows_per_dst=[980, 24, 1702, 14, 2136, 3218, 0, 0] combine_rows_per_src=[6980, 8192, 6150, 7968, 8192, 8124, 8168, 3688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55412 payload_rows_remote=53288 dup_rows=10124 dup_rows_remote=10088 dedup_ratio=0.154480 router_tokens_per_expert=[4056, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 43, 0, 0, 0, 0, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 3984, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 1442, 0, 0, 4091, 0, 0, 0, 0, 0, 39, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3000, 4076, 0] payload_rows_per_dst=[6864, 8184, 5712, 7984, 8190, 2124, 8182, 8172] dup_rows_per_dst=[344, 78, 1046, 10, 2628, 36, 0, 5982] combine_rows_per_src=[6864, 8184, 5712, 7984, 8190, 2124, 8182, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50272 payload_rows_remote=42088 dup_rows=15264 dup_rows_remote=15258 dedup_ratio=0.232910 router_tokens_per_expert=[4091, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1154, 8, 0, 0, 0, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 7, 0, 0, 3497, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 361, 0, 0, 4096, 0, 0, 0, 1, 0, 2226, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 3744, 0] payload_rows_per_dst=[7484, 8184, 7538, 7208, 1472, 8014, 8192, 2180] dup_rows_per_dst=[2440, 6, 5486, 2948, 90, 4294, 0, 0] combine_rows_per_src=[7484, 8184, 7538, 7208, 1472, 8014, 8192, 2180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57196 payload_rows_remote=50192 dup_rows=8340 dup_rows_remote=7350 dedup_ratio=0.127258 router_tokens_per_expert=[4092, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 6, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4017, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 3921, 0] payload_rows_per_dst=[7004, 7784, 6164, 8034, 4680, 8190, 8192, 7148] dup_rows_per_dst=[990, 42, 1194, 8, 0, 6106, 0, 0] combine_rows_per_src=[7004, 7784, 6164, 8034, 4680, 8190, 8192, 7148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=200204288 dispatch_meta_bytes=218208 combine_payload_bytes=200204288 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=200422496 ingress_bytes=313450496 link_bytes=313450496 token_bytes=400626784 weight_movement_bytes=113246208 total_nvlink_bytes=513872992 weight_to_token_ratio=0.282673 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.003328425e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=209887232 dispatch_meta_bytes=233608 combine_payload_bytes=209887232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210120840 ingress_bytes=228761600 link_bytes=228761600 token_bytes=420008072 weight_movement_bytes=18874368 total_nvlink_bytes=438882440 weight_to_token_ratio=0.044938 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.191881093e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=183836672 dispatch_meta_bytes=192256 combine_payload_bytes=183836672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=184028928 ingress_bytes=221585408 link_bytes=221585408 token_bytes=367865600 weight_movement_bytes=37748736 total_nvlink_bytes=405614336 weight_to_token_ratio=0.102616 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.123122353e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=185819136 dispatch_meta_bytes=227048 combine_payload_bytes=185819136 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=186046184 ingress_bytes=223567872 link_bytes=223567872 token_bytes=371865320 weight_movement_bytes=37748736 total_nvlink_bytes=409614056 weight_to_token_ratio=0.101512 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142117347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=202727424 dispatch_meta_bytes=230216 combine_payload_bytes=202727424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202957640 ingress_bytes=221601792 link_bytes=221601792 token_bytes=405685064 weight_movement_bytes=18874368 total_nvlink_bytes=424559432 weight_to_token_ratio=0.046525 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.123279336e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=218267648 dispatch_meta_bytes=253504 combine_payload_bytes=218267648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218521152 ingress_bytes=237142016 link_bytes=237142016 token_bytes=436788800 weight_movement_bytes=18874368 total_nvlink_bytes=455663168 weight_to_token_ratio=0.043212 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272178116e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=172392448 dispatch_meta_bytes=229384 combine_payload_bytes=172392448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=172621832 ingress_bytes=191266816 link_bytes=191266816 token_bytes=345014280 weight_movement_bytes=18874368 total_nvlink_bytes=363888648 weight_to_token_ratio=0.054706 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.832624522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=205586432 dispatch_meta_bytes=230168 combine_payload_bytes=205586432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205816600 ingress_bytes=243335168 link_bytes=243335168 token_bytes=411403032 weight_movement_bytes=37748736 total_nvlink_bytes=449151768 weight_to_token_ratio=0.091756 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331517851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59942 payload_rows_remote=52106 dup_rows=5594 dup_rows_remote=5424 dedup_ratio=0.085358 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 3163, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2666, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0] payload_rows_per_dst=[6334, 6424, 7542, 8192, 7836, 8192, 7744, 7678] dup_rows_per_dst=[44, 0, 2292, 0, 170, 88, 0, 3000] combine_rows_per_src=[6334, 6424, 7542, 8192, 7836, 8192, 7744, 7678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60628 payload_rows_remote=52750 dup_rows=4908 dup_rows_remote=4900 dedup_ratio=0.074890 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3318, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2303, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 77, 0, 0, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 2, 0, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[6678, 8188, 7878, 8178, 7774, 8188, 6788, 6956] dup_rows_per_dst=[0, 4614, 8, 0, 188, 98, 0, 0] combine_rows_per_src=[6678, 8188, 7878, 8178, 7774, 8188, 6788, 6956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56682 payload_rows_remote=48534 dup_rows=8854 dup_rows_remote=726 dedup_ratio=0.135101 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 3717, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0] payload_rows_per_dst=[7434, 7132, 7764, 8192, 7758, 3828, 6426, 8148] dup_rows_per_dst=[110, 0, 40, 0, 576, 0, 0, 8128] combine_rows_per_src=[7434, 7132, 7764, 8192, 7758, 3828, 6426, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54594 payload_rows_remote=47532 dup_rows=10942 dup_rows_remote=7240 dedup_ratio=0.166962 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7, 0, 171, 0, 3318, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 50, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] payload_rows_per_dst=[7632, 7062, 7242, 8182, 8158, 8190, 8124, 4] dup_rows_per_dst=[3032, 3702, 12, 0, 150, 1674, 2372, 0] combine_rows_per_src=[7632, 7062, 7242, 8182, 8158, 8190, 8124, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55722 payload_rows_remote=47530 dup_rows=9814 dup_rows_remote=9368 dedup_ratio=0.149750 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3855, 778, 1, 0, 0, 1, 0, 2557, 0, 0, 3085, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 3930, 0, 0, 0, 2, 0, 0, 0, 0, 22, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 2, 81, 0, 0, 0, 0, 0, 0, 0, 1443, 4094, 0, 3] payload_rows_per_dst=[8192, 2954, 8118, 6198, 8192, 7864, 8192, 6012] dup_rows_per_dst=[692, 44, 6270, 24, 0, 44, 446, 2294] combine_rows_per_src=[8192, 2954, 8118, 6198, 8192, 7864, 8192, 6012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59286 payload_rows_remote=51094 dup_rows=6250 dup_rows_remote=6250 dedup_ratio=0.095367 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 3370, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0] payload_rows_per_dst=[6748, 6932, 7882, 8192, 7782, 8192, 7032, 6526] dup_rows_per_dst=[50, 2610, 3266, 0, 278, 46, 0, 0] combine_rows_per_src=[6748, 6932, 7882, 8192, 7782, 8192, 7032, 6526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59640 payload_rows_remote=51450 dup_rows=5896 dup_rows_remote=5894 dedup_ratio=0.089966 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 3342, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0] payload_rows_per_dst=[6684, 6590, 6930, 8192, 7836, 8190, 7026, 8192] dup_rows_per_dst=[138, 0, 2, 0, 410, 2, 0, 5344] combine_rows_per_src=[6684, 6590, 6930, 8192, 7836, 8190, 7026, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52806 payload_rows_remote=44616 dup_rows=12730 dup_rows_remote=5200 dedup_ratio=0.194244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3758, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1391, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0] payload_rows_per_dst=[8190, 5196, 7854, 8192, 7052, 8192, 8130, 0] dup_rows_per_dst=[7530, 0, 0, 24, 1366, 0, 3810, 0] combine_rows_per_src=[8190, 5196, 7854, 8192, 7052, 8192, 8130, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=216064000 dispatch_meta_bytes=230600 combine_payload_bytes=216064000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216294600 ingress_bytes=234938368 link_bytes=234938368 token_bytes=432358600 weight_movement_bytes=18874368 total_nvlink_bytes=451232968 weight_to_token_ratio=0.043654 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.251063845e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=213426176 dispatch_meta_bytes=230120 combine_payload_bytes=213426176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213656296 ingress_bytes=232300544 link_bytes=232300544 token_bytes=427082472 weight_movement_bytes=18874368 total_nvlink_bytes=445956840 weight_to_token_ratio=0.044194 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225789514e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=198795264 dispatch_meta_bytes=197040 combine_payload_bytes=198795264 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=198992304 ingress_bytes=312041472 link_bytes=312041472 token_bytes=397787568 weight_movement_bytes=113246208 total_nvlink_bytes=511033776 weight_to_token_ratio=0.284690 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.989827851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=194691072 dispatch_meta_bytes=219088 combine_payload_bytes=194691072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194910160 ingress_bytes=213565440 link_bytes=213565440 token_bytes=389601232 weight_movement_bytes=18874368 total_nvlink_bytes=408475600 weight_to_token_ratio=0.048445 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046278965e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=194682880 dispatch_meta_bytes=227592 combine_payload_bytes=194682880 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=194910472 ingress_bytes=289054720 link_bytes=289054720 token_bytes=389593352 weight_movement_bytes=94371840 total_nvlink_bytes=483965192 weight_to_token_ratio=0.242232 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769580103e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=210739200 dispatch_meta_bytes=229376 combine_payload_bytes=210739200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210968576 ingress_bytes=229613568 link_bytes=229613568 token_bytes=421707776 weight_movement_bytes=18874368 total_nvlink_bytes=440582144 weight_to_token_ratio=0.044757 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.200044232e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=209281024 dispatch_meta_bytes=229376 combine_payload_bytes=209281024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209510400 ingress_bytes=228155392 link_bytes=228155392 token_bytes=418791424 weight_movement_bytes=18874368 total_nvlink_bytes=437665792 weight_to_token_ratio=0.045069 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.186072707e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=182747136 dispatch_meta_bytes=199264 combine_payload_bytes=182747136 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=182946400 ingress_bytes=220495872 link_bytes=220495872 token_bytes=365693536 weight_movement_bytes=37748736 total_nvlink_bytes=403442272 weight_to_token_ratio=0.103225 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112682955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57442 payload_rows_remote=49254 dup_rows=8094 dup_rows_remote=8094 dedup_ratio=0.123505 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3993, 4094, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 2, 0, 0, 0, 2756, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2018, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 2, 4091, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3525, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 693] payload_rows_per_dst=[8188, 8160, 8188, 5530, 4036, 8188, 7750, 7402] dup_rows_per_dst=[8, 7020, 0, 22, 4, 0, 0, 1040] combine_rows_per_src=[8188, 8160, 8188, 5530, 4036, 8188, 7750, 7402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59518 payload_rows_remote=51642 dup_rows=6018 dup_rows_remote=2736 dedup_ratio=0.091827 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 4096, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2002, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 27, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1350] payload_rows_per_dst=[8192, 6488, 8192, 4978, 7876, 8160, 7910, 7722] dup_rows_per_dst=[0, 0, 0, 126, 3282, 0, 22, 2588] combine_rows_per_src=[8192, 6488, 8192, 4978, 7876, 8160, 7910, 7722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56862 payload_rows_remote=48964 dup_rows=8674 dup_rows_remote=7702 dedup_ratio=0.132355 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 4094, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4085, 0, 43, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 455] payload_rows_per_dst=[8188, 0, 8182, 8162, 8082, 8172, 8178, 7898] dup_rows_per_dst=[0, 0, 0, 5268, 2340, 10, 84, 972] combine_rows_per_src=[8188, 0, 8182, 8162, 8082, 8172, 8178, 7898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59246 payload_rows_remote=51266 dup_rows=6290 dup_rows_remote=2374 dedup_ratio=0.095978 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 4096, 2727, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3116, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4084, 0, 4, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 901] payload_rows_per_dst=[8192, 7980, 8190, 5918, 6232, 8168, 8158, 6408] dup_rows_per_dst=[0, 3916, 12, 20, 14, 174, 1494, 660] combine_rows_per_src=[8192, 7980, 8190, 5918, 6232, 8168, 8158, 6408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56218 payload_rows_remote=48026 dup_rows=9318 dup_rows_remote=9318 dedup_ratio=0.142181 router_tokens_per_expert=[4093, 0, 0, 0, 2, 0, 0, 0, 3608, 0, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3559, 4, 0, 0, 0, 0, 2420, 0, 1, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3660, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 4094, 0, 0, 0, 0, 0, 323, 0, 0, 0, 98, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 1396, 3958, 0] payload_rows_per_dst=[7718, 8190, 7804, 7322, 8186, 846, 8192, 7960] dup_rows_per_dst=[2016, 46, 4174, 2, 322, 0, 0, 2758] combine_rows_per_src=[7718, 8190, 7804, 7322, 8186, 846, 8192, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=201744384 dispatch_meta_bytes=229392 combine_payload_bytes=201744384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201973776 ingress_bytes=220618752 link_bytes=220618752 token_bytes=403718160 weight_movement_bytes=18874368 total_nvlink_bytes=422592528 weight_to_token_ratio=0.046751 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.113860330e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56176 payload_rows_remote=48036 dup_rows=9360 dup_rows_remote=9360 dedup_ratio=0.142822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 4096, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4068, 0, 23, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1456] payload_rows_per_dst=[8192, 0, 8192, 7998, 7766, 8140, 7926, 7962] dup_rows_per_dst=[0, 0, 0, 3892, 2532, 0, 32, 2904] combine_rows_per_src=[8192, 0, 8192, 7998, 7766, 8140, 7926, 7962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61386 payload_rows_remote=55628 dup_rows=4150 dup_rows_remote=4146 dedup_ratio=0.063324 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 4096, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 5, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 649] payload_rows_per_dst=[8192, 7576, 8192, 5758, 7928, 8152, 7946, 7642] dup_rows_per_dst=[4, 658, 0, 4, 2320, 0, 10, 1154] combine_rows_per_src=[8192, 7576, 8192, 5758, 7928, 8152, 7946, 7642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49958 payload_rows_remote=41766 dup_rows=15578 dup_rows_remote=15576 dedup_ratio=0.237701 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 8, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 213] payload_rows_per_dst=[8192, 8192, 8192, 6294, 2584, 8180, 7842, 482] dup_rows_per_dst=[2, 15542, 0, 6, 22, 0, 6, 0] combine_rows_per_src=[8192, 8192, 8192, 6294, 2584, 8180, 7842, 482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=211525632 dispatch_meta_bytes=217512 combine_payload_bytes=211525632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211743144 ingress_bytes=230400000 link_bytes=230400000 token_bytes=423268776 weight_movement_bytes=18874368 total_nvlink_bytes=442143144 weight_to_token_ratio=0.044592 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17920 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207579436e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200556544 dispatch_meta_bytes=226664 combine_payload_bytes=200556544 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=200783208 ingress_bytes=257179648 link_bytes=257179648 token_bytes=401339752 weight_movement_bytes=56623104 total_nvlink_bytes=457962856 weight_to_token_ratio=0.141085 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.464168847e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209985536 dispatch_meta_bytes=214560 combine_payload_bytes=209985536 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=210200096 ingress_bytes=304357376 link_bytes=304357376 token_bytes=420185632 weight_movement_bytes=94371840 total_nvlink_bytes=514557472 weight_to_token_ratio=0.224596 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=17920 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.916202624e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=196755456 dispatch_meta_bytes=229584 combine_payload_bytes=196755456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=196985040 ingress_bytes=234504192 link_bytes=234504192 token_bytes=393740496 weight_movement_bytes=37748736 total_nvlink_bytes=431489232 weight_to_token_ratio=0.095872 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.246903784e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=196714496 dispatch_meta_bytes=229376 combine_payload_bytes=196714496 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=196943872 ingress_bytes=234463232 link_bytes=234463232 token_bytes=393658368 weight_movement_bytes=37748736 total_nvlink_bytes=431407104 weight_to_token_ratio=0.095892 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.246511326e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=227852288 dispatch_meta_bytes=239096 combine_payload_bytes=227852288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228091384 ingress_bytes=246726656 link_bytes=246726656 token_bytes=455943672 weight_movement_bytes=18874368 total_nvlink_bytes=474818040 weight_to_token_ratio=0.041396 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364013421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=171073536 dispatch_meta_bytes=229368 combine_payload_bytes=171073536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=171302904 ingress_bytes=208822272 link_bytes=208822272 token_bytes=342376440 weight_movement_bytes=37748736 total_nvlink_bytes=380125176 weight_to_token_ratio=0.110255 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000832263e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55954 payload_rows_remote=48168 dup_rows=9582 dup_rows_remote=5156 dedup_ratio=0.146210 router_tokens_per_expert=[24, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 96, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4037, 968, 0, 0, 2211, 6, 0, 0, 0, 0, 0, 3317, 0, 0, 27, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2241, 2, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 2, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 1303, 0, 0, 0, 0, 0] payload_rows_per_dst=[8078, 8188, 7786, 6774, 8154, 8192, 5936, 2846] dup_rows_per_dst=[48, 194, 4426, 350, 58, 4494, 0, 12] combine_rows_per_src=[8078, 8188, 7786, 6774, 8154, 8192, 5936, 2846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56880 payload_rows_remote=48710 dup_rows=8656 dup_rows_remote=8388 dedup_ratio=0.132080 router_tokens_per_expert=[32, 0, 0, 0, 0, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4045, 1239, 0, 0, 1644, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 40, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 7, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 5290, 7674, 8170, 5446, 5748, 8192] dup_rows_per_dst=[88, 508, 476, 4202, 268, 0, 766, 2348] combine_rows_per_src=[8168, 8192, 5290, 7674, 8170, 5446, 5748, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=197296128 dispatch_meta_bytes=213296 combine_payload_bytes=197296128 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=197509424 ingress_bytes=272793600 link_bytes=272793600 token_bytes=394805552 weight_movement_bytes=75497472 total_nvlink_bytes=470303024 weight_to_token_ratio=0.191227 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.613774052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54384 payload_rows_remote=46192 dup_rows=11152 dup_rows_remote=8884 dedup_ratio=0.170166 router_tokens_per_expert=[42, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4057, 42, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 9, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 3, 0, 0, 0, 0, 865, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 5152, 7156, 2556, 8190, 7158, 7822, 8192] dup_rows_per_dst=[258, 626, 38, 18, 294, 2124, 5526, 2268] combine_rows_per_src=[8158, 5152, 7156, 2556, 8190, 7158, 7822, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53164 payload_rows_remote=44972 dup_rows=12372 dup_rows_remote=12196 dedup_ratio=0.188782 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 3681, 2316, 0, 0, 510, 6, 0, 0, 0, 0, 0, 3388, 0, 0, 496, 0, 0, 0, 0, 0, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 0] payload_rows_per_dst=[7378, 8192, 7958, 7046, 8116, 8192, 3716, 2566] dup_rows_per_dst=[4, 176, 4674, 1912, 2, 5602, 0, 2] combine_rows_per_src=[7378, 8192, 7958, 7046, 8116, 8192, 3716, 2566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56410 payload_rows_remote=52016 dup_rows=9126 dup_rows_remote=9126 dedup_ratio=0.139252 router_tokens_per_expert=[43, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4004, 645, 0, 0, 1985, 0, 0, 0, 0, 0, 0, 3440, 0, 0, 23, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 0, 0, 0, 29, 0, 0, 0, 0, 436, 0, 0, 0, 1, 0, 0, 692, 0, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8192, 4996, 6230, 8192, 4394, 8070, 8192] dup_rows_per_dst=[170, 586, 264, 44, 240, 0, 5564, 2258] combine_rows_per_src=[8144, 8192, 4996, 6230, 8192, 4394, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56748 payload_rows_remote=48986 dup_rows=8788 dup_rows_remote=4334 dedup_ratio=0.134094 router_tokens_per_expert=[6, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4012, 1713, 0, 0, 1795, 0, 0, 0, 0, 0, 0, 3168, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 1497, 0, 0, 0, 0, 0] payload_rows_per_dst=[8050, 8192, 6102, 7762, 8186, 5090, 5430, 7936] dup_rows_per_dst=[14, 84, 914, 4454, 24, 948, 0, 2350] combine_rows_per_src=[8050, 8192, 6102, 7762, 8186, 5090, 5430, 7936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59574 payload_rows_remote=51492 dup_rows=5962 dup_rows_remote=5960 dedup_ratio=0.090973 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 2459, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 0, 0, 2, 0, 0, 2, 0, 49, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 28, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 0, 0, 4039, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1706, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0] payload_rows_per_dst=[4920, 7060, 6798, 8192, 8144, 8190, 8082, 8188] dup_rows_per_dst=[12, 26, 26, 0, 58, 2466, 2, 3372] combine_rows_per_src=[4920, 7060, 6798, 8192, 8144, 8190, 8082, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=199516160 dispatch_meta_bytes=228392 combine_payload_bytes=199516160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199744552 ingress_bytes=218390528 link_bytes=218390528 token_bytes=399260712 weight_movement_bytes=18874368 total_nvlink_bytes=418135080 weight_to_token_ratio=0.047273 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.092510584e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59742 payload_rows_remote=51630 dup_rows=5794 dup_rows_remote=3914 dedup_ratio=0.088409 router_tokens_per_expert=[132, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3924, 13, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0] payload_rows_per_dst=[8112, 8192, 7680, 7556, 7334, 8192, 7316, 5360] dup_rows_per_dst=[1880, 514, 1220, 1830, 88, 0, 0, 262] combine_rows_per_src=[8112, 8192, 7680, 7556, 7334, 8192, 7316, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=189202432 dispatch_meta_bytes=220304 combine_payload_bytes=189202432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189422736 ingress_bytes=208076800 link_bytes=208076800 token_bytes=378625168 weight_movement_bytes=18874368 total_nvlink_bytes=397499536 weight_to_token_ratio=0.049850 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.993689517e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=184205312 dispatch_meta_bytes=228672 combine_payload_bytes=184205312 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=184433984 ingress_bytes=203079680 link_bytes=203079680 token_bytes=368639296 weight_movement_bytes=18874368 total_nvlink_bytes=387513664 weight_to_token_ratio=0.051200 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.945809572e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213057536 dispatch_meta_bytes=244568 combine_payload_bytes=213057536 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=213302104 ingress_bytes=269680640 link_bytes=269680640 token_bytes=426359640 weight_movement_bytes=56623104 total_nvlink_bytes=482982744 weight_to_token_ratio=0.132806 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.583947201e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=200646656 dispatch_meta_bytes=213280 combine_payload_bytes=200646656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=200859936 ingress_bytes=257269760 link_bytes=257269760 token_bytes=401506592 weight_movement_bytes=56623104 total_nvlink_bytes=458129696 weight_to_token_ratio=0.141027 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=14976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.465032256e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=210911232 dispatch_meta_bytes=229808 combine_payload_bytes=210911232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211141040 ingress_bytes=229785600 link_bytes=229785600 token_bytes=422052272 weight_movement_bytes=18874368 total_nvlink_bytes=440926640 weight_to_token_ratio=0.044720 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.201692558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57016 payload_rows_remote=53290 dup_rows=8520 dup_rows_remote=8488 dedup_ratio=0.130005 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 2, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2895, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 1, 0, 0, 0, 236, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 3726, 4324, 8192, 8192, 8010, 8192] dup_rows_per_dst=[2548, 5790, 32, 144, 2, 0, 2, 2] combine_rows_per_src=[8192, 8188, 3726, 4324, 8192, 8192, 8010, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=211476480 dispatch_meta_bytes=222176 combine_payload_bytes=211476480 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=211698656 ingress_bytes=249225216 link_bytes=249225216 token_bytes=423175136 weight_movement_bytes=37748736 total_nvlink_bytes=460923872 weight_to_token_ratio=0.089204 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.387953393e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58896 payload_rows_remote=50704 dup_rows=6640 dup_rows_remote=6640 dedup_ratio=0.101318 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 776, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2837, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5872, 7000, 5580, 8192, 8186, 7682, 8192] dup_rows_per_dst=[1556, 1126, 3790, 168, 0, 0, 0, 0] combine_rows_per_src=[8192, 5872, 7000, 5580, 8192, 8186, 7682, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218275840 dispatch_meta_bytes=247112 combine_payload_bytes=218275840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218522952 ingress_bytes=237150208 link_bytes=237150208 token_bytes=436798792 weight_movement_bytes=18874368 total_nvlink_bytes=455673160 weight_to_token_ratio=0.043211 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272256608e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=207683584 dispatch_meta_bytes=229376 combine_payload_bytes=207683584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207912960 ingress_bytes=226557952 link_bytes=226557952 token_bytes=415596544 weight_movement_bytes=18874368 total_nvlink_bytes=434470912 weight_to_token_ratio=0.045415 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.170766823e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54412 payload_rows_remote=46222 dup_rows=11124 dup_rows_remote=11108 dedup_ratio=0.169739 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 704, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3295, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1416, 6590, 5856, 8192, 8192, 8192, 7784, 8190] dup_rows_per_dst=[0, 0, 1340, 5288, 0, 258, 4222, 16] combine_rows_per_src=[1416, 6590, 5856, 8192, 8192, 8192, 7784, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59612 payload_rows_remote=51420 dup_rows=5924 dup_rows_remote=4078 dedup_ratio=0.090393 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1390, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1003, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 318, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7598, 2866, 8192, 8192, 8188, 8192] dup_rows_per_dst=[3046, 1846, 1022, 10, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7598, 2866, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58288 payload_rows_remote=54300 dup_rows=7248 dup_rows_remote=7180 dedup_ratio=0.110596 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2415, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5266, 3988, 8192, 8192, 8074, 8192] dup_rows_per_dst=[2342, 4830, 8, 68, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5266, 3988, 8192, 8192, 8074, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56750 payload_rows_remote=48656 dup_rows=8786 dup_rows_remote=8786 dedup_ratio=0.134064 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 649, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3236, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1848, 6472, 8192, 8088, 8192, 8094, 7688, 8176] dup_rows_per_dst=[320, 0, 4226, 4240, 0, 0, 0, 0] combine_rows_per_src=[1848, 6472, 8192, 8088, 8192, 8094, 7688, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56528 payload_rows_remote=48356 dup_rows=9008 dup_rows_remote=9002 dedup_ratio=0.137451 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2503, 4093, 3314, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3274, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 1, 4093, 0, 4, 0, 0, 4083, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 1, 2, 0, 2, 0, 1339] payload_rows_per_dst=[8186, 4, 8192, 7956, 7780, 8192, 8172, 8046] dup_rows_per_dst=[0, 0, 4, 3604, 2738, 0, 6, 2656] combine_rows_per_src=[8186, 4, 8192, 7956, 7780, 8192, 8172, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53696 payload_rows_remote=45504 dup_rows=11840 dup_rows_remote=426 dedup_ratio=0.180664 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3821, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 1244, 5862, 8186, 8068, 5760, 8192] dup_rows_per_dst=[11414, 6, 0, 420, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 1244, 5862, 8186, 8068, 5760, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210616320 dispatch_meta_bytes=221992 combine_payload_bytes=210616320 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=210838312 ingress_bytes=323862528 link_bytes=323862528 token_bytes=421454632 weight_movement_bytes=113246208 total_nvlink_bytes=534700840 weight_to_token_ratio=0.268703 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.103091393e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=189325312 dispatch_meta_bytes=229320 combine_payload_bytes=189325312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189554632 ingress_bytes=208199680 link_bytes=208199680 token_bytes=378879944 weight_movement_bytes=18874368 total_nvlink_bytes=397754312 weight_to_token_ratio=0.049816 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.994866893e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=222412800 dispatch_meta_bytes=245920 combine_payload_bytes=222412800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222658720 ingress_bytes=241287168 link_bytes=241287168 token_bytes=445071520 weight_movement_bytes=18874368 total_nvlink_bytes=463945888 weight_to_token_ratio=0.042407 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.311894923e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57640 payload_rows_remote=49452 dup_rows=7896 dup_rows_remote=7896 dedup_ratio=0.120483 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 3408, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4095, 0, 0, 3361, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7854, 8188, 8192, 8192, 8192, 7240, 1590] dup_rows_per_dst=[4, 4112, 0, 932, 0, 2848, 0, 0] combine_rows_per_src=[8192, 7854, 8188, 8192, 8192, 8192, 7240, 1590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=199294976 dispatch_meta_bytes=229768 combine_payload_bytes=199294976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199524744 ingress_bytes=218169344 link_bytes=218169344 token_bytes=398819720 weight_movement_bytes=18874368 total_nvlink_bytes=417694088 weight_to_token_ratio=0.047326 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.090391308e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=198066176 dispatch_meta_bytes=229432 combine_payload_bytes=198066176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198295608 ingress_bytes=216940544 link_bytes=216940544 token_bytes=396361784 weight_movement_bytes=18874368 total_nvlink_bytes=415236152 weight_to_token_ratio=0.047619 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.078617551e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=186384384 dispatch_meta_bytes=183720 combine_payload_bytes=186384384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=186568104 ingress_bytes=224133120 link_bytes=224133120 token_bytes=372952488 weight_movement_bytes=37748736 total_nvlink_bytes=410701224 weight_to_token_ratio=0.101216 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.147533275e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57630 payload_rows_remote=49438 dup_rows=7906 dup_rows_remote=7884 dedup_ratio=0.120636 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 3152, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5336, 8192, 8192, 8192, 4896, 7302, 7328] dup_rows_per_dst=[0, 2, 204, 1608, 22, 1010, 0, 5060] combine_rows_per_src=[8192, 5336, 8192, 8192, 8192, 4896, 7302, 7328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202555392 dispatch_meta_bytes=229392 combine_payload_bytes=202555392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=202784784 ingress_bytes=221429760 link_bytes=221429760 token_bytes=405340176 weight_movement_bytes=18874368 total_nvlink_bytes=424214544 weight_to_token_ratio=0.046564 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.121631010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202498048 dispatch_meta_bytes=229288 combine_payload_bytes=202498048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202727336 ingress_bytes=221372416 link_bytes=221372416 token_bytes=405225384 weight_movement_bytes=18874368 total_nvlink_bytes=424099752 weight_to_token_ratio=0.046577 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.121081568e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55906 payload_rows_remote=47726 dup_rows=9630 dup_rows_remote=5920 dedup_ratio=0.146942 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3719, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 65, 0, 0, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 2212, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8186, 6770, 2] dup_rows_per_dst=[4, 3710, 4, 266, 4, 5642, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8186, 6770, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52700 payload_rows_remote=44564 dup_rows=12836 dup_rows_remote=4710 dedup_ratio=0.195862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 4094, 0, 0, 2144, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8172, 6748, 6778, 2328, 8192, 4168, 8178, 8136] dup_rows_per_dst=[4, 0, 166, 64, 120, 0, 4356, 8126] combine_rows_per_src=[8172, 6748, 6778, 2328, 8192, 4168, 8178, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61306 payload_rows_remote=53114 dup_rows=4230 dup_rows_remote=3512 dedup_ratio=0.064545 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 2784, 0, 0, 0, 10, 0, 0] payload_rows_per_dst=[8192, 7378, 8192, 8192, 8192, 8180, 7392, 5588] dup_rows_per_dst=[0, 32, 4, 718, 0, 3476, 0, 0] combine_rows_per_src=[8192, 7378, 8192, 8192, 8192, 8180, 7392, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59000 payload_rows_remote=50842 dup_rows=6536 dup_rows_remote=5052 dedup_ratio=0.099731 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 177, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3485, 3231, 2, 0, 45, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 43, 0, 0, 0, 0, 0, 0, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2, 0, 0, 2843, 0, 0, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 1, 0, 0, 0, 0, 27, 2, 0, 0, 1, 0, 0, 2252, 0, 0, 0, 0, 0] payload_rows_per_dst=[6986, 8192, 6540, 7056, 8186, 5690, 8158, 8192] dup_rows_per_dst=[0, 356, 16, 72, 34, 14, 1484, 4560] combine_rows_per_src=[6986, 8192, 6540, 7056, 8186, 5690, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56768 payload_rows_remote=48576 dup_rows=8768 dup_rows_remote=8766 dedup_ratio=0.133789 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3513, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 1871, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7932, 8192, 8192, 7922, 8190, 8148, 0] dup_rows_per_dst=[2, 3128, 630, 3340, 0, 1668, 0, 0] combine_rows_per_src=[8192, 7932, 8192, 8192, 7922, 8190, 8148, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57820 payload_rows_remote=53278 dup_rows=7716 dup_rows_remote=7716 dedup_ratio=0.117737 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2421, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 2821, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 4846, 8192, 8192, 8192, 4542, 7538, 8126] dup_rows_per_dst=[10, 0, 390, 1558, 116, 0, 0, 5642] combine_rows_per_src=[8192, 4846, 8192, 8192, 8192, 4542, 7538, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=195485696 dispatch_meta_bytes=214584 combine_payload_bytes=195485696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=195700280 ingress_bytes=233234432 link_bytes=233234432 token_bytes=391185976 weight_movement_bytes=37748736 total_nvlink_bytes=428934712 weight_to_token_ratio=0.096498 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.234737569e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=182534144 dispatch_meta_bytes=197096 combine_payload_bytes=182534144 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=182731240 ingress_bytes=258031616 link_bytes=258031616 token_bytes=365265384 weight_movement_bytes=75497472 total_nvlink_bytes=440762856 weight_to_token_ratio=0.206692 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=15616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.472331985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60078 payload_rows_remote=51890 dup_rows=5458 dup_rows_remote=5458 dedup_ratio=0.083282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 0, 0, 0, 0, 178, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6928, 8188, 8188, 8192, 4186, 8012, 8192] dup_rows_per_dst=[2978, 42, 0, 222, 2172, 0, 44, 0] combine_rows_per_src=[8192, 6928, 8188, 8188, 8192, 4186, 8012, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=217554944 dispatch_meta_bytes=226504 combine_payload_bytes=217554944 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=217781448 ingress_bytes=274178048 link_bytes=274178048 token_bytes=435336392 weight_movement_bytes=56623104 total_nvlink_bytes=491959496 weight_to_token_ratio=0.130067 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.627039152e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=208248832 dispatch_meta_bytes=223576 combine_payload_bytes=208248832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208472408 ingress_bytes=227123200 link_bytes=227123200 token_bytes=416721240 weight_movement_bytes=18874368 total_nvlink_bytes=435595608 weight_to_token_ratio=0.045293 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=21120 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.176182751e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=198967296 dispatch_meta_bytes=229368 combine_payload_bytes=198967296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199196664 ingress_bytes=217841664 link_bytes=217841664 token_bytes=398163960 weight_movement_bytes=18874368 total_nvlink_bytes=417038328 weight_to_token_ratio=0.047404 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.087251640e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60648 payload_rows_remote=52456 dup_rows=4888 dup_rows_remote=3350 dedup_ratio=0.074585 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 0, 0, 0, 0, 0, 87, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5038, 7260, 8046, 8192, 8192, 8036, 7692, 8192] dup_rows_per_dst=[178, 32, 0, 24, 1538, 3104, 12, 0] combine_rows_per_src=[5038, 7260, 8046, 8192, 8192, 8036, 7692, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=218226688 dispatch_meta_bytes=243976 combine_payload_bytes=218226688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218470664 ingress_bytes=237101056 link_bytes=237101056 token_bytes=436697352 weight_movement_bytes=18874368 total_nvlink_bytes=455571720 weight_to_token_ratio=0.043221 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.271785658e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=212541440 dispatch_meta_bytes=229392 combine_payload_bytes=212541440 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=212770832 ingress_bytes=288038912 link_bytes=288038912 token_bytes=425312272 weight_movement_bytes=75497472 total_nvlink_bytes=500809744 weight_to_token_ratio=0.177511 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.759847131e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=214859776 dispatch_meta_bytes=223224 combine_payload_bytes=214859776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215083000 ingress_bytes=233734144 link_bytes=233734144 token_bytes=429942776 weight_movement_bytes=18874368 total_nvlink_bytes=448817144 weight_to_token_ratio=0.043900 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239525563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57768 payload_rows_remote=50106 dup_rows=7768 dup_rows_remote=3772 dedup_ratio=0.118530 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 303, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7662, 8192, 8080, 2720, 6544, 8186, 8192] dup_rows_per_dst=[3460, 3996, 188, 24, 88, 0, 12, 0] combine_rows_per_src=[8192, 7662, 8192, 8080, 2720, 6544, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60354 payload_rows_remote=52162 dup_rows=5182 dup_rows_remote=5182 dedup_ratio=0.079071 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 149, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5652, 6660, 8192, 7744, 8192, 8192, 7530, 8192] dup_rows_per_dst=[0, 4, 0, 134, 1152, 3476, 416, 0] combine_rows_per_src=[5652, 6660, 8192, 7744, 8192, 8192, 7530, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60468 payload_rows_remote=52276 dup_rows=5068 dup_rows_remote=4976 dedup_ratio=0.077332 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 0, 0, 0, 214, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6974, 8152, 8192, 8192, 4524, 8050, 8192] dup_rows_per_dst=[3938, 28, 0, 92, 1000, 0, 10, 0] combine_rows_per_src=[8192, 6974, 8152, 8192, 8192, 4524, 8050, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49378 payload_rows_remote=41252 dup_rows=16158 dup_rows_remote=16158 dedup_ratio=0.246552 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 2, 25, 0, 13, 0, 4094, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 120, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 0, 0, 2, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 2, 0, 0, 0, 0, 116, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[84, 240, 8192, 8192, 8192, 8162, 8126, 8190] dup_rows_per_dst=[2, 0, 8044, 8106, 4, 0, 0, 2] combine_rows_per_src=[84, 240, 8192, 8192, 8192, 8162, 8126, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55306 payload_rows_remote=47114 dup_rows=10230 dup_rows_remote=6368 dedup_ratio=0.156097 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 0, 0, 0, 0, 0, 104, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2895, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4690, 3852, 5812, 8184] dup_rows_per_dst=[3862, 6344, 8, 6, 0, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 4690, 3852, 5812, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=205234176 dispatch_meta_bytes=215512 combine_payload_bytes=205234176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205449688 ingress_bytes=224108544 link_bytes=224108544 token_bytes=410683864 weight_movement_bytes=18874368 total_nvlink_bytes=429558232 weight_to_token_ratio=0.045958 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.147297800e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59506 payload_rows_remote=57208 dup_rows=6030 dup_rows_remote=6030 dedup_ratio=0.092010 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2923, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 4096, 0, 0, 0, 0, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 497, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8122, 8188, 2298, 8192, 8192, 8186, 8136, 8192] dup_rows_per_dst=[5012, 0, 0, 24, 990, 0, 0, 4] combine_rows_per_src=[8122, 8188, 2298, 8192, 8192, 8186, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=213655552 dispatch_meta_bytes=229376 combine_payload_bytes=213655552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213884928 ingress_bytes=232529920 link_bytes=232529920 token_bytes=427540480 weight_movement_bytes=18874368 total_nvlink_bytes=446414848 weight_to_token_ratio=0.044146 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227987282e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60314 payload_rows_remote=52122 dup_rows=5222 dup_rows_remote=3044 dedup_ratio=0.079681 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 121, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3787, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4664, 7112, 8190, 8192, 8192, 8192, 7580, 8192] dup_rows_per_dst=[0, 138, 0, 66, 2762, 2178, 78, 0] combine_rows_per_src=[4664, 7112, 8190, 8192, 8192, 8192, 7580, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=214122496 dispatch_meta_bytes=229008 combine_payload_bytes=214122496 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=214351504 ingress_bytes=289619968 link_bytes=289619968 token_bytes=428474000 weight_movement_bytes=75497472 total_nvlink_bytes=503971472 weight_to_token_ratio=0.176201 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.774996031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62368 payload_rows_remote=54180 dup_rows=3168 dup_rows_remote=2440 dedup_ratio=0.048340 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 4096, 0, 0, 0, 0, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5322, 8192, 7902, 8192, 8188, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 2438, 0, 728, 0, 2, 0] combine_rows_per_src=[5322, 8192, 7902, 8192, 8188, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=168968192 dispatch_meta_bytes=229640 combine_payload_bytes=168968192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=169197832 ingress_bytes=187842560 link_bytes=187842560 token_bytes=338166024 weight_movement_bytes=18874368 total_nvlink_bytes=357040392 weight_to_token_ratio=0.055814 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.799814986e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=192978944 dispatch_meta_bytes=213928 combine_payload_bytes=192978944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193192872 ingress_bytes=211853312 link_bytes=211853312 token_bytes=386171816 weight_movement_bytes=18874368 total_nvlink_bytes=405046184 weight_to_token_ratio=0.048876 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.029874197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234323968 dispatch_meta_bytes=252952 combine_payload_bytes=234323968 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234576920 ingress_bytes=290947072 link_bytes=290947072 token_bytes=468900888 weight_movement_bytes=56623104 total_nvlink_bytes=525523992 weight_to_token_ratio=0.120757 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.787711689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=213491712 dispatch_meta_bytes=220664 combine_payload_bytes=213491712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213712376 ingress_bytes=232366080 link_bytes=232366080 token_bytes=427204088 weight_movement_bytes=18874368 total_nvlink_bytes=446078456 weight_to_token_ratio=0.044181 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226417447e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=221921280 dispatch_meta_bytes=226480 combine_payload_bytes=221921280 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=222147760 ingress_bytes=316293120 link_bytes=316293120 token_bytes=444069040 weight_movement_bytes=94371840 total_nvlink_bytes=538440880 weight_to_token_ratio=0.212516 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.030565050e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62952 payload_rows_remote=54760 dup_rows=2584 dup_rows_remote=2584 dedup_ratio=0.039429 router_tokens_per_expert=[8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1910, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 4096, 0, 0, 0, 0, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 567, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7708, 8192, 6394, 8192, 8058, 8184, 8032, 8192] dup_rows_per_dst=[1476, 0, 0, 124, 970, 0, 14, 0] combine_rows_per_src=[7708, 8192, 6394, 8192, 8058, 8184, 8032, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62778 payload_rows_remote=54586 dup_rows=2758 dup_rows_remote=2758 dedup_ratio=0.042084 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 4096, 0, 0, 0, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 124, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5752, 8192, 8030, 8048, 8182, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 2364, 0, 332, 42, 20, 0] combine_rows_per_src=[5752, 8192, 8030, 8048, 8182, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62032 payload_rows_remote=53840 dup_rows=3504 dup_rows_remote=3504 dedup_ratio=0.053467 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 4096, 0, 0, 0, 0, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7674, 8192, 5222, 8192, 8188, 8180, 8192, 8192] dup_rows_per_dst=[2184, 0, 288, 0, 1002, 0, 30, 0] combine_rows_per_src=[7674, 8192, 5222, 8192, 8188, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55736 payload_rows_remote=49114 dup_rows=9800 dup_rows_remote=7810 dedup_ratio=0.149536 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3134, 0, 0, 3, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4094, 10, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 26, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2267, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4094, 0, 0, 3861, 0, 2, 0, 15, 0, 0] payload_rows_per_dst=[8188, 6278, 8192, 5536, 8192, 4538, 6622, 8190] dup_rows_per_dst=[0, 0, 20, 34, 8, 0, 1990, 7748] combine_rows_per_src=[8188, 6278, 8192, 5536, 8192, 4538, 6622, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224296960 dispatch_meta_bytes=229376 combine_payload_bytes=224296960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224526336 ingress_bytes=243171328 link_bytes=243171328 token_bytes=448823296 weight_movement_bytes=18874368 total_nvlink_bytes=467697664 weight_to_token_ratio=0.042053 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329948017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=223584256 dispatch_meta_bytes=229376 combine_payload_bytes=223584256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223813632 ingress_bytes=242458624 link_bytes=242458624 token_bytes=447397888 weight_movement_bytes=18874368 total_nvlink_bytes=466272256 weight_to_token_ratio=0.042187 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.323119238e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59004 payload_rows_remote=50846 dup_rows=6532 dup_rows_remote=3702 dedup_ratio=0.099670 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3620, 4095, 0, 0, 0, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8192, 6810, 8192, 3080, 8192, 8192, 8188] dup_rows_per_dst=[2830, 2, 3556, 0, 144, 0, 0, 0] combine_rows_per_src=[8158, 8192, 6810, 8192, 3080, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60000 payload_rows_remote=51808 dup_rows=5536 dup_rows_remote=2952 dedup_ratio=0.084473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1218, 0, 0, 8, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 2787, 0, 28, 0, 0, 0] payload_rows_per_dst=[8190, 7700, 8192, 8190, 8092, 8192, 5410, 6034] dup_rows_per_dst=[2648, 10, 2584, 0, 0, 4, 0, 290] combine_rows_per_src=[8190, 7700, 8192, 8190, 8092, 8192, 5410, 6034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57848 payload_rows_remote=49660 dup_rows=7688 dup_rows_remote=6166 dedup_ratio=0.117310 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 787, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 8, 0, 2, 0] payload_rows_per_dst=[6816, 8022, 6752, 8192, 8188, 8192, 6692, 4994] dup_rows_per_dst=[1628, 24, 2268, 0, 1522, 4, 2044, 198] combine_rows_per_src=[6816, 8022, 6752, 8192, 8188, 8192, 6692, 4994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=220528640 dispatch_meta_bytes=229376 combine_payload_bytes=220528640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220758016 ingress_bytes=239403008 link_bytes=239403008 token_bytes=441286656 weight_movement_bytes=18874368 total_nvlink_bytes=460161024 weight_to_token_ratio=0.042771 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293841829e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63510 payload_rows_remote=55328 dup_rows=2026 dup_rows_remote=2026 dedup_ratio=0.030914 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 4096, 0, 0, 0, 0, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 91, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6296, 8192, 8086, 8192, 8178, 8182, 8192, 8192] dup_rows_per_dst=[0, 0, 1834, 0, 192, 0, 0, 0] combine_rows_per_src=[6296, 8192, 8086, 8192, 8178, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=201170944 dispatch_meta_bytes=227696 combine_payload_bytes=201170944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201398640 ingress_bytes=220045312 link_bytes=220045312 token_bytes=402569584 weight_movement_bytes=18874368 total_nvlink_bytes=421443952 weight_to_token_ratio=0.046885 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.108365911e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=208265216 dispatch_meta_bytes=218192 combine_payload_bytes=208265216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208483408 ingress_bytes=227139584 link_bytes=227139584 token_bytes=416748624 weight_movement_bytes=18874368 total_nvlink_bytes=435622992 weight_to_token_ratio=0.045290 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.176339734e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=212205568 dispatch_meta_bytes=219040 combine_payload_bytes=212205568 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=212424608 ingress_bytes=306577408 link_bytes=306577408 token_bytes=424630176 weight_movement_bytes=94371840 total_nvlink_bytes=519002016 weight_to_token_ratio=0.222245 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.937473878e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=203407360 dispatch_meta_bytes=223304 combine_payload_bytes=203407360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203630664 ingress_bytes=222281728 link_bytes=222281728 token_bytes=407038024 weight_movement_bytes=18874368 total_nvlink_bytes=425912392 weight_to_token_ratio=0.046370 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.129794148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=226623488 dispatch_meta_bytes=229416 combine_payload_bytes=226623488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226852904 ingress_bytes=245497856 link_bytes=245497856 token_bytes=453476392 weight_movement_bytes=18874368 total_nvlink_bytes=472350760 weight_to_token_ratio=0.041622 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352239664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56484 payload_rows_remote=48444 dup_rows=9052 dup_rows_remote=2766 dedup_ratio=0.138123 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 2679, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 0, 4, 0, 5, 0] payload_rows_per_dst=[1604, 8184, 5888, 8192, 8192, 8192, 8192, 8040] dup_rows_per_dst=[2, 4, 730, 0, 0, 2, 2028, 6286] combine_rows_per_src=[1604, 8184, 5888, 8192, 8192, 8192, 8192, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57646 payload_rows_remote=49516 dup_rows=7890 dup_rows_remote=7870 dedup_ratio=0.120392 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 487, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 584, 0, 0, 0] payload_rows_per_dst=[8180, 8130, 8192, 8190, 7284, 8192, 2132, 7346] dup_rows_per_dst=[3810, 20, 1050, 0, 0, 14, 0, 2996] combine_rows_per_src=[8180, 8130, 8192, 8190, 7284, 8192, 2132, 7346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60624 payload_rows_remote=52436 dup_rows=4912 dup_rows_remote=4912 dedup_ratio=0.074951 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 916, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1329, 0, 0, 4, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 3, 0, 0, 0] payload_rows_per_dst=[8186, 8136, 8192, 8188, 8114, 8192, 6382, 5234] dup_rows_per_dst=[1888, 24, 2730, 0, 0, 54, 0, 216] combine_rows_per_src=[8186, 8136, 8192, 8188, 8114, 8192, 6382, 5234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56284 payload_rows_remote=48108 dup_rows=9252 dup_rows_remote=5336 dedup_ratio=0.141174 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 1495, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 0, 0, 0, 22, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3871, 0, 0, 2174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[2994, 4162, 8188, 8188, 8192, 8192, 8176, 8192] dup_rows_per_dst=[4, 2, 0, 0, 262, 5060, 3916, 8] combine_rows_per_src=[2994, 4162, 8188, 8188, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=198426624 dispatch_meta_bytes=204840 combine_payload_bytes=198426624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198631464 ingress_bytes=217300992 link_bytes=217300992 token_bytes=397058088 weight_movement_bytes=18874368 total_nvlink_bytes=415932456 weight_to_token_ratio=0.047536 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.082071187e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=202817536 dispatch_meta_bytes=229544 combine_payload_bytes=202817536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203047080 ingress_bytes=221691904 link_bytes=221691904 token_bytes=405864616 weight_movement_bytes=18874368 total_nvlink_bytes=424738984 weight_to_token_ratio=0.046504 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.124142745e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54396 payload_rows_remote=46204 dup_rows=11140 dup_rows_remote=9390 dedup_ratio=0.169983 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 2969, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8144, 8192, 8192, 8190, 8098, 3768, 1620] dup_rows_per_dst=[1750, 2012, 7366, 12, 0, 0, 0, 0] combine_rows_per_src=[8192, 8144, 8192, 8192, 8190, 8098, 3768, 1620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64646 payload_rows_remote=56458 dup_rows=890 dup_rows_remote=128 dedup_ratio=0.013580 router_tokens_per_expert=[0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 380, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 0, 0, 0, 0, 166] payload_rows_per_dst=[8190, 8076, 8188, 8192, 8188, 8192, 8038, 7582] dup_rows_per_dst=[0, 0, 762, 8, 2, 2, 0, 116] combine_rows_per_src=[8190, 8076, 8188, 8192, 8188, 8192, 8038, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64354 payload_rows_remote=56162 dup_rows=1182 dup_rows_remote=1176 dedup_ratio=0.018036 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3244, 0, 0, 0, 0, 0, 429] payload_rows_per_dst=[8192, 7990, 8184, 8192, 8192, 8192, 8120, 7292] dup_rows_per_dst=[0, 0, 1078, 14, 6, 30, 0, 54] combine_rows_per_src=[8192, 7990, 8184, 8192, 8192, 8192, 8120, 7292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52624 payload_rows_remote=44432 dup_rows=12912 dup_rows_remote=12900 dedup_ratio=0.197021 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4010, 1, 823, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 0, 0, 0, 1, 0] payload_rows_per_dst=[4616, 8040, 2242, 8192, 8190, 8192, 8192, 4960] dup_rows_per_dst=[36, 6, 504, 0, 252, 12, 11476, 626] combine_rows_per_src=[4616, 8040, 2242, 8192, 8190, 8192, 8192, 4960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=214777856 dispatch_meta_bytes=229392 combine_payload_bytes=214777856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215007248 ingress_bytes=233652224 link_bytes=233652224 token_bytes=429785104 weight_movement_bytes=18874368 total_nvlink_bytes=448659472 weight_to_token_ratio=0.043916 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238740646e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=197050368 dispatch_meta_bytes=213776 combine_payload_bytes=197050368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197264144 ingress_bytes=215924736 link_bytes=215924736 token_bytes=394314512 weight_movement_bytes=18874368 total_nvlink_bytes=413188880 weight_to_token_ratio=0.047866 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.068884579e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=189251584 dispatch_meta_bytes=222376 combine_payload_bytes=189251584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189473960 ingress_bytes=208125952 link_bytes=208125952 token_bytes=378725544 weight_movement_bytes=18874368 total_nvlink_bytes=397599912 weight_to_token_ratio=0.049837 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.994160468e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=231251968 dispatch_meta_bytes=226344 combine_payload_bytes=231251968 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231478312 ingress_bytes=306749440 link_bytes=306749440 token_bytes=462730280 weight_movement_bytes=75497472 total_nvlink_bytes=538227752 weight_to_token_ratio=0.163157 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.939122204e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230039552 dispatch_meta_bytes=229352 combine_payload_bytes=230039552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230268904 ingress_bytes=248913920 link_bytes=248913920 token_bytes=460308456 weight_movement_bytes=18874368 total_nvlink_bytes=479182824 weight_to_token_ratio=0.041004 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384970708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59214 payload_rows_remote=51048 dup_rows=6322 dup_rows_remote=590 dedup_ratio=0.096466 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 0, 0, 0, 0, 0, 42] payload_rows_per_dst=[7834, 7200, 3418, 8192, 8192, 8168, 8044, 8166] dup_rows_per_dst=[0, 0, 136, 12, 2, 28, 412, 5732] combine_rows_per_src=[7834, 7200, 3418, 8192, 8192, 8168, 8044, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=181993472 dispatch_meta_bytes=229328 combine_payload_bytes=181993472 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=182222800 ingress_bytes=219742208 link_bytes=219742208 token_bytes=364216272 weight_movement_bytes=37748736 total_nvlink_bytes=401965008 weight_to_token_ratio=0.103644 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.105461717e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64946 payload_rows_remote=56824 dup_rows=590 dup_rows_remote=590 dedup_ratio=0.009003 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 246] payload_rows_per_dst=[8192, 8122, 8182, 8192, 8192, 8192, 7854, 8020] dup_rows_per_dst=[0, 0, 268, 14, 22, 4, 0, 282] combine_rows_per_src=[8192, 8122, 8182, 8192, 8192, 8192, 7854, 8020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=209092608 dispatch_meta_bytes=206552 combine_payload_bytes=209092608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209299160 ingress_bytes=227966976 link_bytes=227966976 token_bytes=418391768 weight_movement_bytes=18874368 total_nvlink_bytes=437266136 weight_to_token_ratio=0.045112 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.184267397e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232751104 dispatch_meta_bytes=229656 combine_payload_bytes=232751104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232980760 ingress_bytes=251625472 link_bytes=251625472 token_bytes=465731864 weight_movement_bytes=18874368 total_nvlink_bytes=484606232 weight_to_token_ratio=0.040526 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410951465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63662 payload_rows_remote=55470 dup_rows=1874 dup_rows_remote=1846 dedup_ratio=0.028595 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3151, 0, 0, 0, 0, 0, 179] payload_rows_per_dst=[8192, 8028, 8186, 8192, 8192, 8178, 8090, 6604] dup_rows_per_dst=[0, 0, 1784, 28, 4, 2, 0, 56] combine_rows_per_src=[8192, 8028, 8186, 8192, 8192, 8178, 8090, 6604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60640 payload_rows_remote=52448 dup_rows=4896 dup_rows_remote=4532 dedup_ratio=0.074707 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 1472, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 9] payload_rows_per_dst=[8192, 7614, 5676, 8096, 7980, 8158, 7908, 7016] dup_rows_per_dst=[364, 2656, 1834, 0, 0, 28, 0, 14] combine_rows_per_src=[8192, 7614, 5676, 8096, 7980, 8158, 7908, 7016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61978 payload_rows_remote=53786 dup_rows=3558 dup_rows_remote=3408 dedup_ratio=0.054291 router_tokens_per_expert=[0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2497, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2187, 4094, 2, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 462, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[4998, 8190, 7840, 8188, 8186, 8192, 8192, 8192] dup_rows_per_dst=[4, 4, 2474, 0, 926, 0, 150, 0] combine_rows_per_src=[4998, 8190, 7840, 8188, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58404 payload_rows_remote=50212 dup_rows=7132 dup_rows_remote=7132 dedup_ratio=0.108826 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 3305, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 3549, 0, 0, 0, 3767, 0, 0, 0, 0, 1645, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7534, 6620, 8192, 8192, 8192, 3290, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 8, 7114, 0, 0, 4] combine_rows_per_src=[7534, 6620, 8192, 8192, 8192, 3290, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62816 payload_rows_remote=55708 dup_rows=2720 dup_rows_remote=2718 dedup_ratio=0.041504 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 261, 0, 3201, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 3942, 0, 0, 0, 0, 1323, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7896, 6856, 8192, 8188, 7108, 8192, 8192, 8192] dup_rows_per_dst=[2, 72, 0, 0, 2, 2644, 0, 0] combine_rows_per_src=[7896, 6856, 8192, 8188, 7108, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=227205120 dispatch_meta_bytes=229264 combine_payload_bytes=227205120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227434384 ingress_bytes=246079488 link_bytes=246079488 token_bytes=454639504 weight_movement_bytes=18874368 total_nvlink_bytes=473513872 weight_to_token_ratio=0.041515 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357812575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=214827008 dispatch_meta_bytes=227920 combine_payload_bytes=214827008 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=215054928 ingress_bytes=290324480 link_bytes=290324480 token_bytes=429881936 weight_movement_bytes=75497472 total_nvlink_bytes=505379408 weight_to_token_ratio=0.175624 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.781746319e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63572 payload_rows_remote=55418 dup_rows=1964 dup_rows_remote=1962 dedup_ratio=0.029968 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 976, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3279, 0, 0, 0, 0, 0, 227] payload_rows_per_dst=[8192, 7666, 8182, 8192, 8192, 8154, 8054, 6940] dup_rows_per_dst=[0, 0, 1878, 10, 2, 2, 0, 72] combine_rows_per_src=[8192, 7666, 8182, 8192, 8192, 8154, 8054, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=220307456 dispatch_meta_bytes=228776 combine_payload_bytes=220307456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220536232 ingress_bytes=239181824 link_bytes=239181824 token_bytes=440843688 weight_movement_bytes=18874368 total_nvlink_bytes=459718056 weight_to_token_ratio=0.042814 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.291722553e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=205668352 dispatch_meta_bytes=229376 combine_payload_bytes=205668352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205897728 ingress_bytes=224542720 link_bytes=224542720 token_bytes=411566080 weight_movement_bytes=18874368 total_nvlink_bytes=430440448 weight_to_token_ratio=0.045860 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.151457861e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=228179968 dispatch_meta_bytes=233704 combine_payload_bytes=228179968 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228413672 ingress_bytes=284803072 link_bytes=284803072 token_bytes=456593640 weight_movement_bytes=56623104 total_nvlink_bytes=513216744 weight_to_token_ratio=0.124012 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.728842904e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62842 payload_rows_remote=54650 dup_rows=2694 dup_rows_remote=2694 dedup_ratio=0.041107 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 37, 0, 3350, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 3813, 0, 0, 0, 0, 1374, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7630, 6758, 8190, 8186, 7502, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 20, 28, 2640, 4, 0] combine_rows_per_src=[7630, 6758, 8190, 8186, 7502, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58912 payload_rows_remote=52034 dup_rows=6624 dup_rows_remote=3772 dedup_ratio=0.101074 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 497, 0, 3139, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 2120, 0, 0, 0, 3681, 0, 0, 0, 0, 2834, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7364, 6878, 8188, 8184, 6246, 5668, 8192, 8192] dup_rows_per_dst=[10, 2852, 0, 0, 3754, 0, 8, 0] combine_rows_per_src=[7364, 6878, 8188, 8184, 6246, 5668, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=226992128 dispatch_meta_bytes=229520 combine_payload_bytes=226992128 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=227221648 ingress_bytes=245866496 link_bytes=245866496 token_bytes=454213776 weight_movement_bytes=18874368 total_nvlink_bytes=473088144 weight_to_token_ratio=0.041554 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.355771791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=223846400 dispatch_meta_bytes=229376 combine_payload_bytes=223846400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224075776 ingress_bytes=242720768 link_bytes=242720768 token_bytes=447922176 weight_movement_bytes=18874368 total_nvlink_bytes=466796544 weight_to_token_ratio=0.042138 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.325630973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=213131264 dispatch_meta_bytes=223224 combine_payload_bytes=213131264 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=213354488 ingress_bytes=288628736 link_bytes=288628736 token_bytes=426485752 weight_movement_bytes=75497472 total_nvlink_bytes=501983224 weight_to_token_ratio=0.177022 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.765498534e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61316 payload_rows_remote=53124 dup_rows=4220 dup_rows_remote=4220 dedup_ratio=0.064392 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 131, 0, 2973, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 0, 0, 3587, 0, 0, 0, 0, 1934, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7190, 6204, 8192, 8192, 7718, 7436, 8192, 8192] dup_rows_per_dst=[2, 58, 0, 0, 2102, 2056, 2, 0] combine_rows_per_src=[7190, 6204, 8192, 8192, 7718, 7436, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56414 payload_rows_remote=48338 dup_rows=9122 dup_rows_remote=5266 dedup_ratio=0.139191 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 2673, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 0, 3876, 0, 0, 0, 0, 2175, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8076, 8192, 8192, 8192, 3042, 4354, 8178, 8188] dup_rows_per_dst=[3856, 5256, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[8076, 8192, 8192, 8192, 3042, 4354, 8178, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64146 payload_rows_remote=55954 dup_rows=1390 dup_rows_remote=1390 dedup_ratio=0.021210 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 181, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3404, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 6808] dup_rows_per_dst=[476, 32, 0, 32, 0, 780, 70, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 8192, 8192, 6808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63272 payload_rows_remote=55080 dup_rows=2264 dup_rows_remote=2264 dedup_ratio=0.034546 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 108, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 2971, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8188, 8188, 8192, 8184, 8188, 8192, 8192, 5948] dup_rows_per_dst=[486, 18, 0, 326, 0, 1376, 54, 4] combine_rows_per_src=[8188, 8188, 8192, 8184, 8188, 8192, 8192, 5948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55584 payload_rows_remote=47392 dup_rows=9952 dup_rows_remote=8924 dedup_ratio=0.151855 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 3702, 0, 0, 0, 0, 0, 2, 290, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 316, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 2, 0, 4094, 2, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 2, 348, 0, 50, 0] payload_rows_per_dst=[7588, 7078, 644, 8188, 7512, 8192, 8192, 8190] dup_rows_per_dst=[402, 2, 10, 0, 0, 4, 1028, 8506] combine_rows_per_src=[7588, 7078, 644, 8188, 7512, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=197992448 dispatch_meta_bytes=214416 combine_payload_bytes=197992448 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=198206864 ingress_bytes=235741184 link_bytes=235741184 token_bytes=396199312 weight_movement_bytes=37748736 total_nvlink_bytes=433948048 weight_to_token_ratio=0.095277 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.258756033e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=217595904 dispatch_meta_bytes=229376 combine_payload_bytes=217595904 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=217825280 ingress_bytes=255344640 link_bytes=255344640 token_bytes=435421184 weight_movement_bytes=37748736 total_nvlink_bytes=473169920 weight_to_token_ratio=0.086695 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.446586703e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62792 payload_rows_remote=54600 dup_rows=2744 dup_rows_remote=42 dedup_ratio=0.041870 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 4091, 0, 0, 134, 0, 3237, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 0, 3980, 0, 0, 0, 0, 1361, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7960, 6720, 8190, 8182, 7164, 8192, 8192, 8192] dup_rows_per_dst=[10, 30, 0, 0, 2, 2702, 0, 0] combine_rows_per_src=[7960, 6720, 8190, 8182, 7164, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=229187584 dispatch_meta_bytes=229376 combine_payload_bytes=229187584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229416960 ingress_bytes=248061952 link_bytes=248061952 token_bytes=458604544 weight_movement_bytes=18874368 total_nvlink_bytes=477478912 weight_to_token_ratio=0.041156 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376807570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=225607680 dispatch_meta_bytes=229376 combine_payload_bytes=225607680 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=225837056 ingress_bytes=282230784 link_bytes=282230784 token_bytes=451444736 weight_movement_bytes=56623104 total_nvlink_bytes=508067840 weight_to_token_ratio=0.125426 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.704196506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54420 payload_rows_remote=46228 dup_rows=11116 dup_rows_remote=1244 dedup_ratio=0.169617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 698, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 2717, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4494, 6770, 8192, 6638, 8188, 3754, 8192, 8192] dup_rows_per_dst=[768, 16, 14, 76, 26, 168, 176, 9872] combine_rows_per_src=[4494, 6770, 8192, 6638, 8188, 3754, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=194117632 dispatch_meta_bytes=225264 combine_payload_bytes=194117632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=194342896 ingress_bytes=231866368 link_bytes=231866368 token_bytes=388460528 weight_movement_bytes=37748736 total_nvlink_bytes=426209264 weight_to_token_ratio=0.097175 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=14720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221629453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59662 payload_rows_remote=51470 dup_rows=5874 dup_rows_remote=5874 dedup_ratio=0.089630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1177, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8190, 8172, 8182, 8192, 8192, 2354] dup_rows_per_dst=[60, 0, 0, 2, 0, 5802, 10, 0] combine_rows_per_src=[8188, 8192, 8190, 8172, 8182, 8192, 8192, 2354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=223641600 dispatch_meta_bytes=218568 combine_payload_bytes=223641600 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=223860168 ingress_bytes=280264704 link_bytes=280264704 token_bytes=447501768 weight_movement_bytes=56623104 total_nvlink_bytes=504124872 weight_to_token_ratio=0.126532 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.685358495e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=189349888 dispatch_meta_bytes=189888 combine_payload_bytes=189349888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=189539776 ingress_bytes=227098624 link_bytes=227098624 token_bytes=378889664 weight_movement_bytes=37748736 total_nvlink_bytes=416638400 weight_to_token_ratio=0.099630 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.175947276e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=210821120 dispatch_meta_bytes=229376 combine_payload_bytes=210821120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211050496 ingress_bytes=229695488 link_bytes=229695488 token_bytes=421871616 weight_movement_bytes=18874368 total_nvlink_bytes=440745984 weight_to_token_ratio=0.044740 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.200829149e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63692 payload_rows_remote=55500 dup_rows=1844 dup_rows_remote=1108 dedup_ratio=0.028137 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 286, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3430, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 7682, 6860] dup_rows_per_dst=[736, 6, 0, 980, 0, 96, 26, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 7682, 6860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62386 payload_rows_remote=54194 dup_rows=3150 dup_rows_remote=2892 dedup_ratio=0.048065 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 290, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2521, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5042] dup_rows_per_dst=[644, 2, 0, 258, 0, 2234, 12, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64446 payload_rows_remote=56576 dup_rows=1090 dup_rows_remote=1090 dedup_ratio=0.016632 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 58, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 7428, 7870, 8192, 8192, 8192] dup_rows_per_dst=[0, 134, 0, 0, 0, 956, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 7428, 7870, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63996 payload_rows_remote=55804 dup_rows=1540 dup_rows_remote=1536 dedup_ratio=0.023499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3399, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8188, 8168, 8192, 6804, 8072, 8192, 8188, 8192] dup_rows_per_dst=[0, 24, 4, 18, 0, 1490, 0, 4] combine_rows_per_src=[8188, 8168, 8192, 6804, 8072, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64080 payload_rows_remote=55894 dup_rows=1456 dup_rows_remote=626 dedup_ratio=0.022217 router_tokens_per_expert=[0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4088, 2, 0, 211, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 0, 0, 0, 0, 0, 587] payload_rows_per_dst=[8188, 8180, 7376, 8192, 8192, 8188, 8186, 7578] dup_rows_per_dst=[0, 4, 404, 10, 178, 18, 830, 12] combine_rows_per_src=[8188, 8180, 7376, 8192, 8192, 8188, 8186, 7578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=227328000 dispatch_meta_bytes=226432 combine_payload_bytes=227328000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227554432 ingress_bytes=246202368 link_bytes=246202368 token_bytes=454882432 weight_movement_bytes=18874368 total_nvlink_bytes=473756800 weight_to_token_ratio=0.041493 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358989951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=221978624 dispatch_meta_bytes=228344 combine_payload_bytes=221978624 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222206968 ingress_bytes=278601728 link_bytes=278601728 token_bytes=444185592 weight_movement_bytes=56623104 total_nvlink_bytes=500808696 weight_to_token_ratio=0.127476 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.669424677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231735296 dispatch_meta_bytes=230664 combine_payload_bytes=231735296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231965960 ingress_bytes=250609664 link_bytes=250609664 token_bytes=463701256 weight_movement_bytes=18874368 total_nvlink_bytes=482575624 weight_to_token_ratio=0.040704 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401218493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61008 payload_rows_remote=52816 dup_rows=4528 dup_rows_remote=4526 dedup_ratio=0.069092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 5446, 8192, 6744, 7876, 8178, 8188, 8192] dup_rows_per_dst=[0, 4, 0, 0, 2748, 1756, 18, 2] combine_rows_per_src=[8192, 5446, 8192, 6744, 7876, 8178, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63268 payload_rows_remote=55858 dup_rows=2268 dup_rows_remote=1516 dedup_ratio=0.034607 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 164, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 3210, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8182, 7410, 8192, 6720] dup_rows_per_dst=[448, 194, 0, 30, 0, 752, 156, 688] combine_rows_per_src=[8192, 8192, 8188, 8192, 8182, 7410, 8192, 6720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=228573184 dispatch_meta_bytes=229360 combine_payload_bytes=228573184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228802544 ingress_bytes=247447552 link_bytes=247447552 token_bytes=457375728 weight_movement_bytes=18874368 total_nvlink_bytes=476250096 weight_to_token_ratio=0.041267 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.370920691e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=228941824 dispatch_meta_bytes=226080 combine_payload_bytes=228941824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229167904 ingress_bytes=247816192 link_bytes=247816192 token_bytes=458109728 weight_movement_bytes=18874368 total_nvlink_bytes=476984096 weight_to_token_ratio=0.041201 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374452819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63114 payload_rows_remote=54998 dup_rows=2422 dup_rows_remote=2362 dedup_ratio=0.036957 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3964, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8116, 8192, 7936, 8150, 6148, 8188, 8192] dup_rows_per_dst=[0, 60, 0, 2122, 0, 238, 0, 2] combine_rows_per_src=[8192, 8116, 8192, 7936, 8150, 6148, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=216334336 dispatch_meta_bytes=229368 combine_payload_bytes=216334336 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=216563704 ingress_bytes=235208704 link_bytes=235208704 token_bytes=432898040 weight_movement_bytes=18874368 total_nvlink_bytes=451772408 weight_to_token_ratio=0.043600 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.253654072e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228794368 dispatch_meta_bytes=229496 combine_payload_bytes=228794368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229023864 ingress_bytes=266543104 link_bytes=266543104 token_bytes=457818232 weight_movement_bytes=37748736 total_nvlink_bytes=495566968 weight_to_token_ratio=0.082454 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.553884875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=225271808 dispatch_meta_bytes=229440 combine_payload_bytes=225271808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225501248 ingress_bytes=263020544 link_bytes=263020544 token_bytes=450773056 weight_movement_bytes=37748736 total_nvlink_bytes=488521792 weight_to_token_ratio=0.083742 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.520133438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59058 payload_rows_remote=50866 dup_rows=6478 dup_rows_remote=6474 dedup_ratio=0.098846 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7704, 6994, 3410, 8188, 8186] dup_rows_per_dst=[4, 996, 0, 5472, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7704, 6994, 3410, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64134 payload_rows_remote=57284 dup_rows=1402 dup_rows_remote=1400 dedup_ratio=0.021393 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3423, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 6850, 8144, 8192, 8192, 8192] dup_rows_per_dst=[0, 78, 0, 2, 0, 1322, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 6850, 8144, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63370 payload_rows_remote=56986 dup_rows=2166 dup_rows_remote=2166 dedup_ratio=0.033051 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 792, 0, 0, 4066, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0] payload_rows_per_dst=[8192, 8192, 7860, 8192, 6384, 8172, 8186, 8192] dup_rows_per_dst=[0, 210, 0, 0, 0, 1404, 0, 552] combine_rows_per_src=[8192, 8192, 7860, 8192, 6384, 8172, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62316 payload_rows_remote=54414 dup_rows=3220 dup_rows_remote=3220 dedup_ratio=0.049133 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3986, 0, 0, 925, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0] payload_rows_per_dst=[8192, 8192, 7902, 8192, 6756, 6776, 8118, 8188] dup_rows_per_dst=[2, 8, 0, 4, 1268, 1648, 0, 290] combine_rows_per_src=[8192, 8192, 7902, 8192, 6756, 6776, 8118, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=208347136 dispatch_meta_bytes=229360 combine_payload_bytes=208347136 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208576496 ingress_bytes=246095872 link_bytes=246095872 token_bytes=416923632 weight_movement_bytes=37748736 total_nvlink_bytes=454672368 weight_to_token_ratio=0.090541 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357969559e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62208 payload_rows_remote=54020 dup_rows=3328 dup_rows_remote=3324 dedup_ratio=0.050781 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 4093, 0, 2, 536, 0, 3974, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 2421, 0, 0, 0, 3998, 0, 0, 0, 0, 1135, 3947, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8000, 8070, 8190, 8190, 5470, 7908, 8188, 8192] dup_rows_per_dst=[6, 962, 0, 2, 92, 2262, 4, 0] combine_rows_per_src=[8000, 8070, 8190, 8190, 5470, 7908, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=234635264 dispatch_meta_bytes=234736 combine_payload_bytes=234635264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234870000 ingress_bytes=272384000 link_bytes=272384000 token_bytes=469505264 weight_movement_bytes=37748736 total_nvlink_bytes=507254000 weight_to_token_ratio=0.080401 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609849467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233414656 dispatch_meta_bytes=236608 combine_payload_bytes=233414656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233651264 ingress_bytes=252289024 link_bytes=252289024 token_bytes=467065920 weight_movement_bytes=18874368 total_nvlink_bytes=485940288 weight_to_token_ratio=0.040411 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417309294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60406 payload_rows_remote=52214 dup_rows=5130 dup_rows_remote=4722 dedup_ratio=0.078278 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 0, 0, 1407, 0, 0, 4069, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0] payload_rows_per_dst=[8154, 5462, 8056, 8192, 6068, 8090, 8192, 8192] dup_rows_per_dst=[6, 256, 0, 0, 0, 4422, 38, 408] combine_rows_per_src=[8154, 5462, 8056, 8192, 6068, 8090, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222879744 dispatch_meta_bytes=230536 combine_payload_bytes=222879744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223110280 ingress_bytes=241754112 link_bytes=241754112 token_bytes=445990024 weight_movement_bytes=18874368 total_nvlink_bytes=464864392 weight_to_token_ratio=0.042320 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316368951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64750 payload_rows_remote=56558 dup_rows=786 dup_rows_remote=62 dedup_ratio=0.011993 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7688, 7912, 8192, 8190, 8192] dup_rows_per_dst=[0, 62, 0, 0, 0, 724, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7688, 7912, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57106 payload_rows_remote=48918 dup_rows=8430 dup_rows_remote=8430 dedup_ratio=0.128632 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 40, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0] payload_rows_per_dst=[8190, 8188, 8108, 8192, 8192, 80, 7964, 8192] dup_rows_per_dst=[6, 0, 0, 0, 7410, 0, 0, 1014] combine_rows_per_src=[8190, 8188, 8108, 8192, 8192, 80, 7964, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=221265920 dispatch_meta_bytes=229376 combine_payload_bytes=221265920 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=221495296 ingress_bytes=259014656 link_bytes=259014656 token_bytes=442761216 weight_movement_bytes=37748736 total_nvlink_bytes=480509952 weight_to_token_ratio=0.085258 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.481750991e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=213868544 dispatch_meta_bytes=227744 combine_payload_bytes=213868544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214096288 ingress_bytes=232742912 link_bytes=232742912 token_bytes=427964832 weight_movement_bytes=18874368 total_nvlink_bytes=446839200 weight_to_token_ratio=0.044103 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230028066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=231661568 dispatch_meta_bytes=226480 combine_payload_bytes=231661568 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231888048 ingress_bytes=307159040 link_bytes=307159040 token_bytes=463549616 weight_movement_bytes=75497472 total_nvlink_bytes=539047088 weight_to_token_ratio=0.162868 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.943046790e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57900 payload_rows_remote=49708 dup_rows=7636 dup_rows_remote=7634 dedup_ratio=0.116516 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 3151, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0] payload_rows_per_dst=[8192, 8192, 8018, 8190, 6764, 6416, 8192, 3936] dup_rows_per_dst=[2, 1746, 2538, 0, 1944, 906, 452, 48] combine_rows_per_src=[8192, 8192, 8018, 8190, 6764, 6416, 8192, 3936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=200368128 dispatch_meta_bytes=229392 combine_payload_bytes=200368128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200597520 ingress_bytes=219242496 link_bytes=219242496 token_bytes=400965648 weight_movement_bytes=18874368 total_nvlink_bytes=419840016 weight_to_token_ratio=0.047072 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.100673723e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63206 payload_rows_remote=55014 dup_rows=2330 dup_rows_remote=2330 dedup_ratio=0.035553 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 829, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0] payload_rows_per_dst=[8192, 8182, 7760, 8192, 6448, 8064, 8176, 8192] dup_rows_per_dst=[0, 22, 0, 0, 0, 1580, 0, 728] combine_rows_per_src=[8192, 8182, 7760, 8192, 6448, 8064, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63976 payload_rows_remote=55790 dup_rows=1560 dup_rows_remote=1560 dedup_ratio=0.023804 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1144, 0, 4093, 3703, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7568, 7268, 8192, 8186, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 8, 40, 0, 0, 4, 1508] combine_rows_per_src=[8192, 7568, 7268, 8192, 8186, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60400 payload_rows_remote=52240 dup_rows=5136 dup_rows_remote=400 dedup_ratio=0.078369 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3962, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4081, 451, 0, 4096, 3892, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8174, 7924, 8160, 8190, 8192, 8162, 8192, 3406] dup_rows_per_dst=[0, 2, 4736, 92, 108, 58, 8, 132] combine_rows_per_src=[8174, 7924, 8160, 8190, 8192, 8162, 8192, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=203603968 dispatch_meta_bytes=229368 combine_payload_bytes=203603968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203833336 ingress_bytes=241352704 link_bytes=241352704 token_bytes=407437304 weight_movement_bytes=37748736 total_nvlink_bytes=445186040 weight_to_token_ratio=0.092649 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312522857e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=225337344 dispatch_meta_bytes=229376 combine_payload_bytes=225337344 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=225566720 ingress_bytes=244211712 link_bytes=244211712 token_bytes=450904064 weight_movement_bytes=18874368 total_nvlink_bytes=469778432 weight_to_token_ratio=0.041859 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.339916465e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59404 payload_rows_remote=51212 dup_rows=6132 dup_rows_remote=6126 dedup_ratio=0.093567 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 34, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 2, 0, 0, 0, 0, 0, 1096, 0, 2, 0, 0, 1, 4094, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 2430, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8186, 8070, 8190, 2198, 8192, 8188] dup_rows_per_dst=[94, 1162, 0, 0, 4, 6, 6, 4860] combine_rows_per_src=[8192, 8188, 8186, 8070, 8190, 2198, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228515840 dispatch_meta_bytes=229400 combine_payload_bytes=228515840 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=228745240 ingress_bytes=266264576 link_bytes=266264576 token_bytes=457261080 weight_movement_bytes=37748736 total_nvlink_bytes=495009816 weight_to_token_ratio=0.082554 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.551216157e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62708 payload_rows_remote=54684 dup_rows=2828 dup_rows_remote=664 dedup_ratio=0.043152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1729, 0, 4083, 3365, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6664, 7086, 8192, 8166, 8192, 8192, 8024] dup_rows_per_dst=[0, 0, 44, 564, 2, 0, 54, 2164] combine_rows_per_src=[8192, 6664, 7086, 8192, 8166, 8192, 8192, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=213975040 dispatch_meta_bytes=210560 combine_payload_bytes=213975040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=214185600 ingress_bytes=251723776 link_bytes=251723776 token_bytes=428160640 weight_movement_bytes=37748736 total_nvlink_bytes=465909376 weight_to_token_ratio=0.088165 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=25600 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411893366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56700 payload_rows_remote=49074 dup_rows=8836 dup_rows_remote=5344 dedup_ratio=0.134827 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 907, 0, 4096, 4071, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7626, 6324, 8192, 8192, 8168, 8192, 1818] dup_rows_per_dst=[0, 3492, 3992, 1350, 0, 0, 2, 0] combine_rows_per_src=[8188, 7626, 6324, 8192, 8192, 8168, 8192, 1818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63198 payload_rows_remote=55010 dup_rows=2338 dup_rows_remote=868 dedup_ratio=0.035675 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3815, 0, 0, 1013, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0] payload_rows_per_dst=[8192, 8192, 7812, 8192, 6240, 8188, 8190, 8192] dup_rows_per_dst=[26, 448, 0, 2, 0, 1470, 0, 392] combine_rows_per_src=[8192, 8192, 7812, 8192, 6240, 8188, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 2, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=209764352 dispatch_meta_bytes=229352 combine_payload_bytes=209764352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209993704 ingress_bytes=228638720 link_bytes=228638720 token_bytes=419758056 weight_movement_bytes=18874368 total_nvlink_bytes=438632424 weight_to_token_ratio=0.044965 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.190703718e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=223985664 dispatch_meta_bytes=221392 combine_payload_bytes=223985664 prefetch_bytes=0 reduce_bytes=132120576 egress_bytes=224207056 ingress_bytes=356106240 link_bytes=356106240 token_bytes=448192720 weight_movement_bytes=132120576 total_nvlink_bytes=580313296 weight_to_token_ratio=0.294785 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.412034776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=201007104 dispatch_meta_bytes=217672 combine_payload_bytes=201007104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=201224776 ingress_bytes=238755840 link_bytes=238755840 token_bytes=402231880 weight_movement_bytes=37748736 total_nvlink_bytes=439980616 weight_to_token_ratio=0.093848 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287640984e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58116 payload_rows_remote=49924 dup_rows=7420 dup_rows_remote=7380 dedup_ratio=0.113220 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3023, 0, 4082, 3285, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7978, 8040, 3464, 8164, 8192, 8040, 6046] dup_rows_per_dst=[40, 2116, 4854, 410, 0, 0, 0, 0] combine_rows_per_src=[8192, 7978, 8040, 3464, 8164, 8192, 8040, 6046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=225320960 dispatch_meta_bytes=223512 combine_payload_bytes=225320960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225544472 ingress_bytes=281944064 link_bytes=281944064 token_bytes=450865432 weight_movement_bytes=56623104 total_nvlink_bytes=507488536 weight_to_token_ratio=0.125588 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.701449296e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63688 payload_rows_remote=55496 dup_rows=1848 dup_rows_remote=1496 dedup_ratio=0.028198 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 926, 0, 4096, 3863, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 7018, 7614, 8192, 8192, 8192, 8192, 8112] dup_rows_per_dst=[0, 0, 30, 352, 0, 0, 0, 1466] combine_rows_per_src=[8176, 7018, 7614, 8192, 8192, 8192, 8192, 8112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64358 payload_rows_remote=56166 dup_rows=1178 dup_rows_remote=1116 dedup_ratio=0.017975 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 538, 0, 0] payload_rows_per_dst=[8192, 8192, 7014, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 62, 0, 0, 1116] combine_rows_per_src=[8192, 8192, 7014, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57280 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 51, 0, 0] payload_rows_per_dst=[8192, 8140, 8040, 8192, 8192, 8188, 8188, 8188] dup_rows_per_dst=[0, 4, 0, 0, 88, 0, 0, 124] combine_rows_per_src=[8192, 8140, 8040, 8192, 8192, 8188, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=204488704 dispatch_meta_bytes=229216 combine_payload_bytes=204488704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204717920 ingress_bytes=223363072 link_bytes=223363072 token_bytes=409206624 weight_movement_bytes=18874368 total_nvlink_bytes=428080992 weight_to_token_ratio=0.046124 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.140155054e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=227311616 dispatch_meta_bytes=227968 combine_payload_bytes=227311616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227539584 ingress_bytes=246185984 link_bytes=246185984 token_bytes=454851200 weight_movement_bytes=18874368 total_nvlink_bytes=473725568 weight_to_token_ratio=0.041496 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358832968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=230055936 dispatch_meta_bytes=229128 combine_payload_bytes=230055936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230285064 ingress_bytes=248930304 link_bytes=248930304 token_bytes=460341000 weight_movement_bytes=18874368 total_nvlink_bytes=479215368 weight_to_token_ratio=0.041001 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.385127692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63626 payload_rows_remote=55434 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.029144 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 2356, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3380, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 174, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8138, 8188, 6764, 7770, 8192, 8192, 8190] dup_rows_per_dst=[0, 1558, 0, 4, 0, 348, 0, 0] combine_rows_per_src=[8192, 8138, 8188, 6764, 7770, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64886 payload_rows_remote=56694 dup_rows=650 dup_rows_remote=142 dedup_ratio=0.009918 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 197, 0, 0] payload_rows_per_dst=[8188, 8166, 7580, 8186, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 30, 110, 0, 0, 508] combine_rows_per_src=[8188, 8166, 7580, 8186, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64408 payload_rows_remote=56772 dup_rows=1128 dup_rows_remote=1122 dedup_ratio=0.017212 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 102, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 395, 0, 0] payload_rows_per_dst=[8192, 7636, 7630, 8186, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 6, 44, 0, 84, 0, 0, 994] combine_rows_per_src=[8192, 7636, 7630, 8186, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229984 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234848864 ingress_bytes=253493248 link_bytes=253493248 token_bytes=469467744 weight_movement_bytes=18874368 total_nvlink_bytes=488342112 weight_to_token_ratio=0.040204 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428847576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62876 payload_rows_remote=54684 dup_rows=2660 dup_rows_remote=2660 dedup_ratio=0.040588 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1941, 0, 4071, 3326, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6970, 6824, 8192, 8142, 8192, 8192, 8172] dup_rows_per_dst=[0, 0, 38, 180, 0, 0, 80, 2362] combine_rows_per_src=[8192, 6970, 6824, 8192, 8142, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=227057664 dispatch_meta_bytes=229376 combine_payload_bytes=227057664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227287040 ingress_bytes=283680768 link_bytes=283680768 token_bytes=454344704 weight_movement_bytes=56623104 total_nvlink_bytes=510967808 weight_to_token_ratio=0.124626 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.718089539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232218624 dispatch_meta_bytes=227344 combine_payload_bytes=232218624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232445968 ingress_bytes=288841728 link_bytes=288841728 token_bytes=464664592 weight_movement_bytes=56623104 total_nvlink_bytes=521287696 weight_to_token_ratio=0.121858 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.767539319e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58510 payload_rows_remote=50318 dup_rows=7026 dup_rows_remote=7026 dedup_ratio=0.107208 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 196, 0, 0] payload_rows_per_dst=[8192, 8192, 7964, 8192, 7374, 8192, 8186, 2218] dup_rows_per_dst=[0, 826, 5888, 0, 62, 6, 18, 226] combine_rows_per_src=[8192, 8192, 7964, 8192, 7374, 8192, 8186, 2218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232538112 dispatch_meta_bytes=231576 combine_payload_bytes=232538112 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232769688 ingress_bytes=289161216 link_bytes=289161216 token_bytes=465307800 weight_movement_bytes=56623104 total_nvlink_bytes=521930904 weight_to_token_ratio=0.121690 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.770600496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64790 payload_rows_remote=56598 dup_rows=746 dup_rows_remote=746 dedup_ratio=0.011383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 295, 0, 0] payload_rows_per_dst=[8192, 8180, 7458, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 132, 0, 0, 614] combine_rows_per_src=[8192, 8180, 7458, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56234 payload_rows_remote=48042 dup_rows=9302 dup_rows_remote=9302 dedup_ratio=0.141937 router_tokens_per_expert=[0, 0, 0, 70, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 2435, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0] payload_rows_per_dst=[8162, 2960, 6256, 8192, 8192, 6258, 8192, 8022] dup_rows_per_dst=[136, 60, 0, 0, 0, 914, 8192, 0] combine_rows_per_src=[8162, 2960, 6256, 8192, 8192, 6258, 8192, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=223985664 dispatch_meta_bytes=229376 combine_payload_bytes=223985664 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224215040 ingress_bytes=261734400 link_bytes=261734400 token_bytes=448200704 weight_movement_bytes=37748736 total_nvlink_bytes=485949440 weight_to_token_ratio=0.084223 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.507810239e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=206102528 dispatch_meta_bytes=229376 combine_payload_bytes=206102528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206331904 ingress_bytes=224976896 link_bytes=224976896 token_bytes=412434432 weight_movement_bytes=18874368 total_nvlink_bytes=431308800 weight_to_token_ratio=0.045763 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.155617922e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56254 payload_rows_remote=48062 dup_rows=9282 dup_rows_remote=5190 dedup_ratio=0.141632 router_tokens_per_expert=[0, 0, 0, 107, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 2290, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0] payload_rows_per_dst=[7938, 8192, 8192, 8192, 8188, 7526, 0, 8026] dup_rows_per_dst=[40, 2576, 4092, 6, 0, 2568, 0, 0] combine_rows_per_src=[7938, 8192, 8192, 8192, 8188, 7526, 0, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231825408 dispatch_meta_bytes=229376 combine_payload_bytes=231825408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232054784 ingress_bytes=269574144 link_bytes=269574144 token_bytes=463880192 weight_movement_bytes=37748736 total_nvlink_bytes=501628928 weight_to_token_ratio=0.081376 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.582926809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=196780032 dispatch_meta_bytes=229376 combine_payload_bytes=196780032 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=197009408 ingress_bytes=234528768 link_bytes=234528768 token_bytes=393789440 weight_movement_bytes=37748736 total_nvlink_bytes=431538176 weight_to_token_ratio=0.095860 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247139260e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64510 payload_rows_remote=56482 dup_rows=1026 dup_rows_remote=1026 dedup_ratio=0.015656 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 17, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 71, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 414, 0] payload_rows_per_dst=[8188, 8192, 7478, 8188, 8054, 8190, 8028, 8192] dup_rows_per_dst=[2, 54, 4, 0, 8, 130, 0, 828] combine_rows_per_src=[8188, 8192, 7478, 8188, 8054, 8190, 8028, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56228 payload_rows_remote=48408 dup_rows=9308 dup_rows_remote=7404 dedup_ratio=0.142029 router_tokens_per_expert=[0, 0, 0, 249, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 1137, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0] payload_rows_per_dst=[8164, 4228, 5386, 8186, 8192, 6060, 8192, 7820] dup_rows_per_dst=[498, 56, 0, 0, 4, 562, 6284, 1904] combine_rows_per_src=[8164, 4228, 5386, 8186, 8192, 6060, 8192, 7820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=196861952 dispatch_meta_bytes=213008 combine_payload_bytes=196861952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197074960 ingress_bytes=215736320 link_bytes=215736320 token_bytes=393936912 weight_movement_bytes=18874368 total_nvlink_bytes=412811280 weight_to_token_ratio=0.047912 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.067079269e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50822 payload_rows_remote=42630 dup_rows=14714 dup_rows_remote=8230 dedup_ratio=0.224518 router_tokens_per_expert=[0, 0, 0, 9, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 3152, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0] payload_rows_per_dst=[3670, 8192, 8192, 8192, 8192, 6626, 0, 7758] dup_rows_per_dst=[4, 6484, 7950, 0, 0, 276, 0, 0] combine_rows_per_src=[3670, 8192, 8192, 8192, 8192, 6626, 0, 7758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231350272 dispatch_meta_bytes=230032 combine_payload_bytes=231350272 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=231580304 ingress_bytes=306847744 link_bytes=306847744 token_bytes=462930576 weight_movement_bytes=75497472 total_nvlink_bytes=538428048 weight_to_token_ratio=0.163086 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.940064104e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=198279168 dispatch_meta_bytes=223248 combine_payload_bytes=198279168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198502416 ingress_bytes=217153536 link_bytes=217153536 token_bytes=396781584 weight_movement_bytes=18874368 total_nvlink_bytes=415655952 weight_to_token_ratio=0.047569 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.080658336e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 131, 0, 0] payload_rows_per_dst=[8192, 8192, 7838, 8180, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 0, 76, 0, 0, 290] combine_rows_per_src=[8192, 8192, 7838, 8180, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51498 payload_rows_remote=43308 dup_rows=14038 dup_rows_remote=7984 dedup_ratio=0.214203 router_tokens_per_expert=[0, 0, 0, 391, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 0, 0, 1230, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0] payload_rows_per_dst=[8190, 8192, 3142, 8192, 8192, 7542, 0, 8048] dup_rows_per_dst=[6054, 4628, 2066, 0, 0, 1290, 0, 0] combine_rows_per_src=[8190, 8192, 3142, 8192, 8192, 7542, 0, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=174612480 dispatch_meta_bytes=203440 combine_payload_bytes=174612480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=174815920 ingress_bytes=193486848 link_bytes=193486848 token_bytes=349428400 weight_movement_bytes=18874368 total_nvlink_bytes=368302768 weight_to_token_ratio=0.054015 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853895776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54978 payload_rows_remote=46786 dup_rows=10558 dup_rows_remote=10558 dedup_ratio=0.161102 router_tokens_per_expert=[0, 0, 0, 96, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 0, 0, 2007, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0] payload_rows_per_dst=[7734, 7016, 5604, 8192, 8192, 7712, 2702, 7826] dup_rows_per_dst=[50, 1920, 2562, 0, 0, 5740, 286, 0] combine_rows_per_src=[7734, 7016, 5604, 8192, 8192, 7712, 2702, 7826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65294 payload_rows_remote=57102 dup_rows=242 dup_rows_remote=52 dedup_ratio=0.003693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8164, 8192, 8192, 8192, 8190, 8004, 8192] dup_rows_per_dst=[0, 0, 0, 0, 190, 10, 0, 42] combine_rows_per_src=[8168, 8164, 8192, 8192, 8192, 8190, 8004, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229376 combine_payload_bytes=233373696 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233603072 ingress_bytes=271122432 link_bytes=271122432 token_bytes=466976768 weight_movement_bytes=37748736 total_nvlink_bytes=504725504 weight_to_token_ratio=0.080836 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597761743e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65368 payload_rows_remote=57176 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.002563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8186, 8192, 8188, 8192, 8046, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 2, 156, 0, 0, 10] combine_rows_per_src=[8188, 8186, 8192, 8188, 8192, 8046, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=177389568 dispatch_meta_bytes=205168 combine_payload_bytes=177389568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=177594736 ingress_bytes=196263936 link_bytes=196263936 token_bytes=354984304 weight_movement_bytes=18874368 total_nvlink_bytes=373858672 weight_to_token_ratio=0.053170 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.880504467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=191635456 dispatch_meta_bytes=229376 combine_payload_bytes=191635456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191864832 ingress_bytes=210509824 link_bytes=210509824 token_bytes=383500288 weight_movement_bytes=18874368 total_nvlink_bytes=402374656 weight_to_token_ratio=0.049216 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.017001556e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233889792 dispatch_meta_bytes=228616 combine_payload_bytes=233889792 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234118408 ingress_bytes=309387264 link_bytes=309387264 token_bytes=468008200 weight_movement_bytes=75497472 total_nvlink_bytes=543505672 weight_to_token_ratio=0.161317 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.964396536e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57840 payload_rows_remote=49648 dup_rows=7696 dup_rows_remote=7692 dedup_ratio=0.117432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 869, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 2995, 0, 4094, 4073, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8190, 8178, 516, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 1742, 0, 0, 4, 5948] combine_rows_per_src=[8190, 8178, 516, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61060 payload_rows_remote=52868 dup_rows=4476 dup_rows_remote=4470 dedup_ratio=0.068298 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7694, 8190, 8192, 7348, 5380, 8186, 7878, 8192] dup_rows_per_dst=[0, 0, 0, 0, 322, 1336, 2812, 6] combine_rows_per_src=[7694, 8190, 8192, 7348, 5380, 8186, 7878, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234192896 dispatch_meta_bytes=229376 combine_payload_bytes=234192896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234422272 ingress_bytes=253067264 link_bytes=253067264 token_bytes=468615168 weight_movement_bytes=18874368 total_nvlink_bytes=487489536 weight_to_token_ratio=0.040277 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424766007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64912 payload_rows_remote=56722 dup_rows=624 dup_rows_remote=624 dedup_ratio=0.009521 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8190, 8192, 8186, 8192, 7598, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 334, 64, 4, 0, 222] combine_rows_per_src=[8184, 8190, 8192, 8186, 8192, 7598, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=203358208 dispatch_meta_bytes=229360 combine_payload_bytes=203358208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203587568 ingress_bytes=222232576 link_bytes=222232576 token_bytes=406945776 weight_movement_bytes=18874368 total_nvlink_bytes=425820144 weight_to_token_ratio=0.046381 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.129323198e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=216547328 dispatch_meta_bytes=229352 combine_payload_bytes=216547328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216776680 ingress_bytes=235421696 link_bytes=235421696 token_bytes=433324008 weight_movement_bytes=18874368 total_nvlink_bytes=452198376 weight_to_token_ratio=0.043557 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.255694856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232333312 dispatch_meta_bytes=229384 combine_payload_bytes=232333312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232562696 ingress_bytes=251207680 link_bytes=251207680 token_bytes=464896008 weight_movement_bytes=18874368 total_nvlink_bytes=483770376 weight_to_token_ratio=0.040599 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406948388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56394 payload_rows_remote=49510 dup_rows=9142 dup_rows_remote=8468 dedup_ratio=0.139496 router_tokens_per_expert=[0, 0, 0, 125, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 0, 0, 2837, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0] payload_rows_per_dst=[8186, 2438, 6338, 8192, 8190, 6884, 8192, 7974] dup_rows_per_dst=[244, 42, 0, 0, 0, 674, 8182, 0] combine_rows_per_src=[8186, 2438, 6338, 8192, 8190, 6884, 8192, 7974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63362 payload_rows_remote=55170 dup_rows=2174 dup_rows_remote=1606 dedup_ratio=0.033173 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 8192, 6436, 7788] dup_rows_per_dst=[568, 74, 4, 2, 1504, 16, 0, 6] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 8192, 6436, 7788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57128 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8184, 8192, 8192, 8192, 8102, 8078, 8192] dup_rows_per_dst=[0, 0, 0, 0, 182, 0, 0, 34] combine_rows_per_src=[8188, 8184, 8192, 8192, 8192, 8102, 8078, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62716 payload_rows_remote=57336 dup_rows=2820 dup_rows_remote=2820 dedup_ratio=0.043030 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 5380, 8192, 8192, 8192] dup_rows_per_dst=[0, 2812, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8184, 8192, 8192, 8192, 5380, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64848 payload_rows_remote=56660 dup_rows=688 dup_rows_remote=688 dedup_ratio=0.010498 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0] payload_rows_per_dst=[8188, 8192, 8188, 8192, 7512, 8192, 8192, 8192] dup_rows_per_dst=[0, 680, 0, 0, 0, 0, 4, 4] combine_rows_per_src=[8188, 8192, 8188, 8192, 7512, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=202792960 dispatch_meta_bytes=231912 combine_payload_bytes=202792960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203024872 ingress_bytes=221667328 link_bytes=221667328 token_bytes=405817832 weight_movement_bytes=18874368 total_nvlink_bytes=424692200 weight_to_token_ratio=0.046509 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.123907270e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=225976320 dispatch_meta_bytes=227104 combine_payload_bytes=225976320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226203424 ingress_bytes=282599424 link_bytes=282599424 token_bytes=452179744 weight_movement_bytes=56623104 total_nvlink_bytes=508802848 weight_to_token_ratio=0.125223 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.707728633e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229376 combine_payload_bytes=233996288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234225664 ingress_bytes=271745024 link_bytes=271745024 token_bytes=468221952 weight_movement_bytes=37748736 total_nvlink_bytes=505970688 weight_to_token_ratio=0.080621 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603727113e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=240624 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235088880 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469937136 weight_movement_bytes=18874368 total_nvlink_bytes=488811504 weight_to_token_ratio=0.040164 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63186 payload_rows_remote=54994 dup_rows=2350 dup_rows_remote=2350 dedup_ratio=0.035858 router_tokens_per_expert=[0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 3994, 0, 0, 1, 2, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 924, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 133, 0, 0] payload_rows_per_dst=[8192, 7992, 6058, 8186, 8192, 8190, 8192, 8184] dup_rows_per_dst=[4, 6, 22, 0, 204, 0, 0, 2114] combine_rows_per_src=[8192, 7992, 6058, 8186, 8192, 8190, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58618 payload_rows_remote=50430 dup_rows=6918 dup_rows_remote=6912 dedup_ratio=0.105560 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 1324, 8186, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 4, 6906, 2, 0, 6] combine_rows_per_src=[8152, 1324, 8186, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=232079360 dispatch_meta_bytes=229392 combine_payload_bytes=232079360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=232308752 ingress_bytes=269828096 link_bytes=269828096 token_bytes=464388112 weight_movement_bytes=37748736 total_nvlink_bytes=502136848 weight_to_token_ratio=0.081287 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.585360052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64270 payload_rows_remote=56078 dup_rows=1266 dup_rows_remote=8 dedup_ratio=0.019318 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 6938, 8192, 8192, 8188] dup_rows_per_dst=[0, 1258, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8184, 8192, 8192, 8192, 6938, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=225255424 dispatch_meta_bytes=229376 combine_payload_bytes=225255424 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225484800 ingress_bytes=281878528 link_bytes=281878528 token_bytes=450740224 weight_movement_bytes=56623104 total_nvlink_bytes=507363328 weight_to_token_ratio=0.125622 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.700821362e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=206561280 dispatch_meta_bytes=229368 combine_payload_bytes=206561280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206790648 ingress_bytes=225435648 link_bytes=225435648 token_bytes=413351928 weight_movement_bytes=18874368 total_nvlink_bytes=432226296 weight_to_token_ratio=0.045662 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.160013458e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=229695488 dispatch_meta_bytes=224344 combine_payload_bytes=229695488 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=229919832 ingress_bytes=286318592 link_bytes=286318592 token_bytes=459615320 weight_movement_bytes=56623104 total_nvlink_bytes=516238424 weight_to_token_ratio=0.123197 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.743363871e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65046 payload_rows_remote=56854 dup_rows=490 dup_rows_remote=474 dedup_ratio=0.007477 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8162, 8184, 8192, 8138, 8192, 8192, 7794, 8192] dup_rows_per_dst=[0, 0, 0, 0, 400, 16, 0, 74] combine_rows_per_src=[8162, 8184, 8192, 8138, 8192, 8192, 7794, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65166 payload_rows_remote=56974 dup_rows=370 dup_rows_remote=256 dedup_ratio=0.005646 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8078, 8146, 7982, 8192, 8192, 8192] dup_rows_per_dst=[114, 256, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8078, 8146, 7982, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63190 payload_rows_remote=54998 dup_rows=2346 dup_rows_remote=2346 dedup_ratio=0.035797 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 896, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4080, 4096] payload_rows_per_dst=[8192, 5862, 8192, 8192, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 494, 4, 0, 1814, 0, 34] combine_rows_per_src=[8192, 5862, 8192, 8192, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64992 payload_rows_remote=56800 dup_rows=544 dup_rows_remote=544 dedup_ratio=0.008301 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 7652, 8192, 8192, 8192] dup_rows_per_dst=[0, 544, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 7652, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65020 payload_rows_remote=56828 dup_rows=516 dup_rows_remote=496 dedup_ratio=0.007874 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096] payload_rows_per_dst=[8192, 7738, 8192, 8192, 8192, 8190, 8190, 8134] dup_rows_per_dst=[2, 0, 20, 4, 4, 486, 0, 0] combine_rows_per_src=[8192, 7738, 8192, 8192, 8192, 8190, 8190, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232873984 dispatch_meta_bytes=229312 combine_payload_bytes=232873984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233103296 ingress_bytes=251748352 link_bytes=251748352 token_bytes=465977280 weight_movement_bytes=18874368 total_nvlink_bytes=484851648 weight_to_token_ratio=0.040505 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412128841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=233365504 dispatch_meta_bytes=228920 combine_payload_bytes=233365504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233594424 ingress_bytes=252239872 link_bytes=252239872 token_bytes=466959928 weight_movement_bytes=18874368 total_nvlink_bytes=485834296 weight_to_token_ratio=0.040420 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416838344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225271808 dispatch_meta_bytes=229376 combine_payload_bytes=225271808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225501184 ingress_bytes=263020544 link_bytes=263020544 token_bytes=450772992 weight_movement_bytes=37748736 total_nvlink_bytes=488521728 weight_to_token_ratio=0.083742 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.520133438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59364 payload_rows_remote=51178 dup_rows=6172 dup_rows_remote=1682 dedup_ratio=0.094177 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4089, 4096] payload_rows_per_dst=[8190, 5966, 6470, 8178, 8172, 6012, 8190, 8186] dup_rows_per_dst=[0, 0, 182, 0, 4, 1496, 0, 4490] combine_rows_per_src=[8190, 5966, 6470, 8178, 8172, 6012, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=232652800 dispatch_meta_bytes=229376 combine_payload_bytes=232652800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232882176 ingress_bytes=251527168 link_bytes=251527168 token_bytes=465534976 weight_movement_bytes=18874368 total_nvlink_bytes=484409344 weight_to_token_ratio=0.040543 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410009565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56918 payload_rows_remote=48726 dup_rows=8618 dup_rows_remote=1496 dedup_ratio=0.131500 router_tokens_per_expert=[0, 0, 0, 887, 0, 2345, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 0, 0, 2, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 2414, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4075, 2, 0] payload_rows_per_dst=[5962, 7612, 5756, 8192, 8190, 4858, 8192, 8156] dup_rows_per_dst=[504, 940, 0, 0, 4, 46, 7122, 2] combine_rows_per_src=[5962, 7612, 5756, 8192, 8190, 4858, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232767488 dispatch_meta_bytes=229296 combine_payload_bytes=232767488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232996784 ingress_bytes=251641856 link_bytes=251641856 token_bytes=465764272 weight_movement_bytes=18874368 total_nvlink_bytes=484638640 weight_to_token_ratio=0.040523 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411108449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62122 payload_rows_remote=54454 dup_rows=3414 dup_rows_remote=1248 dedup_ratio=0.052094 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1130, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3263, 4096] payload_rows_per_dst=[8192, 7668, 8192, 8192, 8192, 6968, 8190, 6528] dup_rows_per_dst=[0, 2166, 60, 6, 2, 1180, 0, 0] combine_rows_per_src=[8192, 7668, 8192, 8192, 8192, 6968, 8190, 6528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=209625088 dispatch_meta_bytes=211440 combine_payload_bytes=209625088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209836528 ingress_bytes=247373824 link_bytes=247373824 token_bytes=419461616 weight_movement_bytes=37748736 total_nvlink_bytes=457210352 weight_to_token_ratio=0.089993 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.370214266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=199581696 dispatch_meta_bytes=200888 combine_payload_bytes=199581696 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=199782584 ingress_bytes=312827904 link_bytes=312827904 token_bytes=399364280 weight_movement_bytes=113246208 total_nvlink_bytes=512610488 weight_to_token_ratio=0.283566 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.997363055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=223043584 dispatch_meta_bytes=222808 combine_payload_bytes=223043584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223266392 ingress_bytes=241917952 link_bytes=241917952 token_bytes=446309976 weight_movement_bytes=18874368 total_nvlink_bytes=465184344 weight_to_token_ratio=0.042290 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.317938785e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63148 payload_rows_remote=54956 dup_rows=2388 dup_rows_remote=2388 dedup_ratio=0.036438 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 8192, 8192, 8192, 5858, 8192, 8192, 8182] dup_rows_per_dst=[0, 2372, 0, 16, 0, 0, 0, 0] combine_rows_per_src=[8148, 8192, 8192, 8192, 5858, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64474 payload_rows_remote=57344 dup_rows=1062 dup_rows_remote=1062 dedup_ratio=0.016205 router_tokens_per_expert=[0, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7130, 8192, 8192, 8192] dup_rows_per_dst=[1060, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7130, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57992 payload_rows_remote=49800 dup_rows=7544 dup_rows_remote=7544 dedup_ratio=0.115112 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4083, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 668, 8192, 8172] dup_rows_per_dst=[0, 7508, 6, 0, 0, 0, 0, 30] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 668, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64908 payload_rows_remote=56716 dup_rows=628 dup_rows_remote=628 dedup_ratio=0.009583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096] payload_rows_per_dst=[8192, 7588, 8192, 8192, 8192, 8192, 8192, 8168] dup_rows_per_dst=[0, 0, 10, 0, 0, 618, 0, 0] combine_rows_per_src=[8192, 7588, 8192, 8192, 8192, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57232 dup_rows=112 dup_rows_remote=112 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8088, 8192, 8188, 8188] dup_rows_per_dst=[106, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8088, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=203980800 dispatch_meta_bytes=229376 combine_payload_bytes=203980800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204210176 ingress_bytes=222855168 link_bytes=222855168 token_bytes=408190976 weight_movement_bytes=18874368 total_nvlink_bytes=427065344 weight_to_token_ratio=0.046239 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.135288568e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=225099776 dispatch_meta_bytes=229376 combine_payload_bytes=225099776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225329152 ingress_bytes=243974144 link_bytes=243974144 token_bytes=450428928 weight_movement_bytes=18874368 total_nvlink_bytes=469303296 weight_to_token_ratio=0.041903 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337640205e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62450 payload_rows_remote=54258 dup_rows=3086 dup_rows_remote=3086 dedup_ratio=0.047089 router_tokens_per_expert=[0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[5880, 8188, 7916, 8192, 7698, 8192, 8192, 8192] dup_rows_per_dst=[468, 0, 26, 0, 2586, 2, 4, 0] combine_rows_per_src=[5880, 8188, 7916, 8192, 7698, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=233624 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235114648 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469995672 weight_movement_bytes=37748736 total_nvlink_bytes=507744408 weight_to_token_ratio=0.080317 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232308736 dispatch_meta_bytes=229376 combine_payload_bytes=232308736 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=232538112 ingress_bytes=251183104 link_bytes=251183104 token_bytes=464846848 weight_movement_bytes=18874368 total_nvlink_bytes=483721216 weight_to_token_ratio=0.040603 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406712913e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64948 payload_rows_remote=56776 dup_rows=588 dup_rows_remote=588 dedup_ratio=0.008972 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 1, 0, 284, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8188, 8192, 8192, 7648, 8172, 8192] dup_rows_per_dst=[0, 0, 0, 570, 8, 0, 0, 10] combine_rows_per_src=[8192, 8172, 8188, 8192, 8192, 7648, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234422272 dispatch_meta_bytes=229376 combine_payload_bytes=234422272 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234651648 ingress_bytes=309919744 link_bytes=309919744 token_bytes=469073920 weight_movement_bytes=75497472 total_nvlink_bytes=544571392 weight_to_token_ratio=0.160950 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.969498497e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8086, 8192, 8192, 8192] dup_rows_per_dst=[88, 0, 18, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8086, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=222240768 dispatch_meta_bytes=229376 combine_payload_bytes=222240768 prefetch_bytes=0 reduce_bytes=0 egress_bytes=222470144 ingress_bytes=222240768 link_bytes=222470144 token_bytes=444710912 weight_movement_bytes=0 total_nvlink_bytes=444710912 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.131599458e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=232554496 dispatch_meta_bytes=229456 combine_payload_bytes=232554496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232783952 ingress_bytes=251428864 link_bytes=251428864 token_bytes=465338448 weight_movement_bytes=18874368 total_nvlink_bytes=484212816 weight_to_token_ratio=0.040561 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409067664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229376 combine_payload_bytes=234446848 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234676224 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469123072 weight_movement_bytes=18874368 total_nvlink_bytes=487997440 weight_to_token_ratio=0.040233 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65290 payload_rows_remote=57098 dup_rows=246 dup_rows_remote=4 dedup_ratio=0.003754 router_tokens_per_expert=[0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8190, 7948, 8192, 8192, 8192] dup_rows_per_dst=[242, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 7948, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62612 payload_rows_remote=54420 dup_rows=2924 dup_rows_remote=648 dedup_ratio=0.044617 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1126, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 4080, 4096] payload_rows_per_dst=[8192, 5290, 8192, 8192, 8192, 8192, 8190, 8172] dup_rows_per_dst=[0, 0, 532, 4, 10, 2276, 0, 102] combine_rows_per_src=[8192, 5290, 8192, 8192, 8192, 8192, 8190, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57302 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8150, 8192, 8192, 8192] dup_rows_per_dst=[10, 0, 32, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8150, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65450 payload_rows_remote=57258 dup_rows=86 dup_rows_remote=86 dedup_ratio=0.001312 router_tokens_per_expert=[0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 8192, 8192, 8192, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 86, 0, 0, 0, 0] combine_rows_per_src=[8114, 8192, 8192, 8192, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233873408 dispatch_meta_bytes=228408 combine_payload_bytes=233873408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234101816 ingress_bytes=252747776 link_bytes=252747776 token_bytes=467975224 weight_movement_bytes=18874368 total_nvlink_bytes=486849592 weight_to_token_ratio=0.040332 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421704830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 18, 0] combine_rows_per_src=[8192, 8174, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=222904320 dispatch_meta_bytes=220272 combine_payload_bytes=222904320 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=223124592 ingress_bytes=298401792 link_bytes=298401792 token_bytes=446028912 weight_movement_bytes=75497472 total_nvlink_bytes=521526384 weight_to_token_ratio=0.169266 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.859139148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61596 payload_rows_remote=53408 dup_rows=3940 dup_rows_remote=3940 dedup_ratio=0.060120 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8186, 5766, 8192, 8192, 6690, 8190, 8188, 8192] dup_rows_per_dst=[0, 1372, 124, 10, 2426, 0, 0, 8] combine_rows_per_src=[8186, 5766, 8192, 8192, 6690, 8190, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229376 combine_payload_bytes=234708992 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234938368 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469647360 weight_movement_bytes=18874368 total_nvlink_bytes=488521728 weight_to_token_ratio=0.040188 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64826 payload_rows_remote=56634 dup_rows=710 dup_rows_remote=710 dedup_ratio=0.010834 router_tokens_per_expert=[0, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7482, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 710, 0, 0, 0, 0] combine_rows_per_src=[7482, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234528768 dispatch_meta_bytes=229376 combine_payload_bytes=234528768 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234758144 ingress_bytes=234528768 link_bytes=234758144 token_bytes=469286912 weight_movement_bytes=0 total_nvlink_bytes=469286912 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249337028e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229376 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=235036672 ingress_bytes=253681664 link_bytes=253681664 token_bytes=469843968 weight_movement_bytes=18874368 total_nvlink_bytes=488718336 weight_to_token_ratio=0.040172 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430652885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=218759168 dispatch_meta_bytes=229392 combine_payload_bytes=218759168 prefetch_bytes=0 reduce_bytes=0 egress_bytes=218988560 ingress_bytes=218759168 link_bytes=218988560 token_bytes=437747728 weight_movement_bytes=0 total_nvlink_bytes=437747728 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.098240633e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64560 payload_rows_remote=56368 dup_rows=976 dup_rows_remote=0 dedup_ratio=0.014893 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8012, 8192, 7400, 8192, 8192, 8192, 8188] dup_rows_per_dst=[976, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8012, 8192, 7400, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231972864 dispatch_meta_bytes=229376 combine_payload_bytes=231972864 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=232202240 ingress_bytes=307470336 link_bytes=307470336 token_bytes=464175104 weight_movement_bytes=75497472 total_nvlink_bytes=539672576 weight_to_token_ratio=0.162649 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.946029475e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57296 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 8148, 8192, 8192] dup_rows_per_dst=[0, 40, 2, 0, 0, 0, 6, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 8148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65170 payload_rows_remote=56978 dup_rows=366 dup_rows_remote=366 dedup_ratio=0.005585 router_tokens_per_expert=[0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7828, 8192, 8192, 8190] dup_rows_per_dst=[342, 0, 24, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7828, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65258 payload_rows_remote=57066 dup_rows=278 dup_rows_remote=274 dedup_ratio=0.004242 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4093, 4096, 14, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 7922, 8188, 8192] dup_rows_per_dst=[0, 272, 4, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 7922, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=230883328 dispatch_meta_bytes=225472 combine_payload_bytes=230883328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231108800 ingress_bytes=268632064 link_bytes=268632064 token_bytes=461992128 weight_movement_bytes=37748736 total_nvlink_bytes=499740864 weight_to_token_ratio=0.081709 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.573900262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229376 combine_payload_bytes=234684416 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234913792 ingress_bytes=291307520 link_bytes=291307520 token_bytes=469598208 weight_movement_bytes=56623104 total_nvlink_bytes=526221312 weight_to_token_ratio=0.120578 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791165324e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233381888 dispatch_meta_bytes=229376 combine_payload_bytes=233381888 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233611264 ingress_bytes=271130624 link_bytes=271130624 token_bytes=466993152 weight_movement_bytes=37748736 total_nvlink_bytes=504741888 weight_to_token_ratio=0.080834 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597840235e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233742336 dispatch_meta_bytes=229360 combine_payload_bytes=233742336 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233971696 ingress_bytes=271491072 link_bytes=271491072 token_bytes=467714032 weight_movement_bytes=37748736 total_nvlink_bytes=505462768 weight_to_token_ratio=0.080709 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601293870e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63134 payload_rows_remote=54942 dup_rows=2402 dup_rows_remote=2400 dedup_ratio=0.036652 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 5968, 8190, 8192, 8148, 8068, 8192, 8192] dup_rows_per_dst=[0, 114, 0, 8, 0, 0, 2278, 2] combine_rows_per_src=[8184, 5968, 8190, 8192, 8148, 8068, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62044 payload_rows_remote=53852 dup_rows=3492 dup_rows_remote=3484 dedup_ratio=0.053284 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4094, 0, 14, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2668, 4094] payload_rows_per_dst=[8192, 7558, 8192, 8188, 8192, 8186, 8192, 5344] dup_rows_per_dst=[0, 0, 418, 0, 0, 3066, 8, 0] combine_rows_per_src=[8192, 7558, 8192, 8188, 8192, 8186, 8192, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64290 payload_rows_remote=56098 dup_rows=1246 dup_rows_remote=38 dedup_ratio=0.019012 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 4096, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8190, 8192, 7338, 7808, 8192] dup_rows_per_dst=[0, 1208, 0, 0, 30, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8186, 8190, 8192, 7338, 7808, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=225042432 dispatch_meta_bytes=229368 combine_payload_bytes=225042432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225271800 ingress_bytes=243916800 link_bytes=243916800 token_bytes=450314232 weight_movement_bytes=18874368 total_nvlink_bytes=469188600 weight_to_token_ratio=0.041914 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337090763e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=229777408 dispatch_meta_bytes=224544 combine_payload_bytes=229777408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230001952 ingress_bytes=248651776 link_bytes=248651776 token_bytes=459779360 weight_movement_bytes=18874368 total_nvlink_bytes=478653728 weight_to_token_ratio=0.041051 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.382458973e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=220577792 dispatch_meta_bytes=229344 combine_payload_bytes=220577792 prefetch_bytes=0 reduce_bytes=0 egress_bytes=220807136 ingress_bytes=220577792 link_bytes=220807136 token_bytes=441384928 weight_movement_bytes=0 total_nvlink_bytes=441384928 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.115665333e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60480 payload_rows_remote=52288 dup_rows=5056 dup_rows_remote=5054 dedup_ratio=0.077148 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3542, 8192, 8192, 8192, 8192, 7786, 8192, 8192] dup_rows_per_dst=[0, 2420, 0, 2634, 2, 0, 0, 0] combine_rows_per_src=[3542, 8192, 8192, 8192, 8192, 7786, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63556 payload_rows_remote=55364 dup_rows=1980 dup_rows_remote=1978 dedup_ratio=0.030212 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8192, 8192, 8190, 8192, 6224, 8192] dup_rows_per_dst=[2, 6, 40, 0, 0, 1932, 0, 0] combine_rows_per_src=[8192, 8182, 8192, 8192, 8190, 8192, 6224, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65214 payload_rows_remote=57022 dup_rows=322 dup_rows_remote=322 dedup_ratio=0.004913 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4081, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7870, 8192, 8192] dup_rows_per_dst=[0, 320, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7870, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60176 payload_rows_remote=51984 dup_rows=5360 dup_rows_remote=5346 dedup_ratio=0.081787 router_tokens_per_expert=[2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1505, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 3014, 8192, 8192, 8192, 8012, 8192, 8192] dup_rows_per_dst=[3636, 0, 14, 1710, 0, 0, 0, 0] combine_rows_per_src=[8190, 3014, 8192, 8192, 8192, 8012, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=214171648 dispatch_meta_bytes=229368 combine_payload_bytes=214171648 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=214401016 ingress_bytes=233046016 link_bytes=233046016 token_bytes=428572664 weight_movement_bytes=18874368 total_nvlink_bytes=447447032 weight_to_token_ratio=0.044040 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232932260e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=226770944 dispatch_meta_bytes=229368 combine_payload_bytes=226770944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227000312 ingress_bytes=264519680 link_bytes=264519680 token_bytes=453771256 weight_movement_bytes=37748736 total_nvlink_bytes=491519992 weight_to_token_ratio=0.083189 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.534497422e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235102208 ingress_bytes=253747200 link_bytes=253747200 token_bytes=469975040 weight_movement_bytes=18874368 total_nvlink_bytes=488849408 weight_to_token_ratio=0.040160 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431280819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233562112 dispatch_meta_bytes=229376 combine_payload_bytes=233562112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233791488 ingress_bytes=252436480 link_bytes=252436480 token_bytes=467353600 weight_movement_bytes=18874368 total_nvlink_bytes=486227968 weight_to_token_ratio=0.040386 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418722145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=212926464 dispatch_meta_bytes=229320 combine_payload_bytes=212926464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213155784 ingress_bytes=250675200 link_bytes=250675200 token_bytes=426082248 weight_movement_bytes=37748736 total_nvlink_bytes=463830984 weight_to_token_ratio=0.088595 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401846427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56926 payload_rows_remote=48734 dup_rows=8610 dup_rows_remote=8610 dedup_ratio=0.131378 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3562, 8192, 8116, 4610, 8192, 7870, 8192, 8192] dup_rows_per_dst=[0, 4284, 0, 862, 0, 3464, 0, 0] combine_rows_per_src=[3562, 8192, 8116, 4610, 8192, 7870, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63088 payload_rows_remote=55360 dup_rows=2448 dup_rows_remote=2448 dedup_ratio=0.037354 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7728, 8192, 7122, 8192, 7278, 8192, 8192] dup_rows_per_dst=[1700, 0, 16, 726, 4, 0, 2, 0] combine_rows_per_src=[8192, 7728, 8192, 7122, 8192, 7278, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65356 payload_rows_remote=57164 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.002747 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 84, 4094, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 4010, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8022, 8190, 8192, 8192] dup_rows_per_dst=[0, 4, 170, 0, 0, 0, 0, 6] combine_rows_per_src=[8188, 8192, 8192, 8188, 8022, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=199614464 dispatch_meta_bytes=229376 combine_payload_bytes=199614464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199843840 ingress_bytes=218488832 link_bytes=218488832 token_bytes=399458304 weight_movement_bytes=18874368 total_nvlink_bytes=418332672 weight_to_token_ratio=0.047250 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.093452485e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=226754560 dispatch_meta_bytes=231232 combine_payload_bytes=226754560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226985792 ingress_bytes=264503296 link_bytes=264503296 token_bytes=453740352 weight_movement_bytes=37748736 total_nvlink_bytes=491489088 weight_to_token_ratio=0.083195 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.534340439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234143744 dispatch_meta_bytes=229376 combine_payload_bytes=234143744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234373120 ingress_bytes=253018112 link_bytes=253018112 token_bytes=468516864 weight_movement_bytes=18874368 total_nvlink_bytes=487391232 weight_to_token_ratio=0.040285 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424295057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64822 payload_rows_remote=56724 dup_rows=714 dup_rows_remote=210 dedup_ratio=0.010895 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8098, 7572, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 504, 0, 34, 176] combine_rows_per_src=[8192, 8192, 8192, 8192, 8098, 7572, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52898 payload_rows_remote=44706 dup_rows=12638 dup_rows_remote=6 dedup_ratio=0.192841 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 648, 8192, 3408, 8190, 7890, 8190, 8188] dup_rows_per_dst=[12632, 0, 4, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 648, 8192, 3408, 8190, 7890, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60928 payload_rows_remote=52736 dup_rows=4608 dup_rows_remote=3690 dedup_ratio=0.070312 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8144, 3972, 8192, 8192, 8192, 7852, 8192, 8192] dup_rows_per_dst=[3640, 46, 4, 918, 0, 0, 0, 0] combine_rows_per_src=[8144, 3972, 8192, 8192, 8192, 7852, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65436 payload_rows_remote=57332 dup_rows=100 dup_rows_remote=100 dedup_ratio=0.001526 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8192, 8104, 8192, 8190] dup_rows_per_dst=[0, 90, 0, 0, 2, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 8192, 8104, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63904 payload_rows_remote=55712 dup_rows=1632 dup_rows_remote=1628 dedup_ratio=0.024902 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 287, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7890, 6866, 8188, 8192] dup_rows_per_dst=[0, 0, 4, 0, 682, 0, 368, 578] combine_rows_per_src=[8192, 8192, 8192, 8192, 7890, 6866, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=232341504 dispatch_meta_bytes=227736 combine_payload_bytes=232341504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232569240 ingress_bytes=251215872 link_bytes=251215872 token_bytes=464910744 weight_movement_bytes=18874368 total_nvlink_bytes=483785112 weight_to_token_ratio=0.040598 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407026880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=183115776 dispatch_meta_bytes=178848 combine_payload_bytes=183115776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=183294624 ingress_bytes=220864512 link_bytes=220864512 token_bytes=366410400 weight_movement_bytes=37748736 total_nvlink_bytes=404159136 weight_to_token_ratio=0.103023 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.116215082e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=216006656 dispatch_meta_bytes=225704 combine_payload_bytes=216006656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=216232360 ingress_bytes=272629760 link_bytes=272629760 token_bytes=432239016 weight_movement_bytes=56623104 total_nvlink_bytes=488862120 weight_to_token_ratio=0.131000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229728 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061600 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893472 weight_movement_bytes=18874368 total_nvlink_bytes=488767840 weight_to_token_ratio=0.040167 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63878 payload_rows_remote=55686 dup_rows=1658 dup_rows_remote=626 dedup_ratio=0.025299 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 4096] payload_rows_per_dst=[8188, 7950, 8192, 8192, 8092, 6880, 8192, 8192] dup_rows_per_dst=[0, 0, 58, 0, 534, 0, 34, 1032] combine_rows_per_src=[8188, 7950, 8192, 8192, 8092, 6880, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61464 payload_rows_remote=53272 dup_rows=4072 dup_rows_remote=4072 dedup_ratio=0.062134 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 0, 4096] payload_rows_per_dst=[8192, 8192, 8190, 8192, 5240, 7074, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 272, 0, 1338, 2462] combine_rows_per_src=[8192, 8192, 8190, 8192, 5240, 7074, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=228196352 dispatch_meta_bytes=229360 combine_payload_bytes=228196352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228425712 ingress_bytes=247070720 link_bytes=247070720 token_bytes=456622064 weight_movement_bytes=18874368 total_nvlink_bytes=475496432 weight_to_token_ratio=0.041335 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.367310073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57140 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.003296 router_tokens_per_expert=[0, 2, 0, 3996, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7996, 8192, 8192, 8188, 8192, 8192, 8180, 8188] dup_rows_per_dst=[8, 198, 0, 0, 0, 4, 0, 6] combine_rows_per_src=[7996, 8192, 8192, 8188, 8192, 8192, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=225248 combine_payload_bytes=228089856 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=228315104 ingress_bytes=322461696 link_bytes=322461696 token_bytes=456404960 weight_movement_bytes=94371840 total_nvlink_bytes=550776800 weight_to_token_ratio=0.206772 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.089669310e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=218202112 dispatch_meta_bytes=229376 combine_payload_bytes=218202112 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=218431488 ingress_bytes=274825216 link_bytes=274825216 token_bytes=436633600 weight_movement_bytes=56623104 total_nvlink_bytes=493256704 weight_to_token_ratio=0.129681 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.633239997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=234045440 dispatch_meta_bytes=229424 combine_payload_bytes=234045440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234274864 ingress_bytes=271794176 link_bytes=271794176 token_bytes=468320304 weight_movement_bytes=37748736 total_nvlink_bytes=506069040 weight_to_token_ratio=0.080605 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604198063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64438 payload_rows_remote=56246 dup_rows=1098 dup_rows_remote=1072 dedup_ratio=0.016754 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 604, 0, 0, 3547, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7094, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1072, 0, 0, 0, 26, 0, 0, 0] combine_rows_per_src=[8192, 7094, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53848 payload_rows_remote=45656 dup_rows=11688 dup_rows_remote=11688 dedup_ratio=0.178345 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8166, 8044, 2414, 2456] dup_rows_per_dst=[0, 0, 0, 4, 4934, 6676, 56, 18] combine_rows_per_src=[8192, 8192, 8192, 8192, 8166, 8044, 2414, 2456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64230 payload_rows_remote=56038 dup_rows=1306 dup_rows_remote=1306 dedup_ratio=0.019928 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7756, 7322, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 594, 0, 220, 492] combine_rows_per_src=[8192, 8192, 8192, 8192, 7756, 7322, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64592 payload_rows_remote=56400 dup_rows=944 dup_rows_remote=944 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 466, 0, 0, 3627, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7254, 8192, 8192, 8192, 8188, 8192, 8190] dup_rows_per_dst=[932, 0, 0, 0, 10, 0, 2, 0] combine_rows_per_src=[8192, 7254, 8192, 8192, 8192, 8188, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=230383616 dispatch_meta_bytes=229272 combine_payload_bytes=230383616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230612888 ingress_bytes=249257984 link_bytes=249257984 token_bytes=460996504 weight_movement_bytes=18874368 total_nvlink_bytes=479870872 weight_to_token_ratio=0.040943 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388267360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59360 payload_rows_remote=51656 dup_rows=6176 dup_rows_remote=6176 dedup_ratio=0.094238 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2504, 8192, 8192, 8192, 8192, 7704, 8192, 8192] dup_rows_per_dst=[0, 4428, 0, 1748, 0, 0, 0, 0] combine_rows_per_src=[2504, 8192, 8192, 8192, 8192, 7704, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=187006976 dispatch_meta_bytes=229376 combine_payload_bytes=187006976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=187236352 ingress_bytes=205881344 link_bytes=205881344 token_bytes=374243328 weight_movement_bytes=18874368 total_nvlink_bytes=393117696 weight_to_token_ratio=0.050433 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.972653738e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57996 payload_rows_remote=49814 dup_rows=7540 dup_rows_remote=7532 dedup_ratio=0.115051 router_tokens_per_expert=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 333, 0, 0, 3777, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[666, 8192, 8192, 8188, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 7528, 0, 0, 0, 4, 0, 8] combine_rows_per_src=[666, 8192, 8192, 8188, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=229531648 dispatch_meta_bytes=229376 combine_payload_bytes=229531648 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229761024 ingress_bytes=267280384 link_bytes=267280384 token_bytes=459292672 weight_movement_bytes=37748736 total_nvlink_bytes=497041408 weight_to_token_ratio=0.082189 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.560949129e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61726 payload_rows_remote=57336 dup_rows=3810 dup_rows_remote=3810 dedup_ratio=0.058136 router_tokens_per_expert=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1310, 0, 0, 2195, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4390, 8192, 8192, 8192, 8192, 8192, 8184] dup_rows_per_dst=[2620, 0, 0, 0, 1190, 0, 0, 0] combine_rows_per_src=[8192, 4390, 8192, 8192, 8192, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65216 payload_rows_remote=57028 dup_rows=320 dup_rows_remote=262 dedup_ratio=0.004883 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 4094, 0, 0, 4050, 4082, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8136, 8166, 8190, 8192, 7960, 8188, 8192] dup_rows_per_dst=[6, 242, 0, 0, 12, 2, 58, 0] combine_rows_per_src=[8192, 8136, 8166, 8190, 8192, 7960, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=231014400 dispatch_meta_bytes=229376 combine_payload_bytes=231014400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231243776 ingress_bytes=249888768 link_bytes=249888768 token_bytes=462258176 weight_movement_bytes=18874368 total_nvlink_bytes=481132544 weight_to_token_ratio=0.040831 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394311222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=211582976 dispatch_meta_bytes=231328 combine_payload_bytes=211582976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=211814304 ingress_bytes=249331712 link_bytes=249331712 token_bytes=423397280 weight_movement_bytes=37748736 total_nvlink_bytes=461146016 weight_to_token_ratio=0.089157 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388973786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=204038144 dispatch_meta_bytes=229384 combine_payload_bytes=204038144 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=204267528 ingress_bytes=241786880 link_bytes=241786880 token_bytes=408305672 weight_movement_bytes=37748736 total_nvlink_bytes=446054408 weight_to_token_ratio=0.092452 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316682918e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=244584 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235092840 ingress_bytes=291471360 link_bytes=291471360 token_bytes=469941096 weight_movement_bytes=56623104 total_nvlink_bytes=526564200 weight_to_token_ratio=0.120490 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792735159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233586688 dispatch_meta_bytes=229160 combine_payload_bytes=233586688 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=233815848 ingress_bytes=327958528 link_bytes=327958528 token_bytes=467402536 weight_movement_bytes=94371840 total_nvlink_bytes=561774376 weight_to_token_ratio=0.201907 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.142337250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57326 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 18] combine_rows_per_src=[8192, 8192, 8174, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61804 payload_rows_remote=53612 dup_rows=3732 dup_rows_remote=3364 dedup_ratio=0.056946 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 365, 0, 0, 3957, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7926, 8192, 8192, 4818, 8192, 8102, 8190] dup_rows_per_dst=[368, 3362, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 7926, 8192, 8192, 4818, 8192, 8102, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63634 payload_rows_remote=55442 dup_rows=1902 dup_rows_remote=1902 dedup_ratio=0.029022 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 949, 0, 0, 3145, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6290, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1898, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 6290, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65488 payload_rows_remote=57304 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 20, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8184, 8192, 8174, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 44] combine_rows_per_src=[8192, 8174, 8184, 8192, 8174, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234807296 dispatch_meta_bytes=229384 combine_payload_bytes=234807296 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=235036680 ingress_bytes=329179136 link_bytes=329179136 token_bytes=469843976 weight_movement_bytes=94371840 total_nvlink_bytes=564215816 weight_to_token_ratio=0.200858 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154032515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64488 payload_rows_remote=57114 dup_rows=1048 dup_rows_remote=1048 dedup_ratio=0.015991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7962, 7374, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 490, 0, 18, 540] combine_rows_per_src=[8192, 8192, 8192, 8192, 7962, 7374, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=219594752 dispatch_meta_bytes=227904 combine_payload_bytes=219594752 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=219822656 ingress_bytes=295092224 link_bytes=295092224 token_bytes=439417408 weight_movement_bytes=75497472 total_nvlink_bytes=514914880 weight_to_token_ratio=0.171813 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.827428496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65460 payload_rows_remote=57268 dup_rows=76 dup_rows_remote=20 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8174, 8150, 8192, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 12, 0, 0, 0, 4, 4, 56] combine_rows_per_src=[8184, 8174, 8150, 8192, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65380 payload_rows_remote=57236 dup_rows=156 dup_rows_remote=150 dedup_ratio=0.002380 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 71, 0, 0, 0] payload_rows_per_dst=[8192, 8144, 8176, 8192, 8100, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 150] combine_rows_per_src=[8192, 8144, 8176, 8192, 8100, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58418 payload_rows_remote=50226 dup_rows=7118 dup_rows_remote=7118 dedup_ratio=0.108612 router_tokens_per_expert=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3115, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 18, 0, 0, 422, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[1602, 8190, 8192, 8188, 8188, 7674, 8192, 8192] dup_rows_per_dst=[0, 6230, 38, 842, 0, 4, 0, 4] combine_rows_per_src=[1602, 8190, 8192, 8188, 8188, 7674, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=227090432 dispatch_meta_bytes=229376 combine_payload_bytes=227090432 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=227319808 ingress_bytes=283713536 link_bytes=283713536 token_bytes=454410240 weight_movement_bytes=56623104 total_nvlink_bytes=511033344 weight_to_token_ratio=0.124608 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.718403506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234717184 dispatch_meta_bytes=229408 combine_payload_bytes=234717184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234946592 ingress_bytes=253591552 link_bytes=253591552 token_bytes=469663776 weight_movement_bytes=18874368 total_nvlink_bytes=488538144 weight_to_token_ratio=0.040187 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429789476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=233938944 dispatch_meta_bytes=232648 combine_payload_bytes=233938944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234171592 ingress_bytes=271687680 link_bytes=271687680 token_bytes=468110536 weight_movement_bytes=37748736 total_nvlink_bytes=505859272 weight_to_token_ratio=0.080641 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603177671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234569728 dispatch_meta_bytes=229152 combine_payload_bytes=234569728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234798880 ingress_bytes=253444096 link_bytes=253444096 token_bytes=469368608 weight_movement_bytes=18874368 total_nvlink_bytes=488242976 weight_to_token_ratio=0.040212 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428376626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234438656 dispatch_meta_bytes=229544 combine_payload_bytes=234438656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234668200 ingress_bytes=272187392 link_bytes=272187392 token_bytes=469106856 weight_movement_bytes=37748736 total_nvlink_bytes=506855592 weight_to_token_ratio=0.080469 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607965666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=205725696 dispatch_meta_bytes=229376 combine_payload_bytes=205725696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205955072 ingress_bytes=224600064 link_bytes=224600064 token_bytes=411680768 weight_movement_bytes=18874368 total_nvlink_bytes=430555136 weight_to_token_ratio=0.045847 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.152007303e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64658 payload_rows_remote=56466 dup_rows=878 dup_rows_remote=878 dedup_ratio=0.013397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8192, 8186, 8192, 8050, 7478, 8192] dup_rows_per_dst=[0, 4, 6, 862, 0, 0, 6, 0] combine_rows_per_src=[8192, 8176, 8192, 8186, 8192, 8050, 7478, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64572 payload_rows_remote=56380 dup_rows=964 dup_rows_remote=964 dedup_ratio=0.014709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8188, 8192, 8192, 7238] dup_rows_per_dst=[0, 84, 166, 0, 710, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8186, 8192, 8188, 8192, 8192, 7238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=231284736 dispatch_meta_bytes=229376 combine_payload_bytes=231284736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231514112 ingress_bytes=250159104 link_bytes=250159104 token_bytes=462798848 weight_movement_bytes=18874368 total_nvlink_bytes=481673216 weight_to_token_ratio=0.040783 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.396901449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65218 payload_rows_remote=57026 dup_rows=318 dup_rows_remote=236 dedup_ratio=0.004852 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8164, 8188, 8152, 7948, 8190] dup_rows_per_dst=[0, 12, 82, 216, 0, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8164, 8188, 8152, 7948, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 8, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8172, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=230932480 dispatch_meta_bytes=229376 combine_payload_bytes=230932480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231161856 ingress_bytes=249806848 link_bytes=249806848 token_bytes=462094336 weight_movement_bytes=18874368 total_nvlink_bytes=480968704 weight_to_token_ratio=0.040845 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393526305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59194 payload_rows_remote=51002 dup_rows=6342 dup_rows_remote=6336 dedup_ratio=0.096771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8172, 7180, 2978, 8190, 8190, 8102, 8192] dup_rows_per_dst=[0, 6, 4, 78, 6, 3098, 3144, 6] combine_rows_per_src=[8190, 8172, 7180, 2978, 8190, 8190, 8102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64148 payload_rows_remote=55970 dup_rows=1388 dup_rows_remote=1386 dedup_ratio=0.021179 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8192, 8192, 7452, 7558, 8192] dup_rows_per_dst=[0, 2, 152, 1232, 0, 0, 2, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 8192, 7452, 7558, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64136 payload_rows_remote=55944 dup_rows=1400 dup_rows_remote=1400 dedup_ratio=0.021362 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 787, 0, 0, 3396, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6792, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[1392, 0, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[8192, 6792, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=233578496 dispatch_meta_bytes=229048 combine_payload_bytes=233578496 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233807544 ingress_bytes=290201600 link_bytes=290201600 token_bytes=467386040 weight_movement_bytes=56623104 total_nvlink_bytes=524009144 weight_to_token_ratio=0.121148 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.780568943e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61130 payload_rows_remote=52938 dup_rows=4406 dup_rows_remote=692 dedup_ratio=0.067230 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4062, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 4094] payload_rows_per_dst=[8192, 8188, 8128, 8188, 4322, 7728, 8192, 8192] dup_rows_per_dst=[4, 246, 4, 0, 0, 0, 3714, 438] combine_rows_per_src=[8192, 8188, 8128, 8188, 4322, 7728, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229376 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235028480 ingress_bytes=253673472 link_bytes=253673472 token_bytes=469827584 weight_movement_bytes=18874368 total_nvlink_bytes=488701952 weight_to_token_ratio=0.040173 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430574394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=208904192 dispatch_meta_bytes=229352 combine_payload_bytes=208904192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209133544 ingress_bytes=227778560 link_bytes=227778560 token_bytes=418037736 weight_movement_bytes=18874368 total_nvlink_bytes=436912104 weight_to_token_ratio=0.045150 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182462088e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=229253120 dispatch_meta_bytes=229424 combine_payload_bytes=229253120 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229482544 ingress_bytes=267001856 link_bytes=267001856 token_bytes=458735664 weight_movement_bytes=37748736 total_nvlink_bytes=496484400 weight_to_token_ratio=0.082289 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.558280411e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63678 payload_rows_remote=55486 dup_rows=1858 dup_rows_remote=40 dedup_ratio=0.028351 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7144, 8192, 8192, 8192, 7382, 8192, 8192] dup_rows_per_dst=[0, 38, 0, 0, 1818, 0, 0, 2] combine_rows_per_src=[8192, 7144, 8192, 8192, 8192, 7382, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=229146624 dispatch_meta_bytes=229376 combine_payload_bytes=229146624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229376000 ingress_bytes=248020992 link_bytes=248020992 token_bytes=458522624 weight_movement_bytes=18874368 total_nvlink_bytes=477396992 weight_to_token_ratio=0.041163 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376415111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=216834048 dispatch_meta_bytes=214520 combine_payload_bytes=216834048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217048568 ingress_bytes=235708416 link_bytes=235708416 token_bytes=433882616 weight_movement_bytes=18874368 total_nvlink_bytes=452756984 weight_to_token_ratio=0.043501 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.258442066e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65392 payload_rows_remote=57200 dup_rows=144 dup_rows_remote=144 dedup_ratio=0.002197 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8108, 8132, 8192] dup_rows_per_dst=[0, 80, 0, 62, 0, 0, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8108, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=227270656 dispatch_meta_bytes=222104 combine_payload_bytes=227270656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=227492760 ingress_bytes=283893760 link_bytes=283893760 token_bytes=454763416 weight_movement_bytes=56623104 total_nvlink_bytes=511386520 weight_to_token_ratio=0.124511 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.720130324e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63398 payload_rows_remote=55210 dup_rows=2138 dup_rows_remote=2138 dedup_ratio=0.032623 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8120, 8188, 8192, 8192, 6134, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 10, 2120, 0, 8, 0] combine_rows_per_src=[8188, 8120, 8188, 8192, 8192, 6134, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57078 dup_rows=266 dup_rows_remote=42 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8142, 8192, 8192, 8192, 8162, 8006, 8192] dup_rows_per_dst=[0, 6, 36, 224, 0, 0, 0, 0] combine_rows_per_src=[8192, 8142, 8192, 8192, 8192, 8162, 8006, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229376 combine_payload_bytes=234291200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234520576 ingress_bytes=253165568 link_bytes=253165568 token_bytes=468811776 weight_movement_bytes=18874368 total_nvlink_bytes=487686144 weight_to_token_ratio=0.040260 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425707907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57740 payload_rows_remote=49548 dup_rows=7796 dup_rows_remote=7768 dedup_ratio=0.118958 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8186, 8164, 8192, 430, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6, 0, 7754, 8, 28] combine_rows_per_src=[8192, 8186, 8164, 8192, 430, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59032 payload_rows_remote=51330 dup_rows=6504 dup_rows_remote=6504 dedup_ratio=0.099243 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7702, 8192, 8192, 8192, 2178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 106, 6398, 0, 0, 0] combine_rows_per_src=[8192, 7702, 8192, 8192, 8192, 2178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=226140160 dispatch_meta_bytes=229392 combine_payload_bytes=226140160 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=226369552 ingress_bytes=282763264 link_bytes=282763264 token_bytes=452509712 weight_movement_bytes=56623104 total_nvlink_bytes=509132816 weight_to_token_ratio=0.125131 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.709298467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=233791488 dispatch_meta_bytes=228480 combine_payload_bytes=233791488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234019968 ingress_bytes=290414592 link_bytes=290414592 token_bytes=467811456 weight_movement_bytes=56623104 total_nvlink_bytes=524434560 weight_to_token_ratio=0.121038 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.782609728e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61852 payload_rows_remote=53662 dup_rows=3684 dup_rows_remote=3598 dedup_ratio=0.056213 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 1352, 2, 0, 3140, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[6388, 6314, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[1048, 394, 0, 2, 2140, 0, 86, 14] combine_rows_per_src=[6388, 6314, 8192, 8192, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65432 payload_rows_remote=57240 dup_rows=104 dup_rows_remote=104 dedup_ratio=0.001587 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 10, 0, 0, 0] payload_rows_per_dst=[8192, 8162, 8138, 8192, 8172, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 104] combine_rows_per_src=[8192, 8162, 8138, 8192, 8172, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=202948608 dispatch_meta_bytes=229264 combine_payload_bytes=202948608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203177872 ingress_bytes=240697344 link_bytes=240697344 token_bytes=406126480 weight_movement_bytes=37748736 total_nvlink_bytes=443875216 weight_to_token_ratio=0.092948 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.306243520e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=210247680 dispatch_meta_bytes=231336 combine_payload_bytes=210247680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210479016 ingress_bytes=247996416 link_bytes=247996416 token_bytes=420726696 weight_movement_bytes=37748736 total_nvlink_bytes=458475432 weight_to_token_ratio=0.089723 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376179636e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63512 payload_rows_remote=55320 dup_rows=2024 dup_rows_remote=2024 dedup_ratio=0.030884 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6168, 8192, 8192] dup_rows_per_dst=[246, 0, 1778, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60348 payload_rows_remote=52156 dup_rows=5188 dup_rows_remote=5184 dedup_ratio=0.079163 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 7914, 3778, 7810, 8154, 8116] dup_rows_per_dst=[4, 4774, 36, 0, 374, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7914, 3778, 7810, 8154, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=219799552 dispatch_meta_bytes=229040 combine_payload_bytes=219799552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220028592 ingress_bytes=238673920 link_bytes=238673920 token_bytes=439828144 weight_movement_bytes=18874368 total_nvlink_bytes=458702512 weight_to_token_ratio=0.042913 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286856067e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229376 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234684416 ingress_bytes=253329408 link_bytes=253329408 token_bytes=469139456 weight_movement_bytes=18874368 total_nvlink_bytes=488013824 weight_to_token_ratio=0.040232 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427277742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=226590720 dispatch_meta_bytes=229376 combine_payload_bytes=226590720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226820096 ingress_bytes=245465088 link_bytes=245465088 token_bytes=453410816 weight_movement_bytes=18874368 total_nvlink_bytes=472285184 weight_to_token_ratio=0.041628 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.351925697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60532 payload_rows_remote=52340 dup_rows=5004 dup_rows_remote=186 dedup_ratio=0.076355 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8192, 3196, 8192, 8192] dup_rows_per_dst=[186, 0, 4818, 0, 0, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8192, 3196, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64442 payload_rows_remote=56250 dup_rows=1094 dup_rows_remote=1064 dedup_ratio=0.016693 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8144, 8192, 8192, 8192, 7146, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 30, 1062, 0, 0, 0] combine_rows_per_src=[8192, 8144, 8192, 8192, 8192, 7146, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=213630976 dispatch_meta_bytes=229360 combine_payload_bytes=213630976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213860336 ingress_bytes=232505344 link_bytes=232505344 token_bytes=427491312 weight_movement_bytes=18874368 total_nvlink_bytes=446365680 weight_to_token_ratio=0.044151 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227751806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52564 payload_rows_remote=44372 dup_rows=12972 dup_rows_remote=12916 dedup_ratio=0.197937 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[2174, 8172, 1304, 8192, 8146, 8192, 8192, 8192] dup_rows_per_dst=[10, 116, 0, 0, 0, 12790, 0, 56] combine_rows_per_src=[2174, 8172, 1304, 8192, 8146, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63714 payload_rows_remote=55528 dup_rows=1822 dup_rows_remote=1822 dedup_ratio=0.027802 router_tokens_per_expert=[0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 809, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8186, 8192, 8192, 8192, 6380, 8192, 8190] dup_rows_per_dst=[1624, 0, 198, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 8186, 8192, 8192, 8192, 6380, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=214384640 dispatch_meta_bytes=210104 combine_payload_bytes=214384640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214594744 ingress_bytes=233259008 link_bytes=233259008 token_bytes=428979384 weight_movement_bytes=18874368 total_nvlink_bytes=447853752 weight_to_token_ratio=0.043998 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.234973044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=230400000 dispatch_meta_bytes=229256 combine_payload_bytes=230400000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230629256 ingress_bytes=249274368 link_bytes=249274368 token_bytes=461029256 weight_movement_bytes=18874368 total_nvlink_bytes=479903624 weight_to_token_ratio=0.040940 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388424344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64960 payload_rows_remote=56768 dup_rows=576 dup_rows_remote=576 dedup_ratio=0.008789 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 282, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7622, 8188, 8192, 8192] dup_rows_per_dst=[8, 2, 0, 0, 0, 0, 0, 566] combine_rows_per_src=[8192, 8192, 8190, 8192, 7622, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=181747712 dispatch_meta_bytes=229152 combine_payload_bytes=181747712 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=181976864 ingress_bytes=294993920 link_bytes=294993920 token_bytes=363724576 weight_movement_bytes=113246208 total_nvlink_bytes=476970784 weight_to_token_ratio=0.311352 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.826486595e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65342 payload_rows_remote=57174 dup_rows=194 dup_rows_remote=194 dedup_ratio=0.002960 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 8192, 8192, 8168, 8046, 8192] dup_rows_per_dst=[0, 2, 0, 192, 0, 0, 0, 0] combine_rows_per_src=[8192, 8168, 8192, 8192, 8192, 8168, 8046, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61846 payload_rows_remote=53654 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.056305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1452, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5246, 8192, 8192, 8192, 8192, 7448, 8192] dup_rows_per_dst=[0, 324, 0, 0, 0, 992, 2374, 0] combine_rows_per_src=[8192, 5246, 8192, 8192, 8192, 8192, 7448, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=227442688 dispatch_meta_bytes=229400 combine_payload_bytes=227442688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227672088 ingress_bytes=246317056 link_bytes=246317056 token_bytes=455114776 weight_movement_bytes=18874368 total_nvlink_bytes=473989144 weight_to_token_ratio=0.041472 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.360088835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=232521728 dispatch_meta_bytes=229376 combine_payload_bytes=232521728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232751104 ingress_bytes=270270464 link_bytes=270270464 token_bytes=465272832 weight_movement_bytes=37748736 total_nvlink_bytes=503021568 weight_to_token_ratio=0.081132 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589598605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60268 payload_rows_remote=52076 dup_rows=5268 dup_rows_remote=4830 dedup_ratio=0.080383 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 2934, 8192, 8184] dup_rows_per_dst=[438, 0, 4822, 4, 0, 0, 4, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 2934, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=234184704 dispatch_meta_bytes=229472 combine_payload_bytes=234184704 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234414176 ingress_bytes=309682176 link_bytes=309682176 token_bytes=468598880 weight_movement_bytes=75497472 total_nvlink_bytes=544096352 weight_to_token_ratio=0.161113 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.967222237e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59584 payload_rows_remote=51392 dup_rows=5952 dup_rows_remote=5952 dedup_ratio=0.090820 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 1123, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8190, 2246, 8192] dup_rows_per_dst=[0, 5366, 0, 4, 0, 582, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8190, 2246, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=219766784 dispatch_meta_bytes=229376 combine_payload_bytes=219766784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219996160 ingress_bytes=238641152 link_bytes=238641152 token_bytes=439762944 weight_movement_bytes=18874368 total_nvlink_bytes=458637312 weight_to_token_ratio=0.042919 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286542100e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61214 payload_rows_remote=53022 dup_rows=4322 dup_rows_remote=4322 dedup_ratio=0.065948 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 3872, 8192, 8190] dup_rows_per_dst=[818, 0, 3504, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 3872, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=213303296 dispatch_meta_bytes=227624 combine_payload_bytes=213303296 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=213530920 ingress_bytes=269926400 link_bytes=269926400 token_bytes=426834216 weight_movement_bytes=56623104 total_nvlink_bytes=483457320 weight_to_token_ratio=0.132658 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.586301953e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55472 payload_rows_remote=47280 dup_rows=10064 dup_rows_remote=10060 dedup_ratio=0.153564 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 939, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2984, 8180, 8150, 8192, 3390, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 20, 10036, 4] combine_rows_per_src=[8192, 2984, 8180, 8150, 8192, 3390, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=210501632 dispatch_meta_bytes=229376 combine_payload_bytes=210501632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210731008 ingress_bytes=229376000 link_bytes=229376000 token_bytes=421232640 weight_movement_bytes=18874368 total_nvlink_bytes=440107008 weight_to_token_ratio=0.044807 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.197767972e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=217178112 dispatch_meta_bytes=229376 combine_payload_bytes=217178112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=217407488 ingress_bytes=254926848 link_bytes=254926848 token_bytes=434585600 weight_movement_bytes=37748736 total_nvlink_bytes=472334336 weight_to_token_ratio=0.086861 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.442583626e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62944 payload_rows_remote=54752 dup_rows=2592 dup_rows_remote=714 dedup_ratio=0.039551 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2817, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8158, 5634, 8192] dup_rows_per_dst=[0, 1878, 0, 0, 0, 714, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8158, 5634, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61966 payload_rows_remote=55290 dup_rows=3570 dup_rows_remote=3142 dedup_ratio=0.054474 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 366, 0, 0, 0, 2, 0, 0, 1210, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 4094, 0, 4093, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8186, 8190, 6152, 6676, 8188] dup_rows_per_dst=[2, 14, 732, 2388, 2, 4, 428, 0] combine_rows_per_src=[8192, 8190, 8192, 8186, 8190, 6152, 6676, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=193658880 dispatch_meta_bytes=229360 combine_payload_bytes=193658880 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=193888240 ingress_bytes=231407616 link_bytes=231407616 token_bytes=387547120 weight_movement_bytes=37748736 total_nvlink_bytes=425295856 weight_to_token_ratio=0.097404 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217233917e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63298 payload_rows_remote=56150 dup_rows=2238 dup_rows_remote=1530 dedup_ratio=0.034149 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7698, 8192, 8192, 7492, 7148, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 6, 1514, 708, 8, 0] combine_rows_per_src=[8192, 7698, 8192, 8192, 7492, 7148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62812 payload_rows_remote=57254 dup_rows=2724 dup_rows_remote=2724 dedup_ratio=0.041565 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 0, 0, 0, 0, 0, 0, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222] payload_rows_per_dst=[8182, 8112, 8192, 8192, 5558, 8192, 8192, 8192] dup_rows_per_dst=[2, 2278, 0, 0, 0, 0, 0, 444] combine_rows_per_src=[8182, 8112, 8192, 8192, 5558, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=224264192 dispatch_meta_bytes=221864 combine_payload_bytes=224264192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224486056 ingress_bytes=243138560 link_bytes=243138560 token_bytes=448750248 weight_movement_bytes=18874368 total_nvlink_bytes=467624616 weight_to_token_ratio=0.042060 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329634050e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=226467840 dispatch_meta_bytes=233728 combine_payload_bytes=226467840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226701568 ingress_bytes=245342208 link_bytes=245342208 token_bytes=453169408 weight_movement_bytes=18874368 total_nvlink_bytes=472043776 weight_to_token_ratio=0.041650 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.350748321e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57530 payload_rows_remote=49338 dup_rows=8006 dup_rows_remote=8006 dedup_ratio=0.122162 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 95, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 190, 8192] dup_rows_per_dst=[0, 3284, 0, 0, 0, 4722, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=229990400 dispatch_meta_bytes=230720 combine_payload_bytes=229990400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230221120 ingress_bytes=248864768 link_bytes=248864768 token_bytes=460211520 weight_movement_bytes=18874368 total_nvlink_bytes=479085888 weight_to_token_ratio=0.041012 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384499758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=239912 combine_payload_bytes=234512384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234752296 ingress_bytes=253386752 link_bytes=253386752 token_bytes=469264680 weight_movement_bytes=18874368 total_nvlink_bytes=488139048 weight_to_token_ratio=0.040221 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427827184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63722 payload_rows_remote=55530 dup_rows=1814 dup_rows_remote=1814 dedup_ratio=0.027679 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310] payload_rows_per_dst=[8186, 7786, 8192, 8192, 6794, 8188, 8192, 8192] dup_rows_per_dst=[4, 1188, 0, 0, 0, 0, 2, 620] combine_rows_per_src=[8186, 7786, 8192, 8192, 6794, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60458 payload_rows_remote=52266 dup_rows=5078 dup_rows_remote=5078 dedup_ratio=0.077484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1557, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 3114, 8192] dup_rows_per_dst=[0, 4994, 0, 0, 0, 84, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 3114, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=202088448 dispatch_meta_bytes=229376 combine_payload_bytes=202088448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202317824 ingress_bytes=220962816 link_bytes=220962816 token_bytes=404406272 weight_movement_bytes=18874368 total_nvlink_bytes=423280640 weight_to_token_ratio=0.046672 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.117156982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61092 payload_rows_remote=52900 dup_rows=4444 dup_rows_remote=4182 dedup_ratio=0.067810 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4068, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131] payload_rows_per_dst=[8052, 4012, 8192, 8192, 8068, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 4, 80, 4094, 0, 0, 262] combine_rows_per_src=[8052, 4012, 8192, 8192, 8068, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=227450880 dispatch_meta_bytes=229376 combine_payload_bytes=227450880 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=227680256 ingress_bytes=265199616 link_bytes=265199616 token_bytes=455131136 weight_movement_bytes=37748736 total_nvlink_bytes=492879872 weight_to_token_ratio=0.082940 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.541012234e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61224 payload_rows_remote=55690 dup_rows=4312 dup_rows_remote=3892 dedup_ratio=0.065796 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1946] payload_rows_per_dst=[8164, 5534, 8192, 8192, 6566, 8192, 8192, 8192] dup_rows_per_dst=[0, 420, 0, 0, 0, 0, 0, 3892] combine_rows_per_src=[8164, 5534, 8192, 8192, 6566, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=214081536 dispatch_meta_bytes=229376 combine_payload_bytes=214081536 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=214310912 ingress_bytes=270704640 link_bytes=270704640 token_bytes=428392448 weight_movement_bytes=56623104 total_nvlink_bytes=485015552 weight_to_token_ratio=0.132176 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.593758665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62454 payload_rows_remote=54262 dup_rows=3082 dup_rows_remote=3068 dedup_ratio=0.047028 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4094, 0, 14, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094] payload_rows_per_dst=[8192, 5528, 8188, 8192, 7782, 8188, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 120, 0, 2896, 14, 46] combine_rows_per_src=[8192, 5528, 8188, 8192, 7782, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=216678400 dispatch_meta_bytes=228328 combine_payload_bytes=216678400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216906728 ingress_bytes=235552768 link_bytes=235552768 token_bytes=433585128 weight_movement_bytes=18874368 total_nvlink_bytes=452459496 weight_to_token_ratio=0.043531 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.256950724e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64450 payload_rows_remote=57344 dup_rows=1086 dup_rows_remote=1086 dedup_ratio=0.016571 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7106, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 1082, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7106, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=228106240 dispatch_meta_bytes=238328 combine_payload_bytes=228106240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228344568 ingress_bytes=265854976 link_bytes=265854976 token_bytes=456450808 weight_movement_bytes=37748736 total_nvlink_bytes=494199544 weight_to_token_ratio=0.082701 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.547291571e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60244 payload_rows_remote=54410 dup_rows=5292 dup_rows_remote=3120 dedup_ratio=0.080750 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8102, 5348, 8192, 8192, 5834, 8192, 8192] dup_rows_per_dst=[1446, 0, 1650, 0, 24, 2172, 0, 0] combine_rows_per_src=[8192, 8102, 5348, 8192, 8192, 5834, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=222257152 dispatch_meta_bytes=229320 combine_payload_bytes=222257152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222486472 ingress_bytes=241131520 link_bytes=241131520 token_bytes=444743624 weight_movement_bytes=18874368 total_nvlink_bytes=463617992 weight_to_token_ratio=0.042439 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310403581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56842 payload_rows_remote=48812 dup_rows=8694 dup_rows_remote=8692 dedup_ratio=0.132660 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[8030, 8192, 8192, 8112, 6210, 8192, 8190, 1724] dup_rows_per_dst=[2, 8686, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[8030, 8192, 8192, 8112, 6210, 8192, 8190, 1724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65242 payload_rows_remote=57052 dup_rows=294 dup_rows_remote=92 dedup_ratio=0.004486 router_tokens_per_expert=[43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 8192, 7950, 8192, 8192, 8146] dup_rows_per_dst=[86, 0, 202, 0, 0, 0, 4, 2] combine_rows_per_src=[8192, 8188, 8190, 8192, 7950, 8192, 8192, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=233720 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235114744 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469995768 weight_movement_bytes=37748736 total_nvlink_bytes=507744504 weight_to_token_ratio=0.080317 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=222863360 dispatch_meta_bytes=230120 combine_payload_bytes=222863360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=223093480 ingress_bytes=260612096 link_bytes=260612096 token_bytes=445956840 weight_movement_bytes=37748736 total_nvlink_bytes=483705576 weight_to_token_ratio=0.084647 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19840 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.497056875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63486 payload_rows_remote=55294 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.031281 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222] payload_rows_per_dst=[8182, 7938, 8192, 8192, 6406, 8192, 8192, 8192] dup_rows_per_dst=[4, 1602, 0, 0, 0, 0, 0, 444] combine_rows_per_src=[8182, 7938, 8192, 8192, 6406, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=199933952 dispatch_meta_bytes=230016 combine_payload_bytes=199933952 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=200163968 ingress_bytes=275431424 link_bytes=275431424 token_bytes=400097920 weight_movement_bytes=75497472 total_nvlink_bytes=475595392 weight_to_token_ratio=0.188697 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.639048384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233684992 dispatch_meta_bytes=228576 combine_payload_bytes=233684992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233913568 ingress_bytes=252559360 link_bytes=252559360 token_bytes=467598560 weight_movement_bytes=18874368 total_nvlink_bytes=486472928 weight_to_token_ratio=0.040364 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419899521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58456 payload_rows_remote=50294 dup_rows=7080 dup_rows_remote=4976 dedup_ratio=0.108032 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0] payload_rows_per_dst=[6088, 8192, 3362, 8192, 8076, 8192, 8192, 8162] dup_rows_per_dst=[10, 0, 132, 0, 4834, 0, 0, 2104] combine_rows_per_src=[6088, 8192, 3362, 8192, 8076, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64936 payload_rows_remote=56744 dup_rows=600 dup_rows_remote=600 dedup_ratio=0.009155 router_tokens_per_expert=[120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7754, 8192, 8192, 8030] dup_rows_per_dst=[240, 0, 360, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7754, 8192, 8192, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=226484224 dispatch_meta_bytes=229376 combine_payload_bytes=226484224 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226713600 ingress_bytes=283107328 link_bytes=283107328 token_bytes=453197824 weight_movement_bytes=56623104 total_nvlink_bytes=509820928 weight_to_token_ratio=0.124941 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.712595119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58392 payload_rows_remote=50204 dup_rows=7144 dup_rows_remote=7144 dedup_ratio=0.109009 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 123, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4036, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 2, 0, 4072, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8190, 1130, 8192, 8192, 8164, 8188, 8148] dup_rows_per_dst=[1260, 4, 24, 0, 22, 5834, 0, 0] combine_rows_per_src=[8188, 8190, 1130, 8192, 8192, 8164, 8188, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=206004224 dispatch_meta_bytes=221080 combine_payload_bytes=206004224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=206225304 ingress_bytes=262627328 link_bytes=262627328 token_bytes=412229528 weight_movement_bytes=56623104 total_nvlink_bytes=468852632 weight_to_token_ratio=0.137358 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.516365836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232423424 dispatch_meta_bytes=229376 combine_payload_bytes=232423424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232652800 ingress_bytes=251297792 link_bytes=251297792 token_bytes=465076224 weight_movement_bytes=18874368 total_nvlink_bytes=483950592 weight_to_token_ratio=0.040583 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407811797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 326, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62040 payload_rows_remote=53848 dup_rows=3496 dup_rows_remote=2264 dedup_ratio=0.053345 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 1136, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4700, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 1232, 2264, 0] combine_rows_per_src=[8192, 4700, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=205635584 dispatch_meta_bytes=229392 combine_payload_bytes=205635584 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205864976 ingress_bytes=243384320 link_bytes=243384320 token_bytes=411500560 weight_movement_bytes=37748736 total_nvlink_bytes=449249296 weight_to_token_ratio=0.091734 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331988802e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63516 payload_rows_remote=55324 dup_rows=2020 dup_rows_remote=1822 dedup_ratio=0.030823 router_tokens_per_expert=[99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7858, 8192, 8188, 6510] dup_rows_per_dst=[198, 0, 152, 0, 1336, 4, 330, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7858, 8192, 8188, 6510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=4 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 62, 4095, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 8190, 8192, 8192, 8164] dup_rows_per_dst=[0, 0, 34, 0, 4, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 8190, 8192, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65134 payload_rows_remote=56942 dup_rows=402 dup_rows_remote=402 dedup_ratio=0.006134 router_tokens_per_expert=[97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7826, 8192, 8192, 8156] dup_rows_per_dst=[194, 0, 208, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7826, 8192, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=220561408 dispatch_meta_bytes=224448 combine_payload_bytes=220561408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220785856 ingress_bytes=258310144 link_bytes=258310144 token_bytes=441347264 weight_movement_bytes=37748736 total_nvlink_bytes=479096000 weight_to_token_ratio=0.085531 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.475000703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=226607104 dispatch_meta_bytes=228584 combine_payload_bytes=226607104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226835688 ingress_bytes=245481472 link_bytes=245481472 token_bytes=453442792 weight_movement_bytes=18874368 total_nvlink_bytes=472317160 weight_to_token_ratio=0.041625 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352082680e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229240 combine_payload_bytes=234725376 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234954616 ingress_bytes=329097216 link_bytes=329097216 token_bytes=469679992 weight_movement_bytes=94371840 total_nvlink_bytes=564051832 weight_to_token_ratio=0.200928 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153247598e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=38 dedup_ratio=0.001007 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 410, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8138, 8190, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 18, 0, 8, 10, 28] combine_rows_per_src=[8192, 8192, 8138, 8190, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 0, 371, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 0, 24, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233234432 dispatch_meta_bytes=229376 combine_payload_bytes=233234432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233463808 ingress_bytes=252108800 link_bytes=252108800 token_bytes=466698240 weight_movement_bytes=18874368 total_nvlink_bytes=485572608 weight_to_token_ratio=0.040442 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415582476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57786 payload_rows_remote=49602 dup_rows=7750 dup_rows_remote=84 dedup_ratio=0.118256 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 20, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 4094, 2686, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3996, 8192, 8188, 8188, 4658, 8184, 8188] dup_rows_per_dst=[4, 0, 12, 42, 0, 26, 7666, 0] combine_rows_per_src=[8192, 3996, 8192, 8188, 8188, 4658, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229264 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234839952 ingress_bytes=272359424 link_bytes=272359424 token_bytes=469450640 weight_movement_bytes=37748736 total_nvlink_bytes=507199376 weight_to_token_ratio=0.080410 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609613992e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794816 weight_movement_bytes=37748736 total_nvlink_bytes=507543552 weight_to_token_ratio=0.080352 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57078 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7926] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 264, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=203169792 dispatch_meta_bytes=198744 combine_payload_bytes=203169792 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=203368536 ingress_bytes=297541632 link_bytes=297541632 token_bytes=406538328 weight_movement_bytes=94371840 total_nvlink_bytes=500910168 weight_to_token_ratio=0.232135 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=15488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.850897518e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61628 payload_rows_remote=53438 dup_rows=3908 dup_rows_remote=3908 dedup_ratio=0.059631 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123] payload_rows_per_dst=[8178, 4838, 8192, 8192, 7654, 8190, 8192, 8192] dup_rows_per_dst=[2, 500, 0, 0, 3160, 0, 0, 246] combine_rows_per_src=[8178, 4838, 8192, 8192, 7654, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65394 payload_rows_remote=57202 dup_rows=142 dup_rows_remote=138 dedup_ratio=0.002167 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 4093, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8186, 8192, 8060] dup_rows_per_dst=[0, 0, 4, 34, 0, 0, 102, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8186, 8192, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 113, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=233791488 dispatch_meta_bytes=229376 combine_payload_bytes=233791488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234020864 ingress_bytes=252665856 link_bytes=252665856 token_bytes=467812352 weight_movement_bytes=18874368 total_nvlink_bytes=486686720 weight_to_token_ratio=0.040346 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420919913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 48, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=218882048 dispatch_meta_bytes=229384 combine_payload_bytes=218882048 prefetch_bytes=0 reduce_bytes=0 egress_bytes=219111432 ingress_bytes=218882048 link_bytes=219111432 token_bytes=437993480 weight_movement_bytes=0 total_nvlink_bytes=437993480 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.099417932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234299392 dispatch_meta_bytes=229360 combine_payload_bytes=234299392 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234528752 ingress_bytes=234299392 link_bytes=234528752 token_bytes=468828144 weight_movement_bytes=0 total_nvlink_bytes=468828144 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247139106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64980 payload_rows_remote=56788 dup_rows=556 dup_rows_remote=556 dedup_ratio=0.008484 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7636] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 530, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58674 payload_rows_remote=50964 dup_rows=6862 dup_rows_remote=482 dedup_ratio=0.104706 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8098, 8192, 8192, 1906, 7710] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 482, 6380] combine_rows_per_src=[8192, 8192, 8192, 8098, 8192, 8192, 1906, 7710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229360 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077616 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925872 weight_movement_bytes=18874368 total_nvlink_bytes=488800240 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61384 payload_rows_remote=53192 dup_rows=4152 dup_rows_remote=4152 dedup_ratio=0.063354 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 3954, 0, 316, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279] payload_rows_per_dst=[7974, 4450, 8188, 8192, 8010, 8190, 8192, 8188] dup_rows_per_dst=[570, 0, 0, 0, 3016, 8, 0, 558] combine_rows_per_src=[7974, 4450, 8188, 8192, 8010, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232603648 dispatch_meta_bytes=229376 combine_payload_bytes=232603648 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232833024 ingress_bytes=270352384 link_bytes=270352384 token_bytes=465436672 weight_movement_bytes=37748736 total_nvlink_bytes=503185408 weight_to_token_ratio=0.081104 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.590383522e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=208748544 dispatch_meta_bytes=205784 combine_payload_bytes=208748544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=208954328 ingress_bytes=246497280 link_bytes=246497280 token_bytes=417702872 weight_movement_bytes=37748736 total_nvlink_bytes=455451608 weight_to_token_ratio=0.090372 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.361815653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62300 payload_rows_remote=56932 dup_rows=3236 dup_rows_remote=3236 dedup_ratio=0.049377 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7962, 8192, 8192, 8192, 5368, 8192, 8010, 8192] dup_rows_per_dst=[0, 3236, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7962, 8192, 8192, 8192, 5368, 8192, 8010, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=217874432 dispatch_meta_bytes=229376 combine_payload_bytes=217874432 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=218103808 ingress_bytes=236748800 link_bytes=236748800 token_bytes=435978240 weight_movement_bytes=18874368 total_nvlink_bytes=454852608 weight_to_token_ratio=0.043292 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.268410514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64640 payload_rows_remote=56452 dup_rows=896 dup_rows_remote=896 dedup_ratio=0.013672 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7716, 8192, 8188, 8192, 7776, 8192, 8192, 8192] dup_rows_per_dst=[0, 896, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7716, 8192, 8188, 8192, 7776, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=233193472 dispatch_meta_bytes=240672 combine_payload_bytes=233193472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233434144 ingress_bytes=252067840 link_bytes=252067840 token_bytes=466627616 weight_movement_bytes=18874368 total_nvlink_bytes=485501984 weight_to_token_ratio=0.040448 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415190018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65002 payload_rows_remote=56810 dup_rows=534 dup_rows_remote=534 dedup_ratio=0.008148 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7666, 8192, 8192, 8184] dup_rows_per_dst=[0, 0, 534, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7666, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61168 payload_rows_remote=52976 dup_rows=4368 dup_rows_remote=4364 dedup_ratio=0.066650 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8188, 8186, 3834] dup_rows_per_dst=[4, 4, 0, 4, 0, 8, 4348, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8188, 8186, 3834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65430 payload_rows_remote=57238 dup_rows=106 dup_rows_remote=94 dedup_ratio=0.001617 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8086] dup_rows_per_dst=[0, 0, 0, 12, 0, 0, 94, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=231227392 dispatch_meta_bytes=229392 combine_payload_bytes=231227392 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231456784 ingress_bytes=268976128 link_bytes=268976128 token_bytes=462684176 weight_movement_bytes=37748736 total_nvlink_bytes=500432912 weight_to_token_ratio=0.081586 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.577196914e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62524 payload_rows_remote=54332 dup_rows=3012 dup_rows_remote=68 dedup_ratio=0.045959 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7858, 8192, 8192, 8192, 5560, 8188, 8150, 8192] dup_rows_per_dst=[0, 2944, 0, 0, 0, 0, 68, 0] combine_rows_per_src=[7858, 8192, 8192, 8192, 5560, 8188, 8150, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57604 payload_rows_remote=49412 dup_rows=7932 dup_rows_remote=7932 dedup_ratio=0.121033 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7998, 1570, 8192, 8192, 7538, 8182, 7740, 8192] dup_rows_per_dst=[0, 0, 0, 0, 5776, 30, 2126, 0] combine_rows_per_src=[7998, 1570, 8192, 8192, 7538, 8182, 7740, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=232693760 dispatch_meta_bytes=229376 combine_payload_bytes=232693760 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232923136 ingress_bytes=270442496 link_bytes=270442496 token_bytes=465616896 weight_movement_bytes=37748736 total_nvlink_bytes=503365632 weight_to_token_ratio=0.081073 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591246931e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=216989696 dispatch_meta_bytes=229360 combine_payload_bytes=216989696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217219056 ingress_bytes=235864064 link_bytes=235864064 token_bytes=434208752 weight_movement_bytes=18874368 total_nvlink_bytes=453083120 weight_to_token_ratio=0.043468 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259933409e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234446848 dispatch_meta_bytes=229328 combine_payload_bytes=234446848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234676176 ingress_bytes=253321216 link_bytes=253321216 token_bytes=469123024 weight_movement_bytes=18874368 total_nvlink_bytes=487997392 weight_to_token_ratio=0.040233 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427199250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61990 payload_rows_remote=54128 dup_rows=3546 dup_rows_remote=3546 dedup_ratio=0.054108 router_tokens_per_expert=[688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1079, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 4094, 4094, 4094, 0, 0, 0, 10, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4013, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 5126, 8188, 7862, 8050] dup_rows_per_dst=[1376, 4, 2162, 0, 4, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 5126, 8188, 7862, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=222543872 dispatch_meta_bytes=217600 combine_payload_bytes=222543872 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=222761472 ingress_bytes=316915712 link_bytes=316915712 token_bytes=445305344 weight_movement_bytes=94371840 total_nvlink_bytes=539677184 weight_to_token_ratio=0.211926 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.036530420e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65382 payload_rows_remote=57344 dup_rows=154 dup_rows_remote=154 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8038, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 152, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8038, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=202391552 dispatch_meta_bytes=229376 combine_payload_bytes=202391552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202620928 ingress_bytes=221265920 link_bytes=221265920 token_bytes=405012480 weight_movement_bytes=18874368 total_nvlink_bytes=423886848 weight_to_token_ratio=0.046602 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.120061176e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=221708288 dispatch_meta_bytes=230696 combine_payload_bytes=221708288 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=221938984 ingress_bytes=316080128 link_bytes=316080128 token_bytes=443647272 weight_movement_bytes=94371840 total_nvlink_bytes=538019112 weight_to_token_ratio=0.212718 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.028524265e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65456 payload_rows_remote=57264 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8120, 8192, 8188, 8192] dup_rows_per_dst=[2, 2, 0, 0, 0, 76, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8120, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229992 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235111016 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469992040 weight_movement_bytes=18874368 total_nvlink_bytes=488866408 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62490 payload_rows_remote=54388 dup_rows=3046 dup_rows_remote=500 dedup_ratio=0.046478 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8102, 6284, 8192, 8188, 7888, 8192, 7452, 8192] dup_rows_per_dst=[2546, 492, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[8102, 6284, 8192, 8188, 7888, 8192, 7452, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64596 payload_rows_remote=56404 dup_rows=940 dup_rows_remote=940 dedup_ratio=0.014343 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7740, 8192, 8192, 8192, 7756, 8184, 8148, 8192] dup_rows_per_dst=[0, 940, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7740, 8192, 8192, 8192, 7756, 8184, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 691, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234553344 dispatch_meta_bytes=229376 combine_payload_bytes=234553344 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234782720 ingress_bytes=272302080 link_bytes=272302080 token_bytes=469336064 weight_movement_bytes=37748736 total_nvlink_bytes=507084800 weight_to_token_ratio=0.080430 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609064550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8166, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8166, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65280 payload_rows_remote=57088 dup_rows=256 dup_rows_remote=238 dedup_ratio=0.003906 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4096] payload_rows_per_dst=[8190, 8190, 8190, 8192, 7942, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 238, 0, 18] combine_rows_per_src=[8190, 8190, 8190, 8192, 7942, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57330 dup_rows=26 dup_rows_remote=24 dedup_ratio=0.000397 router_tokens_per_expert=[2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2716, 0, 1386, 4094, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8186, 8192, 8192, 8188, 8180, 8192] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 2, 16] combine_rows_per_src=[8192, 8188, 8186, 8192, 8192, 8188, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=222773248 dispatch_meta_bytes=219552 combine_payload_bytes=222773248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=222992800 ingress_bytes=260521984 link_bytes=260521984 token_bytes=445766048 weight_movement_bytes=37748736 total_nvlink_bytes=483514784 weight_to_token_ratio=0.084683 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.496193466e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=231030784 dispatch_meta_bytes=229376 combine_payload_bytes=231030784 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=231260160 ingress_bytes=249905152 link_bytes=249905152 token_bytes=462290944 weight_movement_bytes=18874368 total_nvlink_bytes=481165312 weight_to_token_ratio=0.040828 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394468205e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=291487744 link_bytes=291487744 token_bytes=469958656 weight_movement_bytes=56623104 total_nvlink_bytes=526581760 weight_to_token_ratio=0.120485 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792892142e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64110 payload_rows_remote=55918 dup_rows=1426 dup_rows_remote=442 dedup_ratio=0.021759 router_tokens_per_expert=[0, 0, 0, 0, 4090, 0, 0, 3409, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6818, 8180, 8192, 8192, 8192, 8152, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 984, 442, 0, 0] combine_rows_per_src=[6818, 8180, 8192, 8192, 8192, 8152, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233832448 dispatch_meta_bytes=229304 combine_payload_bytes=233832448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234061752 ingress_bytes=290455552 link_bytes=290455552 token_bytes=467894200 weight_movement_bytes=56623104 total_nvlink_bytes=524517304 weight_to_token_ratio=0.121017 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.783002186e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229416 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053096 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876776 weight_movement_bytes=18874368 total_nvlink_bytes=488751144 weight_to_token_ratio=0.040169 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64062 payload_rows_remote=55870 dup_rows=1474 dup_rows_remote=1474 dedup_ratio=0.022491 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 3379, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6758, 8192, 8192, 8192, 8190, 8166, 8184, 8188] dup_rows_per_dst=[0, 2, 0, 4, 548, 920, 0, 0] combine_rows_per_src=[6758, 8192, 8192, 8192, 8190, 8166, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=229040128 dispatch_meta_bytes=225440 combine_payload_bytes=229040128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229265568 ingress_bytes=266788864 link_bytes=266788864 token_bytes=458305696 weight_movement_bytes=37748736 total_nvlink_bytes=496054432 weight_to_token_ratio=0.082366 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.556239627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64324 payload_rows_remote=56132 dup_rows=1212 dup_rows_remote=1212 dedup_ratio=0.018494 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 7108, 8184, 8074] dup_rows_per_dst=[0, 0, 0, 8, 1118, 84, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 7108, 8184, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=228843520 dispatch_meta_bytes=229376 combine_payload_bytes=228843520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229072896 ingress_bytes=247717888 link_bytes=247717888 token_bytes=457916416 weight_movement_bytes=18874368 total_nvlink_bytes=476790784 weight_to_token_ratio=0.041218 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373510918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65336 payload_rows_remote=57144 dup_rows=200 dup_rows_remote=200 dedup_ratio=0.003052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7992, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 200, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7992, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58916 payload_rows_remote=50724 dup_rows=6620 dup_rows_remote=6620 dedup_ratio=0.101013 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 3228, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8192, 8192, 8192, 1602, 8184, 8192, 8192] dup_rows_per_dst=[6276, 0, 0, 0, 2, 342, 0, 0] combine_rows_per_src=[8170, 8192, 8192, 8192, 1602, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65078 payload_rows_remote=56886 dup_rows=458 dup_rows_remote=458 dedup_ratio=0.006989 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7734] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 456, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63552 payload_rows_remote=55360 dup_rows=1984 dup_rows_remote=1978 dedup_ratio=0.030273 router_tokens_per_expert=[0, 0, 0, 0, 4093, 0, 0, 3165, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6236, 8186, 8192, 8184, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 88, 0, 0, 1850, 32, 8, 6] combine_rows_per_src=[6236, 8186, 8192, 8184, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65500 payload_rows_remote=57308 dup_rows=36 dup_rows_remote=20 dedup_ratio=0.000549 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 4094, 4094, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 2, 0, 0] payload_rows_per_dst=[8188, 8188, 8188, 8192, 8192, 8192, 8192, 8168] dup_rows_per_dst=[0, 0, 0, 14, 4, 2, 16, 0] combine_rows_per_src=[8188, 8188, 8188, 8192, 8192, 8192, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=229916672 dispatch_meta_bytes=229376 combine_payload_bytes=229916672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230146048 ingress_bytes=248791040 link_bytes=248791040 token_bytes=460062720 weight_movement_bytes=18874368 total_nvlink_bytes=478937088 weight_to_token_ratio=0.041026 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.383793332e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63244 payload_rows_remote=57302 dup_rows=2292 dup_rows_remote=2290 dedup_ratio=0.034973 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5942, 8150, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 352, 2, 0, 0, 1938] combine_rows_per_src=[8192, 8192, 8192, 8192, 5942, 8150, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234061824 dispatch_meta_bytes=229376 combine_payload_bytes=234061824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234291200 ingress_bytes=252936192 link_bytes=252936192 token_bytes=468353024 weight_movement_bytes=18874368 total_nvlink_bytes=487227392 weight_to_token_ratio=0.040299 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423510139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=207765504 dispatch_meta_bytes=229376 combine_payload_bytes=207765504 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=207994880 ingress_bytes=245514240 link_bytes=245514240 token_bytes=415760384 weight_movement_bytes=37748736 total_nvlink_bytes=453509120 weight_to_token_ratio=0.090794 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352396647e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=233005056 dispatch_meta_bytes=229376 combine_payload_bytes=233005056 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233234432 ingress_bytes=308502528 link_bytes=308502528 token_bytes=466239488 weight_movement_bytes=75497472 total_nvlink_bytes=541736960 weight_to_token_ratio=0.161929 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.955919431e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=226754560 dispatch_meta_bytes=229352 combine_payload_bytes=226754560 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=226983912 ingress_bytes=264503296 link_bytes=264503296 token_bytes=453738472 weight_movement_bytes=37748736 total_nvlink_bytes=491487208 weight_to_token_ratio=0.083195 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.534340439e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234733568 dispatch_meta_bytes=229312 combine_payload_bytes=234733568 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234962880 ingress_bytes=291356672 link_bytes=291356672 token_bytes=469696448 weight_movement_bytes=56623104 total_nvlink_bytes=526319552 weight_to_token_ratio=0.120553 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791636275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60166 payload_rows_remote=51974 dup_rows=5370 dup_rows_remote=5366 dedup_ratio=0.081940 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1735, 0, 132, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 0, 0, 0, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8008, 8188, 3012] dup_rows_per_dst=[0, 4, 4, 1628, 3732, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8008, 8188, 3012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=238368 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234947360 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469656352 weight_movement_bytes=18874368 total_nvlink_bytes=488530720 weight_to_token_ratio=0.040188 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=212885504 dispatch_meta_bytes=229360 combine_payload_bytes=212885504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213114864 ingress_bytes=231759872 link_bytes=231759872 token_bytes=426000368 weight_movement_bytes=18874368 total_nvlink_bytes=444874736 weight_to_token_ratio=0.044306 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220609061e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57926 payload_rows_remote=49734 dup_rows=7610 dup_rows_remote=1120 dedup_ratio=0.116119 router_tokens_per_expert=[0, 0, 0, 0, 4069, 0, 0, 3245, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8138, 8192, 8192, 5654, 3174, 8192, 8192] dup_rows_per_dst=[6490, 0, 2, 0, 1098, 20, 0, 0] combine_rows_per_src=[8192, 8138, 8192, 8192, 5654, 3174, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64036 payload_rows_remote=55844 dup_rows=1500 dup_rows_remote=1500 dedup_ratio=0.022888 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 3349, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6698, 8192, 8192, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 494, 1006, 0, 0] combine_rows_per_src=[6698, 8192, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58132 payload_rows_remote=49940 dup_rows=7404 dup_rows_remote=7404 dedup_ratio=0.112976 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8120, 8192, 860] dup_rows_per_dst=[0, 0, 0, 4, 7400, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8120, 8192, 860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56408 payload_rows_remote=48216 dup_rows=9128 dup_rows_remote=2212 dedup_ratio=0.139282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 110, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 4224, 3046, 8184, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 252, 0, 1954, 6, 6916] combine_rows_per_src=[8192, 8192, 8186, 4224, 3046, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61370 payload_rows_remote=53220 dup_rows=4166 dup_rows_remote=4154 dedup_ratio=0.063568 router_tokens_per_expert=[0, 126, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1774, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 4438, 8188, 8192, 7832, 8188, 8150, 8192] dup_rows_per_dst=[252, 394, 0, 4, 3504, 0, 12, 0] combine_rows_per_src=[8190, 4438, 8188, 8192, 7832, 8188, 8150, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=203710464 dispatch_meta_bytes=203416 combine_payload_bytes=203710464 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=203913880 ingress_bytes=298082304 link_bytes=298082304 token_bytes=407624344 weight_movement_bytes=94371840 total_nvlink_bytes=501996184 weight_to_token_ratio=0.231517 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.856077971e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62154 payload_rows_remote=53966 dup_rows=3382 dup_rows_remote=3382 dedup_ratio=0.051605 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7172, 8192, 8192, 8192, 6538, 8188, 7488, 8192] dup_rows_per_dst=[0, 3382, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7172, 8192, 8192, 8192, 6538, 8188, 7488, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8192, 8192, 8192, 8180, 8192] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8178, 8192, 8192, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=228737024 dispatch_meta_bytes=229376 combine_payload_bytes=228737024 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=228966400 ingress_bytes=247611392 link_bytes=247611392 token_bytes=457703424 weight_movement_bytes=18874368 total_nvlink_bytes=476577792 weight_to_token_ratio=0.041237 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.372490526e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57324 dup_rows=24 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8188, 8192, 8176, 8192] dup_rows_per_dst=[4, 4, 12, 0, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8188, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=204554240 dispatch_meta_bytes=229376 combine_payload_bytes=204554240 prefetch_bytes=0 reduce_bytes=0 egress_bytes=204783616 ingress_bytes=204554240 link_bytes=204783616 token_bytes=409337856 weight_movement_bytes=0 total_nvlink_bytes=409337856 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.962135849e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=197492736 dispatch_meta_bytes=201712 combine_payload_bytes=197492736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=197694448 ingress_bytes=235241472 link_bytes=235241472 token_bytes=395187184 weight_movement_bytes=37748736 total_nvlink_bytes=432935920 weight_to_token_ratio=0.095521 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.253968039e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=217989120 dispatch_meta_bytes=229496 combine_payload_bytes=217989120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218218616 ingress_bytes=274612224 link_bytes=274612224 token_bytes=436207736 weight_movement_bytes=56623104 total_nvlink_bytes=492830840 weight_to_token_ratio=0.129808 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.631199213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=221044736 dispatch_meta_bytes=229392 combine_payload_bytes=221044736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221274128 ingress_bytes=239919104 link_bytes=239919104 token_bytes=442318864 weight_movement_bytes=18874368 total_nvlink_bytes=461193232 weight_to_token_ratio=0.042671 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.298786807e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229344 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235028448 ingress_bytes=329170944 link_bytes=329170944 token_bytes=469827552 weight_movement_bytes=94371840 total_nvlink_bytes=564199392 weight_to_token_ratio=0.200865 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153954023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61120 payload_rows_remote=52928 dup_rows=4416 dup_rows_remote=4416 dedup_ratio=0.067383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8136, 8192, 3832] dup_rows_per_dst=[0, 0, 0, 886, 3530, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8136, 8192, 3832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62162 payload_rows_remote=53970 dup_rows=3374 dup_rows_remote=2824 dedup_ratio=0.051483 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2428, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5328, 8192, 8192, 7682] dup_rows_per_dst=[0, 0, 0, 550, 502, 0, 0, 2322] combine_rows_per_src=[8192, 8192, 8192, 8192, 5328, 8192, 8192, 7682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=4 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 10, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65174 payload_rows_remote=56982 dup_rows=362 dup_rows_remote=362 dedup_ratio=0.005524 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 1, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 7920, 8148, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 8, 44, 0, 308, 0] combine_rows_per_src=[8192, 8154, 7920, 8148, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=104 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 72, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8088, 8188, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 0, 2, 12, 98] combine_rows_per_src=[8192, 8192, 8192, 8188, 8088, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=216793088 dispatch_meta_bytes=229376 combine_payload_bytes=216793088 prefetch_bytes=0 reduce_bytes=0 egress_bytes=217022464 ingress_bytes=216793088 link_bytes=217022464 token_bytes=433815552 weight_movement_bytes=0 total_nvlink_bytes=433815552 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.079402468e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65224 payload_rows_remote=57032 dup_rows=312 dup_rows_remote=2 dedup_ratio=0.004761 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7882, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 310, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 8192, 7882, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63486 payload_rows_remote=55294 dup_rows=2050 dup_rows_remote=1732 dedup_ratio=0.031281 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7766, 8192, 8192, 8192, 8192, 6568] dup_rows_per_dst=[0, 0, 1344, 26, 318, 0, 2, 360] combine_rows_per_src=[8192, 8192, 7766, 8192, 8192, 8192, 8192, 6568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=221061120 dispatch_meta_bytes=227176 combine_payload_bytes=221061120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221288296 ingress_bytes=239935488 link_bytes=239935488 token_bytes=442349416 weight_movement_bytes=18874368 total_nvlink_bytes=461223784 weight_to_token_ratio=0.042668 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.298943790e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229336 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053016 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876696 weight_movement_bytes=18874368 total_nvlink_bytes=488751064 weight_to_token_ratio=0.040169 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62436 payload_rows_remote=54244 dup_rows=3100 dup_rows_remote=16 dedup_ratio=0.047302 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2551, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8188, 8192, 8190, 8192, 8192, 8192, 5102] dup_rows_per_dst=[0, 0, 3084, 0, 0, 16, 0, 0] combine_rows_per_src=[8188, 8188, 8192, 8190, 8192, 8192, 8192, 5102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=233398272 dispatch_meta_bytes=229376 combine_payload_bytes=233398272 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233627648 ingress_bytes=252272640 link_bytes=252272640 token_bytes=467025920 weight_movement_bytes=18874368 total_nvlink_bytes=485900288 weight_to_token_ratio=0.040414 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417152311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=229328 combine_payload_bytes=234405888 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234635216 ingress_bytes=253280256 link_bytes=253280256 token_bytes=469041104 weight_movement_bytes=18874368 total_nvlink_bytes=487915472 weight_to_token_ratio=0.040240 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426806791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=233603072 dispatch_meta_bytes=228136 combine_payload_bytes=233603072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233831208 ingress_bytes=271351808 link_bytes=271351808 token_bytes=467434280 weight_movement_bytes=37748736 total_nvlink_bytes=505183016 weight_to_token_ratio=0.080757 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.599959511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=226484224 dispatch_meta_bytes=228104 combine_payload_bytes=226484224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226712328 ingress_bytes=245358592 link_bytes=245358592 token_bytes=453196552 weight_movement_bytes=18874368 total_nvlink_bytes=472070920 weight_to_token_ratio=0.041647 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.350905305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=222183424 dispatch_meta_bytes=217040 combine_payload_bytes=222183424 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=222400464 ingress_bytes=316555264 link_bytes=316555264 token_bytes=444583888 weight_movement_bytes=94371840 total_nvlink_bytes=538955728 weight_to_token_ratio=0.212270 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.033076785e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65446 payload_rows_remote=57254 dup_rows=90 dup_rows_remote=66 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8114, 8192, 8190, 8190, 8188] dup_rows_per_dst=[24, 0, 60, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8114, 8192, 8190, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62790 payload_rows_remote=54598 dup_rows=2746 dup_rows_remote=2746 dedup_ratio=0.041901 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8184, 8192, 8192, 5456] dup_rows_per_dst=[0, 0, 2674, 0, 0, 0, 72, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8184, 8192, 8192, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60742 payload_rows_remote=52550 dup_rows=4794 dup_rows_remote=4 dedup_ratio=0.073151 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8188, 3478, 8192, 8192, 8192, 8118, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 4790] combine_rows_per_src=[8190, 8188, 3478, 8192, 8192, 8192, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59880 payload_rows_remote=51688 dup_rows=5656 dup_rows_remote=5656 dedup_ratio=0.086304 router_tokens_per_expert=[0, 0, 0, 2, 4094, 0, 0, 1297, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 1701, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[2598, 8188, 8190, 8192, 8192, 8138, 8192, 8190] dup_rows_per_dst=[4, 0, 0, 4, 3406, 2242, 0, 0] combine_rows_per_src=[2598, 8188, 8190, 8192, 8192, 8138, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229280 combine_payload_bytes=234512384 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234741664 ingress_bytes=272261120 link_bytes=272261120 token_bytes=469254048 weight_movement_bytes=37748736 total_nvlink_bytes=507002784 weight_to_token_ratio=0.080444 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608672091e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57278 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.001068 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8126, 8192, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 70, 0, 0] combine_rows_per_src=[8126, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57292 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000793 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0] payload_rows_per_dst=[8184, 8192, 8192, 8168, 8176, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 0, 42, 10, 0] combine_rows_per_src=[8184, 8192, 8192, 8168, 8176, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64612 payload_rows_remote=56434 dup_rows=924 dup_rows_remote=748 dedup_ratio=0.014099 router_tokens_per_expert=[0, 0, 0, 0, 4089, 0, 0, 3648, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7296, 8178, 8192, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 748, 176, 0, 0] combine_rows_per_src=[7296, 8178, 8192, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229344 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235077600 ingress_bytes=291471360 link_bytes=291471360 token_bytes=469925856 weight_movement_bytes=56623104 total_nvlink_bytes=526548960 weight_to_token_ratio=0.120494 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792735159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=223633408 dispatch_meta_bytes=229376 combine_payload_bytes=223633408 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=223862784 ingress_bytes=261382144 link_bytes=261382144 token_bytes=447496192 weight_movement_bytes=37748736 total_nvlink_bytes=485244928 weight_to_token_ratio=0.084355 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.504435096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=215244800 dispatch_meta_bytes=210216 combine_payload_bytes=215244800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215455016 ingress_bytes=252993536 link_bytes=252993536 token_bytes=430699816 weight_movement_bytes=37748736 total_nvlink_bytes=468448552 weight_to_token_ratio=0.087645 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424059581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=211714048 dispatch_meta_bytes=229376 combine_payload_bytes=211714048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211943424 ingress_bytes=230588416 link_bytes=230588416 token_bytes=423657472 weight_movement_bytes=18874368 total_nvlink_bytes=442531840 weight_to_token_ratio=0.044551 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.209384746e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229392 combine_payload_bytes=234610688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234840080 ingress_bytes=253485056 link_bytes=253485056 token_bytes=469450768 weight_movement_bytes=18874368 total_nvlink_bytes=488325136 weight_to_token_ratio=0.040205 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229376 combine_payload_bytes=234668032 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234897408 ingress_bytes=253542400 link_bytes=253542400 token_bytes=469565440 weight_movement_bytes=18874368 total_nvlink_bytes=488439808 weight_to_token_ratio=0.040195 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429318526e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=231153664 dispatch_meta_bytes=228728 combine_payload_bytes=231153664 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=231382392 ingress_bytes=268902400 link_bytes=268902400 token_bytes=462536056 weight_movement_bytes=37748736 total_nvlink_bytes=500284792 weight_to_token_ratio=0.081613 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.576490489e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62202 payload_rows_remote=54010 dup_rows=3334 dup_rows_remote=3316 dedup_ratio=0.050873 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2436, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8180, 8192, 4872] dup_rows_per_dst=[18, 12, 3304, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8180, 8192, 4872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64562 payload_rows_remote=56370 dup_rows=974 dup_rows_remote=974 dedup_ratio=0.014862 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7218] dup_rows_per_dst=[0, 0, 974, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 16, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=221224960 dispatch_meta_bytes=229304 combine_payload_bytes=221224960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221454264 ingress_bytes=240099328 link_bytes=240099328 token_bytes=442679224 weight_movement_bytes=18874368 total_nvlink_bytes=461553592 weight_to_token_ratio=0.042637 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300513625e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65418 payload_rows_remote=57226 dup_rows=118 dup_rows_remote=110 dedup_ratio=0.001801 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8082, 8188, 8188, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 64, 44, 2, 0, 8] combine_rows_per_src=[8082, 8188, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62564 payload_rows_remote=54374 dup_rows=2972 dup_rows_remote=2972 dedup_ratio=0.045349 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 2, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 3356, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 6792, 6626, 8190, 8192] dup_rows_per_dst=[0, 0, 2, 8, 86, 0, 0, 2876] combine_rows_per_src=[8192, 8188, 8192, 8192, 6792, 6626, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62988 payload_rows_remote=54796 dup_rows=2548 dup_rows_remote=2548 dedup_ratio=0.038879 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1274, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5644] dup_rows_per_dst=[0, 0, 0, 2548, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65398 payload_rows_remote=57206 dup_rows=138 dup_rows_remote=138 dedup_ratio=0.002106 router_tokens_per_expert=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8054] dup_rows_per_dst=[42, 0, 0, 92, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044864 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860352 weight_movement_bytes=18874368 total_nvlink_bytes=488734720 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=230891520 dispatch_meta_bytes=229376 combine_payload_bytes=230891520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231120896 ingress_bytes=249765888 link_bytes=249765888 token_bytes=462012416 weight_movement_bytes=18874368 total_nvlink_bytes=480886784 weight_to_token_ratio=0.040853 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393133846e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61914 payload_rows_remote=53752 dup_rows=3622 dup_rows_remote=3620 dedup_ratio=0.055267 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2156, 0, 144, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4600, 8162, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 290, 0, 2, 0, 3330] combine_rows_per_src=[8192, 8192, 8192, 8192, 4600, 8162, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234397696 dispatch_meta_bytes=229344 combine_payload_bytes=234397696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234627040 ingress_bytes=272146432 link_bytes=272146432 token_bytes=469024736 weight_movement_bytes=37748736 total_nvlink_bytes=506773472 weight_to_token_ratio=0.080483 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607573207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=222715904 dispatch_meta_bytes=229384 combine_payload_bytes=222715904 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=222945288 ingress_bytes=241590272 link_bytes=241590272 token_bytes=445661192 weight_movement_bytes=18874368 total_nvlink_bytes=464535560 weight_to_token_ratio=0.042351 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.314799116e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=224444416 dispatch_meta_bytes=229376 combine_payload_bytes=224444416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224673792 ingress_bytes=243318784 link_bytes=243318784 token_bytes=449118208 weight_movement_bytes=18874368 total_nvlink_bytes=467992576 weight_to_token_ratio=0.042025 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331360868e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234315776 dispatch_meta_bytes=229376 combine_payload_bytes=234315776 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234545152 ingress_bytes=234315776 link_bytes=234545152 token_bytes=468860928 weight_movement_bytes=0 total_nvlink_bytes=468860928 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.247296243e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=220168192 dispatch_meta_bytes=229488 combine_payload_bytes=220168192 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=220397680 ingress_bytes=257916928 link_bytes=257916928 token_bytes=440565872 weight_movement_bytes=37748736 total_nvlink_bytes=478314608 weight_to_token_ratio=0.085682 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.471233101e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65254 payload_rows_remote=57068 dup_rows=282 dup_rows_remote=6 dedup_ratio=0.004303 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 8192, 7926, 8182, 8192] dup_rows_per_dst=[276, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 8192, 7926, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64716 payload_rows_remote=56524 dup_rows=820 dup_rows_remote=820 dedup_ratio=0.012512 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 409, 4096, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7372] dup_rows_per_dst=[14, 0, 0, 806, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57318 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8178, 8192, 8192, 8178, 8180, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 40, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8178, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=233750528 dispatch_meta_bytes=228296 combine_payload_bytes=233750528 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233978824 ingress_bytes=309248000 link_bytes=309248000 token_bytes=467729352 weight_movement_bytes=75497472 total_nvlink_bytes=543226824 weight_to_token_ratio=0.161413 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.963062176e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58412 payload_rows_remote=50220 dup_rows=7124 dup_rows_remote=32 dedup_ratio=0.108704 router_tokens_per_expert=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 544, 4096, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8100, 8180, 8192, 1172, 8192, 8192, 8192, 8192] dup_rows_per_dst=[12, 0, 0, 20, 0, 0, 0, 7092] combine_rows_per_src=[8100, 8180, 8192, 1172, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65218 payload_rows_remote=57302 dup_rows=318 dup_rows_remote=318 dedup_ratio=0.004852 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4084, 0, 129, 0, 0, 33, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 8186, 8188, 8192, 8188, 7916, 8192] dup_rows_per_dst=[0, 8, 240, 66, 0, 0, 0, 4] combine_rows_per_src=[8164, 8192, 8186, 8188, 8192, 8188, 7916, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=56 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 2, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8144, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 4, 0, 52] combine_rows_per_src=[8180, 8144, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8186, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[8190, 8192, 8192, 8186, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=231522304 dispatch_meta_bytes=229376 combine_payload_bytes=231522304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231751680 ingress_bytes=250396672 link_bytes=250396672 token_bytes=463273984 weight_movement_bytes=18874368 total_nvlink_bytes=482148352 weight_to_token_ratio=0.040741 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399177708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229432 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003960 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778488 weight_movement_bytes=18874368 total_nvlink_bytes=488652856 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=22 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 8180, 8192, 8190, 8192] dup_rows_per_dst=[0, 14, 0, 8, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 8180, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=205701120 dispatch_meta_bytes=201008 combine_payload_bytes=205701120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205902128 ingress_bytes=224575488 link_bytes=224575488 token_bytes=411603248 weight_movement_bytes=18874368 total_nvlink_bytes=430477616 weight_to_token_ratio=0.045856 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.151771828e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=230480 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234939472 ingress_bytes=272457728 link_bytes=272457728 token_bytes=469648464 weight_movement_bytes=37748736 total_nvlink_bytes=507397200 weight_to_token_ratio=0.080377 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610555892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229344 combine_payload_bytes=234618880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234848224 ingress_bytes=253493248 link_bytes=253493248 token_bytes=469467104 weight_movement_bytes=18874368 total_nvlink_bytes=488341472 weight_to_token_ratio=0.040204 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428847576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65352 payload_rows_remote=57160 dup_rows=184 dup_rows_remote=184 dedup_ratio=0.002808 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8016] dup_rows_per_dst=[0, 0, 0, 182, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8184, 8192, 8016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229368 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012088 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794808 weight_movement_bytes=18874368 total_nvlink_bytes=488669176 weight_to_token_ratio=0.040176 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21366854424 prefetch_bytes=811597824 reduce_bytes=698351616 weight_movement_bytes=1509949440 egress_bytes=10688934680 ingress_bytes=12187869184 link_bytes=12187869184 total_nvlink_bytes=22876803864 weight_to_token_ratio=0.070668 prefetch_share_of_ingress=0.066591 dedup_ratio=0.053008 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=37 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100405 gemm_inner_pad_upper_bound=32766 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.432878452e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21226904824 prefetch_bytes=717225984 reduce_bytes=962592768 weight_movement_bytes=1679818752 egress_bytes=10618928376 ingress_bytes=12287795200 link_bytes=12288483312 total_nvlink_bytes=22906723576 weight_to_token_ratio=0.079136 prefetch_share_of_ingress=0.058369 dedup_ratio=0.058337 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=51 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100694 gemm_inner_pad_upper_bound=32766 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.452962516e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57336 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 22] combine_rows_per_src=[8192, 8178, 8192, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65212 payload_rows_remote=57020 dup_rows=324 dup_rows_remote=10 dedup_ratio=0.004944 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 157, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7868] dup_rows_per_dst=[10, 0, 0, 314, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=229376 combine_payload_bytes=234127360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234356736 ingress_bytes=253001728 link_bytes=253001728 token_bytes=468484096 weight_movement_bytes=18874368 total_nvlink_bytes=487358464 weight_to_token_ratio=0.040288 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424138073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57282 dup_rows=62 dup_rows_remote=34 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8192, 8192, 8184, 8192, 8162, 8188, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 32, 28] combine_rows_per_src=[8172, 8192, 8192, 8184, 8192, 8162, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62886 payload_rows_remote=54698 dup_rows=2650 dup_rows_remote=2650 dedup_ratio=0.040436 router_tokens_per_expert=[2, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 1319, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8190, 8188, 5894, 8168, 7878, 8188, 8188, 8192] dup_rows_per_dst=[4, 0, 0, 0, 4, 0, 0, 2642] combine_rows_per_src=[8190, 8188, 5894, 8168, 7878, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229432 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077688 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925944 weight_movement_bytes=18874368 total_nvlink_bytes=488800312 weight_to_token_ratio=0.040165 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=233553920 dispatch_meta_bytes=228120 combine_payload_bytes=233553920 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=233782040 ingress_bytes=346800128 link_bytes=346800128 token_bytes=467335960 weight_movement_bytes=113246208 total_nvlink_bytes=580582168 weight_to_token_ratio=0.242323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.322868190e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234627072 dispatch_meta_bytes=229264 combine_payload_bytes=234627072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234856336 ingress_bytes=291250176 link_bytes=291250176 token_bytes=469483408 weight_movement_bytes=56623104 total_nvlink_bytes=526106512 weight_to_token_ratio=0.120607 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790615882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61580 payload_rows_remote=53388 dup_rows=3956 dup_rows_remote=3956 dedup_ratio=0.060364 router_tokens_per_expert=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 4248, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 3952] combine_rows_per_src=[8180, 8192, 4248, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=224043008 dispatch_meta_bytes=229392 combine_payload_bytes=224043008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224272400 ingress_bytes=242917376 link_bytes=242917376 token_bytes=448315408 weight_movement_bytes=18874368 total_nvlink_bytes=467189776 weight_to_token_ratio=0.042101 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.327514774e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20749958368 prefetch_bytes=868220928 reduce_bytes=792723456 weight_movement_bytes=1660944384 egress_bytes=10380442848 ingress_bytes=12030459904 link_bytes=12030689280 total_nvlink_bytes=22410902752 weight_to_token_ratio=0.080046 prefetch_share_of_ingress=0.072169 dedup_ratio=0.077320 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=42 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100550 gemm_inner_pad_upper_bound=32639 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.401503025e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19931346480 prefetch_bytes=811597824 reduce_bytes=981467136 weight_movement_bytes=1793064960 egress_bytes=9970964016 ingress_bytes=11753447424 link_bytes=11753676800 total_nvlink_bytes=21724411440 weight_to_token_ratio=0.089962 prefetch_share_of_ingress=0.069052 dedup_ratio=0.104142 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=52 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100550 gemm_inner_pad_upper_bound=33147 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.346207248e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65348 payload_rows_remote=57156 dup_rows=188 dup_rows_remote=154 dedup_ratio=0.002869 router_tokens_per_expert=[0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8192, 8188, 8192, 8192, 8038] dup_rows_per_dst=[34, 72, 0, 82, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8162, 8192, 8188, 8192, 8192, 8038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=218677248 dispatch_meta_bytes=229376 combine_payload_bytes=218677248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218906624 ingress_bytes=237551616 link_bytes=237551616 token_bytes=437583872 weight_movement_bytes=18874368 total_nvlink_bytes=456458240 weight_to_token_ratio=0.043133 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276102702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65504 payload_rows_remote=57328 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8176, 8192, 8192, 8182, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 26] combine_rows_per_src=[8192, 8186, 8192, 8176, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234110976 dispatch_meta_bytes=229240 combine_payload_bytes=234110976 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234340216 ingress_bytes=309608448 link_bytes=309608448 token_bytes=468451192 weight_movement_bytes=75497472 total_nvlink_bytes=543948664 weight_to_token_ratio=0.161164 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.966515812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0] payload_rows_per_dst=[8150, 8192, 8192, 8172, 8190, 8192, 8192, 8188] dup_rows_per_dst=[0, 4, 4, 0, 0, 60, 0, 0] combine_rows_per_src=[8150, 8192, 8192, 8172, 8190, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229440 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235044928 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860416 weight_movement_bytes=18874368 total_nvlink_bytes=488734784 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20076646920 prefetch_bytes=962592768 reduce_bytes=717225984 weight_movement_bytes=1679818752 egress_bytes=10043683336 ingress_bytes=11712782336 link_bytes=11713011712 total_nvlink_bytes=21756465672 weight_to_token_ratio=0.083670 prefetch_share_of_ingress=0.082183 dedup_ratio=0.097865 prefetch_slots=102 unused_prefetch_slots=1434 slot_utilization=0.066406 reduce_slots=38 remote_demand=51 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100260 gemm_inner_pad_upper_bound=33655 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.338089897e-04 prefetch_bytes_per_gemm_flop=1.921477141e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=0 dedup_ratio=0.000458 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8162, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 30, 0, 0] combine_rows_per_src=[8162, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229376 combine_payload_bytes=234602496 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234831872 ingress_bytes=253476864 link_bytes=253476864 token_bytes=469434368 weight_movement_bytes=18874368 total_nvlink_bytes=488308736 weight_to_token_ratio=0.040207 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428690592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21356774920 prefetch_bytes=792723456 reduce_bytes=792723456 weight_movement_bytes=1585446912 egress_bytes=10683893256 ingress_bytes=12258328576 link_bytes=12258328576 total_nvlink_bytes=22942221832 weight_to_token_ratio=0.074236 prefetch_share_of_ingress=0.064668 dedup_ratio=0.050945 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=42 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=32512 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.446943186e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=229256 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234987400 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469745544 weight_movement_bytes=18874368 total_nvlink_bytes=488619912 weight_to_token_ratio=0.040180 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62172 payload_rows_remote=53984 dup_rows=3364 dup_rows_remote=3360 dedup_ratio=0.051331 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2993, 4094, 0, 0, 3820, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6714, 8190, 8192, 8188, 6312] dup_rows_per_dst=[6, 12, 0, 1870, 0, 4, 4, 1468] combine_rows_per_src=[8192, 8192, 8192, 6714, 8190, 8192, 8188, 6312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62862 payload_rows_remote=54670 dup_rows=2674 dup_rows_remote=2674 dedup_ratio=0.040802 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1348, 4096, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5518] dup_rows_per_dst=[8, 0, 0, 2666, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=221118464 dispatch_meta_bytes=229376 combine_payload_bytes=221118464 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=221347840 ingress_bytes=258867200 link_bytes=258867200 token_bytes=442466304 weight_movement_bytes=37748736 total_nvlink_bytes=480215040 weight_to_token_ratio=0.085314 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.480338140e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=223928320 dispatch_meta_bytes=229376 combine_payload_bytes=223928320 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=224157696 ingress_bytes=261677056 link_bytes=261677056 token_bytes=448086016 weight_movement_bytes=37748736 total_nvlink_bytes=485834752 weight_to_token_ratio=0.084244 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.507260797e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57292 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000854 router_tokens_per_expert=[0, 2, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8148, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 0, 22, 0, 4, 22, 0, 8] combine_rows_per_src=[8192, 8188, 8192, 8148, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229392 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234897424 ingress_bytes=253542400 link_bytes=253542400 token_bytes=469565456 weight_movement_bytes=18874368 total_nvlink_bytes=488439824 weight_to_token_ratio=0.040195 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429318526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65526 payload_rows_remote=57334 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 10] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20609169688 prefetch_bytes=811597824 reduce_bytes=849346560 weight_movement_bytes=1660944384 egress_bytes=10310031640 ingress_bytes=11960082432 link_bytes=11960541168 total_nvlink_bytes=22270114072 weight_to_token_ratio=0.080592 prefetch_share_of_ingress=0.067859 dedup_ratio=0.077949 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=45 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101707 gemm_inner_pad_upper_bound=30988 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.387500427e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234840064 dispatch_meta_bytes=229376 combine_payload_bytes=234840064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235069440 ingress_bytes=272588800 link_bytes=272588800 token_bytes=469909504 weight_movement_bytes=37748736 total_nvlink_bytes=507658240 weight_to_token_ratio=0.080332 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611811760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21212125000 prefetch_bytes=868220928 reduce_bytes=849346560 weight_movement_bytes=1717567488 egress_bytes=10611586888 ingress_bytes=12318105600 link_bytes=12318334984 total_nvlink_bytes=22929692488 weight_to_token_ratio=0.080971 prefetch_share_of_ingress=0.070483 dedup_ratio=0.062041 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=45 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=32512 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.458921350e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none + [2026-08-29 23:37:48] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 772.1 | throughput per GPU (TFLOP/s/GPU): 115.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.765029E+00 | loss scale: 1.0 | grad norm: 3.055 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44868 payload_rows_remote=38964 dup_rows=20668 dup_rows_remote=18042 dedup_ratio=0.315369 router_tokens_per_expert=[137, 0, 0, 0, 169, 199, 77, 2, 0, 0, 299, 336, 650, 410, 253, 376, 15, 237, 0, 501, 230, 346, 0, 0, 451, 121, 126, 179, 600, 1, 424, 3, 292, 433, 8, 75, 6, 46, 260, 172, 348, 530, 606, 322, 267, 26, 265, 258, 687, 491, 547, 0, 348, 451, 23, 18, 694, 137, 372, 1055, 234, 407, 12, 196, 13, 346, 0, 17, 340, 0, 587, 399, 0, 650, 395, 1474, 116, 63, 19, 1227, 167, 14, 95, 218, 12, 286, 0, 0, 589, 335, 106, 158, 264, 477, 128, 189, 0, 120, 131, 160, 0, 340, 498, 50, 216, 20, 313, 413, 540, 255, 403, 0, 376, 595, 201, 79, 249, 1, 851, 105, 749, 200, 672, 28, 355, 0, 436, 0] payload_rows_per_dst=[5562, 5684, 5120, 5354, 5276, 5904, 6230, 5738] dup_rows_per_dst=[2364, 2486, 2708, 2506, 1988, 2626, 3636, 2354] combine_rows_per_src=[5562, 5684, 5120, 5354, 5276, 5904, 6230, 5738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44858 payload_rows_remote=39286 dup_rows=20678 dup_rows_remote=18492 dedup_ratio=0.315521 router_tokens_per_expert=[143, 0, 0, 0, 55, 203, 81, 9, 1, 1, 359, 400, 640, 498, 209, 330, 32, 222, 11, 606, 171, 434, 0, 0, 357, 85, 121, 206, 515, 1, 643, 12, 264, 387, 12, 88, 3, 46, 282, 202, 345, 563, 520, 376, 311, 41, 256, 512, 705, 466, 448, 0, 451, 514, 37, 34, 705, 164, 354, 667, 183, 274, 49, 179, 25, 315, 3, 20, 391, 0, 633, 432, 1, 843, 360, 973, 99, 92, 38, 978, 173, 14, 80, 223, 22, 336, 0, 1, 633, 431, 101, 231, 248, 442, 202, 195, 1, 117, 174, 90, 1, 344, 512, 73, 139, 34, 421, 359, 716, 237, 470, 0, 335, 541, 325, 140, 329, 7, 935, 111, 678, 110, 447, 23, 336, 4, 441, 0] payload_rows_per_dst=[5758, 6066, 5366, 5572, 5526, 5860, 5066, 5644] dup_rows_per_dst=[2802, 2636, 3050, 2186, 2924, 2760, 2310, 2010] combine_rows_per_src=[5758, 6066, 5366, 5572, 5526, 5860, 5066, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44208 payload_rows_remote=38916 dup_rows=21328 dup_rows_remote=18272 dedup_ratio=0.325439 router_tokens_per_expert=[359, 0, 1, 0, 122, 262, 114, 2, 1, 2, 301, 469, 724, 480, 199, 303, 23, 247, 0, 532, 287, 416, 2, 0, 388, 165, 95, 165, 419, 13, 456, 21, 386, 409, 12, 65, 4, 90, 175, 214, 347, 566, 483, 338, 338, 29, 246, 472, 798, 580, 481, 0, 480, 453, 26, 27, 484, 166, 349, 997, 241, 251, 16, 213, 95, 283, 1, 32, 391, 0, 552, 412, 2, 862, 340, 1060, 88, 41, 38, 1217, 124, 23, 116, 247, 40, 356, 8, 0, 538, 323, 84, 259, 357, 301, 134, 256, 0, 143, 151, 67, 0, 370, 607, 57, 186, 37, 238, 315, 465, 257, 381, 0, 440, 339, 283, 172, 343, 0, 775, 111, 877, 78, 530, 42, 363, 2, 254, 1] payload_rows_per_dst=[5908, 5428, 5292, 5684, 5616, 5988, 4720, 5572] dup_rows_per_dst=[3732, 2580, 3056, 2478, 2778, 2778, 1828, 2098] combine_rows_per_src=[5908, 5428, 5292, 5684, 5616, 5988, 4720, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=38676 dup_rows=21104 dup_rows_remote=18128 dedup_ratio=0.322021 router_tokens_per_expert=[114, 1, 1, 6, 109, 256, 52, 6, 0, 0, 429, 423, 559, 423, 139, 377, 14, 248, 6, 646, 138, 346, 0, 0, 376, 52, 178, 170, 510, 0, 404, 3, 373, 411, 9, 135, 3, 68, 348, 203, 250, 729, 690, 289, 392, 23, 205, 503, 722, 435, 593, 2, 439, 504, 44, 19, 663, 160, 283, 808, 236, 344, 39, 218, 18, 258, 0, 15, 304, 1, 662, 378, 1, 853, 480, 814, 73, 85, 24, 1100, 186, 35, 75, 219, 38, 358, 0, 4, 548, 358, 123, 223, 301, 526, 169, 127, 1, 167, 147, 81, 6, 317, 552, 65, 116, 11, 443, 368, 604, 296, 392, 2, 425, 406, 229, 150, 190, 7, 875, 120, 767, 257, 617, 17, 350, 7, 301, 0] payload_rows_per_dst=[5756, 5588, 5692, 5736, 6390, 4724, 5050, 5496] dup_rows_per_dst=[2976, 2344, 3570, 2340, 3742, 1856, 2086, 2190] combine_rows_per_src=[5756, 5588, 5692, 5736, 6390, 4724, 5050, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44858 payload_rows_remote=39476 dup_rows=20678 dup_rows_remote=18390 dedup_ratio=0.315521 router_tokens_per_expert=[162, 0, 0, 0, 75, 216, 48, 3, 1, 0, 416, 459, 630, 482, 154, 383, 57, 313, 1, 551, 164, 347, 0, 0, 443, 59, 128, 211, 473, 4, 490, 9, 246, 454, 12, 87, 5, 57, 309, 179, 375, 657, 614, 313, 406, 32, 268, 638, 640, 598, 466, 0, 370, 607, 33, 36, 582, 160, 346, 598, 245, 295, 45, 305, 24, 219, 0, 26, 354, 0, 504, 390, 4, 787, 387, 1144, 78, 100, 34, 1095, 141, 22, 71, 215, 25, 198, 3, 0, 698, 353, 96, 174, 234, 486, 175, 190, 1, 153, 212, 65, 1, 281, 500, 58, 174, 14, 322, 437, 612, 248, 442, 2, 397, 475, 206, 141, 210, 1, 1057, 105, 876, 148, 484, 23, 301, 2, 336, 0] payload_rows_per_dst=[5138, 5950, 5588, 5618, 5382, 5808, 5792, 5582] dup_rows_per_dst=[2116, 2664, 3716, 2672, 2288, 2544, 2850, 1828] combine_rows_per_src=[5138, 5950, 5588, 5618, 5382, 5808, 5792, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44290 payload_rows_remote=37602 dup_rows=21246 dup_rows_remote=16268 dedup_ratio=0.324188 router_tokens_per_expert=[169, 0, 2, 0, 171, 258, 42, 7, 0, 4, 448, 576, 534, 417, 143, 393, 13, 191, 3, 709, 203, 378, 1, 0, 363, 133, 112, 174, 580, 1, 452, 8, 317, 430, 8, 96, 1, 58, 302, 204, 219, 651, 560, 504, 280, 25, 210, 483, 670, 368, 380, 2, 438, 528, 32, 21, 666, 139, 369, 792, 205, 252, 24, 165, 33, 313, 0, 16, 344, 1, 734, 314, 1, 757, 388, 834, 81, 84, 18, 1058, 196, 13, 63, 282, 35, 254, 4, 2, 649, 350, 94, 211, 345, 423, 146, 193, 1, 120, 153, 79, 4, 287, 553, 41, 204, 26, 539, 378, 499, 266, 316, 0, 484, 390, 269, 158, 315, 3, 943, 102, 824, 295, 689, 17, 358, 2, 333, 0] payload_rows_per_dst=[5374, 5020, 4618, 5580, 5416, 5932, 5662, 6688] dup_rows_per_dst=[2538, 1622, 1656, 2718, 2092, 2704, 2938, 4978] combine_rows_per_src=[5374, 5020, 4618, 5580, 5416, 5932, 5662, 6688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44628 payload_rows_remote=38700 dup_rows=20908 dup_rows_remote=18098 dedup_ratio=0.319031 router_tokens_per_expert=[181, 0, 0, 0, 123, 262, 78, 5, 1, 4, 338, 321, 563, 486, 203, 325, 26, 214, 6, 523, 256, 501, 0, 0, 332, 140, 196, 234, 506, 12, 584, 32, 410, 465, 19, 92, 17, 64, 301, 175, 320, 438, 500, 386, 367, 58, 334, 390, 595, 490, 550, 1, 457, 556, 42, 23, 806, 96, 325, 586, 214, 445, 15, 219, 102, 327, 0, 13, 332, 0, 766, 373, 3, 1037, 326, 692, 107, 80, 25, 868, 177, 30, 92, 436, 22, 312, 2, 0, 656, 383, 112, 192, 387, 377, 213, 198, 0, 222, 202, 136, 2, 250, 617, 57, 196, 28, 461, 337, 335, 223, 426, 0, 369, 300, 315, 104, 261, 2, 807, 108, 874, 174, 411, 27, 367, 2, 305, 2] payload_rows_per_dst=[5688, 5928, 5454, 5778, 5946, 5642, 4840, 5352] dup_rows_per_dst=[2876, 2810, 3218, 2278, 3038, 2654, 2144, 1890] combine_rows_per_src=[5688, 5928, 5454, 5778, 5946, 5642, 4840, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=228024 combine_payload_bytes=159596544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159824568 ingress_bytes=178470912 link_bytes=178470912 token_bytes=319421112 weight_movement_bytes=18874368 total_nvlink_bytes=338295480 weight_to_token_ratio=0.059089 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=7424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.710020466e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160915456 dispatch_meta_bytes=231112 combine_payload_bytes=160915456 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=161146568 ingress_bytes=236412928 link_bytes=236412928 token_bytes=322062024 weight_movement_bytes=75497472 total_nvlink_bytes=397559496 weight_to_token_ratio=0.234419 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265192354e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=228752 combine_payload_bytes=159399936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=159628688 ingress_bytes=197148672 link_bytes=197148672 token_bytes=319028624 weight_movement_bytes=37748736 total_nvlink_bytes=356777360 weight_to_token_ratio=0.118324 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=4096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.888981572e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158416896 dispatch_meta_bytes=227216 combine_payload_bytes=158416896 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=158644112 ingress_bytes=196165632 link_bytes=196165632 token_bytes=317061008 weight_movement_bytes=37748736 total_nvlink_bytes=354809744 weight_to_token_ratio=0.119058 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.072917 gemm_active_groups=19 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.879562566e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161693696 dispatch_meta_bytes=231464 combine_payload_bytes=161693696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=161925160 ingress_bytes=218316800 link_bytes=218316800 token_bytes=323618856 weight_movement_bytes=56623104 total_nvlink_bytes=380241960 weight_to_token_ratio=0.174968 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.091804159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=154017792 dispatch_meta_bytes=215480 combine_payload_bytes=154017792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=154233272 ingress_bytes=191766528 link_bytes=191766528 token_bytes=308251064 weight_movement_bytes=37748736 total_nvlink_bytes=345999800 weight_to_token_ratio=0.122461 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=6528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.837412516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158515200 dispatch_meta_bytes=227192 combine_payload_bytes=158515200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158742392 ingress_bytes=177389568 link_bytes=177389568 token_bytes=317257592 weight_movement_bytes=18874368 total_nvlink_bytes=336131960 weight_to_token_ratio=0.059492 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=6400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.699659559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49584 payload_rows_remote=42566 dup_rows=15952 dup_rows_remote=11772 dedup_ratio=0.243408 router_tokens_per_expert=[0, 43, 0, 0, 1586, 0, 0, 56, 1235, 0, 0, 0, 1045, 1, 0, 3319, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 921, 0, 0, 0, 0, 3562, 80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 846, 0, 2380, 0, 0, 0, 29, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2502, 0, 2292, 2154, 2483, 0, 2, 582, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 26, 0, 0, 0, 0, 0, 70, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 0] payload_rows_per_dst=[5752, 4318, 6136, 7432, 4692, 7018, 6962, 7274] dup_rows_per_dst=[2180, 0, 1614, 1710, 1030, 4180, 3026, 2212] combine_rows_per_src=[5752, 4318, 6136, 7432, 4692, 7018, 6962, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50928 payload_rows_remote=43806 dup_rows=14608 dup_rows_remote=13070 dedup_ratio=0.222900 router_tokens_per_expert=[0, 12, 0, 0, 2114, 0, 0, 45, 820, 0, 0, 0, 583, 0, 0, 3015, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 890, 0, 0, 0, 15, 3291, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1456, 0, 3311, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3014, 0, 1252, 2646, 1608, 0, 38, 421, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 2, 0, 0, 1, 0, 0, 73, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0] payload_rows_per_dst=[5666, 7512, 7498, 7122, 5426, 5398, 5936, 6370] dup_rows_per_dst=[1482, 3816, 3726, 1538, 1316, 1264, 730, 736] combine_rows_per_src=[5666, 7512, 7498, 7122, 5426, 5398, 5936, 6370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49184 payload_rows_remote=41542 dup_rows=16352 dup_rows_remote=8992 dedup_ratio=0.249512 router_tokens_per_expert=[0, 217, 2, 0, 2051, 0, 7, 128, 1445, 0, 1, 0, 1127, 6, 3, 3386, 0, 0, 3, 0, 0, 0, 0, 7, 1, 2, 1, 0, 0, 0, 0, 0, 4, 0, 927, 0, 543, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 46, 602, 0, 0, 0, 0, 3125, 59, 0, 0, 6, 0, 0, 0, 0, 0, 1, 230, 0, 1274, 0, 2736, 4, 0, 0, 14, 1799, 0, 0, 3, 7, 0, 0, 15, 0, 0, 0, 2600, 0, 1335, 2050, 2213, 0, 3, 544, 0, 0, 0, 0, 0, 1, 0, 0, 2737, 20, 0, 0, 7, 7, 1, 57, 0, 160, 3, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1233, 0] payload_rows_per_dst=[7642, 6850, 6466, 6682, 5188, 5046, 5606, 5704] dup_rows_per_dst=[7360, 2478, 1974, 994, 1476, 504, 380, 1186] combine_rows_per_src=[7642, 6850, 6466, 6682, 5188, 5046, 5606, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50482 payload_rows_remote=43426 dup_rows=15054 dup_rows_remote=12542 dedup_ratio=0.229706 router_tokens_per_expert=[0, 51, 0, 0, 933, 0, 0, 41, 998, 0, 0, 0, 977, 0, 1, 3073, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 807, 0, 947, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 35, 1527, 0, 0, 0, 6, 2628, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1354, 0, 3028, 0, 1, 0, 0, 1954, 0, 1, 0, 1, 0, 0, 40, 0, 0, 0, 2610, 0, 2261, 2802, 2120, 0, 7, 499, 2, 0, 0, 0, 0, 0, 0, 0, 2920, 1, 0, 0, 2, 0, 1, 141, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0] payload_rows_per_dst=[4648, 7200, 7056, 6726, 5322, 7004, 6044, 6482] dup_rows_per_dst=[1354, 3644, 2512, 1978, 1304, 2854, 608, 800] combine_rows_per_src=[4648, 7200, 7056, 6726, 5322, 7004, 6044, 6482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51464 payload_rows_remote=47028 dup_rows=14072 dup_rows_remote=13004 dedup_ratio=0.214722 router_tokens_per_expert=[1, 1, 2, 0, 2013, 0, 2, 39, 1013, 0, 1, 0, 193, 2, 0, 3417, 2, 0, 2, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 556, 0, 0, 0, 9, 3424, 542, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 1015, 0, 3776, 0, 0, 0, 2, 1666, 0, 0, 3, 54, 0, 0, 22, 0, 0, 0, 2301, 0, 1244, 2965, 1181, 0, 2, 849, 0, 0, 0, 2, 0, 0, 0, 0, 2905, 0, 0, 0, 0, 1, 1, 17, 0, 554, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0] payload_rows_per_dst=[5376, 6554, 7818, 7254, 4436, 6514, 6122, 7390] dup_rows_per_dst=[1158, 1196, 2456, 1844, 1068, 2878, 840, 2632] combine_rows_per_src=[5376, 6554, 7818, 7254, 4436, 6514, 6122, 7390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=42256 payload_rows_remote=35798 dup_rows=23280 dup_rows_remote=21868 dedup_ratio=0.355225 router_tokens_per_expert=[0, 156, 0, 0, 1171, 0, 2, 56, 1142, 0, 1, 0, 1490, 2, 3, 2907, 14, 0, 0, 0, 0, 0, 5, 2, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 562, 0, 461, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1, 1066, 0, 0, 0, 0, 3697, 29, 0, 0, 2, 0, 0, 0, 0, 0, 0, 197, 0, 1763, 0, 2277, 2, 0, 0, 115, 1810, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2409, 0, 1741, 1549, 2599, 0, 34, 1637, 2, 0, 0, 0, 0, 0, 0, 0, 2244, 104, 0, 0, 0, 3, 0, 279, 2, 187, 3, 2, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 6, 1, 0, 1015, 0] payload_rows_per_dst=[5840, 3150, 1994, 2182, 8126, 7404, 7102, 6458] dup_rows_per_dst=[2206, 2, 62, 14, 11598, 4246, 3740, 1412] combine_rows_per_src=[5840, 3150, 1994, 2182, 8126, 7404, 7102, 6458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174350336 dispatch_meta_bytes=217352 combine_payload_bytes=174350336 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=174567688 ingress_bytes=268722176 link_bytes=268722176 token_bytes=348918024 weight_movement_bytes=94371840 total_nvlink_bytes=443289864 weight_to_token_ratio=0.270470 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.574763671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50236 payload_rows_remote=43002 dup_rows=15300 dup_rows_remote=11738 dedup_ratio=0.233459 router_tokens_per_expert=[0, 68, 0, 0, 1688, 0, 3, 62, 897, 0, 1, 0, 483, 1, 2, 3654, 0, 0, 4, 0, 0, 0, 0, 3, 2, 0, 1, 0, 1, 0, 0, 0, 5, 0, 797, 0, 653, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 270, 1014, 0, 1, 0, 21, 2616, 662, 3, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1079, 4, 3114, 1, 0, 0, 27, 1958, 1, 0, 2, 0, 0, 0, 12, 0, 0, 0, 2654, 0, 1351, 2733, 1653, 0, 58, 467, 0, 0, 1, 0, 0, 5, 0, 0, 3359, 14, 0, 0, 0, 0, 0, 115, 1, 649, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2, 0, 0, 0, 532, 0] payload_rows_per_dst=[5322, 7234, 6802, 7086, 5058, 4340, 6822, 7572] dup_rows_per_dst=[1362, 3562, 2346, 2848, 1194, 1710, 1464, 814] combine_rows_per_src=[5322, 7234, 6802, 7086, 5058, 4340, 6822, 7572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=179429376 dispatch_meta_bytes=227504 combine_payload_bytes=179429376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=179656880 ingress_bytes=217178112 link_bytes=217178112 token_bytes=359086256 weight_movement_bytes=37748736 total_nvlink_bytes=396834992 weight_to_token_ratio=0.105124 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.093750 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.080893811e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=177872896 dispatch_meta_bytes=223872 combine_payload_bytes=177872896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=178096768 ingress_bytes=196747264 link_bytes=196747264 token_bytes=355969664 weight_movement_bytes=18874368 total_nvlink_bytes=374844032 weight_to_token_ratio=0.053022 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.885135478e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=170156032 dispatch_meta_bytes=202136 combine_payload_bytes=170156032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=170358168 ingress_bytes=207904768 link_bytes=207904768 token_bytes=340514200 weight_movement_bytes=37748736 total_nvlink_bytes=378262936 weight_to_token_ratio=0.110858 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.076389 gemm_active_groups=14 gemm_peak_group_rows=15488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.992041191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=192626688 dispatch_meta_bytes=240128 combine_payload_bytes=192626688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=192866816 ingress_bytes=230375424 link_bytes=230375424 token_bytes=385493504 weight_movement_bytes=37748736 total_nvlink_bytes=423242240 weight_to_token_ratio=0.097923 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.072917 gemm_active_groups=16 gemm_peak_group_rows=13312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.207343961e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=146628608 dispatch_meta_bytes=230664 combine_payload_bytes=146628608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=146859272 ingress_bytes=165502976 link_bytes=165502976 token_bytes=293487880 weight_movement_bytes=18874368 total_nvlink_bytes=312362248 weight_to_token_ratio=0.064311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.079861 gemm_active_groups=11 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.585768083e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=176136192 dispatch_meta_bytes=218960 combine_payload_bytes=176136192 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=176355152 ingress_bytes=213884928 link_bytes=213884928 token_bytes=352491344 weight_movement_bytes=37748736 total_nvlink_bytes=390240080 weight_to_token_ratio=0.107091 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.076389 gemm_active_groups=12 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049340142e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45154 payload_rows_remote=38576 dup_rows=20382 dup_rows_remote=16576 dedup_ratio=0.311005 router_tokens_per_expert=[97, 0, 0, 0, 105, 121, 29, 1, 1, 0, 335, 320, 710, 397, 106, 411, 13, 158, 4, 688, 93, 348, 0, 0, 449, 41, 115, 188, 621, 2, 604, 6, 279, 351, 10, 98, 2, 25, 204, 156, 169, 871, 620, 212, 259, 20, 232, 757, 699, 429, 485, 0, 802, 529, 32, 17, 834, 89, 282, 841, 260, 340, 32, 123, 11, 290, 0, 43, 288, 0, 492, 450, 2, 666, 481, 1464, 72, 58, 15, 1229, 100, 2, 101, 182, 9, 150, 0, 0, 549, 313, 125, 228, 174, 435, 137, 144, 0, 45, 228, 63, 0, 326, 400, 62, 138, 21, 360, 493, 821, 403, 368, 0, 301, 572, 161, 155, 246, 3, 1219, 104, 769, 60, 576, 5, 178, 1, 458, 0] payload_rows_per_dst=[4950, 5998, 5084, 6146, 5174, 5486, 6578, 5738] dup_rows_per_dst=[1998, 2732, 2348, 2890, 1960, 2270, 3806, 2378] combine_rows_per_src=[4950, 5998, 5084, 6146, 5174, 5486, 6578, 5738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53336 payload_rows_remote=47640 dup_rows=12200 dup_rows_remote=11236 dedup_ratio=0.186157 router_tokens_per_expert=[0, 0, 0, 673, 841, 0, 4060, 0, 0, 0, 248, 13, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 1846, 0, 9, 0, 0, 3042, 3, 0, 0, 0, 0, 0, 0, 2672, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 157, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 369, 97, 0, 0, 1, 0, 0, 3118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 0, 0, 3707, 0, 0] payload_rows_per_dst=[4020, 5524, 7708, 8152, 8120, 5696, 6586, 7530] dup_rows_per_dst=[830, 1250, 4396, 276, 2962, 964, 584, 938] combine_rows_per_src=[4020, 5524, 7708, 8152, 8120, 5696, 6586, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53120 payload_rows_remote=45372 dup_rows=12416 dup_rows_remote=12022 dedup_ratio=0.189453 router_tokens_per_expert=[0, 0, 0, 490, 302, 0, 4096, 0, 0, 0, 376, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 1412, 24, 0, 0, 1805, 0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 0, 4, 0, 0, 8, 0, 0, 0, 0, 0, 3571, 0, 0, 522, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 489, 95, 0, 0, 0, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 3911, 0, 0] payload_rows_per_dst=[3144, 6104, 6856, 7748, 8192, 7210, 5962, 7904] dup_rows_per_dst=[516, 2356, 3466, 394, 2772, 992, 682, 1238] combine_rows_per_src=[3144, 6104, 6856, 7748, 8192, 7210, 5962, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52230 payload_rows_remote=47226 dup_rows=13306 dup_rows_remote=12934 dedup_ratio=0.203033 router_tokens_per_expert=[0, 0, 0, 273, 670, 1, 4094, 0, 0, 0, 267, 0, 0, 0, 0, 391, 0, 1, 0, 1, 0, 0, 2776, 26, 1, 0, 1631, 0, 0, 0, 0, 3071, 0, 1, 0, 0, 0, 0, 0, 2360, 1, 0, 0, 0, 1, 325, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 11, 0, 0, 6, 0, 0, 0, 0, 0, 3538, 0, 0, 298, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 127, 0, 0, 0, 0, 0, 1745, 0, 0, 0, 0, 1, 0, 0, 0, 0, 556, 0, 0, 0, 0, 3967, 0, 0] payload_rows_per_dst=[3858, 7974, 5004, 7830, 7044, 7116, 5432, 7972] dup_rows_per_dst=[894, 7040, 372, 832, 1700, 568, 824, 1076] combine_rows_per_src=[3858, 7974, 5004, 7830, 7044, 7116, 5432, 7972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49758 payload_rows_remote=41634 dup_rows=15778 dup_rows_remote=9272 dedup_ratio=0.240753 router_tokens_per_expert=[0, 0, 0, 1495, 193, 7, 3883, 0, 0, 0, 331, 5, 0, 0, 1, 702, 0, 0, 0, 0, 0, 0, 1820, 153, 7, 0, 1979, 0, 1, 0, 0, 3171, 0, 1, 0, 1, 6, 0, 0, 2363, 7, 0, 1, 0, 7, 219, 0, 0, 0, 0, 0, 0, 6, 0, 165, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 10, 5, 0, 0, 0, 0, 1784, 1, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 3026, 0, 4, 579, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 143, 198, 0, 0, 11, 0, 0, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 0, 0, 0, 2962, 0, 0] payload_rows_per_dst=[8124, 7886, 4976, 8000, 3598, 6138, 6198, 4838] dup_rows_per_dst=[6506, 7238, 234, 422, 2, 750, 320, 306] combine_rows_per_src=[8124, 7886, 4976, 8000, 3598, 6138, 6198, 4838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53514 payload_rows_remote=45322 dup_rows=12022 dup_rows_remote=6660 dedup_ratio=0.183441 router_tokens_per_expert=[0, 0, 0, 1891, 239, 2, 4091, 0, 0, 0, 366, 1, 0, 0, 0, 1322, 0, 0, 0, 0, 0, 0, 714, 75, 2, 0, 955, 0, 0, 1, 0, 2134, 0, 1, 0, 0, 0, 0, 0, 2551, 2, 0, 0, 0, 2, 124, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2681, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2825, 0, 0, 131, 0, 1, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 1, 0, 190, 32, 0, 0, 2, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 3592, 0, 0] payload_rows_per_dst=[6044, 3094, 7032, 8040, 8192, 5700, 7964, 7448] dup_rows_per_dst=[1598, 400, 2596, 160, 5362, 216, 430, 1260] combine_rows_per_src=[6044, 3094, 7032, 8040, 8192, 5700, 7964, 7448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=47330 payload_rows_remote=40468 dup_rows=18206 dup_rows_remote=17632 dedup_ratio=0.277802 router_tokens_per_expert=[0, 0, 0, 339, 444, 3, 3910, 0, 0, 0, 468, 1, 0, 0, 2, 542, 0, 3, 0, 0, 0, 0, 2783, 67, 3, 0, 2048, 0, 1, 2, 0, 3658, 0, 0, 0, 0, 0, 0, 1, 2805, 3, 0, 1, 0, 3, 625, 0, 0, 0, 0, 0, 0, 2, 0, 322, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 15, 2, 0, 0, 0, 0, 583, 0, 0, 0, 1, 0, 0, 16, 0, 0, 0, 0, 0, 3312, 0, 0, 618, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 1, 0, 440, 327, 0, 0, 19, 0, 0, 1660, 0, 0, 0, 0, 0, 0, 1, 0, 0, 356, 0, 0, 0, 0, 3361, 0, 0] payload_rows_per_dst=[3038, 4230, 8012, 8098, 1202, 7720, 8168, 6862] dup_rows_per_dst=[560, 18, 6180, 588, 2, 5738, 4546, 574] combine_rows_per_src=[3038, 4230, 8012, 8098, 1202, 7720, 8168, 6862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=195133440 dispatch_meta_bytes=235504 combine_payload_bytes=195133440 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=195368944 ingress_bytes=251756544 link_bytes=251756544 token_bytes=390502384 weight_movement_bytes=56623104 total_nvlink_bytes=447125488 weight_to_token_ratio=0.145001 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412207333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50712 payload_rows_remote=43098 dup_rows=14824 dup_rows_remote=10374 dedup_ratio=0.226196 router_tokens_per_expert=[0, 0, 0, 1061, 182, 5, 4057, 0, 0, 0, 378, 1, 0, 0, 4, 809, 0, 0, 0, 0, 0, 0, 1443, 31, 5, 0, 1186, 0, 13, 0, 0, 3354, 0, 0, 0, 0, 0, 0, 0, 1711, 5, 0, 0, 0, 5, 655, 0, 5, 0, 0, 0, 0, 5, 0, 146, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 3412, 0, 0, 1404, 0, 0, 0, 0, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 437, 208, 0, 0, 3, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 3869, 0, 0] payload_rows_per_dst=[8130, 7614, 4176, 6838, 3348, 7028, 5706, 7872] dup_rows_per_dst=[4864, 4450, 586, 256, 2, 2608, 562, 1496] combine_rows_per_src=[8130, 7614, 4176, 6838, 3348, 7028, 5706, 7872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=185843712 dispatch_meta_bytes=229576 combine_payload_bytes=185843712 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=186073288 ingress_bytes=223592448 link_bytes=223592448 token_bytes=371917000 weight_movement_bytes=37748736 total_nvlink_bytes=409665736 weight_to_token_ratio=0.101498 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142352822e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158007296 dispatch_meta_bytes=220608 combine_payload_bytes=158007296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158227904 ingress_bytes=176881664 link_bytes=176881664 token_bytes=316235200 weight_movement_bytes=18874368 total_nvlink_bytes=335109568 weight_to_token_ratio=0.059685 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=4608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694793073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193437696 dispatch_meta_bytes=240640 combine_payload_bytes=193437696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=193678336 ingress_bytes=212312064 link_bytes=212312064 token_bytes=387116032 weight_movement_bytes=18874368 total_nvlink_bytes=405990400 weight_to_token_ratio=0.048756 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.076389 gemm_active_groups=13 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034269733e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=170532864 dispatch_meta_bytes=203624 combine_payload_bytes=170532864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=170736488 ingress_bytes=227155968 link_bytes=227155968 token_bytes=341269352 weight_movement_bytes=56623104 total_nvlink_bytes=397892456 weight_to_token_ratio=0.165919 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.176496718e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=185638912 dispatch_meta_bytes=207928 combine_payload_bytes=185638912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185846840 ingress_bytes=204513280 link_bytes=204513280 token_bytes=371485752 weight_movement_bytes=18874368 total_nvlink_bytes=390360120 weight_to_token_ratio=0.050808 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.959545622e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=165756928 dispatch_meta_bytes=232400 combine_payload_bytes=165756928 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=165989328 ingress_bytes=203505664 link_bytes=203505664 token_bytes=331746256 weight_movement_bytes=37748736 total_nvlink_bytes=369494992 weight_to_token_ratio=0.113788 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.949891141e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=176529408 dispatch_meta_bytes=213888 combine_payload_bytes=176529408 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=176743296 ingress_bytes=233152512 link_bytes=233152512 token_bytes=353272704 weight_movement_bytes=56623104 total_nvlink_bytes=409895808 weight_to_token_ratio=0.160282 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.090278 gemm_active_groups=11 gemm_peak_group_rows=11648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.233952652e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49154 payload_rows_remote=40962 dup_rows=16382 dup_rows_remote=8414 dedup_ratio=0.249969 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 156, 0, 0, 0, 0, 0, 30, 0, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 973, 0, 0, 0, 910, 0, 0, 0, 1, 2100, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 1, 0, 1628, 4095, 0, 29] payload_rows_per_dst=[6430, 3100, 2832, 8192, 4024, 8192, 8192, 8192] dup_rows_per_dst=[6, 0, 124, 1120, 0, 7968, 2658, 4506] combine_rows_per_src=[6430, 3100, 2832, 8192, 4024, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54222 payload_rows_remote=46030 dup_rows=11314 dup_rows_remote=11078 dedup_ratio=0.172638 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 3813, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 557, 0, 0, 0, 0, 0, 208, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1165, 0, 0, 0, 368, 0, 0, 0, 0, 506, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 2860, 4095, 0, 62] payload_rows_per_dst=[7634, 8192, 3692, 8192, 8192, 3790, 8192, 6338] dup_rows_per_dst=[8, 3018, 492, 236, 4132, 288, 2418, 722] combine_rows_per_src=[7634, 8192, 3692, 8192, 8192, 3790, 8192, 6338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50680 payload_rows_remote=43104 dup_rows=14856 dup_rows_remote=11636 dedup_ratio=0.226685 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 2955, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 714, 114, 7, 0, 0, 6, 0, 21, 0, 0, 173, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4086, 0, 0, 1042, 0, 0, 0, 988, 0, 0, 0, 0, 1270, 0, 4052, 0, 0, 0, 0, 0, 0, 2, 1695, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 0, 7, 479, 0, 0, 0, 0, 0, 0, 0, 2429, 4072, 0, 347] payload_rows_per_dst=[7576, 8180, 8118, 8184, 8174, 5392, 3404, 1652] dup_rows_per_dst=[3220, 3720, 1720, 460, 4510, 1208, 4, 14] combine_rows_per_src=[7576, 8180, 8118, 8184, 8174, 5392, 3404, 1652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51832 payload_rows_remote=44206 dup_rows=13704 dup_rows_remote=9222 dedup_ratio=0.209106 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 1, 3722, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 393, 0, 0, 0, 0, 0, 79, 0, 0, 98, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 0, 738, 0, 0, 0, 73, 0, 0, 0, 0, 1598, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 1, 532, 0, 0, 0, 0, 0, 0, 0, 2252, 4095, 0, 35] payload_rows_per_dst=[7790, 8190, 7626, 8192, 8192, 4326, 2978, 4538] dup_rows_per_dst=[496, 4144, 4482, 202, 3302, 492, 316, 270] combine_rows_per_src=[7790, 8190, 7626, 8192, 8192, 4326, 2978, 4538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54978 payload_rows_remote=49774 dup_rows=10558 dup_rows_remote=8806 dedup_ratio=0.161102 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1863, 679, 2, 0, 0, 2, 0, 33, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 1573, 0, 0, 0, 63, 0, 0, 0, 0, 1267, 0, 4090, 0, 0, 0, 0, 0, 0, 1, 378, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 2, 336, 0, 0, 0, 0, 0, 0, 0, 2340, 4092, 0, 201] payload_rows_per_dst=[7822, 8184, 4806, 8190, 5204, 4854, 8192, 7726] dup_rows_per_dst=[0, 1618, 354, 1068, 1752, 958, 750, 4058] combine_rows_per_src=[7822, 8184, 4806, 8190, 5204, 4854, 8192, 7726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54208 payload_rows_remote=46364 dup_rows=11328 dup_rows_remote=9018 dedup_ratio=0.172852 router_tokens_per_expert=[0, 0, 0, 0, 3835, 0, 0, 5, 218, 0, 0, 0, 35, 1, 0, 3591, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 254, 0, 0, 0, 333, 2348, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 181, 0, 4068, 0, 0, 0, 12, 331, 0, 0, 0, 111, 0, 0, 4, 0, 0, 0, 2080, 0, 428, 4003, 74, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 13, 0, 0, 0, 0, 1, 2, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 654, 0] payload_rows_per_dst=[7914, 8018, 5576, 6778, 6082, 4566, 7844, 7430] dup_rows_per_dst=[274, 2, 1500, 3348, 2220, 610, 2310, 1064] combine_rows_per_src=[7914, 8018, 5576, 6778, 6082, 4566, 7844, 7430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48716 payload_rows_remote=40540 dup_rows=16820 dup_rows_remote=14290 dedup_ratio=0.256653 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 3737, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1989, 615, 4, 0, 0, 4, 0, 366, 0, 0, 144, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4088, 0, 0, 941, 0, 0, 0, 431, 0, 0, 0, 0, 554, 0, 4062, 0, 0, 0, 0, 0, 2, 0, 1270, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 5, 816, 0, 0, 0, 0, 0, 0, 0, 1896, 4088, 0, 72] payload_rows_per_dst=[8008, 8176, 8134, 8190, 8190, 3696, 2550, 1772] dup_rows_per_dst=[3282, 2530, 5956, 358, 4520, 156, 4, 14] combine_rows_per_src=[8008, 8176, 8134, 8190, 8190, 3696, 2550, 1772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44046 payload_rows_remote=35854 dup_rows=21490 dup_rows_remote=12454 dedup_ratio=0.327911 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 2577, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1012, 66, 2, 0, 0, 1, 0, 0, 0, 0, 155, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4093, 0, 0, 286, 0, 0, 0, 1038, 0, 0, 0, 1, 1423, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 0, 4092, 3, 1, 0, 0, 0, 3, 369, 0, 0, 0, 0, 0, 0, 0, 2515, 4087, 0, 59] payload_rows_per_dst=[5162, 6400, 2140, 4386, 3088, 6486, 8192, 8192] dup_rows_per_dst=[6, 0, 28, 162, 2, 2812, 9444, 9036] combine_rows_per_src=[5162, 6400, 2140, 4386, 3088, 6486, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=167780352 dispatch_meta_bytes=197504 combine_payload_bytes=167780352 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=167977856 ingress_bytes=224403456 link_bytes=224403456 token_bytes=335758208 weight_movement_bytes=56623104 total_nvlink_bytes=392381312 weight_to_token_ratio=0.168643 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=10112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.150123502e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=188538880 dispatch_meta_bytes=228432 combine_payload_bytes=188538880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=188767312 ingress_bytes=207413248 link_bytes=207413248 token_bytes=377306192 weight_movement_bytes=18874368 total_nvlink_bytes=396180560 weight_to_token_ratio=0.050024 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.987331689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=176553984 dispatch_meta_bytes=218960 combine_payload_bytes=176553984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176772944 ingress_bytes=195428352 link_bytes=195428352 token_bytes=353326928 weight_movement_bytes=18874368 total_nvlink_bytes=372201296 weight_to_token_ratio=0.053419 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872498312e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=181067776 dispatch_meta_bytes=213712 combine_payload_bytes=181067776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=181281488 ingress_bytes=199942144 link_bytes=199942144 token_bytes=362349264 weight_movement_bytes=18874368 total_nvlink_bytes=381223632 weight_to_token_ratio=0.052089 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.915747246e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203874304 dispatch_meta_bytes=234320 combine_payload_bytes=203874304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204108624 ingress_bytes=222748672 link_bytes=222748672 token_bytes=407982928 weight_movement_bytes=18874368 total_nvlink_bytes=426857296 weight_to_token_ratio=0.046263 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.134268176e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=166051840 dispatch_meta_bytes=219320 combine_payload_bytes=166051840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=166271160 ingress_bytes=184926208 link_bytes=184926208 token_bytes=332323000 weight_movement_bytes=18874368 total_nvlink_bytes=351197368 weight_to_token_ratio=0.056795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.771871936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=189906944 dispatch_meta_bytes=221528 combine_payload_bytes=189906944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190128472 ingress_bytes=208781312 link_bytes=208781312 token_bytes=380035416 weight_movement_bytes=18874368 total_nvlink_bytes=398909784 weight_to_token_ratio=0.049665 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.079861 gemm_active_groups=13 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000439805e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=146857984 dispatch_meta_bytes=193232 combine_payload_bytes=146857984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=147051216 ingress_bytes=203481088 link_bytes=203481088 token_bytes=293909200 weight_movement_bytes=56623104 total_nvlink_bytes=350532304 weight_to_token_ratio=0.192655 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.949655666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59410 payload_rows_remote=52182 dup_rows=6126 dup_rows_remote=3076 dedup_ratio=0.093475 router_tokens_per_expert=[4095, 0, 0, 0, 0, 0, 0, 0, 2163, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 1, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 4090, 0, 0, 0, 0, 0, 46, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 733, 3240, 0] payload_rows_per_dst=[4942, 8048, 7736, 7228, 7980, 8192, 8180, 7104] dup_rows_per_dst=[1236, 86, 0, 3050, 356, 200, 0, 1198] combine_rows_per_src=[4942, 8048, 7736, 7228, 7980, 8192, 8180, 7104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59626 payload_rows_remote=51434 dup_rows=5910 dup_rows_remote=4416 dedup_ratio=0.090179 router_tokens_per_expert=[4083, 0, 0, 0, 0, 0, 0, 0, 1716, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 3, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 0, 0, 2, 0, 0, 3970, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 923, 3528, 0] payload_rows_per_dst=[3598, 7986, 7994, 8044, 8094, 8192, 8192, 7526] dup_rows_per_dst=[26, 200, 0, 1912, 642, 1494, 0, 1636] combine_rows_per_src=[3598, 7986, 7994, 8044, 8094, 8192, 8192, 7526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55878 payload_rows_remote=47826 dup_rows=9658 dup_rows_remote=2568 dedup_ratio=0.147369 router_tokens_per_expert=[4039, 0, 0, 0, 7, 0, 0, 0, 1369, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3680, 61, 0, 0, 0, 5, 4057, 0, 4, 7, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2348, 0, 0, 0, 22, 0, 0, 3390, 0, 0, 3497, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 4061, 0, 0, 0, 1, 0, 30, 0, 0, 0, 843, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 1, 0, 0, 350, 792, 3569, 3] payload_rows_per_dst=[8052, 8128, 7516, 4698, 7540, 4088, 8150, 7706] dup_rows_per_dst=[7090, 288, 54, 42, 294, 150, 0, 1740] combine_rows_per_src=[8052, 8128, 7516, 4698, 7540, 4088, 8150, 7706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59930 payload_rows_remote=51840 dup_rows=5606 dup_rows_remote=5418 dedup_ratio=0.085541 router_tokens_per_expert=[4084, 0, 0, 0, 2, 0, 0, 0, 1707, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3969, 18, 0, 0, 0, 0, 4090, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2132, 0, 0, 0, 6, 0, 0, 3944, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 4077, 0, 0, 0, 0, 0, 61, 0, 0, 0, 212, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 263, 420, 3399, 0] payload_rows_per_dst=[3772, 8186, 7978, 8174, 8090, 8192, 8160, 7378] dup_rows_per_dst=[42, 60, 8, 3990, 188, 528, 0, 790] combine_rows_per_src=[3772, 8186, 7978, 8174, 8090, 8192, 8160, 7378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58866 payload_rows_remote=50916 dup_rows=6670 dup_rows_remote=6668 dedup_ratio=0.101776 router_tokens_per_expert=[4090, 0, 0, 0, 1, 0, 0, 0, 2374, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3965, 9, 0, 0, 0, 0, 3985, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 0, 0, 0, 3, 0, 0, 3969, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 4087, 0, 0, 0, 1, 0, 54, 0, 0, 0, 32, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 230, 1150, 2637, 1] payload_rows_per_dst=[8142, 7980, 7950, 3812, 7946, 8192, 8176, 6668] dup_rows_per_dst=[4634, 172, 2, 6, 320, 164, 0, 1372] combine_rows_per_src=[8142, 7980, 7950, 3812, 7946, 8192, 8176, 6668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50924 payload_rows_remote=42738 dup_rows=14612 dup_rows_remote=11226 dedup_ratio=0.222961 router_tokens_per_expert=[0, 0, 0, 2940, 218, 0, 4081, 0, 0, 0, 481, 12, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 178, 0, 1, 0, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3052, 0, 0, 381, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 161, 3, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4070, 0, 0] payload_rows_per_dst=[8074, 358, 3782, 7892, 8108, 6384, 8186, 8140] dup_rows_per_dst=[5336, 0, 160, 10, 4930, 708, 3386, 82] combine_rows_per_src=[8074, 358, 3782, 7892, 8108, 6384, 8186, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58788 payload_rows_remote=50678 dup_rows=6748 dup_rows_remote=6268 dedup_ratio=0.102966 router_tokens_per_expert=[4064, 0, 0, 0, 5, 0, 0, 0, 2472, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 3817, 40, 0, 0, 0, 6, 4047, 0, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 0, 0, 0, 12, 0, 0, 3738, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 4071, 0, 0, 0, 0, 0, 149, 0, 0, 0, 99, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 217, 515, 3109, 0] payload_rows_per_dst=[8012, 8110, 7724, 4164, 7788, 8192, 8154, 6644] dup_rows_per_dst=[4490, 480, 32, 22, 234, 442, 0, 1048] combine_rows_per_src=[8012, 8110, 7724, 4164, 7788, 8192, 8154, 6644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=213737472 dispatch_meta_bytes=221032 combine_payload_bytes=213737472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=213958504 ingress_bytes=270360576 link_bytes=270360576 token_bytes=427695976 weight_movement_bytes=56623104 total_nvlink_bytes=484319080 weight_to_token_ratio=0.132391 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.107639 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.590462014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54816 payload_rows_remote=47592 dup_rows=10720 dup_rows_remote=7578 dedup_ratio=0.163574 router_tokens_per_expert=[3974, 0, 0, 0, 3, 0, 0, 0, 2151, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3823, 59, 0, 0, 0, 0, 3864, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 0, 0, 66, 0, 0, 3842, 0, 0, 3305, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 4072, 0, 0, 0, 2, 0, 29, 0, 0, 0, 532, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 182, 758, 3192, 2] payload_rows_per_dst=[4328, 5168, 7776, 6824, 7202, 8144, 8150, 7224] dup_rows_per_dst=[8, 870, 2594, 2320, 668, 936, 182, 3142] combine_rows_per_src=[4328, 5168, 7776, 6824, 7202, 8144, 8150, 7224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=210673664 dispatch_meta_bytes=223400 combine_payload_bytes=210673664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210897064 ingress_bytes=229548032 link_bytes=229548032 token_bytes=421570728 weight_movement_bytes=18874368 total_nvlink_bytes=440445096 weight_to_token_ratio=0.044772 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.199416298e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=195895296 dispatch_meta_bytes=201576 combine_payload_bytes=195895296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=196096872 ingress_bytes=233644032 link_bytes=233644032 token_bytes=391992168 weight_movement_bytes=37748736 total_nvlink_bytes=429740904 weight_to_token_ratio=0.096300 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238662155e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=212336640 dispatch_meta_bytes=229032 combine_payload_bytes=212336640 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=212565672 ingress_bytes=268959744 link_bytes=268959744 token_bytes=424902312 weight_movement_bytes=56623104 total_nvlink_bytes=481525416 weight_to_token_ratio=0.133261 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.577039931e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=208551936 dispatch_meta_bytes=230336 combine_payload_bytes=208551936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208782272 ingress_bytes=265175040 link_bytes=265175040 token_bytes=417334208 weight_movement_bytes=56623104 total_nvlink_bytes=473957312 weight_to_token_ratio=0.135678 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.540776759e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=207577088 dispatch_meta_bytes=227784 combine_payload_bytes=207577088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207804872 ingress_bytes=226451456 link_bytes=226451456 token_bytes=415381960 weight_movement_bytes=18874368 total_nvlink_bytes=434256328 weight_to_token_ratio=0.045439 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169746430e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175054848 dispatch_meta_bytes=215856 combine_payload_bytes=175054848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175270704 ingress_bytes=193929216 link_bytes=193929216 token_bytes=350325552 weight_movement_bytes=18874368 total_nvlink_bytes=369199920 weight_to_token_ratio=0.053877 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=194936832 dispatch_meta_bytes=220680 combine_payload_bytes=194936832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195157512 ingress_bytes=213811200 link_bytes=213811200 token_bytes=390094344 weight_movement_bytes=18874368 total_nvlink_bytes=408968712 weight_to_token_ratio=0.048384 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048633717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57150 payload_rows_remote=48968 dup_rows=8386 dup_rows_remote=8370 dedup_ratio=0.127960 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 35, 0, 4, 0, 3234, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2507, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3146, 0, 0, 0, 0] payload_rows_per_dst=[7268, 7028, 8192, 8182, 8178, 6296, 6992, 5014] dup_rows_per_dst=[1380, 4, 1018, 16, 5796, 172, 0, 0] combine_rows_per_src=[7268, 7028, 8192, 8182, 8178, 6296, 6992, 5014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55486 payload_rows_remote=47494 dup_rows=10050 dup_rows_remote=4952 dedup_ratio=0.153351 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 3557, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3491, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 7, 0, 0, 7, 0, 5, 0, 0, 0, 0, 3, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 1, 0, 0, 6, 0, 6, 0, 4066, 7, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 3820, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 0, 0] payload_rows_per_dst=[7992, 7162, 8180, 8188, 8146, 8164, 7654, 0] dup_rows_per_dst=[5098, 680, 722, 0, 82, 3466, 2, 0] combine_rows_per_src=[7992, 7162, 8180, 8188, 8146, 8164, 7654, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55688 payload_rows_remote=49578 dup_rows=9848 dup_rows_remote=9848 dedup_ratio=0.150269 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2748, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 0, 0, 0, 0] payload_rows_per_dst=[6062, 6490, 8192, 8192, 5110, 6110, 7354, 8178] dup_rows_per_dst=[1280, 0, 582, 0, 6, 0, 1038, 6942] combine_rows_per_src=[6062, 6490, 8192, 8192, 5110, 6110, 7354, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57870 payload_rows_remote=51194 dup_rows=7666 dup_rows_remote=6792 dedup_ratio=0.116974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2740, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 0, 2, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2828, 0, 0, 0, 1, 0, 2, 0, 3602, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 3741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 0, 0] payload_rows_per_dst=[7094, 5710, 8190, 8190, 6676, 6684, 7486, 7840] dup_rows_per_dst=[2130, 0, 630, 2, 874, 42, 0, 3988] combine_rows_per_src=[7094, 5710, 8190, 8190, 6676, 6684, 7486, 7840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55520 payload_rows_remote=47652 dup_rows=10016 dup_rows_remote=5174 dedup_ratio=0.152832 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 3055, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 5, 0, 0, 5, 0, 144, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3144, 0, 0, 0, 0, 0, 3, 0, 3530, 5, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 383, 0, 3, 0, 0, 0, 0, 0, 3114, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 0, 0] payload_rows_per_dst=[7754, 7868, 8190, 8192, 7070, 5724, 6238, 4484] dup_rows_per_dst=[2436, 4842, 1538, 4, 64, 1128, 4, 0] combine_rows_per_src=[7754, 7868, 8190, 8192, 7070, 5724, 6238, 4484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56860 payload_rows_remote=48668 dup_rows=8676 dup_rows_remote=7534 dedup_ratio=0.132385 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3278, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 1, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3281, 0, 0, 0, 1, 0, 1, 0, 3974, 1, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2592, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2823, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 0, 0] payload_rows_per_dst=[7224, 7586, 8192, 8188, 8104, 6734, 5186, 5646] dup_rows_per_dst=[1274, 2506, 1142, 10, 3742, 2, 0, 0] combine_rows_per_src=[7224, 7586, 8192, 8188, 8104, 6734, 5186, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51166 payload_rows_remote=42974 dup_rows=14370 dup_rows_remote=14320 dedup_ratio=0.219269 router_tokens_per_expert=[0, 1, 0, 0, 0, 5, 0, 4039, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 2804, 0, 0, 0, 0, 0, 1279, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2644, 0, 0, 0, 17, 0, 0, 0, 0, 703, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 12, 4092, 0, 0] payload_rows_per_dst=[8090, 84, 7992, 8174, 2254, 8192, 8192, 8188] dup_rows_per_dst=[6, 0, 7040, 230, 0, 6726, 50, 318] combine_rows_per_src=[8090, 84, 7992, 8174, 2254, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45110 payload_rows_remote=36924 dup_rows=20426 dup_rows_remote=6044 dedup_ratio=0.311676 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 3225, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 3, 0, 0, 3, 0, 17, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1699, 0, 0, 0, 3, 0, 2, 0, 4076, 3, 0, 4, 40, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3471, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2733, 0, 0, 0, 0] payload_rows_per_dst=[2050, 7768, 1914, 8132, 3506, 5472, 8082, 8186] dup_rows_per_dst=[132, 0, 492, 42, 54, 6, 5318, 14382] combine_rows_per_src=[2050, 7768, 1914, 8132, 3506, 5472, 8082, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200572928 dispatch_meta_bytes=229352 combine_payload_bytes=200572928 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=200802280 ingress_bytes=219447296 link_bytes=219447296 token_bytes=401375208 weight_movement_bytes=18874368 total_nvlink_bytes=420249576 weight_to_token_ratio=0.047024 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102636015e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=194535424 dispatch_meta_bytes=209784 combine_payload_bytes=194535424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=194745208 ingress_bytes=232284160 link_bytes=232284160 token_bytes=389280632 weight_movement_bytes=37748736 total_nvlink_bytes=427029368 weight_to_token_ratio=0.096970 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=18048 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225632530e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=203071488 dispatch_meta_bytes=237704 combine_payload_bytes=203071488 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=203309192 ingress_bytes=240820224 link_bytes=240820224 token_bytes=406380680 weight_movement_bytes=37748736 total_nvlink_bytes=444129416 weight_to_token_ratio=0.092890 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307420895e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=209690624 dispatch_meta_bytes=231944 combine_payload_bytes=209690624 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=209922568 ingress_bytes=285188096 link_bytes=285188096 token_bytes=419613192 weight_movement_bytes=75497472 total_nvlink_bytes=495110664 weight_to_token_ratio=0.179922 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.732532015e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=195182592 dispatch_meta_bytes=211304 combine_payload_bytes=195182592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195393896 ingress_bytes=214056960 link_bytes=214056960 token_bytes=390576488 weight_movement_bytes=18874368 total_nvlink_bytes=409450856 weight_to_token_ratio=0.048324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050988468e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=199344128 dispatch_meta_bytes=224808 combine_payload_bytes=199344128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=199568936 ingress_bytes=218218496 link_bytes=218218496 token_bytes=398913064 weight_movement_bytes=18874368 total_nvlink_bytes=417787432 weight_to_token_ratio=0.047314 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.090862258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=176021504 dispatch_meta_bytes=229176 combine_payload_bytes=176021504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=176250680 ingress_bytes=232644608 link_bytes=232644608 token_bytes=352272184 weight_movement_bytes=56623104 total_nvlink_bytes=408895288 weight_to_token_ratio=0.160737 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229086166e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=151240704 dispatch_meta_bytes=171872 combine_payload_bytes=151240704 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=151412576 ingress_bytes=226738176 link_bytes=226738176 token_bytes=302653280 weight_movement_bytes=75497472 total_nvlink_bytes=378150752 weight_to_token_ratio=0.249452 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.172493640e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58778 payload_rows_remote=52070 dup_rows=6758 dup_rows_remote=5416 dedup_ratio=0.103119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 4096, 9, 0, 0, 0, 0, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2384, 0, 0, 0, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2999, 0, 1289, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 3745] payload_rows_per_dst=[7490, 6636, 8192, 6708, 8192, 5998, 7810, 7752] dup_rows_per_dst=[0, 12, 18, 1342, 4, 0, 2260, 3122] combine_rows_per_src=[7490, 6636, 8192, 6708, 8192, 5998, 7810, 7752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49602 payload_rows_remote=41840 dup_rows=15934 dup_rows_remote=12110 dedup_ratio=0.243134 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4038, 4083, 20, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 7, 0, 1409, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 239, 0, 0, 0, 0, 161, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 7, 2515, 0, 2457, 0, 0, 3165, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1923, 0, 0, 0, 0, 0, 2, 7, 0, 2530, 0, 3870] payload_rows_per_dst=[7762, 8182, 7144, 6270, 8192, 7110, 4928, 14] dup_rows_per_dst=[3824, 7324, 30, 1210, 800, 2742, 0, 4] combine_rows_per_src=[7762, 8182, 7144, 6270, 8192, 7110, 4928, 14] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59294 payload_rows_remote=51104 dup_rows=6242 dup_rows_remote=6088 dedup_ratio=0.095245 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3522, 4093, 4, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 2, 0, 2001, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 1, 2834, 0, 450, 0, 0, 3881, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 0, 2, 0, 3189, 0, 4043] payload_rows_per_dst=[8086, 7050, 8190, 6276, 8190, 5674, 7804, 8024] dup_rows_per_dst=[0, 6, 26, 1014, 154, 0, 864, 4178] combine_rows_per_src=[8086, 7050, 8190, 6276, 8190, 5674, 7804, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59722 payload_rows_remote=53212 dup_rows=5814 dup_rows_remote=5814 dedup_ratio=0.088715 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 0, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3255, 0, 697, 0, 0, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 0, 0, 0, 0, 0, 0, 0, 0, 2999, 0, 3905] payload_rows_per_dst=[7810, 7864, 8192, 6800, 8192, 6510, 6488, 7866] dup_rows_per_dst=[0, 48, 36, 1488, 10, 0, 870, 3362] combine_rows_per_src=[7810, 7864, 8192, 6800, 8192, 6510, 6488, 7866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52710 payload_rows_remote=44556 dup_rows=12826 dup_rows_remote=8134 dedup_ratio=0.195709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2614, 4089, 15, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 5, 0, 1587, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 45, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 15, 3361, 0, 1190, 0, 0, 3798, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 5, 0, 3753, 0, 3830] payload_rows_per_dst=[7660, 8154, 8188, 6938, 8192, 7998, 5570, 10] dup_rows_per_dst=[16, 4692, 4, 1184, 144, 6270, 516, 0] combine_rows_per_src=[7660, 8154, 8188, 6938, 8192, 7998, 5570, 10] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57252 payload_rows_remote=49062 dup_rows=8284 dup_rows_remote=8280 dedup_ratio=0.126404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3032, 4094, 5, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2271, 0, 0, 0, 1504, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2795, 0, 1150, 0, 0, 3608, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2682, 0, 0, 0, 0, 0, 1, 1, 0, 3729, 0, 3786] payload_rows_per_dst=[7572, 6066, 8190, 6522, 8190, 7282, 7626, 5804] dup_rows_per_dst=[0, 14, 4, 1034, 10, 2650, 2812, 1760] combine_rows_per_src=[7572, 6066, 8190, 6522, 8190, 7282, 7626, 5804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58670 payload_rows_remote=50478 dup_rows=6866 dup_rows_remote=6866 dedup_ratio=0.104767 router_tokens_per_expert=[4093, 0, 0, 0, 0, 0, 0, 0, 1972, 0, 0, 0, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 4086, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 0, 3349, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 102, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 160, 2010, 0] payload_rows_per_dst=[5924, 8192, 8172, 7576, 8184, 8192, 8192, 4238] dup_rows_per_dst=[1240, 1582, 0, 3674, 10, 238, 0, 122] combine_rows_per_src=[5924, 8192, 8172, 7576, 8184, 8192, 8192, 4238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57092 payload_rows_remote=49404 dup_rows=8444 dup_rows_remote=5624 dedup_ratio=0.128845 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3957, 4086, 57, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2099, 0, 2, 0, 2469, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 0, 1, 2191, 0, 1946, 0, 0, 2947, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2202, 0, 0, 0, 0, 0, 0, 3, 0, 3049, 0, 3588] payload_rows_per_dst=[7176, 7920, 8178, 7144, 6962, 4852, 7172, 7688] dup_rows_per_dst=[0, 116, 12, 2002, 106, 768, 2620, 2820] combine_rows_per_src=[7176, 7920, 8178, 7144, 6962, 4852, 7172, 7688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213278720 dispatch_meta_bytes=229944 combine_payload_bytes=213278720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213508664 ingress_bytes=232153088 link_bytes=232153088 token_bytes=426787384 weight_movement_bytes=18874368 total_nvlink_bytes=445661752 weight_to_token_ratio=0.044224 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224376663e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=171376640 dispatch_meta_bytes=215800 combine_payload_bytes=171376640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=171592440 ingress_bytes=209125376 link_bytes=209125376 token_bytes=342969080 weight_movement_bytes=37748736 total_nvlink_bytes=380717816 weight_to_token_ratio=0.110065 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.003736457e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209321984 dispatch_meta_bytes=228768 combine_payload_bytes=209321984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209550752 ingress_bytes=228196352 link_bytes=228196352 token_bytes=418872736 weight_movement_bytes=18874368 total_nvlink_bytes=437747104 weight_to_token_ratio=0.045060 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.186465165e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=217956352 dispatch_meta_bytes=236104 combine_payload_bytes=217956352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218192456 ingress_bytes=255705088 link_bytes=255705088 token_bytes=436148808 weight_movement_bytes=37748736 total_nvlink_bytes=473897544 weight_to_token_ratio=0.086550 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450040338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=182501376 dispatch_meta_bytes=210760 combine_payload_bytes=182501376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=182712136 ingress_bytes=220250112 link_bytes=220250112 token_bytes=365213512 weight_movement_bytes=37748736 total_nvlink_bytes=402962248 weight_to_token_ratio=0.103361 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.110328203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200957952 dispatch_meta_bytes=229368 combine_payload_bytes=200957952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201187320 ingress_bytes=219832320 link_bytes=219832320 token_bytes=402145272 weight_movement_bytes=18874368 total_nvlink_bytes=421019640 weight_to_token_ratio=0.046934 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.106325126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=206757888 dispatch_meta_bytes=229376 combine_payload_bytes=206757888 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=206987264 ingress_bytes=244506624 link_bytes=244506624 token_bytes=413745152 weight_movement_bytes=37748736 total_nvlink_bytes=451493888 weight_to_token_ratio=0.091237 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.342742166e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=202358784 dispatch_meta_bytes=220112 combine_payload_bytes=202358784 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=202578896 ingress_bytes=296730624 link_bytes=296730624 token_bytes=404937680 weight_movement_bytes=94371840 total_nvlink_bytes=499309520 weight_to_token_ratio=0.233053 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.843126839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55756 payload_rows_remote=49278 dup_rows=9780 dup_rows_remote=7888 dedup_ratio=0.149231 router_tokens_per_expert=[1470, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1584, 2, 0, 0, 19, 0, 0, 0, 0, 0, 0, 2778, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 4096, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 0, 0, 0] payload_rows_per_dst=[7864, 8186, 8192, 6478, 8192, 6490, 5560, 4794] dup_rows_per_dst=[2994, 0, 3208, 1892, 42, 1642, 2, 0] combine_rows_per_src=[7864, 8186, 8192, 6478, 8192, 6490, 5560, 4794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56280 payload_rows_remote=48286 dup_rows=9256 dup_rows_remote=8366 dedup_ratio=0.141235 router_tokens_per_expert=[122, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4047, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 4051, 477, 4, 7, 0, 27, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 7, 0, 4, 5, 0, 0, 1431, 0, 0, 7, 0, 7, 0, 0, 4080, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 3990, 7, 0, 0, 12, 0, 0, 3270, 0, 0, 0, 0, 0] payload_rows_per_dst=[7994, 8186, 8192, 6428, 7936, 2886, 8094, 6564] dup_rows_per_dst=[890, 182, 3686, 4430, 6, 48, 0, 14] combine_rows_per_src=[7994, 8186, 8192, 6428, 7936, 2886, 8094, 6564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61358 payload_rows_remote=53172 dup_rows=4178 dup_rows_remote=4162 dedup_ratio=0.063751 router_tokens_per_expert=[114, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4087, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 692, 6, 2, 0, 9, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 1099, 0, 0, 2, 0, 2, 0, 0, 4092, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 3408, 2, 0, 0, 2, 0, 0, 3206, 0, 0, 0, 0, 0] payload_rows_per_dst=[6924, 8178, 8180, 7186, 8186, 8192, 8096, 6416] dup_rows_per_dst=[164, 2, 1398, 0, 16, 2588, 6, 4] combine_rows_per_src=[6924, 8178, 8180, 7186, 8186, 8192, 8096, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55902 payload_rows_remote=47714 dup_rows=9634 dup_rows_remote=8726 dedup_ratio=0.147003 router_tokens_per_expert=[1667, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2257, 3, 0, 0, 31, 0, 0, 0, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 4094, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 0, 0, 0, 3813, 0, 0, 0, 0, 0, 0, 2126, 0, 0, 0, 0, 0] payload_rows_per_dst=[7978, 8182, 5796, 7110, 8192, 8188, 4014, 6442] dup_rows_per_dst=[3838, 0, 1950, 0, 86, 908, 16, 2836] combine_rows_per_src=[7978, 8182, 5796, 7110, 8192, 8188, 4014, 6442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58976 payload_rows_remote=50842 dup_rows=6560 dup_rows_remote=6550 dedup_ratio=0.100098 router_tokens_per_expert=[336, 0, 0, 0, 0, 73, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4062, 0, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 523, 36, 5, 0, 40, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 2562, 0, 0, 5, 0, 5, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 30, 1, 0, 0, 0, 3457, 5, 0, 0, 14, 0, 0, 3017, 0, 0, 0, 0, 0] payload_rows_per_dst=[6936, 8134, 8172, 8180, 8182, 5138, 8166, 6068] dup_rows_per_dst=[800, 10, 1184, 4514, 8, 30, 10, 4] combine_rows_per_src=[6936, 8134, 8172, 8180, 8182, 5138, 8166, 6068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56466 payload_rows_remote=48274 dup_rows=9070 dup_rows_remote=7498 dedup_ratio=0.138397 router_tokens_per_expert=[1053, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 780, 3, 1, 0, 6, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1940, 0, 0, 1, 0, 1, 0, 0, 4093, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 3928, 1, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0] payload_rows_per_dst=[7994, 8192, 8192, 8186, 8190, 3936, 8052, 3724] dup_rows_per_dst=[2058, 0, 1572, 5366, 44, 18, 12, 0] combine_rows_per_src=[7994, 8192, 8192, 8186, 8190, 3936, 8052, 3724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=201842688 dispatch_meta_bytes=228664 combine_payload_bytes=201842688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=202071352 ingress_bytes=258465792 link_bytes=258465792 token_bytes=403914040 weight_movement_bytes=56623104 total_nvlink_bytes=460537144 weight_to_token_ratio=0.140186 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.476492046e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55532 payload_rows_remote=47362 dup_rows=10004 dup_rows_remote=7992 dedup_ratio=0.152649 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 10, 0, 7, 0, 1026, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] payload_rows_per_dst=[6354, 7542, 8192, 8192, 8124, 8164, 8170, 794] dup_rows_per_dst=[8, 350, 170, 6, 4, 7372, 2012, 82] combine_rows_per_src=[6354, 7542, 8192, 8192, 8124, 8164, 8170, 794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49456 payload_rows_remote=41354 dup_rows=16080 dup_rows_remote=10222 dedup_ratio=0.245361 router_tokens_per_expert=[475, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4080, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 570, 2, 3, 0, 174, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 3, 0, 2, 2, 0, 0, 2142, 0, 0, 3, 0, 3, 0, 0, 4081, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 36, 0, 0, 0, 0, 3799, 3, 0, 0, 3, 0, 0, 2620, 0, 0, 0, 0, 0] payload_rows_per_dst=[7210, 8158, 1498, 0, 8186, 8192, 8110, 8102] dup_rows_per_dst=[1262, 2, 2, 0, 70, 4306, 4580, 5858] combine_rows_per_src=[7210, 8158, 1498, 0, 8186, 8192, 8110, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=197779456 dispatch_meta_bytes=226608 combine_payload_bytes=197779456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198006064 ingress_bytes=216653824 link_bytes=216653824 token_bytes=395785520 weight_movement_bytes=18874368 total_nvlink_bytes=414659888 weight_to_token_ratio=0.047688 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.075870341e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=217792512 dispatch_meta_bytes=229336 combine_payload_bytes=217792512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218021848 ingress_bytes=236666880 link_bytes=236666880 token_bytes=435814360 weight_movement_bytes=18874368 total_nvlink_bytes=454688728 weight_to_token_ratio=0.043308 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.267625597e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=195436544 dispatch_meta_bytes=225760 combine_payload_bytes=195436544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=195662304 ingress_bytes=252059648 link_bytes=252059648 token_bytes=391098848 weight_movement_bytes=56623104 total_nvlink_bytes=447721952 weight_to_token_ratio=0.144780 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415111526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=208248832 dispatch_meta_bytes=229568 combine_payload_bytes=208248832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208478400 ingress_bytes=245997568 link_bytes=245997568 token_bytes=416727232 weight_movement_bytes=37748736 total_nvlink_bytes=454475968 weight_to_token_ratio=0.090584 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357027658e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=197730304 dispatch_meta_bytes=223088 combine_payload_bytes=197730304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=197953392 ingress_bytes=235479040 link_bytes=235479040 token_bytes=395683696 weight_movement_bytes=37748736 total_nvlink_bytes=433432432 weight_to_token_ratio=0.095401 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.256244298e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=193994752 dispatch_meta_bytes=221416 combine_payload_bytes=193994752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=194216168 ingress_bytes=212869120 link_bytes=212869120 token_bytes=388210920 weight_movement_bytes=18874368 total_nvlink_bytes=407085288 weight_to_token_ratio=0.048619 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039607170e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=169385984 dispatch_meta_bytes=206304 combine_payload_bytes=169385984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=169592288 ingress_bytes=207134720 link_bytes=207134720 token_bytes=338978272 weight_movement_bytes=37748736 total_nvlink_bytes=376727008 weight_to_token_ratio=0.111360 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.984662970e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57444 payload_rows_remote=50450 dup_rows=8092 dup_rows_remote=8006 dedup_ratio=0.123474 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1540, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2462, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 46, 0, 0, 0, 2, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3688, 6994, 8192, 5880, 8114, 8192] dup_rows_per_dst=[3082, 4924, 0, 86, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 3688, 6994, 8192, 5880, 8114, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53236 payload_rows_remote=45044 dup_rows=12300 dup_rows_remote=11470 dedup_ratio=0.187683 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 4089, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 1982, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2313, 0, 0, 7, 1349, 0, 0, 0, 100, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 7, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 7, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 2554, 8192, 8164, 2360, 7454, 8142] dup_rows_per_dst=[830, 3992, 0, 7456, 14, 0, 0, 8] combine_rows_per_src=[8192, 8178, 2554, 8192, 8164, 2360, 7454, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54902 payload_rows_remote=46710 dup_rows=10634 dup_rows_remote=10630 dedup_ratio=0.162262 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4094, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 805, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 633, 0, 0, 1, 1327, 0, 0, 0, 8, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2604, 7690, 4162, 8192, 7844, 8030, 8188] dup_rows_per_dst=[328, 82, 4358, 1086, 4, 4774, 0, 2] combine_rows_per_src=[8192, 2604, 7690, 4162, 8192, 7844, 8030, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55676 payload_rows_remote=47872 dup_rows=9860 dup_rows_remote=6798 dedup_ratio=0.150452 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3542, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 75, 0, 0, 0, 22, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7084, 8192, 786, 8192, 7804, 7234, 8192] dup_rows_per_dst=[3902, 0, 2832, 64, 0, 3062, 0, 0] combine_rows_per_src=[8192, 7084, 8192, 786, 8192, 7804, 7234, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58644 payload_rows_remote=50462 dup_rows=6892 dup_rows_remote=4796 dedup_ratio=0.105164 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 4091, 0, 5, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 1048, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 0, 0, 5, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 0, 5, 1299, 0, 0, 0, 40, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 5, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 5, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 5736, 8192, 8192, 4122, 7866, 8162] dup_rows_per_dst=[574, 2096, 14, 4194, 4, 0, 0, 10] combine_rows_per_src=[8192, 8182, 5736, 8192, 8192, 4122, 7866, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=206643200 dispatch_meta_bytes=233824 combine_payload_bytes=206643200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=206877024 ingress_bytes=244391936 link_bytes=244391936 token_bytes=413520224 weight_movement_bytes=37748736 total_nvlink_bytes=451268960 weight_to_token_ratio=0.091286 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341643282e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58488 payload_rows_remote=52968 dup_rows=7048 dup_rows_remote=7038 dedup_ratio=0.107544 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4095, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1439, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 790, 0, 0, 1, 411, 0, 0, 0, 5, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 5520, 8142, 8192, 4032, 8036, 8184] dup_rows_per_dst=[1762, 2878, 10, 2396, 0, 0, 0, 2] combine_rows_per_src=[8192, 8190, 5520, 8142, 8192, 4032, 8036, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57814 payload_rows_remote=49648 dup_rows=7722 dup_rows_remote=7692 dedup_ratio=0.117828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 4096, 7, 0, 0, 0, 0, 0, 0, 455, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3841, 0, 26, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 4070] payload_rows_per_dst=[8140, 1442, 8192, 7810, 8192, 7682, 8166, 8190] dup_rows_per_dst=[0, 10, 940, 1794, 6, 0, 30, 4942] combine_rows_per_src=[8140, 1442, 8192, 7810, 8192, 7682, 8166, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51882 payload_rows_remote=43778 dup_rows=13654 dup_rows_remote=13496 dedup_ratio=0.208344 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, 4093, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2457, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 0, 2, 0, 2116, 0, 0, 3, 1045, 0, 0, 0, 195, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 3, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 3, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1054, 4978, 8192, 6114, 8192, 8124, 7124, 8104] dup_rows_per_dst=[0, 868, 3520, 4076, 6, 1500, 3526, 158] combine_rows_per_src=[1054, 4978, 8192, 6114, 8192, 8124, 7124, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=191324160 dispatch_meta_bytes=229360 combine_payload_bytes=191324160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191553520 ingress_bytes=210198528 link_bytes=210198528 token_bytes=382877680 weight_movement_bytes=18874368 total_nvlink_bytes=401752048 weight_to_token_ratio=0.049296 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.014018871e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=184500224 dispatch_meta_bytes=226056 combine_payload_bytes=184500224 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=184726280 ingress_bytes=259997696 link_bytes=259997696 token_bytes=369226504 weight_movement_bytes=75497472 total_nvlink_bytes=444723976 weight_to_token_ratio=0.204475 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.491169996e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=196083712 dispatch_meta_bytes=218680 combine_payload_bytes=196083712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196302392 ingress_bytes=214958080 link_bytes=214958080 token_bytes=392386104 weight_movement_bytes=18874368 total_nvlink_bytes=411260472 weight_to_token_ratio=0.048102 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.059622557e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=206692352 dispatch_meta_bytes=221032 combine_payload_bytes=206692352 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=206913384 ingress_bytes=263315456 link_bytes=263315456 token_bytes=413605736 weight_movement_bytes=56623104 total_nvlink_bytes=470228840 weight_to_token_ratio=0.136901 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.522959140e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=216956928 dispatch_meta_bytes=240024 combine_payload_bytes=216956928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217196952 ingress_bytes=235831296 link_bytes=235831296 token_bytes=434153880 weight_movement_bytes=18874368 total_nvlink_bytes=453028248 weight_to_token_ratio=0.043474 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259619442e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=203358208 dispatch_meta_bytes=229360 combine_payload_bytes=203358208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203587568 ingress_bytes=241106944 link_bytes=241106944 token_bytes=406945776 weight_movement_bytes=37748736 total_nvlink_bytes=444694512 weight_to_token_ratio=0.092761 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310168105e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=179314688 dispatch_meta_bytes=229096 combine_payload_bytes=179314688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179543784 ingress_bytes=198189056 link_bytes=198189056 token_bytes=358858472 weight_movement_bytes=18874368 total_nvlink_bytes=377732840 weight_to_token_ratio=0.052596 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.898950019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58608 payload_rows_remote=50424 dup_rows=6928 dup_rows_remote=6928 dedup_ratio=0.105713 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 1540, 0, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 4096, 0, 0, 1292, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 7276, 6648, 3732] dup_rows_per_dst=[0, 1760, 8, 0, 3078, 1948, 0, 134] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 7276, 6648, 3732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60750 payload_rows_remote=52558 dup_rows=4786 dup_rows_remote=4754 dedup_ratio=0.073029 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 181, 0, 0, 8, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4056, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2032, 0, 0, 2780, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 0, 0, 0, 0, 4093, 0, 0, 2202, 0, 2, 0, 115, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8114, 8192, 6666, 5976, 7230] dup_rows_per_dst=[4, 380, 24, 0, 32, 1858, 472, 2016] combine_rows_per_src=[8188, 8192, 8192, 8114, 8192, 6666, 5976, 7230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54646 payload_rows_remote=46468 dup_rows=10890 dup_rows_remote=10874 dedup_ratio=0.166168 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 33, 0, 0, 6, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 4089, 124, 0, 0, 0, 0, 0, 0, 7, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4041, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1290, 0, 0, 2547, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 0, 0, 4083, 0, 0, 2109, 0, 7, 0, 162, 0, 0] payload_rows_per_dst=[8178, 8192, 7292, 8192, 2200, 6688, 6588, 7316] dup_rows_per_dst=[16, 1364, 172, 6078, 16, 1000, 0, 2244] combine_rows_per_src=[8178, 8192, 7292, 8192, 2200, 6688, 6588, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62198 payload_rows_remote=54006 dup_rows=3338 dup_rows_remote=3172 dedup_ratio=0.050934 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 0, 81, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 4091, 41, 0, 0, 0, 0, 0, 0, 5, 10, 0, 0, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 1811, 0, 0, 2805, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 0, 0, 4089, 0, 0, 1455, 0, 5, 0, 39, 0, 0] payload_rows_per_dst=[8182, 8192, 8192, 8006, 8192, 7326, 7082, 7026] dup_rows_per_dst=[38, 166, 102, 0, 112, 1916, 0, 1004] combine_rows_per_src=[8182, 8192, 8192, 8006, 8192, 7326, 7082, 7026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57956 payload_rows_remote=55504 dup_rows=7580 dup_rows_remote=7580 dedup_ratio=0.115662 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1230, 0, 0, 1226, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 4096, 0, 0, 3240, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 2452, 7804, 6740] dup_rows_per_dst=[0, 410, 64, 0, 2460, 0, 4246, 400] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 2452, 7804, 6740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61994 payload_rows_remote=53804 dup_rows=3542 dup_rows_remote=3542 dedup_ratio=0.054047 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 207, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 2330, 0, 0, 2431, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 0, 0, 4095, 0, 0, 1842, 0, 1, 0, 55, 0, 0] payload_rows_per_dst=[8190, 8192, 8190, 8170, 8192, 7390, 7370, 6300] dup_rows_per_dst=[0, 422, 0, 0, 54, 2134, 0, 932] combine_rows_per_src=[8190, 8192, 8190, 8170, 8192, 7390, 7370, 6300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=206536704 dispatch_meta_bytes=229408 combine_payload_bytes=206536704 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=206766112 ingress_bytes=263159808 link_bytes=263159808 token_bytes=413302816 weight_movement_bytes=56623104 total_nvlink_bytes=469925920 weight_to_token_ratio=0.137001 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.521467798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50388 payload_rows_remote=42466 dup_rows=15148 dup_rows_remote=12252 dedup_ratio=0.231140 router_tokens_per_expert=[36, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1077, 1066, 0, 0, 13, 1, 0, 0, 0, 0, 0, 3857, 0, 0, 337, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 3263, 0, 0, 0, 0, 0] payload_rows_per_dst=[562, 8124, 4046, 5160, 8192, 8192, 7922, 8190] dup_rows_per_dst=[68, 0, 268, 606, 8, 4782, 2896, 6520] combine_rows_per_src=[562, 8124, 4046, 5160, 8192, 8192, 7922, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56592 payload_rows_remote=49020 dup_rows=8944 dup_rows_remote=6008 dedup_ratio=0.136475 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 53, 0, 0, 12, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 4093, 218, 0, 0, 0, 0, 0, 0, 3, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4075, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 841, 0, 0, 2004, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 4088, 0, 0, 2669, 0, 3, 0, 87, 0, 0] payload_rows_per_dst=[8164, 3504, 8192, 4738, 8192, 8158, 8072, 7572] dup_rows_per_dst=[0, 86, 530, 0, 186, 1680, 3526, 2936] combine_rows_per_src=[8164, 3504, 8192, 4738, 8192, 8158, 8072, 7572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=215277568 dispatch_meta_bytes=229248 combine_payload_bytes=215277568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215506816 ingress_bytes=234151936 link_bytes=234151936 token_bytes=430784384 weight_movement_bytes=18874368 total_nvlink_bytes=449658752 weight_to_token_ratio=0.043814 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.243528641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=190332928 dispatch_meta_bytes=229368 combine_payload_bytes=190332928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190562296 ingress_bytes=209207296 link_bytes=209207296 token_bytes=380895224 weight_movement_bytes=18874368 total_nvlink_bytes=399769592 weight_to_token_ratio=0.049553 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.004521374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=221208576 dispatch_meta_bytes=228712 combine_payload_bytes=221208576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=221437288 ingress_bytes=258957312 link_bytes=258957312 token_bytes=442645864 weight_movement_bytes=37748736 total_nvlink_bytes=480394600 weight_to_token_ratio=0.085280 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.481201549e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=227344384 dispatch_meta_bytes=252336 combine_payload_bytes=227344384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227596720 ingress_bytes=265093120 link_bytes=265093120 token_bytes=454941104 weight_movement_bytes=37748736 total_nvlink_bytes=492689840 weight_to_token_ratio=0.082975 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=14592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.539991842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=220381184 dispatch_meta_bytes=229384 combine_payload_bytes=220381184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220610568 ingress_bytes=258129920 link_bytes=258129920 token_bytes=440991752 weight_movement_bytes=37748736 total_nvlink_bytes=478740488 weight_to_token_ratio=0.085600 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.473273886e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=200785920 dispatch_meta_bytes=220112 combine_payload_bytes=200785920 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=201006032 ingress_bytes=238534656 link_bytes=238534656 token_bytes=401791952 weight_movement_bytes=37748736 total_nvlink_bytes=439540688 weight_to_token_ratio=0.093951 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.285521707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=173940736 dispatch_meta_bytes=218872 combine_payload_bytes=173940736 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=174159608 ingress_bytes=211689472 link_bytes=211689472 token_bytes=348100344 weight_movement_bytes=37748736 total_nvlink_bytes=385849080 weight_to_token_ratio=0.108442 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.028304363e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64272 payload_rows_remote=56088 dup_rows=1264 dup_rows_remote=1264 dedup_ratio=0.019287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 31, 3821, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8002, 8192, 8184, 8192, 8192, 7126, 8192] dup_rows_per_dst=[276, 424, 0, 0, 396, 32, 136, 0] combine_rows_per_src=[8192, 8002, 8192, 8184, 8192, 8192, 7126, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63408 payload_rows_remote=55216 dup_rows=2128 dup_rows_remote=1932 dedup_ratio=0.032471 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 81, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 33, 2766, 0, 4094, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 1, 4079, 0, 0, 2, 0, 0, 0, 0, 2, 0, 96, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 2, 502, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 6160, 8178, 8160, 8192, 8186, 8148, 8192] dup_rows_per_dst=[166, 22, 0, 2, 196, 766, 968, 8] combine_rows_per_src=[8192, 6160, 8178, 8160, 8192, 8186, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51854 payload_rows_remote=43662 dup_rows=13682 dup_rows_remote=13320 dedup_ratio=0.208771 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 0, 174, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 4, 6, 0, 0, 0, 83, 2660, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4089, 0, 0, 4052, 0, 0, 7, 0, 0, 0, 0, 7, 5, 173, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 21, 44, 0, 3, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 8178, 7178, 5972, 5550, 4010, 4654, 8120] dup_rows_per_dst=[362, 6452, 2882, 3572, 62, 236, 88, 28] combine_rows_per_src=[8192, 8178, 7178, 5972, 5550, 4010, 4654, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60706 payload_rows_remote=53706 dup_rows=4830 dup_rows_remote=2882 dedup_ratio=0.073700 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 257, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 23, 2602, 0, 4091, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4091, 0, 0, 3985, 0, 0, 5, 0, 0, 0, 0, 5, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 751, 0, 0, 0, 0, 0, 0, 0, 0, 3, 265, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5] payload_rows_per_dst=[8192, 7000, 4898, 7970, 8192, 8154, 8108, 8192] dup_rows_per_dst=[524, 1948, 0, 26, 282, 1500, 530, 20] combine_rows_per_src=[8192, 7000, 4898, 7970, 8192, 8154, 8108, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64136 payload_rows_remote=55944 dup_rows=1400 dup_rows_remote=1372 dedup_ratio=0.021362 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 10, 3779, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 10, 415, 0, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7656, 8180, 8152, 8192, 8192, 7380, 8192] dup_rows_per_dst=[374, 166, 0, 0, 464, 28, 368, 0] combine_rows_per_src=[8192, 7656, 8180, 8152, 8192, 8192, 7380, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229736448 dispatch_meta_bytes=229408 combine_payload_bytes=229736448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=229965856 ingress_bytes=305233920 link_bytes=305233920 token_bytes=459702304 weight_movement_bytes=75497472 total_nvlink_bytes=535199776 weight_to_token_ratio=0.164231 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28544 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.924601237e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62996 payload_rows_remote=54810 dup_rows=2540 dup_rows_remote=2540 dedup_ratio=0.038757 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 0, 0, 0, 49, 2163, 0, 4095, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4079, 0, 0, 1, 0, 0, 0, 0, 1, 0, 85, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 1, 511, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 5858, 8186, 8158, 8192, 8190, 8028, 8192] dup_rows_per_dst=[660, 62, 0, 0, 174, 714, 926, 4] combine_rows_per_src=[8192, 5858, 8186, 8158, 8192, 8190, 8028, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=226164736 dispatch_meta_bytes=228592 combine_payload_bytes=226164736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226393328 ingress_bytes=245039104 link_bytes=245039104 token_bytes=452558064 weight_movement_bytes=18874368 total_nvlink_bytes=471432432 weight_to_token_ratio=0.041706 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.347844128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52512 payload_rows_remote=44360 dup_rows=13024 dup_rows_remote=13024 dedup_ratio=0.198730 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 3944, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 48, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 1760, 0, 0, 0, 11, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4802, 3272, 8192, 3574, 8192, 8136, 8152, 8192] dup_rows_per_dst=[50, 16, 5038, 22, 0, 7898, 0, 0] combine_rows_per_src=[4802, 3272, 8192, 3574, 8192, 8136, 8152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60128 payload_rows_remote=51936 dup_rows=5408 dup_rows_remote=5396 dedup_ratio=0.082520 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 218, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 1, 0, 0, 0, 0, 26, 2385, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 4063, 0, 0, 3, 0, 0, 0, 0, 3, 1, 504, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 32, 87, 0, 4, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[5416, 5694, 8186, 8126, 8192, 8162, 8160, 8192] dup_rows_per_dst=[328, 20, 2890, 4, 1012, 964, 178, 12] combine_rows_per_src=[5416, 5694, 8186, 8126, 8192, 8162, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=178839552 dispatch_meta_bytes=227928 combine_payload_bytes=178839552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179067480 ingress_bytes=197713920 link_bytes=197713920 token_bytes=357907032 weight_movement_bytes=18874368 total_nvlink_bytes=376781400 weight_to_token_ratio=0.052735 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.894397500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=219979776 dispatch_meta_bytes=226352 combine_payload_bytes=219979776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220206128 ingress_bytes=238854144 link_bytes=238854144 token_bytes=440185904 weight_movement_bytes=18874368 total_nvlink_bytes=459060272 weight_to_token_ratio=0.042878 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.288582884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229146624 dispatch_meta_bytes=229264 combine_payload_bytes=229146624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229375888 ingress_bytes=248020992 link_bytes=248020992 token_bytes=458522512 weight_movement_bytes=18874368 total_nvlink_bytes=477396880 weight_to_token_ratio=0.041163 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376415111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=224501760 dispatch_meta_bytes=229400 combine_payload_bytes=224501760 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=224731160 ingress_bytes=299999232 link_bytes=299999232 token_bytes=449232920 weight_movement_bytes=75497472 total_nvlink_bytes=524730392 weight_to_token_ratio=0.168059 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.874445032e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=181698560 dispatch_meta_bytes=229536 combine_payload_bytes=181698560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=181928096 ingress_bytes=200572928 link_bytes=200572928 token_bytes=363626656 weight_movement_bytes=18874368 total_nvlink_bytes=382501024 weight_to_token_ratio=0.051906 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.921791108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=212729856 dispatch_meta_bytes=229328 combine_payload_bytes=212729856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212959184 ingress_bytes=231604224 link_bytes=231604224 token_bytes=425689040 weight_movement_bytes=18874368 total_nvlink_bytes=444563408 weight_to_token_ratio=0.044338 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219117718e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57292 dup_rows=266 dup_rows_remote=248 dedup_ratio=0.004059 router_tokens_per_expert=[3, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 101, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4078, 4093, 2, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 154, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8160, 8186, 7978, 8184, 8192, 8188] dup_rows_per_dst=[218, 4, 24, 2, 18, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8160, 8186, 7978, 8184, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64194 payload_rows_remote=56002 dup_rows=1342 dup_rows_remote=1342 dedup_ratio=0.020477 router_tokens_per_expert=[12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 452, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8138, 8192, 6930, 8192, 8178, 8180] dup_rows_per_dst=[974, 0, 348, 0, 20, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8138, 8192, 6930, 8192, 8178, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64060 payload_rows_remote=55868 dup_rows=1476 dup_rows_remote=1432 dedup_ratio=0.022522 router_tokens_per_expert=[0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 0, 0, 12, 0, 0, 0, 4083, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4082, 4087, 7, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 11, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8180, 8174, 6826, 8186, 8192, 8130] dup_rows_per_dst=[44, 14, 1404, 12, 2, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8180, 8174, 6826, 8186, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64828 payload_rows_remote=56650 dup_rows=708 dup_rows_remote=698 dedup_ratio=0.010803 router_tokens_per_expert=[5, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 39, 0, 0, 0, 124, 0, 0, 0, 4084, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4037, 4091, 5, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 55, 0, 0, 0, 0, 9, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 5, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8168, 8182, 7626, 8182, 8156, 8144] dup_rows_per_dst=[346, 10, 312, 12, 24, 4, 0, 0] combine_rows_per_src=[8192, 8178, 8168, 8182, 7626, 8182, 8156, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=230160 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234898192 ingress_bytes=310165504 link_bytes=310165504 token_bytes=469566224 weight_movement_bytes=75497472 total_nvlink_bytes=545063696 weight_to_token_ratio=0.160781 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.971853248e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64096 payload_rows_remote=55904 dup_rows=1440 dup_rows_remote=1438 dedup_ratio=0.021973 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 237, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 6794, 8192, 8192, 8152] dup_rows_per_dst=[720, 0, 702, 0, 16, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 6794, 8192, 8192, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64284 payload_rows_remote=56218 dup_rows=1252 dup_rows_remote=952 dedup_ratio=0.019104 router_tokens_per_expert=[5, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 0, 450, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 4095, 1, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3383, 0, 0, 160, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8066, 8190, 7074, 8192, 8190, 8188] dup_rows_per_dst=[930, 0, 300, 2, 16, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8066, 8190, 7074, 8192, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61920 payload_rows_remote=54254 dup_rows=3616 dup_rows_remote=3530 dedup_ratio=0.055176 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 3415, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 4096, 0, 0, 3957, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5862, 8192, 7458, 7666, 8166] dup_rows_per_dst=[0, 100, 2, 0, 54, 1586, 86, 1788] combine_rows_per_src=[8192, 8192, 8192, 5862, 8192, 7458, 7666, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=229384192 dispatch_meta_bytes=229376 combine_payload_bytes=229384192 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=229613568 ingress_bytes=267132928 link_bytes=267132928 token_bytes=458997760 weight_movement_bytes=37748736 total_nvlink_bytes=496746496 weight_to_token_ratio=0.082242 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.559536278e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56134 payload_rows_remote=48086 dup_rows=9402 dup_rows_remote=9082 dedup_ratio=0.143463 router_tokens_per_expert=[0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 139, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4080, 4093, 3, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 70, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[1908, 8186, 5308, 8186, 8122, 8184, 8192, 8048] dup_rows_per_dst=[128, 6, 98, 18, 6282, 22, 2528, 320] combine_rows_per_src=[1908, 8186, 5308, 8186, 8122, 8184, 8192, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=228835328 dispatch_meta_bytes=229200 combine_payload_bytes=228835328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229064528 ingress_bytes=247709696 link_bytes=247709696 token_bytes=457899856 weight_movement_bytes=18874368 total_nvlink_bytes=476774224 weight_to_token_ratio=0.041219 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373432426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232038400 dispatch_meta_bytes=229392 combine_payload_bytes=232038400 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=232267792 ingress_bytes=250912768 link_bytes=250912768 token_bytes=464306192 weight_movement_bytes=18874368 total_nvlink_bytes=483180560 weight_to_token_ratio=0.040651 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404122686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=228982784 dispatch_meta_bytes=229368 combine_payload_bytes=228982784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229212152 ingress_bytes=247857152 link_bytes=247857152 token_bytes=458194936 weight_movement_bytes=18874368 total_nvlink_bytes=477069304 weight_to_token_ratio=0.041193 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374845277e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=222224384 dispatch_meta_bytes=231136 combine_payload_bytes=222224384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222455520 ingress_bytes=241098752 link_bytes=241098752 token_bytes=444679904 weight_movement_bytes=18874368 total_nvlink_bytes=463554272 weight_to_token_ratio=0.042445 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310089614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=230268928 dispatch_meta_bytes=228680 combine_payload_bytes=230268928 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230497608 ingress_bytes=305766400 link_bytes=305766400 token_bytes=460766536 weight_movement_bytes=75497472 total_nvlink_bytes=536264008 weight_to_token_ratio=0.163852 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.929703198e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=196960256 dispatch_meta_bytes=228672 combine_payload_bytes=196960256 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=197188928 ingress_bytes=234708992 link_bytes=234708992 token_bytes=394149184 weight_movement_bytes=37748736 total_nvlink_bytes=431897920 weight_to_token_ratio=0.095773 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.248866077e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59790 payload_rows_remote=51600 dup_rows=5746 dup_rows_remote=5740 dedup_ratio=0.087677 router_tokens_per_expert=[0, 0, 0, 0, 4091, 0, 0, 3674, 0, 0, 0, 0, 0, 2, 928, 0, 0, 2, 0, 0, 0, 1967, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4064, 2, 13, 0, 0, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 106, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 3633, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2, 4092, 0, 0, 0] payload_rows_per_dst=[7644, 3938, 8180, 8188, 8190, 8192, 7270, 8188] dup_rows_per_dst=[1564, 4, 3810, 0, 6, 216, 6, 140] combine_rows_per_src=[7644, 3938, 8180, 8188, 8190, 8192, 7270, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59362 payload_rows_remote=51170 dup_rows=6174 dup_rows_remote=6174 dedup_ratio=0.094208 router_tokens_per_expert=[0, 0, 0, 0, 4071, 0, 0, 2114, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 2744, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 13, 0, 0, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 4096, 0, 0, 0] payload_rows_per_dst=[5118, 6734, 6728, 8192, 8148, 8192, 8058, 8192] dup_rows_per_dst=[552, 2292, 1894, 0, 538, 712, 0, 186] combine_rows_per_src=[5118, 6734, 6728, 8192, 8148, 8192, 8058, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=211353600 dispatch_meta_bytes=229360 combine_payload_bytes=211353600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211582960 ingress_bytes=230227968 link_bytes=230227968 token_bytes=422936560 weight_movement_bytes=18874368 total_nvlink_bytes=441810928 weight_to_token_ratio=0.044627 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.205931110e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51170 payload_rows_remote=43078 dup_rows=14366 dup_rows_remote=6188 dedup_ratio=0.219208 router_tokens_per_expert=[0, 0, 0, 0, 4069, 0, 0, 3130, 0, 0, 0, 0, 0, 7, 1603, 0, 0, 7, 0, 0, 0, 2984, 0, 2, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3970, 6, 180, 0, 0, 351, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 7, 0, 0, 0, 222, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3879, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 7, 4077, 0, 0, 0] payload_rows_per_dst=[8092, 8190, 1060, 8178, 6464, 5424, 7772, 5990] dup_rows_per_dst=[8178, 5760, 18, 0, 2, 370, 10, 28] combine_rows_per_src=[8092, 8190, 1060, 8178, 6464, 5424, 7772, 5990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58316 payload_rows_remote=50134 dup_rows=7220 dup_rows_remote=1350 dedup_ratio=0.110168 router_tokens_per_expert=[0, 0, 0, 0, 4079, 0, 0, 3545, 0, 0, 0, 0, 0, 5, 507, 0, 0, 5, 0, 0, 0, 2981, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4028, 5, 53, 0, 0, 1553, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0, 5, 0, 0, 0, 100, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3413, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 5, 4074, 0, 0, 0] payload_rows_per_dst=[7378, 8182, 3206, 8174, 8190, 8192, 6836, 8158] dup_rows_per_dst=[736, 5870, 18, 0, 18, 210, 10, 358] combine_rows_per_src=[7378, 8182, 3206, 8174, 8190, 8192, 6836, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62562 payload_rows_remote=54524 dup_rows=2974 dup_rows_remote=274 dedup_ratio=0.045380 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 3, 3883, 0, 4096, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3294, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7792, 6766, 7198, 8192, 8192, 8038, 8192] dup_rows_per_dst=[86, 102, 0, 12, 56, 18, 2700, 0] combine_rows_per_src=[8192, 7792, 6766, 7198, 8192, 8192, 8038, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59600 payload_rows_remote=51408 dup_rows=5936 dup_rows_remote=5840 dedup_ratio=0.090576 router_tokens_per_expert=[0, 0, 0, 0, 4076, 0, 0, 3280, 0, 0, 0, 0, 0, 0, 1887, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 23, 0, 0, 1609, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4090, 0, 0, 0] payload_rows_per_dst=[7724, 2966, 8024, 8192, 8158, 8192, 8162, 8182] dup_rows_per_dst=[2610, 4, 2930, 0, 272, 96, 4, 20] combine_rows_per_src=[7724, 2966, 8024, 8192, 8158, 8192, 8162, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58376 payload_rows_remote=53626 dup_rows=7160 dup_rows_remote=7152 dedup_ratio=0.109253 router_tokens_per_expert=[0, 0, 0, 0, 4093, 0, 0, 2868, 0, 0, 0, 0, 0, 1, 604, 0, 0, 1, 0, 0, 0, 2887, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 13, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 65, 0, 0, 0, 1, 0, 0, 0, 180, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 4095, 0, 3, 0] payload_rows_per_dst=[6196, 8184, 4750, 8190, 8188, 8192, 6484, 8192] dup_rows_per_dst=[750, 5730, 8, 0, 156, 362, 2, 152] combine_rows_per_src=[6196, 8184, 4750, 8190, 8188, 8192, 6484, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=209592320 dispatch_meta_bytes=229376 combine_payload_bytes=209592320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209821696 ingress_bytes=228466688 link_bytes=228466688 token_bytes=419414016 weight_movement_bytes=18874368 total_nvlink_bytes=438288384 weight_to_token_ratio=0.045002 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189055392e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57638 payload_rows_remote=49458 dup_rows=7898 dup_rows_remote=7586 dedup_ratio=0.120514 router_tokens_per_expert=[0, 0, 0, 0, 4077, 0, 0, 2570, 0, 0, 0, 0, 0, 3, 1730, 0, 0, 3, 0, 0, 0, 2657, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3971, 2, 88, 0, 0, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 3, 0, 0, 0, 386, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 3870, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3, 4086, 0, 9, 0] payload_rows_per_dst=[3466, 5322, 7976, 8184, 8188, 8192, 8130, 8180] dup_rows_per_dst=[0, 10, 1838, 68, 130, 776, 4764, 312] combine_rows_per_src=[3466, 5322, 7976, 8184, 8188, 8192, 8130, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=176447488 dispatch_meta_bytes=197064 combine_payload_bytes=176447488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=176644552 ingress_bytes=233070592 link_bytes=233070592 token_bytes=353092040 weight_movement_bytes=56623104 total_nvlink_bytes=409715144 weight_to_token_ratio=0.160364 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.233167735e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=205348864 dispatch_meta_bytes=205936 combine_payload_bytes=205348864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205554800 ingress_bytes=224223232 link_bytes=224223232 token_bytes=410903664 weight_movement_bytes=18874368 total_nvlink_bytes=429778032 weight_to_token_ratio=0.045934 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.148396684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=223330304 dispatch_meta_bytes=219192 combine_payload_bytes=223330304 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=223549496 ingress_bytes=242204672 link_bytes=242204672 token_bytes=446879800 weight_movement_bytes=18874368 total_nvlink_bytes=465754168 weight_to_token_ratio=0.042236 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.320685995e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=210567168 dispatch_meta_bytes=228992 combine_payload_bytes=210567168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210796160 ingress_bytes=248315904 link_bytes=248315904 token_bytes=421363328 weight_movement_bytes=37748736 total_nvlink_bytes=459112064 weight_to_token_ratio=0.089587 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379240813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219652096 dispatch_meta_bytes=243112 combine_payload_bytes=219652096 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=219895208 ingress_bytes=276275200 link_bytes=276275200 token_bytes=439547304 weight_movement_bytes=56623104 total_nvlink_bytes=496170408 weight_to_token_ratio=0.128821 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.647133031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58566 payload_rows_remote=50374 dup_rows=6970 dup_rows_remote=6966 dedup_ratio=0.106354 router_tokens_per_expert=[0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 1140, 1, 0, 1071, 1, 0, 0, 0, 0, 2, 11, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 2320] payload_rows_per_dst=[8188, 8146, 2170, 8192, 8192, 8184, 7316, 8178] dup_rows_per_dst=[0, 2284, 0, 12, 4, 0, 0, 4670] combine_rows_per_src=[8188, 8146, 2170, 8192, 8192, 8184, 7316, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=202579968 dispatch_meta_bytes=228176 combine_payload_bytes=202579968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202808144 ingress_bytes=240328704 link_bytes=240328704 token_bytes=405388112 weight_movement_bytes=37748736 total_nvlink_bytes=443136848 weight_to_token_ratio=0.093118 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.302711393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61288 payload_rows_remote=53096 dup_rows=4248 dup_rows_remote=4248 dedup_ratio=0.064819 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 3051, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 2427] payload_rows_per_dst=[8192, 8192, 7478, 8192, 8192, 8132, 7418, 5492] dup_rows_per_dst=[0, 194, 3172, 0, 0, 0, 0, 882] combine_rows_per_src=[8192, 8192, 7478, 8192, 8192, 8132, 7418, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206331904 dispatch_meta_bytes=229360 combine_payload_bytes=206331904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206561264 ingress_bytes=225206272 link_bytes=225206272 token_bytes=412893168 weight_movement_bytes=18874368 total_nvlink_bytes=431767536 weight_to_token_ratio=0.045712 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.157815690e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50624 payload_rows_remote=42446 dup_rows=14912 dup_rows_remote=13116 dedup_ratio=0.227539 router_tokens_per_expert=[0, 0, 0, 4088, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 4, 0, 1952, 1, 0, 508, 3, 0, 0, 0, 0, 7, 24, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 7, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 1952] payload_rows_per_dst=[8178, 8120, 8192, 8076, 6550, 0, 7558, 3950] dup_rows_per_dst=[1796, 3844, 9156, 32, 0, 0, 0, 84] combine_rows_per_src=[8178, 8120, 8192, 8076, 6550, 0, 7558, 3950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58536 payload_rows_remote=50406 dup_rows=7000 dup_rows_remote=3432 dedup_ratio=0.106812 router_tokens_per_expert=[0, 0, 0, 4091, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1786, 5, 0, 675, 0, 0, 0, 0, 0, 5, 360, 0, 0, 0, 0, 0, 4056, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 5, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 1438] payload_rows_per_dst=[8184, 8130, 8186, 8192, 8192, 6786, 7960, 2906] dup_rows_per_dst=[0, 3568, 3334, 40, 6, 0, 0, 52] combine_rows_per_src=[8184, 8130, 8186, 8192, 8192, 6786, 7960, 2906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61404 payload_rows_remote=55688 dup_rows=4132 dup_rows_remote=4132 dedup_ratio=0.063049 router_tokens_per_expert=[315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 4094, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7694, 8188, 7044, 8192, 5716, 8186] dup_rows_per_dst=[2924, 0, 444, 0, 764, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7694, 8188, 7044, 8192, 5716, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57940 payload_rows_remote=49904 dup_rows=7596 dup_rows_remote=4470 dedup_ratio=0.115906 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 137, 0, 0, 2521, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 3052] payload_rows_per_dst=[8192, 4862, 5044, 8192, 8192, 8036, 7278, 8144] dup_rows_per_dst=[0, 214, 12, 0, 0, 3126, 0, 4244] combine_rows_per_src=[8192, 4862, 5044, 8192, 8192, 8036, 7278, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=217481216 dispatch_meta_bytes=229376 combine_payload_bytes=217481216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217710592 ingress_bytes=236355584 link_bytes=236355584 token_bytes=435191808 weight_movement_bytes=18874368 total_nvlink_bytes=454066176 weight_to_token_ratio=0.043370 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264642912e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58044 payload_rows_remote=49862 dup_rows=7492 dup_rows_remote=2584 dedup_ratio=0.114319 router_tokens_per_expert=[0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 1, 0, 2479, 0, 0, 0, 0, 0, 1, 73, 0, 0, 0, 0, 0, 3991, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 788] payload_rows_per_dst=[8190, 8190, 8182, 8192, 8192, 8188, 7296, 1614] dup_rows_per_dst=[0, 2578, 4908, 6, 0, 0, 0, 0] combine_rows_per_src=[8190, 8190, 8182, 8192, 8192, 8188, 7296, 1614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54434 payload_rows_remote=46372 dup_rows=11102 dup_rows_remote=9348 dedup_ratio=0.169403 router_tokens_per_expert=[0, 0, 0, 4092, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2499, 3, 0, 1007, 1, 0, 0, 0, 0, 3, 189, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 894] payload_rows_per_dst=[6424, 5004, 2294, 8192, 8192, 8074, 8192, 8062] dup_rows_per_dst=[0, 6, 106, 26, 0, 1760, 7450, 1754] combine_rows_per_src=[6424, 5004, 2294, 8192, 8192, 8074, 8192, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=173858816 dispatch_meta_bytes=222248 combine_payload_bytes=173858816 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=174081064 ingress_bytes=230481920 link_bytes=230481920 token_bytes=347939880 weight_movement_bytes=56623104 total_nvlink_bytes=404562984 weight_to_token_ratio=0.162738 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.208364353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206462976 dispatch_meta_bytes=215352 combine_payload_bytes=206462976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206678328 ingress_bytes=225337344 link_bytes=225337344 token_bytes=413141304 weight_movement_bytes=18874368 total_nvlink_bytes=432015672 weight_to_token_ratio=0.045685 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159071557e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=228098048 dispatch_meta_bytes=239280 combine_payload_bytes=228098048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228337328 ingress_bytes=284721152 link_bytes=284721152 token_bytes=456435376 weight_movement_bytes=56623104 total_nvlink_bytes=513058480 weight_to_token_ratio=0.124055 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.728057987e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=204406784 dispatch_meta_bytes=217496 combine_payload_bytes=204406784 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=204624280 ingress_bytes=279904256 link_bytes=279904256 token_bytes=409031064 weight_movement_bytes=75497472 total_nvlink_bytes=484528536 weight_to_token_ratio=0.184576 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.681904860e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=204234752 dispatch_meta_bytes=209784 combine_payload_bytes=204234752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204444536 ingress_bytes=241983488 link_bytes=241983488 token_bytes=408679288 weight_movement_bytes=37748736 total_nvlink_bytes=446428024 weight_to_token_ratio=0.092368 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=14464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.318566719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63102 payload_rows_remote=54918 dup_rows=2434 dup_rows_remote=2214 dedup_ratio=0.037140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1396, 2, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 643, 0, 0, 0, 0, 4069, 0, 2, 0, 0, 3670, 0, 0, 0, 2, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 91, 0, 0, 42, 0, 2941, 3499, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7566, 7622, 8190, 8144, 8184, 7016, 8188, 8192] dup_rows_per_dst=[1116, 1008, 0, 4, 220, 70, 16, 0] combine_rows_per_src=[7566, 7622, 8190, 8144, 8184, 7016, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=189939712 dispatch_meta_bytes=222880 combine_payload_bytes=189939712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190162592 ingress_bytes=208814080 link_bytes=208814080 token_bytes=380102304 weight_movement_bytes=18874368 total_nvlink_bytes=398976672 weight_to_token_ratio=0.049656 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.000753772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62802 payload_rows_remote=54680 dup_rows=2734 dup_rows_remote=2734 dedup_ratio=0.041718 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 4061, 0, 0, 4, 0, 3181, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 6, 0, 0, 5, 0, 4073, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 6464, 8192, 8122, 8192, 7268, 8192, 8192] dup_rows_per_dst=[2566, 4, 0, 0, 162, 2, 0, 0] combine_rows_per_src=[8180, 6464, 8192, 8122, 8192, 7268, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=224944128 dispatch_meta_bytes=228528 combine_payload_bytes=224944128 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=225172656 ingress_bytes=281567232 link_bytes=281567232 token_bytes=450116784 weight_movement_bytes=56623104 total_nvlink_bytes=506739888 weight_to_token_ratio=0.125796 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.697838677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63092 payload_rows_remote=55238 dup_rows=2444 dup_rows_remote=1500 dedup_ratio=0.037292 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 860, 7, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 407, 0, 0, 0, 0, 3914, 0, 7, 0, 0, 3948, 0, 0, 0, 7, 0, 4062, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 6, 0, 0, 85, 0, 3525, 3524, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7854, 8154, 8192, 7842, 7730, 7094, 8040, 8186] dup_rows_per_dst=[944, 1084, 104, 10, 158, 142, 2, 0] combine_rows_per_src=[7854, 8154, 8192, 7842, 7730, 7094, 8040, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60108 payload_rows_remote=52250 dup_rows=5428 dup_rows_remote=4626 dedup_ratio=0.082825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2845, 5, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 405, 0, 0, 0, 0, 4054, 0, 5, 55, 0, 3865, 0, 0, 0, 5, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 31, 0, 0, 136, 0, 3379, 1541, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8078, 7858, 8144, 8118, 8182, 3354, 8182, 8192] dup_rows_per_dst=[4390, 802, 0, 10, 190, 10, 26, 0] combine_rows_per_src=[8078, 7858, 8144, 8118, 8182, 3354, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53202 payload_rows_remote=47950 dup_rows=12334 dup_rows_remote=9558 dedup_ratio=0.188202 router_tokens_per_expert=[0, 0, 0, 0, 4084, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 0, 1163, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 3, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 0, 4096, 0, 0, 0] payload_rows_per_dst=[4738, 2350, 8162, 8170, 8170, 8168, 5252, 8192] dup_rows_per_dst=[790, 0, 6934, 0, 718, 50, 2776, 1066] combine_rows_per_src=[4738, 2350, 8162, 8170, 8170, 8168, 5252, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64386 payload_rows_remote=56230 dup_rows=1150 dup_rows_remote=1150 dedup_ratio=0.017548 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 41, 0, 0, 0, 0, 4053, 0, 0, 2, 0, 3776, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 44, 0, 0, 0, 0, 3802, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7736, 7622, 8190, 8106, 8192, 8156, 8192, 8192] dup_rows_per_dst=[354, 16, 0, 0, 736, 0, 44, 0] combine_rows_per_src=[7736, 7622, 8190, 8106, 8192, 8156, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61210 payload_rows_remote=53024 dup_rows=4326 dup_rows_remote=4326 dedup_ratio=0.066010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2168, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 221, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 3665, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 44, 0, 0, 13, 0, 3789, 2357, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8126, 7408, 8186, 8176, 8190, 4742, 8190, 8192] dup_rows_per_dst=[3792, 366, 0, 2, 150, 0, 10, 6] combine_rows_per_src=[8126, 7408, 8186, 8176, 8190, 4742, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=223969280 dispatch_meta_bytes=229656 combine_payload_bytes=223969280 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224198936 ingress_bytes=261718016 link_bytes=261718016 token_bytes=448168216 weight_movement_bytes=37748736 total_nvlink_bytes=485916952 weight_to_token_ratio=0.084229 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.507653256e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214016000 dispatch_meta_bytes=227504 combine_payload_bytes=214016000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=214243504 ingress_bytes=289513472 link_bytes=289513472 token_bytes=428259504 weight_movement_bytes=75497472 total_nvlink_bytes=503756976 weight_to_token_ratio=0.176289 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.773975639e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56682 payload_rows_remote=48490 dup_rows=8854 dup_rows_remote=8854 dedup_ratio=0.135101 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1452, 3, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 175, 0, 0, 0, 0, 4040, 0, 3, 0, 0, 3956, 0, 0, 0, 3, 0, 4069, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 59, 0, 0, 123, 0, 3509, 2751, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2910, 7196, 8146, 8088, 5834, 8130, 8186, 8192] dup_rows_per_dst=[6, 330, 0, 748, 656, 7048, 66, 0] combine_rows_per_src=[2910, 7196, 8146, 8088, 5834, 8130, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=226254848 dispatch_meta_bytes=226952 combine_payload_bytes=226254848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226481800 ingress_bytes=245129216 link_bytes=245129216 token_bytes=452736648 weight_movement_bytes=18874368 total_nvlink_bytes=471611016 weight_to_token_ratio=0.041690 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348707537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=196403200 dispatch_meta_bytes=230032 combine_payload_bytes=196403200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196633232 ingress_bytes=215277568 link_bytes=215277568 token_bytes=393036432 weight_movement_bytes=18874368 total_nvlink_bytes=411910800 weight_to_token_ratio=0.048022 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.062683733e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=230318080 dispatch_meta_bytes=229520 combine_payload_bytes=230318080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=230547600 ingress_bytes=268066816 link_bytes=268066816 token_bytes=460865680 weight_movement_bytes=37748736 total_nvlink_bytes=498614416 weight_to_token_ratio=0.081908 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568484334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=217186304 dispatch_meta_bytes=229400 combine_payload_bytes=217186304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217415704 ingress_bytes=236060672 link_bytes=236060672 token_bytes=434602008 weight_movement_bytes=18874368 total_nvlink_bytes=453476376 weight_to_token_ratio=0.043429 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.261817210e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61410 payload_rows_remote=53222 dup_rows=4126 dup_rows_remote=4122 dedup_ratio=0.062958 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2, 4094, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2746, 0, 2, 0, 0, 2, 4094, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8180, 8144, 8188, 5668, 7244, 7606] dup_rows_per_dst=[0, 10, 0, 0, 4, 1252, 2860, 0] combine_rows_per_src=[8192, 8188, 8180, 8144, 8188, 5668, 7244, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=198615040 dispatch_meta_bytes=229376 combine_payload_bytes=198615040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198844416 ingress_bytes=217489408 link_bytes=217489408 token_bytes=397459456 weight_movement_bytes=18874368 total_nvlink_bytes=416333824 weight_to_token_ratio=0.047488 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.083876496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=217997312 dispatch_meta_bytes=229376 combine_payload_bytes=217997312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218226688 ingress_bytes=236871680 link_bytes=236871680 token_bytes=436224000 weight_movement_bytes=18874368 total_nvlink_bytes=455098368 weight_to_token_ratio=0.043268 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.269587890e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61948 payload_rows_remote=53758 dup_rows=3588 dup_rows_remote=3588 dedup_ratio=0.054749 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 6126, 6672] dup_rows_per_dst=[12, 0, 0, 0, 0, 3576, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 6126, 6672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59968 payload_rows_remote=51786 dup_rows=5568 dup_rows_remote=5546 dedup_ratio=0.084961 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 5, 4091, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 5, 0, 0, 5, 0, 0, 0, 0, 0, 2718, 0, 5, 0, 0, 14, 4091, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 5, 0, 1, 0, 0, 0, 0, 4, 0, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8176, 8180, 8158, 8192, 3826, 7062] dup_rows_per_dst=[4, 22, 0, 48, 10, 5482, 2, 0] combine_rows_per_src=[8192, 8182, 8176, 8180, 8158, 8192, 3826, 7062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59950 payload_rows_remote=51758 dup_rows=5586 dup_rows_remote=5586 dedup_ratio=0.085236 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 10, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 7, 4089, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 7, 0, 0, 7, 0, 0, 0, 0, 0, 2695, 0, 7, 0, 0, 13, 4089, 0, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 7, 0, 2, 0, 0, 0, 0, 7, 0, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8162, 8182, 8148, 8192, 4652, 6244] dup_rows_per_dst=[0, 36, 0, 92, 14, 5440, 4, 0] combine_rows_per_src=[8192, 8178, 8162, 8182, 8148, 8192, 4652, 6244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59602 payload_rows_remote=51416 dup_rows=5934 dup_rows_remote=5934 dedup_ratio=0.090546 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4095, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2948, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8186, 8186, 8190, 8192, 3486, 6980] dup_rows_per_dst=[16, 0, 0, 16, 2, 5898, 2, 0] combine_rows_per_src=[8192, 8190, 8186, 8186, 8190, 8192, 3486, 6980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59146 payload_rows_remote=56602 dup_rows=6390 dup_rows_remote=6390 dedup_ratio=0.097504 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 38, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8192, 2544, 8192, 7456] dup_rows_per_dst=[80, 2, 0, 0, 0, 0, 6308, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8192, 2544, 8192, 7456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51072 payload_rows_remote=46462 dup_rows=14464 dup_rows_remote=14006 dedup_ratio=0.220703 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 1647, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3688] payload_rows_per_dst=[8192, 2204, 3302, 8192, 8192, 8192, 4610, 8188] dup_rows_per_dst=[0, 0, 0, 0, 1632, 5058, 458, 7316] combine_rows_per_src=[8192, 2204, 3302, 8192, 8192, 8192, 4610, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=220192768 dispatch_meta_bytes=229384 combine_payload_bytes=220192768 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=220422152 ingress_bytes=276815872 link_bytes=276815872 token_bytes=440614920 weight_movement_bytes=56623104 total_nvlink_bytes=497238024 weight_to_token_ratio=0.128509 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.652313484e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=212115456 dispatch_meta_bytes=229328 combine_payload_bytes=212115456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212344784 ingress_bytes=230989824 link_bytes=230989824 token_bytes=424460240 weight_movement_bytes=18874368 total_nvlink_bytes=443334608 weight_to_token_ratio=0.044467 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.213230839e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56816 payload_rows_remote=48964 dup_rows=8720 dup_rows_remote=6682 dedup_ratio=0.133057 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 59, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3, 4093, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3, 0, 0, 3, 0, 0, 0, 0, 0, 1775, 0, 3, 0, 0, 4, 4092, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 3, 0, 1, 0, 0, 0, 0, 4, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7924, 8140, 8184, 7860, 8164, 500, 8192, 7852] dup_rows_per_dst=[122, 8, 78, 252, 82, 10, 6130, 2038] combine_rows_per_src=[7924, 8140, 8184, 7860, 8164, 500, 8192, 7852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=212000768 dispatch_meta_bytes=229376 combine_payload_bytes=212000768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212230144 ingress_bytes=230875136 link_bytes=230875136 token_bytes=424230912 weight_movement_bytes=18874368 total_nvlink_bytes=443105280 weight_to_token_ratio=0.044491 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.212131955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=210599936 dispatch_meta_bytes=229400 combine_payload_bytes=210599936 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=210829336 ingress_bytes=267223040 link_bytes=267223040 token_bytes=421429272 weight_movement_bytes=56623104 total_nvlink_bytes=478052376 weight_to_token_ratio=0.134360 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.560399687e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231841792 dispatch_meta_bytes=251968 combine_payload_bytes=231841792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232093760 ingress_bytes=288464896 link_bytes=288464896 token_bytes=463935552 weight_movement_bytes=56623104 total_nvlink_bytes=520558656 weight_to_token_ratio=0.122050 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.763928700e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64176 payload_rows_remote=55986 dup_rows=1360 dup_rows_remote=1360 dedup_ratio=0.020752 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 3146, 348, 0, 0, 0, 0, 2, 0, 2, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 19, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 547, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0] payload_rows_per_dst=[8192, 6992, 8040, 8192, 8190, 8192, 8186, 8192] dup_rows_per_dst=[0, 4, 162, 46, 0, 1094, 0, 54] combine_rows_per_src=[8192, 6992, 8040, 8192, 8190, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=190308352 dispatch_meta_bytes=241872 combine_payload_bytes=190308352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=190550224 ingress_bytes=209182720 link_bytes=209182720 token_bytes=380858576 weight_movement_bytes=18874368 total_nvlink_bytes=399732944 weight_to_token_ratio=0.049557 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.004285899e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200556544 dispatch_meta_bytes=222584 combine_payload_bytes=200556544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200779128 ingress_bytes=219430912 link_bytes=219430912 token_bytes=401335672 weight_movement_bytes=18874368 total_nvlink_bytes=420210040 weight_to_token_ratio=0.047029 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.102479032e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=229318656 dispatch_meta_bytes=229384 combine_payload_bytes=229318656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229548040 ingress_bytes=248193024 link_bytes=248193024 token_bytes=458866696 weight_movement_bytes=18874368 total_nvlink_bytes=477741064 weight_to_token_ratio=0.041133 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.378063437e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63716 payload_rows_remote=55524 dup_rows=1820 dup_rows_remote=1684 dedup_ratio=0.027771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 167, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7344, 8192, 8192, 8192, 7220, 8192, 8192] dup_rows_per_dst=[0, 418, 510, 136, 0, 756, 0, 0] combine_rows_per_src=[8192, 7344, 8192, 8192, 8192, 7220, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59362 payload_rows_remote=51170 dup_rows=6174 dup_rows_remote=774 dedup_ratio=0.094208 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 2127, 568, 0, 0, 0, 0, 5, 0, 5, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 159, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1473, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4078, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0] payload_rows_per_dst=[8192, 8192, 7316, 8192, 8166, 2956, 8164, 8184] dup_rows_per_dst=[170, 5400, 234, 328, 0, 0, 0, 42] combine_rows_per_src=[8192, 8192, 7316, 8192, 8166, 2956, 8164, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57992 payload_rows_remote=49800 dup_rows=7544 dup_rows_remote=7544 dedup_ratio=0.115112 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 2930, 78, 0, 0, 0, 0, 7, 0, 7, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 12, 7, 0, 5, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1158, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0] payload_rows_per_dst=[8192, 8192, 7304, 8192, 8056, 2330, 7684, 8042] dup_rows_per_dst=[0, 6746, 50, 328, 416, 0, 0, 4] combine_rows_per_src=[8192, 8192, 7304, 8192, 8056, 2330, 7684, 8042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58756 payload_rows_remote=50618 dup_rows=6780 dup_rows_remote=6556 dedup_ratio=0.103455 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 2075, 1176, 0, 0, 0, 0, 1, 0, 1, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 19, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 748, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8192, 8192, 8138, 8192, 8162, 1498, 8190, 8192] dup_rows_per_dst=[4, 6504, 224, 40, 0, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8138, 8192, 8162, 1498, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57840 payload_rows_remote=49648 dup_rows=7696 dup_rows_remote=424 dedup_ratio=0.117432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 159, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3636, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 500, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 402, 22, 0, 7272, 0, 0] combine_rows_per_src=[8192, 500, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58920 payload_rows_remote=50728 dup_rows=6616 dup_rows_remote=6606 dedup_ratio=0.100952 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 488, 0, 0, 0, 0, 4083, 0, 0, 258, 0, 4055, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 228, 0, 0, 2, 0, 3002, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1900, 8138, 8192, 8166, 8192, 7948, 8192, 8192] dup_rows_per_dst=[18, 1464, 0, 0, 568, 4556, 10, 0] combine_rows_per_src=[1900, 8138, 8192, 8166, 8192, 7948, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=209592320 dispatch_meta_bytes=207776 combine_payload_bytes=209592320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209800096 ingress_bytes=247341056 link_bytes=247341056 token_bytes=419392416 weight_movement_bytes=37748736 total_nvlink_bytes=457141152 weight_to_token_ratio=0.090008 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.369900299e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=227426304 dispatch_meta_bytes=228832 combine_payload_bytes=227426304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227655136 ingress_bytes=265175040 link_bytes=265175040 token_bytes=455081440 weight_movement_bytes=37748736 total_nvlink_bytes=492830176 weight_to_token_ratio=0.082949 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.540776759e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=203980800 dispatch_meta_bytes=229376 combine_payload_bytes=203980800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204210176 ingress_bytes=222855168 link_bytes=222855168 token_bytes=408190976 weight_movement_bytes=18874368 total_nvlink_bytes=427065344 weight_to_token_ratio=0.046239 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.135288568e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58162 payload_rows_remote=49970 dup_rows=7374 dup_rows_remote=7324 dedup_ratio=0.112518 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1861, 462, 0, 0, 0, 0, 3, 0, 3, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 24, 3, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1804, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0] payload_rows_per_dst=[7608, 4650, 8186, 5174, 7976, 8192, 8184, 8192] dup_rows_per_dst=[0, 8, 82, 56, 0, 6626, 552, 50] combine_rows_per_src=[7608, 4650, 8186, 5174, 7976, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63324 payload_rows_remote=55810 dup_rows=2212 dup_rows_remote=2204 dedup_ratio=0.033752 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 59, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 3014, 0, 0, 4092, 0, 0, 0, 0, 4094, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8188, 8192, 7118, 8188, 7514, 7744, 8188, 8192] dup_rows_per_dst=[42, 60, 120, 0, 8, 1214, 768, 0] combine_rows_per_src=[8188, 8192, 7118, 8188, 7514, 7744, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=207331328 dispatch_meta_bytes=228696 combine_payload_bytes=207331328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207560024 ingress_bytes=226205696 link_bytes=226205696 token_bytes=414891352 weight_movement_bytes=18874368 total_nvlink_bytes=433765720 weight_to_token_ratio=0.045492 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.167391679e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=203358208 dispatch_meta_bytes=200288 combine_payload_bytes=203358208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=203558496 ingress_bytes=241106944 link_bytes=241106944 token_bytes=406916704 weight_movement_bytes=37748736 total_nvlink_bytes=444665440 weight_to_token_ratio=0.092768 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.310168105e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=207781888 dispatch_meta_bytes=229336 combine_payload_bytes=207781888 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208011224 ingress_bytes=245530624 link_bytes=245530624 token_bytes=415793112 weight_movement_bytes=37748736 total_nvlink_bytes=453541848 weight_to_token_ratio=0.090787 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352553631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=228597760 dispatch_meta_bytes=232056 combine_payload_bytes=228597760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228829816 ingress_bytes=247472128 link_bytes=247472128 token_bytes=457427576 weight_movement_bytes=18874368 total_nvlink_bytes=476301944 weight_to_token_ratio=0.041262 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371156167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=204677120 dispatch_meta_bytes=229176 combine_payload_bytes=204677120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204906296 ingress_bytes=242425856 link_bytes=242425856 token_bytes=409583416 weight_movement_bytes=37748736 total_nvlink_bytes=447332152 weight_to_token_ratio=0.092164 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.322805271e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63696 payload_rows_remote=55504 dup_rows=1840 dup_rows_remote=1740 dedup_ratio=0.028076 router_tokens_per_expert=[0, 0, 0, 0, 4091, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 26, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 69, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1529, 0, 0, 2660, 0, 0, 4064, 0, 0, 0, 0, 4091, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4091, 0] payload_rows_per_dst=[8182, 8192, 7160, 8182, 7916, 7690, 8182, 8192] dup_rows_per_dst=[66, 100, 116, 2, 20, 698, 838, 0] combine_rows_per_src=[8182, 8192, 7160, 8182, 7916, 7690, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64180 payload_rows_remote=55988 dup_rows=1356 dup_rows_remote=1356 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 3575, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 7612, 7422, 8192, 8192] dup_rows_per_dst=[584, 60, 256, 0, 0, 94, 362, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 7612, 7422, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53340 payload_rows_remote=45162 dup_rows=12196 dup_rows_remote=11986 dedup_ratio=0.186096 router_tokens_per_expert=[0, 0, 0, 0, 4089, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 280, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2219, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 38, 0, 0, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1811, 0, 0, 3129, 0, 0, 4064, 0, 0, 0, 0, 4089, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0] payload_rows_per_dst=[8178, 8192, 7544, 8178, 8192, 3780, 1084, 8192] dup_rows_per_dst=[210, 858, 3106, 10, 8006, 6, 0, 0] combine_rows_per_src=[8178, 8192, 7544, 8178, 8192, 3780, 1084, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62532 payload_rows_remote=54954 dup_rows=3004 dup_rows_remote=2368 dedup_ratio=0.045837 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 332, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 3189, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 734, 0, 0, 3539, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8190, 8192, 7578, 8190, 6380, 7620, 8190, 8192] dup_rows_per_dst=[104, 32, 636, 2, 4, 928, 1298, 0] combine_rows_per_src=[8190, 8192, 7578, 8190, 6380, 7620, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61874 payload_rows_remote=54316 dup_rows=3662 dup_rows_remote=3232 dedup_ratio=0.055878 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 0, 3638, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8182, 8192, 5174, 7558, 8192, 8192] dup_rows_per_dst=[1010, 160, 946, 0, 0, 430, 1116, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 5174, 7558, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=227344384 dispatch_meta_bytes=228976 combine_payload_bytes=227344384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227573360 ingress_bytes=246218752 link_bytes=246218752 token_bytes=454917744 weight_movement_bytes=18874368 total_nvlink_bytes=473792112 weight_to_token_ratio=0.041490 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.359146934e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59718 payload_rows_remote=51526 dup_rows=5818 dup_rows_remote=5498 dedup_ratio=0.088776 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 2398, 8192, 8168] dup_rows_per_dst=[34, 4, 0, 2, 0, 0, 320, 5458] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 2398, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=229326848 dispatch_meta_bytes=229376 combine_payload_bytes=229326848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229556224 ingress_bytes=248201216 link_bytes=248201216 token_bytes=458883072 weight_movement_bytes=18874368 total_nvlink_bytes=477757440 weight_to_token_ratio=0.041131 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.378141929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=184983552 dispatch_meta_bytes=228592 combine_payload_bytes=184983552 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=185212144 ingress_bytes=260481024 link_bytes=260481024 token_bytes=370195696 weight_movement_bytes=75497472 total_nvlink_bytes=445693168 weight_to_token_ratio=0.203939 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.495801007e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61494 payload_rows_remote=53464 dup_rows=4042 dup_rows_remote=4042 dedup_ratio=0.061676 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 917, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2219, 4093, 0, 4079, 4013, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8078, 8184, 8192, 8190, 8030, 8186, 8192, 4442] dup_rows_per_dst=[0, 0, 2216, 1822, 0, 0, 4, 0] combine_rows_per_src=[8078, 8184, 8192, 8190, 8030, 8186, 8192, 4442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57712 payload_rows_remote=49522 dup_rows=7824 dup_rows_remote=7822 dedup_ratio=0.119385 router_tokens_per_expert=[0, 0, 0, 0, 4093, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 144, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3003, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 110, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 279, 0, 0, 3706, 0, 0, 4080, 0, 0, 0, 0, 4092, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[6172, 5936, 6084, 8186, 7128, 7830, 8186, 8190] dup_rows_per_dst=[32, 924, 154, 22, 12, 3952, 2726, 2] combine_rows_per_src=[6172, 5936, 6084, 8186, 7128, 7830, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222478336 dispatch_meta_bytes=230192 combine_payload_bytes=222478336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=222708528 ingress_bytes=279101440 link_bytes=279101440 token_bytes=445186864 weight_movement_bytes=56623104 total_nvlink_bytes=501809968 weight_to_token_ratio=0.127190 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=20992 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.674212672e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=225091584 dispatch_meta_bytes=229288 combine_payload_bytes=225091584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225320872 ingress_bytes=243965952 link_bytes=243965952 token_bytes=450412456 weight_movement_bytes=18874368 total_nvlink_bytes=469286824 weight_to_token_ratio=0.041905 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.337561713e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=211050496 dispatch_meta_bytes=228096 combine_payload_bytes=211050496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211278592 ingress_bytes=229924864 link_bytes=229924864 token_bytes=422329088 weight_movement_bytes=18874368 total_nvlink_bytes=441203456 weight_to_token_ratio=0.044691 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.203026917e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=218988544 dispatch_meta_bytes=230024 combine_payload_bytes=218988544 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=219218568 ingress_bytes=256737280 link_bytes=256737280 token_bytes=438207112 weight_movement_bytes=37748736 total_nvlink_bytes=475955848 weight_to_token_ratio=0.086144 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.459930294e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=202842112 dispatch_meta_bytes=229376 combine_payload_bytes=202842112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203071488 ingress_bytes=221716480 link_bytes=221716480 token_bytes=405913600 weight_movement_bytes=18874368 total_nvlink_bytes=424787968 weight_to_token_ratio=0.046498 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.124378220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63864 payload_rows_remote=55746 dup_rows=1672 dup_rows_remote=1672 dedup_ratio=0.025513 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4091, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 25, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 608, 0, 0, 2, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3497, 4091, 0, 4060, 4012, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[8006, 8118, 8192, 8136, 8034, 8182, 8192, 7004] dup_rows_per_dst=[0, 0, 412, 1224, 0, 0, 36, 0] combine_rows_per_src=[8006, 8118, 8192, 8136, 8034, 8182, 8192, 7004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57702 payload_rows_remote=49608 dup_rows=7834 dup_rows_remote=7152 dedup_ratio=0.119537 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 4078, 0, 3982, 4054, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8192, 8094, 8108, 8156, 8192, 586] dup_rows_per_dst=[0, 0, 7152, 682, 0, 0, 0, 0] combine_rows_per_src=[8192, 8182, 8192, 8094, 8108, 8156, 8192, 586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61948 payload_rows_remote=53842 dup_rows=3588 dup_rows_remote=3060 dedup_ratio=0.054749 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 28, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 145, 0, 0, 6, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3692, 4089, 0, 4083, 3774, 0, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[8106, 8178, 8006, 8190, 8192, 8178, 8114, 4984] dup_rows_per_dst=[528, 280, 652, 308, 908, 898, 14, 0] combine_rows_per_src=[8106, 8178, 8006, 8190, 8192, 8178, 8114, 4984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61822 payload_rows_remote=53630 dup_rows=3714 dup_rows_remote=1374 dedup_ratio=0.056671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 801, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2354, 4094, 0, 3914, 4060, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8168, 8188, 8192, 8062, 8122, 8188, 8192, 4710] dup_rows_per_dst=[0, 0, 2340, 1372, 0, 0, 2, 0] combine_rows_per_src=[8168, 8188, 8192, 8062, 8122, 8188, 8192, 4710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228335616 dispatch_meta_bytes=229672 combine_payload_bytes=228335616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228565288 ingress_bytes=266084352 link_bytes=266084352 token_bytes=456900904 weight_movement_bytes=37748736 total_nvlink_bytes=494649640 weight_to_token_ratio=0.082619 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.549489339e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64418 payload_rows_remote=56232 dup_rows=1118 dup_rows_remote=1118 dedup_ratio=0.017059 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 4093, 0, 3902, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8154, 7456, 7868, 8190, 8186, 8192, 8182] dup_rows_per_dst=[0, 0, 172, 164, 0, 0, 16, 766] combine_rows_per_src=[8190, 8154, 7456, 7868, 8190, 8186, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59160 payload_rows_remote=50968 dup_rows=6376 dup_rows_remote=6376 dedup_ratio=0.097290 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 1183, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2394, 7616, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[410, 0, 888, 2062, 0, 3016, 0, 0] combine_rows_per_src=[8192, 2394, 7616, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=203194368 dispatch_meta_bytes=227040 combine_payload_bytes=203194368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203421408 ingress_bytes=222068736 link_bytes=222068736 token_bytes=406615776 weight_movement_bytes=18874368 total_nvlink_bytes=425490144 weight_to_token_ratio=0.046418 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.127753364e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=220536832 dispatch_meta_bytes=227608 combine_payload_bytes=220536832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220764440 ingress_bytes=239411200 link_bytes=239411200 token_bytes=441301272 weight_movement_bytes=18874368 total_nvlink_bytes=460175640 weight_to_token_ratio=0.042770 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293920321e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65000 payload_rows_remote=56808 dup_rows=536 dup_rows_remote=532 dedup_ratio=0.008179 router_tokens_per_expert=[0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 2, 3839, 0, 0, 2, 2, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[8192, 7682, 8188, 8184, 8192, 8188, 8186, 8188] dup_rows_per_dst=[4, 12, 504, 0, 4, 0, 0, 12] combine_rows_per_src=[8192, 7682, 8188, 8184, 8192, 8188, 8186, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50570 payload_rows_remote=42378 dup_rows=14966 dup_rows_remote=54 dedup_ratio=0.228363 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 20, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 76, 0, 0, 1, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3582, 4093, 0, 3874, 3877, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8110, 8078, 1892, 160, 7760, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 40, 6, 0, 2, 6, 14912] combine_rows_per_src=[8110, 8078, 1892, 160, 7760, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219668480 dispatch_meta_bytes=220016 combine_payload_bytes=219668480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=219888496 ingress_bytes=276291584 link_bytes=276291584 token_bytes=439556976 weight_movement_bytes=56623104 total_nvlink_bytes=496180080 weight_to_token_ratio=0.128819 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.647290014e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=230326272 dispatch_meta_bytes=229400 combine_payload_bytes=230326272 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230555672 ingress_bytes=268075008 link_bytes=268075008 token_bytes=460881944 weight_movement_bytes=37748736 total_nvlink_bytes=498630680 weight_to_token_ratio=0.081905 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.568562826e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=208764928 dispatch_meta_bytes=229376 combine_payload_bytes=208764928 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=208994304 ingress_bytes=265388032 link_bytes=265388032 token_bytes=417759232 weight_movement_bytes=56623104 total_nvlink_bytes=474382336 weight_to_token_ratio=0.135540 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.542817544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232685568 dispatch_meta_bytes=229360 combine_payload_bytes=232685568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232914928 ingress_bytes=251559936 link_bytes=251559936 token_bytes=465600496 weight_movement_bytes=18874368 total_nvlink_bytes=484474864 weight_to_token_ratio=0.040538 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410323532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63580 payload_rows_remote=55842 dup_rows=1956 dup_rows_remote=1554 dedup_ratio=0.029846 router_tokens_per_expert=[0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4091, 5, 3303, 0, 0, 5, 5, 0, 0, 0, 0, 0, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 5, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 31, 0, 0] payload_rows_per_dst=[8186, 7738, 8182, 8154, 8192, 8182, 8148, 6798] dup_rows_per_dst=[10, 402, 1534, 0, 10, 0, 0, 0] combine_rows_per_src=[8186, 7738, 8182, 8154, 8192, 8182, 8148, 6798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=173580288 dispatch_meta_bytes=169728 combine_payload_bytes=173580288 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=173750016 ingress_bytes=286826496 link_bytes=286826496 token_bytes=347330304 weight_movement_bytes=113246208 total_nvlink_bytes=460576512 weight_to_token_ratio=0.326048 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=14976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.748230357e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65414 payload_rows_remote=57222 dup_rows=122 dup_rows_remote=122 dedup_ratio=0.001862 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 0, 0] payload_rows_per_dst=[8192, 8070, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 114] combine_rows_per_src=[8192, 8070, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57100 payload_rows_remote=48962 dup_rows=8436 dup_rows_remote=8386 dedup_ratio=0.128723 router_tokens_per_expert=[0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4089, 7, 3823, 0, 0, 2, 7, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 7, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8138, 8192, 8178, 8152, 8154, 8188, 8098, 0] dup_rows_per_dst=[50, 7726, 646, 0, 14, 0, 0, 0] combine_rows_per_src=[8138, 8192, 8178, 8152, 8154, 8188, 8098, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=228728832 dispatch_meta_bytes=229584 combine_payload_bytes=228728832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228958416 ingress_bytes=266477568 link_bytes=266477568 token_bytes=457687248 weight_movement_bytes=37748736 total_nvlink_bytes=495435984 weight_to_token_ratio=0.082477 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.553256941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64944 payload_rows_remote=56754 dup_rows=592 dup_rows_remote=16 dedup_ratio=0.009033 router_tokens_per_expert=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 3805, 0, 0, 0, 1, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8186, 7614, 8190, 8188, 8192, 8192, 8192, 8190] dup_rows_per_dst=[2, 2, 576, 0, 2, 0, 0, 10] combine_rows_per_src=[8186, 7614, 8190, 8188, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64822 payload_rows_remote=56630 dup_rows=714 dup_rows_remote=714 dedup_ratio=0.010895 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 314, 0, 0] payload_rows_per_dst=[8192, 7484, 8192, 8186, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 86, 0, 0, 0, 0, 628] combine_rows_per_src=[8192, 7484, 8192, 8186, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=234381312 dispatch_meta_bytes=229376 combine_payload_bytes=234381312 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234610688 ingress_bytes=291004416 link_bytes=291004416 token_bytes=468992000 weight_movement_bytes=56623104 total_nvlink_bytes=525615104 weight_to_token_ratio=0.120734 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.788261131e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64810 payload_rows_remote=56618 dup_rows=726 dup_rows_remote=704 dedup_ratio=0.011078 router_tokens_per_expert=[0, 0, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 712, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8186, 8192, 8182, 8192, 7570, 8104, 8192, 8192] dup_rows_per_dst=[8, 62, 168, 0, 0, 466, 22, 0] combine_rows_per_src=[8186, 8192, 8182, 8192, 7570, 8104, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200548352 dispatch_meta_bytes=229392 combine_payload_bytes=200548352 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=200777744 ingress_bytes=238297088 link_bytes=238297088 token_bytes=401326096 weight_movement_bytes=37748736 total_nvlink_bytes=439074832 weight_to_token_ratio=0.094060 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283245448e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64136 payload_rows_remote=55944 dup_rows=1400 dup_rows_remote=1320 dedup_ratio=0.021362 router_tokens_per_expert=[0, 0, 0, 826, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4094, 31, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3553, 0, 0, 3905, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2, 0] payload_rows_per_dst=[8154, 7986, 8192, 8192, 8192, 7106, 8192, 8122] dup_rows_per_dst=[1312, 0, 8, 0, 80, 0, 0, 0] combine_rows_per_src=[8154, 7986, 8192, 8192, 8192, 7106, 8192, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61798 payload_rows_remote=53612 dup_rows=3738 dup_rows_remote=506 dedup_ratio=0.057037 router_tokens_per_expert=[0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 3, 4002, 0, 0, 0, 3, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8188, 8010, 4722, 8138, 8192, 8192, 8170, 8186] dup_rows_per_dst=[6, 14, 144, 150, 8, 18, 166, 3232] combine_rows_per_src=[8188, 8010, 4722, 8138, 8192, 8192, 8170, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=232464384 dispatch_meta_bytes=227080 combine_payload_bytes=232464384 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=232691464 ingress_bytes=289087488 link_bytes=289087488 token_bytes=465155848 weight_movement_bytes=56623104 total_nvlink_bytes=521778952 weight_to_token_ratio=0.121729 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769894070e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=231956480 dispatch_meta_bytes=229376 combine_payload_bytes=231956480 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232185856 ingress_bytes=269705216 link_bytes=269705216 token_bytes=464142336 weight_movement_bytes=37748736 total_nvlink_bytes=501891072 weight_to_token_ratio=0.081330 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.584182676e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=231907328 dispatch_meta_bytes=229288 combine_payload_bytes=231907328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232136616 ingress_bytes=269656064 link_bytes=269656064 token_bytes=464043944 weight_movement_bytes=37748736 total_nvlink_bytes=501792680 weight_to_token_ratio=0.081347 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.583711726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=229146624 dispatch_meta_bytes=229056 combine_payload_bytes=229146624 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=229375680 ingress_bytes=323518464 link_bytes=323518464 token_bytes=458522304 weight_movement_bytes=94371840 total_nvlink_bytes=552894144 weight_to_token_ratio=0.205817 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.099794741e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61416 payload_rows_remote=53430 dup_rows=4120 dup_rows_remote=4120 dedup_ratio=0.062866 router_tokens_per_expert=[0, 0, 0, 2447, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 4091, 36, 27, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 0, 3635, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 5, 0] payload_rows_per_dst=[8192, 7986, 8192, 8190, 8192, 4412, 8192, 8060] dup_rows_per_dst=[3984, 0, 0, 0, 136, 0, 0, 0] combine_rows_per_src=[8192, 7986, 8192, 8190, 8192, 4412, 8192, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=219594752 dispatch_meta_bytes=216472 combine_payload_bytes=219594752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219811224 ingress_bytes=257343488 link_bytes=257343488 token_bytes=439405976 weight_movement_bytes=37748736 total_nvlink_bytes=477154712 weight_to_token_ratio=0.085909 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.465738681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65164 payload_rows_remote=56972 dup_rows=372 dup_rows_remote=372 dedup_ratio=0.005676 router_tokens_per_expert=[0, 0, 0, 1617, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 31, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 2547, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0] payload_rows_per_dst=[8098, 8192, 8192, 8192, 8192, 7932, 8192, 8174] dup_rows_per_dst=[248, 0, 24, 0, 100, 0, 0, 0] combine_rows_per_src=[8098, 8192, 8192, 8192, 8192, 7932, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56312 payload_rows_remote=48122 dup_rows=9224 dup_rows_remote=6688 dedup_ratio=0.140747 router_tokens_per_expert=[0, 0, 0, 2142, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4088, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 0, 0, 0, 0, 4089, 27, 4, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 3208, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3966, 7, 0] payload_rows_per_dst=[8190, 8192, 1938, 8180, 7300, 6366, 8192, 7954] dup_rows_per_dst=[2536, 6644, 0, 14, 26, 4, 0, 0] combine_rows_per_src=[8190, 8192, 1938, 8180, 7300, 6366, 8192, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=218849280 dispatch_meta_bytes=230200 combine_payload_bytes=218849280 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219079480 ingress_bytes=275472384 link_bytes=275472384 token_bytes=437928760 weight_movement_bytes=56623104 total_nvlink_bytes=494551864 weight_to_token_ratio=0.129298 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.639440843e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233357312 dispatch_meta_bytes=229376 combine_payload_bytes=233357312 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233586688 ingress_bytes=271106048 link_bytes=271106048 token_bytes=466944000 weight_movement_bytes=37748736 total_nvlink_bytes=504692736 weight_to_token_ratio=0.080842 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597604759e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64912 payload_rows_remote=56720 dup_rows=624 dup_rows_remote=624 dedup_ratio=0.009521 router_tokens_per_expert=[0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 124, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 3514, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0] payload_rows_per_dst=[7904, 8138, 8192, 8192, 8192, 7932, 8192, 8170] dup_rows_per_dst=[306, 0, 0, 0, 318, 0, 0, 0] combine_rows_per_src=[7904, 8138, 8192, 8192, 8192, 7932, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=197107712 dispatch_meta_bytes=219240 combine_payload_bytes=197107712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197326952 ingress_bytes=215982080 link_bytes=215982080 token_bytes=394434664 weight_movement_bytes=18874368 total_nvlink_bytes=413309032 weight_to_token_ratio=0.047852 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.069434021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65156 payload_rows_remote=57132 dup_rows=380 dup_rows_remote=380 dedup_ratio=0.005798 router_tokens_per_expert=[0, 0, 0, 1136, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 12, 0, 0, 4096, 86, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 2887, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0] payload_rows_per_dst=[8008, 8192, 8188, 8192, 8192, 8024, 8192, 8168] dup_rows_per_dst=[144, 0, 30, 0, 206, 0, 0, 0] combine_rows_per_src=[8008, 8192, 8188, 8192, 8192, 8024, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57808 payload_rows_remote=49616 dup_rows=7728 dup_rows_remote=7728 dedup_ratio=0.117920 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3687, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8174, 2430, 7096, 8192, 7374, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4162, 0, 0, 0, 3566] combine_rows_per_src=[8158, 8174, 2430, 7096, 8192, 7374, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64028 payload_rows_remote=57086 dup_rows=1508 dup_rows_remote=1508 dedup_ratio=0.023010 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 3976, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 1, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 3469, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7956, 8190, 8188, 8188, 6942, 8184, 8188, 8192] dup_rows_per_dst=[82, 112, 0, 4, 0, 636, 636, 38] combine_rows_per_src=[7956, 8190, 8188, 8188, 6942, 8184, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53162 payload_rows_remote=45126 dup_rows=12374 dup_rows_remote=11712 dedup_ratio=0.188812 router_tokens_per_expert=[0, 0, 0, 1450, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4091, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 0, 0, 0, 0, 4093, 153, 33, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 3552, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3, 0] payload_rows_per_dst=[2912, 1982, 8188, 8192, 7538, 8124, 8190, 8036] dup_rows_per_dst=[4, 0, 2, 0, 354, 11350, 2, 662] combine_rows_per_src=[2912, 1982, 8188, 8192, 7538, 8124, 8190, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=232325120 dispatch_meta_bytes=229376 combine_payload_bytes=232325120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232554496 ingress_bytes=251199488 link_bytes=251199488 token_bytes=464879616 weight_movement_bytes=18874368 total_nvlink_bytes=483753984 weight_to_token_ratio=0.040601 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406869896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234012672 dispatch_meta_bytes=230048 combine_payload_bytes=234012672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234242720 ingress_bytes=271761408 link_bytes=271761408 token_bytes=468255392 weight_movement_bytes=37748736 total_nvlink_bytes=506004128 weight_to_token_ratio=0.080616 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24576 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603884096e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=203227136 dispatch_meta_bytes=229376 combine_payload_bytes=203227136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203456512 ingress_bytes=222101504 link_bytes=222101504 token_bytes=406683648 weight_movement_bytes=18874368 total_nvlink_bytes=425558016 weight_to_token_ratio=0.046410 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.128067331e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233824256 dispatch_meta_bytes=234376 combine_payload_bytes=233824256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234058632 ingress_bytes=290447360 link_bytes=290447360 token_bytes=467882888 weight_movement_bytes=56623104 total_nvlink_bytes=524505992 weight_to_token_ratio=0.121020 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.782923695e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53904 payload_rows_remote=45712 dup_rows=11632 dup_rows_remote=11610 dedup_ratio=0.177490 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3947, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4074, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 0, 0, 0, 0, 0, 3915, 0, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[7904, 8192, 8182, 8192, 8040, 2822, 2380, 8192] dup_rows_per_dst=[10, 22, 0, 7544, 3820, 174, 8, 54] combine_rows_per_src=[7904, 8192, 8182, 8192, 8040, 2822, 2380, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=184836096 dispatch_meta_bytes=227352 combine_payload_bytes=184836096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185063448 ingress_bytes=203710464 link_bytes=203710464 token_bytes=369899544 weight_movement_bytes=18874368 total_nvlink_bytes=388773912 weight_to_token_ratio=0.051026 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.951853434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59156 payload_rows_remote=50964 dup_rows=6380 dup_rows_remote=6380 dedup_ratio=0.097351 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 963, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8110, 8184, 8192, 8192, 1926, 8168, 8192, 8192] dup_rows_per_dst=[36, 0, 0, 0, 0, 4602, 1728, 14] combine_rows_per_src=[8110, 8184, 8192, 8192, 1926, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50918 payload_rows_remote=43682 dup_rows=14618 dup_rows_remote=11792 dedup_ratio=0.223053 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4078, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 3171, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 7, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[7236, 8178, 8178, 8192, 8192, 2824, 126, 7992] dup_rows_per_dst=[2826, 4, 0, 5160, 6600, 0, 0, 28] combine_rows_per_src=[7236, 8178, 8178, 8192, 8192, 2824, 126, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=187236352 dispatch_meta_bytes=229288 combine_payload_bytes=187236352 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=187465640 ingress_bytes=224985088 link_bytes=224985088 token_bytes=374701992 weight_movement_bytes=37748736 total_nvlink_bytes=412450728 weight_to_token_ratio=0.100743 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.155696414e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=208748544 dispatch_meta_bytes=229376 combine_payload_bytes=208748544 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=208977920 ingress_bytes=284246016 link_bytes=284246016 token_bytes=417726464 weight_movement_bytes=75497472 total_nvlink_bytes=493223936 weight_to_token_ratio=0.180734 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.723505468e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63556 payload_rows_remote=55366 dup_rows=1980 dup_rows_remote=1980 dedup_ratio=0.030212 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4010, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3208, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8022, 8186, 8190, 8192, 6418, 8166, 8190, 8192] dup_rows_per_dst=[4, 22, 0, 2, 0, 1082, 858, 12] combine_rows_per_src=[8022, 8186, 8190, 8192, 6418, 8166, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=178921472 dispatch_meta_bytes=221896 combine_payload_bytes=178921472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=179143368 ingress_bytes=197795840 link_bytes=197795840 token_bytes=358064840 weight_movement_bytes=18874368 total_nvlink_bytes=376939208 weight_to_token_ratio=0.052712 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.895182417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58130 payload_rows_remote=49938 dup_rows=7406 dup_rows_remote=704 dedup_ratio=0.113007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 533, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7924, 8186, 8192, 8186, 1066, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 6702, 698, 4] combine_rows_per_src=[7924, 8186, 8192, 8186, 1066, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59928 payload_rows_remote=51736 dup_rows=5608 dup_rows_remote=5608 dedup_ratio=0.085571 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2797, 0, 0] payload_rows_per_dst=[8190, 2590, 8188, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 5602] combine_rows_per_src=[8190, 2590, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63698 payload_rows_remote=55510 dup_rows=1838 dup_rows_remote=1740 dedup_ratio=0.028046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 31, 0, 0, 0, 16, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8086, 6474, 8192, 8192, 8188, 8190, 8188, 8188] dup_rows_per_dst=[0, 0, 986, 746, 98, 0, 0, 8] combine_rows_per_src=[8086, 6474, 8192, 8192, 8188, 8190, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56302 payload_rows_remote=48110 dup_rows=9234 dup_rows_remote=9184 dedup_ratio=0.140900 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3604, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 1, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 3388, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[7214, 8094, 8186, 68, 8180, 8186, 8182, 8192] dup_rows_per_dst=[4, 0, 36, 6, 6776, 2158, 204, 50] combine_rows_per_src=[7214, 8094, 8186, 68, 8180, 8186, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=226779136 dispatch_meta_bytes=229384 combine_payload_bytes=226779136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227008520 ingress_bytes=245653504 link_bytes=245653504 token_bytes=453787656 weight_movement_bytes=18874368 total_nvlink_bytes=472662024 weight_to_token_ratio=0.041593 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.353731006e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=204546048 dispatch_meta_bytes=202568 combine_payload_bytes=204546048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204748616 ingress_bytes=223420416 link_bytes=223420416 token_bytes=409294664 weight_movement_bytes=18874368 total_nvlink_bytes=428169032 weight_to_token_ratio=0.046114 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.140704496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=211910656 dispatch_meta_bytes=229376 combine_payload_bytes=211910656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=212140032 ingress_bytes=268533760 link_bytes=268533760 token_bytes=424050688 weight_movement_bytes=56623104 total_nvlink_bytes=480673792 weight_to_token_ratio=0.133529 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.572958361e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=227368960 dispatch_meta_bytes=229000 combine_payload_bytes=227368960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227597960 ingress_bytes=246243328 link_bytes=246243328 token_bytes=454966920 weight_movement_bytes=18874368 total_nvlink_bytes=473841288 weight_to_token_ratio=0.041485 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.359382410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59046 payload_rows_remote=50854 dup_rows=6490 dup_rows_remote=2916 dedup_ratio=0.099030 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 0, 5, 0, 0, 0, 0, 1787, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 4091, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 39, 0, 0, 0, 29, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4087, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4085, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0] payload_rows_per_dst=[8112, 8192, 8192, 1826, 8182, 8180, 8182, 8180] dup_rows_per_dst=[0, 3574, 2748, 0, 148, 0, 0, 20] combine_rows_per_src=[8112, 8192, 8192, 1826, 8182, 8180, 8182, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=197058560 dispatch_meta_bytes=229176 combine_payload_bytes=197058560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=197287736 ingress_bytes=215932928 link_bytes=215932928 token_bytes=394346296 weight_movement_bytes=18874368 total_nvlink_bytes=413220664 weight_to_token_ratio=0.047862 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.068963070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58524 payload_rows_remote=50868 dup_rows=7012 dup_rows_remote=558 dedup_ratio=0.106995 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 1780, 8192, 7656, 8192, 8168, 8192, 8192] dup_rows_per_dst=[0, 548, 6, 6454, 4, 0, 0, 0] combine_rows_per_src=[8152, 1780, 8192, 7656, 8192, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53090 payload_rows_remote=44916 dup_rows=12446 dup_rows_remote=11810 dedup_ratio=0.189911 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 4089, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 2000, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 4089, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 54, 0, 0, 0, 82, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4081, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[8174, 8192, 1984, 2824, 7408, 8190, 8178, 8140] dup_rows_per_dst=[636, 11606, 14, 0, 126, 0, 36, 28] combine_rows_per_src=[8174, 8192, 1984, 2824, 7408, 8190, 8178, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=208297984 dispatch_meta_bytes=215080 combine_payload_bytes=208297984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208513064 ingress_bytes=264921088 link_bytes=264921088 token_bytes=416811048 weight_movement_bytes=56623104 total_nvlink_bytes=473434152 weight_to_token_ratio=0.135848 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.538343516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=208355328 dispatch_meta_bytes=205704 combine_payload_bytes=208355328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208561032 ingress_bytes=227229696 link_bytes=227229696 token_bytes=416916360 weight_movement_bytes=18874368 total_nvlink_bytes=435790728 weight_to_token_ratio=0.045271 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19712 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.177203143e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61784 payload_rows_remote=53592 dup_rows=3752 dup_rows_remote=3422 dedup_ratio=0.057251 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 49, 0, 0, 0, 5, 0, 1, 0, 0, 4, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8180, 8192, 8192, 4470, 8190, 8182, 8190, 8188] dup_rows_per_dst=[0, 3300, 330, 0, 118, 0, 0, 4] combine_rows_per_src=[8180, 8192, 8192, 4470, 8190, 8182, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=183975936 dispatch_meta_bytes=226904 combine_payload_bytes=183975936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184202840 ingress_bytes=202850304 link_bytes=202850304 token_bytes=368178776 weight_movement_bytes=18874368 total_nvlink_bytes=387053144 weight_to_token_ratio=0.051264 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.943611804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60238 payload_rows_remote=52050 dup_rows=5298 dup_rows_remote=5298 dedup_ratio=0.080841 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8100, 2990, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 5286, 12, 0, 0, 0] combine_rows_per_src=[8100, 2990, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62422 payload_rows_remote=54240 dup_rows=3114 dup_rows_remote=3110 dedup_ratio=0.047516 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2554, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 212, 4093] payload_rows_per_dst=[8192, 8188, 8186, 8188, 8182, 5108, 8188, 8190] dup_rows_per_dst=[0, 136, 4, 2538, 4, 0, 8, 424] combine_rows_per_src=[8192, 8188, 8186, 8188, 8182, 5108, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57782 payload_rows_remote=49590 dup_rows=7754 dup_rows_remote=7754 dedup_ratio=0.118317 router_tokens_per_expert=[0, 0, 0, 358, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 123, 0, 0, 4096, 20, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 4002, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[5154, 8192, 8186, 8192, 8192, 3484, 8192, 8190] dup_rows_per_dst=[390, 32, 6204, 0, 326, 802, 0, 0] combine_rows_per_src=[5154, 8192, 8186, 8192, 8192, 3484, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61654 payload_rows_remote=53470 dup_rows=3882 dup_rows_remote=3862 dedup_ratio=0.059235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 2367, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 4093, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 34, 0, 0, 0, 32, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4082, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4089, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[7912, 4740, 8086, 8192, 8186, 8168, 8186, 8184] dup_rows_per_dst=[0, 0, 454, 3168, 142, 98, 0, 20] combine_rows_per_src=[7912, 4740, 8086, 8192, 8186, 8168, 8186, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=219512832 dispatch_meta_bytes=228056 combine_payload_bytes=219512832 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=219740888 ingress_bytes=295010304 link_bytes=295010304 token_bytes=439253720 weight_movement_bytes=75497472 total_nvlink_bytes=514751192 weight_to_token_ratio=0.171877 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.826643579e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213196800 dispatch_meta_bytes=229392 combine_payload_bytes=213196800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213426192 ingress_bytes=232071168 link_bytes=232071168 token_bytes=426622992 weight_movement_bytes=18874368 total_nvlink_bytes=445497360 weight_to_token_ratio=0.044241 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223591746e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=222167040 dispatch_meta_bytes=229400 combine_payload_bytes=222167040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=222396440 ingress_bytes=259915776 link_bytes=259915776 token_bytes=444563480 weight_movement_bytes=37748736 total_nvlink_bytes=482312216 weight_to_token_ratio=0.084912 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.490385079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61606 payload_rows_remote=53424 dup_rows=3930 dup_rows_remote=3830 dedup_ratio=0.059967 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 183, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 204, 4088] payload_rows_per_dst=[8192, 8182, 8166, 8182, 8130, 4378, 8190, 8186] dup_rows_per_dst=[0, 100, 10, 3090, 302, 0, 20, 408] combine_rows_per_src=[8192, 8182, 8166, 8182, 8130, 4378, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=203120640 dispatch_meta_bytes=229376 combine_payload_bytes=203120640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203350016 ingress_bytes=221995008 link_bytes=221995008 token_bytes=406470656 weight_movement_bytes=18874368 total_nvlink_bytes=425345024 weight_to_token_ratio=0.046435 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.127046938e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=219013120 dispatch_meta_bytes=229328 combine_payload_bytes=219013120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219242448 ingress_bytes=256761856 link_bytes=256761856 token_bytes=438255568 weight_movement_bytes=37748736 total_nvlink_bytes=476004304 weight_to_token_ratio=0.086134 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.460165769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65316 payload_rows_remote=57124 dup_rows=220 dup_rows_remote=194 dedup_ratio=0.003357 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7972, 8192, 8192] dup_rows_per_dst=[0, 10, 0, 26, 2, 0, 0, 182] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7972, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55652 payload_rows_remote=47466 dup_rows=9884 dup_rows_remote=9874 dedup_ratio=0.150818 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 4051, 0, 4086, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 0, 0, 79, 3974] payload_rows_per_dst=[8186, 8112, 8176, 8178, 8134, 3330, 8134, 3402] dup_rows_per_dst=[10, 24, 30, 9592, 46, 0, 28, 154] combine_rows_per_src=[8186, 8112, 8176, 8178, 8134, 3330, 8134, 3402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=218824704 dispatch_meta_bytes=229016 combine_payload_bytes=218824704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219053720 ingress_bytes=237699072 link_bytes=237699072 token_bytes=437878424 weight_movement_bytes=18874368 total_nvlink_bytes=456752792 weight_to_token_ratio=0.043104 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.277515553e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62848 payload_rows_remote=54656 dup_rows=2688 dup_rows_remote=2686 dedup_ratio=0.041016 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2761, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 509, 4088] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8192, 5522, 8192, 8178] dup_rows_per_dst=[0, 36, 2, 1608, 20, 0, 4, 1018] combine_rows_per_src=[8192, 8190, 8192, 8190, 8192, 5522, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=233979904 dispatch_meta_bytes=229272 combine_payload_bytes=233979904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234209176 ingress_bytes=271728640 link_bytes=271728640 token_bytes=468189080 weight_movement_bytes=37748736 total_nvlink_bytes=505937816 weight_to_token_ratio=0.080627 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=22400 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603570130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=194420736 dispatch_meta_bytes=229360 combine_payload_bytes=194420736 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=194650096 ingress_bytes=232169472 link_bytes=232169472 token_bytes=389070832 weight_movement_bytes=37748736 total_nvlink_bytes=426819568 weight_to_token_ratio=0.097023 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224533646e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60648 payload_rows_remote=52862 dup_rows=4888 dup_rows_remote=456 dedup_ratio=0.074585 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 4096] payload_rows_per_dst=[8192, 8192, 8190, 3714, 8190, 7786, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 114, 0, 4432, 0, 342] combine_rows_per_src=[8192, 8192, 8190, 3714, 8190, 7786, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=10 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 14, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8160, 8192, 8192, 8188, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 4, 2, 0, 28, 0, 0, 4] combine_rows_per_src=[8160, 8192, 8192, 8188, 8192, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52892 payload_rows_remote=44700 dup_rows=12644 dup_rows_remote=2882 dedup_ratio=0.192932 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1497, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7674, 8166, 8192, 3610, 5306, 3560, 8192, 8192] dup_rows_per_dst=[532, 0, 0, 0, 2270, 80, 9762, 0] combine_rows_per_src=[7674, 8166, 8192, 3610, 5306, 3560, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=223870976 dispatch_meta_bytes=229368 combine_payload_bytes=223870976 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=224100344 ingress_bytes=261619712 link_bytes=261619712 token_bytes=447971320 weight_movement_bytes=37748736 total_nvlink_bytes=485720056 weight_to_token_ratio=0.084266 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.506711355e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60634 payload_rows_remote=52466 dup_rows=4902 dup_rows_remote=4514 dedup_ratio=0.074799 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2161, 0, 0, 0, 0, 0, 4053, 0, 4092, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 0, 0, 27, 4081] payload_rows_per_dst=[8190, 7958, 8190, 3872, 7886, 8186, 8184, 8168] dup_rows_per_dst=[0, 18, 6, 0, 6, 4472, 12, 388] combine_rows_per_src=[8190, 7958, 8190, 3872, 7886, 8186, 8184, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65064 payload_rows_remote=56872 dup_rows=472 dup_rows_remote=462 dedup_ratio=0.007202 router_tokens_per_expert=[0, 0, 0, 0, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 219, 0, 4089, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088] payload_rows_per_dst=[7740, 8192, 8192, 8182, 8188, 8192, 8192, 8186] dup_rows_per_dst=[0, 10, 14, 0, 438, 0, 0, 10] combine_rows_per_src=[7740, 8192, 8192, 8182, 8188, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=216522752 dispatch_meta_bytes=213272 combine_payload_bytes=216522752 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=216736024 ingress_bytes=329768960 link_bytes=329768960 token_bytes=433258776 weight_movement_bytes=113246208 total_nvlink_bytes=546504984 weight_to_token_ratio=0.261382 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=22272 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.159683918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229264 combine_payload_bytes=234725376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234954640 ingress_bytes=272474112 link_bytes=272474112 token_bytes=469680016 weight_movement_bytes=37748736 total_nvlink_bytes=507428752 weight_to_token_ratio=0.080371 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610712876e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=183091200 dispatch_meta_bytes=190328 combine_payload_bytes=183091200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=183281528 ingress_bytes=201965568 link_bytes=201965568 token_bytes=366372728 weight_movement_bytes=18874368 total_nvlink_bytes=385247096 weight_to_token_ratio=0.051517 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.935134699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64834 payload_rows_remote=56656 dup_rows=702 dup_rows_remote=88 dedup_ratio=0.010712 router_tokens_per_expert=[0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4089] payload_rows_per_dst=[8178, 8192, 8192, 8178, 7568, 8192, 8178, 8156] dup_rows_per_dst=[614, 16, 14, 36, 0, 0, 0, 22] combine_rows_per_src=[8178, 8192, 8192, 8178, 7568, 8192, 8178, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=214900736 dispatch_meta_bytes=227920 combine_payload_bytes=214900736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215128656 ingress_bytes=252649472 link_bytes=252649472 token_bytes=430029392 weight_movement_bytes=37748736 total_nvlink_bytes=467778128 weight_to_token_ratio=0.087782 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420762929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=232947712 dispatch_meta_bytes=229336 combine_payload_bytes=232947712 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233177048 ingress_bytes=289570816 link_bytes=289570816 token_bytes=466124760 weight_movement_bytes=56623104 total_nvlink_bytes=522747864 weight_to_token_ratio=0.121476 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.774525081e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=10 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8178, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 6, 0, 6, 0, 0, 2] combine_rows_per_src=[8178, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=232062976 dispatch_meta_bytes=226976 combine_payload_bytes=232062976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232289952 ingress_bytes=250937344 link_bytes=250937344 token_bytes=464352928 weight_movement_bytes=18874368 total_nvlink_bytes=483227296 weight_to_token_ratio=0.040647 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404358161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59368 payload_rows_remote=51176 dup_rows=6168 dup_rows_remote=6168 dedup_ratio=0.094116 router_tokens_per_expert=[0, 2, 0, 2537, 0, 0, 2, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 754, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5078, 8192, 8192, 8172, 8192, 8192, 8188, 5162] dup_rows_per_dst=[8, 4444, 0, 204, 0, 4, 1508, 0] combine_rows_per_src=[5078, 8192, 8192, 8172, 8192, 8192, 8188, 5162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63264 payload_rows_remote=55072 dup_rows=2272 dup_rows_remote=2272 dedup_ratio=0.034668 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 5948, 8192, 8192, 8192, 8172, 8192, 8192] dup_rows_per_dst=[0, 0, 1776, 338, 158, 0, 0, 0] combine_rows_per_src=[8184, 5948, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229352 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235044840 ingress_bytes=291438592 link_bytes=291438592 token_bytes=469860328 weight_movement_bytes=56623104 total_nvlink_bytes=526483432 weight_to_token_ratio=0.120511 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792421192e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57234 dup_rows=112 dup_rows_remote=106 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092] payload_rows_per_dst=[8184, 8154, 8144, 8186, 8186, 8192, 8188, 8190] dup_rows_per_dst=[0, 6, 12, 0, 68, 0, 20, 6] combine_rows_per_src=[8184, 8154, 8144, 8186, 8186, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58574 payload_rows_remote=54076 dup_rows=6962 dup_rows_remote=6568 dedup_ratio=0.106232 router_tokens_per_expert=[0, 5, 0, 2690, 0, 0, 5, 0, 5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 0, 0, 4091, 0, 5, 4091, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 567, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2730, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4498, 8192, 8164, 8192, 8192, 7684, 5460] dup_rows_per_dst=[5400, 394, 0, 24, 0, 10, 1134, 0] combine_rows_per_src=[8192, 4498, 8192, 8164, 8192, 8192, 7684, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=209616896 dispatch_meta_bytes=229376 combine_payload_bytes=209616896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209846272 ingress_bytes=228491264 link_bytes=228491264 token_bytes=419463168 weight_movement_bytes=18874368 total_nvlink_bytes=438337536 weight_to_token_ratio=0.044996 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.189290867e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=225574912 dispatch_meta_bytes=229376 combine_payload_bytes=225574912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225804288 ingress_bytes=244449280 link_bytes=244449280 token_bytes=451379200 weight_movement_bytes=18874368 total_nvlink_bytes=470253568 weight_to_token_ratio=0.041815 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.342192724e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58218 payload_rows_remote=50026 dup_rows=7318 dup_rows_remote=4706 dedup_ratio=0.111664 router_tokens_per_expert=[0, 0, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6768, 8192, 8192, 8192, 8192, 8192, 8192, 2298] dup_rows_per_dst=[0, 3370, 0, 2612, 0, 0, 1336, 0] combine_rows_per_src=[6768, 8192, 8192, 8192, 8192, 8192, 8192, 2298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56834 payload_rows_remote=48642 dup_rows=8702 dup_rows_remote=682 dedup_ratio=0.132782 router_tokens_per_expert=[0, 7, 0, 3977, 0, 0, 7, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 4089, 0, 7, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 362, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8080, 8190, 8154, 724, 7110] dup_rows_per_dst=[8020, 658, 0, 6, 4, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8080, 8190, 8154, 724, 7110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234430464 dispatch_meta_bytes=229360 combine_payload_bytes=234430464 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234659824 ingress_bytes=272179200 link_bytes=272179200 token_bytes=469090288 weight_movement_bytes=37748736 total_nvlink_bytes=506839024 weight_to_token_ratio=0.080472 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.607887174e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=221495296 dispatch_meta_bytes=242576 combine_payload_bytes=221495296 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=221737872 ingress_bytes=296992768 link_bytes=296992768 token_bytes=443233168 weight_movement_bytes=75497472 total_nvlink_bytes=518730640 weight_to_token_ratio=0.170334 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.845638573e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=204906496 dispatch_meta_bytes=218928 combine_payload_bytes=204906496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205125424 ingress_bytes=223780864 link_bytes=223780864 token_bytes=410031920 weight_movement_bytes=18874368 total_nvlink_bytes=428906288 weight_to_token_ratio=0.046031 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26368 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.144158132e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57986 payload_rows_remote=49794 dup_rows=7550 dup_rows_remote=7548 dedup_ratio=0.115204 router_tokens_per_expert=[0, 1, 0, 2197, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 0, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 395, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5176, 5692, 8192, 8180, 8190, 8192, 8190, 6174] dup_rows_per_dst=[2780, 3662, 2, 312, 0, 4, 790, 0] combine_rows_per_src=[5176, 5692, 8192, 8180, 8190, 8192, 8190, 6174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=199237632 dispatch_meta_bytes=197296 combine_payload_bytes=199237632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=199434928 ingress_bytes=236986368 link_bytes=236986368 token_bytes=398672560 weight_movement_bytes=37748736 total_nvlink_bytes=436421296 weight_to_token_ratio=0.094686 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.270686774e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61694 payload_rows_remote=53566 dup_rows=3842 dup_rows_remote=3836 dedup_ratio=0.058624 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4060, 4093, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2997, 0, 0, 0, 0, 3911, 0, 0, 18, 5, 1, 0, 3981, 0, 2, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7966, 6966, 8192, 8128, 8190, 6824, 7236] dup_rows_per_dst=[4, 12, 0, 0, 6, 0, 2170, 1650] combine_rows_per_src=[8192, 7966, 6966, 8192, 8128, 8190, 6824, 7236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59250 payload_rows_remote=51058 dup_rows=6286 dup_rows_remote=6286 dedup_ratio=0.095917 router_tokens_per_expert=[0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7792, 6672, 8192, 8192, 8192, 8192, 8192, 3826] dup_rows_per_dst=[0, 1136, 0, 3430, 0, 0, 1368, 352] combine_rows_per_src=[7792, 6672, 8192, 8192, 8192, 8192, 8192, 3826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57298 payload_rows_remote=49106 dup_rows=8238 dup_rows_remote=8238 dedup_ratio=0.125702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3079, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 4096] payload_rows_per_dst=[8190, 8192, 8192, 52, 8096, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 154, 6158, 0, 1920] combine_rows_per_src=[8190, 8192, 8192, 52, 8096, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=203956224 dispatch_meta_bytes=229368 combine_payload_bytes=203956224 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=204185592 ingress_bytes=222830592 link_bytes=222830592 token_bytes=408141816 weight_movement_bytes=18874368 total_nvlink_bytes=427016184 weight_to_token_ratio=0.046245 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.135053093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=219406336 dispatch_meta_bytes=229608 combine_payload_bytes=219406336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219635944 ingress_bytes=276029440 link_bytes=276029440 token_bytes=439042280 weight_movement_bytes=56623104 total_nvlink_bytes=495665384 weight_to_token_ratio=0.128970 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.644778279e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60788 payload_rows_remote=52670 dup_rows=4748 dup_rows_remote=4712 dedup_ratio=0.072449 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4030, 4083, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 4055, 0, 0, 12, 0, 0, 0, 4058, 0, 5, 0, 0, 0, 2233, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8118, 7338, 8192, 8118, 8176, 8178, 4476] dup_rows_per_dst=[10, 36, 80, 0, 10, 0, 4612, 0] combine_rows_per_src=[8192, 8118, 7338, 8192, 8118, 8176, 8178, 4476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51496 payload_rows_remote=43310 dup_rows=14040 dup_rows_remote=578 dedup_ratio=0.214233 router_tokens_per_expert=[0, 3, 0, 4015, 0, 0, 3, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 0, 4093, 0, 3, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 279, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2764, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8036, 2474, 8190, 40, 8192, 8192, 8186, 8186] dup_rows_per_dst=[12, 0, 0, 0, 0, 6, 560, 13462] combine_rows_per_src=[8036, 2474, 8190, 40, 8192, 8192, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=209133568 dispatch_meta_bytes=229376 combine_payload_bytes=209133568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209362944 ingress_bytes=228007936 link_bytes=228007936 token_bytes=418496512 weight_movement_bytes=18874368 total_nvlink_bytes=437370880 weight_to_token_ratio=0.045100 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.184659856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62514 payload_rows_remote=54322 dup_rows=3022 dup_rows_remote=3022 dedup_ratio=0.046112 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 0, 0, 0, 0, 4081, 0, 0, 7, 11, 0, 0, 3634, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7268, 8170, 8192, 8184, 8192, 8192, 6124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3022, 0] combine_rows_per_src=[8192, 7268, 8170, 8192, 8184, 8192, 8192, 6124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=201138176 dispatch_meta_bytes=229376 combine_payload_bytes=201138176 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=201367552 ingress_bytes=220012544 link_bytes=220012544 token_bytes=402505728 weight_movement_bytes=18874368 total_nvlink_bytes=421380096 weight_to_token_ratio=0.046892 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.108051944e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57436 payload_rows_remote=49244 dup_rows=8100 dup_rows_remote=8082 dedup_ratio=0.123596 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 7, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3662, 4080, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 4049, 0, 0, 64, 6, 5, 0, 4044, 0, 7, 0, 0, 0, 2040, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8144, 8124, 8186, 8098, 8174, 6466, 2052] dup_rows_per_dst=[18, 2072, 4924, 0, 1064, 0, 22, 0] combine_rows_per_src=[8192, 8144, 8124, 8186, 8098, 8174, 6466, 2052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=215736320 dispatch_meta_bytes=229528 combine_payload_bytes=215736320 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=215965848 ingress_bytes=253485056 link_bytes=253485056 token_bytes=431702168 weight_movement_bytes=37748736 total_nvlink_bytes=469450904 weight_to_token_ratio=0.087442 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428769084e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=177397760 dispatch_meta_bytes=175552 combine_payload_bytes=177397760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=177573312 ingress_bytes=215146496 link_bytes=215146496 token_bytes=354971072 weight_movement_bytes=37748736 total_nvlink_bytes=392719808 weight_to_token_ratio=0.106343 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.061427866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=222502912 dispatch_meta_bytes=229376 combine_payload_bytes=222502912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222732288 ingress_bytes=241377280 link_bytes=241377280 token_bytes=445235200 weight_movement_bytes=18874368 total_nvlink_bytes=464109568 weight_to_token_ratio=0.042392 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312758332e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=201703424 dispatch_meta_bytes=229304 combine_payload_bytes=201703424 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=201932728 ingress_bytes=239452160 link_bytes=239452160 token_bytes=403636152 weight_movement_bytes=37748736 total_nvlink_bytes=441384888 weight_to_token_ratio=0.093522 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.294312779e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61864 payload_rows_remote=53676 dup_rows=3672 dup_rows_remote=3672 dedup_ratio=0.056030 router_tokens_per_expert=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 2264, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4070, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[4530, 8190, 8192, 8192, 8188, 8188, 8192, 8192] dup_rows_per_dst=[0, 14, 3646, 0, 0, 4, 4, 4] combine_rows_per_src=[4530, 8190, 8192, 8192, 8188, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59516 payload_rows_remote=52888 dup_rows=6020 dup_rows_remote=6020 dedup_ratio=0.091858 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4075, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 4069, 0, 0, 6, 0, 0, 0, 4040, 0, 1, 0, 0, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8080, 6628, 8192, 8156, 8192, 8190, 3886] dup_rows_per_dst=[2, 0, 0, 0, 2, 0, 6016, 0] combine_rows_per_src=[8192, 8080, 6628, 8192, 8156, 8192, 8190, 3886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58236 payload_rows_remote=50044 dup_rows=7300 dup_rows_remote=7300 dedup_ratio=0.111389 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7984, 8192, 8192, 8192, 8192, 1100, 8192] dup_rows_per_dst=[0, 0, 0, 0, 18, 0, 0, 7282] combine_rows_per_src=[8192, 7984, 8192, 8192, 8192, 8192, 1100, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=219856896 dispatch_meta_bytes=229392 combine_payload_bytes=219856896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220086288 ingress_bytes=238731264 link_bytes=238731264 token_bytes=439943184 weight_movement_bytes=18874368 total_nvlink_bytes=458817552 weight_to_token_ratio=0.042902 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287405509e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=94 dedup_ratio=0.001434 router_tokens_per_expert=[0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8098, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 94, 0, 0, 0] combine_rows_per_src=[8098, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=216629248 dispatch_meta_bytes=235632 combine_payload_bytes=216629248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216864880 ingress_bytes=235503616 link_bytes=235503616 token_bytes=433494128 weight_movement_bytes=18874368 total_nvlink_bytes=452368496 weight_to_token_ratio=0.043540 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.256479774e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61050 payload_rows_remote=52862 dup_rows=4486 dup_rows_remote=4398 dedup_ratio=0.068451 router_tokens_per_expert=[0, 0, 4091, 2, 0, 0, 0, 0, 0, 2283, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 44, 3982, 0, 0, 1866, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 5, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8188, 3742, 8192, 8182, 8178, 8192, 8192] dup_rows_per_dst=[4364, 88, 2, 4, 0, 10, 8, 10] combine_rows_per_src=[8184, 8188, 3742, 8192, 8182, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61670 payload_rows_remote=53478 dup_rows=3866 dup_rows_remote=142 dedup_ratio=0.058990 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3666, 4089, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 4091, 0, 0, 50, 0, 2, 0, 3962, 0, 3, 0, 1, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 7938, 6936, 8188, 7362, 8184, 6718, 8192] dup_rows_per_dst=[6, 6, 0, 10, 6, 42, 72, 3724] combine_rows_per_src=[8152, 7938, 6936, 8188, 7362, 8184, 6718, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61810 payload_rows_remote=53620 dup_rows=3726 dup_rows_remote=96 dedup_ratio=0.056854 router_tokens_per_expert=[0, 0, 4089, 6, 0, 0, 0, 0, 0, 1786, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4088, 0, 0, 2299, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 4089, 7, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8180, 4612, 8176, 8178, 8176, 8142, 8156] dup_rows_per_dst=[3630, 24, 2, 4, 36, 16, 0, 14] combine_rows_per_src=[8190, 8180, 4612, 8176, 8178, 8176, 8142, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=204980224 dispatch_meta_bytes=229376 combine_payload_bytes=204980224 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205209600 ingress_bytes=242728960 link_bytes=242728960 token_bytes=410189824 weight_movement_bytes=37748736 total_nvlink_bytes=447938560 weight_to_token_ratio=0.092027 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.325709465e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64326 payload_rows_remote=56134 dup_rows=1210 dup_rows_remote=1202 dedup_ratio=0.018463 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6984, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 1188, 8, 0, 0, 12, 0] combine_rows_per_src=[6984, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229376 combine_payload_bytes=234496000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234725376 ingress_bytes=253370368 link_bytes=253370368 token_bytes=469221376 weight_movement_bytes=18874368 total_nvlink_bytes=488095744 weight_to_token_ratio=0.040225 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427670200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=216522752 dispatch_meta_bytes=229040 combine_payload_bytes=216522752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216751792 ingress_bytes=254271488 link_bytes=254271488 token_bytes=433274544 weight_movement_bytes=37748736 total_nvlink_bytes=471023280 weight_to_token_ratio=0.087124 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.436304289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=219045888 dispatch_meta_bytes=214480 combine_payload_bytes=219045888 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=219260368 ingress_bytes=275668992 link_bytes=275668992 token_bytes=438306256 weight_movement_bytes=56623104 total_nvlink_bytes=494929360 weight_to_token_ratio=0.129186 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.641324644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63212 payload_rows_remote=56314 dup_rows=2324 dup_rows_remote=2324 dedup_ratio=0.035461 router_tokens_per_expert=[0, 3585, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 4094, 0, 2, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4094] payload_rows_per_dst=[7174, 8188, 8192, 8188, 6898, 8188, 8192, 8192] dup_rows_per_dst=[4, 16, 4, 0, 0, 1858, 392, 50] combine_rows_per_src=[7174, 8188, 8192, 8188, 6898, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=219627520 dispatch_meta_bytes=214864 combine_payload_bytes=219627520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219842384 ingress_bytes=257376256 link_bytes=257376256 token_bytes=439469904 weight_movement_bytes=37748736 total_nvlink_bytes=477218640 weight_to_token_ratio=0.085896 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.466052648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62334 payload_rows_remote=56810 dup_rows=3202 dup_rows_remote=2676 dedup_ratio=0.048859 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 4085, 0, 0, 950, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7664, 8192, 5524, 8192, 8190, 8188, 8192, 8192] dup_rows_per_dst=[2590, 78, 526, 0, 0, 2, 4, 2] combine_rows_per_src=[7664, 8192, 5524, 8192, 8190, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=229924864 dispatch_meta_bytes=229344 combine_payload_bytes=229924864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=230154208 ingress_bytes=267673600 link_bytes=267673600 token_bytes=460079072 weight_movement_bytes=37748736 total_nvlink_bytes=497827808 weight_to_token_ratio=0.082048 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.564716732e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=230662144 dispatch_meta_bytes=234552 combine_payload_bytes=230662144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230896696 ingress_bytes=249536512 link_bytes=249536512 token_bytes=461558840 weight_movement_bytes=18874368 total_nvlink_bytes=480433208 weight_to_token_ratio=0.040893 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390936078e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=232693760 dispatch_meta_bytes=237944 combine_payload_bytes=232693760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232931704 ingress_bytes=270442496 link_bytes=270442496 token_bytes=465625464 weight_movement_bytes=37748736 total_nvlink_bytes=503374200 weight_to_token_ratio=0.081071 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21632 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.591246931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64928 payload_rows_remote=56736 dup_rows=608 dup_rows_remote=608 dedup_ratio=0.009277 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7584, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 596, 0, 0, 0, 12, 0] combine_rows_per_src=[7584, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55660 payload_rows_remote=47468 dup_rows=9876 dup_rows_remote=8584 dedup_ratio=0.150696 router_tokens_per_expert=[0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[82, 7960, 8192, 6658, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 3270, 0, 0, 1292, 5314] combine_rows_per_src=[82, 7960, 8192, 6658, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61236 payload_rows_remote=53054 dup_rows=4300 dup_rows_remote=3814 dedup_ratio=0.065613 router_tokens_per_expert=[0, 3017, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3988, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3454, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 4091, 0, 5, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 4091] payload_rows_per_dst=[6504, 8182, 7986, 8182, 6918, 7080, 8192, 8192] dup_rows_per_dst=[868, 486, 10, 0, 2, 1842, 1012, 80] combine_rows_per_src=[6504, 8182, 7986, 8182, 6918, 7080, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62222 payload_rows_remote=54030 dup_rows=3314 dup_rows_remote=3308 dedup_ratio=0.050568 router_tokens_per_expert=[0, 0, 4093, 2, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4075, 0, 0, 1560, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4093, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[5100, 7992, 8192, 8184, 8186, 8184, 8192, 8192] dup_rows_per_dst=[0, 4, 3280, 0, 0, 14, 10, 6] combine_rows_per_src=[5100, 7992, 8192, 8184, 8186, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54894 payload_rows_remote=46702 dup_rows=10642 dup_rows_remote=6098 dedup_ratio=0.162384 router_tokens_per_expert=[0, 2264, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3616, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 4089, 0, 7, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 4089] payload_rows_per_dst=[8192, 8178, 8192, 8178, 7788, 3814, 8192, 2360] dup_rows_per_dst=[4544, 210, 14, 0, 5560, 0, 296, 18] combine_rows_per_src=[8192, 8178, 8192, 8178, 7788, 3814, 8192, 2360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=232390656 dispatch_meta_bytes=229376 combine_payload_bytes=232390656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232620032 ingress_bytes=289013760 link_bytes=289013760 token_bytes=465010688 weight_movement_bytes=56623104 total_nvlink_bytes=521633792 weight_to_token_ratio=0.121767 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.769187645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=194428928 dispatch_meta_bytes=224208 combine_payload_bytes=194428928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=194653136 ingress_bytes=232177664 link_bytes=232177664 token_bytes=389082064 weight_movement_bytes=37748736 total_nvlink_bytes=426830800 weight_to_token_ratio=0.097020 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224612138e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65498 payload_rows_remote=57306 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000580 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8154, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 22, 8, 8] combine_rows_per_src=[8192, 8192, 8192, 8192, 8154, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=217309184 dispatch_meta_bytes=227472 combine_payload_bytes=217309184 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=217536656 ingress_bytes=292806656 link_bytes=292806656 token_bytes=434845840 weight_movement_bytes=75497472 total_nvlink_bytes=510343312 weight_to_token_ratio=0.173619 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.805529308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64842 payload_rows_remote=56654 dup_rows=694 dup_rows_remote=694 dedup_ratio=0.010590 router_tokens_per_expert=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2, 0, 4063, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3787, 0, 2, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8186, 8126, 8192, 7580, 8188, 8190, 8188, 8192] dup_rows_per_dst=[28, 0, 0, 4, 0, 0, 654, 8] combine_rows_per_src=[8186, 8126, 8192, 7580, 8188, 8190, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=221306880 dispatch_meta_bytes=229352 combine_payload_bytes=221306880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221536232 ingress_bytes=240181248 link_bytes=240181248 token_bytes=442843112 weight_movement_bytes=18874368 total_nvlink_bytes=461717480 weight_to_token_ratio=0.042621 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.301298542e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=191291392 dispatch_meta_bytes=211200 combine_payload_bytes=191291392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=191502592 ingress_bytes=210165760 link_bytes=210165760 token_bytes=382793984 weight_movement_bytes=18874368 total_nvlink_bytes=401668352 weight_to_token_ratio=0.049307 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.013704904e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64762 payload_rows_remote=56586 dup_rows=774 dup_rows_remote=774 dedup_ratio=0.011810 router_tokens_per_expert=[0, 3857, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4095, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[7716, 8190, 8176, 8192, 7914, 8190, 8192, 8192] dup_rows_per_dst=[2, 12, 0, 0, 0, 536, 210, 14] combine_rows_per_src=[7716, 8190, 8176, 8192, 7914, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234725376 dispatch_meta_bytes=229376 combine_payload_bytes=234725376 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234954752 ingress_bytes=272474112 link_bytes=272474112 token_bytes=469680128 weight_movement_bytes=37748736 total_nvlink_bytes=507428864 weight_to_token_ratio=0.080371 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610712876e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=232054784 dispatch_meta_bytes=229392 combine_payload_bytes=232054784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232284176 ingress_bytes=250929152 link_bytes=250929152 token_bytes=464338960 weight_movement_bytes=18874368 total_nvlink_bytes=483213328 weight_to_token_ratio=0.040648 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404279670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=231776256 dispatch_meta_bytes=229440 combine_payload_bytes=231776256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232005696 ingress_bytes=250650624 link_bytes=250650624 token_bytes=463781952 weight_movement_bytes=18874368 total_nvlink_bytes=482656320 weight_to_token_ratio=0.040697 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401610951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65380 payload_rows_remote=57188 dup_rows=156 dup_rows_remote=14 dedup_ratio=0.002380 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8180, 8192, 8192, 8192, 8048, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 142, 0, 10] combine_rows_per_src=[8180, 8192, 8192, 8192, 8048, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60882 payload_rows_remote=56992 dup_rows=4654 dup_rows_remote=4590 dedup_ratio=0.071014 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1548, 0, 0, 0, 0, 2692, 0, 0, 2, 427, 0, 0, 3956, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7912, 8178, 8192, 8192, 8192, 3890, 8134] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 64, 4588] combine_rows_per_src=[8192, 7912, 8178, 8192, 8192, 8192, 3890, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63690 payload_rows_remote=56710 dup_rows=1846 dup_rows_remote=1846 dedup_ratio=0.028168 router_tokens_per_expert=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3490, 0, 4091, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3766, 0, 5, 0, 0, 0, 0, 5, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 5] payload_rows_per_dst=[8192, 6980, 8192, 7608, 8152, 8192, 8182, 8192] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 1812, 18] combine_rows_per_src=[8192, 6980, 8192, 7608, 8152, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57172 payload_rows_remote=48980 dup_rows=8364 dup_rows_remote=2714 dedup_ratio=0.127625 router_tokens_per_expert=[0, 2978, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 3838, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4093, 0, 3, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[5962, 7446, 8192, 8186, 7684, 6474, 5036, 8192] dup_rows_per_dst=[6, 52, 194, 34, 0, 2414, 14, 5650] combine_rows_per_src=[5962, 7446, 8192, 8186, 7684, 6474, 5036, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59388 payload_rows_remote=51200 dup_rows=6148 dup_rows_remote=6146 dedup_ratio=0.093811 router_tokens_per_expert=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 4089, 0, 4089, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3056, 0, 7, 0, 0, 0, 0, 7, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 7, 0, 0, 0, 0, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[8188, 8178, 8192, 6194, 8160, 8180, 4156, 8140] dup_rows_per_dst=[2, 6082, 0, 46, 0, 0, 4, 14] combine_rows_per_src=[8188, 8178, 8192, 6194, 8160, 8180, 4156, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=233439232 dispatch_meta_bytes=246328 combine_payload_bytes=233439232 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=233685560 ingress_bytes=308936704 link_bytes=308936704 token_bytes=467124792 weight_movement_bytes=75497472 total_nvlink_bytes=542622264 weight_to_token_ratio=0.161622 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.960079491e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234242048 dispatch_meta_bytes=228808 combine_payload_bytes=234242048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234470856 ingress_bytes=290865152 link_bytes=290865152 token_bytes=468712904 weight_movement_bytes=56623104 total_nvlink_bytes=525336008 weight_to_token_ratio=0.120806 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.786926772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63134 payload_rows_remote=54942 dup_rows=2402 dup_rows_remote=2402 dedup_ratio=0.036652 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 2895, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5790, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2392, 0, 0, 0, 0, 0, 10, 0] combine_rows_per_src=[8192, 5790, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=232284160 dispatch_meta_bytes=234224 combine_payload_bytes=232284160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232518384 ingress_bytes=288907264 link_bytes=288907264 token_bytes=464802544 weight_movement_bytes=56623104 total_nvlink_bytes=521425648 weight_to_token_ratio=0.121822 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.768167252e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62882 payload_rows_remote=54712 dup_rows=2654 dup_rows_remote=2454 dedup_ratio=0.040497 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 1221, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 2, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4094, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4080, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5588, 8192, 8170, 8188, 8192, 8168] dup_rows_per_dst=[8, 2446, 0, 0, 200, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5588, 8192, 8170, 8188, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65178 payload_rows_remote=56986 dup_rows=358 dup_rows_remote=358 dedup_ratio=0.005463 router_tokens_per_expert=[0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 4019, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3994, 0, 1, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8038, 8192, 7998, 8188, 8192, 8190, 8192] dup_rows_per_dst=[80, 0, 0, 2, 0, 0, 274, 2] combine_rows_per_src=[8188, 8038, 8192, 7998, 8188, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=209715200 dispatch_meta_bytes=229384 combine_payload_bytes=209715200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=209944584 ingress_bytes=285212672 link_bytes=285212672 token_bytes=419659784 weight_movement_bytes=75497472 total_nvlink_bytes=495157256 weight_to_token_ratio=0.179902 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.732767490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=200622080 dispatch_meta_bytes=206776 combine_payload_bytes=200622080 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=200828856 ingress_bytes=257245184 link_bytes=257245184 token_bytes=401450936 weight_movement_bytes=56623104 total_nvlink_bytes=458074040 weight_to_token_ratio=0.141046 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.464796781e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=225042432 dispatch_meta_bytes=229376 combine_payload_bytes=225042432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225271808 ingress_bytes=262791168 link_bytes=262791168 token_bytes=450314240 weight_movement_bytes=37748736 total_nvlink_bytes=488062976 weight_to_token_ratio=0.083828 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.517935670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=224100352 dispatch_meta_bytes=228664 combine_payload_bytes=224100352 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=224329016 ingress_bytes=299597824 link_bytes=299597824 token_bytes=448429368 weight_movement_bytes=75497472 total_nvlink_bytes=523926840 weight_to_token_ratio=0.168360 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.870598938e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=233414656 dispatch_meta_bytes=229376 combine_payload_bytes=233414656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233644032 ingress_bytes=252289024 link_bytes=252289024 token_bytes=467058688 weight_movement_bytes=18874368 total_nvlink_bytes=485933056 weight_to_token_ratio=0.040411 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.417309294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64054 payload_rows_remote=55862 dup_rows=1482 dup_rows_remote=1482 dedup_ratio=0.022614 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3221, 0, 0, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6716, 8192, 8186, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 8, 1466, 8, 0, 0, 0, 0] combine_rows_per_src=[6716, 8192, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63764 payload_rows_remote=55572 dup_rows=1772 dup_rows_remote=1772 dedup_ratio=0.027039 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 0, 0, 3227, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6454, 8192, 8182, 8168, 8192, 8192, 8192] dup_rows_per_dst=[1750, 0, 0, 0, 0, 0, 18, 4] combine_rows_per_src=[8192, 6454, 8192, 8182, 8168, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61694 payload_rows_remote=53502 dup_rows=3842 dup_rows_remote=364 dedup_ratio=0.058624 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 1732, 0, 0, 0, 7, 4091, 0, 0, 0, 0, 0, 0, 5, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 4091, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 4080, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5212, 8192, 7356, 8182, 8192, 8176] dup_rows_per_dst=[20, 3478, 0, 0, 340, 0, 0, 4] combine_rows_per_src=[8192, 8192, 5212, 8192, 7356, 8182, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=228810752 dispatch_meta_bytes=229376 combine_payload_bytes=228810752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229040128 ingress_bytes=247685120 link_bytes=247685120 token_bytes=457850880 weight_movement_bytes=18874368 total_nvlink_bytes=476725248 weight_to_token_ratio=0.041224 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373196951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60082 payload_rows_remote=51890 dup_rows=5454 dup_rows_remote=5426 dedup_ratio=0.083221 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 2448, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 7, 1461, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4089, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 4079, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2942, 8192, 8022, 8178, 8176, 8188] dup_rows_per_dst=[28, 5012, 0, 0, 404, 0, 0, 10] combine_rows_per_src=[8192, 8192, 2942, 8192, 8022, 8178, 8176, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=227622912 dispatch_meta_bytes=229376 combine_payload_bytes=227622912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227852288 ingress_bytes=246497280 link_bytes=246497280 token_bytes=455475200 weight_movement_bytes=18874368 total_nvlink_bytes=474349568 weight_to_token_ratio=0.041439 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.361815653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56110 payload_rows_remote=47918 dup_rows=9426 dup_rows_remote=9420 dedup_ratio=0.143829 router_tokens_per_expert=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4093, 0, 4093, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3482, 0, 3, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4080, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[10, 8186, 8192, 6988, 8168, 8186, 8188, 8192] dup_rows_per_dst=[0, 3492, 0, 4534, 142, 20, 1232, 6] combine_rows_per_src=[10, 8186, 8192, 6988, 8168, 8186, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=219144192 dispatch_meta_bytes=215464 combine_payload_bytes=219144192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=219359656 ingress_bytes=256892928 link_bytes=256892928 token_bytes=438503848 weight_movement_bytes=37748736 total_nvlink_bytes=476252584 weight_to_token_ratio=0.086085 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.461421637e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65116 payload_rows_remote=56924 dup_rows=420 dup_rows_remote=420 dedup_ratio=0.006409 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7772, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 290, 0, 0, 130, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7772, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60526 payload_rows_remote=52340 dup_rows=5010 dup_rows_remote=4962 dedup_ratio=0.076447 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 22, 2, 0, 0, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 3951, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 3908, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6078, 8190, 7906, 8186, 8136, 8118, 5720] dup_rows_per_dst=[2, 1314, 0, 0, 48, 3646, 0, 0] combine_rows_per_src=[8192, 6078, 8190, 7906, 8186, 8136, 8118, 5720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64218 payload_rows_remote=57174 dup_rows=1318 dup_rows_remote=1318 dedup_ratio=0.020111 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 647, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7044, 8192, 8028, 8190, 8192, 8188] dup_rows_per_dst=[4, 1294, 0, 0, 18, 0, 0, 2] combine_rows_per_src=[8192, 8192, 7044, 8192, 8028, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=212541440 dispatch_meta_bytes=229264 combine_payload_bytes=212541440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212770704 ingress_bytes=231415808 link_bytes=231415808 token_bytes=425312144 weight_movement_bytes=18874368 total_nvlink_bytes=444186512 weight_to_token_ratio=0.044378 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217312409e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=196272128 dispatch_meta_bytes=229352 combine_payload_bytes=196272128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196501480 ingress_bytes=215146496 link_bytes=215146496 token_bytes=392773608 weight_movement_bytes=18874368 total_nvlink_bytes=411647976 weight_to_token_ratio=0.048054 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.061427866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=233160704 dispatch_meta_bytes=229376 combine_payload_bytes=233160704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233390080 ingress_bytes=252035072 link_bytes=252035072 token_bytes=466550784 weight_movement_bytes=18874368 total_nvlink_bytes=485425152 weight_to_token_ratio=0.040455 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414876051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=214384640 dispatch_meta_bytes=229208 combine_payload_bytes=214384640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214613848 ingress_bytes=233259008 link_bytes=233259008 token_bytes=428998488 weight_movement_bytes=18874368 total_nvlink_bytes=447872856 weight_to_token_ratio=0.043996 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.234973044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234184704 dispatch_meta_bytes=233968 combine_payload_bytes=234184704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234418672 ingress_bytes=253059072 link_bytes=253059072 token_bytes=468603376 weight_movement_bytes=18874368 total_nvlink_bytes=487477744 weight_to_token_ratio=0.040278 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424687515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63910 payload_rows_remote=55718 dup_rows=1626 dup_rows_remote=402 dedup_ratio=0.024811 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6566, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 400, 1224, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 6566, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63222 payload_rows_remote=55030 dup_rows=2314 dup_rows_remote=2314 dedup_ratio=0.035309 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7292, 6778, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1226, 840, 0, 248, 0, 0, 0] combine_rows_per_src=[8192, 7292, 6778, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63088 payload_rows_remote=54896 dup_rows=2448 dup_rows_remote=62 dedup_ratio=0.037354 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 5, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 0, 0, 0, 5, 0, 0, 3921, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 5, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4091, 0, 3901, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7852, 8178, 7846, 8062, 6574] dup_rows_per_dst=[0, 2386, 0, 0, 26, 36, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7852, 8178, 7846, 8062, 6574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=228220928 dispatch_meta_bytes=224480 combine_payload_bytes=228220928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228445408 ingress_bytes=247095296 link_bytes=247095296 token_bytes=456666336 weight_movement_bytes=18874368 total_nvlink_bytes=475540704 weight_to_token_ratio=0.041331 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.367545548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=225402880 dispatch_meta_bytes=229376 combine_payload_bytes=225402880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225632256 ingress_bytes=244277248 link_bytes=244277248 token_bytes=451035136 weight_movement_bytes=18874368 total_nvlink_bytes=469909504 weight_to_token_ratio=0.041847 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.340544398e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59354 payload_rows_remote=51162 dup_rows=6182 dup_rows_remote=6178 dedup_ratio=0.094330 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 79, 7, 0, 0, 0, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4083, 0, 0, 0, 7, 0, 0, 3808, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 5, 0, 0, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4089, 0, 3844, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 7652, 7978, 4504, 7998, 6658] dup_rows_per_dst=[4, 2720, 64, 3370, 10, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8180, 7652, 7978, 4504, 7998, 6658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60182 payload_rows_remote=52012 dup_rows=5354 dup_rows_remote=5300 dedup_ratio=0.081696 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1586, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 3, 2564, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 4093, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 4077, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8120, 3180, 8192, 8192, 7952, 8186, 8190, 8170] dup_rows_per_dst=[12, 0, 5152, 0, 82, 36, 18, 54] combine_rows_per_src=[8120, 3180, 8192, 8192, 7952, 8186, 8190, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=224854016 dispatch_meta_bytes=219832 combine_payload_bytes=224854016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225073848 ingress_bytes=243728384 link_bytes=243728384 token_bytes=449927864 weight_movement_bytes=18874368 total_nvlink_bytes=468802232 weight_to_token_ratio=0.041950 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=18560 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.335285454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59728 payload_rows_remote=52000 dup_rows=5808 dup_rows_remote=5808 dedup_ratio=0.088623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7728, 8146, 3022, 8172, 8086] dup_rows_per_dst=[0, 5806, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 7728, 8146, 3022, 8172, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63094 payload_rows_remote=54902 dup_rows=2442 dup_rows_remote=2442 dedup_ratio=0.037262 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 1, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 3983, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 3856, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7968, 8130, 7718, 8174, 6528] dup_rows_per_dst=[0, 2436, 0, 0, 4, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7968, 8130, 7718, 8174, 6528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64388 payload_rows_remote=56494 dup_rows=1148 dup_rows_remote=1148 dedup_ratio=0.017517 router_tokens_per_expert=[0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3730, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4094] payload_rows_per_dst=[8188, 7464, 8188, 8192, 7894, 8078, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 6, 0, 0, 1136, 2] combine_rows_per_src=[8188, 7464, 8188, 8192, 7894, 8078, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=209559552 dispatch_meta_bytes=229360 combine_payload_bytes=209559552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209788912 ingress_bytes=228433920 link_bytes=228433920 token_bytes=419348464 weight_movement_bytes=18874368 total_nvlink_bytes=438222832 weight_to_token_ratio=0.045009 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.188741425e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=213041152 dispatch_meta_bytes=229248 combine_payload_bytes=213041152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213270400 ingress_bytes=231915520 link_bytes=231915520 token_bytes=426311552 weight_movement_bytes=18874368 total_nvlink_bytes=445185920 weight_to_token_ratio=0.044274 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222100403e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=212992000 dispatch_meta_bytes=231232 combine_payload_bytes=212992000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213223232 ingress_bytes=250740736 link_bytes=250740736 token_bytes=426215232 weight_movement_bytes=37748736 total_nvlink_bytes=463963968 weight_to_token_ratio=0.088567 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402474360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61472 payload_rows_remote=53280 dup_rows=4064 dup_rows_remote=4012 dedup_ratio=0.062012 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 64, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 4178, 8192, 8192, 8178, 8192, 8192, 8192] dup_rows_per_dst=[3980, 32, 0, 0, 0, 0, 52, 0] combine_rows_per_src=[8156, 4178, 8192, 8192, 8178, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=224878592 dispatch_meta_bytes=229376 combine_payload_bytes=224878592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=225107968 ingress_bytes=262627328 link_bytes=262627328 token_bytes=449986560 weight_movement_bytes=37748736 total_nvlink_bytes=487735296 weight_to_token_ratio=0.083889 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.516365836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63388 payload_rows_remote=55196 dup_rows=2148 dup_rows_remote=10 dedup_ratio=0.032776 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3710, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7148, 8188, 7420, 8182, 8192, 8188, 7878] dup_rows_per_dst=[0, 4, 0, 0, 0, 2138, 6, 0] combine_rows_per_src=[8192, 7148, 8188, 7420, 8182, 8192, 8188, 7878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=231399424 dispatch_meta_bytes=230568 combine_payload_bytes=231399424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=231629992 ingress_bytes=269148160 link_bytes=269148160 token_bytes=463029416 weight_movement_bytes=37748736 total_nvlink_bytes=500778152 weight_to_token_ratio=0.081526 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.578845240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64298 payload_rows_remote=56622 dup_rows=1238 dup_rows_remote=1204 dedup_ratio=0.018890 router_tokens_per_expert=[0, 0, 0, 0, 4085, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 3833, 0, 4091, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 5, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091] payload_rows_per_dst=[8180, 7676, 8182, 8192, 7564, 8120, 8192, 8192] dup_rows_per_dst=[0, 34, 0, 26, 0, 158, 1020, 0] combine_rows_per_src=[8180, 7676, 8182, 8192, 7564, 8120, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=218234880 dispatch_meta_bytes=229168 combine_payload_bytes=218234880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218464048 ingress_bytes=237109248 link_bytes=237109248 token_bytes=436698928 weight_movement_bytes=18874368 total_nvlink_bytes=455573296 weight_to_token_ratio=0.043221 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.271864149e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=226082816 dispatch_meta_bytes=220824 combine_payload_bytes=226082816 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=226303640 ingress_bytes=320454656 link_bytes=320454656 token_bytes=452386456 weight_movement_bytes=94371840 total_nvlink_bytes=546758296 weight_to_token_ratio=0.208609 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=18432 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.070438840e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61178 payload_rows_remote=52986 dup_rows=4358 dup_rows_remote=4330 dedup_ratio=0.066498 router_tokens_per_expert=[0, 0, 0, 0, 4080, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 3693, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 7, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 4089] payload_rows_per_dst=[8192, 7746, 8178, 8134, 8000, 8004, 4744, 8180] dup_rows_per_dst=[28, 3656, 40, 2, 140, 4, 486, 2] combine_rows_per_src=[8192, 7746, 8178, 8134, 8000, 8004, 4744, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=231923712 dispatch_meta_bytes=231304 combine_payload_bytes=231923712 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=232155016 ingress_bytes=288546816 link_bytes=288546816 token_bytes=464078728 weight_movement_bytes=56623104 total_nvlink_bytes=520701832 weight_to_token_ratio=0.122012 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.764713617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59734 payload_rows_remote=51550 dup_rows=5802 dup_rows_remote=80 dedup_ratio=0.088531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 87, 3, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4079, 0, 0, 0, 3, 0, 0, 3778, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4093, 0, 3549, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0] payload_rows_per_dst=[8182, 3222, 8166, 7562, 8146, 8192, 8080, 8184] dup_rows_per_dst=[4, 16, 8, 0, 2, 50, 0, 5722] combine_rows_per_src=[8182, 3222, 8166, 7562, 8146, 8192, 8080, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65508 payload_rows_remote=57316 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8164, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 28, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8164, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65058 payload_rows_remote=56868 dup_rows=478 dup_rows_remote=478 dedup_ratio=0.007294 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4040, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095] payload_rows_per_dst=[8192, 8082, 8190, 8192, 7832, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 474, 0] combine_rows_per_src=[8192, 8082, 8190, 8192, 7832, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=217030656 dispatch_meta_bytes=229264 combine_payload_bytes=217030656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217259920 ingress_bytes=235905024 link_bytes=235905024 token_bytes=434290576 weight_movement_bytes=18874368 total_nvlink_bytes=453164944 weight_to_token_ratio=0.043460 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260325867e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64790 payload_rows_remote=56598 dup_rows=746 dup_rows_remote=734 dedup_ratio=0.011383 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3728, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 0, 2, 0, 362, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 7460, 8192, 8190, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 4, 6, 0, 12, 0, 0, 724] combine_rows_per_src=[8188, 7460, 8192, 8190, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=211148800 dispatch_meta_bytes=206520 combine_payload_bytes=211148800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211355320 ingress_bytes=230023168 link_bytes=230023168 token_bytes=422504120 weight_movement_bytes=18874368 total_nvlink_bytes=441378488 weight_to_token_ratio=0.044673 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.203968817e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57702 payload_rows_remote=49510 dup_rows=7834 dup_rows_remote=7834 dedup_ratio=0.119537 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 358, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 7824, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 358, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229376 combine_payload_bytes=234766336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234995712 ingress_bytes=253640704 link_bytes=253640704 token_bytes=469762048 weight_movement_bytes=18874368 total_nvlink_bytes=488636416 weight_to_token_ratio=0.040179 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430260427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=232931328 dispatch_meta_bytes=229384 combine_payload_bytes=232931328 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233160712 ingress_bytes=289554432 link_bytes=289554432 token_bytes=466092040 weight_movement_bytes=56623104 total_nvlink_bytes=522715144 weight_to_token_ratio=0.121485 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.774368098e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=231825408 dispatch_meta_bytes=229328 combine_payload_bytes=231825408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232054736 ingress_bytes=250699776 link_bytes=250699776 token_bytes=463880144 weight_movement_bytes=18874368 total_nvlink_bytes=482754512 weight_to_token_ratio=0.040688 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402081902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65480 payload_rows_remote=57320 dup_rows=56 dup_rows_remote=32 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8168, 8192, 8192, 8192, 8160, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 24, 28, 0] combine_rows_per_src=[8192, 8168, 8192, 8192, 8192, 8160, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63976 payload_rows_remote=56736 dup_rows=1560 dup_rows_remote=1076 dedup_ratio=0.023804 router_tokens_per_expert=[0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 3576, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5, 0, 5, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4091, 0, 0, 0, 5, 0, 476, 0, 0, 0, 4091] payload_rows_per_dst=[8178, 7240, 8192, 8192, 8192, 8182, 8182, 7618] dup_rows_per_dst=[0, 484, 48, 0, 40, 0, 36, 952] combine_rows_per_src=[8178, 7240, 8192, 8192, 8192, 8182, 8182, 7618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=202792960 dispatch_meta_bytes=229376 combine_payload_bytes=202792960 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=203022336 ingress_bytes=259416064 link_bytes=259416064 token_bytes=405815296 weight_movement_bytes=56623104 total_nvlink_bytes=462438400 weight_to_token_ratio=0.139529 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.485597085e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229408 combine_payload_bytes=234782720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235012128 ingress_bytes=253657088 link_bytes=253657088 token_bytes=469794848 weight_movement_bytes=18874368 total_nvlink_bytes=488669216 weight_to_token_ratio=0.040176 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430417410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58460 payload_rows_remote=50282 dup_rows=7076 dup_rows_remote=6834 dedup_ratio=0.107971 router_tokens_per_expert=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3382, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 42, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 7, 0, 659, 0, 0, 0, 4089] payload_rows_per_dst=[8178, 8192, 8192, 8182, 8030, 8176, 8178, 1332] dup_rows_per_dst=[242, 6780, 26, 0, 28, 0, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8182, 8030, 8176, 8178, 1332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=232390656 dispatch_meta_bytes=231248 combine_payload_bytes=232390656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232621904 ingress_bytes=251265024 link_bytes=251265024 token_bytes=465012560 weight_movement_bytes=18874368 total_nvlink_bytes=483886928 weight_to_token_ratio=0.040589 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407497830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63816 payload_rows_remote=55624 dup_rows=1720 dup_rows_remote=1712 dedup_ratio=0.026245 router_tokens_per_expert=[0, 0, 0, 0, 4087, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4028, 0, 4093, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 1, 0, 3, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4093] payload_rows_per_dst=[8180, 6648, 8182, 8192, 8116, 8114, 8192, 8192] dup_rows_per_dst=[0, 18, 0, 20, 1414, 2, 258, 8] combine_rows_per_src=[8180, 6648, 8182, 8192, 8116, 8114, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65436 payload_rows_remote=57244 dup_rows=100 dup_rows_remote=100 dedup_ratio=0.001526 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 50, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8092, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 100] combine_rows_per_src=[8192, 8092, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65302 payload_rows_remote=57110 dup_rows=234 dup_rows_remote=234 dedup_ratio=0.003571 router_tokens_per_expert=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3982, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 1, 0, 113, 0, 0, 0, 4095] payload_rows_per_dst=[8188, 7966, 8192, 8192, 8192, 8190, 8190, 8192] dup_rows_per_dst=[0, 4, 0, 0, 4, 0, 0, 226] combine_rows_per_src=[8188, 7966, 8192, 8192, 8192, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61190 payload_rows_remote=53002 dup_rows=4346 dup_rows_remote=4346 dedup_ratio=0.066315 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 210, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 7, 0, 0, 0, 4094, 0, 0, 390, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4094, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8188, 3862, 8188, 8188] dup_rows_per_dst=[424, 4, 12, 794, 0, 0, 98, 3014] combine_rows_per_src=[8192, 8192, 8192, 8188, 8188, 3862, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=205955072 dispatch_meta_bytes=228464 combine_payload_bytes=205955072 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=206183536 ingress_bytes=281452544 link_bytes=281452544 token_bytes=412138608 weight_movement_bytes=75497472 total_nvlink_bytes=487636080 weight_to_token_ratio=0.183185 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.696739793e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=227835904 dispatch_meta_bytes=229344 combine_payload_bytes=227835904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228065248 ingress_bytes=265584640 link_bytes=265584640 token_bytes=455901152 weight_movement_bytes=37748736 total_nvlink_bytes=493649888 weight_to_token_ratio=0.082800 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.544701345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234471424 dispatch_meta_bytes=229376 combine_payload_bytes=234471424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234700800 ingress_bytes=253345792 link_bytes=253345792 token_bytes=469172224 weight_movement_bytes=18874368 total_nvlink_bytes=488046592 weight_to_token_ratio=0.040229 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427434725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58482 payload_rows_remote=50290 dup_rows=7054 dup_rows_remote=6508 dedup_ratio=0.107635 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1638, 8166, 8078, 8166, 8192, 8192, 7858] dup_rows_per_dst=[0, 0, 0, 0, 332, 3848, 546, 2328] combine_rows_per_src=[8192, 1638, 8166, 8078, 8166, 8192, 8192, 7858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=229376 combine_payload_bytes=233922560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234151936 ingress_bytes=252796928 link_bytes=252796928 token_bytes=468074496 weight_movement_bytes=18874368 total_nvlink_bytes=486948864 weight_to_token_ratio=0.040323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422175780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=217096192 dispatch_meta_bytes=229392 combine_payload_bytes=217096192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217325584 ingress_bytes=235970560 link_bytes=235970560 token_bytes=434421776 weight_movement_bytes=18874368 total_nvlink_bytes=453296144 weight_to_token_ratio=0.043447 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260953801e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62386 payload_rows_remote=54194 dup_rows=3150 dup_rows_remote=3150 dedup_ratio=0.048065 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 5042, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 16, 0, 8, 0, 0, 3126] combine_rows_per_src=[8192, 5042, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59446 payload_rows_remote=51254 dup_rows=6090 dup_rows_remote=6090 dedup_ratio=0.092926 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 770, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 5, 0, 5, 0, 0, 25, 0, 0, 0, 4091, 0, 0, 470, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4091, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8182, 8188, 8182, 2136] dup_rows_per_dst=[1550, 0, 30, 990, 0, 3464, 56, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8182, 8188, 8182, 2136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=205987840 dispatch_meta_bytes=227192 combine_payload_bytes=205987840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206215032 ingress_bytes=224862208 link_bytes=224862208 token_bytes=412202872 weight_movement_bytes=18874368 total_nvlink_bytes=431077240 weight_to_token_ratio=0.045789 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.154519038e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=221978624 dispatch_meta_bytes=229376 combine_payload_bytes=221978624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222208000 ingress_bytes=240852992 link_bytes=240852992 token_bytes=444186624 weight_movement_bytes=18874368 total_nvlink_bytes=463060992 weight_to_token_ratio=0.042492 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.307734862e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=209936384 dispatch_meta_bytes=229376 combine_payload_bytes=209936384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210165760 ingress_bytes=228810752 link_bytes=228810752 token_bytes=420102144 weight_movement_bytes=18874368 total_nvlink_bytes=438976512 weight_to_token_ratio=0.044928 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.192352044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63144 payload_rows_remote=54952 dup_rows=2392 dup_rows_remote=158 dedup_ratio=0.036499 router_tokens_per_expert=[0, 0, 0, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3063, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 28, 0, 0, 3, 0, 3, 0, 0, 0, 0, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4093, 0, 0, 0, 3, 0, 993, 0, 0, 0, 4093] payload_rows_per_dst=[7984, 6146, 8072, 8186, 8192, 8186, 8186, 8192] dup_rows_per_dst=[2, 6, 24, 18, 70, 38, 0, 2234] combine_rows_per_src=[7984, 6146, 8072, 8186, 8192, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56136 payload_rows_remote=47944 dup_rows=9400 dup_rows_remote=8730 dedup_ratio=0.143433 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 327, 0, 7, 0, 1, 0, 0, 7, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 6, 0, 0, 0, 7, 0, 7, 0, 0, 73, 0, 0, 0, 4089, 0, 0, 525, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4089, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8178, 8178, 8192, 4244, 2768] dup_rows_per_dst=[670, 6, 40, 5152, 36, 3496, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8178, 8178, 8192, 4244, 2768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62242 payload_rows_remote=54050 dup_rows=3294 dup_rows_remote=3290 dedup_ratio=0.050262 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 5996, 8192, 7094] dup_rows_per_dst=[0, 0, 0, 4, 0, 1078, 16, 2196] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 5996, 8192, 7094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59004 payload_rows_remote=50812 dup_rows=6532 dup_rows_remote=6526 dedup_ratio=0.099670 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 259, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 458, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2509, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4095, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8190, 8192, 8190, 1666] dup_rows_per_dst=[520, 2, 6, 920, 0, 5018, 66, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8190, 8192, 8190, 1666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64070 payload_rows_remote=55884 dup_rows=1466 dup_rows_remote=376 dedup_ratio=0.022369 router_tokens_per_expert=[0, 4091, 0, 4094, 0, 0, 3372, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0] payload_rows_per_dst=[6754, 8192, 8190, 8192, 8186, 8188, 8180, 8188] dup_rows_per_dst=[10, 4, 0, 0, 1090, 6, 0, 356] combine_rows_per_src=[6754, 8192, 8190, 8192, 8186, 8188, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=225083392 dispatch_meta_bytes=220440 combine_payload_bytes=225083392 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=225303832 ingress_bytes=319455232 link_bytes=319455232 token_bytes=450387224 weight_movement_bytes=94371840 total_nvlink_bytes=544759064 weight_to_token_ratio=0.209535 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.060862851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=196378624 dispatch_meta_bytes=226696 combine_payload_bytes=196378624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196605320 ingress_bytes=215252992 link_bytes=215252992 token_bytes=392983944 weight_movement_bytes=18874368 total_nvlink_bytes=411858312 weight_to_token_ratio=0.048028 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.062448258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=221388800 dispatch_meta_bytes=229360 combine_payload_bytes=221388800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=221618160 ingress_bytes=278011904 link_bytes=278011904 token_bytes=443006960 weight_movement_bytes=56623104 total_nvlink_bytes=499630064 weight_to_token_ratio=0.127815 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.663773274e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=56976 dup_rows=368 dup_rows_remote=350 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7834, 8192, 8192, 8192, 8182, 8192, 8192] dup_rows_per_dst=[0, 10, 0, 0, 0, 340, 18, 0] combine_rows_per_src=[8192, 7834, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=208125952 dispatch_meta_bytes=229352 combine_payload_bytes=208125952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208355304 ingress_bytes=227000320 link_bytes=227000320 token_bytes=416481256 weight_movement_bytes=18874368 total_nvlink_bytes=435355624 weight_to_token_ratio=0.045319 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.175005375e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61426 payload_rows_remote=57344 dup_rows=4110 dup_rows_remote=4110 dedup_ratio=0.062714 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 4082, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 34, 0, 0, 0, 4070] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 4082, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=228900864 dispatch_meta_bytes=225040 combine_payload_bytes=228900864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229125904 ingress_bytes=247775232 link_bytes=247775232 token_bytes=458026768 weight_movement_bytes=18874368 total_nvlink_bytes=476901136 weight_to_token_ratio=0.041208 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374060360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64526 payload_rows_remote=56334 dup_rows=1010 dup_rows_remote=1002 dedup_ratio=0.015411 router_tokens_per_expert=[0, 4071, 0, 4091, 0, 0, 3683, 0, 20, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 5, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5, 4087, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0] payload_rows_per_dst=[7376, 8192, 8186, 8192, 8152, 8184, 8062, 8182] dup_rows_per_dst=[40, 8, 0, 0, 474, 16, 0, 472] combine_rows_per_src=[7376, 8192, 8186, 8192, 8152, 8184, 8062, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=233373696 dispatch_meta_bytes=229304 combine_payload_bytes=233373696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233603000 ingress_bytes=252248064 link_bytes=252248064 token_bytes=466976696 weight_movement_bytes=18874368 total_nvlink_bytes=485851064 weight_to_token_ratio=0.040418 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416916835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=245816 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235126840 ingress_bytes=272629760 link_bytes=272629760 token_bytes=470007864 weight_movement_bytes=37748736 total_nvlink_bytes=507756600 weight_to_token_ratio=0.080315 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=15104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=229344 combine_payload_bytes=230744064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230973408 ingress_bytes=249618432 link_bytes=249618432 token_bytes=461717472 weight_movement_bytes=18874368 total_nvlink_bytes=480591840 weight_to_token_ratio=0.040879 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.391720995e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54290 payload_rows_remote=46104 dup_rows=11246 dup_rows_remote=8826 dedup_ratio=0.171600 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 259, 0, 3, 0, 2, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 3, 0, 3, 0, 0, 28, 0, 0, 0, 4093, 0, 0, 318, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4092, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4500, 8174, 8086, 786, 8186, 8188, 8184, 8186] dup_rows_per_dst=[528, 0, 18, 12, 0, 4532, 3736, 2420] combine_rows_per_src=[4500, 8174, 8086, 786, 8186, 8188, 8184, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57264 payload_rows_remote=49186 dup_rows=8272 dup_rows_remote=2002 dedup_ratio=0.126221 router_tokens_per_expert=[0, 4055, 0, 4089, 0, 0, 3342, 0, 62, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 4073, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4089, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0] payload_rows_per_dst=[8078, 8192, 8180, 8192, 1916, 8156, 8180, 6370] dup_rows_per_dst=[6270, 14, 8, 4, 94, 26, 1520, 336] combine_rows_per_src=[8078, 8192, 8180, 8192, 1916, 8156, 8180, 6370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65478 payload_rows_remote=57286 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000885 router_tokens_per_expert=[0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8192, 8192, 8192, 8192, 8192, 8170, 8192] dup_rows_per_dst=[0, 0, 0, 0, 58, 0, 0, 0] combine_rows_per_src=[8156, 8192, 8192, 8192, 8192, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63412 payload_rows_remote=55222 dup_rows=2124 dup_rows_remote=2124 dedup_ratio=0.032410 router_tokens_per_expert=[0, 4094, 0, 4095, 0, 0, 3052, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 955, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0] payload_rows_per_dst=[6108, 8192, 8190, 8192, 8190, 8192, 8158, 8190] dup_rows_per_dst=[4, 0, 0, 0, 1910, 4, 0, 206] combine_rows_per_src=[6108, 8192, 8190, 8192, 8190, 8192, 8158, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64332 payload_rows_remote=57330 dup_rows=1204 dup_rows_remote=1200 dedup_ratio=0.018372 router_tokens_per_expert=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 590, 0, 0, 0, 4094, 4094, 4094, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 7002, 8188, 8188, 8192] dup_rows_per_dst=[0, 4, 4, 0, 4, 0, 1180, 12] combine_rows_per_src=[8186, 8192, 8192, 8192, 7002, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=188841984 dispatch_meta_bytes=219720 combine_payload_bytes=188841984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189061704 ingress_bytes=207716352 link_bytes=207716352 token_bytes=377903688 weight_movement_bytes=18874368 total_nvlink_bytes=396778056 weight_to_token_ratio=0.049945 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=19584 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.990235882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=201465856 dispatch_meta_bytes=204752 combine_payload_bytes=201465856 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=201670608 ingress_bytes=276963328 link_bytes=276963328 token_bytes=403136464 weight_movement_bytes=75497472 total_nvlink_bytes=478633936 weight_to_token_ratio=0.187275 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.653726334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=229376 combine_payload_bytes=234643456 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234872832 ingress_bytes=291266560 link_bytes=291266560 token_bytes=469516288 weight_movement_bytes=56623104 total_nvlink_bytes=526139392 weight_to_token_ratio=0.120599 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790772866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 36, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 72] combine_rows_per_src=[8192, 8120, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65074 payload_rows_remote=56882 dup_rows=462 dup_rows_remote=462 dedup_ratio=0.007050 router_tokens_per_expert=[0, 4096, 0, 4096, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8016, 8192, 8192, 8192, 8192, 8192, 7906, 8192] dup_rows_per_dst=[0, 0, 0, 0, 448, 0, 0, 14] combine_rows_per_src=[8016, 8192, 8192, 8192, 8192, 8192, 7906, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=226189312 dispatch_meta_bytes=229384 combine_payload_bytes=226189312 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=226418696 ingress_bytes=263938048 link_bytes=263938048 token_bytes=452608008 weight_movement_bytes=37748736 total_nvlink_bytes=490356744 weight_to_token_ratio=0.083403 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528924510e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=234120 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235057800 ingress_bytes=272572416 link_bytes=272572416 token_bytes=469881480 weight_movement_bytes=37748736 total_nvlink_bytes=507630216 weight_to_token_ratio=0.080337 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611654776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61182 payload_rows_remote=52990 dup_rows=4354 dup_rows_remote=4344 dedup_ratio=0.066437 router_tokens_per_expert=[4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 5, 0, 0, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 662, 0, 0, 0, 4090, 4091, 4091, 0, 0, 0, 19, 4091, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8192, 8192, 8190, 6852, 8182, 5204, 8192] dup_rows_per_dst=[0, 10, 10, 0, 2940, 36, 1320, 38] combine_rows_per_src=[8178, 8192, 8192, 8190, 6852, 8182, 5204, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234815488 ingress_bytes=272334848 link_bytes=272334848 token_bytes=469401600 weight_movement_bytes=37748736 total_nvlink_bytes=507150336 weight_to_token_ratio=0.080419 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609378516e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=232988672 dispatch_meta_bytes=229376 combine_payload_bytes=232988672 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=233218048 ingress_bytes=251863040 link_bytes=251863040 token_bytes=466206720 weight_movement_bytes=18874368 total_nvlink_bytes=485081088 weight_to_token_ratio=0.040485 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413227725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63926 payload_rows_remote=55740 dup_rows=1610 dup_rows_remote=1164 dedup_ratio=0.024567 router_tokens_per_expert=[0, 4078, 0, 4093, 0, 0, 3348, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 4088, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 1] payload_rows_per_dst=[6692, 8156, 8192, 8188, 8162, 8182, 8168, 8186] dup_rows_per_dst=[12, 10, 14, 0, 1082, 6, 40, 446] combine_rows_per_src=[6692, 8156, 8192, 8188, 8162, 8182, 8168, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=217047040 dispatch_meta_bytes=229336 combine_payload_bytes=217047040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217276376 ingress_bytes=235921408 link_bytes=235921408 token_bytes=434323416 weight_movement_bytes=18874368 total_nvlink_bytes=453197784 weight_to_token_ratio=0.043457 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260482851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57714 payload_rows_remote=49536 dup_rows=7822 dup_rows_remote=7742 dedup_ratio=0.119354 router_tokens_per_expert=[4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 7, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 206, 0, 0, 0, 4089, 4088, 4089, 0, 0, 0, 137, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8188, 8192, 8192, 8192, 8178, 728, 7866] dup_rows_per_dst=[80, 14, 14, 2, 7518, 0, 0, 194] combine_rows_per_src=[8178, 8188, 8192, 8192, 8192, 8178, 728, 7866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63126 payload_rows_remote=54934 dup_rows=2410 dup_rows_remote=2410 dedup_ratio=0.036774 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1207, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 5784, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2410, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 5784, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63698 payload_rows_remote=55506 dup_rows=1838 dup_rows_remote=1836 dedup_ratio=0.028046 router_tokens_per_expert=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 918, 0, 0, 0, 4095, 4090, 4095, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 8192, 8192, 6364, 8190, 8190, 8192] dup_rows_per_dst=[0, 2, 2, 0, 2, 0, 1826, 6] combine_rows_per_src=[8188, 8190, 8192, 8192, 6364, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65116 payload_rows_remote=56924 dup_rows=420 dup_rows_remote=412 dedup_ratio=0.006409 router_tokens_per_expert=[2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 173, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 17, 0, 4094, 3905, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8188, 8192, 8192, 8188, 8148, 7828] dup_rows_per_dst=[0, 42, 346, 0, 8, 0, 4, 20] combine_rows_per_src=[8192, 8188, 8188, 8192, 8192, 8188, 8148, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=228311040 dispatch_meta_bytes=227616 combine_payload_bytes=228311040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228538656 ingress_bytes=247185408 link_bytes=247185408 token_bytes=456849696 weight_movement_bytes=18874368 total_nvlink_bytes=475724064 weight_to_token_ratio=0.041314 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368408957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=202899456 dispatch_meta_bytes=229112 combine_payload_bytes=202899456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203128568 ingress_bytes=221773824 link_bytes=221773824 token_bytes=406028024 weight_movement_bytes=18874368 total_nvlink_bytes=424902392 weight_to_token_ratio=0.046485 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.124927662e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57774 payload_rows_remote=49582 dup_rows=7762 dup_rows_remote=7762 dedup_ratio=0.118439 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7420, 8192, 1204, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 302, 0, 44, 0, 7416] combine_rows_per_src=[8192, 8192, 8192, 7420, 8192, 1204, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64448 payload_rows_remote=56256 dup_rows=1088 dup_rows_remote=1088 dedup_ratio=0.016602 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 544, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7104, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1088, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7104, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=227352576 dispatch_meta_bytes=229368 combine_payload_bytes=227352576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=227581944 ingress_bytes=246226944 link_bytes=246226944 token_bytes=454934520 weight_movement_bytes=18874368 total_nvlink_bytes=473808888 weight_to_token_ratio=0.041488 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.359225426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=225009664 dispatch_meta_bytes=229376 combine_payload_bytes=225009664 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=225239040 ingress_bytes=262758400 link_bytes=262758400 token_bytes=450248704 weight_movement_bytes=37748736 total_nvlink_bytes=487997440 weight_to_token_ratio=0.083840 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.517621704e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233160704 dispatch_meta_bytes=229344 combine_payload_bytes=233160704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233390048 ingress_bytes=252035072 link_bytes=252035072 token_bytes=466550752 weight_movement_bytes=18874368 total_nvlink_bytes=485425120 weight_to_token_ratio=0.040455 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414876051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64866 payload_rows_remote=56684 dup_rows=670 dup_rows_remote=592 dedup_ratio=0.010223 router_tokens_per_expert=[5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 257, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 70, 0, 4091, 3819, 0, 0, 0, 5, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8164, 8192, 8192, 8182, 8026, 7736] dup_rows_per_dst=[0, 78, 510, 0, 20, 0, 10, 52] combine_rows_per_src=[8192, 8182, 8164, 8192, 8192, 8182, 8026, 7736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=203087872 dispatch_meta_bytes=229376 combine_payload_bytes=203087872 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=203317248 ingress_bytes=278585344 link_bytes=278585344 token_bytes=406405120 weight_movement_bytes=75497472 total_nvlink_bytes=481902592 weight_to_token_ratio=0.185769 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.669267694e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=230424576 dispatch_meta_bytes=229376 combine_payload_bytes=230424576 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=230653952 ingress_bytes=268173312 link_bytes=268173312 token_bytes=461078528 weight_movement_bytes=37748736 total_nvlink_bytes=498827264 weight_to_token_ratio=0.081871 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.569504726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=232177664 dispatch_meta_bytes=229104 combine_payload_bytes=232177664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232406768 ingress_bytes=251052032 link_bytes=251052032 token_bytes=464584432 weight_movement_bytes=18874368 total_nvlink_bytes=483458800 weight_to_token_ratio=0.040626 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405457045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63942 payload_rows_remote=55750 dup_rows=1594 dup_rows_remote=1518 dedup_ratio=0.024323 router_tokens_per_expert=[4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 0, 0, 3344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 731, 0, 0, 0, 4093, 4093, 4093, 0, 0, 0, 38, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8164, 8156, 8192, 6716, 8186, 8186, 8192] dup_rows_per_dst=[0, 6, 6, 0, 20, 0, 1486, 76] combine_rows_per_src=[8150, 8164, 8156, 8192, 6716, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65024 payload_rows_remote=56832 dup_rows=512 dup_rows_remote=512 dedup_ratio=0.007812 router_tokens_per_expert=[7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 202, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 129, 0, 4089, 3888, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8174, 8192, 8192, 8148, 7950, 7998] dup_rows_per_dst=[0, 10, 408, 2, 28, 0, 14, 50] combine_rows_per_src=[8192, 8178, 8174, 8192, 8192, 8148, 7950, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57292 dup_rows=54 dup_rows_remote=20 dedup_ratio=0.000824 router_tokens_per_expert=[1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 17, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 16, 0, 4095, 4070, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8192, 8190, 8168, 8168] dup_rows_per_dst=[0, 8, 34, 0, 4, 0, 2, 6] combine_rows_per_src=[8192, 8190, 8190, 8192, 8192, 8190, 8168, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=234 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4094, 4087, 0, 104, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0] payload_rows_per_dst=[7984, 8188, 8188, 8184, 8192, 8178, 8192, 8192] dup_rows_per_dst=[12, 0, 4, 4, 4, 4, 208, 2] combine_rows_per_src=[7984, 8188, 8188, 8184, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 2, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=228352000 dispatch_meta_bytes=229072 combine_payload_bytes=228352000 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=228581072 ingress_bytes=266100736 link_bytes=266100736 token_bytes=456933072 weight_movement_bytes=37748736 total_nvlink_bytes=494681808 weight_to_token_ratio=0.082613 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.549646323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64324 payload_rows_remote=57172 dup_rows=1212 dup_rows_remote=1212 dedup_ratio=0.018494 router_tokens_per_expert=[0, 4096, 0, 4096, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[8020, 8192, 8192, 8192, 8192, 8192, 7152, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1198, 0, 0, 14] combine_rows_per_src=[8020, 8192, 8192, 8192, 8192, 8192, 7152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=232783872 dispatch_meta_bytes=229376 combine_payload_bytes=232783872 prefetch_bytes=0 reduce_bytes=0 egress_bytes=233013248 ingress_bytes=232783872 link_bytes=233013248 token_bytes=465797120 weight_movement_bytes=0 total_nvlink_bytes=465797120 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232618293e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229248 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234897280 ingress_bytes=291291136 link_bytes=291291136 token_bytes=469565312 weight_movement_bytes=56623104 total_nvlink_bytes=526188416 weight_to_token_ratio=0.120586 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791008341e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229360 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234135536 ingress_bytes=252780544 link_bytes=252780544 token_bytes=468041712 weight_movement_bytes=18874368 total_nvlink_bytes=486916080 weight_to_token_ratio=0.040326 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422018797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65034 payload_rows_remote=56852 dup_rows=502 dup_rows_remote=502 dedup_ratio=0.007660 router_tokens_per_expert=[0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3966, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 4091, 4069, 0, 110, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 5, 0, 0] payload_rows_per_dst=[8004, 8182, 8182, 7942, 8192, 8148, 8192, 8192] dup_rows_per_dst=[80, 0, 24, 10, 10, 10, 220, 148] combine_rows_per_src=[8004, 8182, 8182, 7942, 8192, 8148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=234176512 dispatch_meta_bytes=233536 combine_payload_bytes=234176512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234410048 ingress_bytes=290799616 link_bytes=290799616 token_bytes=468586560 weight_movement_bytes=56623104 total_nvlink_bytes=525209664 weight_to_token_ratio=0.120838 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.786298838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=235094016 ingress_bytes=291487744 link_bytes=291487744 token_bytes=469958656 weight_movement_bytes=56623104 total_nvlink_bytes=526581760 weight_to_token_ratio=0.120485 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792892142e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232865792 dispatch_meta_bytes=229416 combine_payload_bytes=232865792 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233095208 ingress_bytes=270614528 link_bytes=270614528 token_bytes=465961000 weight_movement_bytes=37748736 total_nvlink_bytes=503709736 weight_to_token_ratio=0.081013 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592895257e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63406 payload_rows_remote=55268 dup_rows=2130 dup_rows_remote=772 dedup_ratio=0.032501 router_tokens_per_expert=[3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 52, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 71, 0, 4093, 4027, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7694, 6818, 8190, 8120, 8186, 8068, 8138] dup_rows_per_dst=[0, 18, 102, 0, 12, 70, 570, 1358] combine_rows_per_src=[8192, 7694, 6818, 8190, 8120, 8186, 8068, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62412 payload_rows_remote=54220 dup_rows=3124 dup_rows_remote=3116 dedup_ratio=0.047668 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4094, 0, 2616, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 4094, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8110, 8188, 5450, 8192, 7900, 8188, 8192] dup_rows_per_dst=[0, 2742, 36, 64, 8, 0, 274, 0] combine_rows_per_src=[8192, 8110, 8188, 5450, 8192, 7900, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65416 payload_rows_remote=57226 dup_rows=120 dup_rows_remote=120 dedup_ratio=0.001831 router_tokens_per_expert=[0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4095, 4094, 0, 56, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8082, 8190, 8190, 8188, 8192, 8190, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 2, 2, 2, 112, 0] combine_rows_per_src=[8082, 8190, 8190, 8188, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64194 payload_rows_remote=56016 dup_rows=1342 dup_rows_remote=594 dedup_ratio=0.020477 router_tokens_per_expert=[0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4042, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 4089, 4023, 0, 411, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0] payload_rows_per_dst=[8178, 8180, 8178, 8100, 8192, 8076, 7306, 7984] dup_rows_per_dst=[748, 30, 20, 354, 14, 174, 0, 2] combine_rows_per_src=[8178, 8180, 8178, 8100, 8192, 8076, 7306, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=226377728 dispatch_meta_bytes=224160 combine_payload_bytes=226377728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226601888 ingress_bytes=264126464 link_bytes=264126464 token_bytes=452979616 weight_movement_bytes=37748736 total_nvlink_bytes=490728352 weight_to_token_ratio=0.083334 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.530729820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=222085120 dispatch_meta_bytes=229344 combine_payload_bytes=222085120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222314464 ingress_bytes=240959488 link_bytes=240959488 token_bytes=444399584 weight_movement_bytes=18874368 total_nvlink_bytes=463273952 weight_to_token_ratio=0.042472 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.308755255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64382 payload_rows_remote=56190 dup_rows=1154 dup_rows_remote=0 dedup_ratio=0.017609 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 787, 0, 0, 0, 4096, 3886, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7038, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1154, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7038, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234397696 dispatch_meta_bytes=229384 combine_payload_bytes=234397696 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234627080 ingress_bytes=253272064 link_bytes=253272064 token_bytes=469024776 weight_movement_bytes=18874368 total_nvlink_bytes=487899144 weight_to_token_ratio=0.040242 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.426728300e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=229441536 dispatch_meta_bytes=226440 combine_payload_bytes=229441536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229667976 ingress_bytes=248315904 link_bytes=248315904 token_bytes=459109512 weight_movement_bytes=18874368 total_nvlink_bytes=477983880 weight_to_token_ratio=0.041111 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.379240813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64012 payload_rows_remote=55820 dup_rows=1524 dup_rows_remote=890 dedup_ratio=0.023254 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4091, 0, 3726, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 4091, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 7462, 8192, 8024, 7576, 8192] dup_rows_per_dst=[0, 634, 40, 10, 44, 576, 220, 0] combine_rows_per_src=[8192, 8192, 8182, 7462, 8192, 8024, 7576, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 2, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=230154240 dispatch_meta_bytes=224760 combine_payload_bytes=230154240 prefetch_bytes=18874368 reduce_bytes=113246208 egress_bytes=230379000 ingress_bytes=362274816 link_bytes=362274816 token_bytes=460533240 weight_movement_bytes=132120576 total_nvlink_bytes=592653816 weight_to_token_ratio=0.286886 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.471139037e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110400 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991424 weight_movement_bytes=75497472 total_nvlink_bytes=545488896 weight_to_token_ratio=0.160636 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=228638720 dispatch_meta_bytes=226840 combine_payload_bytes=228638720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228865560 ingress_bytes=266387456 link_bytes=266387456 token_bytes=457504280 weight_movement_bytes=37748736 total_nvlink_bytes=495253016 weight_to_token_ratio=0.082510 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.552393533e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65024 payload_rows_remote=56832 dup_rows=512 dup_rows_remote=456 dedup_ratio=0.007812 router_tokens_per_expert=[0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4082, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 4093, 4070, 0, 190, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[7848, 8190, 8164, 8170, 8156, 8146, 8158, 8192] dup_rows_per_dst=[16, 0, 10, 6, 6, 38, 380, 56] combine_rows_per_src=[7848, 8190, 8164, 8170, 8156, 8146, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60472 payload_rows_remote=52284 dup_rows=5064 dup_rows_remote=1464 dedup_ratio=0.077271 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 79, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 724, 0, 0, 4094] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8188, 3144, 8192, 8188] dup_rows_per_dst=[0, 0, 4, 0, 3600, 0, 12, 1448] combine_rows_per_src=[8192, 8192, 8192, 8184, 8188, 3144, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63244 payload_rows_remote=55054 dup_rows=2292 dup_rows_remote=2278 dedup_ratio=0.034973 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4095, 0, 3069, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 4095, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8190, 6140, 8192, 7960, 8192, 8192] dup_rows_per_dst=[0, 2136, 14, 2, 4, 0, 136, 0] combine_rows_per_src=[8186, 8192, 8190, 6140, 8192, 7960, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=232783872 dispatch_meta_bytes=229152 combine_payload_bytes=232783872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233013024 ingress_bytes=270532608 link_bytes=270532608 token_bytes=465796896 weight_movement_bytes=37748736 total_nvlink_bytes=503545632 weight_to_token_ratio=0.081041 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592110340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55072 payload_rows_remote=46880 dup_rows=10464 dup_rows_remote=10262 dedup_ratio=0.159668 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4089, 0, 3022, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 4089, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8086, 6148, 8020, 7976, 266, 8192] dup_rows_per_dst=[202, 8058, 2, 136, 14, 2052, 0, 0] combine_rows_per_src=[8192, 8192, 8086, 6148, 8020, 7976, 266, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61806 payload_rows_remote=57186 dup_rows=3730 dup_rows_remote=3730 dedup_ratio=0.056915 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 4096, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 4620, 8192, 8036, 8192, 8192] dup_rows_per_dst=[0, 3306, 0, 0, 16, 0, 408, 0] combine_rows_per_src=[8190, 8192, 8192, 4620, 8192, 8036, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=214155264 dispatch_meta_bytes=214992 combine_payload_bytes=214155264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214370256 ingress_bytes=233029632 link_bytes=233029632 token_bytes=428525520 weight_movement_bytes=18874368 total_nvlink_bytes=447399888 weight_to_token_ratio=0.044045 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23040 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232775276e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65146 payload_rows_remote=56954 dup_rows=390 dup_rows_remote=390 dedup_ratio=0.005951 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3903, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8188, 8192, 7806] dup_rows_per_dst=[0, 370, 20, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8188, 8192, 7806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61142 payload_rows_remote=53216 dup_rows=4394 dup_rows_remote=4394 dedup_ratio=0.067047 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 4096, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4064, 8192, 8192, 8192, 7926, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4094, 30, 0, 270, 0] combine_rows_per_src=[8192, 4064, 8192, 8192, 8192, 7926, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=225501184 dispatch_meta_bytes=229328 combine_payload_bytes=225501184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225730512 ingress_bytes=244375552 link_bytes=244375552 token_bytes=451231696 weight_movement_bytes=18874368 total_nvlink_bytes=470106064 weight_to_token_ratio=0.041829 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341486299e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59274 payload_rows_remote=51082 dup_rows=6262 dup_rows_remote=6262 dedup_ratio=0.095551 router_tokens_per_expert=[0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 97, 0, 0, 0, 0, 0, 0, 0, 829, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 5, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1149, 0, 0, 4091] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8182, 6422, 8192, 3726] dup_rows_per_dst=[0, 0, 10, 0, 4042, 1032, 30, 1148] combine_rows_per_src=[8192, 8192, 8192, 8176, 8182, 6422, 8192, 3726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=192020480 dispatch_meta_bytes=228568 combine_payload_bytes=192020480 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=192249048 ingress_bytes=229769216 link_bytes=229769216 token_bytes=384269528 weight_movement_bytes=37748736 total_nvlink_bytes=422018264 weight_to_token_ratio=0.098235 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.201535574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=234233856 dispatch_meta_bytes=243664 combine_payload_bytes=234233856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234477520 ingress_bytes=271982592 link_bytes=271982592 token_bytes=468711376 weight_movement_bytes=37748736 total_nvlink_bytes=506460112 weight_to_token_ratio=0.080537 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606003373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233283584 dispatch_meta_bytes=229376 combine_payload_bytes=233283584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233512960 ingress_bytes=252157952 link_bytes=252157952 token_bytes=466796544 weight_movement_bytes=18874368 total_nvlink_bytes=485670912 weight_to_token_ratio=0.040434 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.416053427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=217972736 dispatch_meta_bytes=230440 combine_payload_bytes=217972736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218203176 ingress_bytes=236847104 link_bytes=236847104 token_bytes=436175912 weight_movement_bytes=18874368 total_nvlink_bytes=455050280 weight_to_token_ratio=0.043272 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.269352414e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58606 payload_rows_remote=50414 dup_rows=6930 dup_rows_remote=6930 dedup_ratio=0.105743 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4093, 0, 3275, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 0, 4093, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7998, 2308, 8186, 8192, 8192, 7360, 8178, 8192] dup_rows_per_dst=[0, 0, 36, 6556, 48, 0, 290, 0] combine_rows_per_src=[7998, 2308, 8186, 8192, 8192, 7360, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=209231872 dispatch_meta_bytes=229376 combine_payload_bytes=209231872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209461248 ingress_bytes=228106240 link_bytes=228106240 token_bytes=418693120 weight_movement_bytes=18874368 total_nvlink_bytes=437567488 weight_to_token_ratio=0.045079 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.185601756e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64844 payload_rows_remote=57330 dup_rows=692 dup_rows_remote=688 dedup_ratio=0.010559 router_tokens_per_expert=[0, 0, 0, 1, 4092, 0, 0, 149, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3755, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8188, 8188, 8188, 8192, 7514, 8190, 8192, 8192] dup_rows_per_dst=[300, 0, 0, 8, 4, 380, 0, 0] combine_rows_per_src=[8188, 8188, 8188, 8192, 7514, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=206495744 dispatch_meta_bytes=229376 combine_payload_bytes=206495744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206725120 ingress_bytes=225370112 link_bytes=225370112 token_bytes=413220864 weight_movement_bytes=18874368 total_nvlink_bytes=432095232 weight_to_token_ratio=0.045676 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159385524e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59128 payload_rows_remote=50936 dup_rows=6408 dup_rows_remote=6406 dedup_ratio=0.097778 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1774, 0, 0, 4095] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8190, 1790, 8192, 8190] dup_rows_per_dst=[2, 0, 2, 0, 2852, 0, 4, 3548] combine_rows_per_src=[8192, 8192, 8192, 8190, 8190, 1790, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58982 payload_rows_remote=50790 dup_rows=6554 dup_rows_remote=6554 dedup_ratio=0.100006 router_tokens_per_expert=[0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 4, 0, 0, 0, 4089, 0, 76, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 7, 0, 4087, 0, 0, 0, 6, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 855, 0, 0, 4089] payload_rows_per_dst=[8192, 8192, 8192, 8170, 8180, 8176, 8170, 1710] dup_rows_per_dst=[0, 0, 32, 0, 5526, 956, 40, 0] combine_rows_per_src=[8192, 8192, 8192, 8170, 8180, 8176, 8170, 1710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64852 payload_rows_remote=56660 dup_rows=684 dup_rows_remote=684 dedup_ratio=0.010437 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 250, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7508, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 594, 0, 0, 90] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7508, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=232072 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235055752 ingress_bytes=291446784 link_bytes=291446784 token_bytes=469879432 weight_movement_bytes=56623104 total_nvlink_bytes=526502536 weight_to_token_ratio=0.120506 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.792499684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[6, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61652 payload_rows_remote=54250 dup_rows=3884 dup_rows_remote=794 dedup_ratio=0.059265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 700, 0, 0, 0, 0, 0, 0, 0, 2988, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 285, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5098, 7402, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 224, 3090, 0, 570] combine_rows_per_src=[8192, 8192, 8192, 8192, 5098, 7402, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65318 payload_rows_remote=57136 dup_rows=218 dup_rows_remote=218 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 0, 5, 4087, 0, 0, 41, 0, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 4000, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0] payload_rows_per_dst=[8184, 8182, 8182, 8192, 8012, 8182, 8192, 8192] dup_rows_per_dst=[92, 0, 0, 24, 10, 92, 0, 0] combine_rows_per_src=[8184, 8182, 8182, 8192, 8012, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=208633856 dispatch_meta_bytes=229368 combine_payload_bytes=208633856 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=208863224 ingress_bytes=265256960 link_bytes=265256960 token_bytes=417497080 weight_movement_bytes=56623104 total_nvlink_bytes=474120184 weight_to_token_ratio=0.135625 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.541561676e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=208035840 dispatch_meta_bytes=229376 combine_payload_bytes=208035840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208265216 ingress_bytes=226910208 link_bytes=226910208 token_bytes=416301056 weight_movement_bytes=18874368 total_nvlink_bytes=435175424 weight_to_token_ratio=0.045338 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.174141966e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=232079360 dispatch_meta_bytes=229376 combine_payload_bytes=232079360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232308736 ingress_bytes=250953728 link_bytes=250953728 token_bytes=464388096 weight_movement_bytes=18874368 total_nvlink_bytes=483262464 weight_to_token_ratio=0.040644 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404515145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229376 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235085824 ingress_bytes=310353920 link_bytes=310353920 token_bytes=469942272 weight_movement_bytes=75497472 total_nvlink_bytes=545439744 weight_to_token_ratio=0.160653 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973658558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=222208000 dispatch_meta_bytes=220176 combine_payload_bytes=222208000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=222428176 ingress_bytes=259956736 link_bytes=259956736 token_bytes=444636176 weight_movement_bytes=37748736 total_nvlink_bytes=482384912 weight_to_token_ratio=0.084898 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=14976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.490777538e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59840 payload_rows_remote=51654 dup_rows=5696 dup_rows_remote=1558 dedup_ratio=0.086914 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 59, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2035, 0, 0, 4093] payload_rows_per_dst=[8190, 8192, 8138, 8186, 2652, 8186, 8110, 8186] dup_rows_per_dst=[0, 0, 6, 54, 0, 1480, 18, 4138] combine_rows_per_src=[8190, 8192, 8138, 8186, 2652, 8186, 8110, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234029056 dispatch_meta_bytes=229416 combine_payload_bytes=234029056 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234258472 ingress_bytes=271777792 link_bytes=271777792 token_bytes=468287528 weight_movement_bytes=37748736 total_nvlink_bytes=506036264 weight_to_token_ratio=0.080610 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.604041080e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62336 payload_rows_remote=54144 dup_rows=3200 dup_rows_remote=3184 dedup_ratio=0.048828 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1589, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8192, 4998, 8192, 8192] dup_rows_per_dst=[3178, 0, 0, 6, 16, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 8192, 4998, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65432 payload_rows_remote=57242 dup_rows=104 dup_rows_remote=104 dedup_ratio=0.001587 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 0, 15, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4045, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 8092, 8192, 8192, 8192] dup_rows_per_dst=[30, 0, 0, 2, 2, 70, 0, 0] combine_rows_per_src=[8192, 8190, 8190, 8192, 8092, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=211574784 dispatch_meta_bytes=212848 combine_payload_bytes=211574784 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=211787632 ingress_bytes=268197888 link_bytes=268197888 token_bytes=423362416 weight_movement_bytes=56623104 total_nvlink_bytes=479985520 weight_to_token_ratio=0.133746 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=25856 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.569740201e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62638 payload_rows_remote=54446 dup_rows=2898 dup_rows_remote=272 dedup_ratio=0.044220 router_tokens_per_expert=[0, 0, 0, 7, 4080, 0, 0, 576, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 3410, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8178, 8180, 8168, 6838, 6700, 8192, 8190] dup_rows_per_dst=[2626, 0, 34, 16, 22, 200, 0, 0] combine_rows_per_src=[8192, 8178, 8180, 8168, 6838, 6700, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=221773824 dispatch_meta_bytes=229312 combine_payload_bytes=221773824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222003136 ingress_bytes=240648192 link_bytes=240648192 token_bytes=443776960 weight_movement_bytes=18874368 total_nvlink_bytes=462651328 weight_to_token_ratio=0.042531 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.305772569e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65300 payload_rows_remote=57108 dup_rows=236 dup_rows_remote=236 dedup_ratio=0.003601 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7956, 8192, 8192, 8192] dup_rows_per_dst=[42, 0, 0, 0, 0, 194, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7956, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58288 payload_rows_remote=50096 dup_rows=7248 dup_rows_remote=286 dedup_ratio=0.110596 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 4096, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1410, 8192, 8192, 8192, 7726, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 278, 8, 0, 6962, 0] combine_rows_per_src=[8192, 1410, 8192, 8192, 8192, 7726, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234463232 dispatch_meta_bytes=229384 combine_payload_bytes=234463232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234692616 ingress_bytes=253337600 link_bytes=253337600 token_bytes=469155848 weight_movement_bytes=18874368 total_nvlink_bytes=488030216 weight_to_token_ratio=0.040230 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427356233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65360 payload_rows_remote=57168 dup_rows=176 dup_rows_remote=118 dedup_ratio=0.002686 router_tokens_per_expert=[0, 0, 0, 0, 4081, 0, 0, 73, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8016, 8192, 8192, 8192] dup_rows_per_dst=[116, 0, 0, 2, 0, 58, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8016, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=223010816 dispatch_meta_bytes=218872 combine_payload_bytes=223010816 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=223229688 ingress_bytes=298508288 link_bytes=298508288 token_bytes=446240504 weight_movement_bytes=75497472 total_nvlink_bytes=521737976 weight_to_token_ratio=0.169186 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.860159540e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63008 payload_rows_remote=54828 dup_rows=2528 dup_rows_remote=2528 dedup_ratio=0.038574 router_tokens_per_expert=[5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1252, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 1, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2838, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 5676, 8192, 8192] dup_rows_per_dst=[2504, 0, 0, 12, 12, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 5676, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=233914368 dispatch_meta_bytes=229376 combine_payload_bytes=233914368 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=234143744 ingress_bytes=290537472 link_bytes=290537472 token_bytes=468058112 weight_movement_bytes=56623104 total_nvlink_bytes=524681216 weight_to_token_ratio=0.120975 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.783787103e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=205193216 dispatch_meta_bytes=201528 combine_payload_bytes=205193216 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=205394744 ingress_bytes=280690688 link_bytes=280690688 token_bytes=410587960 weight_movement_bytes=75497472 total_nvlink_bytes=486085432 weight_to_token_ratio=0.183876 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.689440064e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234160128 dispatch_meta_bytes=229144 combine_payload_bytes=234160128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234389272 ingress_bytes=271908864 link_bytes=271908864 token_bytes=468549400 weight_movement_bytes=37748736 total_nvlink_bytes=506298136 weight_to_token_ratio=0.080565 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605296947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=224575488 dispatch_meta_bytes=229424 combine_payload_bytes=224575488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224804912 ingress_bytes=243449856 link_bytes=243449856 token_bytes=449380400 weight_movement_bytes=18874368 total_nvlink_bytes=468254768 weight_to_token_ratio=0.042001 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332616735e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60668 payload_rows_remote=54488 dup_rows=4868 dup_rows_remote=4868 dedup_ratio=0.074280 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 259, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 2836, 0, 0, 0, 0, 1097, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 21, 0, 0, 0, 0, 4094, 0, 1192, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7864, 5672, 8188, 6180, 8188, 8192, 8192] dup_rows_per_dst=[0, 244, 0, 2194, 0, 0, 2426, 4] combine_rows_per_src=[8192, 7864, 5672, 8188, 6180, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59738 payload_rows_remote=51546 dup_rows=5798 dup_rows_remote=5796 dedup_ratio=0.088470 router_tokens_per_expert=[0, 0, 0, 3, 4062, 0, 0, 705, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 3228, 0, 0, 0, 0, 0, 0, 4093, 9, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[3622, 8186, 8186, 8150, 7028, 8184, 8190, 8192] dup_rows_per_dst=[1016, 36, 0, 14, 4330, 398, 2, 2] combine_rows_per_src=[3622, 8186, 8186, 8150, 7028, 8184, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62258 payload_rows_remote=54066 dup_rows=3278 dup_rows_remote=3278 dedup_ratio=0.050018 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1587, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 4918, 8192, 8192] dup_rows_per_dst=[3174, 0, 0, 2, 102, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 4918, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=223182848 dispatch_meta_bytes=237424 combine_payload_bytes=223182848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223420272 ingress_bytes=242057216 link_bytes=242057216 token_bytes=446603120 weight_movement_bytes=18874368 total_nvlink_bytes=465477488 weight_to_token_ratio=0.042262 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.319273144e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=211132416 dispatch_meta_bytes=229368 combine_payload_bytes=211132416 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=211361784 ingress_bytes=248881152 link_bytes=248881152 token_bytes=422494200 weight_movement_bytes=37748736 total_nvlink_bytes=460242936 weight_to_token_ratio=0.089347 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384656741e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63476 payload_rows_remote=55284 dup_rows=2060 dup_rows_remote=72 dedup_ratio=0.031433 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 994, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 1, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3095, 0, 0, 0, 0, 0, 4089, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8192, 8192, 6190, 8148, 8192] dup_rows_per_dst=[1988, 40, 0, 16, 14, 2, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 8192, 6190, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64954 payload_rows_remote=56762 dup_rows=582 dup_rows_remote=582 dedup_ratio=0.008881 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 283, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7610, 8192, 8192] dup_rows_per_dst=[566, 0, 0, 0, 16, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7610, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57358 payload_rows_remote=49166 dup_rows=8178 dup_rows_remote=8178 dedup_ratio=0.124786 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 16, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 8158, 0, 20] combine_rows_per_src=[8192, 8192, 8192, 8192, 16, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62900 payload_rows_remote=55136 dup_rows=2636 dup_rows_remote=428 dedup_ratio=0.040222 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 327, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[5984, 8192, 8192, 8192, 8192, 7764, 8192, 8192] dup_rows_per_dst=[428, 0, 0, 0, 0, 2208, 0, 0] combine_rows_per_src=[5984, 8192, 8192, 8192, 8192, 7764, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=221454336 dispatch_meta_bytes=229376 combine_payload_bytes=221454336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221683712 ingress_bytes=240328704 link_bytes=240328704 token_bytes=443138048 weight_movement_bytes=18874368 total_nvlink_bytes=462012416 weight_to_token_ratio=0.042593 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.302711393e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=226443264 dispatch_meta_bytes=221424 combine_payload_bytes=226443264 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=226664688 ingress_bytes=264192000 link_bytes=264192000 token_bytes=453107952 weight_movement_bytes=37748736 total_nvlink_bytes=490856688 weight_to_token_ratio=0.083311 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.531357753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53398 payload_rows_remote=46040 dup_rows=12138 dup_rows_remote=11410 dedup_ratio=0.185211 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 582, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 3446, 0, 0, 2837, 0, 23, 0, 0, 1180, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2812, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 222, 0, 0, 0, 0, 4091, 0, 1168, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4091, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7358, 7184, 3322, 8192, 8182, 2776, 8192] dup_rows_per_dst=[0, 728, 5130, 626, 5630, 0, 14, 10] combine_rows_per_src=[8192, 7358, 7184, 3322, 8192, 8182, 2776, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=232497152 dispatch_meta_bytes=229376 combine_payload_bytes=232497152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232726528 ingress_bytes=251371520 link_bytes=251371520 token_bytes=465223680 weight_movement_bytes=18874368 total_nvlink_bytes=484098048 weight_to_token_ratio=0.040571 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408518222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=201383936 dispatch_meta_bytes=229376 combine_payload_bytes=201383936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=201613312 ingress_bytes=239132672 link_bytes=239132672 token_bytes=402997248 weight_movement_bytes=37748736 total_nvlink_bytes=440745984 weight_to_token_ratio=0.093670 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.291251602e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=222256 combine_payload_bytes=225837056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=226059312 ingress_bytes=263585792 link_bytes=263585792 token_bytes=451896368 weight_movement_bytes=37748736 total_nvlink_bytes=489645104 weight_to_token_ratio=0.083534 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.525549367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61602 payload_rows_remote=53410 dup_rows=3934 dup_rows_remote=3930 dedup_ratio=0.060028 router_tokens_per_expert=[2, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 1961, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 2378, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 7720, 8192, 8180, 8192, 8178, 8188, 4760] dup_rows_per_dst=[4, 0, 3922, 0, 4, 0, 0, 4] combine_rows_per_src=[8192, 7720, 8192, 8180, 8192, 8178, 8188, 4760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=188579840 dispatch_meta_bytes=229800 combine_payload_bytes=188579840 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=188809640 ingress_bytes=226328576 link_bytes=226328576 token_bytes=377389480 weight_movement_bytes=37748736 total_nvlink_bytes=415138216 weight_to_token_ratio=0.100026 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.168569055e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58782 payload_rows_remote=50590 dup_rows=6754 dup_rows_remote=6754 dedup_ratio=0.103058 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1062, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3033, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0] payload_rows_per_dst=[2130, 8188, 8192, 8150, 7558, 8186, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 6, 6, 6692, 50, 0] combine_rows_per_src=[2130, 8188, 8192, 8150, 7558, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=218767360 dispatch_meta_bytes=229360 combine_payload_bytes=218767360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218996720 ingress_bytes=237641728 link_bytes=237641728 token_bytes=437764080 weight_movement_bytes=18874368 total_nvlink_bytes=456638448 weight_to_token_ratio=0.043115 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276966111e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58030 payload_rows_remote=52032 dup_rows=7506 dup_rows_remote=7506 dedup_ratio=0.114532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 377, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 2999, 0, 0, 0, 0, 1307, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 31, 0, 0, 0, 0, 4095, 0, 1507, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8000, 5998, 8190, 8192, 8190, 3076, 8192] dup_rows_per_dst=[0, 522, 0, 2614, 4366, 0, 2, 2] combine_rows_per_src=[8192, 8000, 5998, 8190, 8192, 8190, 3076, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54302 payload_rows_remote=46110 dup_rows=11234 dup_rows_remote=11234 dedup_ratio=0.171417 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 796, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 2798, 0, 0, 0, 0, 1455, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 0, 0, 0, 0, 4089, 0, 1114, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8096, 8178, 3868, 7334, 8178, 2300, 8156] dup_rows_per_dst=[0, 3046, 3990, 152, 4032, 0, 0, 14] combine_rows_per_src=[8192, 8096, 8178, 3868, 7334, 8178, 2300, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=207216640 dispatch_meta_bytes=229376 combine_payload_bytes=207216640 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=207446016 ingress_bytes=226091008 link_bytes=226091008 token_bytes=414662656 weight_movement_bytes=18874368 total_nvlink_bytes=433537024 weight_to_token_ratio=0.045517 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.166292795e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61528 payload_rows_remote=53336 dup_rows=4008 dup_rows_remote=3450 dedup_ratio=0.061157 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3917, 0, 0, 0, 0, 279, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 7834, 8192, 5916, 8192, 6838, 8192] dup_rows_per_dst=[0, 36, 0, 558, 1352, 0, 2062, 0] combine_rows_per_src=[8192, 8172, 7834, 8192, 5916, 8192, 6838, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65056 payload_rows_remote=56864 dup_rows=480 dup_rows_remote=480 dedup_ratio=0.007324 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 182, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7712, 8192, 8192, 8192] dup_rows_per_dst=[360, 0, 0, 0, 0, 120, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7712, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57942 payload_rows_remote=49750 dup_rows=7594 dup_rows_remote=7594 dedup_ratio=0.115875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3950, 0, 0, 0, 0, 98, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7900, 8192, 890, 8192, 8192, 8192] dup_rows_per_dst=[0, 18, 0, 196, 0, 0, 7380, 0] combine_rows_per_src=[8192, 8192, 7900, 8192, 890, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=213123072 dispatch_meta_bytes=238152 combine_payload_bytes=213123072 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=213361224 ingress_bytes=307494912 link_bytes=307494912 token_bytes=426484296 weight_movement_bytes=94371840 total_nvlink_bytes=520856136 weight_to_token_ratio=0.221279 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.946264950e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=188866560 dispatch_meta_bytes=229376 combine_payload_bytes=188866560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189095936 ingress_bytes=207740928 link_bytes=207740928 token_bytes=377962496 weight_movement_bytes=18874368 total_nvlink_bytes=396836864 weight_to_token_ratio=0.049937 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.990471357e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59738 payload_rows_remote=52696 dup_rows=5798 dup_rows_remote=5798 dedup_ratio=0.088470 router_tokens_per_expert=[5, 0, 0, 0, 4087, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 2884, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 1824, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8184, 7042, 8192, 8144, 8192, 8144, 8182, 3658] dup_rows_per_dst=[10, 0, 5768, 0, 10, 0, 0, 10] combine_rows_per_src=[8184, 7042, 8192, 8144, 8192, 8144, 8182, 3658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=218464256 dispatch_meta_bytes=227144 combine_payload_bytes=218464256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218691400 ingress_bytes=256212992 link_bytes=256212992 token_bytes=437155656 weight_movement_bytes=37748736 total_nvlink_bytes=474904392 weight_to_token_ratio=0.086351 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.454906825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63486 payload_rows_remote=57332 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.031281 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0] payload_rows_per_dst=[8188, 8192, 8192, 8188, 6154, 8192, 8192, 8188] dup_rows_per_dst=[1410, 4, 4, 0, 0, 632, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8188, 6154, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=232914944 dispatch_meta_bytes=229376 combine_payload_bytes=232914944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233144320 ingress_bytes=251789312 link_bytes=251789312 token_bytes=466059264 weight_movement_bytes=18874368 total_nvlink_bytes=484933632 weight_to_token_ratio=0.040498 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412521300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=203776000 dispatch_meta_bytes=229376 combine_payload_bytes=203776000 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=204005376 ingress_bytes=241524736 link_bytes=241524736 token_bytes=407781376 weight_movement_bytes=37748736 total_nvlink_bytes=445530112 weight_to_token_ratio=0.092571 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.314171183e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=215842816 dispatch_meta_bytes=233976 combine_payload_bytes=215842816 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=216076792 ingress_bytes=310214656 link_bytes=310214656 token_bytes=431919608 weight_movement_bytes=94371840 total_nvlink_bytes=526291448 weight_to_token_ratio=0.218494 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.972324199e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60034 payload_rows_remote=51842 dup_rows=5502 dup_rows_remote=5496 dedup_ratio=0.083954 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 644, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 2829, 0, 0, 0, 0, 997, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 4092, 0, 1096, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8144, 5624, 8186, 5322, 8186, 8190, 8192] dup_rows_per_dst=[2, 1256, 0, 1994, 0, 36, 2208, 6] combine_rows_per_src=[8190, 8144, 5624, 8186, 5322, 8186, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=237528 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235069400 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469901272 weight_movement_bytes=18874368 total_nvlink_bytes=488775640 weight_to_token_ratio=0.040167 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62144 payload_rows_remote=53952 dup_rows=3392 dup_rows_remote=6 dedup_ratio=0.051758 router_tokens_per_expert=[1, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 1693, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2594, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8188, 7836, 8192, 8184, 8192, 8172, 8190, 5190] dup_rows_per_dst=[2, 0, 3386, 0, 2, 0, 0, 2] combine_rows_per_src=[8188, 7836, 8192, 8184, 8192, 8172, 8190, 5190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59702 payload_rows_remote=51518 dup_rows=5834 dup_rows_remote=5820 dedup_ratio=0.089020 router_tokens_per_expert=[7, 0, 0, 0, 4085, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 3850, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 1308, 0, 0, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[8184, 8154, 8008, 8184, 8192, 8172, 8178, 2630] dup_rows_per_dst=[14, 3130, 2662, 0, 14, 0, 0, 14] combine_rows_per_src=[8184, 8154, 8008, 8184, 8192, 8172, 8178, 2630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=212344832 dispatch_meta_bytes=229352 combine_payload_bytes=212344832 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=212574184 ingress_bytes=231219200 link_bytes=231219200 token_bytes=424919016 weight_movement_bytes=18874368 total_nvlink_bytes=443793384 weight_to_token_ratio=0.044419 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.215428607e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65510 payload_rows_remote=57318 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8166] dup_rows_per_dst=[0, 0, 26, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61968 payload_rows_remote=53776 dup_rows=3568 dup_rows_remote=3568 dedup_ratio=0.054443 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2312, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[4624, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 468, 3100, 0, 0] combine_rows_per_src=[4624, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60364 payload_rows_remote=52172 dup_rows=5172 dup_rows_remote=5172 dedup_ratio=0.078918 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3114, 8192, 8192, 8192, 8192, 8098] dup_rows_per_dst=[0, 0, 94, 82, 0, 0, 0, 4996] combine_rows_per_src=[8192, 8192, 3114, 8192, 8192, 8192, 8192, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=220987392 dispatch_meta_bytes=215832 combine_payload_bytes=220987392 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=221203224 ingress_bytes=296484864 link_bytes=296484864 token_bytes=442190616 weight_movement_bytes=75497472 total_nvlink_bytes=517688088 weight_to_token_ratio=0.170735 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19456 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.840772087e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63254 payload_rows_remote=55062 dup_rows=2282 dup_rows_remote=2272 dedup_ratio=0.034821 router_tokens_per_expert=[0, 4091, 0, 0, 0, 0, 0, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0] payload_rows_per_dst=[8182, 8192, 8192, 8182, 5942, 8190, 8192, 8182] dup_rows_per_dst=[1852, 10, 10, 0, 0, 410, 0, 0] combine_rows_per_src=[8182, 8192, 8192, 8182, 5942, 8190, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=211017728 dispatch_meta_bytes=229352 combine_payload_bytes=211017728 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=211247080 ingress_bytes=229892096 link_bytes=229892096 token_bytes=422264808 weight_movement_bytes=18874368 total_nvlink_bytes=441139176 weight_to_token_ratio=0.044698 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.202712950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=229376 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235003904 ingress_bytes=253648896 link_bytes=253648896 token_bytes=469778432 weight_movement_bytes=18874368 total_nvlink_bytes=488652800 weight_to_token_ratio=0.040177 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430338918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61898 payload_rows_remote=53708 dup_rows=3638 dup_rows_remote=3638 dedup_ratio=0.055511 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 0, 0, 0, 0, 0, 2894, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3482, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5792, 8192, 6964, 8190, 8192, 8188, 8188] dup_rows_per_dst=[6, 74, 0, 0, 0, 4, 0, 3554] combine_rows_per_src=[8192, 5792, 8192, 6964, 8190, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=220266496 dispatch_meta_bytes=229376 combine_payload_bytes=220266496 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=220495872 ingress_bytes=276889600 link_bytes=276889600 token_bytes=440762368 weight_movement_bytes=56623104 total_nvlink_bytes=497385472 weight_to_token_ratio=0.128466 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.653019909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=213696512 dispatch_meta_bytes=229376 combine_payload_bytes=213696512 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=213925888 ingress_bytes=251445248 link_bytes=251445248 token_bytes=427622400 weight_movement_bytes=37748736 total_nvlink_bytes=465371136 weight_to_token_ratio=0.088276 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409224648e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=225533952 dispatch_meta_bytes=229336 combine_payload_bytes=225533952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225763288 ingress_bytes=244408320 link_bytes=244408320 token_bytes=451297240 weight_movement_bytes=18874368 total_nvlink_bytes=470171608 weight_to_token_ratio=0.041822 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.341800266e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61908 payload_rows_remote=53716 dup_rows=3628 dup_rows_remote=162 dedup_ratio=0.055359 router_tokens_per_expert=[3, 0, 0, 0, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 2870, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 1711, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8176, 7078, 5754, 8186, 8154, 8182, 8186, 8192] dup_rows_per_dst=[6, 0, 0, 0, 8, 112, 36, 3466] combine_rows_per_src=[8176, 7078, 5754, 8186, 8154, 8182, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=219987968 dispatch_meta_bytes=229384 combine_payload_bytes=219987968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220217352 ingress_bytes=238862336 link_bytes=238862336 token_bytes=440205320 weight_movement_bytes=18874368 total_nvlink_bytes=459079688 weight_to_token_ratio=0.042876 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.288661376e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64354 payload_rows_remote=56162 dup_rows=1182 dup_rows_remote=1180 dedup_ratio=0.018036 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 7016, 8192, 8192, 8190] dup_rows_per_dst=[1010, 2, 2, 0, 0, 168, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 7016, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=220020736 dispatch_meta_bytes=215512 combine_payload_bytes=220020736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=220236248 ingress_bytes=257769472 link_bytes=257769472 token_bytes=440256984 weight_movement_bytes=37748736 total_nvlink_bytes=478005720 weight_to_token_ratio=0.085743 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=22144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.469820250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59984 payload_rows_remote=51806 dup_rows=5552 dup_rows_remote=750 dedup_ratio=0.084717 router_tokens_per_expert=[0, 4089, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0] payload_rows_per_dst=[8178, 8192, 8192, 8178, 5008, 5866, 8192, 8178] dup_rows_per_dst=[4802, 14, 14, 2, 0, 720, 0, 0] combine_rows_per_src=[8178, 8192, 8192, 8178, 5008, 5866, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61048 payload_rows_remote=52856 dup_rows=4488 dup_rows_remote=4488 dedup_ratio=0.068481 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 3704, 8192, 8192, 8192] dup_rows_per_dst=[4488, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 3704, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60972 payload_rows_remote=52780 dup_rows=4564 dup_rows_remote=876 dedup_ratio=0.069641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 4013, 0, 0, 0, 0, 436, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 1848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3856, 8026, 8192, 8130, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 872, 0, 0, 3688, 0] combine_rows_per_src=[8192, 3856, 8026, 8192, 8130, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=230039552 dispatch_meta_bytes=229368 combine_payload_bytes=230039552 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=230268920 ingress_bytes=248913920 link_bytes=248913920 token_bytes=460308472 weight_movement_bytes=18874368 total_nvlink_bytes=479182840 weight_to_token_ratio=0.041004 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384970708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62606 payload_rows_remote=54414 dup_rows=2930 dup_rows_remote=2928 dedup_ratio=0.044708 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[7188, 8192, 8192, 8192, 6266, 8192, 8192, 8192] dup_rows_per_dst=[1924, 0, 0, 0, 1004, 2, 0, 0] combine_rows_per_src=[7188, 8192, 8192, 8192, 6266, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61422 payload_rows_remote=53230 dup_rows=4114 dup_rows_remote=28 dedup_ratio=0.062775 router_tokens_per_expert=[0, 0, 0, 0, 4, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 21, 0, 0, 0, 0, 0, 0, 3184, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 2924, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8180, 8182, 8192, 8182, 4110] dup_rows_per_dst=[18, 4086, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8180, 8182, 8192, 8182, 4110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=212197376 dispatch_meta_bytes=210224 combine_payload_bytes=212197376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212407600 ingress_bytes=249946112 link_bytes=249946112 token_bytes=424604976 weight_movement_bytes=37748736 total_nvlink_bytes=462353712 weight_to_token_ratio=0.088903 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=22656 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.394860664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61910 payload_rows_remote=53718 dup_rows=3626 dup_rows_remote=3546 dedup_ratio=0.055328 router_tokens_per_expert=[0, 2, 0, 2329, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 22, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[4662, 8188, 8192, 8192, 8192, 8188, 8104, 8192] dup_rows_per_dst=[0, 3306, 0, 0, 80, 0, 188, 52] combine_rows_per_src=[4662, 8188, 8192, 8192, 8192, 8188, 8104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=216498176 dispatch_meta_bytes=229376 combine_payload_bytes=216498176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=216727552 ingress_bytes=273121280 link_bytes=273121280 token_bytes=433225728 weight_movement_bytes=56623104 total_nvlink_bytes=489848832 weight_to_token_ratio=0.130701 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.616913721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=216186880 dispatch_meta_bytes=214624 combine_payload_bytes=216186880 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=216401504 ingress_bytes=291684352 link_bytes=291684352 token_bytes=432588384 weight_movement_bytes=75497472 total_nvlink_bytes=508085856 weight_to_token_ratio=0.174525 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.794775943e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=222879744 dispatch_meta_bytes=229368 combine_payload_bytes=222879744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223109112 ingress_bytes=241754112 link_bytes=241754112 token_bytes=445988856 weight_movement_bytes=18874368 total_nvlink_bytes=464863224 weight_to_token_ratio=0.042320 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316368951e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57808 payload_rows_remote=49622 dup_rows=7728 dup_rows_remote=7692 dedup_ratio=0.117920 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0] payload_rows_per_dst=[548, 8156, 8156, 8186, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 6, 6, 36, 7300, 344, 0, 36] combine_rows_per_src=[548, 8156, 8156, 8186, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=218030080 dispatch_meta_bytes=213032 combine_payload_bytes=218030080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218243112 ingress_bytes=236904448 link_bytes=236904448 token_bytes=436273192 weight_movement_bytes=18874368 total_nvlink_bytes=455147560 weight_to_token_ratio=0.043263 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.269901856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=220028928 dispatch_meta_bytes=229056 combine_payload_bytes=220028928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220257984 ingress_bytes=238903296 link_bytes=238903296 token_bytes=440286912 weight_movement_bytes=18874368 total_nvlink_bytes=459161280 weight_to_token_ratio=0.042868 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.289053835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59266 payload_rows_remote=51074 dup_rows=6270 dup_rows_remote=6270 dedup_ratio=0.095673 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 3168, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3476, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7044, 8192, 8188, 8192, 8192, 8190, 3076] dup_rows_per_dst=[2, 3204, 0, 3062, 0, 2, 0, 0] combine_rows_per_src=[8192, 7044, 8192, 8188, 8192, 8192, 8190, 3076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=203251712 dispatch_meta_bytes=229256 combine_payload_bytes=203251712 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=203480968 ingress_bytes=222126080 link_bytes=222126080 token_bytes=406732680 weight_movement_bytes=18874368 total_nvlink_bytes=425607048 weight_to_token_ratio=0.046405 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.128302806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21108406528 prefetch_bytes=887095296 reduce_bytes=603979776 weight_movement_bytes=1491075072 egress_bytes=10559715584 ingress_bytes=12039766016 link_bytes=12039766016 total_nvlink_bytes=22599481600 weight_to_token_ratio=0.070639 prefetch_share_of_ingress=0.073680 dedup_ratio=0.063034 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=32 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.099826 gemm_inner_pad_upper_bound=34798 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.403314876e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61494 payload_rows_remote=53302 dup_rows=4042 dup_rows_remote=4018 dedup_ratio=0.061676 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 7, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 84, 0, 0, 0, 0, 0, 0, 3224, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 2758, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8178, 8156, 8178, 4230] dup_rows_per_dst=[24, 3968, 0, 4, 32, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8176, 8178, 8156, 8178, 4230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59430 payload_rows_remote=51582 dup_rows=6106 dup_rows_remote=430 dedup_ratio=0.093170 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3146, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6568, 8192, 7848, 8192, 8192, 8192, 4054] dup_rows_per_dst=[0, 0, 0, 5676, 0, 0, 0, 430] combine_rows_per_src=[8192, 6568, 8192, 7848, 8192, 8192, 8192, 4054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57378 payload_rows_remote=49186 dup_rows=8158 dup_rows_remote=8158 dedup_ratio=0.124481 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 34, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8158] combine_rows_per_src=[8192, 8192, 34, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=209199104 dispatch_meta_bytes=229376 combine_payload_bytes=209199104 prefetch_bytes=0 reduce_bytes=0 egress_bytes=209428480 ingress_bytes=209199104 link_bytes=209428480 token_bytes=418627584 weight_movement_bytes=0 total_nvlink_bytes=418627584 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.006640650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59132 payload_rows_remote=50940 dup_rows=6404 dup_rows_remote=6404 dedup_ratio=0.097717 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2795, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4390, 8192, 5590, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6404] combine_rows_per_src=[8192, 4390, 8192, 5590, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59696 payload_rows_remote=57166 dup_rows=5840 dup_rows_remote=5840 dedup_ratio=0.089111 router_tokens_per_expert=[0, 5, 0, 2724, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 4091, 0, 5, 0, 5, 66, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[8192, 2530, 8192, 8192, 8192, 8182, 8024, 8192] dup_rows_per_dst=[5448, 0, 0, 0, 222, 0, 18, 152] combine_rows_per_src=[8192, 2530, 8192, 8192, 8192, 8182, 8024, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=218324992 dispatch_meta_bytes=229280 combine_payload_bytes=218324992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218554272 ingress_bytes=237199360 link_bytes=237199360 token_bytes=436879264 weight_movement_bytes=18874368 total_nvlink_bytes=455753632 weight_to_token_ratio=0.043203 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272727558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=211279872 dispatch_meta_bytes=208048 combine_payload_bytes=211279872 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=211487920 ingress_bytes=324526080 link_bytes=324526080 token_bytes=422767792 weight_movement_bytes=113246208 total_nvlink_bytes=536014000 weight_to_token_ratio=0.267869 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.109449222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=201465856 dispatch_meta_bytes=229376 combine_payload_bytes=201465856 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=201695232 ingress_bytes=239214592 link_bytes=239214592 token_bytes=403161088 weight_movement_bytes=37748736 total_nvlink_bytes=440909824 weight_to_token_ratio=0.093632 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.292036520e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=208650240 dispatch_meta_bytes=229376 combine_payload_bytes=208650240 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208879616 ingress_bytes=246398976 link_bytes=246398976 token_bytes=417529856 weight_movement_bytes=37748736 total_nvlink_bytes=455278592 weight_to_token_ratio=0.090410 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.360873752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60792 payload_rows_remote=52606 dup_rows=4744 dup_rows_remote=228 dedup_ratio=0.072388 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 89, 0, 0, 0, 0, 0, 0, 2840, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2999, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8132, 5718, 8192, 5998, 8186, 8192, 8188, 8186] dup_rows_per_dst=[10, 152, 0, 26, 0, 6, 34, 4516] combine_rows_per_src=[8132, 5718, 8192, 5998, 8186, 8192, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234151936 dispatch_meta_bytes=252024 combine_payload_bytes=234151936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234403960 ingress_bytes=271900672 link_bytes=271900672 token_bytes=468555896 weight_movement_bytes=37748736 total_nvlink_bytes=506304632 weight_to_token_ratio=0.080564 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605218456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60504 payload_rows_remote=52312 dup_rows=5032 dup_rows_remote=5032 dedup_ratio=0.076782 router_tokens_per_expert=[0, 0, 0, 2096, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 299, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[6398, 5014, 8192, 8192, 8192, 8190, 8134, 8192] dup_rows_per_dst=[3036, 1330, 0, 0, 38, 0, 26, 602] combine_rows_per_src=[6398, 5014, 8192, 8192, 8192, 8190, 8134, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=215474176 dispatch_meta_bytes=211336 combine_payload_bytes=215474176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215685512 ingress_bytes=234348544 link_bytes=234348544 token_bytes=431159688 weight_movement_bytes=18874368 total_nvlink_bytes=450034056 weight_to_token_ratio=0.043776 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.245412442e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20534466008 prefetch_bytes=924844032 reduce_bytes=792723456 weight_movement_bytes=1717567488 egress_bytes=10272658904 ingress_bytes=11979374592 link_bytes=11979374592 total_nvlink_bytes=22252033496 weight_to_token_ratio=0.083643 prefetch_share_of_ingress=0.077203 dedup_ratio=0.085205 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=42 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100550 gemm_inner_pad_upper_bound=34671 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.391259857e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55526 payload_rows_remote=47334 dup_rows=10010 dup_rows_remote=306 dedup_ratio=0.152740 router_tokens_per_expert=[0, 7, 0, 3296, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 7, 41, 0, 0, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[8192, 3426, 8192, 8192, 5102, 8178, 7976, 6268] dup_rows_per_dst=[9704, 42, 0, 0, 152, 36, 6, 70] combine_rows_per_src=[8192, 3426, 8192, 8192, 5102, 8178, 7976, 6268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62918 payload_rows_remote=54726 dup_rows=2618 dup_rows_remote=2618 dedup_ratio=0.039948 router_tokens_per_expert=[0, 0, 0, 2840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5680, 8192, 8192, 8192, 8192, 8192, 8086, 8192] dup_rows_per_dst=[0, 2366, 0, 0, 28, 0, 26, 198] combine_rows_per_src=[5680, 8192, 8192, 8192, 8192, 8192, 8086, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58726 payload_rows_remote=50534 dup_rows=6810 dup_rows_remote=6810 dedup_ratio=0.103912 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[1384, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 6758, 52, 0, 0] combine_rows_per_src=[1384, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=214269952 dispatch_meta_bytes=229376 combine_payload_bytes=214269952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214499328 ingress_bytes=233144320 link_bytes=233144320 token_bytes=428769280 weight_movement_bytes=18874368 total_nvlink_bytes=447643648 weight_to_token_ratio=0.044020 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.233874160e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63888 payload_rows_remote=55696 dup_rows=1648 dup_rows_remote=1648 dedup_ratio=0.025146 router_tokens_per_expert=[0, 0, 0, 3294, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6588, 8192, 8192, 8192, 8192, 8192, 8148, 8192] dup_rows_per_dst=[0, 1538, 0, 0, 72, 0, 22, 16] combine_rows_per_src=[6588, 8192, 8192, 8192, 8192, 8192, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=193880064 dispatch_meta_bytes=190560 combine_payload_bytes=193880064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=194070624 ingress_bytes=250503168 link_bytes=250503168 token_bytes=387950688 weight_movement_bytes=56623104 total_nvlink_bytes=444573792 weight_to_token_ratio=0.145954 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.400198101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=224157696 dispatch_meta_bytes=229376 combine_payload_bytes=224157696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224387072 ingress_bytes=243032064 link_bytes=243032064 token_bytes=448544768 weight_movement_bytes=18874368 total_nvlink_bytes=467419136 weight_to_token_ratio=0.042079 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.328613658e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=206987264 dispatch_meta_bytes=229376 combine_payload_bytes=206987264 prefetch_bytes=0 reduce_bytes=0 egress_bytes=207216640 ingress_bytes=206987264 link_bytes=207216640 token_bytes=414203904 weight_movement_bytes=0 total_nvlink_bytes=414203904 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.985447888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20884587368 prefetch_bytes=736100352 reduce_bytes=943718400 weight_movement_bytes=1679818752 egress_bytes=10447790952 ingress_bytes=12116615168 link_bytes=12116844544 total_nvlink_bytes=22564406120 weight_to_token_ratio=0.080433 prefetch_share_of_ingress=0.060751 dedup_ratio=0.072175 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=50 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100911 gemm_inner_pad_upper_bound=33147 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.418700887e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61472 payload_rows_remote=53280 dup_rows=4064 dup_rows_remote=3374 dedup_ratio=0.062012 router_tokens_per_expert=[0, 3, 0, 2518, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 51, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[5042, 8184, 8192, 8192, 8192, 8186, 7292, 8192] dup_rows_per_dst=[0, 3240, 0, 0, 80, 0, 54, 690] combine_rows_per_src=[5042, 8184, 8192, 8192, 8192, 8186, 7292, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=228130816 dispatch_meta_bytes=229376 combine_payload_bytes=228130816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=228360192 ingress_bytes=265879552 link_bytes=265879552 token_bytes=456491008 weight_movement_bytes=37748736 total_nvlink_bytes=494239744 weight_to_token_ratio=0.082693 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.547527046e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19223201576 prefetch_bytes=792723456 reduce_bytes=830472192 weight_movement_bytes=1623195648 egress_bytes=9616918312 ingress_bytes=11229478912 link_bytes=11229708288 total_nvlink_bytes=20846397224 weight_to_token_ratio=0.084439 prefetch_share_of_ingress=0.070593 dedup_ratio=0.131501 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=44 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101201 gemm_inner_pad_upper_bound=32893 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.241615405e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21056012280 prefetch_bytes=962592768 reduce_bytes=830472192 weight_movement_bytes=1793064960 egress_bytes=10533486584 ingress_bytes=12315590656 link_bytes=12315590656 total_nvlink_bytes=22849077240 weight_to_token_ratio=0.085157 prefetch_share_of_ingress=0.078161 dedup_ratio=0.062988 prefetch_slots=102 unused_prefetch_slots=1434 slot_utilization=0.066406 reduce_slots=44 remote_demand=51 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100911 gemm_inner_pad_upper_bound=33274 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.458373542e-04 prefetch_bytes_per_gemm_flop=1.921477141e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20882792904 prefetch_bytes=924844032 reduce_bytes=1000341504 weight_movement_bytes=1925185536 egress_bytes=10446856648 ingress_bytes=12361121792 link_bytes=12361121792 total_nvlink_bytes=22807978440 weight_to_token_ratio=0.092190 prefetch_share_of_ingress=0.074819 dedup_ratio=0.074137 prefetch_slots=98 unused_prefetch_slots=1438 slot_utilization=0.063802 reduce_slots=53 remote_demand=49 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101635 gemm_inner_pad_upper_bound=32385 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.467462228e-04 prefetch_bytes_per_gemm_flop=1.846125096e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=218234880 dispatch_meta_bytes=226616 combine_payload_bytes=218234880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=218461496 ingress_bytes=255983616 link_bytes=255983616 token_bytes=436696376 weight_movement_bytes=37748736 total_nvlink_bytes=474445112 weight_to_token_ratio=0.086442 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.452709057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63132 payload_rows_remote=54940 dup_rows=2404 dup_rows_remote=2404 dedup_ratio=0.036682 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2912, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8156, 8192, 5824, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2404] combine_rows_per_src=[8192, 8156, 8192, 5824, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19342103432 prefetch_bytes=792723456 reduce_bytes=924844032 weight_movement_bytes=1717567488 egress_bytes=9676338056 ingress_bytes=11383332864 link_bytes=11383332864 total_nvlink_bytes=21059670920 weight_to_token_ratio=0.088799 prefetch_share_of_ingress=0.069639 dedup_ratio=0.127314 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=49 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100839 gemm_inner_pad_upper_bound=33020 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.272281137e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=225034240 dispatch_meta_bytes=229376 combine_payload_bytes=225034240 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=225263616 ingress_bytes=262782976 link_bytes=262782976 token_bytes=450297856 weight_movement_bytes=37748736 total_nvlink_bytes=488046592 weight_to_token_ratio=0.083831 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.517857179e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59934 payload_rows_remote=51746 dup_rows=5602 dup_rows_remote=4358 dedup_ratio=0.085480 router_tokens_per_expert=[0, 0, 0, 1297, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2594, 8192, 8192, 8192, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 1796, 0, 0, 2418, 0, 1244, 144] combine_rows_per_src=[2594, 8192, 8192, 8192, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=211951616 dispatch_meta_bytes=224416 combine_payload_bytes=211951616 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=212176032 ingress_bytes=249700352 link_bytes=249700352 token_bytes=424127648 weight_movement_bytes=37748736 total_nvlink_bytes=461876384 weight_to_token_ratio=0.089003 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392505913e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20166865608 prefetch_bytes=830472192 reduce_bytes=924844032 weight_movement_bytes=1755316224 egress_bytes=10088878792 ingress_bytes=11833303040 link_bytes=11833532416 total_nvlink_bytes=21922181832 weight_to_token_ratio=0.087040 prefetch_share_of_ingress=0.070181 dedup_ratio=0.098361 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=49 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101562 gemm_inner_pad_upper_bound=32004 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.362147607e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none + [2026-08-29 23:37:49] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 757.3 | throughput per GPU (TFLOP/s/GPU): 117.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.756268E+00 | loss scale: 1.0 | grad norm: 2.902 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=39000 dup_rows=21012 dup_rows_remote=18336 dedup_ratio=0.320618 router_tokens_per_expert=[218, 0, 0, 0, 210, 198, 146, 7, 2, 2, 436, 383, 452, 431, 247, 361, 27, 331, 10, 446, 357, 365, 7, 0, 441, 123, 107, 207, 511, 6, 629, 29, 345, 310, 12, 131, 3, 65, 262, 226, 461, 569, 493, 423, 263, 54, 285, 385, 529, 439, 373, 0, 238, 428, 46, 31, 560, 152, 474, 735, 327, 438, 10, 211, 69, 350, 0, 53, 315, 0, 677, 357, 4, 1028, 337, 1076, 141, 70, 45, 952, 194, 49, 105, 240, 46, 305, 7, 0, 469, 260, 96, 198, 334, 332, 175, 214, 0, 181, 144, 124, 2, 321, 521, 75, 155, 37, 217, 399, 436, 232, 403, 0, 392, 523, 309, 155, 356, 3, 766, 111, 876, 287, 492, 27, 411, 0, 348, 0] payload_rows_per_dst=[5294, 5862, 4820, 5818, 5876, 5524, 5296, 6034] dup_rows_per_dst=[2362, 2468, 2616, 2660, 2954, 2676, 2502, 2774] combine_rows_per_src=[5294, 5862, 4820, 5818, 5876, 5524, 5296, 6034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=39084 dup_rows=21114 dup_rows_remote=18604 dedup_ratio=0.322174 router_tokens_per_expert=[186, 0, 0, 0, 92, 192, 47, 6, 0, 1, 417, 370, 526, 383, 179, 420, 33, 204, 2, 600, 209, 452, 0, 0, 301, 119, 126, 201, 522, 1, 647, 6, 293, 435, 10, 128, 3, 62, 255, 194, 329, 646, 561, 407, 336, 54, 333, 603, 655, 505, 479, 0, 465, 483, 34, 30, 515, 187, 329, 616, 219, 416, 38, 224, 19, 315, 0, 14, 358, 0, 464, 391, 3, 970, 385, 1350, 113, 146, 23, 853, 202, 21, 76, 293, 34, 177, 1, 3, 590, 209, 105, 220, 231, 470, 151, 218, 1, 161, 188, 202, 0, 233, 373, 66, 324, 24, 256, 525, 448, 250, 355, 0, 417, 542, 242, 145, 208, 1, 1061, 102, 852, 133, 572, 22, 268, 3, 303, 0] payload_rows_per_dst=[5322, 5850, 5032, 5338, 5582, 5894, 5816, 5588] dup_rows_per_dst=[2858, 2288, 2974, 2510, 2526, 2808, 3118, 2032] combine_rows_per_src=[5322, 5850, 5032, 5338, 5582, 5894, 5816, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44062 payload_rows_remote=38580 dup_rows=21474 dup_rows_remote=18116 dedup_ratio=0.327667 router_tokens_per_expert=[232, 0, 0, 0, 182, 223, 104, 14, 3, 6, 397, 357, 526, 480, 209, 423, 13, 371, 4, 570, 200, 339, 0, 0, 460, 82, 140, 208, 476, 9, 622, 57, 344, 391, 22, 76, 11, 88, 244, 207, 383, 537, 461, 202, 270, 48, 356, 543, 639, 390, 567, 0, 336, 434, 63, 37, 571, 199, 446, 590, 334, 412, 36, 225, 14, 365, 1, 44, 395, 0, 516, 392, 6, 719, 480, 1123, 114, 68, 34, 981, 174, 22, 69, 206, 30, 196, 1, 0, 493, 334, 119, 229, 301, 350, 180, 358, 0, 123, 198, 107, 1, 386, 501, 80, 132, 32, 305, 462, 666, 221, 492, 0, 323, 598, 292, 145, 226, 1, 1058, 103, 707, 110, 370, 26, 316, 2, 302, 0] payload_rows_per_dst=[5524, 5872, 5482, 5482, 4850, 5668, 5874, 5310] dup_rows_per_dst=[3246, 2304, 3358, 2618, 1860, 2702, 3654, 1732] combine_rows_per_src=[5524, 5872, 5482, 5482, 4850, 5668, 5874, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38804 dup_rows=21062 dup_rows_remote=18448 dedup_ratio=0.321381 router_tokens_per_expert=[222, 0, 0, 0, 87, 251, 38, 5, 2, 1, 388, 310, 606, 517, 223, 319, 39, 225, 3, 430, 284, 389, 2, 0, 434, 129, 149, 201, 579, 6, 507, 5, 329, 494, 8, 86, 5, 30, 334, 224, 449, 563, 481, 355, 357, 34, 278, 430, 528, 495, 408, 0, 384, 433, 39, 31, 630, 137, 432, 669, 309, 445, 25, 238, 25, 344, 0, 16, 352, 0, 753, 305, 0, 735, 414, 1176, 97, 60, 29, 1012, 153, 16, 70, 373, 32, 335, 0, 2, 552, 340, 92, 175, 389, 428, 221, 196, 0, 124, 190, 125, 1, 364, 524, 51, 226, 19, 326, 353, 535, 184, 392, 0, 376, 350, 304, 232, 322, 1, 789, 110, 847, 166, 443, 47, 374, 5, 284, 1] payload_rows_per_dst=[5428, 5632, 5112, 5456, 5670, 5962, 5562, 5652] dup_rows_per_dst=[2900, 2258, 2676, 2560, 2614, 3138, 2844, 2072] combine_rows_per_src=[5428, 5632, 5112, 5456, 5670, 5962, 5562, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43948 payload_rows_remote=38120 dup_rows=21588 dup_rows_remote=18192 dedup_ratio=0.329407 router_tokens_per_expert=[222, 0, 0, 0, 61, 193, 69, 4, 1, 1, 369, 524, 789, 396, 235, 356, 17, 241, 3, 526, 169, 386, 0, 0, 470, 89, 147, 188, 429, 3, 541, 5, 319, 383, 18, 100, 7, 59, 326, 203, 331, 584, 556, 315, 349, 32, 251, 692, 636, 529, 517, 0, 273, 566, 20, 25, 625, 166, 374, 756, 250, 357, 28, 214, 9, 285, 0, 14, 347, 0, 667, 365, 1, 1045, 385, 882, 78, 87, 16, 1003, 204, 17, 89, 315, 36, 210, 0, 0, 404, 356, 80, 143, 236, 369, 166, 255, 1, 131, 153, 115, 0, 334, 422, 75, 173, 14, 389, 358, 512, 301, 398, 0, 407, 438, 285, 149, 361, 0, 956, 113, 816, 107, 591, 21, 354, 5, 430, 0] payload_rows_per_dst=[5828, 5798, 5680, 5548, 5588, 4342, 5632, 5532] dup_rows_per_dst=[3396, 2472, 3534, 3168, 2774, 1418, 3032, 1794] combine_rows_per_src=[5828, 5798, 5680, 5548, 5588, 4342, 5632, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38314 dup_rows=21046 dup_rows_remote=17650 dedup_ratio=0.321136 router_tokens_per_expert=[214, 0, 2, 2, 80, 249, 67, 1, 1, 5, 391, 345, 565, 414, 275, 335, 61, 199, 4, 409, 264, 532, 0, 1, 340, 164, 183, 194, 420, 11, 662, 42, 360, 458, 27, 84, 4, 69, 345, 232, 277, 424, 408, 406, 444, 82, 296, 517, 705, 537, 480, 1, 396, 571, 64, 45, 609, 168, 451, 583, 183, 395, 35, 320, 66, 322, 0, 12, 370, 0, 636, 351, 6, 723, 363, 779, 79, 58, 31, 793, 167, 38, 96, 417, 31, 299, 9, 2, 609, 405, 86, 170, 355, 458, 200, 212, 0, 272, 180, 177, 0, 296, 490, 48, 215, 36, 389, 410, 378, 142, 398, 0, 396, 487, 319, 148, 315, 5, 752, 128, 813, 154, 438, 38, 411, 11, 371, 0] payload_rows_per_dst=[4988, 5628, 5302, 6090, 5874, 5476, 4956, 6176] dup_rows_per_dst=[2070, 2192, 2716, 2612, 2964, 3190, 1906, 3396] combine_rows_per_src=[4988, 5628, 5302, 6090, 5874, 5476, 4956, 6176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38396 dup_rows=21202 dup_rows_remote=18684 dedup_ratio=0.323517 router_tokens_per_expert=[238, 0, 0, 1, 114, 210, 40, 9, 2, 5, 389, 278, 507, 521, 215, 273, 66, 232, 4, 532, 231, 434, 1, 0, 424, 164, 198, 257, 470, 8, 481, 6, 422, 479, 14, 128, 40, 92, 376, 208, 291, 720, 484, 423, 285, 22, 208, 374, 623, 554, 517, 0, 270, 542, 54, 29, 596, 149, 330, 885, 245, 409, 39, 314, 109, 325, 1, 20, 405, 0, 594, 384, 1, 812, 390, 989, 83, 95, 33, 825, 146, 27, 66, 392, 37, 274, 4, 1, 881, 317, 85, 180, 371, 399, 205, 125, 0, 191, 169, 121, 15, 419, 442, 84, 311, 27, 274, 323, 356, 219, 400, 0, 399, 460, 287, 138, 291, 0, 660, 104, 804, 110, 415, 46, 308, 4, 383, 0] payload_rows_per_dst=[5496, 5938, 5662, 5534, 4868, 5942, 5322, 5572] dup_rows_per_dst=[3124, 2518, 3680, 2562, 1738, 2954, 2700, 1926] combine_rows_per_src=[5496, 5938, 5662, 5534, 4868, 5942, 5322, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=229344 combine_payload_bytes=159744000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=159973344 ingress_bytes=178618368 link_bytes=178618368 token_bytes=319717344 weight_movement_bytes=18874368 total_nvlink_bytes=338591712 weight_to_token_ratio=0.059035 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=7168 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.711433316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=230752 combine_payload_bytes=160088064 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=160318816 ingress_bytes=235585536 link_bytes=235585536 token_bytes=320406880 weight_movement_bytes=75497472 total_nvlink_bytes=395904352 weight_to_token_ratio=0.235630 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=3968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.257264691e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158023680 dispatch_meta_bytes=226784 combine_payload_bytes=158023680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158250464 ingress_bytes=214646784 link_bytes=214646784 token_bytes=316274144 weight_movement_bytes=56623104 total_nvlink_bytes=372897248 weight_to_token_ratio=0.179032 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=3968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.056639871e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=229008 combine_payload_bytes=158941184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=159170192 ingress_bytes=215564288 link_bytes=215564288 token_bytes=318111376 weight_movement_bytes=56623104 total_nvlink_bytes=374734480 weight_to_token_ratio=0.177998 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=6912 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.065430943e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=225248 combine_payload_bytes=156139520 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=156364768 ingress_bytes=193888256 link_bytes=193888256 token_bytes=312504288 weight_movement_bytes=37748736 total_nvlink_bytes=350253024 weight_to_token_ratio=0.120794 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=5760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857741870e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=223856 combine_payload_bytes=156934144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157158000 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314092144 weight_movement_bytes=37748736 total_nvlink_bytes=351840880 weight_to_token_ratio=0.120184 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=6144 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=228320 combine_payload_bytes=157270016 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=157498336 ingress_bytes=195018752 link_bytes=195018752 token_bytes=314768352 weight_movement_bytes=37748736 total_nvlink_bytes=352517088 weight_to_token_ratio=0.119925 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.076389 gemm_active_groups=19 gemm_peak_group_rows=4864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868573726e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52366 payload_rows_remote=45284 dup_rows=13170 dup_rows_remote=9082 dedup_ratio=0.200958 router_tokens_per_expert=[0, 28, 0, 0, 1617, 0, 0, 80, 184, 0, 0, 0, 10, 3, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 1, 0, 1, 0, 763, 0, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 479, 0, 0, 0, 629, 1757, 2478, 0, 0, 0, 0, 0, 1, 0, 0, 0, 31, 0, 655, 0, 3254, 2, 0, 0, 5, 943, 0, 0, 1, 351, 0, 0, 1317, 0, 0, 0, 2792, 3, 824, 1701, 266, 0, 113, 118, 0, 0, 0, 2, 0, 0, 0, 0, 3859, 28, 5, 0, 0, 1, 16, 569, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 628, 0] payload_rows_per_dst=[5744, 5928, 6464, 7082, 7028, 5782, 6548, 7790] dup_rows_per_dst=[2130, 1352, 842, 4088, 1570, 1254, 748, 1186] combine_rows_per_src=[5744, 5928, 6464, 7082, 7028, 5782, 6548, 7790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49332 payload_rows_remote=44234 dup_rows=16204 dup_rows_remote=14756 dedup_ratio=0.247253 router_tokens_per_expert=[0, 281, 0, 0, 260, 0, 0, 205, 505, 0, 3, 1, 725, 0, 0, 536, 0, 0, 2, 0, 0, 0, 1, 2, 154, 0, 0, 0, 1, 0, 0, 0, 1, 0, 493, 0, 2409, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 11, 1320, 0, 0, 0, 16, 2757, 847, 0, 0, 1, 0, 0, 1, 0, 0, 0, 800, 0, 1238, 0, 623, 3, 0, 0, 582, 1892, 0, 0, 0, 27, 0, 0, 735, 2, 0, 0, 3113, 5, 2182, 300, 1976, 0, 100, 863, 0, 0, 0, 0, 0, 0, 0, 0, 2321, 1183, 0, 0, 0, 0, 3, 1703, 0, 1749, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 1, 828, 0] payload_rows_per_dst=[6204, 7298, 5450, 6938, 5098, 6290, 6648, 5406] dup_rows_per_dst=[2612, 3612, 362, 2968, 1448, 1672, 2628, 902] combine_rows_per_src=[6204, 7298, 5450, 6938, 5098, 6290, 6648, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49128 payload_rows_remote=42558 dup_rows=16408 dup_rows_remote=13904 dedup_ratio=0.250366 router_tokens_per_expert=[0, 259, 0, 0, 1046, 0, 0, 111, 571, 0, 0, 0, 173, 0, 0, 746, 0, 0, 0, 0, 0, 0, 3, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1389, 0, 0, 0, 111, 2032, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 663, 0, 1153, 0, 0, 0, 82, 2538, 0, 0, 0, 511, 0, 0, 712, 0, 0, 0, 3163, 16, 1454, 421, 1824, 0, 29, 449, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 1068, 0, 0, 1, 0, 65, 1026, 0, 2511, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 949, 0] payload_rows_per_dst=[6688, 6426, 6466, 6662, 5388, 6570, 4108, 6820] dup_rows_per_dst=[3416, 764, 1208, 2934, 1648, 2504, 576, 3358] combine_rows_per_src=[6688, 6426, 6466, 6662, 5388, 6570, 4108, 6820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50298 payload_rows_remote=43058 dup_rows=15238 dup_rows_remote=11642 dedup_ratio=0.232513 router_tokens_per_expert=[0, 12, 0, 0, 1145, 0, 0, 101, 226, 0, 0, 0, 50, 0, 0, 788, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1058, 0, 0, 0, 130, 2312, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 0, 865, 0, 2636, 0, 0, 0, 29, 1710, 0, 0, 0, 242, 0, 0, 758, 0, 0, 0, 3079, 5, 1206, 1646, 339, 0, 69, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3015, 86, 0, 0, 0, 0, 77, 587, 0, 3032, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 620, 0] payload_rows_per_dst=[5844, 6576, 7240, 6576, 6290, 4750, 6586, 6436] dup_rows_per_dst=[2220, 2018, 3596, 3130, 1508, 936, 978, 852] combine_rows_per_src=[5844, 6576, 7240, 6576, 6290, 4750, 6586, 6436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49224 payload_rows_remote=42962 dup_rows=16312 dup_rows_remote=13496 dedup_ratio=0.248901 router_tokens_per_expert=[0, 332, 0, 0, 927, 0, 0, 187, 561, 0, 0, 0, 122, 0, 0, 855, 0, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 818, 0, 0, 0, 160, 2990, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 1009, 0, 1246, 0, 0, 0, 6, 1555, 0, 0, 0, 225, 0, 0, 1079, 0, 0, 0, 2935, 3, 1350, 548, 1514, 0, 223, 266, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 434, 0, 0, 0, 0, 3, 1099, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0] payload_rows_per_dst=[6262, 6492, 7804, 4092, 4636, 5902, 7180, 6856] dup_rows_per_dst=[2816, 2118, 4956, 920, 844, 1260, 2082, 1316] combine_rows_per_src=[6262, 6492, 7804, 4092, 4636, 5902, 7180, 6856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46490 payload_rows_remote=39592 dup_rows=19046 dup_rows_remote=15310 dedup_ratio=0.290619 router_tokens_per_expert=[0, 224, 1, 0, 507, 3, 2, 53, 612, 0, 2, 0, 289, 1, 2, 1017, 0, 0, 1, 0, 0, 0, 0, 4, 148, 0, 2, 0, 0, 0, 0, 0, 1, 0, 395, 0, 1660, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 39, 1656, 0, 2, 0, 45, 2221, 1355, 0, 0, 3, 0, 0, 0, 0, 1, 4, 181, 1, 1351, 1, 749, 2, 0, 0, 63, 2745, 0, 0, 1, 108, 0, 0, 849, 0, 0, 0, 3029, 0, 2123, 830, 2164, 0, 138, 666, 0, 0, 0, 0, 1, 0, 0, 0, 2209, 877, 0, 0, 0, 3, 41, 1275, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0] payload_rows_per_dst=[4456, 6140, 3936, 5238, 7416, 6454, 5952, 6898] dup_rows_per_dst=[970, 228, 182, 2220, 6182, 2758, 2770, 3736] combine_rows_per_src=[4456, 6140, 3936, 5238, 7416, 6454, 5952, 6898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=185483264 dispatch_meta_bytes=217464 combine_payload_bytes=185483264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=185700728 ingress_bytes=242106368 link_bytes=242106368 token_bytes=371183992 weight_movement_bytes=56623104 total_nvlink_bytes=427807096 weight_to_token_ratio=0.152547 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=11008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.319744094e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46984 payload_rows_remote=39888 dup_rows=18552 dup_rows_remote=15100 dedup_ratio=0.283081 router_tokens_per_expert=[0, 332, 0, 1, 713, 0, 0, 165, 480, 0, 0, 0, 476, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1280, 0, 0, 0, 38, 2238, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 0, 875, 0, 379, 0, 0, 1, 816, 2321, 1, 0, 0, 45, 0, 0, 888, 0, 0, 0, 2780, 4, 2293, 204, 2759, 0, 116, 1210, 0, 0, 1, 0, 0, 0, 0, 0, 2081, 2287, 0, 1, 5, 0, 20, 1486, 0, 1524, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 533, 0] payload_rows_per_dst=[6868, 7096, 6096, 3368, 4700, 7136, 7002, 4718] dup_rows_per_dst=[3284, 3452, 2226, 696, 1508, 3228, 3644, 514] combine_rows_per_src=[6868, 7096, 6096, 3368, 4700, 7136, 7002, 4718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=181182464 dispatch_meta_bytes=235960 combine_payload_bytes=181182464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=181418424 ingress_bytes=218931200 link_bytes=218931200 token_bytes=362600888 weight_movement_bytes=37748736 total_nvlink_bytes=400349624 weight_to_token_ratio=0.104105 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.076389 gemm_active_groups=15 gemm_peak_group_rows=10752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.097691038e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174317568 dispatch_meta_bytes=225848 combine_payload_bytes=174317568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=174543416 ingress_bytes=249815040 link_bytes=249815040 token_bytes=348860984 weight_movement_bytes=75497472 total_nvlink_bytes=424358456 weight_to_token_ratio=0.216411 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.090278 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393604797e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=176365568 dispatch_meta_bytes=218800 combine_payload_bytes=176365568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176584368 ingress_bytes=195239936 link_bytes=195239936 token_bytes=352949936 weight_movement_bytes=18874368 total_nvlink_bytes=371824304 weight_to_token_ratio=0.053476 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870693003e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=175972352 dispatch_meta_bytes=225832 combine_payload_bytes=175972352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=176198184 ingress_bytes=194846720 link_bytes=194846720 token_bytes=352170536 weight_movement_bytes=18874368 total_nvlink_bytes=371044904 weight_to_token_ratio=0.053594 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.072917 gemm_active_groups=16 gemm_peak_group_rows=10624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=162168832 dispatch_meta_bytes=219608 combine_payload_bytes=162168832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=162388440 ingress_bytes=199917568 link_bytes=199917568 token_bytes=324557272 weight_movement_bytes=37748736 total_nvlink_bytes=362306008 weight_to_token_ratio=0.116308 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.915511771e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=163381248 dispatch_meta_bytes=219952 combine_payload_bytes=163381248 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=163601200 ingress_bytes=201129984 link_bytes=201129984 token_bytes=326982448 weight_movement_bytes=37748736 total_nvlink_bytes=364731184 weight_to_token_ratio=0.115446 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.083333 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.927128545e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38644 dup_rows=21982 dup_rows_remote=19570 dedup_ratio=0.335419 router_tokens_per_expert=[221, 0, 0, 2, 212, 193, 95, 8, 0, 1, 411, 409, 502, 388, 216, 319, 14, 268, 2, 496, 261, 375, 0, 1, 486, 124, 83, 172, 681, 4, 508, 40, 311, 353, 2, 166, 7, 85, 183, 218, 388, 713, 498, 418, 207, 32, 222, 356, 556, 452, 419, 0, 353, 506, 29, 40, 673, 152, 389, 873, 307, 317, 36, 234, 26, 365, 1, 46, 409, 0, 490, 444, 6, 836, 388, 677, 155, 132, 33, 895, 185, 25, 127, 263, 47, 293, 3, 2, 581, 412, 74, 203, 324, 298, 157, 250, 0, 123, 157, 122, 0, 349, 608, 51, 191, 21, 602, 399, 410, 275, 353, 0, 423, 510, 378, 161, 412, 1, 654, 118, 785, 163, 567, 35, 423, 15, 328, 0] payload_rows_per_dst=[5290, 5804, 4616, 5388, 6112, 5178, 4910, 6256] dup_rows_per_dst=[2410, 2652, 2276, 2192, 3686, 2664, 2412, 3690] combine_rows_per_src=[5290, 5804, 4616, 5388, 6112, 5178, 4910, 6256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50746 payload_rows_remote=42602 dup_rows=14790 dup_rows_remote=14258 dedup_ratio=0.225677 router_tokens_per_expert=[0, 1, 0, 417, 3648, 0, 4069, 0, 0, 0, 774, 735, 0, 0, 0, 315, 0, 1, 0, 0, 0, 0, 315, 10, 0, 0, 118, 0, 0, 0, 0, 3519, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 2, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 1, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1674, 0, 0, 240, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 2, 0, 0, 212, 14, 0, 0, 4, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 0, 0, 0, 0, 3894, 0, 0] payload_rows_per_dst=[4462, 6558, 2156, 8144, 8010, 7534, 8032, 5850] dup_rows_per_dst=[1478, 562, 322, 532, 2560, 2772, 5280, 1284] combine_rows_per_src=[4462, 6558, 2156, 8144, 8010, 7534, 8032, 5850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49122 payload_rows_remote=46026 dup_rows=16414 dup_rows_remote=16414 dedup_ratio=0.250458 router_tokens_per_expert=[0, 0, 0, 43, 4048, 3, 3573, 0, 0, 0, 598, 1033, 0, 0, 2, 10, 0, 2, 0, 0, 0, 0, 1728, 5, 3, 0, 470, 0, 0, 0, 0, 3927, 0, 1, 0, 0, 0, 0, 0, 2463, 3, 0, 0, 0, 2, 481, 0, 3, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 158, 2, 0, 0, 0, 0, 818, 0, 0, 301, 0, 0, 0, 0, 3840, 0, 6, 0, 0, 0, 0, 1, 0, 995, 114, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 0, 0, 0, 0, 3177, 0, 0] payload_rows_per_dst=[3144, 3968, 8130, 7644, 3096, 7860, 8134, 7146] dup_rows_per_dst=[234, 448, 5630, 2352, 0, 2378, 2946, 2426] combine_rows_per_src=[3144, 3968, 8130, 7644, 3096, 7860, 8134, 7146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50418 payload_rows_remote=43244 dup_rows=15118 dup_rows_remote=12636 dedup_ratio=0.230682 router_tokens_per_expert=[0, 0, 0, 693, 3659, 0, 4005, 0, 0, 0, 522, 813, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 1450, 18, 0, 0, 905, 0, 0, 0, 0, 3102, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 732, 0, 0, 481, 0, 0, 0, 0, 3118, 0, 13, 0, 0, 0, 0, 0, 0, 187, 138, 0, 0, 0, 0, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 0, 0, 0, 0, 3420, 0, 0] payload_rows_per_dst=[3480, 4156, 7214, 8034, 4772, 7174, 7804, 7784] dup_rows_per_dst=[646, 590, 3804, 814, 0, 2482, 2496, 4286] combine_rows_per_src=[3480, 4156, 7214, 8034, 4772, 7174, 7804, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=48524 payload_rows_remote=45084 dup_rows=17012 dup_rows_remote=16968 dedup_ratio=0.259583 router_tokens_per_expert=[0, 0, 0, 549, 3418, 0, 4021, 0, 0, 0, 924, 612, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 825, 92, 0, 0, 400, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 3722, 0, 0, 0, 0, 10, 0, 136, 0, 0, 0, 0, 0, 1364, 0, 0, 785, 0, 0, 0, 0, 2282, 0, 10, 0, 0, 0, 0, 0, 0, 118, 104, 0, 0, 0, 0, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1708, 0, 0, 0, 0, 3869, 0, 0] payload_rows_per_dst=[7758, 7604, 3440, 8042, 8158, 6128, 3978, 3416] dup_rows_per_dst=[3382, 2736, 44, 1428, 7078, 2214, 128, 2] combine_rows_per_src=[7758, 7604, 3440, 8042, 8158, 6128, 3978, 3416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=40234 payload_rows_remote=32082 dup_rows=25302 dup_rows_remote=9244 dedup_ratio=0.386078 router_tokens_per_expert=[0, 0, 0, 328, 3519, 0, 4090, 0, 0, 0, 423, 1742, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 755, 10, 0, 0, 186, 0, 0, 0, 0, 3471, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 655, 0, 0, 400, 0, 0, 0, 0, 3566, 0, 5, 0, 0, 0, 0, 0, 0, 295, 9, 0, 0, 0, 0, 0, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 0, 0, 0, 0, 3501, 0, 0] payload_rows_per_dst=[8152, 7460, 4890, 2474, 8102, 2014, 5134, 2008] dup_rows_per_dst=[16058, 2772, 40, 298, 5856, 124, 154, 0] combine_rows_per_src=[8152, 7460, 4890, 2474, 8102, 2014, 5134, 2008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=45626 payload_rows_remote=37552 dup_rows=19910 dup_rows_remote=11474 dedup_ratio=0.303802 router_tokens_per_expert=[0, 0, 0, 135, 3861, 4, 3887, 0, 0, 0, 531, 160, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 1205, 144, 4, 0, 358, 0, 0, 2, 0, 3537, 0, 2, 0, 0, 0, 0, 0, 1772, 4, 0, 0, 0, 4, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 42, 4, 0, 0, 0, 0, 1947, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 0, 0, 1793, 0, 0, 915, 0, 0, 0, 0, 3313, 0, 19, 0, 0, 0, 0, 0, 0, 451, 197, 0, 0, 8, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2478, 0, 0, 0, 0, 3543, 0, 0] payload_rows_per_dst=[1588, 3168, 7596, 7842, 3986, 5518, 7854, 8074] dup_rows_per_dst=[80, 260, 3296, 2728, 0, 3498, 1612, 8436] combine_rows_per_src=[1588, 3168, 7596, 7842, 3986, 5518, 7854, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=174497792 dispatch_meta_bytes=227440 combine_payload_bytes=174497792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=174725232 ingress_bytes=193372160 link_bytes=193372160 token_bytes=349223024 weight_movement_bytes=18874368 total_nvlink_bytes=368097392 weight_to_token_ratio=0.054047 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852796892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=188522496 dispatch_meta_bytes=249760 combine_payload_bytes=188522496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=188772256 ingress_bytes=207396864 link_bytes=207396864 token_bytes=377294752 weight_movement_bytes=18874368 total_nvlink_bytes=396169120 weight_to_token_ratio=0.050026 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.987174705e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46864 payload_rows_remote=38678 dup_rows=18672 dup_rows_remote=8152 dedup_ratio=0.284912 router_tokens_per_expert=[0, 0, 0, 103, 3794, 0, 3000, 0, 0, 0, 573, 517, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1552, 278, 0, 0, 313, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 0, 0, 0, 0, 545, 1, 0, 1, 0, 0, 0, 0, 0, 947, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 5, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 1475, 0, 0, 784, 0, 0, 0, 0, 3286, 0, 12, 0, 0, 0, 0, 0, 0, 316, 778, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 1, 10, 0, 2608, 0, 0, 0, 0, 3353, 0, 0] payload_rows_per_dst=[7728, 8186, 5396, 6502, 7264, 4126, 2446, 5216] dup_rows_per_dst=[2252, 10520, 620, 1412, 2358, 1410, 78, 22] combine_rows_per_src=[7728, 8186, 5396, 6502, 7264, 4126, 2446, 5216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158285824 dispatch_meta_bytes=232856 combine_payload_bytes=158285824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158518680 ingress_bytes=177160192 link_bytes=177160192 token_bytes=316804504 weight_movement_bytes=18874368 total_nvlink_bytes=335678872 weight_to_token_ratio=0.059577 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=5248 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.697461791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=177127424 dispatch_meta_bytes=223520 combine_payload_bytes=177127424 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=177350944 ingress_bytes=233750528 link_bytes=233750528 token_bytes=354478368 weight_movement_bytes=56623104 total_nvlink_bytes=411101472 weight_to_token_ratio=0.159736 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239682547e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=184664064 dispatch_meta_bytes=248208 combine_payload_bytes=184664064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184912272 ingress_bytes=203538432 link_bytes=203538432 token_bytes=369576336 weight_movement_bytes=18874368 total_nvlink_bytes=388450704 weight_to_token_ratio=0.051070 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.950205108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=131407872 dispatch_meta_bytes=165304 combine_payload_bytes=131407872 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=131573176 ingress_bytes=188030976 link_bytes=188030976 token_bytes=262981048 weight_movement_bytes=56623104 total_nvlink_bytes=319604152 weight_to_token_ratio=0.215312 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.086806 gemm_active_groups=11 gemm_peak_group_rows=11520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.801620295e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=153812992 dispatch_meta_bytes=196104 combine_payload_bytes=153812992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=154009096 ingress_bytes=191561728 link_bytes=191561728 token_bytes=307822088 weight_movement_bytes=37748736 total_nvlink_bytes=345570824 weight_to_token_ratio=0.122632 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=15744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.835450223e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158425088 dispatch_meta_bytes=187320 combine_payload_bytes=158425088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158612408 ingress_bytes=196173824 link_bytes=196173824 token_bytes=317037496 weight_movement_bytes=37748736 total_nvlink_bytes=354786232 weight_to_token_ratio=0.119067 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.086806 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.879641058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54642 payload_rows_remote=47542 dup_rows=10894 dup_rows_remote=10870 dedup_ratio=0.166229 router_tokens_per_expert=[0, 0, 0, 0, 0, 1309, 0, 3249, 0, 0, 0, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 7, 0, 0, 0, 3550, 0, 0, 0, 0, 1783, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1682, 0, 4, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 26, 3397, 0, 3812] payload_rows_per_dst=[7378, 5788, 7810, 7100, 8182, 8064, 3370, 6950] dup_rows_per_dst=[1738, 2, 1810, 24, 3402, 3294, 60, 564] combine_rows_per_src=[7378, 5788, 7810, 7100, 8182, 8064, 3370, 6950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56878 payload_rows_remote=49024 dup_rows=8658 dup_rows_remote=7764 dedup_ratio=0.132111 router_tokens_per_expert=[0, 0, 0, 0, 0, 3203, 0, 1465, 0, 0, 3, 0, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3090, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2373, 0, 4024, 0, 0, 0, 0, 0, 3, 3, 841, 0, 0, 0, 0, 2349, 3, 0, 0, 0, 0, 3, 449, 0, 0, 0, 0, 0, 0, 0, 64, 3150, 0, 3713] payload_rows_per_dst=[7224, 6300, 7432, 8176, 7854, 7592, 5726, 6574] dup_rows_per_dst=[2118, 526, 24, 20, 894, 3334, 984, 758] combine_rows_per_src=[7224, 6300, 7432, 8176, 7854, 7592, 5726, 6574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57852 payload_rows_remote=50270 dup_rows=7684 dup_rows_remote=7088 dedup_ratio=0.117249 router_tokens_per_expert=[0, 0, 0, 0, 0, 3565, 0, 1504, 0, 0, 0, 0, 3473, 0, 0, 0, 0, 5, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 4, 0, 0, 0, 3942, 0, 0, 0, 0, 437, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 1380, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 84, 3554, 0, 4092] payload_rows_per_dst=[7738, 7154, 8186, 7886, 3864, 7582, 8192, 7250] dup_rows_per_dst=[2400, 240, 420, 0, 0, 596, 3404, 624] combine_rows_per_src=[7738, 7154, 8186, 7886, 3864, 7582, 8192, 7250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58998 payload_rows_remote=50916 dup_rows=6538 dup_rows_remote=6080 dedup_ratio=0.099762 router_tokens_per_expert=[0, 0, 0, 0, 0, 1415, 0, 2915, 0, 0, 0, 0, 3282, 0, 0, 0, 0, 1, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 0, 374, 0, 4095, 0, 0, 0, 0, 0, 15, 0, 9, 0, 0, 0, 0, 2943, 0, 0, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 37, 3622, 0, 4037] payload_rows_per_dst=[7114, 6616, 8082, 7738, 8192, 8022, 5906, 7328] dup_rows_per_dst=[1546, 36, 458, 12, 3192, 674, 28, 592] combine_rows_per_src=[7114, 6616, 8082, 7738, 8192, 8022, 5906, 7328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53102 payload_rows_remote=45092 dup_rows=12434 dup_rows_remote=4756 dedup_ratio=0.189728 router_tokens_per_expert=[0, 0, 0, 0, 0, 2989, 0, 1523, 0, 0, 0, 0, 3503, 0, 0, 0, 0, 2, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 1743, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 2104, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 39, 3229, 0, 3811] payload_rows_per_dst=[8010, 7106, 7634, 7772, 8192, 3486, 4220, 6682] dup_rows_per_dst=[7678, 32, 576, 1178, 2246, 0, 20, 704] combine_rows_per_src=[8010, 7106, 7634, 7772, 8192, 3486, 4220, 6682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51610 payload_rows_remote=44110 dup_rows=13926 dup_rows_remote=8574 dedup_ratio=0.212494 router_tokens_per_expert=[0, 0, 0, 0, 0, 3027, 0, 1238, 0, 0, 4, 0, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 529, 0, 4037, 0, 0, 0, 0, 0, 14, 4, 426, 0, 0, 0, 0, 2659, 4, 0, 0, 0, 0, 4, 233, 0, 0, 0, 0, 0, 0, 0, 719, 3054, 0, 4063] payload_rows_per_dst=[2484, 7708, 6728, 8180, 3708, 7110, 8192, 7500] dup_rows_per_dst=[0, 4188, 18, 14, 0, 738, 3616, 5352] combine_rows_per_src=[2484, 7708, 6728, 8180, 3708, 7110, 8192, 7500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=194732032 dispatch_meta_bytes=233648 combine_payload_bytes=194732032 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=194965680 ingress_bytes=232480768 link_bytes=232480768 token_bytes=389697712 weight_movement_bytes=37748736 total_nvlink_bytes=427446448 weight_to_token_ratio=0.096867 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227516331e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49320 payload_rows_remote=43464 dup_rows=16216 dup_rows_remote=13986 dedup_ratio=0.247437 router_tokens_per_expert=[0, 487, 0, 0, 759, 0, 0, 67, 796, 0, 0, 0, 249, 0, 0, 653, 0, 0, 0, 0, 0, 0, 1, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1306, 0, 0, 0, 39, 2411, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 0, 620, 0, 648, 0, 0, 0, 290, 2574, 0, 0, 2, 156, 0, 0, 403, 0, 0, 0, 2945, 15, 2095, 165, 2159, 0, 4, 462, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 1811, 0, 0, 0, 0, 50, 1399, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0] payload_rows_per_dst=[4772, 5910, 7108, 6534, 6730, 6372, 5856, 6038] dup_rows_per_dst=[1250, 126, 1600, 2090, 3542, 2668, 2230, 2710] combine_rows_per_src=[4772, 5910, 7108, 6534, 6730, 6372, 5856, 6038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=200802304 dispatch_meta_bytes=227152 combine_payload_bytes=200802304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201029456 ingress_bytes=219676672 link_bytes=219676672 token_bytes=401831760 weight_movement_bytes=18874368 total_nvlink_bytes=420706128 weight_to_token_ratio=0.046971 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.104833783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56568 payload_rows_remote=50006 dup_rows=8968 dup_rows_remote=8396 dedup_ratio=0.136841 router_tokens_per_expert=[0, 0, 0, 0, 0, 3277, 0, 507, 0, 6, 0, 0, 2808, 0, 0, 0, 0, 4, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 585, 0, 3799, 0, 0, 0, 0, 0, 13, 0, 1282, 0, 0, 0, 0, 3299, 0, 0, 1, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 436, 2470, 0, 4082] payload_rows_per_dst=[6864, 6562, 8164, 8186, 8078, 5860, 7128, 5726] dup_rows_per_dst=[716, 572, 2, 1222, 2494, 966, 2060, 936] combine_rows_per_src=[6864, 6562, 8164, 8186, 8078, 5860, 7128, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=205905920 dispatch_meta_bytes=229432 combine_payload_bytes=205905920 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=206135352 ingress_bytes=281403392 link_bytes=281403392 token_bytes=412041272 weight_movement_bytes=75497472 total_nvlink_bytes=487538744 weight_to_token_ratio=0.183228 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.696268843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=208551936 dispatch_meta_bytes=227984 combine_payload_bytes=208551936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208779920 ingress_bytes=227426304 link_bytes=227426304 token_bytes=417331856 weight_movement_bytes=18874368 total_nvlink_bytes=436206224 weight_to_token_ratio=0.045226 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.179086944e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=184696832 dispatch_meta_bytes=199392 combine_payload_bytes=184696832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=184896224 ingress_bytes=241319936 link_bytes=241319936 token_bytes=369593056 weight_movement_bytes=56623104 total_nvlink_bytes=426216160 weight_to_token_ratio=0.153204 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.312208890e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=180674560 dispatch_meta_bytes=210736 combine_payload_bytes=180674560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=180885296 ingress_bytes=218423296 link_bytes=218423296 token_bytes=361559856 weight_movement_bytes=37748736 total_nvlink_bytes=399308592 weight_to_token_ratio=0.104405 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.097222 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.092824551e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=204824576 dispatch_meta_bytes=233608 combine_payload_bytes=204824576 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=205058184 ingress_bytes=242573312 link_bytes=242573312 token_bytes=409882760 weight_movement_bytes=37748736 total_nvlink_bytes=447631496 weight_to_token_ratio=0.092096 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.324218122e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=178028544 dispatch_meta_bytes=229800 combine_payload_bytes=178028544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=178258344 ingress_bytes=234651648 link_bytes=234651648 token_bytes=356286888 weight_movement_bytes=56623104 total_nvlink_bytes=412909992 weight_to_token_ratio=0.158926 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.079861 gemm_active_groups=12 gemm_peak_group_rows=12672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.248316635e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57722 payload_rows_remote=52594 dup_rows=7814 dup_rows_remote=7814 dedup_ratio=0.119232 router_tokens_per_expert=[4073, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 3389, 0, 0, 4029, 10, 0, 0, 0, 33, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 0, 0, 6, 0, 0, 2558, 0, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2, 4085, 0, 0, 0, 1, 0, 2, 0, 1, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 48, 459, 0] payload_rows_per_dst=[8146, 6980, 8104, 5128, 8192, 6060, 8186, 6926] dup_rows_per_dst=[432, 236, 40, 0, 6252, 162, 2, 690] combine_rows_per_src=[8146, 6980, 8104, 5128, 8192, 6060, 8186, 6926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59464 payload_rows_remote=51278 dup_rows=6072 dup_rows_remote=4188 dedup_ratio=0.092651 router_tokens_per_expert=[3838, 0, 0, 0, 3, 0, 0, 0, 322, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 0, 0, 0, 3, 0, 0, 861, 0, 0, 3640, 15, 0, 0, 0, 64, 4060, 0, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 0, 0, 0, 7, 0, 0, 3345, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 319, 0, 125, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3933, 512, 527, 0] payload_rows_per_dst=[7784, 7072, 7392, 6702, 8186, 6164, 8126, 8038] dup_rows_per_dst=[558, 994, 64, 2, 1884, 652, 6, 1912] combine_rows_per_src=[7784, 7072, 7392, 6702, 8186, 6164, 8126, 8038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57908 payload_rows_remote=49900 dup_rows=7628 dup_rows_remote=5926 dedup_ratio=0.116394 router_tokens_per_expert=[3975, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 0, 0, 3505, 15, 0, 0, 0, 114, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 2666, 0, 0, 1281, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3, 4076, 0, 0, 0, 0, 0, 7, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 210, 910, 0] payload_rows_per_dst=[7970, 6384, 8008, 5332, 8172, 6998, 8190, 6854] dup_rows_per_dst=[454, 1284, 1702, 0, 2736, 696, 0, 756] combine_rows_per_src=[7970, 6384, 8008, 5332, 8172, 6998, 8190, 6854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57720 payload_rows_remote=50302 dup_rows=7816 dup_rows_remote=5464 dedup_ratio=0.119263 router_tokens_per_expert=[3884, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 2860, 0, 0, 0, 0, 78, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 117, 0, 13, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 548, 1531, 0] payload_rows_per_dst=[7776, 5754, 5788, 6662, 8192, 7418, 8192, 7938] dup_rows_per_dst=[216, 312, 88, 0, 904, 2352, 0, 3944] combine_rows_per_src=[7776, 5754, 5788, 6662, 8192, 7418, 8192, 7938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49464 payload_rows_remote=41370 dup_rows=16072 dup_rows_remote=15658 dedup_ratio=0.245239 router_tokens_per_expert=[4042, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 0, 3922, 5, 0, 0, 0, 13, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 12, 0, 33, 4092, 0, 0, 0, 0, 0, 29, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 426, 538, 0] payload_rows_per_dst=[8094, 8154, 7852, 8192, 2682, 4518, 8184, 1788] dup_rows_per_dst=[414, 7256, 28, 8108, 10, 116, 0, 140] combine_rows_per_src=[8094, 8154, 7852, 8192, 2682, 4518, 8184, 1788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57454 payload_rows_remote=49276 dup_rows=8082 dup_rows_remote=4236 dedup_ratio=0.123322 router_tokens_per_expert=[3927, 0, 0, 0, 4, 0, 0, 0, 698, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 4, 0, 0, 384, 0, 0, 3253, 54, 0, 0, 0, 46, 4074, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 20, 0, 0, 3277, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 275, 0, 6, 4054, 0, 0, 0, 0, 0, 46, 0, 0, 0, 726, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 4054, 391, 530, 0] payload_rows_per_dst=[7798, 6488, 6514, 6594, 6394, 7332, 8156, 8178] dup_rows_per_dst=[1338, 332, 56, 0, 956, 1394, 160, 3846] combine_rows_per_src=[7798, 6488, 6514, 6594, 6394, 7332, 8156, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=215425024 dispatch_meta_bytes=241632 combine_payload_bytes=215425024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=215666656 ingress_bytes=253173760 link_bytes=253173760 token_bytes=431091680 weight_movement_bytes=37748736 total_nvlink_bytes=468840416 weight_to_token_ratio=0.087565 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425786399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=210034688 dispatch_meta_bytes=221864 combine_payload_bytes=210034688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=210256552 ingress_bytes=266657792 link_bytes=266657792 token_bytes=420291240 weight_movement_bytes=56623104 total_nvlink_bytes=476914344 weight_to_token_ratio=0.134723 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.104167 gemm_active_groups=6 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.554983759e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49422 payload_rows_remote=42774 dup_rows=16114 dup_rows_remote=14856 dedup_ratio=0.245880 router_tokens_per_expert=[3488, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 2765, 172, 0, 0, 0, 370, 4071, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 0, 18, 0, 0, 2549, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 834, 0, 35, 3993, 0, 0, 0, 1, 0, 10, 0, 0, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 327, 735, 0] payload_rows_per_dst=[6996, 6648, 7982, 8020, 2084, 7588, 8142, 1962] dup_rows_per_dst=[456, 1258, 5806, 5014, 10, 3408, 0, 162] combine_rows_per_src=[6996, 6648, 7982, 8020, 2084, 7588, 8142, 1962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=204390400 dispatch_meta_bytes=223304 combine_payload_bytes=204390400 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=204613704 ingress_bytes=261013504 link_bytes=261013504 token_bytes=409004104 weight_movement_bytes=56623104 total_nvlink_bytes=465627208 weight_to_token_ratio=0.138441 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.500902969e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=206036992 dispatch_meta_bytes=223064 combine_payload_bytes=206036992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206260056 ingress_bytes=224911360 link_bytes=224911360 token_bytes=412297048 weight_movement_bytes=18874368 total_nvlink_bytes=431171416 weight_to_token_ratio=0.045779 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=23936 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.154989988e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50508 payload_rows_remote=42792 dup_rows=15028 dup_rows_remote=13284 dedup_ratio=0.229309 router_tokens_per_expert=[0, 0, 0, 747, 3583, 0, 3143, 0, 0, 0, 592, 1187, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1723, 114, 0, 0, 1436, 0, 1, 0, 0, 3226, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 1, 2, 0, 804, 0, 0, 0, 0, 0, 632, 0, 0, 616, 0, 0, 0, 0, 3100, 0, 66, 0, 0, 0, 0, 0, 0, 103, 255, 0, 0, 1, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 0, 0, 0, 0, 2067, 0, 0] payload_rows_per_dst=[4332, 5322, 7746, 6540, 4290, 7416, 7716, 7146] dup_rows_per_dst=[762, 1226, 5212, 750, 6, 2888, 1744, 2440] combine_rows_per_src=[4332, 5322, 7746, 6540, 4290, 7416, 7716, 7146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=169451520 dispatch_meta_bytes=228112 combine_payload_bytes=169451520 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=169679632 ingress_bytes=188325888 link_bytes=188325888 token_bytes=339131152 weight_movement_bytes=18874368 total_nvlink_bytes=358005520 weight_to_token_ratio=0.055655 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.804445997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=201834496 dispatch_meta_bytes=214048 combine_payload_bytes=201834496 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=202048544 ingress_bytes=258457600 link_bytes=258457600 token_bytes=403883040 weight_movement_bytes=56623104 total_nvlink_bytes=460506144 weight_to_token_ratio=0.140197 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=21760 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.476413554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=175202304 dispatch_meta_bytes=230520 combine_payload_bytes=175202304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175432824 ingress_bytes=194076672 link_bytes=194076672 token_bytes=350635128 weight_movement_bytes=18874368 total_nvlink_bytes=369509496 weight_to_token_ratio=0.053829 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859547179e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175276032 dispatch_meta_bytes=224304 combine_payload_bytes=175276032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=175500336 ingress_bytes=194150400 link_bytes=194150400 token_bytes=350776368 weight_movement_bytes=18874368 total_nvlink_bytes=369650736 weight_to_token_ratio=0.053807 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.090278 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57850 payload_rows_remote=50298 dup_rows=7686 dup_rows_remote=7686 dedup_ratio=0.117279 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 86, 0, 3, 0, 2735, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3776, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4, 0, 0, 0, 3740, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 1, 29, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0] payload_rows_per_dst=[6000, 8182, 8192, 7552, 8174, 7614, 8090, 4046] dup_rows_per_dst=[632, 6404, 280, 0, 12, 258, 58, 42] combine_rows_per_src=[6000, 8182, 8192, 7552, 8174, 7614, 8090, 4046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61986 payload_rows_remote=53860 dup_rows=3550 dup_rows_remote=3104 dedup_ratio=0.054169 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 184, 0, 2, 0, 3861, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 0, 3, 0, 67, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 1, 0, 3, 0, 4093, 3, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3137, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0] payload_rows_per_dst=[7872, 7878, 8192, 8038, 8126, 8192, 6280, 7408] dup_rows_per_dst=[444, 2092, 172, 0, 446, 6, 0, 390] combine_rows_per_src=[7872, 7878, 8192, 8038, 8126, 8192, 6280, 7408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60080 payload_rows_remote=51888 dup_rows=5456 dup_rows_remote=4878 dedup_ratio=0.083252 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 3628, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0] payload_rows_per_dst=[7500, 8190, 8192, 8066, 8094, 8178, 7632, 4228] dup_rows_per_dst=[968, 3890, 578, 0, 2, 4, 0, 14] combine_rows_per_src=[7500, 8190, 8192, 8066, 8094, 8178, 7632, 4228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57328 payload_rows_remote=49160 dup_rows=8208 dup_rows_remote=8180 dedup_ratio=0.125244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 3664, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 65, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0] payload_rows_per_dst=[7700, 2038, 8192, 8060, 8042, 8168, 6936, 8192] dup_rows_per_dst=[382, 8, 132, 0, 24, 28, 128, 7506] combine_rows_per_src=[7700, 2038, 8192, 8060, 8042, 8168, 6936, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55140 payload_rows_remote=48986 dup_rows=10396 dup_rows_remote=10294 dedup_ratio=0.158630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2905, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 16, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0] payload_rows_per_dst=[6154, 8192, 8192, 7952, 8184, 8112, 8168, 186] dup_rows_per_dst=[102, 3814, 106, 0, 0, 26, 6348, 0] combine_rows_per_src=[6154, 8192, 8192, 7952, 8184, 8112, 8168, 186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=206020608 dispatch_meta_bytes=231936 combine_payload_bytes=206020608 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=206252544 ingress_bytes=262643712 link_bytes=262643712 token_bytes=412273152 weight_movement_bytes=56623104 total_nvlink_bytes=468896256 weight_to_token_ratio=0.137344 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.516522820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54452 payload_rows_remote=46268 dup_rows=11084 dup_rows_remote=3874 dedup_ratio=0.169128 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 557, 0, 0, 0, 3737, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 4, 0, 204, 0, 0, 0, 0, 2, 4092, 0, 3, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 1, 0, 2, 0, 4092, 4, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3040, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[6044, 3284, 6390, 8182, 8088, 8192, 6088, 8184] dup_rows_per_dst=[1396, 0, 334, 1404, 362, 376, 2, 7210] combine_rows_per_src=[6044, 3284, 6390, 8182, 8088, 8192, 6088, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=220610560 dispatch_meta_bytes=227856 combine_payload_bytes=220610560 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=220838416 ingress_bytes=277233664 link_bytes=277233664 token_bytes=441448976 weight_movement_bytes=56623104 total_nvlink_bytes=498072080 weight_to_token_ratio=0.128266 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.656316561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58362 payload_rows_remote=50188 dup_rows=7174 dup_rows_remote=5940 dedup_ratio=0.109467 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1066, 0, 0, 0, 3913, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 4087, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7936, 8174, 8190, 8150, 7836, 8192, 7962, 1922] dup_rows_per_dst=[2400, 1234, 44, 0, 152, 0, 3344, 0] combine_rows_per_src=[7936, 8174, 8190, 8150, 7836, 8192, 7962, 1922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=212533248 dispatch_meta_bytes=227064 combine_payload_bytes=212533248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212760312 ingress_bytes=231407616 link_bytes=231407616 token_bytes=425293560 weight_movement_bytes=18874368 total_nvlink_bytes=444167928 weight_to_token_ratio=0.044380 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217233917e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=201359360 dispatch_meta_bytes=229360 combine_payload_bytes=201359360 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=201588720 ingress_bytes=239108096 link_bytes=239108096 token_bytes=402948080 weight_movement_bytes=37748736 total_nvlink_bytes=440696816 weight_to_token_ratio=0.093681 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.291016127e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54868 payload_rows_remote=46676 dup_rows=10668 dup_rows_remote=6230 dedup_ratio=0.162781 router_tokens_per_expert=[0, 0, 0, 0, 0, 3467, 0, 547, 0, 0, 0, 0, 3474, 0, 0, 0, 0, 21, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 316, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 397, 0, 0, 0, 0, 1822, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 41, 3743, 0, 4095] payload_rows_per_dst=[2254, 7790, 8190, 8024, 5238, 7684, 8192, 7496] dup_rows_per_dst=[274, 4916, 2, 4, 0, 534, 4438, 500] combine_rows_per_src=[2254, 7790, 8190, 8024, 5238, 7684, 8192, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200646656 dispatch_meta_bytes=237120 combine_payload_bytes=200646656 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=200883776 ingress_bytes=238395392 link_bytes=238395392 token_bytes=401530432 weight_movement_bytes=37748736 total_nvlink_bytes=439279168 weight_to_token_ratio=0.094012 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284187348e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=189513728 dispatch_meta_bytes=200568 combine_payload_bytes=189513728 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=189714296 ingress_bytes=246136832 link_bytes=246136832 token_bytes=379228024 weight_movement_bytes=56623104 total_nvlink_bytes=435851128 weight_to_token_ratio=0.149311 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=19200 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358362017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205570048 dispatch_meta_bytes=224512 combine_payload_bytes=205570048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=205794560 ingress_bytes=224444416 link_bytes=224444416 token_bytes=411364608 weight_movement_bytes=18874368 total_nvlink_bytes=430238976 weight_to_token_ratio=0.045882 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.150515961e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=191184896 dispatch_meta_bytes=211624 combine_payload_bytes=191184896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=191396520 ingress_bytes=247808000 link_bytes=247808000 token_bytes=382581416 weight_movement_bytes=56623104 total_nvlink_bytes=439204520 weight_to_token_ratio=0.148003 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.093750 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.374374327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58376 payload_rows_remote=50664 dup_rows=7160 dup_rows_remote=5662 dedup_ratio=0.109253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1497, 4091, 0, 0, 0, 0, 0, 0, 0, 72, 4094, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3533, 0, 0, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 1, 0, 4091, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 4066] payload_rows_per_dst=[8182, 3026, 8188, 7712, 8178, 8188, 8192, 6710] dup_rows_per_dst=[0, 0, 158, 1498, 374, 2, 5062, 66] combine_rows_per_src=[8182, 3026, 8188, 7712, 8178, 8188, 8192, 6710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62538 payload_rows_remote=54412 dup_rows=2998 dup_rows_remote=1030 dedup_ratio=0.045746 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3784, 4090, 0, 0, 0, 0, 1, 0, 0, 8, 4081, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3742, 0, 3, 0, 463, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1209, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 3, 0, 0, 4093, 0, 0, 169, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 3, 0, 3167, 0, 3835] payload_rows_per_dst=[8180, 7574, 8168, 7760, 8126, 8180, 8192, 6358] dup_rows_per_dst=[0, 2, 16, 664, 1968, 6, 338, 4] combine_rows_per_src=[8180, 7574, 8168, 7760, 8126, 8180, 8192, 6358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61132 payload_rows_remote=52954 dup_rows=4404 dup_rows_remote=4388 dedup_ratio=0.067200 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2888, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 1, 0, 4095, 0, 0, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 4074] payload_rows_per_dst=[8192, 5786, 8178, 7442, 8186, 8192, 8192, 6964] dup_rows_per_dst=[2, 2, 16, 430, 1360, 8, 2576, 10] combine_rows_per_src=[8192, 5786, 8178, 7442, 8186, 8192, 8192, 6964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62646 payload_rows_remote=54456 dup_rows=2890 dup_rows_remote=2890 dedup_ratio=0.044098 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 3945, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1087, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4073, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 4054] payload_rows_per_dst=[8192, 7978, 8178, 7258, 8174, 8190, 8182, 6494] dup_rows_per_dst=[0, 12, 54, 388, 2134, 0, 292, 10] combine_rows_per_src=[8192, 7978, 8178, 7258, 8174, 8190, 8182, 6494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55200 payload_rows_remote=47008 dup_rows=10336 dup_rows_remote=10284 dedup_ratio=0.157715 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 3631, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 4086, 0, 0, 3, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 0, 0, 501, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4077, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 4054] payload_rows_per_dst=[8192, 8162, 8174, 7864, 8132, 8192, 1000, 5484] dup_rows_per_dst=[52, 7250, 68, 1642, 1322, 2, 0, 0] combine_rows_per_src=[8192, 8162, 8174, 7864, 8132, 8192, 1000, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=207519744 dispatch_meta_bytes=225304 combine_payload_bytes=207519744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=207745048 ingress_bytes=226394112 link_bytes=226394112 token_bytes=415264792 weight_movement_bytes=18874368 total_nvlink_bytes=434139160 weight_to_token_ratio=0.045451 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.086806 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.169196988e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=222871552 dispatch_meta_bytes=221768 combine_payload_bytes=222871552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223093320 ingress_bytes=241745920 link_bytes=241745920 token_bytes=445964872 weight_movement_bytes=18874368 total_nvlink_bytes=464839240 weight_to_token_ratio=0.042323 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316290459e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56624 payload_rows_remote=48434 dup_rows=8912 dup_rows_remote=908 dedup_ratio=0.135986 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3818, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 4, 0, 153, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 540, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 5, 0, 0, 4092, 0, 0, 376, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3974, 0, 3900] payload_rows_per_dst=[8178, 7644, 8028, 7484, 1088, 8190, 7822, 8190] dup_rows_per_dst=[0, 0, 0, 118, 24, 14, 752, 8004] combine_rows_per_src=[8178, 7644, 8028, 7484, 1088, 8190, 7822, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216899584 dispatch_meta_bytes=229368 combine_payload_bytes=216899584 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=217128952 ingress_bytes=254648320 link_bytes=254648320 token_bytes=434028536 weight_movement_bytes=37748736 total_nvlink_bytes=471777272 weight_to_token_ratio=0.086973 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.439914907e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61632 payload_rows_remote=53728 dup_rows=3904 dup_rows_remote=1882 dedup_ratio=0.059570 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 102, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 9, 0, 0, 4096, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 3365] payload_rows_per_dst=[8174, 7904, 8136, 7554, 8138, 8178, 6196, 7352] dup_rows_per_dst=[0, 2022, 0, 94, 1322, 14, 452, 0] combine_rows_per_src=[8174, 7904, 8136, 7554, 8138, 8178, 6196, 7352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=223051776 dispatch_meta_bytes=229384 combine_payload_bytes=223051776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223281160 ingress_bytes=241926144 link_bytes=241926144 token_bytes=446332936 weight_movement_bytes=18874368 total_nvlink_bytes=465207304 weight_to_token_ratio=0.042288 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.318017277e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57090 payload_rows_remote=48898 dup_rows=8446 dup_rows_remote=8446 dedup_ratio=0.128876 router_tokens_per_expert=[3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 2917, 11, 0, 0, 0, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 332, 0, 19, 4055, 0, 0, 0, 0, 0, 41, 0, 0, 0, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 475, 1828, 0] payload_rows_per_dst=[7104, 6218, 5972, 5782, 8144, 7618, 8192, 8060] dup_rows_per_dst=[0, 166, 140, 0, 738, 3060, 0, 4342] combine_rows_per_src=[7104, 6218, 5972, 5782, 8144, 7618, 8192, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=192544768 dispatch_meta_bytes=229168 combine_payload_bytes=192544768 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=192773936 ingress_bytes=230293504 link_bytes=230293504 token_bytes=385318704 weight_movement_bytes=37748736 total_nvlink_bytes=423067440 weight_to_token_ratio=0.097968 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.206559044e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=198385664 dispatch_meta_bytes=197368 combine_payload_bytes=198385664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=198583032 ingress_bytes=255008768 link_bytes=255008768 token_bytes=396968696 weight_movement_bytes=56623104 total_nvlink_bytes=453591800 weight_to_token_ratio=0.142639 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.443368543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=220069888 dispatch_meta_bytes=222440 combine_payload_bytes=220069888 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=220292328 ingress_bytes=276692992 link_bytes=276692992 token_bytes=440362216 weight_movement_bytes=56623104 total_nvlink_bytes=496985320 weight_to_token_ratio=0.128583 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27648 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.651136108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=200286208 dispatch_meta_bytes=229376 combine_payload_bytes=200286208 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=200515584 ingress_bytes=238034944 link_bytes=238034944 token_bytes=400801792 weight_movement_bytes=37748736 total_nvlink_bytes=438550528 weight_to_token_ratio=0.094183 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.280733713e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59610 payload_rows_remote=51430 dup_rows=5926 dup_rows_remote=954 dedup_ratio=0.090424 router_tokens_per_expert=[3634, 0, 0, 0, 0, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3997, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 6, 57, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4086, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 333, 0, 0, 0, 1495, 0, 0, 0, 10, 0, 0, 281, 0, 0, 0, 0, 0] payload_rows_per_dst=[7268, 7994, 8166, 8180, 8192, 8166, 8192, 3452] dup_rows_per_dst=[0, 2, 116, 4972, 26, 4, 686, 120] combine_rows_per_src=[7268, 7994, 8166, 8180, 8192, 8166, 8192, 3452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62370 payload_rows_remote=54218 dup_rows=3166 dup_rows_remote=3032 dedup_ratio=0.048309 router_tokens_per_expert=[3546, 0, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4064, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 3, 0, 27, 16, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 13, 0, 0, 3, 0, 3, 0, 0, 4073, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 250, 0, 0, 0, 3671, 3, 0, 0, 2, 0, 0, 37, 0, 0, 0, 0, 0] payload_rows_per_dst=[7092, 8134, 8192, 8148, 8152, 7116, 8186, 7350] dup_rows_per_dst=[0, 6, 82, 2260, 134, 30, 578, 76] combine_rows_per_src=[7092, 8134, 8192, 8148, 8152, 7116, 8186, 7350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57946 payload_rows_remote=49758 dup_rows=7590 dup_rows_remote=7570 dedup_ratio=0.115814 router_tokens_per_expert=[3658, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 434, 0, 0, 0, 2629, 0, 0, 0, 1, 0, 0, 83, 0, 0, 0, 0, 0] payload_rows_per_dst=[8066, 7296, 8188, 4576, 8190, 8164, 8192, 5274] dup_rows_per_dst=[4780, 0, 20, 1664, 30, 12, 932, 152] combine_rows_per_src=[8066, 7296, 8188, 4576, 8190, 8164, 8192, 5274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62474 payload_rows_remote=54340 dup_rows=3062 dup_rows_remote=3060 dedup_ratio=0.046722 router_tokens_per_expert=[3936, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 63, 0, 0, 0, 20, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0] payload_rows_per_dst=[7038, 6874, 8192, 8192, 8192, 8134, 8192, 7660] dup_rows_per_dst=[0, 0, 370, 1390, 12, 2, 680, 608] combine_rows_per_src=[7038, 6874, 8192, 8192, 8192, 8134, 8192, 7660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50432 payload_rows_remote=42304 dup_rows=15104 dup_rows_remote=7356 dedup_ratio=0.230469 router_tokens_per_expert=[3948, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 3988, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 43, 0, 0, 0, 52, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 132, 0, 0, 0, 1949, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8168, 7270, 4470, 8192, 8176, 2044, 3984] dup_rows_per_dst=[7748, 7264, 28, 8, 8, 14, 34, 0] combine_rows_per_src=[8128, 8168, 7270, 4470, 8192, 8176, 2044, 3984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=210657280 dispatch_meta_bytes=209536 combine_payload_bytes=210657280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210866816 ingress_bytes=229531648 link_bytes=229531648 token_bytes=421524096 weight_movement_bytes=18874368 total_nvlink_bytes=440398464 weight_to_token_ratio=0.044776 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.199259315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=222076928 dispatch_meta_bytes=229000 combine_payload_bytes=222076928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222305928 ingress_bytes=240951296 link_bytes=240951296 token_bytes=444382856 weight_movement_bytes=18874368 total_nvlink_bytes=463257224 weight_to_token_ratio=0.042473 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.308676763e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52960 payload_rows_remote=44936 dup_rows=12576 dup_rows_remote=7194 dedup_ratio=0.191895 router_tokens_per_expert=[3040, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4045, 0, 4091, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 4063, 0, 0, 4, 0, 25, 6, 0, 0, 0, 0, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 0, 0, 4, 0, 2, 0, 0, 0, 308, 0, 0, 4, 0, 4, 0, 0, 4070, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 247, 0, 0, 0, 3370, 4, 0, 0, 7, 0, 0, 282, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 5264, 8176, 8174, 8060, 7078, 8184, 8024] dup_rows_per_dst=[0, 8, 24, 3458, 28, 3068, 608, 5382] combine_rows_per_src=[0, 5264, 8176, 8174, 8060, 7078, 8184, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=203808768 dispatch_meta_bytes=229312 combine_payload_bytes=203808768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204038080 ingress_bytes=241557504 link_bytes=241557504 token_bytes=407846848 weight_movement_bytes=37748736 total_nvlink_bytes=445595584 weight_to_token_ratio=0.092556 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.097222 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.314485150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56352 payload_rows_remote=48882 dup_rows=9184 dup_rows_remote=9184 dedup_ratio=0.140137 router_tokens_per_expert=[3604, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 165, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4054, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 420, 0, 0, 0, 3870, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7470, 8186, 1076, 8114, 7382, 8192, 7740] dup_rows_per_dst=[7222, 0, 320, 0, 14, 0, 1610, 18] combine_rows_per_src=[8192, 7470, 8186, 1076, 8114, 7382, 8192, 7740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=222576640 dispatch_meta_bytes=229600 combine_payload_bytes=222576640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=222806240 ingress_bytes=260325376 link_bytes=260325376 token_bytes=445382880 weight_movement_bytes=37748736 total_nvlink_bytes=483131616 weight_to_token_ratio=0.084756 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.083333 gemm_active_groups=11 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.494309665e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55308 payload_rows_remote=50814 dup_rows=10228 dup_rows_remote=10222 dedup_ratio=0.156067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 3886, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8048, 2154, 8192, 8132, 7906, 8190, 4494, 8192] dup_rows_per_dst=[1958, 2, 158, 0, 88, 0, 6, 8016] combine_rows_per_src=[8048, 2154, 8192, 8132, 7906, 8190, 4494, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=173277184 dispatch_meta_bytes=198640 combine_payload_bytes=173277184 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=173475824 ingress_bytes=229900288 link_bytes=229900288 token_bytes=346753008 weight_movement_bytes=56623104 total_nvlink_bytes=403376112 weight_to_token_ratio=0.163295 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.202791442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=184057856 dispatch_meta_bytes=208520 combine_payload_bytes=184057856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184266376 ingress_bytes=202932224 link_bytes=202932224 token_bytes=368324232 weight_movement_bytes=18874368 total_nvlink_bytes=387198600 weight_to_token_ratio=0.051244 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.944396721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=200220672 dispatch_meta_bytes=232264 combine_payload_bytes=200220672 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=200452936 ingress_bytes=256843776 link_bytes=256843776 token_bytes=400673608 weight_movement_bytes=56623104 total_nvlink_bytes=457296712 weight_to_token_ratio=0.141320 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.460950687e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=208134144 dispatch_meta_bytes=244144 combine_payload_bytes=208134144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208378288 ingress_bytes=227008512 link_bytes=227008512 token_bytes=416512432 weight_movement_bytes=18874368 total_nvlink_bytes=435386800 weight_to_token_ratio=0.045315 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=26752 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.175083867e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61792 payload_rows_remote=53600 dup_rows=3744 dup_rows_remote=3744 dedup_ratio=0.057129 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2793, 0, 3592, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3756, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 0, 0, 3, 4070, 0, 0, 0, 80, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 3, 3, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7190, 7512, 8050, 8178, 8192, 8192, 6426, 8052] dup_rows_per_dst=[6, 0, 50, 2742, 0, 60, 880, 6] combine_rows_per_src=[7190, 7512, 8050, 8178, 8192, 8192, 6426, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58484 payload_rows_remote=50380 dup_rows=7052 dup_rows_remote=7004 dedup_ratio=0.107605 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 0, 3330, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1514, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 19, 0, 0, 0, 4032, 0, 0, 0, 20, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6786, 3028, 8082, 8104, 8192, 8192, 7924, 8176] dup_rows_per_dst=[2344, 0, 104, 48, 0, 44, 4512, 0] combine_rows_per_src=[6786, 3028, 8082, 8104, 8192, 8192, 7924, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58748 payload_rows_remote=50862 dup_rows=6788 dup_rows_remote=6778 dedup_ratio=0.103577 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 3757, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2420, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3608, 0, 0, 0, 29, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 4840, 7886, 7292, 8192, 8192, 5976, 8190] dup_rows_per_dst=[6642, 0, 10, 20, 0, 116, 0, 0] combine_rows_per_src=[8180, 4840, 7886, 7292, 8192, 8192, 5976, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59762 payload_rows_remote=51570 dup_rows=5774 dup_rows_remote=5748 dedup_ratio=0.088104 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 3818, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3801, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3073, 0, 0, 0, 1, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5320, 7668, 7854, 6152, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2250, 132, 6, 0, 26, 3360, 0] combine_rows_per_src=[5320, 7668, 7854, 6152, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=219545600 dispatch_meta_bytes=229376 combine_payload_bytes=219545600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219774976 ingress_bytes=238419968 link_bytes=238419968 token_bytes=439320576 weight_movement_bytes=18874368 total_nvlink_bytes=458194944 weight_to_token_ratio=0.042963 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284422823e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56542 payload_rows_remote=48372 dup_rows=8994 dup_rows_remote=3884 dedup_ratio=0.137238 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 2678, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2942, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4037, 0, 0, 0, 7, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 6396, 8130, 4792, 8184, 7332, 5346, 8192] dup_rows_per_dst=[5110, 372, 3374, 2, 0, 134, 2, 0] combine_rows_per_src=[8170, 6396, 8130, 4792, 8184, 7332, 5346, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=206356480 dispatch_meta_bytes=229536 combine_payload_bytes=206356480 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=206586016 ingress_bytes=262979584 link_bytes=262979584 token_bytes=412942496 weight_movement_bytes=56623104 total_nvlink_bytes=469565600 weight_to_token_ratio=0.137121 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.519740980e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=208330752 dispatch_meta_bytes=230560 combine_payload_bytes=208330752 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208561312 ingress_bytes=246079488 link_bytes=246079488 token_bytes=416892064 weight_movement_bytes=37748736 total_nvlink_bytes=454640800 weight_to_token_ratio=0.090548 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357812575e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56034 payload_rows_remote=47986 dup_rows=9502 dup_rows_remote=9096 dedup_ratio=0.144989 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 3936, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 3652, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 623, 0, 0, 4, 4016, 0, 0, 0, 46, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4, 4, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8, 7764, 8150, 8188, 8192, 8192, 7492, 8048] dup_rows_per_dst=[0, 1622, 54, 6584, 8, 32, 796, 406] combine_rows_per_src=[8, 7764, 8150, 8188, 8192, 8192, 7492, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57224 payload_rows_remote=49176 dup_rows=8312 dup_rows_remote=8312 dedup_ratio=0.126831 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 3849, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4024, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 3314, 0, 0, 0, 531, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8048, 6030, 7618, 8192, 8192, 2870, 8082] dup_rows_per_dst=[7588, 0, 6, 436, 0, 282, 0, 0] combine_rows_per_src=[8192, 8048, 6030, 7618, 8192, 8192, 2870, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211230720 dispatch_meta_bytes=229272 combine_payload_bytes=211230720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=211459992 ingress_bytes=230105088 link_bytes=230105088 token_bytes=422690712 weight_movement_bytes=18874368 total_nvlink_bytes=441565080 weight_to_token_ratio=0.044653 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.204753734e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61120 payload_rows_remote=52928 dup_rows=4416 dup_rows_remote=4320 dedup_ratio=0.067383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3291, 0, 0, 0, 519, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4093, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 3999] payload_rows_per_dst=[8192, 8172, 8174, 7182, 5606, 8180, 8192, 7422] dup_rows_per_dst=[0, 2, 0, 446, 1404, 0, 96, 2468] combine_rows_per_src=[8192, 8172, 8174, 7182, 5606, 8180, 8192, 7422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=198131712 dispatch_meta_bytes=209024 combine_payload_bytes=198131712 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=198340736 ingress_bytes=273629184 link_bytes=273629184 token_bytes=396472448 weight_movement_bytes=75497472 total_nvlink_bytes=471969920 weight_to_token_ratio=0.190423 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.621780207e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=196550656 dispatch_meta_bytes=228328 combine_payload_bytes=196550656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196778984 ingress_bytes=215425024 link_bytes=215425024 token_bytes=393329640 weight_movement_bytes=18874368 total_nvlink_bytes=412204008 weight_to_token_ratio=0.047986 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.064096584e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=201424896 dispatch_meta_bytes=229952 combine_payload_bytes=201424896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=201654848 ingress_bytes=258048000 link_bytes=258048000 token_bytes=403079744 weight_movement_bytes=56623104 total_nvlink_bytes=459702848 weight_to_token_ratio=0.140476 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.472488968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61758 payload_rows_remote=53570 dup_rows=3778 dup_rows_remote=3772 dedup_ratio=0.057648 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 4093, 75, 0, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 2146, 0, 0, 67, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 560, 0, 3, 0, 763, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8186, 8188, 4432, 8192, 8190] dup_rows_per_dst=[4, 12, 134, 0, 6, 0, 980, 2642] combine_rows_per_src=[8186, 8192, 8192, 8186, 8188, 4432, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216793088 dispatch_meta_bytes=228992 combine_payload_bytes=216793088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=217022080 ingress_bytes=273416192 link_bytes=273416192 token_bytes=433815168 weight_movement_bytes=56623104 total_nvlink_bytes=490438272 weight_to_token_ratio=0.130524 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.619739423e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64744 payload_rows_remote=56570 dup_rows=792 dup_rows_remote=792 dedup_ratio=0.012085 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3078, 0, 0, 711, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 218, 0, 0, 0, 99, 0, 0] payload_rows_per_dst=[8192, 8192, 8086, 8174, 8192, 7532, 8192, 8184] dup_rows_per_dst=[16, 8, 58, 0, 0, 50, 26, 634] combine_rows_per_src=[8192, 8192, 8086, 8174, 8192, 7532, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63464 payload_rows_remote=55302 dup_rows=2072 dup_rows_remote=890 dedup_ratio=0.031616 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 646, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 48, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[8192, 8192, 8162, 8140, 8192, 6202, 8192, 8192] dup_rows_per_dst=[90, 4, 1182, 0, 0, 0, 558, 238] combine_rows_per_src=[8192, 8192, 8162, 8140, 8192, 6202, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61174 payload_rows_remote=53288 dup_rows=4362 dup_rows_remote=700 dedup_ratio=0.066559 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1724, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 2144, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 61, 0, 0, 0, 27, 0, 0] payload_rows_per_dst=[8192, 8192, 4162, 8184, 8174, 7886, 8192, 8192] dup_rows_per_dst=[0, 14, 286, 0, 0, 3662, 224, 176] combine_rows_per_src=[8192, 8192, 4162, 8184, 8174, 7886, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=219422720 dispatch_meta_bytes=229368 combine_payload_bytes=219422720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219652088 ingress_bytes=238297088 link_bytes=238297088 token_bytes=439074808 weight_movement_bytes=18874368 total_nvlink_bytes=457949176 weight_to_token_ratio=0.042987 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.283245448e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52612 payload_rows_remote=44420 dup_rows=12924 dup_rows_remote=12924 dedup_ratio=0.197205 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 282, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 763, 0, 0, 0, 87, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 5472, 4484, 1696] dup_rows_per_dst=[0, 70, 8652, 4022, 0, 42, 134, 4] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 5472, 4484, 1696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231710720 dispatch_meta_bytes=229448 combine_payload_bytes=231710720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=231940168 ingress_bytes=288333824 link_bytes=288333824 token_bytes=463650888 weight_movement_bytes=56623104 total_nvlink_bytes=520273992 weight_to_token_ratio=0.122124 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.762672832e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53522 payload_rows_remote=45330 dup_rows=12014 dup_rows_remote=6702 dedup_ratio=0.183319 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 4092, 241, 0, 0, 0, 0, 0, 0, 4, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 2742, 0, 0, 65, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 57, 0, 4, 0, 669, 0, 0] payload_rows_per_dst=[7990, 8032, 672, 4344, 8102, 7998, 8192, 8192] dup_rows_per_dst=[34, 26, 12, 0, 106, 5602, 922, 5312] combine_rows_per_src=[7990, 8032, 672, 4344, 8102, 7998, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=226516992 dispatch_meta_bytes=224768 combine_payload_bytes=226516992 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226741760 ingress_bytes=283140096 link_bytes=283140096 token_bytes=453258752 weight_movement_bytes=56623104 total_nvlink_bytes=509881856 weight_to_token_ratio=0.124924 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.712909086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59004 payload_rows_remote=50812 dup_rows=6532 dup_rows_remote=6506 dedup_ratio=0.099670 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1501, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2276, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 84, 0, 0, 0, 62, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8156, 8192, 4626, 8192, 5262] dup_rows_per_dst=[56, 26, 5534, 0, 0, 0, 634, 282] combine_rows_per_src=[8192, 8192, 8192, 8156, 8192, 4626, 8192, 5262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=218267648 dispatch_meta_bytes=215952 combine_payload_bytes=218267648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218483600 ingress_bytes=237142016 link_bytes=237142016 token_bytes=436751248 weight_movement_bytes=18874368 total_nvlink_bytes=455625616 weight_to_token_ratio=0.043215 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272178116e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56144 payload_rows_remote=47952 dup_rows=9392 dup_rows_remote=8358 dedup_ratio=0.143311 router_tokens_per_expert=[3864, 0, 0, 0, 0, 4087, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3527, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 7, 0, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 515, 0, 0, 0, 3945, 0, 0, 0, 2, 0, 0, 37, 0, 0, 0, 0, 0] payload_rows_per_dst=[2, 7056, 8170, 8192, 8188, 8170, 8192, 8174] dup_rows_per_dst=[0, 0, 36, 776, 4, 20, 1034, 7522] combine_rows_per_src=[2, 7056, 8170, 8192, 8188, 8170, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=181944320 dispatch_meta_bytes=229376 combine_payload_bytes=181944320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=182173696 ingress_bytes=200818688 link_bytes=200818688 token_bytes=364118016 weight_movement_bytes=18874368 total_nvlink_bytes=382992384 weight_to_token_ratio=0.051836 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.924145859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185671680 dispatch_meta_bytes=208128 combine_payload_bytes=185671680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=185879808 ingress_bytes=223420416 link_bytes=223420416 token_bytes=371551488 weight_movement_bytes=37748736 total_nvlink_bytes=409300224 weight_to_token_ratio=0.101598 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=27264 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.140704496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=208125952 dispatch_meta_bytes=229272 combine_payload_bytes=208125952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208355224 ingress_bytes=245874688 link_bytes=245874688 token_bytes=416481176 weight_movement_bytes=37748736 total_nvlink_bytes=454229912 weight_to_token_ratio=0.090637 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.355850283e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=196411392 dispatch_meta_bytes=225240 combine_payload_bytes=196411392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196636632 ingress_bytes=215285760 link_bytes=215285760 token_bytes=393048024 weight_movement_bytes=18874368 total_nvlink_bytes=411922392 weight_to_token_ratio=0.048021 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.062762225e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64422 payload_rows_remote=56230 dup_rows=1114 dup_rows_remote=628 dedup_ratio=0.016998 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 3652, 0, 4093, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4093, 0, 0, 4021, 0, 0, 3, 0, 0, 0, 0, 3, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 7312, 8186, 8114, 8192, 8192, 8186, 8048] dup_rows_per_dst=[6, 2, 8, 0, 486, 50, 550, 12] combine_rows_per_src=[8192, 7312, 8186, 8114, 8192, 8192, 8186, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65180 payload_rows_remote=57158 dup_rows=356 dup_rows_remote=356 dedup_ratio=0.005432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 5, 0, 0, 0, 18, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 4096, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 7, 77, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 0, 8, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8022, 8070, 8192, 8140, 8180] dup_rows_per_dst=[6, 48, 138, 0, 68, 80, 16, 0] combine_rows_per_src=[8192, 8192, 8192, 8022, 8070, 8192, 8140, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63996 payload_rows_remote=55804 dup_rows=1540 dup_rows_remote=994 dedup_ratio=0.023499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 29, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 282, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 3713, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7508, 8010, 8192, 7518, 8192] dup_rows_per_dst=[4, 62, 546, 0, 818, 106, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 7508, 8010, 8192, 7518, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64140 payload_rows_remote=55948 dup_rows=1396 dup_rows_remote=1392 dedup_ratio=0.021301 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 61, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7868, 8110, 8192, 7202, 8192] dup_rows_per_dst=[4, 14, 122, 0, 1176, 4, 76, 0] combine_rows_per_src=[8192, 8192, 8192, 7868, 8110, 8192, 7202, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230318080 dispatch_meta_bytes=227432 combine_payload_bytes=230318080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230545512 ingress_bytes=249192448 link_bytes=249192448 token_bytes=460863592 weight_movement_bytes=18874368 total_nvlink_bytes=479737960 weight_to_token_ratio=0.040954 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.387639426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234119168 dispatch_meta_bytes=230056 combine_payload_bytes=234119168 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=234349224 ingress_bytes=347365376 link_bytes=347365376 token_bytes=468468392 weight_movement_bytes=113246208 total_nvlink_bytes=581714600 weight_to_token_ratio=0.241737 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.328284118e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57538 payload_rows_remote=49346 dup_rows=7998 dup_rows_remote=7996 dedup_ratio=0.122040 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7674, 8180, 1086, 7896, 8126, 8192] dup_rows_per_dst=[2, 838, 26, 7106, 8, 10, 8, 0] combine_rows_per_src=[8192, 8192, 7674, 8180, 1086, 7896, 8126, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=228573184 dispatch_meta_bytes=227192 combine_payload_bytes=228573184 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=228800376 ingress_bytes=285196288 link_bytes=285196288 token_bytes=457373560 weight_movement_bytes=56623104 total_nvlink_bytes=513996664 weight_to_token_ratio=0.123801 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.732610506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59058 payload_rows_remote=50870 dup_rows=6478 dup_rows_remote=6326 dedup_ratio=0.098846 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 19, 3985, 0, 4092, 0, 0, 0, 0, 0, 0, 149, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 4, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[8168, 8010, 6492, 3666, 8192, 8192, 8150, 8188] dup_rows_per_dst=[2, 6, 266, 24, 544, 48, 5436, 152] combine_rows_per_src=[8168, 8010, 6492, 3666, 8192, 8192, 8150, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62868 payload_rows_remote=54678 dup_rows=2668 dup_rows_remote=2632 dedup_ratio=0.040710 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 20, 4087, 0, 4089, 0, 0, 0, 0, 0, 0, 406, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1013, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1, 0, 4, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7138, 8188, 8178, 6600, 8190] dup_rows_per_dst=[0, 36, 778, 0, 1852, 0, 2, 0] combine_rows_per_src=[8192, 8190, 8192, 7138, 8188, 8178, 6600, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229163008 dispatch_meta_bytes=229360 combine_payload_bytes=229163008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229392368 ingress_bytes=248037376 link_bytes=248037376 token_bytes=458555376 weight_movement_bytes=18874368 total_nvlink_bytes=477429744 weight_to_token_ratio=0.041160 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.376572095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53944 payload_rows_remote=45752 dup_rows=11592 dup_rows_remote=8024 dedup_ratio=0.176880 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4031, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3987, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2671, 0, 0, 0, 11, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 8192, 7618, 5366, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 7844, 6, 8, 0, 166, 3568, 0] combine_rows_per_src=[0, 8192, 7618, 5366, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=202121216 dispatch_meta_bytes=229368 combine_payload_bytes=202121216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202350584 ingress_bytes=220995584 link_bytes=220995584 token_bytes=404471800 weight_movement_bytes=18874368 total_nvlink_bytes=423346168 weight_to_token_ratio=0.046664 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.117470949e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=223961088 dispatch_meta_bytes=229240 combine_payload_bytes=223961088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224190328 ingress_bytes=242835456 link_bytes=242835456 token_bytes=448151416 weight_movement_bytes=18874368 total_nvlink_bytes=467025784 weight_to_token_ratio=0.042116 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.326729857e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=208363520 dispatch_meta_bytes=228784 combine_payload_bytes=208363520 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208592304 ingress_bytes=246112256 link_bytes=246112256 token_bytes=416955824 weight_movement_bytes=37748736 total_nvlink_bytes=454704560 weight_to_token_ratio=0.090534 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.358126542e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=187400192 dispatch_meta_bytes=215104 combine_payload_bytes=187400192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=187615296 ingress_bytes=206274560 link_bytes=206274560 token_bytes=375015488 weight_movement_bytes=18874368 total_nvlink_bytes=393889856 weight_to_token_ratio=0.050330 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.976421341e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64814 payload_rows_remote=56628 dup_rows=722 dup_rows_remote=214 dedup_ratio=0.011017 router_tokens_per_expert=[3949, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3728, 235, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 166, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4079, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[7908, 8190, 7802, 8186, 8186, 8186, 8192, 8164] dup_rows_per_dst=[18, 6, 130, 22, 508, 36, 2, 0] combine_rows_per_src=[7908, 8190, 7802, 8186, 8186, 8186, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61790 payload_rows_remote=53598 dup_rows=3746 dup_rows_remote=3746 dedup_ratio=0.057159 router_tokens_per_expert=[2932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 3097, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 114, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[6490, 8178, 7342, 8192, 8192, 8128, 7120, 8148] dup_rows_per_dst=[1800, 0, 1662, 0, 266, 0, 10, 8] combine_rows_per_src=[6490, 8178, 7342, 8192, 8192, 8128, 7120, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63822 payload_rows_remote=56912 dup_rows=1714 dup_rows_remote=1714 dedup_ratio=0.026154 router_tokens_per_expert=[2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8168, 6910, 8192, 8192, 7848, 8144, 8178] dup_rows_per_dst=[1582, 0, 0, 28, 104, 0, 0, 0] combine_rows_per_src=[8190, 8168, 6910, 8192, 8192, 7848, 8144, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=231948288 dispatch_meta_bytes=227368 combine_payload_bytes=231948288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232175656 ingress_bytes=269697024 link_bytes=269697024 token_bytes=464123944 weight_movement_bytes=37748736 total_nvlink_bytes=501872680 weight_to_token_ratio=0.081333 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.584104185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61010 payload_rows_remote=52908 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.069061 router_tokens_per_expert=[2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3503, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9] payload_rows_per_dst=[4020, 8190, 8180, 8192, 8192, 8102, 7966, 8168] dup_rows_per_dst=[10, 0, 4418, 2, 78, 0, 0, 18] combine_rows_per_src=[4020, 8190, 8180, 8192, 8192, 8102, 7966, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=219537408 dispatch_meta_bytes=229376 combine_payload_bytes=219537408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219766784 ingress_bytes=238411776 link_bytes=238411776 token_bytes=439304192 weight_movement_bytes=18874368 total_nvlink_bytes=458178560 weight_to_token_ratio=0.042964 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284344332e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61020 payload_rows_remote=52832 dup_rows=4516 dup_rows_remote=174 dedup_ratio=0.068909 router_tokens_per_expert=[3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3038, 1982, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8188, 8192, 3918, 8092, 8192, 8182, 8082, 8174] dup_rows_per_dst=[4342, 64, 0, 0, 106, 0, 0, 4] combine_rows_per_src=[8188, 8192, 3918, 8092, 8192, 8182, 8082, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233111552 dispatch_meta_bytes=234504 combine_payload_bytes=233111552 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=233346056 ingress_bytes=308609024 link_bytes=308609024 token_bytes=466457608 weight_movement_bytes=75497472 total_nvlink_bytes=541955080 weight_to_token_ratio=0.161853 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17664 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.956939823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62946 payload_rows_remote=54778 dup_rows=2590 dup_rows_remote=2590 dedup_ratio=0.039520 router_tokens_per_expert=[2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3023, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 10, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5] payload_rows_per_dst=[8178, 8168, 6672, 8192, 8192, 7514, 8188, 7842] dup_rows_per_dst=[2424, 0, 0, 34, 122, 6, 2, 2] combine_rows_per_src=[8178, 8168, 6672, 8192, 8192, 7514, 8188, 7842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61724 payload_rows_remote=53600 dup_rows=3812 dup_rows_remote=3246 dedup_ratio=0.058167 router_tokens_per_expert=[3833, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 4088, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3533, 655, 4, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4050, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4058, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[7674, 8184, 7074, 8180, 6586, 8148, 7754, 8124] dup_rows_per_dst=[10, 8, 16, 24, 380, 1624, 1184, 566] combine_rows_per_src=[7674, 8184, 7074, 8180, 6586, 8148, 7754, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61304 payload_rows_remote=53112 dup_rows=4232 dup_rows_remote=4002 dedup_ratio=0.064575 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2010, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1848, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 9, 0, 0, 0, 67, 0, 0] payload_rows_per_dst=[8192, 8192, 4020, 8138, 8192, 8190, 8192, 8188] dup_rows_per_dst=[8, 0, 0, 0, 0, 3842, 230, 152] combine_rows_per_src=[8192, 8192, 4020, 8138, 8192, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216711168 dispatch_meta_bytes=229736 combine_payload_bytes=216711168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216940904 ingress_bytes=235585536 link_bytes=235585536 token_bytes=433652072 weight_movement_bytes=18874368 total_nvlink_bytes=452526440 weight_to_token_ratio=0.043524 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.257264691e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216399872 dispatch_meta_bytes=212024 combine_payload_bytes=216399872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216611896 ingress_bytes=254148608 link_bytes=254148608 token_bytes=433011768 weight_movement_bytes=37748736 total_nvlink_bytes=470760504 weight_to_token_ratio=0.087177 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.435126913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224370688 dispatch_meta_bytes=229472 combine_payload_bytes=224370688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224600160 ingress_bytes=243245056 link_bytes=243245056 token_bytes=448970848 weight_movement_bytes=18874368 total_nvlink_bytes=467845216 weight_to_token_ratio=0.042039 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330654443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=219545600 dispatch_meta_bytes=227384 combine_payload_bytes=219545600 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=219772984 ingress_bytes=257294336 link_bytes=257294336 token_bytes=439318584 weight_movement_bytes=37748736 total_nvlink_bytes=477067320 weight_to_token_ratio=0.085926 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.465267731e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=217546752 dispatch_meta_bytes=228456 combine_payload_bytes=217546752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217775208 ingress_bytes=236421120 link_bytes=236421120 token_bytes=435321960 weight_movement_bytes=18874368 total_nvlink_bytes=454196328 weight_to_token_ratio=0.043357 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265270845e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58014 payload_rows_remote=54760 dup_rows=7522 dup_rows_remote=7522 dedup_ratio=0.114777 router_tokens_per_expert=[0, 0, 0, 0, 4092, 0, 0, 60, 0, 0, 0, 0, 0, 3, 3841, 0, 0, 3, 0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 1, 48, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 10, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 0, 0] payload_rows_per_dst=[7696, 7934, 8186, 6506, 3254, 8058, 8192, 8188] dup_rows_per_dst=[112, 0, 860, 166, 0, 6372, 12, 0] combine_rows_per_src=[7696, 7934, 8186, 6506, 3254, 8058, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60198 payload_rows_remote=52016 dup_rows=5338 dup_rows_remote=1850 dedup_ratio=0.081451 router_tokens_per_expert=[0, 0, 0, 0, 4087, 0, 0, 198, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 10, 0, 0, 0, 0, 3, 0, 0, 82, 5, 19, 0, 0, 531, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0] payload_rows_per_dst=[7776, 7792, 8192, 8182, 8156, 3732, 8192, 8176] dup_rows_per_dst=[372, 26, 1260, 3488, 4, 0, 144, 44] combine_rows_per_src=[7776, 7792, 8192, 8182, 8156, 3732, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57860 payload_rows_remote=49884 dup_rows=7676 dup_rows_remote=4136 dedup_ratio=0.117126 router_tokens_per_expert=[0, 0, 0, 0, 4087, 0, 0, 51, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 97, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0] payload_rows_per_dst=[7104, 7876, 7976, 8190, 8102, 2242, 8192, 8178] dup_rows_per_dst=[912, 32, 3540, 2720, 2, 0, 458, 12] combine_rows_per_src=[7104, 7876, 7976, 8190, 8102, 2242, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224296960 dispatch_meta_bytes=249128 combine_payload_bytes=224296960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224546088 ingress_bytes=243171328 link_bytes=243171328 token_bytes=448843048 weight_movement_bytes=18874368 total_nvlink_bytes=467717416 weight_to_token_ratio=0.042051 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.329948017e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213057536 dispatch_meta_bytes=215464 combine_payload_bytes=213057536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213273000 ingress_bytes=231931904 link_bytes=231931904 token_bytes=426330536 weight_movement_bytes=18874368 total_nvlink_bytes=445204904 weight_to_token_ratio=0.044272 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222257387e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60316 payload_rows_remote=52184 dup_rows=5220 dup_rows_remote=2860 dedup_ratio=0.079651 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 13, 0, 1, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 75, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8002, 7922, 8192, 3528, 8156, 8132, 8192, 8192] dup_rows_per_dst=[120, 4, 2454, 0, 4, 2360, 242, 36] combine_rows_per_src=[8002, 7922, 8192, 3528, 8156, 8132, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54752 payload_rows_remote=46560 dup_rows=10784 dup_rows_remote=3016 dedup_ratio=0.164551 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 7672, 1642, 8144, 8148, 6292, 6588, 8074] dup_rows_per_dst=[7768, 4, 0, 2292, 0, 630, 80, 10] combine_rows_per_src=[8192, 7672, 1642, 8144, 8148, 6292, 6588, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=204324864 dispatch_meta_bytes=216080 combine_payload_bytes=204324864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204540944 ingress_bytes=223199232 link_bytes=223199232 token_bytes=408865808 weight_movement_bytes=18874368 total_nvlink_bytes=427740176 weight_to_token_ratio=0.046163 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.138585220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53040 payload_rows_remote=45120 dup_rows=12496 dup_rows_remote=12482 dedup_ratio=0.190674 router_tokens_per_expert=[0, 0, 0, 0, 4089, 0, 0, 37, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 5, 249, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 49, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0] payload_rows_per_dst=[8180, 7920, 1562, 8190, 8010, 2816, 8192, 8170] dup_rows_per_dst=[7700, 14, 78, 4292, 230, 0, 140, 42] combine_rows_per_src=[8180, 7920, 1562, 8190, 8010, 2816, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55064 payload_rows_remote=46902 dup_rows=10472 dup_rows_remote=10472 dedup_ratio=0.159790 router_tokens_per_expert=[0, 0, 0, 0, 4091, 0, 0, 66, 0, 0, 0, 0, 0, 4, 2791, 0, 0, 4, 0, 0, 0, 4014, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 217, 1, 57, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 0, 0, 0, 0, 10, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 0, 0, 0] payload_rows_per_dst=[140, 8036, 8184, 6694, 7464, 8192, 8192, 8162] dup_rows_per_dst=[0, 8, 1068, 0, 1284, 8096, 16, 0] combine_rows_per_src=[140, 8036, 8184, 6694, 7464, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62536 payload_rows_remote=57014 dup_rows=3000 dup_rows_remote=3000 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 28, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 172, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7910, 8146, 8192, 5522, 8190] dup_rows_per_dst=[0, 58, 340, 0, 2590, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7910, 8146, 8192, 5522, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213745664 dispatch_meta_bytes=220176 combine_payload_bytes=213745664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=213965840 ingress_bytes=270368768 link_bytes=270368768 token_bytes=427711504 weight_movement_bytes=56623104 total_nvlink_bytes=484334608 weight_to_token_ratio=0.132386 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.100694 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.590540505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=190709760 dispatch_meta_bytes=198304 combine_payload_bytes=190709760 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=190908064 ingress_bytes=285081600 link_bytes=285081600 token_bytes=381617824 weight_movement_bytes=94371840 total_nvlink_bytes=475989664 weight_to_token_ratio=0.247294 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.731511622e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=184811520 dispatch_meta_bytes=230408 combine_payload_bytes=184811520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=185041928 ingress_bytes=203685888 link_bytes=203685888 token_bytes=369853448 weight_movement_bytes=18874368 total_nvlink_bytes=388727816 weight_to_token_ratio=0.051032 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.951617959e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54288 payload_rows_remote=46096 dup_rows=11248 dup_rows_remote=10606 dedup_ratio=0.171631 router_tokens_per_expert=[0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 1190, 3, 0, 62, 2, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 2953, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2569] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 4, 8188, 5140] dup_rows_per_dst=[0, 2402, 6034, 14, 642, 0, 2152, 4] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 4, 8188, 5140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=192110592 dispatch_meta_bytes=229496 combine_payload_bytes=192110592 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=192340088 ingress_bytes=210984960 link_bytes=210984960 token_bytes=384450680 weight_movement_bytes=18874368 total_nvlink_bytes=403325048 weight_to_token_ratio=0.049094 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.021554076e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233529344 dispatch_meta_bytes=240056 combine_payload_bytes=233529344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233769400 ingress_bytes=252403712 link_bytes=252403712 token_bytes=467298744 weight_movement_bytes=18874368 total_nvlink_bytes=486173112 weight_to_token_ratio=0.040390 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.418408178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57062 payload_rows_remote=48870 dup_rows=8474 dup_rows_remote=8464 dedup_ratio=0.129303 router_tokens_per_expert=[0, 0, 0, 4091, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 38, 5, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3915, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4017] payload_rows_per_dst=[8182, 8078, 8192, 8192, 8190, 4, 8180, 8044] dup_rows_per_dst=[0, 2408, 662, 10, 4664, 0, 706, 24] combine_rows_per_src=[8182, 8078, 8192, 8192, 8190, 4, 8180, 8044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58308 payload_rows_remote=50126 dup_rows=7228 dup_rows_remote=6520 dedup_ratio=0.110291 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 151, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 4036] payload_rows_per_dst=[8192, 8182, 8182, 8192, 8192, 2352, 6934, 8082] dup_rows_per_dst=[0, 2580, 708, 12, 1198, 0, 2224, 506] combine_rows_per_src=[8192, 8182, 8182, 8192, 8192, 2352, 6934, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=188809216 dispatch_meta_bytes=226808 combine_payload_bytes=188809216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=189036024 ingress_bytes=207683584 link_bytes=207683584 token_bytes=377845240 weight_movement_bytes=18874368 total_nvlink_bytes=396719608 weight_to_token_ratio=0.049953 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.989921915e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=200171520 dispatch_meta_bytes=229336 combine_payload_bytes=200171520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200400856 ingress_bytes=219045888 link_bytes=219045888 token_bytes=400572376 weight_movement_bytes=18874368 total_nvlink_bytes=419446744 weight_to_token_ratio=0.047118 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.098789922e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55330 payload_rows_remote=47592 dup_rows=10206 dup_rows_remote=3506 dedup_ratio=0.155731 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2200, 0, 0, 325, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 4024] payload_rows_per_dst=[8192, 5170, 1558, 8192, 8192, 7738, 8192, 8096] dup_rows_per_dst=[0, 578, 134, 0, 886, 6700, 1166, 742] combine_rows_per_src=[8192, 5170, 1558, 8192, 8192, 7738, 8192, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=205316096 dispatch_meta_bytes=226584 combine_payload_bytes=205316096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205542680 ingress_bytes=243064832 link_bytes=243064832 token_bytes=410858776 weight_movement_bytes=37748736 total_nvlink_bytes=448607512 weight_to_token_ratio=0.091878 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.328927625e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52500 payload_rows_remote=44308 dup_rows=13036 dup_rows_remote=4764 dedup_ratio=0.198914 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 0, 165, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4038] payload_rows_per_dst=[8192, 8018, 8192, 8112, 3090, 8182, 634, 8080] dup_rows_per_dst=[8272, 3416, 1302, 0, 0, 0, 0, 46] combine_rows_per_src=[8192, 8018, 8192, 8112, 3090, 8182, 634, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57420 payload_rows_remote=49230 dup_rows=8116 dup_rows_remote=5542 dedup_ratio=0.123840 router_tokens_per_expert=[0, 0, 0, 4091, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 532, 11, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 3698] payload_rows_per_dst=[8188, 8190, 8192, 8192, 5460, 7796, 3932, 7470] dup_rows_per_dst=[2724, 2574, 1348, 14, 64, 0, 0, 1392] combine_rows_per_src=[8188, 8190, 8192, 8192, 5460, 7796, 3932, 7470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=46760 payload_rows_remote=41246 dup_rows=18776 dup_rows_remote=16844 dedup_ratio=0.286499 router_tokens_per_expert=[0, 0, 0, 4092, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 4, 0, 1160, 3, 0, 24, 1, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4092, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 2285] payload_rows_per_dst=[0, 2340, 6162, 8192, 8190, 8184, 8178, 5514] dup_rows_per_dst=[0, 8, 3220, 24, 298, 8202, 5092, 1932] combine_rows_per_src=[0, 2340, 6162, 8192, 8190, 8184, 8178, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61360 payload_rows_remote=53174 dup_rows=4176 dup_rows_remote=3470 dedup_ratio=0.063721 router_tokens_per_expert=[2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 3034, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[4702, 8188, 8006, 8192, 8192, 7762, 8186, 8132] dup_rows_per_dst=[0, 0, 3406, 8, 48, 0, 706, 8] combine_rows_per_src=[4702, 8188, 8006, 8192, 8192, 7762, 8186, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=194936832 dispatch_meta_bytes=204392 combine_payload_bytes=194936832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=195141224 ingress_bytes=251559936 link_bytes=251559936 token_bytes=390078056 weight_movement_bytes=56623104 total_nvlink_bytes=446701160 weight_to_token_ratio=0.145158 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=12160 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410323532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=181485568 dispatch_meta_bytes=196288 combine_payload_bytes=181485568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=181681856 ingress_bytes=256983040 link_bytes=256983040 token_bytes=363167424 weight_movement_bytes=75497472 total_nvlink_bytes=438664896 weight_to_token_ratio=0.207886 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.462285046e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=201646080 dispatch_meta_bytes=219088 combine_payload_bytes=201646080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201865168 ingress_bytes=220520448 link_bytes=220520448 token_bytes=403511248 weight_movement_bytes=18874368 total_nvlink_bytes=422385616 weight_to_token_ratio=0.046775 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112918430e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58992 payload_rows_remote=50806 dup_rows=6544 dup_rows_remote=6528 dedup_ratio=0.099854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3064, 3, 0, 0, 3, 0, 0, 0, 0, 0, 4088, 0, 0, 2939, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2052, 0, 0, 0, 3, 0, 4071, 0, 0, 2, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4093, 0, 2139, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1489, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6134, 7636, 8192, 8192, 8186, 4284, 8186, 8182] dup_rows_per_dst=[6, 2352, 1180, 6, 16, 0, 2984, 0] combine_rows_per_src=[6134, 7636, 8192, 8192, 8186, 4284, 8186, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=168943616 dispatch_meta_bytes=232360 combine_payload_bytes=168943616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=169175976 ingress_bytes=187817984 link_bytes=187817984 token_bytes=338119592 weight_movement_bytes=18874368 total_nvlink_bytes=356993960 weight_to_token_ratio=0.055822 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.799579510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=217800704 dispatch_meta_bytes=226576 combine_payload_bytes=217800704 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=218027280 ingress_bytes=274423808 link_bytes=274423808 token_bytes=435827984 weight_movement_bytes=56623104 total_nvlink_bytes=492451088 weight_to_token_ratio=0.129921 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.629393903e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56906 payload_rows_remote=48786 dup_rows=8630 dup_rows_remote=8628 dedup_ratio=0.131683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3199, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7322, 8046, 8120, 8192, 8122, 6408, 2512, 8184] dup_rows_per_dst=[1562, 6766, 2, 18, 0, 162, 120, 0] combine_rows_per_src=[7322, 8046, 8120, 8192, 8122, 6408, 2512, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59966 payload_rows_remote=51774 dup_rows=5570 dup_rows_remote=5568 dedup_ratio=0.084991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 4073, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 2428, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6240, 7948, 8162, 8192, 8180, 4886, 8192, 8166] dup_rows_per_dst=[0, 2982, 2, 2, 0, 72, 2512, 0] combine_rows_per_src=[6240, 7948, 8162, 8192, 8180, 4886, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=208101376 dispatch_meta_bytes=229336 combine_payload_bytes=208101376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208330712 ingress_bytes=226975744 link_bytes=226975744 token_bytes=416432088 weight_movement_bytes=18874368 total_nvlink_bytes=435306456 weight_to_token_ratio=0.045324 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.174769900e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=199827456 dispatch_meta_bytes=229656 combine_payload_bytes=199827456 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=200057112 ingress_bytes=237576192 link_bytes=237576192 token_bytes=399884568 weight_movement_bytes=37748736 total_nvlink_bytes=437633304 weight_to_token_ratio=0.094399 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.276338177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=212066304 dispatch_meta_bytes=229368 combine_payload_bytes=212066304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212295672 ingress_bytes=230940672 link_bytes=230940672 token_bytes=424361976 weight_movement_bytes=18874368 total_nvlink_bytes=443236344 weight_to_token_ratio=0.044477 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.212759889e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60328 payload_rows_remote=53160 dup_rows=5208 dup_rows_remote=3816 dedup_ratio=0.079468 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3299, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5504, 6700, 8192, 8192, 8192, 7168, 8192, 8188] dup_rows_per_dst=[0, 1354, 20, 0, 0, 1392, 2442, 0] combine_rows_per_src=[5504, 6700, 8192, 8192, 8192, 7168, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58986 payload_rows_remote=50794 dup_rows=6550 dup_rows_remote=2562 dedup_ratio=0.099945 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3192, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7496, 8192, 8192, 8190, 6416, 4156, 8152] dup_rows_per_dst=[3988, 2532, 22, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 7496, 8192, 8192, 8190, 6416, 4156, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59356 payload_rows_remote=51914 dup_rows=6180 dup_rows_remote=5218 dedup_ratio=0.094299 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 3852, 0, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3466, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7442, 7720, 8192, 8164, 6932, 4546, 8168] dup_rows_per_dst=[4952, 962, 6, 16, 2, 242, 0, 0] combine_rows_per_src=[8192, 7442, 7720, 8192, 8164, 6932, 4546, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53708 payload_rows_remote=45516 dup_rows=11828 dup_rows_remote=11316 dedup_ratio=0.180481 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 197, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] payload_rows_per_dst=[1460, 8174, 8192, 3144, 8170, 8188, 8192, 8188] dup_rows_per_dst=[2, 2122, 4150, 0, 4552, 426, 512, 64] combine_rows_per_src=[1460, 8174, 8192, 3144, 8170, 8188, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56916 payload_rows_remote=48770 dup_rows=8620 dup_rows_remote=8462 dedup_ratio=0.131531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3864, 4, 0, 0, 4, 0, 0, 0, 0, 0, 4067, 0, 0, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 0, 0, 4, 0, 4047, 0, 0, 13, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3701, 1, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 904, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7736, 1208, 7238, 8128, 8176, 8100, 8184, 8146] dup_rows_per_dst=[8, 0, 292, 18, 146, 5380, 2618, 158] combine_rows_per_src=[7736, 1208, 7238, 8128, 8176, 8100, 8184, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=217743360 dispatch_meta_bytes=227904 combine_payload_bytes=217743360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217971264 ingress_bytes=255492096 link_bytes=255492096 token_bytes=435714624 weight_movement_bytes=37748736 total_nvlink_bytes=473463360 weight_to_token_ratio=0.086636 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.447999554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63034 payload_rows_remote=54850 dup_rows=2502 dup_rows_remote=2496 dedup_ratio=0.038177 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 33, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 4093, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 3, 0, 0, 0, 0, 0, 2921, 0, 3, 0, 0, 3, 4076, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8072, 8188, 8184, 5854, 8172, 8186] dup_rows_per_dst=[2396, 70, 0, 10, 6, 6, 6, 8] combine_rows_per_src=[8192, 8186, 8072, 8188, 8184, 5854, 8172, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=212639744 dispatch_meta_bytes=228528 combine_payload_bytes=212639744 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=212868272 ingress_bytes=288137216 link_bytes=288137216 token_bytes=425508016 weight_movement_bytes=75497472 total_nvlink_bytes=501005488 weight_to_token_ratio=0.177429 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=18816 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.760789031e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=208052224 dispatch_meta_bytes=213424 combine_payload_bytes=208052224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208265648 ingress_bytes=226926592 link_bytes=226926592 token_bytes=416317872 weight_movement_bytes=18874368 total_nvlink_bytes=435192240 weight_to_token_ratio=0.045336 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.174298950e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=186433536 dispatch_meta_bytes=227328 combine_payload_bytes=186433536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=186660864 ingress_bytes=205307904 link_bytes=205307904 token_bytes=373094400 weight_movement_bytes=18874368 total_nvlink_bytes=391968768 weight_to_token_ratio=0.050589 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.967159318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=199761920 dispatch_meta_bytes=228928 combine_payload_bytes=199761920 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=199990848 ingress_bytes=237510656 link_bytes=237510656 token_bytes=399752768 weight_movement_bytes=37748736 total_nvlink_bytes=437501504 weight_to_token_ratio=0.094430 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.275710243e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65348 payload_rows_remote=57184 dup_rows=188 dup_rows_remote=188 dedup_ratio=0.002869 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8164, 8106, 8152, 8158, 8192, 8192] dup_rows_per_dst=[80, 0, 0, 0, 0, 90, 0, 18] combine_rows_per_src=[8192, 8192, 8164, 8106, 8152, 8158, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57296 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 8170, 8164, 8190, 8192, 8192] dup_rows_per_dst=[42, 2, 0, 0, 0, 14, 2, 10] combine_rows_per_src=[8192, 8192, 8174, 8170, 8164, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224665600 dispatch_meta_bytes=229384 combine_payload_bytes=224665600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224894984 ingress_bytes=243539968 link_bytes=243539968 token_bytes=449560584 weight_movement_bytes=18874368 total_nvlink_bytes=468434952 weight_to_token_ratio=0.041984 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.333480144e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234225664 dispatch_meta_bytes=229488 combine_payload_bytes=234225664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234455152 ingress_bytes=271974400 link_bytes=271974400 token_bytes=468680816 weight_movement_bytes=37748736 total_nvlink_bytes=506429552 weight_to_token_ratio=0.080543 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605924881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229464 combine_payload_bytes=234684416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234913880 ingress_bytes=272433152 link_bytes=272433152 token_bytes=469598296 weight_movement_bytes=37748736 total_nvlink_bytes=507347032 weight_to_token_ratio=0.080385 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610320417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65390 payload_rows_remote=57310 dup_rows=146 dup_rows_remote=146 dedup_ratio=0.002228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 8080, 8190, 8192] dup_rows_per_dst=[136, 0, 0, 0, 0, 0, 0, 10] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 8080, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62802 payload_rows_remote=54610 dup_rows=2734 dup_rows_remote=2734 dedup_ratio=0.041718 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 181, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8154, 8052, 7870, 5958, 8192, 8192] dup_rows_per_dst=[2558, 0, 0, 4, 0, 150, 0, 22] combine_rows_per_src=[8192, 8192, 8154, 8052, 7870, 5958, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64060 payload_rows_remote=55868 dup_rows=1476 dup_rows_remote=1186 dedup_ratio=0.022522 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8104, 8192, 8186, 6880, 8188, 8126] dup_rows_per_dst=[290, 0, 544, 194, 448, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8104, 8192, 8186, 6880, 8188, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=52978 payload_rows_remote=44792 dup_rows=12558 dup_rows_remote=10786 dedup_ratio=0.191620 router_tokens_per_expert=[0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1699, 0, 0, 115, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 0, 0, 0, 0, 0, 4046] payload_rows_per_dst=[5406, 3400, 3268, 8192, 8192, 8192, 8186, 8142] dup_rows_per_dst=[0, 0, 190, 6, 86, 8182, 1772, 2322] combine_rows_per_src=[5406, 3400, 3268, 8192, 8192, 8192, 8186, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59194 payload_rows_remote=51010 dup_rows=6342 dup_rows_remote=6334 dedup_ratio=0.096771 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 4092, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4, 0, 0, 4, 0, 0, 0, 0, 0, 3674, 0, 4, 0, 0, 18, 4012, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2256, 8118, 8004, 8178, 8140, 8176, 8138, 8184] dup_rows_per_dst=[50, 8, 0, 66, 8, 6130, 72, 8] combine_rows_per_src=[2256, 8118, 8004, 8178, 8140, 8176, 8138, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60008 payload_rows_remote=51816 dup_rows=5528 dup_rows_remote=5528 dedup_ratio=0.084351 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2146, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0] payload_rows_per_dst=[5868, 5008, 8182, 8192, 8192, 8192, 8182, 8192] dup_rows_per_dst=[0, 6, 0, 6, 0, 4300, 0, 1216] combine_rows_per_src=[5868, 5008, 8182, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229824 combine_payload_bytes=234741760 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234971584 ingress_bytes=329113600 link_bytes=329113600 token_bytes=469713344 weight_movement_bytes=94371840 total_nvlink_bytes=564085184 weight_to_token_ratio=0.200914 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=29056 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.153404581e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=223682560 dispatch_meta_bytes=229376 combine_payload_bytes=223682560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223911936 ingress_bytes=242556928 link_bytes=242556928 token_bytes=447594496 weight_movement_bytes=18874368 total_nvlink_bytes=466468864 weight_to_token_ratio=0.042168 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.324061139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228835328 dispatch_meta_bytes=228216 combine_payload_bytes=228835328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229063544 ingress_bytes=247709696 link_bytes=247709696 token_bytes=457898872 weight_movement_bytes=18874368 total_nvlink_bytes=476773240 weight_to_token_ratio=0.041220 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.373432426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=183468032 dispatch_meta_bytes=222312 combine_payload_bytes=183468032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=183690344 ingress_bytes=202342400 link_bytes=202342400 token_bytes=367158376 weight_movement_bytes=18874368 total_nvlink_bytes=386032744 weight_to_token_ratio=0.051407 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.938745318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=208936960 dispatch_meta_bytes=229376 combine_payload_bytes=208936960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=209166336 ingress_bytes=227811328 link_bytes=227811328 token_bytes=418103296 weight_movement_bytes=18874368 total_nvlink_bytes=436977664 weight_to_token_ratio=0.045143 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.182776055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59416 payload_rows_remote=51250 dup_rows=6120 dup_rows_remote=6120 dedup_ratio=0.093384 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 0] payload_rows_per_dst=[8192, 8108, 8166, 8192, 8192, 2966, 8142, 7458] dup_rows_per_dst=[4854, 0, 0, 22, 32, 520, 0, 692] combine_rows_per_src=[8192, 8108, 8166, 8192, 8192, 2966, 8142, 7458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=212238336 dispatch_meta_bytes=229376 combine_payload_bytes=212238336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212467712 ingress_bytes=231112704 link_bytes=231112704 token_bytes=424706048 weight_movement_bytes=18874368 total_nvlink_bytes=443580416 weight_to_token_ratio=0.044441 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.214408215e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58122 payload_rows_remote=49930 dup_rows=7414 dup_rows_remote=7398 dedup_ratio=0.113129 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0] payload_rows_per_dst=[8186, 7820, 8174, 8192, 8190, 1198, 8170, 8192] dup_rows_per_dst=[6884, 0, 0, 16, 100, 6, 0, 408] combine_rows_per_src=[8186, 7820, 8174, 8192, 8190, 1198, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=209920000 dispatch_meta_bytes=229480 combine_payload_bytes=209920000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210149480 ingress_bytes=228794368 link_bytes=228794368 token_bytes=420069480 weight_movement_bytes=18874368 total_nvlink_bytes=438943848 weight_to_token_ratio=0.044932 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.192195060e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=204513280 dispatch_meta_bytes=229312 combine_payload_bytes=204513280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=204742592 ingress_bytes=242262016 link_bytes=242262016 token_bytes=409255872 weight_movement_bytes=37748736 total_nvlink_bytes=447004608 weight_to_token_ratio=0.092237 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.321235437e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58390 payload_rows_remote=50198 dup_rows=7146 dup_rows_remote=4944 dedup_ratio=0.109039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2428, 0] payload_rows_per_dst=[1330, 7916, 8190, 8192, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 2, 86, 2202, 0, 4856] combine_rows_per_src=[1330, 7916, 8190, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60556 payload_rows_remote=52396 dup_rows=4980 dup_rows_remote=4846 dedup_ratio=0.075989 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 0] payload_rows_per_dst=[8192, 8160, 8162, 8192, 8192, 8082, 8104, 3472] dup_rows_per_dst=[2048, 134, 0, 18, 34, 2746, 0, 0] combine_rows_per_src=[8192, 8160, 8162, 8192, 8192, 8082, 8104, 3472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=50712 payload_rows_remote=42520 dup_rows=14824 dup_rows_remote=7938 dedup_ratio=0.226196 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0] payload_rows_per_dst=[8192, 8192, 8192, 8100, 8192, 1470, 8174, 200] dup_rows_per_dst=[6886, 7808, 110, 0, 20, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8100, 8192, 1470, 8174, 200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56910 payload_rows_remote=48718 dup_rows=8626 dup_rows_remote=5926 dedup_ratio=0.131622 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3364, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5700, 2112, 8192, 8192, 8160, 8170, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 4, 2, 5916, 2700, 0] combine_rows_per_src=[5700, 2112, 8192, 8192, 8160, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57868 payload_rows_remote=49676 dup_rows=7668 dup_rows_remote=4592 dedup_ratio=0.117004 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2074, 0, 0, 0, 0, 0, 4, 0, 0, 50, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1532, 0] payload_rows_per_dst=[4832, 4264, 8174, 8192, 7858, 8192, 8164, 8192] dup_rows_per_dst=[0, 8, 0, 14, 0, 4248, 322, 3076] combine_rows_per_src=[4832, 4264, 8174, 8192, 7858, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62034 payload_rows_remote=53842 dup_rows=3502 dup_rows_remote=3448 dedup_ratio=0.053436 router_tokens_per_expert=[0, 0, 0, 0, 2382, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 1498, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 4093, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 4090, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[4764, 8134, 8192, 8186, 8192, 8190, 8184, 8192] dup_rows_per_dst=[0, 2606, 6, 4, 54, 832, 0, 0] combine_rows_per_src=[4764, 8134, 8192, 8186, 8192, 8190, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=205611008 dispatch_meta_bytes=220568 combine_payload_bytes=205611008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=205831576 ingress_bytes=243359744 link_bytes=243359744 token_bytes=411442584 weight_movement_bytes=37748736 total_nvlink_bytes=449191320 weight_to_token_ratio=0.091747 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.331753327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=214614016 dispatch_meta_bytes=228968 combine_payload_bytes=214614016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214842984 ingress_bytes=233488384 link_bytes=233488384 token_bytes=429457000 weight_movement_bytes=18874368 total_nvlink_bytes=448331368 weight_to_token_ratio=0.043949 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.237170812e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=174161920 dispatch_meta_bytes=201832 combine_payload_bytes=174161920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=174363752 ingress_bytes=193036288 link_bytes=193036288 token_bytes=348525672 weight_movement_bytes=18874368 total_nvlink_bytes=367400040 weight_to_token_ratio=0.054155 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.849578732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=199548928 dispatch_meta_bytes=218576 combine_payload_bytes=199548928 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199767504 ingress_bytes=256172032 link_bytes=256172032 token_bytes=399316432 weight_movement_bytes=56623104 total_nvlink_bytes=455939536 weight_to_token_ratio=0.141800 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.454514366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=203472896 dispatch_meta_bytes=217072 combine_payload_bytes=203472896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=203689968 ingress_bytes=260096000 link_bytes=260096000 token_bytes=407162864 weight_movement_bytes=56623104 total_nvlink_bytes=463785968 weight_to_token_ratio=0.139067 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=23424 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.492111897e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65154 payload_rows_remote=56962 dup_rows=382 dup_rows_remote=382 dedup_ratio=0.005829 router_tokens_per_expert=[0, 0, 0, 0, 3934, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 4031, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[7868, 8160, 8192, 8192, 8192, 8172, 8192, 8186] dup_rows_per_dst=[0, 256, 0, 14, 38, 74, 0, 0] combine_rows_per_src=[7868, 8160, 8192, 8192, 8192, 8172, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220536832 dispatch_meta_bytes=229160 combine_payload_bytes=220536832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220765992 ingress_bytes=239411200 link_bytes=239411200 token_bytes=441302824 weight_movement_bytes=18874368 total_nvlink_bytes=460177192 weight_to_token_ratio=0.042770 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.293920321e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62238 payload_rows_remote=54046 dup_rows=3298 dup_rows_remote=3278 dedup_ratio=0.050323 router_tokens_per_expert=[0, 0, 0, 0, 2535, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0] payload_rows_per_dst=[5070, 8040, 8190, 8192, 8192, 8180, 8192, 8182] dup_rows_per_dst=[0, 1286, 74, 20, 1750, 168, 0, 0] combine_rows_per_src=[5070, 8040, 8190, 8192, 8192, 8180, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233316352 dispatch_meta_bytes=229376 combine_payload_bytes=233316352 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233545728 ingress_bytes=271065088 link_bytes=271065088 token_bytes=466862080 weight_movement_bytes=37748736 total_nvlink_bytes=504610816 weight_to_token_ratio=0.080856 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.597212301e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65088 payload_rows_remote=56896 dup_rows=448 dup_rows_remote=270 dedup_ratio=0.006836 router_tokens_per_expert=[0, 0, 0, 0, 3903, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 86, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[7806, 8132, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 46, 52, 0, 172, 178, 0, 0] combine_rows_per_src=[7806, 8132, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=221372416 dispatch_meta_bytes=229296 combine_payload_bytes=221372416 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=221601712 ingress_bytes=277995520 link_bytes=277995520 token_bytes=442974128 weight_movement_bytes=56623104 total_nvlink_bytes=499597232 weight_to_token_ratio=0.127825 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.663616290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65226 payload_rows_remote=57190 dup_rows=310 dup_rows_remote=276 dedup_ratio=0.004730 router_tokens_per_expert=[0, 0, 0, 0, 4038, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 3930, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0] payload_rows_per_dst=[8076, 8036, 8192, 8192, 8192, 8180, 8190, 8168] dup_rows_per_dst=[0, 34, 0, 26, 0, 250, 0, 0] combine_rows_per_src=[8076, 8036, 8192, 8192, 8192, 8180, 8190, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54466 payload_rows_remote=46284 dup_rows=11070 dup_rows_remote=244 dedup_ratio=0.168915 router_tokens_per_expert=[0, 0, 0, 0, 3777, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8182, 3030, 8192, 8192, 4996, 5490, 8192, 8192] dup_rows_per_dst=[10826, 134, 4, 0, 2, 104, 0, 0] combine_rows_per_src=[8182, 3030, 8192, 8192, 4996, 5490, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65138 payload_rows_remote=56946 dup_rows=398 dup_rows_remote=398 dedup_ratio=0.006073 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8184, 8186, 7840, 8192, 8192] dup_rows_per_dst=[384, 0, 0, 0, 0, 10, 0, 4] combine_rows_per_src=[8192, 8192, 8160, 8184, 8186, 7840, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61306 payload_rows_remote=53116 dup_rows=4230 dup_rows_remote=4182 dedup_ratio=0.064545 router_tokens_per_expert=[0, 0, 0, 0, 2387, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 1280, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 4092, 0, 0, 4092, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 4061, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4091, 0] payload_rows_per_dst=[4774, 7672, 8050, 8110, 8192, 8142, 8176, 8190] dup_rows_per_dst=[0, 1782, 8, 20, 1154, 1192, 26, 48] combine_rows_per_src=[4774, 7672, 8050, 8110, 8192, 8142, 8176, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61836 payload_rows_remote=53644 dup_rows=3700 dup_rows_remote=3696 dedup_ratio=0.056458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 30, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3, 0, 180, 0, 0, 14, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 48, 4091, 0, 3691, 4016, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[5294, 8170, 8092, 8190, 8192, 8182, 7598, 8118] dup_rows_per_dst=[140, 0, 2994, 384, 4, 0, 162, 16] combine_rows_per_src=[5294, 8170, 8092, 8190, 8192, 8182, 7598, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=233046016 dispatch_meta_bytes=228664 combine_payload_bytes=233046016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233274680 ingress_bytes=270794752 link_bytes=270794752 token_bytes=466320696 weight_movement_bytes=37748736 total_nvlink_bytes=504069432 weight_to_token_ratio=0.080950 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594622074e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234250240 dispatch_meta_bytes=229864 combine_payload_bytes=234250240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234480104 ingress_bytes=271998976 link_bytes=271998976 token_bytes=468730344 weight_movement_bytes=37748736 total_nvlink_bytes=506479080 weight_to_token_ratio=0.080534 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.606160356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=189579264 dispatch_meta_bytes=186112 combine_payload_bytes=189579264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=189765376 ingress_bytes=227328000 link_bytes=227328000 token_bytes=379344640 weight_movement_bytes=37748736 total_nvlink_bytes=417093376 weight_to_token_ratio=0.099510 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=27136 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.178145044e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233250816 dispatch_meta_bytes=229376 combine_payload_bytes=233250816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233480192 ingress_bytes=252125184 link_bytes=252125184 token_bytes=466731008 weight_movement_bytes=18874368 total_nvlink_bytes=485605376 weight_to_token_ratio=0.040439 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.415739460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=217563136 dispatch_meta_bytes=229192 combine_payload_bytes=217563136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217792328 ingress_bytes=236437504 link_bytes=236437504 token_bytes=435355464 weight_movement_bytes=18874368 total_nvlink_bytes=454229832 weight_to_token_ratio=0.043354 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.265427829e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60648 payload_rows_remote=56184 dup_rows=4888 dup_rows_remote=4668 dedup_ratio=0.074585 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 112, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 247, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4080, 3618, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8046, 4464, 8192, 8192, 8166, 8160, 7236] dup_rows_per_dst=[3698, 0, 220, 494, 0, 0, 476, 0] combine_rows_per_src=[8192, 8046, 4464, 8192, 8192, 8166, 8160, 7236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219725824 dispatch_meta_bytes=229360 combine_payload_bytes=219725824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219955184 ingress_bytes=238600192 link_bytes=238600192 token_bytes=439681008 weight_movement_bytes=18874368 total_nvlink_bytes=458555376 weight_to_token_ratio=0.042927 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286149641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61896 payload_rows_remote=53704 dup_rows=3640 dup_rows_remote=3350 dedup_ratio=0.055542 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2371, 30, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 145, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 4075, 0, 4086, 4042, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8152, 4742, 8192, 8192, 8156, 8184, 8092] dup_rows_per_dst=[3228, 0, 60, 290, 12, 0, 50, 0] combine_rows_per_src=[8186, 8152, 4742, 8192, 8192, 8156, 8184, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=230129664 dispatch_meta_bytes=243408 combine_payload_bytes=230129664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=230373072 ingress_bytes=286752768 link_bytes=286752768 token_bytes=460502736 weight_movement_bytes=56623104 total_nvlink_bytes=517125840 weight_to_token_ratio=0.122959 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.747523932e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60142 payload_rows_remote=51960 dup_rows=5394 dup_rows_remote=5394 dedup_ratio=0.082306 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4008, 3824, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3526, 8170, 8192, 8192, 8192, 8182, 8040, 7648] dup_rows_per_dst=[0, 0, 4492, 140, 0, 0, 762, 0] combine_rows_per_src=[3526, 8170, 8192, 8192, 8192, 8182, 8040, 7648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219971584 dispatch_meta_bytes=228216 combine_payload_bytes=219971584 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=220199800 ingress_bytes=257720320 link_bytes=257720320 token_bytes=440171384 weight_movement_bytes=37748736 total_nvlink_bytes=477920120 weight_to_token_ratio=0.085759 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.469349300e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59220 payload_rows_remote=51998 dup_rows=6316 dup_rows_remote=6316 dedup_ratio=0.096375 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 535, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 29, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 4012, 2876, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7222, 5480, 8166, 8192, 8172, 8046, 5752] dup_rows_per_dst=[4546, 0, 1040, 92, 0, 0, 638, 0] combine_rows_per_src=[8190, 7222, 5480, 8166, 8192, 8172, 8046, 5752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59142 payload_rows_remote=50950 dup_rows=6394 dup_rows_remote=2536 dedup_ratio=0.097565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2128, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4028, 4077, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4834, 5482, 8176, 8186, 8056, 8024] dup_rows_per_dst=[3858, 1282, 984, 20, 0, 124, 126, 0] combine_rows_per_src=[8192, 8192, 4834, 5482, 8176, 8186, 8056, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57230 payload_rows_remote=49138 dup_rows=8306 dup_rows_remote=8306 dedup_ratio=0.126740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0] payload_rows_per_dst=[72, 8112, 8186, 8192, 8192, 8192, 8092, 8192] dup_rows_per_dst=[0, 0, 0, 14, 64, 5030, 0, 3198] combine_rows_per_src=[72, 8112, 8186, 8192, 8192, 8192, 8092, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59168 payload_rows_remote=51088 dup_rows=6368 dup_rows_remote=1806 dedup_ratio=0.097168 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 111, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4, 0, 636, 0, 0, 32, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 36, 4091, 0, 3997, 3886, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[3868, 8128, 8192, 8188, 8192, 8182, 6338, 8080] dup_rows_per_dst=[0, 0, 212, 1304, 0, 0, 290, 4562] combine_rows_per_src=[3868, 8128, 8192, 8188, 8192, 8182, 6338, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61420 payload_rows_remote=53242 dup_rows=4116 dup_rows_remote=4110 dedup_ratio=0.062805 router_tokens_per_expert=[0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4093, 3, 974, 0, 0, 0, 3, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 562, 0, 0, 3, 0, 0, 0, 0, 0, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2477, 0, 0] payload_rows_per_dst=[8192, 8192, 7986, 8156, 8178, 7972, 7790, 4954] dup_rows_per_dst=[6, 1966, 1124, 0, 6, 0, 1014, 0] combine_rows_per_src=[8192, 8192, 7986, 8156, 8178, 7972, 7790, 4954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=212828160 dispatch_meta_bytes=229416 combine_payload_bytes=212828160 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=213057576 ingress_bytes=250576896 link_bytes=250576896 token_bytes=425885736 weight_movement_bytes=37748736 total_nvlink_bytes=463634472 weight_to_token_ratio=0.088636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.400904526e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=212983808 dispatch_meta_bytes=233256 combine_payload_bytes=212983808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213217064 ingress_bytes=250732544 link_bytes=250732544 token_bytes=426200872 weight_movement_bytes=37748736 total_nvlink_bytes=463949608 weight_to_token_ratio=0.088570 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402395868e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=208691200 dispatch_meta_bytes=213944 combine_payload_bytes=208691200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208905144 ingress_bytes=227565568 link_bytes=227565568 token_bytes=417596344 weight_movement_bytes=18874368 total_nvlink_bytes=436470712 weight_to_token_ratio=0.045198 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.180421303e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=201269248 dispatch_meta_bytes=229776 combine_payload_bytes=201269248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=201499024 ingress_bytes=257892352 link_bytes=257892352 token_bytes=402768272 weight_movement_bytes=56623104 total_nvlink_bytes=459391376 weight_to_token_ratio=0.140585 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.470997626e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64154 payload_rows_remote=56002 dup_rows=1382 dup_rows_remote=1328 dedup_ratio=0.021088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 4096, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3996, 0, 0] payload_rows_per_dst=[8192, 8192, 8152, 8064, 8192, 7204, 8166, 7992] dup_rows_per_dst=[0, 1184, 54, 0, 0, 0, 144, 0] combine_rows_per_src=[8192, 8192, 8152, 8064, 8192, 7204, 8166, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=218079232 dispatch_meta_bytes=229408 combine_payload_bytes=218079232 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=218308640 ingress_bytes=255827968 link_bytes=255827968 token_bytes=436387872 weight_movement_bytes=37748736 total_nvlink_bytes=474136608 weight_to_token_ratio=0.086503 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.451217714e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=209256448 dispatch_meta_bytes=211576 combine_payload_bytes=209256448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=209468024 ingress_bytes=284753920 link_bytes=284753920 token_bytes=418724472 weight_movement_bytes=75497472 total_nvlink_bytes=494221944 weight_to_token_ratio=0.180303 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.728371954e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63568 payload_rows_remote=55410 dup_rows=1968 dup_rows_remote=1968 dedup_ratio=0.030029 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4062, 0, 0] payload_rows_per_dst=[8192, 8192, 7696, 8158, 8192, 6828, 8186, 8124] dup_rows_per_dst=[0, 196, 8, 0, 2, 0, 1762, 0] combine_rows_per_src=[8192, 8192, 7696, 8158, 8192, 6828, 8186, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=229384192 dispatch_meta_bytes=229320 combine_payload_bytes=229384192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229613512 ingress_bytes=267132928 link_bytes=267132928 token_bytes=458997704 weight_movement_bytes=37748736 total_nvlink_bytes=496746440 weight_to_token_ratio=0.082242 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.559536278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60326 payload_rows_remote=52510 dup_rows=5210 dup_rows_remote=5210 dedup_ratio=0.079498 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2623, 0, 0] payload_rows_per_dst=[8192, 6860, 8162, 8122, 8190, 7816, 7738, 5246] dup_rows_per_dst=[0, 2922, 576, 1548, 0, 0, 164, 0] combine_rows_per_src=[8192, 6860, 8162, 8122, 8190, 7816, 7738, 5246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=226959360 dispatch_meta_bytes=229512 combine_payload_bytes=226959360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=227188872 ingress_bytes=264708096 link_bytes=264708096 token_bytes=454148232 weight_movement_bytes=37748736 total_nvlink_bytes=491896968 weight_to_token_ratio=0.083120 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.536302731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60530 payload_rows_remote=52338 dup_rows=5006 dup_rows_remote=140 dedup_ratio=0.076385 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 0, 4096, 0, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 3184, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3059, 0, 0] payload_rows_per_dst=[8192, 8192, 8024, 6368, 8192, 7704, 7740, 6118] dup_rows_per_dst=[0, 4866, 128, 0, 0, 0, 12, 0] combine_rows_per_src=[8192, 8192, 8024, 6368, 8192, 7704, 7740, 6118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60054 payload_rows_remote=51862 dup_rows=5482 dup_rows_remote=5482 dedup_ratio=0.083649 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0] payload_rows_per_dst=[8192, 8192, 8104, 8188, 8192, 8182, 2846, 8158] dup_rows_per_dst=[0, 68, 4, 0, 8, 5328, 74, 0] combine_rows_per_src=[8192, 8192, 8104, 8188, 8192, 8182, 2846, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65482 payload_rows_remote=57290 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8170, 8184, 8192, 8192, 8192, 8172, 8192, 8188] dup_rows_per_dst=[0, 4, 0, 4, 26, 20, 0, 0] combine_rows_per_src=[8170, 8184, 8192, 8192, 8192, 8172, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65160 payload_rows_remote=57262 dup_rows=376 dup_rows_remote=370 dedup_ratio=0.005737 router_tokens_per_expert=[0, 0, 0, 127, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3946, 0, 0, 0, 0, 4093, 4093, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 3, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 7898, 8186, 8192, 8120] dup_rows_per_dst=[254, 0, 0, 0, 6, 116, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 7898, 8186, 8192, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56076 payload_rows_remote=47892 dup_rows=9460 dup_rows_remote=5386 dedup_ratio=0.144348 router_tokens_per_expert=[0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4092, 4, 1082, 0, 0, 1, 4, 0, 0, 0, 0, 0, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 0, 0, 0, 0, 0, 0, 725, 0, 0, 4, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2037, 0, 0] payload_rows_per_dst=[8178, 2196, 5796, 8110, 8188, 7296, 8128, 8184] dup_rows_per_dst=[8, 68, 1548, 14, 8, 0, 3740, 4074] combine_rows_per_src=[8178, 2196, 5796, 8110, 8188, 7296, 8128, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=215080960 dispatch_meta_bytes=230880 combine_payload_bytes=215080960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215311840 ingress_bytes=233955328 link_bytes=233955328 token_bytes=430392800 weight_movement_bytes=18874368 total_nvlink_bytes=449267168 weight_to_token_ratio=0.043854 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.241644840e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=214376448 dispatch_meta_bytes=209912 combine_payload_bytes=214376448 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=214586360 ingress_bytes=270999552 link_bytes=270999552 token_bytes=428962808 weight_movement_bytes=56623104 total_nvlink_bytes=485585912 weight_to_token_ratio=0.132000 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596584367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=212426752 dispatch_meta_bytes=229376 combine_payload_bytes=212426752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212656128 ingress_bytes=231301120 link_bytes=231301120 token_bytes=425082880 weight_movement_bytes=18874368 total_nvlink_bytes=443957248 weight_to_token_ratio=0.044402 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216213525e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=229376 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234889216 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469549056 weight_movement_bytes=18874368 total_nvlink_bytes=488423424 weight_to_token_ratio=0.040197 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57112 dup_rows=232 dup_rows_remote=232 dedup_ratio=0.003540 router_tokens_per_expert=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 7962] dup_rows_per_dst=[16, 14, 0, 0, 0, 202, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 8192, 7962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234545152 dispatch_meta_bytes=230528 combine_payload_bytes=234545152 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234775680 ingress_bytes=328916992 link_bytes=328916992 token_bytes=469320832 weight_movement_bytes=94371840 total_nvlink_bytes=563692672 weight_to_token_ratio=0.201082 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.151520780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=196165632 dispatch_meta_bytes=213112 combine_payload_bytes=196165632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=196378744 ingress_bytes=252788736 link_bytes=252788736 token_bytes=392544376 weight_movement_bytes=56623104 total_nvlink_bytes=449167480 weight_to_token_ratio=0.144246 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422097289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65380 payload_rows_remote=57198 dup_rows=156 dup_rows_remote=156 dedup_ratio=0.002380 router_tokens_per_expert=[0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 20, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4033, 0, 0] payload_rows_per_dst=[8192, 8182, 8192, 8182, 8182, 8192, 8190, 8068] dup_rows_per_dst=[22, 4, 2, 0, 0, 128, 0, 0] combine_rows_per_src=[8192, 8182, 8192, 8182, 8182, 8192, 8190, 8068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233930752 dispatch_meta_bytes=229376 combine_payload_bytes=233930752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234160128 ingress_bytes=252805120 link_bytes=252805120 token_bytes=468090880 weight_movement_bytes=18874368 total_nvlink_bytes=486965248 weight_to_token_ratio=0.040322 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.422254272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=229416 combine_payload_bytes=234283008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234512424 ingress_bytes=253157376 link_bytes=253157376 token_bytes=468795432 weight_movement_bytes=18874368 total_nvlink_bytes=487669800 weight_to_token_ratio=0.040261 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425629416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65494 payload_rows_remote=57302 dup_rows=42 dup_rows_remote=36 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0] payload_rows_per_dst=[8190, 8188, 8192, 8192, 8192, 8192, 8192, 8156] dup_rows_per_dst=[0, 18, 0, 18, 0, 6, 0, 0] combine_rows_per_src=[8190, 8188, 8192, 8192, 8192, 8192, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64242 payload_rows_remote=56050 dup_rows=1294 dup_rows_remote=1294 dedup_ratio=0.019745 router_tokens_per_expert=[0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8168, 8192, 8186, 6928] dup_rows_per_dst=[38, 0, 4, 0, 0, 1252, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8168, 8192, 8186, 6928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63520 payload_rows_remote=55328 dup_rows=2016 dup_rows_remote=2012 dedup_ratio=0.030762 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 6258, 8192, 8110] dup_rows_per_dst=[4, 46, 0, 18, 1948, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 6258, 8192, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60048 payload_rows_remote=52458 dup_rows=5488 dup_rows_remote=4468 dedup_ratio=0.083740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 107, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3679, 3355, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4618, 8174, 8192, 8192, 8192, 8186, 7590, 6904] dup_rows_per_dst=[0, 0, 3346, 18, 0, 0, 1020, 1104] combine_rows_per_src=[4618, 8174, 8192, 8192, 8192, 8186, 7590, 6904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65168 payload_rows_remote=57230 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.005615 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8186, 8188, 7938, 8186, 8098, 8192] dup_rows_per_dst=[132, 0, 0, 6, 0, 154, 0, 76] combine_rows_per_src=[8192, 8188, 8186, 8188, 7938, 8186, 8098, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62624 payload_rows_remote=54814 dup_rows=2912 dup_rows_remote=544 dedup_ratio=0.044434 router_tokens_per_expert=[0, 0, 0, 166, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4091, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4051, 0, 0, 0, 0, 4092, 4091, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 4, 0] payload_rows_per_dst=[7516, 8106, 8182, 8132, 6502, 8184, 8192, 7810] dup_rows_per_dst=[328, 0, 0, 30, 70, 108, 8, 2368] combine_rows_per_src=[7516, 8106, 8182, 8132, 6502, 8184, 8192, 7810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234708992 dispatch_meta_bytes=229352 combine_payload_bytes=234708992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234938344 ingress_bytes=253583360 link_bytes=253583360 token_bytes=469647336 weight_movement_bytes=18874368 total_nvlink_bytes=488521704 weight_to_token_ratio=0.040188 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429710985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=229580800 dispatch_meta_bytes=229376 combine_payload_bytes=229580800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=229810176 ingress_bytes=286203904 link_bytes=286203904 token_bytes=459390976 weight_movement_bytes=56623104 total_nvlink_bytes=516014080 weight_to_token_ratio=0.123257 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.742264987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=226623488 dispatch_meta_bytes=229360 combine_payload_bytes=226623488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226852848 ingress_bytes=245497856 link_bytes=245497856 token_bytes=453476336 weight_movement_bytes=18874368 total_nvlink_bytes=472350704 weight_to_token_ratio=0.041622 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352239664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=214867968 dispatch_meta_bytes=227704 combine_payload_bytes=214867968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215095672 ingress_bytes=233742336 link_bytes=233742336 token_bytes=429963640 weight_movement_bytes=18874368 total_nvlink_bytes=448838008 weight_to_token_ratio=0.043898 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239604055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65376 payload_rows_remote=57184 dup_rows=160 dup_rows_remote=160 dedup_ratio=0.002441 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4046, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8056, 8192, 8172, 8188, 8192, 8192, 8192] dup_rows_per_dst=[28, 0, 0, 0, 0, 42, 0, 90] combine_rows_per_src=[8192, 8056, 8192, 8172, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234414080 dispatch_meta_bytes=230392 combine_payload_bytes=234414080 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=234644472 ingress_bytes=328785920 link_bytes=328785920 token_bytes=469058552 weight_movement_bytes=94371840 total_nvlink_bytes=563430392 weight_to_token_ratio=0.201194 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.150264913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=224518144 dispatch_meta_bytes=221432 combine_payload_bytes=224518144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224739576 ingress_bytes=243392512 link_bytes=243392512 token_bytes=449257720 weight_movement_bytes=18874368 total_nvlink_bytes=468132088 weight_to_token_ratio=0.042012 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332067293e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63554 payload_rows_remote=55388 dup_rows=1982 dup_rows_remote=1982 dedup_ratio=0.030243 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8110, 8192, 8166, 6318, 8192, 8192, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 30, 0, 1938] combine_rows_per_src=[8192, 8110, 8192, 8166, 6318, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234225664 dispatch_meta_bytes=229376 combine_payload_bytes=234225664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234455040 ingress_bytes=253100032 link_bytes=253100032 token_bytes=468680704 weight_movement_bytes=18874368 total_nvlink_bytes=487555072 weight_to_token_ratio=0.040271 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425079974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=226869248 dispatch_meta_bytes=229480 combine_payload_bytes=226869248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=227098728 ingress_bytes=283492352 link_bytes=283492352 token_bytes=453967976 weight_movement_bytes=56623104 total_nvlink_bytes=510591080 weight_to_token_ratio=0.124729 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.716284230e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=68 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4092, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8124, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 6, 62] combine_rows_per_src=[8192, 8192, 8192, 8192, 8124, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65152 payload_rows_remote=57082 dup_rows=384 dup_rows_remote=384 dedup_ratio=0.005859 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8070, 8192, 7934, 8192, 8192, 8192, 8188] dup_rows_per_dst=[34, 0, 0, 0, 0, 90, 0, 260] combine_rows_per_src=[8192, 8070, 8192, 7934, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62562 payload_rows_remote=54370 dup_rows=2974 dup_rows_remote=2972 dedup_ratio=0.045380 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8192, 7876, 8192, 5550] dup_rows_per_dst=[2, 506, 14, 0, 2452, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8176, 8192, 7876, 8192, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59996 payload_rows_remote=51964 dup_rows=5540 dup_rows_remote=5540 dedup_ratio=0.084534 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4096, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2738, 0, 0] payload_rows_per_dst=[8192, 4206, 8158, 7896, 8188, 7206, 8032, 8118] dup_rows_per_dst=[0, 44, 16, 912, 0, 0, 0, 4568] combine_rows_per_src=[8192, 4206, 8158, 7896, 8188, 7206, 8032, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58346 payload_rows_remote=50166 dup_rows=7190 dup_rows_remote=928 dedup_ratio=0.109711 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 4093, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4068, 0, 0, 0, 2, 0, 410, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[8190, 2254, 6984, 8192, 8180, 8188, 8172, 8186] dup_rows_per_dst=[0, 0, 858, 58, 6262, 0, 0, 12] combine_rows_per_src=[8190, 2254, 6984, 8192, 8180, 8188, 8172, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64710 payload_rows_remote=56520 dup_rows=826 dup_rows_remote=756 dedup_ratio=0.012604 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4072, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 4079, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[7834, 8190, 8184, 8152, 7940, 8184, 8036, 8190] dup_rows_per_dst=[144, 0, 0, 362, 0, 6, 244, 70] combine_rows_per_src=[7834, 8190, 8184, 8152, 7940, 8184, 8036, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234594304 dispatch_meta_bytes=229368 combine_payload_bytes=234594304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234823672 ingress_bytes=253468672 link_bytes=253468672 token_bytes=469417976 weight_movement_bytes=18874368 total_nvlink_bytes=488292344 weight_to_token_ratio=0.040208 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428612101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=233807872 dispatch_meta_bytes=229864 combine_payload_bytes=233807872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234037736 ingress_bytes=252682240 link_bytes=252682240 token_bytes=467845608 weight_movement_bytes=18874368 total_nvlink_bytes=486719976 weight_to_token_ratio=0.040343 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.421076896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=222699520 dispatch_meta_bytes=229368 combine_payload_bytes=222699520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=222928888 ingress_bytes=241573888 link_bytes=241573888 token_bytes=445628408 weight_movement_bytes=18874368 total_nvlink_bytes=464502776 weight_to_token_ratio=0.042354 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.314642133e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58524 payload_rows_remote=51698 dup_rows=7012 dup_rows_remote=2886 dedup_ratio=0.106995 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 53, 0, 8, 0, 0, 1811, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7208, 6826, 8192, 3636, 8192, 8192, 8086] dup_rows_per_dst=[0, 2752, 4126, 24, 108, 2, 0, 0] combine_rows_per_src=[8192, 7208, 6826, 8192, 3636, 8192, 8192, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=212844544 dispatch_meta_bytes=230016 combine_payload_bytes=212844544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=213074560 ingress_bytes=250593280 link_bytes=250593280 token_bytes=425919104 weight_movement_bytes=37748736 total_nvlink_bytes=463667840 weight_to_token_ratio=0.088629 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401061509e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=205479936 dispatch_meta_bytes=204376 combine_payload_bytes=205479936 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=205684312 ingress_bytes=280977408 link_bytes=280977408 token_bytes=411164248 weight_movement_bytes=75497472 total_nvlink_bytes=486661720 weight_to_token_ratio=0.183619 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=16256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.692187274e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=231505920 dispatch_meta_bytes=229104 combine_payload_bytes=231505920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231735024 ingress_bytes=250380288 link_bytes=250380288 token_bytes=463240944 weight_movement_bytes=18874368 total_nvlink_bytes=482115312 weight_to_token_ratio=0.040744 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399020725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57386 payload_rows_remote=49196 dup_rows=8150 dup_rows_remote=8108 dedup_ratio=0.124359 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4077, 0, 0, 0, 19, 0, 69, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7104, 8162, 8190, 1200, 8192, 8192, 8154] dup_rows_per_dst=[0, 0, 8062, 42, 44, 0, 2, 0] combine_rows_per_src=[8192, 7104, 8162, 8190, 1200, 8192, 8192, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=211755008 dispatch_meta_bytes=218336 combine_payload_bytes=211755008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=211973344 ingress_bytes=249503744 link_bytes=249503744 token_bytes=423728352 weight_movement_bytes=37748736 total_nvlink_bytes=461477088 weight_to_token_ratio=0.089087 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.390622112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=201506816 dispatch_meta_bytes=229216 combine_payload_bytes=201506816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201736032 ingress_bytes=220381184 link_bytes=220381184 token_bytes=403242848 weight_movement_bytes=18874368 total_nvlink_bytes=422117216 weight_to_token_ratio=0.046806 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.111584071e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62726 payload_rows_remote=54534 dup_rows=2810 dup_rows_remote=2810 dedup_ratio=0.042877 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 7, 0, 0, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7070, 6518, 8178, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 2808, 0, 0, 0] combine_rows_per_src=[8192, 7070, 6518, 8178, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58734 payload_rows_remote=50580 dup_rows=6802 dup_rows_remote=1476 dedup_ratio=0.103790 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 0, 203, 0, 1, 0, 0, 2836, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 4456, 8192, 5672, 8192, 8192, 7684] dup_rows_per_dst=[0, 5326, 0, 1058, 408, 10, 0, 0] combine_rows_per_src=[8192, 8154, 4456, 8192, 5672, 8192, 8192, 7684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64308 payload_rows_remote=56116 dup_rows=1228 dup_rows_remote=1228 dedup_ratio=0.018738 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 31, 556] payload_rows_per_dst=[7180, 8056, 8192, 8122, 8192, 8186, 8188, 8192] dup_rows_per_dst=[0, 28, 6, 2, 0, 6, 12, 1174] combine_rows_per_src=[7180, 8056, 8192, 8122, 8192, 8186, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65352 payload_rows_remote=57160 dup_rows=184 dup_rows_remote=184 dedup_ratio=0.002808 router_tokens_per_expert=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8190, 8192, 8014] dup_rows_per_dst=[16, 22, 4, 4, 0, 138, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8190, 8192, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57174 payload_rows_remote=48982 dup_rows=8362 dup_rows_remote=8352 dedup_ratio=0.127594 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 4, 0, 1, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8150, 6922, 1152, 8184, 8192, 8190] dup_rows_per_dst=[10, 8344, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8150, 6922, 1152, 8184, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58598 payload_rows_remote=50428 dup_rows=6938 dup_rows_remote=6240 dedup_ratio=0.105865 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 964, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 4092, 0, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4071, 0, 0, 0, 19, 0, 350, 0, 0, 3068, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4081, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8184, 1936, 7556, 8192, 8184, 8192, 8184, 8170] dup_rows_per_dst=[0, 0, 8, 82, 6124, 0, 26, 698] combine_rows_per_src=[8184, 1936, 7556, 8192, 8184, 8192, 8184, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=223371264 dispatch_meta_bytes=229376 combine_payload_bytes=223371264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223600640 ingress_bytes=242245632 link_bytes=242245632 token_bytes=446971904 weight_movement_bytes=18874368 total_nvlink_bytes=465846272 weight_to_token_ratio=0.042227 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.321078454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63880 payload_rows_remote=55688 dup_rows=1656 dup_rows_remote=1656 dedup_ratio=0.025269 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 852] payload_rows_per_dst=[7160, 8192, 8192, 7802, 8104, 8190, 8178, 8062] dup_rows_per_dst=[272, 180, 0, 0, 0, 8, 0, 1196] combine_rows_per_src=[7160, 8192, 8192, 7802, 8104, 8190, 8178, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=207175680 dispatch_meta_bytes=208224 combine_payload_bytes=207175680 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=207383904 ingress_bytes=282673152 link_bytes=282673152 token_bytes=414559584 weight_movement_bytes=75497472 total_nvlink_bytes=490057056 weight_to_token_ratio=0.182115 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=22528 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.708435059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=229851136 dispatch_meta_bytes=229376 combine_payload_bytes=229851136 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230080512 ingress_bytes=305348608 link_bytes=305348608 token_bytes=459931648 weight_movement_bytes=75497472 total_nvlink_bytes=535429120 weight_to_token_ratio=0.164149 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.925700121e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=229376 combine_payload_bytes=234127360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234356736 ingress_bytes=253001728 link_bytes=253001728 token_bytes=468484096 weight_movement_bytes=18874368 total_nvlink_bytes=487358464 weight_to_token_ratio=0.040288 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424138073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=200630272 dispatch_meta_bytes=229336 combine_payload_bytes=200630272 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=200859608 ingress_bytes=238379008 link_bytes=238379008 token_bytes=401489880 weight_movement_bytes=37748736 total_nvlink_bytes=439238616 weight_to_token_ratio=0.094022 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.284030365e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62702 payload_rows_remote=54712 dup_rows=2834 dup_rows_remote=2826 dedup_ratio=0.043243 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 359, 806] payload_rows_per_dst=[5618, 8192, 8192, 7990, 8172, 8176, 8170, 8192] dup_rows_per_dst=[0, 484, 0, 8, 0, 4, 0, 2338] combine_rows_per_src=[5618, 8192, 8192, 7990, 8172, 8176, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=206553088 dispatch_meta_bytes=226672 combine_payload_bytes=206553088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206779760 ingress_bytes=225427456 link_bytes=225427456 token_bytes=413332848 weight_movement_bytes=18874368 total_nvlink_bytes=432207216 weight_to_token_ratio=0.045664 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159934966e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=228098048 dispatch_meta_bytes=229376 combine_payload_bytes=228098048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228327424 ingress_bytes=246972416 link_bytes=246972416 token_bytes=456425472 weight_movement_bytes=18874368 total_nvlink_bytes=475299840 weight_to_token_ratio=0.041353 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.366368172e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=224100352 dispatch_meta_bytes=230152 combine_payload_bytes=224100352 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=224330504 ingress_bytes=280723456 link_bytes=280723456 token_bytes=448430856 weight_movement_bytes=56623104 total_nvlink_bytes=505053960 weight_to_token_ratio=0.126269 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=23808 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.689754031e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63836 payload_rows_remote=55644 dup_rows=1700 dup_rows_remote=1700 dedup_ratio=0.025940 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 562] payload_rows_per_dst=[6518, 8190, 8192, 8168, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 64, 0, 0, 0, 0, 0, 1636] combine_rows_per_src=[6518, 8190, 8192, 8168, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62888 payload_rows_remote=55006 dup_rows=2648 dup_rows_remote=2648 dedup_ratio=0.040405 router_tokens_per_expert=[0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 3460, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3938, 0, 4091, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 957] payload_rows_per_dst=[7142, 8188, 6928, 8186, 7882, 8192, 8178, 8192] dup_rows_per_dst=[0, 6, 4, 718, 0, 0, 0, 1920] combine_rows_per_src=[7142, 8188, 6928, 8186, 7882, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60012 payload_rows_remote=51820 dup_rows=5524 dup_rows_remote=5430 dedup_ratio=0.084290 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 1169] payload_rows_per_dst=[8174, 8192, 8192, 8192, 8120, 8172, 8160, 2810] dup_rows_per_dst=[5380, 94, 0, 8, 4, 2, 0, 36] combine_rows_per_src=[8174, 8192, 8192, 8192, 8120, 8172, 8160, 2810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65372 payload_rows_remote=57180 dup_rows=164 dup_rows_remote=164 dedup_ratio=0.002502 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8032, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 6, 0, 154] combine_rows_per_src=[8192, 8032, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57494 payload_rows_remote=49312 dup_rows=8042 dup_rows_remote=878 dedup_ratio=0.122711 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 389] payload_rows_per_dst=[8182, 8192, 8192, 8192, 7514, 8174, 8192, 856] dup_rows_per_dst=[7164, 178, 0, 698, 0, 2, 0, 0] combine_rows_per_src=[8182, 8192, 8192, 8192, 7514, 8174, 8192, 856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62880 payload_rows_remote=54688 dup_rows=2656 dup_rows_remote=1046 dedup_ratio=0.040527 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4092, 1, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 4, 0, 4086, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 9, 796] payload_rows_per_dst=[6560, 7220, 8178, 8172, 8192, 8182, 8184, 8192] dup_rows_per_dst=[0, 70, 8, 4, 852, 40, 72, 1610] combine_rows_per_src=[6560, 7220, 8178, 8172, 8192, 8182, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=227917824 dispatch_meta_bytes=229376 combine_payload_bytes=227917824 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=228147200 ingress_bytes=265666560 link_bytes=265666560 token_bytes=456065024 weight_movement_bytes=37748736 total_nvlink_bytes=493813760 weight_to_token_ratio=0.082771 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.545486262e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=212254720 dispatch_meta_bytes=229000 combine_payload_bytes=212254720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212483720 ingress_bytes=231129088 link_bytes=231129088 token_bytes=424738440 weight_movement_bytes=18874368 total_nvlink_bytes=443612808 weight_to_token_ratio=0.044438 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.214565199e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=225304576 dispatch_meta_bytes=230616 combine_payload_bytes=225304576 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=225535192 ingress_bytes=319676416 link_bytes=319676416 token_bytes=450839768 weight_movement_bytes=94371840 total_nvlink_bytes=545211608 weight_to_token_ratio=0.209325 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.062982127e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65146 payload_rows_remote=56954 dup_rows=390 dup_rows_remote=368 dedup_ratio=0.005951 router_tokens_per_expert=[0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3912, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104] payload_rows_per_dst=[8188, 8192, 8192, 8192, 7824, 8192, 8174, 8192] dup_rows_per_dst=[0, 0, 22, 160, 0, 0, 0, 208] combine_rows_per_src=[8188, 8192, 8192, 8192, 7824, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=229376 combine_payload_bytes=234209280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234438656 ingress_bytes=253083648 link_bytes=253083648 token_bytes=468647936 weight_movement_bytes=18874368 total_nvlink_bytes=487522304 weight_to_token_ratio=0.040274 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424922990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65320 payload_rows_remote=57128 dup_rows=216 dup_rows_remote=192 dedup_ratio=0.003296 router_tokens_per_expert=[0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 3, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84] payload_rows_per_dst=[8162, 8192, 8192, 8192, 8028, 8192, 8170, 8192] dup_rows_per_dst=[0, 0, 24, 24, 0, 0, 0, 168] combine_rows_per_src=[8162, 8192, 8192, 8192, 8028, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=201981952 dispatch_meta_bytes=200760 combine_payload_bytes=201981952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202182712 ingress_bytes=220856320 link_bytes=220856320 token_bytes=404164664 weight_movement_bytes=18874368 total_nvlink_bytes=423039032 weight_to_token_ratio=0.046700 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24064 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.116136590e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=224002048 dispatch_meta_bytes=222936 combine_payload_bytes=224002048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224224984 ingress_bytes=242876416 link_bytes=242876416 token_bytes=448227032 weight_movement_bytes=18874368 total_nvlink_bytes=467101400 weight_to_token_ratio=0.042109 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.327122315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233283584 dispatch_meta_bytes=229288 combine_payload_bytes=233283584 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233512872 ingress_bytes=289906688 link_bytes=289906688 token_bytes=466796456 weight_movement_bytes=56623104 total_nvlink_bytes=523419560 weight_to_token_ratio=0.121301 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.777743241e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229280 combine_payload_bytes=233996288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234225568 ingress_bytes=271745024 link_bytes=271745024 token_bytes=468221856 weight_movement_bytes=37748736 total_nvlink_bytes=505970592 weight_to_token_ratio=0.080621 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603727113e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65282 payload_rows_remote=57092 dup_rows=254 dup_rows_remote=254 dedup_ratio=0.003876 router_tokens_per_expert=[0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68] payload_rows_per_dst=[8078, 8192, 8192, 8192, 8054, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 118, 0, 0, 0, 136] combine_rows_per_src=[8078, 8192, 8192, 8192, 8054, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63820 payload_rows_remote=55628 dup_rows=1716 dup_rows_remote=1716 dedup_ratio=0.026184 router_tokens_per_expert=[0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 0, 0, 0, 4096, 0, 1, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3249, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760] payload_rows_per_dst=[8190, 8192, 8192, 8192, 6498, 8192, 8172, 8192] dup_rows_per_dst=[0, 0, 142, 54, 0, 0, 0, 1520] combine_rows_per_src=[8190, 8192, 8192, 8192, 6498, 8192, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60952 payload_rows_remote=52766 dup_rows=4584 dup_rows_remote=4584 dedup_ratio=0.069946 router_tokens_per_expert=[0, 3, 0, 1529, 0, 0, 3, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 4093, 0, 3, 4093, 0, 0, 0, 108, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4270, 8186, 8190, 8186, 7544] dup_rows_per_dst=[3070, 1286, 216, 0, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 4270, 8186, 8190, 8186, 7544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60148 payload_rows_remote=51956 dup_rows=5388 dup_rows_remote=5388 dedup_ratio=0.082214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 11, 0, 0, 0, 0, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7196, 3830, 8192, 8190, 8192, 8192, 8164] dup_rows_per_dst=[0, 0, 0, 20, 5368, 0, 0, 0] combine_rows_per_src=[8192, 7196, 3830, 8192, 8190, 8192, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57206 payload_rows_remote=49020 dup_rows=8330 dup_rows_remote=6010 dedup_ratio=0.127106 router_tokens_per_expert=[0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3861, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282] payload_rows_per_dst=[8186, 8192, 8192, 6378, 8124, 8192, 8192, 1750] dup_rows_per_dst=[2320, 0, 1860, 202, 3880, 0, 0, 68] combine_rows_per_src=[8186, 8192, 8192, 6378, 8124, 8192, 8192, 1750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=233848832 dispatch_meta_bytes=229384 combine_payload_bytes=233848832 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234078216 ingress_bytes=271597568 link_bytes=271597568 token_bytes=467927048 weight_movement_bytes=37748736 total_nvlink_bytes=505675784 weight_to_token_ratio=0.080672 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602314262e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63008 payload_rows_remote=54816 dup_rows=2528 dup_rows_remote=566 dedup_ratio=0.038574 router_tokens_per_expert=[0, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4092, 11, 0, 0, 0, 3686, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3829, 0, 4092, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 971] payload_rows_per_dst=[7128, 8182, 7380, 8184, 7572, 8192, 8178, 8192] dup_rows_per_dst=[0, 8, 20, 434, 0, 2, 102, 1962] combine_rows_per_src=[7128, 8182, 7380, 8184, 7572, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=227852288 dispatch_meta_bytes=229376 combine_payload_bytes=227852288 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=228081664 ingress_bytes=246726656 link_bytes=246726656 token_bytes=455933952 weight_movement_bytes=18874368 total_nvlink_bytes=474808320 weight_to_token_ratio=0.041397 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.364013421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65254 payload_rows_remote=57064 dup_rows=282 dup_rows_remote=258 dedup_ratio=0.004303 router_tokens_per_expert=[0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7952, 8192, 8188, 8192, 8156] dup_rows_per_dst=[246, 12, 24, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 7952, 8192, 8188, 8192, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=216129536 dispatch_meta_bytes=229400 combine_payload_bytes=216129536 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=216358936 ingress_bytes=253878272 link_bytes=253878272 token_bytes=432488472 weight_movement_bytes=37748736 total_nvlink_bytes=470237208 weight_to_token_ratio=0.087283 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.432536686e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=212811776 dispatch_meta_bytes=229376 combine_payload_bytes=212811776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213041152 ingress_bytes=231686144 link_bytes=231686144 token_bytes=425852928 weight_movement_bytes=18874368 total_nvlink_bytes=444727296 weight_to_token_ratio=0.044321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219902635e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64792 payload_rows_remote=56630 dup_rows=744 dup_rows_remote=744 dedup_ratio=0.011353 router_tokens_per_expert=[0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8162, 8192, 7544, 8190, 8130] dup_rows_per_dst=[64, 646, 32, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8162, 8192, 7544, 8190, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=200785920 dispatch_meta_bytes=220120 combine_payload_bytes=200785920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201006040 ingress_bytes=219660288 link_bytes=219660288 token_bytes=401791960 weight_movement_bytes=18874368 total_nvlink_bytes=420666328 weight_to_token_ratio=0.046975 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.104676800e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=224526336 dispatch_meta_bytes=221528 combine_payload_bytes=224526336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224747864 ingress_bytes=243400704 link_bytes=243400704 token_bytes=449274200 weight_movement_bytes=18874368 total_nvlink_bytes=468148568 weight_to_token_ratio=0.042011 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=29312 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.332145785e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233734144 dispatch_meta_bytes=229288 combine_payload_bytes=233734144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233963432 ingress_bytes=271482880 link_bytes=271482880 token_bytes=467697576 weight_movement_bytes=37748736 total_nvlink_bytes=505446312 weight_to_token_ratio=0.080712 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.601215378e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=231956480 dispatch_meta_bytes=229496 combine_payload_bytes=231956480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232185976 ingress_bytes=250830848 link_bytes=250830848 token_bytes=464142456 weight_movement_bytes=18874368 total_nvlink_bytes=483016824 weight_to_token_ratio=0.040665 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403337769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65082 payload_rows_remote=57090 dup_rows=454 dup_rows_remote=454 dedup_ratio=0.006927 router_tokens_per_expert=[0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7974, 8192, 7992, 8192, 8158] dup_rows_per_dst=[324, 84, 46, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 7974, 8192, 7992, 8192, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63250 payload_rows_remote=55058 dup_rows=2286 dup_rows_remote=2256 dedup_ratio=0.034882 router_tokens_per_expert=[0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8018, 7394, 8192, 8178, 8192, 6892] dup_rows_per_dst=[2172, 30, 82, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8018, 7394, 8192, 8178, 8192, 6892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64436 payload_rows_remote=56244 dup_rows=1100 dup_rows_remote=502 dedup_ratio=0.016785 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 4077, 93, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3842, 0, 0, 1, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8164, 7634, 8192, 8192, 7684, 8192] dup_rows_per_dst=[6, 8, 0, 0, 598, 0, 2, 486] combine_rows_per_src=[8192, 8186, 8164, 7634, 8192, 8192, 7684, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60644 payload_rows_remote=52456 dup_rows=4892 dup_rows_remote=4892 dedup_ratio=0.074646 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 1504] payload_rows_per_dst=[3328, 8182, 8192, 8178, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 22, 0, 0, 2, 10, 0, 4858] combine_rows_per_src=[3328, 8182, 8192, 8178, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62688 payload_rows_remote=54496 dup_rows=2848 dup_rows_remote=2544 dedup_ratio=0.043457 router_tokens_per_expert=[0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6902, 8154, 8192, 8024, 8192, 6840] dup_rows_per_dst=[304, 1424, 0, 0, 0, 1120, 0, 0] combine_rows_per_src=[8192, 8192, 6902, 8154, 8192, 8024, 8192, 6840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56144 payload_rows_remote=48662 dup_rows=9392 dup_rows_remote=4194 dedup_ratio=0.143311 router_tokens_per_expert=[0, 4, 0, 2170, 0, 0, 4, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 4092, 0, 4, 4092, 0, 0, 0, 326, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4348, 3382, 8190, 8182, 8188, 8188, 8184, 7482] dup_rows_per_dst=[16, 1324, 652, 2168, 6, 12, 16, 5198] combine_rows_per_src=[4348, 3382, 8190, 8182, 8188, 8188, 8184, 7482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=233840640 dispatch_meta_bytes=230176 combine_payload_bytes=233840640 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234070816 ingress_bytes=271589376 link_bytes=271589376 token_bytes=467911456 weight_movement_bytes=37748736 total_nvlink_bytes=505660192 weight_to_token_ratio=0.080675 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602235771e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=225517568 dispatch_meta_bytes=229256 combine_payload_bytes=225517568 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=225746824 ingress_bytes=301015040 link_bytes=301015040 token_bytes=451264392 weight_movement_bytes=75497472 total_nvlink_bytes=526761864 weight_to_token_ratio=0.167302 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.884178005e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62618 payload_rows_remote=54426 dup_rows=2918 dup_rows_remote=2918 dedup_ratio=0.044525 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 2726, 0, 0, 47, 0, 2, 0, 4083, 0, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8170, 8168, 8188, 5648, 7872] dup_rows_per_dst=[0, 22, 0, 0, 1860, 0, 228, 808] combine_rows_per_src=[8192, 8188, 8192, 8170, 8168, 8188, 5648, 7872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=230375424 dispatch_meta_bytes=226984 combine_payload_bytes=230375424 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=230602408 ingress_bytes=305872896 link_bytes=305872896 token_bytes=460977832 weight_movement_bytes=75497472 total_nvlink_bytes=536475304 weight_to_token_ratio=0.163777 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=28928 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.930723591e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=214859776 dispatch_meta_bytes=229392 combine_payload_bytes=214859776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215089168 ingress_bytes=233734144 link_bytes=233734144 token_bytes=429948944 weight_movement_bytes=18874368 total_nvlink_bytes=448823312 weight_to_token_ratio=0.043899 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.239525563e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59190 payload_rows_remote=51204 dup_rows=6346 dup_rows_remote=6346 dedup_ratio=0.096832 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4082, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 1041, 0, 0, 2, 0, 5, 0, 4083, 0, 0, 0, 3, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7986, 8190, 8182, 2086, 8172] dup_rows_per_dst=[0, 32, 0, 0, 2570, 0, 10, 3734] combine_rows_per_src=[8192, 8192, 8190, 7986, 8190, 8182, 2086, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=223215616 dispatch_meta_bytes=228160 combine_payload_bytes=223215616 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=223443776 ingress_bytes=279838720 link_bytes=279838720 token_bytes=446659392 weight_movement_bytes=56623104 total_nvlink_bytes=503282496 weight_to_token_ratio=0.126770 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.681276926e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=199319552 dispatch_meta_bytes=211424 combine_payload_bytes=199319552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=199530976 ingress_bytes=255942656 link_bytes=255942656 token_bytes=398850528 weight_movement_bytes=56623104 total_nvlink_bytes=455473632 weight_to_token_ratio=0.141966 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=30208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.452316598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=222928896 dispatch_meta_bytes=229376 combine_payload_bytes=222928896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=223158272 ingress_bytes=241803264 link_bytes=241803264 token_bytes=446087168 weight_movement_bytes=18874368 total_nvlink_bytes=464961536 weight_to_token_ratio=0.042311 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.316839901e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=209731584 dispatch_meta_bytes=230200 combine_payload_bytes=209731584 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=209961784 ingress_bytes=266354688 link_bytes=266354688 token_bytes=419693368 weight_movement_bytes=56623104 total_nvlink_bytes=476316472 weight_to_token_ratio=0.134915 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.552079566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62718 payload_rows_remote=54526 dup_rows=2818 dup_rows_remote=2818 dedup_ratio=0.042999 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2723, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1216, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8114, 8192, 8192, 5454, 8192] dup_rows_per_dst=[0, 0, 0, 0, 386, 0, 0, 2432] combine_rows_per_src=[8190, 8192, 8192, 8114, 8192, 8192, 5454, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57506 payload_rows_remote=49396 dup_rows=8030 dup_rows_remote=8012 dedup_ratio=0.122528 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 4018, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 3713, 0, 0, 95, 0, 8, 0, 4074, 0, 0, 0, 1, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8110, 8188, 8186, 8142, 8178, 8192, 318] dup_rows_per_dst=[0, 18, 0, 0, 424, 0, 7588, 0] combine_rows_per_src=[8192, 8110, 8188, 8186, 8142, 8178, 8192, 318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64954 payload_rows_remote=56776 dup_rows=582 dup_rows_remote=582 dedup_ratio=0.008881 router_tokens_per_expert=[0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3837, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230] payload_rows_per_dst=[8142, 8192, 8192, 8192, 7674, 8192, 8178, 8192] dup_rows_per_dst=[0, 0, 4, 118, 0, 0, 0, 460] combine_rows_per_src=[8142, 8192, 8192, 8192, 7674, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65426 payload_rows_remote=57240 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.001678 router_tokens_per_expert=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4, 10, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8186, 8102, 8192, 8186, 8190, 8192, 8192] dup_rows_per_dst=[0, 90, 2, 0, 0, 6, 6, 6] combine_rows_per_src=[8186, 8186, 8102, 8192, 8186, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57750 payload_rows_remote=49558 dup_rows=7786 dup_rows_remote=7786 dedup_ratio=0.118805 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3015, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8154, 7284, 8192, 8192, 1352] dup_rows_per_dst=[0, 4, 136, 1060, 548, 0, 6038, 0] combine_rows_per_src=[8192, 8192, 8192, 8154, 7284, 8192, 8192, 1352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58588 payload_rows_remote=50396 dup_rows=6948 dup_rows_remote=506 dedup_ratio=0.106018 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 4026, 205, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 3752, 0, 0, 6, 0, 0, 0, 4092, 0, 4, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8160, 8096, 8046, 2218, 8190, 7506, 8192] dup_rows_per_dst=[8, 42, 0, 0, 394, 32, 30, 6442] combine_rows_per_src=[8180, 8160, 8096, 8046, 2218, 8190, 7506, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63516 payload_rows_remote=57290 dup_rows=2020 dup_rows_remote=2020 dedup_ratio=0.030823 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 0, 0, 73, 0, 0, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6226, 8192, 8192, 8138, 8192, 8192] dup_rows_per_dst=[0, 2020, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 6226, 8192, 8192, 8138, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=223338496 dispatch_meta_bytes=229376 combine_payload_bytes=223338496 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=223567872 ingress_bytes=261087232 link_bytes=261087232 token_bytes=446906368 weight_movement_bytes=37748736 total_nvlink_bytes=484655104 weight_to_token_ratio=0.084467 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.501609394e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=202326016 dispatch_meta_bytes=229632 combine_payload_bytes=202326016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=202555648 ingress_bytes=221200384 link_bytes=221200384 token_bytes=404881664 weight_movement_bytes=18874368 total_nvlink_bytes=423756032 weight_to_token_ratio=0.046617 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.119433242e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=232554496 dispatch_meta_bytes=229432 combine_payload_bytes=232554496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232783928 ingress_bytes=251428864 link_bytes=251428864 token_bytes=465338424 weight_movement_bytes=18874368 total_nvlink_bytes=484212792 weight_to_token_ratio=0.040561 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.409067664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64670 payload_rows_remote=56478 dup_rows=866 dup_rows_remote=866 dedup_ratio=0.013214 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7346, 8192, 8192, 8174, 8190, 8192] dup_rows_per_dst=[0, 866, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7346, 8192, 8192, 8174, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234455040 dispatch_meta_bytes=229400 combine_payload_bytes=234455040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234684440 ingress_bytes=253329408 link_bytes=253329408 token_bytes=469139480 weight_movement_bytes=18874368 total_nvlink_bytes=488013848 weight_to_token_ratio=0.040232 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.427277742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=202989568 dispatch_meta_bytes=229376 combine_payload_bytes=202989568 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=203218944 ingress_bytes=221863936 link_bytes=221863936 token_bytes=406208512 weight_movement_bytes=18874368 total_nvlink_bytes=425082880 weight_to_token_ratio=0.046465 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.125791071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=206422016 dispatch_meta_bytes=203608 combine_payload_bytes=206422016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=206625624 ingress_bytes=244170752 link_bytes=244170752 token_bytes=413047640 weight_movement_bytes=37748736 total_nvlink_bytes=450796376 weight_to_token_ratio=0.091391 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.339524006e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=234659840 dispatch_meta_bytes=237240 combine_payload_bytes=234659840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234897080 ingress_bytes=253534208 link_bytes=253534208 token_bytes=469556920 weight_movement_bytes=18874368 total_nvlink_bytes=488431288 weight_to_token_ratio=0.040196 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429240034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=231333888 dispatch_meta_bytes=229376 combine_payload_bytes=231333888 prefetch_bytes=0 reduce_bytes=0 egress_bytes=231563264 ingress_bytes=231333888 link_bytes=231563264 token_bytes=462897152 weight_movement_bytes=0 total_nvlink_bytes=462897152 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.218725259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61076 payload_rows_remote=52884 dup_rows=4460 dup_rows_remote=4460 dedup_ratio=0.068054 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 570, 0, 0, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6366, 5558, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2634, 1826, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 6366, 5558, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60224 payload_rows_remote=52032 dup_rows=5312 dup_rows_remote=5312 dedup_ratio=0.081055 router_tokens_per_expert=[0, 0, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4744, 8192, 8130, 7550, 8192, 8088, 8192, 7136] dup_rows_per_dst=[1096, 4, 482, 3730, 0, 0, 0, 0] combine_rows_per_src=[4744, 8192, 8130, 7550, 8192, 8088, 8192, 7136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59588 payload_rows_remote=51396 dup_rows=5948 dup_rows_remote=0 dedup_ratio=0.090759 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 47, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2278, 8192, 8192, 8158, 8192, 8192] dup_rows_per_dst=[0, 5948, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 2278, 8192, 8192, 8158, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65072 payload_rows_remote=56880 dup_rows=464 dup_rows_remote=26 dedup_ratio=0.007080 router_tokens_per_expert=[0, 3873, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[7752, 8186, 8192, 8180, 8192, 8186, 8192, 8192] dup_rows_per_dst=[6, 0, 6, 0, 438, 2, 6, 6] combine_rows_per_src=[7752, 8186, 8192, 8180, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63860 payload_rows_remote=55668 dup_rows=1676 dup_rows_remote=1670 dedup_ratio=0.025574 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 6536, 8192, 8192, 8192, 8190, 8178] dup_rows_per_dst=[6, 1656, 0, 0, 0, 14, 0, 0] combine_rows_per_src=[8192, 8188, 6536, 8192, 8192, 8192, 8190, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57420 payload_rows_remote=49228 dup_rows=8116 dup_rows_remote=8108 dedup_ratio=0.123840 router_tokens_per_expert=[0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[8192, 100, 8188, 8192, 8184, 8180, 8192, 8192] dup_rows_per_dst=[0, 0, 7990, 0, 14, 104, 0, 8] combine_rows_per_src=[8192, 100, 8188, 8192, 8184, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=216612864 dispatch_meta_bytes=229376 combine_payload_bytes=216612864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=216842240 ingress_bytes=254361600 link_bytes=254361600 token_bytes=433455104 weight_movement_bytes=37748736 total_nvlink_bytes=471203840 weight_to_token_ratio=0.087088 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.437167697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65236 payload_rows_remote=57044 dup_rows=300 dup_rows_remote=300 dedup_ratio=0.004578 router_tokens_per_expert=[0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 150] payload_rows_per_dst=[7892, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 300] combine_rows_per_src=[7892, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=213123072 dispatch_meta_bytes=229376 combine_payload_bytes=213123072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=213352448 ingress_bytes=231997440 link_bytes=231997440 token_bytes=426475520 weight_movement_bytes=18874368 total_nvlink_bytes=445349888 weight_to_token_ratio=0.044257 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222885320e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=210518016 dispatch_meta_bytes=205584 combine_payload_bytes=210518016 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=210723600 ingress_bytes=304889856 link_bytes=304889856 token_bytes=421241616 weight_movement_bytes=94371840 total_nvlink_bytes=515613456 weight_to_token_ratio=0.224033 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=22784 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.921304585e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=232980480 dispatch_meta_bytes=227624 combine_payload_bytes=232980480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233208104 ingress_bytes=251854848 link_bytes=251854848 token_bytes=466188584 weight_movement_bytes=18874368 total_nvlink_bytes=485062952 weight_to_token_ratio=0.040487 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413149233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=228016128 dispatch_meta_bytes=229352 combine_payload_bytes=228016128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=228245480 ingress_bytes=246890496 link_bytes=246890496 token_bytes=456261608 weight_movement_bytes=18874368 total_nvlink_bytes=475135976 weight_to_token_ratio=0.041367 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.365583255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65352 payload_rows_remote=57160 dup_rows=184 dup_rows_remote=170 dedup_ratio=0.002808 router_tokens_per_expert=[0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 85] payload_rows_per_dst=[8022, 8192, 8192, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 14, 4, 0, 0, 166] combine_rows_per_src=[8022, 8192, 8192, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=201637888 dispatch_meta_bytes=229344 combine_payload_bytes=201637888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201867232 ingress_bytes=220512256 link_bytes=220512256 token_bytes=403505120 weight_movement_bytes=18874368 total_nvlink_bytes=422379488 weight_to_token_ratio=0.046776 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.112839938e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=233652224 dispatch_meta_bytes=229376 combine_payload_bytes=233652224 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233881600 ingress_bytes=271400960 link_bytes=271400960 token_bytes=467533824 weight_movement_bytes=37748736 total_nvlink_bytes=505282560 weight_to_token_ratio=0.080740 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600430461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234127360 dispatch_meta_bytes=229320 combine_payload_bytes=234127360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234356680 ingress_bytes=253001728 link_bytes=253001728 token_bytes=468484040 weight_movement_bytes=18874368 total_nvlink_bytes=487358408 weight_to_token_ratio=0.040288 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424138073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65502 payload_rows_remote=57310 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000519 router_tokens_per_expert=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17] payload_rows_per_dst=[8160, 8192, 8192, 8190, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 34] combine_rows_per_src=[8160, 8192, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64190 payload_rows_remote=57318 dup_rows=1346 dup_rows_remote=1340 dedup_ratio=0.020538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 3425, 0, 0, 13, 0, 1, 0, 4083, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8186, 8186, 8190, 8190, 6872, 8188] dup_rows_per_dst=[0, 4, 0, 0, 180, 0, 6, 1156] combine_rows_per_src=[8192, 8186, 8186, 8186, 8190, 8190, 6872, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63854 payload_rows_remote=55662 dup_rows=1682 dup_rows_remote=1682 dedup_ratio=0.025665 router_tokens_per_expert=[0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 840] payload_rows_per_dst=[6510, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 1678] combine_rows_per_src=[6510, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59994 payload_rows_remote=51838 dup_rows=5542 dup_rows_remote=450 dedup_ratio=0.084564 router_tokens_per_expert=[0, 16, 0, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3530, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 3811, 0, 2810, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 3, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[4146, 7060, 8166, 8124, 8156, 7964, 8186, 8192] dup_rows_per_dst=[4, 4, 0, 6, 5092, 0, 404, 32] combine_rows_per_src=[4146, 7060, 8166, 8124, 8156, 7964, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61660 payload_rows_remote=53468 dup_rows=3876 dup_rows_remote=762 dedup_ratio=0.059143 router_tokens_per_expert=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[8192, 8192, 8186, 8192, 7438, 8192, 8192, 5076] dup_rows_per_dst=[3114, 760, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 7438, 8192, 8192, 5076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=229376 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234971136 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469712896 weight_movement_bytes=18874368 total_nvlink_bytes=488587264 weight_to_token_ratio=0.040183 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58816 payload_rows_remote=50624 dup_rows=6720 dup_rows_remote=6672 dedup_ratio=0.102539 router_tokens_per_expert=[0, 2241, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 2517, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3618, 0, 2096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6666, 6672, 8192, 8192, 4568, 8148, 8192, 8186] dup_rows_per_dst=[1478, 3536, 48, 0, 1638, 0, 18, 2] combine_rows_per_src=[6666, 6672, 8192, 8192, 4568, 8148, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64420 payload_rows_remote=56228 dup_rows=1116 dup_rows_remote=374 dedup_ratio=0.017029 router_tokens_per_expert=[0, 3934, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 4, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7876, 7438, 8184, 8184, 8192, 8184, 8170, 8192] dup_rows_per_dst=[8, 0, 8, 6, 308, 28, 16, 742] combine_rows_per_src=[7876, 7438, 8184, 8184, 8192, 8184, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=234774528 dispatch_meta_bytes=234632 combine_payload_bytes=234774528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235009160 ingress_bytes=272523264 link_bytes=272523264 token_bytes=469783688 weight_movement_bytes=37748736 total_nvlink_bytes=507532424 weight_to_token_ratio=0.080353 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611183826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=227991552 dispatch_meta_bytes=229376 combine_payload_bytes=227991552 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=228220928 ingress_bytes=303489024 link_bytes=303489024 token_bytes=456212480 weight_movement_bytes=75497472 total_nvlink_bytes=531709952 weight_to_token_ratio=0.165488 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.907882502e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=212328448 dispatch_meta_bytes=209152 combine_payload_bytes=212328448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=212537600 ingress_bytes=287825920 link_bytes=287825920 token_bytes=424866048 weight_movement_bytes=75497472 total_nvlink_bytes=500363520 weight_to_token_ratio=0.177697 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.757806346e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58242 payload_rows_remote=50050 dup_rows=7294 dup_rows_remote=7294 dedup_ratio=0.111298 router_tokens_per_expert=[0, 3335, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 799, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3968, 0, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7716, 1598, 8192, 8192, 7990, 8170, 8192, 8192] dup_rows_per_dst=[2344, 0, 0, 0, 4924, 0, 2, 24] combine_rows_per_src=[7716, 1598, 8192, 8192, 7990, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=219004928 dispatch_meta_bytes=216920 combine_payload_bytes=219004928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219221848 ingress_bytes=237879296 link_bytes=237879296 token_bytes=438226776 weight_movement_bytes=18874368 total_nvlink_bytes=457101144 weight_to_token_ratio=0.043070 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.279242370e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=207355904 dispatch_meta_bytes=229184 combine_payload_bytes=207355904 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=207585088 ingress_bytes=245104640 link_bytes=245104640 token_bytes=414940992 weight_movement_bytes=37748736 total_nvlink_bytes=452689728 weight_to_token_ratio=0.090974 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.348472061e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=230309888 dispatch_meta_bytes=226408 combine_payload_bytes=230309888 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=230536296 ingress_bytes=286932992 link_bytes=286932992 token_bytes=460846184 weight_movement_bytes=56623104 total_nvlink_bytes=517469288 weight_to_token_ratio=0.122868 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.749250750e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=205004800 dispatch_meta_bytes=229376 combine_payload_bytes=205004800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=205234176 ingress_bytes=261627904 link_bytes=261627904 token_bytes=410238976 weight_movement_bytes=56623104 total_nvlink_bytes=466862080 weight_to_token_ratio=0.138025 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.506789847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62298 payload_rows_remote=54112 dup_rows=3238 dup_rows_remote=3238 dedup_ratio=0.049408 router_tokens_per_expert=[0, 2105, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 3327, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3199, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7452, 6654, 8192, 8190, 7240, 8186, 8192, 8192] dup_rows_per_dst=[1822, 0, 0, 0, 1406, 0, 0, 10] combine_rows_per_src=[7452, 6654, 8192, 8190, 7240, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62244 payload_rows_remote=54054 dup_rows=3292 dup_rows_remote=3292 dedup_ratio=0.050232 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2453, 0, 0, 0, 0, 0, 0, 207, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4906, 8192, 8192, 8192, 8188, 8190, 8192] dup_rows_per_dst=[0, 0, 3292, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 4906, 8192, 8192, 8192, 8188, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59326 payload_rows_remote=51218 dup_rows=6210 dup_rows_remote=3954 dedup_ratio=0.094757 router_tokens_per_expert=[0, 2807, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 3620, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1562, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7326, 8108, 8192, 8192, 3016, 8108, 8192, 8192] dup_rows_per_dst=[3346, 2256, 0, 0, 0, 0, 74, 534] combine_rows_per_src=[7326, 8108, 8192, 8192, 3016, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59648 payload_rows_remote=51462 dup_rows=5888 dup_rows_remote=5156 dedup_ratio=0.089844 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1525, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 3, 2100, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3496, 7082, 8192, 8186, 8186, 8124, 8190] dup_rows_per_dst=[12, 744, 4400, 0, 732, 0, 0, 0] combine_rows_per_src=[8192, 3496, 7082, 8192, 8186, 8186, 8124, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57598 payload_rows_remote=50862 dup_rows=7938 dup_rows_remote=6820 dedup_ratio=0.121124 router_tokens_per_expert=[0, 2701, 0, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 2638, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 0, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6736, 8184, 8192, 8192, 3350, 8176, 7264, 7504] dup_rows_per_dst=[1118, 6118, 0, 686, 0, 0, 16, 0] combine_rows_per_src=[6736, 8184, 8192, 8192, 3350, 8176, 7264, 7504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=221642752 dispatch_meta_bytes=229400 combine_payload_bytes=221642752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=221872152 ingress_bytes=240517120 link_bytes=240517120 token_bytes=443514904 weight_movement_bytes=18874368 total_nvlink_bytes=462389272 weight_to_token_ratio=0.042556 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.304516702e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62944 payload_rows_remote=57314 dup_rows=2592 dup_rows_remote=2592 dedup_ratio=0.039551 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5630, 8192, 8192, 8192, 8162, 8192] dup_rows_per_dst=[0, 916, 0, 0, 1676, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5630, 8192, 8192, 8192, 8162, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59030 payload_rows_remote=50838 dup_rows=6506 dup_rows_remote=6436 dedup_ratio=0.099274 router_tokens_per_expert=[0, 9, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3680, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4007, 0, 1692, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 4, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4, 0, 0, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[4102, 7360, 8170, 7138, 8146, 7738, 8184, 8192] dup_rows_per_dst=[4, 4, 0, 432, 4678, 892, 426, 70] combine_rows_per_src=[4102, 7360, 8170, 7138, 8146, 7738, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=221405184 dispatch_meta_bytes=229384 combine_payload_bytes=221405184 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=221634568 ingress_bytes=240279552 link_bytes=240279552 token_bytes=443039752 weight_movement_bytes=18874368 total_nvlink_bytes=461914120 weight_to_token_ratio=0.042602 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.302240442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=209788928 dispatch_meta_bytes=220688 combine_payload_bytes=209788928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=210009616 ingress_bytes=247537664 link_bytes=247537664 token_bytes=419798544 weight_movement_bytes=37748736 total_nvlink_bytes=457547280 weight_to_token_ratio=0.089921 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=24192 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.371784100e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=210788352 dispatch_meta_bytes=226472 combine_payload_bytes=210788352 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=211014824 ingress_bytes=305160192 link_bytes=305160192 token_bytes=421803176 weight_movement_bytes=94371840 total_nvlink_bytes=516175016 weight_to_token_ratio=0.223734 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=25984 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.923894812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64662 payload_rows_remote=56470 dup_rows=874 dup_rows_remote=874 dedup_ratio=0.013336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 3660, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7352, 8192, 8192, 8192, 8158, 8192] dup_rows_per_dst=[0, 542, 0, 0, 332, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7352, 8192, 8192, 8192, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=208330752 dispatch_meta_bytes=230728 combine_payload_bytes=208330752 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=208561480 ingress_bytes=246079488 link_bytes=246079488 token_bytes=416892232 weight_movement_bytes=37748736 total_nvlink_bytes=454640968 weight_to_token_ratio=0.090548 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=16384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357812575e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=234758144 dispatch_meta_bytes=239624 combine_payload_bytes=234758144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234997768 ingress_bytes=253632512 link_bytes=253632512 token_bytes=469755912 weight_movement_bytes=18874368 total_nvlink_bytes=488630280 weight_to_token_ratio=0.040179 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430181935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=208232448 dispatch_meta_bytes=229096 combine_payload_bytes=208232448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208461544 ingress_bytes=227106816 link_bytes=227106816 token_bytes=416693992 weight_movement_bytes=18874368 total_nvlink_bytes=435568360 weight_to_token_ratio=0.045296 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.176025767e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=231301120 dispatch_meta_bytes=229376 combine_payload_bytes=231301120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231530496 ingress_bytes=250175488 link_bytes=250175488 token_bytes=462831616 weight_movement_bytes=18874368 total_nvlink_bytes=481705984 weight_to_token_ratio=0.040780 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397058432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58992 payload_rows_remote=50800 dup_rows=6544 dup_rows_remote=6544 dedup_ratio=0.099854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1648, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4810, 0, 1734, 0, 0, 0] combine_rows_per_src=[8192, 1648, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65346 payload_rows_remote=57154 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.002899 router_tokens_per_expert=[0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 95] payload_rows_per_dst=[8004, 8192, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 190] combine_rows_per_src=[8004, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57604 payload_rows_remote=49412 dup_rows=7932 dup_rows_remote=280 dedup_ratio=0.121033 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 1448, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3076, 8192, 5458, 8192, 8110, 8192] dup_rows_per_dst=[0, 7652, 0, 0, 280, 0, 0, 0] combine_rows_per_src=[8192, 8192, 3076, 8192, 5458, 8192, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57746 payload_rows_remote=49554 dup_rows=7790 dup_rows_remote=7782 dedup_ratio=0.118866 router_tokens_per_expert=[0, 0, 0, 0, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 3, 0, 0, 0, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3419, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2090, 0, 30, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0] payload_rows_per_dst=[4932, 5022, 6844, 8192, 8192, 8192, 8186, 8186] dup_rows_per_dst=[0, 0, 0, 0, 8, 4246, 3504, 32] combine_rows_per_src=[4932, 5022, 6844, 8192, 8192, 8192, 8186, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58486 payload_rows_remote=51134 dup_rows=7050 dup_rows_remote=7050 dedup_ratio=0.107574 router_tokens_per_expert=[0, 0, 0, 0, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7676, 7994, 7352, 8192, 8192, 2696, 8192, 8192] dup_rows_per_dst=[4444, 504, 0, 0, 0, 0, 2100, 2] combine_rows_per_src=[7676, 7994, 7352, 8192, 8192, 2696, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=208076800 dispatch_meta_bytes=229376 combine_payload_bytes=208076800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208306176 ingress_bytes=226951168 link_bytes=226951168 token_bytes=416382976 weight_movement_bytes=18874368 total_nvlink_bytes=435257344 weight_to_token_ratio=0.045329 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.174534425e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53070 payload_rows_remote=44878 dup_rows=12466 dup_rows_remote=12466 dedup_ratio=0.190216 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3388, 8192, 532, 8192, 8190, 8192] dup_rows_per_dst=[0, 12466, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 3388, 8192, 532, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=234102784 dispatch_meta_bytes=229376 combine_payload_bytes=234102784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234332160 ingress_bytes=252977152 link_bytes=252977152 token_bytes=468434944 weight_movement_bytes=18874368 total_nvlink_bytes=487309312 weight_to_token_ratio=0.040292 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423902598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=202391552 dispatch_meta_bytes=198768 combine_payload_bytes=202391552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=202590320 ingress_bytes=240140288 link_bytes=240140288 token_bytes=404981872 weight_movement_bytes=37748736 total_nvlink_bytes=442730608 weight_to_token_ratio=0.093211 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=17024 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.300906083e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59582 payload_rows_remote=51396 dup_rows=5954 dup_rows_remote=5946 dedup_ratio=0.090851 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 1352, 0, 0, 0, 3, 4092, 0, 0, 2, 0, 0, 0, 4, 1480, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4092, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4092, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 2722, 8192, 8192, 7860, 8184, 8082, 8186] dup_rows_per_dst=[16, 0, 3638, 0, 1938, 14, 340, 8] combine_rows_per_src=[8164, 2722, 8192, 8192, 7860, 8184, 8082, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=202973184 dispatch_meta_bytes=229344 combine_payload_bytes=202973184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=203202528 ingress_bytes=221847552 link_bytes=221847552 token_bytes=406175712 weight_movement_bytes=18874368 total_nvlink_bytes=425050080 weight_to_token_ratio=0.046468 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.125634087e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54912 payload_rows_remote=46730 dup_rows=10624 dup_rows_remote=10624 dedup_ratio=0.162109 router_tokens_per_expert=[0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2844, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 3329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0] payload_rows_per_dst=[1154, 7450, 5688, 8182, 8192, 7862, 8192, 8192] dup_rows_per_dst=[0, 556, 0, 0, 8, 6326, 3664, 70] combine_rows_per_src=[1154, 7450, 5688, 8182, 8192, 7862, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=209444864 dispatch_meta_bytes=232736 combine_payload_bytes=209444864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=209677600 ingress_bytes=247193600 link_bytes=247193600 token_bytes=419122464 weight_movement_bytes=37748736 total_nvlink_bytes=456871200 weight_to_token_ratio=0.090066 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.368487448e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=183820288 dispatch_meta_bytes=229376 combine_payload_bytes=183820288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=184049664 ingress_bytes=202694656 link_bytes=202694656 token_bytes=367869952 weight_movement_bytes=18874368 total_nvlink_bytes=386744320 weight_to_token_ratio=0.051307 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.942120462e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=210518016 dispatch_meta_bytes=229368 combine_payload_bytes=210518016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=210747384 ingress_bytes=229392384 link_bytes=229392384 token_bytes=421265400 weight_movement_bytes=18874368 total_nvlink_bytes=440139768 weight_to_token_ratio=0.044804 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.197924955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=191406080 dispatch_meta_bytes=229416 combine_payload_bytes=191406080 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=191635496 ingress_bytes=229154816 link_bytes=229154816 token_bytes=383041576 weight_movement_bytes=37748736 total_nvlink_bytes=420790312 weight_to_token_ratio=0.098550 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.195648696e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63878 payload_rows_remote=55686 dup_rows=1658 dup_rows_remote=776 dedup_ratio=0.025299 router_tokens_per_expert=[0, 0, 0, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[7286, 8004, 7628, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 882, 768, 8] combine_rows_per_src=[7286, 8004, 7628, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61286 payload_rows_remote=53094 dup_rows=4250 dup_rows_remote=4248 dedup_ratio=0.064850 router_tokens_per_expert=[0, 2734, 0, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 3989, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1077, 0, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7644, 7982, 8192, 8192, 4738, 8154, 8192, 8192] dup_rows_per_dst=[3632, 0, 0, 0, 594, 0, 2, 22] combine_rows_per_src=[7644, 7982, 8192, 8192, 4738, 8154, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51846 payload_rows_remote=43748 dup_rows=13690 dup_rows_remote=6492 dedup_ratio=0.208893 router_tokens_per_expert=[0, 0, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8098, 7218, 8192, 8192, 1596, 2166, 8192] dup_rows_per_dst=[5910, 7198, 0, 0, 0, 0, 188, 394] combine_rows_per_src=[8192, 8098, 7218, 8192, 8192, 1596, 2166, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55898 payload_rows_remote=47712 dup_rows=9638 dup_rows_remote=4124 dedup_ratio=0.147064 router_tokens_per_expert=[0, 0, 0, 0, 3398, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 624, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093] payload_rows_per_dst=[6802, 8192, 0, 8192, 8186, 8186, 8148, 8192] dup_rows_per_dst=[0, 1254, 0, 2, 5514, 2856, 12, 0] combine_rows_per_src=[6802, 8192, 0, 8192, 8186, 8186, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54976 payload_rows_remote=46784 dup_rows=10560 dup_rows_remote=2368 dedup_ratio=0.161133 router_tokens_per_expert=[0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8166, 8192, 8192, 8192, 4590, 4578, 4874, 8192] dup_rows_per_dst=[0, 2368, 8192, 0, 0, 0, 0, 0] combine_rows_per_src=[8166, 8192, 8192, 8192, 4590, 4578, 4874, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=225848 combine_payload_bytes=228089856 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=228315704 ingress_bytes=284712960 link_bytes=284712960 token_bytes=456405560 weight_movement_bytes=56623104 total_nvlink_bytes=513028664 weight_to_token_ratio=0.124063 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.090278 gemm_active_groups=8 gemm_peak_group_rows=20224 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.727979495e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=217473024 dispatch_meta_bytes=229368 combine_payload_bytes=217473024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217702392 ingress_bytes=236347392 link_bytes=236347392 token_bytes=435175416 weight_movement_bytes=18874368 total_nvlink_bytes=454049784 weight_to_token_ratio=0.043372 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.264564420e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=54026 payload_rows_remote=45834 dup_rows=11510 dup_rows_remote=8484 dedup_ratio=0.175629 router_tokens_per_expert=[0, 0, 0, 0, 0, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7424, 8192, 8160, 5044, 1468, 7354] dup_rows_per_dst=[3026, 7708, 640, 32, 0, 0, 0, 104] combine_rows_per_src=[8192, 8192, 7424, 8192, 8160, 5044, 1468, 7354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56620 payload_rows_remote=48436 dup_rows=8916 dup_rows_remote=8698 dedup_ratio=0.136047 router_tokens_per_expert=[0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 4, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3637, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 1374, 0, 1, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0] payload_rows_per_dst=[7568, 8, 8122, 8170, 8192, 8192, 8184, 8184] dup_rows_per_dst=[0, 0, 3690, 0, 16, 2788, 2204, 218] combine_rows_per_src=[7568, 8, 8122, 8170, 8192, 8192, 8184, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=179191808 dispatch_meta_bytes=200960 combine_payload_bytes=179191808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=179392768 ingress_bytes=235814912 link_bytes=235814912 token_bytes=358584576 weight_movement_bytes=56623104 total_nvlink_bytes=415207680 weight_to_token_ratio=0.157907 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=24448 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.259462459e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58600 payload_rows_remote=50426 dup_rows=6936 dup_rows_remote=6936 dedup_ratio=0.105835 router_tokens_per_expert=[0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8160, 8192, 7466, 8174, 7776, 3808, 6832, 8192] dup_rows_per_dst=[0, 472, 1178, 0, 4858, 428, 0, 0] combine_rows_per_src=[8160, 8192, 7466, 8174, 7776, 3808, 6832, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=195428352 dispatch_meta_bytes=207344 combine_payload_bytes=195428352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=195635696 ingress_bytes=214302720 link_bytes=214302720 token_bytes=391064048 weight_movement_bytes=18874368 total_nvlink_bytes=409938416 weight_to_token_ratio=0.048264 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053343220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=191627264 dispatch_meta_bytes=196608 combine_payload_bytes=191627264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=191823872 ingress_bytes=248250368 link_bytes=248250368 token_bytes=383451136 weight_movement_bytes=56623104 total_nvlink_bytes=440074240 weight_to_token_ratio=0.147667 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=16128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.378612879e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=187736064 dispatch_meta_bytes=217272 combine_payload_bytes=187736064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=187953336 ingress_bytes=206610432 link_bytes=206610432 token_bytes=375689400 weight_movement_bytes=18874368 total_nvlink_bytes=394563768 weight_to_token_ratio=0.050239 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=20864 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.979639501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=198393856 dispatch_meta_bytes=228536 combine_payload_bytes=198393856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=198622392 ingress_bytes=217268224 link_bytes=217268224 token_bytes=397016248 weight_movement_bytes=18874368 total_nvlink_bytes=415890616 weight_to_token_ratio=0.047541 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.081757220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=206544896 dispatch_meta_bytes=229448 combine_payload_bytes=206544896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206774344 ingress_bytes=225419264 link_bytes=225419264 token_bytes=413319240 weight_movement_bytes=18874368 total_nvlink_bytes=432193608 weight_to_token_ratio=0.045665 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.159856474e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57070 payload_rows_remote=48878 dup_rows=8466 dup_rows_remote=5436 dedup_ratio=0.129181 router_tokens_per_expert=[0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8122, 7764, 1418, 8192, 8192, 8192, 6998, 8192] dup_rows_per_dst=[0, 4704, 0, 0, 732, 3030, 0, 0] combine_rows_per_src=[8122, 7764, 1418, 8192, 8192, 8192, 6998, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59912 payload_rows_remote=51734 dup_rows=5624 dup_rows_remote=5624 dedup_ratio=0.085815 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 7, 4096, 0, 0, 1, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2582, 8192, 8192, 8192, 8192, 8178, 8192] dup_rows_per_dst=[0, 0, 1584, 0, 4040, 0, 0, 0] combine_rows_per_src=[8192, 2582, 8192, 8192, 8192, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=53820 payload_rows_remote=45628 dup_rows=11716 dup_rows_remote=8192 dedup_ratio=0.178772 router_tokens_per_expert=[0, 0, 0, 0, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 4096, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7452, 8192, 8192, 8160, 626, 7056, 5950, 8192] dup_rows_per_dst=[0, 3524, 8192, 0, 0, 0, 0, 0] combine_rows_per_src=[7452, 8192, 8192, 8160, 626, 7056, 5950, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60458 payload_rows_remote=52266 dup_rows=5078 dup_rows_remote=8 dedup_ratio=0.077484 router_tokens_per_expert=[0, 0, 0, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 3, 0, 0, 0, 0, 2524, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093] payload_rows_per_dst=[3244, 8192, 8192, 8192, 8192, 8186, 8186, 8074] dup_rows_per_dst=[0, 8, 0, 0, 5070, 0, 0, 0] combine_rows_per_src=[3244, 8192, 8192, 8192, 8192, 8186, 8186, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64798 payload_rows_remote=56606 dup_rows=738 dup_rows_remote=738 dedup_ratio=0.011261 router_tokens_per_expert=[0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7474, 8192, 8192, 8186, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 42, 0, 0, 696, 0, 0, 0] combine_rows_per_src=[7474, 8192, 8192, 8186, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=51762 payload_rows_remote=43572 dup_rows=13774 dup_rows_remote=9960 dedup_ratio=0.210175 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 5278, 8192, 8192, 1616, 5618, 6484, 8192] dup_rows_per_dst=[3814, 1768, 8192, 0, 0, 0, 0, 0] combine_rows_per_src=[8190, 5278, 8192, 8192, 1616, 5618, 6484, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=200204288 dispatch_meta_bytes=217256 combine_payload_bytes=200204288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=200421544 ingress_bytes=219078656 link_bytes=219078656 token_bytes=400625832 weight_movement_bytes=18874368 total_nvlink_bytes=419500200 weight_to_token_ratio=0.047112 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=16768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.099103888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=211902464 dispatch_meta_bytes=229432 combine_payload_bytes=211902464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=212131896 ingress_bytes=249651200 link_bytes=249651200 token_bytes=424034360 weight_movement_bytes=37748736 total_nvlink_bytes=461783096 weight_to_token_ratio=0.089023 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.392034962e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=186892288 dispatch_meta_bytes=215280 combine_payload_bytes=186892288 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=187107568 ingress_bytes=262389760 link_bytes=262389760 token_bytes=373999856 weight_movement_bytes=75497472 total_nvlink_bytes=449497328 weight_to_token_ratio=0.201865 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=22016 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.514089577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=49474 payload_rows_remote=41282 dup_rows=16062 dup_rows_remote=16062 dedup_ratio=0.245087 router_tokens_per_expert=[0, 0, 0, 0, 3350, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 907, 0, 4092, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092] payload_rows_per_dst=[6708, 1822, 0, 8192, 8184, 8184, 8192, 8192] dup_rows_per_dst=[0, 50, 0, 4, 6642, 1232, 8134, 0] combine_rows_per_src=[6708, 1822, 0, 8192, 8184, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=214081536 dispatch_meta_bytes=209096 combine_payload_bytes=214081536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=214290632 ingress_bytes=232955904 link_bytes=232955904 token_bytes=428372168 weight_movement_bytes=18874368 total_nvlink_bytes=447246536 weight_to_token_ratio=0.044061 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.093750 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232068851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65316 payload_rows_remote=57148 dup_rows=220 dup_rows_remote=220 dedup_ratio=0.003357 router_tokens_per_expert=[0, 0, 0, 4004, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8010, 8192, 8192, 8168, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 26, 8, 0, 186, 0, 0, 0] combine_rows_per_src=[8010, 8192, 8192, 8168, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=231858176 dispatch_meta_bytes=229376 combine_payload_bytes=231858176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232087552 ingress_bytes=250732544 link_bytes=250732544 token_bytes=463945728 weight_movement_bytes=18874368 total_nvlink_bytes=482820096 weight_to_token_ratio=0.040682 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402395868e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=178470912 dispatch_meta_bytes=214128 combine_payload_bytes=178470912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=178685040 ingress_bytes=197345280 link_bytes=197345280 token_bytes=357155952 weight_movement_bytes=18874368 total_nvlink_bytes=376030320 weight_to_token_ratio=0.052846 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.890865373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=169091072 dispatch_meta_bytes=229376 combine_payload_bytes=169091072 prefetch_bytes=0 reduce_bytes=0 egress_bytes=169320448 ingress_bytes=169091072 link_bytes=169320448 token_bytes=338411520 weight_movement_bytes=0 total_nvlink_bytes=338411520 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.622345222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=229472 combine_payload_bytes=234078208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234307680 ingress_bytes=252952576 link_bytes=252952576 token_bytes=468385888 weight_movement_bytes=18874368 total_nvlink_bytes=487260256 weight_to_token_ratio=0.040297 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.423667123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64578 payload_rows_remote=56386 dup_rows=958 dup_rows_remote=958 dedup_ratio=0.014618 router_tokens_per_expert=[0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096] payload_rows_per_dst=[7244, 8192, 8192, 8192, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 18, 0, 0, 940, 0, 0, 0] combine_rows_per_src=[7244, 8192, 8192, 8192, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61060 payload_rows_remote=52868 dup_rows=4476 dup_rows_remote=4140 dedup_ratio=0.068298 router_tokens_per_expert=[0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0] payload_rows_per_dst=[7156, 4802, 8142, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2788, 0, 0, 1328, 336, 24] combine_rows_per_src=[7156, 4802, 8142, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57320 payload_rows_remote=49134 dup_rows=8216 dup_rows_remote=8216 dedup_ratio=0.125366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2727, 0, 3, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 3, 0, 3, 0, 0, 23, 0, 0, 0, 4093, 0, 0, 42, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8186, 8186, 0, 8186] dup_rows_per_dst=[5466, 0, 18, 130, 0, 0, 0, 2602] combine_rows_per_src=[8192, 8192, 8192, 8186, 8186, 8186, 0, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59536 payload_rows_remote=51344 dup_rows=6000 dup_rows_remote=5992 dedup_ratio=0.091553 router_tokens_per_expert=[0, 0, 0, 3245, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8192, 8192, 8132, 2268, 8192, 8192, 8180] dup_rows_per_dst=[5988, 8, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8132, 2268, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61500 payload_rows_remote=53308 dup_rows=4036 dup_rows_remote=4036 dedup_ratio=0.061584 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 4096, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2079, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 4158] dup_rows_per_dst=[2388, 0, 0, 1646, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8190, 4158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=230957056 dispatch_meta_bytes=229376 combine_payload_bytes=230957056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231186432 ingress_bytes=249831424 link_bytes=249831424 token_bytes=462143488 weight_movement_bytes=18874368 total_nvlink_bytes=481017856 weight_to_token_ratio=0.040841 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393761780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=216547328 dispatch_meta_bytes=228032 combine_payload_bytes=216547328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216775360 ingress_bytes=235421696 link_bytes=235421696 token_bytes=433322688 weight_movement_bytes=18874368 total_nvlink_bytes=452197056 weight_to_token_ratio=0.043557 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=25088 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.255694856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57628 payload_rows_remote=49436 dup_rows=7908 dup_rows_remote=24 dedup_ratio=0.120667 router_tokens_per_expert=[0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8178, 8192, 8192, 318, 8192, 8192, 8172] dup_rows_per_dst=[7884, 10, 0, 0, 0, 0, 14, 0] combine_rows_per_src=[8192, 8178, 8192, 8192, 318, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=201252864 dispatch_meta_bytes=229400 combine_payload_bytes=201252864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201482264 ingress_bytes=220127232 link_bytes=220127232 token_bytes=402735128 weight_movement_bytes=18874368 total_nvlink_bytes=421609496 weight_to_token_ratio=0.046865 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109150828e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=210305024 dispatch_meta_bytes=229344 combine_payload_bytes=210305024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=210534368 ingress_bytes=266928128 link_bytes=266928128 token_bytes=420839392 weight_movement_bytes=56623104 total_nvlink_bytes=477462496 weight_to_token_ratio=0.134548 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.557573986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58766 payload_rows_remote=50852 dup_rows=6770 dup_rows_remote=6266 dedup_ratio=0.103302 router_tokens_per_expert=[0, 0, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 4, 0, 0, 0, 0, 3064, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4092] payload_rows_per_dst=[1838, 8082, 8184, 8188, 8192, 8184, 8184, 7914] dup_rows_per_dst=[0, 8, 0, 96, 6148, 14, 0, 504] combine_rows_per_src=[1838, 8082, 8184, 8188, 8192, 8184, 8184, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59014 payload_rows_remote=50822 dup_rows=6522 dup_rows_remote=6252 dedup_ratio=0.099518 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 3782, 6080] dup_rows_per_dst=[3654, 0, 0, 270, 0, 10, 0, 2588] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 3782, 6080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=218349568 dispatch_meta_bytes=229376 combine_payload_bytes=218349568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=218578944 ingress_bytes=237223936 link_bytes=237223936 token_bytes=436928512 weight_movement_bytes=18874368 total_nvlink_bytes=455802880 weight_to_token_ratio=0.043198 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.272963033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=202489856 dispatch_meta_bytes=197840 combine_payload_bytes=202489856 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=202687696 ingress_bytes=259112960 link_bytes=259112960 token_bytes=405177552 weight_movement_bytes=56623104 total_nvlink_bytes=461800656 weight_to_token_ratio=0.139749 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=24832 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.482692891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=208289792 dispatch_meta_bytes=228472 combine_payload_bytes=208289792 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=208518264 ingress_bytes=283787264 link_bytes=283787264 token_bytes=416808056 weight_movement_bytes=75497472 total_nvlink_bytes=492305528 weight_to_token_ratio=0.181132 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.719109932e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=208166912 dispatch_meta_bytes=228296 combine_payload_bytes=208166912 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=208395208 ingress_bytes=283664384 link_bytes=283664384 token_bytes=416562120 weight_movement_bytes=75497472 total_nvlink_bytes=492059592 weight_to_token_ratio=0.181239 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.717932556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58622 payload_rows_remote=50458 dup_rows=6914 dup_rows_remote=6914 dedup_ratio=0.105499 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 0, 4096, 0, 0, 70, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1344, 8192, 8192, 8192, 8192, 8164, 8154, 8192] dup_rows_per_dst=[0, 0, 0, 1782, 0, 0, 0, 5132] combine_rows_per_src=[1344, 8192, 8192, 8192, 8192, 8164, 8154, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62910 payload_rows_remote=54718 dup_rows=2626 dup_rows_remote=2626 dedup_ratio=0.040070 router_tokens_per_expert=[0, 4093, 0, 4093, 3975, 0, 2, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3905, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3204, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 3976, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0] payload_rows_per_dst=[7960, 8192, 7816, 6414, 8192, 7958, 8192, 8186] dup_rows_per_dst=[4, 0, 0, 0, 0, 6, 2350, 266] combine_rows_per_src=[7960, 8192, 7816, 6414, 8192, 7958, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59070 payload_rows_remote=50968 dup_rows=6466 dup_rows_remote=4656 dedup_ratio=0.098663 router_tokens_per_expert=[0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8122, 3756, 6322, 8192, 8192, 8192, 8102, 8192] dup_rows_per_dst=[0, 8, 0, 44, 428, 4176, 1810, 0] combine_rows_per_src=[8122, 3756, 6322, 8192, 8192, 8192, 8102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61852 payload_rows_remote=53660 dup_rows=3684 dup_rows_remote=3684 dedup_ratio=0.056213 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8180, 4520] dup_rows_per_dst=[1574, 0, 0, 2098, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8180, 4520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64552 payload_rows_remote=56582 dup_rows=984 dup_rows_remote=984 dedup_ratio=0.015015 router_tokens_per_expert=[0, 4018, 0, 4096, 4072, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8036, 7970, 7748, 8192, 8078, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 964, 16] combine_rows_per_src=[8144, 8036, 7970, 7748, 8192, 8078, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=206675968 dispatch_meta_bytes=229488 combine_payload_bytes=206675968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206905456 ingress_bytes=225550336 link_bytes=225550336 token_bytes=413581424 weight_movement_bytes=18874368 total_nvlink_bytes=432455792 weight_to_token_ratio=0.045636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.161112342e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=224124928 dispatch_meta_bytes=229376 combine_payload_bytes=224124928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224354304 ingress_bytes=242999296 link_bytes=242999296 token_bytes=448479232 weight_movement_bytes=18874368 total_nvlink_bytes=467353600 weight_to_token_ratio=0.042085 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.328299691e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61176 payload_rows_remote=52984 dup_rows=4360 dup_rows_remote=1552 dedup_ratio=0.066528 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 4096, 0, 0, 551, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1923, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8178, 8192, 3846] dup_rows_per_dst=[2808, 0, 0, 1552, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8178, 8192, 3846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=208764928 dispatch_meta_bytes=222496 combine_payload_bytes=208764928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=208987424 ingress_bytes=227639296 link_bytes=227639296 token_bytes=417752352 weight_movement_bytes=18874368 total_nvlink_bytes=436626720 weight_to_token_ratio=0.045181 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=27776 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.181127729e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=219791360 dispatch_meta_bytes=229376 combine_payload_bytes=219791360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220020736 ingress_bytes=238665728 link_bytes=238665728 token_bytes=439812096 weight_movement_bytes=18874368 total_nvlink_bytes=458686464 weight_to_token_ratio=0.042915 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286777575e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64384 payload_rows_remote=57122 dup_rows=1152 dup_rows_remote=1152 dedup_ratio=0.017578 router_tokens_per_expert=[0, 4071, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1] payload_rows_per_dst=[8176, 8152, 8052, 7262, 8192, 8166, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1120, 32] combine_rows_per_src=[8176, 8152, 8052, 7262, 8192, 8166, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56154 payload_rows_remote=47970 dup_rows=9382 dup_rows_remote=8230 dedup_ratio=0.143158 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3427, 0, 4, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 4, 0, 0, 84, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6862, 8192, 8150, 216, 8184, 8182, 8184, 8184] dup_rows_per_dst=[16, 0, 24, 6, 0, 0, 8184, 1152] combine_rows_per_src=[6862, 8192, 8150, 216, 8184, 8182, 8184, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=231759872 dispatch_meta_bytes=230264 combine_payload_bytes=231759872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231990136 ingress_bytes=250634240 link_bytes=250634240 token_bytes=463750008 weight_movement_bytes=18874368 total_nvlink_bytes=482624376 weight_to_token_ratio=0.040699 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.401453968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=217022464 dispatch_meta_bytes=218144 combine_payload_bytes=217022464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=217240608 ingress_bytes=235896832 link_bytes=235896832 token_bytes=434263072 weight_movement_bytes=18874368 total_nvlink_bytes=453137440 weight_to_token_ratio=0.043463 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.260247376e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=233971712 dispatch_meta_bytes=233096 combine_payload_bytes=233971712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234204808 ingress_bytes=271720448 link_bytes=271720448 token_bytes=468176520 weight_movement_bytes=37748736 total_nvlink_bytes=505925256 weight_to_token_ratio=0.080629 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.603491638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=196485120 dispatch_meta_bytes=224800 combine_payload_bytes=196485120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196709920 ingress_bytes=215359488 link_bytes=215359488 token_bytes=393195040 weight_movement_bytes=18874368 total_nvlink_bytes=412069408 weight_to_token_ratio=0.048003 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=18688 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.063468651e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64078 payload_rows_remote=55920 dup_rows=1458 dup_rows_remote=1458 dedup_ratio=0.022247 router_tokens_per_expert=[0, 4096, 0, 4096, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8018, 8192, 8034, 7100, 8192, 8158, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1452, 6] combine_rows_per_src=[8018, 8192, 8034, 7100, 8192, 8158, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64998 payload_rows_remote=57324 dup_rows=538 dup_rows_remote=532 dedup_ratio=0.008209 router_tokens_per_expert=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 4093, 0, 0, 0, 260, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 7674, 8186, 8186, 8192] dup_rows_per_dst=[0, 6, 6, 0, 6, 0, 0, 520] combine_rows_per_src=[8184, 8192, 8192, 8192, 7674, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64778 payload_rows_remote=56586 dup_rows=758 dup_rows_remote=758 dedup_ratio=0.011566 router_tokens_per_expert=[0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7474, 8192, 8192, 8190, 8192, 8192, 8192, 8154] dup_rows_per_dst=[0, 4, 0, 0, 754, 0, 0, 0] combine_rows_per_src=[7474, 8192, 8192, 8190, 8192, 8192, 8192, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59324 payload_rows_remote=51468 dup_rows=6212 dup_rows_remote=6212 dedup_ratio=0.094788 router_tokens_per_expert=[0, 3928, 0, 4096, 3837, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0] payload_rows_per_dst=[7680, 7856, 7892, 8188, 8192, 7340, 3984, 8192] dup_rows_per_dst=[10, 0, 0, 4020, 0, 2134, 6, 42] combine_rows_per_src=[7680, 7856, 7892, 8188, 8192, 7340, 3984, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=229048320 dispatch_meta_bytes=229512 combine_payload_bytes=229048320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=229277832 ingress_bytes=247922688 link_bytes=247922688 token_bytes=458326152 weight_movement_bytes=18874368 total_nvlink_bytes=477200520 weight_to_token_ratio=0.041181 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.375473211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=231424 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235030528 ingress_bytes=272547840 link_bytes=272547840 token_bytes=469829632 weight_movement_bytes=37748736 total_nvlink_bytes=507578368 weight_to_token_ratio=0.080346 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611419301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57464 payload_rows_remote=49292 dup_rows=8072 dup_rows_remote=6150 dedup_ratio=0.123169 router_tokens_per_expert=[0, 4095, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8190, 7864, 7542, 8192, 8184, 1772, 7548] dup_rows_per_dst=[1922, 0, 2226, 3924, 0, 0, 0, 0] combine_rows_per_src=[8172, 8190, 7864, 7542, 8192, 8184, 1772, 7548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=231776256 dispatch_meta_bytes=229376 combine_payload_bytes=231776256 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=232005632 ingress_bytes=269524992 link_bytes=269524992 token_bytes=463781888 weight_movement_bytes=37748736 total_nvlink_bytes=501530624 weight_to_token_ratio=0.081393 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.582455859e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=210812928 dispatch_meta_bytes=230720 combine_payload_bytes=210812928 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=211043648 ingress_bytes=248561664 link_bytes=248561664 token_bytes=421856576 weight_movement_bytes=37748736 total_nvlink_bytes=459605312 weight_to_token_ratio=0.089482 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.381595564e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 6] combine_rows_per_src=[8184, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60804 payload_rows_remote=52620 dup_rows=4732 dup_rows_remote=4438 dedup_ratio=0.072205 router_tokens_per_expert=[0, 4081, 0, 4092, 3994, 0, 4, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3977, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 3721, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4092, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0] payload_rows_per_dst=[7406, 8170, 7962, 5248, 8192, 7450, 8192, 8184] dup_rows_per_dst=[4, 598, 0, 0, 0, 8, 3828, 294] combine_rows_per_src=[7406, 8170, 7962, 5248, 8192, 7450, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=201900032 dispatch_meta_bytes=221768 combine_payload_bytes=201900032 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=202121800 ingress_bytes=277397504 link_bytes=277397504 token_bytes=404021832 weight_movement_bytes=75497472 total_nvlink_bytes=479519304 weight_to_token_ratio=0.186865 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.657886395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=253722624 link_bytes=253722624 token_bytes=469925888 weight_movement_bytes=18874368 total_nvlink_bytes=488800256 weight_to_token_ratio=0.040165 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431045344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=215531520 dispatch_meta_bytes=228232 combine_payload_bytes=215531520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=215759752 ingress_bytes=234405888 link_bytes=234405888 token_bytes=431291272 weight_movement_bytes=18874368 total_nvlink_bytes=450165640 weight_to_token_ratio=0.043762 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.245961884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65070 payload_rows_remote=56878 dup_rows=466 dup_rows_remote=454 dedup_ratio=0.007111 router_tokens_per_expert=[3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4093, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 4093, 114, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8158, 8192, 8192, 8186, 7782, 8192] dup_rows_per_dst=[0, 0, 0, 0, 12, 212, 6, 236] combine_rows_per_src=[8192, 8176, 8158, 8192, 8192, 8186, 7782, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=55920 payload_rows_remote=47728 dup_rows=9616 dup_rows_remote=5138 dedup_ratio=0.146729 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[464, 8192, 8192, 6310, 8192, 8190, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 2380, 14, 36, 4478, 2708] combine_rows_per_src=[464, 8192, 8192, 6310, 8192, 8190, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 12, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65524 payload_rows_remote=57332 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=232972288 dispatch_meta_bytes=229328 combine_payload_bytes=232972288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233201616 ingress_bytes=251846656 link_bytes=251846656 token_bytes=466173904 weight_movement_bytes=18874368 total_nvlink_bytes=485048272 weight_to_token_ratio=0.040488 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.413070742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64574 payload_rows_remote=56382 dup_rows=962 dup_rows_remote=938 dedup_ratio=0.014679 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 7244] dup_rows_per_dst=[24, 0, 0, 0, 892, 14, 32, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8178, 7244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57330 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 8186, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 2, 0, 16] combine_rows_per_src=[8192, 8178, 8192, 8186, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=195493888 dispatch_meta_bytes=211464 combine_payload_bytes=195493888 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=195705352 ingress_bytes=270991360 link_bytes=270991360 token_bytes=391199240 weight_movement_bytes=75497472 total_nvlink_bytes=466696712 weight_to_token_ratio=0.192990 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=14208 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.596505875e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235061248 ingress_bytes=272580608 link_bytes=272580608 token_bytes=469893120 weight_movement_bytes=37748736 total_nvlink_bytes=507641856 weight_to_token_ratio=0.080335 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611733268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65082 payload_rows_remote=56890 dup_rows=454 dup_rows_remote=44 dedup_ratio=0.006927 router_tokens_per_expert=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 4, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 4092, 0, 0, 0, 205, 4092, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8178, 8178, 8192, 7794, 8184, 8180, 8192] dup_rows_per_dst=[0, 8, 8, 0, 22, 0, 6, 410] combine_rows_per_src=[8184, 8178, 8178, 8192, 7794, 8184, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=229376 combine_payload_bytes=234831872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235061248 ingress_bytes=253706240 link_bytes=253706240 token_bytes=469893120 weight_movement_bytes=18874368 total_nvlink_bytes=488767488 weight_to_token_ratio=0.040167 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430888360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=230940672 dispatch_meta_bytes=229280 combine_payload_bytes=230940672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=231169952 ingress_bytes=249815040 link_bytes=249815040 token_bytes=462110624 weight_movement_bytes=18874368 total_nvlink_bytes=480984992 weight_to_token_ratio=0.040844 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.393604797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229400 combine_payload_bytes=234823680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235053080 ingress_bytes=253698048 link_bytes=253698048 token_bytes=469876760 weight_movement_bytes=18874368 total_nvlink_bytes=488751128 weight_to_token_ratio=0.040169 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430809869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=233021440 dispatch_meta_bytes=227736 combine_payload_bytes=233021440 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=233249176 ingress_bytes=289644544 link_bytes=289644544 token_bytes=466270616 weight_movement_bytes=56623104 total_nvlink_bytes=522893720 weight_to_token_ratio=0.121438 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.775231507e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64814 payload_rows_remote=56622 dup_rows=722 dup_rows_remote=716 dedup_ratio=0.011017 router_tokens_per_expert=[0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 4093, 162, 0, 22, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8184, 8186, 7502, 8192, 8192, 8192, 8192, 8174] dup_rows_per_dst=[336, 0, 0, 6, 6, 330, 44, 0] combine_rows_per_src=[8184, 8186, 7502, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64624 payload_rows_remote=56432 dup_rows=912 dup_rows_remote=4 dedup_ratio=0.013916 router_tokens_per_expert=[0, 4074, 0, 4096, 4017, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8034, 8150, 7720, 8090, 8192, 8054, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 908, 2] combine_rows_per_src=[8034, 8150, 7720, 8090, 8192, 8054, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=235102208 ingress_bytes=310370304 link_bytes=310370304 token_bytes=469975040 weight_movement_bytes=75497472 total_nvlink_bytes=545472512 weight_to_token_ratio=0.160641 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973815541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57340 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 20, 0, 2] combine_rows_per_src=[8192, 8172, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=231923712 dispatch_meta_bytes=229352 combine_payload_bytes=231923712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232153064 ingress_bytes=250798080 link_bytes=250798080 token_bytes=464076776 weight_movement_bytes=18874368 total_nvlink_bytes=482951144 weight_to_token_ratio=0.040671 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403023802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64782 payload_rows_remote=56590 dup_rows=754 dup_rows_remote=754 dedup_ratio=0.011505 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8158, 8052, 8192, 7612] dup_rows_per_dst=[0, 114, 50, 10, 0, 0, 580, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8158, 8052, 8192, 7612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8192, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 16, 0] combine_rows_per_src=[8182, 8192, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=231145472 dispatch_meta_bytes=225744 combine_payload_bytes=231145472 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=231371216 ingress_bytes=306642944 link_bytes=306642944 token_bytes=462516688 weight_movement_bytes=75497472 total_nvlink_bytes=538014160 weight_to_token_ratio=0.163232 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.938101812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65078 payload_rows_remote=56886 dup_rows=458 dup_rows_remote=138 dedup_ratio=0.006989 router_tokens_per_expert=[4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4092, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 1, 0, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 6, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 23, 0, 4092, 35, 0, 0, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8144, 8178, 8192, 8192, 7980, 8008, 8192] dup_rows_per_dst=[0, 0, 0, 0, 18, 112, 8, 320] combine_rows_per_src=[8192, 8144, 8178, 8192, 8192, 7980, 8008, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235102208 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975040 weight_movement_bytes=37748736 total_nvlink_bytes=507723776 weight_to_token_ratio=0.080321 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229456 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094096 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958736 weight_movement_bytes=18874368 total_nvlink_bytes=488833104 weight_to_token_ratio=0.040162 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=231792640 dispatch_meta_bytes=229376 combine_payload_bytes=231792640 prefetch_bytes=0 reduce_bytes=0 egress_bytes=232022016 ingress_bytes=231792640 link_bytes=232022016 token_bytes=463814656 weight_movement_bytes=0 total_nvlink_bytes=463814656 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223120795e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229376 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235044864 ingress_bytes=272564224 link_bytes=272564224 token_bytes=469860352 weight_movement_bytes=37748736 total_nvlink_bytes=507609088 weight_to_token_ratio=0.080340 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611576284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=233005056 dispatch_meta_bytes=228096 combine_payload_bytes=233005056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233233152 ingress_bytes=270753792 link_bytes=270753792 token_bytes=466238208 weight_movement_bytes=37748736 total_nvlink_bytes=503986944 weight_to_token_ratio=0.080964 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.594229616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65224 payload_rows_remote=57034 dup_rows=312 dup_rows_remote=306 dedup_ratio=0.004761 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 3949, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 138, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8186, 7904, 8190, 8186, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 6, 6, 300, 0, 0] combine_rows_per_src=[8184, 8192, 8186, 7904, 8190, 8186, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63158 payload_rows_remote=54978 dup_rows=2378 dup_rows_remote=2378 dedup_ratio=0.036285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8192, 8192, 5826, 8192, 8192] dup_rows_per_dst=[12, 2, 0, 2322, 22, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8180, 8192, 8192, 5826, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=229352 combine_payload_bytes=234856448 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235085800 ingress_bytes=310353920 link_bytes=310353920 token_bytes=469942248 weight_movement_bytes=75497472 total_nvlink_bytes=545439720 weight_to_token_ratio=0.160653 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973658558e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=233611264 dispatch_meta_bytes=229360 combine_payload_bytes=233611264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233840624 ingress_bytes=252485632 link_bytes=252485632 token_bytes=467451888 weight_movement_bytes=18874368 total_nvlink_bytes=486326256 weight_to_token_ratio=0.040377 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.419193095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 24, 0] combine_rows_per_src=[8168, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=235110400 ingress_bytes=329252864 link_bytes=329252864 token_bytes=469991424 weight_movement_bytes=94371840 total_nvlink_bytes=564363264 weight_to_token_ratio=0.200795 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.154738940e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60750 payload_rows_remote=56846 dup_rows=4786 dup_rows_remote=4340 dedup_ratio=0.073029 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2395, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 3904, 8190, 7714, 8186, 8192] dup_rows_per_dst=[10, 6, 0, 446, 26, 4292, 6, 0] combine_rows_per_src=[8192, 8192, 8180, 3904, 8190, 7714, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64260 payload_rows_remote=56068 dup_rows=1276 dup_rows_remote=1140 dedup_ratio=0.019470 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 7072, 8050, 8192] dup_rows_per_dst=[136, 14, 1116, 10, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 7072, 8050, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=225189888 dispatch_meta_bytes=229424 combine_payload_bytes=225189888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=225419312 ingress_bytes=244064256 link_bytes=244064256 token_bytes=450609200 weight_movement_bytes=18874368 total_nvlink_bytes=469483568 weight_to_token_ratio=0.041886 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.338503614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64438 payload_rows_remote=56252 dup_rows=1098 dup_rows_remote=1074 dedup_ratio=0.016754 router_tokens_per_expert=[0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 4092, 93, 0, 27, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[7552, 8184, 7772, 8168, 8192, 8192, 8192, 8186] dup_rows_per_dst=[214, 0, 0, 8, 8, 790, 54, 24] combine_rows_per_src=[7552, 8184, 7772, 8168, 8192, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234782720 dispatch_meta_bytes=229376 combine_payload_bytes=234782720 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235012096 ingress_bytes=272531456 link_bytes=272531456 token_bytes=469794816 weight_movement_bytes=37748736 total_nvlink_bytes=507543552 weight_to_token_ratio=0.080352 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611262318e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=232841216 dispatch_meta_bytes=244744 combine_payload_bytes=232841216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=233085960 ingress_bytes=270589952 link_bytes=270589952 token_bytes=465927176 weight_movement_bytes=37748736 total_nvlink_bytes=503675912 weight_to_token_ratio=0.081019 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.592659781e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=229654528 dispatch_meta_bytes=228832 combine_payload_bytes=229654528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=229883360 ingress_bytes=267403264 link_bytes=267403264 token_bytes=459537888 weight_movement_bytes=37748736 total_nvlink_bytes=497286624 weight_to_token_ratio=0.082145 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.562126505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=230408192 dispatch_meta_bytes=229304 combine_payload_bytes=230408192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230637496 ingress_bytes=249282560 link_bytes=249282560 token_bytes=461045688 weight_movement_bytes=18874368 total_nvlink_bytes=479920056 weight_to_token_ratio=0.040938 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.388502835e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61042 payload_rows_remote=52850 dup_rows=4494 dup_rows_remote=0 dedup_ratio=0.068573 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3698, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4494, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 3698, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61834 payload_rows_remote=53648 dup_rows=3702 dup_rows_remote=3700 dedup_ratio=0.056488 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 3, 0, 4083, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 614, 0, 0, 4093] payload_rows_per_dst=[8192, 8178, 8192, 8100, 8186, 8186, 8172, 4628] dup_rows_per_dst=[0, 0, 6, 0, 2, 3656, 18, 20] combine_rows_per_src=[8192, 8178, 8192, 8100, 8186, 8186, 8172, 4628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8188, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61412 payload_rows_remote=53220 dup_rows=4124 dup_rows_remote=4124 dedup_ratio=0.062927 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 70, 0, 0, 0, 0, 0, 0, 0, 1991, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8190, 8192, 4562, 8192, 8192, 7794, 8098] dup_rows_per_dst=[0, 0, 0, 0, 2, 4122, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 4562, 8192, 8192, 7794, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=216473600 dispatch_meta_bytes=211400 combine_payload_bytes=216473600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=216685000 ingress_bytes=235347968 link_bytes=235347968 token_bytes=433158600 weight_movement_bytes=18874368 total_nvlink_bytes=452032968 weight_to_token_ratio=0.043574 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=20096 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.254988431e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=219742208 dispatch_meta_bytes=229392 combine_payload_bytes=219742208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=219971600 ingress_bytes=238616576 link_bytes=238616576 token_bytes=439713808 weight_movement_bytes=18874368 total_nvlink_bytes=458588176 weight_to_token_ratio=0.042924 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.286306625e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59712 payload_rows_remote=51520 dup_rows=5824 dup_rows_remote=5824 dedup_ratio=0.088867 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8192, 8192, 2388, 8192, 8192] dup_rows_per_dst=[20, 0, 0, 5708, 90, 0, 6, 0] combine_rows_per_src=[8192, 8192, 8172, 8192, 8192, 2388, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60082 payload_rows_remote=55306 dup_rows=5454 dup_rows_remote=5454 dedup_ratio=0.083221 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8188, 8186, 4776, 8192, 8192, 8086, 6270] dup_rows_per_dst=[0, 0, 0, 0, 10, 5444, 0, 0] combine_rows_per_src=[8192, 8188, 8186, 4776, 8192, 8192, 8086, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094016 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958656 weight_movement_bytes=18874368 total_nvlink_bytes=488833024 weight_to_token_ratio=0.040162 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62680 payload_rows_remote=54488 dup_rows=2856 dup_rows_remote=2690 dedup_ratio=0.043579 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2788, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8036, 5578, 8120, 8192] dup_rows_per_dst=[166, 0, 72, 2618, 0, 0, 0, 0] combine_rows_per_src=[8192, 8180, 8190, 8192, 8036, 5578, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=217989120 dispatch_meta_bytes=229376 combine_payload_bytes=217989120 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=218218496 ingress_bytes=255737856 link_bytes=255737856 token_bytes=436207616 weight_movement_bytes=37748736 total_nvlink_bytes=473956352 weight_to_token_ratio=0.086538 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.450354305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65226 payload_rows_remote=57034 dup_rows=310 dup_rows_remote=308 dedup_ratio=0.004730 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4092, 0, 4046, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7992, 8192, 8184, 8100, 8192, 8184, 8190, 8192] dup_rows_per_dst=[0, 2, 0, 180, 12, 88, 26, 2] combine_rows_per_src=[7992, 8192, 8184, 8100, 8192, 8184, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=211025920 dispatch_meta_bytes=229376 combine_payload_bytes=211025920 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=211255296 ingress_bytes=229900288 link_bytes=229900288 token_bytes=422281216 weight_movement_bytes=18874368 total_nvlink_bytes=441155584 weight_to_token_ratio=0.044696 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.202791442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=226533376 dispatch_meta_bytes=243040 combine_payload_bytes=226533376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=226776416 ingress_bytes=283156480 link_bytes=283156480 token_bytes=453309792 weight_movement_bytes=56623104 total_nvlink_bytes=509932896 weight_to_token_ratio=0.124910 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=26496 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.713066070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=223182848 dispatch_meta_bytes=228712 combine_payload_bytes=223182848 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=223411560 ingress_bytes=298680320 link_bytes=298680320 token_bytes=446594408 weight_movement_bytes=75497472 total_nvlink_bytes=522091880 weight_to_token_ratio=0.169052 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.861807866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=233611264 dispatch_meta_bytes=229368 combine_payload_bytes=233611264 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=233840632 ingress_bytes=271360000 link_bytes=271360000 token_bytes=467451896 weight_movement_bytes=37748736 total_nvlink_bytes=505200632 weight_to_token_ratio=0.080754 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.600038003e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60688 payload_rows_remote=53984 dup_rows=4848 dup_rows_remote=1798 dedup_ratio=0.073975 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3446, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 7572, 8192, 6704, 6752, 6892] dup_rows_per_dst=[0, 0, 0, 0, 0, 3050, 1798, 0] combine_rows_per_src=[8192, 8192, 8192, 7572, 8192, 6704, 6752, 6892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65420 payload_rows_remote=57228 dup_rows=116 dup_rows_remote=6 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 52, 4093, 0, 0, 0, 4092, 0, 0, 0, 3, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[8192, 8184, 8186, 8102, 8192, 8180, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6, 110, 0, 0, 0] combine_rows_per_src=[8192, 8184, 8186, 8102, 8192, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65452 payload_rows_remote=57260 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.001282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 4026, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8192, 8152, 8192, 8180, 8176, 8192] dup_rows_per_dst=[0, 2, 0, 0, 82, 0, 0, 0] combine_rows_per_src=[8192, 8176, 8192, 8152, 8192, 8180, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=221118464 dispatch_meta_bytes=223128 combine_payload_bytes=221118464 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=221341592 ingress_bytes=296615936 link_bytes=296615936 token_bytes=442460056 weight_movement_bytes=75497472 total_nvlink_bytes=517957528 weight_to_token_ratio=0.170631 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.104167 gemm_active_groups=5 gemm_peak_group_rows=17408 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.842027955e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234405888 dispatch_meta_bytes=228936 combine_payload_bytes=234405888 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=234634824 ingress_bytes=309903360 link_bytes=309903360 token_bytes=469040712 weight_movement_bytes=75497472 total_nvlink_bytes=544538184 weight_to_token_ratio=0.160961 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.969341514e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61104 payload_rows_remote=52914 dup_rows=4432 dup_rows_remote=4432 dedup_ratio=0.067627 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 146, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 0, 0, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 12, 0, 0, 4096] payload_rows_per_dst=[8192, 8190, 8188, 8160, 8192, 5694, 6336, 8152] dup_rows_per_dst=[0, 0, 0, 4326, 4, 78, 0, 24] combine_rows_per_src=[8192, 8190, 8188, 8160, 8192, 5694, 6336, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229368 combine_payload_bytes=234872832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235102200 ingress_bytes=272621568 link_bytes=272621568 token_bytes=469975032 weight_movement_bytes=37748736 total_nvlink_bytes=507723768 weight_to_token_ratio=0.080321 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612125726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57296 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 23, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8188, 8192, 8168, 8172, 8192] dup_rows_per_dst=[2, 2, 0, 0, 48, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8188, 8192, 8168, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61872 payload_rows_remote=53680 dup_rows=3664 dup_rows_remote=3664 dedup_ratio=0.055908 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7212, 8006, 5694] dup_rows_per_dst=[0, 42, 0, 3622, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7212, 8006, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234536960 dispatch_meta_bytes=229376 combine_payload_bytes=234536960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234766336 ingress_bytes=253411328 link_bytes=253411328 token_bytes=469303296 weight_movement_bytes=18874368 total_nvlink_bytes=488177664 weight_to_token_ratio=0.040218 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428062659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61174 payload_rows_remote=52990 dup_rows=4362 dup_rows_remote=40 dedup_ratio=0.066559 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 0, 0, 0, 0, 4, 0, 4057, 0, 0, 0, 4, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 0, 308, 0, 0, 4092] payload_rows_per_dst=[8192, 8172, 8188, 8110, 8176, 4030, 8122, 8184] dup_rows_per_dst=[0, 0, 8, 0, 4, 4, 24, 4322] combine_rows_per_src=[8192, 8172, 8188, 8110, 8176, 4030, 8122, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=216735744 dispatch_meta_bytes=229384 combine_payload_bytes=216735744 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=216965128 ingress_bytes=254484480 link_bytes=254484480 token_bytes=433700872 weight_movement_bytes=37748736 total_nvlink_bytes=471449608 weight_to_token_ratio=0.087039 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.438345073e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234684416 dispatch_meta_bytes=229392 combine_payload_bytes=234684416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234913808 ingress_bytes=272433152 link_bytes=272433152 token_bytes=469598224 weight_movement_bytes=37748736 total_nvlink_bytes=507346960 weight_to_token_ratio=0.080385 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.610320417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=219873280 dispatch_meta_bytes=229376 combine_payload_bytes=219873280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220102656 ingress_bytes=238747648 link_bytes=238747648 token_bytes=439975936 weight_movement_bytes=18874368 total_nvlink_bytes=458850304 weight_to_token_ratio=0.042899 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.287562492e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=217047040 dispatch_meta_bytes=212120 combine_payload_bytes=217047040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=217259160 ingress_bytes=254795776 link_bytes=254795776 token_bytes=434306200 weight_movement_bytes=37748736 total_nvlink_bytes=472054936 weight_to_token_ratio=0.086917 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=25216 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.441327758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65468 payload_rows_remote=57292 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 4005, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8174, 8192, 8176, 8164, 8192] dup_rows_per_dst=[14, 0, 0, 0, 54, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8174, 8192, 8176, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65376 payload_rows_remote=57184 dup_rows=160 dup_rows_remote=154 dedup_ratio=0.002441 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 71, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4090, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8044, 8192, 8192, 8192, 8186, 8186, 8192] dup_rows_per_dst=[142, 0, 0, 6, 6, 6, 0, 0] combine_rows_per_src=[8192, 8044, 8192, 8192, 8192, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63406 payload_rows_remote=55214 dup_rows=2130 dup_rows_remote=2108 dedup_ratio=0.032501 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 6074, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 4, 2102, 22, 0] combine_rows_per_src=[8192, 8192, 8180, 6074, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65258 payload_rows_remote=57066 dup_rows=278 dup_rows_remote=278 dedup_ratio=0.004242 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7914, 8192] dup_rows_per_dst=[24, 0, 0, 0, 0, 254, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7914, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229440 combine_payload_bytes=234668032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234897472 ingress_bytes=291291136 link_bytes=291291136 token_bytes=469565504 weight_movement_bytes=56623104 total_nvlink_bytes=526188608 weight_to_token_ratio=0.120586 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.791008341e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234225664 dispatch_meta_bytes=229352 combine_payload_bytes=234225664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234455016 ingress_bytes=253100032 link_bytes=253100032 token_bytes=468680680 weight_movement_bytes=18874368 total_nvlink_bytes=487555048 weight_to_token_ratio=0.040271 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.425079974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57282 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8130, 8192] dup_rows_per_dst=[10, 0, 2, 0, 0, 50, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64684 payload_rows_remote=56574 dup_rows=852 dup_rows_remote=840 dedup_ratio=0.013000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 427, 4088, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8110, 8192, 8180, 8192, 7736, 7890, 8192] dup_rows_per_dst=[0, 12, 0, 0, 840, 0, 0, 0] combine_rows_per_src=[8192, 8110, 8192, 8180, 8192, 7736, 7890, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=226156544 dispatch_meta_bytes=229288 combine_payload_bytes=226156544 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=226385832 ingress_bytes=301654016 link_bytes=301654016 token_bytes=452542376 weight_movement_bytes=75497472 total_nvlink_bytes=528039848 weight_to_token_ratio=0.166830 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.890300358e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=233742336 dispatch_meta_bytes=229376 combine_payload_bytes=233742336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233971712 ingress_bytes=252616704 link_bytes=252616704 token_bytes=467714048 weight_movement_bytes=18874368 total_nvlink_bytes=486588416 weight_to_token_ratio=0.040355 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.420448963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65180 payload_rows_remote=56988 dup_rows=356 dup_rows_remote=354 dedup_ratio=0.005432 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4092, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8180, 7968, 8072, 8192, 8192] dup_rows_per_dst=[2, 118, 0, 224, 12, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8180, 7968, 8072, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64350 payload_rows_remote=56158 dup_rows=1186 dup_rows_remote=1186 dedup_ratio=0.018097 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4092, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 39, 4092, 0, 0, 0, 4085, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8170, 8170, 8184, 8130, 7126, 8188, 8190, 8192] dup_rows_per_dst=[4, 0, 14, 8, 78, 658, 424, 0] combine_rows_per_src=[8170, 8170, 8184, 8130, 7126, 8188, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=231727104 dispatch_meta_bytes=229656 combine_payload_bytes=231727104 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=231956760 ingress_bytes=269475840 link_bytes=269475840 token_bytes=463683864 weight_movement_bytes=37748736 total_nvlink_bytes=501432600 weight_to_token_ratio=0.081411 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.581984908e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234627072 dispatch_meta_bytes=229376 combine_payload_bytes=234627072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234856448 ingress_bytes=253501440 link_bytes=253501440 token_bytes=469483520 weight_movement_bytes=18874368 total_nvlink_bytes=488357888 weight_to_token_ratio=0.040202 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428926068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=233422848 dispatch_meta_bytes=229368 combine_payload_bytes=233422848 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=233652216 ingress_bytes=271171584 link_bytes=271171584 token_bytes=467075064 weight_movement_bytes=37748736 total_nvlink_bytes=504823800 weight_to_token_ratio=0.080819 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.598232693e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=10 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8172, 8192] dup_rows_per_dst=[10, 0, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=230023168 dispatch_meta_bytes=229376 combine_payload_bytes=230023168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=230252544 ingress_bytes=248897536 link_bytes=248897536 token_bytes=460275712 weight_movement_bytes=18874368 total_nvlink_bytes=479150080 weight_to_token_ratio=0.041007 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.384813725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59974 payload_rows_remote=51782 dup_rows=5562 dup_rows_remote=5562 dedup_ratio=0.084869 router_tokens_per_expert=[0, 0, 0, 0, 0, 104, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 2659, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 102, 0, 0, 0, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4089, 0, 0, 0, 0, 3881, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4088, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7922, 3340, 8192, 8186, 7768, 8182] dup_rows_per_dst=[208, 5322, 14, 12, 0, 0, 0, 6] combine_rows_per_src=[8192, 8192, 7922, 3340, 8192, 8186, 7768, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57880 payload_rows_remote=49986 dup_rows=7656 dup_rows_remote=5122 dedup_ratio=0.116821 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 8192, 8116, 8192, 952, 7894, 8150] dup_rows_per_dst=[0, 0, 0, 0, 0, 20, 2534, 5102] combine_rows_per_src=[8192, 8192, 8192, 8116, 8192, 952, 7894, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59880 payload_rows_remote=51690 dup_rows=5656 dup_rows_remote=5600 dedup_ratio=0.086304 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 41, 0, 0, 1, 0, 0, 0, 0, 1301, 0, 0, 0, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8190, 5240, 8192, 8192, 5494, 8192] dup_rows_per_dst=[14, 3038, 56, 2254, 282, 12, 0, 0] combine_rows_per_src=[8192, 8188, 8190, 5240, 8192, 8192, 5494, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229336 combine_payload_bytes=234799104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=235028440 ingress_bytes=272547840 link_bytes=272547840 token_bytes=469827544 weight_movement_bytes=37748736 total_nvlink_bytes=507576280 weight_to_token_ratio=0.080346 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.611419301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=212099072 dispatch_meta_bytes=229376 combine_payload_bytes=212099072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=212328448 ingress_bytes=230973440 link_bytes=230973440 token_bytes=424427520 weight_movement_bytes=18874368 total_nvlink_bytes=443301888 weight_to_token_ratio=0.044470 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.213073856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64786 payload_rows_remote=57310 dup_rows=750 dup_rows_remote=744 dedup_ratio=0.011444 router_tokens_per_expert=[0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 16, 0, 0, 10, 0, 0, 0, 0, 1216, 0, 0, 0, 2525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 4082, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7476, 8188, 8192, 8164, 8192] dup_rows_per_dst=[116, 468, 34, 6, 124, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 7476, 8188, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=204742656 dispatch_meta_bytes=220432 combine_payload_bytes=204742656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=204963088 ingress_bytes=223617024 link_bytes=223617024 token_bytes=409705744 weight_movement_bytes=18874368 total_nvlink_bytes=428580112 weight_to_token_ratio=0.046068 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.097222 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.142588298e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64848 payload_rows_remote=56656 dup_rows=688 dup_rows_remote=688 dedup_ratio=0.010498 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 3984, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7504, 8192] dup_rows_per_dst=[56, 0, 0, 0, 0, 632, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7504, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=211722240 dispatch_meta_bytes=229160 combine_payload_bytes=211722240 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=211951400 ingress_bytes=268345344 link_bytes=268345344 token_bytes=423673640 weight_movement_bytes=56623104 total_nvlink_bytes=480296744 weight_to_token_ratio=0.133648 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.571153052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65474 payload_rows_remote=57282 dup_rows=62 dup_rows_remote=8 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8144, 8178] dup_rows_per_dst=[54, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8144, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63932 payload_rows_remote=55740 dup_rows=1604 dup_rows_remote=1590 dedup_ratio=0.024475 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 25, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4091, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4086, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4092, 0, 0, 0, 0, 0] payload_rows_per_dst=[7814, 8086, 8190, 8178, 8178, 7114, 8180, 8192] dup_rows_per_dst=[50, 254, 0, 8, 8, 60, 1210, 14] combine_rows_per_src=[7814, 8086, 8190, 8178, 8178, 7114, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=232216 combine_payload_bytes=234741760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234973976 ingress_bytes=253616128 link_bytes=253616128 token_bytes=469715736 weight_movement_bytes=18874368 total_nvlink_bytes=488590104 weight_to_token_ratio=0.040183 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=14336 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430024952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=232062976 dispatch_meta_bytes=229376 combine_payload_bytes=232062976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232292352 ingress_bytes=250937344 link_bytes=250937344 token_bytes=464355328 weight_movement_bytes=18874368 total_nvlink_bytes=483229696 weight_to_token_ratio=0.040646 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404358161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62026 payload_rows_remote=53834 dup_rows=3510 dup_rows_remote=3510 dedup_ratio=0.053558 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 427, 0, 0, 0, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6570, 8190, 6312, 8192, 8192, 8188, 8190] dup_rows_per_dst=[24, 954, 18, 6, 62, 0, 2446, 0] combine_rows_per_src=[8192, 6570, 8190, 6312, 8192, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234627072 dispatch_meta_bytes=229160 combine_payload_bytes=234627072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234856232 ingress_bytes=291250176 link_bytes=291250176 token_bytes=469483304 weight_movement_bytes=56623104 total_nvlink_bytes=526106408 weight_to_token_ratio=0.120607 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.790615882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=228311040 dispatch_meta_bytes=229320 combine_payload_bytes=228311040 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=228540360 ingress_bytes=266059776 link_bytes=266059776 token_bytes=456851400 weight_movement_bytes=37748736 total_nvlink_bytes=494600136 weight_to_token_ratio=0.082628 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.549253864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=62974 payload_rows_remote=54782 dup_rows=2562 dup_rows_remote=2556 dedup_ratio=0.039093 router_tokens_per_expert=[3, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1273, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8190, 6476, 8192, 7424, 8192, 8118, 8190, 8192] dup_rows_per_dst=[2, 0, 2546, 0, 6, 0, 0, 8] combine_rows_per_src=[8190, 6476, 8192, 7424, 8192, 8118, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57328 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4054, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8178, 8136, 8192] dup_rows_per_dst=[0, 4, 0, 0, 68, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8178, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=220504064 dispatch_meta_bytes=229376 combine_payload_bytes=220504064 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=220733440 ingress_bytes=258252800 link_bytes=258252800 token_bytes=441237504 weight_movement_bytes=37748736 total_nvlink_bytes=478986240 weight_to_token_ratio=0.085552 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.474451261e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=224387072 dispatch_meta_bytes=229352 combine_payload_bytes=224387072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=224616424 ingress_bytes=243261440 link_bytes=243261440 token_bytes=449003496 weight_movement_bytes=18874368 total_nvlink_bytes=467877864 weight_to_token_ratio=0.042036 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.330811426e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=234815488 dispatch_meta_bytes=229600 combine_payload_bytes=234815488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235045088 ingress_bytes=253689856 link_bytes=253689856 token_bytes=469860576 weight_movement_bytes=18874368 total_nvlink_bytes=488734944 weight_to_token_ratio=0.040170 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.430731377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65448 payload_rows_remote=57340 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.001343 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 42, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8108, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 84, 0, 2, 0, 0, 2] combine_rows_per_src=[8192, 8188, 8192, 8108, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57282 payload_rows_remote=49120 dup_rows=8254 dup_rows_remote=6070 dedup_ratio=0.125946 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3926, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 2232, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 4041, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8162, 8170, 8182, 8192, 8192, 0, 8192] dup_rows_per_dst=[18, 2184, 362, 5690, 0, 0, 0, 0] combine_rows_per_src=[8192, 8162, 8170, 8182, 8192, 8192, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235102208 ingress_bytes=310370304 link_bytes=310370304 token_bytes=469975040 weight_movement_bytes=75497472 total_nvlink_bytes=545472512 weight_to_token_ratio=0.160641 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973815541e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57342 payload_rows_remote=49150 dup_rows=8194 dup_rows_remote=8192 dedup_ratio=0.125031 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 721, 0, 0, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 0, 8192] dup_rows_per_dst=[2, 1066, 18, 7050, 58, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=56148 payload_rows_remote=47964 dup_rows=9388 dup_rows_remote=9374 dedup_ratio=0.143250 router_tokens_per_expert=[0, 0, 0, 0, 0, 124, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 1879, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 3902, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4076, 0, 0, 0, 0, 3997, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4088, 0, 0, 0, 0] payload_rows_per_dst=[7572, 3766, 7810, 4770, 7670, 8184, 8192, 8184] dup_rows_per_dst=[234, 16, 164, 14, 0, 0, 8946, 14] combine_rows_per_src=[7572, 3766, 7810, 4770, 7670, 8184, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229712 combine_payload_bytes=234864640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235094352 ingress_bytes=253739008 link_bytes=253739008 token_bytes=469958992 weight_movement_bytes=18874368 total_nvlink_bytes=488833360 weight_to_token_ratio=0.040162 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431202327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=201195520 dispatch_meta_bytes=220760 combine_payload_bytes=201195520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201416280 ingress_bytes=220069888 link_bytes=220069888 token_bytes=402611800 weight_movement_bytes=18874368 total_nvlink_bytes=421486168 weight_to_token_ratio=0.046880 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=23296 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.108601386e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=201318400 dispatch_meta_bytes=229368 combine_payload_bytes=201318400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=201547768 ingress_bytes=220192768 link_bytes=220192768 token_bytes=402866168 weight_movement_bytes=18874368 total_nvlink_bytes=421740536 weight_to_token_ratio=0.046850 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.109778761e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=196460544 dispatch_meta_bytes=229352 combine_payload_bytes=196460544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=196689896 ingress_bytes=215334912 link_bytes=215334912 token_bytes=393150440 weight_movement_bytes=18874368 total_nvlink_bytes=412024808 weight_to_token_ratio=0.048008 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.063233175e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65032 payload_rows_remote=56840 dup_rows=504 dup_rows_remote=438 dedup_ratio=0.007690 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1860, 0, 0, 0, 213, 0, 0, 0, 0, 3, 2219, 0, 0, 0, 0, 3, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0] payload_rows_per_dst=[8186, 8192, 8192, 8186, 8192, 8170, 7728, 8186] dup_rows_per_dst=[0, 6, 6, 0, 66, 426, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8186, 8192, 8170, 7728, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57344 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.001465 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8096, 8192] dup_rows_per_dst=[32, 0, 0, 0, 0, 64, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8096, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=253755392 link_bytes=253755392 token_bytes=469991424 weight_movement_bytes=18874368 total_nvlink_bytes=488865792 weight_to_token_ratio=0.040159 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.431359311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64966 payload_rows_remote=56774 dup_rows=570 dup_rows_remote=570 dedup_ratio=0.008698 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 123, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7874, 7940, 8192] dup_rows_per_dst=[0, 0, 0, 0, 418, 152, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7874, 7940, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=232816640 dispatch_meta_bytes=229112 combine_payload_bytes=232816640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233045752 ingress_bytes=251691008 link_bytes=251691008 token_bytes=465862392 weight_movement_bytes=18874368 total_nvlink_bytes=484736760 weight_to_token_ratio=0.040515 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.411579399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229760 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=235110784 ingress_bytes=291504128 link_bytes=291504128 token_bytes=469991808 weight_movement_bytes=56623104 total_nvlink_bytes=526614912 weight_to_token_ratio=0.120477 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.793049126e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63304 payload_rows_remote=55112 dup_rows=2232 dup_rows_remote=2232 dedup_ratio=0.034058 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 68, 0, 0, 0, 0, 0, 1475, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8116, 6042, 8190] dup_rows_per_dst=[0, 0, 0, 0, 274, 1958, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8116, 6042, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=232546304 dispatch_meta_bytes=229376 combine_payload_bytes=232546304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=232775680 ingress_bytes=270295040 link_bytes=270295040 token_bytes=465321984 weight_movement_bytes=37748736 total_nvlink_bytes=503070720 weight_to_token_ratio=0.081124 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589834080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61230 payload_rows_remote=53038 dup_rows=4306 dup_rows_remote=4306 dedup_ratio=0.065704 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3934, 8192, 8176, 8192, 8192, 8160] dup_rows_per_dst=[0, 4306, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 3934, 8192, 8176, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=60300 payload_rows_remote=52108 dup_rows=5236 dup_rows_remote=5222 dedup_ratio=0.079895 router_tokens_per_expert=[4, 0, 0, 0, 4088, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1707, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8184, 5616, 6410, 7346, 8192, 8176, 8184, 8192] dup_rows_per_dst=[8, 0, 3408, 1698, 8, 90, 10, 14] combine_rows_per_src=[8184, 5616, 6410, 7346, 8192, 8176, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57344 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 6] combine_rows_per_src=[8192, 8184, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=225738752 dispatch_meta_bytes=229376 combine_payload_bytes=225738752 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=225968128 ingress_bytes=244613120 link_bytes=244613120 token_bytes=451706880 weight_movement_bytes=18874368 total_nvlink_bytes=470581248 weight_to_token_ratio=0.041785 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.343762559e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64882 payload_rows_remote=57094 dup_rows=654 dup_rows_remote=454 dedup_ratio=0.009979 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3527, 0, 0, 0, 103, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7788, 7942, 8192] dup_rows_per_dst=[0, 0, 0, 0, 454, 200, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7788, 7942, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=217243648 dispatch_meta_bytes=229376 combine_payload_bytes=217243648 prefetch_bytes=0 reduce_bytes=0 egress_bytes=217473024 ingress_bytes=217243648 link_bytes=217473024 token_bytes=434716672 weight_movement_bytes=0 total_nvlink_bytes=434716672 weight_to_token_ratio=0.000000 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.083719513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=213434368 dispatch_meta_bytes=229320 combine_payload_bytes=213434368 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=213663688 ingress_bytes=232308736 link_bytes=232308736 token_bytes=427098056 weight_movement_bytes=18874368 total_nvlink_bytes=445972424 weight_to_token_ratio=0.044192 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225868005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229408 combine_payload_bytes=234881024 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=235110432 ingress_bytes=310378496 link_bytes=310378496 token_bytes=469991456 weight_movement_bytes=75497472 total_nvlink_bytes=545488928 weight_to_token_ratio=0.160636 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=30720 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.973894033e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=63532 payload_rows_remote=55346 dup_rows=2004 dup_rows_remote=2004 dedup_ratio=0.030579 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 3174, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 958, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8072, 6354, 8192, 8160, 8186, 8192, 8186, 8190] dup_rows_per_dst=[8, 6, 0, 1650, 0, 6, 0, 334] combine_rows_per_src=[8072, 6354, 8192, 8160, 8186, 8192, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=57360 payload_rows_remote=49168 dup_rows=8176 dup_rows_remote=30 dedup_ratio=0.124756 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 2870, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1228, 8192, 6980, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 14, 12, 0, 0, 8146, 0] combine_rows_per_src=[8192, 1228, 8192, 6980, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=233857024 dispatch_meta_bytes=230192 combine_payload_bytes=233857024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=234087216 ingress_bytes=271605760 link_bytes=271605760 token_bytes=467944240 weight_movement_bytes=37748736 total_nvlink_bytes=505692976 weight_to_token_ratio=0.080669 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=21888 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.602392754e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=94 dedup_ratio=0.001434 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 120, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8114, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 94, 0, 0, 0, 0] combine_rows_per_src=[8176, 8114, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=226697216 dispatch_meta_bytes=229400 combine_payload_bytes=226697216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=226926616 ingress_bytes=245571584 link_bytes=245571584 token_bytes=453623832 weight_movement_bytes=18874368 total_nvlink_bytes=472498200 weight_to_token_ratio=0.041608 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.352946089e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=201392128 dispatch_meta_bytes=196792 combine_payload_bytes=201392128 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=201588920 ingress_bytes=314638336 link_bytes=314638336 token_bytes=402981048 weight_movement_bytes=113246208 total_nvlink_bytes=516227256 weight_to_token_ratio=0.281021 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=22912 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.014709724e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65412 payload_rows_remote=57220 dup_rows=124 dup_rows_remote=2 dedup_ratio=0.001892 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 87, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8102, 8160, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[2, 0, 0, 122, 0, 0, 0, 0] combine_rows_per_src=[8102, 8160, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229376 combine_payload_bytes=234496000 prefetch_bytes=0 reduce_bytes=0 egress_bytes=234725376 ingress_bytes=234496000 link_bytes=234725376 token_bytes=469221376 weight_movement_bytes=0 total_nvlink_bytes=469221376 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.249023061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58642 payload_rows_remote=50450 dup_rows=6894 dup_rows_remote=6872 dedup_ratio=0.105194 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 0, 159, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8178, 8192, 8192, 7900, 7686, 2110] dup_rows_per_dst=[22, 0, 0, 0, 1234, 866, 4772, 0] combine_rows_per_src=[8192, 8192, 8178, 8192, 8192, 7900, 7686, 2110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=58788 payload_rows_remote=50604 dup_rows=6748 dup_rows_remote=650 dedup_ratio=0.102966 router_tokens_per_expert=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1352, 0, 0, 0, 461, 0, 0, 0, 0, 4, 2705, 0, 0, 0, 0, 4, 0, 3628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0] payload_rows_per_dst=[8182, 8178, 8192, 8184, 4768, 5836, 7264, 8184] dup_rows_per_dst=[0, 8, 8, 14, 36, 584, 0, 6098] combine_rows_per_src=[8182, 8178, 8192, 8184, 4768, 5836, 7264, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65054 payload_rows_remote=56862 dup_rows=482 dup_rows_remote=482 dedup_ratio=0.007355 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 0, 655, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8192, 7860, 8044, 8192] dup_rows_per_dst=[0, 0, 0, 0, 342, 140, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8192, 7860, 8044, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234373120 dispatch_meta_bytes=228888 combine_payload_bytes=234373120 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=234602008 ingress_bytes=328744960 link_bytes=328744960 token_bytes=468975128 weight_movement_bytes=94371840 total_nvlink_bytes=563346968 weight_to_token_ratio=0.201230 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=3.149872454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65464 payload_rows_remote=57272 dup_rows=72 dup_rows_remote=72 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 254, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 8164, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 72, 0, 0, 0, 0] combine_rows_per_src=[8148, 8164, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=206643200 dispatch_meta_bytes=229288 combine_payload_bytes=206643200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=206872488 ingress_bytes=225517568 link_bytes=225517568 token_bytes=413515688 weight_movement_bytes=18874368 total_nvlink_bytes=432390056 weight_to_token_ratio=0.045644 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.160798375e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65400 payload_rows_remote=57254 dup_rows=136 dup_rows_remote=130 dedup_ratio=0.002075 router_tokens_per_expert=[0, 3, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4070, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 8, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 8146, 8186, 8114, 8192] dup_rows_per_dst=[12, 84, 0, 0, 6, 0, 6, 28] combine_rows_per_src=[8192, 8186, 8192, 8192, 8146, 8186, 8114, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=207273984 dispatch_meta_bytes=205016 combine_payload_bytes=207273984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=207479000 ingress_bytes=263897088 link_bytes=263897088 token_bytes=414752984 weight_movement_bytes=56623104 total_nvlink_bytes=471376088 weight_to_token_ratio=0.136522 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.528532052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=232906752 dispatch_meta_bytes=229376 combine_payload_bytes=232906752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=233136128 ingress_bytes=251781120 link_bytes=251781120 token_bytes=466042880 weight_movement_bytes=18874368 total_nvlink_bytes=484917248 weight_to_token_ratio=0.040499 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.412442808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65360 payload_rows_remote=57168 dup_rows=176 dup_rows_remote=176 dedup_ratio=0.002686 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8016, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 10, 166] combine_rows_per_src=[8192, 8192, 8192, 8192, 8016, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234586112 dispatch_meta_bytes=229376 combine_payload_bytes=234586112 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=234815488 ingress_bytes=253460480 link_bytes=253460480 token_bytes=469401600 weight_movement_bytes=18874368 total_nvlink_bytes=488275968 weight_to_token_ratio=0.040209 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.428533609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234512384 dispatch_meta_bytes=229536 combine_payload_bytes=234512384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234741920 ingress_bytes=272261120 link_bytes=272261120 token_bytes=469254304 weight_movement_bytes=37748736 total_nvlink_bytes=507003040 weight_to_token_ratio=0.080444 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.097222 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.608672091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229376 combine_payload_bytes=234881024 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=235110400 ingress_bytes=272629760 link_bytes=272629760 token_bytes=469991424 weight_movement_bytes=37748736 total_nvlink_bytes=507740160 weight_to_token_ratio=0.080318 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612204218e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 8156, 8192, 8186, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 12, 6, 28] combine_rows_per_src=[8184, 8192, 8192, 8192, 8156, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234160128 dispatch_meta_bytes=229376 combine_payload_bytes=234160128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234389504 ingress_bytes=253034496 link_bytes=253034496 token_bytes=468549632 weight_movement_bytes=18874368 total_nvlink_bytes=487424000 weight_to_token_ratio=0.040283 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424452040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=59012 payload_rows_remote=50860 dup_rows=6524 dup_rows_remote=5922 dedup_ratio=0.099548 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 90, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8192, 8192, 3262, 8192, 8178, 8192, 6652] dup_rows_per_dst=[602, 5872, 0, 36, 14, 0, 0, 0] combine_rows_per_src=[8152, 8192, 8192, 3262, 8192, 8178, 8192, 6652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=61998 payload_rows_remote=53810 dup_rows=3538 dup_rows_remote=2324 dedup_ratio=0.053986 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 2394, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 1145, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8100, 4796, 8192, 8162, 8184, 8192, 8184, 8188] dup_rows_per_dst=[12, 8, 0, 2282, 0, 8, 14, 1214] combine_rows_per_src=[8100, 4796, 8192, 8162, 8184, 8192, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20969812224 prefetch_bytes=905969664 reduce_bytes=811597824 weight_movement_bytes=1717567488 egress_bytes=10490376448 ingress_bytes=12197003264 link_bytes=12197003264 total_nvlink_bytes=22687379712 weight_to_token_ratio=0.081907 prefetch_share_of_ingress=0.074278 dedup_ratio=0.067321 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=43 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100116 gemm_inner_pad_upper_bound=34290 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.434701749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64972 payload_rows_remote=57266 dup_rows=564 dup_rows_remote=564 dedup_ratio=0.008606 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 372, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 7706, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 564, 0, 0, 0, 0] combine_rows_per_src=[8114, 7706, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234676224 dispatch_meta_bytes=229376 combine_payload_bytes=234676224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234905600 ingress_bytes=253550592 link_bytes=253550592 token_bytes=469581824 weight_movement_bytes=18874368 total_nvlink_bytes=488456192 weight_to_token_ratio=0.040194 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429397018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21137936792 prefetch_bytes=773849088 reduce_bytes=849346560 weight_movement_bytes=1623195648 egress_bytes=10574459288 ingress_bytes=12186673152 link_bytes=12186902528 total_nvlink_bytes=22761132440 weight_to_token_ratio=0.076791 prefetch_share_of_ingress=0.063500 dedup_ratio=0.061981 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=45 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=32893 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.432685494e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=208322560 dispatch_meta_bytes=227128 combine_payload_bytes=208322560 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=208549688 ingress_bytes=246071296 link_bytes=246071296 token_bytes=416872248 weight_movement_bytes=37748736 total_nvlink_bytes=454620984 weight_to_token_ratio=0.090552 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.357734084e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 2] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=220405760 dispatch_meta_bytes=224536 combine_payload_bytes=220405760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=220630296 ingress_bytes=239280128 link_bytes=239280128 token_bytes=441036056 weight_movement_bytes=18874368 total_nvlink_bytes=459910424 weight_to_token_ratio=0.042796 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.292664453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234561536 dispatch_meta_bytes=231320 combine_payload_bytes=234561536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=234792856 ingress_bytes=272310272 link_bytes=272310272 token_bytes=469354392 weight_movement_bytes=37748736 total_nvlink_bytes=507103128 weight_to_token_ratio=0.080427 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.100694 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.609143041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=64862 payload_rows_remote=56744 dup_rows=674 dup_rows_remote=674 dedup_ratio=0.010284 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2706, 0, 0, 0, 405, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 7592, 8118, 8192] dup_rows_per_dst=[0, 0, 0, 0, 600, 74, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 7592, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21040796288 prefetch_bytes=792723456 reduce_bytes=981467136 weight_movement_bytes=1774190592 egress_bytes=10525905536 ingress_bytes=12289081344 link_bytes=12289310720 total_nvlink_bytes=22814986880 weight_to_token_ratio=0.084321 prefetch_share_of_ingress=0.064506 dedup_ratio=0.065172 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=52 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.101345 gemm_inner_pad_upper_bound=32004 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.453127679e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=235094016 ingress_bytes=272613376 link_bytes=272613376 token_bytes=469958656 weight_movement_bytes=37748736 total_nvlink_bytes=507707392 weight_to_token_ratio=0.080324 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.612047235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=232423424 dispatch_meta_bytes=229672 combine_payload_bytes=232423424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=232653096 ingress_bytes=251297792 link_bytes=251297792 token_bytes=465076520 weight_movement_bytes=18874368 total_nvlink_bytes=483950888 weight_to_token_ratio=0.040583 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407811797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65038 payload_rows_remote=56846 dup_rows=498 dup_rows_remote=496 dedup_ratio=0.007599 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 8190, 8190, 7702] dup_rows_per_dst=[2, 0, 0, 0, 268, 0, 228, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 8190, 8190, 7702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=21073416952 prefetch_bytes=868220928 reduce_bytes=887095296 weight_movement_bytes=1755316224 egress_bytes=10542158584 ingress_bytes=12286574592 link_bytes=12286574592 total_nvlink_bytes=22828733176 weight_to_token_ratio=0.083295 prefetch_share_of_ingress=0.070664 dedup_ratio=0.061286 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=47 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100984 gemm_inner_pad_upper_bound=32639 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.452581508e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57106 dup_rows=238 dup_rows_remote=198 dedup_ratio=0.003632 router_tokens_per_expert=[0, 4, 0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4088, 4092, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[8170, 8064, 8192, 8192, 8184, 8184, 8120, 8192] dup_rows_per_dst=[20, 46, 0, 0, 128, 0, 4, 40] combine_rows_per_src=[8170, 8064, 8192, 8192, 8184, 8184, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65360 payload_rows_remote=57168 dup_rows=176 dup_rows_remote=176 dedup_ratio=0.002686 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8122, 8192, 8090, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 2, 170] combine_rows_per_src=[8188, 8192, 8192, 8192, 8122, 8192, 8090, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=232841216 dispatch_meta_bytes=229368 combine_payload_bytes=232841216 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=233070584 ingress_bytes=289464320 link_bytes=289464320 token_bytes=465911800 weight_movement_bytes=56623104 total_nvlink_bytes=522534904 weight_to_token_ratio=0.121532 gemm_rows_defined=33152 gemm_rows_charged=36864 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.100694 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.773504689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=233906176 dispatch_meta_bytes=229216 combine_payload_bytes=233906176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=234135392 ingress_bytes=290529280 link_bytes=290529280 token_bytes=468041568 weight_movement_bytes=56623104 total_nvlink_bytes=524664672 weight_to_token_ratio=0.120979 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.093750 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.783708612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65364 payload_rows_remote=57172 dup_rows=172 dup_rows_remote=172 dedup_ratio=0.002625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 108, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8048, 8164, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 172, 0, 0, 0, 0] combine_rows_per_src=[8048, 8164, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19515059768 prefetch_bytes=773849088 reduce_bytes=849346560 weight_movement_bytes=1623195648 egress_bytes=9762803256 ingress_bytes=11375452160 link_bytes=11375910912 total_nvlink_bytes=21138255416 weight_to_token_ratio=0.083177 prefetch_share_of_ingress=0.068028 dedup_ratio=0.120759 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=45 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100767 gemm_inner_pad_upper_bound=32512 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.270799605e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234160128 dispatch_meta_bytes=229376 combine_payload_bytes=234160128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234389504 ingress_bytes=253034496 link_bytes=253034496 token_bytes=468549632 weight_movement_bytes=18874368 total_nvlink_bytes=487424000 weight_to_token_ratio=0.040283 gemm_rows_defined=33024 gemm_rows_charged=36864 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.104167 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.424452040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=19742586968 prefetch_bytes=887095296 reduce_bytes=736100352 weight_movement_bytes=1623195648 egress_bytes=9876617304 ingress_bytes=11489165312 link_bytes=11489394688 total_nvlink_bytes=21365782616 weight_to_token_ratio=0.082218 prefetch_share_of_ingress=0.077211 dedup_ratio=0.112120 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=39 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100839 gemm_inner_pad_upper_bound=32893 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.293452641e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=234176512 dispatch_meta_bytes=229376 combine_payload_bytes=234176512 prefetch_bytes=37748736 reduce_bytes=0 egress_bytes=234405888 ingress_bytes=271925248 link_bytes=271925248 token_bytes=468582400 weight_movement_bytes=37748736 total_nvlink_bytes=506331136 weight_to_token_ratio=0.080559 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.107639 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.605453931e-04 prefetch_bytes_per_gemm_flop=3.616898148e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20354367768 prefetch_bytes=905969664 reduce_bytes=943718400 weight_movement_bytes=1849688064 egress_bytes=10182582552 ingress_bytes=12021473280 link_bytes=12021473280 total_nvlink_bytes=22204055832 weight_to_token_ratio=0.090874 prefetch_share_of_ingress=0.075363 dedup_ratio=0.089156 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=50 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100550 gemm_inner_pad_upper_bound=33782 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.399663376e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=65478 payload_rows_remote=57286 dup_rows=58 dup_rows_remote=56 dedup_ratio=0.000885 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8134, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 56] combine_rows_per_src=[8192, 8192, 8192, 8192, 8134, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=229368 combine_payload_bytes=234643456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=234872824 ingress_bytes=253517824 link_bytes=253517824 token_bytes=469516280 weight_movement_bytes=18874368 total_nvlink_bytes=488390648 weight_to_token_ratio=0.040200 gemm_rows_defined=32896 gemm_rows_charged=36864 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.107639 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.429083051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=20680553208 prefetch_bytes=905969664 reduce_bytes=754974720 weight_movement_bytes=1660944384 egress_bytes=10345730808 ingress_bytes=11995766784 link_bytes=11995766784 total_nvlink_bytes=22341497592 weight_to_token_ratio=0.080314 prefetch_share_of_ingress=0.075524 dedup_ratio=0.077777 prefetch_slots=96 unused_prefetch_slots=1440 slot_utilization=0.062500 reduce_slots=40 remote_demand=48 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.100767 gemm_inner_pad_upper_bound=32766 gemm_flops_step=50096498540544 bytes_per_gemm_flop=2.394531980e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 anomalies=none + [2026-08-29 23:37:50] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 755.2 | throughput per GPU (TFLOP/s/GPU): 118.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.730852E+00 | loss scale: 1.0 | grad norm: 2.771 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:37:50 +[rank0]:[W829 23:37:51.554325612 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W829 23:37:53.411800003 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W829 23:37:53.753716625 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W829 23:37:53.754018179 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W829 23:37:53.780377774 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W829 23:37:53.879659103 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W829 23:37:53.880598308 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W829 23:37:53.173872866 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:37:57.216000 219093 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:37:57.217000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219279 closing signal SIGTERM +W0829 23:37:57.217000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219280 closing signal SIGTERM +W0829 23:37:57.218000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219281 closing signal SIGTERM +W0829 23:37:57.218000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219283 closing signal SIGTERM +W0829 23:37:57.219000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219284 closing signal SIGTERM +W0829 23:37:57.219000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219285 closing signal SIGTERM +W0829 23:37:57.220000 219093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 219286 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 219093 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e561e584e5b77ec9e017a3286023fd0442f9996 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 3 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 2 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 27 +3, 4 +4, 629 +5, 629 +6, 629 +7, 629 +0, 921 +1, 925 +2, 903 +3, 923 +4, 915 +5, 919 +6, 919 +7, 913 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 15509 +1, 15509 +2, 15509 +3, 15765 +4, 15509 +5, 15369 +6, 15665 +7, 16101 +0, 47207 +1, 47207 +2, 47207 +3, 47207 +4, 47207 +5, 47207 +6, 47207 +7, 47207 +0, 47549 +1, 47551 +2, 47551 +3, 47551 +4, 47551 +5, 47551 +6, 47551 +7, 47551 +0, 48115 +1, 47941 +2, 48143 +3, 47941 +4, 47941 +5, 48143 +6, 47941 +7, 48143 +0, 49011 +1, 48689 +2, 49061 +3, 48691 +4, 48803 +5, 49039 +6, 48685 +7, 49065 +0, 49231 +1, 48889 +2, 49259 +3, 48887 +4, 49023 +5, 49259 +6, 48887 +7, 49259 +0, 49233 +1, 48889 +2, 49261 +3, 48889 +4, 49025 +5, 49261 +6, 48889 +7, 49261 +0, 59143 +1, 59129 +2, 59143 +3, 59147 +4, 59149 +5, 59143 +6, 59131 +7, 59143 +0, 83561 +1, 83497 +2, 83513 +3, 83535 +4, 83539 +5, 83541 +6, 83513 +7, 83517 +0, 83953 +1, 83933 +2, 83951 +3, 83957 +4, 83957 +5, 83951 +6, 83937 +7, 83951 +0, 84343 +1, 84323 +2, 84341 +3, 84347 +4, 84347 +5, 84341 +6, 84327 +7, 84341 +0, 87015 +1, 86951 +2, 87141 +3, 87255 +4, 86939 +5, 87081 +6, 87079 +7, 87021 +0, 95373 +1, 95267 +2, 95385 +3, 95135 +4, 95275 +5, 95395 +6, 95223 +7, 95415 +0, 116515 +1, 116417 +2, 116511 +3, 116283 +4, 116405 +5, 116511 +6, 116359 +7, 116511 +0, 116515 +1, 116417 +2, 116511 +3, 116283 +4, 116405 +5, 116511 +6, 116359 +7, 116511 +0, 116515 +1, 116417 +2, 116511 +3, 116283 +4, 116405 +5, 116511 +6, 116359 +7, 116511 +0, 116515 +1, 116417 +2, 116511 +3, 116283 +4, 116405 +5, 116511 +6, 116359 +7, 116511 +0, 116515 +1, 116417 +2, 116447 +3, 116021 +4, 116405 +5, 116511 +6, 116359 +7, 116511 +0, 115077 +1, 114979 +2, 115073 +3, 114583 +4, 114967 +5, 115073 +6, 114921 +7, 115073 +0, 114043 +1, 113945 +2, 113651 +3, 113161 +4, 113545 +5, 114039 +6, 113499 +7, 113389 +0, 109335 +1, 111833 +2, 108659 +3, 4 +4, 108505 +5, 111927 +6, 90649 +7, 4 +0, 1 +1, 111828 +2, 1 +3, 1 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..3604340b6f3777a92c639651ef01f524690351c8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1db6b34505ebe7ef084062c0e2f3a85afaeaef05 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log @@ -0,0 +1,14340 @@ +[23:38:01] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[23:38:10] WARN: moonep_backend_async_grad_reduce_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:38:11.713000 222421 torch/distributed/run.py:766] +W0829 23:38:11.713000 222421 torch/distributed/run.py:766] ***************************************** +W0829 23:38:11.713000 222421 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:38:11.713000 222421 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 75 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[rank4]:[W829 23:38:30.371041094 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:38:30.371250025 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:38:30.374327440 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W829 23:38:30.374356389 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:38:30.374449306 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:38:30.374467957 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:38:30.374543710 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:38:30.525194406 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.600 seconds +time to initialize megatron (seconds): 7.084 +[after megatron is initialized] datetime: 2026-08-29 23:38:36 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.k_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:38:41 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:38:41 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5437.81, 5443.29) + train/valid/test-data-iterators-setup ..........: (65.14, 74.82) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-29 23:38:41 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 144.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 147.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2048, 2048, 2176, 2176, 2048, 2048, 2176, 2176, 2048, 2048, 2048, 2176, 2176, 2048, 2048, 2048, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [30, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2176, 2048, 2176, 2048, 2048, 2048, 2176, 2176, 2176, 2048, 2048, 2048, 2048, 2176, 2176, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [66, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [48, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [65, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2176, 2176, 2176, 2176, 2176, 2048, 2048, 2176, 2176, 2048, 2048, 2176, 2048, 2048, 2176, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [66, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [51, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [40, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [8, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2048, 2176, 2048, 2048, 2048, 2176, 2048, 2176, 2048, 2176, 2048, 2176, 2048, 2048, 2048, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[70, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [72, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [29, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [65, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [83, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-29 23:39:04] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22979.0 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095326E+01 | loss scale: 1.0 | grad norm: 54.378 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66591.7275390625 | max allocated: 66591.72802734375 | reserved: 68350.0 | max reserved: 68350.0 + [2026-08-29 23:39:05] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 961.0 | throughput per GPU (TFLOP/s/GPU): 92.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095569E+01 | loss scale: 1.0 | grad norm: 56.705 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38184 dup_rows=21936 dup_rows_remote=19308 dedup_ratio=0.334717 router_tokens_per_expert=[241, 260, 265, 256, 269, 254, 240, 237, 253, 248, 268, 239, 234, 279, 239, 250, 250, 271, 249, 248, 239, 279, 252, 296, 230, 264, 251, 247, 265, 243, 273, 254, 260, 253, 276, 276, 237, 244, 237, 252, 246, 261, 267, 232, 237, 257, 250, 236, 255, 251, 257, 224, 274, 263, 249, 236, 295, 267, 278, 260, 266, 253, 253, 286, 231, 262, 265, 234, 238, 270, 269, 261, 258, 265, 259, 247, 252, 245, 227, 239, 249, 249, 270, 253, 236, 256, 229, 261, 266, 252, 264, 259, 287, 216, 265, 264, 252, 249, 274, 270, 236, 259, 259, 252, 256, 256, 250, 269, 262, 283, 286, 277, 249, 249, 262, 274, 250, 283, 248, 253, 252, 238, 259, 292, 259, 272, 251, 258] payload_rows_per_dst=[5440, 5418, 5404, 5530, 5416, 5430, 5502, 5460] dup_rows_per_dst=[2624, 2804, 2638, 2804, 2628, 2722, 2878, 2838] combine_rows_per_src=[5440, 5418, 5404, 5530, 5416, 5430, 5502, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38378 dup_rows=21634 dup_rows_remote=18964 dedup_ratio=0.330109 router_tokens_per_expert=[238, 266, 266, 255, 300, 248, 271, 232, 267, 265, 251, 238, 250, 258, 248, 256, 265, 255, 242, 244, 256, 233, 266, 250, 263, 260, 246, 250, 279, 255, 219, 279, 246, 263, 255, 271, 272, 226, 255, 277, 253, 265, 249, 251, 239, 259, 306, 280, 243, 269, 256, 265, 274, 243, 247, 264, 261, 254, 229, 248, 278, 230, 265, 271, 241, 224, 273, 247, 258, 288, 259, 271, 261, 249, 258, 252, 240, 234, 253, 260, 260, 276, 250, 237, 269, 283, 273, 260, 255, 255, 250, 233, 227, 230, 245, 231, 259, 265, 243, 235, 226, 267, 237, 267, 255, 270, 267, 258, 247, 259, 272, 255, 269, 271, 276, 239, 255, 254, 253, 255, 259, 247, 226, 271, 305, 255, 267, 247] payload_rows_per_dst=[5480, 5446, 5586, 5524, 5426, 5450, 5482, 5508] dup_rows_per_dst=[2738, 2678, 2748, 2670, 2710, 2618, 2682, 2790] combine_rows_per_src=[5480, 5446, 5586, 5524, 5426, 5450, 5482, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38228 dup_rows=21696 dup_rows_remote=18850 dedup_ratio=0.331055 router_tokens_per_expert=[267, 258, 247, 238, 285, 274, 263, 281, 256, 258, 261, 241, 224, 273, 247, 268, 260, 261, 231, 232, 266, 240, 233, 250, 261, 256, 258, 254, 239, 255, 267, 243, 265, 252, 247, 249, 278, 231, 248, 275, 237, 233, 267, 244, 272, 251, 255, 273, 260, 231, 255, 237, 238, 235, 264, 268, 268, 282, 268, 255, 278, 242, 266, 249, 256, 256, 244, 291, 238, 253, 244, 240, 247, 269, 250, 279, 223, 278, 269, 244, 259, 266, 256, 236, 241, 254, 228, 265, 248, 264, 239, 275, 237, 278, 267, 269, 245, 271, 268, 232, 285, 235, 272, 249, 269, 261, 250, 261, 250, 227, 270, 241, 276, 287, 248, 269, 264, 244, 226, 245, 253, 233, 294, 259, 272, 280, 263, 286] payload_rows_per_dst=[5460, 5394, 5404, 5496, 5450, 5492, 5532, 5612] dup_rows_per_dst=[2822, 2618, 2648, 2638, 2712, 2696, 2716, 2846] combine_rows_per_src=[5460, 5394, 5404, 5496, 5450, 5492, 5532, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38064 dup_rows=21828 dup_rows_remote=18920 dedup_ratio=0.333069 router_tokens_per_expert=[259, 285, 256, 235, 288, 251, 255, 245, 241, 240, 234, 249, 250, 273, 247, 247, 252, 280, 250, 250, 230, 252, 236, 259, 278, 250, 228, 237, 249, 256, 265, 257, 256, 214, 243, 271, 272, 261, 261, 242, 266, 268, 259, 240, 237, 242, 259, 236, 257, 261, 243, 254, 288, 259, 226, 230, 272, 262, 250, 260, 258, 241, 274, 240, 243, 276, 271, 273, 253, 226, 262, 258, 263, 250, 277, 243, 245, 243, 260, 263, 282, 247, 281, 279, 293, 249, 252, 269, 264, 264, 260, 268, 264, 281, 258, 265, 233, 267, 247, 265, 259, 259, 264, 276, 273, 261, 249, 240, 248, 253, 259, 266, 257, 247, 275, 231, 268, 271, 261, 261, 254, 236, 245, 233, 262, 259, 252, 269] payload_rows_per_dst=[5406, 5350, 5414, 5484, 5456, 5644, 5500, 5454] dup_rows_per_dst=[2704, 2708, 2640, 2666, 2756, 2908, 2738, 2708] combine_rows_per_src=[5406, 5350, 5414, 5484, 5456, 5644, 5500, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38236 dup_rows=21872 dup_rows_remote=19158 dedup_ratio=0.333740 router_tokens_per_expert=[273, 275, 231, 273, 235, 245, 261, 249, 232, 264, 255, 249, 267, 276, 278, 263, 274, 252, 239, 227, 269, 231, 246, 270, 285, 239, 259, 252, 253, 265, 250, 260, 234, 257, 265, 242, 276, 256, 254, 281, 270, 251, 281, 243, 269, 272, 266, 246, 241, 266, 275, 252, 242, 254, 253, 252, 249, 288, 263, 271, 248, 245, 265, 237, 230, 263, 250, 280, 283, 263, 237, 248, 256, 242, 258, 280, 253, 269, 274, 252, 239, 239, 241, 274, 255, 221, 249, 290, 254, 243, 279, 251, 252, 280, 232, 257, 252, 245, 262, 251, 260, 246, 227, 238, 267, 250, 249, 265, 260, 273, 267, 230, 238, 243, 278, 249, 245, 252, 260, 279, 264, 261, 238, 254, 243, 277, 246, 244] payload_rows_per_dst=[5502, 5428, 5510, 5398, 5510, 5438, 5454, 5424] dup_rows_per_dst=[2750, 2714, 2816, 2804, 2766, 2674, 2630, 2718] combine_rows_per_src=[5502, 5428, 5510, 5398, 5510, 5438, 5454, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38298 dup_rows=21748 dup_rows_remote=19076 dedup_ratio=0.331848 router_tokens_per_expert=[248, 251, 253, 273, 248, 261, 214, 261, 252, 248, 260, 266, 249, 259, 271, 238, 244, 263, 257, 254, 271, 280, 280, 245, 255, 259, 242, 263, 272, 270, 259, 236, 262, 266, 277, 253, 259, 239, 242, 259, 267, 259, 262, 228, 250, 251, 248, 259, 251, 252, 251, 264, 276, 263, 286, 267, 223, 253, 232, 268, 265, 269, 272, 281, 291, 284, 241, 270, 272, 247, 235, 269, 234, 253, 257, 261, 249, 248, 234, 270, 259, 229, 249, 257, 263, 230, 252, 227, 258, 258, 272, 263, 254, 272, 243, 245, 233, 254, 257, 254, 249, 255, 241, 286, 242, 260, 229, 246, 269, 238, 267, 278, 264, 257, 243, 242, 259, 253, 264, 262, 230, 264, 244, 273, 255, 262, 251, 285] payload_rows_per_dst=[5396, 5484, 5490, 5562, 5490, 5440, 5428, 5498] dup_rows_per_dst=[2708, 2816, 2672, 2784, 2740, 2622, 2688, 2718] combine_rows_per_src=[5396, 5484, 5490, 5562, 5490, 5440, 5428, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229968 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156631632 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313033296 weight_movement_bytes=18874368 total_nvlink_bytes=331907664 weight_to_token_ratio=0.060295 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229368 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157425656 ingress_bytes=213819392 link_bytes=213819392 token_bytes=314621944 weight_movement_bytes=56623104 total_nvlink_bytes=371245048 weight_to_token_ratio=0.179972 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048712208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=228312 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156810200 ingress_bytes=194330624 link_bytes=194330624 token_bytes=313392088 weight_movement_bytes=37748736 total_nvlink_bytes=351140824 weight_to_token_ratio=0.120452 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861980423e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=227936 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156138080 ingress_bytes=193658880 link_bytes=193658880 token_bytes=312048224 weight_movement_bytes=37748736 total_nvlink_bytes=349796960 weight_to_token_ratio=0.120971 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855544102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229576 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156844232 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313458888 weight_movement_bytes=18874368 total_nvlink_bytes=332333256 weight_to_token_ratio=0.060213 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229496 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157098104 ingress_bytes=213491712 link_bytes=213491712 token_bytes=313966712 weight_movement_bytes=56623104 total_nvlink_bytes=370589816 weight_to_token_ratio=0.180347 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045572540e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38330 dup_rows=21778 dup_rows_remote=19042 dedup_ratio=0.332306 router_tokens_per_expert=[266, 257, 253, 247, 262, 260, 253, 259, 271, 275, 295, 230, 248, 247, 232, 238, 236, 251, 273, 252, 261, 263, 256, 248, 266, 262, 268, 272, 239, 240, 220, 256, 254, 237, 280, 220, 259, 240, 244, 273, 277, 248, 258, 250, 280, 245, 256, 256, 253, 244, 272, 280, 288, 263, 264, 270, 273, 245, 227, 260, 249, 260, 256, 243, 292, 255, 241, 254, 249, 262, 260, 261, 230, 253, 253, 226, 258, 262, 254, 272, 256, 265, 251, 243, 266, 242, 252, 253, 260, 271, 255, 265, 266, 254, 266, 243, 283, 249, 265, 234, 250, 261, 257, 252, 268, 244, 247, 267, 264, 240, 237, 277, 277, 255, 249, 249, 244, 256, 265, 246, 259, 237, 287, 250, 248, 253, 262, 266] payload_rows_per_dst=[5464, 5464, 5400, 5522, 5428, 5504, 5480, 5496] dup_rows_per_dst=[2722, 2662, 2754, 2772, 2736, 2712, 2710, 2710] combine_rows_per_src=[5464, 5464, 5400, 5522, 5428, 5504, 5480, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43968 payload_rows_remote=38494 dup_rows=21568 dup_rows_remote=18968 dedup_ratio=0.329102 router_tokens_per_expert=[239, 243, 259, 256, 225, 258, 276, 234, 283, 255, 234, 255, 259, 250, 251, 264, 257, 257, 246, 263, 265, 260, 268, 265, 234, 257, 256, 273, 271, 264, 239, 252, 228, 279, 252, 259, 277, 257, 269, 257, 240, 253, 234, 258, 297, 282, 277, 246, 251, 265, 250, 277, 246, 247, 260, 245, 238, 257, 259, 235, 252, 237, 249, 269, 256, 243, 253, 253, 244, 235, 270, 257, 282, 250, 260, 276, 244, 262, 281, 268, 258, 258, 226, 268, 265, 271, 261, 254, 276, 269, 271, 260, 261, 266, 257, 286, 246, 246, 247, 238, 269, 268, 260, 234, 250, 257, 249, 245, 249, 242, 245, 245, 225, 251, 292, 288, 220, 262, 252, 242, 253, 262, 240, 281, 238, 251, 266, 244] payload_rows_per_dst=[5490, 5470, 5604, 5474, 5546, 5584, 5362, 5438] dup_rows_per_dst=[2592, 2784, 2726, 2600, 2722, 2830, 2618, 2696] combine_rows_per_src=[5490, 5470, 5604, 5474, 5546, 5584, 5362, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38062 dup_rows=21758 dup_rows_remote=18658 dedup_ratio=0.332001 router_tokens_per_expert=[252, 267, 267, 260, 246, 251, 262, 279, 246, 218, 264, 250, 271, 277, 298, 246, 255, 258, 247, 228, 253, 253, 251, 234, 258, 250, 238, 250, 255, 258, 272, 266, 265, 234, 254, 235, 285, 254, 259, 233, 258, 251, 267, 237, 260, 272, 251, 263, 254, 265, 253, 256, 229, 286, 263, 237, 247, 234, 243, 229, 268, 283, 255, 260, 239, 265, 212, 285, 263, 250, 278, 272, 255, 257, 251, 244, 248, 257, 258, 260, 242, 235, 270, 244, 230, 289, 269, 277, 282, 270, 266, 282, 253, 248, 255, 250, 231, 256, 253, 251, 269, 255, 257, 257, 262, 259, 282, 242, 244, 243, 273, 232, 256, 246, 277, 247, 251, 258, 273, 250, 253, 227, 262, 266, 258, 274, 267, 261] payload_rows_per_dst=[5448, 5402, 5370, 5276, 5472, 5480, 5614, 5716] dup_rows_per_dst=[2668, 2684, 2478, 2480, 2716, 2588, 3044, 3100] combine_rows_per_src=[5448, 5402, 5370, 5276, 5472, 5480, 5614, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38308 dup_rows=21774 dup_rows_remote=19102 dedup_ratio=0.332245 router_tokens_per_expert=[246, 247, 266, 250, 261, 261, 240, 261, 293, 255, 246, 243, 256, 266, 265, 258, 238, 250, 254, 266, 249, 234, 246, 264, 258, 254, 234, 245, 255, 269, 286, 240, 270, 256, 235, 258, 269, 244, 250, 262, 269, 280, 271, 277, 264, 278, 254, 250, 266, 260, 232, 257, 265, 246, 278, 257, 255, 236, 285, 267, 252, 237, 241, 288, 279, 253, 252, 261, 254, 253, 255, 258, 244, 248, 271, 267, 244, 288, 261, 282, 260, 237, 238, 243, 300, 228, 265, 253, 264, 262, 277, 228, 252, 252, 254, 250, 252, 233, 253, 257, 260, 269, 250, 243, 276, 263, 245, 290, 268, 251, 250, 268, 233, 230, 264, 246, 223, 227, 249, 243, 236, 269, 244, 236, 279, 245, 252, 266] payload_rows_per_dst=[5434, 5478, 5602, 5438, 5556, 5454, 5438, 5362] dup_rows_per_dst=[2794, 2606, 2772, 2806, 2784, 2672, 2818, 2522] combine_rows_per_src=[5434, 5478, 5602, 5438, 5556, 5454, 5438, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38300 dup_rows=21852 dup_rows_remote=19152 dedup_ratio=0.333435 router_tokens_per_expert=[254, 273, 244, 297, 251, 259, 301, 258, 247, 237, 254, 273, 229, 262, 266, 228, 262, 247, 261, 250, 262, 295, 254, 280, 251, 290, 256, 242, 257, 240, 250, 252, 251, 277, 238, 225, 254, 264, 275, 223, 245, 243, 242, 249, 261, 285, 257, 253, 259, 255, 240, 281, 257, 262, 252, 280, 271, 254, 245, 258, 275, 283, 233, 265, 231, 254, 245, 236, 267, 264, 262, 250, 259, 257, 255, 260, 286, 258, 267, 220, 233, 264, 277, 240, 256, 265, 243, 247, 247, 272, 254, 267, 268, 248, 295, 255, 275, 225, 254, 237, 266, 238, 255, 249, 246, 264, 240, 257, 240, 247, 248, 260, 243, 265, 245, 239, 252, 254, 277, 248, 251, 235, 269, 271, 264, 250, 247, 261] payload_rows_per_dst=[5466, 5456, 5384, 5546, 5456, 5484, 5434, 5458] dup_rows_per_dst=[2800, 2842, 2700, 2794, 2686, 2778, 2568, 2684] combine_rows_per_src=[5466, 5456, 5384, 5546, 5456, 5484, 5434, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38256 dup_rows=21812 dup_rows_remote=18906 dedup_ratio=0.332825 router_tokens_per_expert=[239, 278, 264, 292, 241, 240, 240, 273, 278, 249, 260, 250, 276, 234, 260, 260, 258, 245, 256, 265, 289, 268, 280, 264, 270, 262, 236, 249, 260, 250, 278, 257, 252, 267, 277, 287, 245, 222, 269, 269, 246, 233, 253, 262, 255, 248, 305, 242, 273, 216, 276, 249, 275, 269, 265, 226, 277, 266, 271, 254, 245, 255, 258, 243, 237, 239, 265, 303, 254, 261, 251, 238, 242, 260, 252, 264, 255, 234, 231, 244, 282, 249, 261, 269, 262, 241, 260, 265, 279, 287, 248, 246, 238, 236, 274, 264, 240, 262, 264, 261, 269, 240, 244, 239, 233, 258, 236, 272, 263, 252, 230, 230, 246, 246, 246, 257, 223, 260, 259, 291, 244, 236, 250, 247, 263, 260, 255, 230] payload_rows_per_dst=[5598, 5468, 5508, 5398, 5406, 5508, 5408, 5430] dup_rows_per_dst=[2670, 2906, 2756, 2838, 2654, 2814, 2578, 2596] combine_rows_per_src=[5598, 5468, 5508, 5398, 5406, 5508, 5408, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229488 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157229168 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228848 weight_movement_bytes=37748736 total_nvlink_bytes=351977584 weight_to_token_ratio=0.120131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157671424 dispatch_meta_bytes=229848 combine_payload_bytes=157671424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157901272 ingress_bytes=195420160 link_bytes=195420160 token_bytes=315572696 weight_movement_bytes=37748736 total_nvlink_bytes=353321432 weight_to_token_ratio=0.119620 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872419820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=226880 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156128832 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312030784 weight_movement_bytes=18874368 total_nvlink_bytes=330905152 weight_to_token_ratio=0.060489 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229640 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157139208 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048776 weight_movement_bytes=37748736 total_nvlink_bytes=351797512 weight_to_token_ratio=0.120200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229808 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157106608 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313983408 weight_movement_bytes=37748736 total_nvlink_bytes=351732144 weight_to_token_ratio=0.120225 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228648 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925224 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313621800 weight_movement_bytes=18874368 total_nvlink_bytes=332496168 weight_to_token_ratio=0.060182 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38314 dup_rows=21812 dup_rows_remote=19122 dedup_ratio=0.332825 router_tokens_per_expert=[228, 282, 274, 261, 298, 269, 253, 257, 277, 270, 251, 258, 249, 243, 240, 257, 252, 261, 251, 259, 259, 265, 259, 266, 254, 245, 263, 232, 265, 251, 238, 251, 250, 258, 260, 250, 259, 256, 254, 298, 244, 246, 229, 258, 252, 267, 250, 263, 237, 240, 251, 246, 274, 278, 252, 242, 282, 260, 280, 253, 230, 267, 252, 276, 253, 260, 269, 239, 284, 247, 251, 264, 232, 232, 248, 267, 250, 218, 249, 259, 254, 252, 276, 264, 229, 262, 256, 255, 239, 252, 258, 260, 253, 277, 248, 282, 227, 242, 272, 254, 281, 266, 243, 254, 240, 254, 259, 259, 267, 236, 239, 257, 281, 251, 264, 250, 249, 268, 279, 258, 267, 271, 262, 252, 234, 229, 269, 243] payload_rows_per_dst=[5440, 5496, 5526, 5540, 5388, 5472, 5410, 5452] dup_rows_per_dst=[2894, 2646, 2662, 2700, 2656, 2762, 2690, 2802] combine_rows_per_src=[5440, 5496, 5526, 5540, 5388, 5472, 5410, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43486 payload_rows_remote=37978 dup_rows=22050 dup_rows_remote=19402 dedup_ratio=0.336456 router_tokens_per_expert=[253, 228, 270, 261, 289, 256, 253, 244, 278, 248, 266, 221, 255, 265, 261, 240, 261, 247, 234, 210, 263, 258, 245, 235, 243, 229, 257, 296, 234, 254, 283, 248, 249, 264, 251, 257, 246, 253, 257, 277, 272, 257, 240, 257, 272, 250, 269, 240, 280, 259, 240, 243, 232, 225, 253, 270, 231, 245, 258, 288, 270, 245, 279, 260, 254, 245, 253, 249, 254, 280, 277, 243, 226, 262, 280, 256, 221, 265, 285, 239, 263, 278, 276, 250, 259, 223, 290, 261, 238, 245, 274, 249, 261, 259, 257, 268, 269, 246, 289, 245, 269, 250, 240, 267, 270, 229, 278, 281, 256, 276, 218, 266, 235, 249, 230, 253, 263, 259, 282, 237, 263, 255, 258, 249, 278, 300, 254, 240] payload_rows_per_dst=[5404, 5390, 5394, 5508, 5438, 5436, 5480, 5436] dup_rows_per_dst=[2772, 2604, 2828, 2648, 2740, 2866, 2818, 2774] combine_rows_per_src=[5404, 5390, 5394, 5508, 5438, 5436, 5480, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38324 dup_rows=21684 dup_rows_remote=18854 dedup_ratio=0.330872 router_tokens_per_expert=[258, 280, 261, 257, 264, 252, 242, 266, 262, 260, 243, 268, 248, 256, 254, 253, 237, 260, 276, 271, 264, 254, 234, 244, 265, 251, 261, 232, 255, 264, 262, 255, 238, 267, 228, 265, 246, 225, 270, 254, 262, 245, 286, 269, 245, 239, 280, 259, 256, 244, 260, 283, 256, 272, 272, 280, 240, 255, 267, 254, 242, 230, 238, 278, 257, 251, 233, 258, 259, 271, 274, 263, 263, 290, 278, 264, 263, 254, 270, 231, 258, 257, 233, 244, 242, 245, 239, 273, 273, 277, 251, 261, 255, 252, 241, 249, 259, 254, 271, 251, 243, 266, 240, 274, 263, 286, 225, 245, 243, 250, 229, 309, 241, 244, 286, 259, 253, 285, 233, 235, 229, 257, 253, 254, 245, 238, 265, 240] payload_rows_per_dst=[5482, 5522, 5496, 5468, 5528, 5434, 5478, 5444] dup_rows_per_dst=[2766, 2648, 2660, 2786, 2830, 2666, 2738, 2590] combine_rows_per_src=[5482, 5522, 5496, 5468, 5528, 5434, 5478, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38250 dup_rows=21758 dup_rows_remote=18956 dedup_ratio=0.332001 router_tokens_per_expert=[293, 214, 254, 254, 255, 271, 253, 241, 236, 266, 265, 269, 248, 265, 242, 255, 234, 279, 275, 239, 265, 271, 263, 237, 262, 256, 265, 240, 247, 242, 269, 268, 255, 250, 257, 257, 244, 277, 272, 278, 233, 241, 250, 291, 268, 272, 245, 268, 237, 258, 251, 239, 214, 248, 236, 267, 249, 237, 224, 257, 235, 248, 249, 276, 250, 279, 293, 237, 259, 251, 276, 244, 292, 262, 251, 252, 246, 259, 230, 270, 265, 250, 270, 262, 264, 234, 256, 288, 236, 264, 262, 235, 266, 241, 248, 260, 261, 252, 227, 273, 289, 277, 283, 237, 275, 236, 246, 275, 210, 292, 263, 286, 245, 264, 231, 256, 233, 270, 254, 236, 282, 264, 248, 265, 243, 269, 263, 235] payload_rows_per_dst=[5444, 5444, 5448, 5372, 5556, 5478, 5508, 5528] dup_rows_per_dst=[2718, 2780, 2710, 2614, 2746, 2724, 2664, 2802] combine_rows_per_src=[5444, 5444, 5448, 5372, 5556, 5478, 5508, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44012 payload_rows_remote=38628 dup_rows=21524 dup_rows_remote=18866 dedup_ratio=0.328430 router_tokens_per_expert=[269, 225, 243, 259, 260, 260, 278, 260, 264, 265, 241, 246, 249, 255, 241, 252, 296, 285, 256, 254, 246, 258, 269, 254, 269, 242, 288, 281, 239, 268, 250, 246, 245, 256, 240, 248, 260, 223, 262, 275, 269, 279, 286, 227, 267, 261, 262, 259, 250, 273, 259, 265, 281, 245, 228, 242, 262, 246, 272, 273, 247, 250, 249, 269, 260, 276, 240, 232, 252, 234, 242, 248, 250, 268, 252, 271, 272, 271, 249, 248, 252, 262, 240, 275, 239, 257, 241, 260, 263, 270, 251, 247, 235, 240, 240, 249, 290, 256, 243, 256, 259, 276, 262, 239, 261, 260, 261, 285, 249, 237, 268, 227, 249, 228, 227, 255, 249, 262, 251, 240, 228, 283, 271, 245, 256, 275, 268, 268] payload_rows_per_dst=[5488, 5618, 5494, 5500, 5518, 5384, 5536, 5474] dup_rows_per_dst=[2646, 2784, 2744, 2722, 2612, 2658, 2722, 2636] combine_rows_per_src=[5488, 5618, 5494, 5500, 5518, 5384, 5536, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38266 dup_rows=21832 dup_rows_remote=19098 dedup_ratio=0.333130 router_tokens_per_expert=[258, 212, 252, 257, 237, 242, 271, 251, 236, 222, 274, 269, 268, 243, 244, 239, 259, 238, 273, 272, 287, 253, 247, 222, 234, 251, 225, 254, 246, 268, 282, 253, 269, 254, 272, 247, 280, 260, 263, 252, 241, 266, 264, 276, 268, 265, 246, 266, 288, 269, 230, 254, 245, 248, 263, 249, 278, 242, 252, 259, 240, 242, 245, 275, 248, 251, 264, 258, 253, 267, 252, 251, 256, 276, 256, 236, 276, 270, 247, 272, 261, 290, 224, 266, 271, 261, 269, 254, 236, 224, 274, 244, 279, 273, 256, 285, 266, 269, 249, 225, 275, 246, 249, 244, 290, 231, 267, 254, 249, 279, 269, 241, 228, 241, 240, 276, 246, 269, 238, 270, 265, 261, 231, 252, 260, 259, 275, 247] payload_rows_per_dst=[5438, 5612, 5558, 5356, 5582, 5404, 5460, 5294] dup_rows_per_dst=[2734, 2924, 2820, 2580, 2850, 2764, 2746, 2414] combine_rows_per_src=[5438, 5612, 5558, 5356, 5582, 5404, 5460, 5294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38384 dup_rows=21670 dup_rows_remote=19060 dedup_ratio=0.330658 router_tokens_per_expert=[240, 245, 280, 249, 270, 284, 261, 272, 244, 258, 247, 232, 292, 283, 269, 240, 258, 238, 272, 229, 249, 257, 283, 278, 224, 260, 262, 284, 262, 250, 262, 248, 231, 267, 264, 272, 252, 242, 254, 266, 230, 269, 264, 243, 250, 254, 247, 241, 250, 272, 271, 233, 284, 254, 244, 239, 253, 249, 243, 247, 263, 285, 236, 210, 273, 272, 240, 250, 248, 267, 251, 264, 249, 236, 259, 251, 263, 245, 261, 248, 273, 221, 276, 272, 247, 263, 259, 237, 273, 244, 227, 277, 270, 244, 259, 275, 268, 266, 240, 296, 259, 224, 261, 263, 255, 269, 247, 292, 252, 221, 243, 253, 261, 254, 252, 249, 264, 282, 243, 252, 227, 259, 255, 265, 267, 259, 253, 262] payload_rows_per_dst=[5516, 5482, 5482, 5436, 5428, 5450, 5536, 5536] dup_rows_per_dst=[2816, 2750, 2610, 2630, 2726, 2784, 2682, 2672] combine_rows_per_src=[5516, 5482, 5482, 5436, 5428, 5450, 5536, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155557888 dispatch_meta_bytes=229520 combine_payload_bytes=155557888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155787408 ingress_bytes=174432256 link_bytes=174432256 token_bytes=311345296 weight_movement_bytes=18874368 total_nvlink_bytes=330219664 weight_to_token_ratio=0.060622 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671324051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38234 dup_rows=21824 dup_rows_remote=19144 dedup_ratio=0.333008 router_tokens_per_expert=[260, 260, 255, 276, 238, 259, 274, 260, 292, 249, 258, 231, 263, 239, 242, 267, 241, 247, 244, 241, 253, 253, 257, 276, 273, 269, 237, 263, 254, 243, 255, 273, 266, 287, 264, 257, 247, 248, 246, 274, 242, 243, 221, 230, 274, 233, 276, 268, 252, 254, 276, 272, 261, 258, 252, 276, 263, 263, 251, 269, 256, 257, 285, 257, 259, 235, 263, 267, 261, 256, 261, 273, 249, 238, 259, 233, 247, 262, 266, 265, 232, 240, 265, 242, 261, 275, 235, 253, 236, 255, 255, 257, 264, 243, 262, 254, 264, 246, 258, 265, 244, 264, 252, 266, 245, 216, 271, 244, 243, 235, 265, 243, 259, 262, 262, 234, 274, 269, 266, 263, 237, 257, 263, 250, 278, 265, 252, 253] payload_rows_per_dst=[5462, 5478, 5410, 5570, 5458, 5426, 5428, 5480] dup_rows_per_dst=[2784, 2680, 2742, 2834, 2730, 2632, 2614, 2808] combine_rows_per_src=[5462, 5478, 5410, 5570, 5458, 5426, 5428, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=228712 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157203816 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314178920 weight_movement_bytes=37748736 total_nvlink_bytes=351927656 weight_to_token_ratio=0.120150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229744 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163888 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098032 weight_movement_bytes=18874368 total_nvlink_bytes=332972400 weight_to_token_ratio=0.060091 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228824 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156900824 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313572824 weight_movement_bytes=18874368 total_nvlink_bytes=332447192 weight_to_token_ratio=0.060191 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158220288 dispatch_meta_bytes=229976 combine_payload_bytes=158220288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158450264 ingress_bytes=177094656 link_bytes=177094656 token_bytes=316670552 weight_movement_bytes=18874368 total_nvlink_bytes=335544920 weight_to_token_ratio=0.059603 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.696833858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229776 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157450640 ingress_bytes=232718336 link_bytes=232718336 token_bytes=314671504 weight_movement_bytes=75497472 total_nvlink_bytes=390168976 weight_to_token_ratio=0.239925 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229792591e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229512 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156835976 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313442440 weight_movement_bytes=18874368 total_nvlink_bytes=332316808 weight_to_token_ratio=0.060216 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229456 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966992 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704528 weight_movement_bytes=18874368 total_nvlink_bytes=332578896 weight_to_token_ratio=0.060166 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38482 dup_rows=21700 dup_rows_remote=19068 dedup_ratio=0.331116 router_tokens_per_expert=[272, 246, 252, 261, 256, 242, 260, 259, 246, 252, 246, 249, 268, 260, 261, 252, 250, 271, 278, 247, 231, 271, 245, 278, 253, 266, 246, 248, 254, 267, 252, 245, 249, 281, 277, 281, 273, 281, 271, 248, 261, 265, 264, 269, 241, 261, 261, 233, 233, 247, 244, 259, 280, 223, 254, 263, 258, 236, 242, 235, 260, 259, 249, 251, 255, 251, 258, 250, 242, 263, 249, 243, 277, 249, 230, 257, 258, 248, 253, 259, 251, 234, 241, 265, 252, 256, 222, 267, 265, 245, 248, 260, 259, 266, 270, 253, 249, 278, 267, 244, 271, 257, 255, 270, 252, 240, 259, 250, 262, 234, 247, 257, 236, 267, 273, 263, 252, 254, 254, 255, 283, 250, 269, 251, 253, 271, 291, 265] payload_rows_per_dst=[5430, 5462, 5580, 5354, 5464, 5472, 5500, 5574] dup_rows_per_dst=[2734, 2742, 2852, 2632, 2620, 2636, 2684, 2800] combine_rows_per_src=[5430, 5462, 5580, 5354, 5464, 5472, 5500, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38342 dup_rows=21634 dup_rows_remote=18858 dedup_ratio=0.330109 router_tokens_per_expert=[265, 272, 250, 239, 262, 241, 281, 248, 255, 248, 261, 252, 242, 275, 254, 252, 260, 268, 231, 236, 279, 260, 241, 255, 256, 272, 233, 262, 265, 267, 248, 262, 243, 251, 234, 252, 252, 242, 250, 266, 220, 235, 223, 254, 257, 257, 271, 270, 240, 286, 263, 273, 268, 263, 273, 284, 288, 272, 240, 258, 266, 246, 241, 245, 235, 262, 255, 243, 272, 246, 272, 264, 236, 265, 271, 278, 285, 266, 254, 264, 243, 240, 251, 243, 278, 232, 266, 249, 241, 238, 262, 273, 263, 225, 266, 246, 245, 264, 266, 239, 262, 263, 254, 260, 257, 257, 253, 305, 240, 276, 256, 252, 273, 279, 255, 271, 247, 243, 245, 238, 285, 265, 254, 248, 223, 233, 271, 230] payload_rows_per_dst=[5440, 5532, 5408, 5530, 5560, 5410, 5552, 5470] dup_rows_per_dst=[2754, 2658, 2546, 2882, 2776, 2622, 2746, 2650] combine_rows_per_src=[5440, 5532, 5408, 5530, 5560, 5410, 5552, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38254 dup_rows=21772 dup_rows_remote=18940 dedup_ratio=0.332214 router_tokens_per_expert=[254, 261, 254, 247, 275, 233, 269, 258, 250, 270, 256, 252, 266, 274, 244, 247, 250, 284, 244, 234, 293, 231, 267, 232, 242, 236, 246, 279, 266, 279, 268, 260, 277, 241, 249, 266, 265, 260, 251, 268, 255, 257, 261, 275, 256, 248, 258, 261, 245, 270, 255, 280, 248, 275, 243, 258, 266, 253, 238, 238, 211, 206, 258, 254, 271, 235, 278, 246, 248, 258, 246, 253, 269, 274, 249, 265, 282, 269, 265, 235, 254, 259, 224, 265, 271, 256, 275, 254, 277, 263, 272, 266, 258, 226, 251, 300, 276, 256, 279, 262, 251, 242, 257, 230, 246, 235, 234, 241, 247, 252, 310, 246, 250, 238, 256, 232, 234, 250, 242, 276, 263, 235, 255, 255, 269, 263, 253, 252] payload_rows_per_dst=[5508, 5488, 5572, 5348, 5532, 5510, 5410, 5396] dup_rows_per_dst=[2712, 2734, 2724, 2648, 2754, 2832, 2718, 2650] combine_rows_per_src=[5508, 5488, 5572, 5348, 5532, 5510, 5410, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38020 dup_rows=21970 dup_rows_remote=19134 dedup_ratio=0.335236 router_tokens_per_expert=[238, 249, 252, 275, 238, 244, 242, 255, 261, 267, 244, 242, 250, 262, 245, 257, 260, 251, 255, 258, 227, 262, 277, 259, 258, 263, 268, 241, 265, 248, 276, 267, 266, 272, 265, 245, 259, 265, 266, 276, 222, 241, 249, 266, 279, 262, 282, 252, 232, 246, 259, 261, 248, 245, 272, 289, 224, 231, 251, 248, 267, 237, 238, 275, 257, 254, 240, 258, 274, 257, 248, 282, 254, 255, 261, 264, 241, 251, 255, 281, 252, 265, 258, 239, 247, 249, 224, 262, 273, 275, 244, 222, 284, 268, 252, 226, 223, 239, 285, 244, 256, 265, 286, 263, 254, 233, 251, 271, 295, 247, 246, 240, 265, 238, 269, 246, 276, 281, 249, 242, 249, 272, 265, 244, 282, 234, 270, 270] payload_rows_per_dst=[5458, 5272, 5558, 5570, 5264, 5514, 5384, 5546] dup_rows_per_dst=[2584, 2472, 2776, 3002, 2616, 2862, 2822, 2836] combine_rows_per_src=[5458, 5272, 5558, 5570, 5264, 5514, 5384, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157622272 dispatch_meta_bytes=230200 combine_payload_bytes=157622272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157852472 ingress_bytes=176496640 link_bytes=176496640 token_bytes=315474744 weight_movement_bytes=18874368 total_nvlink_bytes=334349112 weight_to_token_ratio=0.059828 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691103963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38332 dup_rows=21780 dup_rows_remote=19190 dedup_ratio=0.332336 router_tokens_per_expert=[232, 268, 264, 262, 269, 272, 252, 267, 263, 253, 261, 260, 243, 247, 262, 260, 250, 242, 264, 253, 275, 240, 246, 276, 268, 291, 240, 262, 300, 251, 260, 261, 248, 225, 255, 261, 237, 246, 243, 221, 257, 292, 250, 246, 261, 273, 218, 274, 258, 267, 229, 250, 262, 266, 253, 261, 278, 281, 239, 242, 243, 244, 275, 254, 272, 241, 270, 274, 261, 236, 262, 243, 285, 239, 262, 252, 261, 252, 254, 258, 262, 256, 260, 277, 237, 254, 234, 258, 265, 246, 229, 274, 246, 252, 256, 241, 273, 255, 262, 241, 282, 236, 248, 277, 239, 270, 274, 273, 249, 244, 219, 254, 285, 247, 249, 283, 239, 253, 256, 279, 226, 260, 266, 258, 237, 241, 255, 246] payload_rows_per_dst=[5484, 5616, 5424, 5438, 5476, 5450, 5470, 5398] dup_rows_per_dst=[2786, 2742, 2590, 2766, 2768, 2644, 2722, 2762] combine_rows_per_src=[5484, 5616, 5424, 5438, 5476, 5450, 5470, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=228800 combine_payload_bytes=157048832 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157277632 ingress_bytes=232546304 link_bytes=232546304 token_bytes=314326464 weight_movement_bytes=75497472 total_nvlink_bytes=389823936 weight_to_token_ratio=0.240188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228144265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38232 dup_rows=21864 dup_rows_remote=19108 dedup_ratio=0.333618 router_tokens_per_expert=[256, 224, 237, 253, 258, 283, 250, 266, 261, 259, 270, 263, 214, 250, 264, 273, 251, 249, 247, 263, 263, 244, 250, 262, 232, 277, 257, 289, 228, 270, 262, 254, 262, 250, 269, 273, 236, 259, 261, 257, 287, 208, 264, 250, 271, 250, 243, 269, 248, 238, 239, 277, 228, 233, 257, 239, 254, 253, 245, 262, 280, 270, 258, 244, 263, 236, 262, 246, 255, 252, 280, 266, 270, 264, 264, 292, 246, 258, 280, 248, 281, 250, 255, 252, 248, 245, 252, 292, 246, 272, 256, 255, 251, 251, 268, 257, 244, 275, 258, 279, 237, 281, 252, 260, 245, 259, 251, 248, 257, 254, 255, 235, 240, 250, 272, 256, 271, 239, 265, 251, 248, 268, 252, 263, 250, 233, 258, 236] payload_rows_per_dst=[5502, 5440, 5406, 5434, 5474, 5478, 5502, 5436] dup_rows_per_dst=[2660, 2756, 2812, 2616, 2890, 2784, 2678, 2668] combine_rows_per_src=[5502, 5440, 5406, 5434, 5474, 5478, 5502, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=228616 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155958536 ingress_bytes=212353024 link_bytes=212353024 token_bytes=311688456 weight_movement_bytes=56623104 total_nvlink_bytes=368311560 weight_to_token_ratio=0.181666 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034662192e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228776 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917160 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605544 weight_movement_bytes=18874368 total_nvlink_bytes=332479912 weight_to_token_ratio=0.060185 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38256 dup_rows=21816 dup_rows_remote=19070 dedup_ratio=0.332886 router_tokens_per_expert=[251, 237, 256, 274, 285, 234, 282, 249, 265, 278, 246, 236, 244, 232, 248, 267, 254, 256, 241, 287, 240, 262, 236, 264, 285, 245, 243, 270, 240, 246, 259, 272, 269, 256, 281, 270, 254, 263, 257, 259, 242, 240, 247, 252, 257, 214, 255, 268, 234, 250, 253, 257, 246, 270, 269, 282, 275, 268, 250, 274, 236, 244, 249, 290, 256, 278, 263, 258, 233, 273, 253, 251, 269, 246, 241, 272, 271, 251, 249, 263, 260, 243, 261, 264, 246, 236, 244, 276, 258, 242, 238, 274, 247, 248, 259, 258, 208, 266, 238, 251, 248, 250, 268, 259, 274, 252, 270, 276, 272, 250, 254, 260, 259, 242, 255, 244, 279, 268, 272, 255, 263, 240, 262, 234, 257, 255, 248, 243] payload_rows_per_dst=[5462, 5476, 5462, 5446, 5504, 5406, 5464, 5500] dup_rows_per_dst=[2706, 2724, 2706, 2830, 2750, 2522, 2746, 2832] combine_rows_per_src=[5462, 5476, 5462, 5446, 5504, 5406, 5464, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=230088 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237960 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245832 weight_movement_bytes=18874368 total_nvlink_bytes=333120200 weight_to_token_ratio=0.060062 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229360 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827632 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425904 weight_movement_bytes=37748736 total_nvlink_bytes=351174640 weight_to_token_ratio=0.120439 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38320 dup_rows=21724 dup_rows_remote=18994 dedup_ratio=0.331482 router_tokens_per_expert=[229, 277, 237, 283, 246, 264, 265, 246, 254, 234, 281, 276, 241, 278, 233, 263, 241, 245, 260, 251, 246, 246, 265, 253, 245, 259, 250, 274, 246, 246, 257, 273, 275, 271, 260, 260, 277, 244, 262, 252, 253, 249, 252, 232, 246, 243, 292, 254, 251, 284, 277, 247, 269, 275, 276, 251, 261, 264, 256, 256, 233, 224, 271, 263, 258, 276, 249, 276, 273, 241, 264, 252, 271, 232, 253, 258, 258, 257, 242, 239, 233, 265, 265, 269, 242, 245, 277, 252, 270, 258, 279, 245, 248, 266, 243, 243, 253, 258, 267, 266, 234, 259, 262, 271, 248, 241, 267, 261, 269, 281, 241, 249, 253, 250, 260, 258, 254, 256, 256, 234, 243, 233, 269, 231, 250, 265, 247, 239] payload_rows_per_dst=[5492, 5454, 5478, 5536, 5458, 5452, 5554, 5388] dup_rows_per_dst=[2730, 2660, 2836, 2780, 2662, 2748, 2700, 2608] combine_rows_per_src=[5492, 5454, 5478, 5536, 5458, 5452, 5554, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38398 dup_rows=21712 dup_rows_remote=19012 dedup_ratio=0.331299 router_tokens_per_expert=[249, 272, 249, 238, 233, 263, 230, 224, 253, 245, 260, 268, 251, 265, 268, 257, 245, 257, 273, 220, 256, 264, 240, 260, 245, 299, 252, 243, 246, 265, 257, 248, 237, 268, 260, 244, 235, 255, 262, 303, 271, 309, 255, 259, 271, 255, 265, 251, 238, 268, 252, 245, 239, 240, 281, 260, 247, 257, 247, 277, 259, 270, 245, 238, 263, 277, 272, 247, 275, 271, 264, 254, 267, 234, 267, 295, 253, 257, 239, 262, 242, 254, 268, 255, 251, 263, 255, 261, 250, 264, 264, 248, 243, 248, 263, 293, 271, 239, 228, 242, 243, 251, 261, 280, 248, 286, 241, 238, 262, 232, 282, 254, 247, 260, 279, 254, 245, 270, 253, 241, 214, 268, 243, 230, 262, 282, 231, 254] payload_rows_per_dst=[5376, 5510, 5600, 5426, 5610, 5498, 5450, 5354] dup_rows_per_dst=[2674, 2630, 2800, 2700, 2784, 2746, 2666, 2712] combine_rows_per_src=[5376, 5510, 5600, 5426, 5610, 5498, 5450, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229304 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925880 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622456 weight_movement_bytes=18874368 total_nvlink_bytes=332496824 weight_to_token_ratio=0.060182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38316 dup_rows=21830 dup_rows_remote=19170 dedup_ratio=0.333099 router_tokens_per_expert=[285, 263, 260, 254, 278, 280, 280, 244, 274, 252, 251, 256, 247, 245, 249, 248, 236, 276, 256, 278, 235, 265, 268, 241, 232, 286, 247, 267, 263, 280, 282, 274, 257, 254, 265, 259, 242, 257, 259, 217, 279, 253, 247, 246, 291, 260, 276, 261, 237, 260, 260, 247, 246, 258, 262, 285, 229, 238, 258, 260, 259, 240, 263, 267, 243, 281, 246, 246, 245, 252, 259, 248, 260, 236, 237, 253, 236, 246, 265, 272, 249, 243, 272, 261, 270, 233, 264, 250, 252, 243, 267, 269, 257, 235, 239, 233, 272, 226, 250, 262, 252, 241, 265, 252, 245, 262, 226, 269, 221, 242, 252, 272, 246, 252, 287, 257, 279, 252, 254, 234, 274, 269, 226, 274, 254, 271, 271, 253] payload_rows_per_dst=[5556, 5482, 5556, 5422, 5390, 5408, 5406, 5486] dup_rows_per_dst=[2776, 2890, 2690, 2716, 2660, 2666, 2612, 2820] combine_rows_per_src=[5556, 5482, 5556, 5422, 5390, 5408, 5406, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44000 payload_rows_remote=38496 dup_rows=21536 dup_rows_remote=18780 dedup_ratio=0.328613 router_tokens_per_expert=[298, 268, 258, 277, 231, 243, 240, 260, 257, 249, 239, 245, 248, 278, 261, 242, 259, 256, 273, 276, 249, 252, 239, 252, 266, 245, 244, 258, 254, 228, 276, 266, 262, 244, 260, 277, 251, 253, 236, 269, 242, 253, 246, 253, 257, 260, 258, 242, 283, 245, 277, 241, 257, 232, 242, 257, 262, 263, 234, 263, 268, 232, 260, 263, 247, 260, 233, 246, 256, 256, 220, 263, 254, 216, 266, 264, 251, 244, 253, 302, 269, 263, 264, 272, 270, 269, 254, 261, 274, 274, 248, 262, 242, 248, 283, 276, 236, 255, 221, 260, 250, 259, 264, 265, 253, 274, 267, 245, 248, 257, 254, 244, 230, 257, 289, 271, 252, 261, 271, 255, 261, 299, 248, 243, 250, 251, 248, 241] payload_rows_per_dst=[5506, 5498, 5480, 5514, 5432, 5570, 5496, 5504] dup_rows_per_dst=[2682, 2662, 2590, 2652, 2630, 2888, 2676, 2756] combine_rows_per_src=[5506, 5498, 5480, 5514, 5432, 5570, 5496, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38394 dup_rows=21756 dup_rows_remote=19050 dedup_ratio=0.331970 router_tokens_per_expert=[247, 288, 225, 236, 241, 233, 216, 294, 246, 273, 277, 258, 261, 265, 264, 251, 261, 247, 242, 257, 247, 234, 238, 258, 239, 220, 266, 252, 273, 254, 224, 260, 247, 237, 259, 265, 234, 258, 269, 247, 241, 247, 266, 233, 262, 271, 261, 280, 251, 265, 279, 260, 252, 241, 246, 251, 262, 271, 238, 277, 261, 268, 255, 258, 257, 224, 236, 283, 255, 236, 265, 257, 288, 247, 260, 268, 252, 242, 281, 255, 273, 264, 250, 231, 261, 234, 252, 241, 275, 235, 265, 262, 237, 259, 268, 239, 256, 253, 278, 267, 264, 243, 257, 264, 265, 287, 249, 275, 251, 266, 264, 277, 265, 262, 290, 249, 253, 252, 248, 245, 251, 260, 276, 256, 269, 251, 261, 253] payload_rows_per_dst=[5512, 5412, 5458, 5490, 5458, 5386, 5610, 5454] dup_rows_per_dst=[2638, 2532, 2696, 2780, 2754, 2706, 2822, 2828] combine_rows_per_src=[5512, 5412, 5458, 5490, 5458, 5386, 5610, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229640 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507848 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314786056 weight_movement_bytes=18874368 total_nvlink_bytes=333660424 weight_to_token_ratio=0.059959 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229256 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157187976 ingress_bytes=213581824 link_bytes=213581824 token_bytes=314146696 weight_movement_bytes=56623104 total_nvlink_bytes=370769800 weight_to_token_ratio=0.180244 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046435949e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38420 dup_rows=21634 dup_rows_remote=18918 dedup_ratio=0.330109 router_tokens_per_expert=[235, 257, 275, 256, 263, 247, 251, 262, 239, 280, 250, 238, 259, 221, 264, 260, 246, 258, 231, 251, 272, 302, 257, 264, 247, 259, 254, 298, 246, 238, 243, 242, 267, 276, 253, 248, 252, 247, 262, 261, 254, 261, 259, 260, 270, 230, 226, 273, 274, 234, 231, 243, 263, 251, 266, 268, 260, 260, 244, 255, 246, 267, 261, 272, 243, 240, 263, 225, 255, 271, 283, 255, 250, 264, 256, 234, 271, 231, 268, 256, 266, 237, 258, 232, 223, 224, 241, 248, 268, 263, 271, 271, 271, 262, 275, 262, 251, 236, 263, 234, 234, 303, 263, 251, 252, 249, 254, 274, 282, 273, 248, 241, 263, 257, 281, 268, 253, 271, 237, 261, 250, 251, 255, 279, 251, 261, 245, 281] payload_rows_per_dst=[5398, 5512, 5482, 5480, 5524, 5446, 5542, 5518] dup_rows_per_dst=[2716, 2704, 2716, 2710, 2606, 2698, 2674, 2810] combine_rows_per_src=[5398, 5512, 5482, 5480, 5524, 5446, 5542, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229944 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157172280 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114616 weight_movement_bytes=18874368 total_nvlink_bytes=332988984 weight_to_token_ratio=0.060088 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157679616 dispatch_meta_bytes=229104 combine_payload_bytes=157679616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157908720 ingress_bytes=214302720 link_bytes=214302720 token_bytes=315588336 weight_movement_bytes=56623104 total_nvlink_bytes=372211440 weight_to_token_ratio=0.179421 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053343220e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38196 dup_rows=21860 dup_rows_remote=19096 dedup_ratio=0.333557 router_tokens_per_expert=[230, 243, 257, 257, 240, 278, 256, 250, 243, 250, 277, 238, 250, 242, 259, 261, 289, 277, 263, 266, 262, 250, 241, 252, 242, 264, 245, 241, 277, 252, 263, 238, 262, 237, 234, 271, 262, 249, 241, 221, 283, 256, 260, 266, 272, 251, 247, 255, 241, 233, 255, 255, 247, 276, 259, 239, 251, 269, 265, 269, 265, 247, 268, 267, 256, 249, 231, 265, 272, 237, 246, 255, 249, 242, 258, 268, 251, 265, 248, 272, 291, 218, 249, 274, 255, 239, 291, 253, 249, 253, 274, 265, 236, 260, 270, 233, 269, 251, 275, 266, 274, 266, 286, 257, 240, 249, 240, 259, 250, 270, 250, 264, 272, 245, 259, 261, 244, 253, 283, 250, 259, 250, 248, 268, 257, 245, 258, 250] payload_rows_per_dst=[5432, 5480, 5456, 5410, 5440, 5450, 5536, 5472] dup_rows_per_dst=[2630, 2764, 2678, 2802, 2688, 2770, 2796, 2732] combine_rows_per_src=[5432, 5480, 5456, 5410, 5440, 5450, 5536, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229776 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491600 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314753424 weight_movement_bytes=18874368 total_nvlink_bytes=333627792 weight_to_token_ratio=0.059966 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229352 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157597672 ingress_bytes=195117056 link_bytes=195117056 token_bytes=314965992 weight_movement_bytes=37748736 total_nvlink_bytes=352714728 weight_to_token_ratio=0.119850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869515627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44048 payload_rows_remote=38566 dup_rows=21488 dup_rows_remote=18768 dedup_ratio=0.327881 router_tokens_per_expert=[253, 260, 240, 237, 258, 266, 265, 251, 251, 223, 247, 246, 234, 265, 267, 254, 247, 253, 246, 260, 255, 267, 258, 249, 259, 270, 271, 281, 230, 267, 250, 243, 258, 266, 238, 281, 248, 285, 265, 260, 242, 248, 262, 266, 252, 232, 238, 265, 300, 260, 243, 262, 257, 285, 265, 252, 237, 273, 272, 256, 258, 220, 258, 256, 262, 266, 267, 264, 231, 257, 262, 253, 245, 237, 275, 244, 235, 257, 258, 253, 250, 241, 235, 268, 263, 278, 262, 235, 278, 240, 251, 272, 280, 277, 265, 240, 262, 222, 272, 266, 247, 276, 275, 251, 263, 262, 244, 265, 275, 234, 227, 260, 255, 249, 244, 282, 273, 259, 250, 222, 258, 239, 238, 272, 275, 243, 250, 274] payload_rows_per_dst=[5482, 5548, 5480, 5516, 5492, 5510, 5482, 5538] dup_rows_per_dst=[2552, 2664, 2732, 2792, 2640, 2760, 2720, 2628] combine_rows_per_src=[5482, 5548, 5480, 5516, 5492, 5510, 5482, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229168 combine_payload_bytes=156450816 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156679984 ingress_bytes=213073920 link_bytes=213073920 token_bytes=313130800 weight_movement_bytes=56623104 total_nvlink_bytes=369753904 weight_to_token_ratio=0.180829 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041569463e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38406 dup_rows=21628 dup_rows_remote=18864 dedup_ratio=0.330017 router_tokens_per_expert=[250, 252, 226, 272, 274, 272, 287, 258, 263, 248, 229, 251, 251, 239, 262, 266, 233, 272, 251, 252, 262, 262, 259, 273, 280, 229, 259, 263, 261, 277, 261, 254, 260, 272, 260, 223, 262, 267, 254, 253, 238, 250, 246, 277, 255, 251, 238, 245, 245, 243, 252, 265, 269, 250, 281, 271, 273, 223, 271, 253, 269, 266, 261, 241, 263, 265, 241, 275, 240, 268, 251, 254, 278, 276, 269, 227, 259, 230, 243, 236, 255, 275, 271, 250, 273, 267, 260, 226, 255, 268, 267, 266, 283, 248, 252, 242, 259, 254, 262, 262, 265, 252, 262, 270, 261, 262, 231, 261, 214, 241, 272, 278, 257, 255, 243, 256, 276, 250, 246, 261, 205, 232, 271, 249, 265, 242, 240, 249] payload_rows_per_dst=[5516, 5540, 5518, 5502, 5470, 5476, 5514, 5372] dup_rows_per_dst=[2684, 2756, 2584, 2764, 2680, 2840, 2698, 2622] combine_rows_per_src=[5516, 5540, 5518, 5502, 5470, 5476, 5514, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38386 dup_rows=21662 dup_rows_remote=18946 dedup_ratio=0.330536 router_tokens_per_expert=[228, 285, 265, 245, 271, 262, 259, 228, 280, 239, 252, 271, 245, 229, 272, 264, 235, 272, 226, 249, 235, 250, 287, 261, 244, 242, 232, 277, 259, 257, 240, 269, 282, 276, 240, 282, 272, 254, 238, 270, 270, 265, 245, 259, 260, 254, 251, 241, 277, 259, 261, 240, 244, 268, 246, 243, 249, 245, 251, 248, 240, 281, 261, 257, 261, 264, 238, 269, 257, 246, 252, 256, 292, 226, 245, 244, 272, 264, 257, 242, 249, 291, 254, 261, 241, 232, 262, 251, 256, 274, 267, 302, 258, 268, 234, 234, 260, 272, 260, 273, 234, 278, 213, 265, 228, 271, 244, 253, 272, 269, 265, 238, 260, 260, 232, 261, 278, 237, 272, 281, 252, 261, 264, 245, 242, 279, 227, 244] payload_rows_per_dst=[5488, 5474, 5584, 5422, 5480, 5490, 5454, 5482] dup_rows_per_dst=[2716, 2670, 2722, 2718, 2614, 2838, 2676, 2708] combine_rows_per_src=[5488, 5474, 5584, 5422, 5480, 5490, 5454, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157966336 dispatch_meta_bytes=229336 combine_payload_bytes=157966336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158195672 ingress_bytes=214589440 link_bytes=214589440 token_bytes=316162008 weight_movement_bytes=56623104 total_nvlink_bytes=372785112 weight_to_token_ratio=0.179095 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.056090429e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38288 dup_rows=21872 dup_rows_remote=19250 dedup_ratio=0.333740 router_tokens_per_expert=[264, 252, 227, 245, 269, 265, 247, 259, 272, 263, 257, 261, 258, 270, 278, 232, 274, 287, 280, 266, 272, 249, 210, 288, 277, 249, 223, 259, 268, 271, 263, 254, 268, 254, 262, 262, 259, 246, 257, 280, 250, 259, 236, 286, 246, 239, 261, 225, 243, 256, 283, 262, 268, 263, 234, 256, 250, 247, 257, 232, 231, 251, 239, 237, 233, 241, 244, 274, 256, 272, 238, 275, 252, 245, 228, 278, 250, 224, 257, 232, 245, 236, 245, 252, 255, 277, 246, 236, 261, 274, 269, 274, 262, 259, 241, 257, 251, 248, 263, 281, 268, 244, 252, 251, 267, 250, 268, 247, 270, 249, 256, 230, 280, 271, 245, 271, 244, 255, 261, 260, 259, 253, 271, 269, 279, 253, 260, 246] payload_rows_per_dst=[5508, 5534, 5402, 5326, 5376, 5404, 5544, 5570] dup_rows_per_dst=[2730, 2846, 2778, 2692, 2622, 2774, 2646, 2784] combine_rows_per_src=[5508, 5534, 5402, 5326, 5376, 5404, 5544, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229080 combine_payload_bytes=157310976 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157540056 ingress_bytes=195059712 link_bytes=195059712 token_bytes=314851032 weight_movement_bytes=37748736 total_nvlink_bytes=352599768 weight_to_token_ratio=0.119894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868966185e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38110 dup_rows=21832 dup_rows_remote=19032 dedup_ratio=0.333130 router_tokens_per_expert=[235, 266, 245, 253, 264, 248, 255, 250, 248, 277, 224, 249, 284, 242, 269, 268, 223, 218, 246, 230, 253, 270, 249, 241, 255, 269, 249, 268, 257, 256, 269, 255, 248, 273, 243, 258, 245, 282, 279, 248, 270, 267, 246, 238, 244, 242, 261, 254, 249, 258, 253, 267, 240, 258, 248, 253, 225, 261, 250, 262, 295, 252, 244, 262, 272, 268, 242, 264, 256, 277, 271, 244, 261, 251, 247, 266, 254, 246, 279, 234, 252, 231, 250, 259, 254, 246, 249, 248, 255, 261, 264, 265, 271, 297, 234, 256, 258, 244, 254, 257, 254, 246, 268, 245, 261, 259, 238, 277, 254, 263, 258, 266, 305, 255, 233, 246, 252, 248, 235, 281, 271, 289, 270, 253, 271, 257, 268, 248] payload_rows_per_dst=[5440, 5360, 5490, 5488, 5496, 5440, 5396, 5594] dup_rows_per_dst=[2714, 2656, 2706, 2654, 2768, 2660, 2874, 2800] combine_rows_per_src=[5440, 5360, 5490, 5488, 5496, 5440, 5396, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38370 dup_rows=21768 dup_rows_remote=19002 dedup_ratio=0.332153 router_tokens_per_expert=[243, 248, 230, 270, 266, 242, 235, 234, 275, 241, 267, 264, 268, 233, 251, 251, 289, 271, 243, 239, 263, 266, 254, 227, 268, 252, 248, 267, 236, 239, 251, 247, 253, 251, 251, 260, 247, 265, 224, 227, 274, 271, 250, 281, 253, 260, 243, 272, 252, 265, 263, 247, 259, 271, 257, 269, 250, 257, 262, 241, 278, 266, 279, 229, 265, 272, 253, 274, 262, 250, 265, 224, 261, 246, 229, 259, 266, 258, 238, 245, 275, 287, 263, 252, 265, 276, 248, 280, 256, 266, 238, 298, 253, 246, 259, 265, 250, 264, 245, 240, 232, 269, 277, 257, 261, 261, 257, 272, 256, 257, 267, 269, 234, 285, 238, 263, 220, 235, 225, 260, 274, 238, 255, 270, 270, 246, 239, 283] payload_rows_per_dst=[5424, 5462, 5398, 5550, 5430, 5586, 5456, 5462] dup_rows_per_dst=[2612, 2658, 2766, 2740, 2704, 2868, 2812, 2608] combine_rows_per_src=[5424, 5462, 5398, 5550, 5430, 5586, 5456, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38226 dup_rows=21830 dup_rows_remote=19146 dedup_ratio=0.333099 router_tokens_per_expert=[262, 225, 256, 259, 241, 288, 280, 248, 255, 270, 258, 241, 276, 230, 273, 250, 245, 264, 253, 283, 229, 235, 276, 243, 263, 270, 242, 252, 248, 254, 253, 257, 270, 276, 260, 265, 241, 253, 256, 241, 239, 261, 264, 255, 245, 268, 258, 269, 232, 265, 259, 252, 261, 238, 242, 261, 254, 230, 266, 271, 287, 240, 267, 258, 249, 260, 263, 251, 254, 254, 268, 261, 245, 258, 251, 255, 254, 259, 251, 241, 260, 268, 253, 272, 275, 230, 277, 286, 242, 273, 262, 230, 226, 258, 238, 232, 260, 263, 274, 282, 279, 257, 260, 269, 242, 234, 228, 232, 285, 263, 236, 267, 255, 247, 260, 242, 246, 265, 268, 260, 234, 245, 261, 261, 272, 262, 252, 268] payload_rows_per_dst=[5526, 5402, 5458, 5418, 5460, 5480, 5546, 5416] dup_rows_per_dst=[2698, 2732, 2784, 2748, 2688, 2684, 2716, 2780] combine_rows_per_src=[5526, 5402, 5458, 5418, 5460, 5480, 5546, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229328 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458384 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687440 weight_movement_bytes=18874368 total_nvlink_bytes=333561808 weight_to_token_ratio=0.059978 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=230152 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057800 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885448 weight_movement_bytes=18874368 total_nvlink_bytes=332759816 weight_to_token_ratio=0.060131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=228568 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156327128 ingress_bytes=193847296 link_bytes=193847296 token_bytes=312425688 weight_movement_bytes=37748736 total_nvlink_bytes=350174424 weight_to_token_ratio=0.120825 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857349411e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229488 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157393008 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556528 weight_movement_bytes=18874368 total_nvlink_bytes=333430896 weight_to_token_ratio=0.060003 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38240 dup_rows=21832 dup_rows_remote=19054 dedup_ratio=0.333130 router_tokens_per_expert=[283, 277, 248, 243, 235, 269, 263, 267, 243, 268, 249, 259, 244, 236, 249, 231, 290, 263, 279, 242, 245, 258, 250, 276, 286, 233, 249, 251, 263, 223, 261, 252, 246, 261, 282, 253, 270, 259, 263, 266, 260, 225, 250, 222, 266, 252, 269, 254, 263, 229, 254, 286, 273, 262, 255, 251, 268, 262, 260, 266, 262, 249, 258, 244, 228, 252, 257, 249, 241, 229, 252, 277, 262, 267, 258, 275, 271, 249, 247, 256, 243, 258, 227, 261, 252, 247, 271, 265, 231, 244, 260, 286, 232, 251, 215, 257, 248, 278, 249, 234, 278, 236, 248, 264, 303, 257, 281, 272, 248, 255, 255, 274, 248, 271, 239, 227, 253, 257, 232, 273, 245, 285, 277, 264, 261, 256, 249, 256] payload_rows_per_dst=[5448, 5464, 5470, 5528, 5458, 5364, 5560, 5412] dup_rows_per_dst=[2680, 2778, 2726, 2756, 2682, 2636, 2800, 2774] combine_rows_per_src=[5448, 5464, 5470, 5528, 5458, 5364, 5560, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229488 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156803184 ingress_bytes=213196800 link_bytes=213196800 token_bytes=313376880 weight_movement_bytes=56623104 total_nvlink_bytes=369999984 weight_to_token_ratio=0.180687 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042746838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38394 dup_rows=21674 dup_rows_remote=18984 dedup_ratio=0.330719 router_tokens_per_expert=[231, 233, 250, 267, 239, 274, 246, 289, 261, 255, 255, 237, 265, 271, 247, 275, 264, 274, 258, 258, 261, 264, 243, 244, 248, 275, 251, 253, 249, 268, 231, 243, 263, 266, 254, 249, 256, 246, 249, 258, 269, 246, 269, 223, 262, 231, 239, 270, 275, 268, 275, 251, 258, 252, 237, 266, 245, 272, 232, 228, 231, 236, 295, 261, 250, 256, 247, 272, 250, 240, 253, 245, 249, 254, 242, 269, 251, 236, 269, 258, 241, 253, 251, 280, 257, 240, 215, 260, 236, 291, 256, 247, 256, 272, 268, 276, 245, 261, 257, 272, 255, 245, 261, 276, 241, 267, 212, 257, 222, 253, 282, 273, 271, 254, 287, 265, 274, 296, 246, 272, 266, 242, 266, 260, 247, 281, 269, 242] payload_rows_per_dst=[5476, 5426, 5440, 5510, 5494, 5470, 5468, 5578] dup_rows_per_dst=[2714, 2668, 2660, 2728, 2588, 2728, 2690, 2898] combine_rows_per_src=[5476, 5426, 5440, 5510, 5494, 5470, 5468, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38258 dup_rows=21814 dup_rows_remote=19054 dedup_ratio=0.332855 router_tokens_per_expert=[270, 259, 249, 271, 256, 265, 247, 247, 242, 287, 280, 251, 283, 282, 253, 232, 243, 256, 246, 242, 268, 256, 258, 262, 265, 245, 244, 256, 289, 244, 294, 244, 255, 251, 242, 265, 259, 258, 281, 275, 219, 244, 235, 256, 266, 260, 251, 256, 250, 263, 268, 269, 250, 241, 259, 241, 250, 259, 264, 252, 254, 258, 266, 268, 265, 266, 253, 242, 267, 245, 259, 260, 257, 292, 226, 260, 221, 240, 271, 236, 239, 277, 244, 264, 243, 242, 238, 280, 266, 285, 220, 252, 271, 273, 246, 252, 261, 244, 294, 249, 274, 267, 253, 241, 259, 249, 232, 249, 264, 240, 248, 232, 227, 235, 270, 257, 244, 268, 269, 245, 262, 275, 242, 263, 268, 250, 263, 251] payload_rows_per_dst=[5502, 5490, 5416, 5464, 5446, 5512, 5476, 5416] dup_rows_per_dst=[2846, 2734, 2730, 2760, 2674, 2672, 2636, 2762] combine_rows_per_src=[5502, 5490, 5416, 5464, 5446, 5512, 5476, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229176 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860216 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491256 weight_movement_bytes=18874368 total_nvlink_bytes=332365624 weight_to_token_ratio=0.060207 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229512 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491336 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314753160 weight_movement_bytes=18874368 total_nvlink_bytes=333627528 weight_to_token_ratio=0.059966 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229248 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156934016 ingress_bytes=232202240 link_bytes=232202240 token_bytes=313638784 weight_movement_bytes=75497472 total_nvlink_bytes=389136256 weight_to_token_ratio=0.240715 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224847613e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43460 payload_rows_remote=38020 dup_rows=22076 dup_rows_remote=19340 dedup_ratio=0.336853 router_tokens_per_expert=[288, 272, 240, 229, 255, 262, 240, 259, 269, 250, 268, 221, 275, 279, 277, 236, 255, 244, 239, 243, 250, 270, 270, 248, 260, 279, 261, 246, 279, 271, 250, 250, 267, 238, 267, 265, 241, 269, 246, 245, 247, 251, 271, 243, 268, 245, 245, 271, 273, 251, 253, 252, 262, 247, 268, 253, 265, 246, 257, 265, 235, 250, 255, 267, 246, 254, 271, 207, 249, 253, 260, 293, 287, 253, 268, 249, 245, 254, 250, 249, 238, 250, 254, 241, 255, 268, 248, 266, 240, 235, 239, 265, 244, 262, 264, 266, 270, 256, 252, 263, 253, 260, 269, 257, 270, 251, 246, 271, 274, 250, 247, 247, 268, 249, 249, 262, 249, 261, 238, 290, 244, 269, 258, 259, 268, 230, 241, 261] payload_rows_per_dst=[5412, 5400, 5480, 5412, 5440, 5376, 5554, 5386] dup_rows_per_dst=[2828, 2830, 2678, 2786, 2736, 2694, 2718, 2806] combine_rows_per_src=[5412, 5400, 5480, 5412, 5440, 5376, 5554, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38290 dup_rows=21776 dup_rows_remote=18952 dedup_ratio=0.332275 router_tokens_per_expert=[236, 268, 267, 265, 224, 271, 265, 250, 228, 254, 238, 271, 231, 276, 235, 258, 254, 250, 264, 274, 261, 259, 274, 263, 279, 239, 258, 246, 247, 249, 291, 256, 254, 243, 245, 255, 264, 263, 258, 267, 269, 243, 257, 265, 246, 237, 262, 261, 253, 252, 261, 257, 275, 273, 260, 219, 266, 257, 262, 258, 230, 254, 243, 219, 252, 283, 247, 241, 251, 270, 267, 242, 258, 280, 260, 263, 264, 246, 276, 235, 238, 263, 262, 246, 257, 244, 279, 245, 233, 234, 273, 257, 254, 258, 261, 258, 264, 234, 255, 251, 293, 254, 274, 273, 229, 241, 238, 247, 267, 271, 233, 271, 258, 291, 232, 249, 268, 291, 258, 261, 284, 258, 272, 256, 229, 247, 250, 243] payload_rows_per_dst=[5470, 5572, 5490, 5440, 5522, 5482, 5418, 5366] dup_rows_per_dst=[2824, 2956, 2698, 2638, 2738, 2642, 2772, 2508] combine_rows_per_src=[5470, 5572, 5490, 5440, 5522, 5482, 5418, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38126 dup_rows=21866 dup_rows_remote=19074 dedup_ratio=0.333649 router_tokens_per_expert=[277, 254, 244, 235, 210, 260, 248, 265, 252, 238, 241, 223, 259, 253, 222, 247, 270, 256, 257, 243, 277, 265, 277, 272, 240, 230, 259, 270, 235, 245, 272, 272, 234, 234, 248, 273, 249, 260, 239, 243, 241, 251, 229, 255, 273, 249, 238, 267, 286, 281, 273, 274, 263, 244, 267, 251, 239, 240, 239, 284, 260, 252, 273, 242, 231, 274, 246, 233, 279, 269, 263, 266, 221, 273, 258, 246, 232, 275, 261, 254, 264, 276, 280, 252, 281, 247, 268, 233, 251, 274, 264, 275, 283, 251, 254, 277, 248, 256, 276, 275, 266, 250, 256, 242, 259, 245, 229, 256, 253, 268, 267, 263, 258, 234, 242, 269, 263, 273, 255, 238, 261, 283, 258, 279, 220, 241, 291, 264] payload_rows_per_dst=[5300, 5526, 5330, 5304, 5404, 5702, 5560, 5544] dup_rows_per_dst=[2538, 2772, 2636, 2474, 2680, 3166, 2808, 2792] combine_rows_per_src=[5300, 5526, 5330, 5304, 5404, 5702, 5560, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38166 dup_rows=21814 dup_rows_remote=19022 dedup_ratio=0.332855 router_tokens_per_expert=[260, 276, 251, 259, 228, 247, 280, 269, 266, 260, 268, 251, 243, 244, 251, 257, 237, 285, 235, 252, 276, 261, 253, 272, 268, 271, 266, 265, 248, 251, 256, 252, 256, 267, 249, 265, 248, 244, 262, 274, 245, 264, 258, 250, 265, 271, 302, 254, 265, 244, 274, 243, 252, 267, 263, 262, 263, 247, 299, 262, 278, 230, 250, 264, 254, 232, 257, 263, 264, 256, 258, 282, 258, 259, 233, 252, 238, 248, 268, 249, 266, 259, 248, 244, 248, 240, 249, 267, 253, 256, 243, 231, 274, 241, 238, 264, 244, 248, 249, 265, 245, 222, 245, 255, 264, 244, 242, 266, 242, 251, 237, 253, 269, 242, 269, 270, 237, 212, 281, 244, 267, 272, 253, 245, 257, 275, 265, 251] payload_rows_per_dst=[5418, 5582, 5556, 5466, 5498, 5346, 5388, 5468] dup_rows_per_dst=[2802, 2714, 2792, 2860, 2644, 2696, 2556, 2750] combine_rows_per_src=[5418, 5582, 5556, 5466, 5498, 5346, 5388, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38300 dup_rows=21830 dup_rows_remote=19122 dedup_ratio=0.333099 router_tokens_per_expert=[227, 291, 256, 233, 257, 245, 240, 233, 264, 249, 236, 284, 267, 265, 270, 252, 263, 237, 282, 284, 275, 245, 251, 240, 252, 243, 260, 255, 248, 235, 245, 243, 261, 265, 274, 257, 241, 264, 256, 253, 261, 260, 253, 225, 247, 232, 260, 283, 237, 273, 253, 262, 266, 269, 248, 249, 277, 265, 272, 270, 252, 252, 263, 256, 265, 264, 289, 269, 261, 240, 260, 253, 258, 262, 256, 236, 252, 253, 264, 248, 252, 253, 239, 258, 264, 252, 273, 247, 229, 264, 247, 277, 250, 245, 278, 229, 229, 250, 261, 253, 242, 225, 277, 264, 249, 249, 250, 258, 269, 255, 252, 252, 252, 289, 285, 247, 235, 268, 264, 246, 260, 269, 256, 275, 258, 241, 268, 250] payload_rows_per_dst=[5432, 5422, 5486, 5548, 5524, 5406, 5392, 5496] dup_rows_per_dst=[2706, 2694, 2698, 2780, 2736, 2708, 2678, 2830] combine_rows_per_src=[5432, 5422, 5486, 5548, 5524, 5406, 5392, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=229440 combine_payload_bytes=155729920 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=155959360 ingress_bytes=193478656 link_bytes=193478656 token_bytes=311689280 weight_movement_bytes=37748736 total_nvlink_bytes=349438016 weight_to_token_ratio=0.121110 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853817284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228800 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156392896 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312556992 weight_movement_bytes=18874368 total_nvlink_bytes=331431360 weight_to_token_ratio=0.060387 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=228968 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157064808 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313900648 weight_movement_bytes=18874368 total_nvlink_bytes=332775016 weight_to_token_ratio=0.060128 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228752 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156556688 ingress_bytes=175202304 link_bytes=175202304 token_bytes=312884624 weight_movement_bytes=18874368 total_nvlink_bytes=331758992 weight_to_token_ratio=0.060324 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678702272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38346 dup_rows=21732 dup_rows_remote=19068 dedup_ratio=0.331604 router_tokens_per_expert=[298, 279, 242, 232, 264, 248, 260, 241, 237, 258, 258, 248, 246, 255, 267, 263, 227, 256, 254, 245, 241, 278, 249, 249, 271, 264, 260, 259, 223, 263, 267, 255, 273, 249, 246, 238, 247, 256, 251, 246, 259, 233, 263, 263, 260, 231, 234, 259, 225, 242, 223, 270, 238, 238, 259, 263, 247, 269, 269, 252, 240, 268, 243, 278, 235, 256, 259, 280, 266, 273, 260, 258, 264, 275, 283, 258, 243, 233, 259, 253, 244, 216, 264, 244, 253, 254, 268, 270, 243, 223, 244, 272, 281, 284, 267, 255, 276, 245, 258, 249, 271, 255, 259, 276, 266, 266, 236, 274, 267, 256, 263, 280, 274, 257, 238, 228, 258, 240, 264, 266, 284, 279, 251, 240, 240, 286, 261, 279] payload_rows_per_dst=[5468, 5458, 5450, 5396, 5546, 5454, 5518, 5514] dup_rows_per_dst=[2724, 2664, 2566, 2652, 2764, 2710, 2876, 2776] combine_rows_per_src=[5468, 5458, 5450, 5396, 5546, 5454, 5518, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229688 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106488 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983288 weight_movement_bytes=18874368 total_nvlink_bytes=332857656 weight_to_token_ratio=0.060113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38374 dup_rows=21654 dup_rows_remote=18910 dedup_ratio=0.330414 router_tokens_per_expert=[263, 284, 250, 254, 265, 259, 276, 279, 212, 269, 258, 269, 261, 255, 222, 249, 235, 272, 256, 263, 257, 245, 274, 241, 269, 279, 276, 256, 237, 267, 249, 261, 237, 254, 245, 264, 273, 240, 264, 231, 249, 269, 250, 244, 251, 215, 283, 238, 260, 261, 266, 267, 250, 268, 268, 274, 243, 258, 284, 234, 276, 243, 254, 220, 284, 264, 240, 266, 253, 244, 284, 276, 283, 259, 274, 235, 257, 240, 267, 286, 259, 259, 266, 247, 265, 260, 271, 271, 231, 248, 256, 263, 266, 260, 252, 262, 237, 258, 238, 249, 262, 264, 239, 263, 286, 260, 261, 205, 273, 267, 238, 237, 253, 226, 239, 239, 247, 275, 259, 233, 240, 240, 268, 254, 238, 265, 249, 263] payload_rows_per_dst=[5518, 5480, 5464, 5508, 5578, 5474, 5430, 5430] dup_rows_per_dst=[2732, 2794, 2550, 2744, 2846, 2798, 2644, 2546] combine_rows_per_src=[5518, 5480, 5464, 5508, 5578, 5474, 5430, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38214 dup_rows=21900 dup_rows_remote=19184 dedup_ratio=0.334167 router_tokens_per_expert=[230, 239, 248, 268, 278, 289, 218, 276, 235, 240, 247, 244, 259, 261, 250, 257, 278, 266, 247, 263, 280, 267, 276, 270, 247, 259, 291, 237, 233, 242, 244, 221, 248, 250, 242, 224, 265, 255, 238, 282, 227, 270, 269, 255, 252, 251, 262, 243, 269, 271, 264, 271, 283, 278, 240, 257, 258, 257, 246, 228, 250, 278, 255, 258, 274, 246, 232, 265, 258, 254, 256, 243, 261, 262, 257, 259, 246, 245, 253, 266, 288, 238, 245, 264, 279, 237, 257, 245, 232, 245, 251, 251, 258, 267, 272, 273, 264, 253, 246, 254, 245, 240, 240, 246, 275, 264, 269, 244, 277, 244, 259, 249, 267, 267, 260, 263, 257, 263, 258, 239, 261, 282, 242, 261, 264, 263, 252, 265] payload_rows_per_dst=[5370, 5500, 5442, 5508, 5424, 5508, 5422, 5462] dup_rows_per_dst=[2708, 2742, 2624, 2818, 2730, 2696, 2716, 2866] combine_rows_per_src=[5370, 5500, 5442, 5508, 5424, 5508, 5422, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229656 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157294872 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314360088 weight_movement_bytes=18874368 total_nvlink_bytes=333234456 weight_to_token_ratio=0.060041 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229136 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409040 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314588944 weight_movement_bytes=18874368 total_nvlink_bytes=333463312 weight_to_token_ratio=0.059997 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229592 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156754136 ingress_bytes=194273280 link_bytes=194273280 token_bytes=313278680 weight_movement_bytes=37748736 total_nvlink_bytes=351027416 weight_to_token_ratio=0.120496 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861430981e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38406 dup_rows=21778 dup_rows_remote=19094 dedup_ratio=0.332306 router_tokens_per_expert=[271, 270, 237, 262, 262, 254, 250, 253, 274, 241, 265, 252, 232, 253, 296, 265, 254, 257, 303, 234, 232, 230, 294, 244, 234, 249, 274, 237, 243, 259, 279, 220, 239, 255, 272, 248, 268, 237, 250, 290, 270, 268, 256, 253, 242, 255, 223, 259, 253, 229, 274, 253, 271, 256, 239, 253, 246, 263, 283, 260, 248, 259, 249, 247, 247, 255, 246, 263, 236, 260, 286, 243, 228, 285, 256, 237, 243, 258, 238, 237, 261, 277, 270, 261, 252, 234, 272, 288, 236, 260, 303, 246, 262, 269, 266, 262, 285, 261, 240, 252, 245, 262, 270, 253, 254, 285, 260, 258, 252, 285, 246, 248, 261, 240, 271, 256, 276, 232, 252, 255, 256, 252, 240, 260, 252, 233, 250, 241] payload_rows_per_dst=[5518, 5460, 5500, 5422, 5352, 5576, 5504, 5426] dup_rows_per_dst=[2756, 2626, 2670, 2744, 2684, 2862, 2808, 2628] combine_rows_per_src=[5518, 5460, 5500, 5422, 5352, 5576, 5504, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38224 dup_rows=21698 dup_rows_remote=18782 dedup_ratio=0.331085 router_tokens_per_expert=[264, 281, 269, 272, 255, 230, 231, 233, 250, 271, 238, 269, 260, 245, 277, 244, 237, 246, 282, 232, 282, 237, 251, 247, 275, 239, 284, 244, 252, 246, 233, 270, 267, 253, 236, 264, 257, 258, 276, 250, 262, 270, 260, 294, 279, 242, 262, 246, 234, 243, 231, 292, 241, 243, 251, 266, 265, 275, 268, 285, 239, 265, 262, 267, 233, 291, 250, 237, 270, 246, 276, 268, 256, 251, 273, 272, 256, 254, 244, 247, 261, 263, 245, 239, 233, 264, 264, 263, 239, 263, 266, 247, 277, 249, 256, 249, 281, 251, 270, 269, 253, 242, 239, 253, 260, 255, 263, 247, 252, 264, 265, 241, 254, 249, 256, 241, 237, 230, 263, 234, 244, 268, 269, 255, 260, 247, 237, 268] payload_rows_per_dst=[5264, 5628, 5520, 5526, 5490, 5432, 5364, 5614] dup_rows_per_dst=[2352, 3048, 2832, 2728, 2754, 2582, 2486, 2916] combine_rows_per_src=[5264, 5628, 5520, 5526, 5490, 5432, 5364, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38326 dup_rows=21764 dup_rows_remote=19114 dedup_ratio=0.332092 router_tokens_per_expert=[267, 275, 292, 269, 251, 282, 251, 232, 261, 250, 230, 267, 257, 281, 252, 246, 261, 247, 265, 272, 284, 262, 260, 255, 225, 265, 243, 244, 266, 260, 252, 267, 254, 247, 251, 204, 237, 304, 257, 265, 252, 244, 223, 270, 260, 270, 271, 239, 262, 249, 274, 260, 279, 248, 236, 269, 225, 282, 265, 242, 279, 254, 255, 251, 260, 242, 248, 261, 245, 229, 249, 252, 270, 263, 242, 237, 250, 264, 287, 269, 246, 252, 244, 234, 247, 275, 275, 244, 243, 278, 261, 273, 267, 254, 257, 225, 256, 259, 267, 241, 277, 227, 268, 279, 254, 247, 247, 236, 253, 231, 288, 270, 268, 267, 254, 242, 229, 258, 262, 266, 257, 253, 259, 239, 235, 227, 252, 288] payload_rows_per_dst=[5532, 5474, 5446, 5548, 5502, 5420, 5470, 5380] dup_rows_per_dst=[2794, 2782, 2650, 2712, 2634, 2730, 2730, 2732] combine_rows_per_src=[5532, 5474, 5446, 5548, 5502, 5420, 5470, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38062 dup_rows=21906 dup_rows_remote=18912 dedup_ratio=0.334259 router_tokens_per_expert=[272, 256, 255, 253, 273, 254, 274, 254, 237, 243, 258, 248, 282, 273, 243, 259, 257, 243, 246, 254, 221, 266, 248, 243, 268, 267, 237, 252, 274, 282, 264, 287, 266, 252, 260, 277, 256, 245, 267, 272, 256, 277, 263, 244, 251, 231, 262, 255, 264, 271, 216, 230, 279, 276, 244, 267, 265, 287, 236, 249, 243, 225, 244, 258, 246, 255, 234, 226, 256, 258, 269, 273, 282, 263, 243, 262, 272, 258, 242, 245, 241, 238, 267, 230, 228, 251, 228, 252, 259, 222, 265, 293, 229, 255, 248, 277, 259, 261, 241, 251, 301, 269, 234, 287, 279, 277, 259, 240, 255, 255, 251, 256, 279, 284, 251, 238, 277, 272, 242, 214, 250, 226, 259, 263, 260, 260, 257, 263] payload_rows_per_dst=[5568, 5480, 5422, 5396, 5590, 5400, 5450, 5324] dup_rows_per_dst=[2994, 2738, 2846, 2712, 2816, 2700, 2662, 2438] combine_rows_per_src=[5568, 5480, 5422, 5396, 5590, 5400, 5450, 5324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38094 dup_rows=21948 dup_rows_remote=19266 dedup_ratio=0.334900 router_tokens_per_expert=[243, 255, 261, 243, 261, 236, 267, 269, 272, 265, 256, 243, 268, 281, 250, 253, 233, 243, 257, 264, 248, 273, 249, 280, 286, 244, 248, 235, 266, 245, 256, 241, 243, 275, 252, 233, 257, 245, 268, 247, 216, 247, 249, 253, 267, 296, 258, 256, 278, 220, 244, 240, 240, 251, 215, 243, 258, 267, 254, 267, 242, 250, 244, 251, 258, 275, 246, 278, 247, 264, 264, 269, 226, 240, 264, 291, 252, 261, 262, 254, 254, 291, 255, 276, 237, 227, 253, 243, 254, 281, 239, 244, 257, 265, 269, 243, 238, 249, 242, 295, 245, 259, 258, 263, 276, 259, 282, 266, 265, 283, 252, 285, 223, 266, 263, 237, 293, 255, 229, 288, 274, 243, 243, 276, 231, 269, 265, 240] payload_rows_per_dst=[5456, 5404, 5468, 5260, 5548, 5494, 5516, 5442] dup_rows_per_dst=[2790, 2732, 2656, 2668, 2754, 2682, 2918, 2748] combine_rows_per_src=[5456, 5404, 5468, 5260, 5548, 5494, 5516, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=230000 combine_payload_bytes=157310976 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157540976 ingress_bytes=213934080 link_bytes=213934080 token_bytes=314851952 weight_movement_bytes=56623104 total_nvlink_bytes=371475056 weight_to_token_ratio=0.179840 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049811092e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228024 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156793528 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313359032 weight_movement_bytes=18874368 total_nvlink_bytes=332233400 weight_to_token_ratio=0.060232 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229760 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213056 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196352 weight_movement_bytes=18874368 total_nvlink_bytes=333070720 weight_to_token_ratio=0.060072 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=227896 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156129848 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312031800 weight_movement_bytes=18874368 total_nvlink_bytes=330906168 weight_to_token_ratio=0.060489 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38498 dup_rows=21636 dup_rows_remote=19108 dedup_ratio=0.330139 router_tokens_per_expert=[259, 273, 244, 277, 280, 271, 255, 265, 260, 294, 246, 265, 269, 289, 260, 227, 255, 251, 235, 242, 264, 263, 224, 254, 236, 251, 236, 244, 263, 251, 254, 242, 273, 277, 254, 265, 259, 249, 265, 234, 252, 239, 256, 246, 237, 288, 256, 238, 233, 245, 242, 252, 257, 233, 253, 259, 264, 242, 237, 239, 273, 247, 285, 265, 223, 293, 285, 263, 252, 280, 227, 250, 234, 247, 257, 248, 248, 272, 251, 272, 288, 263, 297, 260, 239, 267, 247, 260, 242, 275, 268, 249, 235, 264, 259, 239, 293, 253, 240, 252, 277, 250, 256, 240, 259, 230, 261, 239, 274, 262, 271, 249, 252, 260, 263, 263, 238, 253, 259, 244, 271, 268, 260, 236, 304, 251, 239, 234] payload_rows_per_dst=[5574, 5402, 5540, 5400, 5454, 5514, 5540, 5476] dup_rows_per_dst=[2894, 2528, 2636, 2652, 2750, 2790, 2672, 2714] combine_rows_per_src=[5574, 5402, 5540, 5400, 5454, 5514, 5540, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38354 dup_rows=21632 dup_rows_remote=18850 dedup_ratio=0.330078 router_tokens_per_expert=[244, 255, 242, 280, 256, 245, 258, 253, 262, 248, 267, 263, 251, 256, 242, 273, 244, 231, 240, 261, 256, 261, 253, 264, 238, 246, 240, 226, 244, 259, 275, 275, 267, 237, 256, 293, 245, 258, 231, 246, 241, 263, 241, 259, 267, 261, 262, 268, 259, 259, 226, 250, 232, 276, 266, 250, 258, 276, 275, 271, 244, 265, 278, 281, 310, 243, 228, 271, 262, 245, 266, 258, 246, 257, 244, 236, 253, 274, 293, 243, 255, 261, 245, 250, 252, 249, 266, 266, 257, 237, 250, 288, 276, 301, 275, 288, 266, 241, 242, 244, 252, 241, 250, 252, 263, 286, 230, 239, 240, 228, 257, 245, 211, 241, 243, 229, 263, 253, 248, 269, 252, 306, 254, 291, 257, 260, 247, 254] payload_rows_per_dst=[5522, 5418, 5510, 5550, 5496, 5552, 5454, 5402] dup_rows_per_dst=[2668, 2608, 2680, 2782, 2762, 2880, 2498, 2754] combine_rows_per_src=[5522, 5418, 5510, 5550, 5496, 5552, 5454, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=229440 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156262464 ingress_bytes=212656128 link_bytes=212656128 token_bytes=312295488 weight_movement_bytes=56623104 total_nvlink_bytes=368918592 weight_to_token_ratio=0.181313 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037566385e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44012 payload_rows_remote=38492 dup_rows=21524 dup_rows_remote=18726 dedup_ratio=0.328430 router_tokens_per_expert=[251, 226, 245, 243, 260, 251, 240, 272, 254, 233, 260, 245, 244, 251, 278, 257, 279, 256, 266, 237, 228, 258, 282, 271, 245, 232, 255, 276, 268, 252, 279, 273, 256, 259, 266, 269, 252, 257, 259, 232, 213, 265, 241, 262, 263, 250, 230, 267, 251, 261, 270, 264, 238, 228, 250, 226, 261, 267, 248, 256, 264, 235, 249, 258, 264, 280, 249, 265, 244, 244, 275, 255, 238, 250, 276, 252, 273, 270, 237, 248, 236, 263, 240, 256, 255, 274, 258, 265, 249, 282, 254, 285, 242, 249, 261, 251, 250, 239, 295, 247, 242, 270, 304, 268, 251, 243, 241, 271, 262, 252, 236, 288, 249, 267, 266, 263, 267, 258, 283, 254, 234, 266, 213, 270, 266, 267, 256, 256] payload_rows_per_dst=[5384, 5522, 5478, 5484, 5510, 5538, 5520, 5576] dup_rows_per_dst=[2636, 2792, 2604, 2568, 2730, 2702, 2798, 2694] combine_rows_per_src=[5384, 5522, 5478, 5484, 5510, 5538, 5520, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157687808 dispatch_meta_bytes=230424 combine_payload_bytes=157687808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157918232 ingress_bytes=176562176 link_bytes=176562176 token_bytes=315606040 weight_movement_bytes=18874368 total_nvlink_bytes=334480408 weight_to_token_ratio=0.059804 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691731896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=228816 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157326800 ingress_bytes=213721088 link_bytes=213721088 token_bytes=314424784 weight_movement_bytes=56623104 total_nvlink_bytes=371047888 weight_to_token_ratio=0.180085 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047770308e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157663232 dispatch_meta_bytes=228872 combine_payload_bytes=157663232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157892104 ingress_bytes=214286336 link_bytes=214286336 token_bytes=315555336 weight_movement_bytes=56623104 total_nvlink_bytes=372178440 weight_to_token_ratio=0.179440 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053186236e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38184 dup_rows=21912 dup_rows_remote=19150 dedup_ratio=0.334351 router_tokens_per_expert=[283, 259, 245, 260, 226, 270, 266, 237, 246, 253, 247, 271, 263, 269, 274, 225, 266, 266, 263, 245, 246, 276, 235, 242, 296, 257, 255, 234, 237, 215, 256, 251, 229, 266, 259, 255, 265, 242, 268, 239, 255, 256, 255, 274, 237, 268, 257, 265, 271, 249, 258, 268, 284, 238, 253, 256, 247, 253, 269, 263, 247, 239, 249, 264, 246, 259, 261, 269, 254, 263, 237, 245, 236, 256, 251, 279, 261, 258, 267, 259, 243, 280, 264, 228, 271, 262, 266, 279, 262, 272, 271, 261, 278, 235, 258, 238, 256, 246, 242, 251, 273, 260, 262, 251, 266, 237, 247, 261, 281, 235, 273, 244, 251, 276, 231, 229, 263, 255, 245, 255, 257, 266, 265, 244, 277, 251, 274, 243] payload_rows_per_dst=[5416, 5440, 5456, 5426, 5440, 5488, 5502, 5456] dup_rows_per_dst=[2772, 2640, 2724, 2790, 2762, 2848, 2668, 2708] combine_rows_per_src=[5416, 5440, 5456, 5426, 5440, 5488, 5502, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38344 dup_rows=21842 dup_rows_remote=19178 dedup_ratio=0.333282 router_tokens_per_expert=[240, 244, 250, 266, 251, 295, 250, 248, 271, 244, 251, 270, 263, 250, 262, 267, 232, 267, 266, 257, 261, 258, 263, 258, 227, 252, 237, 267, 257, 226, 261, 257, 246, 266, 245, 257, 256, 260, 259, 241, 243, 257, 272, 236, 279, 262, 267, 229, 260, 266, 260, 247, 233, 279, 258, 264, 261, 251, 244, 263, 249, 281, 270, 253, 256, 251, 271, 242, 226, 264, 264, 274, 286, 260, 271, 269, 276, 267, 257, 247, 260, 260, 267, 264, 267, 269, 265, 275, 278, 247, 257, 267, 256, 245, 287, 247, 246, 272, 247, 239, 223, 234, 254, 266, 262, 240, 260, 275, 246, 227, 243, 253, 243, 256, 256, 237, 284, 246, 222, 270, 242, 248, 262, 240, 244, 277, 244, 236] payload_rows_per_dst=[5342, 5436, 5456, 5564, 5546, 5400, 5600, 5350] dup_rows_per_dst=[2688, 2656, 2694, 2714, 2816, 2662, 2948, 2664] combine_rows_per_src=[5342, 5436, 5456, 5564, 5546, 5400, 5600, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38352 dup_rows=21628 dup_rows_remote=18886 dedup_ratio=0.330017 router_tokens_per_expert=[264, 236, 268, 272, 251, 263, 240, 260, 229, 254, 243, 263, 253, 285, 254, 232, 253, 241, 265, 241, 244, 254, 259, 257, 247, 241, 260, 242, 268, 245, 261, 270, 286, 235, 273, 280, 233, 240, 250, 294, 246, 260, 254, 245, 246, 274, 271, 262, 269, 274, 270, 256, 255, 261, 249, 258, 261, 244, 247, 260, 277, 282, 279, 276, 229, 243, 274, 278, 265, 259, 248, 242, 268, 263, 263, 254, 243, 238, 269, 260, 257, 228, 249, 262, 229, 263, 268, 250, 255, 267, 244, 257, 236, 234, 280, 259, 260, 267, 250, 273, 259, 250, 274, 275, 264, 247, 250, 244, 239, 240, 275, 279, 246, 257, 254, 253, 273, 244, 246, 233, 257, 243, 259, 242, 230, 251, 275, 243] payload_rows_per_dst=[5446, 5456, 5556, 5590, 5526, 5430, 5502, 5402] dup_rows_per_dst=[2688, 2640, 2742, 2846, 2666, 2646, 2790, 2610] combine_rows_per_src=[5446, 5456, 5556, 5590, 5526, 5430, 5502, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38118 dup_rows=21880 dup_rows_remote=18882 dedup_ratio=0.333862 router_tokens_per_expert=[262, 223, 238, 262, 247, 260, 276, 270, 286, 259, 289, 282, 246, 261, 231, 253, 242, 229, 250, 249, 225, 250, 278, 258, 229, 254, 241, 252, 281, 264, 238, 258, 281, 245, 246, 262, 262, 246, 238, 244, 259, 223, 256, 260, 260, 245, 224, 241, 250, 250, 245, 261, 298, 285, 259, 287, 293, 242, 277, 246, 230, 237, 251, 267, 237, 266, 271, 245, 257, 249, 248, 236, 246, 273, 259, 270, 272, 254, 264, 283, 242, 251, 255, 254, 272, 284, 222, 248, 239, 260, 264, 264, 262, 238, 248, 239, 267, 242, 272, 272, 265, 252, 267, 297, 251, 270, 231, 242, 262, 275, 243, 238, 262, 252, 258, 239, 286, 249, 275, 252, 247, 264, 255, 263, 279, 264, 250, 242] payload_rows_per_dst=[5538, 5346, 5578, 5574, 5654, 5376, 5196, 5394] dup_rows_per_dst=[2998, 2688, 2796, 2744, 2808, 2708, 2460, 2678] combine_rows_per_src=[5538, 5346, 5578, 5574, 5654, 5376, 5196, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38308 dup_rows=21774 dup_rows_remote=18980 dedup_ratio=0.332245 router_tokens_per_expert=[258, 250, 251, 282, 260, 265, 259, 250, 234, 270, 249, 253, 233, 244, 265, 272, 253, 273, 260, 251, 260, 261, 273, 280, 281, 246, 275, 265, 252, 278, 253, 225, 255, 263, 236, 254, 246, 258, 269, 264, 254, 236, 228, 268, 258, 251, 262, 266, 273, 261, 248, 243, 258, 241, 214, 253, 244, 257, 258, 254, 265, 255, 208, 229, 267, 255, 263, 247, 239, 259, 261, 241, 261, 248, 243, 231, 264, 280, 250, 268, 261, 252, 233, 247, 276, 258, 276, 248, 279, 256, 243, 253, 272, 232, 252, 286, 247, 263, 251, 245, 244, 254, 295, 232, 258, 247, 264, 248, 278, 249, 275, 257, 256, 261, 263, 251, 264, 273, 266, 252, 246, 243, 244, 253, 284, 270, 277, 247] payload_rows_per_dst=[5500, 5538, 5450, 5390, 5438, 5454, 5522, 5470] dup_rows_per_dst=[2690, 2834, 2686, 2532, 2716, 2794, 2692, 2830] combine_rows_per_src=[5500, 5538, 5450, 5390, 5438, 5454, 5522, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229336 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156631000 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032664 weight_movement_bytes=18874368 total_nvlink_bytes=331907032 weight_to_token_ratio=0.060295 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=230088 combine_payload_bytes=157057024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157287112 ingress_bytes=213680128 link_bytes=213680128 token_bytes=314344136 weight_movement_bytes=56623104 total_nvlink_bytes=370967240 weight_to_token_ratio=0.180131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047377849e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=228952 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157318744 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314408536 weight_movement_bytes=18874368 total_nvlink_bytes=333282904 weight_to_token_ratio=0.060031 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38414 dup_rows=21688 dup_rows_remote=18950 dedup_ratio=0.330933 router_tokens_per_expert=[258, 246, 260, 249, 274, 273, 261, 253, 256, 271, 269, 246, 258, 249, 256, 235, 254, 239, 266, 253, 229, 232, 275, 243, 241, 263, 286, 258, 254, 248, 285, 260, 248, 277, 247, 246, 239, 246, 273, 225, 254, 231, 257, 232, 253, 250, 244, 252, 261, 267, 264, 254, 287, 253, 271, 237, 269, 236, 262, 266, 254, 270, 251, 257, 246, 267, 276, 267, 261, 226, 273, 250, 228, 259, 255, 256, 265, 243, 247, 257, 295, 243, 264, 263, 246, 235, 241, 228, 245, 269, 249, 238, 245, 268, 270, 244, 256, 246, 270, 272, 248, 238, 251, 250, 245, 247, 233, 265, 246, 278, 280, 276, 279, 260, 232, 238, 258, 298, 257, 245, 266, 250, 270, 274, 271, 296, 278, 243] payload_rows_per_dst=[5472, 5434, 5386, 5580, 5470, 5474, 5450, 5582] dup_rows_per_dst=[2756, 2738, 2562, 2738, 2682, 2612, 2752, 2848] combine_rows_per_src=[5472, 5434, 5386, 5580, 5470, 5474, 5450, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38192 dup_rows=21882 dup_rows_remote=19162 dedup_ratio=0.333893 router_tokens_per_expert=[261, 236, 250, 245, 255, 273, 238, 283, 286, 247, 250, 262, 261, 255, 274, 258, 264, 285, 230, 269, 256, 262, 248, 288, 269, 246, 244, 250, 270, 274, 256, 253, 249, 263, 253, 262, 264, 267, 267, 252, 237, 271, 231, 275, 267, 254, 256, 267, 269, 254, 268, 251, 262, 227, 244, 234, 245, 269, 250, 260, 272, 243, 264, 279, 248, 271, 247, 240, 244, 260, 258, 254, 213, 247, 252, 248, 259, 272, 242, 266, 256, 246, 274, 276, 239, 274, 270, 243, 236, 251, 257, 227, 242, 255, 262, 234, 253, 238, 245, 234, 234, 283, 242, 248, 280, 225, 263, 270, 276, 264, 276, 245, 278, 226, 279, 256, 246, 234, 258, 249, 292, 276, 272, 248, 244, 244, 230, 273] payload_rows_per_dst=[5466, 5480, 5458, 5462, 5392, 5460, 5404, 5532] dup_rows_per_dst=[2802, 2848, 2812, 2720, 2650, 2624, 2748, 2678] combine_rows_per_src=[5466, 5480, 5458, 5462, 5392, 5460, 5404, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38318 dup_rows=21652 dup_rows_remote=18804 dedup_ratio=0.330383 router_tokens_per_expert=[255, 260, 249, 242, 271, 249, 227, 253, 266, 269, 272, 252, 256, 268, 289, 256, 245, 245, 246, 281, 285, 256, 281, 233, 238, 246, 252, 252, 248, 280, 244, 260, 244, 262, 261, 243, 262, 260, 261, 235, 253, 268, 265, 222, 251, 280, 231, 279, 251, 252, 264, 262, 270, 281, 269, 267, 252, 227, 235, 241, 262, 276, 246, 249, 251, 257, 270, 255, 246, 247, 244, 255, 256, 243, 256, 281, 253, 260, 244, 264, 253, 262, 241, 247, 253, 245, 266, 230, 233, 240, 257, 253, 256, 252, 265, 260, 249, 262, 272, 267, 276, 260, 286, 255, 253, 265, 247, 265, 257, 259, 272, 262, 249, 238, 262, 277, 245, 246, 262, 245, 250, 234, 256, 233, 252, 297, 257, 256] payload_rows_per_dst=[5472, 5488, 5474, 5552, 5454, 5420, 5566, 5458] dup_rows_per_dst=[2796, 2696, 2680, 2656, 2710, 2606, 2848, 2660] combine_rows_per_src=[5472, 5488, 5474, 5552, 5454, 5420, 5566, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228000 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156359328 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312490656 weight_movement_bytes=18874368 total_nvlink_bytes=331365024 weight_to_token_ratio=0.060400 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229152 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157138720 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048288 weight_movement_bytes=37748736 total_nvlink_bytes=351797024 weight_to_token_ratio=0.120200 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229456 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157573200 ingress_bytes=176218112 link_bytes=176218112 token_bytes=314916944 weight_movement_bytes=18874368 total_nvlink_bytes=333791312 weight_to_token_ratio=0.059934 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688435244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229416 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663848 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098280 weight_movement_bytes=18874368 total_nvlink_bytes=331972648 weight_to_token_ratio=0.060283 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228488 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179016 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314129544 weight_movement_bytes=37748736 total_nvlink_bytes=351878280 weight_to_token_ratio=0.120169 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38406 dup_rows=21696 dup_rows_remote=19210 dedup_ratio=0.331055 router_tokens_per_expert=[244, 261, 251, 278, 252, 258, 203, 257, 248, 227, 272, 279, 238, 244, 262, 264, 261, 276, 257, 268, 280, 251, 260, 262, 244, 256, 232, 242, 255, 237, 262, 255, 239, 261, 254, 232, 282, 256, 240, 254, 231, 261, 258, 248, 255, 231, 293, 265, 235, 232, 248, 258, 257, 239, 249, 260, 289, 246, 245, 249, 249, 275, 288, 246, 224, 234, 260, 220, 254, 244, 259, 249, 251, 262, 255, 253, 275, 223, 251, 246, 260, 218, 260, 282, 248, 272, 267, 263, 268, 278, 254, 258, 262, 254, 266, 256, 258, 280, 233, 245, 279, 257, 256, 264, 260, 270, 250, 267, 259, 249, 267, 273, 248, 261, 248, 245, 263, 275, 269, 263, 276, 251, 269, 306, 262, 249, 240, 289] payload_rows_per_dst=[5426, 5518, 5448, 5444, 5434, 5520, 5476, 5574] dup_rows_per_dst=[2650, 2678, 2672, 2686, 2486, 2812, 2858, 2854] combine_rows_per_src=[5426, 5518, 5448, 5444, 5434, 5520, 5476, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38348 dup_rows=21892 dup_rows_remote=19204 dedup_ratio=0.334045 router_tokens_per_expert=[262, 269, 239, 240, 245, 251, 265, 227, 241, 245, 264, 256, 246, 265, 265, 254, 229, 233, 272, 265, 261, 255, 280, 259, 263, 242, 286, 236, 284, 224, 262, 248, 252, 256, 256, 272, 268, 229, 263, 251, 240, 248, 248, 267, 244, 244, 221, 233, 274, 237, 229, 248, 275, 238, 262, 264, 238, 244, 262, 248, 271, 244, 220, 261, 249, 244, 246, 248, 266, 243, 281, 262, 266, 250, 275, 242, 262, 273, 265, 279, 267, 238, 252, 262, 256, 251, 239, 279, 273, 260, 253, 275, 274, 265, 271, 289, 253, 258, 257, 255, 255, 249, 260, 248, 251, 254, 258, 252, 242, 291, 268, 258, 240, 279, 270, 294, 258, 275, 256, 238, 262, 266, 271, 264, 237, 256, 255, 243] payload_rows_per_dst=[5458, 5506, 5296, 5368, 5542, 5526, 5376, 5572] dup_rows_per_dst=[2610, 2692, 2688, 2662, 2760, 2882, 2842, 2756] combine_rows_per_src=[5458, 5506, 5296, 5368, 5542, 5526, 5376, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38326 dup_rows=21806 dup_rows_remote=19100 dedup_ratio=0.332733 router_tokens_per_expert=[250, 253, 275, 258, 245, 245, 247, 266, 241, 270, 255, 248, 247, 271, 239, 263, 260, 266, 258, 236, 245, 250, 288, 260, 252, 235, 263, 221, 262, 273, 275, 258, 216, 246, 259, 272, 255, 239, 246, 232, 221, 269, 270, 229, 260, 263, 260, 259, 257, 259, 268, 247, 299, 277, 261, 275, 253, 253, 270, 230, 246, 244, 263, 243, 268, 283, 230, 276, 258, 285, 265, 246, 237, 256, 275, 237, 285, 247, 262, 256, 246, 255, 232, 269, 256, 244, 267, 257, 271, 267, 240, 259, 258, 265, 255, 254, 238, 296, 265, 279, 248, 253, 257, 252, 256, 265, 243, 246, 271, 268, 243, 258, 243, 260, 233, 288, 254, 236, 270, 236, 274, 250, 270, 257, 238, 245, 255, 244] payload_rows_per_dst=[5438, 5242, 5378, 5534, 5748, 5478, 5508, 5404] dup_rows_per_dst=[2708, 2434, 2614, 2756, 3112, 2712, 2764, 2706] combine_rows_per_src=[5438, 5242, 5378, 5534, 5748, 5478, 5508, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=230464 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157541440 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314852416 weight_movement_bytes=18874368 total_nvlink_bytes=333726784 weight_to_token_ratio=0.059947 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=230208 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303616 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314377024 weight_movement_bytes=18874368 total_nvlink_bytes=333251392 weight_to_token_ratio=0.060037 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38384 dup_rows=21636 dup_rows_remote=18840 dedup_ratio=0.330139 router_tokens_per_expert=[256, 227, 250, 265, 258, 248, 274, 247, 253, 252, 291, 246, 231, 274, 253, 268, 290, 281, 243, 264, 247, 265, 254, 263, 234, 262, 269, 252, 270, 238, 231, 253, 239, 254, 265, 272, 258, 263, 251, 222, 245, 274, 243, 246, 267, 238, 258, 274, 256, 226, 256, 264, 255, 276, 262, 270, 257, 259, 256, 245, 254, 269, 236, 277, 263, 253, 249, 257, 251, 247, 270, 261, 239, 248, 252, 252, 285, 233, 245, 237, 246, 243, 267, 272, 261, 273, 248, 252, 259, 268, 242, 256, 245, 258, 268, 298, 291, 253, 225, 243, 225, 276, 233, 228, 261, 280, 262, 262, 266, 264, 283, 250, 265, 234, 272, 226, 258, 252, 255, 272, 258, 259, 272, 237, 271, 253, 253, 235] payload_rows_per_dst=[5490, 5508, 5472, 5500, 5438, 5516, 5484, 5492] dup_rows_per_dst=[2696, 2724, 2666, 2736, 2646, 2796, 2720, 2652] combine_rows_per_src=[5490, 5508, 5472, 5500, 5438, 5516, 5484, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229704 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157213000 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314196296 weight_movement_bytes=56623104 total_nvlink_bytes=370819400 weight_to_token_ratio=0.180216 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38184 dup_rows=21814 dup_rows_remote=19060 dedup_ratio=0.332855 router_tokens_per_expert=[270, 272, 273, 260, 260, 250, 240, 236, 266, 260, 285, 252, 259, 252, 248, 258, 240, 241, 265, 225, 272, 268, 226, 255, 257, 230, 240, 287, 246, 245, 258, 278, 262, 218, 271, 280, 234, 259, 242, 257, 254, 273, 278, 242, 275, 270, 241, 247, 241, 246, 266, 261, 278, 265, 305, 265, 286, 267, 260, 255, 242, 279, 263, 237, 231, 239, 262, 248, 271, 247, 276, 254, 252, 260, 250, 280, 247, 267, 278, 280, 251, 259, 256, 248, 241, 256, 262, 243, 262, 245, 229, 255, 265, 251, 262, 249, 265, 263, 242, 269, 264, 277, 268, 274, 247, 258, 278, 252, 238, 262, 247, 256, 249, 267, 227, 244, 224, 253, 252, 252, 244, 262, 261, 233, 242, 245, 256, 228] payload_rows_per_dst=[5538, 5408, 5556, 5610, 5514, 5426, 5370, 5300] dup_rows_per_dst=[2754, 2658, 3008, 2816, 2766, 2642, 2592, 2578] combine_rows_per_src=[5538, 5408, 5556, 5610, 5514, 5426, 5370, 5300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38290 dup_rows=21754 dup_rows_remote=18994 dedup_ratio=0.331940 router_tokens_per_expert=[267, 232, 250, 261, 269, 253, 228, 265, 282, 251, 256, 226, 297, 262, 237, 258, 260, 253, 255, 259, 271, 251, 247, 231, 238, 220, 271, 245, 244, 253, 235, 269, 218, 251, 251, 254, 246, 278, 268, 240, 269, 252, 262, 230, 226, 248, 255, 247, 244, 277, 249, 279, 276, 259, 241, 255, 256, 272, 262, 232, 255, 252, 271, 246, 263, 254, 280, 259, 274, 265, 270, 263, 245, 266, 230, 257, 251, 238, 244, 261, 222, 258, 226, 252, 268, 245, 255, 253, 252, 231, 246, 286, 270, 290, 256, 236, 262, 277, 254, 264, 257, 267, 238, 281, 277, 290, 252, 271, 217, 256, 228, 263, 265, 242, 252, 273, 254, 278, 276, 278, 271, 264, 255, 252, 275, 260, 255, 281] payload_rows_per_dst=[5556, 5420, 5374, 5492, 5512, 5384, 5526, 5518] dup_rows_per_dst=[2632, 2584, 2616, 2760, 2728, 2708, 2782, 2944] combine_rows_per_src=[5556, 5420, 5374, 5492, 5512, 5384, 5526, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38326 dup_rows=21718 dup_rows_remote=19040 dedup_ratio=0.331390 router_tokens_per_expert=[276, 247, 273, 267, 267, 265, 237, 251, 212, 246, 283, 235, 266, 279, 222, 278, 258, 236, 284, 270, 254, 272, 237, 246, 272, 257, 224, 229, 260, 263, 241, 282, 302, 261, 230, 266, 251, 266, 233, 241, 239, 251, 257, 285, 223, 250, 275, 249, 276, 243, 259, 271, 246, 276, 215, 227, 228, 230, 270, 258, 262, 255, 273, 256, 263, 236, 209, 258, 243, 242, 257, 221, 266, 226, 284, 262, 238, 256, 266, 274, 263, 256, 250, 261, 260, 258, 250, 267, 247, 245, 269, 262, 286, 249, 260, 285, 230, 241, 282, 236, 257, 254, 290, 265, 243, 241, 255, 262, 260, 260, 258, 239, 266, 261, 249, 294, 262, 259, 252, 257, 256, 306, 281, 236, 263, 273, 236, 262] payload_rows_per_dst=[5562, 5492, 5484, 5418, 5408, 5472, 5372, 5610] dup_rows_per_dst=[2646, 2678, 2762, 2672, 2594, 2776, 2774, 2816] combine_rows_per_src=[5562, 5492, 5484, 5418, 5408, 5472, 5372, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38296 dup_rows=21762 dup_rows_remote=19100 dedup_ratio=0.332062 router_tokens_per_expert=[263, 244, 260, 229, 273, 267, 252, 251, 283, 245, 261, 278, 269, 251, 270, 242, 259, 278, 257, 248, 245, 234, 256, 243, 259, 226, 232, 258, 248, 254, 283, 242, 261, 262, 235, 230, 277, 245, 254, 237, 249, 263, 236, 253, 266, 263, 249, 276, 262, 248, 269, 231, 245, 253, 231, 246, 255, 247, 288, 280, 256, 268, 261, 258, 246, 236, 250, 266, 289, 280, 252, 272, 258, 265, 248, 245, 283, 262, 252, 262, 269, 251, 253, 262, 258, 252, 235, 233, 255, 233, 262, 294, 254, 254, 265, 262, 267, 234, 240, 242, 277, 269, 239, 245, 264, 275, 288, 242, 267, 254, 274, 265, 268, 273, 262, 218, 270, 243, 279, 249, 264, 257, 257, 240, 238, 241, 242, 253] payload_rows_per_dst=[5440, 5458, 5470, 5464, 5524, 5430, 5478, 5510] dup_rows_per_dst=[2608, 2814, 2642, 2732, 2808, 2754, 2662, 2742] combine_rows_per_src=[5440, 5458, 5470, 5464, 5524, 5430, 5478, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=228896 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157449760 ingress_bytes=194969600 link_bytes=194969600 token_bytes=314670624 weight_movement_bytes=37748736 total_nvlink_bytes=352419360 weight_to_token_ratio=0.119963 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868102776e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228976 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156630640 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032304 weight_movement_bytes=37748736 total_nvlink_bytes=350781040 weight_to_token_ratio=0.120591 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229136 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157064976 ingress_bytes=194584576 link_bytes=194584576 token_bytes=313900816 weight_movement_bytes=37748736 total_nvlink_bytes=351649552 weight_to_token_ratio=0.120257 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864413666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229464 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212760 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314196056 weight_movement_bytes=37748736 total_nvlink_bytes=351944792 weight_to_token_ratio=0.120144 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229584 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157090000 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950416 weight_movement_bytes=37748736 total_nvlink_bytes=351699152 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38446 dup_rows=21692 dup_rows_remote=19006 dedup_ratio=0.330994 router_tokens_per_expert=[278, 275, 252, 229, 238, 261, 281, 230, 254, 259, 232, 259, 241, 259, 267, 237, 271, 254, 273, 284, 258, 249, 272, 270, 268, 260, 253, 258, 261, 251, 262, 294, 268, 237, 240, 259, 264, 249, 243, 246, 233, 253, 289, 267, 267, 259, 263, 300, 294, 243, 248, 255, 255, 268, 253, 250, 231, 255, 235, 250, 262, 282, 242, 258, 251, 243, 264, 266, 255, 251, 245, 242, 242, 253, 263, 219, 264, 255, 278, 251, 257, 263, 274, 244, 291, 221, 215, 262, 258, 239, 239, 278, 248, 262, 252, 264, 260, 259, 279, 239, 267, 250, 246, 251, 265, 239, 240, 266, 246, 262, 272, 239, 261, 259, 246, 254, 272, 269, 247, 248, 260, 263, 235, 233, 222, 272, 258, 272] payload_rows_per_dst=[5410, 5614, 5610, 5450, 5398, 5496, 5408, 5458] dup_rows_per_dst=[2694, 2862, 2664, 2712, 2686, 2638, 2752, 2684] combine_rows_per_src=[5410, 5614, 5610, 5450, 5398, 5496, 5408, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38118 dup_rows=21988 dup_rows_remote=19308 dedup_ratio=0.335510 router_tokens_per_expert=[276, 258, 274, 254, 243, 271, 225, 261, 236, 230, 274, 254, 283, 268, 251, 271, 255, 234, 259, 259, 283, 247, 222, 270, 264, 255, 276, 267, 258, 258, 229, 246, 261, 269, 261, 277, 292, 224, 221, 258, 240, 249, 242, 258, 260, 245, 245, 253, 284, 238, 248, 262, 262, 283, 261, 263, 263, 262, 257, 275, 242, 274, 256, 253, 240, 259, 260, 244, 265, 242, 279, 261, 253, 260, 254, 273, 276, 253, 250, 276, 237, 257, 252, 234, 263, 279, 244, 237, 247, 267, 263, 216, 251, 243, 247, 234, 276, 261, 242, 256, 272, 267, 259, 236, 242, 269, 270, 258, 278, 224, 289, 230, 251, 278, 263, 255, 282, 229, 241, 242, 265, 234, 283, 255, 238, 261, 245, 252] payload_rows_per_dst=[5496, 5398, 5430, 5486, 5444, 5364, 5516, 5414] dup_rows_per_dst=[2762, 2766, 2680, 2880, 2846, 2578, 2742, 2734] combine_rows_per_src=[5496, 5398, 5430, 5486, 5444, 5364, 5516, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38216 dup_rows=21864 dup_rows_remote=19260 dedup_ratio=0.333618 router_tokens_per_expert=[260, 240, 252, 247, 252, 258, 280, 284, 224, 263, 279, 260, 270, 238, 247, 233, 257, 226, 234, 246, 253, 244, 245, 279, 257, 256, 256, 278, 249, 248, 268, 267, 252, 259, 227, 262, 246, 269, 250, 234, 254, 270, 230, 289, 277, 255, 264, 298, 245, 292, 285, 246, 246, 252, 258, 254, 245, 243, 252, 249, 269, 245, 270, 263, 256, 266, 266, 267, 237, 248, 243, 261, 280, 280, 258, 252, 232, 246, 258, 246, 263, 257, 255, 278, 259, 229, 272, 253, 223, 269, 242, 254, 263, 232, 266, 224, 273, 294, 258, 249, 241, 282, 239, 265, 244, 274, 270, 242, 272, 251, 273, 276, 252, 250, 242, 259, 261, 256, 297, 253, 235, 233, 253, 262, 261, 244, 238, 234] payload_rows_per_dst=[5486, 5458, 5428, 5218, 5452, 5638, 5536, 5456] dup_rows_per_dst=[2688, 2668, 2844, 2506, 2740, 2944, 2870, 2604] combine_rows_per_src=[5486, 5458, 5428, 5218, 5452, 5638, 5536, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229704 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156361032 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312492360 weight_movement_bytes=18874368 total_nvlink_bytes=331366728 weight_to_token_ratio=0.060399 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229808 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157704624 ingress_bytes=214097920 link_bytes=214097920 token_bytes=315179440 weight_movement_bytes=56623104 total_nvlink_bytes=371802544 weight_to_token_ratio=0.179654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051380927e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229904 combine_payload_bytes=156532736 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156762640 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313295376 weight_movement_bytes=37748736 total_nvlink_bytes=351044112 weight_to_token_ratio=0.120489 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38316 dup_rows=21762 dup_rows_remote=19090 dedup_ratio=0.332062 router_tokens_per_expert=[258, 271, 241, 246, 269, 248, 274, 265, 252, 244, 247, 266, 260, 260, 231, 242, 270, 255, 251, 252, 232, 245, 227, 263, 248, 228, 263, 269, 250, 258, 260, 258, 226, 277, 260, 250, 251, 238, 286, 277, 268, 269, 263, 258, 251, 216, 258, 261, 261, 272, 262, 270, 275, 275, 295, 272, 260, 260, 239, 261, 246, 261, 287, 276, 240, 255, 250, 279, 275, 276, 205, 281, 285, 246, 250, 242, 266, 249, 268, 272, 239, 249, 271, 248, 239, 290, 255, 252, 239, 238, 251, 275, 250, 241, 268, 260, 243, 218, 252, 247, 267, 264, 262, 246, 252, 261, 238, 245, 261, 245, 254, 253, 266, 251, 269, 219, 261, 248, 211, 244, 269, 251, 249, 278, 262, 262, 263, 269] payload_rows_per_dst=[5436, 5366, 5562, 5578, 5544, 5458, 5430, 5400] dup_rows_per_dst=[2712, 2692, 2656, 2966, 2734, 2672, 2586, 2744] combine_rows_per_src=[5436, 5366, 5562, 5578, 5544, 5458, 5430, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38338 dup_rows=21684 dup_rows_remote=18892 dedup_ratio=0.330872 router_tokens_per_expert=[259, 265, 269, 252, 257, 261, 229, 233, 274, 266, 251, 269, 269, 256, 258, 263, 248, 227, 278, 279, 235, 249, 248, 250, 255, 261, 225, 234, 226, 243, 279, 249, 256, 261, 270, 270, 252, 259, 273, 260, 248, 237, 253, 303, 250, 248, 255, 232, 252, 273, 272, 252, 262, 262, 267, 258, 244, 277, 253, 278, 252, 239, 257, 255, 283, 252, 260, 273, 269, 239, 263, 250, 250, 250, 265, 240, 253, 231, 251, 232, 225, 233, 276, 253, 273, 249, 254, 283, 281, 279, 270, 238, 278, 248, 276, 231, 265, 289, 266, 274, 228, 249, 265, 269, 262, 214, 236, 268, 242, 259, 269, 218, 251, 261, 248, 258, 216, 247, 271, 260, 272, 248, 244, 273, 273, 259, 262, 247] payload_rows_per_dst=[5534, 5418, 5540, 5514, 5434, 5458, 5468, 5486] dup_rows_per_dst=[2728, 2554, 2714, 2792, 2688, 2836, 2678, 2694] combine_rows_per_src=[5534, 5418, 5540, 5514, 5434, 5458, 5468, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38232 dup_rows=21980 dup_rows_remote=19358 dedup_ratio=0.335388 router_tokens_per_expert=[256, 266, 280, 260, 267, 212, 275, 277, 263, 277, 267, 242, 235, 257, 251, 253, 246, 230, 266, 253, 238, 243, 257, 235, 244, 250, 243, 291, 252, 245, 257, 223, 249, 261, 267, 271, 275, 250, 254, 233, 271, 241, 269, 279, 251, 259, 237, 253, 267, 237, 243, 237, 279, 279, 236, 269, 280, 256, 298, 259, 261, 257, 246, 257, 262, 297, 264, 234, 236, 277, 245, 268, 241, 254, 256, 261, 253, 274, 245, 280, 283, 263, 267, 272, 265, 255, 264, 230, 250, 259, 227, 230, 243, 239, 244, 267, 267, 247, 244, 247, 274, 262, 231, 244, 280, 274, 242, 263, 239, 237, 278, 248, 241, 277, 249, 254, 238, 245, 272, 259, 257, 246, 264, 264, 280, 260, 237, 251] payload_rows_per_dst=[5420, 5324, 5454, 5442, 5548, 5424, 5490, 5454] dup_rows_per_dst=[2856, 2622, 2786, 2880, 2746, 2692, 2664, 2734] combine_rows_per_src=[5420, 5324, 5454, 5442, 5548, 5424, 5490, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38330 dup_rows=21646 dup_rows_remote=18898 dedup_ratio=0.330292 router_tokens_per_expert=[276, 253, 263, 227, 262, 253, 242, 249, 258, 276, 252, 244, 256, 249, 262, 256, 267, 231, 230, 264, 260, 238, 245, 258, 245, 262, 261, 246, 238, 252, 216, 240, 276, 248, 222, 244, 284, 237, 264, 263, 251, 265, 266, 260, 251, 256, 251, 275, 263, 265, 259, 255, 265, 259, 280, 264, 228, 264, 273, 249, 255, 238, 235, 255, 257, 254, 242, 243, 271, 275, 263, 270, 245, 263, 251, 250, 267, 272, 258, 257, 265, 256, 256, 241, 239, 230, 267, 252, 258, 267, 267, 273, 257, 266, 248, 264, 230, 273, 291, 276, 278, 240, 247, 253, 246, 266, 245, 260, 240, 254, 266, 292, 265, 251, 250, 271, 273, 253, 263, 251, 249, 248, 261, 246, 271, 242, 267, 255] payload_rows_per_dst=[5560, 5360, 5590, 5546, 5346, 5408, 5572, 5508] dup_rows_per_dst=[2748, 2546, 2902, 2882, 2512, 2590, 2742, 2724] combine_rows_per_src=[5560, 5360, 5590, 5546, 5346, 5408, 5572, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38406 dup_rows=21698 dup_rows_remote=19036 dedup_ratio=0.331085 router_tokens_per_expert=[241, 263, 294, 267, 226, 257, 279, 252, 252, 258, 258, 229, 276, 266, 262, 263, 239, 290, 273, 262, 259, 252, 245, 246, 227, 262, 237, 240, 282, 263, 268, 238, 270, 268, 269, 263, 244, 236, 257, 265, 252, 247, 262, 240, 258, 253, 257, 267, 251, 239, 251, 263, 281, 280, 228, 250, 240, 254, 244, 260, 256, 266, 287, 272, 255, 245, 240, 228, 258, 259, 253, 270, 239, 253, 256, 238, 260, 274, 222, 256, 257, 269, 287, 294, 254, 242, 247, 247, 267, 244, 261, 227, 235, 290, 256, 245, 227, 247, 228, 240, 256, 277, 252, 278, 251, 280, 226, 270, 235, 252, 222, 244, 256, 262, 269, 242, 269, 266, 285, 271, 279, 268, 267, 255, 260, 244, 267, 239] payload_rows_per_dst=[5482, 5474, 5472, 5542, 5442, 5428, 5432, 5566] dup_rows_per_dst=[2804, 2692, 2744, 2702, 2570, 2692, 2662, 2832] combine_rows_per_src=[5482, 5474, 5472, 5542, 5442, 5428, 5432, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229624 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171960 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114296 weight_movement_bytes=18874368 total_nvlink_bytes=332988664 weight_to_token_ratio=0.060088 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=230360 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156828632 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426904 weight_movement_bytes=18874368 total_nvlink_bytes=332301272 weight_to_token_ratio=0.060219 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=228920 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261368 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314293816 weight_movement_bytes=18874368 total_nvlink_bytes=333168184 weight_to_token_ratio=0.060053 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=228912 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157228592 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228272 weight_movement_bytes=37748736 total_nvlink_bytes=351977008 weight_to_token_ratio=0.120132 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38390 dup_rows=21632 dup_rows_remote=18884 dedup_ratio=0.330078 router_tokens_per_expert=[279, 260, 266, 255, 264, 259, 257, 278, 231, 246, 269, 285, 256, 277, 256, 273, 279, 260, 242, 243, 252, 271, 248, 264, 254, 248, 253, 258, 269, 253, 245, 253, 257, 259, 245, 277, 245, 247, 262, 254, 268, 241, 282, 251, 256, 241, 276, 270, 268, 249, 253, 276, 288, 270, 250, 253, 245, 261, 237, 277, 237, 229, 243, 256, 252, 252, 261, 240, 226, 247, 264, 214, 239, 275, 256, 259, 260, 257, 253, 270, 289, 228, 257, 232, 250, 250, 238, 259, 239, 249, 255, 277, 271, 264, 274, 270, 244, 257, 250, 273, 224, 218, 265, 268, 241, 251, 284, 242, 234, 248, 245, 249, 251, 254, 260, 261, 266, 282, 248, 271, 262, 254, 226, 293, 245, 232, 261, 256] payload_rows_per_dst=[5592, 5470, 5514, 5452, 5416, 5506, 5408, 5546] dup_rows_per_dst=[2830, 2714, 2748, 2732, 2634, 2698, 2578, 2698] combine_rows_per_src=[5592, 5470, 5514, 5452, 5416, 5506, 5408, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43484 payload_rows_remote=37990 dup_rows=22052 dup_rows_remote=19330 dedup_ratio=0.336487 router_tokens_per_expert=[262, 256, 255, 257, 259, 254, 268, 257, 263, 268, 259, 281, 276, 252, 237, 243, 261, 240, 289, 265, 249, 250, 251, 254, 276, 250, 256, 227, 271, 245, 229, 276, 249, 286, 244, 253, 239, 265, 246, 251, 246, 246, 233, 241, 259, 264, 247, 265, 246, 266, 242, 221, 271, 258, 271, 245, 240, 254, 262, 268, 280, 253, 286, 259, 226, 253, 249, 254, 281, 264, 242, 278, 246, 233, 275, 249, 270, 258, 270, 260, 243, 246, 286, 276, 283, 249, 274, 243, 246, 258, 248, 211, 277, 249, 245, 281, 260, 261, 251, 259, 258, 233, 259, 263, 261, 272, 244, 269, 244, 264, 241, 243, 239, 247, 256, 271, 255, 257, 248, 238, 232, 248, 278, 255, 257, 270, 260, 260] payload_rows_per_dst=[5456, 5470, 5378, 5366, 5494, 5492, 5436, 5392] dup_rows_per_dst=[2838, 2708, 2690, 2878, 2722, 2738, 2728, 2750] combine_rows_per_src=[5456, 5470, 5378, 5366, 5494, 5492, 5436, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229768 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157540744 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314851720 weight_movement_bytes=18874368 total_nvlink_bytes=333726088 weight_to_token_ratio=0.059947 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38348 dup_rows=21662 dup_rows_remote=18930 dedup_ratio=0.330536 router_tokens_per_expert=[254, 248, 261, 257, 256, 270, 293, 245, 252, 252, 262, 258, 256, 267, 249, 268, 239, 253, 271, 259, 270, 265, 264, 277, 265, 274, 264, 256, 250, 249, 240, 262, 278, 255, 256, 237, 253, 260, 268, 242, 249, 258, 247, 241, 273, 263, 263, 241, 239, 238, 218, 241, 244, 238, 247, 255, 276, 248, 234, 279, 271, 258, 245, 254, 283, 266, 268, 276, 254, 283, 257, 279, 255, 248, 280, 254, 273, 240, 230, 285, 266, 268, 250, 275, 241, 261, 260, 249, 259, 245, 211, 242, 273, 232, 239, 258, 239, 241, 288, 277, 283, 236, 240, 238, 240, 258, 250, 256, 254, 247, 260, 274, 248, 262, 240, 252, 251, 228, 259, 209, 262, 249, 269, 243, 271, 287, 284, 238] payload_rows_per_dst=[5476, 5564, 5458, 5438, 5504, 5430, 5478, 5526] dup_rows_per_dst=[2680, 2752, 2842, 2532, 2804, 2636, 2684, 2732] combine_rows_per_src=[5476, 5564, 5458, 5438, 5504, 5430, 5478, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38288 dup_rows=21790 dup_rows_remote=19122 dedup_ratio=0.332489 router_tokens_per_expert=[266, 251, 264, 253, 259, 259, 248, 279, 254, 245, 245, 269, 255, 259, 240, 236, 246, 249, 238, 258, 243, 252, 236, 287, 280, 257, 254, 241, 233, 269, 246, 251, 243, 263, 280, 243, 263, 273, 243, 248, 259, 252, 271, 251, 269, 252, 276, 229, 259, 263, 262, 238, 255, 255, 253, 247, 255, 275, 258, 244, 234, 271, 253, 277, 281, 250, 254, 256, 249, 233, 278, 251, 248, 242, 271, 234, 267, 263, 288, 276, 237, 286, 260, 241, 245, 251, 264, 289, 236, 246, 255, 269, 244, 241, 272, 227, 244, 262, 267, 262, 243, 239, 258, 272, 259, 243, 253, 221, 263, 254, 281, 288, 278, 227, 262, 263, 246, 272, 257, 258, 258, 255, 263, 275, 238, 272, 260, 235] payload_rows_per_dst=[5496, 5438, 5508, 5470, 5450, 5458, 5466, 5460] dup_rows_per_dst=[2668, 2642, 2722, 2728, 2832, 2668, 2752, 2778] combine_rows_per_src=[5496, 5438, 5508, 5470, 5450, 5458, 5466, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38242 dup_rows=21782 dup_rows_remote=19036 dedup_ratio=0.332367 router_tokens_per_expert=[279, 246, 261, 254, 247, 266, 240, 255, 227, 251, 259, 226, 260, 255, 253, 236, 262, 256, 236, 285, 274, 250, 246, 255, 268, 255, 296, 262, 254, 250, 250, 266, 234, 253, 236, 270, 242, 278, 252, 247, 269, 256, 281, 254, 258, 267, 227, 248, 269, 266, 256, 212, 264, 262, 273, 231, 242, 241, 246, 279, 259, 264, 265, 300, 240, 241, 251, 246, 245, 239, 264, 251, 247, 253, 246, 236, 266, 271, 235, 256, 250, 241, 245, 287, 251, 245, 253, 282, 255, 252, 246, 267, 260, 288, 259, 250, 272, 235, 264, 270, 234, 247, 273, 254, 273, 255, 252, 266, 244, 244, 250, 249, 268, 252, 288, 275, 269, 242, 256, 260, 271, 238, 279, 256, 263, 257, 254, 259] payload_rows_per_dst=[5362, 5520, 5376, 5512, 5412, 5556, 5464, 5552] dup_rows_per_dst=[2668, 2810, 2768, 2746, 2562, 2706, 2700, 2822] combine_rows_per_src=[5362, 5520, 5376, 5512, 5412, 5556, 5464, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38294 dup_rows=21772 dup_rows_remote=19072 dedup_ratio=0.332214 router_tokens_per_expert=[236, 284, 241, 269, 287, 220, 261, 250, 255, 257, 271, 245, 276, 268, 279, 215, 271, 249, 251, 254, 274, 253, 275, 247, 249, 256, 240, 278, 254, 216, 252, 266, 248, 267, 274, 268, 271, 267, 264, 254, 264, 247, 236, 223, 237, 267, 242, 262, 266, 243, 244, 241, 255, 259, 292, 253, 255, 279, 270, 289, 243, 255, 252, 250, 255, 235, 258, 254, 254, 246, 267, 239, 266, 266, 277, 257, 264, 267, 243, 256, 261, 259, 267, 222, 276, 232, 273, 244, 257, 256, 254, 283, 242, 263, 243, 248, 269, 233, 248, 256, 264, 241, 275, 239, 243, 246, 265, 236, 289, 248, 247, 230, 270, 245, 259, 251, 250, 272, 281, 253, 279, 246, 250, 254, 258, 252, 245, 254] payload_rows_per_dst=[5394, 5470, 5430, 5480, 5564, 5486, 5452, 5488] dup_rows_per_dst=[2834, 2700, 2752, 2812, 2644, 2674, 2606, 2750] combine_rows_per_src=[5394, 5470, 5430, 5480, 5564, 5486, 5452, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38134 dup_rows=21878 dup_rows_remote=19112 dedup_ratio=0.333832 router_tokens_per_expert=[241, 260, 256, 243, 253, 246, 246, 290, 233, 288, 282, 231, 269, 267, 245, 268, 243, 239, 241, 268, 272, 292, 256, 267, 265, 256, 275, 248, 243, 278, 285, 234, 238, 247, 256, 253, 270, 231, 219, 256, 251, 268, 283, 249, 238, 245, 273, 274, 281, 248, 239, 250, 237, 261, 240, 239, 276, 271, 250, 274, 271, 248, 277, 270, 271, 235, 272, 240, 242, 253, 272, 258, 248, 256, 281, 258, 245, 251, 254, 258, 256, 243, 266, 233, 233, 249, 253, 287, 259, 254, 246, 251, 263, 256, 267, 272, 267, 215, 278, 259, 241, 241, 256, 246, 245, 250, 244, 267, 246, 259, 250, 268, 231, 245, 251, 238, 260, 249, 259, 261, 259, 268, 273, 247, 248, 269, 275, 258] payload_rows_per_dst=[5524, 5578, 5534, 5308, 5454, 5440, 5446, 5374] dup_rows_per_dst=[2766, 2954, 2884, 2482, 2750, 2736, 2618, 2688] combine_rows_per_src=[5524, 5578, 5534, 5308, 5454, 5440, 5446, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=229096 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157474536 ingress_bytes=176119808 link_bytes=176119808 token_bytes=314719976 weight_movement_bytes=18874368 total_nvlink_bytes=333594344 weight_to_token_ratio=0.059972 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687493344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229112 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157302520 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314375928 weight_movement_bytes=37748736 total_nvlink_bytes=352124664 weight_to_token_ratio=0.120075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155607040 dispatch_meta_bytes=229280 combine_payload_bytes=155607040 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=155836320 ingress_bytes=193355776 link_bytes=193355776 token_bytes=311443360 weight_movement_bytes=37748736 total_nvlink_bytes=349192096 weight_to_token_ratio=0.121206 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852639909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229640 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057288 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884936 weight_movement_bytes=37748736 total_nvlink_bytes=351633672 weight_to_token_ratio=0.120263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229112 combine_payload_bytes=156639232 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156868344 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507576 weight_movement_bytes=18874368 total_nvlink_bytes=332381944 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229464 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081688 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933912 weight_movement_bytes=18874368 total_nvlink_bytes=332808280 weight_to_token_ratio=0.060122 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=228984 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156425848 ingress_bytes=193945600 link_bytes=193945600 token_bytes=312622712 weight_movement_bytes=37748736 total_nvlink_bytes=350371448 weight_to_token_ratio=0.120749 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858291312e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38284 dup_rows=21802 dup_rows_remote=19132 dedup_ratio=0.332672 router_tokens_per_expert=[256, 262, 254, 241, 239, 249, 257, 232, 265, 292, 228, 282, 226, 268, 235, 258, 242, 259, 232, 271, 266, 255, 280, 278, 257, 261, 253, 272, 262, 275, 256, 248, 256, 217, 245, 252, 254, 231, 237, 310, 271, 260, 234, 235, 247, 263, 243, 244, 280, 284, 250, 235, 265, 247, 255, 262, 270, 264, 277, 271, 253, 309, 236, 246, 237, 235, 263, 249, 278, 243, 279, 256, 275, 271, 260, 231, 235, 255, 264, 266, 257, 262, 263, 269, 235, 246, 268, 278, 243, 248, 272, 265, 267, 240, 250, 270, 299, 269, 238, 245, 252, 241, 258, 245, 241, 234, 258, 251, 234, 267, 264, 264, 248, 265, 248, 249, 262, 251, 254, 256, 262, 242, 249, 233, 260, 254, 265, 266] payload_rows_per_dst=[5430, 5516, 5368, 5580, 5480, 5480, 5450, 5430] dup_rows_per_dst=[2658, 2818, 2630, 2828, 2714, 2786, 2670, 2698] combine_rows_per_src=[5430, 5516, 5368, 5580, 5480, 5480, 5450, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38290 dup_rows=21782 dup_rows_remote=19046 dedup_ratio=0.332367 router_tokens_per_expert=[247, 250, 251, 270, 278, 253, 258, 247, 248, 280, 288, 252, 210, 263, 246, 251, 264, 251, 244, 221, 257, 255, 223, 270, 241, 251, 248, 262, 268, 275, 266, 242, 242, 250, 259, 264, 280, 270, 263, 256, 251, 257, 252, 242, 252, 265, 260, 237, 251, 222, 251, 246, 259, 218, 256, 250, 277, 242, 279, 248, 249, 265, 247, 250, 257, 247, 286, 252, 272, 249, 258, 285, 278, 262, 243, 264, 277, 255, 283, 265, 244, 261, 274, 253, 243, 276, 267, 258, 265, 270, 243, 262, 250, 244, 249, 257, 240, 238, 256, 237, 260, 238, 276, 309, 244, 262, 248, 253, 288, 263, 253, 255, 257, 239, 257, 242, 248, 233, 258, 248, 262, 284, 259, 266, 288, 241, 235, 242] payload_rows_per_dst=[5458, 5452, 5464, 5378, 5590, 5484, 5510, 5418] dup_rows_per_dst=[2726, 2624, 2736, 2642, 2876, 2748, 2730, 2700] combine_rows_per_src=[5458, 5452, 5464, 5378, 5590, 5484, 5510, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38056 dup_rows=21842 dup_rows_remote=18792 dedup_ratio=0.333282 router_tokens_per_expert=[233, 251, 238, 264, 222, 252, 271, 243, 271, 256, 264, 245, 257, 265, 227, 280, 241, 256, 261, 239, 255, 252, 264, 253, 259, 284, 238, 260, 261, 254, 282, 246, 251, 220, 254, 272, 235, 249, 241, 257, 243, 282, 236, 282, 244, 253, 265, 249, 248, 260, 252, 258, 246, 280, 238, 260, 282, 275, 259, 257, 259, 269, 226, 267, 251, 267, 284, 266, 235, 272, 253, 242, 272, 227, 261, 244, 274, 236, 264, 269, 255, 223, 262, 240, 263, 251, 250, 243, 258, 262, 233, 266, 251, 255, 255, 252, 260, 258, 269, 236, 243, 260, 242, 269, 274, 272, 254, 264, 241, 280, 261, 220, 257, 281, 247, 252, 307, 266, 252, 270, 245, 274, 270, 268, 257, 288, 254, 228] payload_rows_per_dst=[5400, 5492, 5414, 5458, 5418, 5472, 5402, 5638] dup_rows_per_dst=[2678, 2718, 2652, 2800, 2728, 2602, 2614, 3050] combine_rows_per_src=[5400, 5492, 5414, 5458, 5418, 5472, 5402, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229664 combine_payload_bytes=156811264 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157040928 ingress_bytes=213434368 link_bytes=213434368 token_bytes=313852192 weight_movement_bytes=56623104 total_nvlink_bytes=370475296 weight_to_token_ratio=0.180413 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045023098e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38132 dup_rows=21900 dup_rows_remote=19054 dedup_ratio=0.334167 router_tokens_per_expert=[257, 257, 273, 276, 254, 269, 275, 230, 248, 253, 279, 277, 254, 256, 246, 241, 258, 270, 250, 266, 263, 227, 264, 292, 263, 261, 268, 275, 268, 257, 268, 242, 262, 257, 240, 252, 266, 231, 271, 269, 219, 240, 251, 225, 227, 250, 264, 252, 298, 255, 260, 243, 281, 280, 238, 281, 263, 275, 261, 260, 258, 247, 230, 245, 268, 253, 239, 252, 247, 249, 242, 245, 238, 255, 245, 257, 261, 273, 282, 278, 244, 254, 291, 271, 275, 250, 218, 252, 276, 249, 237, 262, 260, 246, 261, 273, 262, 266, 265, 268, 234, 267, 218, 255, 241, 234, 249, 239, 242, 259, 288, 242, 256, 255, 256, 285, 232, 248, 230, 244, 249, 236, 269, 265, 286, 243, 235, 259] payload_rows_per_dst=[5494, 5506, 5448, 5504, 5446, 5464, 5420, 5354] dup_rows_per_dst=[2796, 2878, 2504, 2846, 2722, 2774, 2638, 2742] combine_rows_per_src=[5494, 5506, 5448, 5504, 5446, 5464, 5420, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38098 dup_rows=21966 dup_rows_remote=19124 dedup_ratio=0.335175 router_tokens_per_expert=[265, 237, 296, 271, 275, 231, 224, 251, 242, 251, 248, 243, 225, 258, 263, 283, 271, 248, 243, 262, 242, 256, 264, 242, 242, 264, 242, 264, 263, 246, 250, 280, 268, 257, 243, 256, 236, 281, 242, 259, 269, 244, 271, 271, 225, 253, 267, 253, 246, 275, 255, 270, 231, 249, 256, 264, 272, 260, 275, 246, 269, 225, 248, 287, 246, 282, 238, 245, 267, 272, 277, 231, 263, 263, 241, 274, 262, 254, 278, 239, 273, 262, 270, 246, 242, 252, 296, 247, 254, 276, 252, 263, 262, 250, 243, 269, 243, 272, 264, 252, 283, 249, 256, 243, 249, 295, 241, 245, 241, 283, 266, 234, 270, 243, 256, 252, 232, 253, 266, 261, 242, 254, 245, 249, 238, 252, 255, 230] payload_rows_per_dst=[5372, 5460, 5494, 5450, 5424, 5472, 5476, 5422] dup_rows_per_dst=[2754, 2698, 2696, 2806, 2840, 2842, 2756, 2574] combine_rows_per_src=[5372, 5460, 5494, 5450, 5424, 5472, 5476, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38310 dup_rows=21868 dup_rows_remote=19296 dedup_ratio=0.333679 router_tokens_per_expert=[230, 259, 246, 255, 253, 257, 256, 254, 237, 236, 251, 259, 267, 276, 247, 257, 261, 244, 269, 252, 234, 246, 253, 256, 253, 253, 242, 213, 241, 240, 272, 236, 247, 256, 267, 260, 266, 262, 246, 274, 265, 249, 257, 265, 273, 242, 262, 254, 247, 255, 276, 248, 261, 258, 264, 268, 257, 260, 249, 236, 260, 261, 256, 256, 272, 256, 266, 254, 275, 248, 262, 276, 271, 246, 263, 256, 279, 277, 245, 260, 259, 256, 269, 255, 266, 245, 253, 259, 270, 251, 247, 249, 302, 249, 264, 254, 278, 233, 230, 258, 263, 267, 259, 246, 271, 252, 260, 266, 270, 290, 251, 258, 225, 240, 248, 249, 249, 239, 259, 244, 224, 266, 271, 254, 236, 272, 280, 244] payload_rows_per_dst=[5404, 5358, 5460, 5526, 5534, 5464, 5518, 5404] dup_rows_per_dst=[2676, 2572, 2830, 2698, 2878, 2832, 2786, 2596] combine_rows_per_src=[5404, 5358, 5460, 5526, 5534, 5464, 5518, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38238 dup_rows=21694 dup_rows_remote=18802 dedup_ratio=0.331024 router_tokens_per_expert=[250, 238, 264, 270, 256, 239, 259, 295, 267, 256, 276, 259, 259, 259, 257, 268, 248, 242, 261, 272, 252, 265, 269, 240, 277, 242, 254, 245, 235, 249, 290, 270, 266, 265, 271, 240, 240, 272, 245, 265, 248, 280, 263, 258, 233, 261, 254, 248, 234, 252, 256, 234, 218, 275, 271, 228, 253, 257, 267, 235, 260, 256, 268, 262, 262, 249, 268, 238, 254, 252, 252, 270, 243, 247, 262, 265, 267, 236, 250, 251, 262, 262, 240, 266, 258, 263, 253, 232, 252, 254, 255, 266, 270, 265, 218, 285, 251, 255, 255, 277, 249, 257, 259, 257, 276, 279, 265, 243, 233, 264, 264, 269, 237, 265, 275, 246, 247, 272, 253, 251, 271, 227, 261, 225, 239, 245, 268, 248] payload_rows_per_dst=[5604, 5578, 5714, 5482, 5456, 5266, 5454, 5288] dup_rows_per_dst=[2892, 2824, 3074, 2688, 2676, 2430, 2636, 2474] combine_rows_per_src=[5604, 5578, 5714, 5482, 5456, 5266, 5454, 5288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38210 dup_rows=21828 dup_rows_remote=19102 dedup_ratio=0.333069 router_tokens_per_expert=[247, 254, 265, 260, 260, 225, 240, 267, 239, 265, 264, 243, 257, 219, 266, 237, 279, 265, 218, 269, 297, 259, 224, 269, 240, 294, 216, 258, 278, 255, 253, 259, 229, 263, 235, 253, 235, 254, 272, 242, 271, 272, 269, 269, 249, 247, 253, 269, 258, 254, 274, 240, 268, 276, 247, 274, 255, 271, 283, 244, 261, 258, 273, 260, 265, 237, 236, 258, 279, 251, 249, 252, 247, 242, 268, 259, 254, 271, 276, 268, 245, 266, 250, 253, 263, 243, 252, 271, 236, 238, 250, 285, 254, 262, 258, 241, 265, 242, 258, 235, 234, 254, 292, 264, 286, 259, 243, 265, 249, 246, 289, 278, 247, 274, 223, 231, 244, 279, 235, 250, 243, 257, 257, 244, 261, 254, 249, 263] payload_rows_per_dst=[5462, 5498, 5428, 5544, 5498, 5402, 5520, 5356] dup_rows_per_dst=[2554, 2768, 2736, 2848, 2726, 2732, 2798, 2666] combine_rows_per_src=[5462, 5498, 5428, 5544, 5498, 5402, 5520, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229344 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065184 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313901024 weight_movement_bytes=18874368 total_nvlink_bytes=332775392 weight_to_token_ratio=0.060128 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=227392 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156104768 ingress_bytes=174751744 link_bytes=174751744 token_bytes=311982144 weight_movement_bytes=18874368 total_nvlink_bytes=330856512 weight_to_token_ratio=0.060498 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674385228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=228744 combine_payload_bytes=156188672 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156417416 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312606088 weight_movement_bytes=18874368 total_nvlink_bytes=331480456 weight_to_token_ratio=0.060377 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228888 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156278296 ingress_bytes=193798144 link_bytes=193798144 token_bytes=312327704 weight_movement_bytes=37748736 total_nvlink_bytes=350076440 weight_to_token_ratio=0.120863 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856878461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=230424 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157148184 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314065944 weight_movement_bytes=18874368 total_nvlink_bytes=332940312 weight_to_token_ratio=0.060097 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228160 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156851008 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313473856 weight_movement_bytes=18874368 total_nvlink_bytes=332348224 weight_to_token_ratio=0.060210 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229248 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156737408 ingress_bytes=232005632 link_bytes=232005632 token_bytes=313245568 weight_movement_bytes=75497472 total_nvlink_bytes=388743040 weight_to_token_ratio=0.241017 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222963812e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38034 dup_rows=21948 dup_rows_remote=19074 dedup_ratio=0.334900 router_tokens_per_expert=[262, 254, 233, 258, 221, 253, 269, 250, 259, 246, 260, 269, 278, 268, 252, 290, 271, 253, 216, 214, 278, 249, 267, 228, 266, 234, 231, 278, 289, 274, 260, 245, 261, 256, 233, 235, 249, 242, 258, 268, 269, 241, 227, 265, 229, 243, 261, 233, 250, 253, 244, 251, 240, 260, 271, 224, 253, 237, 259, 241, 285, 268, 257, 264, 264, 247, 263, 258, 279, 243, 258, 266, 262, 252, 245, 255, 249, 262, 267, 272, 261, 244, 255, 245, 254, 261, 268, 243, 255, 260, 238, 267, 252, 233, 243, 257, 249, 263, 294, 229, 253, 273, 247, 248, 266, 260, 310, 254, 268, 267, 290, 243, 274, 283, 277, 261, 282, 243, 276, 264, 248, 237, 273, 249, 285, 257, 248, 217] payload_rows_per_dst=[5470, 5426, 5350, 5262, 5448, 5514, 5554, 5564] dup_rows_per_dst=[2774, 2680, 2590, 2570, 2836, 2840, 2874, 2784] combine_rows_per_src=[5470, 5426, 5350, 5262, 5448, 5514, 5554, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38338 dup_rows=21830 dup_rows_remote=19118 dedup_ratio=0.333099 router_tokens_per_expert=[274, 251, 240, 282, 263, 258, 249, 269, 271, 266, 241, 262, 235, 237, 243, 257, 279, 252, 258, 257, 250, 238, 262, 263, 263, 253, 249, 264, 268, 266, 271, 241, 255, 252, 218, 264, 249, 255, 258, 234, 232, 243, 253, 267, 274, 254, 255, 277, 243, 267, 277, 232, 263, 256, 267, 266, 257, 276, 256, 262, 228, 230, 282, 233, 278, 250, 247, 231, 225, 268, 252, 268, 249, 267, 268, 282, 251, 242, 268, 229, 258, 257, 265, 278, 275, 246, 256, 243, 256, 254, 255, 273, 248, 275, 218, 261, 236, 247, 239, 268, 251, 270, 241, 212, 259, 256, 275, 241, 227, 238, 288, 261, 251, 267, 253, 266, 278, 245, 288, 270, 245, 263, 269, 247, 266, 260, 257, 274] payload_rows_per_dst=[5444, 5516, 5368, 5436, 5416, 5556, 5420, 5550] dup_rows_per_dst=[2752, 2752, 2712, 2754, 2734, 2680, 2598, 2848] combine_rows_per_src=[5444, 5516, 5368, 5436, 5416, 5556, 5420, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=228432 combine_payload_bytes=155787264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156015696 ingress_bytes=212410368 link_bytes=212410368 token_bytes=311802960 weight_movement_bytes=56623104 total_nvlink_bytes=368426064 weight_to_token_ratio=0.181599 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035211634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38172 dup_rows=21786 dup_rows_remote=18948 dedup_ratio=0.332428 router_tokens_per_expert=[262, 268, 272, 242, 262, 255, 245, 251, 259, 253, 292, 257, 274, 238, 257, 271, 235, 267, 252, 229, 242, 271, 264, 273, 263, 248, 270, 285, 244, 249, 227, 272, 249, 259, 262, 252, 266, 234, 262, 235, 265, 258, 261, 258, 242, 276, 245, 254, 261, 240, 270, 283, 240, 245, 244, 258, 248, 226, 284, 236, 274, 252, 264, 268, 241, 252, 239, 243, 264, 245, 251, 245, 290, 275, 261, 292, 292, 236, 249, 235, 243, 245, 266, 228, 249, 245, 252, 243, 270, 278, 253, 252, 240, 242, 266, 259, 251, 270, 234, 244, 253, 247, 255, 264, 247, 265, 240, 267, 257, 263, 245, 280, 268, 233, 278, 249, 275, 240, 257, 252, 274, 250, 249, 231, 283, 268, 261, 257] payload_rows_per_dst=[5376, 5310, 5646, 5492, 5480, 5414, 5454, 5578] dup_rows_per_dst=[2580, 2602, 3038, 2670, 2740, 2506, 2812, 2838] combine_rows_per_src=[5376, 5310, 5646, 5492, 5480, 5414, 5454, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43942 payload_rows_remote=38422 dup_rows=21594 dup_rows_remote=18896 dedup_ratio=0.329498 router_tokens_per_expert=[255, 255, 266, 285, 289, 276, 247, 255, 236, 275, 240, 255, 231, 269, 246, 267, 266, 258, 258, 265, 268, 268, 258, 236, 252, 248, 243, 251, 242, 232, 239, 258, 252, 219, 257, 253, 242, 279, 247, 277, 233, 228, 269, 238, 276, 261, 278, 272, 264, 248, 232, 269, 240, 253, 263, 264, 253, 249, 274, 266, 252, 256, 267, 259, 288, 239, 262, 275, 265, 251, 260, 256, 284, 246, 238, 243, 263, 242, 250, 248, 259, 253, 293, 247, 257, 237, 244, 235, 275, 260, 254, 288, 253, 280, 252, 242, 272, 256, 255, 269, 257, 232, 280, 272, 274, 239, 248, 234, 262, 243, 260, 232, 258, 268, 245, 249, 255, 256, 247, 262, 244, 229, 254, 256, 260, 267, 251, 264] payload_rows_per_dst=[5532, 5446, 5530, 5520, 5544, 5510, 5460, 5400] dup_rows_per_dst=[2762, 2638, 2632, 2698, 2676, 2748, 2710, 2730] combine_rows_per_src=[5532, 5446, 5530, 5520, 5544, 5510, 5460, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38316 dup_rows=21740 dup_rows_remote=19110 dedup_ratio=0.331726 router_tokens_per_expert=[241, 244, 269, 266, 265, 253, 243, 233, 262, 236, 260, 240, 262, 244, 246, 261, 275, 236, 251, 251, 230, 274, 265, 256, 271, 237, 268, 256, 238, 256, 244, 247, 263, 254, 272, 271, 259, 287, 282, 253, 248, 276, 258, 246, 248, 261, 278, 218, 256, 249, 243, 259, 253, 248, 270, 265, 274, 251, 261, 235, 250, 274, 257, 238, 269, 238, 252, 263, 261, 257, 237, 270, 279, 243, 243, 253, 244, 267, 260, 248, 296, 249, 241, 260, 238, 255, 230, 249, 251, 239, 255, 254, 258, 281, 248, 251, 265, 229, 231, 259, 250, 245, 272, 283, 294, 239, 271, 274, 259, 268, 248, 255, 269, 232, 264, 218, 263, 260, 275, 275, 255, 237, 262, 252, 288, 259, 278, 263] payload_rows_per_dst=[5420, 5422, 5552, 5474, 5456, 5480, 5534, 5458] dup_rows_per_dst=[2630, 2688, 2796, 2692, 2712, 2630, 2750, 2842] combine_rows_per_src=[5420, 5422, 5552, 5474, 5456, 5480, 5534, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38118 dup_rows=21924 dup_rows_remote=19030 dedup_ratio=0.334534 router_tokens_per_expert=[248, 269, 242, 260, 229, 235, 256, 240, 257, 251, 256, 261, 266, 259, 246, 246, 276, 278, 253, 261, 270, 248, 285, 253, 285, 247, 241, 246, 274, 268, 257, 252, 274, 256, 284, 261, 223, 261, 274, 242, 218, 240, 232, 219, 283, 238, 232, 231, 282, 243, 259, 251, 281, 295, 278, 246, 286, 250, 261, 233, 244, 253, 262, 248, 288, 236, 258, 237, 262, 221, 259, 225, 260, 272, 279, 260, 257, 245, 271, 243, 259, 267, 238, 218, 242, 272, 284, 249, 283, 250, 260, 254, 261, 236, 249, 244, 261, 252, 257, 279, 248, 260, 255, 260, 258, 258, 295, 257, 233, 259, 259, 257, 283, 253, 279, 253, 257, 248, 255, 254, 260, 253, 286, 248, 261, 242, 252, 242] payload_rows_per_dst=[5358, 5494, 5320, 5498, 5464, 5514, 5484, 5480] dup_rows_per_dst=[2684, 2894, 2616, 2846, 2708, 2618, 2812, 2746] combine_rows_per_src=[5358, 5494, 5320, 5498, 5464, 5514, 5484, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38374 dup_rows=21854 dup_rows_remote=19282 dedup_ratio=0.333466 router_tokens_per_expert=[264, 262, 292, 267, 282, 261, 229, 254, 244, 279, 238, 222, 266, 263, 289, 236, 286, 253, 263, 268, 284, 288, 258, 258, 243, 206, 248, 237, 274, 284, 266, 250, 248, 235, 288, 294, 260, 249, 252, 232, 253, 246, 243, 255, 265, 258, 248, 247, 252, 257, 253, 232, 253, 261, 240, 267, 273, 260, 242, 245, 227, 263, 251, 260, 268, 223, 264, 239, 257, 241, 245, 244, 234, 273, 236, 235, 244, 248, 242, 247, 261, 277, 260, 271, 251, 281, 270, 296, 277, 245, 232, 245, 278, 251, 241, 271, 255, 253, 234, 232, 256, 259, 266, 264, 245, 233, 266, 243, 273, 248, 258, 240, 264, 247, 282, 269, 259, 291, 263, 226, 273, 280, 266, 262, 241, 241, 253, 256] payload_rows_per_dst=[5502, 5564, 5518, 5422, 5308, 5512, 5370, 5486] dup_rows_per_dst=[2794, 2768, 2628, 2650, 2572, 2902, 2680, 2860] combine_rows_per_src=[5502, 5564, 5518, 5422, 5308, 5512, 5370, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38216 dup_rows=21902 dup_rows_remote=19176 dedup_ratio=0.334198 router_tokens_per_expert=[269, 261, 227, 246, 286, 252, 274, 245, 258, 257, 238, 241, 241, 250, 271, 234, 264, 259, 245, 241, 275, 242, 276, 273, 265, 247, 264, 264, 257, 273, 244, 245, 260, 266, 223, 222, 267, 260, 243, 241, 259, 275, 246, 270, 255, 257, 267, 278, 242, 276, 258, 262, 249, 264, 230, 269, 227, 264, 268, 239, 261, 279, 258, 245, 237, 257, 262, 247, 251, 247, 258, 255, 240, 235, 281, 242, 242, 240, 258, 277, 269, 256, 272, 273, 259, 220, 223, 255, 251, 256, 256, 249, 279, 263, 274, 238, 260, 253, 262, 270, 252, 238, 245, 250, 262, 256, 260, 274, 245, 264, 247, 263, 253, 287, 284, 278, 276, 267, 253, 245, 277, 257, 265, 274, 238, 254, 232, 241] payload_rows_per_dst=[5418, 5686, 5570, 5472, 5336, 5444, 5296, 5412] dup_rows_per_dst=[2726, 2970, 2922, 2862, 2538, 2742, 2510, 2632] combine_rows_per_src=[5418, 5686, 5570, 5472, 5336, 5444, 5296, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229824 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157262272 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294720 weight_movement_bytes=18874368 total_nvlink_bytes=333169088 weight_to_token_ratio=0.060053 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228480 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156580992 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312933504 weight_movement_bytes=37748736 total_nvlink_bytes=350682240 weight_to_token_ratio=0.120629 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229272 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157605784 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982296 weight_movement_bytes=18874368 total_nvlink_bytes=333856664 weight_to_token_ratio=0.059922 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229704 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157172040 ingress_bytes=213565440 link_bytes=213565440 token_bytes=314114376 weight_movement_bytes=56623104 total_nvlink_bytes=370737480 weight_to_token_ratio=0.180263 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046278965e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228592 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156359920 ingress_bytes=193880064 link_bytes=193880064 token_bytes=312491248 weight_movement_bytes=37748736 total_nvlink_bytes=350239984 weight_to_token_ratio=0.120799 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857663378e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=230624 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157410528 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314590432 weight_movement_bytes=18874368 total_nvlink_bytes=333464800 weight_to_token_ratio=0.059997 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38140 dup_rows=21922 dup_rows_remote=19278 dedup_ratio=0.334503 router_tokens_per_expert=[215, 255, 280, 258, 269, 214, 240, 245, 230, 250, 233, 249, 284, 275, 267, 263, 267, 290, 253, 268, 248, 273, 250, 263, 256, 240, 242, 273, 247, 274, 260, 246, 261, 263, 236, 244, 268, 248, 295, 260, 225, 230, 245, 261, 220, 252, 266, 245, 281, 252, 243, 221, 253, 238, 261, 246, 262, 264, 267, 237, 248, 265, 243, 258, 238, 265, 254, 255, 224, 272, 275, 271, 240, 270, 271, 261, 272, 252, 265, 238, 268, 259, 253, 261, 256, 265, 259, 248, 270, 282, 262, 275, 234, 249, 269, 270, 247, 242, 245, 254, 281, 278, 235, 230, 261, 260, 239, 237, 282, 276, 228, 264, 251, 264, 239, 251, 220, 270, 319, 271, 247, 274, 257, 242, 265, 260, 272, 269] payload_rows_per_dst=[5366, 5574, 5386, 5424, 5456, 5488, 5474, 5446] dup_rows_per_dst=[2688, 2726, 2652, 2654, 2790, 2872, 2644, 2896] combine_rows_per_src=[5366, 5574, 5386, 5424, 5456, 5488, 5474, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229568 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762304 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295040 weight_movement_bytes=18874368 total_nvlink_bytes=332169408 weight_to_token_ratio=0.060245 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229672 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156451112 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312672552 weight_movement_bytes=18874368 total_nvlink_bytes=331546920 weight_to_token_ratio=0.060365 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38338 dup_rows=21800 dup_rows_remote=19122 dedup_ratio=0.332642 router_tokens_per_expert=[280, 247, 242, 255, 261, 269, 263, 268, 232, 239, 256, 280, 249, 281, 270, 246, 243, 227, 262, 234, 256, 270, 248, 252, 256, 259, 250, 249, 260, 268, 266, 272, 248, 244, 249, 238, 261, 238, 279, 240, 263, 259, 250, 236, 281, 224, 258, 270, 253, 265, 247, 266, 244, 267, 260, 263, 252, 279, 264, 251, 260, 248, 241, 230, 241, 302, 246, 226, 246, 230, 255, 249, 262, 257, 280, 268, 251, 255, 248, 243, 265, 251, 291, 233, 258, 252, 231, 273, 285, 253, 240, 270, 272, 249, 275, 270, 232, 250, 246, 261, 266, 233, 286, 260, 245, 262, 226, 235, 255, 261, 253, 266, 261, 272, 247, 266, 259, 277, 247, 252, 242, 286, 265, 251, 242, 285, 252, 262] payload_rows_per_dst=[5428, 5442, 5398, 5502, 5402, 5492, 5478, 5594] dup_rows_per_dst=[2848, 2702, 2678, 2678, 2716, 2844, 2596, 2738] combine_rows_per_src=[5428, 5442, 5398, 5502, 5402, 5492, 5478, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38160 dup_rows=21888 dup_rows_remote=19182 dedup_ratio=0.333984 router_tokens_per_expert=[244, 267, 273, 281, 251, 237, 277, 264, 256, 269, 246, 257, 216, 254, 258, 247, 246, 261, 259, 225, 279, 243, 247, 279, 260, 261, 248, 255, 247, 276, 270, 245, 247, 240, 260, 266, 266, 250, 256, 268, 271, 261, 251, 247, 241, 250, 254, 272, 277, 263, 260, 249, 240, 257, 255, 267, 255, 268, 254, 260, 262, 239, 252, 248, 264, 258, 246, 266, 260, 256, 242, 269, 280, 246, 265, 245, 247, 246, 239, 236, 236, 243, 251, 268, 259, 265, 274, 263, 265, 269, 265, 264, 285, 248, 249, 237, 270, 253, 258, 267, 252, 251, 259, 262, 234, 268, 279, 264, 257, 235, 214, 255, 279, 261, 255, 253, 249, 246, 236, 250, 265, 258, 266, 262, 235, 239, 265, 261] payload_rows_per_dst=[5468, 5374, 5418, 5492, 5480, 5438, 5490, 5488] dup_rows_per_dst=[2688, 2828, 2622, 2720, 2650, 2810, 2864, 2706] combine_rows_per_src=[5468, 5374, 5418, 5492, 5480, 5438, 5490, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38404 dup_rows=21686 dup_rows_remote=18892 dedup_ratio=0.330902 router_tokens_per_expert=[245, 265, 243, 276, 259, 248, 233, 258, 256, 267, 265, 248, 249, 248, 253, 255, 265, 259, 234, 275, 247, 246, 251, 259, 260, 276, 243, 268, 282, 245, 259, 258, 257, 258, 264, 255, 281, 227, 261, 265, 230, 258, 263, 267, 238, 278, 254, 279, 257, 271, 238, 225, 259, 259, 276, 245, 255, 259, 247, 258, 282, 284, 255, 250, 259, 238, 265, 247, 252, 246, 268, 276, 276, 256, 247, 246, 234, 239, 280, 248, 237, 241, 257, 257, 253, 255, 260, 260, 262, 242, 268, 282, 270, 240, 237, 257, 242, 247, 257, 232, 260, 243, 260, 224, 241, 254, 253, 256, 262, 275, 264, 250, 268, 253, 251, 256, 271, 282, 262, 242, 228, 250, 246, 260, 269, 275, 256, 274] payload_rows_per_dst=[5506, 5500, 5538, 5446, 5514, 5416, 5394, 5536] dup_rows_per_dst=[2630, 2754, 2732, 2794, 2640, 2740, 2646, 2750] combine_rows_per_src=[5506, 5500, 5538, 5446, 5514, 5416, 5394, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38414 dup_rows=21608 dup_rows_remote=18784 dedup_ratio=0.329712 router_tokens_per_expert=[234, 247, 253, 256, 276, 266, 249, 247, 273, 275, 293, 227, 260, 262, 230, 232, 243, 226, 263, 269, 246, 240, 275, 259, 261, 249, 272, 253, 267, 260, 223, 265, 268, 259, 232, 248, 229, 271, 256, 265, 244, 242, 245, 270, 233, 281, 225, 245, 252, 266, 259, 247, 260, 282, 254, 250, 250, 269, 265, 275, 253, 256, 234, 256, 281, 229, 276, 274, 256, 258, 245, 275, 275, 248, 226, 250, 299, 261, 252, 255, 264, 263, 291, 229, 253, 266, 237, 267, 275, 260, 235, 287, 256, 259, 249, 278, 262, 231, 282, 252, 233, 247, 252, 239, 238, 270, 263, 270, 238, 255, 241, 271, 245, 278, 233, 252, 241, 247, 287, 260, 293, 266, 264, 244, 242, 257, 249, 245] payload_rows_per_dst=[5482, 5504, 5390, 5466, 5580, 5514, 5420, 5572] dup_rows_per_dst=[2678, 2638, 2636, 2790, 2740, 2824, 2668, 2634] combine_rows_per_src=[5482, 5504, 5390, 5466, 5580, 5514, 5420, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229840 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157262288 ingress_bytes=213655552 link_bytes=213655552 token_bytes=314294736 weight_movement_bytes=56623104 total_nvlink_bytes=370917840 weight_to_token_ratio=0.180159 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047142374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38094 dup_rows=21924 dup_rows_remote=19330 dedup_ratio=0.334534 router_tokens_per_expert=[239, 250, 257, 269, 246, 266, 246, 274, 265, 248, 272, 277, 270, 242, 262, 259, 239, 242, 253, 254, 249, 248, 232, 263, 261, 263, 251, 242, 281, 278, 260, 240, 266, 262, 263, 239, 251, 271, 260, 255, 271, 281, 265, 254, 261, 267, 249, 259, 266, 268, 269, 256, 238, 253, 246, 245, 267, 272, 243, 240, 267, 257, 262, 264, 244, 252, 245, 272, 241, 270, 242, 234, 246, 230, 280, 264, 262, 244, 259, 250, 227, 238, 276, 260, 250, 266, 238, 255, 287, 252, 239, 265, 261, 270, 240, 252, 270, 264, 215, 262, 239, 260, 266, 285, 273, 263, 235, 251, 267, 245, 248, 251, 270, 237, 276, 258, 240, 255, 276, 250, 241, 274, 247, 253, 239, 258, 257, 247] payload_rows_per_dst=[5490, 5518, 5462, 5486, 5412, 5422, 5412, 5410] dup_rows_per_dst=[2794, 2594, 2886, 2740, 2658, 2730, 2776, 2746] combine_rows_per_src=[5490, 5518, 5462, 5486, 5412, 5422, 5412, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38290 dup_rows=21804 dup_rows_remote=19032 dedup_ratio=0.332703 router_tokens_per_expert=[254, 256, 254, 237, 267, 248, 262, 257, 227, 240, 239, 259, 251, 252, 266, 275, 255, 261, 265, 247, 266, 260, 222, 248, 248, 254, 279, 269, 244, 234, 245, 237, 238, 276, 255, 266, 271, 262, 269, 269, 289, 245, 232, 249, 299, 260, 259, 246, 245, 268, 260, 235, 249, 242, 255, 283, 259, 282, 258, 247, 270, 225, 227, 235, 279, 244, 259, 275, 264, 231, 256, 269, 242, 248, 242, 254, 266, 265, 251, 262, 262, 259, 248, 265, 242, 255, 285, 240, 232, 227, 248, 273, 282, 278, 281, 233, 217, 256, 289, 249, 260, 257, 283, 235, 254, 273, 273, 250, 273, 277, 263, 251, 260, 265, 277, 246, 258, 252, 238, 278, 232, 256, 245, 258, 219, 278, 260, 266] payload_rows_per_dst=[5426, 5456, 5590, 5384, 5442, 5418, 5542, 5474] dup_rows_per_dst=[2662, 2612, 2780, 2696, 2772, 2802, 2778, 2702] combine_rows_per_src=[5426, 5456, 5590, 5384, 5442, 5418, 5542, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229368 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156532728 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312836088 weight_movement_bytes=18874368 total_nvlink_bytes=331710456 weight_to_token_ratio=0.060333 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38216 dup_rows=21922 dup_rows_remote=19364 dedup_ratio=0.334503 router_tokens_per_expert=[240, 254, 256, 249, 241, 266, 244, 258, 269, 241, 220, 253, 233, 281, 246, 227, 271, 253, 266, 260, 260, 255, 247, 249, 270, 272, 255, 262, 271, 257, 248, 236, 247, 249, 247, 268, 259, 268, 234, 263, 255, 277, 239, 275, 264, 241, 252, 263, 250, 285, 248, 282, 227, 229, 265, 247, 267, 260, 269, 247, 280, 231, 254, 234, 270, 282, 246, 239, 250, 271, 268, 261, 259, 285, 266, 238, 222, 237, 221, 273, 275, 252, 262, 273, 250, 251, 235, 251, 250, 249, 278, 267, 237, 251, 265, 259, 292, 266, 255, 267, 246, 257, 244, 233, 249, 250, 247, 287, 263, 258, 223, 250, 275, 277, 258, 284, 236, 274, 259, 238, 263, 265, 309, 240, 256, 247, 243, 278] payload_rows_per_dst=[5398, 5474, 5496, 5482, 5378, 5430, 5662, 5294] dup_rows_per_dst=[2558, 2862, 2706, 2782, 2726, 2666, 3022, 2600] combine_rows_per_src=[5398, 5474, 5496, 5482, 5378, 5430, 5662, 5294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=228792 combine_payload_bytes=157343744 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157572536 ingress_bytes=232841216 link_bytes=232841216 token_bytes=314916280 weight_movement_bytes=75497472 total_nvlink_bytes=390413752 weight_to_token_ratio=0.239738 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230969967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229184 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157531968 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314834752 weight_movement_bytes=18874368 total_nvlink_bytes=333709120 weight_to_token_ratio=0.059950 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38224 dup_rows=21812 dup_rows_remote=19108 dedup_ratio=0.332825 router_tokens_per_expert=[254, 286, 261, 242, 256, 258, 260, 248, 262, 277, 244, 283, 264, 286, 252, 266, 272, 245, 266, 271, 252, 240, 286, 258, 260, 252, 233, 257, 257, 240, 276, 257, 254, 263, 234, 264, 270, 243, 245, 272, 274, 271, 250, 242, 263, 243, 259, 253, 256, 242, 240, 276, 204, 226, 248, 270, 264, 268, 223, 239, 268, 275, 267, 222, 248, 239, 252, 249, 274, 218, 232, 248, 250, 256, 268, 283, 264, 251, 242, 277, 287, 266, 246, 256, 263, 268, 236, 229, 246, 239, 262, 272, 274, 287, 255, 230, 267, 262, 236, 265, 272, 264, 267, 252, 250, 267, 238, 259, 264, 249, 262, 228, 275, 270, 266, 285, 242, 254, 274, 270, 224, 259, 252, 269, 247, 241, 227, 235] payload_rows_per_dst=[5468, 5552, 5488, 5394, 5388, 5518, 5500, 5416] dup_rows_per_dst=[2930, 2692, 2712, 2582, 2714, 2714, 2704, 2764] combine_rows_per_src=[5468, 5552, 5488, 5394, 5388, 5518, 5500, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=229696 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156262720 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312295744 weight_movement_bytes=18874368 total_nvlink_bytes=331170112 weight_to_token_ratio=0.060437 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229288 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065128 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313900968 weight_movement_bytes=18874368 total_nvlink_bytes=332775336 weight_to_token_ratio=0.060128 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=230320 combine_payload_bytes=156532736 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156763056 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313295792 weight_movement_bytes=37748736 total_nvlink_bytes=351044528 weight_to_token_ratio=0.120489 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229328 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156794832 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360336 weight_movement_bytes=56623104 total_nvlink_bytes=369983440 weight_to_token_ratio=0.180696 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38276 dup_rows=21752 dup_rows_remote=18942 dedup_ratio=0.331909 router_tokens_per_expert=[264, 279, 284, 252, 247, 261, 266, 260, 242, 251, 258, 251, 231, 259, 237, 272, 262, 281, 245, 250, 255, 243, 243, 253, 277, 252, 230, 267, 240, 273, 286, 243, 269, 259, 258, 255, 262, 242, 273, 289, 276, 262, 271, 264, 244, 258, 233, 244, 273, 239, 258, 241, 225, 241, 275, 243, 258, 250, 255, 259, 257, 261, 231, 232, 254, 259, 253, 243, 274, 278, 256, 256, 252, 230, 270, 274, 250, 237, 237, 254, 264, 255, 252, 249, 258, 270, 270, 268, 243, 262, 283, 256, 244, 278, 249, 271, 249, 278, 247, 245, 267, 246, 250, 245, 253, 264, 261, 264, 241, 250, 247, 241, 267, 265, 231, 271, 254, 279, 235, 259, 235, 267, 269, 253, 254, 240, 240, 281] payload_rows_per_dst=[5528, 5468, 5508, 5378, 5464, 5530, 5428, 5480] dup_rows_per_dst=[2700, 2732, 2810, 2618, 2690, 2814, 2668, 2720] combine_rows_per_src=[5528, 5468, 5508, 5378, 5464, 5530, 5428, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38130 dup_rows=21864 dup_rows_remote=19036 dedup_ratio=0.333618 router_tokens_per_expert=[255, 253, 249, 252, 245, 283, 223, 275, 270, 234, 264, 249, 266, 255, 262, 274, 254, 257, 241, 266, 287, 227, 267, 260, 269, 251, 261, 269, 282, 249, 237, 264, 262, 268, 249, 250, 256, 259, 243, 266, 248, 261, 246, 245, 257, 245, 246, 234, 264, 227, 253, 268, 270, 271, 276, 261, 247, 229, 252, 232, 246, 249, 248, 261, 280, 234, 258, 233, 242, 270, 242, 248, 241, 242, 232, 268, 250, 251, 251, 257, 251, 262, 273, 254, 265, 260, 251, 260, 269, 260, 255, 253, 230, 253, 271, 270, 257, 280, 254, 280, 280, 266, 251, 252, 257, 268, 260, 262, 275, 246, 252, 272, 228, 258, 274, 246, 248, 249, 284, 248, 248, 261, 250, 246, 283, 226, 261, 271] payload_rows_per_dst=[5444, 5482, 5376, 5466, 5374, 5346, 5642, 5542] dup_rows_per_dst=[2724, 2734, 2700, 2646, 2624, 2610, 2998, 2828] combine_rows_per_src=[5444, 5482, 5376, 5466, 5374, 5346, 5642, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38202 dup_rows=21896 dup_rows_remote=19160 dedup_ratio=0.334106 router_tokens_per_expert=[257, 261, 245, 259, 237, 249, 279, 261, 245, 250, 248, 252, 255, 250, 237, 249, 251, 248, 265, 278, 256, 268, 208, 232, 239, 264, 252, 274, 243, 258, 260, 237, 266, 243, 246, 255, 278, 277, 270, 264, 249, 249, 272, 252, 234, 263, 253, 245, 261, 239, 254, 250, 251, 235, 250, 244, 267, 267, 263, 276, 252, 260, 261, 257, 258, 270, 249, 280, 254, 250, 259, 244, 265, 248, 264, 266, 288, 274, 269, 252, 264, 252, 247, 271, 257, 223, 249, 268, 269, 251, 262, 233, 272, 242, 256, 264, 280, 261, 262, 256, 256, 271, 230, 271, 264, 272, 240, 245, 277, 253, 253, 241, 254, 272, 261, 279, 261, 247, 213, 243, 257, 253, 260, 244, 274, 284, 213, 281] payload_rows_per_dst=[5436, 5374, 5422, 5438, 5564, 5364, 5504, 5538] dup_rows_per_dst=[2632, 2692, 2810, 2736, 2816, 2796, 2760, 2654] combine_rows_per_src=[5436, 5374, 5422, 5438, 5564, 5364, 5504, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38392 dup_rows=21710 dup_rows_remote=19010 dedup_ratio=0.331268 router_tokens_per_expert=[255, 243, 235, 253, 255, 240, 254, 252, 252, 261, 267, 289, 267, 253, 269, 246, 231, 286, 241, 265, 260, 243, 258, 235, 250, 269, 260, 281, 257, 261, 255, 250, 254, 257, 259, 236, 265, 276, 235, 246, 279, 255, 272, 268, 270, 270, 232, 250, 255, 238, 256, 271, 264, 260, 260, 237, 254, 296, 246, 260, 241, 242, 270, 289, 234, 277, 254, 282, 231, 254, 271, 265, 251, 245, 273, 276, 239, 249, 237, 236, 284, 240, 266, 225, 271, 263, 244, 251, 243, 236, 289, 239, 258, 249, 263, 246, 251, 257, 242, 218, 263, 260, 267, 236, 274, 244, 224, 258, 242, 247, 263, 243, 257, 254, 270, 281, 250, 252, 272, 283, 239, 270, 256, 284, 235, 281, 232, 266] payload_rows_per_dst=[5438, 5442, 5488, 5554, 5470, 5434, 5434, 5566] dup_rows_per_dst=[2744, 2762, 2760, 2724, 2678, 2700, 2544, 2798] combine_rows_per_src=[5438, 5442, 5488, 5554, 5470, 5434, 5434, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=228872 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007368 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313785864 weight_movement_bytes=18874368 total_nvlink_bytes=332660232 weight_to_token_ratio=0.060150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38260 dup_rows=21784 dup_rows_remote=19114 dedup_ratio=0.332397 router_tokens_per_expert=[251, 249, 237, 274, 266, 262, 224, 246, 215, 261, 262, 266, 272, 271, 254, 265, 262, 270, 265, 236, 231, 247, 268, 254, 237, 262, 270, 251, 276, 255, 233, 264, 230, 254, 290, 275, 241, 255, 262, 264, 243, 246, 256, 250, 266, 250, 265, 238, 263, 284, 257, 239, 210, 243, 266, 241, 267, 264, 257, 251, 238, 270, 252, 294, 249, 234, 276, 258, 262, 270, 243, 233, 249, 261, 245, 277, 271, 226, 249, 266, 244, 264, 242, 274, 242, 248, 255, 272, 248, 265, 266, 261, 287, 243, 237, 261, 220, 273, 259, 258, 261, 257, 264, 262, 242, 245, 267, 247, 262, 244, 271, 257, 262, 259, 259, 242, 255, 270, 267, 253, 261, 242, 264, 276, 277, 269, 252, 256] payload_rows_per_dst=[5398, 5492, 5426, 5438, 5464, 5510, 5490, 5534] dup_rows_per_dst=[2752, 2670, 2744, 2754, 2720, 2708, 2688, 2748] combine_rows_per_src=[5398, 5492, 5426, 5438, 5464, 5510, 5490, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38214 dup_rows=21868 dup_rows_remote=19178 dedup_ratio=0.333679 router_tokens_per_expert=[235, 274, 245, 235, 279, 276, 262, 250, 254, 250, 258, 248, 282, 264, 221, 250, 242, 245, 242, 252, 265, 280, 244, 246, 253, 268, 251, 240, 254, 256, 250, 251, 258, 246, 263, 252, 245, 262, 268, 239, 222, 235, 242, 240, 260, 241, 295, 263, 277, 235, 280, 266, 251, 267, 284, 252, 242, 262, 259, 262, 270, 237, 273, 259, 251, 248, 256, 260, 252, 257, 261, 231, 259, 266, 275, 246, 269, 248, 273, 220, 270, 243, 248, 252, 277, 251, 251, 253, 271, 268, 261, 286, 235, 238, 248, 267, 239, 256, 259, 265, 273, 274, 254, 234, 248, 258, 240, 279, 253, 263, 252, 263, 236, 271, 270, 281, 251, 268, 267, 281, 265, 248, 235, 254, 230, 272, 274, 235] payload_rows_per_dst=[5472, 5406, 5404, 5516, 5454, 5428, 5472, 5516] dup_rows_per_dst=[2694, 2672, 2658, 2836, 2690, 2810, 2748, 2760] combine_rows_per_src=[5472, 5406, 5404, 5516, 5454, 5428, 5472, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=228664 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156409144 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312589624 weight_movement_bytes=37748736 total_nvlink_bytes=350338360 weight_to_token_ratio=0.120761 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229448 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704840 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313180232 weight_movement_bytes=18874368 total_nvlink_bytes=332054600 weight_to_token_ratio=0.060267 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38170 dup_rows=21890 dup_rows_remote=19104 dedup_ratio=0.334015 router_tokens_per_expert=[246, 276, 261, 239, 257, 250, 251, 267, 273, 237, 283, 270, 261, 257, 270, 263, 264, 271, 248, 264, 260, 231, 287, 245, 230, 253, 258, 270, 278, 273, 246, 233, 255, 261, 241, 255, 268, 242, 253, 249, 268, 255, 251, 240, 265, 250, 265, 267, 243, 248, 264, 246, 255, 263, 265, 252, 262, 246, 231, 258, 254, 239, 265, 242, 257, 265, 236, 277, 262, 249, 235, 236, 248, 267, 253, 266, 263, 242, 250, 250, 274, 246, 239, 250, 277, 244, 247, 230, 261, 284, 257, 252, 266, 250, 255, 238, 246, 264, 268, 249, 238, 273, 277, 243, 258, 251, 271, 245, 247, 296, 248, 257, 248, 288, 249, 259, 246, 266, 258, 253, 283, 244, 264, 268, 233, 261, 250, 251] payload_rows_per_dst=[5478, 5400, 5596, 5372, 5400, 5404, 5476, 5520] dup_rows_per_dst=[2844, 2612, 2784, 2694, 2712, 2736, 2786, 2722] combine_rows_per_src=[5478, 5400, 5596, 5372, 5400, 5404, 5476, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229608 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157483240 ingress_bytes=213876736 link_bytes=213876736 token_bytes=314736872 weight_movement_bytes=56623104 total_nvlink_bytes=371359976 weight_to_token_ratio=0.179906 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049261650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=37980 dup_rows=21868 dup_rows_remote=18932 dedup_ratio=0.333679 router_tokens_per_expert=[251, 286, 235, 268, 240, 260, 243, 277, 244, 257, 268, 237, 239, 248, 290, 240, 241, 215, 247, 236, 263, 235, 251, 249, 286, 264, 267, 269, 276, 222, 274, 251, 260, 251, 251, 233, 255, 258, 270, 244, 257, 247, 267, 252, 252, 244, 262, 242, 254, 261, 253, 238, 268, 260, 264, 214, 270, 275, 263, 236, 258, 248, 261, 253, 258, 257, 243, 251, 278, 270, 247, 259, 243, 252, 266, 293, 251, 259, 278, 298, 248, 256, 252, 235, 231, 255, 276, 272, 246, 251, 280, 251, 241, 287, 259, 271, 261, 285, 251, 262, 261, 238, 269, 269, 260, 255, 270, 249, 286, 244, 232, 233, 258, 274, 248, 247, 256, 241, 253, 271, 270, 240, 257, 282, 207, 273, 246, 256] payload_rows_per_dst=[5688, 5498, 5516, 5418, 5322, 5398, 5380, 5448] dup_rows_per_dst=[2936, 2824, 2766, 2734, 2626, 2594, 2678, 2710] combine_rows_per_src=[5688, 5498, 5516, 5418, 5322, 5398, 5380, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229496 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156942456 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655416 weight_movement_bytes=37748736 total_nvlink_bytes=351404152 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229568 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156754112 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278656 weight_movement_bytes=18874368 total_nvlink_bytes=332153024 weight_to_token_ratio=0.060248 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155566080 dispatch_meta_bytes=227648 combine_payload_bytes=155566080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155793728 ingress_bytes=174440448 link_bytes=174440448 token_bytes=311359808 weight_movement_bytes=18874368 total_nvlink_bytes=330234176 weight_to_token_ratio=0.060619 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671402543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229096 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573416 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917736 weight_movement_bytes=18874368 total_nvlink_bytes=331792104 weight_to_token_ratio=0.060317 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43366 payload_rows_remote=37890 dup_rows=22170 dup_rows_remote=19448 dedup_ratio=0.338287 router_tokens_per_expert=[257, 277, 273, 299, 238, 280, 284, 261, 227, 235, 253, 269, 249, 260, 251, 269, 276, 260, 254, 230, 253, 275, 261, 277, 262, 254, 259, 250, 262, 238, 239, 247, 234, 232, 254, 268, 260, 242, 288, 233, 256, 288, 260, 256, 263, 237, 266, 262, 275, 257, 245, 256, 265, 246, 242, 246, 274, 239, 285, 252, 247, 265, 261, 254, 268, 245, 263, 258, 276, 263, 252, 234, 274, 272, 265, 243, 234, 275, 267, 258, 246, 238, 257, 236, 260, 250, 254, 232, 245, 276, 270, 253, 241, 271, 249, 240, 280, 249, 257, 221, 286, 265, 266, 276, 277, 263, 232, 230, 254, 266, 263, 248, 261, 256, 245, 260, 248, 240, 247, 243, 242, 238, 227, 229, 269, 236, 283, 259] payload_rows_per_dst=[5468, 5388, 5476, 5390, 5410, 5414, 5440, 5380] dup_rows_per_dst=[2896, 2806, 2722, 2828, 2884, 2622, 2826, 2586] combine_rows_per_src=[5468, 5388, 5476, 5390, 5410, 5414, 5440, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38136 dup_rows=21812 dup_rows_remote=18778 dedup_ratio=0.332825 router_tokens_per_expert=[281, 252, 263, 270, 217, 259, 257, 268, 252, 261, 246, 274, 267, 265, 234, 257, 226, 245, 264, 255, 258, 239, 250, 271, 270, 258, 261, 249, 260, 267, 266, 255, 246, 285, 259, 272, 239, 238, 243, 270, 270, 263, 272, 261, 248, 274, 238, 280, 242, 264, 232, 261, 243, 232, 256, 259, 247, 264, 247, 242, 240, 266, 250, 250, 240, 242, 240, 259, 263, 244, 237, 237, 241, 303, 240, 253, 241, 257, 254, 242, 292, 276, 255, 280, 254, 269, 239, 258, 266, 275, 264, 256, 265, 256, 243, 249, 273, 281, 285, 271, 243, 235, 272, 246, 259, 289, 247, 258, 261, 256, 260, 242, 251, 251, 271, 256, 260, 236, 218, 267, 239, 263, 272, 237, 244, 253, 268, 244] payload_rows_per_dst=[5278, 5510, 5482, 5454, 5364, 5510, 5538, 5588] dup_rows_per_dst=[2406, 2678, 2834, 2536, 2622, 2864, 2838, 3034] combine_rows_per_src=[5278, 5510, 5482, 5454, 5364, 5510, 5538, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38338 dup_rows=21662 dup_rows_remote=19000 dedup_ratio=0.330536 router_tokens_per_expert=[262, 242, 247, 260, 271, 262, 261, 271, 272, 248, 268, 254, 249, 267, 260, 244, 274, 269, 282, 246, 251, 249, 238, 233, 273, 255, 274, 269, 277, 238, 251, 262, 265, 244, 269, 229, 235, 261, 233, 251, 231, 243, 250, 257, 261, 260, 238, 267, 252, 262, 275, 275, 277, 252, 283, 245, 240, 245, 255, 228, 270, 249, 267, 224, 251, 247, 231, 266, 271, 260, 261, 254, 262, 248, 229, 297, 235, 280, 228, 234, 249, 275, 276, 228, 260, 239, 253, 259, 247, 256, 251, 252, 227, 266, 264, 251, 266, 252, 261, 271, 261, 269, 240, 253, 252, 267, 268, 266, 281, 265, 258, 287, 258, 241, 244, 258, 269, 253, 274, 258, 249, 284, 266, 254, 255, 235, 238, 236] payload_rows_per_dst=[5542, 5558, 5376, 5536, 5422, 5468, 5530, 5442] dup_rows_per_dst=[2734, 2724, 2612, 2662, 2686, 2638, 2904, 2702] combine_rows_per_src=[5542, 5558, 5376, 5536, 5422, 5468, 5530, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155197440 dispatch_meta_bytes=229352 combine_payload_bytes=155197440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155426792 ingress_bytes=174071808 link_bytes=174071808 token_bytes=310624232 weight_movement_bytes=18874368 total_nvlink_bytes=329498600 weight_to_token_ratio=0.060763 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.667870416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38202 dup_rows=21778 dup_rows_remote=18934 dedup_ratio=0.332306 router_tokens_per_expert=[282, 264, 250, 276, 254, 258, 276, 240, 266, 212, 248, 264, 278, 247, 262, 255, 272, 267, 238, 267, 264, 263, 267, 263, 256, 251, 279, 281, 249, 254, 263, 217, 233, 270, 221, 252, 238, 235, 270, 268, 252, 218, 277, 251, 251, 272, 240, 249, 228, 261, 266, 269, 256, 241, 254, 277, 245, 257, 252, 257, 245, 230, 260, 277, 243, 258, 248, 238, 250, 272, 263, 278, 267, 251, 271, 260, 273, 230, 271, 244, 263, 250, 264, 239, 270, 251, 298, 278, 282, 255, 280, 235, 251, 257, 277, 250, 242, 252, 260, 229, 255, 271, 261, 249, 231, 270, 254, 265, 266, 267, 259, 240, 265, 222, 269, 252, 261, 255, 247, 242, 257, 262, 276, 258, 236, 228, 246, 249] payload_rows_per_dst=[5508, 5546, 5416, 5422, 5538, 5556, 5424, 5348] dup_rows_per_dst=[2756, 2756, 2578, 2728, 2696, 2844, 2718, 2702] combine_rows_per_src=[5508, 5546, 5416, 5422, 5538, 5556, 5424, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38250 dup_rows=21812 dup_rows_remote=18992 dedup_ratio=0.332825 router_tokens_per_expert=[264, 226, 258, 244, 276, 258, 239, 246, 256, 297, 238, 259, 246, 273, 251, 271, 255, 249, 274, 276, 259, 271, 255, 263, 244, 264, 252, 282, 252, 257, 239, 255, 288, 260, 258, 219, 274, 251, 279, 256, 275, 246, 263, 251, 217, 237, 237, 241, 248, 283, 266, 253, 250, 264, 259, 261, 280, 244, 250, 266, 255, 250, 249, 240, 263, 266, 255, 246, 248, 250, 241, 275, 262, 273, 271, 264, 275, 295, 247, 253, 236, 284, 238, 244, 271, 267, 253, 263, 262, 243, 263, 262, 256, 248, 239, 251, 274, 203, 261, 244, 269, 264, 245, 252, 250, 259, 253, 261, 276, 263, 248, 234, 257, 250, 255, 238, 254, 244, 253, 241, 241, 262, 265, 263, 270, 232, 256, 248] payload_rows_per_dst=[5498, 5474, 5374, 5506, 5612, 5442, 5424, 5394] dup_rows_per_dst=[2706, 2820, 2730, 2730, 2756, 2718, 2688, 2664] combine_rows_per_src=[5498, 5474, 5374, 5506, 5612, 5442, 5424, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38212 dup_rows=21894 dup_rows_remote=19258 dedup_ratio=0.334076 router_tokens_per_expert=[263, 269, 258, 252, 271, 250, 264, 251, 255, 248, 239, 247, 251, 260, 268, 244, 265, 253, 242, 276, 273, 246, 282, 224, 256, 256, 266, 284, 248, 270, 256, 262, 248, 244, 291, 265, 241, 239, 265, 258, 249, 259, 278, 276, 247, 246, 254, 221, 241, 234, 263, 254, 268, 267, 230, 264, 243, 252, 282, 261, 278, 269, 242, 255, 238, 241, 285, 242, 264, 283, 277, 236, 240, 211, 244, 278, 231, 248, 270, 245, 248, 233, 247, 257, 270, 234, 263, 237, 258, 243, 263, 269, 272, 256, 276, 278, 254, 234, 272, 254, 262, 247, 235, 260, 254, 261, 287, 266, 248, 255, 250, 265, 258, 283, 271, 253, 236, 254, 264, 259, 244, 247, 250, 268, 246, 276, 225, 260] payload_rows_per_dst=[5536, 5476, 5394, 5412, 5430, 5480, 5448, 5466] dup_rows_per_dst=[2644, 2842, 2768, 2794, 2636, 2728, 2760, 2722] combine_rows_per_src=[5536, 5476, 5394, 5412, 5430, 5480, 5448, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=227656 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156432712 ingress_bytes=175079424 link_bytes=175079424 token_bytes=312637768 weight_movement_bytes=18874368 total_nvlink_bytes=331512136 weight_to_token_ratio=0.060371 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677524896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229352 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261800 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294248 weight_movement_bytes=18874368 total_nvlink_bytes=333168616 weight_to_token_ratio=0.060053 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43430 payload_rows_remote=37896 dup_rows=22106 dup_rows_remote=19404 dedup_ratio=0.337311 router_tokens_per_expert=[241, 241, 253, 249, 239, 294, 250, 286, 245, 251, 258, 262, 248, 253, 263, 269, 268, 276, 276, 269, 250, 250, 295, 254, 240, 227, 254, 278, 265, 263, 282, 251, 236, 238, 254, 237, 268, 288, 259, 251, 286, 249, 235, 257, 286, 259, 262, 267, 229, 251, 250, 243, 252, 256, 223, 214, 238, 242, 265, 263, 227, 262, 257, 244, 269, 272, 237, 248, 260, 223, 263, 248, 272, 239, 256, 279, 239, 251, 260, 251, 267, 282, 256, 272, 236, 258, 254, 272, 255, 253, 246, 282, 271, 246, 242, 254, 272, 256, 273, 254, 248, 234, 285, 260, 250, 244, 264, 223, 284, 272, 271, 228, 243, 250, 237, 265, 229, 266, 284, 233, 259, 255, 280, 265, 268, 245, 269, 241] payload_rows_per_dst=[5412, 5504, 5488, 5294, 5418, 5408, 5534, 5372] dup_rows_per_dst=[2792, 2892, 2776, 2538, 2716, 2884, 2702, 2806] combine_rows_per_src=[5412, 5504, 5488, 5294, 5418, 5408, 5534, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43494 payload_rows_remote=37944 dup_rows=22042 dup_rows_remote=19208 dedup_ratio=0.336334 router_tokens_per_expert=[247, 272, 281, 265, 262, 283, 259, 250, 261, 249, 279, 269, 235, 244, 248, 261, 258, 242, 271, 236, 227, 261, 250, 264, 259, 241, 278, 234, 272, 246, 266, 260, 259, 251, 247, 231, 271, 284, 250, 245, 245, 245, 248, 254, 226, 259, 253, 237, 243, 237, 257, 266, 255, 262, 256, 253, 276, 272, 242, 233, 273, 262, 257, 249, 232, 263, 280, 221, 259, 241, 276, 244, 251, 238, 286, 241, 258, 260, 269, 250, 255, 255, 246, 259, 276, 262, 253, 253, 263, 262, 288, 257, 242, 278, 292, 259, 267, 233, 255, 281, 237, 252, 275, 282, 275, 240, 259, 281, 235, 274, 249, 256, 229, 276, 249, 225, 262, 269, 233, 264, 275, 243, 264, 251, 251, 232, 257, 240] payload_rows_per_dst=[5550, 5362, 5356, 5464, 5616, 5514, 5446, 5186] dup_rows_per_dst=[2834, 2768, 2654, 2722, 3050, 2832, 2856, 2326] combine_rows_per_src=[5550, 5362, 5356, 5464, 5616, 5514, 5446, 5186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=228544 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156703936 ingress_bytes=213098496 link_bytes=213098496 token_bytes=313179328 weight_movement_bytes=56623104 total_nvlink_bytes=369802432 weight_to_token_ratio=0.180801 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041804938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228968 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156900968 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313572968 weight_movement_bytes=37748736 total_nvlink_bytes=351321704 weight_to_token_ratio=0.120383 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229880 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156746232 ingress_bytes=213139456 link_bytes=213139456 token_bytes=313262584 weight_movement_bytes=56623104 total_nvlink_bytes=369885688 weight_to_token_ratio=0.180753 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042197396e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155222016 dispatch_meta_bytes=229200 combine_payload_bytes=155222016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155451216 ingress_bytes=174096384 link_bytes=174096384 token_bytes=310673232 weight_movement_bytes=18874368 total_nvlink_bytes=329547600 weight_to_token_ratio=0.060753 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.668105891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38170 dup_rows=21906 dup_rows_remote=19286 dedup_ratio=0.334259 router_tokens_per_expert=[237, 269, 266, 247, 278, 263, 257, 262, 252, 281, 248, 243, 237, 246, 257, 223, 259, 222, 275, 245, 258, 250, 277, 255, 248, 247, 279, 291, 254, 251, 254, 252, 279, 274, 254, 274, 258, 265, 247, 241, 233, 248, 257, 271, 229, 255, 218, 237, 245, 273, 253, 253, 234, 262, 251, 273, 258, 254, 288, 240, 280, 277, 266, 249, 251, 287, 249, 240, 252, 285, 258, 256, 248, 236, 253, 245, 261, 239, 245, 247, 263, 241, 236, 273, 223, 276, 261, 241, 234, 268, 238, 267, 277, 234, 286, 251, 281, 273, 251, 237, 257, 272, 237, 258, 251, 284, 236, 271, 271, 265, 268, 240, 294, 257, 246, 249, 257, 263, 279, 256, 258, 248, 253, 251, 236, 242, 253, 274] payload_rows_per_dst=[5392, 5454, 5460, 5516, 5352, 5416, 5506, 5534] dup_rows_per_dst=[2740, 2780, 2620, 2796, 2752, 2722, 2798, 2698] combine_rows_per_src=[5392, 5454, 5460, 5516, 5352, 5416, 5506, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=155418624 dispatch_meta_bytes=228608 combine_payload_bytes=155418624 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155647232 ingress_bytes=212041728 link_bytes=212041728 token_bytes=311065856 weight_movement_bytes=56623104 total_nvlink_bytes=367688960 weight_to_token_ratio=0.182029 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.031679507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38412 dup_rows=21680 dup_rows_remote=18994 dedup_ratio=0.330811 router_tokens_per_expert=[262, 247, 255, 230, 255, 258, 258, 236, 239, 258, 247, 287, 251, 253, 242, 258, 234, 250, 242, 259, 257, 263, 261, 262, 236, 284, 279, 245, 261, 258, 251, 287, 281, 265, 276, 250, 257, 274, 241, 251, 251, 248, 278, 270, 256, 252, 245, 257, 275, 242, 258, 254, 243, 275, 255, 240, 248, 274, 229, 239, 250, 247, 255, 233, 287, 254, 251, 294, 228, 253, 240, 240, 257, 263, 243, 284, 255, 243, 267, 259, 265, 246, 284, 274, 269, 273, 220, 274, 255, 259, 241, 250, 243, 263, 263, 257, 270, 254, 266, 267, 276, 277, 233, 237, 290, 260, 249, 266, 231, 260, 244, 250, 258, 289, 286, 228, 245, 261, 253, 231, 246, 230, 266, 267, 242, 229, 268, 251] payload_rows_per_dst=[5472, 5470, 5574, 5414, 5478, 5470, 5534, 5444] dup_rows_per_dst=[2570, 2788, 2730, 2620, 2758, 2802, 2726, 2686] combine_rows_per_src=[5472, 5470, 5574, 5414, 5478, 5470, 5534, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38434 dup_rows=21720 dup_rows_remote=19036 dedup_ratio=0.331421 router_tokens_per_expert=[257, 257, 236, 263, 252, 225, 270, 248, 261, 272, 252, 245, 238, 250, 244, 274, 244, 259, 286, 249, 243, 261, 269, 284, 255, 231, 234, 250, 244, 260, 260, 251, 247, 283, 247, 258, 259, 255, 248, 261, 262, 260, 261, 240, 285, 240, 270, 247, 248, 250, 234, 236, 264, 228, 235, 302, 257, 259, 251, 249, 247, 263, 256, 254, 267, 252, 259, 268, 281, 293, 282, 268, 277, 232, 249, 222, 272, 244, 248, 246, 251, 266, 249, 250, 253, 264, 269, 242, 284, 274, 238, 235, 274, 234, 239, 257, 262, 253, 252, 279, 275, 271, 257, 273, 272, 246, 258, 264, 242, 253, 274, 266, 265, 260, 248, 239, 291, 242, 245, 267, 254, 269, 226, 277, 238, 250, 241, 240] payload_rows_per_dst=[5438, 5506, 5530, 5382, 5560, 5454, 5540, 5406] dup_rows_per_dst=[2650, 2654, 2716, 2684, 2760, 2704, 2854, 2698] combine_rows_per_src=[5438, 5506, 5530, 5382, 5560, 5454, 5540, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229824 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156574144 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312918464 weight_movement_bytes=18874368 total_nvlink_bytes=331792832 weight_to_token_ratio=0.060317 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38142 dup_rows=21830 dup_rows_remote=18942 dedup_ratio=0.333099 router_tokens_per_expert=[278, 249, 259, 263, 249, 302, 260, 243, 261, 236, 236, 249, 242, 262, 267, 277, 245, 245, 234, 246, 248, 242, 275, 255, 256, 261, 227, 253, 244, 258, 270, 242, 250, 239, 248, 247, 275, 274, 238, 249, 265, 257, 238, 260, 237, 241, 266, 280, 260, 275, 250, 275, 239, 260, 224, 256, 232, 284, 260, 260, 234, 255, 254, 233, 259, 258, 265, 256, 239, 277, 263, 244, 270, 223, 243, 252, 239, 259, 246, 250, 269, 285, 237, 284, 267, 246, 282, 264, 299, 249, 284, 254, 256, 270, 250, 230, 263, 259, 233, 256, 252, 236, 249, 265, 238, 269, 242, 239, 241, 279, 248, 259, 244, 265, 275, 268, 259, 251, 271, 258, 258, 251, 268, 261, 262, 266, 273, 292] payload_rows_per_dst=[5508, 5402, 5444, 5432, 5366, 5564, 5460, 5530] dup_rows_per_dst=[2758, 2600, 2684, 2670, 2720, 2888, 2596, 2914] combine_rows_per_src=[5508, 5402, 5444, 5432, 5366, 5564, 5460, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38216 dup_rows=21946 dup_rows_remote=19258 dedup_ratio=0.334869 router_tokens_per_expert=[258, 231, 266, 256, 236, 262, 298, 249, 257, 261, 272, 260, 251, 274, 256, 257, 234, 257, 250, 250, 264, 252, 241, 250, 265, 257, 262, 239, 256, 276, 240, 238, 252, 262, 251, 257, 253, 273, 230, 251, 278, 248, 260, 261, 267, 231, 275, 244, 266, 260, 239, 250, 234, 263, 221, 238, 268, 252, 264, 255, 239, 263, 265, 264, 265, 275, 236, 282, 258, 259, 269, 240, 253, 263, 222, 266, 263, 269, 253, 299, 252, 247, 271, 240, 272, 262, 254, 239, 252, 225, 261, 267, 242, 290, 267, 254, 245, 269, 244, 267, 249, 262, 255, 266, 252, 236, 259, 276, 229, 252, 248, 262, 251, 292, 268, 251, 253, 260, 244, 266, 249, 278, 249, 240, 230, 242, 276, 272] payload_rows_per_dst=[5526, 5374, 5446, 5436, 5452, 5456, 5398, 5502] dup_rows_per_dst=[2762, 2688, 2740, 2646, 2892, 2734, 2744, 2740] combine_rows_per_src=[5526, 5374, 5446, 5436, 5452, 5456, 5398, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38186 dup_rows=21882 dup_rows_remote=19140 dedup_ratio=0.333893 router_tokens_per_expert=[264, 246, 253, 262, 242, 257, 252, 257, 260, 235, 249, 261, 272, 288, 259, 250, 264, 276, 234, 265, 257, 248, 232, 273, 282, 276, 238, 248, 248, 268, 265, 271, 267, 275, 250, 240, 277, 247, 246, 249, 263, 263, 231, 231, 278, 243, 257, 265, 257, 273, 250, 247, 249, 239, 275, 255, 244, 255, 283, 260, 244, 235, 246, 259, 298, 261, 257, 244, 256, 258, 231, 254, 267, 261, 272, 240, 264, 234, 249, 259, 273, 259, 231, 256, 273, 227, 255, 261, 262, 270, 262, 272, 236, 234, 250, 255, 249, 281, 233, 284, 232, 242, 273, 234, 275, 245, 267, 274, 236, 251, 256, 274, 258, 240, 252, 232, 280, 240, 269, 261, 252, 233, 269, 256, 247, 276, 245, 266] payload_rows_per_dst=[5484, 5480, 5446, 5416, 5468, 5412, 5520, 5428] dup_rows_per_dst=[2730, 2810, 2718, 2726, 2742, 2740, 2692, 2724] combine_rows_per_src=[5484, 5480, 5446, 5416, 5468, 5412, 5520, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38298 dup_rows=21702 dup_rows_remote=18904 dedup_ratio=0.331146 router_tokens_per_expert=[285, 265, 265, 256, 266, 261, 243, 248, 272, 252, 248, 240, 248, 287, 242, 262, 230, 251, 243, 292, 224, 241, 255, 259, 259, 235, 258, 285, 260, 290, 241, 278, 249, 241, 238, 269, 271, 258, 232, 230, 286, 256, 243, 257, 262, 262, 269, 252, 245, 217, 234, 267, 245, 264, 240, 267, 265, 253, 271, 288, 251, 255, 275, 262, 251, 240, 248, 283, 261, 285, 243, 231, 276, 255, 230, 273, 247, 245, 241, 259, 241, 258, 255, 262, 243, 277, 255, 248, 234, 255, 258, 250, 257, 241, 277, 257, 274, 264, 246, 267, 269, 250, 254, 248, 254, 259, 274, 266, 263, 262, 276, 241, 239, 241, 248, 258, 261, 260, 228, 240, 267, 261, 254, 249, 245, 269, 276, 254] payload_rows_per_dst=[5488, 5518, 5452, 5480, 5388, 5496, 5536, 5476] dup_rows_per_dst=[2792, 2684, 2698, 2718, 2748, 2640, 2798, 2624] combine_rows_per_src=[5488, 5518, 5452, 5480, 5388, 5496, 5536, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229624 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157565176 ingress_bytes=195084288 link_bytes=195084288 token_bytes=314900728 weight_movement_bytes=37748736 total_nvlink_bytes=352649464 weight_to_token_ratio=0.119875 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869201660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157425664 dispatch_meta_bytes=229880 combine_payload_bytes=157425664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157655544 ingress_bytes=176300032 link_bytes=176300032 token_bytes=315081208 weight_movement_bytes=18874368 total_nvlink_bytes=333955576 weight_to_token_ratio=0.059903 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689220162e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38338 dup_rows=21790 dup_rows_remote=19124 dedup_ratio=0.332489 router_tokens_per_expert=[246, 271, 261, 228, 248, 233, 253, 257, 229, 257, 229, 262, 288, 256, 271, 247, 236, 250, 236, 258, 269, 255, 279, 290, 253, 259, 223, 251, 249, 247, 288, 300, 266, 255, 259, 268, 284, 255, 240, 246, 256, 278, 302, 252, 260, 254, 244, 257, 269, 228, 240, 258, 269, 258, 243, 256, 259, 280, 275, 226, 254, 240, 238, 242, 249, 249, 273, 242, 289, 251, 251, 243, 279, 263, 236, 254, 247, 258, 269, 257, 263, 245, 250, 258, 283, 265, 253, 255, 248, 277, 261, 249, 235, 273, 264, 254, 259, 246, 246, 233, 240, 268, 242, 251, 256, 267, 235, 258, 269, 247, 260, 276, 274, 253, 249, 261, 265, 241, 270, 241, 260, 256, 264, 221, 267, 245, 254, 261] payload_rows_per_dst=[5408, 5584, 5512, 5590, 5544, 5268, 5460, 5380] dup_rows_per_dst=[2666, 2804, 2738, 2942, 2786, 2424, 2646, 2784] combine_rows_per_src=[5408, 5584, 5512, 5590, 5544, 5268, 5460, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=228336 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156457968 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312687600 weight_movement_bytes=18874368 total_nvlink_bytes=331561968 weight_to_token_ratio=0.060362 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229304 combine_payload_bytes=156409856 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156639160 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313049016 weight_movement_bytes=18874368 total_nvlink_bytes=331923384 weight_to_token_ratio=0.060292 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229896 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762632 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295368 weight_movement_bytes=18874368 total_nvlink_bytes=332169736 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=228808 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097416 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966024 weight_movement_bytes=18874368 total_nvlink_bytes=332840392 weight_to_token_ratio=0.060116 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38456 dup_rows=21606 dup_rows_remote=18978 dedup_ratio=0.329681 router_tokens_per_expert=[253, 247, 272, 267, 242, 282, 272, 260, 233, 250, 277, 269, 227, 248, 241, 279, 266, 247, 242, 256, 241, 254, 272, 255, 257, 261, 262, 230, 266, 229, 241, 241, 256, 243, 284, 255, 242, 253, 250, 257, 243, 264, 251, 261, 256, 230, 261, 245, 273, 240, 235, 245, 245, 256, 264, 266, 288, 258, 268, 265, 244, 234, 250, 264, 246, 269, 251, 264, 272, 262, 260, 244, 239, 262, 267, 256, 239, 286, 240, 247, 254, 224, 247, 258, 275, 260, 233, 259, 282, 251, 274, 237, 284, 273, 256, 266, 241, 252, 254, 251, 263, 252, 263, 280, 279, 256, 272, 238, 299, 256, 303, 233, 244, 252, 241, 245, 259, 231, 251, 271, 276, 273, 270, 262, 232, 237, 260, 250] payload_rows_per_dst=[5544, 5404, 5474, 5448, 5526, 5506, 5598, 5430] dup_rows_per_dst=[2694, 2636, 2628, 2742, 2682, 2760, 2786, 2678] combine_rows_per_src=[5544, 5404, 5474, 5448, 5526, 5506, 5598, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229848 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157262296 ingress_bytes=194781184 link_bytes=194781184 token_bytes=314294744 weight_movement_bytes=37748736 total_nvlink_bytes=352043480 weight_to_token_ratio=0.120106 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866297467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157515776 dispatch_meta_bytes=229736 combine_payload_bytes=157515776 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157745512 ingress_bytes=214138880 link_bytes=214138880 token_bytes=315261288 weight_movement_bytes=56623104 total_nvlink_bytes=371884392 weight_to_token_ratio=0.179607 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051773385e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38222 dup_rows=21832 dup_rows_remote=19078 dedup_ratio=0.333130 router_tokens_per_expert=[275, 208, 259, 257, 265, 285, 263, 249, 258, 249, 263, 233, 258, 274, 264, 261, 242, 227, 266, 252, 266, 268, 252, 269, 220, 290, 261, 239, 242, 277, 273, 270, 255, 257, 227, 228, 246, 275, 258, 260, 269, 261, 267, 268, 244, 284, 233, 294, 274, 239, 270, 245, 254, 263, 248, 244, 278, 243, 261, 243, 241, 278, 256, 281, 283, 229, 254, 262, 254, 263, 254, 255, 255, 270, 253, 232, 247, 255, 244, 263, 249, 245, 249, 275, 278, 217, 235, 260, 253, 293, 264, 264, 259, 263, 239, 251, 273, 259, 246, 238, 267, 259, 268, 234, 261, 246, 218, 254, 253, 255, 252, 243, 295, 255, 270, 285, 249, 231, 239, 246, 283, 246, 233, 263, 238, 241, 256, 266] payload_rows_per_dst=[5482, 5466, 5536, 5482, 5420, 5438, 5414, 5466] dup_rows_per_dst=[2760, 2762, 2716, 2754, 2726, 2750, 2638, 2726] combine_rows_per_src=[5482, 5466, 5536, 5482, 5420, 5438, 5414, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38350 dup_rows=21716 dup_rows_remote=18868 dedup_ratio=0.331360 router_tokens_per_expert=[226, 213, 259, 262, 256, 262, 243, 248, 249, 268, 250, 293, 253, 272, 265, 274, 238, 247, 256, 264, 236, 252, 276, 276, 256, 233, 247, 273, 260, 257, 250, 243, 286, 241, 249, 236, 263, 276, 288, 230, 244, 269, 251, 285, 264, 255, 260, 233, 255, 250, 256, 273, 265, 256, 260, 223, 237, 253, 310, 257, 253, 253, 242, 258, 220, 267, 249, 251, 262, 268, 264, 244, 262, 242, 263, 276, 239, 241, 240, 267, 265, 271, 245, 274, 241, 289, 273, 246, 261, 233, 231, 267, 253, 263, 225, 245, 294, 271, 281, 263, 278, 228, 229, 264, 246, 223, 221, 261, 249, 255, 249, 272, 264, 275, 276, 239, 250, 270, 243, 230, 275, 268, 283, 248, 266, 267, 265, 240] payload_rows_per_dst=[5458, 5488, 5338, 5518, 5588, 5432, 5528, 5470] dup_rows_per_dst=[2728, 2640, 2498, 2684, 2822, 2656, 2840, 2848] combine_rows_per_src=[5458, 5488, 5338, 5518, 5588, 5432, 5528, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38350 dup_rows=21818 dup_rows_remote=19130 dedup_ratio=0.332916 router_tokens_per_expert=[246, 262, 245, 233, 255, 259, 258, 280, 264, 241, 256, 251, 275, 264, 232, 259, 255, 236, 242, 245, 259, 251, 263, 295, 240, 254, 242, 271, 273, 258, 256, 261, 262, 268, 262, 284, 261, 261, 241, 250, 261, 263, 271, 263, 262, 252, 262, 283, 231, 245, 269, 247, 238, 279, 253, 253, 255, 261, 262, 264, 274, 260, 257, 239, 240, 245, 280, 264, 233, 259, 272, 269, 278, 230, 277, 264, 271, 273, 234, 263, 261, 277, 253, 271, 243, 241, 288, 227, 243, 257, 234, 255, 236, 252, 237, 253, 262, 277, 250, 245, 262, 263, 266, 255, 252, 253, 236, 254, 269, 251, 256, 238, 250, 276, 229, 240, 238, 282, 253, 258, 260, 244, 248, 236, 275, 239, 270, 227] payload_rows_per_dst=[5470, 5486, 5606, 5466, 5474, 5368, 5458, 5390] dup_rows_per_dst=[2690, 2716, 2806, 2708, 2830, 2688, 2720, 2660] combine_rows_per_src=[5470, 5486, 5606, 5466, 5474, 5368, 5458, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38476 dup_rows=21638 dup_rows_remote=18960 dedup_ratio=0.330170 router_tokens_per_expert=[252, 241, 283, 260, 253, 245, 253, 255, 231, 242, 240, 282, 226, 267, 231, 261, 267, 254, 265, 265, 279, 251, 259, 256, 276, 262, 269, 248, 254, 238, 260, 250, 294, 266, 246, 245, 252, 282, 243, 245, 306, 244, 239, 250, 247, 266, 282, 249, 285, 238, 238, 210, 249, 241, 269, 269, 279, 250, 253, 260, 246, 284, 260, 223, 277, 264, 248, 249, 235, 248, 233, 259, 243, 257, 258, 262, 259, 244, 267, 247, 295, 260, 231, 273, 253, 258, 236, 287, 247, 277, 271, 241, 260, 257, 277, 256, 245, 265, 236, 262, 240, 284, 254, 243, 240, 249, 244, 290, 281, 258, 260, 243, 237, 241, 283, 259, 231, 255, 259, 253, 250, 267, 262, 256, 210, 253, 269, 275] payload_rows_per_dst=[5400, 5540, 5492, 5516, 5422, 5530, 5488, 5510] dup_rows_per_dst=[2644, 2766, 2820, 2592, 2678, 2828, 2700, 2610] combine_rows_per_src=[5400, 5540, 5492, 5516, 5422, 5530, 5488, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38448 dup_rows=21620 dup_rows_remote=18922 dedup_ratio=0.329895 router_tokens_per_expert=[260, 257, 257, 244, 231, 244, 269, 260, 264, 258, 281, 264, 248, 243, 257, 260, 232, 269, 253, 236, 253, 269, 244, 260, 279, 289, 235, 266, 242, 251, 259, 246, 248, 267, 274, 261, 260, 265, 235, 238, 245, 244, 269, 245, 257, 258, 271, 250, 286, 309, 265, 243, 232, 224, 249, 260, 246, 276, 250, 269, 257, 260, 241, 236, 258, 229, 256, 242, 252, 250, 242, 244, 240, 265, 231, 251, 251, 271, 245, 286, 237, 244, 280, 266, 242, 262, 247, 256, 256, 284, 233, 270, 282, 257, 255, 260, 253, 250, 255, 235, 232, 290, 259, 249, 252, 266, 257, 236, 259, 251, 246, 264, 239, 250, 237, 273, 277, 266, 265, 263, 266, 257, 270, 257, 260, 291, 252, 277] payload_rows_per_dst=[5450, 5468, 5494, 5468, 5442, 5538, 5510, 5546] dup_rows_per_dst=[2744, 2698, 2680, 2738, 2584, 2724, 2598, 2854] combine_rows_per_src=[5450, 5468, 5494, 5468, 5442, 5538, 5510, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43550 payload_rows_remote=38150 dup_rows=21986 dup_rows_remote=19348 dedup_ratio=0.335480 router_tokens_per_expert=[278, 242, 236, 267, 290, 242, 252, 263, 277, 248, 290, 274, 253, 266, 266, 277, 250, 259, 284, 242, 306, 271, 275, 261, 274, 248, 267, 229, 273, 249, 247, 262, 280, 266, 250, 255, 249, 255, 280, 246, 230, 276, 235, 233, 257, 252, 240, 227, 253, 271, 256, 255, 277, 260, 229, 258, 232, 261, 271, 239, 260, 237, 282, 286, 278, 260, 248, 258, 262, 248, 244, 262, 256, 233, 261, 283, 255, 239, 258, 267, 265, 230, 265, 261, 247, 275, 254, 249, 256, 241, 270, 227, 245, 231, 240, 250, 266, 251, 272, 249, 235, 238, 233, 264, 269, 252, 240, 262, 261, 229, 238, 260, 276, 260, 264, 269, 221, 254, 242, 251, 249, 241, 278, 247, 231, 259, 251, 262] payload_rows_per_dst=[5502, 5494, 5410, 5444, 5412, 5376, 5400, 5512] dup_rows_per_dst=[2798, 2900, 2652, 2810, 2812, 2636, 2638, 2740] combine_rows_per_src=[5502, 5494, 5410, 5444, 5412, 5376, 5400, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38098 dup_rows=21954 dup_rows_remote=19212 dedup_ratio=0.334991 router_tokens_per_expert=[235, 255, 271, 217, 265, 259, 249, 271, 262, 250, 254, 265, 289, 264, 255, 252, 277, 266, 246, 274, 252, 264, 269, 245, 230, 277, 244, 253, 279, 237, 252, 245, 253, 290, 254, 276, 245, 266, 257, 242, 249, 242, 242, 275, 262, 238, 262, 268, 277, 246, 241, 237, 269, 267, 245, 257, 257, 252, 267, 256, 263, 239, 253, 258, 211, 232, 256, 259, 239, 266, 259, 264, 248, 238, 268, 259, 238, 263, 272, 267, 273, 229, 259, 266, 253, 259, 266, 282, 241, 260, 249, 292, 255, 278, 255, 260, 245, 224, 233, 276, 285, 236, 236, 219, 253, 239, 246, 274, 258, 256, 258, 287, 253, 261, 259, 234, 257, 270, 251, 270, 260, 282, 268, 260, 253, 263, 222, 236] payload_rows_per_dst=[5484, 5460, 5484, 5382, 5596, 5544, 5418, 5214] dup_rows_per_dst=[2742, 2760, 2758, 2786, 3022, 2810, 2632, 2444] combine_rows_per_src=[5484, 5460, 5484, 5382, 5596, 5544, 5418, 5214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229200 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786512 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343824 weight_movement_bytes=18874368 total_nvlink_bytes=332218192 weight_to_token_ratio=0.060235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228872 combine_payload_bytes=157081600 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157310472 ingress_bytes=194830336 link_bytes=194830336 token_bytes=314392072 weight_movement_bytes=37748736 total_nvlink_bytes=352140808 weight_to_token_ratio=0.120069 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866768417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229920 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157311520 ingress_bytes=194830336 link_bytes=194830336 token_bytes=314393120 weight_movement_bytes=37748736 total_nvlink_bytes=352141856 weight_to_token_ratio=0.120069 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866768417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229744 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827440 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315425136 weight_movement_bytes=18874368 total_nvlink_bytes=334299504 weight_to_token_ratio=0.059838 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=229480 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157712488 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315195496 weight_movement_bytes=18874368 total_nvlink_bytes=334069864 weight_to_token_ratio=0.059881 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=229992 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156492392 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312754792 weight_movement_bytes=18874368 total_nvlink_bytes=331629160 weight_to_token_ratio=0.060349 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44024 payload_rows_remote=38608 dup_rows=21512 dup_rows_remote=18864 dedup_ratio=0.328247 router_tokens_per_expert=[242, 260, 263, 264, 245, 255, 249, 256, 248, 223, 247, 243, 279, 255, 253, 221, 263, 283, 267, 269, 268, 294, 245, 235, 263, 256, 262, 289, 257, 256, 263, 245, 268, 233, 222, 266, 263, 258, 236, 268, 256, 274, 246, 234, 265, 251, 248, 244, 222, 245, 252, 263, 286, 247, 267, 269, 290, 258, 264, 279, 253, 257, 236, 241, 243, 245, 245, 269, 244, 242, 276, 261, 271, 268, 284, 247, 249, 262, 258, 237, 266, 266, 252, 266, 243, 273, 279, 264, 272, 253, 262, 231, 252, 246, 245, 248, 270, 257, 244, 240, 250, 239, 259, 236, 254, 260, 255, 229, 271, 286, 269, 253, 289, 250, 263, 273, 269, 238, 252, 245, 264, 263, 263, 249, 220, 249, 259, 252] payload_rows_per_dst=[5366, 5586, 5416, 5542, 5508, 5530, 5616, 5460] dup_rows_per_dst=[2640, 2844, 2648, 2716, 2694, 2706, 2528, 2736] combine_rows_per_src=[5366, 5586, 5416, 5542, 5508, 5530, 5616, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=229240 combine_payload_bytes=156049408 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156278648 ingress_bytes=212672512 link_bytes=212672512 token_bytes=312328056 weight_movement_bytes=56623104 total_nvlink_bytes=368951160 weight_to_token_ratio=0.181294 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037723369e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=158138368 dispatch_meta_bytes=229888 combine_payload_bytes=158138368 prefetch_bytes=0 reduce_bytes=0 egress_bytes=158368256 ingress_bytes=158138368 link_bytes=158368256 token_bytes=316506624 weight_movement_bytes=0 total_nvlink_bytes=316506624 weight_to_token_ratio=0.000000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.517406707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38294 dup_rows=21866 dup_rows_remote=19202 dedup_ratio=0.333649 router_tokens_per_expert=[264, 249, 248, 289, 241, 255, 245, 266, 253, 279, 251, 257, 259, 275, 243, 260, 246, 253, 267, 270, 253, 276, 264, 257, 268, 272, 245, 254, 248, 274, 235, 258, 255, 268, 264, 234, 243, 268, 242, 268, 263, 246, 245, 246, 277, 236, 262, 265, 243, 241, 272, 253, 269, 249, 237, 257, 238, 251, 267, 270, 241, 243, 237, 252, 248, 242, 258, 247, 253, 249, 246, 258, 299, 254, 267, 262, 245, 263, 254, 291, 233, 241, 260, 268, 237, 266, 243, 288, 273, 243, 243, 244, 277, 265, 254, 250, 282, 272, 257, 302, 236, 259, 235, 252, 244, 264, 273, 235, 231, 262, 267, 240, 289, 259, 253, 267, 238, 243, 270, 284, 232, 237, 223, 276, 231, 265, 246, 247] payload_rows_per_dst=[5422, 5522, 5454, 5376, 5514, 5446, 5452, 5484] dup_rows_per_dst=[2846, 2758, 2710, 2664, 2758, 2724, 2770, 2636] combine_rows_per_src=[5422, 5522, 5454, 5376, 5514, 5446, 5452, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38094 dup_rows=21842 dup_rows_remote=19024 dedup_ratio=0.333282 router_tokens_per_expert=[282, 255, 269, 246, 258, 244, 249, 246, 248, 223, 250, 268, 281, 241, 290, 253, 237, 245, 240, 237, 272, 220, 230, 246, 252, 252, 274, 255, 215, 266, 291, 270, 240, 248, 286, 246, 247, 249, 255, 265, 251, 283, 289, 274, 239, 266, 287, 263, 232, 266, 275, 239, 256, 232, 252, 236, 258, 245, 259, 264, 228, 274, 266, 258, 267, 269, 264, 257, 219, 239, 254, 255, 237, 246, 252, 268, 289, 281, 279, 265, 242, 238, 273, 260, 249, 241, 260, 237, 273, 233, 240, 251, 250, 275, 248, 239, 252, 257, 226, 244, 236, 255, 273, 258, 272, 254, 288, 264, 254, 261, 237, 245, 243, 255, 251, 272, 274, 293, 267, 264, 279, 273, 261, 273, 248, 250, 261, 245] payload_rows_per_dst=[5460, 5186, 5512, 5422, 5486, 5394, 5634, 5600] dup_rows_per_dst=[2746, 2452, 2864, 2658, 2796, 2682, 2826, 2818] combine_rows_per_src=[5460, 5186, 5512, 5422, 5486, 5394, 5634, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38416 dup_rows=21702 dup_rows_remote=19080 dedup_ratio=0.331146 router_tokens_per_expert=[259, 238, 243, 248, 232, 260, 227, 257, 262, 241, 235, 269, 253, 244, 285, 292, 242, 241, 249, 259, 263, 256, 253, 255, 244, 242, 274, 248, 275, 240, 254, 278, 262, 274, 260, 247, 253, 251, 238, 280, 249, 274, 262, 251, 257, 245, 251, 253, 263, 274, 256, 241, 275, 238, 265, 270, 246, 256, 250, 258, 247, 246, 244, 263, 242, 270, 273, 250, 242, 245, 252, 239, 251, 264, 266, 281, 276, 256, 273, 290, 248, 270, 227, 267, 253, 231, 245, 245, 227, 238, 239, 256, 287, 246, 288, 253, 278, 269, 268, 240, 263, 270, 265, 254, 269, 264, 247, 260, 248, 235, 250, 243, 243, 265, 263, 252, 287, 290, 249, 244, 262, 246, 265, 254, 263, 251, 246, 258] payload_rows_per_dst=[5412, 5430, 5486, 5518, 5526, 5418, 5526, 5518] dup_rows_per_dst=[2678, 2716, 2728, 2666, 2814, 2622, 2720, 2758] combine_rows_per_src=[5412, 5430, 5486, 5518, 5526, 5418, 5526, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43966 payload_rows_remote=38442 dup_rows=21570 dup_rows_remote=18874 dedup_ratio=0.329132 router_tokens_per_expert=[253, 281, 257, 259, 235, 242, 260, 252, 243, 274, 263, 264, 245, 243, 246, 258, 270, 255, 255, 304, 261, 250, 248, 255, 258, 240, 264, 234, 273, 239, 238, 271, 291, 254, 243, 247, 251, 242, 229, 243, 267, 264, 265, 240, 264, 264, 248, 225, 250, 273, 259, 258, 261, 241, 253, 215, 267, 232, 267, 249, 250, 254, 261, 219, 243, 267, 277, 224, 268, 272, 246, 266, 239, 269, 252, 273, 245, 239, 257, 273, 259, 251, 236, 277, 292, 274, 257, 250, 289, 241, 270, 270, 253, 246, 290, 278, 249, 245, 258, 230, 289, 280, 254, 259, 247, 242, 235, 236, 272, 275, 266, 268, 242, 270, 232, 223, 270, 246, 260, 282, 250, 249, 247, 257, 264, 252, 253, 287] payload_rows_per_dst=[5508, 5426, 5488, 5480, 5524, 5584, 5524, 5432] dup_rows_per_dst=[2642, 2804, 2586, 2538, 2696, 2882, 2686, 2736] combine_rows_per_src=[5508, 5426, 5488, 5480, 5524, 5584, 5524, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38202 dup_rows=21910 dup_rows_remote=19166 dedup_ratio=0.334320 router_tokens_per_expert=[252, 253, 242, 267, 203, 252, 275, 247, 265, 237, 277, 282, 286, 275, 225, 272, 246, 267, 234, 272, 262, 262, 248, 246, 241, 251, 283, 248, 266, 262, 262, 267, 263, 262, 240, 285, 267, 269, 259, 260, 233, 242, 277, 259, 245, 234, 259, 262, 242, 238, 261, 292, 275, 265, 249, 298, 264, 256, 257, 262, 258, 247, 239, 252, 253, 260, 238, 280, 236, 282, 249, 290, 260, 254, 248, 274, 269, 258, 237, 269, 275, 241, 253, 227, 262, 254, 265, 245, 241, 275, 243, 247, 226, 234, 267, 257, 255, 256, 276, 242, 246, 275, 236, 278, 282, 209, 288, 257, 233, 236, 252, 263, 242, 235, 263, 254, 235, 248, 237, 230, 268, 258, 261, 243, 281, 255, 241, 266] payload_rows_per_dst=[5458, 5478, 5520, 5472, 5488, 5400, 5424, 5386] dup_rows_per_dst=[2762, 2756, 2712, 2838, 2826, 2624, 2744, 2648] combine_rows_per_src=[5458, 5478, 5520, 5472, 5488, 5400, 5424, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38318 dup_rows=21704 dup_rows_remote=18924 dedup_ratio=0.331177 router_tokens_per_expert=[276, 274, 263, 243, 248, 262, 226, 264, 252, 273, 284, 267, 246, 262, 259, 267, 272, 268, 263, 265, 258, 274, 257, 256, 236, 246, 233, 285, 252, 253, 259, 270, 253, 279, 262, 227, 253, 250, 257, 267, 242, 272, 252, 274, 259, 250, 243, 274, 247, 263, 281, 277, 266, 291, 252, 288, 265, 228, 228, 228, 233, 260, 266, 276, 261, 262, 241, 231, 249, 283, 263, 279, 267, 239, 277, 224, 235, 250, 254, 250, 283, 237, 246, 261, 249, 253, 268, 258, 229, 250, 259, 266, 255, 249, 239, 241, 260, 248, 252, 258, 251, 278, 255, 232, 258, 252, 244, 230, 260, 257, 237, 239, 226, 237, 250, 238, 261, 275, 265, 230, 246, 279, 263, 284, 228, 280, 248, 263] payload_rows_per_dst=[5568, 5514, 5502, 5456, 5486, 5448, 5398, 5460] dup_rows_per_dst=[2764, 2780, 2726, 2842, 2644, 2638, 2624, 2686] combine_rows_per_src=[5568, 5514, 5502, 5456, 5486, 5448, 5398, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229984 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157082208 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934432 weight_movement_bytes=18874368 total_nvlink_bytes=332808800 weight_to_token_ratio=0.060122 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38080 dup_rows=21904 dup_rows_remote=18984 dedup_ratio=0.334229 router_tokens_per_expert=[260, 242, 245, 234, 278, 262, 268, 254, 246, 269, 219, 240, 261, 251, 264, 274, 239, 239, 235, 252, 245, 270, 241, 245, 255, 284, 246, 249, 235, 258, 259, 262, 283, 270, 269, 276, 251, 263, 259, 242, 263, 256, 273, 252, 256, 271, 235, 251, 252, 261, 235, 242, 268, 271, 268, 249, 258, 240, 256, 220, 272, 263, 263, 243, 240, 247, 268, 277, 261, 243, 259, 238, 266, 262, 278, 249, 253, 248, 269, 258, 243, 255, 249, 268, 273, 246, 293, 248, 269, 256, 298, 240, 268, 272, 250, 243, 253, 260, 260, 290, 258, 235, 241, 247, 243, 253, 242, 265, 261, 245, 246, 266, 298, 288, 266, 232, 255, 271, 257, 253, 243, 232, 271, 239, 235, 254, 239, 271] payload_rows_per_dst=[5552, 5292, 5566, 5520, 5344, 5440, 5430, 5488] dup_rows_per_dst=[2920, 2488, 2902, 2788, 2640, 2746, 2700, 2720] combine_rows_per_src=[5552, 5292, 5566, 5520, 5344, 5440, 5430, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=228472 combine_payload_bytes=156033024 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156261496 ingress_bytes=212656128 link_bytes=212656128 token_bytes=312294520 weight_movement_bytes=56623104 total_nvlink_bytes=368917624 weight_to_token_ratio=0.181313 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037566385e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229984 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581920 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933856 weight_movement_bytes=18874368 total_nvlink_bytes=333808224 weight_to_token_ratio=0.059931 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=229264 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157687696 ingress_bytes=195207168 link_bytes=195207168 token_bytes=315146128 weight_movement_bytes=37748736 total_nvlink_bytes=352894864 weight_to_token_ratio=0.119782 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870379036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229472 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156704864 ingress_bytes=213098496 link_bytes=213098496 token_bytes=313180256 weight_movement_bytes=56623104 total_nvlink_bytes=369803360 weight_to_token_ratio=0.180800 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041804938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228968 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157179496 ingress_bytes=213573632 link_bytes=213573632 token_bytes=314130024 weight_movement_bytes=56623104 total_nvlink_bytes=370753128 weight_to_token_ratio=0.180254 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046357457e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38438 dup_rows=21618 dup_rows_remote=18922 dedup_ratio=0.329865 router_tokens_per_expert=[280, 277, 280, 252, 266, 276, 252, 279, 282, 235, 261, 252, 235, 261, 243, 254, 253, 244, 261, 247, 247, 277, 241, 253, 264, 259, 257, 250, 280, 255, 212, 231, 273, 243, 237, 257, 269, 250, 252, 247, 254, 263, 250, 247, 247, 279, 272, 248, 268, 278, 232, 251, 226, 259, 260, 254, 229, 277, 271, 264, 235, 276, 246, 252, 249, 257, 262, 252, 278, 274, 241, 252, 270, 243, 260, 254, 251, 231, 272, 267, 275, 255, 274, 288, 255, 253, 279, 250, 238, 238, 257, 257, 248, 237, 262, 263, 235, 268, 273, 276, 262, 262, 234, 268, 261, 241, 243, 269, 259, 259, 250, 266, 246, 270, 246, 240, 239, 282, 236, 276, 239, 237, 251, 256, 284, 229, 244, 243] payload_rows_per_dst=[5562, 5412, 5480, 5476, 5572, 5536, 5500, 5380] dup_rows_per_dst=[2808, 2650, 2696, 2680, 2654, 2722, 2752, 2656] combine_rows_per_src=[5562, 5412, 5480, 5476, 5572, 5536, 5500, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228256 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156203936 ingress_bytes=193724416 link_bytes=193724416 token_bytes=312179616 weight_movement_bytes=37748736 total_nvlink_bytes=349928352 weight_to_token_ratio=0.120920 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856172036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229440 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671488 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113536 weight_movement_bytes=18874368 total_nvlink_bytes=333987904 weight_to_token_ratio=0.059897 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38248 dup_rows=21842 dup_rows_remote=19104 dedup_ratio=0.333282 router_tokens_per_expert=[258, 267, 265, 249, 253, 237, 248, 277, 237, 257, 249, 267, 266, 266, 244, 235, 253, 266, 281, 244, 282, 274, 242, 262, 272, 252, 257, 251, 247, 256, 243, 263, 280, 239, 251, 227, 238, 239, 243, 255, 239, 263, 282, 243, 212, 276, 245, 239, 237, 267, 240, 244, 250, 253, 246, 259, 251, 273, 245, 260, 254, 275, 268, 270, 256, 249, 242, 268, 278, 265, 235, 258, 254, 280, 262, 261, 294, 245, 250, 256, 287, 213, 295, 253, 284, 287, 283, 261, 244, 267, 238, 222, 237, 258, 260, 250, 254, 237, 266, 262, 215, 264, 257, 234, 256, 230, 269, 228, 275, 272, 257, 255, 263, 279, 291, 258, 268, 281, 241, 275, 239, 222, 259, 250, 253, 242, 286, 255] payload_rows_per_dst=[5414, 5524, 5318, 5446, 5516, 5476, 5432, 5568] dup_rows_per_dst=[2736, 2766, 2624, 2738, 2790, 2802, 2630, 2756] combine_rows_per_src=[5414, 5524, 5318, 5446, 5516, 5476, 5432, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38296 dup_rows=21780 dup_rows_remote=18976 dedup_ratio=0.332336 router_tokens_per_expert=[219, 240, 266, 249, 251, 259, 262, 281, 259, 256, 234, 277, 277, 265, 246, 258, 278, 248, 269, 279, 263, 257, 229, 246, 241, 264, 245, 262, 246, 280, 242, 286, 245, 265, 262, 246, 237, 253, 234, 243, 243, 265, 243, 256, 266, 266, 272, 242, 229, 276, 265, 254, 226, 263, 244, 257, 244, 249, 254, 276, 269, 252, 255, 237, 262, 227, 253, 241, 234, 248, 245, 256, 234, 256, 250, 245, 284, 277, 252, 258, 255, 238, 235, 280, 240, 251, 288, 246, 281, 256, 265, 236, 240, 260, 267, 300, 248, 266, 261, 276, 263, 262, 273, 245, 257, 275, 252, 259, 259, 221, 254, 283, 265, 265, 269, 243, 267, 256, 238, 276, 239, 267, 279, 245, 269, 261, 243, 250] payload_rows_per_dst=[5310, 5444, 5476, 5606, 5398, 5512, 5550, 5460] dup_rows_per_dst=[2642, 2662, 2600, 2904, 2646, 2764, 2758, 2804] combine_rows_per_src=[5310, 5444, 5476, 5606, 5398, 5512, 5550, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38234 dup_rows=21802 dup_rows_remote=19050 dedup_ratio=0.332672 router_tokens_per_expert=[263, 234, 268, 266, 275, 253, 243, 242, 255, 235, 268, 243, 261, 223, 266, 249, 251, 269, 265, 246, 241, 280, 243, 284, 249, 247, 244, 269, 253, 244, 260, 280, 247, 250, 249, 232, 256, 245, 266, 248, 245, 255, 237, 248, 273, 253, 262, 257, 245, 267, 260, 266, 230, 262, 258, 252, 247, 234, 244, 233, 251, 272, 292, 201, 251, 249, 279, 251, 277, 269, 275, 247, 264, 289, 263, 263, 256, 220, 261, 284, 236, 262, 273, 243, 247, 263, 257, 247, 271, 251, 256, 251, 267, 271, 263, 268, 248, 267, 300, 243, 270, 258, 266, 271, 231, 282, 254, 270, 252, 243, 244, 264, 281, 240, 247, 276, 241, 246, 260, 263, 245, 261, 235, 269, 248, 269, 254, 240] payload_rows_per_dst=[5438, 5560, 5400, 5358, 5526, 5500, 5514, 5438] dup_rows_per_dst=[2650, 2690, 2646, 2670, 2870, 2752, 2812, 2712] combine_rows_per_src=[5438, 5560, 5400, 5358, 5526, 5500, 5514, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38494 dup_rows=21652 dup_rows_remote=19118 dedup_ratio=0.330383 router_tokens_per_expert=[260, 262, 268, 233, 286, 242, 253, 270, 247, 276, 247, 265, 236, 246, 268, 249, 263, 263, 266, 266, 261, 248, 261, 235, 241, 243, 247, 269, 255, 259, 261, 250, 259, 254, 228, 255, 232, 256, 263, 253, 257, 260, 258, 256, 240, 248, 268, 266, 261, 288, 250, 271, 241, 251, 265, 266, 258, 262, 275, 258, 239, 246, 248, 265, 277, 244, 258, 266, 224, 250, 221, 238, 237, 252, 232, 250, 242, 245, 272, 254, 257, 256, 235, 263, 268, 263, 295, 261, 287, 275, 276, 283, 272, 266, 280, 241, 256, 252, 259, 252, 250, 254, 272, 249, 257, 268, 264, 271, 250, 263, 275, 257, 246, 226, 257, 257, 239, 264, 251, 241, 257, 243, 248, 229, 247, 259, 270, 252] payload_rows_per_dst=[5464, 5484, 5430, 5512, 5390, 5634, 5558, 5412] dup_rows_per_dst=[2752, 2692, 2676, 2776, 2534, 2922, 2740, 2560] combine_rows_per_src=[5464, 5484, 5430, 5512, 5390, 5634, 5558, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43516 payload_rows_remote=38104 dup_rows=22020 dup_rows_remote=19250 dedup_ratio=0.335999 router_tokens_per_expert=[285, 236, 247, 250, 263, 234, 263, 262, 252, 252, 262, 244, 272, 266, 243, 268, 266, 243, 266, 258, 243, 240, 232, 242, 273, 254, 272, 249, 232, 264, 267, 252, 270, 264, 241, 279, 229, 288, 254, 259, 248, 250, 271, 257, 252, 263, 247, 259, 251, 239, 241, 272, 241, 260, 275, 259, 233, 258, 239, 238, 266, 264, 272, 249, 261, 269, 258, 235, 274, 253, 246, 268, 231, 253, 247, 234, 267, 242, 251, 242, 253, 273, 282, 264, 273, 254, 240, 243, 255, 258, 260, 270, 274, 243, 238, 242, 248, 295, 246, 262, 265, 238, 262, 249, 270, 244, 242, 251, 236, 254, 266, 263, 248, 253, 230, 276, 260, 276, 284, 274, 274, 247, 258, 243, 254, 259, 267, 281] payload_rows_per_dst=[5380, 5372, 5398, 5454, 5414, 5530, 5412, 5556] dup_rows_per_dst=[2818, 2734, 2864, 2660, 2648, 2714, 2770, 2812] combine_rows_per_src=[5380, 5372, 5398, 5454, 5414, 5530, 5412, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38306 dup_rows=21630 dup_rows_remote=18848 dedup_ratio=0.330048 router_tokens_per_expert=[258, 274, 250, 251, 252, 272, 249, 274, 224, 245, 266, 239, 246, 263, 255, 262, 273, 285, 254, 260, 246, 295, 272, 256, 239, 237, 279, 224, 265, 257, 288, 261, 256, 268, 267, 243, 287, 266, 260, 235, 248, 246, 243, 274, 267, 241, 261, 255, 253, 248, 280, 267, 279, 258, 243, 244, 226, 287, 236, 266, 262, 260, 248, 242, 251, 235, 240, 246, 230, 253, 236, 263, 244, 254, 245, 263, 270, 259, 215, 252, 250, 265, 245, 236, 269, 258, 273, 247, 232, 235, 253, 259, 266, 254, 247, 226, 280, 258, 278, 254, 240, 295, 243, 284, 254, 247, 255, 261, 261, 264, 258, 248, 240, 255, 286, 278, 250, 268, 278, 258, 244, 243, 265, 262, 255, 248, 246, 254] payload_rows_per_dst=[5488, 5600, 5500, 5486, 5378, 5394, 5516, 5544] dup_rows_per_dst=[2672, 2782, 2734, 2712, 2534, 2636, 2844, 2716] combine_rows_per_src=[5488, 5600, 5500, 5486, 5378, 5394, 5516, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229408 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893216 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557024 weight_movement_bytes=18874368 total_nvlink_bytes=332431392 weight_to_token_ratio=0.060194 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38436 dup_rows=21652 dup_rows_remote=19002 dedup_ratio=0.330383 router_tokens_per_expert=[231, 265, 253, 241, 267, 267, 258, 249, 256, 268, 227, 244, 276, 275, 222, 249, 246, 251, 253, 235, 282, 282, 229, 226, 258, 264, 248, 232, 245, 241, 271, 263, 241, 218, 245, 289, 237, 257, 265, 254, 286, 250, 284, 263, 259, 257, 277, 270, 283, 264, 244, 282, 253, 251, 241, 269, 263, 260, 278, 245, 244, 254, 248, 266, 252, 254, 269, 287, 255, 254, 255, 245, 268, 248, 263, 254, 235, 263, 248, 250, 275, 228, 244, 293, 247, 264, 239, 258, 215, 260, 241, 283, 249, 272, 266, 226, 249, 284, 266, 267, 257, 246, 284, 289, 254, 245, 241, 253, 225, 237, 246, 254, 229, 234, 261, 275, 249, 252, 247, 239, 263, 256, 253, 270, 275, 300, 273, 264] payload_rows_per_dst=[5448, 5450, 5616, 5538, 5482, 5474, 5402, 5474] dup_rows_per_dst=[2650, 2724, 2838, 2818, 2652, 2728, 2558, 2684] combine_rows_per_src=[5448, 5450, 5616, 5538, 5482, 5474, 5402, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229088 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157089504 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313949920 weight_movement_bytes=37748736 total_nvlink_bytes=351698656 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229136 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156835600 ingress_bytes=213229568 link_bytes=213229568 token_bytes=313442064 weight_movement_bytes=56623104 total_nvlink_bytes=370065168 weight_to_token_ratio=0.180649 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043060805e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157671424 dispatch_meta_bytes=230448 combine_payload_bytes=157671424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157901872 ingress_bytes=176545792 link_bytes=176545792 token_bytes=315573296 weight_movement_bytes=18874368 total_nvlink_bytes=334447664 weight_to_token_ratio=0.059810 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691574913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38382 dup_rows=21790 dup_rows_remote=19048 dedup_ratio=0.332489 router_tokens_per_expert=[253, 251, 287, 252, 222, 244, 253, 236, 252, 262, 239, 281, 236, 261, 238, 252, 269, 278, 281, 261, 262, 259, 276, 246, 249, 251, 284, 261, 273, 259, 250, 248, 244, 264, 277, 255, 240, 277, 256, 260, 236, 258, 283, 243, 239, 245, 236, 240, 276, 263, 255, 284, 263, 246, 264, 257, 247, 245, 258, 250, 251, 256, 254, 281, 264, 239, 265, 236, 242, 278, 267, 266, 224, 254, 278, 245, 251, 261, 266, 253, 284, 271, 267, 272, 237, 274, 256, 257, 241, 231, 244, 263, 241, 265, 248, 231, 263, 260, 225, 270, 263, 260, 249, 248, 282, 255, 247, 264, 262, 220, 240, 276, 246, 269, 286, 218, 237, 274, 241, 263, 248, 266, 257, 264, 248, 276, 244, 247] payload_rows_per_dst=[5426, 5556, 5364, 5474, 5484, 5506, 5488, 5448] dup_rows_per_dst=[2612, 2858, 2742, 2826, 2694, 2658, 2680, 2720] combine_rows_per_src=[5426, 5556, 5364, 5474, 5484, 5506, 5488, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229416 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156303400 ingress_bytes=212697088 link_bytes=212697088 token_bytes=312377384 weight_movement_bytes=56623104 total_nvlink_bytes=369000488 weight_to_token_ratio=0.181265 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037958844e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228616 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157129992 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031368 weight_movement_bytes=37748736 total_nvlink_bytes=351780104 weight_to_token_ratio=0.120207 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=229752 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157663608 ingress_bytes=195182592 link_bytes=195182592 token_bytes=315097464 weight_movement_bytes=37748736 total_nvlink_bytes=352846200 weight_to_token_ratio=0.119800 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870143561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229720 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442392 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314655064 weight_movement_bytes=18874368 total_nvlink_bytes=333529432 weight_to_token_ratio=0.059984 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38148 dup_rows=21972 dup_rows_remote=19114 dedup_ratio=0.335266 router_tokens_per_expert=[252, 244, 260, 257, 260, 235, 266, 236, 261, 240, 253, 237, 238, 267, 267, 249, 256, 263, 249, 254, 268, 236, 228, 237, 246, 244, 243, 266, 273, 276, 253, 254, 272, 246, 262, 262, 245, 259, 225, 248, 262, 234, 243, 277, 265, 257, 273, 264, 285, 261, 268, 264, 266, 231, 245, 272, 258, 262, 273, 231, 243, 242, 242, 294, 267, 259, 255, 256, 260, 239, 295, 242, 252, 286, 268, 261, 252, 247, 272, 257, 272, 219, 259, 267, 235, 257, 287, 260, 247, 251, 244, 249, 257, 296, 261, 253, 277, 248, 253, 270, 245, 262, 253, 250, 271, 262, 274, 225, 247, 267, 255, 246, 249, 247, 245, 256, 262, 268, 254, 260, 282, 278, 250, 234, 253, 243, 246, 255] payload_rows_per_dst=[5396, 5386, 5540, 5416, 5504, 5450, 5442, 5430] dup_rows_per_dst=[2648, 2706, 2648, 2858, 2832, 2778, 2768, 2734] combine_rows_per_src=[5396, 5386, 5540, 5416, 5504, 5450, 5442, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38414 dup_rows=21706 dup_rows_remote=19000 dedup_ratio=0.331207 router_tokens_per_expert=[257, 247, 251, 256, 263, 265, 260, 222, 278, 258, 252, 278, 245, 255, 293, 266, 251, 249, 252, 270, 261, 259, 246, 254, 264, 232, 246, 254, 267, 248, 239, 271, 224, 288, 237, 270, 253, 264, 275, 264, 257, 235, 273, 277, 285, 246, 269, 265, 239, 252, 241, 238, 267, 267, 272, 274, 235, 259, 263, 248, 257, 258, 261, 283, 267, 238, 247, 268, 253, 271, 240, 255, 226, 254, 263, 263, 287, 249, 252, 262, 253, 234, 249, 259, 251, 256, 256, 265, 245, 276, 237, 251, 231, 232, 269, 239, 263, 253, 234, 253, 263, 273, 287, 257, 266, 237, 254, 251, 252, 260, 267, 245, 257, 253, 232, 218, 246, 276, 248, 259, 267, 265, 230, 260, 255, 271, 258, 255] payload_rows_per_dst=[5426, 5320, 5590, 5474, 5548, 5548, 5508, 5416] dup_rows_per_dst=[2866, 2544, 2774, 2766, 2642, 2686, 2722, 2706] combine_rows_per_src=[5426, 5320, 5590, 5474, 5548, 5548, 5508, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38288 dup_rows=21720 dup_rows_remote=18960 dedup_ratio=0.331421 router_tokens_per_expert=[250, 261, 262, 252, 259, 234, 282, 270, 272, 278, 252, 278, 255, 244, 232, 244, 232, 261, 263, 236, 254, 259, 244, 267, 257, 231, 280, 257, 262, 265, 262, 235, 238, 247, 253, 241, 255, 275, 224, 231, 242, 270, 280, 225, 268, 242, 257, 232, 233, 276, 246, 257, 233, 236, 245, 277, 250, 273, 241, 279, 263, 269, 239, 252, 249, 255, 254, 279, 230, 260, 261, 264, 260, 262, 225, 270, 251, 277, 260, 259, 265, 233, 273, 257, 268, 249, 268, 239, 252, 248, 257, 258, 256, 264, 281, 276, 249, 266, 250, 257, 279, 262, 276, 252, 248, 295, 246, 262, 219, 275, 243, 271, 231, 246, 269, 251, 264, 289, 271, 247, 242, 270, 263, 264, 234, 272, 244, 262] payload_rows_per_dst=[5488, 5390, 5398, 5450, 5532, 5528, 5544, 5486] dup_rows_per_dst=[2762, 2740, 2562, 2688, 2700, 2760, 2756, 2752] combine_rows_per_src=[5488, 5390, 5398, 5450, 5532, 5528, 5544, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38274 dup_rows=21782 dup_rows_remote=19098 dedup_ratio=0.332367 router_tokens_per_expert=[238, 249, 282, 249, 244, 259, 272, 261, 246, 257, 257, 276, 232, 265, 266, 245, 282, 248, 237, 254, 263, 242, 245, 271, 272, 288, 256, 254, 259, 236, 273, 267, 257, 281, 242, 254, 247, 240, 274, 244, 223, 248, 245, 254, 288, 242, 288, 263, 244, 248, 258, 284, 241, 268, 276, 253, 255, 249, 263, 248, 249, 256, 250, 267, 250, 270, 244, 246, 238, 269, 267, 260, 234, 249, 244, 271, 264, 262, 256, 258, 273, 251, 254, 263, 248, 264, 243, 236, 237, 251, 275, 244, 257, 252, 246, 234, 255, 265, 263, 230, 250, 279, 250, 254, 233, 250, 263, 277, 245, 264, 255, 267, 253, 253, 229, 277, 268, 280, 257, 259, 254, 280, 247, 242, 261, 249, 240, 265] payload_rows_per_dst=[5486, 5508, 5424, 5472, 5480, 5404, 5482, 5498] dup_rows_per_dst=[2710, 2786, 2756, 2746, 2684, 2652, 2718, 2730] combine_rows_per_src=[5486, 5508, 5424, 5472, 5480, 5404, 5482, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229048 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156483256 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312737464 weight_movement_bytes=37748736 total_nvlink_bytes=350486200 weight_to_token_ratio=0.120704 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38328 dup_rows=21724 dup_rows_remote=18954 dedup_ratio=0.331482 router_tokens_per_expert=[268, 235, 264, 245, 273, 230, 271, 259, 251, 258, 274, 260, 284, 263, 252, 219, 265, 254, 271, 261, 247, 280, 279, 271, 245, 270, 261, 281, 246, 263, 260, 242, 258, 257, 265, 237, 258, 261, 246, 256, 257, 257, 249, 263, 249, 277, 242, 243, 259, 255, 259, 259, 278, 262, 258, 249, 249, 242, 243, 253, 259, 233, 246, 272, 272, 265, 263, 243, 253, 228, 254, 221, 268, 268, 256, 252, 246, 260, 264, 239, 259, 249, 226, 277, 255, 261, 266, 266, 239, 263, 251, 275, 263, 248, 265, 240, 241, 224, 267, 253, 257, 250, 264, 276, 300, 262, 268, 264, 242, 245, 239, 275, 237, 246, 246, 267, 238, 234, 259, 255, 250, 280, 267, 268, 259, 268, 231, 228] payload_rows_per_dst=[5534, 5560, 5478, 5454, 5428, 5482, 5484, 5392] dup_rows_per_dst=[2678, 2832, 2672, 2698, 2676, 2724, 2770, 2674] combine_rows_per_src=[5534, 5560, 5478, 5454, 5428, 5482, 5484, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38114 dup_rows=21892 dup_rows_remote=19136 dedup_ratio=0.334045 router_tokens_per_expert=[251, 283, 251, 266, 261, 241, 261, 249, 247, 258, 252, 258, 251, 250, 225, 240, 259, 252, 279, 264, 244, 298, 247, 251, 243, 247, 240, 265, 261, 279, 242, 272, 252, 265, 256, 231, 261, 237, 238, 269, 255, 262, 273, 240, 259, 268, 264, 241, 266, 252, 249, 287, 259, 249, 241, 246, 240, 291, 233, 243, 251, 290, 248, 228, 250, 257, 254, 272, 268, 266, 245, 229, 299, 230, 239, 243, 262, 239, 251, 234, 261, 246, 263, 262, 293, 259, 295, 246, 240, 267, 282, 254, 275, 261, 273, 266, 251, 266, 242, 252, 252, 255, 237, 253, 267, 268, 215, 255, 255, 235, 250, 239, 250, 255, 255, 278, 257, 266, 281, 276, 267, 256, 242, 228, 258, 260, 258, 277] payload_rows_per_dst=[5426, 5530, 5420, 5400, 5412, 5564, 5396, 5496] dup_rows_per_dst=[2662, 2756, 2722, 2746, 2664, 2922, 2588, 2832] combine_rows_per_src=[5426, 5530, 5420, 5400, 5412, 5564, 5396, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229656 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157573400 ingress_bytes=213966848 link_bytes=213966848 token_bytes=314917144 weight_movement_bytes=56623104 total_nvlink_bytes=371540248 weight_to_token_ratio=0.179803 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050125059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38116 dup_rows=21794 dup_rows_remote=18992 dedup_ratio=0.332550 router_tokens_per_expert=[248, 271, 229, 256, 262, 250, 255, 260, 250, 255, 260, 254, 271, 258, 241, 235, 282, 275, 231, 272, 265, 243, 279, 234, 269, 262, 263, 200, 251, 230, 252, 266, 260, 245, 261, 212, 255, 254, 253, 278, 241, 251, 287, 257, 260, 248, 271, 236, 277, 252, 228, 266, 239, 269, 250, 248, 248, 220, 248, 241, 254, 278, 271, 271, 255, 275, 246, 279, 251, 256, 266, 275, 238, 249, 238, 254, 229, 242, 250, 250, 233, 274, 261, 263, 253, 257, 260, 230, 266, 234, 279, 232, 232, 245, 280, 253, 248, 247, 277, 260, 244, 251, 274, 268, 257, 244, 258, 255, 256, 281, 257, 289, 253, 256, 293, 268, 270, 247, 276, 291, 244, 250, 268, 264, 280, 272, 247, 260] payload_rows_per_dst=[5626, 5582, 5464, 5632, 5354, 5442, 5226, 5416] dup_rows_per_dst=[2802, 2846, 2728, 2844, 2752, 2662, 2508, 2652] combine_rows_per_src=[5626, 5582, 5464, 5632, 5354, 5442, 5226, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38220 dup_rows=21786 dup_rows_remote=18956 dedup_ratio=0.332428 router_tokens_per_expert=[272, 253, 243, 257, 252, 248, 241, 225, 243, 266, 239, 247, 268, 267, 256, 254, 261, 241, 275, 250, 240, 257, 254, 251, 255, 234, 225, 245, 245, 240, 258, 236, 264, 252, 259, 275, 262, 265, 245, 251, 262, 276, 223, 288, 270, 278, 254, 256, 228, 280, 278, 266, 258, 264, 283, 240, 245, 274, 250, 242, 301, 256, 230, 242, 273, 269, 275, 253, 236, 264, 253, 278, 251, 265, 278, 277, 235, 230, 267, 264, 259, 234, 269, 219, 261, 240, 256, 244, 271, 244, 264, 239, 285, 272, 262, 259, 240, 273, 254, 264, 220, 264, 266, 265, 257, 239, 245, 256, 262, 284, 256, 275, 250, 260, 249, 257, 223, 271, 293, 254, 258, 257, 251, 241, 250, 240, 261, 272] payload_rows_per_dst=[5424, 5294, 5530, 5546, 5536, 5464, 5498, 5458] dup_rows_per_dst=[2638, 2640, 2830, 2728, 2800, 2692, 2742, 2716] combine_rows_per_src=[5424, 5294, 5530, 5546, 5536, 5464, 5498, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228992 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056640 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884288 weight_movement_bytes=18874368 total_nvlink_bytes=332758656 weight_to_token_ratio=0.060132 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229488 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156999792 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770096 weight_movement_bytes=37748736 total_nvlink_bytes=351518832 weight_to_token_ratio=0.120307 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229128 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220616 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212104 weight_movement_bytes=18874368 total_nvlink_bytes=333086472 weight_to_token_ratio=0.060069 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=229000 combine_payload_bytes=156114944 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156343944 ingress_bytes=231612416 link_bytes=231612416 token_bytes=312458888 weight_movement_bytes=75497472 total_nvlink_bytes=387956360 weight_to_token_ratio=0.241624 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219196210e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=228432 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156351568 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312474704 weight_movement_bytes=18874368 total_nvlink_bytes=331349072 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228704 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156777824 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313326944 weight_movement_bytes=37748736 total_nvlink_bytes=351075680 weight_to_token_ratio=0.120477 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38278 dup_rows=21866 dup_rows_remote=19170 dedup_ratio=0.333649 router_tokens_per_expert=[265, 230, 270, 270, 249, 280, 234, 272, 247, 263, 275, 228, 249, 236, 289, 256, 249, 278, 252, 280, 253, 258, 251, 253, 245, 255, 248, 268, 234, 243, 245, 276, 291, 252, 245, 258, 274, 294, 272, 266, 259, 293, 244, 254, 246, 259, 261, 258, 280, 232, 267, 240, 279, 238, 271, 233, 247, 248, 262, 243, 237, 270, 242, 255, 246, 264, 261, 250, 243, 248, 263, 243, 235, 260, 273, 267, 278, 252, 251, 258, 239, 239, 242, 236, 244, 264, 267, 252, 233, 227, 265, 252, 265, 251, 268, 258, 276, 243, 274, 260, 243, 251, 215, 240, 276, 245, 254, 285, 246, 238, 255, 249, 264, 254, 247, 272, 278, 232, 237, 274, 261, 270, 268, 247, 241, 278, 265, 265] payload_rows_per_dst=[5430, 5414, 5620, 5392, 5458, 5382, 5460, 5514] dup_rows_per_dst=[2796, 2762, 2832, 2696, 2726, 2622, 2640, 2792] combine_rows_per_src=[5430, 5414, 5620, 5392, 5458, 5382, 5460, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38130 dup_rows=21776 dup_rows_remote=18960 dedup_ratio=0.332275 router_tokens_per_expert=[283, 285, 256, 238, 261, 278, 241, 252, 243, 264, 255, 271, 269, 277, 297, 271, 270, 232, 258, 265, 279, 246, 263, 259, 276, 268, 268, 268, 242, 259, 239, 244, 266, 266, 247, 269, 226, 263, 290, 244, 251, 268, 257, 242, 268, 229, 239, 253, 246, 263, 247, 266, 257, 251, 222, 249, 248, 230, 264, 232, 269, 258, 258, 253, 233, 247, 247, 264, 266, 252, 259, 252, 260, 235, 242, 244, 278, 250, 212, 238, 243, 248, 258, 274, 266, 252, 240, 286, 238, 239, 273, 267, 242, 247, 270, 252, 273, 283, 281, 270, 258, 241, 259, 255, 239, 257, 257, 263, 265, 256, 261, 241, 267, 251, 276, 249, 249, 241, 263, 248, 271, 278, 230, 273, 261, 235, 252, 223] payload_rows_per_dst=[5546, 5458, 5436, 5346, 5394, 5510, 5440, 5630] dup_rows_per_dst=[2846, 2814, 2594, 2682, 2772, 2646, 2606, 2816] combine_rows_per_src=[5546, 5458, 5436, 5346, 5394, 5510, 5440, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38242 dup_rows=21820 dup_rows_remote=19152 dedup_ratio=0.332947 router_tokens_per_expert=[238, 261, 256, 253, 250, 249, 281, 246, 262, 281, 239, 233, 246, 248, 249, 251, 279, 273, 272, 276, 222, 268, 236, 254, 262, 252, 256, 262, 258, 220, 295, 256, 260, 212, 266, 260, 249, 252, 260, 244, 269, 285, 255, 237, 244, 241, 282, 237, 253, 275, 261, 267, 279, 260, 274, 237, 236, 278, 233, 253, 245, 253, 261, 265, 240, 261, 262, 289, 237, 233, 255, 248, 265, 253, 258, 265, 252, 263, 261, 267, 257, 275, 277, 263, 272, 245, 268, 245, 218, 234, 231, 265, 243, 262, 263, 253, 226, 280, 272, 249, 251, 274, 257, 228, 291, 267, 253, 280, 227, 282, 272, 261, 247, 260, 264, 261, 245, 265, 235, 247, 265, 267, 228, 243, 261, 259, 258, 246] payload_rows_per_dst=[5434, 5486, 5384, 5502, 5466, 5474, 5522, 5448] dup_rows_per_dst=[2652, 2796, 2722, 2758, 2752, 2668, 2818, 2654] combine_rows_per_src=[5434, 5486, 5384, 5502, 5466, 5474, 5522, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38130 dup_rows=21962 dup_rows_remote=19252 dedup_ratio=0.335114 router_tokens_per_expert=[239, 279, 256, 251, 243, 281, 241, 265, 241, 281, 272, 243, 260, 244, 255, 247, 239, 285, 244, 249, 234, 249, 251, 261, 287, 239, 280, 252, 249, 270, 267, 242, 260, 283, 249, 241, 269, 245, 254, 265, 250, 246, 240, 252, 270, 247, 262, 249, 237, 285, 247, 236, 266, 269, 246, 251, 256, 242, 258, 269, 267, 246, 275, 259, 266, 252, 260, 265, 250, 270, 258, 267, 250, 237, 266, 254, 259, 262, 228, 233, 276, 263, 234, 246, 258, 254, 257, 250, 250, 282, 263, 294, 234, 282, 238, 244, 239, 239, 250, 267, 250, 247, 272, 263, 249, 241, 251, 264, 261, 262, 239, 248, 265, 277, 268, 260, 259, 263, 267, 266, 274, 260, 267, 212, 256, 239, 241, 263] payload_rows_per_dst=[5478, 5458, 5456, 5410, 5444, 5506, 5398, 5424] dup_rows_per_dst=[2718, 2738, 2708, 2808, 2710, 2744, 2686, 2850] combine_rows_per_src=[5478, 5458, 5456, 5410, 5444, 5506, 5398, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229792 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016480 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803168 weight_movement_bytes=37748736 total_nvlink_bytes=351551904 weight_to_token_ratio=0.120294 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38388 dup_rows=21712 dup_rows_remote=19050 dedup_ratio=0.331299 router_tokens_per_expert=[262, 289, 257, 266, 256, 263, 257, 281, 263, 265, 238, 224, 262, 248, 260, 249, 270, 274, 265, 243, 245, 241, 262, 239, 268, 262, 261, 234, 263, 261, 270, 246, 249, 240, 252, 247, 267, 225, 257, 280, 278, 283, 224, 221, 258, 250, 251, 264, 273, 266, 246, 229, 258, 249, 249, 262, 275, 225, 260, 259, 270, 228, 249, 271, 219, 243, 238, 219, 270, 268, 263, 266, 279, 251, 238, 249, 228, 263, 248, 252, 258, 286, 289, 247, 254, 266, 274, 263, 255, 254, 263, 244, 245, 262, 252, 262, 267, 228, 223, 249, 257, 274, 273, 273, 246, 262, 253, 253, 254, 241, 240, 256, 255, 267, 288, 248, 252, 264, 239, 262, 253, 261, 283, 294, 247, 266, 237, 276] payload_rows_per_dst=[5524, 5598, 5410, 5452, 5360, 5492, 5436, 5552] dup_rows_per_dst=[2756, 2610, 2682, 2686, 2628, 2856, 2662, 2832] combine_rows_per_src=[5524, 5598, 5410, 5452, 5360, 5492, 5436, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38166 dup_rows=21842 dup_rows_remote=19140 dedup_ratio=0.333282 router_tokens_per_expert=[276, 273, 259, 268, 270, 295, 255, 246, 274, 239, 271, 249, 216, 269, 252, 225, 255, 253, 274, 243, 260, 275, 239, 275, 280, 245, 254, 228, 270, 258, 234, 272, 265, 256, 239, 257, 259, 247, 265, 267, 279, 258, 238, 248, 252, 235, 253, 248, 235, 271, 269, 264, 263, 258, 242, 271, 255, 252, 235, 269, 261, 257, 257, 267, 247, 252, 221, 222, 249, 264, 240, 264, 242, 232, 236, 269, 224, 272, 292, 272, 252, 258, 280, 239, 274, 253, 250, 245, 252, 248, 257, 263, 246, 276, 263, 242, 245, 244, 253, 248, 249, 250, 231, 259, 277, 268, 259, 264, 259, 271, 266, 268, 247, 283, 254, 278, 229, 257, 265, 259, 283, 257, 264, 251, 230, 247, 260, 253] payload_rows_per_dst=[5468, 5528, 5424, 5458, 5368, 5394, 5504, 5550] dup_rows_per_dst=[2806, 2702, 2708, 2794, 2628, 2802, 2718, 2684] combine_rows_per_src=[5468, 5528, 5424, 5458, 5368, 5394, 5504, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=228360 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156408840 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312589320 weight_movement_bytes=18874368 total_nvlink_bytes=331463688 weight_to_token_ratio=0.060381 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38316 dup_rows=21754 dup_rows_remote=19022 dedup_ratio=0.331940 router_tokens_per_expert=[279, 253, 229, 261, 235, 274, 210, 261, 271, 246, 264, 253, 238, 244, 270, 287, 235, 270, 277, 259, 270, 270, 252, 270, 253, 255, 249, 240, 250, 277, 277, 266, 249, 225, 250, 235, 275, 239, 287, 248, 247, 285, 239, 264, 270, 268, 244, 274, 243, 260, 236, 272, 256, 240, 242, 255, 250, 249, 281, 247, 281, 258, 251, 251, 261, 249, 249, 250, 272, 245, 258, 241, 251, 254, 227, 264, 272, 276, 266, 261, 258, 249, 240, 234, 273, 231, 260, 285, 262, 284, 292, 246, 285, 268, 227, 261, 262, 238, 260, 282, 233, 257, 245, 255, 266, 256, 262, 234, 240, 250, 281, 279, 271, 282, 262, 241, 228, 237, 235, 252, 247, 262, 262, 259, 237, 243, 251, 232] payload_rows_per_dst=[5452, 5492, 5466, 5472, 5446, 5558, 5480, 5416] dup_rows_per_dst=[2698, 2848, 2732, 2672, 2746, 2752, 2720, 2586] combine_rows_per_src=[5452, 5492, 5466, 5472, 5446, 5558, 5480, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38092 dup_rows=21940 dup_rows_remote=19146 dedup_ratio=0.334778 router_tokens_per_expert=[241, 253, 273, 243, 229, 271, 246, 260, 261, 275, 239, 279, 244, 258, 266, 250, 282, 257, 273, 273, 220, 244, 252, 286, 261, 261, 234, 243, 252, 265, 247, 233, 263, 289, 268, 271, 251, 263, 249, 244, 248, 259, 244, 259, 252, 246, 226, 257, 222, 268, 265, 263, 267, 239, 257, 269, 231, 264, 262, 277, 226, 254, 237, 251, 265, 255, 240, 272, 254, 288, 233, 226, 251, 276, 260, 257, 269, 261, 289, 252, 269, 238, 262, 256, 274, 227, 264, 251, 274, 282, 262, 257, 249, 264, 233, 257, 256, 241, 264, 263, 234, 251, 249, 256, 265, 247, 269, 253, 235, 252, 268, 271, 256, 278, 298, 247, 280, 231, 240, 254, 256, 269, 242, 265, 259, 234, 266, 240] payload_rows_per_dst=[5504, 5452, 5466, 5432, 5396, 5508, 5420, 5418] dup_rows_per_dst=[2794, 2726, 2712, 2878, 2716, 2730, 2728, 2656] combine_rows_per_src=[5504, 5452, 5466, 5432, 5396, 5508, 5420, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229576 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156868808 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313508040 weight_movement_bytes=37748736 total_nvlink_bytes=351256776 weight_to_token_ratio=0.120408 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229528 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156410008 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312590488 weight_movement_bytes=18874368 total_nvlink_bytes=331464856 weight_to_token_ratio=0.060380 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229752 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157467000 ingress_bytes=194985984 link_bytes=194985984 token_bytes=314704248 weight_movement_bytes=37748736 total_nvlink_bytes=352452984 weight_to_token_ratio=0.119950 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868259760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=229224 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156557160 ingress_bytes=175202304 link_bytes=175202304 token_bytes=312885096 weight_movement_bytes=18874368 total_nvlink_bytes=331759464 weight_to_token_ratio=0.060324 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678702272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229352 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171688 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114024 weight_movement_bytes=18874368 total_nvlink_bytes=332988392 weight_to_token_ratio=0.060088 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228952 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253784 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312278616 weight_movement_bytes=18874368 total_nvlink_bytes=331152984 weight_to_token_ratio=0.060441 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38204 dup_rows=21974 dup_rows_remote=19128 dedup_ratio=0.335297 router_tokens_per_expert=[272, 245, 238, 238, 238, 275, 285, 263, 275, 273, 248, 254, 238, 265, 251, 248, 262, 246, 262, 235, 264, 260, 252, 265, 254, 254, 247, 239, 239, 249, 240, 234, 261, 250, 226, 255, 251, 265, 253, 258, 234, 240, 251, 242, 278, 266, 285, 241, 248, 276, 253, 271, 230, 251, 267, 250, 252, 255, 270, 273, 259, 246, 252, 249, 267, 268, 277, 252, 258, 264, 249, 260, 247, 249, 243, 221, 240, 249, 246, 253, 261, 231, 267, 239, 232, 254, 291, 244, 235, 246, 271, 278, 259, 263, 258, 260, 278, 272, 256, 259, 288, 278, 234, 278, 245, 230, 252, 260, 257, 245, 258, 258, 258, 267, 249, 276, 266, 268, 270, 243, 259, 278, 282, 259, 281, 249, 255, 262] payload_rows_per_dst=[5472, 5446, 5394, 5358, 5374, 5450, 5536, 5532] dup_rows_per_dst=[2740, 2558, 2718, 2846, 2712, 2728, 2760, 2912] combine_rows_per_src=[5472, 5446, 5394, 5358, 5374, 5450, 5536, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38176 dup_rows=21844 dup_rows_remote=19100 dedup_ratio=0.333313 router_tokens_per_expert=[263, 234, 262, 262, 242, 255, 255, 260, 245, 290, 250, 231, 281, 274, 250, 250, 268, 242, 262, 263, 254, 267, 218, 254, 295, 255, 245, 253, 251, 274, 244, 286, 254, 270, 239, 258, 234, 221, 278, 280, 259, 252, 266, 259, 237, 279, 258, 239, 271, 270, 283, 225, 245, 279, 253, 273, 242, 246, 242, 256, 262, 267, 238, 268, 274, 269, 244, 251, 262, 276, 262, 243, 251, 227, 248, 258, 239, 264, 247, 276, 253, 235, 256, 256, 251, 279, 248, 255, 245, 232, 227, 256, 263, 253, 253, 263, 245, 253, 279, 237, 230, 235, 233, 257, 287, 251, 272, 247, 260, 269, 255, 274, 253, 236, 269, 263, 259, 250, 266, 275, 281, 282, 248, 227, 263, 240, 266, 252] payload_rows_per_dst=[5252, 5300, 5630, 5496, 5464, 5572, 5462, 5516] dup_rows_per_dst=[2482, 2550, 3010, 2744, 2718, 2890, 2706, 2744] combine_rows_per_src=[5252, 5300, 5630, 5496, 5464, 5572, 5462, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229328 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156712912 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196496 weight_movement_bytes=18874368 total_nvlink_bytes=332070864 weight_to_token_ratio=0.060264 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38372 dup_rows=21656 dup_rows_remote=18922 dedup_ratio=0.330444 router_tokens_per_expert=[246, 241, 230, 242, 253, 252, 293, 273, 272, 248, 272, 250, 236, 239, 270, 254, 268, 251, 260, 246, 238, 261, 282, 260, 268, 235, 248, 251, 262, 224, 261, 263, 247, 265, 238, 236, 260, 231, 251, 262, 250, 257, 266, 247, 237, 269, 258, 226, 282, 270, 217, 257, 272, 260, 247, 245, 260, 259, 281, 257, 259, 225, 264, 246, 248, 243, 252, 251, 254, 281, 257, 255, 267, 266, 272, 265, 265, 256, 242, 270, 249, 263, 261, 256, 235, 263, 249, 289, 270, 235, 266, 245, 254, 271, 265, 250, 248, 233, 265, 280, 285, 275, 276, 266, 263, 203, 245, 262, 260, 254, 253, 254, 257, 254, 279, 276, 251, 237, 266, 252, 254, 236, 261, 269, 274, 250, 243, 272] payload_rows_per_dst=[5502, 5426, 5418, 5488, 5488, 5508, 5558, 5492] dup_rows_per_dst=[2640, 2730, 2582, 2714, 2800, 2734, 2686, 2770] combine_rows_per_src=[5502, 5426, 5418, 5488, 5488, 5508, 5558, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38382 dup_rows=21648 dup_rows_remote=18974 dedup_ratio=0.330322 router_tokens_per_expert=[254, 269, 252, 256, 241, 279, 252, 258, 256, 250, 293, 228, 263, 253, 276, 282, 239, 262, 246, 224, 249, 266, 247, 253, 280, 253, 265, 277, 271, 257, 254, 251, 240, 251, 278, 216, 273, 232, 282, 242, 236, 259, 269, 255, 248, 241, 262, 258, 267, 252, 254, 268, 256, 245, 252, 284, 272, 263, 252, 224, 255, 265, 297, 274, 241, 261, 263, 242, 254, 252, 249, 250, 260, 257, 254, 265, 249, 265, 249, 279, 244, 247, 248, 270, 249, 236, 248, 267, 268, 241, 245, 249, 237, 261, 267, 247, 295, 261, 275, 267, 252, 233, 252, 257, 245, 270, 267, 232, 245, 267, 263, 248, 234, 248, 281, 259, 230, 245, 231, 254, 250, 233, 247, 247, 290, 270, 293, 235] payload_rows_per_dst=[5528, 5528, 5416, 5586, 5506, 5432, 5436, 5456] dup_rows_per_dst=[2796, 2660, 2668, 2774, 2674, 2616, 2822, 2638] combine_rows_per_src=[5528, 5528, 5416, 5586, 5506, 5432, 5436, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38308 dup_rows=21746 dup_rows_remote=19012 dedup_ratio=0.331818 router_tokens_per_expert=[255, 259, 263, 261, 260, 263, 235, 249, 260, 257, 214, 239, 257, 262, 255, 255, 262, 278, 256, 265, 258, 241, 282, 246, 267, 256, 266, 279, 228, 248, 270, 241, 243, 237, 232, 253, 272, 255, 250, 278, 258, 242, 274, 275, 270, 251, 248, 241, 256, 222, 275, 247, 261, 232, 247, 270, 277, 282, 251, 240, 248, 253, 266, 242, 261, 259, 251, 257, 283, 278, 260, 245, 258, 261, 268, 257, 289, 278, 233, 248, 254, 269, 233, 264, 251, 240, 258, 245, 229, 263, 254, 243, 250, 256, 244, 247, 246, 250, 251, 247, 256, 251, 275, 263, 273, 285, 237, 265, 223, 274, 258, 254, 270, 262, 262, 283, 256, 251, 248, 254, 257, 254, 229, 265, 256, 278, 289, 225] payload_rows_per_dst=[5418, 5522, 5424, 5488, 5530, 5408, 5482, 5518] dup_rows_per_dst=[2670, 2764, 2734, 2650, 2842, 2592, 2734, 2760] combine_rows_per_src=[5418, 5522, 5424, 5488, 5530, 5408, 5482, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38296 dup_rows=21776 dup_rows_remote=19068 dedup_ratio=0.332275 router_tokens_per_expert=[254, 259, 229, 257, 249, 246, 250, 236, 281, 268, 213, 254, 243, 272, 257, 233, 254, 266, 260, 281, 259, 256, 249, 232, 313, 254, 274, 292, 282, 259, 265, 217, 245, 278, 262, 273, 238, 245, 272, 258, 275, 254, 231, 255, 250, 251, 237, 262, 249, 240, 249, 247, 240, 259, 264, 235, 228, 281, 264, 237, 253, 241, 234, 269, 280, 239, 254, 267, 242, 259, 274, 282, 253, 262, 280, 240, 270, 238, 259, 248, 252, 243, 264, 250, 260, 258, 272, 278, 243, 240, 236, 263, 226, 263, 297, 256, 273, 257, 238, 260, 254, 271, 268, 266, 275, 228, 269, 253, 256, 263, 251, 269, 267, 227, 251, 255, 240, 246, 255, 292, 248, 248, 266, 261, 250, 270, 256, 247] payload_rows_per_dst=[5394, 5520, 5464, 5344, 5534, 5476, 5584, 5444] dup_rows_per_dst=[2608, 2906, 2708, 2636, 2760, 2726, 2718, 2714] combine_rows_per_src=[5394, 5520, 5464, 5344, 5534, 5476, 5584, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229104 combine_payload_bytes=156368896 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156598000 ingress_bytes=212992000 link_bytes=212992000 token_bytes=312966896 weight_movement_bytes=56623104 total_nvlink_bytes=369590000 weight_to_token_ratio=0.180924 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040784545e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38354 dup_rows=21646 dup_rows_remote=18842 dedup_ratio=0.330292 router_tokens_per_expert=[269, 272, 246, 258, 267, 255, 252, 268, 257, 262, 256, 268, 273, 269, 264, 290, 241, 251, 271, 265, 261, 230, 298, 271, 283, 259, 250, 234, 259, 254, 280, 263, 233, 249, 242, 247, 275, 264, 271, 222, 245, 272, 257, 266, 225, 239, 245, 244, 246, 240, 247, 270, 271, 260, 245, 251, 238, 246, 296, 283, 270, 259, 253, 237, 279, 266, 229, 270, 250, 253, 250, 267, 262, 236, 270, 265, 230, 257, 226, 268, 287, 229, 270, 240, 252, 254, 239, 253, 265, 262, 217, 225, 253, 258, 260, 266, 255, 258, 249, 243, 243, 266, 267, 250, 222, 305, 256, 245, 253, 252, 244, 221, 268, 290, 272, 261, 259, 257, 268, 260, 212, 269, 270, 251, 243, 257, 263, 227] payload_rows_per_dst=[5702, 5536, 5368, 5478, 5464, 5400, 5444, 5498] dup_rows_per_dst=[2750, 2804, 2624, 2746, 2692, 2660, 2614, 2756] combine_rows_per_src=[5702, 5536, 5368, 5478, 5464, 5400, 5444, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38410 dup_rows=21640 dup_rows_remote=18924 dedup_ratio=0.330200 router_tokens_per_expert=[258, 271, 250, 266, 260, 247, 248, 236, 248, 243, 260, 265, 272, 246, 247, 236, 256, 256, 230, 264, 252, 249, 252, 264, 241, 271, 234, 285, 219, 243, 263, 257, 251, 256, 250, 271, 269, 228, 274, 228, 276, 241, 250, 241, 264, 244, 284, 263, 253, 259, 234, 247, 288, 255, 250, 255, 247, 272, 245, 285, 265, 235, 260, 239, 267, 267, 256, 238, 249, 264, 257, 269, 253, 263, 244, 239, 296, 247, 282, 267, 268, 244, 273, 237, 259, 244, 242, 270, 257, 255, 289, 243, 257, 244, 255, 261, 297, 274, 257, 255, 233, 244, 228, 226, 246, 254, 252, 258, 240, 270, 266, 263, 278, 250, 272, 253, 271, 251, 301, 234, 241, 260, 287, 255, 283, 241, 248, 256] payload_rows_per_dst=[5486, 5470, 5630, 5514, 5744, 5186, 5356, 5510] dup_rows_per_dst=[2716, 2602, 3012, 2664, 3030, 2460, 2316, 2840] combine_rows_per_src=[5486, 5470, 5630, 5514, 5744, 5186, 5356, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229176 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157400888 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572600 weight_movement_bytes=18874368 total_nvlink_bytes=333446968 weight_to_token_ratio=0.060000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229424 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442096 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654768 weight_movement_bytes=18874368 total_nvlink_bytes=333529136 weight_to_token_ratio=0.059984 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229280 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157138848 ingress_bytes=175783936 link_bytes=175783936 token_bytes=314048416 weight_movement_bytes=18874368 total_nvlink_bytes=332922784 weight_to_token_ratio=0.060100 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684275184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229456 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089872 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313950288 weight_movement_bytes=18874368 total_nvlink_bytes=332824656 weight_to_token_ratio=0.060119 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=228784 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157326768 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314424752 weight_movement_bytes=37748736 total_nvlink_bytes=352173488 weight_to_token_ratio=0.120057 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38244 dup_rows=21824 dup_rows_remote=18980 dedup_ratio=0.333008 router_tokens_per_expert=[258, 247, 251, 250, 252, 257, 265, 259, 279, 241, 238, 264, 269, 263, 270, 282, 231, 280, 243, 270, 259, 284, 263, 233, 275, 222, 263, 274, 237, 221, 256, 281, 217, 222, 261, 224, 267, 229, 247, 252, 231, 264, 277, 273, 273, 257, 270, 258, 245, 279, 256, 245, 266, 261, 269, 272, 249, 254, 235, 255, 275, 232, 296, 267, 274, 245, 242, 251, 262, 254, 242, 261, 259, 247, 275, 256, 241, 262, 256, 260, 278, 254, 233, 270, 220, 239, 259, 260, 254, 244, 255, 234, 238, 241, 257, 245, 246, 254, 260, 233, 272, 225, 277, 250, 246, 245, 274, 262, 243, 283, 243, 265, 254, 264, 262, 261, 248, 290, 262, 269, 268, 253, 260, 284, 243, 235, 281, 273] payload_rows_per_dst=[5480, 5450, 5342, 5468, 5450, 5442, 5514, 5566] dup_rows_per_dst=[2810, 2734, 2702, 2844, 2724, 2520, 2642, 2848] combine_rows_per_src=[5480, 5450, 5342, 5468, 5450, 5442, 5514, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229336 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157556696 ingress_bytes=195076096 link_bytes=195076096 token_bytes=314884056 weight_movement_bytes=37748736 total_nvlink_bytes=352632792 weight_to_token_ratio=0.119881 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869123168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38462 dup_rows=21660 dup_rows_remote=18910 dedup_ratio=0.330505 router_tokens_per_expert=[239, 255, 284, 244, 245, 223, 260, 272, 238, 262, 244, 236, 282, 257, 254, 275, 273, 269, 246, 259, 258, 274, 247, 275, 268, 258, 263, 265, 260, 259, 274, 253, 258, 255, 251, 268, 271, 267, 255, 245, 238, 249, 253, 254, 254, 280, 239, 268, 260, 238, 252, 262, 257, 246, 226, 253, 280, 264, 253, 252, 259, 253, 255, 237, 278, 238, 229, 243, 245, 255, 250, 267, 247, 250, 265, 295, 271, 251, 263, 255, 247, 279, 242, 260, 250, 259, 267, 259, 252, 257, 264, 276, 242, 252, 257, 287, 247, 266, 260, 233, 250, 245, 258, 230, 240, 261, 253, 282, 252, 272, 239, 246, 247, 239, 257, 234, 278, 239, 254, 245, 270, 250, 264, 258, 254, 263, 253, 254] payload_rows_per_dst=[5510, 5290, 5536, 5590, 5622, 5530, 5384, 5414] dup_rows_per_dst=[2630, 2412, 2674, 2838, 2928, 2790, 2638, 2750] combine_rows_per_src=[5510, 5290, 5536, 5590, 5622, 5530, 5384, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=228896 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876320 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313523744 weight_movement_bytes=18874368 total_nvlink_bytes=332398112 weight_to_token_ratio=0.060201 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38368 dup_rows=21678 dup_rows_remote=18938 dedup_ratio=0.330780 router_tokens_per_expert=[267, 253, 288, 255, 265, 236, 270, 252, 278, 271, 230, 232, 259, 236, 249, 272, 247, 261, 243, 236, 249, 245, 226, 268, 265, 233, 245, 232, 260, 298, 255, 269, 244, 227, 241, 246, 251, 255, 263, 206, 239, 257, 243, 282, 260, 281, 244, 230, 271, 237, 257, 246, 281, 246, 270, 256, 260, 271, 244, 247, 241, 283, 268, 281, 237, 263, 241, 241, 232, 254, 266, 238, 262, 262, 250, 259, 240, 263, 275, 257, 237, 248, 274, 238, 263, 281, 264, 226, 247, 284, 234, 256, 267, 268, 263, 265, 284, 267, 257, 251, 260, 270, 243, 283, 256, 283, 284, 277, 269, 271, 293, 267, 264, 269, 260, 261, 253, 281, 232, 246, 244, 247, 281, 249, 245, 229, 242, 222] payload_rows_per_dst=[5446, 5386, 5430, 5544, 5446, 5490, 5648, 5468] dup_rows_per_dst=[2780, 2678, 2508, 2774, 2634, 2740, 2982, 2582] combine_rows_per_src=[5446, 5386, 5430, 5544, 5446, 5490, 5648, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38274 dup_rows=21828 dup_rows_remote=19108 dedup_ratio=0.333069 router_tokens_per_expert=[274, 259, 266, 219, 245, 254, 271, 240, 261, 244, 253, 257, 268, 267, 257, 248, 244, 263, 249, 246, 283, 274, 252, 278, 283, 273, 269, 229, 270, 255, 261, 270, 265, 257, 261, 270, 281, 255, 247, 269, 245, 240, 270, 235, 240, 256, 253, 255, 264, 269, 242, 282, 240, 262, 264, 245, 267, 245, 261, 239, 274, 226, 244, 257, 251, 249, 231, 253, 238, 244, 245, 265, 247, 300, 247, 267, 273, 262, 248, 257, 242, 246, 264, 264, 232, 280, 268, 257, 259, 244, 284, 252, 249, 259, 275, 253, 264, 235, 248, 273, 244, 230, 238, 233, 258, 241, 256, 253, 248, 267, 257, 267, 248, 240, 272, 248, 256, 274, 242, 267, 236, 229, 273, 268, 267, 246, 239, 284] payload_rows_per_dst=[5420, 5582, 5488, 5544, 5434, 5458, 5356, 5426] dup_rows_per_dst=[2746, 2816, 2710, 2618, 2720, 2798, 2668, 2752] combine_rows_per_src=[5420, 5582, 5488, 5544, 5434, 5458, 5356, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38334 dup_rows=21772 dup_rows_remote=19070 dedup_ratio=0.332214 router_tokens_per_expert=[234, 263, 288, 268, 262, 241, 264, 263, 238, 274, 239, 261, 246, 242, 274, 255, 266, 250, 256, 262, 256, 276, 245, 276, 275, 235, 289, 263, 292, 283, 254, 267, 269, 263, 257, 229, 255, 272, 254, 240, 256, 252, 258, 257, 267, 252, 236, 249, 271, 232, 257, 274, 224, 268, 259, 232, 241, 260, 260, 264, 243, 229, 249, 228, 243, 242, 244, 275, 261, 266, 231, 248, 238, 235, 284, 261, 260, 262, 249, 230, 251, 257, 235, 262, 277, 264, 253, 275, 249, 269, 245, 270, 255, 257, 247, 302, 265, 260, 242, 240, 257, 257, 285, 234, 277, 228, 269, 273, 243, 260, 254, 264, 244, 248, 272, 231, 263, 252, 259, 217, 262, 261, 255, 240, 244, 257, 273, 271] payload_rows_per_dst=[5506, 5622, 5430, 5362, 5444, 5490, 5474, 5436] dup_rows_per_dst=[2718, 2868, 2702, 2620, 2614, 2846, 2742, 2662] combine_rows_per_src=[5506, 5622, 5430, 5362, 5444, 5490, 5474, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38304 dup_rows=21778 dup_rows_remote=19036 dedup_ratio=0.332306 router_tokens_per_expert=[230, 245, 236, 268, 242, 282, 250, 263, 266, 298, 243, 273, 249, 239, 300, 261, 256, 239, 279, 259, 291, 253, 239, 226, 266, 243, 267, 255, 277, 262, 250, 252, 247, 256, 275, 246, 282, 264, 247, 246, 238, 256, 271, 247, 251, 254, 234, 266, 264, 217, 232, 273, 260, 226, 255, 249, 282, 258, 249, 265, 251, 272, 263, 256, 253, 281, 240, 254, 269, 239, 263, 237, 239, 255, 263, 241, 257, 256, 271, 249, 250, 246, 276, 253, 282, 240, 262, 266, 262, 255, 263, 272, 261, 256, 249, 257, 249, 264, 271, 236, 260, 293, 253, 266, 255, 243, 242, 238, 255, 264, 250, 259, 267, 235, 241, 248, 245, 261, 257, 269, 253, 249, 268, 263, 252, 231, 256, 247] payload_rows_per_dst=[5526, 5470, 5472, 5458, 5426, 5518, 5454, 5434] dup_rows_per_dst=[2764, 2758, 2688, 2686, 2708, 2782, 2742, 2650] combine_rows_per_src=[5526, 5470, 5472, 5458, 5426, 5518, 5454, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157540352 dispatch_meta_bytes=229488 combine_payload_bytes=157540352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157769840 ingress_bytes=176414720 link_bytes=176414720 token_bytes=315310192 weight_movement_bytes=18874368 total_nvlink_bytes=334184560 weight_to_token_ratio=0.059860 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690319046e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38210 dup_rows=21910 dup_rows_remote=19248 dedup_ratio=0.334320 router_tokens_per_expert=[264, 255, 301, 273, 254, 237, 281, 255, 265, 264, 264, 274, 273, 247, 248, 255, 246, 256, 228, 252, 291, 261, 283, 276, 230, 244, 249, 250, 240, 235, 235, 263, 247, 249, 240, 232, 255, 252, 259, 245, 243, 243, 263, 253, 297, 271, 239, 254, 253, 271, 257, 265, 287, 273, 275, 257, 259, 274, 225, 226, 241, 251, 271, 247, 263, 226, 238, 263, 254, 263, 267, 245, 248, 260, 238, 257, 256, 257, 254, 272, 283, 249, 239, 232, 295, 263, 264, 267, 250, 253, 264, 222, 268, 263, 267, 277, 252, 268, 272, 271, 265, 250, 262, 244, 226, 226, 248, 255, 248, 234, 254, 206, 233, 246, 280, 242, 266, 277, 231, 265, 263, 248, 269, 238, 271, 281, 268, 269] payload_rows_per_dst=[5568, 5416, 5448, 5496, 5446, 5474, 5338, 5440] dup_rows_per_dst=[2852, 2662, 2636, 2768, 2676, 2838, 2624, 2854] combine_rows_per_src=[5568, 5416, 5448, 5496, 5446, 5474, 5338, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229224 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157384552 ingress_bytes=213778432 link_bytes=213778432 token_bytes=314539880 weight_movement_bytes=56623104 total_nvlink_bytes=371162984 weight_to_token_ratio=0.180019 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048319750e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229528 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999832 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770136 weight_movement_bytes=18874368 total_nvlink_bytes=332644504 weight_to_token_ratio=0.060153 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38092 dup_rows=21904 dup_rows_remote=19010 dedup_ratio=0.334229 router_tokens_per_expert=[259, 232, 246, 245, 276, 252, 272, 257, 235, 271, 262, 240, 260, 236, 256, 259, 266, 269, 263, 244, 264, 274, 242, 222, 235, 260, 262, 289, 261, 270, 250, 274, 262, 258, 262, 270, 256, 263, 254, 269, 245, 246, 273, 283, 259, 271, 310, 252, 264, 245, 275, 251, 278, 256, 257, 240, 275, 243, 257, 257, 232, 250, 269, 237, 254, 249, 242, 266, 236, 271, 253, 252, 258, 244, 265, 253, 254, 255, 259, 266, 235, 266, 245, 253, 250, 244, 230, 286, 250, 244, 257, 244, 268, 261, 253, 229, 247, 247, 249, 270, 253, 240, 224, 240, 243, 263, 231, 307, 240, 284, 255, 239, 272, 246, 262, 256, 249, 272, 272, 261, 261, 274, 274, 259, 222, 256, 239, 247] payload_rows_per_dst=[5540, 5338, 5654, 5472, 5360, 5374, 5604, 5290] dup_rows_per_dst=[2894, 2634, 3030, 2700, 2658, 2656, 2772, 2560] combine_rows_per_src=[5540, 5338, 5654, 5472, 5360, 5374, 5604, 5290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229616 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245680 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261744 weight_movement_bytes=18874368 total_nvlink_bytes=333136112 weight_to_token_ratio=0.060059 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229360 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122544 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015728 weight_movement_bytes=37748736 total_nvlink_bytes=351764464 weight_to_token_ratio=0.120213 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229832 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156737992 ingress_bytes=232005632 link_bytes=232005632 token_bytes=313246152 weight_movement_bytes=75497472 total_nvlink_bytes=388743624 weight_to_token_ratio=0.241016 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222963812e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43536 payload_rows_remote=38112 dup_rows=22000 dup_rows_remote=19318 dedup_ratio=0.335693 router_tokens_per_expert=[292, 256, 258, 242, 276, 270, 249, 250, 254, 253, 240, 260, 265, 279, 248, 242, 260, 245, 256, 242, 247, 258, 254, 242, 245, 258, 255, 259, 281, 250, 271, 243, 246, 252, 277, 248, 277, 225, 258, 240, 261, 234, 278, 247, 241, 265, 245, 251, 256, 223, 246, 252, 236, 267, 256, 275, 269, 241, 268, 254, 247, 263, 248, 252, 284, 250, 241, 247, 258, 256, 272, 266, 246, 255, 293, 253, 245, 270, 252, 260, 260, 247, 227, 263, 243, 250, 263, 253, 255, 273, 269, 256, 251, 282, 255, 253, 231, 268, 251, 270, 254, 265, 250, 267, 268, 281, 247, 243, 261, 257, 249, 251, 247, 263, 223, 266, 285, 273, 271, 248, 237, 249, 252, 257, 258, 251, 257, 272] payload_rows_per_dst=[5472, 5408, 5336, 5424, 5558, 5474, 5466, 5398] dup_rows_per_dst=[2796, 2724, 2754, 2682, 2738, 2726, 2760, 2820] combine_rows_per_src=[5472, 5408, 5336, 5424, 5558, 5474, 5466, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228408 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253240 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312278072 weight_movement_bytes=18874368 total_nvlink_bytes=331152440 weight_to_token_ratio=0.060441 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38208 dup_rows=21702 dup_rows_remote=18774 dedup_ratio=0.331146 router_tokens_per_expert=[220, 247, 254, 264, 232, 235, 267, 241, 268, 263, 255, 249, 254, 257, 279, 235, 253, 281, 267, 229, 247, 256, 263, 243, 287, 272, 230, 253, 245, 243, 228, 252, 275, 239, 240, 248, 245, 249, 292, 269, 253, 239, 219, 268, 268, 260, 230, 258, 234, 274, 268, 250, 251, 278, 219, 249, 270, 260, 273, 253, 286, 280, 240, 252, 229, 252, 258, 250, 269, 265, 256, 260, 268, 274, 250, 263, 277, 245, 260, 257, 263, 269, 271, 244, 257, 253, 278, 237, 265, 283, 274, 243, 266, 241, 258, 231, 247, 255, 230, 272, 240, 274, 268, 290, 254, 283, 258, 270, 254, 266, 228, 288, 257, 247, 242, 244, 283, 241, 252, 250, 252, 283, 247, 257, 239, 257, 254, 262] payload_rows_per_dst=[5370, 5282, 5494, 5558, 5576, 5556, 5372, 5626] dup_rows_per_dst=[2416, 2438, 2890, 2970, 2788, 2710, 2562, 2928] combine_rows_per_src=[5370, 5282, 5494, 5558, 5576, 5556, 5372, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=229720 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156336472 ingress_bytes=174981120 link_bytes=174981120 token_bytes=312443224 weight_movement_bytes=18874368 total_nvlink_bytes=331317592 weight_to_token_ratio=0.060409 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676582996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38132 dup_rows=21874 dup_rows_remote=19078 dedup_ratio=0.333771 router_tokens_per_expert=[279, 260, 255, 267, 266, 291, 236, 281, 274, 246, 249, 257, 276, 264, 258, 235, 244, 280, 242, 271, 273, 263, 260, 287, 218, 270, 249, 216, 252, 245, 222, 282, 266, 255, 275, 256, 242, 236, 282, 257, 255, 278, 234, 263, 266, 252, 241, 241, 253, 277, 273, 236, 216, 258, 264, 251, 261, 256, 282, 275, 228, 253, 238, 262, 257, 270, 274, 245, 235, 241, 260, 266, 266, 232, 262, 242, 258, 268, 276, 251, 261, 268, 265, 259, 272, 264, 253, 281, 231, 237, 242, 264, 284, 260, 255, 267, 262, 241, 246, 254, 264, 278, 275, 255, 256, 239, 238, 249, 284, 244, 240, 261, 235, 241, 221, 260, 248, 249, 226, 255, 271, 265, 267, 225, 262, 288, 217, 236] payload_rows_per_dst=[5508, 5462, 5432, 5462, 5458, 5530, 5472, 5338] dup_rows_per_dst=[2880, 2686, 2766, 2704, 2748, 2796, 2700, 2594] combine_rows_per_src=[5508, 5462, 5432, 5462, 5458, 5530, 5472, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38400 dup_rows=21650 dup_rows_remote=18998 dedup_ratio=0.330353 router_tokens_per_expert=[241, 273, 236, 245, 247, 265, 272, 256, 257, 260, 258, 226, 252, 256, 296, 261, 269, 264, 258, 246, 289, 248, 273, 243, 240, 246, 264, 261, 263, 287, 243, 277, 272, 273, 250, 239, 267, 242, 266, 242, 255, 235, 258, 251, 253, 273, 250, 298, 274, 241, 269, 247, 244, 260, 254, 253, 264, 266, 239, 274, 259, 267, 249, 232, 251, 252, 268, 270, 274, 240, 239, 237, 256, 272, 246, 251, 218, 269, 268, 258, 257, 255, 262, 253, 234, 258, 264, 263, 231, 230, 240, 248, 261, 253, 263, 253, 254, 247, 265, 258, 255, 255, 280, 268, 249, 260, 272, 252, 277, 248, 254, 286, 233, 262, 228, 258, 240, 243, 275, 271, 237, 260, 243, 236, 249, 245, 262, 264] payload_rows_per_dst=[5466, 5544, 5478, 5474, 5486, 5446, 5576, 5416] dup_rows_per_dst=[2736, 2798, 2770, 2710, 2652, 2604, 2784, 2596] combine_rows_per_src=[5466, 5544, 5478, 5474, 5486, 5446, 5576, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=227928 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156727896 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313227864 weight_movement_bytes=18874368 total_nvlink_bytes=332102232 weight_to_token_ratio=0.060258 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38106 dup_rows=21860 dup_rows_remote=19056 dedup_ratio=0.333557 router_tokens_per_expert=[274, 254, 254, 254, 264, 228, 263, 246, 256, 264, 226, 257, 260, 278, 261, 258, 229, 257, 263, 271, 285, 257, 244, 272, 256, 230, 281, 257, 264, 262, 251, 233, 284, 242, 270, 247, 272, 243, 227, 264, 245, 260, 254, 241, 242, 254, 271, 245, 242, 249, 279, 252, 259, 260, 264, 251, 222, 268, 251, 242, 271, 236, 254, 256, 267, 248, 280, 283, 253, 228, 268, 242, 256, 240, 242, 246, 266, 263, 238, 249, 245, 243, 265, 283, 245, 244, 281, 264, 264, 280, 277, 227, 245, 229, 248, 263, 230, 269, 271, 266, 275, 257, 265, 241, 236, 264, 268, 284, 270, 260, 280, 251, 243, 231, 256, 242, 260, 278, 258, 257, 269, 264, 256, 255, 260, 247, 269, 238] payload_rows_per_dst=[5392, 5514, 5436, 5448, 5492, 5454, 5570, 5370] dup_rows_per_dst=[2802, 2710, 2686, 2664, 2646, 2752, 2804, 2796] combine_rows_per_src=[5392, 5514, 5436, 5448, 5492, 5454, 5570, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38222 dup_rows=21840 dup_rows_remote=19052 dedup_ratio=0.333252 router_tokens_per_expert=[272, 271, 236, 238, 261, 281, 240, 253, 272, 256, 249, 260, 276, 263, 256, 252, 239, 269, 246, 246, 258, 229, 248, 262, 271, 259, 261, 256, 285, 274, 296, 275, 264, 226, 267, 251, 260, 268, 246, 270, 203, 289, 270, 276, 239, 259, 261, 282, 257, 251, 280, 270, 252, 282, 234, 246, 254, 237, 242, 254, 276, 248, 259, 265, 226, 262, 229, 277, 264, 228, 240, 256, 256, 246, 286, 249, 260, 233, 241, 259, 288, 264, 231, 248, 251, 260, 260, 251, 255, 253, 220, 284, 260, 256, 261, 214, 272, 245, 274, 253, 286, 257, 272, 216, 255, 293, 252, 262, 264, 261, 292, 267, 207, 238, 255, 260, 239, 239, 286, 254, 216, 264, 234, 218, 256, 241, 281, 243] payload_rows_per_dst=[5568, 5476, 5474, 5458, 5414, 5412, 5538, 5356] dup_rows_per_dst=[2704, 2872, 2788, 2756, 2610, 2700, 2904, 2506] combine_rows_per_src=[5568, 5476, 5474, 5458, 5414, 5412, 5538, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38230 dup_rows=21812 dup_rows_remote=19104 dedup_ratio=0.332825 router_tokens_per_expert=[254, 235, 226, 238, 233, 256, 265, 261, 256, 259, 275, 248, 264, 260, 259, 268, 257, 255, 245, 273, 255, 263, 261, 257, 261, 281, 249, 236, 239, 247, 271, 251, 273, 243, 265, 251, 265, 252, 248, 265, 263, 257, 238, 260, 225, 249, 271, 276, 266, 279, 259, 269, 271, 242, 245, 244, 271, 260, 249, 241, 259, 257, 242, 247, 226, 257, 264, 266, 223, 270, 255, 268, 258, 241, 266, 248, 292, 240, 261, 242, 252, 265, 241, 249, 244, 266, 251, 258, 262, 259, 279, 240, 252, 242, 262, 271, 251, 267, 252, 251, 279, 285, 260, 232, 268, 245, 262, 251, 245, 273, 260, 281, 235, 261, 259, 264, 259, 240, 243, 251, 273, 230, 256, 272, 234, 284, 249, 266] payload_rows_per_dst=[5496, 5494, 5478, 5478, 5356, 5456, 5536, 5430] dup_rows_per_dst=[2618, 2708, 2724, 2724, 2798, 2730, 2788, 2722] combine_rows_per_src=[5496, 5494, 5478, 5478, 5356, 5456, 5536, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=228840 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156417512 ingress_bytes=193937408 link_bytes=193937408 token_bytes=312606184 weight_movement_bytes=37748736 total_nvlink_bytes=350354920 weight_to_token_ratio=0.120755 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858212820e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229592 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157515992 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802392 weight_movement_bytes=18874368 total_nvlink_bytes=333676760 weight_to_token_ratio=0.059956 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228648 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156310824 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393000 weight_movement_bytes=37748736 total_nvlink_bytes=350141736 weight_to_token_ratio=0.120837 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38370 dup_rows=21698 dup_rows_remote=18974 dedup_ratio=0.331085 router_tokens_per_expert=[296, 266, 244, 247, 257, 228, 289, 247, 278, 234, 261, 229, 254, 248, 260, 258, 246, 248, 245, 258, 258, 256, 240, 287, 258, 280, 252, 266, 246, 269, 254, 264, 246, 260, 251, 265, 267, 261, 290, 242, 271, 277, 252, 247, 297, 273, 238, 261, 252, 278, 249, 257, 244, 273, 255, 265, 257, 245, 257, 258, 225, 248, 252, 252, 247, 249, 253, 258, 247, 279, 257, 251, 266, 274, 258, 262, 234, 259, 260, 244, 239, 236, 269, 255, 252, 244, 281, 267, 231, 266, 252, 249, 241, 249, 264, 259, 273, 245, 238, 278, 261, 245, 270, 266, 242, 255, 237, 267, 258, 222, 258, 284, 257, 239, 225, 254, 267, 257, 273, 266, 266, 224, 255, 240, 231, 254, 271, 250] payload_rows_per_dst=[5468, 5478, 5504, 5488, 5422, 5528, 5572, 5378] dup_rows_per_dst=[2724, 2824, 2892, 2598, 2774, 2580, 2626, 2680] combine_rows_per_src=[5468, 5478, 5504, 5488, 5422, 5528, 5572, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229336 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156819416 ingress_bytes=213213184 link_bytes=213213184 token_bytes=313409496 weight_movement_bytes=56623104 total_nvlink_bytes=370032600 weight_to_token_ratio=0.180668 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042903822e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229096 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786408 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343720 weight_movement_bytes=18874368 total_nvlink_bytes=332218088 weight_to_token_ratio=0.060235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38278 dup_rows=21746 dup_rows_remote=18978 dedup_ratio=0.331818 router_tokens_per_expert=[241, 261, 241, 278, 244, 231, 231, 252, 262, 269, 259, 262, 290, 258, 256, 263, 242, 241, 253, 244, 245, 254, 272, 277, 243, 245, 257, 270, 254, 242, 245, 248, 232, 269, 254, 262, 248, 251, 258, 251, 241, 254, 280, 279, 250, 267, 261, 252, 252, 268, 274, 240, 261, 223, 237, 246, 277, 262, 234, 288, 259, 265, 292, 262, 265, 237, 266, 259, 260, 275, 242, 242, 266, 257, 242, 234, 290, 276, 246, 250, 256, 271, 255, 237, 238, 281, 253, 250, 222, 268, 266, 249, 248, 251, 255, 259, 265, 261, 251, 233, 263, 226, 242, 243, 255, 295, 254, 261, 238, 262, 268, 281, 251, 263, 267, 226, 262, 252, 244, 256, 272, 258, 258, 260, 230, 299, 287, 240] payload_rows_per_dst=[5390, 5458, 5544, 5512, 5500, 5430, 5430, 5526] dup_rows_per_dst=[2806, 2606, 2674, 2768, 2714, 2688, 2766, 2724] combine_rows_per_src=[5390, 5458, 5544, 5512, 5500, 5430, 5430, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229376 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157392896 ingress_bytes=213786624 link_bytes=213786624 token_bytes=314556416 weight_movement_bytes=56623104 total_nvlink_bytes=371179520 weight_to_token_ratio=0.180009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048398242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38300 dup_rows=21606 dup_rows_remote=18822 dedup_ratio=0.329681 router_tokens_per_expert=[264, 254, 245, 269, 248, 249, 248, 246, 246, 235, 276, 248, 265, 268, 274, 248, 283, 279, 274, 286, 231, 272, 240, 245, 242, 260, 268, 291, 267, 242, 218, 272, 242, 269, 254, 268, 254, 257, 247, 271, 229, 252, 247, 260, 237, 275, 287, 275, 238, 273, 239, 242, 251, 275, 253, 223, 255, 246, 285, 234, 260, 257, 261, 260, 265, 255, 269, 251, 246, 261, 269, 261, 253, 289, 247, 252, 260, 261, 265, 245, 208, 250, 272, 253, 256, 238, 254, 251, 275, 242, 269, 278, 261, 276, 255, 219, 236, 265, 241, 265, 249, 259, 269, 268, 282, 258, 231, 230, 246, 275, 240, 228, 246, 245, 269, 274, 259, 282, 280, 239, 264, 252, 277, 229, 232, 260, 240, 243] payload_rows_per_dst=[5106, 5588, 5470, 5502, 5576, 5596, 5462, 5630] dup_rows_per_dst=[2186, 2752, 2778, 2752, 2844, 2888, 2622, 2784] combine_rows_per_src=[5106, 5588, 5470, 5502, 5576, 5596, 5462, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229024 combine_payload_bytes=156786688 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157015712 ingress_bytes=213409792 link_bytes=213409792 token_bytes=313802400 weight_movement_bytes=56623104 total_nvlink_bytes=370425504 weight_to_token_ratio=0.180442 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044787623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38438 dup_rows=21710 dup_rows_remote=19014 dedup_ratio=0.331268 router_tokens_per_expert=[281, 252, 274, 235, 281, 270, 286, 249, 259, 297, 251, 228, 261, 282, 283, 266, 246, 224, 234, 245, 240, 262, 289, 269, 261, 241, 252, 250, 268, 277, 276, 226, 268, 264, 270, 242, 275, 240, 221, 265, 267, 260, 236, 244, 245, 283, 256, 257, 266, 240, 255, 263, 250, 279, 273, 246, 226, 230, 252, 260, 235, 250, 239, 261, 238, 234, 240, 230, 298, 259, 257, 246, 242, 268, 251, 246, 264, 280, 216, 269, 249, 242, 235, 276, 250, 243, 249, 250, 260, 263, 257, 241, 247, 249, 271, 260, 270, 277, 265, 276, 262, 246, 261, 258, 242, 246, 223, 263, 270, 253, 230, 276, 270, 269, 249, 240, 250, 275, 269, 273, 258, 272, 265, 270, 246, 243, 233, 255] payload_rows_per_dst=[5652, 5534, 5456, 5422, 5400, 5388, 5530, 5444] dup_rows_per_dst=[2858, 2586, 2730, 2628, 2676, 2696, 2706, 2830] combine_rows_per_src=[5652, 5534, 5456, 5422, 5400, 5388, 5530, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38448 dup_rows=21618 dup_rows_remote=18924 dedup_ratio=0.329865 router_tokens_per_expert=[251, 263, 223, 264, 245, 256, 273, 258, 256, 249, 252, 287, 265, 265, 275, 256, 268, 287, 251, 267, 255, 247, 265, 249, 246, 221, 236, 236, 229, 255, 244, 248, 236, 268, 302, 266, 252, 251, 260, 268, 262, 247, 239, 251, 239, 242, 247, 293, 250, 258, 249, 283, 255, 248, 286, 250, 269, 255, 255, 292, 246, 271, 250, 241, 254, 251, 249, 251, 249, 259, 266, 265, 228, 260, 275, 245, 295, 259, 253, 223, 254, 258, 259, 256, 258, 289, 253, 253, 228, 251, 254, 262, 278, 247, 255, 257, 263, 273, 268, 279, 285, 265, 233, 238, 263, 249, 238, 251, 236, 241, 261, 252, 242, 236, 282, 242, 291, 276, 258, 239, 255, 248, 245, 226, 257, 261, 241, 257] payload_rows_per_dst=[5506, 5364, 5492, 5570, 5470, 5544, 5504, 5468] dup_rows_per_dst=[2770, 2644, 2754, 2746, 2694, 2680, 2686, 2644] combine_rows_per_src=[5506, 5364, 5492, 5570, 5470, 5544, 5504, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38268 dup_rows=21804 dup_rows_remote=18962 dedup_ratio=0.332703 router_tokens_per_expert=[256, 275, 242, 254, 250, 276, 239, 274, 273, 229, 241, 264, 248, 233, 253, 250, 299, 249, 283, 241, 253, 256, 260, 228, 255, 263, 268, 273, 257, 276, 241, 247, 267, 253, 260, 279, 249, 277, 260, 272, 270, 245, 259, 254, 249, 262, 258, 250, 253, 260, 248, 276, 252, 247, 252, 258, 237, 282, 242, 225, 255, 251, 239, 258, 251, 257, 265, 253, 246, 253, 266, 264, 232, 262, 259, 250, 239, 247, 257, 261, 259, 262, 242, 266, 231, 266, 263, 250, 243, 263, 256, 247, 251, 267, 258, 270, 228, 246, 275, 277, 266, 262, 246, 282, 260, 250, 243, 272, 249, 251, 252, 294, 271, 236, 240, 253, 255, 249, 250, 266, 252, 262, 237, 247, 251, 281, 243, 261] payload_rows_per_dst=[5476, 5466, 5544, 5392, 5390, 5594, 5464, 5406] dup_rows_per_dst=[2638, 2832, 2784, 2678, 2734, 2594, 2842, 2702] combine_rows_per_src=[5476, 5466, 5544, 5392, 5390, 5594, 5464, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=228488 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157105288 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982088 weight_movement_bytes=18874368 total_nvlink_bytes=332856456 weight_to_token_ratio=0.060113 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38216 dup_rows=21728 dup_rows_remote=18906 dedup_ratio=0.331543 router_tokens_per_expert=[256, 248, 261, 235, 238, 229, 256, 277, 276, 257, 285, 253, 264, 248, 237, 240, 247, 246, 254, 272, 235, 241, 252, 286, 258, 265, 273, 227, 237, 276, 232, 260, 277, 263, 247, 265, 280, 256, 252, 245, 250, 284, 288, 263, 256, 251, 262, 268, 246, 299, 272, 264, 241, 283, 250, 246, 256, 284, 266, 278, 238, 248, 255, 260, 259, 223, 263, 253, 253, 246, 252, 208, 259, 246, 254, 289, 250, 260, 265, 268, 256, 255, 250, 258, 248, 254, 262, 252, 278, 213, 271, 235, 238, 284, 245, 259, 232, 252, 277, 241, 261, 247, 256, 244, 232, 254, 237, 251, 287, 261, 265, 243, 252, 239, 270, 248, 252, 249, 274, 259, 259, 278, 243, 286, 237, 259, 266, 237] payload_rows_per_dst=[5436, 5418, 5592, 5546, 5494, 5382, 5440, 5500] dup_rows_per_dst=[2684, 2704, 2822, 2826, 2602, 2734, 2640, 2716] combine_rows_per_src=[5436, 5418, 5592, 5546, 5494, 5382, 5440, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38198 dup_rows=21890 dup_rows_remote=19214 dedup_ratio=0.334015 router_tokens_per_expert=[243, 260, 260, 227, 260, 246, 263, 291, 241, 242, 272, 269, 269, 310, 258, 252, 259, 239, 247, 221, 253, 273, 277, 247, 249, 252, 253, 262, 269, 240, 268, 253, 251, 240, 234, 257, 236, 265, 248, 282, 246, 267, 245, 236, 259, 247, 275, 252, 247, 246, 264, 264, 240, 261, 228, 261, 242, 259, 235, 274, 270, 245, 281, 256, 254, 244, 276, 243, 257, 240, 257, 270, 252, 257, 265, 261, 277, 292, 258, 239, 244, 258, 243, 263, 261, 239, 242, 243, 242, 241, 279, 292, 263, 251, 255, 253, 274, 260, 270, 266, 256, 279, 234, 228, 245, 250, 274, 250, 249, 228, 257, 274, 240, 259, 254, 271, 280, 258, 270, 265, 269, 250, 233, 263, 251, 267, 248, 247] payload_rows_per_dst=[5450, 5448, 5406, 5500, 5514, 5374, 5444, 5510] dup_rows_per_dst=[2876, 2676, 2674, 2646, 2770, 2764, 2744, 2740] combine_rows_per_src=[5450, 5448, 5406, 5500, 5514, 5374, 5444, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229808 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671856 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113904 weight_movement_bytes=18874368 total_nvlink_bytes=333988272 weight_to_token_ratio=0.059897 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=229488 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157712496 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315195504 weight_movement_bytes=18874368 total_nvlink_bytes=334069872 weight_to_token_ratio=0.059881 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228920 combine_payload_bytes=156745728 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156974648 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720376 weight_movement_bytes=37748736 total_nvlink_bytes=351469112 weight_to_token_ratio=0.120326 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38290 dup_rows=21708 dup_rows_remote=18932 dedup_ratio=0.331238 router_tokens_per_expert=[279, 257, 243, 255, 246, 256, 273, 262, 266, 245, 238, 222, 280, 255, 273, 254, 245, 257, 241, 274, 287, 237, 254, 255, 266, 259, 266, 247, 255, 241, 248, 247, 263, 243, 282, 246, 229, 236, 273, 241, 239, 227, 269, 245, 261, 254, 262, 254, 243, 265, 229, 246, 248, 252, 282, 280, 283, 266, 216, 277, 259, 263, 260, 260, 246, 258, 283, 256, 263, 228, 276, 248, 284, 280, 238, 237, 227, 251, 242, 244, 258, 254, 269, 294, 255, 235, 246, 275, 263, 252, 268, 249, 250, 255, 244, 235, 267, 237, 246, 260, 274, 274, 290, 245, 261, 261, 252, 251, 267, 239, 235, 239, 248, 253, 307, 263, 273, 284, 257, 265, 251, 256, 268, 233, 268, 232, 241, 272] payload_rows_per_dst=[5538, 5514, 5460, 5502, 5384, 5496, 5480, 5454] dup_rows_per_dst=[2776, 2748, 2588, 2664, 2832, 2708, 2654, 2738] combine_rows_per_src=[5538, 5514, 5460, 5502, 5384, 5496, 5480, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228488 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156761224 ingress_bytes=232030208 link_bytes=232030208 token_bytes=313293960 weight_movement_bytes=75497472 total_nvlink_bytes=388791432 weight_to_token_ratio=0.240980 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223199287e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229648 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688656 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147664 weight_movement_bytes=18874368 total_nvlink_bytes=332022032 weight_to_token_ratio=0.060273 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38126 dup_rows=21954 dup_rows_remote=19186 dedup_ratio=0.334991 router_tokens_per_expert=[255, 257, 252, 263, 246, 234, 240, 257, 265, 248, 263, 259, 259, 241, 252, 260, 254, 270, 242, 274, 266, 247, 256, 262, 246, 258, 259, 275, 237, 264, 248, 261, 252, 238, 238, 262, 253, 254, 258, 262, 251, 253, 241, 246, 259, 249, 257, 255, 255, 271, 253, 249, 276, 269, 283, 243, 273, 267, 243, 241, 235, 259, 248, 247, 261, 263, 241, 260, 251, 258, 245, 234, 253, 277, 244, 251, 271, 293, 255, 262, 239, 261, 288, 245, 243, 263, 277, 262, 265, 243, 257, 268, 268, 222, 275, 243, 270, 247, 260, 279, 296, 225, 228, 230, 262, 264, 247, 269, 225, 261, 243, 241, 259, 256, 244, 258, 307, 283, 279, 223, 285, 270, 247, 239, 248, 232, 270, 273] payload_rows_per_dst=[5404, 5472, 5382, 5456, 5478, 5434, 5428, 5528] dup_rows_per_dst=[2698, 2766, 2674, 2768, 2760, 2804, 2666, 2818] combine_rows_per_src=[5404, 5472, 5382, 5456, 5478, 5434, 5428, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=228888 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157064728 ingress_bytes=194584576 link_bytes=194584576 token_bytes=313900568 weight_movement_bytes=37748736 total_nvlink_bytes=351649304 weight_to_token_ratio=0.120257 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864413666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229248 combine_payload_bytes=156164096 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156393344 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557440 weight_movement_bytes=18874368 total_nvlink_bytes=331431808 weight_to_token_ratio=0.060387 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38200 dup_rows=21796 dup_rows_remote=19006 dedup_ratio=0.332581 router_tokens_per_expert=[250, 226, 255, 256, 257, 245, 240, 258, 241, 280, 263, 271, 246, 259, 236, 254, 244, 231, 246, 253, 247, 276, 259, 247, 231, 239, 250, 286, 281, 238, 257, 261, 283, 264, 267, 282, 275, 235, 251, 256, 248, 256, 266, 233, 250, 240, 254, 262, 262, 276, 254, 260, 286, 257, 230, 272, 256, 261, 234, 264, 287, 240, 242, 242, 259, 257, 246, 255, 253, 252, 248, 267, 248, 251, 257, 241, 239, 278, 240, 251, 255, 256, 237, 257, 232, 218, 272, 239, 279, 249, 273, 277, 270, 272, 284, 245, 266, 262, 253, 229, 281, 247, 249, 240, 261, 274, 269, 260, 298, 250, 249, 252, 236, 284, 265, 261, 265, 260, 238, 247, 258, 264, 276, 261, 263, 260, 254, 251] payload_rows_per_dst=[5384, 5446, 5356, 5308, 5492, 5666, 5548, 5540] dup_rows_per_dst=[2690, 2646, 2646, 2326, 2790, 3064, 2844, 2790] combine_rows_per_src=[5384, 5446, 5356, 5308, 5492, 5666, 5548, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43512 payload_rows_remote=38188 dup_rows=22024 dup_rows_remote=19304 dedup_ratio=0.336060 router_tokens_per_expert=[245, 234, 263, 253, 265, 246, 228, 275, 281, 284, 275, 260, 241, 250, 275, 245, 250, 293, 274, 254, 260, 262, 279, 263, 274, 242, 269, 264, 261, 243, 239, 257, 271, 248, 245, 249, 242, 272, 229, 278, 264, 240, 266, 271, 264, 255, 259, 267, 264, 263, 243, 264, 254, 293, 252, 274, 233, 266, 254, 260, 254, 245, 249, 246, 252, 223, 258, 265, 259, 244, 247, 250, 263, 275, 259, 237, 244, 271, 273, 258, 263, 253, 258, 244, 241, 242, 270, 245, 235, 255, 252, 256, 228, 264, 258, 258, 286, 241, 251, 216, 256, 219, 261, 237, 270, 233, 265, 238, 256, 288, 251, 248, 268, 237, 249, 267, 243, 269, 254, 290, 243, 275, 260, 261, 273, 245, 247, 233] payload_rows_per_dst=[5444, 5526, 5474, 5482, 5454, 5324, 5346, 5462] dup_rows_per_dst=[2796, 2842, 2766, 2746, 2702, 2720, 2686, 2766] combine_rows_per_src=[5444, 5526, 5474, 5482, 5454, 5324, 5346, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38184 dup_rows=21874 dup_rows_remote=19076 dedup_ratio=0.333771 router_tokens_per_expert=[252, 256, 233, 251, 261, 263, 257, 277, 237, 278, 259, 268, 232, 274, 249, 238, 267, 277, 251, 265, 255, 255, 261, 255, 240, 266, 293, 235, 258, 254, 246, 254, 259, 263, 243, 268, 258, 275, 247, 265, 250, 290, 252, 260, 263, 286, 258, 260, 236, 244, 227, 287, 267, 254, 247, 256, 247, 259, 240, 246, 283, 269, 271, 257, 249, 267, 266, 252, 252, 278, 262, 263, 267, 263, 269, 247, 247, 239, 238, 279, 271, 244, 255, 244, 228, 229, 242, 269, 263, 247, 264, 237, 255, 244, 245, 267, 258, 273, 249, 250, 245, 262, 231, 239, 248, 307, 249, 272, 231, 247, 270, 233, 268, 247, 279, 250, 259, 254, 265, 250, 248, 252, 247, 252, 238, 229, 268, 252] payload_rows_per_dst=[5496, 5422, 5614, 5408, 5478, 5374, 5416, 5454] dup_rows_per_dst=[2674, 2842, 2780, 2772, 2798, 2634, 2712, 2662] combine_rows_per_src=[5496, 5422, 5614, 5408, 5478, 5374, 5416, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38210 dup_rows=21914 dup_rows_remote=19146 dedup_ratio=0.334381 router_tokens_per_expert=[226, 259, 289, 250, 221, 272, 242, 248, 262, 263, 275, 247, 255, 250, 281, 260, 238, 244, 265, 263, 248, 261, 264, 250, 292, 273, 259, 261, 245, 278, 270, 241, 266, 257, 251, 236, 259, 258, 261, 261, 271, 265, 239, 252, 270, 252, 229, 246, 238, 246, 271, 249, 227, 244, 258, 256, 256, 243, 270, 244, 230, 235, 253, 228, 279, 244, 254, 254, 234, 280, 266, 240, 243, 261, 261, 272, 261, 248, 268, 232, 235, 249, 246, 269, 269, 249, 247, 256, 255, 281, 256, 260, 257, 262, 245, 225, 249, 255, 252, 243, 263, 262, 243, 266, 254, 250, 258, 258, 258, 270, 257, 252, 237, 260, 273, 343, 238, 244, 264, 252, 277, 245, 266, 287, 278, 272, 259, 252] payload_rows_per_dst=[5456, 5518, 5424, 5378, 5450, 5444, 5412, 5540] dup_rows_per_dst=[2744, 2786, 2722, 2518, 2744, 2678, 2768, 2954] combine_rows_per_src=[5456, 5518, 5424, 5378, 5450, 5444, 5412, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228824 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696024 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163224 weight_movement_bytes=37748736 total_nvlink_bytes=350911960 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38340 dup_rows=21748 dup_rows_remote=19070 dedup_ratio=0.331848 router_tokens_per_expert=[261, 238, 272, 246, 251, 268, 241, 259, 226, 250, 286, 256, 262, 244, 244, 276, 260, 238, 255, 257, 238, 267, 252, 242, 263, 261, 224, 293, 245, 260, 270, 251, 244, 235, 235, 238, 238, 270, 278, 234, 249, 253, 242, 287, 283, 287, 234, 256, 290, 246, 237, 261, 242, 247, 242, 283, 256, 268, 245, 239, 256, 254, 284, 248, 253, 245, 254, 249, 249, 260, 259, 260, 260, 264, 266, 256, 244, 267, 257, 256, 237, 258, 270, 232, 278, 256, 287, 253, 276, 255, 249, 257, 244, 284, 251, 263, 267, 272, 264, 222, 260, 237, 275, 271, 230, 248, 252, 266, 251, 263, 236, 242, 279, 256, 285, 267, 244, 269, 228, 267, 265, 247, 279, 261, 255, 243, 236, 265] payload_rows_per_dst=[5520, 5422, 5448, 5508, 5468, 5438, 5420, 5564] dup_rows_per_dst=[2640, 2730, 2678, 2688, 2730, 2862, 2692, 2728] combine_rows_per_src=[5520, 5422, 5448, 5508, 5468, 5438, 5420, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229968 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156648016 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313066064 weight_movement_bytes=18874368 total_nvlink_bytes=331940432 weight_to_token_ratio=0.060289 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229040 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156630704 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032368 weight_movement_bytes=18874368 total_nvlink_bytes=331906736 weight_to_token_ratio=0.060295 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38342 dup_rows=21834 dup_rows_remote=19226 dedup_ratio=0.333160 router_tokens_per_expert=[281, 262, 262, 245, 234, 276, 271, 233, 264, 266, 262, 249, 242, 248, 242, 264, 246, 221, 239, 248, 265, 241, 272, 253, 260, 258, 271, 238, 244, 244, 224, 260, 242, 255, 264, 273, 289, 238, 266, 261, 257, 260, 246, 253, 259, 260, 259, 263, 259, 254, 260, 257, 267, 277, 227, 266, 257, 224, 271, 268, 265, 252, 247, 273, 254, 254, 254, 260, 266, 274, 234, 256, 266, 250, 249, 244, 255, 273, 244, 254, 267, 266, 259, 249, 242, 264, 287, 254, 275, 238, 247, 275, 275, 257, 265, 275, 239, 263, 247, 244, 238, 282, 274, 243, 265, 236, 276, 290, 255, 242, 267, 244, 262, 248, 244, 231, 237, 269, 265, 237, 272, 278, 249, 273, 255, 234, 246, 227] payload_rows_per_dst=[5542, 5360, 5524, 5452, 5446, 5544, 5426, 5408] dup_rows_per_dst=[2660, 2608, 2766, 2796, 2728, 2846, 2784, 2646] combine_rows_per_src=[5542, 5360, 5524, 5452, 5446, 5544, 5426, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229424 combine_payload_bytes=156508160 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156737584 ingress_bytes=213131264 link_bytes=213131264 token_bytes=313245744 weight_movement_bytes=56623104 total_nvlink_bytes=369868848 weight_to_token_ratio=0.180763 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042118905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38340 dup_rows=21760 dup_rows_remote=19110 dedup_ratio=0.332031 router_tokens_per_expert=[248, 237, 259, 244, 249, 250, 263, 243, 255, 233, 268, 265, 251, 270, 245, 266, 243, 274, 223, 254, 263, 233, 278, 228, 245, 260, 270, 267, 271, 255, 281, 257, 268, 239, 250, 239, 274, 283, 246, 275, 231, 281, 255, 263, 273, 263, 241, 272, 279, 273, 227, 236, 252, 241, 262, 257, 250, 242, 254, 263, 257, 238, 252, 260, 238, 243, 254, 259, 263, 250, 242, 265, 279, 235, 257, 270, 244, 268, 256, 251, 261, 244, 232, 236, 277, 261, 277, 270, 265, 239, 256, 245, 259, 260, 256, 254, 247, 242, 254, 255, 244, 248, 222, 265, 258, 275, 270, 291, 237, 224, 274, 250, 254, 248, 278, 252, 271, 266, 268, 234, 261, 252, 282, 274, 267, 276, 261, 258] payload_rows_per_dst=[5428, 5440, 5522, 5436, 5402, 5506, 5470, 5572] dup_rows_per_dst=[2664, 2764, 2784, 2650, 2746, 2678, 2642, 2832] combine_rows_per_src=[5428, 5440, 5522, 5436, 5402, 5506, 5470, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229640 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270280 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314310920 weight_movement_bytes=18874368 total_nvlink_bytes=333185288 weight_to_token_ratio=0.060050 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38284 dup_rows=21816 dup_rows_remote=18996 dedup_ratio=0.332886 router_tokens_per_expert=[253, 262, 242, 250, 235, 279, 226, 264, 270, 241, 259, 255, 257, 249, 269, 273, 258, 250, 258, 268, 277, 300, 287, 280, 245, 282, 263, 276, 266, 266, 251, 254, 263, 245, 261, 287, 279, 259, 276, 251, 232, 278, 259, 251, 249, 263, 237, 248, 251, 261, 244, 266, 259, 261, 241, 233, 256, 257, 252, 264, 253, 252, 250, 255, 241, 262, 266, 232, 248, 266, 274, 270, 282, 220, 248, 262, 259, 244, 229, 260, 283, 245, 272, 261, 207, 218, 252, 258, 235, 291, 243, 285, 263, 274, 238, 253, 256, 273, 233, 227, 249, 231, 238, 256, 254, 257, 247, 269, 263, 261, 243, 242, 258, 245, 261, 254, 253, 255, 252, 247, 271, 263, 267, 255, 242, 234, 285, 228] payload_rows_per_dst=[5436, 5670, 5552, 5478, 5426, 5332, 5442, 5384] dup_rows_per_dst=[2820, 2896, 2942, 2632, 2700, 2518, 2552, 2756] combine_rows_per_src=[5436, 5670, 5552, 5478, 5426, 5332, 5442, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=230272 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157279104 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314327936 weight_movement_bytes=18874368 total_nvlink_bytes=333202304 weight_to_token_ratio=0.060047 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229800 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270440 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311080 weight_movement_bytes=18874368 total_nvlink_bytes=333185448 weight_to_token_ratio=0.060050 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229120 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040384 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851648 weight_movement_bytes=18874368 total_nvlink_bytes=332726016 weight_to_token_ratio=0.060138 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38238 dup_rows=21806 dup_rows_remote=19082 dedup_ratio=0.332733 router_tokens_per_expert=[258, 236, 234, 267, 222, 264, 241, 228, 242, 236, 243, 271, 254, 257, 252, 251, 253, 269, 249, 242, 271, 272, 237, 242, 245, 260, 261, 256, 288, 218, 249, 259, 268, 253, 265, 260, 252, 267, 237, 302, 266, 261, 276, 270, 249, 263, 261, 274, 274, 253, 277, 259, 248, 269, 267, 238, 227, 278, 294, 232, 261, 246, 255, 231, 270, 246, 265, 255, 269, 233, 235, 259, 242, 271, 268, 252, 257, 240, 256, 257, 267, 259, 251, 250, 241, 252, 246, 262, 253, 279, 263, 274, 241, 270, 258, 265, 234, 243, 268, 233, 244, 248, 239, 279, 278, 267, 271, 264, 274, 279, 251, 269, 266, 277, 236, 246, 272, 250, 259, 263, 255, 264, 249, 243, 237, 252, 246, 246] payload_rows_per_dst=[5330, 5426, 5556, 5472, 5434, 5486, 5534, 5492] dup_rows_per_dst=[2582, 2716, 2892, 2744, 2606, 2776, 2766, 2724] combine_rows_per_src=[5330, 5426, 5556, 5472, 5434, 5486, 5534, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38314 dup_rows=21684 dup_rows_remote=18988 dedup_ratio=0.330872 router_tokens_per_expert=[248, 236, 261, 252, 249, 232, 259, 251, 242, 256, 245, 285, 252, 229, 263, 236, 232, 259, 239, 254, 251, 227, 252, 273, 250, 254, 246, 251, 268, 259, 236, 226, 233, 273, 253, 230, 254, 266, 219, 265, 256, 264, 277, 241, 270, 254, 260, 270, 239, 283, 255, 272, 272, 262, 237, 252, 258, 259, 247, 259, 255, 255, 284, 250, 269, 264, 249, 248, 238, 246, 276, 243, 245, 273, 250, 252, 255, 297, 251, 265, 262, 256, 280, 266, 237, 241, 234, 276, 265, 261, 258, 275, 249, 217, 251, 289, 252, 285, 247, 265, 303, 253, 266, 242, 249, 267, 264, 240, 271, 279, 249, 270, 261, 221, 244, 253, 259, 264, 244, 261, 279, 278, 250, 268, 256, 250, 266, 277] payload_rows_per_dst=[5386, 5410, 5474, 5560, 5402, 5538, 5556, 5526] dup_rows_per_dst=[2606, 2544, 2696, 2718, 2840, 2696, 2848, 2736] combine_rows_per_src=[5386, 5410, 5474, 5560, 5402, 5538, 5556, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38382 dup_rows=21714 dup_rows_remote=18932 dedup_ratio=0.331329 router_tokens_per_expert=[246, 254, 241, 261, 282, 255, 253, 269, 237, 241, 239, 267, 264, 253, 251, 261, 245, 243, 234, 299, 263, 225, 238, 249, 268, 237, 256, 262, 272, 251, 254, 266, 236, 256, 237, 250, 256, 263, 261, 241, 261, 274, 239, 252, 249, 260, 229, 264, 252, 251, 267, 241, 259, 268, 246, 262, 273, 266, 226, 263, 264, 255, 276, 279, 233, 275, 261, 235, 254, 253, 281, 247, 256, 238, 266, 266, 273, 251, 254, 268, 258, 252, 273, 261, 271, 257, 259, 254, 258, 231, 254, 257, 274, 277, 234, 286, 268, 273, 263, 259, 265, 269, 278, 256, 261, 243, 244, 246, 273, 264, 250, 233, 286, 229, 246, 249, 264, 258, 274, 251, 219, 242, 258, 258, 260, 254, 248, 248] payload_rows_per_dst=[5426, 5508, 5392, 5552, 5440, 5548, 5510, 5446] dup_rows_per_dst=[2722, 2616, 2664, 2744, 2782, 2764, 2780, 2642] combine_rows_per_src=[5426, 5508, 5392, 5552, 5440, 5548, 5510, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38144 dup_rows=21944 dup_rows_remote=19270 dedup_ratio=0.334839 router_tokens_per_expert=[263, 252, 268, 259, 259, 239, 251, 243, 263, 270, 294, 253, 257, 257, 262, 271, 255, 243, 253, 258, 285, 247, 251, 257, 267, 234, 252, 240, 288, 281, 224, 266, 279, 264, 255, 223, 238, 268, 287, 242, 257, 261, 261, 237, 245, 245, 262, 243, 256, 227, 253, 273, 237, 285, 247, 244, 252, 238, 248, 256, 271, 252, 274, 249, 254, 238, 262, 252, 245, 301, 264, 228, 277, 243, 256, 257, 282, 258, 252, 269, 251, 237, 248, 237, 221, 264, 264, 265, 245, 263, 252, 247, 237, 246, 240, 275, 249, 271, 278, 261, 245, 280, 222, 256, 263, 239, 256, 255, 266, 249, 240, 231, 257, 265, 278, 288, 268, 251, 265, 255, 282, 254, 260, 230, 235, 259, 284, 255] payload_rows_per_dst=[5462, 5480, 5398, 5400, 5514, 5342, 5448, 5548] dup_rows_per_dst=[2860, 2722, 2736, 2724, 2762, 2642, 2674, 2824] combine_rows_per_src=[5462, 5480, 5398, 5400, 5514, 5342, 5448, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229280 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156852128 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474976 weight_movement_bytes=18874368 total_nvlink_bytes=332349344 weight_to_token_ratio=0.060210 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38328 dup_rows=21698 dup_rows_remote=19028 dedup_ratio=0.331085 router_tokens_per_expert=[281, 258, 233, 289, 273, 248, 227, 250, 268, 274, 258, 257, 248, 221, 262, 270, 252, 287, 245, 241, 280, 243, 262, 244, 244, 289, 242, 243, 281, 280, 261, 250, 270, 242, 241, 268, 268, 244, 232, 253, 263, 265, 250, 272, 242, 241, 263, 276, 279, 266, 260, 277, 261, 257, 249, 243, 256, 239, 232, 242, 256, 261, 257, 259, 255, 277, 235, 236, 240, 275, 254, 229, 259, 262, 267, 252, 252, 220, 261, 265, 245, 261, 261, 250, 238, 282, 281, 276, 245, 265, 254, 250, 265, 264, 242, 227, 263, 264, 265, 250, 253, 263, 246, 257, 255, 240, 266, 276, 254, 253, 248, 263, 247, 236, 266, 290, 234, 252, 262, 253, 258, 240, 249, 253, 242, 272, 264, 244] payload_rows_per_dst=[5488, 5504, 5510, 5464, 5406, 5538, 5464, 5464] dup_rows_per_dst=[2746, 2784, 2670, 2724, 2672, 2674, 2768, 2660] combine_rows_per_src=[5488, 5504, 5510, 5464, 5406, 5538, 5464, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229208 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157163352 ingress_bytes=213557248 link_bytes=213557248 token_bytes=314097496 weight_movement_bytes=56623104 total_nvlink_bytes=370720600 weight_to_token_ratio=0.180272 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046200474e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229256 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157441928 ingress_bytes=213835776 link_bytes=213835776 token_bytes=314654600 weight_movement_bytes=56623104 total_nvlink_bytes=371277704 weight_to_token_ratio=0.179953 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048869192e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38516 dup_rows=21610 dup_rows_remote=18890 dedup_ratio=0.329742 router_tokens_per_expert=[234, 256, 265, 272, 252, 211, 258, 236, 276, 268, 269, 286, 286, 256, 239, 273, 294, 277, 245, 275, 257, 255, 243, 238, 275, 246, 238, 265, 239, 223, 240, 237, 287, 274, 222, 237, 266, 236, 253, 264, 256, 271, 273, 240, 246, 257, 259, 249, 239, 261, 240, 241, 241, 234, 247, 258, 265, 252, 249, 255, 246, 269, 256, 271, 241, 275, 228, 254, 262, 273, 238, 256, 247, 277, 274, 226, 273, 269, 241, 263, 259, 249, 257, 247, 256, 259, 266, 263, 239, 278, 253, 306, 251, 276, 259, 232, 258, 253, 269, 236, 259, 272, 264, 256, 278, 254, 270, 248, 251, 241, 282, 241, 269, 248, 259, 254, 254, 249, 256, 239, 264, 247, 243, 278, 260, 251, 243, 277] payload_rows_per_dst=[5496, 5410, 5468, 5444, 5524, 5560, 5544, 5480] dup_rows_per_dst=[2780, 2720, 2712, 2604, 2670, 2704, 2718, 2702] combine_rows_per_src=[5496, 5410, 5468, 5444, 5524, 5560, 5544, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38342 dup_rows=21766 dup_rows_remote=19064 dedup_ratio=0.332123 router_tokens_per_expert=[244, 256, 278, 255, 256, 237, 249, 286, 235, 262, 235, 273, 244, 244, 238, 246, 265, 260, 242, 243, 250, 238, 276, 268, 242, 283, 257, 268, 242, 270, 284, 238, 265, 228, 256, 272, 241, 272, 267, 275, 259, 256, 250, 262, 253, 257, 242, 259, 259, 276, 266, 241, 248, 269, 255, 248, 245, 227, 268, 254, 252, 248, 248, 261, 269, 238, 283, 224, 270, 237, 264, 270, 267, 253, 263, 255, 243, 269, 238, 259, 245, 259, 295, 260, 254, 262, 275, 259, 263, 241, 271, 235, 275, 258, 268, 247, 260, 257, 264, 263, 259, 274, 233, 254, 256, 256, 250, 242, 254, 241, 250, 260, 263, 252, 278, 280, 278, 236, 244, 246, 254, 234, 227, 272, 257, 252, 237, 273] payload_rows_per_dst=[5416, 5448, 5458, 5428, 5478, 5586, 5476, 5480] dup_rows_per_dst=[2660, 2804, 2770, 2702, 2726, 2748, 2670, 2686] combine_rows_per_src=[5416, 5448, 5458, 5428, 5478, 5586, 5476, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229656 combine_payload_bytes=156237824 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156467480 ingress_bytes=212860928 link_bytes=212860928 token_bytes=312705304 weight_movement_bytes=56623104 total_nvlink_bytes=369328408 weight_to_token_ratio=0.181075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039528678e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229424 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157220912 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314212400 weight_movement_bytes=37748736 total_nvlink_bytes=351961136 weight_to_token_ratio=0.120138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38162 dup_rows=21872 dup_rows_remote=19042 dedup_ratio=0.333740 router_tokens_per_expert=[259, 279, 243, 243, 252, 273, 245, 262, 243, 254, 264, 258, 276, 245, 252, 262, 249, 261, 266, 244, 254, 233, 258, 266, 253, 251, 260, 250, 266, 235, 277, 235, 249, 230, 267, 247, 278, 253, 266, 259, 266, 248, 252, 251, 240, 259, 234, 292, 261, 279, 250, 248, 272, 273, 274, 264, 244, 245, 270, 250, 243, 265, 266, 229, 281, 232, 275, 258, 239, 254, 246, 229, 267, 257, 239, 262, 278, 281, 272, 254, 279, 242, 264, 231, 257, 230, 259, 259, 285, 277, 259, 253, 278, 276, 265, 270, 256, 247, 266, 234, 257, 290, 248, 236, 283, 237, 276, 242, 265, 264, 256, 271, 261, 241, 230, 224, 256, 284, 220, 244, 222, 260, 243, 270, 249, 244, 252, 240] payload_rows_per_dst=[5502, 5416, 5482, 5472, 5408, 5538, 5498, 5348] dup_rows_per_dst=[2830, 2730, 2700, 2794, 2840, 2750, 2696, 2532] combine_rows_per_src=[5502, 5416, 5482, 5472, 5408, 5538, 5498, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157761536 dispatch_meta_bytes=229624 combine_payload_bytes=157761536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157991160 ingress_bytes=176635904 link_bytes=176635904 token_bytes=315752696 weight_movement_bytes=18874368 total_nvlink_bytes=334627064 weight_to_token_ratio=0.059776 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692438322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229624 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157278456 ingress_bytes=213671936 link_bytes=213671936 token_bytes=314327288 weight_movement_bytes=56623104 total_nvlink_bytes=370950392 weight_to_token_ratio=0.180141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047299358e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38032 dup_rows=21908 dup_rows_remote=18928 dedup_ratio=0.334290 router_tokens_per_expert=[275, 251, 255, 281, 250, 284, 238, 268, 246, 265, 252, 276, 250, 260, 260, 235, 249, 258, 275, 273, 226, 269, 296, 259, 269, 246, 258, 237, 233, 260, 238, 272, 249, 259, 217, 270, 249, 260, 243, 246, 302, 270, 268, 247, 276, 260, 257, 260, 239, 260, 268, 245, 269, 244, 263, 247, 240, 249, 253, 270, 216, 240, 224, 250, 270, 301, 255, 250, 254, 275, 237, 238, 243, 251, 269, 268, 268, 244, 233, 282, 273, 256, 244, 275, 258, 251, 266, 261, 266, 271, 235, 224, 269, 274, 263, 257, 226, 238, 266, 293, 254, 259, 246, 236, 254, 236, 230, 294, 250, 284, 255, 242, 261, 252, 261, 214, 289, 245, 239, 234, 262, 270, 250, 250, 273, 266, 235, 249] payload_rows_per_dst=[5418, 5452, 5274, 5530, 5470, 5340, 5548, 5596] dup_rows_per_dst=[2658, 2550, 2598, 2902, 2826, 2602, 2792, 2980] combine_rows_per_src=[5418, 5452, 5274, 5530, 5470, 5340, 5548, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228816 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156540368 ingress_bytes=194060288 link_bytes=194060288 token_bytes=312851920 weight_movement_bytes=37748736 total_nvlink_bytes=350600656 weight_to_token_ratio=0.120660 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859390196e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38230 dup_rows=21822 dup_rows_remote=18944 dedup_ratio=0.332977 router_tokens_per_expert=[281, 248, 261, 279, 238, 262, 250, 253, 248, 256, 245, 268, 264, 245, 268, 230, 275, 225, 268, 239, 262, 250, 265, 263, 249, 268, 260, 278, 237, 247, 228, 247, 249, 260, 242, 244, 274, 250, 227, 254, 262, 269, 252, 259, 263, 254, 250, 271, 246, 259, 255, 270, 275, 267, 284, 255, 247, 256, 252, 247, 277, 227, 283, 245, 264, 269, 253, 237, 258, 259, 266, 271, 288, 251, 258, 275, 261, 241, 230, 240, 278, 279, 264, 277, 247, 268, 254, 253, 259, 270, 236, 258, 255, 251, 259, 273, 270, 250, 260, 266, 261, 234, 252, 298, 241, 256, 232, 274, 265, 251, 231, 236, 251, 270, 249, 241, 241, 266, 244, 240, 269, 251, 263, 246, 237, 263, 250, 226] payload_rows_per_dst=[5500, 5438, 5472, 5530, 5454, 5484, 5464, 5372] dup_rows_per_dst=[2692, 2684, 2688, 2760, 2788, 2878, 2690, 2642] combine_rows_per_src=[5500, 5438, 5472, 5530, 5454, 5484, 5464, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38154 dup_rows=21944 dup_rows_remote=19184 dedup_ratio=0.334839 router_tokens_per_expert=[267, 271, 283, 258, 267, 275, 258, 248, 262, 256, 260, 249, 253, 277, 258, 244, 252, 268, 255, 259, 258, 242, 274, 261, 258, 271, 256, 257, 279, 281, 293, 252, 267, 254, 292, 243, 264, 250, 262, 256, 272, 240, 263, 265, 255, 255, 261, 241, 250, 251, 243, 245, 252, 255, 246, 264, 283, 254, 292, 257, 243, 245, 234, 253, 256, 255, 253, 273, 261, 282, 252, 257, 227, 255, 260, 270, 264, 246, 256, 232, 242, 247, 295, 266, 247, 255, 254, 260, 256, 245, 233, 255, 230, 247, 250, 240, 240, 262, 219, 262, 233, 237, 222, 259, 250, 243, 263, 269, 268, 271, 251, 264, 266, 243, 241, 246, 254, 264, 252, 258, 252, 227, 242, 251, 263, 227, 254, 285] payload_rows_per_dst=[5456, 5550, 5472, 5480, 5438, 5450, 5356, 5390] dup_rows_per_dst=[2916, 2882, 2808, 2654, 2760, 2594, 2670, 2660] combine_rows_per_src=[5456, 5550, 5472, 5480, 5438, 5450, 5356, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38088 dup_rows=21896 dup_rows_remote=19082 dedup_ratio=0.334106 router_tokens_per_expert=[248, 259, 262, 255, 254, 245, 248, 263, 277, 266, 258, 278, 241, 226, 287, 261, 246, 243, 246, 267, 279, 239, 257, 267, 269, 260, 255, 278, 260, 268, 250, 246, 264, 270, 255, 217, 242, 238, 251, 257, 252, 263, 234, 251, 234, 261, 249, 268, 248, 258, 265, 279, 239, 267, 262, 272, 240, 227, 237, 268, 250, 277, 278, 290, 236, 243, 274, 247, 266, 269, 241, 238, 238, 283, 238, 263, 254, 255, 253, 253, 244, 255, 239, 241, 255, 252, 248, 271, 250, 252, 245, 271, 260, 248, 242, 257, 281, 272, 266, 249, 268, 282, 267, 260, 239, 260, 254, 264, 239, 274, 274, 234, 275, 286, 258, 266, 255, 226, 250, 266, 251, 261, 258, 254, 241, 250, 233, 253] payload_rows_per_dst=[5480, 5460, 5326, 5476, 5474, 5464, 5552, 5408] dup_rows_per_dst=[2776, 2800, 2686, 2838, 2628, 2596, 2814, 2758] combine_rows_per_src=[5480, 5460, 5326, 5476, 5474, 5464, 5552, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=227840 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156006912 ingress_bytes=174653440 link_bytes=174653440 token_bytes=311785984 weight_movement_bytes=18874368 total_nvlink_bytes=330660352 weight_to_token_ratio=0.060536 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673443327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38200 dup_rows=21944 dup_rows_remote=19314 dedup_ratio=0.334839 router_tokens_per_expert=[270, 231, 246, 277, 252, 230, 221, 266, 244, 262, 253, 282, 265, 251, 243, 257, 285, 267, 250, 286, 260, 256, 281, 255, 273, 257, 273, 259, 246, 253, 270, 245, 237, 287, 235, 256, 228, 232, 271, 255, 281, 254, 239, 263, 231, 254, 251, 237, 237, 282, 246, 287, 249, 236, 259, 227, 265, 285, 262, 265, 245, 268, 236, 244, 267, 241, 259, 282, 245, 243, 286, 278, 260, 238, 303, 257, 262, 250, 248, 236, 248, 265, 276, 265, 277, 251, 267, 242, 282, 264, 236, 280, 265, 240, 242, 253, 276, 261, 244, 241, 252, 269, 233, 251, 269, 264, 242, 246, 261, 240, 241, 245, 232, 242, 298, 262, 261, 232, 254, 278, 247, 236, 230, 277, 251, 243, 254, 258] payload_rows_per_dst=[5448, 5514, 5392, 5440, 5480, 5522, 5416, 5380] dup_rows_per_dst=[2652, 2918, 2630, 2746, 2830, 2784, 2654, 2730] combine_rows_per_src=[5448, 5514, 5392, 5440, 5480, 5522, 5416, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228696 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156818776 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313408856 weight_movement_bytes=37748736 total_nvlink_bytes=351157592 weight_to_token_ratio=0.120446 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38102 dup_rows=21872 dup_rows_remote=19066 dedup_ratio=0.333740 router_tokens_per_expert=[274, 261, 273, 263, 259, 284, 247, 266, 262, 246, 242, 243, 270, 279, 254, 246, 246, 251, 270, 266, 258, 245, 272, 265, 224, 265, 252, 257, 259, 259, 272, 238, 277, 274, 270, 273, 268, 252, 241, 272, 245, 258, 258, 255, 230, 229, 282, 252, 285, 262, 245, 253, 278, 267, 257, 266, 234, 276, 268, 286, 247, 244, 261, 255, 256, 279, 245, 253, 232, 260, 248, 230, 253, 264, 252, 265, 276, 247, 233, 239, 239, 231, 237, 246, 237, 239, 269, 230, 269, 268, 233, 275, 266, 249, 248, 259, 258, 246, 238, 259, 274, 254, 289, 242, 228, 255, 267, 241, 244, 242, 261, 252, 249, 255, 250, 230, 262, 236, 230, 259, 250, 302, 283, 247, 270, 289, 241, 250] payload_rows_per_dst=[5562, 5418, 5538, 5562, 5388, 5374, 5386, 5436] dup_rows_per_dst=[2776, 2780, 2734, 2806, 2676, 2616, 2714, 2770] combine_rows_per_src=[5562, 5418, 5538, 5562, 5388, 5374, 5386, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229352 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156508136 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786920 weight_movement_bytes=18874368 total_nvlink_bytes=331661288 weight_to_token_ratio=0.060343 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38164 dup_rows=21818 dup_rows_remote=19136 dedup_ratio=0.332916 router_tokens_per_expert=[243, 248, 239, 242, 267, 265, 248, 228, 235, 225, 256, 242, 246, 244, 265, 235, 252, 237, 259, 270, 254, 230, 275, 247, 224, 272, 267, 284, 297, 243, 242, 265, 278, 265, 291, 263, 251, 255, 260, 265, 242, 248, 235, 264, 267, 236, 260, 247, 231, 263, 252, 238, 242, 234, 270, 287, 262, 273, 234, 272, 250, 263, 265, 266, 244, 279, 255, 258, 261, 305, 231, 223, 258, 250, 247, 243, 277, 256, 234, 268, 246, 252, 268, 259, 263, 267, 261, 268, 244, 227, 285, 268, 276, 284, 244, 272, 259, 257, 261, 262, 261, 278, 247, 252, 271, 219, 225, 293, 292, 260, 252, 268, 274, 237, 250, 266, 243, 245, 275, 257, 270, 261, 232, 214, 269, 235, 263, 272] payload_rows_per_dst=[5304, 5554, 5484, 5498, 5454, 5414, 5538, 5472] dup_rows_per_dst=[2552, 2682, 2770, 2706, 2724, 2954, 2776, 2654] combine_rows_per_src=[5304, 5554, 5484, 5498, 5454, 5414, 5538, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228680 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156237128 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312245576 weight_movement_bytes=18874368 total_nvlink_bytes=331119944 weight_to_token_ratio=0.060447 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=230056 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156697256 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313164456 weight_movement_bytes=18874368 total_nvlink_bytes=332038824 weight_to_token_ratio=0.060270 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38250 dup_rows=21748 dup_rows_remote=19016 dedup_ratio=0.331848 router_tokens_per_expert=[230, 255, 256, 248, 283, 263, 228, 276, 247, 284, 264, 245, 250, 266, 255, 280, 267, 270, 263, 240, 258, 258, 268, 272, 234, 247, 264, 266, 266, 256, 246, 246, 268, 253, 232, 248, 284, 259, 253, 247, 270, 252, 251, 249, 284, 253, 267, 245, 285, 259, 245, 241, 275, 261, 244, 266, 253, 273, 246, 264, 281, 251, 274, 245, 238, 282, 244, 255, 263, 254, 239, 230, 235, 250, 263, 244, 239, 260, 271, 231, 269, 274, 245, 253, 283, 277, 261, 286, 251, 246, 249, 258, 269, 241, 251, 242, 239, 256, 229, 242, 273, 228, 273, 221, 258, 266, 256, 251, 270, 280, 272, 255, 252, 243, 238, 257, 256, 273, 255, 261, 246, 255, 240, 270, 276, 231, 247, 217] payload_rows_per_dst=[5538, 5540, 5494, 5514, 5416, 5524, 5362, 5400] dup_rows_per_dst=[2732, 2722, 2796, 2812, 2580, 2786, 2686, 2634] combine_rows_per_src=[5538, 5540, 5494, 5514, 5416, 5524, 5362, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=228672 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156294464 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312360256 weight_movement_bytes=18874368 total_nvlink_bytes=331234624 weight_to_token_ratio=0.060425 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229200 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156548944 ingress_bytes=212942848 link_bytes=212942848 token_bytes=312868688 weight_movement_bytes=56623104 total_nvlink_bytes=369491792 weight_to_token_ratio=0.180980 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.093750 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040313595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38110 dup_rows=21816 dup_rows_remote=18984 dedup_ratio=0.332886 router_tokens_per_expert=[247, 274, 278, 237, 231, 276, 260, 261, 257, 235, 271, 261, 254, 279, 246, 235, 263, 282, 253, 278, 261, 256, 270, 251, 275, 225, 235, 230, 287, 260, 277, 251, 255, 255, 243, 267, 256, 305, 278, 252, 262, 265, 265, 259, 241, 238, 260, 258, 286, 280, 268, 243, 265, 244, 255, 283, 229, 271, 263, 265, 238, 235, 239, 275, 230, 275, 259, 284, 238, 251, 245, 277, 276, 258, 267, 248, 258, 252, 290, 269, 253, 258, 263, 263, 251, 264, 241, 249, 224, 244, 238, 241, 234, 228, 234, 263, 259, 270, 231, 233, 215, 241, 262, 253, 259, 249, 260, 249, 273, 253, 256, 238, 257, 254, 274, 271, 251, 266, 265, 294, 247, 238, 236, 234, 255, 247, 243, 256] payload_rows_per_dst=[5460, 5302, 5490, 5386, 5490, 5358, 5624, 5610] dup_rows_per_dst=[2668, 2480, 2828, 2626, 2864, 2614, 2904, 2832] combine_rows_per_src=[5460, 5302, 5490, 5386, 5490, 5358, 5624, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229064 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156901064 ingress_bytes=251043840 link_bytes=251043840 token_bytes=313573064 weight_movement_bytes=94371840 total_nvlink_bytes=407944904 weight_to_token_ratio=0.300956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405378554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38326 dup_rows=21728 dup_rows_remote=19036 dedup_ratio=0.331543 router_tokens_per_expert=[277, 245, 245, 248, 266, 234, 271, 252, 254, 260, 260, 273, 246, 259, 273, 284, 286, 244, 283, 257, 234, 253, 262, 242, 248, 259, 240, 278, 279, 244, 248, 265, 242, 264, 269, 268, 264, 241, 268, 290, 225, 277, 232, 237, 250, 240, 225, 262, 262, 290, 270, 244, 271, 258, 270, 252, 246, 276, 244, 250, 277, 252, 259, 239, 253, 242, 270, 293, 250, 259, 248, 250, 243, 247, 242, 266, 257, 261, 274, 278, 251, 250, 248, 273, 287, 273, 242, 246, 242, 239, 257, 260, 248, 259, 256, 256, 249, 286, 240, 239, 236, 244, 221, 246, 245, 247, 244, 237, 276, 241, 245, 234, 242, 296, 236, 242, 261, 277, 261, 278, 244, 258, 262, 257, 240, 279, 245, 257] payload_rows_per_dst=[5524, 5436, 5412, 5596, 5526, 5482, 5300, 5532] dup_rows_per_dst=[2770, 2808, 2696, 2724, 2740, 2692, 2560, 2738] combine_rows_per_src=[5524, 5436, 5412, 5596, 5526, 5482, 5300, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38370 dup_rows=21758 dup_rows_remote=19120 dedup_ratio=0.332001 router_tokens_per_expert=[272, 239, 230, 267, 270, 272, 238, 259, 232, 232, 217, 284, 240, 225, 237, 274, 272, 246, 265, 254, 284, 284, 255, 224, 238, 270, 263, 242, 266, 253, 267, 244, 245, 264, 261, 248, 265, 292, 239, 250, 270, 281, 235, 274, 275, 245, 244, 262, 252, 239, 265, 238, 274, 279, 258, 260, 257, 242, 267, 259, 258, 260, 268, 241, 228, 232, 256, 267, 233, 227, 274, 236, 254, 285, 267, 235, 229, 259, 271, 270, 269, 249, 264, 292, 259, 308, 314, 258, 262, 247, 243, 243, 261, 235, 283, 287, 241, 270, 257, 243, 234, 230, 254, 274, 251, 240, 262, 263, 260, 274, 240, 249, 267, 264, 250, 254, 261, 238, 237, 223, 256, 255, 274, 234, 253, 280, 246, 255] payload_rows_per_dst=[5414, 5488, 5550, 5514, 5408, 5610, 5440, 5354] dup_rows_per_dst=[2562, 2766, 2750, 2720, 2638, 2938, 2644, 2740] combine_rows_per_src=[5414, 5488, 5550, 5514, 5408, 5610, 5440, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38054 dup_rows=21904 dup_rows_remote=19086 dedup_ratio=0.334229 router_tokens_per_expert=[252, 257, 282, 243, 250, 269, 269, 252, 243, 237, 267, 252, 252, 241, 262, 292, 248, 251, 261, 250, 256, 290, 257, 255, 229, 252, 256, 246, 245, 269, 245, 256, 243, 269, 259, 223, 286, 248, 241, 273, 262, 271, 248, 227, 264, 253, 269, 274, 271, 252, 229, 246, 276, 251, 239, 264, 254, 269, 226, 271, 241, 234, 263, 267, 257, 246, 235, 255, 250, 258, 233, 276, 260, 258, 266, 255, 266, 286, 258, 268, 250, 233, 237, 275, 263, 262, 250, 265, 269, 239, 264, 246, 268, 270, 248, 249, 265, 256, 263, 278, 253, 250, 269, 255, 261, 271, 276, 251, 261, 272, 274, 243, 256, 233, 233, 272, 237, 266, 237, 252, 206, 251, 255, 264, 255, 266, 267, 256] payload_rows_per_dst=[5472, 5422, 5482, 5380, 5500, 5418, 5578, 5380] dup_rows_per_dst=[2768, 2710, 2738, 2726, 2754, 2758, 2818, 2632] combine_rows_per_src=[5472, 5422, 5482, 5380, 5500, 5418, 5578, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38282 dup_rows=21744 dup_rows_remote=18982 dedup_ratio=0.331787 router_tokens_per_expert=[267, 261, 258, 277, 257, 243, 250, 279, 251, 272, 240, 258, 248, 247, 245, 242, 253, 273, 237, 246, 245, 260, 269, 258, 266, 274, 250, 258, 264, 248, 280, 225, 249, 267, 232, 244, 234, 282, 258, 297, 247, 266, 300, 234, 271, 242, 263, 250, 260, 262, 291, 235, 251, 258, 255, 242, 276, 243, 238, 260, 236, 268, 266, 288, 262, 278, 255, 240, 261, 233, 233, 265, 269, 271, 264, 273, 225, 258, 271, 230, 268, 229, 247, 245, 254, 262, 264, 277, 265, 266, 278, 248, 236, 238, 249, 265, 248, 232, 252, 269, 272, 260, 221, 252, 254, 258, 254, 236, 253, 241, 278, 262, 280, 279, 258, 245, 256, 257, 257, 285, 252, 263, 231, 238, 233, 258, 234, 255] payload_rows_per_dst=[5478, 5502, 5510, 5542, 5440, 5450, 5388, 5482] dup_rows_per_dst=[2712, 2710, 2762, 2716, 2736, 2732, 2696, 2680] combine_rows_per_src=[5478, 5502, 5510, 5542, 5440, 5450, 5388, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=228376 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156326936 ingress_bytes=174972928 link_bytes=174972928 token_bytes=312425496 weight_movement_bytes=18874368 total_nvlink_bytes=331299864 weight_to_token_ratio=0.060412 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676504504e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38358 dup_rows=21640 dup_rows_remote=18914 dedup_ratio=0.330200 router_tokens_per_expert=[271, 269, 235, 261, 247, 244, 226, 267, 246, 251, 255, 267, 254, 261, 274, 249, 284, 257, 273, 239, 259, 264, 245, 262, 233, 277, 261, 278, 240, 238, 272, 258, 244, 243, 284, 295, 290, 250, 239, 254, 247, 279, 261, 242, 238, 281, 235, 256, 251, 259, 310, 257, 246, 284, 247, 280, 251, 260, 242, 243, 255, 266, 248, 233, 283, 246, 243, 241, 276, 273, 274, 255, 230, 265, 240, 221, 267, 262, 260, 263, 251, 264, 258, 259, 271, 272, 239, 256, 265, 244, 251, 286, 240, 233, 247, 255, 225, 280, 273, 255, 251, 281, 266, 266, 247, 259, 255, 250, 234, 229, 245, 250, 246, 264, 243, 272, 254, 235, 232, 244, 263, 252, 277, 242, 255, 255, 251, 240] payload_rows_per_dst=[5480, 5460, 5564, 5538, 5510, 5462, 5486, 5396] dup_rows_per_dst=[2674, 2820, 2712, 2726, 2688, 2720, 2646, 2654] combine_rows_per_src=[5480, 5460, 5564, 5538, 5510, 5462, 5486, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229448 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212744 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314196040 weight_movement_bytes=37748736 total_nvlink_bytes=351944776 weight_to_token_ratio=0.120144 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38324 dup_rows=21692 dup_rows_remote=19030 dedup_ratio=0.330994 router_tokens_per_expert=[240, 269, 273, 297, 284, 254, 270, 289, 256, 252, 263, 259, 271, 259, 257, 245, 274, 261, 252, 258, 218, 241, 265, 247, 271, 268, 264, 274, 256, 243, 236, 263, 284, 240, 265, 261, 241, 250, 246, 277, 226, 233, 249, 252, 242, 233, 250, 244, 240, 272, 254, 242, 298, 283, 255, 247, 259, 242, 260, 253, 254, 266, 262, 231, 257, 259, 266, 265, 257, 239, 233, 255, 242, 243, 254, 215, 244, 273, 264, 219, 253, 254, 233, 276, 283, 263, 239, 261, 247, 251, 262, 260, 246, 281, 250, 267, 273, 273, 259, 253, 251, 262, 251, 240, 253, 231, 248, 278, 259, 250, 267, 247, 264, 259, 295, 260, 239, 254, 267, 242, 238, 241, 255, 251, 285, 251, 255, 266] payload_rows_per_dst=[5636, 5520, 5440, 5446, 5422, 5390, 5486, 5504] dup_rows_per_dst=[2840, 2662, 2546, 2790, 2548, 2862, 2704, 2740] combine_rows_per_src=[5636, 5520, 5440, 5446, 5422, 5390, 5486, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229960 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157393480 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314557000 weight_movement_bytes=37748736 total_nvlink_bytes=352305736 weight_to_token_ratio=0.120006 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=155869184 dispatch_meta_bytes=228560 combine_payload_bytes=155869184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156097744 ingress_bytes=212492288 link_bytes=212492288 token_bytes=311966928 weight_movement_bytes=56623104 total_nvlink_bytes=368590032 weight_to_token_ratio=0.181504 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035996551e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229056 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157032128 ingress_bytes=232300544 link_bytes=232300544 token_bytes=313835200 weight_movement_bytes=75497472 total_nvlink_bytes=389332672 weight_to_token_ratio=0.240564 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225789514e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38082 dup_rows=21908 dup_rows_remote=19176 dedup_ratio=0.334290 router_tokens_per_expert=[275, 290, 251, 254, 269, 259, 236, 283, 237, 225, 280, 238, 239, 242, 258, 259, 258, 272, 252, 257, 267, 265, 242, 241, 255, 245, 256, 279, 247, 245, 243, 263, 237, 239, 223, 270, 248, 242, 253, 276, 255, 268, 276, 254, 266, 290, 244, 245, 262, 242, 241, 270, 272, 278, 241, 265, 257, 220, 266, 262, 278, 265, 271, 259, 237, 252, 253, 266, 254, 264, 280, 260, 248, 259, 252, 260, 265, 254, 255, 259, 242, 276, 254, 229, 241, 262, 268, 265, 272, 246, 258, 226, 242, 246, 260, 248, 251, 242, 266, 251, 263, 259, 242, 234, 285, 247, 242, 257, 244, 255, 263, 265, 274, 273, 258, 267, 246, 272, 252, 260, 232, 265, 266, 262, 246, 264, 245, 250] payload_rows_per_dst=[5546, 5502, 5384, 5494, 5454, 5406, 5428, 5414] dup_rows_per_dst=[2732, 2896, 2582, 2786, 2782, 2664, 2704, 2762] combine_rows_per_src=[5546, 5502, 5384, 5494, 5454, 5406, 5428, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229088 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343456 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314457824 weight_movement_bytes=37748736 total_nvlink_bytes=352206560 weight_to_token_ratio=0.120044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229416 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157204520 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314179624 weight_movement_bytes=37748736 total_nvlink_bytes=351928360 weight_to_token_ratio=0.120150 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=229032 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156212904 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312196776 weight_movement_bytes=18874368 total_nvlink_bytes=331071144 weight_to_token_ratio=0.060457 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38034 dup_rows=21948 dup_rows_remote=19034 dedup_ratio=0.334900 router_tokens_per_expert=[243, 255, 253, 287, 254, 270, 276, 253, 266, 224, 243, 278, 270, 251, 249, 240, 257, 248, 266, 246, 271, 248, 248, 269, 281, 253, 230, 256, 262, 243, 249, 252, 259, 232, 262, 244, 264, 239, 262, 251, 226, 278, 267, 281, 252, 275, 271, 240, 254, 251, 251, 303, 240, 247, 253, 266, 252, 259, 235, 254, 281, 251, 260, 260, 252, 240, 258, 250, 248, 272, 259, 285, 275, 234, 257, 226, 250, 249, 250, 251, 284, 260, 259, 255, 219, 237, 238, 238, 247, 276, 271, 229, 234, 248, 237, 257, 265, 261, 258, 263, 254, 259, 257, 245, 253, 245, 273, 281, 268, 236, 244, 271, 254, 283, 257, 245, 250, 263, 282, 230, 270, 258, 271, 270, 246, 264, 255, 281] payload_rows_per_dst=[5256, 5480, 5184, 5472, 5624, 5366, 5652, 5554] dup_rows_per_dst=[2452, 2678, 2392, 2828, 2968, 2612, 3104, 2914] combine_rows_per_src=[5256, 5480, 5184, 5472, 5624, 5366, 5652, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38144 dup_rows=21866 dup_rows_remote=19154 dedup_ratio=0.333649 router_tokens_per_expert=[246, 274, 236, 254, 219, 294, 298, 253, 250, 270, 241, 244, 256, 264, 261, 248, 258, 242, 240, 254, 269, 238, 282, 248, 266, 248, 256, 252, 263, 246, 268, 267, 249, 243, 277, 256, 239, 281, 255, 277, 251, 283, 270, 240, 270, 249, 251, 246, 268, 258, 229, 263, 269, 258, 268, 234, 264, 245, 244, 242, 251, 217, 251, 256, 256, 253, 264, 238, 270, 279, 254, 275, 244, 252, 253, 218, 250, 232, 256, 247, 230, 247, 266, 263, 239, 268, 276, 248, 259, 242, 259, 274, 236, 265, 272, 275, 246, 222, 245, 233, 247, 237, 239, 280, 258, 254, 270, 251, 281, 269, 251, 237, 289, 269, 262, 258, 261, 253, 260, 305, 282, 258, 284, 223, 265, 262, 249, 249] payload_rows_per_dst=[5478, 5434, 5536, 5408, 5404, 5526, 5342, 5542] dup_rows_per_dst=[2738, 2760, 2738, 2626, 2678, 2712, 2698, 2916] combine_rows_per_src=[5478, 5434, 5536, 5408, 5404, 5526, 5342, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38274 dup_rows=21840 dup_rows_remote=19074 dedup_ratio=0.333252 router_tokens_per_expert=[285, 307, 256, 271, 259, 230, 283, 270, 243, 277, 276, 270, 254, 249, 277, 259, 252, 251, 258, 278, 266, 239, 284, 251, 274, 245, 230, 234, 264, 256, 249, 237, 269, 262, 235, 279, 247, 224, 256, 251, 225, 237, 243, 252, 266, 271, 248, 269, 267, 274, 264, 253, 256, 279, 251, 243, 244, 226, 261, 240, 244, 246, 284, 237, 266, 264, 242, 241, 254, 235, 235, 261, 260, 254, 258, 277, 270, 276, 257, 244, 265, 230, 226, 277, 256, 227, 244, 245, 261, 266, 280, 252, 278, 251, 237, 275, 236, 253, 253, 227, 269, 247, 255, 274, 243, 274, 283, 247, 277, 259, 255, 251, 244, 261, 232, 248, 242, 273, 241, 265, 230, 249, 264, 243, 269, 280, 267, 256] payload_rows_per_dst=[5662, 5448, 5390, 5450, 5422, 5454, 5448, 5422] dup_rows_per_dst=[2870, 2688, 2678, 2688, 2766, 2686, 2758, 2706] combine_rows_per_src=[5662, 5448, 5390, 5450, 5422, 5454, 5448, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38268 dup_rows=21802 dup_rows_remote=18996 dedup_ratio=0.332672 router_tokens_per_expert=[283, 273, 253, 261, 239, 260, 246, 253, 223, 271, 240, 276, 239, 235, 258, 282, 235, 238, 251, 290, 266, 243, 269, 236, 278, 239, 252, 247, 246, 277, 257, 252, 273, 243, 258, 269, 241, 258, 257, 279, 284, 246, 242, 255, 255, 255, 265, 256, 258, 251, 258, 254, 268, 226, 254, 242, 275, 268, 283, 251, 254, 249, 219, 253, 264, 237, 233, 241, 264, 257, 268, 290, 283, 281, 281, 261, 272, 254, 256, 273, 251, 203, 272, 231, 236, 270, 233, 264, 274, 250, 251, 222, 233, 265, 254, 280, 274, 267, 275, 262, 260, 235, 255, 263, 254, 255, 269, 253, 269, 236, 240, 270, 249, 254, 258, 251, 246, 230, 262, 271, 250, 276, 267, 247, 261, 244, 251, 243] payload_rows_per_dst=[5486, 5380, 5466, 5412, 5618, 5402, 5528, 5442] dup_rows_per_dst=[2698, 2772, 2806, 2714, 2812, 2576, 2746, 2678] combine_rows_per_src=[5486, 5380, 5466, 5412, 5618, 5402, 5528, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38256 dup_rows=21822 dup_rows_remote=19194 dedup_ratio=0.332977 router_tokens_per_expert=[262, 258, 260, 250, 259, 279, 268, 252, 278, 254, 255, 272, 283, 274, 252, 274, 272, 266, 230, 225, 247, 248, 258, 271, 270, 282, 233, 271, 251, 283, 257, 247, 278, 237, 266, 269, 254, 276, 252, 242, 262, 268, 243, 258, 244, 247, 248, 249, 254, 261, 266, 253, 247, 257, 271, 258, 241, 248, 239, 231, 281, 254, 251, 225, 254, 263, 264, 237, 276, 252, 238, 257, 251, 255, 251, 247, 280, 249, 279, 228, 234, 264, 254, 257, 249, 261, 260, 228, 251, 245, 252, 252, 276, 285, 256, 262, 252, 290, 241, 266, 241, 243, 250, 243, 264, 252, 242, 250, 241, 260, 251, 257, 229, 261, 230, 239, 264, 237, 260, 249, 239, 269, 277, 277, 282, 252, 248, 274] payload_rows_per_dst=[5534, 5504, 5428, 5370, 5438, 5510, 5458, 5472] dup_rows_per_dst=[2926, 2718, 2758, 2704, 2724, 2662, 2628, 2702] combine_rows_per_src=[5534, 5504, 5428, 5370, 5438, 5510, 5458, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=228272 combine_payload_bytes=155787264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156015536 ingress_bytes=193536000 link_bytes=193536000 token_bytes=311802800 weight_movement_bytes=37748736 total_nvlink_bytes=349551536 weight_to_token_ratio=0.121066 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854366726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38158 dup_rows=21858 dup_rows_remote=19132 dedup_ratio=0.333527 router_tokens_per_expert=[244, 244, 258, 251, 246, 269, 247, 266, 269, 256, 264, 261, 240, 252, 249, 249, 262, 238, 238, 268, 242, 263, 270, 259, 272, 245, 266, 275, 219, 259, 257, 241, 257, 229, 259, 281, 248, 269, 264, 238, 263, 245, 276, 234, 259, 275, 243, 261, 262, 266, 260, 252, 272, 252, 268, 258, 248, 273, 258, 238, 260, 257, 232, 267, 280, 245, 249, 254, 257, 250, 259, 239, 260, 240, 258, 247, 273, 289, 287, 259, 262, 263, 255, 269, 230, 239, 246, 228, 258, 253, 283, 268, 265, 256, 273, 248, 249, 262, 258, 249, 265, 221, 253, 261, 266, 256, 248, 246, 245, 273, 272, 280, 252, 268, 235, 257, 251, 270, 246, 257, 246, 248, 258, 252, 241, 258, 261, 259] payload_rows_per_dst=[5440, 5424, 5384, 5520, 5472, 5484, 5504, 5450] dup_rows_per_dst=[2690, 2724, 2818, 2726, 2820, 2708, 2704, 2668] combine_rows_per_src=[5440, 5424, 5384, 5520, 5472, 5484, 5504, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229192 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156467016 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312704840 weight_movement_bytes=18874368 total_nvlink_bytes=331579208 weight_to_token_ratio=0.060358 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229392 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999696 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770000 weight_movement_bytes=18874368 total_nvlink_bytes=332644368 weight_to_token_ratio=0.060154 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229056 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974784 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720512 weight_movement_bytes=18874368 total_nvlink_bytes=332594880 weight_to_token_ratio=0.060163 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38062 dup_rows=21942 dup_rows_remote=19136 dedup_ratio=0.334808 router_tokens_per_expert=[233, 236, 261, 257, 277, 252, 271, 249, 232, 269, 261, 247, 282, 260, 239, 228, 251, 268, 263, 241, 250, 274, 294, 271, 263, 291, 237, 275, 230, 264, 247, 250, 263, 271, 274, 257, 259, 259, 264, 271, 247, 276, 237, 277, 229, 262, 264, 263, 277, 302, 222, 268, 271, 270, 253, 252, 218, 281, 247, 286, 244, 224, 233, 278, 264, 236, 248, 270, 263, 231, 251, 239, 250, 273, 262, 278, 254, 249, 255, 266, 253, 257, 233, 278, 246, 249, 237, 259, 246, 242, 247, 272, 249, 258, 236, 264, 271, 263, 252, 246, 244, 242, 243, 246, 260, 232, 235, 278, 281, 266, 225, 264, 261, 231, 271, 256, 247, 242, 257, 297, 235, 258, 218, 250, 255, 263, 274, 268] payload_rows_per_dst=[5452, 5532, 5430, 5464, 5466, 5400, 5352, 5498] dup_rows_per_dst=[2656, 2806, 2916, 2788, 2712, 2652, 2744, 2668] combine_rows_per_src=[5452, 5532, 5430, 5464, 5466, 5400, 5352, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229800 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926376 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622952 weight_movement_bytes=18874368 total_nvlink_bytes=332497320 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229160 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156524328 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312819496 weight_movement_bytes=37748736 total_nvlink_bytes=350568232 weight_to_token_ratio=0.120673 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38118 dup_rows=21764 dup_rows_remote=18712 dedup_ratio=0.332092 router_tokens_per_expert=[260, 271, 259, 264, 266, 252, 246, 247, 265, 235, 249, 262, 252, 257, 255, 263, 262, 223, 256, 255, 235, 251, 242, 224, 243, 275, 279, 259, 290, 242, 259, 253, 249, 269, 235, 265, 252, 274, 234, 277, 258, 247, 252, 272, 255, 274, 237, 254, 248, 269, 259, 269, 256, 246, 257, 250, 251, 256, 246, 240, 266, 260, 262, 251, 271, 267, 262, 267, 250, 261, 262, 271, 238, 284, 235, 255, 227, 269, 230, 225, 245, 289, 256, 257, 254, 254, 265, 256, 241, 264, 290, 284, 252, 243, 252, 275, 255, 248, 279, 260, 250, 263, 272, 226, 242, 256, 241, 277, 254, 256, 246, 282, 231, 224, 256, 273, 252, 276, 264, 244, 246, 267, 245, 261, 252, 248, 249, 281] payload_rows_per_dst=[5654, 5690, 5456, 5456, 5428, 5362, 5238, 5488] dup_rows_per_dst=[3052, 2974, 2824, 2716, 2720, 2388, 2440, 2650] combine_rows_per_src=[5654, 5690, 5456, 5456, 5428, 5362, 5238, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228792 combine_payload_bytes=155901952 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156130744 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312032696 weight_movement_bytes=18874368 total_nvlink_bytes=330907064 weight_to_token_ratio=0.060488 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38192 dup_rows=21780 dup_rows_remote=18820 dedup_ratio=0.332336 router_tokens_per_expert=[262, 267, 248, 273, 261, 276, 239, 248, 230, 276, 262, 264, 249, 257, 242, 276, 242, 268, 264, 273, 246, 262, 240, 247, 268, 230, 226, 265, 222, 265, 251, 296, 258, 262, 239, 259, 271, 224, 266, 232, 250, 257, 231, 246, 257, 239, 279, 244, 251, 226, 243, 248, 268, 266, 266, 251, 239, 273, 249, 229, 264, 260, 253, 281, 260, 271, 269, 217, 248, 253, 267, 273, 253, 263, 248, 257, 268, 260, 250, 294, 269, 244, 278, 258, 257, 260, 247, 247, 246, 241, 255, 251, 255, 264, 226, 292, 237, 279, 253, 261, 224, 270, 308, 255, 277, 266, 273, 272, 243, 247, 256, 247, 268, 232, 259, 247, 284, 288, 238, 261, 251, 246, 241, 235, 261, 243, 266, 263] payload_rows_per_dst=[5568, 5468, 5382, 5288, 5276, 5674, 5536, 5564] dup_rows_per_dst=[2692, 2640, 2646, 2488, 2534, 2998, 2822, 2960] combine_rows_per_src=[5568, 5468, 5382, 5288, 5276, 5674, 5536, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=227320 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156358648 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312489976 weight_movement_bytes=18874368 total_nvlink_bytes=331364344 weight_to_token_ratio=0.060400 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38298 dup_rows=21852 dup_rows_remote=19324 dedup_ratio=0.333435 router_tokens_per_expert=[227, 270, 242, 245, 229, 276, 290, 239, 238, 245, 245, 251, 274, 262, 244, 262, 259, 268, 260, 273, 271, 258, 241, 279, 266, 251, 208, 260, 256, 283, 239, 264, 246, 236, 235, 247, 259, 241, 251, 259, 250, 245, 281, 248, 257, 241, 226, 235, 230, 266, 261, 257, 273, 266, 270, 262, 258, 233, 273, 285, 277, 251, 247, 254, 251, 257, 249, 276, 258, 260, 244, 251, 240, 274, 262, 279, 259, 252, 269, 250, 235, 259, 282, 280, 253, 266, 229, 263, 249, 231, 253, 259, 273, 246, 259, 276, 257, 251, 271, 277, 234, 277, 262, 255, 244, 249, 256, 263, 246, 260, 264, 255, 258, 261, 293, 241, 251, 263, 257, 264, 256, 286, 249, 221, 248, 246, 250, 264] payload_rows_per_dst=[5392, 5510, 5386, 5476, 5436, 5502, 5490, 5492] dup_rows_per_dst=[2686, 2762, 2528, 2850, 2826, 2724, 2752, 2724] combine_rows_per_src=[5392, 5510, 5386, 5476, 5436, 5502, 5490, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38284 dup_rows=21752 dup_rows_remote=19096 dedup_ratio=0.331909 router_tokens_per_expert=[277, 250, 276, 262, 241, 257, 258, 273, 239, 273, 247, 248, 270, 238, 220, 289, 255, 254, 258, 261, 265, 266, 255, 257, 246, 288, 247, 261, 275, 249, 278, 245, 257, 227, 250, 276, 244, 242, 273, 272, 260, 276, 252, 236, 230, 264, 259, 259, 261, 251, 265, 268, 215, 233, 249, 280, 254, 247, 252, 247, 261, 292, 273, 276, 253, 246, 257, 233, 246, 230, 234, 272, 262, 246, 266, 262, 250, 247, 255, 248, 265, 256, 264, 267, 259, 244, 294, 254, 233, 233, 256, 225, 246, 278, 248, 256, 258, 258, 289, 263, 255, 268, 256, 244, 250, 262, 252, 243, 255, 261, 282, 236, 247, 263, 238, 218, 270, 285, 256, 262, 253, 246, 265, 222, 281, 255, 265, 246] payload_rows_per_dst=[5514, 5548, 5430, 5454, 5428, 5500, 5482, 5428] dup_rows_per_dst=[2722, 2772, 2724, 2794, 2586, 2656, 2782, 2716] combine_rows_per_src=[5514, 5548, 5430, 5454, 5428, 5500, 5482, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38246 dup_rows=21736 dup_rows_remote=18828 dedup_ratio=0.331665 router_tokens_per_expert=[233, 251, 246, 248, 233, 260, 244, 275, 271, 283, 259, 268, 248, 285, 255, 249, 260, 250, 251, 248, 281, 267, 232, 242, 259, 273, 255, 246, 253, 269, 251, 264, 230, 258, 252, 295, 259, 258, 257, 236, 262, 251, 279, 230, 229, 273, 233, 264, 251, 256, 266, 258, 273, 254, 261, 278, 227, 265, 268, 278, 259, 261, 270, 259, 224, 246, 252, 285, 245, 247, 246, 256, 274, 241, 251, 277, 260, 259, 259, 267, 254, 255, 240, 262, 262, 249, 251, 224, 262, 247, 275, 259, 278, 260, 243, 238, 247, 261, 264, 245, 249, 268, 257, 238, 259, 243, 225, 260, 268, 270, 280, 268, 248, 266, 231, 259, 289, 272, 226, 257, 260, 280, 216, 225, 257, 265, 259, 249] payload_rows_per_dst=[5440, 5342, 5508, 5556, 5452, 5480, 5554, 5468] dup_rows_per_dst=[2776, 2602, 2624, 2812, 2726, 2638, 2908, 2650] combine_rows_per_src=[5440, 5342, 5508, 5556, 5452, 5480, 5554, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38382 dup_rows=21718 dup_rows_remote=19032 dedup_ratio=0.331390 router_tokens_per_expert=[247, 230, 241, 261, 265, 268, 261, 271, 253, 260, 251, 269, 272, 252, 249, 262, 268, 275, 264, 233, 248, 262, 251, 296, 243, 262, 269, 261, 236, 274, 263, 283, 241, 254, 242, 220, 246, 266, 279, 250, 256, 250, 266, 240, 265, 278, 237, 262, 247, 259, 255, 251, 243, 249, 240, 288, 246, 229, 281, 251, 246, 248, 289, 255, 258, 236, 247, 253, 276, 240, 238, 279, 249, 247, 291, 269, 238, 269, 235, 236, 238, 214, 239, 275, 283, 246, 258, 247, 250, 299, 243, 263, 244, 264, 277, 249, 279, 242, 226, 250, 231, 245, 272, 277, 222, 275, 260, 268, 270, 285, 258, 219, 253, 242, 268, 302, 244, 263, 264, 260, 271, 245, 214, 256, 260, 242, 270, 256] payload_rows_per_dst=[5548, 5562, 5396, 5444, 5436, 5478, 5486, 5468] dup_rows_per_dst=[2676, 2814, 2708, 2710, 2686, 2700, 2672, 2752] combine_rows_per_src=[5548, 5562, 5396, 5444, 5436, 5478, 5486, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38338 dup_rows=21752 dup_rows_remote=19014 dedup_ratio=0.331909 router_tokens_per_expert=[278, 248, 258, 286, 278, 235, 257, 237, 257, 291, 238, 262, 249, 247, 267, 252, 274, 261, 215, 238, 258, 267, 275, 264, 246, 248, 253, 265, 250, 255, 259, 269, 281, 249, 238, 249, 280, 258, 269, 235, 273, 254, 260, 248, 256, 258, 259, 241, 248, 269, 269, 268, 275, 245, 245, 251, 230, 243, 254, 275, 259, 245, 261, 255, 259, 264, 264, 244, 213, 238, 259, 242, 282, 267, 283, 267, 258, 257, 263, 250, 251, 264, 256, 274, 244, 259, 242, 265, 244, 256, 247, 237, 273, 233, 216, 238, 272, 252, 263, 262, 260, 254, 257, 272, 264, 270, 261, 245, 252, 266, 250, 227, 281, 268, 233, 274, 261, 242, 252, 252, 248, 264, 245, 259, 229, 251, 255, 281] payload_rows_per_dst=[5524, 5464, 5482, 5446, 5488, 5484, 5488, 5408] dup_rows_per_dst=[2756, 2730, 2734, 2738, 2732, 2514, 2766, 2782] combine_rows_per_src=[5524, 5464, 5482, 5446, 5488, 5484, 5488, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228048 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156662480 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313096912 weight_movement_bytes=18874368 total_nvlink_bytes=331971280 weight_to_token_ratio=0.060283 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=230488 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157099096 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313967704 weight_movement_bytes=18874368 total_nvlink_bytes=332842072 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38260 dup_rows=21806 dup_rows_remote=19066 dedup_ratio=0.332733 router_tokens_per_expert=[247, 268, 277, 252, 231, 239, 225, 263, 235, 237, 266, 246, 278, 268, 244, 258, 250, 235, 281, 271, 247, 242, 253, 236, 253, 280, 244, 284, 248, 280, 247, 254, 259, 237, 288, 244, 261, 272, 254, 257, 256, 272, 220, 259, 261, 247, 267, 267, 266, 266, 232, 258, 256, 254, 266, 286, 268, 271, 246, 266, 243, 253, 306, 273, 237, 260, 252, 250, 266, 265, 250, 274, 289, 242, 265, 243, 259, 238, 262, 270, 252, 258, 257, 277, 254, 253, 260, 223, 245, 233, 255, 253, 242, 253, 245, 242, 231, 251, 279, 232, 239, 241, 268, 255, 254, 264, 263, 264, 259, 263, 260, 270, 264, 258, 245, 260, 261, 265, 272, 244, 274, 230, 265, 262, 222, 238, 252, 269] payload_rows_per_dst=[5422, 5470, 5502, 5530, 5458, 5402, 5460, 5486] dup_rows_per_dst=[2646, 2740, 2740, 2890, 2786, 2602, 2726, 2676] combine_rows_per_src=[5422, 5470, 5502, 5530, 5458, 5402, 5460, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229520 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040784 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852048 weight_movement_bytes=18874368 total_nvlink_bytes=332726416 weight_to_token_ratio=0.060138 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228296 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156883912 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313539528 weight_movement_bytes=18874368 total_nvlink_bytes=332413896 weight_to_token_ratio=0.060198 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229656 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157442328 ingress_bytes=194961408 link_bytes=194961408 token_bytes=314655000 weight_movement_bytes=37748736 total_nvlink_bytes=352403736 weight_to_token_ratio=0.119969 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868024284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229408 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261856 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294304 weight_movement_bytes=18874368 total_nvlink_bytes=333168672 weight_to_token_ratio=0.060053 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38290 dup_rows=21716 dup_rows_remote=18992 dedup_ratio=0.331360 router_tokens_per_expert=[282, 254, 263, 255, 242, 283, 270, 268, 247, 233, 251, 265, 275, 257, 231, 247, 213, 231, 247, 257, 239, 255, 245, 256, 267, 229, 255, 239, 290, 262, 277, 257, 276, 273, 256, 249, 261, 249, 264, 257, 251, 255, 274, 281, 264, 242, 249, 248, 233, 249, 256, 255, 245, 221, 266, 242, 250, 279, 255, 254, 264, 258, 287, 250, 264, 267, 273, 247, 256, 277, 246, 226, 241, 245, 259, 275, 251, 267, 256, 267, 248, 237, 241, 263, 273, 250, 242, 303, 277, 227, 271, 276, 258, 269, 242, 249, 282, 266, 256, 261, 253, 263, 271, 246, 248, 250, 238, 236, 250, 267, 294, 255, 228, 256, 273, 235, 253, 274, 259, 263, 283, 237, 236, 260, 256, 231, 252, 238] payload_rows_per_dst=[5530, 5424, 5522, 5464, 5480, 5458, 5512, 5430] dup_rows_per_dst=[2724, 2606, 2776, 2664, 2754, 2794, 2760, 2638] combine_rows_per_src=[5530, 5424, 5522, 5464, 5480, 5458, 5512, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229304 combine_payload_bytes=156712960 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156942264 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655224 weight_movement_bytes=37748736 total_nvlink_bytes=351403960 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229128 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157064968 ingress_bytes=213458944 link_bytes=213458944 token_bytes=313900808 weight_movement_bytes=56623104 total_nvlink_bytes=370523912 weight_to_token_ratio=0.180385 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045258573e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43458 payload_rows_remote=37962 dup_rows=22078 dup_rows_remote=19146 dedup_ratio=0.336884 router_tokens_per_expert=[278, 249, 252, 246, 272, 248, 240, 257, 266, 286, 252, 254, 226, 253, 254, 278, 257, 258, 271, 256, 233, 271, 279, 235, 250, 273, 259, 231, 249, 238, 253, 243, 262, 242, 281, 231, 243, 220, 263, 257, 249, 240, 255, 249, 264, 262, 268, 228, 251, 239, 241, 254, 282, 231, 266, 260, 260, 283, 243, 254, 273, 251, 239, 260, 263, 241, 263, 264, 257, 261, 247, 253, 261, 243, 266, 271, 261, 269, 249, 264, 269, 259, 278, 262, 258, 271, 260, 275, 258, 292, 255, 253, 241, 228, 261, 257, 244, 262, 256, 233, 226, 258, 239, 243, 214, 246, 266, 268, 260, 266, 260, 255, 282, 290, 227, 279, 254, 272, 260, 271, 248, 240, 260, 254, 267, 268, 280, 242] payload_rows_per_dst=[5416, 5398, 5346, 5516, 5470, 5486, 5330, 5496] dup_rows_per_dst=[2806, 2714, 2682, 2658, 2796, 2810, 2680, 2932] combine_rows_per_src=[5416, 5398, 5346, 5516, 5470, 5486, 5330, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38328 dup_rows=21788 dup_rows_remote=19110 dedup_ratio=0.332458 router_tokens_per_expert=[269, 260, 241, 280, 263, 259, 258, 255, 223, 245, 256, 235, 240, 258, 262, 259, 254, 267, 252, 257, 258, 274, 271, 252, 241, 255, 242, 269, 255, 279, 265, 267, 248, 278, 249, 252, 238, 267, 248, 243, 268, 231, 286, 251, 258, 231, 236, 265, 262, 267, 270, 253, 248, 248, 234, 281, 257, 246, 250, 270, 260, 249, 272, 235, 273, 225, 284, 284, 251, 243, 275, 240, 266, 238, 266, 256, 237, 262, 259, 229, 246, 251, 252, 229, 287, 228, 292, 280, 226, 248, 272, 251, 272, 281, 243, 250, 279, 260, 249, 254, 270, 260, 240, 228, 243, 230, 253, 275, 265, 239, 249, 259, 260, 270, 241, 273, 250, 237, 251, 254, 259, 279, 250, 255, 274, 283, 251, 260] payload_rows_per_dst=[5458, 5502, 5420, 5474, 5530, 5442, 5444, 5478] dup_rows_per_dst=[2668, 2814, 2678, 2730, 2646, 2774, 2662, 2816] combine_rows_per_src=[5458, 5502, 5420, 5474, 5530, 5442, 5444, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38304 dup_rows=21838 dup_rows_remote=19080 dedup_ratio=0.333221 router_tokens_per_expert=[242, 264, 250, 271, 226, 265, 232, 263, 266, 225, 247, 286, 239, 224, 256, 261, 245, 245, 247, 246, 259, 231, 244, 241, 256, 279, 242, 248, 283, 275, 257, 236, 265, 248, 271, 280, 262, 244, 260, 250, 266, 246, 265, 276, 240, 253, 246, 217, 236, 256, 245, 276, 261, 248, 276, 246, 241, 285, 268, 284, 280, 255, 265, 249, 244, 260, 255, 282, 268, 231, 258, 236, 236, 254, 251, 258, 257, 243, 256, 226, 295, 272, 237, 243, 262, 238, 278, 248, 264, 227, 261, 261, 229, 262, 247, 252, 256, 286, 254, 248, 267, 254, 241, 284, 294, 281, 260, 245, 255, 225, 288, 264, 245, 256, 276, 264, 277, 281, 269, 251, 247, 247, 265, 273, 246, 271, 244, 252] payload_rows_per_dst=[5442, 5384, 5452, 5544, 5382, 5394, 5556, 5544] dup_rows_per_dst=[2592, 2684, 2726, 2798, 2648, 2758, 2848, 2784] combine_rows_per_src=[5442, 5384, 5452, 5544, 5382, 5394, 5556, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38364 dup_rows=21758 dup_rows_remote=19066 dedup_ratio=0.332001 router_tokens_per_expert=[280, 252, 252, 259, 280, 274, 262, 252, 237, 287, 262, 250, 232, 255, 230, 243, 276, 238, 267, 275, 266, 259, 256, 259, 283, 243, 240, 259, 268, 259, 281, 279, 244, 241, 251, 254, 263, 238, 256, 261, 233, 249, 218, 277, 270, 266, 282, 251, 243, 256, 250, 252, 260, 245, 266, 248, 274, 233, 256, 234, 243, 244, 262, 247, 240, 264, 249, 267, 223, 245, 236, 268, 260, 264, 244, 257, 257, 275, 250, 254, 251, 277, 288, 246, 245, 281, 249, 263, 246, 239, 242, 234, 264, 268, 216, 273, 256, 280, 250, 269, 261, 241, 264, 244, 259, 262, 278, 255, 271, 282, 269, 241, 234, 273, 242, 243, 268, 257, 259, 249, 269, 256, 284, 246, 258, 236, 251, 244] payload_rows_per_dst=[5526, 5572, 5430, 5342, 5414, 5474, 5582, 5438] dup_rows_per_dst=[2688, 2844, 2678, 2684, 2692, 2690, 2782, 2700] combine_rows_per_src=[5526, 5572, 5430, 5342, 5414, 5474, 5582, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38288 dup_rows=21810 dup_rows_remote=19132 dedup_ratio=0.332794 router_tokens_per_expert=[220, 256, 245, 255, 250, 282, 248, 302, 292, 283, 242, 253, 269, 276, 247, 273, 256, 258, 248, 239, 256, 258, 230, 278, 250, 271, 255, 226, 249, 249, 288, 260, 277, 250, 255, 267, 264, 286, 251, 232, 267, 270, 273, 235, 258, 254, 273, 268, 244, 231, 249, 262, 254, 238, 241, 275, 230, 258, 229, 237, 275, 235, 245, 262, 266, 250, 261, 260, 253, 267, 251, 233, 248, 250, 237, 262, 238, 273, 263, 234, 264, 271, 272, 249, 265, 258, 229, 257, 265, 244, 251, 261, 261, 257, 275, 242, 247, 270, 250, 229, 272, 242, 244, 259, 247, 262, 242, 253, 258, 260, 269, 254, 249, 280, 251, 247, 278, 264, 250, 242, 252, 286, 257, 234, 256, 261, 249, 278] payload_rows_per_dst=[5576, 5416, 5528, 5354, 5432, 5476, 5438, 5506] dup_rows_per_dst=[2810, 2726, 2832, 2576, 2660, 2766, 2678, 2762] combine_rows_per_src=[5576, 5416, 5528, 5354, 5432, 5476, 5438, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38190 dup_rows=21900 dup_rows_remote=19170 dedup_ratio=0.334167 router_tokens_per_expert=[256, 270, 230, 286, 252, 239, 261, 253, 253, 235, 237, 272, 252, 218, 239, 255, 278, 267, 262, 245, 262, 232, 244, 244, 262, 280, 275, 290, 252, 256, 230, 254, 232, 246, 263, 237, 248, 253, 276, 239, 257, 272, 234, 255, 274, 243, 254, 261, 249, 268, 261, 240, 228, 269, 259, 270, 265, 248, 227, 287, 298, 242, 226, 251, 255, 263, 249, 256, 267, 274, 254, 253, 260, 235, 254, 254, 256, 245, 265, 282, 254, 261, 244, 273, 258, 262, 249, 264, 268, 256, 249, 255, 262, 246, 258, 221, 253, 277, 245, 271, 248, 274, 295, 263, 262, 277, 250, 265, 251, 259, 253, 257, 277, 242, 268, 268, 252, 277, 238, 246, 249, 266, 269, 250, 223, 256, 248, 264] payload_rows_per_dst=[5432, 5506, 5438, 5446, 5420, 5398, 5576, 5420] dup_rows_per_dst=[2584, 2760, 2650, 2730, 2824, 2762, 2824, 2766] combine_rows_per_src=[5432, 5506, 5438, 5446, 5420, 5398, 5576, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=155492352 dispatch_meta_bytes=228432 combine_payload_bytes=155492352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155720784 ingress_bytes=193241088 link_bytes=193241088 token_bytes=311213136 weight_movement_bytes=37748736 total_nvlink_bytes=348961872 weight_to_token_ratio=0.121295 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851541025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38088 dup_rows=21896 dup_rows_remote=19044 dedup_ratio=0.334106 router_tokens_per_expert=[270, 248, 247, 258, 255, 250, 248, 274, 257, 247, 230, 285, 247, 255, 239, 265, 241, 271, 252, 240, 234, 271, 248, 267, 256, 292, 267, 295, 270, 272, 262, 264, 259, 264, 240, 233, 251, 226, 249, 249, 272, 248, 278, 246, 259, 256, 258, 274, 263, 283, 263, 240, 266, 257, 249, 254, 277, 249, 238, 255, 252, 255, 244, 254, 247, 270, 280, 218, 261, 225, 257, 248, 259, 233, 222, 257, 270, 259, 240, 255, 249, 241, 251, 258, 239, 258, 267, 270, 258, 228, 231, 277, 254, 268, 265, 263, 233, 300, 240, 285, 240, 270, 269, 268, 272, 233, 262, 252, 247, 253, 254, 266, 243, 288, 247, 275, 253, 242, 249, 268, 242, 249, 244, 272, 289, 260, 245, 242] payload_rows_per_dst=[5474, 5552, 5472, 5338, 5338, 5418, 5518, 5530] dup_rows_per_dst=[2676, 2852, 2916, 2596, 2664, 2736, 2770, 2686] combine_rows_per_src=[5474, 5552, 5472, 5338, 5338, 5418, 5518, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229752 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157221240 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314212728 weight_movement_bytes=56623104 total_nvlink_bytes=370835832 weight_to_token_ratio=0.180206 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229680 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057328 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884976 weight_movement_bytes=18874368 total_nvlink_bytes=332759344 weight_to_token_ratio=0.060131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229720 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157368664 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314507608 weight_movement_bytes=18874368 total_nvlink_bytes=333381976 weight_to_token_ratio=0.060012 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229536 combine_payload_bytes=156893184 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157122720 ingress_bytes=232390656 link_bytes=232390656 token_bytes=314015904 weight_movement_bytes=75497472 total_nvlink_bytes=389513376 weight_to_token_ratio=0.240426 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226652922e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229440 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655680 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081920 weight_movement_bytes=37748736 total_nvlink_bytes=350830656 weight_to_token_ratio=0.120571 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38136 dup_rows=21860 dup_rows_remote=19130 dedup_ratio=0.333557 router_tokens_per_expert=[268, 268, 260, 232, 263, 257, 259, 243, 242, 271, 265, 283, 248, 246, 255, 259, 266, 234, 258, 238, 231, 261, 241, 237, 253, 227, 252, 270, 258, 283, 258, 271, 277, 250, 298, 242, 258, 244, 246, 267, 267, 237, 281, 249, 258, 285, 270, 245, 264, 258, 261, 256, 251, 255, 258, 253, 272, 240, 238, 244, 242, 261, 259, 226, 274, 267, 248, 256, 265, 246, 272, 261, 245, 268, 253, 264, 272, 263, 244, 256, 254, 245, 270, 238, 243, 259, 277, 247, 266, 277, 237, 248, 276, 256, 253, 255, 267, 268, 255, 270, 252, 261, 246, 290, 239, 226, 231, 274, 251, 257, 257, 254, 263, 243, 250, 255, 250, 250, 245, 255, 253, 247, 254, 239, 275, 264, 256, 247] payload_rows_per_dst=[5540, 5554, 5576, 5412, 5424, 5316, 5504, 5350] dup_rows_per_dst=[2730, 2892, 2866, 2638, 2784, 2516, 2692, 2742] combine_rows_per_src=[5540, 5554, 5576, 5412, 5424, 5316, 5504, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228528 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156236976 ingress_bytes=193757184 link_bytes=193757184 token_bytes=312245424 weight_movement_bytes=37748736 total_nvlink_bytes=349994160 weight_to_token_ratio=0.120894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856486003e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=229064 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156434120 ingress_bytes=193953792 link_bytes=193953792 token_bytes=312639176 weight_movement_bytes=37748736 total_nvlink_bytes=350387912 weight_to_token_ratio=0.120742 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858369804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38304 dup_rows=21726 dup_rows_remote=19002 dedup_ratio=0.331512 router_tokens_per_expert=[266, 246, 231, 248, 233, 242, 241, 264, 268, 281, 257, 278, 250, 255, 246, 262, 259, 248, 265, 271, 244, 265, 272, 269, 220, 242, 263, 288, 216, 217, 263, 234, 273, 254, 235, 259, 275, 252, 255, 259, 248, 240, 243, 292, 270, 281, 235, 262, 253, 279, 254, 251, 263, 248, 290, 276, 265, 266, 236, 264, 247, 262, 240, 233, 247, 278, 258, 277, 242, 234, 250, 236, 241, 259, 242, 259, 254, 239, 252, 247, 274, 282, 274, 240, 221, 262, 252, 256, 269, 262, 254, 244, 268, 280, 256, 248, 282, 249, 230, 267, 258, 250, 264, 252, 267, 279, 263, 281, 260, 289, 241, 239, 228, 245, 288, 273, 255, 256, 252, 246, 242, 257, 255, 266, 256, 245, 277, 235] payload_rows_per_dst=[5466, 5422, 5282, 5580, 5580, 5500, 5474, 5506] dup_rows_per_dst=[2670, 2650, 2574, 2674, 2788, 2856, 2790, 2724] combine_rows_per_src=[5466, 5422, 5282, 5580, 5580, 5500, 5474, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38484 dup_rows=21664 dup_rows_remote=18978 dedup_ratio=0.330566 router_tokens_per_expert=[247, 247, 240, 255, 248, 248, 255, 244, 261, 261, 283, 264, 251, 218, 241, 260, 249, 248, 258, 257, 270, 232, 287, 257, 279, 276, 251, 275, 271, 239, 265, 259, 282, 265, 256, 242, 252, 233, 242, 244, 259, 248, 237, 241, 270, 247, 263, 256, 258, 259, 256, 244, 264, 259, 232, 240, 261, 254, 261, 264, 246, 247, 269, 255, 244, 239, 282, 243, 222, 262, 267, 241, 254, 271, 256, 260, 258, 263, 241, 242, 247, 248, 266, 269, 250, 256, 260, 243, 263, 259, 242, 268, 247, 261, 235, 273, 209, 276, 281, 263, 239, 245, 238, 235, 256, 247, 261, 295, 294, 285, 248, 257, 254, 284, 257, 251, 269, 272, 252, 276, 271, 249, 267, 270, 244, 286, 257, 246] payload_rows_per_dst=[5526, 5610, 5388, 5458, 5466, 5408, 5450, 5566] dup_rows_per_dst=[2520, 2736, 2686, 2680, 2624, 2766, 2808, 2844] combine_rows_per_src=[5526, 5610, 5388, 5458, 5466, 5408, 5450, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38060 dup_rows=21972 dup_rows_remote=19282 dedup_ratio=0.335266 router_tokens_per_expert=[237, 232, 258, 243, 266, 262, 261, 251, 260, 242, 275, 266, 285, 264, 253, 253, 264, 257, 239, 240, 239, 255, 256, 243, 250, 275, 251, 247, 262, 273, 237, 286, 255, 257, 243, 250, 239, 273, 250, 243, 243, 247, 253, 258, 247, 261, 264, 250, 268, 245, 263, 256, 281, 237, 266, 251, 266, 256, 235, 274, 261, 275, 246, 269, 252, 257, 255, 261, 268, 245, 241, 261, 236, 263, 278, 254, 254, 265, 257, 252, 238, 278, 241, 281, 240, 258, 233, 275, 237, 270, 255, 258, 254, 244, 250, 246, 253, 242, 242, 268, 267, 275, 259, 253, 257, 260, 230, 242, 284, 262, 255, 248, 272, 236, 246, 260, 245, 252, 233, 253, 264, 279, 255, 267, 259, 270, 269, 290] payload_rows_per_dst=[5450, 5404, 5434, 5444, 5434, 5438, 5504, 5456] dup_rows_per_dst=[2766, 2744, 2632, 2854, 2764, 2678, 2690, 2844] combine_rows_per_src=[5450, 5404, 5434, 5444, 5434, 5438, 5504, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44036 payload_rows_remote=38528 dup_rows=21500 dup_rows_remote=18896 dedup_ratio=0.328064 router_tokens_per_expert=[269, 247, 252, 258, 240, 244, 271, 265, 216, 269, 232, 301, 255, 248, 249, 276, 268, 226, 257, 256, 244, 247, 261, 254, 253, 250, 261, 276, 259, 231, 249, 269, 262, 243, 264, 267, 256, 248, 263, 271, 264, 246, 251, 242, 253, 245, 310, 238, 282, 281, 239, 243, 262, 259, 260, 264, 255, 236, 237, 245, 271, 240, 277, 272, 235, 253, 249, 262, 250, 253, 257, 264, 244, 245, 278, 245, 244, 227, 256, 294, 270, 255, 284, 287, 287, 270, 268, 251, 240, 263, 249, 274, 227, 253, 237, 291, 266, 243, 257, 254, 249, 251, 259, 236, 243, 266, 241, 233, 263, 254, 256, 258, 248, 236, 273, 241, 283, 239, 266, 256, 257, 276, 255, 233, 242, 273, 246, 254] payload_rows_per_dst=[5498, 5446, 5558, 5492, 5508, 5568, 5452, 5514] dup_rows_per_dst=[2686, 2676, 2688, 2754, 2604, 2844, 2606, 2642] combine_rows_per_src=[5498, 5446, 5558, 5492, 5508, 5568, 5452, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38278 dup_rows=21816 dup_rows_remote=19110 dedup_ratio=0.332886 router_tokens_per_expert=[243, 229, 247, 236, 269, 259, 289, 265, 247, 236, 247, 245, 276, 251, 272, 269, 227, 272, 272, 278, 215, 259, 241, 253, 256, 232, 263, 243, 249, 252, 255, 280, 260, 222, 260, 252, 281, 259, 273, 244, 264, 243, 252, 242, 277, 241, 252, 274, 267, 272, 255, 248, 250, 277, 252, 254, 239, 274, 265, 253, 261, 240, 236, 252, 244, 254, 234, 264, 251, 244, 258, 268, 249, 242, 270, 240, 264, 225, 244, 247, 263, 243, 275, 256, 246, 249, 250, 251, 254, 249, 279, 264, 254, 239, 261, 241, 267, 260, 265, 279, 265, 282, 282, 267, 257, 279, 263, 244, 268, 279, 263, 268, 259, 284, 269, 256, 271, 288, 226, 263, 273, 263, 256, 221, 246, 241, 240, 234] payload_rows_per_dst=[5442, 5400, 5496, 5458, 5338, 5442, 5628, 5516] dup_rows_per_dst=[2718, 2694, 2696, 2732, 2658, 2706, 2948, 2664] combine_rows_per_src=[5442, 5400, 5496, 5458, 5338, 5442, 5628, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38294 dup_rows=21796 dup_rows_remote=19064 dedup_ratio=0.332581 router_tokens_per_expert=[276, 257, 269, 247, 241, 250, 265, 251, 269, 244, 243, 280, 305, 292, 279, 277, 277, 271, 245, 291, 263, 258, 283, 242, 250, 262, 236, 259, 251, 268, 254, 255, 275, 238, 224, 235, 263, 265, 271, 266, 264, 282, 259, 227, 248, 261, 238, 273, 263, 257, 245, 266, 252, 253, 264, 247, 261, 242, 250, 242, 271, 249, 277, 250, 277, 261, 248, 239, 214, 265, 252, 256, 242, 258, 248, 271, 230, 237, 225, 272, 252, 235, 260, 243, 268, 211, 242, 252, 256, 244, 256, 250, 280, 259, 257, 265, 238, 239, 272, 250, 240, 253, 276, 237, 256, 223, 270, 263, 263, 292, 227, 249, 262, 232, 269, 272, 244, 263, 251, 254, 287, 286, 242, 247, 233, 251, 261, 253] payload_rows_per_dst=[5616, 5554, 5438, 5446, 5400, 5442, 5422, 5422] dup_rows_per_dst=[2874, 2776, 2740, 2732, 2590, 2618, 2674, 2792] combine_rows_per_src=[5616, 5554, 5438, 5446, 5400, 5442, 5422, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38254 dup_rows=21876 dup_rows_remote=19174 dedup_ratio=0.333801 router_tokens_per_expert=[261, 256, 241, 265, 238, 249, 288, 278, 264, 251, 248, 262, 235, 244, 256, 254, 277, 252, 242, 240, 263, 260, 250, 242, 239, 259, 230, 292, 260, 257, 235, 256, 281, 270, 290, 229, 247, 270, 272, 243, 243, 269, 255, 265, 252, 268, 269, 252, 250, 244, 281, 261, 267, 259, 284, 249, 267, 276, 251, 236, 230, 270, 234, 256, 250, 274, 249, 251, 267, 251, 258, 250, 263, 278, 278, 247, 257, 275, 225, 244, 241, 240, 262, 260, 252, 249, 228, 244, 257, 263, 241, 245, 238, 275, 251, 273, 271, 265, 237, 225, 251, 265, 263, 263, 241, 259, 259, 293, 256, 236, 242, 242, 271, 250, 270, 250, 241, 246, 245, 265, 257, 283, 265, 283, 268, 243, 235, 258] payload_rows_per_dst=[5422, 5406, 5524, 5464, 5524, 5358, 5432, 5530] dup_rows_per_dst=[2758, 2702, 2826, 2766, 2710, 2680, 2704, 2730] combine_rows_per_src=[5422, 5406, 5524, 5464, 5524, 5358, 5432, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229224 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122408 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015592 weight_movement_bytes=18874368 total_nvlink_bytes=332889960 weight_to_token_ratio=0.060106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157630464 dispatch_meta_bytes=229848 combine_payload_bytes=157630464 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157860312 ingress_bytes=176504832 link_bytes=176504832 token_bytes=315490776 weight_movement_bytes=18874368 total_nvlink_bytes=334365144 weight_to_token_ratio=0.059825 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691182454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229368 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156123128 ingress_bytes=231391232 link_bytes=231391232 token_bytes=312016888 weight_movement_bytes=75497472 total_nvlink_bytes=387514360 weight_to_token_ratio=0.241966 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217076933e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157810688 dispatch_meta_bytes=229696 combine_payload_bytes=157810688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158040384 ingress_bytes=176685056 link_bytes=176685056 token_bytes=315851072 weight_movement_bytes=18874368 total_nvlink_bytes=334725440 weight_to_token_ratio=0.059757 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692909272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229552 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016240 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313802928 weight_movement_bytes=18874368 total_nvlink_bytes=332677296 weight_to_token_ratio=0.060147 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38392 dup_rows=21704 dup_rows_remote=19086 dedup_ratio=0.331177 router_tokens_per_expert=[260, 261, 258, 243, 264, 225, 267, 253, 229, 265, 252, 241, 246, 237, 272, 249, 246, 242, 247, 281, 269, 244, 234, 279, 266, 248, 270, 251, 231, 281, 240, 257, 272, 252, 279, 224, 273, 272, 270, 263, 247, 253, 265, 272, 254, 241, 286, 261, 275, 273, 263, 292, 229, 260, 252, 251, 263, 253, 241, 244, 266, 248, 262, 256, 265, 259, 265, 245, 263, 232, 255, 250, 286, 243, 260, 258, 251, 277, 237, 236, 264, 262, 242, 283, 267, 256, 263, 272, 240, 261, 252, 262, 232, 279, 249, 280, 251, 264, 240, 226, 253, 250, 259, 245, 241, 265, 267, 241, 249, 257, 276, 285, 239, 238, 275, 218, 286, 269, 233, 242, 240, 265, 238, 251, 263, 247, 281, 248] payload_rows_per_dst=[5440, 5512, 5496, 5488, 5472, 5744, 5424, 5256] dup_rows_per_dst=[2618, 2646, 2872, 2768, 2692, 2970, 2714, 2424] combine_rows_per_src=[5440, 5512, 5496, 5488, 5472, 5744, 5424, 5256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229432 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081656 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933880 weight_movement_bytes=18874368 total_nvlink_bytes=332808248 weight_to_token_ratio=0.060122 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229712 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156918096 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606480 weight_movement_bytes=18874368 total_nvlink_bytes=332480848 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229912 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157483544 ingress_bytes=213876736 link_bytes=213876736 token_bytes=314737176 weight_movement_bytes=56623104 total_nvlink_bytes=371360280 weight_to_token_ratio=0.179906 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049261650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38294 dup_rows=21774 dup_rows_remote=19058 dedup_ratio=0.332245 router_tokens_per_expert=[267, 266, 244, 265, 259, 254, 248, 240, 292, 252, 271, 261, 289, 274, 259, 236, 263, 241, 282, 262, 244, 232, 245, 233, 235, 241, 272, 219, 277, 265, 257, 261, 223, 252, 263, 260, 259, 243, 243, 249, 258, 256, 263, 273, 238, 270, 261, 277, 254, 267, 264, 260, 221, 245, 238, 253, 253, 253, 249, 268, 249, 250, 256, 266, 264, 275, 255, 236, 253, 246, 273, 254, 285, 265, 270, 258, 275, 251, 287, 242, 247, 244, 258, 238, 266, 265, 261, 266, 281, 258, 219, 235, 243, 252, 254, 252, 250, 244, 247, 242, 240, 249, 234, 269, 274, 267, 259, 287, 293, 268, 232, 254, 264, 260, 260, 244, 228, 247, 249, 305, 240, 271, 281, 262, 242, 253, 238, 247] payload_rows_per_dst=[5568, 5408, 5488, 5370, 5534, 5420, 5506, 5468] dup_rows_per_dst=[2786, 2650, 2590, 2722, 2842, 2658, 2810, 2716] combine_rows_per_src=[5568, 5408, 5488, 5370, 5534, 5420, 5506, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38350 dup_rows=21904 dup_rows_remote=19254 dedup_ratio=0.334229 router_tokens_per_expert=[263, 265, 244, 288, 258, 240, 229, 259, 249, 232, 245, 278, 240, 244, 271, 272, 233, 255, 233, 267, 285, 270, 267, 269, 254, 259, 261, 268, 239, 241, 245, 255, 229, 223, 235, 261, 245, 267, 255, 249, 273, 255, 248, 265, 261, 259, 267, 257, 269, 252, 282, 260, 287, 259, 234, 280, 222, 260, 235, 293, 246, 237, 252, 255, 245, 256, 252, 288, 235, 257, 226, 279, 261, 259, 297, 255, 257, 265, 257, 254, 293, 259, 283, 250, 279, 209, 253, 308, 241, 274, 280, 246, 222, 253, 249, 232, 274, 285, 253, 245, 239, 230, 249, 237, 244, 209, 279, 265, 246, 252, 211, 248, 269, 278, 253, 260, 267, 236, 284, 275, 268, 257, 251, 213, 268, 267, 265, 267] payload_rows_per_dst=[5500, 5488, 5444, 5492, 5492, 5442, 5282, 5492] dup_rows_per_dst=[2654, 2714, 2654, 2754, 2794, 2820, 2650, 2864] combine_rows_per_src=[5500, 5488, 5444, 5492, 5492, 5442, 5282, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38100 dup_rows=21924 dup_rows_remote=19042 dedup_ratio=0.334534 router_tokens_per_expert=[248, 236, 234, 266, 258, 248, 256, 252, 277, 249, 245, 250, 240, 265, 257, 251, 249, 254, 271, 253, 232, 257, 257, 250, 240, 275, 274, 280, 235, 251, 277, 265, 256, 225, 262, 262, 284, 293, 240, 266, 273, 305, 238, 252, 276, 264, 253, 248, 246, 243, 278, 255, 261, 279, 230, 259, 255, 278, 240, 248, 232, 251, 250, 235, 283, 261, 232, 226, 253, 237, 275, 259, 268, 283, 286, 271, 242, 222, 281, 302, 258, 278, 239, 283, 250, 286, 255, 250, 228, 259, 238, 238, 268, 271, 239, 236, 242, 261, 244, 257, 284, 262, 260, 242, 265, 266, 258, 252, 259, 257, 227, 237, 244, 271, 275, 249, 245, 261, 232, 282, 218, 257, 268, 255, 237, 269, 261, 225] payload_rows_per_dst=[5372, 5388, 5512, 5502, 5560, 5478, 5400, 5400] dup_rows_per_dst=[2692, 2852, 2882, 2578, 2802, 2674, 2746, 2698] combine_rows_per_src=[5372, 5388, 5512, 5502, 5560, 5478, 5400, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38530 dup_rows=21598 dup_rows_remote=18914 dedup_ratio=0.329559 router_tokens_per_expert=[227, 232, 263, 257, 270, 268, 254, 228, 248, 228, 242, 285, 249, 255, 260, 266, 268, 287, 268, 264, 259, 253, 254, 251, 271, 239, 251, 241, 281, 247, 264, 256, 276, 269, 282, 279, 238, 273, 260, 252, 243, 261, 259, 285, 264, 240, 279, 241, 270, 255, 274, 232, 251, 252, 244, 281, 256, 262, 282, 244, 292, 272, 254, 236, 248, 240, 255, 274, 278, 251, 263, 234, 254, 273, 238, 246, 249, 255, 245, 243, 247, 242, 234, 264, 259, 236, 260, 270, 232, 263, 248, 262, 276, 237, 234, 243, 291, 250, 264, 254, 297, 254, 256, 260, 255, 302, 239, 261, 259, 244, 260, 267, 264, 239, 258, 257, 238, 277, 249, 214, 228, 231, 234, 259, 254, 278, 234, 244] payload_rows_per_dst=[5460, 5578, 5608, 5494, 5408, 5410, 5588, 5392] dup_rows_per_dst=[2604, 2730, 2794, 2820, 2684, 2604, 2838, 2524] combine_rows_per_src=[5460, 5578, 5608, 5494, 5408, 5410, 5588, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38162 dup_rows=21852 dup_rows_remote=19022 dedup_ratio=0.333435 router_tokens_per_expert=[240, 245, 265, 269, 260, 251, 231, 246, 273, 281, 251, 244, 269, 270, 260, 244, 240, 259, 255, 256, 239, 241, 271, 259, 256, 259, 243, 258, 271, 227, 226, 272, 293, 224, 279, 263, 263, 250, 256, 244, 263, 252, 245, 259, 271, 255, 256, 245, 287, 243, 235, 253, 254, 263, 259, 250, 268, 231, 254, 238, 263, 238, 254, 272, 242, 235, 255, 287, 256, 243, 291, 259, 240, 230, 263, 246, 242, 251, 235, 262, 256, 249, 255, 282, 265, 279, 275, 246, 225, 259, 267, 253, 267, 254, 285, 259, 266, 264, 256, 284, 276, 256, 274, 247, 234, 253, 246, 273, 250, 256, 241, 254, 263, 253, 269, 263, 245, 251, 262, 247, 239, 261, 246, 237, 250, 284, 281, 263] payload_rows_per_dst=[5488, 5426, 5460, 5440, 5416, 5522, 5474, 5458] dup_rows_per_dst=[2710, 2638, 2776, 2684, 2658, 2830, 2786, 2770] combine_rows_per_src=[5488, 5426, 5460, 5440, 5416, 5522, 5474, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38128 dup_rows=21982 dup_rows_remote=19290 dedup_ratio=0.335419 router_tokens_per_expert=[249, 279, 229, 269, 236, 252, 257, 224, 262, 247, 214, 272, 252, 236, 287, 276, 255, 276, 268, 238, 257, 237, 276, 282, 273, 254, 271, 269, 264, 258, 242, 248, 208, 267, 262, 242, 278, 245, 279, 243, 277, 255, 238, 275, 243, 244, 257, 247, 257, 249, 251, 250, 275, 265, 258, 260, 257, 247, 236, 278, 236, 244, 242, 254, 246, 246, 258, 268, 249, 282, 255, 250, 240, 254, 275, 290, 262, 244, 255, 232, 241, 269, 276, 264, 241, 225, 254, 283, 266, 266, 267, 254, 258, 249, 254, 243, 225, 279, 237, 256, 244, 264, 274, 259, 233, 269, 244, 279, 274, 281, 269, 238, 253, 259, 242, 271, 265, 228, 240, 243, 238, 262, 255, 270, 262, 268, 280, 263] payload_rows_per_dst=[5490, 5444, 5430, 5426, 5380, 5486, 5468, 5430] dup_rows_per_dst=[2592, 2892, 2690, 2692, 2832, 2734, 2782, 2768] combine_rows_per_src=[5490, 5444, 5430, 5426, 5380, 5486, 5468, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229408 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081632 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933856 weight_movement_bytes=37748736 total_nvlink_bytes=351682592 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38372 dup_rows=21736 dup_rows_remote=19078 dedup_ratio=0.331665 router_tokens_per_expert=[224, 265, 248, 276, 248, 289, 291, 234, 238, 246, 244, 267, 251, 245, 279, 263, 256, 211, 266, 266, 268, 247, 258, 248, 240, 275, 246, 246, 262, 257, 233, 264, 251, 248, 251, 258, 255, 265, 257, 248, 249, 259, 260, 258, 266, 249, 257, 277, 240, 247, 273, 261, 255, 292, 276, 239, 281, 236, 258, 252, 252, 271, 264, 246, 259, 240, 266, 256, 251, 274, 257, 284, 255, 260, 248, 238, 265, 242, 257, 253, 236, 267, 263, 265, 242, 258, 260, 255, 247, 276, 254, 243, 268, 249, 269, 252, 265, 259, 281, 264, 247, 242, 239, 249, 260, 249, 248, 261, 264, 209, 275, 257, 239, 271, 258, 259, 264, 250, 285, 249, 268, 230, 237, 238, 282, 277, 258, 223] payload_rows_per_dst=[5496, 5428, 5496, 5582, 5490, 5398, 5468, 5442] dup_rows_per_dst=[2720, 2658, 2720, 2704, 2720, 2810, 2670, 2734] combine_rows_per_src=[5496, 5428, 5496, 5582, 5490, 5398, 5468, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=230416 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157312016 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314393616 weight_movement_bytes=18874368 total_nvlink_bytes=333267984 weight_to_token_ratio=0.060034 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=228568 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156286168 ingress_bytes=193806336 link_bytes=193806336 token_bytes=312343768 weight_movement_bytes=37748736 total_nvlink_bytes=350092504 weight_to_token_ratio=0.120856 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856956953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=229776 combine_payload_bytes=157818880 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=158048656 ingress_bytes=176693248 link_bytes=176693248 token_bytes=315867536 weight_movement_bytes=18874368 total_nvlink_bytes=334741904 weight_to_token_ratio=0.059754 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692987764e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38380 dup_rows=21670 dup_rows_remote=18962 dedup_ratio=0.330658 router_tokens_per_expert=[246, 256, 243, 268, 239, 260, 264, 262, 266, 247, 231, 256, 242, 272, 236, 275, 265, 244, 264, 257, 253, 247, 272, 273, 229, 244, 253, 243, 217, 251, 250, 243, 274, 269, 258, 261, 256, 257, 273, 283, 275, 229, 232, 251, 261, 252, 287, 250, 256, 267, 252, 256, 296, 237, 236, 235, 271, 267, 241, 267, 252, 240, 245, 251, 267, 255, 258, 286, 263, 258, 262, 246, 259, 246, 265, 277, 267, 277, 268, 258, 257, 257, 238, 258, 273, 240, 268, 254, 258, 264, 250, 250, 270, 273, 254, 261, 262, 237, 266, 254, 275, 254, 242, 249, 260, 250, 249, 240, 257, 296, 250, 277, 252, 253, 235, 227, 273, 241, 265, 260, 257, 249, 261, 238, 263, 217, 263, 254] payload_rows_per_dst=[5486, 5394, 5496, 5520, 5700, 5536, 5342, 5392] dup_rows_per_dst=[2708, 2548, 2898, 2730, 2820, 2714, 2628, 2624] combine_rows_per_src=[5486, 5394, 5496, 5520, 5700, 5536, 5342, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228736 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540288 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851840 weight_movement_bytes=18874368 total_nvlink_bytes=331726208 weight_to_token_ratio=0.060330 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229672 combine_payload_bytes=156172288 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156401960 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312574248 weight_movement_bytes=18874368 total_nvlink_bytes=331448616 weight_to_token_ratio=0.060384 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229800 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157401512 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314573224 weight_movement_bytes=18874368 total_nvlink_bytes=333447592 weight_to_token_ratio=0.060000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229368 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157433848 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638328 weight_movement_bytes=18874368 total_nvlink_bytes=333512696 weight_to_token_ratio=0.059988 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38418 dup_rows=21728 dup_rows_remote=19072 dedup_ratio=0.331543 router_tokens_per_expert=[279, 266, 271, 273, 259, 250, 257, 270, 266, 265, 264, 254, 261, 286, 252, 240, 279, 259, 248, 291, 243, 255, 264, 279, 263, 269, 247, 288, 250, 281, 241, 271, 246, 263, 253, 258, 259, 242, 232, 241, 259, 261, 255, 233, 269, 278, 257, 266, 267, 223, 243, 244, 278, 257, 221, 267, 256, 251, 242, 255, 236, 255, 252, 256, 226, 239, 274, 279, 230, 270, 266, 266, 244, 275, 264, 260, 260, 264, 238, 289, 259, 246, 259, 255, 273, 257, 252, 241, 222, 255, 272, 257, 227, 238, 238, 260, 245, 258, 257, 243, 261, 230, 257, 264, 246, 266, 230, 249, 289, 252, 258, 269, 240, 259, 255, 259, 271, 236, 233, 282, 253, 217, 253, 244, 276, 241, 240, 264] payload_rows_per_dst=[5312, 5622, 5454, 5466, 5498, 5404, 5662, 5390] dup_rows_per_dst=[2542, 2834, 2562, 2540, 2790, 2746, 3058, 2656] combine_rows_per_src=[5312, 5622, 5454, 5466, 5498, 5404, 5662, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38194 dup_rows=21844 dup_rows_remote=19166 dedup_ratio=0.333313 router_tokens_per_expert=[284, 263, 274, 281, 278, 223, 263, 255, 255, 270, 278, 240, 259, 266, 260, 230, 288, 256, 249, 265, 260, 233, 249, 266, 273, 248, 275, 262, 264, 250, 263, 257, 287, 265, 223, 252, 228, 248, 243, 270, 271, 257, 252, 267, 259, 270, 263, 233, 273, 251, 243, 237, 270, 242, 242, 243, 257, 242, 256, 276, 229, 255, 249, 238, 271, 272, 257, 254, 246, 267, 257, 237, 250, 275, 258, 265, 260, 291, 275, 263, 259, 256, 279, 251, 264, 269, 236, 270, 260, 252, 243, 240, 243, 279, 254, 252, 242, 264, 247, 230, 278, 235, 247, 272, 259, 221, 257, 259, 261, 228, 255, 252, 245, 254, 277, 236, 255, 250, 259, 256, 245, 248, 242, 261, 223, 260, 239, 278] payload_rows_per_dst=[5508, 5512, 5498, 5356, 5558, 5476, 5390, 5394] dup_rows_per_dst=[2850, 2804, 2678, 2650, 2838, 2738, 2624, 2662] combine_rows_per_src=[5508, 5512, 5498, 5356, 5558, 5476, 5390, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38262 dup_rows=21834 dup_rows_remote=19064 dedup_ratio=0.333160 router_tokens_per_expert=[275, 275, 266, 257, 255, 265, 240, 266, 233, 248, 269, 253, 227, 270, 243, 276, 245, 220, 265, 258, 271, 266, 267, 213, 267, 260, 270, 257, 273, 261, 264, 254, 239, 273, 239, 254, 248, 281, 258, 264, 249, 246, 263, 252, 256, 244, 266, 263, 232, 277, 292, 258, 244, 276, 251, 257, 259, 248, 237, 272, 231, 275, 249, 230, 229, 259, 260, 247, 242, 261, 257, 252, 266, 248, 255, 246, 263, 251, 261, 267, 275, 251, 229, 255, 252, 251, 260, 283, 263, 248, 279, 246, 265, 257, 256, 261, 227, 228, 255, 247, 254, 263, 260, 244, 255, 263, 267, 261, 319, 275, 240, 247, 260, 231, 282, 255, 214, 250, 230, 279, 249, 244, 256, 266, 257, 250, 263, 270] payload_rows_per_dst=[5520, 5422, 5446, 5516, 5464, 5484, 5440, 5410] dup_rows_per_dst=[2716, 2800, 2744, 2660, 2664, 2778, 2770, 2702] combine_rows_per_src=[5520, 5422, 5446, 5516, 5464, 5484, 5440, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38286 dup_rows=21736 dup_rows_remote=19050 dedup_ratio=0.331665 router_tokens_per_expert=[249, 246, 284, 268, 313, 251, 260, 269, 237, 219, 266, 277, 243, 277, 252, 268, 268, 264, 263, 243, 244, 246, 250, 231, 249, 280, 255, 267, 251, 270, 258, 252, 258, 262, 244, 252, 255, 277, 250, 250, 240, 261, 265, 257, 252, 250, 259, 247, 289, 283, 270, 265, 255, 277, 224, 256, 255, 262, 243, 273, 247, 239, 251, 241, 269, 254, 273, 282, 224, 262, 274, 260, 279, 253, 227, 257, 250, 249, 252, 235, 238, 238, 285, 250, 263, 239, 243, 262, 253, 253, 234, 263, 265, 254, 250, 242, 237, 240, 241, 256, 271, 229, 276, 252, 260, 239, 218, 245, 222, 274, 253, 262, 256, 293, 255, 265, 233, 238, 241, 281, 274, 265, 256, 280, 270, 254, 268, 253] payload_rows_per_dst=[5510, 5476, 5444, 5490, 5514, 5424, 5426, 5516] dup_rows_per_dst=[2848, 2706, 2714, 2770, 2686, 2640, 2524, 2848] combine_rows_per_src=[5510, 5476, 5444, 5490, 5514, 5424, 5426, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229960 combine_payload_bytes=157360128 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157590088 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314950216 weight_movement_bytes=37748736 total_nvlink_bytes=352698952 weight_to_token_ratio=0.119856 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38316 dup_rows=21774 dup_rows_remote=19102 dedup_ratio=0.332245 router_tokens_per_expert=[270, 260, 243, 250, 277, 246, 270, 271, 260, 258, 232, 278, 252, 279, 278, 251, 256, 226, 252, 243, 253, 256, 244, 248, 260, 262, 260, 222, 262, 239, 243, 250, 257, 237, 224, 269, 260, 252, 249, 270, 255, 266, 262, 236, 268, 255, 257, 254, 252, 255, 258, 248, 242, 243, 259, 254, 241, 250, 250, 267, 276, 270, 246, 248, 252, 253, 247, 274, 265, 264, 246, 244, 250, 262, 265, 267, 285, 235, 244, 240, 258, 247, 257, 237, 249, 268, 253, 241, 251, 241, 266, 255, 253, 275, 246, 246, 274, 244, 268, 229, 276, 268, 257, 267, 258, 253, 250, 266, 260, 281, 266, 237, 252, 248, 259, 267, 274, 264, 266, 270, 269, 253, 261, 263, 250, 254, 276, 271] payload_rows_per_dst=[5472, 5380, 5446, 5446, 5502, 5436, 5530, 5550] dup_rows_per_dst=[2878, 2572, 2696, 2672, 2684, 2650, 2778, 2844] combine_rows_per_src=[5472, 5380, 5446, 5446, 5502, 5436, 5530, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38234 dup_rows=21850 dup_rows_remote=19174 dedup_ratio=0.333405 router_tokens_per_expert=[257, 258, 253, 276, 263, 262, 231, 223, 266, 253, 259, 258, 290, 252, 275, 265, 238, 260, 261, 241, 271, 253, 258, 240, 253, 263, 240, 279, 231, 274, 250, 267, 263, 255, 255, 230, 238, 276, 294, 245, 252, 257, 287, 273, 268, 245, 274, 281, 271, 278, 255, 249, 240, 239, 241, 260, 278, 258, 256, 235, 275, 229, 271, 240, 267, 263, 263, 280, 252, 268, 260, 252, 245, 270, 253, 255, 270, 241, 255, 274, 225, 256, 266, 235, 254, 247, 258, 246, 246, 252, 267, 266, 259, 255, 256, 276, 280, 268, 259, 242, 246, 258, 257, 252, 272, 238, 242, 241, 256, 239, 215, 231, 222, 231, 252, 279, 259, 262, 261, 235, 247, 258, 263, 255, 245, 284, 257, 242] payload_rows_per_dst=[5462, 5468, 5488, 5558, 5494, 5452, 5356, 5408] dup_rows_per_dst=[2820, 2690, 2898, 2592, 2842, 2676, 2636, 2696] combine_rows_per_src=[5462, 5468, 5488, 5558, 5494, 5452, 5356, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229440 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156672064 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114688 weight_movement_bytes=37748736 total_nvlink_bytes=350863424 weight_to_token_ratio=0.120559 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229304 combine_payload_bytes=156721152 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156950456 ingress_bytes=251092992 link_bytes=251092992 token_bytes=313671608 weight_movement_bytes=94371840 total_nvlink_bytes=408043448 weight_to_token_ratio=0.300862 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405849504e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38200 dup_rows=21914 dup_rows_remote=19336 dedup_ratio=0.334381 router_tokens_per_expert=[263, 232, 252, 255, 278, 262, 215, 257, 253, 274, 226, 269, 249, 274, 261, 253, 273, 268, 241, 247, 255, 260, 259, 260, 235, 231, 262, 248, 239, 220, 259, 243, 246, 271, 281, 245, 258, 260, 279, 251, 243, 223, 261, 246, 266, 267, 263, 257, 242, 240, 240, 256, 240, 278, 276, 279, 266, 276, 255, 259, 213, 242, 253, 278, 254, 253, 231, 260, 285, 231, 258, 265, 251, 249, 265, 298, 275, 268, 269, 262, 262, 250, 257, 252, 266, 273, 292, 254, 244, 277, 245, 259, 283, 264, 307, 239, 258, 253, 247, 235, 257, 256, 246, 245, 239, 230, 247, 253, 258, 290, 234, 229, 245, 239, 243, 259, 249, 259, 277, 261, 277, 248, 264, 259, 257, 227, 253, 293] payload_rows_per_dst=[5412, 5422, 5524, 5452, 5548, 5490, 5332, 5442] dup_rows_per_dst=[2734, 2578, 2710, 2734, 2800, 2958, 2622, 2778] combine_rows_per_src=[5412, 5422, 5524, 5452, 5548, 5490, 5332, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229344 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157048800 ingress_bytes=213442560 link_bytes=213442560 token_bytes=313868256 weight_movement_bytes=56623104 total_nvlink_bytes=370491360 weight_to_token_ratio=0.180404 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045101590e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=37982 dup_rows=21970 dup_rows_remote=19000 dedup_ratio=0.335236 router_tokens_per_expert=[260, 266, 228, 284, 257, 255, 251, 246, 252, 253, 250, 229, 275, 247, 281, 270, 260, 256, 272, 263, 258, 252, 230, 280, 264, 239, 266, 264, 247, 235, 272, 250, 256, 280, 268, 279, 236, 245, 266, 265, 245, 242, 260, 222, 236, 249, 248, 258, 273, 242, 243, 261, 255, 252, 237, 278, 272, 273, 264, 276, 271, 268, 255, 256, 233, 268, 274, 267, 265, 268, 273, 260, 238, 268, 266, 268, 248, 241, 245, 237, 235, 264, 237, 297, 241, 279, 236, 268, 236, 263, 244, 251, 271, 274, 252, 246, 273, 276, 250, 262, 247, 269, 261, 238, 239, 231, 234, 228, 263, 267, 238, 268, 257, 242, 239, 222, 279, 265, 245, 249, 268, 259, 224, 296, 289, 246, 244, 244] payload_rows_per_dst=[5584, 5286, 5618, 5520, 5494, 5444, 5358, 5262] dup_rows_per_dst=[2970, 2628, 3002, 2722, 2798, 2646, 2730, 2474] combine_rows_per_src=[5584, 5286, 5618, 5520, 5494, 5444, 5358, 5262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229672 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=157172008 ingress_bytes=251314176 link_bytes=251314176 token_bytes=314114344 weight_movement_bytes=94371840 total_nvlink_bytes=408486184 weight_to_token_ratio=0.300438 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407968780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229632 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156836096 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313442560 weight_movement_bytes=18874368 total_nvlink_bytes=332316928 weight_to_token_ratio=0.060216 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=230144 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156697344 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313164544 weight_movement_bytes=37748736 total_nvlink_bytes=350913280 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38170 dup_rows=21816 dup_rows_remote=18974 dedup_ratio=0.332886 router_tokens_per_expert=[260, 253, 267, 223, 279, 253, 265, 258, 266, 288, 256, 240, 276, 270, 233, 272, 266, 263, 281, 243, 246, 265, 265, 300, 242, 271, 287, 255, 268, 263, 246, 257, 231, 239, 236, 246, 280, 244, 279, 260, 266, 252, 242, 244, 211, 236, 242, 253, 255, 282, 245, 242, 256, 284, 284, 264, 251, 258, 239, 261, 266, 242, 253, 247, 255, 239, 267, 228, 270, 255, 264, 245, 288, 239, 261, 264, 249, 240, 284, 232, 249, 256, 269, 230, 267, 307, 256, 245, 249, 247, 246, 248, 245, 244, 220, 256, 256, 243, 254, 241, 237, 266, 252, 263, 247, 270, 276, 260, 244, 254, 239, 249, 259, 246, 260, 257, 267, 270, 285, 255, 254, 259, 230, 279, 266, 251, 238, 260] payload_rows_per_dst=[5402, 5354, 5308, 5506, 5530, 5480, 5590, 5550] dup_rows_per_dst=[2690, 2638, 2566, 2752, 2778, 2742, 2808, 2842] combine_rows_per_src=[5402, 5354, 5308, 5506, 5530, 5480, 5590, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=155574272 dispatch_meta_bytes=227928 combine_payload_bytes=155574272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155802200 ingress_bytes=174448640 link_bytes=174448640 token_bytes=311376472 weight_movement_bytes=18874368 total_nvlink_bytes=330250840 weight_to_token_ratio=0.060616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671481034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43512 payload_rows_remote=38056 dup_rows=22024 dup_rows_remote=19236 dedup_ratio=0.336060 router_tokens_per_expert=[271, 254, 254, 262, 257, 271, 266, 254, 282, 248, 267, 242, 284, 251, 243, 250, 259, 256, 237, 249, 262, 246, 235, 277, 225, 255, 266, 273, 258, 274, 262, 229, 255, 257, 237, 266, 275, 270, 239, 246, 261, 273, 238, 261, 263, 240, 273, 268, 253, 272, 256, 271, 234, 261, 247, 264, 238, 223, 256, 274, 223, 277, 236, 255, 257, 254, 233, 263, 281, 268, 235, 238, 267, 241, 261, 262, 234, 247, 258, 249, 255, 267, 264, 238, 269, 238, 274, 233, 260, 253, 256, 249, 255, 262, 276, 263, 278, 274, 277, 242, 256, 239, 265, 256, 239, 254, 242, 264, 270, 266, 263, 259, 269, 263, 271, 239, 234, 230, 246, 258, 259, 265, 264, 265, 268, 266, 242, 244] payload_rows_per_dst=[5468, 5394, 5456, 5442, 5426, 5430, 5462, 5434] dup_rows_per_dst=[2844, 2732, 2788, 2638, 2670, 2794, 2826, 2732] combine_rows_per_src=[5468, 5394, 5456, 5442, 5426, 5430, 5462, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38118 dup_rows=21948 dup_rows_remote=19188 dedup_ratio=0.334900 router_tokens_per_expert=[251, 274, 239, 239, 244, 257, 259, 265, 250, 273, 228, 266, 274, 258, 230, 263, 253, 280, 247, 243, 258, 244, 243, 259, 251, 242, 247, 235, 265, 240, 250, 234, 263, 256, 260, 258, 250, 269, 240, 242, 277, 237, 256, 268, 242, 248, 240, 261, 278, 237, 241, 270, 227, 228, 262, 264, 271, 277, 265, 249, 242, 270, 235, 262, 259, 243, 291, 255, 273, 241, 257, 270, 295, 274, 264, 252, 252, 273, 246, 270, 264, 263, 258, 253, 240, 268, 249, 238, 262, 242, 264, 235, 244, 228, 250, 285, 267, 229, 259, 246, 245, 267, 270, 262, 245, 285, 274, 232, 237, 255, 287, 255, 263, 266, 248, 284, 259, 239, 234, 266, 267, 282, 255, 253, 276, 274, 262, 261] payload_rows_per_dst=[5404, 5374, 5398, 5420, 5562, 5440, 5470, 5520] dup_rows_per_dst=[2736, 2608, 2736, 2736, 2868, 2646, 2760, 2858] combine_rows_per_src=[5404, 5374, 5398, 5420, 5562, 5440, 5470, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228576 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156572896 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917216 weight_movement_bytes=18874368 total_nvlink_bytes=331791584 weight_to_token_ratio=0.060317 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38104 dup_rows=21910 dup_rows_remote=19210 dedup_ratio=0.334320 router_tokens_per_expert=[247, 264, 228, 252, 276, 274, 251, 252, 247, 246, 254, 244, 261, 242, 278, 255, 259, 255, 267, 264, 281, 241, 260, 271, 262, 261, 281, 258, 238, 278, 248, 245, 258, 245, 281, 270, 271, 256, 255, 276, 236, 245, 273, 257, 257, 263, 277, 269, 225, 257, 266, 250, 270, 246, 278, 259, 245, 248, 241, 274, 269, 240, 252, 229, 252, 229, 251, 257, 268, 255, 292, 264, 237, 250, 267, 255, 250, 272, 255, 257, 232, 277, 260, 236, 247, 227, 252, 239, 256, 264, 266, 248, 258, 226, 252, 254, 242, 233, 293, 226, 256, 245, 269, 263, 241, 240, 258, 266, 247, 270, 262, 270, 236, 258, 287, 247, 231, 259, 259, 270, 257, 296, 258, 275, 239, 254, 240, 238] payload_rows_per_dst=[5360, 5444, 5490, 5414, 5522, 5440, 5430, 5526] dup_rows_per_dst=[2782, 2894, 2888, 2684, 2700, 2548, 2732, 2682] combine_rows_per_src=[5360, 5444, 5490, 5414, 5522, 5440, 5430, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=229168 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156106544 ingress_bytes=193626112 link_bytes=193626112 token_bytes=311983920 weight_movement_bytes=37748736 total_nvlink_bytes=349732656 weight_to_token_ratio=0.120996 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855230135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38304 dup_rows=21768 dup_rows_remote=19136 dedup_ratio=0.332153 router_tokens_per_expert=[247, 257, 264, 259, 247, 266, 247, 272, 226, 257, 266, 254, 261, 248, 298, 254, 277, 262, 252, 238, 242, 257, 255, 257, 277, 249, 240, 260, 244, 270, 255, 266, 242, 256, 273, 263, 247, 255, 293, 240, 277, 269, 262, 253, 263, 258, 259, 258, 254, 266, 254, 232, 277, 244, 216, 254, 245, 263, 284, 263, 271, 242, 245, 238, 219, 269, 271, 246, 271, 267, 216, 274, 258, 259, 239, 270, 227, 259, 269, 257, 257, 236, 243, 251, 264, 244, 251, 246, 234, 265, 246, 262, 265, 247, 232, 265, 249, 274, 270, 265, 262, 256, 276, 263, 237, 266, 270, 240, 240, 268, 258, 253, 276, 241, 265, 252, 281, 255, 248, 247, 241, 274, 247, 258, 232, 237, 262, 286] payload_rows_per_dst=[5470, 5484, 5504, 5464, 5468, 5448, 5446, 5484] dup_rows_per_dst=[2776, 2718, 2832, 2632, 2674, 2568, 2848, 2720] combine_rows_per_src=[5470, 5484, 5504, 5464, 5468, 5448, 5446, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38122 dup_rows=21928 dup_rows_remote=19166 dedup_ratio=0.334595 router_tokens_per_expert=[266, 267, 234, 265, 262, 247, 267, 259, 251, 265, 267, 266, 243, 271, 246, 275, 224, 232, 233, 255, 265, 267, 264, 243, 261, 269, 245, 246, 237, 264, 238, 252, 229, 270, 234, 263, 261, 261, 232, 255, 253, 261, 250, 252, 287, 252, 281, 250, 242, 269, 275, 257, 284, 251, 242, 280, 245, 248, 249, 257, 240, 275, 287, 240, 259, 240, 245, 233, 251, 232, 243, 251, 247, 252, 223, 254, 238, 262, 263, 239, 269, 270, 236, 284, 244, 238, 240, 257, 244, 251, 270, 302, 264, 242, 257, 256, 265, 273, 264, 273, 261, 250, 265, 274, 291, 248, 242, 288, 236, 256, 268, 259, 247, 230, 289, 258, 247, 277, 233, 287, 251, 246, 262, 254, 267, 275, 251, 247] payload_rows_per_dst=[5502, 5372, 5428, 5484, 5374, 5486, 5518, 5444] dup_rows_per_dst=[2800, 2618, 2754, 2798, 2490, 2762, 2908, 2798] combine_rows_per_src=[5502, 5372, 5428, 5484, 5374, 5486, 5518, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38366 dup_rows=21818 dup_rows_remote=19150 dedup_ratio=0.332916 router_tokens_per_expert=[246, 280, 269, 211, 250, 266, 254, 249, 289, 260, 244, 251, 251, 238, 261, 245, 233, 252, 248, 243, 252, 266, 251, 246, 261, 225, 232, 252, 272, 245, 253, 279, 286, 258, 289, 263, 250, 251, 274, 270, 239, 251, 255, 215, 251, 262, 279, 260, 278, 268, 267, 239, 304, 243, 264, 258, 253, 270, 249, 264, 271, 274, 266, 260, 268, 247, 265, 266, 268, 239, 263, 291, 242, 234, 214, 226, 288, 220, 258, 251, 272, 273, 227, 275, 275, 256, 283, 262, 256, 253, 296, 251, 249, 256, 269, 263, 267, 249, 240, 255, 230, 256, 253, 233, 243, 236, 253, 229, 253, 251, 245, 281, 262, 280, 233, 242, 250, 262, 256, 235, 248, 259, 241, 271, 274, 250, 247, 273] payload_rows_per_dst=[5468, 5352, 5534, 5552, 5460, 5528, 5398, 5426] dup_rows_per_dst=[2660, 2668, 2772, 2904, 2620, 2904, 2550, 2740] combine_rows_per_src=[5468, 5352, 5534, 5552, 5460, 5528, 5398, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229224 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156360552 ingress_bytes=212754432 link_bytes=212754432 token_bytes=312491880 weight_movement_bytes=56623104 total_nvlink_bytes=369114984 weight_to_token_ratio=0.181199 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038508286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229256 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156303240 ingress_bytes=174948352 link_bytes=174948352 token_bytes=312377224 weight_movement_bytes=18874368 total_nvlink_bytes=331251592 weight_to_token_ratio=0.060422 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676269029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229760 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122944 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314016128 weight_movement_bytes=18874368 total_nvlink_bytes=332890496 weight_to_token_ratio=0.060106 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229152 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376864 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524576 weight_movement_bytes=18874368 total_nvlink_bytes=331398944 weight_to_token_ratio=0.060393 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38210 dup_rows=21844 dup_rows_remote=19092 dedup_ratio=0.333313 router_tokens_per_expert=[260, 273, 260, 257, 252, 251, 227, 236, 262, 245, 284, 232, 257, 255, 257, 263, 246, 204, 262, 255, 258, 303, 253, 256, 242, 280, 254, 259, 248, 260, 271, 273, 246, 266, 275, 244, 251, 283, 262, 263, 270, 272, 279, 231, 265, 270, 261, 254, 246, 297, 212, 270, 239, 244, 237, 243, 246, 238, 267, 249, 241, 253, 261, 278, 297, 241, 257, 255, 275, 260, 227, 284, 249, 280, 262, 242, 246, 276, 230, 270, 240, 249, 236, 254, 270, 267, 232, 245, 262, 254, 265, 252, 250, 281, 247, 249, 276, 264, 245, 255, 240, 272, 247, 246, 264, 267, 288, 230, 238, 293, 221, 253, 270, 255, 257, 258, 260, 252, 264, 236, 269, 231, 252, 244, 250, 253, 245, 261] payload_rows_per_dst=[5482, 5494, 5524, 5440, 5484, 5426, 5414, 5428] dup_rows_per_dst=[2752, 2914, 2864, 2612, 2818, 2680, 2518, 2686] combine_rows_per_src=[5482, 5494, 5524, 5440, 5484, 5426, 5414, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230064 combine_payload_bytes=157147136 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157377200 ingress_bytes=194895872 link_bytes=194895872 token_bytes=314524336 weight_movement_bytes=37748736 total_nvlink_bytes=352273072 weight_to_token_ratio=0.120018 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867396351e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38270 dup_rows=21846 dup_rows_remote=19074 dedup_ratio=0.333344 router_tokens_per_expert=[229, 255, 253, 270, 277, 237, 265, 262, 293, 232, 271, 259, 231, 238, 248, 266, 282, 267, 237, 243, 268, 260, 275, 274, 299, 226, 256, 249, 238, 258, 247, 276, 275, 234, 262, 244, 272, 228, 277, 266, 249, 254, 237, 269, 251, 270, 273, 262, 269, 246, 230, 262, 232, 234, 263, 263, 272, 259, 249, 267, 248, 275, 257, 243, 229, 249, 248, 257, 292, 240, 262, 287, 226, 267, 263, 246, 256, 246, 285, 251, 241, 249, 275, 247, 256, 244, 273, 250, 267, 228, 264, 281, 263, 265, 277, 242, 224, 258, 235, 284, 240, 246, 255, 242, 251, 271, 263, 246, 245, 268, 266, 246, 283, 266, 277, 238, 265, 276, 256, 248, 229, 236, 261, 262, 248, 231, 262, 231] payload_rows_per_dst=[5502, 5444, 5504, 5382, 5440, 5500, 5498, 5420] dup_rows_per_dst=[2670, 2866, 2742, 2588, 2714, 2768, 2726, 2772] combine_rows_per_src=[5502, 5444, 5504, 5382, 5440, 5500, 5498, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38352 dup_rows=21776 dup_rows_remote=19064 dedup_ratio=0.332275 router_tokens_per_expert=[255, 251, 256, 273, 285, 243, 238, 270, 259, 256, 257, 252, 288, 224, 239, 267, 275, 242, 272, 250, 272, 238, 254, 249, 264, 254, 272, 296, 259, 270, 228, 252, 242, 256, 241, 261, 258, 264, 265, 262, 244, 253, 265, 237, 240, 249, 249, 274, 271, 265, 253, 255, 257, 260, 243, 237, 255, 254, 268, 249, 221, 254, 238, 260, 263, 239, 265, 250, 236, 267, 287, 263, 265, 250, 255, 270, 268, 269, 247, 257, 225, 249, 262, 240, 269, 269, 238, 267, 262, 241, 258, 284, 225, 231, 276, 257, 239, 265, 239, 249, 253, 249, 248, 263, 231, 271, 280, 258, 233, 244, 263, 248, 266, 257, 259, 239, 264, 275, 262, 279, 268, 238, 254, 285, 290, 261, 223, 251] payload_rows_per_dst=[5514, 5516, 5408, 5398, 5532, 5414, 5434, 5544] dup_rows_per_dst=[2712, 2778, 2712, 2682, 2770, 2692, 2632, 2798] combine_rows_per_src=[5514, 5516, 5408, 5398, 5532, 5414, 5434, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229208 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737368 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245528 weight_movement_bytes=18874368 total_nvlink_bytes=332119896 weight_to_token_ratio=0.060254 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229376 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983296 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737216 weight_movement_bytes=18874368 total_nvlink_bytes=332611584 weight_to_token_ratio=0.060160 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38172 dup_rows=21750 dup_rows_remote=18812 dedup_ratio=0.331879 router_tokens_per_expert=[269, 269, 258, 282, 254, 242, 293, 257, 244, 236, 276, 279, 271, 272, 272, 275, 239, 234, 244, 248, 234, 248, 288, 268, 267, 274, 224, 265, 249, 240, 252, 241, 255, 252, 223, 250, 256, 242, 255, 241, 272, 267, 254, 234, 278, 264, 264, 260, 235, 255, 236, 240, 252, 270, 255, 243, 267, 256, 270, 270, 286, 259, 237, 245, 256, 262, 246, 246, 260, 270, 245, 256, 257, 250, 239, 271, 247, 254, 260, 255, 242, 262, 245, 262, 257, 215, 293, 276, 242, 252, 243, 236, 291, 255, 277, 259, 290, 244, 240, 270, 240, 251, 252, 260, 269, 254, 254, 251, 246, 235, 241, 225, 283, 256, 282, 230, 285, 253, 251, 253, 265, 243, 299, 251, 237, 267, 241, 262] payload_rows_per_dst=[5362, 5426, 5462, 5504, 5472, 5480, 5614, 5466] dup_rows_per_dst=[2628, 2604, 2672, 2648, 2676, 2734, 2938, 2850] combine_rows_per_src=[5362, 5426, 5462, 5504, 5472, 5480, 5614, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38312 dup_rows=21878 dup_rows_remote=19116 dedup_ratio=0.333832 router_tokens_per_expert=[260, 255, 254, 261, 241, 262, 271, 238, 259, 259, 247, 246, 270, 252, 241, 246, 263, 221, 237, 241, 264, 267, 282, 260, 262, 257, 240, 254, 279, 256, 260, 264, 259, 248, 269, 265, 250, 223, 255, 266, 254, 223, 295, 260, 256, 264, 262, 248, 250, 271, 240, 247, 246, 272, 281, 253, 231, 268, 255, 259, 262, 263, 250, 249, 218, 245, 265, 250, 247, 266, 254, 284, 270, 259, 235, 269, 228, 280, 248, 236, 243, 260, 243, 248, 251, 247, 262, 255, 229, 272, 268, 247, 276, 303, 251, 277, 251, 240, 242, 284, 256, 230, 282, 253, 268, 236, 255, 258, 239, 244, 270, 260, 248, 242, 236, 265, 260, 247, 276, 275, 237, 271, 263, 239, 286, 280, 264, 262] payload_rows_per_dst=[5438, 5444, 5486, 5442, 5346, 5518, 5440, 5544] dup_rows_per_dst=[2686, 2770, 2708, 2752, 2762, 2746, 2696, 2758] combine_rows_per_src=[5438, 5444, 5486, 5442, 5346, 5518, 5440, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229664 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157319456 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314409248 weight_movement_bytes=18874368 total_nvlink_bytes=333283616 weight_to_token_ratio=0.060031 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43458 payload_rows_remote=38038 dup_rows=22078 dup_rows_remote=19350 dedup_ratio=0.336884 router_tokens_per_expert=[273, 253, 269, 255, 265, 248, 261, 255, 273, 269, 242, 286, 274, 250, 275, 242, 260, 245, 277, 257, 266, 257, 245, 252, 258, 240, 237, 248, 248, 236, 274, 258, 248, 254, 253, 253, 274, 308, 266, 279, 278, 267, 247, 254, 233, 234, 254, 258, 244, 275, 256, 254, 260, 275, 265, 242, 267, 258, 268, 235, 251, 246, 248, 230, 252, 288, 298, 254, 289, 269, 247, 272, 260, 278, 258, 269, 251, 247, 279, 281, 233, 248, 245, 262, 251, 231, 226, 236, 248, 258, 254, 249, 232, 232, 251, 251, 248, 258, 260, 272, 263, 256, 284, 255, 270, 251, 254, 247, 242, 254, 216, 261, 239, 256, 249, 242, 269, 252, 245, 269, 238, 246, 260, 265, 253, 227, 242, 244] payload_rows_per_dst=[5454, 5394, 5498, 5420, 5582, 5342, 5466, 5302] dup_rows_per_dst=[2926, 2722, 2822, 2728, 3002, 2472, 2716, 2690] combine_rows_per_src=[5454, 5394, 5498, 5420, 5582, 5342, 5466, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38272 dup_rows=21746 dup_rows_remote=19008 dedup_ratio=0.331818 router_tokens_per_expert=[263, 242, 260, 222, 283, 237, 267, 279, 253, 249, 229, 251, 282, 270, 262, 244, 244, 241, 249, 273, 254, 243, 242, 236, 265, 264, 244, 258, 248, 256, 278, 249, 243, 255, 249, 242, 249, 255, 275, 252, 260, 255, 232, 278, 251, 269, 244, 252, 262, 250, 261, 253, 243, 228, 264, 258, 246, 265, 264, 254, 244, 249, 269, 289, 246, 247, 268, 280, 254, 251, 281, 265, 275, 254, 251, 246, 274, 271, 255, 269, 265, 270, 277, 258, 240, 280, 252, 260, 221, 265, 271, 243, 246, 261, 271, 248, 245, 261, 254, 283, 238, 231, 256, 242, 270, 282, 241, 246, 267, 248, 254, 262, 224, 258, 271, 283, 255, 240, 256, 254, 252, 258, 264, 274, 248, 256, 258, 225] payload_rows_per_dst=[5504, 5392, 5460, 5494, 5518, 5518, 5454, 5450] dup_rows_per_dst=[2682, 2696, 2662, 2704, 2856, 2738, 2706, 2702] combine_rows_per_src=[5504, 5392, 5460, 5494, 5518, 5518, 5454, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38196 dup_rows=21910 dup_rows_remote=19146 dedup_ratio=0.334320 router_tokens_per_expert=[263, 268, 263, 239, 226, 247, 262, 273, 265, 254, 258, 266, 274, 230, 243, 258, 254, 235, 274, 247, 265, 248, 246, 258, 291, 243, 241, 235, 245, 260, 285, 270, 260, 276, 256, 248, 265, 266, 266, 241, 247, 262, 282, 249, 238, 252, 258, 245, 266, 251, 243, 275, 237, 269, 291, 272, 272, 297, 217, 253, 230, 262, 280, 252, 248, 263, 244, 245, 266, 254, 294, 264, 269, 247, 276, 254, 266, 239, 250, 253, 276, 256, 260, 240, 250, 272, 309, 271, 260, 263, 238, 265, 216, 227, 233, 248, 256, 253, 285, 257, 254, 247, 268, 252, 242, 260, 267, 226, 260, 263, 240, 242, 241, 262, 250, 246, 255, 245, 241, 291, 248, 250, 273, 236, 240, 245, 247, 246] payload_rows_per_dst=[5388, 5430, 5460, 5476, 5538, 5444, 5476, 5414] dup_rows_per_dst=[2790, 2764, 2762, 2858, 2726, 2724, 2668, 2618] combine_rows_per_src=[5388, 5430, 5460, 5476, 5538, 5444, 5476, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=227936 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156580448 ingress_bytes=175226880 link_bytes=175226880 token_bytes=312932960 weight_movement_bytes=18874368 total_nvlink_bytes=331807328 weight_to_token_ratio=0.060314 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678937747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229712 combine_payload_bytes=156925952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157155664 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314081616 weight_movement_bytes=37748736 total_nvlink_bytes=351830352 weight_to_token_ratio=0.120188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155803648 dispatch_meta_bytes=229552 combine_payload_bytes=155803648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156033200 ingress_bytes=193552384 link_bytes=193552384 token_bytes=311836848 weight_movement_bytes=37748736 total_nvlink_bytes=349585584 weight_to_token_ratio=0.121053 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854523710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229120 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991232 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753344 weight_movement_bytes=18874368 total_nvlink_bytes=332627712 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38106 dup_rows=21866 dup_rows_remote=18928 dedup_ratio=0.333649 router_tokens_per_expert=[276, 238, 241, 276, 281, 275, 232, 250, 241, 249, 239, 255, 243, 273, 243, 247, 245, 244, 273, 292, 243, 217, 245, 234, 266, 255, 253, 241, 253, 274, 247, 280, 253, 264, 260, 265, 253, 249, 277, 258, 237, 288, 259, 244, 246, 243, 266, 292, 278, 217, 249, 294, 299, 260, 239, 257, 273, 281, 274, 260, 239, 272, 262, 289, 241, 249, 262, 228, 241, 252, 251, 264, 227, 247, 281, 224, 266, 281, 256, 240, 262, 235, 272, 288, 240, 246, 248, 265, 266, 257, 261, 200, 259, 248, 232, 262, 258, 259, 266, 240, 262, 261, 249, 238, 264, 272, 259, 272, 260, 240, 264, 273, 241, 260, 247, 256, 244, 266, 243, 273, 255, 255, 268, 246, 257, 241, 247, 263] payload_rows_per_dst=[5564, 5614, 5526, 5498, 5430, 5518, 5190, 5330] dup_rows_per_dst=[2938, 2812, 2782, 2752, 2590, 2946, 2318, 2728] combine_rows_per_src=[5564, 5614, 5526, 5498, 5430, 5518, 5190, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229368 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680184 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131000 weight_movement_bytes=18874368 total_nvlink_bytes=332005368 weight_to_token_ratio=0.060276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38172 dup_rows=21960 dup_rows_remote=19220 dedup_ratio=0.335083 router_tokens_per_expert=[241, 241, 247, 246, 253, 254, 272, 284, 310, 255, 243, 243, 230, 267, 272, 248, 237, 253, 249, 286, 249, 249, 243, 253, 225, 274, 273, 240, 277, 269, 254, 246, 272, 266, 270, 266, 247, 253, 258, 254, 273, 275, 271, 260, 265, 241, 234, 235, 283, 252, 248, 265, 253, 275, 245, 262, 272, 253, 241, 258, 273, 254, 250, 274, 243, 267, 254, 235, 264, 262, 249, 285, 255, 237, 268, 256, 264, 224, 233, 241, 247, 243, 264, 278, 292, 273, 255, 293, 261, 260, 223, 267, 281, 252, 270, 245, 246, 261, 219, 259, 258, 247, 238, 274, 241, 238, 289, 264, 267, 242, 238, 237, 246, 268, 250, 245, 237, 249, 260, 243, 252, 264, 251, 241, 267, 243, 244, 268] payload_rows_per_dst=[5306, 5514, 5486, 5474, 5404, 5572, 5416, 5404] dup_rows_per_dst=[2556, 2902, 2794, 2842, 2604, 2896, 2626, 2740] combine_rows_per_src=[5306, 5514, 5486, 5474, 5404, 5572, 5416, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38116 dup_rows=21866 dup_rows_remote=19054 dedup_ratio=0.333649 router_tokens_per_expert=[252, 244, 253, 232, 296, 258, 272, 276, 226, 248, 267, 286, 253, 247, 274, 244, 265, 268, 266, 274, 260, 259, 290, 244, 235, 263, 269, 259, 253, 222, 252, 292, 267, 268, 275, 258, 257, 265, 248, 252, 237, 280, 276, 276, 268, 248, 249, 259, 256, 262, 268, 252, 248, 260, 256, 235, 259, 251, 271, 245, 248, 287, 251, 234, 265, 268, 242, 267, 240, 268, 214, 260, 267, 240, 252, 275, 246, 257, 243, 240, 272, 239, 262, 234, 251, 264, 242, 270, 282, 257, 286, 242, 255, 266, 253, 271, 271, 240, 292, 238, 245, 260, 249, 232, 241, 259, 254, 249, 248, 263, 273, 228, 232, 243, 252, 236, 257, 236, 243, 252, 256, 254, 257, 248, 252, 250, 253, 250] payload_rows_per_dst=[5490, 5510, 5554, 5410, 5450, 5488, 5418, 5350] dup_rows_per_dst=[2766, 2832, 2812, 2756, 2638, 2804, 2666, 2592] combine_rows_per_src=[5490, 5510, 5554, 5410, 5450, 5488, 5418, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228136 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156310312 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312392488 weight_movement_bytes=18874368 total_nvlink_bytes=331266856 weight_to_token_ratio=0.060419 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229568 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156582080 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312934592 weight_movement_bytes=37748736 total_nvlink_bytes=350683328 weight_to_token_ratio=0.120628 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38286 dup_rows=21804 dup_rows_remote=19070 dedup_ratio=0.332703 router_tokens_per_expert=[258, 244, 262, 286, 263, 252, 230, 262, 263, 259, 275, 239, 278, 236, 287, 263, 257, 246, 264, 261, 268, 250, 265, 260, 256, 239, 269, 236, 265, 248, 260, 268, 236, 268, 249, 237, 233, 260, 236, 264, 279, 273, 254, 230, 256, 248, 247, 264, 269, 272, 245, 252, 270, 268, 224, 255, 282, 250, 254, 245, 251, 243, 268, 248, 268, 259, 253, 269, 256, 266, 254, 259, 244, 263, 260, 253, 258, 256, 240, 288, 234, 289, 249, 229, 242, 256, 268, 276, 249, 270, 256, 266, 241, 262, 233, 256, 233, 246, 287, 237, 238, 248, 277, 246, 270, 254, 248, 256, 274, 272, 250, 254, 260, 258, 228, 278, 278, 279, 235, 257, 246, 244, 239, 257, 254, 252, 260, 232] payload_rows_per_dst=[5474, 5520, 5366, 5496, 5538, 5450, 5446, 5442] dup_rows_per_dst=[2840, 2704, 2702, 2696, 2754, 2702, 2734, 2672] combine_rows_per_src=[5474, 5520, 5366, 5496, 5538, 5450, 5446, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=228680 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156351816 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312474952 weight_movement_bytes=18874368 total_nvlink_bytes=331349320 weight_to_token_ratio=0.060403 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38246 dup_rows=21778 dup_rows_remote=19192 dedup_ratio=0.332306 router_tokens_per_expert=[260, 218, 246, 251, 243, 243, 258, 232, 245, 273, 250, 249, 269, 251, 272, 268, 243, 272, 228, 260, 254, 265, 231, 259, 241, 252, 263, 252, 287, 260, 254, 260, 245, 247, 288, 259, 250, 242, 272, 256, 250, 260, 262, 265, 275, 278, 279, 275, 238, 263, 244, 252, 271, 262, 271, 252, 244, 256, 254, 234, 278, 250, 241, 239, 236, 300, 251, 247, 250, 247, 274, 261, 222, 266, 240, 214, 250, 267, 259, 274, 247, 263, 263, 259, 261, 241, 239, 242, 241, 247, 266, 266, 233, 249, 248, 266, 280, 273, 246, 254, 277, 273, 262, 247, 282, 280, 254, 250, 259, 240, 286, 272, 252, 238, 246, 263, 245, 255, 262, 244, 259, 257, 272, 245, 263, 274, 250, 258] payload_rows_per_dst=[5340, 5486, 5514, 5512, 5416, 5392, 5622, 5476] dup_rows_per_dst=[2716, 2676, 2892, 2586, 2700, 2670, 2848, 2690] combine_rows_per_src=[5340, 5486, 5514, 5512, 5416, 5392, 5622, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38348 dup_rows=21686 dup_rows_remote=18872 dedup_ratio=0.330902 router_tokens_per_expert=[258, 267, 253, 245, 282, 289, 259, 257, 262, 246, 249, 226, 290, 246, 234, 252, 275, 226, 266, 263, 253, 278, 245, 287, 249, 273, 232, 241, 264, 234, 261, 248, 226, 258, 248, 280, 269, 247, 254, 252, 237, 256, 256, 263, 256, 285, 236, 289, 290, 234, 273, 250, 264, 230, 263, 225, 250, 260, 258, 280, 255, 230, 239, 254, 244, 273, 254, 252, 258, 264, 267, 258, 247, 255, 278, 245, 253, 258, 256, 296, 245, 263, 260, 242, 249, 266, 244, 244, 244, 259, 241, 273, 263, 261, 256, 255, 252, 274, 268, 231, 255, 251, 241, 256, 279, 240, 233, 233, 251, 274, 242, 276, 269, 241, 264, 258, 268, 250, 281, 246, 246, 265, 245, 260, 248, 233, 278, 260] payload_rows_per_dst=[5524, 5496, 5476, 5474, 5502, 5422, 5458, 5498] dup_rows_per_dst=[2706, 2694, 2748, 2636, 2814, 2708, 2654, 2726] combine_rows_per_src=[5524, 5496, 5476, 5474, 5502, 5422, 5458, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38352 dup_rows=21766 dup_rows_remote=18974 dedup_ratio=0.332123 router_tokens_per_expert=[274, 264, 269, 245, 257, 255, 260, 256, 270, 253, 271, 252, 217, 243, 247, 276, 252, 253, 226, 265, 240, 256, 260, 247, 253, 235, 250, 252, 230, 248, 280, 269, 261, 255, 256, 246, 251, 303, 260, 246, 225, 232, 282, 234, 228, 227, 275, 256, 244, 297, 278, 249, 274, 251, 260, 264, 237, 245, 259, 257, 281, 270, 239, 282, 245, 213, 256, 266, 262, 260, 252, 285, 246, 244, 251, 248, 255, 263, 254, 257, 267, 260, 249, 271, 273, 263, 285, 263, 257, 239, 258, 242, 244, 221, 240, 273, 267, 275, 254, 244, 264, 266, 230, 253, 277, 248, 250, 246, 272, 257, 233, 262, 267, 232, 282, 266, 262, 273, 244, 268, 260, 288, 269, 258, 245, 225, 262, 258] payload_rows_per_dst=[5480, 5414, 5502, 5512, 5434, 5418, 5470, 5540] dup_rows_per_dst=[2738, 2618, 2572, 2862, 2680, 2792, 2726, 2778] combine_rows_per_src=[5480, 5414, 5502, 5512, 5434, 5418, 5470, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229424 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048880 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868336 weight_movement_bytes=18874368 total_nvlink_bytes=332742704 weight_to_token_ratio=0.060135 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38274 dup_rows=21754 dup_rows_remote=19146 dedup_ratio=0.331940 router_tokens_per_expert=[278, 252, 279, 246, 220, 247, 277, 263, 275, 248, 230, 262, 258, 221, 241, 276, 234, 239, 245, 247, 262, 233, 250, 241, 295, 260, 281, 265, 252, 262, 249, 243, 269, 261, 275, 253, 247, 252, 262, 250, 249, 269, 248, 252, 255, 241, 259, 240, 270, 265, 232, 222, 274, 260, 273, 265, 264, 247, 240, 243, 257, 236, 270, 241, 234, 244, 293, 283, 233, 254, 259, 289, 248, 261, 230, 261, 252, 248, 256, 262, 259, 249, 269, 264, 231, 253, 246, 258, 266, 267, 238, 269, 252, 261, 268, 267, 245, 274, 260, 272, 267, 231, 279, 282, 261, 280, 277, 259, 247, 241, 265, 268, 263, 261, 254, 249, 243, 281, 255, 216, 272, 275, 243, 258, 233, 257, 265, 239] payload_rows_per_dst=[5430, 5508, 5442, 5438, 5488, 5496, 5546, 5434] dup_rows_per_dst=[2716, 2608, 2722, 2680, 2726, 2738, 2870, 2694] combine_rows_per_src=[5430, 5508, 5442, 5438, 5488, 5496, 5546, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229752 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885368 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540984 weight_movement_bytes=18874368 total_nvlink_bytes=332415352 weight_to_token_ratio=0.060197 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=228880 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157302288 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314375696 weight_movement_bytes=37748736 total_nvlink_bytes=352124432 weight_to_token_ratio=0.120075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229304 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157319096 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314408888 weight_movement_bytes=18874368 total_nvlink_bytes=333283256 weight_to_token_ratio=0.060031 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38288 dup_rows=21716 dup_rows_remote=18896 dedup_ratio=0.331360 router_tokens_per_expert=[218, 242, 233, 246, 258, 271, 255, 253, 259, 254, 255, 250, 246, 263, 272, 243, 250, 267, 259, 271, 254, 261, 280, 259, 243, 256, 242, 243, 263, 272, 268, 266, 258, 250, 236, 255, 277, 246, 268, 237, 259, 231, 226, 252, 249, 248, 257, 263, 234, 226, 256, 252, 270, 282, 257, 251, 246, 251, 247, 236, 245, 282, 237, 244, 227, 275, 249, 281, 280, 264, 272, 235, 264, 249, 269, 255, 286, 279, 281, 274, 291, 250, 257, 242, 279, 274, 260, 263, 248, 261, 254, 271, 245, 255, 261, 249, 250, 230, 277, 272, 247, 253, 267, 257, 243, 262, 246, 253, 242, 258, 253, 253, 269, 261, 242, 276, 259, 252, 239, 243, 243, 253, 275, 244, 253, 281, 253, 262] payload_rows_per_dst=[5532, 5504, 5404, 5610, 5344, 5558, 5428, 5440] dup_rows_per_dst=[2820, 2636, 2632, 2914, 2520, 2762, 2698, 2734] combine_rows_per_src=[5532, 5504, 5404, 5610, 5344, 5558, 5428, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229680 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999984 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770288 weight_movement_bytes=18874368 total_nvlink_bytes=332644656 weight_to_token_ratio=0.060153 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44024 payload_rows_remote=38442 dup_rows=21512 dup_rows_remote=18654 dedup_ratio=0.328247 router_tokens_per_expert=[258, 272, 255, 273, 250, 238, 255, 252, 273, 261, 261, 244, 268, 254, 239, 302, 232, 279, 262, 251, 279, 239, 253, 284, 254, 242, 270, 258, 274, 256, 261, 247, 251, 252, 245, 248, 267, 257, 256, 249, 269, 243, 237, 265, 250, 254, 260, 237, 273, 263, 215, 275, 276, 236, 274, 260, 258, 271, 264, 239, 248, 261, 269, 260, 271, 253, 226, 290, 253, 260, 256, 239, 250, 268, 258, 264, 259, 244, 226, 245, 268, 244, 244, 264, 265, 253, 264, 257, 235, 278, 262, 267, 260, 237, 240, 273, 259, 277, 232, 252, 266, 233, 264, 269, 256, 250, 273, 260, 230, 241, 234, 294, 272, 268, 244, 237, 259, 261, 265, 256, 242, 249, 246, 246, 266, 259, 221, 236] payload_rows_per_dst=[5368, 5500, 5474, 5560, 5302, 5728, 5510, 5582] dup_rows_per_dst=[2454, 2780, 2606, 2724, 2436, 2982, 2672, 2858] combine_rows_per_src=[5368, 5500, 5474, 5560, 5302, 5728, 5510, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38156 dup_rows=21970 dup_rows_remote=19194 dedup_ratio=0.335236 router_tokens_per_expert=[258, 252, 262, 290, 275, 256, 266, 240, 267, 241, 254, 278, 210, 247, 265, 254, 257, 235, 262, 272, 245, 233, 260, 235, 261, 270, 259, 254, 263, 257, 267, 258, 267, 251, 255, 234, 235, 243, 261, 264, 278, 257, 268, 250, 265, 238, 267, 260, 246, 288, 248, 264, 249, 255, 255, 253, 242, 233, 249, 241, 229, 234, 234, 278, 255, 266, 264, 239, 280, 259, 259, 250, 240, 286, 254, 256, 262, 245, 282, 251, 257, 245, 238, 237, 266, 278, 242, 285, 255, 264, 259, 267, 271, 251, 265, 267, 261, 251, 233, 260, 247, 271, 265, 260, 253, 230, 234, 273, 267, 237, 266, 269, 268, 271, 257, 258, 265, 248, 282, 294, 264, 231, 245, 235, 249, 244, 237, 254] payload_rows_per_dst=[5428, 5448, 5410, 5408, 5500, 5474, 5436, 5462] dup_rows_per_dst=[2802, 2728, 2776, 2588, 2796, 2820, 2718, 2742] combine_rows_per_src=[5428, 5448, 5410, 5408, 5500, 5474, 5436, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228736 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157056384 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884032 weight_movement_bytes=37748736 total_nvlink_bytes=351632768 weight_to_token_ratio=0.120263 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=228384 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157686816 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315145248 weight_movement_bytes=18874368 total_nvlink_bytes=334019616 weight_to_token_ratio=0.059891 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38190 dup_rows=21910 dup_rows_remote=19182 dedup_ratio=0.334320 router_tokens_per_expert=[260, 224, 265, 255, 260, 241, 250, 224, 245, 240, 242, 257, 267, 254, 248, 265, 254, 258, 256, 255, 260, 271, 286, 234, 256, 252, 244, 251, 298, 224, 244, 262, 261, 229, 261, 251, 269, 270, 286, 274, 287, 260, 239, 259, 237, 255, 279, 256, 259, 280, 251, 255, 261, 265, 239, 242, 263, 283, 252, 248, 275, 275, 242, 255, 258, 239, 265, 251, 290, 248, 245, 243, 260, 257, 265, 267, 237, 245, 234, 227, 232, 252, 280, 278, 259, 245, 247, 285, 236, 269, 243, 263, 242, 223, 273, 249, 264, 252, 217, 256, 294, 272, 254, 257, 277, 252, 270, 248, 243, 245, 234, 247, 259, 263, 256, 277, 281, 240, 283, 280, 247, 234, 234, 260, 269, 263, 257, 256] payload_rows_per_dst=[5352, 5486, 5500, 5480, 5468, 5458, 5436, 5446] dup_rows_per_dst=[2642, 2724, 2846, 2810, 2594, 2694, 2728, 2872] combine_rows_per_src=[5352, 5486, 5500, 5480, 5468, 5458, 5436, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38362 dup_rows=21628 dup_rows_remote=18850 dedup_ratio=0.330017 router_tokens_per_expert=[236, 247, 249, 257, 232, 268, 246, 241, 241, 234, 244, 248, 268, 262, 273, 246, 205, 260, 250, 255, 263, 289, 240, 265, 241, 263, 270, 256, 236, 255, 260, 243, 260, 296, 255, 281, 260, 240, 240, 237, 253, 241, 286, 257, 267, 252, 264, 231, 254, 250, 278, 234, 265, 281, 277, 266, 251, 270, 251, 257, 268, 247, 246, 267, 243, 254, 259, 246, 257, 229, 250, 252, 263, 241, 259, 227, 285, 253, 235, 270, 269, 277, 236, 245, 249, 249, 258, 235, 295, 273, 270, 268, 281, 250, 250, 243, 260, 279, 300, 254, 274, 261, 237, 273, 282, 246, 263, 265, 236, 227, 263, 251, 238, 238, 269, 250, 245, 251, 265, 270, 247, 241, 251, 280, 255, 271, 261, 269] payload_rows_per_dst=[5422, 5458, 5444, 5546, 5476, 5514, 5584, 5464] dup_rows_per_dst=[2562, 2644, 2796, 2778, 2570, 2782, 2758, 2738] combine_rows_per_src=[5422, 5458, 5444, 5546, 5476, 5514, 5584, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229400 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156516376 ingress_bytes=212910080 link_bytes=212910080 token_bytes=312803352 weight_movement_bytes=56623104 total_nvlink_bytes=369426456 weight_to_token_ratio=0.181018 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039999628e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38158 dup_rows=21996 dup_rows_remote=19408 dedup_ratio=0.335632 router_tokens_per_expert=[269, 252, 257, 226, 257, 252, 257, 259, 239, 286, 252, 264, 258, 265, 274, 288, 244, 269, 268, 242, 243, 237, 255, 232, 239, 298, 259, 232, 281, 246, 251, 272, 274, 261, 260, 254, 245, 252, 251, 251, 251, 255, 253, 256, 263, 264, 275, 256, 245, 235, 243, 222, 259, 268, 280, 261, 245, 242, 252, 222, 255, 269, 278, 274, 250, 244, 261, 235, 235, 282, 217, 260, 270, 250, 253, 276, 239, 220, 244, 249, 259, 248, 246, 282, 282, 254, 259, 257, 246, 250, 269, 240, 269, 262, 258, 243, 253, 280, 230, 251, 239, 252, 233, 260, 254, 266, 275, 247, 256, 273, 242, 258, 267, 264, 274, 267, 273, 267, 262, 268, 233, 235, 254, 264, 274, 263, 284, 247] payload_rows_per_dst=[5414, 5414, 5416, 5428, 5382, 5502, 5426, 5558] dup_rows_per_dst=[2896, 2722, 2826, 2672, 2588, 2746, 2712, 2834] combine_rows_per_src=[5414, 5414, 5416, 5428, 5382, 5502, 5426, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38224 dup_rows=21824 dup_rows_remote=19142 dedup_ratio=0.333008 router_tokens_per_expert=[257, 250, 261, 243, 254, 237, 260, 248, 263, 253, 244, 242, 245, 256, 264, 251, 269, 253, 254, 238, 293, 255, 253, 271, 289, 230, 270, 272, 250, 247, 255, 258, 248, 243, 234, 266, 261, 244, 269, 247, 244, 250, 260, 225, 264, 292, 263, 234, 271, 274, 265, 248, 265, 265, 272, 259, 244, 240, 246, 254, 246, 273, 264, 254, 266, 281, 271, 246, 264, 257, 227, 252, 251, 257, 241, 275, 243, 260, 252, 237, 266, 267, 254, 251, 277, 254, 250, 242, 240, 251, 269, 256, 277, 251, 249, 231, 244, 258, 262, 279, 234, 249, 267, 245, 281, 225, 256, 281, 286, 257, 248, 242, 260, 251, 257, 262, 303, 264, 223, 274, 222, 240, 276, 270, 259, 254, 259, 246] payload_rows_per_dst=[5374, 5496, 5432, 5508, 5464, 5488, 5490, 5460] dup_rows_per_dst=[2682, 2818, 2656, 2772, 2696, 2682, 2738, 2780] combine_rows_per_src=[5374, 5496, 5432, 5508, 5464, 5488, 5490, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229488 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655728 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081968 weight_movement_bytes=18874368 total_nvlink_bytes=331956336 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38164 dup_rows=21896 dup_rows_remote=19144 dedup_ratio=0.334106 router_tokens_per_expert=[239, 270, 252, 253, 256, 235, 245, 265, 267, 245, 256, 252, 267, 266, 256, 275, 228, 259, 268, 282, 262, 254, 255, 254, 253, 231, 282, 260, 257, 238, 282, 249, 257, 253, 247, 279, 259, 263, 243, 268, 244, 274, 270, 257, 248, 261, 282, 261, 264, 259, 243, 253, 270, 253, 251, 234, 256, 245, 275, 252, 242, 262, 245, 253, 264, 282, 267, 295, 247, 256, 247, 250, 220, 270, 268, 300, 236, 278, 280, 247, 234, 262, 225, 249, 240, 262, 237, 246, 252, 220, 245, 251, 243, 254, 258, 238, 253, 260, 255, 255, 235, 251, 224, 246, 257, 273, 278, 242, 233, 277, 274, 257, 253, 281, 258, 303, 260, 259, 255, 244, 277, 251, 253, 256, 230, 259, 249, 251] payload_rows_per_dst=[5466, 5476, 5500, 5482, 5540, 5294, 5408, 5474] dup_rows_per_dst=[2732, 2752, 2832, 2632, 2874, 2538, 2732, 2804] combine_rows_per_src=[5466, 5476, 5500, 5482, 5540, 5294, 5408, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=228848 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157359600 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314490352 weight_movement_bytes=37748736 total_nvlink_bytes=352239088 weight_to_token_ratio=0.120031 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=230264 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156525432 ingress_bytes=231792640 link_bytes=231792640 token_bytes=312820600 weight_movement_bytes=75497472 total_nvlink_bytes=388318072 weight_to_token_ratio=0.241344 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220923027e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229464 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156794968 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360472 weight_movement_bytes=18874368 total_nvlink_bytes=332234840 weight_to_token_ratio=0.060232 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38238 dup_rows=21690 dup_rows_remote=18830 dedup_ratio=0.330963 router_tokens_per_expert=[251, 283, 254, 253, 274, 261, 262, 253, 252, 264, 251, 255, 247, 247, 272, 246, 227, 263, 254, 283, 229, 271, 259, 242, 261, 245, 252, 249, 240, 284, 260, 249, 251, 274, 248, 281, 267, 299, 272, 259, 247, 246, 246, 261, 252, 237, 241, 241, 273, 255, 247, 252, 242, 226, 279, 250, 259, 255, 252, 280, 261, 266, 264, 242, 243, 277, 245, 242, 247, 283, 255, 257, 237, 254, 268, 263, 261, 241, 281, 245, 247, 271, 257, 240, 257, 235, 269, 275, 244, 248, 241, 252, 261, 281, 242, 251, 271, 262, 254, 234, 257, 268, 251, 282, 257, 246, 251, 250, 238, 259, 223, 273, 276, 252, 258, 240, 281, 255, 242, 257, 260, 246, 242, 250, 233, 249, 267, 296] payload_rows_per_dst=[5370, 5474, 5388, 5502, 5574, 5454, 5476, 5608] dup_rows_per_dst=[2620, 2662, 2564, 2704, 2902, 2688, 2690, 2860] combine_rows_per_src=[5370, 5474, 5388, 5502, 5574, 5454, 5476, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229232 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156548976 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868720 weight_movement_bytes=18874368 total_nvlink_bytes=331743088 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38240 dup_rows=21850 dup_rows_remote=19178 dedup_ratio=0.333405 router_tokens_per_expert=[232, 265, 256, 272, 249, 250, 247, 262, 235, 260, 264, 259, 241, 268, 248, 235, 241, 266, 220, 263, 256, 263, 266, 282, 269, 257, 265, 230, 274, 266, 284, 258, 287, 264, 274, 253, 251, 269, 278, 276, 252, 249, 246, 276, 273, 251, 261, 256, 278, 249, 287, 263, 232, 240, 239, 263, 260, 235, 254, 261, 270, 257, 244, 247, 254, 274, 253, 236, 242, 271, 254, 252, 236, 260, 283, 248, 244, 258, 254, 262, 227, 247, 249, 259, 233, 270, 275, 263, 254, 243, 256, 253, 259, 235, 247, 260, 238, 262, 249, 244, 254, 232, 226, 256, 247, 255, 284, 283, 262, 253, 256, 278, 248, 250, 263, 227, 255, 241, 261, 284, 267, 267, 257, 267, 261, 249, 220, 263] payload_rows_per_dst=[5446, 5478, 5590, 5424, 5498, 5370, 5454, 5426] dup_rows_per_dst=[2672, 2842, 2842, 2702, 2664, 2690, 2704, 2734] combine_rows_per_src=[5446, 5478, 5590, 5424, 5498, 5370, 5454, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38124 dup_rows=21800 dup_rows_remote=19106 dedup_ratio=0.332642 router_tokens_per_expert=[253, 221, 241, 263, 251, 245, 255, 242, 271, 280, 250, 246, 237, 226, 256, 238, 257, 257, 260, 222, 275, 258, 264, 268, 272, 251, 269, 275, 229, 240, 249, 244, 274, 263, 247, 250, 241, 270, 247, 241, 255, 279, 285, 266, 243, 278, 260, 254, 258, 284, 263, 237, 276, 247, 305, 228, 257, 256, 283, 277, 268, 252, 238, 259, 285, 256, 246, 238, 239, 263, 241, 243, 251, 258, 282, 280, 274, 251, 294, 263, 258, 256, 247, 258, 278, 267, 275, 245, 222, 254, 265, 266, 231, 237, 229, 254, 267, 231, 258, 263, 245, 243, 265, 272, 246, 243, 267, 267, 277, 253, 248, 243, 256, 239, 247, 238, 266, 227, 246, 265, 268, 253, 281, 271, 242, 243, 248, 278] payload_rows_per_dst=[5348, 5404, 5612, 5594, 5534, 5338, 5446, 5460] dup_rows_per_dst=[2602, 2776, 2694, 2782, 2794, 2746, 2730, 2676] combine_rows_per_src=[5348, 5404, 5612, 5594, 5534, 5338, 5446, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228272 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156851120 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313473968 weight_movement_bytes=18874368 total_nvlink_bytes=332348336 weight_to_token_ratio=0.060210 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38446 dup_rows=21648 dup_rows_remote=18946 dedup_ratio=0.330322 router_tokens_per_expert=[266, 223, 279, 279, 254, 249, 252, 272, 269, 280, 258, 267, 245, 254, 250, 247, 248, 233, 235, 225, 280, 264, 276, 266, 234, 233, 249, 281, 288, 260, 261, 243, 252, 245, 234, 254, 250, 249, 270, 255, 251, 267, 256, 291, 272, 252, 241, 267, 284, 265, 262, 227, 265, 247, 221, 294, 258, 247, 236, 255, 267, 247, 257, 264, 266, 260, 256, 240, 280, 249, 281, 246, 263, 253, 236, 232, 258, 258, 256, 235, 233, 249, 276, 229, 255, 259, 249, 268, 283, 287, 259, 251, 270, 268, 264, 250, 265, 254, 255, 250, 235, 262, 244, 241, 244, 267, 252, 255, 265, 265, 250, 268, 236, 238, 262, 275, 231, 257, 228, 221, 263, 258, 294, 259, 265, 251, 249, 268] payload_rows_per_dst=[5484, 5510, 5524, 5482, 5458, 5582, 5442, 5406] dup_rows_per_dst=[2804, 2642, 2688, 2710, 2680, 2718, 2702, 2704] combine_rows_per_src=[5484, 5510, 5524, 5482, 5458, 5582, 5442, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229672 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156860712 ingress_bytes=213254144 link_bytes=213254144 token_bytes=313491752 weight_movement_bytes=56623104 total_nvlink_bytes=370114856 weight_to_token_ratio=0.180621 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043296280e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38148 dup_rows=21954 dup_rows_remote=19138 dedup_ratio=0.334991 router_tokens_per_expert=[284, 266, 266, 235, 263, 240, 230, 277, 237, 247, 262, 251, 242, 245, 260, 270, 273, 238, 254, 230, 246, 255, 225, 270, 274, 270, 264, 234, 230, 248, 254, 247, 219, 262, 244, 264, 257, 266, 253, 271, 244, 244, 246, 258, 244, 256, 278, 254, 228, 265, 259, 269, 260, 256, 267, 262, 253, 258, 248, 276, 268, 243, 246, 267, 290, 271, 272, 247, 264, 254, 272, 264, 245, 243, 245, 255, 243, 248, 278, 270, 265, 235, 241, 258, 267, 262, 260, 266, 278, 268, 262, 245, 253, 211, 255, 261, 275, 251, 255, 250, 269, 281, 297, 247, 263, 246, 252, 229, 248, 265, 238, 275, 248, 228, 245, 267, 263, 269, 269, 261, 243, 222, 271, 239, 277, 278, 267, 260] payload_rows_per_dst=[5426, 5332, 5462, 5434, 5538, 5472, 5528, 5390] dup_rows_per_dst=[2724, 2692, 2658, 2816, 2784, 2702, 2754, 2824] combine_rows_per_src=[5426, 5332, 5462, 5434, 5538, 5472, 5528, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228920 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156384824 ingress_bytes=193904640 link_bytes=193904640 token_bytes=312540728 weight_movement_bytes=37748736 total_nvlink_bytes=350289464 weight_to_token_ratio=0.120780 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857898853e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38348 dup_rows=21772 dup_rows_remote=19116 dedup_ratio=0.332214 router_tokens_per_expert=[265, 262, 238, 255, 269, 229, 282, 241, 267, 282, 271, 249, 232, 264, 266, 252, 263, 223, 270, 273, 263, 273, 256, 270, 220, 241, 258, 265, 274, 253, 250, 249, 255, 229, 246, 261, 284, 257, 249, 280, 253, 271, 257, 267, 251, 256, 264, 281, 243, 273, 250, 234, 231, 267, 267, 256, 255, 245, 227, 261, 233, 264, 252, 279, 270, 238, 276, 258, 256, 251, 220, 243, 271, 265, 240, 244, 261, 240, 260, 243, 272, 272, 240, 275, 253, 243, 285, 249, 273, 252, 268, 259, 256, 283, 263, 240, 241, 253, 265, 261, 259, 260, 231, 240, 230, 231, 257, 266, 278, 260, 263, 273, 259, 269, 263, 236, 280, 250, 272, 264, 238, 232, 248, 242, 257, 247, 245, 256] payload_rows_per_dst=[5462, 5490, 5498, 5404, 5416, 5534, 5538, 5422] dup_rows_per_dst=[2786, 2712, 2824, 2670, 2656, 2832, 2598, 2694] combine_rows_per_src=[5462, 5490, 5498, 5404, 5416, 5534, 5538, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38196 dup_rows=21912 dup_rows_remote=19260 dedup_ratio=0.334351 router_tokens_per_expert=[231, 271, 256, 259, 274, 257, 251, 238, 292, 260, 232, 272, 264, 255, 267, 268, 260, 272, 236, 244, 279, 237, 287, 239, 272, 241, 243, 253, 269, 230, 248, 254, 225, 260, 270, 255, 284, 264, 235, 232, 246, 264, 254, 285, 235, 251, 260, 266, 246, 231, 263, 273, 271, 250, 264, 269, 256, 272, 254, 275, 229, 254, 259, 243, 263, 262, 245, 242, 278, 256, 251, 264, 236, 239, 251, 247, 254, 233, 269, 227, 261, 242, 270, 267, 240, 273, 245, 257, 263, 240, 249, 248, 258, 247, 239, 241, 265, 271, 236, 258, 241, 266, 276, 265, 286, 235, 255, 256, 276, 246, 262, 266, 248, 242, 254, 260, 227, 275, 240, 286, 258, 248, 259, 305, 285, 245, 248, 265] payload_rows_per_dst=[5466, 5438, 5408, 5510, 5362, 5428, 5518, 5494] dup_rows_per_dst=[2828, 2690, 2764, 2708, 2672, 2652, 2802, 2796] combine_rows_per_src=[5466, 5438, 5408, 5510, 5362, 5428, 5518, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38298 dup_rows=21912 dup_rows_remote=19386 dedup_ratio=0.334351 router_tokens_per_expert=[259, 260, 251, 253, 243, 267, 276, 268, 249, 230, 279, 273, 271, 278, 241, 233, 258, 239, 242, 235, 258, 225, 274, 231, 236, 258, 242, 271, 240, 217, 252, 248, 265, 264, 239, 250, 259, 254, 277, 253, 290, 265, 259, 264, 271, 263, 243, 255, 238, 243, 254, 274, 272, 265, 254, 250, 269, 251, 240, 246, 260, 249, 235, 226, 251, 265, 244, 250, 274, 263, 257, 246, 248, 256, 255, 265, 258, 260, 265, 260, 232, 260, 258, 264, 267, 287, 254, 277, 263, 276, 237, 268, 268, 241, 280, 230, 232, 258, 273, 252, 249, 268, 255, 243, 273, 270, 263, 258, 262, 240, 280, 237, 247, 226, 245, 238, 266, 275, 301, 263, 251, 277, 269, 233, 258, 285, 229, 259] payload_rows_per_dst=[5466, 5326, 5546, 5350, 5402, 5560, 5516, 5458] dup_rows_per_dst=[2796, 2526, 2796, 2702, 2832, 2764, 2710, 2786] combine_rows_per_src=[5466, 5326, 5546, 5350, 5402, 5560, 5516, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229568 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157704384 ingress_bytes=214097920 link_bytes=214097920 token_bytes=315179200 weight_movement_bytes=56623104 total_nvlink_bytes=371802304 weight_to_token_ratio=0.179654 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051380927e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229144 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483352 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737560 weight_movement_bytes=18874368 total_nvlink_bytes=331611928 weight_to_token_ratio=0.060352 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229856 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303264 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376672 weight_movement_bytes=18874368 total_nvlink_bytes=333251040 weight_to_token_ratio=0.060037 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38244 dup_rows=21750 dup_rows_remote=18948 dedup_ratio=0.331879 router_tokens_per_expert=[249, 226, 256, 278, 257, 255, 238, 270, 292, 266, 260, 236, 259, 269, 265, 233, 275, 261, 267, 254, 269, 269, 258, 250, 253, 241, 262, 258, 240, 259, 241, 231, 257, 266, 266, 238, 251, 257, 271, 262, 260, 259, 232, 232, 258, 257, 254, 254, 270, 238, 257, 262, 243, 259, 244, 257, 275, 254, 243, 238, 254, 271, 279, 258, 259, 276, 244, 251, 255, 268, 250, 241, 235, 259, 239, 259, 221, 256, 246, 265, 248, 269, 262, 269, 229, 241, 283, 242, 290, 236, 280, 272, 289, 231, 259, 286, 263, 268, 241, 243, 280, 242, 239, 255, 231, 270, 255, 271, 224, 253, 242, 252, 237, 260, 238, 257, 282, 268, 274, 253, 246, 261, 261, 260, 274, 259, 264, 262] payload_rows_per_dst=[5434, 5456, 5474, 5444, 5442, 5546, 5448, 5542] dup_rows_per_dst=[2784, 2720, 2674, 2728, 2606, 2826, 2610, 2802] combine_rows_per_src=[5434, 5456, 5474, 5444, 5442, 5546, 5448, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229824 combine_payload_bytes=156450816 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156680640 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313131456 weight_movement_bytes=37748736 total_nvlink_bytes=350880192 weight_to_token_ratio=0.120552 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=230736 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157099344 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313967952 weight_movement_bytes=18874368 total_nvlink_bytes=332842320 weight_to_token_ratio=0.060116 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38164 dup_rows=21852 dup_rows_remote=19152 dedup_ratio=0.333435 router_tokens_per_expert=[283, 255, 256, 287, 269, 246, 240, 234, 259, 247, 244, 242, 278, 245, 245, 274, 251, 254, 265, 234, 251, 247, 256, 260, 261, 258, 257, 262, 269, 260, 255, 272, 239, 281, 253, 267, 243, 258, 247, 246, 264, 269, 255, 267, 282, 238, 241, 261, 274, 243, 257, 256, 243, 313, 267, 250, 279, 259, 259, 259, 249, 270, 275, 236, 253, 261, 259, 243, 259, 251, 228, 277, 265, 263, 259, 289, 269, 253, 250, 268, 242, 244, 242, 262, 251, 263, 248, 266, 245, 249, 259, 251, 233, 256, 287, 261, 251, 249, 252, 266, 251, 246, 284, 247, 247, 270, 235, 280, 240, 235, 257, 246, 255, 251, 255, 253, 234, 255, 234, 243, 263, 245, 238, 233, 230, 268, 266, 267] payload_rows_per_dst=[5520, 5506, 5414, 5478, 5518, 5388, 5440, 5420] dup_rows_per_dst=[2700, 2936, 2606, 2874, 2774, 2730, 2672, 2560] combine_rows_per_src=[5520, 5506, 5414, 5478, 5518, 5388, 5440, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38250 dup_rows=21782 dup_rows_remote=19074 dedup_ratio=0.332367 router_tokens_per_expert=[240, 253, 289, 260, 242, 272, 262, 240, 283, 264, 274, 267, 265, 258, 248, 240, 278, 251, 229, 244, 255, 245, 282, 224, 259, 251, 235, 260, 262, 213, 232, 233, 235, 270, 270, 234, 247, 259, 253, 272, 263, 259, 249, 275, 256, 257, 244, 263, 262, 214, 252, 266, 232, 258, 231, 270, 245, 263, 258, 236, 282, 250, 230, 236, 241, 242, 236, 264, 301, 292, 275, 248, 249, 264, 271, 285, 247, 274, 248, 252, 274, 246, 274, 251, 228, 229, 257, 270, 246, 269, 254, 248, 245, 284, 249, 260, 262, 285, 265, 273, 228, 262, 276, 246, 230, 269, 251, 264, 222, 243, 256, 245, 265, 244, 236, 269, 245, 274, 230, 277, 279, 276, 297, 251, 266, 271, 265, 272] payload_rows_per_dst=[5530, 5324, 5504, 5348, 5560, 5444, 5442, 5602] dup_rows_per_dst=[2784, 2582, 2708, 2622, 2818, 2724, 2712, 2832] combine_rows_per_src=[5530, 5324, 5504, 5348, 5560, 5444, 5442, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=228768 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156876192 ingress_bytes=232144896 link_bytes=232144896 token_bytes=313523616 weight_movement_bytes=75497472 total_nvlink_bytes=389021088 weight_to_token_ratio=0.240803 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224298171e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229264 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549008 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868752 weight_movement_bytes=18874368 total_nvlink_bytes=331743120 weight_to_token_ratio=0.060327 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38276 dup_rows=21664 dup_rows_remote=18812 dedup_ratio=0.330566 router_tokens_per_expert=[270, 249, 246, 273, 238, 243, 255, 276, 239, 261, 273, 274, 259, 230, 268, 237, 260, 229, 229, 250, 257, 272, 272, 262, 230, 254, 243, 245, 240, 257, 226, 254, 264, 265, 251, 236, 245, 275, 277, 241, 246, 248, 274, 240, 240, 215, 260, 263, 255, 279, 278, 261, 239, 259, 256, 238, 288, 258, 253, 272, 264, 245, 261, 238, 253, 255, 234, 251, 265, 254, 257, 283, 265, 254, 268, 269, 233, 270, 283, 266, 267, 235, 231, 264, 263, 277, 265, 276, 252, 271, 252, 234, 233, 250, 267, 261, 277, 273, 269, 263, 269, 254, 249, 270, 291, 277, 264, 254, 255, 287, 231, 241, 243, 245, 222, 256, 249, 247, 249, 272, 257, 248, 261, 224, 258, 256, 277, 267] payload_rows_per_dst=[5494, 5376, 5426, 5532, 5538, 5512, 5596, 5398] dup_rows_per_dst=[2660, 2584, 2654, 2756, 2782, 2712, 2852, 2664] combine_rows_per_src=[5494, 5376, 5426, 5532, 5538, 5512, 5596, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229296 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901296 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573296 weight_movement_bytes=18874368 total_nvlink_bytes=332447664 weight_to_token_ratio=0.060191 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38222 dup_rows=21832 dup_rows_remote=19064 dedup_ratio=0.333130 router_tokens_per_expert=[234, 272, 246, 243, 273, 256, 270, 270, 269, 216, 267, 245, 248, 276, 262, 234, 251, 258, 253, 231, 255, 293, 265, 263, 257, 278, 221, 273, 250, 233, 285, 300, 251, 279, 281, 249, 235, 266, 235, 258, 257, 263, 259, 249, 279, 263, 271, 255, 251, 249, 266, 272, 286, 287, 230, 280, 257, 254, 248, 253, 259, 274, 240, 219, 253, 256, 258, 275, 260, 236, 256, 264, 279, 255, 238, 263, 255, 258, 265, 272, 241, 259, 289, 250, 253, 254, 232, 244, 254, 254, 266, 244, 274, 291, 262, 266, 248, 268, 233, 214, 269, 223, 259, 274, 243, 232, 229, 266, 291, 236, 260, 274, 241, 224, 235, 256, 252, 308, 212, 248, 262, 250, 254, 286, 217, 234, 243, 229] payload_rows_per_dst=[5416, 5500, 5542, 5482, 5528, 5480, 5452, 5304] dup_rows_per_dst=[2746, 2832, 2758, 2768, 2758, 2786, 2586, 2598] combine_rows_per_src=[5416, 5500, 5542, 5482, 5528, 5480, 5452, 5304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38242 dup_rows=21848 dup_rows_remote=19088 dedup_ratio=0.333374 router_tokens_per_expert=[261, 256, 267, 250, 273, 253, 255, 275, 251, 253, 263, 263, 245, 255, 252, 271, 229, 242, 269, 251, 245, 273, 250, 255, 252, 264, 256, 254, 250, 244, 260, 258, 251, 277, 223, 285, 268, 281, 267, 242, 267, 237, 279, 259, 252, 278, 250, 239, 233, 271, 240, 246, 257, 255, 234, 258, 249, 251, 250, 241, 234, 225, 258, 237, 246, 247, 277, 270, 260, 263, 262, 233, 275, 273, 228, 233, 284, 265, 243, 244, 270, 267, 258, 239, 278, 249, 272, 237, 259, 236, 262, 246, 246, 261, 259, 225, 255, 273, 243, 267, 264, 258, 284, 257, 243, 271, 247, 246, 258, 262, 261, 279, 278, 260, 270, 267, 242, 253, 231, 280, 257, 235, 278, 259, 246, 259, 280, 249] payload_rows_per_dst=[5526, 5406, 5486, 5318, 5446, 5468, 5534, 5504] dup_rows_per_dst=[2760, 2698, 2824, 2560, 2760, 2660, 2802, 2784] combine_rows_per_src=[5526, 5406, 5486, 5318, 5446, 5468, 5534, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38120 dup_rows=21952 dup_rows_remote=19184 dedup_ratio=0.334961 router_tokens_per_expert=[265, 235, 267, 246, 249, 238, 246, 257, 262, 244, 243, 258, 263, 276, 263, 256, 254, 250, 252, 265, 260, 249, 253, 240, 249, 253, 250, 251, 253, 265, 280, 240, 276, 246, 246, 263, 264, 247, 249, 251, 253, 262, 250, 264, 266, 215, 262, 282, 276, 242, 254, 288, 228, 270, 272, 270, 236, 282, 252, 255, 250, 234, 249, 256, 282, 245, 247, 241, 258, 258, 239, 235, 249, 257, 249, 258, 260, 252, 248, 262, 269, 251, 253, 231, 264, 265, 257, 252, 248, 250, 271, 267, 261, 279, 252, 246, 249, 261, 271, 278, 242, 227, 265, 289, 290, 278, 237, 251, 265, 238, 245, 247, 274, 242, 257, 242, 254, 297, 244, 260, 269, 271, 248, 266, 245, 236, 260, 272] payload_rows_per_dst=[5406, 5426, 5476, 5458, 5412, 5464, 5430, 5512] dup_rows_per_dst=[2730, 2702, 2716, 2770, 2668, 2768, 2836, 2762] combine_rows_per_src=[5406, 5426, 5476, 5458, 5412, 5464, 5430, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38284 dup_rows=21758 dup_rows_remote=19120 dedup_ratio=0.332001 router_tokens_per_expert=[244, 235, 241, 265, 274, 264, 274, 252, 241, 237, 234, 264, 272, 259, 259, 275, 231, 241, 260, 263, 251, 242, 235, 257, 264, 243, 289, 264, 244, 245, 215, 279, 250, 264, 253, 246, 260, 273, 261, 273, 272, 267, 251, 260, 247, 228, 270, 256, 267, 267, 256, 245, 266, 250, 269, 258, 243, 245, 274, 265, 251, 227, 221, 295, 254, 264, 262, 265, 261, 233, 270, 237, 279, 243, 254, 265, 263, 244, 270, 257, 243, 247, 253, 253, 231, 260, 249, 276, 256, 234, 257, 250, 247, 251, 271, 244, 249, 253, 257, 240, 253, 256, 265, 246, 267, 297, 264, 239, 266, 254, 269, 242, 244, 244, 254, 264, 260, 264, 252, 248, 273, 287, 255, 245, 276, 253, 263, 283] payload_rows_per_dst=[5432, 5494, 5496, 5398, 5478, 5430, 5522, 5528] dup_rows_per_dst=[2748, 2638, 2680, 2800, 2764, 2614, 2712, 2802] combine_rows_per_src=[5432, 5494, 5496, 5398, 5478, 5430, 5522, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=228352 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157006848 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313785344 weight_movement_bytes=37748736 total_nvlink_bytes=351534080 weight_to_token_ratio=0.120301 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229144 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786456 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343768 weight_movement_bytes=18874368 total_nvlink_bytes=332218136 weight_to_token_ratio=0.060235 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229320 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868552 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507784 weight_movement_bytes=18874368 total_nvlink_bytes=332382152 weight_to_token_ratio=0.060204 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38530 dup_rows=21636 dup_rows_remote=19062 dedup_ratio=0.330139 router_tokens_per_expert=[253, 264, 274, 253, 264, 270, 251, 280, 257, 256, 246, 263, 275, 234, 253, 266, 254, 229, 269, 279, 266, 264, 235, 239, 267, 263, 251, 255, 214, 278, 244, 243, 233, 254, 262, 239, 259, 273, 233, 275, 244, 272, 242, 255, 289, 264, 245, 233, 260, 271, 250, 250, 269, 250, 235, 251, 240, 247, 249, 259, 255, 262, 262, 254, 216, 253, 285, 281, 272, 290, 287, 256, 262, 263, 270, 280, 260, 242, 265, 239, 223, 257, 243, 246, 259, 247, 283, 269, 232, 231, 259, 271, 268, 249, 264, 253, 224, 271, 267, 231, 264, 239, 243, 275, 273, 273, 255, 267, 268, 290, 258, 278, 235, 237, 240, 253, 255, 244, 255, 252, 256, 253, 227, 233, 253, 231, 276, 272] payload_rows_per_dst=[5452, 5434, 5488, 5472, 5564, 5546, 5574, 5370] dup_rows_per_dst=[2764, 2666, 2656, 2656, 2878, 2664, 2778, 2574] combine_rows_per_src=[5452, 5434, 5488, 5472, 5564, 5546, 5574, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229216 combine_payload_bytes=156139520 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156368736 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312508256 weight_movement_bytes=18874368 total_nvlink_bytes=331382624 weight_to_token_ratio=0.060396 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229616 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040880 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852144 weight_movement_bytes=18874368 total_nvlink_bytes=332726512 weight_to_token_ratio=0.060138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38276 dup_rows=21730 dup_rows_remote=18940 dedup_ratio=0.331573 router_tokens_per_expert=[277, 260, 279, 287, 254, 269, 251, 229, 263, 244, 269, 233, 252, 265, 260, 257, 275, 237, 228, 249, 259, 229, 258, 253, 262, 281, 266, 247, 232, 268, 274, 249, 258, 250, 244, 231, 255, 247, 238, 267, 244, 271, 239, 249, 253, 233, 224, 273, 264, 246, 264, 256, 284, 250, 253, 246, 261, 253, 268, 251, 255, 261, 234, 265, 256, 289, 280, 270, 264, 246, 252, 270, 253, 251, 270, 240, 267, 254, 233, 256, 265, 254, 244, 270, 263, 248, 256, 238, 260, 236, 253, 256, 262, 283, 210, 238, 227, 271, 268, 239, 277, 271, 256, 226, 266, 258, 242, 267, 265, 282, 277, 266, 254, 235, 262, 254, 258, 248, 264, 283, 270, 259, 273, 241, 227, 267, 258, 267] payload_rows_per_dst=[5530, 5432, 5442, 5536, 5462, 5392, 5538, 5474] dup_rows_per_dst=[2790, 2702, 2684, 2714, 2840, 2680, 2728, 2592] combine_rows_per_src=[5530, 5432, 5442, 5536, 5462, 5392, 5538, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38200 dup_rows=21858 dup_rows_remote=19104 dedup_ratio=0.333527 router_tokens_per_expert=[236, 207, 260, 271, 245, 256, 269, 227, 260, 250, 237, 254, 263, 274, 235, 257, 269, 259, 242, 262, 256, 210, 221, 260, 262, 272, 260, 229, 255, 253, 264, 291, 273, 259, 261, 276, 244, 255, 263, 222, 271, 249, 242, 267, 255, 246, 271, 262, 262, 244, 252, 269, 240, 261, 276, 241, 226, 255, 283, 249, 273, 262, 276, 246, 258, 257, 243, 221, 268, 252, 271, 266, 256, 236, 256, 236, 270, 274, 261, 274, 255, 259, 236, 259, 239, 270, 258, 268, 264, 266, 255, 260, 248, 261, 239, 244, 271, 260, 250, 262, 260, 260, 281, 265, 247, 272, 226, 245, 266, 262, 276, 242, 245, 264, 286, 255, 260, 249, 244, 254, 256, 274, 254, 273, 251, 277, 250, 254] payload_rows_per_dst=[5366, 5480, 5478, 5514, 5440, 5432, 5470, 5498] dup_rows_per_dst=[2636, 2650, 2754, 2716, 2758, 2730, 2820, 2794] combine_rows_per_src=[5366, 5480, 5478, 5514, 5440, 5432, 5470, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=230368 combine_payload_bytes=157818880 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=158049248 ingress_bytes=176693248 link_bytes=176693248 token_bytes=315868128 weight_movement_bytes=18874368 total_nvlink_bytes=334742496 weight_to_token_ratio=0.059754 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692987764e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=228864 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157007360 ingress_bytes=213401600 link_bytes=213401600 token_bytes=313785856 weight_movement_bytes=56623104 total_nvlink_bytes=370408960 weight_to_token_ratio=0.180451 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044709131e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229216 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156696416 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313163616 weight_movement_bytes=18874368 total_nvlink_bytes=332037984 weight_to_token_ratio=0.060270 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38212 dup_rows=21820 dup_rows_remote=19104 dedup_ratio=0.332947 router_tokens_per_expert=[247, 275, 232, 276, 260, 248, 265, 270, 229, 240, 243, 264, 229, 266, 253, 258, 258, 262, 280, 278, 263, 270, 252, 284, 251, 265, 250, 265, 282, 262, 260, 284, 240, 238, 235, 261, 239, 230, 258, 259, 262, 277, 257, 261, 254, 257, 268, 225, 250, 255, 263, 270, 234, 246, 269, 266, 237, 277, 270, 260, 275, 294, 268, 260, 274, 267, 270, 257, 246, 235, 254, 226, 247, 246, 247, 257, 252, 263, 260, 261, 256, 253, 273, 234, 247, 257, 286, 274, 224, 255, 233, 236, 262, 259, 222, 261, 261, 238, 287, 263, 242, 239, 277, 229, 281, 257, 262, 268, 266, 230, 258, 252, 253, 252, 262, 271, 241, 252, 242, 245, 238, 232, 266, 241, 234, 240, 260, 299] payload_rows_per_dst=[5422, 5568, 5416, 5560, 5462, 5342, 5504, 5442] dup_rows_per_dst=[2688, 2964, 2626, 2828, 2662, 2722, 2716, 2614] combine_rows_per_src=[5422, 5568, 5416, 5560, 5462, 5342, 5504, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38240 dup_rows=21970 dup_rows_remote=19296 dedup_ratio=0.335236 router_tokens_per_expert=[268, 284, 257, 237, 276, 244, 257, 239, 257, 268, 258, 253, 266, 244, 239, 257, 256, 216, 269, 270, 278, 298, 273, 257, 268, 251, 245, 258, 234, 275, 281, 272, 249, 264, 260, 260, 262, 231, 255, 242, 286, 263, 251, 252, 251, 264, 282, 256, 249, 223, 247, 219, 243, 252, 276, 255, 270, 254, 259, 243, 249, 262, 250, 249, 262, 269, 259, 257, 258, 253, 257, 286, 249, 249, 282, 237, 230, 243, 252, 270, 269, 238, 228, 257, 279, 239, 295, 258, 221, 245, 257, 260, 257, 278, 267, 274, 243, 233, 219, 282, 281, 264, 245, 270, 257, 236, 276, 247, 257, 249, 267, 269, 260, 255, 249, 229, 285, 230, 295, 254, 233, 241, 220, 249, 230, 243, 264, 268] payload_rows_per_dst=[5488, 5478, 5478, 5326, 5504, 5454, 5490, 5348] dup_rows_per_dst=[2720, 2924, 2778, 2674, 2722, 2790, 2700, 2662] combine_rows_per_src=[5488, 5478, 5478, 5326, 5504, 5454, 5490, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38372 dup_rows=21744 dup_rows_remote=19092 dedup_ratio=0.331787 router_tokens_per_expert=[271, 229, 265, 256, 260, 252, 265, 252, 248, 248, 254, 257, 250, 257, 256, 257, 271, 275, 248, 275, 225, 248, 262, 267, 258, 273, 242, 250, 254, 253, 220, 254, 244, 268, 279, 272, 253, 267, 267, 255, 240, 239, 270, 249, 285, 246, 238, 281, 242, 255, 279, 263, 259, 233, 278, 243, 265, 228, 232, 270, 251, 225, 232, 274, 244, 263, 252, 261, 228, 254, 260, 255, 251, 241, 263, 236, 244, 256, 263, 265, 265, 267, 246, 246, 275, 280, 242, 251, 277, 242, 256, 272, 260, 245, 244, 249, 256, 269, 247, 272, 257, 270, 276, 252, 259, 248, 265, 287, 227, 258, 245, 286, 259, 268, 239, 275, 289, 230, 251, 267, 277, 261, 250, 233, 244, 238, 249, 277] payload_rows_per_dst=[5406, 5478, 5578, 5476, 5420, 5496, 5510, 5428] dup_rows_per_dst=[2748, 2672, 2728, 2582, 2652, 2738, 2838, 2786] combine_rows_per_src=[5406, 5478, 5578, 5476, 5420, 5496, 5510, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43976 payload_rows_remote=38534 dup_rows=21560 dup_rows_remote=18940 dedup_ratio=0.328979 router_tokens_per_expert=[261, 229, 254, 215, 269, 265, 265, 247, 243, 282, 272, 259, 245, 251, 297, 238, 249, 269, 272, 248, 244, 280, 247, 278, 252, 239, 246, 277, 261, 286, 239, 223, 235, 255, 247, 258, 231, 251, 248, 248, 268, 269, 301, 249, 256, 252, 273, 235, 254, 261, 286, 259, 253, 261, 261, 238, 276, 228, 262, 262, 270, 256, 244, 277, 253, 271, 272, 246, 228, 272, 273, 256, 260, 259, 251, 255, 236, 265, 251, 260, 251, 236, 274, 245, 268, 256, 238, 248, 250, 275, 241, 234, 244, 266, 259, 246, 266, 250, 264, 268, 254, 280, 280, 266, 235, 256, 234, 242, 250, 255, 243, 268, 246, 270, 260, 255, 270, 298, 251, 269, 252, 254, 232, 245, 249, 262, 244, 235] payload_rows_per_dst=[5490, 5582, 5470, 5500, 5528, 5442, 5526, 5438] dup_rows_per_dst=[2694, 2638, 2682, 2796, 2688, 2620, 2696, 2746] combine_rows_per_src=[5490, 5582, 5470, 5500, 5528, 5442, 5526, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38176 dup_rows=21956 dup_rows_remote=19198 dedup_ratio=0.335022 router_tokens_per_expert=[253, 245, 236, 272, 238, 251, 260, 249, 243, 243, 219, 244, 250, 246, 253, 280, 252, 261, 251, 239, 266, 260, 239, 248, 245, 244, 269, 272, 233, 263, 288, 251, 293, 257, 244, 255, 253, 239, 254, 266, 252, 244, 267, 252, 264, 272, 239, 261, 277, 264, 239, 259, 233, 262, 281, 250, 277, 260, 243, 269, 282, 249, 248, 253, 252, 243, 254, 243, 246, 270, 261, 224, 272, 232, 246, 250, 237, 246, 253, 270, 236, 268, 271, 264, 254, 269, 266, 254, 250, 264, 276, 262, 274, 265, 253, 270, 277, 245, 277, 277, 245, 234, 261, 261, 243, 248, 210, 287, 249, 251, 272, 252, 295, 250, 249, 236, 275, 270, 249, 258, 241, 253, 268, 253, 290, 255, 265, 256] payload_rows_per_dst=[5352, 5404, 5420, 5512, 5382, 5576, 5388, 5546] dup_rows_per_dst=[2612, 2758, 2804, 2780, 2616, 2816, 2790, 2780] combine_rows_per_src=[5352, 5404, 5420, 5512, 5382, 5576, 5388, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229264 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156745616 ingress_bytes=213139456 link_bytes=213139456 token_bytes=313261968 weight_movement_bytes=56623104 total_nvlink_bytes=369885072 weight_to_token_ratio=0.180753 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042197396e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=230144 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156861184 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313492224 weight_movement_bytes=18874368 total_nvlink_bytes=332366592 weight_to_token_ratio=0.060207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38062 dup_rows=21912 dup_rows_remote=19014 dedup_ratio=0.334351 router_tokens_per_expert=[286, 273, 260, 252, 266, 252, 251, 281, 271, 263, 237, 247, 293, 242, 299, 252, 254, 254, 254, 254, 248, 232, 278, 265, 273, 265, 278, 252, 274, 239, 240, 277, 250, 270, 239, 286, 262, 270, 247, 261, 240, 240, 236, 240, 270, 265, 266, 227, 237, 242, 265, 225, 261, 255, 230, 240, 226, 207, 249, 253, 232, 254, 265, 252, 264, 271, 256, 275, 255, 244, 246, 216, 257, 254, 236, 291, 273, 254, 259, 249, 254, 254, 262, 270, 230, 258, 283, 258, 254, 241, 248, 257, 261, 270, 252, 229, 272, 255, 280, 242, 230, 265, 235, 272, 277, 269, 261, 267, 254, 257, 239, 249, 246, 264, 263, 295, 241, 283, 234, 244, 263, 262, 271, 263, 243, 266, 264, 237] payload_rows_per_dst=[5428, 5402, 5446, 5396, 5368, 5436, 5586, 5562] dup_rows_per_dst=[2678, 2552, 2854, 2666, 2660, 2726, 2878, 2898] combine_rows_per_src=[5428, 5402, 5446, 5396, 5368, 5436, 5586, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229856 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157401568 ingress_bytes=194920448 link_bytes=194920448 token_bytes=314573280 weight_movement_bytes=37748736 total_nvlink_bytes=352322016 weight_to_token_ratio=0.120000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867631826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38284 dup_rows=21780 dup_rows_remote=18892 dedup_ratio=0.332336 router_tokens_per_expert=[252, 277, 250, 259, 260, 295, 272, 250, 241, 262, 287, 258, 237, 243, 236, 266, 237, 248, 271, 250, 247, 242, 288, 237, 242, 254, 261, 253, 263, 251, 291, 245, 257, 239, 263, 260, 250, 237, 255, 233, 266, 255, 261, 242, 258, 220, 261, 240, 273, 254, 265, 242, 269, 257, 279, 232, 266, 269, 269, 228, 219, 265, 259, 257, 245, 242, 262, 232, 276, 242, 236, 270, 274, 263, 274, 249, 284, 281, 258, 266, 255, 264, 267, 244, 280, 268, 265, 254, 253, 240, 254, 277, 240, 270, 263, 229, 265, 228, 257, 269, 245, 266, 279, 260, 258, 275, 220, 251, 251, 265, 227, 255, 257, 250, 261, 254, 272, 285, 277, 221, 239, 251, 270, 260, 265, 241, 247, 245] payload_rows_per_dst=[5472, 5428, 5424, 5520, 5588, 5432, 5422, 5470] dup_rows_per_dst=[2888, 2732, 2604, 2636, 2674, 2814, 2712, 2720] combine_rows_per_src=[5472, 5428, 5424, 5520, 5588, 5432, 5422, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157835264 dispatch_meta_bytes=229896 combine_payload_bytes=157835264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158065160 ingress_bytes=176709632 link_bytes=176709632 token_bytes=315900424 weight_movement_bytes=18874368 total_nvlink_bytes=334774792 weight_to_token_ratio=0.059748 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693144747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38254 dup_rows=21838 dup_rows_remote=19136 dedup_ratio=0.333221 router_tokens_per_expert=[246, 259, 278, 255, 285, 261, 256, 229, 267, 254, 273, 271, 248, 241, 253, 255, 251, 267, 236, 257, 253, 275, 248, 247, 263, 271, 256, 240, 276, 295, 244, 271, 269, 267, 267, 231, 260, 249, 271, 260, 261, 266, 226, 254, 270, 244, 237, 241, 293, 267, 270, 268, 273, 236, 246, 225, 247, 240, 260, 261, 221, 244, 258, 239, 268, 238, 276, 234, 247, 242, 279, 269, 233, 243, 252, 244, 270, 253, 229, 284, 257, 246, 256, 227, 256, 270, 271, 265, 270, 237, 271, 260, 263, 254, 242, 285, 250, 262, 272, 255, 272, 275, 232, 252, 235, 274, 257, 249, 270, 279, 233, 260, 259, 263, 240, 250, 264, 236, 239, 259, 277, 244, 269, 256, 248, 235, 247, 262] payload_rows_per_dst=[5508, 5512, 5444, 5474, 5362, 5468, 5480, 5450] dup_rows_per_dst=[2754, 2788, 2702, 2622, 2760, 2792, 2774, 2646] combine_rows_per_src=[5508, 5512, 5444, 5474, 5362, 5468, 5480, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229496 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598392 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967288 weight_movement_bytes=18874368 total_nvlink_bytes=331841656 weight_to_token_ratio=0.060308 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228304 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156130256 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312032208 weight_movement_bytes=18874368 total_nvlink_bytes=330906576 weight_to_token_ratio=0.060489 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228704 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157039968 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851232 weight_movement_bytes=37748736 total_nvlink_bytes=351599968 weight_to_token_ratio=0.120276 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229560 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917944 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606328 weight_movement_bytes=18874368 total_nvlink_bytes=332480696 weight_to_token_ratio=0.060185 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38284 dup_rows=21754 dup_rows_remote=19064 dedup_ratio=0.331940 router_tokens_per_expert=[248, 220, 261, 252, 276, 235, 255, 293, 237, 252, 254, 239, 247, 256, 262, 250, 230, 245, 265, 269, 256, 231, 273, 227, 238, 246, 232, 245, 241, 255, 240, 272, 291, 264, 253, 259, 258, 277, 263, 249, 236, 270, 274, 299, 258, 232, 276, 248, 233, 257, 247, 273, 265, 291, 273, 254, 273, 261, 265, 233, 283, 259, 234, 232, 263, 276, 250, 248, 258, 241, 256, 256, 270, 246, 246, 278, 247, 250, 276, 266, 278, 254, 226, 241, 277, 249, 255, 259, 239, 246, 287, 262, 258, 260, 278, 240, 237, 249, 253, 246, 271, 257, 242, 256, 276, 253, 245, 270, 233, 270, 253, 283, 260, 234, 244, 254, 261, 244, 259, 268, 290, 263, 253, 266, 241, 272, 240, 247] payload_rows_per_dst=[5448, 5382, 5542, 5548, 5424, 5490, 5498, 5450] dup_rows_per_dst=[2626, 2548, 2872, 2718, 2830, 2728, 2690, 2742] combine_rows_per_src=[5448, 5382, 5542, 5548, 5424, 5490, 5498, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43510 payload_rows_remote=38046 dup_rows=22026 dup_rows_remote=19184 dedup_ratio=0.336090 router_tokens_per_expert=[258, 248, 268, 242, 282, 254, 253, 258, 259, 257, 246, 259, 286, 238, 269, 247, 256, 250, 247, 267, 271, 285, 248, 252, 257, 236, 241, 262, 254, 258, 262, 244, 286, 264, 261, 238, 243, 249, 248, 276, 242, 271, 282, 253, 244, 257, 240, 241, 285, 264, 258, 263, 289, 280, 251, 236, 255, 260, 280, 241, 269, 234, 236, 252, 251, 261, 242, 277, 279, 263, 283, 262, 247, 267, 291, 276, 248, 230, 280, 252, 241, 258, 245, 256, 261, 234, 220, 265, 248, 229, 262, 258, 247, 248, 260, 239, 258, 264, 245, 232, 266, 249, 233, 257, 253, 268, 261, 230, 242, 247, 284, 256, 253, 241, 239, 251, 260, 243, 287, 258, 259, 270, 264, 270, 247, 256, 256, 227] payload_rows_per_dst=[5452, 5448, 5438, 5464, 5538, 5300, 5416, 5454] dup_rows_per_dst=[2796, 2732, 2752, 2842, 2880, 2642, 2674, 2708] combine_rows_per_src=[5452, 5448, 5438, 5464, 5538, 5300, 5416, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38030 dup_rows=21982 dup_rows_remote=19130 dedup_ratio=0.335419 router_tokens_per_expert=[267, 261, 283, 243, 259, 242, 241, 263, 244, 269, 268, 220, 237, 270, 247, 258, 284, 264, 241, 258, 222, 259, 271, 275, 255, 269, 257, 252, 258, 247, 264, 255, 239, 256, 255, 252, 242, 240, 264, 248, 264, 249, 261, 255, 248, 244, 239, 273, 258, 241, 249, 249, 291, 280, 252, 246, 270, 274, 270, 269, 259, 233, 266, 252, 278, 266, 259, 275, 240, 263, 256, 254, 249, 266, 251, 246, 255, 285, 286, 259, 236, 274, 253, 275, 225, 250, 283, 262, 274, 238, 264, 242, 250, 267, 278, 260, 258, 236, 262, 231, 239, 234, 245, 252, 232, 249, 216, 250, 287, 262, 264, 235, 257, 255, 267, 278, 281, 263, 248, 254, 234, 274, 250, 287, 237, 255, 229, 237] payload_rows_per_dst=[5478, 5370, 5458, 5536, 5524, 5496, 5234, 5458] dup_rows_per_dst=[2666, 2892, 2600, 2782, 2852, 2766, 2670, 2754] combine_rows_per_src=[5478, 5370, 5458, 5536, 5524, 5496, 5234, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229392 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040656 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851920 weight_movement_bytes=37748736 total_nvlink_bytes=351600656 weight_to_token_ratio=0.120276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38272 dup_rows=21740 dup_rows_remote=19010 dedup_ratio=0.331726 router_tokens_per_expert=[230, 251, 248, 315, 243, 259, 258, 251, 256, 254, 256, 289, 252, 255, 230, 251, 246, 260, 274, 257, 251, 280, 262, 269, 252, 267, 235, 270, 262, 294, 252, 251, 279, 243, 271, 258, 257, 257, 257, 268, 268, 245, 247, 246, 236, 264, 256, 248, 245, 262, 247, 245, 250, 276, 254, 246, 273, 249, 225, 239, 262, 258, 231, 304, 250, 232, 254, 244, 263, 245, 256, 258, 271, 259, 267, 284, 241, 246, 246, 277, 269, 263, 286, 267, 263, 243, 245, 263, 253, 255, 250, 237, 260, 253, 268, 252, 292, 239, 281, 260, 253, 256, 259, 238, 254, 234, 254, 257, 253, 244, 269, 233, 272, 244, 248, 281, 240, 232, 245, 263, 246, 255, 236, 241, 209, 258, 265, 291] payload_rows_per_dst=[5540, 5520, 5454, 5476, 5492, 5524, 5434, 5356] dup_rows_per_dst=[2656, 2844, 2746, 2656, 2694, 2730, 2718, 2696] combine_rows_per_src=[5540, 5520, 5454, 5476, 5492, 5524, 5434, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43524 payload_rows_remote=38058 dup_rows=22012 dup_rows_remote=19288 dedup_ratio=0.335876 router_tokens_per_expert=[262, 280, 265, 267, 277, 271, 253, 264, 259, 235, 236, 245, 276, 267, 265, 271, 271, 244, 234, 253, 266, 273, 246, 259, 284, 285, 231, 262, 262, 231, 238, 256, 248, 252, 272, 272, 274, 269, 254, 280, 279, 248, 226, 241, 262, 261, 238, 249, 238, 255, 249, 240, 264, 265, 275, 248, 263, 247, 245, 250, 241, 290, 244, 254, 227, 247, 242, 265, 289, 278, 233, 281, 268, 242, 262, 264, 242, 240, 260, 258, 277, 219, 262, 248, 279, 246, 233, 255, 245, 278, 254, 237, 253, 240, 257, 243, 245, 262, 260, 231, 279, 257, 249, 246, 235, 271, 247, 248, 287, 245, 253, 274, 253, 238, 278, 263, 227, 253, 245, 249, 255, 263, 246, 274, 255, 252, 265, 258] payload_rows_per_dst=[5496, 5466, 5492, 5480, 5392, 5330, 5434, 5434] dup_rows_per_dst=[2890, 2724, 2758, 2656, 2804, 2722, 2744, 2714] combine_rows_per_src=[5496, 5466, 5492, 5480, 5392, 5330, 5434, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=228920 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156065336 ingress_bytes=193585152 link_bytes=193585152 token_bytes=311901752 weight_movement_bytes=37748736 total_nvlink_bytes=349650488 weight_to_token_ratio=0.121028 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854837677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38228 dup_rows=21648 dup_rows_remote=18718 dedup_ratio=0.330322 router_tokens_per_expert=[244, 259, 277, 291, 246, 270, 250, 276, 241, 252, 239, 254, 223, 288, 251, 265, 245, 242, 279, 236, 250, 274, 259, 255, 269, 246, 255, 257, 268, 258, 243, 238, 254, 248, 236, 253, 259, 255, 280, 249, 264, 257, 255, 266, 242, 239, 272, 270, 235, 264, 238, 248, 235, 250, 281, 215, 244, 269, 272, 277, 253, 231, 285, 270, 254, 240, 240, 288, 272, 245, 265, 288, 249, 248, 247, 229, 269, 259, 245, 261, 253, 277, 262, 246, 255, 266, 231, 266, 273, 252, 279, 254, 265, 267, 260, 261, 268, 252, 255, 271, 277, 238, 242, 262, 246, 262, 242, 250, 243, 252, 271, 268, 257, 242, 270, 246, 256, 266, 262, 258, 257, 236, 226, 233, 270, 235, 262, 261] payload_rows_per_dst=[5402, 5310, 5480, 5534, 5312, 5652, 5538, 5660] dup_rows_per_dst=[2542, 2548, 2718, 2786, 2404, 2838, 2882, 2930] combine_rows_per_src=[5402, 5310, 5480, 5534, 5312, 5652, 5538, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=228640 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155999520 ingress_bytes=212393984 link_bytes=212393984 token_bytes=311770400 weight_movement_bytes=56623104 total_nvlink_bytes=368393504 weight_to_token_ratio=0.181618 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035054650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38264 dup_rows=21684 dup_rows_remote=18926 dedup_ratio=0.330872 router_tokens_per_expert=[267, 266, 260, 252, 258, 243, 270, 281, 267, 244, 264, 265, 295, 261, 240, 233, 224, 257, 255, 250, 253, 257, 252, 265, 252, 255, 266, 272, 251, 291, 270, 249, 237, 241, 221, 255, 213, 283, 258, 274, 250, 260, 266, 251, 256, 242, 286, 261, 243, 255, 269, 270, 274, 228, 261, 276, 263, 279, 246, 252, 248, 251, 247, 246, 224, 262, 254, 244, 226, 253, 276, 256, 247, 260, 270, 264, 260, 247, 284, 265, 237, 268, 247, 262, 242, 257, 250, 257, 253, 272, 250, 274, 236, 252, 278, 277, 283, 258, 257, 255, 249, 256, 241, 263, 246, 252, 238, 265, 266, 269, 276, 247, 248, 245, 239, 258, 259, 254, 233, 249, 270, 238, 261, 266, 247, 237, 244, 248] payload_rows_per_dst=[5588, 5596, 5446, 5550, 5514, 5482, 5348, 5328] dup_rows_per_dst=[2758, 2752, 2668, 2954, 2670, 2722, 2496, 2664] combine_rows_per_src=[5588, 5596, 5446, 5550, 5514, 5482, 5348, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38246 dup_rows=21822 dup_rows_remote=19116 dedup_ratio=0.332977 router_tokens_per_expert=[254, 294, 272, 231, 274, 267, 274, 258, 241, 253, 225, 229, 232, 251, 247, 254, 273, 263, 262, 231, 267, 264, 243, 267, 265, 253, 260, 259, 256, 266, 277, 270, 264, 244, 250, 257, 269, 257, 254, 241, 255, 241, 277, 237, 243, 255, 284, 259, 240, 235, 251, 247, 246, 250, 276, 240, 275, 252, 266, 256, 245, 238, 276, 258, 236, 280, 258, 266, 256, 248, 238, 223, 267, 259, 271, 295, 252, 243, 260, 249, 230, 266, 277, 232, 264, 248, 263, 248, 252, 259, 250, 288, 267, 251, 260, 249, 276, 261, 260, 246, 262, 243, 251, 234, 258, 264, 252, 267, 245, 222, 251, 247, 279, 251, 284, 250, 248, 270, 255, 254, 256, 273, 248, 274, 238, 274, 247, 253] payload_rows_per_dst=[5400, 5516, 5468, 5424, 5500, 5496, 5364, 5546] dup_rows_per_dst=[2712, 2836, 2706, 2678, 2702, 2712, 2714, 2762] combine_rows_per_src=[5400, 5516, 5468, 5424, 5500, 5496, 5364, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=229384 combine_payload_bytes=155885568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156114952 ingress_bytes=193634304 link_bytes=193634304 token_bytes=312000520 weight_movement_bytes=37748736 total_nvlink_bytes=349749256 weight_to_token_ratio=0.120989 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855308627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229128 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156991240 ingress_bytes=156762112 link_bytes=156991240 token_bytes=313753352 weight_movement_bytes=0 total_nvlink_bytes=313753352 weight_to_token_ratio=0.000000 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.504212817e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=227784 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156809672 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313391560 weight_movement_bytes=18874368 total_nvlink_bytes=332265928 weight_to_token_ratio=0.060226 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228760 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958104 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313687448 weight_movement_bytes=37748736 total_nvlink_bytes=351436184 weight_to_token_ratio=0.120339 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38346 dup_rows=21786 dup_rows_remote=18956 dedup_ratio=0.332428 router_tokens_per_expert=[249, 259, 226, 257, 254, 255, 271, 252, 252, 248, 268, 257, 296, 252, 280, 237, 311, 261, 249, 261, 252, 254, 279, 249, 263, 269, 236, 235, 297, 257, 267, 260, 242, 230, 279, 237, 250, 253, 255, 251, 266, 240, 234, 266, 244, 254, 272, 249, 236, 249, 267, 255, 267, 266, 253, 240, 263, 269, 279, 272, 280, 236, 234, 269, 287, 244, 263, 240, 269, 261, 253, 263, 281, 234, 276, 253, 243, 250, 245, 236, 266, 250, 251, 252, 245, 262, 251, 257, 237, 272, 247, 238, 237, 240, 270, 243, 251, 267, 257, 252, 271, 260, 272, 238, 237, 269, 254, 268, 247, 273, 256, 245, 256, 269, 261, 243, 232, 259, 264, 232, 267, 247, 243, 278, 262, 253, 264, 235] payload_rows_per_dst=[5494, 5504, 5416, 5540, 5504, 5412, 5404, 5476] dup_rows_per_dst=[2732, 2896, 2628, 2730, 2692, 2624, 2830, 2654] combine_rows_per_src=[5494, 5504, 5416, 5540, 5504, 5412, 5404, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229448 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156885064 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540680 weight_movement_bytes=56623104 total_nvlink_bytes=370163784 weight_to_token_ratio=0.180593 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38322 dup_rows=21796 dup_rows_remote=19090 dedup_ratio=0.332581 router_tokens_per_expert=[226, 245, 265, 252, 268, 268, 250, 278, 236, 244, 289, 237, 275, 259, 259, 238, 263, 247, 238, 273, 254, 254, 264, 253, 296, 254, 248, 262, 248, 257, 246, 252, 256, 239, 252, 267, 274, 248, 256, 269, 257, 284, 228, 252, 273, 249, 267, 247, 250, 259, 282, 219, 253, 272, 263, 242, 267, 248, 255, 258, 228, 260, 252, 254, 259, 242, 251, 273, 248, 245, 253, 249, 245, 241, 275, 262, 240, 231, 260, 241, 268, 253, 270, 274, 244, 269, 254, 266, 255, 254, 246, 232, 232, 248, 233, 253, 259, 270, 271, 243, 238, 258, 247, 255, 249, 274, 295, 243, 256, 275, 235, 249, 262, 261, 269, 297, 260, 258, 260, 251, 255, 280, 255, 261, 259, 271, 262, 246] payload_rows_per_dst=[5452, 5430, 5522, 5418, 5460, 5424, 5470, 5564] dup_rows_per_dst=[2726, 2788, 2714, 2706, 2570, 2678, 2764, 2850] combine_rows_per_src=[5452, 5430, 5522, 5418, 5460, 5424, 5470, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38200 dup_rows=21816 dup_rows_remote=19076 dedup_ratio=0.332886 router_tokens_per_expert=[259, 293, 236, 248, 267, 243, 252, 268, 238, 260, 255, 255, 223, 281, 268, 252, 275, 265, 263, 249, 251, 271, 237, 263, 293, 242, 242, 257, 255, 262, 256, 260, 253, 251, 249, 266, 243, 250, 223, 265, 252, 241, 266, 246, 270, 269, 243, 225, 258, 234, 282, 249, 248, 260, 246, 227, 253, 309, 257, 259, 247, 236, 263, 240, 282, 279, 262, 283, 237, 248, 263, 247, 246, 254, 258, 279, 240, 244, 251, 257, 279, 254, 263, 272, 259, 286, 260, 257, 277, 248, 243, 278, 259, 271, 281, 270, 241, 240, 248, 251, 259, 256, 271, 249, 248, 222, 245, 280, 275, 228, 239, 289, 217, 263, 257, 275, 245, 251, 242, 244, 263, 250, 275, 251, 271, 232, 248, 237] payload_rows_per_dst=[5428, 5544, 5376, 5466, 5520, 5580, 5378, 5428] dup_rows_per_dst=[2768, 2738, 2648, 2670, 2740, 2934, 2704, 2614] combine_rows_per_src=[5428, 5544, 5376, 5466, 5520, 5580, 5378, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229208 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157294424 ingress_bytes=213688320 link_bytes=213688320 token_bytes=314359640 weight_movement_bytes=56623104 total_nvlink_bytes=370982744 weight_to_token_ratio=0.180122 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.069444 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047456341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229648 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157196560 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314163472 weight_movement_bytes=56623104 total_nvlink_bytes=370786576 weight_to_token_ratio=0.180235 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38388 dup_rows=21716 dup_rows_remote=19040 dedup_ratio=0.331360 router_tokens_per_expert=[254, 269, 248, 285, 237, 272, 271, 259, 250, 259, 267, 292, 242, 262, 244, 271, 245, 285, 227, 271, 245, 245, 249, 275, 276, 253, 248, 249, 258, 240, 250, 238, 250, 244, 250, 264, 267, 257, 300, 242, 223, 238, 249, 281, 228, 253, 248, 253, 279, 262, 250, 227, 257, 288, 251, 267, 232, 243, 246, 271, 251, 255, 236, 271, 266, 277, 245, 263, 271, 247, 266, 269, 246, 276, 255, 262, 235, 266, 277, 271, 224, 266, 272, 261, 208, 246, 279, 220, 240, 259, 249, 280, 287, 275, 273, 244, 239, 279, 256, 270, 210, 247, 269, 279, 262, 265, 258, 247, 251, 241, 246, 253, 244, 269, 228, 271, 266, 249, 267, 247, 262, 250, 259, 225, 231, 272, 261, 251] payload_rows_per_dst=[5544, 5432, 5492, 5450, 5574, 5398, 5510, 5420] dup_rows_per_dst=[2820, 2676, 2808, 2722, 2810, 2562, 2634, 2684] combine_rows_per_src=[5544, 5432, 5492, 5450, 5574, 5398, 5510, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=37890 dup_rows=21924 dup_rows_remote=18742 dedup_ratio=0.334534 router_tokens_per_expert=[252, 248, 245, 261, 265, 286, 231, 235, 240, 258, 245, 286, 272, 225, 249, 244, 243, 264, 235, 275, 246, 239, 250, 252, 244, 247, 239, 263, 250, 267, 241, 270, 284, 234, 267, 254, 247, 258, 265, 241, 238, 255, 244, 238, 273, 267, 263, 243, 277, 257, 241, 276, 224, 264, 264, 254, 251, 271, 262, 270, 244, 250, 250, 279, 271, 230, 260, 252, 261, 275, 234, 249, 260, 258, 277, 258, 241, 253, 272, 259, 242, 241, 269, 276, 262, 280, 256, 232, 248, 260, 264, 265, 242, 238, 272, 235, 255, 268, 269, 246, 249, 268, 292, 264, 245, 272, 233, 250, 252, 245, 226, 282, 284, 262, 264, 274, 241, 269, 246, 249, 309, 250, 234, 255, 237, 267, 281, 266] payload_rows_per_dst=[5384, 5310, 5446, 5336, 5432, 5448, 5534, 5722] dup_rows_per_dst=[2574, 2492, 2638, 2652, 2822, 2740, 2824, 3182] combine_rows_per_src=[5384, 5310, 5446, 5336, 5432, 5448, 5534, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38318 dup_rows=21700 dup_rows_remote=19016 dedup_ratio=0.331116 router_tokens_per_expert=[266, 251, 279, 266, 260, 266, 251, 234, 279, 267, 261, 274, 260, 248, 241, 250, 274, 257, 251, 247, 250, 264, 258, 251, 240, 267, 242, 266, 284, 275, 251, 271, 266, 228, 251, 227, 215, 246, 258, 245, 267, 231, 249, 244, 226, 254, 262, 233, 263, 238, 275, 265, 239, 271, 264, 248, 241, 261, 286, 254, 267, 268, 272, 269, 239, 254, 281, 271, 269, 234, 275, 243, 277, 275, 243, 267, 246, 260, 243, 264, 254, 240, 247, 265, 236, 231, 267, 263, 272, 282, 273, 257, 267, 240, 258, 249, 264, 280, 249, 234, 239, 267, 282, 266, 258, 245, 243, 252, 263, 251, 241, 230, 266, 267, 263, 240, 225, 247, 288, 238, 270, 258, 271, 244, 232, 280, 230, 259] payload_rows_per_dst=[5532, 5478, 5348, 5558, 5518, 5518, 5436, 5448] dup_rows_per_dst=[2774, 2818, 2456, 2804, 2764, 2684, 2692, 2708] combine_rows_per_src=[5532, 5478, 5348, 5558, 5518, 5518, 5436, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=37970 dup_rows=21866 dup_rows_remote=18858 dedup_ratio=0.333649 router_tokens_per_expert=[253, 267, 257, 257, 261, 247, 248, 276, 235, 250, 250, 271, 233, 243, 267, 258, 263, 272, 246, 262, 251, 268, 242, 293, 229, 255, 241, 221, 272, 266, 275, 263, 247, 273, 269, 276, 253, 244, 257, 240, 257, 254, 260, 247, 255, 261, 236, 259, 264, 247, 244, 239, 252, 235, 282, 237, 244, 257, 242, 261, 251, 260, 267, 262, 260, 292, 234, 234, 247, 260, 245, 254, 263, 233, 258, 252, 273, 259, 253, 262, 270, 253, 250, 269, 238, 266, 244, 261, 257, 270, 230, 249, 281, 255, 244, 247, 237, 258, 244, 265, 282, 285, 245, 256, 256, 292, 233, 270, 254, 277, 287, 259, 254, 284, 275, 267, 249, 265, 241, 252, 269, 250, 278, 255, 233, 237, 221, 251] payload_rows_per_dst=[5700, 5664, 5238, 5402, 5592, 5426, 5344, 5304] dup_rows_per_dst=[3008, 3120, 2512, 2692, 2740, 2736, 2592, 2466] combine_rows_per_src=[5700, 5664, 5238, 5402, 5592, 5426, 5344, 5304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229104 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696304 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163504 weight_movement_bytes=37748736 total_nvlink_bytes=350912240 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38392 dup_rows=21656 dup_rows_remote=18968 dedup_ratio=0.330444 router_tokens_per_expert=[246, 245, 257, 276, 238, 250, 244, 265, 265, 248, 274, 265, 257, 243, 244, 251, 261, 258, 248, 271, 272, 279, 283, 262, 257, 267, 257, 261, 221, 273, 271, 236, 260, 259, 246, 248, 270, 256, 265, 242, 282, 267, 236, 268, 227, 246, 246, 270, 253, 260, 252, 228, 251, 248, 225, 263, 236, 262, 249, 288, 270, 270, 278, 269, 240, 243, 264, 244, 247, 243, 255, 252, 233, 274, 278, 277, 263, 274, 257, 233, 283, 276, 245, 278, 255, 244, 267, 252, 272, 260, 279, 245, 261, 262, 260, 250, 247, 260, 236, 250, 236, 253, 258, 262, 273, 258, 269, 238, 260, 249, 252, 273, 245, 233, 247, 258, 228, 267, 248, 253, 257, 232, 241, 255, 254, 249, 256, 270] payload_rows_per_dst=[5440, 5520, 5488, 5536, 5466, 5504, 5490, 5436] dup_rows_per_dst=[2696, 2834, 2688, 2668, 2688, 2874, 2658, 2550] combine_rows_per_src=[5440, 5520, 5488, 5536, 5466, 5504, 5490, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15057780536 prefetch_bytes=792723456 reduce_bytes=641728512 weight_movement_bytes=1434451968 egress_bytes=7534395192 ingress_bytes=8957837312 link_bytes=8957837312 total_nvlink_bytes=16492232504 weight_to_token_ratio=0.095263 prefetch_share_of_ingress=0.088495 dedup_ratio=0.332755 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=34 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081597 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.788116450e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229712 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157466960 ingress_bytes=213860352 link_bytes=213860352 token_bytes=314704208 weight_movement_bytes=56623104 total_nvlink_bytes=371327312 weight_to_token_ratio=0.179925 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049104667e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155197440 dispatch_meta_bytes=226528 combine_payload_bytes=155197440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155423968 ingress_bytes=174071808 link_bytes=174071808 token_bytes=310621408 weight_movement_bytes=18874368 total_nvlink_bytes=329495776 weight_to_token_ratio=0.060763 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.667870416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229336 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179864 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130392 weight_movement_bytes=37748736 total_nvlink_bytes=351879128 weight_to_token_ratio=0.120169 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155525120 dispatch_meta_bytes=227312 combine_payload_bytes=155525120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155752432 ingress_bytes=174399488 link_bytes=174399488 token_bytes=311277552 weight_movement_bytes=18874368 total_nvlink_bytes=330151920 weight_to_token_ratio=0.060635 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671010084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229440 combine_payload_bytes=157253632 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157483072 ingress_bytes=195002368 link_bytes=195002368 token_bytes=314736704 weight_movement_bytes=37748736 total_nvlink_bytes=352485440 weight_to_token_ratio=0.119938 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868416743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38208 dup_rows=21882 dup_rows_remote=19172 dedup_ratio=0.333893 router_tokens_per_expert=[247, 258, 283, 256, 257, 238, 250, 251, 276, 245, 267, 249, 283, 273, 246, 251, 280, 261, 269, 261, 235, 266, 266, 237, 271, 279, 266, 235, 264, 242, 269, 248, 257, 252, 249, 248, 225, 232, 256, 251, 256, 263, 238, 263, 294, 258, 260, 248, 254, 263, 252, 251, 258, 280, 260, 245, 255, 247, 289, 246, 243, 247, 258, 234, 261, 261, 246, 281, 266, 269, 248, 269, 247, 284, 243, 232, 262, 233, 261, 249, 252, 260, 254, 259, 239, 231, 272, 240, 264, 274, 261, 247, 248, 261, 239, 284, 265, 236, 241, 253, 286, 278, 259, 253, 238, 274, 251, 249, 258, 253, 240, 244, 237, 263, 270, 283, 276, 232, 242, 262, 249, 232, 257, 268, 257, 260, 253, 241] payload_rows_per_dst=[5494, 5542, 5382, 5400, 5450, 5484, 5446, 5456] dup_rows_per_dst=[2766, 2756, 2718, 2764, 2774, 2686, 2710, 2708] combine_rows_per_src=[5494, 5542, 5382, 5400, 5450, 5484, 5446, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229520 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156729488 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313229456 weight_movement_bytes=18874368 total_nvlink_bytes=332103824 weight_to_token_ratio=0.060257 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38344 dup_rows=21764 dup_rows_remote=19010 dedup_ratio=0.332092 router_tokens_per_expert=[252, 236, 266, 253, 276, 273, 275, 225, 247, 259, 294, 294, 243, 243, 243, 240, 259, 251, 264, 262, 225, 266, 269, 253, 289, 246, 283, 267, 284, 268, 250, 255, 285, 251, 259, 273, 251, 250, 277, 268, 265, 244, 254, 253, 268, 239, 248, 256, 214, 266, 250, 246, 248, 255, 274, 237, 248, 240, 273, 248, 254, 275, 269, 274, 260, 251, 260, 266, 230, 264, 238, 254, 241, 256, 282, 254, 253, 253, 251, 278, 241, 244, 253, 256, 252, 252, 247, 243, 239, 286, 277, 215, 262, 280, 277, 247, 249, 295, 267, 294, 233, 292, 247, 251, 245, 286, 262, 271, 237, 233, 266, 239, 242, 261, 242, 252, 236, 247, 262, 256, 221, 228, 217, 280, 242, 243, 257, 231] payload_rows_per_dst=[5462, 5476, 5464, 5488, 5428, 5526, 5554, 5374] dup_rows_per_dst=[2776, 2906, 2818, 2654, 2754, 2616, 2780, 2460] combine_rows_per_src=[5462, 5476, 5464, 5488, 5428, 5526, 5554, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15064321056 prefetch_bytes=811597824 reduce_bytes=641728512 weight_movement_bytes=1453326336 egress_bytes=7537667104 ingress_bytes=8979980288 link_bytes=8980210176 total_nvlink_bytes=16517647392 weight_to_token_ratio=0.096475 prefetch_share_of_ingress=0.090379 dedup_ratio=0.332512 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=34 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081308 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.792582403e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38108 dup_rows=21868 dup_rows_remote=19146 dedup_ratio=0.333679 router_tokens_per_expert=[263, 237, 231, 262, 256, 281, 236, 258, 269, 266, 286, 259, 254, 229, 270, 238, 242, 273, 255, 246, 274, 236, 271, 259, 265, 273, 255, 271, 259, 293, 236, 233, 269, 240, 273, 251, 269, 256, 266, 221, 270, 255, 255, 280, 259, 259, 245, 256, 267, 284, 259, 265, 282, 253, 267, 259, 264, 272, 229, 263, 241, 305, 239, 259, 260, 257, 281, 262, 237, 284, 245, 263, 244, 231, 228, 256, 245, 218, 254, 231, 265, 273, 259, 259, 270, 270, 244, 249, 268, 281, 255, 242, 255, 268, 264, 254, 242, 282, 253, 258, 261, 262, 237, 245, 257, 288, 268, 232, 246, 235, 226, 250, 243, 221, 228, 265, 248, 267, 256, 263, 233, 259, 263, 241, 243, 262, 245, 249] payload_rows_per_dst=[5428, 5560, 5506, 5526, 5386, 5548, 5382, 5332] dup_rows_per_dst=[2762, 2722, 2742, 2890, 2606, 2804, 2702, 2640] combine_rows_per_src=[5428, 5560, 5506, 5526, 5386, 5548, 5382, 5332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38080 dup_rows=21952 dup_rows_remote=19072 dedup_ratio=0.334961 router_tokens_per_expert=[266, 237, 271, 248, 240, 269, 263, 255, 270, 266, 262, 263, 252, 247, 265, 256, 269, 229, 269, 273, 228, 270, 238, 230, 284, 231, 254, 243, 261, 253, 271, 232, 218, 254, 243, 286, 263, 247, 288, 268, 280, 267, 233, 223, 268, 291, 247, 249, 261, 241, 244, 264, 267, 262, 257, 252, 243, 242, 251, 244, 257, 262, 259, 260, 268, 250, 202, 225, 280, 231, 229, 246, 279, 239, 241, 260, 262, 278, 271, 232, 235, 230, 258, 256, 269, 264, 242, 262, 257, 268, 249, 238, 291, 259, 276, 264, 284, 259, 294, 254, 258, 249, 239, 260, 221, 247, 244, 289, 250, 252, 265, 249, 258, 269, 285, 266, 276, 264, 226, 262, 258, 247, 275, 277, 259, 240, 269, 256] payload_rows_per_dst=[5452, 5360, 5404, 5408, 5388, 5474, 5594, 5504] dup_rows_per_dst=[2808, 2514, 2844, 2620, 2598, 2858, 2830, 2880] combine_rows_per_src=[5452, 5360, 5404, 5408, 5388, 5474, 5594, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38294 dup_rows=21808 dup_rows_remote=19098 dedup_ratio=0.332764 router_tokens_per_expert=[220, 249, 276, 250, 265, 241, 270, 239, 237, 279, 272, 259, 255, 270, 225, 275, 263, 249, 272, 230, 247, 248, 264, 238, 269, 248, 264, 246, 253, 260, 259, 283, 270, 276, 230, 274, 237, 256, 238, 263, 238, 277, 249, 267, 266, 272, 259, 274, 245, 257, 251, 246, 240, 262, 237, 254, 271, 228, 257, 255, 280, 263, 267, 265, 267, 290, 270, 223, 254, 270, 256, 227, 268, 229, 252, 259, 272, 241, 254, 268, 258, 258, 278, 259, 243, 230, 263, 289, 230, 287, 256, 265, 252, 246, 241, 217, 265, 261, 261, 247, 235, 266, 233, 242, 255, 265, 219, 256, 251, 255, 270, 251, 270, 270, 241, 268, 267, 259, 254, 281, 282, 261, 257, 258, 247, 251, 270, 229] payload_rows_per_dst=[5458, 5492, 5512, 5448, 5530, 5434, 5382, 5472] dup_rows_per_dst=[2706, 2694, 2780, 2708, 2670, 2710, 2682, 2858] combine_rows_per_src=[5458, 5492, 5512, 5448, 5530, 5434, 5382, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229416 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286440 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343464 weight_movement_bytes=18874368 total_nvlink_bytes=333217832 weight_to_token_ratio=0.060044 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38260 dup_rows=21662 dup_rows_remote=18948 dedup_ratio=0.330536 router_tokens_per_expert=[264, 264, 261, 250, 240, 274, 264, 236, 269, 249, 247, 271, 277, 268, 252, 262, 269, 263, 233, 239, 241, 263, 289, 268, 290, 289, 263, 269, 218, 287, 277, 265, 265, 248, 251, 250, 248, 215, 237, 290, 260, 241, 238, 276, 279, 259, 255, 223, 223, 272, 242, 237, 270, 259, 268, 262, 255, 275, 271, 253, 290, 252, 227, 249, 276, 229, 268, 256, 253, 249, 233, 241, 254, 276, 268, 238, 281, 267, 262, 231, 264, 285, 272, 245, 257, 275, 254, 229, 263, 241, 229, 259, 261, 235, 266, 295, 238, 240, 252, 247, 262, 240, 246, 271, 244, 261, 243, 267, 231, 245, 249, 233, 265, 271, 279, 248, 247, 249, 251, 246, 246, 238, 274, 223, 248, 257, 282, 252] payload_rows_per_dst=[5614, 5620, 5538, 5466, 5390, 5500, 5332, 5414] dup_rows_per_dst=[2714, 2954, 2796, 2806, 2522, 2760, 2434, 2676] combine_rows_per_src=[5614, 5620, 5538, 5466, 5390, 5500, 5332, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=229016 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156319384 ingress_bytes=193839104 link_bytes=193839104 token_bytes=312409752 weight_movement_bytes=37748736 total_nvlink_bytes=350158488 weight_to_token_ratio=0.120831 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857270920e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229568 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081792 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934016 weight_movement_bytes=18874368 total_nvlink_bytes=332808384 weight_to_token_ratio=0.060122 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228608 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156204288 ingress_bytes=212598784 link_bytes=212598784 token_bytes=312179968 weight_movement_bytes=56623104 total_nvlink_bytes=368803072 weight_to_token_ratio=0.181380 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037016943e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38250 dup_rows=21736 dup_rows_remote=19016 dedup_ratio=0.331665 router_tokens_per_expert=[262, 254, 251, 267, 236, 257, 252, 277, 238, 261, 285, 264, 256, 263, 249, 249, 248, 230, 246, 261, 235, 267, 243, 265, 254, 248, 229, 255, 260, 255, 278, 252, 238, 247, 259, 243, 284, 249, 275, 244, 253, 261, 288, 276, 275, 244, 227, 247, 267, 276, 255, 233, 265, 253, 266, 250, 260, 243, 248, 248, 246, 275, 246, 256, 242, 242, 228, 275, 278, 257, 236, 237, 265, 266, 258, 261, 253, 263, 258, 267, 269, 258, 285, 226, 261, 222, 248, 236, 259, 260, 253, 245, 261, 258, 268, 249, 272, 249, 248, 228, 249, 295, 245, 264, 237, 258, 268, 260, 248, 276, 263, 275, 268, 264, 254, 239, 229, 271, 255, 281, 264, 261, 271, 220, 260, 303, 234, 271] payload_rows_per_dst=[5520, 5398, 5464, 5504, 5398, 5440, 5550, 5526] dup_rows_per_dst=[2722, 2654, 2756, 2670, 2774, 2676, 2720, 2764] combine_rows_per_src=[5520, 5398, 5464, 5504, 5398, 5440, 5550, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228832 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156941792 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313654752 weight_movement_bytes=37748736 total_nvlink_bytes=351403488 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15038023128 prefetch_bytes=754974720 reduce_bytes=792723456 weight_movement_bytes=1547698176 egress_bytes=7524500952 ingress_bytes=9061220352 link_bytes=9061220352 total_nvlink_bytes=16585721304 weight_to_token_ratio=0.102919 prefetch_share_of_ingress=0.083319 dedup_ratio=0.332450 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=42 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081019 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.808753229e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229064 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901064 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573064 weight_movement_bytes=18874368 total_nvlink_bytes=332447432 weight_to_token_ratio=0.060191 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38376 dup_rows=21668 dup_rows_remote=19026 dedup_ratio=0.330627 router_tokens_per_expert=[245, 230, 269, 265, 252, 265, 229, 271, 267, 238, 282, 270, 258, 276, 268, 260, 229, 254, 245, 262, 268, 247, 264, 265, 240, 261, 254, 270, 257, 260, 235, 259, 236, 241, 266, 265, 280, 241, 259, 249, 270, 247, 295, 258, 235, 267, 257, 261, 277, 266, 268, 243, 236, 262, 236, 258, 240, 259, 233, 285, 246, 257, 256, 268, 282, 244, 250, 252, 255, 242, 260, 262, 261, 238, 237, 253, 257, 254, 260, 260, 254, 246, 262, 243, 243, 250, 252, 253, 261, 280, 284, 251, 272, 244, 254, 242, 272, 263, 258, 244, 233, 275, 245, 256, 239, 261, 273, 285, 241, 228, 252, 277, 253, 274, 257, 254, 257, 252, 234, 241, 242, 256, 275, 252, 274, 242, 253, 260] payload_rows_per_dst=[5502, 5462, 5540, 5448, 5492, 5522, 5490, 5412] dup_rows_per_dst=[2788, 2678, 2714, 2732, 2642, 2660, 2714, 2740] combine_rows_per_src=[5502, 5462, 5540, 5448, 5492, 5522, 5490, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38248 dup_rows=21808 dup_rows_remote=18946 dedup_ratio=0.332764 router_tokens_per_expert=[255, 222, 245, 247, 234, 247, 227, 258, 277, 242, 255, 283, 267, 273, 272, 278, 262, 244, 279, 257, 290, 249, 235, 229, 269, 276, 254, 273, 238, 274, 258, 284, 261, 249, 264, 251, 274, 257, 260, 242, 282, 256, 260, 269, 270, 275, 248, 226, 240, 246, 270, 222, 267, 245, 254, 267, 260, 265, 243, 251, 230, 258, 299, 230, 249, 273, 279, 256, 242, 245, 267, 284, 269, 278, 238, 242, 276, 260, 250, 267, 252, 225, 245, 265, 254, 241, 277, 262, 268, 252, 242, 272, 253, 259, 247, 255, 237, 233, 262, 244, 234, 236, 255, 263, 245, 240, 272, 236, 244, 285, 251, 249, 244, 255, 260, 239, 267, 268, 224, 241, 280, 259, 258, 259, 258, 264, 267, 251] payload_rows_per_dst=[5468, 5480, 5578, 5446, 5558, 5472, 5306, 5420] dup_rows_per_dst=[2696, 2862, 2710, 2648, 2792, 2666, 2666, 2768] combine_rows_per_src=[5468, 5480, 5578, 5446, 5558, 5472, 5306, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38336 dup_rows=21826 dup_rows_remote=19090 dedup_ratio=0.333038 router_tokens_per_expert=[264, 224, 240, 237, 233, 222, 238, 248, 248, 272, 250, 270, 269, 244, 272, 243, 253, 259, 251, 254, 242, 279, 269, 244, 263, 259, 267, 252, 235, 241, 256, 259, 247, 266, 236, 225, 268, 236, 292, 245, 269, 267, 234, 255, 240, 266, 265, 238, 239, 275, 264, 247, 259, 272, 287, 294, 237, 263, 271, 274, 259, 230, 260, 266, 220, 251, 242, 264, 263, 255, 259, 277, 274, 237, 253, 240, 245, 263, 259, 277, 254, 257, 251, 258, 255, 237, 235, 277, 245, 275, 270, 253, 270, 235, 230, 253, 254, 269, 273, 277, 264, 245, 287, 233, 253, 261, 272, 249, 230, 260, 278, 264, 248, 253, 260, 284, 284, 267, 259, 271, 229, 276, 269, 246, 229, 266, 276, 245] payload_rows_per_dst=[5326, 5520, 5406, 5574, 5406, 5374, 5588, 5516] dup_rows_per_dst=[2622, 2646, 2692, 2820, 2752, 2736, 2750, 2808] combine_rows_per_src=[5326, 5520, 5406, 5574, 5406, 5374, 5588, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229608 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417704 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314605800 weight_movement_bytes=18874368 total_nvlink_bytes=333480168 weight_to_token_ratio=0.059994 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38386 dup_rows=21656 dup_rows_remote=18870 dedup_ratio=0.330444 router_tokens_per_expert=[259, 247, 252, 227, 256, 265, 244, 259, 277, 266, 248, 258, 271, 242, 255, 266, 239, 242, 239, 252, 269, 270, 233, 252, 241, 245, 256, 257, 259, 253, 263, 233, 259, 269, 267, 272, 253, 254, 244, 275, 257, 279, 234, 255, 239, 207, 232, 252, 230, 283, 205, 255, 260, 257, 275, 253, 278, 273, 286, 217, 283, 227, 265, 275, 226, 260, 265, 232, 274, 266, 259, 285, 244, 281, 270, 264, 228, 255, 239, 221, 273, 279, 283, 235, 253, 251, 253, 262, 263, 256, 235, 244, 228, 254, 261, 274, 254, 250, 289, 260, 248, 265, 279, 282, 256, 240, 261, 268, 257, 256, 255, 257, 247, 264, 254, 243, 236, 302, 244, 258, 278, 274, 241, 248, 259, 255, 292, 258] payload_rows_per_dst=[5494, 5444, 5422, 5552, 5438, 5444, 5598, 5488] dup_rows_per_dst=[2786, 2562, 2674, 2692, 2604, 2764, 2756, 2818] combine_rows_per_src=[5494, 5444, 5422, 5552, 5438, 5444, 5598, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228776 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156892584 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556392 weight_movement_bytes=37748736 total_nvlink_bytes=351305128 weight_to_token_ratio=0.120389 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229704 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157253960 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314278216 weight_movement_bytes=18874368 total_nvlink_bytes=333152584 weight_to_token_ratio=0.060056 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38240 dup_rows=21792 dup_rows_remote=19062 dedup_ratio=0.332520 router_tokens_per_expert=[230, 256, 228, 274, 258, 283, 261, 295, 253, 263, 257, 254, 265, 240, 283, 267, 251, 249, 252, 246, 246, 255, 256, 246, 251, 232, 259, 275, 222, 265, 262, 263, 250, 248, 256, 232, 264, 274, 250, 238, 266, 237, 250, 247, 264, 269, 269, 245, 239, 273, 276, 258, 276, 242, 255, 222, 250, 288, 238, 242, 285, 238, 262, 279, 264, 238, 262, 262, 266, 253, 245, 251, 250, 244, 264, 257, 251, 269, 248, 246, 266, 249, 252, 275, 250, 252, 283, 267, 252, 266, 266, 261, 247, 260, 255, 273, 247, 266, 268, 254, 250, 243, 237, 249, 281, 262, 270, 264, 240, 270, 280, 236, 268, 238, 249, 281, 247, 245, 254, 252, 257, 235, 227, 231, 278, 259, 237, 270] payload_rows_per_dst=[5570, 5384, 5418, 5518, 5448, 5498, 5504, 5404] dup_rows_per_dst=[2764, 2676, 2700, 2728, 2692, 2850, 2730, 2652] combine_rows_per_src=[5570, 5384, 5418, 5518, 5448, 5498, 5504, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229024 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157458080 ingress_bytes=213852160 link_bytes=213852160 token_bytes=314687136 weight_movement_bytes=56623104 total_nvlink_bytes=371310240 weight_to_token_ratio=0.179935 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049026175e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15070618272 prefetch_bytes=698351616 reduce_bytes=830472192 weight_movement_bytes=1528823808 egress_bytes=7540818592 ingress_bytes=9058623488 link_bytes=9058623488 total_nvlink_bytes=16599442080 weight_to_token_ratio=0.101444 prefetch_share_of_ingress=0.077092 dedup_ratio=0.332490 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=44 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081236 gemm_inner_pad_upper_bound=108331 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.808234857e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15053475792 prefetch_bytes=754974720 reduce_bytes=736100352 weight_movement_bytes=1491075072 egress_bytes=7532244944 ingress_bytes=9012305920 link_bytes=9012305920 total_nvlink_bytes=16544550864 weight_to_token_ratio=0.099052 prefetch_share_of_ingress=0.083772 dedup_ratio=0.332936 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=39 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.798989187e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229208 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860248 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491288 weight_movement_bytes=18874368 total_nvlink_bytes=332365656 weight_to_token_ratio=0.060207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15063578384 prefetch_bytes=792723456 reduce_bytes=754974720 weight_movement_bytes=1547698176 egress_bytes=7537293072 ingress_bytes=9073983488 link_bytes=9074212616 total_nvlink_bytes=16611276560 weight_to_token_ratio=0.102744 prefetch_share_of_ingress=0.087362 dedup_ratio=0.332397 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=40 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.811346677e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38468 dup_rows=21612 dup_rows_remote=19054 dedup_ratio=0.329773 router_tokens_per_expert=[280, 230, 264, 233, 238, 232, 267, 255, 260, 229, 264, 259, 228, 248, 259, 261, 262, 258, 244, 259, 256, 302, 252, 272, 266, 251, 240, 249, 246, 262, 247, 253, 251, 231, 254, 253, 274, 241, 237, 279, 225, 261, 233, 252, 244, 274, 243, 262, 248, 264, 256, 254, 248, 254, 276, 257, 277, 253, 252, 275, 267, 260, 254, 243, 261, 246, 277, 270, 272, 260, 262, 265, 250, 256, 256, 257, 261, 266, 265, 263, 256, 259, 264, 252, 249, 280, 262, 281, 260, 217, 273, 280, 254, 244, 260, 264, 259, 211, 259, 281, 256, 274, 251, 273, 255, 259, 238, 237, 221, 257, 265, 228, 261, 273, 242, 254, 250, 243, 275, 272, 263, 261, 268, 245, 267, 225, 264, 261] payload_rows_per_dst=[5456, 5476, 5628, 5550, 5536, 5292, 5432, 5554] dup_rows_per_dst=[2558, 2762, 2960, 2726, 2838, 2458, 2616, 2694] combine_rows_per_src=[5456, 5476, 5628, 5550, 5536, 5292, 5432, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38266 dup_rows=21802 dup_rows_remote=19128 dedup_ratio=0.332672 router_tokens_per_expert=[280, 266, 263, 234, 247, 247, 224, 254, 250, 277, 239, 274, 258, 270, 236, 249, 242, 233, 271, 241, 233, 281, 284, 256, 265, 278, 266, 231, 222, 282, 244, 231, 237, 257, 256, 256, 223, 273, 291, 262, 265, 242, 264, 268, 267, 252, 270, 259, 265, 240, 236, 267, 271, 264, 275, 273, 242, 246, 253, 263, 249, 283, 252, 242, 257, 273, 261, 255, 261, 231, 242, 250, 257, 260, 254, 240, 263, 269, 269, 262, 287, 261, 232, 265, 255, 257, 239, 236, 261, 272, 254, 264, 274, 264, 246, 257, 257, 263, 231, 253, 228, 246, 270, 234, 266, 257, 281, 281, 250, 225, 264, 265, 223, 240, 244, 234, 244, 253, 269, 248, 276, 252, 255, 272, 272, 232, 254, 310] payload_rows_per_dst=[5416, 5444, 5522, 5470, 5432, 5508, 5468, 5474] dup_rows_per_dst=[2720, 2676, 2762, 2772, 2776, 2740, 2674, 2682] combine_rows_per_src=[5416, 5444, 5522, 5470, 5432, 5508, 5468, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=230088 combine_payload_bytes=157564928 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157795016 ingress_bytes=214188032 link_bytes=214188032 token_bytes=315359944 weight_movement_bytes=56623104 total_nvlink_bytes=371983048 weight_to_token_ratio=0.179551 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.052244336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229576 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967112 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704648 weight_movement_bytes=18874368 total_nvlink_bytes=332579016 weight_to_token_ratio=0.060166 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15049157480 prefetch_bytes=773849088 reduce_bytes=962592768 weight_movement_bytes=1736441856 egress_bytes=7530081128 ingress_bytes=9255518208 link_bytes=9255518208 total_nvlink_bytes=16785599336 weight_to_token_ratio=0.115385 prefetch_share_of_ingress=0.083609 dedup_ratio=0.332858 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=51 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.847537947e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38072 dup_rows=21994 dup_rows_remote=19172 dedup_ratio=0.335602 router_tokens_per_expert=[239, 263, 289, 267, 268, 253, 268, 241, 252, 258, 253, 260, 276, 260, 245, 254, 248, 252, 276, 224, 267, 258, 258, 275, 254, 270, 255, 272, 250, 262, 231, 260, 250, 251, 249, 252, 273, 259, 274, 291, 266, 247, 238, 235, 274, 238, 258, 253, 245, 242, 263, 237, 276, 234, 278, 287, 264, 245, 261, 223, 256, 293, 230, 256, 256, 259, 246, 271, 272, 237, 252, 241, 251, 258, 254, 259, 243, 248, 264, 263, 280, 265, 253, 269, 261, 230, 251, 262, 251, 238, 253, 238, 252, 282, 243, 278, 243, 252, 268, 239, 260, 280, 224, 247, 292, 234, 259, 282, 259, 274, 257, 277, 295, 254, 240, 225, 233, 222, 263, 250, 256, 233, 249, 247, 226, 259, 267, 266] payload_rows_per_dst=[5470, 5466, 5452, 5452, 5408, 5450, 5444, 5400] dup_rows_per_dst=[2822, 2826, 2696, 2728, 2740, 2762, 2850, 2570] combine_rows_per_src=[5470, 5466, 5452, 5452, 5408, 5450, 5444, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=228976 combine_payload_bytes=155942912 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156171888 ingress_bytes=174817280 link_bytes=174817280 token_bytes=312114800 weight_movement_bytes=18874368 total_nvlink_bytes=330989168 weight_to_token_ratio=0.060473 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675013161e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38180 dup_rows=21838 dup_rows_remote=18932 dedup_ratio=0.333221 router_tokens_per_expert=[270, 260, 255, 260, 249, 262, 234, 265, 236, 249, 259, 252, 231, 253, 245, 253, 257, 261, 237, 238, 243, 277, 237, 261, 252, 255, 283, 264, 262, 264, 289, 271, 244, 276, 259, 264, 270, 271, 251, 265, 242, 253, 271, 236, 245, 276, 257, 275, 250, 272, 243, 283, 247, 253, 265, 237, 262, 288, 251, 239, 249, 237, 249, 310, 245, 276, 266, 244, 285, 259, 247, 243, 263, 234, 241, 247, 265, 239, 245, 303, 248, 268, 224, 250, 279, 276, 257, 224, 222, 253, 249, 248, 278, 263, 248, 236, 256, 229, 231, 242, 240, 237, 250, 260, 284, 236, 274, 294, 242, 256, 236, 270, 247, 239, 250, 261, 243, 253, 253, 276, 267, 258, 291, 259, 250, 267, 263, 255] payload_rows_per_dst=[5518, 5510, 5382, 5546, 5598, 5354, 5432, 5358] dup_rows_per_dst=[2906, 2812, 2570, 2724, 2918, 2692, 2642, 2574] combine_rows_per_src=[5518, 5510, 5382, 5546, 5598, 5354, 5432, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228448 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156613728 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999008 weight_movement_bytes=18874368 total_nvlink_bytes=331873376 weight_to_token_ratio=0.060302 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15037163344 prefetch_bytes=811597824 reduce_bytes=830472192 weight_movement_bytes=1642070016 egress_bytes=7524075344 ingress_bytes=9155158016 link_bytes=9155158016 total_nvlink_bytes=16679233360 weight_to_token_ratio=0.109201 prefetch_share_of_ingress=0.088649 dedup_ratio=0.332676 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=44 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082248 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.827504573e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none + [2026-08-29 23:39:06] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 771.4 | throughput per GPU (TFLOP/s/GPU): 115.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.006471E+01 | loss scale: 1.0 | grad norm: 170.101 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38222 dup_rows=21874 dup_rows_remote=19188 dedup_ratio=0.333771 router_tokens_per_expert=[251, 282, 238, 229, 256, 259, 285, 225, 287, 257, 236, 239, 253, 239, 257, 263, 264, 241, 251, 273, 252, 277, 259, 237, 255, 269, 271, 245, 248, 242, 278, 265, 244, 231, 247, 257, 266, 234, 249, 263, 277, 229, 249, 271, 233, 271, 266, 285, 252, 264, 253, 239, 232, 241, 237, 246, 263, 257, 248, 263, 281, 271, 275, 241, 246, 265, 249, 273, 276, 255, 250, 255, 288, 255, 266, 291, 265, 267, 260, 288, 245, 244, 261, 260, 264, 244, 276, 276, 257, 257, 241, 252, 266, 261, 255, 250, 256, 264, 252, 274, 215, 245, 242, 232, 275, 262, 274, 230, 239, 268, 259, 259, 244, 248, 260, 282, 275, 240, 244, 273, 244, 245, 242, 230, 249, 266, 261, 243] payload_rows_per_dst=[5406, 5488, 5434, 5440, 5544, 5438, 5458, 5454] dup_rows_per_dst=[2706, 2766, 2710, 2686, 2954, 2780, 2634, 2638] combine_rows_per_src=[5406, 5488, 5434, 5440, 5544, 5438, 5458, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38144 dup_rows=21996 dup_rows_remote=19204 dedup_ratio=0.335632 router_tokens_per_expert=[268, 248, 243, 242, 252, 246, 239, 258, 255, 242, 227, 257, 266, 271, 220, 269, 237, 233, 257, 246, 237, 290, 255, 256, 251, 281, 240, 251, 260, 238, 263, 261, 253, 259, 255, 240, 267, 260, 253, 241, 268, 267, 275, 257, 268, 239, 266, 284, 267, 240, 268, 244, 243, 259, 249, 230, 264, 263, 214, 245, 263, 273, 240, 256, 247, 283, 258, 267, 240, 243, 245, 242, 262, 265, 249, 239, 283, 286, 240, 245, 250, 271, 235, 272, 255, 253, 256, 240, 269, 257, 242, 257, 272, 253, 271, 241, 264, 249, 248, 269, 276, 271, 259, 222, 277, 259, 259, 248, 302, 280, 270, 237, 241, 250, 245, 244, 263, 263, 257, 246, 245, 274, 270, 270, 282, 285, 279, 247] payload_rows_per_dst=[5382, 5434, 5530, 5368, 5396, 5474, 5494, 5462] dup_rows_per_dst=[2624, 2678, 2774, 2668, 2792, 2714, 2886, 2860] combine_rows_per_src=[5382, 5434, 5530, 5368, 5396, 5474, 5494, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38196 dup_rows=21830 dup_rows_remote=19088 dedup_ratio=0.333099 router_tokens_per_expert=[229, 239, 273, 247, 277, 258, 272, 268, 254, 252, 245, 254, 251, 268, 241, 259, 278, 268, 236, 235, 249, 247, 265, 268, 259, 282, 261, 244, 260, 257, 247, 270, 256, 223, 257, 234, 264, 260, 235, 261, 267, 261, 250, 277, 250, 250, 256, 282, 271, 273, 257, 259, 263, 269, 243, 252, 266, 250, 273, 244, 291, 249, 271, 255, 273, 238, 229, 266, 283, 262, 280, 267, 251, 259, 252, 245, 283, 259, 242, 233, 241, 276, 255, 293, 248, 232, 255, 238, 248, 253, 285, 266, 264, 241, 231, 286, 279, 259, 262, 244, 239, 249, 232, 251, 257, 240, 229, 259, 291, 269, 251, 265, 227, 231, 260, 237, 266, 265, 244, 275, 249, 233, 252, 249, 277, 238, 242, 231] payload_rows_per_dst=[5396, 5510, 5492, 5544, 5524, 5430, 5462, 5348] dup_rows_per_dst=[2778, 2742, 2674, 2828, 2720, 2794, 2690, 2604] combine_rows_per_src=[5396, 5510, 5492, 5544, 5524, 5430, 5462, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38042 dup_rows=21892 dup_rows_remote=18806 dedup_ratio=0.334045 router_tokens_per_expert=[264, 262, 271, 275, 263, 255, 235, 216, 276, 266, 239, 258, 259, 255, 228, 243, 241, 259, 226, 271, 279, 262, 252, 244, 256, 239, 276, 225, 254, 277, 276, 255, 246, 290, 277, 241, 264, 259, 265, 248, 267, 234, 245, 265, 238, 268, 255, 245, 248, 248, 270, 257, 255, 272, 237, 235, 251, 266, 239, 248, 259, 253, 266, 252, 255, 260, 268, 260, 280, 226, 276, 258, 228, 242, 217, 246, 281, 269, 233, 238, 274, 254, 272, 247, 238, 250, 259, 264, 279, 261, 267, 267, 246, 245, 263, 269, 243, 247, 257, 273, 266, 268, 255, 241, 260, 279, 259, 267, 269, 261, 233, 266, 280, 245, 242, 283, 260, 256, 256, 235, 235, 289, 232, 253, 254, 240, 285, 267] payload_rows_per_dst=[5304, 5508, 5414, 5546, 5416, 5534, 5320, 5602] dup_rows_per_dst=[2666, 2676, 2774, 2678, 2606, 2824, 2582, 3086] combine_rows_per_src=[5304, 5508, 5414, 5546, 5416, 5534, 5320, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38310 dup_rows=21832 dup_rows_remote=19148 dedup_ratio=0.333130 router_tokens_per_expert=[259, 266, 256, 270, 258, 256, 284, 244, 267, 249, 264, 274, 240, 257, 228, 300, 245, 252, 273, 250, 240, 258, 244, 247, 252, 240, 264, 245, 260, 241, 237, 294, 262, 269, 258, 272, 241, 266, 232, 249, 243, 271, 234, 264, 251, 245, 240, 242, 273, 239, 246, 253, 273, 268, 257, 245, 284, 244, 242, 257, 249, 262, 280, 266, 266, 230, 266, 231, 248, 247, 269, 245, 259, 259, 263, 250, 233, 260, 245, 270, 238, 276, 231, 267, 287, 251, 268, 286, 248, 250, 239, 261, 265, 244, 260, 240, 265, 292, 245, 264, 243, 258, 262, 262, 272, 263, 256, 243, 233, 252, 254, 242, 261, 250, 273, 272, 276, 233, 270, 276, 257, 260, 268, 247, 266, 272, 238, 200] payload_rows_per_dst=[5518, 5424, 5394, 5508, 5438, 5478, 5470, 5474] dup_rows_per_dst=[2826, 2660, 2684, 2768, 2644, 2744, 2742, 2764] combine_rows_per_src=[5518, 5424, 5394, 5508, 5438, 5478, 5470, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38280 dup_rows=21752 dup_rows_remote=18910 dedup_ratio=0.331909 router_tokens_per_expert=[237, 264, 277, 248, 276, 249, 258, 254, 275, 259, 242, 267, 249, 261, 254, 260, 265, 260, 266, 238, 272, 278, 251, 229, 233, 258, 278, 238, 268, 229, 239, 222, 271, 236, 255, 263, 233, 273, 276, 239, 248, 264, 244, 243, 269, 246, 278, 274, 231, 246, 281, 253, 244, 255, 245, 278, 261, 272, 223, 268, 267, 250, 260, 237, 246, 268, 252, 261, 246, 231, 263, 270, 269, 277, 258, 229, 254, 276, 235, 263, 255, 278, 269, 246, 267, 262, 255, 262, 278, 253, 255, 250, 278, 262, 257, 246, 286, 276, 258, 267, 269, 240, 292, 275, 249, 247, 271, 242, 253, 262, 258, 227, 247, 264, 264, 265, 254, 215, 254, 258, 244, 248, 240, 241, 267, 243, 249, 235] payload_rows_per_dst=[5470, 5432, 5530, 5474, 5458, 5504, 5484, 5432] dup_rows_per_dst=[2790, 2616, 2694, 2668, 2738, 2842, 2860, 2544] combine_rows_per_src=[5470, 5432, 5530, 5474, 5458, 5504, 5484, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229640 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786952 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313344264 weight_movement_bytes=18874368 total_nvlink_bytes=332218632 weight_to_token_ratio=0.060235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229392 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156467216 ingress_bytes=193986560 link_bytes=193986560 token_bytes=312705040 weight_movement_bytes=37748736 total_nvlink_bytes=350453776 weight_to_token_ratio=0.120717 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858683771e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229136 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156679952 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313130768 weight_movement_bytes=18874368 total_nvlink_bytes=332005136 weight_to_token_ratio=0.060276 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229832 combine_payload_bytes=156917760 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157147592 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314065352 weight_movement_bytes=37748736 total_nvlink_bytes=351814088 weight_to_token_ratio=0.120194 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=227392 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156047424 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311867456 weight_movement_bytes=18874368 total_nvlink_bytes=330741824 weight_to_token_ratio=0.060520 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=228760 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157023640 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313818520 weight_movement_bytes=37748736 total_nvlink_bytes=351567256 weight_to_token_ratio=0.120288 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38216 dup_rows=21888 dup_rows_remote=19292 dedup_ratio=0.333984 router_tokens_per_expert=[270, 292, 269, 273, 268, 264, 287, 254, 226, 262, 268, 280, 258, 224, 261, 269, 271, 232, 238, 277, 235, 245, 238, 229, 272, 232, 270, 236, 214, 220, 255, 265, 245, 243, 265, 250, 255, 255, 250, 245, 266, 274, 243, 274, 267, 289, 247, 228, 266, 261, 265, 248, 252, 248, 255, 272, 246, 247, 228, 268, 232, 239, 239, 248, 269, 272, 263, 269, 234, 278, 257, 260, 250, 258, 256, 265, 284, 249, 253, 255, 261, 251, 240, 273, 264, 241, 257, 253, 252, 285, 228, 262, 258, 285, 276, 268, 257, 217, 268, 267, 236, 233, 241, 260, 255, 256, 278, 257, 272, 262, 242, 231, 288, 227, 257, 256, 245, 263, 256, 270, 248, 258, 235, 279, 232, 281, 273, 278] payload_rows_per_dst=[5482, 5352, 5450, 5432, 5514, 5512, 5424, 5482] dup_rows_per_dst=[2968, 2506, 2742, 2596, 2830, 2796, 2640, 2810] combine_rows_per_src=[5482, 5352, 5450, 5432, 5514, 5512, 5424, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38308 dup_rows=21774 dup_rows_remote=19040 dedup_ratio=0.332245 router_tokens_per_expert=[253, 242, 272, 273, 256, 260, 265, 250, 236, 260, 247, 261, 260, 268, 248, 254, 253, 259, 253, 271, 255, 284, 259, 253, 275, 227, 279, 264, 241, 278, 261, 250, 251, 284, 253, 258, 279, 248, 240, 254, 248, 257, 270, 276, 263, 250, 254, 251, 259, 285, 262, 253, 240, 247, 238, 243, 270, 287, 239, 278, 240, 261, 249, 250, 249, 260, 244, 279, 253, 270, 236, 220, 244, 230, 267, 282, 260, 271, 284, 245, 253, 242, 238, 236, 243, 254, 227, 251, 263, 239, 265, 264, 266, 272, 259, 259, 270, 270, 258, 245, 223, 268, 245, 264, 301, 250, 259, 248, 253, 230, 241, 235, 252, 247, 260, 241, 242, 237, 254, 245, 264, 259, 245, 293, 293, 242, 238, 267] payload_rows_per_dst=[5488, 5578, 5536, 5482, 5454, 5322, 5454, 5448] dup_rows_per_dst=[2722, 2746, 2736, 2720, 2734, 2740, 2666, 2710] combine_rows_per_src=[5488, 5578, 5536, 5482, 5454, 5322, 5454, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38308 dup_rows=21734 dup_rows_remote=18862 dedup_ratio=0.331635 router_tokens_per_expert=[263, 238, 249, 247, 258, 268, 253, 253, 258, 240, 238, 257, 243, 257, 248, 276, 281, 268, 266, 283, 260, 286, 265, 234, 247, 258, 254, 246, 246, 259, 264, 266, 281, 232, 244, 252, 248, 254, 220, 254, 251, 266, 260, 257, 256, 253, 247, 275, 259, 282, 231, 263, 252, 261, 270, 272, 232, 242, 253, 267, 260, 252, 222, 234, 279, 243, 238, 271, 280, 251, 249, 250, 277, 264, 259, 261, 248, 258, 247, 291, 295, 230, 272, 265, 272, 269, 252, 258, 280, 258, 243, 262, 239, 259, 285, 219, 242, 243, 240, 234, 237, 258, 245, 282, 257, 248, 271, 256, 253, 257, 271, 247, 255, 254, 270, 245, 271, 279, 234, 286, 226, 270, 256, 259, 254, 236, 232, 245] payload_rows_per_dst=[5422, 5494, 5452, 5472, 5548, 5514, 5430, 5470] dup_rows_per_dst=[2670, 2872, 2648, 2632, 2784, 2802, 2652, 2674] combine_rows_per_src=[5422, 5494, 5452, 5472, 5548, 5514, 5430, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38396 dup_rows=21808 dup_rows_remote=19072 dedup_ratio=0.332764 router_tokens_per_expert=[257, 250, 255, 254, 247, 252, 260, 235, 255, 267, 237, 208, 255, 245, 262, 275, 243, 282, 286, 246, 258, 248, 276, 273, 250, 255, 223, 238, 279, 276, 268, 244, 256, 244, 254, 242, 226, 254, 261, 243, 250, 253, 263, 276, 254, 230, 283, 245, 245, 244, 253, 263, 269, 252, 279, 267, 251, 275, 226, 258, 284, 247, 274, 263, 235, 237, 276, 243, 243, 241, 266, 271, 274, 269, 255, 264, 280, 244, 265, 232, 258, 225, 259, 269, 239, 268, 228, 254, 260, 258, 279, 239, 229, 253, 250, 245, 249, 255, 263, 264, 274, 253, 256, 258, 285, 242, 257, 296, 266, 261, 254, 260, 258, 276, 256, 267, 254, 274, 259, 271, 265, 249, 223, 249, 248, 283, 254, 238] payload_rows_per_dst=[5468, 5484, 5332, 5548, 5486, 5338, 5550, 5522] dup_rows_per_dst=[2560, 2806, 2736, 2752, 2704, 2688, 2836, 2726] combine_rows_per_src=[5468, 5484, 5332, 5548, 5486, 5338, 5550, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38332 dup_rows=21734 dup_rows_remote=18982 dedup_ratio=0.331635 router_tokens_per_expert=[247, 239, 274, 253, 265, 225, 258, 270, 253, 272, 240, 282, 254, 258, 254, 246, 258, 254, 264, 229, 262, 279, 275, 252, 257, 270, 242, 277, 248, 277, 253, 244, 273, 268, 235, 250, 280, 253, 259, 277, 253, 254, 278, 237, 246, 275, 261, 241, 268, 260, 253, 264, 262, 274, 262, 291, 257, 277, 264, 251, 252, 264, 253, 261, 249, 247, 268, 247, 242, 245, 259, 284, 250, 221, 273, 263, 246, 246, 239, 243, 289, 227, 256, 257, 266, 229, 224, 249, 227, 270, 270, 245, 231, 260, 282, 270, 234, 267, 252, 273, 233, 254, 250, 248, 251, 225, 245, 245, 253, 260, 261, 261, 252, 241, 256, 257, 261, 261, 244, 270, 269, 264, 249, 249, 228, 271, 266, 260] payload_rows_per_dst=[5348, 5474, 5532, 5596, 5492, 5456, 5434, 5470] dup_rows_per_dst=[2612, 2648, 2748, 2830, 2600, 2820, 2724, 2752] combine_rows_per_src=[5348, 5474, 5532, 5596, 5492, 5456, 5434, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=230032 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762768 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295504 weight_movement_bytes=18874368 total_nvlink_bytes=332169872 weight_to_token_ratio=0.060245 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38256 dup_rows=21742 dup_rows_remote=18946 dedup_ratio=0.331757 router_tokens_per_expert=[246, 251, 276, 289, 258, 260, 270, 241, 280, 242, 253, 258, 245, 222, 250, 258, 266, 264, 251, 248, 276, 273, 266, 247, 255, 254, 259, 248, 268, 240, 259, 264, 234, 221, 273, 267, 262, 233, 262, 289, 242, 268, 268, 232, 265, 222, 257, 244, 257, 263, 252, 273, 258, 269, 276, 277, 260, 244, 260, 230, 235, 268, 214, 247, 275, 227, 245, 257, 253, 238, 255, 225, 237, 263, 251, 258, 240, 274, 250, 244, 262, 275, 262, 277, 221, 258, 271, 260, 262, 243, 252, 281, 251, 257, 249, 286, 246, 262, 225, 264, 259, 273, 274, 254, 257, 308, 267, 239, 250, 237, 258, 271, 249, 254, 217, 265, 266, 243, 250, 238, 272, 271, 255, 271, 257, 272, 272, 254] payload_rows_per_dst=[5420, 5536, 5398, 5460, 5416, 5538, 5522, 5504] dup_rows_per_dst=[2778, 2740, 2680, 2706, 2568, 2796, 2766, 2708] combine_rows_per_src=[5420, 5536, 5398, 5460, 5416, 5538, 5522, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=228680 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157138248 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314047816 weight_movement_bytes=37748736 total_nvlink_bytes=351796552 weight_to_token_ratio=0.120201 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229392 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157138960 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048528 weight_movement_bytes=37748736 total_nvlink_bytes=351797264 weight_to_token_ratio=0.120200 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229872 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157499888 ingress_bytes=176144384 link_bytes=176144384 token_bytes=314769904 weight_movement_bytes=18874368 total_nvlink_bytes=333644272 weight_to_token_ratio=0.059962 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687728819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43964 payload_rows_remote=38454 dup_rows=21572 dup_rows_remote=18766 dedup_ratio=0.329163 router_tokens_per_expert=[257, 300, 271, 262, 272, 256, 259, 253, 255, 241, 271, 244, 246, 251, 265, 251, 248, 264, 250, 248, 239, 246, 265, 263, 257, 239, 248, 270, 256, 253, 262, 275, 274, 265, 282, 251, 248, 261, 259, 272, 265, 253, 233, 258, 260, 255, 259, 272, 269, 264, 259, 271, 227, 253, 263, 267, 259, 242, 250, 241, 256, 257, 287, 228, 254, 222, 226, 267, 234, 274, 245, 251, 268, 249, 236, 237, 273, 245, 259, 265, 247, 264, 263, 255, 244, 263, 254, 286, 263, 267, 262, 246, 221, 237, 247, 265, 280, 250, 248, 249, 265, 257, 267, 241, 271, 231, 281, 255, 294, 247, 239, 230, 257, 255, 278, 252, 252, 251, 252, 258, 247, 237, 263, 262, 251, 260, 236, 266] payload_rows_per_dst=[5510, 5622, 5586, 5458, 5448, 5328, 5508, 5504] dup_rows_per_dst=[2806, 2878, 2740, 2728, 2562, 2506, 2702, 2650] combine_rows_per_src=[5510, 5622, 5586, 5458, 5448, 5328, 5508, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229256 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157237128 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314245000 weight_movement_bytes=37748736 total_nvlink_bytes=351993736 weight_to_token_ratio=0.120125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38328 dup_rows=21780 dup_rows_remote=18964 dedup_ratio=0.332336 router_tokens_per_expert=[268, 272, 272, 271, 268, 254, 252, 258, 245, 239, 248, 280, 274, 252, 283, 241, 234, 260, 249, 248, 263, 240, 244, 274, 256, 245, 266, 242, 252, 278, 236, 264, 262, 283, 277, 258, 274, 262, 240, 234, 212, 237, 251, 247, 229, 253, 259, 275, 226, 290, 244, 245, 270, 255, 267, 257, 288, 281, 243, 244, 234, 245, 247, 251, 255, 259, 261, 300, 273, 254, 265, 260, 266, 239, 271, 244, 250, 247, 269, 235, 259, 242, 273, 250, 230, 256, 270, 243, 273, 265, 263, 253, 232, 256, 246, 262, 267, 270, 223, 266, 279, 255, 243, 262, 269, 261, 244, 268, 223, 277, 259, 256, 269, 264, 246, 231, 256, 245, 234, 259, 247, 259, 264, 264, 271, 266, 243, 239] payload_rows_per_dst=[5538, 5454, 5358, 5414, 5590, 5446, 5428, 5528] dup_rows_per_dst=[2816, 2648, 2748, 2760, 2706, 2700, 2816, 2586] combine_rows_per_src=[5538, 5454, 5358, 5414, 5590, 5446, 5428, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228808 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925384 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313621960 weight_movement_bytes=18874368 total_nvlink_bytes=332496328 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=228880 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157736464 ingress_bytes=214130688 link_bytes=214130688 token_bytes=315244048 weight_movement_bytes=56623104 total_nvlink_bytes=371867152 weight_to_token_ratio=0.179617 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051694894e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38066 dup_rows=21982 dup_rows_remote=19144 dedup_ratio=0.335419 router_tokens_per_expert=[250, 233, 248, 247, 255, 262, 253, 268, 292, 301, 255, 274, 253, 254, 250, 267, 276, 269, 277, 225, 259, 288, 236, 238, 258, 259, 278, 235, 255, 277, 269, 259, 263, 231, 262, 260, 244, 252, 253, 241, 251, 283, 264, 249, 234, 217, 250, 256, 264, 243, 251, 260, 255, 261, 258, 259, 269, 278, 263, 275, 261, 258, 266, 242, 250, 265, 247, 273, 271, 253, 257, 233, 238, 243, 263, 264, 255, 252, 239, 239, 227, 256, 266, 262, 234, 235, 238, 255, 251, 259, 254, 248, 239, 260, 255, 275, 254, 233, 259, 242, 264, 278, 271, 258, 263, 253, 243, 256, 234, 249, 244, 270, 262, 251, 261, 269, 283, 251, 281, 259, 261, 259, 228, 258, 229, 261, 268, 267] payload_rows_per_dst=[5496, 5504, 5364, 5488, 5366, 5356, 5436, 5544] dup_rows_per_dst=[2828, 2812, 2656, 2838, 2718, 2672, 2706, 2752] combine_rows_per_src=[5496, 5504, 5364, 5488, 5366, 5356, 5436, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38386 dup_rows=21782 dup_rows_remote=19034 dedup_ratio=0.332367 router_tokens_per_expert=[241, 258, 245, 270, 267, 259, 258, 267, 260, 252, 275, 252, 257, 253, 241, 236, 239, 254, 252, 239, 242, 267, 276, 267, 277, 255, 248, 282, 236, 246, 235, 243, 261, 255, 257, 245, 263, 259, 260, 270, 245, 250, 251, 264, 243, 266, 277, 251, 226, 277, 239, 261, 254, 254, 234, 229, 270, 262, 273, 259, 262, 246, 277, 255, 271, 283, 244, 251, 258, 252, 242, 256, 249, 241, 251, 246, 262, 250, 249, 272, 245, 254, 286, 263, 279, 242, 252, 268, 241, 274, 274, 249, 261, 241, 265, 273, 262, 247, 268, 252, 278, 250, 234, 259, 259, 265, 256, 250, 267, 277, 237, 243, 259, 278, 244, 268, 252, 239, 252, 252, 248, 263, 252, 245, 251, 227, 280, 266] payload_rows_per_dst=[5456, 5368, 5566, 5436, 5448, 5510, 5490, 5480] dup_rows_per_dst=[2726, 2748, 2668, 2720, 2706, 2824, 2718, 2672] combine_rows_per_src=[5456, 5368, 5566, 5436, 5448, 5510, 5490, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38270 dup_rows=21762 dup_rows_remote=19004 dedup_ratio=0.332062 router_tokens_per_expert=[254, 266, 254, 242, 268, 268, 260, 242, 251, 251, 268, 259, 243, 251, 276, 254, 261, 259, 265, 274, 278, 223, 233, 262, 253, 261, 244, 272, 227, 277, 231, 259, 229, 253, 230, 205, 257, 223, 279, 284, 261, 250, 258, 258, 237, 277, 258, 273, 253, 263, 253, 290, 266, 252, 241, 249, 292, 255, 276, 255, 244, 274, 259, 240, 259, 290, 244, 244, 257, 281, 238, 277, 260, 254, 241, 263, 281, 271, 240, 231, 261, 274, 275, 245, 258, 251, 257, 222, 258, 280, 254, 249, 285, 254, 270, 230, 244, 248, 249, 244, 247, 243, 260, 243, 249, 237, 274, 267, 239, 269, 265, 284, 282, 263, 238, 267, 255, 275, 239, 285, 245, 249, 246, 232, 274, 258, 220, 244] payload_rows_per_dst=[5448, 5472, 5422, 5526, 5504, 5422, 5442, 5538] dup_rows_per_dst=[2766, 2686, 2642, 2798, 2758, 2824, 2682, 2606] combine_rows_per_src=[5448, 5472, 5422, 5526, 5504, 5422, 5442, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38354 dup_rows=21714 dup_rows_remote=18936 dedup_ratio=0.331329 router_tokens_per_expert=[261, 249, 255, 236, 260, 254, 241, 236, 286, 263, 233, 278, 246, 249, 279, 304, 253, 231, 281, 254, 264, 257, 271, 241, 243, 248, 248, 269, 283, 275, 272, 251, 229, 270, 262, 269, 267, 257, 251, 246, 253, 254, 275, 241, 301, 231, 243, 274, 270, 241, 251, 236, 271, 259, 266, 268, 249, 256, 239, 261, 256, 233, 270, 260, 254, 266, 258, 236, 267, 263, 260, 231, 260, 240, 283, 273, 285, 267, 218, 244, 262, 275, 217, 243, 254, 261, 251, 280, 270, 272, 256, 266, 245, 255, 266, 221, 252, 287, 251, 269, 272, 242, 248, 211, 240, 238, 219, 268, 247, 259, 254, 243, 277, 240, 244, 259, 255, 255, 257, 269, 255, 273, 247, 253, 212, 254, 261, 278] payload_rows_per_dst=[5516, 5494, 5468, 5448, 5526, 5458, 5402, 5510] dup_rows_per_dst=[2744, 2788, 2778, 2724, 2684, 2730, 2598, 2668] combine_rows_per_src=[5516, 5494, 5468, 5448, 5526, 5458, 5402, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38418 dup_rows=21682 dup_rows_remote=18920 dedup_ratio=0.330841 router_tokens_per_expert=[275, 275, 236, 242, 285, 270, 240, 248, 290, 233, 221, 245, 250, 247, 259, 264, 264, 266, 247, 253, 259, 236, 287, 272, 246, 248, 264, 247, 265, 261, 209, 252, 256, 225, 270, 275, 233, 253, 259, 262, 247, 247, 240, 270, 281, 273, 245, 245, 275, 257, 228, 239, 268, 236, 308, 257, 267, 253, 228, 257, 242, 258, 280, 275, 273, 258, 272, 253, 280, 260, 259, 268, 270, 257, 263, 247, 234, 247, 246, 239, 241, 252, 268, 269, 261, 252, 238, 245, 251, 271, 245, 284, 232, 261, 271, 263, 263, 240, 245, 268, 243, 246, 246, 247, 287, 260, 253, 242, 240, 259, 265, 279, 274, 244, 247, 263, 269, 247, 234, 260, 279, 245, 241, 249, 278, 251, 270, 239] payload_rows_per_dst=[5262, 5446, 5490, 5494, 5464, 5618, 5644, 5436] dup_rows_per_dst=[2446, 2706, 2672, 2700, 2776, 2646, 2974, 2762] combine_rows_per_src=[5262, 5446, 5490, 5494, 5464, 5618, 5644, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229168 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157220656 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314212144 weight_movement_bytes=37748736 total_nvlink_bytes=351960880 weight_to_token_ratio=0.120138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228840 combine_payload_bytes=155918336 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156147176 ingress_bytes=212541440 link_bytes=212541440 token_bytes=312065512 weight_movement_bytes=56623104 total_nvlink_bytes=368688616 weight_to_token_ratio=0.181446 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036467501e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229680 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458736 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687792 weight_movement_bytes=18874368 total_nvlink_bytes=333562160 weight_to_token_ratio=0.059978 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229096 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983016 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313736936 weight_movement_bytes=37748736 total_nvlink_bytes=351485672 weight_to_token_ratio=0.120320 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229160 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327144 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425128 weight_movement_bytes=18874368 total_nvlink_bytes=333299496 weight_to_token_ratio=0.060028 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229352 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157589480 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314949608 weight_movement_bytes=37748736 total_nvlink_bytes=352698344 weight_to_token_ratio=0.119856 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38306 dup_rows=21784 dup_rows_remote=19036 dedup_ratio=0.332397 router_tokens_per_expert=[237, 264, 289, 278, 256, 236, 235, 234, 286, 286, 246, 264, 228, 264, 250, 269, 263, 252, 281, 217, 256, 257, 264, 257, 255, 246, 241, 275, 254, 257, 259, 273, 232, 264, 267, 262, 254, 228, 261, 259, 262, 260, 282, 242, 234, 247, 253, 257, 260, 258, 259, 269, 281, 261, 233, 241, 277, 244, 287, 235, 259, 240, 268, 266, 258, 249, 247, 271, 272, 250, 267, 240, 262, 243, 254, 254, 244, 252, 259, 275, 239, 243, 265, 253, 261, 226, 274, 263, 244, 255, 264, 245, 284, 270, 257, 254, 254, 239, 254, 253, 248, 269, 239, 254, 252, 276, 229, 256, 272, 262, 274, 264, 246, 251, 232, 258, 263, 242, 271, 255, 282, 273, 257, 239, 231, 234, 275, 239] payload_rows_per_dst=[5534, 5454, 5428, 5436, 5510, 5446, 5472, 5472] dup_rows_per_dst=[2710, 2760, 2700, 2840, 2684, 2748, 2718, 2624] combine_rows_per_src=[5534, 5454, 5428, 5436, 5510, 5446, 5472, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229368 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130744 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314032120 weight_movement_bytes=37748736 total_nvlink_bytes=351780856 weight_to_token_ratio=0.120207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38174 dup_rows=21790 dup_rows_remote=18986 dedup_ratio=0.332489 router_tokens_per_expert=[248, 237, 254, 258, 270, 281, 270, 273, 253, 266, 269, 263, 270, 263, 240, 273, 249, 245, 253, 268, 228, 266, 268, 261, 254, 234, 256, 254, 253, 253, 241, 232, 276, 238, 264, 272, 226, 238, 274, 248, 274, 265, 259, 261, 271, 274, 276, 255, 224, 260, 241, 256, 284, 228, 257, 256, 238, 235, 222, 291, 255, 276, 249, 246, 280, 259, 250, 265, 270, 251, 260, 270, 258, 270, 250, 266, 247, 242, 259, 242, 256, 263, 278, 253, 272, 281, 280, 254, 247, 266, 232, 240, 245, 269, 244, 246, 217, 260, 262, 266, 237, 264, 248, 255, 270, 239, 258, 241, 237, 245, 251, 249, 234, 269, 263, 260, 268, 257, 288, 247, 254, 229, 266, 275, 261, 253, 251, 237] payload_rows_per_dst=[5572, 5418, 5588, 5406, 5550, 5458, 5348, 5406] dup_rows_per_dst=[2804, 2612, 2854, 2738, 2628, 2794, 2650, 2710] combine_rows_per_src=[5572, 5418, 5588, 5406, 5550, 5458, 5348, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38316 dup_rows=21698 dup_rows_remote=19032 dedup_ratio=0.331085 router_tokens_per_expert=[272, 252, 277, 258, 259, 246, 267, 252, 254, 218, 248, 272, 242, 254, 245, 238, 235, 259, 251, 241, 263, 258, 263, 254, 238, 256, 231, 251, 257, 283, 279, 273, 271, 260, 235, 270, 269, 250, 264, 252, 238, 281, 279, 260, 249, 257, 270, 267, 266, 239, 280, 257, 255, 261, 260, 248, 283, 237, 256, 256, 239, 253, 233, 271, 277, 213, 241, 247, 267, 234, 234, 253, 252, 263, 269, 268, 253, 269, 250, 220, 259, 280, 260, 265, 300, 281, 260, 250, 231, 254, 264, 244, 267, 269, 264, 257, 251, 239, 244, 250, 276, 261, 257, 261, 237, 250, 284, 267, 247, 219, 263, 265, 261, 257, 245, 232, 276, 253, 251, 269, 258, 265, 283, 236, 238, 268, 245, 233] payload_rows_per_dst=[5480, 5536, 5498, 5522, 5426, 5552, 5408, 5416] dup_rows_per_dst=[2628, 2648, 2846, 2666, 2594, 2858, 2734, 2724] combine_rows_per_src=[5480, 5536, 5498, 5522, 5426, 5552, 5408, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38146 dup_rows=21740 dup_rows_remote=18950 dedup_ratio=0.331726 router_tokens_per_expert=[269, 266, 255, 265, 255, 225, 246, 261, 274, 228, 275, 250, 248, 283, 270, 241, 282, 275, 248, 270, 265, 257, 263, 255, 225, 250, 271, 248, 252, 227, 287, 260, 256, 267, 274, 256, 247, 258, 251, 264, 246, 234, 232, 273, 247, 264, 243, 240, 271, 253, 267, 234, 283, 247, 251, 244, 244, 265, 243, 281, 251, 247, 259, 243, 264, 242, 275, 255, 248, 265, 248, 280, 251, 271, 256, 267, 248, 257, 224, 263, 244, 273, 248, 240, 231, 261, 246, 244, 248, 240, 247, 273, 236, 237, 268, 245, 262, 284, 274, 253, 276, 238, 255, 268, 250, 236, 259, 283, 294, 295, 261, 232, 262, 268, 245, 249, 284, 235, 253, 240, 250, 256, 252, 245, 239, 290, 246, 258] payload_rows_per_dst=[5444, 5472, 5424, 5486, 5484, 5424, 5650, 5412] dup_rows_per_dst=[2778, 2798, 2680, 2680, 2744, 2538, 2790, 2732] combine_rows_per_src=[5444, 5472, 5424, 5486, 5484, 5424, 5650, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43954 payload_rows_remote=38442 dup_rows=21582 dup_rows_remote=18900 dedup_ratio=0.329315 router_tokens_per_expert=[288, 260, 272, 266, 271, 236, 287, 264, 259, 271, 252, 259, 259, 249, 251, 265, 273, 295, 251, 269, 253, 259, 231, 253, 264, 275, 248, 258, 254, 244, 220, 250, 237, 271, 248, 237, 252, 271, 253, 249, 251, 273, 226, 265, 240, 235, 236, 255, 250, 256, 258, 256, 257, 261, 236, 263, 259, 236, 258, 254, 270, 266, 255, 243, 263, 262, 260, 252, 247, 230, 244, 252, 267, 242, 259, 255, 251, 252, 295, 253, 253, 243, 264, 261, 233, 275, 259, 258, 244, 259, 244, 248, 256, 263, 271, 247, 288, 252, 240, 240, 265, 262, 246, 275, 271, 267, 264, 251, 268, 239, 248, 272, 232, 259, 264, 270, 262, 255, 260, 253, 270, 240, 251, 245, 237, 257, 246, 274] payload_rows_per_dst=[5630, 5512, 5418, 5456, 5492, 5468, 5488, 5490] dup_rows_per_dst=[2788, 2682, 2580, 2700, 2676, 2688, 2808, 2660] combine_rows_per_src=[5630, 5512, 5418, 5456, 5492, 5468, 5488, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38104 dup_rows=21902 dup_rows_remote=19120 dedup_ratio=0.334198 router_tokens_per_expert=[237, 255, 270, 273, 247, 218, 275, 238, 239, 261, 229, 267, 232, 280, 263, 273, 230, 246, 263, 248, 259, 263, 249, 287, 278, 269, 254, 232, 255, 268, 240, 245, 235, 247, 224, 260, 276, 246, 276, 261, 276, 254, 273, 265, 251, 247, 258, 268, 277, 277, 212, 286, 251, 238, 255, 272, 251, 281, 237, 244, 248, 266, 261, 269, 213, 276, 270, 258, 248, 271, 259, 238, 245, 251, 263, 269, 227, 244, 251, 279, 252, 238, 268, 269, 279, 249, 273, 244, 252, 267, 241, 251, 232, 251, 241, 252, 282, 267, 265, 253, 277, 239, 248, 261, 245, 268, 273, 255, 253, 259, 230, 231, 251, 272, 276, 242, 268, 267, 241, 290, 256, 248, 263, 261, 267, 247, 259, 248] payload_rows_per_dst=[5436, 5484, 5478, 5422, 5426, 5412, 5446, 5530] dup_rows_per_dst=[2678, 2688, 2756, 2828, 2698, 2674, 2798, 2782] combine_rows_per_src=[5436, 5484, 5478, 5422, 5426, 5412, 5446, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38124 dup_rows=21798 dup_rows_remote=18920 dedup_ratio=0.332611 router_tokens_per_expert=[228, 242, 229, 255, 276, 243, 219, 241, 268, 266, 238, 256, 246, 230, 256, 268, 274, 241, 244, 258, 213, 263, 261, 260, 264, 261, 247, 260, 250, 244, 245, 257, 253, 243, 258, 274, 230, 238, 258, 257, 264, 250, 250, 245, 252, 265, 256, 252, 263, 237, 239, 262, 256, 260, 241, 264, 276, 285, 274, 258, 270, 256, 247, 235, 285, 255, 232, 277, 246, 264, 283, 280, 281, 267, 269, 271, 243, 273, 251, 269, 228, 275, 256, 252, 247, 252, 245, 253, 273, 249, 221, 262, 232, 280, 241, 264, 288, 253, 255, 264, 262, 273, 272, 283, 270, 287, 275, 246, 238, 240, 270, 251, 246, 253, 258, 254, 258, 250, 275, 256, 264, 284, 246, 253, 252, 248, 248, 249] payload_rows_per_dst=[5352, 5420, 5364, 5500, 5614, 5388, 5642, 5458] dup_rows_per_dst=[2570, 2664, 2726, 2746, 2878, 2672, 2812, 2730] combine_rows_per_src=[5352, 5420, 5364, 5500, 5614, 5388, 5642, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43478 payload_rows_remote=38006 dup_rows=22058 dup_rows_remote=19418 dedup_ratio=0.336578 router_tokens_per_expert=[253, 272, 295, 261, 271, 246, 239, 255, 221, 243, 244, 302, 248, 238, 244, 274, 242, 232, 238, 266, 247, 282, 261, 260, 262, 274, 272, 282, 267, 262, 252, 250, 285, 257, 260, 248, 237, 245, 259, 246, 238, 257, 281, 247, 246, 236, 287, 227, 270, 244, 258, 258, 234, 244, 252, 260, 270, 247, 260, 271, 264, 282, 277, 240, 270, 239, 243, 235, 259, 250, 234, 259, 247, 255, 274, 240, 224, 288, 263, 261, 255, 247, 265, 266, 264, 278, 264, 237, 274, 267, 269, 259, 258, 262, 252, 258, 229, 269, 247, 247, 244, 244, 271, 244, 252, 275, 273, 256, 249, 240, 260, 229, 256, 256, 268, 248, 246, 247, 268, 238, 249, 270, 259, 247, 242, 263, 265, 259] payload_rows_per_dst=[5412, 5424, 5472, 5472, 5354, 5454, 5428, 5462] dup_rows_per_dst=[2800, 2874, 2640, 2790, 2728, 2896, 2630, 2700] combine_rows_per_src=[5412, 5424, 5472, 5472, 5354, 5454, 5428, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228640 combine_payload_bytes=156360704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156589344 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950048 weight_movement_bytes=37748736 total_nvlink_bytes=350698784 weight_to_token_ratio=0.120622 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229392 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157171728 ingress_bytes=194691072 link_bytes=194691072 token_bytes=314114064 weight_movement_bytes=37748736 total_nvlink_bytes=351862800 weight_to_token_ratio=0.120175 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865434058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=229368 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157687800 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315146232 weight_movement_bytes=18874368 total_nvlink_bytes=334020600 weight_to_token_ratio=0.059891 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38304 dup_rows=21884 dup_rows_remote=19110 dedup_ratio=0.333923 router_tokens_per_expert=[261, 264, 250, 256, 256, 247, 261, 249, 270, 213, 239, 265, 250, 290, 279, 299, 228, 229, 248, 262, 248, 289, 231, 264, 256, 243, 254, 285, 266, 283, 230, 249, 270, 233, 281, 260, 237, 265, 263, 280, 238, 251, 215, 296, 233, 261, 244, 241, 228, 226, 263, 260, 254, 264, 275, 226, 283, 237, 258, 267, 238, 265, 227, 231, 234, 260, 272, 250, 274, 248, 268, 243, 243, 258, 272, 263, 269, 266, 259, 272, 246, 235, 266, 287, 271, 252, 254, 269, 251, 239, 238, 254, 240, 223, 257, 279, 256, 259, 272, 224, 240, 254, 233, 255, 269, 247, 300, 269, 277, 269, 268, 242, 233, 244, 243, 262, 273, 276, 266, 250, 269, 263, 261, 271, 245, 261, 271, 250] payload_rows_per_dst=[5438, 5444, 5450, 5396, 5488, 5348, 5556, 5532] dup_rows_per_dst=[2860, 2686, 2686, 2608, 2814, 2774, 2712, 2744] combine_rows_per_src=[5438, 5444, 5450, 5396, 5488, 5348, 5556, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=228896 combine_payload_bytes=156073984 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156302880 ingress_bytes=212697088 link_bytes=212697088 token_bytes=312376864 weight_movement_bytes=56623104 total_nvlink_bytes=368999968 weight_to_token_ratio=0.181265 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037958844e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228384 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156474400 ingress_bytes=175120384 link_bytes=175120384 token_bytes=312720416 weight_movement_bytes=18874368 total_nvlink_bytes=331594784 weight_to_token_ratio=0.060355 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677917355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228176 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156384080 ingress_bytes=175030272 link_bytes=175030272 token_bytes=312539984 weight_movement_bytes=18874368 total_nvlink_bytes=331414352 weight_to_token_ratio=0.060390 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677053946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155672576 dispatch_meta_bytes=229696 combine_payload_bytes=155672576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155902272 ingress_bytes=174546944 link_bytes=174546944 token_bytes=311574848 weight_movement_bytes=18874368 total_nvlink_bytes=330449216 weight_to_token_ratio=0.060577 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672422935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229656 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122840 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314016024 weight_movement_bytes=37748736 total_nvlink_bytes=351764760 weight_to_token_ratio=0.120213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38308 dup_rows=21738 dup_rows_remote=18984 dedup_ratio=0.331696 router_tokens_per_expert=[253, 232, 254, 254, 246, 229, 291, 228, 256, 238, 240, 253, 251, 242, 238, 247, 240, 259, 274, 238, 261, 272, 290, 259, 257, 260, 255, 228, 266, 258, 264, 270, 274, 250, 252, 283, 263, 256, 260, 263, 255, 227, 250, 232, 255, 251, 230, 265, 274, 250, 291, 245, 237, 272, 247, 266, 271, 234, 230, 254, 274, 269, 263, 245, 238, 286, 272, 279, 264, 251, 293, 259, 236, 239, 254, 255, 285, 273, 268, 258, 238, 233, 269, 276, 243, 220, 261, 240, 280, 230, 258, 245, 279, 266, 249, 230, 270, 260, 238, 273, 258, 273, 253, 247, 262, 239, 238, 261, 243, 255, 270, 277, 259, 241, 271, 255, 267, 243, 272, 250, 252, 269, 265, 251, 259, 253, 260, 266] payload_rows_per_dst=[5370, 5544, 5414, 5490, 5606, 5422, 5476, 5476] dup_rows_per_dst=[2534, 2758, 2718, 2754, 2814, 2612, 2758, 2790] combine_rows_per_src=[5370, 5544, 5414, 5490, 5606, 5422, 5476, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38036 dup_rows=21976 dup_rows_remote=19034 dedup_ratio=0.335327 router_tokens_per_expert=[239, 257, 273, 248, 264, 284, 260, 267, 264, 257, 266, 264, 252, 233, 268, 271, 234, 268, 251, 238, 271, 247, 251, 231, 224, 244, 258, 261, 258, 262, 230, 265, 249, 259, 239, 277, 255, 278, 272, 254, 249, 250, 264, 256, 259, 228, 268, 228, 256, 232, 245, 265, 254, 226, 249, 273, 248, 232, 238, 260, 251, 246, 251, 269, 288, 247, 255, 274, 268, 239, 251, 259, 282, 267, 247, 272, 249, 252, 274, 228, 281, 245, 275, 249, 278, 229, 268, 268, 256, 269, 235, 236, 284, 270, 252, 262, 256, 268, 250, 249, 230, 256, 253, 233, 260, 268, 275, 252, 250, 244, 265, 258, 257, 260, 264, 284, 233, 254, 253, 261, 267, 251, 267, 242, 266, 255, 256, 282] payload_rows_per_dst=[5524, 5464, 5486, 5216, 5510, 5398, 5442, 5520] dup_rows_per_dst=[2942, 2750, 2938, 2520, 2794, 2688, 2692, 2652] combine_rows_per_src=[5524, 5464, 5486, 5216, 5510, 5398, 5442, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38288 dup_rows=21858 dup_rows_remote=19194 dedup_ratio=0.333527 router_tokens_per_expert=[281, 280, 252, 251, 276, 264, 261, 255, 246, 271, 255, 245, 216, 233, 254, 249, 235, 256, 251, 249, 274, 267, 262, 251, 241, 246, 279, 234, 265, 266, 240, 311, 277, 256, 261, 240, 219, 221, 266, 254, 251, 281, 251, 254, 307, 242, 251, 253, 257, 232, 279, 249, 238, 245, 256, 233, 271, 262, 235, 260, 266, 254, 262, 273, 275, 252, 273, 259, 265, 244, 253, 243, 264, 296, 267, 261, 257, 234, 248, 247, 250, 244, 271, 268, 241, 254, 261, 274, 245, 270, 255, 254, 250, 275, 281, 287, 250, 224, 258, 249, 242, 256, 244, 259, 245, 247, 245, 269, 260, 256, 276, 288, 250, 253, 259, 225, 254, 265, 256, 246, 271, 278, 248, 244, 244, 219, 243, 255] payload_rows_per_dst=[5466, 5296, 5384, 5440, 5474, 5774, 5454, 5390] dup_rows_per_dst=[2712, 2354, 2784, 2688, 2736, 3190, 2730, 2664] combine_rows_per_src=[5466, 5296, 5384, 5440, 5474, 5774, 5454, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43500 payload_rows_remote=38064 dup_rows=22036 dup_rows_remote=19294 dedup_ratio=0.336243 router_tokens_per_expert=[265, 245, 277, 237, 260, 273, 261, 236, 255, 279, 220, 262, 236, 236, 280, 268, 255, 273, 271, 272, 248, 218, 245, 269, 265, 232, 256, 272, 251, 244, 246, 272, 244, 268, 261, 253, 262, 275, 271, 282, 267, 263, 253, 266, 259, 247, 280, 249, 235, 249, 244, 251, 255, 248, 272, 269, 235, 286, 280, 257, 268, 298, 271, 251, 243, 243, 248, 234, 263, 242, 256, 243, 256, 238, 229, 280, 243, 274, 279, 246, 257, 241, 259, 218, 250, 239, 261, 251, 262, 261, 302, 221, 234, 288, 284, 258, 269, 267, 234, 250, 246, 259, 246, 247, 275, 284, 256, 245, 252, 242, 234, 256, 249, 211, 252, 266, 265, 252, 244, 223, 221, 246, 305, 286, 246, 263, 253, 273] payload_rows_per_dst=[5538, 5436, 5538, 5328, 5338, 5448, 5382, 5492] dup_rows_per_dst=[2938, 2742, 2862, 2714, 2696, 2724, 2742, 2618] combine_rows_per_src=[5538, 5436, 5538, 5328, 5338, 5448, 5382, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38216 dup_rows=21882 dup_rows_remote=19200 dedup_ratio=0.333893 router_tokens_per_expert=[226, 256, 242, 245, 264, 268, 284, 256, 223, 275, 269, 238, 252, 245, 267, 259, 248, 255, 226, 280, 260, 252, 245, 255, 253, 259, 249, 260, 255, 212, 263, 270, 296, 232, 265, 270, 231, 244, 241, 257, 252, 251, 259, 248, 280, 254, 277, 272, 289, 275, 264, 267, 254, 259, 262, 260, 251, 252, 253, 253, 241, 239, 296, 261, 258, 248, 268, 244, 240, 243, 226, 241, 263, 270, 247, 251, 275, 246, 269, 255, 271, 249, 229, 246, 252, 271, 234, 250, 267, 255, 264, 275, 274, 254, 237, 270, 272, 278, 258, 268, 287, 261, 257, 243, 251, 244, 233, 243, 245, 227, 267, 226, 265, 282, 242, 252, 246, 255, 266, 253, 263, 267, 247, 259, 254, 254, 278, 267] payload_rows_per_dst=[5492, 5468, 5520, 5478, 5388, 5442, 5438, 5428] dup_rows_per_dst=[2646, 2616, 2738, 2874, 2700, 2754, 2682, 2872] combine_rows_per_src=[5492, 5468, 5520, 5478, 5388, 5442, 5438, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38374 dup_rows=21784 dup_rows_remote=19118 dedup_ratio=0.332397 router_tokens_per_expert=[269, 259, 252, 257, 245, 258, 247, 249, 253, 225, 258, 271, 263, 262, 260, 265, 265, 279, 267, 268, 260, 258, 249, 256, 250, 286, 272, 256, 245, 251, 247, 270, 262, 252, 256, 253, 233, 247, 247, 278, 233, 243, 269, 258, 276, 257, 242, 270, 266, 255, 276, 249, 259, 278, 281, 262, 275, 248, 254, 248, 290, 227, 251, 247, 281, 243, 254, 259, 263, 240, 249, 251, 250, 251, 252, 241, 275, 219, 261, 233, 229, 243, 235, 272, 249, 254, 271, 250, 234, 268, 248, 270, 231, 239, 288, 263, 293, 271, 271, 274, 271, 282, 268, 275, 261, 267, 254, 238, 235, 245, 254, 256, 230, 238, 229, 244, 244, 284, 232, 255, 260, 243, 263, 255, 234, 256, 261, 245] payload_rows_per_dst=[5462, 5574, 5432, 5502, 5378, 5416, 5594, 5394] dup_rows_per_dst=[2724, 2784, 2720, 2830, 2666, 2672, 2836, 2552] combine_rows_per_src=[5462, 5574, 5432, 5502, 5378, 5416, 5594, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38240 dup_rows=21862 dup_rows_remote=19288 dedup_ratio=0.333588 router_tokens_per_expert=[262, 280, 251, 251, 249, 257, 253, 246, 271, 257, 246, 212, 252, 257, 250, 253, 285, 266, 228, 259, 243, 261, 244, 255, 278, 259, 257, 283, 239, 268, 250, 255, 256, 220, 257, 225, 242, 264, 255, 256, 255, 240, 247, 265, 255, 243, 279, 245, 282, 278, 293, 261, 256, 242, 283, 237, 269, 260, 290, 271, 254, 217, 264, 254, 244, 258, 265, 247, 251, 226, 277, 269, 248, 261, 250, 253, 274, 279, 223, 261, 274, 241, 239, 242, 237, 252, 268, 235, 254, 273, 253, 224, 256, 276, 276, 238, 248, 267, 233, 253, 277, 273, 240, 270, 272, 284, 268, 274, 239, 256, 258, 260, 236, 251, 261, 255, 257, 262, 263, 253, 271, 251, 269, 235, 242, 253, 272, 249] payload_rows_per_dst=[5418, 5478, 5434, 5540, 5464, 5412, 5456, 5472] dup_rows_per_dst=[2676, 2782, 2574, 2882, 2708, 2664, 2888, 2688] combine_rows_per_src=[5418, 5478, 5434, 5540, 5464, 5412, 5456, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229168 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157138736 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048304 weight_movement_bytes=37748736 total_nvlink_bytes=351797040 weight_to_token_ratio=0.120200 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155795456 dispatch_meta_bytes=228280 combine_payload_bytes=155795456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156023736 ingress_bytes=193544192 link_bytes=193544192 token_bytes=311819192 weight_movement_bytes=37748736 total_nvlink_bytes=349567928 weight_to_token_ratio=0.121060 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854445218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229928 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057576 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313885224 weight_movement_bytes=37748736 total_nvlink_bytes=351633960 weight_to_token_ratio=0.120263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38210 dup_rows=21874 dup_rows_remote=19198 dedup_ratio=0.333771 router_tokens_per_expert=[281, 266, 253, 233, 250, 257, 278, 256, 253, 256, 217, 236, 268, 256, 257, 241, 267, 257, 247, 251, 238, 257, 282, 222, 267, 259, 253, 262, 260, 270, 283, 249, 275, 252, 254, 245, 251, 247, 255, 251, 250, 263, 250, 252, 248, 259, 250, 271, 240, 261, 256, 270, 268, 253, 277, 247, 278, 257, 281, 245, 257, 261, 236, 266, 270, 266, 258, 259, 248, 245, 241, 254, 275, 246, 274, 275, 247, 234, 259, 253, 240, 254, 257, 245, 258, 227, 261, 258, 267, 257, 222, 262, 259, 258, 292, 247, 237, 257, 292, 243, 256, 242, 264, 282, 242, 230, 226, 256, 274, 267, 236, 259, 262, 260, 241, 290, 255, 224, 254, 290, 265, 241, 261, 245, 266, 261, 254, 260] payload_rows_per_dst=[5470, 5488, 5404, 5476, 5434, 5452, 5518, 5420] dup_rows_per_dst=[2646, 2760, 2742, 2830, 2774, 2676, 2608, 2838] combine_rows_per_src=[5470, 5488, 5404, 5476, 5434, 5452, 5518, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=229432 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156139576 ingress_bytes=193658880 link_bytes=193658880 token_bytes=312049720 weight_movement_bytes=37748736 total_nvlink_bytes=349798456 weight_to_token_ratio=0.120970 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855544102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229664 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762400 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295136 weight_movement_bytes=18874368 total_nvlink_bytes=332169504 weight_to_token_ratio=0.060245 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229968 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157409872 ingress_bytes=213803008 link_bytes=213803008 token_bytes=314589776 weight_movement_bytes=56623104 total_nvlink_bytes=371212880 weight_to_token_ratio=0.179990 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048555225e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=230112 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156861152 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313492192 weight_movement_bytes=18874368 total_nvlink_bytes=332366560 weight_to_token_ratio=0.060207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229632 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737792 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245952 weight_movement_bytes=18874368 total_nvlink_bytes=332120320 weight_to_token_ratio=0.060254 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38372 dup_rows=21644 dup_rows_remote=18910 dedup_ratio=0.330261 router_tokens_per_expert=[279, 251, 251, 266, 254, 267, 253, 278, 267, 263, 246, 279, 234, 251, 266, 252, 264, 250, 242, 272, 243, 252, 272, 253, 255, 245, 246, 236, 265, 269, 273, 272, 243, 263, 212, 251, 243, 257, 237, 258, 251, 273, 267, 284, 254, 269, 271, 258, 253, 242, 270, 280, 254, 267, 272, 246, 246, 253, 267, 245, 250, 273, 229, 280, 245, 240, 261, 249, 249, 265, 265, 247, 257, 254, 243, 223, 265, 272, 231, 288, 224, 239, 286, 227, 259, 247, 260, 255, 263, 277, 250, 233, 297, 257, 250, 253, 255, 233, 247, 259, 240, 265, 261, 255, 245, 247, 261, 258, 268, 280, 256, 250, 265, 243, 272, 253, 227, 241, 243, 225, 296, 238, 263, 261, 286, 248, 231, 281] payload_rows_per_dst=[5480, 5530, 5528, 5520, 5424, 5440, 5458, 5512] dup_rows_per_dst=[2834, 2688, 2654, 2734, 2684, 2714, 2702, 2634] combine_rows_per_src=[5480, 5530, 5528, 5520, 5424, 5440, 5458, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38058 dup_rows=21980 dup_rows_remote=19122 dedup_ratio=0.335388 router_tokens_per_expert=[243, 242, 256, 247, 256, 248, 219, 249, 250, 252, 247, 258, 226, 277, 270, 279, 239, 259, 262, 253, 252, 233, 255, 247, 265, 282, 246, 276, 249, 241, 278, 265, 267, 220, 234, 279, 267, 284, 256, 260, 236, 236, 259, 238, 244, 274, 280, 251, 255, 275, 252, 287, 268, 255, 265, 259, 269, 234, 272, 243, 244, 246, 275, 248, 255, 247, 230, 283, 239, 254, 274, 258, 254, 253, 235, 281, 275, 250, 291, 260, 236, 230, 257, 265, 254, 268, 241, 214, 223, 243, 270, 259, 289, 261, 246, 276, 259, 265, 266, 268, 231, 277, 229, 275, 257, 242, 243, 250, 249, 263, 246, 246, 263, 286, 286, 237, 249, 287, 238, 239, 264, 279, 232, 264, 254, 282, 277, 241] payload_rows_per_dst=[5264, 5402, 5324, 5444, 5700, 5508, 5416, 5498] dup_rows_per_dst=[2382, 2622, 2430, 2850, 2994, 2948, 2896, 2858] combine_rows_per_src=[5264, 5402, 5324, 5444, 5700, 5508, 5416, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38262 dup_rows=21658 dup_rows_remote=18886 dedup_ratio=0.330475 router_tokens_per_expert=[245, 242, 250, 254, 251, 263, 273, 269, 258, 248, 243, 241, 243, 263, 252, 282, 259, 254, 257, 247, 231, 261, 272, 256, 253, 258, 263, 252, 253, 270, 244, 257, 278, 269, 256, 249, 271, 231, 282, 247, 245, 253, 254, 246, 272, 249, 263, 243, 220, 269, 299, 245, 268, 255, 249, 260, 249, 234, 260, 270, 263, 283, 281, 245, 231, 243, 242, 263, 261, 221, 239, 271, 252, 279, 232, 252, 257, 229, 248, 244, 265, 256, 216, 276, 244, 282, 257, 240, 248, 266, 249, 264, 263, 234, 261, 244, 227, 270, 251, 274, 274, 258, 261, 249, 263, 221, 278, 273, 244, 277, 255, 282, 273, 276, 272, 250, 250, 276, 273, 256, 250, 261, 251, 245, 272, 263, 249, 243] payload_rows_per_dst=[5616, 5464, 5574, 5512, 5454, 5468, 5366, 5424] dup_rows_per_dst=[2772, 2718, 2780, 2624, 2772, 2670, 2580, 2742] combine_rows_per_src=[5616, 5464, 5574, 5512, 5454, 5468, 5366, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38142 dup_rows=21938 dup_rows_remote=19232 dedup_ratio=0.334747 router_tokens_per_expert=[260, 275, 236, 263, 270, 235, 257, 262, 288, 266, 282, 250, 226, 275, 264, 259, 225, 250, 263, 255, 287, 234, 253, 259, 263, 268, 236, 262, 260, 266, 254, 246, 275, 271, 278, 261, 264, 260, 269, 266, 260, 250, 243, 254, 241, 265, 225, 260, 268, 249, 250, 256, 265, 252, 248, 236, 256, 262, 267, 273, 248, 249, 263, 295, 262, 251, 231, 238, 244, 251, 252, 274, 245, 239, 242, 278, 253, 240, 267, 264, 249, 249, 262, 234, 279, 236, 265, 274, 241, 269, 237, 261, 262, 257, 265, 260, 249, 257, 239, 238, 261, 288, 234, 264, 248, 247, 257, 267, 260, 270, 246, 228, 272, 226, 270, 240, 224, 252, 241, 272, 267, 262, 271, 245, 273, 227, 262, 252] payload_rows_per_dst=[5476, 5456, 5506, 5464, 5380, 5444, 5466, 5406] dup_rows_per_dst=[2860, 2706, 2778, 2810, 2682, 2756, 2640, 2706] combine_rows_per_src=[5476, 5456, 5506, 5464, 5380, 5444, 5466, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38282 dup_rows=21808 dup_rows_remote=19126 dedup_ratio=0.332764 router_tokens_per_expert=[246, 268, 256, 249, 272, 255, 259, 252, 244, 244, 247, 276, 237, 245, 248, 240, 258, 268, 265, 252, 247, 269, 258, 259, 261, 277, 267, 255, 242, 248, 245, 275, 263, 257, 257, 242, 306, 268, 271, 237, 272, 261, 239, 279, 245, 257, 224, 256, 268, 243, 262, 270, 261, 273, 256, 245, 232, 253, 261, 260, 254, 268, 268, 273, 256, 239, 233, 243, 252, 274, 259, 237, 281, 265, 267, 234, 251, 278, 237, 255, 254, 271, 244, 271, 257, 259, 240, 254, 252, 257, 258, 283, 237, 250, 252, 268, 218, 266, 254, 260, 243, 269, 240, 252, 286, 272, 283, 228, 259, 249, 244, 241, 247, 265, 229, 239, 278, 240, 247, 261, 275, 272, 240, 276, 265, 248, 234, 255] payload_rows_per_dst=[5420, 5564, 5524, 5464, 5434, 5464, 5446, 5412] dup_rows_per_dst=[2656, 2728, 2744, 2830, 2688, 2750, 2682, 2730] combine_rows_per_src=[5420, 5564, 5524, 5464, 5434, 5464, 5446, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38222 dup_rows=21872 dup_rows_remote=19146 dedup_ratio=0.333740 router_tokens_per_expert=[254, 239, 255, 249, 252, 257, 237, 266, 278, 255, 264, 262, 288, 251, 257, 249, 259, 242, 289, 242, 268, 278, 250, 245, 252, 249, 248, 266, 271, 257, 244, 316, 269, 278, 260, 297, 279, 290, 259, 237, 259, 257, 258, 270, 248, 271, 245, 267, 251, 268, 230, 264, 271, 246, 250, 259, 258, 276, 273, 284, 263, 256, 249, 267, 256, 276, 290, 257, 263, 230, 231, 254, 240, 252, 234, 271, 260, 262, 237, 271, 222, 258, 280, 233, 248, 258, 240, 249, 253, 250, 258, 233, 244, 245, 244, 238, 244, 248, 268, 256, 244, 231, 249, 271, 247, 267, 243, 226, 262, 253, 261, 227, 243, 235, 246, 251, 254, 291, 228, 256, 245, 239, 248, 266, 248, 269, 253, 264] payload_rows_per_dst=[5484, 5486, 5594, 5522, 5442, 5396, 5334, 5406] dup_rows_per_dst=[2742, 2866, 2894, 2808, 2726, 2510, 2660, 2666] combine_rows_per_src=[5484, 5486, 5594, 5522, 5442, 5396, 5334, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38256 dup_rows=21780 dup_rows_remote=19088 dedup_ratio=0.332336 router_tokens_per_expert=[262, 227, 252, 239, 256, 260, 280, 278, 278, 296, 253, 262, 247, 272, 266, 246, 266, 259, 269, 254, 255, 258, 252, 253, 268, 235, 272, 251, 237, 269, 254, 271, 261, 241, 253, 258, 249, 264, 246, 234, 261, 254, 268, 263, 265, 269, 244, 266, 249, 266, 280, 250, 232, 254, 258, 248, 260, 262, 266, 255, 246, 226, 258, 269, 262, 240, 246, 273, 236, 262, 234, 271, 249, 258, 225, 234, 245, 226, 247, 271, 223, 268, 245, 241, 244, 281, 270, 248, 260, 268, 285, 240, 253, 268, 257, 255, 266, 242, 243, 268, 256, 269, 227, 275, 243, 266, 257, 280, 241, 261, 269, 245, 260, 241, 293, 249, 260, 259, 217, 272, 255, 256, 282, 242, 268, 266, 243, 240] payload_rows_per_dst=[5484, 5472, 5500, 5454, 5366, 5492, 5502, 5486] dup_rows_per_dst=[2864, 2774, 2692, 2704, 2592, 2720, 2714, 2720] combine_rows_per_src=[5484, 5472, 5500, 5454, 5366, 5492, 5502, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229128 combine_payload_bytes=157171712 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157400840 ingress_bytes=194920448 link_bytes=194920448 token_bytes=314572552 weight_movement_bytes=37748736 total_nvlink_bytes=352321288 weight_to_token_ratio=0.120000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867631826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228592 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156949744 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313670896 weight_movement_bytes=18874368 total_nvlink_bytes=332545264 weight_to_token_ratio=0.060173 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38196 dup_rows=21880 dup_rows_remote=19178 dedup_ratio=0.333862 router_tokens_per_expert=[259, 263, 253, 246, 253, 244, 243, 261, 232, 265, 243, 267, 251, 266, 263, 254, 257, 244, 249, 263, 228, 258, 247, 258, 255, 265, 265, 279, 252, 257, 265, 231, 272, 262, 252, 282, 263, 257, 232, 239, 245, 255, 269, 242, 262, 236, 242, 237, 246, 246, 260, 270, 273, 235, 259, 251, 268, 259, 237, 256, 253, 231, 286, 248, 245, 222, 261, 271, 264, 262, 248, 253, 259, 254, 273, 280, 282, 263, 263, 275, 267, 230, 257, 252, 280, 235, 271, 264, 247, 249, 272, 228, 248, 271, 233, 277, 233, 231, 265, 247, 257, 249, 242, 263, 303, 272, 248, 272, 260, 262, 246, 257, 280, 263, 236, 253, 250, 290, 256, 309, 260, 254, 263, 244, 238, 245, 232, 271] payload_rows_per_dst=[5388, 5490, 5432, 5438, 5512, 5460, 5412, 5524] dup_rows_per_dst=[2738, 2656, 2662, 2718, 2838, 2702, 2802, 2764] combine_rows_per_src=[5388, 5490, 5432, 5438, 5512, 5460, 5412, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=228720 combine_payload_bytes=155885568 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156114288 ingress_bytes=155885568 link_bytes=156114288 token_bytes=311999856 weight_movement_bytes=0 total_nvlink_bytes=311999856 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.495810295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229496 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156459128 ingress_bytes=212852736 link_bytes=212852736 token_bytes=312688760 weight_movement_bytes=56623104 total_nvlink_bytes=369311864 weight_to_token_ratio=0.181085 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039450186e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229632 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157032704 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313835776 weight_movement_bytes=56623104 total_nvlink_bytes=370458880 weight_to_token_ratio=0.180423 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229472 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786784 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313344096 weight_movement_bytes=18874368 total_nvlink_bytes=332218464 weight_to_token_ratio=0.060235 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229376 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156925952 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622528 weight_movement_bytes=37748736 total_nvlink_bytes=351371264 weight_to_token_ratio=0.120364 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229496 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680312 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131128 weight_movement_bytes=18874368 total_nvlink_bytes=332005496 weight_to_token_ratio=0.060276 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38270 dup_rows=21776 dup_rows_remote=19034 dedup_ratio=0.332275 router_tokens_per_expert=[243, 241, 255, 246, 260, 271, 242, 264, 244, 267, 267, 237, 279, 248, 256, 226, 262, 266, 251, 258, 278, 251, 255, 294, 239, 260, 255, 252, 241, 285, 242, 250, 260, 260, 266, 265, 257, 248, 247, 245, 267, 237, 268, 245, 249, 281, 277, 234, 260, 252, 249, 250, 263, 258, 250, 266, 245, 286, 237, 275, 258, 259, 256, 252, 233, 272, 245, 241, 269, 245, 270, 254, 253, 286, 300, 247, 273, 241, 250, 244, 257, 263, 254, 246, 265, 242, 262, 238, 242, 258, 279, 256, 293, 255, 255, 248, 253, 235, 256, 263, 245, 273, 236, 271, 256, 260, 255, 239, 248, 243, 253, 263, 260, 220, 268, 267, 256, 238, 245, 242, 273, 238, 268, 272, 266, 244, 261, 258] payload_rows_per_dst=[5450, 5502, 5502, 5490, 5504, 5470, 5372, 5470] dup_rows_per_dst=[2642, 2776, 2710, 2742, 2742, 2756, 2726, 2682] combine_rows_per_src=[5450, 5502, 5502, 5490, 5504, 5470, 5372, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38114 dup_rows=21840 dup_rows_remote=18924 dedup_ratio=0.333252 router_tokens_per_expert=[244, 259, 245, 259, 270, 258, 265, 252, 239, 245, 237, 240, 256, 265, 243, 230, 256, 253, 263, 240, 272, 237, 264, 263, 265, 263, 264, 252, 259, 258, 240, 287, 242, 231, 268, 243, 238, 270, 261, 254, 259, 260, 263, 245, 268, 289, 255, 255, 239, 279, 242, 262, 255, 250, 250, 275, 248, 269, 275, 246, 276, 264, 238, 258, 282, 277, 252, 249, 262, 280, 295, 242, 269, 229, 245, 256, 268, 234, 250, 261, 234, 246, 257, 237, 256, 244, 262, 241, 265, 259, 252, 228, 282, 267, 240, 268, 257, 235, 257, 271, 270, 270, 267, 256, 236, 269, 256, 277, 222, 261, 245, 207, 262, 244, 279, 253, 269, 249, 252, 257, 256, 250, 277, 278, 258, 267, 249, 253] payload_rows_per_dst=[5582, 5596, 5624, 5236, 5504, 5390, 5284, 5480] dup_rows_per_dst=[2916, 2836, 2942, 2532, 2798, 2686, 2464, 2666] combine_rows_per_src=[5582, 5596, 5624, 5236, 5504, 5390, 5284, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=37868 dup_rows=21992 dup_rows_remote=18982 dedup_ratio=0.335571 router_tokens_per_expert=[252, 249, 258, 251, 287, 240, 265, 224, 263, 242, 254, 238, 268, 254, 224, 257, 249, 278, 242, 243, 257, 255, 292, 255, 257, 271, 251, 274, 274, 262, 250, 259, 259, 267, 232, 262, 218, 271, 262, 259, 277, 241, 254, 254, 238, 239, 217, 255, 277, 252, 275, 250, 261, 255, 284, 261, 242, 268, 250, 246, 226, 241, 258, 275, 267, 233, 250, 248, 267, 256, 237, 247, 235, 246, 259, 224, 226, 257, 270, 256, 265, 279, 239, 234, 262, 276, 274, 242, 268, 250, 278, 284, 267, 264, 262, 276, 265, 267, 260, 253, 262, 242, 264, 261, 270, 227, 280, 283, 263, 262, 228, 245, 237, 264, 268, 262, 243, 235, 248, 269, 259, 253, 231, 256, 269, 278, 270, 275] payload_rows_per_dst=[5288, 5342, 5376, 5464, 5396, 5530, 5472, 5676] dup_rows_per_dst=[2638, 2544, 2634, 2744, 2686, 2910, 2826, 3010] combine_rows_per_src=[5288, 5342, 5376, 5464, 5396, 5530, 5472, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38154 dup_rows=21878 dup_rows_remote=19072 dedup_ratio=0.333832 router_tokens_per_expert=[240, 241, 248, 283, 267, 241, 271, 273, 268, 269, 268, 262, 263, 251, 267, 265, 267, 270, 260, 279, 263, 268, 267, 242, 242, 261, 241, 252, 246, 236, 285, 276, 226, 234, 254, 242, 255, 245, 273, 246, 242, 260, 248, 269, 247, 250, 267, 272, 252, 234, 261, 268, 277, 264, 238, 265, 257, 245, 251, 253, 281, 230, 243, 253, 265, 265, 253, 233, 263, 257, 287, 268, 273, 269, 250, 274, 249, 229, 246, 269, 227, 236, 246, 259, 260, 254, 270, 297, 265, 249, 274, 247, 275, 241, 262, 243, 278, 258, 260, 252, 251, 271, 230, 277, 260, 224, 248, 239, 262, 267, 243, 260, 236, 243, 260, 289, 228, 246, 258, 231, 254, 257, 242, 235, 258, 244, 255, 263] payload_rows_per_dst=[5510, 5504, 5420, 5480, 5490, 5404, 5484, 5366] dup_rows_per_dst=[2844, 2806, 2714, 2664, 2810, 2732, 2676, 2632] combine_rows_per_src=[5510, 5504, 5420, 5480, 5490, 5404, 5484, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38368 dup_rows=21660 dup_rows_remote=18822 dedup_ratio=0.330505 router_tokens_per_expert=[224, 267, 236, 230, 268, 254, 251, 265, 253, 263, 265, 265, 255, 287, 225, 234, 273, 245, 269, 278, 226, 264, 259, 245, 260, 239, 266, 269, 267, 212, 256, 226, 236, 261, 240, 259, 257, 235, 244, 246, 266, 243, 271, 251, 245, 239, 240, 249, 268, 241, 255, 254, 265, 248, 253, 246, 264, 273, 279, 254, 233, 289, 275, 250, 229, 262, 253, 280, 267, 256, 236, 261, 251, 250, 244, 243, 242, 242, 274, 283, 254, 268, 248, 265, 259, 245, 236, 225, 234, 243, 252, 269, 268, 257, 244, 232, 268, 241, 256, 276, 268, 246, 272, 256, 257, 264, 262, 267, 275, 272, 230, 263, 273, 314, 252, 263, 232, 273, 300, 276, 252, 275, 247, 268, 269, 270, 261, 273] payload_rows_per_dst=[5468, 5440, 5402, 5594, 5472, 5348, 5508, 5644] dup_rows_per_dst=[2616, 2668, 2562, 2700, 2674, 2650, 2838, 2952] combine_rows_per_src=[5468, 5440, 5402, 5594, 5472, 5348, 5508, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=38014 dup_rows=21978 dup_rows_remote=19092 dedup_ratio=0.335358 router_tokens_per_expert=[253, 223, 265, 245, 251, 295, 271, 268, 258, 246, 280, 252, 289, 264, 238, 244, 273, 268, 242, 259, 250, 263, 267, 252, 256, 269, 275, 226, 241, 262, 260, 300, 256, 254, 246, 246, 224, 248, 272, 267, 209, 222, 276, 254, 266, 252, 235, 278, 254, 275, 274, 250, 236, 271, 253, 248, 266, 265, 246, 273, 243, 258, 242, 267, 270, 250, 281, 266, 261, 291, 259, 252, 265, 261, 264, 273, 262, 240, 254, 266, 243, 265, 234, 269, 263, 266, 250, 257, 252, 263, 259, 264, 271, 269, 262, 253, 245, 219, 245, 265, 261, 255, 242, 246, 244, 270, 268, 254, 237, 260, 239, 278, 213, 234, 249, 249, 244, 266, 246, 242, 244, 244, 243, 239, 262, 269, 256, 254] payload_rows_per_dst=[5454, 5490, 5372, 5474, 5544, 5494, 5346, 5384] dup_rows_per_dst=[2830, 2836, 2638, 2768, 2886, 2786, 2710, 2524] combine_rows_per_src=[5454, 5490, 5372, 5474, 5544, 5494, 5346, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229216 combine_payload_bytes=156753920 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156983136 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737056 weight_movement_bytes=37748736 total_nvlink_bytes=351485792 weight_to_token_ratio=0.120320 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38328 dup_rows=21792 dup_rows_remote=19130 dedup_ratio=0.332520 router_tokens_per_expert=[306, 236, 259, 276, 248, 264, 269, 271, 257, 265, 229, 254, 276, 247, 284, 251, 226, 232, 261, 213, 274, 257, 261, 258, 247, 248, 246, 272, 259, 237, 241, 260, 246, 256, 231, 241, 248, 278, 246, 271, 270, 242, 259, 242, 256, 252, 232, 269, 245, 267, 218, 261, 251, 283, 279, 269, 257, 235, 267, 267, 289, 241, 256, 266, 246, 279, 234, 277, 251, 253, 262, 241, 246, 229, 268, 256, 273, 233, 256, 233, 227, 261, 268, 250, 246, 250, 263, 259, 238, 269, 265, 240, 238, 259, 294, 247, 253, 257, 286, 285, 258, 279, 265, 259, 260, 262, 274, 264, 255, 240, 254, 268, 235, 265, 258, 249, 242, 225, 254, 239, 282, 273, 249, 285, 239, 249, 259, 261] payload_rows_per_dst=[5546, 5402, 5416, 5450, 5514, 5470, 5556, 5390] dup_rows_per_dst=[2838, 2582, 2662, 2852, 2560, 2678, 2882, 2738] combine_rows_per_src=[5546, 5402, 5416, 5450, 5514, 5470, 5556, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228152 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156343096 ingress_bytes=174989312 link_bytes=174989312 token_bytes=312458040 weight_movement_bytes=18874368 total_nvlink_bytes=331332408 weight_to_token_ratio=0.060406 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676661487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155107328 dispatch_meta_bytes=227400 combine_payload_bytes=155107328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155334728 ingress_bytes=173981696 link_bytes=173981696 token_bytes=310442056 weight_movement_bytes=18874368 total_nvlink_bytes=329316424 weight_to_token_ratio=0.060798 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.667007007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38342 dup_rows=21674 dup_rows_remote=18946 dedup_ratio=0.330719 router_tokens_per_expert=[236, 260, 271, 245, 232, 282, 266, 248, 275, 244, 240, 259, 242, 263, 257, 249, 269, 270, 240, 237, 243, 261, 296, 270, 263, 222, 230, 243, 250, 264, 258, 254, 240, 250, 259, 245, 283, 275, 249, 259, 238, 248, 248, 267, 286, 250, 241, 257, 245, 267, 233, 284, 249, 259, 276, 255, 229, 248, 248, 266, 252, 247, 251, 276, 271, 263, 245, 257, 257, 265, 261, 257, 237, 250, 262, 260, 257, 223, 279, 238, 246, 267, 254, 255, 251, 282, 282, 257, 255, 239, 242, 278, 272, 249, 239, 256, 242, 267, 249, 263, 248, 242, 274, 259, 245, 234, 243, 262, 258, 283, 254, 272, 231, 249, 259, 275, 266, 269, 268, 243, 267, 268, 248, 261, 280, 262, 275, 227] payload_rows_per_dst=[5474, 5412, 5468, 5438, 5516, 5520, 5506, 5528] dup_rows_per_dst=[2664, 2728, 2722, 2732, 2648, 2728, 2684, 2768] combine_rows_per_src=[5474, 5412, 5468, 5438, 5516, 5520, 5506, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=228904 combine_payload_bytes=156278784 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156507688 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786472 weight_movement_bytes=18874368 total_nvlink_bytes=331660840 weight_to_token_ratio=0.060343 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=228760 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157384088 ingress_bytes=194904064 link_bytes=194904064 token_bytes=314539416 weight_movement_bytes=37748736 total_nvlink_bytes=352288152 weight_to_token_ratio=0.120013 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867474842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=228424 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155933768 ingress_bytes=174579712 link_bytes=174579712 token_bytes=311639112 weight_movement_bytes=18874368 total_nvlink_bytes=330513480 weight_to_token_ratio=0.060565 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672736902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229832 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157221320 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212808 weight_movement_bytes=18874368 total_nvlink_bytes=333087176 weight_to_token_ratio=0.060069 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229152 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157277984 ingress_bytes=194797568 link_bytes=194797568 token_bytes=314326816 weight_movement_bytes=37748736 total_nvlink_bytes=352075552 weight_to_token_ratio=0.120094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866454450e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38180 dup_rows=21852 dup_rows_remote=19224 dedup_ratio=0.333435 router_tokens_per_expert=[266, 239, 266, 247, 251, 263, 250, 253, 259, 251, 277, 288, 264, 263, 265, 232, 286, 273, 283, 253, 279, 241, 233, 244, 239, 249, 270, 258, 245, 263, 247, 257, 242, 246, 261, 267, 249, 225, 247, 239, 276, 238, 266, 234, 262, 270, 238, 291, 258, 270, 249, 285, 250, 259, 251, 267, 254, 258, 266, 218, 247, 247, 258, 229, 254, 255, 259, 252, 257, 224, 262, 259, 259, 249, 254, 249, 282, 257, 293, 232, 284, 274, 231, 253, 264, 241, 270, 245, 240, 269, 255, 245, 246, 272, 258, 281, 260, 236, 261, 258, 260, 261, 286, 246, 259, 242, 255, 234, 243, 260, 239, 256, 242, 253, 250, 261, 247, 252, 257, 239, 253, 279, 271, 257, 281, 259, 251, 264] payload_rows_per_dst=[5504, 5422, 5400, 5504, 5392, 5552, 5442, 5468] dup_rows_per_dst=[2764, 2818, 2702, 2628, 2802, 2704, 2670, 2764] combine_rows_per_src=[5504, 5422, 5400, 5504, 5392, 5552, 5442, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38048 dup_rows=21996 dup_rows_remote=19244 dedup_ratio=0.335632 router_tokens_per_expert=[272, 278, 249, 257, 268, 256, 254, 260, 265, 221, 268, 256, 259, 219, 255, 259, 257, 250, 254, 271, 245, 254, 247, 251, 249, 225, 231, 267, 270, 291, 242, 268, 258, 246, 247, 245, 280, 251, 258, 264, 249, 273, 259, 265, 258, 260, 290, 293, 248, 261, 285, 282, 260, 254, 237, 244, 271, 260, 276, 255, 243, 270, 253, 239, 229, 251, 249, 253, 247, 260, 250, 269, 246, 265, 238, 278, 280, 240, 254, 244, 253, 244, 280, 253, 245, 255, 282, 235, 266, 221, 264, 249, 266, 239, 259, 253, 264, 245, 246, 270, 253, 221, 257, 247, 236, 243, 274, 257, 264, 242, 276, 258, 247, 251, 242, 275, 253, 240, 234, 244, 261, 248, 265, 275, 265, 259, 261, 276] payload_rows_per_dst=[5192, 5408, 5532, 5486, 5544, 5408, 5478, 5492] dup_rows_per_dst=[2508, 2736, 2860, 2700, 2924, 2758, 2758, 2752] combine_rows_per_src=[5192, 5408, 5532, 5486, 5544, 5408, 5478, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38288 dup_rows=21718 dup_rows_remote=18860 dedup_ratio=0.331390 router_tokens_per_expert=[275, 262, 254, 254, 245, 271, 286, 258, 252, 276, 247, 247, 259, 242, 256, 261, 246, 273, 259, 263, 261, 246, 265, 234, 259, 266, 274, 274, 280, 289, 243, 252, 264, 243, 262, 262, 253, 251, 244, 253, 268, 306, 255, 267, 229, 258, 269, 228, 243, 238, 246, 265, 253, 266, 255, 248, 278, 252, 247, 252, 263, 227, 257, 238, 232, 279, 253, 267, 220, 273, 278, 253, 225, 243, 248, 241, 242, 271, 280, 265, 241, 236, 270, 237, 265, 260, 250, 230, 262, 238, 235, 277, 264, 244, 273, 248, 253, 279, 261, 264, 253, 259, 260, 288, 273, 279, 225, 251, 258, 255, 229, 242, 248, 262, 286, 267, 257, 253, 233, 264, 269, 247, 244, 234, 248, 270, 233, 255] payload_rows_per_dst=[5530, 5494, 5504, 5402, 5462, 5468, 5476, 5482] dup_rows_per_dst=[2858, 2790, 2832, 2528, 2678, 2592, 2782, 2658] combine_rows_per_src=[5530, 5494, 5504, 5402, 5462, 5468, 5476, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38286 dup_rows=21790 dup_rows_remote=19128 dedup_ratio=0.332489 router_tokens_per_expert=[238, 267, 277, 254, 259, 246, 265, 264, 241, 258, 282, 257, 227, 263, 256, 230, 249, 219, 278, 249, 272, 263, 217, 234, 254, 255, 252, 262, 274, 275, 250, 258, 249, 269, 234, 238, 283, 223, 254, 247, 251, 279, 260, 256, 267, 250, 246, 266, 248, 252, 235, 256, 258, 248, 266, 261, 258, 259, 227, 273, 243, 255, 265, 221, 260, 264, 266, 256, 245, 253, 230, 263, 233, 236, 233, 239, 269, 238, 254, 250, 285, 259, 258, 269, 264, 273, 244, 250, 270, 243, 262, 257, 248, 270, 262, 264, 278, 251, 293, 255, 266, 277, 243, 246, 251, 262, 268, 228, 285, 269, 248, 279, 257, 266, 262, 227, 290, 251, 257, 245, 247, 238, 268, 279, 273, 267, 253, 280] payload_rows_per_dst=[5462, 5460, 5438, 5382, 5378, 5512, 5586, 5528] dup_rows_per_dst=[2706, 2662, 2706, 2668, 2600, 2844, 2812, 2792] combine_rows_per_src=[5462, 5460, 5438, 5382, 5378, 5512, 5586, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38272 dup_rows=21770 dup_rows_remote=18988 dedup_ratio=0.332184 router_tokens_per_expert=[262, 247, 275, 238, 254, 261, 251, 229, 256, 289, 268, 254, 235, 241, 263, 253, 270, 246, 235, 270, 255, 240, 265, 252, 249, 276, 264, 245, 226, 256, 246, 257, 251, 268, 244, 267, 247, 273, 250, 247, 275, 244, 261, 261, 249, 245, 251, 270, 257, 253, 249, 257, 242, 271, 250, 242, 238, 268, 234, 280, 255, 269, 250, 255, 232, 215, 262, 266, 283, 256, 239, 227, 252, 244, 238, 273, 279, 250, 252, 260, 266, 262, 241, 257, 271, 284, 281, 276, 246, 283, 247, 280, 261, 236, 247, 255, 224, 279, 255, 263, 240, 239, 286, 257, 254, 258, 262, 260, 261, 274, 260, 266, 279, 272, 240, 228, 280, 246, 267, 256, 242, 260, 254, 223, 290, 235, 249, 287] payload_rows_per_dst=[5486, 5432, 5528, 5458, 5350, 5544, 5494, 5474] dup_rows_per_dst=[2666, 2672, 2678, 2682, 2706, 2842, 2782, 2742] combine_rows_per_src=[5486, 5432, 5528, 5458, 5350, 5544, 5494, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38398 dup_rows=21748 dup_rows_remote=19140 dedup_ratio=0.331848 router_tokens_per_expert=[252, 245, 279, 270, 254, 284, 262, 244, 263, 230, 249, 253, 262, 258, 245, 239, 260, 270, 247, 249, 280, 270, 235, 229, 227, 241, 251, 246, 244, 248, 250, 251, 258, 278, 255, 252, 224, 259, 277, 253, 254, 245, 289, 252, 244, 268, 280, 286, 228, 261, 259, 243, 278, 284, 271, 260, 273, 273, 271, 278, 271, 263, 274, 254, 265, 250, 246, 245, 260, 246, 243, 228, 260, 245, 249, 223, 244, 257, 273, 265, 261, 264, 252, 263, 240, 242, 254, 256, 267, 263, 264, 256, 293, 270, 269, 246, 237, 243, 240, 254, 242, 265, 225, 260, 269, 243, 295, 275, 268, 270, 261, 251, 225, 233, 271, 271, 231, 275, 221, 280, 265, 245, 247, 248, 299, 231, 237, 230] payload_rows_per_dst=[5446, 5350, 5490, 5634, 5390, 5538, 5494, 5446] dup_rows_per_dst=[2732, 2646, 2858, 2848, 2608, 2782, 2702, 2572] combine_rows_per_src=[5446, 5350, 5490, 5634, 5390, 5538, 5494, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229616 combine_payload_bytes=156385280 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156614896 ingress_bytes=213008384 link_bytes=213008384 token_bytes=313000176 weight_movement_bytes=56623104 total_nvlink_bytes=369623280 weight_to_token_ratio=0.180904 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040941529e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38364 dup_rows=21716 dup_rows_remote=19026 dedup_ratio=0.331360 router_tokens_per_expert=[284, 279, 289, 226, 269, 241, 263, 260, 249, 245, 240, 260, 255, 261, 265, 246, 278, 243, 241, 285, 275, 232, 240, 246, 244, 305, 241, 270, 242, 233, 234, 260, 260, 254, 256, 251, 242, 257, 247, 277, 250, 261, 253, 266, 255, 253, 256, 235, 227, 241, 280, 237, 276, 263, 263, 258, 283, 268, 268, 236, 254, 297, 244, 254, 233, 253, 250, 270, 279, 265, 264, 255, 268, 249, 254, 259, 261, 259, 219, 291, 244, 237, 239, 236, 234, 286, 267, 243, 279, 255, 284, 269, 233, 259, 271, 273, 269, 269, 253, 256, 269, 249, 255, 245, 239, 278, 224, 245, 244, 224, 238, 238, 273, 273, 283, 257, 241, 266, 247, 261, 238, 273, 242, 234, 232, 279, 247, 266] payload_rows_per_dst=[5542, 5468, 5456, 5462, 5428, 5490, 5436, 5538] dup_rows_per_dst=[2722, 2670, 2690, 2836, 2830, 2728, 2554, 2686] combine_rows_per_src=[5542, 5468, 5456, 5462, 5428, 5490, 5436, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=155844608 dispatch_meta_bytes=229168 combine_payload_bytes=155844608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156073776 ingress_bytes=174718976 link_bytes=174718976 token_bytes=311918384 weight_movement_bytes=18874368 total_nvlink_bytes=330792752 weight_to_token_ratio=0.060511 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674071261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38230 dup_rows=21776 dup_rows_remote=19046 dedup_ratio=0.332275 router_tokens_per_expert=[259, 224, 273, 266, 265, 250, 266, 223, 274, 255, 267, 260, 241, 256, 280, 250, 259, 273, 277, 294, 262, 283, 243, 271, 287, 245, 277, 233, 251, 242, 246, 256, 266, 228, 259, 233, 239, 222, 237, 260, 253, 228, 238, 259, 267, 228, 249, 235, 246, 233, 274, 269, 256, 231, 259, 213, 241, 270, 249, 278, 264, 243, 237, 233, 248, 282, 261, 278, 244, 271, 263, 248, 279, 274, 240, 252, 240, 234, 266, 279, 270, 270, 272, 260, 266, 252, 260, 256, 245, 221, 242, 256, 228, 279, 288, 265, 258, 249, 245, 244, 284, 237, 280, 263, 268, 267, 276, 265, 246, 258, 268, 246, 256, 263, 264, 276, 269, 241, 263, 252, 256, 278, 252, 221, 254, 265, 241, 269] payload_rows_per_dst=[5460, 5592, 5350, 5350, 5460, 5530, 5560, 5458] dup_rows_per_dst=[2758, 2806, 2452, 2642, 2858, 2730, 2748, 2782] combine_rows_per_src=[5460, 5592, 5350, 5350, 5460, 5530, 5560, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228592 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157056240 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313883888 weight_movement_bytes=37748736 total_nvlink_bytes=351632624 weight_to_token_ratio=0.120263 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229656 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157049112 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868568 weight_movement_bytes=37748736 total_nvlink_bytes=351617304 weight_to_token_ratio=0.120269 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229040 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991152 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753264 weight_movement_bytes=18874368 total_nvlink_bytes=332627632 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=230152 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157508360 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314786568 weight_movement_bytes=18874368 total_nvlink_bytes=333660936 weight_to_token_ratio=0.059959 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229560 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157368504 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314507448 weight_movement_bytes=18874368 total_nvlink_bytes=333381816 weight_to_token_ratio=0.060012 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229104 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156819184 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313409264 weight_movement_bytes=37748736 total_nvlink_bytes=351158000 weight_to_token_ratio=0.120446 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38098 dup_rows=21924 dup_rows_remote=19198 dedup_ratio=0.334534 router_tokens_per_expert=[230, 257, 260, 238, 276, 252, 252, 235, 261, 256, 265, 260, 263, 255, 247, 264, 242, 240, 258, 257, 273, 266, 282, 244, 249, 260, 255, 257, 263, 258, 226, 281, 281, 245, 246, 270, 255, 237, 240, 247, 239, 236, 269, 270, 253, 257, 239, 255, 291, 280, 265, 275, 284, 279, 266, 241, 233, 239, 245, 236, 235, 226, 274, 251, 267, 268, 252, 260, 283, 255, 244, 289, 269, 245, 273, 273, 251, 268, 254, 263, 263, 251, 230, 263, 256, 215, 238, 251, 272, 257, 271, 274, 243, 230, 246, 268, 276, 268, 261, 231, 253, 241, 243, 267, 225, 270, 266, 243, 266, 245, 255, 288, 280, 256, 237, 250, 259, 264, 260, 258, 235, 258, 257, 263, 280, 240, 246, 244] payload_rows_per_dst=[5408, 5434, 5398, 5514, 5554, 5462, 5410, 5432] dup_rows_per_dst=[2734, 2788, 2680, 2726, 2874, 2594, 2786, 2742] combine_rows_per_src=[5408, 5434, 5398, 5514, 5554, 5462, 5410, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38296 dup_rows=21770 dup_rows_remote=18984 dedup_ratio=0.332184 router_tokens_per_expert=[254, 276, 270, 266, 269, 244, 273, 235, 269, 259, 270, 254, 273, 259, 276, 278, 304, 252, 237, 283, 289, 258, 263, 252, 265, 245, 260, 275, 259, 241, 262, 243, 254, 239, 253, 235, 235, 247, 263, 267, 301, 226, 250, 259, 248, 227, 260, 267, 259, 253, 267, 261, 253, 243, 251, 265, 245, 259, 257, 265, 254, 245, 263, 246, 255, 252, 275, 247, 256, 223, 258, 254, 263, 272, 274, 250, 278, 236, 241, 256, 229, 229, 255, 269, 270, 262, 261, 255, 251, 249, 236, 234, 254, 278, 234, 272, 256, 252, 241, 254, 225, 254, 247, 257, 256, 238, 243, 274, 260, 230, 240, 281, 242, 262, 254, 252, 266, 271, 257, 259, 253, 252, 240, 262, 265, 245, 271, 251] payload_rows_per_dst=[5610, 5592, 5414, 5444, 5502, 5436, 5298, 5470] dup_rows_per_dst=[2840, 2784, 2624, 2728, 2692, 2640, 2676, 2786] combine_rows_per_src=[5610, 5592, 5414, 5444, 5502, 5436, 5298, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38344 dup_rows=21692 dup_rows_remote=18980 dedup_ratio=0.330994 router_tokens_per_expert=[239, 251, 222, 277, 274, 258, 245, 244, 277, 287, 242, 267, 234, 267, 247, 259, 265, 250, 267, 262, 274, 261, 268, 234, 259, 271, 239, 244, 264, 250, 261, 263, 273, 245, 254, 256, 273, 245, 298, 239, 233, 248, 256, 246, 282, 245, 274, 244, 270, 279, 242, 253, 247, 263, 254, 240, 244, 272, 237, 271, 245, 275, 248, 254, 243, 283, 268, 253, 226, 239, 256, 235, 241, 235, 256, 248, 256, 244, 257, 242, 257, 246, 251, 278, 300, 259, 282, 279, 283, 245, 250, 257, 236, 252, 260, 254, 258, 267, 244, 256, 281, 233, 243, 259, 280, 272, 240, 255, 249, 262, 274, 258, 265, 251, 258, 238, 253, 269, 232, 257, 221, 270, 235, 248, 281, 241, 249, 271] payload_rows_per_dst=[5500, 5540, 5672, 5534, 5374, 5376, 5396, 5452] dup_rows_per_dst=[2712, 2724, 3150, 2622, 2590, 2542, 2726, 2626] combine_rows_per_src=[5500, 5540, 5672, 5534, 5374, 5376, 5396, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38068 dup_rows=21994 dup_rows_remote=19302 dedup_ratio=0.335602 router_tokens_per_expert=[253, 251, 276, 241, 250, 235, 249, 250, 263, 256, 251, 251, 254, 243, 266, 231, 261, 263, 243, 264, 244, 256, 246, 252, 264, 248, 252, 288, 257, 245, 258, 242, 244, 275, 277, 250, 264, 278, 269, 266, 261, 251, 250, 255, 252, 284, 241, 236, 258, 271, 245, 263, 232, 262, 268, 282, 255, 254, 233, 258, 261, 250, 254, 251, 241, 280, 224, 273, 268, 252, 268, 255, 265, 254, 220, 287, 258, 290, 266, 242, 245, 238, 242, 224, 284, 252, 290, 233, 233, 258, 261, 245, 219, 272, 272, 261, 261, 287, 284, 260, 265, 291, 242, 252, 253, 262, 265, 232, 261, 257, 251, 255, 243, 246, 240, 246, 269, 250, 227, 287, 260, 253, 270, 235, 248, 260, 260, 271] payload_rows_per_dst=[5426, 5474, 5426, 5466, 5460, 5396, 5500, 5394] dup_rows_per_dst=[2614, 2692, 2880, 2728, 2826, 2662, 2856, 2736] combine_rows_per_src=[5426, 5474, 5426, 5466, 5460, 5396, 5500, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38382 dup_rows=21734 dup_rows_remote=18980 dedup_ratio=0.331635 router_tokens_per_expert=[255, 263, 247, 257, 281, 243, 249, 250, 254, 241, 263, 246, 230, 260, 261, 273, 289, 258, 275, 269, 278, 256, 261, 240, 248, 272, 268, 257, 253, 244, 266, 240, 251, 270, 258, 256, 272, 250, 272, 247, 250, 234, 252, 267, 247, 263, 243, 208, 239, 233, 227, 237, 242, 261, 270, 275, 280, 265, 246, 230, 252, 266, 267, 251, 232, 274, 254, 257, 274, 266, 248, 241, 251, 230, 256, 307, 257, 254, 280, 257, 263, 275, 262, 248, 280, 243, 235, 238, 258, 248, 252, 262, 252, 253, 268, 233, 255, 265, 256, 268, 214, 253, 253, 273, 247, 244, 220, 257, 308, 255, 275, 244, 298, 269, 243, 249, 245, 244, 262, 275, 253, 244, 259, 281, 251, 266, 238, 268] payload_rows_per_dst=[5430, 5586, 5422, 5398, 5518, 5474, 5420, 5554] dup_rows_per_dst=[2716, 2762, 2658, 2684, 2758, 2666, 2754, 2736] combine_rows_per_src=[5430, 5586, 5422, 5398, 5518, 5474, 5420, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38244 dup_rows=21746 dup_rows_remote=19058 dedup_ratio=0.331818 router_tokens_per_expert=[236, 273, 289, 227, 251, 247, 257, 256, 241, 265, 278, 258, 227, 287, 251, 232, 275, 253, 280, 268, 233, 254, 252, 254, 261, 237, 262, 228, 273, 266, 256, 241, 254, 232, 252, 258, 235, 247, 251, 255, 250, 248, 267, 273, 256, 262, 255, 267, 265, 236, 244, 247, 259, 250, 260, 245, 260, 282, 231, 245, 274, 262, 249, 265, 236, 275, 261, 259, 266, 256, 237, 253, 267, 265, 255, 254, 259, 272, 247, 255, 233, 250, 251, 271, 249, 250, 247, 259, 221, 279, 260, 254, 263, 258, 270, 282, 250, 262, 253, 256, 257, 266, 281, 259, 243, 253, 244, 280, 266, 235, 273, 229, 275, 254, 249, 259, 279, 271, 251, 250, 265, 237, 260, 251, 250, 270, 238, 284] payload_rows_per_dst=[5446, 5480, 5422, 5512, 5546, 5472, 5408, 5504] dup_rows_per_dst=[2704, 2706, 2702, 2636, 2688, 2722, 2806, 2782] combine_rows_per_src=[5446, 5480, 5422, 5512, 5546, 5472, 5408, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=229184 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156278592 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312328000 weight_movement_bytes=18874368 total_nvlink_bytes=331202368 weight_to_token_ratio=0.060431 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38244 dup_rows=21810 dup_rows_remote=19130 dedup_ratio=0.332794 router_tokens_per_expert=[295, 249, 288, 256, 264, 245, 254, 242, 260, 276, 265, 225, 256, 229, 246, 260, 253, 235, 261, 269, 257, 260, 231, 238, 231, 252, 254, 270, 280, 227, 269, 243, 283, 269, 246, 247, 242, 270, 260, 245, 241, 246, 236, 242, 258, 286, 254, 256, 224, 256, 265, 263, 239, 256, 268, 248, 262, 266, 248, 283, 231, 230, 231, 254, 259, 282, 272, 270, 268, 254, 273, 273, 282, 272, 262, 259, 271, 256, 252, 219, 256, 244, 246, 261, 256, 245, 259, 246, 267, 244, 273, 239, 229, 233, 255, 275, 283, 248, 265, 259, 256, 246, 254, 270, 252, 271, 242, 264, 248, 241, 253, 238, 253, 261, 286, 248, 270, 223, 266, 254, 251, 269, 252, 293, 250, 252, 271, 282] payload_rows_per_dst=[5486, 5396, 5482, 5404, 5536, 5446, 5420, 5556] dup_rows_per_dst=[2734, 2664, 2680, 2644, 2912, 2610, 2760, 2806] combine_rows_per_src=[5486, 5396, 5482, 5404, 5536, 5446, 5420, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229120 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089536 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949952 weight_movement_bytes=18874368 total_nvlink_bytes=332824320 weight_to_token_ratio=0.060119 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38078 dup_rows=21920 dup_rows_remote=19228 dedup_ratio=0.334473 router_tokens_per_expert=[271, 241, 271, 247, 251, 265, 250, 256, 267, 256, 236, 295, 256, 260, 261, 235, 240, 277, 233, 252, 234, 268, 251, 250, 253, 276, 269, 260, 253, 267, 223, 247, 275, 244, 274, 273, 229, 235, 252, 253, 275, 247, 250, 261, 256, 265, 286, 256, 255, 271, 250, 257, 274, 250, 237, 269, 246, 269, 269, 239, 257, 272, 259, 233, 257, 247, 287, 247, 286, 257, 289, 247, 241, 238, 249, 231, 234, 250, 248, 242, 264, 250, 251, 263, 272, 246, 264, 270, 227, 222, 263, 273, 251, 266, 266, 267, 257, 241, 247, 259, 246, 253, 274, 266, 276, 270, 267, 274, 264, 260, 257, 241, 254, 247, 241, 250, 261, 268, 267, 238, 223, 220, 268, 261, 273, 267, 240, 264] payload_rows_per_dst=[5452, 5364, 5434, 5494, 5424, 5538, 5450, 5460] dup_rows_per_dst=[2784, 2742, 2828, 2720, 2676, 2692, 2854, 2624] combine_rows_per_src=[5452, 5364, 5434, 5494, 5424, 5538, 5450, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229296 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157286320 ingress_bytes=194805760 link_bytes=194805760 token_bytes=314343344 weight_movement_bytes=37748736 total_nvlink_bytes=352092080 weight_to_token_ratio=0.120088 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=229480 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156156008 ingress_bytes=174800896 link_bytes=174800896 token_bytes=312082536 weight_movement_bytes=18874368 total_nvlink_bytes=330956904 weight_to_token_ratio=0.060479 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674856178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229448 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157442120 ingress_bytes=194961408 link_bytes=194961408 token_bytes=314654792 weight_movement_bytes=37748736 total_nvlink_bytes=352403528 weight_to_token_ratio=0.119969 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868024284e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229208 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876632 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524056 weight_movement_bytes=37748736 total_nvlink_bytes=351272792 weight_to_token_ratio=0.120401 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229496 combine_payload_bytes=156647424 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156876920 ingress_bytes=213270528 link_bytes=213270528 token_bytes=313524344 weight_movement_bytes=56623104 total_nvlink_bytes=370147448 weight_to_token_ratio=0.180602 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043453264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=229224 combine_payload_bytes=155967488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156196712 ingress_bytes=174841856 link_bytes=174841856 token_bytes=312164200 weight_movement_bytes=18874368 total_nvlink_bytes=331038568 weight_to_token_ratio=0.060463 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675248637e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38098 dup_rows=21954 dup_rows_remote=19106 dedup_ratio=0.334991 router_tokens_per_expert=[242, 274, 263, 265, 264, 272, 242, 233, 255, 257, 240, 249, 260, 261, 276, 259, 256, 259, 249, 237, 246, 243, 247, 278, 244, 273, 235, 239, 274, 264, 281, 249, 283, 230, 249, 266, 256, 252, 251, 244, 248, 291, 278, 271, 262, 245, 235, 235, 278, 269, 247, 241, 248, 253, 267, 287, 243, 268, 239, 263, 285, 264, 231, 283, 261, 264, 259, 266, 257, 224, 271, 278, 239, 258, 242, 245, 262, 256, 264, 275, 263, 274, 258, 252, 253, 235, 255, 228, 252, 250, 253, 246, 252, 261, 278, 257, 219, 241, 263, 259, 267, 256, 249, 260, 245, 245, 276, 263, 278, 235, 263, 246, 282, 260, 255, 217, 256, 268, 246, 246, 245, 249, 269, 265, 251, 262, 246, 250] payload_rows_per_dst=[5450, 5456, 5416, 5484, 5468, 5484, 5410, 5414] dup_rows_per_dst=[2774, 2692, 2776, 2848, 2774, 2650, 2720, 2720] combine_rows_per_src=[5450, 5456, 5416, 5484, 5468, 5484, 5410, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38178 dup_rows=21892 dup_rows_remote=19120 dedup_ratio=0.334045 router_tokens_per_expert=[233, 269, 240, 263, 265, 250, 272, 271, 241, 230, 232, 263, 273, 244, 257, 239, 240, 252, 264, 259, 268, 246, 265, 264, 260, 238, 266, 282, 257, 277, 230, 282, 219, 281, 250, 271, 255, 259, 234, 252, 271, 266, 253, 258, 274, 284, 269, 282, 246, 215, 253, 259, 247, 264, 266, 252, 265, 249, 269, 230, 241, 278, 255, 231, 240, 256, 300, 250, 261, 238, 269, 233, 260, 247, 266, 261, 223, 266, 283, 267, 245, 253, 228, 252, 274, 228, 252, 272, 256, 258, 287, 266, 266, 248, 223, 269, 271, 258, 250, 246, 250, 254, 265, 246, 279, 262, 256, 248, 269, 253, 241, 254, 231, 253, 263, 278, 260, 250, 242, 250, 245, 259, 258, 268, 269, 243, 268, 242] payload_rows_per_dst=[5354, 5524, 5458, 5364, 5442, 5528, 5508, 5466] dup_rows_per_dst=[2730, 2776, 2818, 2676, 2638, 2762, 2720, 2772] combine_rows_per_src=[5354, 5524, 5458, 5364, 5442, 5528, 5508, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228816 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156278224 ingress_bytes=193798144 link_bytes=193798144 token_bytes=312327632 weight_movement_bytes=37748736 total_nvlink_bytes=350076368 weight_to_token_ratio=0.120863 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856878461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38144 dup_rows=21804 dup_rows_remote=18922 dedup_ratio=0.332703 router_tokens_per_expert=[259, 264, 274, 267, 285, 262, 248, 269, 265, 252, 246, 257, 270, 264, 263, 275, 262, 267, 248, 248, 246, 266, 257, 291, 259, 240, 237, 276, 267, 259, 247, 285, 272, 251, 253, 255, 243, 264, 233, 282, 250, 253, 247, 225, 247, 230, 272, 249, 258, 263, 248, 288, 259, 260, 243, 256, 261, 244, 250, 273, 244, 246, 245, 270, 230, 267, 245, 224, 263, 261, 243, 247, 241, 254, 244, 234, 253, 264, 255, 252, 254, 243, 257, 225, 238, 254, 259, 259, 256, 243, 247, 266, 276, 275, 278, 262, 265, 247, 256, 266, 282, 254, 268, 250, 292, 239, 257, 276, 248, 230, 265, 244, 259, 260, 283, 234, 221, 258, 253, 271, 261, 260, 252, 246, 249, 253, 256, 235] payload_rows_per_dst=[5588, 5458, 5644, 5492, 5316, 5344, 5464, 5426] dup_rows_per_dst=[2882, 2832, 2764, 2714, 2638, 2484, 2814, 2676] combine_rows_per_src=[5588, 5458, 5644, 5492, 5316, 5344, 5464, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38178 dup_rows=21806 dup_rows_remote=19104 dedup_ratio=0.332733 router_tokens_per_expert=[271, 248, 267, 237, 273, 241, 238, 258, 249, 248, 234, 225, 252, 286, 251, 264, 240, 246, 257, 237, 260, 247, 260, 260, 256, 242, 244, 245, 244, 273, 270, 262, 276, 264, 258, 276, 252, 273, 253, 276, 248, 279, 242, 247, 245, 296, 278, 248, 246, 256, 269, 276, 274, 249, 276, 245, 260, 256, 253, 262, 230, 256, 229, 247, 267, 253, 218, 249, 272, 286, 254, 239, 260, 259, 242, 262, 268, 283, 283, 253, 255, 256, 238, 243, 251, 241, 268, 276, 263, 263, 257, 228, 268, 246, 265, 270, 253, 265, 247, 245, 248, 264, 245, 252, 250, 266, 242, 268, 248, 245, 274, 225, 264, 265, 240, 239, 276, 247, 258, 268, 256, 244, 263, 232, 264, 258, 279, 262] payload_rows_per_dst=[5414, 5552, 5620, 5414, 5428, 5446, 5360, 5496] dup_rows_per_dst=[2670, 2702, 2802, 2586, 2868, 2730, 2714, 2734] combine_rows_per_src=[5414, 5552, 5620, 5414, 5428, 5446, 5360, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38278 dup_rows=21760 dup_rows_remote=19240 dedup_ratio=0.332031 router_tokens_per_expert=[289, 277, 250, 238, 271, 252, 293, 260, 238, 272, 235, 220, 235, 260, 279, 266, 248, 265, 274, 232, 263, 271, 250, 285, 224, 244, 266, 245, 246, 268, 262, 261, 228, 264, 266, 255, 287, 258, 272, 242, 263, 242, 243, 268, 239, 245, 260, 265, 260, 250, 233, 253, 273, 282, 243, 257, 254, 246, 241, 248, 260, 263, 276, 256, 263, 255, 256, 237, 237, 286, 245, 265, 286, 270, 262, 258, 275, 279, 263, 247, 250, 249, 244, 253, 275, 254, 284, 263, 252, 244, 244, 247, 280, 243, 250, 234, 234, 238, 268, 226, 275, 255, 247, 247, 259, 275, 229, 242, 234, 268, 259, 253, 261, 260, 248, 269, 227, 261, 236, 225, 283, 253, 261, 254, 250, 234, 278, 278] payload_rows_per_dst=[5490, 5410, 5444, 5554, 5470, 5442, 5498, 5468] dup_rows_per_dst=[2780, 2798, 2750, 2636, 2898, 2690, 2520, 2688] combine_rows_per_src=[5490, 5410, 5444, 5554, 5470, 5442, 5498, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38206 dup_rows=21864 dup_rows_remote=19030 dedup_ratio=0.333618 router_tokens_per_expert=[239, 254, 238, 235, 259, 250, 272, 246, 227, 254, 235, 251, 244, 260, 275, 234, 259, 211, 272, 244, 264, 247, 263, 266, 263, 255, 240, 243, 226, 238, 266, 239, 258, 263, 251, 260, 277, 298, 275, 270, 266, 237, 254, 252, 260, 252, 253, 262, 259, 254, 271, 286, 277, 268, 285, 267, 263, 264, 255, 252, 276, 248, 255, 283, 255, 241, 258, 254, 242, 249, 261, 265, 243, 254, 284, 219, 277, 283, 290, 275, 220, 279, 266, 246, 230, 271, 258, 234, 238, 233, 269, 264, 250, 245, 259, 251, 226, 250, 262, 242, 260, 240, 262, 253, 250, 259, 265, 268, 279, 272, 238, 238, 248, 262, 270, 230, 287, 239, 244, 274, 268, 243, 290, 245, 258, 243, 286, 234] payload_rows_per_dst=[5380, 5312, 5566, 5596, 5466, 5442, 5424, 5486] dup_rows_per_dst=[2566, 2680, 2810, 2930, 2834, 2584, 2704, 2756] combine_rows_per_src=[5380, 5312, 5566, 5596, 5466, 5442, 5424, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38292 dup_rows=21712 dup_rows_remote=18934 dedup_ratio=0.331299 router_tokens_per_expert=[263, 261, 271, 227, 242, 285, 267, 267, 287, 255, 267, 256, 259, 241, 251, 255, 262, 249, 268, 249, 239, 231, 244, 246, 267, 243, 253, 238, 245, 246, 265, 249, 271, 267, 256, 263, 248, 249, 256, 252, 260, 276, 258, 238, 247, 272, 278, 264, 295, 253, 236, 258, 255, 253, 242, 274, 296, 228, 248, 252, 277, 245, 272, 228, 279, 265, 252, 290, 252, 265, 262, 254, 264, 282, 258, 241, 250, 276, 261, 277, 224, 257, 256, 257, 254, 247, 272, 262, 250, 270, 234, 261, 264, 253, 271, 251, 238, 245, 247, 245, 276, 250, 262, 261, 243, 269, 245, 244, 254, 239, 270, 265, 263, 241, 277, 238, 233, 257, 266, 226, 231, 237, 258, 253, 251, 252, 268, 238] payload_rows_per_dst=[5522, 5404, 5532, 5492, 5614, 5460, 5422, 5378] dup_rows_per_dst=[2786, 2584, 2778, 2732, 2842, 2706, 2684, 2600] combine_rows_per_src=[5522, 5404, 5532, 5492, 5614, 5460, 5422, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229192 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606280 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983368 weight_movement_bytes=18874368 total_nvlink_bytes=331857736 weight_to_token_ratio=0.060305 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38330 dup_rows=21710 dup_rows_remote=19036 dedup_ratio=0.331268 router_tokens_per_expert=[248, 268, 269, 256, 249, 274, 228, 269, 252, 250, 248, 235, 228, 253, 276, 236, 251, 268, 251, 250, 243, 243, 253, 243, 266, 235, 258, 242, 251, 262, 245, 252, 267, 271, 280, 252, 246, 250, 235, 276, 267, 243, 232, 263, 274, 231, 239, 253, 213, 259, 233, 252, 253, 278, 272, 274, 253, 278, 267, 260, 245, 255, 272, 259, 243, 249, 241, 262, 232, 278, 266, 245, 256, 280, 236, 237, 269, 269, 219, 257, 259, 248, 240, 230, 266, 259, 270, 260, 244, 261, 272, 261, 259, 248, 257, 251, 262, 277, 246, 244, 265, 235, 265, 263, 251, 248, 293, 265, 253, 247, 245, 247, 254, 275, 283, 231, 279, 247, 238, 293, 278, 263, 298, 286, 284, 235, 263, 277] payload_rows_per_dst=[5382, 5360, 5500, 5542, 5418, 5496, 5512, 5616] dup_rows_per_dst=[2696, 2666, 2658, 2704, 2660, 2674, 2700, 2952] combine_rows_per_src=[5382, 5360, 5500, 5542, 5418, 5496, 5512, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=228264 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156466088 ingress_bytes=212860928 link_bytes=212860928 token_bytes=312703912 weight_movement_bytes=56623104 total_nvlink_bytes=369327016 weight_to_token_ratio=0.181076 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039528678e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229128 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606216 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983304 weight_movement_bytes=18874368 total_nvlink_bytes=331857672 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=230072 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016760 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313803448 weight_movement_bytes=18874368 total_nvlink_bytes=332677816 weight_to_token_ratio=0.060147 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228944 combine_payload_bytes=156491776 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156720720 ingress_bytes=231989248 link_bytes=231989248 token_bytes=313212496 weight_movement_bytes=75497472 total_nvlink_bytes=388709968 weight_to_token_ratio=0.241042 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222806829e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=228904 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157072936 ingress_bytes=213467136 link_bytes=213467136 token_bytes=313916968 weight_movement_bytes=56623104 total_nvlink_bytes=370540072 weight_to_token_ratio=0.180376 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045337065e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229464 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157229144 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228824 weight_movement_bytes=18874368 total_nvlink_bytes=333103192 weight_to_token_ratio=0.060066 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38228 dup_rows=21866 dup_rows_remote=19174 dedup_ratio=0.333649 router_tokens_per_expert=[256, 266, 259, 271, 268, 285, 254, 242, 242, 264, 252, 271, 254, 251, 231, 274, 268, 243, 261, 256, 287, 275, 256, 253, 270, 243, 247, 245, 285, 260, 268, 254, 260, 231, 237, 241, 282, 255, 247, 267, 266, 249, 264, 238, 220, 263, 232, 252, 267, 259, 243, 252, 238, 265, 253, 270, 245, 239, 262, 237, 279, 265, 249, 244, 241, 297, 274, 253, 260, 236, 277, 247, 266, 265, 246, 253, 265, 258, 275, 237, 270, 284, 250, 254, 248, 270, 259, 229, 220, 265, 256, 233, 249, 259, 243, 231, 250, 257, 257, 259, 262, 263, 273, 260, 239, 229, 267, 280, 247, 242, 242, 258, 270, 258, 248, 256, 244, 260, 243, 264, 271, 275, 260, 264, 231, 269, 264, 254] payload_rows_per_dst=[5512, 5510, 5434, 5442, 5512, 5344, 5412, 5504] dup_rows_per_dst=[2768, 2832, 2574, 2692, 2788, 2696, 2758, 2758] combine_rows_per_src=[5512, 5510, 5434, 5442, 5512, 5344, 5412, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38178 dup_rows=21910 dup_rows_remote=19172 dedup_ratio=0.334320 router_tokens_per_expert=[241, 243, 270, 269, 245, 264, 277, 259, 258, 243, 260, 301, 255, 268, 235, 241, 280, 256, 263, 253, 262, 276, 246, 268, 272, 225, 280, 249, 229, 267, 262, 203, 254, 270, 254, 257, 260, 244, 247, 228, 230, 261, 282, 250, 244, 266, 258, 258, 300, 263, 271, 235, 245, 252, 255, 251, 266, 261, 245, 249, 263, 212, 274, 229, 275, 256, 273, 275, 223, 270, 264, 252, 246, 263, 280, 274, 275, 258, 255, 281, 251, 233, 280, 230, 270, 282, 258, 264, 255, 252, 290, 283, 257, 235, 257, 232, 257, 242, 260, 250, 281, 238, 242, 268, 250, 240, 283, 257, 229, 242, 249, 237, 263, 258, 280, 268, 227, 250, 264, 274, 239, 234, 231, 261, 243, 232, 248, 268] payload_rows_per_dst=[5470, 5248, 5484, 5364, 5548, 5526, 5538, 5448] dup_rows_per_dst=[2720, 2460, 2800, 2684, 2902, 2732, 2874, 2738] combine_rows_per_src=[5470, 5248, 5484, 5364, 5548, 5526, 5538, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229608 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156811496 ingress_bytes=194330624 link_bytes=194330624 token_bytes=313393384 weight_movement_bytes=37748736 total_nvlink_bytes=351142120 weight_to_token_ratio=0.120452 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861980423e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38502 dup_rows=21610 dup_rows_remote=19032 dedup_ratio=0.329742 router_tokens_per_expert=[252, 272, 262, 237, 248, 250, 251, 240, 252, 240, 248, 267, 244, 212, 243, 230, 222, 261, 263, 243, 255, 278, 260, 290, 239, 276, 234, 281, 258, 280, 270, 249, 298, 258, 261, 252, 288, 257, 255, 275, 261, 271, 257, 250, 268, 244, 264, 248, 219, 256, 256, 216, 249, 248, 251, 264, 264, 244, 263, 235, 246, 245, 223, 254, 297, 237, 252, 246, 254, 256, 242, 273, 260, 245, 257, 293, 244, 248, 240, 252, 267, 244, 242, 247, 261, 272, 242, 311, 268, 250, 263, 288, 246, 272, 252, 256, 234, 230, 250, 240, 262, 290, 264, 270, 252, 264, 273, 263, 271, 252, 261, 261, 245, 256, 245, 218, 258, 244, 282, 258, 242, 268, 273, 276, 272, 259, 267, 244] payload_rows_per_dst=[5424, 5616, 5526, 5516, 5252, 5686, 5436, 5470] dup_rows_per_dst=[2578, 2762, 2828, 2826, 2464, 2898, 2510, 2744] combine_rows_per_src=[5424, 5616, 5526, 5516, 5252, 5686, 5436, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38144 dup_rows=21896 dup_rows_remote=19192 dedup_ratio=0.334106 router_tokens_per_expert=[255, 248, 235, 250, 247, 286, 244, 274, 274, 278, 277, 283, 275, 230, 282, 232, 260, 238, 243, 242, 291, 254, 249, 282, 267, 254, 265, 254, 248, 253, 238, 262, 229, 245, 237, 273, 265, 253, 262, 253, 246, 253, 233, 249, 263, 254, 255, 252, 265, 271, 262, 246, 249, 265, 242, 251, 240, 245, 264, 260, 286, 276, 254, 236, 256, 262, 242, 255, 268, 273, 210, 254, 241, 281, 258, 277, 257, 239, 274, 278, 251, 244, 246, 256, 285, 266, 242, 277, 256, 213, 267, 232, 278, 276, 272, 260, 250, 263, 255, 261, 268, 259, 277, 243, 278, 240, 283, 230, 243, 261, 242, 259, 224, 237, 258, 255, 279, 265, 253, 245, 239, 267, 266, 272, 241, 247, 239, 219] payload_rows_per_dst=[5522, 5496, 5420, 5460, 5478, 5450, 5448, 5366] dup_rows_per_dst=[2818, 2704, 2624, 2764, 2772, 2792, 2776, 2646] combine_rows_per_src=[5522, 5496, 5420, 5460, 5478, 5450, 5448, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229400 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156606488 ingress_bytes=194125824 link_bytes=194125824 token_bytes=312983576 weight_movement_bytes=37748736 total_nvlink_bytes=350732312 weight_to_token_ratio=0.120609 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860018130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38242 dup_rows=21760 dup_rows_remote=18928 dedup_ratio=0.332031 router_tokens_per_expert=[271, 263, 256, 247, 277, 262, 247, 257, 264, 247, 248, 250, 289, 269, 256, 245, 260, 265, 265, 264, 251, 237, 265, 252, 235, 250, 240, 282, 242, 239, 220, 254, 267, 267, 227, 276, 247, 283, 246, 260, 258, 257, 238, 248, 270, 270, 252, 240, 276, 248, 245, 246, 257, 250, 281, 245, 262, 247, 258, 246, 265, 248, 248, 267, 282, 238, 256, 252, 248, 266, 272, 226, 256, 254, 200, 268, 264, 266, 250, 267, 253, 241, 264, 275, 252, 260, 254, 266, 268, 276, 261, 254, 258, 250, 258, 251, 274, 259, 263, 257, 272, 269, 250, 252, 273, 246, 254, 255, 255, 266, 277, 261, 248, 246, 270, 263, 224, 234, 259, 248, 251, 245, 229, 278, 278, 236, 256, 250] payload_rows_per_dst=[5542, 5480, 5434, 5462, 5372, 5532, 5534, 5420] dup_rows_per_dst=[2754, 2562, 2778, 2716, 2758, 2750, 2832, 2610] combine_rows_per_src=[5542, 5480, 5434, 5462, 5372, 5532, 5534, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38142 dup_rows=21926 dup_rows_remote=19326 dedup_ratio=0.334564 router_tokens_per_expert=[253, 278, 264, 242, 241, 226, 260, 257, 256, 252, 265, 251, 262, 246, 257, 257, 255, 252, 235, 271, 250, 280, 267, 256, 277, 255, 235, 238, 245, 267, 256, 254, 252, 256, 261, 253, 279, 229, 244, 267, 231, 271, 243, 283, 245, 263, 255, 275, 237, 238, 240, 251, 294, 264, 256, 259, 279, 287, 265, 279, 273, 258, 243, 251, 224, 249, 262, 268, 260, 265, 269, 260, 228, 244, 247, 245, 257, 241, 268, 247, 253, 254, 240, 253, 255, 257, 245, 229, 257, 270, 248, 253, 258, 270, 270, 256, 278, 242, 258, 263, 242, 248, 253, 257, 261, 292, 283, 272, 286, 263, 255, 235, 251, 227, 253, 244, 276, 258, 242, 240, 259, 260, 263, 235, 253, 238, 257, 281] payload_rows_per_dst=[5430, 5394, 5490, 5496, 5468, 5458, 5510, 5364] dup_rows_per_dst=[2704, 2792, 2724, 2852, 2600, 2678, 2866, 2710] combine_rows_per_src=[5430, 5394, 5490, 5496, 5468, 5458, 5510, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38254 dup_rows=21694 dup_rows_remote=18848 dedup_ratio=0.331024 router_tokens_per_expert=[254, 257, 254, 244, 248, 222, 280, 234, 255, 267, 260, 258, 293, 280, 252, 293, 272, 269, 244, 251, 256, 259, 280, 246, 262, 271, 253, 273, 263, 279, 267, 271, 279, 257, 261, 261, 269, 258, 275, 281, 261, 258, 265, 259, 259, 260, 269, 245, 274, 262, 252, 260, 246, 250, 284, 256, 273, 234, 266, 245, 275, 267, 260, 231, 226, 260, 266, 235, 244, 230, 258, 267, 243, 264, 271, 236, 220, 221, 239, 262, 254, 238, 262, 238, 256, 251, 265, 252, 251, 246, 254, 246, 230, 265, 258, 225, 265, 268, 232, 249, 245, 243, 223, 255, 230, 260, 249, 255, 256, 251, 270, 239, 240, 285, 242, 235, 257, 278, 275, 257, 254, 265, 232, 279, 245, 284, 252, 246] payload_rows_per_dst=[5530, 5578, 5588, 5462, 5436, 5396, 5326, 5526] dup_rows_per_dst=[2772, 2854, 2846, 2808, 2448, 2586, 2654, 2726] combine_rows_per_src=[5530, 5578, 5588, 5462, 5436, 5396, 5326, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38280 dup_rows=21818 dup_rows_remote=19032 dedup_ratio=0.332916 router_tokens_per_expert=[252, 237, 260, 226, 227, 261, 236, 262, 238, 264, 254, 262, 255, 262, 267, 258, 278, 246, 255, 250, 233, 253, 266, 239, 273, 250, 246, 221, 264, 242, 247, 259, 264, 269, 275, 240, 255, 273, 260, 254, 300, 258, 252, 245, 250, 249, 278, 245, 276, 230, 275, 274, 262, 259, 255, 266, 262, 257, 260, 245, 231, 256, 259, 271, 266, 251, 274, 264, 251, 255, 259, 276, 252, 241, 281, 232, 246, 256, 272, 263, 307, 278, 234, 263, 264, 241, 283, 245, 264, 263, 227, 251, 234, 237, 253, 268, 236, 242, 247, 256, 243, 256, 244, 271, 229, 281, 242, 270, 227, 238, 265, 267, 256, 272, 246, 279, 250, 247, 258, 269, 244, 278, 244, 254, 263, 282, 252, 261] payload_rows_per_dst=[5422, 5408, 5486, 5486, 5542, 5438, 5372, 5564] dup_rows_per_dst=[2620, 2636, 2848, 2790, 2736, 2786, 2656, 2746] combine_rows_per_src=[5422, 5408, 5486, 5486, 5542, 5438, 5372, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157704192 dispatch_meta_bytes=230136 combine_payload_bytes=157704192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157934328 ingress_bytes=176578560 link_bytes=176578560 token_bytes=315638520 weight_movement_bytes=18874368 total_nvlink_bytes=334512888 weight_to_token_ratio=0.059797 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691888880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229344 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156467168 ingress_bytes=212860928 link_bytes=212860928 token_bytes=312704992 weight_movement_bytes=56623104 total_nvlink_bytes=369328096 weight_to_token_ratio=0.181075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039528678e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228680 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156867912 ingress_bytes=232136704 link_bytes=232136704 token_bytes=313507144 weight_movement_bytes=75497472 total_nvlink_bytes=389004616 weight_to_token_ratio=0.240816 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224219679e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229872 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156459504 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312689136 weight_movement_bytes=18874368 total_nvlink_bytes=331563504 weight_to_token_ratio=0.060361 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228408 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156916792 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605176 weight_movement_bytes=18874368 total_nvlink_bytes=332479544 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38218 dup_rows=21808 dup_rows_remote=19030 dedup_ratio=0.332764 router_tokens_per_expert=[226, 247, 259, 240, 252, 261, 249, 232, 260, 247, 256, 231, 270, 250, 246, 262, 245, 234, 257, 248, 249, 277, 269, 263, 265, 245, 232, 288, 236, 270, 238, 270, 219, 250, 258, 248, 248, 238, 229, 246, 271, 241, 270, 267, 241, 257, 258, 271, 273, 245, 253, 247, 276, 278, 263, 232, 255, 250, 276, 264, 246, 236, 259, 291, 235, 255, 267, 265, 259, 231, 263, 238, 239, 246, 264, 283, 250, 280, 277, 250, 254, 282, 249, 259, 240, 258, 286, 252, 256, 258, 275, 245, 270, 247, 281, 273, 268, 235, 282, 252, 276, 232, 245, 238, 253, 283, 259, 281, 258, 259, 263, 250, 293, 227, 270, 272, 218, 270, 263, 266, 246, 259, 265, 242, 240, 257, 257, 272] payload_rows_per_dst=[5386, 5462, 5350, 5510, 5492, 5500, 5552, 5476] dup_rows_per_dst=[2590, 2710, 2674, 2778, 2712, 2870, 2716, 2758] combine_rows_per_src=[5386, 5462, 5350, 5510, 5492, 5500, 5552, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229248 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157024128 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313819008 weight_movement_bytes=18874368 total_nvlink_bytes=332693376 weight_to_token_ratio=0.060144 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38074 dup_rows=21762 dup_rows_remote=18814 dedup_ratio=0.332062 router_tokens_per_expert=[272, 257, 265, 252, 229, 300, 243, 275, 240, 254, 253, 263, 258, 268, 263, 263, 270, 276, 274, 247, 278, 266, 242, 255, 237, 274, 232, 268, 257, 235, 275, 244, 267, 250, 263, 248, 253, 263, 251, 267, 261, 261, 266, 233, 269, 267, 238, 288, 264, 238, 244, 247, 261, 247, 258, 230, 249, 282, 280, 275, 234, 250, 253, 230, 262, 245, 236, 250, 255, 237, 269, 257, 246, 237, 250, 252, 256, 294, 233, 252, 232, 255, 254, 269, 219, 252, 274, 240, 241, 282, 250, 261, 259, 261, 267, 231, 265, 242, 215, 251, 246, 260, 245, 270, 258, 267, 269, 255, 239, 250, 282, 244, 258, 247, 275, 273, 248, 281, 255, 265, 247, 258, 275, 260, 267, 252, 243, 256] payload_rows_per_dst=[5546, 5552, 5484, 5278, 5328, 5402, 5484, 5700] dup_rows_per_dst=[2764, 2708, 2806, 2478, 2562, 2692, 2804, 2948] combine_rows_per_src=[5546, 5552, 5484, 5278, 5328, 5402, 5484, 5700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228992 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156769920 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313310848 weight_movement_bytes=37748736 total_nvlink_bytes=351059584 weight_to_token_ratio=0.120483 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=227552 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156178656 ingress_bytes=193699840 link_bytes=193699840 token_bytes=312129760 weight_movement_bytes=37748736 total_nvlink_bytes=349878496 weight_to_token_ratio=0.120939 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855936561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38428 dup_rows=21690 dup_rows_remote=19046 dedup_ratio=0.330963 router_tokens_per_expert=[269, 274, 258, 252, 244, 234, 236, 260, 279, 266, 250, 245, 270, 239, 244, 248, 232, 256, 259, 260, 270, 252, 229, 259, 261, 252, 244, 254, 244, 256, 245, 258, 258, 265, 264, 276, 255, 283, 254, 252, 268, 238, 268, 246, 255, 256, 246, 249, 237, 234, 276, 267, 260, 259, 268, 243, 262, 252, 274, 265, 284, 269, 279, 256, 271, 271, 203, 264, 244, 263, 239, 244, 236, 267, 247, 264, 268, 252, 271, 269, 240, 240, 236, 285, 281, 262, 260, 262, 268, 274, 228, 241, 275, 266, 255, 250, 261, 233, 233, 244, 267, 242, 250, 242, 257, 268, 263, 278, 268, 262, 261, 269, 249, 270, 265, 229, 234, 226, 265, 249, 247, 275, 270, 277, 270, 257, 250, 224] payload_rows_per_dst=[5538, 5418, 5548, 5546, 5496, 5492, 5454, 5354] dup_rows_per_dst=[2776, 2644, 2718, 2824, 2650, 2754, 2742, 2582] combine_rows_per_src=[5538, 5418, 5548, 5546, 5496, 5492, 5454, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=37986 dup_rows=21878 dup_rows_remote=18930 dedup_ratio=0.333832 router_tokens_per_expert=[247, 229, 265, 253, 244, 261, 254, 260, 257, 236, 287, 255, 239, 246, 279, 263, 241, 257, 249, 255, 234, 262, 264, 239, 232, 252, 270, 281, 213, 263, 230, 280, 248, 237, 242, 234, 247, 250, 295, 254, 255, 214, 251, 271, 253, 275, 269, 265, 267, 276, 254, 283, 256, 248, 238, 225, 258, 242, 273, 264, 269, 250, 274, 243, 248, 257, 242, 258, 243, 235, 238, 244, 269, 270, 262, 229, 254, 293, 243, 249, 248, 250, 263, 270, 244, 241, 275, 242, 271, 288, 271, 248, 261, 248, 217, 271, 282, 276, 269, 261, 278, 261, 256, 272, 243, 253, 267, 270, 281, 265, 259, 274, 279, 252, 223, 272, 248, 259, 251, 244, 237, 268, 266, 249, 242, 266, 258, 268] payload_rows_per_dst=[5672, 5618, 5198, 5230, 5416, 5514, 5586, 5424] dup_rows_per_dst=[2948, 2992, 2456, 2532, 2560, 2702, 2948, 2740] combine_rows_per_src=[5672, 5618, 5198, 5230, 5416, 5514, 5586, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38292 dup_rows=21844 dup_rows_remote=19120 dedup_ratio=0.333313 router_tokens_per_expert=[268, 268, 278, 240, 286, 226, 272, 258, 261, 293, 246, 224, 277, 236, 207, 256, 284, 290, 270, 247, 256, 236, 256, 269, 237, 244, 289, 248, 265, 237, 220, 261, 246, 250, 263, 255, 256, 251, 242, 255, 270, 251, 251, 249, 263, 258, 257, 257, 261, 261, 265, 288, 287, 241, 256, 282, 244, 241, 249, 274, 251, 250, 254, 299, 229, 269, 240, 241, 266, 276, 257, 265, 265, 242, 219, 252, 269, 274, 243, 247, 227, 282, 294, 270, 266, 266, 252, 275, 253, 282, 258, 229, 273, 261, 240, 251, 255, 246, 257, 240, 239, 257, 231, 294, 254, 249, 263, 259, 255, 258, 254, 251, 247, 256, 252, 236, 266, 237, 231, 249, 256, 237, 241, 247, 251, 258, 249, 278] payload_rows_per_dst=[5474, 5432, 5476, 5626, 5412, 5482, 5400, 5390] dup_rows_per_dst=[2718, 2786, 2672, 2780, 2696, 2876, 2724, 2592] combine_rows_per_src=[5474, 5432, 5476, 5626, 5412, 5482, 5400, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38138 dup_rows=21880 dup_rows_remote=19162 dedup_ratio=0.333862 router_tokens_per_expert=[260, 245, 264, 248, 234, 244, 232, 239, 276, 271, 246, 270, 250, 257, 225, 240, 272, 267, 246, 212, 266, 235, 251, 244, 260, 265, 238, 273, 252, 282, 259, 258, 255, 238, 231, 278, 263, 269, 251, 252, 241, 242, 237, 246, 254, 246, 268, 231, 266, 260, 274, 246, 248, 252, 235, 269, 298, 271, 233, 226, 245, 259, 226, 272, 254, 279, 265, 240, 248, 246, 260, 266, 250, 271, 273, 257, 260, 261, 252, 236, 239, 252, 269, 260, 260, 271, 265, 270, 285, 261, 228, 245, 248, 291, 272, 270, 258, 271, 253, 260, 277, 247, 245, 222, 256, 276, 243, 246, 216, 279, 253, 237, 245, 255, 272, 277, 245, 279, 272, 273, 258, 293, 271, 279, 274, 263, 241, 265] payload_rows_per_dst=[5384, 5370, 5408, 5450, 5518, 5524, 5418, 5584] dup_rows_per_dst=[2618, 2790, 2596, 2710, 2718, 2848, 2660, 2940] combine_rows_per_src=[5384, 5370, 5408, 5450, 5518, 5524, 5418, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38340 dup_rows=21776 dup_rows_remote=19134 dedup_ratio=0.332275 router_tokens_per_expert=[209, 257, 257, 259, 244, 283, 231, 264, 272, 263, 239, 255, 241, 257, 276, 254, 246, 266, 286, 254, 254, 249, 240, 276, 252, 233, 235, 240, 257, 271, 272, 275, 232, 245, 272, 225, 270, 253, 255, 259, 244, 273, 261, 257, 251, 251, 213, 270, 259, 285, 263, 251, 282, 252, 266, 270, 274, 252, 279, 237, 244, 253, 258, 253, 242, 241, 254, 254, 226, 237, 278, 254, 262, 256, 246, 240, 278, 267, 231, 303, 256, 278, 234, 260, 240, 263, 273, 230, 255, 248, 265, 286, 255, 261, 284, 247, 259, 263, 254, 240, 248, 246, 255, 222, 250, 250, 228, 280, 230, 269, 245, 305, 265, 246, 260, 255, 263, 288, 303, 256, 274, 225, 243, 257, 239, 235, 255, 280] payload_rows_per_dst=[5494, 5470, 5420, 5510, 5446, 5508, 5398, 5514] dup_rows_per_dst=[2628, 2742, 2642, 2846, 2692, 2762, 2690, 2774] combine_rows_per_src=[5494, 5470, 5420, 5510, 5446, 5508, 5398, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38446 dup_rows=21644 dup_rows_remote=19034 dedup_ratio=0.330261 router_tokens_per_expert=[253, 261, 263, 279, 264, 232, 245, 274, 268, 240, 270, 235, 257, 244, 262, 266, 248, 273, 247, 232, 241, 248, 258, 249, 278, 257, 241, 275, 258, 250, 248, 244, 259, 263, 235, 262, 261, 258, 261, 262, 249, 249, 262, 283, 259, 276, 261, 292, 259, 232, 236, 266, 246, 266, 254, 256, 273, 235, 254, 260, 257, 265, 241, 290, 270, 263, 255, 270, 237, 255, 268, 218, 267, 263, 296, 264, 250, 227, 262, 241, 253, 254, 248, 258, 246, 278, 248, 240, 230, 255, 268, 235, 244, 254, 264, 253, 246, 252, 283, 282, 246, 260, 259, 263, 263, 253, 288, 242, 236, 251, 257, 259, 236, 250, 271, 238, 263, 260, 236, 247, 252, 247, 261, 264, 263, 264, 261, 239] payload_rows_per_dst=[5446, 5434, 5536, 5516, 5520, 5446, 5536, 5458] dup_rows_per_dst=[2780, 2660, 2848, 2664, 2692, 2610, 2744, 2646] combine_rows_per_src=[5446, 5434, 5536, 5516, 5520, 5446, 5536, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229896 combine_payload_bytes=157401088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157630984 ingress_bytes=176275456 link_bytes=176275456 token_bytes=315032072 weight_movement_bytes=18874368 total_nvlink_bytes=333906440 weight_to_token_ratio=0.059913 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688984686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=227664 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155818320 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311408976 weight_movement_bytes=18874368 total_nvlink_bytes=330283344 weight_to_token_ratio=0.060610 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229648 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073680 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917712 weight_movement_bytes=18874368 total_nvlink_bytes=332792080 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229200 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156442448 ingress_bytes=212836352 link_bytes=212836352 token_bytes=312655696 weight_movement_bytes=56623104 total_nvlink_bytes=369278800 weight_to_token_ratio=0.181104 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039293203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38286 dup_rows=21814 dup_rows_remote=19012 dedup_ratio=0.332855 router_tokens_per_expert=[294, 247, 253, 248, 266, 251, 277, 241, 265, 242, 248, 261, 274, 238, 258, 261, 226, 252, 257, 252, 262, 255, 268, 242, 250, 237, 248, 244, 287, 255, 263, 261, 270, 247, 260, 241, 238, 294, 243, 278, 253, 222, 281, 245, 260, 237, 238, 251, 271, 261, 273, 289, 268, 260, 239, 231, 267, 263, 263, 259, 261, 263, 223, 228, 263, 264, 276, 237, 227, 247, 245, 230, 284, 268, 273, 258, 242, 246, 248, 254, 254, 245, 256, 240, 245, 287, 272, 254, 263, 259, 261, 283, 269, 292, 268, 243, 272, 221, 237, 251, 234, 242, 270, 267, 257, 261, 236, 252, 264, 262, 237, 234, 244, 227, 295, 289, 253, 229, 251, 257, 264, 267, 267, 267, 246, 265, 257, 280] payload_rows_per_dst=[5540, 5402, 5340, 5436, 5434, 5584, 5424, 5562] dup_rows_per_dst=[2708, 2716, 2776, 2802, 2690, 2798, 2570, 2754] combine_rows_per_src=[5540, 5402, 5340, 5436, 5434, 5584, 5424, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229896 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270536 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311176 weight_movement_bytes=18874368 total_nvlink_bytes=333185544 weight_to_token_ratio=0.060050 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229920 combine_payload_bytes=157474816 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157704736 ingress_bytes=214097920 link_bytes=214097920 token_bytes=315179552 weight_movement_bytes=56623104 total_nvlink_bytes=371802656 weight_to_token_ratio=0.179653 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051380927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38126 dup_rows=21836 dup_rows_remote=18938 dedup_ratio=0.333191 router_tokens_per_expert=[248, 238, 241, 240, 245, 241, 238, 267, 260, 269, 245, 245, 264, 265, 264, 271, 271, 230, 253, 250, 237, 253, 239, 250, 254, 249, 282, 261, 262, 259, 250, 251, 275, 256, 252, 243, 241, 248, 265, 251, 250, 239, 266, 278, 221, 268, 239, 271, 250, 255, 229, 244, 260, 267, 248, 275, 266, 266, 264, 248, 289, 236, 257, 267, 252, 237, 248, 274, 281, 259, 242, 238, 285, 267, 259, 244, 291, 247, 249, 257, 237, 254, 273, 255, 240, 241, 239, 250, 252, 257, 282, 247, 257, 278, 257, 252, 249, 263, 274, 274, 287, 266, 265, 262, 241, 262, 257, 263, 264, 265, 239, 275, 241, 262, 238, 264, 249, 248, 263, 249, 249, 282, 253, 269, 252, 250, 242, 274] payload_rows_per_dst=[5298, 5470, 5422, 5396, 5478, 5414, 5648, 5574] dup_rows_per_dst=[2482, 2632, 2704, 2604, 2762, 2748, 3006, 2898] combine_rows_per_src=[5298, 5470, 5422, 5396, 5478, 5414, 5648, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229192 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157048648 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868104 weight_movement_bytes=37748736 total_nvlink_bytes=351616840 weight_to_token_ratio=0.120269 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228256 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156392352 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312556448 weight_movement_bytes=37748736 total_nvlink_bytes=350305184 weight_to_token_ratio=0.120774 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38300 dup_rows=21792 dup_rows_remote=19098 dedup_ratio=0.332520 router_tokens_per_expert=[258, 240, 239, 264, 264, 257, 267, 252, 302, 245, 230, 266, 271, 273, 230, 260, 247, 249, 250, 251, 269, 250, 263, 241, 266, 279, 258, 236, 254, 257, 245, 244, 273, 224, 245, 255, 244, 261, 257, 262, 300, 244, 277, 238, 245, 260, 218, 284, 265, 256, 266, 283, 252, 266, 243, 221, 238, 266, 261, 276, 238, 263, 268, 246, 248, 242, 274, 262, 247, 258, 281, 286, 226, 255, 274, 243, 266, 243, 238, 266, 288, 255, 252, 250, 230, 260, 241, 244, 234, 261, 237, 267, 225, 243, 257, 262, 270, 263, 247, 269, 242, 248, 289, 251, 272, 256, 269, 244, 240, 274, 245, 228, 267, 268, 260, 249, 251, 261, 288, 253, 251, 245, 276, 258, 250, 257, 259, 281] payload_rows_per_dst=[5488, 5444, 5472, 5442, 5478, 5462, 5512, 5446] dup_rows_per_dst=[2748, 2694, 2810, 2774, 2740, 2550, 2682, 2794] combine_rows_per_src=[5488, 5444, 5472, 5442, 5478, 5462, 5512, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38314 dup_rows=21758 dup_rows_remote=19018 dedup_ratio=0.332001 router_tokens_per_expert=[253, 255, 264, 274, 259, 261, 258, 252, 275, 248, 262, 266, 243, 227, 258, 262, 257, 278, 272, 247, 276, 256, 284, 259, 262, 258, 250, 265, 256, 270, 255, 269, 236, 214, 255, 262, 234, 253, 253, 234, 264, 260, 254, 244, 269, 273, 248, 255, 248, 262, 280, 254, 254, 269, 264, 259, 259, 229, 243, 268, 248, 235, 255, 278, 260, 271, 263, 234, 251, 264, 218, 223, 220, 236, 258, 230, 253, 239, 270, 286, 231, 253, 263, 263, 254, 250, 270, 248, 269, 246, 245, 256, 272, 258, 260, 250, 216, 248, 262, 251, 257, 282, 257, 276, 268, 262, 263, 257, 264, 247, 242, 250, 271, 266, 254, 280, 255, 259, 269, 248, 270, 260, 268, 238, 235, 259, 268, 258] payload_rows_per_dst=[5588, 5540, 5422, 5450, 5378, 5452, 5464, 5484] dup_rows_per_dst=[2646, 2888, 2594, 2760, 2574, 2724, 2740, 2832] combine_rows_per_src=[5588, 5540, 5422, 5450, 5378, 5452, 5464, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38338 dup_rows=21686 dup_rows_remote=18986 dedup_ratio=0.330902 router_tokens_per_expert=[251, 263, 239, 252, 234, 313, 280, 259, 262, 263, 270, 245, 241, 256, 248, 242, 274, 245, 250, 229, 250, 227, 276, 251, 257, 231, 262, 275, 242, 244, 275, 234, 246, 228, 254, 282, 276, 236, 241, 285, 267, 256, 260, 260, 229, 279, 257, 268, 266, 248, 271, 230, 244, 249, 243, 270, 256, 243, 254, 263, 245, 272, 257, 261, 273, 239, 274, 236, 235, 273, 255, 242, 266, 259, 250, 265, 268, 230, 275, 266, 288, 249, 271, 242, 267, 287, 266, 228, 282, 257, 291, 221, 254, 259, 254, 246, 254, 262, 259, 266, 254, 259, 249, 278, 266, 255, 249, 240, 296, 251, 267, 261, 241, 258, 254, 267, 251, 246, 241, 247, 255, 233, 260, 256, 233, 263, 243, 250] payload_rows_per_dst=[5500, 5408, 5478, 5472, 5512, 5570, 5536, 5374] dup_rows_per_dst=[2736, 2636, 2770, 2672, 2700, 2754, 2796, 2622] combine_rows_per_src=[5500, 5408, 5478, 5472, 5512, 5570, 5536, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44042 payload_rows_remote=38590 dup_rows=21494 dup_rows_remote=18932 dedup_ratio=0.327972 router_tokens_per_expert=[241, 233, 284, 258, 251, 245, 236, 241, 238, 252, 260, 268, 253, 231, 259, 257, 266, 203, 253, 263, 239, 248, 263, 284, 250, 272, 268, 274, 254, 266, 258, 237, 266, 256, 260, 265, 240, 267, 274, 236, 267, 264, 254, 226, 233, 251, 285, 257, 235, 266, 223, 252, 260, 258, 262, 262, 231, 243, 241, 254, 225, 272, 257, 272, 251, 272, 270, 245, 268, 256, 270, 272, 262, 293, 229, 242, 259, 266, 275, 247, 275, 264, 280, 263, 256, 287, 264, 252, 234, 264, 266, 227, 224, 256, 250, 236, 243, 290, 230, 255, 250, 249, 248, 272, 244, 271, 247, 252, 263, 240, 265, 252, 251, 292, 249, 251, 276, 276, 270, 250, 268, 266, 273, 257, 251, 269, 266, 238] payload_rows_per_dst=[5452, 5468, 5480, 5498, 5538, 5628, 5482, 5496] dup_rows_per_dst=[2562, 2728, 2722, 2528, 2816, 2850, 2660, 2628] combine_rows_per_src=[5452, 5468, 5480, 5498, 5538, 5628, 5482, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38340 dup_rows=21762 dup_rows_remote=19114 dedup_ratio=0.332062 router_tokens_per_expert=[266, 262, 274, 244, 250, 256, 245, 238, 269, 228, 257, 265, 257, 254, 259, 277, 241, 259, 246, 257, 239, 242, 252, 261, 278, 279, 234, 248, 241, 238, 275, 257, 263, 236, 243, 244, 244, 237, 265, 260, 244, 273, 271, 232, 258, 252, 262, 257, 243, 274, 258, 268, 269, 243, 268, 279, 238, 272, 257, 268, 269, 237, 242, 249, 246, 260, 264, 264, 238, 265, 287, 257, 240, 259, 272, 262, 250, 252, 266, 265, 242, 234, 245, 256, 240, 226, 252, 241, 283, 235, 265, 269, 275, 279, 272, 256, 276, 267, 248, 251, 257, 260, 257, 258, 274, 240, 260, 255, 267, 234, 254, 250, 250, 248, 271, 298, 239, 257, 266, 263, 246, 268, 256, 279, 227, 250, 249, 253] payload_rows_per_dst=[5466, 5392, 5434, 5528, 5572, 5452, 5498, 5432] dup_rows_per_dst=[2736, 2702, 2648, 2740, 2722, 2688, 2718, 2808] combine_rows_per_src=[5466, 5392, 5434, 5528, 5572, 5452, 5498, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38190 dup_rows=21864 dup_rows_remote=19146 dedup_ratio=0.333618 router_tokens_per_expert=[257, 238, 253, 262, 261, 249, 283, 246, 258, 254, 259, 237, 276, 291, 253, 261, 223, 268, 247, 256, 259, 236, 250, 269, 291, 238, 282, 269, 237, 255, 248, 265, 271, 219, 269, 238, 246, 230, 251, 258, 235, 238, 242, 235, 281, 239, 275, 252, 245, 246, 251, 274, 265, 259, 255, 260, 266, 274, 263, 261, 256, 257, 241, 221, 258, 247, 251, 262, 285, 267, 268, 248, 268, 262, 273, 242, 221, 264, 250, 230, 257, 269, 246, 286, 273, 289, 235, 267, 254, 256, 234, 266, 247, 236, 250, 235, 230, 263, 241, 245, 248, 270, 242, 262, 255, 254, 246, 272, 266, 291, 249, 253, 255, 272, 277, 274, 264, 284, 267, 274, 242, 262, 250, 276, 240, 235, 260, 249] payload_rows_per_dst=[5484, 5434, 5304, 5522, 5466, 5482, 5444, 5536] dup_rows_per_dst=[2792, 2752, 2654, 2666, 2726, 2718, 2730, 2826] combine_rows_per_src=[5484, 5434, 5304, 5522, 5466, 5482, 5444, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229592 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106392 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983192 weight_movement_bytes=18874368 total_nvlink_bytes=332857560 weight_to_token_ratio=0.060113 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229328 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163472 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314097616 weight_movement_bytes=18874368 total_nvlink_bytes=332971984 weight_to_token_ratio=0.060091 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229296 combine_payload_bytes=157032448 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157261744 ingress_bytes=194781184 link_bytes=194781184 token_bytes=314294192 weight_movement_bytes=37748736 total_nvlink_bytes=352042928 weight_to_token_ratio=0.120106 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866297467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229816 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270456 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311096 weight_movement_bytes=18874368 total_nvlink_bytes=333185464 weight_to_token_ratio=0.060050 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158064640 dispatch_meta_bytes=230088 combine_payload_bytes=158064640 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=158294728 ingress_bytes=214687744 link_bytes=214687744 token_bytes=316359368 weight_movement_bytes=56623104 total_nvlink_bytes=372982472 weight_to_token_ratio=0.178983 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.057032330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38280 dup_rows=21770 dup_rows_remote=19070 dedup_ratio=0.332184 router_tokens_per_expert=[253, 257, 246, 246, 239, 274, 261, 285, 254, 253, 251, 259, 266, 241, 241, 246, 245, 246, 234, 257, 266, 274, 252, 262, 240, 261, 245, 261, 286, 234, 287, 265, 257, 254, 261, 285, 237, 240, 236, 274, 261, 266, 263, 257, 250, 250, 239, 242, 244, 236, 265, 260, 250, 244, 271, 287, 267, 225, 263, 260, 248, 272, 239, 262, 261, 263, 246, 263, 264, 266, 245, 257, 264, 257, 254, 262, 244, 240, 240, 258, 231, 265, 268, 257, 221, 247, 247, 262, 252, 254, 269, 237, 212, 269, 276, 306, 245, 253, 238, 241, 244, 274, 276, 227, 268, 282, 267, 252, 261, 237, 248, 244, 263, 247, 248, 250, 245, 268, 250, 257, 282, 260, 284, 279, 259, 266, 266, 278] payload_rows_per_dst=[5456, 5482, 5432, 5486, 5478, 5442, 5448, 5542] dup_rows_per_dst=[2688, 2748, 2712, 2700, 2690, 2704, 2666, 2862] combine_rows_per_src=[5456, 5482, 5432, 5486, 5478, 5442, 5448, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229344 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655584 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081824 weight_movement_bytes=37748736 total_nvlink_bytes=350830560 weight_to_token_ratio=0.120571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38088 dup_rows=21840 dup_rows_remote=18958 dedup_ratio=0.333252 router_tokens_per_expert=[280, 256, 263, 238, 259, 253, 243, 270, 268, 291, 262, 256, 264, 276, 239, 263, 283, 253, 246, 258, 239, 243, 270, 258, 242, 228, 247, 255, 254, 252, 239, 231, 250, 271, 264, 242, 247, 249, 276, 234, 268, 284, 266, 263, 268, 249, 276, 252, 257, 276, 233, 260, 258, 269, 254, 262, 233, 224, 264, 257, 253, 248, 264, 252, 227, 263, 257, 269, 240, 230, 252, 253, 266, 238, 279, 267, 233, 250, 259, 268, 257, 276, 261, 262, 258, 248, 251, 260, 221, 232, 258, 242, 269, 229, 259, 251, 278, 227, 246, 243, 268, 253, 238, 263, 247, 264, 243, 291, 258, 248, 274, 258, 294, 264, 238, 252, 263, 293, 252, 272, 226, 263, 267, 257, 248, 278, 259, 256] payload_rows_per_dst=[5478, 5356, 5402, 5482, 5498, 5404, 5468, 5608] dup_rows_per_dst=[2884, 2640, 2712, 2646, 2682, 2602, 2792, 2882] combine_rows_per_src=[5478, 5356, 5402, 5482, 5498, 5404, 5468, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229400 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157024280 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313819160 weight_movement_bytes=37748736 total_nvlink_bytes=351567896 weight_to_token_ratio=0.120288 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228184 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156236632 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312245080 weight_movement_bytes=18874368 total_nvlink_bytes=331119448 weight_to_token_ratio=0.060447 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38090 dup_rows=21960 dup_rows_remote=19192 dedup_ratio=0.335083 router_tokens_per_expert=[249, 270, 246, 249, 264, 245, 251, 212, 274, 253, 255, 284, 258, 256, 273, 251, 274, 219, 253, 266, 252, 270, 265, 262, 284, 238, 238, 252, 255, 267, 268, 264, 252, 242, 262, 267, 230, 269, 257, 274, 262, 265, 233, 258, 264, 236, 271, 245, 245, 256, 259, 261, 271, 255, 234, 252, 238, 267, 285, 277, 259, 236, 277, 259, 284, 276, 275, 253, 272, 229, 244, 258, 261, 234, 222, 275, 245, 224, 255, 295, 259, 241, 257, 255, 256, 265, 272, 264, 268, 232, 251, 272, 250, 248, 251, 248, 229, 274, 227, 260, 275, 250, 269, 231, 221, 242, 253, 253, 254, 289, 267, 264, 255, 247, 251, 270, 260, 256, 264, 246, 259, 244, 258, 265, 243, 263, 245, 258] payload_rows_per_dst=[5502, 5486, 5402, 5520, 5362, 5454, 5412, 5438] dup_rows_per_dst=[2678, 2768, 2772, 2742, 2842, 2724, 2704, 2730] combine_rows_per_src=[5502, 5486, 5402, 5520, 5362, 5454, 5412, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38252 dup_rows=21760 dup_rows_remote=19008 dedup_ratio=0.332031 router_tokens_per_expert=[258, 256, 244, 259, 243, 259, 272, 257, 274, 228, 241, 224, 273, 249, 274, 253, 249, 273, 254, 272, 253, 257, 255, 252, 246, 229, 240, 276, 259, 236, 275, 265, 256, 257, 250, 269, 236, 256, 278, 273, 271, 250, 262, 223, 251, 257, 274, 271, 216, 244, 245, 292, 269, 242, 245, 262, 265, 231, 237, 266, 275, 271, 246, 238, 266, 260, 266, 262, 257, 276, 249, 277, 259, 263, 254, 269, 254, 270, 264, 237, 246, 252, 251, 247, 265, 236, 266, 256, 283, 266, 254, 268, 255, 228, 260, 240, 276, 254, 276, 268, 273, 242, 254, 255, 238, 269, 255, 233, 254, 271, 273, 247, 236, 272, 250, 255, 239, 276, 250, 271, 251, 263, 229, 274, 223, 246, 267, 239] payload_rows_per_dst=[5400, 5456, 5504, 5440, 5556, 5420, 5524, 5476] dup_rows_per_dst=[2728, 2726, 2764, 2648, 2810, 2726, 2752, 2606] combine_rows_per_src=[5400, 5456, 5504, 5440, 5556, 5420, 5524, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38138 dup_rows=21900 dup_rows_remote=19190 dedup_ratio=0.334167 router_tokens_per_expert=[232, 263, 244, 286, 267, 253, 239, 236, 254, 279, 259, 242, 259, 275, 239, 259, 281, 274, 255, 252, 234, 265, 251, 249, 240, 244, 238, 225, 271, 281, 251, 247, 253, 255, 269, 249, 221, 245, 215, 293, 228, 264, 263, 253, 245, 262, 288, 276, 255, 262, 247, 249, 258, 275, 247, 261, 288, 265, 261, 274, 236, 239, 266, 281, 266, 273, 271, 268, 256, 264, 285, 259, 236, 239, 258, 225, 256, 250, 247, 251, 256, 265, 245, 256, 272, 267, 241, 277, 263, 255, 253, 260, 251, 238, 281, 252, 263, 246, 243, 238, 283, 236, 263, 262, 243, 236, 266, 240, 276, 259, 276, 274, 263, 242, 277, 238, 248, 304, 273, 263, 242, 244, 214, 258, 224, 247, 252, 252] payload_rows_per_dst=[5430, 5366, 5436, 5512, 5498, 5532, 5466, 5396] dup_rows_per_dst=[2742, 2750, 2722, 2816, 2710, 2732, 2742, 2686] combine_rows_per_src=[5430, 5366, 5436, 5512, 5498, 5532, 5466, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38286 dup_rows=21750 dup_rows_remote=19058 dedup_ratio=0.331879 router_tokens_per_expert=[214, 243, 244, 265, 238, 266, 244, 272, 245, 226, 264, 209, 240, 248, 252, 269, 268, 244, 244, 240, 289, 262, 247, 266, 262, 241, 251, 260, 237, 269, 241, 272, 269, 253, 251, 261, 259, 241, 257, 281, 243, 256, 258, 266, 250, 267, 239, 245, 269, 278, 247, 265, 241, 267, 288, 282, 265, 274, 235, 265, 275, 262, 248, 259, 246, 246, 274, 246, 259, 291, 229, 270, 236, 282, 245, 266, 265, 284, 239, 258, 277, 267, 252, 259, 245, 254, 267, 283, 239, 254, 274, 235, 263, 256, 240, 279, 263, 290, 233, 268, 260, 269, 253, 267, 269, 227, 273, 256, 275, 258, 280, 277, 253, 257, 268, 254, 234, 253, 247, 223, 260, 231, 239, 228, 242, 245, 250, 238] payload_rows_per_dst=[5360, 5502, 5500, 5556, 5542, 5434, 5556, 5336] dup_rows_per_dst=[2518, 2684, 2692, 2884, 2730, 2854, 2880, 2508] combine_rows_per_src=[5360, 5502, 5500, 5556, 5542, 5434, 5556, 5336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38092 dup_rows=21832 dup_rows_remote=18986 dedup_ratio=0.333130 router_tokens_per_expert=[227, 245, 214, 219, 287, 249, 265, 258, 225, 243, 257, 243, 256, 247, 270, 267, 271, 285, 228, 247, 260, 258, 253, 258, 274, 257, 269, 254, 260, 245, 247, 260, 262, 260, 249, 263, 227, 248, 250, 248, 254, 254, 257, 284, 264, 243, 255, 244, 239, 266, 254, 252, 244, 268, 280, 251, 235, 252, 259, 242, 280, 271, 283, 276, 267, 260, 267, 250, 257, 278, 237, 277, 256, 264, 281, 229, 266, 283, 278, 269, 253, 275, 255, 249, 235, 250, 270, 283, 247, 242, 271, 253, 232, 242, 287, 245, 269, 233, 263, 245, 230, 241, 237, 244, 239, 264, 250, 248, 261, 254, 269, 284, 220, 263, 236, 247, 271, 257, 279, 254, 262, 257, 272, 274, 271, 241, 261, 252] payload_rows_per_dst=[5612, 5554, 5456, 5576, 5492, 5356, 5410, 5248] dup_rows_per_dst=[2846, 2840, 2782, 2762, 2852, 2626, 2652, 2472] combine_rows_per_src=[5612, 5554, 5456, 5576, 5492, 5356, 5410, 5248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38294 dup_rows=21684 dup_rows_remote=18950 dedup_ratio=0.330872 router_tokens_per_expert=[251, 264, 266, 246, 275, 258, 231, 255, 251, 276, 249, 249, 255, 233, 258, 281, 278, 253, 249, 265, 289, 283, 219, 255, 250, 273, 259, 263, 235, 254, 247, 246, 256, 258, 233, 252, 256, 259, 258, 259, 243, 229, 287, 265, 254, 255, 246, 261, 266, 248, 239, 259, 267, 272, 237, 269, 239, 202, 264, 263, 241, 279, 257, 257, 287, 239, 248, 272, 264, 265, 265, 242, 243, 232, 244, 245, 244, 271, 259, 263, 250, 260, 258, 255, 274, 270, 271, 256, 262, 247, 248, 248, 266, 269, 260, 252, 252, 271, 241, 269, 262, 279, 248, 268, 236, 250, 243, 239, 270, 268, 252, 240, 240, 248, 250, 274, 259, 259, 240, 262, 250, 276, 253, 245, 265, 273, 265, 246] payload_rows_per_dst=[5516, 5438, 5448, 5454, 5470, 5558, 5496, 5472] dup_rows_per_dst=[2680, 2798, 2694, 2664, 2696, 2734, 2680, 2738] combine_rows_per_src=[5516, 5438, 5448, 5454, 5470, 5558, 5496, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=229128 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156245768 ingress_bytes=174891008 link_bytes=174891008 token_bytes=312262408 weight_movement_bytes=18874368 total_nvlink_bytes=331136776 weight_to_token_ratio=0.060444 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675719587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229040 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909232 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589424 weight_movement_bytes=18874368 total_nvlink_bytes=332463792 weight_to_token_ratio=0.060188 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229312 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156442560 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312655808 weight_movement_bytes=18874368 total_nvlink_bytes=331530176 weight_to_token_ratio=0.060368 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38380 dup_rows=21746 dup_rows_remote=19120 dedup_ratio=0.331818 router_tokens_per_expert=[269, 252, 270, 233, 277, 241, 252, 229, 251, 236, 263, 261, 248, 239, 251, 255, 229, 260, 241, 224, 233, 261, 266, 237, 256, 246, 274, 258, 235, 265, 247, 258, 250, 236, 253, 254, 276, 266, 267, 259, 255, 267, 248, 263, 248, 267, 247, 256, 251, 246, 251, 264, 245, 257, 233, 223, 262, 265, 273, 266, 241, 265, 235, 241, 280, 241, 260, 240, 270, 253, 267, 219, 268, 281, 240, 263, 263, 258, 252, 249, 281, 252, 251, 268, 264, 256, 267, 271, 256, 271, 255, 276, 307, 265, 244, 270, 284, 267, 273, 219, 268, 256, 239, 265, 224, 277, 284, 268, 256, 250, 254, 251, 263, 275, 255, 250, 252, 265, 258, 250, 257, 266, 256, 243, 271, 259, 256, 252] payload_rows_per_dst=[5368, 5330, 5480, 5410, 5488, 5620, 5516, 5578] dup_rows_per_dst=[2686, 2650, 2744, 2626, 2720, 2888, 2754, 2678] combine_rows_per_src=[5368, 5330, 5480, 5410, 5488, 5620, 5516, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229376 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157048832 ingress_bytes=232316928 link_bytes=232316928 token_bytes=313868288 weight_movement_bytes=75497472 total_nvlink_bytes=389365760 weight_to_token_ratio=0.240539 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225946497e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228312 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253144 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312277976 weight_movement_bytes=18874368 total_nvlink_bytes=331152344 weight_to_token_ratio=0.060441 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38222 dup_rows=21786 dup_rows_remote=18808 dedup_ratio=0.332428 router_tokens_per_expert=[235, 260, 254, 277, 276, 281, 245, 262, 251, 256, 253, 243, 256, 263, 294, 237, 240, 253, 265, 239, 246, 237, 238, 247, 264, 239, 259, 270, 249, 266, 243, 243, 248, 262, 234, 264, 255, 248, 243, 252, 265, 270, 256, 253, 272, 271, 250, 254, 252, 247, 264, 269, 245, 239, 281, 261, 246, 251, 240, 216, 265, 251, 260, 274, 251, 234, 255, 263, 254, 261, 243, 243, 279, 245, 254, 269, 238, 261, 245, 235, 285, 245, 262, 249, 243, 261, 271, 257, 269, 250, 253, 273, 263, 225, 268, 251, 256, 231, 240, 262, 257, 243, 272, 263, 263, 210, 262, 268, 277, 263, 265, 256, 252, 260, 286, 255, 268, 233, 251, 253, 263, 282, 275, 272, 242, 267, 273, 294] payload_rows_per_dst=[5526, 5426, 5400, 5484, 5456, 5470, 5460, 5528] dup_rows_per_dst=[2760, 2570, 2794, 2638, 2604, 2722, 2720, 2978] combine_rows_per_src=[5526, 5426, 5400, 5484, 5456, 5470, 5460, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228976 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157081200 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313933424 weight_movement_bytes=56623104 total_nvlink_bytes=370556528 weight_to_token_ratio=0.180367 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=230000 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434480 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638960 weight_movement_bytes=18874368 total_nvlink_bytes=333513328 weight_to_token_ratio=0.059987 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=228120 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156785432 ingress_bytes=213180416 link_bytes=213180416 token_bytes=313342744 weight_movement_bytes=56623104 total_nvlink_bytes=369965848 weight_to_token_ratio=0.180707 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042589855e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38258 dup_rows=21840 dup_rows_remote=19180 dedup_ratio=0.333252 router_tokens_per_expert=[275, 260, 244, 259, 246, 252, 291, 245, 252, 251, 258, 262, 252, 245, 268, 266, 255, 245, 261, 246, 270, 240, 254, 254, 273, 246, 259, 260, 209, 252, 259, 266, 264, 253, 271, 246, 260, 246, 242, 268, 224, 278, 266, 261, 235, 279, 267, 249, 260, 242, 251, 288, 265, 237, 254, 276, 233, 263, 244, 270, 238, 290, 273, 241, 266, 226, 259, 248, 264, 262, 227, 256, 270, 278, 258, 259, 245, 252, 262, 238, 249, 242, 270, 276, 248, 245, 239, 240, 283, 275, 236, 284, 220, 240, 247, 251, 239, 247, 249, 285, 253, 255, 251, 247, 262, 253, 286, 231, 259, 254, 269, 270, 273, 275, 261, 262, 260, 233, 254, 281, 265, 250, 265, 266, 240, 235, 257, 257] payload_rows_per_dst=[5452, 5438, 5374, 5502, 5502, 5474, 5458, 5496] dup_rows_per_dst=[2800, 2660, 2844, 2748, 2656, 2616, 2762, 2754] combine_rows_per_src=[5452, 5438, 5374, 5502, 5502, 5474, 5458, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38104 dup_rows=21948 dup_rows_remote=19076 dedup_ratio=0.334900 router_tokens_per_expert=[238, 228, 253, 250, 272, 284, 263, 256, 249, 241, 235, 271, 245, 273, 227, 270, 284, 263, 227, 215, 242, 235, 251, 267, 270, 250, 257, 240, 261, 286, 245, 270, 237, 250, 250, 275, 269, 262, 262, 254, 252, 252, 266, 260, 221, 221, 218, 246, 262, 246, 273, 235, 253, 274, 267, 269, 270, 258, 258, 251, 262, 265, 261, 274, 261, 264, 266, 236, 258, 244, 270, 249, 279, 273, 223, 268, 227, 252, 246, 274, 242, 287, 228, 247, 301, 281, 238, 243, 267, 255, 270, 255, 246, 229, 274, 264, 265, 245, 248, 267, 244, 247, 287, 266, 262, 257, 267, 290, 267, 246, 257, 263, 254, 288, 224, 258, 250, 250, 234, 274, 269, 242, 259, 254, 241, 261, 243, 281] payload_rows_per_dst=[5388, 5446, 5406, 5536, 5474, 5396, 5484, 5458] dup_rows_per_dst=[2722, 2680, 2584, 2820, 2706, 2858, 2872, 2706] combine_rows_per_src=[5388, 5446, 5406, 5536, 5474, 5396, 5484, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38374 dup_rows=21730 dup_rows_remote=19060 dedup_ratio=0.331573 router_tokens_per_expert=[261, 256, 251, 276, 266, 277, 255, 255, 251, 275, 264, 273, 253, 249, 250, 232, 240, 264, 265, 266, 267, 270, 276, 227, 263, 277, 233, 269, 281, 254, 240, 244, 229, 236, 241, 242, 263, 227, 251, 244, 295, 243, 232, 276, 269, 260, 247, 269, 238, 242, 237, 273, 277, 299, 252, 246, 246, 260, 257, 261, 237, 244, 264, 273, 282, 239, 245, 250, 236, 249, 265, 246, 231, 263, 240, 247, 296, 238, 261, 263, 250, 255, 255, 255, 263, 240, 256, 268, 230, 244, 284, 249, 261, 258, 265, 266, 244, 241, 243, 266, 236, 239, 251, 277, 270, 235, 269, 270, 225, 261, 257, 263, 262, 276, 265, 267, 270, 251, 256, 247, 271, 255, 259, 298, 265, 243, 246, 230] payload_rows_per_dst=[5498, 5516, 5470, 5430, 5432, 5460, 5458, 5542] dup_rows_per_dst=[2790, 2756, 2578, 2782, 2670, 2738, 2636, 2780] combine_rows_per_src=[5498, 5516, 5470, 5430, 5432, 5460, 5458, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38432 dup_rows=21654 dup_rows_remote=18988 dedup_ratio=0.330414 router_tokens_per_expert=[264, 242, 255, 252, 250, 265, 243, 268, 236, 246, 260, 273, 272, 272, 269, 241, 272, 242, 240, 263, 263, 289, 269, 236, 250, 254, 276, 254, 265, 239, 241, 256, 251, 248, 269, 272, 238, 241, 289, 260, 237, 253, 238, 244, 260, 241, 268, 249, 244, 255, 258, 262, 260, 248, 261, 260, 273, 263, 255, 252, 280, 271, 250, 223, 237, 268, 249, 260, 275, 255, 254, 253, 244, 248, 234, 235, 249, 270, 247, 272, 264, 261, 260, 291, 252, 259, 256, 282, 251, 263, 266, 272, 247, 269, 243, 272, 249, 252, 240, 245, 275, 272, 246, 262, 235, 272, 242, 257, 270, 271, 251, 243, 268, 240, 246, 251, 244, 239, 272, 249, 252, 227, 293, 255, 249, 224, 273, 256] payload_rows_per_dst=[5510, 5478, 5450, 5464, 5448, 5602, 5506, 5424] dup_rows_per_dst=[2706, 2740, 2666, 2766, 2652, 2814, 2658, 2652] combine_rows_per_src=[5510, 5478, 5450, 5464, 5448, 5602, 5506, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38208 dup_rows=21714 dup_rows_remote=18880 dedup_ratio=0.331329 router_tokens_per_expert=[259, 250, 253, 253, 241, 257, 246, 269, 282, 263, 230, 277, 273, 277, 275, 278, 262, 242, 290, 241, 274, 237, 249, 270, 265, 251, 253, 268, 264, 255, 247, 245, 261, 266, 251, 243, 265, 285, 247, 245, 261, 255, 241, 241, 287, 267, 246, 254, 248, 245, 257, 254, 257, 249, 259, 255, 251, 231, 230, 265, 248, 288, 258, 231, 246, 232, 259, 256, 259, 246, 253, 284, 253, 257, 285, 235, 272, 249, 249, 244, 259, 231, 247, 226, 256, 279, 280, 247, 250, 275, 261, 255, 258, 250, 234, 260, 244, 233, 251, 246, 285, 242, 251, 265, 275, 255, 252, 235, 236, 268, 271, 250, 258, 245, 261, 252, 261, 265, 255, 287, 250, 273, 240, 257, 235, 260, 274, 252] payload_rows_per_dst=[5614, 5676, 5652, 5460, 5488, 5438, 5174, 5320] dup_rows_per_dst=[2834, 3052, 3082, 2666, 2588, 2624, 2442, 2426] combine_rows_per_src=[5614, 5676, 5652, 5460, 5488, 5438, 5174, 5320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38380 dup_rows=21698 dup_rows_remote=19054 dedup_ratio=0.331085 router_tokens_per_expert=[277, 244, 241, 256, 255, 234, 280, 229, 251, 247, 264, 243, 261, 235, 231, 266, 243, 273, 280, 264, 275, 271, 269, 266, 265, 252, 254, 239, 238, 270, 267, 248, 275, 253, 250, 265, 228, 236, 258, 238, 252, 257, 257, 252, 263, 244, 247, 256, 254, 233, 281, 255, 265, 242, 260, 257, 249, 243, 257, 281, 256, 266, 246, 284, 254, 265, 252, 237, 271, 240, 254, 238, 276, 278, 263, 259, 264, 242, 243, 251, 262, 268, 252, 246, 275, 246, 230, 243, 257, 256, 260, 241, 235, 244, 262, 274, 264, 271, 266, 265, 250, 278, 266, 245, 235, 254, 275, 258, 264, 260, 241, 290, 256, 244, 264, 262, 266, 235, 263, 270, 260, 251, 266, 254, 233, 268, 271, 237] payload_rows_per_dst=[5442, 5536, 5432, 5488, 5406, 5458, 5580, 5496] dup_rows_per_dst=[2586, 2812, 2630, 2770, 2768, 2644, 2784, 2704] combine_rows_per_src=[5442, 5536, 5432, 5488, 5406, 5458, 5580, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229752 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934520 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639288 weight_movement_bytes=18874368 total_nvlink_bytes=332513656 weight_to_token_ratio=0.060179 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=228720 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156302704 ingress_bytes=193822720 link_bytes=193822720 token_bytes=312376688 weight_movement_bytes=37748736 total_nvlink_bytes=350125424 weight_to_token_ratio=0.120844 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857113936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38288 dup_rows=21876 dup_rows_remote=19392 dedup_ratio=0.333801 router_tokens_per_expert=[278, 275, 258, 239, 258, 253, 302, 235, 279, 257, 244, 240, 255, 274, 254, 274, 238, 259, 289, 260, 271, 262, 225, 271, 271, 252, 256, 222, 260, 226, 230, 245, 228, 257, 261, 227, 264, 256, 238, 259, 227, 266, 273, 273, 272, 291, 249, 272, 243, 217, 251, 230, 235, 250, 275, 253, 238, 222, 245, 234, 254, 253, 281, 247, 254, 261, 283, 224, 269, 278, 251, 263, 262, 256, 254, 256, 246, 265, 257, 263, 234, 264, 243, 254, 255, 270, 303, 267, 247, 263, 244, 259, 236, 239, 257, 261, 282, 241, 258, 276, 278, 242, 256, 258, 250, 288, 264, 261, 260, 235, 256, 256, 224, 255, 274, 245, 278, 251, 265, 246, 270, 260, 298, 257, 240, 272, 254, 227] payload_rows_per_dst=[5522, 5402, 5424, 5372, 5458, 5510, 5500, 5472] dup_rows_per_dst=[2828, 2672, 2802, 2484, 2826, 2682, 2822, 2760] combine_rows_per_src=[5522, 5402, 5424, 5372, 5458, 5510, 5500, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229736 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409640 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589544 weight_movement_bytes=18874368 total_nvlink_bytes=333463912 weight_to_token_ratio=0.059997 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=229680 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157647152 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315064624 weight_movement_bytes=18874368 total_nvlink_bytes=333938992 weight_to_token_ratio=0.059906 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228352 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156728320 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313228288 weight_movement_bytes=37748736 total_nvlink_bytes=350977024 weight_to_token_ratio=0.120515 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38066 dup_rows=21914 dup_rows_remote=19074 dedup_ratio=0.334381 router_tokens_per_expert=[245, 263, 228, 245, 243, 263, 221, 224, 270, 238, 256, 255, 267, 309, 278, 276, 266, 243, 265, 255, 272, 249, 261, 230, 243, 258, 272, 274, 257, 238, 232, 267, 244, 282, 273, 253, 267, 286, 252, 224, 251, 265, 263, 265, 289, 265, 259, 262, 249, 266, 239, 267, 259, 241, 245, 251, 267, 247, 283, 238, 253, 244, 267, 254, 263, 265, 220, 289, 264, 293, 267, 248, 255, 234, 246, 263, 267, 284, 279, 233, 253, 257, 269, 215, 238, 264, 258, 254, 273, 268, 251, 242, 256, 250, 247, 272, 249, 245, 237, 237, 240, 257, 249, 261, 258, 253, 258, 242, 253, 262, 253, 253, 275, 248, 254, 267, 239, 232, 231, 258, 252, 264, 250, 229, 294, 279, 265, 254] payload_rows_per_dst=[5392, 5428, 5486, 5442, 5506, 5422, 5390, 5556] dup_rows_per_dst=[2556, 2736, 2792, 2698, 2834, 2756, 2702, 2840] combine_rows_per_src=[5392, 5428, 5486, 5442, 5506, 5422, 5390, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229736 combine_payload_bytes=157204480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157434216 ingress_bytes=213827584 link_bytes=213827584 token_bytes=314638696 weight_movement_bytes=56623104 total_nvlink_bytes=371261800 weight_to_token_ratio=0.179962 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048790700e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=230720 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157058368 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313886016 weight_movement_bytes=18874368 total_nvlink_bytes=332760384 weight_to_token_ratio=0.060131 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228560 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156146896 ingress_bytes=193667072 link_bytes=193667072 token_bytes=312065232 weight_movement_bytes=37748736 total_nvlink_bytes=349813968 weight_to_token_ratio=0.120964 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855622594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38234 dup_rows=21808 dup_rows_remote=18934 dedup_ratio=0.332764 router_tokens_per_expert=[266, 232, 257, 271, 239, 265, 264, 241, 265, 245, 259, 250, 282, 258, 249, 250, 253, 269, 244, 257, 257, 278, 264, 247, 252, 261, 272, 232, 265, 266, 237, 259, 309, 242, 279, 273, 257, 262, 241, 270, 274, 249, 285, 243, 270, 252, 260, 266, 227, 260, 262, 219, 276, 250, 283, 270, 249, 251, 258, 261, 232, 269, 239, 240, 268, 248, 240, 263, 246, 285, 240, 280, 267, 266, 260, 257, 238, 239, 266, 237, 253, 269, 268, 245, 263, 270, 263, 256, 247, 260, 254, 256, 260, 258, 245, 244, 251, 234, 227, 262, 241, 273, 230, 243, 242, 261, 242, 268, 251, 242, 266, 225, 209, 270, 282, 261, 253, 256, 265, 263, 276, 274, 255, 244, 248, 247, 237, 275] payload_rows_per_dst=[5452, 5494, 5556, 5466, 5418, 5448, 5434, 5460] dup_rows_per_dst=[2734, 2874, 2908, 2704, 2704, 2774, 2482, 2628] combine_rows_per_src=[5452, 5494, 5556, 5466, 5418, 5448, 5434, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38232 dup_rows=21892 dup_rows_remote=19228 dedup_ratio=0.334045 router_tokens_per_expert=[258, 245, 260, 256, 227, 263, 251, 237, 236, 240, 256, 254, 264, 265, 255, 295, 250, 252, 242, 259, 255, 286, 255, 257, 219, 270, 266, 236, 247, 256, 256, 238, 261, 256, 232, 244, 246, 275, 291, 273, 254, 271, 266, 264, 267, 265, 252, 228, 257, 266, 242, 224, 262, 279, 254, 244, 271, 245, 272, 280, 260, 243, 259, 237, 251, 249, 262, 272, 247, 257, 272, 235, 259, 245, 248, 268, 267, 239, 256, 246, 237, 246, 260, 284, 255, 279, 269, 262, 255, 225, 248, 273, 274, 255, 270, 256, 260, 249, 228, 247, 273, 252, 254, 258, 232, 251, 248, 267, 249, 278, 254, 238, 289, 260, 264, 228, 244, 290, 244, 244, 250, 283, 277, 242, 257, 282, 245, 264] payload_rows_per_dst=[5480, 5430, 5454, 5424, 5410, 5528, 5412, 5506] dup_rows_per_dst=[2644, 2658, 2836, 2766, 2736, 2768, 2664, 2820] combine_rows_per_src=[5480, 5430, 5454, 5424, 5410, 5528, 5412, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38180 dup_rows=21876 dup_rows_remote=19240 dedup_ratio=0.333801 router_tokens_per_expert=[273, 248, 275, 258, 289, 254, 244, 270, 247, 239, 260, 272, 257, 276, 229, 250, 242, 255, 257, 250, 266, 255, 251, 243, 250, 269, 235, 256, 268, 273, 240, 274, 237, 286, 243, 262, 233, 258, 254, 249, 229, 259, 242, 240, 264, 269, 258, 258, 269, 272, 257, 267, 265, 261, 260, 258, 242, 256, 263, 248, 263, 257, 247, 228, 228, 258, 256, 246, 248, 237, 274, 252, 239, 245, 255, 250, 254, 266, 277, 273, 245, 257, 250, 249, 264, 265, 252, 279, 285, 267, 267, 255, 260, 242, 256, 264, 230, 254, 253, 256, 253, 251, 258, 238, 264, 247, 285, 229, 278, 269, 272, 247, 261, 249, 244, 275, 246, 266, 255, 265, 241, 246, 246, 234, 273, 252, 271, 266] payload_rows_per_dst=[5570, 5444, 5480, 5456, 5480, 5446, 5368, 5416] dup_rows_per_dst=[2712, 2724, 2602, 2770, 2636, 2868, 2800, 2764] combine_rows_per_src=[5570, 5444, 5480, 5456, 5480, 5446, 5368, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38138 dup_rows=21922 dup_rows_remote=19074 dedup_ratio=0.334503 router_tokens_per_expert=[227, 271, 257, 255, 278, 242, 256, 271, 268, 252, 254, 262, 238, 235, 244, 250, 260, 226, 245, 260, 254, 271, 227, 259, 232, 265, 264, 223, 237, 243, 261, 269, 263, 264, 244, 246, 264, 249, 271, 266, 256, 238, 266, 281, 268, 256, 279, 251, 261, 224, 258, 256, 250, 262, 240, 277, 290, 244, 250, 245, 257, 242, 256, 260, 264, 257, 259, 259, 254, 242, 247, 273, 276, 253, 247, 261, 283, 245, 242, 264, 245, 267, 244, 242, 255, 249, 267, 251, 243, 268, 261, 250, 246, 228, 242, 275, 239, 246, 278, 287, 258, 261, 273, 233, 245, 265, 264, 253, 244, 246, 257, 277, 295, 260, 273, 267, 257, 274, 256, 277, 254, 272, 268, 237, 236, 244, 250, 273] payload_rows_per_dst=[5454, 5360, 5476, 5450, 5490, 5456, 5454, 5474] dup_rows_per_dst=[2666, 2632, 2848, 2694, 2762, 2610, 2798, 2912] combine_rows_per_src=[5454, 5360, 5476, 5450, 5490, 5456, 5454, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38252 dup_rows=21720 dup_rows_remote=18804 dedup_ratio=0.331421 router_tokens_per_expert=[237, 249, 262, 252, 274, 265, 253, 264, 238, 287, 260, 247, 255, 249, 238, 267, 248, 246, 253, 260, 263, 259, 278, 257, 242, 288, 260, 253, 287, 247, 263, 282, 270, 268, 250, 270, 258, 260, 266, 252, 264, 258, 267, 251, 265, 258, 267, 260, 273, 252, 272, 243, 252, 239, 243, 246, 241, 259, 282, 238, 256, 254, 255, 266, 239, 262, 237, 257, 247, 242, 267, 267, 242, 261, 263, 258, 272, 249, 271, 262, 244, 260, 236, 241, 271, 264, 240, 265, 250, 304, 263, 242, 237, 232, 268, 236, 249, 242, 235, 258, 267, 262, 252, 254, 247, 240, 256, 243, 248, 266, 284, 272, 251, 246, 251, 266, 256, 239, 247, 262, 233, 267, 228, 257, 255, 252, 246, 250] payload_rows_per_dst=[5564, 5534, 5492, 5408, 5450, 5382, 5528, 5458] dup_rows_per_dst=[2916, 2896, 2830, 2476, 2742, 2684, 2622, 2554] combine_rows_per_src=[5564, 5534, 5492, 5408, 5450, 5382, 5528, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38168 dup_rows=21954 dup_rows_remote=19136 dedup_ratio=0.334991 router_tokens_per_expert=[236, 228, 267, 246, 260, 277, 270, 240, 227, 256, 269, 254, 258, 254, 243, 275, 253, 284, 249, 257, 254, 246, 254, 269, 238, 248, 276, 259, 212, 251, 244, 252, 264, 262, 257, 261, 261, 236, 237, 235, 258, 237, 258, 253, 257, 246, 279, 248, 279, 264, 281, 242, 244, 276, 269, 268, 251, 223, 218, 251, 257, 259, 243, 246, 259, 252, 270, 272, 241, 252, 279, 245, 239, 264, 259, 258, 238, 241, 261, 256, 268, 229, 234, 273, 259, 255, 263, 273, 213, 255, 251, 265, 276, 271, 259, 272, 265, 271, 228, 280, 245, 256, 252, 269, 262, 237, 265, 281, 243, 277, 249, 262, 256, 287, 253, 266, 288, 250, 247, 268, 251, 263, 271, 242, 277, 261, 269, 249] payload_rows_per_dst=[5452, 5406, 5432, 5464, 5444, 5414, 5482, 5488] dup_rows_per_dst=[2668, 2686, 2666, 2678, 2728, 2818, 2802, 2908] combine_rows_per_src=[5452, 5406, 5432, 5464, 5444, 5414, 5482, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228672 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156835136 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313441600 weight_movement_bytes=18874368 total_nvlink_bytes=332315968 weight_to_token_ratio=0.060217 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229840 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156828112 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426384 weight_movement_bytes=18874368 total_nvlink_bytes=332300752 weight_to_token_ratio=0.060219 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38282 dup_rows=21752 dup_rows_remote=19020 dedup_ratio=0.331909 router_tokens_per_expert=[244, 246, 275, 275, 269, 237, 260, 242, 243, 265, 246, 244, 263, 260, 206, 263, 256, 259, 264, 231, 272, 246, 265, 263, 245, 261, 251, 254, 275, 260, 239, 242, 277, 271, 239, 243, 246, 222, 294, 249, 254, 248, 250, 242, 261, 250, 251, 247, 280, 239, 231, 259, 268, 265, 267, 252, 234, 256, 261, 256, 280, 258, 247, 264, 274, 251, 260, 273, 262, 252, 275, 263, 246, 255, 262, 260, 268, 224, 258, 290, 242, 252, 243, 251, 255, 258, 249, 271, 261, 256, 249, 259, 261, 251, 248, 238, 241, 273, 254, 231, 254, 276, 248, 272, 243, 268, 255, 287, 257, 250, 263, 284, 268, 241, 229, 247, 277, 269, 252, 246, 258, 251, 248, 268, 250, 285, 273, 251] payload_rows_per_dst=[5436, 5468, 5420, 5502, 5522, 5492, 5494, 5450] dup_rows_per_dst=[2640, 2698, 2668, 2732, 2824, 2596, 2818, 2776] combine_rows_per_src=[5436, 5468, 5420, 5502, 5522, 5492, 5494, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229680 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156614960 ingress_bytes=194134016 link_bytes=194134016 token_bytes=313000240 weight_movement_bytes=37748736 total_nvlink_bytes=350748976 weight_to_token_ratio=0.120603 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228848 combine_payload_bytes=156213248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156442096 ingress_bytes=212836352 link_bytes=212836352 token_bytes=312655344 weight_movement_bytes=56623104 total_nvlink_bytes=369278448 weight_to_token_ratio=0.181104 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039293203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=37950 dup_rows=21892 dup_rows_remote=18886 dedup_ratio=0.334045 router_tokens_per_expert=[245, 277, 265, 268, 242, 262, 243, 249, 241, 241, 277, 269, 244, 251, 242, 300, 268, 264, 258, 263, 264, 240, 234, 241, 252, 288, 271, 220, 245, 237, 252, 267, 242, 237, 272, 236, 256, 252, 272, 246, 237, 267, 267, 250, 288, 286, 271, 271, 288, 256, 253, 268, 246, 236, 295, 254, 248, 259, 253, 265, 260, 266, 276, 256, 255, 255, 238, 240, 250, 227, 253, 243, 225, 260, 244, 233, 269, 250, 255, 265, 265, 266, 270, 285, 264, 269, 251, 250, 239, 250, 238, 238, 243, 243, 264, 245, 255, 284, 245, 255, 244, 252, 273, 270, 266, 281, 240, 273, 229, 243, 281, 249, 248, 251, 235, 242, 262, 258, 295, 243, 252, 262, 263, 250, 255, 271, 240, 250] payload_rows_per_dst=[5362, 5424, 5516, 5486, 5350, 5546, 5266, 5694] dup_rows_per_dst=[2584, 2704, 2784, 2872, 2566, 2900, 2476, 3006] combine_rows_per_src=[5362, 5424, 5516, 5486, 5350, 5546, 5266, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228224 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156908416 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313588608 weight_movement_bytes=18874368 total_nvlink_bytes=332462976 weight_to_token_ratio=0.060188 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229216 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156565344 ingress_bytes=194084864 link_bytes=194084864 token_bytes=312901472 weight_movement_bytes=37748736 total_nvlink_bytes=350650208 weight_to_token_ratio=0.120641 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859625671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229208 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032280 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835352 weight_movement_bytes=37748736 total_nvlink_bytes=351584088 weight_to_token_ratio=0.120282 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155443200 dispatch_meta_bytes=227344 combine_payload_bytes=155443200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155670544 ingress_bytes=174317568 link_bytes=174317568 token_bytes=311113744 weight_movement_bytes=18874368 total_nvlink_bytes=329988112 weight_to_token_ratio=0.060667 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670225167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43490 payload_rows_remote=38098 dup_rows=22046 dup_rows_remote=19338 dedup_ratio=0.336395 router_tokens_per_expert=[243, 265, 271, 270, 224, 280, 273, 244, 235, 236, 268, 265, 230, 270, 278, 285, 233, 249, 249, 248, 256, 250, 261, 273, 227, 253, 258, 259, 263, 254, 282, 235, 209, 247, 223, 299, 245, 258, 270, 267, 254, 272, 283, 254, 271, 244, 262, 254, 261, 294, 247, 251, 226, 273, 254, 249, 224, 251, 267, 245, 282, 280, 217, 255, 264, 241, 269, 240, 271, 257, 252, 268, 251, 257, 260, 276, 245, 273, 260, 239, 246, 252, 259, 241, 260, 270, 274, 265, 287, 245, 239, 260, 261, 257, 245, 260, 286, 255, 232, 260, 262, 260, 266, 257, 246, 254, 233, 285, 233, 234, 284, 271, 259, 243, 253, 243, 288, 245, 232, 247, 247, 282, 251, 240, 234, 252, 256, 259] payload_rows_per_dst=[5428, 5392, 5496, 5388, 5462, 5428, 5464, 5432] dup_rows_per_dst=[2846, 2708, 2728, 2764, 2784, 2814, 2772, 2630] combine_rows_per_src=[5428, 5392, 5496, 5388, 5462, 5428, 5464, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38376 dup_rows=21636 dup_rows_remote=18850 dedup_ratio=0.330139 router_tokens_per_expert=[253, 280, 265, 225, 266, 225, 246, 264, 304, 286, 256, 250, 255, 271, 256, 258, 266, 250, 253, 261, 254, 251, 249, 261, 233, 253, 283, 257, 274, 258, 258, 267, 261, 260, 249, 276, 267, 230, 261, 266, 256, 268, 249, 265, 204, 261, 246, 240, 281, 247, 265, 252, 273, 260, 257, 267, 253, 251, 235, 234, 251, 245, 233, 252, 254, 229, 222, 246, 253, 263, 261, 238, 233, 258, 248, 281, 244, 257, 243, 263, 233, 260, 250, 245, 230, 251, 263, 269, 279, 235, 238, 267, 248, 257, 277, 243, 281, 228, 286, 227, 261, 249, 262, 238, 260, 271, 271, 284, 266, 254, 263, 254, 257, 260, 242, 268, 247, 247, 255, 241, 276, 257, 249, 265, 260, 276, 289, 283] payload_rows_per_dst=[5604, 5524, 5470, 5458, 5368, 5426, 5524, 5526] dup_rows_per_dst=[2716, 2732, 2648, 2654, 2618, 2664, 2786, 2818] combine_rows_per_src=[5604, 5524, 5470, 5458, 5368, 5426, 5524, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38154 dup_rows=21794 dup_rows_remote=18978 dedup_ratio=0.332550 router_tokens_per_expert=[248, 274, 257, 260, 276, 296, 294, 258, 256, 240, 273, 260, 267, 253, 278, 268, 260, 251, 249, 262, 254, 277, 281, 230, 225, 244, 257, 248, 241, 235, 267, 273, 268, 267, 248, 254, 249, 258, 256, 259, 256, 230, 261, 250, 232, 264, 254, 246, 235, 262, 253, 272, 262, 245, 245, 240, 246, 261, 261, 256, 257, 248, 242, 243, 270, 274, 254, 231, 251, 258, 268, 278, 241, 282, 260, 258, 257, 274, 276, 270, 271, 257, 239, 275, 268, 254, 263, 277, 239, 276, 259, 245, 234, 262, 277, 289, 252, 247, 241, 229, 248, 278, 278, 225, 248, 260, 259, 284, 249, 239, 212, 281, 242, 237, 242, 256, 240, 244, 259, 250, 245, 251, 259, 254, 241, 231, 269, 239] payload_rows_per_dst=[5518, 5508, 5460, 5404, 5588, 5498, 5404, 5362] dup_rows_per_dst=[2998, 2600, 2644, 2652, 2816, 2872, 2656, 2556] combine_rows_per_src=[5518, 5508, 5460, 5404, 5588, 5498, 5404, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38468 dup_rows=21716 dup_rows_remote=19206 dedup_ratio=0.331360 router_tokens_per_expert=[258, 259, 290, 258, 273, 258, 244, 241, 229, 292, 269, 237, 255, 277, 254, 244, 238, 238, 254, 249, 258, 228, 258, 289, 249, 272, 257, 251, 248, 242, 268, 272, 237, 240, 258, 255, 245, 253, 245, 256, 232, 249, 282, 229, 231, 242, 250, 227, 263, 292, 266, 246, 266, 246, 256, 224, 266, 264, 268, 246, 282, 260, 249, 209, 249, 272, 265, 240, 250, 239, 243, 262, 245, 263, 267, 254, 261, 304, 251, 259, 265, 253, 284, 273, 254, 276, 274, 272, 260, 266, 238, 264, 248, 265, 230, 256, 269, 261, 258, 265, 300, 234, 256, 234, 243, 248, 268, 259, 259, 266, 268, 259, 260, 260, 219, 259, 250, 250, 268, 255, 267, 266, 261, 235, 242, 259, 273, 252] payload_rows_per_dst=[5564, 5480, 5352, 5498, 5448, 5556, 5440, 5482] dup_rows_per_dst=[2712, 2662, 2510, 2708, 2800, 2800, 2854, 2670] combine_rows_per_src=[5564, 5480, 5352, 5498, 5448, 5556, 5440, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38186 dup_rows=21834 dup_rows_remote=19182 dedup_ratio=0.333160 router_tokens_per_expert=[250, 266, 280, 236, 244, 261, 273, 251, 229, 268, 234, 260, 260, 243, 240, 250, 255, 245, 243, 254, 282, 245, 265, 263, 265, 237, 276, 250, 286, 259, 239, 252, 263, 270, 223, 269, 269, 272, 230, 290, 249, 269, 271, 247, 260, 267, 267, 259, 278, 271, 266, 294, 255, 278, 276, 256, 226, 257, 268, 242, 245, 244, 264, 266, 263, 246, 266, 254, 224, 241, 260, 271, 255, 222, 277, 245, 250, 265, 267, 261, 247, 266, 211, 274, 257, 251, 261, 268, 264, 273, 247, 265, 286, 248, 234, 232, 229, 260, 244, 272, 252, 252, 267, 251, 265, 238, 240, 255, 257, 254, 249, 263, 258, 237, 257, 255, 248, 269, 258, 251, 266, 261, 257, 245, 261, 236, 242, 246] payload_rows_per_dst=[5404, 5492, 5448, 5544, 5464, 5516, 5388, 5446] dup_rows_per_dst=[2686, 2740, 2902, 2828, 2670, 2652, 2708, 2648] combine_rows_per_src=[5404, 5492, 5448, 5544, 5464, 5516, 5388, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38410 dup_rows=21708 dup_rows_remote=19034 dedup_ratio=0.331238 router_tokens_per_expert=[288, 241, 263, 257, 265, 260, 262, 230, 236, 252, 268, 253, 235, 246, 238, 233, 255, 234, 224, 255, 270, 249, 242, 254, 263, 240, 270, 250, 272, 257, 262, 273, 270, 302, 257, 242, 253, 245, 258, 249, 254, 261, 266, 251, 251, 266, 248, 225, 268, 268, 255, 242, 271, 239, 274, 278, 248, 254, 237, 252, 222, 257, 259, 258, 266, 239, 242, 251, 267, 228, 274, 272, 250, 256, 249, 257, 261, 254, 265, 279, 260, 247, 280, 248, 247, 265, 242, 247, 248, 281, 260, 239, 256, 254, 255, 255, 289, 258, 243, 259, 268, 247, 255, 252, 249, 260, 263, 270, 244, 253, 280, 276, 253, 284, 260, 248, 254, 276, 262, 227, 252, 281, 254, 244, 239, 271, 264, 262] payload_rows_per_dst=[5418, 5578, 5628, 5558, 5688, 5334, 5390, 5234] dup_rows_per_dst=[2674, 2790, 2928, 2758, 3060, 2606, 2544, 2348] combine_rows_per_src=[5418, 5578, 5628, 5558, 5688, 5334, 5390, 5234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=229744 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156279152 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312328560 weight_movement_bytes=18874368 total_nvlink_bytes=331202928 weight_to_token_ratio=0.060431 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38378 dup_rows=21800 dup_rows_remote=19140 dedup_ratio=0.332642 router_tokens_per_expert=[275, 267, 226, 263, 254, 270, 254, 272, 275, 266, 237, 260, 267, 216, 232, 268, 270, 247, 280, 245, 259, 277, 277, 242, 249, 256, 249, 241, 256, 236, 276, 247, 269, 248, 246, 263, 266, 251, 227, 245, 265, 263, 252, 245, 246, 238, 252, 261, 254, 248, 268, 239, 263, 257, 243, 256, 224, 256, 269, 247, 261, 259, 226, 239, 257, 248, 248, 280, 273, 286, 259, 266, 229, 242, 259, 271, 243, 267, 259, 278, 242, 282, 277, 242, 270, 264, 267, 260, 257, 244, 252, 251, 260, 261, 235, 241, 233, 266, 280, 258, 258, 257, 247, 269, 232, 268, 256, 264, 260, 281, 261, 249, 232, 269, 262, 274, 240, 266, 245, 248, 253, 259, 241, 251, 254, 287, 270, 253] payload_rows_per_dst=[5456, 5500, 5450, 5358, 5558, 5456, 5476, 5482] dup_rows_per_dst=[2748, 2714, 2624, 2660, 2772, 2754, 2802, 2726] combine_rows_per_src=[5456, 5500, 5450, 5358, 5558, 5456, 5476, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=228904 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157417000 ingress_bytes=213811200 link_bytes=213811200 token_bytes=314605096 weight_movement_bytes=56623104 total_nvlink_bytes=371228200 weight_to_token_ratio=0.179982 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048633717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=228528 combine_payload_bytes=156278784 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156507312 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312786096 weight_movement_bytes=37748736 total_nvlink_bytes=350534832 weight_to_token_ratio=0.120685 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44100 payload_rows_remote=38498 dup_rows=21436 dup_rows_remote=18756 dedup_ratio=0.327087 router_tokens_per_expert=[263, 260, 268, 241, 237, 228, 252, 224, 224, 263, 234, 279, 268, 235, 252, 259, 255, 247, 230, 274, 245, 256, 236, 285, 234, 278, 259, 271, 260, 281, 245, 253, 250, 243, 259, 262, 236, 234, 234, 230, 240, 253, 241, 263, 262, 259, 267, 216, 236, 268, 248, 274, 271, 271, 268, 289, 271, 239, 255, 259, 272, 270, 258, 249, 270, 241, 245, 260, 251, 288, 257, 278, 242, 264, 260, 244, 246, 280, 270, 253, 277, 245, 265, 269, 253, 258, 285, 248, 298, 265, 235, 235, 266, 249, 259, 274, 260, 248, 258, 256, 251, 250, 241, 274, 251, 274, 263, 258, 244, 270, 263, 266, 249, 263, 240, 244, 267, 265, 252, 236, 271, 246, 245, 277, 243, 238, 251, 281] payload_rows_per_dst=[5340, 5468, 5434, 5608, 5550, 5590, 5508, 5602] dup_rows_per_dst=[2634, 2750, 2464, 2752, 2746, 2774, 2636, 2680] combine_rows_per_src=[5340, 5468, 5434, 5608, 5550, 5590, 5508, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=230696 combine_payload_bytes=157564928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157795624 ingress_bytes=176439296 link_bytes=176439296 token_bytes=315360552 weight_movement_bytes=18874368 total_nvlink_bytes=334234920 weight_to_token_ratio=0.059850 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690554521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229472 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156639328 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313049184 weight_movement_bytes=37748736 total_nvlink_bytes=350797920 weight_to_token_ratio=0.120584 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229776 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157557136 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314884496 weight_movement_bytes=18874368 total_nvlink_bytes=333758864 weight_to_token_ratio=0.059941 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=230072 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157426360 ingress_bytes=213819392 link_bytes=213819392 token_bytes=314622648 weight_movement_bytes=56623104 total_nvlink_bytes=371245752 weight_to_token_ratio=0.179971 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048712208e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157687808 dispatch_meta_bytes=229016 combine_payload_bytes=157687808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157916824 ingress_bytes=176562176 link_bytes=176562176 token_bytes=315604632 weight_movement_bytes=18874368 total_nvlink_bytes=334479000 weight_to_token_ratio=0.059804 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691731896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38168 dup_rows=21868 dup_rows_remote=19072 dedup_ratio=0.333679 router_tokens_per_expert=[252, 236, 263, 274, 246, 261, 250, 246, 262, 281, 260, 271, 271, 248, 223, 227, 269, 288, 257, 275, 258, 274, 262, 268, 251, 241, 253, 251, 244, 271, 254, 232, 256, 256, 250, 264, 273, 267, 257, 241, 247, 248, 245, 278, 264, 272, 241, 269, 227, 252, 269, 270, 239, 277, 244, 258, 255, 230, 260, 267, 258, 253, 263, 270, 263, 241, 253, 250, 272, 257, 257, 267, 255, 274, 254, 277, 250, 245, 275, 253, 230, 257, 263, 230, 286, 232, 249, 239, 271, 241, 250, 274, 222, 259, 237, 241, 258, 243, 285, 243, 249, 248, 227, 232, 252, 268, 268, 250, 250, 256, 238, 261, 293, 286, 282, 259, 255, 251, 253, 261, 253, 259, 252, 256, 295, 247, 242, 233] payload_rows_per_dst=[5464, 5500, 5558, 5428, 5504, 5324, 5408, 5482] dup_rows_per_dst=[2678, 2796, 2698, 2756, 2782, 2638, 2648, 2872] combine_rows_per_src=[5464, 5500, 5558, 5428, 5504, 5324, 5408, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38312 dup_rows=21760 dup_rows_remote=19046 dedup_ratio=0.332031 router_tokens_per_expert=[257, 266, 250, 279, 241, 242, 287, 276, 259, 262, 244, 246, 258, 238, 245, 265, 218, 266, 239, 263, 249, 231, 241, 249, 273, 257, 272, 255, 246, 255, 255, 248, 270, 260, 244, 232, 248, 248, 246, 233, 284, 241, 276, 245, 259, 238, 258, 267, 287, 242, 252, 246, 296, 247, 243, 254, 260, 268, 255, 246, 253, 247, 255, 250, 244, 262, 269, 287, 249, 258, 268, 239, 264, 243, 232, 251, 261, 256, 283, 260, 257, 261, 281, 258, 257, 257, 272, 263, 242, 262, 260, 264, 280, 252, 272, 257, 250, 269, 254, 251, 235, 264, 263, 282, 263, 269, 219, 269, 242, 256, 277, 226, 261, 242, 259, 267, 238, 290, 227, 269, 254, 238, 251, 253, 267, 264, 245, 251] payload_rows_per_dst=[5520, 5484, 5366, 5454, 5462, 5562, 5464, 5464] dup_rows_per_dst=[2710, 2550, 2732, 2748, 2790, 2828, 2714, 2688] combine_rows_per_src=[5520, 5484, 5366, 5454, 5462, 5562, 5464, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38318 dup_rows=21706 dup_rows_remote=19036 dedup_ratio=0.331207 router_tokens_per_expert=[245, 247, 231, 262, 280, 265, 252, 228, 260, 260, 284, 237, 267, 258, 270, 255, 242, 254, 278, 247, 252, 245, 272, 260, 244, 238, 229, 261, 259, 276, 236, 241, 284, 271, 247, 222, 274, 282, 255, 248, 255, 238, 239, 268, 264, 271, 260, 235, 255, 257, 280, 249, 244, 260, 283, 264, 251, 247, 262, 253, 260, 244, 246, 229, 253, 287, 259, 286, 264, 236, 234, 282, 268, 255, 275, 242, 249, 230, 227, 244, 276, 270, 272, 275, 264, 241, 268, 254, 257, 258, 231, 286, 252, 240, 276, 261, 264, 270, 265, 259, 222, 258, 235, 280, 245, 260, 261, 274, 253, 268, 239, 236, 253, 272, 255, 253, 255, 278, 226, 240, 267, 236, 257, 264, 254, 263, 251, 251] payload_rows_per_dst=[5490, 5466, 5438, 5462, 5512, 5614, 5354, 5494] dup_rows_per_dst=[2712, 2602, 2788, 2706, 2670, 2748, 2824, 2656] combine_rows_per_src=[5490, 5466, 5438, 5462, 5512, 5614, 5354, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38188 dup_rows=21864 dup_rows_remote=19268 dedup_ratio=0.333618 router_tokens_per_expert=[234, 270, 257, 258, 268, 258, 248, 247, 258, 268, 250, 265, 245, 245, 243, 242, 273, 258, 232, 290, 290, 234, 261, 247, 277, 244, 288, 255, 244, 244, 261, 255, 263, 227, 235, 267, 258, 261, 235, 280, 263, 248, 241, 253, 259, 215, 262, 273, 231, 257, 263, 237, 247, 259, 270, 267, 236, 269, 272, 260, 232, 255, 245, 254, 256, 273, 254, 249, 248, 266, 257, 267, 282, 273, 279, 271, 260, 263, 234, 268, 260, 254, 244, 256, 259, 291, 281, 268, 286, 241, 246, 251, 249, 251, 229, 241, 251, 251, 253, 233, 233, 241, 247, 259, 282, 239, 275, 292, 255, 250, 283, 235, 253, 235, 247, 278, 272, 270, 248, 230, 234, 287, 213, 288, 239, 273, 242, 270] payload_rows_per_dst=[5450, 5528, 5484, 5426, 5484, 5470, 5452, 5378] dup_rows_per_dst=[2662, 2778, 2596, 2682, 2916, 2744, 2706, 2780] combine_rows_per_src=[5450, 5528, 5484, 5426, 5484, 5470, 5452, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38290 dup_rows=21804 dup_rows_remote=19060 dedup_ratio=0.332703 router_tokens_per_expert=[247, 262, 239, 255, 246, 246, 270, 255, 253, 252, 241, 263, 258, 261, 246, 266, 271, 239, 254, 259, 249, 247, 274, 217, 260, 245, 251, 268, 259, 274, 235, 296, 269, 231, 251, 267, 275, 252, 238, 258, 283, 257, 231, 270, 265, 284, 276, 255, 267, 272, 269, 254, 274, 270, 253, 262, 279, 253, 263, 254, 231, 260, 260, 278, 247, 275, 248, 252, 236, 265, 254, 265, 239, 255, 242, 220, 236, 270, 248, 243, 248, 234, 252, 251, 272, 266, 258, 282, 241, 257, 263, 271, 242, 225, 264, 267, 276, 240, 267, 267, 245, 230, 253, 259, 257, 272, 221, 267, 249, 265, 267, 244, 250, 243, 252, 247, 272, 269, 257, 254, 237, 239, 265, 237, 266, 263, 273, 258] payload_rows_per_dst=[5444, 5532, 5494, 5518, 5408, 5442, 5438, 5456] dup_rows_per_dst=[2676, 2664, 2830, 2880, 2582, 2744, 2720, 2708] combine_rows_per_src=[5444, 5532, 5494, 5518, 5408, 5442, 5438, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228960 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156565088 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312901216 weight_movement_bytes=18874368 total_nvlink_bytes=331775584 weight_to_token_ratio=0.060321 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38210 dup_rows=21762 dup_rows_remote=19026 dedup_ratio=0.332062 router_tokens_per_expert=[262, 300, 240, 257, 291, 261, 277, 235, 265, 226, 262, 277, 239, 234, 264, 254, 252, 254, 272, 250, 243, 243, 261, 243, 276, 248, 252, 259, 264, 261, 253, 213, 231, 280, 238, 253, 251, 242, 236, 264, 249, 252, 248, 246, 232, 239, 287, 241, 241, 253, 226, 266, 247, 253, 285, 253, 269, 260, 235, 229, 256, 265, 256, 265, 257, 263, 236, 264, 245, 260, 277, 255, 242, 261, 270, 265, 244, 260, 269, 255, 247, 255, 289, 260, 253, 284, 279, 233, 261, 244, 256, 253, 269, 246, 229, 258, 241, 294, 268, 258, 252, 239, 257, 259, 270, 266, 242, 245, 292, 267, 254, 258, 284, 269, 278, 265, 260, 261, 269, 254, 263, 251, 255, 253, 229, 245, 252, 243] payload_rows_per_dst=[5564, 5650, 5444, 5718, 5298, 5384, 5474, 5242] dup_rows_per_dst=[2736, 2994, 2522, 2938, 2700, 2558, 2850, 2464] combine_rows_per_src=[5564, 5650, 5444, 5718, 5298, 5384, 5474, 5242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229432 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157155384 ingress_bytes=175800320 link_bytes=175800320 token_bytes=314081336 weight_movement_bytes=18874368 total_nvlink_bytes=332955704 weight_to_token_ratio=0.060094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684432167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38250 dup_rows=21832 dup_rows_remote=19196 dedup_ratio=0.333130 router_tokens_per_expert=[254, 259, 236, 229, 264, 245, 238, 232, 263, 268, 266, 262, 233, 271, 271, 269, 238, 267, 283, 255, 241, 244, 266, 285, 256, 266, 280, 238, 242, 245, 238, 255, 238, 267, 262, 244, 245, 270, 250, 276, 267, 227, 257, 244, 261, 252, 280, 258, 269, 248, 260, 243, 253, 251, 260, 256, 261, 258, 273, 251, 245, 241, 259, 217, 234, 261, 246, 244, 273, 246, 257, 258, 255, 257, 239, 218, 245, 259, 261, 254, 267, 249, 272, 306, 247, 253, 248, 263, 270, 249, 257, 290, 236, 257, 232, 262, 256, 265, 254, 248, 255, 244, 238, 244, 278, 248, 285, 243, 260, 273, 291, 248, 298, 278, 253, 239, 270, 258, 261, 217, 276, 267, 303, 249, 246, 246, 242, 268] payload_rows_per_dst=[5472, 5488, 5510, 5454, 5398, 5446, 5510, 5426] dup_rows_per_dst=[2648, 2710, 2686, 2636, 2616, 2870, 2750, 2916] combine_rows_per_src=[5472, 5488, 5510, 5454, 5398, 5446, 5510, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229416 combine_payload_bytes=156950528 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157179944 ingress_bytes=213573632 link_bytes=213573632 token_bytes=314130472 weight_movement_bytes=56623104 total_nvlink_bytes=370753576 weight_to_token_ratio=0.180253 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046357457e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38296 dup_rows=21708 dup_rows_remote=19090 dedup_ratio=0.331238 router_tokens_per_expert=[240, 251, 263, 249, 278, 271, 258, 251, 280, 277, 259, 256, 304, 265, 252, 252, 249, 277, 251, 245, 254, 241, 266, 274, 221, 242, 265, 252, 247, 267, 228, 272, 245, 259, 257, 220, 283, 275, 247, 245, 262, 239, 252, 303, 260, 235, 249, 251, 259, 255, 297, 256, 256, 243, 262, 249, 236, 233, 227, 267, 275, 282, 270, 273, 269, 245, 265, 229, 262, 233, 241, 233, 270, 276, 249, 240, 253, 257, 266, 279, 233, 244, 248, 263, 235, 265, 262, 272, 265, 268, 253, 258, 255, 241, 275, 281, 250, 225, 234, 235, 244, 275, 251, 283, 225, 257, 270, 257, 250, 258, 270, 278, 244, 237, 247, 264, 250, 249, 261, 250, 250, 250, 254, 273, 245, 263, 273, 232] payload_rows_per_dst=[5408, 5438, 5390, 5660, 5354, 5536, 5510, 5532] dup_rows_per_dst=[2540, 2664, 2532, 2906, 2780, 2812, 2856, 2618] combine_rows_per_src=[5408, 5438, 5390, 5660, 5354, 5536, 5510, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229824 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156647872 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065920 weight_movement_bytes=18874368 total_nvlink_bytes=331940288 weight_to_token_ratio=0.060289 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229400 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157065240 ingress_bytes=194584576 link_bytes=194584576 token_bytes=313901080 weight_movement_bytes=37748736 total_nvlink_bytes=351649816 weight_to_token_ratio=0.120257 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864413666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=228944 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156737104 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245264 weight_movement_bytes=37748736 total_nvlink_bytes=350994000 weight_to_token_ratio=0.120509 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229784 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901784 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573784 weight_movement_bytes=18874368 total_nvlink_bytes=332448152 weight_to_token_ratio=0.060191 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229544 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089960 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313950376 weight_movement_bytes=18874368 total_nvlink_bytes=332824744 weight_to_token_ratio=0.060119 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38236 dup_rows=21796 dup_rows_remote=19262 dedup_ratio=0.332581 router_tokens_per_expert=[250, 276, 250, 251, 282, 262, 253, 249, 262, 256, 244, 266, 251, 273, 262, 243, 275, 276, 246, 232, 245, 237, 234, 253, 246, 261, 231, 248, 254, 284, 253, 244, 239, 246, 252, 264, 281, 264, 230, 263, 265, 222, 254, 255, 249, 231, 289, 266, 252, 297, 269, 250, 268, 234, 264, 268, 284, 236, 272, 252, 257, 231, 270, 272, 252, 256, 259, 236, 258, 255, 255, 236, 256, 234, 289, 251, 262, 251, 257, 264, 250, 267, 259, 273, 293, 269, 252, 269, 279, 237, 270, 255, 255, 254, 239, 270, 249, 277, 255, 251, 240, 246, 262, 258, 264, 247, 252, 253, 253, 249, 258, 236, 235, 277, 236, 285, 245, 259, 240, 263, 255, 242, 271, 254, 253, 255, 226, 265] payload_rows_per_dst=[5476, 5504, 5462, 5538, 5416, 5492, 5404, 5448] dup_rows_per_dst=[2784, 2534, 2678, 2814, 2878, 2738, 2696, 2674] combine_rows_per_src=[5476, 5504, 5462, 5538, 5416, 5492, 5404, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38038 dup_rows=21964 dup_rows_remote=19064 dedup_ratio=0.335144 router_tokens_per_expert=[264, 289, 251, 241, 268, 244, 263, 227, 254, 240, 258, 238, 277, 261, 250, 254, 257, 254, 242, 240, 269, 273, 240, 259, 242, 236, 222, 253, 259, 279, 276, 255, 281, 243, 256, 244, 239, 252, 257, 228, 254, 257, 265, 251, 264, 246, 246, 245, 236, 282, 223, 254, 265, 256, 255, 236, 254, 250, 234, 257, 264, 269, 251, 292, 245, 241, 283, 256, 238, 269, 252, 267, 259, 275, 245, 249, 266, 233, 269, 250, 251, 243, 216, 260, 252, 242, 270, 251, 293, 259, 275, 251, 250, 266, 259, 262, 269, 251, 266, 265, 269, 265, 245, 248, 307, 269, 289, 276, 249, 262, 268, 243, 268, 273, 277, 278, 242, 247, 264, 251, 250, 239, 263, 270, 255, 220, 246, 246] payload_rows_per_dst=[5410, 5498, 5394, 5436, 5424, 5466, 5534, 5410] dup_rows_per_dst=[2748, 2614, 2662, 2720, 2770, 2734, 2900, 2816] combine_rows_per_src=[5410, 5498, 5394, 5436, 5424, 5466, 5534, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38306 dup_rows=21778 dup_rows_remote=19016 dedup_ratio=0.332306 router_tokens_per_expert=[276, 263, 241, 225, 274, 256, 269, 235, 262, 243, 258, 265, 221, 245, 270, 271, 246, 252, 259, 274, 252, 261, 247, 249, 277, 260, 265, 248, 244, 254, 281, 230, 239, 290, 259, 246, 274, 250, 271, 251, 269, 267, 286, 279, 244, 266, 260, 251, 245, 245, 249, 270, 279, 256, 231, 251, 269, 266, 265, 225, 242, 249, 224, 232, 275, 279, 263, 226, 259, 218, 247, 271, 278, 266, 263, 252, 242, 262, 253, 253, 255, 253, 258, 268, 241, 270, 283, 227, 266, 256, 264, 257, 250, 239, 252, 273, 263, 274, 274, 250, 238, 238, 235, 261, 254, 249, 285, 249, 255, 269, 251, 244, 277, 240, 262, 240, 299, 247, 255, 243, 267, 249, 233, 287, 214, 263, 259, 252] payload_rows_per_dst=[5442, 5468, 5580, 5444, 5452, 5428, 5488, 5456] dup_rows_per_dst=[2706, 2730, 2824, 2552, 2762, 2796, 2690, 2718] combine_rows_per_src=[5442, 5468, 5580, 5444, 5452, 5428, 5488, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38138 dup_rows=21904 dup_rows_remote=19104 dedup_ratio=0.334229 router_tokens_per_expert=[264, 264, 284, 269, 231, 272, 252, 248, 241, 261, 258, 270, 250, 264, 281, 274, 270, 268, 268, 254, 270, 248, 258, 245, 274, 237, 287, 220, 242, 267, 263, 246, 242, 260, 265, 281, 286, 266, 254, 251, 259, 240, 251, 251, 280, 264, 232, 265, 266, 264, 268, 253, 242, 257, 248, 260, 239, 242, 262, 267, 238, 267, 242, 267, 283, 237, 235, 286, 273, 236, 279, 241, 243, 251, 227, 248, 290, 266, 240, 245, 262, 235, 237, 255, 238, 285, 273, 239, 251, 261, 241, 262, 220, 265, 259, 229, 264, 281, 257, 263, 265, 258, 239, 225, 241, 260, 237, 255, 230, 251, 255, 260, 205, 227, 285, 238, 250, 278, 250, 260, 262, 274, 261, 247, 282, 276, 260, 251] payload_rows_per_dst=[5540, 5438, 5494, 5434, 5398, 5422, 5348, 5558] dup_rows_per_dst=[2826, 2796, 2800, 2730, 2762, 2602, 2734, 2654] combine_rows_per_src=[5540, 5438, 5494, 5434, 5398, 5422, 5348, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38050 dup_rows=21960 dup_rows_remote=19230 dedup_ratio=0.335083 router_tokens_per_expert=[222, 241, 256, 249, 255, 247, 275, 279, 236, 258, 276, 268, 248, 273, 255, 259, 274, 233, 241, 226, 254, 258, 258, 279, 256, 277, 253, 263, 249, 254, 232, 278, 230, 260, 265, 276, 251, 269, 249, 247, 258, 242, 264, 271, 262, 271, 254, 249, 240, 220, 257, 223, 232, 283, 261, 249, 227, 229, 263, 237, 268, 236, 264, 257, 258, 263, 242, 279, 267, 265, 249, 294, 269, 251, 222, 259, 266, 257, 250, 254, 261, 259, 248, 274, 254, 260, 254, 278, 240, 258, 272, 238, 256, 271, 265, 240, 265, 259, 254, 268, 263, 243, 266, 251, 265, 278, 273, 241, 252, 273, 242, 272, 252, 275, 263, 248, 223, 242, 262, 258, 275, 219, 232, 288, 234, 277, 248, 288] payload_rows_per_dst=[5488, 5492, 5414, 5230, 5508, 5526, 5522, 5396] dup_rows_per_dst=[2706, 2678, 2822, 2662, 2782, 2730, 2808, 2772] combine_rows_per_src=[5488, 5492, 5414, 5230, 5508, 5526, 5522, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229992 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156844648 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313459304 weight_movement_bytes=18874368 total_nvlink_bytes=332333672 weight_to_token_ratio=0.060213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38334 dup_rows=21768 dup_rows_remote=19002 dedup_ratio=0.332153 router_tokens_per_expert=[267, 268, 254, 250, 255, 261, 242, 252, 233, 261, 237, 305, 246, 281, 241, 230, 264, 270, 265, 237, 273, 239, 268, 247, 231, 272, 242, 241, 298, 249, 271, 219, 240, 241, 260, 256, 236, 232, 259, 257, 254, 259, 251, 247, 248, 223, 261, 276, 275, 264, 268, 259, 253, 248, 241, 259, 245, 254, 242, 261, 255, 265, 267, 277, 269, 250, 272, 254, 240, 260, 270, 250, 240, 232, 277, 259, 258, 251, 259, 239, 271, 259, 258, 243, 252, 263, 304, 258, 277, 275, 252, 250, 223, 253, 253, 218, 269, 250, 262, 278, 266, 235, 264, 273, 260, 245, 266, 280, 262, 249, 255, 284, 258, 253, 220, 286, 237, 239, 239, 260, 246, 268, 295, 250, 261, 268, 234, 265] payload_rows_per_dst=[5434, 5600, 5632, 5434, 5428, 5258, 5550, 5432] dup_rows_per_dst=[2766, 2948, 2920, 2568, 2620, 2408, 2812, 2726] combine_rows_per_src=[5434, 5600, 5632, 5434, 5428, 5258, 5550, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155803648 dispatch_meta_bytes=228408 combine_payload_bytes=155803648 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156032056 ingress_bytes=212426752 link_bytes=212426752 token_bytes=311835704 weight_movement_bytes=56623104 total_nvlink_bytes=368458808 weight_to_token_ratio=0.181580 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035368617e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38078 dup_rows=21898 dup_rows_remote=19124 dedup_ratio=0.334137 router_tokens_per_expert=[281, 270, 246, 241, 270, 269, 258, 272, 249, 262, 270, 252, 244, 275, 256, 271, 247, 281, 249, 233, 240, 257, 279, 243, 290, 272, 276, 255, 248, 241, 261, 240, 269, 271, 240, 242, 238, 273, 243, 269, 246, 271, 255, 277, 252, 236, 233, 255, 268, 253, 293, 251, 258, 276, 253, 250, 287, 243, 257, 268, 245, 250, 246, 269, 300, 267, 260, 289, 244, 246, 250, 265, 250, 244, 248, 267, 236, 255, 251, 250, 254, 236, 260, 259, 245, 259, 251, 258, 252, 247, 262, 219, 278, 272, 280, 234, 253, 223, 270, 251, 245, 241, 260, 258, 226, 221, 215, 273, 273, 271, 262, 271, 238, 249, 253, 228, 265, 248, 227, 268, 282, 231, 241, 265, 255, 258, 246, 278] payload_rows_per_dst=[5514, 5428, 5424, 5560, 5420, 5446, 5450, 5396] dup_rows_per_dst=[2858, 2796, 2716, 2774, 2824, 2686, 2576, 2668] combine_rows_per_src=[5514, 5428, 5424, 5560, 5420, 5446, 5450, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38024 dup_rows=21892 dup_rows_remote=18940 dedup_ratio=0.334045 router_tokens_per_expert=[259, 270, 263, 255, 211, 253, 252, 271, 234, 287, 233, 243, 261, 255, 254, 275, 237, 284, 253, 237, 297, 268, 262, 253, 259, 274, 262, 241, 229, 219, 262, 265, 257, 261, 239, 250, 264, 266, 275, 258, 241, 266, 263, 268, 256, 235, 262, 243, 255, 280, 253, 282, 239, 243, 253, 245, 257, 251, 262, 247, 259, 262, 238, 295, 265, 252, 264, 274, 277, 257, 283, 238, 245, 250, 238, 261, 260, 253, 270, 230, 255, 264, 277, 249, 276, 257, 273, 252, 249, 276, 234, 249, 227, 249, 252, 276, 252, 254, 276, 274, 250, 238, 252, 218, 243, 255, 232, 301, 244, 252, 241, 253, 279, 258, 271, 267, 237, 207, 231, 261, 250, 269, 266, 322, 240, 252, 252, 236] payload_rows_per_dst=[5448, 5424, 5508, 5330, 5372, 5466, 5476, 5620] dup_rows_per_dst=[2704, 2776, 2700, 2536, 2594, 2768, 2862, 2952] combine_rows_per_src=[5448, 5424, 5508, 5330, 5372, 5466, 5476, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229288 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130664 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032040 weight_movement_bytes=18874368 total_nvlink_bytes=332906408 weight_to_token_ratio=0.060103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228968 combine_payload_bytes=156213248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156442216 ingress_bytes=212836352 link_bytes=212836352 token_bytes=312655464 weight_movement_bytes=56623104 total_nvlink_bytes=369278568 weight_to_token_ratio=0.181104 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039293203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=229120 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156081920 ingress_bytes=193601536 link_bytes=193601536 token_bytes=311934720 weight_movement_bytes=37748736 total_nvlink_bytes=349683456 weight_to_token_ratio=0.121015 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854994660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229344 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157245408 ingress_bytes=194764800 link_bytes=194764800 token_bytes=314261472 weight_movement_bytes=37748736 total_nvlink_bytes=352010208 weight_to_token_ratio=0.120119 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866140483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=228808 combine_payload_bytes=155967488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156196296 ingress_bytes=193716224 link_bytes=193716224 token_bytes=312163784 weight_movement_bytes=37748736 total_nvlink_bytes=349912520 weight_to_token_ratio=0.120926 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856093544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=155746304 dispatch_meta_bytes=227856 combine_payload_bytes=155746304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155974160 ingress_bytes=174620672 link_bytes=174620672 token_bytes=311720464 weight_movement_bytes=18874368 total_nvlink_bytes=330594832 weight_to_token_ratio=0.060549 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673129360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38258 dup_rows=21722 dup_rows_remote=18938 dedup_ratio=0.331451 router_tokens_per_expert=[271, 258, 236, 262, 261, 258, 269, 264, 237, 259, 261, 275, 262, 261, 244, 223, 275, 257, 281, 255, 254, 248, 255, 284, 261, 247, 284, 271, 253, 258, 237, 250, 245, 252, 279, 241, 211, 250, 281, 254, 264, 235, 230, 260, 255, 271, 246, 263, 255, 291, 253, 263, 265, 252, 264, 259, 255, 250, 255, 266, 253, 254, 260, 244, 263, 245, 260, 276, 277, 253, 287, 262, 245, 252, 267, 233, 245, 270, 248, 284, 247, 255, 277, 273, 274, 237, 246, 233, 247, 264, 253, 270, 258, 235, 247, 225, 229, 255, 250, 276, 215, 273, 263, 263, 260, 250, 252, 237, 256, 232, 256, 260, 257, 274, 251, 273, 267, 258, 266, 246, 251, 248, 245, 246, 243, 250, 259, 252] payload_rows_per_dst=[5434, 5556, 5492, 5520, 5458, 5440, 5428, 5486] dup_rows_per_dst=[2768, 2784, 2776, 2758, 2682, 2642, 2626, 2686] combine_rows_per_src=[5434, 5556, 5492, 5520, 5458, 5440, 5428, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38326 dup_rows=21706 dup_rows_remote=18974 dedup_ratio=0.331207 router_tokens_per_expert=[267, 262, 243, 288, 283, 269, 271, 262, 263, 252, 264, 257, 266, 240, 242, 274, 229, 230, 241, 233, 254, 253, 290, 282, 243, 252, 255, 249, 256, 241, 215, 274, 260, 227, 271, 214, 242, 254, 254, 242, 265, 255, 299, 267, 237, 250, 220, 279, 258, 266, 254, 271, 268, 222, 261, 282, 267, 279, 253, 254, 279, 261, 273, 255, 262, 252, 249, 253, 272, 254, 233, 268, 251, 248, 251, 271, 261, 238, 242, 258, 267, 272, 243, 239, 241, 265, 256, 229, 279, 237, 262, 260, 231, 231, 272, 274, 265, 273, 275, 266, 278, 248, 253, 274, 264, 227, 249, 233, 276, 228, 247, 262, 254, 243, 248, 258, 249, 262, 238, 252, 258, 283, 253, 270, 225, 271, 266, 260] payload_rows_per_dst=[5550, 5444, 5410, 5568, 5446, 5460, 5504, 5448] dup_rows_per_dst=[2856, 2550, 2662, 2838, 2680, 2656, 2732, 2732] combine_rows_per_src=[5550, 5444, 5410, 5568, 5446, 5460, 5504, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38066 dup_rows=21988 dup_rows_remote=19152 dedup_ratio=0.335510 router_tokens_per_expert=[234, 242, 261, 267, 248, 264, 258, 212, 278, 241, 256, 266, 255, 248, 249, 262, 221, 245, 254, 269, 275, 242, 254, 233, 232, 246, 235, 295, 271, 254, 257, 254, 234, 245, 258, 245, 283, 263, 229, 272, 252, 271, 238, 269, 245, 264, 233, 270, 258, 272, 262, 255, 252, 245, 289, 254, 242, 223, 268, 247, 268, 232, 265, 266, 279, 289, 251, 248, 276, 280, 247, 268, 245, 280, 257, 273, 234, 252, 230, 250, 274, 251, 236, 265, 260, 273, 245, 236, 255, 262, 249, 273, 253, 265, 249, 266, 264, 235, 279, 236, 271, 251, 253, 271, 254, 261, 241, 251, 276, 242, 256, 252, 264, 262, 277, 253, 253, 253, 292, 250, 240, 272, 252, 259, 265, 250, 243, 272] payload_rows_per_dst=[5440, 5326, 5450, 5480, 5482, 5400, 5460, 5510] dup_rows_per_dst=[2642, 2748, 2692, 2716, 2836, 2824, 2726, 2804] combine_rows_per_src=[5440, 5326, 5450, 5480, 5482, 5400, 5460, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38304 dup_rows=21806 dup_rows_remote=19144 dedup_ratio=0.332733 router_tokens_per_expert=[257, 236, 266, 281, 224, 265, 262, 268, 230, 243, 248, 256, 252, 258, 293, 230, 264, 252, 254, 234, 255, 247, 259, 240, 263, 242, 259, 245, 262, 263, 265, 275, 268, 263, 272, 269, 264, 239, 253, 250, 265, 239, 221, 258, 257, 242, 234, 250, 231, 277, 227, 243, 265, 261, 242, 267, 260, 251, 252, 266, 255, 234, 223, 261, 255, 248, 256, 259, 250, 257, 263, 281, 263, 214, 292, 247, 255, 260, 269, 304, 268, 279, 267, 263, 277, 249, 276, 260, 304, 262, 271, 238, 280, 258, 224, 238, 234, 233, 239, 238, 249, 250, 285, 278, 249, 262, 263, 238, 270, 270, 282, 261, 269, 242, 256, 251, 259, 265, 238, 273, 247, 233, 272, 246, 255, 253, 259, 255] payload_rows_per_dst=[5470, 5454, 5426, 5382, 5502, 5570, 5500, 5426] dup_rows_per_dst=[2668, 2704, 2662, 2648, 2844, 2858, 2702, 2720] combine_rows_per_src=[5470, 5454, 5426, 5382, 5502, 5570, 5500, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228784 combine_payload_bytes=156704768 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156933552 ingress_bytes=194453504 link_bytes=194453504 token_bytes=313638320 weight_movement_bytes=37748736 total_nvlink_bytes=351387056 weight_to_token_ratio=0.120358 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863157798e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38104 dup_rows=21958 dup_rows_remote=19210 dedup_ratio=0.335052 router_tokens_per_expert=[237, 230, 258, 259, 227, 257, 249, 255, 248, 254, 256, 233, 257, 256, 262, 239, 229, 233, 253, 247, 273, 270, 238, 245, 259, 284, 250, 267, 289, 246, 262, 264, 275, 233, 214, 226, 250, 235, 257, 256, 238, 262, 263, 248, 254, 257, 262, 254, 277, 252, 270, 255, 265, 268, 272, 269, 241, 264, 271, 267, 248, 254, 254, 242, 280, 280, 245, 255, 271, 237, 268, 259, 278, 241, 273, 260, 249, 265, 230, 268, 275, 265, 262, 277, 246, 250, 217, 264, 268, 250, 250, 255, 264, 239, 258, 271, 259, 273, 258, 265, 243, 271, 279, 231, 272, 263, 275, 241, 308, 259, 250, 270, 249, 250, 258, 237, 251, 233, 259, 253, 249, 258, 246, 268, 274, 247, 263, 247] payload_rows_per_dst=[5362, 5402, 5394, 5456, 5506, 5474, 5536, 5448] dup_rows_per_dst=[2592, 2816, 2574, 2882, 2812, 2748, 2898, 2636] combine_rows_per_src=[5362, 5402, 5394, 5456, 5506, 5474, 5536, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38180 dup_rows=21832 dup_rows_remote=18980 dedup_ratio=0.333130 router_tokens_per_expert=[266, 233, 253, 250, 266, 251, 252, 217, 262, 273, 264, 256, 266, 257, 252, 283, 257, 247, 250, 287, 219, 258, 259, 264, 291, 255, 245, 255, 245, 277, 264, 223, 257, 247, 255, 264, 252, 276, 238, 267, 249, 253, 246, 247, 265, 260, 246, 270, 247, 260, 253, 251, 245, 251, 257, 233, 238, 243, 258, 274, 280, 251, 258, 234, 249, 280, 282, 255, 269, 262, 274, 264, 254, 272, 250, 278, 226, 250, 255, 255, 292, 233, 265, 262, 254, 267, 222, 255, 240, 278, 244, 257, 248, 237, 282, 251, 244, 256, 278, 257, 246, 281, 242, 265, 297, 259, 268, 234, 261, 276, 266, 234, 242, 276, 266, 228, 256, 256, 223, 262, 251, 234, 256, 271, 240, 262, 270, 227] payload_rows_per_dst=[5524, 5518, 5650, 5354, 5322, 5382, 5484, 5470] dup_rows_per_dst=[2852, 2726, 2956, 2540, 2604, 2740, 2844, 2570] combine_rows_per_src=[5524, 5518, 5650, 5354, 5322, 5382, 5484, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38154 dup_rows=21866 dup_rows_remote=19096 dedup_ratio=0.333649 router_tokens_per_expert=[239, 279, 271, 214, 232, 275, 243, 231, 269, 236, 240, 242, 246, 275, 272, 265, 253, 255, 288, 238, 225, 261, 276, 271, 259, 250, 236, 272, 255, 256, 243, 261, 259, 237, 270, 257, 254, 245, 259, 261, 251, 264, 264, 266, 235, 235, 259, 260, 257, 242, 236, 274, 263, 255, 266, 290, 279, 256, 253, 258, 275, 270, 233, 236, 290, 248, 249, 240, 250, 252, 247, 280, 241, 279, 283, 265, 282, 262, 236, 281, 282, 228, 243, 250, 247, 252, 236, 275, 278, 285, 246, 238, 241, 247, 241, 267, 263, 248, 231, 240, 274, 268, 241, 241, 276, 277, 250, 274, 262, 271, 262, 253, 253, 270, 261, 263, 249, 240, 246, 262, 231, 263, 256, 249, 259, 250, 242, 255] payload_rows_per_dst=[5360, 5504, 5464, 5516, 5526, 5388, 5424, 5488] dup_rows_per_dst=[2698, 2694, 2688, 2770, 2844, 2724, 2838, 2610] combine_rows_per_src=[5360, 5504, 5464, 5516, 5526, 5388, 5424, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229200 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212496 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195792 weight_movement_bytes=37748736 total_nvlink_bytes=351944528 weight_to_token_ratio=0.120144 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38014 dup_rows=21852 dup_rows_remote=18988 dedup_ratio=0.333435 router_tokens_per_expert=[247, 269, 266, 280, 255, 254, 266, 263, 251, 253, 251, 253, 260, 259, 261, 248, 250, 216, 266, 284, 270, 268, 277, 245, 270, 228, 259, 259, 254, 235, 250, 259, 238, 247, 248, 244, 241, 253, 261, 308, 258, 223, 240, 235, 256, 242, 287, 242, 274, 259, 255, 263, 243, 270, 227, 280, 270, 268, 222, 261, 286, 245, 232, 250, 258, 229, 256, 251, 259, 260, 240, 252, 258, 274, 267, 285, 253, 254, 263, 249, 290, 268, 254, 254, 251, 232, 248, 254, 253, 263, 238, 257, 277, 255, 277, 250, 272, 253, 242, 284, 239, 244, 253, 252, 278, 242, 270, 282, 240, 236, 287, 224, 271, 249, 284, 259, 253, 253, 244, 264, 248, 216, 258, 252, 267, 240, 274, 255] payload_rows_per_dst=[5328, 5390, 5482, 5398, 5532, 5510, 5374, 5670] dup_rows_per_dst=[2584, 2454, 2900, 2558, 2938, 2732, 2822, 2864] combine_rows_per_src=[5328, 5390, 5482, 5398, 5532, 5510, 5374, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228872 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156147208 ingress_bytes=212541440 link_bytes=212541440 token_bytes=312065544 weight_movement_bytes=56623104 total_nvlink_bytes=368688648 weight_to_token_ratio=0.181446 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036467501e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229792 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122976 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314016160 weight_movement_bytes=18874368 total_nvlink_bytes=332890528 weight_to_token_ratio=0.060106 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229256 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156303240 ingress_bytes=212697088 link_bytes=212697088 token_bytes=312377224 weight_movement_bytes=56623104 total_nvlink_bytes=369000328 weight_to_token_ratio=0.181265 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037958844e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228640 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156613920 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999200 weight_movement_bytes=18874368 total_nvlink_bytes=331873568 weight_to_token_ratio=0.060302 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229000 combine_payload_bytes=156278784 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156507784 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786568 weight_movement_bytes=18874368 total_nvlink_bytes=331660936 weight_to_token_ratio=0.060343 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=228008 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155933352 ingress_bytes=174579712 link_bytes=174579712 token_bytes=311638696 weight_movement_bytes=18874368 total_nvlink_bytes=330513064 weight_to_token_ratio=0.060565 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672736902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38426 dup_rows=21652 dup_rows_remote=18968 dedup_ratio=0.330383 router_tokens_per_expert=[270, 233, 282, 269, 250, 266, 265, 255, 270, 269, 275, 229, 271, 258, 242, 263, 231, 275, 251, 278, 237, 239, 240, 264, 253, 254, 257, 250, 251, 258, 261, 272, 249, 244, 262, 255, 279, 258, 249, 249, 265, 269, 246, 251, 282, 253, 244, 280, 262, 238, 266, 250, 241, 257, 267, 248, 261, 281, 236, 276, 288, 253, 272, 227, 274, 269, 266, 263, 261, 241, 242, 253, 284, 240, 235, 254, 247, 277, 260, 242, 270, 243, 266, 265, 246, 233, 271, 245, 260, 237, 263, 266, 228, 242, 259, 234, 262, 242, 267, 258, 290, 250, 258, 266, 241, 237, 251, 261, 249, 244, 274, 254, 237, 267, 265, 247, 251, 277, 269, 253, 229, 245, 242, 231, 260, 250, 248, 261] payload_rows_per_dst=[5550, 5458, 5558, 5440, 5450, 5478, 5504, 5446] dup_rows_per_dst=[2784, 2684, 2712, 2806, 2766, 2578, 2704, 2618] combine_rows_per_src=[5550, 5458, 5558, 5440, 5450, 5478, 5504, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38268 dup_rows=21778 dup_rows_remote=19034 dedup_ratio=0.332306 router_tokens_per_expert=[247, 277, 232, 275, 245, 246, 270, 295, 239, 229, 263, 254, 244, 261, 273, 235, 280, 273, 273, 228, 254, 274, 246, 243, 261, 237, 243, 262, 292, 241, 285, 268, 254, 256, 277, 266, 220, 240, 257, 217, 254, 245, 288, 251, 224, 234, 267, 258, 274, 241, 275, 269, 246, 276, 255, 258, 256, 233, 238, 273, 282, 261, 257, 278, 241, 228, 268, 256, 234, 231, 251, 267, 254, 244, 264, 230, 252, 252, 240, 272, 252, 252, 271, 243, 244, 271, 245, 242, 302, 261, 273, 242, 251, 262, 285, 278, 230, 255, 269, 250, 270, 246, 268, 253, 244, 236, 265, 277, 262, 259, 248, 285, 276, 252, 252, 264, 249, 251, 228, 281, 238, 251, 245, 254, 262, 243, 254, 268] payload_rows_per_dst=[5508, 5466, 5406, 5484, 5410, 5552, 5490, 5442] dup_rows_per_dst=[2662, 2854, 2610, 2860, 2558, 2796, 2744, 2694] combine_rows_per_src=[5508, 5466, 5406, 5484, 5410, 5552, 5490, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38442 dup_rows=21744 dup_rows_remote=19226 dedup_ratio=0.331787 router_tokens_per_expert=[254, 262, 226, 255, 266, 244, 249, 242, 260, 247, 261, 287, 260, 268, 257, 244, 280, 248, 209, 266, 274, 261, 234, 269, 253, 279, 260, 246, 273, 260, 258, 275, 280, 260, 248, 269, 239, 259, 238, 234, 259, 252, 260, 253, 243, 264, 251, 261, 252, 258, 247, 266, 276, 245, 242, 237, 269, 242, 254, 256, 262, 278, 276, 253, 255, 225, 262, 249, 252, 244, 229, 239, 280, 255, 232, 246, 260, 235, 232, 239, 263, 243, 242, 265, 254, 251, 266, 252, 261, 264, 243, 260, 257, 258, 254, 270, 274, 241, 282, 272, 243, 244, 241, 294, 282, 248, 263, 255, 280, 262, 278, 275, 230, 268, 244, 255, 273, 265, 269, 243, 250, 256, 245, 258, 245, 247, 248, 291] payload_rows_per_dst=[5496, 5524, 5450, 5438, 5350, 5510, 5548, 5476] dup_rows_per_dst=[2668, 2766, 2690, 2788, 2518, 2696, 2920, 2698] combine_rows_per_src=[5496, 5524, 5450, 5438, 5350, 5510, 5548, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43470 payload_rows_remote=38030 dup_rows=22066 dup_rows_remote=19262 dedup_ratio=0.336700 router_tokens_per_expert=[233, 242, 256, 229, 253, 249, 249, 258, 267, 264, 299, 244, 252, 243, 258, 269, 267, 251, 231, 256, 239, 255, 248, 266, 268, 260, 251, 262, 229, 257, 269, 259, 258, 240, 266, 249, 263, 257, 254, 251, 258, 263, 286, 268, 245, 250, 253, 261, 261, 258, 296, 251, 260, 250, 272, 255, 261, 256, 264, 284, 265, 244, 257, 256, 249, 233, 225, 268, 246, 254, 249, 272, 256, 235, 253, 267, 278, 245, 253, 236, 270, 257, 239, 232, 265, 247, 254, 260, 266, 243, 260, 240, 272, 254, 261, 283, 248, 266, 259, 234, 251, 256, 242, 269, 230, 243, 256, 253, 243, 270, 257, 253, 241, 247, 289, 288, 255, 273, 260, 251, 293, 265, 280, 256, 251, 258, 225, 239] payload_rows_per_dst=[5432, 5436, 5440, 5482, 5360, 5432, 5402, 5486] dup_rows_per_dst=[2698, 2700, 2804, 2898, 2678, 2774, 2658, 2856] combine_rows_per_src=[5432, 5436, 5440, 5482, 5360, 5432, 5402, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157392896 dispatch_meta_bytes=229576 combine_payload_bytes=157392896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157622472 ingress_bytes=195141632 link_bytes=195141632 token_bytes=315015368 weight_movement_bytes=37748736 total_nvlink_bytes=352764104 weight_to_token_ratio=0.119831 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869751102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38168 dup_rows=21924 dup_rows_remote=19260 dedup_ratio=0.334534 router_tokens_per_expert=[285, 254, 243, 280, 278, 240, 259, 266, 272, 257, 247, 245, 267, 254, 266, 257, 246, 255, 246, 284, 274, 240, 271, 236, 266, 251, 229, 262, 243, 264, 253, 262, 243, 248, 270, 230, 282, 264, 261, 248, 246, 277, 248, 271, 250, 279, 244, 232, 275, 274, 262, 256, 266, 252, 232, 292, 243, 253, 274, 246, 237, 281, 243, 240, 247, 236, 245, 241, 240, 267, 261, 236, 266, 234, 244, 293, 256, 239, 265, 263, 261, 266, 254, 243, 245, 254, 243, 268, 249, 266, 252, 237, 231, 280, 251, 254, 257, 247, 252, 281, 263, 254, 267, 273, 251, 227, 262, 242, 288, 274, 252, 237, 249, 261, 269, 245, 254, 242, 256, 268, 236, 264, 278, 244, 283, 231, 252, 251] payload_rows_per_dst=[5556, 5466, 5406, 5414, 5408, 5444, 5470, 5448] dup_rows_per_dst=[2784, 2698, 2780, 2838, 2658, 2664, 2784, 2718] combine_rows_per_src=[5556, 5466, 5406, 5414, 5408, 5444, 5470, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38250 dup_rows=21750 dup_rows_remote=18956 dedup_ratio=0.331879 router_tokens_per_expert=[264, 244, 267, 269, 263, 244, 256, 274, 266, 284, 224, 259, 257, 251, 271, 263, 259, 277, 231, 233, 252, 248, 270, 261, 283, 243, 256, 258, 269, 235, 269, 250, 263, 245, 247, 264, 267, 277, 266, 278, 243, 249, 272, 261, 282, 265, 283, 255, 250, 239, 259, 250, 275, 247, 243, 270, 245, 259, 258, 269, 248, 254, 248, 252, 263, 258, 243, 221, 230, 230, 249, 280, 246, 259, 261, 212, 238, 254, 282, 241, 267, 261, 241, 264, 249, 244, 258, 265, 251, 271, 265, 270, 270, 271, 261, 251, 248, 243, 246, 265, 280, 263, 244, 270, 227, 240, 250, 223, 244, 256, 240, 254, 281, 259, 245, 260, 263, 292, 245, 239, 270, 217, 260, 247, 264, 253, 273, 248] payload_rows_per_dst=[5536, 5486, 5538, 5378, 5588, 5352, 5408, 5500] dup_rows_per_dst=[2794, 2790, 2790, 2754, 2874, 2438, 2578, 2732] combine_rows_per_src=[5536, 5486, 5538, 5378, 5588, 5352, 5408, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38296 dup_rows=21714 dup_rows_remote=19054 dedup_ratio=0.331329 router_tokens_per_expert=[295, 256, 252, 269, 293, 266, 288, 253, 235, 282, 230, 254, 254, 238, 252, 256, 249, 267, 299, 267, 255, 270, 234, 284, 261, 269, 234, 233, 268, 258, 247, 243, 276, 260, 241, 244, 264, 267, 223, 229, 237, 255, 295, 275, 261, 290, 262, 232, 276, 276, 246, 245, 253, 232, 248, 263, 259, 244, 233, 266, 257, 263, 271, 261, 238, 260, 248, 265, 260, 246, 250, 272, 251, 258, 257, 255, 270, 247, 228, 281, 271, 243, 247, 242, 258, 264, 258, 259, 241, 264, 266, 253, 241, 237, 248, 247, 258, 251, 246, 241, 251, 250, 234, 243, 252, 281, 276, 259, 252, 267, 271, 232, 250, 251, 233, 266, 240, 277, 278, 252, 243, 256, 284, 239, 262, 267, 222, 244] payload_rows_per_dst=[5484, 5492, 5532, 5526, 5428, 5468, 5476, 5416] dup_rows_per_dst=[2862, 2784, 2690, 2660, 2744, 2610, 2652, 2712] combine_rows_per_src=[5484, 5492, 5532, 5526, 5428, 5468, 5476, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229208 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974936 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720664 weight_movement_bytes=18874368 total_nvlink_bytes=332595032 weight_to_token_ratio=0.060163 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38432 dup_rows=21744 dup_rows_remote=19134 dedup_ratio=0.331787 router_tokens_per_expert=[247, 246, 248, 254, 249, 279, 230, 294, 265, 233, 252, 269, 254, 240, 243, 265, 221, 224, 240, 271, 258, 249, 273, 256, 287, 251, 270, 263, 256, 279, 269, 256, 233, 248, 270, 251, 261, 257, 260, 259, 253, 260, 261, 275, 263, 271, 263, 242, 268, 251, 261, 246, 268, 243, 238, 258, 239, 275, 253, 260, 277, 260, 255, 269, 253, 257, 246, 257, 285, 264, 235, 269, 233, 268, 256, 258, 264, 241, 260, 243, 272, 261, 261, 265, 291, 248, 260, 248, 262, 273, 232, 273, 270, 245, 274, 266, 265, 244, 241, 286, 247, 259, 263, 262, 268, 236, 263, 265, 264, 233, 234, 254, 249, 216, 246, 237, 247, 228, 229, 248, 264, 250, 241, 273, 252, 265, 257, 253] payload_rows_per_dst=[5506, 5534, 5472, 5376, 5586, 5534, 5424, 5360] dup_rows_per_dst=[2630, 2712, 2768, 2544, 2914, 2846, 2720, 2610] combine_rows_per_src=[5506, 5534, 5472, 5376, 5586, 5534, 5424, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=230672 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157689104 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315147536 weight_movement_bytes=18874368 total_nvlink_bytes=334021904 weight_to_token_ratio=0.059891 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=229168 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156000048 ingress_bytes=174645248 link_bytes=174645248 token_bytes=311770928 weight_movement_bytes=18874368 total_nvlink_bytes=330645296 weight_to_token_ratio=0.060539 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673364836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229712 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156565840 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312901968 weight_movement_bytes=18874368 total_nvlink_bytes=331776336 weight_to_token_ratio=0.060320 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228824 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156900824 ingress_bytes=232169472 link_bytes=232169472 token_bytes=313572824 weight_movement_bytes=75497472 total_nvlink_bytes=389070296 weight_to_token_ratio=0.240765 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224533646e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229400 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157089816 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950232 weight_movement_bytes=37748736 total_nvlink_bytes=351698968 weight_to_token_ratio=0.120238 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=230264 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157647736 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315065208 weight_movement_bytes=18874368 total_nvlink_bytes=333939576 weight_to_token_ratio=0.059906 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38458 dup_rows=21646 dup_rows_remote=18968 dedup_ratio=0.330292 router_tokens_per_expert=[260, 262, 266, 225, 276, 246, 236, 264, 252, 242, 256, 278, 256, 270, 275, 244, 236, 235, 266, 259, 252, 263, 285, 268, 250, 273, 247, 246, 220, 257, 256, 242, 250, 241, 259, 250, 237, 249, 268, 218, 268, 274, 257, 265, 265, 281, 244, 263, 279, 269, 242, 237, 253, 264, 306, 258, 247, 234, 262, 251, 235, 252, 235, 255, 269, 246, 247, 239, 251, 254, 251, 259, 256, 244, 213, 239, 290, 239, 292, 237, 247, 237, 252, 289, 226, 268, 284, 288, 267, 237, 247, 258, 273, 242, 263, 222, 262, 266, 264, 268, 262, 249, 261, 258, 263, 261, 267, 245, 262, 230, 256, 269, 261, 248, 275, 267, 258, 278, 279, 304, 244, 264, 268, 257, 232, 235, 236, 262] payload_rows_per_dst=[5542, 5432, 5432, 5492, 5444, 5452, 5536, 5560] dup_rows_per_dst=[2674, 2678, 2746, 2666, 2608, 2748, 2750, 2776] combine_rows_per_src=[5542, 5432, 5432, 5492, 5444, 5452, 5536, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38414 dup_rows=21686 dup_rows_remote=19120 dedup_ratio=0.330902 router_tokens_per_expert=[256, 257, 224, 265, 306, 255, 261, 240, 287, 279, 255, 248, 251, 269, 266, 251, 234, 269, 249, 256, 248, 243, 246, 284, 282, 235, 257, 275, 255, 231, 244, 263, 222, 259, 262, 289, 252, 253, 266, 244, 258, 259, 241, 262, 254, 249, 285, 257, 257, 243, 252, 255, 273, 273, 257, 282, 258, 249, 234, 265, 272, 266, 265, 239, 287, 249, 263, 262, 262, 285, 269, 228, 250, 280, 268, 230, 262, 256, 283, 280, 235, 268, 265, 235, 248, 249, 250, 280, 236, 257, 254, 244, 251, 280, 248, 249, 221, 278, 251, 226, 253, 274, 256, 230, 275, 253, 265, 241, 237, 238, 256, 247, 256, 252, 268, 249, 247, 243, 243, 254, 249, 238, 233, 255, 231, 249, 276, 268] payload_rows_per_dst=[5494, 5482, 5456, 5540, 5590, 5412, 5436, 5440] dup_rows_per_dst=[2846, 2660, 2768, 2740, 2838, 2686, 2566, 2582] combine_rows_per_src=[5494, 5482, 5456, 5540, 5590, 5412, 5436, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157523968 dispatch_meta_bytes=229704 combine_payload_bytes=157523968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157753672 ingress_bytes=176398336 link_bytes=176398336 token_bytes=315277640 weight_movement_bytes=18874368 total_nvlink_bytes=334152008 weight_to_token_ratio=0.059866 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690162062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38338 dup_rows=21746 dup_rows_remote=18946 dedup_ratio=0.331818 router_tokens_per_expert=[243, 258, 262, 266, 283, 244, 243, 251, 220, 258, 241, 263, 255, 235, 257, 255, 234, 243, 285, 264, 231, 260, 286, 285, 256, 266, 249, 264, 227, 242, 264, 239, 256, 260, 269, 268, 247, 257, 233, 239, 254, 270, 256, 254, 246, 257, 285, 275, 251, 237, 267, 240, 267, 257, 276, 237, 283, 248, 253, 247, 286, 232, 255, 264, 269, 261, 267, 259, 232, 252, 237, 266, 232, 273, 264, 262, 267, 300, 264, 231, 266, 281, 256, 247, 233, 220, 240, 233, 229, 240, 273, 278, 260, 252, 255, 268, 284, 268, 270, 257, 253, 260, 244, 240, 244, 239, 250, 287, 232, 279, 229, 272, 258, 247, 279, 257, 243, 251, 255, 248, 242, 269, 235, 278, 287, 270, 258, 261] payload_rows_per_dst=[5428, 5496, 5452, 5442, 5488, 5458, 5524, 5502] dup_rows_per_dst=[2640, 2694, 2800, 2758, 2784, 2604, 2692, 2774] combine_rows_per_src=[5428, 5496, 5452, 5442, 5488, 5458, 5524, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38404 dup_rows=21746 dup_rows_remote=19216 dedup_ratio=0.331818 router_tokens_per_expert=[214, 261, 268, 283, 240, 263, 263, 250, 244, 262, 241, 254, 251, 286, 246, 217, 239, 261, 256, 252, 259, 280, 283, 234, 260, 258, 240, 285, 249, 235, 247, 242, 254, 229, 281, 260, 269, 244, 259, 243, 251, 257, 262, 265, 245, 270, 245, 246, 263, 241, 260, 274, 254, 239, 239, 259, 263, 249, 272, 246, 235, 247, 277, 266, 257, 252, 269, 247, 239, 231, 246, 261, 220, 228, 255, 260, 237, 252, 253, 251, 255, 273, 270, 252, 271, 274, 267, 257, 248, 265, 258, 260, 250, 304, 283, 241, 251, 265, 244, 267, 240, 259, 253, 260, 243, 267, 230, 253, 271, 262, 285, 232, 291, 273, 225, 269, 257, 245, 273, 271, 271, 262, 249, 263, 277, 276, 254, 257] payload_rows_per_dst=[5418, 5474, 5514, 5396, 5386, 5528, 5470, 5604] dup_rows_per_dst=[2668, 2686, 2646, 2772, 2530, 2928, 2694, 2822] combine_rows_per_src=[5418, 5474, 5514, 5396, 5386, 5528, 5470, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38166 dup_rows=21920 dup_rows_remote=19056 dedup_ratio=0.334473 router_tokens_per_expert=[233, 253, 267, 260, 239, 263, 262, 271, 259, 283, 259, 241, 248, 217, 252, 248, 251, 267, 257, 244, 220, 246, 216, 268, 246, 248, 265, 271, 238, 281, 269, 261, 260, 247, 251, 259, 257, 256, 280, 264, 267, 289, 262, 262, 248, 266, 241, 250, 278, 246, 254, 267, 235, 240, 245, 289, 250, 261, 247, 240, 251, 243, 295, 238, 254, 259, 252, 259, 277, 245, 251, 259, 270, 239, 264, 249, 271, 247, 274, 250, 234, 285, 243, 251, 259, 256, 267, 235, 278, 266, 206, 284, 246, 272, 278, 297, 247, 276, 262, 243, 267, 242, 248, 250, 258, 271, 241, 244, 244, 269, 254, 232, 241, 238, 258, 260, 252, 264, 242, 227, 272, 253, 275, 248, 286, 247, 272, 267] payload_rows_per_dst=[5444, 5458, 5508, 5466, 5428, 5450, 5414, 5448] dup_rows_per_dst=[2666, 2638, 2810, 2692, 2812, 2864, 2682, 2756] combine_rows_per_src=[5444, 5458, 5508, 5466, 5428, 5450, 5414, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38282 dup_rows=21828 dup_rows_remote=18958 dedup_ratio=0.333069 router_tokens_per_expert=[249, 237, 249, 251, 263, 236, 272, 264, 261, 285, 270, 226, 279, 261, 264, 243, 261, 269, 265, 304, 245, 243, 259, 270, 230, 269, 252, 245, 255, 259, 240, 262, 262, 225, 242, 262, 258, 251, 263, 250, 247, 233, 256, 248, 238, 252, 245, 257, 252, 262, 254, 239, 270, 240, 254, 256, 274, 267, 267, 261, 244, 240, 244, 263, 260, 248, 260, 279, 252, 254, 265, 242, 286, 234, 242, 258, 267, 280, 242, 257, 265, 263, 277, 253, 255, 254, 236, 261, 239, 255, 262, 257, 256, 267, 239, 261, 239, 277, 249, 266, 253, 270, 260, 274, 277, 277, 220, 238, 245, 255, 261, 286, 241, 256, 266, 238, 256, 261, 280, 259, 264, 249, 240, 269, 251, 246, 243, 262] payload_rows_per_dst=[5426, 5490, 5554, 5456, 5342, 5440, 5524, 5476] dup_rows_per_dst=[2870, 2690, 2944, 2758, 2434, 2676, 2770, 2686] combine_rows_per_src=[5426, 5490, 5554, 5456, 5342, 5440, 5524, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=230136 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157573880 ingress_bytes=176218112 link_bytes=176218112 token_bytes=314917624 weight_movement_bytes=18874368 total_nvlink_bytes=333791992 weight_to_token_ratio=0.059934 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688435244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38282 dup_rows=21726 dup_rows_remote=19122 dedup_ratio=0.331512 router_tokens_per_expert=[269, 272, 273, 257, 267, 254, 254, 254, 240, 240, 266, 242, 224, 256, 232, 243, 258, 262, 253, 256, 267, 263, 257, 247, 253, 234, 273, 263, 276, 246, 267, 260, 256, 251, 262, 266, 266, 286, 274, 247, 246, 257, 247, 256, 250, 268, 256, 255, 254, 264, 258, 235, 256, 251, 254, 267, 238, 245, 246, 262, 263, 257, 252, 264, 242, 260, 271, 236, 245, 277, 272, 259, 241, 247, 247, 258, 262, 270, 255, 277, 247, 244, 251, 262, 265, 256, 236, 275, 250, 237, 265, 230, 259, 261, 247, 284, 269, 232, 251, 256, 277, 230, 253, 281, 273, 280, 245, 247, 233, 259, 286, 271, 228, 269, 238, 268, 259, 246, 237, 240, 260, 239, 266, 260, 251, 279, 255, 255] payload_rows_per_dst=[5408, 5542, 5524, 5528, 5486, 5446, 5490, 5386] dup_rows_per_dst=[2678, 2728, 2762, 2604, 2752, 2692, 2796, 2714] combine_rows_per_src=[5408, 5542, 5524, 5528, 5486, 5446, 5490, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38336 dup_rows=21728 dup_rows_remote=18966 dedup_ratio=0.331543 router_tokens_per_expert=[280, 255, 247, 288, 273, 270, 264, 287, 267, 264, 231, 259, 245, 254, 268, 248, 251, 254, 299, 279, 283, 253, 252, 259, 260, 247, 251, 265, 251, 243, 269, 246, 269, 213, 203, 279, 279, 242, 284, 270, 246, 252, 254, 251, 237, 274, 264, 257, 251, 270, 232, 239, 253, 255, 251, 268, 268, 246, 247, 257, 252, 270, 248, 240, 227, 264, 284, 252, 273, 258, 249, 241, 260, 269, 250, 249, 237, 246, 248, 250, 248, 266, 248, 261, 279, 241, 266, 257, 256, 243, 274, 209, 265, 239, 281, 260, 280, 259, 257, 246, 249, 263, 246, 253, 252, 242, 268, 245, 248, 237, 254, 233, 248, 262, 264, 233, 250, 253, 260, 258, 288, 260, 287, 249, 269, 223, 251, 248] payload_rows_per_dst=[5524, 5438, 5316, 5618, 5560, 5518, 5362, 5472] dup_rows_per_dst=[2786, 2536, 2500, 2874, 2886, 2682, 2702, 2762] combine_rows_per_src=[5524, 5438, 5316, 5618, 5560, 5518, 5362, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229136 combine_payload_bytes=157032448 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157261584 ingress_bytes=194781184 link_bytes=194781184 token_bytes=314294032 weight_movement_bytes=37748736 total_nvlink_bytes=352042768 weight_to_token_ratio=0.120106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866297467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=230480 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157533264 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314836048 weight_movement_bytes=18874368 total_nvlink_bytes=333710416 weight_to_token_ratio=0.059950 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228888 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156556824 ingress_bytes=212951040 link_bytes=212951040 token_bytes=312884760 weight_movement_bytes=56623104 total_nvlink_bytes=369507864 weight_to_token_ratio=0.180971 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040392087e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=228960 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032032 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835104 weight_movement_bytes=18874368 total_nvlink_bytes=332709472 weight_to_token_ratio=0.060141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229616 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032688 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835760 weight_movement_bytes=18874368 total_nvlink_bytes=332710128 weight_to_token_ratio=0.060141 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229208 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157253464 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314277720 weight_movement_bytes=37748736 total_nvlink_bytes=352026456 weight_to_token_ratio=0.120113 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38218 dup_rows=21942 dup_rows_remote=19140 dedup_ratio=0.334808 router_tokens_per_expert=[258, 240, 264, 246, 266, 268, 231, 287, 279, 245, 277, 256, 253, 274, 249, 262, 269, 250, 235, 233, 254, 261, 257, 240, 255, 284, 270, 258, 258, 225, 243, 297, 283, 249, 260, 240, 234, 229, 212, 297, 261, 257, 259, 255, 248, 223, 272, 262, 273, 275, 259, 269, 236, 262, 270, 265, 260, 263, 222, 255, 250, 234, 250, 260, 251, 260, 230, 258, 255, 223, 249, 260, 240, 257, 256, 261, 248, 258, 244, 252, 268, 264, 256, 279, 276, 270, 266, 254, 278, 236, 260, 276, 256, 233, 221, 240, 261, 244, 213, 219, 261, 258, 257, 266, 262, 267, 253, 253, 257, 259, 269, 234, 268, 267, 251, 276, 270, 241, 254, 271, 255, 267, 264, 280, 247, 272, 249, 280] payload_rows_per_dst=[5420, 5376, 5488, 5474, 5366, 5554, 5376, 5540] dup_rows_per_dst=[2890, 2802, 2594, 2732, 2638, 2712, 2690, 2884] combine_rows_per_src=[5420, 5376, 5488, 5474, 5366, 5554, 5376, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38280 dup_rows=21800 dup_rows_remote=19134 dedup_ratio=0.332642 router_tokens_per_expert=[264, 250, 260, 264, 230, 270, 250, 299, 275, 292, 265, 254, 230, 260, 260, 273, 220, 261, 240, 272, 262, 252, 244, 294, 276, 257, 288, 259, 260, 259, 260, 243, 240, 279, 272, 263, 240, 257, 254, 278, 240, 256, 246, 246, 230, 243, 274, 264, 256, 265, 271, 237, 286, 251, 268, 259, 253, 239, 250, 262, 234, 280, 229, 241, 276, 265, 266, 248, 256, 265, 286, 247, 238, 263, 250, 257, 252, 251, 255, 276, 269, 260, 239, 257, 236, 266, 243, 243, 226, 284, 235, 236, 220, 274, 265, 260, 255, 256, 251, 251, 254, 252, 278, 278, 275, 238, 237, 231, 251, 259, 243, 252, 248, 254, 260, 247, 233, 276, 257, 252, 237, 264, 267, 227, 279, 257, 221, 258] payload_rows_per_dst=[5576, 5484, 5488, 5400, 5524, 5396, 5456, 5412] dup_rows_per_dst=[2816, 2810, 2676, 2762, 2778, 2630, 2666, 2662] combine_rows_per_src=[5576, 5484, 5488, 5400, 5524, 5396, 5456, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229432 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156770360 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313311288 weight_movement_bytes=37748736 total_nvlink_bytes=351060024 weight_to_token_ratio=0.120483 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38196 dup_rows=21876 dup_rows_remote=19124 dedup_ratio=0.333801 router_tokens_per_expert=[252, 285, 245, 279, 262, 263, 265, 226, 256, 242, 299, 249, 236, 248, 260, 259, 266, 266, 260, 287, 252, 268, 259, 252, 265, 224, 269, 270, 256, 214, 242, 250, 247, 245, 286, 286, 248, 267, 260, 246, 213, 266, 273, 234, 256, 264, 267, 250, 257, 281, 263, 239, 254, 257, 227, 210, 244, 239, 236, 257, 269, 264, 258, 236, 239, 268, 256, 264, 249, 258, 292, 261, 281, 260, 247, 273, 260, 259, 238, 249, 270, 271, 221, 261, 261, 269, 241, 255, 261, 245, 242, 255, 256, 258, 272, 243, 244, 263, 264, 251, 230, 258, 279, 279, 254, 240, 245, 288, 264, 269, 273, 264, 241, 251, 230, 257, 260, 277, 285, 259, 238, 264, 256, 244, 241, 251, 232, 257] payload_rows_per_dst=[5534, 5416, 5464, 5358, 5534, 5434, 5510, 5410] dup_rows_per_dst=[2718, 2784, 2752, 2624, 2774, 2728, 2820, 2676] combine_rows_per_src=[5534, 5416, 5464, 5358, 5534, 5434, 5510, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38476 dup_rows=21616 dup_rows_remote=18874 dedup_ratio=0.329834 router_tokens_per_expert=[271, 238, 257, 278, 241, 277, 249, 266, 244, 276, 249, 228, 238, 251, 259, 242, 246, 284, 235, 249, 251, 270, 244, 274, 269, 238, 230, 265, 227, 259, 265, 279, 261, 248, 264, 283, 285, 247, 258, 263, 258, 270, 267, 271, 268, 228, 257, 260, 246, 263, 245, 240, 273, 283, 252, 259, 250, 237, 254, 261, 240, 266, 267, 243, 264, 237, 243, 259, 270, 268, 232, 276, 250, 283, 226, 248, 263, 263, 250, 261, 274, 241, 259, 211, 252, 253, 272, 240, 255, 277, 264, 274, 288, 267, 241, 270, 260, 252, 250, 245, 250, 270, 247, 271, 264, 219, 256, 245, 284, 251, 248, 251, 270, 249, 240, 265, 250, 239, 247, 256, 255, 253, 266, 268, 265, 249, 258, 228] payload_rows_per_dst=[5470, 5478, 5562, 5482, 5444, 5590, 5418, 5476] dup_rows_per_dst=[2658, 2692, 2814, 2676, 2742, 2686, 2708, 2640] combine_rows_per_src=[5470, 5478, 5562, 5482, 5444, 5590, 5418, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38294 dup_rows=21718 dup_rows_remote=18854 dedup_ratio=0.331390 router_tokens_per_expert=[280, 249, 252, 235, 222, 228, 273, 265, 247, 262, 260, 255, 247, 269, 267, 237, 284, 248, 248, 246, 260, 263, 260, 264, 246, 262, 274, 272, 282, 263, 242, 262, 265, 261, 266, 214, 283, 252, 267, 241, 260, 271, 265, 280, 230, 230, 263, 241, 278, 250, 241, 246, 227, 254, 265, 252, 265, 257, 285, 242, 250, 272, 277, 243, 259, 236, 233, 255, 231, 259, 269, 246, 244, 272, 272, 246, 235, 250, 220, 265, 253, 271, 299, 267, 262, 259, 235, 260, 259, 265, 230, 272, 261, 274, 249, 278, 269, 240, 238, 239, 259, 266, 232, 242, 267, 249, 270, 233, 233, 265, 265, 239, 263, 254, 241, 257, 261, 276, 258, 280, 266, 244, 265, 265, 246, 249, 252, 282] payload_rows_per_dst=[5516, 5490, 5460, 5484, 5416, 5524, 5428, 5500] dup_rows_per_dst=[2580, 2862, 2718, 2724, 2568, 2864, 2584, 2818] combine_rows_per_src=[5516, 5490, 5460, 5484, 5416, 5524, 5428, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229656 combine_payload_bytes=156794880 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157024536 ingress_bytes=213417984 link_bytes=213417984 token_bytes=313819416 weight_movement_bytes=56623104 total_nvlink_bytes=370442520 weight_to_token_ratio=0.180432 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044866114e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38282 dup_rows=21728 dup_rows_remote=19048 dedup_ratio=0.331543 router_tokens_per_expert=[274, 266, 243, 270, 281, 250, 236, 240, 234, 270, 271, 245, 253, 245, 251, 268, 255, 257, 260, 214, 248, 254, 250, 257, 265, 298, 290, 286, 285, 276, 256, 274, 269, 220, 233, 249, 268, 241, 250, 239, 280, 253, 263, 257, 250, 242, 256, 229, 266, 248, 254, 261, 229, 225, 252, 270, 264, 261, 266, 234, 241, 254, 241, 248, 237, 266, 240, 236, 241, 268, 253, 257, 247, 279, 298, 274, 257, 253, 243, 254, 264, 288, 274, 257, 261, 228, 239, 269, 252, 281, 232, 271, 283, 227, 258, 265, 257, 264, 253, 289, 255, 265, 254, 258, 233, 241, 245, 235, 287, 251, 261, 291, 262, 270, 245, 262, 246, 266, 259, 239, 252, 258, 232, 283, 232, 250, 234, 252] payload_rows_per_dst=[5526, 5578, 5412, 5378, 5448, 5596, 5390, 5480] dup_rows_per_dst=[2680, 2890, 2586, 2650, 2758, 2984, 2576, 2604] combine_rows_per_src=[5526, 5578, 5412, 5378, 5448, 5596, 5390, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38316 dup_rows=21840 dup_rows_remote=19156 dedup_ratio=0.333252 router_tokens_per_expert=[247, 241, 285, 231, 255, 235, 259, 239, 276, 262, 253, 257, 236, 258, 261, 246, 260, 269, 259, 247, 247, 219, 257, 261, 251, 272, 275, 268, 227, 261, 264, 263, 278, 298, 265, 262, 266, 268, 249, 250, 263, 256, 261, 250, 276, 252, 269, 262, 258, 235, 260, 282, 249, 252, 235, 243, 264, 248, 262, 233, 238, 238, 263, 272, 265, 230, 266, 284, 247, 279, 260, 263, 258, 247, 261, 249, 269, 237, 263, 278, 261, 254, 276, 253, 240, 254, 247, 274, 258, 246, 214, 277, 250, 258, 280, 264, 248, 270, 244, 278, 235, 261, 233, 267, 224, 265, 244, 263, 269, 252, 261, 233, 243, 273, 262, 248, 286, 254, 264, 242, 218, 278, 247, 263, 251, 257, 245, 230] payload_rows_per_dst=[5454, 5452, 5588, 5380, 5452, 5468, 5460, 5442] dup_rows_per_dst=[2628, 2748, 2862, 2684, 2860, 2744, 2634, 2680] combine_rows_per_src=[5454, 5452, 5588, 5380, 5452, 5468, 5460, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38032 dup_rows=21966 dup_rows_remote=19204 dedup_ratio=0.335175 router_tokens_per_expert=[244, 233, 259, 246, 236, 241, 240, 242, 263, 243, 276, 268, 268, 267, 255, 240, 264, 256, 252, 263, 278, 256, 261, 278, 238, 235, 249, 251, 241, 224, 276, 269, 239, 249, 255, 254, 235, 281, 292, 252, 262, 227, 249, 280, 258, 262, 251, 252, 246, 271, 251, 259, 253, 254, 266, 265, 257, 284, 269, 264, 246, 246, 245, 248, 268, 255, 261, 275, 256, 253, 261, 272, 236, 242, 232, 250, 274, 259, 243, 258, 254, 256, 259, 265, 280, 217, 257, 245, 253, 265, 268, 267, 244, 270, 249, 285, 236, 265, 269, 229, 271, 238, 269, 288, 233, 266, 264, 268, 257, 237, 241, 251, 257, 242, 265, 254, 248, 233, 263, 258, 257, 269, 250, 264, 266, 265, 249, 283] payload_rows_per_dst=[5390, 5430, 5362, 5514, 5454, 5420, 5462, 5538] dup_rows_per_dst=[2652, 2696, 2576, 2734, 2994, 2756, 2796, 2762] combine_rows_per_src=[5390, 5430, 5362, 5514, 5454, 5420, 5462, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229280 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156680096 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130912 weight_movement_bytes=37748736 total_nvlink_bytes=350879648 weight_to_token_ratio=0.120553 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229400 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827096 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315424792 weight_movement_bytes=18874368 total_nvlink_bytes=334299160 weight_to_token_ratio=0.059838 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228592 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157080816 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933040 weight_movement_bytes=37748736 total_nvlink_bytes=351681776 weight_to_token_ratio=0.120245 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229320 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032392 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835464 weight_movement_bytes=37748736 total_nvlink_bytes=351584200 weight_to_token_ratio=0.120282 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229888 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157172224 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114560 weight_movement_bytes=18874368 total_nvlink_bytes=332988928 weight_to_token_ratio=0.060088 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38272 dup_rows=21816 dup_rows_remote=18950 dedup_ratio=0.332886 router_tokens_per_expert=[270, 246, 256, 227, 229, 249, 231, 258, 240, 248, 255, 244, 272, 277, 241, 260, 278, 244, 262, 262, 277, 241, 279, 235, 275, 288, 238, 275, 239, 240, 256, 268, 270, 236, 266, 267, 266, 248, 240, 259, 266, 264, 264, 228, 249, 261, 243, 248, 284, 278, 254, 276, 236, 254, 237, 240, 267, 258, 266, 240, 271, 249, 243, 258, 249, 260, 271, 274, 256, 300, 267, 234, 251, 219, 274, 244, 265, 258, 237, 250, 254, 243, 272, 257, 263, 265, 241, 256, 281, 247, 270, 256, 264, 250, 248, 289, 246, 255, 236, 249, 241, 271, 238, 256, 274, 259, 253, 253, 239, 242, 241, 281, 264, 263, 256, 258, 257, 255, 262, 251, 261, 255, 267, 239, 246, 262, 275, 252] payload_rows_per_dst=[5358, 5448, 5492, 5450, 5472, 5552, 5450, 5498] dup_rows_per_dst=[2648, 2866, 2658, 2772, 2746, 2760, 2618, 2748] combine_rows_per_src=[5358, 5448, 5492, 5450, 5472, 5552, 5450, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=228944 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156008016 ingress_bytes=193527808 link_bytes=193527808 token_bytes=311787088 weight_movement_bytes=37748736 total_nvlink_bytes=349535824 weight_to_token_ratio=0.121072 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854288235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38204 dup_rows=21904 dup_rows_remote=19110 dedup_ratio=0.334229 router_tokens_per_expert=[244, 245, 238, 262, 273, 256, 225, 267, 261, 260, 245, 253, 246, 250, 260, 227, 241, 242, 233, 272, 240, 278, 253, 256, 239, 259, 256, 251, 258, 254, 266, 293, 272, 271, 244, 294, 275, 243, 258, 305, 235, 236, 243, 238, 234, 271, 217, 255, 264, 277, 244, 279, 267, 272, 265, 271, 261, 263, 287, 258, 239, 243, 271, 266, 264, 261, 245, 238, 250, 284, 269, 267, 263, 257, 242, 230, 278, 230, 254, 258, 261, 261, 238, 241, 272, 290, 279, 276, 236, 240, 256, 252, 247, 262, 246, 246, 275, 260, 256, 248, 260, 254, 275, 260, 262, 251, 275, 258, 236, 232, 284, 225, 252, 258, 257, 268, 218, 248, 250, 272, 245, 244, 275, 261, 263, 248, 244, 240] payload_rows_per_dst=[5384, 5448, 5486, 5510, 5446, 5478, 5428, 5452] dup_rows_per_dst=[2640, 2734, 2696, 2944, 2734, 2728, 2794, 2634] combine_rows_per_src=[5384, 5448, 5486, 5510, 5446, 5478, 5428, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228888 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156991000 ingress_bytes=213385216 link_bytes=213385216 token_bytes=313753112 weight_movement_bytes=56623104 total_nvlink_bytes=370376216 weight_to_token_ratio=0.180470 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044552148e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38344 dup_rows=21818 dup_rows_remote=19082 dedup_ratio=0.332916 router_tokens_per_expert=[245, 265, 279, 265, 268, 244, 257, 250, 262, 236, 263, 255, 250, 292, 253, 244, 246, 269, 243, 266, 265, 267, 271, 288, 247, 274, 254, 274, 270, 267, 251, 231, 248, 253, 263, 237, 241, 270, 270, 269, 247, 226, 240, 259, 257, 245, 240, 290, 248, 222, 279, 279, 262, 270, 237, 270, 248, 222, 269, 260, 258, 269, 258, 281, 231, 246, 255, 244, 271, 236, 224, 227, 247, 270, 251, 262, 238, 240, 242, 238, 270, 266, 240, 243, 258, 258, 285, 231, 251, 278, 258, 262, 261, 239, 252, 251, 268, 250, 277, 287, 284, 263, 273, 238, 254, 278, 300, 249, 250, 257, 268, 231, 237, 284, 254, 246, 285, 234, 245, 218, 259, 248, 242, 257, 272, 259, 242, 236] payload_rows_per_dst=[5496, 5518, 5374, 5524, 5298, 5504, 5540, 5464] dup_rows_per_dst=[2760, 2848, 2736, 2740, 2546, 2702, 2914, 2572] combine_rows_per_src=[5496, 5518, 5374, 5524, 5298, 5504, 5540, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38274 dup_rows=21872 dup_rows_remote=19228 dedup_ratio=0.333740 router_tokens_per_expert=[256, 246, 232, 260, 251, 258, 282, 260, 239, 251, 249, 259, 266, 261, 270, 256, 247, 260, 236, 247, 256, 247, 276, 252, 255, 254, 272, 284, 250, 235, 251, 268, 274, 260, 262, 258, 266, 276, 274, 269, 261, 247, 266, 241, 269, 269, 246, 249, 251, 254, 233, 246, 244, 260, 248, 261, 244, 234, 250, 247, 262, 256, 285, 301, 236, 240, 243, 278, 253, 242, 261, 238, 258, 263, 251, 255, 266, 243, 245, 245, 266, 245, 249, 270, 267, 249, 280, 258, 243, 265, 260, 257, 284, 287, 272, 235, 242, 273, 256, 252, 261, 255, 238, 248, 248, 232, 253, 262, 247, 263, 242, 272, 279, 236, 271, 258, 252, 244, 256, 249, 251, 251, 266, 251, 247, 250, 252, 258] payload_rows_per_dst=[5492, 5460, 5522, 5478, 5390, 5490, 5452, 5380] dup_rows_per_dst=[2700, 2720, 2852, 2674, 2644, 2884, 2636, 2762] combine_rows_per_src=[5492, 5460, 5522, 5478, 5390, 5490, 5452, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229256 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156712840 ingress_bytes=213106688 link_bytes=213106688 token_bytes=313196424 weight_movement_bytes=56623104 total_nvlink_bytes=369819528 weight_to_token_ratio=0.180791 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041883429e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38256 dup_rows=21794 dup_rows_remote=19194 dedup_ratio=0.332550 router_tokens_per_expert=[279, 256, 233, 272, 246, 252, 275, 253, 271, 238, 238, 245, 265, 262, 247, 252, 236, 230, 239, 250, 258, 251, 271, 284, 265, 253, 212, 254, 284, 256, 252, 259, 271, 262, 280, 222, 265, 269, 250, 250, 254, 264, 287, 231, 243, 255, 237, 261, 251, 235, 272, 238, 269, 263, 264, 265, 252, 247, 236, 232, 266, 296, 249, 276, 246, 254, 291, 285, 240, 246, 270, 240, 250, 271, 253, 276, 253, 238, 264, 226, 263, 241, 222, 269, 245, 294, 269, 245, 267, 253, 250, 241, 249, 253, 241, 241, 259, 251, 256, 267, 276, 272, 275, 265, 261, 248, 260, 267, 255, 239, 250, 246, 278, 260, 254, 271, 258, 275, 260, 227, 250, 247, 266, 262, 263, 261, 260, 233] payload_rows_per_dst=[5486, 5440, 5482, 5434, 5468, 5486, 5458, 5488] dup_rows_per_dst=[2682, 2668, 2720, 2788, 2738, 2600, 2836, 2762] combine_rows_per_src=[5486, 5440, 5482, 5434, 5468, 5486, 5458, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38290 dup_rows=21628 dup_rows_remote=18696 dedup_ratio=0.330017 router_tokens_per_expert=[265, 245, 271, 275, 248, 239, 242, 253, 237, 253, 250, 241, 257, 251, 229, 249, 253, 253, 261, 273, 266, 258, 242, 234, 254, 270, 250, 241, 270, 257, 254, 239, 261, 271, 207, 229, 250, 245, 281, 252, 252, 252, 262, 237, 273, 260, 257, 266, 238, 280, 252, 232, 244, 286, 251, 239, 265, 267, 277, 236, 246, 264, 258, 258, 260, 250, 239, 267, 266, 255, 258, 222, 248, 267, 264, 237, 290, 263, 246, 270, 264, 243, 250, 250, 271, 263, 276, 277, 289, 255, 245, 297, 251, 254, 256, 249, 242, 287, 298, 245, 276, 264, 264, 253, 293, 252, 270, 257, 259, 240, 240, 253, 268, 250, 242, 274, 277, 271, 251, 255, 244, 232, 272, 249, 220, 234, 267, 249] payload_rows_per_dst=[5618, 5570, 5370, 5528, 5496, 5310, 5538, 5478] dup_rows_per_dst=[2932, 2706, 2614, 2764, 2708, 2530, 2742, 2632] combine_rows_per_src=[5618, 5570, 5370, 5528, 5496, 5310, 5538, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38382 dup_rows=21702 dup_rows_remote=18874 dedup_ratio=0.331146 router_tokens_per_expert=[248, 245, 267, 225, 259, 269, 237, 292, 218, 242, 275, 249, 231, 267, 270, 259, 272, 276, 266, 236, 269, 265, 252, 255, 258, 257, 274, 265, 264, 231, 244, 253, 256, 288, 252, 266, 264, 289, 274, 250, 304, 218, 253, 256, 262, 273, 253, 236, 278, 266, 242, 258, 282, 240, 272, 248, 251, 276, 249, 263, 234, 247, 259, 275, 224, 269, 237, 253, 272, 247, 242, 254, 284, 249, 241, 266, 229, 249, 230, 270, 272, 270, 230, 267, 253, 255, 253, 236, 258, 247, 257, 269, 229, 269, 255, 282, 234, 223, 263, 242, 268, 287, 283, 251, 256, 256, 255, 290, 242, 224, 290, 246, 273, 271, 227, 236, 250, 267, 244, 257, 227, 264, 264, 254, 243, 242, 253, 244] payload_rows_per_dst=[5378, 5552, 5582, 5452, 5388, 5456, 5534, 5492] dup_rows_per_dst=[2728, 2722, 2806, 2828, 2644, 2748, 2686, 2540] combine_rows_per_src=[5378, 5552, 5582, 5452, 5388, 5456, 5534, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38370 dup_rows=21640 dup_rows_remote=18920 dedup_ratio=0.330200 router_tokens_per_expert=[250, 271, 278, 239, 266, 271, 235, 263, 243, 261, 256, 250, 262, 243, 262, 254, 254, 259, 224, 252, 259, 244, 246, 307, 260, 244, 254, 241, 252, 272, 273, 239, 259, 278, 227, 241, 245, 307, 278, 274, 256, 248, 272, 248, 230, 277, 256, 257, 246, 251, 239, 245, 263, 235, 242, 286, 263, 246, 243, 269, 285, 252, 278, 278, 252, 261, 213, 294, 259, 257, 272, 283, 274, 238, 247, 247, 247, 251, 265, 245, 268, 238, 241, 257, 242, 244, 250, 247, 276, 226, 253, 244, 248, 230, 245, 232, 260, 275, 266, 239, 262, 258, 235, 251, 291, 278, 253, 252, 234, 249, 270, 268, 282, 236, 240, 247, 228, 275, 275, 234, 264, 263, 252, 257, 256, 244, 293, 277] payload_rows_per_dst=[5480, 5480, 5516, 5470, 5440, 5440, 5544, 5526] dup_rows_per_dst=[2728, 2680, 2790, 2772, 2528, 2670, 2752, 2720] combine_rows_per_src=[5480, 5480, 5516, 5470, 5440, 5440, 5544, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229704 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157286728 ingress_bytes=194805760 link_bytes=194805760 token_bytes=314343752 weight_movement_bytes=37748736 total_nvlink_bytes=352092488 weight_to_token_ratio=0.120087 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=230008 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157000312 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770616 weight_movement_bytes=18874368 total_nvlink_bytes=332644984 weight_to_token_ratio=0.060153 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229800 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156926376 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313622952 weight_movement_bytes=56623104 total_nvlink_bytes=370246056 weight_to_token_ratio=0.180545 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=227944 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157063784 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313899624 weight_movement_bytes=18874368 total_nvlink_bytes=332773992 weight_to_token_ratio=0.060129 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229024 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157441696 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654368 weight_movement_bytes=18874368 total_nvlink_bytes=333528736 weight_to_token_ratio=0.059984 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38370 dup_rows=21714 dup_rows_remote=19066 dedup_ratio=0.331329 router_tokens_per_expert=[291, 260, 286, 247, 256, 252, 220, 248, 280, 280, 243, 252, 267, 239, 270, 259, 259, 293, 272, 239, 241, 238, 244, 261, 285, 258, 256, 271, 258, 230, 222, 223, 271, 288, 265, 252, 294, 266, 258, 236, 218, 254, 240, 253, 257, 250, 251, 244, 230, 252, 247, 236, 252, 253, 241, 310, 253, 250, 250, 253, 268, 245, 253, 252, 247, 267, 279, 220, 254, 250, 251, 242, 261, 256, 255, 278, 288, 247, 274, 258, 259, 225, 274, 274, 237, 261, 260, 235, 268, 244, 245, 275, 240, 250, 281, 236, 249, 243, 243, 255, 250, 255, 251, 232, 254, 278, 279, 249, 218, 245, 255, 241, 273, 251, 272, 268, 286, 289, 263, 309, 263, 262, 249, 239, 231, 261, 262, 260] payload_rows_per_dst=[5492, 5452, 5420, 5438, 5544, 5460, 5424, 5592] dup_rows_per_dst=[2808, 2648, 2774, 2652, 2710, 2668, 2570, 2884] combine_rows_per_src=[5492, 5452, 5420, 5438, 5544, 5460, 5424, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229160 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157392680 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314556200 weight_movement_bytes=37748736 total_nvlink_bytes=352304936 weight_to_token_ratio=0.120006 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44008 payload_rows_remote=38448 dup_rows=21528 dup_rows_remote=18818 dedup_ratio=0.328491 router_tokens_per_expert=[246, 238, 259, 259, 238, 254, 248, 246, 258, 237, 264, 244, 252, 278, 258, 242, 268, 260, 245, 264, 257, 264, 273, 238, 234, 251, 267, 262, 267, 257, 265, 248, 255, 269, 253, 244, 272, 287, 236, 246, 270, 260, 304, 249, 207, 281, 265, 260, 253, 270, 264, 244, 275, 236, 240, 276, 239, 276, 249, 229, 240, 248, 273, 276, 240, 266, 236, 264, 266, 272, 258, 255, 271, 235, 240, 256, 230, 281, 268, 280, 258, 253, 257, 240, 243, 260, 266, 229, 254, 272, 257, 261, 258, 244, 248, 241, 239, 248, 254, 249, 284, 244, 253, 276, 243, 261, 253, 239, 266, 269, 291, 266, 255, 271, 263, 246, 242, 276, 241, 255, 242, 247, 280, 245, 280, 258, 247, 239] payload_rows_per_dst=[5412, 5508, 5554, 5492, 5560, 5396, 5560, 5526] dup_rows_per_dst=[2630, 2732, 2762, 2684, 2676, 2686, 2710, 2648] combine_rows_per_src=[5412, 5508, 5554, 5492, 5560, 5396, 5560, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229744 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157393264 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556784 weight_movement_bytes=18874368 total_nvlink_bytes=333431152 weight_to_token_ratio=0.060003 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38102 dup_rows=21896 dup_rows_remote=19108 dedup_ratio=0.334106 router_tokens_per_expert=[234, 275, 254, 270, 243, 249, 257, 252, 264, 267, 267, 265, 256, 233, 254, 246, 261, 247, 247, 266, 258, 257, 239, 257, 252, 266, 223, 251, 258, 263, 238, 245, 259, 210, 260, 276, 264, 262, 268, 283, 236, 278, 268, 266, 265, 243, 273, 252, 254, 282, 265, 264, 255, 254, 259, 271, 260, 267, 266, 270, 248, 264, 256, 274, 261, 279, 242, 236, 259, 261, 233, 273, 274, 281, 256, 277, 244, 247, 264, 240, 267, 261, 265, 240, 256, 243, 257, 216, 257, 283, 270, 255, 239, 245, 254, 270, 258, 251, 279, 268, 278, 264, 226, 223, 239, 251, 270, 259, 256, 251, 255, 250, 244, 256, 241, 233, 248, 260, 267, 264, 246, 251, 245, 245, 263, 227, 257, 252] payload_rows_per_dst=[5438, 5398, 5538, 5496, 5460, 5492, 5422, 5396] dup_rows_per_dst=[2734, 2658, 2788, 2922, 2794, 2664, 2734, 2602] combine_rows_per_src=[5438, 5398, 5538, 5496, 5460, 5492, 5422, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38146 dup_rows=21886 dup_rows_remote=19072 dedup_ratio=0.333954 router_tokens_per_expert=[216, 242, 265, 248, 263, 243, 242, 272, 283, 270, 249, 277, 256, 256, 243, 264, 272, 240, 257, 286, 249, 227, 246, 264, 230, 219, 266, 252, 241, 256, 243, 265, 231, 262, 235, 277, 269, 247, 250, 256, 280, 241, 247, 257, 250, 276, 241, 261, 247, 252, 280, 253, 268, 251, 249, 269, 243, 255, 245, 252, 255, 260, 277, 253, 259, 286, 257, 247, 255, 291, 250, 276, 245, 227, 274, 276, 248, 242, 261, 265, 282, 270, 259, 246, 261, 254, 274, 239, 263, 278, 244, 266, 270, 255, 248, 270, 269, 260, 241, 241, 237, 215, 242, 240, 256, 250, 276, 245, 237, 228, 267, 255, 267, 258, 237, 257, 261, 283, 253, 263, 250, 245, 275, 245, 266, 267, 285, 268] payload_rows_per_dst=[5384, 5422, 5464, 5442, 5504, 5572, 5366, 5496] dup_rows_per_dst=[2794, 2604, 2696, 2776, 2814, 2786, 2552, 2864] combine_rows_per_src=[5384, 5422, 5464, 5442, 5504, 5572, 5366, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38170 dup_rows=21896 dup_rows_remote=19088 dedup_ratio=0.334106 router_tokens_per_expert=[218, 258, 271, 268, 237, 222, 243, 262, 260, 234, 252, 249, 252, 263, 254, 260, 243, 277, 246, 246, 255, 255, 275, 250, 268, 255, 237, 266, 245, 278, 258, 256, 238, 230, 281, 253, 243, 260, 255, 279, 268, 249, 253, 252, 242, 263, 256, 257, 260, 233, 269, 263, 245, 264, 252, 273, 275, 278, 249, 244, 238, 235, 264, 234, 247, 279, 273, 243, 231, 250, 242, 258, 263, 273, 252, 267, 254, 255, 256, 234, 245, 290, 238, 260, 280, 254, 261, 276, 249, 255, 233, 250, 248, 266, 255, 279, 269, 255, 245, 237, 256, 256, 251, 258, 258, 266, 250, 294, 245, 257, 273, 266, 273, 289, 221, 260, 223, 272, 248, 247, 261, 287, 261, 257, 257, 234, 253, 305] payload_rows_per_dst=[5368, 5386, 5444, 5488, 5414, 5470, 5490, 5580] dup_rows_per_dst=[2638, 2834, 2714, 2664, 2740, 2808, 2782, 2716] combine_rows_per_src=[5368, 5386, 5444, 5488, 5414, 5470, 5490, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=229064 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157712072 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315195080 weight_movement_bytes=18874368 total_nvlink_bytes=334069448 weight_to_token_ratio=0.059882 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38238 dup_rows=21822 dup_rows_remote=18986 dedup_ratio=0.332977 router_tokens_per_expert=[249, 266, 240, 261, 276, 264, 250, 256, 284, 257, 258, 276, 265, 257, 237, 260, 259, 226, 267, 233, 262, 254, 248, 226, 260, 265, 227, 245, 268, 271, 258, 307, 249, 268, 277, 256, 241, 288, 235, 217, 247, 266, 265, 256, 277, 225, 239, 245, 228, 269, 258, 280, 265, 296, 253, 263, 230, 278, 254, 236, 266, 221, 246, 248, 245, 269, 247, 215, 253, 239, 276, 266, 277, 236, 278, 267, 247, 257, 251, 244, 281, 263, 243, 277, 261, 240, 253, 261, 265, 238, 255, 240, 263, 261, 257, 236, 249, 249, 266, 253, 282, 249, 273, 242, 282, 255, 246, 250, 253, 250, 249, 261, 257, 251, 240, 270, 248, 251, 269, 271, 267, 297, 243, 194, 257, 268, 269, 272] payload_rows_per_dst=[5476, 5512, 5464, 5446, 5446, 5442, 5518, 5410] dup_rows_per_dst=[2836, 2640, 2802, 2736, 2688, 2582, 2864, 2674] combine_rows_per_src=[5476, 5512, 5464, 5446, 5446, 5442, 5518, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38294 dup_rows=21822 dup_rows_remote=19222 dedup_ratio=0.332977 router_tokens_per_expert=[227, 247, 277, 265, 233, 259, 249, 245, 236, 254, 238, 259, 252, 258, 249, 285, 255, 261, 272, 253, 248, 254, 257, 260, 262, 271, 231, 266, 247, 248, 271, 274, 262, 256, 274, 255, 262, 273, 262, 260, 248, 270, 262, 256, 267, 268, 236, 268, 234, 240, 227, 247, 244, 263, 232, 247, 293, 251, 258, 254, 266, 252, 246, 256, 264, 268, 241, 265, 242, 280, 277, 215, 227, 251, 262, 264, 250, 234, 257, 244, 255, 274, 256, 232, 246, 260, 274, 233, 252, 258, 235, 228, 244, 257, 269, 265, 255, 245, 281, 263, 259, 259, 260, 251, 285, 248, 245, 277, 252, 282, 276, 270, 282, 248, 248, 267, 241, 264, 242, 243, 255, 255, 246, 242, 299, 279, 256, 262] payload_rows_per_dst=[5412, 5508, 5560, 5420, 5430, 5354, 5508, 5522] dup_rows_per_dst=[2654, 2752, 2798, 2600, 2652, 2722, 2908, 2736] combine_rows_per_src=[5412, 5508, 5560, 5420, 5430, 5354, 5508, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=228840 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156294632 ingress_bytes=193814528 link_bytes=193814528 token_bytes=312360424 weight_movement_bytes=37748736 total_nvlink_bytes=350109160 weight_to_token_ratio=0.120850 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857035445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38226 dup_rows=21818 dup_rows_remote=18928 dedup_ratio=0.332916 router_tokens_per_expert=[266, 254, 277, 255, 241, 253, 231, 257, 236, 269, 265, 240, 276, 254, 277, 239, 239, 249, 274, 247, 268, 267, 253, 293, 236, 255, 246, 236, 236, 225, 248, 248, 279, 249, 257, 259, 271, 247, 236, 249, 265, 255, 249, 258, 256, 288, 242, 275, 254, 252, 256, 263, 271, 240, 216, 249, 272, 256, 255, 248, 269, 254, 263, 258, 245, 247, 274, 235, 254, 282, 257, 266, 245, 229, 272, 257, 252, 256, 241, 250, 270, 246, 252, 240, 274, 251, 248, 249, 251, 286, 240, 248, 272, 248, 280, 278, 239, 262, 255, 245, 257, 255, 264, 277, 245, 233, 259, 264, 280, 265, 243, 251, 254, 262, 247, 262, 248, 269, 248, 249, 272, 270, 296, 253, 220, 267, 270, 271] payload_rows_per_dst=[5474, 5458, 5516, 5420, 5406, 5474, 5478, 5492] dup_rows_per_dst=[2706, 2582, 2688, 2628, 2718, 2792, 2814, 2890] combine_rows_per_src=[5474, 5458, 5516, 5420, 5406, 5474, 5478, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228872 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156474888 ingress_bytes=231743488 link_bytes=231743488 token_bytes=312720904 weight_movement_bytes=75497472 total_nvlink_bytes=388218376 weight_to_token_ratio=0.241421 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220452077e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229032 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573352 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917672 weight_movement_bytes=18874368 total_nvlink_bytes=331792040 weight_to_token_ratio=0.060317 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228896 combine_payload_bytes=156622848 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156851744 ingress_bytes=213245952 link_bytes=213245952 token_bytes=313474592 weight_movement_bytes=56623104 total_nvlink_bytes=370097696 weight_to_token_ratio=0.180631 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043217789e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38286 dup_rows=21820 dup_rows_remote=19066 dedup_ratio=0.332947 router_tokens_per_expert=[271, 249, 253, 254, 274, 257, 259, 234, 261, 269, 241, 261, 265, 260, 273, 248, 264, 245, 252, 269, 263, 251, 245, 261, 245, 250, 263, 259, 277, 255, 250, 243, 248, 263, 261, 244, 249, 250, 270, 289, 259, 237, 252, 251, 247, 266, 248, 253, 253, 239, 273, 269, 246, 246, 260, 257, 260, 240, 248, 242, 263, 265, 260, 265, 237, 272, 272, 238, 250, 255, 297, 277, 260, 271, 265, 275, 231, 249, 220, 281, 250, 258, 267, 271, 256, 262, 255, 239, 254, 252, 259, 259, 262, 258, 248, 243, 243, 254, 249, 255, 242, 224, 288, 255, 251, 255, 248, 242, 239, 246, 254, 233, 242, 258, 280, 253, 272, 262, 248, 256, 244, 260, 255, 269, 279, 260, 265, 250] payload_rows_per_dst=[5482, 5430, 5420, 5444, 5496, 5468, 5378, 5598] dup_rows_per_dst=[2776, 2754, 2754, 2728, 2804, 2718, 2578, 2708] combine_rows_per_src=[5482, 5430, 5420, 5444, 5496, 5468, 5378, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=230064 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157082288 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313934512 weight_movement_bytes=56623104 total_nvlink_bytes=370557616 weight_to_token_ratio=0.180366 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=228616 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156802312 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313376008 weight_movement_bytes=37748736 total_nvlink_bytes=351124744 weight_to_token_ratio=0.120458 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38110 dup_rows=21998 dup_rows_remote=19226 dedup_ratio=0.335663 router_tokens_per_expert=[236, 248, 262, 278, 253, 268, 240, 291, 266, 243, 238, 256, 258, 253, 236, 255, 275, 255, 252, 253, 278, 280, 264, 248, 273, 274, 262, 239, 236, 265, 280, 273, 238, 234, 262, 232, 250, 261, 239, 234, 286, 237, 276, 245, 254, 253, 246, 263, 223, 285, 268, 241, 245, 229, 255, 242, 233, 287, 262, 250, 273, 272, 267, 254, 260, 264, 293, 261, 256, 271, 262, 246, 218, 253, 231, 263, 249, 253, 259, 276, 235, 273, 263, 275, 260, 235, 248, 259, 246, 244, 269, 245, 235, 252, 246, 257, 263, 258, 242, 246, 269, 224, 267, 243, 261, 240, 274, 254, 245, 278, 273, 263, 267, 248, 248, 257, 243, 257, 255, 278, 268, 265, 254, 263, 228, 272, 263, 261] payload_rows_per_dst=[5390, 5522, 5368, 5464, 5462, 5398, 5428, 5506] dup_rows_per_dst=[2772, 2892, 2652, 2708, 2768, 2686, 2772, 2748] combine_rows_per_src=[5390, 5522, 5368, 5464, 5462, 5398, 5428, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229408 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048864 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868320 weight_movement_bytes=18874368 total_nvlink_bytes=332742688 weight_to_token_ratio=0.060135 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38256 dup_rows=21788 dup_rows_remote=19106 dedup_ratio=0.332458 router_tokens_per_expert=[257, 270, 273, 302, 240, 237, 270, 274, 257, 258, 237, 249, 275, 233, 255, 270, 233, 264, 278, 256, 227, 237, 252, 235, 268, 253, 230, 230, 249, 242, 240, 267, 264, 262, 300, 262, 207, 264, 270, 251, 229, 239, 245, 247, 250, 265, 282, 250, 244, 256, 246, 241, 271, 242, 281, 269, 257, 261, 277, 254, 247, 261, 239, 287, 262, 239, 247, 251, 253, 272, 257, 279, 248, 278, 280, 238, 232, 231, 240, 258, 244, 248, 216, 272, 258, 260, 262, 274, 271, 219, 253, 253, 271, 251, 276, 275, 251, 261, 233, 234, 252, 248, 260, 249, 235, 267, 255, 294, 269, 263, 272, 279, 253, 260, 252, 271, 241, 232, 282, 258, 254, 246, 276, 250, 271, 256, 268, 270] payload_rows_per_dst=[5576, 5300, 5492, 5520, 5364, 5464, 5500, 5532] dup_rows_per_dst=[2738, 2622, 2682, 2746, 2766, 2742, 2744, 2748] combine_rows_per_src=[5576, 5300, 5492, 5520, 5364, 5464, 5500, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=229344 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156327904 ingress_bytes=174972928 link_bytes=174972928 token_bytes=312426464 weight_movement_bytes=18874368 total_nvlink_bytes=331300832 weight_to_token_ratio=0.060412 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676504504e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38212 dup_rows=21888 dup_rows_remote=19210 dedup_ratio=0.333984 router_tokens_per_expert=[274, 249, 243, 284, 269, 259, 274, 253, 255, 261, 267, 239, 276, 228, 255, 239, 281, 255, 244, 276, 239, 234, 272, 262, 270, 275, 260, 297, 240, 240, 242, 255, 293, 262, 235, 260, 248, 274, 267, 250, 263, 263, 274, 269, 262, 291, 262, 243, 255, 261, 253, 263, 242, 262, 266, 256, 258, 254, 246, 260, 261, 280, 276, 237, 272, 263, 239, 245, 239, 234, 270, 262, 244, 246, 275, 243, 247, 269, 261, 248, 252, 264, 242, 259, 226, 236, 225, 282, 269, 270, 231, 247, 244, 250, 271, 272, 241, 231, 260, 252, 272, 247, 273, 239, 263, 255, 268, 284, 253, 241, 243, 248, 255, 242, 255, 239, 259, 228, 268, 256, 256, 251, 247, 272, 236, 216, 250, 258] payload_rows_per_dst=[5522, 5480, 5562, 5498, 5436, 5350, 5460, 5340] dup_rows_per_dst=[2728, 2804, 2870, 2762, 2678, 2730, 2680, 2636] combine_rows_per_src=[5522, 5480, 5562, 5498, 5436, 5350, 5460, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38268 dup_rows=21880 dup_rows_remote=19128 dedup_ratio=0.333862 router_tokens_per_expert=[261, 270, 251, 289, 227, 258, 263, 255, 280, 265, 283, 248, 263, 259, 256, 269, 237, 237, 272, 248, 253, 247, 247, 257, 239, 249, 254, 267, 279, 282, 232, 265, 227, 248, 237, 265, 274, 251, 280, 269, 241, 267, 275, 257, 253, 257, 214, 247, 297, 243, 277, 276, 267, 272, 259, 246, 262, 259, 250, 293, 257, 251, 227, 271, 245, 255, 241, 255, 266, 225, 249, 280, 236, 287, 262, 252, 279, 215, 284, 255, 236, 251, 261, 253, 270, 257, 248, 242, 237, 268, 259, 239, 248, 224, 286, 291, 242, 243, 273, 275, 251, 262, 273, 248, 233, 248, 262, 252, 232, 270, 222, 255, 250, 234, 263, 252, 252, 266, 263, 257, 264, 222, 259, 252, 266, 225, 242, 273] payload_rows_per_dst=[5538, 5370, 5442, 5598, 5440, 5388, 5424, 5456] dup_rows_per_dst=[2856, 2760, 2682, 2816, 2732, 2752, 2658, 2624] combine_rows_per_src=[5538, 5370, 5442, 5598, 5440, 5388, 5424, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229448 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156926024 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313622600 weight_movement_bytes=56623104 total_nvlink_bytes=370245704 weight_to_token_ratio=0.180545 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=37912 dup_rows=21944 dup_rows_remote=18996 dedup_ratio=0.334839 router_tokens_per_expert=[257, 258, 259, 248, 239, 258, 242, 257, 245, 275, 234, 274, 283, 264, 268, 226, 241, 250, 252, 255, 262, 276, 292, 250, 247, 227, 254, 292, 265, 236, 272, 270, 253, 247, 269, 247, 262, 285, 262, 254, 279, 258, 237, 278, 254, 244, 255, 249, 240, 253, 249, 250, 234, 266, 242, 242, 239, 246, 263, 262, 257, 254, 222, 255, 280, 243, 256, 262, 254, 273, 254, 260, 242, 269, 252, 247, 249, 273, 246, 262, 280, 240, 265, 266, 260, 260, 260, 246, 249, 259, 239, 230, 260, 243, 252, 247, 248, 243, 242, 250, 269, 261, 248, 263, 236, 274, 248, 237, 263, 243, 262, 253, 267, 293, 285, 283, 236, 273, 239, 253, 263, 241, 280, 267, 251, 249, 285, 250] payload_rows_per_dst=[5680, 5282, 5428, 5476, 5474, 5382, 5436, 5434] dup_rows_per_dst=[2948, 2546, 2864, 2716, 2770, 2704, 2644, 2752] combine_rows_per_src=[5680, 5282, 5428, 5476, 5474, 5382, 5436, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38016 dup_rows=21922 dup_rows_remote=18782 dedup_ratio=0.334503 router_tokens_per_expert=[253, 251, 250, 252, 250, 243, 243, 234, 284, 253, 252, 264, 250, 248, 271, 283, 243, 259, 269, 276, 239, 279, 248, 261, 288, 246, 255, 279, 256, 236, 247, 249, 244, 268, 251, 244, 272, 247, 262, 269, 241, 270, 271, 260, 247, 246, 268, 304, 269, 252, 279, 256, 239, 246, 238, 269, 294, 250, 238, 265, 299, 237, 258, 239, 248, 280, 224, 251, 265, 251, 234, 269, 266, 233, 253, 236, 250, 257, 227, 252, 263, 246, 245, 217, 258, 278, 269, 272, 252, 227, 256, 244, 240, 264, 239, 259, 258, 273, 257, 263, 265, 247, 234, 236, 275, 257, 267, 262, 241, 272, 246, 254, 280, 247, 231, 237, 251, 281, 265, 248, 268, 248, 248, 255, 288, 265, 257, 264] payload_rows_per_dst=[5404, 5302, 5460, 5472, 5384, 5480, 5514, 5598] dup_rows_per_dst=[2592, 2486, 2738, 2692, 2718, 2690, 2866, 3140] combine_rows_per_src=[5404, 5302, 5460, 5472, 5384, 5480, 5514, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38244 dup_rows=21796 dup_rows_remote=19084 dedup_ratio=0.332581 router_tokens_per_expert=[260, 264, 244, 235, 265, 250, 231, 241, 253, 240, 239, 260, 249, 264, 256, 237, 258, 264, 274, 263, 231, 257, 270, 249, 276, 263, 218, 296, 255, 234, 264, 236, 254, 237, 273, 255, 257, 239, 289, 250, 268, 246, 247, 251, 244, 242, 263, 273, 268, 245, 237, 254, 248, 245, 250, 257, 262, 254, 255, 264, 256, 265, 287, 257, 255, 289, 276, 277, 239, 280, 254, 266, 280, 263, 243, 267, 262, 273, 271, 224, 230, 259, 256, 250, 266, 256, 253, 264, 264, 241, 242, 273, 272, 260, 239, 236, 237, 265, 270, 262, 228, 277, 232, 262, 249, 267, 259, 280, 244, 266, 257, 262, 245, 279, 235, 254, 263, 238, 267, 256, 254, 247, 274, 256, 256, 257, 273, 229] payload_rows_per_dst=[5386, 5446, 5508, 5496, 5538, 5398, 5528, 5440] dup_rows_per_dst=[2590, 2770, 2668, 2712, 2900, 2724, 2706, 2726] combine_rows_per_src=[5386, 5446, 5508, 5496, 5538, 5398, 5528, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229688 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156746040 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313262392 weight_movement_bytes=18874368 total_nvlink_bytes=332136760 weight_to_token_ratio=0.060251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229584 combine_payload_bytes=156745728 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156975312 ingress_bytes=156745728 link_bytes=156975312 token_bytes=313721040 weight_movement_bytes=0 total_nvlink_bytes=313721040 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.504060203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38272 dup_rows=21798 dup_rows_remote=19020 dedup_ratio=0.332611 router_tokens_per_expert=[284, 239, 240, 260, 270, 240, 276, 252, 274, 268, 272, 237, 239, 274, 244, 266, 261, 256, 247, 249, 253, 257, 232, 279, 242, 278, 238, 260, 233, 298, 270, 269, 241, 254, 297, 293, 273, 245, 263, 272, 271, 246, 266, 248, 277, 264, 259, 269, 252, 255, 239, 257, 250, 270, 255, 264, 249, 248, 269, 215, 274, 235, 221, 271, 242, 262, 264, 272, 263, 276, 257, 246, 275, 265, 265, 240, 269, 262, 221, 253, 263, 249, 254, 268, 249, 250, 265, 263, 229, 255, 267, 250, 242, 253, 270, 272, 273, 245, 230, 244, 241, 241, 252, 271, 276, 254, 267, 237, 265, 241, 247, 232, 264, 244, 263, 230, 266, 242, 255, 240, 234, 247, 264, 243, 244, 242, 245, 279] payload_rows_per_dst=[5476, 5466, 5610, 5324, 5506, 5510, 5380, 5466] dup_rows_per_dst=[2794, 2778, 2866, 2724, 2758, 2688, 2652, 2538] combine_rows_per_src=[5476, 5466, 5610, 5324, 5506, 5510, 5380, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155287552 dispatch_meta_bytes=227632 combine_payload_bytes=155287552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155515184 ingress_bytes=174161920 link_bytes=174161920 token_bytes=310802736 weight_movement_bytes=18874368 total_nvlink_bytes=329677104 weight_to_token_ratio=0.060728 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.668733824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155713536 dispatch_meta_bytes=227192 combine_payload_bytes=155713536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155940728 ingress_bytes=174587904 link_bytes=174587904 token_bytes=311654264 weight_movement_bytes=18874368 total_nvlink_bytes=330528632 weight_to_token_ratio=0.060562 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672815394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229312 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876736 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524160 weight_movement_bytes=37748736 total_nvlink_bytes=351272896 weight_to_token_ratio=0.120401 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38080 dup_rows=21880 dup_rows_remote=19104 dedup_ratio=0.333862 router_tokens_per_expert=[237, 255, 275, 251, 246, 252, 251, 238, 275, 240, 218, 256, 261, 275, 235, 269, 259, 249, 266, 271, 246, 251, 254, 263, 244, 297, 252, 255, 271, 262, 250, 244, 233, 216, 236, 283, 223, 263, 234, 282, 259, 250, 266, 252, 272, 241, 255, 259, 241, 245, 243, 247, 281, 275, 246, 250, 258, 255, 226, 257, 256, 253, 242, 226, 279, 270, 253, 283, 258, 275, 253, 282, 250, 276, 254, 239, 224, 259, 261, 273, 242, 236, 270, 283, 259, 268, 256, 257, 257, 272, 254, 251, 243, 254, 257, 260, 271, 277, 270, 253, 262, 264, 264, 266, 250, 270, 276, 229, 249, 276, 264, 235, 232, 263, 251, 270, 246, 285, 257, 229, 236, 258, 273, 263, 271, 227, 261, 269] payload_rows_per_dst=[5414, 5462, 5356, 5306, 5580, 5552, 5576, 5410] dup_rows_per_dst=[2654, 2806, 2692, 2696, 2798, 2686, 2776, 2772] combine_rows_per_src=[5414, 5462, 5356, 5306, 5580, 5552, 5576, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229168 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156991280 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313753392 weight_movement_bytes=37748736 total_nvlink_bytes=351502128 weight_to_token_ratio=0.120313 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38302 dup_rows=21764 dup_rows_remote=19114 dedup_ratio=0.332092 router_tokens_per_expert=[297, 242, 257, 270, 272, 260, 250, 223, 244, 253, 287, 224, 276, 244, 259, 246, 259, 270, 263, 262, 233, 249, 240, 240, 262, 273, 265, 247, 257, 261, 272, 272, 247, 243, 250, 256, 272, 251, 270, 249, 277, 246, 261, 226, 255, 250, 245, 262, 239, 265, 271, 237, 270, 238, 251, 263, 263, 272, 283, 256, 256, 240, 241, 254, 250, 252, 254, 260, 251, 269, 261, 257, 237, 235, 258, 246, 286, 262, 248, 254, 283, 270, 233, 246, 255, 242, 239, 273, 262, 263, 271, 257, 259, 256, 273, 255, 244, 218, 251, 255, 261, 285, 254, 256, 296, 290, 265, 252, 254, 262, 256, 242, 258, 233, 265, 234, 255, 233, 259, 278, 257, 246, 251, 247, 265, 261, 256, 224] payload_rows_per_dst=[5508, 5496, 5470, 5420, 5442, 5498, 5516, 5422] dup_rows_per_dst=[2700, 2754, 2650, 2778, 2718, 2776, 2766, 2622] combine_rows_per_src=[5508, 5496, 5470, 5420, 5442, 5498, 5516, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228736 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156204416 ingress_bytes=193724416 link_bytes=193724416 token_bytes=312180096 weight_movement_bytes=37748736 total_nvlink_bytes=349928832 weight_to_token_ratio=0.120920 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856172036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38356 dup_rows=21798 dup_rows_remote=19232 dedup_ratio=0.332611 router_tokens_per_expert=[272, 263, 229, 269, 240, 252, 234, 251, 259, 284, 287, 265, 277, 267, 289, 240, 263, 282, 242, 265, 269, 257, 247, 280, 253, 228, 267, 242, 270, 241, 262, 283, 240, 256, 252, 266, 262, 256, 246, 258, 262, 271, 250, 246, 225, 257, 271, 285, 245, 276, 272, 253, 246, 251, 245, 262, 257, 265, 282, 254, 250, 254, 253, 251, 260, 273, 236, 244, 236, 232, 244, 247, 265, 236, 229, 259, 259, 252, 258, 244, 249, 291, 244, 266, 258, 263, 256, 260, 212, 279, 267, 249, 238, 252, 209, 261, 259, 278, 262, 242, 257, 233, 273, 257, 280, 261, 274, 261, 243, 277, 241, 268, 249, 272, 245, 260, 247, 231, 228, 263, 244, 266, 256, 262, 263, 259, 248, 233] payload_rows_per_dst=[5548, 5522, 5480, 5484, 5382, 5390, 5540, 5392] dup_rows_per_dst=[2808, 2780, 2726, 2748, 2566, 2718, 2792, 2660] combine_rows_per_src=[5548, 5522, 5480, 5484, 5382, 5390, 5540, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38192 dup_rows=21840 dup_rows_remote=19066 dedup_ratio=0.333252 router_tokens_per_expert=[254, 245, 265, 282, 249, 242, 247, 258, 267, 231, 272, 266, 250, 277, 262, 259, 259, 263, 256, 283, 279, 245, 242, 247, 247, 246, 259, 223, 231, 269, 270, 250, 263, 258, 256, 257, 236, 236, 272, 244, 248, 227, 237, 259, 267, 250, 232, 239, 255, 257, 242, 270, 238, 258, 262, 260, 284, 272, 245, 234, 231, 278, 233, 261, 253, 250, 262, 281, 234, 223, 262, 276, 301, 251, 238, 229, 250, 256, 251, 293, 288, 257, 263, 264, 249, 238, 265, 263, 270, 235, 259, 250, 268, 245, 285, 240, 237, 281, 250, 278, 269, 239, 254, 262, 253, 250, 278, 269, 271, 253, 256, 263, 261, 240, 262, 282, 241, 266, 264, 252, 266, 234, 267, 267, 253, 250, 272, 223] payload_rows_per_dst=[5468, 5458, 5436, 5476, 5488, 5504, 5438, 5428] dup_rows_per_dst=[2784, 2680, 2526, 2684, 2732, 2774, 2888, 2772] combine_rows_per_src=[5468, 5458, 5436, 5476, 5488, 5504, 5438, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229664 combine_payload_bytes=156884992 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157114656 ingress_bytes=213508096 link_bytes=213508096 token_bytes=313999648 weight_movement_bytes=56623104 total_nvlink_bytes=370622752 weight_to_token_ratio=0.180329 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045729523e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38052 dup_rows=21996 dup_rows_remote=19344 dedup_ratio=0.335632 router_tokens_per_expert=[265, 245, 245, 260, 255, 241, 268, 252, 253, 250, 269, 273, 239, 256, 262, 255, 267, 261, 279, 267, 270, 259, 256, 246, 277, 254, 271, 268, 312, 250, 266, 232, 250, 268, 278, 258, 263, 238, 276, 259, 256, 257, 284, 242, 258, 255, 262, 273, 219, 236, 258, 230, 252, 272, 272, 229, 272, 277, 285, 242, 275, 242, 263, 239, 243, 259, 248, 232, 248, 273, 272, 253, 264, 261, 263, 264, 235, 238, 272, 241, 277, 288, 241, 251, 255, 256, 264, 288, 234, 257, 250, 252, 263, 260, 249, 241, 250, 248, 257, 249, 259, 220, 221, 232, 250, 245, 258, 280, 235, 255, 246, 254, 258, 230, 272, 240, 256, 242, 258, 238, 273, 243, 246, 267, 234, 282, 265, 250] payload_rows_per_dst=[5276, 5486, 5344, 5384, 5610, 5486, 5466, 5488] dup_rows_per_dst=[2420, 2914, 2564, 2770, 3010, 2734, 2932, 2652] combine_rows_per_src=[5276, 5486, 5344, 5384, 5610, 5486, 5466, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38080 dup_rows=21838 dup_rows_remote=18958 dedup_ratio=0.333221 router_tokens_per_expert=[274, 224, 271, 241, 238, 237, 276, 266, 274, 252, 266, 254, 240, 238, 252, 266, 273, 253, 251, 275, 230, 278, 253, 259, 258, 216, 272, 275, 260, 279, 259, 258, 240, 257, 298, 250, 248, 250, 232, 261, 234, 271, 256, 236, 236, 254, 239, 259, 288, 261, 259, 207, 260, 246, 236, 268, 238, 227, 279, 248, 243, 257, 285, 256, 247, 254, 265, 271, 244, 263, 245, 263, 235, 248, 266, 242, 272, 255, 259, 267, 240, 269, 248, 266, 252, 252, 254, 247, 234, 286, 245, 241, 261, 254, 246, 237, 239, 229, 265, 269, 258, 285, 282, 260, 242, 284, 251, 282, 259, 269, 264, 250, 270, 252, 240, 257, 234, 266, 287, 275, 258, 273, 255, 275, 271, 261, 238, 243] payload_rows_per_dst=[5618, 5490, 5402, 5448, 5400, 5456, 5528, 5356] dup_rows_per_dst=[2880, 2942, 2554, 2686, 2778, 2608, 2800, 2590] combine_rows_per_src=[5618, 5490, 5402, 5448, 5400, 5456, 5528, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38116 dup_rows=21996 dup_rows_remote=19254 dedup_ratio=0.335632 router_tokens_per_expert=[266, 265, 266, 247, 239, 258, 231, 278, 239, 267, 255, 261, 251, 256, 267, 270, 251, 265, 244, 268, 243, 256, 265, 239, 261, 267, 280, 254, 257, 239, 255, 271, 247, 258, 270, 269, 220, 271, 249, 254, 278, 268, 252, 249, 249, 261, 248, 239, 254, 250, 254, 254, 258, 253, 256, 243, 268, 230, 259, 255, 271, 260, 261, 257, 225, 275, 236, 243, 275, 237, 243, 251, 277, 242, 277, 226, 275, 266, 239, 234, 272, 285, 275, 267, 242, 244, 257, 251, 270, 262, 256, 259, 245, 240, 262, 263, 224, 280, 258, 253, 268, 241, 237, 249, 245, 277, 263, 242, 266, 256, 238, 233, 273, 265, 250, 272, 282, 269, 250, 237, 231, 266, 278, 261, 261, 256, 272, 248] payload_rows_per_dst=[5436, 5442, 5470, 5424, 5402, 5474, 5394, 5498] dup_rows_per_dst=[2796, 2788, 2694, 2742, 2640, 2826, 2666, 2844] combine_rows_per_src=[5436, 5442, 5470, 5424, 5402, 5474, 5394, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=230352 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157336528 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314442704 weight_movement_bytes=18874368 total_nvlink_bytes=333317072 weight_to_token_ratio=0.060025 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44050 payload_rows_remote=38572 dup_rows=21486 dup_rows_remote=18850 dedup_ratio=0.327850 router_tokens_per_expert=[274, 233, 259, 282, 244, 270, 250, 231, 262, 275, 245, 266, 267, 253, 266, 213, 239, 243, 264, 241, 274, 250, 252, 305, 253, 244, 241, 269, 219, 247, 246, 270, 245, 236, 274, 254, 237, 245, 278, 248, 259, 258, 260, 274, 260, 260, 246, 255, 252, 275, 267, 266, 248, 254, 273, 257, 269, 259, 249, 276, 244, 272, 247, 259, 250, 253, 277, 261, 290, 267, 238, 247, 262, 267, 256, 240, 239, 227, 256, 235, 263, 271, 268, 254, 224, 244, 238, 273, 258, 266, 253, 278, 278, 238, 242, 235, 261, 249, 234, 255, 263, 273, 274, 252, 252, 255, 254, 299, 259, 238, 275, 254, 276, 261, 234, 261, 227, 257, 264, 247, 264, 243, 271, 244, 247, 249, 258, 267] payload_rows_per_dst=[5512, 5478, 5502, 5570, 5462, 5536, 5506, 5484] dup_rows_per_dst=[2668, 2636, 2676, 2764, 2668, 2630, 2788, 2656] combine_rows_per_src=[5512, 5478, 5502, 5570, 5462, 5536, 5506, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229032 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663464 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313097896 weight_movement_bytes=18874368 total_nvlink_bytes=331972264 weight_to_token_ratio=0.060283 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155860992 dispatch_meta_bytes=229584 combine_payload_bytes=155860992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156090576 ingress_bytes=193609728 link_bytes=193609728 token_bytes=311951568 weight_movement_bytes=37748736 total_nvlink_bytes=349700304 weight_to_token_ratio=0.121008 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855073152e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228152 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156203832 ingress_bytes=174850048 link_bytes=174850048 token_bytes=312179512 weight_movement_bytes=18874368 total_nvlink_bytes=331053880 weight_to_token_ratio=0.060460 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675327128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229480 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352616 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475752 weight_movement_bytes=18874368 total_nvlink_bytes=331350120 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38204 dup_rows=21788 dup_rows_remote=19134 dedup_ratio=0.332458 router_tokens_per_expert=[265, 241, 231, 272, 264, 245, 243, 247, 241, 280, 217, 262, 234, 249, 253, 241, 236, 280, 261, 264, 247, 262, 236, 248, 245, 284, 244, 277, 264, 270, 247, 269, 239, 281, 261, 258, 252, 250, 252, 238, 248, 252, 264, 287, 282, 260, 242, 249, 270, 274, 247, 240, 245, 261, 238, 266, 270, 265, 264, 262, 257, 260, 255, 252, 221, 237, 279, 268, 248, 260, 247, 245, 250, 268, 234, 248, 276, 247, 236, 257, 285, 264, 280, 247, 246, 268, 271, 286, 246, 271, 245, 255, 259, 246, 244, 238, 236, 268, 246, 263, 249, 247, 222, 259, 270, 258, 261, 269, 252, 272, 276, 251, 240, 274, 232, 253, 286, 243, 250, 279, 277, 257, 258, 282, 250, 246, 251, 259] payload_rows_per_dst=[5306, 5528, 5488, 5466, 5392, 5524, 5544, 5500] dup_rows_per_dst=[2664, 2740, 2742, 2786, 2650, 2778, 2654, 2774] combine_rows_per_src=[5306, 5528, 5488, 5466, 5392, 5524, 5544, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43990 payload_rows_remote=38466 dup_rows=21546 dup_rows_remote=18838 dedup_ratio=0.328766 router_tokens_per_expert=[253, 246, 271, 255, 250, 271, 258, 256, 248, 232, 246, 251, 259, 257, 254, 265, 251, 245, 234, 249, 294, 253, 255, 257, 262, 264, 249, 235, 275, 265, 241, 279, 252, 273, 267, 265, 240, 282, 260, 257, 268, 227, 256, 267, 261, 251, 239, 251, 223, 237, 250, 271, 254, 281, 262, 234, 268, 262, 220, 275, 260, 240, 264, 277, 230, 227, 240, 248, 279, 245, 242, 251, 253, 280, 257, 237, 263, 256, 279, 259, 246, 280, 235, 282, 270, 276, 270, 258, 278, 265, 271, 250, 243, 243, 266, 271, 239, 289, 277, 235, 258, 230, 271, 239, 249, 260, 252, 247, 259, 247, 223, 277, 257, 242, 246, 260, 282, 281, 229, 232, 260, 245, 250, 268, 243, 259, 267, 271] payload_rows_per_dst=[5482, 5536, 5524, 5444, 5388, 5608, 5510, 5498] dup_rows_per_dst=[2662, 2680, 2708, 2712, 2704, 2800, 2594, 2686] combine_rows_per_src=[5482, 5536, 5524, 5444, 5388, 5608, 5510, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157990912 dispatch_meta_bytes=229688 combine_payload_bytes=157990912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158220600 ingress_bytes=214614016 link_bytes=214614016 token_bytes=316211512 weight_movement_bytes=56623104 total_nvlink_bytes=372834616 weight_to_token_ratio=0.179067 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.056325905e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229352 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156712936 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196520 weight_movement_bytes=18874368 total_nvlink_bytes=332070888 weight_to_token_ratio=0.060264 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=229216 combine_payload_bytes=157556736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157785952 ingress_bytes=195305472 link_bytes=195305472 token_bytes=315342688 weight_movement_bytes=37748736 total_nvlink_bytes=353091424 weight_to_token_ratio=0.119707 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871320936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38252 dup_rows=21808 dup_rows_remote=19024 dedup_ratio=0.332764 router_tokens_per_expert=[245, 255, 239, 246, 223, 251, 267, 295, 247, 279, 248, 252, 289, 239, 252, 261, 273, 273, 266, 261, 272, 274, 276, 227, 261, 262, 258, 244, 250, 262, 265, 243, 254, 253, 254, 256, 259, 269, 284, 257, 262, 277, 236, 298, 231, 249, 256, 252, 277, 223, 249, 256, 237, 260, 262, 266, 251, 245, 279, 260, 231, 247, 250, 252, 238, 286, 258, 254, 245, 260, 253, 269, 264, 267, 249, 233, 265, 261, 266, 262, 249, 250, 249, 281, 230, 263, 265, 257, 244, 259, 240, 272, 247, 267, 252, 283, 248, 232, 265, 274, 254, 256, 256, 240, 266, 252, 249, 243, 262, 237, 258, 243, 271, 245, 264, 235, 262, 271, 253, 245, 252, 239, 239, 227, 273, 242, 253, 277] payload_rows_per_dst=[5452, 5524, 5508, 5382, 5476, 5442, 5458, 5486] dup_rows_per_dst=[2724, 2810, 2786, 2708, 2784, 2774, 2612, 2610] combine_rows_per_src=[5452, 5524, 5508, 5382, 5476, 5442, 5458, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38288 dup_rows=21834 dup_rows_remote=19172 dedup_ratio=0.333160 router_tokens_per_expert=[245, 248, 253, 253, 242, 247, 275, 265, 267, 261, 285, 259, 262, 247, 273, 253, 226, 270, 271, 250, 277, 304, 238, 247, 224, 256, 252, 247, 255, 245, 251, 219, 261, 280, 253, 246, 248, 278, 274, 242, 288, 258, 281, 243, 275, 257, 238, 266, 273, 260, 249, 270, 263, 270, 277, 229, 242, 252, 248, 254, 254, 266, 267, 289, 237, 242, 254, 277, 266, 231, 287, 240, 260, 267, 264, 247, 243, 245, 241, 260, 244, 241, 248, 257, 246, 249, 269, 266, 267, 242, 256, 253, 259, 278, 234, 229, 223, 252, 247, 271, 270, 258, 258, 243, 237, 252, 245, 279, 249, 251, 234, 273, 262, 280, 248, 285, 242, 249, 266, 249, 254, 230, 255, 240, 254, 271, 251, 273] payload_rows_per_dst=[5496, 5438, 5564, 5484, 5396, 5414, 5394, 5516] dup_rows_per_dst=[2774, 2626, 2812, 2842, 2726, 2662, 2690, 2702] combine_rows_per_src=[5496, 5438, 5564, 5484, 5396, 5414, 5394, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38142 dup_rows=21842 dup_rows_remote=19036 dedup_ratio=0.333282 router_tokens_per_expert=[270, 272, 252, 267, 244, 252, 257, 284, 259, 260, 257, 245, 215, 257, 242, 253, 270, 258, 251, 259, 240, 262, 267, 270, 269, 256, 254, 266, 257, 254, 254, 261, 275, 255, 257, 285, 225, 273, 267, 245, 248, 238, 251, 251, 282, 296, 268, 240, 255, 257, 252, 264, 269, 252, 265, 230, 243, 217, 249, 273, 269, 256, 239, 259, 283, 262, 240, 258, 237, 263, 241, 268, 257, 256, 261, 247, 235, 258, 244, 278, 261, 231, 235, 257, 272, 250, 265, 251, 255, 257, 236, 264, 260, 263, 277, 260, 249, 287, 224, 241, 250, 253, 258, 282, 267, 261, 230, 272, 272, 246, 267, 253, 257, 243, 296, 255, 265, 247, 226, 263, 280, 232, 227, 248, 238, 268, 249, 241] payload_rows_per_dst=[5464, 5250, 5408, 5536, 5566, 5434, 5484, 5552] dup_rows_per_dst=[2708, 2398, 2730, 2922, 2784, 2754, 2740, 2806] combine_rows_per_src=[5464, 5250, 5408, 5536, 5566, 5434, 5484, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38220 dup_rows=21814 dup_rows_remote=19168 dedup_ratio=0.332855 router_tokens_per_expert=[285, 269, 265, 268, 291, 256, 247, 267, 258, 271, 223, 246, 230, 246, 247, 259, 246, 270, 256, 274, 252, 263, 230, 268, 254, 240, 233, 263, 250, 263, 256, 259, 264, 228, 231, 266, 229, 266, 220, 257, 284, 237, 287, 257, 264, 252, 261, 259, 240, 267, 261, 256, 255, 252, 239, 284, 242, 260, 261, 246, 263, 239, 253, 256, 281, 282, 256, 238, 262, 262, 275, 258, 248, 266, 262, 241, 229, 260, 251, 269, 257, 284, 265, 229, 257, 264, 252, 257, 244, 228, 258, 251, 252, 259, 273, 253, 261, 252, 273, 269, 255, 254, 243, 281, 268, 240, 261, 250, 240, 256, 240, 244, 259, 255, 253, 263, 273, 244, 255, 239, 253, 253, 280, 247, 274, 269, 257, 243] payload_rows_per_dst=[5456, 5452, 5370, 5502, 5504, 5446, 5528, 5464] dup_rows_per_dst=[2800, 2702, 2754, 2646, 2776, 2720, 2646, 2770] combine_rows_per_src=[5456, 5452, 5370, 5502, 5504, 5446, 5528, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38128 dup_rows=21990 dup_rows_remote=19304 dedup_ratio=0.335541 router_tokens_per_expert=[251, 241, 265, 273, 245, 235, 238, 228, 233, 251, 243, 258, 262, 253, 273, 268, 245, 264, 244, 242, 252, 238, 259, 281, 271, 265, 236, 236, 270, 249, 244, 252, 251, 262, 232, 285, 235, 274, 269, 242, 263, 257, 254, 255, 271, 265, 247, 222, 256, 244, 239, 275, 280, 229, 250, 234, 247, 268, 268, 258, 255, 256, 287, 253, 261, 241, 255, 253, 279, 256, 258, 231, 273, 274, 240, 262, 252, 259, 261, 272, 220, 287, 265, 282, 249, 273, 238, 261, 272, 257, 251, 260, 248, 234, 282, 261, 279, 253, 241, 275, 252, 242, 248, 263, 260, 240, 266, 262, 271, 221, 273, 258, 281, 238, 281, 258, 251, 261, 268, 270, 260, 259, 271, 213, 264, 244, 280, 250] payload_rows_per_dst=[5418, 5530, 5298, 5410, 5542, 5484, 5462, 5402] dup_rows_per_dst=[2686, 2918, 2542, 2694, 3030, 2796, 2746, 2578] combine_rows_per_src=[5418, 5530, 5298, 5410, 5542, 5484, 5462, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229104 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909296 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589488 weight_movement_bytes=18874368 total_nvlink_bytes=332463856 weight_to_token_ratio=0.060188 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38266 dup_rows=21790 dup_rows_remote=19012 dedup_ratio=0.332489 router_tokens_per_expert=[250, 280, 263, 247, 254, 268, 243, 241, 243, 248, 279, 260, 229, 260, 273, 248, 255, 255, 260, 257, 255, 276, 280, 248, 246, 219, 262, 259, 266, 267, 254, 270, 259, 269, 275, 240, 256, 273, 270, 252, 253, 246, 265, 251, 248, 271, 237, 246, 265, 232, 226, 241, 237, 241, 234, 263, 251, 240, 255, 265, 276, 248, 256, 254, 288, 242, 250, 254, 246, 258, 255, 268, 230, 256, 254, 244, 254, 267, 256, 270, 265, 238, 255, 238, 241, 259, 281, 268, 253, 282, 255, 260, 235, 251, 279, 256, 238, 262, 261, 261, 253, 265, 256, 255, 249, 269, 283, 286, 273, 263, 257, 251, 267, 257, 214, 258, 232, 215, 246, 251, 285, 262, 263, 245, 252, 281, 272, 268] payload_rows_per_dst=[5454, 5480, 5444, 5420, 5490, 5498, 5512, 5448] dup_rows_per_dst=[2718, 2778, 2778, 2548, 2694, 2734, 2852, 2688] combine_rows_per_src=[5454, 5480, 5444, 5420, 5490, 5498, 5512, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229840 combine_payload_bytes=156827648 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157057488 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313885136 weight_movement_bytes=37748736 total_nvlink_bytes=351633872 weight_to_token_ratio=0.120263 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=228712 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156458344 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312687976 weight_movement_bytes=18874368 total_nvlink_bytes=331562344 weight_to_token_ratio=0.060362 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229552 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778672 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313327792 weight_movement_bytes=18874368 total_nvlink_bytes=332202160 weight_to_token_ratio=0.060238 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229728 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156402016 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312574304 weight_movement_bytes=18874368 total_nvlink_bytes=331448672 weight_to_token_ratio=0.060384 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38374 dup_rows=21746 dup_rows_remote=19068 dedup_ratio=0.331818 router_tokens_per_expert=[255, 257, 244, 278, 267, 256, 255, 275, 238, 295, 271, 264, 263, 254, 242, 256, 249, 242, 262, 254, 274, 252, 257, 261, 215, 296, 247, 276, 264, 265, 223, 221, 253, 237, 278, 262, 238, 262, 254, 262, 256, 272, 285, 265, 274, 275, 254, 246, 238, 258, 224, 282, 255, 242, 269, 248, 264, 255, 288, 257, 286, 252, 263, 268, 253, 270, 272, 268, 259, 217, 240, 257, 264, 281, 236, 243, 248, 232, 272, 272, 265, 263, 256, 255, 268, 259, 266, 252, 265, 252, 226, 257, 229, 244, 242, 268, 269, 250, 266, 254, 261, 226, 256, 264, 269, 264, 231, 238, 233, 250, 271, 245, 242, 247, 257, 273, 287, 250, 246, 256, 236, 243, 254, 248, 274, 242, 232, 233] payload_rows_per_dst=[5584, 5406, 5572, 5444, 5482, 5436, 5416, 5450] dup_rows_per_dst=[2756, 2574, 2774, 2854, 2686, 2698, 2678, 2726] combine_rows_per_src=[5584, 5406, 5572, 5444, 5482, 5436, 5416, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38208 dup_rows=21878 dup_rows_remote=19120 dedup_ratio=0.333832 router_tokens_per_expert=[243, 261, 264, 235, 260, 252, 237, 259, 252, 250, 235, 253, 285, 227, 262, 241, 271, 266, 251, 243, 243, 249, 263, 276, 242, 274, 265, 254, 257, 252, 313, 274, 234, 241, 282, 231, 264, 245, 250, 269, 274, 233, 272, 268, 270, 246, 275, 250, 242, 236, 268, 246, 229, 238, 264, 286, 292, 269, 272, 263, 269, 261, 265, 250, 241, 270, 227, 236, 268, 205, 235, 240, 252, 250, 266, 263, 247, 285, 245, 264, 248, 267, 246, 255, 259, 279, 249, 258, 247, 248, 304, 263, 241, 252, 245, 251, 258, 230, 231, 275, 251, 281, 260, 272, 260, 261, 268, 223, 261, 251, 256, 249, 266, 260, 262, 245, 241, 260, 271, 260, 255, 274, 256, 250, 275, 245, 267, 225] payload_rows_per_dst=[5394, 5566, 5450, 5516, 5414, 5430, 5418, 5470] dup_rows_per_dst=[2638, 2820, 2758, 2784, 2574, 2794, 2756, 2754] combine_rows_per_src=[5394, 5566, 5450, 5516, 5414, 5430, 5418, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229112 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156966648 ingress_bytes=213360640 link_bytes=213360640 token_bytes=313704184 weight_movement_bytes=56623104 total_nvlink_bytes=370327288 weight_to_token_ratio=0.180498 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044316672e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229768 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157409672 ingress_bytes=194928640 link_bytes=194928640 token_bytes=314589576 weight_movement_bytes=37748736 total_nvlink_bytes=352338312 weight_to_token_ratio=0.119994 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867710318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229312 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156729280 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313229248 weight_movement_bytes=37748736 total_nvlink_bytes=350977984 weight_to_token_ratio=0.120515 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38166 dup_rows=21816 dup_rows_remote=18996 dedup_ratio=0.332886 router_tokens_per_expert=[230, 241, 237, 246, 281, 243, 234, 265, 237, 249, 264, 246, 266, 263, 275, 258, 237, 273, 257, 249, 240, 249, 247, 272, 277, 274, 273, 252, 274, 269, 287, 261, 224, 246, 261, 248, 280, 265, 252, 268, 254, 248, 252, 247, 257, 258, 287, 244, 253, 262, 237, 244, 240, 267, 226, 262, 267, 259, 262, 234, 252, 230, 283, 265, 255, 256, 283, 262, 267, 279, 270, 246, 257, 265, 252, 240, 256, 255, 264, 280, 259, 282, 262, 268, 258, 245, 235, 273, 262, 245, 295, 248, 253, 264, 265, 243, 253, 266, 258, 255, 271, 235, 275, 247, 271, 267, 229, 231, 254, 245, 264, 232, 260, 248, 239, 281, 255, 237, 268, 268, 251, 238, 265, 226, 242, 237, 250, 246] payload_rows_per_dst=[5422, 5506, 5372, 5452, 5554, 5540, 5418, 5456] dup_rows_per_dst=[2648, 2876, 2810, 2634, 2820, 2774, 2688, 2566] combine_rows_per_src=[5422, 5506, 5372, 5452, 5554, 5540, 5418, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38434 dup_rows=21736 dup_rows_remote=19130 dedup_ratio=0.331665 router_tokens_per_expert=[255, 275, 236, 270, 252, 262, 287, 274, 268, 259, 272, 238, 257, 250, 257, 255, 277, 255, 270, 283, 276, 273, 227, 256, 270, 251, 283, 241, 260, 245, 247, 238, 256, 263, 232, 256, 262, 274, 252, 268, 254, 237, 242, 244, 254, 276, 266, 240, 273, 258, 263, 241, 268, 247, 288, 261, 263, 222, 282, 251, 243, 250, 240, 239, 254, 246, 263, 249, 252, 272, 235, 242, 233, 237, 244, 282, 271, 229, 280, 262, 241, 245, 241, 248, 235, 261, 265, 252, 275, 283, 257, 231, 273, 210, 234, 235, 268, 247, 233, 251, 266, 263, 263, 268, 259, 258, 263, 277, 266, 240, 276, 277, 257, 262, 221, 248, 251, 265, 252, 256, 244, 271, 252, 276, 247, 259, 251, 260] payload_rows_per_dst=[5540, 5574, 5488, 5484, 5382, 5366, 5556, 5410] dup_rows_per_dst=[2794, 2730, 2664, 2694, 2720, 2606, 2794, 2734] combine_rows_per_src=[5540, 5574, 5488, 5484, 5382, 5366, 5556, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38238 dup_rows=21780 dup_rows_remote=19032 dedup_ratio=0.332336 router_tokens_per_expert=[246, 244, 258, 249, 233, 268, 242, 236, 269, 263, 263, 249, 241, 227, 226, 249, 255, 272, 241, 277, 240, 263, 243, 304, 262, 271, 242, 242, 267, 261, 227, 259, 240, 263, 251, 226, 245, 244, 277, 254, 258, 275, 274, 229, 241, 276, 278, 273, 244, 236, 256, 278, 255, 278, 269, 253, 287, 265, 254, 268, 241, 261, 282, 238, 248, 273, 283, 266, 277, 268, 268, 241, 243, 253, 276, 254, 261, 244, 239, 267, 217, 254, 261, 254, 264, 251, 237, 279, 233, 257, 251, 228, 268, 265, 262, 292, 241, 281, 245, 258, 239, 249, 267, 258, 236, 258, 252, 268, 271, 257, 278, 271, 248, 236, 254, 251, 255, 242, 279, 259, 238, 255, 261, 234, 257, 253, 269, 256] payload_rows_per_dst=[5378, 5386, 5468, 5394, 5710, 5412, 5490, 5518] dup_rows_per_dst=[2548, 2576, 2716, 2542, 3124, 2734, 2792, 2748] combine_rows_per_src=[5378, 5386, 5468, 5394, 5710, 5412, 5490, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43452 payload_rows_remote=38156 dup_rows=22084 dup_rows_remote=19346 dedup_ratio=0.336975 router_tokens_per_expert=[269, 253, 260, 249, 264, 262, 265, 246, 264, 273, 238, 269, 231, 268, 275, 276, 251, 270, 257, 267, 256, 265, 280, 281, 270, 246, 260, 251, 245, 249, 238, 262, 238, 244, 264, 284, 252, 231, 261, 277, 254, 274, 255, 263, 269, 218, 269, 257, 258, 276, 272, 265, 245, 212, 266, 236, 240, 248, 235, 234, 269, 247, 249, 265, 246, 245, 240, 258, 263, 277, 260, 274, 258, 240, 267, 250, 278, 270, 225, 297, 257, 241, 244, 239, 234, 244, 244, 249, 226, 274, 271, 275, 249, 267, 275, 237, 240, 253, 252, 267, 266, 259, 263, 277, 247, 278, 225, 261, 254, 250, 257, 242, 264, 242, 279, 264, 257, 249, 223, 235, 260, 257, 276, 248, 267, 247, 222, 276] payload_rows_per_dst=[5450, 5520, 5454, 5296, 5476, 5414, 5456, 5386] dup_rows_per_dst=[2874, 2776, 2766, 2738, 2820, 2638, 2726, 2746] combine_rows_per_src=[5450, 5520, 5454, 5296, 5476, 5414, 5456, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38292 dup_rows=21850 dup_rows_remote=19106 dedup_ratio=0.333405 router_tokens_per_expert=[245, 251, 246, 260, 270, 244, 268, 253, 246, 260, 267, 245, 243, 247, 238, 283, 255, 257, 263, 233, 262, 265, 258, 225, 275, 250, 266, 266, 266, 295, 253, 248, 255, 267, 227, 228, 246, 257, 263, 259, 262, 250, 248, 266, 233, 273, 236, 259, 284, 261, 249, 243, 260, 263, 246, 258, 266, 252, 244, 250, 261, 262, 260, 255, 261, 270, 263, 256, 267, 264, 227, 257, 270, 279, 264, 265, 239, 241, 254, 241, 247, 248, 250, 274, 244, 259, 270, 266, 267, 240, 268, 271, 274, 249, 226, 273, 275, 248, 254, 279, 261, 247, 258, 296, 247, 254, 260, 273, 267, 240, 207, 234, 281, 229, 233, 248, 249, 219, 269, 282, 271, 267, 252, 241, 266, 251, 255, 265] payload_rows_per_dst=[5394, 5590, 5402, 5460, 5502, 5556, 5286, 5496] dup_rows_per_dst=[2744, 3028, 2650, 2768, 2734, 2696, 2570, 2660] combine_rows_per_src=[5394, 5590, 5402, 5460, 5502, 5556, 5286, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228648 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156556584 ingress_bytes=175202304 link_bytes=175202304 token_bytes=312884520 weight_movement_bytes=18874368 total_nvlink_bytes=331758888 weight_to_token_ratio=0.060324 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678702272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157425664 dispatch_meta_bytes=230256 combine_payload_bytes=157425664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157655920 ingress_bytes=195174400 link_bytes=195174400 token_bytes=315081584 weight_movement_bytes=37748736 total_nvlink_bytes=352830320 weight_to_token_ratio=0.119806 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870065069e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38356 dup_rows=21662 dup_rows_remote=18988 dedup_ratio=0.330536 router_tokens_per_expert=[257, 236, 239, 267, 238, 245, 268, 288, 253, 270, 236, 242, 249, 282, 277, 248, 246, 234, 242, 241, 272, 273, 270, 247, 282, 239, 226, 290, 255, 255, 257, 267, 254, 256, 249, 267, 260, 249, 257, 286, 280, 239, 254, 252, 267, 254, 246, 290, 277, 228, 271, 239, 266, 250, 227, 249, 258, 264, 246, 255, 255, 257, 243, 266, 228, 256, 236, 231, 269, 258, 259, 267, 245, 242, 263, 253, 244, 239, 247, 258, 276, 286, 261, 277, 257, 249, 235, 251, 234, 267, 227, 262, 262, 267, 258, 260, 275, 272, 221, 275, 257, 250, 263, 264, 296, 257, 265, 244, 242, 259, 254, 243, 257, 276, 282, 291, 250, 243, 255, 251, 231, 255, 254, 244, 232, 253, 260, 271] payload_rows_per_dst=[5720, 5518, 5472, 5448, 5212, 5520, 5496, 5488] dup_rows_per_dst=[2996, 2674, 2848, 2654, 2252, 2738, 2778, 2722] combine_rows_per_src=[5720, 5518, 5472, 5448, 5212, 5520, 5496, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229080 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156851928 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474776 weight_movement_bytes=18874368 total_nvlink_bytes=332349144 weight_to_token_ratio=0.060210 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38246 dup_rows=21816 dup_rows_remote=19172 dedup_ratio=0.332886 router_tokens_per_expert=[279, 256, 276, 248, 253, 258, 254, 256, 240, 281, 257, 255, 241, 254, 265, 248, 280, 232, 234, 245, 243, 255, 238, 270, 236, 252, 252, 261, 252, 272, 282, 262, 256, 285, 279, 266, 237, 267, 255, 243, 263, 262, 260, 261, 238, 250, 231, 269, 263, 246, 251, 259, 224, 252, 262, 277, 265, 257, 280, 259, 253, 250, 227, 278, 245, 259, 276, 267, 242, 264, 255, 258, 280, 266, 261, 247, 243, 245, 238, 287, 225, 270, 249, 229, 230, 271, 244, 268, 271, 242, 244, 237, 244, 245, 247, 294, 238, 252, 244, 254, 263, 261, 249, 222, 242, 246, 263, 241, 248, 273, 304, 259, 249, 243, 276, 263, 243, 259, 269, 270, 267, 257, 253, 265, 251, 266, 268, 255] payload_rows_per_dst=[5446, 5410, 5514, 5550, 5470, 5322, 5474, 5534] dup_rows_per_dst=[2796, 2722, 2730, 2656, 2796, 2698, 2644, 2774] combine_rows_per_src=[5446, 5410, 5514, 5550, 5470, 5322, 5474, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=230008 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156516984 ingress_bytes=194035712 link_bytes=194035712 token_bytes=312803960 weight_movement_bytes=37748736 total_nvlink_bytes=350552696 weight_to_token_ratio=0.120679 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859154721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38274 dup_rows=21816 dup_rows_remote=19044 dedup_ratio=0.332886 router_tokens_per_expert=[262, 260, 255, 258, 219, 244, 260, 251, 244, 246, 268, 258, 244, 252, 258, 251, 259, 256, 255, 251, 258, 267, 261, 237, 282, 256, 252, 258, 238, 242, 257, 276, 268, 259, 269, 240, 281, 251, 277, 247, 267, 242, 246, 267, 276, 245, 257, 217, 280, 260, 250, 272, 261, 237, 242, 254, 263, 272, 284, 249, 267, 290, 269, 271, 229, 256, 257, 275, 253, 246, 261, 269, 236, 263, 247, 251, 249, 267, 259, 250, 256, 246, 257, 259, 270, 252, 273, 259, 254, 259, 240, 242, 236, 271, 237, 255, 244, 249, 265, 245, 252, 240, 247, 250, 277, 258, 292, 240, 256, 293, 250, 271, 274, 249, 243, 266, 235, 248, 235, 253, 261, 249, 245, 245, 267, 258, 235, 277] payload_rows_per_dst=[5454, 5508, 5446, 5600, 5416, 5400, 5462, 5434] dup_rows_per_dst=[2664, 2702, 2772, 2842, 2662, 2732, 2796, 2646] combine_rows_per_src=[5454, 5508, 5446, 5600, 5416, 5400, 5462, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229592 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157073624 ingress_bytes=194592768 link_bytes=194592768 token_bytes=313917656 weight_movement_bytes=37748736 total_nvlink_bytes=351666392 weight_to_token_ratio=0.120250 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864492157e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229376 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157335552 ingress_bytes=213729280 link_bytes=213729280 token_bytes=314441728 weight_movement_bytes=56623104 total_nvlink_bytes=371064832 weight_to_token_ratio=0.180075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047848800e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229672 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885288 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540904 weight_movement_bytes=18874368 total_nvlink_bytes=332415272 weight_to_token_ratio=0.060197 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229272 combine_payload_bytes=156770304 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156999576 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769880 weight_movement_bytes=37748736 total_nvlink_bytes=351518616 weight_to_token_ratio=0.120307 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38180 dup_rows=21794 dup_rows_remote=19090 dedup_ratio=0.332550 router_tokens_per_expert=[249, 264, 231, 246, 266, 210, 268, 247, 253, 270, 242, 270, 255, 275, 248, 278, 239, 250, 256, 240, 256, 212, 253, 263, 253, 253, 250, 294, 257, 250, 279, 256, 253, 270, 236, 242, 253, 261, 262, 245, 256, 258, 275, 253, 259, 238, 250, 258, 257, 268, 281, 247, 249, 258, 237, 226, 275, 248, 256, 243, 270, 247, 273, 251, 250, 252, 291, 242, 267, 256, 254, 240, 243, 233, 283, 247, 254, 253, 289, 279, 237, 266, 255, 248, 283, 234, 258, 233, 278, 281, 269, 262, 259, 268, 256, 246, 281, 261, 249, 243, 231, 252, 252, 270, 257, 284, 240, 276, 244, 228, 266, 252, 256, 232, 261, 251, 270, 249, 246, 279, 260, 248, 269, 247, 265, 302, 244, 249] payload_rows_per_dst=[5408, 5406, 5474, 5500, 5562, 5480, 5456, 5456] dup_rows_per_dst=[2736, 2716, 2664, 2672, 2704, 2786, 2716, 2800] combine_rows_per_src=[5408, 5406, 5474, 5500, 5562, 5480, 5456, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38246 dup_rows=21826 dup_rows_remote=19106 dedup_ratio=0.333038 router_tokens_per_expert=[240, 265, 244, 222, 272, 241, 251, 254, 252, 274, 265, 264, 233, 236, 218, 217, 260, 273, 236, 232, 284, 256, 237, 252, 275, 283, 248, 236, 252, 304, 249, 244, 247, 258, 230, 263, 269, 260, 244, 245, 248, 238, 252, 268, 257, 271, 276, 274, 238, 252, 274, 260, 251, 263, 272, 232, 251, 261, 251, 242, 239, 268, 249, 264, 250, 256, 249, 261, 246, 291, 240, 242, 265, 269, 288, 256, 274, 256, 283, 235, 279, 246, 258, 251, 272, 248, 240, 270, 253, 255, 272, 260, 246, 240, 248, 254, 249, 276, 280, 270, 264, 278, 257, 260, 256, 238, 238, 249, 247, 248, 252, 265, 238, 279, 252, 249, 268, 282, 252, 257, 256, 255, 243, 251, 286, 242, 277, 265] payload_rows_per_dst=[5368, 5496, 5412, 5416, 5544, 5464, 5530, 5480] dup_rows_per_dst=[2528, 2746, 2788, 2718, 2778, 2720, 2724, 2824] combine_rows_per_src=[5368, 5496, 5412, 5416, 5544, 5464, 5530, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38122 dup_rows=21780 dup_rows_remote=18716 dedup_ratio=0.332336 router_tokens_per_expert=[285, 277, 263, 258, 273, 232, 224, 244, 256, 240, 275, 249, 239, 237, 265, 242, 262, 276, 267, 268, 257, 252, 253, 257, 253, 234, 272, 259, 269, 239, 249, 270, 270, 267, 261, 260, 251, 224, 246, 251, 265, 283, 236, 292, 247, 249, 289, 237, 295, 263, 233, 256, 264, 258, 264, 249, 275, 254, 235, 242, 251, 263, 227, 252, 242, 277, 267, 242, 258, 236, 259, 249, 255, 241, 255, 251, 258, 254, 279, 258, 268, 226, 237, 260, 273, 260, 249, 264, 248, 249, 266, 279, 226, 245, 274, 229, 260, 253, 261, 266, 235, 242, 232, 263, 256, 252, 256, 256, 291, 268, 266, 236, 272, 244, 240, 284, 253, 250, 239, 271, 262, 272, 223, 270, 260, 277, 272, 247] payload_rows_per_dst=[5484, 5482, 5452, 5464, 5502, 5416, 5322, 5634] dup_rows_per_dst=[2634, 2778, 2688, 2820, 2660, 2622, 2514, 3064] combine_rows_per_src=[5484, 5482, 5452, 5464, 5502, 5416, 5322, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38392 dup_rows=21610 dup_rows_remote=18916 dedup_ratio=0.329742 router_tokens_per_expert=[225, 279, 259, 237, 249, 236, 262, 221, 250, 280, 221, 232, 271, 259, 266, 270, 278, 258, 272, 283, 254, 255, 234, 260, 224, 261, 246, 255, 256, 255, 275, 256, 279, 217, 243, 271, 255, 258, 233, 246, 255, 256, 295, 274, 220, 258, 245, 232, 270, 242, 260, 248, 273, 266, 249, 283, 242, 248, 261, 247, 282, 241, 247, 255, 268, 282, 243, 275, 246, 244, 292, 243, 246, 246, 250, 237, 259, 237, 214, 258, 272, 284, 256, 260, 262, 251, 263, 247, 275, 274, 251, 288, 282, 266, 252, 246, 265, 273, 272, 236, 269, 261, 256, 234, 244, 268, 275, 218, 255, 267, 244, 243, 229, 268, 268, 246, 269, 250, 266, 282, 247, 254, 267, 245, 267, 267, 260, 244] payload_rows_per_dst=[5406, 5494, 5432, 5534, 5434, 5628, 5494, 5504] dup_rows_per_dst=[2628, 2750, 2642, 2694, 2646, 2830, 2666, 2754] combine_rows_per_src=[5406, 5494, 5432, 5534, 5434, 5628, 5494, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229080 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156614360 ingress_bytes=194134016 link_bytes=194134016 token_bytes=312999640 weight_movement_bytes=37748736 total_nvlink_bytes=350748376 weight_to_token_ratio=0.120603 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38032 dup_rows=21862 dup_rows_remote=18908 dedup_ratio=0.333588 router_tokens_per_expert=[283, 248, 263, 235, 274, 242, 244, 252, 253, 259, 285, 252, 302, 266, 264, 259, 243, 252, 228, 261, 238, 252, 244, 242, 246, 267, 247, 284, 260, 274, 243, 235, 248, 248, 282, 242, 265, 263, 301, 248, 214, 221, 265, 265, 233, 265, 269, 241, 239, 256, 241, 262, 251, 253, 260, 254, 247, 250, 252, 271, 247, 244, 252, 273, 252, 266, 255, 260, 271, 254, 244, 261, 253, 253, 251, 227, 209, 259, 243, 244, 258, 243, 263, 244, 249, 258, 252, 269, 235, 258, 251, 298, 256, 232, 272, 234, 250, 244, 256, 267, 277, 240, 256, 245, 275, 280, 284, 277, 256, 273, 284, 240, 266, 242, 268, 260, 254, 289, 259, 277, 221, 287, 258, 255, 249, 249, 261, 276] payload_rows_per_dst=[5642, 5358, 5498, 5400, 5398, 5372, 5576, 5430] dup_rows_per_dst=[2954, 2688, 2642, 2704, 2606, 2758, 2832, 2678] combine_rows_per_src=[5642, 5358, 5498, 5400, 5398, 5372, 5576, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229408 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885024 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540640 weight_movement_bytes=18874368 total_nvlink_bytes=332415008 weight_to_token_ratio=0.060198 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38332 dup_rows=21712 dup_rows_remote=19024 dedup_ratio=0.331299 router_tokens_per_expert=[244, 266, 253, 272, 261, 240, 242, 251, 256, 230, 272, 248, 277, 262, 270, 253, 246, 260, 276, 282, 227, 270, 245, 247, 271, 257, 241, 248, 242, 258, 272, 248, 269, 238, 258, 252, 251, 271, 268, 263, 276, 261, 247, 241, 286, 229, 240, 262, 246, 251, 224, 248, 271, 256, 257, 246, 261, 274, 267, 234, 239, 268, 275, 240, 262, 275, 250, 249, 253, 231, 263, 208, 226, 243, 273, 261, 252, 273, 261, 248, 269, 275, 237, 229, 240, 269, 246, 254, 269, 252, 271, 244, 272, 245, 263, 240, 232, 264, 270, 270, 265, 262, 277, 269, 258, 270, 268, 252, 301, 259, 251, 239, 265, 242, 254, 228, 233, 255, 277, 281, 237, 242, 230, 273, 287, 271, 253, 274] payload_rows_per_dst=[5452, 5492, 5540, 5420, 5374, 5448, 5548, 5550] dup_rows_per_dst=[2742, 2688, 2684, 2694, 2682, 2702, 2866, 2654] combine_rows_per_src=[5452, 5492, 5540, 5420, 5374, 5448, 5548, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38152 dup_rows=21982 dup_rows_remote=19356 dedup_ratio=0.335419 router_tokens_per_expert=[258, 283, 267, 257, 253, 260, 266, 278, 235, 255, 271, 261, 266, 242, 263, 236, 246, 240, 243, 262, 248, 265, 248, 266, 273, 274, 270, 263, 252, 230, 256, 249, 270, 249, 260, 242, 270, 282, 221, 250, 239, 269, 273, 257, 266, 273, 245, 249, 257, 241, 253, 259, 245, 279, 249, 260, 255, 280, 225, 283, 230, 268, 268, 259, 245, 248, 287, 241, 261, 272, 257, 252, 244, 243, 243, 256, 257, 259, 272, 270, 264, 277, 275, 244, 279, 252, 278, 239, 262, 253, 237, 267, 233, 260, 254, 242, 241, 270, 255, 276, 271, 253, 230, 225, 240, 269, 256, 258, 253, 236, 219, 262, 251, 270, 272, 225, 275, 252, 226, 281, 240, 253, 255, 268, 241, 251, 275, 234] payload_rows_per_dst=[5440, 5368, 5424, 5502, 5502, 5478, 5402, 5438] dup_rows_per_dst=[2862, 2802, 2806, 2720, 2712, 2754, 2626, 2700] combine_rows_per_src=[5440, 5368, 5424, 5502, 5502, 5478, 5402, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=227352 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156375064 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312522776 weight_movement_bytes=18874368 total_nvlink_bytes=331397144 weight_to_token_ratio=0.060394 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44004 payload_rows_remote=38408 dup_rows=21532 dup_rows_remote=18814 dedup_ratio=0.328552 router_tokens_per_expert=[287, 248, 244, 259, 258, 259, 253, 274, 244, 269, 232, 269, 261, 231, 264, 274, 271, 258, 239, 242, 265, 247, 261, 241, 258, 253, 295, 257, 253, 246, 235, 252, 261, 246, 267, 252, 259, 246, 230, 287, 266, 264, 294, 232, 273, 284, 266, 230, 276, 256, 257, 258, 256, 264, 262, 243, 266, 283, 268, 253, 256, 236, 265, 280, 243, 253, 278, 283, 240, 257, 247, 259, 235, 250, 261, 243, 258, 254, 247, 252, 275, 261, 237, 269, 242, 254, 216, 266, 245, 237, 282, 270, 248, 241, 273, 236, 259, 241, 263, 234, 281, 262, 269, 245, 259, 259, 271, 244, 240, 254, 232, 278, 257, 258, 244, 268, 273, 244, 255, 251, 244, 252, 244, 238, 237, 247, 271, 247] payload_rows_per_dst=[5508, 5458, 5596, 5604, 5510, 5458, 5494, 5376] dup_rows_per_dst=[2744, 2688, 2718, 2754, 2610, 2646, 2688, 2684] combine_rows_per_src=[5508, 5458, 5596, 5604, 5510, 5458, 5494, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229232 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157482864 ingress_bytes=176128000 link_bytes=176128000 token_bytes=314736496 weight_movement_bytes=18874368 total_nvlink_bytes=333610864 weight_to_token_ratio=0.059969 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687571836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=227760 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156006832 ingress_bytes=193527808 link_bytes=193527808 token_bytes=311785904 weight_movement_bytes=37748736 total_nvlink_bytes=349534640 weight_to_token_ratio=0.121073 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854288235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229424 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157237296 ingress_bytes=213630976 link_bytes=213630976 token_bytes=314245168 weight_movement_bytes=56623104 total_nvlink_bytes=370868272 weight_to_token_ratio=0.180188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046906899e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=230032 combine_payload_bytes=156270592 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156500624 ingress_bytes=212893696 link_bytes=212893696 token_bytes=312771216 weight_movement_bytes=56623104 total_nvlink_bytes=369394320 weight_to_token_ratio=0.181037 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039842645e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=228888 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157548056 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314867224 weight_movement_bytes=37748736 total_nvlink_bytes=352615960 weight_to_token_ratio=0.119888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38426 dup_rows=21702 dup_rows_remote=18962 dedup_ratio=0.331146 router_tokens_per_expert=[280, 259, 255, 263, 276, 240, 246, 250, 269, 241, 260, 254, 262, 226, 275, 286, 266, 241, 270, 266, 241, 245, 255, 255, 243, 264, 241, 266, 241, 250, 271, 260, 246, 256, 275, 277, 267, 254, 256, 241, 255, 253, 258, 256, 265, 256, 260, 253, 297, 251, 248, 275, 238, 244, 257, 256, 248, 251, 240, 230, 233, 293, 258, 253, 294, 247, 246, 245, 268, 235, 245, 276, 251, 253, 244, 234, 254, 259, 253, 270, 243, 260, 267, 270, 250, 257, 262, 261, 262, 225, 249, 267, 261, 282, 258, 258, 266, 259, 264, 238, 269, 259, 251, 291, 257, 239, 255, 261, 267, 266, 254, 243, 253, 238, 260, 221, 273, 247, 263, 262, 239, 249, 272, 252, 260, 233, 245, 239] payload_rows_per_dst=[5546, 5448, 5524, 5484, 5408, 5488, 5514, 5422] dup_rows_per_dst=[2738, 2702, 2732, 2660, 2740, 2776, 2764, 2590] combine_rows_per_src=[5546, 5448, 5524, 5484, 5408, 5488, 5514, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38212 dup_rows=21798 dup_rows_remote=18994 dedup_ratio=0.332611 router_tokens_per_expert=[243, 262, 261, 245, 244, 252, 252, 265, 240, 271, 259, 291, 258, 286, 229, 239, 259, 285, 231, 257, 245, 260, 264, 267, 247, 257, 254, 228, 255, 284, 243, 243, 265, 282, 262, 268, 232, 245, 277, 242, 285, 236, 279, 249, 253, 228, 224, 254, 274, 256, 230, 292, 262, 237, 263, 252, 245, 256, 273, 237, 254, 249, 246, 247, 260, 282, 237, 256, 259, 240, 255, 246, 263, 248, 256, 245, 249, 257, 261, 269, 259, 231, 247, 249, 245, 266, 252, 277, 268, 246, 255, 274, 253, 265, 293, 285, 250, 279, 284, 249, 263, 250, 254, 284, 243, 269, 269, 242, 246, 266, 268, 262, 266, 226, 239, 254, 249, 261, 276, 246, 262, 258, 223, 269, 239, 269, 249, 226] payload_rows_per_dst=[5460, 5424, 5486, 5460, 5444, 5526, 5558, 5380] dup_rows_per_dst=[2734, 2734, 2676, 2686, 2722, 2804, 2798, 2644] combine_rows_per_src=[5460, 5424, 5486, 5460, 5444, 5526, 5558, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157392896 dispatch_meta_bytes=229552 combine_payload_bytes=157392896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157622448 ingress_bytes=176267264 link_bytes=176267264 token_bytes=315015344 weight_movement_bytes=18874368 total_nvlink_bytes=333889712 weight_to_token_ratio=0.059916 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688906195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38214 dup_rows=21862 dup_rows_remote=19060 dedup_ratio=0.333588 router_tokens_per_expert=[232, 246, 252, 258, 249, 267, 268, 268, 253, 251, 230, 252, 245, 250, 268, 254, 295, 262, 251, 278, 247, 261, 265, 258, 257, 254, 260, 227, 242, 258, 255, 248, 247, 279, 239, 271, 286, 261, 257, 266, 231, 301, 261, 261, 254, 260, 248, 275, 269, 251, 257, 242, 239, 256, 246, 267, 245, 236, 246, 273, 256, 233, 246, 254, 252, 227, 262, 247, 248, 227, 250, 240, 276, 274, 270, 249, 268, 272, 262, 251, 243, 267, 251, 254, 262, 253, 231, 256, 255, 281, 250, 259, 268, 239, 282, 261, 232, 290, 268, 247, 242, 251, 254, 274, 254, 242, 276, 231, 258, 276, 244, 238, 247, 236, 268, 240, 249, 275, 292, 262, 256, 254, 254, 283, 280, 241, 258, 235] payload_rows_per_dst=[5404, 5492, 5490, 5378, 5510, 5518, 5422, 5460] dup_rows_per_dst=[2682, 2744, 2898, 2400, 2640, 2960, 2736, 2802] combine_rows_per_src=[5404, 5492, 5490, 5378, 5510, 5518, 5422, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38126 dup_rows=21914 dup_rows_remote=19106 dedup_ratio=0.334381 router_tokens_per_expert=[263, 259, 255, 250, 233, 258, 248, 264, 246, 247, 267, 260, 271, 259, 260, 268, 282, 236, 250, 256, 284, 252, 283, 238, 283, 278, 275, 246, 254, 252, 228, 251, 241, 268, 242, 266, 248, 221, 258, 252, 254, 235, 252, 266, 267, 252, 245, 258, 270, 264, 297, 253, 258, 263, 272, 247, 252, 259, 240, 267, 238, 265, 260, 247, 257, 235, 256, 248, 247, 256, 248, 252, 248, 271, 252, 267, 237, 260, 259, 234, 261, 277, 253, 237, 274, 242, 263, 240, 244, 261, 242, 259, 246, 253, 247, 248, 227, 275, 259, 289, 252, 257, 250, 257, 267, 248, 246, 238, 232, 266, 246, 231, 277, 254, 258, 245, 258, 251, 296, 250, 256, 258, 291, 268, 246, 264, 295, 254] payload_rows_per_dst=[5478, 5462, 5438, 5496, 5412, 5434, 5392, 5510] dup_rows_per_dst=[2738, 2834, 2612, 2808, 2642, 2660, 2688, 2932] combine_rows_per_src=[5478, 5462, 5438, 5496, 5412, 5434, 5392, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228824 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156745176 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313261528 weight_movement_bytes=37748736 total_nvlink_bytes=351010264 weight_to_token_ratio=0.120502 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38368 dup_rows=21660 dup_rows_remote=18970 dedup_ratio=0.330505 router_tokens_per_expert=[279, 224, 273, 252, 262, 238, 226, 272, 266, 243, 268, 238, 263, 245, 263, 259, 260, 246, 235, 249, 246, 239, 257, 266, 248, 263, 260, 268, 232, 242, 276, 267, 244, 278, 293, 229, 254, 239, 234, 257, 262, 260, 280, 273, 277, 258, 221, 272, 252, 266, 276, 256, 225, 257, 251, 264, 230, 244, 272, 247, 274, 271, 255, 248, 259, 249, 234, 244, 268, 272, 239, 274, 246, 240, 256, 242, 254, 256, 239, 250, 262, 274, 263, 280, 281, 248, 274, 248, 252, 240, 258, 230, 224, 251, 259, 285, 249, 248, 254, 285, 273, 256, 261, 250, 270, 248, 262, 258, 242, 264, 260, 253, 260, 263, 217, 255, 277, 256, 238, 271, 258, 275, 252, 292, 276, 275, 236, 239] payload_rows_per_dst=[5508, 5486, 5578, 5398, 5480, 5574, 5378, 5474] dup_rows_per_dst=[2690, 2748, 2702, 2778, 2564, 2634, 2738, 2806] combine_rows_per_src=[5508, 5486, 5578, 5398, 5480, 5574, 5378, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38318 dup_rows=21740 dup_rows_remote=19148 dedup_ratio=0.331726 router_tokens_per_expert=[244, 240, 240, 230, 252, 255, 270, 235, 240, 240, 245, 254, 255, 252, 249, 242, 231, 252, 243, 241, 258, 261, 261, 261, 268, 236, 264, 274, 255, 236, 241, 261, 281, 236, 228, 246, 253, 253, 252, 257, 283, 240, 254, 240, 255, 259, 280, 269, 267, 267, 277, 270, 265, 274, 285, 288, 266, 262, 260, 256, 273, 243, 247, 249, 240, 268, 258, 260, 268, 260, 291, 241, 256, 275, 285, 260, 232, 252, 268, 272, 246, 248, 271, 272, 261, 248, 236, 249, 249, 261, 251, 246, 264, 248, 247, 281, 244, 255, 233, 245, 263, 256, 246, 255, 268, 270, 260, 269, 237, 245, 241, 248, 254, 273, 268, 275, 257, 267, 242, 266, 245, 252, 260, 258, 252, 245, 261, 273] payload_rows_per_dst=[5340, 5424, 5478, 5532, 5494, 5480, 5478, 5570] dup_rows_per_dst=[2546, 2662, 2694, 2902, 2942, 2676, 2592, 2726] combine_rows_per_src=[5340, 5424, 5478, 5532, 5494, 5480, 5478, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38248 dup_rows=21798 dup_rows_remote=19202 dedup_ratio=0.332611 router_tokens_per_expert=[268, 239, 244, 232, 243, 231, 263, 260, 246, 276, 232, 264, 260, 266, 274, 259, 246, 246, 256, 244, 252, 239, 270, 268, 257, 215, 243, 273, 256, 270, 257, 251, 232, 261, 245, 263, 239, 269, 238, 253, 263, 270, 223, 246, 280, 251, 227, 266, 262, 259, 239, 263, 279, 237, 261, 259, 250, 281, 264, 268, 255, 256, 266, 258, 247, 262, 254, 258, 259, 252, 249, 269, 246, 275, 254, 250, 241, 248, 246, 260, 267, 265, 283, 233, 245, 284, 237, 285, 255, 278, 259, 270, 246, 262, 247, 253, 238, 276, 231, 255, 253, 275, 248, 245, 266, 242, 263, 251, 253, 236, 251, 256, 273, 272, 266, 241, 247, 239, 275, 270, 247, 289, 290, 240, 256, 273, 267, 262] payload_rows_per_dst=[5410, 5490, 5442, 5510, 5428, 5502, 5436, 5520] dup_rows_per_dst=[2704, 2596, 2610, 2804, 2712, 2836, 2642, 2894] combine_rows_per_src=[5410, 5490, 5442, 5510, 5428, 5502, 5436, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38298 dup_rows=21818 dup_rows_remote=19160 dedup_ratio=0.332916 router_tokens_per_expert=[225, 247, 261, 256, 253, 259, 256, 243, 256, 246, 266, 282, 245, 257, 248, 223, 254, 238, 260, 268, 229, 258, 272, 270, 261, 276, 275, 239, 258, 224, 279, 230, 257, 255, 252, 245, 255, 252, 240, 250, 240, 253, 265, 246, 243, 266, 266, 254, 257, 242, 260, 262, 265, 243, 222, 245, 266, 248, 286, 244, 251, 265, 290, 259, 251, 252, 274, 249, 262, 267, 256, 257, 286, 248, 276, 265, 280, 261, 241, 234, 282, 274, 250, 273, 240, 258, 263, 237, 229, 274, 265, 258, 251, 278, 248, 275, 255, 255, 248, 239, 235, 278, 274, 263, 255, 263, 272, 234, 249, 230, 266, 276, 233, 287, 248, 254, 256, 270, 250, 249, 262, 250, 246, 278, 257, 250, 261, 253] payload_rows_per_dst=[5438, 5384, 5420, 5484, 5554, 5504, 5448, 5486] dup_rows_per_dst=[2608, 2798, 2658, 2726, 2764, 2806, 2736, 2722] combine_rows_per_src=[5438, 5384, 5420, 5484, 5554, 5504, 5448, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228928 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156393024 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312557120 weight_movement_bytes=37748736 total_nvlink_bytes=350305856 weight_to_token_ratio=0.120774 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229096 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156753640 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278184 weight_movement_bytes=56623104 total_nvlink_bytes=369901288 weight_to_token_ratio=0.180744 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229352 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384680 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540008 weight_movement_bytes=18874368 total_nvlink_bytes=333414376 weight_to_token_ratio=0.060006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229864 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157180392 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130920 weight_movement_bytes=37748736 total_nvlink_bytes=351879656 weight_to_token_ratio=0.120169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229800 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893608 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557416 weight_movement_bytes=18874368 total_nvlink_bytes=332431784 weight_to_token_ratio=0.060194 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38306 dup_rows=21742 dup_rows_remote=18996 dedup_ratio=0.331757 router_tokens_per_expert=[257, 254, 273, 266, 271, 249, 250, 272, 262, 261, 261, 257, 251, 250, 265, 271, 273, 242, 245, 234, 239, 246, 244, 230, 243, 226, 272, 232, 290, 255, 255, 238, 272, 281, 246, 275, 246, 272, 242, 265, 259, 266, 239, 251, 269, 267, 287, 239, 258, 251, 262, 257, 252, 255, 249, 232, 286, 268, 257, 275, 260, 282, 282, 264, 269, 231, 254, 268, 259, 263, 287, 261, 256, 247, 253, 261, 257, 267, 224, 260, 235, 242, 248, 229, 264, 271, 280, 265, 263, 254, 272, 236, 244, 246, 239, 228, 288, 266, 248, 256, 259, 244, 230, 300, 249, 251, 266, 263, 247, 241, 265, 256, 243, 239, 258, 254, 273, 231, 246, 231, 267, 246, 275, 275, 214, 261, 238, 255] payload_rows_per_dst=[5590, 5326, 5530, 5566, 5488, 5436, 5460, 5398] dup_rows_per_dst=[2750, 2602, 2822, 2814, 2746, 2596, 2798, 2614] combine_rows_per_src=[5590, 5326, 5530, 5566, 5488, 5436, 5460, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229832 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157098440 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313967048 weight_movement_bytes=37748736 total_nvlink_bytes=351715784 weight_to_token_ratio=0.120232 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38448 dup_rows=21734 dup_rows_remote=19072 dedup_ratio=0.331635 router_tokens_per_expert=[235, 263, 265, 262, 291, 278, 249, 239, 229, 250, 307, 233, 257, 255, 282, 247, 263, 229, 253, 255, 248, 261, 260, 239, 260, 274, 261, 267, 274, 246, 235, 280, 256, 249, 248, 255, 238, 231, 273, 252, 243, 253, 286, 284, 255, 235, 257, 262, 248, 244, 250, 247, 308, 275, 274, 267, 235, 229, 254, 231, 241, 260, 273, 261, 257, 246, 262, 277, 234, 287, 242, 236, 246, 285, 239, 256, 253, 258, 239, 221, 256, 250, 253, 240, 239, 259, 267, 246, 229, 262, 253, 249, 250, 255, 246, 254, 254, 247, 242, 275, 245, 270, 269, 283, 296, 237, 246, 271, 263, 249, 255, 256, 253, 289, 250, 240, 256, 244, 285, 272, 243, 252, 269, 243, 274, 254, 259, 260] payload_rows_per_dst=[5528, 5460, 5478, 5488, 5372, 5354, 5544, 5578] dup_rows_per_dst=[2756, 2750, 2676, 2706, 2704, 2662, 2772, 2708] combine_rows_per_src=[5528, 5460, 5478, 5488, 5372, 5354, 5544, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229208 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130584 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031960 weight_movement_bytes=37748736 total_nvlink_bytes=351780696 weight_to_token_ratio=0.120207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38362 dup_rows=21696 dup_rows_remote=18930 dedup_ratio=0.331055 router_tokens_per_expert=[243, 241, 275, 266, 249, 233, 282, 268, 237, 269, 263, 256, 218, 255, 273, 257, 264, 269, 268, 244, 265, 235, 252, 283, 236, 248, 266, 263, 243, 253, 255, 271, 261, 252, 253, 255, 249, 239, 229, 272, 258, 254, 268, 256, 241, 278, 289, 258, 259, 246, 248, 265, 247, 254, 252, 242, 268, 256, 241, 281, 285, 254, 258, 266, 268, 262, 285, 285, 269, 263, 250, 272, 247, 249, 270, 254, 233, 253, 264, 239, 266, 252, 247, 213, 281, 271, 265, 263, 231, 250, 256, 247, 228, 254, 284, 287, 238, 232, 258, 284, 250, 262, 279, 252, 267, 247, 229, 268, 270, 297, 247, 250, 255, 235, 239, 249, 254, 245, 285, 242, 256, 230, 250, 251, 237, 237, 244, 237] payload_rows_per_dst=[5468, 5546, 5424, 5478, 5550, 5510, 5496, 5368] dup_rows_per_dst=[2702, 2684, 2800, 2766, 2776, 2680, 2764, 2524] combine_rows_per_src=[5468, 5546, 5424, 5478, 5550, 5510, 5496, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38152 dup_rows=21858 dup_rows_remote=19078 dedup_ratio=0.333527 router_tokens_per_expert=[273, 227, 256, 264, 240, 246, 237, 282, 242, 260, 254, 242, 279, 240, 270, 240, 248, 238, 234, 281, 248, 243, 261, 276, 255, 255, 264, 255, 253, 256, 259, 250, 275, 267, 244, 229, 263, 244, 252, 272, 279, 252, 245, 237, 239, 243, 247, 266, 228, 253, 252, 300, 283, 243, 270, 257, 269, 266, 248, 278, 233, 230, 258, 271, 246, 258, 253, 259, 277, 279, 239, 254, 244, 243, 246, 239, 262, 267, 250, 244, 258, 271, 278, 247, 251, 261, 267, 260, 255, 263, 267, 245, 242, 240, 273, 249, 239, 281, 258, 250, 267, 256, 273, 272, 243, 281, 233, 248, 242, 261, 266, 252, 225, 259, 266, 258, 249, 271, 259, 277, 240, 246, 253, 264, 305, 267, 263, 236] payload_rows_per_dst=[5364, 5272, 5598, 5462, 5392, 5556, 5508, 5526] dup_rows_per_dst=[2740, 2522, 2864, 2786, 2690, 2698, 2778, 2780] combine_rows_per_src=[5364, 5272, 5598, 5462, 5392, 5556, 5508, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=230080 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157713088 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315196096 weight_movement_bytes=18874368 total_nvlink_bytes=334070464 weight_to_token_ratio=0.059881 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38224 dup_rows=21690 dup_rows_remote=18786 dedup_ratio=0.330963 router_tokens_per_expert=[283, 248, 253, 228, 259, 245, 253, 266, 245, 253, 241, 242, 266, 229, 259, 241, 277, 270, 263, 289, 254, 249, 245, 260, 263, 246, 264, 270, 258, 242, 268, 276, 234, 254, 252, 258, 266, 239, 273, 244, 272, 262, 271, 254, 248, 230, 261, 243, 246, 258, 279, 255, 271, 301, 258, 236, 249, 254, 237, 251, 253, 261, 267, 265, 275, 255, 246, 226, 288, 244, 273, 265, 255, 259, 252, 269, 289, 251, 258, 267, 279, 264, 257, 244, 274, 240, 248, 272, 235, 277, 267, 234, 237, 242, 261, 258, 255, 261, 268, 299, 268, 259, 271, 262, 236, 244, 261, 245, 257, 252, 243, 223, 259, 242, 270, 226, 248, 254, 260, 259, 236, 257, 247, 243, 250, 235, 257, 253] payload_rows_per_dst=[5622, 5632, 5466, 5524, 5304, 5498, 5430, 5370] dup_rows_per_dst=[2904, 2794, 2618, 2758, 2536, 2844, 2614, 2622] combine_rows_per_src=[5622, 5632, 5466, 5524, 5304, 5498, 5430, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38170 dup_rows=21922 dup_rows_remote=19164 dedup_ratio=0.334503 router_tokens_per_expert=[248, 243, 297, 255, 249, 259, 261, 249, 248, 262, 257, 269, 250, 240, 239, 249, 236, 280, 286, 263, 257, 242, 288, 225, 242, 271, 252, 246, 272, 252, 241, 289, 216, 265, 257, 232, 281, 249, 266, 260, 253, 283, 262, 251, 249, 253, 226, 274, 251, 269, 277, 227, 280, 271, 238, 257, 252, 276, 263, 223, 275, 241, 262, 254, 247, 260, 259, 259, 265, 241, 259, 233, 252, 281, 256, 290, 269, 241, 246, 282, 242, 244, 269, 253, 241, 266, 250, 229, 244, 242, 257, 257, 268, 263, 248, 259, 260, 259, 252, 260, 242, 257, 258, 248, 295, 276, 262, 261, 273, 250, 233, 215, 243, 236, 279, 271, 276, 248, 269, 242, 284, 254, 248, 248, 257, 246, 245, 239] payload_rows_per_dst=[5440, 5480, 5426, 5514, 5498, 5402, 5444, 5410] dup_rows_per_dst=[2710, 2804, 2728, 2718, 2782, 2662, 2758, 2760] combine_rows_per_src=[5440, 5480, 5426, 5514, 5498, 5402, 5444, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38448 dup_rows=21596 dup_rows_remote=18756 dedup_ratio=0.329529 router_tokens_per_expert=[248, 272, 268, 250, 259, 252, 256, 241, 250, 283, 232, 257, 266, 267, 240, 254, 269, 264, 262, 267, 269, 262, 257, 229, 280, 240, 256, 256, 305, 256, 239, 255, 233, 265, 245, 252, 294, 242, 255, 257, 235, 235, 262, 273, 260, 256, 246, 263, 294, 250, 266, 266, 272, 256, 254, 252, 281, 245, 270, 236, 260, 247, 259, 220, 260, 274, 253, 252, 256, 244, 246, 241, 255, 269, 278, 252, 282, 273, 288, 252, 263, 261, 229, 234, 241, 245, 235, 259, 223, 250, 262, 241, 255, 255, 264, 241, 237, 270, 260, 248, 276, 274, 252, 230, 256, 267, 276, 248, 239, 229, 260, 242, 260, 240, 271, 267, 243, 250, 257, 268, 264, 249, 237, 250, 245, 284, 272, 252] payload_rows_per_dst=[5516, 5492, 5500, 5510, 5564, 5360, 5502, 5496] dup_rows_per_dst=[2674, 2840, 2646, 2746, 2786, 2556, 2626, 2722] combine_rows_per_src=[5516, 5492, 5500, 5510, 5564, 5360, 5502, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38182 dup_rows=21856 dup_rows_remote=19056 dedup_ratio=0.333496 router_tokens_per_expert=[269, 257, 252, 279, 238, 256, 270, 261, 224, 244, 267, 258, 276, 245, 258, 224, 245, 237, 257, 286, 269, 259, 260, 254, 248, 256, 230, 278, 262, 258, 253, 251, 253, 244, 254, 243, 269, 276, 276, 239, 239, 269, 259, 249, 263, 275, 266, 275, 249, 248, 252, 270, 250, 258, 242, 273, 236, 250, 267, 257, 238, 270, 232, 258, 282, 225, 269, 292, 270, 247, 260, 250, 266, 241, 271, 238, 232, 263, 270, 278, 243, 256, 266, 249, 254, 285, 246, 227, 259, 261, 292, 239, 252, 252, 255, 276, 229, 233, 258, 267, 220, 245, 262, 243, 260, 262, 261, 247, 249, 231, 269, 288, 237, 265, 286, 266, 252, 249, 268, 273, 233, 258, 228, 267, 249, 261, 242, 264] payload_rows_per_dst=[5412, 5478, 5498, 5476, 5508, 5446, 5380, 5482] dup_rows_per_dst=[2744, 2728, 2800, 2624, 2800, 2778, 2668, 2714] combine_rows_per_src=[5412, 5478, 5498, 5476, 5508, 5446, 5380, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229168 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157359920 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314490672 weight_movement_bytes=37748736 total_nvlink_bytes=352239408 weight_to_token_ratio=0.120031 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228920 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156499512 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312770104 weight_movement_bytes=37748736 total_nvlink_bytes=350518840 weight_to_token_ratio=0.120692 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228040 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156793544 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313359048 weight_movement_bytes=18874368 total_nvlink_bytes=332233416 weight_to_token_ratio=0.060232 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229336 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156573656 ingress_bytes=212967424 link_bytes=212967424 token_bytes=312917976 weight_movement_bytes=56623104 total_nvlink_bytes=369541080 weight_to_token_ratio=0.180952 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040549070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=228816 combine_payload_bytes=157483008 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157711824 ingress_bytes=195231744 link_bytes=195231744 token_bytes=315194832 weight_movement_bytes=37748736 total_nvlink_bytes=352943568 weight_to_token_ratio=0.119763 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870614511e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38306 dup_rows=21838 dup_rows_remote=19108 dedup_ratio=0.333221 router_tokens_per_expert=[261, 254, 242, 252, 257, 273, 239, 249, 260, 248, 260, 250, 239, 258, 307, 271, 224, 254, 275, 274, 252, 239, 237, 266, 237, 287, 258, 256, 265, 253, 276, 236, 240, 259, 237, 274, 263, 275, 245, 231, 241, 227, 256, 265, 260, 251, 273, 269, 255, 237, 261, 257, 264, 239, 250, 240, 270, 256, 251, 267, 276, 259, 280, 258, 244, 257, 248, 284, 255, 237, 241, 262, 234, 226, 262, 265, 246, 273, 271, 256, 266, 276, 224, 217, 233, 259, 260, 274, 291, 235, 274, 289, 249, 230, 235, 262, 292, 268, 234, 284, 261, 240, 246, 256, 255, 242, 249, 288, 283, 264, 259, 238, 257, 270, 279, 235, 260, 267, 266, 212, 273, 258, 239, 280, 250, 234, 236, 263] payload_rows_per_dst=[5472, 5480, 5448, 5478, 5392, 5444, 5502, 5482] dup_rows_per_dst=[2768, 2698, 2684, 2762, 2730, 2704, 2816, 2676] combine_rows_per_src=[5472, 5480, 5448, 5478, 5392, 5444, 5502, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228952 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622424 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015896 weight_movement_bytes=18874368 total_nvlink_bytes=331890264 weight_to_token_ratio=0.060298 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38278 dup_rows=21868 dup_rows_remote=19142 dedup_ratio=0.333679 router_tokens_per_expert=[250, 256, 272, 267, 263, 250, 239, 243, 261, 266, 261, 286, 269, 254, 244, 252, 268, 277, 258, 257, 281, 224, 251, 254, 267, 303, 244, 257, 236, 239, 271, 259, 247, 270, 246, 259, 266, 267, 262, 273, 256, 238, 257, 205, 226, 271, 270, 238, 263, 253, 265, 240, 271, 272, 255, 244, 247, 257, 233, 260, 284, 237, 223, 267, 251, 260, 225, 263, 252, 247, 264, 257, 261, 243, 274, 261, 240, 263, 260, 272, 237, 264, 247, 254, 263, 254, 236, 247, 261, 251, 253, 235, 251, 276, 267, 262, 258, 259, 268, 269, 275, 237, 222, 260, 267, 259, 241, 238, 264, 252, 248, 273, 249, 272, 259, 259, 263, 250, 268, 256, 242, 251, 246, 277, 255, 251, 266, 262] payload_rows_per_dst=[5544, 5468, 5450, 5438, 5426, 5390, 5458, 5494] dup_rows_per_dst=[2722, 2824, 2652, 2704, 2760, 2726, 2722, 2758] combine_rows_per_src=[5544, 5468, 5450, 5438, 5426, 5390, 5458, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229656 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157131032 ingress_bytes=213524480 link_bytes=213524480 token_bytes=314032408 weight_movement_bytes=56623104 total_nvlink_bytes=370655512 weight_to_token_ratio=0.180310 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045886507e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38414 dup_rows=21686 dup_rows_remote=18942 dedup_ratio=0.330902 router_tokens_per_expert=[263, 250, 268, 270, 232, 243, 262, 260, 257, 257, 275, 251, 241, 232, 273, 249, 259, 261, 243, 241, 258, 250, 280, 271, 251, 233, 270, 243, 239, 255, 265, 255, 248, 252, 273, 300, 211, 261, 260, 258, 240, 248, 289, 271, 237, 266, 255, 290, 233, 239, 260, 262, 255, 236, 266, 264, 251, 272, 268, 248, 236, 270, 286, 244, 263, 244, 260, 264, 277, 247, 272, 267, 241, 250, 296, 242, 259, 233, 215, 244, 237, 244, 268, 270, 273, 253, 243, 242, 246, 256, 229, 259, 266, 240, 266, 269, 265, 250, 260, 229, 260, 243, 236, 256, 225, 269, 270, 280, 270, 261, 271, 263, 270, 286, 230, 254, 269, 255, 232, 266, 287, 273, 265, 267, 240, 251, 250, 224] payload_rows_per_dst=[5460, 5440, 5552, 5436, 5480, 5426, 5524, 5532] dup_rows_per_dst=[2706, 2708, 2766, 2744, 2668, 2696, 2692, 2706] combine_rows_per_src=[5460, 5440, 5552, 5436, 5480, 5426, 5524, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38184 dup_rows=21894 dup_rows_remote=19026 dedup_ratio=0.334076 router_tokens_per_expert=[243, 244, 245, 268, 263, 249, 259, 241, 266, 248, 261, 259, 232, 263, 224, 260, 247, 287, 244, 257, 257, 274, 237, 250, 260, 238, 257, 259, 238, 279, 271, 261, 252, 260, 238, 243, 249, 216, 267, 272, 266, 256, 239, 255, 240, 259, 239, 249, 261, 255, 251, 257, 257, 246, 262, 245, 261, 274, 244, 252, 292, 295, 245, 259, 240, 272, 262, 267, 267, 263, 240, 263, 222, 244, 237, 262, 271, 261, 252, 240, 253, 274, 271, 251, 235, 273, 270, 272, 217, 290, 250, 254, 232, 261, 249, 237, 266, 279, 260, 255, 262, 237, 262, 283, 267, 251, 275, 243, 254, 252, 244, 266, 270, 278, 250, 259, 255, 275, 244, 257, 288, 267, 237, 237, 256, 262, 264, 264] payload_rows_per_dst=[5414, 5520, 5328, 5468, 5422, 5472, 5560, 5458] dup_rows_per_dst=[2636, 2712, 2672, 2844, 2704, 2706, 2752, 2868] combine_rows_per_src=[5414, 5520, 5328, 5468, 5422, 5472, 5560, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229680 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016368 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313803056 weight_movement_bytes=18874368 total_nvlink_bytes=332677424 weight_to_token_ratio=0.060147 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38058 dup_rows=21936 dup_rows_remote=19168 dedup_ratio=0.334717 router_tokens_per_expert=[226, 252, 249, 252, 294, 278, 257, 247, 267, 259, 272, 264, 246, 259, 241, 280, 259, 227, 255, 246, 285, 274, 253, 252, 244, 248, 259, 240, 266, 234, 269, 258, 269, 261, 266, 283, 275, 267, 244, 254, 244, 258, 238, 262, 247, 256, 238, 244, 235, 249, 255, 233, 248, 258, 231, 254, 267, 254, 260, 258, 268, 239, 245, 261, 259, 279, 263, 250, 266, 266, 254, 267, 288, 253, 239, 246, 250, 255, 230, 239, 249, 232, 254, 235, 269, 245, 263, 262, 246, 250, 247, 267, 245, 270, 259, 271, 262, 247, 249, 241, 270, 259, 256, 268, 277, 249, 254, 272, 285, 245, 276, 302, 244, 291, 258, 246, 260, 243, 224, 269, 232, 286, 231, 253, 275, 233, 250, 260] payload_rows_per_dst=[5542, 5402, 5434, 5418, 5558, 5470, 5406, 5370] dup_rows_per_dst=[2768, 2712, 2778, 2612, 2966, 2658, 2702, 2740] combine_rows_per_src=[5542, 5402, 5434, 5418, 5558, 5470, 5406, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38204 dup_rows=21818 dup_rows_remote=19126 dedup_ratio=0.332916 router_tokens_per_expert=[264, 241, 239, 256, 225, 254, 294, 234, 239, 244, 246, 248, 236, 277, 252, 247, 242, 239, 261, 240, 251, 234, 271, 264, 278, 260, 243, 282, 239, 274, 243, 255, 271, 246, 243, 263, 247, 254, 257, 278, 272, 236, 245, 266, 287, 245, 250, 256, 253, 276, 211, 258, 270, 271, 273, 244, 277, 261, 270, 235, 256, 213, 293, 278, 251, 276, 270, 291, 274, 255, 243, 248, 260, 232, 256, 274, 284, 273, 247, 260, 270, 247, 250, 234, 271, 267, 260, 275, 279, 256, 271, 229, 243, 249, 260, 229, 236, 233, 289, 255, 248, 246, 283, 240, 266, 260, 279, 284, 240, 246, 232, 266, 259, 254, 264, 255, 224, 253, 242, 256, 259, 260, 258, 239, 241, 252, 264, 274] payload_rows_per_dst=[5356, 5438, 5468, 5480, 5520, 5476, 5514, 5466] dup_rows_per_dst=[2636, 2714, 2764, 2798, 2868, 2704, 2692, 2642] combine_rows_per_src=[5356, 5438, 5468, 5480, 5520, 5476, 5514, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38320 dup_rows=21790 dup_rows_remote=19166 dedup_ratio=0.332489 router_tokens_per_expert=[266, 243, 283, 269, 280, 283, 273, 258, 244, 279, 240, 250, 208, 259, 262, 240, 237, 270, 244, 243, 229, 252, 257, 267, 268, 239, 248, 246, 256, 243, 279, 247, 274, 272, 258, 244, 258, 275, 254, 265, 244, 264, 242, 233, 256, 279, 246, 268, 252, 261, 254, 249, 217, 289, 227, 248, 244, 257, 270, 273, 247, 239, 242, 245, 238, 229, 252, 281, 255, 262, 243, 266, 263, 269, 265, 260, 246, 235, 274, 265, 279, 251, 248, 261, 249, 256, 273, 270, 275, 279, 258, 265, 272, 258, 248, 251, 297, 252, 245, 266, 260, 252, 265, 257, 269, 223, 264, 265, 252, 256, 241, 232, 230, 276, 273, 247, 249, 257, 254, 259, 215, 265, 265, 258, 260, 261, 249, 250] payload_rows_per_dst=[5498, 5426, 5476, 5400, 5472, 5522, 5460, 5492] dup_rows_per_dst=[2776, 2624, 2788, 2628, 2734, 2864, 2732, 2644] combine_rows_per_src=[5498, 5426, 5476, 5400, 5472, 5522, 5460, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38362 dup_rows=21668 dup_rows_remote=18946 dedup_ratio=0.330627 router_tokens_per_expert=[275, 243, 243, 260, 251, 242, 235, 269, 266, 268, 274, 234, 279, 277, 269, 269, 269, 258, 261, 231, 256, 264, 242, 252, 270, 253, 262, 249, 263, 235, 249, 254, 237, 273, 262, 273, 263, 246, 259, 280, 270, 246, 263, 257, 228, 247, 261, 249, 265, 245, 240, 257, 230, 253, 255, 253, 218, 253, 263, 254, 260, 260, 257, 276, 236, 230, 245, 273, 251, 252, 255, 266, 262, 257, 264, 289, 258, 262, 266, 249, 245, 267, 277, 262, 261, 274, 277, 256, 246, 257, 269, 257, 245, 256, 251, 269, 270, 261, 240, 252, 248, 262, 244, 247, 271, 252, 277, 243, 254, 237, 257, 261, 250, 248, 259, 266, 234, 254, 241, 242, 249, 279, 259, 266, 237, 254, 257, 238] payload_rows_per_dst=[5514, 5460, 5506, 5438, 5540, 5552, 5448, 5410] dup_rows_per_dst=[2794, 2676, 2722, 2640, 2690, 2786, 2704, 2656] combine_rows_per_src=[5514, 5460, 5506, 5438, 5540, 5552, 5448, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229424 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157573168 ingress_bytes=176218112 link_bytes=176218112 token_bytes=314916912 weight_movement_bytes=18874368 total_nvlink_bytes=333791280 weight_to_token_ratio=0.059934 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688435244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228840 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156630504 ingress_bytes=231899136 link_bytes=231899136 token_bytes=313032168 weight_movement_bytes=75497472 total_nvlink_bytes=388529640 weight_to_token_ratio=0.241181 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221943420e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=228904 combine_payload_bytes=155885568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156114472 ingress_bytes=174759936 link_bytes=174759936 token_bytes=312000040 weight_movement_bytes=18874368 total_nvlink_bytes=330874408 weight_to_token_ratio=0.060495 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674463719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38250 dup_rows=21764 dup_rows_remote=19052 dedup_ratio=0.332092 router_tokens_per_expert=[237, 269, 260, 271, 255, 269, 230, 285, 244, 270, 264, 257, 216, 279, 262, 247, 268, 278, 266, 256, 263, 233, 248, 244, 261, 264, 276, 249, 229, 262, 280, 244, 263, 299, 292, 250, 217, 242, 247, 265, 253, 276, 241, 264, 257, 233, 264, 271, 258, 254, 262, 262, 245, 253, 252, 253, 219, 257, 261, 243, 272, 265, 264, 237, 226, 241, 273, 253, 262, 261, 256, 258, 251, 240, 269, 255, 268, 287, 243, 274, 257, 281, 248, 238, 277, 267, 264, 251, 249, 252, 246, 241, 223, 257, 246, 228, 232, 271, 271, 274, 255, 254, 274, 238, 253, 259, 263, 249, 259, 260, 245, 247, 286, 280, 256, 258, 229, 225, 262, 259, 264, 250, 266, 247, 279, 251, 240, 243] payload_rows_per_dst=[5536, 5468, 5486, 5484, 5522, 5412, 5452, 5412] dup_rows_per_dst=[2694, 2774, 2782, 2630, 2712, 2638, 2756, 2778] combine_rows_per_src=[5536, 5468, 5486, 5484, 5522, 5412, 5452, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229320 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712904 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196488 weight_movement_bytes=37748736 total_nvlink_bytes=350945224 weight_to_token_ratio=0.120527 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229944 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157188664 ingress_bytes=175833088 link_bytes=175833088 token_bytes=314147384 weight_movement_bytes=18874368 total_nvlink_bytes=333021752 weight_to_token_ratio=0.060081 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684746134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229232 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157359984 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314490736 weight_movement_bytes=18874368 total_nvlink_bytes=333365104 weight_to_token_ratio=0.060016 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38090 dup_rows=21912 dup_rows_remote=19114 dedup_ratio=0.334351 router_tokens_per_expert=[241, 277, 266, 257, 253, 257, 267, 264, 270, 249, 237, 257, 253, 266, 270, 227, 252, 285, 263, 240, 260, 255, 259, 257, 256, 232, 259, 253, 266, 252, 260, 244, 262, 275, 259, 229, 276, 237, 239, 204, 242, 229, 262, 255, 242, 231, 218, 294, 269, 245, 269, 231, 264, 257, 243, 257, 259, 280, 278, 264, 263, 269, 272, 230, 265, 238, 278, 262, 296, 273, 243, 258, 292, 236, 275, 225, 267, 232, 244, 244, 265, 284, 259, 272, 261, 232, 262, 271, 265, 260, 283, 232, 234, 263, 275, 248, 230, 248, 238, 246, 227, 234, 271, 250, 250, 249, 242, 248, 268, 284, 263, 252, 252, 232, 250, 268, 257, 290, 283, 281, 247, 274, 272, 235, 254, 252, 278, 241] payload_rows_per_dst=[5484, 5478, 5332, 5488, 5456, 5534, 5410, 5442] dup_rows_per_dst=[2738, 2708, 2576, 2812, 2800, 2798, 2590, 2890] combine_rows_per_src=[5484, 5478, 5332, 5488, 5456, 5534, 5410, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229208 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901208 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573208 weight_movement_bytes=18874368 total_nvlink_bytes=332447576 weight_to_token_ratio=0.060191 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38166 dup_rows=21854 dup_rows_remote=19080 dedup_ratio=0.333466 router_tokens_per_expert=[254, 262, 269, 238, 260, 262, 237, 249, 241, 260, 274, 246, 221, 251, 233, 258, 271, 277, 241, 270, 266, 248, 239, 243, 234, 240, 260, 279, 252, 263, 247, 256, 273, 262, 237, 230, 254, 256, 254, 242, 228, 241, 297, 221, 281, 260, 255, 254, 254, 237, 244, 269, 234, 263, 284, 262, 245, 256, 286, 251, 280, 262, 260, 258, 248, 260, 257, 284, 253, 230, 247, 232, 274, 258, 260, 244, 263, 285, 235, 265, 273, 290, 246, 258, 266, 270, 264, 261, 255, 238, 237, 262, 267, 267, 242, 278, 269, 259, 254, 241, 232, 273, 251, 246, 247, 252, 258, 266, 238, 265, 252, 270, 234, 264, 260, 267, 237, 255, 281, 270, 271, 264, 253, 271, 270, 255, 220, 263] payload_rows_per_dst=[5376, 5466, 5358, 5516, 5540, 5524, 5472, 5430] dup_rows_per_dst=[2654, 2706, 2732, 2774, 2650, 2824, 2674, 2840] combine_rows_per_src=[5376, 5466, 5358, 5516, 5540, 5524, 5472, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=228816 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156245456 ingress_bytes=212639744 link_bytes=212639744 token_bytes=312262096 weight_movement_bytes=56623104 total_nvlink_bytes=368885200 weight_to_token_ratio=0.181332 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037409402e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38166 dup_rows=21816 dup_rows_remote=18858 dedup_ratio=0.332886 router_tokens_per_expert=[239, 268, 245, 264, 245, 239, 248, 254, 243, 262, 262, 233, 285, 276, 232, 255, 261, 263, 245, 265, 240, 256, 242, 226, 256, 251, 249, 255, 253, 258, 248, 254, 245, 235, 286, 256, 246, 261, 269, 227, 255, 265, 282, 246, 247, 265, 264, 248, 266, 263, 266, 262, 261, 282, 254, 261, 275, 271, 254, 273, 234, 266, 236, 248, 243, 228, 237, 252, 264, 266, 270, 248, 249, 286, 265, 253, 250, 252, 263, 255, 255, 269, 259, 278, 247, 229, 281, 243, 257, 242, 247, 274, 271, 249, 214, 274, 226, 247, 248, 246, 272, 261, 248, 224, 238, 243, 234, 246, 259, 274, 254, 281, 269, 249, 266, 259, 257, 259, 295, 266, 271, 276, 256, 246, 289, 244, 274, 280] payload_rows_per_dst=[5486, 5430, 5404, 5314, 5400, 5560, 5572, 5554] dup_rows_per_dst=[2614, 2614, 2618, 2466, 2762, 2790, 2994, 2958] combine_rows_per_src=[5486, 5430, 5404, 5314, 5400, 5560, 5572, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38038 dup_rows=21960 dup_rows_remote=19040 dedup_ratio=0.335083 router_tokens_per_expert=[267, 252, 259, 281, 244, 248, 247, 268, 245, 252, 234, 250, 257, 264, 233, 257, 285, 248, 273, 263, 263, 268, 254, 274, 263, 248, 269, 244, 239, 256, 260, 231, 258, 244, 264, 276, 258, 282, 248, 246, 247, 275, 275, 244, 245, 234, 268, 251, 276, 249, 246, 276, 261, 241, 251, 287, 243, 237, 233, 218, 241, 264, 247, 261, 254, 242, 238, 254, 237, 262, 254, 282, 271, 264, 277, 264, 235, 274, 267, 265, 246, 227, 279, 235, 283, 253, 265, 268, 234, 272, 262, 235, 230, 236, 249, 270, 229, 263, 263, 273, 270, 282, 265, 248, 253, 250, 266, 238, 261, 262, 266, 247, 264, 274, 256, 279, 256, 246, 244, 258, 237, 247, 261, 247, 239, 254, 272, 272] payload_rows_per_dst=[5538, 5544, 5470, 5368, 5472, 5414, 5444, 5326] dup_rows_per_dst=[2920, 2902, 2760, 2710, 2704, 2674, 2812, 2478] combine_rows_per_src=[5538, 5544, 5470, 5368, 5472, 5414, 5444, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38064 dup_rows=21970 dup_rows_remote=19154 dedup_ratio=0.335236 router_tokens_per_expert=[257, 237, 254, 281, 232, 261, 259, 244, 221, 242, 236, 247, 250, 267, 287, 243, 252, 251, 262, 246, 253, 267, 266, 279, 227, 246, 259, 259, 256, 254, 279, 259, 224, 256, 251, 253, 269, 274, 269, 263, 237, 269, 284, 248, 265, 240, 245, 233, 253, 255, 285, 278, 250, 240, 251, 259, 258, 253, 247, 256, 252, 251, 253, 234, 246, 218, 261, 245, 250, 273, 244, 224, 248, 224, 259, 242, 275, 251, 257, 274, 239, 273, 280, 242, 266, 249, 237, 243, 242, 244, 282, 256, 243, 264, 254, 236, 268, 274, 241, 264, 247, 264, 245, 266, 259, 264, 252, 280, 278, 263, 247, 247, 265, 271, 273, 270, 280, 266, 257, 269, 228, 259, 252, 266, 285, 288, 282, 269] payload_rows_per_dst=[5426, 5496, 5474, 5378, 5324, 5384, 5502, 5582] dup_rows_per_dst=[2610, 2734, 2686, 2772, 2658, 2716, 2816, 2978] combine_rows_per_src=[5426, 5496, 5474, 5378, 5324, 5384, 5502, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38442 dup_rows=21622 dup_rows_remote=18912 dedup_ratio=0.329926 router_tokens_per_expert=[236, 239, 280, 265, 263, 254, 251, 242, 271, 262, 223, 270, 250, 247, 291, 260, 268, 258, 260, 256, 244, 262, 243, 261, 261, 276, 242, 250, 264, 248, 273, 225, 294, 244, 247, 260, 253, 257, 252, 253, 287, 240, 230, 259, 272, 265, 271, 253, 308, 233, 257, 261, 249, 282, 270, 252, 283, 239, 272, 253, 263, 265, 236, 226, 261, 246, 251, 250, 229, 255, 243, 253, 250, 271, 268, 268, 242, 264, 260, 249, 255, 271, 269, 250, 274, 255, 225, 243, 262, 255, 250, 234, 252, 265, 259, 251, 222, 251, 271, 284, 254, 242, 274, 265, 273, 245, 248, 266, 218, 267, 248, 251, 251, 237, 241, 245, 256, 252, 262, 284, 224, 264, 261, 252, 277, 252, 245, 275] payload_rows_per_dst=[5516, 5472, 5458, 5532, 5502, 5454, 5460, 5520] dup_rows_per_dst=[2692, 2710, 2816, 2766, 2618, 2686, 2698, 2636] combine_rows_per_src=[5516, 5472, 5458, 5532, 5502, 5454, 5460, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38230 dup_rows=21796 dup_rows_remote=19084 dedup_ratio=0.332581 router_tokens_per_expert=[269, 272, 249, 299, 261, 262, 255, 252, 275, 246, 259, 270, 247, 252, 230, 248, 255, 223, 230, 281, 237, 259, 305, 246, 235, 259, 237, 265, 245, 250, 260, 249, 241, 242, 250, 266, 245, 265, 255, 265, 271, 259, 253, 268, 250, 269, 262, 250, 260, 265, 272, 251, 254, 244, 265, 268, 270, 284, 241, 250, 235, 259, 234, 255, 261, 272, 239, 278, 274, 252, 265, 236, 241, 247, 238, 289, 266, 245, 251, 255, 257, 241, 256, 284, 262, 269, 263, 261, 239, 243, 286, 239, 249, 291, 292, 275, 256, 244, 239, 268, 239, 242, 254, 228, 247, 245, 244, 266, 257, 252, 260, 259, 239, 242, 250, 262, 269, 258, 260, 262, 256, 248, 256, 253, 242, 243, 242, 270] payload_rows_per_dst=[5538, 5362, 5510, 5428, 5454, 5600, 5412, 5436] dup_rows_per_dst=[2754, 2710, 2712, 2786, 2764, 2814, 2588, 2668] combine_rows_per_src=[5538, 5362, 5510, 5428, 5454, 5600, 5412, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228984 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156556920 ingress_bytes=231825408 link_bytes=231825408 token_bytes=312884856 weight_movement_bytes=75497472 total_nvlink_bytes=388382328 weight_to_token_ratio=0.241295 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221236994e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228096 combine_payload_bytes=156327936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156556032 ingress_bytes=194076672 link_bytes=194076672 token_bytes=312883968 weight_movement_bytes=37748736 total_nvlink_bytes=350632704 weight_to_token_ratio=0.120648 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859547179e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=155803648 dispatch_meta_bytes=228312 combine_payload_bytes=155803648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156031960 ingress_bytes=174678016 link_bytes=174678016 token_bytes=311835608 weight_movement_bytes=18874368 total_nvlink_bytes=330709976 weight_to_token_ratio=0.060527 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673678802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38096 dup_rows=21924 dup_rows_remote=19104 dedup_ratio=0.334534 router_tokens_per_expert=[249, 228, 235, 274, 266, 258, 253, 269, 250, 257, 250, 228, 250, 276, 271, 273, 254, 247, 244, 246, 264, 262, 260, 269, 256, 247, 249, 246, 225, 257, 268, 248, 244, 266, 288, 254, 263, 246, 247, 237, 278, 254, 241, 267, 259, 264, 256, 243, 282, 272, 256, 253, 250, 257, 267, 260, 250, 257, 256, 258, 269, 267, 271, 254, 259, 266, 231, 266, 258, 228, 289, 263, 241, 258, 272, 276, 270, 242, 276, 273, 262, 221, 255, 253, 265, 261, 249, 261, 237, 255, 264, 261, 253, 248, 247, 247, 261, 251, 285, 283, 251, 245, 257, 242, 240, 259, 255, 242, 261, 250, 262, 288, 236, 250, 262, 247, 256, 247, 283, 209, 245, 262, 222, 257, 264, 260, 246, 268] payload_rows_per_dst=[5436, 5406, 5448, 5558, 5516, 5382, 5496, 5370] dup_rows_per_dst=[2738, 2678, 2766, 2800, 2820, 2696, 2768, 2658] combine_rows_per_src=[5436, 5406, 5448, 5558, 5516, 5382, 5496, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228872 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156139016 ingress_bytes=193658880 link_bytes=193658880 token_bytes=312049160 weight_movement_bytes=37748736 total_nvlink_bytes=349797896 weight_to_token_ratio=0.120970 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855544102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=229416 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157687848 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315146280 weight_movement_bytes=18874368 total_nvlink_bytes=334020648 weight_to_token_ratio=0.059891 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229256 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156819336 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313409416 weight_movement_bytes=37748736 total_nvlink_bytes=351158152 weight_to_token_ratio=0.120445 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38102 dup_rows=21914 dup_rows_remote=19168 dedup_ratio=0.334381 router_tokens_per_expert=[287, 226, 266, 252, 271, 228, 271, 269, 294, 272, 258, 241, 260, 237, 284, 254, 278, 259, 255, 265, 254, 254, 230, 254, 254, 251, 240, 259, 253, 250, 249, 259, 262, 302, 275, 259, 238, 262, 263, 240, 269, 257, 236, 240, 234, 243, 279, 229, 235, 247, 251, 254, 258, 295, 237, 285, 298, 260, 240, 223, 275, 258, 245, 265, 268, 263, 251, 240, 254, 242, 235, 267, 257, 282, 257, 248, 247, 267, 251, 234, 294, 253, 219, 271, 244, 274, 257, 251, 242, 267, 263, 246, 263, 276, 259, 254, 255, 258, 241, 249, 259, 250, 237, 283, 247, 257, 277, 217, 244, 261, 281, 246, 247, 233, 263, 259, 284, 251, 251, 248, 270, 270, 242, 249, 250, 252, 247, 246] payload_rows_per_dst=[5500, 5436, 5488, 5432, 5408, 5520, 5422, 5416] dup_rows_per_dst=[2840, 2692, 2688, 2820, 2718, 2746, 2702, 2708] combine_rows_per_src=[5500, 5436, 5488, 5432, 5408, 5520, 5422, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156041216 dispatch_meta_bytes=228800 combine_payload_bytes=156041216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156270016 ingress_bytes=193789952 link_bytes=193789952 token_bytes=312311232 weight_movement_bytes=37748736 total_nvlink_bytes=350059968 weight_to_token_ratio=0.120869 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856799969e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229080 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156294872 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312360664 weight_movement_bytes=18874368 total_nvlink_bytes=331235032 weight_to_token_ratio=0.060425 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38274 dup_rows=21776 dup_rows_remote=19104 dedup_ratio=0.332275 router_tokens_per_expert=[249, 249, 244, 249, 273, 283, 272, 248, 253, 237, 277, 259, 276, 295, 224, 254, 260, 255, 266, 250, 269, 269, 262, 275, 229, 258, 238, 237, 263, 265, 252, 254, 233, 238, 244, 280, 244, 254, 233, 236, 249, 216, 247, 242, 242, 253, 260, 260, 267, 274, 245, 271, 242, 272, 258, 272, 253, 247, 247, 255, 257, 243, 253, 223, 253, 264, 262, 278, 240, 243, 272, 248, 264, 269, 261, 255, 268, 240, 257, 241, 250, 242, 257, 275, 276, 282, 258, 273, 246, 245, 228, 271, 248, 237, 253, 284, 268, 255, 286, 259, 268, 255, 265, 270, 278, 258, 271, 262, 264, 243, 268, 251, 238, 259, 256, 232, 247, 268, 235, 280, 266, 278, 279, 259, 241, 231, 248, 236] payload_rows_per_dst=[5540, 5452, 5360, 5486, 5460, 5494, 5560, 5408] dup_rows_per_dst=[2744, 2752, 2502, 2672, 2770, 2756, 2882, 2698] combine_rows_per_src=[5540, 5452, 5360, 5486, 5460, 5494, 5560, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38248 dup_rows=21766 dup_rows_remote=19026 dedup_ratio=0.332123 router_tokens_per_expert=[229, 252, 248, 240, 250, 270, 265, 237, 271, 254, 270, 261, 239, 242, 242, 263, 248, 246, 270, 227, 257, 236, 251, 262, 271, 259, 254, 228, 271, 232, 268, 258, 247, 261, 271, 260, 253, 268, 258, 222, 274, 258, 272, 241, 246, 255, 261, 249, 252, 240, 282, 260, 234, 270, 264, 276, 280, 242, 270, 242, 272, 282, 270, 257, 253, 256, 280, 242, 252, 260, 252, 248, 267, 266, 271, 239, 266, 263, 259, 275, 270, 265, 266, 233, 293, 246, 228, 229, 244, 255, 286, 293, 236, 242, 236, 248, 252, 250, 253, 223, 253, 272, 281, 263, 240, 257, 253, 235, 230, 269, 261, 267, 250, 242, 272, 269, 280, 247, 271, 234, 251, 249, 256, 261, 286, 265, 267, 230] payload_rows_per_dst=[5434, 5420, 5494, 5558, 5454, 5436, 5452, 5522] dup_rows_per_dst=[2632, 2656, 2698, 2826, 2750, 2704, 2760, 2740] combine_rows_per_src=[5434, 5420, 5494, 5558, 5454, 5436, 5452, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38276 dup_rows=21736 dup_rows_remote=18886 dedup_ratio=0.331665 router_tokens_per_expert=[277, 262, 246, 243, 228, 245, 240, 253, 246, 271, 238, 245, 267, 254, 283, 237, 263, 269, 270, 242, 264, 260, 264, 270, 274, 231, 274, 241, 236, 243, 259, 277, 262, 240, 261, 240, 233, 241, 236, 259, 235, 256, 243, 276, 252, 249, 262, 261, 279, 271, 263, 259, 250, 261, 255, 258, 258, 241, 245, 257, 248, 244, 233, 249, 279, 264, 245, 254, 258, 261, 243, 274, 219, 262, 237, 270, 261, 305, 235, 230, 291, 261, 255, 265, 254, 268, 225, 272, 266, 287, 273, 245, 270, 221, 260, 260, 229, 255, 263, 264, 302, 268, 242, 264, 256, 266, 235, 249, 257, 259, 263, 279, 264, 255, 254, 272, 260, 253, 263, 222, 262, 264, 244, 239, 240, 270, 280, 256] payload_rows_per_dst=[5524, 5518, 5358, 5492, 5454, 5664, 5408, 5382] dup_rows_per_dst=[2850, 2756, 2654, 2664, 2726, 2924, 2590, 2572] combine_rows_per_src=[5524, 5518, 5358, 5492, 5454, 5664, 5408, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38308 dup_rows=21674 dup_rows_remote=18820 dedup_ratio=0.330719 router_tokens_per_expert=[247, 250, 253, 243, 233, 241, 243, 249, 257, 257, 260, 283, 234, 264, 263, 266, 267, 289, 277, 246, 235, 260, 258, 249, 243, 254, 269, 249, 240, 258, 265, 275, 225, 234, 272, 226, 239, 241, 257, 233, 258, 263, 260, 253, 274, 259, 253, 260, 244, 253, 253, 260, 238, 241, 256, 266, 255, 268, 279, 249, 272, 259, 260, 247, 265, 250, 238, 288, 251, 253, 254, 234, 268, 270, 247, 259, 267, 267, 244, 261, 233, 266, 244, 250, 284, 272, 238, 229, 260, 263, 239, 258, 255, 256, 282, 263, 265, 273, 248, 260, 266, 250, 279, 281, 261, 257, 259, 254, 256, 256, 259, 280, 263, 250, 253, 235, 264, 253, 261, 241, 261, 248, 264, 249, 241, 259, 252, 278] payload_rows_per_dst=[5418, 5520, 5448, 5516, 5458, 5460, 5554, 5488] dup_rows_per_dst=[2668, 2748, 2566, 2684, 2774, 2724, 2854, 2656] combine_rows_per_src=[5418, 5520, 5448, 5516, 5458, 5460, 5554, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38294 dup_rows=21814 dup_rows_remote=19152 dedup_ratio=0.332855 router_tokens_per_expert=[240, 243, 254, 275, 278, 275, 258, 253, 233, 259, 256, 269, 249, 249, 237, 244, 241, 260, 233, 281, 254, 248, 264, 255, 249, 249, 250, 245, 267, 257, 244, 261, 240, 261, 265, 252, 240, 245, 246, 254, 282, 242, 241, 276, 259, 235, 260, 247, 242, 268, 240, 271, 272, 239, 284, 273, 236, 246, 265, 255, 291, 252, 269, 274, 243, 258, 233, 265, 257, 269, 277, 243, 262, 255, 253, 254, 297, 265, 243, 260, 234, 249, 227, 268, 284, 269, 224, 263, 270, 248, 297, 258, 251, 246, 247, 244, 252, 242, 242, 259, 230, 269, 238, 267, 257, 266, 231, 247, 260, 251, 275, 255, 263, 269, 257, 239, 258, 238, 254, 266, 267, 262, 277, 276, 245, 265, 262, 264] payload_rows_per_dst=[5430, 5458, 5428, 5554, 5444, 5430, 5402, 5576] dup_rows_per_dst=[2714, 2658, 2662, 2800, 2824, 2728, 2680, 2748] combine_rows_per_src=[5430, 5458, 5428, 5554, 5444, 5430, 5402, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43348 payload_rows_remote=37842 dup_rows=22188 dup_rows_remote=19412 dedup_ratio=0.338562 router_tokens_per_expert=[264, 259, 270, 247, 234, 272, 249, 270, 245, 272, 269, 251, 272, 267, 264, 259, 247, 248, 253, 246, 263, 247, 252, 245, 265, 262, 265, 248, 278, 263, 257, 267, 243, 237, 244, 239, 219, 257, 233, 250, 261, 250, 243, 252, 288, 271, 247, 267, 243, 257, 271, 256, 256, 236, 260, 257, 256, 270, 256, 279, 270, 270, 231, 245, 260, 233, 289, 263, 274, 238, 230, 243, 248, 242, 275, 219, 236, 265, 257, 248, 244, 242, 264, 231, 254, 257, 243, 277, 230, 249, 265, 237, 239, 255, 257, 248, 257, 282, 261, 263, 275, 261, 244, 265, 262, 262, 242, 281, 280, 262, 264, 269, 259, 245, 258, 269, 261, 263, 286, 257, 252, 253, 237, 264, 257, 253, 265, 263] payload_rows_per_dst=[5470, 5506, 5320, 5376, 5292, 5306, 5582, 5496] dup_rows_per_dst=[2858, 2776, 2682, 2850, 2748, 2678, 2878, 2718] combine_rows_per_src=[5470, 5506, 5320, 5376, 5292, 5306, 5582, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229512 combine_payload_bytes=156770304 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156999816 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770120 weight_movement_bytes=37748736 total_nvlink_bytes=351518856 weight_to_token_ratio=0.120307 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229096 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156892904 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313556712 weight_movement_bytes=56623104 total_nvlink_bytes=370179816 weight_to_token_ratio=0.180583 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=228648 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007144 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313785640 weight_movement_bytes=18874368 total_nvlink_bytes=332660008 weight_to_token_ratio=0.060151 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38206 dup_rows=21816 dup_rows_remote=18880 dedup_ratio=0.332886 router_tokens_per_expert=[260, 248, 269, 235, 259, 262, 252, 240, 223, 255, 275, 248, 254, 260, 244, 271, 262, 256, 271, 242, 291, 256, 236, 244, 240, 289, 245, 252, 241, 252, 262, 210, 269, 259, 242, 256, 270, 254, 248, 264, 286, 265, 259, 246, 259, 240, 252, 269, 259, 262, 246, 252, 277, 233, 265, 256, 273, 243, 264, 236, 241, 260, 248, 274, 288, 259, 277, 257, 259, 290, 244, 295, 254, 257, 261, 254, 266, 252, 254, 258, 244, 272, 253, 243, 247, 251, 262, 266, 263, 255, 258, 258, 224, 247, 253, 235, 277, 246, 247, 256, 263, 240, 269, 253, 244, 251, 250, 261, 263, 237, 247, 272, 253, 243, 284, 253, 227, 284, 246, 248, 273, 265, 253, 252, 231, 239, 274, 280] payload_rows_per_dst=[5442, 5446, 5464, 5506, 5514, 5430, 5510, 5408] dup_rows_per_dst=[2668, 2652, 2812, 2672, 2936, 2632, 2642, 2802] combine_rows_per_src=[5442, 5446, 5464, 5506, 5514, 5430, 5510, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=228512 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157138080 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314047648 weight_movement_bytes=37748736 total_nvlink_bytes=351796384 weight_to_token_ratio=0.120201 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229784 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157082008 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934232 weight_movement_bytes=18874368 total_nvlink_bytes=332808600 weight_to_token_ratio=0.060122 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38282 dup_rows=21806 dup_rows_remote=19178 dedup_ratio=0.332733 router_tokens_per_expert=[245, 235, 247, 273, 250, 243, 255, 236, 297, 269, 258, 247, 278, 253, 258, 247, 262, 278, 228, 251, 243, 243, 246, 262, 247, 259, 250, 275, 277, 269, 260, 249, 257, 259, 268, 260, 259, 265, 278, 243, 272, 233, 263, 266, 251, 247, 279, 269, 263, 272, 244, 259, 245, 274, 289, 250, 225, 251, 247, 260, 246, 238, 231, 249, 255, 230, 237, 260, 278, 274, 287, 267, 277, 279, 242, 240, 233, 278, 218, 272, 261, 252, 271, 246, 270, 266, 259, 248, 255, 253, 238, 232, 234, 262, 241, 250, 244, 286, 269, 251, 270, 259, 262, 238, 255, 252, 273, 212, 255, 259, 271, 233, 261, 260, 238, 267, 270, 249, 245, 283, 276, 263, 242, 268, 249, 253, 244, 244] payload_rows_per_dst=[5478, 5492, 5454, 5424, 5470, 5448, 5470, 5494] dup_rows_per_dst=[2704, 2706, 2884, 2662, 2784, 2628, 2708, 2730] combine_rows_per_src=[5478, 5492, 5454, 5424, 5470, 5448, 5470, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=155000832 dispatch_meta_bytes=229016 combine_payload_bytes=155000832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155229848 ingress_bytes=173875200 link_bytes=173875200 token_bytes=310230680 weight_movement_bytes=18874368 total_nvlink_bytes=329105048 weight_to_token_ratio=0.060840 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.665986614e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228344 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156720120 ingress_bytes=213114880 link_bytes=213114880 token_bytes=313211896 weight_movement_bytes=56623104 total_nvlink_bytes=369835000 weight_to_token_ratio=0.180782 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041961921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229840 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032912 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835984 weight_movement_bytes=18874368 total_nvlink_bytes=332710352 weight_to_token_ratio=0.060141 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38162 dup_rows=21876 dup_rows_remote=19036 dedup_ratio=0.333801 router_tokens_per_expert=[267, 260, 252, 261, 259, 264, 265, 267, 270, 242, 252, 231, 252, 248, 258, 287, 247, 257, 235, 255, 244, 261, 289, 276, 256, 233, 276, 257, 260, 258, 253, 219, 265, 277, 261, 274, 273, 255, 254, 250, 263, 254, 245, 241, 253, 242, 253, 245, 265, 265, 254, 238, 264, 277, 282, 276, 275, 265, 261, 240, 251, 263, 256, 237, 247, 247, 250, 269, 253, 259, 226, 256, 243, 253, 265, 244, 257, 264, 258, 270, 252, 256, 251, 260, 291, 270, 254, 264, 269, 252, 255, 233, 244, 237, 244, 231, 253, 248, 268, 266, 236, 258, 246, 267, 241, 278, 254, 248, 252, 226, 260, 262, 248, 249, 268, 259, 240, 239, 248, 290, 276, 242, 253, 252, 248, 254, 259, 271] payload_rows_per_dst=[5522, 5522, 5436, 5498, 5436, 5396, 5406, 5444] dup_rows_per_dst=[2748, 2630, 2774, 2840, 2686, 2730, 2720, 2748] combine_rows_per_src=[5522, 5522, 5436, 5498, 5436, 5396, 5406, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38134 dup_rows=21852 dup_rows_remote=18756 dedup_ratio=0.333435 router_tokens_per_expert=[261, 254, 252, 270, 246, 266, 251, 229, 249, 248, 256, 259, 259, 233, 259, 249, 257, 260, 240, 240, 274, 295, 239, 280, 254, 267, 261, 265, 274, 242, 248, 281, 277, 254, 237, 265, 236, 249, 247, 256, 245, 253, 240, 250, 255, 245, 261, 265, 258, 252, 253, 241, 279, 236, 258, 240, 258, 216, 268, 252, 271, 235, 249, 248, 263, 263, 242, 252, 252, 240, 246, 282, 271, 208, 234, 244, 247, 249, 260, 223, 238, 261, 266, 267, 272, 286, 269, 263, 242, 267, 261, 276, 270, 250, 260, 253, 244, 279, 266, 242, 267, 271, 254, 257, 251, 243, 247, 256, 246, 242, 248, 267, 277, 264, 272, 251, 266, 233, 247, 275, 299, 276, 276, 247, 262, 272, 256, 271] payload_rows_per_dst=[5476, 5540, 5344, 5398, 5358, 5516, 5502, 5550] dup_rows_per_dst=[2606, 2814, 2534, 2630, 2592, 2730, 2850, 3096] combine_rows_per_src=[5476, 5540, 5344, 5398, 5358, 5516, 5502, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38274 dup_rows=21736 dup_rows_remote=19062 dedup_ratio=0.331665 router_tokens_per_expert=[255, 247, 249, 273, 303, 248, 258, 276, 258, 249, 222, 220, 265, 245, 278, 231, 252, 257, 254, 266, 287, 236, 251, 272, 257, 250, 260, 239, 266, 257, 265, 264, 262, 270, 270, 249, 275, 273, 239, 253, 256, 243, 269, 259, 274, 262, 271, 231, 267, 257, 262, 254, 257, 271, 264, 253, 255, 247, 249, 280, 234, 245, 239, 260, 282, 251, 265, 241, 253, 247, 242, 242, 283, 274, 243, 234, 242, 204, 228, 229, 274, 230, 270, 229, 269, 246, 262, 287, 262, 262, 255, 268, 264, 265, 270, 261, 260, 265, 240, 262, 262, 251, 263, 243, 245, 252, 258, 239, 268, 257, 249, 263, 260, 231, 256, 279, 244, 248, 249, 262, 255, 248, 274, 227, 263, 269, 251, 281] payload_rows_per_dst=[5526, 5528, 5582, 5566, 5450, 5358, 5362, 5428] dup_rows_per_dst=[2674, 2746, 2954, 2758, 2654, 2588, 2686, 2676] combine_rows_per_src=[5526, 5528, 5582, 5566, 5450, 5358, 5362, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38318 dup_rows=21700 dup_rows_remote=18866 dedup_ratio=0.331116 router_tokens_per_expert=[253, 244, 231, 264, 265, 266, 210, 252, 246, 250, 256, 263, 260, 246, 272, 240, 263, 262, 251, 254, 262, 241, 247, 268, 279, 272, 243, 248, 250, 265, 258, 241, 258, 250, 244, 237, 238, 233, 287, 242, 272, 279, 262, 273, 231, 247, 271, 264, 257, 288, 256, 234, 240, 247, 258, 240, 252, 271, 268, 264, 254, 280, 261, 232, 285, 241, 262, 263, 281, 256, 263, 249, 255, 252, 286, 263, 273, 258, 235, 292, 220, 230, 248, 241, 258, 246, 267, 243, 258, 223, 284, 236, 256, 266, 268, 271, 254, 257, 204, 261, 276, 282, 263, 269, 285, 260, 236, 244, 258, 277, 254, 271, 260, 246, 262, 260, 242, 252, 249, 234, 251, 292, 243, 243, 287, 248, 275, 232] payload_rows_per_dst=[5408, 5508, 5430, 5530, 5548, 5436, 5518, 5458] dup_rows_per_dst=[2628, 2700, 2746, 2624, 2880, 2594, 2834, 2694] combine_rows_per_src=[5408, 5508, 5430, 5530, 5548, 5436, 5518, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38116 dup_rows=21820 dup_rows_remote=19010 dedup_ratio=0.332947 router_tokens_per_expert=[244, 254, 267, 264, 255, 250, 248, 250, 264, 303, 249, 266, 266, 261, 243, 260, 237, 265, 255, 263, 273, 282, 245, 232, 247, 268, 276, 274, 241, 242, 275, 263, 245, 273, 259, 256, 280, 274, 253, 248, 246, 243, 262, 257, 298, 281, 258, 272, 279, 214, 253, 252, 248, 269, 270, 249, 282, 268, 221, 254, 257, 260, 264, 226, 255, 239, 242, 255, 253, 246, 273, 215, 282, 273, 240, 233, 285, 249, 253, 246, 253, 259, 240, 263, 247, 247, 266, 257, 259, 253, 256, 284, 258, 238, 248, 271, 253, 247, 259, 247, 247, 245, 254, 246, 254, 246, 256, 240, 231, 247, 254, 267, 259, 235, 244, 228, 275, 248, 262, 248, 251, 245, 266, 280, 249, 280, 272, 242] payload_rows_per_dst=[5450, 5482, 5600, 5442, 5468, 5486, 5368, 5420] dup_rows_per_dst=[2838, 2794, 2810, 2690, 2610, 2712, 2618, 2748] combine_rows_per_src=[5450, 5482, 5600, 5442, 5468, 5486, 5368, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228792 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540344 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851896 weight_movement_bytes=18874368 total_nvlink_bytes=331726264 weight_to_token_ratio=0.060330 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38172 dup_rows=21898 dup_rows_remote=19058 dedup_ratio=0.334137 router_tokens_per_expert=[257, 242, 257, 241, 247, 259, 247, 268, 253, 291, 249, 248, 273, 251, 216, 259, 269, 268, 266, 284, 241, 244, 233, 237, 251, 267, 252, 257, 289, 286, 243, 266, 273, 255, 254, 245, 257, 228, 261, 216, 233, 265, 242, 290, 259, 256, 256, 276, 261, 245, 219, 261, 249, 227, 249, 255, 251, 242, 283, 258, 246, 242, 244, 270, 252, 246, 270, 239, 269, 239, 259, 254, 254, 273, 264, 251, 277, 298, 252, 276, 252, 257, 226, 280, 258, 268, 268, 257, 284, 248, 279, 252, 261, 280, 216, 295, 278, 256, 273, 233, 250, 239, 265, 235, 278, 267, 242, 291, 251, 252, 249, 259, 235, 256, 236, 245, 252, 244, 218, 251, 278, 262, 238, 251, 250, 260, 245, 296] payload_rows_per_dst=[5434, 5466, 5436, 5406, 5512, 5528, 5404, 5452] dup_rows_per_dst=[2682, 2840, 2696, 2626, 2834, 2834, 2832, 2554] combine_rows_per_src=[5434, 5466, 5436, 5406, 5512, 5528, 5404, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=227560 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156424424 ingress_bytes=193945600 link_bytes=193945600 token_bytes=312621288 weight_movement_bytes=37748736 total_nvlink_bytes=350370024 weight_to_token_ratio=0.120749 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858291312e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38324 dup_rows=21748 dup_rows_remote=19050 dedup_ratio=0.331848 router_tokens_per_expert=[259, 245, 249, 228, 272, 252, 239, 247, 266, 260, 277, 267, 236, 254, 256, 247, 227, 260, 240, 268, 289, 239, 231, 256, 254, 258, 252, 258, 254, 269, 256, 268, 265, 254, 263, 264, 269, 239, 254, 248, 287, 244, 232, 242, 235, 257, 256, 270, 254, 249, 274, 246, 248, 280, 279, 249, 243, 243, 255, 258, 262, 240, 286, 274, 264, 244, 265, 270, 265, 252, 262, 257, 249, 232, 227, 280, 238, 251, 249, 276, 271, 265, 291, 287, 241, 245, 272, 246, 247, 259, 248, 263, 257, 300, 237, 264, 265, 277, 240, 258, 249, 250, 254, 248, 237, 280, 241, 246, 251, 240, 274, 276, 259, 258, 245, 252, 254, 250, 264, 263, 273, 257, 222, 242, 231, 263, 275, 248] payload_rows_per_dst=[5368, 5524, 5456, 5546, 5464, 5534, 5452, 5444] dup_rows_per_dst=[2740, 2634, 2702, 2734, 2698, 2852, 2720, 2668] combine_rows_per_src=[5368, 5524, 5456, 5546, 5464, 5534, 5452, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229344 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999648 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769952 weight_movement_bytes=18874368 total_nvlink_bytes=332644320 weight_to_token_ratio=0.060154 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228736 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179264 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314129792 weight_movement_bytes=18874368 total_nvlink_bytes=333004160 weight_to_token_ratio=0.060085 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38196 dup_rows=21924 dup_rows_remote=19208 dedup_ratio=0.334534 router_tokens_per_expert=[238, 260, 240, 245, 252, 257, 273, 278, 262, 242, 256, 281, 265, 264, 261, 259, 238, 260, 257, 255, 259, 240, 266, 230, 260, 248, 277, 262, 244, 263, 264, 256, 259, 261, 247, 263, 247, 249, 268, 245, 232, 243, 251, 241, 259, 266, 260, 250, 241, 238, 237, 260, 283, 263, 262, 225, 227, 246, 245, 246, 274, 278, 241, 236, 247, 269, 266, 277, 242, 237, 271, 254, 254, 250, 250, 272, 252, 243, 257, 242, 232, 249, 272, 272, 253, 261, 264, 245, 235, 245, 259, 268, 252, 262, 247, 250, 283, 228, 267, 265, 254, 251, 243, 288, 279, 250, 275, 252, 254, 276, 268, 268, 285, 264, 264, 274, 224, 264, 267, 253, 246, 260, 251, 247, 272, 263, 247, 282] payload_rows_per_dst=[5458, 5438, 5426, 5316, 5414, 5416, 5616, 5528] dup_rows_per_dst=[2808, 2720, 2656, 2688, 2752, 2716, 2786, 2798] combine_rows_per_src=[5458, 5438, 5426, 5316, 5414, 5416, 5616, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=228504 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156351640 ingress_bytes=212746240 link_bytes=212746240 token_bytes=312474776 weight_movement_bytes=56623104 total_nvlink_bytes=369097880 weight_to_token_ratio=0.181209 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038429794e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228920 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156581432 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312933944 weight_movement_bytes=37748736 total_nvlink_bytes=350682680 weight_to_token_ratio=0.120628 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229496 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157204600 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314179704 weight_movement_bytes=37748736 total_nvlink_bytes=351928440 weight_to_token_ratio=0.120150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229616 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156680432 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313131248 weight_movement_bytes=37748736 total_nvlink_bytes=350879984 weight_to_token_ratio=0.120552 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38172 dup_rows=21736 dup_rows_remote=18978 dedup_ratio=0.331665 router_tokens_per_expert=[249, 258, 271, 295, 251, 247, 260, 262, 238, 279, 253, 256, 283, 251, 272, 252, 240, 270, 238, 248, 230, 244, 277, 257, 251, 250, 238, 272, 249, 258, 270, 255, 263, 247, 284, 256, 272, 236, 259, 256, 248, 262, 256, 251, 255, 237, 282, 257, 255, 263, 271, 274, 254, 261, 265, 268, 276, 279, 261, 284, 229, 223, 258, 272, 256, 280, 247, 258, 268, 264, 250, 255, 249, 259, 237, 227, 238, 273, 282, 262, 235, 275, 254, 227, 240, 261, 233, 250, 255, 246, 256, 246, 278, 259, 257, 265, 283, 261, 266, 238, 237, 241, 240, 255, 250, 256, 227, 265, 267, 244, 245, 243, 253, 251, 273, 251, 257, 268, 235, 270, 271, 261, 238, 260, 240, 249, 238, 255] payload_rows_per_dst=[5510, 5444, 5484, 5628, 5500, 5418, 5360, 5456] dup_rows_per_dst=[2844, 2650, 2758, 2758, 2710, 2656, 2676, 2684] combine_rows_per_src=[5510, 5444, 5484, 5628, 5500, 5418, 5360, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=37964 dup_rows=21988 dup_rows_remote=19300 dedup_ratio=0.335510 router_tokens_per_expert=[261, 249, 272, 269, 242, 266, 251, 244, 264, 244, 271, 275, 271, 255, 250, 251, 240, 246, 256, 231, 292, 249, 260, 229, 243, 276, 250, 235, 244, 254, 273, 245, 270, 252, 267, 265, 267, 265, 226, 236, 274, 243, 262, 250, 267, 269, 268, 238, 242, 278, 273, 245, 280, 248, 254, 264, 202, 272, 267, 256, 247, 256, 259, 257, 248, 277, 270, 261, 257, 250, 277, 245, 248, 259, 253, 238, 263, 257, 245, 240, 276, 221, 259, 246, 236, 244, 244, 250, 287, 231, 245, 275, 265, 247, 271, 240, 252, 254, 253, 247, 269, 253, 257, 267, 270, 267, 279, 262, 270, 242, 266, 245, 243, 247, 257, 279, 273, 257, 276, 248, 262, 252, 265, 250, 250, 235, 269, 250] payload_rows_per_dst=[5408, 5372, 5528, 5388, 5450, 5318, 5500, 5584] dup_rows_per_dst=[2674, 2634, 2872, 2696, 2862, 2756, 2806, 2688] combine_rows_per_src=[5408, 5372, 5528, 5388, 5450, 5318, 5500, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38280 dup_rows=21754 dup_rows_remote=18900 dedup_ratio=0.331940 router_tokens_per_expert=[286, 260, 252, 259, 254, 269, 247, 251, 258, 243, 272, 250, 275, 248, 262, 266, 244, 255, 250, 262, 269, 244, 244, 259, 248, 285, 252, 251, 254, 270, 238, 257, 262, 270, 282, 254, 259, 273, 257, 275, 276, 247, 237, 263, 235, 239, 293, 249, 238, 280, 250, 265, 263, 257, 279, 259, 245, 254, 251, 268, 256, 241, 269, 255, 237, 239, 226, 271, 280, 254, 265, 247, 277, 240, 259, 254, 239, 280, 254, 277, 236, 252, 230, 244, 273, 262, 210, 235, 229, 251, 263, 253, 249, 252, 252, 264, 246, 252, 248, 263, 255, 240, 247, 254, 254, 233, 233, 270, 250, 224, 254, 240, 250, 252, 265, 267, 253, 236, 277, 280, 278, 264, 297, 265, 247, 270, 256, 259] payload_rows_per_dst=[5502, 5686, 5552, 5548, 5354, 5300, 5404, 5436] dup_rows_per_dst=[2854, 2960, 2810, 2642, 2602, 2610, 2522, 2754] combine_rows_per_src=[5502, 5686, 5552, 5548, 5354, 5300, 5404, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38308 dup_rows=21656 dup_rows_remote=18932 dedup_ratio=0.330444 router_tokens_per_expert=[270, 279, 258, 266, 292, 254, 252, 249, 269, 252, 247, 272, 227, 252, 243, 251, 263, 234, 244, 222, 272, 259, 246, 268, 268, 253, 223, 252, 249, 230, 242, 269, 262, 275, 249, 260, 245, 266, 272, 249, 252, 229, 284, 248, 244, 262, 260, 263, 265, 232, 262, 246, 246, 243, 284, 238, 225, 247, 269, 262, 254, 266, 259, 253, 251, 237, 251, 230, 262, 238, 262, 253, 260, 257, 252, 280, 256, 229, 220, 242, 265, 265, 282, 249, 258, 255, 247, 247, 246, 249, 265, 253, 256, 270, 276, 263, 247, 268, 252, 252, 245, 250, 257, 266, 246, 278, 257, 261, 293, 256, 251, 269, 247, 283, 264, 258, 270, 259, 252, 257, 285, 278, 269, 247, 243, 256, 263, 265] payload_rows_per_dst=[5498, 5450, 5532, 5398, 5354, 5460, 5572, 5616] dup_rows_per_dst=[2768, 2538, 2708, 2704, 2606, 2832, 2724, 2776] combine_rows_per_src=[5498, 5450, 5532, 5398, 5354, 5460, 5572, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228600 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156581112 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312933624 weight_movement_bytes=37748736 total_nvlink_bytes=350682360 weight_to_token_ratio=0.120629 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38126 dup_rows=21920 dup_rows_remote=19222 dedup_ratio=0.334473 router_tokens_per_expert=[274, 244, 249, 242, 265, 267, 274, 261, 264, 248, 231, 260, 255, 265, 260, 270, 254, 248, 261, 256, 255, 232, 254, 243, 254, 230, 265, 241, 248, 257, 245, 292, 281, 255, 235, 221, 259, 272, 247, 225, 279, 271, 261, 262, 255, 255, 265, 251, 223, 257, 254, 265, 224, 229, 240, 284, 260, 282, 255, 236, 265, 283, 269, 274, 251, 272, 250, 230, 265, 237, 258, 267, 257, 269, 242, 240, 268, 268, 245, 260, 261, 253, 245, 246, 234, 235, 257, 246, 276, 301, 225, 262, 256, 233, 237, 269, 240, 244, 251, 275, 285, 271, 252, 271, 234, 262, 266, 261, 230, 299, 265, 232, 292, 231, 270, 270, 262, 282, 249, 241, 266, 271, 257, 240, 244, 255, 262, 265] payload_rows_per_dst=[5482, 5412, 5490, 5404, 5446, 5400, 5464, 5518] dup_rows_per_dst=[2776, 2658, 2698, 2796, 2712, 2672, 2812, 2796] combine_rows_per_src=[5482, 5412, 5490, 5404, 5446, 5400, 5464, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38252 dup_rows=21700 dup_rows_remote=18916 dedup_ratio=0.331116 router_tokens_per_expert=[238, 266, 254, 253, 243, 263, 252, 236, 242, 248, 259, 257, 264, 260, 267, 263, 272, 266, 256, 261, 248, 267, 238, 241, 260, 264, 246, 268, 276, 285, 271, 265, 248, 256, 282, 267, 257, 248, 250, 274, 281, 269, 274, 228, 297, 253, 223, 254, 256, 232, 272, 268, 228, 251, 247, 219, 270, 257, 267, 270, 272, 264, 238, 250, 274, 292, 235, 248, 252, 271, 272, 250, 253, 294, 274, 270, 221, 242, 271, 245, 257, 257, 237, 261, 230, 253, 252, 269, 262, 266, 256, 261, 236, 232, 240, 275, 248, 261, 218, 278, 234, 250, 249, 225, 253, 260, 267, 259, 243, 278, 254, 236, 254, 237, 238, 268, 281, 253, 254, 260, 263, 265, 243, 255, 246, 273, 254, 232] payload_rows_per_dst=[5456, 5584, 5550, 5440, 5556, 5414, 5382, 5454] dup_rows_per_dst=[2674, 2784, 2772, 2682, 2772, 2674, 2644, 2698] combine_rows_per_src=[5456, 5584, 5550, 5440, 5556, 5414, 5382, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=155500544 dispatch_meta_bytes=229056 combine_payload_bytes=155500544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155729600 ingress_bytes=193249280 link_bytes=193249280 token_bytes=311230144 weight_movement_bytes=37748736 total_nvlink_bytes=348978880 weight_to_token_ratio=0.121289 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851619516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=228720 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157023600 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313818480 weight_movement_bytes=18874368 total_nvlink_bytes=332692848 weight_to_token_ratio=0.060144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38198 dup_rows=21846 dup_rows_remote=19180 dedup_ratio=0.333344 router_tokens_per_expert=[289, 251, 240, 241, 249, 265, 244, 254, 231, 270, 264, 244, 239, 266, 255, 258, 252, 256, 266, 248, 250, 306, 241, 276, 250, 236, 277, 283, 260, 234, 244, 269, 249, 247, 265, 256, 250, 266, 225, 262, 242, 259, 266, 244, 266, 255, 246, 258, 256, 243, 257, 259, 274, 227, 215, 262, 255, 252, 251, 276, 288, 266, 236, 284, 228, 251, 252, 263, 271, 255, 247, 268, 257, 253, 270, 272, 256, 235, 253, 248, 250, 273, 252, 262, 245, 247, 213, 260, 265, 227, 254, 258, 288, 249, 255, 246, 273, 285, 227, 269, 254, 249, 274, 259, 273, 249, 275, 274, 255, 238, 269, 255, 264, 264, 238, 250, 257, 233, 269, 231, 271, 287, 252, 248, 232, 273, 269, 264] payload_rows_per_dst=[5446, 5514, 5390, 5442, 5492, 5354, 5584, 5468] dup_rows_per_dst=[2674, 2782, 2722, 2760, 2666, 2734, 2772, 2736] combine_rows_per_src=[5446, 5514, 5390, 5442, 5492, 5354, 5584, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38216 dup_rows=21850 dup_rows_remote=19136 dedup_ratio=0.333405 router_tokens_per_expert=[273, 259, 257, 236, 290, 252, 257, 265, 267, 233, 246, 262, 238, 265, 258, 264, 257, 244, 249, 265, 261, 247, 239, 259, 254, 278, 258, 254, 278, 229, 252, 274, 247, 248, 228, 264, 268, 240, 269, 273, 279, 249, 258, 256, 258, 250, 266, 269, 246, 263, 267, 260, 258, 279, 250, 239, 272, 254, 247, 241, 251, 247, 276, 264, 269, 235, 277, 238, 262, 233, 245, 275, 265, 243, 233, 245, 234, 264, 246, 285, 263, 273, 290, 262, 232, 263, 250, 240, 240, 246, 244, 261, 268, 255, 255, 250, 274, 257, 225, 251, 255, 260, 271, 242, 248, 256, 254, 238, 254, 284, 261, 268, 254, 258, 219, 260, 262, 260, 255, 272, 261, 254, 245, 268, 241, 267, 234, 263] payload_rows_per_dst=[5482, 5464, 5498, 5490, 5438, 5470, 5420, 5424] dup_rows_per_dst=[2762, 2732, 2746, 2738, 2660, 2714, 2776, 2722] combine_rows_per_src=[5482, 5464, 5498, 5490, 5438, 5470, 5420, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=228960 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157138528 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048096 weight_movement_bytes=37748736 total_nvlink_bytes=351796832 weight_to_token_ratio=0.120200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229392 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156393488 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557584 weight_movement_bytes=18874368 total_nvlink_bytes=331431952 weight_to_token_ratio=0.060387 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228672 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156908864 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589056 weight_movement_bytes=37748736 total_nvlink_bytes=351337792 weight_to_token_ratio=0.120376 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229512 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688520 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147528 weight_movement_bytes=18874368 total_nvlink_bytes=332021896 weight_to_token_ratio=0.060273 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229408 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762144 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294880 weight_movement_bytes=18874368 total_nvlink_bytes=332169248 weight_to_token_ratio=0.060245 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38396 dup_rows=21676 dup_rows_remote=19018 dedup_ratio=0.330750 router_tokens_per_expert=[235, 266, 274, 243, 287, 255, 251, 242, 265, 256, 260, 238, 288, 244, 255, 261, 283, 244, 268, 250, 261, 253, 248, 247, 257, 238, 256, 279, 274, 252, 227, 235, 258, 255, 227, 263, 252, 267, 276, 235, 218, 266, 253, 240, 234, 284, 270, 237, 261, 284, 271, 239, 236, 262, 271, 258, 247, 248, 241, 249, 238, 266, 229, 261, 260, 261, 227, 273, 253, 258, 230, 248, 252, 268, 235, 273, 242, 265, 269, 280, 224, 278, 272, 261, 260, 257, 258, 265, 240, 246, 246, 280, 251, 225, 230, 248, 294, 262, 244, 235, 279, 276, 244, 254, 270, 262, 274, 253, 281, 267, 278, 255, 271, 267, 268, 245, 262, 273, 262, 236, 261, 244, 265, 273, 235, 259, 265, 231] payload_rows_per_dst=[5472, 5468, 5426, 5464, 5488, 5422, 5642, 5478] dup_rows_per_dst=[2768, 2676, 2644, 2658, 2700, 2660, 2814, 2756] combine_rows_per_src=[5472, 5468, 5426, 5464, 5488, 5422, 5642, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38026 dup_rows=22040 dup_rows_remote=19182 dedup_ratio=0.336304 router_tokens_per_expert=[257, 284, 270, 258, 234, 241, 258, 250, 245, 270, 278, 251, 228, 281, 254, 265, 270, 281, 232, 251, 258, 269, 249, 243, 234, 245, 235, 236, 271, 255, 255, 229, 239, 256, 276, 264, 260, 245, 265, 259, 263, 265, 260, 258, 238, 251, 262, 269, 244, 246, 234, 275, 246, 246, 273, 279, 241, 228, 252, 238, 264, 284, 265, 256, 250, 259, 260, 264, 248, 295, 282, 246, 220, 264, 248, 266, 234, 245, 264, 258, 269, 255, 254, 270, 270, 279, 255, 256, 256, 253, 242, 265, 278, 245, 244, 240, 267, 253, 245, 259, 246, 264, 297, 251, 264, 280, 241, 244, 237, 254, 240, 256, 263, 247, 251, 269, 256, 253, 234, 265, 273, 261, 252, 242, 244, 279, 252, 257] payload_rows_per_dst=[5288, 5374, 5638, 5418, 5272, 5580, 5456, 5470] dup_rows_per_dst=[2636, 2652, 2814, 2692, 2550, 3066, 2772, 2858] combine_rows_per_src=[5288, 5374, 5638, 5418, 5272, 5580, 5456, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38196 dup_rows=21870 dup_rows_remote=19184 dedup_ratio=0.333710 router_tokens_per_expert=[242, 256, 244, 233, 250, 271, 254, 230, 243, 249, 255, 280, 254, 242, 222, 252, 240, 260, 235, 247, 233, 266, 248, 247, 236, 252, 264, 264, 254, 263, 257, 241, 257, 251, 239, 250, 251, 257, 246, 240, 249, 239, 257, 253, 271, 288, 243, 250, 255, 262, 304, 267, 264, 264, 269, 265, 249, 258, 271, 244, 246, 263, 295, 275, 276, 248, 265, 269, 252, 263, 236, 273, 278, 243, 224, 279, 242, 263, 270, 265, 245, 264, 274, 252, 264, 251, 259, 233, 241, 250, 273, 246, 254, 234, 273, 282, 229, 243, 254, 256, 224, 269, 263, 237, 271, 251, 250, 277, 252, 269, 260, 244, 258, 275, 277, 283, 239, 266, 234, 265, 254, 280, 279, 249, 281, 235, 255, 272] payload_rows_per_dst=[5470, 5574, 5616, 5422, 5438, 5596, 5268, 5282] dup_rows_per_dst=[2686, 3000, 2920, 2626, 2692, 2978, 2446, 2522] combine_rows_per_src=[5470, 5574, 5616, 5422, 5438, 5596, 5268, 5282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38272 dup_rows=21860 dup_rows_remote=19092 dedup_ratio=0.333557 router_tokens_per_expert=[263, 255, 237, 248, 254, 254, 253, 245, 231, 300, 250, 272, 232, 266, 242, 244, 282, 254, 270, 268, 267, 254, 245, 266, 253, 246, 251, 262, 232, 252, 259, 257, 236, 266, 278, 253, 259, 260, 250, 279, 247, 258, 292, 270, 255, 245, 242, 285, 245, 238, 253, 263, 250, 244, 260, 231, 241, 268, 251, 264, 264, 252, 259, 258, 293, 275, 247, 272, 246, 279, 269, 268, 258, 243, 240, 257, 270, 251, 261, 280, 234, 255, 236, 237, 238, 248, 252, 254, 251, 242, 265, 254, 251, 230, 252, 239, 256, 272, 276, 263, 256, 271, 271, 237, 243, 270, 233, 251, 251, 257, 251, 228, 264, 250, 281, 246, 267, 263, 294, 265, 259, 255, 248, 274, 242, 247, 251, 249] payload_rows_per_dst=[5444, 5448, 5554, 5400, 5548, 5346, 5404, 5532] dup_rows_per_dst=[2648, 2788, 2796, 2682, 2870, 2530, 2768, 2778] combine_rows_per_src=[5444, 5448, 5554, 5400, 5548, 5346, 5404, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229656 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157499672 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769688 weight_movement_bytes=56623104 total_nvlink_bytes=371392792 weight_to_token_ratio=0.179887 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38266 dup_rows=21778 dup_rows_remote=19150 dedup_ratio=0.332306 router_tokens_per_expert=[279, 275, 243, 260, 229, 245, 260, 246, 252, 246, 240, 280, 278, 265, 255, 265, 239, 263, 257, 281, 276, 264, 236, 270, 254, 248, 239, 272, 230, 268, 289, 233, 269, 224, 266, 256, 271, 246, 263, 242, 284, 264, 265, 228, 240, 243, 266, 233, 282, 275, 282, 239, 258, 246, 255, 252, 269, 245, 259, 257, 247, 259, 255, 257, 261, 254, 229, 252, 241, 262, 266, 271, 243, 263, 251, 245, 262, 262, 253, 272, 276, 259, 238, 261, 253, 223, 243, 249, 248, 254, 261, 290, 242, 284, 284, 251, 253, 245, 279, 243, 251, 249, 260, 234, 279, 261, 269, 238, 268, 245, 254, 247, 246, 238, 243, 236, 254, 254, 247, 240, 262, 256, 284, 252, 265, 265, 245, 269] payload_rows_per_dst=[5464, 5504, 5492, 5510, 5490, 5440, 5456, 5402] dup_rows_per_dst=[2772, 2734, 2628, 2764, 2684, 2792, 2694, 2710] combine_rows_per_src=[5464, 5504, 5492, 5510, 5490, 5440, 5456, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43982 payload_rows_remote=38564 dup_rows=21554 dup_rows_remote=18962 dedup_ratio=0.328888 router_tokens_per_expert=[252, 260, 236, 273, 248, 250, 246, 274, 261, 272, 268, 252, 233, 257, 264, 235, 257, 240, 269, 253, 279, 268, 274, 239, 245, 214, 271, 269, 226, 233, 248, 220, 260, 258, 263, 276, 249, 244, 254, 243, 263, 252, 260, 259, 243, 255, 242, 272, 254, 260, 246, 269, 238, 272, 244, 236, 268, 268, 280, 269, 283, 256, 262, 288, 254, 266, 275, 260, 250, 287, 227, 257, 243, 227, 258, 297, 266, 245, 230, 255, 284, 259, 252, 261, 287, 279, 234, 242, 240, 265, 251, 240, 240, 261, 267, 259, 270, 252, 260, 266, 263, 268, 235, 242, 241, 283, 267, 272, 249, 254, 256, 219, 263, 245, 266, 232, 268, 260, 238, 276, 261, 280, 277, 248, 236, 271, 218, 242] payload_rows_per_dst=[5474, 5418, 5438, 5624, 5446, 5534, 5474, 5574] dup_rows_per_dst=[2688, 2592, 2748, 2762, 2748, 2708, 2720, 2588] combine_rows_per_src=[5474, 5418, 5438, 5624, 5446, 5534, 5474, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155754496 dispatch_meta_bytes=228832 combine_payload_bytes=155754496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155983328 ingress_bytes=174628864 link_bytes=174628864 token_bytes=311737824 weight_movement_bytes=18874368 total_nvlink_bytes=330612192 weight_to_token_ratio=0.060546 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673207852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229520 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680336 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131152 weight_movement_bytes=18874368 total_nvlink_bytes=332005520 weight_to_token_ratio=0.060276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38200 dup_rows=21808 dup_rows_remote=19066 dedup_ratio=0.332764 router_tokens_per_expert=[253, 269, 279, 254, 211, 219, 241, 264, 241, 253, 235, 248, 259, 289, 237, 227, 247, 254, 257, 280, 248, 269, 272, 248, 252, 246, 277, 235, 250, 258, 275, 262, 263, 261, 277, 264, 240, 252, 251, 247, 265, 288, 216, 280, 278, 299, 222, 251, 226, 293, 262, 261, 234, 247, 229, 285, 264, 248, 248, 283, 263, 252, 257, 249, 264, 259, 274, 228, 270, 271, 248, 264, 281, 264, 245, 263, 245, 233, 240, 286, 248, 286, 255, 272, 255, 241, 241, 273, 254, 252, 242, 276, 239, 273, 247, 251, 258, 248, 275, 258, 256, 257, 248, 262, 242, 226, 254, 276, 242, 225, 260, 258, 266, 272, 259, 240, 230, 287, 270, 250, 264, 255, 251, 246, 265, 263, 256, 245] payload_rows_per_dst=[5330, 5472, 5510, 5454, 5528, 5492, 5438, 5504] dup_rows_per_dst=[2628, 2788, 2798, 2748, 2742, 2718, 2652, 2734] combine_rows_per_src=[5330, 5472, 5510, 5454, 5528, 5492, 5438, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38352 dup_rows=21794 dup_rows_remote=19158 dedup_ratio=0.332550 router_tokens_per_expert=[292, 245, 266, 264, 248, 248, 276, 256, 265, 254, 253, 247, 279, 234, 254, 255, 237, 277, 258, 274, 261, 248, 240, 252, 268, 257, 270, 243, 254, 236, 282, 255, 290, 234, 244, 251, 236, 273, 245, 251, 268, 274, 258, 276, 273, 231, 253, 244, 246, 281, 254, 267, 245, 268, 233, 266, 241, 247, 256, 260, 240, 223, 266, 243, 261, 256, 266, 261, 288, 250, 268, 233, 269, 259, 256, 258, 258, 245, 263, 248, 244, 265, 249, 245, 247, 253, 269, 243, 219, 238, 278, 252, 243, 272, 253, 243, 241, 313, 247, 283, 241, 226, 257, 266, 251, 248, 244, 264, 264, 286, 274, 254, 244, 246, 251, 289, 256, 263, 237, 257, 247, 231, 251, 265, 256, 250, 247, 282] payload_rows_per_dst=[5546, 5482, 5388, 5482, 5514, 5390, 5548, 5392] dup_rows_per_dst=[2726, 2742, 2814, 2590, 2764, 2636, 2770, 2752] combine_rows_per_src=[5546, 5482, 5388, 5482, 5514, 5390, 5548, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229456 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991568 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753680 weight_movement_bytes=18874368 total_nvlink_bytes=332628048 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229664 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967200 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704736 weight_movement_bytes=18874368 total_nvlink_bytes=332579104 weight_to_token_ratio=0.060166 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157958144 dispatch_meta_bytes=230104 combine_payload_bytes=157958144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158188248 ingress_bytes=176832512 link_bytes=176832512 token_bytes=316146392 weight_movement_bytes=18874368 total_nvlink_bytes=335020760 weight_to_token_ratio=0.059701 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694322123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229064 combine_payload_bytes=156467200 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156696264 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163464 weight_movement_bytes=37748736 total_nvlink_bytes=350912200 weight_to_token_ratio=0.120540 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38398 dup_rows=21690 dup_rows_remote=19130 dedup_ratio=0.330963 router_tokens_per_expert=[228, 231, 260, 243, 261, 244, 265, 247, 286, 232, 230, 249, 255, 295, 260, 288, 272, 251, 245, 251, 275, 240, 249, 254, 240, 266, 262, 257, 230, 258, 258, 274, 259, 260, 249, 245, 245, 241, 255, 255, 235, 263, 253, 280, 247, 263, 274, 267, 245, 255, 252, 250, 271, 258, 245, 221, 263, 272, 251, 260, 224, 256, 231, 250, 254, 245, 264, 263, 291, 239, 235, 255, 241, 237, 251, 249, 284, 245, 280, 239, 250, 253, 265, 266, 272, 267, 267, 255, 270, 267, 247, 254, 248, 271, 251, 220, 243, 235, 255, 245, 245, 238, 277, 253, 277, 267, 276, 250, 279, 263, 277, 269, 239, 272, 264, 260, 273, 265, 258, 265, 245, 246, 252, 228, 267, 279, 283, 277] payload_rows_per_dst=[5498, 5398, 5410, 5448, 5476, 5540, 5512, 5564] dup_rows_per_dst=[2650, 2766, 2772, 2560, 2668, 2706, 2786, 2782] combine_rows_per_src=[5498, 5398, 5410, 5448, 5476, 5540, 5512, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=230040 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157319832 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314409624 weight_movement_bytes=18874368 total_nvlink_bytes=333283992 weight_to_token_ratio=0.060031 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43428 payload_rows_remote=37894 dup_rows=22108 dup_rows_remote=19220 dedup_ratio=0.337341 router_tokens_per_expert=[276, 250, 274, 269, 258, 247, 256, 252, 273, 251, 266, 253, 245, 260, 230, 271, 261, 241, 237, 263, 249, 268, 279, 250, 245, 242, 274, 263, 258, 226, 251, 253, 292, 235, 276, 262, 250, 264, 243, 258, 228, 231, 275, 259, 258, 258, 248, 259, 263, 244, 266, 257, 256, 248, 269, 248, 292, 236, 270, 254, 285, 218, 245, 265, 252, 258, 271, 239, 267, 269, 252, 291, 232, 257, 234, 260, 281, 245, 279, 275, 259, 234, 240, 251, 251, 257, 251, 251, 246, 274, 239, 254, 256, 253, 247, 251, 254, 289, 249, 261, 235, 237, 268, 252, 226, 248, 262, 234, 242, 246, 242, 266, 288, 259, 239, 271, 265, 258, 247, 290, 237, 254, 248, 248, 246, 266, 290, 272] payload_rows_per_dst=[5464, 5396, 5380, 5460, 5448, 5394, 5352, 5534] dup_rows_per_dst=[2798, 2724, 2746, 2772, 2688, 2822, 2670, 2888] combine_rows_per_src=[5464, 5396, 5380, 5460, 5448, 5394, 5352, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38240 dup_rows=21794 dup_rows_remote=18982 dedup_ratio=0.332550 router_tokens_per_expert=[232, 248, 250, 239, 276, 206, 250, 278, 242, 237, 227, 266, 254, 269, 242, 240, 230, 273, 278, 243, 257, 236, 244, 226, 279, 247, 251, 257, 246, 267, 282, 275, 232, 241, 271, 262, 265, 268, 264, 272, 286, 262, 254, 280, 288, 254, 243, 258, 269, 261, 253, 230, 264, 250, 267, 250, 238, 250, 247, 245, 273, 259, 247, 268, 256, 244, 228, 254, 258, 285, 259, 267, 248, 265, 283, 250, 261, 273, 257, 275, 246, 257, 253, 268, 258, 228, 241, 262, 280, 246, 269, 274, 256, 270, 240, 257, 274, 245, 275, 251, 244, 228, 268, 286, 243, 258, 237, 246, 244, 248, 246, 252, 254, 280, 258, 277, 252, 266, 253, 254, 249, 281, 274, 261, 255, 237, 238, 248] payload_rows_per_dst=[5502, 5346, 5460, 5680, 5520, 5520, 5396, 5318] dup_rows_per_dst=[2812, 2550, 2856, 2960, 2774, 2690, 2694, 2458] combine_rows_per_src=[5502, 5346, 5460, 5680, 5520, 5520, 5396, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=230112 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157508320 ingress_bytes=195026944 link_bytes=195026944 token_bytes=314786528 weight_movement_bytes=37748736 total_nvlink_bytes=352535264 weight_to_token_ratio=0.119919 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868652218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38098 dup_rows=21886 dup_rows_remote=19000 dedup_ratio=0.333954 router_tokens_per_expert=[252, 220, 257, 236, 240, 286, 261, 261, 224, 252, 274, 270, 258, 261, 253, 249, 240, 230, 254, 287, 259, 239, 247, 263, 251, 256, 243, 257, 262, 264, 248, 278, 247, 246, 258, 263, 259, 249, 233, 247, 265, 239, 245, 254, 260, 265, 254, 236, 246, 248, 256, 247, 261, 266, 251, 233, 271, 263, 268, 219, 259, 248, 297, 266, 236, 229, 240, 257, 258, 235, 264, 242, 270, 257, 248, 257, 247, 264, 252, 265, 289, 255, 276, 257, 244, 233, 253, 267, 254, 255, 277, 236, 266, 262, 252, 257, 263, 300, 283, 242, 303, 231, 273, 249, 262, 269, 269, 251, 269, 254, 241, 260, 253, 275, 284, 214, 267, 268, 260, 261, 265, 289, 263, 251, 237, 267, 243, 247] payload_rows_per_dst=[5374, 5478, 5388, 5476, 5424, 5512, 5552, 5446] dup_rows_per_dst=[2734, 2678, 2652, 2722, 2618, 2754, 2886, 2842] combine_rows_per_src=[5374, 5478, 5388, 5476, 5424, 5512, 5552, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38244 dup_rows=21780 dup_rows_remote=19026 dedup_ratio=0.332336 router_tokens_per_expert=[268, 257, 249, 243, 276, 283, 205, 266, 258, 246, 246, 249, 261, 230, 253, 246, 257, 248, 259, 245, 249, 248, 245, 244, 223, 258, 257, 256, 248, 270, 267, 255, 261, 278, 255, 270, 250, 239, 275, 241, 259, 254, 258, 257, 258, 273, 269, 236, 270, 240, 262, 239, 255, 248, 228, 268, 269, 251, 255, 249, 245, 277, 257, 274, 250, 254, 292, 271, 250, 240, 261, 246, 259, 247, 276, 237, 270, 270, 234, 246, 274, 257, 271, 258, 265, 245, 251, 257, 257, 247, 255, 259, 270, 222, 266, 284, 270, 281, 263, 227, 244, 258, 237, 304, 268, 262, 269, 283, 263, 253, 261, 265, 245, 222, 268, 251, 261, 237, 272, 254, 232, 253, 251, 279, 257, 260, 235, 257] payload_rows_per_dst=[5320, 5408, 5512, 5468, 5476, 5520, 5614, 5438] dup_rows_per_dst=[2752, 2650, 2754, 2706, 2730, 2756, 2802, 2630] combine_rows_per_src=[5320, 5408, 5512, 5468, 5476, 5520, 5614, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38328 dup_rows=21704 dup_rows_remote=19044 dedup_ratio=0.331177 router_tokens_per_expert=[264, 241, 261, 256, 245, 255, 255, 254, 249, 250, 259, 249, 273, 251, 262, 287, 237, 236, 241, 239, 258, 272, 280, 251, 219, 257, 264, 282, 248, 271, 263, 264, 232, 254, 271, 260, 241, 277, 257, 261, 258, 245, 255, 248, 263, 250, 273, 245, 273, 262, 256, 252, 250, 251, 246, 254, 262, 258, 225, 258, 258, 277, 263, 251, 251, 239, 239, 247, 272, 275, 246, 263, 262, 257, 237, 252, 257, 242, 258, 259, 256, 246, 244, 248, 269, 275, 276, 249, 264, 230, 234, 239, 270, 239, 297, 234, 247, 218, 247, 268, 292, 232, 289, 259, 285, 237, 269, 278, 257, 256, 266, 252, 292, 247, 238, 224, 279, 250, 261, 236, 248, 274, 261, 257, 284, 249, 267, 244] payload_rows_per_dst=[5414, 5504, 5498, 5504, 5448, 5474, 5516, 5474] dup_rows_per_dst=[2808, 2660, 2682, 2688, 2664, 2666, 2788, 2748] combine_rows_per_src=[5414, 5504, 5498, 5504, 5448, 5474, 5516, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38228 dup_rows=21856 dup_rows_remote=19216 dedup_ratio=0.333496 router_tokens_per_expert=[245, 267, 242, 279, 270, 268, 254, 260, 264, 251, 246, 242, 238, 250, 274, 255, 242, 265, 244, 257, 271, 242, 255, 300, 243, 268, 258, 258, 274, 252, 265, 243, 272, 277, 250, 268, 245, 256, 263, 270, 236, 267, 223, 230, 277, 265, 260, 270, 237, 252, 235, 281, 260, 267, 254, 255, 282, 252, 249, 277, 242, 232, 263, 234, 261, 247, 244, 232, 226, 263, 271, 281, 248, 260, 255, 244, 266, 249, 254, 245, 254, 240, 255, 243, 242, 261, 242, 258, 246, 252, 279, 233, 249, 266, 273, 242, 272, 257, 256, 255, 268, 275, 274, 254, 243, 237, 290, 251, 249, 281, 263, 275, 249, 279, 263, 230, 265, 228, 235, 264, 247, 279, 261, 219, 277, 255, 239, 254] payload_rows_per_dst=[5462, 5476, 5472, 5460, 5452, 5420, 5542, 5396] dup_rows_per_dst=[2748, 2798, 2786, 2684, 2640, 2650, 2858, 2692] combine_rows_per_src=[5462, 5476, 5472, 5460, 5452, 5420, 5542, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155213824 dispatch_meta_bytes=228456 combine_payload_bytes=155213824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155442280 ingress_bytes=192962560 link_bytes=192962560 token_bytes=310656104 weight_movement_bytes=37748736 total_nvlink_bytes=348404840 weight_to_token_ratio=0.121513 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.848872306e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228888 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156859928 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313490968 weight_movement_bytes=18874368 total_nvlink_bytes=332365336 weight_to_token_ratio=0.060207 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228392 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156277800 ingress_bytes=193798144 link_bytes=193798144 token_bytes=312327208 weight_movement_bytes=37748736 total_nvlink_bytes=350075944 weight_to_token_ratio=0.120863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856878461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229080 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876504 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313523928 weight_movement_bytes=37748736 total_nvlink_bytes=351272664 weight_to_token_ratio=0.120401 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38404 dup_rows=21612 dup_rows_remote=18956 dedup_ratio=0.329773 router_tokens_per_expert=[269, 276, 248, 263, 247, 266, 261, 286, 273, 275, 252, 234, 266, 257, 249, 270, 241, 242, 278, 246, 249, 275, 247, 253, 264, 253, 242, 282, 246, 248, 267, 247, 246, 248, 254, 268, 230, 269, 289, 237, 263, 240, 277, 256, 248, 266, 244, 239, 253, 241, 246, 255, 238, 228, 265, 237, 242, 264, 270, 269, 268, 261, 248, 245, 249, 275, 257, 279, 257, 271, 242, 273, 222, 265, 265, 221, 275, 279, 249, 235, 279, 248, 256, 264, 223, 229, 265, 253, 234, 275, 284, 258, 256, 251, 263, 250, 253, 260, 266, 258, 253, 259, 240, 268, 296, 239, 232, 257, 236, 259, 256, 248, 258, 260, 248, 262, 260, 265, 244, 244, 266, 244, 266, 256, 279, 247, 249, 262] payload_rows_per_dst=[5526, 5520, 5432, 5434, 5576, 5520, 5442, 5474] dup_rows_per_dst=[2858, 2640, 2716, 2626, 2652, 2656, 2718, 2746] combine_rows_per_src=[5526, 5520, 5432, 5434, 5576, 5520, 5442, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229488 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220976 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212464 weight_movement_bytes=18874368 total_nvlink_bytes=333086832 weight_to_token_ratio=0.060069 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229776 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156811664 ingress_bytes=194330624 link_bytes=194330624 token_bytes=313393552 weight_movement_bytes=37748736 total_nvlink_bytes=351142288 weight_to_token_ratio=0.120452 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861980423e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38126 dup_rows=21832 dup_rows_remote=19054 dedup_ratio=0.333130 router_tokens_per_expert=[245, 252, 227, 262, 234, 279, 260, 264, 243, 252, 238, 255, 250, 258, 248, 274, 267, 257, 283, 244, 270, 274, 229, 260, 245, 242, 240, 269, 226, 270, 240, 274, 247, 261, 243, 255, 258, 248, 244, 264, 255, 258, 288, 234, 269, 280, 250, 236, 250, 261, 245, 264, 273, 269, 264, 293, 280, 240, 264, 280, 249, 238, 266, 242, 244, 257, 256, 285, 291, 252, 269, 281, 277, 257, 242, 246, 237, 305, 247, 257, 248, 248, 235, 254, 227, 241, 273, 260, 274, 248, 284, 258, 265, 251, 246, 283, 275, 241, 276, 248, 251, 253, 251, 273, 260, 276, 218, 276, 238, 286, 245, 254, 236, 254, 242, 238, 236, 273, 240, 240, 242, 269, 215, 251, 246, 257, 272, 239] payload_rows_per_dst=[5354, 5426, 5396, 5578, 5554, 5480, 5526, 5390] dup_rows_per_dst=[2728, 2754, 2784, 2778, 2852, 2710, 2716, 2510] combine_rows_per_src=[5354, 5426, 5396, 5578, 5554, 5480, 5526, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229440 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532224 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835008 weight_movement_bytes=18874368 total_nvlink_bytes=333709376 weight_to_token_ratio=0.059950 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228720 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156392816 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312556912 weight_movement_bytes=37748736 total_nvlink_bytes=350305648 weight_to_token_ratio=0.120774 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38158 dup_rows=21742 dup_rows_remote=18748 dedup_ratio=0.331757 router_tokens_per_expert=[239, 262, 232, 244, 274, 245, 257, 258, 233, 286, 260, 253, 268, 256, 269, 264, 248, 250, 256, 256, 252, 245, 270, 273, 238, 249, 242, 238, 257, 267, 259, 259, 234, 252, 254, 256, 283, 265, 243, 260, 256, 311, 264, 258, 254, 248, 251, 272, 259, 236, 224, 249, 274, 267, 257, 253, 244, 255, 257, 258, 271, 281, 276, 241, 249, 270, 240, 253, 242, 267, 238, 264, 239, 227, 272, 247, 257, 264, 254, 240, 269, 256, 291, 238, 258, 277, 267, 277, 254, 243, 252, 270, 273, 252, 243, 256, 256, 284, 282, 256, 263, 265, 242, 280, 247, 257, 247, 248, 239, 240, 257, 242, 260, 244, 229, 269, 242, 257, 264, 237, 249, 248, 257, 269, 251, 276, 245, 245] payload_rows_per_dst=[5510, 5404, 5480, 5480, 5462, 5302, 5520, 5636] dup_rows_per_dst=[2690, 2714, 2842, 2724, 2584, 2504, 2690, 2994] combine_rows_per_src=[5510, 5404, 5480, 5480, 5462, 5302, 5520, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38306 dup_rows=21680 dup_rows_remote=18888 dedup_ratio=0.330811 router_tokens_per_expert=[238, 245, 271, 269, 280, 257, 261, 261, 284, 244, 270, 264, 250, 265, 252, 259, 260, 257, 268, 255, 265, 261, 257, 241, 253, 274, 265, 272, 258, 245, 234, 238, 267, 252, 266, 242, 240, 236, 234, 251, 222, 238, 233, 260, 251, 252, 249, 254, 284, 260, 276, 236, 259, 230, 276, 261, 230, 275, 272, 250, 246, 256, 260, 251, 262, 278, 240, 250, 256, 259, 245, 264, 266, 242, 239, 261, 224, 274, 271, 259, 243, 269, 268, 253, 247, 253, 296, 217, 269, 254, 271, 240, 249, 252, 253, 259, 255, 227, 264, 258, 254, 252, 289, 281, 282, 249, 226, 272, 250, 255, 277, 280, 253, 250, 247, 221, 251, 265, 268, 249, 252, 262, 232, 287, 247, 270, 244, 274] payload_rows_per_dst=[5558, 5548, 5354, 5512, 5526, 5412, 5550, 5396] dup_rows_per_dst=[2782, 2658, 2540, 2732, 2654, 2774, 2792, 2748] combine_rows_per_src=[5558, 5548, 5354, 5512, 5526, 5412, 5550, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38200 dup_rows=21846 dup_rows_remote=19096 dedup_ratio=0.333344 router_tokens_per_expert=[246, 285, 262, 276, 262, 265, 245, 261, 270, 239, 228, 253, 253, 274, 236, 265, 268, 259, 268, 276, 243, 256, 234, 269, 258, 267, 294, 246, 269, 246, 266, 256, 248, 240, 250, 238, 269, 240, 267, 267, 246, 248, 271, 244, 231, 241, 250, 244, 247, 235, 275, 253, 215, 257, 279, 272, 231, 268, 276, 255, 263, 267, 225, 240, 258, 285, 264, 264, 245, 225, 242, 267, 265, 247, 249, 235, 294, 280, 245, 250, 243, 269, 260, 275, 250, 282, 282, 264, 238, 264, 241, 255, 256, 270, 255, 236, 271, 250, 270, 246, 276, 263, 239, 252, 261, 264, 277, 249, 248, 235, 239, 259, 266, 261, 253, 244, 238, 290, 252, 242, 256, 251, 254, 250, 252, 262, 246, 250] payload_rows_per_dst=[5490, 5554, 5330, 5434, 5564, 5630, 5332, 5356] dup_rows_per_dst=[2750, 2796, 2658, 2682, 2666, 3060, 2570, 2664] combine_rows_per_src=[5490, 5554, 5330, 5434, 5564, 5630, 5332, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38338 dup_rows=21736 dup_rows_remote=19018 dedup_ratio=0.331665 router_tokens_per_expert=[278, 270, 233, 267, 250, 258, 254, 276, 255, 249, 265, 248, 257, 231, 232, 278, 260, 259, 231, 231, 238, 238, 279, 251, 245, 262, 249, 268, 251, 254, 246, 271, 251, 292, 254, 242, 266, 240, 254, 236, 263, 281, 274, 253, 284, 228, 208, 264, 265, 240, 251, 246, 265, 252, 251, 232, 256, 258, 256, 213, 269, 250, 275, 253, 240, 276, 254, 295, 244, 248, 255, 293, 281, 246, 275, 268, 281, 254, 246, 261, 270, 237, 282, 276, 258, 257, 253, 241, 264, 273, 247, 286, 269, 258, 248, 278, 266, 248, 238, 272, 253, 261, 277, 242, 247, 240, 248, 258, 274, 249, 247, 269, 267, 274, 263, 270, 283, 242, 231, 239, 250, 253, 226, 251, 253, 251, 243, 213] payload_rows_per_dst=[5452, 5438, 5462, 5380, 5602, 5606, 5432, 5428] dup_rows_per_dst=[2750, 2628, 2718, 2684, 2832, 2788, 2746, 2590] combine_rows_per_src=[5452, 5438, 5462, 5380, 5602, 5606, 5432, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38330 dup_rows=21836 dup_rows_remote=19178 dedup_ratio=0.333191 router_tokens_per_expert=[257, 258, 243, 263, 268, 223, 237, 279, 263, 264, 256, 249, 236, 237, 241, 237, 233, 262, 234, 237, 252, 250, 246, 257, 242, 255, 246, 270, 244, 265, 282, 239, 239, 257, 244, 278, 260, 271, 257, 266, 276, 261, 240, 255, 251, 270, 260, 277, 276, 275, 263, 266, 267, 269, 280, 303, 256, 273, 275, 225, 236, 247, 251, 282, 264, 263, 253, 246, 234, 246, 278, 253, 250, 271, 231, 276, 286, 259, 248, 262, 288, 280, 259, 235, 256, 252, 257, 255, 250, 260, 271, 254, 268, 247, 266, 232, 277, 260, 270, 269, 250, 260, 234, 240, 251, 234, 255, 244, 243, 250, 240, 251, 258, 255, 278, 253, 233, 253, 254, 260, 262, 218, 262, 256, 242, 260, 249, 266] payload_rows_per_dst=[5428, 5370, 5542, 5570, 5480, 5498, 5416, 5396] dup_rows_per_dst=[2788, 2658, 2782, 2724, 2760, 2762, 2640, 2722] combine_rows_per_src=[5428, 5370, 5542, 5570, 5480, 5498, 5416, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38398 dup_rows=21630 dup_rows_remote=18956 dedup_ratio=0.330048 router_tokens_per_expert=[247, 263, 257, 251, 258, 259, 283, 238, 253, 236, 241, 279, 247, 228, 269, 245, 265, 260, 260, 257, 283, 264, 237, 216, 261, 256, 243, 245, 254, 268, 256, 241, 269, 263, 226, 250, 268, 278, 253, 283, 265, 260, 270, 241, 236, 245, 260, 244, 256, 278, 253, 246, 254, 273, 253, 256, 257, 271, 252, 261, 258, 251, 271, 243, 252, 257, 240, 276, 251, 232, 250, 264, 270, 270, 229, 254, 240, 259, 280, 267, 282, 279, 279, 237, 253, 260, 257, 257, 233, 262, 253, 235, 301, 258, 262, 269, 271, 274, 226, 236, 248, 256, 236, 275, 251, 252, 253, 275, 232, 226, 263, 262, 255, 259, 253, 247, 256, 256, 263, 264, 258, 239, 261, 267, 255, 257, 251, 259] payload_rows_per_dst=[5484, 5430, 5444, 5554, 5508, 5586, 5434, 5466] dup_rows_per_dst=[2624, 2702, 2778, 2712, 2674, 2768, 2638, 2734] combine_rows_per_src=[5484, 5430, 5444, 5554, 5508, 5586, 5434, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=227624 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156522792 ingress_bytes=175169536 link_bytes=175169536 token_bytes=312817960 weight_movement_bytes=18874368 total_nvlink_bytes=331692328 weight_to_token_ratio=0.060337 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678388305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38220 dup_rows=21864 dup_rows_remote=19168 dedup_ratio=0.333618 router_tokens_per_expert=[232, 250, 233, 239, 275, 215, 219, 257, 281, 246, 279, 274, 231, 233, 249, 270, 268, 249, 261, 278, 246, 256, 260, 273, 274, 259, 291, 250, 274, 258, 269, 260, 254, 246, 248, 225, 257, 252, 279, 270, 223, 262, 263, 266, 251, 249, 277, 261, 221, 274, 253, 254, 245, 230, 244, 260, 249, 262, 262, 247, 251, 261, 247, 262, 266, 266, 257, 254, 250, 265, 265, 255, 270, 276, 246, 244, 243, 267, 265, 272, 252, 247, 246, 253, 253, 240, 254, 259, 269, 271, 250, 260, 259, 257, 239, 265, 258, 263, 251, 250, 263, 271, 292, 273, 263, 256, 248, 257, 255, 241, 270, 224, 252, 259, 261, 277, 249, 250, 276, 227, 264, 268, 264, 216, 244, 255, 269, 253] payload_rows_per_dst=[5440, 5636, 5332, 5406, 5460, 5452, 5474, 5472] dup_rows_per_dst=[2526, 2816, 2834, 2638, 2862, 2696, 2796, 2696] combine_rows_per_src=[5440, 5636, 5332, 5406, 5460, 5452, 5474, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228776 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130152 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031528 weight_movement_bytes=37748736 total_nvlink_bytes=351780264 weight_to_token_ratio=0.120207 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229184 combine_payload_bytes=156467200 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156696384 ingress_bytes=213090304 link_bytes=213090304 token_bytes=313163584 weight_movement_bytes=56623104 total_nvlink_bytes=369786688 weight_to_token_ratio=0.180810 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041726446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229424 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261872 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294320 weight_movement_bytes=18874368 total_nvlink_bytes=333168688 weight_to_token_ratio=0.060053 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=230032 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157229712 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314229392 weight_movement_bytes=18874368 total_nvlink_bytes=333103760 weight_to_token_ratio=0.060066 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38062 dup_rows=21874 dup_rows_remote=19108 dedup_ratio=0.333771 router_tokens_per_expert=[252, 240, 250, 233, 245, 259, 259, 258, 252, 264, 278, 255, 293, 252, 245, 250, 250, 254, 270, 287, 241, 283, 267, 239, 258, 259, 215, 252, 255, 279, 262, 234, 263, 263, 268, 256, 243, 244, 278, 265, 244, 248, 248, 240, 272, 261, 265, 259, 259, 230, 277, 255, 249, 257, 285, 262, 256, 294, 257, 282, 264, 243, 268, 245, 251, 253, 245, 257, 234, 247, 243, 257, 263, 264, 242, 282, 245, 235, 265, 260, 240, 275, 271, 275, 246, 254, 257, 262, 250, 233, 254, 252, 247, 270, 262, 265, 237, 274, 240, 246, 245, 264, 238, 260, 243, 239, 249, 277, 276, 273, 250, 279, 235, 247, 241, 263, 293, 243, 253, 235, 261, 256, 279, 257, 221, 265, 231, 252] payload_rows_per_dst=[5458, 5456, 5478, 5600, 5390, 5500, 5434, 5346] dup_rows_per_dst=[2712, 2754, 2756, 2766, 2696, 2726, 2746, 2718] combine_rows_per_src=[5458, 5456, 5478, 5600, 5390, 5500, 5434, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229416 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157507624 ingress_bytes=195026944 link_bytes=195026944 token_bytes=314785832 weight_movement_bytes=37748736 total_nvlink_bytes=352534568 weight_to_token_ratio=0.119919 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868652218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229552 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156778672 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313327792 weight_movement_bytes=56623104 total_nvlink_bytes=369950896 weight_to_token_ratio=0.180715 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228680 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156130632 ingress_bytes=193650688 link_bytes=193650688 token_bytes=312032584 weight_movement_bytes=37748736 total_nvlink_bytes=349781320 weight_to_token_ratio=0.120977 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855465610e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38046 dup_rows=21908 dup_rows_remote=18840 dedup_ratio=0.334290 router_tokens_per_expert=[266, 267, 267, 243, 258, 274, 277, 280, 284, 259, 241, 264, 259, 269, 252, 262, 254, 265, 284, 269, 244, 255, 284, 272, 223, 278, 251, 230, 252, 271, 265, 279, 233, 248, 256, 242, 260, 235, 265, 261, 245, 244, 281, 239, 235, 267, 252, 249, 224, 255, 264, 273, 258, 274, 258, 256, 275, 249, 257, 259, 251, 266, 258, 235, 276, 270, 258, 246, 271, 267, 253, 258, 235, 251, 246, 248, 270, 243, 249, 275, 259, 227, 260, 259, 249, 243, 219, 228, 254, 259, 250, 246, 269, 245, 269, 269, 264, 251, 258, 273, 256, 269, 224, 268, 263, 266, 240, 251, 247, 244, 258, 227, 291, 256, 255, 246, 256, 253, 237, 262, 271, 226, 250, 260, 231, 239, 274, 259] payload_rows_per_dst=[5326, 5306, 5414, 5558, 5478, 5572, 5392, 5582] dup_rows_per_dst=[2600, 2494, 2568, 2708, 2754, 2990, 2726, 3068] combine_rows_per_src=[5326, 5306, 5414, 5558, 5478, 5572, 5392, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38322 dup_rows=21842 dup_rows_remote=19276 dedup_ratio=0.333282 router_tokens_per_expert=[261, 246, 224, 254, 268, 246, 224, 261, 252, 256, 263, 253, 260, 236, 246, 252, 276, 246, 237, 259, 281, 263, 267, 267, 249, 278, 271, 267, 272, 252, 271, 258, 264, 259, 253, 262, 265, 245, 266, 247, 235, 251, 256, 241, 265, 262, 246, 275, 263, 271, 280, 257, 276, 253, 250, 268, 250, 265, 255, 252, 286, 258, 243, 295, 261, 233, 237, 271, 252, 263, 268, 242, 280, 259, 271, 275, 277, 244, 251, 252, 255, 299, 261, 239, 243, 230, 248, 250, 250, 298, 262, 275, 236, 239, 269, 245, 276, 257, 227, 236, 250, 244, 250, 240, 232, 242, 289, 246, 222, 266, 235, 257, 249, 258, 223, 266, 237, 263, 236, 259, 288, 236, 238, 256, 236, 263, 269, 257] payload_rows_per_dst=[5340, 5610, 5446, 5514, 5512, 5502, 5372, 5398] dup_rows_per_dst=[2664, 2818, 2738, 2930, 2760, 2696, 2566, 2670] combine_rows_per_src=[5340, 5610, 5446, 5514, 5512, 5502, 5372, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38194 dup_rows=21838 dup_rows_remote=19042 dedup_ratio=0.333221 router_tokens_per_expert=[248, 259, 260, 277, 247, 273, 248, 254, 271, 234, 264, 240, 247, 269, 250, 257, 248, 284, 248, 255, 257, 256, 239, 274, 260, 255, 263, 278, 279, 255, 245, 250, 231, 258, 266, 268, 259, 253, 235, 262, 233, 271, 261, 255, 280, 272, 259, 287, 242, 232, 260, 248, 284, 252, 259, 257, 264, 260, 247, 267, 270, 259, 264, 252, 236, 256, 264, 260, 253, 248, 236, 249, 250, 258, 270, 253, 224, 229, 271, 290, 247, 283, 246, 266, 260, 226, 277, 245, 262, 249, 251, 254, 242, 229, 260, 275, 286, 253, 274, 257, 236, 237, 242, 242, 247, 237, 296, 259, 254, 269, 252, 270, 255, 253, 259, 255, 267, 263, 229, 252, 256, 257, 231, 251, 261, 245, 263, 230] payload_rows_per_dst=[5476, 5482, 5504, 5422, 5446, 5434, 5432, 5502] dup_rows_per_dst=[2720, 2810, 2796, 2812, 2648, 2710, 2790, 2552] combine_rows_per_src=[5476, 5482, 5504, 5422, 5446, 5434, 5432, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38028 dup_rows=21996 dup_rows_remote=19104 dedup_ratio=0.335632 router_tokens_per_expert=[256, 275, 256, 244, 277, 271, 280, 268, 260, 270, 257, 244, 271, 250, 270, 248, 256, 259, 291, 251, 256, 236, 270, 248, 232, 249, 241, 213, 231, 251, 276, 263, 265, 266, 270, 262, 254, 237, 241, 267, 263, 287, 262, 246, 268, 260, 256, 248, 237, 223, 252, 245, 259, 250, 272, 273, 271, 238, 257, 262, 252, 263, 256, 256, 270, 272, 244, 264, 252, 266, 249, 250, 277, 263, 267, 268, 251, 285, 274, 265, 272, 259, 251, 225, 257, 253, 244, 248, 263, 240, 250, 249, 257, 234, 232, 222, 222, 260, 244, 249, 248, 259, 244, 300, 279, 265, 218, 253, 262, 286, 219, 262, 241, 249, 255, 254, 247, 259, 281, 254, 263, 241, 263, 259, 261, 257, 258, 245] payload_rows_per_dst=[5512, 5560, 5508, 5432, 5542, 5300, 5236, 5450] dup_rows_per_dst=[2892, 2868, 2796, 2694, 2888, 2612, 2522, 2724] combine_rows_per_src=[5512, 5560, 5508, 5432, 5542, 5300, 5236, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38206 dup_rows=21910 dup_rows_remote=19222 dedup_ratio=0.334320 router_tokens_per_expert=[257, 277, 279, 261, 261, 244, 254, 248, 265, 242, 251, 240, 285, 233, 271, 230, 252, 260, 247, 261, 237, 260, 269, 267, 251, 257, 249, 281, 226, 245, 228, 264, 267, 248, 255, 280, 274, 256, 273, 251, 293, 257, 293, 269, 269, 250, 246, 281, 265, 256, 276, 240, 252, 237, 245, 260, 274, 280, 261, 249, 228, 230, 263, 251, 258, 235, 259, 254, 246, 255, 245, 259, 275, 264, 260, 244, 263, 247, 265, 266, 286, 258, 214, 259, 257, 245, 259, 246, 259, 256, 261, 248, 278, 244, 239, 255, 233, 253, 235, 239, 294, 275, 265, 244, 222, 271, 226, 251, 295, 251, 273, 246, 261, 262, 242, 273, 238, 240, 257, 236, 262, 284, 258, 266, 219, 275, 240, 242] payload_rows_per_dst=[5440, 5420, 5596, 5456, 5438, 5422, 5458, 5396] dup_rows_per_dst=[2756, 2688, 2928, 2678, 2752, 2706, 2688, 2714] combine_rows_per_src=[5440, 5420, 5596, 5456, 5438, 5422, 5458, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38170 dup_rows=21896 dup_rows_remote=19102 dedup_ratio=0.334106 router_tokens_per_expert=[276, 276, 279, 237, 290, 245, 270, 238, 252, 240, 234, 268, 240, 266, 249, 258, 257, 246, 255, 272, 285, 291, 256, 249, 265, 256, 271, 260, 257, 245, 272, 283, 245, 258, 234, 257, 255, 269, 267, 252, 275, 268, 260, 248, 241, 249, 239, 242, 243, 269, 263, 252, 249, 254, 268, 251, 250, 247, 253, 258, 265, 279, 230, 236, 244, 250, 243, 248, 285, 263, 271, 296, 232, 274, 252, 266, 257, 224, 261, 266, 254, 238, 245, 246, 278, 242, 220, 256, 282, 244, 246, 246, 257, 231, 247, 255, 239, 275, 253, 273, 249, 245, 261, 263, 275, 238, 239, 249, 262, 268, 263, 268, 253, 242, 221, 221, 285, 275, 265, 257, 253, 238, 246, 290, 254, 241, 268, 256] payload_rows_per_dst=[5468, 5590, 5440, 5436, 5470, 5400, 5466, 5370] dup_rows_per_dst=[2768, 2850, 2678, 2698, 2794, 2574, 2774, 2760] combine_rows_per_src=[5468, 5590, 5440, 5436, 5470, 5400, 5466, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=227544 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156063960 ingress_bytes=174710784 link_bytes=174710784 token_bytes=311900376 weight_movement_bytes=18874368 total_nvlink_bytes=330774744 weight_to_token_ratio=0.060514 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673992769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38186 dup_rows=21880 dup_rows_remote=19084 dedup_ratio=0.333862 router_tokens_per_expert=[259, 271, 260, 250, 263, 238, 248, 241, 235, 239, 277, 236, 267, 244, 257, 249, 260, 260, 270, 257, 260, 247, 256, 237, 228, 251, 254, 258, 258, 237, 269, 274, 255, 292, 259, 266, 270, 258, 261, 250, 231, 264, 258, 249, 239, 294, 244, 231, 262, 262, 241, 277, 257, 262, 252, 269, 234, 247, 255, 257, 263, 263, 258, 256, 252, 272, 253, 220, 285, 219, 263, 287, 268, 268, 270, 251, 269, 238, 240, 269, 269, 218, 258, 266, 291, 252, 254, 294, 259, 231, 260, 271, 211, 261, 259, 279, 260, 283, 262, 244, 260, 260, 263, 270, 262, 251, 214, 246, 240, 231, 269, 260, 231, 247, 261, 243, 272, 254, 259, 278, 277, 266, 265, 262, 237, 223, 266, 249] payload_rows_per_dst=[5408, 5520, 5516, 5432, 5388, 5470, 5474, 5448] dup_rows_per_dst=[2660, 2632, 2726, 2798, 2860, 2796, 2676, 2732] combine_rows_per_src=[5408, 5520, 5516, 5432, 5388, 5470, 5474, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=230392 combine_payload_bytes=156966912 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157197304 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314164216 weight_movement_bytes=56623104 total_nvlink_bytes=370787320 weight_to_token_ratio=0.180234 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228944 combine_payload_bytes=156442624 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156671568 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114192 weight_movement_bytes=37748736 total_nvlink_bytes=350862928 weight_to_token_ratio=0.120559 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155762688 dispatch_meta_bytes=228528 combine_payload_bytes=155762688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155991216 ingress_bytes=212385792 link_bytes=212385792 token_bytes=311753904 weight_movement_bytes=56623104 total_nvlink_bytes=368377008 weight_to_token_ratio=0.181628 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034976159e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43486 payload_rows_remote=38088 dup_rows=22050 dup_rows_remote=19330 dedup_ratio=0.336456 router_tokens_per_expert=[253, 264, 252, 273, 254, 258, 221, 264, 249, 264, 236, 261, 233, 234, 249, 266, 263, 261, 254, 273, 277, 264, 261, 287, 277, 264, 262, 276, 266, 273, 275, 274, 252, 266, 256, 247, 272, 253, 280, 266, 281, 267, 248, 252, 264, 259, 282, 243, 241, 247, 261, 260, 292, 243, 250, 283, 229, 249, 238, 274, 258, 235, 257, 242, 253, 223, 260, 260, 248, 267, 240, 270, 265, 253, 254, 255, 268, 244, 278, 258, 277, 246, 252, 240, 234, 246, 256, 273, 247, 250, 257, 261, 236, 239, 268, 249, 240, 268, 241, 249, 252, 232, 230, 236, 224, 255, 265, 269, 259, 234, 257, 222, 274, 247, 244, 256, 263, 239, 274, 259, 248, 254, 245, 272, 241, 262, 278, 267] payload_rows_per_dst=[5348, 5652, 5456, 5398, 5434, 5404, 5304, 5490] dup_rows_per_dst=[2714, 2962, 2920, 2720, 2758, 2658, 2562, 2756] combine_rows_per_src=[5348, 5652, 5456, 5398, 5434, 5404, 5304, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229712 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156721488 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313213264 weight_movement_bytes=37748736 total_nvlink_bytes=350962000 weight_to_token_ratio=0.120521 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229088 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573408 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917728 weight_movement_bytes=18874368 total_nvlink_bytes=331792096 weight_to_token_ratio=0.060317 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229080 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156638936 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048792 weight_movement_bytes=18874368 total_nvlink_bytes=331923160 weight_to_token_ratio=0.060292 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=229672 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156238120 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312246568 weight_movement_bytes=18874368 total_nvlink_bytes=331120936 weight_to_token_ratio=0.060447 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38018 dup_rows=21892 dup_rows_remote=18964 dedup_ratio=0.334045 router_tokens_per_expert=[256, 256, 262, 243, 289, 235, 232, 257, 274, 238, 265, 251, 232, 258, 276, 269, 254, 240, 263, 268, 254, 225, 261, 260, 287, 273, 251, 251, 278, 249, 257, 274, 245, 238, 253, 253, 246, 285, 255, 230, 214, 268, 258, 245, 269, 249, 256, 251, 237, 279, 251, 278, 253, 236, 268, 275, 268, 240, 223, 265, 256, 232, 264, 262, 262, 296, 234, 242, 266, 277, 269, 263, 249, 262, 253, 245, 231, 273, 248, 247, 261, 268, 241, 268, 282, 265, 247, 252, 270, 240, 254, 254, 251, 249, 264, 269, 250, 249, 274, 246, 272, 248, 275, 255, 259, 250, 245, 240, 262, 281, 277, 252, 261, 234, 257, 242, 246, 262, 255, 272, 228, 216, 252, 276, 245, 267, 278, 250] payload_rows_per_dst=[5418, 5382, 5178, 5506, 5476, 5518, 5540, 5626] dup_rows_per_dst=[2768, 2796, 2380, 2668, 2816, 2752, 2784, 2928] combine_rows_per_src=[5418, 5382, 5178, 5506, 5476, 5518, 5540, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38348 dup_rows=21644 dup_rows_remote=18882 dedup_ratio=0.330261 router_tokens_per_expert=[242, 263, 254, 243, 255, 264, 246, 254, 232, 269, 271, 209, 265, 269, 267, 263, 288, 245, 244, 237, 265, 251, 246, 243, 241, 249, 282, 279, 282, 243, 255, 254, 233, 232, 277, 245, 242, 267, 233, 235, 266, 262, 275, 264, 250, 235, 275, 249, 279, 255, 242, 230, 290, 267, 251, 259, 284, 248, 250, 242, 254, 256, 241, 280, 224, 250, 271, 255, 272, 245, 266, 240, 254, 241, 290, 262, 255, 251, 234, 226, 254, 244, 258, 247, 241, 244, 262, 270, 264, 268, 265, 288, 276, 249, 275, 266, 269, 256, 258, 266, 273, 264, 260, 273, 254, 227, 271, 263, 242, 256, 290, 231, 270, 228, 254, 247, 258, 269, 257, 235, 243, 280, 239, 267, 272, 239, 258, 254] payload_rows_per_dst=[5480, 5528, 5396, 5458, 5478, 5490, 5544, 5518] dup_rows_per_dst=[2652, 2680, 2684, 2798, 2594, 2616, 2762, 2858] combine_rows_per_src=[5480, 5528, 5396, 5458, 5478, 5490, 5544, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38154 dup_rows=21894 dup_rows_remote=19194 dedup_ratio=0.334076 router_tokens_per_expert=[233, 259, 264, 251, 254, 267, 237, 255, 260, 278, 245, 256, 251, 235, 244, 253, 256, 241, 238, 266, 228, 266, 290, 272, 237, 257, 261, 269, 254, 291, 262, 263, 258, 238, 243, 258, 262, 269, 250, 265, 253, 281, 238, 273, 262, 240, 271, 233, 266, 258, 229, 262, 241, 268, 271, 259, 251, 225, 279, 263, 245, 258, 270, 285, 250, 256, 256, 249, 243, 252, 246, 249, 266, 232, 238, 228, 281, 243, 269, 237, 260, 249, 259, 263, 247, 257, 246, 242, 247, 265, 247, 238, 266, 289, 267, 269, 262, 252, 272, 290, 257, 284, 287, 278, 284, 234, 229, 253, 244, 255, 238, 283, 237, 247, 254, 273, 235, 242, 252, 271, 256, 253, 236, 254, 252, 290, 244, 247] payload_rows_per_dst=[5396, 5498, 5488, 5464, 5334, 5448, 5562, 5452] dup_rows_per_dst=[2688, 2804, 2700, 2796, 2656, 2774, 2842, 2634] combine_rows_per_src=[5396, 5498, 5488, 5464, 5334, 5448, 5562, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38394 dup_rows=21652 dup_rows_remote=18908 dedup_ratio=0.330383 router_tokens_per_expert=[257, 260, 255, 244, 233, 255, 252, 251, 228, 262, 238, 263, 254, 254, 279, 240, 230, 247, 281, 236, 241, 271, 263, 268, 267, 253, 264, 239, 258, 253, 256, 222, 267, 258, 251, 281, 243, 239, 226, 268, 260, 262, 263, 243, 266, 257, 261, 240, 253, 250, 221, 257, 268, 247, 256, 238, 287, 252, 266, 270, 247, 281, 272, 284, 265, 264, 263, 240, 279, 273, 255, 256, 270, 251, 282, 249, 226, 253, 275, 255, 247, 247, 273, 252, 268, 281, 271, 256, 235, 230, 262, 266, 265, 260, 251, 243, 251, 244, 251, 258, 240, 248, 270, 278, 253, 260, 254, 256, 247, 249, 226, 269, 258, 258, 270, 260, 275, 274, 284, 250, 252, 240, 253, 249, 253, 251, 244, 272] payload_rows_per_dst=[5490, 5498, 5514, 5434, 5594, 5464, 5406, 5484] dup_rows_per_dst=[2744, 2670, 2678, 2848, 2752, 2750, 2592, 2618] combine_rows_per_src=[5490, 5498, 5514, 5434, 5594, 5464, 5406, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38270 dup_rows=21928 dup_rows_remote=19354 dedup_ratio=0.334595 router_tokens_per_expert=[251, 274, 247, 238, 249, 259, 258, 241, 263, 244, 273, 280, 231, 262, 240, 274, 236, 267, 225, 264, 247, 254, 242, 271, 231, 247, 273, 259, 269, 267, 250, 244, 260, 256, 229, 253, 254, 274, 271, 249, 282, 275, 275, 299, 218, 218, 256, 262, 254, 258, 277, 241, 251, 242, 257, 246, 235, 255, 225, 248, 226, 236, 273, 278, 261, 269, 258, 261, 270, 250, 264, 257, 283, 262, 263, 252, 227, 258, 239, 271, 239, 283, 244, 265, 278, 261, 285, 255, 232, 245, 255, 253, 270, 259, 238, 269, 265, 248, 294, 262, 233, 280, 263, 268, 264, 248, 244, 257, 267, 272, 248, 263, 273, 264, 253, 251, 239, 253, 237, 256, 271, 253, 241, 250, 242, 263, 246, 261] payload_rows_per_dst=[5430, 5338, 5552, 5376, 5448, 5426, 5574, 5464] dup_rows_per_dst=[2918, 2574, 2710, 2628, 2842, 2836, 2778, 2642] combine_rows_per_src=[5430, 5338, 5552, 5376, 5448, 5426, 5574, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155721728 dispatch_meta_bytes=227928 combine_payload_bytes=155721728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155949656 ingress_bytes=174596096 link_bytes=174596096 token_bytes=311671384 weight_movement_bytes=18874368 total_nvlink_bytes=330545752 weight_to_token_ratio=0.060559 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672893885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=228920 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157302328 ingress_bytes=213696512 link_bytes=213696512 token_bytes=314375736 weight_movement_bytes=56623104 total_nvlink_bytes=370998840 weight_to_token_ratio=0.180113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047534833e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38340 dup_rows=21804 dup_rows_remote=19108 dedup_ratio=0.332703 router_tokens_per_expert=[244, 233, 249, 254, 232, 282, 257, 270, 261, 234, 263, 264, 293, 247, 248, 268, 248, 279, 254, 243, 258, 216, 263, 279, 253, 286, 256, 281, 265, 244, 261, 246, 258, 266, 255, 247, 246, 252, 284, 260, 258, 237, 264, 267, 268, 256, 253, 249, 246, 251, 241, 256, 249, 271, 277, 235, 224, 235, 275, 262, 245, 259, 295, 271, 270, 224, 225, 236, 266, 252, 255, 251, 237, 276, 246, 258, 258, 260, 265, 265, 251, 252, 257, 256, 281, 278, 242, 258, 265, 256, 242, 265, 265, 247, 281, 271, 258, 251, 266, 228, 267, 265, 244, 276, 232, 268, 244, 239, 253, 239, 262, 259, 247, 269, 246, 251, 265, 236, 263, 244, 247, 243, 286, 252, 278, 240, 246, 250] payload_rows_per_dst=[5484, 5544, 5532, 5498, 5392, 5434, 5396, 5452] dup_rows_per_dst=[2714, 2720, 2708, 2686, 2696, 2900, 2706, 2674] combine_rows_per_src=[5484, 5544, 5532, 5498, 5392, 5434, 5396, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229392 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156508176 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312786960 weight_movement_bytes=37748736 total_nvlink_bytes=350535696 weight_to_token_ratio=0.120685 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229208 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491032 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314752856 weight_movement_bytes=18874368 total_nvlink_bytes=333627224 weight_to_token_ratio=0.059966 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43482 payload_rows_remote=37968 dup_rows=22054 dup_rows_remote=19394 dedup_ratio=0.336517 router_tokens_per_expert=[274, 262, 249, 249, 268, 248, 261, 252, 245, 259, 263, 264, 278, 225, 281, 239, 270, 241, 281, 255, 249, 262, 246, 260, 250, 268, 271, 238, 281, 232, 257, 235, 248, 258, 238, 246, 273, 250, 259, 263, 260, 265, 273, 265, 269, 235, 261, 218, 228, 274, 243, 256, 241, 277, 265, 240, 253, 239, 259, 266, 250, 260, 222, 250, 244, 249, 261, 249, 282, 268, 270, 261, 235, 254, 267, 247, 275, 255, 253, 228, 267, 234, 252, 241, 266, 268, 278, 242, 281, 280, 272, 222, 241, 236, 240, 267, 244, 267, 238, 264, 286, 253, 261, 284, 241, 234, 262, 260, 233, 270, 265, 263, 270, 269, 249, 252, 273, 266, 257, 285, 283, 235, 262, 243, 253, 234, 264, 246] payload_rows_per_dst=[5434, 5438, 5402, 5398, 5430, 5514, 5392, 5474] dup_rows_per_dst=[2800, 2754, 2760, 2648, 2766, 2660, 2858, 2808] combine_rows_per_src=[5434, 5438, 5402, 5398, 5430, 5514, 5392, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38402 dup_rows=21598 dup_rows_remote=18862 dedup_ratio=0.329559 router_tokens_per_expert=[254, 244, 260, 257, 244, 280, 278, 269, 252, 269, 254, 269, 273, 256, 248, 249, 251, 260, 242, 246, 266, 265, 215, 240, 249, 274, 252, 256, 258, 241, 254, 276, 247, 249, 252, 282, 231, 270, 253, 247, 246, 256, 275, 277, 229, 248, 264, 270, 263, 245, 246, 250, 262, 269, 255, 260, 243, 253, 296, 270, 254, 257, 232, 281, 269, 261, 267, 245, 269, 249, 249, 268, 242, 271, 231, 270, 242, 250, 257, 286, 242, 268, 263, 252, 250, 231, 257, 273, 254, 257, 283, 266, 269, 237, 237, 242, 254, 281, 249, 245, 291, 253, 255, 253, 234, 231, 241, 227, 251, 253, 260, 281, 246, 246, 252, 282, 263, 242, 274, 252, 255, 246, 269, 270, 241, 245, 233, 253] payload_rows_per_dst=[5592, 5454, 5500, 5536, 5538, 5464, 5412, 5442] dup_rows_per_dst=[2720, 2636, 2692, 2736, 2714, 2698, 2706, 2696] combine_rows_per_src=[5592, 5454, 5500, 5536, 5538, 5464, 5412, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=230496 combine_payload_bytes=156753920 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156984416 ingress_bytes=213377024 link_bytes=213377024 token_bytes=313738336 weight_movement_bytes=56623104 total_nvlink_bytes=370361440 weight_to_token_ratio=0.180479 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044473656e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229792 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157270432 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314311072 weight_movement_bytes=37748736 total_nvlink_bytes=352059808 weight_to_token_ratio=0.120100 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155516928 dispatch_meta_bytes=229448 combine_payload_bytes=155516928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155746376 ingress_bytes=193265664 link_bytes=193265664 token_bytes=311263304 weight_movement_bytes=37748736 total_nvlink_bytes=349012040 weight_to_token_ratio=0.121276 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851776500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229056 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157523648 ingress_bytes=213917696 link_bytes=213917696 token_bytes=314818240 weight_movement_bytes=56623104 total_nvlink_bytes=371441344 weight_to_token_ratio=0.179860 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049654109e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38252 dup_rows=21760 dup_rows_remote=18806 dedup_ratio=0.332031 router_tokens_per_expert=[263, 268, 265, 242, 239, 253, 261, 274, 250, 260, 269, 263, 248, 251, 249, 261, 241, 283, 245, 251, 265, 266, 247, 234, 272, 280, 243, 261, 251, 307, 243, 271, 247, 276, 257, 288, 261, 262, 227, 270, 259, 243, 265, 273, 254, 235, 255, 287, 249, 276, 281, 268, 274, 235, 277, 235, 223, 233, 253, 228, 279, 247, 248, 244, 255, 253, 267, 281, 237, 250, 274, 257, 266, 237, 240, 269, 240, 259, 250, 239, 267, 236, 257, 274, 231, 261, 252, 264, 225, 268, 261, 264, 276, 229, 243, 250, 214, 236, 236, 281, 232, 231, 237, 268, 241, 283, 225, 264, 281, 239, 241, 309, 260, 243, 277, 251, 257, 266, 267, 292, 263, 252, 229, 268, 237, 232, 269, 270] payload_rows_per_dst=[5548, 5522, 5458, 5420, 5328, 5338, 5638, 5524] dup_rows_per_dst=[2684, 2796, 2740, 2654, 2430, 2594, 2908, 2954] combine_rows_per_src=[5548, 5522, 5458, 5420, 5328, 5338, 5638, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38240 dup_rows=21780 dup_rows_remote=19100 dedup_ratio=0.332336 router_tokens_per_expert=[246, 247, 259, 264, 270, 230, 266, 253, 250, 274, 236, 264, 255, 282, 262, 274, 250, 261, 251, 253, 268, 229, 264, 224, 253, 242, 261, 228, 229, 262, 285, 268, 252, 271, 240, 257, 272, 280, 259, 279, 268, 247, 252, 241, 273, 252, 271, 258, 254, 270, 251, 257, 261, 228, 254, 223, 260, 258, 285, 252, 260, 284, 274, 253, 267, 243, 238, 285, 252, 256, 255, 241, 265, 236, 250, 248, 271, 255, 241, 260, 244, 257, 251, 237, 270, 259, 245, 275, 260, 263, 262, 275, 269, 256, 231, 223, 260, 294, 246, 243, 262, 253, 269, 264, 246, 260, 230, 253, 242, 255, 256, 265, 246, 228, 260, 276, 260, 237, 235, 276, 277, 220, 258, 253, 296, 285, 234, 233] payload_rows_per_dst=[5550, 5412, 5486, 5494, 5420, 5392, 5516, 5486] dup_rows_per_dst=[2698, 2644, 2858, 2754, 2706, 2762, 2680, 2678] combine_rows_per_src=[5550, 5412, 5486, 5494, 5420, 5392, 5516, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38282 dup_rows=21758 dup_rows_remote=18946 dedup_ratio=0.332001 router_tokens_per_expert=[260, 243, 256, 231, 277, 281, 257, 293, 265, 250, 264, 251, 295, 240, 232, 266, 266, 249, 257, 258, 233, 248, 269, 239, 260, 221, 256, 264, 259, 241, 234, 260, 258, 281, 276, 247, 238, 299, 262, 262, 254, 269, 246, 251, 250, 242, 267, 252, 216, 273, 273, 262, 259, 268, 266, 242, 253, 283, 267, 244, 273, 240, 284, 243, 249, 254, 244, 283, 246, 257, 255, 266, 252, 265, 237, 244, 279, 273, 220, 251, 252, 239, 265, 251, 253, 271, 245, 238, 258, 262, 240, 271, 251, 253, 242, 252, 249, 292, 250, 264, 237, 261, 247, 236, 249, 262, 267, 270, 270, 255, 256, 243, 261, 254, 263, 248, 247, 251, 256, 268, 266, 284, 231, 255, 244, 249, 265, 225] payload_rows_per_dst=[5542, 5394, 5496, 5536, 5446, 5448, 5472, 5444] dup_rows_per_dst=[2780, 2634, 2812, 2756, 2704, 2638, 2744, 2690] combine_rows_per_src=[5542, 5394, 5496, 5536, 5446, 5448, 5472, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38174 dup_rows=21692 dup_rows_remote=18732 dedup_ratio=0.330994 router_tokens_per_expert=[269, 271, 264, 242, 256, 249, 261, 264, 245, 255, 252, 249, 226, 253, 262, 251, 243, 240, 247, 301, 272, 257, 244, 249, 243, 266, 261, 253, 266, 276, 271, 248, 243, 261, 243, 252, 258, 238, 253, 241, 231, 252, 239, 240, 229, 248, 256, 268, 267, 282, 251, 265, 246, 253, 260, 236, 279, 251, 229, 277, 247, 266, 248, 246, 276, 247, 232, 248, 267, 270, 270, 279, 255, 249, 231, 220, 263, 261, 272, 236, 260, 279, 232, 272, 263, 273, 257, 253, 269, 275, 282, 247, 258, 255, 269, 266, 267, 272, 275, 262, 259, 241, 254, 252, 267, 269, 274, 240, 226, 264, 260, 276, 227, 255, 269, 258, 244, 242, 244, 272, 240, 265, 256, 241, 283, 264, 248, 255] payload_rows_per_dst=[5670, 5522, 5420, 5442, 5340, 5524, 5458, 5468] dup_rows_per_dst=[2960, 2830, 2642, 2610, 2494, 2754, 2744, 2658] combine_rows_per_src=[5670, 5522, 5420, 5442, 5340, 5524, 5458, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38296 dup_rows=21786 dup_rows_remote=19088 dedup_ratio=0.332428 router_tokens_per_expert=[242, 240, 228, 255, 271, 272, 234, 272, 260, 242, 241, 256, 269, 225, 246, 250, 262, 240, 234, 262, 267, 269, 250, 221, 247, 270, 245, 262, 250, 256, 263, 278, 272, 283, 285, 240, 253, 252, 239, 255, 270, 240, 253, 251, 251, 268, 253, 237, 230, 265, 272, 242, 247, 248, 258, 227, 255, 261, 286, 227, 277, 276, 251, 249, 254, 262, 274, 267, 270, 274, 266, 246, 230, 252, 245, 270, 249, 260, 265, 259, 271, 245, 262, 264, 236, 286, 234, 246, 221, 271, 263, 257, 253, 235, 285, 257, 241, 250, 288, 278, 249, 282, 232, 242, 229, 251, 264, 245, 268, 249, 273, 245, 254, 252, 253, 262, 274, 250, 279, 250, 282, 282, 249, 270, 248, 260, 259, 277] payload_rows_per_dst=[5402, 5454, 5512, 5486, 5460, 5458, 5384, 5594] dup_rows_per_dst=[2604, 2698, 2692, 2656, 2826, 2714, 2788, 2808] combine_rows_per_src=[5402, 5454, 5512, 5486, 5460, 5458, 5384, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=228912 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157031984 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835056 weight_movement_bytes=37748736 total_nvlink_bytes=351583792 weight_to_token_ratio=0.120282 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228232 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156908424 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313588616 weight_movement_bytes=18874368 total_nvlink_bytes=332462984 weight_to_token_ratio=0.060188 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38278 dup_rows=21754 dup_rows_remote=19006 dedup_ratio=0.331940 router_tokens_per_expert=[259, 233, 249, 266, 277, 257, 245, 265, 238, 261, 237, 241, 257, 256, 232, 271, 272, 249, 278, 249, 265, 264, 255, 217, 261, 261, 250, 291, 263, 239, 253, 265, 266, 259, 257, 250, 246, 262, 250, 241, 262, 275, 240, 243, 226, 260, 242, 242, 257, 255, 248, 257, 258, 244, 245, 249, 244, 273, 264, 256, 259, 279, 262, 236, 246, 226, 245, 256, 262, 275, 291, 291, 255, 237, 233, 276, 266, 262, 240, 265, 280, 272, 269, 268, 262, 245, 253, 263, 256, 245, 291, 268, 253, 276, 272, 251, 233, 265, 250, 256, 229, 260, 243, 248, 256, 230, 255, 264, 272, 270, 250, 276, 241, 248, 275, 253, 240, 275, 243, 274, 283, 258, 236, 261, 232, 257, 244, 258] payload_rows_per_dst=[5354, 5552, 5386, 5478, 5504, 5604, 5404, 5500] dup_rows_per_dst=[2734, 2712, 2656, 2694, 2748, 2844, 2710, 2656] combine_rows_per_src=[5354, 5552, 5386, 5478, 5504, 5604, 5404, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229360 combine_payload_bytes=156631040 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156860400 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491440 weight_movement_bytes=18874368 total_nvlink_bytes=332365808 weight_to_token_ratio=0.060207 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38130 dup_rows=21848 dup_rows_remote=19120 dedup_ratio=0.333374 router_tokens_per_expert=[271, 302, 234, 258, 245, 234, 251, 247, 232, 249, 247, 230, 255, 231, 259, 294, 273, 265, 243, 253, 278, 226, 259, 258, 248, 268, 274, 241, 262, 245, 257, 245, 270, 244, 240, 287, 250, 238, 257, 244, 251, 247, 259, 284, 249, 263, 271, 262, 262, 247, 254, 268, 258, 273, 248, 261, 276, 230, 266, 255, 256, 245, 247, 247, 269, 228, 277, 251, 241, 272, 247, 238, 240, 254, 255, 266, 255, 277, 292, 248, 282, 272, 242, 235, 248, 271, 254, 253, 254, 275, 281, 242, 247, 275, 229, 283, 252, 261, 244, 265, 254, 242, 271, 265, 280, 233, 256, 274, 253, 269, 263, 270, 268, 262, 259, 243, 255, 233, 266, 239, 268, 231, 216, 231, 274, 252, 266, 257] payload_rows_per_dst=[5366, 5406, 5484, 5522, 5510, 5558, 5396, 5446] dup_rows_per_dst=[2712, 2784, 2748, 2664, 2710, 2728, 2908, 2594] combine_rows_per_src=[5366, 5406, 5484, 5522, 5510, 5558, 5396, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38300 dup_rows=21746 dup_rows_remote=19108 dedup_ratio=0.331818 router_tokens_per_expert=[257, 268, 253, 234, 277, 246, 279, 289, 242, 254, 272, 236, 251, 264, 286, 261, 259, 241, 266, 248, 264, 242, 247, 246, 296, 248, 266, 246, 252, 264, 241, 253, 257, 260, 248, 249, 257, 259, 248, 266, 239, 269, 263, 277, 252, 248, 254, 266, 268, 238, 236, 238, 245, 259, 254, 239, 250, 269, 274, 264, 289, 255, 246, 240, 240, 236, 260, 210, 251, 237, 259, 263, 243, 250, 251, 249, 242, 263, 228, 239, 269, 270, 244, 244, 241, 244, 276, 254, 283, 265, 260, 232, 236, 258, 268, 247, 249, 265, 298, 285, 263, 254, 244, 282, 263, 263, 279, 243, 250, 263, 255, 246, 262, 274, 229, 250, 250, 263, 227, 277, 259, 276, 273, 264, 266, 253, 255, 252] payload_rows_per_dst=[5506, 5396, 5556, 5490, 5260, 5514, 5554, 5514] dup_rows_per_dst=[2832, 2762, 2668, 2638, 2582, 2668, 2850, 2746] combine_rows_per_src=[5506, 5396, 5556, 5490, 5260, 5514, 5554, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=227624 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156588328 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312949032 weight_movement_bytes=18874368 total_nvlink_bytes=331823400 weight_to_token_ratio=0.060311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229536 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157089952 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950368 weight_movement_bytes=37748736 total_nvlink_bytes=351699104 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229136 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157015824 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802512 weight_movement_bytes=37748736 total_nvlink_bytes=351551248 weight_to_token_ratio=0.120295 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229000 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156409480 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312589960 weight_movement_bytes=37748736 total_nvlink_bytes=350338696 weight_to_token_ratio=0.120761 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229632 combine_payload_bytes=156876800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157106432 ingress_bytes=213499904 link_bytes=213499904 token_bytes=313983232 weight_movement_bytes=56623104 total_nvlink_bytes=370606336 weight_to_token_ratio=0.180338 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045651032e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38412 dup_rows=21612 dup_rows_remote=18814 dedup_ratio=0.329773 router_tokens_per_expert=[281, 250, 253, 254, 272, 242, 243, 265, 251, 262, 238, 245, 281, 258, 255, 239, 268, 246, 246, 274, 274, 261, 258, 300, 257, 244, 234, 234, 262, 243, 254, 253, 241, 259, 269, 287, 240, 256, 245, 272, 273, 238, 238, 284, 267, 272, 251, 263, 259, 270, 254, 265, 250, 261, 231, 242, 265, 270, 241, 251, 265, 261, 276, 243, 257, 221, 255, 257, 253, 268, 266, 244, 260, 250, 253, 253, 271, 254, 273, 253, 262, 272, 266, 238, 238, 254, 269, 268, 258, 234, 256, 242, 265, 239, 245, 269, 251, 253, 256, 258, 268, 261, 296, 248, 268, 274, 259, 244, 252, 231, 252, 266, 254, 260, 278, 225, 263, 237, 256, 244, 229, 236, 250, 237, 257, 250, 272, 264] payload_rows_per_dst=[5482, 5542, 5512, 5506, 5470, 5522, 5522, 5368] dup_rows_per_dst=[2696, 2674, 2798, 2702, 2706, 2628, 2752, 2656] combine_rows_per_src=[5482, 5542, 5512, 5506, 5470, 5522, 5522, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=37890 dup_rows=21956 dup_rows_remote=19016 dedup_ratio=0.335022 router_tokens_per_expert=[265, 243, 271, 278, 241, 270, 248, 255, 239, 271, 246, 242, 284, 244, 272, 226, 257, 260, 274, 259, 244, 245, 253, 242, 244, 243, 243, 282, 253, 251, 237, 228, 256, 250, 236, 281, 238, 263, 272, 265, 228, 263, 263, 272, 273, 274, 246, 239, 262, 241, 268, 263, 271, 275, 264, 269, 253, 256, 278, 260, 229, 263, 283, 263, 268, 267, 256, 278, 253, 277, 218, 260, 245, 263, 249, 233, 244, 259, 277, 256, 246, 255, 251, 233, 252, 286, 260, 230, 261, 238, 283, 260, 259, 254, 266, 239, 252, 253, 256, 251, 269, 245, 250, 244, 246, 267, 262, 266, 249, 270, 268, 268, 246, 271, 243, 243, 281, 247, 249, 269, 267, 246, 245, 235, 256, 232, 265, 254] payload_rows_per_dst=[5454, 5388, 5146, 5306, 5690, 5444, 5462, 5690] dup_rows_per_dst=[2736, 2642, 2548, 2530, 3060, 2636, 2864, 2940] combine_rows_per_src=[5454, 5388, 5146, 5306, 5690, 5444, 5462, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38320 dup_rows=21768 dup_rows_remote=18994 dedup_ratio=0.332153 router_tokens_per_expert=[241, 268, 266, 223, 261, 229, 274, 251, 253, 260, 237, 236, 266, 247, 238, 245, 271, 241, 267, 280, 254, 263, 239, 252, 260, 242, 264, 253, 256, 261, 258, 239, 242, 252, 258, 278, 244, 247, 257, 260, 261, 271, 263, 256, 267, 227, 273, 248, 247, 277, 239, 277, 235, 272, 254, 244, 288, 253, 259, 277, 249, 270, 268, 241, 269, 226, 264, 298, 272, 261, 281, 230, 242, 255, 281, 271, 268, 268, 234, 270, 271, 262, 227, 243, 271, 283, 224, 241, 244, 277, 246, 271, 263, 260, 264, 286, 243, 255, 274, 259, 260, 261, 247, 264, 264, 277, 242, 245, 239, 262, 249, 270, 215, 247, 240, 266, 235, 264, 273, 231, 270, 254, 228, 256, 242, 238, 275, 251] payload_rows_per_dst=[5394, 5514, 5490, 5512, 5538, 5464, 5448, 5408] dup_rows_per_dst=[2596, 2686, 2718, 2788, 2842, 2802, 2774, 2562] combine_rows_per_src=[5394, 5514, 5490, 5512, 5538, 5464, 5448, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38234 dup_rows=21792 dup_rows_remote=18806 dedup_ratio=0.332520 router_tokens_per_expert=[270, 258, 249, 235, 274, 286, 261, 252, 247, 221, 299, 264, 276, 256, 243, 266, 276, 258, 258, 244, 248, 276, 227, 260, 283, 236, 262, 258, 262, 269, 267, 268, 246, 235, 261, 250, 253, 237, 249, 252, 249, 254, 237, 241, 269, 257, 266, 231, 234, 235, 272, 240, 254, 292, 238, 267, 242, 254, 256, 269, 265, 242, 256, 233, 270, 254, 248, 238, 276, 257, 259, 240, 271, 255, 282, 218, 247, 255, 254, 233, 244, 255, 235, 262, 275, 269, 253, 238, 249, 247, 242, 258, 245, 248, 278, 243, 263, 251, 287, 257, 263, 242, 253, 282, 276, 244, 275, 249, 265, 271, 269, 286, 247, 269, 236, 268, 255, 224, 252, 267, 256, 236, 269, 280, 238, 254, 264, 277] payload_rows_per_dst=[5510, 5568, 5390, 5334, 5540, 5416, 5476, 5510] dup_rows_per_dst=[2986, 2910, 2402, 2590, 2756, 2666, 2808, 2674] combine_rows_per_src=[5510, 5568, 5390, 5334, 5540, 5416, 5476, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=228904 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157564456 ingress_bytes=213958656 link_bytes=213958656 token_bytes=314900008 weight_movement_bytes=56623104 total_nvlink_bytes=371523112 weight_to_token_ratio=0.179813 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050046568e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155197440 dispatch_meta_bytes=227624 combine_payload_bytes=155197440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155425064 ingress_bytes=174071808 link_bytes=174071808 token_bytes=310622504 weight_movement_bytes=18874368 total_nvlink_bytes=329496872 weight_to_token_ratio=0.060763 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.667870416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229256 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157187976 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314146696 weight_movement_bytes=37748736 total_nvlink_bytes=351895432 weight_to_token_ratio=0.120163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38186 dup_rows=21814 dup_rows_remote=18978 dedup_ratio=0.332855 router_tokens_per_expert=[256, 238, 258, 272, 268, 275, 224, 245, 246, 262, 244, 271, 240, 263, 258, 288, 287, 241, 249, 288, 277, 256, 273, 281, 264, 241, 275, 234, 249, 256, 248, 267, 271, 278, 256, 250, 266, 267, 266, 227, 256, 280, 262, 269, 259, 237, 266, 242, 248, 244, 249, 252, 263, 265, 239, 235, 223, 206, 259, 260, 275, 264, 272, 285, 223, 242, 247, 240, 265, 254, 235, 217, 258, 240, 270, 256, 270, 234, 266, 248, 263, 256, 238, 243, 272, 273, 257, 284, 254, 265, 252, 264, 271, 259, 255, 263, 247, 244, 263, 275, 257, 250, 243, 258, 223, 252, 254, 275, 250, 271, 248, 248, 250, 237, 251, 247, 231, 254, 252, 261, 275, 256, 274, 217, 254, 260, 281, 291] payload_rows_per_dst=[5454, 5536, 5494, 5418, 5410, 5538, 5420, 5452] dup_rows_per_dst=[2762, 2836, 2810, 2660, 2520, 2800, 2696, 2730] combine_rows_per_src=[5454, 5536, 5494, 5418, 5410, 5538, 5420, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43452 payload_rows_remote=38082 dup_rows=22084 dup_rows_remote=19370 dedup_ratio=0.336975 router_tokens_per_expert=[258, 235, 259, 272, 233, 302, 230, 259, 254, 262, 239, 254, 277, 232, 249, 271, 251, 248, 246, 267, 252, 248, 288, 247, 235, 275, 255, 284, 263, 231, 219, 262, 259, 233, 274, 267, 228, 275, 230, 264, 266, 251, 266, 263, 274, 252, 267, 253, 260, 278, 299, 243, 286, 246, 243, 278, 262, 251, 270, 256, 269, 233, 274, 271, 288, 252, 277, 257, 245, 237, 244, 257, 259, 235, 238, 258, 234, 267, 245, 249, 272, 249, 261, 280, 279, 258, 248, 227, 293, 284, 243, 269, 263, 245, 262, 255, 246, 265, 240, 240, 254, 243, 219, 246, 257, 247, 269, 219, 260, 234, 239, 268, 268, 248, 281, 242, 265, 265, 219, 250, 274, 259, 243, 249, 268, 244, 260, 259] payload_rows_per_dst=[5394, 5436, 5436, 5550, 5370, 5454, 5368, 5444] dup_rows_per_dst=[2778, 2706, 2808, 2888, 2714, 2922, 2524, 2744] combine_rows_per_src=[5394, 5436, 5436, 5550, 5370, 5454, 5368, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228160 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156834624 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441088 weight_movement_bytes=37748736 total_nvlink_bytes=351189824 weight_to_token_ratio=0.120433 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38322 dup_rows=21768 dup_rows_remote=19026 dedup_ratio=0.332153 router_tokens_per_expert=[220, 263, 264, 250, 252, 257, 277, 267, 268, 247, 231, 239, 288, 236, 264, 263, 240, 264, 261, 237, 259, 293, 266, 248, 241, 252, 276, 251, 240, 209, 257, 262, 270, 251, 269, 260, 230, 290, 250, 249, 259, 274, 235, 271, 277, 274, 277, 268, 244, 229, 232, 283, 274, 244, 237, 279, 263, 260, 263, 269, 277, 262, 271, 283, 263, 251, 264, 241, 273, 276, 272, 268, 252, 242, 263, 246, 263, 248, 250, 273, 253, 248, 270, 245, 253, 238, 263, 260, 288, 240, 249, 256, 244, 259, 253, 275, 257, 247, 273, 262, 229, 228, 256, 254, 246, 244, 249, 234, 260, 251, 240, 247, 256, 245, 253, 286, 246, 247, 238, 275, 239, 247, 273, 248, 244, 245, 267, 227] payload_rows_per_dst=[5454, 5490, 5582, 5518, 5484, 5446, 5368, 5426] dup_rows_per_dst=[2718, 2622, 2826, 2822, 2806, 2742, 2586, 2646] combine_rows_per_src=[5454, 5490, 5582, 5518, 5484, 5446, 5368, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38276 dup_rows=21766 dup_rows_remote=18998 dedup_ratio=0.332123 router_tokens_per_expert=[251, 275, 252, 261, 251, 258, 221, 242, 276, 252, 259, 241, 261, 262, 234, 233, 223, 272, 273, 261, 246, 269, 250, 251, 275, 263, 256, 269, 276, 244, 250, 252, 293, 252, 282, 230, 251, 262, 234, 289, 225, 247, 253, 249, 237, 251, 265, 263, 211, 261, 257, 260, 273, 286, 237, 247, 266, 268, 280, 257, 269, 253, 255, 251, 263, 241, 240, 268, 258, 268, 257, 247, 237, 279, 275, 257, 280, 252, 256, 243, 253, 273, 258, 253, 266, 245, 272, 250, 281, 270, 275, 248, 246, 248, 247, 240, 236, 282, 287, 273, 243, 241, 246, 242, 280, 267, 260, 252, 244, 265, 242, 250, 264, 260, 251, 264, 249, 265, 261, 222, 243, 236, 256, 238, 253, 266, 257, 254] payload_rows_per_dst=[5430, 5472, 5468, 5494, 5550, 5518, 5428, 5410] dup_rows_per_dst=[2628, 2788, 2698, 2768, 2692, 2732, 2792, 2668] combine_rows_per_src=[5430, 5472, 5468, 5494, 5550, 5518, 5428, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228656 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156638512 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048368 weight_movement_bytes=18874368 total_nvlink_bytes=331922736 weight_to_token_ratio=0.060292 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=229808 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156213680 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312197552 weight_movement_bytes=18874368 total_nvlink_bytes=331071920 weight_to_token_ratio=0.060456 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229392 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157196304 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314163216 weight_movement_bytes=56623104 total_nvlink_bytes=370786320 weight_to_token_ratio=0.180235 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229096 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007592 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786088 weight_movement_bytes=18874368 total_nvlink_bytes=332660456 weight_to_token_ratio=0.060150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38236 dup_rows=21844 dup_rows_remote=19104 dedup_ratio=0.333313 router_tokens_per_expert=[238, 273, 257, 245, 291, 264, 230, 241, 247, 276, 249, 265, 258, 277, 256, 253, 260, 251, 246, 258, 253, 237, 262, 255, 280, 271, 260, 252, 254, 224, 256, 262, 240, 262, 277, 286, 259, 219, 247, 269, 253, 264, 255, 237, 248, 247, 274, 261, 262, 266, 268, 293, 235, 245, 224, 228, 261, 252, 247, 251, 260, 268, 253, 267, 254, 261, 227, 247, 259, 240, 284, 254, 268, 251, 272, 278, 255, 235, 252, 275, 252, 263, 229, 267, 267, 258, 245, 236, 239, 244, 247, 269, 265, 251, 252, 209, 269, 253, 259, 263, 295, 247, 250, 263, 258, 261, 262, 270, 248, 278, 267, 278, 248, 270, 277, 240, 252, 254, 212, 250, 275, 237, 241, 270, 244, 249, 271, 273] payload_rows_per_dst=[5476, 5454, 5456, 5474, 5474, 5398, 5558, 5402] dup_rows_per_dst=[2764, 2708, 2740, 2686, 2750, 2588, 2884, 2724] combine_rows_per_src=[5476, 5454, 5456, 5474, 5474, 5398, 5558, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38240 dup_rows=21822 dup_rows_remote=18928 dedup_ratio=0.332977 router_tokens_per_expert=[264, 256, 270, 277, 250, 279, 260, 264, 248, 276, 263, 265, 273, 251, 242, 263, 261, 240, 244, 242, 273, 254, 240, 296, 252, 233, 237, 262, 235, 245, 250, 245, 258, 248, 260, 273, 246, 243, 248, 246, 254, 277, 249, 260, 234, 253, 247, 288, 252, 234, 265, 259, 246, 233, 244, 234, 257, 267, 234, 261, 250, 269, 238, 222, 256, 250, 249, 266, 238, 254, 236, 260, 272, 240, 239, 250, 228, 252, 249, 248, 265, 236, 274, 266, 264, 274, 253, 251, 244, 247, 247, 265, 251, 275, 259, 285, 293, 254, 275, 242, 248, 296, 275, 260, 279, 261, 277, 239, 260, 228, 252, 260, 264, 266, 243, 272, 274, 255, 265, 252, 246, 263, 257, 266, 273, 256, 254, 261] payload_rows_per_dst=[5556, 5192, 5484, 5274, 5648, 5528, 5558, 5474] dup_rows_per_dst=[2846, 2322, 2650, 2656, 2830, 2778, 2846, 2894] combine_rows_per_src=[5556, 5192, 5484, 5274, 5648, 5528, 5558, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38292 dup_rows=21874 dup_rows_remote=19336 dedup_ratio=0.333771 router_tokens_per_expert=[270, 288, 261, 264, 259, 241, 246, 236, 268, 261, 258, 260, 288, 263, 260, 216, 257, 237, 275, 256, 246, 260, 254, 261, 271, 251, 235, 281, 251, 258, 289, 243, 272, 267, 256, 292, 251, 240, 252, 263, 257, 227, 267, 262, 251, 243, 296, 246, 243, 270, 260, 250, 265, 270, 258, 238, 252, 296, 276, 238, 245, 249, 260, 249, 252, 287, 260, 255, 261, 266, 258, 267, 252, 262, 263, 256, 249, 253, 258, 288, 261, 293, 286, 265, 268, 261, 223, 253, 271, 242, 272, 210, 254, 227, 259, 261, 249, 239, 227, 259, 262, 246, 248, 263, 243, 238, 263, 246, 276, 226, 234, 235, 255, 251, 258, 240, 225, 238, 260, 264, 274, 258, 235, 236, 260, 249, 244, 249] payload_rows_per_dst=[5484, 5454, 5464, 5494, 5566, 5416, 5370, 5414] dup_rows_per_dst=[2794, 2796, 2820, 2744, 2808, 2796, 2538, 2578] combine_rows_per_src=[5484, 5454, 5464, 5494, 5566, 5416, 5370, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38214 dup_rows=21804 dup_rows_remote=18888 dedup_ratio=0.332703 router_tokens_per_expert=[260, 268, 227, 258, 247, 250, 256, 223, 238, 259, 223, 263, 233, 257, 250, 227, 260, 275, 274, 295, 266, 231, 231, 263, 258, 265, 256, 277, 278, 277, 257, 252, 255, 296, 248, 251, 265, 242, 250, 235, 272, 239, 235, 273, 290, 285, 261, 262, 249, 239, 238, 233, 241, 253, 247, 281, 216, 255, 278, 276, 260, 239, 236, 256, 276, 263, 263, 261, 244, 266, 247, 241, 239, 277, 248, 258, 271, 274, 267, 271, 258, 247, 263, 235, 270, 242, 273, 235, 271, 240, 274, 248, 255, 265, 250, 251, 262, 292, 249, 252, 243, 249, 239, 256, 226, 271, 280, 279, 271, 275, 241, 247, 230, 268, 245, 271, 252, 279, 245, 282, 251, 259, 254, 250, 240, 244, 262, 251] payload_rows_per_dst=[5518, 5342, 5582, 5628, 5486, 5338, 5424, 5414] dup_rows_per_dst=[2916, 2532, 2960, 2822, 2622, 2608, 2592, 2752] combine_rows_per_src=[5518, 5342, 5582, 5628, 5486, 5338, 5424, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229360 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156844016 ingress_bytes=213237760 link_bytes=213237760 token_bytes=313458672 weight_movement_bytes=56623104 total_nvlink_bytes=370081776 weight_to_token_ratio=0.180640 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043139297e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228672 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156859712 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313490752 weight_movement_bytes=37748736 total_nvlink_bytes=351239488 weight_to_token_ratio=0.120414 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=230512 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157074544 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313918576 weight_movement_bytes=18874368 total_nvlink_bytes=332792944 weight_to_token_ratio=0.060125 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38278 dup_rows=21824 dup_rows_remote=19166 dedup_ratio=0.333008 router_tokens_per_expert=[276, 263, 232, 270, 275, 254, 276, 257, 230, 287, 241, 234, 249, 249, 239, 247, 261, 235, 230, 266, 283, 262, 269, 261, 251, 267, 260, 237, 205, 241, 253, 265, 281, 245, 265, 262, 293, 224, 271, 244, 265, 267, 241, 260, 274, 246, 265, 242, 248, 243, 242, 241, 274, 231, 268, 238, 238, 251, 275, 247, 290, 243, 267, 289, 261, 230, 258, 256, 262, 244, 257, 264, 246, 253, 262, 271, 268, 243, 235, 253, 264, 231, 255, 237, 248, 274, 241, 252, 267, 282, 235, 271, 242, 244, 257, 251, 265, 294, 234, 265, 247, 290, 264, 241, 248, 264, 237, 290, 229, 244, 243, 231, 223, 258, 269, 269, 257, 259, 256, 281, 268, 283, 245, 275, 267, 276, 263, 264] payload_rows_per_dst=[5442, 5434, 5470, 5444, 5414, 5474, 5478, 5556] dup_rows_per_dst=[2716, 2658, 2820, 2726, 2712, 2628, 2694, 2870] combine_rows_per_src=[5442, 5434, 5470, 5444, 5414, 5474, 5478, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38460 dup_rows=21638 dup_rows_remote=18988 dedup_ratio=0.330170 router_tokens_per_expert=[240, 241, 259, 227, 223, 248, 257, 257, 261, 254, 258, 261, 260, 217, 258, 275, 255, 257, 225, 281, 264, 230, 255, 262, 249, 254, 285, 272, 261, 245, 274, 265, 258, 257, 244, 258, 261, 275, 239, 266, 240, 275, 269, 273, 270, 255, 242, 247, 254, 262, 268, 272, 267, 266, 302, 237, 246, 245, 244, 264, 266, 266, 270, 268, 259, 232, 264, 258, 234, 242, 270, 273, 259, 257, 247, 262, 259, 243, 244, 241, 243, 269, 237, 262, 259, 242, 264, 259, 230, 274, 274, 202, 253, 238, 246, 264, 242, 250, 278, 244, 236, 284, 259, 244, 284, 254, 255, 249, 267, 252, 255, 256, 290, 289, 274, 233, 250, 246, 248, 252, 257, 256, 244, 268, 261, 240, 247, 288] payload_rows_per_dst=[5430, 5574, 5482, 5512, 5438, 5478, 5420, 5564] dup_rows_per_dst=[2562, 2694, 2776, 2882, 2650, 2554, 2798, 2722] combine_rows_per_src=[5430, 5574, 5482, 5512, 5438, 5478, 5420, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=228408 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156752952 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313277496 weight_movement_bytes=18874368 total_nvlink_bytes=332151864 weight_to_token_ratio=0.060248 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38364 dup_rows=21682 dup_rows_remote=18968 dedup_ratio=0.330841 router_tokens_per_expert=[248, 238, 254, 246, 245, 235, 261, 272, 259, 259, 254, 250, 288, 263, 238, 258, 235, 280, 263, 257, 254, 262, 259, 222, 258, 262, 285, 244, 253, 262, 254, 264, 261, 234, 246, 243, 277, 264, 241, 255, 219, 255, 269, 261, 258, 236, 286, 249, 238, 251, 256, 261, 273, 219, 278, 253, 291, 251, 247, 255, 252, 264, 248, 265, 272, 236, 235, 258, 267, 244, 269, 240, 247, 271, 230, 247, 246, 250, 264, 248, 253, 266, 260, 244, 254, 250, 252, 255, 266, 282, 257, 255, 257, 250, 259, 237, 262, 286, 245, 246, 248, 246, 267, 274, 258, 273, 272, 274, 251, 274, 258, 267, 272, 261, 269, 249, 226, 269, 258, 256, 250, 286, 281, 270, 253, 251, 228, 229] payload_rows_per_dst=[5510, 5536, 5396, 5490, 5384, 5506, 5516, 5516] dup_rows_per_dst=[2626, 2692, 2712, 2714, 2664, 2688, 2886, 2700] combine_rows_per_src=[5510, 5536, 5396, 5490, 5384, 5506, 5516, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38060 dup_rows=21904 dup_rows_remote=19120 dedup_ratio=0.334229 router_tokens_per_expert=[244, 244, 278, 273, 275, 264, 260, 271, 249, 241, 258, 259, 229, 253, 272, 238, 244, 293, 254, 246, 256, 268, 245, 262, 284, 240, 246, 273, 259, 262, 257, 259, 233, 263, 233, 258, 242, 232, 221, 254, 246, 245, 261, 286, 251, 261, 269, 245, 274, 266, 279, 270, 276, 253, 229, 251, 284, 272, 244, 218, 229, 259, 244, 231, 235, 258, 271, 257, 247, 258, 244, 262, 260, 241, 275, 275, 239, 260, 250, 278, 290, 232, 261, 225, 246, 271, 263, 233, 252, 269, 283, 286, 270, 268, 258, 271, 253, 252, 258, 266, 256, 272, 245, 254, 266, 234, 248, 265, 240, 251, 256, 247, 256, 270, 246, 253, 234, 249, 250, 273, 276, 252, 239, 262, 225, 291, 267, 239] payload_rows_per_dst=[5478, 5484, 5348, 5400, 5460, 5572, 5446, 5444] dup_rows_per_dst=[2738, 2812, 2652, 2758, 2760, 2784, 2680, 2720] combine_rows_per_src=[5478, 5484, 5348, 5400, 5460, 5572, 5446, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229776 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016464 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803152 weight_movement_bytes=37748736 total_nvlink_bytes=351551888 weight_to_token_ratio=0.120294 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157532160 dispatch_meta_bytes=229792 combine_payload_bytes=157532160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157761952 ingress_bytes=176406528 link_bytes=176406528 token_bytes=315294112 weight_movement_bytes=18874368 total_nvlink_bytes=334168480 weight_to_token_ratio=0.059863 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690240554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229328 combine_payload_bytes=157138944 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157368272 ingress_bytes=194887680 link_bytes=194887680 token_bytes=314507216 weight_movement_bytes=37748736 total_nvlink_bytes=352255952 weight_to_token_ratio=0.120025 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38408 dup_rows=21788 dup_rows_remote=19082 dedup_ratio=0.332458 router_tokens_per_expert=[282, 254, 242, 258, 266, 233, 258, 255, 246, 229, 259, 265, 255, 264, 247, 257, 258, 254, 248, 270, 276, 250, 286, 264, 266, 247, 252, 270, 250, 258, 230, 260, 259, 237, 245, 245, 288, 258, 249, 258, 247, 260, 246, 248, 222, 227, 265, 269, 251, 257, 241, 247, 268, 247, 249, 249, 267, 262, 259, 247, 257, 241, 246, 282, 255, 249, 246, 259, 281, 246, 258, 247, 254, 242, 266, 279, 283, 262, 262, 243, 272, 233, 263, 250, 273, 256, 272, 252, 265, 273, 251, 251, 263, 225, 288, 259, 271, 224, 276, 252, 251, 231, 273, 231, 234, 251, 288, 261, 298, 256, 235, 246, 227, 257, 273, 262, 291, 245, 246, 243, 236, 254, 260, 258, 287, 290, 242, 239] payload_rows_per_dst=[5432, 5544, 5340, 5458, 5482, 5558, 5482, 5452] dup_rows_per_dst=[2708, 2734, 2706, 2682, 2782, 2734, 2674, 2768] combine_rows_per_src=[5432, 5544, 5340, 5458, 5482, 5558, 5482, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=228720 combine_payload_bytes=155893760 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156122480 ingress_bytes=193642496 link_bytes=193642496 token_bytes=312016240 weight_movement_bytes=37748736 total_nvlink_bytes=349764976 weight_to_token_ratio=0.120983 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855387119e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38150 dup_rows=21778 dup_rows_remote=18986 dedup_ratio=0.332306 router_tokens_per_expert=[264, 264, 231, 255, 247, 283, 257, 245, 241, 268, 241, 258, 256, 255, 262, 238, 273, 253, 258, 239, 253, 265, 243, 248, 246, 267, 246, 222, 257, 252, 237, 258, 231, 245, 241, 256, 264, 243, 280, 263, 263, 252, 276, 264, 264, 268, 264, 262, 252, 249, 277, 265, 254, 285, 233, 266, 280, 238, 258, 250, 231, 275, 261, 263, 252, 249, 233, 258, 242, 232, 279, 246, 258, 262, 265, 269, 250, 285, 234, 276, 249, 272, 256, 261, 252, 234, 279, 246, 251, 258, 229, 255, 268, 250, 256, 276, 269, 281, 260, 266, 272, 285, 243, 252, 265, 254, 251, 248, 264, 277, 254, 259, 245, 241, 243, 251, 254, 239, 239, 282, 259, 245, 267, 269, 254, 249, 232, 262] payload_rows_per_dst=[5428, 5446, 5450, 5526, 5502, 5402, 5608, 5396] dup_rows_per_dst=[2702, 2588, 2732, 2748, 2768, 2782, 2792, 2666] combine_rows_per_src=[5428, 5446, 5450, 5526, 5502, 5402, 5608, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38202 dup_rows=21814 dup_rows_remote=19024 dedup_ratio=0.332855 router_tokens_per_expert=[255, 269, 248, 242, 270, 263, 257, 244, 243, 266, 279, 266, 230, 263, 257, 248, 228, 275, 255, 256, 258, 259, 265, 254, 265, 266, 248, 238, 271, 256, 241, 272, 259, 260, 277, 271, 253, 283, 237, 245, 275, 257, 267, 247, 262, 255, 242, 258, 248, 261, 247, 269, 242, 245, 243, 234, 261, 268, 257, 280, 266, 295, 249, 247, 269, 247, 250, 290, 233, 268, 245, 226, 249, 243, 248, 275, 246, 251, 236, 236, 234, 234, 241, 241, 258, 256, 263, 262, 213, 256, 259, 260, 244, 265, 261, 263, 272, 253, 276, 265, 264, 262, 251, 258, 254, 262, 260, 253, 258, 240, 260, 273, 261, 265, 238, 262, 272, 275, 242, 244, 241, 258, 268, 243, 285, 237, 244, 283] payload_rows_per_dst=[5474, 5436, 5524, 5450, 5438, 5328, 5552, 5520] dup_rows_per_dst=[2726, 2704, 2726, 2686, 2586, 2738, 2858, 2790] combine_rows_per_src=[5474, 5436, 5524, 5450, 5438, 5328, 5552, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229960 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157549128 ingress_bytes=213942272 link_bytes=213942272 token_bytes=314868296 weight_movement_bytes=56623104 total_nvlink_bytes=371491400 weight_to_token_ratio=0.179831 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049889584e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38398 dup_rows=21728 dup_rows_remote=18918 dedup_ratio=0.331543 router_tokens_per_expert=[252, 231, 272, 276, 242, 252, 251, 248, 260, 248, 273, 255, 271, 251, 239, 285, 274, 267, 244, 295, 267, 268, 263, 271, 264, 250, 237, 234, 227, 242, 253, 270, 283, 268, 252, 250, 244, 245, 278, 234, 245, 236, 263, 285, 231, 241, 236, 244, 244, 247, 227, 244, 255, 238, 275, 262, 292, 253, 248, 264, 252, 254, 261, 267, 216, 270, 238, 264, 277, 261, 244, 260, 267, 265, 263, 270, 256, 252, 270, 249, 289, 218, 286, 270, 257, 248, 249, 229, 256, 230, 252, 272, 253, 264, 283, 263, 282, 266, 258, 251, 264, 254, 250, 244, 256, 261, 242, 254, 262, 240, 242, 253, 232, 262, 245, 271, 243, 244, 264, 284, 245, 269, 270, 243, 270, 248, 254, 254] payload_rows_per_dst=[5410, 5572, 5472, 5414, 5468, 5498, 5472, 5502] dup_rows_per_dst=[2810, 2672, 2604, 2746, 2776, 2740, 2686, 2694] combine_rows_per_src=[5410, 5572, 5472, 5414, 5468, 5498, 5472, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228544 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156490944 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753344 weight_movement_bytes=18874368 total_nvlink_bytes=331627712 weight_to_token_ratio=0.060349 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=228904 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704296 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179688 weight_movement_bytes=18874368 total_nvlink_bytes=332054056 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38202 dup_rows=21852 dup_rows_remote=19222 dedup_ratio=0.333435 router_tokens_per_expert=[257, 247, 277, 275, 243, 265, 227, 257, 271, 251, 271, 293, 243, 227, 266, 250, 258, 241, 264, 240, 263, 255, 257, 290, 238, 227, 271, 254, 224, 264, 251, 259, 275, 236, 244, 262, 282, 251, 268, 254, 266, 266, 259, 265, 248, 289, 271, 267, 244, 249, 292, 261, 243, 254, 281, 270, 244, 252, 244, 253, 251, 264, 241, 276, 260, 264, 268, 238, 250, 252, 242, 300, 247, 245, 232, 258, 232, 243, 267, 262, 230, 274, 257, 244, 256, 252, 282, 274, 268, 239, 262, 270, 240, 262, 249, 231, 289, 256, 241, 236, 256, 254, 267, 235, 244, 247, 270, 260, 224, 280, 259, 232, 254, 276, 265, 263, 265, 268, 261, 236, 232, 256, 255, 225, 238, 277, 247, 252] payload_rows_per_dst=[5460, 5482, 5560, 5480, 5404, 5498, 5376, 5424] dup_rows_per_dst=[2780, 2630, 2846, 2758, 2716, 2682, 2724, 2716] combine_rows_per_src=[5460, 5482, 5560, 5480, 5404, 5498, 5376, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38388 dup_rows=21752 dup_rows_remote=19266 dedup_ratio=0.331909 router_tokens_per_expert=[265, 265, 245, 280, 271, 231, 246, 240, 250, 274, 230, 257, 247, 237, 271, 269, 258, 243, 259, 258, 264, 264, 222, 273, 238, 264, 262, 240, 269, 263, 237, 280, 274, 248, 264, 264, 253, 246, 261, 285, 242, 255, 242, 245, 256, 232, 241, 268, 225, 269, 250, 272, 251, 251, 251, 268, 231, 233, 260, 270, 274, 271, 240, 240, 276, 240, 251, 220, 238, 249, 262, 233, 244, 249, 242, 248, 290, 230, 229, 240, 272, 232, 265, 264, 244, 246, 275, 246, 258, 272, 261, 253, 284, 240, 241, 256, 249, 275, 264, 272, 259, 255, 255, 285, 262, 285, 272, 279, 234, 250, 263, 287, 271, 258, 267, 248, 259, 229, 272, 243, 282, 265, 264, 248, 263, 281, 258, 260] payload_rows_per_dst=[5460, 5508, 5436, 5460, 5396, 5438, 5558, 5528] dup_rows_per_dst=[2696, 2680, 2716, 2652, 2486, 2780, 2934, 2808] combine_rows_per_src=[5460, 5508, 5436, 5460, 5396, 5438, 5558, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38376 dup_rows=21758 dup_rows_remote=19076 dedup_ratio=0.332001 router_tokens_per_expert=[279, 250, 230, 262, 281, 259, 241, 263, 245, 273, 252, 259, 279, 261, 257, 268, 266, 241, 262, 228, 245, 241, 249, 274, 254, 241, 268, 244, 245, 251, 251, 274, 256, 260, 252, 281, 271, 257, 268, 283, 256, 250, 233, 252, 254, 267, 247, 244, 248, 236, 280, 245, 256, 228, 247, 265, 262, 237, 235, 276, 285, 245, 225, 272, 266, 242, 253, 275, 275, 241, 250, 253, 254, 253, 279, 249, 277, 245, 266, 255, 261, 275, 267, 276, 246, 247, 241, 254, 231, 241, 253, 256, 216, 286, 242, 266, 290, 255, 262, 265, 242, 235, 259, 264, 247, 255, 260, 271, 275, 237, 239, 257, 268, 243, 277, 265, 260, 234, 222, 257, 257, 253, 267, 259, 250, 278, 263, 245] payload_rows_per_dst=[5512, 5420, 5524, 5402, 5530, 5378, 5594, 5418] dup_rows_per_dst=[2806, 2648, 2738, 2682, 2736, 2738, 2632, 2778] combine_rows_per_src=[5512, 5420, 5524, 5402, 5530, 5378, 5594, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229264 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507472 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314785680 weight_movement_bytes=18874368 total_nvlink_bytes=333660048 weight_to_token_ratio=0.059959 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38486 dup_rows=21632 dup_rows_remote=18920 dedup_ratio=0.330078 router_tokens_per_expert=[251, 257, 242, 247, 263, 265, 259, 276, 257, 246, 263, 249, 252, 249, 242, 246, 237, 252, 280, 262, 275, 250, 260, 291, 233, 242, 243, 250, 264, 269, 271, 271, 233, 278, 267, 226, 271, 249, 248, 255, 272, 267, 260, 238, 266, 275, 258, 253, 229, 258, 246, 268, 249, 259, 280, 246, 265, 286, 262, 270, 252, 264, 247, 254, 238, 251, 262, 273, 271, 264, 259, 246, 269, 230, 266, 246, 241, 262, 255, 254, 258, 257, 252, 254, 242, 232, 272, 261, 229, 261, 259, 261, 247, 282, 253, 245, 270, 235, 262, 260, 242, 262, 244, 271, 243, 269, 255, 261, 246, 253, 243, 253, 246, 246, 239, 260, 262, 247, 260, 276, 270, 247, 248, 261, 254, 242, 233, 291] payload_rows_per_dst=[5448, 5584, 5576, 5496, 5416, 5418, 5496, 5470] dup_rows_per_dst=[2680, 2716, 2656, 2774, 2758, 2712, 2642, 2694] combine_rows_per_src=[5448, 5584, 5576, 5496, 5416, 5418, 5496, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229696 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156705088 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313180480 weight_movement_bytes=18874368 total_nvlink_bytes=332054848 weight_to_token_ratio=0.060267 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229808 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157417904 ingress_bytes=194936832 link_bytes=194936832 token_bytes=314606000 weight_movement_bytes=37748736 total_nvlink_bytes=352354736 weight_to_token_ratio=0.119987 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867788809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=230616 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157467864 ingress_bytes=176111616 link_bytes=176111616 token_bytes=314705112 weight_movement_bytes=18874368 total_nvlink_bytes=333579480 weight_to_token_ratio=0.059975 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687414852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38290 dup_rows=21812 dup_rows_remote=19186 dedup_ratio=0.332825 router_tokens_per_expert=[245, 260, 259, 259, 243, 243, 250, 232, 236, 266, 253, 269, 236, 273, 242, 251, 247, 230, 227, 284, 250, 267, 238, 284, 257, 257, 254, 262, 257, 262, 268, 261, 247, 262, 233, 246, 284, 273, 240, 265, 224, 241, 253, 261, 244, 240, 245, 272, 251, 261, 271, 267, 289, 259, 258, 256, 239, 271, 225, 229, 263, 260, 250, 268, 253, 267, 274, 247, 239, 254, 284, 269, 250, 277, 259, 251, 236, 288, 257, 264, 267, 280, 230, 251, 236, 269, 273, 265, 259, 252, 257, 258, 267, 262, 248, 247, 279, 272, 247, 295, 268, 264, 267, 255, 260, 250, 259, 276, 243, 242, 266, 267, 250, 281, 279, 234, 231, 243, 245, 252, 241, 237, 236, 257, 238, 261, 258, 256] payload_rows_per_dst=[5424, 5420, 5434, 5462, 5528, 5492, 5588, 5376] dup_rows_per_dst=[2610, 2790, 2626, 2772, 2810, 2750, 2832, 2622] combine_rows_per_src=[5424, 5420, 5434, 5462, 5528, 5492, 5588, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38390 dup_rows=21700 dup_rows_remote=19042 dedup_ratio=0.331116 router_tokens_per_expert=[242, 252, 260, 231, 237, 250, 272, 244, 299, 254, 254, 267, 249, 234, 255, 259, 258, 242, 265, 287, 280, 252, 232, 284, 266, 249, 272, 255, 261, 250, 277, 278, 254, 239, 283, 265, 255, 272, 272, 253, 264, 253, 260, 267, 279, 277, 260, 290, 265, 231, 248, 253, 249, 242, 278, 259, 267, 262, 265, 274, 214, 257, 292, 276, 234, 266, 241, 249, 263, 229, 272, 235, 279, 270, 240, 217, 260, 250, 240, 257, 259, 266, 272, 273, 252, 232, 224, 247, 261, 245, 246, 250, 277, 272, 260, 246, 251, 243, 232, 276, 276, 229, 237, 239, 247, 266, 242, 303, 240, 279, 235, 257, 244, 237, 262, 254, 277, 241, 230, 233, 260, 247, 269, 253, 230, 263, 248, 242] payload_rows_per_dst=[5428, 5542, 5586, 5532, 5416, 5490, 5446, 5396] dup_rows_per_dst=[2690, 2780, 2900, 2732, 2682, 2674, 2658, 2584] combine_rows_per_src=[5428, 5542, 5586, 5532, 5416, 5490, 5446, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38088 dup_rows=21914 dup_rows_remote=19038 dedup_ratio=0.334381 router_tokens_per_expert=[262, 260, 257, 239, 224, 247, 256, 268, 269, 247, 277, 245, 236, 266, 253, 255, 257, 253, 260, 272, 242, 243, 254, 234, 246, 257, 257, 264, 256, 252, 254, 250, 248, 250, 269, 245, 243, 283, 296, 230, 235, 259, 270, 249, 278, 252, 247, 287, 256, 240, 284, 263, 266, 278, 265, 273, 269, 249, 261, 254, 272, 262, 266, 257, 266, 243, 272, 252, 271, 275, 229, 266, 268, 282, 268, 277, 260, 242, 267, 248, 246, 260, 274, 253, 263, 258, 251, 255, 248, 263, 278, 264, 250, 256, 269, 259, 247, 238, 237, 221, 223, 240, 238, 258, 243, 249, 257, 246, 232, 239, 272, 240, 250, 265, 253, 275, 252, 260, 256, 262, 259, 246, 240, 270, 244, 240, 251, 264] payload_rows_per_dst=[5406, 5376, 5458, 5496, 5400, 5532, 5420, 5534] dup_rows_per_dst=[2716, 2726, 2824, 2698, 2644, 2762, 2668, 2876] combine_rows_per_src=[5406, 5376, 5458, 5496, 5400, 5532, 5420, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157638656 dispatch_meta_bytes=229624 combine_payload_bytes=157638656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157868280 ingress_bytes=176513024 link_bytes=176513024 token_bytes=315506936 weight_movement_bytes=18874368 total_nvlink_bytes=334381304 weight_to_token_ratio=0.059822 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691260946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15052587520 prefetch_bytes=717225984 reduce_bytes=943718400 weight_movement_bytes=1660944384 egress_bytes=7531799040 ingress_bytes=9181732864 link_bytes=9181732864 total_nvlink_bytes=16713531904 weight_to_token_ratio=0.110343 prefetch_share_of_ingress=0.078114 dedup_ratio=0.332728 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=50 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108458 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.832809304e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229904 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065744 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313901584 weight_movement_bytes=18874368 total_nvlink_bytes=332775952 weight_to_token_ratio=0.060128 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=229728 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157475168 ingress_bytes=194994176 link_bytes=194994176 token_bytes=314720608 weight_movement_bytes=37748736 total_nvlink_bytes=352469344 weight_to_token_ratio=0.119944 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868338251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38204 dup_rows=21886 dup_rows_remote=19192 dedup_ratio=0.333954 router_tokens_per_expert=[216, 245, 228, 243, 226, 290, 248, 258, 276, 258, 268, 249, 263, 272, 245, 246, 237, 253, 262, 240, 285, 277, 256, 275, 232, 239, 252, 254, 278, 275, 271, 238, 257, 284, 232, 246, 271, 262, 241, 238, 252, 243, 261, 268, 255, 273, 276, 255, 254, 253, 278, 256, 253, 257, 244, 269, 272, 258, 252, 262, 249, 266, 253, 251, 262, 228, 259, 256, 272, 285, 239, 254, 271, 229, 270, 267, 231, 254, 241, 279, 280, 277, 281, 239, 243, 246, 264, 265, 253, 262, 239, 255, 267, 249, 242, 269, 265, 261, 250, 259, 241, 270, 262, 263, 250, 270, 266, 235, 245, 256, 240, 252, 274, 229, 253, 252, 247, 253, 243, 240, 252, 281, 227, 240, 277, 258, 264, 269] payload_rows_per_dst=[5446, 5620, 5442, 5508, 5446, 5348, 5414, 5426] dup_rows_per_dst=[2694, 2852, 2728, 2726, 2748, 2690, 2756, 2692] combine_rows_per_src=[5446, 5620, 5442, 5508, 5446, 5348, 5414, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228504 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156236952 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312245400 weight_movement_bytes=18874368 total_nvlink_bytes=331119768 weight_to_token_ratio=0.060447 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38322 dup_rows=21722 dup_rows_remote=19018 dedup_ratio=0.331451 router_tokens_per_expert=[241, 242, 253, 245, 286, 241, 265, 259, 237, 247, 259, 257, 259, 254, 240, 234, 264, 243, 265, 237, 264, 210, 262, 265, 262, 254, 259, 266, 257, 248, 283, 242, 251, 272, 253, 237, 270, 255, 264, 276, 252, 268, 237, 241, 253, 265, 252, 260, 274, 248, 268, 251, 253, 244, 260, 267, 250, 275, 245, 273, 228, 240, 224, 249, 292, 258, 255, 239, 280, 278, 255, 276, 268, 265, 252, 253, 256, 248, 235, 235, 256, 261, 246, 279, 244, 245, 265, 263, 284, 269, 277, 293, 259, 256, 274, 251, 217, 243, 247, 242, 253, 268, 248, 213, 251, 255, 243, 247, 298, 273, 249, 247, 282, 279, 257, 257, 263, 268, 221, 249, 260, 263, 259, 232, 280, 254, 250, 278] payload_rows_per_dst=[5382, 5492, 5496, 5456, 5506, 5588, 5380, 5514] dup_rows_per_dst=[2656, 2704, 2716, 2642, 2784, 2822, 2608, 2790] combine_rows_per_src=[5382, 5492, 5496, 5456, 5506, 5588, 5380, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15059994096 prefetch_bytes=736100352 reduce_bytes=905969664 weight_movement_bytes=1642070016 egress_bytes=7535502832 ingress_bytes=9166561280 link_bytes=9166561280 total_nvlink_bytes=16702064112 weight_to_token_ratio=0.109035 prefetch_share_of_ingress=0.080303 dedup_ratio=0.332575 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=48 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081380 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.829780832e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229584 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156713168 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196752 weight_movement_bytes=18874368 total_nvlink_bytes=332071120 weight_to_token_ratio=0.060264 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38134 dup_rows=21900 dup_rows_remote=19172 dedup_ratio=0.334167 router_tokens_per_expert=[257, 264, 270, 247, 255, 255, 248, 259, 244, 244, 242, 256, 248, 256, 243, 247, 264, 249, 264, 259, 260, 251, 242, 227, 256, 252, 267, 263, 244, 258, 264, 233, 262, 253, 262, 267, 269, 248, 256, 280, 292, 257, 269, 238, 229, 256, 234, 250, 222, 247, 253, 271, 243, 256, 287, 253, 281, 243, 251, 270, 260, 254, 264, 293, 253, 296, 241, 232, 257, 248, 271, 258, 270, 252, 250, 258, 258, 259, 282, 230, 287, 247, 255, 242, 258, 276, 246, 244, 271, 255, 271, 253, 266, 244, 251, 228, 233, 253, 300, 261, 269, 246, 269, 265, 266, 235, 267, 256, 251, 239, 251, 255, 259, 265, 244, 258, 240, 245, 263, 246, 267, 259, 260, 262, 252, 256, 259, 250] payload_rows_per_dst=[5400, 5406, 5534, 5460, 5502, 5400, 5498, 5436] dup_rows_per_dst=[2670, 2700, 2710, 2836, 2728, 2788, 2734, 2734] combine_rows_per_src=[5400, 5406, 5534, 5460, 5502, 5400, 5498, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15010605280 prefetch_bytes=811597824 reduce_bytes=622854144 weight_movement_bytes=1434451968 egress_bytes=7510788320 ingress_bytes=8934268928 link_bytes=8934497648 total_nvlink_bytes=16445057248 weight_to_token_ratio=0.095563 prefetch_share_of_ingress=0.090841 dedup_ratio=0.333384 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=33 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080729 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.783457509e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229360 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196272 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163184 weight_movement_bytes=18874368 total_nvlink_bytes=333037552 weight_to_token_ratio=0.060078 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38066 dup_rows=21954 dup_rows_remote=19120 dedup_ratio=0.334991 router_tokens_per_expert=[287, 234, 279, 286, 228, 264, 286, 240, 243, 238, 279, 254, 258, 267, 257, 271, 278, 262, 240, 234, 234, 244, 251, 241, 269, 280, 266, 244, 231, 260, 267, 257, 254, 252, 260, 207, 240, 251, 242, 271, 262, 284, 227, 279, 245, 244, 251, 269, 225, 232, 236, 252, 235, 224, 289, 240, 253, 263, 257, 251, 304, 283, 252, 264, 254, 244, 246, 234, 287, 237, 261, 258, 265, 246, 252, 252, 268, 293, 225, 265, 274, 285, 263, 224, 275, 241, 274, 251, 273, 262, 269, 250, 248, 263, 261, 262, 244, 237, 244, 240, 250, 285, 250, 250, 249, 278, 277, 244, 241, 264, 257, 243, 281, 277, 269, 259, 267, 258, 274, 235, 237, 244, 238, 257, 256, 264, 244, 266] payload_rows_per_dst=[5536, 5400, 5422, 5362, 5452, 5516, 5422, 5472] dup_rows_per_dst=[2806, 2716, 2654, 2758, 2722, 2834, 2684, 2780] combine_rows_per_src=[5536, 5400, 5422, 5362, 5452, 5516, 5422, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=229224 combine_payload_bytes=156196864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156426088 ingress_bytes=193945600 link_bytes=193945600 token_bytes=312622952 weight_movement_bytes=37748736 total_nvlink_bytes=350371688 weight_to_token_ratio=0.120748 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858291312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44038 payload_rows_remote=38570 dup_rows=21498 dup_rows_remote=18758 dedup_ratio=0.328033 router_tokens_per_expert=[244, 255, 257, 258, 273, 254, 267, 255, 243, 239, 232, 255, 251, 256, 248, 259, 258, 271, 258, 230, 232, 252, 247, 231, 259, 261, 274, 258, 239, 254, 251, 256, 286, 263, 271, 252, 244, 245, 231, 253, 268, 282, 255, 268, 237, 248, 262, 259, 262, 248, 259, 237, 247, 282, 278, 223, 251, 267, 261, 270, 258, 256, 271, 271, 281, 252, 241, 259, 273, 254, 258, 243, 255, 248, 284, 281, 223, 260, 277, 235, 235, 280, 246, 241, 273, 269, 247, 297, 242, 247, 235, 260, 251, 252, 286, 276, 226, 217, 275, 257, 265, 244, 269, 297, 245, 294, 275, 262, 228, 258, 241, 251, 253, 232, 276, 254, 253, 251, 264, 247, 260, 286, 234, 250, 252, 241, 277, 231] payload_rows_per_dst=[5464, 5460, 5508, 5580, 5532, 5566, 5468, 5460] dup_rows_per_dst=[2628, 2602, 2740, 2702, 2716, 2708, 2740, 2662] combine_rows_per_src=[5464, 5460, 5508, 5580, 5532, 5566, 5468, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228744 combine_payload_bytes=155918336 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156147080 ingress_bytes=212541440 link_bytes=212541440 token_bytes=312065416 weight_movement_bytes=56623104 total_nvlink_bytes=368688520 weight_to_token_ratio=0.181446 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036467501e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15059486408 prefetch_bytes=736100352 reduce_bytes=717225984 weight_movement_bytes=1453326336 egress_bytes=7535249096 ingress_bytes=8977563648 link_bytes=8977563648 total_nvlink_bytes=16512812744 weight_to_token_ratio=0.096506 prefetch_share_of_ingress=0.081993 dedup_ratio=0.332625 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=38 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081308 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.792054117e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15052801168 prefetch_bytes=773849088 reduce_bytes=792723456 weight_movement_bytes=1566572544 egress_bytes=7531906192 ingress_bytes=9087467520 link_bytes=9087467520 total_nvlink_bytes=16619373712 weight_to_token_ratio=0.104072 prefetch_share_of_ingress=0.085156 dedup_ratio=0.332850 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=42 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081163 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.813992551e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157982720 dispatch_meta_bytes=229312 combine_payload_bytes=157982720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158212032 ingress_bytes=195731456 link_bytes=195731456 token_bytes=316194752 weight_movement_bytes=37748736 total_nvlink_bytes=353943488 weight_to_token_ratio=0.119384 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.875402506e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38068 dup_rows=21998 dup_rows_remote=19116 dedup_ratio=0.335663 router_tokens_per_expert=[256, 229, 289, 228, 253, 219, 259, 231, 264, 236, 260, 243, 247, 264, 256, 261, 271, 247, 275, 271, 262, 258, 248, 229, 264, 268, 270, 266, 273, 264, 276, 253, 249, 228, 253, 261, 252, 273, 275, 254, 270, 253, 267, 239, 255, 250, 289, 244, 229, 239, 258, 282, 228, 258, 256, 253, 251, 257, 267, 257, 224, 264, 253, 240, 243, 239, 263, 278, 259, 253, 251, 263, 230, 255, 268, 228, 239, 238, 268, 255, 252, 257, 282, 270, 254, 279, 243, 258, 263, 240, 255, 236, 292, 272, 284, 240, 280, 239, 228, 244, 232, 264, 235, 260, 252, 247, 218, 257, 284, 251, 254, 263, 248, 254, 257, 269, 248, 229, 282, 244, 268, 262, 270, 277, 280, 274, 288, 285] payload_rows_per_dst=[5470, 5632, 5334, 5454, 5388, 5558, 5252, 5450] dup_rows_per_dst=[2882, 2990, 2672, 2744, 2672, 2782, 2598, 2658] combine_rows_per_src=[5470, 5632, 5334, 5454, 5388, 5558, 5252, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15046769520 prefetch_bytes=773849088 reduce_bytes=830472192 weight_movement_bytes=1604321280 egress_bytes=7528889200 ingress_bytes=9122201600 link_bytes=9122431184 total_nvlink_bytes=16651090800 weight_to_token_ratio=0.106622 prefetch_share_of_ingress=0.084831 dedup_ratio=0.333054 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=44 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.820971814e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=228736 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156155264 ingress_bytes=174800896 link_bytes=174800896 token_bytes=312081792 weight_movement_bytes=18874368 total_nvlink_bytes=330956160 weight_to_token_ratio=0.060479 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674856178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38150 dup_rows=21938 dup_rows_remote=19226 dedup_ratio=0.334747 router_tokens_per_expert=[254, 249, 262, 231, 279, 264, 274, 292, 235, 226, 264, 240, 274, 280, 270, 261, 230, 252, 271, 272, 271, 248, 269, 243, 242, 278, 242, 242, 256, 236, 252, 235, 242, 282, 266, 262, 252, 246, 273, 249, 276, 238, 240, 250, 238, 233, 245, 258, 265, 258, 263, 268, 269, 234, 275, 299, 254, 258, 255, 254, 256, 269, 235, 232, 292, 260, 253, 240, 228, 261, 272, 261, 248, 205, 256, 248, 259, 258, 266, 238, 244, 275, 269, 257, 260, 261, 247, 265, 257, 270, 249, 272, 258, 273, 263, 285, 243, 251, 252, 272, 237, 252, 267, 277, 272, 281, 234, 243, 243, 266, 235, 255, 261, 266, 226, 241, 242, 250, 244, 272, 277, 250, 268, 254, 250, 268, 255, 226] payload_rows_per_dst=[5518, 5344, 5406, 5550, 5448, 5488, 5448, 5396] dup_rows_per_dst=[2792, 2734, 2694, 2738, 2642, 2922, 2712, 2704] combine_rows_per_src=[5518, 5344, 5406, 5550, 5448, 5488, 5448, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=229504 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156491904 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312754304 weight_movement_bytes=18874368 total_nvlink_bytes=331628672 weight_to_token_ratio=0.060349 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38310 dup_rows=21672 dup_rows_remote=18782 dedup_ratio=0.330688 router_tokens_per_expert=[245, 263, 257, 265, 285, 246, 276, 256, 232, 231, 262, 256, 277, 243, 271, 236, 268, 272, 263, 259, 239, 253, 270, 265, 263, 263, 264, 243, 262, 251, 261, 218, 264, 263, 270, 236, 293, 247, 237, 247, 269, 270, 274, 242, 259, 288, 233, 267, 238, 248, 239, 255, 256, 271, 247, 255, 257, 260, 255, 253, 292, 265, 248, 247, 262, 288, 258, 287, 254, 278, 246, 259, 254, 244, 265, 263, 258, 275, 244, 253, 238, 229, 276, 273, 260, 237, 264, 241, 266, 265, 260, 261, 242, 271, 262, 246, 272, 230, 257, 261, 251, 260, 237, 281, 247, 238, 257, 244, 256, 243, 247, 234, 264, 252, 229, 271, 240, 251, 238, 233, 250, 273, 261, 248, 250, 245, 236, 273] payload_rows_per_dst=[5554, 5460, 5458, 5410, 5732, 5468, 5304, 5478] dup_rows_per_dst=[2890, 2768, 2636, 2744, 3044, 2714, 2326, 2550] combine_rows_per_src=[5554, 5460, 5458, 5410, 5732, 5468, 5304, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=228368 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157146128 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314063888 weight_movement_bytes=18874368 total_nvlink_bytes=332938256 weight_to_token_ratio=0.060097 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15057153440 prefetch_bytes=811597824 reduce_bytes=811597824 weight_movement_bytes=1623195648 egress_bytes=7534079392 ingress_bytes=9146269696 link_bytes=9146269696 total_nvlink_bytes=16680349088 weight_to_token_ratio=0.107802 prefetch_share_of_ingress=0.088735 dedup_ratio=0.332450 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=43 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081308 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.825730333e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38272 dup_rows=21834 dup_rows_remote=18962 dedup_ratio=0.333160 router_tokens_per_expert=[287, 237, 258, 236, 241, 279, 278, 285, 236, 252, 275, 223, 250, 252, 248, 252, 277, 257, 240, 271, 244, 268, 233, 238, 250, 250, 230, 236, 269, 246, 275, 275, 249, 264, 270, 271, 219, 249, 243, 253, 260, 274, 232, 245, 271, 257, 233, 269, 255, 239, 240, 291, 254, 248, 262, 261, 246, 265, 280, 234, 256, 247, 257, 274, 273, 282, 242, 247, 259, 246, 282, 257, 269, 289, 243, 251, 239, 264, 248, 252, 257, 271, 275, 264, 260, 253, 253, 266, 256, 270, 239, 236, 218, 263, 273, 244, 252, 250, 275, 271, 230, 254, 275, 258, 262, 256, 267, 263, 295, 240, 259, 251, 231, 268, 255, 261, 279, 257, 250, 248, 258, 248, 247, 264, 250, 233, 261, 243] payload_rows_per_dst=[5430, 5664, 5528, 5484, 5398, 5198, 5526, 5474] dup_rows_per_dst=[2872, 2996, 2772, 2738, 2706, 2328, 2790, 2632] combine_rows_per_src=[5430, 5664, 5528, 5484, 5398, 5198, 5526, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228936 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991048 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753160 weight_movement_bytes=18874368 total_nvlink_bytes=332627528 weight_to_token_ratio=0.060157 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15041793384 prefetch_bytes=830472192 reduce_bytes=566231040 weight_movement_bytes=1396703232 egress_bytes=7526387048 ingress_bytes=8912109568 link_bytes=8912109568 total_nvlink_bytes=16438496616 weight_to_token_ratio=0.092855 prefetch_share_of_ingress=0.093185 dedup_ratio=0.332401 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=30 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082248 gemm_inner_pad_upper_bound=109220 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.778988518e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none + [2026-08-29 23:39:07] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 785.4 | throughput per GPU (TFLOP/s/GPU): 113.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.599245E+00 | loss scale: 1.0 | grad norm: 8.715 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38476 dup_rows=21680 dup_rows_remote=18864 dedup_ratio=0.330811 router_tokens_per_expert=[247, 271, 242, 241, 253, 259, 255, 240, 269, 272, 235, 253, 265, 244, 272, 222, 269, 275, 286, 265, 263, 284, 243, 244, 265, 260, 272, 279, 282, 249, 246, 245, 272, 273, 261, 257, 229, 253, 270, 266, 262, 237, 231, 287, 265, 289, 275, 256, 258, 255, 225, 254, 260, 238, 269, 260, 252, 252, 270, 255, 254, 241, 283, 272, 262, 256, 227, 236, 236, 254, 265, 267, 243, 245, 228, 262, 248, 241, 259, 265, 281, 277, 249, 235, 251, 248, 264, 248, 271, 256, 260, 234, 250, 252, 267, 260, 271, 269, 241, 243, 292, 239, 256, 231, 274, 250, 288, 280, 257, 262, 252, 262, 251, 278, 250, 248, 264, 258, 220, 221, 244, 237, 244, 252, 259, 239, 241, 250] payload_rows_per_dst=[5442, 5558, 5526, 5380, 5404, 5562, 5598, 5386] dup_rows_per_dst=[2638, 2896, 2840, 2816, 2584, 2644, 2736, 2526] combine_rows_per_src=[5442, 5558, 5526, 5380, 5404, 5562, 5598, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38254 dup_rows=21788 dup_rows_remote=19088 dedup_ratio=0.332458 router_tokens_per_expert=[210, 255, 248, 266, 239, 268, 248, 248, 262, 255, 268, 231, 260, 286, 261, 289, 249, 253, 276, 231, 282, 252, 277, 271, 234, 271, 259, 249, 215, 268, 256, 253, 258, 238, 234, 245, 274, 250, 269, 246, 242, 232, 274, 272, 266, 262, 254, 231, 268, 247, 272, 243, 259, 294, 268, 254, 252, 257, 239, 251, 272, 269, 238, 225, 273, 261, 242, 234, 238, 252, 261, 254, 248, 268, 276, 256, 259, 251, 264, 260, 259, 269, 242, 237, 244, 252, 259, 286, 256, 235, 255, 255, 241, 236, 259, 263, 254, 223, 264, 283, 256, 254, 262, 274, 277, 255, 260, 218, 231, 248, 296, 268, 283, 262, 251, 264, 245, 283, 271, 251, 250, 257, 261, 257, 236, 263, 250, 271] payload_rows_per_dst=[5466, 5466, 5392, 5500, 5494, 5444, 5466, 5520] dup_rows_per_dst=[2722, 2726, 2702, 2716, 2700, 2652, 2780, 2790] combine_rows_per_src=[5466, 5466, 5392, 5500, 5494, 5444, 5466, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38238 dup_rows=21778 dup_rows_remote=19084 dedup_ratio=0.332306 router_tokens_per_expert=[246, 255, 259, 272, 260, 273, 242, 266, 247, 242, 218, 263, 285, 267, 265, 258, 246, 287, 257, 259, 249, 259, 258, 248, 226, 262, 281, 262, 257, 256, 244, 256, 253, 231, 237, 253, 231, 255, 248, 261, 282, 248, 261, 240, 240, 262, 263, 259, 261, 262, 239, 263, 280, 255, 258, 244, 245, 229, 270, 256, 256, 264, 233, 242, 276, 261, 243, 272, 294, 258, 286, 273, 233, 277, 213, 222, 218, 261, 250, 292, 279, 232, 264, 279, 239, 235, 246, 277, 234, 273, 277, 241, 259, 238, 262, 271, 258, 253, 228, 261, 267, 274, 246, 276, 252, 261, 265, 265, 234, 277, 268, 247, 263, 256, 245, 265, 255, 245, 242, 276, 267, 242, 258, 275, 249, 244, 269, 244] payload_rows_per_dst=[5444, 5520, 5386, 5420, 5532, 5438, 5538, 5480] dup_rows_per_dst=[2792, 2694, 2662, 2694, 2726, 2774, 2726, 2710] combine_rows_per_src=[5444, 5520, 5386, 5420, 5532, 5438, 5538, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38280 dup_rows=21798 dup_rows_remote=19008 dedup_ratio=0.332611 router_tokens_per_expert=[269, 249, 263, 265, 263, 236, 260, 223, 245, 237, 262, 248, 246, 243, 280, 258, 262, 265, 266, 260, 237, 254, 242, 248, 254, 280, 256, 256, 266, 245, 282, 258, 242, 250, 234, 255, 264, 257, 261, 255, 273, 281, 222, 253, 269, 270, 240, 284, 279, 261, 268, 248, 222, 228, 241, 247, 259, 265, 248, 258, 260, 244, 236, 257, 263, 250, 259, 253, 258, 275, 262, 260, 249, 261, 264, 256, 258, 270, 274, 283, 238, 264, 285, 265, 225, 263, 273, 248, 262, 262, 292, 267, 267, 254, 245, 220, 285, 255, 252, 264, 270, 223, 244, 280, 241, 242, 226, 242, 288, 256, 268, 244, 250, 246, 271, 258, 250, 265, 258, 271, 253, 226, 270, 235, 256, 259, 241, 245] payload_rows_per_dst=[5420, 5458, 5440, 5444, 5572, 5468, 5478, 5458] dup_rows_per_dst=[2674, 2734, 2692, 2568, 2952, 2676, 2712, 2790] combine_rows_per_src=[5420, 5458, 5440, 5444, 5572, 5468, 5478, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38406 dup_rows=21780 dup_rows_remote=19048 dedup_ratio=0.332336 router_tokens_per_expert=[266, 290, 268, 245, 263, 254, 273, 253, 251, 263, 232, 295, 273, 244, 244, 257, 263, 232, 235, 235, 256, 236, 252, 259, 236, 251, 266, 264, 235, 254, 261, 243, 242, 263, 262, 273, 225, 260, 255, 228, 254, 265, 258, 256, 266, 255, 256, 261, 257, 281, 224, 263, 266, 283, 285, 245, 259, 251, 263, 269, 265, 279, 275, 255, 250, 261, 236, 242, 237, 242, 240, 267, 264, 233, 283, 265, 272, 235, 243, 278, 240, 236, 247, 275, 262, 265, 265, 247, 272, 266, 267, 239, 259, 276, 283, 276, 262, 252, 260, 280, 257, 234, 233, 231, 251, 252, 257, 245, 291, 250, 236, 250, 265, 260, 248, 257, 279, 239, 220, 238, 262, 255, 236, 252, 253, 275, 255, 262] payload_rows_per_dst=[5514, 5414, 5446, 5598, 5442, 5532, 5350, 5460] dup_rows_per_dst=[2828, 2542, 2712, 2842, 2654, 2818, 2732, 2652] combine_rows_per_src=[5514, 5414, 5446, 5598, 5442, 5532, 5350, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38276 dup_rows=21822 dup_rows_remote=19138 dedup_ratio=0.332977 router_tokens_per_expert=[273, 238, 259, 244, 265, 278, 242, 255, 248, 259, 263, 284, 275, 230, 240, 259, 243, 250, 260, 237, 263, 248, 239, 257, 250, 233, 254, 240, 238, 278, 250, 245, 276, 261, 277, 246, 265, 278, 254, 285, 269, 270, 248, 220, 238, 258, 271, 285, 260, 278, 270, 258, 274, 229, 273, 263, 262, 250, 273, 267, 233, 264, 250, 274, 265, 253, 280, 254, 238, 262, 260, 274, 239, 242, 246, 254, 251, 257, 257, 278, 263, 266, 250, 239, 242, 253, 289, 252, 245, 213, 274, 259, 247, 256, 252, 261, 238, 278, 244, 245, 255, 250, 261, 244, 264, 236, 255, 264, 271, 251, 280, 250, 246, 281, 249, 258, 248, 265, 252, 243, 240, 235, 238, 263, 256, 271, 241, 249] payload_rows_per_dst=[5456, 5422, 5546, 5552, 5424, 5438, 5466, 5410] dup_rows_per_dst=[2768, 2548, 2856, 2804, 2796, 2684, 2706, 2660] combine_rows_per_src=[5456, 5422, 5546, 5552, 5424, 5438, 5466, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38232 dup_rows=21834 dup_rows_remote=19080 dedup_ratio=0.333160 router_tokens_per_expert=[252, 257, 242, 225, 242, 269, 280, 271, 253, 252, 257, 236, 280, 222, 290, 240, 256, 253, 243, 278, 231, 271, 260, 250, 282, 248, 299, 234, 249, 295, 264, 267, 264, 266, 247, 227, 236, 265, 250, 261, 266, 270, 259, 248, 253, 283, 268, 249, 244, 276, 257, 232, 260, 251, 244, 244, 257, 233, 266, 255, 258, 262, 264, 250, 231, 253, 253, 261, 232, 248, 266, 251, 261, 234, 276, 248, 259, 278, 265, 250, 251, 259, 259, 244, 255, 234, 228, 241, 268, 254, 252, 246, 271, 240, 290, 250, 269, 248, 268, 249, 248, 243, 258, 264, 250, 232, 260, 270, 243, 265, 272, 264, 264, 265, 252, 224, 259, 241, 268, 259, 274, 298, 253, 273, 260, 257, 260, 237] payload_rows_per_dst=[5452, 5522, 5470, 5474, 5422, 5410, 5426, 5526] dup_rows_per_dst=[2684, 2838, 2754, 2632, 2710, 2674, 2780, 2762] combine_rows_per_src=[5452, 5522, 5470, 5474, 5422, 5410, 5426, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229360 combine_payload_bytes=157597696 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157827056 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315424752 weight_movement_bytes=18874368 total_nvlink_bytes=334299120 weight_to_token_ratio=0.059838 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229368 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917752 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606136 weight_movement_bytes=18874368 total_nvlink_bytes=332480504 weight_to_token_ratio=0.060185 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229288 combine_payload_bytes=156622848 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156852136 ingress_bytes=194371584 link_bytes=194371584 token_bytes=313474984 weight_movement_bytes=37748736 total_nvlink_bytes=351223720 weight_to_token_ratio=0.120420 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862372881e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229152 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157024032 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313818912 weight_movement_bytes=18874368 total_nvlink_bytes=332693280 weight_to_token_ratio=0.060144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229816 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157540792 ingress_bytes=195059712 link_bytes=195059712 token_bytes=314851768 weight_movement_bytes=37748736 total_nvlink_bytes=352600504 weight_to_token_ratio=0.119894 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868966185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229656 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157008152 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786648 weight_movement_bytes=37748736 total_nvlink_bytes=351535384 weight_to_token_ratio=0.120301 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229248 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827520 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425792 weight_movement_bytes=37748736 total_nvlink_bytes=351174528 weight_to_token_ratio=0.120439 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43460 payload_rows_remote=38058 dup_rows=22076 dup_rows_remote=19438 dedup_ratio=0.336853 router_tokens_per_expert=[248, 266, 262, 259, 243, 260, 260, 278, 280, 265, 257, 227, 239, 281, 283, 263, 251, 278, 264, 266, 224, 255, 221, 257, 264, 240, 275, 237, 235, 257, 256, 282, 276, 239, 275, 264, 260, 221, 245, 246, 277, 254, 266, 231, 237, 274, 265, 250, 248, 250, 221, 244, 231, 253, 266, 267, 237, 280, 255, 263, 236, 256, 257, 256, 251, 247, 256, 240, 226, 240, 252, 292, 238, 264, 269, 248, 259, 249, 266, 255, 261, 243, 257, 272, 263, 288, 267, 238, 272, 256, 245, 233, 269, 229, 278, 271, 249, 272, 246, 303, 265, 256, 243, 224, 278, 249, 242, 277, 269, 261, 231, 250, 293, 269, 270, 242, 236, 248, 265, 233, 278, 271, 258, 262, 241, 240, 264, 256] payload_rows_per_dst=[5518, 5370, 5418, 5402, 5378, 5444, 5450, 5480] dup_rows_per_dst=[2824, 2754, 2742, 2638, 2726, 2840, 2780, 2772] combine_rows_per_src=[5518, 5370, 5418, 5402, 5378, 5444, 5450, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38128 dup_rows=21912 dup_rows_remote=18970 dedup_ratio=0.334351 router_tokens_per_expert=[276, 262, 229, 261, 270, 257, 245, 263, 235, 264, 267, 256, 253, 250, 247, 228, 271, 257, 255, 295, 257, 296, 258, 270, 260, 270, 246, 261, 268, 249, 253, 253, 257, 262, 265, 258, 263, 248, 273, 245, 265, 239, 238, 261, 279, 260, 268, 255, 243, 269, 238, 245, 255, 246, 253, 253, 270, 269, 226, 274, 273, 245, 260, 242, 261, 256, 238, 242, 223, 290, 271, 243, 245, 249, 250, 247, 259, 247, 236, 243, 247, 259, 261, 261, 242, 233, 247, 273, 274, 261, 235, 250, 254, 257, 245, 275, 261, 257, 258, 266, 252, 292, 246, 270, 265, 264, 248, 248, 239, 250, 253, 228, 282, 246, 269, 263, 239, 260, 224, 227, 233, 262, 262, 262, 272, 279, 261, 277] payload_rows_per_dst=[5456, 5496, 5510, 5454, 5376, 5454, 5412, 5466] dup_rows_per_dst=[2670, 2942, 2762, 2668, 2624, 2694, 2782, 2770] combine_rows_per_src=[5456, 5496, 5510, 5454, 5376, 5454, 5412, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38218 dup_rows=21778 dup_rows_remote=19090 dedup_ratio=0.332306 router_tokens_per_expert=[246, 293, 264, 258, 230, 242, 274, 263, 276, 234, 249, 232, 259, 263, 288, 261, 236, 246, 242, 258, 274, 283, 227, 290, 246, 267, 252, 246, 238, 248, 261, 228, 261, 232, 241, 241, 236, 263, 292, 264, 241, 244, 248, 254, 247, 261, 274, 258, 272, 231, 286, 245, 237, 230, 247, 222, 256, 261, 278, 285, 255, 263, 297, 279, 256, 257, 253, 279, 262, 262, 264, 288, 212, 251, 255, 275, 260, 248, 243, 249, 248, 230, 244, 247, 260, 259, 257, 260, 247, 261, 257, 270, 257, 276, 264, 250, 264, 241, 247, 257, 207, 237, 281, 258, 256, 236, 236, 231, 273, 249, 261, 263, 254, 263, 259, 258, 240, 255, 269, 295, 251, 270, 265, 277, 286, 264, 245, 244] payload_rows_per_dst=[5416, 5376, 5520, 5564, 5540, 5472, 5362, 5508] dup_rows_per_dst=[2848, 2708, 2594, 2724, 2688, 2702, 2632, 2882] combine_rows_per_src=[5416, 5376, 5520, 5564, 5540, 5472, 5362, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38098 dup_rows=21904 dup_rows_remote=19090 dedup_ratio=0.334229 router_tokens_per_expert=[263, 272, 259, 253, 247, 284, 252, 247, 268, 238, 250, 276, 247, 249, 252, 256, 268, 249, 284, 255, 277, 250, 243, 238, 289, 240, 272, 270, 251, 259, 256, 284, 251, 253, 201, 262, 249, 260, 255, 255, 304, 225, 250, 280, 259, 262, 227, 233, 256, 253, 250, 253, 253, 255, 243, 228, 236, 272, 262, 256, 277, 253, 270, 265, 237, 227, 246, 270, 244, 231, 295, 248, 266, 258, 243, 272, 222, 263, 262, 267, 266, 291, 250, 259, 262, 226, 274, 234, 255, 259, 254, 275, 239, 256, 237, 260, 265, 246, 240, 257, 258, 264, 245, 265, 253, 262, 248, 266, 239, 251, 250, 263, 258, 273, 257, 260, 249, 274, 260, 243, 248, 262, 253, 265, 263, 260, 266, 251] payload_rows_per_dst=[5274, 5272, 5404, 5638, 5444, 5504, 5562, 5534] dup_rows_per_dst=[2408, 2568, 2648, 3070, 2658, 2840, 2898, 2814] combine_rows_per_src=[5274, 5272, 5404, 5638, 5444, 5504, 5562, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38292 dup_rows=21724 dup_rows_remote=19140 dedup_ratio=0.331482 router_tokens_per_expert=[230, 260, 251, 293, 280, 272, 241, 263, 275, 247, 230, 255, 271, 238, 254, 285, 253, 255, 246, 272, 243, 268, 271, 256, 247, 268, 283, 275, 242, 256, 279, 238, 263, 261, 246, 254, 259, 247, 247, 270, 243, 228, 245, 284, 271, 264, 266, 250, 236, 243, 247, 265, 257, 228, 268, 249, 285, 231, 257, 258, 235, 237, 265, 246, 265, 265, 254, 259, 265, 262, 253, 262, 252, 263, 232, 250, 255, 261, 254, 245, 231, 247, 261, 275, 240, 275, 288, 238, 232, 265, 247, 275, 277, 249, 236, 261, 246, 245, 276, 262, 240, 257, 265, 253, 255, 240, 256, 233, 261, 253, 264, 246, 260, 297, 225, 265, 252, 238, 283, 259, 242, 284, 242, 257, 255, 240, 242, 279] payload_rows_per_dst=[5326, 5518, 5502, 5568, 5434, 5440, 5520, 5504] dup_rows_per_dst=[2620, 2786, 2694, 2790, 2760, 2754, 2584, 2736] combine_rows_per_src=[5326, 5518, 5502, 5568, 5434, 5440, 5520, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=229984 combine_payload_bytes=155885568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156115552 ingress_bytes=174759936 link_bytes=174759936 token_bytes=312001120 weight_movement_bytes=18874368 total_nvlink_bytes=330875488 weight_to_token_ratio=0.060495 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674463719e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38324 dup_rows=21852 dup_rows_remote=19082 dedup_ratio=0.333435 router_tokens_per_expert=[259, 271, 292, 273, 256, 255, 257, 242, 286, 251, 273, 231, 300, 249, 271, 265, 241, 278, 274, 248, 256, 252, 262, 251, 251, 262, 260, 265, 269, 248, 258, 263, 281, 268, 238, 255, 264, 259, 258, 246, 222, 238, 269, 244, 254, 269, 290, 237, 242, 268, 252, 237, 248, 249, 251, 237, 236, 253, 242, 247, 260, 250, 245, 260, 243, 248, 262, 244, 253, 259, 254, 260, 246, 254, 280, 239, 245, 236, 262, 245, 241, 253, 246, 224, 251, 260, 262, 278, 249, 249, 232, 260, 274, 266, 270, 250, 249, 286, 271, 268, 259, 247, 255, 272, 256, 264, 256, 254, 256, 252, 232, 237, 255, 246, 256, 282, 257, 262, 228, 238, 227, 260, 251, 269, 274, 297, 252, 267] payload_rows_per_dst=[5560, 5522, 5454, 5344, 5458, 5360, 5468, 5518] dup_rows_per_dst=[2902, 2754, 2730, 2610, 2602, 2770, 2760, 2724] combine_rows_per_src=[5560, 5522, 5454, 5344, 5458, 5360, 5468, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38268 dup_rows=21732 dup_rows_remote=18996 dedup_ratio=0.331604 router_tokens_per_expert=[259, 240, 264, 240, 261, 272, 274, 265, 244, 254, 259, 268, 250, 246, 289, 223, 262, 241, 263, 251, 266, 252, 245, 265, 267, 258, 252, 251, 258, 260, 269, 280, 244, 269, 274, 266, 248, 270, 251, 247, 260, 249, 270, 254, 233, 261, 265, 275, 269, 254, 243, 273, 245, 268, 266, 249, 254, 255, 241, 247, 230, 242, 214, 255, 280, 236, 253, 253, 250, 240, 260, 261, 260, 245, 279, 256, 250, 237, 244, 276, 280, 269, 246, 228, 259, 236, 264, 256, 243, 270, 232, 263, 236, 271, 261, 246, 253, 258, 265, 230, 247, 245, 255, 245, 246, 264, 238, 275, 282, 243, 267, 255, 255, 258, 268, 297, 259, 285, 263, 265, 229, 268, 261, 208, 243, 288, 256, 268] payload_rows_per_dst=[5464, 5452, 5536, 5356, 5500, 5462, 5442, 5592] dup_rows_per_dst=[2752, 2828, 2736, 2654, 2660, 2658, 2694, 2750] combine_rows_per_src=[5464, 5452, 5536, 5356, 5500, 5462, 5442, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=228392 combine_payload_bytes=156172288 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156400680 ingress_bytes=212795392 link_bytes=212795392 token_bytes=312572968 weight_movement_bytes=56623104 total_nvlink_bytes=369196072 weight_to_token_ratio=0.181152 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038900744e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229232 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770160 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311088 weight_movement_bytes=18874368 total_nvlink_bytes=332185456 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228752 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156278160 ingress_bytes=231546880 link_bytes=231546880 token_bytes=312327568 weight_movement_bytes=75497472 total_nvlink_bytes=387825040 weight_to_token_ratio=0.241725 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.218568276e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229728 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073760 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917792 weight_movement_bytes=18874368 total_nvlink_bytes=332792160 weight_to_token_ratio=0.060125 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229624 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204728 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179832 weight_movement_bytes=18874368 total_nvlink_bytes=333054200 weight_to_token_ratio=0.060075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229056 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974784 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720512 weight_movement_bytes=18874368 total_nvlink_bytes=332594880 weight_to_token_ratio=0.060163 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38442 dup_rows=21732 dup_rows_remote=19186 dedup_ratio=0.331604 router_tokens_per_expert=[282, 258, 286, 267, 253, 260, 262, 229, 277, 261, 232, 273, 282, 247, 260, 243, 256, 216, 256, 272, 249, 257, 262, 256, 244, 249, 232, 285, 252, 252, 257, 230, 219, 272, 265, 239, 266, 239, 263, 240, 238, 289, 259, 240, 260, 267, 267, 250, 270, 246, 255, 239, 248, 259, 225, 249, 245, 246, 270, 260, 238, 243, 240, 221, 267, 258, 244, 263, 255, 252, 268, 266, 251, 270, 263, 265, 236, 274, 270, 238, 236, 293, 271, 283, 271, 275, 281, 245, 273, 232, 273, 248, 253, 249, 234, 255, 286, 272, 238, 238, 253, 261, 258, 267, 226, 255, 259, 240, 286, 246, 275, 252, 234, 254, 254, 255, 251, 268, 266, 236, 278, 269, 273, 253, 265, 239, 253, 272] payload_rows_per_dst=[5566, 5436, 5466, 5362, 5548, 5406, 5554, 5466] dup_rows_per_dst=[2778, 2614, 2680, 2546, 2732, 2938, 2670, 2774] combine_rows_per_src=[5566, 5436, 5466, 5362, 5548, 5406, 5554, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38384 dup_rows=21662 dup_rows_remote=18916 dedup_ratio=0.330536 router_tokens_per_expert=[223, 234, 271, 265, 258, 252, 240, 227, 238, 281, 241, 270, 245, 232, 263, 251, 248, 265, 269, 266, 245, 243, 284, 241, 240, 258, 260, 275, 254, 257, 245, 249, 238, 248, 252, 259, 250, 263, 249, 246, 259, 269, 282, 262, 230, 238, 254, 257, 251, 258, 259, 249, 243, 276, 250, 248, 230, 237, 260, 246, 253, 254, 283, 251, 263, 239, 254, 246, 237, 246, 268, 271, 223, 231, 281, 267, 272, 252, 247, 265, 256, 258, 237, 272, 253, 266, 254, 251, 273, 261, 293, 256, 274, 250, 265, 242, 271, 270, 266, 234, 264, 269, 289, 271, 235, 249, 278, 266, 243, 216, 239, 288, 228, 278, 288, 281, 281, 252, 286, 260, 260, 244, 251, 249, 247, 262, 263, 273] payload_rows_per_dst=[5490, 5494, 5440, 5410, 5468, 5566, 5442, 5564] dup_rows_per_dst=[2746, 2704, 2672, 2686, 2656, 2756, 2600, 2842] combine_rows_per_src=[5490, 5494, 5440, 5410, 5468, 5566, 5442, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38088 dup_rows=21886 dup_rows_remote=19122 dedup_ratio=0.333954 router_tokens_per_expert=[263, 256, 262, 241, 259, 266, 292, 264, 289, 257, 269, 228, 275, 258, 242, 292, 267, 229, 264, 254, 257, 267, 266, 252, 270, 260, 268, 264, 231, 301, 237, 276, 221, 248, 247, 275, 259, 269, 254, 267, 247, 237, 271, 247, 252, 262, 232, 254, 213, 268, 284, 265, 254, 225, 266, 286, 258, 247, 237, 268, 220, 255, 238, 241, 243, 247, 255, 262, 221, 265, 254, 264, 243, 258, 261, 283, 278, 263, 252, 290, 227, 269, 237, 241, 250, 239, 261, 260, 243, 254, 271, 258, 281, 254, 260, 238, 217, 229, 256, 248, 264, 253, 244, 257, 248, 258, 266, 263, 256, 249, 241, 253, 265, 257, 239, 261, 250, 276, 282, 258, 282, 243, 236, 258, 263, 253, 254, 264] payload_rows_per_dst=[5552, 5562, 5464, 5394, 5438, 5426, 5348, 5466] dup_rows_per_dst=[2874, 2764, 2620, 2656, 2840, 2660, 2656, 2816] combine_rows_per_src=[5552, 5562, 5464, 5394, 5438, 5426, 5348, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=230512 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157688944 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315147376 weight_movement_bytes=18874368 total_nvlink_bytes=334021744 weight_to_token_ratio=0.059891 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38298 dup_rows=21790 dup_rows_remote=19034 dedup_ratio=0.332489 router_tokens_per_expert=[252, 247, 271, 257, 286, 226, 269, 301, 237, 254, 249, 254, 248, 244, 297, 222, 264, 270, 239, 231, 257, 253, 261, 278, 241, 270, 255, 294, 252, 234, 272, 276, 243, 278, 253, 246, 264, 242, 268, 250, 257, 237, 281, 263, 248, 263, 257, 260, 276, 252, 253, 250, 279, 258, 286, 236, 226, 245, 264, 232, 255, 262, 256, 268, 264, 262, 241, 245, 226, 249, 234, 301, 256, 252, 260, 272, 243, 221, 285, 272, 252, 243, 255, 225, 252, 247, 235, 243, 244, 247, 246, 254, 256, 253, 245, 262, 261, 277, 262, 242, 280, 259, 258, 256, 236, 240, 254, 257, 268, 254, 284, 267, 249, 264, 256, 279, 273, 242, 245, 280, 258, 237, 257, 242, 259, 257, 243, 261] payload_rows_per_dst=[5254, 5360, 5478, 5548, 5448, 5600, 5610, 5448] dup_rows_per_dst=[2410, 2456, 2742, 2970, 2718, 2882, 2856, 2756] combine_rows_per_src=[5254, 5360, 5478, 5548, 5448, 5600, 5610, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38526 dup_rows=21610 dup_rows_remote=19082 dedup_ratio=0.329742 router_tokens_per_expert=[250, 240, 211, 229, 244, 232, 253, 254, 281, 247, 258, 260, 255, 269, 232, 274, 251, 296, 253, 266, 256, 264, 280, 257, 270, 267, 285, 254, 271, 244, 260, 240, 258, 257, 244, 277, 250, 263, 262, 258, 280, 253, 247, 250, 239, 263, 256, 254, 235, 243, 241, 257, 267, 284, 270, 243, 264, 260, 270, 253, 264, 264, 246, 240, 266, 251, 274, 298, 261, 265, 265, 235, 241, 258, 285, 264, 260, 239, 279, 246, 239, 278, 247, 244, 273, 251, 260, 259, 233, 257, 248, 278, 272, 260, 255, 256, 234, 231, 247, 238, 239, 263, 273, 240, 253, 245, 267, 248, 261, 279, 232, 214, 249, 247, 289, 242, 261, 260, 233, 268, 242, 253, 260, 263, 258, 260, 238, 269] payload_rows_per_dst=[5420, 5568, 5516, 5562, 5558, 5498, 5400, 5404] dup_rows_per_dst=[2558, 2860, 2706, 2640, 2816, 2722, 2528, 2780] combine_rows_per_src=[5420, 5568, 5516, 5562, 5558, 5498, 5400, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38264 dup_rows=21824 dup_rows_remote=19294 dedup_ratio=0.333008 router_tokens_per_expert=[253, 242, 244, 264, 239, 266, 252, 253, 270, 260, 234, 227, 274, 262, 280, 260, 236, 238, 258, 271, 256, 241, 264, 241, 278, 241, 262, 260, 271, 231, 267, 281, 259, 271, 264, 289, 253, 275, 237, 243, 258, 234, 248, 274, 236, 271, 280, 233, 257, 241, 240, 255, 280, 245, 267, 249, 259, 266, 250, 249, 274, 253, 281, 251, 255, 234, 243, 264, 246, 229, 260, 262, 229, 250, 269, 266, 238, 265, 216, 263, 234, 270, 238, 254, 221, 246, 263, 274, 255, 263, 265, 274, 240, 265, 253, 238, 261, 255, 287, 220, 238, 275, 239, 261, 237, 245, 273, 257, 284, 236, 282, 285, 264, 252, 282, 255, 270, 245, 257, 266, 247, 275, 243, 246, 239, 259, 287, 286] payload_rows_per_dst=[5398, 5482, 5378, 5420, 5448, 5472, 5554, 5560] dup_rows_per_dst=[2762, 2710, 2872, 2814, 2530, 2634, 2716, 2786] combine_rows_per_src=[5398, 5482, 5378, 5420, 5448, 5472, 5554, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38374 dup_rows=21814 dup_rows_remote=19034 dedup_ratio=0.332855 router_tokens_per_expert=[281, 250, 234, 270, 271, 247, 269, 270, 246, 248, 277, 247, 239, 253, 290, 253, 251, 249, 273, 266, 266, 265, 260, 274, 274, 278, 268, 256, 258, 262, 226, 231, 227, 260, 238, 265, 266, 266, 264, 234, 265, 246, 243, 234, 238, 294, 259, 269, 232, 252, 247, 272, 257, 273, 260, 250, 263, 238, 252, 257, 263, 290, 253, 242, 267, 257, 260, 237, 247, 255, 240, 288, 273, 247, 241, 252, 257, 259, 283, 230, 242, 253, 247, 259, 224, 250, 272, 239, 242, 273, 248, 249, 277, 254, 261, 274, 273, 250, 243, 306, 269, 265, 262, 267, 277, 269, 248, 290, 224, 248, 244, 252, 245, 240, 281, 204, 238, 248, 250, 229, 227, 263, 253, 262, 259, 243, 262, 249] payload_rows_per_dst=[5534, 5508, 5386, 5466, 5484, 5348, 5586, 5410] dup_rows_per_dst=[2756, 2806, 2750, 2736, 2702, 2780, 2788, 2496] combine_rows_per_src=[5534, 5508, 5386, 5466, 5484, 5348, 5586, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229200 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157450064 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314670928 weight_movement_bytes=18874368 total_nvlink_bytes=333545296 weight_to_token_ratio=0.059981 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38450 dup_rows=21684 dup_rows_remote=19044 dedup_ratio=0.330872 router_tokens_per_expert=[276, 278, 246, 221, 266, 240, 271, 244, 250, 265, 256, 261, 271, 260, 298, 245, 243, 279, 235, 256, 228, 267, 267, 255, 252, 246, 300, 280, 255, 255, 311, 258, 249, 223, 255, 284, 245, 251, 249, 248, 271, 247, 262, 246, 245, 250, 244, 252, 242, 268, 240, 257, 277, 266, 232, 261, 268, 254, 265, 215, 264, 234, 249, 269, 259, 256, 267, 254, 259, 268, 250, 241, 282, 254, 272, 253, 261, 220, 240, 234, 250, 249, 262, 275, 233, 239, 274, 275, 256, 260, 253, 235, 233, 233, 253, 253, 269, 282, 285, 258, 269, 255, 254, 269, 256, 277, 239, 250, 224, 253, 233, 266, 255, 258, 270, 239, 262, 264, 257, 258, 278, 241, 264, 232, 268, 250, 278, 235] payload_rows_per_dst=[5528, 5530, 5402, 5472, 5554, 5432, 5508, 5426] dup_rows_per_dst=[2768, 2844, 2640, 2650, 2586, 2634, 2770, 2792] combine_rows_per_src=[5528, 5530, 5402, 5472, 5554, 5432, 5508, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228840 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156237288 ingress_bytes=212631552 link_bytes=212631552 token_bytes=312245736 weight_movement_bytes=56623104 total_nvlink_bytes=368868840 weight_to_token_ratio=0.181341 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037330910e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229328 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=0 egress_bytes=157097936 ingress_bytes=156868608 link_bytes=157097936 token_bytes=313966544 weight_movement_bytes=0 total_nvlink_bytes=313966544 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.505235126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157802496 dispatch_meta_bytes=230432 combine_payload_bytes=157802496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158032928 ingress_bytes=195551232 link_bytes=195551232 token_bytes=315835424 weight_movement_bytes=37748736 total_nvlink_bytes=353584160 weight_to_token_ratio=0.119520 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873675688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=230232 combine_payload_bytes=156729344 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156959576 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313688920 weight_movement_bytes=37748736 total_nvlink_bytes=351437656 weight_to_token_ratio=0.120338 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229632 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409536 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589440 weight_movement_bytes=18874368 total_nvlink_bytes=333463808 weight_to_token_ratio=0.059997 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=229976 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157721176 ingress_bytes=176365568 link_bytes=176365568 token_bytes=315212376 weight_movement_bytes=18874368 total_nvlink_bytes=334086744 weight_to_token_ratio=0.059878 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689848095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38214 dup_rows=21816 dup_rows_remote=18988 dedup_ratio=0.332886 router_tokens_per_expert=[274, 275, 251, 264, 245, 266, 241, 256, 282, 243, 241, 255, 228, 242, 257, 240, 266, 239, 267, 267, 252, 257, 242, 274, 246, 250, 267, 274, 247, 241, 274, 255, 266, 251, 270, 265, 257, 232, 256, 267, 272, 285, 272, 235, 253, 230, 251, 241, 245, 229, 273, 294, 257, 235, 251, 276, 276, 236, 279, 254, 252, 267, 291, 252, 272, 246, 245, 272, 282, 302, 244, 272, 271, 269, 237, 256, 283, 238, 262, 246, 276, 255, 248, 250, 283, 221, 273, 250, 269, 263, 248, 203, 239, 247, 241, 250, 244, 256, 245, 244, 262, 273, 250, 278, 250, 211, 266, 259, 256, 252, 246, 249, 259, 275, 268, 242, 270, 267, 252, 267, 221, 267, 215, 250, 274, 250, 228, 261] payload_rows_per_dst=[5440, 5522, 5480, 5506, 5550, 5350, 5370, 5502] dup_rows_per_dst=[2680, 2714, 2726, 2828, 2844, 2682, 2712, 2630] combine_rows_per_src=[5440, 5522, 5480, 5506, 5550, 5350, 5370, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=228808 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156753352 ingress_bytes=194273280 link_bytes=194273280 token_bytes=313277896 weight_movement_bytes=37748736 total_nvlink_bytes=351026632 weight_to_token_ratio=0.120496 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861430981e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38294 dup_rows=21648 dup_rows_remote=18814 dedup_ratio=0.330322 router_tokens_per_expert=[241, 243, 241, 224, 228, 252, 265, 232, 263, 249, 265, 269, 288, 269, 252, 298, 279, 292, 246, 249, 259, 281, 265, 278, 240, 297, 257, 278, 240, 257, 238, 258, 255, 258, 263, 266, 223, 270, 253, 242, 258, 272, 268, 256, 269, 264, 299, 279, 240, 248, 257, 245, 260, 258, 280, 264, 238, 224, 266, 284, 223, 252, 273, 260, 270, 268, 262, 248, 218, 232, 236, 258, 226, 294, 260, 259, 250, 273, 262, 235, 286, 257, 234, 260, 271, 262, 255, 236, 272, 251, 246, 235, 253, 270, 239, 239, 226, 260, 269, 255, 264, 238, 274, 271, 250, 266, 250, 225, 251, 277, 269, 235, 246, 256, 248, 265, 264, 235, 266, 239, 259, 248, 242, 238, 259, 230, 264, 252] payload_rows_per_dst=[5526, 5594, 5540, 5470, 5432, 5498, 5422, 5406] dup_rows_per_dst=[2632, 2834, 2850, 2674, 2670, 2634, 2738, 2616] combine_rows_per_src=[5526, 5594, 5540, 5470, 5432, 5498, 5422, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38238 dup_rows=21714 dup_rows_remote=18880 dedup_ratio=0.331329 router_tokens_per_expert=[235, 259, 292, 240, 260, 270, 245, 240, 271, 250, 261, 240, 249, 262, 258, 275, 277, 257, 252, 271, 266, 239, 242, 249, 237, 254, 274, 274, 247, 271, 299, 266, 248, 253, 280, 232, 280, 247, 268, 250, 265, 268, 254, 285, 248, 241, 265, 235, 283, 267, 273, 249, 266, 273, 265, 232, 255, 254, 278, 269, 263, 244, 260, 236, 264, 253, 229, 255, 233, 270, 263, 280, 233, 247, 253, 264, 233, 228, 250, 257, 258, 251, 247, 254, 272, 239, 249, 262, 254, 263, 245, 265, 275, 233, 240, 261, 240, 237, 269, 239, 251, 268, 239, 278, 242, 270, 265, 263, 254, 263, 243, 277, 267, 249, 256, 241, 241, 256, 261, 244, 232, 255, 285, 267, 240, 253, 246, 229] payload_rows_per_dst=[5468, 5472, 5228, 5544, 5402, 5558, 5566, 5584] dup_rows_per_dst=[2746, 2546, 2394, 2816, 2664, 2820, 2894, 2834] combine_rows_per_src=[5468, 5472, 5228, 5544, 5402, 5558, 5566, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38118 dup_rows=21904 dup_rows_remote=19088 dedup_ratio=0.334229 router_tokens_per_expert=[256, 248, 249, 232, 255, 271, 251, 248, 260, 248, 246, 243, 251, 250, 235, 266, 253, 267, 247, 263, 241, 234, 254, 291, 270, 264, 271, 256, 262, 290, 235, 246, 230, 251, 230, 255, 246, 242, 263, 273, 244, 261, 244, 243, 263, 271, 240, 257, 244, 249, 288, 266, 260, 241, 282, 257, 253, 264, 240, 277, 277, 258, 267, 258, 252, 253, 272, 262, 261, 265, 256, 263, 258, 255, 233, 258, 270, 248, 267, 254, 249, 288, 261, 271, 243, 278, 230, 279, 243, 246, 238, 230, 261, 240, 242, 237, 265, 271, 262, 242, 256, 252, 280, 280, 249, 277, 260, 256, 235, 280, 254, 246, 239, 229, 235, 258, 266, 280, 261, 256, 243, 257, 295, 232, 250, 256, 275, 261] payload_rows_per_dst=[5450, 5480, 5386, 5582, 5456, 5346, 5514, 5418] dup_rows_per_dst=[2568, 2808, 2640, 2780, 2798, 2726, 2816, 2768] combine_rows_per_src=[5450, 5480, 5386, 5582, 5456, 5346, 5514, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38364 dup_rows=21784 dup_rows_remote=19124 dedup_ratio=0.332397 router_tokens_per_expert=[250, 260, 251, 262, 259, 252, 278, 280, 265, 252, 263, 263, 265, 257, 271, 267, 244, 275, 243, 255, 237, 252, 254, 257, 246, 271, 258, 245, 254, 253, 243, 235, 232, 235, 276, 278, 263, 258, 260, 264, 245, 253, 272, 274, 245, 277, 285, 272, 258, 260, 256, 236, 225, 268, 222, 268, 272, 230, 254, 253, 258, 256, 263, 265, 292, 266, 247, 247, 255, 242, 235, 254, 238, 239, 257, 231, 236, 278, 236, 248, 246, 234, 274, 242, 284, 252, 235, 224, 253, 272, 239, 271, 245, 236, 261, 256, 230, 282, 255, 273, 278, 225, 232, 248, 262, 254, 259, 241, 266, 257, 268, 257, 243, 265, 283, 254, 260, 277, 290, 255, 283, 228, 259, 242, 282, 230, 258, 297] payload_rows_per_dst=[5582, 5364, 5572, 5444, 5388, 5388, 5476, 5538] dup_rows_per_dst=[2808, 2680, 2806, 2644, 2614, 2660, 2698, 2874] combine_rows_per_src=[5582, 5364, 5572, 5444, 5388, 5388, 5476, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38248 dup_rows=21878 dup_rows_remote=19112 dedup_ratio=0.333832 router_tokens_per_expert=[255, 236, 255, 262, 247, 287, 245, 281, 270, 271, 256, 259, 264, 256, 272, 268, 248, 253, 246, 239, 251, 267, 249, 269, 257, 238, 277, 252, 291, 262, 227, 255, 262, 255, 243, 284, 261, 260, 260, 285, 236, 235, 260, 245, 239, 234, 259, 248, 234, 237, 225, 255, 254, 218, 259, 245, 232, 249, 265, 257, 273, 267, 276, 297, 268, 265, 265, 254, 248, 240, 288, 269, 240, 237, 238, 254, 241, 268, 275, 238, 220, 246, 276, 256, 249, 283, 260, 259, 285, 256, 279, 269, 265, 241, 277, 265, 270, 241, 234, 248, 264, 258, 255, 294, 248, 246, 283, 261, 231, 232, 241, 248, 244, 274, 259, 247, 260, 277, 241, 257, 262, 252, 237, 256, 263, 261, 229, 247] payload_rows_per_dst=[5466, 5492, 5468, 5390, 5410, 5508, 5460, 5464] dup_rows_per_dst=[2902, 2670, 2664, 2696, 2766, 2864, 2648, 2668] combine_rows_per_src=[5466, 5492, 5468, 5390, 5410, 5508, 5460, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38082 dup_rows=21876 dup_rows_remote=18956 dedup_ratio=0.333801 router_tokens_per_expert=[245, 276, 280, 264, 235, 259, 262, 261, 265, 272, 230, 254, 255, 266, 252, 270, 256, 246, 261, 264, 254, 265, 237, 264, 235, 274, 274, 254, 253, 241, 247, 270, 257, 240, 245, 258, 236, 270, 258, 243, 270, 279, 232, 259, 282, 249, 257, 240, 282, 260, 272, 245, 243, 251, 241, 236, 234, 258, 265, 263, 249, 239, 243, 247, 273, 272, 219, 269, 262, 272, 274, 240, 248, 257, 255, 262, 285, 250, 270, 250, 249, 231, 231, 263, 233, 260, 270, 278, 261, 248, 246, 235, 286, 269, 239, 272, 271, 242, 230, 257, 265, 271, 246, 267, 244, 264, 249, 275, 237, 258, 255, 264, 272, 256, 261, 259, 248, 237, 299, 247, 280, 233, 244, 225, 248, 246, 276, 269] payload_rows_per_dst=[5578, 5480, 5468, 5406, 5656, 5482, 5490, 5100] dup_rows_per_dst=[2920, 2710, 2818, 2650, 3032, 2660, 2700, 2386] combine_rows_per_src=[5578, 5480, 5468, 5406, 5656, 5482, 5490, 5100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38200 dup_rows=21846 dup_rows_remote=19088 dedup_ratio=0.333344 router_tokens_per_expert=[274, 276, 275, 232, 228, 243, 269, 248, 262, 219, 268, 238, 247, 267, 234, 235, 265, 250, 257, 264, 277, 261, 241, 269, 257, 253, 238, 250, 260, 246, 229, 288, 255, 255, 265, 274, 252, 251, 250, 275, 252, 251, 241, 267, 260, 238, 266, 272, 237, 260, 237, 271, 272, 272, 255, 262, 235, 260, 260, 273, 240, 251, 252, 225, 267, 250, 275, 245, 252, 286, 277, 248, 247, 257, 264, 256, 248, 256, 243, 268, 259, 263, 259, 276, 246, 252, 255, 294, 264, 266, 289, 282, 233, 258, 269, 258, 243, 237, 254, 278, 228, 252, 267, 272, 243, 278, 225, 247, 275, 291, 218, 259, 251, 233, 266, 257, 267, 264, 254, 241, 258, 260, 259, 244, 247, 252, 245, 235] payload_rows_per_dst=[5390, 5514, 5490, 5466, 5480, 5516, 5398, 5436] dup_rows_per_dst=[2640, 2696, 2758, 2658, 2798, 2930, 2736, 2630] combine_rows_per_src=[5390, 5514, 5490, 5466, 5480, 5516, 5398, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228432 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157080656 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313932880 weight_movement_bytes=56623104 total_nvlink_bytes=370555984 weight_to_token_ratio=0.180367 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228472 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156851320 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474168 weight_movement_bytes=18874368 total_nvlink_bytes=332348536 weight_to_token_ratio=0.060210 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228824 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156360152 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312491480 weight_movement_bytes=18874368 total_nvlink_bytes=331365848 weight_to_token_ratio=0.060400 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229952 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157368896 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314507840 weight_movement_bytes=18874368 total_nvlink_bytes=333382208 weight_to_token_ratio=0.060012 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229440 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156893248 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313557056 weight_movement_bytes=37748736 total_nvlink_bytes=351305792 weight_to_token_ratio=0.120389 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38404 dup_rows=21656 dup_rows_remote=18966 dedup_ratio=0.330444 router_tokens_per_expert=[255, 257, 260, 284, 251, 228, 249, 245, 273, 277, 261, 241, 266, 254, 240, 247, 280, 253, 255, 263, 233, 271, 268, 239, 256, 259, 225, 260, 241, 259, 296, 240, 255, 240, 248, 254, 263, 278, 292, 246, 242, 229, 251, 279, 273, 246, 279, 283, 244, 260, 274, 269, 249, 207, 292, 271, 247, 241, 254, 259, 254, 258, 249, 255, 233, 268, 276, 251, 248, 264, 254, 224, 263, 254, 280, 257, 248, 261, 265, 277, 251, 253, 245, 249, 279, 253, 249, 248, 266, 259, 266, 272, 265, 264, 262, 233, 261, 260, 271, 264, 241, 272, 240, 246, 243, 234, 260, 268, 266, 239, 230, 248, 245, 240, 248, 248, 279, 238, 263, 264, 273, 263, 232, 279, 253, 248, 249, 239] payload_rows_per_dst=[5466, 5498, 5524, 5476, 5504, 5454, 5474, 5484] dup_rows_per_dst=[2710, 2698, 2792, 2690, 2742, 2774, 2612, 2638] combine_rows_per_src=[5466, 5498, 5524, 5476, 5504, 5454, 5474, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=228152 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156212024 ingress_bytes=193732608 link_bytes=193732608 token_bytes=312195896 weight_movement_bytes=37748736 total_nvlink_bytes=349944632 weight_to_token_ratio=0.120914 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856250527e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229152 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156696352 ingress_bytes=213090304 link_bytes=213090304 token_bytes=313163552 weight_movement_bytes=56623104 total_nvlink_bytes=369786656 weight_to_token_ratio=0.180810 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041726446e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229480 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157532264 ingress_bytes=195051520 link_bytes=195051520 token_bytes=314835048 weight_movement_bytes=37748736 total_nvlink_bytes=352583784 weight_to_token_ratio=0.119900 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868887693e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38098 dup_rows=21892 dup_rows_remote=18944 dedup_ratio=0.334045 router_tokens_per_expert=[269, 239, 266, 270, 239, 246, 241, 255, 248, 257, 259, 296, 259, 261, 233, 250, 253, 266, 249, 254, 257, 239, 270, 247, 249, 293, 316, 254, 288, 244, 283, 285, 248, 260, 251, 246, 262, 280, 270, 249, 254, 255, 264, 292, 242, 238, 256, 265, 257, 265, 259, 225, 254, 250, 263, 254, 265, 265, 255, 269, 239, 246, 233, 267, 241, 266, 265, 277, 273, 266, 219, 241, 241, 266, 248, 236, 277, 289, 249, 234, 284, 255, 263, 247, 260, 230, 256, 235, 257, 269, 255, 237, 248, 259, 254, 240, 264, 260, 237, 276, 253, 262, 264, 233, 242, 253, 243, 240, 237, 262, 235, 256, 278, 249, 254, 238, 241, 271, 232, 272, 253, 247, 272, 261, 258, 260, 249, 246] payload_rows_per_dst=[5460, 5546, 5458, 5402, 5440, 5460, 5426, 5452] dup_rows_per_dst=[2716, 2948, 2806, 2730, 2736, 2638, 2608, 2710] combine_rows_per_src=[5460, 5546, 5458, 5402, 5440, 5460, 5426, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38114 dup_rows=21902 dup_rows_remote=19150 dedup_ratio=0.334198 router_tokens_per_expert=[268, 278, 264, 260, 247, 243, 263, 267, 238, 237, 251, 263, 247, 240, 207, 228, 279, 265, 260, 266, 258, 261, 256, 247, 237, 252, 261, 230, 234, 290, 253, 266, 263, 275, 263, 244, 276, 263, 249, 248, 273, 235, 256, 264, 262, 285, 257, 280, 286, 252, 245, 234, 255, 247, 244, 266, 286, 271, 241, 260, 267, 248, 239, 236, 267, 272, 246, 264, 267, 234, 273, 299, 265, 265, 259, 252, 240, 249, 262, 239, 238, 244, 261, 254, 255, 229, 253, 267, 236, 255, 251, 261, 242, 273, 256, 249, 280, 261, 241, 260, 249, 246, 304, 271, 286, 278, 249, 247, 251, 241, 222, 250, 283, 248, 239, 231, 255, 247, 253, 246, 250, 283, 249, 281, 240, 280, 238, 246] payload_rows_per_dst=[5372, 5520, 5494, 5438, 5504, 5350, 5520, 5436] dup_rows_per_dst=[2630, 2710, 2866, 2716, 2802, 2724, 2752, 2702] combine_rows_per_src=[5372, 5520, 5494, 5438, 5504, 5350, 5520, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38466 dup_rows=21630 dup_rows_remote=19020 dedup_ratio=0.330048 router_tokens_per_expert=[258, 255, 248, 264, 252, 266, 243, 283, 258, 280, 249, 272, 258, 249, 231, 280, 289, 248, 267, 247, 263, 246, 238, 252, 274, 249, 262, 249, 249, 231, 262, 253, 274, 264, 250, 261, 273, 246, 246, 246, 261, 256, 258, 258, 249, 252, 249, 246, 262, 264, 235, 242, 256, 262, 287, 269, 248, 237, 238, 265, 229, 238, 240, 276, 257, 261, 241, 262, 263, 261, 246, 251, 226, 254, 238, 259, 272, 264, 270, 256, 240, 248, 261, 242, 262, 246, 215, 275, 247, 260, 253, 259, 252, 264, 246, 255, 274, 247, 268, 256, 264, 279, 257, 265, 261, 271, 231, 231, 271, 260, 230, 251, 278, 261, 258, 288, 259, 244, 250, 261, 261, 284, 254, 239, 271, 273, 238, 265] payload_rows_per_dst=[5524, 5498, 5488, 5460, 5486, 5440, 5496, 5514] dup_rows_per_dst=[2768, 2660, 2690, 2636, 2676, 2610, 2736, 2854] combine_rows_per_src=[5524, 5498, 5488, 5460, 5486, 5440, 5496, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38068 dup_rows=21968 dup_rows_remote=19202 dedup_ratio=0.335205 router_tokens_per_expert=[275, 261, 258, 261, 281, 235, 249, 235, 249, 265, 265, 279, 278, 260, 271, 257, 288, 250, 230, 228, 267, 260, 249, 256, 264, 250, 262, 275, 239, 266, 244, 247, 253, 232, 252, 246, 250, 248, 232, 233, 266, 290, 247, 243, 258, 264, 247, 305, 278, 244, 270, 281, 269, 257, 244, 274, 261, 255, 245, 250, 281, 251, 253, 253, 259, 271, 261, 255, 262, 247, 238, 287, 237, 242, 230, 233, 278, 275, 272, 240, 278, 252, 272, 265, 242, 240, 253, 262, 260, 260, 234, 268, 259, 230, 247, 290, 269, 220, 255, 248, 253, 255, 264, 256, 283, 221, 237, 223, 270, 226, 253, 232, 254, 278, 246, 248, 276, 268, 250, 246, 238, 266, 254, 270, 250, 273, 250, 251] payload_rows_per_dst=[5262, 5412, 5262, 5486, 5510, 5686, 5450, 5500] dup_rows_per_dst=[2644, 2670, 2370, 2770, 2778, 3038, 2932, 2766] combine_rows_per_src=[5262, 5412, 5262, 5486, 5510, 5686, 5450, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38164 dup_rows=21970 dup_rows_remote=19298 dedup_ratio=0.335236 router_tokens_per_expert=[244, 278, 287, 250, 249, 264, 257, 253, 265, 259, 251, 261, 260, 250, 283, 268, 244, 274, 253, 280, 246, 270, 272, 237, 260, 242, 265, 249, 242, 247, 283, 274, 240, 248, 232, 255, 299, 282, 261, 281, 241, 237, 296, 251, 228, 273, 240, 262, 260, 249, 262, 241, 284, 240, 248, 242, 261, 273, 262, 269, 274, 241, 233, 254, 272, 226, 237, 240, 231, 251, 267, 240, 278, 227, 235, 253, 274, 272, 278, 256, 284, 248, 266, 271, 262, 258, 254, 226, 234, 234, 247, 277, 248, 270, 259, 243, 262, 238, 267, 243, 277, 235, 235, 263, 250, 266, 245, 255, 259, 262, 255, 253, 231, 257, 238, 244, 274, 263, 260, 238, 242, 222, 273, 243, 266, 254, 265, 279] payload_rows_per_dst=[5468, 5442, 5492, 5436, 5402, 5440, 5448, 5438] dup_rows_per_dst=[2890, 2834, 2760, 2750, 2672, 2722, 2682, 2660] combine_rows_per_src=[5468, 5442, 5492, 5436, 5402, 5440, 5448, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38084 dup_rows=21960 dup_rows_remote=19192 dedup_ratio=0.335083 router_tokens_per_expert=[260, 262, 262, 254, 268, 238, 235, 284, 242, 265, 264, 278, 267, 267, 232, 263, 237, 266, 262, 259, 258, 267, 257, 252, 252, 233, 246, 250, 249, 231, 263, 245, 245, 250, 236, 269, 274, 271, 249, 246, 256, 258, 249, 272, 262, 284, 258, 251, 252, 239, 249, 235, 261, 255, 262, 271, 237, 244, 241, 269, 270, 279, 281, 250, 303, 249, 241, 243, 263, 253, 256, 244, 265, 234, 239, 257, 281, 268, 239, 267, 268, 249, 270, 254, 248, 240, 270, 234, 238, 247, 240, 265, 267, 236, 264, 269, 261, 245, 280, 250, 246, 272, 276, 263, 268, 246, 232, 254, 267, 257, 236, 257, 244, 275, 246, 246, 257, 275, 287, 223, 255, 279, 243, 233, 270, 257, 245, 269] payload_rows_per_dst=[5466, 5346, 5492, 5520, 5462, 5434, 5422, 5434] dup_rows_per_dst=[2816, 2708, 2768, 2670, 2742, 2684, 2798, 2774] combine_rows_per_src=[5466, 5346, 5492, 5520, 5462, 5434, 5422, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228168 combine_payload_bytes=156049408 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156277576 ingress_bytes=212672512 link_bytes=212672512 token_bytes=312326984 weight_movement_bytes=56623104 total_nvlink_bytes=368950088 weight_to_token_ratio=0.181294 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037723369e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38194 dup_rows=21782 dup_rows_remote=18964 dedup_ratio=0.332367 router_tokens_per_expert=[275, 259, 254, 269, 240, 256, 249, 287, 265, 261, 262, 239, 267, 267, 257, 260, 256, 258, 246, 259, 258, 245, 261, 233, 274, 229, 246, 243, 241, 231, 290, 234, 261, 260, 276, 241, 235, 283, 245, 232, 243, 271, 262, 248, 266, 214, 276, 261, 262, 258, 254, 259, 257, 262, 276, 269, 264, 253, 239, 270, 263, 254, 252, 258, 265, 264, 243, 250, 277, 268, 262, 253, 260, 272, 273, 261, 246, 240, 215, 254, 232, 269, 272, 240, 255, 253, 245, 270, 230, 257, 236, 250, 246, 260, 290, 247, 285, 257, 268, 273, 266, 246, 251, 252, 228, 256, 261, 272, 237, 268, 265, 255, 238, 279, 251, 268, 257, 268, 263, 244, 241, 254, 243, 246, 269, 256, 229, 272] payload_rows_per_dst=[5560, 5366, 5608, 5512, 5404, 5444, 5370, 5490] dup_rows_per_dst=[2818, 2670, 2828, 2788, 2730, 2660, 2622, 2666] combine_rows_per_src=[5560, 5366, 5608, 5512, 5404, 5444, 5370, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38328 dup_rows=21746 dup_rows_remote=19018 dedup_ratio=0.331818 router_tokens_per_expert=[255, 250, 227, 246, 261, 242, 244, 259, 240, 234, 260, 274, 243, 292, 264, 245, 264, 253, 260, 274, 289, 254, 276, 265, 252, 264, 276, 236, 264, 259, 277, 265, 255, 254, 271, 273, 256, 260, 265, 239, 241, 257, 268, 274, 275, 247, 253, 256, 256, 254, 234, 250, 262, 246, 260, 274, 292, 257, 269, 259, 279, 228, 236, 239, 280, 309, 256, 269, 252, 221, 267, 236, 223, 271, 241, 274, 255, 253, 250, 233, 234, 213, 272, 240, 257, 262, 267, 237, 243, 227, 268, 224, 264, 275, 278, 237, 260, 207, 258, 240, 266, 257, 221, 286, 288, 241, 231, 272, 254, 239, 283, 269, 246, 286, 236, 279, 248, 228, 292, 244, 257, 239, 256, 278, 280, 261, 224, 251] payload_rows_per_dst=[5518, 5626, 5450, 5462, 5480, 5378, 5446, 5430] dup_rows_per_dst=[2554, 2830, 2838, 2728, 2700, 2618, 2698, 2780] combine_rows_per_src=[5518, 5626, 5450, 5462, 5480, 5378, 5446, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=229944 combine_payload_bytes=157556736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157786680 ingress_bytes=176431104 link_bytes=176431104 token_bytes=315343416 weight_movement_bytes=18874368 total_nvlink_bytes=334217784 weight_to_token_ratio=0.059853 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690476029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=229056 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156344000 ingress_bytes=174989312 link_bytes=174989312 token_bytes=312458944 weight_movement_bytes=18874368 total_nvlink_bytes=331333312 weight_to_token_ratio=0.060406 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676661487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=229080 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156155608 ingress_bytes=193675264 link_bytes=193675264 token_bytes=312082136 weight_movement_bytes=37748736 total_nvlink_bytes=349830872 weight_to_token_ratio=0.120958 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855701085e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229848 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549592 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312869336 weight_movement_bytes=18874368 total_nvlink_bytes=331743704 weight_to_token_ratio=0.060327 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=229104 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156221168 ingress_bytes=193740800 link_bytes=193740800 token_bytes=312213232 weight_movement_bytes=37748736 total_nvlink_bytes=349961968 weight_to_token_ratio=0.120907 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856329019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228632 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156671256 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313113880 weight_movement_bytes=37748736 total_nvlink_bytes=350862616 weight_to_token_ratio=0.120559 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229384 combine_payload_bytes=156991488 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157220872 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314212360 weight_movement_bytes=37748736 total_nvlink_bytes=351961096 weight_to_token_ratio=0.120138 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38160 dup_rows=21818 dup_rows_remote=18980 dedup_ratio=0.332916 router_tokens_per_expert=[248, 243, 234, 248, 266, 295, 291, 255, 277, 247, 238, 263, 219, 250, 255, 235, 263, 251, 275, 258, 255, 274, 255, 274, 278, 249, 260, 281, 250, 248, 258, 269, 237, 271, 225, 256, 257, 272, 243, 246, 256, 247, 247, 241, 249, 243, 242, 241, 245, 244, 232, 274, 265, 232, 251, 250, 225, 262, 250, 253, 239, 258, 271, 253, 243, 264, 254, 249, 285, 268, 261, 266, 219, 258, 272, 263, 243, 282, 277, 266, 267, 236, 241, 279, 270, 247, 268, 232, 275, 264, 253, 251, 261, 258, 244, 259, 257, 242, 261, 263, 239, 271, 253, 238, 243, 252, 251, 259, 247, 269, 261, 235, 295, 271, 261, 266, 265, 277, 279, 253, 249, 244, 275, 256, 275, 251, 263, 233] payload_rows_per_dst=[5436, 5558, 5366, 5434, 5488, 5458, 5486, 5492] dup_rows_per_dst=[2692, 2838, 2580, 2574, 2852, 2752, 2596, 2934] combine_rows_per_src=[5436, 5558, 5366, 5434, 5488, 5458, 5486, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38194 dup_rows=21924 dup_rows_remote=19328 dedup_ratio=0.334534 router_tokens_per_expert=[250, 254, 248, 259, 254, 230, 270, 238, 261, 271, 260, 252, 259, 248, 272, 254, 258, 216, 250, 231, 282, 236, 241, 266, 240, 272, 276, 257, 262, 258, 245, 252, 248, 259, 251, 269, 243, 251, 267, 289, 220, 254, 296, 237, 277, 268, 249, 245, 274, 266, 253, 265, 246, 272, 250, 257, 245, 250, 267, 239, 254, 248, 269, 249, 241, 284, 269, 273, 256, 227, 231, 261, 281, 268, 254, 271, 278, 275, 267, 301, 274, 254, 256, 249, 249, 234, 281, 247, 216, 247, 256, 250, 264, 249, 245, 236, 264, 255, 243, 269, 275, 256, 246, 241, 248, 257, 226, 252, 247, 236, 248, 264, 258, 256, 239, 256, 257, 287, 236, 289, 267, 255, 245, 246, 252, 281, 258, 266] payload_rows_per_dst=[5442, 5410, 5406, 5380, 5588, 5418, 5444, 5524] dup_rows_per_dst=[2718, 2674, 2840, 2828, 2886, 2596, 2610, 2772] combine_rows_per_src=[5442, 5410, 5406, 5380, 5588, 5418, 5444, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38218 dup_rows=21872 dup_rows_remote=19118 dedup_ratio=0.333740 router_tokens_per_expert=[258, 267, 236, 281, 253, 259, 251, 243, 266, 220, 268, 254, 248, 269, 244, 254, 269, 260, 272, 284, 275, 260, 248, 269, 254, 254, 213, 232, 237, 239, 249, 241, 255, 260, 255, 274, 239, 274, 259, 259, 236, 272, 252, 262, 277, 274, 253, 242, 257, 248, 259, 240, 254, 273, 278, 277, 261, 261, 229, 246, 251, 276, 247, 247, 235, 247, 237, 282, 219, 244, 232, 250, 257, 261, 259, 273, 253, 262, 230, 257, 268, 283, 215, 236, 270, 246, 261, 246, 250, 260, 270, 266, 277, 273, 247, 258, 255, 261, 226, 245, 258, 255, 254, 282, 253, 272, 258, 291, 249, 254, 242, 245, 262, 265, 261, 283, 279, 259, 279, 251, 255, 277, 248, 261, 250, 245, 263, 232] payload_rows_per_dst=[5392, 5418, 5556, 5410, 5348, 5530, 5446, 5564] dup_rows_per_dst=[2750, 2694, 2730, 2798, 2648, 2722, 2754, 2776] combine_rows_per_src=[5392, 5418, 5556, 5410, 5348, 5530, 5446, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38362 dup_rows=21730 dup_rows_remote=19086 dedup_ratio=0.331573 router_tokens_per_expert=[245, 246, 251, 240, 255, 248, 274, 253, 285, 235, 258, 248, 286, 288, 240, 253, 266, 250, 277, 271, 238, 267, 270, 250, 253, 263, 238, 247, 237, 253, 242, 243, 252, 238, 257, 265, 264, 207, 272, 270, 274, 274, 273, 250, 258, 271, 251, 257, 249, 267, 267, 246, 250, 274, 291, 254, 273, 248, 257, 269, 225, 262, 261, 256, 263, 255, 268, 264, 258, 246, 239, 259, 242, 252, 254, 242, 247, 247, 265, 243, 263, 269, 283, 249, 251, 250, 241, 259, 274, 269, 272, 262, 249, 226, 228, 217, 252, 271, 264, 268, 251, 265, 280, 255, 258, 255, 274, 237, 227, 243, 251, 255, 261, 244, 274, 262, 236, 245, 259, 227, 257, 245, 270, 304, 270, 245, 251, 254] payload_rows_per_dst=[5464, 5416, 5512, 5528, 5444, 5420, 5492, 5530] dup_rows_per_dst=[2746, 2714, 2754, 2770, 2644, 2704, 2720, 2678] combine_rows_per_src=[5464, 5416, 5512, 5528, 5444, 5420, 5492, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38216 dup_rows=21896 dup_rows_remote=19070 dedup_ratio=0.334106 router_tokens_per_expert=[246, 275, 259, 238, 259, 263, 273, 242, 235, 254, 260, 268, 262, 227, 245, 270, 267, 258, 258, 252, 223, 262, 257, 275, 259, 286, 269, 254, 267, 255, 247, 294, 267, 254, 246, 265, 263, 244, 259, 258, 257, 261, 252, 264, 258, 250, 270, 255, 270, 239, 253, 257, 259, 229, 260, 263, 247, 235, 219, 258, 247, 264, 276, 256, 257, 269, 255, 234, 241, 253, 272, 257, 270, 243, 250, 254, 274, 237, 231, 236, 284, 262, 271, 263, 288, 251, 268, 262, 241, 256, 264, 272, 267, 274, 235, 275, 250, 255, 282, 273, 252, 226, 260, 272, 249, 260, 257, 223, 242, 254, 254, 239, 268, 259, 216, 267, 254, 270, 255, 263, 244, 240, 266, 239, 247, 245, 270, 237] payload_rows_per_dst=[5338, 5504, 5346, 5454, 5392, 5672, 5510, 5424] dup_rows_per_dst=[2814, 2862, 2570, 2610, 2568, 3060, 2586, 2826] combine_rows_per_src=[5338, 5504, 5346, 5454, 5392, 5672, 5510, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228560 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156531920 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312835280 weight_movement_bytes=37748736 total_nvlink_bytes=350584016 weight_to_token_ratio=0.120666 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38210 dup_rows=21852 dup_rows_remote=19054 dedup_ratio=0.333435 router_tokens_per_expert=[248, 248, 239, 234, 237, 249, 267, 239, 255, 244, 236, 246, 280, 245, 237, 251, 254, 286, 253, 249, 240, 225, 255, 294, 253, 270, 269, 287, 292, 242, 228, 250, 247, 253, 278, 255, 262, 220, 268, 258, 283, 265, 266, 259, 245, 255, 241, 281, 275, 260, 257, 267, 262, 243, 248, 261, 268, 253, 254, 251, 297, 247, 258, 265, 277, 241, 267, 264, 246, 248, 261, 259, 268, 230, 295, 253, 271, 284, 220, 222, 277, 252, 286, 252, 236, 262, 243, 268, 247, 273, 269, 251, 260, 252, 244, 235, 239, 251, 277, 242, 263, 236, 234, 268, 247, 278, 278, 272, 252, 245, 256, 258, 252, 262, 232, 243, 231, 250, 254, 298, 269, 250, 257, 234, 274, 254, 239, 256] payload_rows_per_dst=[5382, 5516, 5474, 5500, 5456, 5488, 5454, 5414] dup_rows_per_dst=[2528, 2778, 2798, 2832, 2756, 2726, 2738, 2696] combine_rows_per_src=[5382, 5516, 5474, 5500, 5456, 5488, 5454, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=230088 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156672712 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313115336 weight_movement_bytes=18874368 total_nvlink_bytes=331989704 weight_to_token_ratio=0.060279 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38264 dup_rows=21894 dup_rows_remote=19282 dedup_ratio=0.334076 router_tokens_per_expert=[241, 239, 259, 238, 262, 216, 261, 261, 260, 270, 254, 257, 267, 262, 235, 253, 283, 253, 283, 244, 247, 266, 268, 274, 255, 270, 240, 263, 265, 249, 264, 268, 252, 253, 290, 207, 258, 241, 276, 231, 237, 284, 274, 245, 275, 259, 297, 274, 250, 254, 272, 236, 235, 234, 262, 248, 247, 241, 245, 234, 267, 267, 264, 239, 234, 268, 264, 213, 246, 255, 250, 238, 259, 279, 237, 242, 265, 247, 235, 249, 248, 263, 267, 250, 252, 239, 228, 258, 255, 269, 253, 256, 259, 254, 252, 257, 244, 260, 236, 266, 264, 266, 283, 255, 260, 269, 256, 274, 277, 250, 242, 277, 261, 269, 270, 255, 274, 270, 237, 243, 256, 274, 266, 263, 293, 244, 238, 260] payload_rows_per_dst=[5348, 5536, 5518, 5378, 5342, 5396, 5566, 5558] dup_rows_per_dst=[2722, 2848, 2788, 2612, 2620, 2724, 2792, 2788] combine_rows_per_src=[5348, 5536, 5518, 5378, 5342, 5396, 5566, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229344 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770272 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311200 weight_movement_bytes=18874368 total_nvlink_bytes=332185568 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38288 dup_rows=21766 dup_rows_remote=19010 dedup_ratio=0.332123 router_tokens_per_expert=[281, 243, 228, 241, 242, 242, 241, 257, 258, 256, 260, 248, 251, 243, 278, 282, 231, 261, 262, 249, 257, 258, 275, 245, 236, 232, 258, 247, 263, 275, 280, 246, 268, 269, 287, 259, 255, 248, 266, 251, 266, 273, 272, 272, 256, 265, 255, 246, 256, 259, 264, 268, 249, 241, 245, 258, 251, 257, 268, 268, 256, 255, 249, 243, 273, 251, 246, 261, 294, 265, 222, 247, 253, 255, 275, 227, 275, 308, 283, 256, 271, 253, 239, 268, 231, 245, 232, 268, 245, 268, 264, 244, 251, 258, 258, 233, 248, 237, 255, 228, 247, 276, 271, 253, 258, 237, 275, 277, 240, 221, 253, 268, 240, 250, 241, 261, 258, 250, 246, 260, 252, 251, 260, 273, 263, 260, 264, 255] payload_rows_per_dst=[5482, 5516, 5640, 5446, 5476, 5422, 5384, 5404] dup_rows_per_dst=[2756, 2634, 2894, 2592, 2788, 2634, 2704, 2764] combine_rows_per_src=[5482, 5516, 5640, 5446, 5476, 5422, 5384, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229792 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157360544 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314491296 weight_movement_bytes=37748736 total_nvlink_bytes=352240032 weight_to_token_ratio=0.120031 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229144 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156761880 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294616 weight_movement_bytes=37748736 total_nvlink_bytes=351043352 weight_to_token_ratio=0.120490 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229056 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156737216 ingress_bytes=213131264 link_bytes=213131264 token_bytes=313245376 weight_movement_bytes=56623104 total_nvlink_bytes=369868480 weight_to_token_ratio=0.180763 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042118905e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=230184 combine_payload_bytes=156729344 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156959528 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313688872 weight_movement_bytes=37748736 total_nvlink_bytes=351437608 weight_to_token_ratio=0.120338 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229192 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056840 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884488 weight_movement_bytes=18874368 total_nvlink_bytes=332758856 weight_to_token_ratio=0.060132 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38256 dup_rows=21884 dup_rows_remote=19282 dedup_ratio=0.333923 router_tokens_per_expert=[261, 269, 270, 267, 245, 266, 262, 222, 257, 240, 242, 238, 288, 242, 285, 256, 256, 241, 272, 244, 260, 238, 247, 229, 244, 234, 235, 254, 253, 262, 266, 264, 241, 241, 262, 270, 285, 252, 273, 247, 253, 244, 230, 260, 243, 259, 249, 259, 274, 277, 243, 287, 273, 261, 252, 277, 227, 292, 272, 272, 230, 270, 272, 254, 244, 265, 231, 258, 269, 264, 284, 268, 230, 246, 260, 257, 246, 232, 255, 267, 290, 265, 239, 271, 289, 247, 275, 280, 263, 246, 264, 265, 246, 243, 252, 266, 271, 243, 275, 270, 272, 239, 252, 247, 259, 242, 271, 242, 272, 238, 261, 233, 252, 247, 251, 244, 236, 238, 239, 244, 248, 244, 242, 241, 258, 257, 278, 275] payload_rows_per_dst=[5442, 5396, 5398, 5554, 5490, 5602, 5368, 5402] dup_rows_per_dst=[2778, 2602, 2738, 2912, 2662, 3032, 2574, 2586] combine_rows_per_src=[5442, 5396, 5398, 5554, 5490, 5602, 5368, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=38174 dup_rows=21978 dup_rows_remote=19278 dedup_ratio=0.335358 router_tokens_per_expert=[260, 229, 247, 247, 257, 249, 256, 250, 249, 274, 240, 259, 256, 262, 268, 249, 259, 273, 255, 249, 270, 240, 280, 266, 273, 278, 255, 262, 269, 253, 282, 253, 266, 230, 264, 266, 229, 232, 234, 251, 218, 261, 271, 261, 251, 236, 244, 246, 219, 271, 279, 245, 254, 277, 284, 258, 258, 254, 282, 263, 220, 261, 233, 255, 249, 252, 239, 267, 271, 262, 268, 248, 263, 227, 238, 255, 250, 242, 260, 254, 234, 226, 277, 263, 248, 260, 248, 232, 245, 263, 276, 255, 261, 236, 259, 259, 258, 262, 269, 249, 253, 271, 250, 248, 254, 239, 250, 258, 251, 277, 288, 270, 264, 295, 264, 254, 223, 269, 281, 279, 241, 252, 235, 259, 299, 271, 249, 257] payload_rows_per_dst=[5382, 5606, 5266, 5484, 5390, 5384, 5530, 5516] dup_rows_per_dst=[2722, 2828, 2654, 2742, 2700, 2700, 2764, 2868] combine_rows_per_src=[5382, 5606, 5266, 5484, 5390, 5384, 5530, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38282 dup_rows=21822 dup_rows_remote=19078 dedup_ratio=0.332977 router_tokens_per_expert=[261, 283, 271, 256, 239, 251, 233, 255, 273, 272, 250, 264, 281, 237, 251, 260, 262, 252, 247, 259, 245, 251, 256, 241, 274, 219, 238, 275, 261, 274, 245, 255, 257, 255, 253, 252, 296, 253, 262, 283, 250, 282, 243, 269, 268, 285, 264, 240, 262, 255, 303, 249, 260, 265, 229, 268, 288, 234, 258, 249, 257, 255, 255, 252, 249, 282, 237, 232, 274, 246, 231, 269, 276, 237, 260, 268, 257, 287, 263, 248, 262, 234, 267, 236, 255, 240, 232, 255, 284, 259, 244, 236, 282, 253, 244, 243, 264, 246, 229, 263, 254, 245, 271, 241, 268, 265, 250, 266, 243, 240, 270, 273, 245, 251, 221, 265, 274, 271, 238, 242, 270, 262, 239, 234, 205, 241, 262, 276] payload_rows_per_dst=[5466, 5472, 5596, 5446, 5478, 5378, 5432, 5446] dup_rows_per_dst=[2808, 2636, 2828, 2832, 2754, 2674, 2744, 2546] combine_rows_per_src=[5466, 5472, 5596, 5446, 5478, 5378, 5432, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38288 dup_rows=21806 dup_rows_remote=18984 dedup_ratio=0.332733 router_tokens_per_expert=[258, 255, 247, 256, 272, 258, 249, 280, 270, 245, 263, 244, 228, 248, 243, 290, 264, 244, 254, 246, 242, 260, 260, 257, 260, 233, 250, 264, 255, 253, 258, 250, 269, 239, 267, 242, 279, 270, 241, 271, 259, 284, 275, 254, 245, 253, 262, 273, 218, 235, 244, 271, 260, 250, 262, 268, 260, 237, 256, 252, 270, 260, 250, 246, 279, 254, 256, 247, 268, 276, 269, 249, 266, 273, 244, 250, 249, 262, 229, 261, 247, 224, 273, 246, 250, 261, 269, 266, 257, 249, 257, 242, 258, 274, 247, 249, 234, 256, 256, 267, 247, 265, 264, 247, 271, 260, 275, 253, 243, 229, 254, 248, 253, 270, 287, 282, 250, 254, 263, 263, 236, 272, 248, 257, 276, 225, 258, 226] payload_rows_per_dst=[5496, 5500, 5590, 5408, 5442, 5442, 5428, 5424] dup_rows_per_dst=[2716, 2600, 2776, 2670, 2822, 2696, 2710, 2816] combine_rows_per_src=[5496, 5500, 5590, 5408, 5442, 5442, 5428, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=37952 dup_rows=21896 dup_rows_remote=18834 dedup_ratio=0.334106 router_tokens_per_expert=[275, 249, 245, 252, 235, 252, 230, 272, 287, 231, 266, 283, 246, 271, 255, 266, 237, 249, 252, 260, 259, 256, 236, 234, 272, 249, 271, 258, 271, 282, 241, 243, 235, 253, 243, 250, 247, 244, 247, 246, 224, 285, 240, 259, 236, 291, 271, 233, 283, 238, 248, 229, 243, 244, 232, 266, 277, 246, 253, 294, 280, 227, 257, 247, 271, 230, 257, 271, 259, 266, 267, 259, 258, 269, 283, 243, 259, 269, 239, 251, 252, 243, 252, 244, 235, 252, 236, 255, 252, 247, 254, 259, 269, 248, 236, 250, 242, 250, 288, 228, 267, 247, 277, 276, 225, 264, 274, 236, 287, 278, 251, 260, 247, 278, 246, 270, 265, 270, 269, 284, 280, 240, 305, 243, 264, 250, 248, 271] payload_rows_per_dst=[5402, 5358, 5346, 5444, 5504, 5358, 5540, 5688] dup_rows_per_dst=[2762, 2678, 2542, 2674, 2808, 2610, 2760, 3062] combine_rows_per_src=[5402, 5358, 5346, 5444, 5504, 5358, 5540, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38170 dup_rows=21876 dup_rows_remote=19122 dedup_ratio=0.333801 router_tokens_per_expert=[253, 250, 263, 243, 243, 243, 241, 274, 247, 264, 270, 267, 263, 248, 273, 277, 234, 279, 285, 273, 252, 235, 249, 271, 249, 259, 239, 251, 265, 264, 256, 220, 240, 219, 273, 258, 272, 269, 264, 262, 265, 261, 274, 236, 257, 269, 264, 239, 257, 236, 225, 250, 267, 247, 271, 292, 256, 263, 237, 284, 302, 282, 241, 261, 248, 287, 253, 238, 244, 277, 239, 236, 275, 257, 263, 258, 278, 261, 244, 238, 230, 252, 286, 231, 223, 238, 264, 267, 257, 250, 257, 243, 245, 233, 272, 251, 238, 258, 250, 268, 258, 250, 247, 249, 248, 290, 287, 244, 267, 288, 256, 273, 250, 260, 256, 241, 261, 245, 232, 258, 263, 266, 234, 259, 236, 254, 245, 249] payload_rows_per_dst=[5504, 5376, 5490, 5534, 5496, 5412, 5488, 5360] dup_rows_per_dst=[2734, 2786, 2754, 2808, 2696, 2586, 2854, 2658] combine_rows_per_src=[5504, 5376, 5490, 5534, 5496, 5412, 5488, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38086 dup_rows=21968 dup_rows_remote=19140 dedup_ratio=0.335205 router_tokens_per_expert=[256, 262, 245, 244, 291, 254, 260, 271, 263, 255, 236, 267, 256, 242, 259, 232, 265, 272, 253, 233, 240, 265, 260, 257, 239, 253, 266, 266, 275, 258, 269, 265, 282, 248, 236, 253, 254, 243, 249, 258, 273, 254, 251, 280, 260, 285, 263, 274, 246, 264, 254, 279, 250, 250, 248, 269, 239, 263, 273, 274, 269, 274, 270, 233, 264, 252, 249, 245, 242, 263, 240, 235, 267, 264, 248, 271, 256, 266, 237, 240, 241, 250, 247, 285, 270, 257, 272, 243, 252, 234, 263, 251, 279, 263, 287, 260, 261, 240, 244, 254, 270, 239, 247, 248, 248, 262, 230, 240, 238, 267, 262, 238, 263, 276, 265, 246, 256, 246, 262, 262, 229, 247, 270, 228, 257, 257, 240, 236] payload_rows_per_dst=[5442, 5448, 5496, 5482, 5492, 5544, 5342, 5322] dup_rows_per_dst=[2744, 2824, 2830, 2828, 2586, 2764, 2634, 2758] combine_rows_per_src=[5442, 5448, 5496, 5482, 5492, 5544, 5342, 5322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=230152 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156926728 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313623304 weight_movement_bytes=37748736 total_nvlink_bytes=351372040 weight_to_token_ratio=0.120363 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229808 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156590512 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312951216 weight_movement_bytes=18874368 total_nvlink_bytes=331825584 weight_to_token_ratio=0.060311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229440 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032512 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835584 weight_movement_bytes=37748736 total_nvlink_bytes=351584320 weight_to_token_ratio=0.120282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229088 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056736 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884384 weight_movement_bytes=18874368 total_nvlink_bytes=332758752 weight_to_token_ratio=0.060132 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155451392 dispatch_meta_bytes=227144 combine_payload_bytes=155451392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155678536 ingress_bytes=193200128 link_bytes=193200128 token_bytes=311129928 weight_movement_bytes=37748736 total_nvlink_bytes=348878664 weight_to_token_ratio=0.121328 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851148566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43936 payload_rows_remote=38432 dup_rows=21600 dup_rows_remote=18760 dedup_ratio=0.329590 router_tokens_per_expert=[270, 247, 242, 221, 263, 258, 259, 256, 263, 257, 255, 261, 244, 269, 246, 288, 254, 245, 258, 271, 231, 258, 260, 253, 274, 258, 237, 261, 263, 252, 274, 247, 265, 267, 252, 281, 269, 271, 256, 230, 257, 240, 257, 244, 224, 251, 243, 270, 239, 272, 262, 259, 247, 245, 266, 281, 269, 257, 260, 266, 263, 251, 268, 273, 244, 272, 245, 247, 267, 228, 239, 239, 251, 267, 245, 263, 253, 257, 251, 272, 273, 222, 222, 224, 269, 247, 248, 271, 276, 239, 246, 267, 265, 275, 265, 232, 269, 239, 260, 249, 264, 273, 272, 229, 253, 249, 276, 253, 252, 252, 269, 231, 256, 295, 293, 269, 279, 262, 278, 233, 236, 236, 272, 249, 242, 267, 254, 226] payload_rows_per_dst=[5504, 5396, 5548, 5606, 5494, 5434, 5452, 5502] dup_rows_per_dst=[2840, 2650, 2726, 2750, 2586, 2648, 2728, 2672] combine_rows_per_src=[5504, 5396, 5548, 5606, 5494, 5434, 5452, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229168 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156573488 ingress_bytes=194093056 link_bytes=194093056 token_bytes=312917808 weight_movement_bytes=37748736 total_nvlink_bytes=350666544 weight_to_token_ratio=0.120635 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859704163e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=228904 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156229160 ingress_bytes=174874624 link_bytes=174874624 token_bytes=312229416 weight_movement_bytes=18874368 total_nvlink_bytes=331103784 weight_to_token_ratio=0.060450 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675562603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=228768 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157646240 ingress_bytes=195166208 link_bytes=195166208 token_bytes=315063712 weight_movement_bytes=37748736 total_nvlink_bytes=352812448 weight_to_token_ratio=0.119813 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869986577e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38358 dup_rows=21630 dup_rows_remote=19052 dedup_ratio=0.330048 router_tokens_per_expert=[269, 232, 265, 257, 273, 294, 280, 260, 251, 277, 247, 268, 229, 249, 255, 229, 255, 243, 284, 261, 257, 252, 263, 234, 249, 270, 260, 237, 247, 245, 255, 251, 268, 244, 281, 286, 261, 240, 245, 224, 263, 259, 226, 246, 249, 274, 251, 246, 239, 252, 258, 246, 251, 262, 234, 284, 249, 236, 243, 261, 236, 264, 244, 272, 274, 244, 255, 258, 235, 241, 259, 251, 247, 291, 261, 229, 264, 271, 264, 258, 265, 245, 271, 247, 255, 266, 272, 253, 253, 278, 248, 253, 247, 298, 255, 253, 274, 254, 263, 261, 262, 250, 273, 261, 293, 276, 269, 243, 245, 249, 243, 248, 243, 277, 267, 242, 252, 280, 239, 261, 272, 276, 236, 225, 254, 240, 226, 261] payload_rows_per_dst=[5450, 5548, 5424, 5548, 5402, 5480, 5560, 5494] dup_rows_per_dst=[2820, 2578, 2702, 2514, 2802, 2838, 2768, 2608] combine_rows_per_src=[5450, 5548, 5424, 5548, 5402, 5480, 5560, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38116 dup_rows=21948 dup_rows_remote=19228 dedup_ratio=0.334900 router_tokens_per_expert=[235, 248, 247, 273, 255, 256, 254, 221, 270, 267, 240, 263, 248, 243, 259, 241, 257, 279, 251, 283, 263, 284, 231, 243, 269, 258, 250, 253, 266, 259, 273, 229, 259, 277, 225, 228, 258, 246, 266, 245, 241, 243, 284, 272, 253, 279, 256, 273, 274, 226, 249, 252, 254, 259, 241, 247, 249, 276, 256, 255, 283, 271, 248, 246, 250, 245, 243, 243, 272, 296, 272, 241, 270, 256, 263, 250, 251, 261, 231, 265, 277, 259, 259, 276, 267, 231, 247, 229, 259, 261, 285, 262, 267, 268, 251, 244, 230, 233, 256, 258, 261, 275, 264, 270, 263, 251, 243, 260, 243, 263, 258, 268, 251, 301, 246, 279, 225, 258, 269, 239, 235, 277, 240, 228, 242, 245, 263, 264] payload_rows_per_dst=[5376, 5444, 5420, 5420, 5482, 5480, 5472, 5494] dup_rows_per_dst=[2664, 2852, 2790, 2752, 2736, 2804, 2720, 2630] combine_rows_per_src=[5376, 5444, 5420, 5420, 5482, 5480, 5472, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38400 dup_rows=21730 dup_rows_remote=19046 dedup_ratio=0.331573 router_tokens_per_expert=[304, 245, 280, 289, 263, 258, 254, 243, 231, 268, 243, 282, 238, 271, 263, 248, 249, 289, 271, 255, 259, 272, 254, 238, 252, 248, 263, 238, 257, 252, 251, 264, 246, 248, 252, 223, 227, 222, 261, 264, 245, 243, 266, 260, 257, 241, 265, 247, 249, 252, 223, 255, 229, 263, 273, 241, 264, 254, 280, 286, 234, 263, 267, 270, 283, 271, 265, 268, 260, 278, 250, 275, 240, 274, 260, 255, 261, 258, 263, 290, 243, 238, 255, 263, 241, 273, 245, 269, 270, 253, 254, 249, 239, 249, 253, 251, 215, 279, 255, 226, 267, 241, 271, 239, 237, 266, 263, 213, 280, 243, 232, 284, 265, 224, 262, 252, 275, 258, 243, 253, 260, 258, 246, 250, 251, 263, 265, 274] payload_rows_per_dst=[5536, 5460, 5428, 5480, 5632, 5406, 5364, 5500] dup_rows_per_dst=[2824, 2764, 2506, 2726, 2870, 2684, 2658, 2698] combine_rows_per_src=[5536, 5460, 5428, 5480, 5632, 5406, 5364, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=37870 dup_rows=21924 dup_rows_remote=18888 dedup_ratio=0.334534 router_tokens_per_expert=[279, 248, 244, 274, 271, 267, 257, 265, 245, 248, 265, 273, 246, 260, 269, 262, 284, 256, 271, 278, 257, 259, 258, 250, 233, 253, 264, 264, 269, 264, 252, 239, 261, 242, 267, 300, 235, 274, 274, 244, 254, 263, 244, 269, 250, 268, 257, 226, 240, 273, 242, 270, 276, 251, 272, 252, 232, 253, 237, 244, 256, 265, 271, 259, 249, 236, 250, 255, 266, 258, 244, 197, 262, 238, 252, 248, 245, 257, 245, 265, 249, 264, 263, 270, 249, 282, 229, 284, 211, 267, 225, 270, 257, 247, 253, 262, 262, 230, 235, 249, 270, 237, 243, 249, 216, 262, 237, 279, 248, 261, 253, 248, 286, 278, 272, 256, 252, 253, 258, 256, 258, 279, 257, 254, 235, 261, 272, 268] payload_rows_per_dst=[5374, 5304, 5440, 5540, 5358, 5272, 5582, 5742] dup_rows_per_dst=[2650, 2610, 2816, 2870, 2674, 2406, 2862, 3036] combine_rows_per_src=[5374, 5304, 5440, 5540, 5358, 5272, 5582, 5742] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38232 dup_rows=21792 dup_rows_remote=18952 dedup_ratio=0.332520 router_tokens_per_expert=[223, 233, 258, 236, 234, 247, 274, 250, 254, 280, 263, 252, 265, 247, 275, 258, 247, 234, 264, 248, 274, 248, 271, 263, 249, 283, 247, 232, 246, 265, 262, 264, 246, 263, 258, 254, 243, 259, 262, 256, 283, 234, 261, 262, 255, 240, 248, 273, 261, 248, 239, 280, 268, 247, 258, 230, 250, 290, 268, 244, 226, 242, 248, 249, 283, 263, 269, 250, 255, 256, 234, 274, 288, 256, 258, 253, 264, 259, 269, 245, 295, 276, 262, 256, 259, 234, 246, 271, 248, 273, 247, 255, 279, 259, 271, 262, 245, 252, 254, 246, 239, 255, 281, 277, 241, 243, 260, 255, 239, 232, 253, 246, 250, 265, 250, 271, 248, 257, 246, 256, 258, 232, 240, 263, 288, 257, 240, 269] payload_rows_per_dst=[5434, 5506, 5466, 5468, 5512, 5524, 5416, 5418] dup_rows_per_dst=[2664, 2688, 2728, 2628, 2840, 2862, 2620, 2762] combine_rows_per_src=[5434, 5506, 5466, 5468, 5512, 5524, 5416, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38350 dup_rows=21644 dup_rows_remote=18862 dedup_ratio=0.330261 router_tokens_per_expert=[243, 236, 258, 261, 229, 284, 298, 266, 256, 261, 286, 239, 238, 244, 263, 226, 207, 274, 254, 316, 261, 269, 259, 268, 235, 262, 230, 235, 263, 260, 265, 271, 303, 261, 261, 276, 267, 263, 245, 253, 247, 273, 253, 226, 242, 282, 254, 256, 257, 253, 232, 237, 253, 273, 251, 273, 233, 258, 264, 247, 248, 266, 247, 260, 303, 274, 211, 276, 260, 233, 247, 243, 257, 268, 241, 245, 246, 215, 254, 248, 255, 270, 272, 251, 264, 239, 252, 259, 272, 262, 244, 252, 262, 267, 275, 260, 268, 259, 259, 276, 234, 251, 260, 263, 258, 259, 261, 252, 252, 267, 254, 248, 249, 234, 248, 275, 252, 234, 254, 258, 248, 282, 242, 272, 274, 239, 247, 231] payload_rows_per_dst=[5536, 5530, 5542, 5440, 5430, 5472, 5498, 5444] dup_rows_per_dst=[2640, 2728, 2782, 2664, 2612, 2840, 2744, 2634] combine_rows_per_src=[5536, 5530, 5542, 5440, 5430, 5472, 5498, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38400 dup_rows=21748 dup_rows_remote=19070 dedup_ratio=0.331848 router_tokens_per_expert=[254, 253, 274, 277, 230, 244, 274, 278, 258, 252, 231, 263, 269, 231, 254, 253, 251, 285, 272, 246, 244, 262, 254, 282, 264, 254, 262, 270, 259, 265, 255, 245, 222, 276, 248, 230, 250, 279, 247, 257, 262, 222, 273, 236, 270, 273, 270, 251, 259, 252, 243, 261, 240, 268, 252, 228, 258, 245, 255, 243, 242, 249, 260, 278, 256, 237, 235, 250, 261, 277, 289, 259, 279, 239, 272, 233, 246, 232, 282, 261, 252, 275, 254, 245, 221, 247, 254, 247, 237, 273, 279, 275, 284, 261, 248, 232, 270, 256, 259, 250, 244, 242, 244, 242, 257, 245, 243, 257, 266, 249, 272, 263, 241, 261, 249, 269, 265, 259, 251, 271, 265, 261, 236, 250, 266, 279, 263, 267] payload_rows_per_dst=[5508, 5500, 5462, 5388, 5458, 5520, 5472, 5480] dup_rows_per_dst=[2682, 2840, 2670, 2678, 2758, 2648, 2646, 2826] combine_rows_per_src=[5508, 5500, 5462, 5388, 5458, 5520, 5472, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229640 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157344008 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314458376 weight_movement_bytes=37748736 total_nvlink_bytes=352207112 weight_to_token_ratio=0.120044 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229784 combine_payload_bytes=157286400 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157516184 ingress_bytes=213909504 link_bytes=213909504 token_bytes=314802584 weight_movement_bytes=56623104 total_nvlink_bytes=371425688 weight_to_token_ratio=0.179869 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049575617e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229376 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352512 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475648 weight_movement_bytes=18874368 total_nvlink_bytes=331350016 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228848 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157310448 ingress_bytes=194830336 link_bytes=194830336 token_bytes=314392048 weight_movement_bytes=37748736 total_nvlink_bytes=352140784 weight_to_token_ratio=0.120069 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866768417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=155115520 dispatch_meta_bytes=227032 combine_payload_bytes=155115520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155342552 ingress_bytes=173989888 link_bytes=173989888 token_bytes=310458072 weight_movement_bytes=18874368 total_nvlink_bytes=329332440 weight_to_token_ratio=0.060795 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.667085498e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228736 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827008 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313425280 weight_movement_bytes=18874368 total_nvlink_bytes=332299648 weight_to_token_ratio=0.060220 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229880 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157516280 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802680 weight_movement_bytes=18874368 total_nvlink_bytes=333677048 weight_to_token_ratio=0.059956 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=37942 dup_rows=21888 dup_rows_remote=18832 dedup_ratio=0.333984 router_tokens_per_expert=[255, 261, 254, 236, 266, 265, 301, 277, 281, 268, 233, 267, 243, 273, 244, 264, 239, 247, 246, 239, 243, 241, 260, 247, 255, 259, 237, 253, 257, 260, 270, 253, 256, 255, 270, 271, 274, 245, 276, 238, 258, 260, 269, 253, 243, 271, 242, 268, 286, 249, 289, 240, 297, 260, 277, 279, 263, 227, 246, 284, 240, 248, 233, 241, 278, 233, 253, 257, 255, 242, 238, 281, 235, 263, 281, 259, 277, 266, 265, 260, 262, 249, 258, 256, 260, 236, 222, 253, 254, 254, 238, 244, 261, 259, 245, 246, 288, 272, 261, 253, 227, 245, 236, 260, 264, 239, 245, 250, 246, 268, 245, 225, 258, 251, 273, 249, 252, 250, 257, 260, 242, 269, 241, 262, 241, 290, 265, 242] payload_rows_per_dst=[5706, 5246, 5450, 5458, 5524, 5440, 5588, 5236] dup_rows_per_dst=[3056, 2452, 2866, 2770, 2762, 2554, 2968, 2460] combine_rows_per_src=[5706, 5246, 5450, 5458, 5524, 5440, 5588, 5236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155410432 dispatch_meta_bytes=227096 combine_payload_bytes=155410432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155637528 ingress_bytes=174284800 link_bytes=174284800 token_bytes=311047960 weight_movement_bytes=18874368 total_nvlink_bytes=329922328 weight_to_token_ratio=0.060680 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669911200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38218 dup_rows=21826 dup_rows_remote=19110 dedup_ratio=0.333038 router_tokens_per_expert=[264, 242, 236, 231, 283, 266, 240, 236, 231, 256, 254, 252, 272, 264, 225, 236, 282, 249, 238, 249, 268, 248, 246, 241, 286, 264, 281, 257, 263, 252, 230, 250, 240, 232, 278, 257, 258, 265, 240, 285, 256, 266, 260, 273, 234, 253, 274, 242, 248, 244, 263, 242, 258, 263, 271, 271, 283, 277, 251, 231, 247, 264, 273, 249, 267, 248, 265, 267, 269, 273, 251, 249, 243, 267, 284, 247, 300, 257, 268, 264, 253, 259, 266, 241, 247, 234, 269, 250, 262, 243, 265, 235, 246, 263, 273, 231, 265, 255, 220, 246, 263, 255, 258, 259, 278, 261, 247, 257, 252, 252, 231, 250, 257, 236, 248, 273, 282, 249, 248, 238, 256, 243, 286, 255, 232, 246, 297, 277] payload_rows_per_dst=[5402, 5492, 5450, 5520, 5504, 5470, 5412, 5460] dup_rows_per_dst=[2574, 2716, 2776, 2750, 2934, 2604, 2686, 2786] combine_rows_per_src=[5402, 5492, 5450, 5520, 5504, 5470, 5412, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38122 dup_rows=21912 dup_rows_remote=18996 dedup_ratio=0.334351 router_tokens_per_expert=[241, 256, 261, 246, 250, 247, 263, 243, 247, 266, 246, 266, 247, 269, 267, 275, 252, 247, 245, 245, 258, 266, 228, 247, 250, 253, 247, 270, 270, 267, 253, 270, 264, 259, 261, 273, 265, 286, 229, 249, 255, 254, 271, 270, 286, 235, 250, 230, 270, 243, 264, 279, 278, 261, 246, 249, 261, 231, 269, 223, 268, 238, 264, 255, 214, 261, 231, 250, 258, 259, 271, 249, 255, 255, 265, 261, 258, 253, 277, 257, 254, 253, 289, 250, 246, 250, 244, 249, 277, 260, 266, 251, 248, 268, 250, 234, 286, 277, 299, 242, 266, 252, 274, 265, 234, 268, 261, 259, 249, 261, 262, 254, 238, 240, 258, 242, 240, 239, 274, 255, 242, 267, 226, 225, 256, 243, 266, 291] payload_rows_per_dst=[5490, 5440, 5558, 5418, 5446, 5372, 5502, 5398] dup_rows_per_dst=[2690, 2696, 2716, 2780, 2702, 2806, 2916, 2606] combine_rows_per_src=[5490, 5440, 5558, 5418, 5446, 5372, 5502, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38154 dup_rows=21856 dup_rows_remote=19118 dedup_ratio=0.333496 router_tokens_per_expert=[261, 238, 250, 257, 240, 233, 239, 255, 243, 264, 268, 285, 253, 257, 232, 258, 271, 250, 225, 277, 251, 248, 249, 249, 274, 245, 253, 291, 229, 239, 261, 271, 247, 254, 278, 257, 269, 288, 258, 256, 262, 287, 256, 236, 242, 281, 247, 274, 282, 260, 260, 242, 261, 259, 274, 255, 276, 243, 236, 253, 238, 249, 265, 262, 243, 235, 272, 260, 266, 246, 274, 254, 265, 227, 250, 242, 261, 242, 274, 271, 244, 277, 308, 241, 258, 254, 265, 249, 254, 266, 263, 254, 250, 248, 228, 273, 220, 257, 260, 269, 284, 248, 253, 245, 265, 260, 219, 284, 260, 256, 244, 247, 265, 217, 246, 259, 238, 246, 250, 264, 233, 253, 261, 267, 263, 275, 265, 258] payload_rows_per_dst=[5396, 5454, 5542, 5452, 5488, 5526, 5416, 5406] dup_rows_per_dst=[2670, 2712, 2842, 2778, 2676, 2738, 2726, 2714] combine_rows_per_src=[5396, 5454, 5542, 5452, 5488, 5526, 5416, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38328 dup_rows=21668 dup_rows_remote=18898 dedup_ratio=0.330627 router_tokens_per_expert=[239, 293, 280, 263, 220, 229, 238, 278, 244, 272, 236, 235, 295, 255, 264, 258, 290, 266, 264, 267, 249, 243, 262, 254, 262, 224, 229, 264, 276, 241, 245, 276, 249, 282, 233, 238, 244, 251, 240, 271, 278, 260, 265, 273, 272, 283, 249, 267, 279, 276, 234, 258, 254, 261, 251, 263, 268, 277, 248, 263, 249, 255, 289, 230, 222, 259, 231, 264, 262, 274, 252, 246, 227, 241, 251, 287, 264, 256, 274, 266, 268, 258, 262, 235, 286, 247, 237, 248, 274, 253, 260, 241, 241, 263, 227, 247, 245, 278, 259, 258, 238, 250, 253, 253, 239, 238, 266, 238, 255, 262, 227, 269, 270, 265, 250, 275, 265, 256, 247, 256, 252, 260, 230, 237, 248, 272, 257, 256] payload_rows_per_dst=[5446, 5524, 5540, 5550, 5440, 5438, 5444, 5486] dup_rows_per_dst=[2752, 2700, 2770, 2760, 2712, 2656, 2612, 2706] combine_rows_per_src=[5446, 5524, 5540, 5550, 5440, 5438, 5444, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38038 dup_rows=21758 dup_rows_remote=18706 dedup_ratio=0.332001 router_tokens_per_expert=[243, 271, 273, 257, 268, 264, 237, 261, 255, 264, 279, 243, 224, 254, 250, 238, 257, 258, 267, 282, 253, 244, 240, 250, 230, 244, 248, 264, 267, 248, 247, 246, 260, 277, 265, 279, 266, 249, 261, 261, 254, 255, 244, 248, 265, 260, 223, 272, 226, 258, 232, 256, 226, 254, 260, 263, 231, 233, 259, 275, 248, 273, 230, 254, 299, 272, 260, 276, 269, 258, 270, 231, 265, 262, 252, 246, 283, 266, 258, 264, 277, 254, 273, 239, 245, 262, 224, 276, 233, 245, 241, 262, 238, 239, 240, 250, 280, 269, 236, 269, 268, 267, 271, 237, 238, 241, 240, 262, 243, 256, 287, 244, 235, 253, 275, 260, 263, 277, 266, 266, 282, 289, 255, 253, 242, 240, 281, 251] payload_rows_per_dst=[5522, 5458, 5270, 5418, 5438, 5522, 5410, 5740] dup_rows_per_dst=[2640, 2632, 2368, 2762, 2682, 2798, 2824, 3052] combine_rows_per_src=[5522, 5458, 5270, 5418, 5438, 5522, 5410, 5740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38238 dup_rows=21790 dup_rows_remote=19036 dedup_ratio=0.332489 router_tokens_per_expert=[234, 274, 290, 275, 246, 260, 282, 270, 244, 236, 286, 233, 274, 271, 261, 250, 293, 269, 270, 255, 270, 256, 242, 251, 250, 272, 266, 222, 263, 251, 277, 271, 269, 258, 255, 260, 242, 250, 270, 256, 257, 254, 229, 262, 266, 221, 261, 239, 238, 262, 281, 247, 250, 264, 266, 269, 257, 237, 247, 257, 245, 271, 261, 259, 283, 265, 262, 265, 235, 262, 248, 269, 259, 246, 241, 266, 278, 289, 242, 221, 263, 270, 275, 267, 280, 256, 243, 269, 246, 240, 217, 253, 257, 268, 267, 269, 250, 274, 254, 249, 246, 255, 235, 264, 245, 241, 231, 271, 251, 261, 253, 264, 248, 229, 229, 251, 244, 264, 222, 252, 257, 241, 228, 232, 248, 286, 248, 250] payload_rows_per_dst=[5556, 5538, 5470, 5442, 5508, 5486, 5432, 5314] dup_rows_per_dst=[2816, 2818, 2628, 2780, 2754, 2794, 2656, 2544] combine_rows_per_src=[5556, 5538, 5470, 5442, 5508, 5486, 5432, 5314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38408 dup_rows=21610 dup_rows_remote=19010 dedup_ratio=0.329742 router_tokens_per_expert=[270, 251, 249, 244, 239, 256, 246, 246, 259, 254, 258, 252, 258, 271, 254, 267, 255, 268, 256, 271, 247, 248, 246, 273, 264, 306, 238, 268, 252, 248, 237, 265, 258, 253, 244, 272, 255, 237, 242, 247, 239, 267, 248, 249, 255, 260, 255, 258, 264, 262, 240, 249, 240, 250, 258, 252, 245, 271, 233, 270, 230, 258, 264, 273, 248, 240, 249, 280, 244, 250, 234, 243, 243, 249, 259, 265, 279, 234, 264, 258, 263, 246, 251, 257, 286, 276, 262, 234, 278, 266, 241, 251, 244, 270, 257, 253, 247, 261, 253, 239, 242, 257, 280, 252, 251, 229, 236, 247, 264, 293, 267, 263, 244, 261, 246, 244, 261, 251, 278, 272, 296, 267, 283, 268, 248, 262, 275, 243] payload_rows_per_dst=[5422, 5564, 5420, 5518, 5426, 5520, 5462, 5594] dup_rows_per_dst=[2726, 2720, 2658, 2600, 2652, 2750, 2700, 2804] combine_rows_per_src=[5422, 5564, 5420, 5518, 5426, 5520, 5462, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229312 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770240 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311168 weight_movement_bytes=18874368 total_nvlink_bytes=332185536 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228472 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376184 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312523896 weight_movement_bytes=18874368 total_nvlink_bytes=331398264 weight_to_token_ratio=0.060393 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229088 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156507872 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786656 weight_movement_bytes=18874368 total_nvlink_bytes=331661024 weight_to_token_ratio=0.060343 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=228904 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157220392 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314211880 weight_movement_bytes=56623104 total_nvlink_bytes=370834984 weight_to_token_ratio=0.180207 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38240 dup_rows=21822 dup_rows_remote=19042 dedup_ratio=0.332977 router_tokens_per_expert=[265, 239, 251, 250, 235, 281, 254, 275, 252, 261, 246, 270, 273, 235, 273, 260, 264, 281, 258, 217, 265, 237, 250, 259, 269, 276, 287, 270, 249, 227, 260, 288, 253, 258, 264, 293, 251, 253, 265, 250, 244, 259, 248, 237, 229, 303, 258, 261, 263, 246, 245, 262, 233, 251, 260, 242, 261, 263, 226, 253, 267, 225, 252, 277, 248, 241, 264, 262, 281, 249, 267, 273, 262, 254, 254, 223, 280, 273, 242, 293, 263, 243, 234, 271, 249, 221, 230, 242, 243, 288, 268, 239, 274, 248, 240, 267, 259, 267, 269, 255, 257, 265, 244, 231, 261, 248, 251, 224, 267, 237, 229, 244, 254, 264, 255, 271, 252, 242, 296, 248, 233, 270, 264, 275, 264, 256, 257, 244] payload_rows_per_dst=[5474, 5568, 5406, 5390, 5500, 5698, 5332, 5346] dup_rows_per_dst=[2780, 2746, 2846, 2648, 2832, 2844, 2654, 2472] combine_rows_per_src=[5474, 5568, 5406, 5390, 5500, 5698, 5332, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155803648 dispatch_meta_bytes=226976 combine_payload_bytes=155803648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156030624 ingress_bytes=174678016 link_bytes=174678016 token_bytes=311834272 weight_movement_bytes=18874368 total_nvlink_bytes=330708640 weight_to_token_ratio=0.060527 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673678802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229096 combine_payload_bytes=156622848 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156851944 ingress_bytes=250994688 link_bytes=250994688 token_bytes=313474792 weight_movement_bytes=94371840 total_nvlink_bytes=407846632 weight_to_token_ratio=0.301051 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404907603e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229672 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157548840 ingress_bytes=176193536 link_bytes=176193536 token_bytes=314868008 weight_movement_bytes=18874368 total_nvlink_bytes=333742376 weight_to_token_ratio=0.059944 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688199769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229128 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860168 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491208 weight_movement_bytes=37748736 total_nvlink_bytes=351239944 weight_to_token_ratio=0.120414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38250 dup_rows=21866 dup_rows_remote=19134 dedup_ratio=0.333649 router_tokens_per_expert=[259, 275, 240, 256, 244, 251, 280, 250, 222, 254, 268, 284, 252, 256, 253, 278, 266, 252, 261, 259, 248, 242, 265, 225, 249, 256, 265, 297, 245, 237, 269, 240, 250, 280, 269, 282, 254, 236, 247, 281, 246, 254, 276, 246, 240, 276, 240, 246, 266, 229, 272, 256, 253, 270, 249, 255, 272, 254, 268, 226, 256, 260, 245, 264, 269, 262, 238, 260, 254, 253, 231, 241, 241, 261, 264, 275, 275, 271, 224, 274, 251, 254, 230, 286, 256, 259, 267, 249, 273, 258, 270, 238, 300, 246, 294, 275, 262, 267, 263, 270, 273, 263, 254, 258, 250, 262, 222, 259, 247, 256, 243, 224, 256, 252, 259, 247, 280, 230, 255, 221, 231, 237, 266, 286, 229, 236, 242, 253] payload_rows_per_dst=[5538, 5420, 5424, 5522, 5482, 5516, 5442, 5326] dup_rows_per_dst=[2706, 2732, 2822, 2668, 2704, 2896, 2704, 2634] combine_rows_per_src=[5538, 5420, 5424, 5522, 5482, 5516, 5442, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38368 dup_rows=21678 dup_rows_remote=19010 dedup_ratio=0.330780 router_tokens_per_expert=[258, 259, 260, 255, 268, 277, 240, 273, 245, 230, 273, 250, 252, 252, 248, 261, 235, 257, 250, 234, 260, 239, 270, 242, 254, 260, 247, 252, 226, 243, 261, 252, 239, 271, 259, 248, 241, 269, 261, 265, 259, 271, 235, 272, 253, 261, 279, 269, 231, 253, 240, 253, 234, 273, 279, 233, 264, 272, 251, 258, 259, 261, 252, 232, 260, 265, 261, 256, 266, 242, 256, 259, 281, 267, 237, 268, 238, 265, 274, 265, 255, 241, 256, 251, 269, 243, 267, 237, 250, 260, 250, 253, 261, 284, 261, 262, 264, 267, 252, 271, 248, 246, 267, 281, 259, 230, 235, 232, 248, 259, 251, 269, 249, 282, 254, 256, 270, 268, 255, 258, 255, 252, 259, 262, 241, 272, 270, 246] payload_rows_per_dst=[5532, 5416, 5558, 5456, 5482, 5466, 5490, 5458] dup_rows_per_dst=[2670, 2548, 2746, 2634, 2838, 2734, 2668, 2840] combine_rows_per_src=[5532, 5416, 5558, 5456, 5482, 5466, 5490, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38286 dup_rows=21798 dup_rows_remote=19134 dedup_ratio=0.332611 router_tokens_per_expert=[262, 240, 263, 247, 240, 260, 256, 255, 254, 233, 264, 244, 251, 230, 275, 272, 255, 248, 276, 239, 260, 279, 264, 273, 263, 271, 240, 251, 252, 271, 269, 275, 272, 251, 281, 252, 257, 275, 279, 275, 251, 275, 244, 248, 242, 245, 242, 256, 231, 252, 284, 267, 273, 276, 244, 274, 252, 276, 270, 258, 257, 240, 269, 243, 245, 272, 253, 273, 256, 259, 232, 226, 250, 254, 250, 279, 243, 255, 268, 247, 253, 268, 278, 249, 263, 266, 230, 239, 289, 269, 240, 242, 257, 251, 202, 262, 252, 257, 230, 241, 257, 246, 261, 248, 232, 254, 252, 241, 249, 276, 251, 257, 251, 249, 240, 269, 270, 241, 280, 249, 283, 253, 279, 242, 236, 256, 245, 258] payload_rows_per_dst=[5450, 5548, 5474, 5540, 5400, 5452, 5352, 5522] dup_rows_per_dst=[2642, 2824, 2816, 2792, 2724, 2664, 2656, 2680] combine_rows_per_src=[5450, 5548, 5474, 5540, 5400, 5452, 5352, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38282 dup_rows=21840 dup_rows_remote=19132 dedup_ratio=0.333252 router_tokens_per_expert=[259, 256, 262, 247, 259, 258, 244, 253, 247, 275, 243, 241, 253, 256, 273, 237, 271, 282, 252, 253, 248, 250, 271, 237, 295, 263, 273, 238, 263, 266, 278, 263, 258, 252, 250, 268, 244, 268, 261, 252, 267, 248, 216, 286, 270, 258, 235, 228, 270, 276, 269, 218, 202, 266, 244, 242, 224, 225, 248, 281, 237, 267, 269, 246, 215, 295, 252, 272, 248, 253, 252, 261, 269, 271, 239, 237, 235, 295, 241, 246, 244, 245, 259, 265, 236, 225, 259, 262, 292, 231, 257, 283, 283, 254, 261, 262, 253, 272, 274, 273, 266, 272, 274, 258, 241, 257, 257, 248, 253, 270, 233, 235, 260, 257, 252, 229, 258, 289, 258, 235, 262, 261, 267, 257, 255, 268, 241, 273] payload_rows_per_dst=[5428, 5510, 5414, 5434, 5428, 5486, 5516, 5480] dup_rows_per_dst=[2698, 2896, 2708, 2534, 2734, 2750, 2756, 2764] combine_rows_per_src=[5428, 5510, 5414, 5434, 5428, 5486, 5516, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38212 dup_rows=21848 dup_rows_remote=18982 dedup_ratio=0.333374 router_tokens_per_expert=[263, 271, 256, 269, 239, 285, 236, 248, 248, 237, 269, 276, 256, 273, 245, 271, 280, 260, 264, 245, 229, 260, 269, 262, 232, 250, 251, 254, 250, 254, 264, 255, 261, 274, 235, 246, 240, 259, 256, 246, 253, 253, 262, 264, 277, 249, 263, 270, 251, 239, 285, 266, 262, 252, 254, 265, 244, 241, 245, 243, 233, 250, 267, 249, 240, 260, 246, 254, 261, 237, 275, 235, 239, 245, 249, 262, 281, 221, 261, 249, 276, 268, 252, 266, 236, 271, 280, 258, 252, 242, 243, 239, 250, 249, 268, 255, 293, 264, 260, 235, 280, 262, 238, 251, 257, 247, 258, 264, 267, 275, 248, 264, 251, 255, 244, 263, 252, 251, 264, 265, 256, 252, 281, 270, 247, 260, 230, 269] payload_rows_per_dst=[5438, 5422, 5404, 5408, 5418, 5544, 5578, 5476] dup_rows_per_dst=[2846, 2614, 2504, 2684, 2736, 2668, 2930, 2866] combine_rows_per_src=[5438, 5422, 5404, 5408, 5418, 5544, 5578, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229536 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156901536 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313573536 weight_movement_bytes=37748736 total_nvlink_bytes=351322272 weight_to_token_ratio=0.120382 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38318 dup_rows=21742 dup_rows_remote=18922 dedup_ratio=0.331757 router_tokens_per_expert=[245, 267, 270, 237, 250, 290, 253, 257, 256, 286, 267, 223, 239, 241, 273, 259, 261, 268, 239, 251, 239, 252, 275, 249, 261, 262, 269, 274, 246, 239, 223, 247, 275, 233, 253, 261, 250, 271, 259, 272, 243, 216, 249, 256, 258, 252, 245, 258, 280, 283, 215, 252, 256, 253, 276, 257, 264, 257, 258, 245, 229, 261, 260, 227, 252, 278, 252, 281, 259, 250, 254, 239, 264, 267, 231, 294, 281, 252, 239, 255, 228, 257, 243, 260, 296, 269, 257, 237, 259, 262, 258, 267, 245, 267, 291, 275, 253, 258, 227, 264, 278, 252, 254, 272, 253, 261, 249, 280, 255, 256, 243, 258, 232, 263, 276, 255, 246, 254, 283, 238, 253, 237, 255, 241, 265, 280, 235, 231] payload_rows_per_dst=[5560, 5432, 5488, 5424, 5476, 5576, 5416, 5422] dup_rows_per_dst=[2666, 2678, 2614, 2722, 2820, 2766, 2810, 2666] combine_rows_per_src=[5560, 5432, 5488, 5424, 5476, 5576, 5416, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38446 dup_rows=21702 dup_rows_remote=18940 dedup_ratio=0.331146 router_tokens_per_expert=[239, 290, 255, 255, 258, 284, 282, 265, 239, 250, 249, 256, 252, 292, 265, 236, 245, 238, 250, 253, 257, 277, 227, 292, 260, 265, 253, 228, 249, 277, 254, 246, 254, 267, 284, 256, 246, 250, 209, 282, 238, 268, 253, 268, 273, 251, 253, 234, 263, 293, 274, 240, 251, 243, 256, 290, 257, 237, 258, 250, 244, 242, 235, 242, 256, 250, 233, 244, 271, 274, 239, 231, 260, 256, 278, 243, 273, 284, 267, 246, 273, 261, 217, 263, 269, 257, 261, 289, 266, 259, 253, 249, 235, 268, 251, 249, 256, 234, 244, 262, 283, 248, 262, 276, 261, 237, 241, 268, 213, 247, 250, 278, 239, 256, 253, 242, 254, 276, 252, 223, 266, 275, 240, 252, 269, 249, 273, 265] payload_rows_per_dst=[5524, 5516, 5500, 5388, 5518, 5486, 5418, 5484] dup_rows_per_dst=[2810, 2626, 2672, 2762, 2692, 2754, 2702, 2684] combine_rows_per_src=[5524, 5516, 5500, 5388, 5518, 5486, 5418, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229512 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384840 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540168 weight_movement_bytes=18874368 total_nvlink_bytes=333414536 weight_to_token_ratio=0.060006 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229680 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157049136 ingress_bytes=232316928 link_bytes=232316928 token_bytes=313868592 weight_movement_bytes=75497472 total_nvlink_bytes=389366064 weight_to_token_ratio=0.240538 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225946497e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38340 dup_rows=21688 dup_rows_remote=18814 dedup_ratio=0.330933 router_tokens_per_expert=[233, 254, 255, 245, 276, 271, 270, 242, 250, 241, 298, 280, 267, 292, 247, 250, 240, 264, 214, 278, 251, 258, 264, 264, 241, 252, 266, 244, 268, 255, 247, 282, 246, 271, 276, 265, 259, 267, 274, 255, 260, 246, 224, 261, 236, 263, 266, 243, 237, 239, 252, 274, 245, 267, 247, 255, 247, 279, 261, 272, 277, 252, 234, 250, 228, 273, 255, 255, 257, 259, 241, 275, 267, 257, 260, 268, 240, 239, 238, 242, 259, 278, 255, 245, 264, 263, 263, 251, 251, 284, 239, 262, 242, 260, 254, 239, 246, 282, 245, 266, 265, 288, 236, 267, 293, 231, 250, 272, 248, 271, 247, 241, 234, 240, 279, 219, 264, 265, 270, 250, 227, 232, 262, 264, 274, 238, 233, 247] payload_rows_per_dst=[5508, 5510, 5456, 5468, 5406, 5532, 5562, 5406] dup_rows_per_dst=[2874, 2742, 2710, 2708, 2702, 2628, 2734, 2590] combine_rows_per_src=[5508, 5510, 5456, 5468, 5406, 5532, 5562, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229656 combine_payload_bytes=156803072 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157032728 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313835800 weight_movement_bytes=56623104 total_nvlink_bytes=370458904 weight_to_token_ratio=0.180423 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228776 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156745128 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313261480 weight_movement_bytes=18874368 total_nvlink_bytes=332135848 weight_to_token_ratio=0.060251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229544 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157704360 ingress_bytes=195223552 link_bytes=195223552 token_bytes=315179176 weight_movement_bytes=37748736 total_nvlink_bytes=352927912 weight_to_token_ratio=0.119769 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870536019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228960 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179488 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130016 weight_movement_bytes=18874368 total_nvlink_bytes=333004384 weight_to_token_ratio=0.060085 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=228616 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157269256 ingress_bytes=213663744 link_bytes=213663744 token_bytes=314309896 weight_movement_bytes=56623104 total_nvlink_bytes=370933000 weight_to_token_ratio=0.180151 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047220866e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38358 dup_rows=21766 dup_rows_remote=19110 dedup_ratio=0.332123 router_tokens_per_expert=[278, 263, 255, 270, 276, 253, 253, 255, 252, 260, 273, 237, 271, 251, 234, 272, 250, 251, 264, 245, 274, 249, 253, 268, 243, 262, 237, 271, 220, 236, 249, 262, 242, 261, 280, 258, 255, 289, 261, 243, 271, 289, 252, 235, 273, 260, 241, 284, 264, 286, 252, 259, 260, 258, 235, 241, 258, 267, 272, 254, 263, 279, 255, 224, 273, 276, 247, 214, 269, 277, 265, 264, 257, 232, 266, 241, 244, 291, 258, 246, 272, 265, 239, 237, 211, 227, 235, 254, 265, 237, 288, 264, 232, 238, 273, 254, 254, 222, 276, 272, 242, 243, 262, 267, 242, 241, 269, 247, 255, 281, 266, 260, 261, 228, 256, 249, 252, 259, 279, 237, 255, 245, 250, 246, 279, 256, 250, 248] payload_rows_per_dst=[5520, 5412, 5558, 5472, 5510, 5350, 5510, 5438] dup_rows_per_dst=[2786, 2656, 2830, 2782, 2730, 2632, 2688, 2662] combine_rows_per_src=[5520, 5412, 5558, 5472, 5510, 5350, 5510, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38182 dup_rows=21860 dup_rows_remote=19192 dedup_ratio=0.333557 router_tokens_per_expert=[250, 261, 254, 234, 235, 265, 273, 232, 266, 255, 246, 267, 263, 239, 235, 260, 268, 269, 269, 259, 279, 267, 285, 241, 252, 260, 253, 242, 268, 267, 261, 242, 250, 269, 267, 248, 222, 266, 247, 258, 266, 250, 255, 254, 282, 252, 245, 250, 244, 245, 256, 272, 264, 269, 255, 252, 244, 266, 286, 250, 252, 263, 234, 240, 250, 247, 278, 245, 259, 232, 248, 240, 276, 241, 263, 242, 260, 235, 257, 241, 243, 260, 264, 253, 237, 292, 254, 242, 237, 280, 253, 249, 247, 267, 239, 253, 276, 248, 234, 258, 267, 250, 250, 247, 285, 238, 250, 244, 266, 261, 267, 240, 248, 254, 278, 259, 238, 290, 267, 275, 270, 262, 260, 253, 264, 268, 249, 278] payload_rows_per_dst=[5396, 5542, 5450, 5480, 5380, 5382, 5494, 5552] dup_rows_per_dst=[2674, 2822, 2702, 2704, 2658, 2758, 2668, 2874] combine_rows_per_src=[5396, 5542, 5450, 5480, 5380, 5382, 5494, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38310 dup_rows=21728 dup_rows_remote=19076 dedup_ratio=0.331543 router_tokens_per_expert=[280, 247, 268, 263, 258, 267, 249, 279, 260, 267, 269, 245, 222, 255, 275, 266, 261, 268, 250, 261, 281, 260, 246, 257, 250, 256, 249, 254, 243, 253, 284, 252, 251, 275, 261, 271, 241, 271, 268, 251, 268, 293, 271, 266, 237, 275, 248, 271, 248, 285, 270, 288, 251, 242, 248, 250, 238, 270, 243, 264, 235, 245, 290, 296, 243, 258, 239, 267, 223, 242, 251, 245, 237, 244, 247, 220, 240, 284, 251, 278, 256, 232, 273, 246, 252, 263, 240, 223, 254, 286, 262, 240, 274, 275, 250, 249, 256, 234, 240, 272, 243, 254, 264, 216, 261, 233, 258, 272, 240, 247, 250, 255, 231, 236, 264, 282, 255, 269, 245, 259, 260, 244, 231, 278, 249, 268, 244, 238] payload_rows_per_dst=[5534, 5522, 5578, 5576, 5344, 5498, 5336, 5420] dup_rows_per_dst=[2806, 2728, 2858, 2750, 2594, 2652, 2654, 2686] combine_rows_per_src=[5534, 5522, 5578, 5576, 5344, 5498, 5336, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229872 combine_payload_bytes=157114368 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157344240 ingress_bytes=232611840 link_bytes=232611840 token_bytes=314458608 weight_movement_bytes=75497472 total_nvlink_bytes=389956080 weight_to_token_ratio=0.240087 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228772199e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38058 dup_rows=21938 dup_rows_remote=19062 dedup_ratio=0.334747 router_tokens_per_expert=[262, 248, 246, 259, 274, 261, 260, 247, 260, 260, 231, 254, 265, 243, 246, 268, 254, 236, 233, 261, 268, 223, 307, 275, 258, 263, 267, 246, 239, 268, 266, 246, 255, 260, 239, 247, 295, 268, 283, 272, 276, 270, 253, 252, 281, 253, 256, 248, 222, 256, 258, 268, 293, 228, 263, 238, 262, 240, 267, 246, 279, 249, 237, 226, 254, 254, 232, 259, 236, 273, 270, 260, 242, 258, 249, 246, 246, 228, 259, 264, 232, 269, 253, 238, 270, 249, 216, 266, 297, 248, 225, 246, 246, 268, 269, 275, 304, 250, 258, 259, 249, 258, 235, 249, 235, 270, 258, 238, 280, 255, 258, 300, 253, 247, 266, 256, 265, 273, 275, 235, 235, 276, 248, 246, 229, 257, 267, 253] payload_rows_per_dst=[5420, 5452, 5540, 5428, 5404, 5416, 5468, 5470] dup_rows_per_dst=[2748, 2768, 2876, 2636, 2656, 2718, 2844, 2692] combine_rows_per_src=[5420, 5452, 5540, 5428, 5404, 5416, 5468, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38352 dup_rows=21708 dup_rows_remote=19072 dedup_ratio=0.331238 router_tokens_per_expert=[272, 252, 249, 235, 225, 252, 247, 227, 240, 271, 244, 253, 241, 265, 272, 249, 251, 245, 262, 260, 258, 267, 269, 272, 282, 272, 252, 290, 255, 238, 258, 274, 278, 238, 240, 255, 254, 226, 256, 254, 247, 258, 250, 235, 239, 268, 258, 269, 257, 245, 253, 270, 281, 257, 271, 238, 239, 257, 247, 256, 233, 228, 255, 269, 234, 261, 240, 265, 259, 272, 253, 282, 249, 245, 254, 273, 246, 244, 255, 253, 256, 253, 249, 231, 257, 281, 259, 241, 240, 253, 251, 263, 234, 282, 256, 236, 252, 265, 237, 254, 273, 271, 273, 278, 281, 251, 275, 244, 249, 267, 269, 272, 264, 271, 244, 272, 253, 265, 273, 249, 248, 270, 252, 274, 268, 223, 264, 260] payload_rows_per_dst=[5416, 5574, 5424, 5476, 5406, 5424, 5548, 5560] dup_rows_per_dst=[2572, 2836, 2626, 2636, 2764, 2660, 2874, 2740] combine_rows_per_src=[5416, 5574, 5424, 5476, 5406, 5424, 5548, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43962 payload_rows_remote=38400 dup_rows=21574 dup_rows_remote=18960 dedup_ratio=0.329193 router_tokens_per_expert=[273, 268, 251, 262, 236, 273, 231, 254, 272, 283, 264, 221, 250, 260, 268, 258, 254, 258, 249, 241, 266, 254, 254, 272, 276, 273, 272, 236, 271, 263, 259, 225, 292, 285, 261, 254, 248, 268, 235, 263, 261, 239, 253, 260, 233, 260, 262, 232, 256, 276, 274, 250, 268, 265, 242, 258, 287, 251, 261, 289, 278, 284, 225, 238, 239, 259, 285, 229, 242, 258, 247, 275, 291, 261, 264, 250, 267, 227, 276, 255, 259, 259, 266, 249, 234, 256, 240, 252, 252, 241, 264, 239, 251, 253, 257, 250, 254, 246, 256, 249, 275, 252, 239, 253, 250, 233, 271, 260, 239, 214, 250, 259, 271, 253, 254, 248, 244, 236, 249, 255, 245, 263, 281, 255, 245, 257, 238, 272] payload_rows_per_dst=[5402, 5426, 5310, 5384, 5758, 5608, 5512, 5562] dup_rows_per_dst=[2706, 2602, 2472, 2714, 3062, 2752, 2652, 2614] combine_rows_per_src=[5402, 5426, 5310, 5384, 5758, 5608, 5512, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38380 dup_rows=21766 dup_rows_remote=19114 dedup_ratio=0.332123 router_tokens_per_expert=[270, 255, 253, 266, 244, 240, 280, 307, 231, 250, 270, 250, 249, 229, 278, 279, 243, 247, 279, 280, 261, 247, 289, 260, 261, 231, 234, 257, 244, 252, 255, 284, 267, 237, 243, 277, 255, 264, 248, 266, 259, 231, 263, 272, 260, 249, 262, 232, 256, 283, 256, 258, 242, 276, 247, 239, 245, 230, 282, 283, 274, 246, 247, 255, 241, 252, 262, 242, 243, 246, 243, 271, 252, 261, 255, 275, 252, 228, 252, 246, 262, 258, 262, 260, 241, 249, 268, 264, 248, 243, 258, 273, 249, 227, 246, 266, 263, 250, 241, 272, 256, 231, 263, 267, 263, 268, 265, 263, 241, 263, 280, 225, 262, 265, 265, 251, 269, 282, 237, 250, 254, 245, 249, 243, 281, 229, 245, 256] payload_rows_per_dst=[5590, 5532, 5464, 5454, 5390, 5460, 5484, 5396] dup_rows_per_dst=[2712, 2716, 2706, 2784, 2652, 2688, 2738, 2770] combine_rows_per_src=[5590, 5532, 5464, 5454, 5390, 5460, 5484, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229496 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622968 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313016440 weight_movement_bytes=18874368 total_nvlink_bytes=331890808 weight_to_token_ratio=0.060298 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229544 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157147304 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314065064 weight_movement_bytes=18874368 total_nvlink_bytes=332939432 weight_to_token_ratio=0.060097 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38298 dup_rows=21630 dup_rows_remote=18846 dedup_ratio=0.330048 router_tokens_per_expert=[286, 284, 239, 261, 247, 265, 260, 248, 245, 263, 267, 241, 275, 269, 240, 260, 243, 234, 269, 243, 252, 262, 269, 259, 248, 235, 228, 246, 268, 231, 249, 260, 274, 288, 273, 269, 265, 265, 273, 262, 262, 263, 274, 239, 250, 263, 267, 231, 251, 244, 265, 253, 264, 235, 272, 253, 275, 256, 252, 227, 249, 236, 240, 258, 269, 268, 212, 229, 264, 254, 256, 266, 278, 270, 254, 226, 266, 245, 273, 265, 227, 249, 274, 235, 263, 264, 241, 250, 239, 285, 252, 255, 258, 267, 261, 259, 261, 268, 279, 261, 239, 224, 293, 225, 272, 258, 254, 281, 272, 264, 272, 268, 273, 232, 238, 232, 243, 260, 238, 228, 267, 253, 283, 245, 267, 268, 228, 254] payload_rows_per_dst=[5608, 5406, 5602, 5418, 5426, 5494, 5560, 5392] dup_rows_per_dst=[2784, 2610, 2872, 2642, 2610, 2664, 2822, 2626] combine_rows_per_src=[5608, 5406, 5602, 5418, 5426, 5494, 5560, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=228480 combine_payload_bytes=155885568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156114048 ingress_bytes=193634304 link_bytes=193634304 token_bytes=311999616 weight_movement_bytes=37748736 total_nvlink_bytes=349748352 weight_to_token_ratio=0.120990 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855308627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229696 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157319488 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314409280 weight_movement_bytes=18874368 total_nvlink_bytes=333283648 weight_to_token_ratio=0.060031 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229440 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157515840 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802240 weight_movement_bytes=18874368 total_nvlink_bytes=333676608 weight_to_token_ratio=0.059956 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229976 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434456 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638936 weight_movement_bytes=18874368 total_nvlink_bytes=333513304 weight_to_token_ratio=0.059987 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=228576 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097184 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313965792 weight_movement_bytes=18874368 total_nvlink_bytes=332840160 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38098 dup_rows=21934 dup_rows_remote=19132 dedup_ratio=0.334686 router_tokens_per_expert=[260, 227, 250, 256, 291, 286, 238, 224, 267, 255, 251, 242, 287, 255, 269, 242, 261, 261, 226, 268, 274, 243, 243, 271, 265, 273, 275, 270, 255, 261, 270, 237, 249, 232, 254, 272, 257, 223, 255, 300, 241, 273, 271, 263, 235, 239, 255, 238, 231, 264, 217, 268, 252, 273, 262, 241, 237, 268, 264, 268, 246, 260, 255, 256, 238, 231, 283, 245, 243, 279, 256, 260, 255, 236, 254, 269, 237, 258, 257, 253, 250, 254, 249, 242, 255, 259, 266, 240, 246, 262, 255, 264, 277, 242, 262, 270, 259, 266, 258, 262, 256, 269, 269, 279, 251, 256, 272, 248, 236, 297, 250, 261, 240, 260, 253, 256, 277, 274, 249, 233, 245, 234, 258, 245, 258, 263, 259, 256] payload_rows_per_dst=[5452, 5504, 5444, 5430, 5420, 5410, 5508, 5434] dup_rows_per_dst=[2748, 2802, 2670, 2694, 2688, 2776, 2870, 2686] combine_rows_per_src=[5452, 5504, 5444, 5430, 5420, 5410, 5508, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38260 dup_rows=21766 dup_rows_remote=18940 dedup_ratio=0.332123 router_tokens_per_expert=[223, 260, 250, 247, 244, 288, 270, 233, 252, 251, 250, 270, 261, 235, 258, 267, 259, 261, 275, 235, 261, 264, 270, 267, 282, 266, 274, 256, 252, 246, 250, 268, 266, 255, 258, 278, 271, 256, 266, 283, 247, 275, 272, 274, 279, 263, 247, 277, 244, 243, 272, 251, 228, 267, 252, 252, 242, 258, 243, 267, 229, 242, 254, 251, 245, 254, 234, 255, 261, 279, 258, 256, 235, 213, 256, 249, 247, 222, 265, 260, 275, 267, 240, 265, 223, 234, 274, 249, 239, 232, 263, 222, 270, 244, 278, 248, 260, 289, 248, 260, 273, 256, 243, 269, 242, 278, 245, 247, 280, 258, 249, 271, 286, 263, 251, 274, 226, 239, 244, 252, 268, 250, 250, 245, 255, 253, 264, 261] payload_rows_per_dst=[5532, 5550, 5612, 5358, 5370, 5370, 5510, 5468] dup_rows_per_dst=[2586, 2792, 2952, 2632, 2608, 2676, 2826, 2694] combine_rows_per_src=[5532, 5550, 5612, 5358, 5370, 5370, 5510, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38424 dup_rows=21660 dup_rows_remote=18904 dedup_ratio=0.330505 router_tokens_per_expert=[277, 279, 242, 245, 266, 264, 242, 273, 265, 261, 235, 244, 235, 295, 259, 254, 232, 244, 249, 272, 246, 251, 266, 260, 233, 259, 263, 263, 263, 241, 281, 253, 256, 253, 283, 252, 229, 270, 238, 232, 257, 246, 259, 265, 262, 248, 236, 254, 246, 255, 266, 247, 257, 242, 241, 229, 282, 277, 263, 247, 252, 268, 284, 253, 246, 256, 256, 280, 258, 264, 264, 258, 246, 268, 246, 265, 263, 251, 246, 273, 268, 233, 259, 260, 233, 250, 279, 263, 254, 240, 278, 243, 251, 257, 283, 253, 240, 232, 259, 249, 265, 238, 246, 255, 277, 234, 225, 268, 222, 276, 285, 233, 238, 250, 271, 247, 270, 278, 269, 274, 269, 240, 264, 286, 268, 251, 241, 243] payload_rows_per_dst=[5536, 5320, 5464, 5570, 5546, 5452, 5460, 5528] dup_rows_per_dst=[2736, 2832, 2616, 2648, 2734, 2756, 2548, 2790] combine_rows_per_src=[5536, 5320, 5464, 5570, 5546, 5452, 5460, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228920 combine_payload_bytes=156049408 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156278328 ingress_bytes=212672512 link_bytes=212672512 token_bytes=312327736 weight_movement_bytes=56623104 total_nvlink_bytes=368950840 weight_to_token_ratio=0.181294 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037723369e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38284 dup_rows=21708 dup_rows_remote=18906 dedup_ratio=0.331238 router_tokens_per_expert=[289, 292, 244, 252, 253, 239, 292, 243, 240, 228, 247, 242, 260, 232, 263, 276, 248, 253, 265, 256, 256, 262, 261, 256, 236, 266, 228, 259, 227, 262, 273, 285, 272, 259, 290, 288, 260, 229, 270, 254, 253, 250, 253, 271, 256, 242, 260, 244, 240, 308, 230, 261, 272, 261, 268, 245, 274, 270, 264, 260, 241, 277, 257, 245, 242, 256, 253, 223, 242, 242, 275, 235, 275, 272, 245, 265, 259, 256, 258, 236, 273, 257, 244, 244, 249, 241, 245, 266, 297, 251, 224, 237, 246, 250, 236, 254, 231, 248, 270, 262, 256, 263, 250, 254, 286, 272, 241, 243, 265, 257, 247, 246, 266, 254, 264, 240, 245, 244, 273, 233, 257, 282, 258, 259, 257, 277, 267, 244] payload_rows_per_dst=[5438, 5542, 5512, 5544, 5410, 5390, 5448, 5544] dup_rows_per_dst=[2746, 2644, 2790, 2802, 2658, 2638, 2734, 2696] combine_rows_per_src=[5438, 5542, 5512, 5544, 5410, 5390, 5448, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38340 dup_rows=21726 dup_rows_remote=19098 dedup_ratio=0.331512 router_tokens_per_expert=[255, 240, 228, 269, 271, 283, 260, 254, 231, 279, 260, 247, 241, 262, 275, 264, 251, 253, 253, 236, 243, 240, 271, 241, 262, 240, 265, 278, 256, 260, 273, 262, 236, 260, 243, 246, 275, 241, 221, 298, 237, 245, 242, 274, 267, 242, 252, 270, 252, 228, 257, 249, 250, 280, 227, 223, 254, 261, 264, 288, 268, 236, 281, 246, 271, 280, 270, 232, 272, 252, 266, 227, 266, 276, 242, 279, 248, 255, 294, 252, 265, 243, 246, 242, 238, 261, 282, 235, 252, 260, 251, 230, 272, 260, 264, 288, 235, 232, 282, 231, 257, 257, 277, 249, 244, 266, 251, 273, 288, 243, 218, 264, 258, 279, 263, 270, 250, 247, 274, 250, 235, 272, 254, 240, 256, 252, 259, 255] payload_rows_per_dst=[5542, 5452, 5470, 5440, 5584, 5422, 5350, 5550] dup_rows_per_dst=[2696, 2716, 2628, 2688, 2780, 2756, 2784, 2678] combine_rows_per_src=[5542, 5452, 5470, 5440, 5584, 5422, 5350, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38290 dup_rows=21760 dup_rows_remote=19050 dedup_ratio=0.332031 router_tokens_per_expert=[268, 246, 281, 264, 249, 293, 285, 240, 262, 221, 251, 253, 241, 251, 243, 246, 281, 255, 241, 263, 272, 263, 240, 278, 240, 242, 238, 258, 244, 250, 265, 272, 258, 236, 233, 255, 243, 268, 251, 261, 257, 259, 258, 232, 263, 239, 248, 252, 279, 255, 253, 251, 258, 257, 252, 281, 223, 256, 273, 267, 260, 266, 281, 252, 262, 256, 261, 253, 258, 274, 247, 247, 272, 256, 257, 273, 262, 248, 265, 266, 271, 257, 270, 261, 263, 245, 261, 237, 252, 225, 239, 242, 259, 261, 236, 255, 275, 255, 288, 267, 237, 261, 251, 235, 244, 223, 262, 285, 252, 277, 264, 245, 229, 244, 246, 250, 275, 271, 227, 239, 270, 257, 267, 241, 254, 253, 286, 274] payload_rows_per_dst=[5352, 5266, 5550, 5654, 5610, 5356, 5502, 5486] dup_rows_per_dst=[2534, 2374, 2992, 2976, 2722, 2712, 2740, 2710] combine_rows_per_src=[5352, 5266, 5550, 5654, 5610, 5356, 5502, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38156 dup_rows=21820 dup_rows_remote=19082 dedup_ratio=0.332947 router_tokens_per_expert=[280, 230, 255, 254, 255, 263, 265, 255, 259, 257, 275, 251, 277, 238, 256, 265, 246, 259, 249, 265, 248, 291, 261, 283, 236, 247, 242, 282, 268, 253, 273, 250, 274, 245, 249, 274, 232, 262, 281, 247, 257, 235, 282, 262, 205, 257, 261, 241, 242, 254, 280, 249, 236, 250, 277, 247, 273, 224, 219, 230, 264, 262, 267, 234, 258, 259, 254, 252, 252, 304, 262, 232, 242, 244, 279, 288, 278, 232, 263, 250, 236, 266, 274, 248, 264, 268, 237, 270, 246, 224, 251, 266, 256, 240, 265, 251, 252, 252, 228, 267, 238, 255, 275, 249, 255, 244, 266, 268, 250, 269, 247, 264, 243, 252, 262, 248, 243, 247, 262, 282, 258, 263, 259, 275, 250, 253, 284, 237] payload_rows_per_dst=[5470, 5516, 5360, 5440, 5560, 5380, 5482, 5508] dup_rows_per_dst=[2800, 2790, 2768, 2576, 2738, 2744, 2676, 2728] combine_rows_per_src=[5470, 5516, 5360, 5440, 5560, 5380, 5482, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228800 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156941760 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313654720 weight_movement_bytes=37748736 total_nvlink_bytes=351403456 weight_to_token_ratio=0.120351 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38242 dup_rows=21870 dup_rows_remote=19080 dedup_ratio=0.333710 router_tokens_per_expert=[260, 256, 243, 226, 274, 251, 269, 265, 248, 248, 242, 273, 247, 236, 255, 243, 291, 266, 273, 252, 264, 284, 216, 257, 229, 220, 240, 248, 230, 271, 282, 269, 267, 274, 275, 284, 270, 251, 275, 246, 250, 236, 238, 298, 239, 252, 281, 307, 249, 244, 267, 278, 239, 258, 246, 259, 227, 235, 267, 246, 272, 277, 258, 248, 268, 246, 252, 272, 235, 283, 264, 280, 209, 242, 270, 258, 262, 241, 271, 255, 267, 243, 256, 275, 278, 243, 272, 285, 272, 256, 276, 266, 260, 260, 237, 243, 214, 241, 247, 235, 264, 236, 239, 257, 268, 275, 247, 275, 265, 242, 246, 277, 260, 252, 270, 248, 254, 260, 215, 267, 260, 248, 253, 236, 247, 239, 231, 262] payload_rows_per_dst=[5424, 5470, 5590, 5546, 5478, 5272, 5444, 5442] dup_rows_per_dst=[2790, 2742, 2896, 3054, 2738, 2476, 2612, 2562] combine_rows_per_src=[5424, 5470, 5590, 5546, 5478, 5272, 5444, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157384704 dispatch_meta_bytes=229312 combine_payload_bytes=157384704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157614016 ingress_bytes=176259072 link_bytes=176259072 token_bytes=314998720 weight_movement_bytes=18874368 total_nvlink_bytes=333873088 weight_to_token_ratio=0.059919 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688827703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228760 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157040024 ingress_bytes=213434368 link_bytes=213434368 token_bytes=313851288 weight_movement_bytes=56623104 total_nvlink_bytes=370474392 weight_to_token_ratio=0.180414 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045023098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229752 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270392 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311032 weight_movement_bytes=18874368 total_nvlink_bytes=333185400 weight_to_token_ratio=0.060050 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229360 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157065200 ingress_bytes=194584576 link_bytes=194584576 token_bytes=313901040 weight_movement_bytes=37748736 total_nvlink_bytes=351649776 weight_to_token_ratio=0.120257 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864413666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228952 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156515928 ingress_bytes=194035712 link_bytes=194035712 token_bytes=312802904 weight_movement_bytes=37748736 total_nvlink_bytes=350551640 weight_to_token_ratio=0.120679 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859154721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229288 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868520 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507752 weight_movement_bytes=18874368 total_nvlink_bytes=332382120 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38284 dup_rows=21794 dup_rows_remote=19032 dedup_ratio=0.332550 router_tokens_per_expert=[265, 253, 246, 270, 264, 251, 273, 214, 255, 290, 270, 248, 250, 277, 263, 244, 269, 244, 261, 250, 256, 244, 272, 251, 275, 275, 244, 254, 231, 254, 257, 273, 254, 264, 251, 248, 278, 246, 270, 249, 222, 257, 247, 271, 256, 240, 277, 271, 250, 282, 263, 264, 226, 258, 263, 245, 245, 252, 286, 248, 249, 239, 246, 253, 231, 238, 262, 270, 252, 249, 259, 264, 298, 245, 254, 278, 254, 233, 283, 248, 251, 236, 260, 253, 265, 252, 277, 248, 258, 250, 244, 262, 257, 262, 290, 254, 244, 251, 247, 239, 237, 297, 241, 230, 244, 256, 273, 246, 235, 250, 247, 270, 261, 279, 252, 259, 271, 240, 275, 243, 236, 281, 259, 250, 234, 270, 261, 240] payload_rows_per_dst=[5536, 5458, 5518, 5442, 5382, 5486, 5408, 5512] dup_rows_per_dst=[2730, 2762, 2684, 2696, 2854, 2752, 2606, 2710] combine_rows_per_src=[5536, 5458, 5518, 5442, 5382, 5486, 5408, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38160 dup_rows=21968 dup_rows_remote=19146 dedup_ratio=0.335205 router_tokens_per_expert=[258, 268, 262, 277, 247, 259, 238, 246, 256, 251, 250, 243, 250, 286, 244, 266, 243, 254, 258, 235, 257, 276, 272, 232, 244, 267, 252, 258, 264, 246, 269, 227, 247, 274, 280, 273, 256, 259, 266, 268, 227, 240, 253, 255, 259, 259, 262, 273, 255, 239, 251, 249, 239, 278, 253, 220, 266, 253, 264, 259, 263, 266, 281, 263, 262, 253, 297, 247, 275, 289, 266, 262, 257, 268, 254, 254, 249, 270, 245, 248, 252, 242, 262, 276, 255, 255, 260, 231, 246, 252, 276, 242, 231, 214, 292, 246, 261, 279, 239, 246, 256, 252, 253, 260, 260, 286, 275, 235, 265, 260, 246, 242, 250, 273, 217, 260, 242, 259, 250, 244, 262, 272, 240, 251, 254, 246, 253, 247] payload_rows_per_dst=[5458, 5358, 5496, 5522, 5574, 5312, 5408, 5440] dup_rows_per_dst=[2744, 2750, 2806, 2676, 2818, 2752, 2822, 2600] combine_rows_per_src=[5458, 5358, 5496, 5522, 5574, 5312, 5408, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38288 dup_rows=21878 dup_rows_remote=19216 dedup_ratio=0.333832 router_tokens_per_expert=[244, 268, 273, 269, 226, 251, 257, 252, 253, 277, 273, 259, 256, 243, 243, 225, 263, 273, 254, 241, 233, 255, 263, 238, 253, 254, 270, 243, 251, 236, 260, 252, 277, 276, 236, 261, 256, 289, 228, 267, 276, 257, 278, 250, 257, 252, 248, 283, 230, 295, 249, 254, 240, 259, 254, 270, 258, 263, 254, 230, 262, 269, 273, 252, 254, 267, 278, 252, 243, 246, 245, 285, 274, 236, 261, 273, 219, 258, 245, 222, 257, 256, 255, 258, 247, 264, 263, 242, 263, 246, 241, 237, 246, 238, 259, 244, 245, 263, 235, 243, 273, 243, 269, 265, 282, 264, 246, 256, 259, 271, 257, 265, 271, 268, 255, 272, 250, 245, 274, 269, 258, 238, 257, 259, 256, 245, 262, 268] payload_rows_per_dst=[5366, 5438, 5616, 5468, 5424, 5370, 5506, 5470] dup_rows_per_dst=[2772, 2640, 2766, 2756, 2692, 2662, 2766, 2824] combine_rows_per_src=[5366, 5438, 5616, 5468, 5424, 5370, 5506, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38318 dup_rows=21778 dup_rows_remote=19024 dedup_ratio=0.332306 router_tokens_per_expert=[266, 248, 253, 234, 281, 248, 242, 274, 242, 242, 268, 251, 273, 253, 235, 312, 263, 261, 248, 263, 249, 247, 303, 261, 281, 259, 254, 233, 260, 259, 238, 276, 273, 264, 249, 267, 270, 264, 231, 254, 264, 266, 241, 281, 282, 230, 246, 225, 248, 283, 246, 259, 242, 262, 254, 262, 245, 279, 216, 284, 259, 266, 244, 248, 275, 256, 268, 229, 225, 269, 255, 248, 236, 243, 273, 253, 272, 253, 267, 244, 266, 269, 268, 249, 277, 240, 259, 263, 243, 244, 259, 287, 235, 267, 246, 256, 255, 259, 255, 239, 269, 261, 240, 229, 249, 273, 244, 245, 262, 263, 232, 246, 288, 239, 246, 258, 258, 243, 277, 235, 228, 282, 259, 249, 239, 273, 259, 239] payload_rows_per_dst=[5434, 5568, 5384, 5440, 5480, 5560, 5442, 5450] dup_rows_per_dst=[2810, 2742, 2830, 2754, 2652, 2696, 2600, 2694] combine_rows_per_src=[5434, 5568, 5384, 5440, 5480, 5560, 5442, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229264 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040528 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851792 weight_movement_bytes=18874368 total_nvlink_bytes=332726160 weight_to_token_ratio=0.060138 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43964 payload_rows_remote=38466 dup_rows=21572 dup_rows_remote=18878 dedup_ratio=0.329163 router_tokens_per_expert=[256, 243, 240, 265, 281, 265, 276, 255, 266, 249, 250, 266, 253, 259, 264, 242, 259, 254, 228, 244, 229, 266, 241, 237, 264, 255, 246, 235, 248, 279, 236, 255, 272, 260, 274, 245, 251, 237, 251, 277, 239, 292, 269, 247, 248, 252, 225, 257, 280, 258, 246, 253, 264, 261, 288, 249, 283, 252, 286, 250, 276, 260, 246, 226, 228, 266, 260, 240, 241, 267, 252, 239, 250, 275, 238, 251, 255, 249, 270, 229, 262, 267, 262, 261, 234, 258, 196, 254, 267, 275, 239, 249, 262, 269, 271, 249, 255, 257, 269, 275, 277, 272, 225, 260, 303, 248, 246, 264, 262, 262, 251, 273, 265, 252, 268, 242, 256, 254, 257, 272, 261, 256, 232, 247, 259, 252, 258, 273] payload_rows_per_dst=[5536, 5416, 5498, 5580, 5372, 5472, 5558, 5532] dup_rows_per_dst=[2724, 2536, 2694, 2776, 2648, 2678, 2840, 2676] combine_rows_per_src=[5536, 5416, 5498, 5580, 5372, 5472, 5558, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=37934 dup_rows=21988 dup_rows_remote=19264 dedup_ratio=0.335510 router_tokens_per_expert=[260, 257, 256, 267, 238, 238, 247, 279, 264, 222, 284, 254, 233, 272, 276, 267, 241, 293, 271, 247, 240, 255, 272, 241, 249, 260, 269, 278, 268, 277, 252, 245, 254, 257, 242, 246, 270, 286, 237, 242, 235, 265, 256, 287, 255, 238, 275, 234, 224, 250, 252, 215, 234, 255, 244, 265, 268, 253, 244, 267, 216, 244, 242, 256, 224, 268, 274, 289, 242, 264, 244, 263, 255, 265, 257, 245, 250, 240, 252, 284, 246, 255, 238, 243, 246, 276, 265, 234, 271, 257, 267, 282, 248, 236, 263, 260, 277, 251, 272, 256, 258, 258, 271, 296, 220, 243, 265, 270, 263, 273, 260, 230, 237, 266, 257, 254, 262, 261, 260, 251, 255, 269, 244, 259, 261, 266, 263, 257] payload_rows_per_dst=[5478, 5500, 5398, 5138, 5490, 5512, 5418, 5614] dup_rows_per_dst=[2750, 2816, 2760, 2452, 2742, 2930, 2814, 2724] combine_rows_per_src=[5478, 5500, 5398, 5138, 5490, 5512, 5418, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38354 dup_rows=21668 dup_rows_remote=18984 dedup_ratio=0.330627 router_tokens_per_expert=[258, 254, 255, 256, 266, 248, 243, 287, 244, 250, 266, 256, 271, 267, 251, 242, 242, 286, 249, 238, 226, 231, 249, 239, 250, 259, 251, 279, 254, 244, 271, 301, 266, 251, 249, 245, 246, 227, 266, 258, 230, 247, 251, 252, 231, 260, 244, 239, 279, 250, 266, 273, 246, 270, 280, 274, 250, 260, 290, 261, 265, 256, 233, 247, 248, 229, 255, 254, 260, 253, 259, 256, 255, 254, 268, 254, 251, 266, 259, 278, 249, 240, 279, 257, 288, 253, 262, 244, 266, 257, 257, 282, 247, 258, 258, 273, 228, 252, 245, 288, 242, 252, 279, 250, 262, 261, 255, 271, 275, 250, 250, 249, 225, 260, 258, 243, 242, 241, 224, 257, 264, 291, 242, 278, 236, 243, 267, 274] payload_rows_per_dst=[5520, 5480, 5358, 5524, 5514, 5528, 5476, 5468] dup_rows_per_dst=[2708, 2658, 2566, 2876, 2684, 2812, 2742, 2622] combine_rows_per_src=[5520, 5480, 5358, 5524, 5514, 5528, 5476, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229224 combine_payload_bytes=156303360 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156532584 ingress_bytes=212926464 link_bytes=212926464 token_bytes=312835944 weight_movement_bytes=56623104 total_nvlink_bytes=369459048 weight_to_token_ratio=0.180999 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040156612e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38534 dup_rows=21714 dup_rows_remote=19094 dedup_ratio=0.331329 router_tokens_per_expert=[258, 243, 248, 242, 267, 251, 241, 248, 256, 231, 227, 239, 257, 245, 265, 229, 264, 258, 262, 274, 262, 282, 259, 234, 250, 280, 250, 261, 231, 263, 233, 252, 265, 279, 249, 248, 272, 243, 260, 255, 240, 248, 232, 235, 265, 224, 262, 233, 267, 277, 294, 229, 257, 250, 251, 242, 255, 269, 251, 247, 272, 270, 255, 225, 255, 275, 253, 253, 277, 270, 243, 221, 236, 276, 258, 252, 254, 267, 259, 245, 261, 236, 282, 255, 276, 274, 252, 237, 253, 268, 264, 263, 269, 249, 280, 272, 265, 281, 276, 256, 266, 236, 281, 280, 253, 279, 273, 246, 223, 272, 254, 243, 277, 251, 258, 239, 286, 261, 247, 265, 262, 220, 281, 254, 250, 263, 242, 260] payload_rows_per_dst=[5288, 5510, 5402, 5484, 5530, 5592, 5504, 5512] dup_rows_per_dst=[2620, 2706, 2618, 2738, 2658, 2790, 2864, 2720] combine_rows_per_src=[5288, 5510, 5402, 5484, 5530, 5592, 5504, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=230016 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057664 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885312 weight_movement_bytes=18874368 total_nvlink_bytes=332759680 weight_to_token_ratio=0.060131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229368 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179896 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130424 weight_movement_bytes=18874368 total_nvlink_bytes=333004792 weight_to_token_ratio=0.060084 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=229376 combine_payload_bytes=157556736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157786112 ingress_bytes=195305472 link_bytes=195305472 token_bytes=315342848 weight_movement_bytes=37748736 total_nvlink_bytes=353091584 weight_to_token_ratio=0.119707 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871320936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155377664 dispatch_meta_bytes=228792 combine_payload_bytes=155377664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155606456 ingress_bytes=174252032 link_bytes=174252032 token_bytes=310984120 weight_movement_bytes=18874368 total_nvlink_bytes=329858488 weight_to_token_ratio=0.060692 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669597233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229352 combine_payload_bytes=157097984 prefetch_bytes=0 reduce_bytes=0 egress_bytes=157327336 ingress_bytes=157097984 link_bytes=157327336 token_bytes=314425320 weight_movement_bytes=0 total_nvlink_bytes=314425320 weight_to_token_ratio=0.000000 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.507433124e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157835264 dispatch_meta_bytes=230512 combine_payload_bytes=157835264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158065776 ingress_bytes=195584000 link_bytes=195584000 token_bytes=315901040 weight_movement_bytes=37748736 total_nvlink_bytes=353649776 weight_to_token_ratio=0.119495 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873989655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38328 dup_rows=21730 dup_rows_remote=19102 dedup_ratio=0.331573 router_tokens_per_expert=[272, 271, 253, 255, 270, 251, 271, 262, 260, 273, 260, 270, 253, 261, 255, 250, 252, 263, 238, 240, 226, 240, 285, 258, 271, 230, 246, 268, 273, 268, 259, 236, 249, 238, 257, 256, 302, 275, 268, 247, 251, 255, 261, 243, 276, 249, 257, 244, 239, 239, 275, 266, 270, 260, 253, 260, 238, 254, 269, 254, 250, 293, 244, 263, 251, 260, 291, 280, 258, 258, 249, 217, 245, 222, 243, 255, 263, 275, 253, 283, 251, 250, 268, 256, 235, 278, 253, 269, 247, 249, 257, 264, 243, 248, 251, 252, 250, 236, 253, 251, 247, 234, 263, 232, 249, 260, 253, 270, 261, 255, 235, 282, 235, 241, 253, 254, 242, 264, 264, 247, 258, 266, 243, 250, 252, 269, 266, 264] payload_rows_per_dst=[5524, 5478, 5430, 5538, 5474, 5422, 5440, 5500] dup_rows_per_dst=[2850, 2628, 2826, 2716, 2732, 2720, 2622, 2636] combine_rows_per_src=[5524, 5478, 5430, 5538, 5474, 5422, 5440, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38108 dup_rows=21910 dup_rows_remote=19150 dedup_ratio=0.334320 router_tokens_per_expert=[235, 271, 258, 281, 254, 260, 254, 275, 253, 238, 260, 255, 248, 270, 240, 242, 279, 232, 239, 246, 270, 237, 244, 260, 256, 244, 249, 260, 258, 255, 240, 239, 250, 277, 238, 236, 255, 240, 257, 278, 252, 250, 277, 264, 267, 256, 240, 266, 256, 269, 246, 231, 264, 286, 249, 251, 234, 260, 247, 250, 243, 261, 256, 263, 269, 248, 237, 261, 245, 238, 270, 231, 268, 256, 269, 270, 239, 246, 269, 245, 248, 268, 264, 282, 256, 250, 228, 255, 284, 285, 252, 253, 268, 284, 235, 274, 267, 237, 266, 253, 255, 284, 251, 242, 273, 252, 238, 265, 281, 268, 245, 262, 223, 246, 292, 263, 260, 255, 233, 262, 247, 274, 272, 254, 255, 268, 236, 271] payload_rows_per_dst=[5502, 5338, 5378, 5456, 5480, 5490, 5518, 5464] dup_rows_per_dst=[2686, 2678, 2716, 2676, 2754, 2882, 2760, 2758] combine_rows_per_src=[5502, 5338, 5378, 5456, 5480, 5490, 5518, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38272 dup_rows=21768 dup_rows_remote=19074 dedup_ratio=0.332153 router_tokens_per_expert=[262, 255, 248, 255, 253, 242, 257, 263, 241, 243, 277, 248, 254, 265, 286, 267, 255, 260, 302, 271, 253, 236, 254, 277, 242, 270, 257, 249, 250, 249, 255, 249, 276, 284, 257, 262, 237, 264, 245, 254, 274, 254, 249, 265, 254, 278, 255, 240, 285, 267, 239, 279, 276, 246, 255, 229, 259, 264, 266, 241, 247, 265, 270, 233, 278, 239, 247, 248, 224, 253, 261, 245, 260, 239, 255, 261, 263, 248, 237, 247, 258, 236, 250, 273, 284, 237, 243, 249, 280, 247, 267, 242, 244, 259, 257, 269, 226, 260, 264, 266, 266, 272, 259, 224, 284, 284, 274, 267, 240, 234, 254, 254, 237, 248, 224, 268, 216, 257, 297, 257, 233, 254, 254, 271, 244, 247, 244, 275] payload_rows_per_dst=[5440, 5460, 5558, 5514, 5400, 5496, 5520, 5380] dup_rows_per_dst=[2792, 2798, 2738, 2728, 2610, 2694, 2736, 2672] combine_rows_per_src=[5440, 5460, 5558, 5514, 5400, 5496, 5520, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38346 dup_rows=21700 dup_rows_remote=19060 dedup_ratio=0.331116 router_tokens_per_expert=[231, 243, 248, 240, 256, 266, 252, 279, 233, 254, 244, 249, 256, 257, 266, 249, 268, 256, 228, 272, 267, 261, 264, 254, 290, 246, 270, 284, 227, 266, 253, 269, 247, 255, 259, 267, 265, 268, 255, 258, 238, 261, 247, 250, 258, 272, 258, 252, 263, 244, 240, 241, 261, 247, 267, 252, 253, 271, 270, 274, 248, 217, 280, 237, 269, 238, 266, 262, 254, 237, 258, 249, 248, 254, 223, 273, 234, 251, 275, 237, 271, 275, 217, 274, 275, 268, 228, 260, 247, 246, 256, 247, 247, 258, 258, 259, 279, 267, 263, 239, 262, 245, 266, 233, 248, 254, 257, 258, 250, 238, 285, 280, 270, 248, 250, 267, 279, 261, 249, 255, 263, 240, 270, 264, 244, 274, 253, 270] payload_rows_per_dst=[5410, 5508, 5426, 5490, 5500, 5458, 5552, 5492] dup_rows_per_dst=[2636, 2842, 2794, 2640, 2556, 2714, 2696, 2822] combine_rows_per_src=[5410, 5508, 5426, 5490, 5500, 5458, 5552, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38172 dup_rows=21818 dup_rows_remote=19146 dedup_ratio=0.332916 router_tokens_per_expert=[268, 216, 253, 247, 267, 252, 276, 259, 264, 242, 263, 220, 261, 261, 247, 267, 249, 238, 256, 247, 235, 246, 255, 268, 267, 227, 271, 277, 262, 226, 254, 266, 251, 232, 229, 260, 278, 267, 254, 254, 249, 257, 266, 245, 247, 257, 284, 279, 272, 237, 233, 256, 262, 257, 255, 258, 254, 257, 252, 251, 282, 277, 233, 238, 265, 247, 235, 259, 305, 254, 265, 250, 279, 260, 241, 237, 259, 244, 247, 242, 249, 270, 248, 237, 238, 264, 273, 270, 221, 251, 243, 258, 269, 240, 242, 258, 274, 266, 282, 260, 271, 230, 274, 261, 257, 239, 233, 267, 260, 236, 248, 267, 253, 257, 238, 250, 239, 262, 250, 289, 282, 259, 304, 288, 271, 276, 232, 283] payload_rows_per_dst=[5422, 5404, 5546, 5502, 5478, 5422, 5442, 5502] dup_rows_per_dst=[2704, 2684, 2672, 2646, 2700, 2640, 2808, 2964] combine_rows_per_src=[5422, 5404, 5546, 5502, 5478, 5422, 5442, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229720 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157221208 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212696 weight_movement_bytes=18874368 total_nvlink_bytes=333087064 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38244 dup_rows=21840 dup_rows_remote=19152 dedup_ratio=0.333252 router_tokens_per_expert=[267, 282, 245, 266, 273, 255, 272, 233, 258, 262, 244, 260, 278, 253, 295, 237, 248, 257, 240, 243, 267, 262, 261, 244, 281, 247, 257, 261, 253, 265, 255, 280, 247, 249, 270, 234, 237, 265, 269, 275, 280, 240, 242, 263, 232, 252, 241, 246, 266, 222, 252, 253, 285, 259, 245, 227, 247, 242, 242, 277, 255, 235, 249, 254, 272, 266, 262, 268, 255, 263, 248, 254, 257, 262, 230, 263, 260, 237, 261, 243, 258, 265, 252, 286, 258, 266, 250, 270, 274, 270, 269, 245, 218, 249, 257, 258, 246, 259, 240, 255, 248, 284, 267, 258, 252, 242, 258, 263, 252, 284, 258, 233, 259, 268, 243, 243, 222, 269, 244, 241, 280, 256, 253, 252, 255, 254, 259, 272] payload_rows_per_dst=[5488, 5462, 5306, 5382, 5594, 5550, 5462, 5452] dup_rows_per_dst=[2566, 2780, 2610, 2638, 3082, 2740, 2736, 2688] combine_rows_per_src=[5488, 5462, 5306, 5382, 5594, 5550, 5462, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=229032 combine_payload_bytes=156090368 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156319400 ingress_bytes=212713472 link_bytes=212713472 token_bytes=312409768 weight_movement_bytes=56623104 total_nvlink_bytes=369032872 weight_to_token_ratio=0.181246 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038115827e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38248 dup_rows=21822 dup_rows_remote=19194 dedup_ratio=0.332977 router_tokens_per_expert=[223, 240, 257, 264, 259, 264, 268, 252, 232, 256, 277, 251, 277, 261, 268, 256, 262, 254, 284, 289, 264, 252, 232, 272, 254, 252, 272, 247, 262, 258, 258, 239, 275, 263, 248, 250, 284, 236, 239, 269, 263, 242, 249, 243, 274, 267, 285, 281, 290, 243, 250, 223, 277, 247, 255, 248, 252, 240, 267, 268, 267, 233, 252, 258, 249, 251, 242, 254, 282, 256, 241, 245, 278, 267, 243, 246, 253, 252, 247, 241, 234, 239, 274, 258, 251, 259, 241, 255, 247, 247, 279, 256, 243, 233, 262, 261, 236, 239, 229, 284, 252, 250, 249, 268, 269, 263, 263, 264, 274, 267, 252, 279, 273, 245, 227, 263, 270, 242, 238, 254, 274, 242, 279, 246, 242, 228, 265, 262] payload_rows_per_dst=[5456, 5534, 5468, 5434, 5466, 5436, 5512, 5408] dup_rows_per_dst=[2754, 2768, 2868, 2706, 2628, 2642, 2764, 2692] combine_rows_per_src=[5456, 5534, 5468, 5434, 5466, 5436, 5512, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38144 dup_rows=21856 dup_rows_remote=18828 dedup_ratio=0.333496 router_tokens_per_expert=[287, 274, 271, 263, 263, 256, 260, 242, 263, 253, 258, 279, 257, 263, 266, 250, 296, 268, 251, 232, 259, 251, 239, 251, 255, 239, 240, 287, 246, 259, 262, 271, 232, 256, 240, 256, 257, 255, 249, 242, 242, 254, 252, 260, 241, 243, 245, 256, 243, 249, 258, 238, 272, 270, 254, 269, 246, 264, 260, 263, 267, 258, 279, 269, 257, 264, 265, 256, 231, 262, 249, 255, 279, 251, 266, 267, 228, 260, 231, 259, 274, 264, 272, 237, 283, 276, 238, 261, 211, 237, 268, 256, 254, 259, 275, 255, 251, 211, 257, 255, 267, 242, 263, 251, 239, 261, 260, 286, 284, 255, 252, 249, 236, 264, 236, 215, 267, 242, 253, 273, 240, 278, 254, 252, 268, 264, 254, 239] payload_rows_per_dst=[5536, 5402, 5454, 5528, 5368, 5656, 5358, 5378] dup_rows_per_dst=[3028, 2810, 2506, 2790, 2638, 2986, 2500, 2598] combine_rows_per_src=[5536, 5402, 5454, 5528, 5368, 5656, 5358, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229384 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156991496 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313753608 weight_movement_bytes=37748736 total_nvlink_bytes=351502344 weight_to_token_ratio=0.120313 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229624 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157294840 ingress_bytes=194813952 link_bytes=194813952 token_bytes=314360056 weight_movement_bytes=37748736 total_nvlink_bytes=352108792 weight_to_token_ratio=0.120081 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866611434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229272 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156581784 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312934296 weight_movement_bytes=37748736 total_nvlink_bytes=350683032 weight_to_token_ratio=0.120628 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229584 combine_payload_bytes=156647424 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156877008 ingress_bytes=156647424 link_bytes=156877008 token_bytes=313524432 weight_movement_bytes=0 total_nvlink_bytes=313524432 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.503118302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229768 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893576 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557384 weight_movement_bytes=18874368 total_nvlink_bytes=332431752 weight_to_token_ratio=0.060194 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=227888 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156465712 ingress_bytes=193986560 link_bytes=193986560 token_bytes=312703536 weight_movement_bytes=37748736 total_nvlink_bytes=350452272 weight_to_token_ratio=0.120717 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858683771e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38224 dup_rows=21778 dup_rows_remote=19072 dedup_ratio=0.332306 router_tokens_per_expert=[262, 265, 269, 227, 263, 259, 247, 268, 252, 243, 244, 276, 242, 248, 253, 250, 248, 249, 278, 267, 252, 245, 255, 266, 234, 286, 243, 275, 261, 240, 258, 263, 266, 233, 243, 250, 277, 275, 247, 254, 301, 268, 245, 259, 238, 265, 235, 298, 193, 246, 273, 262, 270, 266, 236, 258, 271, 250, 249, 277, 294, 262, 255, 276, 227, 246, 247, 271, 244, 280, 239, 266, 257, 247, 232, 252, 234, 247, 278, 282, 247, 238, 262, 267, 266, 246, 242, 252, 252, 238, 234, 269, 232, 267, 274, 265, 247, 256, 246, 248, 254, 242, 234, 246, 249, 285, 252, 248, 272, 257, 261, 269, 274, 252, 241, 263, 262, 246, 275, 245, 230, 261, 263, 251, 256, 256, 270, 277] payload_rows_per_dst=[5564, 5534, 5456, 5448, 5312, 5428, 5478, 5538] dup_rows_per_dst=[2884, 2706, 2852, 2828, 2474, 2674, 2654, 2706] combine_rows_per_src=[5564, 5534, 5456, 5448, 5312, 5428, 5478, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38318 dup_rows=21682 dup_rows_remote=18982 dedup_ratio=0.330841 router_tokens_per_expert=[271, 262, 245, 245, 261, 264, 248, 235, 242, 242, 238, 245, 280, 260, 245, 261, 259, 249, 253, 261, 260, 273, 249, 247, 222, 254, 230, 256, 275, 244, 258, 260, 245, 269, 278, 243, 251, 253, 281, 269, 277, 233, 261, 269, 230, 257, 256, 254, 252, 275, 257, 265, 275, 271, 262, 265, 256, 250, 226, 245, 248, 245, 244, 277, 283, 252, 267, 253, 269, 255, 269, 221, 262, 264, 268, 258, 279, 256, 289, 285, 257, 244, 244, 254, 277, 221, 240, 254, 277, 257, 246, 257, 244, 287, 255, 250, 227, 276, 264, 263, 269, 269, 244, 247, 259, 268, 265, 257, 252, 262, 249, 247, 225, 259, 246, 247, 250, 261, 256, 259, 259, 260, 230, 259, 269, 236, 257, 250] payload_rows_per_dst=[5420, 5436, 5482, 5532, 5600, 5432, 5536, 5416] dup_rows_per_dst=[2668, 2664, 2770, 2694, 2860, 2696, 2700, 2630] combine_rows_per_src=[5420, 5436, 5482, 5532, 5600, 5432, 5536, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38300 dup_rows=21722 dup_rows_remote=18990 dedup_ratio=0.331451 router_tokens_per_expert=[272, 232, 251, 233, 225, 255, 228, 249, 241, 264, 275, 250, 259, 269, 270, 277, 281, 251, 256, 271, 272, 258, 247, 260, 251, 248, 276, 302, 249, 259, 246, 269, 237, 256, 247, 254, 248, 241, 238, 244, 245, 263, 239, 261, 284, 251, 262, 244, 269, 255, 235, 269, 257, 217, 230, 251, 297, 268, 247, 253, 267, 261, 252, 284, 288, 218, 247, 267, 280, 256, 232, 240, 262, 274, 255, 270, 263, 250, 267, 259, 224, 268, 273, 225, 251, 254, 269, 245, 240, 256, 270, 273, 257, 267, 267, 284, 236, 282, 247, 253, 274, 248, 234, 234, 223, 240, 264, 268, 271, 278, 240, 231, 270, 262, 256, 235, 238, 259, 264, 267, 255, 253, 256, 267, 255, 263, 251, 271] payload_rows_per_dst=[5496, 5546, 5412, 5508, 5406, 5514, 5396, 5536] dup_rows_per_dst=[2604, 2846, 2616, 2716, 2850, 2732, 2650, 2708] combine_rows_per_src=[5496, 5546, 5412, 5508, 5406, 5514, 5396, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38152 dup_rows=21808 dup_rows_remote=18982 dedup_ratio=0.332764 router_tokens_per_expert=[303, 229, 262, 272, 245, 251, 270, 252, 243, 260, 229, 258, 255, 249, 275, 246, 278, 257, 261, 243, 243, 257, 227, 263, 225, 256, 232, 269, 247, 270, 261, 254, 252, 254, 256, 243, 232, 258, 237, 277, 239, 248, 251, 255, 275, 284, 286, 253, 266, 274, 244, 286, 242, 262, 268, 259, 239, 271, 285, 226, 266, 287, 251, 275, 254, 265, 243, 246, 275, 247, 229, 262, 249, 237, 284, 258, 259, 261, 277, 286, 237, 239, 240, 245, 254, 242, 264, 272, 265, 263, 234, 255, 260, 266, 232, 251, 256, 223, 274, 279, 258, 234, 265, 261, 224, 238, 232, 259, 251, 220, 242, 247, 272, 278, 282, 261, 271, 275, 243, 258, 235, 263, 273, 245, 276, 293, 229, 257] payload_rows_per_dst=[5496, 5444, 5394, 5576, 5502, 5366, 5408, 5542] dup_rows_per_dst=[2702, 2642, 2806, 2826, 2762, 2672, 2518, 2880] combine_rows_per_src=[5496, 5444, 5394, 5576, 5502, 5366, 5408, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38270 dup_rows=21908 dup_rows_remote=19236 dedup_ratio=0.334290 router_tokens_per_expert=[261, 249, 263, 259, 251, 265, 233, 247, 251, 273, 216, 274, 265, 256, 249, 245, 262, 251, 227, 260, 261, 259, 243, 250, 277, 277, 281, 276, 259, 269, 264, 259, 228, 240, 260, 243, 247, 260, 280, 242, 267, 265, 257, 242, 235, 269, 239, 241, 246, 254, 250, 245, 260, 262, 253, 271, 267, 265, 242, 240, 255, 236, 262, 254, 269, 271, 276, 248, 212, 272, 255, 257, 269, 244, 283, 264, 252, 248, 245, 241, 248, 267, 230, 232, 249, 282, 264, 250, 281, 255, 247, 253, 264, 218, 247, 256, 264, 248, 244, 245, 277, 239, 248, 267, 284, 240, 288, 285, 267, 262, 264, 224, 260, 266, 273, 267, 259, 238, 269, 253, 267, 225, 256, 269, 289, 274, 232, 267] payload_rows_per_dst=[5438, 5468, 5358, 5408, 5490, 5398, 5536, 5532] dup_rows_per_dst=[2676, 2882, 2672, 2716, 2722, 2688, 2756, 2796] combine_rows_per_src=[5438, 5468, 5358, 5408, 5490, 5398, 5536, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229184 combine_payload_bytes=156565504 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156794688 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360192 weight_movement_bytes=56623104 total_nvlink_bytes=369983296 weight_to_token_ratio=0.180697 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38330 dup_rows=21728 dup_rows_remote=18898 dedup_ratio=0.331543 router_tokens_per_expert=[234, 285, 266, 279, 243, 266, 244, 235, 258, 252, 254, 223, 255, 259, 263, 281, 255, 240, 257, 267, 271, 262, 274, 281, 267, 250, 240, 269, 271, 229, 261, 248, 243, 265, 234, 244, 266, 248, 265, 238, 262, 271, 247, 260, 246, 244, 293, 259, 237, 296, 274, 252, 245, 248, 291, 255, 263, 248, 273, 242, 229, 258, 233, 235, 274, 266, 254, 279, 252, 278, 256, 249, 272, 271, 268, 251, 265, 268, 267, 275, 214, 273, 239, 244, 259, 261, 246, 236, 220, 267, 261, 254, 302, 228, 249, 244, 223, 259, 266, 250, 273, 258, 282, 288, 244, 272, 260, 224, 232, 233, 250, 211, 233, 261, 265, 234, 251, 257, 265, 273, 265, 245, 257, 280, 264, 235, 270, 243] payload_rows_per_dst=[5442, 5354, 5364, 5414, 5686, 5648, 5422, 5478] dup_rows_per_dst=[2752, 2298, 2806, 2564, 2984, 2854, 2640, 2830] combine_rows_per_src=[5442, 5354, 5364, 5414, 5686, 5648, 5422, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229200 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179728 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130256 weight_movement_bytes=37748736 total_nvlink_bytes=351878992 weight_to_token_ratio=0.120169 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38240 dup_rows=21884 dup_rows_remote=19088 dedup_ratio=0.333923 router_tokens_per_expert=[268, 230, 225, 254, 240, 259, 256, 219, 248, 253, 244, 254, 244, 243, 241, 277, 245, 274, 250, 270, 262, 251, 270, 248, 265, 262, 266, 245, 238, 239, 257, 269, 248, 285, 232, 248, 233, 233, 256, 271, 282, 268, 280, 271, 252, 255, 274, 244, 255, 262, 240, 252, 237, 231, 246, 274, 261, 229, 281, 234, 265, 246, 307, 271, 228, 266, 251, 249, 238, 273, 246, 273, 272, 281, 249, 251, 278, 250, 260, 239, 253, 230, 288, 246, 247, 263, 267, 230, 273, 251, 268, 281, 240, 241, 255, 240, 249, 243, 280, 248, 237, 289, 229, 281, 265, 244, 257, 239, 264, 274, 260, 268, 260, 260, 259, 292, 244, 295, 248, 254, 261, 253, 256, 263, 260, 257, 242, 271] payload_rows_per_dst=[5378, 5526, 5490, 5406, 5412, 5430, 5504, 5506] dup_rows_per_dst=[2532, 2696, 2774, 2776, 2796, 2716, 2750, 2844] combine_rows_per_src=[5378, 5526, 5490, 5406, 5412, 5430, 5504, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43460 payload_rows_remote=38066 dup_rows=22076 dup_rows_remote=19440 dedup_ratio=0.336853 router_tokens_per_expert=[255, 258, 219, 252, 246, 242, 255, 250, 260, 256, 282, 247, 231, 236, 244, 253, 291, 262, 296, 251, 264, 241, 226, 262, 278, 247, 265, 250, 250, 264, 241, 258, 239, 257, 282, 283, 251, 249, 260, 277, 245, 273, 268, 254, 260, 273, 235, 242, 241, 274, 267, 275, 260, 265, 279, 246, 269, 266, 259, 271, 231, 275, 249, 242, 254, 237, 247, 246, 266, 237, 247, 272, 272, 276, 256, 269, 264, 258, 268, 268, 272, 255, 290, 236, 251, 259, 244, 264, 244, 267, 255, 258, 250, 254, 277, 245, 230, 253, 270, 239, 253, 232, 265, 250, 248, 261, 257, 253, 263, 248, 240, 247, 258, 270, 266, 270, 241, 272, 264, 225, 238, 238, 253, 268, 256, 231, 254, 248] payload_rows_per_dst=[5394, 5546, 5370, 5390, 5502, 5506, 5326, 5426] dup_rows_per_dst=[2636, 2992, 2704, 2894, 2772, 2736, 2664, 2678] combine_rows_per_src=[5394, 5546, 5370, 5390, 5502, 5506, 5326, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229160 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157105960 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982760 weight_movement_bytes=18874368 total_nvlink_bytes=332857128 weight_to_token_ratio=0.060113 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228536 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156499128 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312769720 weight_movement_bytes=37748736 total_nvlink_bytes=350518456 weight_to_token_ratio=0.120692 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=230024 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983944 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737864 weight_movement_bytes=18874368 total_nvlink_bytes=332612232 weight_to_token_ratio=0.060160 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=228912 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157228592 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228272 weight_movement_bytes=37748736 total_nvlink_bytes=351977008 weight_to_token_ratio=0.120132 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229312 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860352 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491392 weight_movement_bytes=37748736 total_nvlink_bytes=351240128 weight_to_token_ratio=0.120414 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=230024 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156148360 ingress_bytes=174792704 link_bytes=174792704 token_bytes=312066696 weight_movement_bytes=18874368 total_nvlink_bytes=330941064 weight_to_token_ratio=0.060482 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674777686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38264 dup_rows=21772 dup_rows_remote=19084 dedup_ratio=0.332214 router_tokens_per_expert=[252, 243, 271, 261, 260, 262, 238, 257, 269, 290, 229, 265, 238, 256, 253, 256, 251, 258, 278, 266, 262, 238, 272, 234, 263, 257, 260, 246, 222, 266, 278, 243, 243, 264, 222, 241, 250, 241, 224, 245, 251, 276, 239, 256, 264, 220, 229, 274, 241, 260, 292, 263, 245, 266, 243, 251, 268, 237, 259, 246, 257, 246, 256, 274, 280, 289, 276, 259, 258, 243, 264, 258, 254, 257, 251, 264, 294, 270, 273, 263, 251, 281, 267, 253, 249, 263, 249, 254, 260, 266, 284, 246, 233, 251, 227, 234, 248, 247, 251, 258, 262, 244, 272, 251, 235, 245, 264, 259, 285, 274, 232, 283, 258, 245, 263, 234, 254, 254, 227, 258, 249, 260, 277, 265, 263, 288, 269, 236] payload_rows_per_dst=[5444, 5500, 5328, 5486, 5562, 5458, 5512, 5474] dup_rows_per_dst=[2756, 2688, 2550, 2722, 2944, 2678, 2708, 2726] combine_rows_per_src=[5444, 5500, 5328, 5486, 5562, 5458, 5512, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38080 dup_rows=21906 dup_rows_remote=19020 dedup_ratio=0.334259 router_tokens_per_expert=[271, 258, 231, 261, 241, 268, 278, 257, 276, 250, 275, 231, 270, 219, 271, 248, 260, 250, 262, 254, 248, 248, 260, 267, 228, 242, 238, 258, 253, 237, 247, 215, 285, 260, 273, 263, 271, 281, 249, 235, 257, 259, 253, 261, 239, 237, 262, 245, 244, 257, 243, 244, 254, 287, 271, 243, 250, 245, 285, 284, 242, 266, 263, 267, 260, 229, 284, 233, 284, 257, 241, 260, 260, 281, 273, 232, 262, 248, 269, 262, 272, 256, 230, 277, 201, 291, 272, 285, 264, 243, 244, 247, 234, 262, 278, 267, 265, 220, 251, 253, 245, 258, 238, 261, 241, 241, 258, 273, 254, 268, 277, 256, 269, 251, 249, 239, 251, 259, 273, 253, 249, 247, 274, 256, 251, 263, 258, 262] payload_rows_per_dst=[5462, 5380, 5472, 5400, 5478, 5494, 5550, 5394] dup_rows_per_dst=[2748, 2554, 2788, 2572, 2792, 2752, 2886, 2814] combine_rows_per_src=[5462, 5380, 5472, 5400, 5478, 5494, 5550, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38220 dup_rows=21882 dup_rows_remote=19066 dedup_ratio=0.333893 router_tokens_per_expert=[270, 236, 260, 245, 259, 252, 249, 254, 243, 250, 264, 258, 273, 220, 252, 264, 225, 253, 232, 246, 238, 264, 279, 260, 240, 273, 257, 272, 246, 272, 265, 231, 237, 258, 253, 263, 249, 272, 252, 259, 217, 269, 252, 255, 257, 253, 246, 251, 267, 277, 281, 255, 283, 269, 276, 258, 265, 251, 264, 260, 273, 243, 255, 259, 243, 271, 230, 251, 241, 249, 273, 259, 285, 254, 276, 238, 245, 288, 262, 281, 255, 278, 244, 274, 241, 243, 267, 255, 251, 267, 252, 266, 253, 256, 278, 245, 243, 249, 289, 228, 243, 241, 262, 262, 256, 268, 265, 238, 271, 258, 240, 250, 267, 239, 257, 251, 289, 267, 251, 238, 247, 251, 250, 265, 256, 240, 255, 230] payload_rows_per_dst=[5474, 5438, 5422, 5638, 5448, 5434, 5380, 5420] dup_rows_per_dst=[2624, 2668, 2664, 2834, 2844, 2816, 2746, 2686] combine_rows_per_src=[5474, 5438, 5422, 5638, 5448, 5434, 5380, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38414 dup_rows=21648 dup_rows_remote=18880 dedup_ratio=0.330322 router_tokens_per_expert=[255, 252, 229, 255, 236, 256, 282, 257, 263, 279, 266, 249, 257, 242, 255, 233, 253, 270, 248, 255, 247, 240, 248, 269, 266, 234, 284, 263, 248, 246, 249, 252, 254, 243, 262, 278, 242, 259, 285, 241, 275, 245, 263, 253, 251, 246, 260, 265, 260, 260, 263, 275, 256, 267, 250, 283, 270, 264, 254, 263, 240, 237, 240, 239, 255, 282, 270, 237, 277, 267, 268, 255, 269, 241, 273, 252, 281, 263, 251, 248, 225, 244, 259, 253, 244, 252, 246, 279, 264, 283, 241, 269, 259, 266, 248, 256, 256, 268, 253, 231, 255, 256, 271, 252, 247, 239, 254, 284, 247, 252, 263, 249, 254, 240, 238, 250, 242, 222, 277, 256, 255, 240, 261, 255, 251, 264, 271, 257] payload_rows_per_dst=[5426, 5518, 5486, 5474, 5594, 5490, 5452, 5448] dup_rows_per_dst=[2706, 2626, 2758, 2768, 2784, 2686, 2702, 2618] combine_rows_per_src=[5426, 5518, 5486, 5474, 5594, 5490, 5452, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38322 dup_rows=21660 dup_rows_remote=18918 dedup_ratio=0.330505 router_tokens_per_expert=[242, 253, 256, 254, 243, 272, 231, 248, 257, 245, 265, 251, 256, 269, 262, 252, 271, 263, 270, 264, 227, 248, 256, 276, 263, 249, 256, 259, 246, 254, 256, 244, 281, 265, 284, 258, 244, 278, 272, 234, 261, 254, 253, 251, 256, 245, 240, 272, 251, 253, 262, 261, 260, 275, 300, 238, 268, 251, 254, 260, 246, 258, 274, 256, 241, 257, 266, 235, 252, 265, 247, 279, 259, 250, 205, 235, 242, 265, 251, 275, 262, 236, 279, 274, 255, 280, 262, 250, 267, 275, 271, 274, 248, 282, 260, 230, 262, 270, 246, 235, 256, 237, 258, 266, 264, 236, 272, 239, 264, 223, 251, 251, 228, 214, 281, 264, 262, 246, 291, 252, 228, 231, 246, 279, 260, 259, 253, 242] payload_rows_per_dst=[5428, 5480, 5554, 5538, 5460, 5536, 5458, 5422] dup_rows_per_dst=[2684, 2724, 2742, 2796, 2588, 2874, 2602, 2650] combine_rows_per_src=[5428, 5480, 5554, 5538, 5460, 5536, 5458, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38280 dup_rows=21846 dup_rows_remote=19076 dedup_ratio=0.333344 router_tokens_per_expert=[253, 251, 259, 251, 261, 253, 248, 227, 243, 262, 261, 287, 265, 246, 265, 270, 252, 255, 279, 278, 256, 248, 242, 266, 269, 221, 280, 273, 274, 249, 256, 274, 274, 244, 251, 266, 262, 243, 259, 251, 241, 271, 255, 259, 249, 230, 281, 278, 250, 254, 303, 241, 252, 252, 266, 260, 241, 242, 247, 252, 258, 274, 245, 265, 243, 234, 239, 262, 276, 242, 254, 251, 269, 256, 250, 264, 261, 270, 257, 251, 270, 265, 249, 270, 246, 279, 231, 273, 253, 287, 248, 246, 248, 235, 274, 262, 263, 245, 253, 241, 296, 235, 237, 258, 257, 226, 241, 270, 249, 241, 236, 236, 266, 276, 246, 286, 230, 223, 251, 253, 243, 289, 253, 257, 265, 257, 266, 218] payload_rows_per_dst=[5430, 5536, 5526, 5216, 5492, 5496, 5584, 5410] dup_rows_per_dst=[2636, 2808, 2702, 2382, 2758, 2800, 2990, 2770] combine_rows_per_src=[5430, 5536, 5526, 5216, 5492, 5496, 5584, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229392 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958736 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688080 weight_movement_bytes=18874368 total_nvlink_bytes=332562448 weight_to_token_ratio=0.060169 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228400 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156204080 ingress_bytes=174850048 link_bytes=174850048 token_bytes=312179760 weight_movement_bytes=18874368 total_nvlink_bytes=331054128 weight_to_token_ratio=0.060460 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675327128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38210 dup_rows=21842 dup_rows_remote=19000 dedup_ratio=0.333282 router_tokens_per_expert=[251, 281, 272, 265, 244, 260, 268, 234, 260, 283, 254, 261, 263, 258, 260, 252, 271, 273, 234, 238, 247, 265, 240, 267, 238, 251, 226, 234, 247, 260, 274, 257, 261, 254, 276, 236, 249, 274, 270, 258, 225, 263, 267, 262, 266, 260, 250, 231, 268, 229, 276, 249, 245, 259, 246, 263, 270, 256, 280, 252, 234, 272, 282, 233, 271, 247, 265, 267, 266, 268, 252, 262, 245, 264, 254, 266, 262, 272, 238, 264, 274, 245, 237, 246, 249, 277, 253, 233, 255, 266, 251, 262, 232, 272, 252, 260, 264, 269, 260, 237, 253, 243, 262, 242, 258, 248, 264, 241, 259, 263, 253, 229, 247, 250, 271, 263, 266, 243, 238, 257, 242, 270, 237, 262, 269, 268, 233, 276] payload_rows_per_dst=[5504, 5426, 5438, 5486, 5484, 5438, 5412, 5506] dup_rows_per_dst=[2828, 2618, 2766, 2742, 2842, 2690, 2678, 2678] combine_rows_per_src=[5504, 5426, 5438, 5486, 5484, 5438, 5412, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38452 dup_rows=21638 dup_rows_remote=18976 dedup_ratio=0.330170 router_tokens_per_expert=[221, 256, 241, 261, 236, 249, 254, 247, 279, 281, 252, 280, 256, 252, 247, 226, 258, 266, 263, 257, 278, 250, 252, 249, 278, 255, 229, 249, 255, 254, 254, 257, 268, 271, 245, 248, 258, 261, 242, 282, 235, 280, 268, 259, 251, 249, 252, 234, 263, 287, 243, 263, 236, 251, 270, 246, 240, 241, 239, 283, 258, 258, 248, 252, 253, 257, 271, 264, 248, 245, 271, 256, 245, 249, 233, 242, 235, 219, 249, 266, 263, 257, 263, 255, 259, 278, 265, 256, 279, 249, 245, 231, 281, 213, 249, 242, 243, 265, 264, 258, 285, 288, 258, 280, 260, 254, 252, 270, 276, 260, 239, 267, 239, 272, 283, 248, 246, 276, 248, 253, 254, 235, 275, 278, 245, 282, 244, 260] payload_rows_per_dst=[5446, 5532, 5496, 5586, 5342, 5478, 5500, 5518] dup_rows_per_dst=[2662, 2682, 2710, 2820, 2664, 2668, 2674, 2758] combine_rows_per_src=[5446, 5532, 5496, 5586, 5342, 5478, 5500, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229144 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156778264 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313327384 weight_movement_bytes=37748736 total_nvlink_bytes=351076120 weight_to_token_ratio=0.120477 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229176 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157572920 ingress_bytes=176218112 link_bytes=176218112 token_bytes=314916664 weight_movement_bytes=18874368 total_nvlink_bytes=333791032 weight_to_token_ratio=0.059934 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688435244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=228960 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157195872 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314162784 weight_movement_bytes=18874368 total_nvlink_bytes=333037152 weight_to_token_ratio=0.060078 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229424 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157024304 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313819184 weight_movement_bytes=18874368 total_nvlink_bytes=332693552 weight_to_token_ratio=0.060144 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=228840 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156737000 ingress_bytes=213131264 link_bytes=213131264 token_bytes=313245160 weight_movement_bytes=56623104 total_nvlink_bytes=369868264 weight_to_token_ratio=0.180763 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042118905e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157499392 dispatch_meta_bytes=229712 combine_payload_bytes=157499392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157729104 ingress_bytes=195248128 link_bytes=195248128 token_bytes=315228496 weight_movement_bytes=37748736 total_nvlink_bytes=352977232 weight_to_token_ratio=0.119750 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870771494e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38370 dup_rows=21608 dup_rows_remote=18804 dedup_ratio=0.329712 router_tokens_per_expert=[268, 246, 273, 273, 251, 263, 283, 257, 290, 254, 285, 280, 254, 214, 267, 247, 264, 280, 270, 273, 274, 290, 257, 258, 256, 233, 250, 257, 270, 239, 260, 250, 269, 247, 252, 259, 257, 266, 286, 251, 227, 232, 226, 250, 244, 264, 231, 261, 235, 250, 238, 260, 250, 233, 270, 269, 260, 226, 232, 231, 238, 247, 280, 261, 257, 245, 274, 243, 257, 264, 273, 243, 252, 237, 246, 250, 258, 257, 279, 266, 237, 247, 293, 256, 245, 264, 235, 261, 278, 260, 266, 216, 269, 224, 253, 270, 268, 258, 256, 243, 255, 251, 248, 245, 270, 259, 281, 259, 272, 272, 231, 246, 268, 267, 270, 247, 248, 274, 236, 260, 276, 241, 269, 269, 238, 247, 232, 249] payload_rows_per_dst=[5574, 5558, 5444, 5398, 5454, 5500, 5542, 5458] dup_rows_per_dst=[2836, 2804, 2600, 2562, 2748, 2648, 2686, 2724] combine_rows_per_src=[5574, 5558, 5444, 5398, 5454, 5500, 5542, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38280 dup_rows=21870 dup_rows_remote=19084 dedup_ratio=0.333710 router_tokens_per_expert=[265, 268, 274, 243, 232, 233, 250, 248, 288, 281, 283, 244, 258, 243, 274, 268, 278, 283, 272, 238, 268, 273, 232, 272, 244, 231, 224, 242, 245, 259, 247, 258, 285, 254, 251, 252, 270, 241, 260, 252, 261, 258, 255, 258, 267, 250, 262, 254, 239, 247, 280, 262, 246, 245, 268, 252, 287, 244, 258, 234, 240, 231, 269, 251, 260, 278, 268, 249, 250, 244, 260, 249, 259, 248, 269, 245, 224, 245, 223, 269, 272, 265, 268, 235, 256, 250, 240, 223, 237, 260, 249, 242, 259, 277, 276, 262, 245, 276, 250, 253, 257, 233, 258, 278, 268, 254, 237, 258, 248, 273, 253, 245, 289, 256, 233, 255, 268, 246, 260, 271, 255, 268, 274, 276, 247, 258, 259, 255] payload_rows_per_dst=[5646, 5408, 5478, 5386, 5410, 5428, 5386, 5524] dup_rows_per_dst=[2658, 2724, 2782, 2720, 2670, 2714, 2786, 2816] combine_rows_per_src=[5646, 5408, 5478, 5386, 5410, 5428, 5386, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43512 payload_rows_remote=38074 dup_rows=22024 dup_rows_remote=19316 dedup_ratio=0.336060 router_tokens_per_expert=[242, 229, 262, 251, 235, 263, 224, 256, 256, 247, 265, 256, 248, 234, 250, 256, 243, 273, 250, 250, 255, 249, 277, 268, 256, 284, 270, 253, 291, 272, 259, 269, 247, 275, 236, 267, 265, 288, 264, 258, 240, 260, 275, 226, 253, 253, 247, 242, 224, 234, 264, 249, 270, 273, 254, 279, 245, 263, 239, 258, 282, 280, 274, 258, 262, 255, 244, 245, 240, 272, 245, 250, 245, 260, 271, 278, 237, 273, 251, 270, 273, 272, 247, 236, 258, 253, 282, 239, 241, 272, 240, 249, 237, 267, 271, 236, 247, 256, 252, 260, 238, 225, 285, 266, 258, 259, 231, 258, 266, 277, 243, 259, 267, 262, 245, 249, 268, 249, 270, 278, 234, 217, 270, 245, 236, 265, 271, 256] payload_rows_per_dst=[5376, 5458, 5476, 5404, 5494, 5438, 5426, 5440] dup_rows_per_dst=[2572, 2980, 2716, 2888, 2702, 2708, 2734, 2724] combine_rows_per_src=[5376, 5458, 5476, 5404, 5494, 5438, 5426, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38574 dup_rows=21578 dup_rows_remote=18888 dedup_ratio=0.329254 router_tokens_per_expert=[260, 249, 253, 238, 265, 264, 267, 298, 265, 252, 231, 261, 234, 241, 300, 253, 243, 250, 275, 273, 269, 243, 259, 264, 283, 237, 260, 260, 257, 272, 247, 264, 289, 257, 234, 276, 242, 255, 230, 241, 283, 250, 241, 252, 269, 248, 284, 256, 235, 291, 272, 280, 259, 238, 230, 241, 248, 262, 264, 223, 228, 259, 249, 258, 272, 248, 256, 245, 266, 291, 250, 269, 246, 261, 256, 249, 257, 262, 243, 227, 283, 260, 250, 249, 245, 234, 278, 256, 250, 234, 300, 259, 234, 248, 249, 266, 234, 220, 277, 270, 242, 264, 246, 261, 267, 262, 237, 268, 258, 271, 252, 270, 236, 276, 270, 266, 234, 238, 248, 288, 251, 230, 245, 246, 253, 249, 260, 255] payload_rows_per_dst=[5536, 5582, 5562, 5384, 5490, 5446, 5494, 5464] dup_rows_per_dst=[2726, 2730, 2652, 2690, 2706, 2744, 2704, 2626] combine_rows_per_src=[5536, 5582, 5562, 5384, 5490, 5446, 5494, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38270 dup_rows=21876 dup_rows_remote=19246 dedup_ratio=0.333801 router_tokens_per_expert=[287, 250, 254, 265, 270, 261, 259, 284, 236, 244, 268, 254, 255, 265, 250, 253, 259, 238, 246, 244, 268, 263, 234, 248, 263, 250, 254, 250, 250, 261, 260, 245, 277, 245, 248, 281, 265, 265, 238, 213, 249, 264, 230, 248, 250, 253, 235, 249, 261, 249, 254, 275, 256, 246, 269, 270, 252, 230, 242, 285, 239, 256, 261, 250, 260, 262, 255, 245, 271, 305, 252, 261, 248, 251, 246, 248, 232, 251, 279, 229, 256, 265, 220, 242, 236, 253, 261, 265, 248, 283, 238, 257, 258, 252, 248, 252, 255, 254, 262, 277, 260, 286, 236, 238, 267, 245, 291, 256, 256, 281, 263, 260, 232, 245, 262, 238, 240, 265, 286, 252, 281, 271, 278, 271, 234, 278, 263, 263] payload_rows_per_dst=[5494, 5424, 5390, 5484, 5474, 5376, 5508, 5510] dup_rows_per_dst=[2816, 2642, 2630, 2706, 2716, 2692, 2866, 2808] combine_rows_per_src=[5494, 5424, 5390, 5484, 5474, 5376, 5508, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38420 dup_rows=21644 dup_rows_remote=18782 dedup_ratio=0.330261 router_tokens_per_expert=[271, 250, 285, 262, 265, 261, 269, 231, 254, 240, 261, 234, 266, 243, 271, 253, 272, 276, 275, 219, 238, 288, 253, 289, 248, 236, 240, 228, 244, 257, 279, 248, 260, 270, 251, 241, 238, 276, 262, 251, 285, 261, 244, 262, 249, 270, 247, 245, 233, 265, 246, 248, 232, 271, 262, 265, 256, 228, 264, 238, 255, 231, 249, 266, 217, 261, 243, 253, 276, 245, 250, 217, 257, 257, 259, 259, 277, 257, 271, 255, 242, 236, 246, 249, 267, 242, 262, 259, 270, 262, 266, 276, 284, 244, 264, 254, 262, 263, 271, 279, 260, 239, 257, 268, 239, 264, 277, 264, 276, 266, 267, 257, 260, 255, 243, 259, 263, 246, 280, 270, 224, 249, 248, 252, 274, 239, 242, 251] payload_rows_per_dst=[5408, 5384, 5552, 5548, 5432, 5562, 5534, 5472] dup_rows_per_dst=[2614, 2572, 2644, 2680, 2704, 2684, 2884, 2862] combine_rows_per_src=[5408, 5384, 5552, 5548, 5432, 5562, 5534, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229456 combine_payload_bytes=156794880 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157024336 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313819216 weight_movement_bytes=18874368 total_nvlink_bytes=332693584 weight_to_token_ratio=0.060144 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=228696 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157392216 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314555736 weight_movement_bytes=37748736 total_nvlink_bytes=352304472 weight_to_token_ratio=0.120007 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38282 dup_rows=21808 dup_rows_remote=19126 dedup_ratio=0.332764 router_tokens_per_expert=[250, 293, 256, 247, 235, 250, 258, 234, 249, 273, 256, 266, 261, 240, 268, 267, 267, 254, 245, 268, 246, 255, 247, 276, 271, 263, 279, 238, 268, 266, 271, 263, 246, 264, 252, 265, 257, 271, 263, 240, 261, 227, 253, 287, 269, 259, 259, 262, 256, 236, 277, 255, 265, 245, 276, 251, 249, 266, 253, 249, 224, 311, 258, 281, 250, 254, 243, 254, 275, 239, 286, 260, 232, 252, 282, 236, 237, 252, 256, 256, 250, 262, 255, 257, 256, 261, 268, 263, 264, 263, 261, 272, 222, 257, 263, 232, 261, 246, 256, 267, 242, 269, 257, 265, 248, 255, 247, 258, 233, 236, 232, 246, 255, 250, 224, 274, 249, 230, 263, 224, 263, 237, 248, 275, 252, 245, 277, 247] payload_rows_per_dst=[5470, 5572, 5518, 5496, 5446, 5428, 5414, 5384] dup_rows_per_dst=[2736, 2782, 2752, 2808, 2682, 2784, 2622, 2642] combine_rows_per_src=[5470, 5572, 5518, 5496, 5446, 5428, 5414, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=37994 dup_rows=21958 dup_rows_remote=18906 dedup_ratio=0.335052 router_tokens_per_expert=[246, 266, 237, 258, 267, 249, 247, 232, 280, 267, 264, 235, 234, 262, 245, 270, 260, 261, 238, 264, 263, 260, 244, 250, 269, 260, 248, 263, 269, 241, 265, 260, 264, 256, 250, 263, 237, 227, 224, 253, 239, 262, 244, 256, 230, 250, 267, 272, 256, 295, 253, 253, 278, 252, 259, 265, 272, 263, 249, 233, 280, 260, 244, 268, 268, 259, 273, 261, 251, 243, 278, 267, 254, 260, 255, 275, 261, 262, 259, 277, 270, 236, 255, 247, 269, 255, 228, 288, 297, 252, 278, 243, 235, 248, 225, 251, 287, 226, 287, 268, 240, 284, 213, 233, 289, 258, 274, 270, 262, 274, 233, 253, 275, 277, 258, 247, 257, 240, 237, 260, 273, 230, 236, 236, 240, 239, 222, 262] payload_rows_per_dst=[5584, 5450, 5460, 5538, 5358, 5470, 5446, 5272] dup_rows_per_dst=[3052, 2780, 2804, 2858, 2530, 2684, 2820, 2430] combine_rows_per_src=[5584, 5450, 5460, 5538, 5358, 5470, 5446, 5272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=230064 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983984 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737904 weight_movement_bytes=37748736 total_nvlink_bytes=351486640 weight_to_token_ratio=0.120319 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=228808 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157597128 ingress_bytes=195117056 link_bytes=195117056 token_bytes=314965448 weight_movement_bytes=37748736 total_nvlink_bytes=352714184 weight_to_token_ratio=0.119850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869515627e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=229560 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156180664 ingress_bytes=174825472 link_bytes=174825472 token_bytes=312131768 weight_movement_bytes=18874368 total_nvlink_bytes=331006136 weight_to_token_ratio=0.060469 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675091653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157999104 dispatch_meta_bytes=229848 combine_payload_bytes=157999104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158228952 ingress_bytes=176873472 link_bytes=176873472 token_bytes=316228056 weight_movement_bytes=18874368 total_nvlink_bytes=335102424 weight_to_token_ratio=0.059686 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694714582e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229632 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157032704 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313835776 weight_movement_bytes=56623104 total_nvlink_bytes=370458880 weight_to_token_ratio=0.180423 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155623424 dispatch_meta_bytes=227600 combine_payload_bytes=155623424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155851024 ingress_bytes=174497792 link_bytes=174497792 token_bytes=311474448 weight_movement_bytes=18874368 total_nvlink_bytes=330348816 weight_to_token_ratio=0.060597 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671951985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38326 dup_rows=21768 dup_rows_remote=19174 dedup_ratio=0.332153 router_tokens_per_expert=[236, 259, 269, 246, 241, 243, 235, 233, 278, 256, 238, 257, 283, 240, 243, 259, 235, 233, 262, 259, 244, 293, 250, 259, 253, 286, 253, 229, 253, 260, 258, 279, 256, 279, 285, 260, 266, 263, 252, 262, 252, 258, 241, 249, 264, 282, 256, 259, 252, 257, 266, 270, 260, 246, 240, 282, 275, 241, 228, 257, 254, 282, 239, 227, 283, 286, 245, 267, 230, 268, 264, 269, 258, 227, 259, 243, 270, 263, 275, 257, 241, 238, 265, 226, 261, 275, 255, 244, 247, 297, 287, 258, 250, 256, 224, 240, 242, 291, 228, 225, 240, 214, 270, 264, 276, 256, 241, 228, 250, 281, 244, 268, 249, 252, 243, 256, 259, 267, 275, 286, 281, 272, 266, 250, 260, 235, 272, 217] payload_rows_per_dst=[5392, 5446, 5494, 5414, 5528, 5452, 5442, 5600] dup_rows_per_dst=[2640, 2766, 2874, 2738, 2718, 2676, 2594, 2762] combine_rows_per_src=[5392, 5446, 5494, 5414, 5528, 5452, 5442, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38302 dup_rows=21774 dup_rows_remote=19050 dedup_ratio=0.332245 router_tokens_per_expert=[281, 242, 267, 251, 237, 259, 247, 229, 249, 264, 251, 242, 262, 252, 260, 268, 277, 281, 260, 247, 274, 245, 253, 236, 241, 249, 276, 242, 261, 285, 238, 227, 254, 230, 257, 254, 257, 239, 258, 259, 236, 251, 227, 262, 244, 231, 275, 245, 261, 255, 243, 258, 274, 268, 264, 249, 236, 267, 280, 246, 265, 259, 253, 240, 268, 281, 255, 241, 229, 258, 249, 252, 271, 256, 251, 267, 276, 248, 254, 266, 229, 242, 252, 255, 262, 275, 261, 251, 247, 231, 279, 266, 248, 282, 257, 266, 259, 255, 294, 258, 241, 222, 267, 241, 268, 238, 261, 282, 257, 242, 244, 269, 239, 257, 254, 279, 266, 266, 274, 230, 256, 268, 256, 262, 269, 288, 259, 272] payload_rows_per_dst=[5458, 5460, 5376, 5604, 5538, 5556, 5418, 5352] dup_rows_per_dst=[2664, 2724, 2582, 3000, 2706, 2650, 2778, 2670] combine_rows_per_src=[5458, 5460, 5376, 5604, 5538, 5556, 5418, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=230000 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213296 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196592 weight_movement_bytes=18874368 total_nvlink_bytes=333070960 weight_to_token_ratio=0.060072 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=37966 dup_rows=21880 dup_rows_remote=18860 dedup_ratio=0.333862 router_tokens_per_expert=[274, 263, 270, 264, 254, 248, 303, 246, 248, 261, 243, 302, 266, 281, 268, 279, 235, 270, 257, 263, 252, 270, 260, 255, 237, 240, 267, 250, 265, 244, 261, 235, 257, 236, 225, 279, 238, 246, 254, 247, 247, 250, 238, 251, 267, 272, 276, 276, 243, 260, 244, 266, 239, 250, 264, 265, 245, 242, 259, 208, 272, 261, 268, 255, 270, 263, 230, 222, 278, 267, 252, 242, 245, 259, 267, 241, 256, 248, 245, 265, 267, 251, 249, 237, 237, 282, 262, 251, 274, 272, 241, 238, 269, 270, 243, 261, 227, 276, 260, 268, 243, 268, 271, 261, 259, 253, 268, 249, 223, 247, 262, 256, 273, 271, 264, 300, 242, 242, 240, 252, 253, 251, 253, 211, 250, 283, 254, 253] payload_rows_per_dst=[5576, 5268, 5406, 5400, 5364, 5458, 5494, 5690] dup_rows_per_dst=[2804, 2478, 2696, 2682, 2586, 2766, 2848, 3020] combine_rows_per_src=[5576, 5268, 5406, 5400, 5364, 5458, 5494, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38248 dup_rows=21780 dup_rows_remote=18988 dedup_ratio=0.332336 router_tokens_per_expert=[248, 245, 257, 280, 240, 258, 234, 210, 233, 282, 250, 237, 266, 258, 248, 277, 261, 272, 298, 274, 265, 254, 249, 257, 229, 257, 263, 249, 273, 258, 273, 273, 256, 273, 254, 248, 273, 266, 273, 236, 275, 246, 259, 274, 256, 243, 258, 260, 234, 259, 239, 270, 248, 259, 253, 244, 245, 284, 286, 252, 274, 249, 258, 239, 245, 248, 274, 256, 250, 263, 242, 265, 252, 263, 278, 255, 251, 260, 261, 265, 270, 262, 228, 261, 255, 261, 264, 271, 237, 273, 253, 262, 253, 272, 240, 284, 254, 251, 251, 269, 235, 263, 241, 253, 244, 262, 252, 267, 218, 273, 271, 262, 244, 243, 227, 213, 271, 284, 257, 243, 229, 247, 255, 247, 243, 253, 267, 234] payload_rows_per_dst=[5426, 5584, 5508, 5460, 5430, 5494, 5504, 5350] dup_rows_per_dst=[2620, 2826, 2792, 2726, 2826, 2798, 2628, 2564] combine_rows_per_src=[5426, 5584, 5508, 5460, 5430, 5494, 5504, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38208 dup_rows=21898 dup_rows_remote=19158 dedup_ratio=0.334137 router_tokens_per_expert=[243, 267, 263, 267, 253, 256, 250, 259, 249, 241, 265, 239, 269, 269, 241, 234, 264, 247, 284, 238, 259, 263, 258, 276, 278, 269, 244, 262, 232, 275, 242, 249, 258, 244, 227, 264, 241, 252, 257, 204, 278, 253, 267, 259, 267, 255, 269, 273, 271, 265, 258, 252, 254, 229, 242, 234, 245, 260, 245, 221, 238, 258, 237, 248, 249, 273, 248, 285, 227, 229, 266, 258, 243, 255, 290, 245, 261, 239, 274, 254, 287, 251, 245, 268, 258, 280, 257, 253, 245, 231, 261, 261, 237, 237, 251, 263, 224, 266, 287, 259, 274, 248, 263, 233, 231, 243, 270, 269, 266, 232, 249, 294, 274, 252, 278, 273, 258, 289, 266, 249, 285, 271, 247, 292, 246, 266, 229, 274] payload_rows_per_dst=[5412, 5498, 5468, 5376, 5428, 5430, 5500, 5526] dup_rows_per_dst=[2718, 2782, 2668, 2538, 2764, 2740, 2716, 2972] combine_rows_per_src=[5412, 5498, 5468, 5376, 5428, 5430, 5500, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38292 dup_rows=21940 dup_rows_remote=19332 dedup_ratio=0.334778 router_tokens_per_expert=[247, 256, 276, 272, 259, 231, 248, 285, 242, 284, 233, 277, 230, 223, 249, 254, 266, 248, 259, 279, 276, 257, 284, 240, 239, 242, 248, 241, 236, 282, 258, 230, 262, 256, 265, 262, 275, 264, 272, 248, 258, 254, 260, 289, 264, 260, 249, 263, 220, 237, 242, 281, 215, 264, 237, 245, 251, 256, 238, 247, 255, 267, 260, 241, 237, 255, 258, 263, 261, 267, 239, 270, 248, 247, 264, 257, 250, 254, 233, 262, 272, 256, 289, 276, 245, 278, 264, 261, 285, 239, 241, 256, 292, 269, 223, 231, 264, 244, 272, 248, 278, 274, 254, 274, 241, 232, 285, 251, 247, 260, 259, 259, 238, 244, 266, 267, 277, 246, 240, 256, 263, 246, 262, 264, 251, 246, 222, 288] payload_rows_per_dst=[5420, 5428, 5556, 5304, 5464, 5458, 5484, 5482] dup_rows_per_dst=[2712, 2742, 2846, 2608, 2666, 2896, 2800, 2670] combine_rows_per_src=[5420, 5428, 5556, 5304, 5464, 5458, 5484, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229408 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114400 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999392 weight_movement_bytes=37748736 total_nvlink_bytes=351748128 weight_to_token_ratio=0.120219 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38218 dup_rows=21784 dup_rows_remote=19106 dedup_ratio=0.332397 router_tokens_per_expert=[233, 229, 268, 247, 251, 252, 259, 269, 229, 237, 246, 268, 282, 252, 276, 283, 250, 253, 246, 273, 260, 244, 247, 264, 239, 244, 275, 266, 213, 252, 283, 233, 271, 253, 260, 245, 289, 252, 239, 234, 273, 294, 262, 265, 237, 253, 271, 249, 221, 232, 298, 254, 248, 246, 271, 211, 251, 264, 257, 254, 274, 263, 245, 253, 260, 280, 272, 262, 226, 273, 267, 244, 240, 256, 263, 241, 256, 261, 243, 262, 283, 231, 243, 250, 284, 282, 254, 266, 306, 245, 253, 255, 240, 252, 257, 244, 252, 252, 253, 238, 267, 262, 269, 249, 262, 240, 255, 249, 254, 272, 244, 255, 240, 259, 267, 285, 268, 279, 248, 241, 251, 239, 256, 261, 277, 236, 271, 254] payload_rows_per_dst=[5514, 5378, 5492, 5418, 5534, 5496, 5420, 5500] dup_rows_per_dst=[2648, 2706, 2802, 2666, 2678, 2794, 2726, 2764] combine_rows_per_src=[5514, 5378, 5492, 5418, 5534, 5496, 5420, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38290 dup_rows=21718 dup_rows_remote=18926 dedup_ratio=0.331390 router_tokens_per_expert=[266, 257, 255, 249, 259, 271, 264, 234, 274, 265, 270, 232, 258, 267, 248, 291, 277, 275, 271, 274, 263, 253, 265, 243, 246, 251, 265, 265, 257, 260, 246, 266, 260, 246, 287, 233, 235, 268, 265, 228, 227, 256, 250, 244, 258, 272, 248, 242, 290, 270, 239, 267, 242, 231, 254, 283, 234, 235, 233, 220, 256, 246, 261, 261, 263, 236, 242, 279, 245, 262, 230, 273, 253, 252, 247, 274, 255, 250, 273, 267, 255, 267, 258, 261, 250, 251, 241, 263, 266, 252, 277, 242, 255, 277, 250, 290, 254, 251, 252, 227, 244, 258, 258, 277, 236, 283, 245, 255, 249, 256, 240, 267, 256, 252, 244, 282, 278, 252, 271, 223, 256, 256, 242, 256, 279, 245, 243, 247] payload_rows_per_dst=[5528, 5554, 5576, 5532, 5282, 5432, 5398, 5516] dup_rows_per_dst=[2792, 3076, 2808, 2790, 2298, 2600, 2706, 2648] combine_rows_per_src=[5528, 5554, 5576, 5532, 5282, 5432, 5398, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155508736 dispatch_meta_bytes=227304 combine_payload_bytes=155508736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155736040 ingress_bytes=193257472 link_bytes=193257472 token_bytes=311244776 weight_movement_bytes=37748736 total_nvlink_bytes=348993512 weight_to_token_ratio=0.121283 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851698008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228944 combine_payload_bytes=156663808 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156892752 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556560 weight_movement_bytes=37748736 total_nvlink_bytes=351305296 weight_to_token_ratio=0.120389 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229464 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156729432 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313229400 weight_movement_bytes=37748736 total_nvlink_bytes=350978136 weight_to_token_ratio=0.120515 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=230496 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157074528 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313918560 weight_movement_bytes=18874368 total_nvlink_bytes=332792928 weight_to_token_ratio=0.060125 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229296 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156770224 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313311152 weight_movement_bytes=37748736 total_nvlink_bytes=351059888 weight_to_token_ratio=0.120483 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38144 dup_rows=21990 dup_rows_remote=19326 dedup_ratio=0.335541 router_tokens_per_expert=[250, 256, 261, 253, 277, 226, 270, 263, 254, 255, 264, 272, 240, 274, 237, 273, 266, 262, 251, 274, 256, 260, 260, 276, 252, 278, 255, 266, 264, 239, 248, 241, 257, 261, 262, 224, 246, 250, 240, 283, 278, 238, 262, 249, 263, 253, 241, 259, 266, 260, 266, 242, 284, 248, 260, 251, 259, 260, 229, 264, 255, 265, 267, 247, 273, 262, 254, 270, 256, 252, 266, 264, 279, 280, 243, 215, 259, 243, 249, 243, 255, 268, 238, 234, 258, 257, 261, 254, 259, 256, 265, 277, 262, 261, 267, 264, 253, 246, 228, 263, 231, 269, 242, 248, 255, 248, 260, 268, 257, 277, 241, 247, 241, 236, 211, 272, 262, 256, 269, 235, 226, 271, 262, 279, 244, 273, 235, 257] payload_rows_per_dst=[5478, 5508, 5410, 5484, 5454, 5408, 5402, 5402] dup_rows_per_dst=[2772, 2788, 2722, 2762, 2762, 2864, 2664, 2656] combine_rows_per_src=[5478, 5508, 5410, 5484, 5454, 5408, 5402, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=228864 combine_payload_bytes=156835840 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157064704 ingress_bytes=213458944 link_bytes=213458944 token_bytes=313900544 weight_movement_bytes=56623104 total_nvlink_bytes=370523648 weight_to_token_ratio=0.180385 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045258573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38318 dup_rows=21790 dup_rows_remote=19088 dedup_ratio=0.332489 router_tokens_per_expert=[245, 251, 278, 253, 258, 247, 247, 271, 257, 256, 231, 255, 260, 216, 265, 276, 216, 279, 257, 264, 268, 252, 259, 252, 262, 253, 247, 243, 271, 235, 254, 253, 234, 256, 262, 253, 235, 248, 281, 255, 283, 290, 253, 278, 275, 238, 247, 253, 265, 248, 275, 276, 237, 234, 326, 261, 254, 265, 248, 262, 257, 232, 258, 240, 254, 225, 237, 246, 253, 268, 264, 273, 273, 290, 243, 274, 274, 246, 244, 247, 261, 253, 264, 233, 253, 268, 217, 265, 278, 245, 261, 269, 244, 244, 257, 248, 246, 252, 250, 293, 245, 274, 267, 260, 269, 245, 246, 227, 253, 251, 232, 268, 298, 239, 236, 271, 247, 280, 251, 258, 263, 242, 249, 253, 248, 269, 265, 240] payload_rows_per_dst=[5466, 5428, 5526, 5480, 5490, 5456, 5438, 5462] dup_rows_per_dst=[2666, 2702, 2756, 2796, 2732, 2664, 2718, 2756] combine_rows_per_src=[5466, 5428, 5526, 5480, 5490, 5456, 5438, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229880 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156467704 ingress_bytes=193986560 link_bytes=193986560 token_bytes=312705528 weight_movement_bytes=37748736 total_nvlink_bytes=350454264 weight_to_token_ratio=0.120717 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858683771e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38060 dup_rows=21810 dup_rows_remote=18744 dedup_ratio=0.332794 router_tokens_per_expert=[223, 274, 278, 253, 280, 267, 248, 267, 271, 240, 253, 288, 255, 249, 271, 227, 289, 268, 262, 251, 253, 245, 243, 248, 258, 261, 243, 239, 276, 247, 220, 240, 263, 275, 251, 265, 286, 256, 257, 273, 239, 243, 256, 253, 247, 260, 248, 211, 254, 256, 268, 262, 248, 263, 251, 271, 245, 273, 248, 269, 240, 262, 242, 254, 266, 268, 219, 281, 250, 244, 245, 268, 291, 280, 226, 257, 270, 284, 250, 235, 259, 259, 256, 245, 282, 274, 230, 247, 259, 255, 258, 244, 244, 243, 256, 239, 248, 258, 245, 253, 287, 229, 260, 286, 247, 244, 231, 261, 253, 279, 254, 256, 274, 252, 272, 255, 257, 248, 244, 263, 234, 251, 266, 268, 259, 236, 273, 265] payload_rows_per_dst=[5504, 5392, 5194, 5506, 5504, 5372, 5588, 5666] dup_rows_per_dst=[2784, 2548, 2472, 2852, 2764, 2612, 2712, 3066] combine_rows_per_src=[5504, 5392, 5194, 5506, 5504, 5372, 5588, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229624 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157180152 ingress_bytes=213573632 link_bytes=213573632 token_bytes=314130680 weight_movement_bytes=56623104 total_nvlink_bytes=370753784 weight_to_token_ratio=0.180253 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046357457e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38184 dup_rows=21852 dup_rows_remote=19078 dedup_ratio=0.333435 router_tokens_per_expert=[262, 249, 264, 277, 257, 251, 269, 261, 267, 243, 229, 259, 227, 274, 279, 264, 229, 243, 268, 285, 254, 282, 251, 249, 229, 234, 254, 253, 270, 289, 268, 251, 246, 263, 242, 248, 268, 244, 251, 273, 270, 269, 270, 274, 251, 256, 248, 264, 235, 228, 257, 271, 246, 261, 261, 217, 245, 249, 243, 255, 237, 268, 261, 239, 283, 258, 251, 248, 243, 231, 226, 242, 269, 258, 251, 256, 253, 246, 279, 270, 259, 269, 298, 242, 246, 243, 260, 271, 267, 247, 274, 255, 249, 255, 262, 264, 257, 266, 237, 292, 259, 260, 257, 240, 227, 287, 268, 231, 267, 227, 249, 295, 259, 255, 248, 250, 239, 248, 269, 241, 255, 260, 241, 263, 226, 268, 277, 274] payload_rows_per_dst=[5448, 5458, 5500, 5382, 5368, 5566, 5532, 5430] dup_rows_per_dst=[2816, 2760, 2774, 2564, 2760, 2756, 2706, 2716] combine_rows_per_src=[5448, 5458, 5500, 5382, 5368, 5566, 5532, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38312 dup_rows=21734 dup_rows_remote=19048 dedup_ratio=0.331635 router_tokens_per_expert=[258, 232, 274, 225, 256, 246, 264, 247, 236, 275, 229, 276, 258, 236, 240, 245, 231, 268, 265, 240, 253, 285, 263, 255, 253, 275, 227, 244, 276, 251, 275, 287, 242, 243, 273, 264, 258, 250, 261, 246, 278, 267, 246, 268, 257, 247, 235, 254, 252, 254, 251, 268, 281, 271, 255, 232, 282, 262, 287, 250, 260, 237, 257, 261, 261, 225, 271, 255, 235, 257, 228, 258, 240, 291, 263, 234, 257, 228, 251, 251, 283, 262, 236, 263, 259, 289, 234, 246, 236, 262, 257, 263, 249, 249, 254, 246, 261, 260, 235, 275, 273, 274, 268, 254, 249, 274, 259, 242, 250, 286, 246, 284, 241, 226, 284, 284, 259, 281, 241, 257, 265, 259, 242, 252, 263, 241, 275, 221] payload_rows_per_dst=[5414, 5502, 5448, 5460, 5396, 5490, 5616, 5476] dup_rows_per_dst=[2580, 2794, 2730, 2860, 2614, 2686, 2764, 2706] combine_rows_per_src=[5414, 5502, 5448, 5460, 5396, 5490, 5616, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43488 payload_rows_remote=38086 dup_rows=22048 dup_rows_remote=19322 dedup_ratio=0.336426 router_tokens_per_expert=[259, 264, 259, 259, 247, 268, 273, 246, 259, 246, 239, 277, 267, 264, 258, 249, 264, 263, 242, 250, 247, 256, 234, 272, 255, 245, 237, 246, 260, 250, 251, 259, 244, 257, 274, 244, 243, 268, 264, 248, 242, 255, 290, 272, 250, 278, 247, 273, 291, 258, 253, 257, 236, 229, 264, 249, 262, 273, 260, 243, 255, 264, 216, 254, 240, 240, 243, 266, 235, 248, 264, 256, 249, 283, 283, 225, 248, 253, 260, 262, 248, 256, 251, 270, 269, 262, 258, 235, 240, 275, 254, 264, 248, 260, 279, 278, 268, 284, 258, 250, 271, 255, 251, 277, 271, 235, 247, 266, 271, 245, 233, 225, 251, 250, 250, 271, 227, 278, 263, 271, 261, 225, 251, 252, 293, 234, 270, 234] payload_rows_per_dst=[5510, 5382, 5488, 5402, 5396, 5462, 5414, 5434] dup_rows_per_dst=[2758, 2680, 2810, 2726, 2714, 2832, 2800, 2728] combine_rows_per_src=[5510, 5382, 5488, 5402, 5396, 5462, 5414, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43936 payload_rows_remote=38476 dup_rows=21600 dup_rows_remote=18958 dedup_ratio=0.329590 router_tokens_per_expert=[278, 260, 265, 273, 248, 258, 260, 279, 231, 249, 257, 267, 261, 254, 277, 244, 249, 243, 239, 254, 273, 264, 265, 256, 255, 247, 265, 267, 284, 247, 274, 259, 292, 236, 257, 259, 249, 230, 240, 263, 253, 261, 284, 275, 275, 250, 250, 246, 259, 249, 243, 244, 275, 229, 246, 240, 252, 251, 273, 260, 244, 284, 250, 261, 257, 275, 265, 274, 264, 264, 249, 229, 240, 238, 276, 232, 245, 265, 245, 233, 279, 243, 278, 258, 289, 254, 243, 264, 266, 256, 265, 243, 259, 243, 249, 232, 267, 245, 246, 274, 243, 267, 263, 272, 230, 254, 247, 260, 275, 250, 275, 235, 253, 257, 232, 258, 255, 240, 227, 253, 233, 287, 240, 269, 250, 250, 254, 253] payload_rows_per_dst=[5594, 5576, 5464, 5410, 5460, 5548, 5500, 5384] dup_rows_per_dst=[2728, 2706, 2776, 2710, 2642, 2694, 2706, 2638] combine_rows_per_src=[5594, 5576, 5464, 5410, 5460, 5548, 5500, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38406 dup_rows=21644 dup_rows_remote=18926 dedup_ratio=0.330261 router_tokens_per_expert=[258, 278, 243, 235, 262, 249, 223, 258, 267, 265, 257, 236, 259, 269, 255, 242, 258, 264, 247, 243, 253, 260, 217, 239, 239, 271, 254, 242, 254, 259, 279, 236, 252, 247, 257, 264, 270, 250, 260, 289, 269, 269, 252, 240, 250, 257, 268, 256, 268, 251, 261, 260, 269, 245, 245, 283, 261, 252, 250, 246, 240, 249, 269, 258, 267, 260, 272, 260, 264, 254, 286, 245, 256, 258, 262, 277, 262, 239, 270, 254, 259, 261, 244, 270, 263, 270, 237, 233, 267, 222, 243, 231, 252, 258, 222, 266, 243, 260, 239, 240, 240, 259, 248, 244, 251, 244, 256, 292, 233, 258, 247, 248, 263, 271, 249, 268, 280, 275, 266, 272, 289, 276, 248, 248, 241, 267, 277, 264] payload_rows_per_dst=[5486, 5436, 5524, 5484, 5496, 5620, 5374, 5472] dup_rows_per_dst=[2718, 2648, 2786, 2730, 2730, 2778, 2620, 2634] combine_rows_per_src=[5486, 5436, 5524, 5484, 5496, 5620, 5374, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=227216 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156120976 ingress_bytes=174768128 link_bytes=174768128 token_bytes=312014736 weight_movement_bytes=18874368 total_nvlink_bytes=330889104 weight_to_token_ratio=0.060492 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674542211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229048 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156630712 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032376 weight_movement_bytes=37748736 total_nvlink_bytes=350781112 weight_to_token_ratio=0.120591 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229440 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157155392 ingress_bytes=213549056 link_bytes=213549056 token_bytes=314081344 weight_movement_bytes=56623104 total_nvlink_bytes=370704448 weight_to_token_ratio=0.180282 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046121982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229632 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156229888 ingress_bytes=212623360 link_bytes=212623360 token_bytes=312230144 weight_movement_bytes=56623104 total_nvlink_bytes=368853248 weight_to_token_ratio=0.181351 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037252418e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229736 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827432 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315425128 weight_movement_bytes=18874368 total_nvlink_bytes=334299496 weight_to_token_ratio=0.059838 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38282 dup_rows=21786 dup_rows_remote=19078 dedup_ratio=0.332428 router_tokens_per_expert=[265, 246, 260, 262, 256, 285, 234, 272, 263, 252, 295, 289, 237, 290, 278, 280, 264, 259, 247, 242, 244, 238, 254, 268, 272, 239, 270, 269, 266, 246, 255, 229, 261, 273, 272, 266, 244, 252, 246, 227, 242, 265, 270, 258, 261, 254, 250, 262, 264, 256, 277, 268, 267, 288, 236, 250, 248, 228, 252, 256, 283, 269, 244, 265, 237, 260, 266, 280, 249, 261, 249, 242, 248, 263, 225, 267, 254, 259, 249, 241, 272, 263, 232, 259, 247, 256, 248, 256, 232, 265, 268, 275, 241, 258, 242, 234, 279, 255, 257, 231, 255, 226, 245, 262, 252, 256, 250, 283, 254, 260, 261, 262, 230, 253, 247, 252, 236, 268, 248, 243, 270, 245, 256, 246, 259, 242, 246, 261] payload_rows_per_dst=[5624, 5460, 5412, 5486, 5408, 5414, 5468, 5478] dup_rows_per_dst=[2904, 2664, 2794, 2816, 2692, 2682, 2708, 2526] combine_rows_per_src=[5624, 5460, 5412, 5486, 5408, 5414, 5468, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229328 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157540304 ingress_bytes=195059712 link_bytes=195059712 token_bytes=314851280 weight_movement_bytes=37748736 total_nvlink_bytes=352600016 weight_to_token_ratio=0.119894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868966185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38176 dup_rows=21832 dup_rows_remote=19140 dedup_ratio=0.333130 router_tokens_per_expert=[236, 284, 291, 285, 216, 252, 240, 256, 251, 253, 248, 261, 233, 282, 265, 243, 263, 238, 250, 255, 254, 262, 251, 245, 251, 257, 253, 270, 289, 251, 273, 248, 242, 251, 233, 275, 259, 221, 258, 266, 246, 282, 233, 256, 260, 249, 252, 257, 273, 265, 249, 249, 242, 251, 256, 265, 284, 257, 265, 247, 258, 289, 264, 265, 291, 231, 251, 265, 262, 253, 266, 239, 256, 233, 243, 261, 279, 264, 243, 243, 271, 279, 245, 278, 259, 260, 245, 248, 246, 264, 250, 261, 226, 233, 240, 258, 253, 262, 250, 230, 255, 235, 288, 261, 234, 255, 252, 257, 251, 264, 242, 248, 280, 239, 268, 255, 268, 253, 284, 214, 305, 283, 248, 241, 245, 267, 269, 244] payload_rows_per_dst=[5446, 5528, 5400, 5516, 5448, 5444, 5380, 5542] dup_rows_per_dst=[2746, 2692, 2680, 2842, 2712, 2682, 2694, 2784] combine_rows_per_src=[5446, 5528, 5400, 5516, 5448, 5444, 5380, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229440 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032512 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835584 weight_movement_bytes=37748736 total_nvlink_bytes=351584320 weight_to_token_ratio=0.120282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38146 dup_rows=21810 dup_rows_remote=18964 dedup_ratio=0.332794 router_tokens_per_expert=[243, 260, 274, 275, 275, 290, 267, 266, 238, 244, 271, 265, 251, 242, 253, 262, 254, 252, 268, 255, 246, 252, 257, 254, 267, 252, 242, 258, 273, 249, 229, 245, 247, 257, 258, 255, 240, 246, 259, 237, 254, 250, 216, 250, 240, 272, 247, 277, 255, 298, 271, 270, 256, 225, 265, 241, 261, 251, 265, 271, 252, 255, 250, 257, 259, 249, 244, 251, 270, 251, 265, 252, 260, 260, 253, 261, 239, 240, 250, 263, 255, 255, 260, 235, 237, 270, 267, 290, 243, 260, 246, 281, 238, 235, 227, 223, 288, 263, 257, 265, 261, 231, 274, 269, 236, 251, 248, 235, 265, 255, 256, 246, 265, 270, 270, 275, 278, 262, 266, 248, 256, 249, 270, 268, 237, 268, 246, 274] payload_rows_per_dst=[5342, 5458, 5394, 5350, 5582, 5562, 5458, 5580] dup_rows_per_dst=[2476, 2648, 2616, 2500, 3064, 2794, 2866, 2846] combine_rows_per_src=[5342, 5458, 5394, 5350, 5582, 5562, 5458, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229264 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598160 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967056 weight_movement_bytes=18874368 total_nvlink_bytes=331841424 weight_to_token_ratio=0.060308 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38324 dup_rows=21766 dup_rows_remote=19092 dedup_ratio=0.332123 router_tokens_per_expert=[276, 230, 254, 255, 270, 253, 242, 288, 250, 250, 263, 267, 258, 260, 244, 265, 250, 259, 232, 251, 267, 254, 239, 256, 260, 255, 248, 255, 277, 262, 260, 257, 253, 268, 244, 263, 266, 249, 219, 262, 240, 255, 258, 275, 226, 263, 260, 259, 255, 290, 285, 253, 243, 256, 257, 254, 266, 246, 236, 276, 257, 238, 248, 251, 248, 254, 250, 244, 257, 254, 255, 247, 227, 271, 223, 262, 257, 262, 263, 267, 269, 262, 287, 292, 256, 244, 228, 233, 259, 271, 270, 278, 227, 262, 270, 242, 239, 267, 285, 248, 264, 256, 239, 259, 255, 237, 250, 261, 254, 250, 259, 270, 247, 273, 247, 261, 247, 288, 252, 243, 249, 275, 250, 247, 282, 242, 256, 247] payload_rows_per_dst=[5466, 5472, 5446, 5500, 5420, 5446, 5504, 5516] dup_rows_per_dst=[2784, 2692, 2674, 2722, 2662, 2854, 2682, 2696] combine_rows_per_src=[5466, 5472, 5446, 5500, 5420, 5446, 5504, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38286 dup_rows=21682 dup_rows_remote=18992 dedup_ratio=0.330841 router_tokens_per_expert=[242, 245, 244, 245, 280, 244, 251, 267, 256, 231, 246, 282, 239, 298, 290, 278, 248, 270, 283, 259, 259, 273, 222, 258, 249, 234, 254, 263, 251, 271, 256, 263, 232, 267, 261, 264, 246, 254, 266, 232, 258, 238, 266, 265, 232, 245, 246, 252, 243, 233, 250, 257, 232, 252, 256, 276, 246, 248, 277, 241, 284, 270, 249, 251, 250, 276, 281, 273, 271, 252, 255, 277, 238, 245, 248, 267, 250, 237, 242, 240, 244, 258, 268, 272, 273, 246, 274, 232, 252, 257, 284, 268, 250, 251, 265, 235, 276, 254, 258, 227, 251, 234, 263, 253, 268, 250, 249, 246, 215, 260, 261, 280, 275, 266, 248, 229, 263, 256, 279, 241, 258, 262, 239, 258, 265, 258, 281, 274] payload_rows_per_dst=[5530, 5478, 5436, 5436, 5480, 5568, 5428, 5498] dup_rows_per_dst=[2746, 2748, 2612, 2694, 2724, 2690, 2662, 2806] combine_rows_per_src=[5530, 5478, 5436, 5436, 5480, 5568, 5428, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38174 dup_rows=21924 dup_rows_remote=19188 dedup_ratio=0.334534 router_tokens_per_expert=[223, 233, 239, 277, 271, 242, 270, 259, 229, 257, 242, 278, 260, 239, 259, 260, 243, 264, 270, 259, 253, 245, 260, 266, 229, 261, 266, 234, 274, 277, 255, 245, 265, 264, 262, 244, 251, 251, 274, 249, 255, 246, 254, 264, 241, 262, 266, 266, 264, 240, 256, 267, 250, 243, 269, 265, 260, 246, 266, 261, 253, 249, 228, 270, 256, 267, 248, 237, 256, 258, 245, 278, 261, 274, 230, 239, 285, 243, 219, 286, 242, 255, 254, 262, 256, 266, 254, 260, 260, 270, 253, 250, 261, 248, 271, 245, 288, 257, 251, 264, 250, 265, 226, 272, 236, 288, 244, 236, 271, 265, 228, 235, 281, 279, 259, 282, 285, 248, 256, 240, 240, 237, 271, 267, 252, 257, 260, 249] payload_rows_per_dst=[5412, 5440, 5476, 5438, 5414, 5512, 5396, 5524] dup_rows_per_dst=[2664, 2762, 2752, 2736, 2750, 2702, 2756, 2802] combine_rows_per_src=[5412, 5440, 5476, 5438, 5414, 5512, 5396, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38422 dup_rows=21698 dup_rows_remote=19058 dedup_ratio=0.331085 router_tokens_per_expert=[236, 249, 248, 253, 275, 230, 249, 253, 213, 296, 276, 259, 253, 250, 241, 258, 270, 243, 218, 254, 239, 252, 273, 263, 226, 261, 258, 252, 248, 261, 236, 292, 286, 274, 265, 254, 256, 263, 248, 251, 261, 250, 240, 293, 269, 251, 237, 265, 254, 247, 252, 273, 238, 268, 273, 267, 274, 260, 254, 245, 267, 230, 268, 248, 245, 252, 253, 261, 248, 249, 280, 247, 237, 240, 251, 261, 272, 253, 229, 250, 248, 259, 277, 238, 267, 261, 245, 261, 263, 252, 233, 257, 268, 239, 250, 223, 272, 257, 246, 250, 288, 239, 265, 277, 270, 279, 250, 247, 264, 254, 283, 243, 269, 270, 259, 276, 283, 267, 235, 236, 242, 251, 242, 281, 257, 262, 275, 244] payload_rows_per_dst=[5418, 5498, 5518, 5546, 5416, 5402, 5540, 5500] dup_rows_per_dst=[2660, 2594, 2808, 2690, 2640, 2680, 2828, 2798] combine_rows_per_src=[5418, 5498, 5518, 5546, 5416, 5402, 5540, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38050 dup_rows=21868 dup_rows_remote=19084 dedup_ratio=0.333679 router_tokens_per_expert=[267, 283, 242, 280, 246, 272, 266, 273, 255, 254, 264, 261, 247, 282, 246, 260, 271, 262, 261, 228, 217, 258, 256, 257, 267, 254, 253, 246, 267, 259, 248, 258, 251, 286, 238, 265, 258, 269, 256, 269, 230, 245, 244, 255, 254, 238, 285, 242, 255, 229, 232, 239, 259, 255, 243, 222, 260, 253, 245, 256, 236, 261, 228, 252, 244, 278, 257, 251, 257, 251, 274, 232, 253, 245, 268, 268, 277, 248, 253, 264, 238, 258, 278, 257, 269, 253, 239, 267, 234, 245, 254, 258, 273, 260, 259, 268, 265, 261, 280, 279, 260, 241, 235, 274, 267, 266, 230, 252, 267, 254, 259, 245, 269, 269, 258, 295, 221, 284, 249, 272, 254, 266, 250, 251, 259, 219, 267, 250] payload_rows_per_dst=[5618, 5478, 5424, 5586, 5472, 5344, 5486, 5260] dup_rows_per_dst=[2784, 2830, 2740, 2854, 2768, 2692, 2784, 2416] combine_rows_per_src=[5618, 5478, 5424, 5586, 5472, 5344, 5486, 5260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228440 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156474456 ingress_bytes=212869120 link_bytes=212869120 token_bytes=312720472 weight_movement_bytes=56623104 total_nvlink_bytes=369343576 weight_to_token_ratio=0.181066 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039607170e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229664 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204768 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179872 weight_movement_bytes=18874368 total_nvlink_bytes=333054240 weight_to_token_ratio=0.060075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38262 dup_rows=21946 dup_rows_remote=19342 dedup_ratio=0.334869 router_tokens_per_expert=[253, 255, 260, 285, 264, 250, 245, 255, 264, 249, 262, 249, 259, 250, 278, 270, 275, 283, 284, 279, 253, 257, 268, 254, 257, 243, 250, 246, 249, 237, 254, 262, 233, 269, 280, 244, 248, 243, 272, 228, 245, 241, 258, 290, 268, 256, 298, 273, 270, 257, 241, 265, 257, 266, 246, 272, 264, 265, 251, 252, 238, 272, 236, 260, 234, 242, 274, 279, 249, 250, 253, 250, 245, 243, 282, 234, 246, 271, 242, 281, 249, 241, 213, 233, 263, 249, 262, 267, 263, 282, 266, 255, 248, 258, 236, 254, 257, 266, 252, 235, 253, 227, 258, 233, 241, 255, 253, 252, 254, 244, 226, 260, 243, 265, 254, 231, 278, 272, 278, 246, 255, 252, 276, 246, 270, 247, 271, 247] payload_rows_per_dst=[5454, 5508, 5552, 5468, 5436, 5424, 5328, 5420] dup_rows_per_dst=[2842, 2794, 2740, 2756, 2714, 2654, 2604, 2842] combine_rows_per_src=[5454, 5508, 5552, 5468, 5436, 5424, 5328, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229112 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048568 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868024 weight_movement_bytes=18874368 total_nvlink_bytes=332742392 weight_to_token_ratio=0.060135 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229448 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156590152 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950856 weight_movement_bytes=18874368 total_nvlink_bytes=331825224 weight_to_token_ratio=0.060311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229920 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157606432 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982944 weight_movement_bytes=18874368 total_nvlink_bytes=333857312 weight_to_token_ratio=0.059922 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=228536 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156081336 ingress_bytes=193601536 link_bytes=193601536 token_bytes=311934136 weight_movement_bytes=37748736 total_nvlink_bytes=349682872 weight_to_token_ratio=0.121015 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854994660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38172 dup_rows=21868 dup_rows_remote=19106 dedup_ratio=0.333679 router_tokens_per_expert=[234, 270, 280, 268, 265, 275, 233, 259, 267, 259, 264, 257, 238, 247, 242, 242, 281, 246, 276, 249, 240, 241, 267, 248, 230, 260, 243, 261, 243, 297, 296, 251, 267, 255, 250, 246, 255, 260, 237, 229, 229, 241, 298, 263, 255, 299, 250, 257, 227, 257, 244, 268, 281, 258, 225, 270, 257, 253, 252, 263, 248, 242, 231, 244, 287, 270, 275, 273, 257, 240, 254, 242, 250, 261, 243, 254, 232, 281, 254, 253, 258, 265, 280, 279, 251, 267, 285, 264, 255, 280, 246, 243, 240, 260, 242, 229, 236, 257, 273, 261, 258, 259, 262, 253, 243, 230, 273, 273, 267, 260, 260, 254, 249, 222, 242, 273, 270, 256, 262, 243, 246, 256, 237, 282, 249, 211, 275, 266] payload_rows_per_dst=[5486, 5496, 5420, 5488, 5448, 5448, 5460, 5422] dup_rows_per_dst=[2714, 2762, 2762, 2552, 2804, 2840, 2778, 2656] combine_rows_per_src=[5486, 5496, 5420, 5488, 5448, 5448, 5460, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=230416 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156951568 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313672720 weight_movement_bytes=18874368 total_nvlink_bytes=332547088 weight_to_token_ratio=0.060172 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229112 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156581624 ingress_bytes=212975616 link_bytes=212975616 token_bytes=312934136 weight_movement_bytes=56623104 total_nvlink_bytes=369557240 weight_to_token_ratio=0.180943 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040627562e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38130 dup_rows=21882 dup_rows_remote=19122 dedup_ratio=0.333893 router_tokens_per_expert=[253, 260, 257, 265, 249, 285, 251, 258, 249, 227, 242, 264, 234, 259, 249, 232, 249, 251, 232, 233, 264, 250, 250, 271, 250, 260, 253, 270, 274, 263, 239, 261, 256, 239, 240, 269, 292, 279, 256, 270, 247, 250, 252, 268, 265, 243, 262, 254, 264, 266, 238, 263, 272, 261, 259, 244, 253, 265, 258, 268, 238, 253, 282, 236, 288, 268, 269, 244, 248, 244, 266, 246, 250, 258, 249, 269, 247, 253, 260, 274, 234, 270, 260, 242, 254, 250, 259, 272, 248, 268, 249, 246, 262, 256, 250, 255, 258, 247, 281, 261, 247, 247, 248, 256, 297, 258, 266, 253, 264, 230, 260, 275, 234, 256, 228, 263, 227, 245, 262, 224, 275, 264, 283, 275, 266, 240, 245, 259] payload_rows_per_dst=[5356, 5442, 5524, 5454, 5512, 5522, 5478, 5366] dup_rows_per_dst=[2712, 2698, 2760, 2786, 2754, 2628, 2818, 2726] combine_rows_per_src=[5356, 5442, 5524, 5454, 5512, 5522, 5478, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38352 dup_rows=21688 dup_rows_remote=18908 dedup_ratio=0.330933 router_tokens_per_expert=[244, 272, 260, 279, 252, 259, 248, 249, 255, 280, 239, 256, 250, 267, 267, 223, 283, 255, 284, 238, 283, 257, 239, 256, 258, 264, 296, 254, 227, 251, 254, 282, 256, 245, 258, 219, 218, 267, 239, 240, 257, 267, 272, 244, 236, 272, 244, 270, 263, 266, 248, 272, 274, 252, 257, 264, 260, 251, 261, 243, 265, 254, 250, 275, 245, 244, 248, 249, 277, 229, 253, 270, 260, 242, 243, 246, 263, 233, 236, 256, 250, 256, 261, 234, 283, 254, 251, 263, 252, 257, 258, 272, 254, 260, 246, 265, 272, 254, 262, 253, 269, 254, 246, 248, 249, 261, 237, 233, 274, 265, 231, 272, 277, 252, 235, 257, 299, 275, 239, 280, 263, 229, 251, 237, 257, 257, 262, 268] payload_rows_per_dst=[5552, 5516, 5374, 5520, 5374, 5508, 5508, 5496] dup_rows_per_dst=[2648, 2726, 2592, 2790, 2614, 2776, 2762, 2780] combine_rows_per_src=[5552, 5516, 5374, 5520, 5374, 5508, 5508, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38230 dup_rows=21852 dup_rows_remote=19176 dedup_ratio=0.333435 router_tokens_per_expert=[241, 261, 243, 244, 235, 279, 278, 263, 264, 256, 252, 231, 268, 243, 265, 256, 241, 254, 250, 273, 250, 264, 245, 270, 272, 261, 255, 256, 260, 250, 254, 254, 269, 246, 272, 242, 235, 263, 235, 267, 263, 230, 271, 273, 267, 239, 262, 253, 273, 229, 277, 271, 226, 245, 247, 261, 277, 254, 279, 270, 241, 226, 249, 240, 244, 240, 263, 274, 261, 265, 243, 268, 257, 282, 251, 257, 239, 271, 251, 269, 238, 263, 256, 254, 256, 267, 246, 241, 272, 261, 267, 222, 260, 239, 237, 267, 272, 276, 258, 279, 258, 239, 261, 259, 229, 238, 251, 267, 265, 252, 272, 254, 262, 252, 302, 240, 242, 239, 257, 246, 259, 272, 268, 241, 280, 239, 267, 251] payload_rows_per_dst=[5458, 5444, 5444, 5454, 5476, 5408, 5464, 5536] dup_rows_per_dst=[2700, 2774, 2730, 2676, 2794, 2684, 2796, 2698] combine_rows_per_src=[5458, 5444, 5444, 5454, 5476, 5408, 5464, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38296 dup_rows=21718 dup_rows_remote=19014 dedup_ratio=0.331390 router_tokens_per_expert=[268, 254, 267, 235, 247, 277, 244, 251, 225, 252, 254, 241, 278, 254, 237, 245, 263, 249, 250, 236, 258, 241, 287, 236, 244, 260, 268, 243, 280, 265, 245, 237, 271, 245, 264, 249, 270, 255, 253, 263, 264, 263, 232, 277, 257, 261, 260, 244, 266, 290, 234, 238, 216, 260, 222, 276, 255, 252, 233, 249, 265, 232, 268, 230, 269, 291, 260, 296, 274, 257, 281, 258, 235, 267, 232, 232, 264, 254, 259, 260, 256, 268, 256, 248, 258, 257, 286, 246, 256, 262, 235, 240, 281, 248, 261, 255, 268, 235, 256, 279, 260, 267, 272, 289, 237, 251, 239, 273, 273, 242, 269, 245, 230, 281, 244, 249, 254, 236, 264, 280, 273, 242, 242, 279, 263, 248, 252, 269] payload_rows_per_dst=[5406, 5424, 5510, 5402, 5532, 5522, 5534, 5488] dup_rows_per_dst=[2652, 2700, 2746, 2570, 2846, 2704, 2776, 2724] combine_rows_per_src=[5406, 5424, 5510, 5402, 5532, 5522, 5534, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38404 dup_rows=21718 dup_rows_remote=18972 dedup_ratio=0.331390 router_tokens_per_expert=[226, 261, 299, 257, 246, 258, 268, 234, 251, 238, 234, 223, 224, 250, 281, 270, 261, 232, 276, 269, 239, 244, 232, 247, 261, 273, 249, 248, 272, 255, 265, 265, 275, 228, 271, 280, 278, 263, 252, 246, 268, 264, 267, 246, 264, 267, 251, 248, 250, 241, 247, 254, 268, 254, 294, 217, 280, 228, 253, 245, 262, 260, 256, 273, 250, 240, 243, 275, 274, 248, 261, 275, 258, 246, 261, 245, 248, 258, 260, 238, 246, 251, 297, 278, 225, 250, 284, 240, 258, 246, 273, 263, 279, 298, 261, 270, 251, 247, 257, 241, 263, 233, 265, 234, 277, 259, 247, 230, 239, 268, 259, 280, 245, 250, 294, 251, 258, 263, 254, 232, 237, 248, 255, 256, 242, 261, 267, 248] payload_rows_per_dst=[5470, 5494, 5522, 5512, 5414, 5506, 5444, 5456] dup_rows_per_dst=[2570, 2682, 2814, 2652, 2746, 2932, 2656, 2666] combine_rows_per_src=[5470, 5494, 5522, 5512, 5414, 5506, 5444, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38268 dup_rows=21846 dup_rows_remote=19114 dedup_ratio=0.333344 router_tokens_per_expert=[239, 266, 250, 254, 246, 234, 254, 265, 242, 244, 274, 256, 253, 271, 245, 244, 234, 275, 252, 256, 251, 254, 241, 252, 258, 288, 255, 272, 272, 256, 256, 256, 268, 238, 256, 237, 260, 271, 237, 244, 277, 280, 258, 251, 278, 275, 269, 282, 261, 261, 242, 265, 257, 283, 267, 251, 288, 237, 287, 241, 271, 249, 258, 261, 250, 247, 253, 263, 257, 256, 240, 255, 227, 261, 247, 267, 240, 250, 254, 270, 248, 253, 238, 269, 254, 229, 271, 238, 270, 220, 279, 254, 267, 253, 259, 245, 279, 265, 255, 245, 240, 248, 260, 258, 229, 260, 254, 292, 280, 267, 265, 249, 237, 249, 266, 269, 252, 245, 251, 256, 245, 246, 231, 250, 257, 273, 247, 239] payload_rows_per_dst=[5422, 5422, 5580, 5520, 5504, 5458, 5370, 5414] dup_rows_per_dst=[2732, 2782, 2782, 2838, 2956, 2608, 2536, 2612] combine_rows_per_src=[5422, 5422, 5580, 5520, 5504, 5458, 5370, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229008 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156409488 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312589968 weight_movement_bytes=37748736 total_nvlink_bytes=350338704 weight_to_token_ratio=0.120761 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229624 combine_payload_bytes=156590080 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156819704 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313409784 weight_movement_bytes=37748736 total_nvlink_bytes=351158520 weight_to_token_ratio=0.120445 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229040 combine_payload_bytes=157089792 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157318832 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314408624 weight_movement_bytes=18874368 total_nvlink_bytes=333282992 weight_to_token_ratio=0.060031 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38378 dup_rows=21778 dup_rows_remote=19150 dedup_ratio=0.332306 router_tokens_per_expert=[235, 251, 280, 269, 261, 280, 282, 252, 291, 253, 268, 269, 264, 250, 251, 265, 246, 263, 244, 243, 229, 266, 262, 249, 264, 262, 264, 243, 232, 266, 293, 246, 243, 246, 239, 280, 247, 239, 268, 268, 286, 269, 256, 265, 237, 264, 251, 247, 255, 270, 256, 285, 270, 267, 253, 247, 235, 237, 261, 277, 241, 229, 277, 241, 267, 264, 249, 244, 271, 258, 252, 274, 285, 239, 261, 264, 257, 275, 249, 243, 236, 271, 267, 254, 263, 241, 272, 275, 256, 259, 249, 240, 226, 259, 250, 282, 237, 279, 252, 235, 244, 237, 251, 261, 272, 239, 248, 233, 248, 245, 266, 257, 274, 235, 262, 250, 248, 251, 235, 254, 272, 256, 264, 254, 231, 236, 240, 251] payload_rows_per_dst=[5504, 5520, 5516, 5432, 5432, 5516, 5380, 5458] dup_rows_per_dst=[2938, 2624, 2694, 2770, 2872, 2684, 2628, 2568] combine_rows_per_src=[5504, 5520, 5516, 5432, 5432, 5516, 5380, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229240 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157089656 ingress_bytes=213483520 link_bytes=213483520 token_bytes=313950072 weight_movement_bytes=56623104 total_nvlink_bytes=370573176 weight_to_token_ratio=0.180357 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045494048e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229504 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157532288 ingress_bytes=195051520 link_bytes=195051520 token_bytes=314835072 weight_movement_bytes=37748736 total_nvlink_bytes=352583808 weight_to_token_ratio=0.119900 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868887693e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229528 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156975256 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720984 weight_movement_bytes=18874368 total_nvlink_bytes=332595352 weight_to_token_ratio=0.060163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38334 dup_rows=21708 dup_rows_remote=18976 dedup_ratio=0.331238 router_tokens_per_expert=[246, 231, 235, 267, 274, 256, 245, 267, 253, 256, 249, 253, 227, 274, 291, 269, 248, 233, 227, 256, 231, 286, 256, 290, 248, 262, 285, 293, 243, 232, 278, 245, 244, 220, 252, 242, 259, 267, 248, 258, 252, 280, 243, 250, 231, 230, 284, 251, 248, 247, 302, 256, 263, 260, 257, 239, 269, 273, 233, 240, 258, 266, 263, 245, 265, 289, 258, 217, 235, 260, 232, 283, 272, 254, 282, 272, 245, 277, 234, 264, 268, 258, 250, 232, 279, 253, 240, 242, 263, 241, 261, 278, 244, 269, 265, 252, 247, 244, 270, 257, 239, 274, 229, 268, 273, 287, 257, 262, 265, 241, 257, 260, 246, 274, 241, 250, 260, 269, 240, 246, 264, 242, 256, 230, 251, 255, 285, 259] payload_rows_per_dst=[5472, 5494, 5442, 5490, 5496, 5444, 5522, 5468] dup_rows_per_dst=[2714, 2732, 2580, 2748, 2782, 2746, 2738, 2668] combine_rows_per_src=[5472, 5494, 5442, 5490, 5496, 5444, 5522, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=230112 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157426400 ingress_bytes=176070656 link_bytes=176070656 token_bytes=314622688 weight_movement_bytes=18874368 total_nvlink_bytes=333497056 weight_to_token_ratio=0.059990 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687022394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229240 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157245304 ingress_bytes=194764800 link_bytes=194764800 token_bytes=314261368 weight_movement_bytes=37748736 total_nvlink_bytes=352010104 weight_to_token_ratio=0.120119 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866140483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44040 payload_rows_remote=38548 dup_rows=21496 dup_rows_remote=18668 dedup_ratio=0.328003 router_tokens_per_expert=[261, 253, 258, 264, 252, 254, 258, 257, 246, 273, 251, 212, 230, 275, 257, 278, 256, 218, 248, 253, 276, 245, 262, 261, 269, 266, 240, 241, 260, 275, 270, 255, 265, 251, 277, 261, 252, 284, 231, 238, 279, 260, 254, 247, 235, 289, 250, 287, 275, 248, 256, 247, 277, 241, 265, 270, 278, 262, 262, 235, 239, 242, 255, 259, 270, 248, 234, 244, 249, 240, 266, 235, 253, 265, 238, 283, 238, 240, 235, 268, 270, 262, 261, 250, 248, 227, 257, 250, 253, 228, 242, 242, 279, 268, 266, 280, 262, 268, 232, 240, 266, 250, 285, 246, 251, 253, 247, 255, 288, 249, 251, 271, 235, 278, 243, 250, 242, 271, 239, 256, 263, 257, 272, 251, 263, 249, 282, 269] payload_rows_per_dst=[5474, 5536, 5492, 5518, 5472, 5530, 5464, 5554] dup_rows_per_dst=[2684, 2654, 2828, 2704, 2540, 2636, 2764, 2686] combine_rows_per_src=[5474, 5536, 5492, 5518, 5472, 5530, 5464, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38316 dup_rows=21764 dup_rows_remote=19164 dedup_ratio=0.332092 router_tokens_per_expert=[267, 246, 241, 241, 239, 269, 253, 246, 212, 259, 243, 245, 260, 261, 250, 227, 243, 271, 271, 275, 266, 243, 273, 271, 215, 253, 259, 251, 268, 239, 251, 229, 272, 262, 273, 257, 254, 247, 254, 302, 264, 256, 248, 261, 250, 253, 258, 251, 254, 243, 246, 252, 276, 240, 256, 242, 264, 244, 255, 234, 265, 220, 273, 264, 269, 255, 271, 232, 257, 273, 238, 255, 298, 256, 281, 249, 257, 244, 268, 277, 254, 235, 229, 285, 242, 251, 255, 284, 270, 229, 256, 252, 255, 270, 256, 257, 254, 270, 260, 256, 273, 229, 256, 270, 274, 267, 239, 281, 259, 271, 256, 264, 243, 228, 286, 238, 246, 255, 281, 253, 280, 272, 245, 259, 254, 237, 247, 278] payload_rows_per_dst=[5386, 5414, 5582, 5456, 5528, 5390, 5562, 5454] dup_rows_per_dst=[2532, 2742, 2742, 2600, 2832, 2770, 2796, 2750] combine_rows_per_src=[5386, 5414, 5582, 5456, 5528, 5390, 5562, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38460 dup_rows=21598 dup_rows_remote=18906 dedup_ratio=0.329559 router_tokens_per_expert=[256, 258, 274, 268, 250, 254, 254, 282, 236, 266, 282, 238, 232, 259, 236, 237, 252, 253, 241, 236, 255, 253, 237, 249, 267, 247, 232, 252, 239, 232, 270, 269, 286, 281, 233, 256, 241, 285, 257, 261, 263, 269, 266, 268, 261, 291, 268, 244, 286, 233, 283, 248, 264, 256, 263, 261, 263, 252, 261, 261, 264, 276, 243, 247, 255, 257, 260, 234, 269, 244, 258, 264, 268, 259, 248, 264, 249, 246, 277, 259, 264, 262, 235, 260, 249, 261, 280, 291, 237, 272, 266, 240, 271, 265, 281, 274, 239, 246, 239, 251, 259, 240, 231, 227, 256, 231, 258, 240, 257, 267, 245, 263, 303, 245, 239, 257, 230, 248, 253, 244, 248, 251, 261, 255, 258, 250, 246, 255] payload_rows_per_dst=[5260, 5376, 5582, 5450, 5588, 5586, 5618, 5478] dup_rows_per_dst=[2382, 2508, 2878, 2644, 2862, 2830, 2802, 2692] combine_rows_per_src=[5260, 5376, 5582, 5450, 5588, 5586, 5618, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38498 dup_rows=21692 dup_rows_remote=19020 dedup_ratio=0.330994 router_tokens_per_expert=[260, 261, 259, 276, 240, 238, 262, 315, 260, 253, 246, 250, 280, 265, 253, 274, 253, 253, 241, 245, 266, 254, 254, 266, 266, 319, 255, 278, 273, 237, 260, 276, 252, 255, 247, 235, 249, 255, 277, 252, 240, 276, 272, 279, 258, 243, 239, 245, 237, 248, 237, 252, 256, 245, 288, 281, 248, 236, 281, 235, 279, 255, 233, 248, 267, 246, 264, 246, 265, 229, 287, 254, 236, 253, 257, 254, 220, 251, 232, 246, 250, 230, 252, 239, 260, 259, 242, 244, 261, 225, 246, 274, 256, 267, 239, 265, 246, 229, 250, 262, 238, 261, 236, 282, 286, 247, 249, 239, 250, 253, 255, 253, 272, 231, 247, 260, 276, 282, 264, 264, 269, 254, 270, 261, 280, 280, 253, 232] payload_rows_per_dst=[5578, 5566, 5518, 5486, 5438, 5346, 5430, 5482] dup_rows_per_dst=[2806, 2826, 2630, 2632, 2576, 2672, 2642, 2908] combine_rows_per_src=[5578, 5566, 5518, 5486, 5438, 5346, 5430, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38064 dup_rows=21966 dup_rows_remote=19228 dedup_ratio=0.335175 router_tokens_per_expert=[251, 232, 266, 242, 243, 268, 266, 243, 256, 256, 263, 271, 236, 252, 269, 245, 259, 251, 264, 257, 272, 248, 274, 232, 250, 294, 228, 252, 270, 270, 269, 260, 256, 246, 255, 237, 243, 256, 264, 285, 246, 264, 269, 265, 255, 251, 250, 249, 243, 285, 247, 244, 255, 269, 259, 258, 261, 227, 252, 258, 275, 279, 261, 264, 287, 256, 258, 264, 276, 242, 250, 257, 249, 245, 260, 267, 280, 245, 247, 239, 261, 251, 275, 263, 251, 265, 271, 241, 240, 239, 225, 256, 279, 245, 259, 255, 244, 265, 247, 231, 251, 261, 265, 229, 293, 251, 246, 268, 244, 238, 244, 265, 239, 258, 260, 241, 256, 247, 255, 262, 266, 281, 258, 246, 276, 250, 242, 254] payload_rows_per_dst=[5462, 5512, 5454, 5368, 5506, 5486, 5382, 5400] dup_rows_per_dst=[2656, 2788, 2728, 2906, 2738, 2666, 2702, 2782] combine_rows_per_src=[5462, 5512, 5454, 5368, 5506, 5486, 5382, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157892608 dispatch_meta_bytes=228864 combine_payload_bytes=157892608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158121472 ingress_bytes=195641344 link_bytes=195641344 token_bytes=316014080 weight_movement_bytes=37748736 total_nvlink_bytes=353762816 weight_to_token_ratio=0.119453 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874539097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229920 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157172256 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114592 weight_movement_bytes=18874368 total_nvlink_bytes=332988960 weight_to_token_ratio=0.060088 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38226 dup_rows=21792 dup_rows_remote=19184 dedup_ratio=0.332520 router_tokens_per_expert=[232, 267, 270, 239, 258, 287, 238, 260, 244, 273, 259, 236, 256, 240, 249, 255, 249, 244, 239, 260, 264, 263, 269, 242, 262, 256, 240, 273, 249, 241, 269, 251, 263, 224, 234, 244, 246, 256, 279, 271, 272, 255, 249, 242, 262, 275, 259, 280, 247, 288, 240, 251, 248, 251, 275, 258, 249, 267, 284, 241, 274, 227, 248, 277, 237, 251, 261, 248, 266, 256, 286, 253, 260, 258, 255, 257, 253, 247, 250, 251, 245, 252, 255, 280, 267, 234, 214, 244, 247, 230, 254, 256, 252, 255, 295, 232, 285, 223, 290, 281, 257, 269, 284, 246, 251, 250, 254, 242, 298, 268, 242, 225, 265, 255, 254, 240, 263, 286, 267, 267, 254, 242, 257, 276, 236, 264, 248, 258] payload_rows_per_dst=[5518, 5632, 5550, 5522, 5418, 5402, 5388, 5314] dup_rows_per_dst=[2608, 2910, 2898, 2788, 2760, 2622, 2716, 2490] combine_rows_per_src=[5518, 5632, 5550, 5522, 5418, 5402, 5388, 5314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157532160 dispatch_meta_bytes=229464 combine_payload_bytes=157532160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157761624 ingress_bytes=176406528 link_bytes=176406528 token_bytes=315293784 weight_movement_bytes=18874368 total_nvlink_bytes=334168152 weight_to_token_ratio=0.059863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690240554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157687808 dispatch_meta_bytes=230072 combine_payload_bytes=157687808 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157917880 ingress_bytes=176562176 link_bytes=176562176 token_bytes=315605688 weight_movement_bytes=18874368 total_nvlink_bytes=334480056 weight_to_token_ratio=0.059804 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691731896e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38280 dup_rows=21736 dup_rows_remote=19028 dedup_ratio=0.331665 router_tokens_per_expert=[262, 254, 260, 256, 254, 237, 251, 243, 289, 253, 215, 259, 280, 274, 255, 259, 250, 272, 254, 269, 265, 276, 275, 262, 248, 274, 272, 241, 247, 254, 255, 253, 250, 276, 269, 274, 271, 275, 239, 264, 246, 270, 262, 254, 260, 266, 239, 270, 271, 235, 260, 237, 236, 271, 252, 267, 249, 264, 257, 254, 250, 263, 237, 256, 253, 270, 266, 254, 256, 244, 255, 264, 244, 237, 252, 236, 252, 269, 230, 269, 257, 255, 249, 264, 233, 246, 276, 246, 243, 267, 239, 248, 289, 271, 237, 290, 278, 250, 247, 256, 265, 256, 253, 266, 252, 271, 276, 250, 266, 256, 211, 261, 244, 240, 228, 241, 240, 248, 218, 269, 303, 244, 253, 233, 253, 237, 244, 286] payload_rows_per_dst=[5446, 5494, 5562, 5432, 5476, 5494, 5520, 5376] dup_rows_per_dst=[2756, 2840, 2808, 2686, 2626, 2726, 2708, 2586] combine_rows_per_src=[5446, 5494, 5562, 5432, 5476, 5494, 5520, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=229168 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156139312 ingress_bytes=174784512 link_bytes=174784512 token_bytes=312049456 weight_movement_bytes=18874368 total_nvlink_bytes=330923824 weight_to_token_ratio=0.060485 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674699195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38330 dup_rows=21720 dup_rows_remote=19044 dedup_ratio=0.331421 router_tokens_per_expert=[276, 267, 285, 248, 245, 263, 270, 232, 272, 260, 251, 268, 251, 238, 282, 267, 223, 262, 281, 244, 258, 255, 252, 256, 230, 252, 235, 292, 261, 243, 252, 285, 247, 250, 225, 248, 251, 235, 247, 266, 281, 238, 229, 267, 241, 238, 262, 268, 284, 267, 257, 240, 250, 242, 271, 235, 267, 260, 276, 248, 232, 255, 272, 250, 230, 233, 253, 237, 254, 253, 250, 254, 264, 262, 262, 266, 257, 243, 259, 263, 288, 257, 259, 256, 249, 249, 255, 281, 253, 265, 252, 236, 278, 239, 262, 250, 256, 248, 230, 273, 258, 255, 251, 262, 239, 268, 260, 259, 280, 272, 232, 283, 265, 272, 260, 240, 241, 259, 243, 263, 269, 243, 206, 302, 244, 281, 268, 262] payload_rows_per_dst=[5528, 5486, 5420, 5476, 5400, 5548, 5506, 5452] dup_rows_per_dst=[2822, 2676, 2566, 2736, 2680, 2710, 2746, 2784] combine_rows_per_src=[5528, 5486, 5420, 5476, 5400, 5548, 5506, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229640 combine_payload_bytes=156573696 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156803336 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313377032 weight_movement_bytes=37748736 total_nvlink_bytes=351125768 weight_to_token_ratio=0.120458 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229232 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157024112 ingress_bytes=232292352 link_bytes=232292352 token_bytes=313818992 weight_movement_bytes=75497472 total_nvlink_bytes=389316464 weight_to_token_ratio=0.240576 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225711022e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229496 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157229176 ingress_bytes=232497152 link_bytes=232497152 token_bytes=314228856 weight_movement_bytes=75497472 total_nvlink_bytes=389726328 weight_to_token_ratio=0.240263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227673315e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38496 dup_rows=21606 dup_rows_remote=18962 dedup_ratio=0.329681 router_tokens_per_expert=[261, 249, 252, 250, 212, 254, 262, 254, 269, 241, 259, 246, 289, 256, 238, 267, 264, 260, 258, 233, 259, 253, 255, 249, 263, 261, 264, 277, 236, 241, 266, 258, 236, 247, 246, 251, 268, 248, 262, 243, 244, 277, 246, 276, 258, 246, 253, 238, 247, 245, 251, 227, 267, 247, 265, 221, 265, 264, 247, 259, 270, 287, 277, 251, 248, 231, 244, 262, 267, 260, 263, 262, 266, 247, 260, 254, 256, 257, 251, 249, 245, 256, 248, 266, 225, 252, 256, 248, 290, 259, 292, 240, 237, 253, 264, 254, 247, 247, 257, 246, 277, 287, 247, 245, 263, 285, 256, 254, 283, 258, 273, 273, 245, 254, 249, 270, 253, 234, 279, 267, 247, 271, 267, 254, 253, 248, 264, 268] payload_rows_per_dst=[5462, 5530, 5434, 5508, 5468, 5528, 5490, 5510] dup_rows_per_dst=[2656, 2664, 2644, 2672, 2686, 2642, 2906, 2736] combine_rows_per_src=[5462, 5530, 5434, 5508, 5468, 5528, 5490, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38404 dup_rows=21666 dup_rows_remote=19104 dedup_ratio=0.330597 router_tokens_per_expert=[262, 253, 284, 276, 271, 251, 253, 266, 256, 276, 272, 265, 246, 274, 268, 270, 258, 240, 254, 290, 253, 237, 239, 253, 263, 271, 272, 269, 270, 257, 273, 276, 257, 248, 266, 250, 246, 255, 248, 258, 256, 246, 270, 245, 249, 245, 252, 248, 255, 236, 252, 231, 247, 263, 249, 235, 270, 247, 282, 275, 243, 246, 227, 256, 271, 249, 258, 255, 261, 247, 249, 246, 270, 287, 252, 229, 242, 249, 257, 246, 262, 284, 251, 258, 268, 252, 252, 257, 266, 241, 230, 263, 255, 281, 224, 268, 276, 229, 272, 256, 231, 249, 251, 250, 275, 258, 282, 244, 248, 268, 259, 244, 239, 251, 255, 262, 247, 254, 267, 231, 252, 251, 248, 248, 279, 260, 212, 269] payload_rows_per_dst=[5552, 5584, 5458, 5466, 5360, 5502, 5504, 5444] dup_rows_per_dst=[2934, 2766, 2620, 2562, 2776, 2722, 2680, 2606] combine_rows_per_src=[5552, 5584, 5458, 5466, 5360, 5502, 5504, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38328 dup_rows=21662 dup_rows_remote=18938 dedup_ratio=0.330536 router_tokens_per_expert=[281, 249, 241, 271, 266, 244, 242, 264, 251, 249, 255, 255, 270, 218, 250, 236, 242, 256, 235, 282, 245, 228, 241, 256, 257, 285, 257, 262, 258, 241, 259, 282, 240, 247, 267, 253, 262, 258, 256, 271, 249, 252, 291, 274, 262, 229, 238, 270, 278, 236, 233, 254, 281, 279, 261, 272, 262, 274, 274, 262, 250, 250, 244, 258, 260, 236, 251, 241, 259, 227, 249, 271, 257, 267, 222, 256, 260, 256, 277, 254, 289, 266, 261, 268, 250, 274, 238, 245, 260, 253, 236, 255, 268, 263, 279, 250, 266, 264, 247, 248, 243, 262, 250, 267, 252, 255, 237, 249, 257, 247, 259, 240, 249, 252, 261, 248, 268, 257, 272, 245, 250, 238, 237, 268, 254, 257, 265, 291] payload_rows_per_dst=[5408, 5410, 5498, 5590, 5448, 5550, 5424, 5546] dup_rows_per_dst=[2676, 2528, 2738, 2796, 2776, 2760, 2664, 2724] combine_rows_per_src=[5408, 5410, 5498, 5590, 5448, 5550, 5424, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38358 dup_rows=21696 dup_rows_remote=19184 dedup_ratio=0.331055 router_tokens_per_expert=[213, 251, 268, 242, 269, 247, 249, 267, 248, 260, 262, 270, 275, 279, 241, 272, 262, 257, 276, 282, 263, 249, 240, 238, 248, 283, 265, 260, 250, 261, 254, 246, 255, 261, 246, 248, 256, 268, 257, 255, 246, 261, 263, 246, 242, 287, 246, 249, 269, 276, 265, 236, 242, 279, 266, 266, 256, 254, 253, 241, 267, 226, 286, 289, 271, 254, 246, 269, 245, 256, 268, 265, 275, 226, 248, 271, 236, 248, 254, 263, 248, 242, 237, 224, 241, 247, 276, 237, 267, 255, 254, 245, 268, 238, 241, 277, 262, 269, 239, 259, 246, 271, 274, 231, 255, 235, 265, 235, 269, 258, 245, 269, 290, 232, 274, 256, 271, 248, 278, 259, 238, 233, 248, 264, 256, 234, 238, 271] payload_rows_per_dst=[5474, 5540, 5440, 5532, 5454, 5482, 5446, 5472] dup_rows_per_dst=[2752, 2728, 2732, 2810, 2736, 2512, 2718, 2708] combine_rows_per_src=[5474, 5540, 5440, 5532, 5454, 5482, 5446, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43390 payload_rows_remote=37950 dup_rows=22146 dup_rows_remote=19398 dedup_ratio=0.337921 router_tokens_per_expert=[271, 249, 270, 270, 252, 240, 267, 283, 225, 259, 281, 266, 270, 290, 240, 247, 244, 257, 252, 254, 253, 238, 255, 299, 265, 273, 281, 254, 271, 258, 259, 271, 253, 254, 242, 271, 251, 261, 239, 233, 278, 241, 265, 255, 276, 266, 258, 241, 233, 265, 255, 252, 225, 260, 212, 250, 265, 243, 247, 255, 230, 240, 240, 265, 250, 249, 255, 267, 263, 261, 251, 237, 257, 251, 258, 253, 271, 268, 267, 236, 252, 254, 278, 261, 259, 254, 259, 263, 253, 241, 255, 217, 251, 281, 248, 245, 242, 261, 257, 247, 290, 246, 245, 248, 280, 288, 256, 248, 257, 237, 237, 274, 257, 241, 263, 267, 241, 264, 246, 256, 252, 250, 276, 258, 262, 252, 250, 270] payload_rows_per_dst=[5574, 5560, 5396, 5276, 5440, 5350, 5356, 5438] dup_rows_per_dst=[2786, 2808, 2772, 2598, 2748, 2792, 2870, 2772] combine_rows_per_src=[5574, 5560, 5396, 5276, 5440, 5350, 5356, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157679616 dispatch_meta_bytes=229832 combine_payload_bytes=157679616 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157909448 ingress_bytes=176553984 link_bytes=176553984 token_bytes=315589064 weight_movement_bytes=18874368 total_nvlink_bytes=334463432 weight_to_token_ratio=0.059807 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691653405e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=230032 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532816 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835600 weight_movement_bytes=18874368 total_nvlink_bytes=333709968 weight_to_token_ratio=0.059950 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38002 dup_rows=21914 dup_rows_remote=18936 dedup_ratio=0.334381 router_tokens_per_expert=[271, 249, 258, 272, 237, 288, 223, 263, 263, 244, 257, 249, 266, 270, 252, 261, 262, 253, 273, 249, 243, 256, 260, 254, 231, 231, 254, 265, 264, 246, 254, 250, 243, 239, 258, 244, 255, 243, 250, 254, 245, 261, 234, 249, 268, 254, 259, 240, 257, 262, 230, 263, 258, 253, 253, 256, 248, 272, 268, 255, 254, 287, 242, 268, 283, 266, 266, 240, 279, 290, 275, 237, 257, 230, 261, 281, 297, 276, 250, 227, 290, 259, 260, 253, 265, 262, 264, 256, 242, 248, 253, 245, 232, 235, 261, 234, 272, 269, 245, 224, 259, 259, 277, 275, 249, 229, 256, 259, 264, 264, 260, 258, 266, 256, 245, 248, 259, 244, 251, 265, 229, 265, 280, 257, 260, 259, 270, 231] payload_rows_per_dst=[5620, 5386, 5390, 5492, 5448, 5284, 5546, 5456] dup_rows_per_dst=[2978, 2706, 2602, 2920, 2730, 2574, 2692, 2712] combine_rows_per_src=[5620, 5386, 5390, 5492, 5448, 5284, 5546, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229064 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220552 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212040 weight_movement_bytes=18874368 total_nvlink_bytes=333086408 weight_to_token_ratio=0.060069 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38240 dup_rows=21776 dup_rows_remote=19022 dedup_ratio=0.332275 router_tokens_per_expert=[240, 270, 266, 243, 269, 264, 282, 245, 213, 281, 229, 246, 268, 242, 250, 255, 240, 253, 250, 254, 250, 246, 250, 276, 253, 285, 246, 255, 242, 285, 271, 252, 264, 266, 272, 245, 276, 229, 258, 238, 244, 243, 245, 266, 276, 235, 274, 261, 227, 254, 251, 266, 276, 250, 252, 255, 260, 248, 264, 258, 276, 257, 280, 229, 245, 253, 263, 246, 240, 262, 247, 280, 268, 257, 266, 224, 267, 239, 275, 264, 260, 250, 243, 258, 252, 242, 255, 247, 260, 256, 261, 246, 256, 230, 255, 266, 242, 264, 274, 251, 245, 246, 258, 256, 270, 260, 258, 274, 280, 271, 245, 243, 264, 247, 263, 266, 295, 241, 265, 244, 256, 260, 272, 271, 258, 227, 252, 251] payload_rows_per_dst=[5384, 5536, 5408, 5522, 5476, 5420, 5520, 5494] dup_rows_per_dst=[2742, 2680, 2776, 2684, 2716, 2654, 2754, 2770] combine_rows_per_src=[5384, 5536, 5408, 5522, 5476, 5420, 5520, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=230168 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157344536 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314458904 weight_movement_bytes=18874368 total_nvlink_bytes=333333272 weight_to_token_ratio=0.060022 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38296 dup_rows=21792 dup_rows_remote=19146 dedup_ratio=0.332520 router_tokens_per_expert=[263, 255, 240, 274, 262, 285, 280, 240, 272, 255, 254, 241, 238, 253, 283, 270, 222, 248, 264, 257, 251, 237, 233, 247, 271, 272, 268, 272, 256, 259, 245, 245, 247, 248, 259, 241, 261, 244, 233, 230, 254, 258, 255, 242, 243, 256, 239, 255, 253, 253, 294, 278, 286, 262, 262, 259, 276, 256, 255, 246, 253, 228, 279, 278, 232, 272, 263, 279, 251, 256, 227, 247, 232, 268, 270, 301, 249, 276, 242, 265, 267, 269, 252, 269, 255, 272, 243, 265, 277, 244, 255, 247, 264, 257, 262, 272, 265, 233, 236, 246, 251, 251, 257, 248, 256, 255, 245, 251, 249, 271, 267, 263, 265, 255, 242, 247, 264, 243, 231, 247, 252, 261, 257, 255, 250, 254, 254, 252] payload_rows_per_dst=[5502, 5448, 5350, 5644, 5542, 5454, 5414, 5390] dup_rows_per_dst=[2828, 2646, 2580, 2792, 2718, 2886, 2674, 2668] combine_rows_per_src=[5502, 5448, 5350, 5644, 5542, 5454, 5414, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155443200 dispatch_meta_bytes=229392 combine_payload_bytes=155443200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155672592 ingress_bytes=174317568 link_bytes=174317568 token_bytes=311115792 weight_movement_bytes=18874368 total_nvlink_bytes=329990160 weight_to_token_ratio=0.060667 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670225167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=155656192 dispatch_meta_bytes=227752 combine_payload_bytes=155656192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155883944 ingress_bytes=174530560 link_bytes=174530560 token_bytes=311540136 weight_movement_bytes=18874368 total_nvlink_bytes=330414504 weight_to_token_ratio=0.060584 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672265952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229048 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860088 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491128 weight_movement_bytes=18874368 total_nvlink_bytes=332365496 weight_to_token_ratio=0.060207 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229768 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157090184 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950600 weight_movement_bytes=37748736 total_nvlink_bytes=351699336 weight_to_token_ratio=0.120238 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38404 dup_rows=21728 dup_rows_remote=18968 dedup_ratio=0.331543 router_tokens_per_expert=[264, 278, 282, 248, 241, 259, 211, 261, 276, 273, 266, 266, 254, 257, 271, 257, 269, 265, 272, 226, 251, 265, 252, 241, 244, 255, 269, 242, 232, 269, 255, 259, 255, 265, 240, 249, 264, 256, 244, 240, 244, 267, 250, 254, 277, 255, 263, 259, 258, 284, 254, 241, 265, 258, 250, 252, 232, 214, 246, 285, 258, 255, 256, 238, 271, 257, 233, 280, 254, 263, 263, 231, 256, 269, 251, 246, 261, 252, 237, 259, 248, 261, 263, 259, 256, 264, 280, 240, 249, 263, 262, 243, 246, 248, 277, 251, 218, 273, 251, 260, 241, 267, 265, 264, 229, 251, 245, 263, 277, 266, 275, 232, 240, 226, 255, 265, 257, 277, 259, 263, 247, 251, 244, 291, 246, 246, 319, 254] payload_rows_per_dst=[5516, 5444, 5404, 5482, 5472, 5442, 5496, 5552] dup_rows_per_dst=[2812, 2688, 2760, 2610, 2694, 2778, 2658, 2728] combine_rows_per_src=[5516, 5444, 5404, 5482, 5472, 5442, 5496, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38198 dup_rows=21778 dup_rows_remote=19004 dedup_ratio=0.332306 router_tokens_per_expert=[257, 240, 237, 248, 248, 237, 237, 248, 241, 264, 236, 270, 287, 244, 240, 272, 246, 264, 244, 262, 255, 244, 281, 253, 279, 276, 260, 252, 273, 268, 289, 247, 275, 274, 270, 267, 252, 222, 244, 266, 245, 252, 270, 249, 253, 237, 250, 281, 246, 253, 260, 243, 264, 277, 287, 270, 274, 277, 245, 272, 243, 265, 253, 238, 282, 257, 247, 274, 267, 256, 255, 253, 254, 272, 236, 236, 254, 248, 238, 235, 273, 230, 256, 248, 244, 256, 250, 259, 244, 244, 231, 251, 243, 266, 273, 249, 253, 254, 269, 275, 266, 267, 244, 226, 269, 246, 276, 264, 256, 268, 258, 245, 268, 271, 255, 239, 268, 252, 255, 264, 257, 234, 273, 242, 240, 252, 258, 250] payload_rows_per_dst=[5336, 5502, 5502, 5560, 5468, 5388, 5552, 5450] dup_rows_per_dst=[2676, 2884, 2712, 2774, 2660, 2646, 2720, 2706] combine_rows_per_src=[5336, 5502, 5502, 5560, 5468, 5388, 5552, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38304 dup_rows=21704 dup_rows_remote=18930 dedup_ratio=0.331177 router_tokens_per_expert=[273, 259, 253, 257, 246, 249, 254, 226, 273, 263, 276, 248, 281, 264, 246, 272, 243, 245, 274, 260, 236, 273, 251, 255, 256, 260, 243, 240, 255, 240, 257, 238, 264, 285, 242, 289, 246, 283, 220, 286, 253, 253, 282, 227, 264, 269, 253, 257, 259, 235, 244, 276, 257, 258, 244, 272, 242, 241, 253, 247, 249, 268, 259, 248, 278, 274, 254, 246, 238, 250, 252, 223, 248, 284, 247, 247, 254, 253, 235, 259, 254, 268, 239, 253, 257, 239, 244, 265, 256, 237, 256, 251, 263, 273, 247, 271, 260, 259, 240, 280, 248, 240, 266, 246, 251, 278, 247, 251, 263, 262, 254, 268, 271, 236, 315, 261, 259, 259, 232, 265, 247, 252, 257, 266, 271, 244, 249, 265] payload_rows_per_dst=[5222, 5404, 5656, 5410, 5470, 5532, 5610, 5528] dup_rows_per_dst=[2348, 2616, 3110, 2694, 2642, 2712, 2808, 2774] combine_rows_per_src=[5222, 5404, 5656, 5410, 5470, 5532, 5610, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38278 dup_rows=21890 dup_rows_remote=19206 dedup_ratio=0.334015 router_tokens_per_expert=[250, 259, 277, 256, 237, 250, 276, 263, 243, 254, 271, 234, 255, 231, 280, 244, 281, 263, 252, 241, 276, 252, 255, 235, 267, 283, 273, 225, 240, 254, 286, 258, 253, 254, 267, 277, 228, 266, 283, 260, 261, 235, 277, 255, 295, 270, 257, 261, 262, 274, 246, 274, 242, 240, 242, 240, 267, 268, 253, 266, 278, 234, 264, 255, 253, 236, 264, 280, 276, 260, 250, 243, 269, 297, 264, 268, 239, 225, 272, 238, 240, 261, 248, 285, 242, 252, 254, 257, 255, 246, 243, 240, 269, 234, 254, 246, 265, 243, 236, 242, 253, 253, 232, 250, 273, 253, 257, 266, 244, 256, 279, 239, 265, 250, 255, 225, 289, 266, 257, 254, 244, 277, 239, 218, 247, 254, 251, 251] payload_rows_per_dst=[5428, 5460, 5620, 5498, 5494, 5368, 5400, 5378] dup_rows_per_dst=[2732, 2822, 2778, 2712, 2774, 2684, 2682, 2706] combine_rows_per_src=[5428, 5460, 5620, 5498, 5494, 5368, 5400, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229488 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532272 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835056 weight_movement_bytes=18874368 total_nvlink_bytes=333709424 weight_to_token_ratio=0.059950 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38398 dup_rows=21658 dup_rows_remote=18858 dedup_ratio=0.330475 router_tokens_per_expert=[254, 254, 256, 263, 252, 239, 252, 242, 257, 243, 243, 277, 261, 277, 234, 275, 252, 249, 242, 245, 269, 265, 245, 258, 252, 254, 238, 255, 264, 270, 249, 277, 239, 262, 248, 246, 242, 243, 253, 237, 236, 268, 226, 274, 237, 240, 291, 236, 273, 258, 253, 259, 282, 267, 251, 273, 270, 245, 244, 281, 262, 253, 252, 256, 255, 271, 254, 239, 241, 260, 273, 264, 265, 262, 276, 246, 278, 253, 226, 277, 255, 269, 271, 245, 270, 263, 264, 233, 251, 260, 240, 264, 244, 273, 251, 256, 233, 229, 249, 261, 282, 253, 261, 247, 257, 253, 272, 247, 238, 246, 268, 277, 261, 272, 252, 231, 268, 263, 234, 234, 258, 245, 271, 267, 254, 294, 269, 253] payload_rows_per_dst=[5422, 5496, 5432, 5506, 5480, 5514, 5542, 5486] dup_rows_per_dst=[2736, 2672, 2524, 2852, 2800, 2704, 2604, 2766] combine_rows_per_src=[5422, 5496, 5432, 5506, 5480, 5514, 5542, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228808 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156687816 ingress_bytes=194207744 link_bytes=194207744 token_bytes=313146824 weight_movement_bytes=37748736 total_nvlink_bytes=350895560 weight_to_token_ratio=0.120546 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860803047e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38220 dup_rows=21814 dup_rows_remote=18868 dedup_ratio=0.332855 router_tokens_per_expert=[276, 251, 266, 236, 235, 237, 252, 239, 257, 251, 249, 248, 290, 276, 255, 275, 279, 257, 268, 244, 250, 272, 263, 266, 248, 231, 265, 246, 250, 237, 264, 249, 242, 273, 259, 243, 278, 259, 267, 252, 286, 229, 249, 282, 250, 256, 262, 252, 222, 237, 232, 236, 262, 261, 275, 304, 273, 281, 270, 273, 271, 255, 249, 230, 262, 235, 231, 244, 235, 259, 244, 257, 254, 242, 256, 258, 246, 256, 236, 267, 296, 253, 293, 282, 253, 270, 233, 253, 266, 263, 253, 259, 234, 248, 242, 255, 249, 267, 246, 241, 261, 270, 269, 276, 268, 255, 245, 240, 243, 284, 259, 229, 255, 239, 251, 236, 260, 260, 272, 233, 251, 269, 266, 226, 266, 270, 259, 266] payload_rows_per_dst=[5502, 5492, 5392, 5474, 5420, 5508, 5472, 5462] dup_rows_per_dst=[2946, 2690, 2628, 2784, 2544, 2794, 2732, 2696] combine_rows_per_src=[5502, 5492, 5392, 5474, 5420, 5508, 5472, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228936 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157122120 ingress_bytes=213516288 link_bytes=213516288 token_bytes=314015304 weight_movement_bytes=56623104 total_nvlink_bytes=370638408 weight_to_token_ratio=0.180320 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045808015e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38270 dup_rows=21874 dup_rows_remote=19208 dedup_ratio=0.333771 router_tokens_per_expert=[236, 250, 269, 243, 276, 279, 243, 229, 236, 252, 254, 260, 243, 283, 251, 262, 276, 274, 287, 267, 260, 252, 245, 248, 232, 275, 277, 271, 262, 239, 269, 246, 246, 256, 229, 285, 254, 247, 249, 253, 256, 249, 243, 245, 256, 252, 268, 258, 268, 249, 259, 243, 252, 263, 290, 267, 269, 228, 247, 268, 266, 266, 237, 257, 256, 261, 270, 285, 235, 239, 280, 262, 238, 261, 260, 248, 241, 286, 262, 254, 244, 268, 254, 251, 241, 257, 281, 252, 251, 243, 249, 274, 226, 233, 264, 281, 220, 232, 257, 277, 253, 254, 230, 258, 251, 242, 254, 255, 283, 265, 229, 269, 263, 269, 254, 248, 254, 251, 248, 233, 262, 263, 275, 259, 258, 270, 249, 255] payload_rows_per_dst=[5430, 5534, 5372, 5496, 5524, 5478, 5392, 5436] dup_rows_per_dst=[2702, 2826, 2720, 2762, 2752, 2660, 2666, 2786] combine_rows_per_src=[5430, 5534, 5372, 5496, 5524, 5478, 5392, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229936 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016624 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313803312 weight_movement_bytes=18874368 total_nvlink_bytes=332677680 weight_to_token_ratio=0.060147 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38378 dup_rows=21704 dup_rows_remote=18986 dedup_ratio=0.331177 router_tokens_per_expert=[238, 244, 253, 257, 283, 245, 219, 253, 260, 263, 260, 263, 238, 251, 266, 272, 249, 252, 266, 250, 289, 246, 263, 255, 252, 255, 241, 226, 264, 269, 243, 266, 257, 260, 264, 258, 259, 241, 246, 244, 272, 249, 241, 264, 253, 270, 257, 287, 262, 232, 252, 233, 256, 257, 228, 260, 261, 285, 260, 276, 243, 239, 241, 263, 268, 275, 230, 241, 256, 280, 247, 239, 257, 276, 261, 282, 214, 225, 248, 270, 273, 281, 252, 260, 259, 241, 267, 264, 261, 244, 245, 235, 280, 258, 255, 257, 251, 271, 270, 256, 246, 241, 255, 273, 236, 243, 253, 258, 244, 255, 286, 272, 287, 243, 277, 256, 239, 265, 254, 251, 246, 253, 263, 280, 249, 241, 258, 274] payload_rows_per_dst=[5436, 5454, 5478, 5460, 5460, 5474, 5522, 5548] dup_rows_per_dst=[2694, 2718, 2766, 2636, 2678, 2790, 2698, 2724] combine_rows_per_src=[5436, 5454, 5478, 5460, 5460, 5474, 5522, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229024 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507232 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314785440 weight_movement_bytes=18874368 total_nvlink_bytes=333659808 weight_to_token_ratio=0.059959 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228352 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156777472 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313326592 weight_movement_bytes=37748736 total_nvlink_bytes=351075328 weight_to_token_ratio=0.120477 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229912 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983832 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737752 weight_movement_bytes=18874368 total_nvlink_bytes=332612120 weight_to_token_ratio=0.060160 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229456 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157425744 ingress_bytes=194945024 link_bytes=194945024 token_bytes=314622032 weight_movement_bytes=37748736 total_nvlink_bytes=352370768 weight_to_token_ratio=0.119981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867867301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38320 dup_rows=21698 dup_rows_remote=18864 dedup_ratio=0.331085 router_tokens_per_expert=[241, 263, 261, 266, 256, 262, 259, 232, 241, 256, 242, 273, 289, 254, 270, 259, 269, 280, 236, 251, 250, 235, 242, 233, 228, 258, 263, 263, 242, 233, 255, 309, 252, 276, 234, 270, 246, 269, 277, 246, 255, 255, 262, 297, 264, 280, 247, 246, 254, 256, 282, 245, 225, 271, 232, 233, 234, 276, 264, 238, 233, 246, 260, 236, 259, 254, 233, 282, 267, 275, 243, 262, 270, 249, 258, 266, 268, 273, 239, 247, 262, 246, 240, 257, 243, 256, 216, 247, 263, 263, 259, 249, 252, 266, 241, 275, 254, 270, 249, 252, 286, 247, 280, 256, 253, 272, 276, 263, 254, 276, 247, 243, 273, 265, 255, 266, 256, 231, 260, 215, 259, 250, 251, 242, 282, 233, 283, 257] payload_rows_per_dst=[5622, 5424, 5518, 5370, 5488, 5482, 5516, 5418] dup_rows_per_dst=[2626, 2670, 2834, 2600, 2802, 2588, 2840, 2738] combine_rows_per_src=[5622, 5424, 5518, 5370, 5488, 5482, 5516, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38338 dup_rows=21668 dup_rows_remote=19010 dedup_ratio=0.330627 router_tokens_per_expert=[243, 240, 283, 266, 248, 239, 254, 260, 246, 264, 246, 262, 269, 234, 278, 259, 260, 261, 242, 258, 232, 241, 270, 247, 237, 259, 280, 253, 271, 274, 267, 271, 268, 257, 245, 236, 237, 282, 253, 272, 272, 265, 261, 274, 253, 250, 254, 232, 248, 244, 259, 268, 247, 245, 249, 264, 260, 243, 243, 288, 253, 271, 277, 235, 240, 292, 256, 267, 273, 267, 263, 251, 275, 256, 270, 264, 232, 289, 246, 216, 246, 245, 264, 261, 256, 283, 240, 229, 239, 240, 247, 240, 289, 247, 280, 272, 266, 269, 235, 243, 248, 273, 270, 248, 251, 228, 258, 249, 245, 267, 257, 269, 242, 246, 240, 240, 267, 253, 260, 266, 229, 264, 251, 255, 273, 242, 247, 263] payload_rows_per_dst=[5418, 5472, 5534, 5530, 5512, 5474, 5508, 5420] dup_rows_per_dst=[2764, 2774, 2688, 2658, 2802, 2682, 2644, 2656] combine_rows_per_src=[5418, 5472, 5534, 5530, 5512, 5474, 5508, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38130 dup_rows=21770 dup_rows_remote=18846 dedup_ratio=0.332184 router_tokens_per_expert=[248, 221, 239, 249, 272, 225, 246, 269, 241, 277, 245, 240, 286, 238, 271, 267, 252, 250, 254, 256, 254, 242, 252, 250, 269, 251, 238, 289, 233, 241, 243, 266, 268, 275, 243, 252, 251, 272, 222, 224, 240, 273, 239, 249, 243, 259, 239, 250, 273, 256, 231, 274, 271, 261, 269, 248, 247, 268, 252, 242, 259, 275, 254, 259, 289, 245, 236, 266, 253, 245, 248, 289, 258, 237, 260, 253, 240, 262, 243, 256, 250, 252, 260, 276, 257, 265, 274, 230, 237, 235, 247, 277, 255, 260, 241, 242, 251, 242, 263, 234, 246, 278, 256, 267, 269, 256, 273, 267, 288, 284, 259, 258, 272, 280, 273, 272, 250, 243, 283, 278, 257, 268, 267, 268, 258, 251, 274, 233] payload_rows_per_dst=[5450, 5302, 5414, 5502, 5388, 5428, 5646, 5636] dup_rows_per_dst=[2618, 2466, 2574, 2776, 2676, 2688, 3048, 2924] combine_rows_per_src=[5450, 5302, 5414, 5502, 5388, 5428, 5646, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38338 dup_rows=21658 dup_rows_remote=18828 dedup_ratio=0.330475 router_tokens_per_expert=[283, 233, 257, 269, 263, 224, 237, 255, 244, 261, 267, 281, 251, 265, 252, 237, 247, 255, 240, 270, 285, 253, 252, 253, 277, 262, 262, 253, 263, 245, 270, 224, 247, 223, 249, 244, 268, 284, 253, 258, 267, 253, 243, 254, 253, 257, 262, 280, 255, 268, 271, 248, 251, 256, 276, 245, 250, 261, 258, 264, 256, 265, 265, 249, 253, 236, 251, 284, 273, 258, 280, 230, 270, 247, 255, 270, 258, 285, 242, 237, 236, 257, 271, 267, 263, 262, 259, 278, 267, 249, 261, 245, 278, 281, 261, 250, 241, 239, 231, 244, 240, 268, 259, 245, 288, 233, 234, 245, 245, 248, 232, 261, 250, 249, 235, 260, 247, 258, 258, 282, 256, 258, 246, 254, 263, 253, 242, 267] payload_rows_per_dst=[5458, 5458, 5520, 5554, 5484, 5540, 5370, 5494] dup_rows_per_dst=[2700, 2764, 2670, 2722, 2774, 2830, 2536, 2662] combine_rows_per_src=[5458, 5458, 5520, 5554, 5484, 5540, 5370, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228736 combine_payload_bytes=156958720 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157187456 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314146176 weight_movement_bytes=37748736 total_nvlink_bytes=351894912 weight_to_token_ratio=0.120163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229392 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261840 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294288 weight_movement_bytes=18874368 total_nvlink_bytes=333168656 weight_to_token_ratio=0.060053 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38178 dup_rows=21882 dup_rows_remote=19220 dedup_ratio=0.333893 router_tokens_per_expert=[235, 247, 251, 276, 250, 237, 267, 264, 250, 272, 260, 251, 232, 256, 275, 239, 255, 263, 286, 272, 239, 237, 243, 259, 255, 266, 261, 253, 260, 244, 262, 280, 272, 249, 255, 278, 267, 264, 247, 250, 257, 259, 261, 283, 267, 248, 250, 262, 292, 284, 244, 251, 239, 273, 274, 264, 266, 241, 240, 266, 279, 254, 272, 232, 246, 219, 263, 269, 264, 271, 272, 267, 230, 255, 250, 233, 253, 260, 251, 266, 241, 284, 244, 253, 265, 261, 250, 268, 234, 236, 255, 244, 266, 245, 229, 258, 253, 275, 228, 232, 271, 281, 265, 257, 237, 273, 225, 248, 224, 261, 270, 238, 279, 264, 256, 226, 271, 261, 246, 272, 252, 233, 238, 270, 252, 266, 246, 259] payload_rows_per_dst=[5460, 5430, 5570, 5544, 5476, 5308, 5406, 5460] dup_rows_per_dst=[2664, 2840, 2768, 2798, 2662, 2758, 2670, 2722] combine_rows_per_src=[5460, 5430, 5570, 5544, 5476, 5308, 5406, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38324 dup_rows=21696 dup_rows_remote=18934 dedup_ratio=0.331055 router_tokens_per_expert=[254, 243, 256, 267, 260, 259, 254, 280, 253, 249, 267, 252, 282, 236, 264, 240, 288, 232, 236, 261, 244, 297, 243, 252, 245, 280, 236, 245, 238, 276, 217, 257, 258, 242, 249, 266, 295, 272, 245, 234, 245, 239, 261, 292, 232, 264, 265, 242, 259, 248, 250, 243, 281, 284, 266, 264, 237, 282, 261, 259, 269, 275, 227, 279, 266, 262, 262, 250, 267, 242, 285, 232, 251, 248, 253, 276, 273, 264, 241, 247, 239, 255, 240, 257, 258, 231, 251, 257, 254, 256, 268, 255, 252, 259, 265, 253, 260, 236, 242, 244, 247, 234, 276, 236, 236, 250, 265, 259, 271, 268, 259, 254, 265, 259, 255, 223, 245, 318, 276, 262, 243, 275, 233, 238, 233, 259, 269, 261] payload_rows_per_dst=[5516, 5452, 5468, 5508, 5484, 5556, 5334, 5522] dup_rows_per_dst=[2762, 2664, 2734, 2860, 2754, 2726, 2490, 2706] combine_rows_per_src=[5516, 5452, 5468, 5508, 5484, 5556, 5334, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=227904 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156408384 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312588864 weight_movement_bytes=37748736 total_nvlink_bytes=350337600 weight_to_token_ratio=0.120762 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38252 dup_rows=21878 dup_rows_remote=19140 dedup_ratio=0.333832 router_tokens_per_expert=[266, 236, 261, 248, 264, 258, 237, 241, 246, 266, 276, 257, 264, 262, 267, 247, 247, 248, 251, 248, 247, 237, 254, 252, 236, 250, 286, 255, 275, 258, 245, 253, 233, 236, 273, 255, 253, 244, 241, 233, 301, 272, 265, 271, 241, 241, 256, 242, 266, 259, 253, 256, 261, 237, 254, 263, 260, 248, 246, 257, 247, 271, 237, 272, 294, 250, 249, 275, 262, 253, 287, 281, 252, 246, 238, 257, 268, 273, 268, 253, 228, 237, 252, 247, 269, 247, 251, 239, 260, 263, 251, 276, 264, 276, 290, 260, 274, 274, 256, 237, 251, 258, 267, 233, 281, 221, 268, 232, 239, 244, 270, 267, 251, 259, 236, 258, 257, 253, 249, 253, 267, 252, 254, 256, 263, 266, 266, 258] payload_rows_per_dst=[5436, 5386, 5424, 5484, 5536, 5436, 5406, 5550] dup_rows_per_dst=[2756, 2698, 2690, 2690, 2792, 2784, 2738, 2730] combine_rows_per_src=[5436, 5386, 5424, 5484, 5536, 5436, 5406, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=228664 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261112 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314293560 weight_movement_bytes=18874368 total_nvlink_bytes=333167928 weight_to_token_ratio=0.060053 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38344 dup_rows=21766 dup_rows_remote=19224 dedup_ratio=0.332123 router_tokens_per_expert=[249, 269, 263, 263, 237, 253, 229, 245, 275, 266, 274, 245, 254, 253, 251, 296, 256, 240, 236, 256, 261, 245, 235, 232, 271, 232, 272, 250, 256, 220, 259, 263, 266, 248, 223, 296, 263, 250, 241, 265, 291, 263, 275, 278, 272, 248, 243, 256, 230, 252, 273, 259, 254, 248, 240, 238, 271, 256, 235, 271, 265, 233, 283, 250, 266, 264, 250, 254, 260, 254, 248, 271, 226, 270, 257, 280, 238, 262, 240, 251, 249, 239, 259, 253, 264, 226, 281, 264, 233, 269, 254, 260, 243, 271, 225, 241, 245, 246, 237, 277, 271, 250, 265, 254, 278, 271, 245, 252, 234, 280, 262, 264, 265, 283, 265, 263, 246, 259, 255, 224, 253, 249, 256, 274, 241, 261, 284, 295] payload_rows_per_dst=[5434, 5426, 5554, 5456, 5406, 5452, 5500, 5542] dup_rows_per_dst=[2810, 2542, 2802, 2660, 2776, 2610, 2762, 2804] combine_rows_per_src=[5434, 5426, 5554, 5456, 5406, 5452, 5500, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229032 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204136 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179240 weight_movement_bytes=18874368 total_nvlink_bytes=333053608 weight_to_token_ratio=0.060075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229592 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156606680 ingress_bytes=213000192 link_bytes=213000192 token_bytes=312983768 weight_movement_bytes=56623104 total_nvlink_bytes=369606872 weight_to_token_ratio=0.180914 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040863037e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229568 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909760 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589952 weight_movement_bytes=37748736 total_nvlink_bytes=351338688 weight_to_token_ratio=0.120376 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=230272 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157287296 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314344320 weight_movement_bytes=18874368 total_nvlink_bytes=333218688 weight_to_token_ratio=0.060044 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38274 dup_rows=21872 dup_rows_remote=19218 dedup_ratio=0.333740 router_tokens_per_expert=[227, 248, 270, 281, 259, 238, 252, 272, 256, 265, 266, 258, 230, 231, 266, 230, 239, 267, 233, 260, 254, 246, 246, 242, 249, 265, 250, 231, 245, 242, 253, 260, 283, 256, 236, 239, 240, 246, 264, 243, 233, 249, 251, 265, 256, 261, 245, 255, 258, 251, 266, 256, 245, 242, 284, 240, 278, 239, 260, 258, 261, 285, 264, 269, 277, 284, 243, 230, 279, 281, 250, 239, 261, 261, 271, 244, 265, 236, 246, 286, 245, 263, 248, 258, 269, 244, 257, 280, 259, 278, 247, 273, 245, 269, 256, 249, 271, 242, 247, 254, 259, 258, 281, 262, 287, 269, 263, 248, 272, 250, 252, 269, 257, 276, 246, 259, 238, 238, 273, 267, 238, 256, 245, 252, 273, 247, 261, 256] payload_rows_per_dst=[5430, 5298, 5390, 5504, 5530, 5474, 5586, 5452] dup_rows_per_dst=[2668, 2666, 2654, 2808, 2776, 2806, 2782, 2712] combine_rows_per_src=[5430, 5298, 5390, 5504, 5530, 5474, 5586, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38198 dup_rows=21920 dup_rows_remote=19096 dedup_ratio=0.334473 router_tokens_per_expert=[248, 247, 276, 268, 251, 239, 264, 258, 286, 232, 234, 259, 226, 249, 257, 257, 246, 266, 257, 257, 240, 253, 231, 267, 237, 269, 258, 272, 257, 268, 226, 273, 236, 242, 264, 238, 273, 266, 274, 251, 268, 238, 234, 251, 272, 314, 257, 260, 244, 271, 237, 244, 272, 248, 252, 286, 264, 266, 260, 231, 249, 271, 264, 262, 218, 253, 274, 244, 265, 275, 259, 251, 253, 270, 249, 276, 244, 286, 240, 247, 261, 270, 257, 257, 258, 244, 257, 247, 250, 261, 258, 255, 249, 251, 242, 264, 261, 243, 244, 252, 265, 266, 248, 230, 270, 261, 270, 256, 274, 243, 266, 263, 254, 246, 251, 297, 283, 258, 249, 244, 255, 254, 256, 245, 240, 259, 251, 242] payload_rows_per_dst=[5428, 5440, 5552, 5418, 5452, 5410, 5480, 5436] dup_rows_per_dst=[2674, 2714, 2724, 2824, 2756, 2752, 2744, 2732] combine_rows_per_src=[5428, 5440, 5552, 5418, 5452, 5410, 5480, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38266 dup_rows=21666 dup_rows_remote=18750 dedup_ratio=0.330597 router_tokens_per_expert=[265, 256, 256, 242, 283, 243, 271, 250, 243, 241, 257, 256, 259, 249, 250, 242, 270, 254, 263, 273, 250, 258, 258, 244, 248, 294, 267, 242, 248, 239, 269, 272, 254, 282, 256, 254, 244, 246, 281, 232, 287, 255, 245, 283, 245, 262, 247, 251, 241, 273, 284, 246, 227, 291, 229, 239, 275, 263, 255, 273, 240, 253, 258, 238, 268, 287, 244, 229, 266, 278, 275, 262, 264, 242, 266, 236, 258, 256, 240, 263, 262, 240, 251, 280, 244, 267, 270, 249, 244, 280, 258, 242, 285, 257, 238, 257, 268, 246, 260, 241, 227, 253, 256, 248, 237, 273, 259, 251, 257, 273, 265, 283, 246, 247, 241, 244, 257, 263, 233, 248, 261, 257, 264, 219, 262, 250, 236, 264] payload_rows_per_dst=[5474, 5560, 5508, 5512, 5218, 5538, 5456, 5604] dup_rows_per_dst=[2652, 2738, 2740, 2658, 2514, 2666, 2782, 2916] combine_rows_per_src=[5474, 5560, 5508, 5512, 5218, 5538, 5456, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38162 dup_rows=21786 dup_rows_remote=19020 dedup_ratio=0.332428 router_tokens_per_expert=[255, 281, 216, 266, 259, 241, 243, 243, 262, 266, 248, 203, 264, 237, 263, 253, 257, 254, 254, 259, 244, 223, 255, 256, 253, 276, 265, 269, 270, 267, 270, 226, 260, 281, 264, 261, 283, 247, 262, 259, 265, 245, 244, 270, 271, 260, 253, 260, 294, 258, 266, 227, 257, 253, 250, 247, 270, 257, 256, 257, 275, 259, 225, 256, 262, 259, 256, 258, 269, 244, 249, 251, 221, 269, 250, 257, 251, 245, 260, 250, 249, 241, 268, 260, 248, 270, 249, 262, 248, 272, 271, 239, 291, 268, 285, 256, 224, 256, 252, 244, 263, 274, 273, 254, 247, 286, 294, 275, 250, 248, 263, 268, 265, 273, 230, 255, 261, 263, 251, 229, 263, 224, 267, 227, 245, 227, 259, 240] payload_rows_per_dst=[5370, 5496, 5430, 5506, 5450, 5588, 5524, 5386] dup_rows_per_dst=[2630, 2700, 2940, 2708, 2652, 2766, 2818, 2572] combine_rows_per_src=[5370, 5496, 5430, 5506, 5450, 5588, 5524, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229968 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157000272 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770576 weight_movement_bytes=37748736 total_nvlink_bytes=351519312 weight_to_token_ratio=0.120307 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229176 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156688184 ingress_bytes=213082112 link_bytes=213082112 token_bytes=313147192 weight_movement_bytes=56623104 total_nvlink_bytes=369770296 weight_to_token_ratio=0.180819 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041647954e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228064 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156965600 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313703136 weight_movement_bytes=18874368 total_nvlink_bytes=332577504 weight_to_token_ratio=0.060166 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38278 dup_rows=21786 dup_rows_remote=19000 dedup_ratio=0.332428 router_tokens_per_expert=[248, 254, 257, 226, 275, 256, 232, 278, 257, 249, 242, 247, 237, 249, 276, 267, 269, 268, 255, 216, 247, 258, 260, 271, 258, 245, 278, 253, 249, 257, 277, 257, 263, 247, 263, 261, 231, 274, 288, 230, 238, 250, 260, 244, 262, 243, 278, 263, 259, 249, 225, 249, 235, 271, 238, 299, 256, 245, 262, 251, 254, 251, 223, 263, 250, 245, 247, 242, 267, 291, 256, 252, 264, 243, 261, 246, 260, 282, 253, 270, 275, 290, 256, 261, 271, 254, 256, 264, 239, 259, 279, 260, 259, 236, 280, 226, 256, 250, 274, 260, 259, 250, 262, 281, 259, 251, 275, 218, 279, 240, 247, 253, 278, 268, 247, 249, 257, 236, 258, 261, 261, 229, 265, 257, 257, 229, 271, 244] payload_rows_per_dst=[5434, 5522, 5458, 5368, 5472, 5500, 5490, 5506] dup_rows_per_dst=[2666, 2714, 2732, 2692, 2786, 2830, 2738, 2628] combine_rows_per_src=[5434, 5522, 5458, 5368, 5472, 5500, 5490, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38250 dup_rows=21762 dup_rows_remote=18908 dedup_ratio=0.332062 router_tokens_per_expert=[286, 265, 226, 285, 295, 236, 258, 252, 249, 248, 269, 280, 253, 274, 258, 242, 253, 253, 251, 264, 265, 263, 267, 238, 254, 263, 300, 263, 243, 258, 235, 267, 245, 239, 289, 253, 255, 258, 245, 255, 265, 229, 247, 239, 253, 243, 247, 268, 231, 239, 255, 262, 239, 240, 271, 259, 257, 265, 262, 245, 226, 228, 278, 222, 256, 248, 271, 242, 247, 278, 268, 265, 244, 238, 244, 247, 256, 255, 259, 243, 279, 239, 255, 252, 244, 239, 270, 279, 271, 244, 228, 276, 252, 248, 264, 242, 276, 252, 286, 243, 256, 246, 249, 248, 256, 245, 259, 256, 249, 250, 261, 263, 279, 272, 269, 254, 263, 267, 279, 243, 269, 235, 261, 262, 298, 248, 262, 247] payload_rows_per_dst=[5524, 5466, 5380, 5500, 5498, 5452, 5502, 5452] dup_rows_per_dst=[2854, 2808, 2680, 2602, 2624, 2712, 2688, 2794] combine_rows_per_src=[5524, 5466, 5380, 5500, 5498, 5452, 5502, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38308 dup_rows=21802 dup_rows_remote=19086 dedup_ratio=0.332672 router_tokens_per_expert=[225, 257, 263, 272, 248, 260, 256, 270, 267, 228, 244, 273, 264, 254, 268, 254, 252, 247, 264, 238, 255, 246, 261, 255, 249, 252, 262, 290, 252, 252, 246, 259, 280, 251, 274, 245, 256, 253, 251, 282, 254, 250, 237, 241, 258, 231, 266, 281, 282, 266, 242, 258, 253, 242, 242, 247, 259, 255, 256, 267, 244, 240, 263, 262, 242, 281, 263, 227, 245, 266, 292, 241, 277, 240, 253, 245, 276, 269, 256, 273, 246, 254, 281, 263, 249, 260, 242, 245, 281, 267, 261, 251, 249, 247, 270, 278, 267, 261, 251, 250, 272, 271, 268, 264, 259, 270, 245, 264, 256, 252, 261, 269, 256, 262, 243, 233, 202, 249, 257, 258, 243, 222, 237, 258, 227, 265, 251, 264] payload_rows_per_dst=[5480, 5462, 5442, 5502, 5562, 5466, 5426, 5394] dup_rows_per_dst=[2726, 2698, 2778, 2654, 2730, 2822, 2716, 2678] combine_rows_per_src=[5480, 5462, 5442, 5502, 5562, 5466, 5426, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228728 combine_payload_bytes=156311552 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156540280 ingress_bytes=231809024 link_bytes=231809024 token_bytes=312851832 weight_movement_bytes=75497472 total_nvlink_bytes=388349304 weight_to_token_ratio=0.241320 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221080011e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38368 dup_rows=21680 dup_rows_remote=18958 dedup_ratio=0.330811 router_tokens_per_expert=[229, 255, 252, 244, 284, 248, 254, 265, 264, 267, 258, 279, 296, 274, 240, 247, 241, 268, 249, 259, 255, 231, 262, 252, 301, 247, 246, 268, 270, 266, 263, 227, 269, 240, 277, 268, 260, 250, 266, 268, 253, 242, 267, 272, 241, 218, 216, 255, 286, 263, 260, 253, 235, 237, 241, 249, 263, 261, 279, 243, 251, 249, 255, 286, 239, 235, 242, 262, 273, 264, 255, 274, 257, 260, 237, 265, 269, 249, 232, 265, 262, 269, 254, 262, 244, 253, 267, 277, 231, 236, 251, 267, 271, 271, 264, 248, 248, 238, 249, 279, 266, 252, 242, 236, 230, 269, 279, 223, 256, 244, 256, 258, 262, 234, 242, 236, 248, 288, 255, 263, 272, 276, 264, 251, 233, 244, 266, 270] payload_rows_per_dst=[5566, 5488, 5394, 5468, 5578, 5452, 5420, 5490] dup_rows_per_dst=[2746, 2722, 2730, 2754, 2578, 2802, 2630, 2718] combine_rows_per_src=[5566, 5488, 5394, 5468, 5578, 5452, 5420, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229112 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157015800 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802488 weight_movement_bytes=37748736 total_nvlink_bytes=351551224 weight_to_token_ratio=0.120295 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228632 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156900632 ingress_bytes=232169472 link_bytes=232169472 token_bytes=313572632 weight_movement_bytes=75497472 total_nvlink_bytes=389070104 weight_to_token_ratio=0.240766 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224533646e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229576 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157139144 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048712 weight_movement_bytes=37748736 total_nvlink_bytes=351797448 weight_to_token_ratio=0.120200 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38266 dup_rows=21800 dup_rows_remote=19060 dedup_ratio=0.332642 router_tokens_per_expert=[252, 280, 246, 240, 248, 273, 238, 237, 258, 273, 276, 216, 226, 245, 282, 240, 241, 257, 257, 249, 253, 249, 304, 263, 254, 270, 247, 268, 241, 257, 239, 252, 244, 250, 258, 259, 268, 241, 273, 266, 233, 246, 227, 262, 270, 261, 275, 272, 251, 276, 263, 236, 261, 269, 269, 279, 224, 267, 244, 268, 251, 249, 251, 269, 272, 248, 258, 252, 247, 250, 257, 276, 269, 243, 281, 274, 256, 263, 231, 244, 235, 263, 241, 268, 239, 266, 267, 256, 250, 283, 235, 276, 253, 282, 241, 248, 267, 269, 254, 260, 250, 249, 262, 270, 262, 262, 236, 239, 238, 287, 253, 234, 236, 271, 248, 266, 262, 260, 237, 250, 259, 239, 259, 269, 267, 253, 255, 258] payload_rows_per_dst=[5352, 5494, 5470, 5520, 5474, 5480, 5504, 5442] dup_rows_per_dst=[2708, 2708, 2740, 2734, 2768, 2726, 2680, 2736] combine_rows_per_src=[5352, 5494, 5470, 5520, 5474, 5480, 5504, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229304 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384632 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314539960 weight_movement_bytes=18874368 total_nvlink_bytes=333414328 weight_to_token_ratio=0.060006 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38220 dup_rows=21844 dup_rows_remote=19092 dedup_ratio=0.333313 router_tokens_per_expert=[248, 257, 242, 258, 259, 237, 251, 244, 261, 266, 257, 244, 263, 258, 270, 255, 233, 222, 277, 248, 245, 242, 261, 240, 269, 244, 255, 250, 275, 252, 269, 240, 248, 277, 247, 246, 257, 274, 234, 266, 272, 246, 262, 229, 265, 268, 264, 267, 279, 251, 272, 239, 247, 262, 270, 235, 251, 274, 259, 257, 237, 246, 273, 260, 272, 243, 275, 254, 279, 263, 250, 251, 224, 252, 255, 273, 244, 265, 240, 257, 260, 274, 240, 266, 261, 271, 246, 242, 237, 257, 262, 285, 234, 259, 262, 273, 244, 240, 273, 267, 242, 250, 251, 248, 245, 249, 236, 272, 274, 242, 275, 267, 261, 258, 243, 252, 249, 277, 240, 237, 293, 260, 269, 261, 269, 267, 242, 263] payload_rows_per_dst=[5402, 5342, 5474, 5472, 5588, 5496, 5430, 5488] dup_rows_per_dst=[2738, 2702, 2770, 2752, 2606, 2762, 2720, 2794] combine_rows_per_src=[5402, 5342, 5474, 5472, 5588, 5496, 5430, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=37950 dup_rows=21916 dup_rows_remote=18834 dedup_ratio=0.334412 router_tokens_per_expert=[231, 245, 246, 277, 249, 271, 287, 267, 254, 246, 281, 250, 235, 266, 255, 246, 260, 229, 301, 228, 238, 265, 248, 217, 247, 260, 258, 258, 249, 265, 268, 265, 242, 268, 256, 231, 247, 262, 275, 237, 284, 262, 286, 255, 238, 263, 245, 258, 273, 265, 276, 259, 248, 235, 221, 291, 251, 280, 227, 257, 233, 247, 259, 266, 256, 263, 254, 255, 266, 257, 237, 267, 258, 262, 260, 263, 277, 242, 261, 232, 258, 284, 217, 261, 256, 235, 276, 254, 229, 254, 244, 261, 235, 248, 226, 257, 252, 282, 233, 245, 276, 260, 263, 256, 254, 290, 266, 275, 277, 261, 253, 261, 269, 251, 243, 274, 235, 268, 259, 281, 266, 248, 251, 251, 251, 220, 280, 253] payload_rows_per_dst=[5462, 5408, 5458, 5480, 5474, 5336, 5332, 5670] dup_rows_per_dst=[2704, 2704, 2760, 2642, 2730, 2670, 2624, 3082] combine_rows_per_src=[5462, 5408, 5458, 5480, 5474, 5336, 5332, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38160 dup_rows=21878 dup_rows_remote=19136 dedup_ratio=0.333832 router_tokens_per_expert=[226, 270, 277, 247, 257, 257, 266, 245, 267, 245, 275, 249, 228, 264, 231, 267, 246, 260, 243, 244, 229, 259, 263, 254, 283, 233, 238, 267, 255, 260, 270, 269, 260, 269, 248, 278, 270, 252, 267, 249, 252, 260, 261, 251, 238, 228, 263, 253, 236, 259, 246, 257, 260, 282, 245, 283, 267, 242, 248, 253, 250, 287, 230, 253, 271, 255, 235, 253, 256, 245, 225, 239, 218, 269, 264, 267, 245, 285, 267, 266, 265, 251, 256, 279, 245, 231, 289, 256, 262, 261, 243, 268, 255, 262, 249, 248, 267, 273, 242, 242, 248, 300, 268, 240, 277, 261, 270, 255, 286, 250, 255, 249, 271, 251, 268, 270, 231, 258, 256, 218, 244, 237, 260, 267, 259, 262, 251, 261] payload_rows_per_dst=[5418, 5402, 5450, 5540, 5380, 5498, 5548, 5422] dup_rows_per_dst=[2724, 2744, 2748, 2656, 2740, 2742, 2818, 2706] combine_rows_per_src=[5418, 5402, 5450, 5540, 5380, 5498, 5548, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229304 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966840 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704376 weight_movement_bytes=37748736 total_nvlink_bytes=351453112 weight_to_token_ratio=0.120332 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229248 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778368 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313327488 weight_movement_bytes=18874368 total_nvlink_bytes=332201856 weight_to_token_ratio=0.060238 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155443200 dispatch_meta_bytes=227136 combine_payload_bytes=155443200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155670336 ingress_bytes=174317568 link_bytes=174317568 token_bytes=311113536 weight_movement_bytes=18874368 total_nvlink_bytes=329987904 weight_to_token_ratio=0.060667 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670225167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38220 dup_rows=21818 dup_rows_remote=19072 dedup_ratio=0.332916 router_tokens_per_expert=[270, 252, 227, 254, 253, 257, 278, 252, 260, 290, 263, 276, 233, 265, 257, 268, 229, 240, 261, 250, 251, 261, 238, 262, 250, 255, 276, 210, 255, 245, 249, 230, 264, 213, 242, 248, 250, 253, 263, 278, 238, 267, 269, 253, 285, 274, 270, 263, 252, 242, 256, 270, 269, 234, 270, 247, 255, 261, 246, 227, 259, 232, 249, 288, 260, 237, 285, 256, 264, 262, 249, 281, 258, 261, 245, 253, 243, 266, 236, 266, 257, 235, 237, 245, 229, 274, 237, 292, 259, 263, 266, 281, 247, 271, 265, 245, 262, 241, 261, 300, 277, 253, 262, 275, 245, 240, 251, 270, 271, 258, 266, 281, 243, 261, 245, 242, 243, 220, 275, 283, 234, 252, 258, 236, 249, 238, 294, 253] payload_rows_per_dst=[5532, 5328, 5520, 5432, 5498, 5500, 5548, 5360] dup_rows_per_dst=[2778, 2596, 2740, 2682, 2746, 2706, 2878, 2692] combine_rows_per_src=[5532, 5328, 5520, 5432, 5498, 5500, 5548, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38266 dup_rows=21782 dup_rows_remote=18988 dedup_ratio=0.332367 router_tokens_per_expert=[290, 260, 279, 246, 241, 257, 268, 239, 273, 249, 270, 269, 264, 255, 273, 265, 259, 252, 251, 260, 242, 237, 249, 259, 291, 266, 260, 243, 272, 271, 273, 245, 233, 238, 252, 242, 273, 255, 254, 247, 272, 275, 252, 243, 258, 244, 239, 243, 255, 270, 277, 255, 277, 236, 240, 256, 250, 252, 239, 224, 229, 217, 234, 280, 274, 254, 247, 273, 284, 243, 252, 249, 273, 239, 262, 261, 233, 246, 266, 246, 264, 265, 257, 247, 258, 236, 237, 234, 252, 245, 287, 248, 255, 270, 240, 268, 270, 242, 258, 246, 269, 250, 253, 272, 242, 248, 287, 247, 260, 264, 268, 265, 249, 264, 276, 261, 256, 266, 253, 238, 234, 274, 267, 231, 285, 234, 278, 257] payload_rows_per_dst=[5532, 5506, 5404, 5344, 5466, 5436, 5488, 5578] dup_rows_per_dst=[2864, 2754, 2636, 2638, 2682, 2690, 2794, 2724] combine_rows_per_src=[5532, 5506, 5404, 5344, 5466, 5436, 5488, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38236 dup_rows=21634 dup_rows_remote=18636 dedup_ratio=0.330109 router_tokens_per_expert=[257, 275, 252, 242, 260, 251, 245, 264, 266, 246, 285, 256, 267, 267, 240, 258, 278, 259, 254, 252, 260, 245, 263, 261, 256, 241, 244, 247, 268, 220, 263, 286, 258, 256, 261, 282, 277, 250, 258, 239, 252, 255, 246, 248, 275, 242, 233, 262, 277, 268, 240, 275, 257, 242, 255, 265, 257, 265, 285, 246, 242, 251, 275, 258, 256, 239, 289, 286, 253, 257, 240, 284, 264, 243, 245, 234, 266, 271, 260, 269, 292, 258, 223, 235, 245, 264, 252, 265, 245, 249, 264, 223, 289, 237, 250, 271, 280, 261, 275, 240, 244, 250, 219, 230, 222, 285, 227, 244, 266, 232, 228, 274, 259, 267, 263, 237, 269, 239, 250, 251, 261, 244, 263, 288, 262, 267, 236, 237] payload_rows_per_dst=[5666, 5482, 5424, 5376, 5504, 5652, 5460, 5338] dup_rows_per_dst=[2998, 2782, 2694, 2606, 2740, 2826, 2494, 2494] combine_rows_per_src=[5666, 5482, 5424, 5376, 5504, 5652, 5460, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38196 dup_rows=21844 dup_rows_remote=19040 dedup_ratio=0.333313 router_tokens_per_expert=[277, 265, 226, 246, 247, 273, 266, 247, 239, 267, 268, 266, 274, 237, 258, 273, 261, 256, 263, 253, 265, 245, 248, 271, 242, 262, 294, 257, 259, 251, 263, 260, 222, 259, 275, 246, 260, 238, 251, 267, 297, 266, 258, 224, 259, 274, 250, 263, 260, 281, 225, 275, 264, 219, 263, 260, 242, 241, 263, 263, 265, 229, 263, 261, 266, 235, 253, 261, 221, 246, 233, 240, 256, 266, 271, 233, 276, 284, 276, 259, 249, 290, 263, 260, 252, 261, 280, 251, 249, 261, 247, 241, 247, 236, 240, 267, 229, 260, 239, 238, 263, 259, 287, 252, 293, 267, 243, 281, 271, 257, 259, 254, 252, 244, 253, 233, 258, 258, 282, 241, 259, 247, 248, 230, 236, 247, 260, 236] payload_rows_per_dst=[5474, 5496, 5452, 5426, 5492, 5460, 5522, 5370] dup_rows_per_dst=[2784, 2804, 2766, 2722, 2660, 2728, 2782, 2598] combine_rows_per_src=[5474, 5496, 5452, 5426, 5492, 5460, 5522, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229184 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156532544 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312835904 weight_movement_bytes=18874368 total_nvlink_bytes=331710272 weight_to_token_ratio=0.060333 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229168 combine_payload_bytes=156549120 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156778288 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313327408 weight_movement_bytes=37748736 total_nvlink_bytes=351076144 weight_to_token_ratio=0.120477 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229016 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966552 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704088 weight_movement_bytes=37748736 total_nvlink_bytes=351452824 weight_to_token_ratio=0.120332 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=227488 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156842144 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313456800 weight_movement_bytes=18874368 total_nvlink_bytes=332331168 weight_to_token_ratio=0.060214 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38204 dup_rows=21820 dup_rows_remote=19096 dedup_ratio=0.332947 router_tokens_per_expert=[264, 282, 246, 267, 255, 240, 261, 238, 264, 264, 259, 267, 232, 237, 243, 245, 263, 254, 280, 252, 276, 282, 242, 256, 268, 251, 228, 268, 238, 267, 277, 273, 274, 249, 273, 304, 259, 256, 240, 270, 239, 250, 258, 235, 239, 273, 250, 249, 251, 254, 272, 232, 241, 247, 259, 248, 254, 266, 257, 268, 267, 234, 262, 251, 267, 233, 234, 259, 222, 269, 264, 279, 250, 267, 247, 261, 253, 268, 259, 249, 253, 268, 278, 268, 243, 239, 281, 251, 276, 240, 241, 246, 272, 243, 233, 242, 250, 263, 281, 250, 260, 246, 252, 245, 238, 282, 252, 266, 248, 301, 258, 275, 239, 256, 254, 250, 254, 234, 268, 242, 245, 255, 225, 259, 265, 261, 267, 252] payload_rows_per_dst=[5476, 5506, 5512, 5436, 5448, 5450, 5542, 5346] dup_rows_per_dst=[2652, 2844, 2724, 2690, 2714, 2698, 2792, 2706] combine_rows_per_src=[5476, 5506, 5512, 5436, 5448, 5450, 5542, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228944 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679760 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130576 weight_movement_bytes=37748736 total_nvlink_bytes=350879312 weight_to_token_ratio=0.120553 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38132 dup_rows=21974 dup_rows_remote=19250 dedup_ratio=0.335297 router_tokens_per_expert=[250, 242, 244, 270, 238, 240, 246, 239, 259, 250, 232, 256, 245, 226, 244, 280, 269, 255, 260, 253, 238, 253, 266, 268, 254, 278, 257, 275, 236, 263, 270, 240, 254, 269, 233, 255, 262, 268, 282, 253, 245, 266, 261, 246, 248, 249, 266, 272, 256, 268, 243, 273, 226, 242, 279, 247, 240, 297, 243, 227, 239, 267, 261, 269, 241, 254, 257, 242, 251, 238, 258, 271, 259, 277, 279, 241, 246, 229, 245, 246, 268, 252, 271, 249, 259, 236, 263, 270, 247, 252, 224, 279, 240, 274, 261, 270, 255, 256, 267, 234, 268, 252, 279, 266, 259, 240, 249, 259, 258, 240, 238, 245, 268, 246, 279, 255, 267, 230, 256, 272, 276, 293, 274, 259, 270, 273, 259, 275] payload_rows_per_dst=[5340, 5418, 5478, 5430, 5414, 5460, 5446, 5576] dup_rows_per_dst=[2582, 2852, 2780, 2724, 2654, 2770, 2684, 2928] combine_rows_per_src=[5340, 5418, 5478, 5430, 5414, 5460, 5446, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38000 dup_rows=21982 dup_rows_remote=18998 dedup_ratio=0.335419 router_tokens_per_expert=[253, 251, 258, 245, 257, 238, 222, 259, 242, 254, 234, 276, 303, 254, 254, 245, 253, 263, 291, 241, 294, 239, 239, 279, 244, 260, 256, 257, 236, 243, 265, 234, 287, 265, 260, 250, 227, 262, 258, 260, 258, 244, 253, 262, 264, 245, 270, 260, 256, 246, 254, 275, 237, 264, 248, 271, 240, 263, 245, 235, 269, 256, 286, 238, 259, 284, 256, 250, 255, 268, 276, 264, 252, 265, 281, 273, 262, 241, 247, 248, 253, 264, 268, 280, 265, 224, 257, 260, 259, 280, 272, 260, 241, 309, 227, 261, 259, 245, 247, 214, 257, 284, 260, 221, 264, 260, 253, 246, 247, 260, 267, 266, 263, 237, 241, 264, 225, 255, 264, 282, 263, 231, 222, 234, 262, 258, 245, 264] payload_rows_per_dst=[5376, 5350, 5362, 5556, 5460, 5500, 5396, 5554] dup_rows_per_dst=[2682, 2502, 2616, 2946, 2656, 2882, 2714, 2984] combine_rows_per_src=[5376, 5350, 5362, 5556, 5460, 5500, 5396, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229200 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712784 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196368 weight_movement_bytes=37748736 total_nvlink_bytes=350945104 weight_to_token_ratio=0.120527 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229528 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156418200 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312606872 weight_movement_bytes=18874368 total_nvlink_bytes=331481240 weight_to_token_ratio=0.060377 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155648000 dispatch_meta_bytes=227992 combine_payload_bytes=155648000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155875992 ingress_bytes=174522368 link_bytes=174522368 token_bytes=311523992 weight_movement_bytes=18874368 total_nvlink_bytes=330398360 weight_to_token_ratio=0.060587 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672187460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38276 dup_rows=21810 dup_rows_remote=19074 dedup_ratio=0.332794 router_tokens_per_expert=[251, 246, 268, 256, 256, 281, 254, 256, 239, 264, 256, 276, 257, 247, 257, 254, 250, 252, 234, 245, 252, 248, 287, 288, 255, 251, 246, 275, 270, 264, 274, 243, 293, 256, 275, 248, 257, 242, 260, 235, 234, 239, 256, 252, 256, 266, 261, 277, 233, 274, 267, 245, 257, 274, 256, 248, 245, 245, 251, 270, 232, 258, 253, 250, 276, 263, 282, 246, 279, 264, 239, 241, 269, 236, 258, 266, 233, 268, 253, 250, 251, 244, 248, 235, 275, 283, 277, 278, 256, 245, 240, 268, 253, 244, 255, 241, 251, 244, 254, 271, 252, 238, 235, 272, 269, 241, 241, 258, 243, 263, 263, 257, 246, 245, 274, 261, 265, 246, 267, 253, 246, 286, 253, 260, 257, 235, 248, 241] payload_rows_per_dst=[5514, 5520, 5446, 5422, 5472, 5450, 5470, 5432] dup_rows_per_dst=[2722, 2748, 2768, 2694, 2774, 2736, 2634, 2734] combine_rows_per_src=[5514, 5520, 5446, 5422, 5472, 5450, 5470, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43460 payload_rows_remote=37918 dup_rows=22076 dup_rows_remote=19222 dedup_ratio=0.336853 router_tokens_per_expert=[264, 268, 265, 259, 282, 258, 238, 243, 251, 253, 239, 267, 235, 268, 239, 282, 234, 271, 258, 267, 275, 228, 235, 234, 244, 255, 254, 270, 238, 260, 263, 276, 252, 271, 265, 257, 223, 258, 277, 262, 268, 253, 278, 256, 263, 271, 264, 250, 268, 241, 270, 272, 261, 260, 251, 265, 259, 244, 226, 267, 235, 252, 265, 242, 266, 251, 237, 285, 255, 247, 268, 262, 277, 232, 271, 270, 247, 279, 266, 285, 249, 232, 282, 272, 271, 263, 229, 260, 257, 262, 255, 218, 274, 251, 265, 257, 228, 276, 223, 268, 277, 239, 250, 242, 263, 241, 269, 247, 271, 270, 266, 256, 256, 259, 239, 252, 251, 220, 236, 269, 254, 240, 250, 237, 252, 257, 264, 232] payload_rows_per_dst=[5444, 5376, 5498, 5406, 5542, 5468, 5418, 5308] dup_rows_per_dst=[2778, 2748, 2838, 2750, 2854, 2726, 2754, 2628] combine_rows_per_src=[5444, 5376, 5498, 5406, 5542, 5468, 5418, 5308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=37962 dup_rows=21960 dup_rows_remote=19116 dedup_ratio=0.335083 router_tokens_per_expert=[248, 241, 258, 266, 271, 267, 255, 284, 280, 270, 272, 273, 253, 231, 258, 277, 279, 256, 243, 243, 275, 251, 281, 247, 268, 257, 247, 233, 267, 266, 257, 273, 260, 246, 237, 236, 269, 277, 222, 277, 227, 226, 268, 269, 288, 255, 262, 258, 248, 245, 251, 231, 239, 227, 240, 255, 257, 266, 247, 276, 260, 264, 262, 275, 266, 259, 257, 250, 231, 292, 261, 236, 261, 243, 250, 256, 224, 241, 221, 277, 249, 270, 250, 246, 248, 253, 258, 252, 253, 238, 255, 243, 229, 279, 277, 261, 244, 267, 273, 256, 269, 272, 275, 247, 269, 285, 242, 249, 255, 280, 262, 284, 234, 234, 228, 258, 261, 268, 258, 266, 235, 242, 239, 276, 240, 269, 231, 247] payload_rows_per_dst=[5478, 5514, 5410, 5408, 5366, 5432, 5614, 5354] dup_rows_per_dst=[2930, 2772, 2744, 2678, 2684, 2690, 2844, 2618] combine_rows_per_src=[5478, 5514, 5410, 5408, 5366, 5432, 5614, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38396 dup_rows=21616 dup_rows_remote=18944 dedup_ratio=0.329834 router_tokens_per_expert=[233, 271, 288, 221, 232, 265, 278, 261, 278, 294, 267, 232, 277, 249, 272, 270, 246, 249, 290, 254, 256, 254, 265, 238, 285, 239, 245, 259, 272, 248, 248, 250, 294, 242, 257, 279, 254, 255, 272, 259, 253, 252, 234, 252, 262, 215, 260, 244, 229, 272, 251, 266, 253, 264, 258, 288, 248, 282, 241, 254, 237, 259, 241, 246, 264, 240, 236, 259, 242, 297, 258, 256, 250, 244, 273, 244, 254, 275, 272, 263, 243, 248, 269, 248, 256, 260, 253, 229, 265, 261, 284, 240, 277, 268, 265, 251, 243, 270, 227, 266, 231, 277, 260, 266, 260, 248, 238, 257, 234, 241, 236, 268, 233, 260, 275, 241, 233, 268, 225, 259, 244, 271, 251, 255, 256, 253, 277, 242] payload_rows_per_dst=[5546, 5524, 5500, 5446, 5544, 5460, 5434, 5466] dup_rows_per_dst=[2830, 2672, 2668, 2732, 2710, 2774, 2610, 2620] combine_rows_per_src=[5546, 5524, 5500, 5446, 5544, 5460, 5434, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38024 dup_rows=21872 dup_rows_remote=18918 dedup_ratio=0.333740 router_tokens_per_expert=[256, 264, 266, 272, 262, 226, 252, 236, 274, 236, 259, 262, 291, 234, 249, 259, 237, 242, 243, 282, 269, 248, 261, 246, 247, 289, 241, 268, 250, 261, 260, 266, 256, 265, 277, 271, 258, 298, 260, 282, 250, 253, 267, 256, 238, 265, 227, 233, 257, 248, 271, 270, 246, 278, 245, 270, 278, 251, 244, 242, 262, 255, 252, 244, 294, 260, 250, 256, 270, 226, 235, 248, 260, 242, 260, 262, 249, 234, 243, 290, 272, 283, 263, 258, 239, 242, 254, 248, 239, 263, 265, 265, 262, 242, 218, 270, 264, 252, 271, 252, 244, 251, 244, 267, 264, 289, 239, 268, 243, 236, 261, 255, 248, 245, 230, 255, 280, 225, 231, 288, 284, 223, 267, 238, 246, 260, 264, 245] payload_rows_per_dst=[5640, 5592, 5344, 5460, 5518, 5340, 5362, 5408] dup_rows_per_dst=[2954, 2952, 2648, 2842, 2796, 2590, 2440, 2650] combine_rows_per_src=[5640, 5592, 5344, 5460, 5518, 5340, 5362, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229400 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007896 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786392 weight_movement_bytes=37748736 total_nvlink_bytes=351535128 weight_to_token_ratio=0.120301 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155312128 dispatch_meta_bytes=228560 combine_payload_bytes=155312128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155540688 ingress_bytes=193060864 link_bytes=193060864 token_bytes=310852816 weight_movement_bytes=37748736 total_nvlink_bytes=348601552 weight_to_token_ratio=0.121436 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.849814207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38376 dup_rows=21724 dup_rows_remote=18974 dedup_ratio=0.331482 router_tokens_per_expert=[236, 253, 269, 247, 247, 273, 279, 275, 237, 261, 259, 255, 277, 276, 235, 243, 252, 241, 220, 238, 256, 286, 252, 244, 263, 242, 233, 256, 253, 261, 244, 255, 263, 253, 262, 263, 242, 256, 252, 237, 266, 255, 264, 246, 269, 251, 252, 262, 275, 241, 235, 259, 277, 287, 266, 251, 271, 239, 266, 251, 259, 262, 254, 273, 236, 267, 258, 275, 247, 298, 256, 269, 259, 244, 244, 237, 236, 264, 252, 279, 226, 262, 227, 276, 250, 255, 262, 262, 246, 274, 260, 264, 224, 243, 258, 223, 271, 252, 270, 264, 258, 233, 246, 262, 241, 271, 245, 277, 247, 237, 258, 262, 268, 263, 254, 232, 263, 277, 279, 251, 259, 275, 275, 249, 245, 244, 268, 262] payload_rows_per_dst=[5520, 5414, 5436, 5574, 5490, 5396, 5422, 5560] dup_rows_per_dst=[2724, 2578, 2750, 2758, 2752, 2628, 2766, 2768] combine_rows_per_src=[5520, 5414, 5436, 5574, 5490, 5396, 5422, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=155492352 dispatch_meta_bytes=228312 combine_payload_bytes=155492352 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155720664 ingress_bytes=212115456 link_bytes=212115456 token_bytes=311213016 weight_movement_bytes=56623104 total_nvlink_bytes=367836120 weight_to_token_ratio=0.181943 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.032385932e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229360 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157499376 ingress_bytes=176144384 link_bytes=176144384 token_bytes=314769392 weight_movement_bytes=18874368 total_nvlink_bytes=333643760 weight_to_token_ratio=0.059963 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687728819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155746304 dispatch_meta_bytes=227768 combine_payload_bytes=155746304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155974072 ingress_bytes=174620672 link_bytes=174620672 token_bytes=311720376 weight_movement_bytes=18874368 total_nvlink_bytes=330594744 weight_to_token_ratio=0.060549 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673129360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38244 dup_rows=21916 dup_rows_remote=19322 dedup_ratio=0.334412 router_tokens_per_expert=[288, 246, 258, 249, 265, 254, 264, 268, 262, 266, 250, 275, 283, 249, 242, 282, 242, 257, 254, 279, 282, 253, 248, 262, 262, 272, 243, 250, 260, 262, 246, 249, 259, 230, 233, 260, 261, 258, 260, 257, 234, 296, 246, 261, 235, 256, 252, 253, 262, 264, 239, 232, 238, 222, 271, 255, 248, 273, 236, 246, 256, 246, 261, 236, 229, 234, 264, 267, 262, 236, 289, 243, 263, 281, 262, 225, 267, 280, 251, 243, 241, 233, 268, 238, 263, 281, 261, 224, 275, 260, 271, 257, 254, 250, 276, 252, 266, 287, 272, 228, 281, 258, 254, 235, 261, 234, 241, 268, 288, 239, 266, 268, 247, 250, 278, 250, 228, 243, 254, 269, 257, 250, 256, 244, 249, 272, 250, 267] payload_rows_per_dst=[5542, 5484, 5462, 5376, 5380, 5442, 5492, 5442] dup_rows_per_dst=[2860, 2758, 2640, 2594, 2812, 2766, 2800, 2686] combine_rows_per_src=[5542, 5484, 5462, 5376, 5380, 5442, 5492, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43952 payload_rows_remote=38416 dup_rows=21584 dup_rows_remote=18756 dedup_ratio=0.329346 router_tokens_per_expert=[250, 252, 235, 257, 223, 293, 248, 242, 268, 245, 259, 253, 259, 247, 229, 273, 254, 237, 223, 250, 242, 253, 247, 239, 249, 258, 266, 267, 256, 258, 282, 266, 279, 266, 259, 260, 256, 251, 275, 257, 261, 259, 228, 271, 242, 241, 287, 249, 254, 253, 218, 256, 274, 277, 262, 270, 250, 261, 257, 238, 290, 240, 276, 262, 239, 235, 245, 277, 231, 272, 253, 237, 262, 256, 255, 210, 265, 242, 257, 239, 253, 258, 253, 255, 260, 214, 288, 249, 279, 260, 277, 268, 278, 249, 256, 272, 275, 239, 258, 264, 249, 258, 244, 255, 274, 244, 259, 257, 257, 256, 262, 277, 259, 251, 236, 269, 274, 262, 289, 254, 241, 263, 271, 256, 246, 246, 262, 258] payload_rows_per_dst=[5420, 5420, 5560, 5532, 5410, 5526, 5548, 5536] dup_rows_per_dst=[2612, 2674, 2736, 2744, 2540, 2812, 2638, 2828] combine_rows_per_src=[5420, 5420, 5560, 5532, 5410, 5526, 5548, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229400 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157417496 ingress_bytes=194936832 link_bytes=194936832 token_bytes=314605592 weight_movement_bytes=37748736 total_nvlink_bytes=352354328 weight_to_token_ratio=0.119987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867788809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=230264 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156877688 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313525112 weight_movement_bytes=18874368 total_nvlink_bytes=332399480 weight_to_token_ratio=0.060200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=228688 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157580624 ingress_bytes=195100672 link_bytes=195100672 token_bytes=314932560 weight_movement_bytes=37748736 total_nvlink_bytes=352681296 weight_to_token_ratio=0.119863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869358644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38340 dup_rows=21732 dup_rows_remote=18998 dedup_ratio=0.331604 router_tokens_per_expert=[253, 246, 253, 237, 271, 235, 237, 279, 286, 265, 258, 277, 255, 240, 251, 251, 258, 233, 286, 231, 246, 259, 258, 264, 266, 260, 250, 259, 243, 262, 221, 262, 257, 256, 265, 263, 285, 247, 235, 265, 249, 250, 263, 240, 270, 255, 248, 249, 288, 252, 238, 282, 232, 248, 249, 251, 253, 258, 279, 259, 256, 238, 259, 260, 266, 279, 246, 283, 248, 263, 228, 224, 268, 245, 253, 268, 290, 218, 258, 233, 288, 213, 237, 269, 269, 270, 251, 268, 278, 265, 248, 250, 264, 252, 238, 239, 266, 272, 253, 232, 270, 248, 231, 263, 270, 243, 285, 248, 274, 236, 268, 247, 244, 253, 269, 240, 259, 242, 283, 262, 264, 240, 246, 282, 249, 287, 243, 279] payload_rows_per_dst=[5432, 5446, 5486, 5468, 5448, 5464, 5562, 5498] dup_rows_per_dst=[2756, 2670, 2708, 2736, 2692, 2734, 2650, 2786] combine_rows_per_src=[5432, 5446, 5486, 5468, 5448, 5464, 5562, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38170 dup_rows=21904 dup_rows_remote=19212 dedup_ratio=0.334229 router_tokens_per_expert=[241, 269, 260, 243, 255, 258, 256, 246, 249, 247, 239, 296, 239, 253, 275, 257, 230, 307, 287, 281, 231, 242, 291, 260, 275, 242, 251, 248, 243, 279, 280, 258, 251, 249, 245, 249, 261, 255, 268, 271, 262, 259, 261, 228, 263, 254, 232, 269, 269, 227, 253, 234, 220, 239, 269, 268, 242, 235, 283, 242, 240, 280, 237, 253, 268, 230, 275, 273, 238, 262, 248, 268, 263, 255, 226, 236, 269, 268, 270, 228, 268, 262, 258, 238, 270, 244, 273, 250, 268, 235, 242, 278, 252, 259, 241, 270, 263, 269, 266, 264, 248, 280, 281, 254, 261, 270, 245, 263, 234, 262, 270, 252, 240, 263, 259, 268, 226, 255, 243, 259, 272, 254, 230, 282, 254, 237, 243, 260] payload_rows_per_dst=[5434, 5534, 5468, 5348, 5462, 5440, 5474, 5472] dup_rows_per_dst=[2732, 2876, 2686, 2634, 2692, 2776, 2890, 2618] combine_rows_per_src=[5434, 5534, 5468, 5348, 5462, 5440, 5474, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38298 dup_rows=21682 dup_rows_remote=18970 dedup_ratio=0.330841 router_tokens_per_expert=[292, 253, 281, 287, 248, 257, 262, 252, 250, 259, 252, 244, 239, 242, 261, 251, 259, 258, 259, 259, 255, 275, 250, 245, 266, 228, 260, 260, 267, 253, 243, 255, 239, 259, 243, 259, 235, 256, 245, 266, 227, 250, 238, 248, 266, 255, 261, 245, 244, 289, 271, 274, 234, 236, 248, 247, 262, 265, 246, 262, 295, 266, 256, 253, 256, 255, 236, 275, 271, 260, 255, 245, 241, 249, 247, 277, 258, 241, 254, 233, 263, 271, 237, 236, 274, 229, 282, 281, 258, 235, 280, 278, 256, 216, 260, 226, 282, 256, 253, 237, 242, 268, 281, 265, 269, 244, 243, 249, 261, 239, 262, 283, 246, 233, 271, 281, 251, 285, 249, 271, 239, 273, 239, 244, 287, 267, 257, 244] payload_rows_per_dst=[5538, 5430, 5368, 5592, 5414, 5448, 5556, 5508] dup_rows_per_dst=[2722, 2754, 2616, 2704, 2692, 2716, 2712, 2766] combine_rows_per_src=[5538, 5430, 5368, 5592, 5414, 5448, 5556, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43992 payload_rows_remote=38420 dup_rows=21544 dup_rows_remote=18800 dedup_ratio=0.328735 router_tokens_per_expert=[261, 277, 246, 257, 259, 280, 240, 251, 240, 257, 273, 230, 256, 243, 264, 235, 237, 302, 259, 243, 239, 264, 255, 240, 261, 268, 265, 256, 283, 266, 245, 275, 262, 244, 262, 229, 285, 236, 299, 272, 249, 275, 259, 241, 279, 249, 257, 239, 289, 247, 254, 258, 237, 255, 264, 266, 259, 238, 251, 251, 277, 253, 229, 277, 242, 224, 257, 275, 242, 252, 231, 270, 270, 313, 226, 248, 262, 238, 237, 269, 271, 254, 258, 261, 276, 271, 226, 227, 255, 228, 257, 253, 268, 277, 254, 247, 262, 266, 250, 243, 245, 253, 249, 249, 276, 237, 258, 256, 262, 249, 274, 268, 289, 245, 268, 265, 222, 246, 229, 242, 224, 258, 283, 240, 252, 256, 284, 260] payload_rows_per_dst=[5472, 5572, 5504, 5454, 5480, 5466, 5504, 5540] dup_rows_per_dst=[2666, 2744, 2770, 2756, 2632, 2700, 2690, 2586] combine_rows_per_src=[5472, 5572, 5504, 5454, 5480, 5466, 5504, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38298 dup_rows=21768 dup_rows_remote=19000 dedup_ratio=0.332153 router_tokens_per_expert=[250, 255, 231, 249, 261, 258, 259, 236, 260, 268, 246, 261, 273, 283, 266, 262, 277, 248, 275, 303, 270, 247, 259, 273, 247, 267, 247, 256, 252, 231, 261, 264, 276, 243, 260, 247, 251, 260, 254, 215, 264, 274, 247, 260, 286, 262, 228, 275, 244, 267, 274, 253, 238, 266, 295, 246, 249, 246, 284, 252, 271, 257, 276, 247, 232, 260, 255, 261, 253, 258, 230, 255, 246, 256, 232, 277, 252, 257, 261, 251, 230, 257, 260, 261, 244, 275, 226, 297, 258, 269, 236, 246, 291, 264, 238, 235, 248, 277, 248, 251, 275, 248, 236, 242, 244, 258, 250, 255, 241, 285, 250, 286, 269, 253, 228, 239, 239, 248, 244, 276, 260, 258, 268, 230, 244, 245, 228, 260] payload_rows_per_dst=[5470, 5640, 5362, 5472, 5418, 5480, 5552, 5374] dup_rows_per_dst=[2768, 2988, 2668, 2758, 2652, 2694, 2636, 2604] combine_rows_per_src=[5470, 5640, 5362, 5472, 5418, 5480, 5552, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229352 combine_payload_bytes=157040640 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157269992 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314310632 weight_movement_bytes=18874368 total_nvlink_bytes=333185000 weight_to_token_ratio=0.060050 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38164 dup_rows=21858 dup_rows_remote=19064 dedup_ratio=0.333527 router_tokens_per_expert=[256, 239, 222, 262, 249, 258, 273, 275, 284, 262, 252, 284, 287, 246, 252, 241, 218, 283, 287, 273, 251, 262, 270, 232, 266, 230, 243, 271, 258, 238, 267, 263, 301, 248, 286, 252, 266, 263, 233, 277, 254, 232, 249, 250, 258, 248, 280, 257, 257, 238, 246, 232, 257, 274, 250, 275, 271, 246, 258, 253, 256, 251, 243, 251, 259, 245, 270, 239, 274, 275, 240, 295, 250, 275, 246, 279, 267, 244, 226, 237, 238, 247, 249, 250, 246, 266, 259, 256, 233, 283, 261, 235, 251, 251, 239, 235, 264, 253, 261, 287, 258, 237, 272, 268, 244, 246, 264, 243, 265, 231, 256, 245, 248, 245, 236, 257, 244, 282, 281, 265, 246, 262, 281, 248, 243, 261, 243, 246] payload_rows_per_dst=[5478, 5470, 5514, 5466, 5526, 5348, 5450, 5426] dup_rows_per_dst=[2806, 2754, 2794, 2650, 2716, 2650, 2738, 2750] combine_rows_per_src=[5478, 5470, 5514, 5466, 5526, 5348, 5450, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229528 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573848 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312918168 weight_movement_bytes=18874368 total_nvlink_bytes=331792536 weight_to_token_ratio=0.060317 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38170 dup_rows=21758 dup_rows_remote=18932 dedup_ratio=0.332001 router_tokens_per_expert=[250, 265, 263, 273, 244, 255, 247, 229, 254, 253, 226, 264, 292, 242, 232, 254, 255, 278, 244, 239, 245, 263, 241, 258, 270, 288, 230, 239, 248, 233, 274, 283, 264, 248, 266, 260, 290, 253, 239, 272, 233, 237, 242, 265, 243, 245, 266, 235, 256, 273, 250, 275, 262, 238, 269, 286, 254, 258, 277, 281, 242, 267, 276, 253, 243, 259, 239, 278, 237, 275, 240, 252, 262, 263, 232, 257, 241, 267, 224, 242, 243, 275, 250, 253, 254, 234, 276, 275, 252, 279, 270, 272, 254, 260, 244, 265, 264, 268, 258, 260, 268, 264, 272, 238, 244, 272, 268, 280, 274, 240, 257, 238, 253, 232, 260, 223, 261, 253, 283, 246, 259, 252, 274, 235, 263, 240, 246, 250] payload_rows_per_dst=[5438, 5496, 5400, 5608, 5362, 5542, 5506, 5426] dup_rows_per_dst=[2648, 2680, 2716, 2826, 2660, 2770, 2824, 2634] combine_rows_per_src=[5438, 5496, 5400, 5608, 5362, 5542, 5506, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229072 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097680 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966288 weight_movement_bytes=18874368 total_nvlink_bytes=332840656 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38130 dup_rows=21832 dup_rows_remote=19034 dedup_ratio=0.333130 router_tokens_per_expert=[239, 251, 267, 240, 265, 274, 287, 268, 250, 277, 257, 279, 261, 250, 238, 234, 254, 262, 253, 251, 247, 289, 277, 265, 242, 246, 261, 239, 271, 273, 250, 246, 273, 247, 269, 256, 263, 251, 239, 244, 244, 284, 254, 245, 254, 258, 267, 255, 238, 266, 232, 262, 236, 266, 251, 289, 268, 268, 249, 269, 259, 255, 233, 248, 244, 272, 245, 258, 258, 266, 249, 259, 279, 246, 254, 255, 261, 245, 243, 264, 231, 261, 257, 244, 248, 263, 249, 252, 255, 289, 267, 244, 284, 259, 279, 240, 272, 236, 274, 260, 245, 275, 259, 219, 263, 246, 226, 238, 261, 253, 249, 250, 250, 284, 256, 267, 257, 248, 231, 268, 247, 250, 273, 236, 255, 241, 238, 266] payload_rows_per_dst=[5196, 5366, 5486, 5620, 5446, 5548, 5468, 5574] dup_rows_per_dst=[2288, 2644, 2854, 2896, 2750, 2868, 2734, 2798] combine_rows_per_src=[5196, 5366, 5486, 5620, 5446, 5548, 5468, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=228880 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157597200 ingress_bytes=213991424 link_bytes=213991424 token_bytes=314965520 weight_movement_bytes=56623104 total_nvlink_bytes=371588624 weight_to_token_ratio=0.179776 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050360534e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229192 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157097800 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966408 weight_movement_bytes=37748736 total_nvlink_bytes=351715144 weight_to_token_ratio=0.120232 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228912 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548656 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868400 weight_movement_bytes=18874368 total_nvlink_bytes=331742768 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228408 combine_payload_bytes=156344320 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156572728 ingress_bytes=231841792 link_bytes=231841792 token_bytes=312917048 weight_movement_bytes=75497472 total_nvlink_bytes=388414520 weight_to_token_ratio=0.241270 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221393978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=228656 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156409136 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312589616 weight_movement_bytes=18874368 total_nvlink_bytes=331463984 weight_to_token_ratio=0.060381 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38206 dup_rows=21862 dup_rows_remote=19124 dedup_ratio=0.333588 router_tokens_per_expert=[249, 276, 265, 269, 275, 264, 233, 276, 261, 259, 284, 243, 272, 249, 230, 250, 234, 256, 260, 253, 214, 238, 287, 263, 236, 269, 252, 283, 255, 275, 269, 270, 276, 239, 257, 257, 269, 252, 245, 271, 256, 261, 257, 261, 254, 254, 269, 261, 259, 243, 261, 234, 238, 244, 274, 258, 263, 238, 235, 269, 253, 262, 281, 248, 269, 247, 250, 252, 259, 291, 270, 257, 252, 280, 254, 241, 243, 267, 297, 245, 253, 251, 284, 249, 257, 276, 269, 234, 257, 246, 273, 253, 241, 245, 281, 234, 237, 235, 211, 243, 249, 261, 274, 249, 235, 237, 226, 266, 266, 252, 277, 258, 236, 277, 277, 222, 235, 234, 259, 225, 265, 255, 256, 263, 253, 275, 254, 261] payload_rows_per_dst=[5488, 5478, 5492, 5498, 5564, 5468, 5324, 5362] dup_rows_per_dst=[2822, 2750, 2786, 2622, 2784, 2738, 2628, 2732] combine_rows_per_src=[5488, 5478, 5492, 5498, 5564, 5468, 5324, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43942 payload_rows_remote=38560 dup_rows=21594 dup_rows_remote=19080 dedup_ratio=0.329498 router_tokens_per_expert=[272, 234, 255, 254, 275, 276, 271, 229, 253, 289, 257, 278, 261, 247, 256, 260, 247, 266, 263, 274, 252, 275, 266, 254, 244, 254, 259, 239, 269, 271, 271, 242, 261, 268, 263, 253, 249, 269, 234, 236, 233, 231, 249, 257, 273, 238, 287, 249, 225, 293, 272, 249, 274, 248, 266, 227, 257, 278, 265, 224, 250, 273, 258, 268, 222, 249, 245, 230, 244, 259, 238, 253, 270, 243, 229, 227, 269, 266, 271, 233, 250, 280, 234, 252, 242, 265, 267, 214, 265, 256, 241, 248, 250, 248, 253, 268, 255, 260, 289, 279, 254, 299, 249, 275, 234, 260, 258, 224, 266, 254, 243, 274, 261, 241, 265, 281, 250, 251, 234, 246, 252, 251, 289, 259, 278, 255, 258, 253] payload_rows_per_dst=[5570, 5542, 5460, 5570, 5382, 5464, 5530, 5424] dup_rows_per_dst=[2764, 2750, 2640, 2684, 2514, 2602, 2816, 2824] combine_rows_per_src=[5570, 5542, 5460, 5570, 5382, 5464, 5530, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38144 dup_rows=21808 dup_rows_remote=19058 dedup_ratio=0.332764 router_tokens_per_expert=[280, 254, 264, 280, 259, 225, 270, 250, 246, 251, 261, 268, 256, 235, 235, 272, 261, 269, 259, 229, 277, 261, 260, 251, 244, 245, 253, 265, 252, 283, 253, 271, 247, 270, 257, 258, 269, 266, 223, 253, 255, 238, 261, 253, 234, 263, 229, 254, 264, 250, 262, 276, 230, 242, 284, 274, 259, 267, 258, 301, 245, 245, 249, 253, 235, 253, 260, 261, 277, 248, 229, 253, 284, 251, 253, 250, 274, 256, 249, 256, 224, 264, 276, 255, 257, 231, 239, 269, 277, 244, 219, 268, 247, 257, 261, 245, 272, 269, 274, 259, 263, 250, 256, 244, 252, 253, 279, 256, 261, 259, 270, 250, 259, 257, 256, 257, 254, 272, 234, 227, 236, 222, 267, 265, 268, 252, 267, 258] payload_rows_per_dst=[5474, 5454, 5400, 5512, 5498, 5396, 5584, 5410] dup_rows_per_dst=[2738, 2812, 2660, 2806, 2680, 2670, 2750, 2692] combine_rows_per_src=[5474, 5454, 5400, 5512, 5498, 5396, 5584, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38386 dup_rows=21766 dup_rows_remote=19074 dedup_ratio=0.332123 router_tokens_per_expert=[266, 273, 256, 276, 252, 255, 275, 268, 268, 251, 240, 254, 236, 240, 268, 248, 268, 228, 248, 257, 271, 249, 238, 270, 229, 256, 268, 242, 268, 267, 244, 235, 227, 246, 267, 257, 283, 234, 261, 244, 281, 260, 268, 273, 269, 263, 246, 265, 264, 250, 265, 219, 263, 291, 246, 260, 261, 259, 263, 266, 260, 268, 260, 242, 272, 253, 253, 242, 266, 250, 255, 250, 237, 277, 265, 268, 252, 239, 249, 273, 249, 236, 256, 258, 277, 264, 261, 228, 225, 250, 286, 269, 281, 270, 272, 268, 263, 264, 267, 250, 250, 235, 254, 237, 256, 248, 247, 264, 254, 255, 244, 250, 260, 249, 258, 226, 287, 251, 268, 231, 273, 228, 243, 259, 250, 259, 256, 236] payload_rows_per_dst=[5548, 5384, 5538, 5494, 5490, 5524, 5368, 5424] dup_rows_per_dst=[2704, 2692, 2750, 2780, 2712, 2776, 2708, 2644] combine_rows_per_src=[5548, 5384, 5538, 5494, 5490, 5524, 5368, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229320 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721096 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313212872 weight_movement_bytes=18874368 total_nvlink_bytes=332087240 weight_to_token_ratio=0.060261 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38298 dup_rows=21834 dup_rows_remote=19140 dedup_ratio=0.333160 router_tokens_per_expert=[271, 274, 244, 257, 222, 227, 275, 265, 245, 250, 260, 247, 270, 262, 249, 231, 268, 240, 277, 229, 289, 241, 253, 276, 246, 262, 251, 243, 263, 253, 256, 253, 242, 233, 266, 243, 272, 263, 291, 224, 240, 266, 251, 244, 279, 263, 248, 253, 265, 268, 257, 270, 268, 285, 250, 263, 261, 250, 220, 261, 242, 266, 263, 252, 263, 255, 270, 243, 264, 254, 279, 244, 256, 240, 274, 252, 226, 237, 256, 248, 261, 240, 242, 257, 271, 254, 269, 231, 270, 249, 249, 281, 247, 277, 259, 239, 279, 259, 285, 269, 258, 232, 255, 256, 246, 272, 232, 251, 275, 234, 260, 234, 256, 287, 266, 246, 240, 286, 245, 276, 270, 238, 250, 233, 266, 261, 252, 274] payload_rows_per_dst=[5404, 5530, 5542, 5468, 5438, 5358, 5468, 5494] dup_rows_per_dst=[2694, 2808, 2728, 2814, 2684, 2696, 2612, 2798] combine_rows_per_src=[5404, 5530, 5542, 5468, 5438, 5358, 5468, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38204 dup_rows=21900 dup_rows_remote=19200 dedup_ratio=0.334167 router_tokens_per_expert=[267, 271, 264, 250, 260, 258, 272, 266, 247, 250, 255, 243, 254, 250, 239, 243, 269, 264, 268, 267, 252, 254, 264, 265, 254, 255, 269, 266, 250, 246, 253, 246, 270, 270, 246, 251, 245, 218, 266, 232, 267, 276, 251, 229, 276, 265, 262, 242, 284, 261, 242, 248, 250, 256, 270, 261, 236, 248, 260, 251, 295, 255, 234, 261, 243, 247, 251, 299, 260, 244, 255, 264, 254, 274, 268, 249, 254, 233, 256, 237, 241, 284, 243, 265, 299, 259, 269, 234, 256, 251, 281, 219, 280, 238, 281, 262, 256, 242, 258, 243, 267, 243, 243, 247, 250, 275, 203, 259, 251, 258, 258, 245, 261, 235, 243, 239, 261, 278, 243, 264, 272, 240, 272, 283, 282, 245, 242, 251] payload_rows_per_dst=[5468, 5542, 5432, 5398, 5476, 5476, 5410, 5434] dup_rows_per_dst=[2710, 2742, 2700, 2826, 2700, 2848, 2586, 2788] combine_rows_per_src=[5468, 5542, 5432, 5398, 5476, 5476, 5410, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38190 dup_rows=21886 dup_rows_remote=19242 dedup_ratio=0.333954 router_tokens_per_expert=[249, 242, 287, 241, 254, 277, 268, 230, 255, 253, 278, 240, 250, 258, 249, 261, 262, 268, 258, 269, 276, 252, 258, 268, 266, 246, 229, 240, 286, 248, 267, 266, 227, 252, 251, 252, 250, 268, 250, 264, 256, 243, 258, 242, 271, 270, 238, 283, 271, 240, 264, 253, 225, 269, 260, 247, 265, 258, 251, 257, 248, 275, 231, 238, 250, 245, 268, 243, 261, 293, 265, 266, 262, 241, 261, 252, 248, 294, 273, 258, 274, 266, 245, 255, 252, 241, 286, 272, 246, 237, 244, 250, 289, 262, 236, 242, 269, 275, 229, 245, 238, 267, 255, 256, 265, 251, 262, 264, 265, 257, 241, 239, 230, 243, 241, 250, 242, 284, 275, 250, 246, 244, 272, 251, 264, 237, 270, 236] payload_rows_per_dst=[5388, 5484, 5456, 5460, 5506, 5482, 5464, 5410] dup_rows_per_dst=[2796, 2834, 2694, 2644, 2854, 2712, 2692, 2660] combine_rows_per_src=[5388, 5484, 5456, 5460, 5506, 5482, 5464, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157941760 dispatch_meta_bytes=230560 combine_payload_bytes=157941760 prefetch_bytes=0 reduce_bytes=0 egress_bytes=158172320 ingress_bytes=157941760 link_bytes=158172320 token_bytes=316114080 weight_movement_bytes=0 total_nvlink_bytes=316114080 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.515529345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=228808 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156466632 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312704456 weight_movement_bytes=18874368 total_nvlink_bytes=331578824 weight_to_token_ratio=0.060358 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38296 dup_rows=21682 dup_rows_remote=18828 dedup_ratio=0.330841 router_tokens_per_expert=[246, 241, 246, 288, 260, 259, 242, 283, 282, 239, 254, 258, 230, 254, 263, 263, 245, 263, 246, 251, 252, 245, 250, 249, 267, 277, 276, 267, 271, 251, 262, 247, 263, 248, 244, 259, 245, 259, 272, 285, 217, 258, 254, 230, 232, 257, 241, 265, 284, 259, 243, 254, 237, 271, 268, 285, 267, 258, 241, 246, 252, 247, 237, 272, 262, 264, 266, 247, 266, 240, 260, 266, 276, 261, 228, 278, 259, 230, 270, 262, 236, 243, 230, 254, 235, 267, 251, 252, 255, 265, 261, 254, 260, 264, 275, 236, 271, 283, 260, 244, 264, 225, 269, 220, 235, 256, 261, 246, 276, 247, 247, 253, 243, 250, 254, 250, 272, 245, 266, 277, 269, 222, 264, 273, 300, 245, 284, 247] payload_rows_per_dst=[5490, 5500, 5362, 5414, 5500, 5492, 5538, 5558] dup_rows_per_dst=[2726, 2738, 2696, 2544, 2770, 2588, 2766, 2854] combine_rows_per_src=[5490, 5500, 5362, 5414, 5500, 5492, 5538, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229840 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458896 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687952 weight_movement_bytes=18874368 total_nvlink_bytes=333562320 weight_to_token_ratio=0.059978 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229752 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157098360 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966968 weight_movement_bytes=37748736 total_nvlink_bytes=351715704 weight_to_token_ratio=0.120232 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229616 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156713200 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196784 weight_movement_bytes=18874368 total_nvlink_bytes=332071152 weight_to_token_ratio=0.060264 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229728 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655968 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313082208 weight_movement_bytes=18874368 total_nvlink_bytes=331956576 weight_to_token_ratio=0.060286 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228496 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157088912 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949328 weight_movement_bytes=18874368 total_nvlink_bytes=332823696 weight_to_token_ratio=0.060119 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38176 dup_rows=21782 dup_rows_remote=19094 dedup_ratio=0.332367 router_tokens_per_expert=[231, 266, 257, 251, 265, 268, 275, 245, 233, 243, 261, 248, 259, 253, 264, 229, 234, 255, 291, 248, 252, 273, 264, 289, 260, 262, 242, 268, 251, 257, 265, 211, 244, 223, 270, 246, 258, 229, 270, 285, 238, 244, 255, 257, 258, 261, 251, 241, 271, 267, 249, 240, 240, 248, 267, 217, 259, 281, 275, 221, 257, 252, 289, 235, 257, 249, 252, 259, 257, 259, 245, 278, 268, 275, 244, 300, 247, 235, 239, 246, 260, 249, 253, 273, 250, 266, 224, 263, 253, 280, 234, 269, 249, 272, 265, 273, 258, 255, 246, 245, 269, 255, 245, 231, 246, 252, 262, 245, 259, 270, 250, 261, 255, 252, 281, 248, 247, 257, 289, 265, 249, 280, 269, 237, 284, 254, 263, 278] payload_rows_per_dst=[5418, 5482, 5448, 5430, 5438, 5578, 5380, 5580] dup_rows_per_dst=[2678, 2762, 2612, 2706, 2782, 2688, 2718, 2836] combine_rows_per_src=[5418, 5482, 5448, 5430, 5438, 5578, 5380, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38306 dup_rows=21742 dup_rows_remote=19122 dedup_ratio=0.331757 router_tokens_per_expert=[273, 260, 242, 289, 274, 256, 261, 245, 250, 247, 241, 269, 258, 259, 251, 247, 249, 247, 269, 267, 268, 255, 273, 224, 268, 268, 228, 246, 262, 234, 254, 264, 263, 258, 265, 261, 267, 285, 246, 259, 242, 259, 291, 266, 260, 279, 263, 237, 278, 269, 262, 251, 263, 265, 276, 258, 257, 246, 272, 226, 246, 256, 231, 217, 261, 238, 274, 231, 251, 252, 278, 249, 264, 222, 240, 252, 250, 259, 281, 252, 237, 255, 242, 250, 257, 271, 244, 249, 232, 247, 251, 278, 243, 252, 255, 261, 255, 254, 271, 248, 255, 270, 235, 266, 255, 239, 272, 303, 269, 250, 223, 237, 246, 244, 251, 248, 249, 270, 255, 273, 275, 245, 253, 257, 262, 274, 240, 274] payload_rows_per_dst=[5476, 5448, 5540, 5474, 5488, 5396, 5480, 5492] dup_rows_per_dst=[2768, 2704, 2862, 2672, 2620, 2652, 2724, 2740] combine_rows_per_src=[5476, 5448, 5540, 5474, 5488, 5396, 5480, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38254 dup_rows=21806 dup_rows_remote=19232 dedup_ratio=0.332733 router_tokens_per_expert=[250, 243, 284, 249, 244, 246, 266, 254, 271, 250, 266, 259, 251, 249, 246, 269, 271, 286, 278, 276, 230, 259, 263, 262, 231, 261, 254, 282, 254, 272, 281, 268, 238, 252, 251, 276, 246, 252, 251, 245, 240, 266, 256, 272, 226, 252, 258, 237, 262, 250, 251, 267, 284, 271, 284, 265, 261, 271, 243, 237, 279, 258, 244, 249, 244, 256, 240, 280, 257, 269, 254, 231, 238, 251, 238, 241, 276, 224, 244, 255, 269, 258, 232, 269, 261, 255, 245, 249, 250, 272, 272, 276, 282, 233, 253, 256, 242, 243, 243, 258, 265, 249, 245, 287, 245, 252, 260, 262, 238, 221, 245, 270, 278, 268, 249, 257, 285, 224, 237, 286, 255, 266, 248, 245, 266, 242, 250, 238] payload_rows_per_dst=[5452, 5572, 5396, 5574, 5322, 5476, 5476, 5462] dup_rows_per_dst=[2742, 2884, 2640, 2778, 2674, 2788, 2574, 2726] combine_rows_per_src=[5452, 5572, 5396, 5574, 5322, 5476, 5476, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38318 dup_rows=21752 dup_rows_remote=19134 dedup_ratio=0.331909 router_tokens_per_expert=[254, 226, 239, 257, 251, 232, 268, 234, 231, 245, 270, 249, 267, 281, 236, 246, 227, 259, 244, 245, 247, 254, 285, 241, 260, 245, 281, 244, 243, 236, 256, 275, 268, 231, 271, 257, 259, 272, 249, 259, 242, 246, 232, 248, 253, 261, 267, 265, 260, 262, 281, 251, 244, 247, 259, 257, 278, 269, 267, 232, 224, 239, 264, 245, 255, 245, 247, 257, 265, 257, 266, 258, 263, 230, 287, 263, 261, 254, 265, 272, 264, 245, 241, 259, 256, 251, 238, 251, 265, 278, 260, 264, 266, 269, 276, 258, 253, 263, 255, 305, 259, 264, 273, 297, 244, 250, 263, 241, 254, 261, 270, 266, 255, 254, 244, 248, 239, 286, 251, 260, 251, 256, 257, 236, 240, 267, 266, 267] payload_rows_per_dst=[5434, 5466, 5474, 5406, 5506, 5508, 5540, 5450] dup_rows_per_dst=[2538, 2618, 2686, 2752, 2784, 2774, 2896, 2704] combine_rows_per_src=[5434, 5466, 5474, 5406, 5506, 5508, 5540, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229080 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156597976 ingress_bytes=212992000 link_bytes=212992000 token_bytes=312966872 weight_movement_bytes=56623104 total_nvlink_bytes=369589976 weight_to_token_ratio=0.180924 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040784545e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38092 dup_rows=21898 dup_rows_remote=19160 dedup_ratio=0.334137 router_tokens_per_expert=[278, 242, 274, 250, 270, 225, 257, 245, 273, 289, 243, 271, 265, 277, 263, 264, 257, 271, 277, 255, 258, 269, 230, 239, 268, 251, 277, 269, 256, 266, 229, 258, 277, 221, 232, 255, 269, 268, 267, 246, 235, 261, 256, 299, 289, 239, 238, 305, 270, 249, 255, 259, 293, 241, 263, 263, 253, 261, 226, 272, 249, 256, 269, 263, 238, 246, 269, 232, 264, 250, 276, 285, 278, 224, 266, 246, 288, 250, 259, 224, 266, 257, 236, 243, 253, 250, 265, 237, 240, 241, 274, 269, 235, 247, 276, 257, 229, 263, 237, 246, 257, 242, 252, 266, 269, 246, 249, 263, 271, 245, 246, 224, 256, 239, 257, 231, 256, 258, 255, 269, 261, 223, 253, 246, 243, 271, 240, 249] payload_rows_per_dst=[5480, 5532, 5446, 5546, 5458, 5416, 5360, 5400] dup_rows_per_dst=[2892, 2728, 2868, 2738, 2732, 2676, 2650, 2614] combine_rows_per_src=[5480, 5532, 5446, 5546, 5458, 5416, 5360, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44076 payload_rows_remote=38504 dup_rows=21460 dup_rows_remote=18758 dedup_ratio=0.327454 router_tokens_per_expert=[256, 259, 272, 266, 256, 243, 252, 265, 252, 236, 258, 245, 230, 283, 253, 246, 265, 256, 249, 260, 248, 274, 249, 224, 227, 248, 271, 273, 278, 271, 257, 242, 288, 272, 248, 280, 238, 281, 256, 229, 253, 241, 282, 245, 266, 237, 262, 259, 250, 222, 252, 251, 283, 255, 255, 236, 266, 264, 248, 259, 249, 248, 248, 268, 255, 237, 257, 260, 273, 248, 250, 229, 255, 270, 270, 250, 261, 266, 262, 266, 262, 249, 274, 244, 245, 229, 259, 256, 251, 287, 267, 228, 262, 241, 246, 265, 240, 237, 259, 236, 247, 291, 286, 261, 262, 256, 260, 273, 280, 273, 271, 257, 244, 246, 274, 249, 258, 215, 262, 301, 228, 254, 251, 255, 283, 248, 233, 249] payload_rows_per_dst=[5422, 5438, 5572, 5558, 5466, 5542, 5638, 5440] dup_rows_per_dst=[2722, 2746, 2702, 2550, 2752, 2588, 2740, 2660] combine_rows_per_src=[5422, 5438, 5572, 5558, 5466, 5542, 5638, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38250 dup_rows=21862 dup_rows_remote=19086 dedup_ratio=0.333588 router_tokens_per_expert=[245, 249, 265, 253, 254, 262, 241, 261, 254, 272, 240, 237, 282, 272, 269, 241, 253, 257, 247, 259, 263, 237, 253, 267, 244, 247, 251, 260, 269, 249, 243, 248, 248, 263, 254, 272, 275, 258, 241, 255, 263, 292, 238, 241, 244, 259, 231, 268, 248, 264, 253, 270, 253, 272, 243, 259, 269, 262, 260, 260, 235, 252, 246, 264, 269, 260, 249, 279, 267, 236, 283, 243, 278, 251, 252, 263, 240, 273, 246, 240, 239, 277, 260, 248, 253, 238, 271, 247, 258, 269, 272, 241, 258, 243, 257, 259, 254, 240, 247, 252, 238, 289, 230, 262, 249, 271, 233, 248, 274, 248, 238, 256, 271, 277, 282, 258, 261, 266, 264, 232, 238, 269, 243, 273, 251, 241, 256, 282] payload_rows_per_dst=[5424, 5444, 5448, 5530, 5606, 5490, 5394, 5338] dup_rows_per_dst=[2776, 2728, 2678, 2794, 3072, 2684, 2560, 2570] combine_rows_per_src=[5424, 5444, 5448, 5530, 5606, 5490, 5394, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38088 dup_rows=21866 dup_rows_remote=18908 dedup_ratio=0.333649 router_tokens_per_expert=[255, 268, 252, 270, 253, 256, 267, 251, 290, 271, 266, 259, 238, 239, 256, 232, 270, 240, 290, 228, 228, 260, 272, 265, 250, 288, 224, 267, 245, 256, 266, 273, 279, 278, 275, 279, 252, 227, 233, 259, 211, 240, 249, 255, 219, 266, 250, 240, 234, 246, 286, 262, 216, 275, 239, 257, 269, 224, 257, 227, 250, 285, 226, 274, 274, 254, 278, 266, 249, 237, 245, 248, 247, 293, 242, 278, 262, 256, 255, 277, 245, 259, 272, 245, 277, 242, 267, 238, 263, 248, 246, 239, 249, 259, 260, 247, 247, 270, 263, 253, 260, 244, 250, 289, 261, 259, 259, 283, 251, 264, 247, 285, 251, 261, 263, 243, 236, 243, 261, 266, 242, 256, 272, 273, 262, 239, 268, 246] payload_rows_per_dst=[5452, 5242, 5610, 5392, 5560, 5282, 5550, 5582] dup_rows_per_dst=[2794, 2402, 2880, 2662, 2874, 2454, 2842, 2958] combine_rows_per_src=[5452, 5242, 5610, 5392, 5560, 5282, 5550, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229712 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157131088 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314032464 weight_movement_bytes=37748736 total_nvlink_bytes=351781200 weight_to_token_ratio=0.120206 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229944 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156918328 ingress_bytes=232185856 link_bytes=232185856 token_bytes=313606712 weight_movement_bytes=75497472 total_nvlink_bytes=389104184 weight_to_token_ratio=0.240739 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224690630e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229808 combine_payload_bytes=156950528 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157180336 ingress_bytes=213573632 link_bytes=213573632 token_bytes=314130864 weight_movement_bytes=56623104 total_nvlink_bytes=370753968 weight_to_token_ratio=0.180253 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046357457e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=229008 combine_payload_bytes=156024832 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156253840 ingress_bytes=212647936 link_bytes=212647936 token_bytes=312278672 weight_movement_bytes=56623104 total_nvlink_bytes=368901776 weight_to_token_ratio=0.181322 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037487893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157712384 dispatch_meta_bytes=229048 combine_payload_bytes=157712384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157941432 ingress_bytes=176586752 link_bytes=176586752 token_bytes=315653816 weight_movement_bytes=18874368 total_nvlink_bytes=334528184 weight_to_token_ratio=0.059795 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691967372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229344 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156901344 ingress_bytes=213295104 link_bytes=213295104 token_bytes=313573344 weight_movement_bytes=56623104 total_nvlink_bytes=370196448 weight_to_token_ratio=0.180574 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043688739e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=227984 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156236432 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312244880 weight_movement_bytes=18874368 total_nvlink_bytes=331119248 weight_to_token_ratio=0.060447 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38196 dup_rows=21866 dup_rows_remote=19082 dedup_ratio=0.333649 router_tokens_per_expert=[254, 236, 257, 247, 259, 247, 260, 281, 270, 239, 269, 234, 258, 265, 260, 257, 233, 264, 272, 257, 251, 288, 239, 271, 256, 269, 251, 276, 238, 256, 252, 236, 281, 242, 250, 224, 242, 260, 222, 253, 254, 257, 251, 250, 267, 266, 246, 203, 245, 256, 269, 254, 266, 259, 268, 266, 272, 247, 250, 251, 245, 265, 252, 261, 272, 249, 219, 253, 246, 269, 273, 258, 239, 279, 236, 282, 259, 268, 253, 269, 260, 209, 252, 244, 255, 232, 279, 280, 264, 264, 269, 264, 268, 264, 260, 265, 257, 250, 252, 251, 256, 276, 238, 238, 260, 248, 276, 294, 255, 258, 276, 272, 247, 260, 272, 269, 231, 254, 266, 235, 261, 250, 246, 259, 239, 259, 262, 252] payload_rows_per_dst=[5468, 5458, 5348, 5470, 5472, 5474, 5572, 5408] dup_rows_per_dst=[2718, 2760, 2588, 2782, 2776, 2784, 2742, 2716] combine_rows_per_src=[5468, 5458, 5348, 5470, 5472, 5474, 5572, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43360 payload_rows_remote=37984 dup_rows=22176 dup_rows_remote=19568 dedup_ratio=0.338379 router_tokens_per_expert=[249, 242, 277, 261, 257, 257, 291, 248, 260, 269, 283, 229, 281, 211, 276, 281, 235, 255, 273, 260, 216, 252, 259, 255, 239, 232, 246, 260, 263, 278, 251, 268, 239, 259, 284, 266, 258, 262, 260, 238, 248, 276, 280, 258, 245, 248, 284, 234, 252, 254, 259, 267, 276, 272, 273, 226, 272, 260, 246, 256, 266, 229, 247, 276, 237, 293, 254, 246, 255, 275, 250, 265, 249, 267, 238, 258, 268, 244, 287, 245, 246, 291, 281, 274, 259, 241, 257, 241, 217, 243, 239, 239, 261, 257, 294, 239, 252, 233, 238, 272, 248, 255, 245, 256, 268, 274, 241, 231, 219, 245, 257, 258, 241, 271, 295, 267, 225, 254, 254, 258, 284, 241, 245, 258, 258, 217, 262, 252] payload_rows_per_dst=[5480, 5370, 5484, 5414, 5470, 5354, 5376, 5412] dup_rows_per_dst=[2864, 2714, 2794, 2848, 2792, 2804, 2608, 2752] combine_rows_per_src=[5480, 5370, 5484, 5414, 5470, 5354, 5376, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38312 dup_rows=21872 dup_rows_remote=19242 dedup_ratio=0.333740 router_tokens_per_expert=[278, 252, 245, 250, 260, 236, 252, 279, 233, 284, 252, 280, 261, 275, 270, 262, 282, 255, 239, 265, 259, 271, 265, 271, 274, 262, 252, 276, 288, 246, 270, 236, 260, 232, 290, 245, 270, 266, 245, 267, 263, 249, 268, 256, 245, 250, 256, 269, 239, 251, 240, 245, 221, 248, 274, 278, 244, 243, 232, 243, 253, 249, 234, 244, 242, 247, 266, 246, 262, 245, 256, 225, 251, 247, 251, 237, 258, 256, 260, 242, 259, 258, 265, 250, 250, 253, 262, 248, 273, 237, 272, 262, 271, 244, 247, 236, 250, 254, 267, 258, 271, 263, 236, 271, 240, 257, 272, 254, 245, 257, 292, 263, 268, 251, 268, 260, 256, 260, 263, 270, 251, 258, 247, 241, 273, 252, 230, 243] payload_rows_per_dst=[5456, 5540, 5474, 5354, 5352, 5450, 5566, 5472] dup_rows_per_dst=[2882, 2882, 2788, 2522, 2630, 2724, 2734, 2710] combine_rows_per_src=[5456, 5540, 5474, 5354, 5352, 5450, 5566, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229112 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679928 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130744 weight_movement_bytes=37748736 total_nvlink_bytes=350879480 weight_to_token_ratio=0.120553 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38366 dup_rows=21834 dup_rows_remote=19198 dedup_ratio=0.333160 router_tokens_per_expert=[266, 256, 295, 237, 248, 225, 256, 246, 244, 233, 258, 297, 252, 252, 227, 268, 256, 275, 249, 234, 280, 249, 258, 236, 264, 216, 248, 224, 270, 236, 261, 230, 253, 284, 245, 257, 251, 266, 251, 221, 266, 248, 250, 225, 240, 228, 257, 273, 273, 257, 278, 245, 245, 288, 261, 273, 242, 284, 274, 244, 247, 272, 299, 243, 251, 247, 247, 277, 266, 276, 263, 247, 274, 249, 272, 268, 261, 252, 241, 250, 248, 274, 266, 240, 256, 298, 256, 274, 283, 281, 251, 256, 240, 273, 243, 259, 273, 213, 244, 233, 271, 253, 292, 247, 262, 256, 256, 228, 244, 232, 232, 252, 277, 284, 281, 213, 266, 270, 240, 240, 277, 242, 260, 255, 256, 253, 298, 243] payload_rows_per_dst=[5390, 5336, 5444, 5504, 5578, 5562, 5372, 5516] dup_rows_per_dst=[2730, 2636, 2586, 2946, 2704, 2834, 2604, 2794] combine_rows_per_src=[5390, 5336, 5444, 5504, 5578, 5562, 5372, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38218 dup_rows=21826 dup_rows_remote=19050 dedup_ratio=0.333038 router_tokens_per_expert=[260, 251, 241, 247, 283, 265, 273, 252, 265, 283, 270, 271, 281, 246, 275, 271, 260, 265, 267, 268, 249, 260, 288, 269, 250, 268, 256, 262, 251, 275, 244, 244, 255, 243, 240, 237, 243, 263, 256, 277, 250, 234, 247, 259, 258, 243, 243, 266, 259, 217, 246, 275, 259, 260, 284, 258, 274, 281, 256, 234, 259, 240, 266, 266, 244, 265, 257, 278, 244, 246, 256, 232, 244, 250, 258, 218, 211, 256, 268, 262, 253, 247, 235, 269, 258, 272, 258, 281, 247, 250, 248, 270, 247, 265, 245, 223, 237, 240, 271, 280, 222, 251, 255, 246, 265, 244, 264, 236, 241, 238, 222, 241, 260, 252, 250, 269, 283, 231, 257, 263, 256, 269, 248, 266, 283, 268, 257, 288] payload_rows_per_dst=[5586, 5514, 5368, 5492, 5354, 5438, 5390, 5568] dup_rows_per_dst=[2882, 2838, 2660, 2776, 2624, 2698, 2516, 2832] combine_rows_per_src=[5586, 5514, 5368, 5492, 5354, 5438, 5390, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38416 dup_rows=21678 dup_rows_remote=18940 dedup_ratio=0.330780 router_tokens_per_expert=[268, 246, 252, 251, 232, 273, 245, 291, 266, 246, 234, 264, 246, 302, 240, 289, 227, 252, 267, 273, 256, 245, 252, 265, 221, 255, 236, 240, 266, 250, 246, 260, 242, 268, 256, 258, 248, 281, 235, 243, 263, 273, 248, 259, 250, 268, 244, 254, 245, 297, 277, 232, 278, 239, 258, 265, 267, 251, 250, 255, 235, 244, 256, 273, 216, 241, 220, 265, 255, 294, 279, 246, 288, 244, 261, 248, 233, 265, 268, 234, 253, 266, 215, 250, 266, 272, 263, 261, 235, 265, 240, 226, 260, 261, 247, 274, 256, 266, 244, 267, 266, 265, 270, 262, 265, 247, 249, 289, 256, 262, 258, 259, 255, 256, 253, 275, 257, 231, 229, 252, 259, 242, 283, 289, 235, 283, 251, 258] payload_rows_per_dst=[5486, 5454, 5442, 5514, 5394, 5518, 5522, 5528] dup_rows_per_dst=[2804, 2568, 2738, 2730, 2720, 2590, 2840, 2688] combine_rows_per_src=[5486, 5454, 5442, 5514, 5394, 5518, 5522, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38366 dup_rows=21712 dup_rows_remote=18846 dedup_ratio=0.331299 router_tokens_per_expert=[253, 238, 263, 261, 267, 279, 241, 252, 275, 255, 271, 254, 245, 273, 275, 260, 244, 268, 246, 235, 253, 267, 284, 245, 241, 259, 253, 243, 259, 258, 228, 237, 247, 294, 257, 258, 265, 261, 225, 270, 276, 248, 249, 245, 246, 267, 237, 230, 245, 258, 268, 252, 263, 253, 241, 270, 230, 254, 244, 248, 266, 263, 232, 240, 237, 276, 241, 281, 263, 267, 272, 275, 247, 251, 284, 286, 260, 269, 267, 246, 239, 266, 255, 274, 261, 261, 254, 254, 253, 242, 256, 274, 255, 264, 275, 273, 213, 242, 270, 237, 254, 272, 230, 249, 267, 250, 263, 258, 258, 256, 257, 271, 241, 273, 273, 254, 263, 233, 244, 251, 294, 238, 265, 244, 233, 246, 245, 262] payload_rows_per_dst=[5458, 5456, 5468, 5426, 5532, 5576, 5460, 5448] dup_rows_per_dst=[2866, 2584, 2682, 2628, 2912, 2736, 2634, 2670] combine_rows_per_src=[5458, 5456, 5468, 5426, 5532, 5576, 5460, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155582464 dispatch_meta_bytes=230208 combine_payload_bytes=155582464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155812672 ingress_bytes=174456832 link_bytes=174456832 token_bytes=311395136 weight_movement_bytes=18874368 total_nvlink_bytes=330269504 weight_to_token_ratio=0.060612 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671559526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38168 dup_rows=21862 dup_rows_remote=18880 dedup_ratio=0.333588 router_tokens_per_expert=[231, 265, 253, 260, 243, 245, 272, 265, 267, 258, 267, 242, 266, 246, 262, 246, 268, 260, 270, 248, 219, 255, 243, 244, 251, 262, 254, 262, 248, 253, 251, 245, 243, 253, 276, 261, 246, 251, 248, 245, 266, 249, 263, 252, 237, 252, 272, 262, 268, 263, 220, 268, 255, 246, 267, 258, 255, 258, 254, 248, 256, 255, 281, 257, 303, 279, 266, 263, 257, 268, 270, 269, 261, 240, 235, 257, 262, 275, 240, 232, 272, 242, 254, 248, 271, 260, 250, 249, 264, 256, 250, 272, 267, 247, 256, 230, 254, 248, 271, 265, 255, 264, 261, 216, 277, 273, 256, 235, 256, 244, 254, 223, 257, 254, 264, 260, 239, 257, 268, 249, 250, 280, 270, 254, 279, 241, 275, 248] payload_rows_per_dst=[5296, 5470, 5586, 5308, 5544, 5356, 5608, 5506] dup_rows_per_dst=[2354, 2596, 3004, 2446, 2854, 2710, 2916, 2982] combine_rows_per_src=[5296, 5470, 5586, 5308, 5544, 5356, 5608, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230256 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157377392 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314524528 weight_movement_bytes=18874368 total_nvlink_bytes=333398896 weight_to_token_ratio=0.060009 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=230216 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157156168 ingress_bytes=175800320 link_bytes=175800320 token_bytes=314082120 weight_movement_bytes=18874368 total_nvlink_bytes=332956488 weight_to_token_ratio=0.060094 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684432167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229072 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770000 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313310928 weight_movement_bytes=18874368 total_nvlink_bytes=332185296 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229424 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581360 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933296 weight_movement_bytes=18874368 total_nvlink_bytes=333807664 weight_to_token_ratio=0.059931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=228848 combine_payload_bytes=157147136 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157375984 ingress_bytes=232644608 link_bytes=232644608 token_bytes=314523120 weight_movement_bytes=75497472 total_nvlink_bytes=390020592 weight_to_token_ratio=0.240038 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229086166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228192 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156564320 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312900448 weight_movement_bytes=18874368 total_nvlink_bytes=331774816 weight_to_token_ratio=0.060321 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38264 dup_rows=21812 dup_rows_remote=19098 dedup_ratio=0.332825 router_tokens_per_expert=[277, 263, 244, 273, 255, 262, 283, 209, 261, 245, 277, 257, 256, 263, 256, 244, 283, 250, 273, 267, 251, 241, 249, 236, 244, 276, 230, 247, 275, 242, 245, 251, 242, 270, 243, 262, 241, 246, 251, 251, 260, 256, 265, 282, 252, 241, 229, 261, 296, 262, 227, 250, 268, 240, 239, 306, 250, 262, 231, 244, 251, 237, 285, 241, 244, 231, 261, 242, 272, 277, 286, 270, 279, 264, 273, 257, 270, 240, 250, 237, 255, 271, 266, 240, 264, 234, 279, 257, 256, 263, 245, 237, 258, 253, 230, 279, 276, 249, 238, 263, 247, 282, 249, 253, 269, 264, 258, 229, 255, 256, 269, 251, 255, 265, 270, 282, 262, 257, 281, 245, 235, 275, 236, 225, 250, 234, 247, 275] payload_rows_per_dst=[5412, 5486, 5494, 5480, 5512, 5460, 5402, 5478] dup_rows_per_dst=[2838, 2634, 2610, 2698, 2794, 2714, 2814, 2710] combine_rows_per_src=[5412, 5486, 5494, 5480, 5512, 5460, 5402, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38226 dup_rows=21852 dup_rows_remote=19010 dedup_ratio=0.333435 router_tokens_per_expert=[264, 270, 239, 277, 249, 258, 280, 266, 246, 255, 250, 274, 253, 254, 278, 258, 233, 249, 253, 253, 274, 242, 243, 266, 249, 288, 224, 261, 265, 269, 245, 271, 268, 239, 285, 255, 265, 234, 258, 278, 278, 253, 259, 242, 269, 269, 246, 241, 262, 240, 284, 256, 253, 269, 246, 258, 257, 263, 265, 264, 271, 276, 241, 266, 277, 226, 284, 246, 246, 216, 215, 259, 253, 238, 240, 237, 242, 254, 243, 215, 235, 260, 256, 304, 250, 262, 247, 256, 255, 258, 248, 237, 278, 265, 262, 279, 242, 257, 242, 299, 275, 251, 261, 230, 266, 242, 282, 245, 249, 280, 269, 260, 214, 243, 276, 254, 262, 242, 261, 254, 256, 230, 238, 244, 254, 266, 275, 240] payload_rows_per_dst=[5504, 5482, 5512, 5498, 5354, 5456, 5458, 5420] dup_rows_per_dst=[2838, 2688, 2766, 2844, 2428, 2848, 2842, 2598] combine_rows_per_src=[5504, 5482, 5512, 5498, 5354, 5456, 5458, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229448 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958792 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313688136 weight_movement_bytes=37748736 total_nvlink_bytes=351436872 weight_to_token_ratio=0.120338 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38138 dup_rows=22002 dup_rows_remote=19346 dedup_ratio=0.335724 router_tokens_per_expert=[259, 259, 241, 261, 252, 263, 278, 238, 257, 254, 259, 262, 247, 258, 235, 251, 251, 246, 234, 228, 245, 257, 274, 229, 255, 274, 235, 259, 251, 246, 266, 276, 230, 260, 277, 277, 269, 290, 280, 259, 262, 257, 254, 260, 253, 259, 275, 236, 263, 284, 258, 264, 279, 245, 259, 262, 260, 250, 249, 243, 275, 252, 285, 261, 251, 263, 257, 270, 264, 280, 281, 248, 269, 224, 273, 251, 249, 273, 252, 258, 250, 259, 252, 241, 224, 260, 248, 249, 273, 270, 263, 238, 238, 241, 263, 234, 260, 265, 268, 265, 247, 239, 237, 254, 218, 275, 237, 256, 240, 249, 261, 248, 254, 270, 257, 258, 250, 248, 217, 239, 259, 268, 274, 254, 261, 262, 257, 268] payload_rows_per_dst=[5410, 5396, 5508, 5490, 5534, 5342, 5384, 5470] dup_rows_per_dst=[2738, 2656, 2888, 2888, 2792, 2664, 2654, 2722] combine_rows_per_src=[5410, 5396, 5508, 5490, 5534, 5342, 5384, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38120 dup_rows=21994 dup_rows_remote=19318 dedup_ratio=0.335602 router_tokens_per_expert=[260, 253, 252, 270, 250, 241, 219, 274, 234, 251, 286, 260, 258, 239, 248, 246, 291, 275, 251, 265, 263, 234, 268, 273, 265, 271, 272, 238, 258, 257, 256, 248, 247, 260, 261, 249, 253, 239, 259, 237, 281, 252, 271, 260, 246, 257, 287, 220, 242, 274, 252, 239, 268, 263, 281, 279, 254, 256, 279, 265, 239, 231, 245, 266, 274, 262, 252, 250, 257, 251, 260, 248, 233, 232, 242, 236, 248, 275, 278, 251, 276, 236, 252, 258, 252, 256, 259, 270, 266, 275, 237, 257, 284, 233, 264, 250, 263, 231, 248, 277, 261, 256, 274, 259, 254, 259, 256, 221, 256, 287, 234, 270, 266, 274, 244, 250, 239, 257, 246, 273, 263, 243, 261, 251, 244, 262, 241, 236] payload_rows_per_dst=[5440, 5542, 5432, 5450, 5422, 5426, 5444, 5386] dup_rows_per_dst=[2642, 2828, 2726, 2816, 2676, 2824, 2768, 2714] combine_rows_per_src=[5440, 5542, 5432, 5450, 5422, 5426, 5444, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38256 dup_rows=21824 dup_rows_remote=19156 dedup_ratio=0.333008 router_tokens_per_expert=[247, 265, 296, 259, 234, 224, 255, 235, 246, 256, 259, 225, 260, 234, 275, 261, 247, 253, 248, 254, 259, 250, 251, 244, 264, 269, 255, 235, 262, 280, 259, 274, 267, 223, 228, 242, 274, 277, 254, 252, 248, 223, 241, 253, 284, 251, 254, 255, 264, 280, 255, 229, 256, 254, 268, 246, 256, 253, 234, 280, 265, 216, 270, 236, 250, 235, 283, 270, 265, 257, 238, 266, 236, 230, 240, 285, 272, 253, 256, 273, 282, 247, 257, 235, 245, 262, 276, 262, 257, 249, 267, 300, 221, 269, 270, 255, 266, 271, 259, 282, 260, 263, 276, 270, 285, 238, 249, 270, 242, 250, 266, 257, 255, 238, 240, 255, 245, 260, 259, 247, 257, 264, 270, 259, 261, 250, 255, 263] payload_rows_per_dst=[5440, 5446, 5314, 5456, 5530, 5514, 5496, 5516] dup_rows_per_dst=[2622, 2762, 2738, 2668, 2688, 2794, 2912, 2640] combine_rows_per_src=[5440, 5446, 5314, 5456, 5530, 5514, 5496, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38368 dup_rows=21664 dup_rows_remote=18908 dedup_ratio=0.330566 router_tokens_per_expert=[271, 250, 261, 273, 236, 251, 256, 252, 269, 262, 246, 273, 235, 235, 257, 252, 269, 290, 240, 265, 259, 243, 264, 246, 258, 248, 266, 233, 241, 279, 258, 272, 268, 253, 264, 272, 259, 275, 246, 259, 245, 262, 246, 274, 278, 251, 226, 252, 237, 287, 249, 230, 234, 242, 269, 257, 277, 237, 253, 248, 271, 251, 268, 258, 231, 259, 245, 243, 258, 265, 275, 228, 249, 263, 263, 265, 261, 278, 243, 274, 264, 231, 242, 254, 251, 292, 235, 269, 255, 241, 237, 256, 246, 221, 259, 274, 278, 257, 252, 263, 268, 260, 265, 253, 250, 256, 268, 253, 274, 252, 258, 256, 260, 249, 275, 225, 232, 255, 248, 247, 251, 273, 280, 278, 242, 239, 258, 258] payload_rows_per_dst=[5430, 5546, 5504, 5476, 5444, 5490, 5504, 5478] dup_rows_per_dst=[2728, 2716, 2756, 2660, 2756, 2564, 2822, 2662] combine_rows_per_src=[5430, 5546, 5504, 5476, 5444, 5490, 5504, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=228944 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156802640 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376336 weight_movement_bytes=18874368 total_nvlink_bytes=332250704 weight_to_token_ratio=0.060229 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38256 dup_rows=21756 dup_rows_remote=19028 dedup_ratio=0.331970 router_tokens_per_expert=[254, 243, 262, 256, 238, 258, 266, 263, 241, 269, 258, 249, 251, 242, 241, 261, 241, 260, 259, 254, 250, 251, 250, 240, 265, 259, 239, 242, 241, 260, 243, 277, 256, 252, 256, 261, 251, 249, 286, 288, 253, 251, 247, 232, 254, 253, 253, 259, 244, 254, 250, 231, 295, 253, 278, 292, 250, 231, 252, 252, 256, 273, 259, 262, 271, 276, 233, 246, 255, 275, 250, 222, 280, 264, 278, 255, 255, 243, 308, 278, 253, 262, 254, 273, 264, 239, 264, 262, 284, 269, 256, 240, 250, 262, 274, 248, 226, 280, 259, 248, 250, 234, 265, 244, 274, 234, 268, 238, 276, 279, 266, 256, 239, 267, 278, 236, 266, 240, 252, 244, 248, 269, 253, 233, 242, 249, 249, 247] payload_rows_per_dst=[5524, 5458, 5572, 5378, 5448, 5450, 5492, 5458] dup_rows_per_dst=[2728, 2642, 2848, 2668, 2912, 2690, 2702, 2566] combine_rows_per_src=[5524, 5458, 5572, 5378, 5448, 5450, 5492, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38252 dup_rows=21784 dup_rows_remote=18914 dedup_ratio=0.332397 router_tokens_per_expert=[267, 253, 261, 264, 251, 256, 269, 266, 241, 247, 289, 252, 251, 234, 248, 249, 244, 245, 239, 254, 284, 250, 261, 273, 255, 256, 284, 240, 236, 274, 243, 248, 270, 262, 244, 245, 275, 229, 289, 259, 262, 247, 230, 267, 263, 250, 259, 245, 230, 234, 260, 237, 252, 270, 252, 254, 279, 233, 235, 238, 256, 260, 228, 251, 249, 252, 271, 272, 285, 293, 267, 248, 272, 258, 240, 271, 229, 257, 236, 261, 261, 264, 260, 253, 266, 267, 270, 271, 256, 244, 250, 244, 277, 298, 241, 272, 225, 252, 243, 249, 228, 260, 258, 231, 261, 262, 245, 244, 260, 282, 242, 238, 249, 241, 294, 275, 244, 246, 272, 260, 256, 269, 273, 278, 289, 256, 250, 232] payload_rows_per_dst=[5224, 5398, 5280, 5502, 5560, 5728, 5560, 5500] dup_rows_per_dst=[2534, 2546, 2446, 2692, 2762, 3070, 2864, 2870] combine_rows_per_src=[5224, 5398, 5280, 5502, 5560, 5728, 5560, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229936 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156443184 ingress_bytes=193961984 link_bytes=193961984 token_bytes=312656432 weight_movement_bytes=37748736 total_nvlink_bytes=350405168 weight_to_token_ratio=0.120736 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858448295e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229752 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156369272 ingress_bytes=193888256 link_bytes=193888256 token_bytes=312508792 weight_movement_bytes=37748736 total_nvlink_bytes=350257528 weight_to_token_ratio=0.120793 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857741870e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229648 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926224 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622800 weight_movement_bytes=18874368 total_nvlink_bytes=332497168 weight_to_token_ratio=0.060182 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229136 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925712 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622288 weight_movement_bytes=18874368 total_nvlink_bytes=332496656 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229104 combine_payload_bytes=157155328 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157384432 ingress_bytes=213778432 link_bytes=213778432 token_bytes=314539760 weight_movement_bytes=56623104 total_nvlink_bytes=371162864 weight_to_token_ratio=0.180019 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048319750e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38334 dup_rows=21758 dup_rows_remote=19114 dedup_ratio=0.332001 router_tokens_per_expert=[271, 268, 249, 262, 265, 257, 252, 246, 269, 273, 264, 276, 266, 253, 225, 255, 221, 255, 263, 236, 256, 271, 244, 245, 253, 253, 276, 264, 272, 255, 268, 278, 278, 242, 275, 239, 268, 249, 295, 270, 273, 262, 235, 252, 240, 251, 248, 241, 262, 250, 253, 241, 259, 250, 237, 246, 256, 239, 236, 260, 237, 277, 257, 251, 243, 264, 283, 210, 240, 246, 260, 251, 254, 264, 251, 268, 264, 246, 230, 243, 250, 230, 275, 228, 262, 254, 270, 238, 239, 272, 262, 255, 275, 244, 240, 250, 257, 295, 263, 261, 298, 242, 248, 227, 259, 272, 277, 261, 243, 267, 266, 255, 236, 273, 238, 229, 255, 296, 271, 256, 254, 264, 256, 251, 252, 276, 284, 235] payload_rows_per_dst=[5494, 5504, 5460, 5364, 5432, 5444, 5570, 5510] dup_rows_per_dst=[2808, 2716, 2776, 2658, 2602, 2644, 2812, 2742] combine_rows_per_src=[5494, 5504, 5460, 5364, 5432, 5444, 5570, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228664 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156908856 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589048 weight_movement_bytes=37748736 total_nvlink_bytes=351337784 weight_to_token_ratio=0.120376 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229792 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245856 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261920 weight_movement_bytes=18874368 total_nvlink_bytes=333136288 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38276 dup_rows=21720 dup_rows_remote=19064 dedup_ratio=0.331421 router_tokens_per_expert=[262, 253, 249, 258, 223, 252, 283, 260, 262, 249, 240, 239, 262, 256, 261, 230, 264, 282, 285, 286, 256, 257, 256, 248, 244, 284, 227, 270, 227, 261, 255, 247, 258, 262, 280, 240, 259, 228, 251, 254, 281, 253, 244, 245, 236, 258, 296, 257, 248, 290, 264, 278, 262, 248, 286, 276, 256, 257, 245, 259, 235, 235, 233, 253, 258, 231, 269, 256, 248, 259, 243, 244, 238, 271, 265, 273, 246, 229, 228, 310, 269, 234, 259, 255, 227, 263, 261, 266, 253, 260, 259, 233, 258, 254, 262, 252, 250, 239, 261, 236, 277, 253, 267, 253, 235, 254, 269, 261, 245, 245, 289, 264, 245, 249, 244, 254, 242, 270, 260, 280, 248, 248, 269, 275, 234, 256, 277, 271] payload_rows_per_dst=[5400, 5482, 5480, 5508, 5432, 5460, 5540, 5514] dup_rows_per_dst=[2678, 2816, 2724, 2742, 2704, 2670, 2656, 2730] combine_rows_per_src=[5400, 5482, 5480, 5508, 5432, 5460, 5540, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38298 dup_rows=21780 dup_rows_remote=19246 dedup_ratio=0.332336 router_tokens_per_expert=[250, 264, 238, 261, 272, 249, 245, 273, 302, 255, 262, 304, 261, 266, 243, 249, 254, 249, 275, 241, 263, 222, 241, 237, 286, 260, 219, 236, 256, 252, 238, 267, 260, 273, 261, 247, 271, 251, 241, 238, 253, 246, 280, 231, 259, 250, 255, 258, 241, 266, 250, 239, 284, 239, 259, 255, 264, 255, 252, 256, 230, 266, 236, 278, 227, 276, 292, 277, 273, 226, 238, 245, 277, 257, 245, 274, 232, 250, 245, 232, 235, 242, 253, 250, 248, 273, 263, 270, 239, 241, 255, 282, 280, 232, 237, 233, 241, 257, 252, 255, 284, 282, 243, 276, 265, 239, 241, 262, 262, 278, 261, 256, 267, 274, 271, 277, 255, 283, 248, 255, 278, 264, 213, 250, 259, 242, 256, 289] payload_rows_per_dst=[5544, 5458, 5444, 5406, 5498, 5452, 5440, 5514] dup_rows_per_dst=[2844, 2534, 2704, 2734, 2634, 2614, 2868, 2848] combine_rows_per_src=[5544, 5458, 5444, 5406, 5498, 5452, 5440, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38406 dup_rows=21632 dup_rows_remote=18946 dedup_ratio=0.330078 router_tokens_per_expert=[246, 244, 272, 250, 266, 253, 254, 260, 256, 260, 251, 223, 229, 258, 271, 278, 254, 240, 250, 266, 221, 289, 229, 244, 253, 264, 255, 283, 241, 228, 253, 268, 260, 274, 264, 284, 261, 249, 238, 265, 260, 252, 247, 238, 294, 272, 254, 253, 262, 300, 242, 262, 246, 290, 260, 217, 240, 235, 261, 246, 252, 262, 253, 258, 220, 238, 259, 257, 241, 267, 285, 252, 249, 230, 260, 257, 261, 253, 292, 271, 273, 278, 244, 257, 249, 219, 252, 262, 257, 268, 257, 232, 282, 271, 257, 251, 266, 276, 251, 242, 254, 255, 250, 275, 258, 236, 214, 229, 226, 259, 263, 254, 270, 249, 249, 258, 266, 247, 249, 276, 266, 265, 246, 273, 281, 264, 263, 277] payload_rows_per_dst=[5494, 5408, 5554, 5494, 5498, 5498, 5364, 5594] dup_rows_per_dst=[2648, 2668, 2776, 2678, 2686, 2720, 2652, 2804] combine_rows_per_src=[5494, 5408, 5554, 5494, 5498, 5498, 5364, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38242 dup_rows=21852 dup_rows_remote=19152 dedup_ratio=0.333435 router_tokens_per_expert=[248, 234, 247, 252, 270, 269, 271, 267, 252, 237, 268, 269, 278, 256, 232, 228, 244, 248, 253, 292, 265, 274, 234, 261, 249, 253, 264, 267, 248, 261, 269, 244, 230, 248, 225, 261, 253, 252, 249, 253, 260, 259, 236, 246, 234, 236, 257, 264, 251, 273, 240, 278, 256, 248, 276, 258, 218, 257, 281, 261, 233, 259, 251, 231, 249, 237, 265, 266, 248, 240, 276, 264, 260, 276, 253, 259, 307, 257, 230, 288, 276, 247, 259, 296, 260, 261, 257, 263, 240, 262, 265, 265, 236, 255, 264, 261, 258, 239, 252, 277, 240, 279, 272, 235, 263, 245, 259, 284, 267, 256, 236, 268, 241, 255, 249, 279, 260, 249, 255, 224, 255, 273, 220, 235, 277, 247, 247, 292] payload_rows_per_dst=[5470, 5488, 5328, 5442, 5518, 5532, 5480, 5426] dup_rows_per_dst=[2686, 2764, 2598, 2700, 2832, 2802, 2780, 2690] combine_rows_per_src=[5470, 5488, 5328, 5442, 5518, 5532, 5480, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229360 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007856 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786352 weight_movement_bytes=37748736 total_nvlink_bytes=351535088 weight_to_token_ratio=0.120301 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38110 dup_rows=21946 dup_rows_remote=19154 dedup_ratio=0.334869 router_tokens_per_expert=[268, 260, 252, 261, 236, 224, 244, 264, 252, 246, 248, 253, 258, 273, 273, 261, 260, 251, 253, 274, 255, 273, 234, 238, 249, 269, 260, 259, 252, 261, 250, 267, 265, 282, 249, 242, 263, 259, 254, 272, 289, 239, 249, 224, 250, 276, 257, 244, 237, 254, 250, 272, 251, 262, 274, 260, 255, 239, 258, 252, 221, 248, 287, 261, 260, 242, 266, 217, 265, 263, 265, 232, 250, 250, 222, 255, 229, 249, 259, 280, 258, 262, 264, 271, 259, 264, 254, 286, 277, 268, 271, 256, 285, 251, 240, 245, 265, 281, 248, 274, 255, 256, 252, 279, 237, 252, 285, 246, 242, 226, 247, 246, 262, 269, 288, 244, 232, 241, 261, 252, 250, 248, 256, 245, 270, 254, 261, 256] payload_rows_per_dst=[5480, 5496, 5472, 5548, 5358, 5572, 5210, 5454] dup_rows_per_dst=[2792, 2714, 2756, 3084, 2650, 2926, 2300, 2724] combine_rows_per_src=[5480, 5496, 5472, 5548, 5358, 5572, 5210, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38086 dup_rows=21998 dup_rows_remote=19356 dedup_ratio=0.335663 router_tokens_per_expert=[256, 242, 252, 251, 245, 249, 276, 257, 232, 260, 265, 223, 247, 255, 234, 275, 280, 238, 274, 253, 254, 253, 247, 271, 251, 263, 251, 274, 243, 265, 267, 264, 246, 249, 251, 256, 263, 246, 267, 259, 271, 223, 259, 240, 250, 271, 245, 251, 269, 251, 229, 244, 248, 239, 247, 284, 281, 248, 262, 285, 283, 275, 244, 254, 261, 230, 252, 258, 234, 274, 247, 244, 252, 254, 254, 253, 263, 238, 265, 239, 276, 261, 253, 269, 264, 232, 238, 268, 265, 292, 265, 256, 255, 261, 280, 252, 250, 251, 257, 232, 212, 259, 261, 267, 252, 270, 264, 238, 269, 268, 260, 252, 261, 255, 223, 269, 265, 272, 226, 249, 244, 291, 279, 266, 278, 259, 254, 253] payload_rows_per_dst=[5412, 5496, 5452, 5468, 5408, 5468, 5420, 5414] dup_rows_per_dst=[2626, 2800, 2642, 2818, 2628, 2906, 2704, 2874] combine_rows_per_src=[5412, 5496, 5452, 5468, 5408, 5468, 5420, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38282 dup_rows=21794 dup_rows_remote=18982 dedup_ratio=0.332550 router_tokens_per_expert=[270, 256, 215, 246, 210, 248, 258, 243, 252, 230, 246, 266, 280, 261, 262, 265, 240, 249, 248, 229, 254, 244, 256, 250, 252, 256, 266, 244, 249, 262, 240, 271, 285, 253, 244, 256, 261, 228, 261, 256, 285, 247, 260, 245, 248, 267, 258, 274, 264, 258, 256, 274, 274, 264, 240, 246, 255, 264, 281, 246, 269, 263, 272, 238, 234, 233, 289, 262, 263, 249, 271, 243, 271, 244, 260, 228, 267, 237, 247, 257, 271, 254, 256, 242, 294, 238, 254, 269, 267, 263, 270, 281, 283, 256, 259, 279, 252, 267, 254, 244, 260, 243, 249, 257, 249, 222, 258, 261, 260, 264, 250, 245, 268, 265, 282, 262, 252, 267, 271, 239, 240, 269, 278, 273, 247, 245, 264, 210] payload_rows_per_dst=[5442, 5444, 5430, 5536, 5436, 5586, 5408, 5460] dup_rows_per_dst=[2574, 2576, 2770, 2804, 2674, 2886, 2698, 2812] combine_rows_per_src=[5442, 5444, 5430, 5536, 5436, 5586, 5408, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=230176 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098784 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313967392 weight_movement_bytes=18874368 total_nvlink_bytes=332841760 weight_to_token_ratio=0.060116 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229408 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157540384 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314851360 weight_movement_bytes=18874368 total_nvlink_bytes=333725728 weight_to_token_ratio=0.059947 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229576 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156868808 ingress_bytes=232136704 link_bytes=232136704 token_bytes=313508040 weight_movement_bytes=75497472 total_nvlink_bytes=389005512 weight_to_token_ratio=0.240815 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224219679e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38366 dup_rows=21848 dup_rows_remote=19228 dedup_ratio=0.333374 router_tokens_per_expert=[278, 241, 279, 236, 252, 258, 275, 266, 237, 243, 256, 259, 249, 266, 232, 263, 250, 260, 289, 272, 269, 247, 267, 253, 256, 253, 289, 252, 287, 242, 247, 231, 260, 279, 272, 266, 258, 274, 259, 252, 251, 244, 263, 257, 257, 254, 271, 247, 238, 244, 249, 236, 267, 248, 251, 250, 281, 259, 241, 264, 255, 239, 260, 223, 254, 215, 282, 279, 246, 266, 232, 269, 270, 257, 299, 253, 259, 277, 248, 257, 250, 253, 223, 237, 262, 219, 272, 256, 247, 233, 243, 229, 267, 248, 274, 258, 247, 256, 264, 238, 240, 252, 257, 247, 286, 264, 224, 262, 262, 295, 258, 269, 217, 258, 249, 247, 237, 266, 264, 245, 276, 274, 237, 257, 270, 260, 263, 270] payload_rows_per_dst=[5446, 5502, 5542, 5370, 5546, 5322, 5428, 5532] dup_rows_per_dst=[2734, 2826, 2786, 2640, 2780, 2620, 2814, 2648] combine_rows_per_src=[5446, 5502, 5542, 5370, 5546, 5322, 5428, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=229056 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156327616 ingress_bytes=174972928 link_bytes=174972928 token_bytes=312426176 weight_movement_bytes=18874368 total_nvlink_bytes=331300544 weight_to_token_ratio=0.060412 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676504504e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229768 combine_payload_bytes=156000256 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156230024 ingress_bytes=212623360 link_bytes=212623360 token_bytes=312230280 weight_movement_bytes=56623104 total_nvlink_bytes=368853384 weight_to_token_ratio=0.181350 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037252418e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229056 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032128 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835200 weight_movement_bytes=37748736 total_nvlink_bytes=351583936 weight_to_token_ratio=0.120282 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38366 dup_rows=21748 dup_rows_remote=19008 dedup_ratio=0.331848 router_tokens_per_expert=[229, 243, 284, 279, 249, 263, 263, 261, 303, 263, 247, 249, 277, 244, 262, 265, 259, 254, 261, 258, 285, 258, 268, 274, 264, 243, 252, 277, 220, 259, 268, 254, 245, 269, 289, 244, 278, 242, 253, 240, 262, 253, 266, 233, 260, 242, 270, 248, 253, 234, 248, 264, 266, 273, 262, 260, 274, 222, 284, 287, 242, 230, 243, 251, 256, 262, 275, 243, 255, 235, 242, 265, 255, 259, 234, 271, 239, 229, 297, 265, 234, 241, 240, 232, 266, 240, 268, 264, 267, 280, 268, 259, 227, 269, 238, 249, 198, 248, 261, 266, 271, 237, 272, 283, 274, 251, 249, 234, 252, 260, 262, 263, 239, 229, 222, 286, 264, 283, 254, 265, 238, 252, 233, 255, 248, 253, 260, 260] payload_rows_per_dst=[5566, 5594, 5508, 5452, 5418, 5442, 5422, 5386] dup_rows_per_dst=[2796, 2714, 2680, 2734, 2746, 2642, 2740, 2696] combine_rows_per_src=[5566, 5594, 5508, 5452, 5418, 5442, 5422, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230376 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157377512 ingress_bytes=213770240 link_bytes=213770240 token_bytes=314524648 weight_movement_bytes=56623104 total_nvlink_bytes=371147752 weight_to_token_ratio=0.180028 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048241258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44012 payload_rows_remote=38486 dup_rows=21524 dup_rows_remote=18732 dedup_ratio=0.328430 router_tokens_per_expert=[268, 285, 281, 259, 280, 254, 227, 240, 239, 257, 259, 274, 255, 258, 250, 261, 240, 279, 240, 270, 265, 282, 235, 281, 272, 262, 253, 263, 226, 259, 269, 263, 251, 242, 277, 229, 260, 235, 248, 277, 223, 246, 238, 262, 254, 259, 250, 219, 276, 250, 267, 262, 241, 263, 251, 270, 247, 250, 256, 256, 270, 261, 272, 276, 252, 275, 252, 258, 239, 282, 288, 255, 261, 281, 260, 233, 258, 257, 253, 230, 271, 260, 242, 259, 235, 245, 232, 257, 265, 274, 245, 282, 266, 267, 280, 230, 249, 236, 235, 277, 230, 238, 263, 263, 278, 223, 273, 258, 245, 253, 233, 267, 252, 252, 259, 273, 250, 251, 247, 228, 259, 254, 254, 250, 242, 255, 251, 282] payload_rows_per_dst=[5538, 5526, 5430, 5610, 5520, 5464, 5464, 5460] dup_rows_per_dst=[2756, 2792, 2510, 2726, 2748, 2756, 2578, 2658] combine_rows_per_src=[5538, 5526, 5430, 5610, 5520, 5464, 5464, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229496 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157376632 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314523768 weight_movement_bytes=18874368 total_nvlink_bytes=333398136 weight_to_token_ratio=0.060009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38330 dup_rows=21732 dup_rows_remote=18932 dedup_ratio=0.331604 router_tokens_per_expert=[266, 253, 272, 241, 254, 264, 256, 243, 262, 226, 266, 267, 251, 274, 253, 247, 251, 246, 249, 253, 246, 242, 229, 276, 245, 238, 242, 224, 245, 261, 245, 278, 268, 265, 255, 228, 224, 271, 268, 246, 244, 264, 258, 272, 247, 277, 265, 254, 272, 258, 256, 240, 254, 244, 277, 266, 256, 235, 252, 248, 233, 260, 251, 274, 257, 256, 247, 257, 254, 292, 271, 248, 261, 253, 249, 267, 232, 239, 276, 278, 242, 293, 272, 261, 277, 270, 237, 261, 264, 263, 285, 274, 298, 261, 230, 272, 224, 249, 293, 243, 263, 265, 229, 261, 277, 275, 246, 251, 249, 257, 246, 267, 295, 257, 268, 251, 233, 235, 244, 268, 249, 258, 238, 254, 246, 212, 267, 254] payload_rows_per_dst=[5484, 5410, 5474, 5442, 5474, 5562, 5492, 5466] dup_rows_per_dst=[2706, 2530, 2738, 2710, 2800, 2958, 2698, 2592] combine_rows_per_src=[5484, 5410, 5474, 5442, 5474, 5562, 5492, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38260 dup_rows=21846 dup_rows_remote=19102 dedup_ratio=0.333344 router_tokens_per_expert=[266, 245, 246, 211, 259, 250, 262, 248, 248, 255, 282, 273, 243, 250, 252, 268, 245, 280, 256, 276, 259, 265, 256, 270, 238, 264, 254, 244, 227, 248, 305, 238, 241, 269, 255, 254, 265, 252, 230, 267, 251, 260, 250, 309, 258, 231, 268, 267, 253, 243, 259, 254, 266, 266, 254, 269, 266, 259, 258, 227, 242, 261, 238, 272, 257, 273, 242, 261, 265, 272, 254, 240, 254, 237, 231, 234, 240, 238, 263, 283, 237, 256, 277, 275, 241, 248, 265, 261, 297, 260, 247, 271, 295, 255, 243, 287, 268, 270, 226, 277, 271, 257, 251, 273, 242, 238, 244, 256, 264, 261, 289, 255, 252, 271, 247, 284, 245, 245, 296, 227, 218, 241, 236, 247, 218, 236, 253, 254] payload_rows_per_dst=[5496, 5432, 5558, 5430, 5408, 5496, 5508, 5362] dup_rows_per_dst=[2620, 2818, 2696, 2744, 2680, 2934, 2776, 2578] combine_rows_per_src=[5496, 5432, 5558, 5430, 5408, 5496, 5508, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44016 payload_rows_remote=38370 dup_rows=21520 dup_rows_remote=18738 dedup_ratio=0.328369 router_tokens_per_expert=[270, 255, 267, 255, 237, 286, 276, 277, 279, 272, 267, 256, 258, 268, 259, 226, 251, 234, 244, 250, 263, 278, 287, 270, 266, 230, 258, 275, 261, 261, 264, 258, 235, 236, 272, 250, 223, 263, 267, 255, 224, 252, 223, 239, 257, 240, 272, 233, 273, 262, 257, 231, 274, 285, 223, 264, 261, 251, 269, 221, 253, 219, 238, 274, 248, 240, 251, 264, 250, 287, 262, 233, 275, 272, 260, 255, 279, 229, 253, 256, 271, 255, 279, 271, 246, 260, 268, 268, 247, 239, 254, 253, 258, 270, 253, 254, 275, 274, 276, 253, 234, 263, 267, 270, 250, 258, 264, 260, 260, 262, 230, 230, 256, 242, 226, 273, 245, 259, 246, 266, 230, 238, 268, 236, 277, 266, 250, 250] payload_rows_per_dst=[5646, 5684, 5324, 5464, 5382, 5552, 5524, 5440] dup_rows_per_dst=[2782, 2958, 2558, 2634, 2504, 2740, 2728, 2616] combine_rows_per_src=[5646, 5684, 5324, 5464, 5382, 5552, 5524, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38198 dup_rows=21916 dup_rows_remote=19212 dedup_ratio=0.334412 router_tokens_per_expert=[235, 245, 272, 266, 238, 267, 236, 262, 246, 244, 281, 234, 215, 270, 277, 257, 253, 230, 282, 254, 247, 250, 251, 229, 259, 266, 302, 269, 249, 241, 278, 265, 260, 280, 237, 256, 257, 227, 257, 266, 240, 244, 264, 263, 265, 254, 257, 236, 277, 273, 254, 271, 249, 238, 247, 242, 235, 261, 268, 230, 243, 231, 251, 235, 237, 243, 260, 240, 281, 285, 267, 255, 253, 234, 249, 245, 297, 250, 274, 301, 241, 275, 231, 260, 255, 291, 248, 225, 266, 263, 244, 266, 253, 277, 250, 237, 253, 240, 243, 232, 249, 269, 264, 276, 276, 253, 272, 282, 261, 272, 247, 244, 246, 255, 256, 273, 263, 237, 255, 238, 256, 307, 265, 252, 278, 243, 260, 260] payload_rows_per_dst=[5470, 5526, 5422, 5404, 5534, 5340, 5500, 5424] dup_rows_per_dst=[2620, 2724, 2704, 2606, 2808, 2824, 2766, 2864] combine_rows_per_src=[5470, 5526, 5422, 5404, 5534, 5340, 5500, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157638656 dispatch_meta_bytes=228872 combine_payload_bytes=157638656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157867528 ingress_bytes=195387392 link_bytes=195387392 token_bytes=315506184 weight_movement_bytes=37748736 total_nvlink_bytes=353254920 weight_to_token_ratio=0.119645 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872105854e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38094 dup_rows=21856 dup_rows_remote=18892 dedup_ratio=0.333496 router_tokens_per_expert=[255, 269, 266, 260, 259, 269, 234, 261, 282, 244, 266, 229, 279, 241, 252, 267, 250, 234, 258, 261, 255, 235, 272, 243, 242, 274, 249, 288, 269, 244, 254, 257, 293, 282, 257, 234, 245, 247, 270, 263, 263, 270, 254, 249, 251, 247, 271, 287, 231, 252, 260, 252, 225, 266, 244, 260, 275, 260, 261, 238, 240, 253, 249, 270, 257, 229, 235, 269, 263, 250, 257, 255, 279, 242, 274, 258, 245, 269, 220, 272, 237, 229, 259, 259, 222, 258, 259, 274, 265, 252, 272, 258, 243, 254, 233, 241, 272, 248, 226, 261, 277, 274, 256, 264, 229, 216, 249, 235, 243, 245, 251, 244, 267, 255, 251, 270, 237, 276, 252, 293, 263, 283, 282, 268, 277, 258, 248, 272] payload_rows_per_dst=[5488, 5348, 5488, 5436, 5428, 5462, 5444, 5586] dup_rows_per_dst=[2774, 2598, 2878, 2580, 2620, 2682, 2760, 2964] combine_rows_per_src=[5488, 5348, 5488, 5436, 5428, 5462, 5444, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229048 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228728 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228408 weight_movement_bytes=18874368 total_nvlink_bytes=333102776 weight_to_token_ratio=0.060066 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229448 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156942408 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655368 weight_movement_bytes=37748736 total_nvlink_bytes=351404104 weight_to_token_ratio=0.120351 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38398 dup_rows=21672 dup_rows_remote=18880 dedup_ratio=0.330688 router_tokens_per_expert=[261, 246, 277, 268, 259, 242, 266, 268, 277, 242, 252, 249, 261, 240, 264, 237, 240, 268, 230, 258, 265, 262, 234, 257, 267, 225, 263, 261, 259, 266, 237, 261, 250, 246, 250, 235, 265, 256, 248, 244, 266, 281, 246, 267, 239, 254, 237, 265, 259, 281, 248, 270, 247, 283, 220, 265, 263, 256, 276, 268, 265, 275, 237, 251, 223, 263, 263, 250, 274, 233, 281, 251, 243, 251, 240, 250, 277, 254, 252, 265, 279, 242, 262, 253, 261, 267, 270, 260, 246, 263, 268, 250, 237, 273, 237, 261, 284, 264, 257, 246, 246, 233, 256, 247, 257, 254, 252, 270, 264, 264, 261, 259, 242, 249, 255, 280, 278, 233, 257, 231, 245, 243, 277, 268, 257, 241, 260, 264] payload_rows_per_dst=[5544, 5434, 5428, 5540, 5424, 5466, 5500, 5528] dup_rows_per_dst=[2674, 2672, 2670, 2788, 2716, 2792, 2728, 2632] combine_rows_per_src=[5544, 5434, 5428, 5540, 5424, 5466, 5500, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=228432 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157391952 ingress_bytes=213786624 link_bytes=213786624 token_bytes=314555472 weight_movement_bytes=56623104 total_nvlink_bytes=371178576 weight_to_token_ratio=0.180010 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048398242e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229640 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688648 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147656 weight_movement_bytes=18874368 total_nvlink_bytes=332022024 weight_to_token_ratio=0.060273 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=227944 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156260968 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312293992 weight_movement_bytes=18874368 total_nvlink_bytes=331168360 weight_to_token_ratio=0.060438 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229112 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157507320 ingress_bytes=195026944 link_bytes=195026944 token_bytes=314785528 weight_movement_bytes=37748736 total_nvlink_bytes=352534264 weight_to_token_ratio=0.119919 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868652218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38286 dup_rows=21770 dup_rows_remote=19062 dedup_ratio=0.332184 router_tokens_per_expert=[278, 262, 272, 249, 254, 265, 244, 262, 262, 256, 254, 264, 233, 241, 223, 244, 261, 270, 231, 257, 248, 219, 245, 252, 250, 268, 250, 273, 242, 273, 247, 226, 271, 263, 272, 241, 250, 260, 265, 253, 265, 233, 294, 273, 282, 239, 250, 222, 274, 256, 284, 264, 237, 251, 250, 261, 266, 277, 276, 242, 280, 259, 270, 258, 263, 261, 275, 248, 242, 231, 255, 244, 255, 270, 247, 243, 273, 266, 261, 297, 241, 240, 256, 258, 220, 253, 256, 262, 243, 245, 274, 271, 232, 217, 268, 247, 262, 239, 265, 266, 236, 240, 262, 265, 239, 252, 266, 241, 255, 270, 273, 263, 253, 248, 277, 249, 272, 251, 281, 244, 271, 274, 271, 271, 235, 253, 266, 231] payload_rows_per_dst=[5446, 5416, 5422, 5536, 5608, 5358, 5480, 5500] dup_rows_per_dst=[2680, 2608, 2594, 2874, 2904, 2586, 2708, 2816] combine_rows_per_src=[5446, 5416, 5422, 5536, 5608, 5358, 5480, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38204 dup_rows=21840 dup_rows_remote=19134 dedup_ratio=0.333252 router_tokens_per_expert=[281, 255, 275, 273, 276, 258, 267, 243, 270, 228, 264, 255, 247, 241, 260, 269, 273, 246, 262, 251, 265, 270, 256, 259, 257, 220, 264, 289, 250, 255, 226, 256, 252, 256, 271, 239, 234, 263, 266, 247, 281, 277, 275, 272, 266, 242, 243, 232, 276, 294, 263, 232, 278, 230, 233, 248, 255, 240, 243, 264, 285, 261, 263, 255, 278, 238, 266, 253, 240, 240, 256, 255, 248, 269, 226, 268, 258, 225, 283, 254, 249, 265, 261, 251, 271, 259, 238, 243, 248, 252, 268, 245, 239, 229, 249, 258, 258, 254, 259, 255, 228, 251, 293, 266, 259, 261, 249, 247, 261, 262, 249, 276, 266, 227, 263, 237, 264, 261, 248, 274, 255, 228, 270, 253, 251, 246, 250, 268] payload_rows_per_dst=[5544, 5492, 5460, 5524, 5440, 5366, 5424, 5446] dup_rows_per_dst=[2780, 2706, 2772, 2716, 2674, 2684, 2832, 2676] combine_rows_per_src=[5544, 5492, 5460, 5524, 5440, 5366, 5424, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229392 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157048848 ingress_bytes=213442560 link_bytes=213442560 token_bytes=313868304 weight_movement_bytes=56623104 total_nvlink_bytes=370491408 weight_to_token_ratio=0.180404 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045101590e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38272 dup_rows=21798 dup_rows_remote=18980 dedup_ratio=0.332611 router_tokens_per_expert=[251, 251, 250, 254, 270, 237, 238, 244, 259, 241, 261, 250, 246, 299, 259, 243, 249, 254, 264, 251, 237, 244, 273, 264, 261, 255, 264, 241, 269, 283, 275, 229, 244, 236, 256, 246, 241, 248, 246, 268, 265, 260, 238, 259, 254, 263, 277, 240, 238, 267, 260, 251, 243, 269, 275, 257, 273, 257, 254, 277, 262, 261, 252, 246, 245, 278, 261, 264, 260, 285, 273, 242, 245, 277, 241, 269, 261, 296, 241, 231, 259, 239, 256, 247, 244, 242, 244, 254, 259, 283, 263, 268, 264, 262, 247, 203, 246, 255, 270, 263, 251, 215, 271, 260, 269, 274, 257, 284, 247, 256, 261, 277, 250, 245, 236, 235, 278, 247, 275, 257, 262, 241, 247, 251, 257, 251, 276, 252] payload_rows_per_dst=[5460, 5508, 5464, 5466, 5528, 5414, 5474, 5424] dup_rows_per_dst=[2646, 2718, 2618, 2818, 2810, 2654, 2838, 2696] combine_rows_per_src=[5460, 5508, 5464, 5466, 5528, 5414, 5474, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38306 dup_rows=21760 dup_rows_remote=19082 dedup_ratio=0.332031 router_tokens_per_expert=[256, 244, 257, 245, 237, 265, 274, 237, 247, 201, 249, 247, 258, 254, 255, 246, 286, 261, 276, 259, 230, 284, 241, 255, 243, 234, 244, 276, 227, 259, 240, 262, 255, 241, 250, 273, 264, 265, 264, 262, 280, 277, 259, 256, 243, 246, 213, 263, 262, 249, 283, 275, 255, 272, 218, 267, 260, 228, 287, 260, 292, 263, 228, 243, 279, 245, 251, 262, 230, 218, 247, 275, 262, 254, 295, 272, 242, 246, 248, 248, 277, 269, 260, 239, 256, 257, 263, 272, 261, 262, 260, 259, 266, 265, 281, 259, 271, 268, 261, 253, 270, 252, 273, 278, 256, 265, 229, 249, 226, 261, 283, 241, 247, 267, 224, 250, 299, 255, 251, 245, 243, 246, 270, 271, 264, 238, 237, 243] payload_rows_per_dst=[5350, 5564, 5488, 5484, 5470, 5592, 5480, 5348] dup_rows_per_dst=[2594, 2590, 2734, 2800, 2678, 2820, 2792, 2752] combine_rows_per_src=[5350, 5564, 5488, 5484, 5470, 5592, 5480, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38396 dup_rows=21664 dup_rows_remote=18922 dedup_ratio=0.330566 router_tokens_per_expert=[238, 262, 245, 252, 261, 253, 267, 272, 270, 238, 242, 282, 259, 250, 260, 254, 246, 266, 268, 266, 286, 267, 270, 245, 251, 296, 278, 248, 248, 270, 258, 267, 283, 257, 256, 263, 271, 237, 251, 257, 277, 257, 263, 255, 237, 256, 264, 261, 244, 264, 230, 247, 254, 252, 253, 257, 245, 239, 245, 254, 262, 254, 260, 252, 258, 261, 262, 236, 261, 215, 261, 253, 263, 271, 275, 258, 266, 258, 249, 262, 258, 251, 240, 250, 244, 224, 251, 285, 263, 257, 227, 254, 241, 246, 233, 242, 268, 265, 236, 265, 260, 264, 245, 238, 238, 253, 243, 269, 266, 259, 244, 259, 250, 260, 264, 236, 244, 283, 255, 251, 268, 274, 239, 264, 267, 253, 255, 266] payload_rows_per_dst=[5476, 5576, 5588, 5438, 5692, 5280, 5492, 5330] dup_rows_per_dst=[2742, 2884, 2694, 2586, 3018, 2652, 2652, 2436] combine_rows_per_src=[5476, 5576, 5588, 5438, 5692, 5280, 5492, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38252 dup_rows=21712 dup_rows_remote=18798 dedup_ratio=0.331299 router_tokens_per_expert=[244, 292, 274, 247, 247, 250, 243, 268, 271, 218, 269, 260, 275, 239, 230, 254, 277, 285, 229, 263, 265, 238, 245, 223, 252, 271, 253, 254, 252, 243, 247, 273, 268, 257, 267, 282, 256, 271, 249, 276, 251, 265, 270, 265, 274, 273, 265, 254, 257, 254, 251, 270, 273, 248, 254, 265, 235, 244, 245, 271, 251, 244, 248, 275, 275, 227, 236, 243, 221, 263, 271, 254, 212, 246, 262, 274, 296, 233, 264, 241, 234, 300, 242, 264, 267, 275, 239, 244, 287, 274, 279, 274, 264, 260, 253, 257, 240, 252, 293, 245, 256, 243, 245, 261, 233, 265, 240, 256, 230, 262, 264, 269, 263, 248, 248, 233, 264, 247, 245, 263, 250, 278, 246, 254, 240, 251, 248, 226] payload_rows_per_dst=[5504, 5444, 5572, 5518, 5344, 5614, 5410, 5418] dup_rows_per_dst=[2658, 2696, 2914, 2652, 2692, 2812, 2698, 2590] combine_rows_per_src=[5504, 5444, 5572, 5518, 5344, 5614, 5410, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229352 combine_payload_bytes=156483584 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156712936 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196520 weight_movement_bytes=37748736 total_nvlink_bytes=350945256 weight_to_token_ratio=0.120527 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=37986 dup_rows=21896 dup_rows_remote=18762 dedup_ratio=0.334106 router_tokens_per_expert=[275, 245, 268, 255, 240, 272, 251, 260, 262, 237, 243, 241, 244, 250, 254, 225, 258, 272, 244, 250, 264, 275, 282, 241, 255, 242, 283, 241, 283, 264, 255, 270, 284, 249, 242, 270, 245, 246, 242, 270, 282, 267, 257, 226, 254, 272, 280, 251, 251, 261, 264, 242, 248, 271, 261, 260, 266, 261, 274, 264, 241, 276, 266, 248, 232, 262, 277, 262, 264, 227, 250, 246, 263, 251, 248, 241, 209, 251, 242, 224, 273, 235, 249, 263, 248, 260, 248, 280, 258, 287, 250, 258, 258, 261, 265, 246, 242, 272, 232, 274, 251, 275, 249, 245, 264, 243, 242, 262, 228, 257, 256, 260, 244, 241, 266, 240, 286, 267, 261, 246, 265, 257, 262, 262, 273, 266, 245, 255] payload_rows_per_dst=[5440, 5438, 5254, 5486, 5614, 5260, 5494, 5654] dup_rows_per_dst=[2604, 2868, 2480, 2822, 2824, 2544, 2620, 3134] combine_rows_per_src=[5440, 5438, 5254, 5486, 5614, 5260, 5494, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38232 dup_rows=21816 dup_rows_remote=19142 dedup_ratio=0.332886 router_tokens_per_expert=[240, 260, 260, 242, 242, 273, 249, 237, 256, 229, 258, 207, 274, 295, 263, 255, 273, 264, 284, 235, 272, 250, 243, 245, 244, 257, 257, 263, 267, 262, 267, 259, 264, 240, 259, 266, 253, 256, 247, 290, 246, 250, 248, 228, 260, 290, 263, 230, 262, 258, 253, 253, 266, 243, 244, 257, 270, 261, 270, 248, 239, 278, 257, 247, 254, 235, 248, 252, 248, 277, 243, 278, 244, 243, 271, 247, 255, 249, 236, 258, 262, 260, 268, 254, 254, 246, 247, 247, 248, 250, 245, 276, 237, 271, 258, 258, 263, 273, 257, 270, 257, 242, 293, 252, 274, 273, 286, 257, 257, 271, 264, 247, 247, 244, 235, 254, 252, 217, 282, 299, 258, 245, 224, 276, 266, 257, 243, 236] payload_rows_per_dst=[5424, 5472, 5480, 5394, 5466, 5488, 5618, 5378] dup_rows_per_dst=[2656, 2812, 2700, 2818, 2610, 2674, 2854, 2692] combine_rows_per_src=[5424, 5472, 5480, 5394, 5466, 5488, 5618, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229008 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156991120 ingress_bytes=213385216 link_bytes=213385216 token_bytes=313753232 weight_movement_bytes=56623104 total_nvlink_bytes=370376336 weight_to_token_ratio=0.180470 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044552148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229552 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130928 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314032304 weight_movement_bytes=37748736 total_nvlink_bytes=351781040 weight_to_token_ratio=0.120207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229272 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157499288 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769304 weight_movement_bytes=56623104 total_nvlink_bytes=371392408 weight_to_token_ratio=0.179888 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228200 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156908392 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313588584 weight_movement_bytes=37748736 total_nvlink_bytes=351337320 weight_to_token_ratio=0.120377 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=226992 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155817648 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311408304 weight_movement_bytes=18874368 total_nvlink_bytes=330282672 weight_to_token_ratio=0.060610 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38334 dup_rows=21780 dup_rows_remote=19188 dedup_ratio=0.332336 router_tokens_per_expert=[275, 278, 274, 259, 255, 251, 234, 229, 237, 264, 248, 243, 269, 250, 245, 261, 251, 260, 260, 265, 281, 256, 271, 265, 251, 249, 271, 254, 264, 292, 255, 264, 242, 283, 241, 255, 253, 245, 253, 237, 241, 235, 253, 259, 254, 285, 255, 255, 243, 257, 225, 285, 274, 246, 263, 261, 286, 263, 252, 266, 253, 263, 254, 260, 242, 266, 249, 233, 267, 262, 260, 283, 255, 258, 279, 273, 250, 307, 251, 264, 227, 250, 253, 246, 261, 250, 267, 255, 248, 266, 253, 240, 246, 222, 243, 263, 246, 240, 229, 287, 238, 267, 244, 266, 198, 259, 237, 269, 234, 257, 270, 266, 258, 274, 248, 274, 263, 221, 239, 270, 263, 258, 276, 258, 229, 260, 228, 275] payload_rows_per_dst=[5438, 5566, 5480, 5480, 5512, 5344, 5422, 5514] dup_rows_per_dst=[2706, 2852, 2612, 2822, 2886, 2636, 2592, 2674] combine_rows_per_src=[5438, 5566, 5480, 5480, 5512, 5344, 5422, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229496 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827768 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313426040 weight_movement_bytes=37748736 total_nvlink_bytes=351174776 weight_to_token_ratio=0.120439 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38332 dup_rows=21832 dup_rows_remote=19198 dedup_ratio=0.333130 router_tokens_per_expert=[245, 240, 245, 281, 263, 265, 279, 273, 267, 228, 243, 257, 268, 254, 238, 267, 263, 233, 253, 259, 244, 259, 247, 226, 255, 262, 258, 256, 243, 225, 252, 268, 278, 237, 265, 292, 246, 268, 270, 262, 270, 270, 236, 253, 251, 275, 265, 263, 255, 240, 248, 252, 269, 266, 254, 257, 227, 254, 266, 267, 249, 238, 263, 260, 251, 288, 263, 251, 248, 225, 265, 263, 246, 259, 271, 289, 246, 278, 264, 251, 246, 276, 267, 248, 265, 250, 248, 231, 257, 241, 244, 270, 249, 250, 253, 241, 259, 257, 240, 243, 243, 230, 265, 253, 271, 263, 239, 248, 251, 271, 269, 245, 266, 272, 256, 286, 276, 243, 266, 261, 260, 243, 235, 262, 260, 244, 255, 260] payload_rows_per_dst=[5468, 5372, 5546, 5454, 5516, 5460, 5438, 5450] dup_rows_per_dst=[2758, 2634, 2856, 2676, 2800, 2612, 2656, 2840] combine_rows_per_src=[5468, 5372, 5546, 5454, 5516, 5460, 5438, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=230088 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157246152 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314262216 weight_movement_bytes=18874368 total_nvlink_bytes=333136584 weight_to_token_ratio=0.060059 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38334 dup_rows=21826 dup_rows_remote=19206 dedup_ratio=0.333038 router_tokens_per_expert=[249, 294, 234, 252, 260, 282, 270, 279, 269, 273, 249, 264, 274, 261, 259, 258, 254, 253, 273, 256, 242, 274, 248, 254, 276, 232, 253, 263, 245, 268, 245, 256, 276, 267, 274, 265, 256, 240, 239, 233, 274, 273, 221, 262, 262, 270, 258, 257, 249, 220, 262, 278, 255, 230, 258, 230, 245, 254, 257, 238, 234, 263, 268, 257, 264, 225, 247, 275, 233, 236, 245, 247, 266, 270, 252, 256, 246, 240, 252, 257, 267, 235, 259, 292, 224, 245, 232, 268, 276, 248, 275, 251, 250, 262, 241, 249, 299, 261, 251, 250, 258, 237, 252, 273, 242, 255, 276, 253, 282, 263, 242, 234, 275, 243, 263, 229, 258, 258, 271, 253, 267, 230, 269, 303, 244, 235, 276, 237] payload_rows_per_dst=[5566, 5454, 5492, 5376, 5410, 5426, 5454, 5532] dup_rows_per_dst=[2888, 2730, 2762, 2620, 2612, 2722, 2802, 2690] combine_rows_per_src=[5566, 5454, 5492, 5376, 5410, 5426, 5454, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38250 dup_rows=21886 dup_rows_remote=19170 dedup_ratio=0.333954 router_tokens_per_expert=[250, 243, 228, 259, 258, 254, 245, 280, 268, 240, 241, 228, 269, 238, 254, 264, 248, 238, 266, 249, 289, 241, 275, 238, 241, 253, 250, 255, 248, 247, 283, 270, 291, 277, 263, 270, 275, 253, 260, 277, 250, 275, 257, 227, 274, 246, 260, 268, 259, 270, 288, 289, 278, 224, 262, 236, 246, 244, 276, 244, 260, 256, 240, 256, 245, 257, 257, 257, 257, 247, 277, 258, 248, 259, 243, 260, 261, 257, 239, 236, 245, 261, 263, 262, 245, 255, 275, 254, 283, 252, 250, 279, 246, 238, 285, 267, 238, 253, 252, 271, 263, 259, 270, 249, 236, 234, 246, 265, 246, 237, 283, 274, 289, 284, 241, 258, 259, 244, 269, 243, 250, 231, 236, 236, 227, 249, 238, 259] payload_rows_per_dst=[5432, 5468, 5500, 5442, 5400, 5570, 5460, 5378] dup_rows_per_dst=[2606, 2714, 2946, 2814, 2716, 2750, 2692, 2648] combine_rows_per_src=[5432, 5468, 5500, 5442, 5400, 5570, 5460, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=230120 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237992 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245864 weight_movement_bytes=18874368 total_nvlink_bytes=333120232 weight_to_token_ratio=0.060062 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38262 dup_rows=21810 dup_rows_remote=19162 dedup_ratio=0.332794 router_tokens_per_expert=[241, 236, 229, 272, 271, 244, 258, 257, 248, 283, 242, 238, 270, 262, 270, 241, 274, 240, 240, 244, 257, 232, 268, 252, 255, 227, 249, 261, 265, 263, 233, 273, 249, 291, 247, 271, 257, 286, 225, 242, 247, 234, 264, 252, 243, 271, 236, 241, 249, 238, 237, 257, 252, 251, 255, 262, 228, 271, 269, 269, 255, 245, 269, 252, 252, 263, 246, 265, 268, 246, 245, 261, 261, 260, 265, 252, 250, 259, 280, 275, 254, 254, 263, 249, 288, 282, 245, 274, 249, 274, 249, 266, 226, 255, 247, 255, 255, 258, 254, 255, 258, 261, 260, 264, 255, 263, 296, 253, 246, 234, 251, 253, 267, 265, 226, 245, 239, 287, 265, 269, 251, 279, 260, 241, 271, 259, 274, 266] payload_rows_per_dst=[5426, 5396, 5464, 5458, 5464, 5508, 5534, 5476] dup_rows_per_dst=[2698, 2670, 2648, 2660, 2832, 2752, 2698, 2852] combine_rows_per_src=[5426, 5396, 5464, 5458, 5464, 5508, 5534, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=37968 dup_rows=21932 dup_rows_remote=18984 dedup_ratio=0.334656 router_tokens_per_expert=[264, 266, 246, 245, 279, 260, 246, 268, 224, 243, 242, 281, 264, 246, 254, 248, 277, 258, 241, 253, 241, 283, 277, 268, 272, 263, 248, 266, 281, 250, 236, 231, 250, 284, 234, 272, 244, 277, 235, 277, 253, 266, 274, 251, 276, 235, 259, 262, 221, 273, 255, 253, 243, 274, 250, 262, 269, 252, 264, 246, 256, 260, 261, 278, 247, 254, 238, 271, 266, 257, 220, 253, 268, 225, 283, 246, 242, 245, 257, 239, 262, 263, 266, 251, 245, 247, 257, 265, 224, 293, 255, 297, 294, 250, 232, 258, 235, 237, 239, 250, 259, 268, 270, 243, 264, 287, 238, 244, 263, 259, 251, 256, 270, 253, 254, 268, 222, 265, 266, 236, 276, 227, 267, 251, 236, 251, 254, 252] payload_rows_per_dst=[5636, 5604, 5540, 5468, 5472, 5188, 5344, 5352] dup_rows_per_dst=[2948, 2850, 2758, 2766, 2832, 2536, 2618, 2624] combine_rows_per_src=[5636, 5604, 5540, 5468, 5472, 5188, 5344, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38288 dup_rows=21796 dup_rows_remote=19126 dedup_ratio=0.332581 router_tokens_per_expert=[258, 282, 267, 277, 256, 263, 250, 261, 238, 254, 266, 232, 280, 253, 239, 283, 256, 258, 254, 262, 268, 282, 242, 214, 239, 261, 247, 273, 265, 255, 224, 238, 242, 240, 266, 256, 234, 262, 282, 255, 226, 273, 251, 280, 265, 283, 269, 269, 243, 259, 243, 244, 259, 221, 266, 268, 248, 260, 277, 258, 259, 263, 255, 251, 261, 290, 271, 272, 235, 267, 228, 237, 247, 260, 251, 253, 222, 245, 247, 247, 264, 265, 275, 246, 240, 290, 280, 277, 263, 265, 242, 267, 283, 258, 284, 243, 236, 268, 254, 257, 261, 221, 260, 263, 235, 256, 231, 254, 250, 264, 255, 261, 227, 255, 253, 264, 266, 259, 252, 246, 236, 230, 270, 253, 253, 271, 251, 257] payload_rows_per_dst=[5550, 5456, 5494, 5444, 5158, 5650, 5536, 5452] dup_rows_per_dst=[2768, 2620, 2806, 2618, 2332, 3068, 2914, 2670] combine_rows_per_src=[5550, 5456, 5494, 5444, 5158, 5650, 5536, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38356 dup_rows=21770 dup_rows_remote=19118 dedup_ratio=0.332184 router_tokens_per_expert=[247, 260, 244, 244, 259, 249, 253, 218, 239, 276, 257, 242, 261, 278, 267, 254, 249, 251, 257, 254, 238, 280, 251, 245, 279, 272, 237, 257, 257, 265, 239, 241, 255, 235, 238, 291, 272, 266, 257, 275, 268, 263, 265, 240, 278, 264, 267, 259, 232, 271, 260, 257, 245, 289, 254, 234, 249, 252, 264, 266, 267, 265, 252, 266, 235, 271, 271, 251, 232, 241, 230, 243, 251, 260, 254, 300, 266, 269, 263, 251, 262, 228, 267, 219, 264, 236, 277, 277, 249, 259, 256, 244, 243, 271, 241, 238, 288, 235, 269, 268, 246, 290, 225, 266, 239, 252, 252, 247, 264, 239, 256, 229, 235, 268, 226, 274, 236, 256, 290, 272, 251, 259, 276, 274, 279, 258, 252, 242] payload_rows_per_dst=[5434, 5396, 5574, 5498, 5472, 5410, 5508, 5474] dup_rows_per_dst=[2662, 2748, 2812, 2748, 2704, 2652, 2622, 2822] combine_rows_per_src=[5434, 5396, 5574, 5498, 5472, 5410, 5508, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=230160 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157246224 ingress_bytes=194764800 link_bytes=194764800 token_bytes=314262288 weight_movement_bytes=37748736 total_nvlink_bytes=352011024 weight_to_token_ratio=0.120119 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866140483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229680 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901680 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573680 weight_movement_bytes=18874368 total_nvlink_bytes=332448048 weight_to_token_ratio=0.060191 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229696 combine_payload_bytes=156721152 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156950848 ingress_bytes=194469888 link_bytes=194469888 token_bytes=313672000 weight_movement_bytes=37748736 total_nvlink_bytes=351420736 weight_to_token_ratio=0.120345 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863314782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38176 dup_rows=21936 dup_rows_remote=19290 dedup_ratio=0.334717 router_tokens_per_expert=[245, 274, 243, 235, 241, 268, 241, 261, 231, 254, 277, 242, 279, 262, 270, 234, 285, 253, 280, 297, 247, 275, 248, 268, 262, 246, 274, 261, 270, 261, 256, 272, 277, 239, 251, 254, 277, 262, 245, 256, 273, 256, 285, 250, 222, 254, 235, 260, 255, 286, 281, 255, 257, 215, 263, 240, 250, 259, 258, 273, 281, 275, 249, 267, 271, 278, 252, 270, 268, 235, 255, 246, 241, 248, 293, 235, 261, 256, 255, 244, 236, 243, 272, 241, 253, 268, 254, 253, 236, 255, 253, 261, 227, 238, 254, 250, 259, 261, 251, 217, 279, 265, 244, 242, 249, 242, 250, 256, 254, 257, 230, 279, 267, 263, 268, 235, 240, 240, 265, 253, 248, 269, 236, 252, 257, 268, 215, 283] payload_rows_per_dst=[5426, 5488, 5488, 5516, 5410, 5472, 5424, 5376] dup_rows_per_dst=[2688, 2844, 2704, 2812, 2806, 2694, 2646, 2742] combine_rows_per_src=[5426, 5488, 5488, 5516, 5410, 5472, 5424, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=155516928 dispatch_meta_bytes=227808 combine_payload_bytes=155516928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155744736 ingress_bytes=174391296 link_bytes=174391296 token_bytes=311261664 weight_movement_bytes=18874368 total_nvlink_bytes=330136032 weight_to_token_ratio=0.060638 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670931592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229656 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057304 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884952 weight_movement_bytes=37748736 total_nvlink_bytes=351633688 weight_to_token_ratio=0.120263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229896 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157336072 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314442248 weight_movement_bytes=18874368 total_nvlink_bytes=333316616 weight_to_token_ratio=0.060025 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38186 dup_rows=21850 dup_rows_remote=19222 dedup_ratio=0.333405 router_tokens_per_expert=[259, 246, 275, 273, 262, 262, 236, 255, 266, 256, 259, 261, 260, 251, 227, 260, 246, 252, 227, 247, 241, 273, 250, 241, 256, 259, 270, 241, 274, 278, 247, 262, 249, 224, 260, 251, 234, 267, 233, 266, 254, 277, 263, 244, 263, 244, 234, 252, 275, 253, 241, 264, 248, 254, 264, 269, 265, 252, 277, 249, 242, 248, 251, 239, 247, 246, 262, 250, 250, 249, 273, 229, 280, 270, 253, 256, 254, 261, 259, 263, 255, 285, 250, 251, 245, 258, 267, 260, 269, 254, 240, 212, 243, 285, 254, 245, 260, 261, 244, 265, 280, 246, 265, 252, 264, 272, 270, 288, 236, 248, 276, 241, 255, 253, 247, 282, 291, 257, 238, 252, 230, 254, 246, 268, 267, 291, 252, 264] payload_rows_per_dst=[5506, 5500, 5416, 5446, 5422, 5450, 5448, 5498] dup_rows_per_dst=[2710, 2628, 2614, 2736, 2782, 2696, 2888, 2796] combine_rows_per_src=[5506, 5500, 5416, 5446, 5422, 5450, 5448, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229864 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598760 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967656 weight_movement_bytes=18874368 total_nvlink_bytes=331842024 weight_to_token_ratio=0.060308 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38224 dup_rows=21864 dup_rows_remote=19096 dedup_ratio=0.333618 router_tokens_per_expert=[267, 279, 260, 256, 229, 259, 244, 241, 270, 260, 248, 266, 292, 273, 256, 220, 261, 293, 278, 268, 223, 250, 248, 270, 212, 261, 280, 271, 244, 251, 254, 256, 274, 247, 263, 259, 236, 241, 249, 278, 248, 220, 250, 240, 249, 259, 263, 266, 232, 278, 247, 226, 271, 269, 247, 260, 281, 286, 245, 230, 259, 259, 259, 259, 258, 262, 280, 272, 244, 239, 280, 266, 260, 261, 257, 262, 257, 244, 271, 265, 236, 260, 247, 251, 266, 271, 277, 271, 292, 262, 248, 250, 257, 243, 227, 262, 234, 282, 275, 232, 233, 275, 261, 286, 239, 241, 246, 258, 265, 251, 245, 239, 265, 260, 263, 244, 237, 253, 244, 253, 257, 241, 235, 271, 245, 231, 260, 259] payload_rows_per_dst=[5470, 5448, 5382, 5448, 5532, 5464, 5480, 5448] dup_rows_per_dst=[2770, 2792, 2702, 2768, 2824, 2776, 2644, 2588] combine_rows_per_src=[5470, 5448, 5382, 5448, 5532, 5464, 5480, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229632 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156639488 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313049344 weight_movement_bytes=37748736 total_nvlink_bytes=350798080 weight_to_token_ratio=0.120584 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38300 dup_rows=21722 dup_rows_remote=18976 dedup_ratio=0.331451 router_tokens_per_expert=[270, 250, 249, 250, 286, 237, 290, 271, 229, 259, 250, 248, 249, 239, 249, 253, 253, 272, 251, 250, 249, 273, 257, 244, 267, 275, 248, 229, 286, 259, 247, 288, 261, 266, 231, 273, 251, 236, 256, 257, 258, 241, 268, 250, 296, 251, 297, 238, 275, 256, 245, 249, 249, 253, 258, 258, 277, 258, 245, 271, 265, 254, 279, 254, 277, 261, 269, 257, 259, 255, 263, 264, 274, 238, 261, 257, 254, 239, 265, 264, 246, 251, 282, 252, 251, 250, 230, 224, 264, 222, 241, 234, 262, 233, 264, 262, 254, 256, 272, 294, 224, 263, 225, 239, 247, 248, 235, 272, 307, 254, 259, 235, 255, 270, 245, 239, 240, 260, 250, 266, 233, 239, 252, 263, 277, 267, 250, 250] payload_rows_per_dst=[5508, 5484, 5514, 5476, 5490, 5406, 5488, 5448] dup_rows_per_dst=[2650, 2812, 2746, 2816, 2824, 2530, 2680, 2664] combine_rows_per_src=[5508, 5484, 5514, 5476, 5490, 5406, 5488, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38250 dup_rows=21816 dup_rows_remote=19076 dedup_ratio=0.332886 router_tokens_per_expert=[272, 246, 261, 229, 260, 232, 252, 246, 265, 264, 272, 260, 266, 254, 267, 270, 273, 288, 259, 238, 254, 246, 243, 243, 234, 235, 270, 261, 270, 235, 230, 261, 240, 269, 252, 269, 274, 259, 258, 253, 274, 254, 270, 270, 244, 260, 239, 248, 254, 245, 241, 250, 233, 255, 281, 250, 247, 236, 274, 266, 266, 270, 282, 253, 272, 246, 258, 244, 236, 244, 266, 232, 283, 255, 268, 251, 271, 262, 260, 257, 241, 264, 232, 242, 245, 246, 252, 259, 257, 252, 250, 247, 263, 250, 269, 262, 254, 278, 239, 274, 235, 260, 259, 266, 251, 268, 245, 261, 276, 255, 254, 257, 285, 227, 286, 239, 261, 267, 267, 261, 235, 256, 239, 254, 252, 269, 245, 265] payload_rows_per_dst=[5474, 5446, 5516, 5452, 5470, 5420, 5518, 5424] dup_rows_per_dst=[2758, 2634, 2750, 2754, 2740, 2642, 2746, 2792] combine_rows_per_src=[5474, 5446, 5516, 5452, 5470, 5420, 5518, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38312 dup_rows=21598 dup_rows_remote=18740 dedup_ratio=0.329559 router_tokens_per_expert=[294, 253, 236, 276, 242, 223, 282, 264, 277, 238, 276, 250, 259, 240, 250, 241, 251, 230, 263, 283, 241, 256, 268, 247, 269, 234, 244, 289, 240, 234, 268, 267, 260, 282, 279, 249, 266, 253, 272, 244, 259, 279, 232, 263, 238, 257, 254, 254, 242, 270, 261, 253, 282, 258, 258, 245, 266, 240, 263, 275, 242, 247, 265, 264, 243, 235, 259, 289, 246, 244, 244, 231, 288, 256, 250, 266, 275, 265, 269, 231, 260, 275, 268, 272, 243, 256, 244, 261, 254, 252, 258, 260, 272, 255, 258, 275, 262, 228, 253, 249, 248, 260, 238, 238, 265, 247, 255, 265, 248, 260, 233, 239, 269, 238, 248, 251, 273, 264, 236, 227, 229, 275, 268, 242, 251, 266, 262, 270] payload_rows_per_dst=[5328, 5238, 5522, 5396, 5522, 5788, 5518, 5626] dup_rows_per_dst=[2546, 2396, 2616, 2526, 2798, 3072, 2786, 2858] combine_rows_per_src=[5328, 5238, 5522, 5396, 5522, 5788, 5518, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38192 dup_rows=21896 dup_rows_remote=19134 dedup_ratio=0.334106 router_tokens_per_expert=[232, 248, 273, 293, 248, 260, 248, 256, 267, 247, 255, 276, 252, 237, 259, 254, 241, 260, 273, 249, 229, 282, 268, 254, 248, 273, 276, 245, 281, 225, 243, 243, 274, 238, 267, 273, 262, 254, 239, 251, 283, 266, 250, 266, 256, 251, 247, 258, 272, 230, 271, 235, 245, 258, 245, 256, 272, 260, 290, 247, 256, 271, 249, 271, 223, 274, 243, 246, 266, 267, 253, 259, 267, 260, 262, 245, 263, 231, 263, 216, 246, 261, 247, 253, 262, 267, 267, 237, 258, 268, 283, 258, 282, 241, 270, 260, 263, 247, 275, 257, 264, 235, 234, 249, 248, 230, 244, 262, 261, 239, 265, 278, 284, 224, 249, 275, 261, 269, 245, 254, 241, 240, 225, 261, 246, 277, 263, 247] payload_rows_per_dst=[5448, 5420, 5696, 5538, 5436, 5276, 5428, 5398] dup_rows_per_dst=[2762, 2782, 2974, 2696, 2640, 2644, 2674, 2724] combine_rows_per_src=[5448, 5420, 5696, 5538, 5436, 5276, 5428, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38250 dup_rows=21846 dup_rows_remote=19152 dedup_ratio=0.333344 router_tokens_per_expert=[236, 268, 272, 233, 282, 232, 280, 272, 257, 246, 255, 268, 273, 270, 266, 237, 256, 284, 264, 251, 271, 262, 276, 275, 260, 265, 269, 260, 262, 253, 254, 279, 254, 251, 261, 253, 254, 226, 249, 241, 262, 265, 247, 262, 290, 250, 234, 246, 238, 247, 261, 242, 255, 241, 247, 269, 246, 258, 263, 245, 268, 257, 263, 254, 228, 245, 244, 270, 260, 253, 278, 254, 262, 272, 243, 256, 256, 254, 247, 258, 241, 263, 242, 248, 249, 256, 270, 244, 221, 249, 252, 286, 274, 260, 269, 243, 219, 248, 252, 252, 260, 256, 239, 254, 294, 284, 248, 266, 245, 266, 248, 223, 257, 256, 245, 249, 274, 263, 255, 243, 234, 261, 244, 248, 255, 252, 266, 278] payload_rows_per_dst=[5562, 5580, 5398, 5460, 5436, 5440, 5436, 5378] dup_rows_per_dst=[2732, 2902, 2692, 2648, 2724, 2694, 2672, 2782] combine_rows_per_src=[5562, 5580, 5398, 5460, 5436, 5440, 5436, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229280 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156794784 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360288 weight_movement_bytes=18874368 total_nvlink_bytes=332234656 weight_to_token_ratio=0.060232 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229104 combine_payload_bytes=156876800 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157105904 ingress_bytes=232374272 link_bytes=232374272 token_bytes=313982704 weight_movement_bytes=75497472 total_nvlink_bytes=389480176 weight_to_token_ratio=0.240451 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226495939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229304 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156901304 ingress_bytes=213295104 link_bytes=213295104 token_bytes=313573304 weight_movement_bytes=56623104 total_nvlink_bytes=370196408 weight_to_token_ratio=0.180574 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043688739e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43960 payload_rows_remote=38440 dup_rows=21576 dup_rows_remote=18822 dedup_ratio=0.329224 router_tokens_per_expert=[289, 225, 275, 276, 266, 262, 282, 252, 256, 239, 246, 264, 255, 243, 265, 250, 272, 259, 263, 241, 243, 233, 247, 253, 250, 264, 254, 244, 227, 303, 242, 263, 269, 257, 237, 238, 263, 239, 260, 257, 251, 251, 266, 256, 260, 271, 226, 247, 251, 256, 260, 205, 262, 248, 251, 252, 228, 246, 242, 274, 264, 250, 235, 272, 275, 256, 248, 253, 264, 236, 271, 248, 302, 234, 299, 277, 238, 259, 266, 277, 248, 248, 234, 252, 253, 255, 257, 236, 259, 267, 262, 285, 256, 260, 264, 268, 261, 293, 286, 229, 253, 274, 258, 265, 248, 279, 241, 227, 265, 261, 262, 235, 226, 264, 250, 256, 237, 265, 264, 254, 250, 271, 261, 259, 266, 245, 243, 266] payload_rows_per_dst=[5584, 5436, 5444, 5464, 5572, 5486, 5520, 5454] dup_rows_per_dst=[2694, 2680, 2652, 2540, 2834, 2722, 2754, 2700] combine_rows_per_src=[5584, 5436, 5444, 5464, 5572, 5486, 5520, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228208 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157154160 ingress_bytes=175800320 link_bytes=175800320 token_bytes=314080112 weight_movement_bytes=18874368 total_nvlink_bytes=332954480 weight_to_token_ratio=0.060094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684432167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229304 combine_payload_bytes=156434432 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156663736 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098168 weight_movement_bytes=18874368 total_nvlink_bytes=331972536 weight_to_token_ratio=0.060283 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229608 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901608 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573608 weight_movement_bytes=18874368 total_nvlink_bytes=332447976 weight_to_token_ratio=0.060191 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38100 dup_rows=21946 dup_rows_remote=19194 dedup_ratio=0.334869 router_tokens_per_expert=[259, 251, 251, 259, 250, 279, 257, 275, 270, 270, 278, 243, 272, 256, 262, 253, 254, 254, 255, 251, 278, 240, 262, 280, 280, 280, 256, 232, 243, 260, 233, 263, 260, 249, 263, 308, 220, 278, 253, 255, 265, 252, 270, 207, 249, 244, 240, 266, 259, 242, 264, 262, 273, 230, 245, 264, 253, 256, 273, 232, 248, 241, 269, 244, 284, 259, 251, 253, 244, 288, 230, 280, 255, 292, 241, 252, 305, 266, 264, 249, 285, 265, 286, 253, 258, 250, 249, 253, 258, 265, 243, 236, 232, 238, 245, 270, 262, 261, 252, 256, 264, 244, 257, 235, 256, 242, 256, 268, 280, 251, 248, 248, 260, 231, 237, 254, 264, 270, 237, 232, 254, 262, 249, 224, 239, 256, 260, 220] payload_rows_per_dst=[5550, 5490, 5476, 5400, 5448, 5400, 5476, 5350] dup_rows_per_dst=[2820, 2752, 2876, 2710, 2784, 2772, 2684, 2548] combine_rows_per_src=[5550, 5490, 5476, 5400, 5448, 5400, 5476, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157450240 dispatch_meta_bytes=229048 combine_payload_bytes=157450240 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157679288 ingress_bytes=176324608 link_bytes=176324608 token_bytes=315129528 weight_movement_bytes=18874368 total_nvlink_bytes=334003896 weight_to_token_ratio=0.059894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689455637e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38188 dup_rows=21806 dup_rows_remote=18986 dedup_ratio=0.332733 router_tokens_per_expert=[257, 277, 254, 250, 269, 237, 279, 258, 249, 282, 259, 280, 265, 257, 262, 257, 307, 244, 288, 283, 234, 238, 264, 269, 255, 262, 237, 223, 272, 249, 239, 271, 268, 272, 243, 233, 233, 257, 270, 246, 261, 226, 231, 279, 254, 247, 251, 245, 258, 271, 241, 249, 261, 277, 253, 259, 235, 296, 264, 262, 261, 275, 253, 266, 276, 256, 239, 273, 263, 225, 263, 246, 274, 263, 243, 228, 259, 259, 261, 262, 278, 231, 239, 264, 220, 247, 266, 250, 245, 265, 264, 236, 250, 271, 268, 266, 248, 243, 250, 280, 244, 239, 246, 257, 258, 285, 268, 228, 237, 214, 264, 255, 253, 230, 217, 245, 258, 245, 257, 247, 275, 263, 251, 261, 286, 286, 233, 271] payload_rows_per_dst=[5474, 5494, 5412, 5542, 5456, 5486, 5362, 5504] dup_rows_per_dst=[2910, 2776, 2620, 2820, 2724, 2634, 2670, 2652] combine_rows_per_src=[5474, 5494, 5412, 5542, 5456, 5486, 5362, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=229176 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156286776 ingress_bytes=193806336 link_bytes=193806336 token_bytes=312344376 weight_movement_bytes=37748736 total_nvlink_bytes=350093112 weight_to_token_ratio=0.120856 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856956953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38424 dup_rows=21616 dup_rows_remote=18940 dedup_ratio=0.329834 router_tokens_per_expert=[270, 274, 266, 246, 262, 286, 279, 259, 286, 270, 230, 244, 245, 255, 261, 268, 255, 255, 222, 249, 236, 276, 265, 267, 235, 267, 283, 234, 265, 250, 229, 254, 261, 249, 241, 283, 230, 279, 260, 244, 260, 264, 259, 248, 246, 246, 264, 252, 236, 253, 234, 268, 254, 245, 265, 249, 268, 268, 271, 262, 235, 230, 233, 262, 234, 245, 224, 250, 247, 248, 266, 242, 271, 265, 223, 266, 247, 254, 267, 261, 246, 268, 260, 255, 269, 267, 231, 226, 273, 283, 229, 268, 278, 249, 237, 250, 267, 251, 255, 263, 265, 217, 275, 278, 239, 251, 255, 273, 249, 261, 255, 253, 281, 250, 266, 275, 270, 262, 269, 270, 261, 252, 271, 227, 264, 255, 263, 264] payload_rows_per_dst=[5614, 5426, 5496, 5428, 5422, 5474, 5518, 5542] dup_rows_per_dst=[2788, 2658, 2676, 2638, 2598, 2704, 2696, 2858] combine_rows_per_src=[5614, 5426, 5496, 5428, 5422, 5474, 5518, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38122 dup_rows=21952 dup_rows_remote=19190 dedup_ratio=0.334961 router_tokens_per_expert=[263, 251, 261, 246, 248, 254, 280, 266, 256, 238, 241, 245, 252, 270, 243, 226, 249, 298, 255, 228, 239, 237, 268, 230, 283, 252, 279, 286, 245, 267, 262, 228, 255, 261, 251, 249, 272, 271, 271, 251, 232, 261, 256, 259, 239, 308, 267, 246, 272, 259, 255, 249, 261, 253, 259, 266, 259, 257, 246, 266, 276, 253, 267, 243, 250, 217, 240, 254, 270, 230, 240, 302, 267, 237, 253, 263, 283, 250, 292, 264, 267, 250, 233, 248, 254, 273, 255, 249, 248, 245, 235, 282, 229, 264, 265, 260, 260, 257, 247, 252, 257, 267, 239, 261, 258, 296, 258, 258, 256, 258, 254, 264, 264, 273, 237, 246, 231, 273, 254, 241, 274, 242, 259, 259, 242, 251, 251, 224] payload_rows_per_dst=[5410, 5410, 5494, 5590, 5462, 5366, 5488, 5364] dup_rows_per_dst=[2670, 2802, 2804, 2692, 2762, 2748, 2796, 2678] combine_rows_per_src=[5410, 5410, 5494, 5590, 5462, 5366, 5488, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43520 payload_rows_remote=37994 dup_rows=22016 dup_rows_remote=19176 dedup_ratio=0.335938 router_tokens_per_expert=[247, 234, 274, 266, 262, 257, 238, 244, 260, 248, 250, 240, 245, 247, 249, 229, 250, 260, 258, 242, 275, 254, 247, 249, 264, 262, 240, 239, 254, 246, 291, 248, 231, 258, 234, 251, 245, 253, 228, 261, 266, 277, 286, 266, 244, 275, 251, 252, 249, 243, 260, 261, 240, 262, 243, 264, 263, 290, 236, 239, 267, 247, 243, 224, 290, 246, 264, 297, 280, 270, 241, 257, 244, 265, 298, 262, 253, 235, 253, 248, 277, 252, 273, 235, 263, 230, 252, 251, 268, 255, 265, 257, 261, 255, 257, 244, 275, 285, 248, 257, 265, 246, 273, 270, 265, 291, 273, 256, 250, 235, 243, 225, 261, 248, 265, 248, 279, 258, 263, 258, 264, 254, 272, 238, 261, 275, 250, 241] payload_rows_per_dst=[5074, 5390, 5408, 5636, 5476, 5474, 5536, 5526] dup_rows_per_dst=[2386, 2672, 2748, 2946, 2830, 2816, 2778, 2840] combine_rows_per_src=[5074, 5390, 5408, 5636, 5476, 5474, 5536, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38126 dup_rows=21872 dup_rows_remote=19028 dedup_ratio=0.333740 router_tokens_per_expert=[242, 279, 240, 273, 304, 275, 249, 228, 250, 250, 282, 265, 265, 232, 266, 261, 254, 263, 240, 253, 236, 251, 256, 248, 255, 269, 253, 249, 280, 243, 294, 267, 240, 254, 251, 252, 250, 245, 235, 267, 239, 247, 259, 240, 264, 243, 271, 250, 267, 250, 254, 256, 239, 229, 248, 252, 244, 248, 277, 252, 278, 251, 251, 268, 246, 274, 240, 263, 276, 261, 251, 250, 258, 239, 265, 275, 241, 247, 250, 256, 246, 256, 266, 237, 277, 253, 255, 240, 279, 259, 257, 265, 277, 270, 282, 272, 257, 252, 265, 256, 263, 261, 232, 237, 230, 243, 258, 282, 256, 240, 260, 248, 259, 278, 223, 276, 260, 256, 240, 253, 272, 253, 276, 248, 237, 267, 263, 241] payload_rows_per_dst=[5500, 5454, 5340, 5384, 5456, 5538, 5526, 5466] dup_rows_per_dst=[2822, 2768, 2674, 2744, 2728, 2844, 2554, 2738] combine_rows_per_src=[5500, 5454, 5340, 5384, 5456, 5538, 5526, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=37986 dup_rows=21908 dup_rows_remote=18904 dedup_ratio=0.334290 router_tokens_per_expert=[253, 271, 250, 246, 281, 239, 237, 267, 267, 243, 263, 246, 238, 260, 256, 259, 257, 239, 251, 285, 269, 239, 248, 269, 226, 246, 266, 249, 280, 222, 284, 246, 242, 268, 252, 251, 239, 223, 248, 259, 274, 258, 239, 285, 230, 245, 249, 240, 261, 256, 261, 228, 247, 271, 270, 234, 258, 252, 262, 233, 246, 283, 271, 262, 270, 254, 246, 258, 265, 248, 233, 260, 248, 248, 222, 256, 269, 269, 269, 263, 254, 271, 296, 273, 246, 273, 244, 237, 277, 259, 250, 266, 275, 254, 251, 274, 283, 240, 242, 266, 264, 239, 245, 263, 292, 239, 251, 262, 233, 261, 259, 269, 271, 273, 237, 267, 272, 233, 260, 253, 258, 256, 272, 270, 259, 239, 263, 250] payload_rows_per_dst=[5642, 5412, 5384, 5276, 5490, 5550, 5374, 5500] dup_rows_per_dst=[3004, 2740, 2804, 2490, 2666, 2850, 2588, 2766] combine_rows_per_src=[5642, 5412, 5384, 5276, 5490, 5550, 5374, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228696 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156646744 ingress_bytes=194166784 link_bytes=194166784 token_bytes=313064792 weight_movement_bytes=37748736 total_nvlink_bytes=350813528 weight_to_token_ratio=0.120578 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860410588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157384704 dispatch_meta_bytes=229456 combine_payload_bytes=157384704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157614160 ingress_bytes=176259072 link_bytes=176259072 token_bytes=314998864 weight_movement_bytes=18874368 total_nvlink_bytes=333873232 weight_to_token_ratio=0.059919 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688827703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229248 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376960 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524672 weight_movement_bytes=18874368 total_nvlink_bytes=331399040 weight_to_token_ratio=0.060393 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155623424 dispatch_meta_bytes=228680 combine_payload_bytes=155623424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155852104 ingress_bytes=174497792 link_bytes=174497792 token_bytes=311475528 weight_movement_bytes=18874368 total_nvlink_bytes=330349896 weight_to_token_ratio=0.060597 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671951985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38216 dup_rows=21820 dup_rows_remote=19138 dedup_ratio=0.332947 router_tokens_per_expert=[253, 244, 263, 235, 258, 269, 264, 273, 264, 267, 257, 262, 259, 249, 268, 256, 248, 250, 263, 261, 247, 257, 269, 242, 244, 238, 242, 282, 224, 241, 256, 282, 261, 254, 244, 274, 259, 255, 231, 264, 228, 274, 281, 272, 249, 247, 248, 277, 267, 241, 233, 277, 243, 250, 261, 277, 242, 264, 233, 245, 246, 266, 236, 264, 275, 238, 269, 228, 276, 238, 282, 267, 264, 273, 231, 241, 288, 270, 239, 242, 248, 280, 298, 254, 251, 274, 260, 233, 239, 255, 252, 233, 247, 269, 284, 248, 249, 243, 244, 270, 247, 255, 258, 272, 274, 262, 271, 255, 256, 251, 248, 236, 248, 233, 270, 258, 238, 260, 253, 272, 237, 277, 248, 244, 273, 258, 260, 252] payload_rows_per_dst=[5514, 5386, 5452, 5472, 5460, 5494, 5500, 5438] dup_rows_per_dst=[2768, 2706, 2784, 2618, 2782, 2756, 2682, 2724] combine_rows_per_src=[5514, 5386, 5452, 5472, 5460, 5494, 5500, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228616 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156392712 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312556808 weight_movement_bytes=37748736 total_nvlink_bytes=350305544 weight_to_token_ratio=0.120774 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=227560 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155818216 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311408872 weight_movement_bytes=18874368 total_nvlink_bytes=330283240 weight_to_token_ratio=0.060610 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38200 dup_rows=21842 dup_rows_remote=19178 dedup_ratio=0.333282 router_tokens_per_expert=[264, 238, 243, 260, 245, 252, 260, 238, 271, 264, 268, 265, 249, 259, 231, 215, 286, 232, 235, 255, 271, 247, 261, 265, 272, 258, 265, 254, 235, 257, 245, 241, 273, 252, 245, 260, 249, 269, 247, 254, 300, 248, 255, 303, 256, 264, 290, 250, 238, 245, 260, 247, 240, 259, 251, 272, 256, 231, 262, 236, 271, 224, 255, 268, 260, 243, 256, 258, 286, 233, 241, 278, 261, 289, 264, 297, 272, 243, 266, 263, 257, 254, 268, 236, 268, 248, 271, 248, 244, 243, 238, 242, 275, 261, 281, 263, 274, 265, 265, 219, 235, 263, 272, 252, 271, 252, 264, 236, 260, 244, 276, 256, 250, 249, 270, 248, 259, 238, 221, 287, 251, 243, 249, 237, 255, 240, 288, 241] payload_rows_per_dst=[5468, 5494, 5538, 5344, 5542, 5420, 5440, 5448] dup_rows_per_dst=[2576, 2664, 2892, 2686, 2878, 2774, 2768, 2604] combine_rows_per_src=[5468, 5494, 5538, 5344, 5542, 5420, 5440, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229416 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762152 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294888 weight_movement_bytes=18874368 total_nvlink_bytes=332169256 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38276 dup_rows=21870 dup_rows_remote=19224 dedup_ratio=0.333710 router_tokens_per_expert=[278, 232, 257, 254, 273, 256, 248, 257, 250, 261, 241, 258, 264, 273, 263, 248, 254, 243, 244, 276, 254, 269, 236, 223, 272, 268, 250, 251, 225, 276, 243, 252, 234, 256, 271, 264, 263, 258, 259, 273, 228, 257, 244, 256, 249, 256, 267, 284, 278, 269, 244, 256, 252, 220, 254, 272, 246, 258, 249, 250, 222, 250, 239, 259, 260, 269, 257, 262, 248, 231, 254, 240, 273, 248, 235, 242, 252, 247, 272, 248, 252, 295, 257, 267, 284, 238, 247, 249, 289, 275, 265, 291, 275, 263, 257, 232, 260, 279, 270, 243, 245, 234, 277, 229, 270, 266, 266, 259, 230, 275, 250, 266, 248, 260, 253, 242, 274, 251, 262, 262, 250, 254, 255, 224, 268, 235, 269, 282] payload_rows_per_dst=[5462, 5446, 5410, 5390, 5412, 5594, 5512, 5440] dup_rows_per_dst=[2764, 2626, 2828, 2646, 2664, 2878, 2726, 2738] combine_rows_per_src=[5462, 5446, 5410, 5390, 5412, 5594, 5512, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229512 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696712 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163912 weight_movement_bytes=37748736 total_nvlink_bytes=350912648 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38410 dup_rows=21662 dup_rows_remote=18946 dedup_ratio=0.330536 router_tokens_per_expert=[258, 255, 255, 256, 251, 240, 261, 251, 249, 243, 284, 263, 291, 245, 252, 254, 258, 267, 258, 248, 250, 246, 277, 259, 291, 240, 281, 242, 237, 261, 259, 259, 262, 263, 249, 260, 234, 271, 249, 235, 260, 256, 278, 266, 235, 266, 262, 244, 242, 271, 259, 248, 274, 272, 269, 252, 277, 259, 262, 250, 273, 253, 232, 247, 270, 260, 265, 224, 262, 276, 248, 250, 254, 265, 262, 241, 232, 256, 239, 296, 265, 272, 243, 262, 233, 245, 249, 234, 274, 276, 258, 252, 268, 262, 228, 282, 263, 231, 270, 276, 242, 262, 267, 269, 256, 255, 269, 223, 241, 275, 253, 249, 243, 232, 240, 252, 245, 271, 265, 221, 264, 249, 256, 245, 244, 259, 247, 260] payload_rows_per_dst=[5386, 5528, 5464, 5526, 5520, 5542, 5466, 5442] dup_rows_per_dst=[2830, 2738, 2716, 2754, 2680, 2664, 2736, 2544] combine_rows_per_src=[5386, 5528, 5464, 5526, 5520, 5542, 5466, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38134 dup_rows=21856 dup_rows_remote=19016 dedup_ratio=0.333496 router_tokens_per_expert=[244, 268, 252, 267, 254, 250, 246, 266, 265, 254, 251, 249, 256, 271, 255, 230, 251, 260, 295, 285, 263, 251, 245, 286, 296, 246, 239, 259, 282, 225, 252, 250, 262, 245, 274, 251, 246, 262, 279, 262, 259, 241, 242, 252, 273, 241, 231, 209, 237, 261, 259, 269, 261, 259, 259, 267, 241, 262, 254, 265, 250, 239, 261, 244, 273, 283, 239, 259, 286, 261, 297, 222, 283, 274, 258, 250, 263, 251, 238, 256, 242, 245, 242, 262, 270, 237, 263, 253, 257, 262, 230, 237, 249, 270, 243, 266, 269, 261, 259, 262, 266, 259, 243, 245, 244, 267, 282, 273, 237, 238, 260, 263, 245, 231, 258, 252, 256, 230, 241, 277, 268, 247, 224, 252, 269, 259, 262, 268] payload_rows_per_dst=[5432, 5538, 5498, 5438, 5546, 5386, 5446, 5396] dup_rows_per_dst=[2724, 2832, 2560, 2738, 2840, 2670, 2810, 2682] combine_rows_per_src=[5432, 5538, 5498, 5438, 5546, 5386, 5446, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38296 dup_rows=21646 dup_rows_remote=18856 dedup_ratio=0.330292 router_tokens_per_expert=[258, 274, 264, 272, 273, 257, 250, 283, 227, 248, 270, 283, 266, 259, 282, 269, 268, 244, 260, 254, 237, 284, 247, 256, 266, 273, 271, 245, 272, 272, 240, 260, 257, 244, 275, 257, 256, 256, 251, 261, 275, 261, 266, 247, 240, 264, 247, 239, 244, 223, 254, 267, 254, 253, 290, 271, 255, 266, 241, 257, 246, 275, 253, 238, 272, 259, 239, 258, 250, 260, 271, 256, 234, 243, 253, 240, 246, 279, 254, 232, 255, 272, 260, 242, 239, 277, 241, 236, 225, 268, 266, 241, 257, 260, 251, 263, 244, 259, 255, 243, 222, 268, 256, 250, 258, 262, 246, 248, 255, 271, 256, 245, 264, 266, 249, 249, 250, 242, 243, 256, 223, 278, 283, 265, 246, 251, 255, 244] payload_rows_per_dst=[5314, 5394, 5464, 5452, 5470, 5572, 5630, 5594] dup_rows_per_dst=[2362, 2720, 2568, 2738, 2806, 2778, 2884, 2790] combine_rows_per_src=[5314, 5394, 5464, 5452, 5470, 5572, 5630, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=230000 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008496 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786992 weight_movement_bytes=18874368 total_nvlink_bytes=332661360 weight_to_token_ratio=0.060150 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38350 dup_rows=21740 dup_rows_remote=19040 dedup_ratio=0.331726 router_tokens_per_expert=[269, 252, 272, 268, 252, 243, 283, 279, 250, 267, 266, 259, 272, 236, 263, 277, 263, 254, 269, 273, 271, 241, 277, 240, 232, 230, 250, 248, 264, 242, 272, 248, 239, 232, 252, 239, 267, 224, 267, 259, 270, 274, 248, 255, 273, 268, 246, 246, 257, 263, 245, 253, 277, 239, 268, 262, 253, 280, 269, 278, 260, 285, 271, 267, 253, 238, 260, 258, 257, 262, 232, 265, 259, 242, 222, 226, 257, 267, 239, 275, 252, 250, 252, 262, 239, 263, 275, 264, 238, 265, 255, 260, 240, 227, 240, 291, 255, 270, 241, 254, 236, 269, 214, 252, 238, 262, 260, 227, 274, 248, 241, 286, 303, 246, 264, 243, 231, 274, 247, 253, 265, 241, 271, 253, 250, 254, 235, 258] payload_rows_per_dst=[5562, 5468, 5412, 5570, 5410, 5446, 5432, 5496] dup_rows_per_dst=[2854, 2680, 2706, 2884, 2614, 2700, 2622, 2680] combine_rows_per_src=[5562, 5468, 5412, 5570, 5410, 5446, 5432, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38068 dup_rows=21940 dup_rows_remote=19178 dedup_ratio=0.334778 router_tokens_per_expert=[252, 263, 241, 251, 241, 232, 274, 272, 261, 235, 261, 251, 252, 240, 246, 205, 280, 275, 267, 252, 239, 243, 248, 263, 228, 269, 260, 263, 245, 242, 244, 259, 252, 264, 285, 236, 260, 269, 251, 235, 283, 278, 253, 249, 253, 246, 261, 256, 235, 261, 267, 243, 261, 258, 269, 257, 233, 259, 268, 264, 257, 259, 264, 239, 288, 250, 285, 255, 246, 250, 276, 262, 267, 272, 221, 276, 238, 252, 265, 253, 263, 244, 262, 302, 263, 266, 261, 282, 249, 252, 252, 251, 253, 249, 258, 250, 246, 239, 249, 247, 238, 268, 258, 257, 251, 247, 274, 248, 264, 242, 241, 251, 282, 234, 256, 282, 246, 236, 264, 248, 256, 304, 254, 263, 258, 236, 264, 273] payload_rows_per_dst=[5528, 5572, 5356, 5480, 5466, 5496, 5356, 5342] dup_rows_per_dst=[2762, 2948, 2540, 2708, 2846, 2818, 2684, 2634] combine_rows_per_src=[5528, 5572, 5356, 5480, 5466, 5496, 5356, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38164 dup_rows=21894 dup_rows_remote=19120 dedup_ratio=0.334076 router_tokens_per_expert=[251, 254, 247, 243, 260, 261, 267, 267, 270, 265, 256, 247, 275, 240, 249, 262, 254, 253, 244, 237, 257, 251, 259, 252, 278, 248, 263, 238, 257, 275, 256, 278, 271, 247, 243, 260, 263, 265, 248, 236, 228, 249, 282, 268, 257, 258, 247, 232, 248, 239, 263, 242, 246, 265, 302, 274, 272, 267, 261, 262, 268, 246, 235, 246, 278, 253, 242, 244, 250, 275, 244, 258, 269, 233, 252, 278, 245, 263, 227, 281, 270, 267, 269, 259, 239, 234, 254, 247, 244, 251, 262, 275, 249, 245, 252, 253, 228, 260, 254, 248, 297, 241, 266, 267, 256, 254, 232, 245, 305, 280, 260, 233, 243, 249, 274, 263, 231, 263, 259, 237, 267, 250, 266, 246, 251, 282, 243, 252] payload_rows_per_dst=[5474, 5446, 5478, 5524, 5436, 5352, 5478, 5454] dup_rows_per_dst=[2754, 2754, 2630, 2748, 2748, 2788, 2774, 2698] combine_rows_per_src=[5474, 5446, 5478, 5524, 5436, 5352, 5478, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229424 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157556784 ingress_bytes=195076096 link_bytes=195076096 token_bytes=314884144 weight_movement_bytes=37748736 total_nvlink_bytes=352632880 weight_to_token_ratio=0.119881 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869123168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=228600 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156425464 ingress_bytes=231694336 link_bytes=231694336 token_bytes=312622328 weight_movement_bytes=75497472 total_nvlink_bytes=388119800 weight_to_token_ratio=0.241497 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219981127e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228608 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089024 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949440 weight_movement_bytes=18874368 total_nvlink_bytes=332823808 weight_to_token_ratio=0.060119 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229560 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311160 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392760 weight_movement_bytes=18874368 total_nvlink_bytes=333267128 weight_to_token_ratio=0.060034 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38450 dup_rows=21686 dup_rows_remote=19022 dedup_ratio=0.330902 router_tokens_per_expert=[265, 278, 265, 250, 290, 253, 259, 271, 305, 264, 218, 258, 218, 267, 260, 262, 250, 242, 258, 228, 245, 265, 266, 255, 241, 251, 264, 247, 251, 250, 266, 253, 253, 251, 272, 263, 214, 235, 262, 255, 243, 240, 292, 261, 235, 235, 266, 267, 212, 253, 280, 253, 282, 245, 256, 253, 277, 237, 252, 255, 270, 273, 263, 246, 255, 240, 252, 264, 274, 268, 248, 257, 229, 294, 274, 246, 228, 256, 267, 263, 290, 266, 264, 237, 265, 271, 235, 251, 271, 228, 267, 248, 267, 241, 264, 249, 233, 285, 270, 247, 229, 265, 255, 269, 266, 265, 246, 269, 235, 278, 269, 229, 265, 247, 230, 266, 274, 255, 268, 240, 245, 237, 255, 254, 263, 255, 247, 262] payload_rows_per_dst=[5518, 5400, 5408, 5412, 5552, 5508, 5532, 5520] dup_rows_per_dst=[2848, 2664, 2680, 2802, 2678, 2720, 2688, 2606] combine_rows_per_src=[5518, 5400, 5408, 5412, 5552, 5508, 5532, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=228984 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156155512 ingress_bytes=174800896 link_bytes=174800896 token_bytes=312082040 weight_movement_bytes=18874368 total_nvlink_bytes=330956408 weight_to_token_ratio=0.060479 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674856178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229136 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548880 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868624 weight_movement_bytes=18874368 total_nvlink_bytes=331742992 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38218 dup_rows=21728 dup_rows_remote=18824 dedup_ratio=0.331543 router_tokens_per_expert=[250, 249, 253, 274, 261, 267, 230, 232, 299, 239, 263, 260, 264, 268, 259, 265, 267, 256, 250, 247, 276, 258, 279, 252, 251, 250, 232, 261, 259, 251, 248, 277, 290, 268, 219, 286, 240, 238, 274, 289, 232, 244, 249, 277, 257, 257, 227, 223, 255, 260, 247, 280, 276, 238, 251, 257, 241, 272, 282, 258, 288, 281, 287, 274, 274, 259, 251, 251, 286, 274, 247, 267, 251, 254, 227, 271, 248, 285, 272, 230, 247, 272, 249, 267, 253, 239, 269, 239, 243, 225, 228, 272, 237, 240, 229, 282, 241, 255, 250, 232, 265, 246, 244, 253, 287, 258, 226, 224, 246, 239, 269, 241, 264, 258, 259, 250, 231, 259, 255, 256, 259, 285, 239, 252, 226, 283, 242, 272] payload_rows_per_dst=[5548, 5464, 5500, 5590, 5486, 5390, 5368, 5462] dup_rows_per_dst=[2718, 2764, 2640, 2904, 2808, 2592, 2584, 2718] combine_rows_per_src=[5548, 5464, 5500, 5590, 5486, 5390, 5368, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=229888 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157721088 ingress_bytes=214114304 link_bytes=214114304 token_bytes=315212288 weight_movement_bytes=56623104 total_nvlink_bytes=371835392 weight_to_token_ratio=0.179635 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051537910e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38470 dup_rows=21638 dup_rows_remote=19006 dedup_ratio=0.330170 router_tokens_per_expert=[262, 274, 252, 243, 233, 243, 244, 237, 265, 257, 243, 254, 253, 239, 258, 254, 247, 271, 244, 271, 263, 275, 236, 293, 226, 243, 246, 275, 262, 274, 269, 278, 246, 251, 252, 248, 222, 277, 265, 267, 231, 245, 229, 258, 285, 263, 260, 231, 233, 281, 270, 255, 243, 263, 258, 224, 252, 247, 259, 243, 243, 269, 241, 269, 270, 262, 278, 245, 279, 267, 247, 246, 262, 265, 268, 237, 288, 255, 264, 277, 266, 257, 273, 239, 259, 253, 264, 280, 262, 280, 268, 248, 267, 253, 274, 267, 242, 273, 230, 234, 263, 245, 249, 257, 265, 262, 227, 241, 279, 257, 270, 254, 249, 246, 259, 238, 277, 254, 256, 246, 270, 230, 258, 260, 239, 229, 256, 269] payload_rows_per_dst=[5394, 5496, 5428, 5532, 5618, 5554, 5432, 5444] dup_rows_per_dst=[2628, 2850, 2632, 2568, 2802, 2866, 2664, 2628] combine_rows_per_src=[5394, 5496, 5428, 5532, 5618, 5554, 5432, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228168 combine_payload_bytes=156540928 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156769096 ingress_bytes=213164032 link_bytes=213164032 token_bytes=313310024 weight_movement_bytes=56623104 total_nvlink_bytes=369933128 weight_to_token_ratio=0.180725 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042432871e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38378 dup_rows=21670 dup_rows_remote=19042 dedup_ratio=0.330658 router_tokens_per_expert=[241, 269, 245, 297, 251, 249, 255, 234, 256, 245, 260, 269, 250, 258, 292, 280, 265, 284, 238, 247, 261, 256, 277, 289, 257, 250, 274, 224, 260, 263, 247, 233, 243, 284, 263, 238, 255, 227, 240, 235, 275, 293, 277, 265, 259, 263, 249, 233, 263, 276, 252, 222, 268, 262, 235, 258, 238, 248, 244, 260, 269, 226, 250, 264, 253, 236, 260, 276, 278, 251, 221, 252, 274, 232, 268, 243, 231, 270, 259, 254, 232, 257, 241, 273, 273, 241, 231, 250, 240, 246, 248, 258, 249, 261, 240, 265, 282, 249, 221, 278, 267, 220, 242, 287, 263, 252, 258, 271, 241, 248, 282, 245, 271, 260, 249, 261, 270, 267, 262, 293, 251, 258, 283, 287, 257, 232, 250, 238] payload_rows_per_dst=[5494, 5432, 5498, 5428, 5488, 5374, 5510, 5642] dup_rows_per_dst=[2808, 2818, 2700, 2642, 2628, 2636, 2702, 2736] combine_rows_per_src=[5494, 5432, 5498, 5428, 5488, 5374, 5510, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38198 dup_rows=21768 dup_rows_remote=18954 dedup_ratio=0.332153 router_tokens_per_expert=[256, 259, 245, 256, 263, 253, 257, 249, 237, 248, 247, 223, 299, 242, 255, 274, 279, 219, 292, 264, 249, 260, 276, 255, 254, 249, 296, 263, 248, 251, 238, 263, 268, 246, 241, 275, 248, 252, 214, 285, 274, 256, 274, 225, 274, 253, 227, 261, 246, 256, 259, 251, 267, 287, 249, 249, 260, 243, 269, 275, 253, 268, 261, 231, 262, 268, 250, 272, 251, 263, 280, 236, 228, 255, 249, 263, 256, 228, 244, 264, 249, 264, 266, 266, 219, 249, 265, 267, 274, 275, 269, 268, 239, 266, 254, 250, 266, 225, 263, 257, 258, 232, 256, 264, 241, 249, 250, 243, 238, 261, 258, 252, 238, 283, 267, 257, 248, 247, 243, 268, 269, 256, 275, 242, 260, 251, 265, 261] payload_rows_per_dst=[5406, 5376, 5546, 5520, 5428, 5436, 5486, 5570] dup_rows_per_dst=[2720, 2530, 2748, 2728, 2680, 2750, 2798, 2814] combine_rows_per_src=[5406, 5376, 5546, 5520, 5428, 5436, 5486, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38196 dup_rows=21800 dup_rows_remote=19032 dedup_ratio=0.332642 router_tokens_per_expert=[278, 229, 249, 257, 264, 239, 253, 247, 246, 262, 266, 251, 259, 246, 253, 247, 270, 260, 240, 261, 251, 265, 247, 272, 274, 256, 300, 265, 247, 272, 264, 259, 255, 276, 268, 265, 275, 254, 235, 250, 251, 247, 263, 247, 233, 237, 261, 269, 262, 258, 247, 242, 242, 245, 283, 257, 229, 253, 240, 250, 257, 261, 255, 240, 253, 244, 257, 254, 241, 261, 254, 243, 247, 276, 242, 258, 255, 240, 247, 251, 255, 262, 289, 270, 247, 267, 267, 287, 223, 250, 294, 254, 252, 243, 240, 254, 263, 259, 232, 240, 276, 253, 269, 253, 247, 255, 273, 277, 236, 272, 261, 268, 265, 261, 258, 275, 254, 247, 273, 225, 259, 257, 250, 241, 229, 277, 267, 263] payload_rows_per_dst=[5418, 5556, 5374, 5450, 5450, 5540, 5448, 5500] dup_rows_per_dst=[2674, 2850, 2798, 2592, 2596, 2768, 2820, 2702] combine_rows_per_src=[5418, 5556, 5374, 5450, 5450, 5540, 5448, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38212 dup_rows=21800 dup_rows_remote=18978 dedup_ratio=0.332642 router_tokens_per_expert=[241, 313, 269, 248, 248, 226, 248, 263, 292, 238, 275, 244, 253, 283, 270, 253, 249, 269, 240, 234, 241, 253, 239, 248, 244, 258, 253, 267, 249, 271, 274, 293, 285, 248, 255, 254, 306, 281, 244, 278, 254, 269, 265, 263, 278, 222, 250, 246, 265, 290, 252, 264, 277, 259, 246, 272, 296, 291, 242, 260, 229, 262, 262, 246, 260, 229, 235, 250, 268, 254, 259, 230, 245, 259, 246, 254, 269, 238, 254, 288, 247, 268, 263, 226, 272, 252, 252, 243, 234, 245, 239, 232, 253, 283, 265, 242, 283, 244, 251, 243, 242, 238, 266, 265, 252, 221, 263, 278, 239, 253, 229, 238, 262, 252, 219, 277, 252, 275, 253, 219, 256, 265, 250, 242, 255, 241, 268, 266] payload_rows_per_dst=[5524, 5464, 5560, 5570, 5416, 5436, 5344, 5422] dup_rows_per_dst=[2822, 2804, 2714, 2856, 2660, 2596, 2666, 2682] combine_rows_per_src=[5524, 5464, 5560, 5570, 5416, 5436, 5344, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229904 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157803024 ingress_bytes=176447488 link_bytes=176447488 token_bytes=315376144 weight_movement_bytes=18874368 total_nvlink_bytes=334250512 weight_to_token_ratio=0.059847 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690633012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38226 dup_rows=21746 dup_rows_remote=19120 dedup_ratio=0.331818 router_tokens_per_expert=[253, 267, 246, 250, 241, 259, 236, 235, 235, 263, 264, 257, 275, 260, 250, 273, 239, 259, 247, 258, 256, 265, 267, 265, 254, 252, 241, 258, 263, 240, 286, 286, 269, 257, 253, 258, 245, 250, 265, 279, 257, 273, 238, 243, 261, 267, 256, 246, 251, 258, 260, 228, 258, 242, 244, 260, 247, 252, 262, 269, 260, 286, 230, 247, 270, 264, 266, 240, 248, 280, 233, 216, 269, 232, 270, 267, 282, 254, 271, 267, 267, 252, 272, 258, 257, 243, 269, 252, 238, 239, 251, 259, 257, 296, 231, 265, 250, 241, 240, 259, 266, 283, 239, 241, 271, 261, 253, 255, 247, 260, 259, 270, 235, 267, 247, 278, 252, 252, 237, 264, 247, 262, 246, 242, 261, 237, 269, 271] payload_rows_per_dst=[5458, 5538, 5388, 5436, 5494, 5444, 5564, 5468] dup_rows_per_dst=[2670, 2734, 2846, 2672, 2764, 2768, 2626, 2666] combine_rows_per_src=[5458, 5538, 5388, 5436, 5494, 5444, 5564, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229680 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157425968 ingress_bytes=176070656 link_bytes=176070656 token_bytes=314622256 weight_movement_bytes=18874368 total_nvlink_bytes=333496624 weight_to_token_ratio=0.059991 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687022394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228608 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156687616 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313146624 weight_movement_bytes=18874368 total_nvlink_bytes=332020992 weight_to_token_ratio=0.060273 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38518 dup_rows=21684 dup_rows_remote=19026 dedup_ratio=0.330872 router_tokens_per_expert=[227, 226, 268, 254, 246, 263, 258, 263, 243, 252, 245, 272, 266, 242, 235, 258, 252, 269, 230, 237, 254, 258, 243, 234, 241, 245, 259, 259, 244, 238, 247, 286, 265, 212, 281, 253, 250, 247, 275, 240, 285, 245, 232, 247, 258, 273, 252, 272, 250, 258, 295, 270, 275, 267, 246, 257, 271, 258, 242, 250, 254, 261, 249, 255, 269, 254, 264, 259, 251, 259, 240, 282, 274, 265, 245, 274, 240, 277, 271, 244, 285, 257, 251, 252, 239, 272, 241, 249, 253, 243, 266, 247, 225, 260, 280, 260, 273, 224, 261, 261, 256, 246, 257, 245, 255, 236, 249, 301, 261, 254, 285, 293, 254, 264, 233, 258, 266, 244, 254, 259, 250, 255, 279, 268, 275, 255, 249, 241] payload_rows_per_dst=[5442, 5334, 5512, 5500, 5550, 5472, 5550, 5492] dup_rows_per_dst=[2594, 2658, 2662, 2816, 2786, 2688, 2764, 2716] combine_rows_per_src=[5442, 5334, 5512, 5500, 5550, 5472, 5550, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228912 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156679728 ingress_bytes=231948288 link_bytes=231948288 token_bytes=313130544 weight_movement_bytes=75497472 total_nvlink_bytes=388628016 weight_to_token_ratio=0.241105 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222414370e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228760 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156745112 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313261464 weight_movement_bytes=37748736 total_nvlink_bytes=351010200 weight_to_token_ratio=0.120502 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38358 dup_rows=21754 dup_rows_remote=19042 dedup_ratio=0.331940 router_tokens_per_expert=[257, 254, 240, 248, 292, 275, 254, 235, 258, 258, 276, 280, 232, 234, 273, 270, 254, 255, 255, 245, 212, 246, 239, 262, 265, 294, 289, 260, 262, 254, 245, 257, 261, 253, 264, 255, 258, 252, 272, 270, 258, 241, 268, 241, 267, 287, 264, 257, 252, 243, 253, 273, 227, 271, 247, 279, 252, 240, 243, 254, 239, 247, 270, 278, 225, 245, 231, 271, 243, 299, 283, 261, 239, 236, 274, 275, 256, 256, 254, 243, 249, 268, 275, 246, 249, 250, 263, 273, 241, 237, 258, 259, 269, 254, 252, 279, 272, 260, 281, 255, 250, 245, 253, 219, 239, 269, 246, 266, 245, 261, 256, 253, 284, 256, 253, 239, 259, 253, 245, 252, 241, 259, 242, 217, 249, 278, 240, 252] payload_rows_per_dst=[5552, 5444, 5506, 5424, 5492, 5538, 5468, 5358] dup_rows_per_dst=[2720, 2744, 2830, 2712, 2690, 2706, 2672, 2680] combine_rows_per_src=[5552, 5444, 5506, 5424, 5492, 5538, 5468, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229384 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803080 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376776 weight_movement_bytes=18874368 total_nvlink_bytes=332251144 weight_to_token_ratio=0.060229 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157769728 dispatch_meta_bytes=230176 combine_payload_bytes=157769728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157999904 ingress_bytes=195518464 link_bytes=195518464 token_bytes=315769632 weight_movement_bytes=37748736 total_nvlink_bytes=353518368 weight_to_token_ratio=0.119545 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873361721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229600 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343968 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314458336 weight_movement_bytes=37748736 total_nvlink_bytes=352207072 weight_to_token_ratio=0.120044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38410 dup_rows=21714 dup_rows_remote=19106 dedup_ratio=0.331329 router_tokens_per_expert=[258, 275, 245, 270, 270, 250, 247, 258, 245, 275, 252, 230, 234, 264, 252, 264, 264, 251, 295, 264, 259, 255, 244, 280, 279, 220, 261, 255, 237, 273, 260, 227, 242, 248, 249, 270, 244, 248, 244, 253, 265, 246, 263, 248, 246, 241, 263, 240, 255, 257, 244, 254, 257, 264, 248, 238, 282, 260, 277, 254, 252, 239, 245, 252, 273, 269, 265, 238, 266, 266, 267, 236, 272, 258, 258, 262, 255, 250, 258, 239, 261, 284, 266, 249, 282, 252, 255, 243, 251, 249, 256, 273, 279, 262, 267, 287, 228, 258, 248, 283, 232, 259, 231, 266, 277, 256, 251, 250, 250, 239, 228, 255, 268, 257, 248, 261, 248, 273, 243, 262, 280, 254, 239, 245, 252, 268, 258, 252] payload_rows_per_dst=[5494, 5496, 5412, 5428, 5488, 5592, 5438, 5474] dup_rows_per_dst=[2684, 2752, 2608, 2728, 2776, 2840, 2584, 2742] combine_rows_per_src=[5494, 5496, 5412, 5428, 5488, 5592, 5438, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38354 dup_rows=21676 dup_rows_remote=18956 dedup_ratio=0.330750 router_tokens_per_expert=[246, 278, 225, 261, 261, 265, 258, 268, 275, 236, 259, 230, 262, 269, 240, 273, 259, 250, 252, 270, 247, 236, 262, 269, 246, 240, 261, 251, 244, 263, 282, 282, 273, 232, 264, 281, 294, 260, 264, 244, 265, 242, 254, 278, 241, 267, 267, 264, 254, 246, 255, 249, 254, 244, 269, 267, 270, 253, 253, 269, 254, 237, 250, 249, 266, 244, 280, 273, 270, 266, 255, 249, 246, 238, 248, 277, 255, 248, 235, 263, 235, 281, 241, 251, 237, 280, 254, 255, 271, 258, 257, 268, 276, 244, 250, 270, 281, 273, 267, 249, 225, 244, 260, 224, 249, 262, 235, 246, 267, 225, 252, 274, 255, 254, 250, 246, 264, 231, 250, 236, 242, 258, 262, 268, 230, 252, 268, 245] payload_rows_per_dst=[5464, 5524, 5570, 5484, 5506, 5472, 5488, 5352] dup_rows_per_dst=[2748, 2704, 2810, 2662, 2720, 2784, 2578, 2670] combine_rows_per_src=[5464, 5524, 5570, 5484, 5506, 5472, 5488, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38298 dup_rows=21796 dup_rows_remote=19064 dedup_ratio=0.332581 router_tokens_per_expert=[260, 246, 231, 269, 229, 251, 269, 277, 254, 218, 238, 256, 260, 257, 256, 271, 260, 281, 231, 250, 251, 267, 252, 253, 247, 269, 273, 264, 270, 262, 251, 226, 270, 261, 269, 262, 262, 271, 259, 259, 245, 287, 223, 233, 275, 266, 260, 253, 293, 257, 253, 255, 267, 226, 269, 241, 263, 232, 237, 269, 285, 254, 266, 259, 276, 242, 244, 265, 245, 236, 248, 263, 254, 277, 250, 255, 272, 266, 265, 246, 245, 254, 256, 270, 212, 254, 256, 270, 265, 280, 278, 269, 297, 244, 271, 239, 229, 261, 256, 238, 257, 257, 252, 263, 243, 238, 261, 262, 247, 281, 243, 238, 250, 247, 233, 275, 266, 246, 258, 264, 267, 262, 236, 253, 237, 245, 236, 273] payload_rows_per_dst=[5434, 5448, 5494, 5432, 5362, 5548, 5580, 5442] dup_rows_per_dst=[2650, 2766, 2816, 2734, 2486, 2780, 2832, 2732] combine_rows_per_src=[5434, 5448, 5494, 5432, 5362, 5548, 5580, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38396 dup_rows=21746 dup_rows_remote=19010 dedup_ratio=0.331818 router_tokens_per_expert=[245, 243, 242, 247, 274, 228, 253, 255, 254, 254, 311, 243, 244, 239, 254, 288, 234, 247, 272, 280, 279, 257, 237, 246, 257, 244, 263, 246, 250, 262, 289, 261, 244, 230, 261, 266, 238, 277, 274, 251, 232, 240, 265, 258, 266, 249, 275, 278, 261, 261, 257, 280, 221, 275, 244, 257, 249, 266, 291, 238, 261, 264, 278, 233, 244, 265, 255, 262, 238, 264, 277, 251, 248, 256, 257, 245, 261, 254, 252, 244, 232, 270, 268, 236, 250, 238, 268, 250, 268, 257, 268, 266, 254, 265, 258, 217, 259, 261, 263, 265, 233, 243, 271, 263, 252, 277, 248, 245, 265, 239, 253, 267, 264, 257, 240, 266, 277, 271, 249, 266, 235, 276, 244, 272, 227, 232, 265, 247] payload_rows_per_dst=[5488, 5512, 5496, 5564, 5448, 5394, 5410, 5478] dup_rows_per_dst=[2660, 2736, 2712, 2708, 2698, 2736, 2798, 2698] combine_rows_per_src=[5488, 5512, 5496, 5564, 5448, 5394, 5410, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38168 dup_rows=21930 dup_rows_remote=19060 dedup_ratio=0.334625 router_tokens_per_expert=[227, 233, 263, 278, 255, 251, 228, 251, 278, 232, 254, 247, 253, 263, 247, 264, 282, 252, 243, 253, 230, 256, 272, 248, 236, 255, 276, 269, 244, 268, 240, 251, 243, 241, 271, 240, 251, 254, 243, 250, 255, 261, 244, 252, 248, 253, 277, 247, 235, 243, 253, 249, 254, 269, 249, 254, 281, 251, 283, 288, 270, 238, 271, 239, 246, 282, 264, 249, 247, 265, 246, 265, 272, 250, 236, 264, 245, 278, 267, 233, 287, 271, 252, 261, 263, 266, 265, 247, 243, 266, 267, 256, 239, 243, 248, 261, 268, 269, 236, 232, 250, 259, 245, 272, 244, 269, 259, 239, 246, 265, 262, 253, 259, 237, 249, 263, 269, 252, 275, 273, 263, 285, 253, 237, 278, 252, 277, 278] payload_rows_per_dst=[5438, 5462, 5552, 5522, 5268, 5352, 5394, 5618] dup_rows_per_dst=[2870, 2738, 3052, 2732, 2378, 2658, 2720, 2782] combine_rows_per_src=[5438, 5462, 5552, 5522, 5268, 5352, 5394, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=230064 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157557424 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314884784 weight_movement_bytes=18874368 total_nvlink_bytes=333759152 weight_to_token_ratio=0.059941 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229240 combine_payload_bytes=157097984 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157327224 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314425208 weight_movement_bytes=37748736 total_nvlink_bytes=352173944 weight_to_token_ratio=0.120056 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38112 dup_rows=21884 dup_rows_remote=18974 dedup_ratio=0.333923 router_tokens_per_expert=[246, 250, 277, 250, 235, 257, 247, 256, 257, 260, 243, 258, 251, 228, 250, 257, 286, 278, 250, 239, 265, 256, 259, 273, 263, 263, 242, 259, 271, 269, 262, 258, 264, 247, 257, 264, 265, 248, 247, 242, 261, 250, 243, 242, 273, 251, 263, 253, 272, 245, 267, 239, 270, 272, 274, 258, 275, 254, 268, 264, 237, 261, 256, 255, 252, 240, 221, 257, 259, 263, 257, 240, 216, 251, 255, 269, 253, 253, 242, 227, 250, 250, 261, 258, 270, 245, 269, 262, 246, 252, 284, 228, 241, 260, 270, 234, 242, 265, 264, 259, 254, 240, 301, 261, 241, 242, 267, 284, 296, 284, 274, 251, 242, 241, 264, 230, 262, 251, 268, 261, 271, 266, 233, 229, 251, 254, 248, 285] payload_rows_per_dst=[5430, 5560, 5406, 5336, 5548, 5450, 5540, 5382] dup_rows_per_dst=[2614, 2826, 2734, 2540, 2820, 2710, 2910, 2730] combine_rows_per_src=[5430, 5560, 5406, 5336, 5548, 5450, 5540, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38278 dup_rows=21886 dup_rows_remote=19084 dedup_ratio=0.333954 router_tokens_per_expert=[250, 279, 276, 266, 232, 267, 236, 253, 285, 251, 279, 257, 250, 235, 292, 269, 270, 282, 255, 271, 260, 270, 231, 255, 274, 241, 252, 243, 232, 227, 264, 260, 257, 240, 252, 230, 246, 241, 283, 250, 269, 257, 217, 261, 252, 274, 244, 245, 260, 250, 250, 264, 255, 242, 255, 238, 263, 254, 251, 277, 253, 275, 230, 247, 275, 250, 254, 230, 268, 265, 237, 259, 264, 241, 266, 257, 302, 249, 258, 247, 263, 229, 291, 276, 254, 267, 253, 275, 273, 266, 264, 254, 260, 235, 261, 276, 232, 248, 266, 268, 258, 228, 251, 277, 276, 265, 263, 265, 270, 252, 231, 273, 238, 255, 244, 242, 233, 273, 254, 240, 260, 271, 262, 227, 254, 233, 240, 254] payload_rows_per_dst=[5516, 5372, 5420, 5458, 5478, 5568, 5464, 5374] dup_rows_per_dst=[2838, 2802, 2616, 2670, 2766, 2826, 2782, 2586] combine_rows_per_src=[5516, 5372, 5420, 5458, 5478, 5568, 5464, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229448 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098056 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966664 weight_movement_bytes=18874368 total_nvlink_bytes=332841032 weight_to_token_ratio=0.060116 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229624 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157499640 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769656 weight_movement_bytes=56623104 total_nvlink_bytes=371392760 weight_to_token_ratio=0.179887 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38226 dup_rows=21758 dup_rows_remote=19048 dedup_ratio=0.332001 router_tokens_per_expert=[247, 281, 251, 237, 289, 231, 267, 280, 248, 265, 266, 257, 241, 272, 255, 259, 277, 243, 254, 243, 254, 248, 251, 266, 255, 251, 261, 276, 246, 271, 242, 257, 256, 256, 274, 265, 254, 273, 266, 258, 248, 247, 257, 267, 250, 239, 244, 240, 248, 249, 255, 269, 242, 268, 261, 252, 310, 266, 279, 243, 259, 233, 267, 230, 237, 254, 273, 250, 237, 259, 239, 258, 264, 247, 287, 261, 257, 265, 266, 245, 237, 264, 274, 245, 244, 251, 240, 276, 254, 238, 247, 276, 254, 233, 266, 263, 261, 249, 256, 234, 253, 259, 208, 259, 267, 231, 267, 253, 266, 263, 255, 247, 246, 269, 257, 288, 272, 239, 265, 236, 238, 256, 263, 242, 237, 253, 259, 293] payload_rows_per_dst=[5494, 5518, 5422, 5552, 5476, 5414, 5442, 5460] dup_rows_per_dst=[2798, 2672, 2766, 2710, 2722, 2710, 2614, 2766] combine_rows_per_src=[5494, 5518, 5422, 5552, 5476, 5414, 5442, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228912 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156565040 ingress_bytes=194084864 link_bytes=194084864 token_bytes=312901168 weight_movement_bytes=37748736 total_nvlink_bytes=350649904 weight_to_token_ratio=0.120641 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859625671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228344 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156335096 ingress_bytes=193855488 link_bytes=193855488 token_bytes=312441848 weight_movement_bytes=37748736 total_nvlink_bytes=350190584 weight_to_token_ratio=0.120818 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857427903e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229448 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016136 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802824 weight_movement_bytes=37748736 total_nvlink_bytes=351551560 weight_to_token_ratio=0.120294 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229096 combine_payload_bytes=156573696 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156802792 ingress_bytes=232071168 link_bytes=232071168 token_bytes=313376488 weight_movement_bytes=75497472 total_nvlink_bytes=388873960 weight_to_token_ratio=0.240916 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223591746e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38232 dup_rows=21778 dup_rows_remote=19008 dedup_ratio=0.332306 router_tokens_per_expert=[245, 241, 256, 253, 231, 258, 293, 259, 232, 245, 242, 246, 244, 257, 235, 252, 265, 245, 240, 252, 263, 259, 247, 254, 240, 282, 273, 213, 246, 249, 260, 252, 227, 280, 259, 247, 254, 252, 262, 251, 286, 246, 267, 285, 259, 256, 255, 262, 256, 256, 269, 254, 260, 252, 278, 289, 278, 284, 271, 232, 262, 248, 268, 262, 263, 264, 229, 260, 252, 271, 244, 243, 249, 255, 254, 269, 255, 262, 241, 246, 255, 264, 274, 228, 279, 240, 272, 260, 234, 257, 270, 276, 262, 257, 265, 260, 235, 237, 222, 267, 234, 257, 244, 275, 261, 275, 269, 279, 254, 265, 246, 264, 271, 236, 263, 262, 250, 260, 299, 234, 234, 252, 253, 252, 242, 259, 254, 257] payload_rows_per_dst=[5374, 5458, 5526, 5602, 5426, 5482, 5412, 5478] dup_rows_per_dst=[2604, 2622, 2770, 2836, 2688, 2824, 2756, 2678] combine_rows_per_src=[5374, 5458, 5526, 5602, 5426, 5482, 5412, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44030 payload_rows_remote=38584 dup_rows=21506 dup_rows_remote=19044 dedup_ratio=0.328156 router_tokens_per_expert=[280, 247, 294, 258, 242, 268, 238, 252, 252, 267, 248, 212, 227, 256, 262, 289, 248, 297, 252, 261, 252, 268, 277, 259, 258, 250, 291, 261, 256, 248, 256, 263, 250, 262, 261, 251, 228, 226, 287, 265, 271, 271, 263, 258, 289, 226, 265, 251, 280, 255, 255, 256, 261, 277, 242, 246, 244, 260, 260, 231, 241, 222, 265, 249, 267, 273, 233, 257, 246, 274, 232, 207, 251, 243, 226, 238, 259, 244, 258, 246, 246, 268, 234, 220, 269, 270, 237, 253, 246, 313, 272, 257, 269, 243, 276, 256, 262, 253, 240, 262, 269, 265, 251, 252, 241, 244, 263, 255, 243, 273, 278, 281, 277, 253, 262, 261, 246, 266, 251, 260, 282, 252, 261, 264, 234, 241, 238, 248] payload_rows_per_dst=[5534, 5542, 5494, 5482, 5446, 5482, 5516, 5534] dup_rows_per_dst=[2650, 2852, 2754, 2606, 2462, 2776, 2748, 2658] combine_rows_per_src=[5534, 5542, 5494, 5482, 5446, 5482, 5516, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38148 dup_rows=21774 dup_rows_remote=18862 dedup_ratio=0.332245 router_tokens_per_expert=[275, 259, 258, 269, 236, 245, 251, 237, 269, 259, 253, 238, 259, 239, 245, 258, 255, 246, 256, 271, 262, 270, 265, 259, 247, 270, 247, 263, 247, 266, 255, 289, 236, 253, 261, 246, 250, 286, 286, 278, 283, 262, 256, 271, 261, 243, 263, 277, 258, 256, 282, 256, 240, 259, 274, 248, 257, 269, 271, 226, 262, 263, 252, 273, 245, 252, 236, 262, 256, 253, 250, 264, 254, 238, 240, 230, 230, 245, 250, 222, 251, 237, 277, 227, 258, 225, 227, 289, 207, 244, 278, 270, 280, 280, 269, 264, 273, 269, 260, 251, 247, 252, 246, 270, 254, 244, 246, 257, 275, 248, 251, 226, 263, 267, 249, 265, 276, 258, 242, 237, 268, 246, 253, 290, 274, 244, 227, 254] payload_rows_per_dst=[5368, 5426, 5434, 5506, 5530, 5440, 5444, 5614] dup_rows_per_dst=[2726, 2610, 2598, 2622, 2820, 2726, 2760, 2912] combine_rows_per_src=[5368, 5426, 5434, 5506, 5530, 5440, 5444, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38206 dup_rows=21956 dup_rows_remote=19318 dedup_ratio=0.335022 router_tokens_per_expert=[285, 260, 275, 278, 264, 260, 248, 252, 262, 260, 253, 273, 236, 266, 230, 253, 230, 260, 229, 239, 264, 237, 284, 267, 276, 251, 254, 231, 264, 269, 255, 272, 277, 273, 253, 248, 220, 267, 290, 256, 235, 257, 254, 236, 277, 247, 247, 282, 258, 255, 288, 251, 250, 262, 241, 284, 253, 248, 248, 254, 259, 252, 267, 261, 223, 270, 274, 265, 252, 268, 242, 242, 245, 242, 234, 253, 253, 265, 237, 251, 261, 256, 250, 241, 230, 253, 227, 241, 251, 265, 234, 259, 261, 269, 255, 253, 270, 242, 255, 290, 264, 287, 235, 276, 250, 236, 214, 256, 227, 260, 283, 273, 237, 265, 274, 276, 250, 258, 295, 262, 265, 267, 269, 262, 244, 215, 250, 252] payload_rows_per_dst=[5494, 5458, 5498, 5458, 5370, 5374, 5470, 5458] dup_rows_per_dst=[2816, 2706, 2740, 2804, 2662, 2638, 2766, 2824] combine_rows_per_src=[5494, 5458, 5498, 5458, 5370, 5374, 5470, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38214 dup_rows=21892 dup_rows_remote=19106 dedup_ratio=0.334045 router_tokens_per_expert=[264, 264, 263, 244, 252, 308, 237, 262, 271, 228, 226, 290, 238, 239, 256, 266, 247, 272, 249, 240, 280, 242, 289, 257, 243, 223, 282, 272, 266, 234, 243, 246, 234, 257, 248, 246, 267, 260, 266, 264, 288, 268, 244, 251, 265, 263, 259, 275, 267, 238, 259, 257, 284, 254, 267, 254, 259, 229, 270, 247, 233, 236, 258, 252, 243, 263, 249, 262, 265, 280, 224, 267, 259, 256, 235, 268, 223, 251, 248, 246, 250, 271, 250, 243, 252, 272, 251, 258, 253, 268, 261, 271, 256, 265, 267, 242, 247, 249, 276, 247, 241, 250, 226, 247, 274, 273, 263, 234, 239, 275, 260, 265, 271, 257, 244, 264, 244, 263, 265, 285, 237, 235, 276, 272, 253, 276, 245, 234] payload_rows_per_dst=[5430, 5510, 5490, 5356, 5430, 5464, 5450, 5514] dup_rows_per_dst=[2786, 2728, 2820, 2704, 2648, 2796, 2682, 2728] combine_rows_per_src=[5430, 5510, 5490, 5356, 5430, 5464, 5450, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228960 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156827232 ingress_bytes=232095744 link_bytes=232095744 token_bytes=313425504 weight_movement_bytes=75497472 total_nvlink_bytes=388922976 weight_to_token_ratio=0.240879 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223827221e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38332 dup_rows=21708 dup_rows_remote=18948 dedup_ratio=0.331238 router_tokens_per_expert=[239, 283, 280, 256, 280, 294, 250, 249, 243, 262, 249, 250, 255, 279, 258, 253, 256, 255, 247, 263, 286, 256, 258, 264, 268, 249, 236, 272, 265, 250, 260, 282, 231, 242, 260, 229, 240, 243, 250, 240, 279, 256, 247, 271, 257, 285, 259, 258, 259, 252, 222, 244, 245, 239, 232, 287, 244, 247, 248, 267, 265, 276, 285, 257, 251, 263, 236, 264, 233, 240, 237, 255, 263, 242, 226, 248, 271, 281, 284, 268, 268, 255, 261, 248, 224, 245, 275, 244, 273, 242, 245, 262, 232, 266, 261, 256, 274, 255, 269, 273, 273, 259, 246, 232, 266, 232, 252, 265, 270, 257, 252, 253, 267, 248, 247, 277, 291, 259, 257, 212, 252, 241, 245, 271, 231, 279, 219, 262] payload_rows_per_dst=[5630, 5450, 5352, 5418, 5532, 5482, 5496, 5468] dup_rows_per_dst=[2730, 2746, 2742, 2720, 2592, 2770, 2760, 2648] combine_rows_per_src=[5630, 5450, 5352, 5418, 5532, 5482, 5496, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=158040064 dispatch_meta_bytes=230512 combine_payload_bytes=158040064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158270576 ingress_bytes=195788800 link_bytes=195788800 token_bytes=316310640 weight_movement_bytes=37748736 total_nvlink_bytes=354059376 weight_to_token_ratio=0.119341 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.875951947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38242 dup_rows=21852 dup_rows_remote=19108 dedup_ratio=0.333435 router_tokens_per_expert=[274, 261, 249, 249, 248, 237, 261, 239, 277, 254, 249, 238, 282, 257, 251, 274, 268, 274, 250, 251, 249, 278, 243, 235, 234, 269, 267, 269, 224, 282, 247, 253, 288, 234, 249, 269, 274, 232, 270, 241, 261, 257, 284, 236, 253, 241, 231, 258, 255, 243, 257, 250, 273, 261, 292, 248, 250, 236, 239, 228, 267, 251, 243, 287, 226, 257, 247, 253, 247, 238, 274, 271, 235, 236, 252, 234, 250, 253, 248, 265, 243, 262, 263, 243, 254, 247, 232, 251, 267, 255, 280, 296, 265, 249, 263, 239, 294, 274, 250, 263, 260, 220, 233, 266, 255, 244, 273, 273, 282, 255, 226, 265, 261, 278, 273, 252, 247, 263, 246, 257, 262, 265, 258, 268, 282, 265, 260, 252] payload_rows_per_dst=[5496, 5442, 5436, 5434, 5444, 5476, 5464, 5492] dup_rows_per_dst=[2704, 2744, 2720, 2726, 2528, 2742, 2802, 2886] combine_rows_per_src=[5496, 5442, 5436, 5434, 5444, 5476, 5464, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38116 dup_rows=21828 dup_rows_remote=18932 dedup_ratio=0.333069 router_tokens_per_expert=[274, 248, 253, 223, 242, 253, 281, 243, 251, 240, 254, 237, 241, 264, 248, 251, 238, 254, 260, 270, 267, 264, 245, 257, 263, 259, 278, 245, 254, 249, 249, 268, 263, 278, 260, 275, 259, 274, 239, 260, 264, 211, 271, 259, 270, 276, 246, 245, 266, 267, 264, 255, 247, 262, 286, 251, 265, 264, 268, 273, 286, 246, 251, 293, 219, 247, 231, 257, 268, 274, 259, 252, 270, 241, 258, 247, 273, 249, 239, 285, 258, 240, 253, 238, 264, 269, 291, 276, 256, 245, 227, 255, 242, 268, 241, 263, 242, 275, 233, 255, 260, 245, 270, 225, 296, 244, 251, 252, 254, 236, 227, 303, 234, 233, 245, 269, 236, 257, 240, 258, 249, 252, 249, 268, 244, 280, 258, 256] payload_rows_per_dst=[5370, 5462, 5502, 5592, 5506, 5464, 5450, 5362] dup_rows_per_dst=[2636, 2778, 2798, 2896, 2632, 2708, 2686, 2694] combine_rows_per_src=[5370, 5462, 5502, 5592, 5506, 5464, 5450, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228040 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156482248 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312736456 weight_movement_bytes=18874368 total_nvlink_bytes=331610824 weight_to_token_ratio=0.060352 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229280 combine_payload_bytes=156524544 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156753824 ingress_bytes=232022016 link_bytes=232022016 token_bytes=313278368 weight_movement_bytes=75497472 total_nvlink_bytes=388775840 weight_to_token_ratio=0.240992 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223120795e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=230096 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721872 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313213648 weight_movement_bytes=18874368 total_nvlink_bytes=332088016 weight_to_token_ratio=0.060260 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229120 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157236992 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314244864 weight_movement_bytes=37748736 total_nvlink_bytes=351993600 weight_to_token_ratio=0.120125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229400 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156868632 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313507864 weight_movement_bytes=37748736 total_nvlink_bytes=351256600 weight_to_token_ratio=0.120408 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=228192 combine_payload_bytes=156123136 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156351328 ingress_bytes=212746240 link_bytes=212746240 token_bytes=312474464 weight_movement_bytes=56623104 total_nvlink_bytes=369097568 weight_to_token_ratio=0.181209 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038429794e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38474 dup_rows=21672 dup_rows_remote=18982 dedup_ratio=0.330688 router_tokens_per_expert=[254, 261, 260, 257, 230, 229, 246, 250, 223, 247, 244, 291, 257, 259, 259, 270, 267, 247, 271, 235, 228, 251, 256, 293, 288, 269, 255, 257, 237, 259, 277, 235, 250, 220, 250, 260, 260, 280, 261, 251, 250, 241, 264, 246, 250, 268, 238, 251, 250, 282, 289, 260, 238, 269, 256, 269, 252, 249, 270, 235, 264, 252, 271, 266, 241, 283, 255, 230, 278, 233, 260, 253, 258, 267, 243, 245, 297, 222, 290, 265, 254, 249, 250, 247, 275, 250, 273, 250, 284, 257, 237, 233, 236, 262, 248, 255, 271, 267, 280, 258, 246, 248, 233, 238, 269, 267, 239, 253, 250, 256, 229, 285, 278, 244, 260, 254, 260, 246, 266, 267, 247, 256, 281, 252, 251, 258, 256, 249] payload_rows_per_dst=[5482, 5480, 5390, 5604, 5530, 5466, 5424, 5488] dup_rows_per_dst=[2592, 2770, 2690, 2740, 2710, 2654, 2754, 2762] combine_rows_per_src=[5482, 5480, 5390, 5604, 5530, 5466, 5424, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38306 dup_rows=21812 dup_rows_remote=19130 dedup_ratio=0.332825 router_tokens_per_expert=[276, 273, 246, 279, 275, 230, 253, 251, 248, 271, 237, 234, 273, 274, 241, 273, 262, 241, 254, 269, 253, 266, 280, 227, 255, 298, 243, 245, 286, 245, 280, 261, 267, 284, 270, 255, 258, 217, 256, 262, 238, 246, 235, 245, 265, 252, 238, 275, 275, 247, 263, 249, 271, 228, 282, 256, 250, 233, 265, 250, 231, 275, 255, 268, 272, 244, 244, 223, 243, 272, 249, 217, 256, 269, 223, 241, 287, 274, 247, 289, 270, 271, 248, 243, 255, 268, 265, 214, 259, 263, 258, 231, 252, 266, 241, 220, 241, 298, 238, 249, 251, 281, 267, 255, 251, 254, 263, 265, 240, 251, 269, 262, 253, 244, 247, 245, 249, 255, 281, 276, 276, 272, 227, 234, 259, 266, 274, 241] payload_rows_per_dst=[5548, 5496, 5366, 5426, 5418, 5478, 5568, 5424] dup_rows_per_dst=[2720, 2834, 2760, 2770, 2682, 2570, 2702, 2774] combine_rows_per_src=[5548, 5496, 5366, 5426, 5418, 5478, 5568, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38130 dup_rows=21972 dup_rows_remote=19174 dedup_ratio=0.335266 router_tokens_per_expert=[288, 265, 248, 262, 225, 283, 259, 261, 269, 242, 286, 251, 236, 235, 277, 245, 261, 262, 260, 291, 243, 256, 276, 252, 248, 269, 273, 249, 251, 247, 249, 271, 239, 254, 232, 246, 258, 248, 287, 243, 278, 241, 271, 232, 266, 259, 248, 250, 254, 265, 245, 249, 266, 265, 227, 245, 249, 257, 266, 277, 263, 268, 275, 273, 248, 247, 251, 270, 261, 255, 243, 239, 273, 270, 273, 241, 259, 252, 274, 242, 261, 241, 254, 259, 244, 251, 246, 260, 257, 272, 247, 257, 229, 247, 257, 266, 240, 249, 228, 281, 242, 224, 258, 247, 245, 281, 253, 258, 264, 251, 253, 276, 238, 252, 234, 238, 276, 254, 254, 267, 240, 285, 235, 271, 263, 269, 246, 264] payload_rows_per_dst=[5480, 5330, 5412, 5436, 5448, 5568, 5456, 5434] dup_rows_per_dst=[2784, 2448, 2692, 2790, 2748, 3066, 2646, 2798] combine_rows_per_src=[5480, 5330, 5412, 5436, 5448, 5568, 5456, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157589504 dispatch_meta_bytes=229824 combine_payload_bytes=157589504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157819328 ingress_bytes=176463872 link_bytes=176463872 token_bytes=315408832 weight_movement_bytes=18874368 total_nvlink_bytes=334283200 weight_to_token_ratio=0.059841 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690789996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38270 dup_rows=21794 dup_rows_remote=19108 dedup_ratio=0.332550 router_tokens_per_expert=[262, 277, 256, 262, 269, 239, 253, 254, 237, 253, 267, 241, 233, 247, 248, 241, 241, 261, 258, 277, 268, 247, 213, 254, 243, 282, 267, 248, 267, 241, 255, 271, 282, 270, 249, 254, 250, 226, 232, 272, 247, 280, 245, 269, 255, 268, 255, 257, 235, 223, 258, 249, 274, 271, 261, 253, 223, 249, 275, 234, 252, 265, 263, 269, 236, 251, 251, 250, 257, 256, 255, 266, 277, 242, 239, 260, 231, 261, 248, 280, 258, 271, 245, 253, 254, 243, 253, 242, 267, 242, 287, 248, 262, 252, 254, 248, 286, 272, 257, 273, 259, 266, 255, 256, 266, 260, 252, 255, 256, 259, 253, 275, 265, 246, 270, 246, 261, 257, 252, 268, 264, 259, 271, 251, 262, 266, 247, 247] payload_rows_per_dst=[5396, 5476, 5482, 5412, 5400, 5472, 5552, 5552] dup_rows_per_dst=[2682, 2710, 2740, 2696, 2720, 2686, 2848, 2712] combine_rows_per_src=[5396, 5476, 5482, 5412, 5400, 5472, 5552, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229744 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157131120 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032496 weight_movement_bytes=18874368 total_nvlink_bytes=332906864 weight_to_token_ratio=0.060103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38106 dup_rows=21946 dup_rows_remote=19118 dedup_ratio=0.334869 router_tokens_per_expert=[273, 251, 269, 271, 247, 232, 257, 263, 269, 262, 256, 271, 255, 255, 268, 253, 255, 247, 249, 256, 260, 254, 243, 275, 269, 268, 261, 267, 265, 246, 262, 259, 264, 246, 229, 278, 222, 238, 246, 268, 282, 290, 252, 238, 281, 266, 271, 281, 249, 231, 249, 282, 273, 239, 274, 261, 256, 253, 259, 280, 289, 245, 235, 255, 243, 267, 251, 228, 251, 254, 258, 242, 233, 260, 255, 273, 249, 228, 264, 257, 260, 267, 248, 259, 243, 245, 242, 235, 266, 235, 240, 252, 213, 272, 233, 235, 255, 276, 283, 250, 226, 260, 270, 255, 268, 245, 257, 250, 260, 264, 264, 273, 261, 260, 251, 276, 247, 239, 255, 266, 234, 273, 237, 256, 253, 261, 252, 263] payload_rows_per_dst=[5510, 5486, 5530, 5460, 5380, 5300, 5484, 5440] dup_rows_per_dst=[2794, 2786, 2774, 2800, 2646, 2590, 2828, 2728] combine_rows_per_src=[5510, 5486, 5530, 5460, 5380, 5300, 5484, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38254 dup_rows=21790 dup_rows_remote=19018 dedup_ratio=0.332489 router_tokens_per_expert=[234, 249, 263, 260, 269, 248, 280, 255, 232, 229, 247, 227, 250, 270, 261, 248, 260, 263, 259, 241, 269, 260, 253, 255, 237, 280, 227, 283, 283, 264, 272, 249, 269, 246, 247, 258, 246, 282, 264, 253, 267, 233, 240, 259, 239, 257, 263, 275, 263, 266, 236, 240, 246, 241, 282, 284, 241, 251, 247, 241, 276, 248, 246, 229, 274, 266, 241, 244, 250, 244, 265, 268, 261, 267, 273, 260, 223, 276, 247, 266, 255, 275, 254, 227, 244, 253, 256, 275, 257, 255, 252, 244, 258, 247, 310, 213, 258, 255, 262, 279, 244, 265, 266, 241, 246, 259, 247, 225, 224, 240, 262, 283, 255, 274, 261, 241, 269, 246, 268, 248, 271, 270, 270, 245, 256, 274, 261, 291] payload_rows_per_dst=[5492, 5514, 5440, 5600, 5480, 5188, 5452, 5580] dup_rows_per_dst=[2772, 2800, 2756, 2878, 2766, 2338, 2660, 2820] combine_rows_per_src=[5492, 5514, 5440, 5600, 5480, 5188, 5452, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38372 dup_rows=21744 dup_rows_remote=19088 dedup_ratio=0.331787 router_tokens_per_expert=[264, 275, 242, 252, 264, 274, 274, 263, 221, 265, 256, 254, 235, 255, 276, 229, 244, 262, 273, 253, 240, 260, 236, 235, 270, 252, 224, 263, 250, 256, 272, 248, 235, 248, 218, 264, 254, 247, 254, 255, 236, 246, 271, 256, 251, 271, 245, 287, 271, 227, 247, 258, 248, 262, 269, 257, 240, 247, 290, 298, 267, 259, 239, 237, 246, 256, 254, 251, 276, 241, 262, 243, 240, 267, 292, 235, 249, 289, 263, 222, 246, 281, 249, 252, 255, 271, 299, 259, 256, 277, 250, 236, 251, 267, 274, 266, 235, 277, 235, 259, 276, 262, 267, 243, 281, 249, 265, 263, 280, 272, 261, 265, 253, 257, 259, 273, 233, 256, 234, 235, 264, 266, 253, 227, 238, 253, 255, 256] payload_rows_per_dst=[5500, 5420, 5378, 5510, 5488, 5528, 5572, 5396] dup_rows_per_dst=[2698, 2656, 2698, 2722, 2684, 2850, 2808, 2628] combine_rows_per_src=[5500, 5420, 5378, 5510, 5488, 5528, 5572, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38346 dup_rows=21742 dup_rows_remote=19092 dedup_ratio=0.331757 router_tokens_per_expert=[260, 248, 264, 252, 249, 254, 270, 271, 256, 256, 247, 260, 267, 263, 233, 270, 271, 264, 239, 250, 265, 255, 245, 247, 273, 263, 262, 237, 241, 264, 266, 280, 256, 273, 231, 269, 247, 268, 237, 235, 259, 254, 244, 261, 274, 246, 270, 235, 238, 238, 251, 249, 275, 244, 250, 252, 253, 272, 244, 240, 257, 258, 263, 265, 264, 260, 264, 247, 268, 247, 245, 246, 227, 247, 248, 224, 241, 253, 261, 259, 255, 254, 265, 260, 247, 246, 256, 273, 255, 244, 280, 271, 240, 263, 241, 265, 257, 275, 248, 274, 228, 248, 254, 269, 293, 250, 249, 262, 271, 261, 262, 273, 261, 255, 259, 271, 245, 260, 265, 266, 270, 252, 250, 275, 235, 243, 250, 271] payload_rows_per_dst=[5478, 5518, 5534, 5448, 5418, 5454, 5510, 5434] dup_rows_per_dst=[2762, 2726, 2584, 2650, 2584, 2776, 2838, 2822] combine_rows_per_src=[5478, 5518, 5534, 5448, 5418, 5454, 5510, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229216 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156409696 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312590176 weight_movement_bytes=37748736 total_nvlink_bytes=350338912 weight_to_token_ratio=0.120761 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229512 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983432 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737352 weight_movement_bytes=18874368 total_nvlink_bytes=332611720 weight_to_token_ratio=0.060160 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228896 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156311072 ingress_bytes=212705280 link_bytes=212705280 token_bytes=312393248 weight_movement_bytes=56623104 total_nvlink_bytes=369016352 weight_to_token_ratio=0.181256 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038037335e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229088 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917472 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605856 weight_movement_bytes=18874368 total_nvlink_bytes=332480224 weight_to_token_ratio=0.060185 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229840 combine_payload_bytes=157171712 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157401552 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314573264 weight_movement_bytes=18874368 total_nvlink_bytes=333447632 weight_to_token_ratio=0.060000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229752 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157294968 ingress_bytes=194813952 link_bytes=194813952 token_bytes=314360184 weight_movement_bytes=37748736 total_nvlink_bytes=352108920 weight_to_token_ratio=0.120081 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866611434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38234 dup_rows=21854 dup_rows_remote=19162 dedup_ratio=0.333466 router_tokens_per_expert=[248, 264, 252, 239, 228, 278, 259, 271, 269, 285, 271, 243, 255, 233, 268, 255, 248, 255, 244, 253, 258, 248, 256, 253, 279, 242, 271, 285, 257, 244, 274, 238, 265, 243, 251, 260, 236, 260, 264, 261, 263, 249, 255, 263, 272, 242, 234, 252, 268, 263, 268, 247, 260, 239, 243, 261, 275, 252, 277, 272, 252, 275, 243, 235, 280, 257, 245, 254, 237, 275, 252, 277, 249, 288, 243, 273, 235, 243, 246, 253, 236, 253, 241, 268, 254, 254, 279, 280, 301, 257, 237, 262, 268, 224, 232, 252, 260, 238, 257, 259, 248, 263, 260, 209, 261, 261, 259, 256, 268, 273, 231, 227, 288, 243, 285, 258, 259, 264, 215, 265, 231, 244, 263, 267, 261, 259, 246, 262] payload_rows_per_dst=[5472, 5476, 5448, 5514, 5466, 5518, 5328, 5460] dup_rows_per_dst=[2764, 2734, 2692, 2746, 2748, 2678, 2732, 2760] combine_rows_per_src=[5472, 5476, 5448, 5514, 5466, 5518, 5328, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43530 payload_rows_remote=38080 dup_rows=22006 dup_rows_remote=19152 dedup_ratio=0.335785 router_tokens_per_expert=[252, 228, 270, 262, 282, 237, 250, 264, 238, 250, 250, 264, 256, 243, 260, 253, 259, 282, 271, 250, 270, 263, 269, 263, 234, 265, 259, 232, 234, 263, 242, 253, 273, 265, 251, 254, 254, 230, 240, 284, 249, 244, 281, 266, 250, 263, 249, 272, 256, 254, 273, 250, 236, 258, 232, 271, 279, 255, 256, 255, 276, 278, 245, 245, 283, 272, 257, 262, 255, 270, 247, 261, 263, 255, 259, 270, 253, 237, 269, 239, 269, 274, 253, 249, 251, 264, 252, 251, 241, 244, 243, 244, 256, 253, 247, 273, 222, 274, 248, 258, 257, 252, 240, 239, 271, 258, 269, 267, 256, 243, 251, 264, 269, 263, 256, 254, 247, 267, 242, 245, 265, 250, 241, 263, 258, 245, 259, 247] payload_rows_per_dst=[5430, 5406, 5440, 5512, 5450, 5420, 5410, 5462] dup_rows_per_dst=[2688, 2812, 2810, 2726, 2854, 2708, 2728, 2680] combine_rows_per_src=[5430, 5406, 5440, 5512, 5450, 5420, 5410, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229584 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156836048 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313442512 weight_movement_bytes=18874368 total_nvlink_bytes=332316880 weight_to_token_ratio=0.060216 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38122 dup_rows=21830 dup_rows_remote=19018 dedup_ratio=0.333099 router_tokens_per_expert=[246, 237, 236, 257, 242, 292, 246, 259, 267, 257, 235, 253, 248, 247, 239, 265, 278, 253, 257, 226, 253, 246, 257, 233, 269, 266, 263, 270, 263, 257, 267, 250, 245, 256, 278, 253, 260, 281, 255, 234, 265, 240, 250, 245, 282, 248, 272, 276, 249, 259, 286, 259, 288, 251, 251, 272, 240, 263, 227, 262, 238, 271, 243, 239, 227, 260, 261, 265, 278, 258, 275, 240, 255, 268, 249, 244, 271, 269, 244, 238, 248, 228, 268, 258, 264, 264, 244, 289, 229, 248, 273, 240, 278, 260, 240, 238, 260, 275, 260, 262, 257, 255, 259, 282, 238, 262, 238, 262, 273, 247, 267, 247, 247, 247, 224, 273, 255, 239, 272, 250, 270, 263, 256, 261, 233, 269, 253, 269] payload_rows_per_dst=[5374, 5392, 5554, 5310, 5452, 5576, 5464, 5584] dup_rows_per_dst=[2678, 2682, 2868, 2630, 2752, 2786, 2622, 2812] combine_rows_per_src=[5374, 5392, 5554, 5310, 5452, 5576, 5464, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38328 dup_rows=21664 dup_rows_remote=18874 dedup_ratio=0.330566 router_tokens_per_expert=[244, 265, 251, 265, 285, 238, 281, 267, 273, 256, 248, 265, 272, 265, 267, 255, 247, 221, 255, 286, 236, 250, 241, 250, 277, 266, 249, 249, 240, 275, 293, 255, 253, 264, 256, 259, 276, 282, 227, 226, 238, 222, 227, 250, 243, 266, 247, 258, 256, 266, 250, 268, 244, 256, 262, 235, 252, 241, 262, 259, 262, 253, 234, 257, 247, 221, 276, 293, 243, 271, 280, 268, 259, 234, 278, 225, 234, 245, 217, 249, 269, 254, 273, 276, 249, 276, 261, 247, 241, 233, 271, 259, 264, 268, 263, 263, 263, 229, 234, 273, 271, 244, 265, 249, 268, 295, 235, 259, 268, 239, 255, 280, 246, 281, 253, 251, 259, 260, 250, 264, 267, 237, 266, 252, 278, 253, 247, 232] payload_rows_per_dst=[5554, 5408, 5446, 5452, 5458, 5544, 5526, 5484] dup_rows_per_dst=[2840, 2772, 2542, 2662, 2622, 2790, 2728, 2708] combine_rows_per_src=[5554, 5408, 5446, 5452, 5458, 5544, 5526, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228928 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156204608 ingress_bytes=174850048 link_bytes=174850048 token_bytes=312180288 weight_movement_bytes=18874368 total_nvlink_bytes=331054656 weight_to_token_ratio=0.060460 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675327128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38150 dup_rows=21812 dup_rows_remote=19016 dedup_ratio=0.332825 router_tokens_per_expert=[266, 247, 270, 265, 232, 281, 271, 271, 279, 233, 266, 227, 242, 254, 267, 247, 279, 250, 237, 220, 282, 257, 257, 243, 277, 230, 253, 234, 253, 257, 260, 253, 262, 274, 243, 242, 273, 269, 270, 255, 256, 259, 244, 267, 267, 253, 259, 252, 265, 264, 235, 257, 242, 256, 256, 224, 250, 267, 211, 241, 280, 280, 257, 269, 278, 257, 253, 261, 277, 252, 244, 272, 244, 232, 229, 234, 288, 234, 221, 263, 226, 257, 243, 245, 251, 255, 266, 268, 259, 275, 247, 247, 252, 263, 253, 267, 265, 254, 254, 270, 271, 280, 253, 263, 227, 252, 270, 276, 234, 243, 253, 256, 261, 249, 269, 272, 262, 229, 254, 254, 277, 264, 270, 273, 250, 269, 273, 249] payload_rows_per_dst=[5554, 5400, 5444, 5370, 5400, 5438, 5574, 5544] dup_rows_per_dst=[2682, 2684, 2718, 2738, 2678, 2710, 2796, 2806] combine_rows_per_src=[5554, 5400, 5444, 5370, 5400, 5438, 5574, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38254 dup_rows=21814 dup_rows_remote=19080 dedup_ratio=0.332855 router_tokens_per_expert=[282, 262, 274, 250, 249, 256, 236, 276, 269, 263, 260, 273, 272, 262, 262, 272, 251, 258, 249, 234, 219, 252, 245, 266, 237, 244, 279, 262, 260, 240, 269, 253, 257, 233, 249, 257, 241, 252, 250, 233, 241, 261, 252, 287, 245, 242, 271, 268, 255, 251, 246, 256, 272, 265, 272, 251, 249, 249, 270, 260, 255, 251, 263, 236, 270, 239, 277, 271, 250, 262, 244, 247, 255, 253, 247, 252, 224, 267, 249, 244, 244, 264, 225, 216, 259, 264, 223, 241, 238, 295, 286, 243, 248, 257, 298, 258, 256, 251, 252, 264, 255, 256, 244, 261, 256, 276, 252, 247, 244, 244, 270, 260, 251, 256, 251, 274, 268, 256, 256, 254, 250, 274, 258, 262, 262, 279, 263, 280] payload_rows_per_dst=[5602, 5364, 5474, 5468, 5320, 5434, 5460, 5600] dup_rows_per_dst=[2834, 2672, 2604, 2734, 2782, 2684, 2716, 2788] combine_rows_per_src=[5602, 5364, 5474, 5468, 5320, 5434, 5460, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38294 dup_rows=21692 dup_rows_remote=19008 dedup_ratio=0.330994 router_tokens_per_expert=[241, 270, 262, 249, 286, 264, 252, 282, 266, 264, 293, 261, 265, 276, 238, 258, 277, 259, 243, 258, 268, 292, 240, 252, 260, 218, 255, 247, 266, 250, 258, 274, 263, 255, 237, 260, 266, 268, 238, 250, 240, 255, 222, 248, 255, 249, 258, 255, 263, 284, 278, 264, 259, 272, 274, 246, 273, 250, 267, 214, 248, 252, 267, 252, 236, 246, 216, 275, 247, 237, 255, 258, 259, 240, 250, 245, 235, 254, 254, 260, 269, 273, 241, 259, 263, 268, 241, 243, 274, 224, 262, 246, 240, 250, 254, 260, 237, 250, 255, 259, 238, 245, 241, 279, 236, 260, 286, 240, 260, 266, 259, 251, 258, 251, 260, 250, 261, 263, 277, 260, 258, 246, 251, 254, 271, 246, 288, 252] payload_rows_per_dst=[5510, 5550, 5410, 5572, 5424, 5440, 5478, 5460] dup_rows_per_dst=[2944, 2684, 2628, 2754, 2622, 2694, 2646, 2720] combine_rows_per_src=[5510, 5550, 5410, 5572, 5424, 5440, 5478, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38128 dup_rows=21844 dup_rows_remote=18896 dedup_ratio=0.333313 router_tokens_per_expert=[236, 244, 249, 266, 261, 263, 257, 256, 215, 239, 255, 249, 242, 241, 273, 263, 272, 281, 240, 245, 294, 257, 264, 245, 264, 270, 260, 269, 244, 236, 242, 253, 252, 267, 262, 262, 246, 222, 230, 257, 255, 232, 257, 275, 241, 234, 259, 251, 244, 261, 247, 232, 259, 260, 275, 268, 259, 245, 258, 283, 255, 227, 257, 237, 262, 259, 247, 238, 276, 275, 268, 268, 236, 265, 266, 290, 247, 275, 275, 269, 271, 270, 256, 274, 236, 267, 230, 279, 252, 280, 249, 245, 276, 264, 231, 243, 289, 240, 245, 236, 270, 240, 272, 245, 263, 259, 253, 254, 240, 253, 278, 250, 266, 249, 280, 241, 263, 244, 287, 265, 255, 242, 243, 255, 273, 232, 271, 262] payload_rows_per_dst=[5564, 5390, 5464, 5402, 5574, 5366, 5408, 5524] dup_rows_per_dst=[2948, 2550, 2738, 2534, 2908, 2668, 2766, 2732] combine_rows_per_src=[5564, 5390, 5464, 5402, 5574, 5366, 5408, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228560 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376272 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312523984 weight_movement_bytes=18874368 total_nvlink_bytes=331398352 weight_to_token_ratio=0.060393 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=228808 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220296 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314211784 weight_movement_bytes=18874368 total_nvlink_bytes=333086152 weight_to_token_ratio=0.060069 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228664 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156491064 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753464 weight_movement_bytes=18874368 total_nvlink_bytes=331627832 weight_to_token_ratio=0.060349 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229336 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156917720 ingress_bytes=232185856 link_bytes=232185856 token_bytes=313606104 weight_movement_bytes=75497472 total_nvlink_bytes=389103576 weight_to_token_ratio=0.240740 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224690630e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229208 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081432 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933656 weight_movement_bytes=18874368 total_nvlink_bytes=332808024 weight_to_token_ratio=0.060122 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=228096 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156400384 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312572672 weight_movement_bytes=18874368 total_nvlink_bytes=331447040 weight_to_token_ratio=0.060384 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38212 dup_rows=21946 dup_rows_remote=19158 dedup_ratio=0.334869 router_tokens_per_expert=[237, 255, 243, 258, 250, 235, 254, 242, 244, 259, 296, 256, 280, 229, 241, 242, 258, 249, 269, 274, 271, 249, 264, 244, 214, 277, 252, 245, 253, 267, 256, 278, 268, 252, 255, 238, 266, 267, 255, 256, 255, 257, 262, 250, 242, 257, 266, 237, 255, 239, 280, 260, 270, 261, 257, 245, 280, 287, 232, 234, 265, 261, 247, 240, 252, 242, 255, 263, 263, 266, 241, 262, 254, 249, 256, 260, 240, 273, 258, 255, 258, 236, 266, 275, 260, 263, 255, 249, 263, 282, 265, 251, 254, 271, 266, 274, 229, 246, 237, 230, 255, 262, 263, 274, 280, 252, 239, 253, 250, 265, 216, 258, 270, 236, 270, 259, 256, 276, 233, 247, 263, 273, 259, 266, 282, 246, 245, 264] payload_rows_per_dst=[5406, 5492, 5378, 5446, 5508, 5454, 5414, 5492] dup_rows_per_dst=[2636, 2748, 2788, 2780, 2670, 2922, 2604, 2798] combine_rows_per_src=[5406, 5492, 5378, 5446, 5508, 5454, 5414, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38422 dup_rows=21608 dup_rows_remote=18792 dedup_ratio=0.329712 router_tokens_per_expert=[235, 265, 234, 278, 254, 238, 256, 249, 261, 234, 274, 273, 264, 251, 260, 231, 276, 252, 264, 256, 229, 293, 233, 240, 257, 267, 273, 229, 274, 239, 243, 274, 258, 248, 277, 253, 244, 242, 237, 273, 254, 269, 237, 256, 258, 260, 244, 246, 274, 259, 265, 238, 254, 246, 290, 256, 239, 251, 265, 244, 254, 246, 242, 228, 256, 271, 250, 267, 268, 235, 270, 253, 262, 272, 271, 281, 244, 266, 244, 251, 256, 233, 282, 247, 272, 270, 252, 230, 277, 262, 253, 235, 243, 264, 232, 248, 267, 252, 277, 240, 239, 257, 258, 305, 236, 271, 257, 254, 267, 279, 235, 292, 250, 231, 237, 273, 268, 244, 264, 258, 241, 271, 269, 238, 262, 252, 284, 260] payload_rows_per_dst=[5480, 5540, 5454, 5384, 5506, 5460, 5604, 5500] dup_rows_per_dst=[2634, 2658, 2658, 2718, 2816, 2652, 2768, 2704] combine_rows_per_src=[5480, 5540, 5454, 5384, 5506, 5460, 5604, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15060027760 prefetch_bytes=736100352 reduce_bytes=924844032 weight_movement_bytes=1660944384 egress_bytes=7535520112 ingress_bytes=9185452032 link_bytes=9185452032 total_nvlink_bytes=16720972144 weight_to_token_ratio=0.110288 prefetch_share_of_ingress=0.080138 dedup_ratio=0.332682 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=49 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081742 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.833551705e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229480 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156745832 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313262184 weight_movement_bytes=37748736 total_nvlink_bytes=351010920 weight_to_token_ratio=0.120502 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43508 payload_rows_remote=38062 dup_rows=22028 dup_rows_remote=19162 dedup_ratio=0.336121 router_tokens_per_expert=[249, 267, 251, 301, 248, 266, 274, 276, 251, 280, 254, 255, 251, 251, 269, 261, 259, 239, 249, 247, 236, 228, 249, 295, 257, 281, 253, 245, 261, 246, 248, 256, 253, 270, 269, 285, 244, 236, 253, 283, 258, 252, 262, 251, 248, 248, 244, 283, 232, 247, 276, 267, 278, 230, 243, 278, 269, 236, 280, 266, 266, 267, 266, 228, 259, 244, 255, 219, 243, 278, 241, 257, 252, 284, 260, 247, 273, 248, 224, 250, 262, 270, 274, 254, 262, 240, 252, 244, 254, 242, 256, 266, 228, 271, 249, 284, 274, 260, 239, 250, 256, 249, 255, 249, 254, 235, 246, 223, 248, 233, 250, 232, 274, 242, 279, 275, 253, 274, 272, 279, 272, 247, 223, 254, 273, 238, 230, 267] payload_rows_per_dst=[5376, 5398, 5398, 5494, 5402, 5486, 5508, 5446] dup_rows_per_dst=[2594, 2700, 2778, 2764, 2666, 2722, 2938, 2866] combine_rows_per_src=[5376, 5398, 5398, 5494, 5402, 5486, 5508, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=228856 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157605368 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314981880 weight_movement_bytes=18874368 total_nvlink_bytes=333856248 weight_to_token_ratio=0.059922 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38204 dup_rows=21808 dup_rows_remote=19034 dedup_ratio=0.332764 router_tokens_per_expert=[269, 240, 235, 275, 245, 264, 270, 277, 250, 266, 263, 239, 243, 246, 277, 255, 269, 241, 249, 255, 258, 246, 249, 256, 293, 268, 259, 265, 275, 245, 246, 235, 248, 265, 244, 252, 238, 262, 282, 251, 227, 249, 254, 272, 255, 245, 280, 261, 263, 284, 261, 271, 246, 268, 248, 274, 249, 237, 265, 258, 259, 257, 255, 249, 279, 234, 246, 259, 236, 259, 253, 241, 262, 239, 271, 232, 254, 257, 255, 245, 270, 276, 243, 270, 252, 241, 248, 249, 274, 234, 243, 235, 245, 260, 265, 268, 249, 264, 248, 245, 257, 241, 253, 265, 274, 249, 289, 279, 262, 234, 248, 292, 282, 252, 264, 222, 262, 231, 255, 269, 239, 275, 250, 270, 257, 254, 251, 239] payload_rows_per_dst=[5532, 5436, 5432, 5498, 5408, 5472, 5524, 5426] dup_rows_per_dst=[2666, 2782, 2738, 2790, 2636, 2704, 2774, 2718] combine_rows_per_src=[5532, 5436, 5432, 5498, 5408, 5472, 5524, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38404 dup_rows=21764 dup_rows_remote=19094 dedup_ratio=0.332092 router_tokens_per_expert=[257, 242, 241, 268, 266, 245, 254, 255, 269, 246, 261, 266, 238, 245, 235, 284, 248, 255, 284, 265, 253, 265, 261, 246, 281, 262, 264, 258, 242, 273, 211, 258, 270, 248, 292, 251, 263, 268, 240, 276, 279, 274, 246, 235, 255, 263, 251, 241, 305, 251, 210, 251, 258, 247, 241, 273, 257, 254, 272, 254, 283, 239, 251, 259, 256, 271, 262, 242, 260, 256, 276, 257, 254, 259, 223, 237, 250, 280, 249, 295, 253, 247, 219, 255, 254, 271, 272, 234, 253, 239, 260, 251, 240, 272, 249, 250, 223, 271, 223, 267, 257, 248, 236, 240, 256, 275, 265, 254, 275, 238, 254, 240, 288, 247, 269, 268, 230, 257, 260, 262, 279, 277, 263, 218, 262, 241, 274, 250] payload_rows_per_dst=[5434, 5508, 5540, 5462, 5490, 5368, 5446, 5524] dup_rows_per_dst=[2710, 2744, 2764, 2748, 2764, 2670, 2598, 2766] combine_rows_per_src=[5434, 5508, 5540, 5462, 5490, 5368, 5446, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38206 dup_rows=21834 dup_rows_remote=19188 dedup_ratio=0.333160 router_tokens_per_expert=[290, 265, 231, 238, 285, 249, 262, 284, 210, 268, 242, 258, 256, 253, 255, 250, 272, 247, 266, 242, 253, 233, 245, 233, 271, 252, 268, 267, 269, 248, 231, 274, 246, 273, 258, 255, 269, 241, 252, 286, 254, 267, 250, 250, 230, 271, 272, 250, 255, 254, 263, 256, 287, 249, 232, 250, 244, 250, 251, 275, 278, 238, 269, 253, 253, 239, 261, 261, 255, 253, 269, 246, 270, 249, 230, 285, 272, 249, 253, 263, 250, 275, 237, 266, 233, 248, 245, 256, 263, 310, 255, 269, 266, 261, 255, 230, 252, 263, 254, 239, 262, 246, 274, 264, 281, 250, 239, 252, 253, 250, 278, 259, 217, 254, 268, 270, 264, 243, 241, 245, 240, 265, 246, 266, 252, 262, 249, 248] payload_rows_per_dst=[5518, 5496, 5530, 5506, 5444, 5388, 5428, 5392] dup_rows_per_dst=[2674, 2646, 2718, 2702, 2772, 2850, 2804, 2668] combine_rows_per_src=[5518, 5496, 5530, 5506, 5444, 5388, 5428, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38444 dup_rows=21666 dup_rows_remote=18876 dedup_ratio=0.330597 router_tokens_per_expert=[229, 234, 270, 239, 273, 237, 276, 264, 297, 259, 244, 258, 254, 245, 254, 275, 242, 282, 235, 236, 241, 238, 268, 279, 249, 254, 245, 253, 265, 267, 265, 258, 251, 283, 230, 274, 238, 233, 228, 255, 283, 255, 296, 287, 264, 283, 272, 246, 244, 251, 249, 268, 275, 271, 263, 279, 266, 263, 263, 271, 255, 251, 255, 262, 237, 274, 239, 249, 271, 254, 237, 259, 269, 259, 237, 268, 278, 245, 238, 259, 247, 281, 254, 248, 240, 241, 238, 234, 268, 252, 242, 242, 238, 236, 242, 268, 290, 260, 219, 238, 255, 261, 244, 236, 259, 242, 285, 255, 271, 241, 226, 261, 254, 255, 241, 266, 266, 257, 271, 257, 248, 269, 245, 276, 260, 275, 276, 216] payload_rows_per_dst=[5426, 5536, 5522, 5562, 5478, 5534, 5344, 5468] dup_rows_per_dst=[2790, 2642, 2810, 2810, 2668, 2672, 2478, 2796] combine_rows_per_src=[5426, 5536, 5522, 5562, 5478, 5534, 5344, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228896 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156130848 ingress_bytes=212525056 link_bytes=212525056 token_bytes=312032800 weight_movement_bytes=56623104 total_nvlink_bytes=368655904 weight_to_token_ratio=0.181465 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036310518e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228952 combine_payload_bytes=156483584 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156712536 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196120 weight_movement_bytes=37748736 total_nvlink_bytes=350944856 weight_to_token_ratio=0.120527 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229992 combine_payload_bytes=157302784 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157532776 ingress_bytes=195051520 link_bytes=195051520 token_bytes=314835560 weight_movement_bytes=37748736 total_nvlink_bytes=352584296 weight_to_token_ratio=0.119900 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868887693e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229576 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721352 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313213128 weight_movement_bytes=18874368 total_nvlink_bytes=332087496 weight_to_token_ratio=0.060260 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38244 dup_rows=21826 dup_rows_remote=19036 dedup_ratio=0.333038 router_tokens_per_expert=[256, 246, 258, 266, 258, 262, 249, 284, 259, 269, 258, 266, 260, 248, 260, 250, 269, 247, 251, 276, 253, 275, 259, 260, 265, 223, 239, 257, 294, 241, 277, 238, 263, 262, 232, 250, 276, 233, 257, 243, 269, 235, 275, 270, 267, 262, 279, 255, 236, 257, 270, 245, 252, 261, 229, 260, 263, 257, 272, 237, 253, 249, 260, 258, 241, 251, 247, 257, 249, 265, 250, 259, 259, 264, 259, 235, 252, 252, 256, 235, 272, 261, 253, 254, 248, 267, 285, 250, 251, 243, 241, 233, 250, 219, 280, 259, 260, 275, 264, 242, 275, 266, 264, 260, 248, 223, 260, 269, 252, 245, 289, 245, 246, 258, 258, 239, 258, 244, 247, 243, 232, 282, 262, 251, 285, 241, 256, 272] payload_rows_per_dst=[5472, 5496, 5466, 5432, 5396, 5400, 5558, 5490] dup_rows_per_dst=[2826, 2752, 2790, 2686, 2666, 2732, 2716, 2658] combine_rows_per_src=[5472, 5496, 5466, 5432, 5396, 5400, 5558, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229280 combine_payload_bytes=157466624 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157695904 ingress_bytes=176340992 link_bytes=176340992 token_bytes=315162528 weight_movement_bytes=18874368 total_nvlink_bytes=334036896 weight_to_token_ratio=0.059888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689612620e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15045081344 prefetch_bytes=811597824 reduce_bytes=660602880 weight_movement_bytes=1472200704 egress_bytes=7528044800 ingress_bytes=8989237248 link_bytes=8989237248 total_nvlink_bytes=16517282048 weight_to_token_ratio=0.097853 prefetch_share_of_ingress=0.090286 dedup_ratio=0.333043 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=35 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.794384340e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38204 dup_rows=21840 dup_rows_remote=18982 dedup_ratio=0.333252 router_tokens_per_expert=[276, 260, 243, 249, 213, 264, 238, 264, 261, 240, 256, 248, 254, 258, 250, 253, 243, 247, 236, 250, 299, 279, 265, 273, 236, 253, 262, 222, 252, 253, 241, 241, 262, 265, 279, 240, 242, 266, 279, 280, 249, 263, 224, 262, 241, 274, 259, 270, 236, 247, 239, 256, 239, 258, 273, 240, 243, 273, 268, 253, 235, 249, 263, 248, 263, 264, 245, 276, 244, 255, 260, 280, 256, 255, 254, 272, 284, 245, 258, 264, 244, 259, 274, 259, 250, 277, 263, 282, 251, 232, 294, 248, 252, 244, 261, 257, 261, 241, 246, 269, 255, 256, 265, 254, 254, 252, 236, 259, 256, 242, 268, 234, 254, 271, 272, 239, 249, 254, 267, 272, 267, 257, 246, 258, 255, 247, 265, 271] payload_rows_per_dst=[5394, 5446, 5522, 5406, 5492, 5458, 5468, 5510] dup_rows_per_dst=[2660, 2658, 2788, 2634, 2858, 2836, 2628, 2778] combine_rows_per_src=[5394, 5446, 5522, 5406, 5492, 5458, 5468, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229120 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876544 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313523968 weight_movement_bytes=18874368 total_nvlink_bytes=332398336 weight_to_token_ratio=0.060201 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15063139968 prefetch_bytes=698351616 reduce_bytes=1113587712 weight_movement_bytes=1811939328 egress_bytes=7537075840 ingress_bytes=9338003456 link_bytes=9338003456 total_nvlink_bytes=16875079296 weight_to_token_ratio=0.120290 prefetch_share_of_ingress=0.074786 dedup_ratio=0.332357 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=59 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=108331 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.864003219e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43490 payload_rows_remote=37952 dup_rows=22046 dup_rows_remote=19154 dedup_ratio=0.336395 router_tokens_per_expert=[280, 248, 261, 242, 247, 249, 274, 239, 277, 241, 235, 253, 227, 268, 239, 241, 258, 247, 278, 247, 270, 237, 275, 266, 259, 272, 271, 234, 269, 261, 263, 294, 250, 276, 267, 274, 272, 250, 253, 226, 234, 266, 244, 270, 263, 254, 260, 254, 245, 267, 274, 266, 265, 246, 257, 284, 240, 254, 250, 243, 236, 273, 262, 244, 254, 242, 226, 244, 249, 263, 240, 262, 239, 258, 270, 244, 270, 243, 264, 268, 244, 255, 271, 238, 243, 229, 243, 278, 237, 254, 261, 269, 259, 234, 227, 242, 253, 261, 233, 256, 265, 240, 261, 285, 253, 264, 258, 230, 265, 247, 285, 256, 242, 240, 270, 279, 268, 262, 253, 276, 263, 270, 258, 259, 263, 263, 286, 243] payload_rows_per_dst=[5224, 5460, 5444, 5350, 5480, 5552, 5442, 5538] dup_rows_per_dst=[2258, 2942, 2782, 2624, 2790, 2976, 2782, 2892] combine_rows_per_src=[5224, 5460, 5444, 5350, 5480, 5552, 5442, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228744 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156712328 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313195912 weight_movement_bytes=18874368 total_nvlink_bytes=332070280 weight_to_token_ratio=0.060264 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15068805000 prefetch_bytes=736100352 reduce_bytes=905969664 weight_movement_bytes=1642070016 egress_bytes=7539906440 ingress_bytes=9170968576 link_bytes=9170968576 total_nvlink_bytes=16710875016 weight_to_token_ratio=0.108971 prefetch_share_of_ingress=0.080264 dedup_ratio=0.332159 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=48 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082031 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.830660594e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38378 dup_rows=21660 dup_rows_remote=19060 dedup_ratio=0.330505 router_tokens_per_expert=[254, 271, 282, 260, 291, 241, 260, 265, 257, 266, 243, 250, 263, 254, 255, 264, 252, 249, 266, 278, 242, 287, 258, 238, 267, 263, 265, 245, 272, 251, 234, 275, 261, 240, 269, 251, 253, 257, 234, 285, 242, 258, 251, 283, 241, 243, 277, 230, 262, 247, 271, 230, 232, 266, 257, 259, 260, 276, 279, 256, 280, 244, 244, 244, 268, 244, 242, 257, 245, 250, 240, 249, 266, 264, 258, 249, 262, 270, 250, 242, 249, 246, 255, 254, 268, 226, 227, 264, 256, 282, 222, 257, 279, 236, 260, 268, 278, 241, 275, 258, 259, 273, 277, 263, 214, 250, 282, 230, 227, 262, 257, 253, 241, 259, 241, 253, 257, 245, 269, 257, 280, 254, 251, 255, 256, 233, 257, 256] payload_rows_per_dst=[5582, 5510, 5456, 5494, 5456, 5498, 5470, 5410] dup_rows_per_dst=[2770, 2774, 2694, 2720, 2656, 2600, 2728, 2718] combine_rows_per_src=[5582, 5510, 5456, 5494, 5456, 5498, 5470, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38232 dup_rows=21852 dup_rows_remote=19154 dedup_ratio=0.333435 router_tokens_per_expert=[232, 271, 233, 274, 245, 249, 263, 260, 238, 250, 245, 279, 247, 256, 247, 249, 252, 266, 257, 248, 271, 234, 263, 284, 246, 269, 262, 244, 259, 242, 266, 253, 233, 272, 256, 281, 263, 256, 272, 261, 260, 291, 267, 281, 267, 225, 252, 266, 268, 260, 262, 243, 241, 260, 249, 262, 264, 241, 237, 262, 273, 255, 271, 237, 258, 267, 248, 255, 237, 237, 260, 262, 249, 220, 243, 268, 236, 262, 222, 260, 283, 233, 235, 241, 241, 241, 261, 227, 222, 262, 269, 237, 262, 261, 222, 254, 250, 286, 247, 272, 246, 253, 272, 261, 271, 274, 274, 275, 253, 252, 256, 278, 248, 265, 276, 287, 271, 232, 274, 254, 242, 274, 266, 257, 259, 276, 249, 241] payload_rows_per_dst=[5452, 5608, 5582, 5390, 5568, 5362, 5410, 5312] dup_rows_per_dst=[2698, 2924, 2860, 2670, 2814, 2540, 2730, 2616] combine_rows_per_src=[5452, 5608, 5582, 5390, 5568, 5362, 5410, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155451392 dispatch_meta_bytes=228424 combine_payload_bytes=155451392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155679816 ingress_bytes=193200128 link_bytes=193200128 token_bytes=311131208 weight_movement_bytes=37748736 total_nvlink_bytes=348879944 weight_to_token_ratio=0.121327 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851148566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229752 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157426040 ingress_bytes=176070656 link_bytes=176070656 token_bytes=314622328 weight_movement_bytes=18874368 total_nvlink_bytes=333496696 weight_to_token_ratio=0.059991 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687022394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229544 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827816 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426088 weight_movement_bytes=18874368 total_nvlink_bytes=332300456 weight_to_token_ratio=0.060220 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15023973016 prefetch_bytes=849346560 reduce_bytes=452984832 weight_movement_bytes=1302331392 egress_bytes=7517471384 ingress_bytes=8808833024 link_bytes=8809291936 total_nvlink_bytes=16326304408 weight_to_token_ratio=0.086684 prefetch_share_of_ingress=0.096420 dedup_ratio=0.332765 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=24 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080512 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.758464602e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15066634296 prefetch_bytes=811597824 reduce_bytes=698351616 weight_movement_bytes=1509949440 egress_bytes=7538825272 ingress_bytes=9037758464 link_bytes=9037758464 total_nvlink_bytes=16576583736 weight_to_token_ratio=0.100218 prefetch_share_of_ingress=0.089801 dedup_ratio=0.332531 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=37 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080295 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.804069891e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38250 dup_rows=21784 dup_rows_remote=19006 dedup_ratio=0.332397 router_tokens_per_expert=[239, 279, 266, 273, 260, 289, 267, 273, 263, 256, 241, 251, 254, 251, 256, 233, 249, 282, 249, 287, 258, 243, 249, 247, 271, 244, 222, 231, 243, 259, 234, 254, 249, 262, 249, 248, 275, 246, 247, 267, 269, 215, 261, 278, 279, 249, 224, 251, 243, 278, 274, 269, 290, 293, 257, 265, 249, 264, 248, 267, 260, 259, 246, 253, 256, 259, 259, 236, 243, 274, 241, 263, 256, 276, 258, 231, 277, 264, 259, 288, 249, 247, 250, 271, 272, 241, 269, 276, 233, 238, 267, 237, 259, 259, 266, 242, 252, 265, 289, 262, 263, 256, 247, 249, 237, 247, 241, 237, 233, 275, 234, 258, 251, 250, 251, 259, 251, 243, 266, 261, 256, 254, 260, 277, 261, 224, 223, 263] payload_rows_per_dst=[5526, 5428, 5458, 5560, 5502, 5396, 5458, 5424] dup_rows_per_dst=[2776, 2616, 2680, 2870, 2778, 2756, 2632, 2676] combine_rows_per_src=[5526, 5428, 5458, 5560, 5502, 5396, 5458, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229024 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901024 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573024 weight_movement_bytes=18874368 total_nvlink_bytes=332447392 weight_to_token_ratio=0.060191 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38310 dup_rows=21742 dup_rows_remote=18994 dedup_ratio=0.331757 router_tokens_per_expert=[259, 246, 229, 246, 256, 253, 265, 246, 274, 256, 246, 253, 292, 271, 251, 245, 294, 259, 258, 248, 286, 223, 270, 255, 241, 264, 268, 258, 237, 264, 257, 261, 241, 241, 251, 239, 251, 283, 245, 241, 266, 297, 260, 264, 250, 243, 265, 260, 257, 282, 246, 272, 267, 244, 256, 249, 253, 252, 228, 257, 232, 287, 264, 252, 258, 247, 260, 277, 240, 247, 260, 257, 270, 240, 257, 270, 200, 266, 288, 245, 280, 248, 266, 232, 240, 267, 250, 229, 247, 239, 254, 250, 247, 264, 262, 271, 240, 257, 277, 250, 246, 270, 263, 238, 240, 253, 231, 245, 280, 252, 265, 269, 273, 255, 247, 268, 268, 261, 251, 282, 238, 259, 250, 256, 257, 260, 266, 247] payload_rows_per_dst=[5484, 5476, 5472, 5460, 5550, 5420, 5354, 5578] dup_rows_per_dst=[2748, 2810, 2722, 2680, 2830, 2672, 2582, 2698] combine_rows_per_src=[5484, 5476, 5472, 5460, 5550, 5420, 5354, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15058290024 prefetch_bytes=792723456 reduce_bytes=641728512 weight_movement_bytes=1434451968 egress_bytes=7534650728 ingress_bytes=8958091264 link_bytes=8958551176 total_nvlink_bytes=16492741992 weight_to_token_ratio=0.095260 prefetch_share_of_ingress=0.088492 dedup_ratio=0.332668 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=34 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082610 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.788258948e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229216 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157146976 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314064736 weight_movement_bytes=18874368 total_nvlink_bytes=332939104 weight_to_token_ratio=0.060097 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38240 dup_rows=21786 dup_rows_remote=18900 dedup_ratio=0.332428 router_tokens_per_expert=[248, 232, 215, 267, 269, 245, 243, 266, 287, 289, 237, 260, 241, 242, 283, 258, 257, 265, 268, 263, 253, 254, 237, 234, 262, 270, 248, 280, 240, 243, 256, 274, 274, 278, 257, 238, 262, 258, 236, 275, 263, 235, 268, 241, 257, 269, 263, 257, 299, 243, 258, 287, 293, 263, 253, 250, 259, 259, 263, 276, 227, 248, 260, 258, 292, 244, 254, 237, 228, 246, 224, 241, 232, 249, 271, 260, 269, 238, 272, 243, 231, 237, 271, 227, 263, 259, 253, 269, 241, 253, 271, 228, 286, 269, 286, 246, 272, 263, 222, 259, 263, 257, 269, 247, 225, 265, 255, 261, 254, 249, 226, 258, 259, 277, 280, 249, 257, 289, 250, 258, 255, 261, 238, 249, 248, 239, 257, 254] payload_rows_per_dst=[5510, 5522, 5448, 5574, 5620, 5410, 5456, 5210] dup_rows_per_dst=[2886, 2758, 2814, 2696, 2940, 2698, 2634, 2360] combine_rows_per_src=[5510, 5522, 5448, 5574, 5620, 5410, 5456, 5210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228560 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156859600 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313490640 weight_movement_bytes=37748736 total_nvlink_bytes=351239376 weight_to_token_ratio=0.120414 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38186 dup_rows=21840 dup_rows_remote=19074 dedup_ratio=0.333252 router_tokens_per_expert=[282, 254, 251, 249, 258, 229, 235, 248, 256, 269, 259, 273, 264, 270, 245, 250, 256, 264, 210, 248, 254, 238, 261, 276, 230, 245, 252, 263, 268, 236, 246, 272, 249, 243, 270, 243, 263, 239, 257, 250, 239, 255, 253, 245, 269, 262, 235, 275, 240, 260, 273, 250, 273, 256, 258, 279, 246, 220, 251, 282, 247, 247, 253, 268, 282, 291, 244, 249, 266, 287, 267, 261, 239, 255, 268, 241, 283, 271, 259, 262, 271, 251, 242, 264, 231, 252, 261, 264, 250, 256, 257, 240, 235, 244, 267, 262, 228, 275, 262, 264, 260, 248, 272, 256, 237, 287, 253, 275, 241, 265, 242, 255, 288, 249, 269, 255, 248, 246, 250, 258, 258, 265, 240, 227, 273, 265, 244, 280] payload_rows_per_dst=[5510, 5428, 5518, 5440, 5540, 5382, 5396, 5482] dup_rows_per_dst=[2766, 2734, 2752, 2662, 2910, 2712, 2648, 2656] combine_rows_per_src=[5510, 5428, 5518, 5440, 5540, 5382, 5396, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229040 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638896 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048752 weight_movement_bytes=37748736 total_nvlink_bytes=350797488 weight_to_token_ratio=0.120584 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15042271576 prefetch_bytes=773849088 reduce_bytes=811597824 weight_movement_bytes=1585446912 egress_bytes=7526627672 ingress_bytes=9101090816 link_bytes=9101090816 total_nvlink_bytes=16627718488 weight_to_token_ratio=0.105399 prefetch_share_of_ingress=0.085028 dedup_ratio=0.332599 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=43 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.816711962e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none + [2026-08-29 23:39:07] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 783.8 | throughput per GPU (TFLOP/s/GPU): 113.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.308677E+00 | loss scale: 1.0 | grad norm: 4.113 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38104 dup_rows=21868 dup_rows_remote=18998 dedup_ratio=0.333679 router_tokens_per_expert=[224, 299, 223, 248, 241, 258, 251, 267, 260, 231, 272, 256, 251, 258, 247, 233, 278, 244, 274, 261, 256, 266, 265, 261, 236, 246, 259, 253, 247, 272, 244, 278, 252, 253, 239, 225, 245, 244, 220, 256, 247, 253, 260, 255, 251, 242, 267, 241, 267, 228, 261, 254, 295, 239, 281, 270, 231, 273, 265, 298, 233, 272, 290, 260, 255, 267, 257, 267, 253, 267, 247, 262, 257, 263, 262, 272, 257, 258, 260, 234, 255, 276, 249, 272, 258, 252, 258, 273, 245, 279, 247, 261, 251, 240, 247, 250, 286, 267, 261, 245, 271, 258, 269, 234, 267, 250, 246, 283, 247, 273, 257, 254, 265, 240, 248, 257, 266, 231, 269, 250, 271, 237, 239, 238, 258, 244, 263, 247] payload_rows_per_dst=[5404, 5520, 5324, 5564, 5506, 5486, 5502, 5362] dup_rows_per_dst=[2634, 2760, 2576, 2870, 2770, 2740, 2834, 2684] combine_rows_per_src=[5404, 5520, 5324, 5564, 5506, 5486, 5502, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38266 dup_rows=21866 dup_rows_remote=19198 dedup_ratio=0.333649 router_tokens_per_expert=[268, 276, 247, 267, 247, 250, 270, 243, 248, 240, 253, 246, 246, 258, 269, 230, 279, 246, 269, 264, 245, 254, 240, 263, 241, 251, 223, 280, 245, 279, 262, 250, 256, 229, 239, 250, 228, 263, 258, 289, 285, 260, 233, 260, 261, 227, 251, 247, 263, 262, 285, 281, 245, 267, 244, 246, 237, 263, 269, 232, 243, 242, 271, 303, 256, 248, 229, 249, 266, 281, 233, 258, 257, 257, 258, 257, 265, 252, 286, 278, 272, 239, 251, 233, 248, 252, 267, 235, 245, 255, 240, 247, 235, 254, 230, 258, 280, 272, 276, 252, 273, 269, 258, 269, 255, 255, 256, 264, 267, 280, 251, 237, 281, 241, 233, 256, 244, 259, 278, 239, 261, 267, 251, 257, 260, 271, 260, 267] payload_rows_per_dst=[5430, 5480, 5404, 5486, 5510, 5388, 5532, 5440] dup_rows_per_dst=[2686, 2702, 2668, 2820, 2750, 2534, 2896, 2810] combine_rows_per_src=[5430, 5480, 5404, 5486, 5510, 5388, 5532, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38530 dup_rows=21628 dup_rows_remote=19072 dedup_ratio=0.330017 router_tokens_per_expert=[247, 212, 252, 244, 280, 261, 266, 270, 273, 249, 259, 233, 271, 241, 292, 271, 228, 282, 243, 239, 252, 234, 221, 228, 257, 280, 263, 251, 257, 234, 251, 247, 249, 259, 252, 263, 240, 256, 238, 247, 248, 264, 239, 266, 286, 285, 265, 241, 274, 268, 240, 254, 305, 243, 229, 273, 232, 250, 285, 246, 291, 228, 260, 249, 239, 252, 258, 272, 252, 255, 257, 249, 267, 279, 274, 277, 262, 262, 253, 247, 264, 236, 254, 269, 235, 265, 273, 239, 235, 235, 267, 251, 288, 269, 271, 248, 260, 251, 274, 245, 229, 243, 280, 263, 250, 244, 270, 244, 262, 267, 263, 239, 262, 266, 236, 259, 242, 233, 293, 239, 244, 297, 247, 246, 253, 270, 273, 257] payload_rows_per_dst=[5554, 5378, 5496, 5504, 5518, 5500, 5472, 5486] dup_rows_per_dst=[2688, 2556, 2700, 2750, 2792, 2698, 2696, 2748] combine_rows_per_src=[5554, 5378, 5496, 5504, 5518, 5500, 5472, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38254 dup_rows=21812 dup_rows_remote=19094 dedup_ratio=0.332825 router_tokens_per_expert=[269, 261, 268, 261, 263, 224, 247, 244, 278, 258, 250, 246, 234, 267, 234, 273, 222, 268, 260, 261, 269, 268, 242, 265, 258, 255, 277, 258, 259, 255, 236, 244, 281, 254, 266, 240, 246, 278, 285, 260, 267, 256, 248, 247, 232, 280, 261, 244, 247, 273, 272, 260, 228, 263, 242, 279, 270, 240, 218, 259, 257, 260, 244, 242, 282, 265, 260, 257, 230, 246, 254, 251, 241, 252, 264, 278, 284, 240, 239, 251, 255, 271, 237, 256, 257, 276, 294, 264, 256, 258, 276, 258, 257, 227, 244, 254, 249, 257, 257, 243, 263, 250, 225, 259, 275, 276, 278, 258, 272, 245, 262, 256, 265, 260, 233, 240, 242, 269, 266, 261, 230, 239, 259, 257, 254, 277, 261, 223] payload_rows_per_dst=[5430, 5464, 5498, 5410, 5470, 5460, 5518, 5474] dup_rows_per_dst=[2724, 2730, 2792, 2698, 2718, 2820, 2732, 2598] combine_rows_per_src=[5430, 5464, 5498, 5410, 5470, 5460, 5518, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38162 dup_rows=21726 dup_rows_remote=18800 dedup_ratio=0.331512 router_tokens_per_expert=[295, 264, 254, 269, 260, 251, 270, 231, 234, 248, 265, 255, 261, 250, 242, 286, 275, 228, 251, 297, 262, 254, 275, 248, 277, 265, 270, 258, 238, 258, 249, 273, 224, 253, 256, 246, 270, 306, 236, 278, 230, 248, 234, 264, 254, 266, 257, 253, 229, 258, 264, 262, 256, 262, 257, 250, 247, 270, 242, 229, 244, 258, 253, 246, 266, 261, 283, 238, 249, 240, 271, 236, 259, 260, 249, 260, 233, 283, 271, 257, 254, 270, 232, 271, 286, 253, 225, 271, 250, 237, 266, 281, 251, 266, 247, 251, 267, 260, 244, 240, 285, 266, 289, 239, 267, 258, 238, 271, 257, 236, 210, 265, 261, 246, 257, 257, 236, 281, 257, 223, 238, 244, 254, 278, 261, 248, 245, 248] payload_rows_per_dst=[5546, 5406, 5398, 5368, 5432, 5512, 5500, 5648] dup_rows_per_dst=[2724, 2488, 2752, 2574, 2800, 2710, 2752, 2926] combine_rows_per_src=[5546, 5406, 5398, 5368, 5432, 5512, 5500, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38476 dup_rows=21596 dup_rows_remote=18932 dedup_ratio=0.329529 router_tokens_per_expert=[282, 254, 236, 258, 240, 253, 256, 259, 246, 268, 267, 266, 260, 262, 241, 249, 254, 264, 241, 263, 264, 271, 235, 245, 252, 253, 257, 269, 272, 235, 247, 229, 277, 247, 275, 281, 275, 243, 260, 252, 283, 246, 244, 266, 246, 285, 256, 271, 270, 255, 247, 236, 274, 272, 277, 267, 267, 255, 263, 252, 253, 262, 274, 234, 254, 237, 259, 246, 254, 278, 264, 240, 274, 250, 236, 252, 240, 275, 236, 248, 259, 255, 256, 258, 244, 262, 262, 242, 276, 252, 273, 268, 245, 248, 239, 225, 255, 274, 225, 269, 284, 279, 248, 272, 217, 295, 264, 263, 226, 227, 237, 270, 245, 234, 247, 260, 263, 273, 259, 247, 257, 250, 249, 235, 277, 254, 253, 240] payload_rows_per_dst=[5478, 5430, 5656, 5506, 5484, 5464, 5486, 5436] dup_rows_per_dst=[2716, 2672, 2758, 2810, 2602, 2664, 2724, 2650] combine_rows_per_src=[5478, 5430, 5656, 5506, 5484, 5464, 5486, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=228408 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156302392 ingress_bytes=193822720 link_bytes=193822720 token_bytes=312376376 weight_movement_bytes=37748736 total_nvlink_bytes=350125112 weight_to_token_ratio=0.120844 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857113936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=230408 combine_payload_bytes=157818880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158049288 ingress_bytes=214441984 link_bytes=214441984 token_bytes=315868168 weight_movement_bytes=56623104 total_nvlink_bytes=372491272 weight_to_token_ratio=0.179262 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.054677579e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229856 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967392 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704928 weight_movement_bytes=18874368 total_nvlink_bytes=332579296 weight_to_token_ratio=0.060166 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229392 combine_payload_bytes=156688384 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156917776 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606160 weight_movement_bytes=37748736 total_nvlink_bytes=351354896 weight_to_token_ratio=0.120370 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=227848 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156539400 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312850952 weight_movement_bytes=18874368 total_nvlink_bytes=331725320 weight_to_token_ratio=0.060330 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229632 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827328 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315425024 weight_movement_bytes=18874368 total_nvlink_bytes=334299392 weight_to_token_ratio=0.059838 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43532 payload_rows_remote=38216 dup_rows=22004 dup_rows_remote=19404 dedup_ratio=0.335754 router_tokens_per_expert=[278, 253, 247, 240, 268, 262, 268, 243, 252, 253, 269, 254, 248, 232, 224, 232, 299, 269, 276, 239, 280, 273, 248, 251, 276, 265, 269, 266, 296, 260, 255, 253, 245, 268, 257, 277, 241, 269, 259, 261, 260, 251, 241, 251, 266, 234, 234, 265, 271, 287, 255, 288, 253, 254, 281, 257, 253, 263, 263, 276, 250, 227, 276, 274, 235, 238, 253, 245, 258, 268, 265, 253, 260, 265, 244, 254, 260, 266, 247, 259, 253, 254, 259, 253, 238, 254, 264, 247, 276, 264, 241, 246, 256, 264, 255, 275, 243, 240, 259, 235, 268, 264, 238, 232, 242, 262, 255, 262, 225, 261, 241, 231, 248, 261, 287, 281, 262, 256, 250, 230, 260, 265, 246, 263, 222, 247, 226, 232] payload_rows_per_dst=[5434, 5562, 5448, 5400, 5424, 5446, 5316, 5502] dup_rows_per_dst=[2612, 2988, 2710, 2680, 2716, 2752, 2600, 2946] combine_rows_per_src=[5434, 5562, 5448, 5400, 5424, 5446, 5316, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38120 dup_rows=21912 dup_rows_remote=19046 dedup_ratio=0.334351 router_tokens_per_expert=[290, 284, 252, 266, 257, 253, 240, 272, 290, 249, 236, 244, 250, 260, 269, 251, 247, 267, 262, 280, 277, 243, 264, 245, 273, 274, 262, 219, 269, 258, 254, 291, 262, 252, 240, 267, 247, 282, 223, 253, 265, 273, 261, 245, 264, 269, 292, 260, 235, 212, 226, 264, 262, 247, 256, 272, 301, 253, 230, 245, 247, 290, 278, 267, 264, 242, 238, 285, 243, 253, 215, 247, 261, 275, 252, 293, 260, 243, 249, 237, 221, 262, 258, 254, 252, 245, 234, 256, 251, 263, 221, 264, 299, 238, 250, 237, 267, 258, 233, 257, 234, 274, 246, 262, 266, 273, 266, 243, 253, 268, 276, 235, 267, 250, 254, 246, 270, 239, 243, 232, 263, 254, 228, 246, 271, 225, 248, 271] payload_rows_per_dst=[5488, 5504, 5488, 5430, 5482, 5388, 5494, 5350] dup_rows_per_dst=[2838, 2866, 2822, 2740, 2632, 2622, 2728, 2664] combine_rows_per_src=[5488, 5504, 5488, 5430, 5482, 5388, 5494, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38320 dup_rows=21822 dup_rows_remote=19186 dedup_ratio=0.332977 router_tokens_per_expert=[243, 230, 273, 278, 247, 239, 260, 260, 238, 230, 256, 279, 266, 243, 255, 276, 260, 247, 258, 260, 239, 258, 275, 231, 278, 259, 256, 273, 256, 250, 248, 250, 249, 269, 240, 269, 232, 233, 251, 253, 249, 258, 261, 228, 239, 276, 248, 260, 252, 242, 258, 257, 239, 239, 269, 259, 279, 254, 251, 260, 250, 276, 252, 269, 246, 267, 241, 253, 239, 262, 259, 284, 243, 260, 251, 265, 266, 246, 241, 243, 272, 272, 284, 247, 260, 229, 239, 257, 237, 272, 253, 275, 269, 248, 264, 268, 286, 266, 233, 249, 265, 262, 274, 259, 260, 264, 247, 253, 252, 272, 261, 252, 251, 248, 235, 261, 265, 264, 252, 259, 266, 298, 250, 219, 267, 236, 253, 285] payload_rows_per_dst=[5442, 5516, 5394, 5468, 5408, 5514, 5450, 5522] dup_rows_per_dst=[2704, 2680, 2636, 2744, 2724, 2778, 2860, 2696] combine_rows_per_src=[5442, 5516, 5394, 5468, 5408, 5514, 5450, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38160 dup_rows=21996 dup_rows_remote=19364 dedup_ratio=0.335632 router_tokens_per_expert=[287, 237, 244, 249, 229, 251, 232, 258, 259, 263, 270, 233, 257, 248, 227, 242, 255, 269, 253, 235, 256, 271, 264, 242, 253, 254, 249, 268, 250, 260, 260, 249, 245, 281, 270, 235, 262, 242, 255, 245, 277, 267, 266, 252, 269, 272, 246, 252, 255, 229, 261, 224, 259, 245, 253, 258, 254, 262, 234, 270, 250, 242, 254, 256, 262, 273, 241, 263, 281, 255, 265, 252, 277, 274, 240, 255, 282, 259, 240, 256, 237, 254, 244, 226, 234, 266, 273, 256, 248, 259, 244, 267, 277, 265, 261, 262, 261, 275, 266, 272, 270, 273, 256, 300, 262, 278, 253, 247, 285, 266, 268, 226, 244, 262, 256, 255, 256, 279, 247, 248, 236, 247, 249, 267, 245, 239, 253, 263] payload_rows_per_dst=[5338, 5426, 5498, 5380, 5504, 5402, 5568, 5424] dup_rows_per_dst=[2634, 2750, 2774, 2632, 2846, 2744, 2948, 2668] combine_rows_per_src=[5338, 5426, 5498, 5380, 5504, 5402, 5568, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38340 dup_rows=21730 dup_rows_remote=18972 dedup_ratio=0.331573 router_tokens_per_expert=[251, 261, 245, 262, 236, 213, 252, 248, 246, 251, 244, 251, 252, 261, 258, 248, 251, 239, 258, 252, 213, 270, 241, 268, 259, 262, 267, 251, 259, 277, 266, 257, 259, 274, 260, 297, 253, 236, 258, 279, 280, 238, 234, 252, 240, 248, 258, 255, 230, 248, 274, 266, 266, 234, 277, 255, 271, 254, 267, 253, 273, 249, 260, 225, 226, 278, 258, 262, 272, 274, 262, 284, 230, 249, 266, 273, 254, 243, 247, 234, 261, 244, 249, 261, 248, 258, 265, 257, 232, 232, 241, 240, 255, 277, 259, 263, 285, 301, 235, 261, 303, 259, 254, 265, 248, 241, 254, 247, 245, 240, 260, 286, 247, 270, 255, 265, 253, 252, 260, 264, 266, 272, 257, 256, 255, 243, 276, 247] payload_rows_per_dst=[5364, 5486, 5530, 5472, 5466, 5480, 5548, 5460] dup_rows_per_dst=[2594, 2694, 2712, 2732, 2758, 2604, 2820, 2816] combine_rows_per_src=[5364, 5486, 5530, 5472, 5466, 5480, 5548, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=230480 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156763216 ingress_bytes=213155840 link_bytes=213155840 token_bytes=313295952 weight_movement_bytes=56623104 total_nvlink_bytes=369919056 weight_to_token_ratio=0.180734 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042354380e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38160 dup_rows=21734 dup_rows_remote=18876 dedup_ratio=0.331635 router_tokens_per_expert=[256, 253, 273, 231, 259, 258, 264, 246, 238, 267, 248, 247, 280, 251, 300, 270, 270, 265, 248, 239, 268, 261, 231, 236, 234, 259, 244, 243, 274, 271, 255, 233, 254, 240, 252, 242, 264, 277, 249, 243, 267, 249, 251, 252, 240, 271, 256, 275, 243, 236, 278, 261, 287, 264, 276, 265, 237, 276, 239, 270, 277, 267, 269, 243, 268, 243, 255, 243, 257, 254, 263, 277, 249, 249, 267, 253, 263, 280, 260, 227, 252, 280, 225, 216, 251, 278, 260, 267, 265, 229, 250, 251, 249, 254, 279, 263, 237, 286, 265, 247, 272, 256, 262, 234, 244, 256, 257, 266, 271, 268, 246, 253, 270, 255, 241, 259, 250, 249, 243, 239, 283, 232, 244, 260, 240, 248, 277, 239] payload_rows_per_dst=[5466, 5446, 5298, 5542, 5452, 5588, 5368, 5642] dup_rows_per_dst=[2816, 2616, 2502, 2828, 2764, 2914, 2436, 2858] combine_rows_per_src=[5466, 5446, 5298, 5542, 5452, 5588, 5368, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38480 dup_rows=21644 dup_rows_remote=18920 dedup_ratio=0.330261 router_tokens_per_expert=[256, 256, 264, 235, 286, 239, 241, 279, 255, 285, 256, 281, 256, 264, 253, 258, 234, 228, 245, 275, 267, 241, 263, 245, 270, 242, 244, 274, 271, 247, 234, 266, 282, 268, 236, 255, 266, 262, 242, 242, 303, 272, 258, 246, 235, 245, 271, 256, 282, 245, 258, 256, 249, 258, 263, 267, 259, 294, 236, 256, 253, 248, 244, 241, 271, 244, 231, 273, 252, 259, 249, 276, 264, 238, 260, 240, 269, 257, 257, 241, 241, 244, 251, 264, 267, 253, 246, 250, 253, 254, 283, 248, 257, 252, 242, 263, 267, 235, 250, 280, 248, 257, 263, 258, 246, 241, 264, 249, 240, 253, 263, 264, 238, 255, 240, 290, 263, 256, 261, 242, 234, 262, 258, 238, 268, 262, 251, 265] payload_rows_per_dst=[5570, 5496, 5516, 5532, 5440, 5412, 5494, 5432] dup_rows_per_dst=[2758, 2596, 2762, 2686, 2722, 2724, 2662, 2734] combine_rows_per_src=[5570, 5496, 5516, 5532, 5440, 5412, 5494, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228664 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156368184 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507704 weight_movement_bytes=18874368 total_nvlink_bytes=331382072 weight_to_token_ratio=0.060396 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229248 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157269888 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314310528 weight_movement_bytes=18874368 total_nvlink_bytes=333184896 weight_to_token_ratio=0.060050 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=230024 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157188744 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314147464 weight_movement_bytes=37748736 total_nvlink_bytes=351896200 weight_to_token_ratio=0.120162 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=230096 combine_payload_bytes=156303360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156533456 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312836816 weight_movement_bytes=37748736 total_nvlink_bytes=350585552 weight_to_token_ratio=0.120666 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228144 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156531504 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312834864 weight_movement_bytes=18874368 total_nvlink_bytes=331709232 weight_to_token_ratio=0.060333 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157614080 dispatch_meta_bytes=229600 combine_payload_bytes=157614080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157843680 ingress_bytes=176488448 link_bytes=176488448 token_bytes=315457760 weight_movement_bytes=18874368 total_nvlink_bytes=334332128 weight_to_token_ratio=0.059832 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691025471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38020 dup_rows=21982 dup_rows_remote=19040 dedup_ratio=0.335419 router_tokens_per_expert=[257, 234, 262, 236, 287, 252, 248, 253, 272, 279, 268, 274, 265, 287, 227, 240, 248, 256, 227, 253, 297, 241, 277, 249, 265, 247, 274, 252, 292, 269, 260, 280, 271, 253, 242, 270, 249, 259, 269, 259, 274, 273, 249, 258, 250, 262, 224, 251, 272, 251, 248, 235, 256, 272, 245, 256, 233, 221, 262, 275, 255, 244, 254, 269, 268, 278, 255, 247, 248, 269, 242, 255, 261, 267, 223, 257, 263, 260, 258, 249, 270, 259, 284, 254, 263, 235, 253, 235, 254, 278, 262, 271, 229, 264, 236, 276, 272, 273, 224, 279, 267, 262, 283, 238, 247, 260, 216, 284, 243, 264, 235, 296, 254, 259, 247, 234, 262, 221, 248, 271, 240, 208, 246, 238, 245, 285, 240, 215] payload_rows_per_dst=[5534, 5580, 5522, 5444, 5374, 5464, 5360, 5276] dup_rows_per_dst=[2942, 2820, 2834, 2652, 2670, 2898, 2616, 2550] combine_rows_per_src=[5534, 5580, 5522, 5444, 5374, 5464, 5360, 5276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38426 dup_rows=21700 dup_rows_remote=19022 dedup_ratio=0.331116 router_tokens_per_expert=[247, 245, 247, 223, 277, 261, 274, 266, 216, 276, 251, 226, 261, 263, 263, 246, 252, 231, 246, 249, 240, 281, 244, 282, 245, 233, 238, 262, 250, 256, 269, 266, 275, 237, 263, 270, 247, 251, 244, 244, 249, 257, 280, 265, 276, 248, 262, 248, 239, 234, 240, 261, 244, 262, 255, 245, 262, 262, 245, 251, 232, 258, 255, 276, 266, 247, 235, 270, 258, 251, 279, 270, 263, 270, 260, 227, 296, 240, 228, 231, 270, 307, 281, 252, 253, 265, 281, 247, 264, 239, 245, 267, 240, 250, 257, 282, 254, 238, 257, 266, 226, 234, 235, 266, 223, 248, 266, 270, 265, 276, 276, 244, 261, 279, 270, 251, 246, 288, 236, 288, 257, 267, 238, 268, 262, 259, 264, 276] payload_rows_per_dst=[5424, 5410, 5468, 5462, 5454, 5576, 5470, 5572] dup_rows_per_dst=[2660, 2678, 2764, 2580, 2728, 2824, 2618, 2848] combine_rows_per_src=[5424, 5410, 5468, 5462, 5454, 5576, 5470, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38176 dup_rows=21886 dup_rows_remote=19126 dedup_ratio=0.333954 router_tokens_per_expert=[265, 275, 262, 263, 231, 285, 264, 238, 261, 237, 262, 257, 251, 261, 241, 253, 273, 254, 249, 248, 245, 249, 226, 262, 261, 247, 262, 238, 237, 240, 268, 254, 266, 255, 243, 283, 251, 238, 261, 242, 284, 258, 254, 296, 259, 292, 283, 232, 250, 241, 254, 251, 283, 249, 254, 280, 268, 283, 256, 243, 215, 265, 256, 267, 269, 275, 238, 263, 273, 270, 236, 225, 212, 252, 229, 267, 256, 243, 244, 236, 268, 252, 260, 240, 252, 256, 238, 264, 234, 283, 261, 253, 271, 281, 267, 255, 252, 261, 252, 258, 248, 264, 268, 238, 258, 256, 258, 228, 257, 285, 280, 254, 260, 252, 266, 265, 253, 251, 254, 241, 253, 265, 260, 241, 255, 240, 271, 270] payload_rows_per_dst=[5452, 5384, 5556, 5458, 5398, 5492, 5474, 5436] dup_rows_per_dst=[2760, 2642, 2838, 2772, 2578, 2778, 2760, 2758] combine_rows_per_src=[5452, 5384, 5556, 5458, 5398, 5492, 5474, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38142 dup_rows=21900 dup_rows_remote=19118 dedup_ratio=0.334167 router_tokens_per_expert=[265, 262, 282, 232, 252, 272, 263, 228, 278, 256, 252, 275, 272, 253, 263, 240, 273, 238, 243, 257, 222, 242, 265, 254, 257, 248, 280, 242, 275, 241, 257, 252, 228, 284, 280, 257, 256, 251, 253, 240, 251, 264, 276, 251, 245, 296, 275, 258, 253, 228, 237, 235, 264, 233, 264, 257, 245, 251, 291, 224, 244, 269, 272, 262, 279, 240, 286, 256, 257, 247, 266, 259, 276, 221, 276, 254, 274, 252, 241, 254, 255, 245, 259, 245, 254, 254, 242, 249, 249, 249, 249, 262, 291, 252, 268, 232, 242, 252, 278, 274, 248, 252, 265, 262, 240, 251, 264, 243, 274, 229, 261, 252, 247, 259, 283, 218, 256, 272, 243, 241, 272, 262, 281, 250, 276, 254, 228, 261] payload_rows_per_dst=[5542, 5378, 5488, 5336, 5494, 5468, 5446, 5484] dup_rows_per_dst=[2748, 2714, 2842, 2722, 2782, 2642, 2728, 2722] combine_rows_per_src=[5542, 5378, 5488, 5336, 5494, 5468, 5446, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38190 dup_rows=21814 dup_rows_remote=19042 dedup_ratio=0.332855 router_tokens_per_expert=[260, 242, 241, 255, 243, 273, 228, 260, 258, 259, 251, 268, 249, 262, 243, 303, 270, 245, 269, 268, 241, 269, 260, 257, 245, 249, 259, 229, 249, 281, 255, 247, 267, 288, 266, 260, 250, 274, 243, 257, 265, 253, 254, 246, 264, 265, 251, 249, 231, 250, 226, 322, 275, 236, 251, 264, 303, 253, 257, 242, 246, 248, 264, 245, 270, 238, 289, 260, 271, 228, 256, 262, 264, 242, 258, 229, 251, 239, 253, 249, 245, 261, 247, 234, 254, 234, 272, 237, 248, 245, 242, 264, 261, 246, 261, 279, 217, 265, 242, 259, 238, 228, 260, 255, 248, 252, 278, 250, 241, 251, 256, 276, 262, 282, 269, 282, 275, 263, 245, 267, 253, 261, 268, 240, 274, 250, 270, 249] payload_rows_per_dst=[5418, 5524, 5532, 5444, 5460, 5384, 5410, 5550] dup_rows_per_dst=[2772, 2662, 2772, 2782, 2658, 2676, 2622, 2870] combine_rows_per_src=[5418, 5524, 5532, 5444, 5460, 5384, 5410, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38176 dup_rows=21928 dup_rows_remote=19210 dedup_ratio=0.334595 router_tokens_per_expert=[254, 258, 275, 251, 230, 281, 250, 286, 244, 279, 260, 262, 256, 244, 268, 260, 248, 242, 243, 236, 269, 291, 255, 229, 264, 248, 235, 285, 235, 266, 272, 274, 240, 239, 248, 270, 245, 260, 253, 228, 277, 263, 245, 263, 268, 243, 257, 279, 259, 237, 258, 240, 241, 282, 274, 276, 257, 265, 236, 273, 286, 269, 232, 281, 263, 270, 258, 270, 248, 226, 235, 276, 267, 257, 244, 255, 267, 242, 250, 214, 236, 259, 250, 228, 244, 276, 258, 259, 252, 278, 243, 246, 258, 259, 224, 261, 252, 277, 269, 269, 265, 257, 268, 272, 267, 249, 271, 250, 251, 231, 268, 221, 254, 281, 237, 287, 276, 255, 251, 241, 233, 249, 241, 224, 256, 264, 248, 267] payload_rows_per_dst=[5404, 5506, 5410, 5500, 5456, 5428, 5472, 5432] dup_rows_per_dst=[2678, 2678, 2746, 2832, 2862, 2634, 2780, 2718] combine_rows_per_src=[5404, 5506, 5410, 5500, 5456, 5428, 5472, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38286 dup_rows=21822 dup_rows_remote=19212 dedup_ratio=0.332977 router_tokens_per_expert=[262, 263, 247, 256, 231, 256, 263, 271, 245, 241, 252, 245, 260, 249, 267, 269, 258, 269, 281, 273, 257, 248, 270, 265, 254, 260, 249, 261, 258, 237, 245, 270, 253, 244, 256, 237, 228, 279, 258, 248, 229, 244, 260, 256, 258, 279, 247, 253, 278, 244, 251, 241, 217, 258, 243, 259, 237, 245, 261, 263, 241, 256, 240, 285, 253, 243, 225, 261, 246, 257, 283, 275, 232, 239, 275, 265, 271, 243, 262, 248, 233, 282, 266, 247, 237, 243, 250, 266, 279, 242, 267, 247, 244, 267, 251, 256, 247, 262, 257, 250, 261, 251, 270, 283, 245, 264, 263, 261, 267, 269, 251, 249, 254, 274, 278, 260, 271, 265, 244, 260, 271, 285, 252, 268, 230, 270, 257, 244] payload_rows_per_dst=[5460, 5510, 5404, 5428, 5466, 5392, 5536, 5518] dup_rows_per_dst=[2694, 2800, 2654, 2610, 2690, 2762, 2764, 2848] combine_rows_per_src=[5460, 5510, 5404, 5428, 5466, 5392, 5536, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43406 payload_rows_remote=38066 dup_rows=22130 dup_rows_remote=19420 dedup_ratio=0.337677 router_tokens_per_expert=[278, 249, 251, 228, 244, 258, 273, 282, 239, 262, 251, 253, 253, 244, 253, 268, 255, 251, 286, 250, 253, 239, 266, 282, 245, 236, 264, 232, 246, 264, 226, 274, 236, 287, 226, 263, 249, 264, 277, 251, 256, 270, 254, 240, 260, 253, 257, 229, 230, 263, 273, 268, 238, 240, 273, 247, 228, 247, 256, 242, 273, 241, 260, 227, 263, 252, 257, 258, 236, 257, 256, 256, 270, 280, 284, 282, 249, 282, 268, 272, 283, 265, 266, 239, 248, 256, 247, 253, 230, 242, 249, 254, 244, 268, 238, 243, 285, 268, 274, 263, 232, 239, 273, 251, 262, 234, 255, 261, 280, 280, 271, 255, 220, 245, 273, 280, 280, 231, 257, 247, 251, 233, 250, 255, 277, 275, 267, 264] payload_rows_per_dst=[5416, 5442, 5386, 5340, 5534, 5340, 5478, 5470] dup_rows_per_dst=[2756, 2696, 2758, 2672, 2910, 2710, 2888, 2740] combine_rows_per_src=[5416, 5442, 5386, 5340, 5534, 5340, 5478, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157392896 dispatch_meta_bytes=229792 combine_payload_bytes=157392896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157622688 ingress_bytes=176267264 link_bytes=176267264 token_bytes=315015584 weight_movement_bytes=18874368 total_nvlink_bytes=333889952 weight_to_token_ratio=0.059916 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688906195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229040 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156458672 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312688304 weight_movement_bytes=18874368 total_nvlink_bytes=331562672 weight_to_token_ratio=0.060362 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=229944 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156148280 ingress_bytes=174792704 link_bytes=174792704 token_bytes=312066616 weight_movement_bytes=18874368 total_nvlink_bytes=330940984 weight_to_token_ratio=0.060482 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674777686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229208 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156598104 ingress_bytes=250740736 link_bytes=250740736 token_bytes=312967000 weight_movement_bytes=94371840 total_nvlink_bytes=407338840 weight_to_token_ratio=0.301539 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402474360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=228240 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155958160 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311688080 weight_movement_bytes=18874368 total_nvlink_bytes=330562448 weight_to_token_ratio=0.060555 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228928 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655168 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081408 weight_movement_bytes=37748736 total_nvlink_bytes=350830144 weight_to_token_ratio=0.120572 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229544 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156598440 ingress_bytes=231866368 link_bytes=231866368 token_bytes=312967336 weight_movement_bytes=75497472 total_nvlink_bytes=388464808 weight_to_token_ratio=0.241231 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221629453e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229992 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157049448 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868904 weight_movement_bytes=18874368 total_nvlink_bytes=332743272 weight_to_token_ratio=0.060135 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38294 dup_rows=21740 dup_rows_remote=18914 dedup_ratio=0.331726 router_tokens_per_expert=[244, 234, 247, 262, 276, 235, 269, 255, 260, 275, 251, 239, 262, 292, 239, 269, 235, 258, 246, 229, 270, 272, 280, 248, 233, 261, 261, 271, 258, 277, 281, 284, 244, 253, 250, 242, 258, 273, 251, 256, 259, 283, 231, 243, 250, 249, 289, 254, 247, 231, 247, 274, 267, 259, 245, 256, 258, 282, 250, 265, 222, 238, 249, 274, 276, 246, 247, 253, 267, 265, 287, 279, 261, 277, 239, 248, 248, 258, 230, 250, 253, 278, 307, 245, 274, 275, 224, 261, 289, 267, 257, 246, 252, 247, 268, 226, 258, 241, 231, 266, 248, 234, 233, 238, 227, 251, 245, 225, 265, 242, 273, 245, 239, 256, 270, 250, 255, 258, 269, 261, 268, 240, 223, 263, 276, 243, 263, 290] payload_rows_per_dst=[5502, 5502, 5444, 5484, 5500, 5518, 5342, 5504] dup_rows_per_dst=[2716, 2826, 2726, 2644, 2762, 2820, 2502, 2744] combine_rows_per_src=[5502, 5502, 5444, 5484, 5500, 5518, 5342, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38350 dup_rows=21716 dup_rows_remote=19080 dedup_ratio=0.331360 router_tokens_per_expert=[258, 253, 254, 272, 266, 253, 253, 251, 268, 271, 234, 243, 245, 277, 239, 264, 264, 246, 269, 227, 231, 265, 249, 265, 246, 263, 267, 243, 278, 237, 234, 235, 255, 254, 256, 273, 251, 266, 225, 288, 278, 266, 274, 247, 252, 280, 249, 245, 249, 286, 252, 249, 235, 269, 241, 273, 240, 271, 277, 248, 245, 264, 253, 276, 219, 254, 257, 264, 248, 281, 246, 266, 235, 253, 282, 245, 251, 245, 247, 260, 214, 266, 247, 236, 248, 246, 274, 246, 251, 253, 257, 259, 269, 249, 209, 238, 232, 265, 241, 289, 254, 252, 280, 266, 268, 268, 232, 256, 283, 259, 258, 260, 239, 242, 273, 265, 257, 270, 266, 256, 280, 272, 266, 233, 256, 263, 284, 261] payload_rows_per_dst=[5492, 5420, 5562, 5516, 5470, 5340, 5500, 5520] dup_rows_per_dst=[2710, 2618, 2756, 2740, 2636, 2584, 2826, 2846] combine_rows_per_src=[5492, 5420, 5562, 5516, 5470, 5340, 5500, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38226 dup_rows=21810 dup_rows_remote=19052 dedup_ratio=0.332794 router_tokens_per_expert=[265, 260, 272, 265, 282, 241, 261, 276, 275, 248, 254, 240, 287, 269, 237, 236, 242, 240, 250, 269, 247, 275, 246, 268, 242, 261, 267, 264, 276, 235, 265, 261, 243, 244, 253, 244, 254, 258, 227, 249, 249, 272, 256, 248, 251, 275, 231, 272, 253, 249, 227, 256, 246, 243, 264, 215, 254, 253, 240, 242, 251, 270, 251, 251, 259, 219, 257, 268, 272, 235, 265, 281, 267, 252, 280, 280, 273, 250, 247, 265, 277, 246, 258, 245, 250, 254, 234, 238, 260, 281, 268, 267, 272, 286, 219, 274, 234, 273, 264, 250, 266, 265, 246, 281, 238, 249, 261, 276, 250, 274, 253, 265, 254, 233, 256, 261, 272, 274, 253, 230, 244, 289, 233, 251, 278, 265, 219, 245] payload_rows_per_dst=[5496, 5406, 5428, 5338, 5536, 5500, 5558, 5464] dup_rows_per_dst=[2840, 2810, 2624, 2592, 2804, 2758, 2732, 2650] combine_rows_per_src=[5496, 5406, 5428, 5338, 5536, 5500, 5558, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38258 dup_rows=21684 dup_rows_remote=18846 dedup_ratio=0.330872 router_tokens_per_expert=[244, 250, 271, 227, 265, 243, 283, 254, 244, 238, 231, 238, 258, 240, 285, 242, 269, 263, 239, 258, 247, 256, 269, 253, 227, 256, 262, 267, 237, 252, 265, 265, 253, 263, 272, 260, 244, 252, 256, 257, 288, 276, 260, 250, 256, 251, 296, 282, 255, 259, 247, 273, 276, 261, 262, 251, 218, 274, 247, 252, 272, 278, 255, 247, 255, 229, 272, 268, 256, 264, 260, 270, 276, 257, 246, 265, 250, 232, 265, 219, 269, 237, 242, 248, 276, 265, 264, 270, 259, 260, 269, 265, 243, 249, 246, 250, 255, 250, 279, 264, 259, 258, 258, 259, 236, 253, 244, 256, 261, 261, 250, 264, 253, 246, 249, 275, 250, 248, 260, 267, 235, 208, 236, 241, 271, 269, 261, 255] payload_rows_per_dst=[5440, 5450, 5594, 5482, 5440, 5506, 5508, 5432] dup_rows_per_dst=[2586, 2720, 2838, 2772, 2728, 2718, 2706, 2616] combine_rows_per_src=[5440, 5450, 5594, 5482, 5440, 5506, 5508, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38076 dup_rows=21988 dup_rows_remote=19302 dedup_ratio=0.335510 router_tokens_per_expert=[238, 251, 262, 247, 253, 252, 267, 226, 260, 251, 283, 262, 240, 263, 255, 256, 252, 230, 266, 265, 258, 246, 249, 253, 263, 252, 254, 249, 266, 283, 231, 238, 263, 242, 275, 247, 245, 254, 272, 260, 242, 242, 253, 248, 263, 257, 257, 259, 273, 248, 267, 272, 257, 279, 256, 280, 274, 248, 263, 243, 240, 254, 281, 255, 273, 257, 280, 259, 263, 228, 217, 227, 255, 267, 261, 247, 261, 242, 247, 242, 261, 301, 276, 242, 236, 241, 271, 267, 263, 238, 243, 243, 269, 277, 276, 268, 263, 248, 237, 240, 263, 258, 257, 258, 243, 263, 252, 269, 260, 257, 260, 251, 252, 245, 264, 273, 281, 255, 266, 246, 247, 264, 238, 263, 258, 278, 226, 245] payload_rows_per_dst=[5402, 5428, 5454, 5554, 5282, 5538, 5472, 5418] dup_rows_per_dst=[2730, 2682, 2704, 2826, 2770, 2806, 2686, 2784] combine_rows_per_src=[5402, 5428, 5454, 5554, 5282, 5538, 5472, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38068 dup_rows=21962 dup_rows_remote=19196 dedup_ratio=0.335114 router_tokens_per_expert=[274, 250, 264, 261, 254, 274, 261, 239, 247, 268, 251, 259, 285, 257, 258, 251, 257, 249, 248, 239, 228, 233, 265, 260, 271, 238, 250, 246, 285, 264, 265, 252, 287, 253, 267, 256, 276, 253, 237, 275, 238, 287, 255, 282, 264, 237, 243, 260, 265, 258, 246, 266, 254, 238, 270, 214, 269, 261, 237, 269, 282, 263, 244, 246, 237, 232, 249, 269, 250, 265, 249, 251, 270, 255, 233, 257, 246, 235, 253, 259, 232, 268, 260, 279, 232, 232, 261, 229, 230, 256, 256, 250, 270, 238, 239, 264, 226, 240, 265, 252, 283, 274, 255, 264, 275, 264, 251, 255, 270, 256, 285, 256, 262, 248, 246, 248, 275, 280, 249, 269, 241, 277, 266, 251, 262, 251, 248, 263] payload_rows_per_dst=[5498, 5428, 5482, 5440, 5340, 5346, 5534, 5506] dup_rows_per_dst=[2678, 2672, 2916, 2724, 2680, 2646, 2880, 2766] combine_rows_per_src=[5498, 5428, 5482, 5440, 5340, 5346, 5534, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38348 dup_rows=21710 dup_rows_remote=19014 dedup_ratio=0.331268 router_tokens_per_expert=[248, 267, 289, 275, 262, 260, 251, 258, 243, 273, 234, 267, 257, 263, 262, 267, 268, 230, 291, 245, 254, 254, 233, 234, 242, 238, 230, 248, 232, 260, 235, 256, 257, 245, 285, 280, 254, 247, 270, 238, 277, 243, 255, 284, 233, 257, 258, 254, 272, 275, 213, 239, 269, 265, 251, 229, 253, 259, 271, 256, 248, 266, 256, 265, 252, 245, 273, 253, 263, 253, 241, 282, 263, 242, 261, 280, 264, 242, 246, 266, 255, 239, 251, 258, 275, 239, 249, 244, 242, 243, 265, 240, 262, 235, 247, 244, 255, 243, 245, 260, 272, 272, 261, 257, 262, 294, 256, 212, 269, 250, 244, 245, 245, 251, 260, 238, 249, 235, 290, 264, 273, 279, 271, 270, 264, 277, 259, 282] payload_rows_per_dst=[5484, 5402, 5536, 5478, 5480, 5448, 5452, 5546] dup_rows_per_dst=[2868, 2498, 2738, 2696, 2772, 2528, 2742, 2868] combine_rows_per_src=[5484, 5402, 5536, 5478, 5480, 5448, 5452, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228832 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081056 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933280 weight_movement_bytes=18874368 total_nvlink_bytes=332807648 weight_to_token_ratio=0.060122 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38112 dup_rows=21860 dup_rows_remote=19112 dedup_ratio=0.333557 router_tokens_per_expert=[242, 249, 268, 263, 242, 241, 278, 263, 228, 228, 252, 267, 272, 253, 247, 280, 254, 267, 267, 255, 264, 255, 252, 239, 260, 265, 268, 265, 275, 264, 245, 269, 253, 255, 243, 262, 230, 264, 268, 238, 275, 268, 259, 246, 264, 244, 266, 263, 302, 234, 275, 261, 247, 289, 249, 247, 215, 292, 269, 231, 277, 227, 251, 247, 247, 268, 262, 258, 236, 249, 251, 258, 242, 265, 255, 282, 258, 271, 252, 245, 227, 239, 255, 258, 247, 247, 258, 222, 266, 285, 293, 271, 276, 232, 236, 236, 271, 256, 234, 237, 255, 277, 267, 270, 237, 291, 246, 244, 262, 254, 256, 278, 246, 237, 253, 244, 258, 267, 264, 249, 267, 253, 255, 250, 269, 244, 234, 248] payload_rows_per_dst=[5564, 5564, 5424, 5404, 5514, 5434, 5340, 5432] dup_rows_per_dst=[2748, 2870, 2786, 2716, 2848, 2662, 2586, 2644] combine_rows_per_src=[5564, 5564, 5424, 5404, 5514, 5434, 5340, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229720 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311320 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392920 weight_movement_bytes=18874368 total_nvlink_bytes=333267288 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229112 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156802808 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376504 weight_movement_bytes=18874368 total_nvlink_bytes=332250872 weight_to_token_ratio=0.060229 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228416 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156933184 ingress_bytes=194453504 link_bytes=194453504 token_bytes=313637952 weight_movement_bytes=37748736 total_nvlink_bytes=351386688 weight_to_token_ratio=0.120358 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863157798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155959296 dispatch_meta_bytes=229512 combine_payload_bytes=155959296 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156188808 ingress_bytes=193708032 link_bytes=193708032 token_bytes=312148104 weight_movement_bytes=37748736 total_nvlink_bytes=349896840 weight_to_token_ratio=0.120932 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856015052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=229056 combine_payload_bytes=155926528 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156155584 ingress_bytes=212549632 link_bytes=212549632 token_bytes=312082112 weight_movement_bytes=56623104 total_nvlink_bytes=368705216 weight_to_token_ratio=0.181437 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036545993e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229448 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157302856 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376264 weight_movement_bytes=18874368 total_nvlink_bytes=333250632 weight_to_token_ratio=0.060038 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228896 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156335648 ingress_bytes=174981120 link_bytes=174981120 token_bytes=312442400 weight_movement_bytes=18874368 total_nvlink_bytes=331316768 weight_to_token_ratio=0.060409 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676582996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38232 dup_rows=21816 dup_rows_remote=19062 dedup_ratio=0.332886 router_tokens_per_expert=[272, 244, 250, 245, 246, 261, 235, 266, 255, 275, 226, 242, 274, 251, 261, 290, 262, 249, 265, 250, 261, 244, 237, 312, 264, 243, 244, 263, 263, 258, 248, 258, 259, 262, 253, 276, 262, 275, 247, 257, 283, 269, 287, 234, 241, 233, 247, 237, 251, 257, 267, 276, 278, 249, 263, 256, 266, 258, 257, 265, 250, 240, 242, 264, 256, 252, 246, 238, 237, 259, 270, 253, 251, 241, 250, 260, 254, 241, 262, 265, 251, 258, 282, 250, 261, 300, 262, 276, 236, 253, 270, 246, 251, 263, 245, 264, 267, 267, 216, 230, 268, 256, 263, 241, 252, 275, 270, 228, 225, 267, 277, 256, 258, 253, 250, 267, 263, 258, 262, 277, 234, 227, 258, 263, 254, 226, 221, 261] payload_rows_per_dst=[5418, 5488, 5550, 5524, 5394, 5542, 5464, 5340] dup_rows_per_dst=[2768, 2754, 2694, 2754, 2676, 2794, 2652, 2724] combine_rows_per_src=[5418, 5488, 5550, 5524, 5394, 5542, 5464, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38276 dup_rows=21852 dup_rows_remote=19060 dedup_ratio=0.333435 router_tokens_per_expert=[248, 265, 248, 290, 272, 256, 244, 258, 256, 253, 241, 268, 276, 250, 248, 275, 275, 252, 260, 253, 246, 242, 264, 247, 258, 243, 226, 270, 228, 244, 297, 229, 241, 271, 249, 254, 254, 255, 283, 256, 260, 256, 254, 245, 236, 289, 241, 261, 250, 251, 248, 235, 273, 233, 253, 222, 273, 259, 243, 262, 289, 274, 242, 280, 282, 269, 230, 249, 220, 269, 254, 273, 221, 248, 275, 264, 253, 285, 258, 250, 250, 230, 263, 252, 221, 244, 259, 258, 254, 255, 277, 248, 270, 225, 292, 235, 274, 253, 286, 238, 244, 235, 255, 278, 251, 274, 275, 272, 266, 247, 240, 247, 254, 244, 232, 275, 288, 293, 240, 250, 232, 262, 248, 243, 249, 268, 258, 290] payload_rows_per_dst=[5530, 5426, 5448, 5482, 5408, 5444, 5438, 5508] dup_rows_per_dst=[2766, 2642, 2762, 2692, 2792, 2622, 2832, 2744] combine_rows_per_src=[5530, 5426, 5448, 5482, 5408, 5444, 5438, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38330 dup_rows=21774 dup_rows_remote=18994 dedup_ratio=0.332245 router_tokens_per_expert=[261, 253, 233, 259, 276, 284, 262, 270, 256, 247, 226, 264, 247, 240, 257, 274, 259, 260, 252, 236, 256, 254, 240, 292, 251, 236, 260, 255, 258, 268, 259, 243, 252, 246, 263, 261, 274, 249, 243, 246, 222, 259, 250, 249, 254, 253, 266, 255, 246, 266, 258, 261, 274, 231, 280, 269, 278, 265, 266, 251, 260, 253, 255, 244, 240, 256, 233, 267, 249, 271, 234, 231, 257, 260, 265, 257, 257, 262, 248, 256, 246, 251, 258, 256, 250, 237, 258, 243, 263, 250, 249, 270, 259, 272, 273, 271, 270, 273, 251, 271, 229, 272, 243, 266, 249, 259, 244, 257, 261, 259, 293, 280, 267, 285, 252, 264, 258, 256, 271, 230, 263, 248, 235, 239, 271, 243, 237, 236] payload_rows_per_dst=[5484, 5490, 5482, 5486, 5394, 5432, 5556, 5438] dup_rows_per_dst=[2734, 2668, 2602, 2828, 2692, 2780, 2798, 2672] combine_rows_per_src=[5484, 5490, 5482, 5486, 5394, 5432, 5556, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38358 dup_rows=21748 dup_rows_remote=19018 dedup_ratio=0.331848 router_tokens_per_expert=[260, 256, 235, 253, 249, 267, 242, 269, 243, 229, 251, 266, 232, 233, 242, 264, 241, 255, 235, 258, 275, 273, 249, 260, 243, 283, 287, 284, 278, 261, 261, 284, 242, 247, 259, 222, 238, 303, 257, 287, 266, 296, 228, 230, 260, 247, 240, 258, 266, 261, 254, 254, 262, 250, 268, 242, 237, 244, 250, 254, 236, 231, 287, 230, 286, 275, 246, 279, 245, 278, 244, 254, 247, 247, 262, 265, 239, 242, 257, 236, 252, 242, 236, 227, 268, 276, 266, 260, 214, 225, 259, 267, 251, 257, 266, 255, 254, 294, 253, 249, 258, 248, 270, 251, 236, 271, 268, 261, 257, 285, 260, 270, 268, 233, 276, 276, 259, 259, 257, 281, 290, 240, 253, 247, 226, 247, 265, 259] payload_rows_per_dst=[5390, 5552, 5430, 5384, 5480, 5468, 5572, 5512] dup_rows_per_dst=[2592, 2902, 2730, 2668, 2724, 2574, 2798, 2760] combine_rows_per_src=[5390, 5552, 5430, 5384, 5480, 5468, 5572, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229176 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827448 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425720 weight_movement_bytes=37748736 total_nvlink_bytes=351174456 weight_to_token_ratio=0.120439 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38222 dup_rows=21730 dup_rows_remote=19000 dedup_ratio=0.331573 router_tokens_per_expert=[285, 259, 238, 279, 246, 264, 273, 235, 251, 269, 261, 287, 221, 256, 278, 243, 247, 221, 234, 237, 254, 263, 268, 283, 226, 264, 272, 259, 262, 257, 245, 257, 265, 249, 263, 257, 245, 239, 279, 278, 267, 255, 265, 265, 266, 254, 282, 240, 255, 243, 266, 253, 271, 249, 240, 274, 243, 257, 252, 262, 258, 248, 226, 293, 246, 238, 258, 293, 251, 250, 275, 262, 244, 246, 253, 253, 262, 275, 255, 247, 279, 259, 248, 307, 243, 255, 234, 262, 252, 251, 281, 240, 214, 254, 255, 269, 249, 270, 273, 238, 247, 251, 233, 257, 238, 248, 255, 262, 253, 270, 256, 277, 267, 240, 237, 255, 281, 226, 230, 257, 270, 257, 257, 257, 238, 241, 280, 234] payload_rows_per_dst=[5478, 5264, 5368, 5558, 5648, 5530, 5376, 5584] dup_rows_per_dst=[2812, 2470, 2672, 2898, 2932, 2698, 2518, 2730] combine_rows_per_src=[5478, 5264, 5368, 5558, 5648, 5530, 5376, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38196 dup_rows=21922 dup_rows_remote=19200 dedup_ratio=0.334503 router_tokens_per_expert=[253, 293, 252, 249, 274, 247, 271, 242, 250, 236, 245, 236, 241, 242, 240, 239, 278, 268, 267, 250, 252, 250, 279, 262, 224, 252, 253, 253, 286, 273, 256, 257, 273, 242, 241, 268, 230, 238, 263, 268, 244, 272, 263, 267, 236, 279, 270, 272, 246, 243, 253, 276, 246, 269, 269, 239, 256, 255, 254, 244, 254, 249, 256, 261, 270, 245, 268, 268, 238, 242, 242, 238, 239, 245, 274, 240, 260, 270, 257, 247, 249, 257, 280, 251, 248, 261, 238, 267, 253, 274, 236, 267, 259, 257, 244, 271, 240, 252, 275, 276, 232, 270, 236, 270, 265, 241, 229, 261, 263, 229, 247, 296, 284, 259, 266, 258, 240, 245, 242, 304, 273, 246, 231, 253, 284, 249, 267, 264] payload_rows_per_dst=[5400, 5536, 5470, 5418, 5440, 5430, 5414, 5506] dup_rows_per_dst=[2620, 2784, 2782, 2722, 2646, 2794, 2750, 2824] combine_rows_per_src=[5400, 5536, 5470, 5418, 5440, 5430, 5414, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38328 dup_rows=21608 dup_rows_remote=18832 dedup_ratio=0.329712 router_tokens_per_expert=[258, 240, 243, 248, 241, 246, 276, 260, 250, 254, 258, 259, 241, 237, 248, 234, 253, 274, 244, 247, 248, 290, 281, 240, 262, 243, 234, 259, 228, 241, 257, 248, 259, 272, 269, 247, 263, 259, 289, 243, 261, 251, 255, 271, 260, 234, 241, 289, 261, 244, 237, 236, 246, 273, 292, 269, 276, 247, 259, 258, 290, 251, 256, 242, 246, 275, 258, 229, 234, 272, 241, 286, 248, 247, 224, 280, 261, 288, 251, 239, 239, 258, 257, 253, 259, 264, 244, 269, 264, 254, 271, 260, 265, 231, 261, 259, 262, 243, 256, 260, 266, 250, 295, 257, 274, 274, 273, 262, 264, 279, 244, 229, 254, 285, 269, 238, 249, 262, 267, 255, 248, 244, 244, 257, 233, 264, 225, 257] payload_rows_per_dst=[5428, 5418, 5560, 5506, 5466, 5522, 5600, 5428] dup_rows_per_dst=[2558, 2680, 2766, 2768, 2692, 2694, 2776, 2674] combine_rows_per_src=[5428, 5418, 5560, 5506, 5466, 5522, 5600, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229344 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007840 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786336 weight_movement_bytes=18874368 total_nvlink_bytes=332660704 weight_to_token_ratio=0.060150 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229296 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157228976 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228656 weight_movement_bytes=37748736 total_nvlink_bytes=351977392 weight_to_token_ratio=0.120131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229504 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157343872 ingress_bytes=232611840 link_bytes=232611840 token_bytes=314458240 weight_movement_bytes=75497472 total_nvlink_bytes=389955712 weight_to_token_ratio=0.240087 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228772199e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=228888 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786200 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343512 weight_movement_bytes=18874368 total_nvlink_bytes=332217880 weight_to_token_ratio=0.060235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38212 dup_rows=21762 dup_rows_remote=18892 dedup_ratio=0.332062 router_tokens_per_expert=[246, 291, 246, 268, 249, 274, 261, 263, 273, 244, 239, 263, 270, 253, 261, 266, 250, 248, 289, 255, 260, 253, 241, 264, 256, 255, 252, 267, 246, 274, 215, 248, 273, 265, 235, 258, 247, 261, 266, 274, 248, 286, 264, 257, 234, 245, 249, 254, 264, 277, 230, 236, 251, 245, 269, 253, 220, 258, 239, 270, 268, 281, 262, 282, 233, 230, 259, 235, 243, 248, 267, 264, 231, 239, 264, 231, 259, 279, 253, 260, 243, 272, 237, 223, 260, 232, 259, 281, 240, 238, 274, 289, 252, 246, 258, 253, 244, 278, 255, 260, 236, 268, 258, 240, 260, 239, 260, 266, 227, 278, 276, 248, 263, 251, 278, 238, 227, 307, 302, 246, 247, 236, 244, 238, 261, 293, 260, 271] payload_rows_per_dst=[5562, 5586, 5452, 5602, 5430, 5506, 5458, 5178] dup_rows_per_dst=[2870, 2842, 2702, 2846, 2560, 2850, 2708, 2384] combine_rows_per_src=[5562, 5586, 5452, 5602, 5430, 5506, 5458, 5178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229584 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680400 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131216 weight_movement_bytes=18874368 total_nvlink_bytes=332005584 weight_to_token_ratio=0.060276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=228640 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157220128 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314211616 weight_movement_bytes=37748736 total_nvlink_bytes=351960352 weight_to_token_ratio=0.120138 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43470 payload_rows_remote=38074 dup_rows=22066 dup_rows_remote=19364 dedup_ratio=0.336700 router_tokens_per_expert=[262, 282, 256, 268, 276, 236, 253, 225, 270, 256, 236, 263, 256, 267, 268, 247, 284, 259, 260, 270, 260, 237, 256, 241, 264, 261, 252, 226, 240, 241, 240, 258, 242, 264, 234, 263, 280, 234, 246, 246, 290, 273, 255, 250, 266, 252, 257, 246, 275, 262, 246, 222, 250, 243, 257, 253, 236, 266, 251, 259, 273, 250, 242, 265, 271, 255, 280, 250, 260, 239, 268, 256, 246, 244, 250, 261, 267, 243, 253, 264, 237, 278, 257, 261, 240, 258, 261, 263, 272, 267, 240, 276, 229, 282, 257, 285, 242, 236, 239, 248, 232, 265, 276, 265, 255, 260, 256, 278, 276, 278, 243, 260, 269, 259, 249, 256, 235, 260, 252, 267, 277, 251, 269, 249, 237, 263, 233, 245] payload_rows_per_dst=[5530, 5396, 5364, 5410, 5450, 5444, 5458, 5418] dup_rows_per_dst=[2712, 2702, 2832, 2690, 2764, 2882, 2760, 2724] combine_rows_per_src=[5530, 5396, 5364, 5410, 5450, 5444, 5458, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228416 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156744768 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313261120 weight_movement_bytes=37748736 total_nvlink_bytes=351009856 weight_to_token_ratio=0.120502 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38194 dup_rows=21884 dup_rows_remote=19144 dedup_ratio=0.333923 router_tokens_per_expert=[275, 249, 253, 259, 262, 262, 254, 266, 244, 242, 239, 232, 249, 266, 217, 269, 237, 260, 250, 247, 273, 237, 267, 242, 260, 258, 229, 263, 240, 285, 272, 268, 281, 238, 272, 266, 252, 252, 239, 248, 261, 273, 235, 239, 249, 257, 290, 239, 279, 255, 249, 247, 252, 266, 252, 246, 275, 256, 274, 264, 263, 265, 254, 268, 262, 272, 269, 266, 244, 267, 239, 261, 262, 265, 247, 263, 246, 236, 237, 263, 259, 243, 254, 236, 244, 248, 257, 244, 263, 279, 261, 270, 253, 252, 239, 272, 239, 275, 269, 242, 285, 274, 264, 270, 237, 238, 243, 255, 258, 264, 302, 238, 268, 246, 232, 273, 260, 262, 216, 253, 248, 251, 263, 262, 261, 242, 265, 258] payload_rows_per_dst=[5422, 5416, 5452, 5502, 5458, 5424, 5480, 5498] dup_rows_per_dst=[2654, 2760, 2730, 2828, 2740, 2724, 2826, 2622] combine_rows_per_src=[5422, 5416, 5452, 5502, 5458, 5424, 5480, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=229752 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156180856 ingress_bytes=193699840 link_bytes=193699840 token_bytes=312131960 weight_movement_bytes=37748736 total_nvlink_bytes=349880696 weight_to_token_ratio=0.120938 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855936561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38224 dup_rows=21752 dup_rows_remote=19008 dedup_ratio=0.331909 router_tokens_per_expert=[251, 247, 274, 256, 257, 242, 244, 245, 261, 256, 265, 245, 261, 290, 259, 263, 282, 264, 249, 242, 257, 249, 240, 265, 264, 259, 244, 247, 239, 261, 257, 249, 274, 234, 271, 252, 252, 268, 249, 239, 272, 262, 280, 242, 278, 249, 255, 252, 229, 248, 269, 260, 261, 253, 240, 268, 266, 261, 207, 288, 267, 285, 297, 290, 220, 270, 251, 266, 238, 226, 291, 272, 260, 245, 228, 253, 244, 245, 248, 238, 249, 277, 266, 236, 257, 292, 249, 256, 261, 257, 279, 264, 259, 240, 257, 253, 251, 254, 256, 261, 222, 278, 232, 257, 247, 260, 245, 232, 245, 240, 265, 277, 277, 264, 260, 247, 278, 261, 281, 245, 260, 245, 221, 285, 232, 244, 247, 250] payload_rows_per_dst=[5506, 5430, 5492, 5566, 5326, 5560, 5394, 5510] dup_rows_per_dst=[2726, 2706, 2766, 2812, 2664, 2744, 2650, 2684] combine_rows_per_src=[5506, 5430, 5492, 5566, 5326, 5560, 5394, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38168 dup_rows=21880 dup_rows_remote=19178 dedup_ratio=0.333862 router_tokens_per_expert=[260, 252, 267, 270, 262, 234, 267, 259, 256, 262, 245, 250, 234, 281, 250, 263, 270, 264, 258, 240, 249, 237, 286, 278, 252, 273, 271, 309, 245, 263, 237, 250, 250, 277, 272, 248, 232, 254, 266, 239, 258, 263, 230, 240, 251, 274, 241, 240, 261, 261, 261, 260, 237, 256, 293, 219, 245, 267, 249, 254, 282, 246, 242, 262, 229, 258, 287, 250, 255, 237, 262, 261, 262, 256, 254, 246, 254, 275, 247, 236, 240, 263, 248, 264, 266, 269, 240, 257, 304, 253, 246, 249, 247, 260, 275, 252, 254, 257, 261, 232, 265, 273, 262, 240, 246, 259, 272, 265, 259, 237, 228, 250, 275, 261, 232, 292, 239, 225, 256, 243, 237, 252, 231, 278, 249, 255, 285, 272] payload_rows_per_dst=[5482, 5412, 5488, 5450, 5422, 5462, 5452, 5488] dup_rows_per_dst=[2742, 2646, 2888, 2740, 2716, 2804, 2642, 2702] combine_rows_per_src=[5482, 5412, 5488, 5450, 5422, 5462, 5452, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38128 dup_rows=21954 dup_rows_remote=19390 dedup_ratio=0.334991 router_tokens_per_expert=[259, 271, 255, 253, 253, 263, 255, 260, 248, 261, 282, 263, 267, 254, 246, 215, 238, 239, 239, 260, 240, 272, 253, 286, 239, 254, 263, 267, 272, 212, 285, 297, 248, 281, 250, 238, 248, 255, 223, 255, 226, 259, 257, 254, 278, 260, 227, 250, 242, 257, 233, 232, 259, 254, 231, 242, 242, 273, 262, 260, 258, 259, 270, 266, 258, 231, 252, 268, 249, 234, 268, 262, 270, 272, 247, 289, 266, 272, 272, 270, 273, 273, 252, 262, 259, 249, 253, 252, 226, 263, 251, 267, 259, 249, 246, 248, 255, 252, 253, 249, 258, 274, 281, 275, 258, 256, 246, 235, 274, 256, 257, 237, 249, 271, 250, 266, 268, 279, 246, 222, 247, 243, 268, 257, 247, 281, 265, 261] payload_rows_per_dst=[5474, 5420, 5454, 5430, 5490, 5468, 5424, 5422] dup_rows_per_dst=[2736, 2812, 2564, 2650, 2870, 2696, 2808, 2818] combine_rows_per_src=[5474, 5420, 5454, 5430, 5490, 5468, 5424, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229352 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671976 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114600 weight_movement_bytes=18874368 total_nvlink_bytes=331988968 weight_to_token_ratio=0.060279 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38208 dup_rows=21802 dup_rows_remote=19108 dedup_ratio=0.332672 router_tokens_per_expert=[241, 239, 269, 285, 249, 244, 232, 251, 252, 273, 261, 257, 260, 292, 240, 249, 259, 250, 254, 268, 261, 246, 271, 249, 239, 260, 271, 277, 256, 268, 263, 230, 256, 283, 299, 299, 257, 232, 249, 262, 282, 238, 275, 256, 235, 237, 248, 244, 243, 253, 243, 254, 279, 248, 247, 267, 263, 255, 247, 258, 268, 260, 271, 254, 261, 265, 262, 256, 242, 228, 248, 236, 258, 249, 254, 249, 231, 262, 256, 267, 264, 261, 256, 263, 255, 263, 268, 271, 260, 261, 258, 239, 264, 257, 225, 246, 274, 268, 236, 260, 241, 265, 252, 255, 267, 239, 254, 253, 258, 253, 251, 267, 234, 240, 245, 248, 275, 247, 266, 268, 264, 265, 260, 249, 247, 247, 230, 277] payload_rows_per_dst=[5432, 5466, 5552, 5526, 5440, 5482, 5386, 5450] dup_rows_per_dst=[2756, 2778, 2752, 2694, 2608, 2740, 2800, 2674] combine_rows_per_src=[5432, 5466, 5552, 5526, 5440, 5482, 5386, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38308 dup_rows=21702 dup_rows_remote=18906 dedup_ratio=0.331146 router_tokens_per_expert=[258, 257, 262, 220, 272, 246, 289, 245, 286, 232, 299, 291, 262, 224, 259, 275, 243, 278, 268, 260, 287, 245, 289, 250, 236, 252, 228, 245, 261, 267, 291, 278, 272, 263, 248, 264, 264, 281, 245, 254, 235, 268, 251, 254, 262, 240, 257, 258, 217, 240, 225, 229, 249, 230, 254, 223, 256, 273, 248, 262, 269, 250, 243, 291, 260, 256, 250, 228, 262, 252, 233, 247, 250, 267, 240, 235, 267, 275, 261, 257, 252, 254, 251, 258, 256, 258, 248, 239, 272, 248, 262, 236, 270, 280, 271, 240, 256, 291, 253, 266, 239, 264, 246, 255, 279, 252, 267, 247, 241, 251, 286, 268, 269, 275, 249, 252, 247, 282, 239, 241, 240, 258, 258, 256, 238, 230, 249, 259] payload_rows_per_dst=[5540, 5550, 5484, 5384, 5448, 5520, 5526, 5382] dup_rows_per_dst=[2814, 2806, 2748, 2534, 2632, 2670, 2796, 2702] combine_rows_per_src=[5540, 5550, 5484, 5384, 5448, 5520, 5526, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229384 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156565512 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312901640 weight_movement_bytes=18874368 total_nvlink_bytes=331776008 weight_to_token_ratio=0.060320 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228928 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156794432 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313359936 weight_movement_bytes=56623104 total_nvlink_bytes=369983040 weight_to_token_ratio=0.180697 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=230072 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156402360 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312574648 weight_movement_bytes=18874368 total_nvlink_bytes=331449016 weight_to_token_ratio=0.060384 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229264 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156729232 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313229200 weight_movement_bytes=37748736 total_nvlink_bytes=350977936 weight_to_token_ratio=0.120515 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=228856 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157138424 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314047992 weight_movement_bytes=37748736 total_nvlink_bytes=351796728 weight_to_token_ratio=0.120201 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.093750 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38218 dup_rows=21862 dup_rows_remote=19078 dedup_ratio=0.333588 router_tokens_per_expert=[256, 241, 250, 263, 265, 261, 259, 246, 255, 282, 280, 242, 255, 248, 266, 245, 244, 244, 245, 265, 251, 242, 281, 286, 248, 267, 271, 261, 243, 234, 275, 246, 254, 242, 221, 252, 271, 236, 256, 254, 251, 270, 251, 244, 248, 244, 265, 264, 276, 263, 246, 268, 247, 276, 240, 260, 276, 257, 272, 261, 245, 262, 239, 260, 254, 257, 260, 271, 280, 269, 245, 242, 270, 276, 253, 250, 225, 234, 252, 276, 244, 280, 291, 251, 269, 230, 246, 251, 236, 271, 269, 275, 242, 248, 256, 269, 266, 254, 267, 267, 253, 258, 275, 271, 231, 238, 246, 251, 233, 269, 274, 262, 250, 235, 269, 257, 249, 274, 248, 233, 236, 258, 243, 264, 250, 260, 247, 250] payload_rows_per_dst=[5456, 5618, 5220, 5520, 5502, 5642, 5358, 5358] dup_rows_per_dst=[2784, 3068, 2346, 2836, 2728, 2966, 2520, 2614] combine_rows_per_src=[5456, 5618, 5220, 5520, 5502, 5642, 5358, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38292 dup_rows=21846 dup_rows_remote=19104 dedup_ratio=0.333344 router_tokens_per_expert=[241, 259, 264, 233, 264, 247, 268, 240, 252, 241, 269, 271, 265, 277, 263, 264, 253, 263, 271, 262, 255, 219, 254, 244, 267, 246, 281, 242, 247, 232, 263, 271, 249, 253, 282, 231, 268, 261, 233, 254, 224, 253, 220, 244, 261, 268, 255, 263, 247, 244, 268, 241, 260, 272, 234, 241, 253, 273, 239, 222, 286, 267, 270, 255, 261, 280, 271, 264, 272, 243, 256, 235, 279, 273, 257, 264, 275, 257, 237, 247, 264, 254, 226, 271, 251, 247, 268, 265, 234, 266, 246, 266, 265, 238, 258, 262, 271, 276, 261, 254, 253, 269, 250, 260, 271, 248, 267, 269, 254, 254, 253, 251, 255, 259, 243, 256, 269, 270, 265, 243, 228, 242, 260, 249, 257, 264, 261, 255] payload_rows_per_dst=[5488, 5398, 5422, 5470, 5580, 5472, 5436, 5424] dup_rows_per_dst=[2748, 2742, 2628, 2674, 2762, 2690, 2874, 2728] combine_rows_per_src=[5488, 5398, 5422, 5470, 5580, 5472, 5436, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38302 dup_rows=21838 dup_rows_remote=19204 dedup_ratio=0.333221 router_tokens_per_expert=[229, 292, 236, 251, 261, 285, 269, 267, 227, 239, 276, 249, 273, 269, 250, 252, 262, 252, 253, 258, 252, 240, 256, 251, 267, 280, 270, 258, 260, 262, 241, 256, 271, 256, 243, 248, 271, 207, 279, 246, 242, 286, 233, 272, 253, 244, 278, 259, 250, 248, 242, 267, 237, 249, 236, 246, 264, 276, 249, 256, 261, 238, 254, 264, 237, 248, 241, 267, 243, 232, 261, 239, 283, 252, 265, 253, 274, 223, 259, 238, 257, 239, 255, 241, 253, 273, 243, 265, 257, 256, 221, 302, 247, 234, 259, 266, 283, 270, 232, 242, 258, 267, 250, 287, 260, 245, 254, 244, 279, 258, 248, 289, 267, 292, 228, 264, 269, 271, 231, 271, 276, 272, 228, 218, 240, 267, 270, 287] payload_rows_per_dst=[5478, 5520, 5390, 5394, 5396, 5454, 5510, 5556] dup_rows_per_dst=[2772, 2716, 2786, 2680, 2634, 2682, 2822, 2746] combine_rows_per_src=[5478, 5520, 5390, 5394, 5396, 5454, 5510, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229184 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156770112 ingress_bytes=213164032 link_bytes=213164032 token_bytes=313311040 weight_movement_bytes=56623104 total_nvlink_bytes=369934144 weight_to_token_ratio=0.180725 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042432871e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229584 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073616 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917648 weight_movement_bytes=18874368 total_nvlink_bytes=332792016 weight_to_token_ratio=0.060125 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38272 dup_rows=21734 dup_rows_remote=18976 dedup_ratio=0.331635 router_tokens_per_expert=[263, 249, 264, 250, 241, 250, 261, 270, 277, 256, 262, 231, 276, 245, 285, 254, 239, 252, 265, 244, 284, 264, 270, 268, 258, 257, 249, 268, 294, 246, 240, 253, 294, 242, 257, 236, 266, 240, 272, 245, 222, 232, 244, 282, 239, 265, 258, 249, 266, 247, 257, 245, 266, 281, 268, 259, 264, 246, 276, 267, 235, 239, 231, 254, 239, 262, 265, 268, 273, 238, 271, 262, 274, 259, 268, 229, 275, 240, 284, 236, 251, 249, 247, 258, 243, 247, 233, 261, 238, 260, 245, 236, 228, 241, 263, 238, 257, 224, 265, 248, 224, 247, 265, 264, 231, 245, 288, 285, 278, 250, 277, 272, 285, 241, 289, 237, 263, 264, 257, 250, 248, 234, 242, 271, 278, 262, 256, 261] payload_rows_per_dst=[5330, 5534, 5428, 5468, 5570, 5464, 5478, 5530] dup_rows_per_dst=[2590, 2768, 2658, 2734, 2680, 2760, 2786, 2758] combine_rows_per_src=[5330, 5534, 5428, 5468, 5570, 5464, 5478, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=230024 combine_payload_bytes=156884992 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157115016 ingress_bytes=213508096 link_bytes=213508096 token_bytes=314000008 weight_movement_bytes=56623104 total_nvlink_bytes=370623112 weight_to_token_ratio=0.180328 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045729523e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38326 dup_rows=21814 dup_rows_remote=19020 dedup_ratio=0.332855 router_tokens_per_expert=[268, 236, 261, 259, 255, 269, 236, 271, 273, 288, 266, 276, 252, 271, 272, 268, 267, 255, 231, 264, 222, 264, 237, 251, 247, 235, 249, 234, 229, 268, 249, 241, 269, 257, 247, 260, 223, 254, 239, 270, 254, 269, 287, 258, 251, 249, 264, 249, 275, 260, 244, 262, 272, 264, 226, 227, 251, 249, 279, 265, 245, 251, 255, 238, 250, 266, 267, 249, 239, 253, 259, 272, 235, 269, 260, 236, 252, 223, 246, 244, 253, 254, 240, 230, 288, 257, 281, 282, 228, 260, 244, 283, 258, 237, 259, 241, 294, 245, 258, 267, 265, 273, 243, 246, 262, 266, 243, 243, 259, 266, 256, 270, 265, 254, 252, 267, 231, 266, 262, 262, 249, 269, 272, 264, 254, 279, 276, 248] payload_rows_per_dst=[5618, 5406, 5374, 5438, 5430, 5396, 5554, 5506] dup_rows_per_dst=[2824, 2480, 2826, 2688, 2610, 2794, 2758, 2834] combine_rows_per_src=[5618, 5406, 5374, 5438, 5430, 5396, 5554, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38374 dup_rows=21698 dup_rows_remote=19006 dedup_ratio=0.331085 router_tokens_per_expert=[260, 268, 260, 256, 267, 241, 272, 266, 245, 266, 256, 257, 260, 274, 244, 263, 242, 251, 253, 246, 253, 263, 292, 267, 271, 265, 270, 263, 261, 243, 265, 250, 263, 245, 266, 237, 247, 242, 268, 278, 259, 280, 248, 269, 272, 255, 240, 237, 267, 270, 262, 259, 275, 256, 251, 276, 241, 233, 272, 243, 255, 240, 240, 238, 246, 273, 267, 237, 255, 213, 260, 281, 225, 246, 266, 265, 264, 268, 267, 249, 267, 255, 254, 263, 247, 248, 238, 261, 268, 231, 275, 251, 250, 257, 235, 239, 279, 237, 241, 217, 273, 255, 261, 254, 261, 255, 266, 247, 255, 238, 250, 240, 237, 256, 269, 258, 276, 256, 255, 272, 267, 251, 261, 246, 268, 250, 250, 252] payload_rows_per_dst=[5468, 5516, 5506, 5464, 5500, 5470, 5384, 5530] dup_rows_per_dst=[2842, 2794, 2706, 2692, 2664, 2608, 2674, 2718] combine_rows_per_src=[5468, 5516, 5506, 5464, 5500, 5470, 5384, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38368 dup_rows=21704 dup_rows_remote=19024 dedup_ratio=0.331177 router_tokens_per_expert=[260, 239, 264, 252, 252, 271, 258, 241, 276, 234, 258, 251, 220, 234, 248, 235, 298, 252, 266, 290, 255, 270, 267, 248, 255, 239, 226, 234, 259, 281, 262, 257, 233, 243, 245, 261, 245, 249, 261, 240, 247, 267, 267, 254, 256, 276, 255, 273, 267, 239, 284, 254, 255, 259, 274, 230, 246, 264, 269, 270, 241, 277, 261, 256, 243, 253, 235, 246, 249, 267, 272, 269, 244, 244, 274, 260, 273, 241, 257, 246, 264, 275, 246, 275, 262, 237, 271, 280, 271, 271, 255, 236, 264, 265, 252, 240, 249, 245, 269, 249, 275, 251, 264, 266, 265, 231, 241, 262, 278, 263, 217, 261, 285, 253, 268, 248, 226, 271, 266, 250, 266, 248, 221, 257, 262, 251, 249, 254] payload_rows_per_dst=[5442, 5528, 5464, 5478, 5478, 5560, 5436, 5446] dup_rows_per_dst=[2544, 2790, 2680, 2814, 2668, 2768, 2736, 2704] combine_rows_per_src=[5442, 5528, 5464, 5478, 5478, 5560, 5436, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38256 dup_rows=21732 dup_rows_remote=18956 dedup_ratio=0.331604 router_tokens_per_expert=[254, 240, 240, 245, 247, 231, 264, 280, 267, 265, 269, 268, 265, 243, 258, 235, 228, 276, 286, 265, 235, 249, 238, 254, 262, 291, 245, 264, 228, 260, 266, 261, 262, 289, 256, 214, 262, 282, 247, 239, 259, 250, 268, 242, 273, 240, 251, 265, 234, 251, 269, 271, 241, 256, 240, 285, 263, 283, 254, 266, 242, 224, 266, 253, 253, 247, 247, 255, 281, 270, 254, 251, 232, 258, 247, 224, 282, 227, 260, 247, 265, 285, 271, 252, 262, 248, 256, 246, 261, 250, 266, 258, 259, 277, 234, 248, 241, 252, 280, 265, 243, 259, 256, 234, 262, 287, 261, 283, 248, 256, 266, 269, 258, 257, 246, 245, 246, 246, 239, 264, 261, 273, 253, 266, 258, 234, 272, 239] payload_rows_per_dst=[5414, 5490, 5490, 5446, 5458, 5488, 5548, 5470] dup_rows_per_dst=[2728, 2726, 2708, 2750, 2612, 2788, 2776, 2644] combine_rows_per_src=[5414, 5490, 5490, 5446, 5458, 5488, 5548, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228992 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156991104 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313753216 weight_movement_bytes=37748736 total_nvlink_bytes=351501952 weight_to_token_ratio=0.120313 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229384 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157212680 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314195976 weight_movement_bytes=18874368 total_nvlink_bytes=333070344 weight_to_token_ratio=0.060072 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229520 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409424 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589328 weight_movement_bytes=18874368 total_nvlink_bytes=333463696 weight_to_token_ratio=0.059997 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229568 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384896 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540224 weight_movement_bytes=18874368 total_nvlink_bytes=333414592 weight_to_token_ratio=0.060006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228848 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156925424 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313622000 weight_movement_bytes=56623104 total_nvlink_bytes=370245104 weight_to_token_ratio=0.180546 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38188 dup_rows=21904 dup_rows_remote=19158 dedup_ratio=0.334229 router_tokens_per_expert=[251, 258, 249, 266, 253, 269, 271, 248, 236, 248, 236, 253, 248, 250, 254, 264, 238, 259, 260, 254, 256, 254, 242, 266, 291, 227, 261, 267, 218, 254, 292, 253, 271, 268, 266, 251, 275, 258, 270, 247, 249, 259, 272, 240, 257, 256, 257, 269, 258, 271, 233, 273, 261, 242, 271, 232, 234, 264, 268, 267, 241, 262, 273, 252, 244, 244, 254, 252, 280, 268, 270, 259, 275, 241, 221, 246, 246, 256, 295, 264, 273, 253, 229, 291, 269, 255, 267, 267, 257, 251, 263, 267, 247, 252, 247, 272, 233, 264, 239, 275, 236, 245, 255, 245, 277, 223, 268, 238, 257, 273, 240, 257, 237, 257, 249, 279, 250, 254, 242, 263, 256, 243, 269, 249, 262, 241, 245, 259] payload_rows_per_dst=[5444, 5514, 5532, 5446, 5414, 5500, 5346, 5436] dup_rows_per_dst=[2746, 2690, 2782, 2758, 2816, 2738, 2700, 2674] combine_rows_per_src=[5444, 5514, 5532, 5446, 5414, 5500, 5346, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44064 payload_rows_remote=38606 dup_rows=21472 dup_rows_remote=18828 dedup_ratio=0.327637 router_tokens_per_expert=[249, 248, 255, 276, 263, 236, 245, 275, 264, 261, 245, 268, 266, 264, 283, 255, 256, 250, 230, 277, 267, 254, 260, 268, 245, 256, 273, 241, 248, 237, 250, 239, 265, 241, 247, 272, 255, 247, 258, 246, 246, 263, 238, 262, 244, 255, 267, 251, 252, 240, 270, 243, 292, 237, 267, 256, 267, 250, 276, 256, 265, 241, 235, 263, 261, 285, 270, 250, 241, 239, 252, 266, 266, 261, 248, 261, 260, 271, 287, 277, 259, 251, 266, 251, 238, 253, 273, 275, 242, 278, 253, 255, 279, 280, 264, 270, 241, 287, 282, 251, 243, 264, 248, 235, 247, 243, 263, 242, 233, 214, 284, 248, 271, 213, 252, 228, 248, 240, 260, 252, 246, 248, 253, 260, 272, 241, 257, 249] payload_rows_per_dst=[5528, 5458, 5500, 5568, 5562, 5514, 5476, 5458] dup_rows_per_dst=[2778, 2644, 2614, 2652, 2828, 2860, 2574, 2522] combine_rows_per_src=[5528, 5458, 5500, 5568, 5562, 5514, 5476, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38358 dup_rows=21718 dup_rows_remote=18986 dedup_ratio=0.331390 router_tokens_per_expert=[265, 253, 242, 241, 257, 206, 252, 252, 246, 233, 250, 259, 245, 232, 259, 252, 244, 231, 251, 238, 258, 265, 289, 264, 247, 250, 255, 250, 326, 224, 269, 285, 256, 230, 259, 274, 238, 265, 244, 276, 233, 243, 309, 260, 273, 273, 262, 255, 262, 246, 252, 256, 268, 278, 264, 290, 257, 250, 277, 266, 273, 281, 287, 239, 261, 259, 259, 258, 254, 250, 242, 247, 260, 267, 262, 231, 246, 266, 285, 249, 251, 257, 278, 271, 257, 262, 284, 246, 252, 262, 219, 246, 244, 244, 235, 247, 268, 244, 263, 242, 260, 255, 258, 254, 235, 287, 250, 251, 268, 259, 269, 249, 250, 226, 241, 264, 249, 256, 254, 261, 249, 252, 253, 245, 246, 259, 265, 249] payload_rows_per_dst=[5356, 5538, 5546, 5646, 5460, 5424, 5444, 5404] dup_rows_per_dst=[2532, 2754, 2754, 2846, 2732, 2686, 2780, 2634] combine_rows_per_src=[5356, 5538, 5546, 5646, 5460, 5424, 5444, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229384 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156647432 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065480 weight_movement_bytes=18874368 total_nvlink_bytes=331939848 weight_to_token_ratio=0.060289 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158130176 dispatch_meta_bytes=229736 combine_payload_bytes=158130176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158359912 ingress_bytes=177004544 link_bytes=177004544 token_bytes=316490088 weight_movement_bytes=18874368 total_nvlink_bytes=335364456 weight_to_token_ratio=0.059637 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.695970449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38110 dup_rows=21888 dup_rows_remote=18954 dedup_ratio=0.333984 router_tokens_per_expert=[255, 246, 267, 235, 267, 230, 249, 267, 231, 258, 279, 260, 255, 253, 236, 255, 264, 248, 243, 250, 262, 255, 247, 258, 240, 213, 285, 265, 242, 270, 255, 268, 243, 248, 277, 264, 258, 246, 256, 253, 256, 261, 294, 250, 235, 251, 264, 236, 260, 269, 261, 246, 274, 256, 276, 259, 278, 265, 244, 233, 275, 234, 249, 246, 243, 253, 237, 245, 272, 234, 260, 283, 278, 261, 253, 230, 246, 250, 256, 238, 252, 253, 257, 254, 254, 242, 261, 259, 250, 242, 254, 252, 245, 269, 268, 273, 256, 261, 256, 252, 265, 255, 242, 255, 268, 238, 267, 254, 264, 267, 262, 242, 278, 252, 239, 280, 265, 271, 275, 244, 253, 278, 258, 275, 281, 243, 265, 258] payload_rows_per_dst=[5422, 5404, 5354, 5374, 5424, 5606, 5526, 5538] dup_rows_per_dst=[2664, 2726, 2520, 2542, 2654, 2864, 2984, 2934] combine_rows_per_src=[5422, 5404, 5354, 5374, 5424, 5606, 5526, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229376 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343744 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314458112 weight_movement_bytes=37748736 total_nvlink_bytes=352206848 weight_to_token_ratio=0.120044 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38310 dup_rows=21822 dup_rows_remote=19160 dedup_ratio=0.332977 router_tokens_per_expert=[245, 263, 240, 270, 257, 267, 278, 244, 260, 232, 262, 286, 269, 272, 223, 262, 243, 250, 251, 229, 254, 253, 253, 246, 276, 263, 282, 245, 246, 242, 221, 263, 236, 274, 254, 276, 252, 263, 249, 264, 274, 253, 264, 256, 271, 241, 231, 263, 272, 241, 247, 279, 229, 259, 257, 243, 224, 247, 272, 233, 259, 252, 251, 268, 275, 259, 257, 272, 282, 228, 264, 254, 257, 271, 269, 254, 241, 255, 281, 260, 264, 264, 254, 238, 247, 234, 227, 240, 263, 237, 243, 274, 280, 247, 257, 259, 275, 255, 251, 268, 273, 254, 250, 262, 271, 261, 254, 245, 253, 268, 249, 227, 241, 268, 258, 260, 261, 262, 263, 272, 247, 278, 265, 249, 266, 266, 254, 234] payload_rows_per_dst=[5480, 5492, 5444, 5404, 5480, 5444, 5430, 5540] dup_rows_per_dst=[2780, 2542, 2798, 2662, 2878, 2612, 2802, 2748] combine_rows_per_src=[5480, 5492, 5444, 5404, 5480, 5444, 5430, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38350 dup_rows=21716 dup_rows_remote=19000 dedup_ratio=0.331360 router_tokens_per_expert=[262, 260, 244, 247, 261, 239, 260, 234, 250, 262, 276, 237, 291, 255, 257, 232, 271, 243, 246, 226, 230, 288, 259, 275, 274, 251, 272, 274, 297, 241, 239, 248, 264, 234, 276, 256, 259, 229, 265, 277, 250, 235, 286, 274, 261, 231, 260, 246, 234, 252, 271, 275, 281, 252, 249, 274, 244, 270, 273, 264, 270, 229, 228, 242, 253, 244, 263, 243, 258, 259, 252, 267, 252, 249, 268, 278, 257, 229, 271, 231, 262, 276, 260, 266, 246, 270, 252, 283, 255, 241, 251, 240, 257, 244, 259, 231, 246, 234, 262, 251, 261, 259, 246, 241, 284, 247, 281, 258, 250, 249, 254, 261, 227, 264, 271, 249, 279, 259, 286, 257, 248, 251, 217, 247, 261, 266, 263, 260] payload_rows_per_dst=[5466, 5494, 5498, 5436, 5440, 5470, 5506, 5510] dup_rows_per_dst=[2668, 2774, 2708, 2780, 2708, 2716, 2662, 2700] combine_rows_per_src=[5466, 5494, 5498, 5436, 5440, 5470, 5506, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38088 dup_rows=21962 dup_rows_remote=19130 dedup_ratio=0.335114 router_tokens_per_expert=[261, 264, 260, 268, 252, 217, 274, 266, 244, 257, 229, 275, 267, 269, 260, 245, 272, 252, 253, 234, 271, 252, 260, 285, 245, 245, 266, 256, 246, 245, 259, 275, 249, 269, 268, 244, 244, 254, 272, 249, 262, 245, 236, 249, 264, 265, 255, 260, 253, 240, 284, 239, 245, 216, 247, 270, 253, 236, 266, 246, 232, 240, 243, 262, 266, 275, 272, 250, 274, 244, 239, 251, 251, 266, 269, 247, 266, 246, 249, 257, 244, 255, 262, 255, 242, 235, 260, 256, 258, 247, 266, 305, 268, 269, 244, 250, 250, 245, 302, 224, 272, 257, 252, 244, 248, 298, 272, 267, 262, 264, 228, 274, 232, 264, 292, 266, 244, 241, 264, 266, 223, 279, 264, 216, 246, 251, 254, 288] payload_rows_per_dst=[5440, 5494, 5402, 5340, 5520, 5440, 5486, 5452] dup_rows_per_dst=[2776, 2738, 2768, 2604, 2724, 2792, 2832, 2728] combine_rows_per_src=[5440, 5494, 5402, 5340, 5520, 5440, 5486, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38232 dup_rows=21726 dup_rows_remote=18936 dedup_ratio=0.331512 router_tokens_per_expert=[261, 256, 247, 251, 226, 261, 259, 247, 265, 246, 258, 273, 268, 291, 256, 257, 252, 264, 261, 275, 252, 236, 258, 251, 254, 230, 262, 255, 249, 259, 282, 244, 256, 252, 267, 265, 246, 250, 239, 258, 257, 243, 267, 263, 304, 266, 259, 292, 258, 258, 249, 278, 252, 257, 246, 234, 244, 245, 260, 261, 279, 226, 245, 252, 247, 277, 282, 248, 276, 245, 254, 255, 254, 241, 261, 268, 242, 264, 246, 259, 242, 258, 245, 250, 246, 265, 261, 228, 276, 252, 238, 268, 271, 259, 241, 266, 237, 282, 253, 252, 260, 282, 266, 249, 241, 260, 229, 222, 252, 232, 252, 241, 253, 275, 255, 267, 262, 284, 239, 269, 234, 260, 257, 252, 263, 267, 250, 252] payload_rows_per_dst=[5498, 5506, 5578, 5394, 5470, 5474, 5396, 5494] dup_rows_per_dst=[2746, 2662, 2790, 2694, 2768, 2658, 2624, 2784] combine_rows_per_src=[5498, 5506, 5578, 5394, 5470, 5474, 5396, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=228256 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156326816 ingress_bytes=193847296 link_bytes=193847296 token_bytes=312425376 weight_movement_bytes=37748736 total_nvlink_bytes=350174112 weight_to_token_ratio=0.120825 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857349411e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229880 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157147640 ingress_bytes=213540864 link_bytes=213540864 token_bytes=314065400 weight_movement_bytes=56623104 total_nvlink_bytes=370688504 weight_to_token_ratio=0.180291 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046043490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228872 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156237320 ingress_bytes=212631552 link_bytes=212631552 token_bytes=312245768 weight_movement_bytes=56623104 total_nvlink_bytes=368868872 weight_to_token_ratio=0.181341 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037330910e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229400 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311000 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392600 weight_movement_bytes=18874368 total_nvlink_bytes=333266968 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228672 combine_payload_bytes=156598272 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156826944 ingress_bytes=213221376 link_bytes=213221376 token_bytes=313425216 weight_movement_bytes=56623104 total_nvlink_bytes=370048320 weight_to_token_ratio=0.180659 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042982313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38186 dup_rows=21862 dup_rows_remote=19128 dedup_ratio=0.333588 router_tokens_per_expert=[258, 261, 229, 237, 271, 229, 272, 278, 246, 252, 266, 240, 289, 250, 261, 248, 276, 247, 273, 254, 272, 254, 264, 286, 237, 253, 242, 259, 252, 260, 243, 230, 257, 284, 244, 250, 258, 248, 241, 238, 236, 230, 223, 253, 261, 234, 257, 288, 247, 237, 277, 269, 248, 232, 260, 272, 238, 262, 244, 247, 282, 233, 250, 280, 254, 242, 264, 255, 266, 260, 231, 258, 261, 263, 257, 289, 243, 230, 260, 249, 247, 250, 247, 245, 253, 282, 248, 246, 264, 258, 242, 259, 276, 259, 280, 259, 268, 286, 257, 243, 242, 266, 235, 234, 239, 267, 291, 267, 267, 238, 258, 238, 260, 276, 273, 227, 279, 276, 255, 284, 248, 253, 273, 270, 264, 258, 242, 268] payload_rows_per_dst=[5488, 5504, 5404, 5452, 5560, 5306, 5398, 5562] dup_rows_per_dst=[2734, 2872, 2600, 2590, 2958, 2524, 2734, 2850] combine_rows_per_src=[5488, 5504, 5404, 5452, 5560, 5306, 5398, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38186 dup_rows=21864 dup_rows_remote=19098 dedup_ratio=0.333618 router_tokens_per_expert=[246, 253, 265, 226, 236, 262, 244, 225, 244, 269, 254, 261, 251, 264, 267, 251, 252, 231, 253, 257, 249, 249, 277, 257, 272, 254, 267, 272, 254, 248, 253, 281, 266, 225, 252, 268, 251, 238, 280, 276, 274, 256, 266, 264, 260, 252, 233, 275, 253, 236, 262, 252, 280, 264, 249, 240, 243, 270, 266, 280, 237, 266, 276, 248, 237, 264, 248, 235, 254, 276, 256, 267, 261, 283, 239, 286, 234, 249, 239, 220, 233, 246, 251, 271, 243, 260, 259, 266, 242, 261, 251, 261, 264, 263, 245, 251, 240, 263, 261, 254, 246, 260, 259, 246, 278, 259, 279, 268, 278, 259, 258, 278, 241, 251, 273, 240, 271, 253, 238, 258, 265, 240, 232, 282, 245, 268, 256, 252] payload_rows_per_dst=[5370, 5486, 5480, 5566, 5398, 5438, 5502, 5432] dup_rows_per_dst=[2666, 2766, 2792, 2678, 2698, 2696, 2870, 2698] combine_rows_per_src=[5370, 5486, 5480, 5566, 5398, 5438, 5502, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38298 dup_rows=21816 dup_rows_remote=19028 dedup_ratio=0.332886 router_tokens_per_expert=[265, 260, 283, 264, 260, 211, 244, 261, 272, 261, 264, 274, 249, 264, 253, 259, 263, 232, 266, 274, 282, 263, 266, 261, 297, 250, 257, 275, 248, 231, 235, 260, 276, 258, 257, 226, 226, 253, 265, 270, 238, 267, 246, 244, 253, 272, 251, 276, 266, 227, 264, 236, 223, 257, 241, 269, 238, 239, 257, 221, 266, 267, 256, 253, 270, 247, 252, 255, 275, 273, 261, 210, 278, 250, 265, 233, 270, 272, 258, 236, 283, 291, 243, 247, 265, 265, 275, 275, 255, 231, 244, 253, 253, 287, 254, 255, 244, 239, 241, 241, 267, 270, 252, 260, 253, 256, 278, 239, 231, 235, 250, 248, 290, 253, 252, 224, 272, 230, 241, 265, 241, 248, 291, 249, 296, 235, 266, 268] payload_rows_per_dst=[5502, 5522, 5452, 5412, 5422, 5562, 5378, 5470] dup_rows_per_dst=[2786, 2798, 2704, 2548, 2788, 2790, 2630, 2772] combine_rows_per_src=[5502, 5522, 5452, 5412, 5422, 5562, 5378, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229256 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156639112 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048968 weight_movement_bytes=18874368 total_nvlink_bytes=331923336 weight_to_token_ratio=0.060292 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229136 combine_payload_bytes=156409856 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156638992 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048848 weight_movement_bytes=18874368 total_nvlink_bytes=331923216 weight_to_token_ratio=0.060292 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229304 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097912 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966520 weight_movement_bytes=18874368 total_nvlink_bytes=332840888 weight_to_token_ratio=0.060116 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38268 dup_rows=21834 dup_rows_remote=19028 dedup_ratio=0.333160 router_tokens_per_expert=[253, 241, 244, 294, 260, 246, 279, 262, 248, 285, 258, 259, 245, 235, 237, 251, 254, 260, 263, 247, 240, 264, 269, 266, 256, 263, 267, 246, 245, 263, 295, 264, 248, 229, 244, 258, 256, 259, 238, 234, 267, 256, 264, 251, 266, 262, 259, 273, 235, 244, 282, 248, 248, 246, 241, 264, 263, 243, 265, 240, 256, 270, 257, 243, 259, 264, 246, 254, 257, 238, 260, 232, 244, 262, 246, 289, 249, 282, 256, 259, 281, 249, 265, 260, 224, 262, 277, 260, 273, 228, 261, 274, 268, 270, 291, 256, 250, 244, 263, 269, 262, 231, 270, 257, 273, 271, 248, 229, 261, 254, 238, 263, 230, 239, 259, 237, 258, 248, 270, 262, 265, 245, 265, 228, 251, 245, 250, 269] payload_rows_per_dst=[5272, 5480, 5414, 5618, 5474, 5490, 5520, 5434] dup_rows_per_dst=[2264, 2834, 2674, 2932, 2742, 2718, 2864, 2806] combine_rows_per_src=[5272, 5480, 5414, 5618, 5474, 5490, 5520, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38300 dup_rows=21708 dup_rows_remote=19036 dedup_ratio=0.331238 router_tokens_per_expert=[243, 248, 250, 233, 259, 269, 265, 262, 262, 282, 260, 264, 252, 273, 261, 263, 253, 243, 249, 257, 242, 223, 243, 252, 257, 263, 225, 265, 270, 274, 278, 267, 255, 222, 268, 279, 254, 248, 261, 271, 233, 263, 259, 242, 286, 238, 258, 252, 248, 269, 228, 245, 274, 268, 264, 266, 262, 262, 252, 244, 261, 249, 249, 259, 249, 248, 260, 283, 237, 254, 260, 264, 251, 222, 267, 256, 234, 283, 274, 263, 222, 243, 242, 254, 264, 236, 262, 237, 258, 261, 262, 271, 265, 231, 261, 265, 262, 248, 239, 268, 249, 253, 243, 271, 237, 257, 263, 263, 261, 274, 254, 253, 245, 258, 253, 255, 266, 264, 266, 273, 257, 272, 246, 241, 259, 273, 260, 250] payload_rows_per_dst=[5462, 5370, 5508, 5528, 5548, 5490, 5450, 5472] dup_rows_per_dst=[2830, 2752, 2670, 2672, 2662, 2578, 2740, 2804] combine_rows_per_src=[5462, 5370, 5508, 5528, 5548, 5490, 5450, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38182 dup_rows=21876 dup_rows_remote=19220 dedup_ratio=0.333801 router_tokens_per_expert=[272, 239, 257, 283, 238, 240, 254, 268, 256, 235, 252, 272, 235, 282, 235, 271, 259, 275, 258, 273, 239, 250, 254, 268, 245, 270, 251, 247, 247, 263, 263, 250, 294, 260, 242, 268, 279, 250, 250, 273, 274, 255, 257, 253, 262, 252, 250, 264, 241, 282, 241, 268, 271, 268, 243, 258, 272, 243, 288, 280, 239, 243, 245, 242, 245, 244, 253, 224, 264, 232, 231, 262, 274, 256, 242, 266, 244, 262, 253, 247, 271, 265, 258, 262, 266, 265, 252, 238, 247, 224, 250, 230, 243, 266, 246, 269, 282, 268, 249, 230, 230, 256, 256, 250, 248, 275, 246, 250, 232, 295, 234, 266, 265, 244, 242, 254, 262, 271, 248, 263, 251, 288, 240, 281, 277, 259, 258, 239] payload_rows_per_dst=[5436, 5506, 5536, 5494, 5352, 5398, 5478, 5460] dup_rows_per_dst=[2742, 2718, 2830, 2754, 2646, 2706, 2656, 2824] combine_rows_per_src=[5436, 5506, 5536, 5494, 5352, 5398, 5478, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38270 dup_rows=21844 dup_rows_remote=19140 dedup_ratio=0.333313 router_tokens_per_expert=[261, 258, 284, 278, 254, 259, 288, 254, 243, 279, 265, 253, 239, 255, 267, 243, 272, 234, 247, 259, 269, 262, 237, 246, 252, 240, 224, 283, 255, 248, 250, 270, 248, 259, 238, 279, 254, 286, 258, 270, 247, 242, 237, 255, 260, 239, 254, 257, 240, 286, 250, 254, 229, 257, 261, 260, 269, 251, 267, 233, 238, 264, 256, 256, 256, 269, 303, 254, 258, 254, 245, 252, 268, 230, 245, 268, 239, 276, 255, 279, 262, 238, 240, 265, 255, 255, 267, 253, 281, 241, 268, 257, 227, 248, 255, 251, 263, 252, 256, 247, 285, 248, 249, 256, 248, 261, 256, 211, 242, 266, 263, 242, 274, 250, 264, 286, 242, 247, 245, 246, 250, 261, 268, 249, 273, 266, 254, 252] payload_rows_per_dst=[5566, 5460, 5474, 5476, 5474, 5422, 5370, 5450] dup_rows_per_dst=[2794, 2636, 2692, 2666, 2828, 2704, 2720, 2804] combine_rows_per_src=[5566, 5460, 5474, 5476, 5474, 5422, 5370, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43946 payload_rows_remote=38500 dup_rows=21590 dup_rows_remote=18748 dedup_ratio=0.329437 router_tokens_per_expert=[251, 263, 255, 269, 260, 258, 248, 261, 276, 257, 263, 235, 273, 237, 265, 248, 249, 275, 259, 285, 253, 262, 235, 249, 249, 257, 243, 237, 271, 277, 245, 261, 281, 237, 264, 258, 247, 252, 283, 268, 231, 280, 244, 240, 268, 286, 261, 244, 258, 251, 247, 258, 270, 247, 250, 252, 278, 283, 289, 237, 245, 256, 257, 235, 233, 255, 271, 251, 245, 263, 246, 240, 259, 258, 263, 244, 235, 238, 258, 269, 272, 268, 262, 262, 255, 255, 255, 233, 258, 242, 259, 262, 277, 241, 244, 243, 241, 275, 281, 239, 249, 246, 275, 274, 239, 263, 263, 261, 252, 235, 243, 246, 262, 233, 248, 250, 262, 251, 257, 264, 262, 254, 234, 246, 254, 262, 296, 252] payload_rows_per_dst=[5510, 5612, 5446, 5492, 5448, 5446, 5466, 5526] dup_rows_per_dst=[2728, 2602, 2842, 2734, 2608, 2730, 2698, 2648] combine_rows_per_src=[5510, 5612, 5446, 5492, 5448, 5446, 5466, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229184 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974912 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720640 weight_movement_bytes=18874368 total_nvlink_bytes=332595008 weight_to_token_ratio=0.060163 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229344 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106144 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982944 weight_movement_bytes=18874368 total_nvlink_bytes=332857312 weight_to_token_ratio=0.060113 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38240 dup_rows=21784 dup_rows_remote=19120 dedup_ratio=0.332397 router_tokens_per_expert=[240, 253, 240, 245, 233, 236, 249, 242, 236, 273, 247, 273, 271, 269, 293, 246, 243, 251, 257, 295, 292, 266, 237, 241, 253, 261, 213, 238, 228, 230, 276, 244, 279, 258, 217, 260, 233, 243, 268, 262, 249, 240, 245, 220, 269, 249, 224, 257, 255, 251, 269, 238, 259, 258, 260, 246, 247, 259, 247, 272, 245, 263, 236, 279, 265, 280, 285, 228, 249, 270, 242, 275, 261, 240, 264, 259, 271, 263, 305, 245, 252, 269, 294, 256, 239, 248, 276, 265, 233, 254, 262, 243, 259, 242, 255, 246, 276, 271, 251, 267, 264, 224, 294, 260, 268, 253, 256, 269, 261, 281, 253, 244, 244, 274, 249, 265, 277, 260, 294, 238, 253, 240, 237, 265, 266, 290, 257, 244] payload_rows_per_dst=[5512, 5444, 5566, 5622, 5498, 5496, 5268, 5346] dup_rows_per_dst=[2664, 2668, 2932, 2976, 2822, 2690, 2502, 2530] combine_rows_per_src=[5512, 5444, 5566, 5622, 5498, 5496, 5268, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229608 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156623080 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313016552 weight_movement_bytes=18874368 total_nvlink_bytes=331890920 weight_to_token_ratio=0.060298 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229640 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156983560 ingress_bytes=213377024 link_bytes=213377024 token_bytes=313737480 weight_movement_bytes=56623104 total_nvlink_bytes=370360584 weight_to_token_ratio=0.180479 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044473656e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157696000 dispatch_meta_bytes=228992 combine_payload_bytes=157696000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157924992 ingress_bytes=214319104 link_bytes=214319104 token_bytes=315620992 weight_movement_bytes=56623104 total_nvlink_bytes=372244096 weight_to_token_ratio=0.179402 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053500203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38304 dup_rows=21810 dup_rows_remote=19042 dedup_ratio=0.332794 router_tokens_per_expert=[243, 268, 240, 236, 239, 233, 259, 247, 262, 270, 275, 260, 242, 286, 276, 248, 272, 266, 259, 251, 266, 252, 245, 254, 278, 245, 276, 285, 271, 281, 274, 262, 250, 271, 245, 256, 249, 262, 236, 271, 282, 210, 268, 234, 243, 255, 265, 245, 256, 267, 283, 249, 240, 260, 242, 280, 264, 266, 234, 256, 255, 242, 239, 258, 281, 272, 253, 218, 257, 273, 246, 260, 263, 263, 248, 258, 272, 244, 230, 257, 249, 258, 251, 248, 242, 245, 266, 261, 246, 259, 260, 265, 252, 244, 275, 267, 257, 245, 265, 260, 262, 271, 243, 262, 245, 255, 279, 254, 240, 238, 225, 263, 247, 301, 251, 264, 258, 257, 228, 239, 252, 264, 247, 236, 248, 251, 256, 268] payload_rows_per_dst=[5444, 5592, 5422, 5468, 5422, 5496, 5480, 5402] dup_rows_per_dst=[2724, 2882, 2662, 2714, 2768, 2680, 2648, 2732] combine_rows_per_src=[5444, 5592, 5422, 5468, 5422, 5496, 5480, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38190 dup_rows=21916 dup_rows_remote=19134 dedup_ratio=0.334412 router_tokens_per_expert=[245, 264, 243, 257, 255, 247, 268, 260, 249, 261, 250, 275, 267, 258, 245, 257, 248, 256, 288, 253, 236, 263, 260, 277, 248, 255, 272, 258, 253, 256, 230, 253, 277, 245, 219, 277, 250, 239, 260, 272, 274, 252, 264, 269, 241, 270, 254, 248, 251, 249, 219, 247, 268, 275, 253, 265, 239, 244, 283, 271, 254, 229, 250, 284, 235, 229, 228, 238, 241, 266, 270, 247, 249, 233, 271, 244, 275, 234, 250, 274, 258, 269, 245, 274, 265, 240, 243, 260, 242, 254, 266, 248, 287, 248, 259, 268, 267, 256, 269, 271, 262, 251, 260, 242, 264, 250, 273, 267, 263, 261, 255, 243, 248, 256, 245, 264, 272, 249, 244, 286, 257, 259, 247, 267, 246, 241, 232, 292] payload_rows_per_dst=[5496, 5430, 5446, 5472, 5396, 5398, 5526, 5456] dup_rows_per_dst=[2706, 2782, 2776, 2690, 2572, 2854, 2782, 2754] combine_rows_per_src=[5496, 5430, 5446, 5472, 5396, 5398, 5526, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229440 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860480 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491520 weight_movement_bytes=37748736 total_nvlink_bytes=351240256 weight_to_token_ratio=0.120414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229384 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122568 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015752 weight_movement_bytes=37748736 total_nvlink_bytes=351764488 weight_to_token_ratio=0.120213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229296 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655536 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081776 weight_movement_bytes=37748736 total_nvlink_bytes=350830512 weight_to_token_ratio=0.120571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38252 dup_rows=21864 dup_rows_remote=19210 dedup_ratio=0.333618 router_tokens_per_expert=[286, 240, 253, 255, 289, 262, 227, 244, 216, 248, 244, 266, 265, 249, 259, 241, 261, 231, 259, 237, 261, 249, 251, 246, 275, 273, 244, 263, 256, 258, 270, 249, 263, 261, 271, 252, 233, 267, 250, 276, 267, 260, 224, 258, 243, 276, 271, 260, 257, 281, 249, 247, 247, 231, 268, 234, 247, 262, 256, 256, 254, 226, 258, 264, 255, 273, 271, 274, 261, 276, 259, 259, 257, 273, 265, 274, 208, 241, 260, 257, 273, 266, 259, 283, 237, 256, 265, 265, 235, 275, 261, 245, 251, 252, 259, 245, 237, 266, 256, 264, 235, 248, 243, 272, 285, 250, 254, 256, 274, 273, 256, 255, 254, 218, 298, 239, 253, 269, 274, 246, 233, 265, 260, 224, 260, 257, 240, 268] payload_rows_per_dst=[5432, 5414, 5502, 5420, 5468, 5486, 5516, 5434] dup_rows_per_dst=[2656, 2752, 2762, 2654, 2858, 2768, 2732, 2682] combine_rows_per_src=[5432, 5414, 5502, 5420, 5468, 5486, 5516, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38112 dup_rows=21982 dup_rows_remote=19186 dedup_ratio=0.335419 router_tokens_per_expert=[267, 242, 250, 277, 259, 240, 271, 260, 238, 235, 274, 247, 236, 240, 285, 272, 255, 236, 238, 244, 262, 231, 227, 289, 258, 245, 294, 256, 255, 246, 270, 247, 256, 270, 256, 258, 276, 272, 250, 254, 250, 250, 277, 258, 228, 239, 267, 262, 259, 256, 240, 252, 271, 279, 242, 258, 247, 249, 263, 266, 261, 260, 249, 266, 231, 254, 248, 261, 260, 257, 259, 281, 264, 241, 255, 246, 240, 267, 254, 259, 286, 264, 237, 242, 258, 299, 228, 253, 249, 262, 241, 284, 289, 225, 250, 257, 275, 268, 254, 227, 234, 263, 235, 271, 284, 245, 262, 254, 264, 226, 237, 263, 254, 223, 271, 252, 251, 256, 278, 244, 256, 250, 260, 275, 252, 262, 266, 268] payload_rows_per_dst=[5430, 5424, 5516, 5412, 5396, 5518, 5416, 5442] dup_rows_per_dst=[2756, 2682, 2730, 2788, 2792, 2730, 2708, 2796] combine_rows_per_src=[5430, 5424, 5516, 5412, 5396, 5518, 5416, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38292 dup_rows=21716 dup_rows_remote=19048 dedup_ratio=0.331360 router_tokens_per_expert=[256, 232, 239, 260, 296, 274, 287, 245, 246, 266, 264, 270, 244, 262, 287, 263, 264, 270, 255, 275, 238, 265, 249, 257, 274, 248, 243, 262, 255, 258, 232, 278, 291, 250, 281, 243, 265, 235, 238, 268, 248, 253, 225, 268, 258, 251, 279, 258, 230, 238, 262, 266, 245, 252, 238, 238, 240, 240, 218, 270, 231, 264, 274, 238, 267, 257, 252, 270, 269, 284, 239, 235, 267, 262, 272, 265, 243, 227, 258, 241, 264, 242, 255, 249, 266, 253, 268, 274, 251, 252, 279, 247, 260, 281, 263, 244, 266, 236, 257, 240, 270, 283, 252, 281, 242, 235, 276, 249, 262, 249, 244, 256, 252, 252, 268, 248, 233, 263, 261, 242, 261, 231, 243, 249, 287, 249, 267, 239] payload_rows_per_dst=[5480, 5538, 5480, 5350, 5506, 5468, 5528, 5470] dup_rows_per_dst=[2902, 2708, 2742, 2538, 2710, 2828, 2668, 2620] combine_rows_per_src=[5480, 5538, 5480, 5350, 5506, 5468, 5528, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38116 dup_rows=21988 dup_rows_remote=19306 dedup_ratio=0.335510 router_tokens_per_expert=[250, 228, 279, 241, 277, 253, 270, 228, 266, 267, 271, 245, 251, 275, 248, 262, 265, 252, 236, 253, 265, 260, 260, 248, 249, 248, 275, 253, 243, 263, 257, 253, 258, 247, 250, 239, 254, 250, 257, 267, 261, 226, 263, 260, 267, 279, 224, 255, 243, 248, 260, 235, 266, 263, 261, 266, 274, 277, 257, 272, 262, 209, 241, 260, 260, 271, 251, 287, 254, 268, 283, 250, 272, 256, 268, 250, 263, 229, 259, 284, 249, 273, 268, 235, 271, 266, 224, 226, 260, 274, 255, 267, 269, 249, 230, 241, 255, 264, 238, 256, 277, 243, 237, 244, 271, 269, 231, 260, 236, 232, 281, 239, 310, 217, 241, 260, 224, 223, 276, 276, 283, 289, 243, 258, 268, 266, 254, 243] payload_rows_per_dst=[5408, 5368, 5400, 5474, 5516, 5432, 5440, 5510] dup_rows_per_dst=[2814, 2792, 2714, 2714, 2894, 2682, 2626, 2752] combine_rows_per_src=[5408, 5368, 5400, 5474, 5516, 5432, 5440, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38540 dup_rows=21650 dup_rows_remote=19046 dedup_ratio=0.330353 router_tokens_per_expert=[248, 263, 264, 266, 267, 258, 263, 262, 262, 243, 258, 261, 231, 258, 247, 262, 238, 250, 287, 239, 264, 263, 264, 262, 243, 284, 253, 245, 250, 239, 273, 259, 216, 264, 246, 260, 279, 254, 238, 254, 259, 267, 236, 251, 232, 239, 260, 220, 241, 260, 287, 237, 258, 277, 265, 234, 238, 263, 265, 270, 257, 251, 280, 241, 269, 247, 252, 276, 237, 271, 282, 254, 275, 280, 282, 277, 232, 257, 267, 262, 227, 248, 242, 262, 252, 241, 248, 248, 256, 243, 241, 269, 252, 282, 249, 231, 250, 239, 254, 245, 259, 247, 250, 249, 268, 246, 267, 244, 248, 243, 263, 271, 246, 236, 265, 261, 267, 282, 254, 271, 271, 235, 275, 272, 242, 275, 274, 263] payload_rows_per_dst=[5496, 5530, 5346, 5528, 5550, 5400, 5448, 5588] dup_rows_per_dst=[2730, 2696, 2604, 2720, 2890, 2582, 2638, 2790] combine_rows_per_src=[5496, 5530, 5346, 5528, 5550, 5400, 5448, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229848 combine_payload_bytes=156680192 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156910040 ingress_bytes=213303296 link_bytes=213303296 token_bytes=313590232 weight_movement_bytes=56623104 total_nvlink_bytes=370213336 weight_to_token_ratio=0.180564 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043767231e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=229192 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156335944 ingress_bytes=212729856 link_bytes=212729856 token_bytes=312442696 weight_movement_bytes=56623104 total_nvlink_bytes=369065800 weight_to_token_ratio=0.181227 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038272811e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38182 dup_rows=21742 dup_rows_remote=18914 dedup_ratio=0.331757 router_tokens_per_expert=[254, 284, 272, 266, 274, 234, 259, 264, 276, 271, 258, 252, 251, 254, 249, 255, 260, 265, 265, 243, 244, 266, 235, 238, 230, 233, 289, 258, 241, 273, 236, 268, 272, 246, 232, 267, 262, 237, 257, 251, 273, 243, 266, 260, 253, 274, 221, 266, 262, 273, 274, 272, 271, 240, 268, 239, 250, 287, 206, 252, 263, 248, 260, 280, 241, 258, 270, 258, 252, 255, 275, 235, 251, 283, 251, 269, 267, 242, 286, 245, 259, 244, 266, 239, 243, 265, 229, 253, 280, 268, 237, 255, 255, 222, 275, 262, 277, 291, 243, 260, 270, 275, 248, 213, 257, 242, 235, 258, 243, 267, 275, 245, 260, 260, 275, 243, 240, 270, 246, 258, 235, 249, 235, 251, 255, 260, 249, 251] payload_rows_per_dst=[5612, 5464, 5422, 5536, 5444, 5482, 5468, 5366] dup_rows_per_dst=[2828, 2854, 2644, 2754, 2690, 2622, 2730, 2620] combine_rows_per_src=[5612, 5464, 5422, 5536, 5444, 5482, 5468, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229360 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073392 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917424 weight_movement_bytes=18874368 total_nvlink_bytes=332791792 weight_to_token_ratio=0.060125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229688 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352824 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475960 weight_movement_bytes=18874368 total_nvlink_bytes=331350328 weight_to_token_ratio=0.060403 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157859840 dispatch_meta_bytes=230344 combine_payload_bytes=157859840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158090184 ingress_bytes=176734208 link_bytes=176734208 token_bytes=315950024 weight_movement_bytes=18874368 total_nvlink_bytes=334824392 weight_to_token_ratio=0.059738 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693380222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38422 dup_rows=21764 dup_rows_remote=19212 dedup_ratio=0.332092 router_tokens_per_expert=[249, 232, 241, 251, 256, 219, 260, 274, 212, 272, 259, 252, 256, 275, 279, 266, 245, 274, 252, 280, 264, 255, 244, 253, 286, 262, 219, 268, 243, 251, 257, 268, 231, 248, 252, 250, 227, 238, 255, 281, 250, 279, 246, 247, 249, 229, 260, 253, 301, 266, 257, 267, 245, 268, 239, 263, 258, 268, 258, 227, 277, 266, 241, 266, 242, 251, 251, 252, 251, 247, 209, 245, 267, 256, 245, 252, 243, 234, 256, 250, 278, 278, 255, 243, 269, 238, 247, 238, 260, 278, 264, 245, 268, 271, 252, 251, 264, 275, 251, 242, 254, 255, 252, 268, 241, 250, 280, 270, 277, 282, 277, 243, 264, 247, 272, 243, 277, 247, 270, 256, 242, 280, 272, 257, 249, 271, 248, 270] payload_rows_per_dst=[5420, 5488, 5380, 5582, 5350, 5492, 5534, 5526] dup_rows_per_dst=[2686, 2754, 2610, 2752, 2552, 2778, 2828, 2804] combine_rows_per_src=[5420, 5488, 5380, 5582, 5350, 5492, 5534, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38402 dup_rows=21622 dup_rows_remote=19026 dedup_ratio=0.329926 router_tokens_per_expert=[281, 249, 278, 274, 268, 281, 258, 243, 266, 264, 258, 232, 276, 270, 268, 246, 243, 269, 251, 255, 242, 254, 269, 257, 243, 240, 274, 253, 228, 257, 244, 275, 253, 247, 244, 249, 276, 254, 268, 260, 256, 252, 258, 239, 246, 254, 231, 242, 266, 264, 256, 289, 279, 267, 257, 254, 282, 236, 274, 254, 227, 270, 279, 235, 277, 255, 262, 255, 252, 257, 246, 250, 262, 263, 285, 245, 237, 253, 262, 277, 245, 263, 238, 258, 227, 284, 226, 259, 265, 267, 274, 259, 239, 256, 243, 246, 244, 245, 259, 272, 253, 222, 257, 233, 249, 268, 249, 248, 213, 259, 259, 254, 273, 256, 250, 258, 254, 256, 286, 241, 268, 247, 243, 258, 243, 267, 259, 254] payload_rows_per_dst=[5506, 5512, 5430, 5534, 5562, 5414, 5440, 5516] dup_rows_per_dst=[2918, 2596, 2628, 2844, 2714, 2684, 2528, 2710] combine_rows_per_src=[5506, 5512, 5430, 5534, 5562, 5414, 5440, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228384 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156621856 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015328 weight_movement_bytes=18874368 total_nvlink_bytes=331889696 weight_to_token_ratio=0.060299 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=230536 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157607048 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314983560 weight_movement_bytes=18874368 total_nvlink_bytes=333857928 weight_to_token_ratio=0.059922 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38366 dup_rows=21740 dup_rows_remote=18946 dedup_ratio=0.331726 router_tokens_per_expert=[271, 273, 251, 255, 271, 241, 286, 247, 247, 260, 246, 234, 244, 245, 243, 245, 243, 271, 273, 268, 272, 235, 262, 283, 261, 257, 239, 265, 246, 250, 266, 264, 274, 251, 254, 240, 248, 249, 238, 250, 238, 263, 246, 254, 256, 212, 247, 246, 236, 279, 271, 249, 248, 255, 246, 240, 279, 275, 249, 255, 271, 237, 265, 257, 228, 250, 273, 256, 273, 223, 239, 249, 262, 264, 248, 271, 230, 254, 258, 221, 244, 269, 249, 254, 255, 257, 233, 265, 268, 270, 272, 251, 274, 241, 269, 282, 250, 241, 296, 258, 296, 235, 259, 286, 241, 286, 262, 257, 231, 236, 266, 253, 276, 262, 259, 273, 292, 265, 269, 242, 265, 240, 236, 260, 281, 244, 244, 263] payload_rows_per_dst=[5416, 5502, 5390, 5430, 5414, 5546, 5478, 5620] dup_rows_per_dst=[2702, 2808, 2542, 2794, 2584, 2760, 2828, 2722] combine_rows_per_src=[5416, 5502, 5390, 5430, 5414, 5546, 5478, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229712 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157524304 ingress_bytes=176168960 link_bytes=176168960 token_bytes=314818896 weight_movement_bytes=18874368 total_nvlink_bytes=333693264 weight_to_token_ratio=0.059953 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687964294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38190 dup_rows=21882 dup_rows_remote=19034 dedup_ratio=0.333893 router_tokens_per_expert=[225, 250, 280, 235, 252, 287, 250, 274, 277, 285, 221, 258, 263, 269, 258, 260, 253, 254, 256, 275, 257, 246, 259, 262, 245, 253, 274, 259, 256, 249, 268, 255, 246, 253, 245, 250, 262, 266, 278, 270, 261, 246, 278, 258, 238, 258, 270, 277, 216, 251, 248, 285, 248, 271, 265, 237, 256, 239, 236, 250, 279, 272, 238, 241, 255, 259, 258, 223, 228, 276, 249, 259, 232, 241, 244, 260, 278, 244, 245, 282, 246, 272, 219, 235, 244, 269, 279, 244, 248, 249, 248, 251, 257, 241, 245, 267, 258, 227, 241, 255, 265, 223, 292, 247, 239, 259, 282, 261, 270, 252, 271, 271, 262, 269, 259, 255, 270, 249, 262, 271, 248, 284, 291, 253, 230, 247, 261, 244] payload_rows_per_dst=[5244, 5518, 5504, 5398, 5424, 5426, 5676, 5464] dup_rows_per_dst=[2522, 2724, 2806, 2666, 2642, 2602, 3072, 2848] combine_rows_per_src=[5244, 5518, 5504, 5398, 5424, 5426, 5676, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38256 dup_rows=21786 dup_rows_remote=19138 dedup_ratio=0.332428 router_tokens_per_expert=[266, 237, 269, 237, 261, 250, 263, 262, 271, 251, 248, 250, 246, 251, 265, 262, 257, 254, 267, 243, 248, 300, 241, 268, 254, 258, 263, 267, 231, 253, 244, 261, 235, 272, 257, 240, 265, 245, 251, 254, 261, 226, 276, 231, 267, 257, 279, 234, 266, 261, 269, 280, 266, 277, 248, 251, 265, 241, 235, 267, 277, 241, 246, 263, 249, 256, 260, 249, 254, 279, 263, 259, 255, 224, 242, 247, 254, 247, 242, 260, 257, 253, 243, 244, 242, 263, 258, 263, 260, 273, 241, 269, 220, 282, 256, 265, 250, 273, 247, 246, 238, 267, 252, 267, 264, 228, 266, 287, 230, 252, 268, 236, 267, 265, 250, 271, 274, 272, 263, 272, 252, 241, 243, 261, 269, 253, 253, 261] payload_rows_per_dst=[5450, 5420, 5398, 5522, 5444, 5486, 5494, 5536] dup_rows_per_dst=[2728, 2798, 2702, 2784, 2636, 2692, 2648, 2798] combine_rows_per_src=[5450, 5420, 5398, 5522, 5444, 5486, 5494, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229248 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157376384 ingress_bytes=194895872 link_bytes=194895872 token_bytes=314523520 weight_movement_bytes=37748736 total_nvlink_bytes=352272256 weight_to_token_ratio=0.120019 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867396351e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38190 dup_rows=21876 dup_rows_remote=19010 dedup_ratio=0.333801 router_tokens_per_expert=[244, 282, 282, 247, 233, 251, 253, 265, 243, 249, 260, 263, 271, 255, 259, 257, 263, 279, 261, 237, 251, 263, 256, 268, 258, 249, 254, 260, 272, 246, 254, 254, 251, 229, 256, 263, 255, 279, 260, 260, 277, 231, 265, 287, 268, 258, 267, 265, 258, 253, 235, 233, 270, 247, 258, 263, 238, 261, 251, 240, 258, 277, 241, 277, 275, 236, 272, 208, 217, 253, 263, 264, 250, 281, 264, 238, 248, 261, 238, 260, 243, 259, 283, 259, 264, 240, 263, 265, 264, 265, 267, 259, 282, 273, 242, 240, 258, 237, 251, 266, 255, 244, 252, 238, 260, 246, 254, 226, 265, 276, 262, 265, 243, 264, 261, 261, 240, 255, 231, 275, 244, 275, 249, 249, 271, 253, 237, 239] payload_rows_per_dst=[5494, 5476, 5590, 5358, 5418, 5470, 5396, 5458] dup_rows_per_dst=[2734, 2774, 2752, 2762, 2638, 2866, 2714, 2636] combine_rows_per_src=[5494, 5476, 5590, 5358, 5418, 5470, 5396, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228896 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156655136 ingress_bytes=250798080 link_bytes=250798080 token_bytes=313081376 weight_movement_bytes=94371840 total_nvlink_bytes=407453216 weight_to_token_ratio=0.301429 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403023802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38010 dup_rows=21948 dup_rows_remote=19182 dedup_ratio=0.334900 router_tokens_per_expert=[252, 273, 272, 261, 260, 291, 237, 259, 258, 293, 279, 240, 272, 254, 259, 272, 225, 255, 231, 252, 251, 250, 258, 231, 239, 256, 269, 261, 248, 258, 251, 264, 257, 269, 263, 263, 259, 252, 275, 239, 234, 268, 274, 265, 257, 266, 252, 279, 263, 285, 254, 253, 283, 243, 268, 281, 257, 250, 249, 235, 260, 260, 258, 271, 255, 267, 251, 240, 274, 265, 234, 274, 255, 266, 234, 259, 257, 284, 265, 263, 286, 251, 281, 232, 259, 234, 268, 267, 244, 256, 222, 276, 262, 245, 218, 247, 260, 267, 244, 274, 258, 241, 265, 227, 231, 272, 258, 264, 243, 248, 226, 231, 240, 241, 238, 256, 245, 257, 248, 221, 248, 263, 259, 221, 251, 294, 263, 250] payload_rows_per_dst=[5572, 5334, 5578, 5524, 5436, 5400, 5394, 5350] dup_rows_per_dst=[2892, 2664, 2766, 2816, 2850, 2696, 2624, 2640] combine_rows_per_src=[5572, 5334, 5578, 5524, 5436, 5400, 5394, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44010 payload_rows_remote=38358 dup_rows=21526 dup_rows_remote=18774 dedup_ratio=0.328461 router_tokens_per_expert=[270, 226, 251, 268, 281, 270, 256, 261, 260, 263, 247, 242, 247, 250, 277, 282, 249, 225, 247, 259, 268, 260, 271, 267, 252, 264, 225, 238, 231, 248, 246, 236, 243, 287, 234, 250, 261, 265, 238, 287, 271, 270, 266, 250, 239, 222, 234, 266, 250, 252, 292, 268, 252, 265, 244, 272, 280, 261, 246, 291, 276, 241, 245, 228, 260, 255, 263, 272, 278, 244, 255, 245, 269, 270, 259, 230, 243, 264, 248, 260, 252, 255, 246, 256, 249, 276, 267, 260, 258, 273, 265, 257, 253, 278, 243, 251, 257, 259, 284, 220, 269, 233, 271, 250, 254, 273, 247, 248, 253, 248, 239, 261, 254, 251, 261, 246, 244, 262, 247, 296, 250, 239, 248, 253, 267, 247, 256, 244] payload_rows_per_dst=[5652, 5386, 5474, 5534, 5508, 5498, 5482, 5476] dup_rows_per_dst=[2752, 2586, 2692, 2792, 2722, 2678, 2650, 2654] combine_rows_per_src=[5652, 5386, 5474, 5534, 5508, 5498, 5482, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229576 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926152 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622728 weight_movement_bytes=18874368 total_nvlink_bytes=332497096 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228800 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655040 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081280 weight_movement_bytes=18874368 total_nvlink_bytes=331955648 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38274 dup_rows=21734 dup_rows_remote=18978 dedup_ratio=0.331635 router_tokens_per_expert=[265, 270, 264, 239, 259, 249, 282, 266, 253, 250, 231, 251, 261, 262, 250, 247, 239, 250, 254, 272, 268, 253, 242, 256, 270, 268, 237, 255, 260, 264, 255, 246, 274, 270, 269, 243, 238, 260, 251, 251, 267, 248, 267, 232, 281, 236, 243, 260, 268, 287, 262, 271, 247, 269, 256, 250, 264, 248, 236, 239, 260, 263, 219, 243, 272, 271, 280, 249, 263, 253, 267, 239, 263, 251, 253, 262, 250, 277, 246, 246, 248, 268, 278, 256, 270, 267, 241, 260, 256, 257, 252, 261, 260, 270, 240, 249, 240, 289, 257, 277, 216, 254, 237, 280, 239, 236, 264, 234, 270, 282, 224, 292, 257, 250, 238, 258, 273, 254, 248, 241, 247, 262, 252, 258, 224, 281, 245, 254] payload_rows_per_dst=[5518, 5464, 5442, 5528, 5528, 5518, 5410, 5394] dup_rows_per_dst=[2680, 2714, 2738, 2636, 2756, 2748, 2772, 2690] combine_rows_per_src=[5518, 5464, 5442, 5528, 5528, 5518, 5410, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=228768 combine_payload_bytes=155688960 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=155917728 ingress_bytes=174563328 link_bytes=174563328 token_bytes=311606688 weight_movement_bytes=18874368 total_nvlink_bytes=330481056 weight_to_token_ratio=0.060571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672579918e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38254 dup_rows=21868 dup_rows_remote=19120 dedup_ratio=0.333679 router_tokens_per_expert=[285, 276, 254, 240, 225, 274, 221, 239, 244, 261, 257, 261, 256, 250, 247, 265, 248, 252, 246, 280, 248, 219, 248, 256, 240, 248, 274, 288, 252, 256, 275, 251, 277, 262, 264, 245, 230, 246, 269, 260, 256, 264, 261, 251, 234, 268, 230, 265, 272, 241, 259, 290, 249, 237, 261, 258, 275, 266, 277, 236, 255, 253, 246, 270, 218, 238, 245, 270, 279, 262, 271, 235, 242, 257, 280, 268, 295, 225, 243, 236, 254, 254, 255, 247, 260, 271, 243, 249, 276, 267, 257, 265, 261, 268, 257, 249, 248, 264, 227, 257, 255, 281, 246, 260, 234, 258, 284, 262, 270, 245, 262, 283, 260, 285, 240, 253, 251, 260, 233, 279, 246, 252, 257, 249, 259, 244, 254, 250] payload_rows_per_dst=[5414, 5414, 5490, 5504, 5444, 5450, 5456, 5496] dup_rows_per_dst=[2696, 2748, 2938, 2786, 2684, 2552, 2816, 2648] combine_rows_per_src=[5414, 5414, 5490, 5504, 5444, 5450, 5456, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=228528 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157342896 ingress_bytes=213737472 link_bytes=213737472 token_bytes=314457264 weight_movement_bytes=56623104 total_nvlink_bytes=371080368 weight_to_token_ratio=0.180066 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047927291e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38152 dup_rows=21818 dup_rows_remote=19014 dedup_ratio=0.332916 router_tokens_per_expert=[233, 255, 284, 255, 231, 274, 246, 239, 269, 251, 249, 296, 250, 240, 242, 261, 249, 252, 256, 234, 260, 241, 264, 260, 245, 246, 276, 276, 250, 257, 259, 269, 247, 249, 254, 243, 257, 277, 269, 276, 230, 260, 260, 251, 251, 244, 238, 276, 275, 266, 266, 275, 243, 238, 283, 268, 251, 246, 263, 253, 270, 264, 257, 267, 272, 247, 249, 256, 291, 250, 268, 278, 259, 249, 263, 255, 252, 252, 252, 270, 266, 262, 257, 283, 237, 254, 258, 253, 260, 244, 271, 250, 261, 254, 255, 250, 241, 247, 260, 267, 245, 252, 278, 277, 245, 235, 267, 248, 271, 270, 241, 243, 261, 224, 247, 246, 236, 239, 243, 247, 250, 246, 237, 244, 247, 281, 253, 266] payload_rows_per_dst=[5478, 5454, 5432, 5566, 5568, 5456, 5458, 5306] dup_rows_per_dst=[2672, 2734, 2732, 2804, 2758, 2774, 2716, 2628] combine_rows_per_src=[5478, 5454, 5432, 5566, 5568, 5456, 5458, 5306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229008 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156999312 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769616 weight_movement_bytes=37748736 total_nvlink_bytes=351518352 weight_to_token_ratio=0.120307 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38124 dup_rows=21844 dup_rows_remote=19048 dedup_ratio=0.333313 router_tokens_per_expert=[253, 262, 276, 250, 259, 239, 255, 226, 258, 256, 277, 277, 255, 261, 228, 256, 269, 281, 264, 239, 235, 263, 247, 271, 263, 252, 241, 246, 258, 236, 268, 235, 244, 236, 255, 250, 264, 240, 256, 257, 276, 234, 254, 254, 244, 261, 257, 242, 260, 254, 254, 241, 270, 222, 256, 265, 256, 263, 261, 258, 292, 246, 277, 222, 268, 257, 299, 247, 268, 253, 274, 284, 246, 230, 263, 254, 229, 277, 265, 253, 273, 259, 260, 281, 270, 257, 261, 248, 261, 262, 284, 277, 262, 244, 261, 247, 244, 245, 280, 271, 245, 246, 259, 258, 253, 233, 245, 270, 231, 261, 236, 279, 261, 270, 282, 221, 238, 239, 274, 241, 246, 243, 262, 250, 285, 251, 264, 234] payload_rows_per_dst=[5420, 5472, 5354, 5374, 5426, 5692, 5386, 5568] dup_rows_per_dst=[2724, 2664, 2694, 2554, 2698, 2988, 2726, 2796] combine_rows_per_src=[5420, 5472, 5354, 5374, 5426, 5692, 5386, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229496 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917880 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606264 weight_movement_bytes=18874368 total_nvlink_bytes=332480632 weight_to_token_ratio=0.060185 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38326 dup_rows=21856 dup_rows_remote=19142 dedup_ratio=0.333496 router_tokens_per_expert=[265, 253, 251, 275, 252, 244, 255, 256, 252, 268, 258, 302, 265, 246, 258, 241, 241, 266, 230, 226, 260, 260, 252, 249, 273, 232, 277, 244, 265, 222, 252, 257, 259, 254, 278, 247, 268, 276, 250, 252, 237, 243, 222, 256, 255, 234, 245, 288, 284, 254, 266, 290, 255, 277, 259, 274, 266, 263, 228, 252, 282, 240, 256, 252, 275, 267, 240, 255, 268, 248, 242, 258, 265, 270, 250, 237, 265, 268, 264, 276, 236, 227, 235, 255, 254, 269, 220, 269, 250, 265, 252, 235, 276, 249, 272, 257, 271, 240, 240, 259, 268, 247, 243, 274, 263, 229, 220, 224, 274, 254, 263, 265, 252, 247, 260, 283, 243, 278, 254, 256, 267, 256, 253, 255, 256, 261, 262, 273] payload_rows_per_dst=[5512, 5422, 5424, 5520, 5544, 5416, 5354, 5488] dup_rows_per_dst=[2770, 2590, 2704, 2876, 2752, 2626, 2714, 2824] combine_rows_per_src=[5512, 5422, 5424, 5520, 5544, 5416, 5354, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228664 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156499256 ingress_bytes=212893696 link_bytes=212893696 token_bytes=312769848 weight_movement_bytes=56623104 total_nvlink_bytes=369392952 weight_to_token_ratio=0.181038 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039842645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38222 dup_rows=21880 dup_rows_remote=19106 dedup_ratio=0.333862 router_tokens_per_expert=[259, 275, 264, 242, 286, 283, 254, 269, 265, 251, 264, 258, 240, 260, 264, 226, 232, 252, 249, 238, 226, 273, 248, 234, 240, 302, 219, 275, 260, 242, 257, 262, 273, 255, 250, 257, 233, 234, 279, 248, 248, 236, 252, 270, 228, 239, 252, 249, 258, 281, 249, 261, 248, 249, 247, 290, 272, 255, 252, 254, 245, 295, 266, 261, 231, 235, 252, 261, 274, 223, 240, 259, 264, 257, 248, 284, 225, 242, 277, 251, 287, 266, 243, 272, 247, 245, 256, 274, 248, 233, 268, 240, 256, 269, 248, 252, 260, 241, 240, 270, 254, 252, 287, 248, 252, 247, 283, 249, 279, 278, 253, 245, 264, 262, 258, 249, 269, 263, 253, 250, 254, 257, 251, 226, 268, 252, 268, 304] payload_rows_per_dst=[5540, 5424, 5338, 5492, 5402, 5434, 5506, 5520] dup_rows_per_dst=[2780, 2594, 2668, 2874, 2644, 2774, 2770, 2776] combine_rows_per_src=[5540, 5424, 5338, 5492, 5402, 5434, 5506, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228688 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156384592 ingress_bytes=175030272 link_bytes=175030272 token_bytes=312540496 weight_movement_bytes=18874368 total_nvlink_bytes=331414864 weight_to_token_ratio=0.060390 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677053946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38436 dup_rows=21710 dup_rows_remote=19112 dedup_ratio=0.331268 router_tokens_per_expert=[280, 271, 269, 271, 258, 273, 252, 302, 264, 242, 253, 252, 246, 233, 242, 238, 245, 228, 243, 261, 257, 245, 242, 264, 264, 279, 243, 284, 257, 239, 254, 276, 252, 228, 250, 262, 228, 269, 272, 245, 253, 259, 247, 253, 239, 266, 215, 256, 219, 238, 269, 259, 255, 266, 239, 251, 262, 272, 283, 256, 266, 250, 270, 249, 244, 297, 281, 270, 265, 249, 255, 260, 223, 243, 254, 265, 297, 227, 276, 266, 271, 259, 273, 252, 262, 269, 248, 269, 244, 254, 264, 263, 254, 264, 255, 268, 247, 268, 260, 261, 232, 265, 270, 267, 270, 262, 258, 223, 247, 275, 234, 265, 255, 238, 251, 239, 265, 239, 269, 266, 220, 252, 235, 267, 219, 247, 267, 269] payload_rows_per_dst=[5562, 5390, 5390, 5498, 5544, 5524, 5554, 5364] dup_rows_per_dst=[2730, 2772, 2598, 2710, 2800, 2814, 2654, 2632] combine_rows_per_src=[5562, 5390, 5390, 5498, 5544, 5524, 5554, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229872 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213168 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196464 weight_movement_bytes=18874368 total_nvlink_bytes=333070832 weight_to_token_ratio=0.060072 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38106 dup_rows=21916 dup_rows_remote=19164 dedup_ratio=0.334412 router_tokens_per_expert=[262, 259, 263, 249, 257, 230, 264, 233, 248, 253, 286, 232, 268, 235, 257, 278, 238, 259, 242, 277, 255, 254, 251, 269, 253, 246, 243, 257, 248, 237, 259, 252, 257, 257, 239, 266, 251, 265, 283, 244, 257, 251, 282, 260, 248, 278, 265, 255, 250, 261, 250, 261, 242, 255, 248, 250, 246, 258, 245, 263, 269, 270, 256, 270, 273, 240, 231, 273, 223, 253, 261, 289, 244, 263, 283, 239, 247, 253, 272, 243, 265, 269, 220, 271, 237, 232, 244, 276, 256, 227, 244, 292, 243, 267, 246, 250, 250, 257, 242, 253, 256, 268, 264, 244, 239, 253, 248, 249, 261, 241, 266, 266, 287, 263, 256, 230, 256, 283, 288, 240, 268, 270, 240, 300, 262, 260, 266, 250] payload_rows_per_dst=[5514, 5342, 5588, 5420, 5334, 5546, 5572, 5304] dup_rows_per_dst=[2752, 2632, 2968, 2756, 2600, 2786, 2862, 2560] combine_rows_per_src=[5514, 5342, 5588, 5420, 5334, 5546, 5572, 5304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229312 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156786624 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313343936 weight_movement_bytes=37748736 total_nvlink_bytes=351092672 weight_to_token_ratio=0.120471 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38360 dup_rows=21746 dup_rows_remote=19106 dedup_ratio=0.331818 router_tokens_per_expert=[244, 257, 228, 267, 271, 246, 246, 245, 240, 283, 266, 269, 251, 244, 273, 251, 271, 236, 282, 260, 260, 266, 260, 246, 242, 239, 252, 264, 229, 252, 248, 247, 247, 211, 271, 243, 245, 255, 252, 282, 228, 260, 249, 280, 258, 257, 255, 237, 233, 246, 253, 252, 267, 258, 284, 237, 260, 264, 258, 268, 258, 234, 253, 259, 256, 242, 222, 273, 251, 250, 283, 255, 262, 233, 219, 291, 258, 240, 263, 237, 275, 278, 258, 259, 232, 255, 282, 254, 267, 239, 285, 243, 249, 265, 236, 262, 265, 227, 280, 250, 256, 262, 258, 274, 282, 276, 273, 249, 249, 270, 243, 289, 263, 253, 283, 250, 270, 290, 238, 247, 244, 263, 271, 253, 254, 255, 260, 248] payload_rows_per_dst=[5530, 5440, 5408, 5484, 5430, 5526, 5518, 5454] dup_rows_per_dst=[2632, 2668, 2652, 2684, 2640, 2752, 2888, 2830] combine_rows_per_src=[5530, 5440, 5408, 5484, 5430, 5526, 5518, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=230192 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157664048 ingress_bytes=176308224 link_bytes=176308224 token_bytes=315097904 weight_movement_bytes=18874368 total_nvlink_bytes=333972272 weight_to_token_ratio=0.059900 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689298653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38412 dup_rows=21730 dup_rows_remote=19056 dedup_ratio=0.331573 router_tokens_per_expert=[256, 261, 241, 251, 289, 273, 245, 267, 274, 229, 235, 249, 260, 242, 247, 256, 257, 223, 229, 250, 259, 271, 248, 258, 237, 261, 274, 249, 273, 227, 246, 272, 276, 262, 255, 245, 271, 276, 258, 255, 255, 284, 266, 242, 245, 267, 262, 230, 274, 221, 268, 265, 269, 258, 270, 261, 260, 254, 290, 262, 259, 254, 270, 234, 259, 259, 262, 240, 246, 245, 261, 256, 262, 222, 243, 253, 249, 254, 258, 267, 243, 250, 293, 279, 272, 258, 242, 268, 254, 256, 266, 257, 230, 248, 264, 270, 270, 243, 255, 254, 250, 261, 264, 236, 248, 244, 241, 250, 281, 265, 273, 288, 262, 249, 259, 260, 249, 276, 261, 258, 226, 259, 255, 259, 235, 236, 251, 237] payload_rows_per_dst=[5408, 5394, 5562, 5520, 5406, 5592, 5488, 5436] dup_rows_per_dst=[2742, 2674, 2736, 2818, 2666, 2708, 2758, 2628] combine_rows_per_src=[5408, 5394, 5562, 5520, 5406, 5592, 5488, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38234 dup_rows=21874 dup_rows_remote=19198 dedup_ratio=0.333771 router_tokens_per_expert=[255, 289, 247, 257, 268, 236, 236, 256, 239, 238, 241, 233, 272, 265, 245, 269, 248, 249, 242, 241, 262, 258, 275, 272, 253, 254, 245, 264, 246, 259, 245, 244, 241, 245, 263, 256, 273, 268, 254, 252, 284, 275, 251, 233, 244, 254, 278, 258, 257, 263, 239, 227, 240, 260, 251, 266, 283, 268, 245, 256, 262, 238, 260, 237, 243, 250, 246, 273, 264, 249, 269, 242, 248, 246, 255, 243, 252, 275, 284, 226, 250, 268, 254, 244, 260, 258, 250, 274, 255, 250, 267, 307, 275, 292, 252, 242, 250, 252, 259, 257, 237, 259, 257, 253, 270, 241, 251, 288, 294, 243, 271, 245, 250, 233, 270, 261, 261, 275, 263, 245, 269, 238, 238, 287, 254, 249, 244, 257] payload_rows_per_dst=[5412, 5410, 5512, 5428, 5408, 5570, 5460, 5462] dup_rows_per_dst=[2680, 2704, 2746, 2676, 2722, 2826, 2794, 2726] combine_rows_per_src=[5412, 5410, 5512, 5428, 5408, 5570, 5460, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229080 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156311256 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312393432 weight_movement_bytes=18874368 total_nvlink_bytes=331267800 weight_to_token_ratio=0.060419 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229864 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157352424 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474984 weight_movement_bytes=18874368 total_nvlink_bytes=333349352 weight_to_token_ratio=0.060019 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38332 dup_rows=21724 dup_rows_remote=19170 dedup_ratio=0.331482 router_tokens_per_expert=[257, 261, 262, 241, 270, 246, 260, 227, 248, 282, 254, 257, 251, 248, 265, 260, 259, 279, 240, 261, 243, 269, 223, 250, 247, 256, 265, 238, 233, 276, 283, 259, 257, 244, 277, 269, 277, 291, 245, 286, 237, 246, 241, 296, 241, 260, 247, 235, 265, 272, 265, 246, 257, 244, 281, 240, 261, 267, 247, 261, 250, 254, 265, 241, 293, 247, 239, 256, 248, 264, 269, 228, 282, 261, 268, 252, 245, 262, 248, 239, 259, 252, 249, 292, 260, 253, 277, 251, 253, 269, 244, 257, 238, 274, 256, 273, 291, 235, 265, 266, 252, 262, 238, 272, 230, 264, 253, 239, 275, 221, 237, 258, 242, 245, 246, 263, 233, 251, 246, 272, 254, 269, 254, 217, 267, 277, 241, 240] payload_rows_per_dst=[5432, 5460, 5458, 5416, 5532, 5508, 5526, 5480] dup_rows_per_dst=[2746, 2702, 2614, 2680, 2896, 2806, 2726, 2554] combine_rows_per_src=[5432, 5460, 5458, 5416, 5532, 5508, 5526, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229872 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157565424 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900976 weight_movement_bytes=18874368 total_nvlink_bytes=333775344 weight_to_token_ratio=0.059937 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229728 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156836192 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313442656 weight_movement_bytes=37748736 total_nvlink_bytes=351191392 weight_to_token_ratio=0.120433 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38298 dup_rows=21728 dup_rows_remote=18996 dedup_ratio=0.331543 router_tokens_per_expert=[223, 281, 245, 240, 266, 239, 282, 248, 269, 234, 284, 232, 263, 249, 240, 268, 246, 262, 260, 266, 250, 270, 236, 251, 243, 269, 254, 266, 285, 243, 227, 285, 239, 269, 264, 286, 250, 248, 230, 260, 222, 216, 249, 239, 259, 240, 271, 271, 248, 270, 269, 234, 269, 232, 287, 228, 258, 261, 269, 225, 261, 262, 267, 264, 243, 275, 261, 262, 247, 257, 269, 273, 229, 241, 251, 245, 278, 269, 253, 259, 248, 250, 241, 258, 242, 255, 249, 261, 255, 270, 247, 278, 236, 266, 253, 253, 269, 250, 254, 282, 277, 263, 250, 265, 249, 281, 240, 235, 241, 267, 276, 222, 261, 274, 247, 261, 268, 268, 263, 293, 258, 275, 249, 270, 241, 242, 254, 256] payload_rows_per_dst=[5408, 5516, 5386, 5514, 5516, 5352, 5510, 5606] dup_rows_per_dst=[2718, 2710, 2640, 2694, 2708, 2772, 2732, 2754] combine_rows_per_src=[5408, 5516, 5386, 5514, 5516, 5352, 5510, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38400 dup_rows=21746 dup_rows_remote=19072 dedup_ratio=0.331818 router_tokens_per_expert=[251, 238, 269, 277, 238, 231, 254, 258, 262, 249, 269, 243, 243, 229, 263, 253, 274, 272, 280, 288, 261, 261, 216, 263, 237, 270, 268, 247, 264, 251, 256, 282, 260, 232, 291, 265, 263, 245, 246, 267, 255, 254, 250, 245, 237, 294, 274, 264, 256, 262, 257, 251, 263, 263, 218, 272, 283, 246, 264, 251, 267, 249, 275, 293, 261, 254, 247, 250, 246, 253, 259, 255, 253, 213, 251, 272, 243, 241, 250, 240, 246, 269, 239, 224, 264, 237, 268, 270, 256, 229, 258, 273, 247, 276, 270, 206, 271, 257, 247, 288, 271, 276, 259, 252, 243, 243, 273, 261, 257, 266, 278, 238, 216, 232, 245, 232, 263, 247, 283, 236, 256, 259, 258, 265, 245, 275, 263, 264] payload_rows_per_dst=[5448, 5510, 5522, 5586, 5392, 5390, 5552, 5390] dup_rows_per_dst=[2606, 2870, 2762, 2754, 2584, 2674, 2808, 2688] combine_rows_per_src=[5448, 5510, 5522, 5586, 5392, 5390, 5552, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=230008 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157237880 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314245752 weight_movement_bytes=37748736 total_nvlink_bytes=351994488 weight_to_token_ratio=0.120125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38270 dup_rows=21764 dup_rows_remote=19052 dedup_ratio=0.332092 router_tokens_per_expert=[250, 259, 245, 270, 264, 253, 267, 236, 262, 277, 250, 265, 249, 239, 287, 261, 234, 278, 254, 247, 238, 267, 245, 251, 262, 228, 277, 257, 243, 245, 238, 222, 250, 268, 245, 263, 277, 233, 279, 256, 232, 248, 263, 276, 241, 263, 264, 249, 251, 250, 255, 244, 277, 252, 251, 273, 245, 241, 245, 259, 244, 262, 250, 268, 254, 239, 278, 278, 274, 271, 271, 250, 229, 247, 243, 250, 263, 266, 261, 248, 230, 295, 243, 243, 232, 222, 245, 265, 241, 239, 258, 244, 248, 226, 246, 246, 258, 250, 241, 267, 282, 245, 253, 259, 291, 262, 259, 288, 271, 262, 255, 254, 285, 223, 261, 295, 277, 261, 277, 235, 264, 239, 250, 277, 286, 270, 261, 271] payload_rows_per_dst=[5468, 5350, 5502, 5482, 5496, 5342, 5538, 5594] dup_rows_per_dst=[2800, 2622, 2712, 2652, 2748, 2504, 2856, 2870] combine_rows_per_src=[5468, 5350, 5502, 5482, 5496, 5342, 5538, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229176 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157097784 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966392 weight_movement_bytes=37748736 total_nvlink_bytes=351715128 weight_to_token_ratio=0.120232 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38280 dup_rows=21804 dup_rows_remote=19108 dedup_ratio=0.332703 router_tokens_per_expert=[225, 259, 228, 255, 249, 275, 245, 242, 276, 257, 283, 276, 242, 253, 238, 267, 255, 245, 259, 276, 256, 243, 265, 227, 256, 246, 272, 256, 260, 252, 253, 231, 251, 254, 277, 300, 269, 250, 258, 267, 256, 269, 267, 255, 250, 272, 247, 274, 237, 293, 247, 262, 248, 254, 234, 278, 256, 236, 236, 231, 230, 255, 228, 239, 256, 272, 253, 267, 252, 261, 270, 270, 240, 259, 250, 272, 253, 255, 283, 273, 256, 277, 279, 228, 266, 244, 244, 251, 222, 292, 260, 279, 249, 260, 254, 275, 238, 240, 226, 238, 230, 267, 234, 274, 267, 237, 276, 261, 223, 273, 234, 260, 278, 273, 265, 240, 256, 239, 267, 254, 265, 282, 253, 256, 255, 233, 272, 278] payload_rows_per_dst=[5452, 5504, 5570, 5258, 5658, 5558, 5356, 5376] dup_rows_per_dst=[2696, 2870, 2862, 2422, 2998, 2802, 2600, 2554] combine_rows_per_src=[5452, 5504, 5570, 5258, 5658, 5558, 5356, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229888 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157516288 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802688 weight_movement_bytes=37748736 total_nvlink_bytes=352551424 weight_to_token_ratio=0.119912 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38296 dup_rows=21722 dup_rows_remote=18922 dedup_ratio=0.331451 router_tokens_per_expert=[255, 261, 260, 248, 266, 277, 219, 239, 239, 297, 276, 256, 221, 230, 255, 284, 240, 260, 249, 248, 250, 250, 278, 256, 285, 254, 239, 258, 277, 258, 239, 261, 247, 242, 253, 252, 256, 275, 255, 266, 254, 241, 264, 237, 258, 235, 241, 223, 253, 249, 266, 263, 236, 281, 246, 285, 270, 263, 255, 262, 260, 249, 258, 274, 227, 253, 255, 283, 269, 259, 264, 250, 253, 249, 260, 290, 251, 260, 261, 275, 259, 232, 277, 275, 232, 243, 274, 230, 255, 271, 241, 259, 255, 238, 244, 260, 258, 259, 246, 271, 281, 303, 241, 275, 259, 249, 257, 234, 260, 247, 261, 229, 235, 243, 259, 268, 266, 238, 259, 273, 223, 259, 260, 264, 259, 245, 271, 258] payload_rows_per_dst=[5496, 5472, 5390, 5538, 5518, 5432, 5488, 5480] dup_rows_per_dst=[2670, 2732, 2608, 2802, 2800, 2658, 2772, 2680] combine_rows_per_src=[5496, 5472, 5390, 5538, 5518, 5432, 5488, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229288 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983208 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737128 weight_movement_bytes=37748736 total_nvlink_bytes=351485864 weight_to_token_ratio=0.120320 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38298 dup_rows=21756 dup_rows_remote=19076 dedup_ratio=0.331970 router_tokens_per_expert=[260, 256, 264, 264, 241, 264, 263, 260, 271, 259, 286, 242, 244, 241, 264, 278, 262, 228, 273, 265, 279, 264, 250, 308, 237, 222, 247, 250, 249, 240, 260, 247, 267, 257, 236, 274, 233, 246, 262, 269, 275, 250, 264, 250, 241, 272, 230, 271, 245, 252, 242, 253, 263, 262, 252, 267, 255, 257, 264, 249, 260, 292, 251, 218, 250, 249, 262, 262, 251, 244, 261, 270, 240, 247, 271, 247, 244, 260, 248, 261, 258, 264, 252, 233, 257, 265, 257, 252, 277, 252, 247, 233, 283, 233, 275, 264, 254, 249, 261, 231, 276, 255, 235, 264, 252, 242, 228, 263, 262, 272, 265, 271, 255, 256, 248, 253, 257, 245, 256, 237, 254, 287, 282, 254, 254, 245, 236, 283] payload_rows_per_dst=[5570, 5482, 5676, 5288, 5434, 5450, 5456, 5424] dup_rows_per_dst=[2744, 2680, 2888, 2552, 2700, 2754, 2704, 2734] combine_rows_per_src=[5570, 5482, 5676, 5288, 5434, 5450, 5456, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38294 dup_rows=21780 dup_rows_remote=19022 dedup_ratio=0.332336 router_tokens_per_expert=[279, 254, 266, 279, 256, 275, 253, 274, 213, 255, 271, 248, 252, 251, 251, 250, 272, 260, 246, 270, 264, 243, 239, 275, 274, 244, 240, 281, 272, 273, 257, 252, 251, 245, 260, 244, 268, 252, 270, 243, 249, 249, 239, 253, 245, 249, 250, 231, 267, 258, 241, 253, 268, 253, 275, 271, 230, 266, 246, 246, 258, 252, 292, 234, 282, 263, 276, 247, 262, 240, 246, 262, 265, 267, 251, 267, 239, 270, 243, 259, 234, 273, 263, 248, 263, 224, 263, 271, 246, 258, 243, 242, 261, 283, 258, 269, 254, 278, 227, 279, 250, 243, 257, 238, 247, 280, 240, 251, 239, 267, 238, 247, 272, 261, 255, 243, 247, 269, 245, 249, 270, 274, 262, 257, 244, 264, 244, 242] payload_rows_per_dst=[5540, 5472, 5436, 5462, 5520, 5466, 5422, 5438] dup_rows_per_dst=[2714, 2852, 2560, 2758, 2758, 2732, 2648, 2758] combine_rows_per_src=[5540, 5472, 5436, 5462, 5520, 5466, 5422, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229552 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157024432 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313819312 weight_movement_bytes=37748736 total_nvlink_bytes=351568048 weight_to_token_ratio=0.120288 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228872 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157089288 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313949704 weight_movement_bytes=37748736 total_nvlink_bytes=351698440 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38220 dup_rows=21750 dup_rows_remote=18838 dedup_ratio=0.331879 router_tokens_per_expert=[237, 253, 275, 251, 287, 254, 263, 265, 249, 241, 251, 238, 261, 277, 247, 217, 232, 281, 268, 254, 266, 247, 263, 246, 251, 235, 253, 272, 241, 283, 270, 237, 236, 249, 259, 278, 264, 262, 246, 279, 267, 255, 260, 255, 263, 264, 237, 264, 248, 273, 282, 285, 253, 281, 264, 239, 245, 255, 234, 279, 248, 297, 244, 269, 270, 272, 252, 257, 251, 258, 249, 253, 248, 253, 276, 265, 243, 276, 225, 238, 242, 270, 234, 229, 238, 256, 242, 268, 255, 272, 256, 288, 243, 243, 267, 268, 267, 255, 234, 259, 262, 257, 235, 254, 259, 261, 266, 236, 255, 235, 248, 262, 265, 260, 242, 243, 254, 266, 248, 214, 233, 287, 234, 275, 269, 254, 255, 268] payload_rows_per_dst=[5390, 5486, 5554, 5540, 5398, 5436, 5416, 5566] dup_rows_per_dst=[2554, 2704, 2722, 2852, 2706, 2782, 2518, 2912] combine_rows_per_src=[5390, 5486, 5554, 5540, 5398, 5436, 5416, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229264 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081488 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933712 weight_movement_bytes=37748736 total_nvlink_bytes=351682448 weight_to_token_ratio=0.120244 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229496 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157098104 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966712 weight_movement_bytes=37748736 total_nvlink_bytes=351715448 weight_to_token_ratio=0.120232 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38242 dup_rows=21816 dup_rows_remote=19066 dedup_ratio=0.332886 router_tokens_per_expert=[259, 242, 286, 275, 243, 240, 264, 250, 253, 253, 262, 262, 247, 248, 234, 256, 259, 260, 264, 268, 230, 261, 264, 272, 241, 261, 287, 251, 252, 252, 264, 263, 265, 252, 235, 273, 254, 266, 248, 259, 256, 224, 255, 257, 250, 269, 249, 256, 237, 255, 242, 263, 274, 254, 265, 251, 264, 231, 272, 246, 278, 260, 236, 223, 246, 282, 237, 277, 279, 256, 275, 256, 265, 230, 267, 224, 251, 254, 262, 297, 284, 257, 268, 241, 248, 247, 237, 224, 257, 267, 240, 261, 215, 257, 280, 254, 262, 248, 243, 239, 241, 277, 257, 262, 268, 246, 255, 251, 290, 252, 240, 283, 261, 275, 243, 268, 269, 250, 242, 257, 265, 250, 262, 295, 264, 228, 219, 269] payload_rows_per_dst=[5524, 5558, 5422, 5424, 5452, 5378, 5478, 5484] dup_rows_per_dst=[2624, 2740, 2714, 2678, 2864, 2696, 2750, 2750] combine_rows_per_src=[5524, 5558, 5422, 5424, 5452, 5378, 5478, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38206 dup_rows=21898 dup_rows_remote=19186 dedup_ratio=0.334137 router_tokens_per_expert=[264, 252, 259, 234, 293, 235, 249, 281, 257, 250, 245, 249, 287, 265, 247, 251, 219, 273, 267, 256, 259, 249, 255, 237, 258, 244, 241, 254, 256, 251, 279, 248, 253, 257, 241, 266, 241, 268, 261, 249, 245, 231, 263, 241, 243, 268, 235, 256, 249, 273, 280, 279, 247, 249, 266, 263, 288, 284, 264, 255, 266, 244, 254, 240, 259, 252, 263, 256, 243, 254, 243, 275, 236, 224, 254, 290, 249, 275, 244, 259, 243, 275, 207, 255, 256, 272, 259, 289, 259, 237, 258, 246, 255, 251, 240, 270, 272, 243, 241, 259, 253, 265, 244, 273, 252, 238, 248, 261, 249, 251, 248, 258, 250, 277, 281, 247, 250, 266, 288, 275, 254, 262, 275, 262, 247, 265, 241, 242] payload_rows_per_dst=[5452, 5434, 5372, 5572, 5496, 5432, 5334, 5546] dup_rows_per_dst=[2784, 2658, 2664, 2830, 2656, 2712, 2776, 2818] combine_rows_per_src=[5452, 5434, 5372, 5572, 5496, 5432, 5334, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38306 dup_rows=21786 dup_rows_remote=19198 dedup_ratio=0.332428 router_tokens_per_expert=[245, 261, 243, 256, 270, 241, 271, 256, 258, 263, 267, 278, 264, 243, 233, 258, 295, 249, 230, 266, 251, 265, 282, 261, 254, 257, 228, 270, 242, 262, 265, 267, 236, 253, 252, 222, 233, 283, 276, 274, 247, 272, 241, 252, 240, 238, 249, 248, 231, 266, 268, 244, 252, 251, 235, 276, 261, 225, 217, 251, 246, 243, 241, 268, 249, 261, 261, 223, 239, 254, 270, 264, 272, 252, 239, 264, 244, 243, 280, 233, 249, 257, 241, 231, 251, 259, 267, 254, 272, 277, 264, 261, 245, 268, 274, 283, 258, 260, 247, 258, 237, 263, 227, 278, 252, 255, 287, 258, 278, 266, 245, 273, 260, 247, 256, 255, 277, 260, 254, 260, 250, 261, 294, 260, 265, 251, 274, 259] payload_rows_per_dst=[5440, 5490, 5444, 5328, 5410, 5574, 5494, 5570] dup_rows_per_dst=[2774, 2798, 2588, 2622, 2686, 2732, 2790, 2796] combine_rows_per_src=[5440, 5490, 5444, 5328, 5410, 5574, 5494, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228232 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156777352 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313326472 weight_movement_bytes=37748736 total_nvlink_bytes=351075208 weight_to_token_ratio=0.120477 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229232 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156868464 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313507696 weight_movement_bytes=56623104 total_nvlink_bytes=370130800 weight_to_token_ratio=0.180612 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38256 dup_rows=21774 dup_rows_remote=19084 dedup_ratio=0.332245 router_tokens_per_expert=[232, 276, 238, 254, 234, 261, 268, 258, 289, 250, 255, 251, 252, 241, 269, 262, 260, 264, 253, 253, 262, 264, 270, 268, 258, 265, 262, 244, 272, 249, 238, 231, 234, 224, 277, 255, 267, 270, 266, 249, 235, 265, 263, 227, 235, 251, 284, 265, 257, 261, 232, 254, 242, 249, 235, 248, 264, 261, 266, 268, 261, 248, 261, 272, 263, 255, 241, 250, 267, 246, 260, 240, 238, 251, 249, 257, 268, 233, 252, 260, 252, 264, 278, 253, 266, 242, 242, 233, 260, 259, 281, 262, 239, 270, 246, 278, 262, 237, 263, 264, 255, 227, 268, 250, 266, 235, 274, 240, 263, 256, 253, 272, 258, 256, 245, 289, 268, 250, 271, 262, 262, 264, 238, 269, 244, 271, 265, 267] payload_rows_per_dst=[5506, 5602, 5630, 5366, 5378, 5312, 5414, 5554] dup_rows_per_dst=[2690, 2864, 3066, 2636, 2682, 2376, 2656, 2804] combine_rows_per_src=[5506, 5602, 5630, 5366, 5378, 5312, 5414, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229568 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721344 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313213120 weight_movement_bytes=18874368 total_nvlink_bytes=332087488 weight_to_token_ratio=0.060260 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=230016 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157131392 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032768 weight_movement_bytes=18874368 total_nvlink_bytes=332907136 weight_to_token_ratio=0.060103 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38226 dup_rows=21790 dup_rows_remote=19184 dedup_ratio=0.332489 router_tokens_per_expert=[256, 240, 289, 228, 246, 231, 246, 255, 259, 243, 256, 251, 243, 235, 303, 261, 231, 260, 274, 248, 253, 256, 222, 245, 242, 245, 268, 268, 253, 250, 268, 296, 251, 284, 238, 275, 255, 265, 243, 262, 272, 259, 255, 259, 242, 261, 252, 268, 253, 272, 230, 277, 239, 275, 215, 249, 249, 236, 242, 262, 252, 298, 268, 270, 273, 252, 256, 263, 229, 240, 247, 247, 247, 253, 244, 264, 257, 250, 258, 283, 268, 259, 262, 276, 240, 257, 267, 264, 275, 242, 250, 260, 242, 273, 231, 258, 254, 240, 265, 267, 264, 252, 238, 267, 247, 298, 246, 250, 262, 241, 259, 255, 246, 250, 269, 268, 269, 280, 283, 258, 280, 229, 232, 228, 267, 272, 232, 264] payload_rows_per_dst=[5378, 5428, 5478, 5488, 5520, 5478, 5510, 5466] dup_rows_per_dst=[2706, 2730, 2804, 2686, 2606, 2770, 2700, 2788] combine_rows_per_src=[5378, 5428, 5478, 5488, 5520, 5478, 5510, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38178 dup_rows=21914 dup_rows_remote=19240 dedup_ratio=0.334381 router_tokens_per_expert=[234, 234, 258, 281, 256, 302, 237, 304, 255, 273, 237, 271, 247, 281, 284, 272, 251, 261, 240, 241, 257, 266, 257, 253, 264, 254, 239, 204, 293, 243, 292, 253, 253, 266, 247, 238, 275, 258, 252, 274, 267, 271, 274, 253, 243, 264, 229, 236, 242, 272, 262, 283, 230, 272, 234, 242, 243, 262, 256, 247, 243, 251, 266, 254, 270, 282, 254, 231, 251, 228, 240, 252, 248, 276, 249, 246, 231, 234, 254, 236, 246, 254, 290, 251, 240, 240, 248, 232, 233, 281, 270, 268, 230, 251, 265, 233, 242, 262, 272, 249, 249, 256, 248, 289, 277, 255, 262, 261, 278, 233, 247, 275, 263, 263, 286, 269, 265, 261, 258, 269, 262, 252, 245, 232, 246, 261, 262, 252] payload_rows_per_dst=[5546, 5424, 5440, 5444, 5328, 5450, 5514, 5476] dup_rows_per_dst=[2906, 2712, 2760, 2674, 2636, 2614, 2796, 2816] combine_rows_per_src=[5546, 5424, 5440, 5444, 5328, 5450, 5514, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38194 dup_rows=21846 dup_rows_remote=19024 dedup_ratio=0.333344 router_tokens_per_expert=[262, 231, 273, 242, 261, 258, 251, 251, 243, 263, 279, 268, 272, 253, 280, 232, 248, 273, 277, 237, 233, 271, 278, 260, 245, 265, 279, 273, 239, 284, 244, 253, 273, 235, 231, 269, 234, 272, 253, 247, 226, 280, 267, 266, 239, 246, 240, 237, 250, 257, 269, 261, 248, 266, 237, 243, 235, 217, 264, 292, 247, 276, 259, 256, 245, 273, 255, 243, 245, 232, 292, 261, 271, 281, 239, 266, 260, 263, 259, 278, 243, 289, 246, 244, 233, 270, 235, 247, 233, 265, 247, 267, 266, 246, 270, 254, 239, 265, 277, 255, 263, 286, 255, 231, 252, 252, 263, 231, 242, 257, 255, 254, 256, 235, 271, 272, 246, 255, 262, 259, 268, 275, 255, 256, 249, 230, 275, 239] payload_rows_per_dst=[5464, 5496, 5372, 5462, 5518, 5500, 5428, 5450] dup_rows_per_dst=[2774, 2822, 2658, 2692, 2808, 2610, 2726, 2756] combine_rows_per_src=[5464, 5496, 5372, 5462, 5518, 5500, 5428, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229360 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156925936 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622512 weight_movement_bytes=37748736 total_nvlink_bytes=351371248 weight_to_token_ratio=0.120364 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229640 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803336 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313377032 weight_movement_bytes=18874368 total_nvlink_bytes=332251400 weight_to_token_ratio=0.060229 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38182 dup_rows=21872 dup_rows_remote=19054 dedup_ratio=0.333740 router_tokens_per_expert=[281, 256, 226, 291, 252, 225, 239, 275, 294, 264, 253, 232, 239, 254, 259, 280, 237, 252, 265, 241, 239, 246, 245, 276, 249, 262, 250, 269, 242, 245, 251, 256, 267, 255, 265, 265, 265, 258, 256, 239, 239, 233, 254, 268, 266, 269, 240, 222, 260, 259, 235, 240, 271, 263, 253, 242, 249, 277, 236, 258, 266, 268, 271, 276, 269, 249, 256, 269, 253, 239, 251, 270, 237, 258, 247, 289, 265, 243, 221, 277, 263, 253, 240, 254, 265, 253, 248, 260, 247, 261, 264, 276, 252, 256, 252, 284, 266, 256, 258, 250, 261, 283, 279, 258, 260, 261, 245, 269, 252, 238, 256, 288, 250, 244, 279, 264, 269, 248, 250, 231, 236, 239, 260, 285, 250, 246, 253, 233] payload_rows_per_dst=[5350, 5360, 5514, 5526, 5460, 5502, 5470, 5482] dup_rows_per_dst=[2664, 2690, 2608, 2722, 2726, 2740, 2904, 2818] combine_rows_per_src=[5350, 5360, 5514, 5526, 5460, 5502, 5470, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229672 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606760 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983848 weight_movement_bytes=18874368 total_nvlink_bytes=331858216 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228872 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671496 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114120 weight_movement_bytes=18874368 total_nvlink_bytes=331988488 weight_to_token_ratio=0.060280 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38312 dup_rows=21662 dup_rows_remote=18876 dedup_ratio=0.330536 router_tokens_per_expert=[267, 261, 239, 284, 246, 227, 270, 263, 256, 256, 233, 249, 257, 265, 254, 221, 242, 234, 264, 264, 251, 248, 268, 252, 257, 235, 243, 264, 244, 261, 266, 242, 278, 264, 259, 244, 243, 249, 231, 270, 223, 281, 249, 240, 255, 242, 257, 264, 249, 286, 249, 255, 244, 254, 253, 273, 261, 266, 301, 275, 277, 248, 277, 257, 259, 242, 231, 261, 265, 232, 261, 267, 262, 267, 276, 246, 269, 274, 253, 255, 281, 256, 276, 235, 238, 263, 266, 255, 273, 246, 264, 253, 253, 239, 235, 275, 283, 287, 253, 276, 239, 270, 250, 220, 261, 253, 285, 255, 250, 273, 248, 271, 242, 258, 257, 266, 250, 245, 266, 231, 263, 225, 266, 260, 254, 254, 237, 235] payload_rows_per_dst=[5440, 5456, 5464, 5586, 5508, 5480, 5562, 5378] dup_rows_per_dst=[2656, 2614, 2634, 2864, 2732, 2736, 2786, 2640] combine_rows_per_src=[5440, 5456, 5464, 5586, 5508, 5480, 5562, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38058 dup_rows=21930 dup_rows_remote=19160 dedup_ratio=0.334625 router_tokens_per_expert=[242, 245, 271, 241, 266, 253, 258, 252, 256, 255, 264, 265, 248, 243, 268, 252, 267, 254, 263, 259, 256, 252, 266, 240, 241, 218, 258, 253, 248, 265, 261, 284, 247, 248, 271, 242, 244, 274, 247, 273, 240, 251, 263, 265, 273, 233, 268, 218, 254, 273, 268, 232, 250, 225, 269, 234, 253, 277, 268, 259, 283, 253, 240, 251, 250, 288, 277, 256, 266, 267, 275, 249, 257, 230, 256, 276, 260, 251, 259, 231, 265, 254, 249, 250, 262, 259, 296, 269, 231, 282, 247, 237, 250, 284, 269, 255, 263, 269, 259, 257, 231, 232, 298, 250, 261, 247, 228, 272, 260, 252, 235, 243, 259, 288, 236, 229, 262, 245, 241, 270, 225, 276, 282, 252, 246, 271, 264, 248] payload_rows_per_dst=[5456, 5472, 5406, 5432, 5460, 5548, 5422, 5410] dup_rows_per_dst=[2702, 2698, 2708, 2746, 2836, 2770, 2692, 2778] combine_rows_per_src=[5456, 5472, 5406, 5432, 5460, 5548, 5422, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38054 dup_rows=21974 dup_rows_remote=19172 dedup_ratio=0.335297 router_tokens_per_expert=[241, 263, 221, 238, 259, 266, 225, 264, 288, 254, 248, 262, 285, 251, 252, 249, 260, 283, 249, 265, 256, 244, 244, 276, 255, 265, 243, 276, 247, 239, 232, 266, 263, 276, 243, 237, 242, 257, 267, 277, 265, 279, 244, 276, 257, 250, 261, 261, 260, 266, 261, 261, 262, 257, 251, 260, 240, 256, 266, 248, 285, 249, 232, 251, 251, 247, 236, 275, 248, 265, 234, 259, 272, 247, 246, 257, 281, 244, 255, 275, 253, 247, 246, 267, 258, 256, 266, 255, 300, 248, 213, 255, 263, 272, 254, 240, 242, 268, 255, 304, 253, 257, 273, 240, 266, 244, 269, 279, 262, 237, 262, 239, 239, 264, 266, 247, 256, 237, 260, 229, 259, 247, 253, 234, 262, 254, 251, 249] payload_rows_per_dst=[5446, 5438, 5508, 5442, 5422, 5418, 5498, 5390] dup_rows_per_dst=[2686, 2762, 2802, 2768, 2762, 2768, 2802, 2624] combine_rows_per_src=[5446, 5438, 5508, 5442, 5422, 5418, 5498, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228944 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622416 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015888 weight_movement_bytes=18874368 total_nvlink_bytes=331890256 weight_to_token_ratio=0.060298 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228752 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157154704 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314080656 weight_movement_bytes=37748736 total_nvlink_bytes=351829392 weight_to_token_ratio=0.120188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38198 dup_rows=21776 dup_rows_remote=18854 dedup_ratio=0.332275 router_tokens_per_expert=[262, 255, 235, 262, 229, 264, 250, 252, 267, 257, 271, 229, 234, 257, 250, 263, 251, 263, 279, 244, 233, 251, 226, 270, 241, 257, 270, 260, 235, 272, 267, 238, 259, 238, 230, 248, 255, 274, 251, 268, 264, 265, 239, 253, 265, 252, 255, 257, 248, 266, 253, 245, 259, 250, 267, 247, 278, 261, 245, 267, 268, 248, 254, 242, 263, 259, 240, 265, 265, 259, 264, 258, 283, 262, 238, 235, 289, 253, 239, 278, 262, 239, 282, 257, 248, 237, 267, 254, 274, 250, 287, 250, 280, 248, 251, 271, 255, 230, 246, 269, 238, 262, 265, 253, 262, 264, 280, 255, 268, 251, 220, 245, 280, 274, 279, 270, 249, 278, 252, 267, 263, 243, 243, 240, 241, 258, 248, 248] payload_rows_per_dst=[5562, 5484, 5480, 5536, 5598, 5524, 5114, 5462] dup_rows_per_dst=[2922, 2950, 2734, 2718, 2702, 2740, 2282, 2728] combine_rows_per_src=[5562, 5484, 5480, 5536, 5598, 5524, 5114, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=228872 combine_payload_bytes=155885568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156114440 ingress_bytes=231383040 link_bytes=231383040 token_bytes=312000008 weight_movement_bytes=75497472 total_nvlink_bytes=387497480 weight_to_token_ratio=0.241979 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216998442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155869184 dispatch_meta_bytes=228904 combine_payload_bytes=155869184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156098088 ingress_bytes=174743552 link_bytes=174743552 token_bytes=311967272 weight_movement_bytes=18874368 total_nvlink_bytes=330841640 weight_to_token_ratio=0.060501 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674306736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38372 dup_rows=21744 dup_rows_remote=19134 dedup_ratio=0.331787 router_tokens_per_expert=[268, 250, 264, 267, 254, 274, 241, 229, 255, 260, 228, 270, 270, 268, 270, 262, 253, 255, 258, 258, 221, 245, 250, 260, 251, 230, 282, 244, 283, 247, 280, 254, 240, 256, 267, 232, 232, 254, 261, 265, 261, 273, 269, 248, 261, 235, 260, 237, 258, 251, 258, 254, 269, 273, 210, 218, 271, 260, 233, 267, 261, 250, 246, 263, 263, 240, 269, 268, 230, 257, 236, 278, 246, 263, 266, 216, 253, 244, 241, 245, 271, 240, 270, 249, 264, 272, 261, 258, 251, 246, 251, 260, 266, 276, 271, 258, 282, 243, 249, 256, 244, 261, 234, 271, 275, 260, 267, 259, 239, 257, 263, 254, 255, 271, 269, 273, 265, 276, 267, 244, 254, 242, 290, 243, 277, 268, 234, 253] payload_rows_per_dst=[5482, 5512, 5406, 5406, 5420, 5550, 5524, 5492] dup_rows_per_dst=[2778, 2630, 2696, 2678, 2610, 2778, 2704, 2870] combine_rows_per_src=[5482, 5512, 5406, 5406, 5420, 5550, 5524, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38184 dup_rows=21832 dup_rows_remote=19028 dedup_ratio=0.333130 router_tokens_per_expert=[226, 257, 217, 250, 239, 232, 242, 268, 244, 287, 234, 270, 244, 271, 268, 248, 256, 251, 270, 222, 262, 248, 253, 254, 239, 238, 274, 260, 239, 242, 270, 303, 272, 239, 254, 265, 251, 235, 233, 238, 242, 269, 240, 257, 287, 260, 266, 227, 272, 215, 240, 258, 260, 252, 276, 275, 274, 285, 283, 250, 257, 242, 262, 261, 271, 260, 261, 257, 243, 266, 255, 273, 238, 260, 240, 228, 272, 253, 240, 248, 257, 268, 250, 264, 279, 246, 263, 258, 280, 250, 268, 285, 252, 266, 241, 250, 287, 265, 283, 283, 231, 251, 268, 241, 259, 243, 272, 260, 249, 243, 254, 276, 260, 245, 252, 274, 261, 248, 280, 235, 256, 255, 255, 239, 248, 250, 288, 240] payload_rows_per_dst=[5476, 5406, 5398, 5520, 5440, 5528, 5500, 5436] dup_rows_per_dst=[2518, 2756, 2672, 2804, 2690, 2826, 2830, 2736] combine_rows_per_src=[5476, 5406, 5398, 5520, 5440, 5528, 5500, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38404 dup_rows=21578 dup_rows_remote=18854 dedup_ratio=0.329254 router_tokens_per_expert=[259, 276, 240, 248, 252, 234, 260, 259, 262, 245, 244, 269, 268, 265, 236, 239, 270, 230, 285, 256, 260, 245, 257, 260, 251, 246, 280, 281, 240, 252, 266, 260, 240, 272, 260, 261, 245, 240, 256, 264, 257, 257, 230, 258, 248, 266, 242, 288, 241, 255, 276, 273, 247, 257, 272, 253, 244, 267, 243, 286, 260, 238, 255, 268, 259, 243, 248, 274, 226, 253, 274, 261, 251, 263, 271, 254, 291, 263, 270, 246, 275, 257, 250, 244, 241, 254, 261, 251, 241, 249, 256, 234, 253, 279, 282, 260, 275, 275, 282, 261, 242, 253, 267, 214, 235, 267, 246, 235, 243, 231, 263, 242, 268, 255, 273, 227, 260, 282, 232, 250, 253, 243, 270, 253, 261, 238, 280, 244] payload_rows_per_dst=[5452, 5554, 5496, 5518, 5510, 5518, 5400, 5510] dup_rows_per_dst=[2660, 2724, 2672, 2752, 2784, 2656, 2662, 2668] combine_rows_per_src=[5452, 5554, 5496, 5518, 5510, 5518, 5400, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228208 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156687216 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313146224 weight_movement_bytes=18874368 total_nvlink_bytes=332020592 weight_to_token_ratio=0.060273 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=230024 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157401736 ingress_bytes=194920448 link_bytes=194920448 token_bytes=314573448 weight_movement_bytes=37748736 total_nvlink_bytes=352322184 weight_to_token_ratio=0.120000 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867631826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38370 dup_rows=21750 dup_rows_remote=18982 dedup_ratio=0.331879 router_tokens_per_expert=[250, 239, 249, 269, 248, 248, 281, 231, 251, 261, 262, 235, 249, 229, 278, 254, 270, 285, 270, 283, 272, 264, 227, 263, 242, 259, 269, 252, 279, 258, 239, 248, 236, 251, 277, 232, 285, 268, 277, 272, 257, 253, 236, 263, 232, 294, 274, 271, 262, 274, 256, 287, 258, 232, 247, 267, 241, 268, 256, 273, 247, 278, 249, 275, 233, 243, 274, 235, 220, 255, 258, 260, 232, 270, 256, 252, 254, 252, 248, 263, 237, 276, 271, 234, 244, 293, 269, 243, 261, 249, 255, 254, 252, 260, 266, 256, 269, 265, 254, 241, 247, 271, 259, 253, 250, 233, 237, 255, 250, 250, 248, 255, 241, 244, 252, 260, 245, 266, 237, 243, 227, 239, 260, 239, 284, 267, 269, 271] payload_rows_per_dst=[5404, 5432, 5496, 5556, 5570, 5490, 5422, 5416] dup_rows_per_dst=[2664, 2700, 2764, 2784, 2632, 2750, 2688, 2768] combine_rows_per_src=[5404, 5432, 5496, 5556, 5570, 5490, 5422, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228848 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156630512 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032176 weight_movement_bytes=18874368 total_nvlink_bytes=331906544 weight_to_token_ratio=0.060295 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229032 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157531816 ingress_bytes=213925888 link_bytes=213925888 token_bytes=314834600 weight_movement_bytes=56623104 total_nvlink_bytes=371457704 weight_to_token_ratio=0.179850 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049732601e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38014 dup_rows=21988 dup_rows_remote=19132 dedup_ratio=0.335510 router_tokens_per_expert=[246, 252, 273, 266, 260, 237, 257, 236, 256, 246, 262, 271, 251, 256, 272, 249, 266, 267, 250, 233, 259, 271, 243, 265, 245, 256, 262, 274, 245, 261, 246, 280, 258, 245, 287, 239, 252, 269, 235, 246, 232, 240, 247, 289, 255, 264, 270, 254, 252, 246, 269, 258, 244, 236, 266, 266, 271, 249, 273, 250, 250, 240, 265, 257, 266, 237, 234, 239, 251, 243, 261, 249, 250, 251, 243, 234, 249, 255, 249, 235, 257, 291, 268, 254, 228, 268, 276, 258, 248, 233, 304, 260, 277, 240, 261, 262, 241, 258, 266, 262, 250, 269, 282, 264, 223, 288, 283, 246, 270, 260, 250, 283, 254, 274, 251, 237, 262, 264, 240, 254, 235, 266, 273, 252, 244, 243, 251, 255] payload_rows_per_dst=[5410, 5428, 5470, 5468, 5344, 5492, 5534, 5402] dup_rows_per_dst=[2770, 2818, 2694, 2716, 2548, 2878, 2856, 2708] combine_rows_per_src=[5410, 5428, 5470, 5468, 5344, 5492, 5534, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38250 dup_rows=21810 dup_rows_remote=19004 dedup_ratio=0.332794 router_tokens_per_expert=[243, 243, 264, 257, 261, 254, 268, 269, 261, 255, 270, 264, 253, 251, 279, 222, 257, 271, 284, 246, 262, 227, 244, 252, 226, 264, 235, 290, 253, 270, 246, 242, 242, 262, 255, 260, 281, 246, 253, 261, 234, 267, 283, 257, 241, 261, 263, 274, 275, 263, 235, 227, 248, 251, 257, 233, 215, 236, 243, 258, 240, 243, 251, 257, 238, 261, 267, 262, 227, 247, 255, 250, 267, 277, 291, 257, 272, 260, 245, 242, 275, 241, 293, 273, 248, 248, 247, 258, 250, 252, 268, 270, 246, 256, 248, 268, 255, 261, 252, 252, 281, 266, 253, 269, 251, 261, 245, 247, 254, 266, 269, 248, 257, 273, 272, 256, 274, 248, 244, 238, 227, 257, 283, 277, 254, 248, 280, 236] payload_rows_per_dst=[5478, 5480, 5474, 5292, 5484, 5476, 5560, 5482] dup_rows_per_dst=[2750, 2658, 2806, 2572, 2752, 2806, 2700, 2766] combine_rows_per_src=[5478, 5480, 5474, 5292, 5484, 5476, 5560, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38394 dup_rows=21652 dup_rows_remote=18988 dedup_ratio=0.330383 router_tokens_per_expert=[248, 249, 262, 250, 265, 298, 257, 274, 239, 238, 246, 249, 244, 268, 252, 248, 267, 259, 243, 262, 246, 271, 268, 247, 265, 255, 258, 280, 280, 266, 245, 268, 250, 230, 228, 253, 273, 250, 236, 266, 251, 290, 266, 256, 239, 264, 282, 243, 243, 252, 260, 238, 296, 238, 249, 243, 241, 248, 256, 248, 239, 251, 254, 246, 266, 284, 251, 263, 256, 251, 273, 265, 248, 238, 210, 251, 243, 276, 262, 248, 236, 261, 268, 244, 261, 258, 263, 270, 292, 250, 263, 255, 246, 247, 252, 246, 259, 279, 270, 251, 260, 272, 233, 246, 244, 274, 281, 260, 256, 227, 279, 256, 258, 251, 277, 241, 272, 241, 245, 250, 240, 265, 264, 249, 264, 255, 227, 279] payload_rows_per_dst=[5492, 5570, 5490, 5310, 5514, 5500, 5546, 5462] dup_rows_per_dst=[2682, 2790, 2664, 2694, 2656, 2724, 2748, 2694] combine_rows_per_src=[5492, 5570, 5490, 5310, 5514, 5500, 5546, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229408 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157392928 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556448 weight_movement_bytes=18874368 total_nvlink_bytes=333430816 weight_to_token_ratio=0.060003 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=228584 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155933928 ingress_bytes=212328448 link_bytes=212328448 token_bytes=311639272 weight_movement_bytes=56623104 total_nvlink_bytes=368262376 weight_to_token_ratio=0.181694 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034426717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38330 dup_rows=21652 dup_rows_remote=18894 dedup_ratio=0.330383 router_tokens_per_expert=[238, 275, 251, 265, 234, 251, 256, 273, 285, 230, 263, 257, 243, 252, 276, 263, 292, 236, 260, 264, 287, 238, 267, 251, 255, 225, 243, 253, 290, 264, 264, 258, 282, 250, 253, 233, 259, 247, 239, 241, 245, 262, 259, 247, 242, 264, 241, 252, 244, 252, 233, 244, 242, 250, 269, 228, 242, 236, 270, 263, 260, 245, 248, 229, 274, 251, 258, 262, 230, 261, 259, 265, 241, 240, 267, 238, 274, 249, 280, 258, 270, 296, 257, 262, 269, 242, 244, 253, 238, 244, 227, 258, 270, 228, 274, 248, 274, 290, 248, 289, 253, 263, 255, 232, 262, 245, 280, 257, 262, 257, 273, 245, 263, 255, 265, 256, 243, 270, 276, 252, 254, 265, 268, 251, 257, 280, 234, 277] payload_rows_per_dst=[5554, 5512, 5636, 5364, 5448, 5460, 5574, 5336] dup_rows_per_dst=[2758, 2694, 2900, 2578, 2766, 2700, 2796, 2460] combine_rows_per_src=[5554, 5512, 5636, 5364, 5448, 5460, 5574, 5336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229016 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156901016 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313573016 weight_movement_bytes=37748736 total_nvlink_bytes=351321752 weight_to_token_ratio=0.120383 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43956 payload_rows_remote=38482 dup_rows=21580 dup_rows_remote=18904 dedup_ratio=0.329285 router_tokens_per_expert=[256, 258, 279, 237, 244, 243, 250, 260, 251, 269, 229, 248, 271, 231, 242, 290, 261, 229, 277, 254, 245, 281, 253, 247, 244, 280, 250, 243, 258, 244, 255, 259, 274, 285, 236, 269, 259, 245, 292, 268, 257, 251, 258, 269, 254, 236, 253, 278, 233, 278, 236, 267, 242, 266, 245, 246, 267, 271, 244, 265, 225, 251, 269, 218, 249, 244, 273, 235, 279, 263, 249, 263, 250, 234, 261, 245, 256, 251, 270, 253, 249, 248, 241, 261, 266, 232, 277, 257, 265, 246, 287, 245, 253, 264, 260, 257, 259, 240, 239, 283, 254, 257, 227, 284, 222, 279, 277, 271, 262, 254, 255, 261, 260, 263, 274, 255, 243, 275, 233, 246, 257, 246, 278, 275, 249, 273, 244, 245] payload_rows_per_dst=[5462, 5478, 5558, 5410, 5474, 5486, 5480, 5608] dup_rows_per_dst=[2654, 2682, 2810, 2636, 2676, 2730, 2768, 2624] combine_rows_per_src=[5462, 5478, 5558, 5410, 5474, 5486, 5480, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229528 combine_payload_bytes=157261824 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157491352 ingress_bytes=195010560 link_bytes=195010560 token_bytes=314753176 weight_movement_bytes=37748736 total_nvlink_bytes=352501912 weight_to_token_ratio=0.119931 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868495235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38196 dup_rows=21896 dup_rows_remote=19182 dedup_ratio=0.334106 router_tokens_per_expert=[256, 253, 242, 249, 215, 246, 229, 253, 247, 273, 263, 255, 257, 252, 273, 282, 259, 256, 251, 268, 259, 238, 247, 271, 256, 265, 273, 237, 264, 237, 271, 263, 222, 265, 261, 270, 279, 254, 254, 276, 255, 283, 265, 228, 262, 243, 267, 239, 250, 258, 235, 242, 248, 268, 262, 269, 226, 249, 264, 283, 261, 252, 253, 259, 248, 271, 246, 254, 276, 241, 293, 273, 256, 244, 257, 266, 265, 251, 261, 257, 282, 254, 230, 288, 238, 256, 240, 255, 236, 263, 254, 236, 271, 239, 271, 272, 242, 271, 241, 239, 248, 243, 258, 274, 273, 242, 260, 272, 255, 267, 238, 251, 244, 259, 267, 261, 233, 254, 275, 245, 262, 272, 256, 246, 269, 241, 251, 253] payload_rows_per_dst=[5386, 5508, 5448, 5444, 5500, 5420, 5444, 5490] dup_rows_per_dst=[2704, 2722, 2798, 2714, 2818, 2750, 2704, 2686] combine_rows_per_src=[5386, 5508, 5448, 5444, 5500, 5420, 5444, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38396 dup_rows=21720 dup_rows_remote=18954 dedup_ratio=0.331421 router_tokens_per_expert=[240, 270, 270, 257, 254, 266, 282, 254, 231, 249, 257, 261, 231, 239, 289, 275, 260, 272, 236, 251, 242, 250, 275, 246, 243, 254, 244, 254, 260, 254, 292, 260, 261, 260, 238, 253, 267, 283, 223, 251, 257, 242, 267, 262, 256, 254, 236, 277, 283, 242, 237, 256, 237, 260, 269, 239, 284, 262, 278, 232, 245, 246, 233, 247, 266, 271, 262, 249, 247, 287, 290, 238, 266, 271, 286, 246, 242, 248, 246, 267, 247, 220, 233, 242, 252, 277, 252, 279, 249, 240, 259, 264, 241, 265, 229, 246, 260, 250, 239, 273, 274, 237, 246, 248, 250, 243, 256, 252, 246, 259, 253, 287, 282, 260, 263, 243, 248, 262, 275, 256, 266, 242, 275, 258, 276, 252, 245, 260] payload_rows_per_dst=[5550, 5420, 5512, 5572, 5460, 5410, 5408, 5484] dup_rows_per_dst=[2700, 2766, 2662, 2708, 2724, 2580, 2738, 2842] combine_rows_per_src=[5550, 5420, 5512, 5572, 5460, 5410, 5408, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=228896 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157228576 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228256 weight_movement_bytes=37748736 total_nvlink_bytes=351976992 weight_to_token_ratio=0.120132 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157622272 dispatch_meta_bytes=229544 combine_payload_bytes=157622272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157851816 ingress_bytes=195371008 link_bytes=195371008 token_bytes=315474088 weight_movement_bytes=37748736 total_nvlink_bytes=353222824 weight_to_token_ratio=0.119657 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871948870e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229512 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680328 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131144 weight_movement_bytes=18874368 total_nvlink_bytes=332005512 weight_to_token_ratio=0.060276 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38300 dup_rows=21712 dup_rows_remote=18804 dedup_ratio=0.331299 router_tokens_per_expert=[272, 235, 256, 258, 231, 255, 254, 282, 257, 259, 241, 273, 251, 235, 247, 269, 244, 259, 245, 250, 256, 269, 261, 246, 237, 272, 262, 252, 258, 256, 257, 231, 245, 267, 237, 248, 265, 257, 254, 254, 261, 245, 249, 272, 245, 234, 258, 266, 273, 255, 239, 227, 220, 264, 263, 265, 252, 262, 274, 228, 267, 241, 263, 281, 249, 247, 237, 239, 247, 249, 278, 248, 250, 274, 228, 262, 285, 288, 272, 269, 243, 263, 245, 255, 231, 276, 271, 271, 267, 265, 250, 267, 222, 254, 257, 239, 274, 264, 272, 236, 255, 261, 240, 264, 279, 240, 250, 247, 241, 287, 253, 255, 259, 237, 254, 265, 257, 249, 272, 245, 278, 267, 262, 246, 269, 276, 261, 294] payload_rows_per_dst=[5326, 5322, 5600, 5440, 5584, 5516, 5512, 5524] dup_rows_per_dst=[2424, 2450, 2812, 2708, 2948, 2738, 2724, 2908] combine_rows_per_src=[5326, 5322, 5600, 5440, 5584, 5516, 5512, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229400 combine_payload_bytes=157270016 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157499416 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769432 weight_movement_bytes=56623104 total_nvlink_bytes=371392536 weight_to_token_ratio=0.179888 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38350 dup_rows=21738 dup_rows_remote=18956 dedup_ratio=0.331696 router_tokens_per_expert=[236, 255, 261, 281, 272, 250, 236, 265, 240, 268, 231, 260, 264, 235, 272, 261, 256, 232, 236, 250, 290, 248, 271, 269, 234, 222, 250, 267, 253, 242, 261, 234, 259, 242, 245, 260, 275, 268, 276, 231, 246, 246, 250, 264, 254, 263, 249, 260, 256, 251, 260, 281, 252, 248, 263, 213, 243, 255, 254, 252, 255, 266, 230, 261, 286, 255, 266, 242, 268, 254, 268, 258, 264, 258, 266, 257, 261, 284, 254, 273, 246, 237, 262, 216, 270, 266, 283, 256, 293, 264, 271, 225, 240, 254, 275, 241, 231, 253, 252, 260, 263, 252, 292, 271, 234, 240, 251, 280, 250, 313, 220, 253, 285, 236, 261, 239, 241, 268, 235, 251, 250, 269, 253, 260, 266, 247, 285, 264] payload_rows_per_dst=[5482, 5424, 5466, 5450, 5538, 5504, 5448, 5486] dup_rows_per_dst=[2692, 2606, 2710, 2630, 2890, 2694, 2782, 2734] combine_rows_per_src=[5482, 5424, 5466, 5450, 5538, 5504, 5448, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43518 payload_rows_remote=38098 dup_rows=22018 dup_rows_remote=19194 dedup_ratio=0.335968 router_tokens_per_expert=[247, 251, 262, 252, 269, 264, 229, 241, 260, 267, 262, 269, 244, 242, 261, 250, 243, 243, 253, 252, 253, 272, 238, 243, 280, 264, 260, 290, 237, 294, 277, 268, 253, 253, 286, 243, 258, 281, 246, 248, 264, 287, 225, 258, 215, 255, 262, 236, 276, 254, 270, 258, 291, 286, 228, 286, 250, 239, 249, 278, 250, 258, 224, 231, 275, 240, 265, 256, 254, 240, 258, 237, 260, 237, 277, 228, 266, 257, 265, 260, 286, 255, 251, 262, 256, 269, 248, 248, 264, 236, 261, 287, 257, 257, 249, 236, 250, 264, 248, 263, 254, 255, 245, 239, 258, 254, 272, 257, 247, 254, 231, 265, 277, 262, 247, 252, 233, 271, 245, 260, 261, 249, 240, 246, 265, 262, 262, 248] payload_rows_per_dst=[5372, 5590, 5470, 5448, 5388, 5420, 5424, 5406] dup_rows_per_dst=[2768, 2744, 2670, 2808, 2762, 2824, 2688, 2754] combine_rows_per_src=[5372, 5590, 5470, 5448, 5388, 5420, 5424, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38240 dup_rows=21862 dup_rows_remote=19216 dedup_ratio=0.333588 router_tokens_per_expert=[253, 249, 262, 258, 266, 261, 276, 281, 269, 257, 259, 274, 257, 280, 254, 260, 256, 240, 277, 248, 273, 262, 260, 250, 264, 246, 255, 242, 267, 265, 287, 254, 243, 249, 247, 264, 273, 245, 266, 242, 269, 239, 268, 251, 237, 250, 247, 250, 238, 248, 254, 237, 232, 261, 257, 245, 284, 228, 260, 271, 237, 251, 255, 255, 270, 255, 262, 242, 267, 264, 275, 241, 257, 253, 254, 243, 245, 262, 247, 240, 259, 274, 260, 248, 285, 263, 257, 268, 242, 244, 289, 239, 255, 239, 233, 252, 245, 239, 234, 258, 262, 270, 243, 222, 279, 274, 256, 236, 263, 263, 240, 248, 240, 228, 285, 270, 239, 261, 261, 270, 250, 248, 242, 281, 245, 270, 284, 263] payload_rows_per_dst=[5512, 5544, 5434, 5364, 5438, 5492, 5428, 5462] dup_rows_per_dst=[2920, 2748, 2646, 2662, 2716, 2722, 2636, 2812] combine_rows_per_src=[5512, 5544, 5434, 5364, 5438, 5492, 5428, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=228416 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157105216 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313982016 weight_movement_bytes=37748736 total_nvlink_bytes=351730752 weight_to_token_ratio=0.120226 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229224 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157310824 ingress_bytes=194830336 link_bytes=194830336 token_bytes=314392424 weight_movement_bytes=37748736 total_nvlink_bytes=352141160 weight_to_token_ratio=0.120069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866768417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43992 payload_rows_remote=38470 dup_rows=21544 dup_rows_remote=18788 dedup_ratio=0.328735 router_tokens_per_expert=[280, 258, 255, 279, 221, 273, 256, 223, 265, 266, 271, 256, 251, 282, 263, 231, 234, 256, 261, 252, 241, 269, 238, 265, 253, 239, 246, 248, 264, 265, 294, 268, 242, 275, 270, 237, 257, 237, 263, 243, 251, 238, 265, 257, 242, 260, 237, 233, 247, 225, 229, 260, 263, 243, 243, 247, 264, 265, 255, 258, 274, 246, 284, 246, 237, 257, 250, 218, 259, 266, 261, 244, 254, 280, 257, 249, 268, 264, 241, 276, 250, 265, 270, 286, 248, 235, 252, 280, 274, 253, 265, 293, 248, 263, 264, 248, 251, 276, 250, 277, 265, 251, 258, 250, 248, 261, 258, 262, 252, 271, 275, 223, 270, 269, 229, 264, 259, 257, 282, 262, 255, 225, 245, 244, 249, 280, 265, 231] payload_rows_per_dst=[5522, 5474, 5570, 5524, 5690, 5474, 5218, 5520] dup_rows_per_dst=[2756, 2784, 2744, 2746, 2872, 2524, 2466, 2652] combine_rows_per_src=[5522, 5474, 5570, 5524, 5690, 5474, 5218, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=229168 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156278576 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312327984 weight_movement_bytes=18874368 total_nvlink_bytes=331202352 weight_to_token_ratio=0.060431 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38156 dup_rows=21940 dup_rows_remote=19198 dedup_ratio=0.334778 router_tokens_per_expert=[225, 276, 239, 242, 245, 242, 254, 274, 250, 241, 281, 267, 255, 261, 258, 256, 259, 258, 258, 273, 242, 261, 266, 226, 265, 263, 251, 266, 230, 230, 270, 262, 262, 234, 268, 256, 262, 251, 285, 259, 259, 251, 255, 244, 252, 275, 247, 285, 251, 222, 238, 268, 244, 251, 235, 287, 250, 259, 251, 232, 241, 278, 264, 257, 275, 262, 262, 255, 231, 261, 264, 255, 260, 251, 234, 245, 262, 277, 240, 257, 259, 244, 241, 270, 224, 264, 284, 254, 249, 271, 246, 283, 252, 217, 265, 250, 267, 254, 260, 263, 262, 260, 231, 259, 268, 258, 255, 243, 259, 242, 274, 258, 263, 267, 266, 263, 243, 276, 245, 252, 246, 246, 256, 246, 291, 281, 276, 255] payload_rows_per_dst=[5470, 5368, 5438, 5398, 5440, 5470, 5464, 5548] dup_rows_per_dst=[2662, 2792, 2852, 2658, 2742, 2676, 2762, 2796] combine_rows_per_src=[5470, 5368, 5438, 5398, 5440, 5470, 5464, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229824 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860864 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491904 weight_movement_bytes=18874368 total_nvlink_bytes=332366272 weight_to_token_ratio=0.060207 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38246 dup_rows=21746 dup_rows_remote=18972 dedup_ratio=0.331818 router_tokens_per_expert=[238, 246, 266, 259, 227, 246, 268, 255, 249, 250, 262, 292, 251, 274, 246, 239, 262, 240, 284, 269, 241, 256, 218, 233, 259, 238, 267, 268, 251, 255, 250, 253, 256, 273, 239, 287, 253, 277, 280, 243, 244, 259, 245, 286, 274, 267, 249, 258, 270, 257, 284, 248, 267, 263, 269, 255, 265, 246, 250, 249, 271, 252, 260, 253, 252, 247, 251, 248, 228, 252, 281, 240, 233, 274, 274, 265, 235, 246, 270, 270, 264, 257, 252, 252, 262, 237, 259, 250, 239, 242, 281, 274, 262, 266, 260, 273, 261, 243, 283, 255, 232, 239, 233, 270, 280, 231, 249, 279, 255, 251, 251, 255, 238, 241, 240, 275, 254, 251, 257, 256, 281, 269, 258, 239, 218, 242, 267, 258] payload_rows_per_dst=[5408, 5404, 5540, 5544, 5508, 5490, 5470, 5426] dup_rows_per_dst=[2728, 2684, 2840, 2774, 2624, 2770, 2664, 2662] combine_rows_per_src=[5408, 5404, 5540, 5544, 5508, 5490, 5470, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38218 dup_rows=21916 dup_rows_remote=19380 dedup_ratio=0.334412 router_tokens_per_expert=[258, 283, 292, 243, 252, 271, 267, 246, 245, 282, 270, 266, 240, 263, 250, 255, 255, 232, 232, 251, 246, 279, 264, 237, 267, 242, 242, 245, 234, 253, 224, 266, 292, 248, 233, 255, 265, 265, 265, 267, 269, 233, 273, 251, 244, 256, 248, 249, 249, 262, 262, 270, 245, 285, 272, 275, 233, 252, 243, 260, 219, 248, 249, 238, 287, 254, 260, 222, 244, 259, 271, 257, 247, 269, 241, 250, 231, 257, 222, 244, 274, 262, 252, 248, 268, 244, 271, 271, 274, 257, 271, 243, 259, 286, 278, 245, 259, 225, 260, 251, 279, 265, 243, 265, 245, 237, 274, 259, 286, 276, 249, 262, 249, 236, 279, 245, 275, 262, 243, 271, 256, 259, 262, 257, 261, 241, 264, 228] payload_rows_per_dst=[5504, 5402, 5484, 5426, 5384, 5474, 5476, 5470] dup_rows_per_dst=[2862, 2536, 2742, 2698, 2646, 2932, 2794, 2706] combine_rows_per_src=[5504, 5402, 5484, 5426, 5384, 5474, 5476, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229032 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157802152 ingress_bytes=195321856 link_bytes=195321856 token_bytes=315375272 weight_movement_bytes=37748736 total_nvlink_bytes=353124008 weight_to_token_ratio=0.119695 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871477920e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229416 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156516392 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312803368 weight_movement_bytes=18874368 total_nvlink_bytes=331677736 weight_to_token_ratio=0.060339 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228872 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156884488 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540104 weight_movement_bytes=56623104 total_nvlink_bytes=370163208 weight_to_token_ratio=0.180593 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38364 dup_rows=21660 dup_rows_remote=18748 dedup_ratio=0.330505 router_tokens_per_expert=[259, 262, 259, 228, 242, 290, 237, 273, 240, 255, 269, 257, 255, 276, 264, 280, 237, 280, 268, 267, 268, 250, 265, 257, 260, 260, 289, 263, 239, 240, 241, 272, 233, 250, 220, 254, 225, 285, 288, 275, 250, 245, 253, 240, 248, 264, 242, 244, 246, 267, 273, 244, 221, 268, 271, 255, 234, 269, 248, 276, 266, 258, 249, 266, 251, 285, 241, 264, 232, 235, 245, 259, 277, 240, 243, 236, 240, 246, 256, 278, 271, 250, 262, 239, 245, 234, 244, 249, 252, 269, 238, 280, 235, 230, 242, 275, 248, 250, 279, 269, 283, 254, 272, 274, 241, 232, 251, 250, 234, 266, 244, 279, 288, 249, 289, 261, 240, 258, 253, 261, 267, 258, 258, 251, 272, 250, 261, 254] payload_rows_per_dst=[5522, 5594, 5432, 5432, 5470, 5400, 5514, 5512] dup_rows_per_dst=[2770, 2718, 2600, 2776, 2586, 2630, 2668, 2912] combine_rows_per_src=[5522, 5594, 5432, 5432, 5470, 5400, 5514, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=230392 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156771320 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313312248 weight_movement_bytes=18874368 total_nvlink_bytes=332186616 weight_to_token_ratio=0.060241 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38198 dup_rows=21956 dup_rows_remote=19288 dedup_ratio=0.335022 router_tokens_per_expert=[237, 261, 287, 283, 267, 262, 242, 241, 269, 253, 247, 261, 259, 270, 264, 266, 255, 267, 266, 276, 266, 263, 270, 283, 282, 245, 245, 238, 271, 237, 250, 254, 265, 247, 253, 261, 241, 245, 251, 259, 252, 268, 241, 243, 246, 246, 259, 269, 228, 256, 264, 244, 256, 227, 230, 251, 240, 275, 267, 233, 241, 245, 247, 255, 259, 254, 253, 276, 261, 248, 259, 277, 281, 253, 228, 263, 270, 243, 259, 265, 253, 295, 250, 261, 256, 314, 260, 244, 256, 253, 267, 250, 246, 233, 214, 247, 268, 231, 245, 255, 234, 245, 242, 253, 265, 263, 244, 246, 236, 294, 244, 260, 220, 249, 252, 283, 269, 271, 231, 254, 276, 272, 258, 264, 269, 264, 248, 273] payload_rows_per_dst=[5556, 5478, 5406, 5314, 5506, 5464, 5382, 5474] dup_rows_per_dst=[2782, 2858, 2686, 2604, 2792, 2734, 2668, 2832] combine_rows_per_src=[5556, 5478, 5406, 5314, 5506, 5464, 5382, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38354 dup_rows=21724 dup_rows_remote=18938 dedup_ratio=0.331482 router_tokens_per_expert=[232, 247, 261, 265, 277, 282, 256, 269, 245, 242, 252, 246, 266, 261, 244, 262, 280, 244, 265, 272, 237, 264, 247, 245, 260, 255, 255, 224, 263, 263, 252, 262, 254, 242, 226, 246, 279, 260, 273, 267, 266, 275, 228, 246, 276, 246, 266, 244, 266, 247, 237, 256, 270, 254, 255, 267, 254, 265, 254, 247, 267, 259, 266, 263, 284, 243, 229, 223, 273, 263, 256, 272, 242, 246, 261, 249, 268, 265, 238, 267, 265, 289, 238, 278, 269, 242, 262, 248, 241, 246, 263, 253, 267, 259, 243, 259, 237, 235, 253, 270, 239, 275, 232, 322, 262, 238, 237, 240, 275, 240, 256, 244, 244, 230, 243, 246, 251, 270, 243, 232, 286, 262, 248, 276, 276, 259, 281, 249] payload_rows_per_dst=[5514, 5470, 5450, 5534, 5458, 5458, 5420, 5508] dup_rows_per_dst=[2700, 2706, 2738, 2720, 2700, 2786, 2690, 2684] combine_rows_per_src=[5514, 5470, 5450, 5534, 5458, 5458, 5420, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38340 dup_rows=21742 dup_rows_remote=19164 dedup_ratio=0.331757 router_tokens_per_expert=[261, 268, 244, 242, 258, 264, 257, 250, 233, 265, 248, 231, 259, 262, 272, 252, 267, 258, 270, 264, 267, 240, 269, 224, 236, 263, 246, 232, 284, 252, 246, 246, 249, 216, 264, 225, 247, 248, 259, 259, 243, 259, 261, 233, 269, 253, 269, 262, 275, 264, 258, 249, 270, 255, 254, 264, 279, 283, 246, 268, 262, 257, 279, 261, 227, 300, 272, 252, 248, 272, 242, 252, 243, 236, 268, 250, 257, 233, 239, 255, 254, 243, 289, 243, 233, 276, 255, 283, 231, 256, 261, 255, 243, 266, 250, 245, 256, 242, 263, 264, 244, 260, 264, 272, 256, 250, 252, 288, 266, 257, 247, 252, 270, 272, 261, 242, 247, 283, 245, 269, 230, 277, 269, 235, 266, 272, 247, 251] payload_rows_per_dst=[5438, 5436, 5454, 5584, 5426, 5498, 5500, 5458] dup_rows_per_dst=[2694, 2692, 2578, 2864, 2666, 2668, 2766, 2814] combine_rows_per_src=[5438, 5436, 5454, 5584, 5426, 5498, 5500, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=228448 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157367392 ingress_bytes=194887680 link_bytes=194887680 token_bytes=314506336 weight_movement_bytes=37748736 total_nvlink_bytes=352255072 weight_to_token_ratio=0.120025 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229944 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688952 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147960 weight_movement_bytes=18874368 total_nvlink_bytes=332022328 weight_to_token_ratio=0.060273 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229168 combine_payload_bytes=157097984 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157327152 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314425136 weight_movement_bytes=37748736 total_nvlink_bytes=352173872 weight_to_token_ratio=0.120056 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38366 dup_rows=21794 dup_rows_remote=19204 dedup_ratio=0.332550 router_tokens_per_expert=[232, 276, 234, 248, 259, 270, 286, 268, 276, 260, 268, 243, 265, 254, 240, 276, 269, 267, 280, 264, 254, 252, 236, 263, 283, 251, 265, 247, 265, 223, 255, 229, 274, 275, 278, 264, 280, 259, 221, 213, 235, 291, 239, 251, 256, 288, 248, 239, 261, 247, 272, 235, 261, 245, 243, 242, 265, 241, 244, 247, 265, 235, 241, 239, 267, 233, 263, 260, 276, 246, 275, 256, 248, 274, 268, 248, 264, 264, 246, 250, 245, 238, 263, 270, 263, 234, 235, 283, 260, 242, 242, 266, 246, 244, 260, 244, 262, 272, 234, 262, 245, 259, 260, 255, 268, 251, 243, 265, 256, 261, 259, 275, 247, 239, 254, 262, 266, 265, 254, 262, 267, 266, 280, 240, 255, 248, 265, 246] payload_rows_per_dst=[5516, 5452, 5484, 5376, 5520, 5418, 5534, 5442] dup_rows_per_dst=[2794, 2754, 2738, 2590, 2756, 2652, 2720, 2790] combine_rows_per_src=[5516, 5452, 5484, 5376, 5520, 5418, 5534, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=38072 dup_rows=21978 dup_rows_remote=19232 dedup_ratio=0.335358 router_tokens_per_expert=[277, 240, 279, 263, 300, 259, 242, 282, 236, 250, 258, 236, 246, 247, 240, 236, 247, 257, 288, 271, 282, 259, 271, 241, 277, 257, 269, 258, 268, 238, 271, 248, 275, 280, 237, 279, 267, 255, 248, 247, 249, 268, 236, 261, 253, 232, 265, 286, 250, 250, 236, 253, 255, 254, 252, 278, 262, 283, 256, 260, 267, 269, 263, 258, 259, 268, 265, 285, 252, 226, 255, 271, 268, 253, 224, 241, 228, 277, 273, 271, 258, 238, 245, 263, 257, 248, 263, 252, 256, 246, 263, 259, 251, 254, 246, 237, 260, 273, 246, 257, 228, 239, 247, 284, 268, 248, 236, 250, 263, 242, 270, 262, 242, 270, 224, 238, 267, 252, 256, 240, 246, 254, 250, 231, 234, 249, 271, 242] payload_rows_per_dst=[5470, 5496, 5488, 5450, 5486, 5418, 5434, 5316] dup_rows_per_dst=[2712, 2908, 2788, 2842, 2746, 2654, 2712, 2616] combine_rows_per_src=[5470, 5496, 5488, 5450, 5486, 5418, 5434, 5316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38164 dup_rows=21772 dup_rows_remote=18846 dedup_ratio=0.332214 router_tokens_per_expert=[235, 256, 256, 247, 234, 252, 261, 276, 248, 252, 250, 256, 261, 274, 230, 246, 243, 263, 249, 239, 251, 275, 254, 293, 240, 270, 253, 260, 245, 284, 258, 269, 243, 261, 278, 234, 266, 261, 249, 246, 254, 266, 265, 269, 255, 282, 262, 273, 245, 229, 254, 268, 259, 241, 261, 275, 256, 286, 244, 256, 231, 254, 250, 262, 276, 263, 267, 258, 254, 272, 267, 276, 269, 254, 271, 274, 260, 265, 248, 263, 246, 262, 240, 238, 258, 254, 243, 222, 229, 258, 260, 253, 270, 273, 239, 271, 247, 275, 244, 259, 249, 257, 216, 268, 268, 252, 290, 257, 254, 245, 255, 256, 252, 253, 253, 264, 236, 240, 230, 283, 247, 271, 248, 262, 248, 249, 236, 236] payload_rows_per_dst=[5600, 5610, 5508, 5576, 5446, 5262, 5366, 5396] dup_rows_per_dst=[2926, 2684, 2818, 2994, 2600, 2474, 2656, 2620] combine_rows_per_src=[5600, 5610, 5508, 5576, 5446, 5262, 5366, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=230016 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270656 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311296 weight_movement_bytes=18874368 total_nvlink_bytes=333185664 weight_to_token_ratio=0.060050 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38328 dup_rows=21758 dup_rows_remote=19032 dedup_ratio=0.332001 router_tokens_per_expert=[258, 251, 255, 276, 259, 223, 240, 269, 271, 242, 249, 262, 271, 226, 249, 283, 256, 254, 280, 267, 242, 272, 246, 234, 258, 292, 236, 262, 229, 257, 253, 250, 232, 257, 266, 256, 260, 232, 253, 247, 280, 244, 230, 239, 257, 258, 235, 241, 255, 266, 263, 250, 254, 240, 248, 280, 240, 258, 266, 243, 243, 245, 263, 235, 270, 270, 297, 273, 239, 270, 238, 245, 280, 234, 264, 248, 273, 248, 265, 240, 246, 254, 267, 230, 271, 250, 263, 280, 236, 254, 262, 231, 266, 249, 245, 277, 256, 287, 277, 284, 243, 236, 239, 248, 253, 272, 262, 267, 284, 258, 272, 241, 270, 256, 252, 256, 256, 263, 261, 273, 280, 246, 263, 240, 268, 235, 283, 244] payload_rows_per_dst=[5396, 5450, 5428, 5428, 5538, 5542, 5482, 5514] dup_rows_per_dst=[2772, 2726, 2546, 2670, 2770, 2620, 2876, 2778] combine_rows_per_src=[5396, 5450, 5428, 5428, 5538, 5542, 5482, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230280 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157377416 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314524552 weight_movement_bytes=18874368 total_nvlink_bytes=333398920 weight_to_token_ratio=0.060009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=229216 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156172128 ingress_bytes=193691648 link_bytes=193691648 token_bytes=312115040 weight_movement_bytes=37748736 total_nvlink_bytes=349863776 weight_to_token_ratio=0.120945 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855858069e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228040 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156547784 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312867528 weight_movement_bytes=18874368 total_nvlink_bytes=331741896 weight_to_token_ratio=0.060327 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38020 dup_rows=21924 dup_rows_remote=19052 dedup_ratio=0.334534 router_tokens_per_expert=[261, 214, 245, 232, 269, 264, 255, 248, 251, 252, 232, 236, 266, 278, 224, 261, 247, 244, 254, 237, 254, 243, 293, 253, 263, 264, 250, 249, 266, 280, 256, 258, 273, 203, 249, 274, 246, 228, 251, 234, 269, 253, 258, 284, 231, 281, 240, 275, 237, 249, 248, 248, 256, 258, 285, 278, 268, 254, 252, 264, 238, 261, 241, 247, 264, 283, 255, 223, 276, 222, 260, 260, 257, 277, 242, 249, 263, 268, 222, 266, 267, 256, 261, 290, 239, 262, 276, 254, 262, 249, 237, 270, 257, 259, 256, 250, 287, 258, 265, 279, 249, 253, 252, 252, 280, 263, 262, 270, 264, 260, 244, 249, 255, 272, 246, 288, 245, 257, 253, 269, 268, 231, 263, 238, 269, 276, 250, 237] payload_rows_per_dst=[5188, 5332, 5408, 5548, 5246, 5766, 5532, 5592] dup_rows_per_dst=[2478, 2590, 2710, 2812, 2458, 3044, 2960, 2872] combine_rows_per_src=[5188, 5332, 5408, 5548, 5246, 5766, 5532, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229440 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157220928 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314212416 weight_movement_bytes=56623104 total_nvlink_bytes=370835520 weight_to_token_ratio=0.180206 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38196 dup_rows=21822 dup_rows_remote=19058 dedup_ratio=0.332977 router_tokens_per_expert=[276, 238, 257, 249, 264, 252, 238, 258, 242, 260, 227, 254, 282, 244, 255, 267, 255, 281, 259, 264, 236, 264, 273, 249, 243, 253, 234, 242, 261, 254, 259, 265, 252, 240, 243, 285, 248, 262, 248, 261, 243, 260, 288, 241, 255, 256, 256, 254, 271, 241, 236, 270, 229, 252, 258, 231, 251, 260, 245, 240, 258, 247, 268, 263, 259, 265, 263, 254, 270, 245, 265, 249, 233, 261, 268, 294, 247, 264, 274, 254, 256, 274, 258, 268, 243, 262, 254, 243, 246, 288, 243, 226, 269, 244, 235, 265, 263, 248, 273, 263, 246, 257, 268, 265, 237, 258, 279, 241, 264, 264, 263, 252, 264, 242, 258, 248, 289, 258, 250, 271, 282, 251, 257, 240, 271, 280, 224, 236] payload_rows_per_dst=[5442, 5470, 5474, 5404, 5522, 5446, 5518, 5438] dup_rows_per_dst=[2684, 2714, 2710, 2636, 2808, 2702, 2764, 2804] combine_rows_per_src=[5442, 5470, 5474, 5404, 5522, 5446, 5518, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38318 dup_rows=21798 dup_rows_remote=19196 dedup_ratio=0.332611 router_tokens_per_expert=[257, 256, 279, 252, 270, 268, 260, 237, 285, 262, 256, 251, 280, 260, 282, 266, 252, 261, 260, 243, 251, 253, 247, 260, 249, 262, 236, 265, 243, 257, 283, 272, 244, 233, 256, 243, 235, 284, 250, 280, 248, 228, 241, 274, 268, 230, 276, 246, 260, 280, 257, 243, 288, 260, 257, 251, 248, 268, 263, 247, 251, 251, 249, 243, 245, 258, 232, 265, 258, 271, 284, 283, 241, 238, 281, 238, 258, 261, 253, 259, 223, 258, 276, 226, 254, 238, 254, 250, 246, 240, 263, 253, 252, 273, 246, 259, 255, 267, 265, 247, 250, 241, 270, 254, 243, 282, 212, 242, 245, 255, 263, 238, 276, 249, 276, 269, 251, 242, 259, 251, 278, 269, 229, 279, 255, 235, 246, 272] payload_rows_per_dst=[5604, 5430, 5432, 5504, 5494, 5420, 5368, 5486] dup_rows_per_dst=[2838, 2758, 2640, 2728, 2756, 2602, 2690, 2786] combine_rows_per_src=[5604, 5430, 5432, 5504, 5494, 5420, 5368, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=228288 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155958208 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311688128 weight_movement_bytes=18874368 total_nvlink_bytes=330562496 weight_to_token_ratio=0.060555 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38264 dup_rows=21822 dup_rows_remote=18980 dedup_ratio=0.332977 router_tokens_per_expert=[262, 236, 232, 285, 261, 260, 280, 205, 250, 256, 253, 257, 246, 254, 293, 250, 255, 265, 265, 262, 261, 279, 232, 249, 251, 254, 255, 293, 244, 257, 253, 258, 268, 238, 270, 273, 256, 271, 274, 257, 258, 272, 254, 245, 250, 267, 247, 246, 250, 277, 245, 222, 266, 260, 244, 246, 278, 289, 241, 256, 259, 271, 249, 283, 260, 260, 242, 264, 277, 242, 254, 266, 230, 248, 256, 250, 264, 250, 288, 263, 250, 241, 253, 249, 294, 263, 260, 248, 244, 222, 244, 265, 250, 220, 257, 275, 239, 238, 254, 243, 248, 281, 270, 265, 266, 241, 261, 245, 277, 239, 231, 226, 267, 243, 234, 252, 244, 260, 262, 235, 256, 276, 231, 286, 271, 235, 269, 279] payload_rows_per_dst=[5444, 5484, 5450, 5536, 5462, 5428, 5408, 5502] dup_rows_per_dst=[2716, 2782, 2842, 2736, 2766, 2642, 2640, 2698] combine_rows_per_src=[5444, 5484, 5450, 5536, 5462, 5428, 5408, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38206 dup_rows=21816 dup_rows_remote=18886 dedup_ratio=0.332886 router_tokens_per_expert=[253, 286, 242, 257, 271, 267, 246, 241, 243, 262, 250, 271, 237, 234, 276, 286, 274, 254, 250, 300, 232, 270, 246, 216, 249, 253, 253, 256, 228, 263, 273, 267, 249, 250, 236, 230, 249, 263, 247, 238, 260, 271, 257, 244, 256, 236, 264, 291, 251, 252, 283, 264, 258, 279, 264, 285, 252, 268, 232, 297, 273, 251, 248, 265, 240, 279, 257, 239, 250, 244, 265, 249, 251, 248, 241, 254, 272, 267, 258, 252, 255, 247, 239, 252, 262, 252, 251, 282, 264, 254, 246, 268, 262, 245, 258, 265, 255, 248, 278, 262, 257, 253, 275, 259, 251, 263, 241, 265, 266, 266, 227, 267, 248, 272, 227, 255, 261, 237, 253, 240, 260, 257, 252, 236, 211, 244, 256, 289] payload_rows_per_dst=[5500, 5518, 5470, 5514, 5436, 5464, 5466, 5352] dup_rows_per_dst=[2744, 2650, 2612, 2930, 2696, 2740, 2800, 2644] combine_rows_per_src=[5500, 5518, 5470, 5514, 5436, 5464, 5466, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229016 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156679832 ingress_bytes=213073920 link_bytes=213073920 token_bytes=313130648 weight_movement_bytes=56623104 total_nvlink_bytes=369753752 weight_to_token_ratio=0.180829 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041569463e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=230056 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157180584 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314131112 weight_movement_bytes=18874368 total_nvlink_bytes=333005480 weight_to_token_ratio=0.060084 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38302 dup_rows=21858 dup_rows_remote=19258 dedup_ratio=0.333527 router_tokens_per_expert=[253, 277, 274, 270, 236, 262, 256, 238, 251, 265, 255, 251, 261, 251, 238, 259, 269, 281, 254, 246, 252, 259, 265, 268, 234, 247, 223, 258, 271, 222, 249, 250, 252, 246, 270, 242, 263, 267, 265, 260, 269, 303, 258, 234, 222, 235, 264, 275, 244, 250, 264, 257, 248, 259, 249, 266, 248, 270, 226, 273, 269, 260, 237, 254, 278, 281, 248, 220, 238, 242, 253, 238, 254, 247, 249, 240, 242, 255, 256, 247, 280, 278, 252, 241, 234, 274, 266, 253, 278, 250, 273, 258, 250, 274, 275, 251, 245, 269, 230, 258, 278, 260, 235, 240, 253, 283, 253, 250, 239, 273, 284, 265, 242, 265, 300, 255, 251, 264, 264, 267, 258, 241, 244, 251, 267, 278, 234, 253] payload_rows_per_dst=[5446, 5432, 5518, 5416, 5376, 5590, 5440, 5460] dup_rows_per_dst=[2748, 2664, 2732, 2732, 2600, 2784, 2790, 2808] combine_rows_per_src=[5446, 5432, 5518, 5416, 5376, 5590, 5440, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228976 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958320 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687664 weight_movement_bytes=18874368 total_nvlink_bytes=332562032 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38170 dup_rows=21776 dup_rows_remote=18946 dedup_ratio=0.332275 router_tokens_per_expert=[268, 266, 262, 243, 259, 271, 289, 243, 242, 273, 274, 268, 244, 281, 259, 261, 260, 285, 257, 253, 275, 268, 250, 271, 250, 228, 258, 252, 258, 242, 256, 257, 251, 261, 233, 263, 250, 269, 269, 240, 241, 246, 243, 261, 229, 260, 255, 241, 275, 230, 250, 263, 263, 291, 252, 241, 240, 246, 256, 240, 278, 263, 258, 254, 256, 267, 264, 251, 266, 279, 252, 264, 282, 241, 247, 245, 239, 248, 286, 240, 241, 247, 255, 262, 273, 248, 270, 238, 242, 269, 260, 245, 244, 281, 282, 268, 262, 251, 292, 276, 240, 249, 278, 238, 276, 268, 240, 277, 237, 261, 246, 233, 231, 223, 261, 267, 253, 276, 237, 239, 265, 255, 253, 252, 234, 245, 205, 261] payload_rows_per_dst=[5590, 5490, 5418, 5684, 5258, 5468, 5574, 5278] dup_rows_per_dst=[2830, 2736, 2606, 3020, 2492, 2782, 2958, 2352] combine_rows_per_src=[5590, 5490, 5418, 5684, 5258, 5468, 5574, 5278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38094 dup_rows=21940 dup_rows_remote=19100 dedup_ratio=0.334778 router_tokens_per_expert=[269, 238, 266, 283, 244, 266, 224, 249, 259, 226, 284, 268, 249, 241, 248, 251, 239, 246, 246, 269, 257, 269, 278, 251, 262, 231, 286, 260, 259, 288, 263, 267, 244, 237, 260, 254, 212, 259, 247, 269, 265, 243, 252, 267, 235, 238, 249, 262, 273, 274, 272, 256, 259, 224, 254, 255, 262, 254, 236, 236, 260, 276, 274, 263, 261, 263, 290, 252, 255, 256, 266, 254, 239, 256, 260, 218, 243, 245, 258, 261, 238, 264, 237, 261, 237, 268, 268, 262, 244, 248, 257, 246, 278, 258, 275, 260, 262, 250, 229, 251, 241, 237, 259, 273, 235, 296, 270, 253, 270, 232, 295, 247, 251, 281, 268, 268, 261, 251, 245, 258, 242, 244, 267, 262, 298, 251, 241, 245] payload_rows_per_dst=[5404, 5502, 5470, 5480, 5420, 5384, 5432, 5504] dup_rows_per_dst=[2726, 2840, 2696, 2776, 2734, 2638, 2768, 2762] combine_rows_per_src=[5404, 5502, 5470, 5480, 5420, 5384, 5432, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228368 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156720144 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313211920 weight_movement_bytes=37748736 total_nvlink_bytes=350960656 weight_to_token_ratio=0.120521 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=230240 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157115232 ingress_bytes=175759360 link_bytes=175759360 token_bytes=314000224 weight_movement_bytes=18874368 total_nvlink_bytes=332874592 weight_to_token_ratio=0.060109 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38344 dup_rows=21730 dup_rows_remote=19004 dedup_ratio=0.331573 router_tokens_per_expert=[247, 259, 270, 245, 272, 265, 254, 267, 240, 242, 251, 267, 247, 261, 244, 235, 247, 286, 254, 254, 250, 251, 261, 243, 286, 259, 227, 252, 244, 241, 249, 248, 257, 246, 253, 232, 274, 256, 267, 246, 252, 253, 239, 255, 260, 265, 250, 271, 243, 234, 255, 271, 251, 255, 275, 262, 259, 236, 259, 240, 247, 234, 256, 251, 266, 257, 283, 254, 272, 254, 241, 254, 249, 238, 264, 248, 251, 264, 282, 278, 297, 275, 263, 271, 263, 263, 233, 272, 260, 262, 242, 228, 261, 261, 269, 274, 283, 228, 273, 261, 262, 261, 239, 262, 254, 250, 262, 243, 288, 234, 236, 267, 259, 274, 267, 266, 252, 234, 269, 251, 250, 271, 266, 254, 237, 240, 243, 261] payload_rows_per_dst=[5428, 5432, 5482, 5290, 5632, 5562, 5518, 5462] dup_rows_per_dst=[2704, 2672, 2670, 2414, 3030, 2826, 2688, 2726] combine_rows_per_src=[5428, 5432, 5482, 5290, 5632, 5562, 5518, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228464 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156572784 ingress_bytes=212967424 link_bytes=212967424 token_bytes=312917104 weight_movement_bytes=56623104 total_nvlink_bytes=369540208 weight_to_token_ratio=0.180952 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040549070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=228776 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156261800 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312294824 weight_movement_bytes=18874368 total_nvlink_bytes=331169192 weight_to_token_ratio=0.060438 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38326 dup_rows=21750 dup_rows_remote=19032 dedup_ratio=0.331879 router_tokens_per_expert=[263, 259, 254, 261, 260, 241, 271, 250, 257, 280, 263, 271, 273, 260, 236, 256, 266, 241, 252, 253, 248, 235, 234, 250, 242, 284, 250, 235, 256, 246, 260, 264, 259, 280, 258, 272, 243, 261, 228, 279, 249, 276, 253, 251, 252, 234, 258, 257, 239, 284, 245, 282, 248, 236, 266, 243, 222, 239, 268, 238, 258, 280, 257, 275, 259, 261, 242, 263, 253, 255, 241, 272, 252, 263, 264, 240, 266, 255, 244, 270, 258, 269, 249, 275, 273, 265, 255, 276, 255, 257, 247, 252, 257, 249, 260, 240, 255, 264, 264, 257, 263, 253, 238, 250, 245, 234, 251, 257, 279, 257, 259, 263, 222, 256, 254, 241, 269, 257, 264, 247, 282, 249, 238, 267, 255, 249, 263, 268] payload_rows_per_dst=[5512, 5368, 5528, 5502, 5460, 5516, 5460, 5440] dup_rows_per_dst=[2798, 2664, 2692, 2658, 2736, 2762, 2718, 2722] combine_rows_per_src=[5512, 5368, 5528, 5502, 5460, 5516, 5460, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38164 dup_rows=21912 dup_rows_remote=19052 dedup_ratio=0.334351 router_tokens_per_expert=[272, 240, 264, 271, 234, 239, 282, 249, 271, 253, 270, 245, 239, 242, 265, 246, 254, 292, 231, 221, 241, 276, 273, 248, 242, 259, 271, 260, 227, 252, 243, 248, 274, 236, 260, 251, 250, 268, 242, 270, 272, 242, 267, 233, 257, 251, 237, 296, 237, 244, 269, 247, 254, 263, 260, 241, 248, 286, 267, 264, 243, 243, 245, 272, 259, 228, 256, 267, 268, 271, 234, 241, 263, 248, 248, 242, 247, 216, 253, 269, 268, 258, 282, 238, 241, 242, 238, 295, 269, 285, 269, 257, 249, 268, 248, 253, 252, 286, 265, 228, 256, 269, 270, 281, 256, 273, 255, 247, 280, 243, 243, 273, 252, 274, 245, 258, 255, 248, 281, 262, 261, 265, 243, 246, 263, 270, 240, 249] payload_rows_per_dst=[5432, 5410, 5482, 5484, 5386, 5460, 5488, 5482] dup_rows_per_dst=[2732, 2666, 2730, 2682, 2634, 2860, 2866, 2742] combine_rows_per_src=[5432, 5410, 5482, 5484, 5386, 5460, 5488, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229392 combine_payload_bytes=157057024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157286416 ingress_bytes=232554496 link_bytes=232554496 token_bytes=314343440 weight_movement_bytes=75497472 total_nvlink_bytes=389840912 weight_to_token_ratio=0.240175 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228222757e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38374 dup_rows=21674 dup_rows_remote=19034 dedup_ratio=0.330719 router_tokens_per_expert=[222, 267, 260, 290, 257, 248, 260, 238, 226, 251, 277, 245, 277, 286, 244, 243, 290, 269, 262, 250, 247, 254, 228, 228, 272, 251, 246, 247, 256, 256, 223, 250, 269, 259, 255, 296, 247, 271, 260, 238, 250, 236, 263, 234, 240, 243, 262, 241, 260, 255, 260, 241, 266, 270, 278, 272, 254, 252, 256, 267, 246, 265, 271, 253, 280, 216, 276, 259, 257, 230, 282, 283, 249, 247, 277, 245, 266, 241, 266, 242, 277, 236, 261, 249, 248, 293, 276, 269, 226, 249, 274, 256, 255, 245, 250, 258, 256, 234, 250, 269, 279, 254, 227, 252, 235, 247, 250, 270, 243, 270, 268, 226, 266, 249, 252, 258, 276, 254, 242, 268, 254, 265, 267, 232, 262, 268, 286, 251] payload_rows_per_dst=[5454, 5430, 5488, 5480, 5534, 5538, 5456, 5482] dup_rows_per_dst=[2728, 2628, 2640, 2852, 2698, 2706, 2604, 2818] combine_rows_per_src=[5454, 5430, 5488, 5480, 5534, 5538, 5456, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38408 dup_rows=21642 dup_rows_remote=19022 dedup_ratio=0.330231 router_tokens_per_expert=[264, 258, 242, 251, 263, 267, 261, 259, 252, 269, 248, 269, 239, 252, 267, 247, 267, 256, 241, 229, 297, 258, 266, 240, 232, 246, 251, 256, 266, 259, 252, 250, 247, 241, 254, 272, 269, 291, 268, 260, 251, 245, 259, 283, 267, 258, 262, 254, 253, 248, 238, 254, 275, 273, 260, 249, 246, 266, 259, 251, 262, 243, 221, 255, 252, 274, 269, 268, 219, 239, 257, 250, 240, 222, 253, 256, 257, 253, 222, 257, 282, 273, 274, 264, 270, 228, 262, 252, 265, 263, 240, 254, 290, 250, 276, 259, 252, 277, 251, 289, 268, 255, 255, 244, 266, 266, 286, 271, 222, 244, 253, 237, 262, 252, 267, 254, 248, 280, 240, 232, 238, 255, 244, 260, 266, 256, 251, 229] payload_rows_per_dst=[5492, 5430, 5500, 5486, 5394, 5602, 5578, 5412] dup_rows_per_dst=[2724, 2702, 2862, 2620, 2582, 2802, 2694, 2656] combine_rows_per_src=[5492, 5430, 5500, 5486, 5394, 5602, 5578, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229432 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157212728 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196024 weight_movement_bytes=18874368 total_nvlink_bytes=333070392 weight_to_token_ratio=0.060072 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38216 dup_rows=21914 dup_rows_remote=19220 dedup_ratio=0.334381 router_tokens_per_expert=[242, 244, 260, 267, 264, 256, 269, 256, 275, 239, 274, 242, 218, 241, 250, 252, 236, 264, 272, 244, 258, 250, 269, 248, 239, 274, 263, 246, 267, 275, 270, 294, 240, 242, 263, 257, 224, 295, 257, 254, 248, 265, 269, 248, 235, 275, 251, 263, 263, 248, 253, 229, 300, 268, 240, 253, 231, 237, 289, 264, 251, 272, 247, 219, 253, 235, 271, 238, 227, 229, 275, 252, 257, 277, 242, 254, 283, 244, 229, 284, 233, 266, 256, 273, 237, 262, 282, 260, 270, 258, 269, 255, 277, 239, 263, 262, 273, 255, 242, 260, 263, 234, 243, 269, 242, 276, 257, 269, 261, 235, 255, 277, 280, 250, 256, 231, 242, 260, 293, 247, 259, 244, 240, 267, 252, 247, 236, 273] payload_rows_per_dst=[5440, 5496, 5404, 5444, 5406, 5512, 5482, 5438] dup_rows_per_dst=[2658, 2842, 2768, 2684, 2694, 2812, 2740, 2716] combine_rows_per_src=[5440, 5496, 5404, 5444, 5406, 5512, 5482, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228864 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156548608 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868352 weight_movement_bytes=37748736 total_nvlink_bytes=350617088 weight_to_token_ratio=0.120654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229632 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409536 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589440 weight_movement_bytes=18874368 total_nvlink_bytes=333463808 weight_to_token_ratio=0.059997 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38236 dup_rows=21912 dup_rows_remote=19162 dedup_ratio=0.334351 router_tokens_per_expert=[249, 235, 269, 245, 257, 240, 249, 257, 255, 251, 284, 265, 254, 253, 247, 245, 251, 265, 262, 285, 272, 284, 293, 251, 237, 228, 271, 240, 274, 242, 251, 252, 263, 240, 268, 246, 256, 244, 244, 257, 267, 250, 260, 251, 250, 289, 246, 244, 230, 270, 248, 244, 280, 264, 250, 275, 261, 225, 270, 266, 241, 247, 240, 259, 275, 264, 263, 231, 267, 219, 293, 265, 235, 221, 237, 231, 272, 292, 245, 262, 268, 249, 256, 267, 276, 257, 247, 229, 241, 267, 253, 258, 252, 236, 264, 263, 267, 254, 251, 268, 236, 260, 288, 234, 264, 276, 251, 263, 269, 233, 282, 254, 273, 251, 248, 278, 282, 244, 266, 270, 244, 267, 267, 246, 241, 248, 247, 233] payload_rows_per_dst=[5388, 5624, 5452, 5440, 5588, 5416, 5434, 5282] dup_rows_per_dst=[2750, 2926, 2752, 2648, 2968, 2720, 2632, 2516] combine_rows_per_src=[5388, 5624, 5452, 5440, 5588, 5416, 5434, 5282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38302 dup_rows=21846 dup_rows_remote=19256 dedup_ratio=0.333344 router_tokens_per_expert=[242, 235, 275, 286, 236, 243, 261, 265, 255, 250, 245, 241, 271, 278, 285, 298, 256, 240, 223, 261, 253, 243, 231, 262, 251, 252, 282, 257, 248, 246, 247, 237, 263, 240, 248, 240, 265, 256, 249, 284, 252, 255, 252, 228, 266, 245, 221, 260, 241, 236, 243, 267, 278, 236, 279, 269, 248, 285, 260, 225, 214, 286, 230, 259, 258, 248, 254, 254, 257, 253, 249, 268, 273, 270, 282, 271, 269, 236, 260, 246, 256, 262, 268, 260, 259, 266, 289, 247, 255, 251, 256, 270, 262, 263, 238, 255, 265, 261, 263, 265, 260, 242, 234, 244, 253, 242, 265, 241, 269, 294, 267, 260, 278, 233, 241, 252, 263, 257, 257, 265, 237, 252, 264, 255, 285, 255, 263, 246] payload_rows_per_dst=[5552, 5388, 5428, 5426, 5480, 5492, 5460, 5464] dup_rows_per_dst=[2780, 2590, 2620, 2686, 2816, 2822, 2790, 2742] combine_rows_per_src=[5552, 5388, 5428, 5426, 5480, 5492, 5460, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229720 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157548888 ingress_bytes=176193536 link_bytes=176193536 token_bytes=314868056 weight_movement_bytes=18874368 total_nvlink_bytes=333742424 weight_to_token_ratio=0.059944 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688199769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229744 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762480 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295216 weight_movement_bytes=18874368 total_nvlink_bytes=332169584 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38258 dup_rows=21754 dup_rows_remote=18906 dedup_ratio=0.331940 router_tokens_per_expert=[275, 263, 250, 233, 252, 259, 260, 253, 288, 278, 256, 242, 240, 267, 246, 251, 262, 250, 260, 243, 260, 229, 248, 227, 252, 269, 240, 237, 249, 236, 246, 262, 276, 265, 257, 280, 265, 271, 281, 266, 275, 254, 249, 262, 232, 276, 273, 261, 250, 231, 234, 232, 258, 263, 248, 255, 251, 239, 267, 238, 261, 288, 247, 270, 252, 273, 264, 252, 262, 264, 256, 261, 239, 253, 248, 237, 244, 273, 262, 258, 230, 250, 261, 247, 238, 235, 240, 262, 259, 278, 224, 263, 234, 243, 273, 268, 273, 245, 253, 231, 258, 243, 276, 273, 263, 213, 248, 265, 291, 261, 287, 246, 254, 262, 256, 299, 287, 256, 271, 246, 225, 259, 303, 271, 226, 243, 270, 253] payload_rows_per_dst=[5348, 5352, 5598, 5480, 5468, 5476, 5536, 5524] dup_rows_per_dst=[2680, 2588, 2836, 2626, 2728, 2732, 2716, 2848] combine_rows_per_src=[5348, 5352, 5598, 5480, 5468, 5476, 5536, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229592 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156844248 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313458904 weight_movement_bytes=18874368 total_nvlink_bytes=332333272 weight_to_token_ratio=0.060213 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=230232 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157115224 ingress_bytes=175759360 link_bytes=175759360 token_bytes=314000216 weight_movement_bytes=18874368 total_nvlink_bytes=332874584 weight_to_token_ratio=0.060109 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38198 dup_rows=21982 dup_rows_remote=19204 dedup_ratio=0.335419 router_tokens_per_expert=[273, 258, 259, 250, 249, 257, 280, 245, 268, 238, 260, 238, 253, 236, 260, 247, 246, 254, 245, 216, 263, 261, 269, 279, 264, 271, 241, 248, 276, 245, 288, 258, 252, 253, 270, 275, 267, 246, 268, 264, 266, 213, 248, 253, 261, 248, 227, 285, 266, 239, 250, 260, 258, 238, 258, 260, 243, 270, 245, 278, 260, 239, 252, 258, 268, 253, 253, 248, 271, 275, 244, 256, 262, 240, 236, 241, 263, 242, 297, 241, 298, 254, 258, 273, 250, 234, 247, 271, 249, 258, 257, 254, 248, 263, 218, 275, 245, 254, 243, 236, 245, 246, 252, 279, 263, 265, 289, 264, 275, 218, 212, 281, 236, 278, 257, 248, 263, 268, 258, 261, 243, 235, 271, 247, 278, 263, 280, 253] payload_rows_per_dst=[5432, 5514, 5444, 5404, 5524, 5428, 5356, 5452] dup_rows_per_dst=[2710, 2734, 2748, 2744, 2656, 2786, 2778, 2826] combine_rows_per_src=[5432, 5514, 5444, 5404, 5524, 5428, 5356, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228656 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156933424 ingress_bytes=213327872 link_bytes=213327872 token_bytes=313638192 weight_movement_bytes=56623104 total_nvlink_bytes=370261296 weight_to_token_ratio=0.180536 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044002706e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38556 dup_rows=21612 dup_rows_remote=19002 dedup_ratio=0.329773 router_tokens_per_expert=[263, 272, 256, 261, 251, 254, 269, 248, 251, 263, 255, 275, 264, 262, 259, 270, 265, 250, 250, 241, 231, 257, 268, 280, 242, 258, 272, 252, 257, 269, 261, 272, 251, 262, 208, 228, 266, 253, 274, 259, 285, 235, 250, 245, 243, 231, 274, 225, 245, 258, 283, 236, 255, 236, 249, 271, 283, 268, 223, 275, 260, 269, 259, 271, 256, 237, 253, 262, 281, 231, 247, 251, 225, 239, 274, 256, 251, 247, 250, 271, 251, 243, 264, 262, 263, 246, 227, 266, 252, 251, 242, 245, 251, 252, 265, 252, 279, 233, 235, 254, 246, 260, 258, 252, 265, 273, 251, 236, 252, 268, 260, 265, 330, 277, 246, 250, 253, 250, 255, 271, 255, 247, 239, 281, 259, 254, 268, 255] payload_rows_per_dst=[5516, 5450, 5368, 5582, 5406, 5476, 5532, 5594] dup_rows_per_dst=[2830, 2800, 2610, 2700, 2656, 2588, 2642, 2786] combine_rows_per_src=[5516, 5450, 5368, 5582, 5406, 5476, 5532, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38244 dup_rows=21806 dup_rows_remote=19064 dedup_ratio=0.332733 router_tokens_per_expert=[242, 277, 254, 269, 227, 251, 239, 260, 240, 261, 259, 267, 231, 274, 255, 254, 259, 264, 248, 261, 256, 259, 288, 250, 272, 250, 245, 254, 290, 230, 291, 277, 228, 243, 251, 240, 263, 239, 253, 284, 250, 264, 260, 266, 272, 259, 264, 244, 275, 243, 264, 268, 271, 261, 273, 252, 263, 251, 277, 240, 260, 277, 225, 269, 240, 245, 270, 230, 242, 285, 229, 219, 242, 248, 272, 266, 256, 268, 251, 261, 263, 245, 268, 261, 253, 227, 264, 275, 241, 258, 243, 261, 275, 278, 272, 230, 244, 215, 251, 242, 265, 259, 247, 228, 280, 266, 252, 254, 273, 264, 268, 271, 251, 244, 235, 246, 252, 278, 253, 273, 238, 240, 256, 269, 261, 249, 267, 236] payload_rows_per_dst=[5424, 5558, 5412, 5506, 5446, 5486, 5504, 5394] dup_rows_per_dst=[2696, 2830, 2748, 2832, 2602, 2742, 2654, 2702] combine_rows_per_src=[5424, 5558, 5412, 5506, 5446, 5486, 5504, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43968 payload_rows_remote=38394 dup_rows=21568 dup_rows_remote=18930 dedup_ratio=0.329102 router_tokens_per_expert=[267, 233, 257, 255, 269, 268, 293, 252, 279, 236, 269, 255, 244, 272, 251, 198, 251, 276, 274, 245, 265, 252, 264, 237, 272, 257, 255, 244, 292, 269, 234, 223, 265, 279, 266, 286, 243, 249, 269, 220, 276, 244, 256, 238, 254, 289, 264, 256, 247, 257, 248, 251, 269, 229, 250, 263, 266, 264, 254, 299, 249, 243, 249, 268, 254, 210, 265, 258, 264, 236, 244, 272, 247, 259, 255, 245, 231, 243, 279, 248, 277, 278, 249, 240, 250, 263, 300, 240, 254, 274, 273, 244, 255, 276, 247, 258, 272, 251, 250, 280, 268, 252, 248, 245, 225, 260, 251, 271, 239, 255, 258, 250, 249, 259, 240, 255, 259, 238, 255, 240, 262, 257, 262, 278, 234, 241, 250, 258] payload_rows_per_dst=[5456, 5480, 5500, 5574, 5458, 5530, 5498, 5472] dup_rows_per_dst=[2740, 2740, 2808, 2638, 2562, 2826, 2652, 2602] combine_rows_per_src=[5456, 5480, 5500, 5574, 5458, 5530, 5498, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229608 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156688616 ingress_bytes=194207744 link_bytes=194207744 token_bytes=313147624 weight_movement_bytes=37748736 total_nvlink_bytes=350896360 weight_to_token_ratio=0.120546 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860803047e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38392 dup_rows=21708 dup_rows_remote=19102 dedup_ratio=0.331238 router_tokens_per_expert=[242, 235, 257, 253, 237, 249, 244, 274, 263, 246, 242, 248, 237, 259, 290, 226, 253, 241, 244, 249, 251, 276, 265, 277, 247, 240, 269, 232, 260, 268, 277, 283, 254, 251, 237, 268, 276, 251, 274, 267, 253, 274, 246, 220, 284, 243, 268, 257, 274, 255, 266, 257, 266, 224, 255, 251, 259, 250, 296, 252, 253, 290, 237, 241, 240, 270, 263, 241, 258, 231, 259, 269, 240, 256, 231, 253, 253, 240, 263, 254, 300, 271, 283, 249, 252, 228, 247, 239, 263, 218, 264, 252, 247, 258, 261, 246, 255, 279, 258, 272, 272, 249, 237, 244, 258, 241, 257, 234, 264, 256, 262, 269, 254, 250, 242, 280, 249, 260, 273, 266, 256, 266, 258, 254, 245, 292, 269, 265] payload_rows_per_dst=[5400, 5582, 5482, 5456, 5436, 5494, 5452, 5526] dup_rows_per_dst=[2604, 2682, 2764, 2796, 2606, 2662, 2762, 2832] combine_rows_per_src=[5400, 5582, 5482, 5456, 5436, 5494, 5452, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157925376 dispatch_meta_bytes=230232 combine_payload_bytes=157925376 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=158155608 ingress_bytes=214548480 link_bytes=214548480 token_bytes=316080984 weight_movement_bytes=56623104 total_nvlink_bytes=372704088 weight_to_token_ratio=0.179141 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.055697971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229232 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876656 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524080 weight_movement_bytes=18874368 total_nvlink_bytes=332398448 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38268 dup_rows=21836 dup_rows_remote=18970 dedup_ratio=0.333191 router_tokens_per_expert=[248, 238, 223, 258, 280, 245, 264, 249, 277, 244, 289, 265, 256, 288, 287, 238, 292, 250, 289, 262, 263, 250, 286, 274, 251, 243, 268, 248, 243, 238, 248, 266, 271, 282, 254, 245, 261, 255, 228, 268, 244, 252, 278, 268, 272, 296, 266, 241, 258, 243, 264, 253, 241, 255, 231, 253, 265, 244, 261, 259, 277, 234, 263, 277, 285, 251, 253, 274, 263, 266, 252, 259, 238, 284, 237, 253, 277, 254, 266, 259, 237, 269, 215, 242, 249, 227, 255, 236, 246, 284, 253, 255, 284, 274, 247, 253, 276, 280, 234, 268, 215, 280, 242, 232, 256, 239, 242, 239, 235, 250, 253, 260, 256, 253, 237, 254, 263, 239, 233, 253, 243, 267, 270, 224, 286, 226, 257, 230] payload_rows_per_dst=[5432, 5526, 5564, 5466, 5540, 5414, 5386, 5372] dup_rows_per_dst=[2866, 2816, 2798, 2690, 2802, 2638, 2616, 2610] combine_rows_per_src=[5432, 5526, 5564, 5466, 5540, 5414, 5386, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229296 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491120 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314752944 weight_movement_bytes=18874368 total_nvlink_bytes=333627312 weight_to_token_ratio=0.059966 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38184 dup_rows=21826 dup_rows_remote=19080 dedup_ratio=0.333038 router_tokens_per_expert=[277, 265, 259, 260, 263, 238, 264, 223, 236, 258, 263, 256, 243, 253, 277, 263, 279, 245, 248, 259, 288, 255, 253, 265, 246, 274, 244, 261, 258, 240, 257, 264, 289, 260, 239, 263, 252, 254, 241, 239, 268, 281, 254, 275, 264, 279, 281, 251, 228, 271, 266, 231, 268, 252, 238, 256, 234, 265, 266, 255, 259, 244, 266, 252, 266, 247, 215, 251, 234, 248, 229, 268, 252, 288, 264, 249, 254, 273, 249, 278, 241, 256, 246, 279, 260, 247, 239, 247, 225, 257, 259, 243, 268, 293, 242, 292, 242, 253, 250, 243, 278, 256, 267, 239, 273, 246, 269, 248, 263, 256, 248, 269, 237, 235, 253, 269, 261, 241, 259, 229, 234, 274, 245, 231, 285, 264, 241, 276] payload_rows_per_dst=[5488, 5526, 5514, 5410, 5510, 5432, 5418, 5412] dup_rows_per_dst=[2708, 2746, 2866, 2692, 2620, 2756, 2782, 2656] combine_rows_per_src=[5488, 5526, 5514, 5410, 5510, 5432, 5418, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38150 dup_rows=21820 dup_rows_remote=18758 dedup_ratio=0.332947 router_tokens_per_expert=[264, 245, 236, 257, 242, 241, 233, 262, 243, 249, 241, 254, 255, 219, 247, 277, 244, 271, 258, 257, 276, 255, 250, 222, 262, 265, 252, 234, 267, 252, 265, 271, 249, 254, 260, 276, 246, 268, 259, 278, 252, 286, 240, 273, 252, 279, 245, 258, 276, 270, 267, 239, 244, 259, 280, 250, 238, 277, 244, 262, 233, 241, 237, 246, 234, 251, 258, 238, 273, 278, 264, 259, 258, 257, 255, 267, 264, 254, 243, 228, 277, 264, 271, 263, 263, 267, 243, 262, 266, 256, 258, 246, 264, 270, 249, 262, 264, 258, 261, 249, 268, 237, 272, 256, 245, 268, 264, 286, 254, 253, 262, 259, 279, 265, 253, 245, 275, 256, 227, 259, 241, 218, 254, 281, 240, 236, 266, 251] payload_rows_per_dst=[5362, 5402, 5508, 5556, 5410, 5536, 5376, 5566] dup_rows_per_dst=[2520, 2556, 2626, 2858, 2752, 2860, 2586, 3062] combine_rows_per_src=[5362, 5402, 5508, 5556, 5410, 5536, 5376, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229976 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157483608 ingress_bytes=176128000 link_bytes=176128000 token_bytes=314737240 weight_movement_bytes=18874368 total_nvlink_bytes=333611608 weight_to_token_ratio=0.059969 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687571836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228952 combine_payload_bytes=156745728 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156974680 ingress_bytes=232243200 link_bytes=232243200 token_bytes=313720408 weight_movement_bytes=75497472 total_nvlink_bytes=389217880 weight_to_token_ratio=0.240652 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225240072e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229056 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156630720 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032384 weight_movement_bytes=37748736 total_nvlink_bytes=350781120 weight_to_token_ratio=0.120591 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38282 dup_rows=21768 dup_rows_remote=19014 dedup_ratio=0.332153 router_tokens_per_expert=[271, 239, 248, 266, 253, 247, 242, 249, 265, 251, 257, 254, 273, 259, 257, 258, 260, 267, 244, 253, 239, 265, 242, 284, 251, 262, 245, 275, 263, 267, 280, 250, 261, 263, 254, 248, 272, 250, 275, 265, 253, 244, 250, 261, 230, 255, 251, 237, 250, 256, 265, 295, 253, 267, 244, 258, 276, 245, 250, 239, 264, 247, 271, 271, 262, 258, 258, 281, 244, 300, 222, 225, 263, 250, 248, 274, 265, 266, 264, 232, 250, 258, 271, 258, 261, 229, 228, 254, 233, 262, 229, 258, 250, 270, 244, 257, 245, 276, 280, 250, 252, 243, 268, 239, 272, 246, 247, 223, 269, 274, 271, 265, 272, 240, 251, 282, 267, 227, 247, 268, 249, 244, 240, 244, 256, 225, 291, 265] payload_rows_per_dst=[5470, 5486, 5480, 5554, 5508, 5378, 5486, 5406] dup_rows_per_dst=[2708, 2808, 2658, 2748, 2716, 2646, 2754, 2730] combine_rows_per_src=[5470, 5486, 5480, 5554, 5508, 5378, 5486, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=227632 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156490032 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312752432 weight_movement_bytes=18874368 total_nvlink_bytes=331626800 weight_to_token_ratio=0.060349 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38170 dup_rows=21944 dup_rows_remote=19260 dedup_ratio=0.334839 router_tokens_per_expert=[260, 257, 245, 257, 274, 242, 277, 260, 266, 268, 250, 230, 244, 261, 252, 255, 286, 252, 242, 240, 245, 227, 259, 248, 263, 235, 252, 280, 274, 269, 248, 285, 274, 250, 222, 273, 256, 259, 258, 264, 250, 236, 246, 244, 265, 255, 254, 247, 255, 242, 259, 261, 260, 272, 264, 225, 259, 260, 252, 256, 250, 236, 248, 262, 249, 286, 270, 290, 258, 240, 258, 301, 258, 235, 248, 279, 249, 251, 267, 260, 244, 261, 270, 245, 268, 251, 247, 252, 247, 258, 253, 267, 266, 274, 243, 256, 254, 257, 259, 235, 254, 262, 271, 259, 256, 254, 249, 250, 277, 243, 265, 260, 254, 266, 251, 246, 229, 271, 242, 254, 254, 249, 248, 269, 262, 259, 261, 230] payload_rows_per_dst=[5440, 5438, 5422, 5404, 5558, 5434, 5502, 5394] dup_rows_per_dst=[2756, 2772, 2684, 2718, 2840, 2770, 2708, 2696] combine_rows_per_src=[5440, 5438, 5422, 5404, 5558, 5434, 5502, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38414 dup_rows=21666 dup_rows_remote=19008 dedup_ratio=0.330597 router_tokens_per_expert=[268, 242, 277, 312, 268, 276, 272, 288, 257, 259, 263, 269, 268, 245, 263, 264, 238, 246, 257, 256, 278, 276, 258, 276, 236, 242, 259, 286, 292, 258, 254, 238, 235, 257, 240, 269, 255, 261, 262, 254, 259, 262, 242, 255, 255, 244, 257, 253, 257, 241, 248, 239, 249, 234, 247, 265, 252, 262, 254, 260, 229, 255, 228, 248, 238, 251, 283, 272, 245, 250, 247, 236, 220, 258, 279, 237, 233, 246, 244, 261, 248, 261, 243, 245, 255, 257, 218, 255, 250, 289, 262, 272, 251, 254, 251, 246, 251, 254, 279, 266, 251, 270, 258, 271, 259, 281, 248, 261, 224, 264, 255, 243, 248, 273, 236, 279, 261, 236, 261, 268, 269, 251, 245, 266, 241, 269, 235, 269] payload_rows_per_dst=[5658, 5498, 5472, 5414, 5384, 5456, 5494, 5494] dup_rows_per_dst=[2924, 2802, 2648, 2522, 2616, 2658, 2776, 2720] combine_rows_per_src=[5658, 5498, 5472, 5414, 5384, 5456, 5494, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38294 dup_rows=21830 dup_rows_remote=19052 dedup_ratio=0.333099 router_tokens_per_expert=[270, 252, 279, 266, 239, 238, 282, 295, 249, 262, 264, 245, 277, 279, 283, 267, 262, 257, 254, 270, 255, 222, 254, 241, 243, 233, 243, 241, 228, 246, 237, 282, 250, 283, 251, 271, 266, 251, 264, 263, 283, 240, 248, 256, 286, 269, 268, 257, 259, 255, 294, 258, 263, 277, 269, 249, 248, 245, 240, 274, 244, 249, 229, 242, 243, 270, 239, 268, 256, 231, 244, 262, 246, 251, 266, 242, 254, 240, 261, 252, 257, 260, 246, 255, 265, 239, 263, 280, 279, 241, 243, 238, 257, 254, 222, 265, 242, 265, 247, 239, 268, 256, 256, 258, 248, 243, 233, 252, 261, 230, 243, 239, 283, 255, 253, 260, 278, 228, 279, 262, 282, 263, 285, 243, 239, 269, 237, 267] payload_rows_per_dst=[5574, 5360, 5574, 5412, 5406, 5488, 5386, 5506] dup_rows_per_dst=[2920, 2576, 2838, 2778, 2644, 2640, 2574, 2860] combine_rows_per_src=[5574, 5360, 5574, 5412, 5406, 5488, 5386, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229184 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032256 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835328 weight_movement_bytes=37748736 total_nvlink_bytes=351584064 weight_to_token_ratio=0.120282 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229720 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156574040 ingress_bytes=212967424 link_bytes=212967424 token_bytes=312918360 weight_movement_bytes=56623104 total_nvlink_bytes=369541464 weight_to_token_ratio=0.180952 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040549070e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229688 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157573432 ingress_bytes=195092480 link_bytes=195092480 token_bytes=314917176 weight_movement_bytes=37748736 total_nvlink_bytes=352665912 weight_to_token_ratio=0.119869 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869280152e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38324 dup_rows=21768 dup_rows_remote=19082 dedup_ratio=0.332153 router_tokens_per_expert=[256, 254, 252, 295, 271, 274, 262, 267, 247, 240, 233, 255, 263, 262, 262, 240, 238, 262, 252, 277, 253, 283, 261, 271, 289, 242, 242, 248, 237, 218, 258, 248, 246, 251, 237, 263, 250, 256, 241, 286, 245, 258, 263, 292, 257, 255, 268, 255, 267, 257, 243, 274, 237, 259, 248, 269, 244, 241, 264, 240, 238, 237, 236, 262, 248, 258, 273, 267, 237, 238, 272, 245, 250, 241, 241, 277, 258, 241, 270, 249, 257, 269, 288, 247, 252, 220, 251, 237, 241, 251, 234, 258, 254, 258, 259, 253, 236, 271, 266, 301, 269, 291, 247, 257, 282, 242, 244, 259, 215, 254, 266, 267, 257, 240, 250, 282, 245, 278, 259, 290, 270, 237, 253, 260, 245, 239, 263, 288] payload_rows_per_dst=[5546, 5478, 5456, 5368, 5444, 5424, 5572, 5480] dup_rows_per_dst=[2720, 2680, 2790, 2664, 2686, 2634, 2762, 2832] combine_rows_per_src=[5546, 5478, 5456, 5368, 5444, 5424, 5572, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38262 dup_rows=21718 dup_rows_remote=18926 dedup_ratio=0.331390 router_tokens_per_expert=[263, 248, 276, 272, 249, 269, 245, 261, 269, 288, 258, 254, 246, 231, 253, 271, 259, 253, 263, 242, 250, 242, 255, 251, 265, 249, 239, 239, 272, 259, 243, 242, 247, 227, 252, 225, 243, 249, 266, 239, 251, 254, 250, 237, 245, 245, 230, 251, 243, 268, 275, 257, 252, 234, 252, 303, 264, 278, 247, 259, 253, 232, 280, 247, 251, 277, 264, 244, 272, 237, 231, 240, 251, 251, 253, 277, 263, 256, 279, 244, 266, 261, 264, 247, 283, 256, 229, 262, 220, 248, 271, 263, 242, 261, 260, 260, 259, 272, 257, 282, 250, 261, 268, 239, 235, 266, 259, 260, 275, 274, 288, 264, 247, 264, 249, 279, 256, 252, 263, 240, 282, 256, 274, 268, 253, 259, 226, 277] payload_rows_per_dst=[5556, 5596, 5586, 5506, 5442, 5390, 5392, 5350] dup_rows_per_dst=[2792, 2848, 2768, 2832, 2738, 2380, 2694, 2666] combine_rows_per_src=[5556, 5596, 5586, 5506, 5442, 5390, 5392, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229384 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081608 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933832 weight_movement_bytes=37748736 total_nvlink_bytes=351682568 weight_to_token_ratio=0.120244 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38340 dup_rows=21712 dup_rows_remote=18956 dedup_ratio=0.331299 router_tokens_per_expert=[278, 238, 242, 248, 238, 268, 225, 259, 248, 246, 228, 238, 244, 241, 246, 261, 243, 226, 242, 244, 254, 250, 253, 265, 258, 264, 253, 280, 287, 272, 271, 258, 246, 267, 271, 237, 245, 280, 260, 247, 245, 247, 256, 251, 232, 243, 247, 258, 262, 243, 295, 241, 230, 275, 229, 261, 262, 233, 265, 272, 239, 239, 266, 269, 268, 264, 282, 261, 282, 240, 249, 288, 249, 244, 275, 264, 225, 265, 244, 252, 287, 275, 258, 258, 265, 286, 263, 242, 273, 254, 282, 210, 248, 281, 252, 262, 257, 248, 279, 241, 272, 252, 240, 269, 269, 257, 264, 272, 247, 262, 232, 247, 278, 257, 271, 252, 242, 284, 249, 267, 234, 276, 230, 261, 257, 260, 261, 252] payload_rows_per_dst=[5346, 5484, 5424, 5522, 5532, 5554, 5508, 5454] dup_rows_per_dst=[2550, 2756, 2640, 2640, 2772, 2838, 2708, 2808] combine_rows_per_src=[5346, 5484, 5424, 5522, 5532, 5554, 5508, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38184 dup_rows=21876 dup_rows_remote=19198 dedup_ratio=0.333801 router_tokens_per_expert=[259, 253, 284, 253, 239, 260, 264, 270, 254, 286, 241, 235, 280, 260, 241, 259, 250, 216, 239, 247, 251, 267, 237, 243, 242, 244, 257, 262, 235, 273, 253, 230, 249, 239, 268, 269, 278, 258, 248, 214, 247, 252, 245, 262, 262, 264, 252, 273, 250, 275, 254, 244, 249, 292, 264, 258, 255, 234, 268, 268, 265, 258, 276, 222, 268, 244, 231, 244, 263, 237, 265, 274, 257, 250, 261, 262, 257, 278, 255, 262, 260, 280, 261, 258, 253, 265, 248, 259, 249, 270, 279, 289, 263, 276, 242, 250, 234, 293, 288, 259, 256, 260, 258, 240, 239, 261, 245, 264, 248, 276, 263, 239, 255, 257, 239, 260, 267, 266, 233, 264, 240, 265, 243, 253, 229, 255, 261, 252] payload_rows_per_dst=[5250, 5520, 5374, 5506, 5458, 5364, 5712, 5476] dup_rows_per_dst=[2486, 2912, 2598, 2870, 2758, 2488, 3086, 2678] combine_rows_per_src=[5250, 5520, 5374, 5506, 5458, 5364, 5712, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229624 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157204728 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179832 weight_movement_bytes=18874368 total_nvlink_bytes=333054200 weight_to_token_ratio=0.060075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228752 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156949904 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671056 weight_movement_bytes=18874368 total_nvlink_bytes=332545424 weight_to_token_ratio=0.060172 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43932 payload_rows_remote=38368 dup_rows=21604 dup_rows_remote=18862 dedup_ratio=0.329651 router_tokens_per_expert=[251, 270, 260, 257, 282, 240, 246, 244, 285, 249, 296, 258, 260, 253, 264, 289, 260, 257, 276, 253, 243, 251, 252, 253, 251, 262, 251, 268, 258, 243, 265, 250, 228, 248, 260, 243, 255, 250, 261, 245, 235, 241, 263, 241, 251, 264, 256, 255, 259, 246, 232, 287, 270, 269, 263, 272, 229, 252, 267, 239, 251, 241, 226, 258, 282, 257, 246, 249, 252, 248, 265, 265, 260, 250, 290, 251, 262, 238, 232, 249, 259, 253, 253, 255, 257, 257, 256, 264, 245, 260, 261, 257, 255, 254, 262, 271, 295, 256, 249, 261, 307, 280, 258, 260, 252, 269, 245, 258, 243, 262, 248, 250, 234, 253, 289, 255, 243, 251, 256, 234, 229, 262, 258, 225, 240, 253, 281, 243] payload_rows_per_dst=[5502, 5416, 5456, 5474, 5550, 5516, 5564, 5454] dup_rows_per_dst=[2906, 2770, 2536, 2648, 2642, 2722, 2742, 2638] combine_rows_per_src=[5502, 5416, 5456, 5474, 5550, 5516, 5564, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229184 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157269824 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314310464 weight_movement_bytes=18874368 total_nvlink_bytes=333184832 weight_to_token_ratio=0.060050 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229528 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156631192 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032856 weight_movement_bytes=37748736 total_nvlink_bytes=350781592 weight_to_token_ratio=0.120590 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43480 payload_rows_remote=38030 dup_rows=22056 dup_rows_remote=19292 dedup_ratio=0.336548 router_tokens_per_expert=[265, 236, 251, 243, 264, 219, 273, 259, 227, 269, 255, 246, 260, 257, 262, 294, 249, 259, 258, 236, 236, 226, 250, 249, 277, 244, 281, 266, 275, 251, 291, 238, 274, 253, 242, 260, 256, 257, 291, 253, 235, 240, 252, 259, 260, 254, 265, 256, 263, 238, 236, 272, 257, 289, 254, 249, 243, 235, 267, 239, 255, 246, 248, 246, 260, 271, 257, 277, 284, 272, 273, 264, 279, 257, 292, 212, 242, 236, 266, 246, 286, 267, 267, 262, 249, 256, 253, 240, 261, 241, 239, 245, 262, 238, 261, 258, 241, 277, 242, 238, 278, 226, 252, 263, 247, 250, 283, 221, 250, 251, 251, 273, 261, 272, 256, 249, 249, 246, 252, 268, 265, 257, 283, 250, 269, 267, 245, 253] payload_rows_per_dst=[5372, 5448, 5450, 5372, 5564, 5380, 5398, 5496] dup_rows_per_dst=[2788, 2724, 2764, 2702, 2812, 2790, 2688, 2788] combine_rows_per_src=[5372, 5448, 5450, 5372, 5564, 5380, 5398, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38114 dup_rows=21902 dup_rows_remote=19078 dedup_ratio=0.334198 router_tokens_per_expert=[247, 231, 271, 252, 232, 227, 246, 269, 274, 240, 268, 277, 259, 244, 277, 251, 256, 260, 242, 247, 219, 271, 272, 277, 222, 235, 255, 245, 256, 265, 258, 275, 239, 240, 255, 277, 266, 256, 261, 230, 253, 232, 251, 247, 258, 270, 236, 256, 282, 222, 256, 231, 258, 244, 249, 265, 301, 276, 246, 265, 267, 283, 266, 260, 264, 260, 247, 263, 262, 235, 286, 259, 273, 228, 258, 288, 232, 258, 268, 255, 251, 266, 264, 271, 261, 254, 250, 288, 268, 246, 275, 257, 264, 253, 257, 247, 283, 213, 220, 262, 236, 295, 258, 244, 257, 252, 257, 245, 259, 221, 259, 261, 249, 273, 237, 273, 257, 257, 244, 246, 293, 266, 243, 262, 239, 258, 263, 260] payload_rows_per_dst=[5466, 5456, 5330, 5524, 5448, 5520, 5434, 5456] dup_rows_per_dst=[2664, 2654, 2724, 2818, 2824, 2824, 2610, 2784] combine_rows_per_src=[5466, 5456, 5330, 5524, 5448, 5520, 5434, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=228920 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157384248 ingress_bytes=194904064 link_bytes=194904064 token_bytes=314539576 weight_movement_bytes=37748736 total_nvlink_bytes=352288312 weight_to_token_ratio=0.120013 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867474842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38232 dup_rows=21828 dup_rows_remote=19168 dedup_ratio=0.333069 router_tokens_per_expert=[250, 254, 286, 234, 269, 265, 239, 235, 249, 248, 262, 241, 284, 263, 272, 258, 254, 255, 246, 261, 244, 255, 258, 259, 252, 246, 258, 258, 247, 275, 257, 273, 250, 267, 264, 239, 261, 277, 270, 241, 263, 262, 241, 265, 266, 265, 262, 255, 263, 238, 250, 252, 267, 257, 249, 266, 226, 230, 251, 268, 267, 253, 278, 253, 270, 246, 247, 253, 279, 240, 254, 257, 273, 246, 262, 275, 249, 266, 252, 254, 259, 265, 240, 226, 241, 257, 285, 255, 260, 236, 220, 237, 238, 245, 231, 237, 242, 268, 271, 254, 246, 244, 238, 280, 274, 234, 284, 253, 255, 234, 257, 274, 281, 244, 256, 258, 254, 266, 267, 250, 241, 271, 252, 285, 262, 263, 257, 275] payload_rows_per_dst=[5514, 5436, 5484, 5476, 5528, 5318, 5478, 5474] dup_rows_per_dst=[2704, 2760, 2812, 2660, 2718, 2546, 2738, 2890] combine_rows_per_src=[5514, 5436, 5484, 5476, 5528, 5318, 5478, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=229288 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156000168 ingress_bytes=212393984 link_bytes=212393984 token_bytes=311771048 weight_movement_bytes=56623104 total_nvlink_bytes=368394152 weight_to_token_ratio=0.181618 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035054650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38262 dup_rows=21748 dup_rows_remote=19046 dedup_ratio=0.331848 router_tokens_per_expert=[253, 258, 243, 262, 252, 279, 268, 268, 257, 239, 261, 283, 239, 230, 242, 240, 258, 279, 270, 272, 279, 274, 267, 281, 254, 300, 270, 248, 264, 254, 238, 250, 276, 249, 262, 239, 256, 272, 220, 235, 262, 233, 230, 308, 237, 242, 284, 263, 243, 245, 243, 235, 241, 264, 254, 245, 263, 272, 239, 285, 230, 249, 245, 235, 254, 266, 257, 285, 237, 276, 281, 255, 241, 218, 232, 258, 273, 267, 267, 247, 235, 249, 235, 246, 278, 260, 248, 268, 262, 283, 244, 286, 236, 241, 237, 233, 280, 258, 285, 291, 263, 254, 250, 262, 259, 259, 246, 259, 266, 246, 237, 248, 234, 269, 243, 243, 276, 241, 270, 260, 229, 255, 268, 277, 259, 256, 219, 263] payload_rows_per_dst=[5490, 5548, 5456, 5368, 5526, 5414, 5538, 5448] dup_rows_per_dst=[2658, 2968, 2680, 2608, 2702, 2668, 2788, 2676] combine_rows_per_src=[5490, 5548, 5456, 5368, 5526, 5414, 5538, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228768 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156343712 ingress_bytes=193863680 link_bytes=193863680 token_bytes=312458656 weight_movement_bytes=37748736 total_nvlink_bytes=350207392 weight_to_token_ratio=0.120812 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857506395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38440 dup_rows=21656 dup_rows_remote=18878 dedup_ratio=0.330444 router_tokens_per_expert=[241, 235, 292, 267, 238, 264, 284, 231, 240, 273, 241, 255, 262, 270, 243, 263, 258, 250, 265, 253, 237, 259, 274, 236, 243, 238, 235, 231, 277, 256, 229, 264, 261, 256, 260, 275, 258, 244, 249, 261, 241, 266, 251, 264, 250, 256, 264, 276, 262, 264, 247, 244, 254, 245, 270, 266, 248, 248, 269, 262, 235, 272, 234, 261, 276, 233, 225, 236, 234, 256, 275, 275, 252, 257, 267, 269, 254, 238, 277, 264, 249, 237, 236, 270, 240, 217, 236, 256, 243, 255, 233, 266, 255, 272, 239, 281, 251, 231, 248, 276, 252, 254, 239, 254, 247, 253, 262, 277, 263, 294, 291, 257, 251, 289, 276, 269, 284, 260, 276, 235, 250, 261, 247, 243, 262, 289, 268, 269] payload_rows_per_dst=[5440, 5492, 5516, 5398, 5710, 5480, 5428, 5416] dup_rows_per_dst=[2778, 2724, 2728, 2764, 2924, 2490, 2540, 2708] combine_rows_per_src=[5440, 5492, 5516, 5398, 5710, 5480, 5428, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229600 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827872 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426144 weight_movement_bytes=18874368 total_nvlink_bytes=332300512 weight_to_token_ratio=0.060220 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38332 dup_rows=21786 dup_rows_remote=18996 dedup_ratio=0.332428 router_tokens_per_expert=[271, 252, 253, 250, 271, 253, 255, 251, 244, 268, 246, 246, 252, 229, 248, 257, 276, 259, 275, 270, 239, 253, 247, 247, 257, 267, 261, 242, 251, 258, 232, 270, 306, 250, 250, 266, 265, 264, 247, 261, 266, 232, 260, 238, 239, 239, 249, 279, 276, 241, 253, 261, 235, 270, 246, 230, 260, 250, 234, 251, 282, 260, 280, 242, 265, 224, 264, 255, 270, 244, 240, 262, 249, 290, 257, 256, 249, 284, 274, 259, 266, 254, 265, 233, 270, 264, 246, 252, 268, 248, 224, 272, 255, 276, 268, 250, 276, 231, 280, 268, 251, 252, 247, 270, 265, 279, 251, 248, 250, 250, 276, 246, 257, 260, 248, 257, 256, 284, 252, 210, 250, 254, 245, 231, 277, 249, 256, 257] payload_rows_per_dst=[5396, 5418, 5448, 5456, 5614, 5538, 5504, 5376] dup_rows_per_dst=[2696, 2790, 2774, 2686, 2670, 2684, 2776, 2710] combine_rows_per_src=[5396, 5418, 5448, 5456, 5614, 5538, 5504, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=37942 dup_rows=21936 dup_rows_remote=19022 dedup_ratio=0.334717 router_tokens_per_expert=[257, 266, 260, 280, 267, 230, 275, 260, 259, 233, 276, 269, 242, 267, 228, 256, 273, 246, 283, 239, 284, 242, 261, 251, 289, 271, 246, 264, 275, 284, 251, 266, 253, 252, 270, 261, 273, 244, 236, 278, 247, 285, 246, 240, 242, 242, 245, 256, 262, 262, 256, 273, 266, 240, 245, 258, 253, 279, 255, 260, 260, 273, 265, 237, 287, 235, 256, 258, 240, 220, 264, 266, 272, 249, 217, 256, 231, 255, 271, 270, 258, 280, 221, 226, 224, 262, 245, 253, 278, 236, 243, 249, 248, 269, 224, 254, 250, 266, 260, 234, 261, 240, 218, 280, 254, 290, 245, 267, 245, 261, 244, 240, 264, 251, 255, 273, 265, 245, 265, 264, 258, 241, 257, 274, 247, 247, 257, 269] payload_rows_per_dst=[5418, 5400, 5364, 5488, 5276, 5524, 5472, 5658] dup_rows_per_dst=[2656, 2660, 2776, 2800, 2494, 2982, 2654, 2914] combine_rows_per_src=[5418, 5400, 5364, 5488, 5276, 5524, 5472, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229232 combine_payload_bytes=156721152 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156950384 ingress_bytes=213344256 link_bytes=213344256 token_bytes=313671536 weight_movement_bytes=56623104 total_nvlink_bytes=370294640 weight_to_token_ratio=0.180517 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044159689e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157450240 dispatch_meta_bytes=229272 combine_payload_bytes=157450240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157679512 ingress_bytes=195198976 link_bytes=195198976 token_bytes=315129752 weight_movement_bytes=37748736 total_nvlink_bytes=352878488 weight_to_token_ratio=0.119788 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870300544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38530 dup_rows=21626 dup_rows_remote=19060 dedup_ratio=0.329987 router_tokens_per_expert=[248, 243, 251, 282, 263, 245, 261, 279, 276, 264, 283, 284, 251, 269, 253, 251, 268, 238, 258, 268, 250, 252, 290, 249, 270, 275, 232, 279, 246, 231, 264, 236, 269, 217, 246, 275, 267, 263, 292, 248, 280, 261, 258, 270, 254, 252, 279, 248, 290, 257, 265, 266, 271, 241, 266, 268, 278, 257, 234, 250, 252, 256, 254, 256, 235, 244, 267, 244, 286, 269, 248, 252, 283, 240, 274, 230, 255, 250, 240, 244, 234, 272, 249, 244, 274, 234, 234, 278, 233, 250, 309, 244, 264, 268, 260, 236, 253, 234, 275, 255, 250, 233, 250, 240, 236, 262, 255, 241, 261, 245, 251, 232, 235, 232, 255, 248, 237, 271, 241, 246, 290, 260, 233, 256, 264, 221, 244, 269] payload_rows_per_dst=[5452, 5570, 5620, 5524, 5464, 5528, 5380, 5372] dup_rows_per_dst=[2724, 2872, 2738, 2798, 2658, 2638, 2566, 2632] combine_rows_per_src=[5452, 5570, 5620, 5524, 5464, 5528, 5380, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229312 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157237184 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314245056 weight_movement_bytes=37748736 total_nvlink_bytes=351993792 weight_to_token_ratio=0.120125 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38306 dup_rows=21790 dup_rows_remote=18950 dedup_ratio=0.332489 router_tokens_per_expert=[249, 276, 263, 278, 257, 254, 235, 256, 273, 262, 252, 263, 259, 259, 271, 217, 261, 213, 258, 255, 259, 262, 258, 245, 258, 272, 260, 240, 228, 273, 242, 254, 247, 263, 241, 260, 260, 230, 239, 307, 252, 281, 270, 252, 279, 249, 264, 246, 262, 247, 236, 220, 251, 290, 267, 262, 256, 245, 261, 269, 287, 259, 268, 266, 299, 250, 269, 264, 246, 250, 262, 262, 254, 257, 253, 237, 258, 258, 258, 249, 252, 267, 243, 260, 258, 273, 261, 294, 247, 229, 220, 241, 259, 229, 250, 266, 261, 249, 269, 232, 259, 269, 229, 270, 252, 263, 243, 256, 264, 250, 252, 245, 263, 245, 237, 240, 244, 258, 261, 251, 240, 282, 265, 267, 261, 253, 257, 258] payload_rows_per_dst=[5520, 5444, 5440, 5550, 5512, 5380, 5442, 5458] dup_rows_per_dst=[2728, 2632, 2840, 2742, 2740, 2718, 2684, 2706] combine_rows_per_src=[5520, 5444, 5440, 5550, 5512, 5380, 5442, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155410432 dispatch_meta_bytes=227856 combine_payload_bytes=155410432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155638288 ingress_bytes=174284800 link_bytes=174284800 token_bytes=311048720 weight_movement_bytes=18874368 total_nvlink_bytes=329923088 weight_to_token_ratio=0.060680 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669911200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38442 dup_rows=21656 dup_rows_remote=18948 dedup_ratio=0.330444 router_tokens_per_expert=[262, 278, 252, 272, 248, 281, 272, 304, 263, 233, 261, 265, 241, 235, 240, 245, 266, 251, 287, 258, 259, 246, 251, 266, 257, 238, 277, 241, 232, 271, 266, 261, 276, 238, 269, 251, 246, 233, 267, 268, 226, 257, 250, 241, 257, 254, 260, 250, 267, 266, 232, 285, 257, 244, 250, 255, 270, 266, 270, 256, 252, 252, 240, 254, 253, 278, 264, 271, 277, 256, 246, 279, 255, 265, 244, 249, 265, 234, 285, 268, 242, 264, 245, 224, 264, 264, 265, 265, 240, 246, 280, 248, 257, 254, 261, 254, 239, 251, 244, 234, 242, 254, 243, 225, 255, 253, 256, 266, 270, 255, 276, 261, 269, 285, 258, 269, 247, 220, 251, 247, 263, 241, 271, 239, 262, 225, 276, 221] payload_rows_per_dst=[5494, 5538, 5534, 5486, 5526, 5438, 5382, 5482] dup_rows_per_dst=[2810, 2716, 2552, 2746, 2852, 2708, 2666, 2606] combine_rows_per_src=[5494, 5538, 5534, 5486, 5526, 5438, 5382, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=230360 combine_payload_bytes=157818880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158049240 ingress_bytes=176693248 link_bytes=176693248 token_bytes=315868120 weight_movement_bytes=18874368 total_nvlink_bytes=334742488 weight_to_token_ratio=0.059754 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692987764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38306 dup_rows=21740 dup_rows_remote=19026 dedup_ratio=0.331726 router_tokens_per_expert=[251, 234, 277, 268, 224, 247, 257, 255, 258, 261, 245, 259, 241, 247, 261, 263, 282, 247, 259, 253, 254, 266, 270, 270, 241, 250, 261, 249, 279, 256, 273, 248, 239, 254, 245, 239, 265, 266, 237, 275, 251, 267, 232, 225, 261, 269, 263, 281, 266, 242, 258, 237, 256, 226, 243, 266, 276, 259, 277, 264, 243, 250, 287, 252, 257, 244, 246, 268, 245, 240, 230, 266, 247, 251, 232, 211, 256, 244, 272, 290, 248, 268, 286, 228, 260, 271, 252, 248, 268, 246, 257, 262, 232, 274, 258, 255, 265, 270, 278, 235, 264, 281, 243, 273, 273, 257, 254, 238, 268, 238, 238, 229, 285, 251, 284, 264, 257, 246, 279, 225, 244, 273, 275, 265, 272, 243, 271, 241] payload_rows_per_dst=[5474, 5566, 5446, 5490, 5368, 5466, 5486, 5500] dup_rows_per_dst=[2622, 2750, 2692, 2714, 2630, 2760, 2722, 2850] combine_rows_per_src=[5474, 5566, 5446, 5490, 5368, 5466, 5486, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229024 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130400 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314031776 weight_movement_bytes=18874368 total_nvlink_bytes=332906144 weight_to_token_ratio=0.060103 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=229560 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157687992 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315146424 weight_movement_bytes=18874368 total_nvlink_bytes=334020792 weight_to_token_ratio=0.059891 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43552 payload_rows_remote=38178 dup_rows=21984 dup_rows_remote=19286 dedup_ratio=0.335449 router_tokens_per_expert=[276, 271, 260, 243, 245, 261, 233, 267, 265, 245, 245, 255, 229, 247, 245, 249, 236, 262, 252, 265, 270, 263, 251, 241, 249, 249, 266, 255, 255, 258, 244, 247, 284, 254, 270, 259, 253, 257, 266, 265, 258, 242, 280, 263, 208, 245, 264, 257, 248, 254, 300, 270, 275, 252, 258, 263, 245, 279, 270, 251, 245, 271, 252, 255, 256, 272, 253, 271, 265, 273, 266, 282, 258, 263, 267, 281, 252, 269, 249, 265, 268, 225, 249, 246, 234, 260, 241, 258, 241, 283, 271, 238, 227, 262, 238, 255, 273, 243, 243, 244, 246, 240, 229, 261, 265, 259, 269, 296, 263, 232, 277, 272, 252, 273, 238, 265, 265, 224, 250, 246, 251, 245, 227, 247, 273, 242, 247, 261] payload_rows_per_dst=[5374, 5412, 5454, 5542, 5630, 5284, 5476, 5380] dup_rows_per_dst=[2698, 2714, 2796, 2834, 2962, 2600, 2748, 2632] combine_rows_per_src=[5374, 5412, 5454, 5542, 5630, 5284, 5476, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38394 dup_rows=21796 dup_rows_remote=19098 dedup_ratio=0.332581 router_tokens_per_expert=[299, 291, 270, 258, 256, 278, 263, 267, 262, 242, 234, 274, 261, 256, 282, 268, 263, 256, 246, 233, 231, 239, 258, 241, 259, 241, 240, 257, 248, 258, 256, 261, 242, 252, 265, 250, 271, 271, 230, 250, 285, 222, 250, 258, 239, 256, 248, 273, 260, 258, 294, 273, 269, 271, 278, 259, 298, 278, 279, 238, 231, 256, 255, 255, 247, 245, 259, 259, 256, 252, 242, 253, 220, 277, 229, 245, 234, 268, 285, 251, 278, 249, 266, 275, 264, 270, 239, 243, 256, 258, 259, 278, 246, 244, 242, 239, 231, 232, 267, 263, 266, 217, 257, 272, 277, 251, 252, 245, 243, 243, 258, 245, 260, 264, 261, 242, 250, 287, 234, 247, 263, 237, 231, 248, 248, 272, 273, 242] payload_rows_per_dst=[5592, 5430, 5448, 5614, 5346, 5502, 5346, 5462] dup_rows_per_dst=[2930, 2544, 2676, 2890, 2698, 2710, 2692, 2656] combine_rows_per_src=[5592, 5430, 5448, 5614, 5346, 5502, 5346, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229328 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130704 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314032080 weight_movement_bytes=37748736 total_nvlink_bytes=351780816 weight_to_token_ratio=0.120207 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38246 dup_rows=21844 dup_rows_remote=19082 dedup_ratio=0.333313 router_tokens_per_expert=[250, 260, 233, 262, 256, 256, 243, 250, 252, 259, 254, 302, 249, 229, 260, 264, 251, 235, 256, 273, 245, 249, 249, 254, 271, 263, 260, 243, 271, 268, 263, 253, 265, 248, 277, 244, 274, 241, 255, 266, 250, 247, 280, 253, 255, 246, 239, 243, 248, 272, 251, 250, 232, 245, 248, 260, 259, 239, 267, 252, 242, 253, 267, 286, 273, 253, 248, 277, 265, 246, 251, 251, 238, 279, 246, 275, 264, 263, 269, 250, 266, 258, 265, 261, 242, 246, 241, 258, 259, 269, 246, 279, 276, 238, 253, 262, 258, 256, 256, 245, 253, 251, 268, 251, 248, 230, 269, 296, 251, 241, 252, 277, 268, 226, 242, 276, 255, 252, 247, 251, 224, 265, 264, 273, 270, 234, 267, 248] payload_rows_per_dst=[5436, 5446, 5464, 5446, 5536, 5464, 5466, 5434] dup_rows_per_dst=[2722, 2762, 2702, 2696, 2760, 2774, 2738, 2690] combine_rows_per_src=[5436, 5446, 5464, 5446, 5536, 5464, 5466, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38010 dup_rows=22040 dup_rows_remote=19268 dedup_ratio=0.336304 router_tokens_per_expert=[246, 229, 276, 266, 275, 277, 249, 233, 266, 247, 267, 226, 267, 239, 233, 246, 275, 265, 238, 259, 261, 228, 259, 261, 272, 239, 271, 229, 242, 257, 247, 260, 233, 245, 237, 260, 261, 256, 220, 259, 277, 276, 258, 255, 254, 253, 260, 259, 243, 232, 251, 270, 249, 277, 267, 256, 248, 284, 252, 255, 258, 241, 271, 251, 249, 275, 256, 290, 262, 273, 268, 262, 280, 244, 268, 260, 257, 265, 261, 253, 262, 241, 297, 274, 243, 251, 259, 250, 252, 255, 247, 257, 274, 263, 256, 236, 246, 273, 255, 262, 244, 255, 282, 251, 249, 252, 298, 238, 265, 278, 242, 224, 285, 248, 265, 232, 286, 260, 233, 234, 222, 262, 271, 245, 257, 284, 225, 232] payload_rows_per_dst=[5382, 5418, 5364, 5486, 5434, 5404, 5522, 5486] dup_rows_per_dst=[2650, 2708, 2762, 2742, 2802, 2672, 2932, 2772] combine_rows_per_src=[5382, 5418, 5364, 5486, 5434, 5404, 5522, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229856 combine_payload_bytes=156377088 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156606944 ingress_bytes=194125824 link_bytes=194125824 token_bytes=312984032 weight_movement_bytes=37748736 total_nvlink_bytes=350732768 weight_to_token_ratio=0.120609 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860018130e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229968 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157491792 ingress_bytes=195010560 link_bytes=195010560 token_bytes=314753616 weight_movement_bytes=37748736 total_nvlink_bytes=352502352 weight_to_token_ratio=0.119931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868495235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38166 dup_rows=21850 dup_rows_remote=19014 dedup_ratio=0.333405 router_tokens_per_expert=[266, 276, 235, 267, 251, 272, 283, 249, 257, 272, 244, 246, 245, 265, 258, 276, 259, 243, 251, 249, 250, 249, 272, 258, 262, 262, 240, 257, 256, 227, 250, 250, 256, 260, 273, 228, 264, 258, 237, 267, 258, 254, 246, 215, 276, 274, 265, 262, 254, 268, 267, 257, 265, 238, 240, 228, 281, 269, 240, 259, 253, 234, 254, 241, 269, 245, 270, 269, 245, 232, 273, 267, 259, 270, 240, 242, 266, 245, 250, 258, 257, 299, 250, 239, 266, 264, 252, 253, 266, 243, 273, 249, 236, 236, 261, 271, 256, 276, 259, 276, 255, 260, 273, 286, 270, 262, 258, 243, 258, 252, 259, 235, 242, 262, 246, 249, 211, 254, 251, 247, 247, 285, 273, 265, 260, 260, 239, 246] payload_rows_per_dst=[5494, 5436, 5478, 5420, 5420, 5550, 5520, 5368] dup_rows_per_dst=[2830, 2550, 2708, 2676, 2780, 2764, 2836, 2706] combine_rows_per_src=[5494, 5436, 5478, 5420, 5420, 5550, 5520, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229312 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156884928 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540544 weight_movement_bytes=18874368 total_nvlink_bytes=332414912 weight_to_token_ratio=0.060198 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=229112 combine_payload_bytes=155688960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155918072 ingress_bytes=193437696 link_bytes=193437696 token_bytes=311607032 weight_movement_bytes=37748736 total_nvlink_bytes=349355768 weight_to_token_ratio=0.121142 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853424826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38310 dup_rows=21804 dup_rows_remote=19234 dedup_ratio=0.332703 router_tokens_per_expert=[269, 246, 252, 250, 257, 280, 252, 246, 237, 286, 257, 250, 288, 276, 267, 264, 252, 268, 249, 267, 247, 254, 257, 243, 275, 258, 280, 260, 244, 248, 239, 246, 249, 249, 235, 234, 249, 248, 265, 251, 260, 230, 254, 249, 270, 243, 258, 252, 236, 257, 267, 259, 271, 267, 242, 253, 267, 266, 246, 259, 251, 266, 257, 282, 255, 293, 259, 255, 235, 254, 280, 241, 244, 238, 223, 273, 258, 270, 238, 266, 258, 241, 258, 236, 266, 248, 238, 233, 251, 253, 264, 275, 255, 259, 239, 258, 248, 273, 250, 265, 277, 234, 267, 227, 244, 266, 271, 250, 281, 258, 274, 269, 230, 249, 280, 289, 272, 264, 239, 260, 282, 225, 269, 250, 248, 246, 278, 213] payload_rows_per_dst=[5502, 5468, 5422, 5522, 5516, 5380, 5452, 5470] dup_rows_per_dst=[2852, 2706, 2570, 2770, 2648, 2684, 2856, 2718] combine_rows_per_src=[5502, 5468, 5422, 5522, 5516, 5380, 5452, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38272 dup_rows=21878 dup_rows_remote=19206 dedup_ratio=0.333832 router_tokens_per_expert=[254, 241, 281, 256, 248, 254, 226, 251, 253, 247, 260, 256, 269, 257, 275, 256, 268, 242, 227, 260, 250, 238, 260, 288, 257, 272, 280, 256, 246, 242, 266, 256, 266, 284, 258, 263, 257, 250, 230, 255, 250, 237, 242, 233, 255, 263, 262, 275, 253, 255, 278, 260, 233, 228, 265, 250, 270, 265, 258, 255, 256, 257, 236, 245, 293, 283, 276, 278, 278, 260, 259, 254, 253, 276, 237, 258, 252, 246, 252, 268, 242, 242, 255, 244, 277, 254, 255, 276, 240, 248, 247, 270, 244, 252, 241, 242, 246, 267, 259, 257, 247, 254, 230, 268, 241, 235, 258, 269, 276, 223, 255, 239, 279, 248, 273, 284, 242, 256, 247, 275, 232, 288, 261, 250, 248, 261, 261, 251] payload_rows_per_dst=[5436, 5496, 5502, 5432, 5514, 5386, 5398, 5494] dup_rows_per_dst=[2732, 2720, 2658, 2696, 2932, 2672, 2650, 2818] combine_rows_per_src=[5436, 5496, 5502, 5432, 5514, 5386, 5398, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38158 dup_rows=21894 dup_rows_remote=19178 dedup_ratio=0.334076 router_tokens_per_expert=[248, 257, 240, 239, 266, 255, 272, 264, 254, 249, 276, 268, 258, 263, 250, 242, 255, 252, 279, 244, 248, 249, 227, 246, 252, 248, 271, 252, 266, 219, 285, 235, 231, 287, 245, 259, 262, 237, 243, 243, 248, 269, 245, 284, 244, 263, 274, 251, 257, 263, 230, 230, 251, 261, 289, 244, 264, 282, 250, 254, 285, 231, 246, 263, 247, 271, 235, 234, 262, 280, 258, 242, 271, 254, 255, 257, 243, 265, 262, 231, 233, 256, 264, 272, 264, 234, 240, 224, 274, 264, 291, 244, 251, 266, 260, 265, 268, 263, 289, 261, 256, 268, 249, 278, 238, 254, 254, 274, 227, 258, 260, 259, 257, 291, 254, 258, 277, 258, 244, 284, 245, 255, 269, 249, 229, 266, 255, 238] payload_rows_per_dst=[5432, 5372, 5470, 5484, 5450, 5462, 5506, 5466] dup_rows_per_dst=[2770, 2684, 2700, 2716, 2684, 2742, 2806, 2792] combine_rows_per_src=[5432, 5372, 5470, 5484, 5450, 5462, 5506, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228720 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156556656 ingress_bytes=212951040 link_bytes=212951040 token_bytes=312884592 weight_movement_bytes=56623104 total_nvlink_bytes=369507696 weight_to_token_ratio=0.180971 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040392087e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=230176 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157147936 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314065696 weight_movement_bytes=18874368 total_nvlink_bytes=332940064 weight_to_token_ratio=0.060097 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229912 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156992024 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313754136 weight_movement_bytes=37748736 total_nvlink_bytes=351502872 weight_to_token_ratio=0.120313 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43550 payload_rows_remote=37926 dup_rows=21986 dup_rows_remote=18932 dedup_ratio=0.335480 router_tokens_per_expert=[249, 256, 243, 268, 270, 245, 266, 271, 310, 281, 238, 248, 279, 243, 246, 255, 242, 250, 260, 281, 242, 257, 240, 250, 247, 247, 243, 244, 262, 266, 263, 229, 259, 262, 256, 247, 248, 267, 230, 265, 269, 233, 247, 279, 261, 269, 242, 267, 243, 260, 257, 275, 271, 279, 267, 243, 282, 259, 253, 257, 252, 275, 269, 248, 252, 251, 242, 257, 238, 262, 246, 249, 266, 282, 238, 234, 265, 247, 258, 253, 226, 239, 261, 255, 259, 234, 269, 232, 257, 247, 267, 276, 266, 251, 259, 279, 261, 270, 250, 235, 252, 237, 257, 269, 263, 263, 252, 270, 251, 232, 250, 264, 242, 267, 238, 257, 257, 266, 226, 272, 229, 264, 254, 254, 255, 276, 280, 256] payload_rows_per_dst=[5624, 5396, 5448, 5368, 5430, 5598, 5408, 5278] dup_rows_per_dst=[3054, 2650, 2768, 2670, 2636, 2828, 2744, 2636] combine_rows_per_src=[5624, 5396, 5448, 5368, 5430, 5598, 5408, 5278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229344 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156524512 ingress_bytes=175169536 link_bytes=175169536 token_bytes=312819680 weight_movement_bytes=18874368 total_nvlink_bytes=331694048 weight_to_token_ratio=0.060336 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678388305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43952 payload_rows_remote=38402 dup_rows=21584 dup_rows_remote=18892 dedup_ratio=0.329346 router_tokens_per_expert=[277, 262, 280, 258, 255, 263, 218, 251, 276, 262, 233, 269, 257, 259, 270, 243, 227, 242, 283, 267, 253, 263, 264, 279, 256, 274, 271, 247, 256, 267, 261, 227, 231, 243, 254, 282, 235, 236, 237, 258, 260, 278, 265, 246, 260, 246, 260, 228, 251, 250, 249, 257, 250, 245, 236, 240, 264, 264, 276, 242, 262, 253, 261, 251, 267, 249, 244, 263, 268, 234, 247, 273, 254, 285, 260, 285, 257, 226, 253, 256, 278, 247, 249, 264, 235, 277, 270, 237, 281, 260, 289, 255, 232, 266, 266, 270, 257, 242, 261, 264, 263, 256, 253, 235, 258, 230, 261, 267, 225, 265, 238, 258, 268, 251, 239, 243, 262, 263, 264, 255, 258, 241, 270, 264, 261, 270, 233, 256] payload_rows_per_dst=[5448, 5554, 5432, 5470, 5550, 5504, 5490, 5504] dup_rows_per_dst=[2818, 2720, 2606, 2632, 2692, 2848, 2576, 2692] combine_rows_per_src=[5448, 5554, 5432, 5470, 5550, 5504, 5490, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38114 dup_rows=21994 dup_rows_remote=19212 dedup_ratio=0.335602 router_tokens_per_expert=[264, 259, 255, 233, 271, 234, 257, 259, 243, 258, 256, 282, 236, 272, 253, 260, 232, 264, 261, 270, 270, 235, 246, 256, 240, 277, 261, 270, 246, 247, 256, 274, 247, 277, 266, 256, 257, 232, 273, 276, 257, 267, 237, 244, 256, 250, 256, 264, 245, 222, 253, 243, 265, 239, 251, 263, 255, 250, 252, 253, 285, 241, 283, 261, 224, 246, 263, 223, 259, 278, 251, 233, 226, 245, 244, 264, 246, 275, 268, 252, 246, 262, 239, 261, 247, 243, 250, 294, 259, 260, 278, 261, 259, 251, 249, 282, 247, 278, 255, 268, 255, 243, 259, 263, 252, 246, 253, 268, 227, 260, 240, 280, 259, 239, 246, 269, 271, 272, 288, 274, 260, 285, 284, 237, 246, 239, 253, 241] payload_rows_per_dst=[5454, 5428, 5430, 5442, 5374, 5482, 5458, 5474] dup_rows_per_dst=[2730, 2782, 2800, 2680, 2620, 2800, 2730, 2852] combine_rows_per_src=[5454, 5428, 5430, 5442, 5374, 5482, 5458, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38208 dup_rows=21786 dup_rows_remote=18876 dedup_ratio=0.332428 router_tokens_per_expert=[262, 237, 238, 234, 235, 246, 254, 260, 268, 255, 271, 302, 246, 259, 257, 237, 252, 241, 268, 264, 274, 250, 245, 304, 258, 287, 212, 250, 251, 255, 246, 217, 236, 252, 239, 266, 253, 264, 247, 271, 279, 234, 233, 256, 237, 254, 277, 249, 263, 261, 256, 248, 245, 256, 255, 266, 253, 260, 263, 269, 265, 264, 252, 262, 249, 242, 259, 281, 260, 249, 251, 260, 269, 291, 263, 237, 268, 250, 252, 250, 242, 263, 269, 243, 238, 251, 265, 283, 241, 269, 253, 272, 230, 244, 265, 236, 224, 234, 251, 263, 244, 244, 274, 241, 241, 261, 257, 262, 261, 273, 263, 260, 257, 243, 262, 274, 275, 289, 263, 273, 254, 240, 259, 258, 262, 262, 273, 256] payload_rows_per_dst=[5452, 5406, 5468, 5516, 5420, 5424, 5522, 5542] dup_rows_per_dst=[2670, 2742, 2626, 2760, 2594, 2652, 2832, 2910] combine_rows_per_src=[5452, 5406, 5468, 5516, 5420, 5424, 5522, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155344896 dispatch_meta_bytes=227432 combine_payload_bytes=155344896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155572328 ingress_bytes=193093632 link_bytes=193093632 token_bytes=310917224 weight_movement_bytes=37748736 total_nvlink_bytes=348665960 weight_to_token_ratio=0.121411 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.850128174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229176 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157523768 ingress_bytes=195043328 link_bytes=195043328 token_bytes=314818360 weight_movement_bytes=37748736 total_nvlink_bytes=352567096 weight_to_token_ratio=0.119906 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868809202e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38282 dup_rows=21780 dup_rows_remote=19034 dedup_ratio=0.332336 router_tokens_per_expert=[228, 261, 265, 266, 227, 256, 264, 239, 251, 261, 260, 283, 233, 258, 262, 254, 248, 265, 245, 253, 238, 239, 241, 273, 250, 226, 281, 253, 258, 267, 236, 240, 280, 240, 274, 255, 272, 243, 219, 246, 267, 272, 263, 259, 269, 264, 241, 242, 243, 240, 277, 247, 225, 242, 228, 249, 232, 287, 252, 286, 279, 271, 247, 256, 253, 242, 271, 273, 253, 255, 261, 239, 246, 266, 281, 221, 272, 230, 273, 255, 268, 251, 252, 273, 266, 263, 261, 245, 256, 247, 257, 246, 258, 239, 278, 263, 266, 235, 249, 249, 254, 263, 252, 263, 270, 267, 295, 250, 238, 247, 259, 253, 266, 247, 244, 277, 238, 276, 255, 251, 262, 273, 264, 271, 260, 287, 265, 260] payload_rows_per_dst=[5472, 5444, 5484, 5418, 5412, 5466, 5474, 5586] dup_rows_per_dst=[2664, 2582, 2728, 2704, 2770, 2780, 2746, 2806] combine_rows_per_src=[5472, 5444, 5484, 5418, 5412, 5466, 5474, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=229304 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156344248 ingress_bytes=174989312 link_bytes=174989312 token_bytes=312459192 weight_movement_bytes=18874368 total_nvlink_bytes=331333560 weight_to_token_ratio=0.060406 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676661487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38368 dup_rows=21778 dup_rows_remote=19008 dedup_ratio=0.332306 router_tokens_per_expert=[279, 255, 250, 274, 261, 232, 276, 234, 225, 275, 287, 234, 228, 259, 213, 267, 273, 273, 250, 272, 236, 251, 254, 271, 241, 250, 261, 267, 254, 256, 254, 255, 262, 254, 254, 260, 261, 233, 265, 223, 263, 269, 254, 272, 245, 246, 290, 229, 261, 270, 253, 300, 285, 296, 259, 287, 241, 254, 259, 249, 225, 264, 262, 232, 270, 257, 275, 250, 248, 302, 239, 274, 268, 281, 252, 248, 277, 229, 246, 263, 238, 235, 260, 256, 249, 247, 263, 262, 263, 268, 259, 247, 275, 282, 261, 254, 258, 231, 267, 247, 252, 261, 267, 275, 262, 242, 261, 227, 249, 220, 239, 280, 242, 242, 235, 229, 238, 264, 241, 255, 230, 260, 258, 273, 251, 241, 266, 263] payload_rows_per_dst=[5454, 5536, 5390, 5522, 5624, 5502, 5444, 5286] dup_rows_per_dst=[2644, 2700, 2770, 2872, 2734, 2736, 2632, 2690] combine_rows_per_src=[5454, 5536, 5390, 5522, 5624, 5502, 5444, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228336 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156728304 ingress_bytes=213123072 link_bytes=213123072 token_bytes=313228272 weight_movement_bytes=56623104 total_nvlink_bytes=369851376 weight_to_token_ratio=0.180773 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042040413e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38184 dup_rows=21804 dup_rows_remote=18896 dedup_ratio=0.332703 router_tokens_per_expert=[271, 252, 233, 250, 251, 266, 229, 287, 249, 271, 254, 239, 244, 239, 239, 249, 261, 251, 258, 251, 259, 249, 274, 263, 245, 262, 264, 281, 236, 222, 257, 258, 247, 225, 262, 261, 255, 253, 254, 281, 263, 234, 299, 236, 255, 251, 240, 283, 247, 233, 281, 261, 255, 263, 248, 254, 242, 275, 246, 257, 239, 228, 258, 245, 244, 258, 258, 262, 271, 255, 236, 238, 246, 256, 261, 263, 245, 256, 259, 251, 265, 268, 239, 229, 270, 257, 264, 273, 282, 271, 244, 260, 267, 295, 286, 258, 243, 255, 268, 262, 260, 267, 282, 259, 247, 255, 250, 232, 261, 247, 246, 241, 258, 286, 263, 258, 256, 237, 283, 257, 249, 251, 271, 256, 267, 273, 238, 258] payload_rows_per_dst=[5420, 5492, 5422, 5364, 5476, 5548, 5484, 5526] dup_rows_per_dst=[2626, 2690, 2776, 2700, 2642, 2908, 2666, 2796] combine_rows_per_src=[5420, 5492, 5422, 5364, 5476, 5548, 5484, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38318 dup_rows=21652 dup_rows_remote=18988 dedup_ratio=0.330383 router_tokens_per_expert=[264, 264, 267, 265, 267, 252, 246, 266, 245, 269, 276, 249, 225, 274, 255, 249, 253, 249, 219, 259, 273, 250, 262, 221, 239, 258, 253, 286, 248, 272, 258, 254, 231, 259, 224, 269, 237, 262, 227, 238, 267, 244, 284, 236, 262, 262, 256, 238, 271, 265, 261, 258, 279, 251, 247, 237, 259, 242, 235, 280, 246, 251, 270, 263, 264, 250, 264, 259, 262, 259, 251, 242, 251, 253, 255, 253, 240, 287, 258, 236, 251, 272, 279, 258, 258, 275, 272, 251, 258, 270, 252, 234, 267, 235, 233, 270, 240, 261, 268, 267, 295, 256, 259, 241, 287, 260, 264, 244, 234, 278, 244, 225, 250, 224, 267, 255, 237, 273, 261, 247, 253, 255, 269, 256, 289, 270, 249, 273] payload_rows_per_dst=[5492, 5466, 5446, 5566, 5508, 5522, 5432, 5452] dup_rows_per_dst=[2774, 2642, 2546, 2664, 2660, 2748, 2814, 2804] combine_rows_per_src=[5492, 5466, 5446, 5566, 5508, 5522, 5432, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229264 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032336 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835408 weight_movement_bytes=18874368 total_nvlink_bytes=332709776 weight_to_token_ratio=0.060141 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229504 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384832 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540160 weight_movement_bytes=18874368 total_nvlink_bytes=333414528 weight_to_token_ratio=0.060006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38124 dup_rows=21968 dup_rows_remote=19284 dedup_ratio=0.335205 router_tokens_per_expert=[243, 255, 232, 257, 235, 254, 262, 267, 251, 255, 242, 246, 279, 259, 238, 270, 252, 253, 257, 239, 256, 284, 240, 276, 263, 258, 257, 252, 272, 245, 286, 230, 241, 245, 249, 249, 259, 256, 240, 267, 231, 253, 250, 257, 273, 248, 286, 238, 228, 235, 269, 252, 250, 269, 263, 282, 265, 260, 266, 257, 270, 255, 275, 262, 269, 252, 234, 263, 261, 240, 268, 258, 246, 261, 251, 248, 281, 264, 249, 273, 295, 256, 261, 250, 251, 292, 261, 251, 258, 250, 264, 243, 254, 237, 274, 255, 264, 257, 261, 255, 261, 255, 238, 263, 256, 247, 269, 233, 266, 249, 253, 274, 223, 242, 237, 254, 237, 238, 252, 256, 270, 273, 254, 265, 246, 276, 240, 269] payload_rows_per_dst=[5444, 5396, 5472, 5480, 5502, 5484, 5352, 5438] dup_rows_per_dst=[2684, 2844, 2718, 2894, 2676, 2820, 2706, 2626] combine_rows_per_src=[5444, 5396, 5472, 5480, 5502, 5484, 5352, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228320 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156629984 ingress_bytes=231899136 link_bytes=231899136 token_bytes=313031648 weight_movement_bytes=75497472 total_nvlink_bytes=388529120 weight_to_token_ratio=0.241182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221943420e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229224 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179752 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130280 weight_movement_bytes=18874368 total_nvlink_bytes=333004648 weight_to_token_ratio=0.060085 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38230 dup_rows=21770 dup_rows_remote=18978 dedup_ratio=0.332184 router_tokens_per_expert=[294, 277, 267, 264, 231, 266, 259, 261, 286, 265, 231, 243, 262, 271, 248, 259, 236, 224, 240, 245, 253, 260, 242, 241, 250, 270, 282, 248, 246, 263, 252, 237, 263, 234, 258, 258, 264, 249, 268, 240, 248, 240, 263, 253, 251, 241, 287, 246, 257, 258, 254, 252, 274, 244, 250, 268, 248, 267, 217, 264, 252, 259, 243, 243, 254, 244, 261, 254, 241, 250, 276, 292, 239, 263, 273, 245, 255, 270, 264, 283, 265, 268, 261, 241, 228, 240, 258, 289, 262, 265, 261, 257, 242, 259, 263, 228, 253, 276, 234, 259, 254, 247, 246, 234, 226, 295, 238, 231, 256, 249, 290, 248, 257, 293, 249, 272, 284, 285, 253, 256, 242, 259, 264, 247, 261, 260, 258, 255] payload_rows_per_dst=[5530, 5404, 5430, 5466, 5536, 5468, 5406, 5526] dup_rows_per_dst=[2838, 2574, 2696, 2634, 2792, 2706, 2666, 2864] combine_rows_per_src=[5530, 5404, 5430, 5466, 5536, 5468, 5406, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38316 dup_rows=21722 dup_rows_remote=18862 dedup_ratio=0.331451 router_tokens_per_expert=[239, 240, 235, 250, 255, 275, 245, 264, 259, 252, 258, 250, 257, 256, 253, 247, 283, 266, 274, 256, 227, 253, 250, 241, 272, 263, 263, 246, 259, 279, 283, 264, 265, 252, 247, 278, 267, 266, 261, 265, 256, 292, 250, 256, 244, 264, 226, 236, 257, 264, 274, 241, 255, 243, 249, 242, 235, 258, 232, 249, 266, 262, 257, 248, 261, 266, 257, 252, 248, 255, 244, 301, 258, 273, 291, 260, 239, 239, 224, 238, 261, 286, 223, 254, 233, 231, 262, 243, 272, 264, 253, 263, 288, 266, 282, 261, 254, 251, 294, 254, 214, 233, 254, 254, 264, 234, 246, 224, 248, 270, 259, 288, 272, 253, 264, 241, 286, 234, 275, 252, 255, 261, 251, 262, 288, 237, 242, 235] payload_rows_per_dst=[5490, 5498, 5486, 5444, 5506, 5494, 5416, 5480] dup_rows_per_dst=[2580, 2860, 2764, 2620, 2706, 2790, 2666, 2736] combine_rows_per_src=[5490, 5498, 5486, 5444, 5506, 5494, 5416, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229632 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156385536 ingress_bytes=193904640 link_bytes=193904640 token_bytes=312541440 weight_movement_bytes=37748736 total_nvlink_bytes=350290176 weight_to_token_ratio=0.120780 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857898853e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38124 dup_rows=21948 dup_rows_remote=19228 dedup_ratio=0.334900 router_tokens_per_expert=[270, 250, 298, 249, 254, 258, 219, 281, 260, 287, 266, 249, 227, 272, 254, 230, 274, 249, 275, 249, 242, 247, 238, 239, 259, 244, 230, 256, 280, 264, 232, 294, 264, 284, 252, 259, 239, 250, 271, 238, 253, 266, 234, 267, 270, 260, 279, 245, 258, 254, 275, 239, 236, 284, 273, 255, 275, 228, 234, 266, 277, 257, 266, 237, 216, 250, 240, 253, 262, 261, 259, 259, 254, 228, 256, 258, 249, 253, 258, 251, 256, 283, 296, 249, 249, 255, 232, 254, 260, 278, 251, 259, 251, 265, 238, 248, 229, 259, 237, 263, 262, 252, 266, 256, 268, 230, 280, 271, 251, 256, 250, 274, 261, 248, 281, 253, 249, 243, 247, 266, 269, 269, 244, 266, 254, 226, 259, 257] payload_rows_per_dst=[5314, 5422, 5474, 5568, 5426, 5460, 5460, 5464] dup_rows_per_dst=[2514, 2694, 2788, 3080, 2588, 2728, 2836, 2720] combine_rows_per_src=[5314, 5422, 5474, 5568, 5426, 5460, 5460, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228832 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156818912 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313408992 weight_movement_bytes=18874368 total_nvlink_bytes=332283360 weight_to_token_ratio=0.060223 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38224 dup_rows=21958 dup_rows_remote=19240 dedup_ratio=0.335052 router_tokens_per_expert=[236, 264, 246, 256, 260, 278, 255, 253, 275, 252, 252, 253, 237, 260, 262, 254, 240, 220, 271, 248, 256, 253, 251, 280, 275, 268, 273, 262, 264, 262, 244, 260, 259, 253, 256, 261, 265, 233, 257, 227, 292, 275, 246, 265, 248, 265, 262, 253, 221, 264, 219, 265, 230, 249, 248, 258, 244, 257, 253, 236, 256, 269, 240, 256, 242, 273, 245, 262, 277, 247, 259, 268, 262, 274, 276, 263, 284, 252, 271, 218, 261, 272, 264, 261, 236, 236, 291, 268, 240, 253, 283, 251, 268, 238, 268, 239, 237, 253, 231, 262, 263, 242, 258, 247, 244, 230, 280, 257, 271, 248, 263, 250, 273, 247, 259, 250, 255, 267, 246, 247, 215, 242, 294, 265, 285, 254, 288, 241] payload_rows_per_dst=[5426, 5476, 5514, 5312, 5468, 5540, 5354, 5488] dup_rows_per_dst=[2760, 2778, 2720, 2618, 2878, 2718, 2718, 2768] combine_rows_per_src=[5426, 5476, 5514, 5312, 5468, 5540, 5354, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=228712 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157171048 ingress_bytes=194691072 link_bytes=194691072 token_bytes=314113384 weight_movement_bytes=37748736 total_nvlink_bytes=351862120 weight_to_token_ratio=0.120176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865434058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38218 dup_rows=21880 dup_rows_remote=19104 dedup_ratio=0.333862 router_tokens_per_expert=[232, 242, 246, 252, 230, 269, 240, 261, 273, 250, 267, 275, 261, 249, 283, 266, 263, 233, 215, 251, 273, 243, 256, 259, 243, 266, 252, 227, 256, 266, 252, 251, 264, 252, 250, 246, 243, 255, 265, 268, 255, 253, 277, 273, 265, 251, 247, 243, 255, 271, 248, 257, 279, 241, 258, 260, 247, 256, 279, 247, 240, 241, 236, 263, 243, 262, 244, 263, 252, 247, 280, 237, 245, 251, 261, 265, 246, 251, 240, 232, 248, 290, 268, 259, 275, 282, 257, 283, 285, 226, 251, 256, 256, 247, 260, 249, 251, 279, 248, 279, 275, 254, 254, 252, 263, 259, 273, 237, 235, 253, 248, 280, 260, 232, 267, 243, 256, 249, 230, 268, 278, 273, 238, 286, 254, 250, 279, 267] payload_rows_per_dst=[5430, 5428, 5438, 5480, 5432, 5514, 5460, 5474] dup_rows_per_dst=[2762, 2584, 2776, 2676, 2606, 2870, 2820, 2786] combine_rows_per_src=[5430, 5428, 5438, 5480, 5432, 5514, 5460, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38262 dup_rows=21710 dup_rows_remote=18898 dedup_ratio=0.331268 router_tokens_per_expert=[261, 268, 247, 254, 290, 260, 241, 246, 257, 235, 263, 261, 262, 238, 234, 241, 263, 259, 219, 256, 249, 265, 244, 261, 269, 275, 238, 243, 243, 254, 275, 251, 249, 274, 259, 248, 260, 265, 254, 280, 292, 252, 261, 253, 256, 264, 265, 264, 255, 295, 242, 248, 280, 277, 275, 228, 273, 283, 248, 272, 231, 258, 256, 267, 256, 257, 271, 222, 228, 254, 259, 241, 250, 277, 238, 240, 246, 254, 250, 268, 280, 236, 272, 255, 270, 249, 235, 235, 269, 270, 260, 291, 254, 298, 261, 246, 234, 274, 241, 267, 261, 218, 223, 272, 265, 255, 260, 259, 238, 257, 262, 248, 240, 270, 242, 267, 259, 241, 259, 274, 261, 258, 242, 239, 248, 238, 251, 247] payload_rows_per_dst=[5430, 5440, 5574, 5564, 5460, 5576, 5408, 5374] dup_rows_per_dst=[2686, 2688, 2818, 2812, 2562, 2786, 2660, 2698] combine_rows_per_src=[5430, 5440, 5574, 5564, 5460, 5576, 5408, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38088 dup_rows=21848 dup_rows_remote=19058 dedup_ratio=0.333374 router_tokens_per_expert=[243, 244, 273, 276, 238, 228, 258, 267, 245, 241, 256, 250, 263, 245, 228, 251, 252, 263, 282, 254, 237, 290, 242, 265, 248, 247, 268, 264, 284, 268, 246, 249, 266, 237, 240, 245, 236, 246, 253, 269, 272, 248, 257, 246, 252, 263, 239, 231, 252, 270, 216, 236, 249, 257, 248, 269, 254, 254, 252, 257, 243, 269, 262, 246, 286, 251, 242, 257, 261, 264, 277, 245, 268, 272, 232, 270, 253, 261, 240, 249, 261, 269, 276, 234, 259, 293, 242, 264, 233, 263, 269, 275, 253, 253, 260, 291, 280, 254, 254, 269, 233, 298, 263, 272, 274, 236, 219, 243, 267, 199, 267, 258, 242, 258, 246, 270, 244, 262, 262, 285, 256, 274, 256, 252, 275, 263, 253, 262] payload_rows_per_dst=[5400, 5458, 5380, 5384, 5530, 5600, 5416, 5520] dup_rows_per_dst=[2612, 2860, 2620, 2684, 2726, 2790, 2756, 2800] combine_rows_per_src=[5400, 5458, 5380, 5384, 5530, 5600, 5416, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229408 combine_payload_bytes=156155904 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156385312 ingress_bytes=212779008 link_bytes=212779008 token_bytes=312541216 weight_movement_bytes=56623104 total_nvlink_bytes=369164320 weight_to_token_ratio=0.181170 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038743761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229856 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156795360 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360864 weight_movement_bytes=18874368 total_nvlink_bytes=332235232 weight_to_token_ratio=0.060232 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229288 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770216 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311144 weight_movement_bytes=18874368 total_nvlink_bytes=332185512 weight_to_token_ratio=0.060242 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38220 dup_rows=21890 dup_rows_remote=19246 dedup_ratio=0.334015 router_tokens_per_expert=[262, 261, 266, 249, 270, 259, 249, 244, 242, 272, 256, 234, 234, 228, 269, 240, 238, 247, 255, 250, 257, 240, 269, 254, 256, 272, 248, 279, 253, 256, 277, 267, 275, 258, 266, 259, 253, 247, 262, 250, 274, 226, 251, 259, 255, 247, 239, 222, 279, 253, 247, 246, 247, 242, 248, 275, 265, 237, 274, 234, 265, 244, 261, 271, 237, 249, 253, 256, 240, 243, 271, 259, 290, 248, 249, 251, 256, 265, 259, 259, 273, 249, 274, 248, 280, 268, 257, 243, 269, 259, 308, 247, 265, 248, 274, 246, 244, 281, 263, 237, 254, 241, 244, 253, 241, 236, 273, 244, 254, 240, 273, 250, 285, 264, 240, 262, 256, 262, 265, 266, 275, 258, 262, 266, 252, 247, 254, 249] payload_rows_per_dst=[5426, 5496, 5600, 5366, 5416, 5566, 5450, 5326] dup_rows_per_dst=[2644, 3000, 3018, 2550, 2754, 2850, 2606, 2468] combine_rows_per_src=[5426, 5496, 5600, 5366, 5416, 5566, 5450, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228640 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156949792 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313670944 weight_movement_bytes=18874368 total_nvlink_bytes=332545312 weight_to_token_ratio=0.060173 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228584 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156237032 ingress_bytes=212631552 link_bytes=212631552 token_bytes=312245480 weight_movement_bytes=56623104 total_nvlink_bytes=368868584 weight_to_token_ratio=0.181342 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037330910e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38344 dup_rows=21760 dup_rows_remote=19058 dedup_ratio=0.332031 router_tokens_per_expert=[264, 246, 258, 239, 247, 265, 269, 237, 244, 223, 257, 223, 249, 231, 282, 250, 270, 249, 257, 226, 264, 257, 270, 228, 228, 262, 257, 255, 270, 251, 267, 234, 255, 248, 272, 258, 269, 244, 273, 263, 270, 262, 270, 277, 273, 274, 275, 283, 251, 268, 249, 242, 242, 258, 245, 250, 276, 260, 231, 241, 276, 240, 264, 257, 254, 214, 270, 271, 254, 272, 228, 270, 250, 275, 241, 245, 234, 251, 280, 258, 227, 265, 269, 251, 229, 260, 263, 272, 266, 248, 270, 254, 257, 266, 252, 248, 264, 278, 259, 256, 262, 243, 260, 258, 292, 244, 256, 250, 276, 278, 261, 269, 267, 248, 239, 265, 249, 287, 228, 276, 247, 272, 266, 231, 240, 252, 237, 249] payload_rows_per_dst=[5400, 5384, 5610, 5446, 5432, 5518, 5578, 5408] dup_rows_per_dst=[2568, 2706, 2922, 2654, 2702, 2676, 2834, 2698] combine_rows_per_src=[5400, 5384, 5610, 5446, 5432, 5518, 5578, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38300 dup_rows=21688 dup_rows_remote=19102 dedup_ratio=0.330933 router_tokens_per_expert=[266, 263, 265, 257, 244, 280, 243, 284, 249, 230, 254, 283, 248, 267, 248, 271, 252, 217, 237, 245, 244, 260, 270, 280, 258, 236, 252, 270, 266, 259, 258, 263, 276, 268, 264, 287, 259, 269, 247, 224, 272, 257, 281, 269, 252, 277, 267, 245, 275, 256, 285, 271, 255, 257, 218, 273, 275, 253, 230, 252, 259, 253, 273, 239, 229, 255, 252, 255, 225, 268, 258, 274, 243, 245, 273, 278, 268, 251, 297, 235, 276, 244, 261, 224, 245, 242, 226, 250, 274, 258, 244, 262, 274, 249, 270, 273, 234, 247, 245, 242, 259, 250, 269, 223, 222, 271, 250, 265, 243, 261, 267, 252, 248, 231, 284, 256, 258, 230, 250, 270, 267, 231, 235, 264, 231, 243, 274, 261] payload_rows_per_dst=[5468, 5548, 5558, 5524, 5504, 5476, 5386, 5384] dup_rows_per_dst=[2836, 2586, 2870, 2724, 2708, 2668, 2614, 2682] combine_rows_per_src=[5468, 5548, 5558, 5524, 5504, 5476, 5386, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229864 combine_payload_bytes=156549120 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156778984 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313328104 weight_movement_bytes=56623104 total_nvlink_bytes=369951208 weight_to_token_ratio=0.180715 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229608 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286632 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343656 weight_movement_bytes=18874368 total_nvlink_bytes=333218024 weight_to_token_ratio=0.060044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43948 payload_rows_remote=38386 dup_rows=21588 dup_rows_remote=18832 dedup_ratio=0.329407 router_tokens_per_expert=[238, 244, 276, 269, 233, 286, 257, 250, 257, 279, 235, 258, 274, 264, 266, 242, 265, 270, 264, 250, 256, 249, 266, 275, 262, 266, 248, 277, 265, 279, 252, 268, 270, 260, 274, 236, 237, 260, 271, 256, 242, 257, 261, 280, 246, 246, 227, 250, 254, 260, 246, 251, 236, 248, 266, 243, 239, 242, 274, 242, 215, 259, 250, 243, 245, 270, 260, 255, 252, 252, 275, 250, 243, 252, 274, 230, 254, 260, 279, 271, 229, 265, 267, 256, 257, 243, 234, 262, 270, 259, 281, 237, 235, 253, 254, 289, 260, 216, 238, 249, 260, 243, 255, 256, 249, 256, 267, 246, 236, 264, 275, 249, 280, 242, 239, 272, 263, 259, 251, 259, 256, 268, 252, 250, 248, 274, 279, 263] payload_rows_per_dst=[5460, 5458, 5232, 5472, 5572, 5574, 5618, 5562] dup_rows_per_dst=[2796, 2796, 2414, 2680, 2826, 2616, 2704, 2756] combine_rows_per_src=[5460, 5458, 5232, 5472, 5572, 5574, 5618, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38228 dup_rows=21848 dup_rows_remote=19126 dedup_ratio=0.333374 router_tokens_per_expert=[275, 250, 255, 261, 243, 246, 251, 291, 275, 256, 271, 265, 258, 252, 253, 284, 255, 228, 237, 285, 247, 256, 267, 277, 285, 264, 249, 273, 249, 260, 260, 294, 261, 251, 238, 254, 244, 216, 253, 242, 248, 255, 263, 269, 239, 239, 241, 270, 249, 261, 252, 281, 257, 278, 250, 254, 238, 263, 244, 265, 272, 247, 243, 257, 272, 261, 259, 268, 259, 266, 267, 262, 256, 231, 265, 250, 249, 277, 246, 274, 275, 267, 276, 251, 233, 218, 250, 237, 258, 244, 261, 229, 262, 251, 257, 249, 244, 247, 257, 257, 243, 265, 253, 246, 248, 266, 241, 281, 268, 273, 257, 245, 247, 218, 223, 267, 254, 243, 248, 243, 250, 269, 255, 281, 244, 259, 264, 266] payload_rows_per_dst=[5574, 5500, 5390, 5468, 5514, 5412, 5460, 5370] dup_rows_per_dst=[2798, 2872, 2576, 2754, 2810, 2624, 2722, 2692] combine_rows_per_src=[5574, 5500, 5390, 5468, 5514, 5412, 5460, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229608 combine_payload_bytes=156876800 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157106408 ingress_bytes=213499904 link_bytes=213499904 token_bytes=313983208 weight_movement_bytes=56623104 total_nvlink_bytes=370606312 weight_to_token_ratio=0.180338 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045651032e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38120 dup_rows=21916 dup_rows_remote=19224 dedup_ratio=0.334412 router_tokens_per_expert=[264, 260, 225, 281, 252, 268, 241, 249, 230, 264, 252, 250, 241, 227, 279, 280, 251, 242, 257, 264, 254, 242, 255, 254, 261, 236, 254, 280, 250, 259, 264, 281, 254, 271, 259, 276, 263, 241, 238, 285, 255, 244, 257, 244, 244, 246, 257, 262, 264, 262, 270, 241, 243, 267, 276, 285, 233, 262, 229, 230, 247, 254, 274, 286, 279, 258, 263, 274, 254, 245, 259, 264, 241, 280, 274, 264, 227, 243, 242, 245, 256, 242, 260, 239, 271, 238, 256, 275, 262, 243, 268, 270, 239, 262, 232, 261, 248, 225, 244, 244, 250, 283, 251, 238, 266, 257, 251, 249, 243, 249, 249, 277, 269, 274, 260, 269, 268, 255, 263, 258, 245, 273, 259, 234, 262, 267, 242, 274] payload_rows_per_dst=[5442, 5398, 5500, 5444, 5506, 5392, 5422, 5516] dup_rows_per_dst=[2684, 2810, 2692, 2802, 2718, 2756, 2626, 2828] combine_rows_per_src=[5442, 5398, 5500, 5444, 5506, 5392, 5422, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38334 dup_rows=21710 dup_rows_remote=19130 dedup_ratio=0.331268 router_tokens_per_expert=[248, 256, 249, 260, 254, 248, 240, 290, 254, 274, 239, 279, 255, 256, 238, 255, 251, 273, 247, 257, 227, 258, 230, 228, 286, 260, 260, 285, 266, 238, 277, 259, 235, 261, 253, 277, 258, 266, 247, 247, 222, 270, 254, 251, 243, 290, 232, 263, 226, 258, 253, 251, 246, 240, 249, 238, 254, 267, 254, 256, 252, 253, 276, 263, 245, 243, 271, 267, 246, 259, 267, 263, 246, 246, 263, 261, 279, 228, 263, 283, 238, 240, 256, 292, 258, 261, 269, 259, 253, 269, 244, 266, 291, 255, 264, 245, 242, 228, 273, 244, 253, 286, 249, 263, 238, 268, 241, 255, 281, 264, 272, 247, 266, 278, 268, 234, 224, 230, 262, 253, 265, 244, 280, 252, 243, 246, 244, 283] payload_rows_per_dst=[5472, 5490, 5446, 5492, 5458, 5578, 5470, 5420] dup_rows_per_dst=[2718, 2714, 2692, 2580, 2802, 2742, 2738, 2724] combine_rows_per_src=[5472, 5490, 5446, 5492, 5458, 5578, 5470, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38206 dup_rows=21784 dup_rows_remote=18960 dedup_ratio=0.332397 router_tokens_per_expert=[247, 253, 241, 254, 235, 251, 258, 287, 243, 242, 261, 245, 280, 246, 287, 241, 247, 239, 233, 270, 267, 276, 253, 258, 268, 263, 265, 237, 278, 226, 269, 260, 255, 229, 268, 276, 294, 292, 229, 248, 240, 276, 257, 239, 239, 255, 260, 253, 241, 267, 252, 266, 274, 235, 256, 244, 228, 248, 268, 270, 241, 249, 243, 254, 251, 236, 246, 228, 236, 264, 235, 247, 266, 262, 222, 268, 253, 276, 230, 241, 280, 269, 254, 263, 277, 281, 236, 244, 260, 251, 301, 259, 236, 245, 261, 268, 271, 263, 247, 254, 264, 251, 245, 263, 238, 250, 248, 264, 243, 273, 258, 266, 246, 280, 261, 270, 263, 255, 255, 238, 287, 242, 298, 249, 274, 256, 264, 260] payload_rows_per_dst=[5442, 5544, 5492, 5376, 5338, 5546, 5506, 5508] dup_rows_per_dst=[2700, 2674, 2728, 2696, 2584, 2824, 2690, 2888] combine_rows_per_src=[5442, 5544, 5492, 5376, 5338, 5546, 5506, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=228872 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157457928 ingress_bytes=194977792 link_bytes=194977792 token_bytes=314686984 weight_movement_bytes=37748736 total_nvlink_bytes=352435720 weight_to_token_ratio=0.119956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868181268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229416 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156811304 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313393192 weight_movement_bytes=18874368 total_nvlink_bytes=332267560 weight_to_token_ratio=0.060226 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229376 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156368896 ingress_bytes=231636992 link_bytes=231636992 token_bytes=312508416 weight_movement_bytes=75497472 total_nvlink_bytes=388005888 weight_to_token_ratio=0.241585 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219431685e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38194 dup_rows=21774 dup_rows_remote=18910 dedup_ratio=0.332245 router_tokens_per_expert=[254, 261, 250, 262, 233, 256, 248, 245, 254, 269, 240, 263, 260, 248, 264, 251, 257, 247, 267, 240, 246, 259, 245, 225, 238, 262, 241, 233, 272, 238, 258, 256, 249, 281, 258, 238, 255, 268, 258, 252, 245, 264, 240, 260, 248, 246, 259, 249, 283, 228, 254, 272, 245, 228, 265, 237, 227, 274, 248, 274, 279, 259, 250, 271, 260, 250, 270, 274, 263, 264, 267, 239, 229, 281, 262, 241, 271, 256, 257, 252, 268, 268, 245, 262, 261, 266, 239, 261, 258, 256, 252, 281, 256, 254, 248, 277, 238, 261, 253, 293, 256, 247, 256, 263, 263, 273, 238, 278, 261, 262, 261, 256, 261, 231, 248, 271, 256, 251, 253, 257, 267, 247, 253, 279, 257, 269, 250, 265] payload_rows_per_dst=[5568, 5454, 5556, 5592, 5514, 5278, 5512, 5288] dup_rows_per_dst=[2864, 2722, 2856, 2856, 2816, 2392, 2806, 2462] combine_rows_per_src=[5568, 5454, 5556, 5592, 5514, 5278, 5512, 5288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229856 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245920 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261984 weight_movement_bytes=18874368 total_nvlink_bytes=333136352 weight_to_token_ratio=0.060059 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38236 dup_rows=21830 dup_rows_remote=19112 dedup_ratio=0.333099 router_tokens_per_expert=[287, 243, 238, 235, 284, 263, 250, 263, 252, 265, 243, 260, 259, 256, 246, 292, 261, 254, 263, 266, 258, 268, 251, 235, 250, 249, 247, 267, 272, 256, 253, 278, 247, 260, 244, 245, 236, 231, 248, 249, 258, 252, 230, 268, 258, 251, 275, 264, 265, 239, 262, 243, 241, 273, 270, 263, 248, 243, 233, 263, 263, 245, 261, 272, 264, 258, 246, 267, 242, 263, 241, 264, 271, 241, 256, 249, 275, 275, 235, 247, 257, 265, 232, 242, 262, 251, 228, 241, 260, 274, 282, 239, 258, 246, 272, 246, 272, 252, 287, 257, 246, 275, 226, 282, 255, 258, 249, 274, 237, 259, 256, 267, 269, 231, 233, 238, 283, 266, 250, 269, 258, 234, 259, 268, 252, 259, 266, 268] payload_rows_per_dst=[5514, 5494, 5386, 5492, 5470, 5486, 5460, 5404] dup_rows_per_dst=[2758, 2762, 2646, 2676, 2718, 2624, 2844, 2802] combine_rows_per_src=[5514, 5494, 5386, 5492, 5470, 5486, 5460, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228664 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156720440 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313212216 weight_movement_bytes=18874368 total_nvlink_bytes=332086584 weight_to_token_ratio=0.060261 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38138 dup_rows=21888 dup_rows_remote=19242 dedup_ratio=0.333984 router_tokens_per_expert=[261, 261, 254, 262, 234, 237, 255, 237, 268, 271, 260, 247, 257, 260, 245, 241, 274, 254, 246, 255, 272, 267, 239, 243, 247, 256, 261, 244, 263, 242, 253, 262, 268, 261, 259, 263, 255, 236, 255, 232, 273, 263, 241, 233, 251, 253, 283, 234, 267, 257, 256, 227, 272, 264, 241, 260, 279, 245, 261, 246, 250, 235, 259, 264, 256, 267, 273, 262, 256, 266, 257, 226, 262, 268, 264, 252, 252, 249, 286, 242, 274, 254, 246, 240, 253, 266, 244, 238, 222, 255, 262, 259, 272, 281, 266, 269, 259, 242, 268, 237, 232, 254, 258, 286, 224, 266, 270, 251, 278, 267, 229, 249, 256, 259, 272, 258, 290, 273, 248, 292, 250, 264, 226, 258, 271, 271, 248, 252] payload_rows_per_dst=[5452, 5510, 5408, 5408, 5504, 5496, 5412, 5458] dup_rows_per_dst=[2648, 2646, 2712, 2758, 2772, 2706, 2728, 2918] combine_rows_per_src=[5452, 5510, 5408, 5408, 5504, 5496, 5412, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228416 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671040 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313113664 weight_movement_bytes=18874368 total_nvlink_bytes=331988032 weight_to_token_ratio=0.060280 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229392 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156844048 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313458704 weight_movement_bytes=18874368 total_nvlink_bytes=332333072 weight_to_token_ratio=0.060213 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229520 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156442768 ingress_bytes=193961984 link_bytes=193961984 token_bytes=312656016 weight_movement_bytes=37748736 total_nvlink_bytes=350404752 weight_to_token_ratio=0.120736 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858448295e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38128 dup_rows=21872 dup_rows_remote=19062 dedup_ratio=0.333740 router_tokens_per_expert=[247, 249, 250, 276, 256, 268, 261, 283, 274, 282, 265, 263, 245, 254, 232, 230, 281, 253, 271, 269, 244, 233, 260, 270, 246, 260, 241, 257, 285, 243, 248, 255, 257, 225, 227, 246, 243, 256, 249, 262, 257, 241, 275, 238, 266, 252, 255, 256, 252, 246, 238, 247, 265, 263, 268, 243, 250, 274, 249, 230, 273, 253, 264, 255, 264, 263, 254, 281, 247, 256, 261, 237, 249, 259, 268, 259, 238, 235, 260, 238, 261, 284, 267, 263, 241, 257, 285, 246, 273, 262, 231, 259, 243, 244, 241, 266, 244, 251, 277, 275, 258, 253, 241, 258, 256, 230, 269, 246, 254, 257, 254, 254, 242, 285, 269, 247, 248, 244, 242, 277, 255, 239, 269, 304, 285, 244, 268, 255] payload_rows_per_dst=[5266, 5414, 5354, 5650, 5502, 5534, 5408, 5536] dup_rows_per_dst=[2544, 2694, 2656, 2950, 2636, 2836, 2746, 2810] combine_rows_per_src=[5266, 5414, 5354, 5650, 5502, 5534, 5408, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38328 dup_rows=21656 dup_rows_remote=19076 dedup_ratio=0.330444 router_tokens_per_expert=[250, 265, 249, 263, 238, 255, 246, 247, 264, 228, 262, 268, 232, 256, 284, 261, 265, 256, 241, 254, 243, 256, 274, 253, 268, 233, 273, 250, 258, 257, 273, 253, 247, 228, 270, 268, 274, 224, 262, 273, 251, 294, 258, 225, 274, 262, 232, 224, 296, 244, 231, 271, 266, 270, 246, 269, 255, 274, 253, 273, 265, 293, 240, 247, 258, 259, 266, 243, 276, 224, 263, 241, 264, 261, 249, 235, 241, 246, 279, 248, 260, 293, 274, 251, 264, 234, 241, 223, 283, 259, 262, 241, 255, 253, 285, 279, 242, 258, 248, 266, 242, 249, 246, 260, 253, 256, 267, 255, 246, 253, 250, 246, 258, 277, 260, 253, 275, 249, 246, 240, 247, 265, 259, 263, 239, 251, 251, 254] payload_rows_per_dst=[5462, 5486, 5552, 5540, 5488, 5530, 5402, 5420] dup_rows_per_dst=[2674, 2728, 2580, 2846, 2618, 2784, 2672, 2754] combine_rows_per_src=[5462, 5486, 5552, 5540, 5488, 5530, 5402, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38144 dup_rows=21828 dup_rows_remote=19010 dedup_ratio=0.333069 router_tokens_per_expert=[250, 240, 236, 257, 229, 262, 271, 243, 274, 258, 222, 236, 240, 246, 234, 260, 242, 245, 229, 272, 251, 237, 272, 271, 261, 255, 264, 279, 275, 244, 259, 267, 270, 242, 272, 237, 266, 272, 253, 262, 264, 265, 247, 267, 267, 288, 260, 265, 292, 273, 237, 236, 254, 230, 267, 255, 259, 252, 260, 273, 270, 252, 262, 240, 254, 273, 245, 260, 261, 259, 276, 241, 275, 256, 285, 250, 256, 261, 256, 252, 232, 253, 257, 246, 229, 249, 242, 256, 242, 252, 233, 272, 242, 295, 247, 247, 267, 251, 263, 244, 263, 240, 271, 290, 259, 272, 277, 266, 272, 267, 256, 233, 270, 251, 247, 246, 248, 258, 240, 256, 243, 249, 246, 268, 277, 237, 222, 275] payload_rows_per_dst=[5340, 5468, 5542, 5450, 5462, 5432, 5564, 5450] dup_rows_per_dst=[2576, 2778, 2852, 2774, 2858, 2556, 2818, 2616] combine_rows_per_src=[5340, 5468, 5542, 5450, 5462, 5432, 5564, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38254 dup_rows=21802 dup_rows_remote=18996 dedup_ratio=0.332672 router_tokens_per_expert=[265, 271, 264, 261, 233, 282, 251, 257, 261, 293, 259, 262, 256, 247, 218, 260, 243, 267, 267, 285, 273, 229, 252, 255, 287, 293, 266, 244, 273, 233, 242, 256, 266, 270, 265, 232, 254, 266, 238, 257, 265, 265, 278, 239, 231, 274, 255, 269, 242, 254, 276, 239, 245, 292, 241, 241, 310, 269, 261, 257, 244, 269, 251, 252, 252, 259, 258, 246, 242, 233, 246, 259, 282, 262, 248, 233, 245, 241, 260, 230, 260, 242, 281, 226, 241, 252, 255, 230, 270, 213, 266, 264, 264, 263, 257, 261, 261, 244, 243, 244, 272, 261, 266, 252, 253, 231, 260, 256, 241, 262, 291, 249, 254, 253, 260, 257, 260, 256, 252, 243, 268, 247, 264, 265, 261, 238, 244, 247] payload_rows_per_dst=[5478, 5526, 5516, 5480, 5408, 5364, 5508, 5454] dup_rows_per_dst=[2802, 2804, 2732, 2806, 2584, 2726, 2664, 2684] combine_rows_per_src=[5478, 5526, 5516, 5480, 5408, 5364, 5508, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38408 dup_rows=21640 dup_rows_remote=18992 dedup_ratio=0.330200 router_tokens_per_expert=[262, 252, 299, 270, 287, 248, 273, 218, 252, 265, 268, 258, 279, 240, 279, 258, 263, 240, 258, 256, 259, 275, 240, 264, 265, 241, 280, 242, 262, 251, 239, 262, 233, 263, 248, 243, 271, 260, 240, 304, 239, 261, 272, 254, 251, 254, 280, 251, 235, 263, 266, 239, 244, 237, 243, 236, 244, 244, 280, 282, 253, 263, 239, 245, 237, 255, 250, 264, 244, 236, 251, 288, 258, 246, 247, 253, 247, 249, 250, 248, 272, 270, 228, 263, 227, 237, 263, 276, 243, 246, 256, 258, 241, 270, 254, 264, 276, 262, 242, 238, 270, 260, 262, 261, 246, 236, 265, 247, 257, 266, 264, 268, 249, 255, 278, 241, 268, 256, 255, 254, 240, 250, 245, 271, 256, 266, 258, 273] payload_rows_per_dst=[5618, 5484, 5462, 5438, 5372, 5488, 5552, 5482] dup_rows_per_dst=[2798, 2710, 2786, 2588, 2674, 2648, 2688, 2748] combine_rows_per_src=[5618, 5484, 5462, 5438, 5372, 5488, 5552, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=228760 combine_payload_bytes=156172288 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156401048 ingress_bytes=212795392 link_bytes=212795392 token_bytes=312573336 weight_movement_bytes=56623104 total_nvlink_bytes=369196440 weight_to_token_ratio=0.181151 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038900744e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229616 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157221104 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212592 weight_movement_bytes=18874368 total_nvlink_bytes=333086960 weight_to_token_ratio=0.060069 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=228616 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156466440 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312704264 weight_movement_bytes=18874368 total_nvlink_bytes=331578632 weight_to_token_ratio=0.060359 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229000 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917384 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605768 weight_movement_bytes=18874368 total_nvlink_bytes=332480136 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38108 dup_rows=21932 dup_rows_remote=19000 dedup_ratio=0.334656 router_tokens_per_expert=[262, 252, 275, 245, 256, 242, 262, 256, 254, 275, 259, 274, 255, 236, 285, 255, 269, 242, 253, 283, 238, 236, 251, 264, 249, 259, 262, 309, 265, 266, 248, 242, 270, 274, 261, 259, 263, 260, 256, 238, 264, 249, 276, 264, 259, 243, 250, 267, 247, 270, 234, 234, 233, 233, 277, 242, 262, 277, 261, 278, 235, 245, 227, 269, 232, 245, 256, 216, 255, 233, 280, 256, 269, 265, 257, 273, 269, 269, 254, 252, 275, 244, 253, 258, 246, 236, 277, 235, 245, 271, 256, 255, 307, 252, 252, 253, 291, 253, 248, 246, 264, 275, 272, 261, 229, 248, 244, 255, 237, 261, 268, 251, 271, 247, 260, 260, 227, 248, 241, 218, 260, 245, 226, 283, 265, 266, 239, 257] payload_rows_per_dst=[5496, 5530, 5670, 5416, 5462, 5276, 5448, 5306] dup_rows_per_dst=[2932, 2744, 3056, 2632, 2734, 2534, 2758, 2542] combine_rows_per_src=[5496, 5530, 5670, 5416, 5462, 5276, 5448, 5306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43478 payload_rows_remote=38062 dup_rows=22058 dup_rows_remote=19232 dedup_ratio=0.336578 router_tokens_per_expert=[274, 257, 246, 275, 261, 253, 265, 273, 261, 228, 224, 251, 254, 264, 277, 249, 215, 277, 278, 253, 249, 256, 233, 283, 269, 246, 259, 283, 239, 261, 263, 267, 258, 254, 258, 239, 258, 263, 238, 245, 241, 232, 244, 242, 292, 283, 258, 250, 259, 252, 242, 270, 257, 257, 286, 263, 241, 258, 253, 254, 276, 238, 274, 227, 263, 267, 243, 262, 274, 254, 259, 281, 234, 246, 252, 260, 267, 271, 244, 244, 254, 278, 271, 250, 232, 247, 269, 249, 252, 283, 265, 273, 258, 227, 237, 270, 281, 262, 259, 254, 269, 261, 257, 261, 242, 266, 283, 245, 252, 240, 240, 283, 231, 253, 245, 282, 236, 247, 255, 257, 236, 274, 228, 221, 248, 251, 254, 254] payload_rows_per_dst=[5414, 5466, 5458, 5462, 5416, 5442, 5464, 5356] dup_rows_per_dst=[2810, 2796, 2652, 2752, 2826, 2788, 2846, 2588] combine_rows_per_src=[5414, 5466, 5458, 5462, 5416, 5442, 5464, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38224 dup_rows=21796 dup_rows_remote=19122 dedup_ratio=0.332581 router_tokens_per_expert=[256, 272, 244, 247, 254, 270, 251, 269, 261, 256, 251, 282, 253, 241, 237, 266, 263, 264, 269, 254, 211, 262, 282, 277, 261, 240, 242, 245, 258, 264, 214, 289, 243, 274, 289, 252, 234, 245, 262, 275, 238, 270, 224, 262, 267, 235, 257, 260, 242, 250, 298, 253, 281, 267, 227, 260, 261, 263, 243, 286, 266, 249, 210, 266, 241, 267, 257, 259, 276, 264, 248, 253, 250, 223, 248, 244, 266, 255, 242, 246, 259, 259, 300, 276, 275, 208, 254, 214, 268, 275, 270, 258, 215, 268, 258, 267, 244, 248, 266, 252, 257, 266, 238, 246, 287, 257, 266, 274, 249, 261, 252, 245, 254, 272, 227, 263, 267, 236, 273, 250, 263, 263, 257, 227, 250, 279, 256, 246] payload_rows_per_dst=[5466, 5516, 5454, 5502, 5434, 5480, 5412, 5476] dup_rows_per_dst=[2754, 2674, 2720, 2742, 2644, 2768, 2804, 2690] combine_rows_per_src=[5466, 5516, 5454, 5502, 5434, 5480, 5412, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229600 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157548768 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314867936 weight_movement_bytes=37748736 total_nvlink_bytes=352616672 weight_to_token_ratio=0.119888 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=228432 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156318800 ingress_bytes=193839104 link_bytes=193839104 token_bytes=312409168 weight_movement_bytes=37748736 total_nvlink_bytes=350157904 weight_to_token_ratio=0.120831 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857270920e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=229176 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156131128 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312033080 weight_movement_bytes=18874368 total_nvlink_bytes=330907448 weight_to_token_ratio=0.060488 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229384 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156794888 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360392 weight_movement_bytes=56623104 total_nvlink_bytes=369983496 weight_to_token_ratio=0.180696 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38132 dup_rows=21882 dup_rows_remote=18994 dedup_ratio=0.333893 router_tokens_per_expert=[284, 252, 256, 244, 270, 247, 243, 277, 257, 246, 239, 288, 254, 258, 264, 244, 255, 238, 245, 268, 280, 232, 243, 251, 241, 243, 258, 246, 275, 268, 256, 285, 274, 251, 289, 233, 257, 257, 242, 251, 251, 258, 231, 250, 241, 257, 229, 244, 254, 273, 252, 246, 268, 264, 261, 239, 253, 257, 244, 241, 258, 229, 255, 255, 246, 240, 256, 269, 257, 254, 257, 257, 232, 267, 247, 271, 265, 231, 277, 260, 247, 255, 288, 261, 231, 234, 266, 262, 271, 240, 264, 266, 244, 260, 246, 241, 266, 257, 269, 248, 285, 274, 245, 301, 250, 234, 276, 244, 254, 275, 276, 270, 230, 248, 267, 256, 239, 259, 253, 235, 268, 266, 261, 278, 293, 236, 274, 248] payload_rows_per_dst=[5388, 5380, 5508, 5426, 5388, 5428, 5614, 5522] dup_rows_per_dst=[2702, 2600, 2678, 2672, 2784, 2724, 2834, 2888] combine_rows_per_src=[5388, 5380, 5508, 5426, 5388, 5428, 5614, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38326 dup_rows=21848 dup_rows_remote=19100 dedup_ratio=0.333374 router_tokens_per_expert=[254, 243, 256, 247, 242, 261, 283, 254, 261, 249, 249, 273, 242, 255, 270, 262, 261, 268, 243, 249, 237, 263, 241, 271, 262, 266, 276, 257, 260, 241, 247, 244, 259, 262, 237, 256, 229, 237, 238, 268, 265, 253, 264, 270, 251, 247, 248, 271, 251, 273, 237, 252, 237, 273, 242, 237, 264, 266, 293, 244, 240, 265, 232, 241, 260, 228, 264, 231, 246, 243, 260, 256, 267, 260, 262, 232, 269, 241, 254, 251, 244, 247, 247, 261, 278, 261, 252, 236, 245, 274, 273, 259, 255, 278, 275, 240, 261, 260, 276, 245, 270, 245, 245, 280, 267, 233, 258, 258, 257, 258, 253, 258, 276, 251, 267, 269, 267, 243, 253, 238, 273, 274, 248, 272, 268, 277, 246, 284] payload_rows_per_dst=[5508, 5440, 5362, 5410, 5394, 5552, 5440, 5582] dup_rows_per_dst=[2694, 2732, 2748, 2684, 2654, 2698, 2808, 2830] combine_rows_per_src=[5508, 5440, 5362, 5410, 5394, 5552, 5440, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38352 dup_rows=21648 dup_rows_remote=18990 dedup_ratio=0.330322 router_tokens_per_expert=[226, 256, 269, 284, 271, 241, 258, 246, 279, 265, 257, 259, 243, 288, 249, 248, 245, 231, 254, 247, 285, 270, 228, 270, 269, 286, 278, 270, 240, 254, 258, 253, 272, 232, 247, 261, 239, 268, 266, 269, 271, 255, 271, 277, 263, 265, 247, 274, 246, 271, 260, 232, 269, 253, 221, 232, 255, 245, 254, 259, 250, 256, 248, 273, 272, 252, 281, 230, 257, 259, 267, 279, 274, 276, 236, 243, 258, 237, 243, 253, 267, 249, 266, 259, 265, 269, 254, 248, 231, 287, 259, 272, 253, 245, 219, 280, 251, 248, 276, 273, 262, 259, 282, 280, 265, 268, 228, 245, 244, 251, 234, 231, 239, 265, 260, 235, 253, 242, 237, 238, 244, 253, 235, 236, 253, 230, 276, 257] payload_rows_per_dst=[5542, 5568, 5524, 5366, 5552, 5432, 5536, 5368] dup_rows_per_dst=[2736, 2708, 2830, 2682, 2682, 2814, 2658, 2538] combine_rows_per_src=[5542, 5568, 5524, 5366, 5552, 5432, 5536, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38238 dup_rows=21782 dup_rows_remote=18966 dedup_ratio=0.332367 router_tokens_per_expert=[271, 232, 261, 250, 237, 241, 237, 245, 282, 256, 253, 254, 249, 253, 255, 273, 262, 237, 278, 249, 256, 268, 238, 266, 243, 252, 261, 261, 234, 257, 259, 262, 266, 259, 250, 260, 263, 235, 248, 244, 257, 269, 252, 264, 247, 259, 231, 279, 273, 245, 255, 275, 264, 289, 258, 233, 252, 256, 256, 251, 274, 241, 281, 263, 251, 255, 240, 265, 283, 255, 270, 234, 273, 240, 247, 239, 268, 265, 262, 254, 254, 272, 250, 242, 237, 237, 248, 230, 246, 256, 253, 242, 259, 254, 256, 274, 245, 246, 277, 254, 302, 252, 268, 226, 275, 246, 258, 246, 252, 233, 261, 243, 269, 254, 284, 269, 258, 267, 259, 272, 243, 252, 257, 259, 258, 255, 273, 263] payload_rows_per_dst=[5460, 5410, 5490, 5516, 5450, 5390, 5468, 5570] dup_rows_per_dst=[2638, 2756, 2676, 2816, 2752, 2630, 2700, 2814] combine_rows_per_src=[5460, 5410, 5490, 5516, 5450, 5390, 5468, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=228504 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156417176 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312605848 weight_movement_bytes=18874368 total_nvlink_bytes=331480216 weight_to_token_ratio=0.060378 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229704 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213000 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196296 weight_movement_bytes=18874368 total_nvlink_bytes=333070664 weight_to_token_ratio=0.060072 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38178 dup_rows=21896 dup_rows_remote=19204 dedup_ratio=0.334106 router_tokens_per_expert=[236, 260, 253, 262, 266, 255, 251, 266, 268, 242, 272, 268, 261, 271, 235, 262, 275, 264, 249, 263, 284, 249, 251, 248, 262, 257, 239, 221, 250, 275, 251, 266, 290, 221, 269, 262, 271, 252, 248, 236, 229, 265, 270, 251, 237, 258, 268, 263, 248, 251, 273, 283, 230, 240, 267, 226, 242, 284, 255, 257, 259, 269, 253, 262, 267, 242, 240, 274, 232, 271, 253, 256, 242, 249, 250, 236, 251, 253, 275, 249, 259, 257, 263, 274, 264, 223, 244, 256, 258, 272, 251, 277, 235, 240, 252, 252, 255, 257, 261, 272, 254, 266, 271, 266, 256, 249, 268, 262, 276, 255, 257, 274, 258, 250, 242, 223, 242, 257, 241, 247, 270, 239, 258, 270, 235, 261, 281, 257] payload_rows_per_dst=[5422, 5542, 5394, 5462, 5408, 5462, 5550, 5400] dup_rows_per_dst=[2834, 2666, 2786, 2736, 2672, 2692, 2848, 2662] combine_rows_per_src=[5422, 5542, 5394, 5462, 5408, 5462, 5550, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228816 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156851664 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474512 weight_movement_bytes=18874368 total_nvlink_bytes=332348880 weight_to_token_ratio=0.060210 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229368 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157319160 ingress_bytes=194838528 link_bytes=194838528 token_bytes=314408952 weight_movement_bytes=37748736 total_nvlink_bytes=352157688 weight_to_token_ratio=0.120063 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866846909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=37944 dup_rows=21952 dup_rows_remote=18884 dedup_ratio=0.334961 router_tokens_per_expert=[265, 252, 272, 243, 255, 283, 240, 280, 238, 248, 252, 271, 267, 265, 247, 242, 255, 243, 298, 263, 255, 279, 238, 263, 259, 239, 244, 251, 260, 256, 260, 266, 264, 237, 260, 262, 253, 240, 278, 256, 266, 226, 277, 245, 240, 251, 286, 241, 219, 270, 273, 256, 223, 248, 259, 252, 246, 288, 242, 245, 260, 263, 254, 261, 240, 264, 268, 251, 273, 269, 260, 284, 251, 251, 244, 259, 258, 290, 256, 234, 233, 273, 250, 255, 250, 255, 259, 255, 263, 272, 237, 240, 251, 283, 268, 226, 236, 247, 256, 271, 240, 258, 287, 237, 245, 248, 245, 249, 273, 246, 265, 251, 259, 283, 272, 261, 254, 254, 257, 228, 255, 282, 244, 252, 266, 255, 255, 225] payload_rows_per_dst=[5640, 5468, 5328, 5472, 5470, 5416, 5410, 5380] dup_rows_per_dst=[3068, 2790, 2404, 2646, 2834, 2724, 2698, 2788] combine_rows_per_src=[5640, 5468, 5328, 5472, 5470, 5416, 5410, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38288 dup_rows=21842 dup_rows_remote=19076 dedup_ratio=0.333282 router_tokens_per_expert=[257, 277, 251, 272, 259, 282, 220, 258, 262, 268, 266, 215, 240, 271, 263, 234, 261, 276, 253, 253, 232, 228, 273, 240, 268, 244, 249, 235, 235, 269, 250, 256, 255, 250, 264, 252, 269, 293, 244, 271, 266, 259, 248, 242, 261, 270, 250, 252, 269, 280, 247, 274, 248, 261, 249, 247, 262, 240, 251, 238, 241, 232, 217, 279, 232, 244, 259, 287, 257, 258, 291, 225, 244, 244, 280, 247, 232, 283, 260, 243, 280, 243, 245, 256, 238, 248, 275, 263, 280, 272, 273, 251, 295, 253, 237, 258, 268, 262, 247, 255, 229, 258, 274, 297, 267, 268, 277, 249, 251, 244, 254, 251, 248, 261, 280, 281, 246, 273, 252, 235, 252, 266, 261, 213, 252, 272, 230, 244] payload_rows_per_dst=[5420, 5458, 5492, 5354, 5406, 5576, 5550, 5438] dup_rows_per_dst=[2770, 2586, 2800, 2716, 2766, 2758, 2752, 2694] combine_rows_per_src=[5420, 5458, 5492, 5354, 5406, 5576, 5550, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38396 dup_rows=21678 dup_rows_remote=18904 dedup_ratio=0.330780 router_tokens_per_expert=[285, 262, 243, 259, 252, 250, 281, 284, 269, 270, 273, 259, 243, 224, 258, 219, 282, 259, 241, 270, 249, 256, 263, 278, 276, 239, 258, 248, 271, 239, 233, 256, 240, 257, 282, 233, 249, 285, 259, 226, 243, 248, 289, 257, 251, 278, 241, 263, 248, 242, 226, 283, 244, 247, 261, 263, 246, 283, 239, 242, 238, 252, 258, 263, 243, 280, 247, 263, 283, 267, 251, 241, 245, 278, 286, 251, 247, 256, 262, 281, 267, 234, 271, 233, 270, 269, 267, 244, 257, 273, 232, 260, 260, 244, 261, 239, 249, 245, 234, 252, 262, 246, 256, 258, 246, 252, 253, 247, 253, 238, 252, 249, 256, 272, 251, 259, 248, 271, 257, 267, 238, 304, 239, 269, 232, 267, 251, 248] payload_rows_per_dst=[5540, 5462, 5490, 5464, 5590, 5472, 5390, 5450] dup_rows_per_dst=[2722, 2774, 2712, 2606, 2772, 2690, 2594, 2808] combine_rows_per_src=[5540, 5462, 5490, 5464, 5590, 5472, 5390, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229528 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606616 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983704 weight_movement_bytes=18874368 total_nvlink_bytes=331858072 weight_to_token_ratio=0.060305 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155418624 dispatch_meta_bytes=227312 combine_payload_bytes=155418624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155645936 ingress_bytes=174292992 link_bytes=174292992 token_bytes=311064560 weight_movement_bytes=18874368 total_nvlink_bytes=329938928 weight_to_token_ratio=0.060677 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669989692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229456 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057104 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884752 weight_movement_bytes=37748736 total_nvlink_bytes=351633488 weight_to_token_ratio=0.120263 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229200 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157499216 ingress_bytes=176144384 link_bytes=176144384 token_bytes=314769232 weight_movement_bytes=18874368 total_nvlink_bytes=333643600 weight_to_token_ratio=0.059963 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687728819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38284 dup_rows=21794 dup_rows_remote=19134 dedup_ratio=0.332550 router_tokens_per_expert=[239, 262, 223, 277, 271, 263, 245, 234, 266, 251, 265, 266, 261, 260, 278, 219, 253, 239, 269, 225, 254, 257, 268, 259, 234, 284, 258, 271, 285, 252, 286, 229, 262, 249, 273, 269, 273, 234, 266, 251, 235, 269, 289, 265, 250, 257, 251, 242, 272, 229, 251, 247, 249, 246, 269, 229, 284, 238, 282, 255, 238, 248, 263, 253, 266, 283, 250, 259, 250, 250, 261, 273, 239, 250, 255, 272, 254, 286, 253, 262, 252, 251, 259, 272, 253, 249, 266, 234, 253, 274, 222, 262, 262, 261, 253, 247, 265, 270, 250, 257, 241, 246, 251, 262, 275, 224, 257, 261, 266, 261, 287, 212, 237, 247, 252, 215, 283, 258, 284, 244, 270, 258, 241, 258, 240, 259, 248, 265] payload_rows_per_dst=[5296, 5472, 5496, 5436, 5540, 5458, 5586, 5458] dup_rows_per_dst=[2386, 2774, 2720, 2670, 2786, 2736, 3062, 2660] combine_rows_per_src=[5296, 5472, 5496, 5436, 5540, 5458, 5586, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38214 dup_rows=21836 dup_rows_remote=19112 dedup_ratio=0.333191 router_tokens_per_expert=[265, 242, 259, 288, 253, 238, 263, 251, 236, 248, 255, 260, 304, 265, 267, 254, 264, 259, 247, 252, 246, 252, 240, 270, 256, 279, 274, 259, 272, 238, 261, 267, 236, 255, 281, 270, 256, 261, 258, 230, 272, 261, 249, 274, 268, 237, 260, 237, 258, 235, 287, 272, 236, 247, 254, 237, 226, 242, 255, 254, 274, 266, 261, 291, 241, 243, 257, 278, 254, 261, 272, 262, 262, 242, 233, 274, 267, 254, 249, 266, 265, 255, 276, 250, 290, 258, 264, 232, 274, 234, 249, 231, 251, 265, 241, 253, 240, 259, 219, 251, 246, 258, 221, 240, 233, 245, 245, 253, 254, 268, 252, 273, 256, 279, 281, 234, 265, 241, 277, 241, 249, 251, 223, 253, 276, 256, 275, 267] payload_rows_per_dst=[5494, 5456, 5486, 5462, 5516, 5408, 5366, 5512] dup_rows_per_dst=[2802, 2816, 2724, 2728, 2714, 2768, 2548, 2736] combine_rows_per_src=[5494, 5456, 5486, 5462, 5516, 5408, 5366, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38264 dup_rows=21766 dup_rows_remote=18916 dedup_ratio=0.332123 router_tokens_per_expert=[229, 243, 244, 250, 285, 239, 220, 259, 266, 255, 256, 292, 248, 256, 246, 252, 246, 250, 272, 285, 268, 281, 242, 231, 256, 259, 224, 265, 252, 268, 293, 264, 279, 250, 250, 233, 267, 233, 258, 235, 228, 262, 267, 243, 258, 259, 267, 269, 252, 267, 275, 259, 250, 280, 251, 256, 256, 248, 286, 242, 261, 269, 277, 249, 255, 225, 270, 262, 236, 281, 269, 258, 222, 248, 250, 267, 284, 224, 255, 244, 221, 263, 241, 206, 256, 257, 261, 269, 241, 274, 239, 232, 230, 251, 268, 276, 274, 248, 258, 289, 299, 263, 267, 276, 251, 262, 245, 280, 265, 222, 237, 255, 250, 254, 269, 259, 241, 264, 243, 261, 250, 268, 262, 244, 281, 257, 262, 245] payload_rows_per_dst=[5398, 5572, 5432, 5506, 5388, 5418, 5588, 5468] dup_rows_per_dst=[2682, 2740, 2684, 2850, 2712, 2552, 2794, 2752] combine_rows_per_src=[5398, 5572, 5432, 5506, 5388, 5418, 5588, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38034 dup_rows=21968 dup_rows_remote=19174 dedup_ratio=0.335205 router_tokens_per_expert=[284, 256, 252, 260, 242, 269, 256, 258, 252, 262, 253, 253, 261, 228, 233, 246, 252, 247, 254, 252, 274, 243, 245, 244, 266, 257, 254, 248, 279, 249, 244, 257, 247, 261, 250, 269, 230, 249, 280, 266, 247, 280, 259, 283, 258, 247, 262, 267, 267, 250, 262, 255, 248, 263, 280, 254, 257, 249, 241, 259, 252, 216, 266, 258, 256, 225, 253, 244, 251, 238, 243, 241, 250, 254, 260, 266, 271, 277, 269, 268, 261, 269, 245, 248, 284, 220, 292, 253, 246, 268, 251, 232, 265, 271, 253, 258, 288, 267, 232, 250, 231, 286, 249, 294, 285, 259, 267, 250, 274, 258, 245, 229, 254, 263, 233, 218, 276, 239, 279, 240, 276, 277, 245, 272, 238, 264, 244, 242] payload_rows_per_dst=[5402, 5468, 5538, 5458, 5356, 5422, 5534, 5390] dup_rows_per_dst=[2728, 2662, 2772, 2696, 2776, 2810, 2794, 2730] combine_rows_per_src=[5402, 5468, 5538, 5458, 5356, 5422, 5534, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229304 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156753848 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278392 weight_movement_bytes=56623104 total_nvlink_bytes=369901496 weight_to_token_ratio=0.180744 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38174 dup_rows=21796 dup_rows_remote=19034 dedup_ratio=0.332581 router_tokens_per_expert=[270, 276, 229, 254, 265, 279, 252, 266, 274, 271, 259, 278, 248, 280, 258, 284, 245, 260, 281, 254, 245, 264, 225, 272, 248, 223, 260, 236, 261, 246, 263, 255, 247, 276, 224, 253, 237, 258, 255, 257, 272, 245, 240, 250, 229, 242, 259, 242, 270, 256, 268, 251, 248, 262, 272, 267, 256, 222, 250, 280, 294, 255, 246, 237, 263, 258, 216, 254, 243, 266, 259, 274, 265, 231, 270, 271, 263, 253, 241, 247, 251, 253, 272, 255, 286, 267, 262, 268, 263, 266, 233, 278, 246, 254, 256, 254, 253, 266, 260, 244, 256, 243, 228, 268, 262, 240, 275, 253, 264, 229, 226, 254, 264, 262, 268, 229, 283, 266, 265, 255, 256, 260, 269, 239, 242, 259, 243, 248] payload_rows_per_dst=[5560, 5372, 5304, 5504, 5458, 5566, 5448, 5528] dup_rows_per_dst=[2926, 2704, 2668, 2764, 2690, 2762, 2594, 2688] combine_rows_per_src=[5560, 5372, 5304, 5504, 5458, 5566, 5448, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229672 combine_payload_bytes=156811264 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157040936 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313852200 weight_movement_bytes=37748736 total_nvlink_bytes=351600936 weight_to_token_ratio=0.120276 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38140 dup_rows=21790 dup_rows_remote=19062 dedup_ratio=0.332489 router_tokens_per_expert=[242, 272, 291, 231, 253, 249, 268, 233, 281, 256, 274, 253, 245, 265, 279, 261, 250, 239, 258, 254, 245, 254, 264, 225, 243, 254, 264, 246, 250, 284, 264, 262, 258, 262, 253, 284, 277, 255, 254, 231, 250, 264, 285, 240, 275, 270, 271, 233, 259, 251, 255, 256, 248, 257, 248, 232, 227, 250, 256, 288, 266, 238, 247, 257, 252, 238, 255, 271, 260, 253, 250, 231, 257, 238, 256, 244, 255, 266, 248, 227, 264, 256, 285, 243, 243, 278, 222, 243, 266, 230, 222, 248, 271, 252, 272, 267, 244, 255, 261, 276, 253, 247, 266, 246, 251, 248, 264, 271, 269, 285, 264, 269, 270, 276, 235, 261, 261, 272, 259, 223, 266, 260, 249, 264, 251, 238, 269, 276] payload_rows_per_dst=[5606, 5418, 5544, 5412, 5492, 5354, 5464, 5456] dup_rows_per_dst=[2728, 2710, 2958, 2658, 2796, 2484, 2758, 2698] combine_rows_per_src=[5606, 5418, 5544, 5412, 5492, 5354, 5464, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228720 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958064 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313687408 weight_movement_bytes=37748736 total_nvlink_bytes=351436144 weight_to_token_ratio=0.120339 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=228832 combine_payload_bytes=155787264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156016096 ingress_bytes=231284736 link_bytes=231284736 token_bytes=311803360 weight_movement_bytes=75497472 total_nvlink_bytes=387300832 weight_to_token_ratio=0.242132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216056541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38380 dup_rows=21754 dup_rows_remote=19004 dedup_ratio=0.331940 router_tokens_per_expert=[250, 248, 238, 255, 232, 251, 241, 253, 279, 246, 260, 261, 253, 286, 273, 253, 268, 264, 263, 291, 266, 273, 274, 254, 267, 266, 260, 240, 279, 262, 260, 242, 279, 246, 263, 275, 244, 220, 244, 265, 238, 294, 233, 258, 265, 251, 226, 254, 263, 261, 265, 277, 267, 247, 261, 265, 251, 241, 284, 267, 282, 259, 236, 281, 272, 259, 274, 242, 282, 255, 240, 268, 240, 251, 258, 263, 225, 251, 240, 256, 255, 252, 269, 290, 239, 205, 219, 254, 259, 264, 251, 247, 270, 266, 261, 257, 272, 256, 284, 250, 232, 255, 230, 259, 253, 244, 217, 269, 262, 249, 234, 257, 265, 244, 278, 253, 240, 238, 247, 266, 282, 232, 252, 238, 263, 230, 263, 250] payload_rows_per_dst=[5496, 5616, 5448, 5516, 5402, 5416, 5420, 5468] dup_rows_per_dst=[2662, 2842, 2662, 2898, 2750, 2700, 2626, 2614] combine_rows_per_src=[5496, 5616, 5448, 5516, 5402, 5416, 5420, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38152 dup_rows=21814 dup_rows_remote=19022 dedup_ratio=0.332855 router_tokens_per_expert=[258, 302, 254, 233, 258, 269, 263, 257, 258, 242, 268, 240, 274, 271, 239, 270, 271, 232, 242, 262, 269, 246, 267, 276, 248, 273, 254, 270, 276, 273, 258, 264, 272, 236, 292, 239, 262, 228, 275, 266, 237, 278, 262, 276, 242, 243, 239, 255, 267, 269, 243, 211, 253, 254, 255, 249, 233, 249, 277, 238, 273, 264, 251, 241, 253, 260, 271, 265, 260, 275, 240, 263, 250, 233, 270, 233, 243, 227, 277, 238, 253, 278, 249, 260, 256, 254, 247, 255, 249, 276, 250, 248, 273, 242, 249, 250, 270, 228, 250, 261, 247, 252, 283, 272, 248, 275, 243, 238, 258, 280, 264, 262, 310, 237, 257, 231, 252, 260, 240, 283, 271, 211, 252, 239, 233, 253, 255, 240] payload_rows_per_dst=[5558, 5570, 5480, 5394, 5382, 5460, 5478, 5400] dup_rows_per_dst=[2754, 2792, 2724, 2660, 2734, 2718, 2784, 2648] combine_rows_per_src=[5558, 5570, 5480, 5394, 5382, 5460, 5478, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228832 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156589536 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950240 weight_movement_bytes=37748736 total_nvlink_bytes=350698976 weight_to_token_ratio=0.120622 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228808 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156450248 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312671688 weight_movement_bytes=37748736 total_nvlink_bytes=350420424 weight_to_token_ratio=0.120730 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229536 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434016 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638496 weight_movement_bytes=18874368 total_nvlink_bytes=333512864 weight_to_token_ratio=0.059987 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228696 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156499288 ingress_bytes=212893696 link_bytes=212893696 token_bytes=312769880 weight_movement_bytes=56623104 total_nvlink_bytes=369392984 weight_to_token_ratio=0.181038 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039842645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38190 dup_rows=21890 dup_rows_remote=19156 dedup_ratio=0.334015 router_tokens_per_expert=[274, 275, 270, 270, 261, 249, 221, 264, 276, 277, 268, 273, 271, 277, 228, 235, 251, 257, 277, 251, 209, 282, 270, 252, 253, 266, 239, 272, 270, 243, 293, 239, 264, 260, 248, 244, 264, 264, 235, 268, 248, 202, 257, 296, 266, 270, 248, 261, 265, 268, 262, 252, 269, 262, 251, 222, 249, 242, 241, 252, 276, 247, 240, 246, 253, 249, 239, 273, 237, 249, 224, 240, 268, 261, 252, 239, 271, 252, 259, 248, 243, 251, 233, 267, 271, 254, 239, 267, 235, 227, 253, 255, 247, 239, 249, 255, 270, 282, 265, 262, 279, 265, 269, 233, 244, 257, 278, 274, 269, 264, 239, 267, 241, 235, 263, 232, 241, 262, 273, 280, 279, 260, 258, 278, 241, 259, 260, 238] payload_rows_per_dst=[5478, 5498, 5456, 5406, 5400, 5418, 5516, 5474] dup_rows_per_dst=[2900, 2750, 2734, 2682, 2628, 2552, 2918, 2726] combine_rows_per_src=[5478, 5498, 5456, 5406, 5400, 5418, 5516, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38094 dup_rows=21758 dup_rows_remote=18722 dedup_ratio=0.332001 router_tokens_per_expert=[246, 266, 264, 233, 261, 268, 249, 253, 256, 241, 223, 257, 237, 262, 240, 279, 271, 278, 266, 244, 258, 259, 264, 250, 257, 265, 228, 256, 229, 261, 215, 240, 237, 263, 269, 246, 255, 282, 241, 277, 244, 260, 248, 280, 262, 283, 237, 248, 253, 237, 262, 254, 260, 267, 253, 249, 276, 254, 273, 256, 269, 251, 238, 266, 243, 254, 251, 277, 247, 264, 279, 268, 233, 243, 255, 260, 262, 274, 249, 245, 263, 266, 283, 259, 268, 252, 238, 246, 225, 241, 227, 238, 277, 261, 267, 247, 260, 270, 264, 254, 255, 248, 259, 258, 285, 240, 246, 249, 247, 264, 249, 263, 258, 220, 247, 264, 261, 246, 239, 283, 268, 269, 300, 261, 274, 270, 254, 255] payload_rows_per_dst=[5402, 5042, 5512, 5292, 5704, 5678, 5464, 5684] dup_rows_per_dst=[2594, 2128, 2826, 2398, 3034, 2984, 2758, 3036] combine_rows_per_src=[5402, 5042, 5512, 5292, 5704, 5678, 5464, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38236 dup_rows=21864 dup_rows_remote=19082 dedup_ratio=0.333618 router_tokens_per_expert=[261, 261, 257, 259, 250, 255, 236, 259, 260, 264, 245, 263, 259, 243, 275, 264, 246, 261, 280, 257, 239, 271, 279, 222, 268, 262, 248, 263, 218, 284, 249, 272, 263, 274, 236, 245, 250, 254, 271, 265, 251, 272, 265, 265, 244, 276, 263, 261, 252, 274, 253, 256, 280, 235, 261, 274, 234, 251, 251, 259, 282, 223, 240, 284, 272, 246, 284, 292, 261, 224, 248, 255, 263, 258, 238, 263, 276, 266, 251, 239, 269, 234, 259, 248, 249, 285, 228, 276, 268, 245, 259, 249, 286, 254, 226, 245, 244, 263, 267, 230, 246, 250, 261, 261, 219, 257, 267, 261, 228, 258, 244, 272, 279, 234, 227, 230, 262, 291, 265, 251, 253, 276, 239, 265, 221, 232, 221, 284] payload_rows_per_dst=[5444, 5566, 5548, 5436, 5464, 5418, 5386, 5410] dup_rows_per_dst=[2778, 2672, 2762, 2782, 2808, 2742, 2670, 2650] combine_rows_per_src=[5444, 5566, 5548, 5436, 5464, 5418, 5386, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38454 dup_rows=21606 dup_rows_remote=18844 dedup_ratio=0.329681 router_tokens_per_expert=[289, 254, 261, 275, 277, 240, 252, 256, 222, 249, 259, 273, 254, 260, 233, 280, 252, 239, 252, 246, 257, 259, 274, 266, 273, 245, 258, 265, 250, 293, 245, 262, 256, 246, 252, 271, 269, 272, 248, 260, 277, 267, 280, 266, 257, 262, 256, 286, 272, 257, 245, 242, 239, 228, 251, 244, 243, 257, 263, 244, 273, 257, 244, 233, 287, 272, 244, 272, 238, 216, 249, 251, 269, 247, 281, 253, 259, 260, 257, 222, 238, 264, 264, 270, 233, 251, 265, 282, 267, 274, 278, 260, 269, 277, 243, 236, 259, 242, 249, 284, 245, 230, 273, 257, 253, 266, 241, 213, 233, 258, 225, 265, 257, 259, 252, 265, 243, 257, 260, 284, 222, 238, 242, 271, 252, 258, 242, 238] payload_rows_per_dst=[5420, 5480, 5588, 5396, 5450, 5600, 5476, 5520] dup_rows_per_dst=[2596, 2792, 2862, 2588, 2704, 2742, 2762, 2560] combine_rows_per_src=[5420, 5480, 5588, 5396, 5450, 5600, 5476, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229384 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655624 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081864 weight_movement_bytes=18874368 total_nvlink_bytes=331956232 weight_to_token_ratio=0.060286 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38306 dup_rows=21736 dup_rows_remote=18932 dedup_ratio=0.331665 router_tokens_per_expert=[238, 223, 251, 280, 217, 263, 260, 280, 249, 262, 266, 251, 247, 246, 250, 237, 246, 270, 262, 266, 280, 218, 258, 247, 269, 259, 289, 240, 274, 265, 253, 242, 258, 276, 263, 247, 249, 264, 242, 254, 245, 248, 299, 266, 255, 257, 250, 242, 225, 279, 253, 285, 244, 274, 245, 276, 269, 251, 258, 232, 249, 292, 274, 257, 248, 262, 265, 283, 256, 258, 236, 250, 258, 254, 253, 245, 256, 258, 249, 218, 269, 262, 281, 256, 268, 255, 256, 261, 264, 251, 232, 270, 267, 246, 258, 253, 239, 234, 253, 243, 224, 293, 250, 250, 262, 242, 269, 273, 267, 243, 246, 246, 251, 273, 263, 249, 277, 248, 248, 272, 268, 252, 256, 271, 260, 246, 238, 228] payload_rows_per_dst=[5374, 5548, 5468, 5512, 5450, 5494, 5438, 5516] dup_rows_per_dst=[2666, 2728, 2762, 2814, 2648, 2804, 2630, 2684] combine_rows_per_src=[5374, 5548, 5468, 5512, 5450, 5494, 5438, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=227264 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156260288 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312293312 weight_movement_bytes=18874368 total_nvlink_bytes=331167680 weight_to_token_ratio=0.060438 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38364 dup_rows=21712 dup_rows_remote=19038 dedup_ratio=0.331299 router_tokens_per_expert=[251, 233, 253, 263, 260, 265, 257, 274, 235, 253, 240, 258, 261, 244, 244, 267, 277, 270, 261, 260, 265, 264, 265, 271, 256, 247, 241, 286, 242, 263, 220, 249, 257, 247, 268, 271, 269, 237, 268, 256, 249, 271, 260, 249, 263, 228, 271, 245, 246, 245, 260, 238, 261, 230, 247, 248, 236, 266, 247, 256, 259, 244, 260, 262, 250, 250, 278, 271, 252, 245, 252, 258, 246, 256, 254, 273, 246, 263, 267, 272, 238, 254, 242, 238, 265, 232, 246, 279, 277, 267, 240, 256, 249, 242, 294, 238, 287, 252, 253, 269, 242, 272, 257, 240, 258, 260, 265, 256, 267, 253, 246, 250, 260, 267, 258, 295, 253, 281, 272, 286, 259, 257, 259, 255, 217, 225, 253, 245] payload_rows_per_dst=[5460, 5496, 5536, 5552, 5672, 5474, 5162, 5472] dup_rows_per_dst=[2674, 2768, 2682, 2696, 3120, 2706, 2254, 2812] combine_rows_per_src=[5460, 5496, 5536, 5552, 5672, 5474, 5162, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229272 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156843928 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313458584 weight_movement_bytes=18874368 total_nvlink_bytes=332332952 weight_to_token_ratio=0.060213 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=229192 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157736776 ingress_bytes=176381952 link_bytes=176381952 token_bytes=315244360 weight_movement_bytes=18874368 total_nvlink_bytes=334118728 weight_to_token_ratio=0.059872 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690005079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38306 dup_rows=21740 dup_rows_remote=19092 dedup_ratio=0.331726 router_tokens_per_expert=[273, 250, 248, 250, 247, 245, 258, 268, 268, 223, 261, 258, 275, 243, 264, 252, 266, 260, 240, 252, 249, 250, 256, 255, 251, 263, 283, 254, 244, 257, 247, 253, 249, 271, 275, 228, 225, 262, 245, 255, 239, 252, 278, 259, 250, 243, 266, 233, 236, 275, 264, 257, 239, 260, 267, 255, 262, 250, 265, 250, 248, 261, 280, 256, 240, 286, 258, 251, 234, 226, 267, 255, 255, 279, 252, 246, 255, 245, 258, 262, 244, 251, 265, 230, 238, 264, 258, 252, 279, 269, 253, 253, 237, 261, 268, 263, 253, 235, 257, 273, 265, 278, 253, 255, 248, 283, 266, 274, 233, 239, 241, 298, 270, 251, 292, 269, 248, 264, 228, 262, 262, 270, 248, 256, 264, 245, 263, 253] payload_rows_per_dst=[5474, 5486, 5398, 5456, 5490, 5470, 5506, 5516] dup_rows_per_dst=[2692, 2674, 2662, 2794, 2648, 2700, 2796, 2774] combine_rows_per_src=[5474, 5486, 5398, 5456, 5490, 5470, 5506, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38048 dup_rows=21970 dup_rows_remote=19168 dedup_ratio=0.335236 router_tokens_per_expert=[237, 235, 268, 243, 260, 255, 237, 240, 243, 247, 235, 259, 257, 254, 254, 265, 267, 275, 273, 247, 264, 272, 252, 255, 267, 274, 288, 231, 261, 243, 228, 263, 227, 255, 255, 234, 222, 276, 257, 245, 273, 289, 253, 252, 250, 237, 253, 278, 267, 296, 249, 261, 280, 248, 260, 231, 268, 269, 258, 242, 255, 252, 256, 271, 252, 278, 264, 271, 267, 247, 231, 255, 254, 266, 270, 255, 273, 270, 265, 247, 243, 240, 255, 264, 270, 254, 258, 261, 236, 253, 261, 259, 261, 268, 237, 257, 241, 234, 247, 267, 238, 267, 279, 252, 224, 237, 257, 255, 239, 284, 267, 258, 268, 264, 236, 248, 264, 252, 256, 246, 266, 263, 242, 258, 268, 252, 234, 295] payload_rows_per_dst=[5318, 5518, 5454, 5460, 5534, 5410, 5382, 5490] dup_rows_per_dst=[2660, 2802, 2658, 2866, 2796, 2744, 2710, 2734] combine_rows_per_src=[5318, 5518, 5454, 5460, 5534, 5410, 5382, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228952 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130328 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314031704 weight_movement_bytes=18874368 total_nvlink_bytes=332906072 weight_to_token_ratio=0.060103 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229608 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157368552 ingress_bytes=194887680 link_bytes=194887680 token_bytes=314507496 weight_movement_bytes=37748736 total_nvlink_bytes=352256232 weight_to_token_ratio=0.120025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229592 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157130968 ingress_bytes=232398848 link_bytes=232398848 token_bytes=314032344 weight_movement_bytes=75497472 total_nvlink_bytes=389529816 weight_to_token_ratio=0.240413 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226731414e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155844608 dispatch_meta_bytes=228864 combine_payload_bytes=155844608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156073472 ingress_bytes=174718976 link_bytes=174718976 token_bytes=311918080 weight_movement_bytes=18874368 total_nvlink_bytes=330792448 weight_to_token_ratio=0.060511 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674071261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38286 dup_rows=21716 dup_rows_remote=18892 dedup_ratio=0.331360 router_tokens_per_expert=[228, 290, 268, 246, 262, 256, 244, 264, 264, 247, 257, 242, 261, 269, 252, 258, 257, 224, 265, 253, 269, 255, 235, 238, 270, 261, 246, 277, 256, 255, 244, 252, 263, 262, 256, 269, 266, 273, 262, 251, 261, 274, 265, 249, 272, 242, 260, 254, 239, 258, 265, 273, 250, 249, 273, 229, 241, 260, 239, 263, 266, 256, 273, 250, 252, 271, 264, 233, 275, 261, 262, 249, 250, 235, 267, 232, 250, 243, 269, 275, 233, 264, 217, 262, 222, 269, 243, 257, 241, 270, 262, 283, 251, 265, 254, 248, 261, 277, 281, 264, 229, 286, 244, 263, 251, 229, 262, 257, 232, 273, 245, 270, 238, 267, 272, 261, 272, 230, 273, 260, 234, 245, 260, 246, 254, 269, 247, 259] payload_rows_per_dst=[5434, 5460, 5534, 5454, 5508, 5484, 5488, 5458] dup_rows_per_dst=[2782, 2654, 2824, 2714, 2668, 2598, 2760, 2716] combine_rows_per_src=[5434, 5460, 5534, 5454, 5508, 5484, 5488, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38110 dup_rows=21778 dup_rows_remote=18774 dedup_ratio=0.332306 router_tokens_per_expert=[243, 264, 242, 241, 246, 269, 270, 239, 250, 261, 280, 251, 271, 254, 255, 239, 278, 282, 253, 270, 257, 243, 247, 255, 258, 279, 242, 236, 264, 262, 259, 270, 234, 248, 258, 268, 252, 256, 232, 273, 254, 231, 257, 249, 256, 236, 250, 235, 271, 254, 238, 233, 228, 260, 254, 271, 272, 247, 240, 277, 258, 262, 263, 231, 273, 242, 264, 245, 255, 268, 256, 237, 256, 266, 220, 259, 253, 259, 272, 273, 279, 271, 268, 275, 259, 261, 260, 233, 272, 260, 238, 246, 259, 257, 279, 257, 250, 239, 269, 233, 245, 234, 264, 261, 239, 256, 254, 253, 255, 255, 273, 264, 268, 230, 249, 239, 273, 268, 267, 244, 269, 276, 257, 273, 280, 264, 255, 262] payload_rows_per_dst=[5420, 5552, 5358, 5406, 5488, 5554, 5332, 5648] dup_rows_per_dst=[2730, 2758, 2568, 2712, 2708, 2794, 2504, 3004] combine_rows_per_src=[5420, 5552, 5358, 5406, 5488, 5554, 5332, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38520 dup_rows=21610 dup_rows_remote=18982 dedup_ratio=0.329742 router_tokens_per_expert=[253, 250, 253, 243, 244, 250, 264, 232, 260, 279, 251, 255, 246, 256, 255, 257, 265, 266, 259, 280, 245, 265, 269, 258, 251, 260, 249, 270, 221, 214, 251, 253, 287, 266, 264, 265, 261, 267, 259, 253, 240, 226, 263, 239, 256, 258, 261, 242, 232, 254, 239, 265, 271, 247, 254, 267, 227, 262, 238, 241, 259, 242, 265, 254, 245, 240, 273, 258, 244, 257, 278, 248, 261, 236, 265, 272, 263, 248, 258, 249, 247, 250, 261, 263, 281, 268, 249, 266, 267, 266, 235, 263, 247, 240, 237, 269, 279, 289, 280, 286, 254, 271, 241, 240, 239, 274, 255, 277, 250, 252, 249, 281, 252, 268, 248, 253, 257, 246, 246, 278, 275, 234, 260, 248, 276, 247, 266, 245] payload_rows_per_dst=[5528, 5392, 5540, 5406, 5496, 5490, 5564, 5510] dup_rows_per_dst=[2568, 2760, 2674, 2628, 2694, 2728, 2870, 2688] combine_rows_per_src=[5528, 5392, 5540, 5406, 5496, 5490, 5564, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38290 dup_rows=21756 dup_rows_remote=19122 dedup_ratio=0.331970 router_tokens_per_expert=[254, 253, 245, 231, 267, 293, 259, 270, 260, 245, 273, 299, 254, 263, 263, 281, 252, 257, 268, 268, 253, 239, 252, 262, 259, 251, 254, 280, 250, 273, 250, 247, 236, 252, 247, 253, 271, 243, 277, 256, 273, 264, 261, 274, 254, 261, 269, 264, 273, 279, 275, 254, 258, 262, 260, 258, 257, 238, 247, 240, 253, 258, 263, 236, 241, 269, 265, 253, 257, 254, 256, 252, 247, 233, 269, 260, 261, 230, 246, 255, 257, 262, 258, 242, 275, 260, 251, 259, 268, 241, 250, 237, 276, 255, 254, 233, 245, 273, 257, 261, 262, 255, 259, 253, 246, 260, 231, 237, 241, 280, 266, 236, 273, 291, 228, 236, 227, 252, 276, 255, 233, 224, 256, 255, 242, 246, 261, 234] payload_rows_per_dst=[5590, 5398, 5520, 5462, 5446, 5442, 5490, 5432] dup_rows_per_dst=[2830, 2766, 2790, 2742, 2716, 2732, 2634, 2546] combine_rows_per_src=[5590, 5398, 5520, 5462, 5446, 5442, 5490, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=228712 combine_payload_bytes=156819456 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157048168 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313867624 weight_movement_bytes=18874368 total_nvlink_bytes=332741992 weight_to_token_ratio=0.060135 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38200 dup_rows=21970 dup_rows_remote=19340 dedup_ratio=0.335236 router_tokens_per_expert=[276, 250, 249, 226, 242, 258, 256, 259, 255, 251, 231, 252, 249, 291, 256, 272, 234, 275, 265, 258, 245, 274, 284, 233, 249, 255, 230, 231, 255, 254, 274, 240, 273, 275, 258, 266, 270, 261, 273, 248, 249, 260, 229, 252, 288, 265, 248, 254, 253, 223, 236, 231, 257, 274, 236, 243, 258, 252, 239, 258, 268, 245, 258, 256, 280, 264, 250, 281, 256, 273, 272, 260, 258, 260, 242, 280, 241, 262, 241, 272, 232, 238, 265, 240, 272, 229, 256, 233, 241, 243, 268, 267, 247, 250, 253, 264, 288, 286, 217, 276, 260, 265, 272, 244, 268, 260, 256, 290, 270, 302, 249, 263, 252, 236, 247, 262, 275, 277, 249, 255, 272, 248, 232, 242, 275, 227, 234, 244] payload_rows_per_dst=[5442, 5390, 5488, 5362, 5540, 5366, 5564, 5414] dup_rows_per_dst=[2704, 2722, 2850, 2612, 2844, 2630, 2968, 2640] combine_rows_per_src=[5442, 5390, 5488, 5362, 5540, 5366, 5564, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=227536 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156326096 ingress_bytes=174972928 link_bytes=174972928 token_bytes=312424656 weight_movement_bytes=18874368 total_nvlink_bytes=331299024 weight_to_token_ratio=0.060413 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676504504e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38176 dup_rows=21786 dup_rows_remote=18854 dedup_ratio=0.332428 router_tokens_per_expert=[257, 272, 248, 252, 237, 251, 273, 279, 264, 279, 217, 284, 234, 265, 276, 234, 269, 255, 247, 249, 283, 234, 257, 259, 247, 260, 229, 261, 251, 235, 287, 236, 270, 247, 269, 273, 249, 263, 278, 249, 279, 250, 269, 264, 271, 234, 238, 237, 253, 262, 254, 249, 304, 268, 277, 240, 245, 238, 264, 246, 259, 271, 263, 253, 254, 245, 267, 272, 250, 245, 268, 257, 255, 238, 252, 248, 274, 259, 238, 258, 242, 249, 240, 281, 245, 271, 246, 238, 233, 250, 255, 288, 276, 220, 265, 244, 268, 248, 266, 276, 228, 248, 263, 268, 281, 259, 250, 232, 249, 265, 274, 245, 246, 244, 293, 253, 272, 263, 246, 248, 263, 244, 264, 262, 248, 239, 240, 233] payload_rows_per_dst=[5574, 5488, 5404, 5646, 5386, 5328, 5480, 5444] dup_rows_per_dst=[2932, 2736, 2624, 2870, 2556, 2652, 2760, 2656] combine_rows_per_src=[5574, 5488, 5404, 5646, 5386, 5328, 5480, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157777920 dispatch_meta_bytes=230008 combine_payload_bytes=157777920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158007928 ingress_bytes=176652288 link_bytes=176652288 token_bytes=315785848 weight_movement_bytes=18874368 total_nvlink_bytes=334660216 weight_to_token_ratio=0.059770 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692595305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229648 combine_payload_bytes=156835840 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157065488 ingress_bytes=213458944 link_bytes=213458944 token_bytes=313901328 weight_movement_bytes=56623104 total_nvlink_bytes=370524432 weight_to_token_ratio=0.180385 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045258573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38286 dup_rows=21748 dup_rows_remote=19152 dedup_ratio=0.331848 router_tokens_per_expert=[251, 246, 252, 262, 250, 268, 254, 273, 264, 277, 258, 259, 236, 263, 273, 263, 244, 278, 274, 294, 266, 256, 233, 274, 242, 261, 262, 239, 243, 212, 251, 220, 242, 272, 262, 248, 231, 235, 251, 248, 274, 263, 238, 256, 224, 258, 255, 248, 257, 247, 270, 289, 251, 253, 256, 264, 254, 251, 225, 285, 226, 265, 266, 227, 249, 261, 240, 282, 258, 259, 273, 243, 252, 265, 255, 249, 278, 257, 229, 265, 256, 271, 266, 251, 266, 265, 264, 270, 264, 266, 273, 238, 254, 261, 270, 240, 249, 267, 237, 263, 244, 257, 272, 236, 245, 261, 271, 257, 235, 242, 261, 280, 271, 275, 256, 249, 243, 278, 255, 283, 257, 262, 243, 227, 245, 258, 251, 259] payload_rows_per_dst=[5512, 5502, 5390, 5600, 5438, 5526, 5354, 5466] dup_rows_per_dst=[2786, 2596, 2620, 2820, 2792, 2824, 2552, 2758] combine_rows_per_src=[5512, 5502, 5390, 5600, 5438, 5526, 5354, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38426 dup_rows=21740 dup_rows_remote=19094 dedup_ratio=0.331726 router_tokens_per_expert=[291, 283, 264, 251, 276, 272, 250, 261, 234, 310, 252, 255, 238, 240, 263, 265, 249, 240, 269, 243, 264, 270, 257, 244, 257, 257, 254, 254, 223, 265, 249, 245, 252, 275, 259, 258, 257, 260, 252, 227, 248, 232, 279, 259, 260, 242, 254, 261, 241, 261, 250, 246, 257, 285, 263, 240, 224, 261, 234, 252, 242, 257, 234, 255, 250, 234, 277, 248, 256, 250, 252, 235, 229, 266, 226, 249, 251, 271, 261, 253, 277, 276, 282, 245, 256, 245, 248, 272, 270, 274, 251, 241, 267, 255, 280, 212, 264, 257, 287, 261, 273, 279, 262, 283, 293, 254, 273, 246, 246, 277, 247, 244, 246, 243, 274, 248, 223, 237, 254, 279, 256, 244, 266, 266, 238, 268, 260, 239] payload_rows_per_dst=[5572, 5422, 5430, 5392, 5370, 5550, 5660, 5400] dup_rows_per_dst=[2838, 2658, 2720, 2612, 2646, 2752, 2832, 2682] combine_rows_per_src=[5572, 5422, 5430, 5392, 5370, 5550, 5660, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=230160 combine_payload_bytes=156467200 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156697360 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313164560 weight_movement_bytes=18874368 total_nvlink_bytes=332038928 weight_to_token_ratio=0.060270 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=228120 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156597016 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312965912 weight_movement_bytes=37748736 total_nvlink_bytes=350714648 weight_to_token_ratio=0.120616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229752 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157049208 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868664 weight_movement_bytes=18874368 total_nvlink_bytes=332743032 weight_to_token_ratio=0.060135 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157392896 dispatch_meta_bytes=230080 combine_payload_bytes=157392896 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157622976 ingress_bytes=195141632 link_bytes=195141632 token_bytes=315015872 weight_movement_bytes=37748736 total_nvlink_bytes=352764608 weight_to_token_ratio=0.119831 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869751102e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38260 dup_rows=21672 dup_rows_remote=18874 dedup_ratio=0.330688 router_tokens_per_expert=[240, 251, 267, 263, 251, 253, 255, 249, 231, 248, 243, 256, 260, 263, 250, 270, 241, 243, 262, 288, 264, 248, 269, 247, 283, 260, 274, 255, 244, 264, 267, 249, 266, 271, 253, 277, 259, 261, 273, 265, 250, 272, 271, 247, 257, 279, 242, 258, 249, 252, 227, 265, 265, 238, 253, 272, 239, 265, 241, 271, 263, 241, 252, 274, 257, 268, 246, 298, 261, 261, 258, 240, 228, 255, 222, 263, 258, 251, 237, 260, 250, 222, 250, 242, 269, 250, 227, 273, 243, 262, 245, 260, 273, 261, 263, 260, 245, 253, 249, 257, 254, 254, 255, 271, 276, 276, 235, 258, 264, 281, 236, 249, 233, 266, 261, 270, 251, 266, 255, 272, 245, 242, 266, 246, 266, 232, 230, 265] payload_rows_per_dst=[5478, 5594, 5604, 5468, 5436, 5408, 5428, 5448] dup_rows_per_dst=[2622, 2722, 2798, 2666, 2690, 2692, 2798, 2684] combine_rows_per_src=[5478, 5594, 5604, 5468, 5436, 5408, 5428, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38062 dup_rows=21886 dup_rows_remote=18964 dedup_ratio=0.333954 router_tokens_per_expert=[253, 260, 276, 260, 254, 261, 252, 238, 279, 275, 247, 262, 277, 249, 263, 266, 271, 274, 279, 252, 277, 258, 221, 292, 270, 255, 237, 244, 249, 219, 254, 230, 275, 239, 252, 251, 280, 269, 261, 261, 250, 231, 256, 245, 275, 245, 274, 259, 280, 222, 271, 245, 248, 272, 239, 277, 239, 254, 244, 246, 253, 246, 252, 252, 267, 251, 218, 255, 268, 294, 234, 252, 257, 240, 279, 273, 257, 246, 250, 270, 265, 238, 263, 247, 259, 257, 240, 264, 248, 232, 237, 275, 236, 248, 248, 251, 234, 255, 251, 227, 253, 264, 250, 229, 255, 247, 249, 277, 269, 254, 245, 292, 252, 253, 264, 266, 264, 258, 263, 264, 272, 267, 259, 276, 270, 253, 240, 260] payload_rows_per_dst=[5288, 5540, 5496, 5376, 5458, 5342, 5562, 5588] dup_rows_per_dst=[2572, 2624, 2750, 2704, 2748, 2690, 2876, 2922] combine_rows_per_src=[5288, 5540, 5496, 5376, 5458, 5342, 5562, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38264 dup_rows=21782 dup_rows_remote=19166 dedup_ratio=0.332367 router_tokens_per_expert=[236, 253, 247, 276, 261, 234, 252, 252, 240, 291, 263, 258, 250, 255, 257, 265, 240, 271, 268, 288, 207, 246, 243, 240, 261, 263, 258, 252, 224, 262, 278, 276, 273, 252, 254, 258, 268, 293, 239, 251, 283, 275, 254, 259, 285, 241, 252, 254, 251, 250, 269, 257, 254, 256, 210, 254, 247, 243, 267, 251, 239, 255, 274, 276, 256, 283, 284, 239, 249, 254, 261, 262, 261, 274, 268, 260, 250, 269, 263, 228, 267, 261, 233, 245, 261, 253, 263, 262, 276, 227, 260, 230, 220, 271, 258, 268, 260, 255, 241, 257, 254, 254, 249, 248, 262, 274, 237, 262, 247, 256, 268, 226, 238, 240, 252, 305, 274, 235, 239, 280, 247, 258, 244, 247, 263, 277, 254, 238] payload_rows_per_dst=[5430, 5436, 5580, 5490, 5500, 5450, 5362, 5506] dup_rows_per_dst=[2750, 2718, 2802, 2616, 2822, 2660, 2738, 2676] combine_rows_per_src=[5430, 5436, 5580, 5490, 5500, 5450, 5362, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228536 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156941496 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313654456 weight_movement_bytes=18874368 total_nvlink_bytes=332528824 weight_to_token_ratio=0.060176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38404 dup_rows=21726 dup_rows_remote=19030 dedup_ratio=0.331512 router_tokens_per_expert=[277, 247, 267, 252, 235, 279, 244, 249, 257, 261, 242, 231, 247, 236, 262, 259, 221, 253, 264, 266, 231, 224, 281, 273, 241, 257, 244, 260, 261, 255, 238, 263, 260, 242, 254, 261, 249, 260, 251, 263, 247, 271, 272, 259, 263, 283, 230, 270, 239, 286, 260, 265, 273, 253, 282, 239, 275, 264, 276, 283, 243, 246, 242, 255, 238, 242, 238, 252, 269, 271, 280, 252, 283, 245, 260, 267, 256, 236, 266, 277, 250, 252, 252, 253, 260, 240, 262, 246, 271, 267, 266, 262, 253, 277, 260, 279, 259, 272, 263, 245, 249, 252, 231, 261, 240, 257, 234, 249, 288, 248, 240, 263, 237, 249, 276, 269, 242, 256, 236, 264, 259, 264, 244, 254, 259, 262, 223, 248] payload_rows_per_dst=[5494, 5466, 5494, 5568, 5470, 5468, 5406, 5444] dup_rows_per_dst=[2596, 2598, 2776, 2794, 2794, 2832, 2696, 2640] combine_rows_per_src=[5494, 5466, 5494, 5568, 5470, 5468, 5406, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38154 dup_rows=21848 dup_rows_remote=19010 dedup_ratio=0.333374 router_tokens_per_expert=[244, 261, 239, 271, 227, 237, 273, 230, 285, 276, 251, 249, 260, 254, 276, 229, 259, 254, 235, 238, 263, 255, 258, 245, 235, 255, 284, 259, 275, 259, 241, 286, 253, 251, 273, 270, 250, 278, 267, 255, 251, 272, 231, 260, 259, 261, 245, 245, 258, 232, 237, 247, 243, 268, 257, 238, 269, 225, 251, 277, 285, 241, 259, 239, 281, 254, 280, 278, 278, 252, 263, 263, 230, 229, 289, 260, 272, 274, 275, 248, 270, 276, 248, 249, 277, 248, 241, 268, 263, 267, 245, 293, 268, 268, 239, 266, 235, 264, 272, 277, 245, 247, 246, 238, 251, 254, 248, 247, 276, 245, 244, 251, 270, 257, 256, 262, 235, 271, 246, 228, 244, 226, 250, 266, 250, 253, 251, 241] payload_rows_per_dst=[5422, 5424, 5488, 5404, 5558, 5534, 5392, 5466] dup_rows_per_dst=[2702, 2778, 2754, 2648, 2894, 2838, 2688, 2546] combine_rows_per_src=[5422, 5424, 5488, 5404, 5558, 5534, 5392, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228104 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156130056 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312032008 weight_movement_bytes=18874368 total_nvlink_bytes=330906376 weight_to_token_ratio=0.060489 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38304 dup_rows=21752 dup_rows_remote=19080 dedup_ratio=0.331909 router_tokens_per_expert=[277, 260, 255, 252, 234, 263, 268, 237, 271, 255, 249, 225, 234, 258, 276, 261, 264, 266, 281, 268, 262, 263, 278, 258, 249, 297, 253, 268, 274, 230, 259, 266, 240, 230, 256, 262, 231, 256, 232, 267, 250, 259, 251, 240, 264, 268, 253, 260, 247, 248, 240, 253, 265, 275, 276, 245, 262, 265, 259, 275, 261, 252, 295, 256, 239, 252, 246, 286, 285, 241, 264, 252, 257, 253, 243, 255, 249, 247, 239, 263, 238, 263, 235, 263, 246, 252, 233, 264, 239, 264, 281, 249, 280, 268, 265, 244, 267, 284, 236, 249, 238, 241, 280, 256, 253, 259, 237, 272, 275, 222, 247, 249, 243, 269, 250, 254, 240, 255, 245, 229, 240, 250, 278, 259, 265, 240, 263, 264] payload_rows_per_dst=[5480, 5614, 5380, 5616, 5440, 5466, 5326, 5462] dup_rows_per_dst=[2672, 2932, 2658, 2788, 2702, 2702, 2672, 2626] combine_rows_per_src=[5480, 5614, 5380, 5616, 5440, 5466, 5326, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229720 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156959064 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688408 weight_movement_bytes=18874368 total_nvlink_bytes=332562776 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38198 dup_rows=21890 dup_rows_remote=19082 dedup_ratio=0.334015 router_tokens_per_expert=[247, 265, 244, 251, 244, 271, 229, 265, 247, 281, 250, 266, 273, 247, 292, 264, 256, 252, 246, 293, 216, 296, 227, 256, 276, 263, 258, 259, 236, 280, 248, 266, 245, 258, 263, 279, 275, 266, 294, 272, 258, 250, 214, 279, 262, 243, 256, 241, 240, 264, 241, 244, 259, 271, 256, 272, 271, 242, 226, 257, 263, 273, 263, 237, 283, 247, 241, 230, 263, 284, 269, 282, 249, 238, 258, 245, 267, 248, 257, 259, 255, 249, 232, 288, 227, 253, 275, 237, 253, 247, 260, 257, 241, 259, 240, 252, 239, 263, 260, 246, 289, 247, 263, 252, 237, 266, 227, 255, 275, 232, 263, 257, 249, 258, 234, 258, 245, 276, 252, 263, 248, 269, 247, 241, 240, 260, 252, 262] payload_rows_per_dst=[5504, 5448, 5506, 5434, 5546, 5444, 5394, 5370] dup_rows_per_dst=[2768, 2808, 2804, 2724, 2694, 2606, 2748, 2738] combine_rows_per_src=[5504, 5448, 5506, 5434, 5546, 5444, 5394, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229736 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157532520 ingress_bytes=195051520 link_bytes=195051520 token_bytes=314835304 weight_movement_bytes=37748736 total_nvlink_bytes=352584040 weight_to_token_ratio=0.119900 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868887693e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38192 dup_rows=21876 dup_rows_remote=19172 dedup_ratio=0.333801 router_tokens_per_expert=[244, 245, 266, 241, 280, 269, 267, 250, 249, 240, 247, 269, 242, 253, 249, 261, 246, 272, 276, 241, 257, 265, 237, 271, 254, 223, 253, 240, 264, 247, 255, 253, 254, 252, 256, 248, 224, 243, 226, 274, 255, 262, 259, 266, 277, 259, 241, 232, 254, 274, 242, 277, 252, 256, 242, 282, 280, 228, 263, 249, 254, 229, 250, 268, 265, 247, 239, 208, 287, 246, 256, 247, 260, 282, 250, 253, 244, 272, 252, 278, 257, 260, 273, 275, 242, 265, 269, 276, 268, 266, 271, 256, 266, 252, 264, 230, 262, 251, 289, 282, 256, 225, 270, 272, 232, 265, 262, 262, 278, 213, 260, 257, 283, 226, 247, 251, 267, 233, 265, 251, 254, 225, 269, 286, 260, 282, 237, 266] payload_rows_per_dst=[5420, 5498, 5402, 5476, 5468, 5522, 5424, 5450] dup_rows_per_dst=[2724, 2610, 2654, 2724, 2704, 2858, 2848, 2754] combine_rows_per_src=[5420, 5498, 5402, 5476, 5468, 5522, 5424, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=228656 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156507440 ingress_bytes=212901888 link_bytes=212901888 token_bytes=312786224 weight_movement_bytes=56623104 total_nvlink_bytes=369409328 weight_to_token_ratio=0.181028 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039921137e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229536 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122720 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015904 weight_movement_bytes=37748736 total_nvlink_bytes=351764640 weight_to_token_ratio=0.120213 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229120 combine_payload_bytes=156459008 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156688128 ingress_bytes=194207744 link_bytes=194207744 token_bytes=313147136 weight_movement_bytes=37748736 total_nvlink_bytes=350895872 weight_to_token_ratio=0.120546 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860803047e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38390 dup_rows=21628 dup_rows_remote=18864 dedup_ratio=0.330017 router_tokens_per_expert=[252, 240, 264, 250, 230, 236, 246, 253, 263, 281, 228, 233, 263, 243, 271, 250, 239, 276, 251, 255, 244, 248, 253, 222, 270, 265, 259, 252, 252, 267, 246, 283, 279, 283, 277, 268, 258, 261, 273, 256, 252, 243, 244, 236, 272, 263, 231, 245, 260, 273, 237, 272, 259, 252, 246, 238, 278, 238, 243, 270, 275, 254, 241, 264, 272, 245, 235, 254, 266, 292, 262, 263, 255, 249, 265, 230, 252, 306, 244, 227, 275, 227, 253, 275, 237, 264, 256, 286, 274, 235, 229, 266, 256, 233, 256, 227, 308, 268, 272, 240, 269, 287, 255, 256, 258, 279, 247, 277, 248, 239, 254, 238, 254, 264, 226, 254, 254, 248, 278, 242, 243, 261, 242, 266, 286, 226, 257, 280] payload_rows_per_dst=[5436, 5504, 5518, 5458, 5424, 5510, 5604, 5454] dup_rows_per_dst=[2570, 2660, 2764, 2742, 2810, 2588, 2786, 2708] combine_rows_per_src=[5436, 5504, 5518, 5458, 5424, 5510, 5604, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229456 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156663888 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313098320 weight_movement_bytes=37748736 total_nvlink_bytes=350847056 weight_to_token_ratio=0.120565 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38246 dup_rows=21732 dup_rows_remote=18956 dedup_ratio=0.331604 router_tokens_per_expert=[228, 261, 275, 273, 261, 244, 242, 298, 262, 233, 251, 262, 255, 263, 272, 256, 230, 264, 272, 256, 237, 255, 275, 245, 250, 271, 247, 264, 272, 256, 270, 245, 260, 255, 257, 247, 266, 272, 234, 248, 260, 234, 251, 254, 262, 239, 254, 251, 261, 234, 213, 251, 235, 231, 249, 226, 253, 247, 260, 267, 268, 282, 269, 250, 258, 268, 296, 253, 236, 257, 254, 272, 268, 238, 243, 247, 263, 249, 276, 231, 258, 249, 270, 276, 214, 288, 245, 281, 271, 257, 250, 221, 241, 264, 232, 242, 262, 261, 244, 238, 249, 251, 251, 254, 265, 272, 274, 253, 269, 292, 247, 266, 268, 261, 276, 268, 262, 266, 234, 247, 291, 257, 239, 269, 236, 256, 251, 286] payload_rows_per_dst=[5530, 5354, 5522, 5342, 5300, 5608, 5590, 5558] dup_rows_per_dst=[2742, 2660, 2566, 2650, 2566, 2994, 2778, 2776] combine_rows_per_src=[5530, 5354, 5522, 5342, 5300, 5608, 5590, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38280 dup_rows=21758 dup_rows_remote=19016 dedup_ratio=0.332001 router_tokens_per_expert=[261, 236, 254, 234, 264, 262, 265, 262, 245, 266, 234, 237, 255, 276, 252, 258, 244, 276, 232, 257, 241, 227, 247, 247, 232, 237, 253, 284, 276, 264, 239, 262, 244, 278, 257, 257, 250, 227, 268, 264, 242, 198, 286, 257, 248, 264, 251, 247, 235, 241, 269, 267, 272, 257, 289, 261, 262, 248, 237, 246, 290, 248, 254, 244, 246, 277, 260, 251, 261, 240, 251, 259, 278, 252, 268, 266, 238, 278, 234, 230, 303, 263, 272, 304, 256, 240, 273, 230, 252, 274, 273, 258, 247, 265, 256, 231, 276, 274, 227, 243, 278, 284, 259, 240, 240, 256, 232, 258, 261, 257, 257, 255, 242, 277, 256, 247, 263, 249, 260, 261, 259, 266, 260, 277, 234, 299, 265, 233] payload_rows_per_dst=[5432, 5394, 5430, 5498, 5454, 5554, 5510, 5506] dup_rows_per_dst=[2690, 2642, 2646, 2742, 2724, 2840, 2684, 2790] combine_rows_per_src=[5432, 5394, 5430, 5498, 5454, 5554, 5510, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=229016 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157474456 ingress_bytes=176119808 link_bytes=176119808 token_bytes=314719896 weight_movement_bytes=18874368 total_nvlink_bytes=333594264 weight_to_token_ratio=0.059972 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687493344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38402 dup_rows=21646 dup_rows_remote=18994 dedup_ratio=0.330292 router_tokens_per_expert=[297, 255, 295, 277, 229, 247, 261, 249, 262, 240, 265, 242, 232, 279, 276, 230, 253, 235, 230, 236, 238, 247, 277, 225, 265, 257, 294, 244, 245, 233, 250, 253, 232, 235, 252, 238, 248, 245, 266, 285, 243, 242, 270, 234, 264, 261, 241, 266, 252, 290, 265, 267, 226, 257, 271, 260, 248, 271, 245, 233, 249, 242, 278, 253, 261, 259, 267, 265, 275, 276, 259, 221, 261, 276, 240, 248, 258, 287, 257, 249, 253, 262, 256, 255, 246, 255, 258, 245, 262, 270, 276, 281, 270, 272, 250, 264, 260, 264, 226, 242, 274, 214, 261, 258, 227, 281, 283, 248, 258, 269, 267, 238, 263, 236, 259, 242, 262, 256, 251, 256, 268, 266, 271, 266, 273, 242, 260, 246] payload_rows_per_dst=[5496, 5404, 5484, 5462, 5534, 5550, 5488, 5472] dup_rows_per_dst=[2776, 2560, 2560, 2752, 2784, 2800, 2652, 2762] combine_rows_per_src=[5496, 5404, 5484, 5462, 5534, 5550, 5488, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228808 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156884424 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540040 weight_movement_bytes=37748736 total_nvlink_bytes=351288776 weight_to_token_ratio=0.120395 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38246 dup_rows=21874 dup_rows_remote=19186 dedup_ratio=0.333771 router_tokens_per_expert=[265, 250, 250, 277, 253, 252, 299, 256, 272, 258, 245, 252, 252, 265, 262, 249, 260, 254, 248, 265, 275, 251, 270, 265, 260, 242, 253, 274, 251, 262, 255, 246, 225, 272, 248, 253, 239, 234, 265, 243, 241, 236, 264, 292, 280, 254, 244, 234, 249, 245, 261, 261, 266, 281, 279, 238, 273, 263, 248, 267, 262, 227, 281, 286, 275, 259, 246, 231, 252, 249, 233, 276, 257, 289, 270, 216, 249, 258, 263, 243, 257, 252, 266, 256, 248, 243, 242, 205, 260, 274, 250, 250, 267, 272, 246, 264, 242, 249, 266, 273, 255, 248, 222, 263, 252, 263, 257, 254, 254, 243, 229, 239, 249, 255, 268, 259, 269, 246, 269, 278, 224, 242, 275, 267, 250, 268, 256, 267] payload_rows_per_dst=[5486, 5472, 5430, 5502, 5488, 5416, 5386, 5482] dup_rows_per_dst=[2828, 2790, 2618, 2872, 2644, 2688, 2632, 2802] combine_rows_per_src=[5486, 5472, 5430, 5502, 5488, 5416, 5386, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229184 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157024064 ingress_bytes=232292352 link_bytes=232292352 token_bytes=313818944 weight_movement_bytes=75497472 total_nvlink_bytes=389316416 weight_to_token_ratio=0.240577 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225711022e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=37974 dup_rows=21948 dup_rows_remote=18994 dedup_ratio=0.334900 router_tokens_per_expert=[268, 259, 258, 228, 275, 257, 259, 246, 253, 251, 261, 262, 264, 262, 243, 244, 252, 246, 275, 233, 223, 263, 258, 265, 276, 264, 269, 250, 271, 262, 251, 244, 265, 252, 243, 238, 279, 277, 245, 278, 252, 268, 238, 247, 253, 248, 258, 240, 269, 260, 250, 263, 255, 247, 240, 243, 242, 264, 252, 256, 243, 245, 255, 268, 265, 283, 272, 238, 249, 250, 276, 241, 219, 266, 245, 237, 268, 273, 270, 242, 245, 233, 267, 262, 277, 280, 259, 289, 249, 274, 239, 247, 278, 252, 258, 257, 293, 250, 290, 265, 273, 261, 254, 272, 221, 279, 274, 235, 254, 238, 231, 294, 267, 231, 254, 267, 250, 244, 262, 259, 223, 238, 275, 254, 239, 251, 254, 231] payload_rows_per_dst=[5614, 5392, 5424, 5374, 5346, 5512, 5544, 5382] dup_rows_per_dst=[2954, 2826, 2738, 2746, 2430, 2814, 2824, 2616] combine_rows_per_src=[5614, 5392, 5424, 5374, 5346, 5512, 5544, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38376 dup_rows=21632 dup_rows_remote=18882 dedup_ratio=0.330078 router_tokens_per_expert=[244, 242, 273, 241, 255, 243, 240, 264, 283, 238, 275, 235, 235, 247, 243, 246, 245, 262, 256, 273, 252, 273, 274, 293, 254, 260, 247, 238, 253, 271, 237, 251, 254, 250, 274, 257, 254, 265, 277, 251, 256, 261, 256, 245, 229, 233, 243, 243, 251, 277, 258, 255, 260, 262, 254, 258, 283, 249, 257, 265, 243, 256, 257, 277, 233, 240, 285, 232, 261, 245, 294, 251, 258, 265, 272, 231, 242, 247, 235, 263, 260, 283, 258, 253, 280, 251, 252, 256, 271, 260, 247, 231, 262, 258, 243, 277, 265, 258, 257, 249, 270, 249, 262, 269, 259, 269, 264, 240, 260, 244, 257, 250, 256, 268, 256, 255, 258, 256, 223, 267, 273, 223, 269, 269, 249, 274, 253, 248] payload_rows_per_dst=[5394, 5528, 5428, 5656, 5442, 5536, 5442, 5478] dup_rows_per_dst=[2646, 2750, 2668, 2668, 2730, 2716, 2738, 2716] combine_rows_per_src=[5394, 5528, 5428, 5656, 5442, 5536, 5442, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229584 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157524176 ingress_bytes=195043328 link_bytes=195043328 token_bytes=314818768 weight_movement_bytes=37748736 total_nvlink_bytes=352567504 weight_to_token_ratio=0.119906 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868809202e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38438 dup_rows=21684 dup_rows_remote=19038 dedup_ratio=0.330872 router_tokens_per_expert=[243, 221, 248, 260, 245, 243, 269, 266, 247, 269, 232, 289, 240, 241, 289, 254, 221, 229, 237, 261, 249, 228, 252, 258, 249, 268, 234, 239, 256, 256, 264, 250, 266, 246, 219, 242, 240, 286, 246, 238, 257, 264, 291, 256, 264, 245, 267, 251, 257, 254, 267, 260, 268, 259, 286, 247, 265, 250, 290, 272, 250, 260, 253, 239, 273, 267, 245, 276, 224, 224, 268, 248, 245, 258, 235, 262, 261, 251, 247, 246, 264, 228, 295, 259, 244, 243, 256, 243, 262, 272, 298, 260, 259, 238, 278, 275, 281, 285, 289, 246, 257, 260, 245, 293, 258, 273, 236, 279, 232, 280, 231, 261, 285, 227, 228, 252, 282, 244, 249, 247, 277, 264, 275, 265, 252, 268, 245, 236] payload_rows_per_dst=[5540, 5334, 5478, 5512, 5414, 5528, 5574, 5472] dup_rows_per_dst=[2572, 2568, 2678, 2842, 2646, 2820, 2838, 2720] combine_rows_per_src=[5540, 5334, 5478, 5512, 5414, 5528, 5574, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229728 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885344 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540960 weight_movement_bytes=18874368 total_nvlink_bytes=332415328 weight_to_token_ratio=0.060197 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38184 dup_rows=21858 dup_rows_remote=19142 dedup_ratio=0.333527 router_tokens_per_expert=[258, 265, 266, 257, 242, 247, 252, 253, 262, 253, 277, 265, 269, 253, 221, 261, 265, 266, 273, 250, 259, 254, 252, 250, 277, 276, 264, 252, 221, 252, 267, 247, 261, 266, 246, 249, 248, 250, 245, 262, 275, 256, 275, 257, 265, 237, 262, 251, 240, 281, 255, 272, 283, 266, 238, 267, 248, 266, 267, 284, 263, 246, 275, 253, 241, 250, 259, 253, 236, 253, 244, 232, 232, 262, 238, 269, 258, 245, 248, 244, 258, 240, 243, 283, 258, 215, 251, 248, 252, 237, 271, 253, 259, 287, 255, 256, 276, 264, 248, 256, 235, 254, 277, 252, 247, 260, 273, 288, 272, 261, 246, 246, 246, 238, 254, 259, 271, 236, 261, 261, 254, 248, 265, 257, 243, 231, 265, 259] payload_rows_per_dst=[5440, 5486, 5494, 5516, 5336, 5484, 5494, 5428] dup_rows_per_dst=[2762, 2764, 2716, 2892, 2592, 2648, 2816, 2668] combine_rows_per_src=[5440, 5486, 5494, 5516, 5336, 5484, 5494, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155541504 dispatch_meta_bytes=227872 combine_payload_bytes=155541504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155769376 ingress_bytes=174415872 link_bytes=174415872 token_bytes=311310880 weight_movement_bytes=18874368 total_nvlink_bytes=330185248 weight_to_token_ratio=0.060629 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671167068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229032 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417128 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314605224 weight_movement_bytes=18874368 total_nvlink_bytes=333479592 weight_to_token_ratio=0.059994 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38386 dup_rows=21610 dup_rows_remote=18896 dedup_ratio=0.329742 router_tokens_per_expert=[271, 249, 261, 256, 238, 245, 229, 239, 255, 266, 237, 248, 251, 250, 257, 234, 280, 266, 275, 253, 241, 266, 283, 246, 245, 264, 273, 256, 256, 279, 252, 268, 279, 236, 265, 267, 241, 239, 240, 252, 256, 248, 247, 275, 278, 229, 252, 268, 226, 258, 245, 274, 231, 259, 247, 271, 251, 239, 270, 268, 274, 273, 243, 272, 258, 240, 260, 250, 250, 228, 253, 272, 280, 261, 245, 267, 262, 255, 246, 263, 253, 256, 257, 262, 283, 234, 258, 238, 271, 230, 281, 243, 256, 246, 276, 262, 239, 254, 231, 270, 284, 249, 269, 247, 251, 263, 233, 281, 252, 252, 256, 266, 226, 255, 261, 251, 233, 264, 240, 243, 265, 288, 265, 269, 284, 250, 257, 262] payload_rows_per_dst=[5386, 5608, 5442, 5490, 5468, 5544, 5448, 5540] dup_rows_per_dst=[2586, 2798, 2702, 2642, 2712, 2668, 2788, 2714] combine_rows_per_src=[5386, 5608, 5442, 5490, 5468, 5544, 5448, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229904 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671952 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315114000 weight_movement_bytes=18874368 total_nvlink_bytes=333988368 weight_to_token_ratio=0.059897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38170 dup_rows=21916 dup_rows_remote=19136 dedup_ratio=0.334412 router_tokens_per_expert=[286, 277, 269, 267, 248, 242, 261, 254, 254, 254, 251, 262, 265, 270, 271, 255, 275, 254, 282, 241, 247, 264, 236, 255, 259, 284, 272, 261, 252, 276, 237, 269, 253, 272, 248, 258, 279, 253, 265, 260, 236, 233, 247, 248, 254, 235, 252, 239, 260, 263, 277, 276, 260, 231, 239, 257, 278, 258, 240, 232, 279, 235, 272, 258, 258, 247, 256, 286, 232, 233, 266, 253, 268, 227, 281, 257, 221, 274, 264, 258, 252, 272, 254, 263, 257, 244, 250, 243, 245, 261, 243, 241, 248, 252, 253, 234, 269, 273, 237, 262, 258, 255, 264, 251, 236, 270, 253, 238, 246, 266, 283, 250, 272, 258, 240, 263, 254, 283, 247, 253, 253, 242, 270, 250, 242, 228, 242, 270] payload_rows_per_dst=[5600, 5572, 5356, 5450, 5366, 5386, 5482, 5408] dup_rows_per_dst=[2772, 2756, 2708, 2780, 2796, 2638, 2740, 2726] combine_rows_per_src=[5600, 5572, 5356, 5450, 5366, 5386, 5482, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229304 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156630968 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032632 weight_movement_bytes=18874368 total_nvlink_bytes=331907000 weight_to_token_ratio=0.060295 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38288 dup_rows=21832 dup_rows_remote=19124 dedup_ratio=0.333130 router_tokens_per_expert=[265, 252, 253, 234, 257, 237, 252, 279, 250, 228, 270, 257, 254, 264, 263, 233, 263, 271, 267, 293, 238, 255, 235, 258, 289, 249, 267, 269, 263, 256, 242, 281, 245, 246, 263, 263, 262, 226, 269, 273, 253, 256, 268, 267, 237, 247, 237, 246, 258, 255, 255, 234, 239, 253, 273, 258, 243, 232, 236, 267, 260, 256, 275, 271, 251, 268, 263, 246, 262, 247, 266, 264, 261, 246, 262, 257, 290, 243, 237, 251, 227, 242, 247, 250, 248, 262, 234, 248, 274, 255, 243, 263, 275, 281, 261, 239, 262, 247, 250, 294, 223, 268, 257, 248, 243, 244, 249, 251, 245, 261, 275, 245, 263, 270, 275, 266, 267, 251, 237, 265, 258, 269, 255, 245, 303, 256, 269, 227] payload_rows_per_dst=[5418, 5556, 5376, 5434, 5518, 5388, 5416, 5598] dup_rows_per_dst=[2678, 2836, 2740, 2696, 2710, 2710, 2708, 2754] combine_rows_per_src=[5418, 5556, 5376, 5434, 5518, 5388, 5416, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229128 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458184 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687240 weight_movement_bytes=18874368 total_nvlink_bytes=333561608 weight_to_token_ratio=0.059978 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229224 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573544 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917864 weight_movement_bytes=18874368 total_nvlink_bytes=331792232 weight_to_token_ratio=0.060317 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38248 dup_rows=21818 dup_rows_remote=19106 dedup_ratio=0.332916 router_tokens_per_expert=[260, 247, 255, 248, 270, 240, 281, 269, 250, 255, 282, 247, 266, 240, 270, 262, 255, 242, 241, 275, 241, 251, 272, 239, 237, 239, 262, 262, 288, 269, 270, 241, 264, 233, 257, 260, 249, 269, 239, 231, 268, 271, 279, 269, 262, 280, 247, 259, 255, 232, 241, 266, 269, 264, 264, 253, 259, 248, 257, 251, 269, 256, 254, 242, 256, 266, 249, 229, 251, 234, 257, 256, 259, 283, 268, 244, 272, 266, 284, 256, 237, 264, 263, 264, 265, 259, 237, 276, 236, 250, 254, 242, 284, 247, 253, 260, 233, 285, 260, 281, 231, 267, 252, 263, 238, 264, 247, 259, 259, 217, 277, 282, 250, 243, 227, 251, 242, 230, 222, 263, 243, 270, 264, 251, 263, 265, 260, 245] payload_rows_per_dst=[5510, 5526, 5462, 5414, 5496, 5470, 5504, 5336] dup_rows_per_dst=[2774, 2642, 2812, 2746, 2764, 2712, 2726, 2642] combine_rows_per_src=[5510, 5526, 5462, 5414, 5496, 5470, 5504, 5336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38304 dup_rows=21700 dup_rows_remote=18856 dedup_ratio=0.331116 router_tokens_per_expert=[246, 260, 250, 267, 261, 244, 295, 256, 264, 245, 263, 245, 252, 241, 290, 275, 268, 260, 257, 231, 276, 291, 274, 248, 262, 266, 272, 228, 270, 264, 247, 257, 243, 278, 254, 262, 254, 207, 271, 230, 279, 283, 252, 238, 272, 266, 240, 209, 277, 251, 242, 271, 248, 249, 264, 255, 273, 273, 232, 251, 273, 233, 224, 215, 263, 286, 244, 297, 239, 243, 254, 229, 249, 280, 236, 264, 259, 253, 245, 267, 254, 260, 252, 247, 222, 249, 241, 237, 234, 257, 242, 287, 255, 224, 269, 286, 288, 268, 270, 263, 245, 227, 255, 252, 260, 266, 292, 259, 233, 272, 256, 272, 261, 236, 272, 252, 238, 272, 237, 256, 258, 255, 222, 274, 250, 249, 265, 275] payload_rows_per_dst=[5532, 5648, 5438, 5636, 5448, 5346, 5336, 5452] dup_rows_per_dst=[2844, 2912, 2790, 2962, 2550, 2466, 2484, 2692] combine_rows_per_src=[5532, 5648, 5438, 5636, 5448, 5346, 5336, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38210 dup_rows=21854 dup_rows_remote=19090 dedup_ratio=0.333466 router_tokens_per_expert=[267, 257, 241, 265, 251, 266, 253, 252, 244, 248, 263, 261, 246, 253, 275, 281, 286, 255, 251, 240, 240, 257, 265, 256, 252, 247, 274, 279, 272, 246, 238, 260, 291, 252, 249, 292, 267, 235, 264, 239, 288, 237, 294, 244, 247, 228, 254, 253, 242, 254, 250, 261, 256, 241, 258, 279, 266, 261, 265, 264, 226, 272, 268, 263, 234, 272, 251, 256, 256, 237, 250, 266, 253, 274, 213, 245, 284, 270, 257, 244, 260, 229, 279, 261, 263, 260, 260, 251, 252, 245, 240, 288, 253, 259, 246, 271, 246, 232, 251, 234, 267, 258, 284, 272, 238, 240, 246, 233, 236, 258, 264, 259, 256, 258, 249, 273, 256, 239, 270, 272, 255, 274, 252, 236, 250, 251, 241, 238] payload_rows_per_dst=[5484, 5472, 5446, 5552, 5454, 5500, 5384, 5390] dup_rows_per_dst=[2762, 2764, 2822, 2700, 2670, 2734, 2652, 2750] combine_rows_per_src=[5484, 5472, 5446, 5552, 5454, 5500, 5384, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229648 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057296 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884944 weight_movement_bytes=37748736 total_nvlink_bytes=351633680 weight_to_token_ratio=0.120263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38350 dup_rows=21718 dup_rows_remote=18888 dedup_ratio=0.331390 router_tokens_per_expert=[262, 251, 271, 243, 283, 267, 236, 253, 254, 231, 246, 276, 246, 245, 261, 275, 260, 255, 224, 251, 235, 283, 258, 288, 271, 264, 242, 268, 256, 260, 259, 248, 257, 247, 258, 270, 251, 235, 244, 252, 285, 273, 254, 270, 235, 262, 248, 288, 258, 284, 268, 243, 230, 271, 225, 244, 270, 251, 261, 245, 243, 248, 286, 258, 260, 280, 252, 280, 258, 256, 265, 259, 258, 261, 274, 235, 240, 223, 246, 302, 265, 256, 255, 257, 254, 255, 249, 258, 222, 252, 219, 271, 269, 236, 228, 242, 269, 270, 243, 259, 248, 253, 272, 239, 250, 244, 219, 243, 258, 259, 253, 294, 236, 239, 270, 256, 249, 268, 254, 285, 250, 257, 250, 287, 249, 281, 242, 249] payload_rows_per_dst=[5516, 5474, 5536, 5476, 5468, 5412, 5400, 5536] dup_rows_per_dst=[2684, 2770, 2722, 2694, 2830, 2564, 2746, 2708] combine_rows_per_src=[5516, 5474, 5536, 5476, 5468, 5412, 5400, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229416 combine_payload_bytes=156663808 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156893224 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313557032 weight_movement_bytes=37748736 total_nvlink_bytes=351305768 weight_to_token_ratio=0.120389 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228640 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157121824 ingress_bytes=213516288 link_bytes=213516288 token_bytes=314015008 weight_movement_bytes=56623104 total_nvlink_bytes=370638112 weight_to_token_ratio=0.180320 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045808015e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38418 dup_rows=21668 dup_rows_remote=18914 dedup_ratio=0.330627 router_tokens_per_expert=[283, 259, 267, 249, 268, 263, 260, 257, 228, 251, 247, 277, 250, 278, 288, 276, 272, 267, 275, 261, 247, 259, 265, 257, 289, 256, 278, 288, 250, 244, 262, 270, 260, 247, 273, 271, 287, 258, 243, 264, 253, 245, 257, 243, 273, 233, 273, 222, 271, 267, 266, 226, 230, 238, 238, 243, 262, 265, 238, 272, 247, 240, 252, 271, 232, 233, 272, 262, 245, 253, 260, 255, 246, 274, 248, 236, 243, 242, 215, 278, 250, 240, 230, 272, 238, 227, 284, 237, 247, 247, 281, 254, 251, 282, 283, 207, 264, 252, 258, 252, 257, 235, 240, 256, 255, 248, 256, 228, 224, 281, 268, 254, 266, 230, 255, 263, 281, 244, 282, 249, 264, 283, 244, 254, 243, 256, 278, 255] payload_rows_per_dst=[5580, 5572, 5450, 5462, 5412, 5420, 5444, 5528] dup_rows_per_dst=[2822, 2908, 2754, 2590, 2576, 2640, 2612, 2766] combine_rows_per_src=[5580, 5572, 5450, 5462, 5412, 5420, 5444, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229200 combine_payload_bytes=156508160 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156737360 ingress_bytes=250880000 link_bytes=250880000 token_bytes=313245520 weight_movement_bytes=94371840 total_nvlink_bytes=407617360 weight_to_token_ratio=0.301271 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.403808719e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43468 payload_rows_remote=37976 dup_rows=22068 dup_rows_remote=19272 dedup_ratio=0.336731 router_tokens_per_expert=[251, 249, 237, 260, 272, 249, 264, 256, 245, 240, 260, 242, 246, 255, 268, 271, 252, 279, 280, 281, 247, 271, 245, 261, 258, 268, 285, 262, 252, 262, 262, 220, 274, 241, 251, 243, 258, 231, 262, 219, 240, 268, 249, 254, 269, 265, 265, 259, 264, 260, 258, 250, 242, 257, 258, 240, 277, 258, 253, 239, 276, 249, 271, 244, 229, 245, 246, 266, 273, 236, 237, 262, 241, 285, 278, 253, 279, 239, 251, 278, 263, 256, 252, 246, 283, 211, 251, 260, 270, 267, 290, 257, 255, 247, 243, 262, 251, 252, 241, 245, 274, 265, 282, 223, 261, 257, 243, 257, 241, 235, 252, 249, 228, 269, 256, 268, 261, 261, 245, 251, 264, 258, 283, 243, 246, 242, 277, 283] payload_rows_per_dst=[5284, 5246, 5442, 5416, 5528, 5508, 5552, 5492] dup_rows_per_dst=[2472, 2316, 2808, 2800, 3032, 2938, 2906, 2796] combine_rows_per_src=[5284, 5246, 5442, 5416, 5528, 5508, 5552, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228952 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157310552 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392152 weight_movement_bytes=18874368 total_nvlink_bytes=333266520 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38124 dup_rows=21890 dup_rows_remote=19198 dedup_ratio=0.334015 router_tokens_per_expert=[272, 266, 259, 233, 245, 248, 277, 256, 234, 251, 264, 281, 226, 236, 245, 287, 248, 270, 263, 249, 231, 220, 265, 269, 266, 258, 252, 277, 223, 271, 244, 250, 282, 285, 239, 228, 265, 249, 257, 232, 236, 256, 237, 225, 290, 281, 272, 236, 269, 265, 233, 269, 269, 254, 237, 236, 235, 274, 239, 289, 282, 267, 237, 252, 272, 248, 222, 258, 226, 264, 272, 257, 230, 248, 240, 246, 265, 259, 277, 253, 243, 284, 257, 247, 265, 259, 251, 247, 258, 234, 268, 267, 278, 250, 278, 254, 277, 253, 264, 227, 241, 268, 261, 276, 291, 281, 263, 241, 253, 244, 263, 265, 255, 284, 243, 233, 264, 244, 245, 257, 265, 282, 238, 279, 268, 261, 238, 254] payload_rows_per_dst=[5334, 5410, 5454, 5522, 5464, 5442, 5570, 5450] dup_rows_per_dst=[2826, 2702, 2686, 2692, 2610, 2838, 2766, 2770] combine_rows_per_src=[5334, 5410, 5454, 5522, 5464, 5442, 5570, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229328 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589456 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314949584 weight_movement_bytes=18874368 total_nvlink_bytes=333823952 weight_to_token_ratio=0.059928 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155549696 dispatch_meta_bytes=228992 combine_payload_bytes=155549696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155778688 ingress_bytes=174424064 link_bytes=174424064 token_bytes=311328384 weight_movement_bytes=18874368 total_nvlink_bytes=330202752 weight_to_token_ratio=0.060625 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671245559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38030 dup_rows=21926 dup_rows_remote=18916 dedup_ratio=0.334564 router_tokens_per_expert=[256, 250, 250, 273, 256, 273, 229, 245, 246, 245, 269, 251, 272, 269, 262, 254, 242, 231, 254, 263, 238, 248, 256, 262, 218, 261, 243, 280, 255, 254, 256, 267, 269, 245, 225, 230, 254, 265, 248, 215, 249, 248, 260, 258, 246, 287, 256, 278, 244, 245, 234, 258, 243, 237, 279, 265, 252, 245, 240, 284, 256, 250, 243, 286, 237, 255, 224, 257, 252, 258, 246, 247, 258, 262, 256, 262, 248, 255, 260, 252, 258, 237, 244, 256, 270, 253, 253, 273, 247, 245, 271, 232, 254, 274, 263, 257, 270, 269, 266, 260, 243, 272, 292, 248, 263, 291, 269, 259, 274, 259, 276, 284, 255, 269, 270, 266, 250, 267, 272, 220, 252, 251, 244, 270, 232, 248, 267, 302] payload_rows_per_dst=[5474, 5374, 5450, 5460, 5408, 5436, 5580, 5428] dup_rows_per_dst=[2726, 2682, 2616, 2662, 2650, 2738, 3010, 2842] combine_rows_per_src=[5474, 5374, 5450, 5460, 5408, 5436, 5580, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229288 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156385192 ingress_bytes=231653376 link_bytes=231653376 token_bytes=312541096 weight_movement_bytes=75497472 total_nvlink_bytes=388038568 weight_to_token_ratio=0.241560 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219588668e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38246 dup_rows=21860 dup_rows_remote=19038 dedup_ratio=0.333557 router_tokens_per_expert=[230, 257, 231, 270, 241, 263, 253, 248, 255, 257, 249, 242, 244, 215, 251, 263, 263, 271, 253, 275, 250, 273, 254, 266, 222, 268, 221, 239, 275, 271, 261, 231, 254, 252, 276, 247, 243, 268, 269, 246, 237, 245, 271, 266, 275, 269, 246, 227, 272, 245, 283, 243, 241, 262, 261, 255, 283, 265, 241, 260, 272, 244, 286, 250, 252, 259, 253, 263, 270, 243, 253, 278, 250, 228, 258, 279, 279, 267, 261, 278, 262, 244, 225, 269, 272, 248, 254, 234, 238, 254, 266, 252, 265, 262, 262, 240, 240, 261, 244, 252, 280, 264, 251, 256, 254, 294, 277, 256, 261, 271, 278, 279, 244, 257, 242, 257, 267, 280, 249, 242, 274, 223, 243, 230, 256, 258, 246, 248] payload_rows_per_dst=[5430, 5510, 5460, 5498, 5480, 5530, 5430, 5338] dup_rows_per_dst=[2822, 2676, 2794, 2878, 2862, 2564, 2692, 2572] combine_rows_per_src=[5430, 5510, 5460, 5498, 5480, 5530, 5430, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38278 dup_rows=21744 dup_rows_remote=18956 dedup_ratio=0.331787 router_tokens_per_expert=[233, 253, 263, 226, 253, 242, 252, 265, 273, 274, 241, 254, 245, 216, 258, 281, 265, 261, 273, 246, 237, 281, 247, 229, 278, 289, 235, 281, 261, 259, 248, 250, 237, 259, 242, 265, 254, 248, 290, 264, 255, 256, 269, 294, 235, 250, 223, 242, 232, 277, 261, 257, 278, 279, 274, 241, 241, 272, 259, 236, 245, 258, 249, 253, 254, 253, 258, 253, 225, 252, 253, 247, 249, 247, 248, 209, 235, 256, 234, 264, 257, 267, 244, 249, 244, 265, 282, 246, 269, 287, 248, 260, 261, 249, 267, 256, 266, 264, 259, 256, 235, 253, 246, 256, 274, 281, 263, 274, 236, 253, 277, 272, 266, 264, 240, 263, 256, 252, 266, 276, 274, 257, 250, 247, 261, 264, 274, 241] payload_rows_per_dst=[5426, 5522, 5488, 5488, 5366, 5514, 5570, 5418] dup_rows_per_dst=[2632, 2758, 2678, 2736, 2508, 2788, 2760, 2884] combine_rows_per_src=[5426, 5522, 5488, 5488, 5366, 5514, 5570, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38170 dup_rows=21908 dup_rows_remote=19286 dedup_ratio=0.334290 router_tokens_per_expert=[265, 257, 239, 278, 251, 257, 246, 268, 253, 243, 271, 257, 252, 261, 266, 265, 215, 269, 251, 278, 234, 281, 239, 258, 258, 260, 249, 273, 244, 273, 235, 223, 268, 264, 241, 252, 286, 274, 246, 244, 256, 244, 234, 234, 243, 244, 255, 262, 240, 267, 248, 287, 261, 264, 264, 275, 255, 268, 239, 253, 233, 248, 273, 272, 283, 253, 226, 268, 273, 242, 238, 241, 232, 252, 289, 262, 267, 236, 248, 251, 248, 247, 264, 240, 255, 230, 263, 246, 243, 277, 253, 266, 280, 251, 264, 248, 244, 216, 251, 288, 221, 267, 253, 273, 257, 275, 281, 259, 275, 237, 267, 250, 258, 272, 251, 264, 238, 257, 296, 263, 273, 248, 268, 244, 244, 258, 251, 270] payload_rows_per_dst=[5454, 5458, 5440, 5510, 5418, 5474, 5380, 5494] dup_rows_per_dst=[2804, 2622, 2654, 2784, 2704, 2676, 2848, 2816] combine_rows_per_src=[5454, 5458, 5440, 5510, 5418, 5474, 5380, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=227784 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155998664 ingress_bytes=212393984 link_bytes=212393984 token_bytes=311769544 weight_movement_bytes=56623104 total_nvlink_bytes=368392648 weight_to_token_ratio=0.181618 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035054650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229136 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156884752 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540368 weight_movement_bytes=18874368 total_nvlink_bytes=332414736 weight_to_token_ratio=0.060198 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38372 dup_rows=21758 dup_rows_remote=19030 dedup_ratio=0.332001 router_tokens_per_expert=[248, 226, 268, 256, 241, 231, 258, 236, 295, 279, 262, 253, 290, 264, 226, 260, 263, 249, 270, 248, 261, 247, 257, 247, 234, 263, 248, 240, 279, 268, 253, 231, 275, 248, 260, 252, 241, 268, 267, 249, 255, 231, 244, 249, 250, 239, 247, 258, 243, 243, 251, 269, 231, 280, 301, 249, 259, 260, 249, 265, 273, 270, 267, 259, 268, 234, 260, 252, 268, 256, 250, 242, 244, 287, 258, 243, 254, 251, 240, 260, 256, 259, 262, 264, 233, 258, 248, 271, 255, 256, 240, 270, 263, 251, 280, 239, 252, 260, 260, 281, 249, 250, 229, 244, 254, 257, 269, 256, 260, 245, 260, 249, 254, 274, 246, 274, 257, 262, 240, 276, 279, 256, 265, 263, 240, 267, 268, 247] payload_rows_per_dst=[5406, 5446, 5452, 5546, 5406, 5482, 5460, 5580] dup_rows_per_dst=[2780, 2670, 2614, 2792, 2728, 2728, 2690, 2756] combine_rows_per_src=[5406, 5446, 5452, 5546, 5406, 5482, 5460, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=228936 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157015624 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313802312 weight_movement_bytes=18874368 total_nvlink_bytes=332676680 weight_to_token_ratio=0.060147 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38368 dup_rows=21692 dup_rows_remote=19108 dedup_ratio=0.330994 router_tokens_per_expert=[249, 264, 287, 246, 271, 268, 238, 263, 246, 278, 236, 281, 243, 243, 232, 257, 241, 299, 222, 255, 271, 250, 243, 248, 256, 260, 271, 261, 294, 245, 241, 238, 214, 253, 256, 242, 256, 251, 254, 256, 259, 249, 254, 239, 262, 243, 285, 257, 268, 257, 265, 252, 280, 263, 265, 261, 266, 247, 265, 274, 237, 264, 217, 252, 231, 245, 272, 241, 255, 293, 267, 251, 246, 240, 270, 248, 261, 261, 248, 258, 269, 247, 227, 241, 283, 249, 271, 236, 257, 286, 253, 272, 263, 249, 251, 243, 242, 226, 271, 241, 265, 261, 238, 239, 248, 246, 264, 274, 283, 242, 241, 277, 259, 237, 229, 275, 252, 279, 290, 256, 317, 249, 272, 231, 257, 256, 246, 261] payload_rows_per_dst=[5514, 5448, 5476, 5484, 5492, 5448, 5440, 5542] dup_rows_per_dst=[2690, 2742, 2584, 2782, 2682, 2746, 2676, 2790] combine_rows_per_src=[5514, 5448, 5476, 5484, 5492, 5448, 5440, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229824 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156574144 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312918464 weight_movement_bytes=18874368 total_nvlink_bytes=331792832 weight_to_token_ratio=0.060317 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38380 dup_rows=21750 dup_rows_remote=19088 dedup_ratio=0.331879 router_tokens_per_expert=[252, 261, 249, 253, 288, 268, 248, 261, 271, 244, 253, 260, 225, 268, 291, 257, 238, 274, 277, 239, 267, 261, 275, 266, 247, 257, 250, 271, 254, 244, 266, 234, 264, 260, 256, 269, 263, 261, 248, 280, 267, 276, 242, 260, 242, 236, 266, 248, 264, 241, 263, 242, 251, 262, 281, 267, 269, 277, 250, 247, 236, 267, 270, 264, 259, 234, 248, 231, 236, 254, 246, 263, 255, 272, 265, 253, 270, 257, 260, 220, 277, 234, 255, 245, 247, 269, 286, 246, 259, 271, 214, 273, 251, 253, 279, 244, 264, 247, 251, 246, 271, 263, 241, 254, 256, 233, 235, 254, 277, 273, 261, 248, 258, 247, 215, 289, 248, 252, 262, 247, 249, 253, 248, 238, 241, 243, 248, 272] payload_rows_per_dst=[5470, 5498, 5530, 5434, 5552, 5444, 5452, 5406] dup_rows_per_dst=[2826, 2742, 2746, 2542, 2802, 2762, 2668, 2662] combine_rows_per_src=[5470, 5498, 5530, 5434, 5552, 5444, 5452, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38224 dup_rows=21800 dup_rows_remote=19010 dedup_ratio=0.332642 router_tokens_per_expert=[245, 271, 259, 265, 280, 246, 257, 270, 282, 260, 274, 251, 232, 274, 261, 271, 233, 258, 272, 255, 260, 241, 268, 282, 240, 257, 253, 268, 248, 264, 260, 249, 233, 240, 250, 241, 246, 275, 258, 231, 234, 253, 249, 253, 275, 268, 240, 257, 282, 254, 261, 267, 264, 240, 258, 236, 277, 251, 255, 256, 267, 285, 255, 243, 277, 259, 238, 251, 271, 288, 259, 260, 293, 257, 250, 235, 252, 269, 251, 264, 274, 266, 235, 245, 236, 252, 275, 241, 241, 270, 254, 276, 255, 268, 234, 238, 266, 262, 237, 251, 259, 288, 256, 253, 214, 250, 281, 235, 282, 237, 244, 237, 253, 243, 235, 260, 263, 247, 272, 257, 260, 262, 216, 264, 256, 257, 249, 228] payload_rows_per_dst=[5508, 5444, 5430, 5512, 5506, 5480, 5404, 5452] dup_rows_per_dst=[2888, 2772, 2576, 2790, 2842, 2640, 2700, 2592] combine_rows_per_src=[5508, 5444, 5430, 5512, 5506, 5480, 5404, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229608 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157401320 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314573032 weight_movement_bytes=18874368 total_nvlink_bytes=333447400 weight_to_token_ratio=0.060000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229904 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157385232 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540560 weight_movement_bytes=18874368 total_nvlink_bytes=333414928 weight_to_token_ratio=0.060006 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38400 dup_rows=21710 dup_rows_remote=18968 dedup_ratio=0.331268 router_tokens_per_expert=[253, 250, 270, 243, 264, 275, 256, 252, 263, 270, 256, 247, 245, 249, 247, 255, 250, 235, 244, 276, 264, 281, 233, 284, 254, 264, 267, 235, 247, 237, 282, 252, 241, 270, 241, 234, 258, 246, 263, 257, 264, 271, 251, 252, 239, 243, 248, 265, 265, 251, 262, 256, 279, 244, 247, 275, 263, 251, 309, 246, 265, 268, 258, 238, 249, 262, 260, 261, 255, 280, 245, 264, 238, 259, 277, 243, 258, 270, 269, 276, 218, 223, 254, 297, 259, 262, 256, 249, 288, 232, 271, 230, 262, 247, 261, 277, 265, 278, 224, 270, 266, 258, 279, 234, 239, 241, 247, 255, 237, 260, 288, 243, 260, 261, 242, 285, 235, 283, 248, 261, 237, 248, 246, 251, 241, 241, 228, 245] payload_rows_per_dst=[5504, 5498, 5408, 5530, 5536, 5450, 5426, 5474] dup_rows_per_dst=[2686, 2712, 2678, 2824, 2796, 2722, 2742, 2550] combine_rows_per_src=[5504, 5498, 5408, 5530, 5536, 5450, 5426, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229872 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434352 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638832 weight_movement_bytes=18874368 total_nvlink_bytes=333513200 weight_to_token_ratio=0.059987 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228936 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156794440 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313359944 weight_movement_bytes=18874368 total_nvlink_bytes=332234312 weight_to_token_ratio=0.060232 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=37954 dup_rows=21944 dup_rows_remote=18978 dedup_ratio=0.334839 router_tokens_per_expert=[253, 256, 236, 263, 247, 255, 272, 241, 244, 249, 277, 252, 251, 252, 220, 264, 268, 247, 240, 274, 272, 242, 268, 263, 246, 227, 250, 252, 256, 268, 251, 266, 244, 271, 251, 244, 273, 260, 281, 254, 252, 255, 251, 265, 240, 243, 257, 251, 280, 247, 238, 247, 243, 256, 266, 245, 255, 254, 256, 250, 261, 248, 249, 264, 253, 245, 262, 272, 248, 241, 272, 259, 255, 251, 242, 266, 273, 258, 250, 240, 267, 258, 275, 253, 290, 249, 265, 270, 259, 268, 270, 225, 249, 273, 270, 254, 244, 263, 259, 279, 261, 232, 246, 278, 240, 259, 254, 260, 272, 238, 251, 249, 253, 241, 246, 259, 262, 248, 269, 274, 254, 245, 272, 259, 272, 232, 256, 286] payload_rows_per_dst=[5638, 5528, 5554, 5308, 5626, 5374, 5096, 5468] dup_rows_per_dst=[2966, 2912, 2960, 2488, 3074, 2468, 2288, 2788] combine_rows_per_src=[5638, 5528, 5554, 5308, 5626, 5374, 5096, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38170 dup_rows=21866 dup_rows_remote=19212 dedup_ratio=0.333649 router_tokens_per_expert=[237, 260, 246, 239, 271, 254, 288, 278, 270, 256, 256, 234, 258, 242, 251, 290, 237, 267, 255, 259, 262, 270, 253, 267, 234, 253, 265, 243, 229, 274, 278, 259, 255, 243, 228, 232, 268, 289, 264, 271, 262, 277, 245, 252, 249, 280, 251, 223, 249, 266, 273, 252, 237, 233, 263, 256, 261, 265, 252, 266, 256, 264, 262, 250, 248, 268, 249, 263, 258, 254, 267, 267, 252, 245, 259, 253, 269, 242, 268, 266, 227, 258, 224, 273, 255, 239, 229, 238, 271, 247, 261, 284, 263, 265, 265, 278, 282, 253, 262, 241, 238, 260, 272, 260, 256, 258, 249, 260, 261, 256, 267, 232, 243, 236, 237, 262, 240, 254, 238, 248, 265, 274, 266, 244, 276, 235, 247, 262] payload_rows_per_dst=[5502, 5452, 5446, 5472, 5492, 5500, 5414, 5392] dup_rows_per_dst=[2758, 2758, 2732, 2738, 2764, 2654, 2800, 2662] combine_rows_per_src=[5502, 5452, 5446, 5472, 5492, 5500, 5414, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38194 dup_rows=21830 dup_rows_remote=19058 dedup_ratio=0.333099 router_tokens_per_expert=[254, 258, 241, 274, 259, 262, 233, 233, 247, 288, 276, 268, 246, 221, 239, 272, 239, 246, 250, 277, 243, 263, 256, 266, 250, 286, 285, 270, 256, 266, 235, 254, 268, 235, 250, 272, 254, 273, 271, 259, 254, 249, 235, 241, 254, 279, 262, 251, 258, 246, 248, 257, 259, 258, 264, 239, 233, 275, 256, 266, 251, 278, 242, 240, 255, 264, 253, 251, 262, 249, 291, 234, 265, 252, 225, 238, 272, 243, 258, 276, 254, 285, 281, 261, 228, 258, 244, 257, 218, 262, 265, 253, 253, 235, 267, 260, 275, 261, 264, 253, 241, 264, 246, 248, 297, 255, 243, 258, 260, 247, 236, 242, 259, 257, 255, 253, 246, 234, 283, 264, 239, 242, 235, 267, 265, 279, 273, 268] payload_rows_per_dst=[5484, 5512, 5494, 5412, 5420, 5436, 5482, 5466] dup_rows_per_dst=[2658, 2772, 2720, 2728, 2756, 2726, 2698, 2772] combine_rows_per_src=[5484, 5512, 5494, 5412, 5420, 5436, 5482, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229472 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157515872 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802272 weight_movement_bytes=18874368 total_nvlink_bytes=333676640 weight_to_token_ratio=0.059956 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38082 dup_rows=21980 dup_rows_remote=19294 dedup_ratio=0.335388 router_tokens_per_expert=[233, 259, 240, 280, 239, 236, 276, 250, 278, 245, 301, 272, 241, 279, 268, 264, 258, 268, 265, 244, 271, 262, 260, 260, 253, 260, 288, 252, 246, 271, 270, 266, 258, 241, 229, 237, 257, 235, 260, 243, 264, 239, 264, 249, 242, 269, 265, 239, 239, 259, 268, 255, 257, 248, 250, 274, 263, 256, 262, 267, 252, 227, 256, 265, 253, 261, 263, 264, 243, 262, 237, 246, 237, 271, 216, 258, 286, 268, 269, 246, 256, 244, 271, 270, 246, 275, 247, 254, 221, 258, 247, 263, 259, 245, 265, 246, 246, 243, 235, 255, 254, 251, 268, 248, 274, 263, 267, 264, 225, 275, 282, 277, 249, 263, 257, 241, 268, 240, 255, 244, 254, 228, 243, 243, 250, 295, 275, 245] payload_rows_per_dst=[5502, 5536, 5374, 5450, 5474, 5346, 5452, 5422] dup_rows_per_dst=[2820, 2852, 2608, 2746, 2686, 2788, 2802, 2678] combine_rows_per_src=[5502, 5536, 5374, 5450, 5474, 5346, 5452, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=155459584 dispatch_meta_bytes=227728 combine_payload_bytes=155459584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155687312 ingress_bytes=174333952 link_bytes=174333952 token_bytes=311146896 weight_movement_bytes=18874368 total_nvlink_bytes=330021264 weight_to_token_ratio=0.060661 gemm_rows_defined=33280 gemm_rows_charged=36864 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.097222 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670382150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229528 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156573848 ingress_bytes=194093056 link_bytes=194093056 token_bytes=312918168 weight_movement_bytes=37748736 total_nvlink_bytes=350666904 weight_to_token_ratio=0.120635 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859704163e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38234 dup_rows=21880 dup_rows_remote=19306 dedup_ratio=0.333862 router_tokens_per_expert=[243, 260, 257, 276, 235, 251, 243, 279, 268, 248, 263, 271, 234, 286, 271, 273, 230, 250, 245, 279, 258, 222, 250, 273, 244, 269, 247, 262, 247, 254, 301, 264, 248, 242, 245, 250, 242, 272, 245, 260, 260, 244, 240, 264, 257, 234, 255, 240, 234, 251, 247, 294, 234, 243, 250, 260, 246, 259, 246, 258, 247, 283, 271, 252, 260, 253, 260, 254, 258, 260, 266, 249, 266, 245, 247, 239, 244, 272, 251, 276, 251, 262, 251, 253, 262, 254, 275, 264, 250, 253, 261, 278, 230, 256, 269, 255, 265, 273, 243, 281, 246, 261, 235, 252, 263, 264, 256, 250, 257, 271, 287, 263, 267, 243, 254, 239, 254, 218, 232, 249, 254, 243, 273, 272, 267, 263, 272, 251] payload_rows_per_dst=[5512, 5514, 5422, 5438, 5376, 5474, 5488, 5432] dup_rows_per_dst=[2804, 2676, 2574, 2712, 2824, 2774, 2846, 2670] combine_rows_per_src=[5512, 5514, 5422, 5438, 5376, 5474, 5488, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229008 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156671632 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114256 weight_movement_bytes=37748736 total_nvlink_bytes=350862992 weight_to_token_ratio=0.120559 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=37990 dup_rows=21874 dup_rows_remote=18886 dedup_ratio=0.333771 router_tokens_per_expert=[240, 248, 285, 250, 273, 237, 234, 261, 262, 248, 257, 243, 255, 264, 251, 230, 239, 255, 240, 287, 274, 243, 244, 261, 246, 255, 253, 259, 277, 223, 250, 265, 273, 263, 248, 255, 285, 268, 247, 255, 247, 234, 279, 241, 256, 261, 249, 255, 276, 259, 288, 254, 272, 257, 262, 253, 266, 236, 249, 240, 258, 274, 256, 269, 260, 265, 258, 269, 260, 269, 239, 272, 270, 281, 232, 267, 230, 237, 234, 228, 264, 242, 261, 274, 244, 231, 273, 269, 279, 251, 250, 261, 263, 277, 253, 261, 246, 233, 247, 280, 258, 265, 230, 247, 246, 250, 267, 233, 250, 245, 290, 256, 247, 235, 293, 311, 256, 243, 233, 252, 252, 254, 250, 267, 257, 239, 260, 258] payload_rows_per_dst=[5166, 5228, 5642, 5512, 5430, 5522, 5490, 5672] dup_rows_per_dst=[2390, 2468, 3068, 2826, 2656, 2800, 2678, 2988] combine_rows_per_src=[5166, 5228, 5642, 5512, 5430, 5522, 5490, 5672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38358 dup_rows=21720 dup_rows_remote=18998 dedup_ratio=0.331421 router_tokens_per_expert=[296, 256, 274, 241, 252, 251, 254, 272, 252, 249, 242, 253, 254, 229, 270, 275, 237, 248, 231, 288, 279, 237, 252, 251, 277, 261, 254, 243, 250, 243, 252, 269, 246, 260, 261, 256, 215, 247, 293, 255, 245, 252, 232, 259, 269, 241, 231, 269, 260, 259, 226, 242, 238, 258, 276, 253, 250, 307, 254, 246, 285, 259, 234, 243, 284, 264, 275, 261, 233, 261, 272, 276, 239, 263, 238, 265, 247, 268, 250, 233, 267, 252, 269, 265, 243, 249, 275, 256, 246, 243, 237, 259, 256, 260, 260, 272, 259, 249, 281, 238, 248, 271, 236, 245, 241, 278, 255, 253, 279, 239, 276, 270, 242, 263, 269, 265, 240, 267, 260, 251, 264, 271, 248, 241, 262, 250, 259, 247] payload_rows_per_dst=[5458, 5474, 5450, 5458, 5558, 5480, 5430, 5508] dup_rows_per_dst=[2782, 2670, 2612, 2722, 2700, 2738, 2806, 2690] combine_rows_per_src=[5458, 5474, 5450, 5458, 5558, 5480, 5430, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=229504 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156213376 ingress_bytes=212606976 link_bytes=212606976 token_bytes=312197248 weight_movement_bytes=56623104 total_nvlink_bytes=368820352 weight_to_token_ratio=0.181370 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037095435e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=230160 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156836624 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313443088 weight_movement_bytes=18874368 total_nvlink_bytes=332317456 weight_to_token_ratio=0.060216 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38410 dup_rows=21668 dup_rows_remote=18922 dedup_ratio=0.330627 router_tokens_per_expert=[231, 253, 260, 266, 242, 242, 231, 263, 246, 252, 278, 252, 267, 279, 249, 269, 266, 255, 241, 253, 280, 247, 266, 261, 265, 257, 245, 261, 242, 230, 266, 242, 263, 253, 266, 272, 257, 261, 241, 306, 247, 249, 259, 264, 247, 275, 260, 273, 275, 244, 261, 249, 236, 272, 233, 253, 256, 238, 296, 278, 254, 251, 237, 258, 257, 247, 246, 277, 224, 235, 268, 257, 258, 239, 278, 244, 256, 261, 254, 262, 256, 236, 283, 256, 281, 270, 290, 251, 242, 240, 269, 247, 269, 258, 235, 237, 237, 261, 264, 268, 262, 272, 264, 236, 236, 252, 226, 261, 267, 261, 254, 281, 252, 244, 232, 254, 233, 266, 260, 252, 244, 252, 269, 256, 273, 252, 244, 259] payload_rows_per_dst=[5472, 5458, 5572, 5450, 5456, 5522, 5458, 5480] dup_rows_per_dst=[2688, 2696, 2814, 2732, 2670, 2718, 2746, 2604] combine_rows_per_src=[5472, 5458, 5572, 5450, 5456, 5522, 5458, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155607040 dispatch_meta_bytes=227504 combine_payload_bytes=155607040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155834544 ingress_bytes=174481408 link_bytes=174481408 token_bytes=311441584 weight_movement_bytes=18874368 total_nvlink_bytes=330315952 weight_to_token_ratio=0.060603 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671795001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229424 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343792 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314458160 weight_movement_bytes=37748736 total_nvlink_bytes=352206896 weight_to_token_ratio=0.120044 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38188 dup_rows=21942 dup_rows_remote=19134 dedup_ratio=0.334808 router_tokens_per_expert=[255, 255, 254, 263, 230, 269, 249, 278, 253, 238, 261, 283, 252, 232, 262, 250, 249, 239, 264, 274, 291, 249, 266, 274, 258, 251, 261, 242, 235, 227, 302, 271, 275, 236, 263, 244, 223, 256, 267, 258, 261, 274, 253, 239, 233, 252, 255, 267, 246, 250, 282, 249, 269, 264, 240, 246, 264, 260, 230, 260, 240, 264, 240, 240, 234, 256, 240, 265, 222, 258, 251, 275, 265, 262, 261, 258, 262, 255, 253, 236, 265, 286, 246, 258, 283, 272, 252, 280, 230, 261, 252, 232, 269, 213, 251, 257, 274, 276, 253, 241, 263, 269, 275, 254, 255, 223, 261, 272, 259, 275, 286, 261, 280, 276, 243, 265, 242, 226, 258, 255, 250, 263, 265, 255, 252, 279, 247, 218] payload_rows_per_dst=[5482, 5470, 5460, 5366, 5424, 5406, 5530, 5456] dup_rows_per_dst=[2686, 2836, 2652, 2722, 2682, 2808, 2864, 2692] combine_rows_per_src=[5482, 5470, 5460, 5366, 5424, 5406, 5530, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38238 dup_rows=21832 dup_rows_remote=18986 dedup_ratio=0.333130 router_tokens_per_expert=[269, 260, 249, 262, 257, 241, 272, 266, 275, 253, 273, 266, 267, 252, 264, 226, 261, 286, 248, 269, 228, 230, 243, 259, 298, 265, 249, 263, 255, 242, 250, 256, 259, 267, 260, 236, 242, 265, 253, 236, 272, 267, 257, 249, 281, 247, 265, 264, 268, 249, 282, 228, 255, 253, 274, 240, 241, 223, 236, 259, 261, 248, 240, 262, 267, 263, 270, 260, 235, 268, 235, 245, 252, 278, 280, 259, 223, 242, 267, 255, 262, 269, 224, 270, 273, 261, 264, 274, 261, 258, 259, 276, 265, 228, 250, 267, 269, 253, 240, 256, 237, 246, 279, 255, 271, 255, 241, 261, 243, 267, 253, 234, 256, 222, 228, 259, 263, 256, 276, 264, 242, 261, 275, 246, 257, 275, 236, 239] payload_rows_per_dst=[5466, 5464, 5536, 5410, 5452, 5510, 5426, 5440] dup_rows_per_dst=[2846, 2732, 2704, 2628, 2746, 2812, 2694, 2670] combine_rows_per_src=[5466, 5464, 5536, 5410, 5452, 5510, 5426, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229328 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157556688 ingress_bytes=213950464 link_bytes=213950464 token_bytes=314884048 weight_movement_bytes=56623104 total_nvlink_bytes=371507152 weight_to_token_ratio=0.179822 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049968076e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38396 dup_rows=21750 dup_rows_remote=19050 dedup_ratio=0.331879 router_tokens_per_expert=[226, 242, 235, 252, 257, 265, 250, 248, 257, 232, 239, 225, 271, 244, 229, 243, 247, 234, 223, 307, 232, 255, 262, 247, 241, 246, 253, 244, 256, 268, 267, 263, 278, 256, 249, 258, 248, 257, 267, 233, 249, 235, 232, 264, 235, 286, 251, 259, 251, 284, 240, 283, 247, 256, 226, 249, 243, 272, 241, 260, 253, 277, 261, 275, 250, 240, 261, 244, 244, 270, 249, 279, 261, 256, 241, 266, 252, 267, 277, 280, 280, 249, 252, 245, 271, 235, 265, 263, 287, 250, 247, 284, 258, 271, 248, 241, 257, 238, 257, 262, 246, 271, 276, 273, 275, 247, 272, 265, 282, 254, 253, 260, 276, 261, 263, 277, 265, 260, 259, 282, 223, 249, 240, 266, 293, 276, 226, 246] payload_rows_per_dst=[5330, 5390, 5442, 5474, 5516, 5524, 5542, 5568] dup_rows_per_dst=[2500, 2700, 2672, 2762, 2758, 2768, 2834, 2756] combine_rows_per_src=[5330, 5390, 5442, 5474, 5516, 5524, 5542, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38188 dup_rows=21884 dup_rows_remote=19140 dedup_ratio=0.333923 router_tokens_per_expert=[234, 249, 240, 262, 254, 256, 258, 260, 264, 274, 267, 258, 233, 264, 262, 264, 278, 253, 264, 266, 268, 251, 252, 263, 271, 233, 257, 256, 255, 252, 278, 242, 261, 256, 259, 277, 258, 264, 244, 249, 269, 228, 249, 250, 239, 269, 249, 257, 259, 271, 281, 236, 265, 259, 231, 269, 237, 279, 246, 279, 251, 280, 218, 253, 248, 262, 261, 255, 276, 254, 244, 244, 266, 277, 257, 221, 254, 264, 267, 254, 256, 261, 245, 245, 244, 277, 256, 248, 273, 240, 264, 251, 283, 271, 230, 263, 263, 245, 250, 287, 241, 255, 244, 266, 251, 241, 248, 252, 256, 217, 274, 241, 225, 246, 264, 272, 251, 250, 255, 246, 265, 263, 249, 262, 265, 272, 253, 258] payload_rows_per_dst=[5514, 5520, 5454, 5438, 5464, 5386, 5412, 5464] dup_rows_per_dst=[2684, 2758, 2702, 2790, 2744, 2828, 2650, 2728] combine_rows_per_src=[5514, 5520, 5454, 5438, 5464, 5386, 5412, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229288 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156647336 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065384 weight_movement_bytes=18874368 total_nvlink_bytes=331939752 weight_to_token_ratio=0.060289 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228896 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156851744 ingress_bytes=213245952 link_bytes=213245952 token_bytes=313474592 weight_movement_bytes=56623104 total_nvlink_bytes=370097696 weight_to_token_ratio=0.180631 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043217789e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38244 dup_rows=21742 dup_rows_remote=18948 dedup_ratio=0.331757 router_tokens_per_expert=[255, 276, 240, 263, 240, 291, 264, 266, 245, 233, 223, 259, 276, 265, 257, 298, 223, 266, 259, 255, 273, 263, 253, 277, 226, 277, 248, 244, 266, 258, 243, 215, 272, 250, 253, 264, 258, 288, 259, 246, 249, 256, 276, 262, 245, 273, 273, 248, 264, 244, 277, 254, 269, 287, 253, 253, 237, 243, 243, 267, 246, 256, 260, 244, 241, 264, 256, 292, 246, 255, 254, 285, 251, 236, 263, 249, 258, 263, 242, 240, 245, 251, 254, 224, 270, 256, 218, 246, 251, 248, 257, 252, 255, 245, 266, 253, 270, 258, 272, 268, 244, 253, 261, 238, 255, 272, 245, 249, 260, 272, 278, 281, 235, 258, 250, 259, 264, 236, 264, 254, 272, 239, 289, 238, 231, 263, 253, 235] payload_rows_per_dst=[5482, 5418, 5550, 5468, 5514, 5430, 5512, 5420] dup_rows_per_dst=[2820, 2674, 2794, 2726, 2676, 2552, 2840, 2660] combine_rows_per_src=[5482, 5418, 5550, 5468, 5514, 5430, 5512, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229784 combine_payload_bytes=157270016 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157499800 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769816 weight_movement_bytes=56623104 total_nvlink_bytes=371392920 weight_to_token_ratio=0.179887 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38122 dup_rows=21830 dup_rows_remote=19106 dedup_ratio=0.333099 router_tokens_per_expert=[256, 256, 235, 243, 252, 261, 269, 240, 254, 290, 269, 258, 258, 256, 264, 264, 267, 248, 266, 239, 275, 252, 270, 271, 251, 249, 262, 240, 271, 245, 243, 242, 260, 225, 245, 261, 274, 264, 239, 235, 262, 293, 267, 253, 243, 264, 247, 246, 280, 284, 249, 269, 249, 261, 240, 233, 271, 273, 273, 249, 255, 236, 249, 261, 272, 276, 221, 298, 269, 233, 243, 267, 249, 250, 276, 218, 240, 250, 245, 281, 258, 244, 271, 255, 226, 264, 245, 247, 257, 259, 265, 263, 253, 248, 257, 266, 259, 254, 244, 277, 227, 232, 247, 266, 250, 248, 242, 257, 284, 251, 237, 251, 239, 292, 240, 281, 258, 246, 279, 232, 257, 278, 275, 283, 245, 248, 240, 257] payload_rows_per_dst=[5464, 5482, 5448, 5532, 5398, 5392, 5406, 5584] dup_rows_per_dst=[2786, 2650, 2708, 2732, 2778, 2778, 2674, 2724] combine_rows_per_src=[5464, 5482, 5448, 5532, 5398, 5392, 5406, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38050 dup_rows=21888 dup_rows_remote=18966 dedup_ratio=0.333984 router_tokens_per_expert=[220, 241, 266, 282, 264, 265, 244, 242, 239, 251, 266, 242, 271, 263, 257, 273, 264, 263, 256, 255, 229, 262, 261, 245, 234, 239, 263, 249, 278, 241, 270, 221, 253, 257, 278, 243, 230, 254, 249, 265, 247, 269, 246, 258, 250, 253, 267, 266, 274, 293, 260, 237, 259, 267, 288, 271, 264, 277, 262, 256, 260, 259, 266, 267, 237, 258, 244, 240, 267, 246, 267, 245, 254, 266, 251, 223, 266, 256, 248, 273, 234, 233, 250, 251, 249, 260, 238, 234, 270, 256, 248, 275, 257, 243, 243, 264, 282, 284, 270, 267, 259, 243, 271, 273, 235, 269, 233, 269, 289, 277, 266, 257, 258, 228, 284, 269, 241, 277, 271, 250, 228, 242, 248, 233, 250, 220, 264, 254] payload_rows_per_dst=[5424, 5374, 5530, 5598, 5426, 5348, 5578, 5370] dup_rows_per_dst=[2748, 2686, 2640, 2922, 2656, 2662, 2910, 2664] combine_rows_per_src=[5424, 5374, 5530, 5598, 5426, 5348, 5578, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229312 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156647360 ingress_bytes=194166784 link_bytes=194166784 token_bytes=313065408 weight_movement_bytes=37748736 total_nvlink_bytes=350814144 weight_to_token_ratio=0.120578 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860410588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=228768 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876192 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313523616 weight_movement_bytes=18874368 total_nvlink_bytes=332397984 weight_to_token_ratio=0.060201 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38224 dup_rows=21838 dup_rows_remote=19164 dedup_ratio=0.333221 router_tokens_per_expert=[261, 266, 291, 273, 259, 275, 261, 266, 262, 277, 250, 250, 257, 236, 265, 287, 254, 266, 270, 253, 250, 252, 230, 245, 281, 253, 281, 246, 241, 256, 230, 282, 247, 245, 238, 278, 253, 307, 271, 270, 263, 257, 268, 256, 283, 245, 256, 272, 251, 252, 248, 230, 221, 205, 250, 255, 273, 273, 236, 228, 258, 268, 265, 259, 272, 250, 244, 247, 254, 256, 259, 265, 275, 263, 260, 259, 254, 253, 249, 238, 253, 253, 269, 244, 232, 239, 255, 239, 243, 251, 241, 240, 241, 260, 266, 255, 271, 242, 258, 264, 251, 253, 222, 259, 246, 265, 237, 268, 254, 259, 236, 289, 255, 263, 283, 248, 254, 247, 261, 257, 244, 246, 274, 267, 249, 245, 266, 249] payload_rows_per_dst=[5640, 5480, 5562, 5304, 5410, 5354, 5474, 5474] dup_rows_per_dst=[2832, 2700, 2856, 2640, 2786, 2608, 2674, 2742] combine_rows_per_src=[5640, 5480, 5562, 5304, 5410, 5354, 5474, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228912 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376624 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524336 weight_movement_bytes=18874368 total_nvlink_bytes=331398704 weight_to_token_ratio=0.060393 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=228064 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156080864 ingress_bytes=212475904 link_bytes=212475904 token_bytes=311933664 weight_movement_bytes=56623104 total_nvlink_bytes=368556768 weight_to_token_ratio=0.181523 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035839567e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38252 dup_rows=21760 dup_rows_remote=19074 dedup_ratio=0.332031 router_tokens_per_expert=[241, 232, 250, 254, 264, 258, 240, 261, 228, 253, 251, 272, 243, 251, 245, 268, 227, 266, 260, 239, 252, 261, 271, 244, 269, 274, 260, 269, 246, 257, 232, 257, 249, 285, 261, 237, 256, 256, 253, 257, 253, 254, 250, 247, 256, 275, 258, 241, 244, 244, 262, 254, 259, 271, 246, 259, 273, 271, 260, 248, 228, 266, 251, 240, 225, 252, 276, 258, 263, 238, 259, 260, 262, 273, 273, 259, 275, 268, 256, 248, 227, 254, 254, 261, 258, 271, 284, 256, 241, 253, 253, 284, 242, 250, 271, 246, 254, 266, 274, 242, 247, 295, 267, 271, 243, 252, 244, 261, 262, 278, 254, 268, 250, 288, 251, 272, 240, 250, 223, 260, 248, 255, 256, 267, 263, 246, 252, 260] payload_rows_per_dst=[5366, 5440, 5466, 5504, 5548, 5524, 5506, 5422] dup_rows_per_dst=[2656, 2728, 2710, 2648, 2742, 2686, 2850, 2740] combine_rows_per_src=[5366, 5440, 5466, 5504, 5548, 5524, 5506, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38256 dup_rows=21810 dup_rows_remote=19068 dedup_ratio=0.332794 router_tokens_per_expert=[258, 233, 227, 235, 253, 242, 266, 253, 247, 263, 255, 267, 247, 244, 282, 276, 235, 260, 283, 233, 255, 264, 262, 246, 246, 240, 256, 274, 276, 241, 257, 258, 241, 260, 251, 261, 253, 259, 263, 247, 243, 271, 248, 253, 250, 262, 276, 281, 248, 261, 259, 258, 262, 237, 247, 282, 272, 249, 248, 267, 278, 280, 266, 255, 241, 229, 265, 256, 261, 260, 276, 258, 275, 242, 270, 243, 242, 247, 290, 227, 231, 290, 238, 258, 259, 279, 291, 249, 272, 227, 265, 233, 273, 254, 228, 250, 278, 275, 217, 244, 264, 250, 252, 223, 248, 259, 264, 250, 250, 246, 249, 229, 273, 288, 240, 274, 246, 260, 257, 249, 263, 267, 287, 260, 264, 251, 266, 224] payload_rows_per_dst=[5426, 5470, 5448, 5588, 5448, 5436, 5382, 5528] dup_rows_per_dst=[2670, 2742, 2790, 2710, 2716, 2758, 2614, 2810] combine_rows_per_src=[5426, 5470, 5448, 5588, 5448, 5436, 5382, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38202 dup_rows=21924 dup_rows_remote=19322 dedup_ratio=0.334534 router_tokens_per_expert=[220, 241, 258, 281, 267, 243, 253, 232, 247, 237, 253, 253, 259, 260, 262, 240, 249, 277, 234, 253, 249, 257, 282, 247, 254, 259, 265, 258, 244, 225, 283, 246, 250, 257, 260, 246, 254, 273, 256, 245, 245, 259, 274, 229, 272, 265, 256, 264, 268, 252, 253, 265, 264, 247, 264, 262, 263, 246, 257, 247, 250, 245, 251, 257, 273, 241, 286, 259, 247, 248, 239, 253, 241, 282, 252, 269, 254, 269, 261, 274, 281, 264, 263, 222, 259, 261, 258, 248, 219, 266, 260, 285, 284, 255, 257, 263, 255, 268, 260, 238, 271, 259, 278, 248, 232, 238, 251, 266, 260, 283, 260, 258, 239, 237, 267, 259, 262, 247, 256, 250, 272, 253, 272, 233, 249, 259, 248, 263] payload_rows_per_dst=[5410, 5616, 5494, 5380, 5498, 5380, 5458, 5376] dup_rows_per_dst=[2602, 2818, 2842, 2676, 2798, 2722, 2710, 2756] combine_rows_per_src=[5410, 5616, 5494, 5380, 5498, 5380, 5458, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229552 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156795056 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313360560 weight_movement_bytes=37748736 total_nvlink_bytes=351109296 weight_to_token_ratio=0.120464 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38118 dup_rows=21874 dup_rows_remote=19082 dedup_ratio=0.333771 router_tokens_per_expert=[251, 264, 265, 237, 249, 219, 263, 245, 259, 258, 286, 249, 261, 251, 278, 249, 247, 256, 225, 262, 248, 247, 251, 241, 320, 233, 309, 276, 247, 260, 243, 257, 237, 262, 224, 238, 256, 229, 266, 244, 235, 224, 263, 258, 242, 254, 265, 238, 241, 259, 281, 249, 232, 294, 227, 238, 280, 267, 244, 284, 260, 246, 265, 257, 262, 250, 243, 272, 240, 284, 268, 244, 270, 263, 249, 262, 269, 242, 269, 281, 262, 269, 246, 269, 245, 271, 241, 254, 301, 268, 232, 244, 242, 261, 236, 272, 219, 255, 267, 275, 281, 254, 272, 257, 257, 268, 269, 227, 248, 249, 240, 225, 256, 259, 246, 255, 267, 237, 265, 253, 294, 260, 263, 222, 287, 276, 259, 260] payload_rows_per_dst=[5522, 5500, 5266, 5454, 5544, 5502, 5458, 5416] dup_rows_per_dst=[2646, 2744, 2604, 2794, 2792, 2724, 2668, 2902] combine_rows_per_src=[5522, 5500, 5266, 5454, 5544, 5502, 5458, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229304 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909496 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589688 weight_movement_bytes=37748736 total_nvlink_bytes=351338424 weight_to_token_ratio=0.120376 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38242 dup_rows=21778 dup_rows_remote=19006 dedup_ratio=0.332306 router_tokens_per_expert=[262, 247, 235, 256, 254, 251, 258, 274, 250, 284, 247, 289, 255, 258, 235, 303, 277, 269, 253, 217, 256, 242, 248, 253, 286, 284, 239, 274, 272, 234, 241, 244, 274, 279, 256, 278, 239, 284, 270, 254, 245, 265, 236, 242, 274, 241, 241, 266, 269, 246, 254, 297, 265, 280, 247, 226, 248, 241, 277, 260, 247, 271, 254, 237, 258, 245, 283, 230, 231, 252, 252, 240, 239, 232, 237, 256, 252, 268, 261, 284, 250, 260, 259, 250, 247, 248, 239, 269, 263, 238, 272, 275, 247, 264, 261, 263, 271, 266, 281, 271, 260, 265, 254, 212, 273, 284, 241, 257, 272, 256, 253, 252, 248, 265, 280, 307, 239, 238, 245, 232, 258, 246, 241, 237, 248, 232, 222, 227] payload_rows_per_dst=[5528, 5426, 5516, 5560, 5408, 5468, 5480, 5372] dup_rows_per_dst=[2788, 2752, 2772, 2678, 2632, 2742, 2856, 2558] combine_rows_per_src=[5528, 5426, 5516, 5560, 5408, 5468, 5480, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229296 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925872 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622448 weight_movement_bytes=18874368 total_nvlink_bytes=332496816 weight_to_token_ratio=0.060182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=230096 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156705488 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313180880 weight_movement_bytes=18874368 total_nvlink_bytes=332055248 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38222 dup_rows=21756 dup_rows_remote=18924 dedup_ratio=0.331970 router_tokens_per_expert=[225, 290, 257, 258, 282, 245, 260, 253, 256, 245, 281, 295, 260, 268, 249, 263, 237, 298, 256, 234, 234, 251, 253, 245, 244, 267, 202, 245, 250, 242, 237, 270, 266, 294, 253, 248, 283, 243, 252, 266, 254, 264, 263, 274, 241, 264, 266, 260, 258, 248, 271, 251, 271, 243, 221, 247, 265, 245, 235, 236, 267, 235, 277, 243, 243, 246, 286, 244, 239, 274, 255, 250, 228, 272, 251, 255, 253, 246, 250, 254, 265, 274, 247, 263, 234, 234, 267, 257, 248, 272, 247, 274, 262, 271, 290, 250, 275, 235, 240, 275, 230, 264, 254, 259, 288, 266, 259, 230, 276, 261, 286, 259, 248, 219, 264, 246, 248, 247, 255, 279, 250, 241, 251, 252, 262, 263, 281, 248] payload_rows_per_dst=[5542, 5374, 5532, 5342, 5406, 5474, 5552, 5558] dup_rows_per_dst=[2760, 2556, 2850, 2596, 2686, 2714, 2762, 2832] combine_rows_per_src=[5542, 5374, 5532, 5342, 5406, 5474, 5552, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38370 dup_rows=21692 dup_rows_remote=19066 dedup_ratio=0.330994 router_tokens_per_expert=[266, 264, 248, 249, 251, 250, 241, 237, 240, 237, 269, 248, 277, 264, 273, 241, 251, 247, 227, 242, 248, 244, 257, 225, 245, 274, 241, 256, 264, 260, 259, 245, 262, 234, 239, 279, 274, 254, 247, 267, 269, 282, 266, 256, 269, 244, 262, 270, 230, 272, 267, 243, 247, 242, 238, 269, 254, 264, 269, 265, 236, 252, 243, 259, 269, 271, 249, 260, 284, 243, 243, 242, 241, 221, 257, 262, 245, 274, 233, 272, 222, 254, 266, 285, 258, 268, 279, 297, 236, 257, 250, 251, 244, 263, 270, 279, 254, 284, 269, 249, 256, 266, 263, 262, 246, 252, 239, 250, 247, 259, 280, 268, 237, 231, 257, 250, 252, 261, 275, 268, 262, 265, 247, 239, 280, 293, 247, 251] payload_rows_per_dst=[5454, 5384, 5522, 5474, 5408, 5586, 5544, 5472] dup_rows_per_dst=[2656, 2586, 2826, 2626, 2724, 2772, 2744, 2758] combine_rows_per_src=[5454, 5384, 5522, 5474, 5408, 5586, 5544, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228800 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156360128 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312491456 weight_movement_bytes=18874368 total_nvlink_bytes=331365824 weight_to_token_ratio=0.060400 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228992 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868224 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507456 weight_movement_bytes=18874368 total_nvlink_bytes=332381824 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38414 dup_rows=21680 dup_rows_remote=18956 dedup_ratio=0.330811 router_tokens_per_expert=[243, 248, 267, 266, 264, 234, 273, 251, 229, 248, 262, 266, 239, 253, 278, 282, 248, 267, 262, 260, 258, 294, 269, 270, 261, 244, 235, 251, 273, 242, 247, 244, 278, 249, 244, 265, 246, 278, 235, 243, 237, 257, 258, 250, 231, 263, 256, 242, 247, 267, 267, 246, 280, 274, 247, 259, 270, 254, 240, 254, 256, 261, 241, 245, 243, 281, 245, 258, 253, 238, 259, 252, 250, 238, 270, 272, 259, 253, 264, 248, 245, 248, 230, 252, 251, 269, 276, 265, 269, 285, 259, 257, 270, 237, 275, 274, 265, 246, 284, 247, 239, 240, 230, 261, 232, 253, 283, 255, 250, 261, 268, 269, 244, 258, 237, 238, 266, 249, 242, 247, 288, 250, 254, 242, 260, 287, 266, 244] payload_rows_per_dst=[5548, 5574, 5428, 5498, 5432, 5534, 5442, 5400] dup_rows_per_dst=[2658, 2676, 2636, 2718, 2734, 2790, 2724, 2744] combine_rows_per_src=[5548, 5574, 5428, 5498, 5432, 5534, 5442, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=228584 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156785896 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343208 weight_movement_bytes=18874368 total_nvlink_bytes=332217576 weight_to_token_ratio=0.060235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229744 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157393264 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556784 weight_movement_bytes=18874368 total_nvlink_bytes=333431152 weight_to_token_ratio=0.060003 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38050 dup_rows=21898 dup_rows_remote=18960 dedup_ratio=0.334137 router_tokens_per_expert=[272, 243, 254, 224, 266, 257, 219, 262, 236, 248, 257, 262, 230, 242, 250, 253, 274, 277, 283, 231, 245, 239, 224, 271, 259, 258, 283, 243, 270, 272, 280, 242, 265, 255, 241, 243, 256, 241, 270, 265, 243, 260, 269, 235, 240, 225, 230, 267, 274, 243, 242, 257, 255, 283, 247, 245, 282, 261, 282, 274, 260, 273, 258, 251, 248, 247, 274, 254, 251, 237, 240, 258, 229, 247, 252, 255, 248, 265, 263, 266, 249, 271, 259, 248, 263, 303, 274, 271, 274, 276, 262, 245, 283, 251, 263, 271, 269, 232, 261, 241, 236, 231, 273, 273, 278, 223, 261, 253, 281, 274, 263, 258, 247, 253, 254, 241, 250, 264, 260, 269, 244, 253, 216, 257, 261, 262, 264, 251] payload_rows_per_dst=[5306, 5430, 5422, 5544, 5372, 5588, 5550, 5426] dup_rows_per_dst=[2644, 2872, 2588, 2830, 2696, 2938, 2664, 2666] combine_rows_per_src=[5306, 5430, 5422, 5544, 5372, 5588, 5550, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229480 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157573224 ingress_bytes=195092480 link_bytes=195092480 token_bytes=314916968 weight_movement_bytes=37748736 total_nvlink_bytes=352665704 weight_to_token_ratio=0.119869 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869280152e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38416 dup_rows=21712 dup_rows_remote=18928 dedup_ratio=0.331299 router_tokens_per_expert=[283, 270, 253, 238, 270, 243, 294, 225, 261, 256, 280, 285, 260, 249, 252, 259, 259, 243, 240, 250, 212, 252, 216, 258, 253, 254, 255, 258, 258, 239, 258, 259, 261, 274, 280, 253, 250, 255, 266, 239, 262, 257, 255, 257, 274, 269, 266, 255, 261, 229, 258, 255, 235, 240, 268, 232, 254, 251, 247, 282, 271, 234, 238, 237, 281, 250, 288, 254, 238, 244, 227, 270, 252, 269, 245, 294, 265, 258, 271, 254, 262, 246, 260, 256, 253, 260, 243, 249, 277, 264, 263, 251, 292, 238, 257, 253, 276, 240, 271, 256, 249, 262, 247, 247, 262, 239, 270, 234, 249, 271, 254, 268, 266, 245, 258, 258, 248, 252, 255, 272, 253, 277, 272, 228, 252, 240, 249, 257] payload_rows_per_dst=[5556, 5408, 5472, 5372, 5562, 5548, 5480, 5426] dup_rows_per_dst=[2800, 2784, 2610, 2612, 2758, 2700, 2710, 2738] combine_rows_per_src=[5556, 5408, 5472, 5372, 5562, 5548, 5480, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43428 payload_rows_remote=38004 dup_rows=22108 dup_rows_remote=19368 dedup_ratio=0.337341 router_tokens_per_expert=[250, 267, 235, 261, 246, 252, 252, 247, 255, 275, 264, 263, 263, 252, 245, 255, 238, 272, 276, 280, 221, 244, 269, 276, 258, 224, 301, 256, 249, 262, 216, 242, 256, 259, 271, 239, 259, 258, 262, 239, 238, 269, 240, 242, 267, 249, 252, 249, 237, 254, 253, 253, 242, 268, 230, 257, 261, 279, 243, 254, 235, 263, 252, 246, 255, 259, 279, 248, 250, 266, 275, 264, 234, 279, 255, 258, 272, 256, 267, 269, 288, 266, 271, 285, 261, 275, 260, 246, 248, 262, 253, 252, 258, 229, 251, 282, 245, 225, 267, 261, 254, 240, 276, 272, 276, 255, 247, 265, 234, 276, 274, 272, 253, 257, 265, 266, 258, 255, 237, 245, 233, 236, 248, 214, 259, 237, 295, 256] payload_rows_per_dst=[5424, 5464, 5344, 5476, 5472, 5482, 5360, 5406] dup_rows_per_dst=[2740, 2812, 2754, 2902, 2792, 2892, 2594, 2622] combine_rows_per_src=[5424, 5464, 5344, 5476, 5472, 5482, 5360, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=228040 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156080840 ingress_bytes=212475904 link_bytes=212475904 token_bytes=311933640 weight_movement_bytes=56623104 total_nvlink_bytes=368556744 weight_to_token_ratio=0.181523 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035839567e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38224 dup_rows=21792 dup_rows_remote=19032 dedup_ratio=0.332520 router_tokens_per_expert=[241, 256, 236, 268, 270, 243, 262, 247, 249, 256, 260, 269, 263, 246, 262, 268, 254, 234, 275, 279, 245, 243, 266, 228, 252, 277, 258, 257, 234, 241, 231, 261, 260, 234, 268, 265, 248, 236, 251, 263, 254, 255, 250, 300, 256, 245, 278, 262, 239, 244, 261, 259, 283, 260, 260, 271, 266, 281, 268, 240, 296, 239, 264, 252, 263, 237, 256, 280, 284, 224, 300, 253, 263, 255, 274, 248, 242, 272, 238, 251, 256, 249, 273, 271, 273, 256, 243, 225, 256, 236, 245, 273, 259, 252, 282, 257, 225, 233, 251, 260, 237, 262, 265, 254, 221, 243, 256, 258, 254, 249, 251, 237, 249, 280, 238, 255, 241, 257, 265, 261, 269, 252, 280, 241, 268, 247, 254, 270] payload_rows_per_dst=[5504, 5340, 5484, 5490, 5520, 5530, 5368, 5508] dup_rows_per_dst=[2688, 2730, 2766, 2876, 2760, 2682, 2544, 2746] combine_rows_per_src=[5504, 5340, 5484, 5490, 5520, 5530, 5368, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229376 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581312 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933248 weight_movement_bytes=18874368 total_nvlink_bytes=333807616 weight_to_token_ratio=0.059931 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43528 payload_rows_remote=37972 dup_rows=22008 dup_rows_remote=19190 dedup_ratio=0.335815 router_tokens_per_expert=[260, 259, 238, 281, 241, 258, 253, 252, 269, 267, 262, 232, 261, 253, 242, 271, 248, 238, 268, 219, 258, 271, 278, 226, 272, 264, 239, 242, 227, 251, 256, 240, 264, 254, 237, 255, 248, 247, 254, 279, 273, 284, 269, 250, 267, 263, 272, 271, 261, 260, 268, 244, 254, 224, 253, 245, 253, 253, 260, 230, 250, 235, 240, 269, 250, 266, 282, 239, 239, 255, 247, 261, 286, 243, 254, 218, 262, 249, 273, 261, 257, 247, 270, 282, 266, 264, 262, 252, 285, 251, 239, 251, 247, 271, 268, 259, 258, 241, 250, 236, 258, 276, 220, 273, 230, 274, 293, 245, 239, 253, 268, 255, 243, 261, 271, 263, 268, 271, 245, 251, 291, 258, 286, 231, 275, 217, 266, 264] payload_rows_per_dst=[5472, 5288, 5556, 5352, 5438, 5530, 5450, 5442] dup_rows_per_dst=[2726, 2706, 2818, 2646, 2732, 2812, 2688, 2880] combine_rows_per_src=[5472, 5288, 5556, 5352, 5438, 5530, 5450, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38438 dup_rows=21654 dup_rows_remote=18974 dedup_ratio=0.330414 router_tokens_per_expert=[252, 256, 241, 226, 247, 240, 260, 224, 245, 268, 256, 226, 273, 256, 249, 230, 252, 244, 263, 249, 284, 230, 254, 264, 269, 257, 230, 286, 261, 264, 237, 268, 247, 271, 267, 255, 246, 281, 273, 281, 243, 268, 266, 245, 276, 265, 277, 249, 260, 253, 243, 231, 241, 272, 276, 256, 263, 271, 277, 283, 280, 241, 255, 256, 258, 260, 255, 249, 244, 255, 260, 247, 273, 238, 285, 270, 248, 280, 239, 250, 253, 281, 274, 204, 244, 264, 255, 234, 238, 244, 281, 222, 245, 296, 250, 245, 263, 283, 257, 267, 274, 255, 269, 270, 258, 269, 228, 263, 222, 255, 238, 268, 260, 266, 266, 235, 242, 254, 251, 248, 251, 252, 261, 252, 249, 267, 244, 261] payload_rows_per_dst=[5356, 5494, 5608, 5530, 5446, 5458, 5546, 5444] dup_rows_per_dst=[2542, 2730, 2806, 2786, 2684, 2694, 2732, 2680] combine_rows_per_src=[5356, 5494, 5608, 5530, 5446, 5458, 5546, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43990 payload_rows_remote=38404 dup_rows=21546 dup_rows_remote=18842 dedup_ratio=0.328766 router_tokens_per_expert=[266, 252, 254, 234, 271, 285, 278, 248, 252, 239, 244, 254, 242, 253, 242, 232, 246, 240, 266, 239, 255, 254, 267, 238, 276, 263, 232, 255, 254, 241, 254, 255, 272, 258, 234, 274, 273, 254, 251, 294, 258, 241, 246, 267, 272, 246, 250, 252, 253, 292, 257, 244, 259, 265, 278, 229, 259, 255, 254, 246, 299, 272, 241, 242, 256, 276, 262, 243, 247, 268, 255, 249, 267, 247, 236, 269, 256, 221, 235, 255, 249, 258, 259, 249, 224, 284, 256, 245, 251, 270, 234, 255, 279, 262, 234, 262, 264, 262, 284, 258, 269, 268, 247, 237, 253, 245, 255, 283, 251, 257, 278, 263, 265, 244, 270, 253, 263, 236, 262, 238, 256, 267, 268, 231, 270, 263, 263, 264] payload_rows_per_dst=[5472, 5458, 5514, 5586, 5404, 5480, 5528, 5548] dup_rows_per_dst=[2620, 2612, 2770, 2704, 2680, 2662, 2820, 2678] combine_rows_per_src=[5472, 5458, 5514, 5586, 5404, 5480, 5528, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155664384 dispatch_meta_bytes=229488 combine_payload_bytes=155664384 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=155893872 ingress_bytes=212287488 link_bytes=212287488 token_bytes=311558256 weight_movement_bytes=56623104 total_nvlink_bytes=368181360 weight_to_token_ratio=0.181742 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034034258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229024 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156794528 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360032 weight_movement_bytes=56623104 total_nvlink_bytes=369983136 weight_to_token_ratio=0.180697 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38242 dup_rows=21870 dup_rows_remote=19150 dedup_ratio=0.333710 router_tokens_per_expert=[273, 244, 263, 256, 260, 259, 238, 262, 275, 289, 266, 242, 251, 252, 268, 287, 252, 233, 267, 240, 280, 246, 245, 243, 235, 239, 239, 239, 250, 269, 271, 279, 260, 241, 259, 255, 239, 262, 248, 269, 270, 244, 254, 252, 232, 245, 240, 263, 249, 270, 249, 276, 265, 287, 252, 265, 265, 247, 248, 268, 240, 252, 292, 231, 236, 266, 263, 259, 268, 263, 238, 273, 267, 233, 237, 253, 251, 255, 245, 255, 268, 261, 246, 251, 246, 248, 278, 261, 246, 259, 285, 256, 249, 270, 270, 216, 243, 286, 241, 270, 248, 237, 254, 235, 239, 262, 263, 243, 285, 242, 275, 249, 254, 265, 235, 236, 258, 276, 259, 266, 262, 269, 247, 251, 250, 260, 262, 273] payload_rows_per_dst=[5530, 5410, 5308, 5576, 5440, 5484, 5424, 5494] dup_rows_per_dst=[2840, 2644, 2758, 2736, 2684, 2736, 2720, 2752] combine_rows_per_src=[5530, 5410, 5308, 5576, 5440, 5484, 5424, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155533312 dispatch_meta_bytes=228648 combine_payload_bytes=155533312 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=155761960 ingress_bytes=193282048 link_bytes=193282048 token_bytes=311295272 weight_movement_bytes=37748736 total_nvlink_bytes=349044008 weight_to_token_ratio=0.121263 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.851933483e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229648 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157671696 ingress_bytes=214065152 link_bytes=214065152 token_bytes=315113744 weight_movement_bytes=56623104 total_nvlink_bytes=371736848 weight_to_token_ratio=0.179691 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051066960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=228984 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157531768 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314834552 weight_movement_bytes=18874368 total_nvlink_bytes=333708920 weight_to_token_ratio=0.059950 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38266 dup_rows=21670 dup_rows_remote=18940 dedup_ratio=0.330658 router_tokens_per_expert=[241, 268, 256, 258, 248, 248, 250, 257, 270, 263, 254, 265, 274, 264, 264, 235, 269, 253, 244, 280, 267, 262, 277, 253, 274, 259, 261, 200, 249, 241, 248, 236, 238, 258, 240, 243, 232, 245, 267, 269, 300, 252, 233, 260, 242, 251, 237, 259, 265, 279, 256, 256, 235, 263, 260, 258, 240, 249, 258, 274, 247, 258, 285, 251, 265, 231, 257, 284, 236, 263, 260, 264, 227, 283, 249, 237, 261, 252, 249, 229, 231, 264, 266, 254, 255, 279, 268, 250, 260, 288, 277, 281, 238, 249, 256, 249, 261, 272, 276, 252, 270, 275, 248, 267, 247, 265, 251, 238, 250, 247, 238, 253, 245, 252, 253, 256, 249, 239, 252, 239, 267, 270, 291, 262, 251, 291, 245, 236] payload_rows_per_dst=[5438, 5454, 5434, 5544, 5396, 5600, 5492, 5508] dup_rows_per_dst=[2792, 2692, 2618, 2724, 2698, 2730, 2728, 2688] combine_rows_per_src=[5438, 5454, 5434, 5544, 5396, 5600, 5492, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229568 combine_payload_bytes=156639232 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156868800 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313508032 weight_movement_bytes=37748736 total_nvlink_bytes=351256768 weight_to_token_ratio=0.120408 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38288 dup_rows=21772 dup_rows_remote=19098 dedup_ratio=0.332214 router_tokens_per_expert=[278, 256, 239, 276, 269, 244, 244, 264, 245, 242, 251, 241, 219, 285, 265, 228, 239, 236, 266, 240, 247, 246, 261, 257, 263, 271, 276, 263, 230, 241, 248, 291, 259, 251, 258, 226, 276, 254, 263, 270, 279, 242, 238, 265, 255, 250, 261, 243, 224, 300, 254, 263, 255, 258, 258, 265, 257, 270, 251, 246, 251, 240, 271, 261, 272, 239, 252, 247, 278, 245, 292, 271, 265, 245, 238, 241, 280, 256, 264, 268, 248, 263, 294, 290, 247, 245, 251, 267, 267, 256, 251, 248, 242, 236, 271, 257, 259, 232, 267, 264, 266, 227, 237, 250, 280, 247, 240, 287, 253, 251, 253, 266, 279, 255, 246, 263, 261, 274, 252, 250, 257, 253, 228, 249, 267, 275, 241, 218] payload_rows_per_dst=[5428, 5476, 5482, 5430, 5526, 5520, 5416, 5486] dup_rows_per_dst=[2664, 2674, 2698, 2818, 2780, 2746, 2742, 2650] combine_rows_per_src=[5428, 5476, 5482, 5430, 5526, 5520, 5416, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38180 dup_rows=21734 dup_rows_remote=18868 dedup_ratio=0.331635 router_tokens_per_expert=[227, 246, 250, 251, 251, 282, 246, 280, 242, 259, 266, 247, 264, 268, 270, 235, 247, 236, 289, 267, 286, 241, 292, 245, 241, 232, 250, 290, 253, 264, 278, 280, 247, 252, 259, 273, 276, 254, 265, 230, 229, 256, 237, 254, 247, 254, 259, 259, 290, 239, 264, 283, 252, 247, 226, 264, 250, 270, 264, 269, 283, 256, 263, 240, 221, 272, 252, 256, 245, 262, 253, 254, 261, 236, 266, 253, 262, 237, 289, 244, 233, 264, 234, 234, 260, 248, 255, 274, 281, 257, 270, 257, 238, 265, 272, 275, 246, 262, 226, 252, 230, 232, 263, 258, 242, 249, 286, 259, 227, 269, 260, 277, 250, 251, 255, 234, 246, 255, 229, 246, 256, 269, 257, 243, 283, 262, 266, 262] payload_rows_per_dst=[5622, 5488, 5516, 5482, 5390, 5464, 5334, 5506] dup_rows_per_dst=[2866, 2894, 2948, 2584, 2584, 2662, 2574, 2622] combine_rows_per_src=[5622, 5488, 5516, 5482, 5390, 5464, 5334, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228824 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966360 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313703896 weight_movement_bytes=37748736 total_nvlink_bytes=351452632 weight_to_token_ratio=0.120332 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38196 dup_rows=21866 dup_rows_remote=19078 dedup_ratio=0.333649 router_tokens_per_expert=[224, 261, 277, 258, 252, 227, 245, 245, 294, 273, 228, 261, 253, 263, 298, 260, 241, 263, 256, 280, 242, 240, 247, 239, 271, 264, 249, 264, 242, 266, 242, 255, 245, 240, 274, 272, 256, 271, 248, 246, 245, 235, 245, 242, 243, 259, 241, 246, 245, 265, 259, 260, 229, 270, 260, 255, 246, 289, 248, 274, 270, 240, 246, 231, 225, 264, 234, 270, 244, 271, 253, 243, 272, 263, 268, 276, 256, 262, 271, 259, 292, 252, 279, 259, 257, 277, 241, 278, 268, 268, 242, 238, 273, 255, 252, 289, 230, 251, 277, 259, 228, 263, 251, 258, 243, 247, 279, 230, 282, 244, 278, 236, 264, 250, 247, 234, 261, 214, 262, 260, 241, 252, 254, 270, 283, 278, 241, 275] payload_rows_per_dst=[5454, 5424, 5400, 5488, 5474, 5558, 5430, 5442] dup_rows_per_dst=[2784, 2698, 2616, 2686, 2788, 2882, 2682, 2730] combine_rows_per_src=[5454, 5424, 5400, 5488, 5474, 5558, 5430, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229544 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057192 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884840 weight_movement_bytes=18874368 total_nvlink_bytes=332759208 weight_to_token_ratio=0.060132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43508 payload_rows_remote=38056 dup_rows=22028 dup_rows_remote=19230 dedup_ratio=0.336121 router_tokens_per_expert=[266, 252, 247, 237, 246, 259, 246, 236, 260, 261, 253, 267, 282, 232, 241, 267, 265, 234, 260, 280, 263, 247, 267, 282, 252, 272, 240, 267, 253, 238, 220, 266, 282, 246, 266, 258, 243, 273, 249, 246, 260, 252, 293, 247, 282, 249, 229, 250, 233, 243, 260, 258, 252, 237, 263, 259, 251, 257, 251, 260, 238, 251, 288, 245, 235, 267, 259, 223, 245, 243, 268, 267, 215, 271, 227, 299, 257, 246, 252, 257, 260, 282, 247, 252, 272, 259, 236, 247, 271, 267, 244, 289, 272, 236, 260, 253, 231, 256, 283, 230, 245, 243, 248, 259, 273, 216, 257, 273, 243, 241, 252, 264, 279, 252, 223, 276, 260, 266, 279, 267, 261, 283, 270, 244, 284, 285, 242, 276] payload_rows_per_dst=[5436, 5422, 5452, 5398, 5336, 5478, 5388, 5598] dup_rows_per_dst=[2668, 2790, 2798, 2694, 2726, 2816, 2640, 2896] combine_rows_per_src=[5436, 5422, 5452, 5398, 5336, 5478, 5388, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38286 dup_rows=21666 dup_rows_remote=19036 dedup_ratio=0.330597 router_tokens_per_expert=[230, 249, 263, 260, 274, 248, 259, 260, 251, 271, 262, 255, 265, 237, 260, 266, 268, 232, 256, 274, 235, 248, 236, 274, 248, 252, 252, 266, 253, 235, 250, 229, 248, 263, 258, 242, 255, 269, 276, 255, 257, 268, 271, 261, 258, 270, 247, 206, 253, 281, 257, 285, 255, 238, 271, 289, 215, 250, 275, 244, 236, 272, 284, 248, 249, 267, 275, 255, 285, 222, 247, 278, 272, 243, 259, 254, 262, 238, 241, 244, 241, 246, 279, 290, 265, 258, 265, 272, 257, 244, 261, 255, 231, 242, 257, 252, 260, 253, 255, 256, 286, 251, 289, 236, 271, 227, 235, 251, 273, 242, 252, 243, 272, 259, 245, 271, 248, 279, 258, 261, 257, 242, 258, 255, 241, 246, 268, 247] payload_rows_per_dst=[5452, 5428, 5434, 5504, 5534, 5370, 5564, 5584] dup_rows_per_dst=[2666, 2588, 2774, 2802, 2718, 2618, 2870, 2630] combine_rows_per_src=[5452, 5428, 5434, 5504, 5534, 5370, 5564, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38154 dup_rows=21858 dup_rows_remote=19164 dedup_ratio=0.333527 router_tokens_per_expert=[280, 232, 288, 245, 246, 242, 263, 246, 264, 235, 279, 227, 279, 267, 238, 264, 257, 272, 273, 259, 281, 231, 267, 278, 224, 247, 257, 232, 239, 264, 236, 270, 277, 255, 237, 278, 257, 251, 269, 268, 242, 230, 249, 259, 229, 278, 260, 254, 281, 266, 269, 265, 258, 243, 260, 290, 243, 250, 251, 258, 224, 229, 275, 247, 233, 267, 261, 261, 243, 264, 267, 250, 240, 258, 268, 256, 269, 244, 229, 263, 248, 226, 282, 244, 256, 275, 261, 252, 258, 248, 271, 251, 276, 285, 248, 258, 231, 240, 267, 260, 250, 277, 279, 248, 248, 257, 260, 247, 256, 249, 289, 246, 240, 263, 250, 267, 269, 238, 265, 254, 272, 233, 262, 243, 230, 240, 269, 273] payload_rows_per_dst=[5450, 5422, 5412, 5524, 5440, 5452, 5474, 5504] dup_rows_per_dst=[2740, 2752, 2774, 2694, 2706, 2826, 2734, 2632] combine_rows_per_src=[5450, 5422, 5412, 5524, 5440, 5452, 5474, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228192 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156613472 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312998752 weight_movement_bytes=18874368 total_nvlink_bytes=331873120 weight_to_token_ratio=0.060302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229096 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156679912 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313130728 weight_movement_bytes=18874368 total_nvlink_bytes=332005096 weight_to_token_ratio=0.060276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38322 dup_rows=21618 dup_rows_remote=19036 dedup_ratio=0.329865 router_tokens_per_expert=[237, 282, 247, 261, 232, 242, 250, 220, 285, 257, 255, 255, 260, 260, 236, 258, 245, 277, 242, 275, 236, 260, 235, 261, 272, 285, 283, 260, 259, 241, 272, 253, 279, 245, 278, 256, 277, 250, 264, 244, 298, 247, 255, 253, 241, 253, 249, 269, 256, 272, 240, 251, 235, 271, 250, 263, 243, 242, 242, 252, 273, 247, 266, 245, 255, 245, 248, 265, 275, 257, 244, 254, 262, 289, 253, 245, 279, 238, 253, 262, 256, 241, 255, 263, 242, 239, 257, 261, 252, 250, 262, 249, 259, 266, 255, 255, 254, 244, 240, 282, 244, 250, 271, 252, 251, 244, 264, 276, 242, 258, 256, 261, 243, 273, 249, 237, 234, 257, 287, 287, 273, 256, 239, 226, 275, 258, 251, 249] payload_rows_per_dst=[5432, 5504, 5552, 5444, 5434, 5436, 5596, 5520] dup_rows_per_dst=[2642, 2808, 2764, 2652, 2814, 2688, 2582, 2668] combine_rows_per_src=[5432, 5504, 5552, 5444, 5434, 5436, 5596, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=229144 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156106520 ingress_bytes=174751744 link_bytes=174751744 token_bytes=311983896 weight_movement_bytes=18874368 total_nvlink_bytes=330858264 weight_to_token_ratio=0.060498 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674385228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229288 combine_payload_bytes=156819456 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157048744 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868200 weight_movement_bytes=37748736 total_nvlink_bytes=351616936 weight_to_token_ratio=0.120269 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43956 payload_rows_remote=38412 dup_rows=21580 dup_rows_remote=18780 dedup_ratio=0.329285 router_tokens_per_expert=[217, 273, 260, 248, 295, 232, 232, 267, 265, 243, 247, 267, 256, 241, 282, 223, 266, 249, 244, 241, 269, 276, 279, 257, 251, 248, 255, 267, 247, 251, 279, 237, 258, 288, 245, 241, 251, 248, 228, 277, 243, 251, 237, 245, 244, 278, 235, 278, 272, 256, 233, 289, 259, 252, 274, 267, 268, 240, 245, 244, 268, 246, 233, 264, 253, 257, 223, 259, 292, 257, 248, 269, 238, 240, 284, 275, 258, 249, 251, 272, 279, 258, 270, 266, 262, 265, 269, 242, 247, 258, 245, 275, 263, 229, 277, 267, 277, 229, 260, 250, 256, 261, 245, 249, 241, 264, 244, 264, 247, 266, 241, 257, 241, 250, 260, 257, 265, 242, 262, 259, 262, 260, 251, 256, 260, 260, 256, 258] payload_rows_per_dst=[5472, 5480, 5446, 5542, 5534, 5544, 5452, 5486] dup_rows_per_dst=[2624, 2752, 2648, 2678, 2716, 2800, 2650, 2712] combine_rows_per_src=[5472, 5480, 5446, 5542, 5534, 5544, 5452, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229272 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156508056 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786840 weight_movement_bytes=18874368 total_nvlink_bytes=331661208 weight_to_token_ratio=0.060343 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38516 dup_rows=21644 dup_rows_remote=19020 dedup_ratio=0.330261 router_tokens_per_expert=[241, 271, 291, 264, 264, 225, 233, 258, 262, 266, 227, 257, 234, 256, 259, 269, 241, 240, 273, 256, 256, 277, 232, 225, 235, 242, 215, 259, 268, 259, 268, 254, 228, 240, 262, 256, 261, 245, 229, 241, 245, 280, 246, 273, 254, 280, 250, 239, 238, 276, 248, 242, 240, 257, 249, 268, 262, 258, 249, 258, 286, 252, 266, 245, 256, 242, 259, 253, 265, 277, 273, 269, 274, 269, 269, 252, 261, 270, 267, 234, 253, 262, 238, 270, 246, 250, 262, 243, 258, 277, 271, 262, 262, 231, 236, 242, 254, 239, 269, 270, 239, 253, 259, 254, 280, 262, 292, 281, 243, 252, 273, 277, 229, 242, 245, 250, 274, 264, 251, 245, 280, 258, 257, 261, 260, 262, 261, 279] payload_rows_per_dst=[5478, 5376, 5406, 5500, 5536, 5500, 5596, 5500] dup_rows_per_dst=[2676, 2624, 2652, 2688, 2844, 2626, 2798, 2736] combine_rows_per_src=[5478, 5376, 5406, 5500, 5536, 5500, 5596, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229432 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196344 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163256 weight_movement_bytes=18874368 total_nvlink_bytes=333037624 weight_to_token_ratio=0.060078 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=228768 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157564320 ingress_bytes=232833024 link_bytes=232833024 token_bytes=314899872 weight_movement_bytes=75497472 total_nvlink_bytes=390397344 weight_to_token_ratio=0.239751 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230891475e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38354 dup_rows=21724 dup_rows_remote=19020 dedup_ratio=0.331482 router_tokens_per_expert=[244, 243, 280, 258, 275, 260, 260, 272, 229, 252, 260, 247, 253, 242, 245, 252, 284, 254, 247, 254, 266, 256, 222, 267, 220, 227, 246, 264, 252, 259, 246, 256, 277, 268, 239, 270, 246, 268, 254, 234, 232, 265, 238, 268, 261, 234, 249, 310, 222, 248, 253, 232, 276, 238, 276, 261, 256, 252, 273, 253, 263, 266, 258, 235, 241, 243, 276, 265, 260, 254, 252, 275, 254, 279, 245, 278, 261, 252, 270, 263, 234, 255, 252, 280, 265, 258, 273, 264, 276, 260, 285, 249, 271, 216, 236, 254, 263, 234, 261, 276, 247, 249, 250, 268, 231, 245, 262, 278, 255, 272, 269, 251, 241, 260, 272, 237, 268, 239, 265, 272, 246, 279, 252, 261, 249, 251, 256, 246] payload_rows_per_dst=[5458, 5584, 5552, 5252, 5696, 5396, 5372, 5502] dup_rows_per_dst=[2704, 2972, 2960, 2356, 2934, 2566, 2546, 2686] combine_rows_per_src=[5458, 5584, 5552, 5252, 5696, 5396, 5372, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157761536 dispatch_meta_bytes=230144 combine_payload_bytes=157761536 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157991680 ingress_bytes=195510272 link_bytes=195510272 token_bytes=315753216 weight_movement_bytes=37748736 total_nvlink_bytes=353501952 weight_to_token_ratio=0.119551 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873283229e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38450 dup_rows=21662 dup_rows_remote=19070 dedup_ratio=0.330536 router_tokens_per_expert=[236, 269, 269, 256, 259, 277, 265, 247, 261, 290, 226, 246, 241, 267, 222, 250, 251, 255, 251, 255, 261, 248, 236, 291, 254, 267, 219, 257, 263, 268, 269, 278, 280, 257, 233, 255, 268, 255, 270, 279, 237, 243, 286, 238, 261, 252, 264, 234, 255, 235, 247, 251, 252, 259, 277, 287, 252, 236, 241, 257, 255, 235, 251, 284, 250, 264, 250, 257, 258, 247, 271, 224, 232, 232, 236, 237, 256, 269, 247, 278, 277, 230, 231, 244, 246, 232, 259, 273, 271, 252, 263, 274, 258, 267, 235, 269, 237, 251, 255, 246, 283, 269, 258, 245, 276, 249, 244, 229, 255, 262, 265, 267, 270, 261, 268, 282, 253, 264, 242, 247, 270, 240, 265, 267, 252, 283, 257, 277] payload_rows_per_dst=[5476, 5502, 5380, 5462, 5424, 5522, 5454, 5654] dup_rows_per_dst=[2686, 2744, 2844, 2686, 2592, 2640, 2728, 2742] combine_rows_per_src=[5476, 5502, 5380, 5462, 5424, 5522, 5454, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38198 dup_rows=21924 dup_rows_remote=19224 dedup_ratio=0.334534 router_tokens_per_expert=[268, 246, 254, 255, 264, 277, 259, 242, 241, 280, 262, 285, 254, 249, 275, 264, 257, 278, 276, 277, 241, 247, 250, 251, 258, 250, 262, 245, 263, 292, 241, 246, 253, 246, 252, 249, 262, 259, 270, 237, 235, 240, 261, 252, 250, 261, 277, 253, 279, 266, 259, 259, 237, 260, 254, 251, 239, 255, 239, 250, 271, 258, 269, 265, 239, 217, 245, 239, 253, 242, 265, 258, 267, 252, 243, 272, 266, 244, 263, 240, 246, 229, 264, 276, 299, 240, 266, 257, 287, 239, 258, 271, 242, 224, 263, 240, 272, 233, 270, 282, 270, 244, 242, 255, 217, 262, 239, 254, 259, 273, 272, 256, 285, 250, 222, 258, 262, 235, 279, 242, 258, 254, 263, 248, 250, 284, 248, 247] payload_rows_per_dst=[5524, 5482, 5414, 5458, 5320, 5488, 5464, 5462] dup_rows_per_dst=[2826, 2786, 2700, 2764, 2690, 2714, 2736, 2708] combine_rows_per_src=[5524, 5482, 5414, 5458, 5320, 5488, 5464, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38182 dup_rows=21812 dup_rows_remote=18958 dedup_ratio=0.332825 router_tokens_per_expert=[271, 256, 262, 242, 259, 223, 245, 210, 246, 251, 285, 217, 245, 276, 215, 281, 249, 246, 275, 267, 230, 236, 260, 246, 265, 255, 268, 260, 266, 267, 259, 248, 223, 250, 255, 267, 224, 249, 276, 244, 227, 263, 256, 253, 275, 281, 249, 311, 281, 264, 253, 265, 291, 215, 278, 260, 254, 252, 275, 265, 263, 263, 255, 255, 279, 265, 244, 248, 269, 250, 244, 240, 253, 245, 234, 247, 261, 255, 255, 268, 252, 245, 253, 268, 279, 285, 268, 236, 246, 225, 251, 248, 277, 232, 258, 257, 265, 260, 251, 265, 261, 252, 254, 296, 244, 250, 286, 250, 260, 221, 297, 237, 268, 258, 280, 265, 271, 259, 242, 264, 242, 213, 259, 246, 265, 248, 294, 235] payload_rows_per_dst=[5294, 5452, 5464, 5556, 5440, 5458, 5518, 5542] dup_rows_per_dst=[2496, 2656, 2742, 2894, 2674, 2680, 2816, 2854] combine_rows_per_src=[5294, 5452, 5464, 5556, 5440, 5458, 5518, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38314 dup_rows=21860 dup_rows_remote=19174 dedup_ratio=0.333557 router_tokens_per_expert=[243, 255, 262, 261, 223, 265, 223, 250, 289, 250, 247, 258, 236, 279, 279, 254, 279, 246, 281, 237, 249, 256, 263, 265, 286, 268, 216, 265, 267, 260, 242, 268, 238, 262, 273, 288, 254, 270, 259, 254, 277, 249, 260, 259, 237, 229, 263, 234, 248, 256, 247, 240, 251, 241, 244, 281, 280, 243, 245, 210, 275, 236, 259, 268, 277, 246, 249, 267, 249, 254, 250, 267, 256, 267, 274, 243, 263, 244, 266, 233, 246, 263, 244, 253, 254, 254, 243, 228, 240, 280, 248, 270, 267, 263, 269, 233, 253, 264, 277, 269, 260, 276, 257, 239, 228, 260, 247, 275, 265, 239, 263, 272, 234, 242, 269, 246, 243, 281, 236, 263, 250, 273, 229, 270, 304, 255, 250, 267] payload_rows_per_dst=[5420, 5512, 5422, 5362, 5424, 5480, 5566, 5490] dup_rows_per_dst=[2728, 2784, 2790, 2686, 2786, 2630, 2722, 2734] combine_rows_per_src=[5420, 5512, 5422, 5362, 5424, 5480, 5566, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229496 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157327480 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314425464 weight_movement_bytes=37748736 total_nvlink_bytes=352174200 weight_to_token_ratio=0.120056 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=230080 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157721280 ingress_bytes=176365568 link_bytes=176365568 token_bytes=315212480 weight_movement_bytes=18874368 total_nvlink_bytes=334086848 weight_to_token_ratio=0.059878 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689848095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229688 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688696 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147704 weight_movement_bytes=18874368 total_nvlink_bytes=332022072 weight_to_token_ratio=0.060273 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38172 dup_rows=21896 dup_rows_remote=19170 dedup_ratio=0.334106 router_tokens_per_expert=[264, 263, 258, 245, 280, 262, 297, 241, 262, 249, 238, 264, 237, 255, 250, 253, 244, 259, 245, 266, 254, 262, 264, 240, 265, 234, 261, 252, 259, 242, 281, 262, 262, 262, 243, 255, 262, 268, 262, 251, 282, 263, 258, 249, 225, 243, 250, 268, 256, 279, 263, 276, 254, 256, 244, 248, 262, 247, 253, 246, 234, 240, 240, 252, 244, 250, 229, 280, 227, 265, 245, 264, 264, 246, 244, 274, 263, 271, 268, 256, 241, 275, 261, 260, 271, 287, 257, 243, 257, 262, 267, 276, 277, 241, 255, 265, 269, 267, 263, 254, 254, 249, 237, 279, 267, 261, 209, 262, 261, 240, 242, 283, 258, 238, 253, 253, 256, 242, 257, 258, 252, 251, 239, 270, 245, 244, 246, 263] payload_rows_per_dst=[5432, 5446, 5464, 5362, 5412, 5590, 5468, 5466] dup_rows_per_dst=[2804, 2734, 2742, 2738, 2768, 2800, 2726, 2584] combine_rows_per_src=[5432, 5446, 5464, 5362, 5412, 5590, 5468, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228560 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156622032 ingress_bytes=213016576 link_bytes=213016576 token_bytes=313015504 weight_movement_bytes=56623104 total_nvlink_bytes=369638608 weight_to_token_ratio=0.180896 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041020021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38422 dup_rows=21620 dup_rows_remote=18798 dedup_ratio=0.329895 router_tokens_per_expert=[243, 245, 241, 262, 259, 237, 247, 233, 264, 251, 240, 231, 247, 299, 286, 285, 248, 258, 259, 276, 266, 255, 263, 240, 279, 266, 253, 235, 250, 249, 240, 252, 271, 242, 253, 285, 258, 234, 268, 304, 279, 249, 264, 252, 254, 251, 262, 258, 265, 253, 256, 241, 215, 253, 266, 235, 263, 254, 230, 242, 262, 254, 253, 236, 245, 244, 250, 219, 231, 264, 246, 245, 237, 265, 239, 268, 261, 288, 257, 255, 269, 268, 270, 243, 250, 273, 253, 267, 260, 254, 263, 265, 255, 253, 258, 257, 253, 272, 259, 269, 245, 273, 260, 261, 252, 230, 242, 260, 250, 274, 225, 262, 255, 256, 281, 299, 262, 265, 237, 262, 277, 267, 254, 269, 249, 260, 253, 242] payload_rows_per_dst=[5382, 5478, 5660, 5422, 5466, 5494, 5470, 5544] dup_rows_per_dst=[2758, 2700, 2708, 2534, 2562, 2822, 2704, 2832] combine_rows_per_src=[5382, 5478, 5660, 5422, 5466, 5494, 5470, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229952 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157164096 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098240 weight_movement_bytes=18874368 total_nvlink_bytes=332972608 weight_to_token_ratio=0.060091 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38210 dup_rows=21828 dup_rows_remote=19068 dedup_ratio=0.333069 router_tokens_per_expert=[228, 227, 259, 242, 263, 245, 258, 266, 239, 262, 258, 248, 247, 278, 271, 258, 241, 271, 260, 288, 268, 260, 267, 264, 241, 244, 253, 236, 254, 266, 251, 265, 243, 280, 271, 268, 276, 275, 232, 270, 233, 246, 275, 257, 270, 234, 265, 254, 271, 244, 272, 234, 256, 251, 245, 248, 240, 245, 264, 253, 238, 255, 224, 288, 265, 283, 235, 257, 239, 265, 257, 264, 248, 262, 237, 252, 251, 258, 250, 258, 256, 259, 274, 237, 253, 253, 277, 251, 248, 252, 243, 236, 266, 237, 260, 264, 223, 253, 256, 271, 254, 246, 249, 229, 267, 272, 227, 254, 286, 247, 246, 242, 280, 271, 286, 245, 268, 287, 248, 264, 257, 245, 279, 245, 284, 270, 265, 250] payload_rows_per_dst=[5438, 5498, 5486, 5382, 5456, 5506, 5366, 5576] dup_rows_per_dst=[2660, 2760, 2812, 2674, 2706, 2626, 2678, 2912] combine_rows_per_src=[5438, 5498, 5486, 5382, 5456, 5506, 5366, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229368 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156581880 ingress_bytes=175226880 link_bytes=175226880 token_bytes=312934392 weight_movement_bytes=18874368 total_nvlink_bytes=331808760 weight_to_token_ratio=0.060314 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678937747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=228880 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157605392 ingress_bytes=195125248 link_bytes=195125248 token_bytes=314981904 weight_movement_bytes=37748736 total_nvlink_bytes=352730640 weight_to_token_ratio=0.119844 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869594119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38242 dup_rows=21674 dup_rows_remote=18816 dedup_ratio=0.330719 router_tokens_per_expert=[262, 266, 283, 284, 244, 264, 233, 247, 270, 236, 284, 253, 224, 274, 254, 260, 269, 252, 279, 227, 266, 250, 278, 251, 254, 230, 208, 275, 256, 264, 243, 260, 253, 257, 262, 253, 253, 255, 239, 260, 252, 254, 268, 264, 263, 263, 248, 258, 261, 226, 261, 272, 248, 253, 265, 267, 242, 251, 251, 249, 240, 236, 221, 271, 272, 248, 264, 263, 240, 256, 258, 262, 262, 271, 275, 243, 265, 254, 270, 271, 272, 242, 281, 272, 274, 232, 247, 250, 251, 240, 272, 208, 290, 234, 251, 259, 272, 253, 241, 253, 260, 270, 282, 247, 281, 271, 252, 246, 241, 249, 235, 264, 247, 250, 248, 280, 263, 248, 274, 272, 255, 268, 251, 230, 239, 251, 255, 255] payload_rows_per_dst=[5620, 5604, 5294, 5540, 5494, 5338, 5542, 5430] dup_rows_per_dst=[2858, 3024, 2406, 2868, 2668, 2432, 2790, 2628] combine_rows_per_src=[5620, 5604, 5294, 5540, 5494, 5338, 5542, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38078 dup_rows=21948 dup_rows_remote=19132 dedup_ratio=0.334900 router_tokens_per_expert=[261, 254, 258, 266, 294, 238, 273, 225, 243, 264, 248, 257, 264, 267, 271, 263, 260, 252, 254, 243, 280, 229, 268, 251, 253, 268, 252, 231, 259, 261, 247, 243, 258, 223, 251, 252, 271, 276, 280, 265, 251, 256, 262, 266, 230, 261, 242, 257, 246, 257, 249, 288, 237, 242, 263, 248, 230, 232, 283, 274, 264, 265, 239, 260, 250, 258, 284, 258, 266, 270, 268, 231, 243, 272, 287, 264, 254, 282, 234, 242, 236, 270, 238, 262, 259, 238, 243, 257, 252, 224, 251, 271, 250, 246, 279, 273, 245, 231, 286, 251, 263, 261, 287, 261, 278, 246, 243, 230, 273, 258, 268, 242, 271, 248, 257, 265, 275, 225, 258, 248, 261, 244, 260, 242, 270, 247, 236, 251] payload_rows_per_dst=[5476, 5430, 5402, 5392, 5510, 5458, 5418, 5502] dup_rows_per_dst=[2816, 2672, 2800, 2762, 2816, 2640, 2828, 2614] combine_rows_per_src=[5476, 5430, 5402, 5392, 5510, 5458, 5418, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229112 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156737272 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245432 weight_movement_bytes=37748736 total_nvlink_bytes=350994168 weight_to_token_ratio=0.120508 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38326 dup_rows=21744 dup_rows_remote=19058 dedup_ratio=0.331787 router_tokens_per_expert=[275, 272, 269, 268, 235, 258, 235, 243, 258, 250, 238, 248, 259, 236, 261, 262, 230, 278, 277, 262, 256, 257, 238, 258, 253, 269, 253, 247, 255, 244, 266, 246, 256, 253, 278, 262, 241, 254, 267, 236, 248, 260, 246, 243, 262, 260, 246, 243, 261, 236, 221, 265, 273, 250, 213, 233, 283, 272, 271, 254, 248, 293, 268, 254, 246, 282, 261, 239, 277, 296, 254, 274, 275, 249, 255, 273, 271, 242, 236, 234, 243, 253, 250, 262, 275, 265, 249, 249, 276, 251, 261, 271, 228, 255, 267, 252, 252, 264, 248, 268, 273, 252, 260, 253, 244, 253, 274, 233, 268, 270, 260, 243, 250, 224, 240, 260, 260, 231, 271, 264, 268, 270, 257, 252, 252, 233, 260, 284] payload_rows_per_dst=[5448, 5444, 5260, 5716, 5436, 5482, 5540, 5466] dup_rows_per_dst=[2686, 2734, 2342, 2982, 2752, 2732, 2830, 2686] combine_rows_per_src=[5448, 5444, 5260, 5716, 5436, 5482, 5540, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38304 dup_rows=21806 dup_rows_remote=19060 dedup_ratio=0.332733 router_tokens_per_expert=[273, 249, 242, 267, 262, 270, 256, 255, 270, 242, 285, 238, 255, 232, 244, 260, 256, 279, 254, 278, 262, 243, 257, 254, 244, 268, 250, 244, 259, 285, 246, 255, 277, 247, 258, 271, 227, 247, 268, 239, 261, 262, 260, 264, 244, 265, 246, 250, 281, 224, 264, 266, 212, 250, 292, 233, 257, 245, 265, 252, 262, 264, 283, 246, 265, 264, 263, 254, 228, 253, 260, 243, 260, 236, 252, 269, 249, 259, 251, 258, 271, 261, 266, 273, 273, 249, 244, 246, 248, 263, 293, 268, 269, 242, 230, 235, 240, 241, 231, 242, 258, 265, 283, 249, 231, 261, 254, 267, 250, 269, 250, 285, 254, 251, 263, 256, 234, 252, 245, 263, 256, 275, 269, 258, 244, 253, 249, 259] payload_rows_per_dst=[5448, 5448, 5426, 5518, 5478, 5506, 5402, 5504] dup_rows_per_dst=[2752, 2820, 2746, 2674, 2650, 2756, 2750, 2658] combine_rows_per_src=[5448, 5448, 5426, 5518, 5478, 5506, 5402, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228232 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156867464 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313506696 weight_movement_bytes=18874368 total_nvlink_bytes=332381064 weight_to_token_ratio=0.060204 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=228840 combine_payload_bytes=155967488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156196328 ingress_bytes=193716224 link_bytes=193716224 token_bytes=312163816 weight_movement_bytes=37748736 total_nvlink_bytes=349912552 weight_to_token_ratio=0.120926 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856093544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38326 dup_rows=21808 dup_rows_remote=19032 dedup_ratio=0.332764 router_tokens_per_expert=[255, 266, 247, 250, 252, 267, 247, 255, 224, 273, 272, 285, 241, 253, 256, 249, 255, 268, 260, 278, 237, 254, 251, 254, 264, 272, 264, 221, 266, 284, 261, 288, 264, 286, 247, 215, 298, 273, 249, 274, 243, 260, 248, 276, 236, 261, 261, 274, 237, 240, 265, 270, 274, 267, 266, 254, 258, 279, 226, 242, 248, 265, 232, 266, 246, 280, 252, 271, 233, 264, 266, 234, 237, 253, 255, 251, 288, 273, 270, 278, 230, 254, 291, 244, 243, 267, 259, 262, 243, 255, 247, 278, 227, 248, 276, 226, 244, 260, 258, 241, 251, 227, 272, 247, 282, 248, 277, 249, 225, 243, 267, 244, 255, 246, 261, 202, 257, 240, 246, 243, 244, 241, 252, 276, 247, 259, 280, 260] payload_rows_per_dst=[5468, 5530, 5522, 5402, 5566, 5392, 5394, 5454] dup_rows_per_dst=[2716, 2824, 2808, 2776, 2736, 2708, 2676, 2564] combine_rows_per_src=[5468, 5530, 5522, 5402, 5566, 5392, 5394, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229536 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157212832 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314196128 weight_movement_bytes=56623104 total_nvlink_bytes=370819232 weight_to_token_ratio=0.180216 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229456 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122640 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015824 weight_movement_bytes=37748736 total_nvlink_bytes=351764560 weight_to_token_ratio=0.120213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38256 dup_rows=21780 dup_rows_remote=19084 dedup_ratio=0.332336 router_tokens_per_expert=[260, 266, 270, 255, 259, 271, 273, 230, 258, 239, 276, 270, 249, 235, 263, 274, 263, 263, 267, 238, 275, 242, 225, 247, 275, 287, 254, 269, 260, 269, 236, 250, 245, 249, 259, 237, 239, 241, 237, 257, 264, 239, 244, 264, 233, 236, 238, 263, 243, 256, 258, 299, 234, 262, 281, 254, 266, 284, 245, 243, 251, 267, 241, 277, 241, 262, 239, 275, 277, 269, 241, 282, 259, 246, 263, 278, 255, 236, 264, 280, 249, 251, 284, 246, 243, 276, 237, 268, 257, 245, 253, 223, 269, 271, 245, 268, 285, 265, 234, 248, 249, 277, 250, 262, 259, 252, 241, 215, 286, 245, 256, 274, 263, 249, 248, 263, 261, 252, 244, 252, 224, 256, 254, 254, 249, 284, 246, 245] payload_rows_per_dst=[5502, 5482, 5332, 5480, 5556, 5506, 5500, 5398] dup_rows_per_dst=[2794, 2758, 2558, 2842, 2778, 2664, 2696, 2690] combine_rows_per_src=[5502, 5482, 5332, 5480, 5556, 5506, 5500, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38178 dup_rows=21818 dup_rows_remote=18968 dedup_ratio=0.332916 router_tokens_per_expert=[270, 260, 253, 259, 275, 246, 250, 257, 266, 251, 252, 255, 243, 239, 247, 269, 269, 274, 270, 228, 263, 245, 263, 253, 251, 265, 257, 266, 245, 270, 264, 242, 243, 268, 284, 246, 242, 300, 275, 269, 236, 266, 258, 233, 261, 248, 252, 274, 254, 274, 248, 251, 216, 244, 244, 256, 247, 248, 244, 251, 243, 254, 235, 268, 248, 268, 257, 279, 234, 257, 262, 236, 263, 251, 244, 255, 275, 248, 253, 253, 257, 261, 260, 278, 259, 271, 252, 285, 256, 241, 256, 257, 246, 268, 277, 271, 269, 228, 247, 259, 237, 256, 275, 256, 226, 269, 255, 248, 273, 285, 220, 251, 252, 243, 269, 241, 268, 244, 266, 269, 245, 281, 285, 251, 228, 238, 262, 245] payload_rows_per_dst=[5452, 5514, 5510, 5386, 5430, 5540, 5448, 5438] dup_rows_per_dst=[2732, 2736, 2800, 2568, 2736, 2850, 2660, 2736] combine_rows_per_src=[5452, 5514, 5510, 5386, 5430, 5540, 5448, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229432 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157212728 ingress_bytes=232480768 link_bytes=232480768 token_bytes=314196024 weight_movement_bytes=75497472 total_nvlink_bytes=389693496 weight_to_token_ratio=0.240288 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227516331e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38362 dup_rows=21716 dup_rows_remote=19034 dedup_ratio=0.331360 router_tokens_per_expert=[271, 263, 267, 277, 252, 246, 281, 237, 263, 283, 254, 280, 255, 270, 241, 233, 252, 243, 268, 256, 259, 231, 252, 250, 264, 274, 272, 257, 238, 258, 249, 247, 272, 261, 235, 272, 264, 245, 274, 272, 260, 266, 245, 215, 268, 249, 264, 249, 271, 256, 244, 258, 269, 271, 254, 286, 268, 243, 258, 258, 276, 249, 270, 239, 245, 250, 232, 260, 244, 267, 280, 237, 262, 270, 280, 249, 255, 257, 270, 254, 264, 245, 290, 244, 238, 222, 272, 242, 241, 284, 241, 240, 275, 262, 259, 253, 262, 257, 262, 208, 276, 230, 275, 231, 248, 258, 250, 238, 262, 232, 244, 252, 277, 252, 213, 250, 250, 261, 241, 237, 254, 249, 282, 255, 237, 256, 273, 288] payload_rows_per_dst=[5506, 5458, 5496, 5514, 5470, 5502, 5386, 5488] dup_rows_per_dst=[2840, 2682, 2726, 2826, 2754, 2642, 2584, 2662] combine_rows_per_src=[5506, 5458, 5496, 5514, 5470, 5502, 5386, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229360 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156925936 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622512 weight_movement_bytes=37748736 total_nvlink_bytes=351371248 weight_to_token_ratio=0.120364 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228584 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156605672 ingress_bytes=194125824 link_bytes=194125824 token_bytes=312982760 weight_movement_bytes=37748736 total_nvlink_bytes=350731496 weight_to_token_ratio=0.120610 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860018130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38224 dup_rows=21762 dup_rows_remote=18752 dedup_ratio=0.332062 router_tokens_per_expert=[261, 260, 262, 275, 271, 249, 274, 270, 260, 271, 293, 258, 253, 252, 288, 275, 241, 246, 238, 249, 267, 264, 274, 278, 231, 246, 254, 256, 258, 239, 254, 234, 286, 239, 244, 253, 255, 251, 252, 253, 235, 226, 217, 262, 248, 245, 249, 264, 244, 259, 263, 252, 251, 253, 283, 272, 243, 244, 258, 238, 228, 226, 246, 242, 279, 214, 262, 246, 232, 241, 258, 246, 278, 282, 268, 260, 267, 216, 268, 253, 251, 228, 282, 274, 273, 276, 275, 253, 248, 280, 273, 255, 276, 262, 283, 265, 265, 243, 267, 259, 236, 259, 274, 240, 263, 257, 267, 267, 240, 248, 257, 238, 256, 251, 249, 255, 254, 253, 259, 239, 251, 264, 268, 275, 256, 263, 221, 268] payload_rows_per_dst=[5550, 5630, 5540, 5426, 5460, 5270, 5442, 5456] dup_rows_per_dst=[3010, 2924, 2758, 2680, 2680, 2402, 2718, 2590] combine_rows_per_src=[5550, 5630, 5540, 5426, 5460, 5270, 5442, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229584 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157360336 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314491088 weight_movement_bytes=37748736 total_nvlink_bytes=352239824 weight_to_token_ratio=0.120031 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38528 dup_rows=21612 dup_rows_remote=19056 dedup_ratio=0.329773 router_tokens_per_expert=[247, 246, 266, 267, 265, 277, 254, 255, 243, 244, 250, 246, 277, 256, 270, 258, 269, 253, 251, 258, 251, 263, 281, 208, 257, 265, 243, 228, 264, 278, 236, 277, 244, 259, 269, 259, 277, 251, 266, 247, 249, 256, 255, 241, 262, 273, 259, 254, 256, 263, 259, 271, 257, 274, 249, 245, 268, 278, 252, 258, 285, 261, 269, 251, 248, 241, 258, 236, 235, 226, 253, 251, 243, 268, 267, 233, 248, 272, 233, 264, 268, 270, 270, 242, 240, 240, 257, 239, 282, 257, 242, 265, 242, 259, 266, 244, 249, 249, 273, 248, 276, 234, 227, 257, 233, 254, 272, 259, 249, 265, 278, 244, 245, 252, 251, 240, 255, 252, 268, 270, 261, 239, 259, 280, 255, 279, 273, 243] payload_rows_per_dst=[5512, 5496, 5516, 5574, 5396, 5482, 5464, 5484] dup_rows_per_dst=[2730, 2668, 2726, 2818, 2556, 2684, 2670, 2760] combine_rows_per_src=[5512, 5496, 5516, 5574, 5396, 5482, 5464, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38360 dup_rows=21660 dup_rows_remote=18934 dedup_ratio=0.330505 router_tokens_per_expert=[264, 251, 251, 257, 207, 257, 249, 238, 265, 245, 258, 237, 278, 273, 259, 251, 273, 254, 234, 240, 245, 257, 275, 268, 248, 268, 270, 252, 265, 267, 291, 251, 253, 262, 268, 200, 272, 262, 245, 281, 262, 264, 255, 239, 256, 272, 242, 247, 268, 263, 254, 257, 255, 259, 246, 250, 240, 264, 249, 269, 270, 242, 275, 253, 264, 259, 235, 257, 240, 233, 252, 262, 276, 262, 261, 255, 245, 235, 287, 234, 290, 254, 245, 286, 254, 268, 257, 240, 255, 278, 228, 253, 255, 258, 253, 275, 260, 253, 246, 268, 245, 261, 263, 230, 268, 239, 263, 247, 249, 264, 254, 253, 254, 276, 269, 261, 242, 268, 230, 249, 238, 253, 227, 257, 251, 282, 272, 278] payload_rows_per_dst=[5430, 5350, 5230, 5452, 5612, 5660, 5626, 5516] dup_rows_per_dst=[2650, 2430, 2416, 2748, 3038, 2880, 2772, 2726] combine_rows_per_src=[5430, 5350, 5230, 5452, 5612, 5660, 5626, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38402 dup_rows=21638 dup_rows_remote=18922 dedup_ratio=0.330170 router_tokens_per_expert=[274, 279, 248, 248, 256, 254, 242, 242, 275, 265, 243, 244, 240, 266, 225, 278, 246, 262, 258, 254, 269, 263, 256, 235, 251, 276, 270, 236, 251, 235, 264, 263, 295, 258, 254, 234, 267, 255, 255, 245, 247, 248, 246, 249, 243, 258, 270, 282, 254, 257, 250, 262, 251, 226, 232, 269, 273, 271, 280, 255, 274, 273, 264, 271, 241, 259, 247, 268, 254, 242, 267, 239, 257, 240, 267, 272, 241, 257, 250, 258, 266, 273, 246, 245, 247, 246, 250, 256, 260, 249, 257, 242, 270, 244, 260, 267, 233, 257, 259, 242, 265, 269, 274, 240, 251, 262, 270, 237, 229, 286, 270, 255, 281, 255, 269, 254, 256, 238, 252, 261, 253, 251, 255, 262, 260, 242, 271, 236] payload_rows_per_dst=[5504, 5518, 5496, 5594, 5488, 5428, 5400, 5470] dup_rows_per_dst=[2654, 2660, 2716, 2730, 2630, 2728, 2798, 2722] combine_rows_per_src=[5504, 5518, 5496, 5594, 5488, 5428, 5400, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=227904 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156793408 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313358912 weight_movement_bytes=56623104 total_nvlink_bytes=369982016 weight_to_token_ratio=0.180697 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38288 dup_rows=21886 dup_rows_remote=19170 dedup_ratio=0.333954 router_tokens_per_expert=[258, 250, 262, 274, 264, 284, 265, 257, 247, 248, 258, 230, 238, 282, 270, 272, 255, 278, 267, 254, 246, 259, 248, 269, 249, 262, 223, 240, 256, 258, 257, 257, 247, 258, 279, 234, 262, 261, 275, 261, 270, 281, 271, 256, 250, 225, 256, 271, 248, 262, 232, 265, 238, 267, 250, 266, 263, 255, 260, 251, 221, 232, 246, 283, 256, 269, 235, 274, 246, 265, 231, 267, 278, 241, 224, 266, 259, 233, 254, 243, 242, 270, 274, 240, 266, 230, 262, 244, 273, 253, 273, 245, 220, 274, 267, 260, 269, 266, 223, 257, 240, 252, 253, 262, 248, 263, 242, 255, 269, 241, 237, 294, 248, 248, 274, 272, 256, 277, 250, 241, 260, 264, 263, 277, 238, 272, 253, 237] payload_rows_per_dst=[5492, 5510, 5416, 5362, 5352, 5558, 5474, 5486] dup_rows_per_dst=[2826, 2646, 2898, 2716, 2730, 2628, 2668, 2774] combine_rows_per_src=[5492, 5510, 5416, 5362, 5352, 5558, 5474, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157810688 dispatch_meta_bytes=230336 combine_payload_bytes=157810688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158041024 ingress_bytes=176685056 link_bytes=176685056 token_bytes=315851712 weight_movement_bytes=18874368 total_nvlink_bytes=334726080 weight_to_token_ratio=0.059757 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692909272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38476 dup_rows=21638 dup_rows_remote=18996 dedup_ratio=0.330170 router_tokens_per_expert=[237, 248, 226, 262, 244, 251, 270, 250, 257, 271, 256, 271, 253, 246, 245, 264, 263, 251, 265, 246, 248, 254, 281, 259, 276, 280, 235, 248, 260, 247, 265, 251, 262, 270, 263, 247, 300, 282, 282, 245, 245, 280, 238, 235, 279, 264, 247, 250, 243, 251, 262, 243, 274, 258, 267, 249, 282, 237, 227, 268, 257, 259, 250, 246, 241, 259, 254, 271, 283, 243, 265, 243, 267, 251, 241, 269, 269, 239, 233, 233, 269, 248, 248, 234, 263, 273, 284, 266, 236, 251, 262, 244, 267, 258, 262, 251, 259, 265, 248, 228, 264, 223, 240, 265, 250, 244, 247, 259, 279, 260, 254, 247, 248, 273, 276, 252, 256, 266, 267, 236, 254, 266, 226, 254, 242, 258, 264, 279] payload_rows_per_dst=[5422, 5540, 5524, 5482, 5462, 5532, 5422, 5514] dup_rows_per_dst=[2680, 2718, 2782, 2736, 2660, 2700, 2642, 2720] combine_rows_per_src=[5422, 5540, 5524, 5482, 5462, 5532, 5422, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229296 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157523888 ingress_bytes=213917696 link_bytes=213917696 token_bytes=314818480 weight_movement_bytes=56623104 total_nvlink_bytes=371441584 weight_to_token_ratio=0.179860 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049654109e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229176 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157351736 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474296 weight_movement_bytes=18874368 total_nvlink_bytes=333348664 weight_to_token_ratio=0.060019 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38356 dup_rows=21624 dup_rows_remote=18872 dedup_ratio=0.329956 router_tokens_per_expert=[227, 248, 232, 242, 255, 234, 274, 256, 253, 224, 250, 243, 258, 251, 265, 235, 228, 279, 264, 243, 276, 275, 248, 273, 267, 268, 267, 296, 216, 275, 254, 244, 247, 244, 247, 251, 269, 267, 288, 239, 261, 275, 261, 266, 248, 283, 236, 260, 259, 271, 250, 266, 248, 249, 234, 274, 264, 270, 274, 260, 265, 238, 251, 245, 209, 254, 241, 254, 275, 254, 249, 249, 266, 250, 250, 228, 240, 273, 266, 229, 252, 262, 260, 254, 282, 233, 260, 239, 252, 260, 238, 265, 269, 272, 273, 283, 279, 247, 253, 246, 254, 255, 259, 258, 250, 253, 272, 261, 236, 309, 272, 252, 274, 256, 266, 262, 241, 248, 264, 276, 277, 264, 258, 209, 243, 249, 252, 252] payload_rows_per_dst=[5404, 5590, 5572, 5434, 5306, 5556, 5576, 5474] dup_rows_per_dst=[2490, 2756, 2712, 2802, 2668, 2752, 2736, 2708] combine_rows_per_src=[5404, 5590, 5572, 5434, 5306, 5556, 5576, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229832 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057480 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885128 weight_movement_bytes=18874368 total_nvlink_bytes=332759496 weight_to_token_ratio=0.060131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229888 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827584 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315425280 weight_movement_bytes=18874368 total_nvlink_bytes=334299648 weight_to_token_ratio=0.059838 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38274 dup_rows=21862 dup_rows_remote=19150 dedup_ratio=0.333588 router_tokens_per_expert=[249, 259, 247, 263, 256, 262, 252, 259, 258, 275, 258, 243, 237, 254, 259, 250, 243, 250, 230, 246, 271, 239, 249, 229, 257, 264, 242, 274, 297, 263, 242, 260, 241, 263, 266, 268, 272, 267, 245, 233, 258, 271, 258, 278, 245, 287, 256, 238, 255, 269, 283, 268, 278, 252, 262, 239, 274, 283, 251, 260, 242, 250, 257, 260, 271, 245, 267, 254, 254, 254, 237, 261, 254, 243, 246, 248, 257, 249, 255, 278, 256, 237, 242, 255, 241, 263, 246, 269, 255, 268, 249, 269, 257, 232, 243, 252, 237, 256, 278, 264, 272, 238, 260, 274, 259, 293, 233, 262, 255, 255, 236, 275, 265, 252, 229, 224, 253, 248, 263, 273, 286, 230, 267, 284, 241, 256, 218, 259] payload_rows_per_dst=[5454, 5400, 5516, 5502, 5464, 5430, 5520, 5388] dup_rows_per_dst=[2708, 2712, 2776, 2864, 2682, 2638, 2774, 2708] combine_rows_per_src=[5454, 5400, 5516, 5502, 5464, 5430, 5520, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=228912 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157335088 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314441264 weight_movement_bytes=18874368 total_nvlink_bytes=333315632 weight_to_token_ratio=0.060025 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38196 dup_rows=21822 dup_rows_remote=18886 dedup_ratio=0.332977 router_tokens_per_expert=[284, 274, 244, 250, 270, 284, 271, 263, 222, 258, 269, 283, 245, 241, 257, 279, 264, 258, 278, 237, 279, 267, 257, 255, 248, 262, 283, 258, 275, 242, 251, 247, 257, 256, 258, 241, 249, 292, 237, 268, 265, 217, 241, 244, 300, 237, 249, 234, 226, 230, 251, 293, 250, 225, 259, 255, 271, 265, 246, 254, 246, 241, 237, 267, 241, 246, 252, 261, 258, 258, 261, 268, 241, 271, 265, 231, 265, 259, 252, 266, 262, 250, 269, 266, 256, 267, 266, 260, 248, 252, 232, 257, 239, 262, 250, 236, 224, 251, 247, 225, 230, 246, 257, 238, 253, 252, 252, 226, 247, 279, 248, 282, 277, 252, 260, 242, 278, 270, 277, 278, 243, 260, 260, 276, 273, 263, 251, 268] payload_rows_per_dst=[5518, 5518, 5618, 5426, 5588, 5384, 5288, 5374] dup_rows_per_dst=[2936, 2804, 2872, 2606, 2920, 2760, 2560, 2364] combine_rows_per_src=[5518, 5518, 5618, 5426, 5588, 5384, 5288, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38348 dup_rows=21770 dup_rows_remote=19042 dedup_ratio=0.332184 router_tokens_per_expert=[266, 258, 253, 234, 253, 291, 244, 275, 252, 251, 223, 287, 252, 259, 222, 271, 243, 258, 277, 257, 255, 275, 249, 257, 245, 236, 257, 242, 275, 233, 245, 235, 266, 249, 239, 275, 257, 258, 237, 287, 268, 246, 248, 253, 241, 254, 237, 258, 243, 253, 241, 262, 253, 216, 275, 284, 250, 231, 289, 253, 236, 263, 263, 270, 243, 258, 239, 243, 246, 271, 270, 227, 263, 251, 254, 258, 268, 274, 244, 264, 285, 290, 223, 286, 267, 250, 286, 272, 277, 247, 231, 280, 256, 257, 264, 279, 240, 262, 258, 287, 293, 265, 259, 266, 244, 243, 245, 256, 265, 258, 266, 248, 274, 271, 228, 231, 273, 253, 253, 254, 253, 247, 242, 234, 249, 246, 244, 253] payload_rows_per_dst=[5454, 5536, 5428, 5440, 5418, 5616, 5486, 5388] dup_rows_per_dst=[2728, 2542, 2718, 2724, 2728, 2884, 2824, 2622] combine_rows_per_src=[5454, 5536, 5428, 5440, 5418, 5616, 5486, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229696 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157000000 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770304 weight_movement_bytes=18874368 total_nvlink_bytes=332644672 weight_to_token_ratio=0.060153 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38294 dup_rows=21816 dup_rows_remote=19108 dedup_ratio=0.332886 router_tokens_per_expert=[275, 247, 277, 260, 241, 254, 247, 248, 244, 234, 275, 247, 266, 279, 271, 245, 261, 284, 256, 263, 262, 247, 267, 268, 274, 248, 233, 273, 229, 252, 239, 235, 249, 232, 251, 241, 274, 243, 289, 262, 273, 262, 248, 224, 257, 256, 258, 248, 263, 255, 241, 230, 259, 245, 259, 266, 251, 245, 260, 257, 281, 290, 243, 242, 259, 243, 217, 273, 286, 259, 225, 271, 280, 242, 239, 239, 244, 263, 270, 236, 255, 260, 258, 268, 248, 237, 257, 262, 255, 256, 253, 261, 270, 241, 247, 228, 248, 257, 252, 267, 248, 259, 235, 242, 260, 255, 260, 245, 255, 250, 267, 258, 285, 276, 262, 266, 252, 257, 274, 274, 239, 265, 269, 257, 273, 267, 271, 266] payload_rows_per_dst=[5502, 5374, 5426, 5468, 5438, 5420, 5504, 5588] dup_rows_per_dst=[2718, 2808, 2708, 2706, 2654, 2692, 2612, 2918] combine_rows_per_src=[5502, 5374, 5426, 5468, 5438, 5420, 5504, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38228 dup_rows=21760 dup_rows_remote=19124 dedup_ratio=0.332031 router_tokens_per_expert=[250, 269, 230, 262, 261, 262, 279, 272, 238, 258, 265, 224, 243, 239, 263, 276, 242, 265, 288, 267, 241, 258, 257, 242, 267, 223, 279, 254, 246, 272, 283, 294, 258, 246, 262, 198, 234, 255, 277, 241, 266, 239, 256, 255, 261, 277, 240, 265, 270, 264, 240, 217, 275, 249, 239, 266, 243, 260, 265, 275, 245, 255, 269, 236, 240, 246, 259, 259, 235, 251, 251, 269, 285, 248, 263, 263, 272, 249, 263, 275, 251, 277, 268, 273, 250, 227, 269, 264, 233, 257, 225, 288, 268, 273, 270, 241, 280, 264, 263, 253, 249, 246, 241, 259, 225, 247, 268, 238, 254, 254, 274, 232, 277, 266, 257, 255, 263, 233, 248, 253, 256, 275, 280, 293, 236, 213, 238, 249] payload_rows_per_dst=[5368, 5488, 5410, 5474, 5506, 5496, 5486, 5548] dup_rows_per_dst=[2634, 2868, 2650, 2798, 2750, 2740, 2684, 2636] combine_rows_per_src=[5368, 5488, 5410, 5474, 5506, 5496, 5486, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38322 dup_rows=21748 dup_rows_remote=19064 dedup_ratio=0.331848 router_tokens_per_expert=[215, 263, 273, 284, 246, 276, 237, 285, 230, 269, 251, 247, 282, 257, 250, 260, 254, 256, 244, 250, 257, 262, 247, 263, 230, 234, 239, 249, 237, 259, 280, 244, 247, 265, 260, 261, 260, 238, 267, 260, 257, 267, 273, 272, 260, 228, 260, 248, 284, 252, 267, 276, 244, 264, 240, 247, 254, 226, 241, 244, 257, 266, 264, 249, 280, 245, 262, 256, 255, 283, 274, 287, 246, 244, 251, 233, 252, 266, 265, 257, 244, 248, 254, 267, 235, 272, 233, 243, 238, 276, 274, 267, 247, 258, 267, 262, 271, 257, 260, 273, 259, 270, 253, 243, 219, 272, 258, 235, 261, 240, 263, 248, 254, 256, 257, 268, 281, 247, 258, 265, 256, 265, 268, 242, 262, 235, 250, 253] payload_rows_per_dst=[5510, 5380, 5442, 5466, 5478, 5514, 5436, 5562] dup_rows_per_dst=[2740, 2630, 2804, 2684, 2834, 2656, 2728, 2672] combine_rows_per_src=[5510, 5380, 5442, 5466, 5478, 5514, 5436, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228328 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679144 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313129960 weight_movement_bytes=37748736 total_nvlink_bytes=350878696 weight_to_token_ratio=0.120553 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229560 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157302968 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376376 weight_movement_bytes=18874368 total_nvlink_bytes=333250744 weight_to_token_ratio=0.060037 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229608 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081832 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934056 weight_movement_bytes=18874368 total_nvlink_bytes=332808424 weight_to_token_ratio=0.060122 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38072 dup_rows=21954 dup_rows_remote=19086 dedup_ratio=0.334991 router_tokens_per_expert=[233, 252, 242, 272, 262, 255, 252, 258, 268, 257, 227, 250, 268, 261, 243, 244, 241, 270, 238, 272, 279, 257, 260, 253, 232, 276, 270, 251, 257, 248, 253, 242, 281, 248, 236, 261, 248, 280, 269, 240, 252, 256, 262, 276, 251, 255, 234, 252, 226, 279, 229, 252, 242, 248, 274, 261, 280, 257, 271, 257, 290, 265, 231, 251, 264, 288, 231, 280, 226, 245, 249, 255, 256, 233, 275, 249, 265, 246, 266, 248, 243, 229, 256, 248, 263, 276, 242, 227, 257, 236, 250, 258, 233, 267, 259, 251, 246, 269, 283, 273, 272, 270, 288, 254, 253, 264, 224, 273, 253, 267, 244, 256, 259, 268, 262, 255, 275, 261, 242, 257, 256, 269, 251, 262, 250, 273, 257, 254] payload_rows_per_dst=[5426, 5400, 5412, 5434, 5522, 5370, 5510, 5508] dup_rows_per_dst=[2662, 2652, 2790, 2792, 2776, 2620, 2868, 2794] combine_rows_per_src=[5426, 5400, 5412, 5434, 5522, 5370, 5510, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229408 combine_payload_bytes=156581888 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156811296 ingress_bytes=194330624 link_bytes=194330624 token_bytes=313393184 weight_movement_bytes=37748736 total_nvlink_bytes=351141920 weight_to_token_ratio=0.120452 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.093750 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861980423e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38198 dup_rows=21866 dup_rows_remote=19162 dedup_ratio=0.333649 router_tokens_per_expert=[262, 256, 272, 253, 266, 277, 271, 273, 259, 244, 236, 268, 248, 268, 258, 241, 260, 265, 205, 255, 247, 253, 277, 261, 266, 288, 243, 249, 255, 248, 255, 254, 220, 287, 284, 222, 236, 269, 252, 263, 233, 257, 264, 237, 272, 254, 267, 244, 259, 252, 235, 261, 263, 255, 249, 234, 259, 256, 250, 267, 250, 231, 253, 295, 273, 245, 249, 265, 241, 246, 256, 252, 260, 241, 249, 240, 260, 271, 268, 285, 253, 243, 256, 239, 246, 254, 270, 263, 241, 261, 248, 252, 271, 253, 280, 258, 263, 268, 283, 253, 260, 241, 262, 257, 236, 230, 281, 270, 232, 241, 255, 256, 249, 252, 253, 273, 264, 261, 253, 281, 238, 258, 240, 277, 248, 267, 254, 260] payload_rows_per_dst=[5498, 5418, 5418, 5418, 5484, 5472, 5428, 5534] dup_rows_per_dst=[2806, 2744, 2704, 2720, 2718, 2704, 2748, 2722] combine_rows_per_src=[5498, 5418, 5418, 5418, 5484, 5472, 5428, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229544 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196456 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163368 weight_movement_bytes=18874368 total_nvlink_bytes=333037736 weight_to_token_ratio=0.060078 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=228632 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156171544 ingress_bytes=174817280 link_bytes=174817280 token_bytes=312114456 weight_movement_bytes=18874368 total_nvlink_bytes=330988824 weight_to_token_ratio=0.060473 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675013161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38178 dup_rows=21898 dup_rows_remote=19300 dedup_ratio=0.334137 router_tokens_per_expert=[275, 236, 231, 270, 254, 237, 248, 241, 248, 257, 259, 235, 253, 266, 279, 255, 260, 258, 244, 245, 240, 219, 225, 252, 279, 269, 266, 243, 239, 274, 268, 248, 229, 237, 246, 250, 264, 253, 277, 242, 261, 244, 282, 253, 258, 231, 266, 273, 267, 274, 261, 233, 237, 256, 262, 275, 241, 288, 282, 243, 245, 256, 251, 278, 249, 263, 263, 257, 228, 278, 251, 248, 256, 258, 255, 270, 288, 260, 241, 264, 253, 252, 259, 276, 247, 220, 252, 225, 256, 281, 263, 271, 271, 260, 237, 268, 268, 228, 242, 240, 265, 250, 271, 248, 274, 248, 252, 249, 266, 267, 271, 259, 272, 252, 255, 252, 265, 252, 260, 248, 252, 272, 268, 259, 254, 234, 296, 271] payload_rows_per_dst=[5444, 5460, 5388, 5470, 5510, 5416, 5444, 5506] dup_rows_per_dst=[2644, 2598, 2744, 2858, 2748, 2736, 2752, 2818] combine_rows_per_src=[5444, 5460, 5388, 5470, 5510, 5416, 5444, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229440 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156688448 ingress_bytes=231956480 link_bytes=231956480 token_bytes=313147456 weight_movement_bytes=75497472 total_nvlink_bytes=388644928 weight_to_token_ratio=0.241092 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222492862e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15058287240 prefetch_bytes=849346560 reduce_bytes=622854144 weight_movement_bytes=1472200704 egress_bytes=7534647944 ingress_bytes=8995840000 link_bytes=8995840000 total_nvlink_bytes=16530487944 weight_to_token_ratio=0.097767 prefetch_share_of_ingress=0.094415 dedup_ratio=0.332506 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=33 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.795702347e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38334 dup_rows=21754 dup_rows_remote=19144 dedup_ratio=0.331940 router_tokens_per_expert=[287, 247, 270, 250, 248, 237, 255, 259, 261, 253, 257, 272, 240, 263, 236, 253, 253, 245, 256, 247, 270, 237, 282, 294, 243, 280, 244, 253, 242, 211, 272, 241, 261, 267, 242, 245, 288, 246, 227, 257, 274, 260, 267, 248, 251, 268, 263, 258, 278, 233, 248, 256, 232, 254, 241, 219, 238, 267, 257, 270, 239, 261, 251, 242, 233, 237, 262, 276, 262, 247, 241, 251, 266, 235, 242, 248, 231, 260, 287, 251, 264, 226, 294, 274, 251, 274, 249, 259, 235, 283, 254, 269, 298, 258, 285, 265, 254, 295, 258, 267, 228, 255, 279, 245, 240, 262, 246, 236, 298, 280, 265, 232, 243, 280, 282, 218, 243, 258, 256, 264, 261, 260, 248, 246, 238, 275, 267, 256] payload_rows_per_dst=[5412, 5446, 5494, 5420, 5448, 5534, 5516, 5512] dup_rows_per_dst=[2764, 2694, 2750, 2552, 2610, 2942, 2764, 2678] combine_rows_per_src=[5412, 5446, 5494, 5420, 5448, 5534, 5516, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38248 dup_rows=21744 dup_rows_remote=18910 dedup_ratio=0.331787 router_tokens_per_expert=[260, 291, 248, 252, 288, 232, 262, 264, 253, 269, 228, 261, 243, 256, 250, 239, 234, 261, 258, 251, 252, 258, 277, 248, 259, 262, 264, 238, 235, 253, 259, 244, 252, 250, 275, 235, 242, 263, 288, 271, 263, 256, 267, 252, 248, 240, 254, 235, 246, 254, 266, 241, 249, 254, 270, 272, 252, 240, 261, 254, 231, 279, 260, 279, 251, 286, 255, 283, 257, 248, 241, 258, 267, 262, 246, 252, 240, 256, 265, 242, 265, 258, 261, 251, 240, 220, 261, 262, 255, 270, 256, 230, 258, 271, 245, 253, 236, 237, 260, 269, 290, 257, 270, 235, 259, 250, 257, 277, 272, 249, 250, 264, 265, 258, 240, 260, 269, 228, 252, 284, 259, 256, 234, 283, 250, 265, 252, 268] payload_rows_per_dst=[5544, 5344, 5572, 5458, 5532, 5478, 5494, 5370] dup_rows_per_dst=[2834, 2582, 2862, 2758, 2702, 2620, 2770, 2616] combine_rows_per_src=[5544, 5344, 5572, 5458, 5532, 5478, 5494, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38468 dup_rows=21654 dup_rows_remote=19088 dedup_ratio=0.330414 router_tokens_per_expert=[278, 277, 247, 232, 246, 252, 248, 227, 270, 268, 230, 243, 245, 229, 284, 271, 234, 294, 296, 252, 253, 263, 258, 242, 254, 236, 242, 280, 252, 260, 259, 272, 258, 269, 224, 259, 242, 262, 246, 249, 233, 264, 283, 231, 265, 229, 252, 224, 237, 257, 275, 272, 251, 266, 265, 240, 274, 256, 260, 245, 269, 222, 247, 238, 239, 294, 268, 263, 243, 252, 256, 242, 271, 247, 229, 235, 251, 238, 274, 244, 247, 283, 243, 257, 283, 268, 246, 262, 274, 268, 267, 281, 240, 251, 269, 256, 241, 269, 242, 269, 253, 266, 270, 256, 253, 258, 238, 276, 233, 240, 246, 262, 277, 288, 264, 261, 256, 270, 267, 237, 247, 271, 245, 290, 243, 265, 249, 267] payload_rows_per_dst=[5462, 5530, 5414, 5492, 5416, 5642, 5406, 5520] dup_rows_per_dst=[2632, 2764, 2566, 2656, 2676, 2748, 2738, 2874] combine_rows_per_src=[5462, 5530, 5414, 5492, 5416, 5642, 5406, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229912 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156607000 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312984088 weight_movement_bytes=18874368 total_nvlink_bytes=331858456 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38360 dup_rows=21824 dup_rows_remote=19120 dedup_ratio=0.333008 router_tokens_per_expert=[260, 275, 239, 251, 250, 242, 251, 278, 244, 290, 274, 233, 236, 235, 248, 263, 258, 277, 253, 270, 276, 270, 268, 269, 263, 245, 257, 263, 247, 262, 236, 286, 231, 277, 259, 268, 253, 259, 245, 251, 261, 249, 240, 247, 265, 277, 261, 253, 245, 267, 278, 258, 237, 240, 231, 253, 281, 235, 261, 249, 263, 251, 256, 254, 280, 265, 265, 272, 248, 267, 253, 266, 255, 257, 267, 260, 276, 255, 255, 266, 261, 250, 267, 265, 252, 245, 250, 237, 244, 244, 238, 256, 278, 241, 244, 285, 262, 277, 255, 240, 274, 261, 240, 240, 248, 266, 230, 266, 235, 280, 245, 233, 269, 229, 280, 240, 265, 247, 251, 218, 246, 295, 234, 247, 263, 262, 269, 213] payload_rows_per_dst=[5438, 5572, 5428, 5442, 5574, 5470, 5436, 5352] dup_rows_per_dst=[2700, 2828, 2764, 2676, 2840, 2644, 2668, 2704] combine_rows_per_src=[5438, 5572, 5428, 5442, 5574, 5470, 5436, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38178 dup_rows=21798 dup_rows_remote=19056 dedup_ratio=0.332611 router_tokens_per_expert=[236, 237, 250, 256, 229, 252, 272, 250, 278, 249, 231, 270, 242, 265, 253, 249, 262, 260, 259, 254, 271, 240, 280, 260, 257, 240, 243, 242, 243, 237, 270, 253, 280, 247, 253, 248, 256, 260, 259, 259, 242, 230, 273, 246, 238, 285, 262, 229, 254, 247, 265, 246, 300, 242, 271, 273, 269, 244, 249, 272, 257, 248, 264, 245, 274, 254, 248, 240, 262, 264, 255, 247, 251, 253, 223, 255, 226, 274, 265, 264, 262, 268, 248, 290, 264, 247, 290, 271, 277, 254, 232, 261, 265, 281, 248, 253, 274, 252, 246, 268, 255, 270, 256, 252, 252, 257, 249, 247, 280, 260, 262, 271, 255, 247, 279, 250, 259, 239, 242, 269, 252, 239, 234, 252, 248, 277, 265, 241] payload_rows_per_dst=[5356, 5420, 5506, 5492, 5410, 5592, 5560, 5402] dup_rows_per_dst=[2682, 2722, 2628, 2800, 2700, 2830, 2742, 2694] combine_rows_per_src=[5356, 5420, 5506, 5492, 5410, 5592, 5560, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229912 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157245976 ingress_bytes=194764800 link_bytes=194764800 token_bytes=314262040 weight_movement_bytes=37748736 total_nvlink_bytes=352010776 weight_to_token_ratio=0.120119 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866140483e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228632 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156892440 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313556248 weight_movement_bytes=18874368 total_nvlink_bytes=332430616 weight_to_token_ratio=0.060195 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15062763056 prefetch_bytes=754974720 reduce_bytes=773849088 weight_movement_bytes=1528823808 egress_bytes=7536887344 ingress_bytes=9054699520 link_bytes=9054699520 total_nvlink_bytes=16591586864 weight_to_token_ratio=0.101497 prefetch_share_of_ingress=0.083379 dedup_ratio=0.332550 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=41 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081380 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.807451575e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=230224 combine_payload_bytes=157564928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157795152 ingress_bytes=176439296 link_bytes=176439296 token_bytes=315360080 weight_movement_bytes=18874368 total_nvlink_bytes=334234448 weight_to_token_ratio=0.059850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690554521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229920 combine_payload_bytes=157122560 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157352480 ingress_bytes=232620032 link_bytes=232620032 token_bytes=314475040 weight_movement_bytes=75497472 total_nvlink_bytes=389972512 weight_to_token_ratio=0.240075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228850690e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38214 dup_rows=21910 dup_rows_remote=19164 dedup_ratio=0.334320 router_tokens_per_expert=[257, 236, 239, 253, 277, 281, 272, 271, 238, 231, 258, 273, 224, 291, 263, 277, 252, 281, 279, 251, 249, 233, 254, 272, 261, 267, 253, 261, 266, 236, 270, 269, 247, 247, 255, 266, 245, 250, 236, 262, 283, 234, 265, 277, 256, 243, 266, 262, 238, 260, 254, 252, 241, 278, 258, 247, 266, 251, 242, 234, 253, 264, 288, 262, 250, 248, 259, 241, 240, 234, 246, 251, 277, 248, 240, 292, 273, 264, 263, 262, 228, 246, 264, 256, 263, 241, 256, 242, 250, 229, 255, 266, 245, 287, 268, 283, 291, 272, 236, 246, 246, 245, 236, 224, 232, 253, 257, 244, 252, 255, 242, 257, 264, 262, 234, 231, 258, 265, 265, 269, 266, 221, 232, 261, 257, 271, 283, 297] payload_rows_per_dst=[5454, 5450, 5522, 5500, 5428, 5412, 5320, 5540] dup_rows_per_dst=[2828, 2858, 2666, 2676, 2748, 2746, 2656, 2732] combine_rows_per_src=[5454, 5450, 5522, 5500, 5428, 5412, 5320, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228936 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606024 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983112 weight_movement_bytes=18874368 total_nvlink_bytes=331857480 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15067420800 prefetch_bytes=811597824 reduce_bytes=603979776 weight_movement_bytes=1415577600 egress_bytes=7539218560 ingress_bytes=8943779840 link_bytes=8943779840 total_nvlink_bytes=16482998400 weight_to_token_ratio=0.093950 prefetch_share_of_ingress=0.090744 dedup_ratio=0.332553 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=32 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080150 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.785310371e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15062207288 prefetch_bytes=792723456 reduce_bytes=622854144 weight_movement_bytes=1415577600 egress_bytes=7536610104 ingress_bytes=8941174784 link_bytes=8941174784 total_nvlink_bytes=16477784888 weight_to_token_ratio=0.093982 prefetch_share_of_ingress=0.088660 dedup_ratio=0.332518 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=33 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081814 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.784790363e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15032965480 prefetch_bytes=698351616 reduce_bytes=1038090240 weight_movement_bytes=1736441856 egress_bytes=7521974632 ingress_bytes=9247432704 link_bytes=9247432704 total_nvlink_bytes=16769407336 weight_to_token_ratio=0.115509 prefetch_share_of_ingress=0.075518 dedup_ratio=0.332738 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=55 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081453 gemm_inner_pad_upper_bound=108331 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.845923961e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229512 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156754056 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278600 weight_movement_bytes=56623104 total_nvlink_bytes=369901704 weight_to_token_ratio=0.180744 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38288 dup_rows=21606 dup_rows_remote=18704 dedup_ratio=0.329681 router_tokens_per_expert=[240, 248, 259, 255, 261, 246, 274, 277, 280, 234, 250, 270, 245, 254, 247, 263, 257, 280, 262, 252, 295, 252, 280, 242, 281, 245, 252, 278, 251, 273, 287, 228, 260, 252, 278, 265, 286, 240, 248, 246, 249, 272, 251, 260, 275, 259, 258, 278, 259, 238, 239, 258, 263, 274, 274, 253, 240, 254, 261, 260, 242, 236, 256, 250, 273, 264, 262, 220, 259, 241, 246, 238, 249, 250, 248, 262, 260, 255, 235, 247, 246, 234, 256, 221, 265, 256, 260, 248, 267, 244, 268, 266, 252, 263, 258, 262, 256, 266, 260, 242, 278, 240, 243, 256, 240, 230, 288, 238, 259, 265, 215, 262, 266, 246, 235, 265, 255, 278, 252, 267, 248, 245, 269, 239, 256, 298, 241, 243] payload_rows_per_dst=[5642, 5654, 5608, 5304, 5464, 5388, 5414, 5456] dup_rows_per_dst=[2902, 2868, 2790, 2336, 2554, 2744, 2662, 2750] combine_rows_per_src=[5642, 5654, 5608, 5304, 5464, 5388, 5414, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15051400264 prefetch_bytes=811597824 reduce_bytes=868220928 weight_movement_bytes=1679818752 egress_bytes=7531201608 ingress_bytes=9200017408 link_bytes=9200017408 total_nvlink_bytes=16731219016 weight_to_token_ratio=0.111605 prefetch_share_of_ingress=0.088217 dedup_ratio=0.332794 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=46 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.079427 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.836459169e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38300 dup_rows=21764 dup_rows_remote=19000 dedup_ratio=0.332092 router_tokens_per_expert=[246, 249, 269, 240, 273, 243, 252, 252, 271, 274, 251, 264, 232, 274, 232, 239, 245, 253, 215, 278, 286, 267, 234, 244, 260, 242, 234, 237, 240, 268, 225, 262, 252, 286, 255, 262, 256, 255, 282, 241, 276, 246, 231, 275, 252, 265, 273, 235, 267, 249, 229, 268, 246, 267, 216, 260, 244, 251, 243, 264, 274, 252, 263, 224, 251, 235, 268, 244, 263, 251, 258, 261, 256, 252, 254, 276, 260, 238, 268, 266, 277, 235, 262, 276, 296, 259, 247, 272, 265, 259, 281, 250, 243, 263, 254, 243, 268, 240, 276, 248, 264, 241, 236, 271, 240, 261, 256, 262, 274, 277, 262, 242, 244, 270, 265, 260, 227, 280, 279, 254, 267, 259, 253, 251, 247, 262, 268, 271] payload_rows_per_dst=[5434, 5370, 5556, 5428, 5486, 5558, 5472, 5468] dup_rows_per_dst=[2688, 2610, 2728, 2606, 2716, 2806, 2764, 2846] combine_rows_per_src=[5434, 5370, 5556, 5428, 5486, 5558, 5472, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=227968 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157055616 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313883264 weight_movement_bytes=18874368 total_nvlink_bytes=332757632 weight_to_token_ratio=0.060132 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229200 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106000 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982800 weight_movement_bytes=18874368 total_nvlink_bytes=332857168 weight_to_token_ratio=0.060113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15056285888 prefetch_bytes=773849088 reduce_bytes=962592768 weight_movement_bytes=1736441856 egress_bytes=7533646016 ingress_bytes=9259081728 link_bytes=9259081728 total_nvlink_bytes=16792727744 weight_to_token_ratio=0.115330 prefetch_share_of_ingress=0.083577 dedup_ratio=0.332560 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=51 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080729 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.848249278e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38134 dup_rows=21906 dup_rows_remote=19092 dedup_ratio=0.334259 router_tokens_per_expert=[240, 247, 234, 276, 244, 260, 239, 274, 279, 277, 261, 268, 256, 285, 258, 248, 238, 275, 251, 271, 249, 250, 219, 246, 274, 241, 272, 238, 257, 276, 231, 268, 246, 242, 240, 251, 249, 234, 246, 254, 240, 261, 289, 237, 248, 267, 269, 265, 268, 245, 239, 276, 236, 216, 257, 251, 260, 269, 257, 222, 260, 262, 245, 263, 263, 259, 280, 244, 251, 256, 242, 257, 264, 245, 292, 267, 251, 273, 269, 251, 213, 246, 247, 264, 280, 264, 244, 279, 278, 272, 270, 241, 248, 274, 304, 241, 261, 234, 284, 238, 258, 250, 203, 265, 278, 254, 247, 236, 230, 250, 256, 269, 250, 230, 254, 243, 250, 260, 259, 271, 282, 261, 247, 252, 287, 284, 264, 266] payload_rows_per_dst=[5496, 5500, 5530, 5342, 5628, 5352, 5336, 5446] dup_rows_per_dst=[2814, 2848, 2804, 2710, 2874, 2484, 2690, 2682] combine_rows_per_src=[5496, 5500, 5530, 5342, 5628, 5352, 5336, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=228904 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156425768 ingress_bytes=212819968 link_bytes=212819968 token_bytes=312622632 weight_movement_bytes=56623104 total_nvlink_bytes=369245736 weight_to_token_ratio=0.181123 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039136219e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38060 dup_rows=21912 dup_rows_remote=19090 dedup_ratio=0.334351 router_tokens_per_expert=[271, 239, 231, 254, 261, 247, 254, 274, 243, 257, 247, 252, 256, 273, 255, 244, 266, 237, 255, 258, 239, 270, 283, 248, 256, 252, 254, 271, 242, 251, 269, 257, 256, 282, 256, 265, 257, 242, 240, 253, 276, 251, 242, 263, 251, 232, 285, 237, 222, 262, 257, 284, 238, 245, 238, 249, 258, 259, 252, 240, 264, 278, 246, 231, 240, 263, 226, 236, 267, 236, 252, 255, 265, 256, 253, 255, 258, 276, 281, 239, 255, 261, 220, 265, 279, 256, 241, 253, 294, 264, 241, 282, 283, 262, 257, 267, 244, 275, 248, 264, 247, 265, 271, 293, 265, 260, 254, 243, 257, 246, 264, 238, 257, 252, 234, 260, 264, 276, 246, 280, 247, 267, 243, 279, 261, 254, 256, 243] payload_rows_per_dst=[5564, 5554, 5464, 5640, 5380, 5190, 5610, 5222] dup_rows_per_dst=[2822, 2792, 2806, 2970, 2644, 2522, 2826, 2530] combine_rows_per_src=[5564, 5554, 5464, 5640, 5380, 5190, 5610, 5222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=228600 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156122360 ingress_bytes=174768128 link_bytes=174768128 token_bytes=312016120 weight_movement_bytes=18874368 total_nvlink_bytes=330890488 weight_to_token_ratio=0.060492 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674542211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15030375080 prefetch_bytes=811597824 reduce_bytes=811597824 weight_movement_bytes=1623195648 egress_bytes=7520678568 ingress_bytes=9132892160 link_bytes=9132892160 total_nvlink_bytes=16653570728 weight_to_token_ratio=0.107994 prefetch_share_of_ingress=0.088865 dedup_ratio=0.332879 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=43 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081163 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.823059979e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none + [2026-08-29 23:39:08] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 777.5 | throughput per GPU (TFLOP/s/GPU): 114.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.287969E+00 | loss scale: 1.0 | grad norm: 7.846 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38286 dup_rows=21794 dup_rows_remote=19160 dedup_ratio=0.332550 router_tokens_per_expert=[265, 235, 258, 236, 271, 251, 253, 261, 292, 277, 272, 260, 221, 293, 255, 229, 249, 261, 252, 239, 294, 267, 267, 241, 244, 270, 279, 248, 249, 247, 255, 233, 249, 274, 260, 249, 252, 254, 274, 271, 251, 272, 269, 255, 252, 244, 241, 254, 245, 251, 246, 267, 237, 277, 252, 264, 239, 234, 261, 249, 269, 242, 270, 242, 252, 270, 260, 270, 261, 245, 249, 253, 261, 205, 267, 256, 265, 244, 311, 252, 280, 276, 260, 247, 255, 237, 238, 262, 269, 268, 229, 253, 267, 273, 252, 272, 247, 267, 247, 244, 248, 220, 243, 252, 273, 259, 250, 241, 241, 260, 259, 224, 268, 268, 265, 267, 238, 242, 254, 268, 239, 253, 261, 266, 258, 260, 270, 267] payload_rows_per_dst=[5416, 5498, 5490, 5456, 5478, 5490, 5408, 5506] dup_rows_per_dst=[2842, 2692, 2752, 2634, 2764, 2786, 2542, 2782] combine_rows_per_src=[5416, 5498, 5490, 5456, 5478, 5490, 5408, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38162 dup_rows=21908 dup_rows_remote=19240 dedup_ratio=0.334290 router_tokens_per_expert=[224, 253, 257, 258, 277, 253, 268, 265, 252, 274, 256, 279, 265, 264, 265, 241, 242, 259, 287, 273, 230, 247, 239, 248, 268, 253, 255, 255, 267, 244, 242, 256, 249, 247, 262, 249, 248, 254, 253, 271, 285, 259, 233, 249, 234, 254, 281, 239, 247, 256, 289, 233, 252, 244, 252, 240, 232, 264, 255, 265, 241, 265, 263, 242, 271, 237, 264, 279, 256, 266, 269, 235, 282, 258, 243, 252, 261, 268, 243, 266, 241, 259, 275, 252, 283, 264, 261, 261, 254, 240, 257, 275, 247, 250, 273, 255, 265, 267, 241, 261, 242, 266, 252, 266, 235, 290, 243, 258, 239, 261, 236, 267, 271, 285, 235, 242, 262, 238, 253, 262, 229, 240, 291, 261, 251, 226, 264, 249] payload_rows_per_dst=[5420, 5438, 5466, 5402, 5462, 5500, 5486, 5454] dup_rows_per_dst=[2882, 2692, 2668, 2678, 2838, 2794, 2692, 2664] combine_rows_per_src=[5420, 5438, 5466, 5402, 5462, 5500, 5486, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38332 dup_rows=21752 dup_rows_remote=19072 dedup_ratio=0.331909 router_tokens_per_expert=[263, 227, 245, 260, 253, 268, 259, 233, 267, 262, 265, 253, 263, 290, 270, 274, 251, 291, 260, 278, 239, 233, 235, 234, 265, 225, 248, 286, 263, 241, 283, 205, 261, 236, 261, 274, 255, 263, 237, 258, 275, 262, 264, 247, 265, 254, 263, 228, 268, 276, 254, 272, 282, 221, 252, 251, 268, 244, 265, 265, 252, 249, 250, 255, 263, 250, 253, 253, 249, 259, 257, 241, 276, 250, 258, 236, 248, 248, 265, 260, 276, 252, 238, 263, 254, 263, 262, 255, 270, 253, 229, 235, 230, 267, 226, 243, 258, 259, 260, 282, 239, 264, 262, 230, 259, 271, 249, 252, 250, 283, 239, 265, 266, 274, 264, 266, 219, 276, 254, 257, 277, 242, 265, 240, 245, 285, 249, 269] payload_rows_per_dst=[5574, 5406, 5508, 5440, 5452, 5376, 5542, 5486] dup_rows_per_dst=[2730, 2668, 2698, 2808, 2680, 2656, 2702, 2810] combine_rows_per_src=[5574, 5406, 5508, 5440, 5452, 5376, 5542, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38344 dup_rows=21776 dup_rows_remote=19180 dedup_ratio=0.332275 router_tokens_per_expert=[251, 253, 250, 241, 267, 263, 271, 258, 253, 253, 243, 258, 265, 257, 263, 235, 257, 249, 255, 278, 237, 248, 246, 250, 250, 247, 263, 238, 241, 249, 264, 234, 240, 272, 249, 226, 265, 273, 271, 248, 250, 269, 241, 279, 249, 244, 262, 269, 251, 247, 261, 241, 281, 243, 254, 263, 308, 243, 236, 238, 265, 244, 249, 251, 203, 263, 218, 253, 248, 250, 263, 252, 279, 247, 273, 266, 248, 252, 268, 267, 231, 253, 283, 253, 265, 253, 252, 240, 263, 259, 246, 263, 250, 225, 259, 273, 255, 277, 261, 259, 269, 254, 262, 245, 285, 271, 263, 257, 264, 268, 280, 247, 259, 266, 251, 246, 275, 259, 286, 280, 253, 243, 257, 259, 256, 260, 255, 259] payload_rows_per_dst=[5428, 5416, 5472, 5444, 5476, 5466, 5552, 5506] dup_rows_per_dst=[2734, 2596, 2742, 2706, 2624, 2670, 2882, 2822] combine_rows_per_src=[5428, 5416, 5472, 5444, 5476, 5466, 5552, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38148 dup_rows=21838 dup_rows_remote=18918 dedup_ratio=0.333221 router_tokens_per_expert=[246, 254, 232, 256, 256, 261, 249, 221, 249, 231, 261, 292, 259, 257, 258, 243, 256, 269, 275, 227, 261, 265, 228, 251, 290, 215, 238, 279, 253, 258, 257, 259, 251, 269, 257, 231, 259, 288, 253, 260, 269, 253, 240, 264, 252, 261, 220, 286, 266, 267, 224, 266, 257, 254, 248, 279, 241, 264, 276, 246, 250, 236, 261, 279, 266, 274, 267, 232, 223, 280, 231, 252, 237, 287, 257, 251, 242, 278, 274, 251, 251, 232, 235, 263, 281, 264, 241, 267, 269, 240, 262, 242, 246, 270, 237, 258, 260, 267, 234, 246, 253, 243, 267, 252, 253, 268, 229, 261, 266, 276, 256, 238, 262, 264, 260, 265, 255, 249, 263, 253, 281, 278, 269, 266, 255, 252, 276, 258] payload_rows_per_dst=[5414, 5442, 5564, 5524, 5404, 5370, 5430, 5550] dup_rows_per_dst=[2636, 2720, 2658, 2696, 2808, 2692, 2708, 2920] combine_rows_per_src=[5414, 5442, 5564, 5524, 5404, 5370, 5430, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38240 dup_rows=21780 dup_rows_remote=19068 dedup_ratio=0.332336 router_tokens_per_expert=[277, 249, 246, 255, 240, 242, 238, 258, 261, 242, 257, 238, 248, 247, 237, 245, 254, 259, 272, 273, 269, 252, 271, 266, 252, 264, 250, 249, 259, 246, 228, 292, 283, 275, 272, 272, 225, 242, 252, 261, 271, 235, 249, 253, 285, 243, 249, 241, 224, 265, 261, 229, 244, 268, 294, 269, 267, 280, 262, 263, 239, 257, 271, 261, 283, 274, 257, 266, 254, 278, 219, 233, 243, 259, 258, 244, 255, 245, 262, 250, 279, 236, 244, 263, 264, 241, 245, 260, 263, 259, 238, 297, 253, 257, 253, 262, 225, 267, 242, 232, 257, 239, 277, 220, 262, 254, 242, 257, 244, 265, 251, 248, 247, 276, 274, 262, 286, 247, 233, 279, 247, 256, 268, 257, 252, 269, 272, 269] payload_rows_per_dst=[5366, 5518, 5434, 5452, 5468, 5516, 5448, 5554] dup_rows_per_dst=[2594, 2794, 2782, 2856, 2692, 2712, 2516, 2834] combine_rows_per_src=[5366, 5518, 5434, 5452, 5468, 5516, 5448, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38174 dup_rows=21844 dup_rows_remote=19140 dedup_ratio=0.333313 router_tokens_per_expert=[267, 256, 250, 242, 256, 227, 253, 255, 249, 244, 260, 257, 261, 241, 265, 281, 258, 248, 245, 272, 255, 242, 249, 258, 268, 236, 253, 223, 263, 266, 261, 280, 256, 256, 302, 279, 259, 277, 270, 234, 283, 270, 272, 258, 265, 256, 244, 252, 238, 244, 262, 259, 276, 261, 259, 258, 258, 238, 245, 274, 233, 245, 267, 259, 251, 261, 232, 262, 258, 273, 269, 236, 219, 271, 283, 249, 257, 257, 249, 277, 256, 223, 248, 286, 265, 235, 300, 254, 215, 245, 245, 258, 258, 246, 273, 255, 249, 272, 271, 233, 282, 250, 243, 268, 268, 258, 240, 264, 283, 253, 227, 250, 257, 260, 254, 260, 239, 250, 278, 223, 251, 250, 265, 242, 224, 268, 261, 259] payload_rows_per_dst=[5464, 5458, 5598, 5464, 5452, 5320, 5518, 5418] dup_rows_per_dst=[2664, 2696, 2868, 2688, 2756, 2804, 2704, 2664] combine_rows_per_src=[5464, 5458, 5598, 5464, 5452, 5320, 5518, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229784 combine_payload_bytes=156819456 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157049240 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868696 weight_movement_bytes=37748736 total_nvlink_bytes=351617432 weight_to_token_ratio=0.120269 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229608 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156541160 ingress_bytes=212934656 link_bytes=212934656 token_bytes=312852712 weight_movement_bytes=56623104 total_nvlink_bytes=369475816 weight_to_token_ratio=0.180990 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040235103e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229616 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237488 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245360 weight_movement_bytes=18874368 total_nvlink_bytes=333119728 weight_to_token_ratio=0.060063 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=230096 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157287120 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314344144 weight_movement_bytes=18874368 total_nvlink_bytes=333218512 weight_to_token_ratio=0.060044 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228264 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156482472 ingress_bytes=212877312 link_bytes=212877312 token_bytes=312736680 weight_movement_bytes=56623104 total_nvlink_bytes=369359784 weight_to_token_ratio=0.181057 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039685661e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229232 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860272 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491312 weight_movement_bytes=37748736 total_nvlink_bytes=351240048 weight_to_token_ratio=0.120414 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229256 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156589960 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950664 weight_movement_bytes=18874368 total_nvlink_bytes=331825032 weight_to_token_ratio=0.060311 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38274 dup_rows=21724 dup_rows_remote=18980 dedup_ratio=0.331482 router_tokens_per_expert=[241, 258, 277, 227, 285, 269, 230, 255, 254, 252, 256, 285, 272, 227, 258, 258, 259, 262, 236, 254, 230, 253, 252, 252, 269, 239, 260, 254, 235, 268, 227, 253, 284, 291, 255, 252, 238, 253, 227, 238, 250, 269, 244, 285, 258, 265, 236, 246, 228, 251, 266, 245, 248, 278, 260, 255, 261, 272, 257, 282, 252, 268, 258, 260, 280, 293, 256, 249, 296, 266, 252, 272, 259, 242, 256, 260, 262, 236, 265, 224, 271, 249, 269, 244, 272, 244, 249, 227, 239, 294, 258, 250, 242, 268, 263, 254, 281, 254, 241, 281, 210, 254, 268, 246, 239, 259, 243, 259, 261, 250, 265, 258, 254, 246, 227, 243, 271, 262, 270, 259, 265, 250, 250, 242, 252, 288, 263, 257] payload_rows_per_dst=[5550, 5374, 5440, 5538, 5512, 5508, 5446, 5444] dup_rows_per_dst=[2658, 2632, 2742, 2744, 2824, 2678, 2692, 2754] combine_rows_per_src=[5550, 5374, 5440, 5538, 5512, 5508, 5446, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38098 dup_rows=21888 dup_rows_remote=19040 dedup_ratio=0.333984 router_tokens_per_expert=[237, 207, 260, 248, 264, 265, 258, 295, 263, 260, 251, 274, 296, 254, 251, 242, 256, 241, 257, 260, 248, 240, 241, 258, 250, 299, 224, 263, 252, 266, 240, 253, 256, 251, 250, 250, 276, 256, 256, 247, 264, 250, 293, 268, 275, 273, 262, 272, 259, 258, 258, 246, 265, 266, 259, 269, 231, 255, 273, 262, 234, 264, 279, 232, 272, 280, 259, 264, 266, 265, 257, 256, 275, 272, 256, 243, 262, 255, 274, 288, 246, 255, 259, 279, 226, 266, 262, 250, 294, 246, 263, 264, 250, 246, 256, 237, 251, 233, 263, 279, 243, 258, 236, 268, 259, 258, 245, 225, 258, 258, 245, 229, 249, 250, 236, 240, 262, 230, 230, 258, 234, 251, 243, 201, 266, 264, 256, 265] payload_rows_per_dst=[5444, 5416, 5550, 5458, 5516, 5544, 5412, 5308] dup_rows_per_dst=[2806, 2680, 2848, 2762, 2972, 2654, 2604, 2562] combine_rows_per_src=[5444, 5416, 5550, 5458, 5516, 5544, 5412, 5308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38122 dup_rows=21964 dup_rows_remote=19238 dedup_ratio=0.335144 router_tokens_per_expert=[260, 238, 257, 245, 262, 252, 264, 240, 263, 248, 250, 237, 284, 257, 243, 251, 263, 288, 239, 284, 235, 242, 270, 239, 259, 261, 268, 236, 277, 262, 277, 278, 272, 273, 286, 241, 245, 252, 270, 275, 254, 264, 247, 282, 250, 245, 241, 253, 233, 247, 274, 264, 243, 229, 219, 240, 230, 275, 244, 279, 279, 250, 264, 238, 254, 263, 262, 229, 231, 263, 258, 254, 253, 245, 267, 242, 267, 270, 251, 279, 246, 250, 287, 265, 275, 262, 254, 231, 239, 259, 255, 229, 268, 277, 247, 260, 267, 235, 275, 230, 254, 256, 282, 255, 260, 252, 225, 265, 248, 238, 261, 287, 242, 271, 268, 297, 259, 256, 247, 218, 213, 251, 264, 276, 255, 249, 261, 272] payload_rows_per_dst=[5368, 5458, 5468, 5438, 5450, 5520, 5400, 5470] dup_rows_per_dst=[2734, 2898, 2832, 2578, 2726, 2688, 2780, 2728] combine_rows_per_src=[5368, 5458, 5468, 5438, 5450, 5520, 5400, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38234 dup_rows=21870 dup_rows_remote=19226 dedup_ratio=0.333710 router_tokens_per_expert=[248, 252, 231, 264, 258, 292, 243, 257, 261, 232, 249, 231, 250, 273, 234, 247, 244, 258, 236, 259, 272, 241, 252, 283, 256, 258, 235, 252, 260, 230, 240, 262, 245, 261, 240, 244, 255, 261, 247, 272, 253, 219, 277, 285, 244, 261, 240, 232, 265, 266, 253, 245, 285, 279, 249, 273, 255, 246, 246, 251, 247, 243, 246, 277, 270, 246, 271, 259, 219, 266, 261, 257, 266, 255, 286, 239, 259, 257, 246, 259, 236, 234, 257, 249, 237, 269, 257, 261, 250, 230, 260, 268, 245, 299, 263, 265, 251, 262, 247, 257, 268, 280, 262, 237, 233, 276, 271, 258, 271, 249, 263, 253, 271, 264, 247, 266, 274, 249, 240, 268, 252, 263, 243, 279, 270, 272, 293, 261] payload_rows_per_dst=[5390, 5432, 5430, 5458, 5472, 5468, 5494, 5522] dup_rows_per_dst=[2654, 2644, 2642, 2794, 2760, 2692, 2782, 2902] combine_rows_per_src=[5390, 5432, 5430, 5458, 5472, 5468, 5494, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38164 dup_rows=21832 dup_rows_remote=19038 dedup_ratio=0.333130 router_tokens_per_expert=[238, 255, 265, 243, 256, 247, 247, 262, 246, 262, 248, 279, 248, 263, 266, 238, 298, 264, 265, 263, 243, 269, 260, 255, 255, 273, 246, 248, 270, 259, 235, 266, 283, 259, 237, 247, 242, 260, 236, 263, 271, 270, 267, 238, 211, 264, 259, 260, 248, 253, 257, 244, 253, 262, 288, 264, 257, 275, 259, 254, 259, 246, 258, 257, 246, 230, 246, 271, 294, 254, 245, 240, 247, 257, 266, 258, 271, 246, 255, 257, 261, 275, 255, 236, 241, 231, 259, 283, 278, 260, 248, 263, 226, 214, 255, 267, 231, 247, 229, 276, 235, 266, 261, 247, 257, 264, 237, 283, 263, 248, 243, 256, 247, 277, 277, 277, 226, 274, 278, 271, 273, 247, 260, 272, 269, 223, 257, 229] payload_rows_per_dst=[5490, 5456, 5424, 5386, 5332, 5562, 5514, 5540] dup_rows_per_dst=[2606, 2838, 2710, 2660, 2452, 2978, 2794, 2794] combine_rows_per_src=[5490, 5456, 5424, 5386, 5332, 5562, 5514, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38144 dup_rows=21972 dup_rows_remote=19238 dedup_ratio=0.335266 router_tokens_per_expert=[271, 255, 266, 262, 268, 231, 250, 289, 245, 268, 249, 269, 263, 237, 258, 255, 252, 258, 263, 258, 268, 256, 260, 268, 230, 264, 257, 250, 267, 265, 252, 254, 242, 258, 244, 241, 264, 252, 231, 255, 245, 284, 267, 247, 237, 274, 229, 262, 264, 258, 242, 273, 260, 246, 256, 253, 278, 274, 236, 236, 257, 268, 279, 256, 234, 255, 272, 264, 257, 241, 240, 242, 231, 257, 277, 256, 244, 249, 260, 257, 250, 243, 235, 229, 272, 250, 258, 263, 251, 233, 270, 257, 281, 260, 266, 259, 268, 269, 249, 259, 267, 249, 248, 276, 260, 256, 241, 255, 252, 257, 243, 252, 271, 251, 251, 259, 247, 267, 271, 271, 261, 257, 241, 251, 246, 272, 245, 267] payload_rows_per_dst=[5522, 5506, 5358, 5506, 5370, 5420, 5444, 5438] dup_rows_per_dst=[2750, 2738, 2706, 2766, 2702, 2734, 2758, 2818] combine_rows_per_src=[5522, 5506, 5358, 5506, 5370, 5420, 5444, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38360 dup_rows=21650 dup_rows_remote=18900 dedup_ratio=0.330353 router_tokens_per_expert=[287, 269, 264, 259, 224, 243, 234, 268, 252, 262, 284, 253, 253, 255, 239, 266, 260, 246, 226, 265, 262, 281, 267, 248, 247, 253, 267, 260, 249, 272, 263, 239, 257, 248, 272, 275, 289, 224, 273, 237, 237, 265, 237, 246, 277, 224, 248, 267, 225, 261, 253, 247, 260, 280, 255, 225, 276, 251, 267, 227, 243, 246, 294, 277, 257, 246, 281, 267, 230, 263, 249, 261, 241, 260, 255, 251, 260, 285, 266, 268, 273, 252, 244, 221, 260, 238, 231, 266, 261, 253, 244, 237, 285, 251, 287, 236, 255, 270, 268, 256, 243, 247, 276, 268, 250, 255, 283, 261, 229, 274, 248, 255, 277, 263, 230, 239, 254, 273, 255, 281, 250, 245, 254, 235, 248, 286, 235, 246] payload_rows_per_dst=[5498, 5514, 5506, 5464, 5538, 5412, 5526, 5428] dup_rows_per_dst=[2726, 2696, 2646, 2710, 2742, 2666, 2750, 2714] combine_rows_per_src=[5498, 5514, 5506, 5464, 5538, 5412, 5526, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229016 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156999320 ingress_bytes=213393408 link_bytes=213393408 token_bytes=313769624 weight_movement_bytes=56623104 total_nvlink_bytes=370392728 weight_to_token_ratio=0.180461 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044630639e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228552 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156277960 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312327368 weight_movement_bytes=18874368 total_nvlink_bytes=331201736 weight_to_token_ratio=0.060431 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229440 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156377152 ingress_bytes=193896448 link_bytes=193896448 token_bytes=312524864 weight_movement_bytes=37748736 total_nvlink_bytes=350273600 weight_to_token_ratio=0.120786 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857820362e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229840 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156836304 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313442768 weight_movement_bytes=37748736 total_nvlink_bytes=351191504 weight_to_token_ratio=0.120433 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228808 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156548552 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868296 weight_movement_bytes=37748736 total_nvlink_bytes=350617032 weight_to_token_ratio=0.120654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229528 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156467352 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312705176 weight_movement_bytes=18874368 total_nvlink_bytes=331579544 weight_to_token_ratio=0.060358 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229040 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157351600 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474160 weight_movement_bytes=18874368 total_nvlink_bytes=333348528 weight_to_token_ratio=0.060019 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38320 dup_rows=21796 dup_rows_remote=19054 dedup_ratio=0.332581 router_tokens_per_expert=[257, 241, 260, 246, 234, 284, 273, 236, 244, 253, 247, 250, 257, 266, 253, 267, 261, 276, 255, 274, 239, 277, 239, 280, 238, 268, 232, 287, 252, 229, 282, 251, 257, 244, 282, 258, 268, 226, 262, 266, 260, 264, 257, 249, 265, 255, 261, 246, 264, 275, 242, 250, 254, 266, 268, 248, 247, 272, 248, 247, 245, 252, 250, 253, 287, 275, 238, 246, 267, 249, 265, 262, 242, 254, 255, 284, 263, 246, 251, 236, 273, 250, 254, 225, 275, 271, 243, 254, 269, 243, 238, 226, 263, 237, 237, 244, 284, 272, 261, 268, 247, 264, 264, 283, 254, 250, 253, 232, 260, 242, 269, 260, 249, 276, 240, 279, 224, 252, 239, 238, 243, 247, 250, 256, 305, 262, 242, 272] payload_rows_per_dst=[5432, 5446, 5542, 5420, 5534, 5396, 5538, 5432] dup_rows_per_dst=[2704, 2834, 2698, 2742, 2706, 2608, 2788, 2716] combine_rows_per_src=[5432, 5446, 5542, 5420, 5534, 5396, 5538, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38262 dup_rows=21886 dup_rows_remote=19130 dedup_ratio=0.333954 router_tokens_per_expert=[248, 269, 225, 263, 244, 235, 263, 262, 263, 271, 242, 262, 265, 263, 256, 241, 252, 250, 236, 244, 286, 259, 247, 259, 243, 251, 259, 270, 273, 268, 229, 259, 248, 276, 254, 250, 230, 261, 279, 247, 243, 245, 256, 254, 275, 260, 244, 250, 251, 255, 253, 264, 244, 236, 268, 282, 247, 247, 237, 265, 254, 259, 270, 250, 252, 230, 262, 244, 274, 254, 273, 286, 245, 257, 253, 225, 263, 245, 266, 265, 229, 227, 254, 252, 275, 249, 260, 238, 259, 279, 284, 256, 270, 268, 260, 268, 255, 246, 269, 242, 255, 242, 273, 243, 257, 266, 277, 252, 256, 276, 245, 259, 245, 277, 265, 260, 251, 271, 264, 281, 237, 255, 252, 252, 270, 236, 248, 258] payload_rows_per_dst=[5376, 5454, 5388, 5400, 5494, 5526, 5490, 5522] dup_rows_per_dst=[2768, 2716, 2756, 2764, 2694, 2730, 2736, 2722] combine_rows_per_src=[5376, 5454, 5388, 5400, 5494, 5526, 5490, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38080 dup_rows=21952 dup_rows_remote=19326 dedup_ratio=0.334961 router_tokens_per_expert=[258, 270, 246, 232, 264, 266, 270, 261, 245, 251, 238, 238, 249, 273, 233, 244, 252, 280, 285, 260, 249, 255, 272, 241, 256, 254, 290, 236, 278, 251, 241, 252, 287, 235, 240, 232, 278, 283, 280, 257, 221, 275, 255, 261, 231, 243, 265, 256, 258, 235, 246, 247, 286, 227, 242, 267, 279, 265, 254, 243, 293, 271, 242, 283, 254, 239, 261, 248, 239, 240, 251, 255, 256, 248, 268, 253, 247, 266, 255, 285, 274, 246, 225, 258, 258, 244, 285, 250, 263, 283, 253, 241, 262, 235, 258, 257, 263, 254, 270, 272, 255, 220, 271, 241, 273, 255, 257, 268, 256, 260, 267, 231, 233, 257, 257, 256, 252, 258, 263, 229, 284, 236, 230, 281, 259, 271, 235, 270] payload_rows_per_dst=[5378, 5544, 5440, 5394, 5504, 5462, 5428, 5434] dup_rows_per_dst=[2698, 2760, 2758, 2882, 2626, 2722, 2798, 2708] combine_rows_per_src=[5378, 5544, 5440, 5394, 5504, 5462, 5428, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38322 dup_rows=21820 dup_rows_remote=19306 dedup_ratio=0.332947 router_tokens_per_expert=[277, 275, 249, 256, 264, 259, 247, 239, 233, 246, 243, 279, 255, 255, 267, 267, 249, 258, 278, 267, 260, 242, 251, 270, 259, 247, 233, 277, 270, 262, 281, 282, 240, 248, 291, 288, 254, 220, 256, 258, 255, 276, 245, 240, 261, 269, 270, 246, 290, 282, 249, 273, 259, 242, 237, 244, 264, 255, 242, 242, 238, 232, 237, 273, 254, 233, 267, 239, 233, 285, 266, 270, 247, 236, 260, 275, 238, 244, 261, 283, 263, 269, 240, 279, 217, 250, 276, 276, 265, 279, 252, 268, 266, 262, 277, 260, 260, 234, 231, 251, 277, 232, 281, 260, 249, 251, 250, 243, 236, 228, 236, 235, 277, 239, 260, 278, 249, 250, 251, 250, 211, 242, 302, 258, 253, 245, 242, 244] payload_rows_per_dst=[5438, 5496, 5494, 5402, 5460, 5542, 5394, 5490] dup_rows_per_dst=[2784, 2876, 2740, 2716, 2722, 2856, 2514, 2612] combine_rows_per_src=[5438, 5496, 5494, 5402, 5460, 5542, 5394, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38230 dup_rows=21794 dup_rows_remote=18912 dedup_ratio=0.332550 router_tokens_per_expert=[245, 278, 267, 254, 261, 270, 272, 231, 249, 261, 259, 249, 263, 273, 244, 267, 262, 259, 261, 244, 264, 263, 228, 249, 283, 263, 258, 271, 284, 240, 227, 234, 290, 283, 224, 267, 240, 259, 262, 236, 227, 252, 251, 257, 222, 254, 267, 238, 242, 291, 281, 249, 237, 251, 266, 261, 240, 264, 254, 260, 267, 241, 234, 260, 228, 274, 256, 243, 289, 264, 245, 254, 249, 265, 250, 250, 273, 287, 258, 274, 246, 251, 250, 266, 249, 283, 246, 255, 256, 257, 265, 238, 237, 247, 236, 254, 263, 271, 243, 261, 246, 255, 287, 268, 256, 251, 250, 246, 249, 232, 260, 242, 240, 263, 250, 245, 254, 268, 244, 263, 278, 283, 270, 246, 261, 249, 267, 252] payload_rows_per_dst=[5482, 5274, 5394, 5494, 5592, 5430, 5564, 5512] dup_rows_per_dst=[2778, 2518, 2664, 2718, 2906, 2568, 2760, 2882] combine_rows_per_src=[5482, 5274, 5394, 5494, 5592, 5430, 5564, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38260 dup_rows=21724 dup_rows_remote=19060 dedup_ratio=0.331482 router_tokens_per_expert=[248, 259, 286, 234, 256, 231, 271, 283, 253, 234, 266, 239, 274, 261, 252, 256, 266, 243, 219, 267, 232, 261, 287, 268, 259, 257, 258, 262, 241, 244, 292, 252, 262, 261, 233, 241, 252, 239, 275, 259, 271, 268, 237, 275, 256, 256, 271, 259, 244, 240, 250, 244, 256, 263, 247, 258, 265, 250, 275, 250, 266, 238, 287, 251, 209, 236, 247, 270, 275, 255, 266, 267, 273, 286, 249, 286, 239, 239, 263, 233, 265, 266, 256, 242, 246, 240, 249, 241, 247, 276, 262, 272, 259, 252, 288, 273, 254, 260, 251, 260, 211, 257, 256, 280, 256, 275, 235, 235, 268, 242, 261, 259, 278, 260, 254, 243, 250, 268, 246, 238, 254, 263, 230, 257, 241, 247, 285, 257] payload_rows_per_dst=[5514, 5552, 5456, 5444, 5484, 5454, 5480, 5428] dup_rows_per_dst=[2692, 2664, 2774, 2724, 2702, 2814, 2640, 2714] combine_rows_per_src=[5514, 5552, 5456, 5444, 5484, 5454, 5480, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38148 dup_rows=21912 dup_rows_remote=19192 dedup_ratio=0.334351 router_tokens_per_expert=[256, 248, 277, 264, 253, 243, 242, 264, 248, 274, 238, 256, 276, 278, 244, 252, 261, 266, 267, 259, 252, 288, 225, 286, 240, 259, 237, 259, 243, 266, 264, 232, 248, 255, 273, 230, 263, 244, 282, 233, 273, 247, 269, 249, 249, 256, 232, 274, 223, 243, 275, 257, 265, 259, 249, 248, 238, 271, 294, 269, 260, 289, 252, 256, 245, 229, 272, 237, 258, 248, 242, 234, 253, 260, 232, 245, 276, 231, 265, 264, 285, 252, 229, 258, 292, 229, 249, 264, 269, 282, 266, 225, 233, 249, 255, 261, 240, 270, 229, 267, 259, 249, 257, 253, 282, 253, 245, 257, 276, 266, 271, 241, 268, 241, 221, 248, 262, 260, 287, 255, 244, 271, 263, 256, 254, 262, 272, 258] payload_rows_per_dst=[5482, 5412, 5466, 5504, 5328, 5476, 5484, 5472] dup_rows_per_dst=[2744, 2796, 2688, 2792, 2654, 2720, 2746, 2772] combine_rows_per_src=[5482, 5412, 5466, 5504, 5328, 5476, 5484, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229496 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157188216 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314146936 weight_movement_bytes=37748736 total_nvlink_bytes=351895672 weight_to_token_ratio=0.120163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229568 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950720 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671872 weight_movement_bytes=18874368 total_nvlink_bytes=332546240 weight_to_token_ratio=0.060172 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=229624 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156205304 ingress_bytes=174850048 link_bytes=174850048 token_bytes=312180984 weight_movement_bytes=18874368 total_nvlink_bytes=331055352 weight_to_token_ratio=0.060460 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675327128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=230512 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157197424 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314164336 weight_movement_bytes=18874368 total_nvlink_bytes=333038704 weight_to_token_ratio=0.060078 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228568 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156818648 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313408728 weight_movement_bytes=18874368 total_nvlink_bytes=332283096 weight_to_token_ratio=0.060223 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229280 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156942240 ingress_bytes=232210432 link_bytes=232210432 token_bytes=313655200 weight_movement_bytes=75497472 total_nvlink_bytes=389152672 weight_to_token_ratio=0.240702 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224926105e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229360 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156483568 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312737776 weight_movement_bytes=37748736 total_nvlink_bytes=350486512 weight_to_token_ratio=0.120704 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43994 payload_rows_remote=38548 dup_rows=21542 dup_rows_remote=18702 dedup_ratio=0.328705 router_tokens_per_expert=[268, 243, 257, 236, 257, 281, 272, 254, 265, 261, 225, 260, 254, 227, 258, 261, 261, 251, 235, 243, 261, 262, 249, 253, 225, 230, 294, 280, 257, 268, 253, 245, 230, 269, 265, 266, 256, 260, 238, 266, 249, 262, 271, 282, 250, 239, 270, 244, 268, 256, 245, 265, 241, 273, 230, 298, 271, 262, 271, 257, 256, 263, 231, 256, 257, 265, 242, 255, 223, 260, 259, 253, 257, 261, 258, 243, 253, 255, 258, 251, 240, 243, 246, 236, 265, 237, 270, 266, 255, 269, 263, 276, 264, 227, 254, 273, 256, 285, 270, 242, 266, 239, 256, 273, 263, 274, 247, 265, 269, 269, 279, 255, 238, 281, 239, 251, 248, 239, 249, 258, 217, 244, 267, 266, 252, 258, 251, 262] payload_rows_per_dst=[5496, 5504, 5506, 5446, 5506, 5504, 5630, 5402] dup_rows_per_dst=[2662, 2630, 2728, 2840, 2594, 2664, 2786, 2638] combine_rows_per_src=[5496, 5504, 5506, 5446, 5506, 5504, 5630, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38306 dup_rows=21824 dup_rows_remote=19118 dedup_ratio=0.333008 router_tokens_per_expert=[258, 234, 239, 245, 275, 258, 235, 262, 262, 264, 273, 272, 242, 269, 252, 286, 257, 231, 248, 279, 251, 250, 277, 233, 259, 259, 266, 264, 234, 254, 250, 257, 217, 250, 241, 242, 254, 244, 247, 245, 268, 244, 283, 253, 260, 245, 277, 286, 224, 239, 257, 247, 283, 283, 237, 251, 264, 268, 223, 290, 259, 264, 241, 221, 267, 264, 246, 261, 249, 245, 245, 242, 233, 255, 256, 275, 252, 241, 254, 262, 257, 230, 248, 256, 271, 268, 242, 243, 270, 266, 260, 268, 246, 275, 265, 248, 249, 262, 267, 252, 245, 255, 270, 228, 268, 278, 296, 273, 241, 252, 272, 262, 247, 257, 256, 285, 244, 227, 267, 244, 266, 256, 276, 259, 250, 271, 269, 262] payload_rows_per_dst=[5510, 5432, 5406, 5418, 5448, 5474, 5498, 5526] dup_rows_per_dst=[2742, 2706, 2706, 2684, 2646, 2752, 2842, 2746] combine_rows_per_src=[5510, 5432, 5406, 5418, 5448, 5474, 5498, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38052 dup_rows=21942 dup_rows_remote=19112 dedup_ratio=0.334808 router_tokens_per_expert=[242, 247, 243, 244, 252, 249, 246, 251, 239, 249, 252, 257, 238, 257, 295, 253, 258, 257, 237, 255, 276, 259, 243, 275, 260, 264, 267, 242, 271, 226, 267, 291, 255, 283, 263, 232, 259, 267, 276, 266, 241, 248, 245, 247, 249, 249, 228, 281, 268, 230, 260, 239, 267, 248, 261, 275, 258, 251, 278, 268, 273, 277, 229, 263, 242, 260, 254, 263, 270, 274, 260, 258, 259, 230, 259, 258, 254, 271, 250, 229, 245, 268, 280, 256, 254, 266, 256, 279, 271, 261, 257, 238, 281, 286, 240, 254, 223, 259, 224, 232, 257, 283, 251, 255, 259, 248, 280, 292, 278, 251, 236, 277, 242, 274, 219, 252, 282, 215, 234, 253, 247, 293, 239, 235, 230, 269, 234, 266] payload_rows_per_dst=[5542, 5602, 5254, 5444, 5432, 5550, 5562, 5208] dup_rows_per_dst=[2830, 2900, 2582, 2844, 2750, 2834, 2844, 2358] combine_rows_per_src=[5542, 5602, 5254, 5444, 5432, 5550, 5562, 5208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38302 dup_rows=21836 dup_rows_remote=19156 dedup_ratio=0.333191 router_tokens_per_expert=[271, 263, 249, 268, 233, 249, 240, 275, 263, 284, 243, 267, 262, 276, 247, 252, 248, 235, 259, 250, 260, 235, 287, 275, 250, 235, 260, 235, 246, 271, 261, 252, 270, 225, 247, 254, 263, 266, 288, 266, 252, 279, 247, 278, 286, 245, 267, 278, 256, 254, 266, 255, 260, 228, 244, 234, 271, 252, 235, 256, 268, 225, 258, 268, 249, 257, 245, 251, 267, 264, 249, 243, 246, 261, 262, 238, 245, 249, 246, 267, 266, 250, 254, 252, 251, 275, 255, 274, 221, 247, 298, 272, 254, 252, 261, 251, 249, 240, 261, 248, 228, 294, 244, 247, 258, 264, 252, 254, 247, 226, 252, 283, 251, 254, 258, 278, 265, 267, 261, 241, 254, 267, 265, 277, 271, 241, 234, 223] payload_rows_per_dst=[5464, 5458, 5498, 5454, 5398, 5514, 5406, 5508] dup_rows_per_dst=[2820, 2660, 2924, 2606, 2680, 2752, 2688, 2706] combine_rows_per_src=[5464, 5458, 5498, 5454, 5398, 5514, 5406, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38100 dup_rows=21992 dup_rows_remote=19186 dedup_ratio=0.335571 router_tokens_per_expert=[234, 262, 239, 261, 245, 243, 273, 259, 272, 241, 256, 250, 223, 251, 249, 256, 251, 271, 237, 247, 255, 253, 227, 260, 240, 242, 268, 239, 261, 284, 261, 259, 273, 242, 261, 250, 238, 259, 243, 276, 258, 248, 240, 246, 266, 289, 254, 261, 269, 264, 270, 271, 266, 265, 243, 248, 279, 245, 260, 234, 277, 245, 279, 241, 267, 262, 261, 290, 261, 246, 257, 253, 260, 252, 256, 240, 253, 247, 248, 241, 245, 245, 264, 264, 241, 252, 256, 253, 251, 267, 263, 260, 269, 263, 266, 266, 257, 267, 253, 253, 251, 270, 285, 247, 255, 255, 266, 262, 259, 240, 270, 235, 271, 242, 265, 275, 255, 248, 244, 242, 265, 247, 279, 237, 249, 249, 262, 265] payload_rows_per_dst=[5318, 5376, 5428, 5520, 5472, 5468, 5444, 5518] dup_rows_per_dst=[2710, 2734, 2780, 2792, 2716, 2782, 2806, 2672] combine_rows_per_src=[5318, 5376, 5428, 5520, 5472, 5468, 5444, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38268 dup_rows=21748 dup_rows_remote=18896 dedup_ratio=0.331848 router_tokens_per_expert=[275, 242, 262, 278, 261, 274, 252, 234, 217, 219, 274, 265, 258, 262, 254, 264, 254, 285, 245, 271, 255, 288, 294, 256, 260, 254, 270, 263, 230, 278, 261, 249, 269, 255, 231, 260, 240, 220, 265, 270, 246, 242, 258, 240, 270, 240, 257, 251, 256, 254, 261, 246, 248, 244, 260, 264, 267, 252, 269, 286, 255, 277, 251, 237, 272, 250, 283, 250, 267, 261, 239, 249, 247, 262, 247, 254, 265, 258, 268, 255, 249, 280, 242, 230, 257, 237, 263, 231, 262, 247, 228, 277, 262, 248, 259, 244, 279, 262, 256, 238, 256, 250, 249, 252, 263, 288, 212, 252, 237, 236, 242, 261, 266, 270, 237, 272, 259, 238, 267, 237, 238, 284, 266, 254, 259, 274, 269, 257] payload_rows_per_dst=[5478, 5550, 5380, 5494, 5484, 5428, 5454, 5520] dup_rows_per_dst=[2652, 2876, 2560, 2682, 2770, 2604, 2752, 2852] combine_rows_per_src=[5478, 5550, 5380, 5494, 5484, 5428, 5454, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43426 payload_rows_remote=38028 dup_rows=22110 dup_rows_remote=19398 dedup_ratio=0.337372 router_tokens_per_expert=[257, 273, 275, 238, 255, 269, 253, 255, 246, 249, 242, 254, 247, 245, 255, 257, 242, 241, 236, 234, 280, 226, 269, 253, 238, 242, 253, 259, 259, 274, 281, 268, 250, 245, 235, 231, 273, 227, 260, 255, 231, 270, 263, 250, 235, 281, 235, 262, 253, 261, 260, 251, 265, 260, 236, 238, 283, 277, 261, 277, 304, 246, 255, 255, 292, 245, 253, 268, 267, 262, 257, 244, 251, 233, 250, 238, 255, 255, 249, 238, 231, 288, 245, 257, 215, 249, 250, 252, 282, 261, 267, 249, 253, 262, 265, 253, 251, 226, 251, 258, 265, 256, 243, 260, 243, 275, 259, 281, 275, 232, 241, 274, 239, 248, 272, 257, 271, 278, 246, 293, 288, 269, 248, 261, 280, 262, 271, 249] payload_rows_per_dst=[5468, 5398, 5364, 5464, 5376, 5374, 5390, 5592] dup_rows_per_dst=[2672, 2712, 2642, 2900, 2738, 2784, 2790, 2872] combine_rows_per_src=[5468, 5398, 5364, 5464, 5376, 5374, 5390, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38320 dup_rows=21646 dup_rows_remote=18894 dedup_ratio=0.330292 router_tokens_per_expert=[249, 268, 249, 245, 263, 285, 256, 266, 246, 246, 260, 265, 261, 234, 261, 253, 255, 247, 260, 235, 246, 253, 248, 247, 258, 248, 237, 260, 250, 269, 229, 244, 281, 252, 260, 257, 248, 257, 276, 261, 267, 246, 260, 263, 254, 270, 267, 255, 254, 276, 281, 241, 263, 277, 250, 247, 257, 276, 259, 255, 254, 240, 235, 226, 281, 273, 270, 280, 260, 250, 266, 214, 228, 263, 254, 256, 285, 259, 241, 240, 282, 263, 265, 252, 259, 255, 255, 249, 273, 251, 261, 290, 266, 261, 237, 242, 246, 247, 233, 254, 249, 244, 241, 264, 247, 256, 244, 265, 268, 254, 243, 252, 265, 224, 246, 279, 278, 249, 253, 274, 266, 285, 258, 245, 233, 261, 249, 257] payload_rows_per_dst=[5526, 5308, 5500, 5504, 5512, 5570, 5444, 5526] dup_rows_per_dst=[2688, 2664, 2848, 2678, 2728, 2752, 2570, 2718] combine_rows_per_src=[5526, 5308, 5500, 5504, 5512, 5570, 5444, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157892608 dispatch_meta_bytes=229000 combine_payload_bytes=157892608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158121608 ingress_bytes=195641344 link_bytes=195641344 token_bytes=316014216 weight_movement_bytes=37748736 total_nvlink_bytes=353762952 weight_to_token_ratio=0.119453 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874539097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229696 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157131072 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314032448 weight_movement_bytes=37748736 total_nvlink_bytes=351781184 weight_to_token_ratio=0.120206 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229832 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114824 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999816 weight_movement_bytes=18874368 total_nvlink_bytes=332874184 weight_to_token_ratio=0.060109 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=229144 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156286744 ingress_bytes=193806336 link_bytes=193806336 token_bytes=312344344 weight_movement_bytes=37748736 total_nvlink_bytes=350093080 weight_to_token_ratio=0.120856 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856956953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228656 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156974384 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720112 weight_movement_bytes=37748736 total_nvlink_bytes=351468848 weight_to_token_ratio=0.120326 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155762688 dispatch_meta_bytes=229704 combine_payload_bytes=155762688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155992392 ingress_bytes=174637056 link_bytes=174637056 token_bytes=311755080 weight_movement_bytes=18874368 total_nvlink_bytes=330629448 weight_to_token_ratio=0.060542 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673286344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228856 combine_payload_bytes=156958720 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157187576 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314146296 weight_movement_bytes=37748736 total_nvlink_bytes=351895032 weight_to_token_ratio=0.120163 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155860992 dispatch_meta_bytes=228656 combine_payload_bytes=155860992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156089648 ingress_bytes=174735360 link_bytes=174735360 token_bytes=311950640 weight_movement_bytes=18874368 total_nvlink_bytes=330825008 weight_to_token_ratio=0.060504 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674228244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38402 dup_rows=21690 dup_rows_remote=18938 dedup_ratio=0.330963 router_tokens_per_expert=[266, 259, 252, 248, 266, 255, 267, 249, 251, 227, 245, 259, 257, 238, 231, 261, 269, 267, 287, 275, 265, 264, 288, 266, 246, 260, 276, 252, 258, 246, 252, 256, 261, 255, 253, 284, 234, 268, 256, 259, 250, 258, 242, 259, 269, 248, 232, 284, 243, 250, 290, 239, 267, 260, 251, 255, 262, 272, 253, 259, 241, 227, 274, 255, 271, 265, 260, 238, 261, 258, 252, 264, 251, 230, 233, 248, 269, 255, 244, 264, 239, 257, 260, 242, 242, 268, 261, 269, 257, 241, 253, 250, 256, 242, 281, 271, 251, 246, 270, 258, 266, 245, 261, 262, 264, 249, 266, 264, 251, 265, 240, 229, 262, 243, 260, 258, 237, 227, 253, 252, 260, 255, 262, 267, 273, 277, 236, 239] payload_rows_per_dst=[5426, 5524, 5480, 5444, 5434, 5524, 5542, 5472] dup_rows_per_dst=[2636, 2930, 2744, 2752, 2692, 2654, 2632, 2650] combine_rows_per_src=[5426, 5524, 5480, 5444, 5434, 5524, 5542, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38246 dup_rows=21824 dup_rows_remote=19128 dedup_ratio=0.333008 router_tokens_per_expert=[252, 238, 274, 248, 258, 266, 256, 245, 274, 242, 239, 286, 284, 230, 268, 274, 231, 250, 239, 260, 260, 257, 244, 252, 241, 265, 260, 252, 253, 235, 245, 242, 256, 249, 258, 257, 256, 246, 245, 232, 274, 265, 239, 268, 273, 247, 244, 272, 267, 249, 262, 254, 273, 234, 259, 276, 235, 268, 252, 241, 243, 246, 279, 247, 271, 227, 237, 252, 285, 256, 249, 261, 290, 262, 253, 283, 252, 250, 224, 244, 261, 276, 279, 296, 251, 270, 249, 264, 268, 250, 261, 253, 262, 262, 242, 245, 289, 251, 260, 260, 275, 273, 276, 259, 276, 248, 256, 250, 230, 244, 251, 226, 241, 255, 264, 236, 252, 251, 273, 255, 260, 271, 272, 242, 258, 233, 257, 253] payload_rows_per_dst=[5534, 5358, 5466, 5426, 5478, 5520, 5484, 5446] dup_rows_per_dst=[2734, 2614, 2696, 2744, 2714, 2858, 2764, 2700] combine_rows_per_src=[5534, 5358, 5466, 5426, 5478, 5520, 5484, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38098 dup_rows=21820 dup_rows_remote=19046 dedup_ratio=0.332947 router_tokens_per_expert=[240, 247, 259, 245, 254, 252, 290, 240, 250, 253, 266, 305, 255, 255, 273, 240, 263, 294, 274, 268, 254, 238, 280, 249, 266, 244, 249, 222, 254, 247, 265, 295, 264, 245, 276, 269, 247, 266, 242, 250, 245, 290, 241, 251, 249, 288, 242, 241, 235, 264, 271, 270, 250, 248, 253, 253, 257, 288, 259, 250, 231, 266, 220, 259, 279, 249, 266, 263, 272, 274, 253, 278, 265, 258, 282, 260, 256, 247, 232, 262, 251, 255, 301, 240, 270, 241, 254, 282, 244, 267, 252, 252, 244, 263, 240, 241, 252, 252, 236, 268, 228, 257, 257, 260, 225, 260, 256, 250, 215, 264, 243, 241, 253, 249, 265, 254, 274, 221, 259, 207, 286, 259, 260, 253, 256, 221, 259, 269] payload_rows_per_dst=[5430, 5540, 5390, 5466, 5618, 5518, 5312, 5442] dup_rows_per_dst=[2818, 2784, 2822, 2682, 2774, 2676, 2616, 2648] combine_rows_per_src=[5430, 5540, 5390, 5466, 5618, 5518, 5312, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229360 combine_payload_bytes=157294592 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157523952 ingress_bytes=213917696 link_bytes=213917696 token_bytes=314818544 weight_movement_bytes=56623104 total_nvlink_bytes=371441648 weight_to_token_ratio=0.179859 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049654109e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38380 dup_rows=21728 dup_rows_remote=18980 dedup_ratio=0.331543 router_tokens_per_expert=[252, 266, 273, 278, 231, 296, 243, 271, 250, 254, 256, 245, 276, 249, 275, 243, 252, 234, 257, 253, 277, 243, 240, 256, 296, 255, 283, 243, 255, 275, 259, 235, 240, 270, 250, 263, 243, 266, 238, 232, 268, 261, 284, 243, 251, 232, 264, 257, 254, 286, 258, 276, 235, 261, 241, 254, 263, 253, 259, 269, 241, 273, 264, 237, 239, 243, 245, 253, 262, 278, 258, 269, 248, 239, 255, 270, 281, 238, 254, 268, 253, 255, 250, 243, 262, 255, 260, 270, 285, 242, 250, 263, 275, 266, 227, 251, 250, 249, 273, 225, 274, 250, 282, 230, 271, 251, 257, 250, 287, 230, 240, 269, 271, 239, 257, 275, 252, 249, 227, 226, 255, 265, 249, 263, 235, 259, 246, 248] payload_rows_per_dst=[5614, 5444, 5490, 5492, 5530, 5454, 5428, 5356] dup_rows_per_dst=[2702, 2782, 2634, 2756, 2670, 2760, 2748, 2676] combine_rows_per_src=[5614, 5444, 5490, 5492, 5530, 5454, 5428, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38082 dup_rows=21836 dup_rows_remote=18978 dedup_ratio=0.333191 router_tokens_per_expert=[274, 256, 241, 256, 245, 249, 233, 250, 278, 252, 273, 256, 239, 273, 267, 243, 243, 251, 242, 305, 266, 254, 251, 315, 227, 229, 245, 247, 261, 272, 252, 235, 255, 245, 242, 262, 242, 245, 248, 271, 243, 251, 262, 258, 252, 254, 224, 252, 268, 257, 264, 231, 275, 250, 256, 251, 261, 279, 232, 252, 251, 272, 239, 295, 239, 264, 261, 246, 268, 265, 243, 270, 272, 257, 285, 250, 266, 263, 229, 257, 273, 263, 277, 252, 267, 266, 265, 261, 268, 244, 255, 278, 243, 269, 239, 243, 260, 248, 263, 258, 251, 269, 243, 290, 276, 219, 266, 263, 275, 270, 263, 251, 240, 258, 258, 247, 259, 257, 239, 241, 252, 263, 269, 237, 248, 253, 244, 221] payload_rows_per_dst=[5506, 5196, 5396, 5380, 5534, 5516, 5554, 5618] dup_rows_per_dst=[2664, 2360, 2616, 2764, 2744, 2924, 2906, 2858] combine_rows_per_src=[5506, 5196, 5396, 5380, 5534, 5516, 5554, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38284 dup_rows=21772 dup_rows_remote=18916 dedup_ratio=0.332214 router_tokens_per_expert=[239, 264, 233, 235, 239, 263, 236, 243, 233, 262, 232, 243, 276, 254, 256, 253, 272, 277, 269, 255, 276, 253, 236, 255, 255, 238, 234, 264, 276, 265, 277, 266, 258, 228, 239, 270, 267, 255, 238, 235, 261, 220, 244, 278, 262, 248, 234, 288, 273, 273, 253, 244, 276, 278, 265, 270, 268, 259, 265, 252, 251, 246, 262, 244, 228, 257, 271, 263, 267, 295, 270, 240, 246, 264, 217, 287, 252, 244, 258, 256, 274, 244, 259, 283, 246, 276, 247, 262, 236, 254, 262, 261, 274, 260, 271, 255, 247, 264, 242, 227, 238, 230, 272, 253, 255, 253, 226, 273, 258, 231, 238, 288, 243, 258, 274, 259, 281, 283, 253, 269, 254, 237, 265, 242, 267, 244, 259, 273] payload_rows_per_dst=[5350, 5480, 5378, 5506, 5528, 5538, 5440, 5544] dup_rows_per_dst=[2572, 2856, 2672, 2852, 2702, 2790, 2550, 2778] combine_rows_per_src=[5350, 5480, 5378, 5506, 5528, 5538, 5440, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229496 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885112 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540728 weight_movement_bytes=18874368 total_nvlink_bytes=332415096 weight_to_token_ratio=0.060198 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38074 dup_rows=21992 dup_rows_remote=19284 dedup_ratio=0.335571 router_tokens_per_expert=[262, 256, 249, 222, 220, 272, 231, 237, 235, 269, 249, 269, 267, 281, 277, 261, 260, 252, 278, 277, 256, 274, 266, 249, 261, 260, 273, 261, 276, 252, 238, 249, 252, 258, 246, 264, 272, 251, 236, 233, 276, 266, 263, 250, 256, 263, 254, 252, 274, 254, 269, 254, 240, 271, 242, 243, 257, 227, 264, 266, 218, 227, 248, 249, 264, 256, 279, 247, 267, 251, 258, 276, 281, 267, 242, 276, 246, 238, 260, 247, 275, 249, 238, 216, 267, 253, 261, 252, 267, 257, 300, 245, 275, 242, 246, 246, 237, 271, 256, 254, 258, 264, 250, 247, 255, 286, 245, 246, 267, 259, 237, 253, 252, 278, 257, 252, 238, 250, 246, 236, 294, 259, 255, 249, 244, 274, 245, 276] payload_rows_per_dst=[5532, 5460, 5474, 5322, 5410, 5470, 5436, 5440] dup_rows_per_dst=[2582, 2904, 2710, 2684, 2900, 2708, 2734, 2770] combine_rows_per_src=[5532, 5460, 5474, 5322, 5410, 5470, 5436, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228576 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156277984 ingress_bytes=193798144 link_bytes=193798144 token_bytes=312327392 weight_movement_bytes=37748736 total_nvlink_bytes=350076128 weight_to_token_ratio=0.120863 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856878461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229440 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157433920 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638400 weight_movement_bytes=18874368 total_nvlink_bytes=333512768 weight_to_token_ratio=0.059987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=228240 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156212112 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312195984 weight_movement_bytes=18874368 total_nvlink_bytes=331070352 weight_to_token_ratio=0.060457 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228800 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157040064 ingress_bytes=213434368 link_bytes=213434368 token_bytes=313851328 weight_movement_bytes=56623104 total_nvlink_bytes=370474432 weight_to_token_ratio=0.180414 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045023098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38124 dup_rows=21900 dup_rows_remote=19072 dedup_ratio=0.334167 router_tokens_per_expert=[271, 290, 257, 262, 234, 255, 277, 280, 291, 235, 239, 255, 263, 249, 270, 242, 221, 246, 276, 280, 299, 255, 257, 241, 248, 268, 279, 241, 243, 240, 279, 254, 265, 279, 264, 262, 246, 234, 219, 233, 246, 248, 257, 266, 258, 257, 235, 253, 261, 247, 256, 284, 253, 280, 253, 288, 259, 259, 244, 272, 238, 263, 264, 270, 268, 272, 245, 243, 259, 249, 270, 253, 260, 251, 255, 240, 255, 259, 254, 251, 252, 207, 248, 253, 257, 279, 229, 252, 247, 229, 266, 280, 258, 245, 247, 257, 260, 267, 255, 246, 263, 249, 248, 259, 249, 256, 264, 245, 226, 250, 264, 269, 240, 274, 254, 255, 239, 234, 281, 258, 263, 274, 261, 269, 259, 249, 254, 234] payload_rows_per_dst=[5512, 5476, 5462, 5522, 5458, 5382, 5426, 5398] dup_rows_per_dst=[2828, 2778, 2772, 2752, 2710, 2630, 2714, 2716] combine_rows_per_src=[5512, 5476, 5462, 5522, 5458, 5382, 5426, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=229432 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156180536 ingress_bytes=193699840 link_bytes=193699840 token_bytes=312131640 weight_movement_bytes=37748736 total_nvlink_bytes=349880376 weight_to_token_ratio=0.120939 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855936561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38236 dup_rows=21876 dup_rows_remote=19206 dedup_ratio=0.333801 router_tokens_per_expert=[283, 262, 247, 229, 254, 243, 266, 277, 267, 235, 252, 268, 257, 252, 260, 235, 287, 250, 273, 252, 256, 279, 262, 238, 244, 254, 226, 272, 226, 233, 235, 272, 301, 258, 267, 249, 254, 271, 241, 239, 252, 267, 246, 246, 271, 253, 273, 245, 234, 240, 269, 247, 259, 260, 242, 273, 238, 246, 291, 245, 266, 241, 244, 252, 252, 242, 258, 245, 242, 267, 272, 253, 274, 269, 259, 242, 279, 256, 228, 257, 242, 232, 272, 272, 245, 228, 252, 251, 258, 272, 240, 263, 271, 289, 222, 283, 273, 280, 255, 288, 259, 264, 253, 244, 260, 228, 250, 262, 254, 253, 240, 260, 264, 277, 234, 242, 243, 260, 241, 264, 246, 291, 278, 239, 254, 280, 279, 240] payload_rows_per_dst=[5396, 5432, 5468, 5424, 5410, 5534, 5506, 5490] dup_rows_per_dst=[2778, 2686, 2798, 2670, 2780, 2650, 2740, 2774] combine_rows_per_src=[5396, 5432, 5468, 5424, 5410, 5534, 5506, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228784 combine_payload_bytes=156155904 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156384688 ingress_bytes=193904640 link_bytes=193904640 token_bytes=312540592 weight_movement_bytes=37748736 total_nvlink_bytes=350289328 weight_to_token_ratio=0.120780 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857898853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38428 dup_rows=21634 dup_rows_remote=18956 dedup_ratio=0.330109 router_tokens_per_expert=[279, 270, 275, 267, 260, 235, 261, 269, 251, 269, 264, 238, 238, 218, 256, 257, 268, 248, 244, 243, 255, 259, 244, 246, 238, 241, 258, 258, 236, 261, 256, 265, 252, 272, 253, 254, 241, 243, 259, 240, 245, 277, 257, 272, 250, 246, 269, 246, 224, 256, 249, 255, 253, 272, 284, 262, 238, 298, 245, 257, 229, 243, 267, 270, 241, 270, 267, 247, 220, 259, 253, 269, 230, 263, 284, 265, 250, 267, 236, 264, 265, 271, 260, 289, 280, 255, 240, 259, 253, 257, 234, 261, 256, 276, 239, 247, 238, 273, 265, 257, 250, 260, 268, 238, 257, 254, 280, 253, 244, 255, 260, 253, 256, 267, 243, 257, 264, 260, 250, 291, 263, 264, 276, 255, 244, 249, 222, 270] payload_rows_per_dst=[5510, 5358, 5474, 5524, 5514, 5508, 5462, 5552] dup_rows_per_dst=[2704, 2682, 2678, 2680, 2656, 2776, 2748, 2710] combine_rows_per_src=[5510, 5358, 5474, 5524, 5514, 5508, 5462, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38344 dup_rows=21804 dup_rows_remote=19106 dedup_ratio=0.332703 router_tokens_per_expert=[272, 236, 260, 242, 249, 247, 268, 252, 249, 242, 262, 237, 282, 258, 283, 289, 264, 263, 247, 274, 272, 254, 269, 242, 249, 260, 251, 253, 240, 230, 252, 260, 253, 249, 262, 266, 219, 233, 277, 228, 261, 285, 274, 242, 243, 255, 265, 262, 261, 252, 239, 227, 294, 271, 242, 250, 252, 258, 269, 244, 260, 272, 224, 243, 234, 256, 239, 274, 233, 237, 258, 267, 230, 260, 272, 246, 247, 261, 266, 263, 267, 258, 277, 253, 254, 257, 237, 249, 243, 272, 285, 275, 268, 244, 258, 271, 246, 247, 254, 247, 261, 230, 261, 263, 248, 248, 258, 273, 275, 247, 272, 251, 277, 268, 221, 271, 233, 285, 280, 239, 244, 239, 250, 269, 271, 249, 264, 276] payload_rows_per_dst=[5490, 5410, 5422, 5406, 5388, 5584, 5484, 5548] dup_rows_per_dst=[2766, 2750, 2726, 2710, 2698, 2752, 2678, 2724] combine_rows_per_src=[5490, 5410, 5422, 5406, 5388, 5584, 5484, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229768 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156844424 ingress_bytes=194363392 link_bytes=194363392 token_bytes=313459080 weight_movement_bytes=37748736 total_nvlink_bytes=351207816 weight_to_token_ratio=0.120426 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862294389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38274 dup_rows=21740 dup_rows_remote=18926 dedup_ratio=0.331726 router_tokens_per_expert=[245, 252, 249, 241, 265, 259, 274, 243, 240, 243, 244, 253, 251, 244, 248, 265, 278, 257, 291, 274, 265, 251, 259, 250, 268, 261, 236, 239, 244, 237, 238, 254, 255, 262, 274, 259, 251, 244, 259, 254, 255, 239, 270, 270, 251, 257, 268, 266, 274, 264, 238, 251, 266, 254, 253, 257, 273, 271, 251, 255, 260, 260, 258, 265, 271, 256, 265, 245, 256, 237, 279, 256, 250, 242, 259, 266, 261, 229, 275, 250, 215, 277, 247, 262, 251, 269, 239, 241, 255, 239, 246, 248, 248, 250, 247, 264, 294, 252, 270, 267, 256, 248, 256, 268, 245, 245, 281, 259, 250, 254, 274, 249, 270, 253, 262, 250, 281, 259, 254, 259, 269, 261, 230, 251, 249, 233, 248, 274] payload_rows_per_dst=[5370, 5512, 5494, 5506, 5534, 5332, 5522, 5526] dup_rows_per_dst=[2662, 2692, 2774, 2794, 2660, 2664, 2814, 2680] combine_rows_per_src=[5370, 5512, 5494, 5506, 5534, 5332, 5522, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229536 combine_payload_bytes=157401088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157630624 ingress_bytes=195149824 link_bytes=195149824 token_bytes=315031712 weight_movement_bytes=37748736 total_nvlink_bytes=352780448 weight_to_token_ratio=0.119825 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869829594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38324 dup_rows=21818 dup_rows_remote=19108 dedup_ratio=0.332916 router_tokens_per_expert=[248, 248, 241, 238, 254, 252, 253, 254, 266, 248, 256, 252, 285, 218, 234, 263, 246, 223, 263, 247, 257, 259, 246, 261, 257, 220, 240, 255, 263, 284, 279, 254, 260, 254, 240, 269, 251, 264, 227, 246, 258, 260, 251, 284, 256, 259, 288, 281, 252, 258, 265, 235, 248, 272, 281, 274, 235, 260, 259, 263, 249, 278, 241, 262, 261, 283, 266, 250, 239, 233, 246, 247, 267, 273, 254, 263, 248, 277, 267, 248, 233, 312, 275, 278, 254, 262, 283, 259, 248, 280, 252, 273, 257, 254, 239, 248, 251, 259, 286, 239, 243, 262, 251, 241, 259, 241, 250, 264, 256, 236, 257, 248, 222, 259, 266, 264, 256, 259, 248, 252, 253, 237, 262, 285, 246, 251, 242, 250] payload_rows_per_dst=[5432, 5476, 5498, 5442, 5456, 5542, 5478, 5394] dup_rows_per_dst=[2588, 2632, 2696, 2788, 2788, 2906, 2710, 2710] combine_rows_per_src=[5432, 5476, 5498, 5442, 5456, 5542, 5478, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38194 dup_rows=21888 dup_rows_remote=19228 dedup_ratio=0.333984 router_tokens_per_expert=[270, 249, 243, 269, 234, 270, 267, 255, 278, 253, 227, 242, 274, 250, 236, 222, 253, 247, 285, 277, 274, 262, 236, 251, 272, 281, 249, 230, 251, 224, 228, 237, 234, 247, 246, 250, 246, 250, 255, 244, 249, 285, 244, 239, 266, 251, 248, 268, 244, 258, 258, 276, 260, 282, 251, 268, 246, 266, 259, 244, 259, 242, 267, 246, 275, 244, 253, 262, 292, 256, 258, 259, 260, 263, 236, 259, 249, 284, 283, 235, 247, 255, 283, 248, 264, 264, 264, 282, 252, 259, 257, 273, 240, 232, 272, 247, 251, 267, 225, 283, 266, 219, 247, 246, 279, 287, 260, 265, 259, 250, 266, 249, 252, 246, 282, 249, 266, 274, 263, 244, 275, 257, 243, 253, 273, 243, 228, 250] payload_rows_per_dst=[5452, 5454, 5398, 5478, 5502, 5418, 5468, 5478] dup_rows_per_dst=[2626, 2660, 2646, 2774, 2834, 2860, 2770, 2718] combine_rows_per_src=[5452, 5454, 5398, 5478, 5502, 5418, 5468, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38206 dup_rows=21912 dup_rows_remote=19202 dedup_ratio=0.334351 router_tokens_per_expert=[262, 267, 259, 258, 282, 267, 249, 236, 239, 244, 241, 292, 246, 258, 230, 257, 260, 289, 251, 267, 266, 248, 248, 252, 263, 242, 256, 255, 240, 255, 272, 254, 276, 256, 275, 245, 259, 247, 249, 224, 267, 269, 263, 273, 247, 249, 268, 267, 273, 283, 244, 262, 226, 267, 283, 245, 253, 251, 247, 254, 287, 238, 243, 244, 243, 261, 266, 230, 252, 239, 243, 254, 289, 263, 277, 257, 223, 266, 253, 247, 232, 259, 260, 253, 241, 245, 248, 245, 252, 269, 251, 240, 257, 268, 265, 279, 260, 291, 230, 279, 233, 263, 256, 278, 260, 256, 236, 254, 257, 255, 250, 228, 275, 258, 264, 250, 262, 268, 258, 254, 257, 244, 269, 254, 250, 256, 242, 255] payload_rows_per_dst=[5458, 5452, 5480, 5462, 5464, 5418, 5444, 5446] dup_rows_per_dst=[2716, 2784, 2788, 2738, 2662, 2710, 2728, 2786] combine_rows_per_src=[5458, 5452, 5480, 5462, 5464, 5418, 5444, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229800 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286824 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343848 weight_movement_bytes=18874368 total_nvlink_bytes=333218216 weight_to_token_ratio=0.060044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=228800 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156999104 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769408 weight_movement_bytes=37748736 total_nvlink_bytes=351518144 weight_to_token_ratio=0.120307 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229728 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157204832 ingress_bytes=213598208 link_bytes=213598208 token_bytes=314179936 weight_movement_bytes=56623104 total_nvlink_bytes=370803040 weight_to_token_ratio=0.180225 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046592932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229688 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156672312 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114936 weight_movement_bytes=18874368 total_nvlink_bytes=331989304 weight_to_token_ratio=0.060279 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38208 dup_rows=21828 dup_rows_remote=19062 dedup_ratio=0.333069 router_tokens_per_expert=[249, 255, 248, 258, 251, 264, 265, 267, 287, 251, 243, 267, 258, 262, 268, 240, 274, 257, 271, 262, 248, 269, 238, 254, 243, 240, 254, 249, 262, 251, 264, 261, 267, 258, 269, 249, 264, 247, 265, 264, 245, 269, 227, 268, 281, 235, 234, 264, 270, 261, 250, 244, 259, 255, 262, 257, 245, 248, 266, 252, 266, 248, 249, 271, 251, 249, 260, 233, 259, 283, 276, 251, 277, 241, 249, 251, 258, 234, 249, 244, 257, 257, 249, 245, 280, 242, 231, 256, 255, 275, 293, 229, 244, 256, 269, 278, 253, 240, 254, 259, 251, 249, 263, 248, 252, 276, 272, 274, 242, 248, 261, 246, 229, 247, 260, 244, 255, 224, 267, 256, 249, 258, 281, 266, 240, 284, 256, 244] payload_rows_per_dst=[5500, 5470, 5476, 5440, 5444, 5444, 5490, 5444] dup_rows_per_dst=[2766, 2724, 2802, 2700, 2686, 2788, 2686, 2676] combine_rows_per_src=[5500, 5470, 5476, 5440, 5444, 5444, 5490, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229632 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156721408 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313213184 weight_movement_bytes=37748736 total_nvlink_bytes=350961920 weight_to_token_ratio=0.120521 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38252 dup_rows=21842 dup_rows_remote=19196 dedup_ratio=0.333282 router_tokens_per_expert=[255, 267, 244, 262, 265, 245, 249, 264, 260, 257, 271, 255, 250, 241, 255, 228, 267, 269, 253, 237, 281, 276, 232, 264, 268, 247, 253, 262, 240, 255, 283, 266, 241, 234, 244, 293, 275, 259, 269, 284, 258, 270, 258, 275, 246, 266, 254, 221, 245, 263, 288, 257, 267, 228, 250, 234, 245, 247, 226, 266, 257, 260, 256, 255, 234, 268, 254, 273, 251, 247, 259, 284, 256, 247, 279, 244, 260, 242, 248, 246, 257, 240, 253, 240, 256, 247, 264, 263, 257, 258, 261, 239, 250, 253, 250, 262, 267, 250, 260, 254, 245, 256, 239, 243, 252, 268, 262, 245, 262, 275, 259, 271, 246, 248, 273, 260, 249, 281, 239, 260, 246, 268, 241, 288, 273, 234, 260, 240] payload_rows_per_dst=[5386, 5536, 5496, 5442, 5398, 5444, 5524, 5468] dup_rows_per_dst=[2750, 2770, 2798, 2646, 2786, 2656, 2692, 2744] combine_rows_per_src=[5386, 5536, 5496, 5442, 5398, 5444, 5524, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38194 dup_rows=21872 dup_rows_remote=19060 dedup_ratio=0.333740 router_tokens_per_expert=[268, 254, 270, 257, 265, 271, 245, 252, 271, 264, 260, 260, 240, 232, 254, 260, 225, 254, 272, 280, 244, 245, 273, 283, 248, 235, 266, 260, 244, 240, 249, 220, 253, 257, 253, 256, 234, 261, 261, 276, 237, 257, 265, 288, 286, 258, 254, 245, 230, 260, 246, 264, 262, 221, 252, 230, 266, 267, 218, 241, 285, 289, 279, 234, 252, 247, 208, 271, 254, 291, 236, 250, 265, 271, 272, 267, 236, 237, 274, 259, 264, 259, 288, 248, 272, 258, 258, 259, 248, 270, 286, 260, 276, 263, 254, 251, 282, 246, 232, 250, 239, 262, 260, 254, 251, 252, 246, 269, 226, 264, 253, 294, 223, 263, 260, 226, 243, 282, 258, 271, 280, 229, 256, 247, 234, 280, 235, 251] payload_rows_per_dst=[5454, 5372, 5470, 5456, 5454, 5560, 5432, 5466] dup_rows_per_dst=[2792, 2704, 2812, 2632, 2726, 2868, 2728, 2610] combine_rows_per_src=[5454, 5372, 5470, 5456, 5454, 5560, 5432, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229080 combine_payload_bytes=156499968 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156729048 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313229016 weight_movement_bytes=37748736 total_nvlink_bytes=350977752 weight_to_token_ratio=0.120515 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38358 dup_rows=21694 dup_rows_remote=19036 dedup_ratio=0.331024 router_tokens_per_expert=[269, 249, 235, 250, 257, 252, 286, 282, 267, 270, 261, 250, 257, 277, 261, 243, 262, 245, 270, 267, 244, 270, 274, 225, 220, 243, 258, 282, 261, 259, 252, 253, 280, 283, 247, 276, 252, 244, 267, 268, 258, 256, 239, 275, 255, 243, 251, 257, 257, 294, 277, 247, 252, 267, 247, 250, 256, 245, 263, 239, 250, 261, 237, 263, 238, 265, 255, 244, 269, 235, 255, 253, 250, 258, 270, 242, 258, 241, 244, 294, 266, 232, 238, 226, 271, 245, 276, 261, 232, 258, 244, 244, 241, 275, 256, 251, 259, 256, 266, 261, 241, 247, 262, 244, 233, 253, 262, 248, 269, 247, 272, 263, 241, 274, 250, 251, 278, 251, 253, 268, 268, 243, 277, 249, 246, 249, 232, 261] payload_rows_per_dst=[5536, 5496, 5544, 5420, 5484, 5466, 5444, 5452] dup_rows_per_dst=[2796, 2674, 2758, 2790, 2658, 2566, 2722, 2730] combine_rows_per_src=[5536, 5496, 5544, 5420, 5484, 5466, 5444, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38410 dup_rows=21656 dup_rows_remote=18890 dedup_ratio=0.330444 router_tokens_per_expert=[260, 242, 248, 269, 258, 224, 238, 226, 226, 232, 276, 270, 267, 219, 241, 275, 262, 251, 222, 280, 245, 256, 255, 238, 279, 272, 258, 260, 232, 258, 281, 266, 250, 256, 274, 237, 255, 251, 261, 261, 264, 259, 241, 266, 249, 264, 264, 273, 250, 271, 256, 259, 239, 248, 239, 255, 263, 283, 285, 265, 273, 282, 235, 252, 259, 241, 286, 252, 250, 266, 265, 254, 254, 266, 256, 255, 233, 242, 268, 255, 262, 249, 243, 228, 243, 287, 258, 253, 258, 255, 263, 233, 270, 279, 259, 235, 260, 243, 236, 258, 265, 258, 258, 268, 266, 245, 265, 264, 248, 255, 272, 257, 231, 245, 295, 273, 264, 261, 248, 262, 248, 247, 268, 251, 256, 266, 244, 248] payload_rows_per_dst=[5354, 5516, 5520, 5534, 5456, 5496, 5470, 5534] dup_rows_per_dst=[2588, 2714, 2730, 2776, 2622, 2654, 2766, 2806] combine_rows_per_src=[5354, 5516, 5520, 5534, 5456, 5496, 5470, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229792 combine_payload_bytes=156680192 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156909984 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313590176 weight_movement_bytes=37748736 total_nvlink_bytes=351338912 weight_to_token_ratio=0.120376 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229016 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671640 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114264 weight_movement_bytes=18874368 total_nvlink_bytes=331988632 weight_to_token_ratio=0.060279 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38042 dup_rows=21874 dup_rows_remote=18904 dedup_ratio=0.333771 router_tokens_per_expert=[300, 243, 229, 254, 284, 257, 253, 252, 250, 264, 233, 254, 254, 248, 275, 249, 245, 270, 273, 273, 255, 267, 244, 260, 267, 258, 238, 261, 249, 232, 268, 242, 259, 248, 270, 266, 256, 224, 223, 266, 238, 250, 212, 264, 233, 265, 244, 272, 237, 252, 277, 266, 290, 262, 273, 261, 249, 261, 245, 261, 277, 242, 273, 262, 260, 250, 269, 251, 266, 247, 265, 253, 246, 248, 263, 280, 262, 278, 269, 241, 231, 265, 264, 252, 251, 243, 262, 257, 252, 258, 281, 269, 258, 260, 251, 243, 257, 276, 253, 271, 281, 249, 242, 238, 285, 249, 236, 247, 287, 248, 238, 257, 247, 290, 249, 252, 270, 232, 271, 228, 246, 239, 247, 271, 239, 259, 218, 272] payload_rows_per_dst=[5452, 5300, 5362, 5562, 5492, 5434, 5440, 5620] dup_rows_per_dst=[2718, 2524, 2618, 2782, 2654, 2792, 2816, 2970] combine_rows_per_src=[5452, 5300, 5362, 5562, 5492, 5434, 5440, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38254 dup_rows=21852 dup_rows_remote=19166 dedup_ratio=0.333435 router_tokens_per_expert=[228, 256, 250, 257, 269, 258, 266, 240, 272, 293, 258, 265, 249, 268, 259, 268, 272, 228, 263, 235, 259, 262, 248, 261, 243, 261, 296, 246, 238, 237, 241, 268, 260, 244, 260, 254, 246, 259, 267, 237, 283, 263, 246, 245, 236, 262, 226, 268, 273, 240, 223, 264, 288, 243, 269, 259, 234, 252, 252, 267, 262, 246, 276, 240, 262, 251, 266, 276, 265, 265, 288, 237, 256, 241, 242, 240, 264, 238, 281, 272, 250, 274, 247, 225, 240, 264, 254, 276, 259, 268, 253, 251, 274, 251, 265, 275, 251, 283, 237, 262, 263, 242, 282, 297, 256, 253, 255, 252, 245, 248, 254, 262, 253, 256, 242, 249, 245, 263, 280, 254, 233, 251, 251, 253, 256, 253, 241, 218] payload_rows_per_dst=[5510, 5430, 5434, 5476, 5478, 5464, 5544, 5348] dup_rows_per_dst=[2802, 2686, 2678, 2700, 2810, 2788, 2740, 2648] combine_rows_per_src=[5510, 5430, 5434, 5476, 5478, 5464, 5544, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38432 dup_rows=21668 dup_rows_remote=18946 dedup_ratio=0.330627 router_tokens_per_expert=[263, 272, 249, 259, 252, 274, 260, 256, 255, 243, 277, 279, 234, 270, 249, 255, 260, 263, 255, 246, 272, 234, 257, 251, 265, 237, 239, 245, 260, 266, 252, 290, 237, 265, 265, 267, 254, 251, 256, 259, 241, 231, 261, 242, 233, 228, 273, 280, 251, 245, 238, 249, 246, 263, 264, 256, 237, 266, 274, 277, 263, 233, 240, 247, 276, 276, 266, 241, 258, 282, 287, 254, 263, 244, 257, 265, 235, 284, 240, 249, 249, 270, 253, 264, 266, 254, 244, 238, 261, 271, 250, 258, 228, 260, 270, 243, 239, 244, 268, 258, 230, 238, 265, 255, 277, 268, 258, 283, 258, 261, 274, 253, 277, 249, 257, 261, 256, 220, 252, 252, 256, 250, 234, 269, 240, 256, 258, 265] payload_rows_per_dst=[5552, 5528, 5406, 5514, 5540, 5436, 5472, 5420] dup_rows_per_dst=[2742, 2656, 2680, 2584, 2814, 2722, 2786, 2684] combine_rows_per_src=[5552, 5528, 5406, 5514, 5540, 5436, 5472, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229576 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343944 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314458312 weight_movement_bytes=37748736 total_nvlink_bytes=352207048 weight_to_token_ratio=0.120044 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229200 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157556560 ingress_bytes=195076096 link_bytes=195076096 token_bytes=314883920 weight_movement_bytes=37748736 total_nvlink_bytes=352632656 weight_to_token_ratio=0.119881 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869123168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=227784 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156047816 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311867848 weight_movement_bytes=18874368 total_nvlink_bytes=330742216 weight_to_token_ratio=0.060520 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229680 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156918064 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606448 weight_movement_bytes=37748736 total_nvlink_bytes=351355184 weight_to_token_ratio=0.120370 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=229512 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157646984 ingress_bytes=195166208 link_bytes=195166208 token_bytes=315064456 weight_movement_bytes=37748736 total_nvlink_bytes=352813192 weight_to_token_ratio=0.119813 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869986577e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38206 dup_rows=21848 dup_rows_remote=19000 dedup_ratio=0.333374 router_tokens_per_expert=[270, 250, 255, 271, 269, 268, 278, 247, 274, 272, 256, 231, 261, 250, 259, 250, 254, 247, 270, 254, 252, 241, 273, 210, 242, 263, 246, 273, 265, 257, 242, 266, 260, 255, 258, 241, 276, 283, 260, 239, 254, 268, 259, 254, 240, 272, 249, 265, 243, 230, 246, 262, 256, 240, 248, 261, 257, 248, 269, 241, 255, 234, 264, 260, 254, 295, 250, 243, 253, 257, 234, 267, 237, 277, 261, 216, 272, 270, 253, 269, 224, 273, 246, 269, 250, 242, 271, 260, 220, 264, 272, 265, 253, 253, 249, 263, 282, 236, 257, 246, 255, 254, 252, 274, 249, 253, 255, 268, 251, 270, 261, 281, 276, 236, 264, 246, 269, 254, 249, 246, 265, 233, 236, 238, 255, 256, 264, 292] payload_rows_per_dst=[5482, 5598, 5540, 5462, 5604, 5430, 5340, 5232] dup_rows_per_dst=[2848, 2930, 2726, 2592, 2864, 2684, 2696, 2508] combine_rows_per_src=[5482, 5598, 5540, 5462, 5604, 5430, 5340, 5232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38206 dup_rows=21798 dup_rows_remote=19028 dedup_ratio=0.332611 router_tokens_per_expert=[254, 259, 273, 265, 255, 239, 257, 260, 223, 264, 246, 225, 262, 248, 238, 280, 259, 251, 240, 278, 236, 250, 265, 285, 249, 245, 271, 229, 233, 282, 276, 244, 251, 237, 227, 272, 259, 257, 227, 255, 256, 232, 256, 238, 242, 264, 255, 255, 265, 244, 259, 275, 257, 249, 266, 276, 256, 248, 241, 292, 259, 256, 254, 254, 237, 252, 253, 251, 275, 252, 267, 272, 235, 257, 266, 237, 279, 236, 259, 257, 253, 263, 265, 256, 252, 272, 249, 256, 276, 281, 271, 271, 248, 252, 263, 249, 237, 267, 245, 219, 253, 281, 263, 263, 251, 274, 269, 261, 257, 274, 253, 258, 270, 222, 285, 292, 261, 263, 264, 284, 223, 240, 250, 252, 241, 253, 252, 254] payload_rows_per_dst=[5398, 5476, 5346, 5532, 5474, 5598, 5504, 5410] dup_rows_per_dst=[2698, 2710, 2620, 2770, 2696, 2756, 2746, 2802] combine_rows_per_src=[5398, 5476, 5346, 5532, 5474, 5598, 5504, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38106 dup_rows=22002 dup_rows_remote=19220 dedup_ratio=0.335724 router_tokens_per_expert=[265, 263, 270, 281, 259, 235, 270, 239, 270, 263, 263, 259, 247, 259, 281, 251, 255, 265, 265, 258, 257, 256, 248, 257, 241, 262, 231, 266, 247, 256, 262, 263, 250, 255, 243, 269, 283, 260, 261, 236, 243, 242, 272, 259, 285, 257, 247, 243, 209, 240, 236, 271, 278, 260, 245, 274, 284, 237, 241, 250, 237, 238, 279, 272, 256, 238, 254, 230, 258, 239, 251, 231, 257, 287, 241, 251, 248, 258, 271, 264, 256, 273, 258, 286, 251, 260, 274, 254, 249, 261, 235, 247, 221, 231, 250, 257, 238, 285, 266, 238, 247, 273, 235, 289, 283, 256, 285, 281, 245, 277, 241, 229, 258, 271, 241, 251, 244, 250, 271, 230, 254, 254, 282, 248, 278, 252, 259, 240] payload_rows_per_dst=[5454, 5424, 5428, 5388, 5418, 5464, 5492, 5466] dup_rows_per_dst=[2896, 2754, 2782, 2714, 2650, 2662, 2844, 2700] combine_rows_per_src=[5454, 5424, 5428, 5388, 5418, 5464, 5492, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228824 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156720600 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313212376 weight_movement_bytes=37748736 total_nvlink_bytes=350961112 weight_to_token_ratio=0.120521 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38276 dup_rows=21832 dup_rows_remote=19116 dedup_ratio=0.333130 router_tokens_per_expert=[238, 280, 238, 258, 263, 256, 253, 241, 243, 279, 236, 270, 263, 283, 258, 268, 266, 266, 247, 254, 267, 257, 280, 277, 248, 266, 249, 251, 230, 250, 261, 264, 274, 256, 235, 263, 279, 273, 239, 257, 243, 264, 267, 258, 244, 264, 240, 236, 256, 293, 259, 230, 242, 247, 267, 264, 284, 272, 267, 265, 246, 253, 223, 268, 271, 252, 238, 277, 256, 252, 281, 243, 260, 247, 232, 243, 264, 264, 247, 245, 260, 284, 238, 245, 266, 265, 254, 244, 255, 254, 265, 270, 240, 242, 268, 267, 257, 258, 249, 237, 236, 262, 256, 256, 260, 252, 278, 258, 270, 256, 268, 240, 246, 219, 276, 260, 243, 238, 267, 256, 238, 249, 247, 263, 234, 232, 280, 250] payload_rows_per_dst=[5528, 5476, 5434, 5502, 5428, 5488, 5450, 5398] dup_rows_per_dst=[2726, 2790, 2750, 2770, 2716, 2746, 2736, 2598] combine_rows_per_src=[5528, 5476, 5434, 5502, 5428, 5488, 5450, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228936 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156720712 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313212488 weight_movement_bytes=37748736 total_nvlink_bytes=350961224 weight_to_token_ratio=0.120521 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229304 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311480 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393656 weight_movement_bytes=37748736 total_nvlink_bytes=350142392 weight_to_token_ratio=0.120837 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38208 dup_rows=21870 dup_rows_remote=19256 dedup_ratio=0.333710 router_tokens_per_expert=[253, 282, 274, 258, 234, 241, 253, 262, 237, 255, 253, 228, 257, 261, 263, 259, 258, 266, 249, 254, 285, 313, 252, 249, 264, 254, 268, 269, 254, 238, 271, 234, 264, 239, 254, 285, 265, 255, 243, 259, 259, 262, 249, 273, 267, 263, 252, 265, 275, 251, 245, 263, 266, 255, 269, 248, 242, 259, 239, 243, 263, 239, 278, 280, 270, 262, 273, 249, 247, 248, 238, 252, 223, 267, 238, 245, 261, 267, 249, 254, 246, 243, 252, 268, 245, 246, 257, 246, 245, 266, 253, 222, 263, 277, 262, 253, 228, 256, 261, 283, 274, 255, 251, 250, 240, 266, 228, 253, 251, 250, 241, 249, 253, 232, 260, 257, 270, 281, 265, 263, 264, 257, 253, 255, 269, 246, 242, 261] payload_rows_per_dst=[5448, 5528, 5484, 5422, 5468, 5432, 5458, 5426] dup_rows_per_dst=[2692, 2828, 2824, 2808, 2618, 2656, 2614, 2830] combine_rows_per_src=[5448, 5528, 5484, 5422, 5468, 5432, 5458, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38228 dup_rows=21832 dup_rows_remote=18934 dedup_ratio=0.333130 router_tokens_per_expert=[262, 248, 259, 226, 272, 255, 282, 247, 235, 252, 256, 252, 263, 236, 252, 285, 272, 232, 233, 234, 231, 245, 239, 249, 255, 229, 256, 251, 270, 238, 272, 274, 261, 282, 268, 280, 255, 273, 245, 266, 238, 255, 256, 248, 249, 265, 262, 250, 245, 235, 254, 240, 235, 267, 243, 251, 250, 282, 258, 261, 294, 235, 289, 254, 267, 263, 239, 237, 261, 243, 248, 265, 265, 249, 257, 280, 265, 224, 271, 257, 241, 250, 217, 269, 282, 286, 281, 259, 247, 241, 257, 245, 249, 253, 269, 258, 279, 220, 258, 259, 261, 268, 267, 274, 253, 251, 261, 240, 260, 255, 287, 227, 261, 272, 260, 259, 299, 258, 260, 253, 230, 260, 269, 279, 248, 233, 236, 268] payload_rows_per_dst=[5334, 5336, 5518, 5310, 5546, 5638, 5546, 5476] dup_rows_per_dst=[2420, 2616, 2704, 2730, 2782, 2980, 2702, 2898] combine_rows_per_src=[5334, 5336, 5518, 5310, 5546, 5638, 5546, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38362 dup_rows=21774 dup_rows_remote=19114 dedup_ratio=0.332245 router_tokens_per_expert=[241, 257, 248, 256, 264, 249, 250, 265, 262, 278, 246, 235, 251, 258, 247, 272, 291, 251, 230, 278, 225, 243, 251, 258, 262, 246, 246, 267, 243, 288, 243, 285, 269, 243, 265, 248, 265, 230, 284, 257, 256, 249, 249, 277, 256, 252, 261, 267, 272, 301, 244, 261, 242, 264, 245, 248, 269, 259, 229, 243, 235, 266, 269, 263, 274, 284, 240, 245, 235, 271, 278, 233, 256, 240, 243, 230, 263, 239, 254, 246, 220, 242, 246, 252, 250, 254, 255, 250, 240, 267, 260, 241, 278, 273, 265, 237, 251, 260, 254, 254, 246, 287, 257, 282, 250, 272, 268, 280, 259, 239, 277, 260, 271, 264, 254, 273, 253, 244, 273, 229, 231, 261, 223, 254, 247, 244, 292, 274] payload_rows_per_dst=[5424, 5496, 5484, 5502, 5366, 5400, 5578, 5512] dup_rows_per_dst=[2734, 2718, 2772, 2718, 2696, 2660, 2814, 2662] combine_rows_per_src=[5424, 5496, 5484, 5502, 5366, 5400, 5578, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38142 dup_rows=21970 dup_rows_remote=19190 dedup_ratio=0.335236 router_tokens_per_expert=[264, 280, 261, 253, 266, 260, 268, 265, 260, 263, 256, 270, 250, 259, 246, 245, 288, 267, 268, 299, 256, 224, 245, 246, 232, 241, 247, 250, 252, 263, 269, 255, 237, 239, 248, 258, 229, 270, 277, 256, 259, 258, 229, 243, 259, 264, 283, 267, 254, 254, 268, 257, 252, 236, 253, 246, 247, 279, 240, 263, 247, 243, 264, 284, 260, 258, 247, 265, 243, 259, 285, 270, 245, 247, 270, 263, 265, 258, 229, 267, 248, 241, 239, 236, 259, 279, 262, 243, 263, 246, 237, 245, 229, 264, 250, 257, 244, 260, 274, 251, 255, 272, 279, 257, 256, 257, 279, 234, 237, 259, 227, 253, 250, 257, 263, 268, 254, 265, 253, 244, 259, 245, 250, 248, 230, 247, 285, 296] payload_rows_per_dst=[5476, 5424, 5426, 5466, 5558, 5316, 5454, 5446] dup_rows_per_dst=[2856, 2780, 2726, 2708, 2704, 2680, 2734, 2782] combine_rows_per_src=[5476, 5424, 5426, 5466, 5558, 5316, 5454, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229568 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008064 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786560 weight_movement_bytes=18874368 total_nvlink_bytes=332660928 weight_to_token_ratio=0.060150 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229856 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156729824 ingress_bytes=231997440 link_bytes=231997440 token_bytes=313229792 weight_movement_bytes=75497472 total_nvlink_bytes=388727264 weight_to_token_ratio=0.241029 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222885320e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=228648 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156810536 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313392424 weight_movement_bytes=18874368 total_nvlink_bytes=332266792 weight_to_token_ratio=0.060226 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229904 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157360656 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314491408 weight_movement_bytes=18874368 total_nvlink_bytes=333365776 weight_to_token_ratio=0.060016 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229328 combine_payload_bytes=156229632 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156458960 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312688592 weight_movement_bytes=18874368 total_nvlink_bytes=331562960 weight_to_token_ratio=0.060362 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38304 dup_rows=21616 dup_rows_remote=18796 dedup_ratio=0.329834 router_tokens_per_expert=[265, 271, 244, 256, 257, 274, 234, 260, 248, 273, 250, 272, 243, 263, 244, 240, 242, 248, 260, 266, 237, 241, 276, 261, 269, 273, 279, 267, 247, 229, 281, 275, 261, 269, 242, 203, 240, 244, 256, 256, 252, 253, 273, 236, 258, 258, 264, 274, 269, 252, 260, 275, 260, 255, 248, 253, 265, 276, 252, 255, 264, 245, 248, 257, 257, 262, 265, 263, 237, 292, 253, 263, 241, 254, 260, 270, 258, 262, 266, 246, 251, 249, 230, 256, 218, 280, 247, 247, 242, 234, 246, 248, 261, 232, 254, 242, 265, 284, 273, 266, 262, 241, 247, 271, 259, 247, 262, 247, 268, 255, 280, 268, 260, 255, 252, 248, 251, 296, 258, 270, 243, 237, 247, 255, 256, 229, 283, 229] payload_rows_per_dst=[5616, 5566, 5574, 5510, 5350, 5294, 5562, 5448] dup_rows_per_dst=[2820, 2738, 2994, 2756, 2458, 2332, 2828, 2690] combine_rows_per_src=[5616, 5566, 5574, 5510, 5350, 5294, 5562, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38154 dup_rows=21964 dup_rows_remote=19278 dedup_ratio=0.335144 router_tokens_per_expert=[262, 274, 266, 268, 247, 250, 245, 257, 252, 279, 262, 270, 285, 249, 270, 277, 265, 263, 262, 246, 255, 213, 229, 264, 258, 261, 244, 235, 230, 261, 265, 229, 250, 286, 291, 266, 289, 255, 267, 278, 253, 278, 264, 251, 251, 249, 238, 270, 273, 252, 271, 263, 272, 246, 253, 248, 249, 226, 255, 244, 259, 245, 236, 260, 273, 258, 285, 256, 251, 253, 224, 262, 260, 268, 251, 247, 234, 241, 265, 255, 246, 270, 248, 234, 238, 247, 251, 253, 246, 268, 273, 260, 263, 258, 248, 262, 256, 241, 253, 266, 263, 252, 231, 241, 242, 249, 273, 249, 258, 260, 266, 252, 266, 272, 244, 232, 231, 222, 271, 248, 258, 268, 264, 256, 286, 267, 244, 258] payload_rows_per_dst=[5596, 5364, 5566, 5418, 5452, 5426, 5346, 5404] dup_rows_per_dst=[2830, 2596, 2906, 2686, 2714, 2704, 2758, 2770] combine_rows_per_src=[5596, 5364, 5566, 5418, 5452, 5426, 5346, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38200 dup_rows=21936 dup_rows_remote=19186 dedup_ratio=0.334717 router_tokens_per_expert=[251, 251, 265, 251, 271, 260, 263, 253, 252, 261, 254, 265, 240, 256, 263, 256, 265, 264, 272, 221, 254, 251, 263, 252, 267, 261, 234, 235, 276, 275, 232, 274, 228, 220, 275, 268, 253, 252, 256, 260, 283, 243, 273, 260, 245, 264, 260, 235, 240, 218, 268, 246, 268, 248, 265, 283, 287, 247, 274, 291, 283, 266, 238, 236, 245, 238, 245, 268, 251, 274, 258, 265, 240, 267, 261, 250, 255, 249, 254, 263, 260, 278, 228, 242, 279, 257, 261, 230, 280, 248, 267, 264, 286, 237, 261, 261, 233, 231, 258, 266, 240, 262, 258, 264, 269, 264, 242, 257, 267, 250, 246, 258, 240, 239, 240, 250, 266, 270, 244, 259, 245, 275, 243, 253, 255, 227, 255, 279] payload_rows_per_dst=[5414, 5416, 5400, 5578, 5460, 5440, 5438, 5454] dup_rows_per_dst=[2810, 2776, 2750, 2738, 2706, 2838, 2692, 2626] combine_rows_per_src=[5414, 5416, 5400, 5578, 5460, 5440, 5438, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38086 dup_rows=21922 dup_rows_remote=19168 dedup_ratio=0.334503 router_tokens_per_expert=[259, 255, 243, 278, 250, 268, 275, 255, 236, 280, 252, 229, 265, 244, 250, 274, 269, 244, 253, 231, 247, 251, 239, 239, 245, 233, 237, 273, 275, 286, 254, 204, 242, 228, 257, 251, 263, 256, 256, 271, 231, 259, 256, 240, 266, 239, 271, 252, 239, 256, 252, 239, 239, 244, 255, 262, 244, 265, 257, 265, 250, 250, 250, 274, 251, 255, 300, 247, 253, 287, 257, 277, 263, 254, 238, 247, 231, 253, 270, 258, 243, 251, 248, 240, 267, 267, 264, 257, 227, 289, 274, 269, 238, 259, 281, 232, 222, 259, 259, 290, 278, 265, 268, 256, 241, 271, 247, 260, 270, 270, 243, 260, 260, 261, 267, 273, 268, 259, 252, 273, 275, 238, 312, 224, 247, 255, 275, 251] payload_rows_per_dst=[5440, 5346, 5396, 5496, 5528, 5434, 5460, 5514] dup_rows_per_dst=[2786, 2614, 2680, 2586, 2754, 2778, 2858, 2866] combine_rows_per_src=[5440, 5346, 5396, 5496, 5528, 5434, 5460, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43444 payload_rows_remote=37998 dup_rows=22092 dup_rows_remote=19330 dedup_ratio=0.337097 router_tokens_per_expert=[249, 243, 231, 283, 240, 255, 292, 236, 257, 251, 261, 250, 262, 249, 262, 265, 256, 241, 244, 258, 244, 245, 266, 269, 238, 249, 213, 280, 253, 256, 266, 269, 257, 267, 292, 235, 266, 253, 256, 264, 259, 234, 260, 245, 205, 279, 247, 260, 245, 267, 256, 259, 277, 258, 261, 237, 288, 273, 244, 287, 236, 250, 251, 237, 266, 265, 263, 235, 242, 255, 245, 263, 250, 266, 259, 260, 249, 260, 258, 261, 244, 262, 253, 279, 254, 264, 259, 243, 259, 249, 256, 254, 253, 255, 267, 237, 253, 254, 252, 256, 281, 267, 236, 230, 268, 258, 245, 278, 255, 259, 238, 274, 229, 248, 253, 280, 258, 251, 246, 248, 268, 273, 256, 289, 250, 266, 271, 255] payload_rows_per_dst=[5430, 5416, 5448, 5410, 5362, 5418, 5446, 5514] dup_rows_per_dst=[2660, 2678, 2792, 2842, 2832, 2758, 2762, 2768] combine_rows_per_src=[5430, 5416, 5448, 5410, 5362, 5418, 5446, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228400 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157121584 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314014768 weight_movement_bytes=18874368 total_nvlink_bytes=332889136 weight_to_token_ratio=0.060107 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229728 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156508512 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312787296 weight_movement_bytes=37748736 total_nvlink_bytes=350536032 weight_to_token_ratio=0.120685 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229544 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156696744 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313163944 weight_movement_bytes=18874368 total_nvlink_bytes=332038312 weight_to_token_ratio=0.060270 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38070 dup_rows=21798 dup_rows_remote=18900 dedup_ratio=0.332611 router_tokens_per_expert=[267, 263, 257, 269, 262, 260, 241, 255, 258, 248, 276, 253, 229, 286, 261, 267, 239, 256, 231, 254, 245, 253, 249, 261, 238, 258, 257, 243, 255, 279, 253, 265, 243, 265, 233, 236, 252, 270, 256, 236, 254, 250, 257, 247, 240, 212, 253, 244, 284, 232, 273, 285, 238, 262, 252, 276, 265, 248, 267, 255, 262, 244, 249, 252, 237, 255, 271, 224, 244, 283, 210, 257, 249, 252, 245, 273, 231, 255, 273, 258, 287, 239, 279, 273, 255, 243, 266, 270, 257, 261, 254, 248, 236, 253, 268, 253, 264, 267, 272, 247, 241, 253, 247, 280, 238, 251, 270, 225, 244, 246, 241, 281, 277, 308, 249, 276, 263, 245, 270, 294, 259, 264, 256, 257, 255, 231, 255, 303] payload_rows_per_dst=[5298, 5452, 5558, 5444, 5400, 5452, 5466, 5668] dup_rows_per_dst=[2524, 2620, 2820, 2844, 2592, 2832, 2668, 2898] combine_rows_per_src=[5298, 5452, 5558, 5444, 5400, 5452, 5466, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38478 dup_rows=21628 dup_rows_remote=18938 dedup_ratio=0.330017 router_tokens_per_expert=[269, 277, 257, 256, 254, 271, 271, 251, 240, 240, 241, 250, 257, 278, 233, 242, 254, 238, 274, 232, 273, 278, 254, 259, 276, 287, 268, 239, 244, 250, 262, 253, 250, 237, 241, 245, 222, 248, 239, 282, 271, 248, 258, 247, 269, 248, 251, 245, 276, 244, 255, 268, 269, 238, 231, 231, 263, 244, 264, 244, 256, 268, 275, 246, 260, 274, 276, 230, 279, 275, 237, 252, 268, 286, 274, 269, 258, 260, 267, 250, 256, 239, 246, 257, 244, 244, 262, 239, 238, 276, 278, 271, 249, 248, 259, 254, 235, 281, 241, 245, 233, 278, 258, 274, 254, 230, 260, 220, 252, 238, 275, 238, 264, 246, 269, 251, 281, 238, 254, 263, 285, 254, 271, 252, 249, 262, 278, 263] payload_rows_per_dst=[5518, 5546, 5410, 5430, 5584, 5430, 5460, 5530] dup_rows_per_dst=[2656, 2736, 2592, 2714, 2846, 2690, 2564, 2830] combine_rows_per_src=[5518, 5546, 5410, 5430, 5584, 5430, 5460, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38252 dup_rows=21814 dup_rows_remote=19142 dedup_ratio=0.332855 router_tokens_per_expert=[259, 266, 280, 248, 236, 245, 291, 255, 255, 231, 261, 252, 277, 281, 255, 233, 248, 260, 264, 264, 239, 232, 246, 262, 231, 270, 282, 249, 238, 256, 269, 247, 241, 251, 244, 252, 262, 263, 263, 263, 249, 258, 257, 246, 263, 251, 254, 280, 257, 266, 242, 255, 245, 255, 271, 223, 271, 238, 259, 255, 263, 268, 252, 269, 257, 268, 253, 273, 214, 251, 231, 228, 252, 249, 283, 293, 265, 273, 255, 257, 262, 246, 259, 280, 214, 259, 305, 261, 258, 254, 241, 264, 234, 253, 284, 236, 267, 239, 238, 246, 255, 289, 270, 251, 259, 241, 248, 263, 249, 260, 270, 249, 273, 271, 273, 258, 272, 262, 261, 252, 248, 250, 283, 254, 244, 232, 238, 223] payload_rows_per_dst=[5540, 5470, 5516, 5398, 5460, 5404, 5450, 5484] dup_rows_per_dst=[2710, 2672, 2678, 2780, 2744, 2816, 2738, 2676] combine_rows_per_src=[5540, 5470, 5516, 5398, 5460, 5404, 5450, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229016 combine_payload_bytes=156000256 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156229272 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312229528 weight_movement_bytes=37748736 total_nvlink_bytes=349978264 weight_to_token_ratio=0.120901 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155639808 dispatch_meta_bytes=229312 combine_payload_bytes=155639808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155869120 ingress_bytes=174514176 link_bytes=174514176 token_bytes=311508928 weight_movement_bytes=18874368 total_nvlink_bytes=330383296 weight_to_token_ratio=0.060590 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672108968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=227880 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156162600 ingress_bytes=231432192 link_bytes=231432192 token_bytes=312097320 weight_movement_bytes=75497472 total_nvlink_bytes=387594792 weight_to_token_ratio=0.241904 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217469392e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157605888 dispatch_meta_bytes=229664 combine_payload_bytes=157605888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157835552 ingress_bytes=176480256 link_bytes=176480256 token_bytes=315441440 weight_movement_bytes=18874368 total_nvlink_bytes=334315808 weight_to_token_ratio=0.059835 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690946979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229576 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909768 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589960 weight_movement_bytes=18874368 total_nvlink_bytes=332464328 weight_to_token_ratio=0.060188 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38414 dup_rows=21658 dup_rows_remote=19002 dedup_ratio=0.330475 router_tokens_per_expert=[229, 289, 275, 257, 263, 246, 259, 275, 262, 225, 254, 268, 238, 234, 253, 272, 275, 244, 285, 255, 242, 261, 258, 251, 233, 283, 252, 250, 270, 261, 234, 259, 259, 263, 252, 254, 257, 255, 238, 230, 264, 247, 231, 266, 258, 261, 276, 276, 252, 242, 248, 235, 252, 266, 259, 239, 265, 256, 242, 256, 278, 300, 231, 239, 243, 248, 227, 266, 263, 244, 283, 266, 273, 257, 261, 257, 274, 252, 270, 274, 239, 258, 248, 256, 268, 276, 257, 228, 264, 281, 250, 253, 284, 278, 279, 258, 245, 278, 272, 225, 282, 243, 251, 259, 261, 243, 261, 249, 239, 216, 280, 260, 251, 256, 257, 241, 266, 252, 254, 216, 252, 225, 281, 263, 235, 238, 257, 266] payload_rows_per_dst=[5486, 5488, 5508, 5464, 5522, 5554, 5454, 5402] dup_rows_per_dst=[2712, 2738, 2666, 2656, 2794, 2800, 2674, 2618] combine_rows_per_src=[5486, 5488, 5508, 5464, 5522, 5554, 5454, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38210 dup_rows=21858 dup_rows_remote=19110 dedup_ratio=0.333527 router_tokens_per_expert=[243, 251, 267, 229, 236, 280, 257, 266, 256, 251, 271, 248, 237, 237, 253, 286, 235, 286, 236, 234, 251, 264, 269, 253, 264, 272, 238, 269, 257, 287, 252, 256, 253, 253, 221, 252, 258, 239, 252, 273, 285, 261, 255, 253, 250, 245, 267, 291, 273, 270, 266, 234, 252, 219, 245, 251, 289, 225, 240, 271, 270, 254, 273, 257, 254, 214, 242, 240, 279, 244, 282, 259, 264, 234, 278, 224, 269, 253, 227, 248, 262, 231, 251, 241, 250, 256, 271, 258, 253, 245, 280, 254, 268, 255, 239, 273, 252, 236, 287, 269, 259, 229, 245, 260, 247, 264, 269, 271, 244, 275, 271, 267, 219, 251, 258, 275, 265, 258, 270, 252, 258, 249, 286, 240, 284, 262, 243, 267] payload_rows_per_dst=[5428, 5466, 5468, 5502, 5386, 5520, 5480, 5428] dup_rows_per_dst=[2708, 2780, 2748, 2676, 2636, 2654, 2810, 2846] combine_rows_per_src=[5428, 5466, 5468, 5502, 5386, 5520, 5480, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38044 dup_rows=21886 dup_rows_remote=18904 dedup_ratio=0.333954 router_tokens_per_expert=[246, 250, 252, 256, 250, 255, 263, 218, 257, 242, 281, 258, 268, 254, 245, 275, 252, 253, 271, 273, 249, 284, 249, 263, 264, 254, 240, 249, 240, 252, 255, 248, 257, 294, 253, 261, 227, 261, 238, 234, 257, 254, 270, 248, 267, 265, 228, 267, 220, 251, 282, 270, 239, 243, 246, 258, 249, 278, 259, 225, 265, 279, 248, 254, 239, 249, 227, 265, 240, 247, 250, 280, 247, 254, 249, 262, 247, 263, 245, 255, 263, 256, 276, 259, 250, 255, 274, 252, 256, 266, 237, 275, 246, 253, 270, 277, 257, 262, 270, 272, 266, 266, 265, 279, 242, 238, 250, 240, 272, 266, 287, 242, 255, 273, 274, 260, 240, 235, 280, 255, 246, 247, 263, 276, 265, 227, 257, 244] payload_rows_per_dst=[5606, 5584, 5492, 5380, 5412, 5382, 5400, 5394] dup_rows_per_dst=[2982, 2972, 2704, 2760, 2778, 2500, 2584, 2606] combine_rows_per_src=[5606, 5584, 5492, 5380, 5412, 5382, 5400, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38184 dup_rows=21898 dup_rows_remote=19202 dedup_ratio=0.334137 router_tokens_per_expert=[250, 279, 258, 238, 259, 252, 267, 264, 258, 262, 231, 265, 270, 260, 247, 273, 257, 257, 272, 235, 251, 272, 259, 261, 257, 240, 275, 271, 231, 248, 231, 268, 247, 280, 253, 252, 246, 264, 263, 254, 265, 250, 246, 281, 253, 261, 276, 252, 233, 254, 241, 248, 220, 272, 256, 275, 247, 257, 263, 231, 259, 244, 250, 261, 263, 232, 272, 241, 282, 242, 260, 280, 259, 257, 233, 256, 245, 265, 241, 247, 257, 233, 264, 252, 268, 267, 281, 258, 245, 258, 224, 254, 251, 245, 238, 236, 251, 244, 255, 263, 241, 285, 243, 261, 269, 282, 300, 264, 256, 258, 264, 245, 261, 234, 261, 255, 271, 274, 270, 238, 278, 238, 258, 244, 258, 264, 267, 238] payload_rows_per_dst=[5440, 5496, 5532, 5410, 5454, 5338, 5540, 5428] dup_rows_per_dst=[2826, 2674, 2754, 2612, 2696, 2724, 2822, 2790] combine_rows_per_src=[5440, 5496, 5532, 5410, 5454, 5338, 5540, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229664 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157573408 ingress_bytes=213966848 link_bytes=213966848 token_bytes=314917152 weight_movement_bytes=56623104 total_nvlink_bytes=371540256 weight_to_token_ratio=0.179803 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050125059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229280 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737440 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245600 weight_movement_bytes=18874368 total_nvlink_bytes=332119968 weight_to_token_ratio=0.060254 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38362 dup_rows=21712 dup_rows_remote=19004 dedup_ratio=0.331299 router_tokens_per_expert=[275, 231, 251, 259, 242, 266, 239, 276, 265, 287, 267, 250, 252, 246, 266, 254, 244, 252, 249, 251, 242, 276, 273, 263, 259, 262, 268, 274, 267, 238, 249, 256, 243, 267, 290, 237, 258, 251, 271, 256, 238, 249, 257, 255, 266, 260, 258, 244, 260, 243, 232, 274, 250, 260, 263, 262, 262, 271, 239, 292, 267, 278, 257, 242, 236, 276, 275, 244, 234, 272, 247, 263, 258, 257, 249, 278, 233, 268, 223, 242, 252, 253, 273, 250, 221, 284, 246, 248, 269, 220, 240, 233, 255, 260, 236, 275, 248, 260, 247, 268, 249, 260, 238, 249, 271, 252, 275, 269, 265, 265, 242, 264, 264, 254, 258, 251, 284, 235, 291, 238, 238, 272, 240, 247, 250, 280, 234, 239] payload_rows_per_dst=[5528, 5468, 5424, 5558, 5486, 5402, 5462, 5496] dup_rows_per_dst=[2724, 2778, 2776, 2746, 2624, 2628, 2708, 2728] combine_rows_per_src=[5528, 5468, 5424, 5558, 5486, 5402, 5462, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43492 payload_rows_remote=37854 dup_rows=22044 dup_rows_remote=19106 dedup_ratio=0.336365 router_tokens_per_expert=[244, 246, 247, 247, 259, 264, 257, 271, 227, 267, 261, 258, 258, 240, 240, 276, 282, 251, 225, 243, 246, 245, 238, 238, 243, 273, 244, 240, 251, 251, 287, 267, 265, 274, 259, 262, 249, 255, 282, 257, 271, 264, 225, 235, 254, 246, 257, 239, 275, 270, 245, 285, 267, 288, 281, 243, 267, 238, 261, 229, 300, 261, 274, 267, 254, 277, 245, 253, 249, 279, 255, 270, 228, 255, 247, 246, 239, 216, 245, 249, 263, 259, 267, 230, 272, 265, 238, 261, 279, 269, 267, 264, 264, 246, 276, 249, 270, 235, 253, 250, 256, 280, 258, 245, 261, 260, 267, 276, 251, 256, 274, 248, 231, 234, 254, 272, 234, 258, 242, 271, 242, 254, 281, 272, 246, 238, 237, 255] payload_rows_per_dst=[5318, 5246, 5486, 5432, 5448, 5570, 5354, 5638] dup_rows_per_dst=[2634, 2422, 2702, 2792, 2890, 3112, 2554, 2938] combine_rows_per_src=[5318, 5246, 5486, 5432, 5448, 5570, 5354, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155828224 dispatch_meta_bytes=227792 combine_payload_bytes=155828224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156056016 ingress_bytes=174702592 link_bytes=174702592 token_bytes=311884240 weight_movement_bytes=18874368 total_nvlink_bytes=330758608 weight_to_token_ratio=0.060517 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673914278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38394 dup_rows=21788 dup_rows_remote=19068 dedup_ratio=0.332458 router_tokens_per_expert=[244, 212, 288, 255, 257, 282, 253, 285, 249, 272, 267, 252, 261, 251, 265, 277, 277, 245, 268, 236, 253, 256, 275, 261, 271, 266, 282, 259, 273, 234, 271, 259, 262, 218, 230, 228, 271, 266, 265, 222, 239, 264, 242, 258, 247, 255, 256, 264, 236, 249, 240, 245, 261, 270, 252, 237, 251, 324, 256, 262, 286, 303, 261, 260, 269, 253, 290, 238, 269, 268, 257, 268, 275, 244, 223, 250, 258, 247, 269, 267, 247, 235, 283, 262, 250, 259, 245, 223, 240, 244, 271, 259, 265, 238, 267, 249, 255, 244, 219, 237, 263, 248, 268, 260, 246, 253, 261, 252, 272, 267, 229, 279, 237, 280, 266, 255, 235, 244, 243, 249, 226, 232, 233, 268, 257, 260, 252, 260] payload_rows_per_dst=[5564, 5544, 5370, 5568, 5558, 5354, 5412, 5378] dup_rows_per_dst=[2776, 2828, 2604, 2818, 2732, 2720, 2694, 2616] combine_rows_per_src=[5564, 5544, 5370, 5568, 5558, 5354, 5412, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38334 dup_rows=21794 dup_rows_remote=18972 dedup_ratio=0.332550 router_tokens_per_expert=[266, 248, 229, 266, 264, 253, 244, 300, 267, 252, 287, 235, 232, 239, 252, 282, 266, 235, 265, 254, 252, 248, 258, 286, 288, 290, 261, 248, 232, 232, 252, 248, 262, 235, 276, 240, 269, 261, 245, 266, 233, 249, 241, 231, 264, 268, 295, 257, 242, 238, 267, 271, 242, 246, 261, 265, 241, 246, 254, 261, 260, 237, 251, 248, 265, 259, 256, 264, 274, 244, 232, 250, 237, 248, 241, 240, 263, 281, 286, 266, 255, 222, 277, 228, 266, 266, 275, 236, 261, 261, 232, 226, 253, 245, 295, 274, 258, 284, 279, 279, 252, 254, 265, 260, 272, 248, 278, 242, 235, 245, 249, 284, 280, 252, 232, 248, 266, 249, 226, 265, 259, 239, 261, 258, 265, 250, 243, 260] payload_rows_per_dst=[5476, 5408, 5506, 5430, 5496, 5456, 5546, 5424] dup_rows_per_dst=[2756, 2822, 2678, 2630, 2716, 2688, 2822, 2682] combine_rows_per_src=[5476, 5408, 5506, 5430, 5496, 5456, 5546, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229544 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156631208 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032872 weight_movement_bytes=18874368 total_nvlink_bytes=331907240 weight_to_token_ratio=0.060295 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=155049984 dispatch_meta_bytes=227840 combine_payload_bytes=155049984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155277824 ingress_bytes=173924352 link_bytes=173924352 token_bytes=310327808 weight_movement_bytes=18874368 total_nvlink_bytes=329202176 weight_to_token_ratio=0.060821 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.666457565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229464 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157360216 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314490968 weight_movement_bytes=37748736 total_nvlink_bytes=352239704 weight_to_token_ratio=0.120031 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229848 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491672 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314753496 weight_movement_bytes=18874368 total_nvlink_bytes=333627864 weight_to_token_ratio=0.059966 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229224 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157245288 ingress_bytes=213639168 link_bytes=213639168 token_bytes=314261352 weight_movement_bytes=56623104 total_nvlink_bytes=370884456 weight_to_token_ratio=0.180178 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046985391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38276 dup_rows=21810 dup_rows_remote=19106 dedup_ratio=0.332794 router_tokens_per_expert=[260, 267, 242, 282, 249, 255, 271, 235, 235, 242, 270, 251, 270, 248, 274, 250, 273, 267, 243, 276, 268, 216, 222, 261, 258, 242, 267, 267, 287, 244, 269, 285, 252, 252, 264, 301, 262, 258, 252, 248, 274, 248, 272, 297, 255, 236, 253, 242, 226, 277, 278, 284, 231, 295, 230, 293, 230, 239, 253, 235, 260, 276, 238, 232, 253, 252, 250, 263, 266, 254, 239, 233, 247, 266, 264, 236, 225, 263, 250, 233, 266, 243, 283, 239, 266, 250, 255, 276, 241, 267, 271, 246, 271, 241, 251, 237, 271, 234, 262, 264, 280, 265, 261, 244, 242, 237, 264, 250, 242, 269, 255, 239, 241, 242, 276, 227, 267, 275, 269, 257, 240, 255, 267, 264, 252, 235, 270, 266] payload_rows_per_dst=[5478, 5516, 5538, 5450, 5332, 5446, 5516, 5450] dup_rows_per_dst=[2724, 2774, 2794, 2704, 2656, 2760, 2642, 2756] combine_rows_per_src=[5478, 5516, 5538, 5450, 5332, 5446, 5516, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38280 dup_rows=21702 dup_rows_remote=18890 dedup_ratio=0.331146 router_tokens_per_expert=[238, 250, 209, 261, 276, 252, 253, 258, 278, 287, 266, 244, 247, 267, 253, 256, 261, 267, 250, 283, 244, 255, 275, 235, 242, 238, 250, 259, 237, 245, 254, 248, 276, 227, 231, 279, 260, 261, 257, 254, 282, 257, 248, 259, 281, 269, 274, 268, 252, 253, 263, 273, 246, 265, 253, 258, 298, 256, 266, 252, 246, 262, 250, 252, 249, 258, 235, 249, 259, 274, 241, 286, 278, 269, 245, 270, 245, 224, 248, 286, 261, 235, 253, 229, 252, 247, 256, 249, 252, 250, 251, 242, 263, 263, 265, 254, 255, 247, 271, 258, 226, 290, 231, 237, 241, 258, 251, 268, 247, 243, 262, 293, 245, 277, 284, 253, 259, 240, 230, 273, 265, 276, 233, 239, 234, 265, 249, 264] payload_rows_per_dst=[5464, 5412, 5554, 5458, 5500, 5458, 5460, 5528] dup_rows_per_dst=[2726, 2674, 2812, 2832, 2732, 2586, 2696, 2644] combine_rows_per_src=[5464, 5412, 5554, 5458, 5500, 5458, 5460, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38408 dup_rows=21684 dup_rows_remote=19024 dedup_ratio=0.330872 router_tokens_per_expert=[251, 253, 249, 267, 251, 272, 235, 231, 260, 229, 256, 282, 255, 246, 246, 265, 255, 294, 270, 277, 239, 265, 248, 229, 255, 270, 266, 272, 277, 265, 276, 263, 283, 238, 248, 273, 230, 250, 258, 267, 253, 234, 236, 237, 261, 264, 277, 259, 268, 261, 255, 263, 278, 272, 268, 239, 245, 227, 252, 252, 271, 260, 239, 265, 266, 237, 250, 257, 287, 239, 258, 261, 244, 255, 254, 253, 239, 272, 244, 266, 266, 250, 260, 236, 232, 268, 289, 274, 247, 249, 280, 264, 286, 226, 248, 267, 241, 246, 264, 254, 250, 266, 262, 267, 234, 248, 246, 257, 244, 269, 265, 261, 219, 254, 259, 265, 279, 231, 259, 257, 251, 256, 260, 254, 232, 233, 234, 275] payload_rows_per_dst=[5444, 5662, 5446, 5540, 5456, 5504, 5360, 5440] dup_rows_per_dst=[2660, 2968, 2784, 2682, 2708, 2686, 2600, 2596] combine_rows_per_src=[5444, 5662, 5446, 5540, 5456, 5504, 5360, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229528 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008024 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786520 weight_movement_bytes=18874368 total_nvlink_bytes=332660888 weight_to_token_ratio=0.060150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38152 dup_rows=21884 dup_rows_remote=19220 dedup_ratio=0.333923 router_tokens_per_expert=[273, 252, 264, 230, 240, 257, 262, 277, 245, 256, 232, 245, 278, 273, 266, 250, 253, 259, 243, 257, 257, 251, 212, 246, 257, 243, 254, 265, 273, 269, 272, 258, 258, 270, 279, 248, 274, 259, 229, 273, 242, 253, 275, 259, 254, 232, 251, 257, 265, 257, 263, 245, 221, 260, 249, 267, 282, 250, 246, 295, 271, 220, 259, 224, 249, 274, 247, 270, 254, 266, 236, 250, 262, 252, 295, 236, 234, 255, 262, 240, 231, 270, 254, 232, 245, 264, 265, 270, 237, 259, 253, 244, 258, 229, 254, 265, 243, 255, 273, 258, 234, 285, 247, 247, 245, 252, 277, 257, 254, 263, 275, 276, 273, 279, 258, 257, 271, 247, 261, 242, 247, 253, 235, 260, 276, 264, 261, 275] payload_rows_per_dst=[5452, 5390, 5496, 5414, 5500, 5436, 5470, 5494] dup_rows_per_dst=[2748, 2748, 2730, 2734, 2664, 2624, 2812, 2824] combine_rows_per_src=[5452, 5390, 5496, 5414, 5500, 5436, 5470, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=228680 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157023560 ingress_bytes=232292352 link_bytes=232292352 token_bytes=313818440 weight_movement_bytes=75497472 total_nvlink_bytes=389315912 weight_to_token_ratio=0.240577 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225711022e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44010 payload_rows_remote=38438 dup_rows=21526 dup_rows_remote=18746 dedup_ratio=0.328461 router_tokens_per_expert=[280, 258, 250, 265, 255, 269, 255, 252, 260, 235, 286, 249, 255, 272, 263, 226, 247, 270, 252, 257, 240, 242, 242, 233, 239, 267, 253, 270, 267, 263, 227, 271, 262, 285, 227, 251, 262, 256, 286, 252, 256, 273, 243, 261, 265, 248, 253, 232, 263, 269, 248, 248, 271, 235, 254, 240, 264, 239, 281, 263, 245, 262, 256, 239, 270, 260, 248, 239, 256, 250, 240, 254, 267, 252, 217, 292, 265, 268, 258, 230, 239, 250, 236, 257, 234, 254, 278, 254, 271, 260, 281, 259, 238, 249, 284, 240, 250, 240, 246, 288, 253, 231, 289, 258, 237, 237, 242, 265, 236, 271, 273, 286, 247, 255, 240, 272, 268, 245, 260, 287, 247, 222, 256, 292, 256, 252, 274, 284] payload_rows_per_dst=[5516, 5406, 5342, 5478, 5482, 5680, 5534, 5572] dup_rows_per_dst=[2744, 2584, 2342, 2692, 2738, 2976, 2670, 2780] combine_rows_per_src=[5516, 5406, 5342, 5478, 5482, 5680, 5534, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43966 payload_rows_remote=38560 dup_rows=21570 dup_rows_remote=18878 dedup_ratio=0.329132 router_tokens_per_expert=[260, 250, 262, 246, 251, 289, 234, 248, 245, 264, 265, 247, 254, 258, 263, 260, 242, 272, 258, 270, 259, 286, 234, 253, 278, 259, 256, 248, 268, 259, 268, 248, 263, 264, 227, 266, 268, 274, 256, 259, 238, 292, 262, 274, 251, 285, 235, 264, 251, 238, 254, 236, 273, 281, 252, 225, 287, 265, 267, 246, 233, 251, 255, 247, 272, 255, 271, 237, 271, 260, 236, 259, 242, 247, 259, 266, 265, 241, 267, 278, 264, 209, 251, 281, 272, 249, 271, 238, 234, 248, 251, 262, 233, 264, 244, 246, 267, 232, 268, 260, 248, 259, 235, 256, 257, 266, 257, 238, 222, 262, 250, 272, 253, 254, 264, 271, 262, 286, 255, 250, 264, 254, 255, 247, 235, 218, 258, 257] payload_rows_per_dst=[5522, 5610, 5562, 5490, 5490, 5426, 5406, 5460] dup_rows_per_dst=[2670, 2706, 2794, 2632, 2762, 2608, 2692, 2706] combine_rows_per_src=[5522, 5610, 5562, 5490, 5490, 5426, 5406, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38330 dup_rows=21718 dup_rows_remote=18982 dedup_ratio=0.331390 router_tokens_per_expert=[307, 252, 253, 259, 247, 257, 262, 265, 248, 261, 241, 259, 240, 274, 246, 257, 236, 270, 243, 240, 234, 237, 262, 265, 270, 253, 273, 269, 269, 233, 248, 265, 237, 240, 271, 250, 285, 264, 241, 250, 243, 261, 259, 248, 245, 249, 285, 280, 244, 277, 281, 260, 251, 265, 264, 264, 260, 258, 255, 259, 242, 227, 254, 272, 259, 245, 248, 270, 248, 266, 272, 257, 260, 230, 279, 238, 238, 274, 267, 248, 242, 264, 237, 267, 249, 280, 230, 243, 259, 263, 248, 264, 251, 284, 260, 271, 257, 249, 236, 258, 260, 232, 255, 213, 278, 240, 243, 231, 274, 237, 265, 251, 254, 247, 266, 247, 244, 270, 276, 270, 288, 241, 250, 275, 233, 279, 272, 230] payload_rows_per_dst=[5512, 5434, 5436, 5510, 5468, 5488, 5400, 5570] dup_rows_per_dst=[2744, 2700, 2780, 2756, 2730, 2736, 2558, 2714] combine_rows_per_src=[5512, 5434, 5436, 5510, 5468, 5488, 5400, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229728 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157548896 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314868064 weight_movement_bytes=37748736 total_nvlink_bytes=352616800 weight_to_token_ratio=0.119887 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38342 dup_rows=21606 dup_rows_remote=18868 dedup_ratio=0.329681 router_tokens_per_expert=[248, 280, 256, 248, 251, 233, 288, 219, 245, 242, 232, 242, 268, 254, 264, 237, 261, 236, 242, 241, 291, 259, 256, 249, 297, 261, 268, 279, 229, 251, 268, 275, 277, 262, 232, 264, 247, 252, 233, 300, 270, 256, 231, 236, 239, 254, 237, 270, 233, 265, 235, 254, 237, 270, 269, 287, 248, 253, 233, 250, 263, 268, 279, 263, 289, 249, 275, 275, 299, 271, 229, 259, 255, 270, 234, 289, 269, 244, 260, 248, 260, 259, 246, 255, 271, 231, 251, 231, 271, 250, 274, 242, 248, 256, 263, 239, 253, 276, 248, 278, 244, 268, 259, 247, 249, 259, 240, 272, 259, 267, 255, 245, 217, 274, 275, 255, 247, 250, 232, 242, 261, 275, 242, 273, 276, 267, 226, 238] payload_rows_per_dst=[5468, 5588, 5454, 5536, 5520, 5442, 5562, 5360] dup_rows_per_dst=[2546, 2738, 2666, 2678, 2910, 2652, 2676, 2740] combine_rows_per_src=[5468, 5588, 5454, 5536, 5520, 5442, 5562, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229488 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156500080 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312770672 weight_movement_bytes=37748736 total_nvlink_bytes=350519408 weight_to_token_ratio=0.120691 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=228736 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157670784 ingress_bytes=195190784 link_bytes=195190784 token_bytes=315112832 weight_movement_bytes=37748736 total_nvlink_bytes=352861568 weight_to_token_ratio=0.119794 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870222052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157941760 dispatch_meta_bytes=229752 combine_payload_bytes=157941760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158171512 ingress_bytes=176816128 link_bytes=176816128 token_bytes=316113272 weight_movement_bytes=18874368 total_nvlink_bytes=334987640 weight_to_token_ratio=0.059708 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694165140e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229248 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228928 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228608 weight_movement_bytes=18874368 total_nvlink_bytes=333102976 weight_to_token_ratio=0.060066 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38400 dup_rows=21728 dup_rows_remote=19012 dedup_ratio=0.331543 router_tokens_per_expert=[259, 203, 254, 252, 253, 239, 257, 285, 254, 242, 260, 265, 261, 225, 260, 255, 260, 245, 231, 238, 264, 289, 271, 250, 258, 277, 249, 262, 276, 242, 273, 258, 256, 269, 249, 260, 258, 265, 281, 281, 256, 249, 268, 252, 265, 238, 244, 274, 259, 248, 248, 254, 259, 245, 301, 264, 251, 215, 279, 265, 240, 241, 252, 241, 261, 273, 217, 241, 264, 275, 270, 250, 261, 241, 249, 237, 249, 253, 270, 251, 255, 254, 269, 251, 254, 265, 272, 270, 263, 253, 267, 223, 273, 231, 256, 286, 240, 219, 249, 248, 249, 254, 255, 284, 257, 276, 227, 258, 247, 258, 317, 263, 247, 254, 251, 264, 237, 279, 276, 255, 273, 258, 255, 249, 235, 262, 234, 240] payload_rows_per_dst=[5430, 5532, 5532, 5408, 5420, 5540, 5502, 5444] dup_rows_per_dst=[2618, 2754, 2798, 2716, 2704, 2744, 2700, 2694] combine_rows_per_src=[5430, 5532, 5532, 5408, 5420, 5540, 5502, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=228840 combine_payload_bytes=157048832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157277672 ingress_bytes=194797568 link_bytes=194797568 token_bytes=314326504 weight_movement_bytes=37748736 total_nvlink_bytes=352075240 weight_to_token_ratio=0.120094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866454450e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38396 dup_rows=21644 dup_rows_remote=18906 dedup_ratio=0.330261 router_tokens_per_expert=[262, 239, 274, 256, 243, 255, 253, 248, 231, 237, 238, 227, 262, 265, 234, 246, 269, 291, 263, 276, 282, 261, 239, 237, 246, 242, 267, 273, 224, 244, 274, 234, 272, 287, 246, 242, 252, 246, 266, 228, 262, 263, 232, 274, 258, 270, 264, 255, 282, 256, 257, 248, 254, 255, 264, 236, 261, 270, 262, 274, 240, 272, 296, 275, 247, 273, 270, 263, 241, 227, 254, 284, 255, 251, 260, 261, 244, 261, 232, 263, 279, 279, 263, 257, 248, 266, 296, 234, 243, 257, 251, 241, 239, 268, 272, 270, 261, 253, 268, 239, 256, 243, 245, 261, 252, 255, 246, 242, 263, 274, 249, 241, 263, 273, 261, 267, 240, 264, 260, 247, 242, 263, 223, 244, 263, 266, 236, 248] payload_rows_per_dst=[5332, 5542, 5496, 5606, 5484, 5526, 5436, 5470] dup_rows_per_dst=[2608, 2702, 2738, 2798, 2688, 2800, 2660, 2650] combine_rows_per_src=[5332, 5542, 5496, 5606, 5484, 5526, 5436, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229648 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157516048 ingress_bytes=232783872 link_bytes=232783872 token_bytes=314802448 weight_movement_bytes=75497472 total_nvlink_bytes=390299920 weight_to_token_ratio=0.239825 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230420525e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229208 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157499224 ingress_bytes=176144384 link_bytes=176144384 token_bytes=314769240 weight_movement_bytes=18874368 total_nvlink_bytes=333643608 weight_to_token_ratio=0.059963 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687728819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38266 dup_rows=21766 dup_rows_remote=18818 dedup_ratio=0.332123 router_tokens_per_expert=[263, 274, 269, 268, 231, 261, 262, 260, 267, 259, 238, 269, 277, 237, 249, 260, 262, 295, 214, 274, 244, 249, 275, 273, 238, 264, 259, 252, 262, 223, 264, 242, 260, 274, 250, 256, 239, 250, 265, 232, 244, 273, 277, 245, 276, 240, 238, 290, 235, 272, 271, 242, 239, 260, 255, 241, 248, 269, 261, 252, 250, 278, 237, 271, 278, 245, 299, 270, 251, 246, 265, 245, 260, 269, 243, 236, 287, 278, 262, 274, 227, 265, 258, 248, 277, 258, 246, 236, 232, 236, 250, 245, 257, 251, 292, 252, 243, 245, 261, 253, 253, 244, 283, 238, 259, 258, 264, 232, 238, 281, 247, 242, 273, 279, 248, 252, 220, 262, 263, 233, 257, 244, 265, 242, 278, 251, 247, 251] payload_rows_per_dst=[5504, 5492, 5560, 5464, 5552, 5406, 5310, 5482] dup_rows_per_dst=[2948, 2688, 2678, 2732, 2864, 2654, 2554, 2648] combine_rows_per_src=[5504, 5492, 5560, 5464, 5552, 5406, 5310, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38182 dup_rows=21806 dup_rows_remote=19020 dedup_ratio=0.332733 router_tokens_per_expert=[290, 260, 280, 232, 287, 264, 262, 242, 250, 259, 265, 259, 254, 246, 266, 224, 280, 245, 248, 271, 293, 264, 258, 267, 237, 217, 262, 233, 274, 258, 247, 252, 251, 258, 240, 253, 280, 233, 251, 225, 255, 239, 234, 253, 248, 245, 276, 276, 267, 266, 254, 255, 244, 254, 257, 223, 248, 258, 247, 259, 258, 243, 256, 262, 227, 263, 258, 249, 294, 250, 252, 260, 258, 278, 260, 265, 242, 263, 278, 270, 257, 261, 256, 233, 281, 263, 267, 250, 265, 225, 248, 242, 249, 256, 270, 238, 228, 262, 232, 255, 256, 242, 273, 269, 282, 249, 239, 262, 249, 285, 241, 254, 241, 261, 252, 266, 269, 250, 257, 273, 256, 295, 286, 248, 245, 252, 260, 237] payload_rows_per_dst=[5512, 5494, 5432, 5408, 5548, 5384, 5418, 5534] dup_rows_per_dst=[2768, 2718, 2602, 2694, 2786, 2738, 2738, 2762] combine_rows_per_src=[5512, 5494, 5432, 5408, 5548, 5384, 5418, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=37960 dup_rows=21918 dup_rows_remote=18892 dedup_ratio=0.334442 router_tokens_per_expert=[254, 290, 274, 263, 218, 240, 273, 263, 235, 261, 257, 236, 283, 268, 275, 231, 247, 246, 249, 248, 249, 267, 277, 271, 237, 273, 254, 282, 268, 263, 250, 232, 262, 257, 247, 268, 225, 283, 258, 259, 248, 248, 257, 259, 263, 239, 244, 242, 229, 267, 257, 278, 252, 252, 242, 275, 255, 271, 251, 233, 257, 265, 256, 243, 225, 257, 263, 231, 260, 286, 243, 248, 232, 229, 255, 256, 273, 260, 253, 267, 269, 261, 257, 256, 247, 244, 263, 271, 257, 270, 259, 260, 271, 256, 270, 245, 262, 280, 245, 250, 239, 247, 270, 289, 252, 262, 240, 227, 241, 282, 274, 245, 273, 260, 253, 241, 267, 249, 267, 263, 263, 245, 240, 235, 270, 269, 240, 258] payload_rows_per_dst=[5482, 5254, 5448, 5446, 5410, 5414, 5506, 5658] dup_rows_per_dst=[2760, 2474, 2670, 2720, 2666, 2682, 2920, 3026] combine_rows_per_src=[5482, 5254, 5448, 5446, 5410, 5414, 5506, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38296 dup_rows=21808 dup_rows_remote=19030 dedup_ratio=0.332764 router_tokens_per_expert=[266, 249, 244, 260, 284, 271, 283, 253, 260, 228, 243, 255, 249, 256, 241, 259, 261, 242, 268, 282, 307, 210, 226, 261, 237, 234, 255, 270, 254, 264, 243, 278, 234, 260, 252, 260, 257, 276, 230, 256, 230, 265, 263, 267, 253, 246, 242, 259, 288, 266, 263, 251, 290, 281, 277, 260, 236, 255, 245, 256, 268, 251, 260, 246, 282, 247, 246, 261, 260, 274, 250, 237, 264, 251, 249, 272, 263, 269, 259, 250, 235, 246, 262, 248, 251, 261, 277, 260, 270, 274, 255, 254, 249, 274, 256, 233, 257, 230, 247, 263, 270, 236, 259, 246, 257, 258, 250, 260, 247, 266, 245, 288, 280, 255, 233, 238, 235, 234, 261, 258, 223, 257, 238, 277, 275, 255, 246, 249] payload_rows_per_dst=[5534, 5412, 5432, 5572, 5438, 5432, 5450, 5458] dup_rows_per_dst=[2668, 2772, 2668, 2814, 2830, 2778, 2708, 2570] combine_rows_per_src=[5534, 5412, 5432, 5572, 5438, 5432, 5450, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38406 dup_rows=21700 dup_rows_remote=18912 dedup_ratio=0.331116 router_tokens_per_expert=[280, 291, 232, 279, 240, 264, 253, 237, 224, 278, 261, 259, 269, 242, 264, 274, 237, 258, 253, 242, 262, 274, 267, 250, 290, 268, 251, 250, 255, 241, 285, 262, 265, 255, 262, 243, 265, 242, 245, 230, 254, 247, 255, 266, 258, 237, 277, 258, 249, 248, 244, 251, 267, 243, 264, 282, 249, 269, 249, 272, 254, 268, 277, 261, 250, 241, 266, 263, 246, 267, 240, 252, 237, 257, 242, 253, 269, 251, 246, 212, 254, 260, 239, 279, 252, 244, 273, 269, 270, 272, 262, 242, 257, 273, 252, 230, 242, 227, 264, 268, 267, 263, 266, 225, 247, 262, 267, 266, 287, 263, 238, 257, 242, 245, 251, 234, 233, 248, 267, 273, 253, 252, 244, 274, 257, 266, 254, 248] payload_rows_per_dst=[5538, 5448, 5488, 5590, 5330, 5518, 5430, 5494] dup_rows_per_dst=[2756, 2786, 2630, 2704, 2654, 2738, 2788, 2644] combine_rows_per_src=[5538, 5448, 5488, 5590, 5330, 5518, 5430, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228336 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156965872 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313703408 weight_movement_bytes=37748736 total_nvlink_bytes=351452144 weight_to_token_ratio=0.120333 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38244 dup_rows=21920 dup_rows_remote=19260 dedup_ratio=0.334473 router_tokens_per_expert=[256, 269, 295, 221, 256, 246, 267, 275, 262, 234, 259, 233, 268, 274, 281, 253, 250, 269, 241, 258, 261, 229, 254, 264, 240, 207, 253, 239, 240, 273, 273, 265, 253, 254, 280, 243, 274, 260, 287, 268, 239, 255, 283, 275, 253, 257, 250, 217, 250, 270, 263, 269, 248, 249, 269, 249, 255, 262, 272, 246, 242, 246, 251, 290, 249, 257, 266, 264, 254, 260, 247, 253, 277, 267, 254, 267, 237, 244, 253, 252, 256, 245, 274, 240, 271, 283, 223, 261, 270, 277, 261, 285, 242, 254, 254, 270, 257, 241, 249, 254, 259, 273, 262, 249, 265, 235, 246, 231, 250, 259, 264, 245, 269, 247, 244, 256, 255, 277, 270, 240, 231, 237, 248, 254, 237, 256, 262, 235] payload_rows_per_dst=[5474, 5372, 5500, 5460, 5540, 5492, 5412, 5366] dup_rows_per_dst=[2824, 2660, 2796, 2802, 2662, 2840, 2666, 2670] combine_rows_per_src=[5474, 5372, 5500, 5460, 5540, 5492, 5412, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228808 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622280 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015752 weight_movement_bytes=18874368 total_nvlink_bytes=331890120 weight_to_token_ratio=0.060298 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155484160 dispatch_meta_bytes=227408 combine_payload_bytes=155484160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155711568 ingress_bytes=174358528 link_bytes=174358528 token_bytes=311195728 weight_movement_bytes=18874368 total_nvlink_bytes=330070096 weight_to_token_ratio=0.060651 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670617626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43490 payload_rows_remote=38024 dup_rows=22046 dup_rows_remote=19378 dedup_ratio=0.336395 router_tokens_per_expert=[253, 286, 242, 252, 280, 246, 235, 240, 265, 248, 258, 254, 271, 270, 239, 242, 249, 248, 266, 258, 265, 261, 238, 258, 243, 250, 239, 248, 253, 261, 259, 263, 268, 241, 259, 249, 268, 292, 252, 265, 246, 236, 256, 260, 278, 248, 265, 239, 235, 248, 249, 265, 275, 275, 251, 266, 275, 251, 242, 255, 263, 237, 256, 224, 287, 249, 240, 277, 260, 263, 252, 240, 260, 252, 271, 248, 276, 256, 264, 252, 262, 259, 270, 237, 277, 254, 264, 248, 258, 242, 222, 255, 281, 234, 280, 241, 244, 271, 277, 248, 267, 257, 263, 250, 238, 259, 265, 278, 257, 281, 250, 241, 265, 257, 252, 258, 258, 241, 222, 258, 252, 244, 289, 254, 268, 253, 244, 247] payload_rows_per_dst=[5414, 5406, 5434, 5466, 5420, 5422, 5556, 5372] dup_rows_per_dst=[2748, 2712, 2810, 2668, 2874, 2746, 2736, 2752] combine_rows_per_src=[5414, 5406, 5434, 5466, 5420, 5422, 5556, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229304 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157089720 ingress_bytes=213483520 link_bytes=213483520 token_bytes=313950136 weight_movement_bytes=56623104 total_nvlink_bytes=370573240 weight_to_token_ratio=0.180357 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045494048e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229272 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157540248 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314851224 weight_movement_bytes=18874368 total_nvlink_bytes=333725592 weight_to_token_ratio=0.059947 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43950 payload_rows_remote=38456 dup_rows=21586 dup_rows_remote=18992 dedup_ratio=0.329376 router_tokens_per_expert=[244, 244, 235, 252, 280, 274, 259, 287, 272, 270, 274, 286, 229, 254, 230, 260, 245, 242, 249, 248, 231, 253, 261, 277, 267, 260, 239, 251, 246, 237, 234, 291, 243, 236, 263, 258, 236, 256, 253, 261, 264, 242, 234, 271, 253, 244, 253, 277, 238, 283, 255, 245, 250, 274, 263, 290, 230, 269, 271, 219, 251, 263, 245, 268, 261, 255, 211, 262, 290, 250, 228, 248, 221, 258, 260, 254, 244, 250, 263, 254, 246, 275, 246, 270, 282, 276, 268, 261, 235, 274, 259, 256, 248, 266, 289, 265, 294, 253, 234, 264, 246, 255, 277, 243, 257, 262, 227, 224, 251, 296, 266, 275, 250, 257, 266, 222, 259, 266, 269, 257, 240, 267, 269, 250, 266, 217, 286, 239] payload_rows_per_dst=[5500, 5428, 5494, 5556, 5464, 5534, 5496, 5478] dup_rows_per_dst=[2800, 2634, 2594, 2672, 2554, 2898, 2752, 2682] combine_rows_per_src=[5500, 5428, 5494, 5556, 5464, 5534, 5496, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=230016 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156877440 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524864 weight_movement_bytes=37748736 total_nvlink_bytes=351273600 weight_to_token_ratio=0.120401 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155746304 dispatch_meta_bytes=229608 combine_payload_bytes=155746304 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155975912 ingress_bytes=212369408 link_bytes=212369408 token_bytes=311722216 weight_movement_bytes=56623104 total_nvlink_bytes=368345320 weight_to_token_ratio=0.181646 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034819175e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157515776 dispatch_meta_bytes=229792 combine_payload_bytes=157515776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157745568 ingress_bytes=176390144 link_bytes=176390144 token_bytes=315261344 weight_movement_bytes=18874368 total_nvlink_bytes=334135712 weight_to_token_ratio=0.059869 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690083570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38204 dup_rows=21830 dup_rows_remote=19008 dedup_ratio=0.333099 router_tokens_per_expert=[257, 252, 252, 275, 245, 228, 256, 257, 247, 261, 227, 276, 263, 270, 266, 265, 282, 240, 266, 271, 253, 234, 257, 256, 272, 273, 232, 249, 278, 263, 273, 249, 242, 268, 253, 255, 243, 264, 234, 253, 261, 253, 253, 240, 249, 246, 239, 259, 263, 234, 247, 267, 245, 231, 252, 261, 246, 270, 288, 274, 264, 252, 278, 272, 227, 261, 239, 270, 252, 249, 280, 272, 259, 232, 255, 255, 257, 242, 254, 254, 270, 231, 237, 248, 257, 247, 275, 247, 234, 240, 249, 244, 241, 270, 243, 260, 254, 279, 259, 268, 277, 255, 250, 225, 249, 279, 239, 254, 234, 249, 266, 252, 264, 264, 280, 284, 255, 246, 253, 284, 249, 268, 269, 241, 276, 232, 287, 275] payload_rows_per_dst=[5502, 5710, 5392, 5484, 5456, 5364, 5586, 5212] dup_rows_per_dst=[2822, 3124, 2632, 2688, 2646, 2752, 2844, 2322] combine_rows_per_src=[5502, 5710, 5392, 5484, 5456, 5364, 5586, 5212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38324 dup_rows=21610 dup_rows_remote=18784 dedup_ratio=0.329742 router_tokens_per_expert=[250, 251, 247, 243, 251, 255, 263, 238, 242, 249, 254, 267, 246, 262, 246, 263, 255, 244, 259, 252, 238, 260, 236, 250, 236, 264, 240, 225, 255, 246, 260, 247, 226, 253, 277, 266, 250, 259, 301, 293, 238, 277, 250, 250, 235, 254, 251, 275, 267, 277, 255, 244, 234, 292, 277, 279, 249, 271, 276, 240, 259, 269, 259, 245, 241, 236, 276, 272, 277, 284, 254, 279, 283, 272, 284, 279, 268, 252, 211, 246, 240, 253, 243, 251, 257, 229, 246, 267, 266, 258, 250, 256, 265, 274, 256, 260, 266, 270, 251, 250, 262, 225, 237, 239, 241, 258, 256, 234, 267, 266, 243, 281, 260, 253, 264, 271, 251, 227, 257, 230, 258, 264, 221, 269, 270, 276, 260, 264] payload_rows_per_dst=[5474, 5324, 5570, 5556, 5602, 5488, 5428, 5484] dup_rows_per_dst=[2580, 2610, 2740, 2830, 2826, 2654, 2664, 2706] combine_rows_per_src=[5474, 5324, 5570, 5556, 5602, 5488, 5428, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38130 dup_rows=21866 dup_rows_remote=19254 dedup_ratio=0.333649 router_tokens_per_expert=[260, 240, 237, 239, 257, 275, 252, 253, 280, 267, 245, 280, 266, 266, 285, 260, 289, 269, 255, 247, 250, 266, 268, 245, 223, 264, 267, 241, 260, 262, 258, 232, 261, 276, 244, 265, 256, 227, 240, 246, 265, 249, 258, 243, 265, 228, 258, 247, 264, 267, 253, 239, 239, 273, 233, 259, 213, 263, 268, 283, 224, 262, 257, 238, 273, 242, 256, 245, 261, 246, 299, 244, 240, 276, 254, 257, 270, 263, 269, 293, 266, 287, 248, 251, 245, 231, 242, 255, 281, 260, 251, 251, 272, 282, 249, 267, 228, 247, 262, 266, 265, 261, 261, 269, 235, 268, 252, 234, 242, 254, 268, 254, 278, 299, 235, 259, 290, 259, 264, 262, 268, 229, 227, 236, 223, 269, 219, 238] payload_rows_per_dst=[5502, 5422, 5374, 5384, 5570, 5446, 5432, 5540] dup_rows_per_dst=[2812, 2770, 2682, 2566, 2894, 2814, 2716, 2612] combine_rows_per_src=[5502, 5422, 5374, 5384, 5570, 5446, 5432, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38156 dup_rows=21852 dup_rows_remote=19140 dedup_ratio=0.333435 router_tokens_per_expert=[276, 262, 267, 259, 299, 250, 253, 256, 240, 253, 220, 262, 261, 267, 287, 251, 229, 263, 262, 254, 278, 265, 256, 293, 272, 241, 257, 267, 268, 260, 269, 266, 256, 258, 263, 255, 266, 264, 204, 231, 241, 248, 264, 290, 253, 234, 238, 238, 257, 258, 283, 246, 258, 254, 246, 260, 254, 239, 248, 261, 238, 280, 247, 242, 280, 250, 242, 259, 243, 269, 273, 242, 273, 247, 237, 263, 248, 255, 271, 239, 249, 281, 238, 260, 261, 261, 252, 292, 236, 241, 253, 246, 251, 266, 273, 237, 260, 250, 281, 260, 256, 244, 253, 285, 268, 273, 253, 251, 246, 250, 251, 239, 258, 251, 237, 242, 242, 253, 250, 254, 217, 245, 233, 283, 256, 261, 279, 262] payload_rows_per_dst=[5504, 5586, 5354, 5356, 5488, 5408, 5528, 5460] dup_rows_per_dst=[2822, 2814, 2652, 2786, 2694, 2786, 2712, 2586] combine_rows_per_src=[5504, 5586, 5354, 5356, 5488, 5408, 5528, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38406 dup_rows=21620 dup_rows_remote=18932 dedup_ratio=0.329895 router_tokens_per_expert=[248, 266, 254, 233, 275, 220, 277, 254, 247, 256, 278, 275, 268, 271, 262, 235, 250, 252, 253, 265, 265, 257, 256, 284, 246, 238, 287, 249, 269, 250, 258, 237, 247, 265, 256, 264, 244, 250, 245, 244, 240, 226, 280, 245, 247, 270, 260, 239, 254, 245, 252, 268, 258, 236, 267, 276, 254, 276, 248, 276, 260, 269, 246, 275, 275, 254, 280, 253, 257, 249, 271, 229, 245, 288, 256, 242, 243, 305, 259, 235, 247, 233, 272, 252, 269, 255, 278, 248, 241, 242, 252, 258, 260, 266, 253, 273, 220, 268, 255, 247, 247, 251, 262, 238, 255, 232, 268, 233, 275, 255, 245, 233, 261, 269, 245, 254, 273, 260, 255, 236, 252, 248, 268, 288, 257, 240, 271, 250] payload_rows_per_dst=[5544, 5448, 5414, 5538, 5508, 5510, 5390, 5564] dup_rows_per_dst=[2694, 2784, 2630, 2782, 2774, 2688, 2578, 2690] combine_rows_per_src=[5544, 5448, 5414, 5538, 5508, 5510, 5390, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228848 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712432 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196016 weight_movement_bytes=37748736 total_nvlink_bytes=350944752 weight_to_token_ratio=0.120528 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38354 dup_rows=21712 dup_rows_remote=19016 dedup_ratio=0.331299 router_tokens_per_expert=[240, 276, 268, 242, 235, 252, 229, 261, 249, 229, 265, 254, 244, 255, 241, 256, 286, 254, 268, 263, 267, 244, 260, 249, 237, 246, 241, 295, 237, 233, 267, 236, 256, 251, 253, 274, 252, 266, 269, 245, 248, 276, 241, 263, 228, 273, 251, 245, 247, 252, 266, 252, 242, 265, 274, 260, 271, 258, 257, 254, 257, 250, 287, 228, 257, 272, 274, 237, 258, 268, 267, 275, 257, 251, 251, 250, 268, 281, 237, 235, 240, 262, 263, 244, 277, 279, 260, 257, 240, 262, 249, 255, 278, 259, 240, 268, 256, 251, 243, 254, 266, 273, 256, 267, 236, 266, 266, 263, 257, 229, 269, 254, 257, 278, 253, 255, 245, 235, 259, 263, 254, 267, 270, 246, 261, 245, 271, 242] payload_rows_per_dst=[5384, 5470, 5500, 5466, 5562, 5500, 5470, 5472] dup_rows_per_dst=[2608, 2696, 2682, 2774, 2714, 2766, 2742, 2730] combine_rows_per_src=[5384, 5470, 5500, 5466, 5562, 5500, 5470, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=228432 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157203536 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314178640 weight_movement_bytes=37748736 total_nvlink_bytes=351927376 weight_to_token_ratio=0.120151 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229536 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156410016 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312590496 weight_movement_bytes=18874368 total_nvlink_bytes=331464864 weight_to_token_ratio=0.060380 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38284 dup_rows=21646 dup_rows_remote=18906 dedup_ratio=0.330292 router_tokens_per_expert=[223, 314, 239, 275, 264, 277, 261, 246, 250, 242, 252, 275, 270, 251, 240, 250, 230, 248, 279, 249, 255, 256, 253, 263, 260, 272, 256, 261, 264, 273, 246, 240, 257, 246, 264, 252, 224, 262, 278, 239, 234, 276, 248, 264, 265, 230, 273, 272, 244, 270, 258, 276, 294, 252, 260, 258, 287, 266, 258, 263, 285, 205, 240, 257, 227, 281, 237, 250, 230, 265, 260, 250, 265, 263, 255, 284, 268, 247, 261, 254, 253, 224, 245, 238, 241, 255, 273, 268, 225, 253, 256, 281, 259, 255, 268, 228, 234, 250, 257, 278, 229, 268, 283, 261, 259, 272, 278, 249, 250, 262, 249, 238, 248, 244, 262, 257, 253, 255, 255, 244, 273, 266, 246, 250, 253, 243, 235, 257] payload_rows_per_dst=[5506, 5518, 5410, 5606, 5552, 5414, 5478, 5406] dup_rows_per_dst=[2752, 2692, 2758, 2740, 2642, 2630, 2756, 2676] combine_rows_per_src=[5506, 5518, 5410, 5606, 5552, 5414, 5478, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229184 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156516160 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312803136 weight_movement_bytes=18874368 total_nvlink_bytes=331677504 weight_to_token_ratio=0.060339 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38348 dup_rows=21686 dup_rows_remote=18880 dedup_ratio=0.330902 router_tokens_per_expert=[254, 252, 265, 278, 248, 243, 238, 266, 234, 240, 249, 231, 279, 241, 245, 279, 270, 249, 250, 254, 276, 253, 270, 266, 273, 244, 258, 246, 244, 258, 248, 251, 227, 242, 278, 266, 265, 276, 251, 277, 265, 253, 255, 269, 244, 250, 271, 265, 242, 257, 272, 256, 222, 271, 234, 241, 262, 287, 270, 255, 264, 263, 274, 283, 257, 253, 243, 276, 240, 266, 240, 270, 276, 238, 275, 244, 265, 259, 259, 240, 245, 267, 251, 248, 242, 244, 232, 255, 261, 259, 231, 251, 265, 256, 262, 270, 245, 274, 260, 261, 251, 236, 264, 253, 247, 244, 240, 231, 243, 264, 246, 264, 254, 308, 269, 275, 262, 271, 277, 263, 273, 245, 245, 238, 227, 234, 255, 250] payload_rows_per_dst=[5394, 5520, 5502, 5532, 5484, 5494, 5462, 5462] dup_rows_per_dst=[2690, 2700, 2806, 2774, 2718, 2584, 2584, 2830] combine_rows_per_src=[5394, 5520, 5502, 5532, 5484, 5494, 5462, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229352 combine_payload_bytes=157310976 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157540328 ingress_bytes=213934080 link_bytes=213934080 token_bytes=314851304 weight_movement_bytes=56623104 total_nvlink_bytes=371474408 weight_to_token_ratio=0.179841 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049811092e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229480 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327464 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425448 weight_movement_bytes=18874368 total_nvlink_bytes=333299816 weight_to_token_ratio=0.060028 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228760 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040024 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851288 weight_movement_bytes=37748736 total_nvlink_bytes=351600024 weight_to_token_ratio=0.120276 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=228912 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157302320 ingress_bytes=213696512 link_bytes=213696512 token_bytes=314375728 weight_movement_bytes=56623104 total_nvlink_bytes=370998832 weight_to_token_ratio=0.180113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047534833e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38310 dup_rows=21830 dup_rows_remote=19058 dedup_ratio=0.333099 router_tokens_per_expert=[253, 250, 257, 231, 261, 226, 256, 279, 237, 237, 251, 260, 262, 255, 252, 294, 271, 282, 260, 271, 253, 273, 273, 254, 261, 243, 289, 288, 259, 255, 235, 227, 280, 224, 268, 248, 266, 277, 254, 247, 242, 284, 264, 263, 268, 228, 238, 259, 253, 264, 272, 249, 255, 271, 295, 258, 223, 253, 236, 255, 258, 290, 236, 262, 265, 238, 245, 233, 238, 263, 264, 249, 287, 253, 262, 259, 273, 264, 242, 249, 269, 258, 259, 252, 240, 241, 270, 242, 274, 253, 242, 257, 248, 282, 258, 257, 222, 295, 236, 252, 235, 250, 243, 269, 274, 259, 238, 242, 266, 260, 218, 253, 271, 254, 228, 243, 242, 276, 256, 258, 242, 272, 264, 258, 231, 273, 254, 253] payload_rows_per_dst=[5426, 5526, 5552, 5514, 5396, 5478, 5354, 5460] dup_rows_per_dst=[2696, 2862, 2668, 2746, 2772, 2726, 2670, 2690] combine_rows_per_src=[5426, 5526, 5552, 5514, 5396, 5478, 5354, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38232 dup_rows=21852 dup_rows_remote=19144 dedup_ratio=0.333435 router_tokens_per_expert=[250, 230, 266, 248, 280, 236, 263, 263, 263, 280, 251, 252, 250, 246, 237, 265, 258, 255, 265, 273, 259, 246, 272, 240, 272, 266, 265, 228, 297, 301, 254, 238, 255, 269, 252, 249, 234, 273, 259, 265, 260, 241, 255, 241, 278, 248, 262, 238, 259, 286, 254, 252, 265, 273, 283, 279, 266, 235, 259, 252, 258, 252, 251, 275, 271, 256, 220, 228, 245, 263, 268, 246, 232, 270, 263, 248, 256, 272, 239, 272, 238, 258, 232, 252, 233, 259, 241, 266, 235, 243, 270, 245, 258, 256, 277, 245, 236, 237, 243, 278, 268, 262, 274, 230, 264, 246, 258, 241, 254, 247, 289, 275, 252, 234, 255, 257, 261, 267, 261, 241, 247, 251, 273, 261, 274, 233, 237, 258] payload_rows_per_dst=[5452, 5532, 5536, 5484, 5410, 5398, 5444, 5428] dup_rows_per_dst=[2708, 2846, 2778, 2758, 2688, 2618, 2760, 2696] combine_rows_per_src=[5452, 5532, 5536, 5484, 5410, 5398, 5444, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38126 dup_rows=21680 dup_rows_remote=18804 dedup_ratio=0.330811 router_tokens_per_expert=[283, 243, 266, 254, 249, 256, 237, 249, 259, 267, 265, 268, 257, 239, 276, 283, 239, 244, 273, 245, 245, 262, 264, 258, 266, 253, 250, 245, 243, 256, 272, 264, 233, 246, 303, 285, 266, 266, 221, 274, 248, 238, 260, 253, 255, 240, 272, 258, 257, 250, 248, 264, 268, 265, 227, 264, 263, 236, 296, 266, 260, 251, 248, 263, 273, 238, 231, 250, 273, 246, 239, 250, 238, 287, 264, 249, 253, 266, 261, 266, 243, 251, 257, 263, 257, 250, 266, 249, 268, 238, 252, 271, 260, 261, 223, 280, 293, 244, 245, 243, 265, 247, 291, 274, 229, 233, 272, 255, 253, 240, 260, 239, 247, 248, 257, 253, 239, 249, 259, 223, 255, 238, 261, 242, 286, 255, 269, 257] payload_rows_per_dst=[5478, 5360, 5288, 5308, 5470, 5670, 5552, 5730] dup_rows_per_dst=[2594, 2690, 2556, 2544, 2712, 2886, 2822, 2876] combine_rows_per_src=[5478, 5360, 5288, 5308, 5470, 5670, 5552, 5730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38382 dup_rows=21634 dup_rows_remote=18858 dedup_ratio=0.330109 router_tokens_per_expert=[242, 244, 268, 247, 235, 284, 250, 261, 270, 262, 244, 246, 248, 277, 270, 290, 233, 259, 255, 257, 263, 232, 237, 240, 225, 239, 243, 257, 260, 277, 251, 221, 247, 239, 248, 281, 266, 262, 273, 244, 255, 276, 268, 272, 276, 215, 242, 253, 258, 262, 260, 256, 233, 279, 253, 273, 263, 225, 246, 244, 231, 263, 249, 242, 277, 284, 258, 288, 249, 261, 253, 264, 254, 267, 259, 261, 240, 263, 257, 265, 254, 275, 241, 253, 239, 278, 279, 268, 262, 253, 261, 238, 283, 267, 238, 240, 282, 231, 284, 264, 253, 270, 268, 271, 268, 283, 264, 250, 235, 278, 236, 211, 253, 257, 239, 251, 255, 260, 271, 271, 270, 249, 236, 244, 269, 265, 236, 224] payload_rows_per_dst=[5552, 5426, 5452, 5372, 5616, 5502, 5520, 5462] dup_rows_per_dst=[2724, 2472, 2782, 2570, 2784, 2756, 2776, 2770] combine_rows_per_src=[5552, 5426, 5452, 5372, 5616, 5502, 5520, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229472 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157147232 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314064992 weight_movement_bytes=18874368 total_nvlink_bytes=332939360 weight_to_token_ratio=0.060097 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38284 dup_rows=21790 dup_rows_remote=19048 dedup_ratio=0.332489 router_tokens_per_expert=[276, 253, 285, 224, 244, 238, 260, 290, 251, 255, 283, 272, 258, 234, 293, 248, 260, 272, 226, 249, 241, 248, 261, 271, 253, 256, 273, 258, 251, 272, 244, 251, 246, 238, 280, 299, 264, 249, 255, 273, 259, 252, 250, 260, 266, 258, 253, 252, 263, 247, 241, 267, 236, 265, 280, 251, 248, 261, 265, 269, 242, 279, 240, 246, 244, 246, 263, 244, 270, 262, 243, 286, 254, 265, 277, 248, 248, 261, 253, 273, 250, 264, 254, 247, 243, 258, 247, 267, 280, 261, 271, 238, 262, 260, 248, 252, 280, 259, 227, 249, 245, 261, 267, 234, 264, 228, 269, 254, 269, 241, 262, 233, 259, 264, 262, 226, 242, 217, 245, 252, 251, 251, 263, 256, 273, 229, 237, 256] payload_rows_per_dst=[5568, 5454, 5534, 5462, 5484, 5462, 5456, 5326] dup_rows_per_dst=[2760, 2718, 2774, 2738, 2790, 2742, 2628, 2640] combine_rows_per_src=[5568, 5454, 5534, 5462, 5484, 5462, 5456, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38092 dup_rows=21878 dup_rows_remote=19070 dedup_ratio=0.333832 router_tokens_per_expert=[243, 253, 257, 276, 251, 254, 262, 261, 254, 242, 222, 260, 270, 269, 238, 242, 264, 257, 241, 276, 223, 223, 260, 264, 257, 264, 303, 290, 274, 254, 259, 278, 237, 241, 251, 278, 263, 246, 273, 270, 246, 251, 252, 260, 278, 262, 278, 229, 292, 273, 251, 243, 261, 235, 260, 243, 268, 228, 275, 273, 286, 251, 283, 252, 275, 233, 233, 225, 248, 268, 257, 274, 281, 237, 239, 258, 285, 225, 244, 238, 235, 271, 244, 256, 234, 254, 258, 248, 257, 264, 262, 248, 273, 242, 279, 269, 244, 258, 257, 268, 282, 254, 262, 277, 240, 258, 269, 241, 287, 255, 256, 234, 251, 241, 228, 233, 240, 239, 233, 252, 248, 237, 287, 263, 243, 254, 270, 263] payload_rows_per_dst=[5458, 5566, 5460, 5536, 5366, 5436, 5494, 5342] dup_rows_per_dst=[2650, 2808, 2770, 2812, 2674, 2752, 2790, 2622] combine_rows_per_src=[5458, 5566, 5460, 5536, 5366, 5436, 5494, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229504 combine_payload_bytes=156598272 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156827776 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313426048 weight_movement_bytes=37748736 total_nvlink_bytes=351174784 weight_to_token_ratio=0.120439 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=227720 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156391816 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312555912 weight_movement_bytes=18874368 total_nvlink_bytes=331430280 weight_to_token_ratio=0.060387 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38416 dup_rows=21608 dup_rows_remote=18828 dedup_ratio=0.329712 router_tokens_per_expert=[246, 240, 249, 248, 250, 254, 256, 253, 219, 241, 278, 268, 277, 271, 264, 251, 254, 243, 247, 239, 245, 276, 266, 226, 252, 263, 253, 251, 248, 271, 245, 234, 254, 262, 274, 291, 288, 263, 266, 250, 231, 266, 267, 265, 268, 215, 219, 282, 284, 252, 256, 292, 273, 255, 293, 253, 247, 234, 258, 249, 234, 266, 253, 247, 271, 259, 261, 237, 266, 256, 272, 260, 241, 237, 255, 248, 261, 235, 282, 230, 241, 225, 262, 277, 250, 269, 235, 254, 250, 274, 249, 238, 243, 267, 249, 272, 243, 262, 255, 253, 262, 296, 295, 264, 260, 253, 247, 239, 253, 284, 276, 242, 251, 272, 245, 264, 285, 230, 252, 262, 226, 234, 261, 268, 250, 245, 236, 292] payload_rows_per_dst=[5542, 5418, 5524, 5512, 5510, 5450, 5518, 5454] dup_rows_per_dst=[2588, 2608, 2798, 2780, 2632, 2660, 2850, 2692] combine_rows_per_src=[5542, 5418, 5524, 5512, 5510, 5450, 5518, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=228960 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157441632 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654304 weight_movement_bytes=18874368 total_nvlink_bytes=333528672 weight_to_token_ratio=0.059984 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38200 dup_rows=21772 dup_rows_remote=19074 dedup_ratio=0.332214 router_tokens_per_expert=[270, 247, 243, 271, 261, 278, 240, 251, 263, 254, 254, 240, 262, 265, 257, 248, 235, 283, 244, 231, 261, 253, 256, 239, 272, 231, 255, 249, 281, 240, 235, 242, 250, 274, 270, 259, 230, 242, 280, 275, 263, 252, 276, 258, 236, 250, 248, 268, 260, 258, 252, 260, 251, 273, 256, 257, 253, 248, 263, 274, 215, 255, 250, 258, 249, 251, 216, 232, 276, 253, 256, 226, 270, 226, 248, 261, 235, 259, 248, 248, 264, 274, 265, 284, 260, 265, 263, 275, 265, 258, 275, 269, 255, 255, 257, 228, 251, 275, 244, 255, 249, 254, 275, 262, 250, 254, 262, 276, 234, 242, 238, 284, 234, 280, 249, 281, 283, 254, 275, 260, 264, 251, 250, 276, 261, 248, 256, 250] payload_rows_per_dst=[5408, 5394, 5564, 5480, 5374, 5544, 5494, 5506] dup_rows_per_dst=[2800, 2620, 2698, 2686, 2534, 2880, 2716, 2838] combine_rows_per_src=[5408, 5394, 5564, 5480, 5374, 5544, 5494, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229328 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040592 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851856 weight_movement_bytes=18874368 total_nvlink_bytes=332726224 weight_to_token_ratio=0.060138 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228648 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156253480 ingress_bytes=193773568 link_bytes=193773568 token_bytes=312278312 weight_movement_bytes=37748736 total_nvlink_bytes=350027048 weight_to_token_ratio=0.120882 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856642986e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=228976 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157580912 ingress_bytes=195100672 link_bytes=195100672 token_bytes=314932848 weight_movement_bytes=37748736 total_nvlink_bytes=352681584 weight_to_token_ratio=0.119863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869358644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229096 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696296 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163496 weight_movement_bytes=37748736 total_nvlink_bytes=350912232 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38230 dup_rows=21788 dup_rows_remote=19040 dedup_ratio=0.332458 router_tokens_per_expert=[265, 258, 234, 288, 253, 241, 262, 252, 253, 274, 258, 240, 255, 238, 275, 277, 254, 269, 268, 254, 257, 263, 260, 273, 273, 245, 246, 250, 236, 254, 265, 289, 279, 246, 252, 240, 239, 262, 281, 239, 244, 274, 253, 269, 248, 268, 275, 262, 225, 239, 268, 241, 252, 240, 253, 245, 292, 251, 270, 273, 232, 281, 255, 282, 280, 234, 255, 284, 257, 261, 259, 254, 218, 285, 266, 270, 234, 255, 248, 273, 263, 256, 254, 260, 245, 217, 252, 271, 276, 261, 250, 248, 240, 247, 261, 275, 232, 271, 233, 268, 233, 228, 263, 269, 254, 264, 246, 233, 234, 268, 272, 229, 282, 244, 231, 214, 289, 271, 244, 278, 253, 250, 254, 254, 246, 248, 259, 236] payload_rows_per_dst=[5490, 5436, 5440, 5478, 5518, 5442, 5466, 5478] dup_rows_per_dst=[2756, 2876, 2822, 2720, 2748, 2710, 2528, 2628] combine_rows_per_src=[5490, 5436, 5440, 5478, 5518, 5442, 5466, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38102 dup_rows=21966 dup_rows_remote=19200 dedup_ratio=0.335175 router_tokens_per_expert=[271, 255, 254, 252, 211, 252, 257, 269, 261, 247, 248, 238, 241, 290, 255, 257, 255, 227, 270, 252, 242, 247, 261, 261, 230, 263, 262, 241, 219, 253, 238, 264, 257, 258, 220, 264, 253, 258, 254, 253, 260, 254, 275, 241, 256, 268, 272, 234, 257, 225, 269, 274, 236, 259, 275, 256, 262, 265, 299, 265, 238, 264, 271, 225, 269, 242, 257, 271, 264, 236, 242, 245, 247, 261, 276, 244, 276, 243, 279, 236, 242, 305, 224, 234, 259, 253, 284, 291, 281, 254, 278, 250, 266, 255, 246, 278, 239, 276, 295, 252, 266, 259, 242, 251, 234, 262, 259, 236, 254, 257, 258, 266, 257, 254, 263, 251, 271, 249, 264, 259, 268, 259, 280, 275, 245, 244, 247, 228] payload_rows_per_dst=[5400, 5380, 5468, 5342, 5558, 5538, 5416, 5468] dup_rows_per_dst=[2716, 2590, 2686, 2626, 2930, 2862, 2790, 2766] combine_rows_per_src=[5400, 5380, 5468, 5342, 5558, 5538, 5416, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38244 dup_rows=21838 dup_rows_remote=19070 dedup_ratio=0.333221 router_tokens_per_expert=[259, 244, 247, 258, 215, 222, 262, 237, 249, 257, 259, 279, 249, 233, 286, 248, 298, 260, 268, 242, 250, 266, 232, 249, 252, 237, 260, 284, 281, 269, 242, 238, 262, 270, 244, 240, 252, 250, 261, 288, 270, 241, 262, 268, 225, 277, 255, 253, 270, 235, 258, 260, 276, 244, 269, 287, 254, 233, 285, 248, 227, 233, 255, 252, 272, 247, 243, 264, 247, 248, 239, 267, 287, 278, 274, 258, 250, 269, 265, 225, 259, 247, 264, 254, 255, 274, 267, 253, 246, 241, 261, 250, 256, 245, 279, 258, 285, 238, 254, 237, 282, 231, 253, 259, 235, 260, 252, 265, 260, 237, 261, 259, 274, 246, 265, 262, 232, 286, 264, 269, 249, 228, 284, 238, 276, 239, 227, 283] payload_rows_per_dst=[5454, 5568, 5464, 5432, 5422, 5414, 5554, 5390] dup_rows_per_dst=[2768, 2688, 2772, 2740, 2630, 2804, 2888, 2548] combine_rows_per_src=[5454, 5568, 5464, 5432, 5422, 5414, 5554, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38120 dup_rows=21938 dup_rows_remote=19284 dedup_ratio=0.334747 router_tokens_per_expert=[243, 227, 279, 252, 264, 238, 234, 238, 249, 229, 256, 256, 261, 256, 240, 239, 265, 250, 239, 237, 273, 269, 245, 264, 266, 288, 251, 272, 247, 265, 241, 250, 243, 251, 264, 259, 248, 273, 253, 262, 280, 259, 279, 288, 278, 249, 240, 239, 266, 279, 267, 272, 238, 253, 268, 249, 262, 225, 255, 237, 268, 257, 249, 261, 275, 290, 249, 261, 269, 255, 250, 259, 238, 246, 268, 246, 251, 261, 239, 253, 248, 293, 261, 246, 259, 230, 248, 254, 254, 246, 241, 281, 256, 262, 257, 292, 270, 225, 243, 238, 275, 267, 278, 242, 246, 251, 261, 257, 249, 255, 262, 247, 255, 261, 276, 251, 248, 264, 256, 265, 264, 256, 235, 258, 241, 256, 262, 262] payload_rows_per_dst=[5316, 5430, 5466, 5458, 5496, 5466, 5478, 5488] dup_rows_per_dst=[2606, 2814, 2864, 2754, 2724, 2790, 2654, 2732] combine_rows_per_src=[5316, 5430, 5466, 5458, 5496, 5466, 5478, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229080 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819160 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409240 weight_movement_bytes=18874368 total_nvlink_bytes=332283608 weight_to_token_ratio=0.060223 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38174 dup_rows=21822 dup_rows_remote=19032 dedup_ratio=0.332977 router_tokens_per_expert=[252, 239, 242, 262, 273, 273, 272, 253, 288, 267, 242, 290, 263, 233, 229, 255, 247, 257, 267, 239, 261, 277, 275, 244, 258, 237, 261, 272, 263, 253, 228, 265, 251, 236, 261, 240, 245, 267, 261, 267, 257, 246, 264, 267, 259, 276, 234, 250, 227, 252, 254, 233, 261, 265, 266, 270, 291, 233, 268, 263, 282, 255, 228, 247, 252, 266, 268, 261, 248, 258, 232, 260, 249, 242, 257, 241, 253, 256, 293, 245, 251, 256, 266, 255, 280, 235, 261, 268, 270, 272, 251, 234, 255, 287, 260, 264, 263, 245, 240, 254, 240, 236, 271, 247, 255, 240, 240, 309, 276, 273, 225, 240, 228, 245, 265, 263, 227, 238, 255, 229, 248, 248, 266, 239, 285, 288, 269, 262] payload_rows_per_dst=[5506, 5458, 5436, 5480, 5492, 5540, 5408, 5394] dup_rows_per_dst=[2760, 2750, 2726, 2710, 2670, 2790, 2700, 2716] combine_rows_per_src=[5506, 5458, 5436, 5480, 5492, 5540, 5408, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229208 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156295000 ingress_bytes=212688896 link_bytes=212688896 token_bytes=312360792 weight_movement_bytes=56623104 total_nvlink_bytes=368983896 weight_to_token_ratio=0.181275 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037880352e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38342 dup_rows=21826 dup_rows_remote=19144 dedup_ratio=0.333038 router_tokens_per_expert=[256, 246, 246, 265, 270, 240, 241, 263, 261, 260, 242, 261, 248, 236, 279, 254, 210, 273, 278, 250, 252, 238, 256, 260, 266, 228, 255, 245, 261, 260, 236, 257, 266, 247, 239, 269, 264, 260, 265, 253, 269, 219, 269, 239, 256, 245, 228, 252, 245, 256, 245, 247, 274, 247, 255, 254, 272, 263, 270, 236, 289, 256, 273, 237, 246, 281, 255, 293, 255, 247, 268, 286, 245, 247, 232, 278, 253, 277, 236, 228, 251, 263, 243, 266, 246, 251, 275, 260, 262, 254, 260, 250, 263, 268, 261, 253, 257, 258, 243, 267, 278, 248, 264, 258, 270, 243, 268, 253, 270, 253, 253, 261, 274, 268, 259, 286, 249, 260, 254, 265, 256, 239, 272, 258, 236, 233, 252, 258] payload_rows_per_dst=[5484, 5368, 5444, 5478, 5436, 5506, 5532, 5462] dup_rows_per_dst=[2652, 2682, 2636, 2760, 2818, 2746, 2756, 2776] combine_rows_per_src=[5484, 5368, 5444, 5478, 5436, 5506, 5532, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38224 dup_rows=21828 dup_rows_remote=19084 dedup_ratio=0.333069 router_tokens_per_expert=[231, 235, 280, 255, 264, 258, 264, 239, 264, 251, 246, 268, 272, 248, 252, 279, 247, 223, 260, 230, 220, 231, 262, 281, 255, 259, 237, 235, 232, 272, 228, 292, 288, 266, 253, 247, 244, 272, 229, 226, 244, 268, 238, 267, 271, 247, 253, 272, 266, 256, 249, 263, 251, 228, 259, 241, 248, 272, 261, 255, 279, 255, 274, 257, 264, 237, 248, 261, 245, 290, 254, 240, 269, 279, 269, 260, 237, 241, 271, 263, 280, 238, 259, 238, 262, 261, 282, 227, 241, 239, 275, 250, 235, 282, 244, 258, 274, 236, 255, 256, 249, 277, 293, 234, 245, 269, 276, 272, 296, 247, 274, 278, 259, 248, 260, 269, 259, 278, 256, 244, 235, 241, 250, 250, 271, 250, 220, 279] payload_rows_per_dst=[5508, 5406, 5424, 5484, 5436, 5418, 5618, 5414] dup_rows_per_dst=[2704, 2522, 2746, 2744, 2820, 2724, 2844, 2724] combine_rows_per_src=[5508, 5406, 5424, 5484, 5436, 5418, 5618, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229256 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876680 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524104 weight_movement_bytes=18874368 total_nvlink_bytes=332398472 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38132 dup_rows=21896 dup_rows_remote=19138 dedup_ratio=0.334106 router_tokens_per_expert=[239, 280, 295, 239, 263, 259, 251, 239, 256, 229, 238, 272, 244, 273, 277, 280, 246, 262, 286, 261, 272, 224, 256, 222, 284, 256, 259, 274, 255, 256, 229, 255, 258, 258, 256, 243, 254, 281, 256, 263, 242, 260, 277, 256, 233, 254, 271, 271, 264, 247, 254, 271, 244, 274, 260, 247, 245, 232, 268, 250, 254, 269, 233, 248, 249, 239, 268, 251, 264, 280, 239, 250, 257, 221, 272, 278, 265, 255, 248, 229, 242, 251, 251, 256, 264, 271, 278, 254, 256, 238, 267, 261, 284, 265, 253, 262, 231, 266, 240, 249, 259, 249, 246, 242, 226, 276, 246, 263, 278, 266, 244, 248, 254, 251, 268, 252, 241, 252, 264, 261, 244, 243, 256, 256, 247, 272, 251, 285] payload_rows_per_dst=[5506, 5522, 5508, 5308, 5402, 5530, 5418, 5446] dup_rows_per_dst=[2762, 2672, 2758, 2812, 2728, 2776, 2640, 2748] combine_rows_per_src=[5506, 5522, 5508, 5308, 5402, 5530, 5418, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229616 combine_payload_bytes=156139520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156369136 ingress_bytes=193888256 link_bytes=193888256 token_bytes=312508656 weight_movement_bytes=37748736 total_nvlink_bytes=350257392 weight_to_token_ratio=0.120793 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857741870e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228824 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156589528 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950232 weight_movement_bytes=37748736 total_nvlink_bytes=350698968 weight_to_token_ratio=0.120622 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229944 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157278776 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314327608 weight_movement_bytes=18874368 total_nvlink_bytes=333201976 weight_to_token_ratio=0.060047 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229232 combine_payload_bytes=156565504 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156794736 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360240 weight_movement_bytes=56623104 total_nvlink_bytes=369983344 weight_to_token_ratio=0.180697 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229080 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156417752 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312606424 weight_movement_bytes=18874368 total_nvlink_bytes=331480792 weight_to_token_ratio=0.060377 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38192 dup_rows=21836 dup_rows_remote=19132 dedup_ratio=0.333191 router_tokens_per_expert=[246, 278, 261, 250, 258, 222, 279, 242, 240, 266, 254, 272, 245, 250, 243, 251, 272, 257, 228, 252, 258, 240, 280, 251, 253, 237, 268, 229, 257, 295, 242, 258, 264, 252, 236, 251, 246, 229, 258, 245, 255, 250, 262, 263, 260, 278, 266, 290, 241, 256, 249, 250, 244, 251, 265, 224, 269, 256, 254, 255, 287, 246, 266, 244, 278, 269, 236, 247, 252, 225, 261, 269, 228, 258, 242, 259, 252, 277, 283, 270, 272, 271, 244, 265, 255, 259, 254, 243, 275, 251, 273, 264, 279, 249, 256, 277, 246, 260, 226, 237, 258, 271, 239, 252, 283, 273, 254, 221, 265, 247, 259, 278, 249, 254, 256, 237, 244, 248, 264, 253, 288, 269, 252, 268, 279, 246, 255, 248] payload_rows_per_dst=[5438, 5460, 5462, 5468, 5508, 5478, 5416, 5470] dup_rows_per_dst=[2676, 2694, 2748, 2646, 2704, 2896, 2722, 2750] combine_rows_per_src=[5438, 5460, 5462, 5468, 5508, 5478, 5416, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43530 payload_rows_remote=38062 dup_rows=22006 dup_rows_remote=19248 dedup_ratio=0.335785 router_tokens_per_expert=[247, 245, 261, 267, 275, 270, 252, 258, 270, 261, 275, 249, 272, 268, 221, 250, 254, 281, 263, 233, 230, 259, 234, 254, 250, 278, 265, 255, 254, 256, 234, 248, 247, 223, 251, 251, 254, 247, 273, 252, 228, 274, 255, 232, 265, 258, 246, 246, 274, 259, 263, 259, 259, 264, 254, 243, 274, 262, 251, 255, 251, 249, 253, 268, 249, 266, 251, 272, 267, 252, 261, 285, 251, 282, 255, 261, 262, 250, 262, 223, 277, 289, 267, 268, 271, 271, 241, 261, 266, 275, 251, 269, 251, 261, 273, 258, 258, 264, 260, 274, 264, 251, 259, 247, 261, 235, 258, 249, 258, 243, 259, 227, 272, 257, 242, 223, 255, 248, 242, 256, 275, 253, 223, 266, 241, 251, 232, 238] payload_rows_per_dst=[5432, 5422, 5292, 5450, 5422, 5530, 5514, 5468] dup_rows_per_dst=[2850, 2674, 2712, 2770, 2654, 2872, 2716, 2758] combine_rows_per_src=[5432, 5422, 5292, 5450, 5422, 5530, 5514, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38430 dup_rows=21636 dup_rows_remote=18874 dedup_ratio=0.330139 router_tokens_per_expert=[263, 247, 274, 243, 271, 265, 252, 246, 264, 230, 238, 249, 279, 258, 230, 249, 260, 281, 234, 283, 256, 269, 274, 236, 280, 252, 263, 280, 284, 258, 252, 268, 258, 258, 250, 248, 274, 265, 250, 286, 255, 258, 264, 270, 249, 246, 242, 248, 212, 261, 237, 268, 277, 267, 258, 256, 256, 251, 216, 241, 270, 253, 260, 269, 237, 263, 244, 250, 261, 251, 267, 277, 256, 216, 276, 239, 274, 258, 270, 284, 270, 268, 259, 268, 230, 258, 264, 249, 239, 236, 245, 284, 243, 231, 239, 252, 269, 251, 230, 231, 246, 261, 264, 282, 258, 255, 275, 278, 249, 276, 243, 233, 242, 252, 238, 224, 264, 226, 255, 256, 291, 256, 259, 256, 253, 251, 263, 262] payload_rows_per_dst=[5470, 5616, 5526, 5324, 5548, 5390, 5516, 5510] dup_rows_per_dst=[2762, 3000, 2780, 2474, 2828, 2520, 2686, 2586] combine_rows_per_src=[5470, 5616, 5526, 5324, 5548, 5390, 5516, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38340 dup_rows=21684 dup_rows_remote=18918 dedup_ratio=0.330872 router_tokens_per_expert=[266, 262, 246, 267, 253, 236, 254, 225, 272, 283, 258, 234, 258, 227, 260, 247, 271, 244, 256, 268, 266, 257, 288, 260, 237, 279, 247, 285, 253, 257, 230, 262, 245, 246, 233, 246, 236, 240, 254, 257, 240, 250, 244, 259, 255, 233, 268, 242, 269, 262, 258, 275, 275, 244, 256, 263, 267, 286, 271, 261, 259, 232, 240, 253, 274, 269, 246, 290, 227, 254, 269, 238, 295, 247, 258, 252, 242, 253, 254, 233, 248, 242, 265, 255, 262, 255, 263, 241, 236, 256, 240, 274, 235, 244, 261, 275, 289, 251, 238, 274, 249, 274, 251, 259, 242, 265, 285, 255, 223, 258, 265, 261, 251, 237, 251, 276, 256, 262, 255, 238, 264, 278, 270, 269, 268, 265, 271, 238] payload_rows_per_dst=[5434, 5498, 5326, 5618, 5454, 5498, 5512, 5512] dup_rows_per_dst=[2662, 2822, 2570, 2724, 2748, 2606, 2766, 2786] combine_rows_per_src=[5434, 5498, 5326, 5618, 5454, 5498, 5512, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229296 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156663728 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313098160 weight_movement_bytes=37748736 total_nvlink_bytes=350846896 weight_to_token_ratio=0.120565 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38256 dup_rows=21844 dup_rows_remote=19092 dedup_ratio=0.333313 router_tokens_per_expert=[245, 266, 265, 264, 244, 243, 291, 252, 250, 246, 231, 251, 266, 281, 269, 271, 260, 265, 250, 274, 270, 281, 263, 254, 264, 263, 245, 280, 230, 259, 260, 265, 232, 276, 289, 237, 251, 272, 241, 213, 270, 271, 274, 260, 254, 265, 244, 240, 270, 268, 250, 263, 271, 251, 268, 257, 247, 273, 237, 252, 250, 239, 263, 262, 242, 250, 265, 250, 269, 268, 236, 256, 276, 257, 248, 240, 257, 232, 263, 244, 254, 250, 245, 235, 302, 257, 238, 250, 244, 253, 258, 247, 254, 261, 266, 280, 231, 255, 263, 273, 266, 276, 244, 229, 260, 257, 270, 244, 254, 242, 248, 277, 220, 249, 255, 227, 227, 252, 276, 254, 261, 244, 265, 257, 234, 236, 265, 282] payload_rows_per_dst=[5570, 5536, 5458, 5442, 5414, 5436, 5454, 5382] dup_rows_per_dst=[2700, 2830, 2720, 2800, 2692, 2752, 2724, 2626] combine_rows_per_src=[5570, 5536, 5458, 5442, 5414, 5436, 5454, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44042 payload_rows_remote=38632 dup_rows=21494 dup_rows_remote=19054 dedup_ratio=0.327972 router_tokens_per_expert=[272, 256, 239, 274, 250, 255, 272, 255, 251, 277, 246, 270, 266, 247, 293, 256, 257, 252, 248, 222, 255, 254, 248, 260, 263, 247, 252, 258, 265, 239, 273, 254, 260, 227, 284, 260, 275, 254, 265, 261, 264, 245, 261, 263, 275, 264, 233, 252, 243, 243, 264, 250, 225, 251, 252, 267, 237, 236, 249, 253, 278, 236, 228, 213, 247, 259, 234, 246, 260, 243, 258, 268, 259, 280, 227, 251, 271, 254, 253, 244, 257, 220, 272, 261, 258, 283, 258, 226, 252, 294, 261, 243, 267, 232, 248, 266, 268, 265, 286, 244, 266, 263, 254, 260, 281, 268, 265, 249, 252, 267, 249, 243, 259, 261, 279, 275, 225, 233, 286, 227, 243, 269, 257, 274, 262, 239, 275, 278] payload_rows_per_dst=[5516, 5466, 5614, 5410, 5450, 5500, 5556, 5530] dup_rows_per_dst=[2842, 2628, 2672, 2440, 2658, 2696, 2804, 2754] combine_rows_per_src=[5516, 5466, 5614, 5410, 5450, 5500, 5556, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38374 dup_rows=21578 dup_rows_remote=18868 dedup_ratio=0.329254 router_tokens_per_expert=[260, 252, 258, 250, 257, 244, 262, 251, 244, 265, 270, 237, 237, 265, 248, 262, 250, 249, 260, 287, 265, 250, 268, 274, 254, 263, 267, 246, 287, 253, 245, 229, 287, 267, 264, 254, 277, 269, 265, 250, 263, 222, 272, 225, 250, 265, 260, 236, 242, 256, 272, 278, 244, 266, 249, 260, 284, 234, 244, 269, 264, 252, 252, 269, 257, 257, 286, 250, 274, 259, 264, 258, 276, 255, 262, 252, 268, 286, 259, 263, 253, 284, 276, 250, 224, 253, 254, 277, 266, 262, 217, 234, 233, 246, 245, 270, 236, 253, 247, 262, 240, 262, 225, 254, 246, 239, 276, 241, 253, 255, 257, 255, 245, 244, 222, 240, 270, 261, 240, 297, 245, 261, 239, 257, 228, 247, 283, 248] payload_rows_per_dst=[5512, 5584, 5480, 5530, 5572, 5412, 5406, 5462] dup_rows_per_dst=[2612, 2710, 2772, 2740, 2880, 2676, 2596, 2592] combine_rows_per_src=[5512, 5584, 5480, 5530, 5572, 5412, 5406, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38196 dup_rows=21990 dup_rows_remote=19290 dedup_ratio=0.335541 router_tokens_per_expert=[273, 257, 234, 248, 242, 238, 263, 271, 268, 267, 250, 258, 259, 251, 244, 270, 270, 227, 241, 261, 249, 245, 281, 272, 279, 262, 281, 232, 275, 235, 251, 273, 223, 250, 246, 233, 247, 258, 264, 269, 257, 263, 259, 269, 225, 265, 228, 269, 258, 249, 243, 243, 254, 275, 266, 236, 255, 261, 264, 247, 256, 250, 255, 242, 278, 221, 256, 251, 268, 249, 249, 262, 251, 253, 271, 254, 264, 259, 269, 254, 268, 240, 253, 236, 261, 277, 247, 248, 267, 244, 294, 235, 288, 246, 252, 241, 248, 251, 228, 255, 254, 241, 266, 245, 262, 268, 239, 256, 281, 291, 264, 275, 249, 256, 241, 254, 244, 245, 265, 263, 278, 270, 275, 276, 241, 272, 252, 251] payload_rows_per_dst=[5438, 5484, 5350, 5404, 5478, 5432, 5534, 5426] dup_rows_per_dst=[2748, 2784, 2700, 2704, 2740, 2762, 2714, 2838] combine_rows_per_src=[5438, 5484, 5350, 5404, 5478, 5432, 5534, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=229240 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156131192 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312033144 weight_movement_bytes=18874368 total_nvlink_bytes=330907512 weight_to_token_ratio=0.060488 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229216 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157638496 ingress_bytes=176283648 link_bytes=176283648 token_bytes=315047776 weight_movement_bytes=18874368 total_nvlink_bytes=333922144 weight_to_token_ratio=0.059910 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689063178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229032 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157269672 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314310312 weight_movement_bytes=18874368 total_nvlink_bytes=333184680 weight_to_token_ratio=0.060050 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229392 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156925968 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622544 weight_movement_bytes=37748736 total_nvlink_bytes=351371280 weight_to_token_ratio=0.120364 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=158236672 dispatch_meta_bytes=230744 combine_payload_bytes=158236672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158467416 ingress_bytes=177111040 link_bytes=177111040 token_bytes=316704088 weight_movement_bytes=18874368 total_nvlink_bytes=335578456 weight_to_token_ratio=0.059596 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.696990841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=228968 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157408872 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314588776 weight_movement_bytes=18874368 total_nvlink_bytes=333463144 weight_to_token_ratio=0.059997 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229944 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680760 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131576 weight_movement_bytes=18874368 total_nvlink_bytes=332005944 weight_to_token_ratio=0.060276 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38384 dup_rows=21716 dup_rows_remote=19024 dedup_ratio=0.331360 router_tokens_per_expert=[256, 231, 264, 264, 266, 263, 276, 241, 273, 249, 276, 279, 238, 290, 269, 255, 261, 246, 280, 244, 258, 252, 250, 249, 259, 246, 256, 235, 251, 259, 257, 261, 200, 266, 255, 241, 256, 283, 246, 252, 270, 247, 267, 255, 251, 266, 269, 254, 274, 279, 272, 261, 264, 239, 248, 260, 234, 273, 272, 267, 256, 277, 247, 246, 267, 251, 234, 243, 255, 240, 252, 234, 259, 287, 287, 246, 245, 244, 262, 258, 257, 257, 239, 257, 226, 240, 227, 262, 269, 238, 238, 255, 247, 248, 256, 281, 248, 229, 231, 280, 263, 258, 283, 242, 248, 217, 264, 244, 263, 256, 267, 235, 251, 282, 264, 280, 268, 225, 270, 268, 266, 237, 292, 248, 251, 251, 284, 241] payload_rows_per_dst=[5506, 5446, 5504, 5484, 5436, 5444, 5468, 5532] dup_rows_per_dst=[2874, 2682, 2652, 2854, 2692, 2550, 2588, 2824] combine_rows_per_src=[5506, 5446, 5504, 5484, 5436, 5444, 5468, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38318 dup_rows=21686 dup_rows_remote=18828 dedup_ratio=0.330902 router_tokens_per_expert=[240, 267, 240, 253, 254, 269, 225, 237, 264, 235, 227, 269, 272, 252, 243, 225, 267, 247, 236, 258, 265, 257, 265, 273, 251, 270, 284, 249, 261, 270, 259, 248, 239, 275, 239, 225, 247, 280, 273, 243, 239, 247, 251, 262, 245, 285, 262, 265, 263, 235, 267, 282, 267, 236, 254, 263, 267, 251, 247, 275, 270, 231, 230, 242, 246, 258, 238, 275, 272, 253, 259, 253, 255, 260, 260, 236, 260, 250, 295, 257, 253, 240, 238, 259, 256, 249, 227, 272, 262, 235, 267, 269, 257, 250, 256, 270, 246, 250, 269, 257, 262, 245, 261, 258, 284, 233, 248, 261, 241, 261, 261, 296, 236, 272, 252, 242, 256, 239, 240, 249, 277, 257, 266, 276, 273, 283, 277, 264] payload_rows_per_dst=[5160, 5572, 5460, 5460, 5698, 5526, 5442, 5532] dup_rows_per_dst=[2240, 2782, 2694, 2700, 2884, 2764, 2764, 2858] combine_rows_per_src=[5160, 5572, 5460, 5460, 5698, 5526, 5442, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229632 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157450496 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314671360 weight_movement_bytes=18874368 total_nvlink_bytes=333545728 weight_to_token_ratio=0.059981 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38272 dup_rows=21714 dup_rows_remote=19048 dedup_ratio=0.331329 router_tokens_per_expert=[247, 260, 256, 269, 262, 262, 259, 264, 252, 305, 261, 244, 253, 262, 261, 261, 247, 238, 256, 238, 266, 246, 255, 246, 230, 278, 269, 236, 246, 260, 259, 236, 241, 263, 256, 253, 227, 252, 247, 250, 275, 231, 250, 261, 250, 261, 244, 269, 259, 227, 259, 243, 251, 282, 231, 257, 255, 263, 259, 232, 254, 248, 258, 255, 282, 257, 223, 250, 245, 259, 276, 255, 243, 254, 228, 275, 259, 246, 253, 252, 253, 253, 249, 277, 272, 251, 265, 251, 269, 244, 239, 270, 290, 256, 273, 263, 245, 266, 249, 262, 287, 245, 248, 244, 277, 235, 281, 255, 263, 262, 240, 249, 272, 252, 272, 268, 257, 271, 254, 247, 276, 272, 251, 248, 256, 248, 271, 266] payload_rows_per_dst=[5540, 5310, 5446, 5406, 5458, 5550, 5550, 5562] dup_rows_per_dst=[2816, 2702, 2614, 2660, 2656, 2800, 2666, 2800] combine_rows_per_src=[5540, 5310, 5446, 5406, 5458, 5550, 5550, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38238 dup_rows=21814 dup_rows_remote=19148 dedup_ratio=0.332855 router_tokens_per_expert=[261, 254, 221, 244, 271, 257, 242, 276, 278, 228, 242, 268, 272, 267, 252, 241, 265, 240, 259, 227, 255, 246, 292, 271, 279, 255, 245, 256, 248, 240, 254, 236, 233, 249, 257, 265, 233, 260, 251, 254, 279, 265, 249, 265, 244, 271, 247, 239, 294, 238, 235, 254, 251, 266, 275, 246, 247, 248, 234, 270, 229, 247, 283, 251, 254, 270, 243, 247, 266, 242, 222, 292, 252, 268, 259, 246, 236, 271, 224, 246, 275, 245, 248, 248, 231, 273, 233, 284, 284, 248, 253, 268, 275, 238, 266, 218, 261, 254, 243, 248, 265, 262, 268, 252, 284, 222, 284, 263, 270, 251, 261, 280, 254, 253, 250, 250, 249, 270, 259, 240, 250, 271, 281, 270, 284, 267, 274, 282] payload_rows_per_dst=[5484, 5584, 5636, 5428, 5250, 5350, 5438, 5552] dup_rows_per_dst=[2666, 2848, 2970, 2708, 2342, 2526, 2898, 2856] combine_rows_per_src=[5484, 5584, 5636, 5428, 5250, 5350, 5438, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38276 dup_rows=21888 dup_rows_remote=19190 dedup_ratio=0.333984 router_tokens_per_expert=[241, 261, 253, 265, 281, 264, 246, 251, 250, 242, 248, 244, 286, 281, 252, 268, 253, 261, 278, 255, 263, 249, 243, 222, 236, 252, 244, 279, 250, 261, 279, 281, 274, 215, 262, 218, 249, 246, 244, 271, 274, 263, 264, 247, 266, 232, 239, 305, 267, 276, 255, 259, 257, 274, 235, 263, 274, 244, 261, 282, 275, 238, 247, 243, 245, 252, 255, 258, 261, 241, 265, 246, 260, 261, 255, 271, 261, 263, 249, 255, 253, 261, 237, 233, 241, 252, 255, 249, 257, 274, 254, 242, 261, 248, 251, 267, 250, 263, 262, 264, 240, 265, 262, 266, 248, 213, 269, 268, 284, 268, 238, 244, 244, 256, 265, 261, 236, 241, 263, 264, 241, 248, 261, 260, 259, 268, 250, 256] payload_rows_per_dst=[5490, 5488, 5436, 5498, 5508, 5372, 5454, 5402] dup_rows_per_dst=[2776, 2724, 2702, 2802, 2688, 2698, 2754, 2744] combine_rows_per_src=[5490, 5488, 5436, 5498, 5508, 5372, 5454, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38304 dup_rows=21696 dup_rows_remote=19042 dedup_ratio=0.331055 router_tokens_per_expert=[275, 258, 272, 272, 266, 248, 254, 251, 254, 283, 227, 239, 263, 233, 240, 240, 240, 230, 262, 242, 268, 262, 297, 244, 238, 278, 244, 257, 261, 263, 249, 260, 232, 265, 238, 270, 248, 244, 271, 269, 284, 251, 266, 241, 268, 252, 265, 253, 240, 292, 242, 248, 244, 252, 252, 250, 244, 258, 263, 263, 253, 272, 262, 248, 251, 271, 247, 269, 260, 215, 264, 228, 263, 240, 254, 269, 239, 227, 254, 248, 247, 259, 271, 248, 256, 256, 257, 266, 267, 258, 269, 244, 272, 266, 255, 257, 259, 266, 252, 212, 256, 298, 273, 260, 259, 239, 246, 252, 248, 258, 252, 267, 226, 257, 271, 251, 248, 281, 243, 258, 266, 262, 253, 240, 284, 257, 288, 269] payload_rows_per_dst=[5422, 5536, 5468, 5434, 5370, 5528, 5538, 5544] dup_rows_per_dst=[2728, 2654, 2766, 2732, 2628, 2768, 2656, 2764] combine_rows_per_src=[5422, 5536, 5468, 5434, 5370, 5528, 5538, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38154 dup_rows=21942 dup_rows_remote=19262 dedup_ratio=0.334808 router_tokens_per_expert=[280, 251, 274, 284, 274, 254, 221, 255, 234, 246, 265, 251, 228, 253, 258, 277, 275, 259, 268, 256, 239, 287, 249, 278, 250, 252, 269, 278, 251, 240, 238, 259, 262, 238, 265, 254, 243, 255, 284, 270, 258, 250, 277, 250, 238, 260, 233, 243, 252, 244, 233, 251, 253, 253, 240, 276, 261, 258, 254, 280, 264, 253, 239, 249, 254, 257, 249, 244, 260, 288, 240, 257, 258, 253, 251, 272, 251, 270, 248, 284, 255, 260, 267, 240, 256, 270, 247, 273, 262, 245, 240, 226, 237, 243, 238, 247, 275, 263, 243, 259, 263, 256, 257, 236, 250, 221, 249, 273, 256, 269, 221, 271, 258, 270, 251, 261, 267, 282, 252, 243, 291, 260, 242, 266, 281, 266, 219, 262] payload_rows_per_dst=[5476, 5470, 5482, 5440, 5490, 5332, 5424, 5480] dup_rows_per_dst=[2734, 2826, 2678, 2680, 2782, 2680, 2700, 2862] combine_rows_per_src=[5476, 5470, 5482, 5440, 5490, 5332, 5424, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43500 payload_rows_remote=38064 dup_rows=22036 dup_rows_remote=19230 dedup_ratio=0.336243 router_tokens_per_expert=[249, 271, 234, 248, 253, 242, 287, 270, 242, 265, 277, 259, 254, 254, 249, 258, 246, 235, 234, 265, 267, 231, 270, 305, 222, 265, 276, 258, 215, 259, 262, 274, 272, 259, 258, 247, 254, 253, 289, 248, 261, 282, 257, 230, 239, 239, 275, 258, 258, 238, 226, 264, 295, 267, 274, 274, 246, 256, 248, 232, 245, 248, 255, 254, 251, 239, 281, 240, 263, 297, 236, 230, 270, 235, 243, 261, 274, 235, 270, 266, 259, 257, 267, 276, 228, 260, 263, 253, 231, 248, 254, 272, 261, 269, 260, 258, 269, 267, 258, 268, 272, 252, 232, 235, 244, 269, 241, 256, 257, 248, 264, 240, 235, 246, 279, 276, 228, 282, 243, 249, 268, 262, 264, 247, 249, 257, 239, 268] payload_rows_per_dst=[5446, 5436, 5436, 5420, 5444, 5508, 5412, 5398] dup_rows_per_dst=[2778, 2732, 2806, 2740, 2738, 2724, 2732, 2786] combine_rows_per_src=[5446, 5436, 5436, 5420, 5444, 5508, 5412, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228584 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157179112 ingress_bytes=213573632 link_bytes=213573632 token_bytes=314129640 weight_movement_bytes=56623104 total_nvlink_bytes=370752744 weight_to_token_ratio=0.180254 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046357457e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229544 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156852392 ingress_bytes=194371584 link_bytes=194371584 token_bytes=313475240 weight_movement_bytes=37748736 total_nvlink_bytes=351223976 weight_to_token_ratio=0.120420 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862372881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229280 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156991392 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313753504 weight_movement_bytes=37748736 total_nvlink_bytes=351502240 weight_to_token_ratio=0.120313 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229864 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008360 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786856 weight_movement_bytes=18874368 total_nvlink_bytes=332661224 weight_to_token_ratio=0.060150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229384 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122568 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015752 weight_movement_bytes=18874368 total_nvlink_bytes=332890120 weight_to_token_ratio=0.060106 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=229176 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156139320 ingress_bytes=174784512 link_bytes=174784512 token_bytes=312049464 weight_movement_bytes=18874368 total_nvlink_bytes=330923832 weight_to_token_ratio=0.060485 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674699195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229664 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156508448 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312787232 weight_movement_bytes=37748736 total_nvlink_bytes=350535968 weight_to_token_ratio=0.120685 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38328 dup_rows=21804 dup_rows_remote=19042 dedup_ratio=0.332703 router_tokens_per_expert=[268, 236, 256, 263, 285, 265, 241, 269, 268, 282, 244, 272, 247, 257, 238, 272, 269, 253, 268, 264, 249, 270, 244, 269, 243, 262, 268, 242, 241, 252, 266, 238, 255, 231, 274, 245, 281, 256, 269, 257, 252, 251, 250, 229, 263, 262, 277, 236, 258, 248, 230, 254, 277, 252, 268, 239, 253, 276, 245, 244, 272, 224, 276, 255, 262, 250, 268, 259, 249, 254, 259, 243, 267, 266, 258, 259, 251, 259, 252, 227, 268, 267, 276, 257, 282, 261, 264, 263, 258, 239, 264, 242, 264, 218, 268, 229, 275, 242, 250, 250, 236, 228, 239, 264, 257, 284, 248, 210, 254, 254, 249, 278, 281, 232, 268, 259, 275, 253, 281, 223, 272, 249, 266, 231, 263, 261, 261, 252] payload_rows_per_dst=[5564, 5510, 5404, 5508, 5404, 5476, 5362, 5504] dup_rows_per_dst=[2762, 2686, 2772, 2634, 2762, 2764, 2674, 2750] combine_rows_per_src=[5564, 5510, 5404, 5508, 5404, 5476, 5362, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229480 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220968 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212456 weight_movement_bytes=18874368 total_nvlink_bytes=333086824 weight_to_token_ratio=0.060069 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43934 payload_rows_remote=38306 dup_rows=21602 dup_rows_remote=18506 dedup_ratio=0.329620 router_tokens_per_expert=[254, 265, 262, 236, 272, 260, 289, 230, 265, 264, 286, 252, 281, 252, 234, 259, 245, 249, 229, 285, 262, 231, 241, 269, 230, 264, 251, 247, 246, 257, 243, 261, 256, 258, 256, 270, 240, 216, 252, 255, 275, 273, 253, 246, 247, 271, 266, 244, 270, 255, 252, 273, 258, 269, 239, 263, 265, 276, 235, 274, 274, 260, 255, 261, 215, 255, 244, 242, 236, 278, 237, 238, 267, 242, 270, 254, 256, 259, 289, 250, 246, 238, 253, 289, 256, 229, 262, 244, 257, 261, 265, 253, 282, 254, 228, 262, 260, 238, 245, 252, 238, 251, 250, 255, 251, 240, 292, 266, 239, 253, 259, 234, 281, 266, 244, 269, 255, 260, 237, 269, 272, 289, 252, 268, 269, 278, 265, 232] payload_rows_per_dst=[5352, 5478, 5526, 5588, 5408, 5516, 5438, 5628] dup_rows_per_dst=[2460, 2542, 2824, 2770, 2656, 2646, 2608, 3096] combine_rows_per_src=[5352, 5478, 5526, 5588, 5408, 5516, 5438, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38360 dup_rows=21626 dup_rows_remote=18888 dedup_ratio=0.329987 router_tokens_per_expert=[269, 248, 271, 254, 254, 215, 266, 271, 258, 255, 248, 239, 256, 309, 271, 259, 267, 273, 247, 274, 282, 258, 257, 262, 226, 238, 263, 238, 234, 248, 287, 258, 260, 265, 280, 238, 270, 258, 257, 255, 221, 238, 247, 255, 268, 274, 277, 250, 260, 233, 261, 267, 263, 268, 278, 272, 230, 248, 247, 259, 238, 277, 274, 266, 260, 274, 265, 223, 236, 283, 272, 229, 230, 241, 245, 239, 271, 273, 243, 254, 249, 258, 245, 246, 271, 235, 235, 275, 248, 270, 261, 259, 213, 241, 243, 253, 255, 251, 268, 258, 238, 257, 259, 242, 273, 252, 251, 252, 237, 247, 278, 279, 287, 261, 274, 254, 246, 246, 285, 249, 230, 244, 279, 248, 260, 242, 266, 251] payload_rows_per_dst=[5550, 5520, 5468, 5498, 5412, 5428, 5480, 5554] dup_rows_per_dst=[2738, 2704, 2756, 2784, 2664, 2576, 2714, 2690] combine_rows_per_src=[5550, 5520, 5468, 5498, 5412, 5428, 5480, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38262 dup_rows=21796 dup_rows_remote=19146 dedup_ratio=0.332581 router_tokens_per_expert=[255, 275, 238, 227, 261, 249, 280, 258, 257, 250, 245, 257, 243, 251, 269, 234, 265, 251, 250, 279, 280, 266, 256, 262, 250, 230, 248, 236, 240, 268, 275, 242, 291, 234, 262, 239, 264, 283, 261, 275, 283, 265, 241, 231, 278, 265, 244, 237, 261, 245, 258, 234, 238, 262, 254, 247, 236, 239, 272, 267, 239, 263, 247, 264, 273, 231, 266, 249, 279, 242, 252, 262, 259, 255, 258, 232, 246, 279, 247, 245, 264, 287, 282, 235, 252, 253, 256, 282, 261, 241, 280, 259, 256, 278, 228, 257, 230, 250, 244, 264, 248, 247, 250, 271, 254, 272, 253, 243, 288, 245, 247, 258, 234, 250, 257, 276, 247, 257, 229, 264, 238, 257, 264, 290, 274, 272, 257, 266] payload_rows_per_dst=[5406, 5434, 5548, 5394, 5468, 5526, 5478, 5486] dup_rows_per_dst=[2692, 2762, 2758, 2658, 2682, 2816, 2650, 2778] combine_rows_per_src=[5406, 5434, 5548, 5394, 5468, 5526, 5478, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38256 dup_rows=21738 dup_rows_remote=18942 dedup_ratio=0.331696 router_tokens_per_expert=[258, 255, 264, 261, 262, 246, 271, 242, 257, 272, 271, 253, 249, 224, 267, 238, 256, 249, 253, 264, 258, 239, 238, 264, 259, 261, 276, 248, 262, 265, 250, 261, 246, 273, 244, 262, 230, 250, 247, 251, 257, 244, 240, 237, 263, 270, 279, 266, 231, 253, 270, 241, 267, 263, 288, 257, 251, 258, 267, 290, 247, 267, 258, 238, 244, 254, 245, 284, 261, 270, 230, 263, 278, 252, 256, 262, 252, 220, 240, 265, 240, 238, 269, 222, 266, 262, 259, 260, 237, 262, 275, 259, 263, 293, 295, 269, 249, 253, 263, 232, 268, 252, 237, 254, 229, 278, 245, 252, 228, 251, 243, 235, 249, 280, 257, 252, 247, 235, 264, 271, 301, 246, 247, 265, 240, 273, 271, 258] payload_rows_per_dst=[5458, 5448, 5444, 5586, 5524, 5542, 5318, 5478] dup_rows_per_dst=[2722, 2758, 2674, 2706, 2628, 2796, 2620, 2834] combine_rows_per_src=[5458, 5448, 5444, 5586, 5524, 5542, 5318, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38182 dup_rows=21794 dup_rows_remote=19116 dedup_ratio=0.332550 router_tokens_per_expert=[257, 223, 273, 259, 279, 284, 251, 240, 275, 269, 256, 264, 250, 258, 249, 271, 256, 243, 262, 299, 252, 264, 257, 272, 267, 254, 226, 253, 233, 264, 256, 261, 233, 227, 288, 255, 238, 268, 241, 249, 262, 247, 247, 243, 281, 235, 234, 276, 265, 252, 250, 252, 221, 246, 238, 275, 254, 282, 271, 247, 246, 244, 293, 246, 263, 229, 247, 249, 231, 232, 260, 272, 244, 272, 258, 262, 240, 253, 291, 275, 244, 263, 245, 256, 278, 261, 263, 242, 225, 254, 293, 257, 270, 289, 269, 260, 279, 261, 263, 241, 233, 256, 256, 266, 234, 258, 253, 263, 261, 281, 236, 242, 250, 271, 251, 238, 240, 286, 238, 244, 247, 270, 267, 248, 253, 250, 260, 242] payload_rows_per_dst=[5530, 5560, 5430, 5428, 5488, 5440, 5608, 5258] dup_rows_per_dst=[2786, 2678, 2618, 2736, 2668, 2898, 2912, 2498] combine_rows_per_src=[5530, 5560, 5430, 5428, 5488, 5440, 5608, 5258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38204 dup_rows=21778 dup_rows_remote=18928 dedup_ratio=0.332306 router_tokens_per_expert=[251, 252, 233, 269, 250, 261, 231, 259, 263, 244, 241, 246, 250, 246, 293, 259, 256, 252, 245, 232, 266, 229, 252, 252, 231, 250, 230, 232, 263, 240, 261, 262, 267, 265, 267, 253, 312, 253, 231, 259, 268, 279, 266, 251, 249, 275, 241, 266, 285, 239, 251, 270, 250, 239, 241, 258, 239, 240, 283, 235, 235, 273, 260, 289, 265, 253, 249, 269, 253, 258, 261, 272, 243, 270, 263, 235, 237, 230, 245, 252, 267, 237, 235, 248, 259, 231, 252, 282, 241, 280, 270, 261, 262, 259, 268, 238, 262, 249, 269, 261, 252, 254, 273, 251, 241, 258, 269, 274, 265, 252, 285, 242, 263, 267, 285, 260, 284, 246, 246, 262, 272, 247, 223, 261, 273, 257, 251, 279] payload_rows_per_dst=[5482, 5378, 5554, 5458, 5382, 5474, 5492, 5538] dup_rows_per_dst=[2614, 2528, 2850, 2716, 2728, 2706, 2822, 2814] combine_rows_per_src=[5482, 5378, 5554, 5458, 5382, 5474, 5492, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38344 dup_rows=21806 dup_rows_remote=19100 dedup_ratio=0.332733 router_tokens_per_expert=[243, 265, 246, 247, 253, 262, 261, 266, 244, 244, 249, 277, 263, 266, 238, 252, 270, 255, 265, 273, 252, 276, 274, 263, 272, 253, 277, 250, 266, 266, 271, 258, 244, 250, 268, 244, 265, 265, 269, 262, 251, 246, 253, 226, 265, 274, 254, 247, 239, 250, 274, 271, 220, 271, 299, 260, 250, 234, 273, 242, 231, 233, 246, 253, 239, 241, 242, 279, 264, 266, 277, 282, 262, 288, 272, 251, 234, 253, 289, 276, 254, 264, 260, 253, 237, 242, 262, 242, 274, 262, 271, 253, 249, 239, 276, 266, 213, 230, 243, 238, 239, 236, 266, 259, 253, 256, 253, 266, 245, 252, 257, 259, 242, 270, 255, 254, 240, 266, 266, 251, 255, 239, 255, 251, 258, 246, 257, 233] payload_rows_per_dst=[5456, 5596, 5386, 5386, 5584, 5502, 5402, 5418] dup_rows_per_dst=[2696, 2886, 2780, 2706, 2846, 2706, 2528, 2658] combine_rows_per_src=[5456, 5596, 5386, 5386, 5584, 5502, 5402, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=227248 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157128624 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314030000 weight_movement_bytes=37748736 total_nvlink_bytes=351778736 weight_to_token_ratio=0.120207 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=228992 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157351552 ingress_bytes=213745664 link_bytes=213745664 token_bytes=314474112 weight_movement_bytes=56623104 total_nvlink_bytes=371097216 weight_to_token_ratio=0.180056 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048005783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229632 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950784 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671936 weight_movement_bytes=18874368 total_nvlink_bytes=332546304 weight_to_token_ratio=0.060172 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228792 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156925368 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313621944 weight_movement_bytes=56623104 total_nvlink_bytes=370245048 weight_to_token_ratio=0.180546 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229192 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622664 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313016136 weight_movement_bytes=18874368 total_nvlink_bytes=331890504 weight_to_token_ratio=0.060298 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228528 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712112 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313195696 weight_movement_bytes=37748736 total_nvlink_bytes=350944432 weight_to_token_ratio=0.120528 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229776 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286800 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343824 weight_movement_bytes=18874368 total_nvlink_bytes=333218192 weight_to_token_ratio=0.060044 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38310 dup_rows=21822 dup_rows_remote=19128 dedup_ratio=0.332977 router_tokens_per_expert=[230, 247, 259, 275, 253, 300, 264, 258, 256, 237, 258, 266, 243, 264, 230, 280, 229, 251, 256, 279, 231, 252, 267, 249, 255, 232, 262, 285, 253, 235, 274, 262, 251, 240, 250, 253, 258, 245, 248, 247, 245, 244, 257, 237, 273, 267, 271, 277, 231, 222, 264, 257, 265, 232, 257, 311, 273, 245, 289, 244, 256, 238, 281, 230, 270, 264, 252, 267, 242, 265, 274, 224, 231, 239, 243, 235, 268, 244, 273, 258, 247, 264, 292, 264, 270, 256, 238, 244, 275, 247, 266, 266, 258, 244, 268, 238, 263, 250, 256, 231, 281, 236, 256, 266, 247, 270, 262, 257, 258, 258, 261, 282, 247, 269, 251, 263, 261, 254, 249, 264, 243, 253, 228, 259, 245, 302, 269, 241] payload_rows_per_dst=[5488, 5430, 5466, 5458, 5404, 5452, 5542, 5474] dup_rows_per_dst=[2752, 2714, 2660, 2732, 2694, 2822, 2726, 2722] combine_rows_per_src=[5488, 5430, 5466, 5458, 5404, 5452, 5542, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229752 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157147512 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314065272 weight_movement_bytes=37748736 total_nvlink_bytes=351814008 weight_to_token_ratio=0.120194 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38250 dup_rows=21754 dup_rows_remote=18996 dedup_ratio=0.331940 router_tokens_per_expert=[269, 268, 246, 253, 273, 237, 269, 262, 257, 235, 248, 272, 245, 268, 240, 250, 246, 261, 262, 268, 271, 242, 249, 247, 253, 237, 246, 279, 246, 228, 256, 262, 242, 274, 270, 258, 248, 252, 267, 232, 275, 232, 268, 260, 254, 252, 271, 244, 284, 264, 291, 283, 261, 257, 251, 239, 254, 249, 283, 249, 254, 261, 268, 243, 259, 254, 241, 254, 235, 267, 264, 235, 258, 247, 236, 263, 264, 279, 263, 280, 263, 261, 226, 230, 260, 217, 259, 232, 242, 271, 243, 234, 260, 277, 284, 267, 232, 288, 286, 247, 241, 241, 246, 252, 235, 260, 245, 258, 253, 268, 255, 256, 248, 294, 274, 236, 219, 243, 283, 281, 243, 252, 280, 250, 266, 252, 258, 266] payload_rows_per_dst=[5490, 5454, 5436, 5536, 5328, 5586, 5420, 5532] dup_rows_per_dst=[2694, 2652, 2762, 2846, 2362, 2974, 2706, 2758] combine_rows_per_src=[5490, 5454, 5436, 5536, 5328, 5586, 5420, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38186 dup_rows=21638 dup_rows_remote=18838 dedup_ratio=0.330170 router_tokens_per_expert=[232, 253, 261, 264, 249, 254, 246, 255, 244, 253, 250, 249, 245, 272, 247, 259, 290, 228, 275, 259, 246, 272, 258, 274, 271, 240, 246, 259, 271, 256, 264, 249, 269, 244, 235, 251, 291, 241, 259, 234, 249, 214, 248, 273, 282, 269, 290, 257, 250, 245, 251, 261, 254, 258, 247, 232, 245, 275, 284, 265, 253, 269, 242, 260, 264, 257, 271, 261, 247, 254, 254, 238, 262, 244, 246, 236, 235, 258, 254, 267, 256, 234, 240, 234, 244, 256, 256, 274, 251, 263, 244, 269, 238, 251, 250, 259, 262, 241, 251, 254, 275, 261, 267, 258, 280, 243, 252, 266, 250, 257, 267, 276, 255, 263, 277, 266, 271, 255, 260, 234, 254, 270, 259, 243, 265, 251, 263, 267] payload_rows_per_dst=[5712, 5706, 5402, 5572, 5440, 5196, 5444, 5426] dup_rows_per_dst=[2800, 3006, 2562, 2700, 2656, 2416, 2786, 2712] combine_rows_per_src=[5712, 5706, 5402, 5572, 5440, 5196, 5444, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38288 dup_rows=21824 dup_rows_remote=19154 dedup_ratio=0.333008 router_tokens_per_expert=[274, 276, 246, 241, 264, 280, 272, 252, 235, 278, 233, 249, 246, 266, 268, 270, 265, 242, 259, 260, 245, 248, 255, 257, 257, 255, 257, 250, 254, 267, 248, 234, 274, 263, 289, 251, 270, 273, 268, 239, 259, 250, 236, 243, 284, 255, 213, 234, 259, 266, 217, 259, 271, 252, 267, 273, 220, 260, 251, 279, 244, 261, 266, 268, 267, 268, 256, 274, 245, 282, 235, 272, 248, 278, 227, 263, 240, 279, 259, 252, 264, 226, 264, 281, 261, 249, 258, 250, 250, 241, 270, 275, 264, 252, 227, 252, 249, 231, 227, 243, 277, 259, 260, 249, 248, 241, 256, 283, 244, 252, 250, 278, 254, 256, 260, 272, 269, 258, 268, 245, 223, 249, 224, 249, 293, 273, 242, 240] payload_rows_per_dst=[5540, 5372, 5442, 5500, 5488, 5476, 5424, 5470] dup_rows_per_dst=[2760, 2734, 2760, 2726, 2802, 2692, 2670, 2680] combine_rows_per_src=[5540, 5372, 5442, 5500, 5488, 5476, 5424, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38252 dup_rows=21798 dup_rows_remote=19036 dedup_ratio=0.332611 router_tokens_per_expert=[250, 269, 244, 242, 265, 251, 248, 256, 258, 262, 251, 253, 250, 248, 242, 256, 248, 240, 273, 265, 242, 226, 249, 261, 272, 259, 264, 271, 277, 275, 245, 257, 290, 271, 265, 271, 259, 243, 265, 262, 232, 215, 238, 247, 237, 270, 275, 259, 276, 279, 234, 245, 250, 264, 299, 272, 266, 256, 244, 259, 251, 259, 228, 250, 252, 276, 277, 232, 269, 250, 267, 244, 243, 257, 251, 242, 263, 243, 257, 236, 263, 244, 268, 236, 217, 255, 252, 234, 252, 260, 254, 258, 248, 247, 225, 278, 272, 244, 243, 236, 270, 310, 244, 258, 236, 257, 267, 279, 293, 234, 249, 265, 251, 279, 284, 245, 271, 248, 292, 242, 252, 263, 250, 254, 253, 253, 260, 264] payload_rows_per_dst=[5368, 5486, 5488, 5526, 5504, 5362, 5510, 5494] dup_rows_per_dst=[2722, 2762, 2710, 2738, 2614, 2620, 2804, 2828] combine_rows_per_src=[5368, 5486, 5488, 5526, 5504, 5362, 5510, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43520 payload_rows_remote=38254 dup_rows=22016 dup_rows_remote=19322 dedup_ratio=0.335938 router_tokens_per_expert=[253, 243, 243, 238, 282, 238, 279, 262, 249, 292, 244, 258, 251, 272, 253, 261, 252, 282, 262, 280, 281, 285, 254, 253, 276, 226, 267, 241, 285, 264, 253, 255, 267, 260, 252, 254, 251, 253, 264, 263, 287, 244, 243, 257, 246, 229, 255, 240, 253, 251, 218, 260, 260, 244, 263, 251, 280, 250, 236, 241, 251, 256, 259, 245, 247, 260, 263, 238, 263, 278, 254, 243, 237, 265, 254, 248, 256, 238, 252, 287, 252, 233, 267, 247, 262, 245, 257, 267, 222, 255, 223, 249, 252, 243, 236, 270, 247, 265, 255, 266, 272, 257, 251, 243, 240, 271, 259, 256, 274, 258, 259, 276, 245, 281, 278, 257, 262, 231, 268, 272, 265, 232, 258, 250, 248, 254, 276, 262] payload_rows_per_dst=[5434, 5616, 5398, 5422, 5430, 5266, 5472, 5482] dup_rows_per_dst=[2802, 2816, 2732, 2614, 2736, 2694, 2826, 2796] combine_rows_per_src=[5434, 5616, 5398, 5422, 5430, 5266, 5472, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38434 dup_rows=21606 dup_rows_remote=18956 dedup_ratio=0.329681 router_tokens_per_expert=[285, 272, 229, 286, 228, 256, 279, 239, 251, 246, 271, 267, 278, 254, 261, 243, 245, 256, 240, 264, 236, 262, 277, 247, 283, 276, 272, 247, 260, 274, 244, 229, 251, 251, 279, 254, 209, 249, 265, 255, 238, 290, 270, 241, 251, 273, 248, 249, 283, 251, 259, 242, 259, 270, 271, 237, 231, 229, 233, 257, 281, 249, 260, 253, 228, 238, 234, 252, 231, 231, 221, 247, 276, 252, 280, 251, 263, 234, 277, 263, 256, 258, 268, 244, 291, 276, 265, 270, 252, 253, 284, 260, 248, 269, 273, 259, 255, 270, 243, 241, 275, 268, 267, 269, 253, 244, 247, 264, 259, 278, 276, 230, 237, 266, 263, 257, 241, 270, 255, 250, 231, 244, 278, 259, 248, 242, 255, 234] payload_rows_per_dst=[5594, 5478, 5496, 5492, 5374, 5564, 5548, 5384] dup_rows_per_dst=[2696, 2746, 2650, 2638, 2582, 2888, 2730, 2676] combine_rows_per_src=[5594, 5478, 5496, 5492, 5374, 5564, 5548, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43946 payload_rows_remote=38398 dup_rows=21590 dup_rows_remote=18818 dedup_ratio=0.329437 router_tokens_per_expert=[275, 253, 239, 277, 236, 233, 243, 264, 242, 243, 273, 256, 226, 268, 279, 209, 268, 266, 250, 251, 234, 236, 257, 260, 246, 253, 231, 252, 245, 250, 251, 251, 234, 257, 271, 259, 283, 269, 257, 260, 243, 217, 264, 264, 275, 241, 255, 263, 259, 244, 259, 293, 251, 246, 272, 236, 261, 257, 243, 248, 271, 286, 248, 286, 260, 284, 275, 266, 261, 244, 256, 261, 270, 274, 257, 262, 252, 261, 266, 249, 243, 249, 263, 245, 264, 266, 269, 247, 246, 241, 276, 225, 270, 244, 260, 253, 310, 265, 246, 265, 243, 254, 243, 258, 233, 289, 253, 254, 263, 268, 269, 242, 244, 256, 265, 265, 217, 248, 274, 262, 263, 284, 238, 277, 251, 262, 222, 237] payload_rows_per_dst=[5438, 5444, 5462, 5548, 5566, 5468, 5540, 5480] dup_rows_per_dst=[2594, 2558, 2762, 2772, 2830, 2654, 2770, 2650] combine_rows_per_src=[5438, 5444, 5462, 5548, 5566, 5468, 5540, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228984 combine_payload_bytes=156672000 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156900984 ingress_bytes=213295104 link_bytes=213295104 token_bytes=313572984 weight_movement_bytes=56623104 total_nvlink_bytes=370196088 weight_to_token_ratio=0.180574 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043688739e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228096 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156637952 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313047808 weight_movement_bytes=18874368 total_nvlink_bytes=331922176 weight_to_token_ratio=0.060292 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229768 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157057416 ingress_bytes=213450752 link_bytes=213450752 token_bytes=313885064 weight_movement_bytes=56623104 total_nvlink_bytes=370508168 weight_to_token_ratio=0.180394 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045180081e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38316 dup_rows=21702 dup_rows_remote=18948 dedup_ratio=0.331146 router_tokens_per_expert=[270, 238, 237, 217, 240, 262, 224, 251, 273, 236, 257, 260, 268, 240, 241, 263, 240, 274, 243, 250, 236, 250, 256, 250, 255, 260, 268, 268, 249, 243, 235, 250, 288, 263, 265, 251, 252, 222, 253, 256, 278, 253, 266, 248, 249, 282, 292, 274, 253, 279, 252, 240, 252, 247, 242, 266, 254, 242, 261, 265, 254, 248, 271, 231, 210, 265, 243, 248, 256, 232, 264, 286, 261, 283, 241, 263, 258, 282, 268, 276, 261, 257, 247, 283, 289, 242, 263, 241, 262, 249, 261, 254, 277, 251, 243, 269, 285, 257, 240, 259, 250, 249, 248, 274, 257, 272, 263, 247, 234, 232, 236, 271, 243, 253, 250, 253, 269, 265, 286, 294, 224, 270, 239, 257, 246, 287, 257, 263] payload_rows_per_dst=[5430, 5392, 5534, 5356, 5518, 5568, 5476, 5560] dup_rows_per_dst=[2524, 2662, 2850, 2758, 2754, 2730, 2672, 2752] combine_rows_per_src=[5430, 5392, 5534, 5356, 5518, 5568, 5476, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229152 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909344 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589536 weight_movement_bytes=18874368 total_nvlink_bytes=332463904 weight_to_token_ratio=0.060188 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=230304 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156918688 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313607072 weight_movement_bytes=18874368 total_nvlink_bytes=332481440 weight_to_token_ratio=0.060185 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157425664 dispatch_meta_bytes=229560 combine_payload_bytes=157425664 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157655224 ingress_bytes=232923136 link_bytes=232923136 token_bytes=315080888 weight_movement_bytes=75497472 total_nvlink_bytes=390578360 weight_to_token_ratio=0.239613 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.231754884e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=228864 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507072 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314785280 weight_movement_bytes=18874368 total_nvlink_bytes=333659648 weight_to_token_ratio=0.059960 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229056 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171392 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314113728 weight_movement_bytes=18874368 total_nvlink_bytes=332988096 weight_to_token_ratio=0.060088 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38402 dup_rows=21668 dup_rows_remote=18840 dedup_ratio=0.330627 router_tokens_per_expert=[255, 281, 235, 266, 263, 269, 269, 261, 243, 268, 256, 275, 237, 285, 265, 284, 251, 257, 282, 246, 265, 230, 259, 268, 236, 276, 278, 254, 269, 237, 239, 266, 267, 267, 277, 257, 249, 244, 250, 276, 239, 266, 265, 238, 293, 270, 251, 250, 255, 252, 251, 248, 254, 233, 234, 249, 287, 261, 254, 259, 283, 242, 242, 259, 252, 286, 251, 246, 262, 258, 248, 279, 225, 252, 267, 256, 260, 231, 251, 233, 247, 268, 246, 266, 246, 261, 226, 257, 281, 251, 240, 250, 263, 259, 278, 255, 287, 261, 238, 233, 251, 241, 269, 225, 244, 264, 236, 267, 242, 255, 246, 239, 238, 252, 253, 239, 265, 232, 261, 252, 270, 266, 246, 280, 250, 247, 276, 245] payload_rows_per_dst=[5464, 5462, 5514, 5474, 5510, 5528, 5450, 5466] dup_rows_per_dst=[2810, 2764, 2756, 2652, 2626, 2660, 2572, 2828] combine_rows_per_src=[5464, 5462, 5514, 5474, 5510, 5528, 5450, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38260 dup_rows=21838 dup_rows_remote=19106 dedup_ratio=0.333221 router_tokens_per_expert=[281, 249, 258, 278, 228, 266, 242, 235, 218, 248, 250, 249, 242, 251, 265, 244, 265, 231, 230, 242, 258, 269, 270, 236, 245, 255, 281, 251, 263, 233, 262, 253, 288, 257, 271, 270, 231, 256, 256, 257, 266, 270, 256, 230, 266, 237, 261, 244, 266, 297, 257, 266, 287, 233, 276, 275, 263, 281, 255, 264, 257, 258, 252, 258, 245, 262, 249, 237, 226, 250, 266, 256, 242, 272, 295, 237, 239, 230, 282, 263, 241, 260, 258, 214, 261, 255, 283, 270, 226, 238, 243, 262, 260, 230, 264, 249, 244, 254, 240, 247, 267, 268, 235, 265, 275, 242, 253, 231, 272, 266, 281, 255, 277, 241, 254, 259, 275, 275, 269, 247, 264, 268, 240, 264, 270, 265, 253, 278] payload_rows_per_dst=[5438, 5442, 5634, 5376, 5414, 5370, 5508, 5516] dup_rows_per_dst=[2732, 2742, 2926, 2624, 2688, 2562, 2682, 2882] combine_rows_per_src=[5438, 5442, 5634, 5376, 5414, 5370, 5508, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38122 dup_rows=21940 dup_rows_remote=19184 dedup_ratio=0.334778 router_tokens_per_expert=[272, 270, 262, 241, 236, 249, 243, 300, 249, 223, 276, 243, 271, 244, 279, 266, 268, 260, 258, 280, 282, 255, 276, 260, 258, 233, 241, 257, 261, 254, 244, 274, 258, 232, 277, 255, 243, 248, 258, 254, 237, 267, 273, 260, 260, 272, 285, 230, 303, 237, 241, 244, 256, 275, 259, 267, 235, 261, 268, 241, 242, 270, 246, 276, 249, 298, 258, 271, 241, 295, 262, 235, 263, 246, 274, 239, 243, 253, 244, 236, 267, 251, 240, 261, 258, 260, 237, 266, 239, 228, 251, 240, 266, 254, 251, 221, 265, 251, 287, 270, 228, 236, 253, 240, 256, 260, 266, 244, 250, 252, 271, 286, 224, 245, 278, 251, 258, 246, 260, 257, 261, 266, 258, 240, 262, 256, 245, 234] payload_rows_per_dst=[5398, 5536, 5470, 5544, 5428, 5370, 5474, 5376] dup_rows_per_dst=[2850, 2786, 2748, 2698, 2786, 2610, 2756, 2706] combine_rows_per_src=[5398, 5536, 5470, 5544, 5428, 5370, 5474, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38288 dup_rows=21830 dup_rows_remote=19064 dedup_ratio=0.333099 router_tokens_per_expert=[253, 262, 267, 247, 266, 251, 244, 267, 237, 265, 281, 231, 254, 247, 248, 273, 239, 261, 220, 231, 262, 276, 258, 238, 275, 274, 253, 254, 280, 264, 251, 256, 223, 254, 276, 244, 254, 273, 272, 242, 255, 270, 234, 265, 266, 248, 283, 239, 261, 245, 260, 268, 245, 247, 289, 265, 266, 247, 202, 237, 281, 226, 268, 274, 254, 278, 271, 246, 261, 267, 266, 249, 251, 244, 244, 275, 256, 253, 237, 273, 272, 252, 241, 260, 266, 257, 255, 241, 270, 256, 245, 263, 236, 252, 268, 270, 279, 228, 242, 251, 245, 197, 283, 241, 259, 285, 272, 270, 238, 256, 263, 242, 255, 251, 271, 251, 264, 239, 270, 273, 233, 240, 267, 252, 242, 260, 283, 273] payload_rows_per_dst=[5464, 5418, 5440, 5448, 5556, 5470, 5436, 5474] dup_rows_per_dst=[2722, 2766, 2756, 2714, 2694, 2738, 2666, 2774] combine_rows_per_src=[5464, 5418, 5440, 5448, 5556, 5470, 5436, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38230 dup_rows=21820 dup_rows_remote=18974 dedup_ratio=0.332947 router_tokens_per_expert=[256, 241, 244, 257, 273, 263, 293, 249, 252, 243, 285, 266, 236, 262, 263, 264, 266, 272, 253, 233, 265, 259, 256, 265, 265, 258, 262, 229, 233, 262, 241, 248, 277, 277, 266, 275, 229, 242, 249, 263, 265, 255, 239, 255, 238, 263, 252, 260, 270, 242, 262, 224, 288, 250, 233, 237, 229, 253, 254, 254, 243, 285, 244, 252, 219, 238, 259, 227, 244, 245, 233, 256, 255, 279, 253, 265, 276, 250, 251, 262, 260, 298, 264, 254, 278, 254, 238, 242, 254, 271, 258, 235, 263, 261, 279, 257, 245, 239, 286, 255, 240, 274, 261, 254, 269, 264, 288, 264, 274, 260, 243, 262, 258, 236, 261, 257, 265, 285, 240, 271, 258, 259, 246, 276, 244, 233, 251, 233] payload_rows_per_dst=[5496, 5478, 5462, 5350, 5438, 5486, 5572, 5434] dup_rows_per_dst=[2798, 2656, 2748, 2690, 2586, 2846, 2784, 2712] combine_rows_per_src=[5496, 5478, 5462, 5350, 5438, 5486, 5572, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=228968 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157523560 ingress_bytes=176168960 link_bytes=176168960 token_bytes=314818152 weight_movement_bytes=18874368 total_nvlink_bytes=333692520 weight_to_token_ratio=0.059953 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687964294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38140 dup_rows=21886 dup_rows_remote=19188 dedup_ratio=0.333954 router_tokens_per_expert=[230, 239, 283, 254, 282, 261, 275, 249, 250, 269, 271, 259, 252, 276, 234, 258, 236, 259, 230, 263, 266, 264, 251, 255, 239, 282, 283, 248, 214, 229, 284, 252, 275, 240, 263, 269, 267, 253, 249, 254, 284, 256, 246, 250, 237, 244, 262, 255, 229, 263, 250, 249, 257, 241, 241, 258, 277, 273, 261, 269, 257, 268, 259, 227, 250, 280, 267, 250, 247, 270, 248, 245, 269, 267, 244, 252, 286, 273, 247, 264, 248, 277, 234, 251, 277, 254, 260, 259, 264, 257, 303, 263, 259, 262, 245, 229, 229, 236, 273, 239, 264, 257, 256, 265, 266, 258, 259, 271, 238, 265, 231, 229, 241, 270, 268, 261, 253, 276, 248, 259, 255, 259, 249, 237, 232, 244, 252, 247] payload_rows_per_dst=[5462, 5394, 5510, 5438, 5534, 5486, 5378, 5448] dup_rows_per_dst=[2822, 2716, 2698, 2720, 2784, 2798, 2694, 2654] combine_rows_per_src=[5462, 5394, 5510, 5438, 5534, 5486, 5378, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43530 payload_rows_remote=38050 dup_rows=22006 dup_rows_remote=19280 dedup_ratio=0.335785 router_tokens_per_expert=[246, 230, 254, 263, 236, 237, 267, 244, 245, 245, 242, 284, 234, 225, 235, 252, 277, 284, 255, 251, 262, 259, 256, 269, 257, 266, 256, 259, 242, 272, 273, 253, 288, 251, 248, 255, 253, 246, 262, 257, 240, 243, 264, 266, 245, 233, 205, 266, 273, 255, 280, 283, 244, 264, 232, 241, 220, 246, 251, 270, 245, 279, 257, 263, 235, 232, 260, 263, 267, 218, 277, 275, 241, 227, 260, 291, 273, 266, 239, 271, 261, 268, 240, 251, 264, 249, 260, 266, 257, 256, 255, 254, 257, 271, 252, 245, 231, 248, 266, 261, 267, 267, 253, 254, 264, 262, 244, 263, 255, 260, 282, 264, 253, 250, 279, 260, 248, 259, 226, 264, 272, 264, 287, 243, 253, 273, 252, 288] payload_rows_per_dst=[5332, 5506, 5414, 5480, 5396, 5444, 5440, 5518] dup_rows_per_dst=[2546, 2876, 2630, 2726, 2794, 2768, 2842, 2824] combine_rows_per_src=[5332, 5506, 5414, 5480, 5396, 5444, 5440, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38158 dup_rows=21942 dup_rows_remote=19268 dedup_ratio=0.334808 router_tokens_per_expert=[233, 263, 276, 241, 228, 240, 262, 242, 244, 257, 252, 243, 263, 233, 261, 261, 252, 239, 236, 247, 251, 251, 237, 245, 267, 250, 248, 258, 254, 256, 279, 239, 274, 257, 260, 267, 269, 257, 268, 249, 261, 250, 278, 255, 238, 237, 245, 255, 269, 246, 252, 256, 249, 274, 258, 249, 267, 241, 261, 244, 227, 248, 302, 265, 257, 261, 232, 271, 289, 229, 239, 244, 231, 257, 254, 251, 246, 275, 245, 274, 256, 252, 268, 255, 258, 301, 269, 269, 247, 262, 252, 265, 285, 238, 287, 233, 293, 284, 280, 264, 247, 249, 251, 251, 253, 240, 272, 269, 263, 229, 262, 255, 267, 257, 258, 274, 261, 261, 275, 251, 266, 249, 259, 227, 265, 253, 258, 237] payload_rows_per_dst=[5374, 5454, 5424, 5438, 5436, 5504, 5522, 5442] dup_rows_per_dst=[2624, 2564, 2816, 2778, 2674, 2890, 2802, 2794] combine_rows_per_src=[5374, 5454, 5424, 5438, 5436, 5504, 5522, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229464 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942424 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655384 weight_movement_bytes=18874368 total_nvlink_bytes=332529752 weight_to_token_ratio=0.060175 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229224 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376936 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524648 weight_movement_bytes=18874368 total_nvlink_bytes=331399016 weight_to_token_ratio=0.060393 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229408 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057056 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884704 weight_movement_bytes=18874368 total_nvlink_bytes=332759072 weight_to_token_ratio=0.060132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228816 combine_payload_bytes=156590080 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156818896 ingress_bytes=213213184 link_bytes=213213184 token_bytes=313408976 weight_movement_bytes=56623104 total_nvlink_bytes=370032080 weight_to_token_ratio=0.180668 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042903822e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229312 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156450752 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312672192 weight_movement_bytes=37748736 total_nvlink_bytes=350420928 weight_to_token_ratio=0.120729 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=229320 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156082120 ingress_bytes=193601536 link_bytes=193601536 token_bytes=311934920 weight_movement_bytes=37748736 total_nvlink_bytes=349683656 weight_to_token_ratio=0.121015 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854994660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229704 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156524872 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312820040 weight_movement_bytes=37748736 total_nvlink_bytes=350568776 weight_to_token_ratio=0.120672 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38274 dup_rows=21832 dup_rows_remote=19172 dedup_ratio=0.333130 router_tokens_per_expert=[286, 262, 257, 244, 268, 267, 226, 261, 279, 259, 240, 241, 231, 253, 248, 244, 237, 297, 239, 266, 230, 271, 249, 245, 255, 243, 279, 257, 244, 268, 251, 244, 257, 270, 239, 241, 261, 273, 263, 269, 253, 228, 236, 269, 269, 267, 254, 247, 274, 258, 252, 257, 252, 298, 280, 259, 253, 249, 266, 261, 259, 289, 262, 248, 251, 244, 283, 252, 247, 258, 276, 268, 225, 254, 261, 247, 278, 258, 263, 251, 265, 239, 244, 246, 262, 254, 253, 250, 256, 266, 243, 276, 288, 225, 244, 260, 239, 277, 241, 258, 235, 234, 250, 246, 253, 253, 242, 287, 259, 279, 264, 270, 256, 247, 235, 249, 233, 247, 227, 257, 258, 268, 280, 261, 272, 250, 247, 253] payload_rows_per_dst=[5422, 5354, 5404, 5638, 5468, 5520, 5468, 5430] dup_rows_per_dst=[2710, 2796, 2670, 2914, 2610, 2776, 2696, 2660] combine_rows_per_src=[5422, 5354, 5404, 5638, 5468, 5520, 5468, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38130 dup_rows=21932 dup_rows_remote=19112 dedup_ratio=0.334656 router_tokens_per_expert=[254, 271, 279, 253, 241, 257, 236, 233, 256, 248, 271, 265, 259, 243, 221, 251, 266, 272, 230, 272, 254, 236, 260, 266, 247, 249, 257, 244, 263, 276, 258, 240, 245, 263, 242, 240, 250, 222, 234, 243, 265, 267, 225, 263, 259, 256, 249, 278, 252, 251, 259, 242, 284, 250, 240, 273, 261, 279, 262, 257, 249, 248, 256, 249, 270, 224, 254, 274, 265, 261, 267, 243, 239, 232, 263, 249, 279, 246, 250, 267, 260, 269, 272, 261, 232, 293, 269, 236, 246, 262, 263, 243, 244, 229, 306, 250, 244, 273, 270, 261, 273, 266, 280, 268, 237, 258, 237, 245, 244, 276, 280, 235, 244, 282, 228, 247, 254, 274, 258, 281, 255, 256, 239, 274, 252, 277, 274, 267] payload_rows_per_dst=[5460, 5448, 5356, 5468, 5434, 5488, 5474, 5476] dup_rows_per_dst=[2616, 2732, 2646, 2756, 2732, 2782, 2820, 2848] combine_rows_per_src=[5460, 5448, 5356, 5468, 5434, 5488, 5474, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38258 dup_rows=21728 dup_rows_remote=18838 dedup_ratio=0.331543 router_tokens_per_expert=[268, 263, 265, 255, 283, 238, 271, 260, 263, 258, 263, 271, 246, 262, 271, 283, 303, 269, 240, 268, 236, 248, 239, 256, 235, 254, 257, 274, 278, 261, 268, 253, 257, 278, 253, 251, 247, 246, 255, 253, 249, 265, 237, 283, 251, 249, 240, 254, 241, 247, 265, 233, 242, 279, 282, 249, 246, 261, 228, 250, 275, 236, 262, 272, 231, 236, 268, 253, 247, 274, 242, 268, 272, 271, 232, 264, 263, 275, 247, 253, 259, 244, 238, 288, 266, 261, 255, 241, 250, 231, 255, 244, 232, 288, 261, 260, 238, 284, 244, 241, 273, 227, 253, 266, 236, 240, 227, 270, 240, 242, 258, 285, 281, 252, 254, 219, 234, 257, 269, 230, 262, 260, 249, 250, 260, 274, 258, 271] payload_rows_per_dst=[5550, 5568, 5548, 5622, 5424, 5448, 5480, 5168] dup_rows_per_dst=[2890, 2710, 2910, 2926, 2768, 2698, 2568, 2258] combine_rows_per_src=[5550, 5568, 5548, 5622, 5424, 5448, 5480, 5168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38102 dup_rows=21950 dup_rows_remote=19182 dedup_ratio=0.334930 router_tokens_per_expert=[254, 260, 262, 264, 256, 225, 237, 244, 258, 263, 248, 282, 253, 270, 265, 259, 261, 275, 262, 251, 229, 272, 259, 281, 274, 220, 265, 267, 245, 257, 241, 267, 243, 252, 272, 289, 223, 266, 253, 258, 257, 284, 252, 248, 273, 264, 271, 244, 239, 276, 231, 255, 271, 260, 257, 257, 277, 245, 245, 220, 272, 263, 213, 275, 253, 244, 273, 239, 280, 247, 242, 252, 241, 240, 280, 272, 270, 229, 271, 257, 235, 251, 234, 213, 249, 268, 240, 234, 258, 294, 249, 251, 255, 244, 236, 248, 233, 245, 233, 245, 268, 276, 281, 268, 264, 269, 245, 277, 246, 257, 272, 274, 252, 238, 250, 275, 256, 257, 256, 243, 287, 263, 258, 246, 268, 272, 262, 252] payload_rows_per_dst=[5498, 5484, 5482, 5388, 5490, 5308, 5454, 5482] dup_rows_per_dst=[2702, 2768, 2816, 2724, 2690, 2610, 2852, 2788] combine_rows_per_src=[5498, 5484, 5482, 5388, 5490, 5308, 5454, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229784 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157000088 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770392 weight_movement_bytes=37748736 total_nvlink_bytes=351519128 weight_to_token_ratio=0.120307 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38302 dup_rows=21724 dup_rows_remote=18930 dedup_ratio=0.331482 router_tokens_per_expert=[239, 253, 235, 266, 230, 264, 238, 276, 248, 244, 236, 251, 254, 264, 226, 238, 263, 256, 257, 245, 239, 258, 253, 257, 245, 249, 267, 269, 276, 266, 241, 272, 256, 243, 252, 270, 259, 258, 233, 266, 256, 270, 247, 269, 269, 266, 270, 268, 251, 283, 271, 242, 272, 260, 236, 274, 277, 279, 252, 267, 253, 260, 238, 243, 240, 248, 273, 235, 243, 288, 282, 261, 250, 285, 279, 286, 253, 246, 236, 244, 264, 232, 272, 255, 234, 266, 246, 273, 228, 246, 281, 275, 234, 227, 248, 269, 247, 261, 257, 268, 266, 259, 253, 258, 242, 237, 276, 261, 254, 232, 278, 271, 265, 266, 245, 238, 263, 242, 253, 246, 259, 257, 226, 241, 278, 261, 248, 276] payload_rows_per_dst=[5408, 5514, 5510, 5558, 5522, 5376, 5514, 5410] dup_rows_per_dst=[2516, 2712, 2794, 2758, 2776, 2724, 2726, 2718] combine_rows_per_src=[5408, 5514, 5510, 5558, 5522, 5376, 5514, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38138 dup_rows=21926 dup_rows_remote=19152 dedup_ratio=0.334564 router_tokens_per_expert=[274, 239, 274, 249, 260, 248, 281, 251, 277, 262, 274, 268, 245, 264, 250, 258, 271, 245, 239, 259, 283, 256, 276, 248, 232, 245, 267, 262, 254, 271, 227, 270, 255, 295, 258, 257, 283, 253, 265, 270, 249, 275, 253, 226, 247, 279, 285, 248, 238, 255, 254, 238, 243, 263, 240, 243, 286, 266, 240, 276, 233, 268, 249, 264, 264, 270, 261, 276, 241, 248, 243, 255, 252, 258, 237, 280, 250, 275, 239, 261, 262, 246, 270, 258, 287, 215, 245, 281, 254, 245, 252, 249, 262, 273, 248, 276, 264, 241, 239, 253, 253, 274, 236, 243, 248, 268, 250, 257, 265, 220, 289, 244, 263, 208, 255, 248, 236, 269, 235, 247, 239, 252, 252, 259, 233, 257, 242, 263] payload_rows_per_dst=[5504, 5352, 5576, 5392, 5486, 5472, 5454, 5374] dup_rows_per_dst=[2844, 2858, 2820, 2720, 2734, 2774, 2634, 2542] combine_rows_per_src=[5504, 5352, 5576, 5392, 5486, 5472, 5454, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38350 dup_rows=21658 dup_rows_remote=19020 dedup_ratio=0.330475 router_tokens_per_expert=[247, 248, 245, 244, 264, 225, 256, 243, 289, 242, 250, 270, 259, 288, 265, 266, 233, 273, 243, 234, 271, 253, 268, 250, 228, 253, 247, 242, 221, 275, 275, 246, 241, 258, 246, 250, 266, 249, 273, 250, 249, 237, 274, 262, 255, 255, 267, 271, 246, 272, 254, 267, 239, 265, 238, 248, 257, 233, 260, 251, 259, 283, 253, 258, 267, 258, 250, 260, 211, 241, 266, 250, 270, 246, 266, 284, 269, 244, 240, 267, 254, 256, 282, 253, 250, 295, 255, 244, 234, 247, 275, 263, 289, 255, 220, 260, 263, 278, 246, 245, 229, 265, 266, 263, 240, 244, 256, 247, 264, 247, 265, 266, 259, 240, 282, 250, 246, 244, 265, 261, 250, 245, 293, 255, 261, 288, 250, 275] payload_rows_per_dst=[5514, 5380, 5428, 5528, 5450, 5612, 5430, 5536] dup_rows_per_dst=[2688, 2644, 2778, 2638, 2728, 2652, 2738, 2792] combine_rows_per_src=[5514, 5380, 5428, 5528, 5450, 5612, 5430, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38180 dup_rows=21846 dup_rows_remote=19106 dedup_ratio=0.333344 router_tokens_per_expert=[266, 266, 264, 258, 240, 264, 264, 259, 273, 258, 253, 262, 271, 256, 279, 264, 256, 264, 241, 246, 242, 275, 245, 286, 248, 257, 224, 244, 247, 238, 258, 240, 237, 252, 269, 262, 279, 238, 256, 270, 252, 266, 250, 255, 259, 275, 248, 257, 252, 254, 268, 253, 260, 254, 284, 263, 236, 234, 248, 242, 242, 310, 269, 249, 263, 259, 260, 245, 218, 252, 274, 264, 244, 275, 267, 273, 261, 246, 260, 264, 265, 275, 282, 240, 273, 231, 275, 248, 234, 241, 241, 249, 236, 256, 248, 241, 225, 253, 254, 291, 248, 260, 253, 247, 249, 276, 240, 263, 253, 239, 268, 238, 259, 247, 272, 248, 272, 260, 261, 239, 271, 245, 265, 249, 230, 277, 241, 264] payload_rows_per_dst=[5526, 5372, 5488, 5544, 5510, 5388, 5364, 5498] dup_rows_per_dst=[2868, 2650, 2762, 2692, 2740, 2682, 2750, 2702] combine_rows_per_src=[5526, 5372, 5488, 5544, 5510, 5388, 5364, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=228968 combine_payload_bytes=156180480 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156409448 ingress_bytes=193929216 link_bytes=193929216 token_bytes=312589928 weight_movement_bytes=37748736 total_nvlink_bytes=350338664 weight_to_token_ratio=0.120761 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858134329e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228384 combine_payload_bytes=156704768 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156933152 ingress_bytes=232202240 link_bytes=232202240 token_bytes=313637920 weight_movement_bytes=75497472 total_nvlink_bytes=389135392 weight_to_token_ratio=0.240715 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224847613e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229136 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156294928 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312360720 weight_movement_bytes=18874368 total_nvlink_bytes=331235088 weight_to_token_ratio=0.060425 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=228928 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157113920 ingress_bytes=213508096 link_bytes=213508096 token_bytes=313998912 weight_movement_bytes=56623104 total_nvlink_bytes=370622016 weight_to_token_ratio=0.180329 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045729523e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229160 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156442408 ingress_bytes=193961984 link_bytes=193961984 token_bytes=312655656 weight_movement_bytes=37748736 total_nvlink_bytes=350404392 weight_to_token_ratio=0.120736 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858448295e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229480 combine_payload_bytes=157081600 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157311080 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392680 weight_movement_bytes=18874368 total_nvlink_bytes=333267048 weight_to_token_ratio=0.060034 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229144 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156614424 ingress_bytes=194134016 link_bytes=194134016 token_bytes=312999704 weight_movement_bytes=37748736 total_nvlink_bytes=350748440 weight_to_token_ratio=0.120603 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38150 dup_rows=21892 dup_rows_remote=18992 dedup_ratio=0.334045 router_tokens_per_expert=[255, 256, 252, 258, 232, 258, 237, 260, 276, 257, 256, 260, 246, 258, 242, 265, 229, 271, 253, 240, 254, 249, 259, 260, 250, 283, 237, 253, 270, 263, 281, 263, 264, 277, 253, 234, 238, 268, 292, 293, 269, 273, 230, 262, 252, 251, 264, 238, 263, 231, 262, 282, 260, 222, 265, 251, 256, 235, 245, 261, 264, 241, 265, 254, 250, 258, 296, 259, 251, 249, 251, 241, 241, 229, 275, 253, 271, 267, 247, 252, 251, 280, 258, 226, 226, 247, 233, 267, 264, 251, 251, 276, 236, 249, 261, 209, 260, 265, 251, 273, 262, 253, 239, 250, 263, 256, 267, 253, 251, 271, 273, 246, 273, 247, 292, 271, 246, 272, 266, 298, 261, 250, 260, 227, 236, 255, 256, 252] payload_rows_per_dst=[5460, 5416, 5474, 5414, 5322, 5432, 5632, 5494] dup_rows_per_dst=[2676, 2814, 2772, 2700, 2612, 2538, 2880, 2900] combine_rows_per_src=[5460, 5416, 5474, 5414, 5322, 5432, 5632, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38386 dup_rows=21726 dup_rows_remote=18962 dedup_ratio=0.331512 router_tokens_per_expert=[258, 245, 261, 241, 239, 282, 285, 270, 258, 260, 274, 256, 285, 236, 261, 266, 242, 269, 236, 261, 235, 263, 275, 254, 254, 246, 237, 269, 253, 257, 220, 246, 253, 282, 284, 224, 233, 247, 261, 244, 221, 292, 239, 275, 245, 244, 268, 260, 257, 281, 221, 255, 266, 248, 261, 228, 244, 294, 246, 277, 268, 268, 240, 239, 275, 242, 269, 261, 270, 266, 276, 256, 245, 265, 265, 265, 250, 257, 251, 255, 260, 254, 244, 262, 246, 251, 227, 243, 265, 269, 269, 266, 271, 246, 249, 270, 268, 248, 274, 238, 231, 267, 244, 291, 247, 244, 264, 237, 251, 257, 253, 280, 268, 264, 249, 267, 238, 260, 257, 218, 256, 266, 250, 254, 232, 240, 271, 265] payload_rows_per_dst=[5566, 5384, 5438, 5456, 5550, 5534, 5424, 5458] dup_rows_per_dst=[2788, 2650, 2706, 2730, 2786, 2650, 2764, 2652] combine_rows_per_src=[5566, 5384, 5438, 5456, 5550, 5534, 5424, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228568 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156490968 ingress_bytes=231759872 link_bytes=231759872 token_bytes=312753368 weight_movement_bytes=75497472 total_nvlink_bytes=388250840 weight_to_token_ratio=0.241396 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220609061e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43504 payload_rows_remote=37978 dup_rows=22032 dup_rows_remote=19188 dedup_ratio=0.336182 router_tokens_per_expert=[252, 237, 262, 240, 257, 285, 231, 254, 294, 254, 268, 274, 296, 270, 249, 262, 238, 251, 272, 268, 267, 248, 263, 263, 252, 241, 229, 218, 253, 238, 263, 257, 252, 271, 237, 248, 259, 235, 272, 241, 251, 241, 266, 234, 270, 278, 259, 254, 236, 298, 259, 304, 241, 248, 263, 233, 227, 305, 276, 265, 235, 282, 238, 271, 276, 287, 257, 244, 254, 247, 235, 264, 248, 252, 250, 218, 232, 252, 280, 266, 264, 253, 265, 260, 250, 266, 254, 278, 269, 273, 246, 234, 255, 232, 260, 242, 291, 248, 239, 239, 246, 246, 272, 240, 237, 264, 257, 257, 271, 251, 248, 244, 253, 258, 255, 255, 269, 228, 257, 237, 250, 255, 270, 236, 255, 261, 282, 279] payload_rows_per_dst=[5526, 5416, 5458, 5442, 5542, 5246, 5622, 5252] dup_rows_per_dst=[2844, 2800, 2678, 2806, 2852, 2626, 2978, 2448] combine_rows_per_src=[5526, 5416, 5458, 5442, 5542, 5246, 5622, 5252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38166 dup_rows=21794 dup_rows_remote=18900 dedup_ratio=0.332550 router_tokens_per_expert=[274, 254, 280, 226, 254, 266, 278, 233, 266, 244, 244, 274, 246, 265, 259, 269, 276, 240, 227, 276, 235, 283, 257, 266, 276, 240, 249, 255, 240, 257, 263, 277, 266, 253, 310, 255, 238, 255, 251, 280, 248, 286, 260, 260, 281, 257, 270, 265, 271, 272, 289, 246, 237, 270, 278, 258, 266, 236, 242, 254, 238, 221, 235, 238, 263, 245, 248, 239, 243, 224, 242, 269, 260, 246, 251, 216, 259, 260, 261, 263, 253, 235, 275, 240, 252, 241, 237, 281, 250, 258, 241, 269, 255, 251, 269, 250, 250, 268, 296, 257, 260, 267, 272, 266, 256, 232, 224, 245, 261, 242, 276, 276, 242, 278, 260, 247, 229, 281, 244, 228, 275, 248, 256, 239, 261, 271, 224, 256] payload_rows_per_dst=[5516, 5460, 5576, 5442, 5394, 5464, 5484, 5406] dup_rows_per_dst=[2748, 2774, 2894, 2660, 2584, 2650, 2812, 2672] combine_rows_per_src=[5516, 5460, 5576, 5442, 5394, 5464, 5484, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38328 dup_rows=21734 dup_rows_remote=18968 dedup_ratio=0.331635 router_tokens_per_expert=[251, 257, 221, 236, 244, 279, 264, 245, 242, 242, 274, 261, 237, 260, 248, 245, 260, 268, 273, 234, 272, 254, 249, 254, 262, 291, 251, 260, 226, 260, 236, 270, 260, 281, 242, 262, 233, 242, 267, 278, 239, 240, 300, 264, 238, 239, 273, 250, 270, 255, 258, 250, 252, 279, 251, 249, 265, 232, 247, 264, 253, 270, 255, 247, 276, 264, 249, 264, 262, 246, 257, 247, 250, 260, 271, 237, 280, 255, 243, 223, 273, 265, 250, 259, 260, 253, 252, 257, 255, 240, 250, 268, 225, 247, 260, 230, 229, 242, 259, 265, 240, 244, 256, 252, 255, 265, 243, 256, 235, 264, 274, 266, 250, 269, 246, 265, 266, 262, 260, 294, 259, 224, 276, 271, 263, 268, 309, 282] payload_rows_per_dst=[5468, 5474, 5494, 5468, 5522, 5408, 5422, 5546] dup_rows_per_dst=[2544, 2766, 2722, 2726, 2646, 2680, 2668, 2982] combine_rows_per_src=[5468, 5474, 5494, 5468, 5522, 5408, 5422, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38220 dup_rows=21942 dup_rows_remote=19196 dedup_ratio=0.334808 router_tokens_per_expert=[237, 249, 283, 249, 249, 249, 247, 258, 250, 268, 274, 288, 251, 249, 269, 237, 244, 242, 276, 268, 270, 263, 264, 269, 262, 220, 248, 257, 293, 239, 263, 307, 282, 240, 290, 272, 247, 269, 249, 237, 280, 255, 245, 261, 258, 267, 252, 255, 253, 267, 254, 237, 273, 269, 227, 242, 232, 242, 233, 273, 257, 263, 236, 272, 269, 276, 259, 262, 239, 243, 245, 259, 244, 276, 244, 261, 242, 246, 256, 211, 260, 255, 258, 256, 227, 261, 272, 243, 272, 236, 235, 259, 268, 264, 254, 240, 261, 257, 264, 250, 223, 230, 227, 261, 272, 244, 249, 261, 260, 280, 240, 264, 284, 216, 265, 261, 255, 246, 288, 262, 268, 275, 265, 253, 257, 241, 271, 245] payload_rows_per_dst=[5364, 5600, 5588, 5438, 5374, 5374, 5414, 5442] dup_rows_per_dst=[2850, 2770, 2730, 2622, 2690, 2746, 2672, 2862] combine_rows_per_src=[5364, 5600, 5588, 5438, 5374, 5374, 5414, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38344 dup_rows=21634 dup_rows_remote=19026 dedup_ratio=0.330109 router_tokens_per_expert=[270, 205, 263, 247, 267, 275, 268, 255, 268, 267, 239, 246, 244, 235, 253, 260, 246, 261, 281, 276, 246, 235, 271, 251, 235, 258, 279, 263, 258, 251, 252, 259, 270, 255, 254, 237, 252, 241, 271, 251, 225, 246, 280, 262, 254, 260, 238, 261, 266, 283, 249, 267, 243, 251, 237, 242, 239, 242, 257, 254, 269, 273, 238, 273, 275, 272, 249, 246, 290, 224, 253, 268, 260, 253, 265, 258, 257, 243, 262, 261, 257, 245, 267, 250, 249, 244, 259, 273, 244, 255, 251, 265, 255, 276, 276, 265, 252, 276, 252, 255, 251, 229, 275, 259, 255, 225, 251, 250, 254, 255, 259, 272, 271, 273, 257, 273, 261, 255, 243, 215, 256, 251, 250, 251, 262, 261, 267, 261] payload_rows_per_dst=[5464, 5488, 5418, 5558, 5508, 5544, 5468, 5454] dup_rows_per_dst=[2660, 2756, 2696, 2608, 2764, 2718, 2672, 2760] combine_rows_per_src=[5464, 5488, 5418, 5558, 5508, 5544, 5468, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38170 dup_rows=21898 dup_rows_remote=19068 dedup_ratio=0.334137 router_tokens_per_expert=[271, 263, 239, 250, 230, 250, 239, 268, 251, 250, 257, 250, 274, 226, 254, 262, 245, 253, 240, 281, 221, 248, 280, 263, 242, 273, 252, 278, 254, 238, 252, 268, 250, 256, 244, 233, 277, 252, 251, 245, 285, 281, 257, 275, 242, 255, 247, 276, 254, 273, 255, 247, 267, 301, 253, 255, 242, 260, 232, 223, 277, 265, 253, 242, 258, 265, 270, 261, 257, 266, 239, 259, 264, 252, 292, 236, 219, 258, 266, 287, 243, 241, 274, 267, 233, 231, 245, 246, 255, 241, 268, 263, 248, 270, 268, 244, 251, 273, 264, 237, 269, 258, 258, 253, 283, 264, 255, 268, 254, 244, 242, 275, 260, 268, 254, 254, 257, 277, 250, 233, 282, 285, 239, 244, 280, 226, 237, 241] payload_rows_per_dst=[5382, 5492, 5446, 5492, 5468, 5446, 5464, 5448] dup_rows_per_dst=[2686, 2684, 2806, 2706, 2830, 2628, 2832, 2726] combine_rows_per_src=[5382, 5492, 5446, 5492, 5468, 5446, 5464, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229392 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458448 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687504 weight_movement_bytes=18874368 total_nvlink_bytes=333561872 weight_to_token_ratio=0.059978 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155557888 dispatch_meta_bytes=228664 combine_payload_bytes=155557888 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=155786552 ingress_bytes=212180992 link_bytes=212180992 token_bytes=311344440 weight_movement_bytes=56623104 total_nvlink_bytes=367967544 weight_to_token_ratio=0.181866 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.033013866e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228264 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156556200 ingress_bytes=194076672 link_bytes=194076672 token_bytes=312884136 weight_movement_bytes=37748736 total_nvlink_bytes=350632872 weight_to_token_ratio=0.120648 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859547179e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229184 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220672 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212160 weight_movement_bytes=18874368 total_nvlink_bytes=333086528 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229664 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778784 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313327904 weight_movement_bytes=18874368 total_nvlink_bytes=332202272 weight_to_token_ratio=0.060238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229480 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286504 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343528 weight_movement_bytes=18874368 total_nvlink_bytes=333217896 weight_to_token_ratio=0.060044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228952 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573272 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917592 weight_movement_bytes=18874368 total_nvlink_bytes=331791960 weight_to_token_ratio=0.060317 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38030 dup_rows=21952 dup_rows_remote=19100 dedup_ratio=0.334961 router_tokens_per_expert=[255, 261, 242, 264, 287, 250, 247, 257, 249, 256, 259, 263, 236, 240, 284, 271, 251, 250, 266, 230, 264, 228, 228, 272, 253, 266, 269, 307, 275, 258, 252, 276, 279, 246, 266, 279, 288, 267, 289, 290, 265, 249, 244, 245, 261, 253, 257, 241, 248, 250, 241, 250, 266, 284, 242, 243, 227, 258, 211, 228, 242, 266, 248, 250, 236, 252, 264, 248, 247, 252, 251, 270, 270, 250, 252, 261, 250, 266, 268, 251, 266, 239, 243, 263, 258, 215, 278, 259, 242, 258, 253, 268, 270, 269, 241, 247, 238, 253, 231, 254, 276, 258, 250, 261, 258, 245, 253, 271, 262, 259, 241, 249, 270, 263, 253, 250, 278, 235, 249, 250, 274, 248, 232, 250, 269, 264, 274, 254] payload_rows_per_dst=[5520, 5298, 5442, 5524, 5374, 5460, 5412, 5554] dup_rows_per_dst=[2722, 2396, 2736, 2962, 2992, 2678, 2614, 2852] combine_rows_per_src=[5520, 5298, 5442, 5524, 5374, 5460, 5412, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=228520 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155999400 ingress_bytes=174645248 link_bytes=174645248 token_bytes=311770280 weight_movement_bytes=18874368 total_nvlink_bytes=330644648 weight_to_token_ratio=0.060539 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673364836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38272 dup_rows=21664 dup_rows_remote=18766 dedup_ratio=0.330566 router_tokens_per_expert=[239, 265, 265, 276, 249, 261, 234, 273, 281, 263, 237, 249, 247, 255, 264, 234, 256, 274, 253, 259, 259, 265, 268, 281, 278, 248, 255, 268, 253, 278, 248, 241, 260, 238, 248, 265, 257, 266, 255, 231, 233, 243, 234, 260, 262, 260, 267, 271, 248, 235, 236, 207, 253, 282, 256, 260, 225, 220, 276, 256, 247, 259, 255, 294, 241, 227, 242, 241, 260, 256, 283, 248, 256, 246, 230, 270, 275, 275, 241, 269, 268, 261, 246, 275, 268, 269, 260, 272, 257, 264, 243, 261, 240, 263, 245, 261, 267, 275, 245, 272, 281, 262, 277, 280, 280, 268, 272, 270, 255, 252, 240, 253, 279, 243, 253, 248, 246, 259, 242, 228, 225, 258, 252, 252, 258, 238, 230, 260] payload_rows_per_dst=[5484, 5524, 5288, 5578, 5464, 5550, 5600, 5384] dup_rows_per_dst=[2700, 2844, 2426, 2826, 2656, 2756, 2898, 2558] combine_rows_per_src=[5484, 5524, 5288, 5578, 5464, 5550, 5600, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38264 dup_rows=21748 dup_rows_remote=19062 dedup_ratio=0.331848 router_tokens_per_expert=[298, 273, 250, 250, 262, 248, 267, 262, 263, 257, 252, 255, 238, 272, 236, 263, 248, 261, 255, 219, 260, 244, 249, 283, 263, 240, 259, 274, 268, 252, 230, 278, 251, 280, 257, 291, 228, 227, 257, 264, 253, 227, 266, 273, 266, 254, 250, 261, 253, 238, 272, 261, 266, 253, 272, 232, 257, 249, 272, 253, 271, 265, 245, 242, 261, 261, 250, 248, 231, 233, 260, 234, 235, 229, 267, 283, 250, 269, 234, 270, 232, 268, 247, 254, 256, 252, 264, 250, 248, 244, 236, 268, 268, 243, 286, 255, 256, 258, 251, 260, 234, 249, 264, 262, 266, 268, 279, 268, 265, 280, 249, 263, 269, 271, 266, 235, 273, 237, 274, 213, 225, 247, 278, 252, 264, 257, 252, 262] payload_rows_per_dst=[5474, 5410, 5524, 5544, 5392, 5390, 5592, 5462] dup_rows_per_dst=[2818, 2756, 2686, 2658, 2638, 2752, 2752, 2688] combine_rows_per_src=[5474, 5410, 5524, 5544, 5392, 5390, 5592, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38294 dup_rows=21740 dup_rows_remote=19004 dedup_ratio=0.331726 router_tokens_per_expert=[274, 288, 248, 266, 236, 250, 254, 262, 244, 257, 272, 260, 250, 274, 249, 228, 232, 285, 248, 246, 271, 240, 253, 235, 248, 218, 264, 264, 229, 258, 226, 261, 265, 257, 259, 250, 271, 266, 246, 281, 236, 275, 245, 294, 249, 236, 231, 257, 251, 261, 258, 264, 256, 255, 255, 228, 267, 266, 254, 257, 241, 262, 246, 245, 234, 259, 264, 276, 293, 284, 273, 269, 257, 235, 268, 264, 255, 250, 288, 258, 267, 250, 271, 230, 296, 264, 261, 250, 250, 262, 253, 246, 246, 270, 251, 235, 245, 293, 276, 248, 241, 255, 224, 283, 249, 288, 268, 241, 237, 252, 271, 245, 257, 255, 246, 231, 240, 241, 261, 261, 275, 267, 260, 268, 240, 239, 273, 235] payload_rows_per_dst=[5502, 5440, 5444, 5442, 5620, 5530, 5506, 5312] dup_rows_per_dst=[2736, 2672, 2668, 2730, 3014, 2674, 2686, 2560] combine_rows_per_src=[5502, 5440, 5444, 5442, 5620, 5530, 5506, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38356 dup_rows=21674 dup_rows_remote=18934 dedup_ratio=0.330719 router_tokens_per_expert=[240, 264, 269, 273, 245, 254, 251, 272, 268, 254, 260, 258, 245, 238, 229, 217, 273, 252, 240, 241, 258, 264, 255, 269, 249, 254, 264, 268, 252, 275, 243, 266, 272, 282, 253, 241, 266, 263, 289, 257, 252, 242, 259, 253, 262, 235, 277, 261, 268, 223, 255, 252, 275, 251, 228, 234, 250, 253, 252, 247, 262, 235, 259, 282, 258, 260, 246, 241, 250, 247, 267, 263, 244, 261, 273, 247, 289, 248, 261, 222, 257, 257, 242, 254, 246, 251, 250, 271, 251, 268, 262, 239, 243, 266, 288, 267, 277, 249, 274, 265, 259, 252, 241, 260, 270, 224, 268, 256, 258, 240, 274, 237, 262, 242, 280, 256, 251, 233, 270, 227, 275, 283, 245, 250, 251, 262, 274, 264] payload_rows_per_dst=[5462, 5506, 5540, 5430, 5418, 5488, 5516, 5502] dup_rows_per_dst=[2612, 2740, 2788, 2622, 2736, 2736, 2692, 2748] combine_rows_per_src=[5462, 5506, 5540, 5430, 5418, 5488, 5516, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38492 dup_rows=21620 dup_rows_remote=19024 dedup_ratio=0.329895 router_tokens_per_expert=[250, 262, 272, 239, 263, 282, 261, 251, 237, 272, 248, 245, 261, 243, 256, 250, 274, 265, 238, 255, 270, 267, 240, 261, 260, 287, 266, 248, 265, 258, 254, 274, 245, 267, 256, 242, 268, 288, 266, 260, 242, 243, 251, 229, 219, 256, 266, 279, 241, 268, 250, 228, 256, 234, 257, 227, 274, 234, 263, 239, 265, 269, 255, 242, 244, 260, 251, 238, 260, 250, 245, 262, 250, 257, 263, 267, 258, 255, 249, 259, 229, 240, 257, 246, 274, 257, 261, 236, 267, 243, 276, 221, 247, 238, 246, 272, 252, 272, 254, 264, 264, 241, 259, 258, 285, 251, 279, 290, 252, 255, 256, 259, 254, 267, 238, 274, 275, 238, 255, 261, 271, 247, 263, 246, 271, 267, 247, 272] payload_rows_per_dst=[5506, 5578, 5478, 5438, 5448, 5424, 5536, 5508] dup_rows_per_dst=[2678, 2786, 2676, 2566, 2688, 2596, 2846, 2784] combine_rows_per_src=[5506, 5578, 5478, 5438, 5448, 5424, 5536, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38216 dup_rows=21850 dup_rows_remote=19098 dedup_ratio=0.333405 router_tokens_per_expert=[262, 287, 256, 240, 275, 256, 237, 294, 273, 266, 240, 256, 227, 235, 276, 232, 240, 258, 263, 265, 226, 241, 265, 289, 247, 242, 258, 240, 238, 229, 274, 272, 259, 273, 267, 241, 228, 240, 257, 242, 229, 251, 242, 260, 263, 250, 269, 235, 279, 261, 242, 266, 250, 248, 248, 265, 259, 261, 240, 239, 283, 267, 243, 260, 255, 279, 272, 245, 262, 272, 244, 261, 265, 273, 274, 249, 273, 251, 265, 260, 242, 237, 262, 247, 274, 240, 254, 250, 241, 293, 270, 280, 269, 249, 279, 267, 248, 228, 268, 263, 251, 280, 262, 216, 260, 255, 272, 259, 265, 264, 269, 251, 246, 244, 259, 254, 278, 256, 218, 238, 273, 235, 246, 236, 226, 281, 261, 276] payload_rows_per_dst=[5388, 5470, 5362, 5470, 5568, 5534, 5468, 5426] dup_rows_per_dst=[2836, 2624, 2650, 2752, 2832, 2774, 2754, 2628] combine_rows_per_src=[5388, 5470, 5362, 5470, 5568, 5534, 5468, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38180 dup_rows=21846 dup_rows_remote=18982 dedup_ratio=0.333344 router_tokens_per_expert=[262, 263, 268, 254, 254, 269, 266, 268, 287, 250, 248, 254, 230, 290, 258, 281, 278, 265, 233, 277, 249, 239, 239, 257, 241, 229, 228, 272, 259, 273, 242, 257, 265, 248, 247, 259, 265, 266, 258, 247, 241, 260, 232, 251, 239, 254, 247, 250, 253, 243, 286, 239, 249, 239, 257, 240, 261, 267, 267, 286, 242, 267, 242, 283, 254, 262, 259, 264, 249, 251, 279, 273, 251, 246, 245, 275, 263, 270, 289, 257, 259, 270, 242, 243, 241, 276, 241, 242, 274, 266, 254, 257, 254, 279, 253, 260, 252, 286, 240, 238, 269, 269, 246, 254, 255, 232, 251, 266, 280, 261, 236, 281, 257, 238, 243, 271, 226, 263, 250, 242, 247, 244, 244, 235, 225, 258, 250, 271] payload_rows_per_dst=[5498, 5426, 5410, 5516, 5510, 5456, 5488, 5386] dup_rows_per_dst=[2906, 2650, 2648, 2726, 2864, 2766, 2744, 2542] combine_rows_per_src=[5498, 5426, 5410, 5516, 5510, 5456, 5488, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228152 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156990264 ingress_bytes=232259584 link_bytes=232259584 token_bytes=313752376 weight_movement_bytes=75497472 total_nvlink_bytes=389249848 weight_to_token_ratio=0.240628 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225397055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229304 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958648 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687992 weight_movement_bytes=18874368 total_nvlink_bytes=332562360 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229192 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081416 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933640 weight_movement_bytes=18874368 total_nvlink_bytes=332808008 weight_to_token_ratio=0.060122 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229160 combine_payload_bytes=157106176 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157335336 ingress_bytes=213729280 link_bytes=213729280 token_bytes=314441512 weight_movement_bytes=56623104 total_nvlink_bytes=371064616 weight_to_token_ratio=0.180075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047848800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157663232 dispatch_meta_bytes=230064 combine_payload_bytes=157663232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157893296 ingress_bytes=176537600 link_bytes=176537600 token_bytes=315556528 weight_movement_bytes=18874368 total_nvlink_bytes=334430896 weight_to_token_ratio=0.059813 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691496421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228648 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156613928 ingress_bytes=194134016 link_bytes=194134016 token_bytes=312999208 weight_movement_bytes=37748736 total_nvlink_bytes=350747944 weight_to_token_ratio=0.120603 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229256 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156761992 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294728 weight_movement_bytes=37748736 total_nvlink_bytes=351043464 weight_to_token_ratio=0.120490 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38014 dup_rows=21736 dup_rows_remote=18536 dedup_ratio=0.331665 router_tokens_per_expert=[263, 267, 242, 205, 248, 261, 275, 257, 252, 225, 256, 250, 275, 238, 242, 274, 268, 250, 250, 230, 270, 243, 235, 259, 231, 244, 230, 245, 253, 276, 251, 248, 258, 259, 250, 296, 271, 276, 258, 269, 259, 228, 265, 228, 226, 244, 266, 230, 250, 242, 247, 248, 244, 252, 264, 261, 271, 243, 279, 256, 274, 247, 271, 230, 251, 267, 271, 286, 248, 254, 248, 250, 238, 273, 249, 255, 251, 288, 225, 253, 253, 262, 253, 262, 242, 258, 255, 270, 268, 237, 268, 265, 243, 251, 254, 244, 261, 274, 260, 257, 260, 274, 302, 252, 261, 256, 239, 256, 270, 240, 261, 240, 263, 251, 268, 275, 274, 281, 269, 274, 257, 272, 275, 271, 247, 259, 254, 248] payload_rows_per_dst=[5434, 5456, 5456, 5444, 5246, 5456, 5522, 5786] dup_rows_per_dst=[2620, 2510, 2710, 2714, 2458, 2714, 2810, 3200] combine_rows_per_src=[5434, 5456, 5456, 5444, 5246, 5456, 5522, 5786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=226200 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155931544 ingress_bytes=174579712 link_bytes=174579712 token_bytes=311636888 weight_movement_bytes=18874368 total_nvlink_bytes=330511256 weight_to_token_ratio=0.060565 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672736902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43934 payload_rows_remote=38490 dup_rows=21602 dup_rows_remote=18800 dedup_ratio=0.329620 router_tokens_per_expert=[245, 236, 241, 226, 248, 259, 253, 263, 284, 251, 242, 263, 290, 248, 281, 230, 249, 234, 259, 264, 252, 260, 250, 260, 271, 254, 236, 250, 277, 246, 271, 252, 240, 247, 254, 268, 255, 261, 269, 244, 259, 216, 288, 236, 264, 270, 266, 267, 267, 275, 259, 255, 277, 280, 241, 252, 255, 275, 267, 263, 236, 267, 258, 258, 244, 239, 283, 268, 256, 240, 267, 276, 238, 257, 247, 259, 236, 258, 278, 233, 257, 264, 233, 235, 263, 247, 228, 229, 271, 242, 294, 251, 254, 259, 242, 245, 245, 273, 268, 253, 264, 259, 265, 249, 267, 252, 264, 258, 263, 252, 260, 231, 249, 254, 259, 256, 248, 270, 268, 248, 287, 275, 240, 254, 256, 246, 261, 247] payload_rows_per_dst=[5468, 5442, 5540, 5622, 5488, 5414, 5444, 5516] dup_rows_per_dst=[2652, 2728, 2668, 2748, 2596, 2614, 2802, 2794] combine_rows_per_src=[5468, 5442, 5540, 5622, 5488, 5414, 5444, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38224 dup_rows=21848 dup_rows_remote=19116 dedup_ratio=0.333374 router_tokens_per_expert=[250, 234, 257, 263, 252, 268, 252, 258, 256, 247, 269, 269, 255, 256, 241, 254, 257, 254, 237, 271, 256, 252, 265, 261, 249, 269, 278, 236, 255, 250, 253, 266, 286, 248, 265, 260, 264, 245, 280, 238, 257, 272, 268, 249, 238, 234, 247, 247, 251, 262, 258, 230, 241, 267, 257, 261, 254, 275, 279, 254, 271, 268, 240, 267, 246, 259, 256, 247, 251, 259, 264, 267, 253, 282, 238, 262, 243, 277, 263, 263, 252, 249, 259, 262, 231, 237, 250, 272, 257, 257, 238, 281, 268, 259, 269, 249, 282, 233, 292, 257, 261, 255, 233, 258, 263, 234, 255, 267, 241, 260, 259, 245, 229, 275, 237, 261, 240, 239, 229, 258, 249, 265, 266, 234, 274, 283, 238, 253] payload_rows_per_dst=[5414, 5482, 5464, 5488, 5488, 5486, 5410, 5456] dup_rows_per_dst=[2748, 2736, 2732, 2782, 2772, 2694, 2780, 2604] combine_rows_per_src=[5414, 5482, 5464, 5488, 5488, 5486, 5410, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38348 dup_rows=21736 dup_rows_remote=18930 dedup_ratio=0.331665 router_tokens_per_expert=[252, 250, 261, 258, 253, 254, 261, 259, 245, 266, 270, 272, 263, 244, 266, 224, 249, 271, 257, 274, 233, 261, 253, 264, 273, 227, 242, 240, 239, 266, 254, 261, 267, 251, 239, 277, 266, 260, 251, 254, 232, 261, 251, 246, 256, 256, 270, 233, 259, 270, 265, 245, 245, 270, 261, 274, 279, 235, 265, 273, 233, 255, 253, 237, 256, 271, 249, 231, 258, 247, 267, 254, 253, 273, 261, 249, 261, 264, 259, 291, 269, 268, 258, 253, 228, 234, 231, 251, 235, 289, 271, 251, 283, 267, 243, 265, 251, 278, 238, 246, 252, 240, 250, 252, 259, 248, 249, 273, 272, 248, 253, 236, 264, 258, 267, 242, 263, 286, 250, 249, 280, 237, 246, 237, 250, 300, 244, 259] payload_rows_per_dst=[5452, 5594, 5354, 5536, 5508, 5400, 5400, 5556] dup_rows_per_dst=[2806, 2796, 2534, 2866, 2780, 2556, 2690, 2708] combine_rows_per_src=[5452, 5594, 5354, 5536, 5508, 5400, 5400, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38256 dup_rows=21742 dup_rows_remote=18886 dedup_ratio=0.331757 router_tokens_per_expert=[262, 245, 270, 273, 250, 240, 234, 270, 254, 256, 265, 240, 260, 254, 227, 263, 259, 257, 276, 269, 263, 249, 259, 255, 266, 247, 238, 240, 241, 263, 261, 238, 244, 262, 262, 274, 255, 290, 225, 244, 244, 258, 300, 254, 264, 241, 269, 248, 264, 260, 268, 278, 266, 258, 244, 257, 242, 245, 261, 278, 258, 282, 256, 241, 233, 261, 237, 275, 266, 271, 260, 227, 258, 258, 269, 288, 258, 265, 265, 243, 216, 275, 252, 246, 239, 273, 265, 262, 233, 259, 273, 255, 251, 257, 231, 286, 262, 262, 267, 271, 252, 241, 283, 285, 215, 248, 265, 247, 275, 238, 255, 271, 261, 238, 256, 254, 218, 259, 224, 272, 263, 236, 261, 261, 252, 247, 254, 232] payload_rows_per_dst=[5498, 5538, 5540, 5434, 5492, 5424, 5518, 5350] dup_rows_per_dst=[2628, 2856, 2728, 2650, 2776, 2722, 2756, 2626] combine_rows_per_src=[5498, 5538, 5540, 5434, 5492, 5424, 5518, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38270 dup_rows=21762 dup_rows_remote=19124 dedup_ratio=0.332062 router_tokens_per_expert=[271, 254, 258, 254, 264, 268, 259, 252, 290, 230, 232, 252, 268, 252, 261, 252, 250, 239, 239, 236, 248, 250, 263, 247, 255, 249, 231, 226, 261, 279, 255, 240, 270, 263, 263, 248, 249, 268, 270, 239, 242, 247, 254, 288, 275, 233, 264, 249, 249, 253, 248, 246, 237, 279, 268, 246, 250, 285, 239, 243, 244, 260, 270, 266, 250, 232, 237, 271, 253, 268, 278, 264, 276, 259, 250, 245, 274, 247, 264, 259, 254, 254, 258, 243, 262, 269, 238, 277, 262, 245, 224, 265, 258, 247, 268, 247, 270, 253, 259, 260, 265, 282, 299, 246, 265, 252, 277, 252, 295, 275, 251, 261, 244, 253, 270, 230, 246, 236, 252, 254, 228, 254, 241, 263, 267, 294, 238, 248] payload_rows_per_dst=[5440, 5338, 5452, 5474, 5502, 5504, 5680, 5384] dup_rows_per_dst=[2794, 2598, 2792, 2692, 2752, 2638, 2844, 2652] combine_rows_per_src=[5440, 5338, 5452, 5474, 5502, 5504, 5680, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44026 payload_rows_remote=38558 dup_rows=21510 dup_rows_remote=18822 dedup_ratio=0.328217 router_tokens_per_expert=[229, 250, 269, 245, 251, 263, 244, 276, 269, 228, 239, 251, 276, 257, 247, 280, 244, 247, 259, 252, 271, 266, 238, 247, 258, 256, 239, 248, 258, 246, 257, 268, 260, 259, 233, 276, 237, 245, 274, 210, 262, 237, 253, 221, 277, 251, 233, 265, 243, 268, 233, 272, 254, 252, 273, 248, 243, 241, 261, 263, 273, 245, 236, 273, 249, 285, 261, 229, 260, 272, 253, 248, 249, 216, 290, 277, 274, 256, 268, 242, 248, 274, 270, 270, 275, 258, 244, 263, 277, 250, 263, 229, 241, 244, 250, 279, 249, 265, 246, 268, 235, 266, 264, 263, 261, 265, 254, 254, 262, 263, 274, 284, 261, 264, 222, 297, 264, 254, 269, 265, 240, 251, 253, 257, 260, 274, 255, 246] payload_rows_per_dst=[5494, 5464, 5424, 5468, 5608, 5490, 5554, 5524] dup_rows_per_dst=[2654, 2644, 2562, 2688, 2650, 2780, 2792, 2740] combine_rows_per_src=[5494, 5464, 5424, 5468, 5608, 5490, 5554, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38260 dup_rows=21942 dup_rows_remote=19194 dedup_ratio=0.334808 router_tokens_per_expert=[256, 252, 231, 245, 252, 259, 255, 253, 239, 256, 270, 268, 234, 266, 234, 242, 223, 248, 270, 239, 263, 251, 271, 264, 270, 274, 255, 269, 272, 275, 268, 281, 270, 269, 249, 274, 257, 258, 286, 252, 305, 267, 266, 253, 258, 258, 264, 235, 251, 257, 263, 257, 269, 255, 233, 244, 232, 251, 221, 245, 236, 259, 226, 267, 219, 242, 244, 262, 260, 253, 248, 245, 268, 278, 222, 265, 256, 251, 262, 266, 247, 254, 261, 251, 252, 255, 270, 261, 247, 254, 263, 229, 248, 279, 237, 234, 270, 233, 262, 258, 264, 288, 260, 266, 238, 260, 250, 302, 253, 264, 291, 268, 235, 253, 245, 243, 273, 267, 232, 258, 276, 254, 240, 274, 275, 249, 245, 247] payload_rows_per_dst=[5442, 5592, 5504, 5346, 5334, 5456, 5470, 5450] dup_rows_per_dst=[2582, 2794, 2938, 2586, 2748, 2628, 2984, 2682] combine_rows_per_src=[5442, 5592, 5504, 5346, 5334, 5456, 5470, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157655040 dispatch_meta_bytes=229160 combine_payload_bytes=157655040 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=157884200 ingress_bytes=252026880 link_bytes=252026880 token_bytes=315539240 weight_movement_bytes=94371840 total_nvlink_bytes=409911080 weight_to_token_ratio=0.299081 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.414797559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229360 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156794864 ingress_bytes=213188608 link_bytes=213188608 token_bytes=313360368 weight_movement_bytes=56623104 total_nvlink_bytes=369983472 weight_to_token_ratio=0.180696 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042668347e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229112 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157302520 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314375928 weight_movement_bytes=37748736 total_nvlink_bytes=352124664 weight_to_token_ratio=0.120075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=37982 dup_rows=21868 dup_rows_remote=18852 dedup_ratio=0.333679 router_tokens_per_expert=[250, 254, 262, 277, 244, 240, 257, 257, 242, 274, 271, 224, 243, 234, 265, 247, 283, 269, 254, 257, 270, 273, 250, 243, 276, 249, 255, 265, 274, 263, 243, 241, 262, 252, 274, 260, 245, 266, 243, 255, 260, 276, 269, 271, 268, 268, 263, 245, 258, 276, 249, 252, 228, 243, 232, 277, 240, 259, 258, 242, 242, 256, 268, 262, 283, 262, 240, 258, 240, 264, 255, 239, 237, 282, 237, 266, 263, 290, 257, 280, 259, 269, 256, 234, 247, 268, 245, 248, 247, 242, 276, 250, 242, 275, 243, 257, 241, 230, 242, 269, 256, 231, 259, 246, 245, 247, 264, 255, 276, 265, 272, 259, 237, 249, 245, 244, 253, 273, 234, 269, 258, 238, 257, 286, 251, 285, 265, 231] payload_rows_per_dst=[5436, 5288, 5510, 5390, 5464, 5458, 5436, 5686] dup_rows_per_dst=[2646, 2490, 2810, 2694, 2876, 2652, 2684, 3016] combine_rows_per_src=[5436, 5288, 5510, 5390, 5464, 5458, 5436, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228568 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925144 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313621720 weight_movement_bytes=18874368 total_nvlink_bytes=332496088 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229576 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983496 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737416 weight_movement_bytes=18874368 total_nvlink_bytes=332611784 weight_to_token_ratio=0.060160 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157933568 dispatch_meta_bytes=229520 combine_payload_bytes=157933568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158163088 ingress_bytes=176807936 link_bytes=176807936 token_bytes=316096656 weight_movement_bytes=18874368 total_nvlink_bytes=334971024 weight_to_token_ratio=0.059711 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694086648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229816 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942776 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655736 weight_movement_bytes=18874368 total_nvlink_bytes=332530104 weight_to_token_ratio=0.060175 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155574272 dispatch_meta_bytes=227336 combine_payload_bytes=155574272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155801608 ingress_bytes=174448640 link_bytes=174448640 token_bytes=311375880 weight_movement_bytes=18874368 total_nvlink_bytes=330250248 weight_to_token_ratio=0.060616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671481034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38262 dup_rows=21898 dup_rows_remote=19178 dedup_ratio=0.334137 router_tokens_per_expert=[246, 225, 228, 254, 245, 228, 256, 284, 315, 263, 259, 252, 286, 244, 288, 245, 221, 248, 285, 265, 246, 281, 232, 259, 240, 250, 262, 250, 239, 256, 257, 228, 246, 290, 273, 272, 246, 261, 250, 273, 257, 243, 245, 246, 281, 272, 276, 244, 267, 257, 266, 256, 245, 236, 244, 266, 256, 282, 221, 258, 267, 260, 281, 235, 288, 257, 250, 265, 256, 288, 265, 254, 242, 269, 252, 242, 245, 253, 232, 263, 264, 272, 232, 275, 254, 273, 253, 254, 241, 271, 270, 248, 282, 233, 247, 263, 265, 265, 207, 250, 247, 271, 246, 271, 257, 229, 254, 262, 269, 245, 248, 262, 248, 256, 258, 236, 254, 243, 246, 251, 242, 256, 273, 255, 269, 263, 250, 258] payload_rows_per_dst=[5486, 5296, 5534, 5446, 5492, 5482, 5376, 5526] dup_rows_per_dst=[2750, 2552, 2816, 2748, 2750, 2782, 2720, 2780] combine_rows_per_src=[5486, 5296, 5534, 5446, 5492, 5482, 5376, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38042 dup_rows=21970 dup_rows_remote=19346 dedup_ratio=0.335236 router_tokens_per_expert=[251, 223, 244, 244, 246, 233, 262, 254, 275, 262, 233, 235, 247, 290, 259, 235, 250, 250, 252, 271, 251, 273, 252, 277, 235, 261, 270, 274, 246, 240, 254, 267, 261, 250, 237, 285, 231, 250, 251, 241, 271, 261, 264, 236, 270, 247, 281, 238, 262, 272, 268, 287, 267, 254, 255, 248, 291, 270, 245, 261, 234, 246, 251, 242, 256, 254, 253, 247, 276, 270, 235, 250, 251, 249, 259, 257, 238, 259, 256, 271, 258, 243, 271, 250, 260, 243, 270, 271, 255, 251, 261, 283, 243, 249, 274, 254, 278, 234, 278, 265, 266, 239, 256, 241, 241, 296, 266, 268, 267, 275, 247, 257, 262, 267, 252, 261, 245, 250, 245, 248, 247, 270, 243, 252, 237, 278, 229, 248] payload_rows_per_dst=[5366, 5420, 5524, 5524, 5400, 5496, 5456, 5380] dup_rows_per_dst=[2620, 2826, 2624, 2782, 2762, 2776, 2892, 2688] combine_rows_per_src=[5366, 5420, 5524, 5524, 5400, 5496, 5456, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38218 dup_rows=21796 dup_rows_remote=18698 dedup_ratio=0.332581 router_tokens_per_expert=[259, 261, 265, 240, 254, 223, 243, 262, 279, 241, 262, 261, 271, 260, 262, 247, 253, 248, 242, 240, 277, 249, 245, 232, 235, 215, 247, 274, 238, 238, 287, 245, 256, 262, 250, 281, 239, 229, 259, 256, 265, 267, 248, 286, 248, 241, 260, 268, 261, 244, 262, 241, 298, 265, 251, 242, 277, 263, 259, 266, 247, 279, 262, 281, 250, 272, 247, 255, 239, 260, 242, 253, 268, 263, 247, 270, 272, 273, 249, 259, 260, 272, 248, 259, 285, 264, 265, 239, 252, 256, 237, 281, 246, 256, 261, 229, 251, 243, 260, 244, 244, 261, 251, 260, 249, 258, 272, 252, 238, 258, 244, 240, 269, 263, 248, 288, 264, 271, 255, 278, 271, 230, 242, 272, 246, 239, 259, 251] payload_rows_per_dst=[5522, 5544, 5456, 5382, 5538, 5444, 5426, 5428] dup_rows_per_dst=[3098, 2690, 2818, 2574, 2736, 2472, 2580, 2828] combine_rows_per_src=[5522, 5544, 5456, 5382, 5538, 5444, 5426, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38240 dup_rows=21768 dup_rows_remote=18976 dedup_ratio=0.332153 router_tokens_per_expert=[265, 269, 274, 236, 267, 253, 301, 274, 249, 246, 240, 257, 243, 258, 277, 262, 266, 264, 229, 259, 254, 250, 259, 246, 276, 290, 244, 277, 270, 258, 275, 243, 264, 237, 254, 275, 242, 249, 261, 248, 263, 231, 241, 239, 267, 276, 263, 264, 261, 255, 260, 243, 266, 230, 290, 268, 273, 274, 240, 228, 258, 276, 265, 281, 262, 237, 255, 248, 255, 266, 270, 227, 243, 253, 278, 243, 238, 238, 222, 261, 281, 248, 251, 218, 221, 224, 265, 276, 255, 279, 245, 243, 266, 265, 250, 277, 239, 260, 273, 250, 220, 249, 305, 216, 232, 259, 254, 231, 250, 267, 260, 260, 244, 226, 289, 245, 256, 252, 254, 272, 278, 277, 267, 252, 247, 248, 251, 252] payload_rows_per_dst=[5504, 5528, 5452, 5536, 5404, 5512, 5406, 5426] dup_rows_per_dst=[2838, 2792, 2696, 2800, 2588, 2616, 2644, 2794] combine_rows_per_src=[5504, 5528, 5452, 5536, 5404, 5512, 5406, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38308 dup_rows=21780 dup_rows_remote=19060 dedup_ratio=0.332336 router_tokens_per_expert=[251, 239, 257, 263, 257, 256, 237, 275, 239, 248, 222, 249, 265, 258, 260, 257, 262, 249, 261, 253, 249, 268, 275, 256, 306, 260, 279, 246, 253, 256, 257, 278, 233, 245, 240, 248, 268, 277, 265, 269, 265, 267, 262, 277, 266, 260, 256, 276, 276, 256, 286, 261, 263, 263, 257, 249, 257, 232, 264, 204, 265, 221, 270, 246, 243, 261, 238, 255, 241, 251, 242, 253, 239, 261, 246, 286, 239, 259, 246, 258, 287, 244, 271, 275, 268, 241, 258, 250, 237, 241, 243, 249, 233, 268, 256, 263, 256, 288, 262, 275, 272, 266, 242, 256, 279, 282, 247, 252, 276, 234, 272, 252, 238, 229, 288, 236, 268, 279, 246, 258, 252, 272, 241, 233, 232, 245, 233, 220] payload_rows_per_dst=[5406, 5626, 5526, 5408, 5422, 5448, 5550, 5370] dup_rows_per_dst=[2660, 2790, 2822, 2732, 2614, 2720, 2872, 2570] combine_rows_per_src=[5406, 5626, 5526, 5408, 5422, 5448, 5550, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229760 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156950912 ingress_bytes=213344256 link_bytes=213344256 token_bytes=313672064 weight_movement_bytes=56623104 total_nvlink_bytes=370295168 weight_to_token_ratio=0.180517 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044159689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38160 dup_rows=21998 dup_rows_remote=19328 dedup_ratio=0.335663 router_tokens_per_expert=[261, 225, 272, 267, 253, 251, 245, 276, 270, 270, 264, 268, 257, 249, 247, 257, 293, 253, 234, 258, 252, 258, 270, 257, 255, 252, 279, 234, 262, 263, 245, 279, 244, 287, 220, 262, 246, 251, 288, 273, 268, 248, 248, 236, 250, 262, 254, 235, 213, 269, 266, 272, 287, 278, 235, 256, 238, 236, 259, 238, 274, 249, 213, 241, 262, 255, 235, 248, 258, 264, 252, 273, 254, 252, 245, 249, 257, 246, 260, 257, 254, 251, 264, 244, 258, 251, 296, 275, 244, 253, 253, 255, 242, 256, 267, 238, 272, 247, 259, 264, 247, 246, 243, 251, 248, 258, 235, 253, 270, 274, 240, 248, 298, 249, 248, 280, 275, 241, 262, 263, 264, 251, 232, 292, 257, 268, 252, 241] payload_rows_per_dst=[5548, 5494, 5442, 5378, 5390, 5456, 5354, 5476] dup_rows_per_dst=[2716, 2794, 2702, 2670, 2744, 2746, 2756, 2870] combine_rows_per_src=[5548, 5494, 5442, 5378, 5390, 5456, 5354, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38248 dup_rows=21760 dup_rows_remote=18932 dedup_ratio=0.332031 router_tokens_per_expert=[246, 264, 244, 224, 268, 288, 260, 256, 264, 271, 222, 232, 233, 255, 266, 253, 253, 243, 244, 239, 278, 270, 276, 257, 253, 274, 266, 252, 250, 270, 240, 251, 267, 253, 228, 246, 269, 259, 248, 253, 244, 246, 250, 224, 258, 282, 266, 269, 264, 238, 269, 270, 264, 243, 302, 271, 265, 274, 262, 232, 276, 239, 245, 236, 255, 270, 239, 244, 257, 278, 269, 292, 261, 258, 239, 267, 258, 247, 255, 289, 249, 256, 251, 237, 248, 238, 231, 256, 242, 263, 212, 257, 250, 248, 262, 265, 248, 270, 241, 261, 252, 265, 259, 263, 282, 244, 247, 307, 258, 242, 259, 254, 236, 243, 244, 251, 272, 245, 283, 247, 243, 265, 268, 239, 258, 247, 279, 279] payload_rows_per_dst=[5402, 5500, 5472, 5552, 5528, 5326, 5526, 5470] dup_rows_per_dst=[2690, 2732, 2652, 2748, 2828, 2604, 2778, 2728] combine_rows_per_src=[5402, 5500, 5472, 5552, 5528, 5326, 5526, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=229552 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156049584 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311869616 weight_movement_bytes=18874368 total_nvlink_bytes=330743984 weight_to_token_ratio=0.060520 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38158 dup_rows=21798 dup_rows_remote=18834 dedup_ratio=0.332611 router_tokens_per_expert=[280, 249, 285, 240, 246, 261, 248, 263, 271, 241, 274, 258, 271, 233, 237, 283, 239, 268, 270, 240, 272, 252, 252, 274, 243, 266, 256, 265, 261, 297, 249, 235, 265, 247, 244, 234, 276, 259, 266, 264, 217, 243, 250, 231, 259, 252, 223, 245, 215, 276, 264, 286, 245, 237, 247, 256, 267, 218, 257, 235, 265, 254, 248, 261, 262, 253, 254, 237, 251, 269, 288, 245, 260, 263, 243, 265, 246, 245, 278, 260, 269, 271, 259, 279, 253, 239, 217, 267, 259, 239, 240, 242, 243, 244, 273, 253, 265, 259, 269, 229, 262, 227, 248, 247, 263, 262, 264, 253, 250, 241, 252, 287, 272, 263, 268, 237, 249, 274, 268, 255, 282, 308, 251, 266, 265, 242, 273, 266] payload_rows_per_dst=[5528, 5296, 5378, 5346, 5518, 5648, 5444, 5580] dup_rows_per_dst=[2718, 2450, 2572, 2550, 2886, 2978, 2680, 2964] combine_rows_per_src=[5528, 5296, 5378, 5346, 5518, 5648, 5444, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=227664 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156768592 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313309520 weight_movement_bytes=18874368 total_nvlink_bytes=332183888 weight_to_token_ratio=0.060242 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228864 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156859904 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313490944 weight_movement_bytes=37748736 total_nvlink_bytes=351239680 weight_to_token_ratio=0.120414 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229472 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157139040 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048608 weight_movement_bytes=37748736 total_nvlink_bytes=351797344 weight_to_token_ratio=0.120200 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229952 combine_payload_bytes=156303360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156533312 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312836672 weight_movement_bytes=37748736 total_nvlink_bytes=350585408 weight_to_token_ratio=0.120666 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228720 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156892528 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556336 weight_movement_bytes=37748736 total_nvlink_bytes=351305072 weight_to_token_ratio=0.120389 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=227968 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156523136 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312818304 weight_movement_bytes=37748736 total_nvlink_bytes=350567040 weight_to_token_ratio=0.120673 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38084 dup_rows=21940 dup_rows_remote=19086 dedup_ratio=0.334778 router_tokens_per_expert=[249, 275, 291, 269, 272, 243, 267, 258, 255, 230, 266, 278, 249, 255, 246, 238, 259, 249, 249, 233, 257, 249, 260, 264, 268, 259, 254, 259, 244, 269, 249, 247, 268, 236, 256, 241, 243, 241, 243, 266, 240, 257, 252, 243, 254, 280, 249, 264, 263, 241, 289, 294, 250, 275, 268, 267, 226, 256, 257, 253, 243, 265, 261, 274, 244, 248, 241, 261, 256, 289, 224, 257, 247, 226, 264, 282, 242, 270, 270, 255, 250, 266, 243, 246, 248, 252, 285, 270, 252, 248, 247, 244, 280, 258, 253, 256, 270, 286, 254, 252, 292, 256, 247, 264, 253, 261, 241, 258, 253, 268, 251, 277, 247, 253, 235, 248, 242, 273, 241, 229, 268, 236, 270, 240, 245, 257, 258, 244] payload_rows_per_dst=[5418, 5434, 5414, 5572, 5462, 5430, 5512, 5354] dup_rows_per_dst=[2864, 2642, 2652, 2792, 2690, 2828, 2854, 2618] combine_rows_per_src=[5418, 5434, 5414, 5572, 5462, 5430, 5512, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38270 dup_rows=21910 dup_rows_remote=19138 dedup_ratio=0.334320 router_tokens_per_expert=[266, 261, 279, 253, 269, 277, 260, 244, 251, 254, 250, 251, 238, 233, 264, 225, 265, 263, 282, 266, 236, 245, 288, 284, 285, 251, 283, 262, 245, 239, 277, 269, 237, 274, 273, 246, 247, 259, 242, 250, 258, 237, 248, 267, 249, 261, 254, 262, 244, 238, 252, 253, 250, 263, 242, 298, 238, 250, 260, 252, 268, 268, 234, 291, 235, 245, 256, 267, 234, 239, 256, 276, 271, 242, 260, 242, 246, 256, 252, 247, 240, 238, 264, 264, 264, 268, 259, 270, 274, 271, 267, 259, 250, 278, 271, 235, 231, 276, 224, 262, 236, 266, 266, 237, 238, 249, 229, 277, 269, 232, 274, 286, 276, 240, 253, 232, 228, 270, 253, 267, 239, 246, 256, 265, 237, 277, 258, 243] payload_rows_per_dst=[5504, 5586, 5356, 5414, 5406, 5566, 5440, 5354] dup_rows_per_dst=[2646, 2894, 2772, 2788, 2642, 2778, 2664, 2726] combine_rows_per_src=[5504, 5586, 5356, 5414, 5406, 5566, 5440, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38128 dup_rows=21928 dup_rows_remote=19108 dedup_ratio=0.334595 router_tokens_per_expert=[287, 259, 261, 249, 243, 235, 260, 259, 244, 235, 256, 235, 244, 256, 261, 241, 268, 260, 287, 253, 262, 251, 228, 237, 259, 249, 254, 211, 245, 260, 250, 239, 240, 248, 238, 263, 284, 250, 276, 239, 262, 240, 269, 262, 228, 290, 255, 263, 273, 302, 257, 244, 257, 260, 259, 263, 261, 235, 241, 242, 243, 262, 263, 271, 269, 259, 246, 244, 259, 256, 279, 227, 254, 274, 250, 251, 255, 254, 259, 246, 247, 259, 254, 268, 253, 247, 271, 260, 259, 257, 271, 258, 292, 270, 271, 245, 241, 256, 239, 242, 255, 234, 267, 267, 237, 254, 251, 254, 259, 265, 268, 280, 238, 262, 268, 258, 266, 287, 270, 264, 258, 267, 251, 255, 265, 240, 250, 258] payload_rows_per_dst=[5480, 5530, 5302, 5554, 5504, 5626, 5450, 5162] dup_rows_per_dst=[2820, 2860, 2550, 2710, 2804, 3114, 2688, 2382] combine_rows_per_src=[5480, 5530, 5302, 5554, 5504, 5626, 5450, 5162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38396 dup_rows=21626 dup_rows_remote=18862 dedup_ratio=0.329987 router_tokens_per_expert=[252, 269, 249, 263, 280, 242, 233, 285, 233, 239, 259, 242, 280, 245, 252, 260, 234, 258, 272, 254, 262, 271, 254, 263, 262, 280, 246, 269, 272, 242, 240, 260, 260, 264, 257, 256, 254, 237, 252, 228, 250, 246, 245, 243, 271, 237, 285, 241, 241, 257, 237, 264, 294, 237, 263, 281, 246, 280, 245, 249, 245, 272, 233, 252, 265, 243, 239, 256, 282, 262, 247, 256, 284, 255, 243, 266, 258, 264, 244, 253, 263, 262, 274, 231, 229, 269, 241, 250, 227, 257, 257, 270, 236, 261, 266, 257, 245, 241, 252, 269, 251, 260, 227, 286, 284, 275, 243, 234, 262, 258, 252, 254, 231, 241, 256, 252, 274, 281, 264, 236, 288, 274, 259, 268, 246, 273, 257, 264] payload_rows_per_dst=[5474, 5514, 5408, 5448, 5522, 5452, 5526, 5566] dup_rows_per_dst=[2692, 2764, 2644, 2744, 2712, 2648, 2660, 2762] combine_rows_per_src=[5474, 5514, 5408, 5448, 5522, 5452, 5526, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=228680 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156220744 ingress_bytes=193740800 link_bytes=193740800 token_bytes=312212808 weight_movement_bytes=37748736 total_nvlink_bytes=349961544 weight_to_token_ratio=0.120907 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856329019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229632 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983552 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737472 weight_movement_bytes=18874368 total_nvlink_bytes=332611840 weight_to_token_ratio=0.060160 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38422 dup_rows=21726 dup_rows_remote=19096 dedup_ratio=0.331512 router_tokens_per_expert=[271, 253, 295, 271, 265, 243, 268, 282, 250, 269, 263, 271, 275, 266, 262, 253, 235, 262, 263, 266, 277, 233, 239, 228, 243, 258, 253, 257, 231, 261, 277, 228, 256, 273, 250, 251, 234, 264, 251, 256, 260, 254, 273, 251, 251, 273, 294, 239, 232, 256, 223, 247, 272, 230, 235, 245, 247, 249, 271, 246, 263, 272, 277, 244, 263, 235, 225, 238, 232, 264, 257, 257, 264, 241, 266, 272, 255, 263, 290, 265, 244, 281, 280, 251, 252, 249, 255, 254, 254, 272, 264, 282, 237, 260, 243, 255, 239, 238, 255, 275, 275, 266, 254, 240, 258, 254, 259, 281, 245, 257, 263, 265, 252, 248, 255, 230, 266, 260, 267, 235, 226, 239, 257, 263, 243, 266, 249, 261] payload_rows_per_dst=[5620, 5450, 5496, 5388, 5496, 5486, 5462, 5412] dup_rows_per_dst=[2894, 2572, 2764, 2630, 2678, 2780, 2786, 2622] combine_rows_per_src=[5620, 5450, 5496, 5388, 5496, 5486, 5462, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43948 payload_rows_remote=38480 dup_rows=21588 dup_rows_remote=18976 dedup_ratio=0.329407 router_tokens_per_expert=[262, 249, 260, 282, 252, 254, 259, 245, 256, 254, 238, 253, 224, 243, 268, 249, 254, 268, 248, 278, 241, 248, 245, 257, 247, 277, 262, 274, 256, 281, 242, 252, 257, 254, 261, 276, 241, 252, 277, 234, 251, 263, 278, 267, 262, 256, 276, 265, 260, 264, 255, 281, 278, 255, 291, 246, 257, 243, 266, 230, 252, 241, 265, 252, 249, 249, 249, 257, 258, 249, 256, 251, 241, 283, 252, 259, 288, 293, 256, 267, 251, 241, 269, 250, 254, 253, 246, 254, 260, 257, 258, 264, 264, 231, 247, 241, 240, 240, 285, 290, 259, 237, 236, 246, 262, 237, 268, 242, 243, 250, 258, 236, 264, 234, 281, 233, 254, 225, 253, 279, 246, 256, 249, 259, 261, 259, 243, 262] payload_rows_per_dst=[5450, 5508, 5512, 5538, 5536, 5468, 5452, 5484] dup_rows_per_dst=[2646, 2752, 2828, 2734, 2778, 2612, 2606, 2632] combine_rows_per_src=[5450, 5508, 5512, 5538, 5536, 5468, 5452, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38084 dup_rows=21956 dup_rows_remote=19284 dedup_ratio=0.335022 router_tokens_per_expert=[280, 220, 279, 272, 251, 239, 263, 268, 275, 254, 257, 268, 270, 285, 286, 278, 249, 236, 218, 246, 224, 263, 291, 227, 241, 251, 247, 261, 245, 258, 247, 234, 245, 253, 278, 264, 259, 235, 257, 291, 266, 246, 217, 269, 260, 258, 256, 260, 246, 264, 276, 254, 290, 279, 214, 266, 275, 256, 265, 258, 275, 241, 285, 249, 276, 241, 250, 272, 245, 248, 260, 276, 245, 249, 237, 257, 271, 260, 246, 251, 256, 251, 235, 256, 247, 246, 267, 254, 242, 274, 259, 249, 254, 254, 230, 295, 263, 255, 254, 225, 254, 247, 220, 240, 249, 251, 239, 269, 256, 266, 267, 227, 255, 282, 267, 275, 257, 272, 236, 244, 245, 263, 275, 237, 284, 277, 233, 241] payload_rows_per_dst=[5602, 5332, 5466, 5488, 5496, 5376, 5374, 5446] dup_rows_per_dst=[2888, 2544, 2762, 2898, 2672, 2762, 2590, 2840] combine_rows_per_src=[5602, 5332, 5466, 5488, 5496, 5376, 5374, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38324 dup_rows=21700 dup_rows_remote=18862 dedup_ratio=0.331116 router_tokens_per_expert=[242, 270, 278, 238, 232, 271, 250, 258, 270, 258, 259, 241, 241, 244, 254, 280, 293, 225, 250, 271, 268, 262, 261, 255, 244, 272, 230, 243, 261, 229, 268, 248, 242, 252, 255, 255, 272, 261, 246, 271, 255, 255, 268, 236, 274, 250, 218, 250, 258, 237, 253, 233, 254, 240, 255, 251, 243, 223, 256, 268, 240, 255, 232, 263, 274, 267, 226, 265, 275, 275, 281, 241, 247, 257, 250, 255, 266, 246, 232, 229, 264, 255, 260, 268, 270, 265, 259, 240, 279, 237, 246, 259, 244, 254, 269, 263, 295, 268, 240, 270, 276, 263, 248, 278, 258, 257, 230, 259, 267, 254, 259, 266, 233, 280, 238, 272, 262, 241, 268, 269, 251, 260, 273, 275, 273, 251, 267, 262] payload_rows_per_dst=[5112, 5526, 5670, 5616, 5424, 5464, 5512, 5512] dup_rows_per_dst=[2108, 2634, 2906, 2754, 2748, 2800, 2912, 2838] combine_rows_per_src=[5112, 5526, 5670, 5616, 5424, 5464, 5512, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=228944 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156401232 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312573520 weight_movement_bytes=18874368 total_nvlink_bytes=331447888 weight_to_token_ratio=0.060384 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229032 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157499048 ingress_bytes=195018752 link_bytes=195018752 token_bytes=314769064 weight_movement_bytes=37748736 total_nvlink_bytes=352517800 weight_to_token_ratio=0.119925 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868573726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=230072 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157606584 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314983096 weight_movement_bytes=18874368 total_nvlink_bytes=333857464 weight_to_token_ratio=0.059922 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157614080 dispatch_meta_bytes=229824 combine_payload_bytes=157614080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157843904 ingress_bytes=176488448 link_bytes=176488448 token_bytes=315457984 weight_movement_bytes=18874368 total_nvlink_bytes=334332352 weight_to_token_ratio=0.059832 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691025471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=229472 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156221536 ingress_bytes=174866432 link_bytes=174866432 token_bytes=312213600 weight_movement_bytes=18874368 total_nvlink_bytes=331087968 weight_to_token_ratio=0.060453 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675484112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=228744 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157203848 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314178952 weight_movement_bytes=37748736 total_nvlink_bytes=351927688 weight_to_token_ratio=0.120150 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38176 dup_rows=21860 dup_rows_remote=19240 dedup_ratio=0.333557 router_tokens_per_expert=[273, 257, 247, 258, 265, 259, 265, 272, 259, 257, 265, 277, 239, 245, 230, 240, 233, 270, 252, 251, 249, 261, 232, 238, 265, 285, 242, 258, 242, 260, 236, 287, 228, 241, 281, 244, 269, 228, 240, 257, 251, 253, 245, 278, 263, 255, 294, 249, 244, 246, 230, 258, 247, 253, 228, 270, 252, 277, 255, 264, 268, 267, 238, 247, 285, 258, 263, 263, 273, 231, 256, 264, 271, 246, 255, 269, 285, 269, 264, 246, 249, 248, 236, 247, 280, 247, 243, 282, 253, 265, 253, 235, 240, 274, 262, 243, 260, 270, 217, 257, 255, 286, 230, 252, 251, 242, 266, 243, 247, 269, 248, 267, 276, 298, 262, 248, 243, 264, 249, 231, 275, 240, 288, 258, 250, 260, 252, 270] payload_rows_per_dst=[5394, 5416, 5478, 5402, 5566, 5426, 5500, 5494] dup_rows_per_dst=[2822, 2706, 2674, 2686, 2830, 2688, 2620, 2834] combine_rows_per_src=[5394, 5416, 5478, 5402, 5566, 5426, 5500, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38236 dup_rows=21758 dup_rows_remote=18924 dedup_ratio=0.332001 router_tokens_per_expert=[273, 261, 273, 261, 228, 233, 272, 253, 251, 284, 237, 257, 237, 252, 260, 242, 242, 281, 253, 264, 261, 246, 245, 247, 280, 251, 277, 257, 245, 227, 250, 274, 271, 268, 248, 281, 270, 264, 244, 281, 263, 266, 242, 269, 248, 268, 276, 229, 237, 274, 256, 231, 267, 272, 267, 266, 233, 251, 277, 225, 238, 240, 253, 256, 261, 280, 257, 240, 244, 229, 251, 265, 270, 257, 236, 253, 245, 260, 283, 237, 261, 258, 257, 263, 238, 256, 243, 262, 248, 241, 259, 252, 268, 246, 258, 243, 250, 259, 257, 259, 283, 258, 237, 263, 251, 264, 236, 266, 288, 264, 266, 245, 272, 236, 239, 260, 279, 240, 266, 256, 246, 252, 251, 255, 231, 260, 269, 284] payload_rows_per_dst=[5482, 5470, 5542, 5344, 5498, 5498, 5514, 5430] dup_rows_per_dst=[2666, 2730, 2834, 2742, 2638, 2608, 2778, 2762] combine_rows_per_src=[5482, 5470, 5542, 5344, 5498, 5498, 5514, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38156 dup_rows=21780 dup_rows_remote=18984 dedup_ratio=0.332336 router_tokens_per_expert=[266, 270, 257, 239, 253, 260, 252, 275, 254, 272, 221, 268, 262, 243, 240, 251, 257, 209, 287, 265, 255, 270, 232, 254, 244, 274, 241, 268, 245, 275, 268, 261, 253, 271, 259, 238, 244, 256, 266, 259, 253, 280, 244, 262, 278, 295, 256, 249, 237, 266, 261, 235, 249, 237, 249, 272, 247, 249, 264, 250, 252, 275, 260, 259, 279, 236, 246, 239, 242, 258, 243, 255, 276, 271, 270, 280, 268, 229, 271, 243, 256, 217, 259, 243, 253, 269, 245, 247, 249, 301, 273, 251, 244, 256, 265, 266, 263, 247, 259, 264, 276, 253, 278, 243, 241, 246, 256, 265, 256, 238, 251, 229, 233, 248, 257, 250, 260, 275, 266, 249, 246, 265, 265, 264, 230, 263, 270, 249] payload_rows_per_dst=[5600, 5456, 5518, 5496, 5484, 5552, 5218, 5432] dup_rows_per_dst=[2796, 2754, 2808, 2628, 2728, 2846, 2472, 2748] combine_rows_per_src=[5600, 5456, 5518, 5496, 5484, 5552, 5218, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229664 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598560 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967456 weight_movement_bytes=18874368 total_nvlink_bytes=331841824 weight_to_token_ratio=0.060308 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=228640 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156843296 ingress_bytes=194363392 link_bytes=194363392 token_bytes=313457952 weight_movement_bytes=37748736 total_nvlink_bytes=351206688 weight_to_token_ratio=0.120427 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862294389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38298 dup_rows=21796 dup_rows_remote=19104 dedup_ratio=0.332581 router_tokens_per_expert=[276, 271, 261, 247, 263, 271, 258, 255, 251, 258, 223, 272, 254, 261, 240, 264, 270, 230, 242, 236, 220, 260, 243, 293, 268, 242, 291, 250, 248, 266, 259, 249, 278, 244, 288, 247, 229, 268, 241, 276, 232, 219, 261, 240, 275, 239, 272, 258, 278, 256, 260, 257, 278, 270, 224, 254, 247, 261, 261, 235, 231, 239, 253, 249, 260, 226, 248, 255, 252, 252, 264, 252, 283, 265, 243, 264, 239, 281, 268, 254, 259, 272, 256, 244, 298, 275, 251, 250, 246, 262, 233, 259, 257, 248, 244, 267, 274, 253, 261, 254, 235, 255, 232, 258, 271, 254, 272, 225, 240, 258, 286, 263, 262, 240, 271, 277, 276, 271, 291, 227, 254, 251, 251, 244, 275, 265, 250, 233] payload_rows_per_dst=[5500, 5442, 5428, 5448, 5454, 5530, 5442, 5496] dup_rows_per_dst=[2750, 2692, 2706, 2658, 2758, 2712, 2740, 2780] combine_rows_per_src=[5500, 5442, 5428, 5448, 5454, 5530, 5442, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38266 dup_rows=21880 dup_rows_remote=19108 dedup_ratio=0.333862 router_tokens_per_expert=[248, 265, 276, 260, 272, 252, 273, 237, 279, 246, 270, 273, 262, 275, 243, 255, 262, 274, 264, 236, 239, 248, 243, 241, 275, 269, 273, 250, 246, 278, 242, 244, 268, 264, 245, 236, 247, 286, 249, 233, 259, 229, 234, 263, 231, 268, 254, 261, 283, 239, 259, 244, 276, 244, 285, 254, 275, 239, 234, 290, 228, 252, 256, 223, 249, 282, 274, 255, 264, 248, 241, 286, 259, 275, 261, 253, 251, 247, 253, 251, 250, 253, 271, 249, 250, 303, 255, 242, 261, 237, 248, 264, 231, 254, 247, 238, 276, 251, 261, 253, 229, 262, 245, 250, 251, 239, 285, 224, 233, 266, 266, 226, 263, 266, 255, 274, 268, 244, 240, 252, 264, 263, 276, 260, 247, 274, 260, 265] payload_rows_per_dst=[5592, 5498, 5412, 5390, 5490, 5406, 5376, 5492] dup_rows_per_dst=[2780, 2670, 2642, 2772, 2808, 2700, 2658, 2850] combine_rows_per_src=[5592, 5498, 5412, 5390, 5490, 5406, 5376, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38276 dup_rows=21894 dup_rows_remote=19150 dedup_ratio=0.334076 router_tokens_per_expert=[266, 263, 256, 264, 240, 278, 252, 303, 217, 254, 268, 210, 267, 282, 301, 280, 263, 266, 270, 255, 256, 240, 271, 278, 260, 231, 254, 241, 261, 241, 261, 263, 247, 272, 248, 256, 246, 254, 304, 250, 267, 234, 239, 250, 252, 242, 248, 230, 257, 215, 245, 256, 248, 275, 283, 241, 258, 250, 265, 285, 278, 258, 265, 236, 238, 225, 262, 255, 254, 258, 259, 266, 252, 251, 245, 265, 252, 273, 258, 260, 242, 255, 240, 280, 255, 261, 276, 217, 249, 265, 255, 235, 232, 259, 267, 267, 258, 258, 252, 289, 241, 280, 268, 252, 239, 248, 261, 248, 262, 271, 284, 233, 255, 237, 245, 241, 266, 260, 269, 251, 257, 270, 248, 237, 263, 254, 232, 245] payload_rows_per_dst=[5522, 5506, 5470, 5484, 5404, 5366, 5496, 5394] dup_rows_per_dst=[2880, 2716, 2608, 2746, 2742, 2744, 2792, 2666] combine_rows_per_src=[5522, 5506, 5470, 5484, 5404, 5366, 5496, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38294 dup_rows=21808 dup_rows_remote=19106 dedup_ratio=0.332764 router_tokens_per_expert=[252, 251, 289, 278, 261, 276, 218, 275, 244, 238, 259, 247, 253, 261, 283, 238, 268, 263, 277, 237, 254, 257, 256, 273, 240, 252, 244, 262, 251, 255, 268, 258, 212, 240, 256, 237, 269, 251, 273, 264, 261, 250, 233, 244, 279, 271, 251, 251, 262, 260, 233, 277, 256, 234, 268, 248, 285, 256, 268, 247, 265, 234, 284, 254, 241, 260, 281, 247, 257, 239, 241, 252, 238, 218, 279, 266, 231, 226, 233, 252, 272, 263, 269, 296, 274, 261, 250, 257, 280, 257, 277, 266, 251, 262, 246, 262, 256, 253, 244, 238, 262, 232, 285, 265, 267, 246, 243, 268, 264, 254, 263, 247, 254, 265, 255, 260, 254, 233, 243, 236, 237, 253, 284, 254, 276, 247, 276, 239] payload_rows_per_dst=[5440, 5458, 5432, 5516, 5380, 5602, 5466, 5434] dup_rows_per_dst=[2806, 2768, 2598, 2800, 2542, 2884, 2708, 2702] combine_rows_per_src=[5440, 5458, 5432, 5516, 5380, 5602, 5466, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38274 dup_rows=21760 dup_rows_remote=19004 dedup_ratio=0.332031 router_tokens_per_expert=[238, 249, 257, 272, 238, 258, 259, 242, 253, 251, 252, 248, 242, 237, 250, 261, 254, 249, 223, 272, 263, 241, 245, 276, 264, 268, 243, 262, 275, 232, 250, 273, 247, 248, 245, 246, 272, 235, 279, 255, 270, 271, 251, 281, 253, 270, 258, 272, 278, 292, 240, 285, 277, 262, 254, 249, 242, 268, 245, 249, 239, 272, 256, 235, 234, 276, 232, 278, 232, 260, 267, 265, 285, 250, 245, 245, 256, 260, 288, 256, 236, 252, 257, 286, 263, 260, 237, 258, 259, 257, 261, 270, 279, 263, 212, 232, 276, 249, 250, 270, 269, 243, 229, 229, 251, 256, 258, 250, 261, 243, 272, 259, 265, 248, 248, 231, 268, 279, 260, 253, 249, 258, 249, 226, 274, 261, 269, 261] payload_rows_per_dst=[5372, 5464, 5504, 5520, 5502, 5520, 5430, 5464] dup_rows_per_dst=[2642, 2716, 2802, 2766, 2756, 2644, 2700, 2734] combine_rows_per_src=[5372, 5464, 5504, 5520, 5502, 5520, 5430, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228560 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156515536 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312802512 weight_movement_bytes=18874368 total_nvlink_bytes=331676880 weight_to_token_ratio=0.060340 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229704 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157008200 ingress_bytes=213401600 link_bytes=213401600 token_bytes=313786696 weight_movement_bytes=56623104 total_nvlink_bytes=370409800 weight_to_token_ratio=0.180451 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044709131e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229608 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098216 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966824 weight_movement_bytes=18874368 total_nvlink_bytes=332841192 weight_to_token_ratio=0.060116 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229496 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156967032 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704568 weight_movement_bytes=37748736 total_nvlink_bytes=351453304 weight_to_token_ratio=0.120332 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229600 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=157081824 ingress_bytes=251224064 link_bytes=251224064 token_bytes=313934048 weight_movement_bytes=94371840 total_nvlink_bytes=408305888 weight_to_token_ratio=0.300610 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407105371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229112 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999416 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769720 weight_movement_bytes=18874368 total_nvlink_bytes=332644088 weight_to_token_ratio=0.060154 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38208 dup_rows=21816 dup_rows_remote=19020 dedup_ratio=0.332886 router_tokens_per_expert=[281, 256, 264, 245, 263, 252, 251, 268, 254, 256, 263, 234, 277, 253, 254, 247, 260, 245, 246, 279, 227, 242, 253, 252, 239, 232, 268, 248, 249, 241, 252, 295, 293, 222, 254, 232, 247, 243, 245, 260, 256, 256, 249, 272, 274, 264, 258, 247, 269, 249, 257, 263, 248, 237, 261, 275, 276, 273, 256, 283, 234, 240, 253, 250, 257, 277, 275, 240, 257, 267, 237, 285, 244, 262, 241, 228, 247, 263, 252, 252, 241, 253, 277, 252, 247, 252, 254, 265, 239, 237, 268, 277, 241, 241, 259, 230, 273, 253, 241, 267, 288, 253, 266, 245, 254, 269, 268, 262, 237, 244, 265, 269, 276, 270, 279, 256, 266, 245, 249, 266, 272, 245, 242, 264, 247, 260, 259, 259] payload_rows_per_dst=[5418, 5474, 5446, 5518, 5408, 5430, 5512, 5514] dup_rows_per_dst=[2818, 2582, 2698, 2730, 2760, 2636, 2796, 2796] combine_rows_per_src=[5418, 5474, 5446, 5518, 5408, 5430, 5512, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38190 dup_rows=21896 dup_rows_remote=19148 dedup_ratio=0.334106 router_tokens_per_expert=[259, 258, 241, 255, 283, 252, 248, 248, 221, 245, 272, 245, 241, 243, 281, 238, 247, 271, 249, 240, 259, 247, 257, 251, 252, 266, 251, 268, 270, 249, 283, 261, 270, 244, 250, 272, 244, 248, 277, 258, 264, 257, 242, 226, 261, 267, 268, 251, 266, 245, 251, 287, 269, 240, 270, 253, 282, 247, 269, 254, 242, 235, 240, 275, 250, 237, 274, 259, 273, 255, 268, 251, 232, 278, 263, 255, 261, 275, 258, 245, 246, 273, 250, 255, 253, 234, 236, 271, 266, 236, 255, 233, 257, 257, 259, 256, 248, 251, 273, 276, 248, 234, 254, 239, 249, 258, 272, 274, 226, 244, 272, 279, 225, 271, 246, 264, 281, 243, 266, 247, 281, 261, 242, 242, 272, 290, 234, 260] payload_rows_per_dst=[5364, 5530, 5450, 5552, 5508, 5398, 5444, 5394] dup_rows_per_dst=[2696, 2712, 2748, 2698, 2760, 2676, 2750, 2856] combine_rows_per_src=[5364, 5530, 5450, 5552, 5508, 5398, 5444, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228912 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156728880 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313228848 weight_movement_bytes=18874368 total_nvlink_bytes=332103216 weight_to_token_ratio=0.060257 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229352 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655592 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081832 weight_movement_bytes=37748736 total_nvlink_bytes=350830568 weight_to_token_ratio=0.120571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38264 dup_rows=21902 dup_rows_remote=19190 dedup_ratio=0.334198 router_tokens_per_expert=[243, 232, 276, 232, 253, 272, 269, 260, 252, 247, 256, 253, 245, 245, 242, 264, 271, 269, 255, 232, 237, 276, 249, 244, 270, 250, 280, 236, 254, 250, 270, 250, 251, 248, 264, 273, 244, 263, 230, 254, 267, 258, 239, 254, 248, 252, 237, 295, 275, 252, 260, 261, 282, 244, 255, 278, 285, 274, 274, 253, 282, 274, 239, 265, 257, 245, 252, 239, 268, 255, 258, 295, 268, 274, 254, 264, 283, 265, 278, 242, 271, 218, 257, 255, 254, 247, 252, 279, 247, 275, 281, 260, 237, 264, 268, 264, 222, 271, 223, 236, 247, 232, 254, 269, 240, 246, 252, 248, 234, 249, 248, 259, 250, 244, 249, 217, 257, 263, 262, 297, 250, 253, 242, 270, 249, 261, 245, 239] payload_rows_per_dst=[5370, 5698, 5582, 5476, 5544, 5504, 5118, 5342] dup_rows_per_dst=[2712, 3036, 2788, 2616, 2878, 2874, 2244, 2754] combine_rows_per_src=[5370, 5698, 5582, 5476, 5544, 5504, 5118, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38256 dup_rows=21846 dup_rows_remote=19160 dedup_ratio=0.333344 router_tokens_per_expert=[240, 274, 241, 253, 255, 271, 265, 273, 238, 256, 256, 273, 233, 250, 242, 260, 240, 246, 257, 261, 264, 291, 231, 275, 266, 262, 257, 244, 277, 231, 247, 269, 229, 264, 240, 264, 249, 281, 238, 274, 244, 256, 258, 246, 250, 262, 262, 252, 259, 294, 272, 239, 224, 268, 260, 251, 288, 264, 279, 273, 281, 244, 281, 235, 247, 219, 260, 256, 271, 263, 224, 303, 246, 258, 267, 245, 239, 266, 247, 256, 277, 238, 261, 260, 257, 244, 241, 241, 257, 255, 229, 258, 263, 253, 234, 292, 275, 256, 278, 284, 255, 246, 267, 262, 250, 214, 244, 268, 241, 238, 253, 264, 269, 243, 265, 262, 253, 293, 236, 221, 250, 277, 260, 267, 221, 251, 257, 242] payload_rows_per_dst=[5468, 5430, 5462, 5592, 5410, 5434, 5462, 5432] dup_rows_per_dst=[2692, 2806, 2676, 2832, 2724, 2686, 2728, 2702] combine_rows_per_src=[5468, 5430, 5462, 5592, 5410, 5434, 5462, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38284 dup_rows=21842 dup_rows_remote=19084 dedup_ratio=0.333282 router_tokens_per_expert=[259, 254, 275, 262, 267, 277, 247, 264, 267, 235, 250, 256, 272, 244, 257, 260, 230, 265, 269, 260, 297, 292, 267, 245, 288, 260, 273, 229, 246, 224, 242, 273, 257, 240, 265, 299, 272, 252, 250, 219, 242, 238, 267, 302, 239, 272, 236, 265, 243, 234, 262, 265, 237, 273, 272, 269, 249, 270, 271, 243, 244, 257, 237, 258, 260, 244, 258, 254, 256, 253, 234, 255, 252, 263, 245, 233, 252, 240, 239, 249, 249, 247, 259, 263, 269, 248, 269, 250, 246, 240, 264, 252, 250, 254, 258, 268, 249, 256, 240, 277, 237, 239, 248, 257, 274, 249, 286, 235, 259, 261, 244, 257, 219, 235, 273, 246, 252, 243, 254, 269, 279, 268, 276, 250, 268, 268, 252, 270] payload_rows_per_dst=[5496, 5512, 5434, 5410, 5362, 5474, 5436, 5570] dup_rows_per_dst=[2796, 2808, 2796, 2758, 2612, 2698, 2700, 2674] combine_rows_per_src=[5496, 5512, 5434, 5410, 5362, 5474, 5436, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43452 payload_rows_remote=37986 dup_rows=22084 dup_rows_remote=19296 dedup_ratio=0.336975 router_tokens_per_expert=[240, 223, 256, 259, 246, 248, 284, 251, 254, 253, 247, 258, 254, 245, 219, 254, 238, 255, 244, 276, 234, 286, 250, 259, 247, 264, 258, 274, 243, 267, 254, 278, 243, 247, 281, 269, 251, 266, 258, 281, 246, 245, 277, 266, 266, 234, 282, 243, 241, 253, 275, 240, 264, 246, 272, 284, 280, 219, 251, 240, 238, 268, 282, 254, 280, 245, 264, 258, 251, 260, 255, 230, 216, 277, 291, 242, 247, 275, 248, 264, 281, 268, 233, 264, 249, 223, 251, 271, 287, 253, 250, 260, 238, 260, 244, 263, 252, 264, 239, 255, 252, 274, 262, 258, 258, 254, 238, 253, 253, 250, 260, 241, 262, 259, 222, 258, 309, 249, 267, 239, 247, 238, 253, 252, 270, 254, 244, 304] payload_rows_per_dst=[5420, 5466, 5422, 5440, 5422, 5428, 5362, 5492] dup_rows_per_dst=[2562, 2788, 2888, 2774, 2784, 2762, 2764, 2762] combine_rows_per_src=[5420, 5466, 5422, 5440, 5422, 5428, 5362, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38180 dup_rows=21772 dup_rows_remote=18838 dedup_ratio=0.332214 router_tokens_per_expert=[260, 240, 255, 266, 276, 243, 267, 251, 239, 270, 272, 263, 255, 273, 254, 253, 221, 269, 258, 269, 249, 246, 237, 218, 276, 257, 271, 265, 258, 276, 228, 247, 238, 247, 251, 256, 231, 257, 262, 247, 256, 286, 258, 300, 242, 258, 273, 258, 260, 237, 258, 267, 235, 261, 280, 244, 255, 241, 268, 261, 245, 265, 251, 269, 243, 244, 250, 247, 274, 284, 254, 252, 264, 251, 240, 249, 273, 279, 267, 224, 266, 244, 229, 275, 277, 249, 248, 264, 226, 230, 263, 237, 268, 259, 263, 262, 260, 230, 230, 262, 244, 258, 262, 272, 265, 258, 282, 242, 261, 248, 238, 279, 252, 264, 265, 237, 254, 252, 255, 264, 261, 269, 270, 246, 251, 263, 257, 263] payload_rows_per_dst=[5458, 5414, 5420, 5532, 5422, 5452, 5482, 5584] dup_rows_per_dst=[2730, 2568, 2598, 2770, 2768, 2674, 2730, 2934] combine_rows_per_src=[5458, 5414, 5420, 5532, 5422, 5452, 5482, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44048 payload_rows_remote=38712 dup_rows=21488 dup_rows_remote=18854 dedup_ratio=0.327881 router_tokens_per_expert=[245, 257, 289, 253, 289, 261, 240, 250, 266, 231, 271, 248, 243, 252, 241, 264, 273, 307, 246, 241, 262, 248, 263, 239, 259, 255, 246, 260, 268, 261, 251, 266, 253, 275, 267, 248, 240, 276, 270, 246, 263, 282, 238, 244, 243, 262, 250, 272, 255, 246, 251, 278, 254, 248, 270, 272, 247, 277, 259, 268, 273, 264, 290, 233, 277, 243, 234, 224, 284, 252, 229, 255, 294, 269, 236, 229, 246, 259, 234, 220, 270, 260, 260, 257, 263, 227, 245, 238, 247, 289, 272, 251, 251, 269, 256, 253, 237, 263, 225, 288, 276, 231, 292, 266, 256, 264, 272, 250, 238, 257, 247, 265, 261, 264, 263, 225, 229, 261, 234, 267, 223, 241, 271, 247, 261, 255, 250, 237] payload_rows_per_dst=[5524, 5492, 5536, 5588, 5336, 5550, 5556, 5466] dup_rows_per_dst=[2676, 2798, 2722, 2782, 2634, 2666, 2698, 2512] combine_rows_per_src=[5524, 5492, 5536, 5588, 5336, 5550, 5556, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229816 combine_payload_bytes=156729344 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156959160 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688504 weight_movement_bytes=18874368 total_nvlink_bytes=332562872 weight_to_token_ratio=0.060169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229664 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926240 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622816 weight_movement_bytes=18874368 total_nvlink_bytes=332497184 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229472 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040736 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852000 weight_movement_bytes=18874368 total_nvlink_bytes=332726368 weight_to_token_ratio=0.060138 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=229128 combine_payload_bytes=155590656 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=155819784 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311410440 weight_movement_bytes=18874368 total_nvlink_bytes=330284808 weight_to_token_ratio=0.060609 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228072 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156613352 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312998632 weight_movement_bytes=18874368 total_nvlink_bytes=331873000 weight_to_token_ratio=0.060302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=158564352 dispatch_meta_bytes=230264 combine_payload_bytes=158564352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158794616 ingress_bytes=177438720 link_bytes=177438720 token_bytes=317358968 weight_movement_bytes=18874368 total_nvlink_bytes=336233336 weight_to_token_ratio=0.059473 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.700130510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38438 dup_rows=21626 dup_rows_remote=18966 dedup_ratio=0.329987 router_tokens_per_expert=[234, 262, 252, 233, 249, 275, 258, 254, 237, 269, 260, 246, 254, 260, 271, 257, 251, 255, 281, 245, 282, 255, 207, 252, 264, 244, 244, 266, 246, 251, 255, 253, 247, 271, 256, 286, 262, 264, 263, 257, 281, 249, 228, 287, 243, 249, 238, 267, 239, 232, 256, 294, 231, 247, 259, 282, 260, 286, 281, 264, 269, 258, 236, 249, 267, 277, 247, 277, 271, 247, 269, 260, 263, 255, 263, 250, 263, 257, 257, 263, 268, 260, 232, 249, 246, 244, 264, 231, 245, 255, 270, 274, 275, 239, 262, 268, 254, 265, 238, 264, 251, 225, 259, 241, 251, 257, 284, 257, 267, 264, 235, 254, 238, 256, 261, 248, 252, 250, 255, 254, 266, 225, 250, 252, 251, 261, 249, 253] payload_rows_per_dst=[5460, 5458, 5586, 5516, 5580, 5468, 5472, 5370] dup_rows_per_dst=[2682, 2644, 2710, 2770, 2792, 2696, 2660, 2672] combine_rows_per_src=[5460, 5458, 5586, 5516, 5580, 5468, 5472, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38220 dup_rows=21822 dup_rows_remote=19070 dedup_ratio=0.332977 router_tokens_per_expert=[255, 240, 251, 264, 233, 244, 236, 256, 241, 260, 256, 254, 249, 247, 267, 262, 252, 266, 224, 260, 260, 253, 251, 223, 286, 261, 254, 241, 257, 260, 256, 249, 260, 274, 228, 269, 258, 258, 245, 257, 251, 243, 233, 256, 284, 262, 275, 270, 281, 257, 250, 267, 254, 256, 248, 254, 254, 254, 242, 244, 239, 283, 270, 270, 243, 244, 276, 258, 266, 265, 256, 246, 225, 241, 252, 254, 262, 211, 254, 283, 260, 270, 221, 267, 241, 254, 258, 292, 245, 252, 279, 266, 242, 282, 266, 259, 234, 294, 247, 239, 237, 241, 265, 249, 282, 283, 263, 282, 255, 278, 275, 262, 256, 262, 238, 249, 255, 257, 257, 255, 277, 285, 247, 240, 232, 259, 272, 237] payload_rows_per_dst=[5362, 5454, 5494, 5484, 5436, 5550, 5518, 5416] dup_rows_per_dst=[2668, 2652, 2752, 2762, 2636, 2758, 2854, 2740] combine_rows_per_src=[5362, 5454, 5494, 5484, 5436, 5550, 5518, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229160 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156778280 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313327400 weight_movement_bytes=37748736 total_nvlink_bytes=351076136 weight_to_token_ratio=0.120477 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229616 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157671664 ingress_bytes=195190784 link_bytes=195190784 token_bytes=315113712 weight_movement_bytes=37748736 total_nvlink_bytes=352862448 weight_to_token_ratio=0.119794 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870222052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38054 dup_rows=21874 dup_rows_remote=19006 dedup_ratio=0.333771 router_tokens_per_expert=[252, 282, 287, 255, 260, 271, 246, 242, 261, 276, 260, 256, 248, 268, 248, 259, 237, 265, 255, 234, 250, 271, 250, 295, 256, 253, 230, 252, 261, 278, 252, 252, 245, 259, 255, 241, 256, 241, 252, 257, 280, 225, 224, 250, 257, 233, 247, 281, 250, 278, 247, 246, 241, 247, 238, 240, 244, 276, 226, 283, 237, 257, 238, 263, 263, 250, 239, 246, 303, 265, 283, 283, 277, 250, 234, 260, 272, 253, 237, 248, 262, 277, 264, 266, 255, 273, 251, 244, 277, 276, 268, 260, 267, 241, 268, 284, 269, 272, 265, 244, 252, 247, 285, 239, 263, 247, 265, 265, 241, 252, 295, 239, 256, 249, 229, 249, 263, 230, 235, 236, 278, 236, 237, 241, 242, 259, 272, 244] payload_rows_per_dst=[5608, 5510, 5256, 5594, 5546, 5514, 5300, 5334] dup_rows_per_dst=[2868, 2952, 2490, 2830, 2780, 2818, 2558, 2578] combine_rows_per_src=[5608, 5510, 5256, 5594, 5546, 5514, 5300, 5334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38210 dup_rows=21858 dup_rows_remote=19116 dedup_ratio=0.333527 router_tokens_per_expert=[283, 263, 249, 271, 266, 239, 290, 251, 248, 246, 252, 280, 265, 279, 254, 264, 243, 251, 243, 273, 233, 274, 257, 256, 291, 257, 253, 259, 288, 260, 252, 262, 262, 287, 246, 234, 259, 274, 239, 264, 258, 292, 236, 248, 272, 273, 250, 257, 255, 261, 264, 252, 269, 269, 254, 245, 243, 247, 262, 269, 259, 247, 254, 255, 256, 264, 253, 248, 248, 247, 237, 258, 297, 246, 247, 280, 256, 276, 260, 241, 249, 215, 209, 250, 234, 257, 275, 245, 252, 272, 225, 236, 248, 272, 266, 250, 254, 239, 228, 249, 255, 263, 249, 286, 253, 250, 252, 254, 266, 254, 271, 290, 250, 260, 224, 262, 238, 258, 267, 256, 199, 242, 259, 260, 259, 242, 254, 248] payload_rows_per_dst=[5526, 5500, 5512, 5468, 5444, 5338, 5544, 5346] dup_rows_per_dst=[2874, 2804, 2790, 2742, 2784, 2572, 2682, 2610] combine_rows_per_src=[5526, 5500, 5512, 5468, 5444, 5338, 5544, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38112 dup_rows=21902 dup_rows_remote=19080 dedup_ratio=0.334198 router_tokens_per_expert=[271, 238, 235, 296, 254, 259, 225, 270, 221, 253, 257, 248, 282, 258, 252, 246, 253, 252, 254, 257, 243, 257, 270, 266, 262, 244, 247, 241, 273, 264, 277, 244, 258, 243, 281, 271, 240, 236, 258, 275, 231, 257, 249, 256, 254, 260, 247, 248, 254, 222, 253, 264, 248, 267, 236, 259, 230, 244, 276, 261, 255, 240, 260, 251, 256, 248, 268, 254, 255, 263, 266, 251, 276, 240, 276, 233, 236, 258, 263, 243, 270, 243, 267, 237, 251, 260, 262, 266, 251, 262, 264, 254, 288, 244, 282, 271, 257, 265, 257, 261, 250, 274, 277, 250, 253, 267, 268, 283, 233, 250, 267, 245, 268, 233, 273, 284, 267, 266, 264, 239, 243, 247, 242, 261, 240, 269, 256, 248] payload_rows_per_dst=[5390, 5514, 5432, 5404, 5398, 5522, 5534, 5440] dup_rows_per_dst=[2740, 2694, 2696, 2636, 2774, 2822, 2780, 2760] combine_rows_per_src=[5390, 5514, 5432, 5404, 5398, 5522, 5534, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38262 dup_rows=21866 dup_rows_remote=19200 dedup_ratio=0.333649 router_tokens_per_expert=[251, 259, 276, 265, 258, 252, 256, 273, 252, 246, 260, 252, 261, 277, 261, 286, 238, 261, 253, 258, 274, 252, 255, 251, 278, 250, 260, 257, 249, 234, 247, 220, 244, 238, 239, 263, 253, 292, 263, 265, 269, 265, 312, 268, 250, 269, 256, 255, 254, 257, 235, 268, 276, 251, 266, 256, 233, 276, 217, 224, 269, 235, 275, 253, 275, 280, 246, 246, 272, 241, 220, 230, 262, 252, 252, 274, 300, 250, 276, 239, 238, 254, 244, 260, 267, 230, 272, 255, 244, 273, 260, 259, 232, 260, 265, 283, 262, 268, 247, 272, 251, 219, 265, 273, 249, 274, 264, 271, 260, 228, 218, 273, 266, 268, 246, 210, 239, 233, 250, 245, 238, 248, 274, 255, 231, 234, 279, 279] payload_rows_per_dst=[5564, 5408, 5500, 5370, 5448, 5494, 5470, 5416] dup_rows_per_dst=[2806, 2666, 2902, 2720, 2782, 2698, 2718, 2574] combine_rows_per_src=[5564, 5408, 5500, 5370, 5448, 5494, 5470, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38286 dup_rows=21730 dup_rows_remote=18944 dedup_ratio=0.331573 router_tokens_per_expert=[280, 263, 276, 237, 243, 255, 238, 251, 245, 262, 237, 288, 238, 247, 255, 267, 249, 283, 259, 274, 267, 232, 266, 234, 235, 264, 264, 261, 259, 247, 265, 228, 234, 283, 265, 239, 249, 253, 228, 246, 223, 261, 272, 246, 254, 264, 245, 262, 262, 273, 265, 265, 253, 279, 279, 254, 252, 276, 274, 261, 270, 260, 237, 271, 250, 242, 245, 241, 264, 245, 257, 255, 251, 262, 254, 235, 240, 219, 243, 266, 253, 260, 254, 250, 248, 264, 253, 249, 247, 279, 247, 249, 282, 253, 294, 255, 277, 243, 285, 251, 266, 266, 267, 245, 247, 253, 276, 277, 263, 226, 251, 254, 260, 269, 287, 270, 244, 252, 261, 261, 243, 244, 225, 249, 245, 252, 241, 288] payload_rows_per_dst=[5380, 5486, 5276, 5584, 5562, 5502, 5496, 5520] dup_rows_per_dst=[2768, 2688, 2236, 2894, 2912, 2746, 2700, 2786] combine_rows_per_src=[5380, 5486, 5276, 5584, 5562, 5502, 5496, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38202 dup_rows=21846 dup_rows_remote=19068 dedup_ratio=0.333344 router_tokens_per_expert=[257, 244, 252, 261, 246, 259, 300, 260, 264, 265, 247, 275, 269, 263, 259, 236, 261, 244, 263, 229, 253, 254, 254, 260, 234, 271, 249, 236, 253, 265, 247, 258, 294, 280, 276, 228, 269, 238, 258, 262, 244, 245, 254, 272, 259, 255, 255, 241, 247, 244, 242, 269, 260, 237, 275, 238, 248, 268, 276, 244, 230, 257, 250, 260, 235, 251, 253, 226, 270, 256, 224, 259, 271, 257, 268, 292, 244, 264, 283, 280, 261, 256, 251, 259, 264, 282, 255, 251, 255, 257, 247, 250, 253, 283, 240, 277, 252, 266, 251, 246, 244, 264, 240, 253, 257, 281, 265, 235, 268, 243, 274, 270, 281, 286, 257, 243, 223, 261, 235, 241, 236, 271, 260, 251, 263, 216, 241, 257] payload_rows_per_dst=[5466, 5468, 5460, 5420, 5488, 5518, 5504, 5366] dup_rows_per_dst=[2848, 2594, 2800, 2670, 2778, 2764, 2714, 2678] combine_rows_per_src=[5466, 5468, 5460, 5420, 5488, 5518, 5504, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155869184 dispatch_meta_bytes=228240 combine_payload_bytes=155869184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156097424 ingress_bytes=174743552 link_bytes=174743552 token_bytes=311966608 weight_movement_bytes=18874368 total_nvlink_bytes=330840976 weight_to_token_ratio=0.060501 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674306736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229304 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156737464 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245624 weight_movement_bytes=37748736 total_nvlink_bytes=350994360 weight_to_token_ratio=0.120508 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228768 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156335520 ingress_bytes=193855488 link_bytes=193855488 token_bytes=312442272 weight_movement_bytes=37748736 total_nvlink_bytes=350191008 weight_to_token_ratio=0.120818 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857427903e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229848 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156951000 ingress_bytes=194469888 link_bytes=194469888 token_bytes=313672152 weight_movement_bytes=37748736 total_nvlink_bytes=351420888 weight_to_token_ratio=0.120345 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863314782e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=228920 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048376 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313867832 weight_movement_bytes=18874368 total_nvlink_bytes=332742200 weight_to_token_ratio=0.060135 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229080 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704472 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179864 weight_movement_bytes=18874368 total_nvlink_bytes=332054232 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38296 dup_rows=21710 dup_rows_remote=18874 dedup_ratio=0.331268 router_tokens_per_expert=[232, 259, 249, 253, 260, 267, 273, 277, 270, 254, 275, 226, 248, 257, 272, 246, 258, 251, 261, 253, 271, 263, 247, 272, 244, 237, 253, 249, 238, 261, 238, 252, 252, 253, 279, 269, 265, 253, 256, 273, 277, 263, 256, 268, 265, 218, 248, 288, 268, 254, 251, 286, 250, 240, 237, 254, 277, 243, 246, 245, 255, 248, 262, 237, 245, 268, 256, 256, 272, 243, 257, 240, 260, 245, 268, 236, 270, 269, 294, 225, 282, 247, 262, 264, 254, 259, 262, 257, 273, 249, 241, 247, 244, 237, 267, 270, 261, 246, 273, 237, 271, 289, 259, 265, 199, 277, 262, 242, 226, 270, 262, 263, 249, 260, 258, 238, 257, 264, 248, 261, 242, 247, 242, 245, 246, 273, 250, 265] payload_rows_per_dst=[5486, 5454, 5530, 5430, 5468, 5490, 5504, 5464] dup_rows_per_dst=[2750, 2642, 2836, 2676, 2740, 2740, 2700, 2626] combine_rows_per_src=[5486, 5454, 5530, 5430, 5468, 5490, 5504, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38160 dup_rows=21910 dup_rows_remote=19168 dedup_ratio=0.334320 router_tokens_per_expert=[243, 242, 233, 276, 236, 274, 262, 243, 279, 262, 264, 267, 270, 258, 257, 239, 277, 281, 248, 271, 230, 262, 298, 246, 263, 253, 257, 265, 261, 248, 276, 228, 284, 250, 258, 246, 229, 271, 236, 259, 253, 279, 263, 279, 242, 227, 229, 259, 280, 268, 249, 243, 290, 247, 275, 240, 231, 222, 254, 240, 276, 258, 232, 263, 242, 248, 245, 265, 255, 269, 253, 246, 263, 219, 245, 261, 281, 249, 230, 245, 258, 262, 281, 278, 272, 251, 253, 255, 260, 233, 250, 264, 249, 257, 238, 273, 265, 247, 280, 253, 286, 252, 231, 261, 250, 278, 256, 259, 253, 257, 235, 241, 240, 252, 264, 236, 256, 253, 240, 257, 287, 257, 258, 260, 256, 257, 256, 284] payload_rows_per_dst=[5452, 5486, 5404, 5404, 5434, 5504, 5466, 5476] dup_rows_per_dst=[2758, 2842, 2724, 2732, 2598, 2764, 2742, 2750] combine_rows_per_src=[5452, 5486, 5404, 5404, 5434, 5504, 5466, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228680 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157089096 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313949512 weight_movement_bytes=37748736 total_nvlink_bytes=351698248 weight_to_token_ratio=0.120238 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229312 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156532672 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312836032 weight_movement_bytes=37748736 total_nvlink_bytes=350584768 weight_to_token_ratio=0.120666 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38224 dup_rows=21706 dup_rows_remote=18908 dedup_ratio=0.331207 router_tokens_per_expert=[267, 260, 287, 247, 263, 281, 259, 248, 263, 269, 272, 286, 261, 258, 232, 247, 252, 279, 260, 244, 276, 252, 270, 252, 248, 275, 232, 274, 255, 251, 252, 273, 234, 257, 278, 255, 246, 252, 271, 265, 250, 250, 264, 242, 279, 246, 263, 265, 263, 268, 285, 249, 247, 256, 259, 254, 273, 242, 263, 246, 255, 238, 235, 248, 254, 276, 265, 244, 276, 261, 237, 277, 252, 241, 278, 261, 236, 262, 236, 273, 255, 266, 254, 253, 268, 258, 248, 263, 223, 237, 266, 254, 241, 262, 234, 252, 250, 236, 232, 248, 230, 261, 244, 251, 279, 253, 249, 275, 270, 229, 265, 248, 255, 224, 241, 231, 295, 249, 267, 255, 266, 254, 245, 254, 268, 254, 249, 235] payload_rows_per_dst=[5606, 5510, 5676, 5550, 5268, 5430, 5432, 5358] dup_rows_per_dst=[2798, 2900, 3036, 2608, 2512, 2638, 2608, 2606] combine_rows_per_src=[5606, 5510, 5676, 5550, 5268, 5430, 5432, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38394 dup_rows=21762 dup_rows_remote=19104 dedup_ratio=0.332062 router_tokens_per_expert=[231, 266, 238, 278, 251, 268, 243, 227, 251, 272, 253, 274, 260, 239, 261, 276, 245, 286, 254, 245, 236, 269, 250, 253, 261, 249, 230, 257, 261, 254, 261, 254, 234, 276, 251, 263, 231, 294, 248, 250, 256, 280, 241, 300, 261, 258, 238, 258, 263, 264, 247, 232, 260, 247, 241, 242, 249, 247, 278, 250, 246, 262, 249, 242, 262, 244, 243, 248, 273, 277, 248, 288, 234, 250, 252, 262, 236, 252, 269, 233, 240, 271, 263, 265, 263, 237, 271, 269, 248, 263, 280, 247, 263, 271, 270, 246, 266, 270, 245, 281, 261, 256, 267, 267, 250, 255, 280, 259, 282, 285, 249, 246, 255, 288, 282, 229, 256, 267, 233, 237, 268, 240, 261, 249, 205, 247, 259, 224] payload_rows_per_dst=[5488, 5402, 5486, 5380, 5452, 5552, 5638, 5376] dup_rows_per_dst=[2688, 2728, 2792, 2658, 2690, 2782, 2800, 2624] combine_rows_per_src=[5488, 5402, 5486, 5380, 5452, 5552, 5638, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38356 dup_rows=21766 dup_rows_remote=19058 dedup_ratio=0.332123 router_tokens_per_expert=[279, 259, 271, 221, 260, 258, 278, 267, 276, 244, 267, 246, 242, 260, 260, 230, 260, 233, 252, 256, 260, 244, 237, 282, 254, 249, 257, 247, 258, 254, 271, 232, 256, 280, 268, 268, 250, 247, 242, 258, 271, 278, 266, 279, 239, 264, 274, 245, 256, 260, 235, 236, 269, 267, 281, 276, 238, 249, 244, 251, 245, 274, 247, 271, 249, 265, 253, 269, 245, 268, 257, 238, 256, 229, 256, 252, 259, 263, 247, 258, 243, 253, 226, 254, 266, 252, 262, 235, 268, 279, 271, 243, 252, 263, 268, 226, 257, 261, 263, 247, 238, 255, 250, 264, 268, 253, 253, 222, 260, 250, 225, 261, 250, 261, 268, 246, 280, 254, 294, 274, 235, 265, 260, 239, 268, 257, 275, 242] payload_rows_per_dst=[5476, 5422, 5550, 5504, 5472, 5414, 5450, 5482] dup_rows_per_dst=[2760, 2670, 2820, 2694, 2656, 2708, 2604, 2854] combine_rows_per_src=[5476, 5422, 5550, 5504, 5472, 5414, 5450, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38268 dup_rows=21836 dup_rows_remote=19116 dedup_ratio=0.333191 router_tokens_per_expert=[272, 251, 260, 264, 249, 250, 263, 249, 239, 267, 244, 250, 261, 263, 249, 265, 264, 267, 258, 243, 265, 253, 272, 251, 283, 269, 233, 254, 210, 236, 248, 270, 297, 243, 237, 231, 251, 288, 246, 301, 287, 251, 277, 287, 259, 254, 229, 272, 270, 256, 254, 282, 243, 268, 253, 261, 257, 257, 274, 269, 270, 248, 249, 246, 267, 275, 275, 274, 261, 256, 254, 239, 222, 227, 248, 263, 259, 243, 265, 243, 268, 258, 265, 249, 245, 270, 263, 269, 283, 270, 241, 242, 230, 260, 271, 234, 236, 252, 244, 254, 234, 247, 249, 247, 251, 222, 238, 233, 265, 241, 252, 252, 274, 272, 232, 243, 263, 251, 277, 235, 242, 255, 277, 264, 264, 259, 250, 265] payload_rows_per_dst=[5480, 5432, 5618, 5480, 5404, 5428, 5352, 5506] dup_rows_per_dst=[2712, 2720, 2802, 2834, 2738, 2808, 2482, 2740] combine_rows_per_src=[5480, 5432, 5618, 5480, 5404, 5428, 5352, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38298 dup_rows=21786 dup_rows_remote=19074 dedup_ratio=0.332428 router_tokens_per_expert=[263, 251, 241, 273, 264, 253, 252, 236, 268, 260, 250, 259, 256, 271, 258, 243, 236, 231, 273, 234, 248, 267, 249, 256, 255, 262, 279, 258, 297, 267, 262, 254, 242, 261, 265, 236, 265, 237, 273, 236, 249, 262, 283, 263, 270, 269, 261, 264, 246, 267, 244, 278, 261, 237, 272, 295, 251, 244, 254, 286, 248, 274, 241, 216, 254, 241, 246, 249, 274, 238, 254, 251, 241, 270, 236, 254, 259, 264, 265, 257, 273, 251, 238, 261, 244, 262, 267, 252, 247, 274, 248, 254, 276, 251, 262, 284, 267, 252, 236, 236, 243, 246, 251, 240, 255, 277, 257, 271, 275, 244, 230, 240, 234, 264, 267, 254, 271, 242, 258, 255, 259, 262, 247, 257, 256, 237, 254, 258] payload_rows_per_dst=[5458, 5400, 5422, 5690, 5444, 5472, 5412, 5452] dup_rows_per_dst=[2738, 2642, 2562, 2930, 2662, 2802, 2738, 2712] combine_rows_per_src=[5458, 5400, 5422, 5690, 5444, 5472, 5412, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38418 dup_rows=21732 dup_rows_remote=19020 dedup_ratio=0.331604 router_tokens_per_expert=[256, 271, 247, 248, 249, 238, 272, 240, 285, 252, 253, 239, 233, 242, 267, 236, 259, 240, 271, 246, 261, 266, 259, 278, 263, 271, 245, 244, 238, 260, 258, 240, 259, 253, 256, 247, 258, 266, 262, 259, 249, 266, 281, 271, 249, 263, 269, 274, 250, 268, 236, 244, 244, 262, 281, 245, 266, 280, 262, 287, 248, 253, 244, 259, 239, 288, 261, 247, 255, 280, 259, 239, 222, 251, 280, 239, 233, 233, 282, 241, 249, 246, 270, 260, 281, 260, 217, 276, 256, 254, 258, 266, 247, 242, 261, 267, 233, 260, 239, 247, 236, 253, 254, 251, 245, 270, 232, 259, 251, 253, 258, 229, 276, 263, 281, 253, 282, 256, 260, 237, 268, 266, 277, 270, 257, 247, 261, 247] payload_rows_per_dst=[5442, 5462, 5614, 5510, 5386, 5456, 5430, 5504] dup_rows_per_dst=[2614, 2736, 2750, 2748, 2712, 2764, 2510, 2898] combine_rows_per_src=[5442, 5462, 5614, 5510, 5386, 5456, 5430, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228528 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156794032 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313359536 weight_movement_bytes=37748736 total_nvlink_bytes=351108272 weight_to_token_ratio=0.120465 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229992 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157491816 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314753640 weight_movement_bytes=18874368 total_nvlink_bytes=333628008 weight_to_token_ratio=0.059966 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229656 combine_payload_bytes=157106176 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157335832 ingress_bytes=213729280 link_bytes=213729280 token_bytes=314442008 weight_movement_bytes=56623104 total_nvlink_bytes=371065112 weight_to_token_ratio=0.180075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047848800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38274 dup_rows=21686 dup_rows_remote=18846 dedup_ratio=0.330902 router_tokens_per_expert=[252, 256, 287, 251, 267, 257, 258, 244, 250, 261, 269, 240, 235, 214, 246, 219, 254, 245, 273, 267, 267, 287, 225, 259, 216, 240, 265, 244, 252, 273, 261, 238, 264, 265, 271, 262, 260, 246, 256, 239, 286, 271, 257, 277, 258, 266, 262, 268, 259, 264, 272, 261, 231, 257, 261, 244, 256, 268, 273, 284, 230, 285, 265, 236, 252, 239, 269, 259, 269, 240, 259, 221, 269, 254, 220, 252, 246, 255, 248, 251, 241, 268, 258, 244, 263, 253, 234, 258, 277, 269, 245, 255, 264, 237, 261, 272, 256, 273, 266, 259, 258, 232, 246, 263, 239, 254, 268, 245, 272, 239, 278, 248, 218, 253, 273, 272, 266, 248, 252, 257, 245, 251, 291, 270, 257, 267, 254, 270] payload_rows_per_dst=[5318, 5462, 5576, 5530, 5406, 5582, 5458, 5518] dup_rows_per_dst=[2694, 2670, 2840, 2762, 2600, 2616, 2734, 2770] combine_rows_per_src=[5318, 5462, 5576, 5530, 5406, 5582, 5458, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229536 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156975264 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720992 weight_movement_bytes=18874368 total_nvlink_bytes=332595360 weight_to_token_ratio=0.060163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229488 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098096 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966704 weight_movement_bytes=18874368 total_nvlink_bytes=332841072 weight_to_token_ratio=0.060116 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229752 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589880 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314950008 weight_movement_bytes=18874368 total_nvlink_bytes=333824376 weight_to_token_ratio=0.059928 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38264 dup_rows=21794 dup_rows_remote=19052 dedup_ratio=0.332550 router_tokens_per_expert=[241, 274, 259, 275, 250, 245, 255, 234, 223, 268, 268, 264, 251, 257, 274, 226, 262, 266, 248, 252, 265, 263, 254, 287, 250, 252, 243, 263, 243, 247, 270, 268, 264, 271, 236, 244, 269, 252, 231, 283, 274, 252, 248, 264, 232, 242, 250, 252, 252, 251, 272, 227, 247, 274, 268, 265, 264, 249, 251, 252, 251, 245, 250, 252, 245, 230, 252, 242, 253, 275, 263, 278, 258, 262, 261, 247, 260, 248, 246, 267, 274, 273, 250, 243, 259, 255, 243, 273, 250, 269, 232, 255, 259, 251, 264, 239, 243, 246, 247, 248, 248, 273, 268, 262, 285, 299, 258, 262, 255, 243, 249, 224, 296, 258, 271, 255, 263, 269, 236, 247, 268, 259, 252, 270, 245, 238, 266, 258] payload_rows_per_dst=[5418, 5530, 5452, 5424, 5420, 5462, 5478, 5558] dup_rows_per_dst=[2710, 2736, 2676, 2716, 2754, 2716, 2742, 2744] combine_rows_per_src=[5418, 5530, 5452, 5424, 5420, 5462, 5478, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=228480 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156998784 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769088 weight_movement_bytes=37748736 total_nvlink_bytes=351517824 weight_to_token_ratio=0.120307 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229264 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958608 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313687952 weight_movement_bytes=37748736 total_nvlink_bytes=351436688 weight_to_token_ratio=0.120338 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38364 dup_rows=21630 dup_rows_remote=18878 dedup_ratio=0.330048 router_tokens_per_expert=[248, 254, 264, 268, 281, 285, 256, 245, 264, 254, 245, 273, 249, 260, 242, 251, 247, 271, 254, 245, 290, 239, 234, 244, 262, 264, 272, 271, 296, 253, 262, 255, 252, 233, 233, 231, 250, 255, 239, 269, 241, 242, 263, 251, 258, 270, 245, 258, 227, 267, 252, 254, 271, 264, 272, 257, 254, 245, 261, 255, 255, 258, 239, 237, 234, 223, 251, 256, 245, 250, 246, 254, 268, 257, 282, 247, 255, 267, 255, 245, 266, 285, 260, 263, 273, 234, 277, 233, 249, 274, 262, 250, 245, 247, 245, 248, 276, 259, 251, 260, 250, 278, 226, 246, 280, 249, 252, 255, 253, 251, 265, 252, 255, 263, 266, 256, 235, 283, 280, 279, 265, 254, 260, 261, 264, 222, 255, 265] payload_rows_per_dst=[5542, 5538, 5396, 5484, 5498, 5464, 5502, 5482] dup_rows_per_dst=[2752, 2982, 2584, 2652, 2722, 2594, 2704, 2640] combine_rows_per_src=[5542, 5538, 5396, 5484, 5498, 5464, 5502, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38190 dup_rows=21906 dup_rows_remote=19048 dedup_ratio=0.334259 router_tokens_per_expert=[275, 262, 286, 223, 244, 250, 253, 235, 264, 256, 237, 249, 258, 252, 266, 236, 263, 252, 257, 233, 270, 265, 275, 236, 235, 228, 276, 267, 237, 282, 226, 256, 272, 246, 267, 276, 246, 237, 271, 276, 263, 263, 253, 250, 279, 235, 255, 237, 245, 254, 288, 261, 285, 235, 282, 265, 253, 276, 258, 243, 264, 243, 244, 253, 251, 256, 256, 238, 236, 257, 242, 258, 253, 275, 252, 251, 224, 262, 258, 242, 281, 260, 259, 254, 249, 240, 301, 238, 262, 259, 267, 265, 233, 241, 238, 243, 273, 247, 271, 265, 263, 265, 251, 244, 268, 266, 239, 267, 259, 251, 268, 286, 263, 255, 256, 250, 251, 265, 240, 243, 291, 256, 244, 246, 262, 273, 254, 256] payload_rows_per_dst=[5436, 5446, 5412, 5440, 5392, 5442, 5566, 5496] dup_rows_per_dst=[2656, 2670, 2840, 2858, 2630, 2738, 2800, 2714] combine_rows_per_src=[5436, 5446, 5412, 5440, 5392, 5442, 5566, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38242 dup_rows=21726 dup_rows_remote=19046 dedup_ratio=0.331512 router_tokens_per_expert=[258, 263, 251, 267, 232, 249, 262, 254, 241, 248, 259, 254, 230, 223, 235, 246, 250, 250, 266, 257, 221, 259, 248, 270, 254, 221, 291, 252, 246, 249, 260, 237, 242, 252, 264, 238, 267, 259, 264, 266, 273, 243, 285, 241, 273, 268, 254, 258, 256, 262, 281, 244, 265, 260, 224, 250, 264, 250, 245, 263, 264, 264, 286, 256, 254, 258, 240, 283, 231, 266, 261, 294, 259, 265, 264, 239, 236, 249, 282, 234, 260, 233, 242, 271, 242, 249, 280, 256, 254, 261, 288, 250, 228, 240, 302, 268, 248, 237, 254, 248, 270, 272, 234, 273, 260, 267, 289, 259, 230, 272, 270, 229, 265, 232, 258, 254, 257, 284, 278, 245, 265, 228, 256, 277, 272, 237, 256, 269] payload_rows_per_dst=[5322, 5404, 5544, 5530, 5430, 5568, 5498, 5514] dup_rows_per_dst=[2622, 2658, 2750, 2738, 2800, 2680, 2726, 2752] combine_rows_per_src=[5322, 5404, 5544, 5530, 5430, 5568, 5498, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38160 dup_rows=21832 dup_rows_remote=18862 dedup_ratio=0.333130 router_tokens_per_expert=[284, 271, 241, 266, 253, 257, 282, 238, 249, 243, 221, 249, 256, 256, 252, 275, 275, 255, 257, 286, 235, 262, 293, 255, 251, 276, 254, 272, 267, 293, 253, 273, 249, 257, 256, 241, 245, 240, 262, 257, 246, 264, 260, 242, 250, 251, 258, 264, 229, 272, 234, 214, 257, 255, 255, 257, 270, 279, 248, 257, 234, 260, 226, 258, 251, 275, 224, 264, 246, 244, 259, 261, 251, 254, 257, 253, 244, 236, 270, 244, 257, 268, 279, 283, 267, 239, 248, 237, 259, 280, 226, 264, 288, 262, 242, 261, 242, 268, 271, 252, 259, 285, 253, 226, 270, 286, 270, 266, 237, 228, 288, 249, 229, 239, 293, 250, 243, 242, 240, 225, 245, 292, 243, 229, 250, 284, 253, 271] payload_rows_per_dst=[5436, 5544, 5428, 5440, 5440, 5550, 5478, 5388] dup_rows_per_dst=[2750, 2970, 2656, 2570, 2626, 2770, 2822, 2668] combine_rows_per_src=[5436, 5544, 5428, 5440, 5440, 5550, 5478, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38202 dup_rows=21844 dup_rows_remote=19086 dedup_ratio=0.333313 router_tokens_per_expert=[235, 272, 290, 254, 235, 279, 255, 257, 253, 241, 232, 282, 276, 254, 268, 237, 246, 246, 263, 248, 264, 241, 268, 256, 268, 258, 286, 230, 258, 258, 239, 273, 262, 285, 238, 283, 265, 259, 261, 264, 245, 284, 244, 257, 236, 276, 249, 269, 257, 247, 268, 222, 243, 241, 263, 234, 255, 258, 283, 255, 244, 267, 254, 259, 264, 246, 212, 252, 272, 241, 242, 258, 264, 260, 243, 255, 273, 252, 237, 262, 238, 254, 271, 257, 276, 264, 284, 267, 273, 273, 247, 269, 269, 243, 250, 265, 290, 259, 250, 250, 243, 248, 269, 244, 226, 259, 260, 249, 264, 249, 234, 269, 243, 257, 251, 227, 267, 267, 260, 255, 258, 262, 261, 231, 246, 248, 241, 249] payload_rows_per_dst=[5506, 5488, 5520, 5260, 5520, 5478, 5430, 5490] dup_rows_per_dst=[2734, 2716, 2834, 2444, 2936, 2720, 2702, 2758] combine_rows_per_src=[5506, 5488, 5520, 5260, 5520, 5478, 5430, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38244 dup_rows=21854 dup_rows_remote=19108 dedup_ratio=0.333466 router_tokens_per_expert=[267, 272, 254, 240, 254, 271, 273, 260, 271, 276, 254, 274, 246, 230, 242, 260, 235, 243, 272, 228, 272, 291, 258, 240, 277, 261, 223, 248, 283, 252, 258, 248, 249, 247, 239, 266, 314, 264, 262, 274, 266, 243, 256, 253, 242, 277, 250, 230, 255, 268, 253, 238, 247, 288, 270, 254, 233, 268, 243, 265, 273, 241, 272, 263, 261, 253, 264, 238, 267, 245, 252, 247, 267, 270, 256, 250, 242, 259, 255, 266, 269, 232, 260, 291, 249, 222, 273, 254, 272, 273, 251, 260, 245, 248, 259, 231, 248, 226, 251, 267, 245, 260, 260, 268, 259, 249, 228, 266, 270, 274, 275, 231, 245, 277, 250, 238, 250, 248, 240, 255, 254, 258, 249, 233, 262, 252, 237, 266] payload_rows_per_dst=[5524, 5432, 5474, 5538, 5438, 5484, 5430, 5362] dup_rows_per_dst=[2764, 2746, 2790, 2724, 2746, 2694, 2724, 2666] combine_rows_per_src=[5524, 5432, 5474, 5538, 5438, 5484, 5430, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=228968 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157367912 ingress_bytes=232636416 link_bytes=232636416 token_bytes=314506856 weight_movement_bytes=75497472 total_nvlink_bytes=390004328 weight_to_token_ratio=0.240050 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229007674e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38376 dup_rows=21714 dup_rows_remote=19056 dedup_ratio=0.331329 router_tokens_per_expert=[250, 246, 251, 278, 248, 244, 252, 239, 254, 246, 259, 254, 240, 255, 240, 253, 257, 266, 250, 251, 234, 239, 250, 267, 249, 245, 265, 261, 255, 255, 253, 253, 261, 254, 285, 235, 249, 252, 258, 263, 227, 247, 237, 248, 266, 246, 255, 269, 254, 263, 273, 228, 260, 257, 292, 268, 262, 253, 244, 275, 279, 262, 293, 247, 239, 266, 274, 271, 214, 250, 244, 248, 243, 284, 244, 225, 261, 267, 287, 253, 259, 261, 254, 256, 250, 266, 258, 238, 249, 274, 247, 245, 249, 280, 259, 276, 265, 267, 211, 285, 273, 255, 287, 242, 268, 252, 255, 277, 273, 261, 255, 247, 239, 261, 263, 253, 254, 249, 228, 264, 257, 278, 249, 255, 265, 262, 256, 250] payload_rows_per_dst=[5440, 5474, 5446, 5536, 5444, 5500, 5506, 5476] dup_rows_per_dst=[2578, 2626, 2658, 2884, 2696, 2742, 2840, 2690] combine_rows_per_src=[5440, 5474, 5446, 5536, 5444, 5500, 5506, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228952 combine_payload_bytes=156426240 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156655192 ingress_bytes=213049344 link_bytes=213049344 token_bytes=313081432 weight_movement_bytes=56623104 total_nvlink_bytes=369704536 weight_to_token_ratio=0.180857 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041333987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229152 combine_payload_bytes=156639232 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156868384 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313507616 weight_movement_bytes=56623104 total_nvlink_bytes=370130720 weight_to_token_ratio=0.180612 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228088 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156531448 ingress_bytes=212926464 link_bytes=212926464 token_bytes=312834808 weight_movement_bytes=56623104 total_nvlink_bytes=369457912 weight_to_token_ratio=0.181000 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040156612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38102 dup_rows=21904 dup_rows_remote=19226 dedup_ratio=0.334229 router_tokens_per_expert=[230, 251, 259, 245, 259, 276, 239, 280, 257, 245, 276, 268, 249, 240, 262, 270, 266, 272, 262, 236, 271, 249, 268, 232, 250, 267, 230, 247, 274, 263, 278, 274, 278, 241, 275, 250, 243, 274, 225, 226, 273, 239, 252, 254, 262, 249, 243, 256, 265, 261, 275, 252, 261, 278, 240, 241, 268, 238, 269, 290, 271, 237, 240, 258, 257, 249, 254, 267, 272, 227, 262, 252, 270, 258, 260, 242, 240, 236, 248, 238, 248, 267, 260, 248, 258, 254, 269, 253, 248, 241, 233, 211, 259, 259, 258, 244, 266, 281, 242, 258, 270, 231, 255, 261, 238, 253, 254, 267, 260, 254, 254, 260, 278, 249, 251, 252, 253, 292, 263, 272, 222, 278, 246, 258, 285, 292, 245, 257] payload_rows_per_dst=[5482, 5486, 5416, 5490, 5336, 5356, 5530, 5536] dup_rows_per_dst=[2730, 2792, 2664, 2798, 2728, 2664, 2678, 2850] combine_rows_per_src=[5482, 5486, 5416, 5490, 5336, 5356, 5530, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229152 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704544 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179936 weight_movement_bytes=18874368 total_nvlink_bytes=332054304 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229408 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876832 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524256 weight_movement_bytes=18874368 total_nvlink_bytes=332398624 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229728 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417824 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314605920 weight_movement_bytes=18874368 total_nvlink_bytes=333480288 weight_to_token_ratio=0.059994 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229312 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156295104 ingress_bytes=212688896 link_bytes=212688896 token_bytes=312360896 weight_movement_bytes=56623104 total_nvlink_bytes=368984000 weight_to_token_ratio=0.181275 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037880352e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38104 dup_rows=21878 dup_rows_remote=19054 dedup_ratio=0.333832 router_tokens_per_expert=[241, 229, 250, 243, 261, 258, 259, 248, 284, 213, 227, 252, 242, 277, 238, 246, 262, 242, 270, 268, 260, 240, 262, 249, 264, 267, 269, 278, 258, 281, 212, 226, 258, 275, 261, 247, 245, 252, 280, 248, 257, 238, 249, 264, 242, 250, 253, 265, 262, 248, 234, 276, 244, 258, 249, 253, 282, 287, 249, 251, 259, 280, 253, 236, 249, 269, 261, 255, 261, 257, 265, 256, 226, 269, 233, 273, 257, 269, 261, 247, 257, 269, 263, 240, 226, 245, 270, 254, 245, 275, 275, 276, 250, 269, 269, 258, 244, 264, 275, 249, 268, 255, 296, 257, 247, 233, 236, 262, 250, 227, 257, 257, 277, 268, 281, 260, 264, 261, 251, 269, 242, 241, 229, 272, 268, 255, 268, 255] payload_rows_per_dst=[5554, 5434, 5422, 5304, 5528, 5482, 5500, 5434] dup_rows_per_dst=[2824, 2782, 2756, 2496, 2886, 2816, 2782, 2536] combine_rows_per_src=[5554, 5434, 5422, 5304, 5528, 5482, 5500, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38440 dup_rows=21650 dup_rows_remote=18974 dedup_ratio=0.330353 router_tokens_per_expert=[270, 251, 242, 243, 263, 267, 268, 279, 285, 239, 250, 256, 259, 255, 274, 258, 265, 272, 248, 258, 238, 274, 268, 236, 254, 258, 257, 264, 254, 284, 246, 262, 255, 257, 274, 245, 262, 235, 228, 263, 270, 234, 238, 253, 257, 270, 260, 248, 235, 243, 272, 244, 255, 264, 240, 261, 231, 251, 261, 270, 262, 245, 255, 272, 256, 258, 254, 266, 258, 249, 275, 247, 234, 225, 274, 237, 245, 250, 252, 255, 274, 250, 241, 252, 260, 232, 223, 283, 246, 273, 245, 283, 254, 261, 249, 237, 259, 268, 270, 262, 253, 264, 272, 273, 245, 267, 239, 249, 265, 251, 280, 274, 239, 242, 222, 266, 250, 249, 286, 266, 261, 257, 254, 245, 246, 280, 253, 256] payload_rows_per_dst=[5580, 5490, 5422, 5446, 5492, 5466, 5560, 5430] dup_rows_per_dst=[2738, 2786, 2676, 2676, 2578, 2660, 2822, 2714] combine_rows_per_src=[5580, 5490, 5422, 5446, 5492, 5466, 5560, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38292 dup_rows=21792 dup_rows_remote=19112 dedup_ratio=0.332520 router_tokens_per_expert=[252, 255, 259, 262, 250, 267, 260, 238, 230, 243, 263, 258, 265, 278, 247, 231, 283, 237, 233, 270, 245, 233, 297, 274, 256, 253, 267, 242, 240, 262, 253, 272, 245, 300, 237, 281, 272, 248, 270, 268, 263, 249, 228, 247, 252, 253, 260, 240, 276, 251, 312, 240, 254, 259, 256, 290, 267, 237, 232, 270, 255, 256, 258, 280, 276, 286, 247, 253, 259, 258, 223, 277, 273, 255, 257, 244, 286, 252, 257, 255, 268, 261, 252, 233, 236, 269, 263, 234, 252, 282, 269, 245, 249, 263, 248, 242, 236, 255, 260, 227, 274, 247, 240, 270, 242, 261, 268, 269, 226, 265, 239, 249, 235, 254, 270, 262, 258, 239, 264, 222, 262, 268, 274, 253, 252, 249, 262, 211] payload_rows_per_dst=[5464, 5462, 5506, 5526, 5534, 5452, 5382, 5418] dup_rows_per_dst=[2652, 2772, 2720, 2860, 2782, 2680, 2674, 2652] combine_rows_per_src=[5464, 5462, 5506, 5526, 5534, 5452, 5382, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38354 dup_rows=21704 dup_rows_remote=19094 dedup_ratio=0.331177 router_tokens_per_expert=[241, 243, 237, 270, 258, 276, 235, 278, 239, 241, 253, 258, 259, 271, 243, 252, 244, 259, 261, 258, 270, 207, 247, 246, 266, 254, 279, 234, 261, 263, 252, 243, 245, 230, 284, 271, 247, 254, 263, 216, 235, 228, 249, 275, 268, 257, 259, 264, 254, 247, 256, 259, 255, 270, 249, 251, 254, 245, 252, 251, 284, 269, 259, 262, 253, 269, 267, 260, 288, 246, 257, 254, 255, 267, 270, 238, 257, 216, 243, 256, 242, 269, 260, 238, 273, 264, 268, 264, 287, 255, 283, 225, 254, 248, 250, 257, 253, 248, 236, 264, 240, 249, 285, 254, 241, 275, 268, 279, 260, 266, 301, 272, 261, 285, 279, 268, 231, 253, 227, 244, 250, 264, 265, 252, 244, 252, 271, 238] payload_rows_per_dst=[5440, 5478, 5466, 5482, 5488, 5472, 5556, 5450] dup_rows_per_dst=[2668, 2610, 2624, 2752, 2704, 2802, 2826, 2718] combine_rows_per_src=[5440, 5478, 5466, 5482, 5488, 5472, 5556, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38260 dup_rows=21762 dup_rows_remote=19036 dedup_ratio=0.332062 router_tokens_per_expert=[240, 257, 259, 264, 281, 240, 277, 276, 282, 235, 241, 248, 230, 271, 280, 251, 257, 249, 259, 243, 277, 251, 239, 267, 231, 262, 267, 242, 293, 268, 263, 261, 241, 242, 248, 280, 257, 285, 262, 249, 266, 232, 235, 289, 260, 246, 265, 286, 283, 241, 253, 254, 246, 272, 277, 251, 256, 237, 266, 233, 236, 273, 221, 263, 230, 258, 266, 256, 273, 275, 243, 265, 286, 235, 261, 252, 263, 274, 229, 254, 282, 230, 252, 253, 249, 261, 241, 254, 243, 254, 223, 262, 257, 259, 259, 254, 243, 245, 249, 266, 259, 255, 273, 249, 281, 237, 260, 255, 224, 273, 245, 271, 241, 252, 232, 288, 295, 251, 246, 244, 285, 214, 221, 245, 255, 268, 265, 262] payload_rows_per_dst=[5502, 5496, 5458, 5468, 5514, 5412, 5454, 5470] dup_rows_per_dst=[2762, 2762, 2828, 2656, 2726, 2654, 2716, 2658] combine_rows_per_src=[5502, 5496, 5458, 5468, 5514, 5412, 5454, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38208 dup_rows=21734 dup_rows_remote=18880 dedup_ratio=0.331635 router_tokens_per_expert=[262, 249, 245, 241, 262, 268, 247, 284, 255, 235, 256, 258, 238, 228, 267, 265, 258, 278, 264, 271, 244, 245, 242, 257, 244, 278, 253, 254, 230, 245, 246, 250, 253, 266, 256, 252, 261, 272, 257, 237, 276, 263, 282, 244, 244, 272, 254, 244, 234, 216, 261, 275, 276, 241, 260, 254, 272, 250, 255, 243, 261, 243, 262, 257, 270, 259, 270, 244, 221, 282, 238, 231, 240, 259, 262, 259, 286, 245, 245, 252, 270, 268, 249, 256, 235, 231, 259, 239, 230, 263, 253, 269, 278, 248, 294, 250, 271, 229, 269, 242, 243, 284, 261, 263, 245, 261, 254, 250, 255, 261, 267, 266, 261, 239, 291, 257, 260, 251, 269, 269, 248, 266, 253, 273, 276, 257, 238, 272] payload_rows_per_dst=[5236, 5566, 5448, 5426, 5458, 5584, 5490, 5594] dup_rows_per_dst=[2422, 2746, 2818, 2582, 2668, 2868, 2776, 2854] combine_rows_per_src=[5236, 5566, 5448, 5426, 5458, 5584, 5490, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=228632 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156302616 ingress_bytes=174948352 link_bytes=174948352 token_bytes=312376600 weight_movement_bytes=18874368 total_nvlink_bytes=331250968 weight_to_token_ratio=0.060422 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676269029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38326 dup_rows=21730 dup_rows_remote=18978 dedup_ratio=0.331573 router_tokens_per_expert=[274, 236, 261, 241, 252, 223, 271, 248, 260, 278, 260, 249, 218, 258, 262, 270, 257, 259, 261, 271, 251, 256, 266, 247, 255, 255, 273, 272, 246, 248, 274, 259, 251, 268, 267, 269, 244, 239, 261, 274, 241, 270, 268, 240, 276, 266, 221, 261, 261, 254, 260, 245, 281, 263, 239, 228, 271, 238, 224, 235, 258, 276, 251, 240, 268, 272, 259, 255, 249, 245, 268, 242, 237, 284, 260, 246, 271, 257, 263, 246, 264, 267, 253, 261, 241, 262, 252, 255, 230, 229, 242, 227, 289, 258, 243, 249, 276, 244, 250, 232, 254, 251, 233, 256, 261, 283, 274, 265, 277, 272, 260, 255, 237, 247, 246, 252, 243, 253, 260, 280, 247, 271, 281, 277, 264, 272, 252, 248] payload_rows_per_dst=[5420, 5540, 5480, 5414, 5548, 5434, 5492, 5478] dup_rows_per_dst=[2702, 2760, 2752, 2634, 2696, 2610, 2794, 2782] combine_rows_per_src=[5420, 5540, 5480, 5414, 5548, 5434, 5492, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157450240 dispatch_meta_bytes=229656 combine_payload_bytes=157450240 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157679896 ingress_bytes=195198976 link_bytes=195198976 token_bytes=315130136 weight_movement_bytes=37748736 total_nvlink_bytes=352878872 weight_to_token_ratio=0.119788 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870300544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38336 dup_rows=21640 dup_rows_remote=18972 dedup_ratio=0.330200 router_tokens_per_expert=[259, 270, 236, 257, 261, 253, 252, 260, 243, 264, 286, 247, 254, 268, 280, 252, 291, 273, 259, 262, 238, 275, 269, 243, 266, 239, 257, 265, 237, 246, 274, 250, 230, 262, 250, 269, 248, 245, 260, 250, 246, 253, 249, 252, 245, 256, 244, 263, 265, 284, 258, 265, 276, 248, 238, 265, 249, 221, 252, 259, 240, 237, 238, 254, 261, 226, 259, 247, 239, 247, 278, 273, 254, 263, 246, 248, 266, 259, 259, 261, 236, 262, 252, 260, 276, 244, 281, 268, 256, 256, 229, 227, 268, 229, 252, 270, 254, 246, 286, 237, 275, 231, 245, 253, 263, 229, 271, 263, 275, 259, 257, 270, 269, 257, 251, 251, 262, 298, 268, 236, 264, 261, 230, 240, 272, 265, 255, 266] payload_rows_per_dst=[5474, 5504, 5466, 5458, 5500, 5420, 5560, 5514] dup_rows_per_dst=[2810, 2784, 2578, 2640, 2672, 2712, 2668, 2776] combine_rows_per_src=[5474, 5504, 5466, 5458, 5500, 5420, 5560, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229616 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157073648 ingress_bytes=194592768 link_bytes=194592768 token_bytes=313917680 weight_movement_bytes=37748736 total_nvlink_bytes=351666416 weight_to_token_ratio=0.120250 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864492157e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229184 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942144 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655104 weight_movement_bytes=18874368 total_nvlink_bytes=332529472 weight_to_token_ratio=0.060176 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229792 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327776 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425760 weight_movement_bytes=18874368 total_nvlink_bytes=333300128 weight_to_token_ratio=0.060028 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228352 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156728320 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313228288 weight_movement_bytes=37748736 total_nvlink_bytes=350977024 weight_to_token_ratio=0.120515 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229216 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157212512 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314195808 weight_movement_bytes=18874368 total_nvlink_bytes=333070176 weight_to_token_ratio=0.060072 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229232 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157253488 ingress_bytes=213647360 link_bytes=213647360 token_bytes=314277744 weight_movement_bytes=56623104 total_nvlink_bytes=370900848 weight_to_token_ratio=0.180169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047063882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38314 dup_rows=21720 dup_rows_remote=19068 dedup_ratio=0.331421 router_tokens_per_expert=[262, 243, 231, 258, 220, 262, 240, 261, 246, 261, 243, 240, 265, 282, 267, 296, 239, 258, 241, 256, 254, 279, 268, 266, 277, 262, 258, 271, 256, 236, 265, 263, 287, 255, 262, 246, 242, 224, 258, 262, 229, 219, 256, 233, 255, 245, 258, 272, 248, 263, 258, 247, 253, 243, 257, 229, 242, 251, 236, 253, 252, 249, 267, 267, 249, 253, 294, 245, 237, 261, 252, 260, 230, 250, 242, 270, 267, 282, 252, 266, 250, 255, 287, 263, 279, 259, 245, 261, 253, 270, 246, 276, 250, 263, 231, 246, 262, 278, 277, 264, 272, 246, 228, 264, 253, 237, 255, 244, 264, 270, 258, 251, 262, 240, 251, 259, 269, 262, 243, 273, 253, 274, 271, 284, 293, 268, 221, 234] payload_rows_per_dst=[5502, 5578, 5408, 5340, 5568, 5480, 5416, 5524] dup_rows_per_dst=[2652, 2740, 2598, 2690, 2792, 2768, 2690, 2790] combine_rows_per_src=[5502, 5578, 5408, 5340, 5568, 5480, 5416, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38242 dup_rows=21816 dup_rows_remote=19032 dedup_ratio=0.332886 router_tokens_per_expert=[265, 242, 275, 284, 241, 249, 270, 254, 265, 247, 248, 251, 230, 267, 247, 238, 245, 249, 249, 271, 268, 241, 257, 263, 250, 239, 262, 272, 239, 266, 282, 235, 244, 284, 243, 220, 263, 263, 227, 262, 260, 284, 263, 246, 281, 237, 273, 241, 266, 249, 256, 282, 250, 273, 291, 260, 246, 230, 235, 241, 269, 254, 257, 272, 243, 285, 257, 242, 229, 286, 236, 236, 263, 252, 280, 281, 265, 261, 268, 250, 266, 263, 226, 274, 243, 256, 256, 280, 221, 260, 234, 265, 240, 249, 269, 260, 247, 250, 277, 267, 268, 238, 256, 247, 277, 242, 247, 222, 266, 256, 252, 239, 256, 277, 246, 241, 262, 287, 250, 250, 268, 239, 279, 266, 240, 238, 286, 253] payload_rows_per_dst=[5440, 5440, 5434, 5478, 5500, 5420, 5462, 5546] dup_rows_per_dst=[2706, 2736, 2748, 2784, 2768, 2704, 2640, 2730] combine_rows_per_src=[5440, 5440, 5434, 5478, 5500, 5420, 5462, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38270 dup_rows=21798 dup_rows_remote=19050 dedup_ratio=0.332611 router_tokens_per_expert=[265, 279, 266, 223, 249, 269, 241, 269, 268, 266, 296, 259, 273, 239, 257, 285, 254, 271, 268, 270, 272, 235, 248, 238, 264, 249, 250, 251, 266, 257, 276, 253, 265, 270, 239, 226, 254, 254, 238, 262, 247, 254, 243, 263, 248, 245, 253, 249, 217, 262, 254, 241, 266, 272, 239, 244, 274, 264, 246, 234, 243, 268, 293, 258, 251, 269, 279, 253, 257, 250, 242, 229, 255, 238, 254, 262, 231, 260, 264, 260, 244, 251, 272, 248, 261, 271, 253, 247, 259, 248, 265, 270, 255, 273, 256, 235, 272, 282, 247, 237, 249, 242, 226, 264, 293, 248, 259, 245, 255, 279, 258, 252, 258, 251, 260, 261, 253, 256, 250, 263, 239, 241, 270, 245, 267, 263, 249, 261] payload_rows_per_dst=[5522, 5462, 5414, 5472, 5420, 5468, 5514, 5466] dup_rows_per_dst=[2886, 2782, 2606, 2678, 2688, 2748, 2702, 2708] combine_rows_per_src=[5522, 5462, 5414, 5472, 5420, 5468, 5514, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38294 dup_rows=21744 dup_rows_remote=19016 dedup_ratio=0.331787 router_tokens_per_expert=[284, 276, 241, 219, 263, 283, 245, 251, 251, 251, 256, 263, 271, 259, 290, 248, 230, 266, 250, 247, 280, 243, 260, 246, 273, 272, 261, 257, 260, 243, 227, 264, 226, 251, 271, 274, 272, 241, 232, 279, 260, 229, 269, 258, 215, 250, 252, 220, 286, 237, 262, 233, 234, 258, 268, 241, 244, 276, 260, 266, 275, 244, 251, 264, 271, 263, 265, 218, 296, 262, 278, 231, 276, 261, 211, 247, 251, 275, 254, 254, 270, 236, 259, 248, 270, 256, 240, 272, 274, 229, 246, 254, 264, 286, 255, 249, 240, 295, 262, 273, 231, 236, 262, 279, 250, 249, 260, 296, 285, 262, 246, 277, 266, 222, 252, 252, 228, 259, 266, 229, 262, 251, 260, 254, 247, 258, 253, 257] payload_rows_per_dst=[5524, 5496, 5346, 5512, 5498, 5488, 5542, 5386] dup_rows_per_dst=[2778, 2662, 2652, 2686, 2728, 2728, 2864, 2646] combine_rows_per_src=[5524, 5496, 5346, 5512, 5498, 5488, 5542, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38378 dup_rows=21760 dup_rows_remote=19134 dedup_ratio=0.332031 router_tokens_per_expert=[260, 268, 248, 264, 277, 256, 254, 271, 281, 246, 259, 286, 265, 234, 268, 270, 256, 268, 262, 228, 225, 238, 229, 235, 263, 267, 251, 255, 277, 250, 236, 272, 246, 230, 254, 252, 243, 253, 264, 252, 245, 227, 261, 241, 255, 245, 243, 250, 239, 282, 256, 261, 300, 246, 256, 281, 262, 225, 280, 235, 235, 273, 254, 218, 256, 262, 268, 267, 252, 246, 240, 255, 261, 252, 252, 252, 248, 235, 269, 241, 252, 252, 273, 250, 247, 257, 275, 240, 242, 238, 284, 235, 258, 262, 291, 269, 238, 278, 272, 250, 265, 291, 255, 240, 277, 282, 256, 228, 253, 253, 271, 263, 245, 252, 243, 262, 239, 281, 265, 284, 254, 265, 252, 265, 259, 276, 231, 259] payload_rows_per_dst=[5562, 5398, 5398, 5444, 5408, 5506, 5488, 5572] dup_rows_per_dst=[2852, 2626, 2616, 2762, 2704, 2744, 2764, 2692] combine_rows_per_src=[5562, 5398, 5398, 5444, 5408, 5506, 5488, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229528 combine_payload_bytes=156934144 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157163672 ingress_bytes=213557248 link_bytes=213557248 token_bytes=314097816 weight_movement_bytes=56623104 total_nvlink_bytes=370720920 weight_to_token_ratio=0.180272 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046200474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38294 dup_rows=21748 dup_rows_remote=19010 dedup_ratio=0.331848 router_tokens_per_expert=[240, 230, 233, 263, 217, 263, 252, 266, 253, 287, 248, 251, 264, 224, 260, 279, 279, 267, 263, 269, 255, 253, 258, 285, 234, 275, 284, 290, 210, 237, 236, 247, 273, 240, 253, 256, 251, 229, 240, 255, 251, 241, 252, 235, 281, 236, 260, 258, 259, 245, 270, 250, 270, 258, 269, 268, 257, 248, 238, 234, 281, 288, 264, 286, 266, 276, 289, 265, 263, 262, 271, 272, 251, 243, 251, 256, 259, 250, 278, 246, 266, 251, 245, 258, 236, 257, 261, 247, 256, 253, 249, 272, 238, 253, 257, 267, 270, 226, 253, 248, 251, 259, 258, 244, 271, 253, 268, 243, 292, 254, 236, 225, 264, 250, 274, 256, 251, 228, 259, 269, 241, 259, 239, 273, 252, 248, 266, 256] payload_rows_per_dst=[5334, 5520, 5402, 5504, 5520, 5434, 5580, 5494] dup_rows_per_dst=[2480, 2764, 2620, 2866, 2814, 2698, 2768, 2738] combine_rows_per_src=[5334, 5520, 5402, 5504, 5520, 5434, 5580, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38246 dup_rows=21752 dup_rows_remote=19108 dedup_ratio=0.331909 router_tokens_per_expert=[274, 256, 276, 285, 255, 266, 271, 265, 249, 269, 289, 245, 263, 242, 238, 272, 254, 242, 225, 245, 238, 248, 282, 254, 254, 230, 259, 255, 266, 284, 285, 276, 254, 251, 256, 275, 267, 257, 248, 248, 254, 247, 245, 255, 240, 244, 268, 282, 263, 251, 254, 250, 248, 257, 266, 255, 283, 275, 231, 272, 277, 224, 259, 277, 245, 265, 232, 253, 252, 255, 256, 287, 268, 261, 233, 275, 220, 229, 250, 270, 268, 283, 260, 240, 265, 218, 247, 257, 248, 263, 243, 254, 267, 238, 239, 268, 264, 253, 251, 242, 251, 262, 233, 257, 237, 268, 256, 236, 258, 250, 236, 233, 276, 269, 232, 258, 249, 265, 247, 269, 274, 256, 275, 258, 236, 267, 237, 259] payload_rows_per_dst=[5582, 5434, 5538, 5474, 5450, 5440, 5392, 5474] dup_rows_per_dst=[2848, 2760, 2644, 2810, 2652, 2676, 2582, 2780] combine_rows_per_src=[5582, 5434, 5538, 5474, 5450, 5440, 5392, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229096 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156868328 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313507560 weight_movement_bytes=37748736 total_nvlink_bytes=351256296 weight_to_token_ratio=0.120408 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38286 dup_rows=21814 dup_rows_remote=19026 dedup_ratio=0.332855 router_tokens_per_expert=[247, 264, 267, 260, 245, 239, 265, 243, 262, 283, 274, 253, 257, 251, 250, 286, 255, 248, 239, 249, 276, 269, 258, 251, 238, 255, 248, 254, 291, 254, 227, 249, 228, 280, 278, 248, 251, 259, 266, 243, 249, 276, 277, 225, 254, 250, 259, 238, 264, 261, 272, 233, 278, 260, 267, 267, 253, 281, 266, 262, 253, 263, 238, 271, 263, 234, 247, 251, 265, 238, 258, 252, 274, 270, 267, 255, 282, 243, 253, 249, 246, 259, 251, 249, 258, 238, 253, 267, 250, 254, 247, 229, 238, 267, 245, 249, 261, 268, 262, 276, 240, 255, 247, 254, 247, 269, 274, 253, 278, 236, 243, 249, 254, 237, 276, 231, 257, 269, 232, 277, 272, 248, 242, 270, 235, 265, 246, 267] payload_rows_per_dst=[5520, 5422, 5490, 5540, 5480, 5388, 5436, 5446] dup_rows_per_dst=[2772, 2700, 2672, 2838, 2722, 2612, 2788, 2710] combine_rows_per_src=[5520, 5422, 5490, 5540, 5480, 5388, 5436, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229280 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983200 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737120 weight_movement_bytes=18874368 total_nvlink_bytes=332611488 weight_to_token_ratio=0.060160 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229240 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157081464 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313933688 weight_movement_bytes=56623104 total_nvlink_bytes=370556792 weight_to_token_ratio=0.180366 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=230048 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157426336 ingress_bytes=176070656 link_bytes=176070656 token_bytes=314622624 weight_movement_bytes=18874368 total_nvlink_bytes=333496992 weight_to_token_ratio=0.059990 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687022394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229216 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081440 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933664 weight_movement_bytes=37748736 total_nvlink_bytes=351682400 weight_to_token_ratio=0.120244 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229416 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156885032 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540648 weight_movement_bytes=37748736 total_nvlink_bytes=351289384 weight_to_token_ratio=0.120395 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229248 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157048704 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868160 weight_movement_bytes=37748736 total_nvlink_bytes=351616896 weight_to_token_ratio=0.120269 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38326 dup_rows=21800 dup_rows_remote=19204 dedup_ratio=0.332642 router_tokens_per_expert=[239, 243, 249, 256, 225, 243, 248, 255, 275, 246, 260, 271, 247, 238, 264, 241, 249, 259, 264, 226, 222, 259, 264, 270, 249, 269, 239, 245, 222, 296, 267, 248, 270, 263, 247, 279, 280, 244, 243, 251, 254, 259, 239, 278, 248, 254, 273, 257, 249, 251, 264, 236, 261, 271, 243, 240, 272, 257, 246, 268, 254, 232, 250, 249, 260, 250, 259, 271, 244, 257, 237, 211, 262, 275, 267, 289, 276, 263, 249, 227, 247, 274, 263, 240, 256, 253, 255, 267, 240, 255, 261, 265, 239, 241, 254, 284, 272, 241, 265, 278, 259, 277, 266, 255, 274, 262, 269, 244, 310, 248, 248, 246, 242, 273, 235, 281, 268, 254, 257, 244, 295, 237, 259, 258, 253, 272, 244, 261] payload_rows_per_dst=[5410, 5498, 5462, 5338, 5484, 5478, 5544, 5522] dup_rows_per_dst=[2596, 2820, 2836, 2526, 2710, 2684, 2884, 2744] combine_rows_per_src=[5410, 5498, 5462, 5338, 5484, 5478, 5544, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38122 dup_rows=21872 dup_rows_remote=19154 dedup_ratio=0.333740 router_tokens_per_expert=[220, 254, 245, 262, 266, 261, 286, 287, 268, 254, 283, 242, 281, 245, 253, 264, 260, 261, 262, 233, 248, 264, 236, 261, 265, 258, 250, 267, 252, 272, 261, 239, 247, 252, 265, 249, 292, 266, 274, 236, 234, 224, 245, 265, 250, 245, 245, 259, 245, 269, 257, 278, 265, 254, 258, 261, 234, 262, 255, 217, 280, 272, 258, 265, 236, 251, 244, 256, 268, 275, 253, 247, 257, 255, 249, 248, 243, 272, 290, 274, 254, 250, 251, 203, 257, 259, 251, 261, 249, 231, 263, 259, 244, 267, 251, 258, 272, 218, 263, 234, 262, 256, 249, 229, 245, 282, 267, 286, 274, 246, 262, 254, 272, 252, 249, 238, 271, 258, 246, 270, 256, 267, 288, 253, 261, 224, 260, 240] payload_rows_per_dst=[5496, 5418, 5428, 5542, 5460, 5390, 5394, 5536] dup_rows_per_dst=[2846, 2760, 2668, 2718, 2776, 2626, 2804, 2674] combine_rows_per_src=[5496, 5418, 5428, 5542, 5460, 5390, 5394, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38238 dup_rows=21786 dup_rows_remote=19056 dedup_ratio=0.332428 router_tokens_per_expert=[265, 254, 273, 269, 260, 276, 251, 256, 241, 291, 245, 241, 253, 239, 247, 266, 237, 257, 226, 290, 264, 246, 228, 254, 233, 255, 245, 261, 269, 279, 271, 279, 250, 257, 238, 281, 281, 253, 231, 263, 243, 231, 263, 263, 268, 261, 252, 249, 253, 253, 242, 252, 259, 242, 252, 236, 267, 251, 256, 254, 287, 275, 265, 265, 238, 260, 260, 243, 251, 266, 241, 234, 207, 270, 270, 260, 236, 265, 233, 267, 248, 259, 261, 231, 246, 261, 257, 264, 268, 268, 260, 242, 267, 254, 282, 253, 236, 246, 270, 246, 255, 257, 260, 278, 233, 254, 229, 248, 221, 259, 242, 277, 251, 275, 262, 256, 290, 262, 292, 255, 258, 225, 273, 267, 265, 265, 256, 269] payload_rows_per_dst=[5508, 5480, 5458, 5462, 5372, 5512, 5414, 5544] dup_rows_per_dst=[2746, 2708, 2710, 2756, 2630, 2730, 2608, 2898] combine_rows_per_src=[5508, 5480, 5458, 5462, 5372, 5512, 5414, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38216 dup_rows=21862 dup_rows_remote=19130 dedup_ratio=0.333588 router_tokens_per_expert=[239, 249, 247, 220, 265, 273, 241, 265, 271, 236, 265, 253, 246, 266, 263, 263, 244, 280, 249, 246, 233, 231, 228, 285, 256, 251, 258, 246, 246, 243, 254, 243, 261, 258, 266, 242, 273, 253, 253, 265, 253, 266, 238, 238, 236, 276, 261, 254, 255, 266, 212, 264, 257, 247, 234, 242, 265, 258, 259, 265, 265, 240, 254, 257, 244, 252, 276, 276, 255, 238, 249, 261, 259, 249, 268, 246, 247, 254, 258, 263, 255, 233, 283, 299, 253, 291, 257, 283, 282, 252, 254, 253, 261, 262, 250, 234, 259, 262, 257, 248, 269, 277, 238, 248, 268, 264, 253, 280, 266, 235, 233, 240, 264, 288, 255, 281, 240, 279, 258, 271, 270, 229, 249, 253, 271, 251, 281, 246] payload_rows_per_dst=[5386, 5330, 5486, 5396, 5458, 5584, 5504, 5530] dup_rows_per_dst=[2738, 2656, 2700, 2684, 2732, 2820, 2690, 2842] combine_rows_per_src=[5386, 5330, 5486, 5396, 5458, 5584, 5504, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38304 dup_rows=21742 dup_rows_remote=18988 dedup_ratio=0.331757 router_tokens_per_expert=[244, 247, 264, 241, 269, 275, 243, 264, 303, 271, 238, 255, 239, 244, 233, 257, 282, 257, 255, 256, 231, 266, 261, 287, 266, 259, 235, 263, 250, 254, 242, 258, 244, 267, 267, 264, 251, 258, 234, 233, 244, 265, 252, 270, 260, 260, 225, 280, 271, 269, 226, 243, 265, 250, 246, 258, 269, 262, 292, 234, 248, 244, 230, 248, 254, 283, 247, 233, 237, 266, 287, 270, 250, 258, 256, 266, 278, 263, 227, 265, 256, 246, 241, 270, 242, 259, 245, 239, 262, 266, 263, 265, 249, 266, 276, 247, 258, 223, 286, 295, 240, 272, 256, 249, 245, 260, 256, 265, 218, 219, 248, 270, 262, 275, 253, 260, 233, 237, 259, 254, 288, 254, 253, 238, 276, 274, 260, 262] payload_rows_per_dst=[5522, 5490, 5458, 5424, 5516, 5476, 5408, 5500] dup_rows_per_dst=[2652, 2754, 2690, 2686, 2764, 2708, 2712, 2776] combine_rows_per_src=[5522, 5490, 5458, 5424, 5516, 5476, 5408, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38258 dup_rows=21790 dup_rows_remote=19172 dedup_ratio=0.332489 router_tokens_per_expert=[254, 284, 261, 235, 267, 228, 256, 255, 250, 257, 245, 280, 268, 255, 289, 247, 254, 233, 277, 240, 278, 248, 267, 239, 281, 280, 286, 257, 246, 229, 255, 262, 252, 234, 240, 260, 266, 246, 278, 261, 262, 250, 234, 252, 254, 235, 270, 259, 255, 275, 251, 241, 247, 245, 242, 278, 265, 235, 244, 258, 261, 254, 246, 234, 236, 262, 220, 264, 247, 260, 276, 250, 234, 249, 263, 255, 245, 237, 271, 210, 293, 249, 256, 264, 236, 238, 247, 266, 266, 269, 257, 249, 279, 271, 278, 265, 248, 260, 287, 267, 288, 254, 272, 261, 259, 260, 263, 262, 261, 267, 269, 247, 257, 269, 265, 258, 239, 247, 239, 250, 250, 257, 252, 235, 229, 247, 281, 259] payload_rows_per_dst=[5536, 5504, 5488, 5434, 5352, 5528, 5548, 5356] dup_rows_per_dst=[2726, 2760, 2618, 2628, 2606, 2838, 2902, 2712] combine_rows_per_src=[5536, 5504, 5488, 5434, 5352, 5528, 5548, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=230120 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157213416 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314196712 weight_movement_bytes=37748736 total_nvlink_bytes=351945448 weight_to_token_ratio=0.120144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38330 dup_rows=21796 dup_rows_remote=19124 dedup_ratio=0.332581 router_tokens_per_expert=[250, 265, 252, 259, 245, 262, 290, 244, 247, 256, 262, 252, 256, 232, 248, 281, 281, 240, 290, 246, 269, 244, 242, 249, 264, 256, 283, 249, 242, 258, 267, 260, 264, 257, 226, 254, 244, 250, 244, 254, 265, 266, 249, 264, 254, 243, 251, 264, 234, 276, 258, 256, 249, 286, 267, 244, 274, 238, 243, 283, 271, 253, 250, 248, 278, 234, 238, 250, 250, 242, 250, 286, 270, 254, 235, 243, 251, 259, 273, 263, 258, 273, 235, 236, 256, 243, 248, 266, 277, 260, 287, 261, 253, 245, 272, 277, 277, 256, 240, 254, 228, 283, 262, 274, 226, 271, 263, 247, 213, 249, 268, 273, 256, 247, 252, 264, 246, 254, 257, 264, 248, 237, 256, 273, 247, 247, 233, 260] payload_rows_per_dst=[5490, 5492, 5288, 5468, 5478, 5458, 5656, 5410] dup_rows_per_dst=[2712, 2788, 2468, 2792, 2674, 2836, 2854, 2672] combine_rows_per_src=[5490, 5492, 5288, 5468, 5478, 5458, 5656, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229104 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376816 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524528 weight_movement_bytes=18874368 total_nvlink_bytes=331398896 weight_to_token_ratio=0.060393 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38212 dup_rows=21902 dup_rows_remote=19142 dedup_ratio=0.334198 router_tokens_per_expert=[251, 273, 261, 242, 277, 264, 255, 270, 253, 225, 255, 247, 243, 239, 226, 280, 253, 237, 244, 266, 258, 267, 250, 238, 252, 242, 239, 228, 267, 264, 234, 235, 253, 260, 283, 252, 271, 275, 274, 266, 260, 286, 233, 235, 225, 260, 272, 228, 256, 245, 239, 253, 246, 251, 253, 263, 246, 268, 259, 262, 249, 250, 227, 241, 271, 249, 285, 268, 274, 280, 248, 259, 258, 245, 253, 236, 236, 255, 255, 238, 278, 249, 286, 258, 280, 262, 285, 260, 241, 281, 259, 246, 259, 276, 293, 223, 240, 268, 256, 267, 223, 267, 268, 278, 249, 261, 261, 253, 241, 274, 233, 252, 278, 255, 277, 250, 262, 275, 240, 274, 294, 274, 262, 253, 236, 234, 254, 237] payload_rows_per_dst=[5384, 5412, 5540, 5372, 5494, 5466, 5422, 5544] dup_rows_per_dst=[2738, 2536, 2726, 2644, 2726, 3006, 2760, 2766] combine_rows_per_src=[5384, 5412, 5540, 5372, 5494, 5466, 5422, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229176 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156852024 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474872 weight_movement_bytes=18874368 total_nvlink_bytes=332349240 weight_to_token_ratio=0.060210 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229168 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122352 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015536 weight_movement_bytes=18874368 total_nvlink_bytes=332889904 weight_to_token_ratio=0.060106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229384 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156762120 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294856 weight_movement_bytes=37748736 total_nvlink_bytes=351043592 weight_to_token_ratio=0.120489 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229720 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156934488 ingress_bytes=194453504 link_bytes=194453504 token_bytes=313639256 weight_movement_bytes=37748736 total_nvlink_bytes=351387992 weight_to_token_ratio=0.120357 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863157798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229816 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157229496 ingress_bytes=232497152 link_bytes=232497152 token_bytes=314229176 weight_movement_bytes=75497472 total_nvlink_bytes=389726648 weight_to_token_ratio=0.240262 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227673315e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229416 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156745768 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313262120 weight_movement_bytes=18874368 total_nvlink_bytes=332136488 weight_to_token_ratio=0.060251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38258 dup_rows=21608 dup_rows_remote=18664 dedup_ratio=0.329712 router_tokens_per_expert=[245, 239, 250, 266, 241, 244, 246, 270, 259, 261, 258, 250, 256, 266, 261, 242, 244, 257, 246, 257, 269, 269, 225, 242, 274, 251, 255, 262, 275, 248, 284, 239, 247, 277, 270, 238, 262, 243, 263, 240, 234, 239, 268, 283, 216, 237, 254, 248, 238, 255, 273, 251, 242, 219, 242, 232, 267, 245, 243, 262, 279, 266, 268, 266, 265, 261, 239, 234, 257, 263, 274, 245, 251, 275, 261, 234, 275, 229, 269, 261, 240, 255, 254, 255, 273, 227, 262, 255, 267, 294, 269, 251, 277, 242, 262, 253, 275, 257, 259, 244, 273, 272, 296, 250, 269, 268, 260, 256, 262, 234, 269, 249, 306, 250, 243, 252, 240, 258, 261, 274, 226, 249, 271, 288, 240, 245, 265, 260] payload_rows_per_dst=[5670, 5492, 5438, 5428, 5624, 5292, 5436, 5548] dup_rows_per_dst=[2944, 2736, 2600, 2668, 2888, 2440, 2624, 2708] combine_rows_per_src=[5670, 5492, 5438, 5428, 5624, 5292, 5436, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38308 dup_rows=21720 dup_rows_remote=19136 dedup_ratio=0.331421 router_tokens_per_expert=[259, 252, 252, 256, 270, 238, 266, 245, 252, 238, 232, 248, 254, 239, 278, 236, 255, 235, 270, 248, 235, 284, 250, 268, 279, 274, 266, 252, 255, 248, 250, 248, 245, 271, 241, 263, 251, 252, 253, 279, 260, 258, 242, 276, 278, 242, 270, 247, 229, 249, 260, 231, 249, 258, 256, 251, 247, 254, 269, 246, 260, 245, 279, 263, 255, 251, 252, 234, 257, 288, 252, 249, 253, 266, 247, 287, 252, 267, 265, 256, 249, 220, 232, 260, 264, 259, 266, 244, 267, 255, 257, 276, 270, 241, 274, 266, 244, 239, 285, 270, 229, 255, 277, 238, 265, 266, 233, 234, 242, 252, 269, 260, 277, 269, 264, 242, 274, 213, 262, 244, 242, 296, 256, 266, 261, 273, 281, 253] payload_rows_per_dst=[5382, 5478, 5478, 5508, 5508, 5510, 5440, 5512] dup_rows_per_dst=[2648, 2756, 2778, 2584, 2754, 2690, 2676, 2834] combine_rows_per_src=[5382, 5478, 5478, 5508, 5508, 5510, 5440, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38316 dup_rows=21722 dup_rows_remote=18990 dedup_ratio=0.331451 router_tokens_per_expert=[267, 229, 220, 250, 258, 232, 247, 247, 261, 274, 263, 277, 253, 254, 244, 262, 231, 239, 242, 241, 248, 254, 302, 246, 243, 238, 242, 269, 256, 266, 255, 303, 273, 246, 232, 261, 239, 233, 298, 274, 257, 264, 272, 262, 279, 278, 251, 249, 252, 254, 268, 265, 270, 249, 250, 244, 263, 244, 226, 247, 250, 251, 218, 245, 273, 265, 249, 250, 260, 269, 244, 246, 277, 257, 253, 273, 267, 250, 249, 240, 237, 225, 275, 257, 263, 233, 272, 237, 270, 268, 278, 280, 264, 248, 228, 280, 239, 285, 251, 263, 242, 266, 259, 268, 224, 257, 232, 252, 244, 251, 250, 233, 239, 262, 260, 297, 236, 261, 261, 271, 270, 288, 261, 268, 274, 271, 255, 264] payload_rows_per_dst=[5414, 5436, 5640, 5366, 5536, 5498, 5384, 5540] dup_rows_per_dst=[2662, 2714, 2696, 2626, 2708, 2732, 2648, 2936] combine_rows_per_src=[5414, 5436, 5640, 5366, 5536, 5498, 5384, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38248 dup_rows=21790 dup_rows_remote=18990 dedup_ratio=0.332489 router_tokens_per_expert=[263, 304, 233, 254, 251, 268, 262, 224, 243, 229, 240, 250, 251, 246, 276, 245, 264, 263, 265, 232, 276, 262, 248, 250, 260, 282, 230, 254, 256, 274, 269, 264, 233, 241, 243, 252, 227, 243, 264, 271, 290, 269, 282, 251, 277, 257, 271, 258, 245, 261, 263, 267, 229, 219, 253, 268, 281, 194, 261, 266, 241, 243, 241, 245, 256, 251, 241, 272, 243, 243, 262, 261, 244, 268, 276, 264, 293, 227, 275, 268, 267, 241, 262, 236, 228, 268, 256, 249, 261, 237, 253, 234, 248, 271, 241, 245, 267, 261, 262, 277, 274, 250, 245, 290, 267, 257, 251, 245, 263, 261, 250, 267, 264, 269, 267, 241, 269, 259, 258, 242, 243, 253, 298, 261, 265, 263, 252, 242] payload_rows_per_dst=[5444, 5498, 5488, 5406, 5502, 5378, 5512, 5518] dup_rows_per_dst=[2634, 2800, 2770, 2548, 2786, 2616, 2862, 2774] combine_rows_per_src=[5444, 5498, 5488, 5406, 5502, 5378, 5512, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38138 dup_rows=21938 dup_rows_remote=19260 dedup_ratio=0.334747 router_tokens_per_expert=[267, 251, 270, 224, 238, 233, 233, 258, 264, 249, 216, 262, 270, 252, 257, 248, 260, 259, 243, 280, 226, 244, 254, 279, 274, 250, 268, 235, 248, 251, 249, 246, 232, 286, 243, 255, 251, 256, 272, 258, 236, 260, 274, 263, 265, 282, 268, 208, 243, 255, 263, 262, 260, 250, 290, 255, 252, 248, 261, 261, 250, 249, 259, 251, 246, 245, 243, 249, 276, 233, 268, 253, 279, 227, 257, 263, 248, 286, 258, 238, 263, 241, 232, 251, 273, 239, 262, 238, 246, 254, 260, 254, 255, 266, 236, 258, 254, 265, 257, 253, 251, 264, 261, 204, 288, 239, 269, 243, 285, 251, 255, 259, 263, 307, 285, 255, 290, 264, 238, 260, 253, 241, 262, 281, 279, 286, 270, 263] payload_rows_per_dst=[5314, 5468, 5470, 5464, 5460, 5360, 5432, 5630] dup_rows_per_dst=[2670, 2664, 2748, 2754, 2678, 2696, 2764, 2964] combine_rows_per_src=[5314, 5468, 5470, 5464, 5460, 5360, 5432, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38294 dup_rows=21826 dup_rows_remote=19124 dedup_ratio=0.333038 router_tokens_per_expert=[252, 238, 257, 230, 224, 253, 253, 263, 243, 225, 231, 264, 249, 250, 282, 222, 230, 240, 264, 227, 250, 257, 255, 270, 256, 257, 264, 242, 248, 258, 262, 238, 246, 249, 277, 270, 251, 256, 253, 246, 259, 274, 263, 242, 242, 240, 251, 240, 260, 275, 242, 270, 240, 280, 253, 254, 259, 245, 250, 253, 258, 257, 253, 294, 252, 257, 272, 258, 310, 262, 248, 257, 251, 267, 248, 242, 280, 257, 254, 246, 277, 245, 266, 239, 241, 269, 269, 276, 268, 290, 247, 250, 264, 239, 255, 276, 252, 276, 270, 247, 255, 232, 234, 242, 268, 256, 259, 261, 275, 239, 277, 249, 249, 254, 266, 246, 252, 257, 275, 299, 264, 282, 246, 260, 240, 254, 283, 261] payload_rows_per_dst=[5360, 5364, 5416, 5526, 5520, 5488, 5492, 5544] dup_rows_per_dst=[2512, 2672, 2702, 2760, 2802, 2854, 2692, 2832] combine_rows_per_src=[5360, 5364, 5416, 5526, 5520, 5488, 5492, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=227688 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156932456 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313637224 weight_movement_bytes=18874368 total_nvlink_bytes=332511592 weight_to_token_ratio=0.060179 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229776 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157139344 ingress_bytes=175783936 link_bytes=175783936 token_bytes=314048912 weight_movement_bytes=18874368 total_nvlink_bytes=332923280 weight_to_token_ratio=0.060100 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684275184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38186 dup_rows=21934 dup_rows_remote=19222 dedup_ratio=0.334686 router_tokens_per_expert=[275, 240, 253, 235, 216, 246, 230, 250, 241, 244, 278, 253, 274, 242, 267, 291, 240, 284, 264, 266, 272, 246, 276, 277, 228, 280, 257, 228, 274, 257, 235, 249, 239, 240, 261, 290, 256, 275, 222, 258, 263, 260, 239, 253, 259, 262, 252, 239, 264, 256, 251, 250, 287, 253, 257, 237, 245, 273, 245, 230, 246, 279, 244, 258, 263, 279, 261, 237, 269, 254, 272, 269, 270, 263, 288, 254, 271, 263, 270, 252, 231, 260, 260, 230, 253, 236, 257, 244, 273, 240, 240, 253, 263, 258, 273, 238, 265, 242, 247, 234, 237, 273, 241, 265, 269, 264, 239, 258, 259, 257, 242, 272, 278, 253, 242, 244, 277, 259, 261, 257, 282, 248, 253, 254, 278, 234, 280, 249] payload_rows_per_dst=[5342, 5462, 5490, 5468, 5590, 5366, 5416, 5468] dup_rows_per_dst=[2728, 2804, 2646, 2682, 2880, 2652, 2712, 2830] combine_rows_per_src=[5342, 5462, 5490, 5468, 5590, 5366, 5416, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38098 dup_rows=21862 dup_rows_remote=18882 dedup_ratio=0.333588 router_tokens_per_expert=[250, 253, 253, 258, 252, 264, 238, 263, 244, 248, 270, 250, 223, 234, 259, 244, 259, 237, 256, 267, 265, 246, 265, 250, 262, 247, 249, 293, 259, 269, 272, 281, 226, 284, 284, 240, 258, 247, 291, 239, 268, 253, 261, 247, 265, 227, 236, 262, 271, 259, 269, 231, 251, 267, 282, 296, 243, 242, 222, 269, 229, 232, 260, 268, 261, 227, 230, 296, 234, 247, 240, 240, 281, 296, 238, 239, 237, 271, 255, 253, 248, 262, 309, 271, 250, 242, 238, 266, 272, 249, 269, 236, 263, 245, 236, 273, 238, 270, 254, 247, 249, 246, 264, 277, 266, 228, 232, 286, 255, 252, 274, 246, 252, 253, 288, 246, 260, 266, 275, 230, 251, 241, 265, 281, 281, 265, 226, 271] payload_rows_per_dst=[5364, 5534, 5338, 5446, 5402, 5486, 5528, 5576] dup_rows_per_dst=[2642, 2792, 2584, 2736, 2688, 2772, 2668, 2980] combine_rows_per_src=[5364, 5534, 5338, 5446, 5402, 5486, 5528, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229224 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157171560 ingress_bytes=194691072 link_bytes=194691072 token_bytes=314113896 weight_movement_bytes=37748736 total_nvlink_bytes=351862632 weight_to_token_ratio=0.120175 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865434058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228952 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156892760 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313556568 weight_movement_bytes=18874368 total_nvlink_bytes=332430936 weight_to_token_ratio=0.060194 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229592 combine_payload_bytes=156213248 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156442840 ingress_bytes=193961984 link_bytes=193961984 token_bytes=312656088 weight_movement_bytes=37748736 total_nvlink_bytes=350404824 weight_to_token_ratio=0.120736 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858448295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229672 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081896 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313934120 weight_movement_bytes=37748736 total_nvlink_bytes=351682856 weight_to_token_ratio=0.120244 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229632 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156639488 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313049344 weight_movement_bytes=37748736 total_nvlink_bytes=350798080 weight_to_token_ratio=0.120584 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=227920 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156277328 ingress_bytes=193798144 link_bytes=193798144 token_bytes=312326736 weight_movement_bytes=37748736 total_nvlink_bytes=350075472 weight_to_token_ratio=0.120863 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856878461e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38092 dup_rows=21930 dup_rows_remote=18976 dedup_ratio=0.334625 router_tokens_per_expert=[239, 266, 246, 265, 258, 244, 286, 248, 270, 261, 257, 273, 240, 270, 243, 241, 249, 253, 270, 281, 242, 262, 248, 265, 227, 260, 235, 259, 274, 263, 253, 275, 289, 269, 252, 229, 251, 252, 224, 238, 228, 276, 258, 250, 260, 240, 271, 251, 251, 251, 266, 242, 209, 271, 232, 262, 282, 267, 268, 248, 262, 242, 246, 253, 248, 266, 253, 231, 261, 267, 236, 285, 278, 272, 282, 254, 240, 265, 221, 289, 266, 256, 259, 253, 248, 254, 265, 273, 256, 286, 271, 227, 285, 264, 252, 231, 235, 265, 232, 252, 253, 245, 269, 263, 232, 232, 272, 248, 253, 259, 300, 282, 263, 281, 230, 260, 247, 261, 271, 238, 267, 225, 266, 246, 247, 247, 270, 250] payload_rows_per_dst=[5514, 5474, 5390, 5424, 5484, 5490, 5358, 5472] dup_rows_per_dst=[2954, 2832, 2464, 2648, 2812, 2866, 2688, 2666] combine_rows_per_src=[5514, 5474, 5390, 5424, 5484, 5490, 5358, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38252 dup_rows=21906 dup_rows_remote=19210 dedup_ratio=0.334259 router_tokens_per_expert=[262, 265, 254, 256, 267, 271, 259, 227, 261, 254, 252, 256, 295, 251, 283, 243, 240, 266, 252, 285, 267, 264, 253, 245, 263, 262, 285, 248, 259, 254, 292, 284, 275, 257, 247, 246, 227, 248, 261, 267, 253, 257, 267, 249, 258, 232, 261, 240, 259, 227, 237, 232, 253, 275, 258, 243, 253, 277, 246, 248, 255, 252, 268, 254, 236, 240, 252, 242, 251, 255, 245, 245, 258, 244, 261, 267, 257, 274, 226, 259, 256, 240, 254, 259, 253, 255, 225, 233, 226, 261, 273, 252, 281, 270, 246, 266, 251, 264, 264, 229, 274, 278, 246, 275, 277, 259, 247, 253, 259, 256, 246, 271, 230, 249, 265, 233, 275, 242, 248, 270, 278, 264, 257, 261, 269, 238, 256, 265] payload_rows_per_dst=[5496, 5594, 5440, 5378, 5370, 5398, 5510, 5444] dup_rows_per_dst=[2816, 2844, 2650, 2696, 2654, 2702, 2788, 2756] combine_rows_per_src=[5496, 5594, 5440, 5378, 5370, 5398, 5510, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38358 dup_rows=21706 dup_rows_remote=18834 dedup_ratio=0.331207 router_tokens_per_expert=[289, 265, 249, 237, 249, 245, 260, 249, 247, 240, 238, 254, 253, 256, 248, 263, 256, 266, 254, 261, 260, 245, 266, 251, 263, 275, 236, 274, 269, 247, 240, 238, 250, 253, 263, 253, 267, 263, 252, 262, 241, 275, 290, 276, 281, 258, 256, 252, 237, 250, 254, 241, 241, 255, 232, 265, 238, 231, 258, 264, 243, 258, 269, 264, 233, 261, 285, 249, 293, 245, 222, 240, 290, 270, 255, 233, 260, 268, 249, 236, 243, 223, 257, 293, 260, 281, 275, 243, 254, 248, 291, 259, 264, 251, 258, 272, 251, 250, 251, 253, 282, 246, 243, 247, 296, 262, 242, 262, 253, 258, 232, 265, 277, 252, 262, 241, 264, 268, 272, 253, 253, 249, 242, 237, 239, 270, 257, 243] payload_rows_per_dst=[5460, 5482, 5528, 5380, 5550, 5472, 5482, 5476] dup_rows_per_dst=[2624, 2720, 2856, 2620, 2628, 2872, 2704, 2682] combine_rows_per_src=[5460, 5482, 5528, 5380, 5550, 5472, 5482, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43984 payload_rows_remote=38598 dup_rows=21552 dup_rows_remote=18876 dedup_ratio=0.328857 router_tokens_per_expert=[251, 227, 279, 274, 282, 252, 260, 266, 243, 245, 273, 258, 235, 266, 245, 244, 248, 269, 264, 249, 249, 243, 265, 257, 272, 231, 237, 263, 238, 253, 235, 258, 270, 268, 254, 242, 239, 249, 239, 260, 263, 253, 253, 251, 248, 243, 268, 280, 230, 261, 255, 247, 261, 272, 251, 268, 233, 251, 247, 226, 275, 268, 251, 295, 277, 253, 261, 273, 251, 273, 254, 262, 253, 233, 236, 253, 269, 265, 232, 284, 259, 261, 253, 237, 253, 269, 281, 229, 242, 236, 265, 256, 278, 258, 254, 256, 220, 273, 255, 248, 246, 267, 249, 253, 252, 229, 224, 249, 285, 282, 310, 247, 263, 268, 249, 245, 257, 226, 281, 258, 251, 252, 275, 259, 286, 254, 246, 291] payload_rows_per_dst=[5604, 5386, 5516, 5532, 5510, 5512, 5486, 5438] dup_rows_per_dst=[2956, 2676, 2644, 2650, 2748, 2662, 2692, 2524] combine_rows_per_src=[5604, 5386, 5516, 5532, 5510, 5512, 5486, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38192 dup_rows=21910 dup_rows_remote=19266 dedup_ratio=0.334320 router_tokens_per_expert=[222, 262, 249, 244, 269, 242, 266, 254, 255, 254, 248, 261, 283, 247, 279, 234, 273, 240, 247, 229, 256, 246, 234, 274, 272, 233, 279, 257, 264, 251, 244, 240, 266, 237, 258, 251, 251, 250, 263, 255, 285, 276, 235, 272, 251, 264, 248, 257, 245, 242, 253, 219, 284, 271, 278, 269, 255, 232, 254, 258, 265, 241, 258, 259, 258, 268, 243, 266, 247, 277, 256, 264, 236, 248, 263, 241, 242, 229, 237, 264, 278, 250, 260, 265, 288, 273, 249, 266, 253, 240, 271, 255, 273, 261, 276, 271, 242, 250, 250, 291, 256, 266, 272, 240, 250, 246, 243, 255, 242, 261, 282, 268, 246, 254, 241, 260, 250, 258, 232, 279, 249, 301, 239, 225, 263, 271, 261, 247] payload_rows_per_dst=[5444, 5374, 5450, 5486, 5434, 5566, 5448, 5424] dup_rows_per_dst=[2694, 2704, 2788, 2680, 2644, 2892, 2780, 2728] combine_rows_per_src=[5444, 5374, 5450, 5486, 5434, 5566, 5448, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228272 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253104 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312277936 weight_movement_bytes=18874368 total_nvlink_bytes=331152304 weight_to_token_ratio=0.060441 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229848 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156910040 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313590232 weight_movement_bytes=18874368 total_nvlink_bytes=332464600 weight_to_token_ratio=0.060188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38304 dup_rows=21742 dup_rows_remote=18996 dedup_ratio=0.331757 router_tokens_per_expert=[251, 284, 248, 261, 262, 265, 235, 270, 250, 240, 237, 261, 247, 258, 248, 261, 262, 289, 281, 287, 277, 253, 234, 272, 266, 280, 283, 262, 254, 240, 265, 273, 244, 259, 279, 236, 239, 270, 243, 251, 241, 223, 267, 247, 242, 247, 253, 261, 234, 245, 248, 237, 268, 240, 276, 257, 248, 248, 258, 223, 246, 255, 249, 249, 267, 274, 263, 239, 263, 220, 248, 243, 260, 273, 278, 245, 237, 245, 259, 261, 255, 292, 255, 266, 233, 246, 256, 232, 258, 259, 249, 263, 278, 262, 271, 264, 276, 262, 261, 243, 241, 268, 244, 263, 247, 250, 272, 256, 232, 271, 268, 264, 256, 249, 272, 254, 237, 261, 278, 234, 265, 251, 229, 259, 266, 270, 248, 268] payload_rows_per_dst=[5486, 5626, 5366, 5358, 5418, 5532, 5490, 5518] dup_rows_per_dst=[2670, 2930, 2638, 2604, 2732, 2746, 2746, 2676] combine_rows_per_src=[5486, 5626, 5366, 5358, 5418, 5532, 5490, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38188 dup_rows=21912 dup_rows_remote=19214 dedup_ratio=0.334351 router_tokens_per_expert=[272, 247, 249, 251, 250, 254, 246, 268, 252, 273, 238, 272, 231, 260, 239, 264, 250, 274, 271, 264, 271, 247, 261, 249, 271, 269, 247, 273, 251, 252, 249, 257, 233, 224, 276, 253, 293, 254, 272, 266, 236, 273, 262, 270, 246, 243, 226, 240, 239, 283, 258, 267, 264, 251, 276, 231, 259, 249, 284, 227, 243, 233, 221, 265, 278, 266, 256, 248, 228, 263, 241, 265, 277, 229, 241, 236, 265, 253, 249, 252, 253, 245, 290, 271, 281, 264, 244, 269, 251, 261, 276, 267, 266, 253, 248, 230, 245, 270, 256, 271, 236, 272, 246, 253, 260, 262, 280, 246, 244, 265, 281, 224, 276, 283, 257, 237, 248, 260, 250, 257, 257, 237, 237, 271, 251, 268, 278, 235] payload_rows_per_dst=[5422, 5504, 5436, 5432, 5406, 5530, 5454, 5440] dup_rows_per_dst=[2710, 2808, 2698, 2668, 2688, 2808, 2768, 2764] combine_rows_per_src=[5422, 5504, 5436, 5432, 5406, 5530, 5454, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38376 dup_rows=21668 dup_rows_remote=18950 dedup_ratio=0.330627 router_tokens_per_expert=[234, 262, 261, 244, 243, 282, 259, 267, 233, 289, 261, 271, 258, 268, 251, 242, 252, 276, 270, 256, 241, 251, 255, 281, 235, 242, 267, 250, 256, 283, 244, 267, 280, 276, 259, 258, 251, 262, 293, 249, 267, 244, 249, 220, 265, 248, 272, 249, 261, 236, 254, 243, 247, 244, 223, 236, 267, 256, 260, 244, 289, 237, 260, 268, 253, 266, 269, 251, 259, 246, 259, 236, 268, 238, 253, 219, 243, 273, 279, 236, 262, 264, 237, 243, 252, 251, 263, 247, 232, 251, 263, 264, 266, 256, 233, 291, 244, 243, 254, 264, 257, 263, 273, 258, 253, 229, 271, 280, 289, 233, 271, 249, 255, 238, 261, 251, 262, 263, 228, 244, 275, 261, 283, 256, 281, 265, 241, 232] payload_rows_per_dst=[5446, 5354, 5542, 5568, 5502, 5468, 5496, 5492] dup_rows_per_dst=[2804, 2468, 2742, 2912, 2594, 2682, 2748, 2718] combine_rows_per_src=[5446, 5354, 5542, 5568, 5502, 5468, 5496, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=228768 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157343136 ingress_bytes=194863104 link_bytes=194863104 token_bytes=314457504 weight_movement_bytes=37748736 total_nvlink_bytes=352206240 weight_to_token_ratio=0.120044 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867082384e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=158097408 dispatch_meta_bytes=229896 combine_payload_bytes=158097408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158327304 ingress_bytes=195846144 link_bytes=195846144 token_bytes=316424712 weight_movement_bytes=37748736 total_nvlink_bytes=354173448 weight_to_token_ratio=0.119298 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.876501389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229832 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156664264 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098696 weight_movement_bytes=18874368 total_nvlink_bytes=331973064 weight_to_token_ratio=0.060282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229200 combine_payload_bytes=156893184 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157122384 ingress_bytes=213516288 link_bytes=213516288 token_bytes=314015568 weight_movement_bytes=56623104 total_nvlink_bytes=370638672 weight_to_token_ratio=0.180319 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045808015e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229608 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156647656 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065704 weight_movement_bytes=18874368 total_nvlink_bytes=331940072 weight_to_token_ratio=0.060289 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229304 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157417400 ingress_bytes=194936832 link_bytes=194936832 token_bytes=314605496 weight_movement_bytes=37748736 total_nvlink_bytes=352354232 weight_to_token_ratio=0.119988 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867788809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38386 dup_rows=21762 dup_rows_remote=19146 dedup_ratio=0.332062 router_tokens_per_expert=[246, 257, 261, 282, 260, 251, 233, 265, 244, 234, 259, 244, 263, 297, 272, 264, 280, 257, 253, 278, 258, 255, 282, 276, 245, 260, 246, 249, 267, 261, 285, 270, 267, 241, 267, 262, 277, 250, 235, 252, 253, 234, 253, 268, 215, 265, 237, 224, 249, 247, 250, 239, 269, 258, 275, 223, 251, 284, 236, 248, 257, 226, 246, 244, 248, 247, 249, 256, 242, 267, 252, 258, 265, 256, 238, 236, 266, 274, 261, 250, 232, 270, 280, 248, 270, 228, 251, 237, 246, 261, 250, 283, 251, 253, 234, 268, 281, 270, 247, 275, 261, 261, 244, 242, 246, 239, 284, 265, 238, 264, 278, 275, 280, 243, 245, 262, 252, 281, 261, 291, 250, 238, 249, 261, 270, 240, 236, 256] payload_rows_per_dst=[5502, 5528, 5370, 5388, 5478, 5478, 5486, 5544] dup_rows_per_dst=[2762, 2916, 2630, 2616, 2652, 2646, 2854, 2686] combine_rows_per_src=[5502, 5528, 5370, 5388, 5478, 5478, 5486, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38176 dup_rows=21936 dup_rows_remote=19188 dedup_ratio=0.334717 router_tokens_per_expert=[226, 259, 266, 243, 279, 258, 248, 246, 237, 274, 266, 249, 260, 247, 259, 236, 277, 292, 265, 244, 259, 277, 233, 230, 244, 237, 244, 236, 232, 266, 240, 241, 233, 272, 264, 278, 247, 237, 251, 258, 250, 270, 257, 277, 239, 251, 274, 244, 241, 250, 264, 244, 260, 236, 226, 246, 301, 298, 267, 286, 281, 263, 269, 272, 287, 267, 261, 275, 250, 251, 241, 229, 247, 243, 244, 287, 296, 248, 304, 244, 260, 243, 271, 248, 262, 217, 259, 249, 273, 250, 215, 293, 242, 264, 238, 242, 226, 264, 238, 268, 257, 282, 271, 289, 250, 272, 240, 255, 260, 246, 252, 251, 256, 248, 251, 260, 288, 262, 279, 244, 235, 251, 260, 215, 268, 247, 249, 258] payload_rows_per_dst=[5424, 5350, 5668, 5570, 5598, 5376, 5302, 5312] dup_rows_per_dst=[2748, 2684, 3036, 2838, 2970, 2676, 2374, 2610] combine_rows_per_src=[5424, 5350, 5668, 5570, 5598, 5376, 5302, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=230128 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157459184 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314688240 weight_movement_bytes=18874368 total_nvlink_bytes=333562608 weight_to_token_ratio=0.059978 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38248 dup_rows=21784 dup_rows_remote=19066 dedup_ratio=0.332397 router_tokens_per_expert=[238, 233, 289, 261, 252, 276, 280, 243, 248, 260, 244, 267, 255, 243, 251, 282, 258, 230, 233, 249, 258, 263, 250, 276, 270, 244, 239, 234, 253, 259, 263, 254, 266, 250, 264, 275, 247, 263, 261, 269, 260, 277, 251, 264, 256, 254, 253, 257, 244, 255, 219, 241, 265, 237, 272, 218, 280, 276, 230, 276, 279, 253, 251, 271, 248, 239, 260, 276, 236, 269, 250, 253, 235, 269, 262, 244, 247, 244, 270, 249, 235, 265, 244, 257, 255, 261, 243, 258, 245, 254, 283, 264, 267, 274, 250, 256, 282, 252, 239, 254, 207, 264, 233, 270, 250, 248, 275, 235, 281, 262, 290, 258, 247, 254, 249, 259, 300, 257, 248, 262, 276, 257, 257, 280, 265, 230, 236, 240] payload_rows_per_dst=[5490, 5462, 5474, 5474, 5392, 5504, 5504, 5452] dup_rows_per_dst=[2754, 2604, 2860, 2660, 2710, 2718, 2696, 2782] combine_rows_per_src=[5490, 5462, 5474, 5474, 5392, 5504, 5504, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229456 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156598352 ingress_bytes=212992000 link_bytes=212992000 token_bytes=312967248 weight_movement_bytes=56623104 total_nvlink_bytes=369590352 weight_to_token_ratio=0.180923 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040784545e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38400 dup_rows=21754 dup_rows_remote=19096 dedup_ratio=0.331940 router_tokens_per_expert=[247, 276, 278, 231, 258, 273, 266, 265, 252, 252, 279, 260, 240, 238, 244, 271, 238, 277, 238, 224, 245, 243, 226, 279, 259, 249, 272, 258, 256, 261, 275, 220, 245, 270, 257, 265, 238, 253, 245, 251, 259, 253, 277, 238, 239, 293, 240, 296, 268, 257, 284, 259, 248, 271, 235, 262, 266, 259, 279, 249, 235, 263, 261, 257, 234, 257, 278, 247, 249, 240, 281, 286, 221, 281, 238, 273, 267, 250, 263, 243, 247, 249, 262, 255, 262, 240, 240, 295, 227, 277, 267, 245, 247, 251, 235, 260, 271, 248, 225, 259, 273, 257, 273, 254, 277, 226, 265, 256, 262, 244, 244, 254, 255, 274, 245, 281, 269, 216, 227, 258, 294, 249, 268, 226, 261, 238, 262, 268] payload_rows_per_dst=[5500, 5382, 5444, 5540, 5552, 5392, 5534, 5438] dup_rows_per_dst=[2760, 2658, 2794, 2766, 2664, 2726, 2642, 2744] combine_rows_per_src=[5500, 5382, 5444, 5540, 5552, 5392, 5534, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38336 dup_rows=21634 dup_rows_remote=18850 dedup_ratio=0.330109 router_tokens_per_expert=[257, 265, 258, 251, 246, 245, 272, 251, 275, 290, 266, 242, 248, 282, 268, 254, 268, 277, 264, 250, 265, 243, 285, 246, 247, 259, 257, 250, 274, 260, 240, 247, 260, 251, 263, 280, 289, 263, 258, 229, 274, 255, 246, 250, 247, 243, 256, 261, 234, 250, 269, 242, 231, 260, 256, 263, 280, 289, 250, 233, 256, 231, 236, 226, 259, 260, 284, 269, 265, 276, 279, 277, 243, 263, 263, 262, 243, 228, 234, 270, 247, 226, 214, 216, 287, 236, 253, 266, 260, 246, 244, 253, 276, 233, 254, 275, 253, 288, 239, 253, 259, 253, 244, 275, 271, 248, 246, 232, 233, 265, 257, 234, 288, 260, 251, 236, 257, 244, 275, 250, 267, 250, 255, 271, 248, 261, 253, 258] payload_rows_per_dst=[5544, 5502, 5468, 5444, 5566, 5430, 5464, 5484] dup_rows_per_dst=[2796, 2762, 2782, 2568, 2784, 2542, 2636, 2764] combine_rows_per_src=[5544, 5502, 5468, 5444, 5566, 5430, 5464, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38242 dup_rows=21794 dup_rows_remote=19068 dedup_ratio=0.332550 router_tokens_per_expert=[252, 226, 274, 273, 267, 261, 296, 228, 265, 249, 248, 267, 252, 232, 262, 270, 262, 271, 226, 299, 231, 249, 229, 229, 236, 272, 263, 266, 258, 267, 248, 278, 283, 259, 288, 252, 254, 273, 234, 221, 249, 241, 239, 263, 261, 247, 293, 273, 263, 234, 226, 283, 263, 263, 254, 252, 255, 231, 248, 265, 252, 248, 250, 240, 265, 246, 276, 234, 255, 257, 250, 291, 252, 293, 255, 272, 239, 269, 229, 266, 232, 249, 245, 243, 286, 244, 267, 271, 234, 259, 230, 241, 272, 260, 261, 252, 245, 239, 259, 258, 256, 260, 280, 257, 234, 268, 284, 263, 245, 274, 236, 255, 271, 263, 270, 251, 231, 281, 250, 264, 244, 249, 248, 249, 262, 277, 225, 262] payload_rows_per_dst=[5448, 5412, 5520, 5434, 5512, 5468, 5500, 5448] dup_rows_per_dst=[2718, 2756, 2740, 2698, 2786, 2624, 2726, 2746] combine_rows_per_src=[5448, 5412, 5520, 5434, 5512, 5468, 5500, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38378 dup_rows=21714 dup_rows_remote=18936 dedup_ratio=0.331329 router_tokens_per_expert=[260, 263, 248, 274, 271, 253, 272, 229, 236, 264, 248, 235, 241, 259, 283, 284, 242, 238, 239, 227, 272, 242, 270, 266, 249, 252, 236, 243, 277, 256, 263, 259, 286, 249, 258, 240, 271, 236, 269, 228, 287, 258, 253, 241, 238, 246, 277, 274, 260, 248, 293, 275, 257, 244, 248, 252, 263, 231, 234, 243, 258, 296, 247, 233, 255, 267, 257, 225, 273, 276, 252, 225, 256, 300, 243, 262, 263, 247, 254, 261, 268, 247, 256, 242, 250, 247, 237, 262, 257, 275, 259, 235, 254, 258, 259, 249, 252, 255, 262, 267, 255, 228, 252, 266, 270, 234, 256, 263, 240, 299, 260, 297, 260, 269, 262, 261, 249, 274, 241, 245, 254, 251, 259, 252, 258, 245, 238, 279] payload_rows_per_dst=[5510, 5466, 5444, 5434, 5438, 5460, 5538, 5532] dup_rows_per_dst=[2730, 2596, 2778, 2730, 2794, 2650, 2774, 2662] combine_rows_per_src=[5510, 5466, 5444, 5434, 5438, 5460, 5538, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38230 dup_rows=21778 dup_rows_remote=19038 dedup_ratio=0.332306 router_tokens_per_expert=[279, 260, 271, 252, 253, 227, 239, 249, 263, 258, 239, 238, 254, 249, 258, 285, 256, 235, 243, 253, 260, 276, 247, 239, 265, 254, 245, 250, 275, 259, 254, 245, 270, 259, 251, 265, 258, 236, 250, 281, 254, 245, 260, 233, 258, 250, 235, 253, 268, 253, 240, 264, 257, 254, 289, 264, 229, 243, 258, 246, 281, 232, 232, 294, 266, 299, 242, 257, 267, 259, 252, 249, 261, 240, 261, 263, 248, 236, 255, 267, 246, 260, 271, 248, 282, 260, 254, 256, 265, 249, 277, 262, 252, 266, 240, 236, 236, 256, 246, 292, 253, 257, 256, 252, 264, 255, 264, 250, 250, 277, 246, 280, 260, 234, 241, 231, 237, 267, 273, 259, 251, 268, 268, 265, 257, 262, 288, 235] payload_rows_per_dst=[5236, 5446, 5414, 5682, 5490, 5482, 5480, 5528] dup_rows_per_dst=[2386, 2574, 2702, 3052, 2754, 2766, 2804, 2740] combine_rows_per_src=[5236, 5446, 5414, 5682, 5490, 5482, 5480, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229256 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156893064 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556872 weight_movement_bytes=37748736 total_nvlink_bytes=351305608 weight_to_token_ratio=0.120389 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229984 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157516384 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802784 weight_movement_bytes=37748736 total_nvlink_bytes=352551520 weight_to_token_ratio=0.119912 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=228744 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157253000 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314277256 weight_movement_bytes=18874368 total_nvlink_bytes=333151624 weight_to_token_ratio=0.060056 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229240 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156868472 ingress_bytes=251011072 link_bytes=251011072 token_bytes=313507704 weight_movement_bytes=94371840 total_nvlink_bytes=407879544 weight_to_token_ratio=0.301019 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405064587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229256 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157425544 ingress_bytes=194945024 link_bytes=194945024 token_bytes=314621832 weight_movement_bytes=37748736 total_nvlink_bytes=352370568 weight_to_token_ratio=0.119981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867867301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229072 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819152 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409232 weight_movement_bytes=18874368 total_nvlink_bytes=332283600 weight_to_token_ratio=0.060223 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43518 payload_rows_remote=38032 dup_rows=22018 dup_rows_remote=19224 dedup_ratio=0.335968 router_tokens_per_expert=[287, 246, 252, 271, 213, 270, 256, 252, 269, 261, 244, 257, 254, 244, 271, 249, 232, 250, 252, 255, 291, 258, 254, 250, 259, 256, 253, 281, 261, 255, 234, 239, 252, 269, 250, 272, 288, 287, 243, 248, 261, 227, 240, 280, 254, 263, 249, 263, 272, 251, 247, 249, 280, 285, 239, 259, 242, 256, 257, 255, 276, 236, 253, 283, 261, 259, 240, 235, 261, 239, 237, 227, 243, 275, 242, 217, 243, 255, 273, 261, 277, 257, 259, 259, 253, 285, 257, 299, 226, 236, 268, 264, 261, 233, 267, 243, 253, 265, 248, 260, 241, 268, 285, 257, 276, 277, 246, 273, 258, 235, 269, 274, 248, 249, 249, 231, 252, 221, 268, 267, 249, 268, 250, 267, 239, 244, 258, 249] payload_rows_per_dst=[5460, 5418, 5472, 5486, 5336, 5432, 5476, 5438] dup_rows_per_dst=[2732, 2742, 2820, 2794, 2600, 2856, 2894, 2580] combine_rows_per_src=[5460, 5418, 5472, 5486, 5336, 5432, 5476, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38186 dup_rows=21800 dup_rows_remote=18918 dedup_ratio=0.332642 router_tokens_per_expert=[247, 250, 239, 243, 244, 281, 255, 254, 282, 247, 293, 241, 264, 286, 255, 267, 264, 228, 250, 264, 246, 214, 253, 270, 268, 244, 235, 279, 262, 241, 232, 281, 275, 270, 264, 221, 263, 262, 310, 248, 251, 234, 259, 246, 267, 240, 259, 287, 272, 260, 263, 244, 227, 252, 245, 273, 252, 229, 239, 231, 258, 269, 271, 246, 243, 246, 231, 234, 285, 260, 231, 264, 236, 250, 258, 245, 260, 259, 253, 249, 239, 265, 236, 264, 244, 235, 282, 249, 251, 247, 246, 258, 264, 247, 254, 262, 266, 291, 261, 283, 269, 280, 268, 240, 247, 267, 269, 262, 265, 261, 278, 261, 232, 247, 241, 215, 272, 262, 272, 272, 253, 250, 262, 267, 267, 271, 265, 239] payload_rows_per_dst=[5550, 5438, 5668, 5328, 5372, 5586, 5462, 5332] dup_rows_per_dst=[2882, 2626, 2956, 2624, 2608, 3008, 2566, 2530] combine_rows_per_src=[5550, 5438, 5668, 5328, 5372, 5586, 5462, 5332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=229024 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156008096 ingress_bytes=174653440 link_bytes=174653440 token_bytes=311787168 weight_movement_bytes=18874368 total_nvlink_bytes=330661536 weight_to_token_ratio=0.060536 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673443327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38272 dup_rows=21760 dup_rows_remote=19008 dedup_ratio=0.332031 router_tokens_per_expert=[270, 258, 248, 256, 249, 273, 278, 258, 267, 242, 246, 267, 267, 262, 246, 251, 263, 290, 243, 270, 266, 275, 255, 262, 251, 231, 246, 248, 238, 235, 241, 255, 264, 240, 254, 244, 282, 257, 272, 263, 260, 238, 252, 240, 241, 273, 283, 255, 275, 269, 235, 250, 261, 253, 249, 262, 250, 251, 262, 240, 274, 273, 272, 258, 238, 245, 266, 266, 248, 269, 243, 245, 242, 249, 261, 257, 281, 258, 231, 259, 251, 237, 235, 275, 271, 233, 243, 262, 261, 241, 259, 275, 268, 290, 259, 268, 265, 262, 212, 242, 252, 228, 241, 271, 259, 251, 251, 277, 260, 262, 258, 272, 272, 248, 229, 246, 274, 268, 243, 247, 238, 263, 234, 262, 266, 272, 255, 243] payload_rows_per_dst=[5572, 5482, 5470, 5484, 5422, 5504, 5444, 5398] dup_rows_per_dst=[2704, 2656, 2766, 2784, 2694, 2752, 2682, 2722] combine_rows_per_src=[5572, 5482, 5470, 5484, 5422, 5504, 5444, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228416 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638272 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048128 weight_movement_bytes=37748736 total_nvlink_bytes=350796864 weight_to_token_ratio=0.120584 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38172 dup_rows=21914 dup_rows_remote=19236 dedup_ratio=0.334381 router_tokens_per_expert=[271, 228, 261, 253, 250, 250, 260, 252, 260, 243, 251, 250, 259, 251, 243, 281, 294, 247, 259, 243, 258, 242, 274, 252, 271, 242, 269, 258, 244, 262, 258, 232, 249, 272, 282, 242, 256, 259, 262, 264, 253, 243, 254, 274, 269, 279, 246, 276, 270, 254, 234, 293, 261, 265, 234, 251, 271, 259, 261, 262, 264, 254, 248, 251, 264, 240, 267, 234, 268, 279, 241, 245, 256, 253, 232, 256, 253, 238, 271, 267, 257, 260, 263, 246, 238, 243, 262, 258, 236, 232, 225, 219, 255, 231, 240, 211, 268, 261, 268, 283, 284, 247, 276, 280, 297, 257, 247, 276, 271, 242, 252, 262, 251, 231, 253, 241, 237, 245, 260, 306, 259, 258, 269, 237, 270, 266, 237, 257] payload_rows_per_dst=[5416, 5428, 5496, 5522, 5450, 5340, 5554, 5416] dup_rows_per_dst=[2710, 2782, 2864, 2742, 2678, 2412, 2988, 2738] combine_rows_per_src=[5416, 5428, 5496, 5522, 5450, 5340, 5554, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38414 dup_rows=21678 dup_rows_remote=19018 dedup_ratio=0.330780 router_tokens_per_expert=[243, 260, 258, 273, 256, 267, 245, 250, 276, 267, 245, 276, 266, 256, 237, 240, 235, 280, 259, 250, 281, 247, 225, 254, 245, 249, 279, 252, 245, 247, 244, 260, 258, 250, 278, 232, 254, 265, 239, 237, 242, 261, 258, 261, 252, 259, 275, 267, 262, 270, 264, 255, 263, 241, 228, 281, 257, 257, 250, 264, 249, 244, 264, 244, 256, 256, 245, 240, 222, 239, 219, 270, 261, 263, 235, 255, 278, 276, 265, 265, 273, 254, 252, 234, 260, 279, 270, 233, 253, 261, 266, 231, 256, 256, 262, 268, 256, 251, 255, 254, 248, 253, 265, 259, 262, 241, 273, 237, 304, 282, 263, 285, 264, 277, 275, 255, 238, 273, 241, 260, 237, 247, 243, 254, 251, 264, 269, 231] payload_rows_per_dst=[5440, 5444, 5454, 5534, 5468, 5526, 5470, 5522] dup_rows_per_dst=[2790, 2660, 2722, 2652, 2622, 2690, 2906, 2636] combine_rows_per_src=[5440, 5444, 5454, 5534, 5468, 5526, 5470, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38226 dup_rows=21738 dup_rows_remote=18854 dedup_ratio=0.331696 router_tokens_per_expert=[277, 277, 253, 257, 273, 241, 295, 263, 261, 246, 294, 270, 244, 267, 243, 256, 275, 233, 244, 273, 262, 254, 244, 241, 261, 248, 229, 257, 261, 287, 244, 272, 281, 274, 250, 237, 252, 256, 235, 241, 264, 239, 250, 266, 277, 260, 285, 252, 247, 267, 229, 254, 264, 227, 254, 234, 228, 262, 252, 248, 260, 276, 261, 219, 253, 239, 269, 273, 259, 262, 240, 233, 260, 246, 251, 251, 253, 252, 266, 272, 278, 245, 256, 232, 265, 236, 251, 246, 257, 236, 229, 245, 254, 254, 253, 278, 245, 238, 283, 267, 261, 259, 271, 283, 290, 247, 256, 299, 248, 275, 248, 272, 249, 267, 243, 243, 259, 249, 248, 249, 254, 228, 248, 273, 260, 255, 252, 252] payload_rows_per_dst=[5582, 5438, 5480, 5374, 5448, 5450, 5572, 5454] dup_rows_per_dst=[2852, 2732, 2758, 2590, 2710, 2580, 2884, 2632] combine_rows_per_src=[5582, 5438, 5480, 5374, 5448, 5450, 5572, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38246 dup_rows=21708 dup_rows_remote=19002 dedup_ratio=0.331238 router_tokens_per_expert=[245, 279, 248, 227, 288, 254, 250, 238, 253, 255, 266, 254, 238, 281, 283, 268, 263, 243, 257, 267, 284, 262, 257, 253, 297, 257, 269, 254, 265, 241, 240, 260, 251, 301, 256, 263, 285, 273, 270, 261, 246, 238, 226, 262, 267, 257, 242, 246, 250, 266, 265, 235, 248, 231, 248, 261, 279, 251, 235, 240, 261, 263, 219, 241, 250, 265, 264, 238, 270, 268, 282, 271, 239, 268, 251, 241, 250, 247, 263, 254, 217, 252, 246, 272, 243, 262, 249, 254, 256, 255, 251, 261, 258, 282, 243, 244, 254, 261, 238, 254, 259, 275, 227, 253, 254, 250, 261, 284, 274, 280, 251, 236, 265, 244, 265, 234, 261, 260, 269, 274, 219, 279, 226, 259, 252, 263, 241, 247] payload_rows_per_dst=[5504, 5574, 5582, 5336, 5502, 5396, 5488, 5446] dup_rows_per_dst=[2750, 2764, 2706, 2650, 2740, 2694, 2734, 2670] combine_rows_per_src=[5504, 5574, 5582, 5336, 5502, 5396, 5488, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229120 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991232 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753344 weight_movement_bytes=18874368 total_nvlink_bytes=332627712 weight_to_token_ratio=0.060157 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38150 dup_rows=21790 dup_rows_remote=18824 dedup_ratio=0.332489 router_tokens_per_expert=[271, 261, 271, 236, 261, 262, 231, 267, 262, 264, 257, 254, 250, 236, 246, 261, 242, 249, 248, 266, 265, 252, 265, 258, 270, 281, 272, 259, 217, 271, 270, 245, 234, 243, 252, 250, 269, 265, 237, 255, 257, 275, 266, 252, 270, 249, 277, 281, 305, 218, 277, 276, 268, 252, 250, 284, 242, 240, 249, 239, 240, 272, 234, 246, 259, 255, 227, 245, 250, 246, 238, 267, 242, 257, 286, 273, 262, 247, 248, 236, 275, 268, 252, 270, 274, 271, 252, 232, 259, 256, 255, 252, 272, 243, 270, 250, 250, 269, 276, 252, 230, 260, 259, 262, 261, 277, 244, 257, 283, 242, 232, 261, 268, 266, 256, 267, 263, 248, 234, 228, 242, 255, 236, 225, 272, 273, 273, 214] payload_rows_per_dst=[5352, 5480, 5422, 5618, 5392, 5574, 5312, 5596] dup_rows_per_dst=[2452, 2808, 2690, 2840, 2684, 2942, 2408, 2966] combine_rows_per_src=[5352, 5480, 5422, 5618, 5392, 5574, 5312, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229632 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156582144 ingress_bytes=175226880 link_bytes=175226880 token_bytes=312934656 weight_movement_bytes=18874368 total_nvlink_bytes=331809024 weight_to_token_ratio=0.060314 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678937747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229728 combine_payload_bytes=157343744 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157573472 ingress_bytes=195092480 link_bytes=195092480 token_bytes=314917216 weight_movement_bytes=37748736 total_nvlink_bytes=352665952 weight_to_token_ratio=0.119869 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869280152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=228320 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156802016 ingress_bytes=213196800 link_bytes=213196800 token_bytes=313375712 weight_movement_bytes=56623104 total_nvlink_bytes=369998816 weight_to_token_ratio=0.180688 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042746838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228992 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156884608 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540224 weight_movement_bytes=18874368 total_nvlink_bytes=332414592 weight_to_token_ratio=0.060198 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38192 dup_rows=21868 dup_rows_remote=19186 dedup_ratio=0.333679 router_tokens_per_expert=[247, 257, 241, 274, 269, 305, 245, 270, 246, 278, 244, 260, 248, 245, 257, 266, 246, 273, 265, 249, 226, 254, 250, 263, 260, 258, 255, 272, 245, 254, 260, 279, 271, 280, 234, 225, 263, 255, 267, 257, 254, 235, 267, 241, 265, 284, 277, 257, 285, 243, 238, 262, 246, 256, 250, 255, 247, 253, 227, 276, 241, 272, 259, 269, 267, 215, 261, 253, 263, 256, 254, 231, 230, 254, 239, 247, 272, 257, 258, 251, 263, 229, 260, 260, 277, 239, 267, 242, 246, 257, 240, 277, 267, 291, 256, 249, 279, 247, 265, 233, 224, 265, 250, 267, 235, 253, 260, 260, 242, 269, 295, 245, 247, 245, 263, 247, 249, 246, 261, 258, 267, 244, 277, 264, 223, 280, 275, 233] payload_rows_per_dst=[5462, 5486, 5476, 5476, 5358, 5490, 5468, 5452] dup_rows_per_dst=[2842, 2732, 2788, 2682, 2658, 2750, 2710, 2706] combine_rows_per_src=[5462, 5486, 5476, 5476, 5358, 5490, 5468, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=227896 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156490296 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312752696 weight_movement_bytes=18874368 total_nvlink_bytes=331627064 weight_to_token_ratio=0.060349 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43462 payload_rows_remote=38038 dup_rows=22074 dup_rows_remote=19336 dedup_ratio=0.336823 router_tokens_per_expert=[247, 250, 258, 252, 236, 263, 249, 246, 252, 239, 261, 271, 233, 249, 278, 252, 222, 257, 266, 271, 252, 256, 242, 251, 277, 233, 277, 233, 254, 255, 263, 232, 261, 273, 222, 273, 243, 264, 257, 270, 266, 287, 255, 260, 265, 289, 231, 268, 238, 273, 288, 250, 263, 274, 272, 260, 249, 253, 257, 233, 250, 267, 224, 265, 253, 263, 252, 259, 284, 243, 238, 268, 256, 252, 267, 243, 276, 240, 241, 257, 270, 260, 260, 288, 259, 222, 253, 234, 258, 296, 255, 265, 263, 274, 253, 260, 254, 236, 283, 243, 253, 255, 265, 257, 237, 273, 221, 251, 274, 252, 234, 248, 222, 255, 259, 258, 288, 280, 258, 256, 245, 273, 236, 258, 249, 241, 272, 243] payload_rows_per_dst=[5424, 5532, 5484, 5600, 5456, 5338, 5398, 5230] dup_rows_per_dst=[2738, 2970, 2972, 3090, 2648, 2672, 2674, 2310] combine_rows_per_src=[5424, 5532, 5484, 5600, 5456, 5338, 5398, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38148 dup_rows=21874 dup_rows_remote=19120 dedup_ratio=0.333771 router_tokens_per_expert=[265, 259, 226, 254, 265, 274, 254, 256, 288, 254, 245, 265, 255, 268, 245, 266, 247, 278, 270, 254, 241, 243, 252, 223, 268, 241, 233, 270, 251, 249, 265, 255, 264, 235, 269, 247, 237, 246, 254, 236, 265, 270, 263, 229, 282, 240, 243, 250, 273, 251, 231, 278, 274, 250, 253, 275, 263, 261, 252, 257, 270, 258, 251, 277, 281, 251, 269, 249, 258, 253, 256, 264, 260, 238, 253, 245, 245, 250, 257, 249, 268, 241, 250, 257, 258, 251, 283, 270, 244, 253, 255, 279, 242, 279, 240, 264, 242, 267, 256, 282, 223, 265, 240, 277, 238, 230, 268, 227, 266, 230, 275, 293, 271, 262, 245, 240, 267, 252, 275, 217, 243, 278, 242, 248, 257, 257, 290, 250] payload_rows_per_dst=[5494, 5388, 5418, 5526, 5436, 5514, 5466, 5420] dup_rows_per_dst=[2784, 2692, 2642, 2822, 2720, 2754, 2692, 2768] combine_rows_per_src=[5494, 5388, 5418, 5526, 5436, 5514, 5466, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229512 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663944 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098376 weight_movement_bytes=18874368 total_nvlink_bytes=331972744 weight_to_token_ratio=0.060283 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=155803648 dispatch_meta_bytes=229496 combine_payload_bytes=155803648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156033144 ingress_bytes=174678016 link_bytes=174678016 token_bytes=311836792 weight_movement_bytes=18874368 total_nvlink_bytes=330711160 weight_to_token_ratio=0.060526 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673678802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38284 dup_rows=21776 dup_rows_remote=19072 dedup_ratio=0.332275 router_tokens_per_expert=[261, 241, 257, 289, 260, 228, 236, 262, 269, 258, 230, 247, 268, 256, 278, 272, 272, 271, 263, 266, 268, 247, 267, 242, 261, 245, 252, 271, 269, 260, 253, 277, 281, 249, 243, 272, 249, 262, 268, 283, 244, 244, 252, 265, 239, 263, 263, 261, 270, 254, 252, 267, 270, 238, 269, 262, 259, 262, 247, 243, 244, 255, 256, 258, 228, 255, 273, 247, 280, 255, 269, 257, 266, 261, 241, 254, 227, 260, 259, 258, 241, 256, 250, 259, 272, 273, 260, 257, 238, 239, 257, 250, 260, 252, 242, 263, 198, 247, 253, 243, 250, 271, 263, 239, 241, 264, 291, 252, 252, 235, 252, 251, 284, 250, 251, 250, 237, 251, 270, 237, 243, 277, 269, 257, 235, 256, 266, 234] payload_rows_per_dst=[5448, 5600, 5520, 5454, 5476, 5374, 5384, 5504] dup_rows_per_dst=[2776, 2768, 2756, 2758, 2704, 2764, 2620, 2630] combine_rows_per_src=[5448, 5600, 5520, 5454, 5476, 5374, 5384, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229072 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483280 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737488 weight_movement_bytes=18874368 total_nvlink_bytes=331611856 weight_to_token_ratio=0.060352 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38268 dup_rows=21710 dup_rows_remote=18942 dedup_ratio=0.331268 router_tokens_per_expert=[254, 267, 242, 294, 263, 261, 247, 229, 249, 237, 221, 278, 267, 278, 237, 263, 258, 245, 229, 238, 263, 246, 269, 248, 244, 248, 268, 254, 246, 286, 294, 257, 250, 257, 247, 265, 242, 271, 262, 243, 244, 234, 248, 247, 270, 245, 234, 253, 266, 255, 257, 236, 261, 238, 236, 269, 277, 227, 275, 270, 258, 234, 229, 254, 266, 271, 273, 261, 248, 228, 258, 255, 259, 264, 271, 253, 260, 244, 277, 285, 241, 276, 256, 243, 234, 265, 227, 248, 283, 256, 251, 257, 269, 255, 255, 283, 254, 275, 268, 242, 259, 255, 247, 266, 265, 273, 251, 268, 269, 252, 271, 248, 275, 251, 246, 266, 248, 231, 265, 270, 255, 257, 268, 242, 253, 247, 243, 282] payload_rows_per_dst=[5534, 5496, 5394, 5418, 5528, 5494, 5558, 5404] dup_rows_per_dst=[2640, 2690, 2630, 2666, 2818, 2704, 2768, 2794] combine_rows_per_src=[5534, 5496, 5394, 5418, 5528, 5494, 5558, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38290 dup_rows=21836 dup_rows_remote=19082 dedup_ratio=0.333191 router_tokens_per_expert=[276, 247, 246, 272, 253, 245, 255, 222, 247, 281, 226, 243, 278, 255, 272, 273, 260, 237, 248, 270, 263, 241, 264, 260, 219, 257, 265, 242, 285, 268, 253, 250, 250, 269, 277, 274, 262, 250, 251, 240, 256, 239, 275, 272, 278, 242, 261, 255, 269, 243, 245, 236, 259, 268, 251, 238, 244, 234, 256, 248, 277, 250, 260, 270, 278, 261, 237, 262, 229, 265, 272, 289, 262, 268, 234, 248, 230, 239, 259, 257, 241, 243, 259, 284, 280, 248, 227, 263, 256, 278, 274, 263, 270, 258, 277, 261, 237, 248, 259, 263, 272, 237, 254, 261, 250, 268, 229, 250, 233, 264, 230, 296, 257, 250, 246, 259, 266, 243, 246, 267, 243, 270, 254, 276, 229, 259, 256, 252] payload_rows_per_dst=[5476, 5410, 5538, 5488, 5434, 5500, 5400, 5454] dup_rows_per_dst=[2706, 2754, 2764, 2608, 2746, 2864, 2702, 2692] combine_rows_per_src=[5476, 5410, 5538, 5488, 5434, 5500, 5400, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43518 payload_rows_remote=38040 dup_rows=22018 dup_rows_remote=19070 dedup_ratio=0.335968 router_tokens_per_expert=[272, 248, 227, 270, 265, 242, 242, 287, 247, 257, 244, 261, 280, 242, 270, 275, 257, 279, 233, 275, 230, 265, 242, 256, 274, 271, 255, 266, 233, 240, 238, 280, 282, 257, 203, 259, 273, 261, 251, 284, 273, 270, 239, 263, 238, 261, 281, 265, 259, 263, 263, 273, 260, 254, 256, 234, 245, 253, 270, 260, 258, 261, 274, 262, 226, 245, 251, 248, 254, 243, 266, 262, 231, 234, 280, 251, 272, 271, 265, 270, 262, 255, 272, 238, 252, 282, 271, 252, 226, 248, 268, 259, 262, 255, 277, 247, 274, 254, 260, 244, 250, 248, 237, 252, 257, 261, 246, 282, 241, 241, 265, 272, 256, 252, 248, 226, 256, 267, 238, 224, 242, 247, 235, 255, 247, 254, 265, 249] payload_rows_per_dst=[5060, 5300, 5630, 5582, 5480, 5486, 5502, 5478] dup_rows_per_dst=[2300, 2522, 2940, 2852, 2676, 2918, 2862, 2948] combine_rows_per_src=[5060, 5300, 5630, 5582, 5480, 5486, 5502, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38366 dup_rows=21734 dup_rows_remote=19204 dedup_ratio=0.331635 router_tokens_per_expert=[268, 256, 257, 245, 257, 248, 260, 249, 282, 264, 258, 276, 267, 243, 248, 277, 268, 250, 228, 244, 250, 260, 223, 262, 277, 259, 272, 239, 261, 279, 238, 258, 265, 249, 232, 245, 258, 268, 224, 237, 268, 255, 256, 226, 243, 267, 234, 256, 268, 269, 280, 246, 261, 261, 231, 261, 252, 252, 241, 255, 233, 266, 243, 250, 255, 251, 284, 249, 241, 238, 263, 252, 280, 253, 256, 259, 264, 285, 243, 255, 230, 256, 258, 258, 301, 268, 276, 252, 250, 234, 272, 255, 240, 278, 241, 278, 256, 280, 240, 255, 248, 238, 262, 270, 255, 281, 231, 259, 276, 246, 233, 257, 260, 269, 265, 275, 262, 261, 258, 247, 242, 261, 250, 244, 272, 271, 240, 254] payload_rows_per_dst=[5494, 5420, 5436, 5490, 5482, 5542, 5450, 5488] dup_rows_per_dst=[2816, 2716, 2530, 2648, 2774, 2752, 2724, 2774] combine_rows_per_src=[5494, 5420, 5436, 5490, 5482, 5542, 5450, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229424 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040688 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851952 weight_movement_bytes=18874368 total_nvlink_bytes=332726320 weight_to_token_ratio=0.060138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228840 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974568 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720296 weight_movement_bytes=18874368 total_nvlink_bytes=332594664 weight_to_token_ratio=0.060163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38326 dup_rows=21766 dup_rows_remote=18968 dedup_ratio=0.332123 router_tokens_per_expert=[253, 251, 228, 239, 263, 251, 243, 260, 261, 257, 272, 271, 275, 261, 267, 255, 247, 289, 277, 267, 263, 249, 264, 266, 233, 262, 260, 254, 258, 247, 258, 286, 248, 229, 246, 239, 241, 251, 263, 257, 255, 227, 258, 247, 275, 247, 259, 264, 251, 240, 237, 276, 238, 251, 258, 269, 279, 285, 244, 289, 276, 260, 235, 233, 251, 240, 227, 251, 255, 270, 255, 260, 250, 240, 297, 245, 266, 291, 262, 235, 280, 218, 270, 271, 257, 258, 265, 276, 267, 221, 247, 254, 266, 240, 262, 246, 264, 244, 260, 248, 269, 250, 269, 276, 252, 248, 254, 227, 245, 236, 267, 255, 278, 259, 260, 223, 268, 272, 259, 273, 270, 253, 236, 250, 230, 287, 226, 253] payload_rows_per_dst=[5436, 5530, 5374, 5444, 5528, 5492, 5466, 5500] dup_rows_per_dst=[2778, 2830, 2638, 2798, 2662, 2704, 2662, 2694] combine_rows_per_src=[5436, 5530, 5374, 5444, 5528, 5492, 5466, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229488 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065328 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313901168 weight_movement_bytes=18874368 total_nvlink_bytes=332775536 weight_to_token_ratio=0.060128 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155811840 dispatch_meta_bytes=228440 combine_payload_bytes=155811840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156040280 ingress_bytes=174686208 link_bytes=174686208 token_bytes=311852120 weight_movement_bytes=18874368 total_nvlink_bytes=330726488 weight_to_token_ratio=0.060523 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673757294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230280 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157377416 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314524552 weight_movement_bytes=18874368 total_nvlink_bytes=333398920 weight_to_token_ratio=0.060009 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38132 dup_rows=21862 dup_rows_remote=18864 dedup_ratio=0.333588 router_tokens_per_expert=[291, 229, 242, 218, 266, 228, 241, 233, 280, 255, 242, 247, 250, 265, 288, 230, 240, 270, 246, 256, 260, 237, 266, 232, 253, 250, 249, 265, 248, 261, 250, 235, 243, 290, 267, 248, 280, 258, 239, 247, 243, 277, 277, 255, 250, 253, 252, 250, 265, 267, 291, 234, 255, 256, 257, 238, 262, 244, 240, 288, 249, 271, 271, 251, 272, 243, 300, 227, 245, 252, 254, 226, 270, 267, 247, 260, 277, 277, 275, 254, 269, 273, 272, 278, 271, 231, 270, 234, 271, 232, 252, 276, 276, 278, 268, 248, 256, 260, 244, 237, 245, 224, 286, 255, 256, 244, 259, 262, 264, 264, 269, 265, 254, 265, 239, 259, 258, 278, 231, 249, 264, 261, 242, 235, 257, 241, 263, 246] payload_rows_per_dst=[5542, 5412, 5522, 5632, 5496, 5430, 5434, 5206] dup_rows_per_dst=[2998, 2638, 2832, 3018, 2686, 2690, 2652, 2348] combine_rows_per_src=[5542, 5412, 5522, 5632, 5496, 5430, 5434, 5206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43932 payload_rows_remote=38456 dup_rows=21604 dup_rows_remote=18892 dedup_ratio=0.329651 router_tokens_per_expert=[277, 282, 239, 256, 249, 237, 275, 256, 267, 218, 274, 257, 263, 246, 262, 237, 240, 273, 261, 281, 238, 255, 250, 237, 279, 246, 257, 268, 247, 246, 265, 259, 262, 269, 247, 247, 274, 252, 235, 250, 270, 251, 236, 231, 272, 234, 262, 243, 254, 265, 266, 274, 267, 254, 257, 263, 244, 254, 238, 250, 266, 258, 251, 251, 270, 255, 270, 264, 257, 247, 253, 255, 246, 291, 242, 253, 237, 257, 258, 287, 251, 264, 250, 236, 270, 236, 258, 251, 268, 255, 296, 243, 239, 234, 278, 265, 272, 280, 261, 257, 266, 279, 254, 247, 265, 261, 255, 268, 262, 246, 245, 250, 301, 214, 247, 212, 258, 251, 282, 256, 237, 251, 246, 246, 244, 269, 280, 226] payload_rows_per_dst=[5482, 5504, 5362, 5548, 5520, 5476, 5554, 5486] dup_rows_per_dst=[2708, 2700, 2708, 2676, 2764, 2712, 2782, 2554] combine_rows_per_src=[5482, 5504, 5362, 5548, 5520, 5476, 5554, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229176 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157212472 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314195768 weight_movement_bytes=56623104 total_nvlink_bytes=370818872 weight_to_token_ratio=0.180216 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=227984 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156416656 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312605328 weight_movement_bytes=18874368 total_nvlink_bytes=331479696 weight_to_token_ratio=0.060378 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38474 dup_rows=21616 dup_rows_remote=18856 dedup_ratio=0.329834 router_tokens_per_expert=[247, 235, 263, 235, 256, 248, 256, 251, 247, 257, 260, 254, 256, 242, 273, 265, 241, 266, 228, 272, 228, 278, 270, 279, 262, 248, 272, 254, 274, 282, 261, 251, 264, 290, 267, 259, 256, 254, 227, 269, 241, 268, 222, 248, 277, 263, 265, 245, 246, 255, 263, 268, 269, 242, 271, 228, 293, 246, 261, 244, 259, 251, 245, 266, 251, 253, 247, 260, 250, 290, 237, 266, 252, 261, 232, 249, 265, 254, 285, 251, 233, 265, 250, 265, 286, 241, 272, 250, 277, 266, 264, 278, 250, 250, 263, 231, 252, 219, 235, 267, 256, 280, 262, 269, 255, 242, 255, 274, 248, 243, 242, 249, 262, 249, 246, 247, 272, 246, 242, 246, 264, 230, 264, 256, 266, 251, 249, 253] payload_rows_per_dst=[5488, 5554, 5566, 5532, 5446, 5474, 5390, 5470] dup_rows_per_dst=[2602, 2778, 2664, 2682, 2760, 2808, 2706, 2616] combine_rows_per_src=[5488, 5554, 5566, 5532, 5446, 5474, 5390, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157515776 dispatch_meta_bytes=229392 combine_payload_bytes=157515776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157745168 ingress_bytes=176390144 link_bytes=176390144 token_bytes=315260944 weight_movement_bytes=18874368 total_nvlink_bytes=334135312 weight_to_token_ratio=0.059869 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690083570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38194 dup_rows=21862 dup_rows_remote=19222 dedup_ratio=0.333588 router_tokens_per_expert=[269, 253, 236, 234, 269, 263, 242, 261, 276, 279, 275, 234, 279, 250, 275, 252, 279, 233, 248, 290, 279, 251, 238, 281, 239, 261, 243, 291, 279, 243, 221, 267, 267, 241, 256, 263, 247, 229, 245, 264, 248, 249, 250, 238, 273, 249, 251, 294, 278, 245, 281, 238, 270, 265, 225, 264, 251, 247, 245, 265, 240, 272, 254, 261, 253, 242, 262, 247, 275, 238, 265, 258, 232, 250, 256, 259, 264, 260, 234, 263, 245, 241, 252, 272, 236, 259, 268, 267, 271, 252, 267, 245, 276, 280, 236, 256, 268, 262, 252, 253, 230, 267, 252, 250, 258, 289, 245, 250, 230, 277, 238, 239, 243, 263, 264, 260, 252, 237, 251, 271, 260, 262, 252, 256, 248, 239, 250, 264] payload_rows_per_dst=[5530, 5476, 5440, 5408, 5486, 5422, 5480, 5432] dup_rows_per_dst=[2764, 2810, 2688, 2794, 2630, 2824, 2640, 2712] combine_rows_per_src=[5530, 5476, 5440, 5408, 5486, 5422, 5480, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38244 dup_rows=21672 dup_rows_remote=18868 dedup_ratio=0.330688 router_tokens_per_expert=[245, 262, 243, 229, 256, 270, 250, 250, 266, 232, 244, 272, 255, 254, 243, 268, 222, 259, 266, 245, 267, 241, 264, 252, 239, 262, 265, 277, 281, 263, 241, 271, 243, 290, 262, 242, 240, 222, 268, 260, 271, 246, 279, 268, 259, 253, 285, 254, 260, 260, 268, 273, 292, 263, 259, 270, 244, 264, 241, 245, 241, 267, 235, 262, 255, 254, 289, 214, 266, 254, 253, 270, 274, 229, 248, 230, 243, 267, 234, 253, 217, 265, 233, 223, 255, 256, 255, 249, 298, 277, 261, 258, 267, 257, 258, 239, 277, 230, 260, 260, 245, 248, 244, 296, 274, 248, 258, 275, 260, 270, 268, 236, 247, 221, 240, 274, 232, 287, 248, 248, 253, 256, 271, 257, 254, 250, 268, 272] payload_rows_per_dst=[5340, 5334, 5476, 5446, 5530, 5606, 5512, 5620] dup_rows_per_dst=[2738, 2554, 2808, 2590, 2688, 2700, 2790, 2804] combine_rows_per_src=[5340, 5334, 5476, 5446, 5530, 5606, 5512, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38418 dup_rows=21696 dup_rows_remote=19078 dedup_ratio=0.331055 router_tokens_per_expert=[286, 258, 274, 267, 273, 278, 255, 259, 254, 271, 293, 271, 276, 272, 246, 232, 243, 264, 236, 231, 247, 266, 254, 264, 257, 238, 266, 258, 275, 226, 244, 251, 268, 219, 250, 252, 265, 234, 241, 269, 256, 241, 262, 232, 259, 269, 251, 240, 249, 271, 260, 256, 237, 247, 277, 260, 266, 294, 263, 251, 263, 260, 267, 259, 246, 256, 252, 252, 263, 261, 254, 278, 257, 261, 258, 280, 266, 248, 232, 242, 266, 278, 284, 265, 254, 238, 242, 250, 252, 256, 246, 289, 250, 248, 263, 255, 245, 223, 254, 251, 278, 240, 208, 255, 247, 258, 256, 269, 268, 241, 239, 250, 254, 247, 267, 244, 228, 270, 252, 273, 268, 231, 253, 259, 238, 278, 256, 253] payload_rows_per_dst=[5668, 5422, 5402, 5580, 5474, 5444, 5366, 5484] dup_rows_per_dst=[2862, 2618, 2614, 2780, 2738, 2828, 2598, 2658] combine_rows_per_src=[5668, 5422, 5402, 5580, 5474, 5444, 5366, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38114 dup_rows=21866 dup_rows_remote=19000 dedup_ratio=0.333649 router_tokens_per_expert=[230, 249, 255, 240, 242, 280, 239, 257, 254, 263, 245, 258, 266, 285, 240, 280, 247, 257, 266, 241, 279, 251, 243, 271, 239, 272, 265, 260, 268, 231, 269, 240, 288, 295, 281, 278, 273, 275, 265, 231, 274, 270, 234, 258, 254, 250, 271, 214, 250, 269, 248, 239, 243, 242, 259, 246, 245, 272, 263, 249, 224, 243, 282, 241, 270, 282, 237, 232, 238, 244, 248, 261, 268, 277, 251, 285, 272, 277, 244, 253, 261, 243, 250, 283, 258, 255, 240, 248, 231, 252, 263, 267, 243, 251, 259, 251, 288, 245, 225, 254, 244, 259, 272, 250, 247, 236, 258, 253, 260, 249, 250, 275, 254, 244, 250, 268, 248, 263, 280, 252, 263, 264, 238, 271, 252, 243, 286, 225] payload_rows_per_dst=[5464, 5484, 5556, 5398, 5520, 5428, 5360, 5460] dup_rows_per_dst=[2702, 2714, 2866, 2632, 2758, 2682, 2770, 2742] combine_rows_per_src=[5464, 5484, 5556, 5398, 5520, 5428, 5360, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157589504 dispatch_meta_bytes=229320 combine_payload_bytes=157589504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157818824 ingress_bytes=176463872 link_bytes=176463872 token_bytes=315408328 weight_movement_bytes=18874368 total_nvlink_bytes=334282696 weight_to_token_ratio=0.059841 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690789996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38430 dup_rows=21784 dup_rows_remote=19068 dedup_ratio=0.332397 router_tokens_per_expert=[250, 274, 271, 241, 251, 244, 234, 266, 220, 264, 252, 237, 244, 259, 281, 255, 261, 255, 233, 251, 266, 250, 234, 272, 267, 264, 280, 275, 254, 233, 247, 239, 259, 258, 263, 254, 246, 274, 269, 250, 244, 280, 245, 265, 253, 295, 277, 269, 243, 275, 221, 243, 271, 268, 262, 259, 245, 253, 261, 217, 267, 251, 249, 234, 281, 246, 268, 254, 273, 238, 266, 206, 250, 242, 226, 266, 246, 241, 259, 241, 273, 265, 241, 241, 268, 265, 255, 275, 238, 270, 271, 252, 271, 272, 252, 251, 250, 230, 268, 281, 218, 252, 261, 274, 245, 243, 271, 270, 259, 258, 254, 267, 284, 256, 260, 232, 264, 247, 263, 260, 261, 278, 258, 283, 269, 233, 261, 251] payload_rows_per_dst=[5424, 5494, 5562, 5322, 5434, 5528, 5456, 5532] dup_rows_per_dst=[2662, 2668, 2840, 2716, 2572, 2792, 2746, 2788] combine_rows_per_src=[5424, 5494, 5562, 5322, 5434, 5528, 5456, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229664 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156672288 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114912 weight_movement_bytes=37748736 total_nvlink_bytes=350863648 weight_to_token_ratio=0.120559 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=228448 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156875872 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313523296 weight_movement_bytes=18874368 total_nvlink_bytes=332397664 weight_to_token_ratio=0.060201 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229984 combine_payload_bytes=157360128 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157590112 ingress_bytes=232857600 link_bytes=232857600 token_bytes=314950240 weight_movement_bytes=75497472 total_nvlink_bytes=390447712 weight_to_token_ratio=0.239712 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.231126950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228456 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156343400 ingress_bytes=212738048 link_bytes=212738048 token_bytes=312458344 weight_movement_bytes=56623104 total_nvlink_bytes=369081448 weight_to_token_ratio=0.181218 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038351302e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38266 dup_rows=21762 dup_rows_remote=18966 dedup_ratio=0.332062 router_tokens_per_expert=[279, 264, 300, 255, 261, 238, 264, 261, 270, 252, 255, 264, 210, 253, 247, 237, 268, 221, 268, 290, 260, 248, 229, 282, 268, 262, 263, 262, 254, 262, 268, 270, 243, 257, 267, 263, 277, 244, 267, 253, 240, 242, 248, 298, 237, 268, 284, 261, 256, 236, 265, 283, 235, 259, 255, 264, 229, 263, 250, 242, 270, 253, 243, 261, 289, 250, 256, 255, 255, 240, 234, 269, 266, 249, 241, 237, 244, 272, 271, 265, 266, 261, 221, 236, 257, 252, 237, 272, 280, 250, 217, 260, 237, 274, 255, 267, 264, 254, 264, 275, 258, 242, 230, 281, 256, 241, 250, 254, 273, 225, 238, 263, 233, 265, 234, 253, 225, 246, 286, 265, 260, 267, 237, 258, 264, 244, 251, 279] payload_rows_per_dst=[5508, 5612, 5500, 5472, 5658, 5358, 5318, 5348] dup_rows_per_dst=[2796, 2888, 2730, 2656, 2890, 2510, 2652, 2640] combine_rows_per_src=[5508, 5612, 5500, 5472, 5658, 5358, 5318, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229992 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157639272 ingress_bytes=176283648 link_bytes=176283648 token_bytes=315048552 weight_movement_bytes=18874368 total_nvlink_bytes=333922920 weight_to_token_ratio=0.059909 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689063178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38154 dup_rows=21942 dup_rows_remote=19158 dedup_ratio=0.334808 router_tokens_per_expert=[245, 266, 248, 257, 265, 258, 249, 265, 263, 240, 259, 256, 246, 231, 244, 272, 291, 214, 259, 259, 260, 276, 267, 263, 254, 230, 245, 260, 261, 252, 265, 258, 234, 267, 250, 248, 269, 263, 264, 248, 244, 233, 238, 257, 269, 251, 248, 279, 275, 242, 251, 276, 253, 248, 253, 223, 278, 267, 283, 259, 257, 253, 256, 233, 232, 259, 254, 245, 256, 250, 278, 256, 230, 268, 247, 237, 243, 278, 242, 287, 258, 256, 265, 302, 247, 233, 256, 252, 258, 240, 245, 278, 236, 242, 267, 277, 252, 288, 265, 269, 245, 252, 249, 263, 248, 255, 268, 268, 239, 240, 237, 286, 258, 247, 255, 259, 267, 245, 248, 269, 241, 250, 268, 272, 265, 256, 264, 259] payload_rows_per_dst=[5432, 5502, 5426, 5450, 5360, 5440, 5506, 5478] dup_rows_per_dst=[2696, 2726, 2698, 2764, 2764, 2784, 2742, 2768] combine_rows_per_src=[5432, 5502, 5426, 5450, 5360, 5440, 5506, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228928 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966464 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704000 weight_movement_bytes=18874368 total_nvlink_bytes=332578368 weight_to_token_ratio=0.060166 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229248 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156508032 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312786816 weight_movement_bytes=37748736 total_nvlink_bytes=350535552 weight_to_token_ratio=0.120685 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38240 dup_rows=21884 dup_rows_remote=19098 dedup_ratio=0.333923 router_tokens_per_expert=[229, 241, 280, 249, 270, 245, 268, 253, 267, 234, 259, 268, 263, 266, 260, 264, 273, 231, 252, 234, 277, 260, 274, 245, 259, 255, 258, 257, 266, 263, 275, 249, 253, 241, 263, 252, 237, 256, 250, 266, 255, 236, 262, 258, 262, 276, 304, 245, 247, 276, 233, 241, 271, 248, 238, 251, 261, 261, 240, 234, 272, 226, 254, 247, 274, 241, 268, 223, 240, 272, 225, 258, 245, 281, 268, 270, 255, 250, 245, 284, 273, 246, 234, 268, 249, 272, 248, 251, 233, 267, 245, 247, 299, 231, 241, 244, 263, 269, 256, 281, 251, 255, 267, 249, 241, 239, 256, 276, 268, 232, 261, 244, 246, 250, 264, 260, 248, 254, 274, 264, 282, 276, 260, 235, 253, 247, 268, 272] payload_rows_per_dst=[5456, 5468, 5454, 5330, 5412, 5446, 5506, 5580] dup_rows_per_dst=[2776, 2788, 2778, 2670, 2786, 2650, 2710, 2726] combine_rows_per_src=[5456, 5468, 5454, 5330, 5412, 5446, 5506, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38334 dup_rows=21732 dup_rows_remote=18980 dedup_ratio=0.331604 router_tokens_per_expert=[273, 243, 277, 251, 235, 263, 252, 260, 267, 265, 259, 260, 259, 256, 286, 224, 272, 265, 279, 283, 230, 269, 264, 234, 251, 251, 257, 250, 243, 255, 254, 262, 273, 272, 241, 253, 231, 269, 257, 245, 237, 259, 272, 261, 234, 257, 241, 299, 248, 249, 265, 273, 266, 287, 265, 250, 262, 249, 244, 251, 255, 246, 289, 271, 253, 257, 273, 251, 277, 272, 258, 254, 273, 241, 249, 265, 280, 254, 265, 242, 231, 261, 265, 237, 276, 248, 273, 252, 208, 254, 246, 291, 201, 235, 258, 255, 233, 252, 283, 279, 248, 249, 247, 263, 241, 247, 251, 255, 271, 260, 246, 234, 259, 261, 244, 250, 256, 251, 277, 215, 271, 244, 245, 239, 272, 239, 264, 247] payload_rows_per_dst=[5522, 5456, 5418, 5540, 5584, 5358, 5470, 5456] dup_rows_per_dst=[2738, 2678, 2784, 2800, 2744, 2624, 2752, 2612] combine_rows_per_src=[5522, 5456, 5418, 5540, 5584, 5358, 5470, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38174 dup_rows=21860 dup_rows_remote=19140 dedup_ratio=0.333557 router_tokens_per_expert=[264, 254, 265, 257, 289, 255, 288, 239, 271, 259, 229, 268, 273, 232, 293, 290, 250, 227, 255, 262, 270, 267, 259, 238, 247, 257, 287, 252, 262, 271, 274, 252, 266, 261, 248, 236, 250, 262, 252, 239, 250, 278, 247, 239, 276, 271, 250, 270, 245, 258, 261, 249, 246, 266, 267, 232, 253, 217, 243, 264, 258, 270, 261, 250, 244, 245, 244, 238, 265, 260, 246, 244, 253, 272, 267, 230, 253, 255, 256, 251, 259, 261, 232, 255, 252, 252, 273, 291, 247, 249, 255, 252, 241, 269, 227, 249, 246, 229, 259, 283, 227, 272, 249, 259, 248, 255, 262, 291, 250, 244, 266, 239, 255, 227, 259, 248, 268, 262, 246, 259, 256, 263, 252, 253, 271, 275, 260, 257] payload_rows_per_dst=[5398, 5288, 5416, 5590, 5400, 5394, 5688, 5502] dup_rows_per_dst=[2680, 2398, 2722, 2864, 2698, 2734, 3044, 2720] combine_rows_per_src=[5398, 5288, 5416, 5590, 5400, 5394, 5688, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38148 dup_rows=21882 dup_rows_remote=19136 dedup_ratio=0.333893 router_tokens_per_expert=[275, 257, 263, 253, 257, 253, 236, 243, 251, 240, 266, 273, 276, 244, 229, 244, 248, 250, 253, 264, 259, 233, 265, 276, 259, 253, 264, 267, 252, 271, 246, 266, 243, 254, 268, 258, 255, 260, 271, 252, 237, 232, 275, 247, 258, 247, 232, 242, 261, 241, 258, 279, 281, 266, 229, 267, 233, 269, 239, 268, 251, 282, 273, 246, 237, 306, 241, 274, 265, 255, 267, 244, 231, 236, 246, 268, 273, 218, 279, 254, 271, 289, 278, 280, 263, 262, 253, 251, 262, 271, 254, 257, 224, 236, 252, 239, 274, 284, 264, 255, 249, 262, 245, 243, 286, 256, 261, 216, 284, 252, 276, 274, 258, 254, 251, 260, 245, 234, 255, 244, 244, 267, 258, 240, 250, 237, 255, 239] payload_rows_per_dst=[5460, 5506, 5430, 5496, 5400, 5476, 5526, 5360] dup_rows_per_dst=[2660, 2746, 2770, 2790, 2788, 2670, 2836, 2622] combine_rows_per_src=[5460, 5506, 5430, 5496, 5400, 5476, 5526, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38242 dup_rows=21950 dup_rows_remote=19322 dedup_ratio=0.334930 router_tokens_per_expert=[261, 224, 262, 250, 257, 262, 263, 253, 251, 240, 229, 259, 277, 282, 250, 249, 269, 246, 245, 265, 277, 247, 258, 256, 257, 262, 247, 266, 248, 284, 262, 273, 242, 245, 260, 254, 233, 253, 227, 257, 242, 236, 252, 254, 247, 271, 253, 260, 241, 259, 242, 265, 249, 249, 220, 243, 256, 262, 245, 258, 244, 258, 283, 270, 269, 259, 260, 257, 262, 241, 230, 272, 264, 269, 239, 271, 260, 254, 243, 265, 289, 251, 250, 246, 279, 262, 278, 256, 295, 247, 233, 251, 267, 249, 260, 288, 254, 235, 252, 270, 251, 260, 250, 259, 286, 237, 264, 279, 245, 266, 263, 260, 269, 218, 265, 251, 273, 258, 223, 242, 269, 248, 266, 263, 257, 240, 256, 262] payload_rows_per_dst=[5418, 5484, 5344, 5432, 5500, 5504, 5484, 5420] dup_rows_per_dst=[2720, 2840, 2628, 2656, 2730, 2898, 2778, 2700] combine_rows_per_src=[5418, 5484, 5344, 5432, 5500, 5504, 5484, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38286 dup_rows=21890 dup_rows_remote=19200 dedup_ratio=0.334015 router_tokens_per_expert=[236, 262, 275, 263, 254, 262, 275, 265, 254, 240, 257, 242, 263, 246, 272, 226, 253, 287, 263, 244, 246, 249, 253, 260, 244, 259, 251, 256, 246, 259, 248, 257, 276, 249, 255, 225, 256, 271, 250, 263, 253, 265, 268, 242, 263, 247, 269, 271, 261, 261, 225, 277, 258, 258, 240, 219, 245, 269, 235, 242, 244, 278, 251, 262, 270, 252, 232, 261, 275, 265, 273, 259, 269, 247, 252, 232, 259, 227, 256, 227, 238, 256, 282, 281, 286, 235, 263, 270, 254, 276, 254, 260, 272, 242, 268, 265, 254, 262, 267, 279, 234, 254, 243, 270, 262, 254, 268, 236, 247, 272, 260, 280, 248, 262, 261, 261, 242, 269, 229, 255, 278, 251, 216, 230, 264, 255, 268, 264] payload_rows_per_dst=[5396, 5468, 5486, 5360, 5422, 5546, 5550, 5418] dup_rows_per_dst=[2788, 2682, 2760, 2690, 2690, 2858, 2734, 2688] combine_rows_per_src=[5396, 5468, 5486, 5360, 5422, 5546, 5550, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229352 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860392 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491432 weight_movement_bytes=37748736 total_nvlink_bytes=351240168 weight_to_token_ratio=0.120414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229256 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245320 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261384 weight_movement_bytes=18874368 total_nvlink_bytes=333135752 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229256 combine_payload_bytes=156360704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156589960 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950664 weight_movement_bytes=37748736 total_nvlink_bytes=350699400 weight_to_token_ratio=0.120622 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229136 combine_payload_bytes=156254208 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156483344 ingress_bytes=212877312 link_bytes=212877312 token_bytes=312737552 weight_movement_bytes=56623104 total_nvlink_bytes=369360656 weight_to_token_ratio=0.181056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039685661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=230256 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156869488 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313508720 weight_movement_bytes=18874368 total_nvlink_bytes=332383088 weight_to_token_ratio=0.060204 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38044 dup_rows=21938 dup_rows_remote=18986 dedup_ratio=0.334747 router_tokens_per_expert=[283, 258, 258, 247, 258, 271, 255, 250, 277, 244, 256, 250, 269, 256, 288, 251, 252, 228, 262, 250, 254, 252, 258, 270, 263, 256, 240, 232, 257, 242, 271, 256, 258, 284, 256, 250, 273, 258, 249, 265, 267, 255, 240, 262, 249, 265, 262, 259, 252, 251, 269, 258, 260, 272, 252, 235, 251, 245, 259, 255, 243, 250, 227, 271, 219, 257, 232, 242, 268, 270, 246, 262, 244, 258, 268, 295, 262, 265, 231, 264, 263, 252, 269, 243, 263, 302, 239, 230, 244, 269, 251, 283, 254, 244, 237, 246, 272, 243, 265, 262, 267, 221, 267, 254, 275, 256, 261, 249, 273, 254, 253, 251, 248, 265, 253, 255, 269, 265, 244, 245, 255, 258, 255, 252, 228, 254, 238, 273] payload_rows_per_dst=[5554, 5418, 5574, 5448, 5422, 5480, 5298, 5404] dup_rows_per_dst=[2952, 2668, 2790, 2652, 2792, 2810, 2564, 2710] combine_rows_per_src=[5554, 5418, 5574, 5448, 5422, 5480, 5298, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229944 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157049400 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868856 weight_movement_bytes=18874368 total_nvlink_bytes=332743224 weight_to_token_ratio=0.060135 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38356 dup_rows=21644 dup_rows_remote=18810 dedup_ratio=0.330261 router_tokens_per_expert=[251, 269, 229, 253, 251, 254, 231, 253, 244, 255, 256, 255, 278, 257, 263, 270, 264, 241, 263, 266, 271, 265, 276, 243, 259, 242, 237, 259, 257, 274, 270, 243, 252, 260, 275, 247, 263, 229, 260, 246, 232, 227, 255, 243, 234, 246, 235, 236, 235, 247, 280, 276, 256, 243, 256, 273, 280, 258, 262, 259, 265, 247, 249, 270, 280, 274, 241, 269, 264, 240, 258, 235, 251, 267, 254, 258, 249, 260, 265, 245, 271, 263, 283, 266, 253, 263, 261, 240, 270, 239, 254, 262, 277, 261, 234, 288, 244, 232, 253, 240, 245, 280, 242, 267, 261, 228, 246, 247, 258, 266, 243, 262, 282, 255, 266, 256, 257, 272, 227, 242, 266, 232, 263, 259, 292, 269, 264, 262] payload_rows_per_dst=[5472, 5508, 5330, 5546, 5514, 5536, 5418, 5568] dup_rows_per_dst=[2666, 2752, 2550, 2766, 2706, 2834, 2610, 2760] combine_rows_per_src=[5472, 5508, 5330, 5546, 5514, 5536, 5418, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=155828224 dispatch_meta_bytes=228120 combine_payload_bytes=155828224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156056344 ingress_bytes=193576960 link_bytes=193576960 token_bytes=311884568 weight_movement_bytes=37748736 total_nvlink_bytes=349633304 weight_to_token_ratio=0.121034 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854759185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=228664 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157334840 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314441016 weight_movement_bytes=18874368 total_nvlink_bytes=333315384 weight_to_token_ratio=0.060025 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43984 payload_rows_remote=38414 dup_rows=21552 dup_rows_remote=18912 dedup_ratio=0.328857 router_tokens_per_expert=[243, 224, 226, 253, 270, 249, 263, 249, 262, 250, 239, 259, 263, 245, 280, 239, 229, 243, 235, 274, 274, 234, 268, 233, 258, 235, 277, 286, 266, 248, 264, 244, 257, 254, 294, 257, 276, 281, 244, 233, 238, 269, 248, 261, 252, 288, 260, 237, 259, 267, 253, 235, 256, 267, 267, 246, 254, 253, 269, 256, 249, 254, 264, 236, 267, 270, 252, 260, 275, 242, 281, 246, 245, 263, 248, 248, 262, 254, 240, 252, 261, 249, 269, 241, 262, 272, 254, 266, 235, 238, 260, 265, 280, 272, 240, 256, 239, 277, 262, 258, 263, 250, 268, 261, 238, 268, 254, 243, 243, 229, 260, 247, 273, 252, 259, 251, 281, 243, 279, 293, 259, 246, 247, 259, 263, 248, 260, 254] payload_rows_per_dst=[5454, 5464, 5514, 5520, 5570, 5512, 5404, 5546] dup_rows_per_dst=[2574, 2672, 2784, 2650, 2640, 2728, 2716, 2788] combine_rows_per_src=[5454, 5464, 5514, 5520, 5570, 5512, 5404, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38230 dup_rows=21830 dup_rows_remote=19106 dedup_ratio=0.333099 router_tokens_per_expert=[239, 247, 252, 263, 252, 246, 235, 278, 263, 227, 274, 264, 260, 233, 256, 249, 268, 261, 265, 264, 245, 250, 275, 258, 257, 274, 246, 256, 265, 262, 292, 252, 257, 263, 255, 246, 231, 261, 237, 237, 262, 244, 271, 229, 242, 268, 229, 271, 247, 259, 266, 262, 262, 240, 269, 265, 234, 244, 273, 250, 219, 254, 245, 258, 272, 262, 266, 289, 264, 249, 258, 263, 233, 263, 256, 242, 238, 225, 250, 262, 244, 261, 277, 261, 253, 257, 289, 252, 252, 265, 265, 249, 245, 252, 252, 273, 237, 253, 234, 259, 247, 261, 259, 267, 276, 237, 275, 248, 243, 280, 254, 270, 261, 267, 246, 265, 247, 276, 245, 249, 279, 238, 259, 263, 277, 248, 278, 253] payload_rows_per_dst=[5452, 5516, 5370, 5438, 5432, 5538, 5476, 5484] dup_rows_per_dst=[2624, 2864, 2636, 2656, 2752, 2756, 2724, 2818] combine_rows_per_src=[5452, 5516, 5370, 5438, 5432, 5538, 5476, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38206 dup_rows=21862 dup_rows_remote=19016 dedup_ratio=0.333588 router_tokens_per_expert=[267, 236, 287, 244, 243, 242, 265, 283, 266, 253, 244, 263, 231, 265, 273, 236, 267, 243, 228, 229, 233, 258, 248, 247, 250, 256, 266, 264, 224, 265, 260, 285, 263, 252, 279, 259, 253, 263, 251, 233, 250, 251, 272, 232, 235, 280, 247, 267, 256, 267, 293, 260, 246, 269, 260, 276, 265, 254, 255, 241, 259, 235, 232, 252, 265, 258, 239, 253, 258, 249, 253, 270, 277, 280, 256, 237, 236, 270, 233, 251, 253, 240, 254, 264, 254, 265, 267, 235, 234, 274, 263, 231, 260, 272, 268, 286, 230, 279, 228, 264, 246, 261, 286, 255, 288, 283, 262, 230, 255, 234, 227, 254, 272, 259, 278, 236, 252, 248, 278, 252, 242, 269, 275, 265, 245, 243, 277, 262] payload_rows_per_dst=[5428, 5450, 5418, 5486, 5478, 5390, 5556, 5468] dup_rows_per_dst=[2768, 2596, 2756, 2754, 2684, 2528, 2930, 2846] combine_rows_per_src=[5428, 5450, 5418, 5486, 5478, 5390, 5556, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38264 dup_rows=21704 dup_rows_remote=18872 dedup_ratio=0.331177 router_tokens_per_expert=[254, 256, 266, 256, 239, 247, 238, 232, 223, 258, 253, 254, 222, 278, 263, 283, 261, 268, 232, 267, 247, 295, 233, 232, 248, 239, 229, 233, 273, 278, 241, 264, 240, 257, 285, 267, 251, 270, 237, 261, 251, 268, 276, 262, 279, 267, 247, 282, 232, 259, 266, 272, 267, 263, 240, 256, 254, 293, 257, 255, 266, 242, 244, 247, 271, 288, 269, 247, 275, 242, 235, 248, 250, 218, 266, 246, 250, 260, 244, 225, 289, 264, 284, 253, 248, 266, 246, 250, 236, 278, 230, 244, 252, 285, 243, 248, 220, 236, 253, 274, 259, 238, 269, 248, 261, 259, 251, 257, 252, 250, 269, 291, 246, 262, 284, 259, 261, 257, 253, 257, 273, 261, 221, 265, 247, 268, 293, 249] payload_rows_per_dst=[5438, 5396, 5568, 5586, 5464, 5452, 5400, 5528] dup_rows_per_dst=[2606, 2684, 2832, 2640, 2604, 2780, 2774, 2784] combine_rows_per_src=[5438, 5396, 5568, 5586, 5464, 5452, 5400, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38106 dup_rows=21890 dup_rows_remote=19226 dedup_ratio=0.334015 router_tokens_per_expert=[256, 236, 262, 249, 231, 248, 282, 263, 272, 260, 259, 242, 299, 234, 282, 255, 229, 269, 263, 264, 246, 262, 237, 273, 249, 258, 268, 249, 244, 269, 268, 254, 247, 257, 268, 273, 243, 261, 226, 239, 259, 270, 257, 241, 271, 276, 266, 246, 258, 252, 242, 244, 258, 255, 245, 268, 250, 266, 238, 242, 282, 255, 252, 244, 285, 266, 264, 257, 257, 250, 241, 240, 243, 271, 260, 242, 294, 256, 268, 259, 255, 246, 231, 243, 265, 253, 258, 256, 228, 240, 264, 243, 222, 262, 266, 276, 262, 199, 256, 256, 277, 249, 236, 243, 246, 241, 247, 247, 256, 257, 280, 266, 245, 278, 254, 248, 232, 265, 270, 277, 269, 242, 265, 258, 257, 295, 277, 274] payload_rows_per_dst=[5528, 5540, 5502, 5354, 5458, 5364, 5378, 5522] dup_rows_per_dst=[2732, 2664, 2698, 2748, 2848, 2652, 2658, 2890] combine_rows_per_src=[5528, 5540, 5502, 5354, 5458, 5364, 5378, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38324 dup_rows=21630 dup_rows_remote=18948 dedup_ratio=0.330048 router_tokens_per_expert=[254, 243, 258, 238, 232, 246, 265, 252, 283, 242, 241, 232, 272, 255, 249, 283, 275, 242, 247, 245, 239, 239, 273, 231, 262, 250, 278, 252, 258, 284, 243, 239, 262, 267, 261, 272, 237, 226, 261, 236, 257, 260, 269, 251, 272, 256, 251, 287, 253, 241, 272, 273, 243, 232, 254, 251, 252, 264, 272, 276, 251, 269, 270, 259, 276, 260, 266, 276, 244, 226, 229, 252, 244, 276, 263, 247, 247, 251, 267, 251, 240, 292, 251, 257, 241, 264, 228, 250, 252, 260, 246, 273, 236, 260, 266, 246, 263, 240, 275, 260, 232, 277, 270, 245, 268, 254, 234, 259, 230, 255, 279, 271, 280, 243, 239, 244, 256, 270, 271, 259, 268, 231, 273, 278, 262, 274, 254, 258] payload_rows_per_dst=[5366, 5442, 5506, 5582, 5448, 5468, 5510, 5584] dup_rows_per_dst=[2724, 2672, 2744, 2682, 2702, 2656, 2714, 2736] combine_rows_per_src=[5366, 5442, 5506, 5582, 5448, 5468, 5510, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229304 combine_payload_bytes=157343744 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157573048 ingress_bytes=195092480 link_bytes=195092480 token_bytes=314916792 weight_movement_bytes=37748736 total_nvlink_bytes=352665528 weight_to_token_ratio=0.119869 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869280152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228888 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156720664 ingress_bytes=213114880 link_bytes=213114880 token_bytes=313212440 weight_movement_bytes=56623104 total_nvlink_bytes=369835544 weight_to_token_ratio=0.180782 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041961921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229344 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819424 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409504 weight_movement_bytes=18874368 total_nvlink_bytes=332283872 weight_to_token_ratio=0.060223 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38292 dup_rows=21722 dup_rows_remote=18964 dedup_ratio=0.331451 router_tokens_per_expert=[271, 264, 281, 249, 263, 270, 257, 277, 253, 277, 275, 234, 239, 236, 244, 250, 280, 234, 270, 292, 242, 250, 288, 262, 269, 255, 270, 253, 244, 239, 278, 275, 214, 244, 256, 246, 267, 271, 249, 250, 248, 224, 259, 246, 255, 267, 244, 235, 220, 252, 236, 245, 277, 246, 247, 271, 219, 263, 222, 256, 269, 274, 227, 257, 251, 235, 247, 260, 258, 259, 231, 234, 255, 279, 271, 250, 259, 285, 275, 247, 283, 258, 245, 240, 233, 244, 260, 280, 247, 243, 249, 281, 256, 271, 232, 282, 264, 258, 255, 266, 295, 242, 265, 255, 260, 269, 265, 245, 230, 264, 262, 282, 246, 250, 233, 244, 272, 264, 264, 252, 250, 267, 279, 249, 260, 262, 265, 237] payload_rows_per_dst=[5522, 5522, 5340, 5432, 5670, 5498, 5326, 5504] dup_rows_per_dst=[2758, 2880, 2610, 2530, 3000, 2710, 2550, 2684] combine_rows_per_src=[5522, 5522, 5340, 5432, 5670, 5498, 5326, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228544 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156957888 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687232 weight_movement_bytes=18874368 total_nvlink_bytes=332561600 weight_to_token_ratio=0.060169 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229088 combine_payload_bytes=156975104 prefetch_bytes=0 reduce_bytes=113246208 egress_bytes=157204192 ingress_bytes=270221312 link_bytes=270221312 token_bytes=314179296 weight_movement_bytes=113246208 total_nvlink_bytes=427425504 weight_to_token_ratio=0.360451 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.589127654e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229328 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311504 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393680 weight_movement_bytes=37748736 total_nvlink_bytes=350142416 weight_to_token_ratio=0.120837 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43992 payload_rows_remote=38396 dup_rows=21544 dup_rows_remote=18804 dedup_ratio=0.328735 router_tokens_per_expert=[265, 257, 246, 229, 261, 255, 241, 270, 253, 261, 274, 237, 225, 274, 241, 232, 253, 258, 276, 250, 269, 253, 252, 257, 253, 246, 263, 281, 254, 236, 269, 252, 239, 243, 275, 250, 230, 260, 254, 260, 268, 256, 256, 263, 244, 282, 242, 251, 246, 246, 250, 250, 231, 270, 268, 261, 273, 252, 272, 268, 244, 262, 255, 241, 283, 251, 264, 260, 277, 253, 233, 260, 241, 232, 243, 237, 273, 245, 290, 280, 260, 256, 253, 252, 250, 252, 270, 245, 274, 271, 254, 257, 298, 265, 249, 262, 261, 231, 256, 241, 239, 253, 256, 238, 230, 265, 258, 244, 256, 270, 269, 269, 254, 260, 262, 273, 238, 230, 244, 259, 260, 249, 284, 266, 249, 257, 272, 280] payload_rows_per_dst=[5416, 5556, 5442, 5490, 5488, 5596, 5462, 5542] dup_rows_per_dst=[2626, 2688, 2704, 2688, 2756, 2740, 2610, 2732] combine_rows_per_src=[5416, 5556, 5442, 5490, 5488, 5596, 5462, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229024 combine_payload_bytes=156844032 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157073056 ingress_bytes=232341504 link_bytes=232341504 token_bytes=313917088 weight_movement_bytes=75497472 total_nvlink_bytes=389414560 weight_to_token_ratio=0.240501 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226181972e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=228800 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157498816 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314768832 weight_movement_bytes=56623104 total_nvlink_bytes=371391936 weight_to_token_ratio=0.179888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38358 dup_rows=21796 dup_rows_remote=19092 dedup_ratio=0.332581 router_tokens_per_expert=[244, 251, 230, 292, 245, 279, 268, 251, 244, 257, 255, 287, 255, 260, 246, 247, 232, 263, 241, 271, 271, 267, 255, 237, 271, 237, 234, 272, 251, 256, 271, 264, 259, 260, 254, 261, 264, 221, 281, 248, 234, 260, 241, 247, 254, 238, 234, 239, 264, 254, 254, 274, 255, 270, 227, 261, 261, 253, 272, 271, 259, 281, 268, 250, 273, 257, 252, 286, 249, 256, 260, 265, 268, 227, 292, 264, 268, 239, 252, 281, 263, 264, 279, 252, 236, 251, 250, 263, 250, 246, 270, 261, 278, 262, 279, 258, 266, 280, 270, 240, 239, 254, 243, 263, 251, 258, 229, 242, 274, 211, 250, 258, 267, 237, 246, 249, 266, 238, 262, 270, 250, 228, 267, 240, 262, 255, 220, 259] payload_rows_per_dst=[5500, 5496, 5390, 5330, 5704, 5472, 5466, 5382] dup_rows_per_dst=[2722, 2690, 2600, 2528, 3016, 2818, 2718, 2704] combine_rows_per_src=[5500, 5496, 5390, 5330, 5704, 5472, 5466, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38354 dup_rows=21736 dup_rows_remote=19050 dedup_ratio=0.331665 router_tokens_per_expert=[281, 286, 237, 265, 251, 237, 263, 254, 246, 245, 243, 263, 268, 244, 239, 266, 263, 241, 245, 258, 235, 266, 250, 244, 262, 252, 262, 247, 257, 238, 246, 218, 254, 252, 272, 256, 242, 263, 279, 267, 272, 247, 244, 260, 259, 261, 227, 238, 263, 244, 282, 270, 272, 274, 260, 246, 268, 239, 247, 269, 286, 248, 269, 248, 260, 252, 258, 274, 277, 221, 253, 252, 268, 268, 261, 242, 247, 235, 253, 245, 260, 274, 263, 269, 285, 254, 260, 264, 250, 268, 219, 258, 276, 253, 264, 296, 233, 268, 264, 233, 261, 239, 265, 269, 251, 260, 244, 236, 279, 256, 245, 280, 267, 235, 255, 247, 246, 231, 247, 260, 278, 246, 260, 257, 257, 254, 270, 246] payload_rows_per_dst=[5446, 5370, 5478, 5576, 5446, 5546, 5448, 5490] dup_rows_per_dst=[2730, 2598, 2708, 2794, 2686, 2880, 2718, 2622] combine_rows_per_src=[5446, 5370, 5478, 5576, 5446, 5546, 5448, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38142 dup_rows=21844 dup_rows_remote=19034 dedup_ratio=0.333313 router_tokens_per_expert=[242, 268, 278, 243, 260, 258, 231, 272, 262, 266, 236, 254, 235, 274, 251, 250, 272, 236, 262, 244, 249, 268, 263, 277, 239, 259, 254, 258, 269, 237, 258, 252, 248, 264, 237, 236, 252, 237, 259, 231, 254, 265, 252, 272, 251, 274, 251, 234, 245, 286, 248, 250, 253, 285, 276, 265, 264, 278, 257, 280, 254, 266, 247, 260, 261, 284, 239, 271, 261, 262, 228, 256, 240, 242, 244, 249, 266, 264, 244, 237, 241, 260, 266, 242, 247, 243, 241, 274, 249, 239, 274, 238, 251, 264, 269, 244, 284, 289, 241, 256, 248, 283, 233, 284, 253, 240, 270, 248, 247, 257, 275, 272, 277, 244, 248, 250, 263, 251, 267, 277, 262, 261, 230, 254, 253, 232, 256, 265] payload_rows_per_dst=[5406, 5474, 5384, 5490, 5444, 5464, 5550, 5480] dup_rows_per_dst=[2754, 2720, 2650, 2938, 2652, 2620, 2810, 2700] combine_rows_per_src=[5406, 5474, 5384, 5490, 5444, 5464, 5550, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38070 dup_rows=21950 dup_rows_remote=19222 dedup_ratio=0.334930 router_tokens_per_expert=[242, 244, 251, 262, 246, 234, 248, 254, 213, 279, 257, 272, 257, 268, 271, 261, 250, 254, 249, 237, 265, 248, 246, 267, 255, 246, 248, 280, 267, 236, 263, 260, 240, 251, 257, 286, 271, 264, 261, 267, 263, 228, 252, 239, 253, 256, 249, 268, 267, 246, 263, 251, 230, 252, 246, 248, 273, 268, 265, 263, 275, 263, 262, 250, 246, 267, 281, 246, 255, 278, 278, 249, 242, 247, 245, 260, 311, 277, 270, 230, 238, 227, 257, 250, 252, 263, 235, 273, 264, 253, 234, 233, 245, 272, 267, 262, 229, 282, 247, 257, 260, 258, 237, 259, 251, 250, 240, 253, 262, 254, 255, 226, 281, 265, 243, 240, 254, 261, 292, 240, 255, 264, 270, 266, 274, 259, 266, 254] payload_rows_per_dst=[5364, 5458, 5498, 5516, 5526, 5378, 5322, 5524] dup_rows_per_dst=[2754, 2684, 2712, 2728, 2838, 2672, 2718, 2844] combine_rows_per_src=[5364, 5458, 5498, 5516, 5526, 5378, 5322, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38294 dup_rows=21814 dup_rows_remote=19062 dedup_ratio=0.332855 router_tokens_per_expert=[283, 242, 276, 256, 255, 247, 251, 239, 257, 289, 259, 241, 236, 265, 261, 255, 279, 252, 250, 280, 276, 258, 246, 253, 268, 263, 227, 270, 254, 248, 254, 264, 280, 226, 249, 250, 250, 271, 275, 264, 275, 249, 251, 244, 281, 245, 246, 234, 251, 230, 257, 231, 243, 284, 253, 241, 290, 255, 256, 240, 257, 264, 276, 252, 256, 234, 245, 268, 258, 262, 259, 249, 237, 274, 252, 282, 239, 268, 270, 227, 270, 232, 234, 255, 241, 265, 241, 262, 242, 262, 252, 270, 272, 258, 260, 271, 258, 278, 258, 261, 227, 280, 242, 260, 237, 249, 251, 245, 262, 260, 255, 252, 269, 266, 237, 263, 280, 259, 269, 264, 219, 247, 238, 266, 242, 244, 277, 262] payload_rows_per_dst=[5496, 5468, 5428, 5426, 5474, 5476, 5498, 5456] dup_rows_per_dst=[2728, 2816, 2752, 2734, 2686, 2698, 2652, 2748] combine_rows_per_src=[5496, 5468, 5428, 5426, 5474, 5476, 5498, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38240 dup_rows=21830 dup_rows_remote=19078 dedup_ratio=0.333099 router_tokens_per_expert=[283, 273, 259, 268, 256, 278, 282, 271, 266, 249, 246, 257, 277, 247, 258, 255, 278, 220, 254, 289, 263, 253, 270, 244, 245, 260, 259, 262, 257, 236, 273, 246, 260, 253, 255, 262, 244, 239, 246, 245, 254, 243, 251, 255, 264, 257, 263, 244, 225, 259, 261, 256, 261, 261, 240, 251, 255, 266, 268, 275, 258, 278, 219, 244, 283, 253, 252, 252, 271, 274, 272, 236, 241, 288, 277, 231, 260, 280, 258, 265, 245, 279, 242, 245, 230, 255, 257, 252, 253, 267, 230, 271, 256, 228, 241, 270, 240, 241, 266, 244, 263, 236, 248, 264, 246, 259, 230, 262, 237, 248, 262, 257, 252, 244, 249, 233, 260, 279, 277, 262, 263, 235, 266, 272, 235, 245, 295, 238] payload_rows_per_dst=[5564, 5466, 5530, 5430, 5556, 5366, 5360, 5434] dup_rows_per_dst=[2886, 2752, 2540, 2724, 2830, 2676, 2646, 2776] combine_rows_per_src=[5564, 5466, 5530, 5430, 5556, 5366, 5360, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229800 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157344168 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314458536 weight_movement_bytes=18874368 total_nvlink_bytes=333332904 weight_to_token_ratio=0.060022 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=228704 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156458336 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312687968 weight_movement_bytes=18874368 total_nvlink_bytes=331562336 weight_to_token_ratio=0.060362 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38040 dup_rows=21870 dup_rows_remote=18996 dedup_ratio=0.333710 router_tokens_per_expert=[264, 290, 260, 261, 272, 258, 270, 265, 229, 280, 263, 282, 263, 272, 255, 259, 284, 284, 271, 253, 239, 263, 263, 275, 255, 253, 239, 263, 257, 302, 256, 230, 241, 269, 252, 244, 248, 244, 268, 256, 252, 252, 245, 242, 242, 241, 254, 264, 250, 245, 241, 235, 258, 230, 259, 246, 247, 277, 220, 289, 244, 260, 283, 255, 260, 228, 246, 242, 269, 256, 246, 248, 236, 266, 257, 265, 249, 254, 249, 279, 277, 233, 255, 268, 233, 250, 255, 237, 262, 216, 265, 260, 239, 248, 248, 257, 239, 259, 266, 251, 249, 271, 261, 270, 273, 248, 246, 270, 229, 251, 248, 240, 296, 263, 255, 254, 253, 257, 249, 252, 250, 265, 259, 276, 252, 252, 263, 265] payload_rows_per_dst=[5626, 5564, 5400, 5388, 5404, 5338, 5440, 5506] dup_rows_per_dst=[2874, 2796, 2628, 2690, 2696, 2668, 2702, 2816] combine_rows_per_src=[5626, 5564, 5400, 5388, 5404, 5338, 5440, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229616 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327600 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425584 weight_movement_bytes=18874368 total_nvlink_bytes=333299952 weight_to_token_ratio=0.060028 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=229168 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156163888 ingress_bytes=193683456 link_bytes=193683456 token_bytes=312098608 weight_movement_bytes=37748736 total_nvlink_bytes=349847344 weight_to_token_ratio=0.120951 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855779577e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38304 dup_rows=21690 dup_rows_remote=18980 dedup_ratio=0.330963 router_tokens_per_expert=[277, 246, 266, 287, 251, 232, 237, 262, 238, 263, 258, 263, 250, 240, 267, 258, 273, 278, 223, 280, 289, 220, 286, 277, 272, 276, 240, 245, 263, 253, 259, 260, 261, 266, 263, 264, 250, 286, 244, 235, 268, 246, 249, 221, 264, 246, 255, 245, 258, 254, 247, 258, 279, 248, 283, 229, 252, 276, 255, 241, 224, 240, 254, 272, 250, 274, 267, 244, 278, 238, 244, 259, 249, 256, 262, 242, 244, 256, 242, 286, 253, 244, 267, 255, 268, 267, 275, 277, 240, 239, 228, 280, 274, 258, 258, 243, 264, 271, 248, 259, 244, 244, 239, 222, 261, 233, 256, 243, 255, 253, 256, 267, 249, 254, 270, 257, 231, 252, 268, 258, 217, 269, 278, 240, 262, 292, 268, 249] payload_rows_per_dst=[5434, 5576, 5418, 5470, 5438, 5542, 5378, 5590] dup_rows_per_dst=[2756, 2812, 2708, 2670, 2744, 2710, 2652, 2638] combine_rows_per_src=[5434, 5576, 5418, 5470, 5438, 5542, 5378, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229272 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860312 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491352 weight_movement_bytes=37748736 total_nvlink_bytes=351240088 weight_to_token_ratio=0.120414 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229424 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157081648 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313933872 weight_movement_bytes=56623104 total_nvlink_bytes=370556976 weight_to_token_ratio=0.180366 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155811840 dispatch_meta_bytes=228144 combine_payload_bytes=155811840 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156039984 ingress_bytes=231309312 link_bytes=231309312 token_bytes=311851824 weight_movement_bytes=75497472 total_nvlink_bytes=387349296 weight_to_token_ratio=0.242094 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216292016e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229136 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122320 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015504 weight_movement_bytes=37748736 total_nvlink_bytes=351764240 weight_to_token_ratio=0.120213 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38290 dup_rows=21722 dup_rows_remote=18914 dedup_ratio=0.331451 router_tokens_per_expert=[272, 259, 271, 251, 251, 253, 253, 231, 253, 273, 248, 225, 244, 253, 248, 244, 255, 242, 234, 254, 263, 278, 282, 245, 261, 256, 250, 251, 286, 224, 243, 278, 246, 247, 234, 246, 249, 255, 275, 263, 251, 281, 241, 262, 257, 275, 238, 247, 270, 257, 291, 249, 230, 275, 267, 281, 222, 254, 252, 256, 262, 253, 266, 260, 264, 246, 268, 241, 248, 272, 252, 265, 268, 250, 245, 248, 270, 250, 253, 286, 249, 273, 215, 256, 267, 267, 243, 265, 258, 259, 264, 268, 249, 239, 273, 253, 259, 249, 237, 282, 232, 247, 256, 250, 263, 250, 255, 234, 250, 271, 270, 238, 224, 240, 247, 270, 261, 247, 282, 264, 267, 287, 271, 274, 255, 242, 255, 272] payload_rows_per_dst=[5390, 5358, 5430, 5510, 5550, 5516, 5536, 5524] dup_rows_per_dst=[2668, 2434, 2684, 2626, 2860, 2680, 2962, 2808] combine_rows_per_src=[5390, 5358, 5430, 5510, 5550, 5516, 5536, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38074 dup_rows=21878 dup_rows_remote=18974 dedup_ratio=0.333832 router_tokens_per_expert=[240, 219, 267, 243, 236, 252, 257, 277, 270, 268, 253, 260, 254, 261, 272, 251, 253, 271, 230, 242, 247, 257, 266, 253, 254, 254, 280, 262, 250, 233, 262, 267, 255, 228, 256, 257, 242, 266, 260, 280, 244, 250, 229, 247, 267, 252, 277, 251, 257, 255, 265, 237, 273, 268, 250, 225, 262, 272, 267, 264, 260, 270, 248, 246, 257, 237, 250, 256, 242, 261, 261, 265, 267, 219, 271, 265, 256, 262, 239, 238, 262, 263, 260, 236, 263, 246, 245, 274, 229, 239, 268, 236, 243, 239, 244, 252, 250, 272, 268, 289, 243, 280, 279, 273, 247, 243, 272, 239, 265, 286, 272, 266, 270, 283, 263, 257, 252, 245, 232, 237, 256, 253, 261, 257, 237, 281, 274, 280] payload_rows_per_dst=[5418, 5392, 5470, 5490, 5406, 5454, 5584, 5444] dup_rows_per_dst=[2742, 2770, 2652, 2748, 2686, 2544, 2904, 2832] combine_rows_per_src=[5418, 5392, 5470, 5490, 5406, 5454, 5584, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38332 dup_rows=21730 dup_rows_remote=19090 dedup_ratio=0.331573 router_tokens_per_expert=[237, 256, 258, 240, 261, 263, 265, 264, 270, 272, 244, 231, 243, 275, 248, 241, 248, 274, 243, 250, 242, 286, 267, 259, 247, 274, 250, 229, 276, 253, 272, 283, 265, 254, 237, 271, 245, 246, 290, 226, 258, 255, 258, 279, 243, 246, 260, 250, 237, 232, 262, 281, 250, 250, 240, 268, 283, 252, 279, 242, 267, 238, 254, 265, 268, 244, 233, 253, 258, 246, 287, 253, 244, 245, 239, 256, 237, 264, 255, 275, 262, 235, 252, 241, 282, 281, 261, 213, 267, 266, 246, 280, 258, 243, 249, 259, 248, 258, 231, 237, 252, 234, 251, 220, 247, 262, 271, 283, 256, 252, 296, 241, 242, 265, 252, 250, 272, 252, 254, 278, 250, 270, 253, 275, 255, 244, 282, 279] payload_rows_per_dst=[5402, 5496, 5472, 5468, 5474, 5514, 5432, 5548] dup_rows_per_dst=[2734, 2810, 2694, 2732, 2640, 2676, 2646, 2798] combine_rows_per_src=[5402, 5496, 5472, 5468, 5474, 5514, 5432, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38226 dup_rows=21850 dup_rows_remote=18990 dedup_ratio=0.333405 router_tokens_per_expert=[274, 266, 249, 234, 250, 283, 241, 237, 277, 249, 247, 232, 277, 246, 268, 232, 270, 224, 253, 270, 236, 251, 274, 260, 283, 257, 253, 258, 246, 220, 262, 248, 236, 271, 251, 274, 246, 247, 242, 250, 268, 262, 279, 243, 274, 240, 288, 252, 277, 288, 237, 270, 264, 232, 258, 252, 260, 245, 263, 255, 263, 272, 247, 277, 247, 251, 270, 263, 232, 231, 253, 234, 281, 284, 242, 259, 233, 273, 243, 266, 283, 243, 249, 277, 276, 253, 239, 279, 253, 237, 240, 271, 263, 274, 265, 277, 274, 247, 257, 267, 244, 281, 257, 272, 233, 264, 252, 258, 241, 245, 256, 242, 238, 226, 240, 268, 256, 269, 249, 244, 225, 269, 278, 286, 248, 241, 251, 239] payload_rows_per_dst=[5448, 5352, 5496, 5460, 5512, 5510, 5496, 5412] dup_rows_per_dst=[2676, 2778, 2750, 2860, 2612, 2848, 2684, 2642] combine_rows_per_src=[5448, 5352, 5496, 5460, 5512, 5510, 5496, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38418 dup_rows=21684 dup_rows_remote=18972 dedup_ratio=0.330872 router_tokens_per_expert=[252, 262, 233, 275, 274, 247, 272, 268, 251, 251, 218, 261, 214, 247, 255, 276, 261, 258, 232, 272, 268, 265, 229, 249, 239, 249, 239, 261, 253, 265, 257, 276, 258, 249, 234, 268, 252, 242, 256, 271, 266, 242, 249, 261, 250, 260, 248, 254, 261, 264, 258, 283, 261, 255, 260, 240, 274, 291, 266, 282, 295, 271, 288, 240, 240, 238, 219, 273, 294, 263, 263, 240, 251, 257, 274, 259, 240, 248, 256, 266, 252, 263, 269, 257, 233, 242, 244, 239, 284, 260, 244, 219, 242, 258, 269, 277, 257, 271, 257, 287, 274, 272, 252, 237, 246, 259, 228, 255, 251, 265, 252, 232, 242, 241, 225, 267, 234, 244, 237, 278, 236, 256, 278, 278, 263, 265, 260, 258] payload_rows_per_dst=[5468, 5434, 5482, 5576, 5442, 5442, 5544, 5464] dup_rows_per_dst=[2644, 2712, 2638, 3002, 2720, 2662, 2646, 2660] combine_rows_per_src=[5468, 5434, 5482, 5576, 5442, 5442, 5544, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38156 dup_rows=21782 dup_rows_remote=18924 dedup_ratio=0.332367 router_tokens_per_expert=[284, 219, 274, 255, 268, 264, 238, 251, 266, 270, 247, 266, 277, 282, 260, 253, 234, 250, 235, 259, 246, 253, 255, 258, 267, 224, 269, 275, 238, 227, 272, 249, 235, 262, 242, 245, 274, 238, 285, 272, 287, 293, 279, 279, 245, 261, 264, 267, 248, 286, 255, 243, 261, 245, 251, 271, 243, 272, 237, 265, 219, 254, 241, 257, 279, 256, 254, 299, 234, 256, 257, 268, 256, 258, 260, 241, 246, 275, 257, 261, 240, 248, 264, 262, 227, 263, 234, 282, 260, 230, 259, 252, 263, 246, 269, 253, 220, 260, 240, 264, 249, 264, 252, 278, 254, 242, 255, 259, 221, 263, 274, 278, 287, 258, 238, 243, 256, 249, 250, 242, 263, 253, 238, 255, 224, 250, 252, 267] payload_rows_per_dst=[5532, 5406, 5598, 5392, 5528, 5430, 5446, 5422] dup_rows_per_dst=[2816, 2616, 2858, 2704, 2786, 2674, 2700, 2628] combine_rows_per_src=[5532, 5406, 5598, 5392, 5528, 5430, 5446, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=228816 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157064656 ingress_bytes=194584576 link_bytes=194584576 token_bytes=313900496 weight_movement_bytes=37748736 total_nvlink_bytes=351649232 weight_to_token_ratio=0.120257 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864413666e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38358 dup_rows=21664 dup_rows_remote=18876 dedup_ratio=0.330566 router_tokens_per_expert=[247, 262, 269, 195, 258, 262, 260, 265, 264, 263, 271, 263, 238, 246, 240, 272, 287, 240, 230, 256, 259, 274, 240, 234, 239, 249, 249, 266, 235, 259, 248, 262, 257, 274, 249, 258, 226, 244, 228, 237, 234, 257, 278, 255, 261, 260, 255, 271, 281, 266, 246, 230, 254, 259, 257, 244, 232, 250, 259, 253, 252, 253, 273, 293, 257, 234, 248, 264, 218, 253, 262, 246, 271, 251, 291, 260, 259, 258, 281, 266, 225, 278, 256, 253, 284, 266, 272, 261, 252, 268, 258, 278, 282, 262, 237, 241, 248, 247, 256, 244, 248, 257, 272, 253, 281, 302, 227, 256, 242, 252, 261, 283, 239, 283, 258, 247, 263, 285, 245, 277, 264, 254, 253, 257, 242, 244, 256, 232] payload_rows_per_dst=[5514, 5464, 5660, 5454, 5514, 5338, 5440, 5488] dup_rows_per_dst=[2788, 2590, 2932, 2750, 2724, 2504, 2666, 2710] combine_rows_per_src=[5514, 5464, 5660, 5454, 5514, 5338, 5440, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=228192 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156179296 ingress_bytes=174825472 link_bytes=174825472 token_bytes=312130400 weight_movement_bytes=18874368 total_nvlink_bytes=331004768 weight_to_token_ratio=0.060469 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675091653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229688 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157237560 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314245432 weight_movement_bytes=37748736 total_nvlink_bytes=351994168 weight_to_token_ratio=0.120125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=228864 combine_payload_bytes=156573696 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156802560 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376256 weight_movement_bytes=18874368 total_nvlink_bytes=332250624 weight_to_token_ratio=0.060229 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38252 dup_rows=21884 dup_rows_remote=19226 dedup_ratio=0.333923 router_tokens_per_expert=[267, 244, 228, 244, 294, 285, 242, 270, 244, 262, 248, 266, 248, 240, 236, 266, 267, 232, 233, 271, 288, 285, 272, 269, 247, 275, 248, 279, 268, 249, 261, 277, 284, 240, 254, 283, 251, 273, 267, 250, 295, 235, 249, 241, 267, 258, 233, 238, 277, 261, 278, 265, 246, 256, 256, 235, 249, 240, 250, 224, 267, 250, 254, 241, 273, 254, 233, 245, 250, 290, 244, 260, 263, 269, 249, 241, 261, 272, 261, 244, 244, 258, 272, 277, 246, 262, 239, 266, 265, 244, 264, 245, 231, 271, 228, 217, 249, 256, 256, 285, 244, 231, 262, 268, 253, 274, 248, 270, 240, 233, 261, 258, 258, 248, 246, 265, 234, 241, 270, 290, 268, 266, 263, 253, 239, 263, 225, 241] payload_rows_per_dst=[5442, 5604, 5470, 5398, 5482, 5400, 5448, 5408] dup_rows_per_dst=[2726, 2838, 2766, 2700, 2736, 2658, 2728, 2732] combine_rows_per_src=[5442, 5604, 5470, 5398, 5482, 5400, 5448, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229560 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589688 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314949816 weight_movement_bytes=18874368 total_nvlink_bytes=333824184 weight_to_token_ratio=0.059928 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228320 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156515296 ingress_bytes=212910080 link_bytes=212910080 token_bytes=312802272 weight_movement_bytes=56623104 total_nvlink_bytes=369425376 weight_to_token_ratio=0.181019 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039999628e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=228936 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157343304 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314457672 weight_movement_bytes=18874368 total_nvlink_bytes=333332040 weight_to_token_ratio=0.060022 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229912 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156910104 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313590296 weight_movement_bytes=18874368 total_nvlink_bytes=332464664 weight_to_token_ratio=0.060188 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38004 dup_rows=21988 dup_rows_remote=19114 dedup_ratio=0.335510 router_tokens_per_expert=[247, 247, 239, 255, 258, 253, 249, 258, 254, 231, 277, 253, 250, 251, 261, 248, 250, 261, 272, 276, 250, 246, 245, 240, 266, 282, 256, 258, 261, 275, 260, 255, 276, 286, 275, 262, 291, 228, 258, 266, 245, 231, 240, 234, 240, 241, 260, 220, 245, 274, 260, 246, 257, 235, 263, 265, 288, 240, 247, 268, 256, 239, 245, 258, 264, 241, 272, 254, 266, 276, 258, 246, 261, 290, 263, 251, 280, 252, 239, 240, 224, 275, 237, 259, 250, 251, 249, 253, 237, 249, 228, 254, 270, 256, 265, 270, 282, 258, 269, 248, 263, 268, 238, 258, 256, 237, 248, 272, 245, 258, 273, 251, 258, 244, 275, 273, 255, 269, 238, 264, 262, 252, 250, 269, 235, 271, 239, 287] payload_rows_per_dst=[5392, 5474, 5324, 5344, 5406, 5462, 5602, 5544] dup_rows_per_dst=[2626, 2832, 2626, 2692, 2694, 2806, 2838, 2874] combine_rows_per_src=[5392, 5474, 5324, 5344, 5406, 5462, 5602, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38254 dup_rows=21764 dup_rows_remote=19086 dedup_ratio=0.332092 router_tokens_per_expert=[288, 244, 256, 247, 256, 253, 244, 248, 240, 240, 274, 244, 260, 240, 255, 249, 268, 280, 288, 256, 243, 292, 264, 250, 262, 248, 237, 248, 241, 277, 249, 275, 229, 266, 278, 251, 257, 263, 233, 265, 243, 276, 251, 255, 258, 252, 252, 262, 285, 256, 239, 262, 261, 278, 268, 258, 240, 271, 256, 284, 263, 260, 265, 284, 258, 242, 242, 258, 239, 256, 261, 249, 241, 264, 253, 264, 266, 275, 256, 240, 235, 234, 234, 258, 258, 267, 231, 243, 252, 279, 257, 252, 226, 255, 249, 226, 216, 234, 261, 262, 277, 267, 273, 240, 265, 243, 237, 294, 266, 292, 227, 244, 262, 251, 270, 256, 257, 258, 250, 271, 240, 251, 272, 247, 235, 266, 263, 264] payload_rows_per_dst=[5392, 5510, 5530, 5552, 5432, 5366, 5518, 5472] dup_rows_per_dst=[2684, 2846, 2652, 2908, 2696, 2546, 2678, 2754] combine_rows_per_src=[5392, 5510, 5530, 5552, 5432, 5366, 5518, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38188 dup_rows=21838 dup_rows_remote=18956 dedup_ratio=0.333221 router_tokens_per_expert=[254, 264, 239, 259, 243, 232, 258, 260, 260, 241, 251, 274, 225, 258, 289, 275, 246, 280, 265, 279, 238, 257, 260, 277, 238, 255, 256, 258, 244, 253, 251, 271, 260, 253, 273, 257, 268, 283, 264, 236, 241, 267, 261, 259, 253, 274, 249, 236, 252, 267, 253, 256, 258, 259, 266, 258, 248, 257, 228, 234, 247, 260, 243, 241, 266, 268, 262, 249, 269, 278, 264, 283, 287, 258, 249, 234, 250, 263, 265, 251, 253, 292, 253, 270, 249, 255, 241, 254, 226, 265, 257, 262, 259, 265, 249, 257, 242, 249, 250, 242, 252, 257, 270, 263, 249, 240, 245, 259, 254, 264, 252, 245, 275, 261, 257, 229, 240, 263, 256, 240, 273, 259, 257, 250, 252, 222, 269, 258] payload_rows_per_dst=[5496, 5490, 5488, 5418, 5510, 5510, 5334, 5452] dup_rows_per_dst=[2668, 2766, 2780, 2636, 2882, 2704, 2732, 2670] combine_rows_per_src=[5496, 5490, 5488, 5418, 5510, 5510, 5334, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38238 dup_rows=21770 dup_rows_remote=18874 dedup_ratio=0.332184 router_tokens_per_expert=[232, 271, 277, 285, 254, 247, 251, 262, 237, 248, 239, 276, 275, 260, 271, 242, 266, 266, 250, 276, 269, 261, 274, 279, 221, 248, 245, 238, 256, 258, 274, 247, 256, 290, 252, 255, 222, 258, 250, 248, 267, 266, 273, 251, 263, 249, 224, 245, 277, 247, 251, 305, 289, 275, 284, 238, 259, 246, 251, 264, 256, 253, 274, 243, 264, 259, 249, 228, 250, 254, 245, 248, 272, 226, 255, 243, 266, 244, 250, 234, 264, 250, 273, 247, 226, 242, 240, 253, 270, 217, 242, 257, 265, 247, 269, 265, 263, 256, 247, 278, 264, 240, 281, 260, 258, 249, 258, 235, 259, 239, 278, 259, 248, 269, 270, 232, 235, 267, 232, 268, 240, 253, 257, 272, 278, 259, 251, 263] payload_rows_per_dst=[5558, 5484, 5460, 5528, 5358, 5402, 5526, 5450] dup_rows_per_dst=[2696, 2772, 2678, 2896, 2616, 2652, 2722, 2738] combine_rows_per_src=[5558, 5484, 5460, 5528, 5358, 5402, 5526, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155664384 dispatch_meta_bytes=228472 combine_payload_bytes=155664384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155892856 ingress_bytes=174538752 link_bytes=174538752 token_bytes=311557240 weight_movement_bytes=18874368 total_nvlink_bytes=330431608 weight_to_token_ratio=0.060581 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672344443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38312 dup_rows=21696 dup_rows_remote=18858 dedup_ratio=0.331055 router_tokens_per_expert=[240, 237, 260, 265, 239, 258, 231, 253, 241, 228, 255, 236, 276, 267, 232, 273, 254, 274, 260, 249, 243, 255, 273, 252, 255, 263, 252, 274, 302, 240, 273, 264, 288, 247, 252, 238, 243, 241, 245, 275, 267, 265, 257, 245, 256, 274, 236, 256, 291, 264, 271, 274, 279, 268, 262, 256, 251, 258, 229, 257, 239, 240, 259, 265, 264, 252, 261, 246, 260, 265, 242, 246, 261, 230, 259, 274, 257, 242, 290, 261, 242, 267, 261, 281, 239, 279, 249, 233, 250, 246, 280, 260, 273, 254, 254, 267, 259, 266, 244, 266, 251, 266, 237, 251, 257, 246, 256, 260, 249, 230, 243, 263, 270, 256, 254, 253, 267, 277, 247, 230, 217, 288, 262, 243, 251, 234, 253, 255] payload_rows_per_dst=[5384, 5528, 5496, 5536, 5506, 5526, 5408, 5456] dup_rows_per_dst=[2598, 2838, 2674, 2790, 2714, 2744, 2680, 2658] combine_rows_per_src=[5384, 5528, 5496, 5536, 5506, 5526, 5408, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38164 dup_rows=21842 dup_rows_remote=18996 dedup_ratio=0.333282 router_tokens_per_expert=[271, 260, 262, 245, 242, 255, 246, 259, 283, 255, 248, 264, 234, 216, 241, 260, 264, 250, 275, 249, 246, 240, 254, 253, 267, 242, 276, 266, 239, 268, 241, 250, 299, 237, 279, 261, 253, 270, 252, 242, 251, 273, 250, 250, 259, 282, 228, 302, 254, 257, 267, 281, 271, 262, 248, 272, 233, 242, 256, 257, 252, 274, 237, 270, 259, 215, 273, 241, 245, 249, 279, 247, 254, 260, 267, 263, 247, 249, 254, 232, 258, 252, 232, 290, 254, 232, 232, 254, 256, 240, 235, 268, 240, 274, 267, 263, 258, 270, 252, 274, 255, 249, 246, 268, 258, 250, 263, 251, 255, 263, 269, 264, 255, 260, 285, 275, 248, 264, 289, 241, 221, 263, 247, 257, 249, 247, 245, 254] payload_rows_per_dst=[5384, 5358, 5530, 5544, 5422, 5448, 5534, 5474] dup_rows_per_dst=[2698, 2802, 2846, 2722, 2646, 2646, 2756, 2726] combine_rows_per_src=[5384, 5358, 5530, 5544, 5422, 5448, 5534, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229360 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917744 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606128 weight_movement_bytes=18874368 total_nvlink_bytes=332480496 weight_to_token_ratio=0.060185 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38206 dup_rows=21852 dup_rows_remote=19014 dedup_ratio=0.333435 router_tokens_per_expert=[270, 251, 276, 251, 243, 233, 238, 257, 254, 242, 272, 256, 260, 250, 277, 251, 266, 258, 245, 253, 263, 262, 274, 257, 276, 267, 273, 264, 275, 266, 251, 282, 282, 259, 225, 258, 249, 244, 241, 269, 280, 239, 268, 254, 250, 265, 269, 259, 243, 238, 280, 274, 249, 263, 283, 245, 233, 266, 254, 268, 275, 260, 255, 244, 268, 251, 255, 214, 249, 267, 258, 270, 267, 258, 244, 286, 259, 247, 264, 271, 242, 274, 268, 255, 269, 238, 245, 233, 238, 244, 244, 255, 254, 265, 261, 257, 238, 253, 253, 255, 232, 252, 245, 241, 250, 257, 244, 233, 239, 282, 245, 266, 261, 244, 268, 265, 249, 268, 250, 255, 237, 248, 244, 262, 253, 247, 250, 258] payload_rows_per_dst=[5478, 5644, 5420, 5506, 5442, 5528, 5364, 5302] dup_rows_per_dst=[2838, 2992, 2640, 2754, 2660, 2986, 2606, 2376] combine_rows_per_src=[5478, 5644, 5420, 5506, 5442, 5528, 5364, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228448 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156851296 ingress_bytes=232120320 link_bytes=232120320 token_bytes=313474144 weight_movement_bytes=75497472 total_nvlink_bytes=388971616 weight_to_token_ratio=0.240841 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224062696e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228576 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156646624 ingress_bytes=194166784 link_bytes=194166784 token_bytes=313064672 weight_movement_bytes=37748736 total_nvlink_bytes=350813408 weight_to_token_ratio=0.120578 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860410588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38218 dup_rows=21848 dup_rows_remote=19082 dedup_ratio=0.333374 router_tokens_per_expert=[250, 261, 253, 241, 264, 233, 267, 249, 250, 291, 277, 285, 238, 269, 239, 246, 246, 223, 239, 270, 255, 277, 277, 263, 263, 263, 269, 273, 253, 256, 241, 244, 257, 266, 258, 252, 234, 284, 249, 282, 233, 281, 250, 251, 247, 271, 254, 265, 268, 237, 246, 278, 215, 244, 215, 279, 246, 238, 257, 232, 267, 287, 239, 267, 245, 229, 259, 261, 250, 247, 241, 241, 278, 243, 264, 247, 257, 260, 267, 257, 254, 281, 243, 252, 263, 220, 262, 260, 242, 257, 274, 262, 240, 268, 267, 273, 237, 245, 273, 250, 263, 250, 273, 248, 255, 240, 236, 257, 280, 276, 227, 272, 257, 272, 233, 257, 250, 230, 299, 287, 259, 259, 273, 266, 237, 244, 255, 270] payload_rows_per_dst=[5516, 5492, 5494, 5388, 5406, 5470, 5430, 5492] dup_rows_per_dst=[2710, 2732, 2774, 2642, 2686, 2766, 2734, 2804] combine_rows_per_src=[5516, 5492, 5494, 5388, 5406, 5470, 5430, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228680 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157154632 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314080584 weight_movement_bytes=37748736 total_nvlink_bytes=351829320 weight_to_token_ratio=0.120188 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228640 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156548384 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868128 weight_movement_bytes=37748736 total_nvlink_bytes=350616864 weight_to_token_ratio=0.120654 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228880 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156720656 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313212432 weight_movement_bytes=37748736 total_nvlink_bytes=350961168 weight_to_token_ratio=0.120521 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229200 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770128 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311056 weight_movement_bytes=18874368 total_nvlink_bytes=332185424 weight_to_token_ratio=0.060242 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38254 dup_rows=21794 dup_rows_remote=19062 dedup_ratio=0.332550 router_tokens_per_expert=[269, 246, 274, 279, 239, 245, 251, 289, 265, 247, 256, 272, 256, 253, 247, 256, 238, 259, 247, 244, 239, 245, 267, 223, 229, 274, 255, 261, 268, 254, 249, 271, 265, 250, 247, 262, 262, 285, 270, 244, 240, 259, 242, 260, 252, 262, 278, 284, 271, 249, 241, 248, 252, 240, 243, 248, 289, 277, 283, 271, 274, 267, 259, 207, 262, 257, 273, 245, 238, 252, 272, 273, 231, 248, 256, 261, 247, 255, 238, 239, 258, 226, 269, 275, 259, 255, 267, 216, 245, 254, 264, 258, 257, 261, 228, 256, 256, 248, 245, 234, 271, 239, 265, 254, 273, 265, 271, 246, 265, 269, 265, 249, 233, 277, 247, 247, 272, 253, 260, 266, 249, 266, 234, 276, 275, 260, 265, 230] payload_rows_per_dst=[5304, 5482, 5372, 5410, 5582, 5580, 5524, 5488] dup_rows_per_dst=[2436, 2564, 2596, 2674, 2868, 3064, 2860, 2732] combine_rows_per_src=[5304, 5482, 5372, 5410, 5582, 5580, 5524, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38284 dup_rows=21870 dup_rows_remote=19268 dedup_ratio=0.333710 router_tokens_per_expert=[287, 261, 284, 231, 246, 259, 275, 282, 287, 249, 240, 269, 268, 234, 238, 268, 234, 249, 245, 235, 228, 280, 237, 226, 226, 257, 280, 253, 245, 262, 243, 263, 240, 274, 281, 305, 238, 254, 266, 259, 273, 266, 256, 273, 247, 255, 251, 278, 266, 261, 298, 239, 289, 259, 238, 257, 286, 255, 254, 253, 263, 266, 270, 265, 242, 275, 249, 272, 241, 243, 255, 258, 249, 263, 271, 234, 246, 264, 258, 249, 262, 230, 247, 275, 272, 262, 256, 271, 269, 257, 246, 245, 257, 260, 212, 250, 245, 274, 249, 271, 244, 247, 264, 248, 277, 238, 231, 235, 262, 255, 244, 208, 259, 278, 246, 247, 262, 250, 275, 252, 263, 239, 236, 258, 252, 228, 272, 243] payload_rows_per_dst=[5542, 5264, 5526, 5562, 5402, 5522, 5382, 5466] dup_rows_per_dst=[2730, 2662, 2906, 2876, 2736, 2704, 2602, 2654] combine_rows_per_src=[5542, 5264, 5526, 5562, 5402, 5522, 5382, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38264 dup_rows=21778 dup_rows_remote=19048 dedup_ratio=0.332306 router_tokens_per_expert=[266, 260, 252, 275, 273, 258, 227, 275, 251, 253, 250, 226, 249, 277, 254, 261, 246, 275, 237, 273, 260, 273, 243, 239, 261, 254, 237, 282, 248, 267, 265, 196, 236, 271, 283, 261, 241, 297, 248, 250, 274, 229, 259, 242, 254, 239, 272, 272, 270, 241, 236, 285, 257, 286, 255, 273, 243, 268, 263, 253, 229, 233, 256, 264, 263, 259, 282, 271, 248, 243, 219, 236, 267, 244, 261, 274, 237, 250, 237, 246, 284, 231, 259, 252, 251, 267, 251, 262, 256, 264, 253, 286, 257, 255, 240, 267, 227, 302, 254, 239, 290, 257, 267, 239, 243, 238, 237, 268, 268, 247, 285, 266, 240, 239, 252, 279, 243, 232, 226, 248, 257, 248, 246, 274, 247, 280, 307, 248] payload_rows_per_dst=[5512, 5440, 5494, 5494, 5362, 5494, 5452, 5510] dup_rows_per_dst=[2702, 2672, 2762, 2730, 2712, 2776, 2802, 2622] combine_rows_per_src=[5512, 5440, 5494, 5494, 5362, 5494, 5452, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38200 dup_rows=21876 dup_rows_remote=19044 dedup_ratio=0.333801 router_tokens_per_expert=[291, 267, 287, 264, 229, 244, 270, 262, 250, 242, 263, 249, 251, 253, 242, 261, 276, 244, 245, 265, 259, 265, 261, 258, 238, 240, 255, 244, 260, 272, 270, 243, 257, 277, 244, 266, 256, 256, 262, 262, 251, 255, 257, 265, 240, 248, 259, 268, 239, 251, 248, 248, 237, 257, 267, 260, 266, 267, 256, 246, 246, 255, 221, 246, 263, 217, 273, 282, 265, 255, 262, 243, 263, 274, 239, 260, 257, 264, 275, 254, 278, 275, 252, 248, 244, 268, 269, 250, 224, 251, 264, 271, 268, 286, 253, 242, 257, 270, 244, 231, 207, 275, 252, 220, 250, 234, 256, 254, 266, 253, 235, 260, 253, 275, 262, 252, 259, 286, 253, 263, 255, 260, 259, 277, 252, 240, 252, 264] payload_rows_per_dst=[5426, 5496, 5478, 5332, 5460, 5492, 5392, 5584] dup_rows_per_dst=[2824, 2694, 2768, 2688, 2832, 2794, 2536, 2740] combine_rows_per_src=[5426, 5496, 5478, 5332, 5460, 5492, 5392, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229264 combine_payload_bytes=156688384 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156917648 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606032 weight_movement_bytes=37748736 total_nvlink_bytes=351354768 weight_to_token_ratio=0.120370 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43452 payload_rows_remote=38064 dup_rows=22084 dup_rows_remote=19408 dedup_ratio=0.336975 router_tokens_per_expert=[250, 299, 258, 222, 237, 258, 252, 245, 264, 253, 267, 262, 244, 249, 266, 262, 274, 273, 275, 256, 260, 227, 259, 244, 263, 246, 245, 235, 266, 238, 264, 248, 233, 246, 254, 266, 253, 246, 244, 261, 230, 253, 249, 273, 211, 295, 256, 262, 249, 261, 273, 233, 268, 284, 237, 271, 279, 237, 256, 205, 232, 288, 254, 259, 237, 263, 233, 281, 235, 248, 258, 246, 249, 257, 241, 265, 253, 267, 233, 232, 253, 235, 251, 265, 258, 252, 267, 252, 244, 259, 266, 270, 241, 245, 257, 243, 278, 252, 260, 253, 269, 267, 238, 284, 267, 276, 283, 252, 267, 243, 248, 244, 272, 240, 253, 250, 277, 273, 242, 287, 282, 268, 250, 253, 267, 292, 293, 253] payload_rows_per_dst=[5390, 5428, 5388, 5478, 5292, 5398, 5490, 5588] dup_rows_per_dst=[2786, 2718, 2676, 2694, 2704, 2718, 2872, 2916] combine_rows_per_src=[5390, 5428, 5388, 5478, 5292, 5398, 5490, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38344 dup_rows=21760 dup_rows_remote=19142 dedup_ratio=0.332031 router_tokens_per_expert=[276, 249, 252, 261, 255, 239, 284, 259, 278, 284, 249, 233, 256, 274, 244, 274, 273, 272, 240, 248, 265, 250, 264, 232, 228, 222, 252, 259, 271, 277, 240, 232, 275, 242, 269, 245, 251, 235, 282, 264, 252, 270, 245, 263, 273, 242, 283, 284, 259, 284, 263, 273, 248, 257, 238, 264, 258, 292, 258, 263, 245, 255, 248, 231, 293, 250, 256, 237, 259, 267, 225, 260, 256, 266, 236, 232, 243, 239, 256, 267, 257, 252, 294, 265, 234, 251, 249, 264, 266, 238, 267, 280, 252, 243, 232, 261, 217, 270, 261, 249, 246, 247, 242, 256, 264, 236, 249, 268, 279, 258, 259, 242, 260, 250, 239, 258, 271, 234, 292, 246, 278, 265, 257, 231, 231, 247, 271, 245] payload_rows_per_dst=[5580, 5432, 5470, 5440, 5446, 5520, 5498, 5390] dup_rows_per_dst=[2754, 2618, 2880, 2832, 2638, 2690, 2588, 2760] combine_rows_per_src=[5580, 5432, 5470, 5440, 5446, 5520, 5498, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=230208 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157041472 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852736 weight_movement_bytes=18874368 total_nvlink_bytes=332727104 weight_to_token_ratio=0.060138 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=37986 dup_rows=21954 dup_rows_remote=19128 dedup_ratio=0.334991 router_tokens_per_expert=[255, 249, 228, 259, 255, 267, 268, 257, 274, 266, 213, 269, 265, 249, 253, 244, 274, 236, 258, 259, 250, 246, 249, 254, 267, 241, 284, 286, 280, 262, 252, 247, 251, 260, 275, 269, 267, 265, 246, 253, 270, 262, 255, 285, 254, 231, 257, 233, 254, 254, 264, 241, 274, 263, 248, 264, 257, 251, 234, 262, 271, 268, 282, 289, 249, 262, 270, 253, 261, 248, 290, 252, 241, 289, 255, 235, 239, 272, 235, 258, 240, 276, 252, 280, 252, 242, 251, 254, 248, 256, 253, 285, 265, 245, 267, 247, 239, 252, 271, 266, 250, 233, 279, 230, 247, 256, 276, 230, 240, 256, 247, 281, 234, 237, 243, 254, 223, 253, 239, 242, 259, 272, 248, 274, 263, 224, 254, 249] payload_rows_per_dst=[5596, 5430, 5588, 5588, 5346, 5350, 5418, 5266] dup_rows_per_dst=[2826, 2608, 3036, 2974, 2662, 2550, 2688, 2610] combine_rows_per_src=[5596, 5430, 5588, 5588, 5346, 5350, 5418, 5266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229248 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958592 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313687936 weight_movement_bytes=37748736 total_nvlink_bytes=351436672 weight_to_token_ratio=0.120339 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38316 dup_rows=21758 dup_rows_remote=19098 dedup_ratio=0.332001 router_tokens_per_expert=[248, 244, 278, 263, 249, 219, 259, 236, 289, 260, 285, 235, 276, 260, 255, 251, 231, 217, 268, 223, 271, 270, 261, 249, 284, 273, 261, 247, 274, 229, 250, 246, 257, 279, 247, 267, 264, 240, 244, 263, 237, 234, 237, 256, 244, 265, 280, 260, 282, 274, 259, 265, 246, 247, 284, 252, 222, 249, 254, 265, 244, 253, 262, 256, 284, 272, 247, 240, 232, 254, 276, 257, 253, 257, 250, 237, 254, 286, 256, 253, 278, 249, 250, 266, 284, 241, 261, 234, 277, 238, 234, 267, 239, 245, 247, 251, 262, 270, 247, 262, 249, 275, 220, 252, 281, 247, 270, 244, 249, 253, 244, 261, 276, 269, 271, 241, 258, 264, 279, 253, 238, 251, 241, 267, 250, 255, 272, 279] payload_rows_per_dst=[5462, 5386, 5464, 5474, 5516, 5462, 5502, 5512] dup_rows_per_dst=[2752, 2722, 2684, 2754, 2700, 2660, 2670, 2816] combine_rows_per_src=[5462, 5386, 5464, 5474, 5516, 5462, 5502, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228976 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156696176 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313163376 weight_movement_bytes=18874368 total_nvlink_bytes=332037744 weight_to_token_ratio=0.060270 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=229888 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156140032 ingress_bytes=174784512 link_bytes=174784512 token_bytes=312050176 weight_movement_bytes=18874368 total_nvlink_bytes=330924544 weight_to_token_ratio=0.060485 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674699195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229944 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286968 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343992 weight_movement_bytes=18874368 total_nvlink_bytes=333218360 weight_to_token_ratio=0.060044 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=228456 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155819112 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311409768 weight_movement_bytes=18874368 total_nvlink_bytes=330284136 weight_to_token_ratio=0.060609 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15066484304 prefetch_bytes=717225984 reduce_bytes=1019215872 weight_movement_bytes=1736441856 egress_bytes=7538749008 ingress_bytes=9264177152 link_bytes=9264177152 total_nvlink_bytes=16802926160 weight_to_token_ratio=0.115252 prefetch_share_of_ingress=0.077419 dedup_ratio=0.332464 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=54 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081163 gemm_inner_pad_upper_bound=108458 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.849266400e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229656 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171992 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114328 weight_movement_bytes=18874368 total_nvlink_bytes=332988696 weight_to_token_ratio=0.060088 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38368 dup_rows=21616 dup_rows_remote=18848 dedup_ratio=0.329834 router_tokens_per_expert=[275, 272, 261, 262, 277, 240, 273, 245, 259, 244, 258, 255, 237, 277, 250, 250, 227, 232, 252, 293, 249, 259, 251, 252, 288, 249, 276, 252, 242, 263, 297, 248, 274, 264, 277, 247, 240, 242, 267, 241, 257, 252, 282, 268, 240, 241, 248, 241, 289, 257, 242, 241, 235, 246, 273, 251, 261, 249, 241, 281, 289, 273, 254, 283, 258, 223, 261, 224, 248, 232, 246, 253, 224, 273, 251, 232, 241, 241, 270, 270, 252, 273, 265, 275, 267, 268, 238, 279, 249, 260, 277, 258, 226, 243, 262, 263, 258, 231, 231, 251, 274, 243, 258, 277, 247, 250, 224, 246, 248, 278, 258, 227, 246, 263, 252, 240, 279, 256, 266, 256, 266, 262, 253, 272, 252, 256, 250, 285] payload_rows_per_dst=[5474, 5502, 5484, 5400, 5372, 5570, 5566, 5552] dup_rows_per_dst=[2796, 2612, 2678, 2626, 2522, 2740, 2874, 2768] combine_rows_per_src=[5474, 5502, 5484, 5400, 5372, 5570, 5566, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15047599880 prefetch_bytes=868220928 reduce_bytes=736100352 weight_movement_bytes=1604321280 egress_bytes=7529301768 ingress_bytes=9122619392 link_bytes=9122619392 total_nvlink_bytes=16651921160 weight_to_token_ratio=0.106616 prefetch_share_of_ingress=0.095172 dedup_ratio=0.332833 prefetch_slots=92 unused_prefetch_slots=1444 slot_utilization=0.059896 reduce_slots=39 remote_demand=46 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080874 gemm_inner_pad_upper_bound=109474 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.821009383e-04 prefetch_bytes_per_gemm_flop=1.733097029e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38234 dup_rows=21894 dup_rows_remote=19252 dedup_ratio=0.334076 router_tokens_per_expert=[268, 255, 256, 259, 228, 244, 260, 264, 265, 262, 248, 258, 273, 245, 246, 251, 281, 283, 252, 253, 244, 245, 249, 275, 250, 255, 265, 272, 248, 280, 272, 272, 242, 263, 234, 259, 260, 238, 244, 258, 266, 246, 239, 246, 237, 235, 265, 251, 259, 267, 261, 284, 255, 241, 267, 259, 246, 284, 259, 251, 279, 273, 285, 244, 261, 258, 272, 234, 256, 269, 252, 243, 221, 258, 267, 240, 252, 293, 267, 252, 238, 255, 247, 277, 251, 252, 260, 268, 267, 263, 245, 232, 242, 265, 261, 249, 229, 246, 263, 250, 249, 257, 243, 243, 259, 239, 259, 268, 248, 257, 265, 250, 238, 264, 265, 256, 265, 255, 218, 251, 279, 254, 242, 283, 270, 259, 237, 265] payload_rows_per_dst=[5398, 5514, 5400, 5570, 5460, 5404, 5408, 5488] dup_rows_per_dst=[2766, 2878, 2566, 2858, 2730, 2740, 2642, 2714] combine_rows_per_src=[5398, 5514, 5400, 5570, 5460, 5404, 5408, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38140 dup_rows=21886 dup_rows_remote=19022 dedup_ratio=0.333954 router_tokens_per_expert=[255, 233, 266, 255, 246, 287, 248, 247, 233, 257, 261, 271, 264, 254, 248, 255, 234, 237, 275, 236, 259, 257, 250, 263, 270, 259, 266, 247, 259, 257, 261, 229, 244, 236, 268, 244, 224, 257, 261, 288, 276, 267, 243, 250, 257, 254, 258, 255, 276, 214, 231, 273, 265, 206, 258, 248, 245, 235, 243, 240, 282, 268, 273, 263, 272, 272, 262, 268, 234, 234, 244, 266, 280, 239, 296, 261, 261, 280, 248, 270, 242, 265, 262, 245, 256, 263, 286, 250, 280, 266, 280, 243, 290, 254, 263, 260, 241, 225, 244, 243, 243, 254, 249, 267, 255, 250, 252, 270, 265, 253, 227, 239, 245, 293, 247, 249, 226, 256, 274, 258, 252, 297, 273, 242, 267, 262, 260, 257] payload_rows_per_dst=[5416, 5438, 5414, 5388, 5510, 5580, 5382, 5522] dup_rows_per_dst=[2744, 2680, 2750, 2652, 2864, 2830, 2572, 2794] combine_rows_per_src=[5416, 5438, 5414, 5388, 5510, 5580, 5382, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=228864 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157384192 ingress_bytes=232652800 link_bytes=232652800 token_bytes=314539520 weight_movement_bytes=75497472 total_nvlink_bytes=390036992 weight_to_token_ratio=0.240025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229164657e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229944 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156836408 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313442872 weight_movement_bytes=18874368 total_nvlink_bytes=332317240 weight_to_token_ratio=0.060216 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38282 dup_rows=21762 dup_rows_remote=19142 dedup_ratio=0.332062 router_tokens_per_expert=[279, 248, 273, 253, 233, 251, 261, 277, 238, 270, 274, 267, 257, 277, 238, 271, 256, 263, 263, 257, 265, 261, 246, 255, 243, 245, 251, 262, 266, 219, 262, 242, 233, 260, 264, 269, 232, 281, 244, 256, 271, 237, 237, 256, 266, 262, 245, 249, 256, 279, 267, 267, 247, 262, 254, 276, 257, 209, 264, 283, 272, 263, 278, 249, 271, 244, 263, 237, 249, 260, 281, 289, 264, 247, 256, 275, 219, 249, 254, 237, 260, 233, 241, 261, 268, 266, 223, 261, 249, 239, 252, 264, 247, 241, 281, 236, 293, 262, 244, 260, 274, 264, 255, 257, 235, 229, 233, 268, 272, 259, 243, 288, 260, 242, 238, 272, 247, 256, 231, 283, 250, 249, 239, 269, 227, 292, 251, 241] payload_rows_per_dst=[5490, 5492, 5444, 5520, 5466, 5392, 5512, 5458] dup_rows_per_dst=[2844, 2620, 2680, 2846, 2724, 2652, 2760, 2636] combine_rows_per_src=[5490, 5492, 5444, 5520, 5466, 5392, 5512, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38056 dup_rows=21846 dup_rows_remote=18938 dedup_ratio=0.333344 router_tokens_per_expert=[262, 247, 266, 263, 257, 266, 270, 300, 267, 240, 244, 263, 259, 258, 265, 244, 247, 273, 267, 259, 265, 260, 252, 248, 260, 247, 273, 264, 274, 249, 266, 262, 270, 268, 264, 261, 259, 251, 254, 254, 258, 272, 234, 242, 262, 239, 249, 226, 280, 278, 245, 263, 252, 273, 257, 242, 268, 261, 224, 249, 287, 241, 260, 272, 255, 214, 263, 247, 261, 244, 274, 239, 223, 253, 272, 250, 259, 242, 252, 265, 241, 261, 257, 255, 229, 242, 269, 251, 225, 262, 266, 235, 252, 250, 276, 241, 267, 256, 253, 251, 239, 232, 267, 271, 247, 259, 241, 263, 262, 265, 256, 275, 235, 238, 269, 243, 257, 280, 240, 277, 268, 250, 249, 249, 272, 236, 264, 260] payload_rows_per_dst=[5634, 5590, 5576, 5234, 5442, 5384, 5436, 5394] dup_rows_per_dst=[2908, 2870, 2854, 2438, 2584, 2640, 2772, 2780] combine_rows_per_src=[5634, 5590, 5576, 5234, 5442, 5384, 5436, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38300 dup_rows=21824 dup_rows_remote=19056 dedup_ratio=0.333008 router_tokens_per_expert=[251, 287, 256, 228, 251, 250, 267, 237, 264, 252, 248, 255, 262, 250, 244, 273, 274, 278, 272, 238, 254, 289, 281, 269, 263, 252, 252, 299, 233, 238, 274, 250, 245, 231, 279, 243, 263, 249, 252, 251, 255, 250, 239, 256, 249, 223, 263, 239, 238, 249, 277, 247, 261, 267, 225, 250, 277, 275, 262, 250, 242, 274, 268, 252, 234, 272, 256, 237, 265, 243, 248, 292, 240, 257, 260, 247, 276, 242, 270, 278, 261, 268, 246, 265, 257, 257, 255, 260, 248, 251, 267, 250, 264, 229, 266, 246, 256, 240, 258, 262, 241, 275, 251, 251, 240, 253, 268, 256, 232, 243, 245, 256, 240, 265, 269, 257, 263, 280, 270, 249, 245, 254, 247, 236, 260, 284, 249, 274] payload_rows_per_dst=[5452, 5584, 5408, 5480, 5496, 5412, 5362, 5518] dup_rows_per_dst=[2698, 2848, 2566, 2748, 2738, 2768, 2692, 2766] combine_rows_per_src=[5452, 5584, 5408, 5480, 5496, 5412, 5362, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228648 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156450088 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312671528 weight_movement_bytes=18874368 total_nvlink_bytes=331545896 weight_to_token_ratio=0.060365 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15029531664 prefetch_bytes=792723456 reduce_bytes=905969664 weight_movement_bytes=1698693120 egress_bytes=7520252944 ingress_bytes=9207971840 link_bytes=9207971840 total_nvlink_bytes=16728224784 weight_to_token_ratio=0.113024 prefetch_share_of_ingress=0.086091 dedup_ratio=0.332676 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=48 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.838046991e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15053762344 prefetch_bytes=849346560 reduce_bytes=811597824 weight_movement_bytes=1660944384 egress_bytes=7532384040 ingress_bytes=9182322688 link_bytes=9182322688 total_nvlink_bytes=16714706728 weight_to_token_ratio=0.110334 prefetch_share_of_ingress=0.092498 dedup_ratio=0.332634 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=43 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081308 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.832927042e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229696 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032768 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835840 weight_movement_bytes=18874368 total_nvlink_bytes=332710208 weight_to_token_ratio=0.060141 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229424 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157106224 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313983024 weight_movement_bytes=37748736 total_nvlink_bytes=351731760 weight_to_token_ratio=0.120225 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=227976 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156105352 ingress_bytes=174751744 link_bytes=174751744 token_bytes=311982728 weight_movement_bytes=18874368 total_nvlink_bytes=330857096 weight_to_token_ratio=0.060498 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674385228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15053764280 prefetch_bytes=830472192 reduce_bytes=452984832 weight_movement_bytes=1283457024 egress_bytes=7532385976 ingress_bytes=8804835328 link_bytes=8804835328 total_nvlink_bytes=16337221304 weight_to_token_ratio=0.085258 prefetch_share_of_ingress=0.094320 dedup_ratio=0.332769 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=24 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081453 gemm_inner_pad_upper_bound=109220 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.757574997e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15064661808 prefetch_bytes=754974720 reduce_bytes=792723456 weight_movement_bytes=1547698176 egress_bytes=7537835824 ingress_bytes=9074524160 link_bytes=9074524160 total_nvlink_bytes=16612359984 weight_to_token_ratio=0.102737 prefetch_share_of_ingress=0.083197 dedup_ratio=0.332466 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=42 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082248 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.811408866e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38190 dup_rows=21874 dup_rows_remote=19164 dedup_ratio=0.333771 router_tokens_per_expert=[249, 240, 252, 247, 258, 244, 266, 253, 271, 266, 256, 245, 294, 260, 228, 254, 285, 260, 253, 264, 252, 240, 256, 245, 258, 285, 260, 271, 228, 235, 237, 262, 274, 273, 247, 270, 268, 255, 248, 256, 231, 247, 259, 271, 256, 270, 256, 261, 273, 261, 262, 263, 231, 254, 254, 239, 254, 248, 250, 287, 238, 273, 249, 263, 249, 234, 272, 263, 280, 285, 265, 246, 252, 270, 248, 305, 239, 219, 255, 261, 270, 268, 252, 260, 268, 253, 250, 241, 245, 245, 293, 210, 276, 251, 245, 238, 250, 258, 238, 257, 278, 253, 262, 264, 251, 257, 238, 258, 256, 251, 268, 244, 264, 270, 251, 261, 216, 255, 273, 227, 258, 261, 286, 237, 230, 256, 244, 273] payload_rows_per_dst=[5396, 5472, 5466, 5458, 5532, 5444, 5418, 5476] dup_rows_per_dst=[2770, 2710, 2822, 2740, 2754, 2686, 2748, 2644] combine_rows_per_src=[5396, 5472, 5466, 5458, 5532, 5444, 5418, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38068 dup_rows=21864 dup_rows_remote=18778 dedup_ratio=0.333618 router_tokens_per_expert=[257, 284, 305, 254, 259, 247, 255, 245, 256, 269, 242, 278, 266, 214, 245, 301, 250, 259, 262, 234, 246, 259, 238, 232, 233, 244, 244, 250, 271, 276, 255, 267, 264, 267, 264, 259, 258, 266, 250, 284, 285, 258, 272, 262, 244, 247, 243, 247, 271, 234, 263, 271, 244, 239, 239, 230, 241, 237, 264, 286, 276, 246, 264, 262, 256, 264, 250, 264, 250, 272, 248, 235, 270, 227, 279, 260, 264, 258, 289, 261, 261, 273, 235, 255, 266, 258, 228, 227, 248, 241, 249, 230, 272, 261, 263, 258, 253, 261, 262, 228, 269, 276, 244, 249, 243, 251, 232, 249, 233, 257, 247, 255, 263, 253, 235, 257, 228, 285, 249, 255, 284, 279, 277, 255, 233, 266, 273, 261] payload_rows_per_dst=[5604, 5384, 5568, 5478, 5362, 5584, 5426, 5266] dup_rows_per_dst=[3086, 2658, 2770, 2656, 2596, 3016, 2592, 2490] combine_rows_per_src=[5604, 5384, 5568, 5478, 5362, 5584, 5426, 5266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229416 combine_payload_bytes=156426240 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156655656 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081896 weight_movement_bytes=37748736 total_nvlink_bytes=350830632 weight_to_token_ratio=0.120571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=227384 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156153912 ingress_bytes=174800896 link_bytes=174800896 token_bytes=312080440 weight_movement_bytes=18874368 total_nvlink_bytes=330954808 weight_to_token_ratio=0.060479 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674856178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15061320104 prefetch_bytes=754974720 reduce_bytes=736100352 weight_movement_bytes=1491075072 egress_bytes=7536165288 ingress_bytes=9016229888 link_bytes=9016229888 total_nvlink_bytes=16552395176 weight_to_token_ratio=0.099000 prefetch_share_of_ingress=0.083735 dedup_ratio=0.332516 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=39 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.799772469e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38212 dup_rows=21870 dup_rows_remote=19078 dedup_ratio=0.333710 router_tokens_per_expert=[264, 261, 268, 247, 247, 271, 239, 272, 252, 264, 261, 248, 257, 247, 271, 254, 243, 241, 245, 260, 259, 233, 250, 242, 236, 257, 270, 228, 248, 244, 230, 276, 246, 270, 284, 241, 240, 265, 261, 260, 251, 237, 235, 267, 259, 255, 237, 234, 264, 246, 250, 264, 263, 258, 261, 297, 256, 224, 252, 241, 260, 254, 284, 260, 280, 275, 253, 265, 273, 264, 257, 244, 267, 230, 275, 267, 243, 255, 255, 248, 245, 247, 254, 263, 269, 245, 269, 251, 243, 257, 282, 265, 257, 236, 271, 263, 258, 247, 257, 253, 252, 245, 273, 255, 257, 259, 267, 261, 262, 258, 256, 269, 265, 261, 232, 258, 267, 249, 271, 237, 254, 294, 266, 264, 225, 256, 268, 243] payload_rows_per_dst=[5454, 5462, 5324, 5472, 5512, 5484, 5544, 5414] dup_rows_per_dst=[2792, 2730, 2492, 2796, 2790, 2750, 2714, 2806] combine_rows_per_src=[5454, 5462, 5324, 5472, 5512, 5484, 5544, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229160 combine_payload_bytes=156516352 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156745512 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313261864 weight_movement_bytes=37748736 total_nvlink_bytes=351010600 weight_to_token_ratio=0.120502 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38338 dup_rows=21636 dup_rows_remote=18808 dedup_ratio=0.330139 router_tokens_per_expert=[226, 272, 237, 257, 275, 279, 259, 280, 245, 268, 266, 261, 240, 278, 260, 270, 271, 246, 269, 259, 273, 260, 252, 267, 271, 247, 266, 239, 245, 287, 266, 252, 274, 260, 232, 262, 251, 246, 258, 244, 237, 252, 227, 232, 259, 271, 284, 238, 268, 243, 249, 242, 275, 274, 239, 269, 237, 261, 231, 230, 275, 265, 277, 236, 263, 266, 242, 280, 277, 248, 244, 266, 240, 261, 244, 246, 240, 262, 245, 280, 243, 266, 265, 264, 273, 245, 244, 249, 277, 255, 217, 261, 246, 237, 255, 236, 294, 223, 234, 254, 259, 252, 256, 251, 252, 262, 262, 257, 288, 254, 255, 276, 231, 261, 276, 258, 233, 249, 252, 265, 252, 259, 269, 257, 261, 267, 224, 247] payload_rows_per_dst=[5562, 5650, 5492, 5228, 5534, 5402, 5556, 5476] dup_rows_per_dst=[2828, 3088, 2612, 2482, 2674, 2664, 2702, 2586] combine_rows_per_src=[5562, 5650, 5492, 5228, 5534, 5402, 5556, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=228584 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157261032 ingress_bytes=194781184 link_bytes=194781184 token_bytes=314293480 weight_movement_bytes=37748736 total_nvlink_bytes=352042216 weight_to_token_ratio=0.120107 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866297467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38152 dup_rows=21906 dup_rows_remote=19148 dedup_ratio=0.334259 router_tokens_per_expert=[254, 249, 284, 256, 237, 261, 260, 237, 258, 228, 240, 256, 281, 268, 236, 266, 236, 271, 266, 258, 246, 248, 261, 268, 279, 257, 212, 231, 268, 242, 227, 250, 253, 250, 264, 256, 231, 243, 248, 262, 258, 270, 267, 238, 267, 270, 264, 259, 254, 285, 255, 235, 241, 268, 253, 267, 268, 259, 230, 231, 260, 292, 272, 240, 264, 247, 240, 245, 260, 264, 268, 263, 254, 220, 285, 255, 258, 245, 296, 244, 278, 266, 257, 244, 238, 268, 232, 265, 228, 261, 255, 244, 291, 258, 270, 256, 256, 269, 270, 285, 269, 248, 245, 256, 248, 243, 279, 253, 233, 244, 239, 291, 272, 254, 256, 237, 260, 257, 298, 239, 246, 255, 246, 279, 272, 231, 262, 256] payload_rows_per_dst=[5478, 5430, 5608, 5460, 5604, 5368, 5422, 5260] dup_rows_per_dst=[2758, 2610, 3054, 2760, 2884, 2608, 2834, 2398] combine_rows_per_src=[5478, 5430, 5608, 5460, 5604, 5368, 5422, 5260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229200 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156499792 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312770384 weight_movement_bytes=18874368 total_nvlink_bytes=331644752 weight_to_token_ratio=0.060346 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15032618608 prefetch_bytes=736100352 reduce_bytes=849346560 weight_movement_bytes=1585446912 egress_bytes=7521799792 ingress_bytes=9096265728 link_bytes=9096265728 total_nvlink_bytes=16618065520 weight_to_token_ratio=0.105467 prefetch_share_of_ingress=0.080923 dedup_ratio=0.332796 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=45 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.083406 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.815748803e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none + [2026-08-29 23:39:09] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 1053.1 | throughput per GPU (TFLOP/s/GPU): 84.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.095760E+00 | loss scale: 1.0 | grad norm: 3.443 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38242 dup_rows=21792 dup_rows_remote=19076 dedup_ratio=0.332520 router_tokens_per_expert=[262, 271, 272, 259, 227, 256, 245, 239, 214, 277, 253, 241, 245, 254, 264, 269, 258, 254, 255, 246, 278, 247, 245, 254, 238, 259, 254, 235, 259, 240, 256, 274, 250, 249, 238, 275, 275, 248, 259, 243, 255, 237, 242, 245, 255, 244, 236, 259, 255, 269, 240, 254, 263, 221, 241, 241, 296, 245, 275, 246, 269, 264, 282, 248, 284, 234, 250, 267, 272, 261, 258, 239, 267, 239, 253, 250, 253, 269, 242, 262, 264, 263, 248, 254, 261, 254, 259, 283, 253, 250, 259, 250, 242, 260, 281, 267, 247, 246, 249, 239, 244, 258, 250, 270, 236, 279, 258, 274, 259, 258, 245, 263, 298, 292, 242, 264, 262, 248, 289, 271, 260, 232, 257, 250, 251, 277, 260, 273] payload_rows_per_dst=[5474, 5412, 5374, 5502, 5460, 5486, 5520, 5516] dup_rows_per_dst=[2622, 2692, 2646, 2716, 2740, 2810, 2630, 2936] combine_rows_per_src=[5474, 5412, 5374, 5502, 5460, 5486, 5520, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38332 dup_rows=21696 dup_rows_remote=18920 dedup_ratio=0.331055 router_tokens_per_expert=[273, 262, 253, 248, 276, 278, 243, 292, 248, 254, 268, 240, 273, 253, 292, 248, 246, 280, 250, 236, 239, 268, 266, 256, 249, 245, 247, 283, 259, 244, 276, 246, 241, 263, 272, 215, 221, 265, 233, 269, 248, 247, 243, 251, 267, 221, 235, 247, 246, 276, 254, 259, 245, 250, 247, 270, 263, 257, 239, 248, 264, 239, 262, 277, 288, 279, 261, 262, 247, 268, 251, 269, 243, 246, 276, 261, 267, 261, 252, 297, 236, 275, 269, 224, 251, 261, 254, 259, 257, 265, 256, 260, 250, 257, 261, 280, 262, 240, 280, 250, 259, 238, 247, 223, 279, 232, 221, 232, 263, 243, 240, 249, 269, 248, 251, 251, 292, 255, 250, 236, 264, 254, 269, 262, 273, 260, 263, 245] payload_rows_per_dst=[5550, 5452, 5336, 5558, 5616, 5472, 5348, 5508] dup_rows_per_dst=[2852, 2728, 2540, 2634, 2840, 2758, 2568, 2776] combine_rows_per_src=[5550, 5452, 5336, 5558, 5616, 5472, 5348, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38062 dup_rows=21992 dup_rows_remote=19218 dedup_ratio=0.335571 router_tokens_per_expert=[265, 264, 243, 263, 224, 257, 235, 254, 255, 279, 271, 244, 270, 253, 228, 267, 246, 283, 238, 267, 242, 269, 277, 239, 262, 254, 275, 238, 258, 259, 255, 243, 241, 252, 216, 247, 249, 267, 232, 254, 231, 255, 281, 249, 239, 262, 241, 251, 250, 231, 256, 264, 247, 294, 278, 250, 241, 269, 277, 239, 253, 252, 234, 256, 238, 234, 281, 236, 249, 268, 264, 245, 251, 277, 256, 281, 250, 287, 274, 237, 263, 271, 246, 266, 261, 266, 260, 246, 267, 258, 257, 259, 240, 240, 262, 224, 253, 241, 280, 259, 278, 258, 253, 252, 255, 236, 272, 294, 264, 272, 252, 237, 259, 298, 269, 247, 276, 268, 280, 223, 232, 255, 242, 252, 243, 253, 290, 276] payload_rows_per_dst=[5450, 5522, 5400, 5424, 5482, 5412, 5456, 5398] dup_rows_per_dst=[2694, 2688, 2534, 2758, 2774, 2760, 2856, 2928] combine_rows_per_src=[5450, 5522, 5400, 5424, 5482, 5412, 5456, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38272 dup_rows=21730 dup_rows_remote=19008 dedup_ratio=0.331573 router_tokens_per_expert=[269, 259, 224, 253, 259, 247, 249, 262, 238, 246, 230, 265, 264, 273, 251, 247, 262, 259, 250, 244, 219, 251, 256, 281, 246, 248, 266, 252, 268, 243, 254, 262, 242, 264, 251, 235, 259, 285, 218, 248, 271, 284, 265, 224, 243, 252, 247, 237, 269, 239, 263, 266, 285, 257, 235, 253, 233, 279, 284, 250, 268, 261, 271, 253, 261, 262, 277, 262, 278, 261, 263, 266, 271, 254, 254, 247, 237, 251, 241, 248, 244, 256, 258, 240, 247, 257, 256, 254, 249, 242, 268, 252, 254, 233, 278, 274, 244, 288, 229, 257, 244, 252, 261, 262, 261, 252, 204, 257, 268, 286, 303, 260, 231, 250, 268, 247, 277, 256, 250, 257, 287, 254, 270, 260, 276, 229, 281, 264] payload_rows_per_dst=[5374, 5418, 5410, 5568, 5540, 5478, 5534, 5484] dup_rows_per_dst=[2698, 2704, 2640, 2764, 2726, 2646, 2722, 2830] combine_rows_per_src=[5374, 5418, 5410, 5568, 5540, 5478, 5534, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38260 dup_rows=21758 dup_rows_remote=19014 dedup_ratio=0.332001 router_tokens_per_expert=[258, 230, 280, 275, 244, 262, 253, 271, 258, 262, 278, 260, 242, 252, 259, 246, 244, 255, 253, 257, 239, 236, 264, 255, 260, 254, 267, 262, 252, 265, 250, 234, 253, 264, 262, 275, 282, 264, 267, 268, 244, 240, 273, 240, 228, 270, 249, 278, 262, 236, 229, 241, 274, 235, 260, 263, 244, 237, 265, 236, 249, 242, 249, 261, 260, 265, 229, 235, 283, 233, 255, 260, 261, 256, 266, 238, 250, 270, 263, 267, 284, 239, 263, 249, 275, 255, 261, 249, 276, 248, 237, 239, 272, 249, 258, 277, 254, 241, 267, 270, 251, 270, 259, 254, 273, 263, 273, 232, 236, 278, 239, 250, 239, 256, 280, 251, 264, 274, 227, 256, 254, 233, 264, 271, 251, 271, 268, 260] payload_rows_per_dst=[5488, 5440, 5518, 5372, 5492, 5518, 5474, 5476] dup_rows_per_dst=[2772, 2654, 2796, 2594, 2690, 2744, 2746, 2762] combine_rows_per_src=[5488, 5440, 5518, 5372, 5492, 5518, 5474, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38246 dup_rows=21746 dup_rows_remote=18950 dedup_ratio=0.331818 router_tokens_per_expert=[241, 243, 259, 246, 270, 264, 277, 249, 225, 264, 238, 241, 236, 229, 242, 249, 251, 234, 281, 237, 304, 285, 258, 272, 225, 258, 237, 229, 248, 252, 252, 256, 250, 259, 271, 265, 265, 274, 252, 254, 252, 252, 270, 245, 289, 262, 260, 250, 268, 272, 248, 270, 270, 257, 239, 233, 252, 261, 264, 257, 241, 252, 269, 233, 252, 262, 267, 264, 242, 246, 290, 272, 248, 248, 249, 256, 283, 252, 263, 273, 249, 269, 249, 245, 288, 268, 228, 237, 256, 259, 258, 239, 236, 240, 268, 234, 257, 266, 247, 243, 292, 250, 237, 253, 265, 238, 237, 247, 260, 254, 277, 275, 306, 267, 264, 281, 247, 249, 241, 253, 265, 257, 255, 230, 260, 286, 255, 256] payload_rows_per_dst=[5358, 5448, 5544, 5428, 5584, 5382, 5502, 5544] dup_rows_per_dst=[2588, 2710, 2796, 2744, 2750, 2664, 2694, 2800] combine_rows_per_src=[5358, 5448, 5544, 5428, 5584, 5382, 5502, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38502 dup_rows=21628 dup_rows_remote=19020 dedup_ratio=0.330017 router_tokens_per_expert=[230, 234, 267, 256, 264, 280, 244, 261, 261, 250, 265, 241, 271, 263, 233, 247, 259, 224, 263, 257, 250, 260, 265, 238, 256, 231, 259, 287, 234, 253, 229, 242, 266, 269, 269, 266, 252, 259, 248, 272, 249, 262, 251, 272, 250, 255, 267, 251, 232, 255, 275, 263, 244, 253, 248, 262, 278, 286, 244, 246, 274, 256, 277, 267, 234, 264, 245, 267, 256, 247, 246, 246, 260, 237, 259, 223, 257, 230, 274, 249, 275, 231, 271, 242, 245, 240, 258, 266, 290, 257, 240, 267, 252, 263, 254, 245, 268, 242, 258, 262, 232, 244, 251, 260, 262, 273, 290, 269, 266, 259, 283, 268, 276, 260, 269, 246, 251, 259, 248, 224, 272, 284, 267, 218, 225, 269, 261, 270] payload_rows_per_dst=[5428, 5406, 5526, 5560, 5458, 5486, 5592, 5452] dup_rows_per_dst=[2706, 2608, 2790, 2760, 2530, 2706, 2782, 2746] combine_rows_per_src=[5428, 5406, 5526, 5560, 5458, 5486, 5592, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229272 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868504 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507736 weight_movement_bytes=18874368 total_nvlink_bytes=332382104 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229008 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157236880 ingress_bytes=232505344 link_bytes=232505344 token_bytes=314244752 weight_movement_bytes=75497472 total_nvlink_bytes=389742224 weight_to_token_ratio=0.240251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227751806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=229120 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156131072 ingress_bytes=193650688 link_bytes=193650688 token_bytes=312033024 weight_movement_bytes=37748736 total_nvlink_bytes=349781760 weight_to_token_ratio=0.120977 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855465610e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229120 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156991232 ingress_bytes=213385216 link_bytes=213385216 token_bytes=313753344 weight_movement_bytes=56623104 total_nvlink_bytes=370376448 weight_to_token_ratio=0.180470 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044552148e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229096 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942056 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655016 weight_movement_bytes=18874368 total_nvlink_bytes=332529384 weight_to_token_ratio=0.060176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228784 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156884400 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540016 weight_movement_bytes=18874368 total_nvlink_bytes=332414384 weight_to_token_ratio=0.060198 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157704192 dispatch_meta_bytes=230088 combine_payload_bytes=157704192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157934280 ingress_bytes=176578560 link_bytes=176578560 token_bytes=315638472 weight_movement_bytes=18874368 total_nvlink_bytes=334512840 weight_to_token_ratio=0.059797 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691888880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38204 dup_rows=21948 dup_rows_remote=19352 dedup_ratio=0.334900 router_tokens_per_expert=[271, 227, 264, 247, 248, 250, 220, 279, 267, 264, 256, 262, 263, 241, 249, 278, 224, 256, 239, 251, 232, 257, 233, 279, 263, 268, 244, 249, 251, 260, 279, 243, 237, 269, 261, 256, 247, 271, 257, 268, 250, 239, 249, 249, 251, 245, 281, 248, 245, 245, 237, 258, 262, 259, 269, 241, 234, 254, 267, 256, 233, 242, 235, 253, 260, 280, 268, 261, 273, 266, 267, 254, 253, 267, 258, 282, 228, 245, 281, 270, 288, 229, 298, 266, 260, 259, 267, 243, 252, 259, 270, 231, 278, 269, 263, 266, 249, 284, 257, 252, 248, 277, 284, 237, 252, 247, 256, 265, 254, 263, 248, 248, 297, 253, 233, 266, 293, 251, 223, 263, 234, 238, 256, 230, 254, 280, 247, 236] payload_rows_per_dst=[5468, 5400, 5440, 5384, 5522, 5464, 5468, 5442] dup_rows_per_dst=[2704, 2656, 2716, 2596, 2904, 2932, 2774, 2666] combine_rows_per_src=[5468, 5400, 5440, 5384, 5522, 5464, 5468, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38188 dup_rows=21860 dup_rows_remote=19096 dedup_ratio=0.333557 router_tokens_per_expert=[260, 261, 252, 213, 254, 229, 266, 235, 243, 277, 242, 249, 260, 252, 246, 259, 278, 251, 246, 223, 272, 249, 261, 285, 263, 261, 237, 250, 270, 239, 266, 239, 256, 247, 248, 261, 263, 274, 239, 236, 249, 256, 238, 262, 254, 267, 272, 264, 255, 281, 254, 266, 254, 247, 271, 269, 248, 246, 222, 247, 276, 274, 249, 278, 226, 257, 258, 259, 271, 261, 266, 251, 285, 241, 263, 272, 266, 249, 264, 245, 257, 245, 247, 249, 266, 259, 271, 223, 254, 252, 274, 266, 231, 259, 265, 257, 240, 297, 234, 243, 265, 251, 266, 235, 266, 270, 263, 240, 269, 288, 279, 256, 248, 255, 256, 240, 250, 269, 269, 246, 246, 243, 287, 265, 246, 232, 267, 267] payload_rows_per_dst=[5346, 5376, 5444, 5598, 5422, 5408, 5594, 5488] dup_rows_per_dst=[2650, 2804, 2728, 2676, 2766, 2732, 2740, 2764] combine_rows_per_src=[5346, 5376, 5444, 5598, 5422, 5408, 5594, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38294 dup_rows=21728 dup_rows_remote=19000 dedup_ratio=0.331543 router_tokens_per_expert=[243, 272, 280, 245, 262, 273, 242, 265, 247, 275, 268, 277, 241, 257, 247, 284, 261, 238, 263, 273, 245, 245, 258, 266, 250, 283, 237, 244, 234, 260, 241, 243, 269, 231, 273, 246, 236, 246, 225, 278, 252, 257, 262, 253, 250, 246, 274, 226, 250, 245, 252, 243, 254, 239, 268, 238, 264, 278, 265, 263, 256, 258, 235, 231, 258, 261, 265, 239, 273, 266, 262, 263, 273, 272, 241, 294, 214, 263, 238, 239, 266, 219, 243, 237, 261, 278, 276, 245, 246, 285, 262, 251, 262, 262, 251, 281, 269, 247, 258, 234, 269, 266, 244, 222, 262, 272, 288, 236, 247, 281, 247, 260, 273, 236, 284, 254, 260, 249, 258, 269, 243, 257, 251, 248, 240, 285, 242, 289] payload_rows_per_dst=[5602, 5458, 5390, 5394, 5514, 5490, 5452, 5508] dup_rows_per_dst=[2754, 2624, 2658, 2684, 2728, 2760, 2752, 2768] combine_rows_per_src=[5602, 5458, 5390, 5394, 5514, 5490, 5452, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38124 dup_rows=21952 dup_rows_remote=19108 dedup_ratio=0.334961 router_tokens_per_expert=[281, 252, 245, 263, 227, 264, 289, 249, 252, 243, 259, 233, 257, 256, 270, 239, 250, 272, 298, 289, 276, 255, 232, 260, 236, 275, 239, 270, 288, 241, 253, 247, 252, 258, 229, 260, 259, 250, 264, 267, 250, 267, 244, 279, 261, 245, 247, 240, 278, 247, 274, 249, 245, 252, 263, 256, 264, 246, 238, 240, 262, 270, 242, 272, 258, 238, 238, 257, 239, 251, 239, 272, 227, 249, 261, 244, 249, 243, 262, 258, 264, 292, 255, 242, 249, 269, 265, 272, 272, 220, 246, 251, 257, 266, 278, 247, 263, 245, 254, 282, 255, 243, 270, 280, 260, 255, 238, 260, 256, 265, 255, 271, 248, 255, 247, 259, 231, 256, 267, 279, 250, 243, 240, 281, 238, 246, 266, 250] payload_rows_per_dst=[5470, 5606, 5406, 5418, 5324, 5488, 5460, 5412] dup_rows_per_dst=[2688, 2756, 2738, 2778, 2646, 2802, 2844, 2700] combine_rows_per_src=[5470, 5606, 5406, 5418, 5324, 5488, 5460, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38314 dup_rows=21686 dup_rows_remote=19038 dedup_ratio=0.330902 router_tokens_per_expert=[247, 248, 239, 251, 254, 267, 240, 241, 259, 227, 264, 239, 253, 255, 274, 263, 258, 235, 263, 242, 266, 258, 254, 231, 270, 262, 239, 257, 249, 244, 255, 242, 279, 252, 255, 246, 244, 265, 232, 231, 290, 254, 267, 280, 258, 242, 224, 260, 251, 278, 254, 236, 246, 227, 281, 255, 271, 276, 254, 269, 246, 240, 246, 264, 279, 273, 269, 276, 250, 267, 292, 294, 270, 247, 238, 283, 274, 264, 268, 267, 269, 239, 250, 273, 285, 279, 223, 235, 248, 242, 260, 260, 263, 236, 280, 250, 266, 260, 234, 279, 242, 258, 264, 276, 240, 267, 259, 269, 255, 225, 239, 252, 239, 261, 256, 230, 258, 246, 266, 265, 250, 277, 266, 254, 252, 261, 244, 236] payload_rows_per_dst=[5410, 5406, 5496, 5520, 5634, 5536, 5432, 5416] dup_rows_per_dst=[2632, 2644, 2662, 2668, 2988, 2648, 2738, 2706] combine_rows_per_src=[5410, 5406, 5496, 5520, 5634, 5536, 5432, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=230224 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156713808 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313197392 weight_movement_bytes=18874368 total_nvlink_bytes=332071760 weight_to_token_ratio=0.060263 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38330 dup_rows=21724 dup_rows_remote=19050 dedup_ratio=0.331482 router_tokens_per_expert=[237, 242, 262, 276, 247, 220, 250, 258, 250, 257, 268, 269, 245, 253, 261, 267, 276, 267, 277, 251, 243, 264, 249, 241, 238, 243, 236, 263, 258, 260, 287, 265, 248, 251, 265, 229, 231, 263, 266, 266, 265, 264, 250, 260, 264, 236, 253, 267, 274, 286, 242, 265, 270, 249, 260, 272, 257, 240, 267, 271, 245, 254, 274, 258, 263, 262, 266, 237, 249, 239, 227, 256, 234, 259, 266, 241, 272, 239, 264, 282, 249, 263, 254, 247, 246, 282, 255, 249, 257, 286, 261, 264, 228, 257, 267, 257, 267, 240, 255, 248, 245, 239, 242, 261, 243, 246, 261, 224, 269, 256, 228, 277, 246, 262, 243, 274, 273, 278, 257, 238, 267, 270, 254, 256, 272, 233, 260, 264] payload_rows_per_dst=[5452, 5472, 5482, 5540, 5430, 5540, 5418, 5478] dup_rows_per_dst=[2672, 2764, 2674, 2828, 2682, 2704, 2584, 2816] combine_rows_per_src=[5452, 5472, 5482, 5540, 5430, 5540, 5418, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38308 dup_rows=21752 dup_rows_remote=19006 dedup_ratio=0.331909 router_tokens_per_expert=[246, 273, 254, 228, 287, 245, 273, 236, 247, 242, 254, 265, 277, 258, 267, 248, 286, 237, 281, 275, 254, 244, 265, 251, 245, 238, 245, 285, 237, 262, 245, 261, 269, 270, 268, 253, 237, 260, 261, 261, 215, 285, 259, 274, 232, 306, 247, 240, 303, 247, 232, 229, 253, 248, 253, 260, 266, 264, 279, 256, 240, 243, 229, 257, 265, 231, 266, 257, 236, 259, 249, 267, 266, 246, 227, 241, 244, 256, 267, 253, 255, 248, 242, 251, 253, 249, 273, 241, 264, 226, 251, 243, 244, 245, 273, 268, 274, 253, 270, 238, 290, 255, 266, 268, 280, 258, 263, 265, 246, 242, 260, 264, 210, 287, 252, 264, 256, 248, 265, 221, 263, 243, 256, 260, 264, 300, 294, 230] payload_rows_per_dst=[5434, 5476, 5538, 5478, 5444, 5376, 5564, 5474] dup_rows_per_dst=[2766, 2746, 2736, 2640, 2616, 2676, 2820, 2752] combine_rows_per_src=[5434, 5476, 5538, 5478, 5444, 5376, 5564, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229136 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156647184 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065232 weight_movement_bytes=18874368 total_nvlink_bytes=331939600 weight_to_token_ratio=0.060289 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229176 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081400 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933624 weight_movement_bytes=37748736 total_nvlink_bytes=351682360 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229408 combine_payload_bytes=156934144 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157163552 ingress_bytes=213557248 link_bytes=213557248 token_bytes=314097696 weight_movement_bytes=56623104 total_nvlink_bytes=370720800 weight_to_token_ratio=0.180272 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046200474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228928 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156384832 ingress_bytes=231653376 link_bytes=231653376 token_bytes=312540736 weight_movement_bytes=75497472 total_nvlink_bytes=388038208 weight_to_token_ratio=0.241560 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219588668e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229520 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157229200 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228880 weight_movement_bytes=18874368 total_nvlink_bytes=333103248 weight_to_token_ratio=0.060066 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229256 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157138824 ingress_bytes=175783936 link_bytes=175783936 token_bytes=314048392 weight_movement_bytes=18874368 total_nvlink_bytes=332922760 weight_to_token_ratio=0.060100 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684275184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38106 dup_rows=21892 dup_rows_remote=19082 dedup_ratio=0.334045 router_tokens_per_expert=[257, 270, 258, 233, 257, 300, 257, 255, 259, 268, 261, 248, 283, 279, 287, 258, 222, 222, 260, 256, 230, 234, 258, 258, 263, 241, 261, 265, 250, 242, 275, 246, 253, 231, 282, 249, 259, 267, 243, 249, 242, 250, 261, 254, 245, 272, 262, 289, 249, 279, 241, 245, 262, 268, 252, 270, 257, 261, 282, 261, 273, 258, 252, 264, 273, 259, 250, 268, 249, 249, 258, 266, 260, 260, 279, 252, 257, 251, 240, 256, 250, 257, 265, 235, 253, 265, 246, 255, 274, 265, 277, 249, 247, 230, 244, 264, 252, 230, 245, 241, 242, 287, 212, 260, 280, 263, 257, 263, 287, 257, 256, 230, 249, 286, 279, 252, 241, 225, 243, 227, 253, 249, 262, 220, 241, 268, 262, 251] payload_rows_per_dst=[5526, 5414, 5448, 5538, 5442, 5436, 5416, 5424] dup_rows_per_dst=[2934, 2552, 2768, 2810, 2812, 2716, 2708, 2592] combine_rows_per_src=[5526, 5414, 5448, 5538, 5442, 5436, 5416, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=37998 dup_rows=21924 dup_rows_remote=19056 dedup_ratio=0.334534 router_tokens_per_expert=[261, 241, 249, 269, 284, 248, 244, 233, 284, 280, 253, 237, 259, 277, 252, 257, 258, 266, 235, 254, 259, 242, 271, 271, 249, 246, 267, 244, 264, 270, 244, 274, 252, 262, 260, 269, 259, 231, 264, 261, 255, 244, 243, 250, 261, 269, 298, 265, 261, 259, 235, 259, 272, 241, 233, 259, 237, 257, 255, 273, 242, 251, 283, 242, 226, 272, 243, 274, 256, 233, 261, 264, 251, 241, 261, 273, 280, 264, 262, 255, 264, 252, 258, 254, 281, 271, 257, 246, 242, 236, 250, 244, 248, 250, 248, 252, 251, 255, 288, 250, 265, 241, 236, 253, 261, 244, 284, 272, 253, 260, 238, 269, 228, 267, 267, 245, 265, 236, 273, 255, 240, 254, 241, 250, 239, 249, 261, 265] payload_rows_per_dst=[5614, 5604, 5636, 5414, 5320, 5530, 5452, 5042] dup_rows_per_dst=[2868, 3050, 3050, 2740, 2534, 2684, 2660, 2338] combine_rows_per_src=[5614, 5604, 5636, 5414, 5320, 5530, 5452, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228752 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156310928 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312393104 weight_movement_bytes=18874368 total_nvlink_bytes=331267472 weight_to_token_ratio=0.060419 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38272 dup_rows=21740 dup_rows_remote=18846 dedup_ratio=0.331726 router_tokens_per_expert=[241, 256, 244, 236, 228, 244, 245, 255, 241, 255, 256, 242, 280, 265, 249, 252, 264, 262, 254, 261, 281, 292, 273, 245, 259, 240, 269, 276, 264, 273, 250, 256, 246, 254, 293, 271, 242, 265, 248, 255, 266, 247, 240, 272, 247, 246, 245, 284, 213, 252, 249, 234, 261, 251, 257, 252, 226, 234, 260, 265, 273, 295, 229, 269, 253, 271, 261, 261, 242, 267, 277, 257, 246, 264, 257, 269, 250, 288, 242, 252, 253, 253, 243, 286, 269, 312, 269, 240, 247, 256, 241, 258, 256, 249, 221, 266, 258, 245, 269, 249, 262, 236, 225, 274, 247, 238, 233, 253, 263, 247, 263, 253, 263, 244, 280, 254, 257, 250, 279, 253, 254, 274, 275, 231, 258, 231, 276, 249] payload_rows_per_dst=[5396, 5498, 5414, 5416, 5350, 5552, 5646, 5524] dup_rows_per_dst=[2396, 2940, 2700, 2774, 2452, 2686, 2898, 2894] combine_rows_per_src=[5396, 5498, 5414, 5416, 5350, 5552, 5646, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38402 dup_rows=21896 dup_rows_remote=19220 dedup_ratio=0.334106 router_tokens_per_expert=[270, 266, 224, 272, 286, 258, 243, 239, 270, 252, 252, 236, 251, 229, 247, 255, 238, 236, 278, 254, 272, 263, 244, 250, 254, 253, 261, 262, 270, 240, 251, 252, 254, 261, 279, 268, 255, 267, 294, 277, 227, 295, 257, 255, 278, 238, 274, 252, 254, 250, 256, 263, 249, 242, 274, 262, 265, 291, 248, 274, 256, 230, 241, 249, 238, 254, 266, 246, 248, 244, 240, 262, 273, 241, 263, 261, 276, 294, 281, 250, 255, 240, 247, 259, 247, 244, 245, 261, 258, 260, 271, 233, 278, 243, 260, 236, 232, 239, 261, 244, 258, 228, 223, 238, 270, 251, 253, 259, 225, 255, 284, 229, 255, 258, 254, 257, 273, 262, 248, 263, 258, 277, 259, 259, 275, 249, 266, 269] payload_rows_per_dst=[5418, 5410, 5582, 5526, 5528, 5402, 5238, 5536] dup_rows_per_dst=[2682, 2746, 2880, 2682, 2730, 2672, 2676, 2828] combine_rows_per_src=[5418, 5410, 5582, 5526, 5528, 5402, 5238, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38248 dup_rows=21774 dup_rows_remote=19208 dedup_ratio=0.332245 router_tokens_per_expert=[238, 265, 267, 269, 255, 246, 269, 266, 275, 238, 247, 262, 235, 245, 251, 252, 262, 272, 245, 250, 233, 235, 262, 258, 254, 242, 256, 266, 241, 261, 261, 275, 269, 282, 240, 259, 258, 244, 224, 254, 273, 283, 245, 244, 247, 219, 264, 254, 268, 230, 268, 281, 268, 250, 278, 257, 270, 252, 240, 269, 251, 240, 261, 234, 268, 259, 260, 276, 265, 247, 244, 265, 263, 231, 253, 225, 278, 248, 285, 259, 270, 249, 273, 271, 245, 229, 259, 245, 248, 246, 252, 259, 227, 263, 251, 253, 255, 262, 257, 255, 261, 263, 264, 254, 230, 242, 256, 262, 248, 279, 299, 248, 257, 268, 236, 262, 258, 267, 249, 274, 240, 242, 286, 250, 290, 258, 265, 236] payload_rows_per_dst=[5508, 5432, 5384, 5392, 5506, 5514, 5548, 5478] dup_rows_per_dst=[2652, 2714, 2734, 2842, 2746, 2566, 2722, 2798] combine_rows_per_src=[5508, 5432, 5384, 5392, 5506, 5514, 5548, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38248 dup_rows=21798 dup_rows_remote=19036 dedup_ratio=0.332611 router_tokens_per_expert=[233, 263, 262, 242, 256, 259, 246, 251, 255, 255, 253, 266, 249, 236, 264, 234, 284, 243, 284, 245, 238, 247, 273, 245, 278, 259, 251, 250, 286, 268, 282, 259, 251, 272, 269, 270, 262, 265, 264, 269, 259, 247, 279, 279, 227, 251, 247, 267, 276, 267, 256, 228, 238, 272, 277, 255, 253, 261, 270, 242, 253, 234, 250, 262, 236, 272, 286, 271, 267, 256, 242, 239, 237, 286, 284, 272, 244, 236, 242, 256, 256, 234, 252, 247, 259, 265, 236, 236, 262, 260, 265, 277, 246, 266, 236, 226, 260, 227, 245, 272, 262, 258, 267, 234, 262, 251, 240, 262, 242, 264, 280, 287, 257, 235, 251, 279, 241, 257, 227, 246, 262, 238, 246, 276, 244, 239, 271, 249] payload_rows_per_dst=[5392, 5630, 5504, 5426, 5490, 5412, 5510, 5374] dup_rows_per_dst=[2656, 2754, 2852, 2762, 2762, 2634, 2716, 2662] combine_rows_per_src=[5392, 5630, 5504, 5426, 5490, 5412, 5510, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38416 dup_rows=21726 dup_rows_remote=18988 dedup_ratio=0.331512 router_tokens_per_expert=[260, 295, 258, 240, 264, 264, 282, 290, 245, 263, 276, 272, 226, 266, 267, 257, 233, 250, 257, 254, 249, 258, 259, 254, 247, 274, 244, 257, 258, 245, 256, 270, 259, 246, 254, 234, 254, 260, 262, 252, 252, 255, 254, 251, 275, 261, 251, 246, 241, 263, 235, 245, 259, 266, 260, 242, 267, 232, 244, 222, 250, 261, 240, 253, 236, 274, 264, 275, 257, 256, 280, 272, 235, 246, 270, 265, 251, 266, 243, 253, 248, 252, 256, 246, 224, 250, 245, 246, 251, 252, 276, 241, 259, 254, 253, 252, 274, 250, 241, 260, 262, 255, 276, 272, 289, 246, 222, 245, 243, 262, 257, 257, 281, 274, 260, 301, 238, 252, 257, 255, 238, 271, 243, 293, 282, 240, 250, 238] payload_rows_per_dst=[5544, 5476, 5394, 5414, 5478, 5438, 5532, 5534] dup_rows_per_dst=[2906, 2654, 2738, 2546, 2808, 2572, 2690, 2812] combine_rows_per_src=[5544, 5476, 5394, 5414, 5478, 5438, 5532, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38116 dup_rows=21832 dup_rows_remote=19136 dedup_ratio=0.333130 router_tokens_per_expert=[271, 247, 266, 279, 247, 240, 245, 272, 256, 252, 261, 265, 256, 268, 261, 276, 271, 251, 239, 279, 283, 245, 247, 269, 255, 253, 269, 269, 251, 257, 235, 269, 234, 223, 280, 255, 230, 246, 228, 293, 263, 254, 263, 265, 257, 239, 252, 242, 241, 250, 271, 250, 245, 244, 268, 226, 256, 230, 284, 252, 250, 273, 229, 261, 256, 245, 263, 270, 259, 257, 255, 282, 254, 231, 264, 266, 251, 236, 270, 271, 273, 242, 229, 277, 259, 232, 254, 242, 271, 223, 274, 275, 259, 245, 257, 254, 268, 260, 266, 270, 240, 221, 258, 265, 237, 281, 274, 268, 272, 257, 235, 251, 283, 267, 294, 261, 239, 269, 240, 235, 245, 286, 247, 249, 251, 237, 246, 242] payload_rows_per_dst=[5484, 5588, 5398, 5416, 5486, 5398, 5452, 5482] dup_rows_per_dst=[2840, 2696, 2650, 2644, 2774, 2734, 2794, 2700] combine_rows_per_src=[5484, 5588, 5398, 5416, 5486, 5398, 5452, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228472 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156990584 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313752696 weight_movement_bytes=37748736 total_nvlink_bytes=351501432 weight_to_token_ratio=0.120314 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=230488 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157525080 ingress_bytes=176168960 link_bytes=176168960 token_bytes=314819672 weight_movement_bytes=18874368 total_nvlink_bytes=333694040 weight_to_token_ratio=0.059953 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687964294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229824 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893632 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557440 weight_movement_bytes=18874368 total_nvlink_bytes=332431808 weight_to_token_ratio=0.060194 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229616 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157581552 ingress_bytes=195100672 link_bytes=195100672 token_bytes=314933488 weight_movement_bytes=37748736 total_nvlink_bytes=352682224 weight_to_token_ratio=0.119863 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869358644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229008 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352144 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475280 weight_movement_bytes=18874368 total_nvlink_bytes=331349648 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229136 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156892944 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313556752 weight_movement_bytes=56623104 total_nvlink_bytes=370179856 weight_to_token_ratio=0.180583 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155639808 dispatch_meta_bytes=228216 combine_payload_bytes=155639808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155868024 ingress_bytes=174514176 link_bytes=174514176 token_bytes=311507832 weight_movement_bytes=18874368 total_nvlink_bytes=330382200 weight_to_token_ratio=0.060590 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672108968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38318 dup_rows=21746 dup_rows_remote=19002 dedup_ratio=0.331818 router_tokens_per_expert=[230, 240, 250, 251, 242, 245, 253, 252, 251, 260, 255, 259, 249, 233, 246, 255, 249, 246, 244, 250, 262, 242, 240, 236, 272, 274, 273, 257, 260, 247, 248, 257, 224, 264, 263, 245, 264, 261, 241, 274, 256, 273, 269, 264, 267, 267, 272, 261, 250, 252, 265, 270, 230, 253, 248, 259, 263, 231, 281, 240, 262, 274, 264, 266, 252, 260, 259, 259, 241, 238, 271, 284, 247, 250, 247, 251, 261, 270, 248, 273, 281, 290, 269, 255, 232, 267, 290, 265, 282, 250, 256, 274, 252, 272, 247, 248, 245, 232, 273, 249, 254, 233, 247, 243, 247, 264, 249, 253, 261, 246, 247, 238, 256, 224, 267, 272, 264, 262, 248, 268, 254, 265, 264, 250, 245, 254, 297, 255] payload_rows_per_dst=[5376, 5400, 5546, 5472, 5482, 5584, 5420, 5510] dup_rows_per_dst=[2566, 2714, 2784, 2744, 2740, 2876, 2542, 2780] combine_rows_per_src=[5376, 5400, 5546, 5472, 5482, 5584, 5420, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229280 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179808 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130336 weight_movement_bytes=37748736 total_nvlink_bytes=351879072 weight_to_token_ratio=0.120169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38188 dup_rows=21822 dup_rows_remote=18970 dedup_ratio=0.332977 router_tokens_per_expert=[263, 264, 241, 271, 247, 296, 278, 294, 284, 224, 260, 246, 257, 263, 273, 260, 258, 269, 240, 266, 260, 264, 262, 237, 239, 275, 255, 245, 275, 278, 265, 266, 235, 235, 247, 251, 233, 241, 239, 264, 273, 301, 260, 233, 250, 261, 271, 253, 261, 256, 240, 251, 276, 280, 244, 247, 246, 250, 248, 243, 265, 242, 281, 243, 216, 266, 245, 268, 253, 259, 252, 241, 247, 262, 248, 256, 271, 244, 256, 259, 233, 228, 247, 266, 275, 251, 246, 247, 241, 240, 262, 275, 243, 255, 259, 266, 246, 245, 244, 263, 244, 264, 264, 243, 286, 268, 268, 245, 243, 238, 246, 253, 255, 279, 267, 236, 275, 274, 249, 267, 270, 260, 232, 268, 266, 224, 266, 248] payload_rows_per_dst=[5506, 5490, 5398, 5452, 5492, 5206, 5644, 5526] dup_rows_per_dst=[2936, 2712, 2696, 2694, 2590, 2372, 2970, 2852] combine_rows_per_src=[5506, 5490, 5398, 5452, 5492, 5206, 5644, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38252 dup_rows=21838 dup_rows_remote=19046 dedup_ratio=0.333221 router_tokens_per_expert=[254, 271, 265, 262, 263, 245, 256, 247, 258, 243, 264, 268, 230, 245, 267, 245, 254, 254, 257, 254, 249, 292, 259, 254, 256, 235, 255, 265, 247, 249, 250, 231, 254, 215, 249, 266, 275, 251, 254, 263, 260, 287, 266, 266, 266, 268, 249, 267, 223, 267, 241, 247, 285, 265, 255, 256, 250, 245, 242, 240, 258, 237, 247, 257, 252, 231, 226, 264, 277, 287, 258, 242, 237, 271, 252, 267, 281, 252, 249, 234, 258, 263, 267, 230, 257, 252, 271, 268, 258, 254, 243, 235, 273, 244, 279, 285, 239, 273, 274, 249, 259, 245, 283, 249, 244, 278, 247, 236, 243, 255, 273, 242, 272, 239, 278, 266, 255, 247, 259, 256, 265, 250, 263, 278, 246, 269, 269, 235] payload_rows_per_dst=[5438, 5472, 5506, 5412, 5480, 5460, 5446, 5484] dup_rows_per_dst=[2728, 2650, 2806, 2618, 2680, 2754, 2792, 2810] combine_rows_per_src=[5438, 5472, 5506, 5412, 5480, 5460, 5446, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43528 payload_rows_remote=38140 dup_rows=22008 dup_rows_remote=19282 dedup_ratio=0.335815 router_tokens_per_expert=[242, 283, 257, 266, 274, 235, 244, 205, 256, 257, 257, 229, 264, 257, 257, 290, 269, 236, 228, 249, 249, 260, 256, 287, 240, 291, 250, 287, 257, 270, 280, 235, 251, 249, 257, 264, 249, 286, 268, 238, 263, 241, 238, 265, 241, 248, 265, 234, 239, 221, 272, 238, 295, 244, 273, 256, 243, 248, 290, 230, 247, 294, 263, 254, 220, 231, 242, 257, 272, 255, 257, 276, 296, 256, 234, 232, 267, 253, 277, 260, 267, 245, 292, 252, 282, 255, 264, 259, 251, 266, 267, 236, 252, 251, 248, 272, 259, 267, 244, 259, 241, 257, 241, 274, 278, 249, 256, 236, 244, 247, 256, 268, 229, 276, 270, 239, 266, 246, 240, 262, 228, 266, 251, 274, 265, 250, 245, 260] payload_rows_per_dst=[5364, 5530, 5388, 5398, 5402, 5610, 5428, 5408] dup_rows_per_dst=[2782, 2758, 2726, 2816, 2768, 2708, 2724, 2726] combine_rows_per_src=[5364, 5530, 5388, 5398, 5402, 5610, 5428, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38226 dup_rows=21920 dup_rows_remote=19178 dedup_ratio=0.334473 router_tokens_per_expert=[248, 249, 274, 249, 249, 261, 262, 261, 265, 274, 249, 261, 270, 238, 226, 243, 289, 253, 253, 254, 230, 259, 280, 262, 253, 265, 264, 234, 241, 235, 256, 238, 246, 251, 266, 260, 236, 245, 266, 272, 288, 245, 282, 260, 225, 264, 269, 245, 268, 266, 267, 281, 247, 249, 231, 253, 245, 271, 276, 251, 234, 246, 261, 265, 267, 265, 263, 242, 252, 247, 276, 257, 260, 271, 253, 246, 257, 245, 262, 258, 224, 254, 278, 250, 242, 264, 251, 252, 252, 268, 233, 256, 286, 265, 239, 256, 250, 238, 236, 245, 255, 273, 274, 251, 265, 254, 281, 287, 246, 263, 257, 279, 231, 242, 276, 266, 254, 240, 238, 235, 268, 245, 235, 276, 288, 263, 237, 253] payload_rows_per_dst=[5408, 5390, 5472, 5512, 5476, 5464, 5490, 5404] dup_rows_per_dst=[2750, 2742, 2768, 2710, 2766, 2676, 2818, 2690] combine_rows_per_src=[5408, 5390, 5472, 5512, 5476, 5464, 5490, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38158 dup_rows=21838 dup_rows_remote=19046 dedup_ratio=0.333221 router_tokens_per_expert=[292, 234, 241, 285, 240, 259, 238, 249, 266, 283, 245, 231, 237, 260, 249, 237, 295, 243, 244, 263, 269, 231, 293, 252, 265, 274, 244, 227, 253, 282, 223, 250, 245, 226, 231, 287, 290, 262, 246, 245, 240, 259, 239, 245, 237, 257, 239, 222, 219, 273, 273, 258, 261, 269, 235, 259, 262, 246, 266, 253, 259, 259, 287, 280, 246, 250, 265, 279, 260, 277, 253, 236, 282, 284, 265, 230, 286, 241, 257, 258, 258, 272, 253, 269, 264, 250, 273, 262, 284, 246, 256, 267, 253, 232, 279, 248, 276, 257, 269, 239, 258, 240, 253, 251, 283, 239, 269, 257, 287, 239, 243, 215, 261, 243, 244, 254, 266, 268, 278, 248, 276, 261, 251, 254, 261, 233, 216, 261] payload_rows_per_dst=[5374, 5438, 5352, 5538, 5592, 5540, 5378, 5486] dup_rows_per_dst=[2718, 2778, 2588, 2780, 2746, 2792, 2772, 2664] combine_rows_per_src=[5374, 5438, 5352, 5538, 5592, 5540, 5378, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43522 payload_rows_remote=38166 dup_rows=22014 dup_rows_remote=19236 dedup_ratio=0.335907 router_tokens_per_expert=[283, 243, 239, 256, 263, 242, 203, 266, 261, 253, 262, 265, 252, 258, 261, 273, 257, 260, 256, 262, 238, 269, 268, 243, 249, 243, 238, 253, 234, 248, 260, 241, 292, 266, 239, 269, 265, 241, 261, 270, 262, 268, 262, 288, 263, 274, 250, 236, 267, 255, 241, 268, 240, 268, 261, 254, 249, 255, 273, 277, 262, 244, 264, 254, 262, 278, 267, 246, 281, 260, 246, 264, 254, 260, 244, 225, 252, 252, 240, 236, 260, 237, 261, 261, 272, 273, 242, 274, 266, 274, 256, 238, 265, 264, 284, 256, 246, 246, 237, 255, 250, 229, 254, 242, 247, 281, 274, 221, 227, 274, 247, 252, 247, 263, 287, 257, 265, 254, 252, 249, 240, 249, 268, 220, 275, 255, 247, 271] payload_rows_per_dst=[5438, 5348, 5522, 5464, 5356, 5536, 5386, 5472] dup_rows_per_dst=[2722, 2690, 2890, 2800, 2778, 2830, 2578, 2726] combine_rows_per_src=[5438, 5348, 5522, 5464, 5356, 5536, 5386, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228632 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156646680 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313064728 weight_movement_bytes=18874368 total_nvlink_bytes=331939096 weight_to_token_ratio=0.060289 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229192 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909384 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589576 weight_movement_bytes=18874368 total_nvlink_bytes=332463944 weight_to_token_ratio=0.060188 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229688 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156451128 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312672568 weight_movement_bytes=18874368 total_nvlink_bytes=331546936 weight_to_token_ratio=0.060365 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38120 dup_rows=21830 dup_rows_remote=18928 dedup_ratio=0.333099 router_tokens_per_expert=[265, 264, 268, 271, 248, 257, 289, 228, 244, 244, 257, 270, 228, 279, 244, 256, 253, 236, 264, 243, 254, 268, 268, 254, 270, 229, 260, 262, 239, 279, 284, 258, 254, 272, 264, 260, 251, 217, 242, 271, 286, 255, 268, 270, 242, 270, 272, 267, 237, 278, 266, 251, 253, 250, 260, 231, 272, 252, 262, 311, 254, 264, 259, 242, 274, 245, 262, 254, 245, 247, 217, 249, 243, 261, 242, 250, 249, 230, 251, 236, 273, 253, 240, 224, 285, 263, 232, 246, 281, 265, 260, 244, 267, 263, 263, 266, 240, 253, 295, 242, 264, 262, 243, 246, 247, 242, 275, 233, 267, 258, 245, 267, 246, 245, 279, 261, 250, 250, 264, 241, 248, 252, 237, 265, 262, 254, 249, 270] payload_rows_per_dst=[5586, 5518, 5586, 5552, 5356, 5290, 5384, 5434] dup_rows_per_dst=[2902, 2830, 2842, 2782, 2560, 2690, 2512, 2712] combine_rows_per_src=[5586, 5518, 5586, 5552, 5356, 5290, 5384, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229616 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156803312 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313377008 weight_movement_bytes=37748736 total_nvlink_bytes=351125744 weight_to_token_ratio=0.120458 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38356 dup_rows=21786 dup_rows_remote=18956 dedup_ratio=0.332428 router_tokens_per_expert=[283, 258, 238, 259, 257, 251, 254, 239, 237, 287, 275, 237, 246, 230, 247, 216, 250, 257, 221, 268, 251, 236, 255, 272, 263, 244, 261, 241, 285, 292, 245, 266, 234, 270, 267, 243, 277, 275, 252, 237, 267, 254, 236, 263, 281, 271, 285, 278, 236, 280, 246, 269, 255, 267, 243, 232, 288, 259, 250, 245, 280, 243, 254, 265, 254, 260, 224, 242, 254, 245, 231, 233, 238, 252, 261, 259, 264, 274, 249, 272, 247, 250, 260, 277, 238, 258, 248, 228, 257, 263, 244, 241, 258, 231, 282, 275, 268, 255, 253, 244, 265, 258, 264, 250, 260, 267, 257, 263, 239, 263, 272, 260, 233, 266, 222, 232, 288, 263, 267, 262, 265, 278, 252, 261, 255, 259, 295, 240] payload_rows_per_dst=[5454, 5502, 5538, 5394, 5434, 5442, 5464, 5522] dup_rows_per_dst=[2574, 2712, 2842, 2830, 2590, 2672, 2812, 2754] combine_rows_per_src=[5454, 5502, 5538, 5394, 5434, 5442, 5464, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=228816 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156523984 ingress_bytes=231792640 link_bytes=231792640 token_bytes=312819152 weight_movement_bytes=75497472 total_nvlink_bytes=388316624 weight_to_token_ratio=0.241345 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220923027e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=229608 combine_payload_bytes=156327936 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156557544 ingress_bytes=194076672 link_bytes=194076672 token_bytes=312885480 weight_movement_bytes=37748736 total_nvlink_bytes=350634216 weight_to_token_ratio=0.120647 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859547179e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228192 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156367712 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507232 weight_movement_bytes=18874368 total_nvlink_bytes=331381600 weight_to_token_ratio=0.060397 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229248 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157335424 ingress_bytes=194854912 link_bytes=194854912 token_bytes=314441600 weight_movement_bytes=37748736 total_nvlink_bytes=352190336 weight_to_token_ratio=0.120050 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867003892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43936 payload_rows_remote=38492 dup_rows=21600 dup_rows_remote=18984 dedup_ratio=0.329590 router_tokens_per_expert=[258, 245, 269, 249, 254, 270, 303, 256, 227, 259, 258, 262, 272, 247, 252, 294, 236, 266, 265, 238, 279, 250, 244, 279, 281, 245, 215, 249, 252, 285, 239, 250, 232, 247, 252, 272, 243, 283, 239, 219, 272, 250, 276, 244, 228, 244, 250, 279, 265, 236, 253, 278, 247, 245, 249, 265, 238, 244, 267, 271, 275, 271, 226, 245, 256, 229, 258, 258, 254, 266, 266, 264, 230, 257, 282, 245, 250, 259, 279, 277, 241, 243, 279, 248, 254, 274, 235, 268, 259, 257, 261, 238, 263, 259, 265, 264, 248, 248, 260, 284, 274, 270, 234, 264, 256, 276, 246, 256, 246, 251, 249, 270, 240, 263, 291, 240, 225, 229, 261, 254, 282, 272, 260, 246, 235, 229, 274, 244] payload_rows_per_dst=[5582, 5404, 5444, 5494, 5512, 5480, 5538, 5482] dup_rows_per_dst=[2768, 2742, 2616, 2656, 2748, 2736, 2726, 2608] combine_rows_per_src=[5582, 5404, 5444, 5494, 5512, 5480, 5538, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38384 dup_rows=21606 dup_rows_remote=18792 dedup_ratio=0.329681 router_tokens_per_expert=[256, 254, 274, 243, 269, 243, 237, 249, 263, 248, 264, 230, 257, 251, 245, 264, 239, 234, 234, 273, 274, 270, 259, 260, 244, 249, 285, 259, 274, 276, 256, 222, 257, 266, 250, 252, 268, 253, 256, 257, 277, 255, 249, 245, 261, 260, 269, 234, 236, 252, 255, 238, 241, 260, 257, 249, 260, 250, 260, 269, 280, 243, 274, 256, 256, 254, 261, 279, 261, 255, 247, 265, 248, 269, 227, 258, 267, 292, 255, 252, 265, 272, 252, 287, 276, 241, 234, 242, 261, 242, 241, 266, 258, 228, 272, 243, 233, 253, 234, 270, 242, 242, 262, 263, 260, 262, 267, 249, 254, 259, 239, 257, 272, 261, 270, 257, 236, 279, 253, 258, 287, 242, 272, 270, 269, 245, 237, 244] payload_rows_per_dst=[5516, 5524, 5534, 5498, 5454, 5486, 5372, 5546] dup_rows_per_dst=[2578, 2692, 2684, 2662, 2654, 2802, 2720, 2814] combine_rows_per_src=[5516, 5524, 5534, 5498, 5454, 5486, 5372, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38130 dup_rows=21956 dup_rows_remote=19070 dedup_ratio=0.335022 router_tokens_per_expert=[251, 256, 229, 262, 244, 263, 262, 247, 260, 256, 236, 238, 263, 279, 248, 225, 257, 268, 250, 244, 250, 238, 252, 253, 266, 244, 271, 251, 242, 307, 250, 255, 244, 253, 263, 264, 236, 255, 239, 240, 249, 243, 250, 265, 259, 272, 255, 254, 263, 253, 249, 258, 228, 271, 217, 252, 267, 256, 256, 239, 272, 260, 273, 256, 267, 250, 279, 260, 262, 244, 235, 262, 268, 274, 251, 267, 258, 266, 261, 257, 269, 223, 248, 270, 248, 255, 258, 266, 272, 270, 259, 242, 292, 243, 247, 258, 234, 255, 256, 268, 279, 236, 263, 289, 244, 249, 290, 243, 253, 251, 272, 286, 256, 236, 260, 257, 231, 266, 280, 262, 249, 250, 269, 253, 242, 264, 253, 263] payload_rows_per_dst=[5384, 5494, 5378, 5406, 5476, 5508, 5450, 5484] dup_rows_per_dst=[2654, 2702, 2704, 2734, 2846, 2732, 2886, 2698] combine_rows_per_src=[5384, 5494, 5378, 5406, 5476, 5508, 5450, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38304 dup_rows=21854 dup_rows_remote=19160 dedup_ratio=0.333466 router_tokens_per_expert=[261, 256, 260, 227, 260, 261, 270, 252, 263, 255, 240, 266, 247, 266, 261, 271, 248, 243, 244, 261, 238, 254, 273, 248, 253, 269, 254, 257, 243, 249, 241, 261, 280, 254, 270, 265, 280, 257, 240, 262, 257, 249, 251, 245, 247, 248, 244, 235, 247, 287, 233, 251, 281, 266, 250, 240, 277, 232, 249, 253, 282, 259, 250, 252, 259, 287, 271, 242, 260, 283, 259, 267, 262, 267, 254, 260, 241, 263, 249, 236, 215, 261, 271, 242, 272, 239, 252, 263, 260, 250, 271, 256, 264, 261, 269, 230, 278, 252, 262, 279, 259, 219, 224, 268, 243, 249, 261, 277, 286, 258, 263, 249, 251, 247, 261, 281, 248, 253, 274, 234, 273, 281, 219, 218, 264, 271, 249, 236] payload_rows_per_dst=[5492, 5378, 5392, 5488, 5510, 5500, 5462, 5460] dup_rows_per_dst=[2740, 2694, 2776, 2730, 2810, 2652, 2792, 2660] combine_rows_per_src=[5492, 5378, 5392, 5488, 5510, 5500, 5462, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38198 dup_rows=21940 dup_rows_remote=19244 dedup_ratio=0.334778 router_tokens_per_expert=[251, 244, 226, 252, 240, 269, 231, 274, 248, 256, 272, 243, 232, 265, 275, 280, 269, 252, 234, 284, 249, 242, 262, 258, 255, 266, 256, 268, 248, 240, 273, 234, 258, 258, 271, 246, 252, 234, 273, 267, 245, 276, 261, 267, 280, 253, 240, 222, 256, 262, 278, 239, 243, 250, 258, 243, 276, 264, 277, 289, 265, 269, 251, 263, 265, 256, 250, 261, 260, 232, 240, 253, 283, 258, 230, 221, 249, 254, 257, 260, 269, 267, 231, 256, 256, 263, 254, 268, 263, 265, 242, 244, 251, 267, 220, 231, 277, 255, 262, 237, 237, 288, 245, 265, 268, 268, 265, 257, 244, 260, 243, 222, 254, 254, 281, 268, 258, 248, 266, 266, 244, 251, 262, 249, 274, 249, 269, 272] payload_rows_per_dst=[5414, 5412, 5470, 5488, 5372, 5398, 5484, 5558] dup_rows_per_dst=[2702, 2768, 2736, 2878, 2686, 2696, 2702, 2772] combine_rows_per_src=[5414, 5412, 5470, 5488, 5372, 5398, 5484, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38122 dup_rows=21808 dup_rows_remote=19124 dedup_ratio=0.332764 router_tokens_per_expert=[259, 264, 230, 258, 264, 257, 258, 247, 265, 261, 266, 234, 269, 229, 256, 259, 276, 249, 266, 262, 243, 229, 263, 263, 233, 252, 255, 248, 242, 235, 247, 247, 253, 244, 258, 272, 256, 233, 240, 265, 247, 256, 259, 247, 264, 248, 263, 274, 269, 280, 236, 281, 230, 268, 289, 283, 249, 262, 267, 252, 252, 239, 248, 271, 272, 267, 269, 259, 255, 237, 241, 256, 288, 245, 272, 268, 268, 252, 250, 246, 258, 250, 271, 258, 266, 275, 276, 269, 250, 243, 259, 262, 243, 261, 246, 265, 258, 256, 237, 244, 263, 258, 244, 233, 301, 259, 244, 253, 259, 255, 253, 266, 283, 254, 241, 253, 290, 243, 272, 251, 244, 235, 247, 222, 248, 244, 255, 265] payload_rows_per_dst=[5364, 5418, 5480, 5586, 5606, 5486, 5408, 5380] dup_rows_per_dst=[2788, 2602, 2678, 2766, 2684, 2818, 2758, 2714] combine_rows_per_src=[5364, 5418, 5480, 5586, 5606, 5486, 5408, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157663232 dispatch_meta_bytes=229904 combine_payload_bytes=157663232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157893136 ingress_bytes=176537600 link_bytes=176537600 token_bytes=315556368 weight_movement_bytes=18874368 total_nvlink_bytes=334430736 weight_to_token_ratio=0.059813 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691496421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=228704 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157449568 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314670432 weight_movement_bytes=18874368 total_nvlink_bytes=333544800 weight_to_token_ratio=0.059981 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=228800 combine_payload_bytes=156180480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156409280 ingress_bytes=212803584 link_bytes=212803584 token_bytes=312589760 weight_movement_bytes=56623104 total_nvlink_bytes=369212864 weight_to_token_ratio=0.181142 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038979236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229856 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157123040 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314016224 weight_movement_bytes=18874368 total_nvlink_bytes=332890592 weight_to_token_ratio=0.060106 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229768 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688776 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147784 weight_movement_bytes=18874368 total_nvlink_bytes=332022152 weight_to_token_ratio=0.060273 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38270 dup_rows=21752 dup_rows_remote=19022 dedup_ratio=0.331909 router_tokens_per_expert=[243, 269, 245, 253, 239, 254, 266, 275, 282, 246, 237, 240, 276, 252, 271, 240, 249, 266, 259, 262, 266, 247, 243, 259, 266, 252, 249, 260, 264, 263, 265, 241, 254, 265, 268, 249, 252, 255, 244, 251, 236, 241, 253, 257, 243, 243, 266, 272, 246, 272, 263, 226, 268, 255, 255, 264, 262, 266, 257, 244, 283, 244, 239, 278, 252, 282, 250, 236, 275, 258, 265, 255, 247, 259, 268, 265, 218, 266, 274, 258, 243, 246, 273, 273, 259, 254, 248, 296, 252, 272, 252, 270, 244, 285, 234, 240, 261, 250, 248, 254, 251, 255, 253, 241, 247, 223, 252, 221, 251, 283, 274, 234, 245, 259, 237, 257, 286, 248, 226, 278, 255, 262, 254, 256, 276, 278, 255, 259] payload_rows_per_dst=[5464, 5430, 5454, 5514, 5566, 5492, 5348, 5516] dup_rows_per_dst=[2712, 2792, 2644, 2730, 2690, 2790, 2648, 2746] combine_rows_per_src=[5464, 5430, 5454, 5514, 5566, 5492, 5348, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228984 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156376696 ingress_bytes=231645184 link_bytes=231645184 token_bytes=312524408 weight_movement_bytes=75497472 total_nvlink_bytes=388021880 weight_to_token_ratio=0.241573 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219510177e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38146 dup_rows=21910 dup_rows_remote=19188 dedup_ratio=0.334320 router_tokens_per_expert=[225, 272, 261, 262, 234, 255, 247, 245, 258, 245, 264, 265, 254, 265, 282, 267, 268, 234, 243, 234, 233, 239, 267, 264, 256, 241, 247, 237, 285, 240, 228, 266, 257, 243, 245, 253, 233, 268, 253, 243, 264, 246, 234, 256, 244, 261, 263, 282, 231, 291, 260, 277, 273, 256, 251, 250, 262, 253, 273, 259, 247, 278, 260, 253, 252, 284, 216, 262, 264, 256, 261, 246, 234, 275, 265, 256, 263, 283, 237, 266, 250, 278, 264, 262, 278, 275, 261, 268, 276, 265, 236, 245, 249, 268, 272, 266, 258, 238, 245, 241, 270, 258, 271, 241, 248, 239, 253, 261, 236, 257, 243, 236, 227, 225, 239, 267, 283, 265, 279, 275, 237, 235, 273, 280, 260, 253, 279, 261] payload_rows_per_dst=[5480, 5356, 5460, 5460, 5468, 5658, 5416, 5328] dup_rows_per_dst=[2722, 2676, 2630, 2888, 2704, 3146, 2574, 2570] combine_rows_per_src=[5480, 5356, 5460, 5460, 5468, 5658, 5416, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38164 dup_rows=21820 dup_rows_remote=18980 dedup_ratio=0.332947 router_tokens_per_expert=[253, 263, 253, 240, 266, 256, 267, 268, 252, 240, 243, 228, 239, 262, 252, 256, 256, 242, 269, 246, 235, 277, 277, 234, 268, 271, 257, 230, 268, 245, 240, 269, 296, 256, 262, 265, 254, 243, 269, 260, 273, 278, 264, 254, 231, 270, 261, 260, 276, 271, 237, 236, 253, 256, 258, 232, 250, 255, 245, 269, 250, 250, 265, 269, 264, 241, 256, 301, 241, 253, 261, 238, 266, 247, 274, 257, 251, 275, 255, 255, 250, 251, 296, 249, 238, 256, 247, 228, 265, 232, 274, 249, 220, 247, 250, 240, 252, 261, 259, 241, 251, 259, 236, 268, 259, 251, 246, 274, 278, 246, 272, 261, 275, 254, 268, 242, 264, 285, 241, 259, 239, 268, 235, 273, 249, 260, 250, 275] payload_rows_per_dst=[5454, 5452, 5552, 5410, 5472, 5384, 5526, 5466] dup_rows_per_dst=[2622, 2716, 2840, 2734, 2798, 2600, 2702, 2808] combine_rows_per_src=[5454, 5452, 5552, 5410, 5472, 5384, 5526, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229168 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983088 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737008 weight_movement_bytes=18874368 total_nvlink_bytes=332611376 weight_to_token_ratio=0.060160 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=229336 combine_payload_bytes=156246016 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156475352 ingress_bytes=212869120 link_bytes=212869120 token_bytes=312721368 weight_movement_bytes=56623104 total_nvlink_bytes=369344472 weight_to_token_ratio=0.181066 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039607170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38246 dup_rows=21796 dup_rows_remote=18986 dedup_ratio=0.332581 router_tokens_per_expert=[271, 240, 262, 254, 273, 277, 252, 248, 254, 248, 287, 265, 240, 275, 248, 239, 279, 259, 224, 270, 226, 241, 258, 267, 270, 270, 245, 264, 235, 244, 270, 257, 261, 254, 246, 228, 246, 233, 238, 256, 234, 262, 269, 261, 257, 270, 217, 261, 249, 243, 259, 256, 250, 268, 228, 252, 261, 238, 261, 261, 247, 247, 244, 251, 255, 235, 224, 256, 252, 264, 237, 256, 265, 270, 269, 247, 275, 266, 266, 260, 278, 256, 234, 255, 281, 248, 238, 256, 260, 252, 281, 256, 241, 276, 266, 248, 286, 267, 266, 262, 261, 264, 275, 277, 254, 243, 259, 263, 251, 251, 258, 247, 267, 263, 260, 221, 238, 288, 282, 249, 264, 267, 274, 229, 266, 260, 266, 247] payload_rows_per_dst=[5448, 5314, 5522, 5382, 5480, 5500, 5600, 5494] dup_rows_per_dst=[2650, 2464, 2844, 2648, 2692, 2752, 2936, 2810] combine_rows_per_src=[5448, 5314, 5522, 5382, 5480, 5500, 5600, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38220 dup_rows=21768 dup_rows_remote=19000 dedup_ratio=0.332153 router_tokens_per_expert=[283, 276, 265, 232, 253, 254, 249, 263, 237, 264, 246, 262, 266, 231, 245, 259, 271, 255, 263, 268, 232, 281, 246, 287, 253, 246, 241, 260, 240, 269, 262, 244, 255, 251, 275, 281, 257, 253, 258, 255, 216, 244, 242, 269, 269, 281, 240, 258, 258, 271, 284, 238, 254, 251, 224, 227, 258, 257, 238, 250, 239, 245, 258, 292, 260, 269, 276, 274, 255, 245, 247, 260, 254, 242, 273, 233, 247, 270, 276, 238, 281, 280, 246, 251, 244, 260, 245, 256, 262, 241, 245, 233, 258, 241, 264, 251, 239, 261, 262, 252, 259, 264, 267, 290, 235, 246, 266, 269, 265, 279, 230, 274, 262, 256, 250, 264, 253, 257, 253, 225, 278, 258, 249, 263, 245, 279, 253, 237] payload_rows_per_dst=[5388, 5480, 5478, 5438, 5524, 5446, 5548, 5466] dup_rows_per_dst=[2782, 2756, 2730, 2650, 2714, 2670, 2768, 2698] combine_rows_per_src=[5388, 5480, 5478, 5438, 5524, 5446, 5548, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38252 dup_rows=21772 dup_rows_remote=19052 dedup_ratio=0.332214 router_tokens_per_expert=[235, 230, 278, 255, 232, 286, 267, 239, 262, 277, 238, 267, 261, 244, 240, 261, 268, 234, 244, 249, 255, 249, 253, 273, 266, 245, 237, 245, 283, 269, 272, 274, 231, 259, 267, 253, 255, 252, 228, 262, 237, 232, 279, 245, 227, 256, 244, 234, 262, 240, 262, 253, 265, 260, 257, 252, 267, 240, 261, 254, 247, 256, 254, 243, 266, 259, 276, 228, 251, 260, 262, 238, 251, 255, 286, 232, 264, 240, 262, 224, 256, 282, 260, 248, 257, 244, 245, 249, 232, 275, 259, 239, 234, 244, 267, 278, 250, 278, 266, 261, 267, 277, 270, 248, 258, 275, 248, 270, 251, 288, 269, 231, 275, 290, 277, 250, 243, 277, 248, 247, 269, 252, 266, 255, 260, 268, 280, 259] payload_rows_per_dst=[5420, 5512, 5346, 5518, 5384, 5518, 5512, 5554] dup_rows_per_dst=[2724, 2720, 2576, 2628, 2724, 2704, 2818, 2878] combine_rows_per_src=[5420, 5512, 5346, 5518, 5384, 5518, 5512, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38400 dup_rows=21658 dup_rows_remote=19006 dedup_ratio=0.330475 router_tokens_per_expert=[246, 257, 246, 251, 231, 266, 274, 279, 257, 237, 287, 233, 230, 251, 248, 254, 278, 269, 253, 242, 231, 250, 253, 262, 251, 236, 265, 251, 278, 253, 272, 240, 256, 258, 241, 231, 238, 263, 265, 277, 247, 288, 246, 238, 259, 261, 280, 276, 254, 261, 278, 244, 260, 240, 276, 254, 246, 276, 254, 274, 262, 249, 248, 243, 243, 276, 241, 279, 268, 265, 252, 226, 254, 265, 232, 239, 234, 276, 272, 271, 235, 240, 247, 255, 279, 256, 264, 279, 241, 256, 271, 273, 236, 237, 269, 227, 249, 256, 262, 254, 246, 279, 268, 254, 246, 282, 269, 253, 253, 264, 252, 263, 235, 258, 260, 264, 254, 258, 256, 265, 266, 251, 277, 237, 255, 247, 237, 266] payload_rows_per_dst=[5438, 5484, 5480, 5510, 5458, 5478, 5564, 5466] dup_rows_per_dst=[2656, 2684, 2768, 2728, 2728, 2652, 2736, 2706] combine_rows_per_src=[5438, 5484, 5480, 5510, 5458, 5478, 5564, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228576 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548320 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868064 weight_movement_bytes=18874368 total_nvlink_bytes=331742432 weight_to_token_ratio=0.060327 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228928 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156884544 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540160 weight_movement_bytes=56623104 total_nvlink_bytes=370163264 weight_to_token_ratio=0.180593 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38152 dup_rows=21844 dup_rows_remote=19018 dedup_ratio=0.333313 router_tokens_per_expert=[258, 263, 240, 241, 288, 241, 265, 266, 247, 225, 275, 247, 252, 260, 277, 274, 269, 246, 265, 249, 249, 266, 260, 241, 271, 266, 267, 269, 249, 262, 264, 254, 244, 243, 262, 264, 238, 272, 245, 220, 250, 228, 256, 252, 251, 286, 212, 269, 256, 271, 231, 264, 265, 232, 252, 230, 247, 250, 262, 260, 272, 251, 267, 240, 269, 248, 254, 235, 274, 282, 251, 267, 270, 272, 252, 284, 232, 274, 234, 285, 240, 244, 246, 253, 262, 254, 254, 244, 269, 234, 233, 241, 254, 257, 253, 272, 254, 274, 238, 254, 274, 249, 270, 227, 277, 246, 255, 271, 257, 246, 277, 284, 250, 254, 260, 260, 253, 277, 281, 239, 265, 220, 263, 236, 269, 278, 274, 235] payload_rows_per_dst=[5458, 5486, 5454, 5424, 5540, 5326, 5510, 5494] dup_rows_per_dst=[2780, 2808, 2530, 2676, 2826, 2694, 2796, 2734] combine_rows_per_src=[5458, 5486, 5454, 5424, 5540, 5326, 5510, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228880 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156778000 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313327120 weight_movement_bytes=56623104 total_nvlink_bytes=369950224 weight_to_token_ratio=0.180716 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229216 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909408 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589600 weight_movement_bytes=37748736 total_nvlink_bytes=351338336 weight_to_token_ratio=0.120376 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229624 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157516024 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802424 weight_movement_bytes=18874368 total_nvlink_bytes=333676792 weight_to_token_ratio=0.059956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38276 dup_rows=21786 dup_rows_remote=19176 dedup_ratio=0.332428 router_tokens_per_expert=[271, 222, 255, 276, 257, 253, 238, 254, 247, 236, 255, 268, 261, 231, 259, 254, 244, 250, 254, 248, 227, 254, 241, 265, 273, 246, 277, 241, 263, 282, 238, 273, 238, 267, 252, 294, 267, 274, 259, 263, 225, 260, 257, 263, 246, 267, 270, 268, 261, 253, 250, 256, 228, 237, 249, 262, 254, 245, 277, 231, 255, 280, 243, 261, 267, 251, 266, 228, 283, 276, 276, 242, 271, 270, 240, 265, 248, 256, 282, 237, 247, 248, 258, 247, 302, 272, 256, 264, 249, 265, 255, 266, 267, 247, 260, 241, 254, 246, 216, 258, 258, 264, 270, 243, 273, 276, 273, 284, 238, 252, 256, 259, 251, 263, 261, 259, 257, 267, 246, 248, 265, 224, 225, 236, 239, 267, 251, 262] payload_rows_per_dst=[5404, 5434, 5550, 5474, 5538, 5472, 5500, 5378] dup_rows_per_dst=[2670, 2718, 2790, 2610, 2778, 2816, 2740, 2664] combine_rows_per_src=[5404, 5434, 5550, 5474, 5538, 5472, 5500, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228680 combine_payload_bytes=156270592 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156499272 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312769864 weight_movement_bytes=18874368 total_nvlink_bytes=331644232 weight_to_token_ratio=0.060346 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38216 dup_rows=21794 dup_rows_remote=18890 dedup_ratio=0.332550 router_tokens_per_expert=[266, 255, 276, 252, 263, 243, 247, 292, 247, 230, 243, 252, 237, 307, 248, 243, 261, 245, 262, 250, 296, 265, 246, 263, 264, 294, 262, 243, 268, 263, 255, 269, 256, 232, 240, 225, 255, 261, 266, 259, 266, 242, 255, 263, 238, 263, 286, 217, 273, 239, 277, 273, 265, 264, 246, 232, 261, 237, 278, 247, 286, 273, 258, 254, 266, 276, 254, 254, 243, 268, 249, 274, 235, 278, 259, 244, 259, 227, 256, 266, 249, 250, 254, 274, 243, 267, 229, 249, 258, 245, 246, 283, 263, 283, 254, 246, 259, 270, 270, 251, 254, 273, 255, 257, 229, 254, 226, 255, 256, 254, 259, 252, 241, 230, 248, 274, 246, 245, 235, 247, 226, 258, 240, 263, 251, 254, 248, 293] payload_rows_per_dst=[5526, 5552, 5400, 5490, 5504, 5432, 5408, 5430] dup_rows_per_dst=[2904, 2872, 2694, 2636, 2684, 2696, 2740, 2568] combine_rows_per_src=[5526, 5552, 5400, 5490, 5504, 5432, 5408, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229808 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008304 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786800 weight_movement_bytes=18874368 total_nvlink_bytes=332661168 weight_to_token_ratio=0.060150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38256 dup_rows=21802 dup_rows_remote=19062 dedup_ratio=0.332672 router_tokens_per_expert=[244, 248, 259, 257, 284, 221, 246, 246, 256, 279, 273, 255, 274, 234, 291, 286, 246, 248, 253, 281, 275, 256, 260, 210, 235, 253, 261, 266, 250, 260, 253, 239, 250, 256, 272, 242, 277, 236, 256, 263, 299, 242, 272, 259, 263, 259, 236, 227, 250, 242, 257, 255, 261, 254, 274, 258, 265, 259, 269, 252, 243, 231, 223, 246, 250, 260, 245, 260, 232, 232, 258, 258, 275, 277, 252, 247, 278, 258, 286, 285, 244, 269, 234, 265, 233, 240, 238, 249, 263, 247, 259, 261, 263, 246, 274, 267, 268, 247, 256, 268, 289, 280, 264, 284, 265, 257, 247, 231, 274, 264, 231, 238, 275, 257, 249, 251, 256, 247, 244, 254, 244, 272, 255, 225, 272, 252, 233, 267] payload_rows_per_dst=[5506, 5442, 5478, 5368, 5506, 5464, 5560, 5410] dup_rows_per_dst=[2800, 2650, 2740, 2710, 2800, 2640, 2766, 2696] combine_rows_per_src=[5506, 5442, 5478, 5368, 5506, 5464, 5560, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38228 dup_rows=21790 dup_rows_remote=18928 dedup_ratio=0.332489 router_tokens_per_expert=[258, 237, 253, 235, 266, 258, 241, 247, 233, 277, 283, 248, 249, 264, 283, 249, 249, 246, 279, 263, 294, 243, 239, 250, 284, 232, 238, 282, 249, 236, 254, 238, 244, 257, 220, 271, 271, 256, 265, 269, 244, 252, 246, 244, 239, 248, 239, 242, 243, 251, 268, 262, 260, 243, 243, 248, 253, 247, 263, 258, 262, 258, 257, 234, 249, 253, 244, 278, 263, 252, 239, 248, 281, 281, 248, 272, 270, 249, 238, 272, 244, 255, 256, 253, 254, 286, 257, 252, 268, 254, 246, 264, 275, 261, 238, 235, 240, 259, 259, 249, 247, 257, 260, 273, 255, 269, 260, 260, 273, 241, 264, 267, 271, 266, 249, 256, 259, 243, 261, 278, 250, 258, 270, 253, 244, 268, 276, 284] payload_rows_per_dst=[5390, 5428, 5186, 5672, 5496, 5472, 5584, 5518] dup_rows_per_dst=[2772, 2724, 2286, 2970, 2686, 2724, 2766, 2862] combine_rows_per_src=[5390, 5428, 5186, 5672, 5496, 5472, 5584, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228424 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156761160 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313293896 weight_movement_bytes=18874368 total_nvlink_bytes=332168264 weight_to_token_ratio=0.060245 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38276 dup_rows=21752 dup_rows_remote=19080 dedup_ratio=0.331909 router_tokens_per_expert=[242, 260, 246, 245, 244, 254, 250, 254, 258, 258, 262, 257, 269, 264, 260, 256, 242, 264, 268, 269, 219, 277, 236, 266, 215, 284, 250, 233, 294, 241, 245, 268, 257, 249, 241, 241, 269, 259, 268, 242, 277, 251, 253, 262, 282, 263, 230, 250, 246, 244, 255, 261, 266, 255, 264, 251, 260, 256, 226, 244, 283, 245, 244, 242, 227, 258, 282, 253, 239, 250, 262, 237, 262, 285, 291, 265, 246, 272, 273, 257, 262, 278, 254, 276, 240, 223, 250, 272, 279, 258, 257, 248, 258, 243, 249, 254, 238, 232, 259, 247, 267, 248, 268, 240, 260, 282, 267, 231, 253, 276, 269, 253, 268, 256, 264, 248, 267, 269, 262, 261, 269, 238, 245, 263, 233, 272, 266, 251] payload_rows_per_dst=[5484, 5404, 5472, 5414, 5536, 5458, 5508, 5508] dup_rows_per_dst=[2674, 2738, 2712, 2674, 2782, 2744, 2672, 2756] combine_rows_per_src=[5484, 5404, 5472, 5414, 5536, 5458, 5508, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38350 dup_rows=21652 dup_rows_remote=18872 dedup_ratio=0.330383 router_tokens_per_expert=[267, 234, 247, 227, 267, 254, 257, 230, 230, 230, 221, 263, 257, 255, 263, 254, 249, 254, 280, 278, 249, 260, 273, 256, 257, 273, 250, 260, 273, 226, 263, 256, 275, 293, 256, 259, 267, 271, 252, 249, 270, 292, 246, 253, 277, 262, 253, 274, 244, 252, 269, 259, 237, 243, 276, 256, 279, 253, 249, 248, 264, 248, 266, 257, 248, 262, 256, 275, 288, 244, 231, 231, 229, 264, 240, 228, 256, 249, 260, 245, 261, 239, 253, 259, 295, 267, 286, 254, 251, 236, 237, 266, 246, 256, 235, 258, 252, 253, 254, 238, 261, 246, 266, 218, 280, 241, 257, 264, 296, 265, 240, 255, 245, 248, 259, 245, 275, 262, 263, 223, 290, 254, 262, 254, 270, 252, 264, 249] payload_rows_per_dst=[5392, 5534, 5612, 5528, 5406, 5446, 5462, 5504] dup_rows_per_dst=[2520, 2780, 2886, 2672, 2606, 2752, 2710, 2726] combine_rows_per_src=[5392, 5534, 5612, 5528, 5406, 5446, 5462, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38180 dup_rows=21854 dup_rows_remote=19208 dedup_ratio=0.333466 router_tokens_per_expert=[237, 233, 282, 265, 266, 283, 251, 260, 254, 244, 246, 239, 283, 258, 248, 238, 270, 253, 265, 229, 263, 235, 244, 256, 287, 268, 276, 281, 254, 228, 260, 260, 249, 253, 249, 243, 234, 281, 233, 262, 269, 249, 253, 262, 256, 247, 268, 244, 248, 253, 252, 238, 266, 271, 247, 257, 250, 242, 261, 256, 276, 275, 262, 252, 252, 254, 239, 240, 266, 231, 260, 256, 260, 237, 304, 256, 246, 247, 272, 259, 247, 243, 247, 244, 281, 260, 232, 288, 256, 279, 245, 260, 210, 274, 277, 231, 268, 270, 259, 273, 238, 285, 231, 261, 238, 246, 263, 279, 241, 268, 264, 267, 267, 247, 248, 267, 269, 221, 248, 266, 251, 264, 261, 256, 237, 257, 271, 260] payload_rows_per_dst=[5440, 5494, 5420, 5390, 5444, 5502, 5518, 5474] dup_rows_per_dst=[2734, 2764, 2684, 2822, 2714, 2646, 2784, 2706] combine_rows_per_src=[5440, 5494, 5420, 5390, 5444, 5502, 5518, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229272 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156925848 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622424 weight_movement_bytes=37748736 total_nvlink_bytes=351371160 weight_to_token_ratio=0.120364 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=228624 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156810512 ingress_bytes=232079360 link_bytes=232079360 token_bytes=313392400 weight_movement_bytes=75497472 total_nvlink_bytes=388889872 weight_to_token_ratio=0.240904 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223670237e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229424 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007920 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786416 weight_movement_bytes=37748736 total_nvlink_bytes=351535152 weight_to_token_ratio=0.120301 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43922 payload_rows_remote=38536 dup_rows=21614 dup_rows_remote=18984 dedup_ratio=0.329803 router_tokens_per_expert=[246, 260, 274, 271, 264, 236, 265, 274, 281, 294, 225, 244, 226, 278, 255, 233, 262, 261, 244, 243, 216, 235, 255, 245, 261, 259, 262, 233, 273, 266, 277, 240, 260, 252, 251, 237, 273, 254, 262, 256, 240, 281, 240, 254, 265, 250, 262, 272, 266, 287, 247, 272, 227, 289, 257, 235, 247, 259, 246, 248, 246, 243, 249, 259, 268, 261, 273, 227, 261, 248, 242, 244, 230, 260, 263, 269, 239, 243, 241, 239, 266, 238, 236, 258, 243, 244, 266, 266, 249, 250, 273, 262, 286, 247, 274, 249, 241, 264, 257, 255, 274, 260, 277, 278, 266, 268, 271, 265, 253, 243, 268, 247, 253, 241, 263, 260, 253, 245, 236, 248, 263, 255, 258, 277, 256, 285, 283, 246] payload_rows_per_dst=[5546, 5470, 5500, 5530, 5386, 5424, 5558, 5508] dup_rows_per_dst=[2706, 2594, 2718, 2624, 2630, 2790, 2816, 2736] combine_rows_per_src=[5546, 5470, 5500, 5530, 5386, 5424, 5558, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228888 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157310488 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392088 weight_movement_bytes=18874368 total_nvlink_bytes=333266456 weight_to_token_ratio=0.060034 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229552 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614832 ingress_bytes=175259648 link_bytes=175259648 token_bytes=313000112 weight_movement_bytes=18874368 total_nvlink_bytes=331874480 weight_to_token_ratio=0.060301 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38306 dup_rows=21706 dup_rows_remote=18960 dedup_ratio=0.331207 router_tokens_per_expert=[279, 245, 269, 264, 275, 277, 238, 275, 262, 263, 259, 240, 252, 265, 239, 256, 268, 212, 238, 261, 278, 247, 259, 222, 221, 246, 250, 235, 260, 257, 263, 251, 243, 272, 253, 229, 246, 270, 254, 266, 237, 251, 259, 266, 257, 272, 273, 261, 233, 300, 258, 261, 249, 253, 228, 279, 280, 261, 261, 286, 253, 238, 236, 259, 278, 251, 255, 245, 256, 265, 260, 235, 272, 261, 235, 260, 250, 240, 227, 251, 242, 266, 233, 265, 242, 267, 281, 244, 246, 232, 240, 269, 272, 251, 287, 280, 269, 241, 262, 235, 259, 235, 270, 269, 271, 266, 268, 273, 241, 264, 276, 250, 256, 231, 267, 245, 236, 278, 255, 257, 265, 260, 265, 260, 235, 265, 246, 270] payload_rows_per_dst=[5536, 5352, 5470, 5524, 5428, 5498, 5536, 5486] dup_rows_per_dst=[2780, 2584, 2748, 2746, 2654, 2736, 2762, 2696] combine_rows_per_src=[5536, 5352, 5470, 5524, 5428, 5498, 5536, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38192 dup_rows=21680 dup_rows_remote=18818 dedup_ratio=0.330811 router_tokens_per_expert=[278, 281, 266, 272, 248, 241, 272, 250, 226, 285, 245, 247, 279, 253, 273, 270, 240, 270, 243, 246, 260, 255, 264, 248, 225, 251, 264, 235, 254, 240, 261, 248, 290, 247, 277, 261, 238, 274, 267, 273, 235, 266, 216, 258, 243, 264, 263, 253, 248, 251, 270, 265, 271, 273, 268, 278, 254, 255, 261, 220, 257, 251, 255, 252, 238, 253, 280, 277, 273, 264, 238, 244, 263, 266, 245, 248, 253, 266, 264, 272, 253, 252, 263, 249, 271, 238, 252, 272, 245, 234, 242, 264, 246, 259, 258, 266, 293, 242, 240, 272, 257, 255, 247, 273, 269, 249, 215, 253, 245, 272, 258, 234, 250, 250, 243, 247, 247, 249, 238, 277, 240, 264, 274, 241, 236, 279, 254, 253] payload_rows_per_dst=[5664, 5570, 5516, 5430, 5492, 5412, 5410, 5362] dup_rows_per_dst=[2862, 2922, 2748, 2496, 2662, 2716, 2552, 2722] combine_rows_per_src=[5664, 5570, 5516, 5430, 5492, 5412, 5410, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157843456 dispatch_meta_bytes=230080 combine_payload_bytes=157843456 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=158073536 ingress_bytes=195592192 link_bytes=195592192 token_bytes=315916992 weight_movement_bytes=37748736 total_nvlink_bytes=353665728 weight_to_token_ratio=0.119489 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874068146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38148 dup_rows=21954 dup_rows_remote=19240 dedup_ratio=0.334991 router_tokens_per_expert=[271, 262, 250, 267, 267, 256, 261, 242, 237, 253, 272, 266, 262, 263, 262, 254, 319, 248, 244, 241, 265, 247, 256, 263, 255, 271, 258, 285, 271, 244, 248, 242, 267, 248, 256, 253, 269, 258, 258, 258, 249, 247, 230, 268, 233, 297, 233, 250, 257, 267, 253, 245, 245, 273, 248, 240, 255, 237, 246, 237, 252, 255, 269, 226, 283, 281, 265, 263, 248, 251, 242, 239, 240, 256, 274, 252, 277, 268, 241, 272, 246, 237, 246, 248, 262, 229, 244, 250, 250, 264, 230, 242, 254, 288, 284, 278, 254, 249, 243, 234, 278, 246, 266, 257, 249, 292, 270, 250, 247, 271, 241, 254, 253, 271, 281, 249, 230, 253, 265, 269, 267, 261, 257, 238, 251, 251, 261, 225] payload_rows_per_dst=[5484, 5448, 5434, 5352, 5478, 5398, 5510, 5478] dup_rows_per_dst=[2806, 2866, 2714, 2658, 2826, 2706, 2692, 2686] combine_rows_per_src=[5484, 5448, 5434, 5352, 5478, 5398, 5510, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229064 combine_payload_bytes=156901376 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157130440 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031816 weight_movement_bytes=37748736 total_nvlink_bytes=351780552 weight_to_token_ratio=0.120207 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38140 dup_rows=21914 dup_rows_remote=19174 dedup_ratio=0.334381 router_tokens_per_expert=[246, 270, 273, 255, 262, 288, 294, 273, 262, 252, 281, 246, 240, 248, 260, 242, 260, 246, 245, 270, 249, 251, 234, 273, 247, 276, 253, 242, 262, 247, 249, 247, 208, 234, 243, 238, 255, 259, 269, 281, 261, 249, 260, 248, 244, 252, 269, 246, 235, 245, 246, 266, 262, 296, 250, 264, 232, 238, 253, 261, 276, 219, 266, 247, 266, 243, 233, 261, 253, 262, 282, 280, 248, 252, 261, 281, 274, 259, 266, 244, 237, 273, 242, 262, 264, 251, 246, 229, 266, 257, 286, 245, 236, 264, 270, 262, 246, 256, 255, 244, 248, 244, 265, 267, 264, 238, 256, 281, 245, 250, 263, 289, 264, 265, 249, 266, 273, 245, 266, 240, 275, 264, 232, 270, 236, 263, 232, 247] payload_rows_per_dst=[5582, 5400, 5346, 5450, 5486, 5488, 5482, 5388] dup_rows_per_dst=[2802, 2702, 2686, 2662, 2844, 2692, 2740, 2786] combine_rows_per_src=[5582, 5400, 5346, 5450, 5486, 5488, 5482, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38276 dup_rows=21826 dup_rows_remote=19076 dedup_ratio=0.333038 router_tokens_per_expert=[257, 210, 236, 271, 285, 249, 251, 267, 246, 283, 261, 212, 260, 246, 260, 231, 280, 269, 256, 252, 263, 246, 274, 277, 259, 238, 236, 232, 275, 274, 268, 282, 281, 250, 277, 287, 261, 259, 230, 236, 280, 264, 238, 244, 257, 278, 222, 240, 240, 240, 261, 259, 216, 256, 258, 245, 261, 258, 289, 272, 228, 249, 230, 272, 257, 253, 261, 270, 238, 247, 249, 269, 279, 289, 285, 270, 250, 232, 236, 250, 263, 265, 230, 244, 258, 238, 273, 260, 252, 233, 241, 255, 285, 264, 241, 258, 229, 263, 257, 285, 277, 245, 244, 261, 268, 248, 287, 264, 261, 294, 289, 239, 228, 247, 263, 234, 274, 230, 232, 250, 264, 268, 231, 272, 272, 261, 228, 264] payload_rows_per_dst=[5248, 5500, 5594, 5448, 5402, 5502, 5582, 5434] dup_rows_per_dst=[2274, 2862, 3024, 2608, 2678, 2672, 2958, 2750] combine_rows_per_src=[5248, 5500, 5594, 5448, 5402, 5502, 5582, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228040 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156662472 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313096904 weight_movement_bytes=18874368 total_nvlink_bytes=331971272 weight_to_token_ratio=0.060283 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229552 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483760 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737968 weight_movement_bytes=18874368 total_nvlink_bytes=331612336 weight_to_token_ratio=0.060352 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38366 dup_rows=21836 dup_rows_remote=19228 dedup_ratio=0.333191 router_tokens_per_expert=[246, 275, 251, 238, 280, 277, 293, 271, 248, 246, 255, 256, 270, 292, 261, 264, 222, 264, 281, 245, 255, 257, 246, 228, 227, 228, 272, 261, 240, 222, 223, 225, 260, 256, 249, 239, 249, 238, 244, 275, 250, 273, 283, 250, 268, 239, 263, 252, 243, 257, 233, 268, 274, 264, 251, 269, 290, 242, 248, 251, 284, 273, 239, 254, 263, 251, 266, 249, 264, 254, 234, 253, 243, 269, 244, 248, 254, 248, 273, 274, 227, 258, 258, 265, 253, 265, 268, 262, 258, 246, 247, 269, 257, 254, 298, 257, 248, 264, 254, 256, 254, 271, 262, 258, 233, 281, 247, 254, 250, 261, 250, 294, 219, 231, 246, 251, 275, 253, 268, 236, 267, 241, 252, 270, 256, 271, 263, 256] payload_rows_per_dst=[5620, 5334, 5456, 5532, 5412, 5508, 5454, 5384] dup_rows_per_dst=[2826, 2608, 2720, 2748, 2762, 2776, 2670, 2726] combine_rows_per_src=[5620, 5334, 5456, 5532, 5412, 5508, 5454, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38206 dup_rows=21840 dup_rows_remote=19048 dedup_ratio=0.333252 router_tokens_per_expert=[244, 270, 283, 253, 246, 248, 252, 249, 271, 236, 255, 267, 263, 251, 260, 261, 258, 247, 250, 261, 252, 236, 266, 254, 252, 259, 254, 263, 237, 233, 262, 269, 264, 245, 283, 243, 226, 247, 231, 266, 281, 253, 249, 250, 263, 277, 264, 235, 262, 255, 236, 263, 270, 273, 257, 260, 255, 257, 260, 262, 215, 244, 269, 260, 254, 276, 277, 271, 267, 251, 225, 250, 234, 248, 254, 227, 260, 257, 292, 256, 263, 250, 246, 263, 222, 263, 264, 246, 246, 261, 277, 267, 250, 281, 262, 280, 260, 270, 236, 267, 249, 271, 274, 253, 237, 263, 244, 274, 226, 298, 249, 228, 242, 252, 282, 260, 238, 286, 269, 250, 247, 273, 258, 249, 234, 257, 249, 246] payload_rows_per_dst=[5512, 5472, 5472, 5450, 5386, 5490, 5444, 5470] dup_rows_per_dst=[2706, 2634, 2682, 2746, 2812, 2792, 2754, 2714] combine_rows_per_src=[5512, 5472, 5472, 5450, 5386, 5490, 5444, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229256 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156450696 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312672136 weight_movement_bytes=37748736 total_nvlink_bytes=350420872 weight_to_token_ratio=0.120729 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229408 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007904 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786400 weight_movement_bytes=18874368 total_nvlink_bytes=332660768 weight_to_token_ratio=0.060150 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38348 dup_rows=21688 dup_rows_remote=18948 dedup_ratio=0.330933 router_tokens_per_expert=[234, 255, 256, 248, 258, 249, 259, 240, 262, 263, 285, 276, 275, 262, 248, 240, 246, 267, 235, 256, 243, 275, 236, 243, 236, 283, 220, 255, 274, 249, 281, 219, 246, 267, 260, 273, 271, 243, 262, 234, 269, 236, 252, 244, 259, 263, 263, 231, 262, 276, 262, 273, 228, 240, 236, 258, 279, 261, 255, 279, 259, 269, 266, 230, 246, 250, 241, 264, 268, 285, 259, 259, 267, 260, 255, 265, 259, 223, 268, 251, 228, 254, 275, 260, 275, 247, 266, 240, 216, 269, 267, 241, 252, 242, 261, 267, 296, 245, 241, 235, 280, 252, 238, 258, 261, 264, 267, 237, 261, 255, 254, 249, 271, 261, 263, 278, 260, 248, 247, 260, 227, 274, 255, 234, 261, 288, 246, 288] payload_rows_per_dst=[5418, 5462, 5486, 5462, 5500, 5452, 5504, 5564] dup_rows_per_dst=[2802, 2574, 2660, 2804, 2740, 2668, 2682, 2758] combine_rows_per_src=[5418, 5462, 5486, 5462, 5500, 5452, 5504, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38368 dup_rows=21618 dup_rows_remote=18834 dedup_ratio=0.329865 router_tokens_per_expert=[292, 261, 226, 261, 240, 251, 228, 271, 245, 229, 269, 268, 251, 259, 266, 252, 239, 238, 244, 252, 259, 234, 265, 259, 225, 258, 256, 251, 236, 265, 238, 258, 263, 249, 262, 259, 253, 285, 230, 261, 258, 250, 253, 257, 240, 264, 300, 261, 264, 273, 288, 272, 249, 230, 238, 266, 243, 273, 253, 292, 253, 253, 258, 262, 251, 243, 244, 281, 280, 268, 232, 263, 258, 258, 243, 283, 263, 253, 245, 287, 241, 261, 270, 234, 239, 255, 235, 268, 227, 263, 234, 275, 257, 287, 269, 256, 268, 229, 237, 284, 264, 224, 254, 259, 257, 244, 266, 224, 259, 266, 265, 276, 252, 247, 234, 259, 270, 293, 253, 257, 242, 272, 252, 264, 230, 245, 268, 273] payload_rows_per_dst=[5504, 5414, 5454, 5550, 5566, 5470, 5468, 5492] dup_rows_per_dst=[2634, 2540, 2836, 2784, 2738, 2672, 2684, 2730] combine_rows_per_src=[5504, 5414, 5454, 5550, 5566, 5470, 5468, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=230376 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157377512 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314524648 weight_movement_bytes=18874368 total_nvlink_bytes=333399016 weight_to_token_ratio=0.060009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229016 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156720792 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313212568 weight_movement_bytes=18874368 total_nvlink_bytes=332086936 weight_to_token_ratio=0.060261 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38204 dup_rows=21694 dup_rows_remote=18792 dedup_ratio=0.331024 router_tokens_per_expert=[261, 278, 278, 251, 267, 266, 295, 268, 259, 241, 256, 277, 261, 250, 265, 260, 245, 278, 242, 242, 252, 288, 253, 234, 257, 259, 247, 287, 247, 277, 258, 218, 253, 253, 266, 239, 280, 262, 283, 261, 269, 253, 252, 256, 243, 282, 244, 263, 243, 233, 249, 257, 256, 249, 278, 235, 265, 249, 257, 248, 249, 244, 251, 250, 239, 239, 230, 262, 222, 265, 262, 233, 258, 253, 239, 273, 250, 254, 248, 243, 271, 266, 231, 270, 258, 226, 237, 236, 241, 233, 283, 257, 235, 268, 241, 241, 292, 248, 256, 272, 245, 239, 261, 286, 254, 292, 260, 253, 250, 271, 225, 264, 283, 236, 262, 280, 245, 275, 261, 273, 255, 254, 244, 268, 260, 242, 255, 254] payload_rows_per_dst=[5638, 5536, 5508, 5654, 5418, 5612, 5126, 5350] dup_rows_per_dst=[2902, 2902, 2810, 2892, 2522, 2918, 2398, 2350] combine_rows_per_src=[5638, 5536, 5508, 5654, 5418, 5612, 5126, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229184 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157302592 ingress_bytes=213696512 link_bytes=213696512 token_bytes=314376000 weight_movement_bytes=56623104 total_nvlink_bytes=370999104 weight_to_token_ratio=0.180113 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047534833e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=228808 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157384136 ingress_bytes=213778432 link_bytes=213778432 token_bytes=314539464 weight_movement_bytes=56623104 total_nvlink_bytes=371162568 weight_to_token_ratio=0.180019 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048319750e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38220 dup_rows=21814 dup_rows_remote=19072 dedup_ratio=0.332855 router_tokens_per_expert=[260, 251, 243, 252, 271, 259, 250, 266, 248, 257, 255, 251, 277, 249, 249, 247, 247, 250, 284, 263, 236, 246, 260, 256, 233, 244, 261, 272, 266, 218, 278, 242, 247, 235, 268, 257, 273, 267, 281, 267, 275, 251, 276, 235, 230, 267, 246, 247, 253, 238, 254, 259, 247, 267, 223, 272, 269, 244, 239, 241, 258, 262, 244, 258, 267, 246, 284, 233, 245, 250, 252, 269, 272, 259, 245, 280, 272, 241, 265, 274, 228, 253, 270, 238, 221, 247, 264, 274, 227, 255, 289, 254, 270, 263, 254, 295, 262, 241, 246, 277, 238, 260, 267, 261, 253, 239, 266, 263, 268, 249, 261, 271, 252, 244, 256, 246, 244, 271, 242, 275, 244, 258, 254, 270, 319, 240, 227, 257] payload_rows_per_dst=[5454, 5438, 5502, 5360, 5594, 5490, 5476, 5408] dup_rows_per_dst=[2716, 2674, 2742, 2696, 2714, 2714, 2768, 2790] combine_rows_per_src=[5454, 5438, 5502, 5360, 5594, 5490, 5476, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38380 dup_rows=21700 dup_rows_remote=19034 dedup_ratio=0.331116 router_tokens_per_expert=[293, 247, 244, 257, 261, 258, 269, 239, 292, 252, 248, 254, 264, 256, 256, 247, 260, 232, 270, 273, 272, 238, 270, 259, 259, 279, 235, 243, 257, 260, 255, 252, 257, 258, 251, 241, 251, 279, 258, 225, 245, 255, 259, 247, 231, 261, 281, 256, 269, 257, 255, 253, 255, 251, 234, 247, 268, 254, 273, 263, 267, 244, 241, 282, 257, 287, 257, 242, 274, 256, 248, 238, 242, 235, 264, 242, 251, 254, 264, 235, 276, 271, 280, 268, 239, 279, 262, 252, 275, 238, 259, 242, 247, 263, 265, 268, 263, 253, 255, 264, 230, 259, 268, 226, 252, 245, 247, 240, 274, 255, 258, 272, 241, 240, 244, 248, 286, 262, 249, 246, 261, 242, 263, 249, 256, 247, 265, 259] payload_rows_per_dst=[5494, 5448, 5476, 5504, 5416, 5564, 5456, 5478] dup_rows_per_dst=[2780, 2780, 2634, 2722, 2676, 2804, 2666, 2638] combine_rows_per_src=[5494, 5448, 5476, 5504, 5416, 5564, 5456, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38170 dup_rows=21856 dup_rows_remote=18976 dedup_ratio=0.333496 router_tokens_per_expert=[262, 268, 254, 269, 251, 238, 237, 257, 270, 241, 243, 249, 262, 239, 297, 236, 283, 268, 261, 275, 233, 253, 228, 260, 254, 299, 239, 260, 280, 238, 266, 265, 268, 283, 265, 256, 227, 259, 241, 244, 249, 248, 277, 260, 235, 262, 256, 269, 277, 261, 278, 271, 238, 251, 268, 245, 250, 246, 262, 226, 243, 276, 243, 269, 241, 263, 257, 262, 251, 276, 249, 265, 225, 258, 260, 256, 263, 227, 287, 244, 278, 262, 240, 257, 262, 267, 236, 242, 241, 242, 245, 260, 265, 249, 280, 235, 269, 263, 245, 277, 264, 237, 248, 275, 210, 233, 236, 244, 248, 265, 290, 256, 268, 266, 251, 252, 260, 244, 259, 250, 234, 270, 237, 258, 284, 260, 265, 267] payload_rows_per_dst=[5454, 5534, 5428, 5504, 5420, 5484, 5346, 5510] dup_rows_per_dst=[2692, 2774, 2770, 2720, 2644, 2742, 2634, 2880] combine_rows_per_src=[5454, 5534, 5428, 5504, 5420, 5484, 5346, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=227984 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156711568 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313195152 weight_movement_bytes=37748736 total_nvlink_bytes=350943888 weight_to_token_ratio=0.120528 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229168 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778288 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313327408 weight_movement_bytes=18874368 total_nvlink_bytes=332201776 weight_to_token_ratio=0.060238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229656 combine_payload_bytes=157204480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157434136 ingress_bytes=213827584 link_bytes=213827584 token_bytes=314638616 weight_movement_bytes=56623104 total_nvlink_bytes=371261720 weight_to_token_ratio=0.179962 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048790700e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43532 payload_rows_remote=38084 dup_rows=22004 dup_rows_remote=19308 dedup_ratio=0.335754 router_tokens_per_expert=[260, 247, 275, 257, 262, 265, 270, 261, 245, 251, 260, 254, 245, 269, 256, 254, 274, 251, 246, 265, 237, 262, 240, 243, 243, 248, 253, 265, 230, 282, 271, 262, 258, 254, 237, 252, 256, 240, 254, 251, 267, 252, 264, 267, 227, 283, 274, 260, 274, 261, 254, 237, 271, 244, 265, 268, 236, 254, 242, 285, 227, 275, 262, 281, 247, 253, 261, 237, 250, 235, 233, 262, 253, 268, 242, 261, 259, 270, 252, 261, 240, 263, 275, 263, 235, 273, 253, 249, 250, 265, 254, 244, 238, 248, 245, 252, 220, 270, 292, 265, 263, 263, 255, 240, 246, 250, 261, 248, 248, 274, 260, 283, 269, 258, 251, 273, 268, 233, 266, 263, 290, 233, 262, 242, 233, 236, 258, 269] payload_rows_per_dst=[5532, 5448, 5404, 5446, 5418, 5412, 5446, 5426] dup_rows_per_dst=[2730, 2696, 2788, 2826, 2670, 2682, 2830, 2782] combine_rows_per_src=[5532, 5448, 5404, 5446, 5418, 5412, 5446, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38324 dup_rows=21736 dup_rows_remote=18932 dedup_ratio=0.331665 router_tokens_per_expert=[275, 253, 246, 283, 252, 287, 286, 243, 254, 226, 223, 261, 236, 264, 251, 260, 258, 282, 284, 248, 258, 257, 275, 258, 281, 240, 271, 241, 230, 243, 244, 254, 251, 260, 241, 225, 253, 267, 259, 247, 238, 253, 245, 262, 243, 271, 250, 271, 247, 293, 247, 265, 231, 253, 236, 291, 241, 262, 239, 280, 240, 272, 229, 263, 250, 276, 250, 255, 231, 249, 277, 251, 275, 237, 276, 259, 232, 271, 275, 246, 252, 259, 284, 256, 266, 265, 244, 292, 260, 262, 260, 257, 279, 232, 263, 209, 251, 240, 264, 250, 226, 252, 258, 244, 268, 252, 257, 238, 262, 268, 255, 262, 258, 224, 262, 257, 271, 265, 268, 256, 256, 260, 252, 249, 258, 264, 253, 269] payload_rows_per_dst=[5522, 5482, 5360, 5504, 5436, 5476, 5466, 5554] dup_rows_per_dst=[2678, 2766, 2712, 2674, 2784, 2804, 2628, 2690] combine_rows_per_src=[5522, 5482, 5360, 5504, 5436, 5476, 5466, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228584 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156572904 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917224 weight_movement_bytes=18874368 total_nvlink_bytes=331791592 weight_to_token_ratio=0.060317 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38298 dup_rows=21768 dup_rows_remote=18964 dedup_ratio=0.332153 router_tokens_per_expert=[267, 223, 265, 250, 241, 240, 255, 285, 253, 258, 242, 262, 256, 227, 270, 247, 253, 248, 254, 255, 258, 273, 253, 252, 255, 260, 278, 242, 226, 242, 264, 241, 270, 231, 264, 258, 239, 275, 258, 242, 263, 253, 232, 258, 272, 253, 279, 290, 266, 258, 254, 262, 278, 270, 246, 258, 258, 249, 255, 237, 263, 246, 255, 282, 227, 248, 261, 246, 272, 235, 252, 238, 278, 266, 261, 282, 275, 268, 251, 243, 256, 264, 267, 289, 260, 259, 267, 276, 253, 243, 270, 204, 258, 279, 262, 277, 252, 265, 276, 261, 246, 263, 245, 243, 242, 242, 253, 260, 236, 260, 259, 264, 293, 243, 235, 234, 236, 248, 246, 253, 271, 268, 246, 267, 258, 244, 254, 249] payload_rows_per_dst=[5490, 5466, 5464, 5470, 5476, 5560, 5432, 5410] dup_rows_per_dst=[2592, 2642, 2810, 2804, 2730, 2808, 2702, 2680] combine_rows_per_src=[5490, 5466, 5464, 5470, 5476, 5560, 5432, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38232 dup_rows=21794 dup_rows_remote=18990 dedup_ratio=0.332550 router_tokens_per_expert=[255, 251, 256, 273, 250, 268, 263, 217, 266, 248, 269, 267, 230, 244, 257, 272, 252, 259, 266, 238, 238, 248, 275, 254, 249, 264, 280, 249, 269, 253, 254, 270, 252, 283, 268, 250, 266, 232, 274, 250, 246, 241, 240, 249, 233, 252, 253, 223, 248, 275, 287, 255, 275, 252, 266, 236, 286, 240, 276, 249, 255, 252, 231, 254, 251, 269, 244, 263, 282, 260, 245, 267, 236, 254, 256, 274, 274, 280, 261, 241, 268, 257, 279, 219, 275, 252, 272, 256, 232, 252, 271, 242, 231, 243, 261, 219, 248, 264, 268, 270, 292, 260, 262, 251, 233, 271, 279, 268, 236, 283, 245, 271, 261, 263, 259, 241, 233, 268, 251, 230, 241, 268, 240, 251, 243, 278, 245, 256] payload_rows_per_dst=[5502, 5526, 5380, 5470, 5510, 5458, 5524, 5372] dup_rows_per_dst=[2670, 2710, 2644, 2804, 2804, 2600, 2878, 2684] combine_rows_per_src=[5502, 5526, 5380, 5470, 5510, 5458, 5524, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=229568 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156221632 ingress_bytes=193740800 link_bytes=193740800 token_bytes=312213696 weight_movement_bytes=37748736 total_nvlink_bytes=349962432 weight_to_token_ratio=0.120907 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856329019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229024 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204128 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179232 weight_movement_bytes=18874368 total_nvlink_bytes=333053600 weight_to_token_ratio=0.060075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43516 payload_rows_remote=37998 dup_rows=22020 dup_rows_remote=19040 dedup_ratio=0.335999 router_tokens_per_expert=[225, 250, 254, 263, 228, 228, 246, 264, 243, 243, 243, 237, 267, 285, 261, 275, 287, 272, 270, 243, 236, 245, 249, 232, 261, 261, 268, 257, 238, 265, 249, 277, 246, 256, 254, 246, 261, 260, 270, 271, 261, 248, 259, 253, 277, 281, 289, 268, 257, 254, 260, 276, 279, 251, 248, 253, 261, 267, 225, 247, 236, 247, 239, 239, 280, 282, 243, 243, 272, 251, 221, 261, 279, 275, 254, 264, 229, 257, 259, 244, 239, 244, 271, 248, 239, 280, 256, 238, 258, 269, 253, 252, 276, 247, 247, 266, 248, 264, 255, 243, 234, 263, 248, 249, 259, 253, 249, 228, 265, 275, 277, 223, 264, 278, 235, 252, 269, 293, 263, 242, 243, 257, 288, 308, 228, 268, 216, 273] payload_rows_per_dst=[5518, 5498, 5564, 5406, 5464, 5512, 5180, 5374] dup_rows_per_dst=[2980, 2864, 2938, 2672, 2764, 2674, 2412, 2716] combine_rows_per_src=[5518, 5498, 5564, 5406, 5464, 5512, 5180, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229048 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157097656 ingress_bytes=213491712 link_bytes=213491712 token_bytes=313966264 weight_movement_bytes=56623104 total_nvlink_bytes=370589368 weight_to_token_ratio=0.180348 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045572540e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38438 dup_rows=21758 dup_rows_remote=19066 dedup_ratio=0.332001 router_tokens_per_expert=[229, 232, 233, 248, 230, 267, 270, 270, 214, 257, 249, 278, 276, 273, 285, 262, 250, 254, 240, 264, 256, 247, 303, 296, 239, 255, 263, 244, 273, 273, 251, 254, 229, 251, 230, 247, 248, 244, 260, 265, 251, 252, 250, 250, 249, 262, 279, 249, 272, 269, 244, 263, 259, 238, 228, 267, 286, 261, 254, 276, 253, 254, 249, 275, 271, 256, 255, 251, 287, 269, 243, 279, 257, 227, 234, 253, 235, 259, 248, 270, 241, 264, 262, 264, 257, 247, 267, 251, 248, 250, 240, 260, 277, 261, 254, 277, 268, 212, 241, 257, 274, 245, 254, 261, 263, 265, 228, 276, 266, 279, 244, 263, 230, 224, 269, 261, 261, 219, 280, 248, 246, 280, 263, 220, 250, 265, 267, 276] payload_rows_per_dst=[5478, 5596, 5340, 5538, 5522, 5432, 5416, 5456] dup_rows_per_dst=[2668, 2728, 2692, 2758, 2666, 2808, 2776, 2662] combine_rows_per_src=[5478, 5596, 5340, 5538, 5522, 5432, 5416, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228888 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827160 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425432 weight_movement_bytes=37748736 total_nvlink_bytes=351174168 weight_to_token_ratio=0.120439 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38234 dup_rows=21676 dup_rows_remote=18970 dedup_ratio=0.330750 router_tokens_per_expert=[231, 251, 258, 246, 270, 271, 278, 227, 244, 234, 246, 255, 244, 258, 260, 266, 246, 242, 217, 272, 268, 276, 249, 259, 261, 274, 235, 223, 273, 242, 217, 258, 274, 250, 262, 270, 284, 229, 272, 253, 244, 219, 243, 272, 262, 272, 258, 270, 250, 275, 259, 278, 245, 248, 254, 253, 251, 260, 255, 285, 223, 282, 247, 249, 273, 254, 258, 232, 237, 239, 253, 265, 275, 256, 279, 258, 276, 252, 275, 242, 260, 251, 248, 233, 265, 251, 247, 278, 258, 273, 243, 254, 263, 246, 256, 252, 270, 235, 244, 274, 251, 255, 279, 260, 271, 253, 270, 261, 285, 245, 282, 231, 231, 254, 281, 259, 263, 257, 251, 251, 257, 265, 258, 249, 265, 265, 232, 263] payload_rows_per_dst=[5458, 5358, 5588, 5468, 5456, 5426, 5626, 5480] dup_rows_per_dst=[2620, 2666, 2680, 2760, 2792, 2730, 2706, 2722] combine_rows_per_src=[5458, 5358, 5588, 5468, 5456, 5426, 5626, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38256 dup_rows=21718 dup_rows_remote=19000 dedup_ratio=0.331390 router_tokens_per_expert=[268, 274, 233, 246, 239, 268, 256, 274, 264, 274, 237, 237, 241, 236, 271, 258, 254, 254, 238, 246, 240, 253, 263, 279, 256, 235, 252, 253, 235, 269, 252, 228, 258, 261, 281, 245, 256, 283, 266, 295, 251, 262, 258, 253, 271, 263, 249, 250, 240, 246, 260, 244, 278, 239, 225, 253, 254, 245, 247, 258, 275, 251, 273, 241, 266, 252, 261, 280, 271, 248, 268, 253, 243, 236, 246, 258, 271, 268, 248, 270, 234, 264, 272, 236, 261, 247, 262, 278, 270, 249, 238, 256, 261, 246, 260, 265, 270, 235, 238, 281, 266, 267, 259, 252, 251, 265, 256, 248, 250, 256, 243, 239, 252, 269, 254, 266, 258, 263, 241, 276, 240, 252, 231, 240, 266, 283, 258, 291] payload_rows_per_dst=[5422, 5446, 5454, 5388, 5548, 5522, 5476, 5562] dup_rows_per_dst=[2730, 2568, 2792, 2556, 2774, 2834, 2746, 2718] combine_rows_per_src=[5422, 5446, 5454, 5388, 5548, 5522, 5476, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155639808 dispatch_meta_bytes=228152 combine_payload_bytes=155639808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155867960 ingress_bytes=174514176 link_bytes=174514176 token_bytes=311507768 weight_movement_bytes=18874368 total_nvlink_bytes=330382136 weight_to_token_ratio=0.060590 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672108968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=230016 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157672064 ingress_bytes=195190784 link_bytes=195190784 token_bytes=315114112 weight_movement_bytes=37748736 total_nvlink_bytes=352862848 weight_to_token_ratio=0.119794 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870222052e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228816 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156835280 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313441744 weight_movement_bytes=18874368 total_nvlink_bytes=332316112 weight_to_token_ratio=0.060217 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38284 dup_rows=21762 dup_rows_remote=19100 dedup_ratio=0.332062 router_tokens_per_expert=[239, 264, 280, 235, 265, 262, 282, 243, 266, 249, 235, 234, 240, 258, 265, 256, 242, 224, 273, 253, 253, 266, 251, 278, 252, 243, 240, 266, 273, 279, 249, 220, 270, 258, 253, 268, 261, 257, 236, 252, 267, 256, 249, 245, 271, 224, 243, 257, 266, 260, 251, 270, 260, 250, 286, 244, 238, 232, 240, 237, 244, 256, 275, 258, 279, 275, 274, 274, 275, 264, 263, 243, 255, 231, 249, 224, 219, 266, 246, 254, 273, 255, 248, 268, 231, 268, 276, 284, 242, 280, 256, 241, 252, 277, 273, 254, 275, 267, 227, 256, 286, 250, 274, 255, 243, 255, 243, 288, 267, 274, 254, 261, 225, 220, 252, 290, 259, 258, 247, 265, 235, 263, 265, 258, 262, 255, 251, 250] payload_rows_per_dst=[5470, 5490, 5458, 5394, 5460, 5546, 5522, 5434] dup_rows_per_dst=[2676, 2662, 2676, 2740, 2722, 2782, 2828, 2676] combine_rows_per_src=[5470, 5490, 5458, 5394, 5460, 5546, 5522, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229024 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156925600 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622176 weight_movement_bytes=37748736 total_nvlink_bytes=351370912 weight_to_token_ratio=0.120364 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38070 dup_rows=21994 dup_rows_remote=19304 dedup_ratio=0.335602 router_tokens_per_expert=[259, 263, 267, 277, 261, 250, 266, 275, 240, 269, 245, 230, 268, 232, 250, 267, 256, 255, 261, 239, 266, 277, 252, 246, 254, 234, 252, 270, 241, 253, 256, 258, 264, 259, 272, 264, 243, 236, 264, 239, 248, 275, 263, 260, 257, 254, 257, 229, 272, 283, 273, 289, 239, 251, 278, 250, 257, 228, 245, 255, 259, 268, 276, 253, 284, 267, 239, 242, 275, 248, 254, 239, 226, 240, 252, 257, 257, 283, 253, 264, 252, 267, 234, 247, 236, 265, 259, 252, 271, 258, 266, 272, 240, 257, 245, 260, 250, 242, 258, 243, 243, 240, 277, 237, 252, 278, 252, 265, 234, 252, 252, 271, 290, 263, 260, 269, 256, 228, 238, 262, 273, 252, 269, 232, 245, 265, 241, 269] payload_rows_per_dst=[5514, 5468, 5430, 5464, 5360, 5472, 5372, 5462] dup_rows_per_dst=[2724, 2672, 2738, 2888, 2800, 2690, 2720, 2762] combine_rows_per_src=[5514, 5468, 5430, 5464, 5360, 5472, 5372, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38216 dup_rows=21852 dup_rows_remote=19034 dedup_ratio=0.333435 router_tokens_per_expert=[252, 249, 237, 275, 241, 272, 246, 246, 257, 239, 268, 258, 266, 262, 240, 246, 224, 258, 266, 283, 247, 253, 247, 253, 236, 245, 271, 254, 256, 250, 256, 261, 295, 258, 244, 263, 280, 245, 241, 258, 263, 259, 230, 233, 297, 242, 271, 250, 259, 247, 254, 248, 254, 288, 240, 255, 247, 288, 248, 247, 249, 284, 265, 270, 276, 269, 271, 290, 283, 273, 260, 246, 206, 276, 251, 269, 265, 282, 226, 249, 251, 252, 283, 258, 239, 248, 235, 248, 262, 251, 241, 239, 246, 252, 249, 239, 268, 243, 260, 251, 268, 274, 245, 284, 261, 248, 252, 256, 244, 254, 268, 260, 252, 259, 292, 270, 240, 217, 254, 259, 285, 251, 223, 228, 263, 247, 259, 262] payload_rows_per_dst=[5394, 5432, 5518, 5468, 5558, 5388, 5506, 5420] dup_rows_per_dst=[2714, 2688, 2740, 2818, 2826, 2598, 2766, 2702] combine_rows_per_src=[5394, 5432, 5518, 5468, 5558, 5388, 5506, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229536 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040800 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852064 weight_movement_bytes=18874368 total_nvlink_bytes=332726432 weight_to_token_ratio=0.060138 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38340 dup_rows=21660 dup_rows_remote=18894 dedup_ratio=0.330505 router_tokens_per_expert=[285, 259, 233, 260, 241, 244, 270, 248, 265, 268, 261, 259, 260, 239, 239, 262, 235, 262, 264, 276, 269, 284, 243, 259, 236, 264, 263, 252, 237, 251, 246, 264, 256, 252, 258, 293, 258, 255, 247, 247, 247, 240, 260, 245, 247, 244, 221, 287, 242, 247, 262, 243, 267, 265, 248, 264, 246, 269, 245, 250, 240, 252, 254, 272, 259, 265, 268, 278, 273, 248, 260, 258, 260, 259, 254, 258, 257, 234, 253, 267, 262, 252, 253, 296, 291, 228, 238, 271, 239, 268, 223, 233, 252, 234, 249, 250, 263, 231, 245, 240, 268, 253, 249, 224, 265, 258, 256, 275, 266, 264, 277, 245, 269, 259, 282, 231, 255, 251, 285, 268, 261, 241, 266, 277, 266, 260, 260, 247] payload_rows_per_dst=[5542, 5480, 5466, 5472, 5536, 5412, 5422, 5546] dup_rows_per_dst=[2644, 2730, 2648, 2660, 2766, 2666, 2736, 2810] combine_rows_per_src=[5542, 5480, 5466, 5472, 5536, 5412, 5422, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=229496 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156164216 ingress_bytes=193683456 link_bytes=193683456 token_bytes=312098936 weight_movement_bytes=37748736 total_nvlink_bytes=349847672 weight_to_token_ratio=0.120951 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855779577e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229000 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156761736 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294472 weight_movement_bytes=18874368 total_nvlink_bytes=332168840 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38338 dup_rows=21826 dup_rows_remote=19058 dedup_ratio=0.333038 router_tokens_per_expert=[260, 275, 246, 263, 252, 244, 234, 257, 274, 240, 294, 214, 256, 259, 240, 251, 255, 246, 236, 262, 260, 277, 246, 258, 279, 251, 253, 284, 265, 253, 251, 232, 255, 240, 236, 267, 257, 221, 248, 235, 230, 269, 281, 244, 245, 255, 269, 270, 272, 257, 257, 241, 260, 268, 271, 243, 279, 235, 259, 261, 271, 282, 265, 263, 268, 240, 268, 253, 259, 216, 246, 268, 264, 259, 239, 248, 274, 260, 254, 248, 244, 274, 244, 258, 258, 260, 252, 260, 226, 270, 233, 270, 249, 269, 262, 250, 270, 248, 264, 272, 260, 251, 270, 245, 279, 266, 254, 263, 237, 261, 259, 245, 238, 271, 247, 287, 234, 271, 237, 243, 261, 277, 271, 268, 246, 249, 237, 271] payload_rows_per_dst=[5372, 5676, 5452, 5546, 5506, 5374, 5306, 5478] dup_rows_per_dst=[2768, 3062, 2592, 2800, 2622, 2784, 2460, 2738] combine_rows_per_src=[5372, 5676, 5452, 5546, 5506, 5374, 5306, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38264 dup_rows=21786 dup_rows_remote=19040 dedup_ratio=0.332428 router_tokens_per_expert=[242, 292, 250, 233, 259, 259, 266, 254, 250, 263, 246, 247, 251, 278, 262, 260, 280, 240, 250, 234, 251, 276, 233, 247, 251, 244, 264, 275, 272, 268, 256, 257, 247, 270, 265, 245, 253, 271, 255, 260, 295, 250, 250, 244, 225, 275, 276, 235, 264, 252, 245, 290, 261, 279, 250, 251, 250, 241, 266, 263, 255, 240, 252, 250, 264, 248, 268, 255, 268, 281, 233, 285, 267, 253, 248, 258, 246, 247, 266, 256, 235, 246, 242, 249, 272, 267, 248, 266, 233, 241, 266, 248, 243, 273, 264, 258, 251, 232, 264, 234, 280, 263, 230, 270, 252, 261, 250, 254, 238, 235, 232, 262, 293, 279, 272, 238, 252, 230, 248, 276, 258, 253, 249, 251, 288, 255, 213, 276] payload_rows_per_dst=[5510, 5386, 5486, 5520, 5544, 5378, 5392, 5534] dup_rows_per_dst=[2714, 2810, 2746, 2698, 2742, 2724, 2624, 2728] combine_rows_per_src=[5510, 5386, 5486, 5520, 5544, 5378, 5392, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38408 dup_rows=21758 dup_rows_remote=19156 dedup_ratio=0.332001 router_tokens_per_expert=[239, 239, 286, 241, 257, 244, 244, 260, 266, 267, 267, 263, 271, 251, 264, 256, 244, 264, 246, 256, 242, 264, 255, 269, 248, 246, 261, 225, 278, 249, 263, 249, 247, 268, 243, 278, 294, 235, 275, 240, 261, 267, 252, 255, 273, 233, 285, 261, 259, 271, 233, 261, 257, 263, 248, 286, 245, 230, 255, 238, 291, 253, 250, 241, 254, 228, 241, 261, 296, 247, 239, 265, 266, 230, 233, 259, 285, 256, 250, 276, 271, 275, 239, 250, 250, 280, 276, 269, 246, 239, 238, 239, 232, 250, 263, 282, 276, 260, 242, 247, 234, 262, 255, 236, 244, 240, 228, 243, 257, 253, 255, 254, 275, 254, 268, 285, 259, 252, 249, 252, 282, 274, 252, 248, 262, 253, 252, 258] payload_rows_per_dst=[5508, 5420, 5538, 5474, 5418, 5524, 5370, 5526] dup_rows_per_dst=[2722, 2698, 2796, 2688, 2754, 2674, 2602, 2824] combine_rows_per_src=[5508, 5420, 5538, 5474, 5418, 5524, 5370, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=228936 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157269576 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310216 weight_movement_bytes=37748736 total_nvlink_bytes=352058952 weight_to_token_ratio=0.120100 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38176 dup_rows=21826 dup_rows_remote=19102 dedup_ratio=0.333038 router_tokens_per_expert=[259, 282, 241, 243, 248, 254, 246, 262, 223, 278, 273, 236, 241, 263, 228, 256, 261, 283, 266, 267, 249, 262, 246, 254, 236, 230, 273, 261, 231, 277, 247, 257, 257, 273, 275, 271, 262, 251, 265, 271, 244, 280, 248, 267, 254, 263, 254, 258, 262, 247, 241, 248, 263, 244, 256, 255, 228, 279, 264, 237, 287, 257, 264, 236, 252, 265, 241, 253, 281, 282, 249, 259, 231, 248, 236, 233, 251, 259, 256, 238, 263, 230, 243, 235, 280, 297, 252, 252, 267, 240, 254, 257, 262, 244, 234, 237, 267, 241, 277, 242, 268, 228, 290, 281, 256, 276, 276, 243, 269, 236, 246, 272, 222, 268, 261, 268, 235, 261, 257, 260, 255, 265, 268, 298, 250, 255, 246, 256] payload_rows_per_dst=[5378, 5498, 5300, 5470, 5432, 5524, 5574, 5534] dup_rows_per_dst=[2600, 2702, 2588, 2666, 2778, 2918, 2850, 2724] combine_rows_per_src=[5378, 5498, 5300, 5470, 5432, 5524, 5574, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229584 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157262032 ingress_bytes=213655552 link_bytes=213655552 token_bytes=314294480 weight_movement_bytes=56623104 total_nvlink_bytes=370917584 weight_to_token_ratio=0.180159 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047142374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229216 combine_payload_bytes=156729344 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156958560 ingress_bytes=232226816 link_bytes=232226816 token_bytes=313687904 weight_movement_bytes=75497472 total_nvlink_bytes=389185376 weight_to_token_ratio=0.240677 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225083088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=230256 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157549424 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314868592 weight_movement_bytes=37748736 total_nvlink_bytes=352617328 weight_to_token_ratio=0.119887 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38272 dup_rows=21746 dup_rows_remote=19094 dedup_ratio=0.331818 router_tokens_per_expert=[279, 266, 242, 252, 252, 237, 254, 262, 267, 249, 274, 245, 241, 261, 264, 265, 242, 249, 253, 243, 268, 248, 262, 252, 257, 242, 264, 250, 236, 307, 263, 249, 247, 250, 266, 242, 271, 261, 265, 290, 256, 244, 268, 242, 269, 259, 267, 266, 257, 258, 257, 256, 250, 246, 304, 225, 255, 257, 265, 240, 251, 256, 274, 257, 267, 260, 268, 270, 233, 258, 267, 271, 230, 259, 254, 241, 235, 246, 245, 281, 286, 244, 258, 257, 260, 238, 266, 267, 262, 259, 272, 241, 227, 252, 265, 235, 247, 256, 233, 254, 274, 229, 274, 238, 238, 258, 262, 265, 293, 248, 249, 276, 247, 229, 253, 251, 274, 254, 272, 264, 241, 283, 229, 235, 222, 253, 253, 274] payload_rows_per_dst=[5496, 5518, 5416, 5444, 5444, 5502, 5478, 5492] dup_rows_per_dst=[2724, 2652, 2910, 2772, 2726, 2676, 2710, 2576] combine_rows_per_src=[5496, 5518, 5416, 5444, 5444, 5502, 5478, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38142 dup_rows=21970 dup_rows_remote=19196 dedup_ratio=0.335236 router_tokens_per_expert=[274, 253, 273, 253, 240, 279, 269, 275, 272, 256, 251, 261, 253, 255, 252, 245, 264, 254, 281, 262, 244, 262, 259, 297, 255, 250, 276, 230, 250, 219, 241, 226, 260, 246, 280, 257, 265, 274, 264, 282, 267, 262, 244, 264, 266, 256, 246, 261, 229, 239, 238, 259, 287, 250, 245, 260, 239, 276, 226, 265, 264, 250, 261, 280, 261, 273, 250, 247, 239, 248, 246, 273, 213, 245, 247, 248, 244, 227, 268, 277, 268, 247, 233, 283, 238, 264, 265, 258, 241, 237, 234, 258, 253, 258, 269, 293, 256, 247, 274, 252, 282, 255, 256, 234, 258, 262, 250, 252, 242, 288, 263, 252, 259, 259, 250, 273, 275, 288, 252, 265, 211, 258, 246, 259, 230, 231, 253, 238] payload_rows_per_dst=[5540, 5414, 5500, 5406, 5366, 5424, 5464, 5452] dup_rows_per_dst=[2782, 2726, 2888, 2730, 2646, 2774, 2782, 2642] combine_rows_per_src=[5540, 5414, 5500, 5406, 5366, 5424, 5464, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229112 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156598008 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312966904 weight_movement_bytes=37748736 total_nvlink_bytes=350715640 weight_to_token_ratio=0.120616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38344 dup_rows=21720 dup_rows_remote=19022 dedup_ratio=0.331421 router_tokens_per_expert=[252, 253, 265, 242, 242, 266, 252, 266, 271, 278, 256, 240, 251, 228, 244, 251, 255, 239, 257, 234, 277, 260, 270, 256, 241, 230, 289, 268, 231, 279, 266, 262, 239, 242, 248, 293, 252, 283, 251, 275, 250, 277, 278, 271, 256, 247, 278, 263, 213, 285, 262, 252, 279, 281, 249, 260, 245, 247, 280, 238, 261, 258, 224, 251, 240, 261, 283, 250, 259, 238, 261, 245, 244, 274, 262, 255, 267, 255, 256, 232, 263, 239, 252, 242, 260, 278, 251, 246, 248, 233, 272, 247, 267, 224, 256, 241, 260, 282, 243, 221, 260, 274, 270, 256, 267, 240, 269, 267, 255, 259, 271, 239, 271, 240, 265, 250, 258, 244, 241, 252, 287, 296, 255, 236, 233, 255, 257, 235] payload_rows_per_dst=[5444, 5528, 5592, 5472, 5482, 5370, 5508, 5420] dup_rows_per_dst=[2670, 2700, 2814, 2698, 2682, 2668, 2758, 2730] combine_rows_per_src=[5444, 5528, 5592, 5472, 5482, 5370, 5508, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229464 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991576 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753688 weight_movement_bytes=18874368 total_nvlink_bytes=332628056 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38334 dup_rows=21736 dup_rows_remote=19114 dedup_ratio=0.331665 router_tokens_per_expert=[248, 273, 265, 267, 270, 250, 256, 262, 266, 254, 241, 254, 239, 273, 237, 262, 262, 272, 251, 262, 247, 251, 248, 254, 232, 270, 253, 247, 273, 236, 250, 248, 238, 244, 252, 263, 277, 250, 243, 258, 251, 253, 284, 260, 245, 243, 257, 267, 251, 249, 241, 259, 213, 221, 281, 269, 249, 242, 258, 251, 235, 273, 282, 262, 266, 256, 239, 251, 236, 262, 254, 249, 259, 251, 263, 271, 244, 276, 224, 243, 260, 242, 282, 267, 273, 258, 258, 265, 251, 242, 241, 270, 273, 224, 274, 243, 265, 279, 259, 271, 269, 267, 245, 256, 267, 268, 251, 286, 257, 258, 280, 251, 279, 261, 254, 246, 266, 264, 234, 228, 252, 245, 218, 277, 258, 258, 265, 273] payload_rows_per_dst=[5464, 5430, 5458, 5462, 5466, 5478, 5600, 5442] dup_rows_per_dst=[2770, 2682, 2712, 2610, 2622, 2768, 2858, 2714] combine_rows_per_src=[5464, 5430, 5458, 5462, 5466, 5478, 5600, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229352 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156458984 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312688616 weight_movement_bytes=18874368 total_nvlink_bytes=331562984 weight_to_token_ratio=0.060362 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229464 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157286488 ingress_bytes=213680128 link_bytes=213680128 token_bytes=314343512 weight_movement_bytes=56623104 total_nvlink_bytes=370966616 weight_to_token_ratio=0.180131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047377849e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38278 dup_rows=21784 dup_rows_remote=19074 dedup_ratio=0.332397 router_tokens_per_expert=[224, 294, 266, 244, 257, 235, 245, 252, 258, 271, 256, 267, 268, 266, 269, 286, 251, 236, 246, 262, 234, 283, 274, 264, 253, 233, 242, 253, 231, 249, 254, 246, 249, 263, 250, 255, 249, 248, 289, 250, 247, 260, 241, 262, 249, 256, 250, 274, 267, 255, 268, 259, 256, 288, 257, 256, 240, 237, 252, 274, 267, 259, 235, 245, 232, 305, 290, 261, 288, 272, 250, 258, 240, 251, 271, 249, 237, 245, 253, 247, 258, 253, 253, 280, 264, 256, 247, 271, 291, 251, 264, 217, 263, 250, 267, 251, 259, 251, 245, 248, 250, 252, 265, 257, 233, 250, 274, 251, 241, 247, 244, 286, 247, 242, 265, 267, 250, 247, 255, 282, 252, 248, 243, 255, 249, 247, 238, 267] payload_rows_per_dst=[5492, 5342, 5474, 5446, 5524, 5550, 5482, 5442] dup_rows_per_dst=[2824, 2680, 2710, 2784, 2774, 2722, 2624, 2666] combine_rows_per_src=[5492, 5342, 5474, 5446, 5524, 5550, 5482, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38142 dup_rows=21890 dup_rows_remote=19220 dedup_ratio=0.334015 router_tokens_per_expert=[248, 252, 272, 225, 282, 270, 237, 269, 274, 272, 246, 232, 270, 212, 266, 239, 249, 250, 264, 261, 228, 275, 247, 257, 285, 238, 252, 288, 254, 268, 266, 268, 261, 274, 261, 266, 252, 274, 274, 288, 225, 235, 229, 283, 257, 237, 247, 262, 235, 281, 264, 244, 271, 259, 248, 267, 226, 244, 288, 263, 246, 257, 251, 241, 260, 229, 269, 243, 256, 263, 255, 271, 251, 273, 206, 236, 253, 270, 260, 269, 257, 246, 254, 239, 244, 263, 246, 290, 240, 245, 261, 254, 251, 273, 278, 241, 262, 249, 259, 269, 255, 244, 242, 240, 278, 275, 253, 274, 248, 272, 255, 270, 255, 258, 261, 250, 239, 302, 279, 253, 235, 252, 219, 256, 254, 233, 256, 249] payload_rows_per_dst=[5504, 5450, 5520, 5356, 5342, 5492, 5564, 5418] dup_rows_per_dst=[2670, 2856, 2886, 2616, 2804, 2672, 2702, 2684] combine_rows_per_src=[5504, 5450, 5520, 5356, 5342, 5492, 5564, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38456 dup_rows=21644 dup_rows_remote=19132 dedup_ratio=0.330261 router_tokens_per_expert=[230, 230, 278, 247, 261, 250, 248, 253, 270, 261, 242, 264, 292, 261, 255, 234, 246, 272, 228, 280, 270, 267, 271, 245, 235, 300, 273, 265, 241, 290, 244, 287, 245, 272, 246, 245, 276, 242, 250, 243, 248, 271, 234, 270, 240, 240, 291, 269, 246, 277, 237, 283, 249, 253, 254, 249, 248, 284, 241, 232, 278, 246, 254, 268, 235, 250, 267, 253, 250, 231, 257, 264, 277, 270, 265, 228, 254, 261, 281, 239, 260, 253, 256, 273, 261, 256, 257, 264, 230, 254, 265, 235, 280, 259, 252, 233, 252, 255, 273, 227, 253, 246, 221, 271, 254, 235, 249, 245, 263, 244, 233, 253, 263, 258, 259, 262, 254, 254, 242, 273, 257, 244, 264, 267, 270, 273, 259, 254] payload_rows_per_dst=[5460, 5578, 5438, 5450, 5468, 5494, 5436, 5568] dup_rows_per_dst=[2692, 2850, 2726, 2748, 2696, 2682, 2512, 2738] combine_rows_per_src=[5460, 5578, 5438, 5450, 5468, 5494, 5436, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229792 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245856 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261920 weight_movement_bytes=18874368 total_nvlink_bytes=333136288 weight_to_token_ratio=0.060059 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38220 dup_rows=21742 dup_rows_remote=18858 dedup_ratio=0.331757 router_tokens_per_expert=[257, 258, 255, 252, 246, 262, 278, 251, 247, 229, 238, 252, 268, 265, 254, 284, 235, 264, 246, 277, 239, 239, 253, 238, 254, 251, 265, 248, 239, 246, 258, 263, 287, 260, 238, 272, 223, 265, 237, 246, 286, 249, 257, 234, 260, 269, 273, 254, 249, 275, 257, 278, 276, 244, 247, 285, 271, 228, 256, 263, 262, 244, 248, 257, 276, 251, 255, 262, 254, 261, 258, 263, 238, 270, 259, 270, 266, 279, 258, 264, 275, 243, 269, 261, 257, 246, 233, 267, 240, 277, 243, 252, 243, 237, 272, 255, 252, 248, 260, 246, 248, 270, 247, 261, 240, 219, 231, 294, 276, 263, 253, 240, 267, 243, 232, 267, 245, 248, 245, 248, 274, 264, 254, 264, 274, 276, 245, 259] payload_rows_per_dst=[5512, 5478, 5482, 5372, 5402, 5452, 5522, 5574] dup_rows_per_dst=[2680, 2552, 2690, 2660, 2640, 2736, 2900, 2884] combine_rows_per_src=[5512, 5478, 5482, 5372, 5402, 5452, 5522, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229408 combine_payload_bytes=156786688 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157016096 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802784 weight_movement_bytes=37748736 total_nvlink_bytes=351551520 weight_to_token_ratio=0.120294 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=229448 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156459080 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312688712 weight_movement_bytes=18874368 total_nvlink_bytes=331563080 weight_to_token_ratio=0.060362 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157515776 dispatch_meta_bytes=230352 combine_payload_bytes=157515776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157746128 ingress_bytes=176390144 link_bytes=176390144 token_bytes=315261904 weight_movement_bytes=18874368 total_nvlink_bytes=334136272 weight_to_token_ratio=0.059869 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690083570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38112 dup_rows=21956 dup_rows_remote=19250 dedup_ratio=0.335022 router_tokens_per_expert=[258, 262, 238, 277, 258, 261, 252, 281, 267, 282, 254, 252, 240, 254, 236, 238, 251, 245, 263, 232, 266, 263, 278, 254, 247, 242, 231, 264, 257, 255, 251, 288, 233, 266, 259, 248, 259, 248, 254, 240, 259, 272, 248, 224, 261, 269, 269, 279, 253, 260, 258, 266, 249, 254, 258, 244, 216, 269, 283, 264, 247, 253, 252, 252, 245, 256, 288, 257, 279, 231, 236, 262, 258, 268, 231, 270, 257, 283, 258, 280, 261, 263, 235, 239, 253, 240, 247, 267, 253, 260, 258, 249, 271, 250, 288, 261, 246, 272, 250, 272, 252, 260, 266, 248, 246, 262, 272, 274, 226, 244, 262, 271, 243, 279, 212, 263, 273, 233, 286, 281, 251, 269, 198, 253, 259, 245, 232, 251] payload_rows_per_dst=[5526, 5468, 5458, 5410, 5492, 5330, 5520, 5376] dup_rows_per_dst=[2694, 2706, 2718, 2746, 2826, 2860, 2726, 2680] combine_rows_per_src=[5526, 5468, 5458, 5410, 5492, 5330, 5520, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38296 dup_rows=21776 dup_rows_remote=18990 dedup_ratio=0.332275 router_tokens_per_expert=[218, 248, 227, 271, 271, 266, 265, 220, 213, 251, 241, 235, 240, 269, 278, 243, 256, 223, 254, 241, 254, 233, 277, 266, 253, 256, 284, 241, 280, 250, 227, 262, 255, 265, 283, 261, 255, 232, 241, 253, 282, 260, 277, 290, 240, 245, 233, 257, 257, 248, 234, 271, 253, 244, 263, 278, 248, 293, 248, 277, 275, 249, 260, 281, 252, 282, 269, 251, 243, 256, 252, 254, 276, 261, 239, 256, 275, 240, 279, 243, 268, 285, 278, 225, 253, 248, 252, 236, 243, 262, 234, 248, 266, 263, 308, 256, 265, 288, 258, 255, 243, 260, 273, 263, 257, 253, 243, 249, 259, 245, 271, 277, 262, 267, 216, 283, 249, 258, 266, 248, 246, 217, 230, 248, 226, 243, 281, 295] payload_rows_per_dst=[5370, 5494, 5506, 5592, 5442, 5464, 5498, 5394] dup_rows_per_dst=[2542, 2620, 2752, 2766, 2814, 2786, 2820, 2676] combine_rows_per_src=[5370, 5494, 5506, 5592, 5442, 5464, 5498, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228312 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156777432 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313326552 weight_movement_bytes=18874368 total_nvlink_bytes=332200920 weight_to_token_ratio=0.060239 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38442 dup_rows=21686 dup_rows_remote=19064 dedup_ratio=0.330902 router_tokens_per_expert=[263, 239, 246, 257, 244, 237, 266, 263, 274, 279, 266, 233, 234, 245, 286, 264, 242, 252, 247, 257, 279, 226, 234, 242, 238, 252, 271, 251, 259, 282, 274, 247, 262, 272, 255, 272, 264, 244, 246, 262, 261, 267, 276, 270, 277, 262, 224, 259, 234, 261, 267, 242, 239, 227, 258, 258, 244, 263, 247, 247, 263, 273, 231, 261, 247, 242, 247, 256, 238, 253, 281, 260, 235, 237, 267, 250, 255, 268, 270, 247, 265, 250, 280, 231, 233, 247, 269, 254, 267, 238, 252, 249, 248, 258, 249, 295, 271, 238, 254, 246, 226, 265, 259, 289, 242, 280, 259, 260, 254, 287, 275, 275, 256, 263, 279, 246, 278, 265, 273, 253, 256, 259, 259, 240, 246, 228, 246, 266] payload_rows_per_dst=[5450, 5436, 5548, 5408, 5448, 5434, 5576, 5550] dup_rows_per_dst=[2742, 2670, 2798, 2622, 2658, 2736, 2784, 2676] combine_rows_per_src=[5450, 5436, 5548, 5408, 5448, 5434, 5576, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=229448 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156336200 ingress_bytes=174981120 link_bytes=174981120 token_bytes=312442952 weight_movement_bytes=18874368 total_nvlink_bytes=331317320 weight_to_token_ratio=0.060409 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676582996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229144 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089560 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949976 weight_movement_bytes=18874368 total_nvlink_bytes=332824344 weight_to_token_ratio=0.060119 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38182 dup_rows=21950 dup_rows_remote=19246 dedup_ratio=0.334930 router_tokens_per_expert=[252, 253, 294, 260, 257, 266, 248, 253, 265, 231, 269, 289, 235, 256, 248, 291, 249, 228, 257, 266, 251, 265, 251, 257, 270, 263, 282, 258, 237, 274, 253, 249, 223, 274, 252, 258, 260, 250, 249, 231, 258, 234, 260, 256, 229, 272, 261, 259, 236, 295, 287, 249, 256, 252, 240, 268, 278, 252, 254, 246, 250, 265, 225, 245, 241, 261, 245, 252, 255, 280, 241, 244, 262, 251, 247, 253, 272, 242, 253, 255, 260, 261, 241, 236, 248, 271, 276, 256, 281, 245, 264, 249, 253, 262, 283, 234, 230, 263, 267, 232, 228, 257, 280, 253, 238, 268, 233, 258, 245, 267, 268, 264, 244, 260, 242, 253, 286, 250, 255, 265, 242, 249, 289, 283, 270, 222, 275, 257] payload_rows_per_dst=[5500, 5464, 5366, 5466, 5404, 5496, 5436, 5454] dup_rows_per_dst=[2834, 2756, 2686, 2730, 2704, 2744, 2666, 2830] combine_rows_per_src=[5500, 5464, 5366, 5466, 5404, 5496, 5436, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=230024 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157688456 ingress_bytes=176332800 link_bytes=176332800 token_bytes=315146888 weight_movement_bytes=18874368 total_nvlink_bytes=334021256 weight_to_token_ratio=0.059891 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689534128e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38158 dup_rows=21848 dup_rows_remote=19158 dedup_ratio=0.333374 router_tokens_per_expert=[266, 235, 253, 252, 290, 286, 268, 247, 244, 241, 252, 255, 242, 252, 256, 259, 251, 257, 252, 254, 250, 258, 257, 268, 250, 252, 244, 249, 265, 216, 281, 250, 277, 279, 250, 250, 247, 250, 249, 245, 268, 249, 268, 268, 249, 261, 244, 256, 273, 258, 253, 234, 255, 255, 275, 255, 253, 272, 242, 261, 241, 253, 242, 264, 270, 231, 276, 271, 273, 274, 261, 254, 294, 257, 245, 257, 255, 232, 257, 272, 268, 261, 288, 259, 241, 249, 247, 252, 241, 252, 253, 273, 227, 257, 223, 259, 263, 274, 269, 264, 254, 240, 298, 260, 279, 237, 243, 254, 263, 255, 242, 291, 254, 264, 249, 239, 253, 268, 254, 241, 266, 230, 247, 234, 226, 259, 242, 279] payload_rows_per_dst=[5432, 5452, 5530, 5428, 5490, 5466, 5506, 5384] dup_rows_per_dst=[2764, 2656, 2690, 2744, 2868, 2634, 2866, 2626] combine_rows_per_src=[5432, 5452, 5530, 5428, 5490, 5466, 5506, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38308 dup_rows=21840 dup_rows_remote=19208 dedup_ratio=0.333252 router_tokens_per_expert=[264, 263, 218, 286, 247, 220, 233, 235, 261, 245, 281, 258, 270, 239, 263, 260, 251, 251, 255, 242, 268, 248, 264, 244, 237, 260, 258, 232, 235, 260, 245, 243, 261, 283, 257, 244, 269, 237, 224, 247, 248, 251, 264, 269, 246, 262, 280, 262, 248, 230, 244, 261, 232, 265, 313, 273, 247, 281, 275, 265, 264, 251, 250, 271, 250, 260, 265, 257, 274, 249, 246, 236, 285, 256, 252, 259, 253, 253, 259, 252, 250, 258, 264, 261, 267, 236, 251, 276, 284, 287, 247, 252, 261, 264, 265, 262, 231, 244, 272, 288, 259, 260, 246, 224, 246, 237, 234, 260, 268, 272, 239, 230, 259, 220, 268, 274, 268, 235, 275, 280, 251, 262, 276, 257, 260, 262, 256, 254] payload_rows_per_dst=[5416, 5376, 5454, 5520, 5520, 5512, 5388, 5510] dup_rows_per_dst=[2670, 2610, 2754, 2820, 2692, 2858, 2632, 2804] combine_rows_per_src=[5416, 5376, 5454, 5520, 5520, 5512, 5388, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38120 dup_rows=21764 dup_rows_remote=18908 dedup_ratio=0.332092 router_tokens_per_expert=[268, 239, 268, 243, 236, 265, 293, 250, 258, 256, 264, 277, 279, 250, 250, 262, 247, 265, 239, 250, 226, 251, 255, 258, 259, 246, 247, 246, 254, 262, 254, 282, 256, 268, 256, 270, 276, 266, 286, 263, 242, 264, 251, 234, 262, 244, 259, 229, 232, 284, 244, 252, 259, 275, 243, 235, 234, 284, 282, 251, 261, 256, 257, 236, 238, 243, 259, 232, 256, 252, 274, 259, 264, 232, 255, 261, 275, 247, 249, 264, 251, 262, 258, 271, 250, 278, 270, 241, 270, 264, 271, 253, 267, 258, 230, 271, 256, 267, 215, 266, 271, 242, 251, 262, 228, 260, 266, 237, 240, 252, 270, 245, 266, 240, 259, 256, 273, 240, 278, 277, 249, 253, 266, 234, 258, 281, 247, 228] payload_rows_per_dst=[5652, 5558, 5552, 5498, 5342, 5332, 5406, 5432] dup_rows_per_dst=[2856, 3006, 2748, 2678, 2586, 2516, 2650, 2724] combine_rows_per_src=[5652, 5558, 5552, 5498, 5342, 5332, 5406, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38188 dup_rows=21778 dup_rows_remote=18862 dedup_ratio=0.332306 router_tokens_per_expert=[268, 256, 220, 268, 265, 250, 244, 276, 247, 237, 263, 263, 258, 253, 261, 233, 228, 234, 251, 236, 260, 261, 257, 275, 277, 242, 271, 274, 249, 226, 256, 266, 266, 250, 259, 237, 251, 255, 265, 225, 240, 275, 260, 250, 266, 227, 227, 235, 286, 249, 265, 255, 211, 253, 250, 248, 262, 270, 292, 252, 261, 284, 226, 247, 258, 263, 243, 275, 254, 240, 259, 242, 260, 246, 269, 280, 264, 265, 263, 224, 274, 248, 272, 242, 237, 273, 240, 292, 288, 257, 255, 241, 265, 245, 265, 251, 241, 263, 247, 240, 242, 282, 245, 245, 250, 274, 267, 240, 266, 240, 265, 248, 270, 276, 275, 265, 243, 263, 280, 252, 264, 269, 250, 267, 278, 253, 260, 274] payload_rows_per_dst=[5508, 5388, 5364, 5482, 5476, 5544, 5426, 5570] dup_rows_per_dst=[2616, 2738, 2612, 2740, 2734, 2746, 2676, 2916] combine_rows_per_src=[5508, 5388, 5364, 5482, 5476, 5544, 5426, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229712 combine_payload_bytes=156393472 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156623184 ingress_bytes=194142208 link_bytes=194142208 token_bytes=313016656 weight_movement_bytes=37748736 total_nvlink_bytes=350765392 weight_to_token_ratio=0.120597 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860175113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38210 dup_rows=21840 dup_rows_remote=19082 dedup_ratio=0.333252 router_tokens_per_expert=[263, 270, 244, 255, 257, 256, 251, 260, 273, 256, 259, 254, 243, 272, 252, 260, 238, 258, 252, 275, 254, 267, 268, 242, 273, 255, 261, 246, 281, 267, 255, 230, 270, 263, 253, 227, 263, 231, 242, 252, 240, 261, 237, 268, 276, 242, 247, 269, 262, 260, 254, 214, 252, 264, 256, 263, 251, 253, 259, 275, 251, 254, 223, 246, 268, 230, 253, 255, 272, 267, 260, 229, 276, 249, 277, 249, 256, 245, 259, 277, 258, 248, 239, 242, 266, 258, 271, 237, 254, 250, 266, 230, 289, 243, 265, 258, 239, 288, 266, 243, 250, 273, 265, 259, 282, 257, 255, 273, 295, 251, 261, 258, 252, 256, 279, 246, 238, 234, 234, 246, 252, 270, 247, 243, 248, 255, 279, 253] payload_rows_per_dst=[5558, 5486, 5378, 5440, 5470, 5442, 5536, 5386] dup_rows_per_dst=[2692, 2758, 2704, 2634, 2774, 2706, 2894, 2678] combine_rows_per_src=[5558, 5486, 5378, 5440, 5470, 5442, 5536, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229264 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156524432 ingress_bytes=175169536 link_bytes=175169536 token_bytes=312819600 weight_movement_bytes=18874368 total_nvlink_bytes=331693968 weight_to_token_ratio=0.060336 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678388305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=230064 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157139632 ingress_bytes=232407040 link_bytes=232407040 token_bytes=314049200 weight_movement_bytes=75497472 total_nvlink_bytes=389546672 weight_to_token_ratio=0.240400 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226809906e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228112 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156367632 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507152 weight_movement_bytes=18874368 total_nvlink_bytes=331381520 weight_to_token_ratio=0.060397 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228200 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156646248 ingress_bytes=213041152 link_bytes=213041152 token_bytes=313064296 weight_movement_bytes=56623104 total_nvlink_bytes=369687400 weight_to_token_ratio=0.180867 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041255496e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38220 dup_rows=21872 dup_rows_remote=19286 dedup_ratio=0.333740 router_tokens_per_expert=[274, 269, 267, 237, 254, 273, 268, 264, 279, 221, 234, 256, 256, 236, 226, 263, 261, 264, 276, 257, 223, 264, 255, 245, 275, 276, 265, 246, 269, 288, 238, 258, 270, 254, 255, 265, 236, 263, 250, 222, 258, 260, 238, 258, 248, 284, 243, 246, 266, 254, 251, 285, 257, 239, 262, 246, 250, 243, 221, 238, 262, 266, 272, 233, 258, 248, 271, 244, 279, 258, 261, 291, 238, 273, 247, 236, 253, 234, 266, 254, 246, 266, 260, 268, 236, 264, 260, 248, 254, 234, 270, 242, 260, 239, 227, 241, 253, 245, 253, 245, 250, 252, 272, 273, 275, 287, 262, 297, 232, 249, 248, 285, 258, 241, 272, 248, 242, 276, 279, 260, 251, 265, 266, 239, 273, 250, 252, 260] payload_rows_per_dst=[5410, 5506, 5362, 5406, 5472, 5444, 5612, 5452] dup_rows_per_dst=[2744, 2814, 2738, 2684, 2750, 2586, 2744, 2812] combine_rows_per_src=[5410, 5506, 5362, 5406, 5472, 5444, 5612, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38230 dup_rows=21780 dup_rows_remote=18998 dedup_ratio=0.332336 router_tokens_per_expert=[267, 272, 255, 259, 236, 256, 250, 256, 242, 251, 266, 228, 236, 231, 271, 276, 233, 248, 247, 260, 257, 291, 282, 259, 234, 243, 257, 240, 261, 257, 255, 251, 278, 258, 238, 260, 273, 236, 229, 246, 226, 282, 276, 277, 247, 237, 222, 217, 257, 286, 264, 259, 269, 246, 263, 235, 273, 280, 243, 259, 270, 246, 241, 263, 229, 239, 241, 259, 246, 239, 238, 268, 249, 270, 233, 266, 280, 244, 271, 242, 272, 279, 271, 252, 256, 274, 261, 259, 269, 260, 254, 250, 235, 287, 264, 262, 243, 271, 247, 223, 246, 234, 277, 261, 252, 260, 273, 254, 273, 304, 262, 271, 268, 262, 258, 277, 254, 269, 269, 233, 263, 260, 242, 248, 250, 227, 251, 284] payload_rows_per_dst=[5428, 5398, 5420, 5526, 5368, 5582, 5524, 5510] dup_rows_per_dst=[2676, 2752, 2584, 2782, 2660, 2828, 2778, 2720] combine_rows_per_src=[5428, 5398, 5420, 5526, 5368, 5582, 5524, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229168 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737328 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245488 weight_movement_bytes=18874368 total_nvlink_bytes=332119856 weight_to_token_ratio=0.060254 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=230024 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156779144 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313328264 weight_movement_bytes=18874368 total_nvlink_bytes=332202632 weight_to_token_ratio=0.060238 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228912 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156818992 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409072 weight_movement_bytes=18874368 total_nvlink_bytes=332283440 weight_to_token_ratio=0.060223 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38302 dup_rows=21802 dup_rows_remote=19170 dedup_ratio=0.332672 router_tokens_per_expert=[269, 253, 258, 256, 266, 250, 253, 251, 282, 271, 265, 266, 265, 254, 261, 269, 278, 254, 246, 305, 272, 238, 247, 264, 249, 253, 259, 277, 267, 252, 255, 249, 277, 266, 254, 243, 245, 249, 228, 275, 226, 267, 260, 288, 254, 259, 242, 232, 255, 235, 230, 283, 229, 243, 259, 251, 266, 241, 256, 255, 265, 260, 279, 255, 261, 239, 264, 273, 215, 248, 225, 244, 256, 254, 251, 245, 264, 267, 272, 254, 256, 225, 264, 275, 275, 256, 266, 252, 264, 277, 241, 262, 253, 248, 251, 228, 228, 233, 238, 257, 228, 275, 257, 276, 248, 241, 269, 282, 289, 247, 246, 252, 246, 256, 255, 253, 231, 274, 261, 248, 269, 255, 269, 277, 235, 255, 252, 260] payload_rows_per_dst=[5492, 5550, 5470, 5468, 5432, 5482, 5380, 5460] dup_rows_per_dst=[2886, 2780, 2660, 2656, 2632, 2704, 2752, 2732] combine_rows_per_src=[5492, 5550, 5470, 5468, 5432, 5482, 5380, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38316 dup_rows=21768 dup_rows_remote=19086 dedup_ratio=0.332153 router_tokens_per_expert=[254, 248, 258, 238, 246, 246, 262, 247, 256, 257, 252, 272, 269, 239, 228, 250, 283, 239, 248, 247, 252, 261, 263, 262, 263, 245, 246, 255, 265, 243, 250, 253, 224, 284, 241, 259, 268, 256, 245, 266, 236, 229, 251, 255, 258, 255, 283, 257, 229, 264, 227, 264, 297, 262, 269, 230, 243, 267, 231, 259, 271, 268, 285, 249, 237, 252, 279, 243, 261, 244, 284, 261, 241, 263, 275, 238, 225, 264, 263, 247, 253, 269, 269, 254, 274, 261, 266, 276, 255, 277, 252, 239, 271, 269, 265, 254, 253, 248, 240, 265, 236, 248, 269, 255, 253, 271, 286, 229, 250, 259, 282, 255, 275, 251, 239, 249, 273, 256, 252, 236, 273, 267, 274, 245, 267, 238, 250, 264] payload_rows_per_dst=[5450, 5450, 5452, 5504, 5426, 5560, 5422, 5504] dup_rows_per_dst=[2594, 2700, 2682, 2726, 2728, 2848, 2776, 2714] combine_rows_per_src=[5450, 5450, 5452, 5504, 5426, 5560, 5422, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38208 dup_rows=21838 dup_rows_remote=19146 dedup_ratio=0.333221 router_tokens_per_expert=[240, 251, 267, 255, 275, 276, 254, 272, 232, 271, 254, 287, 257, 289, 248, 234, 271, 247, 289, 247, 274, 248, 286, 255, 236, 269, 257, 250, 279, 255, 247, 238, 271, 258, 280, 256, 242, 266, 233, 253, 239, 253, 215, 275, 256, 247, 249, 266, 268, 247, 284, 253, 252, 265, 256, 263, 258, 244, 254, 239, 231, 268, 259, 261, 251, 258, 257, 266, 239, 260, 233, 231, 301, 251, 271, 258, 266, 260, 252, 237, 275, 281, 270, 248, 241, 245, 228, 257, 225, 268, 266, 269, 228, 227, 262, 252, 240, 266, 262, 246, 249, 266, 248, 285, 273, 270, 255, 258, 220, 256, 250, 247, 265, 270, 264, 235, 250, 252, 219, 239, 280, 248, 240, 272, 261, 242, 269, 267] payload_rows_per_dst=[5516, 5470, 5410, 5480, 5478, 5434, 5490, 5420] dup_rows_per_dst=[2808, 2826, 2708, 2724, 2704, 2650, 2692, 2726] combine_rows_per_src=[5516, 5470, 5410, 5480, 5478, 5434, 5490, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38330 dup_rows=21652 dup_rows_remote=18946 dedup_ratio=0.330383 router_tokens_per_expert=[246, 264, 237, 251, 243, 262, 231, 245, 262, 263, 245, 254, 252, 269, 265, 254, 264, 256, 241, 259, 234, 248, 262, 275, 254, 289, 250, 226, 257, 273, 246, 257, 273, 258, 267, 230, 258, 284, 255, 254, 267, 246, 258, 264, 240, 243, 239, 225, 260, 251, 285, 251, 287, 246, 251, 288, 235, 252, 247, 263, 260, 258, 248, 249, 265, 242, 250, 232, 276, 275, 241, 261, 265, 247, 250, 241, 274, 259, 275, 258, 222, 253, 268, 265, 249, 264, 250, 264, 250, 266, 239, 246, 255, 270, 259, 270, 280, 263, 249, 241, 269, 268, 262, 259, 256, 240, 246, 250, 251, 266, 241, 272, 269, 258, 252, 263, 273, 237, 279, 242, 269, 247, 265, 274, 259, 258, 242, 241] payload_rows_per_dst=[5450, 5478, 5470, 5476, 5500, 5482, 5474, 5554] dup_rows_per_dst=[2636, 2704, 2652, 2786, 2722, 2672, 2774, 2706] combine_rows_per_src=[5450, 5478, 5470, 5476, 5500, 5482, 5474, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38302 dup_rows=21742 dup_rows_remote=19014 dedup_ratio=0.331757 router_tokens_per_expert=[274, 290, 241, 235, 264, 266, 256, 267, 248, 241, 246, 265, 256, 249, 230, 275, 232, 270, 250, 265, 254, 277, 224, 262, 274, 258, 259, 246, 221, 264, 243, 273, 238, 231, 253, 259, 285, 246, 240, 261, 265, 254, 218, 240, 236, 258, 243, 250, 249, 238, 243, 242, 258, 245, 283, 265, 271, 254, 260, 258, 269, 277, 252, 261, 259, 274, 282, 251, 233, 278, 259, 256, 249, 270, 260, 253, 274, 230, 244, 235, 239, 256, 242, 252, 241, 278, 268, 246, 272, 252, 262, 247, 272, 258, 261, 247, 252, 288, 254, 239, 245, 248, 267, 260, 307, 262, 231, 277, 248, 250, 251, 269, 258, 277, 261, 257, 268, 260, 240, 233, 259, 279, 241, 270, 268, 268, 251, 253] payload_rows_per_dst=[5492, 5562, 5394, 5526, 5472, 5476, 5370, 5502] dup_rows_per_dst=[2728, 2918, 2546, 2784, 2742, 2710, 2590, 2724] combine_rows_per_src=[5492, 5562, 5394, 5526, 5472, 5476, 5370, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229888 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114880 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999872 weight_movement_bytes=37748736 total_nvlink_bytes=351748608 weight_to_token_ratio=0.120219 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43982 payload_rows_remote=38546 dup_rows=21554 dup_rows_remote=18932 dedup_ratio=0.328888 router_tokens_per_expert=[263, 259, 265, 254, 260, 200, 259, 248, 255, 258, 260, 265, 243, 263, 240, 269, 240, 271, 259, 254, 238, 261, 269, 265, 241, 259, 254, 277, 276, 261, 268, 247, 245, 271, 266, 264, 260, 256, 266, 230, 248, 262, 241, 281, 277, 229, 273, 246, 244, 273, 254, 249, 279, 256, 259, 282, 248, 247, 254, 256, 258, 257, 246, 258, 261, 251, 255, 280, 249, 248, 249, 245, 265, 254, 266, 240, 260, 278, 245, 250, 249, 274, 253, 217, 246, 260, 279, 236, 253, 283, 257, 271, 251, 269, 249, 238, 255, 248, 247, 247, 279, 267, 256, 261, 271, 242, 241, 240, 247, 229, 250, 232, 255, 259, 262, 276, 266, 255, 270, 248, 275, 237, 263, 242, 277, 244, 239, 271] payload_rows_per_dst=[5582, 5436, 5532, 5460, 5498, 5474, 5432, 5568] dup_rows_per_dst=[2762, 2622, 2698, 2780, 2694, 2696, 2592, 2710] combine_rows_per_src=[5582, 5436, 5532, 5460, 5498, 5474, 5432, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229608 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171944 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114280 weight_movement_bytes=18874368 total_nvlink_bytes=332988648 weight_to_token_ratio=0.060088 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229416 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156729384 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313229352 weight_movement_bytes=18874368 total_nvlink_bytes=332103720 weight_to_token_ratio=0.060257 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229104 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157228784 ingress_bytes=213622784 link_bytes=213622784 token_bytes=314228464 weight_movement_bytes=56623104 total_nvlink_bytes=370851568 weight_to_token_ratio=0.180197 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046828407e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229264 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114256 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999248 weight_movement_bytes=37748736 total_nvlink_bytes=351747984 weight_to_token_ratio=0.120219 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38174 dup_rows=21840 dup_rows_remote=19098 dedup_ratio=0.333252 router_tokens_per_expert=[246, 264, 238, 233, 270, 223, 241, 251, 264, 240, 238, 266, 247, 250, 262, 249, 261, 258, 277, 259, 285, 239, 247, 267, 263, 269, 268, 245, 250, 275, 264, 275, 250, 225, 293, 275, 302, 279, 260, 239, 272, 284, 268, 269, 275, 262, 245, 271, 257, 240, 257, 253, 268, 246, 247, 262, 270, 265, 225, 278, 265, 252, 212, 245, 248, 256, 241, 266, 232, 258, 251, 258, 267, 246, 255, 267, 237, 263, 246, 239, 270, 255, 255, 228, 270, 266, 267, 238, 270, 295, 250, 244, 254, 272, 272, 226, 236, 270, 270, 261, 281, 243, 269, 255, 256, 241, 248, 250, 242, 233, 243, 270, 268, 278, 255, 259, 236, 278, 254, 270, 234, 220, 239, 265, 249, 230, 248, 260] payload_rows_per_dst=[5362, 5528, 5578, 5452, 5408, 5522, 5418, 5428] dup_rows_per_dst=[2602, 2876, 2960, 2632, 2652, 2742, 2718, 2658] combine_rows_per_src=[5362, 5528, 5578, 5452, 5408, 5522, 5418, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38382 dup_rows=21694 dup_rows_remote=18954 dedup_ratio=0.331024 router_tokens_per_expert=[233, 268, 227, 271, 244, 248, 273, 281, 273, 276, 230, 247, 261, 233, 250, 260, 271, 250, 268, 264, 274, 255, 238, 244, 258, 293, 259, 274, 254, 239, 263, 253, 259, 277, 242, 246, 248, 262, 209, 258, 266, 248, 263, 262, 257, 256, 268, 239, 282, 250, 276, 212, 251, 255, 240, 260, 261, 250, 275, 265, 251, 236, 264, 272, 247, 255, 260, 266, 272, 250, 265, 255, 248, 251, 250, 265, 254, 268, 252, 291, 216, 252, 260, 246, 223, 219, 263, 230, 272, 250, 284, 287, 241, 247, 235, 264, 266, 238, 274, 247, 278, 250, 238, 245, 250, 269, 260, 248, 269, 285, 282, 260, 238, 259, 249, 275, 256, 278, 230, 271, 245, 252, 270, 254, 272, 227, 248, 255] payload_rows_per_dst=[5418, 5504, 5462, 5460, 5532, 5452, 5462, 5552] dup_rows_per_dst=[2732, 2810, 2658, 2740, 2766, 2526, 2856, 2606] combine_rows_per_src=[5418, 5504, 5462, 5460, 5532, 5452, 5462, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=229912 combine_payload_bytes=157884416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158114328 ingress_bytes=195633152 link_bytes=195633152 token_bytes=315998744 weight_movement_bytes=37748736 total_nvlink_bytes=353747480 weight_to_token_ratio=0.119459 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874460605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229088 combine_payload_bytes=156360704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156589792 ingress_bytes=231858176 link_bytes=231858176 token_bytes=312950496 weight_movement_bytes=75497472 total_nvlink_bytes=388447968 weight_to_token_ratio=0.241244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221550961e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229344 combine_payload_bytes=157212672 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157442016 ingress_bytes=194961408 link_bytes=194961408 token_bytes=314654688 weight_movement_bytes=37748736 total_nvlink_bytes=352403424 weight_to_token_ratio=0.119969 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868024284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38234 dup_rows=21716 dup_rows_remote=18930 dedup_ratio=0.331360 router_tokens_per_expert=[256, 242, 261, 238, 240, 246, 255, 228, 269, 257, 252, 256, 257, 249, 237, 260, 257, 221, 273, 256, 261, 280, 237, 269, 248, 260, 294, 276, 255, 250, 263, 269, 230, 267, 244, 260, 265, 261, 269, 268, 264, 246, 268, 254, 242, 257, 270, 260, 231, 264, 260, 268, 258, 274, 266, 227, 251, 249, 256, 269, 231, 250, 234, 248, 249, 258, 270, 245, 272, 241, 254, 293, 269, 276, 279, 248, 258, 269, 251, 254, 241, 263, 237, 247, 249, 250, 255, 223, 236, 239, 291, 257, 242, 279, 274, 248, 272, 258, 254, 220, 234, 235, 240, 291, 275, 275, 265, 278, 228, 259, 256, 253, 236, 253, 265, 266, 255, 252, 241, 275, 229, 272, 256, 273, 257, 255, 269, 271] payload_rows_per_dst=[5378, 5568, 5478, 5416, 5586, 5416, 5412, 5566] dup_rows_per_dst=[2628, 2770, 2772, 2656, 2786, 2646, 2774, 2684] combine_rows_per_src=[5378, 5568, 5478, 5416, 5586, 5416, 5412, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38392 dup_rows=21644 dup_rows_remote=18850 dedup_ratio=0.330261 router_tokens_per_expert=[244, 264, 246, 247, 257, 228, 284, 266, 252, 260, 246, 257, 275, 263, 281, 229, 244, 245, 243, 264, 250, 262, 267, 264, 252, 250, 249, 262, 272, 265, 255, 270, 237, 283, 271, 274, 241, 245, 261, 252, 264, 275, 222, 245, 258, 281, 281, 257, 259, 252, 277, 232, 238, 254, 261, 256, 211, 242, 241, 251, 268, 278, 259, 267, 267, 242, 252, 239, 278, 244, 225, 272, 264, 259, 272, 266, 256, 272, 242, 244, 259, 269, 228, 248, 265, 237, 231, 254, 237, 260, 279, 270, 262, 254, 237, 256, 241, 254, 272, 264, 254, 253, 267, 241, 257, 271, 246, 241, 245, 254, 243, 246, 277, 259, 260, 268, 255, 271, 244, 274, 241, 252, 256, 274, 251, 280, 264, 247] payload_rows_per_dst=[5436, 5508, 5500, 5386, 5454, 5524, 5544, 5540] dup_rows_per_dst=[2762, 2720, 2794, 2706, 2734, 2568, 2554, 2806] combine_rows_per_src=[5436, 5508, 5500, 5386, 5454, 5524, 5544, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38272 dup_rows=21832 dup_rows_remote=19046 dedup_ratio=0.333130 router_tokens_per_expert=[248, 246, 260, 256, 245, 257, 234, 254, 273, 243, 262, 261, 237, 259, 262, 289, 221, 267, 284, 237, 262, 228, 246, 244, 257, 305, 239, 263, 284, 271, 270, 231, 247, 247, 244, 247, 250, 242, 216, 261, 272, 253, 281, 255, 249, 232, 253, 243, 259, 254, 267, 288, 260, 251, 247, 237, 268, 289, 249, 270, 241, 269, 265, 260, 273, 273, 258, 232, 241, 282, 247, 257, 248, 250, 258, 266, 277, 259, 244, 227, 306, 245, 248, 266, 252, 272, 247, 252, 236, 258, 261, 262, 257, 237, 262, 254, 237, 234, 275, 270, 254, 243, 291, 251, 256, 258, 266, 255, 233, 258, 261, 279, 251, 231, 249, 258, 266, 261, 302, 234, 238, 254, 261, 267, 250, 248, 256, 253] payload_rows_per_dst=[5254, 5454, 5376, 5518, 5514, 5670, 5486, 5432] dup_rows_per_dst=[2340, 2764, 2554, 2770, 2670, 3138, 2810, 2786] combine_rows_per_src=[5254, 5454, 5376, 5518, 5514, 5670, 5486, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43966 payload_rows_remote=38436 dup_rows=21570 dup_rows_remote=18940 dedup_ratio=0.329132 router_tokens_per_expert=[245, 264, 270, 249, 243, 255, 244, 235, 275, 263, 251, 254, 241, 246, 256, 274, 253, 275, 238, 271, 298, 259, 259, 241, 231, 260, 247, 267, 241, 260, 267, 258, 268, 246, 261, 264, 244, 274, 240, 256, 239, 227, 253, 224, 268, 264, 255, 277, 232, 272, 265, 267, 233, 290, 274, 255, 243, 259, 257, 261, 258, 284, 277, 253, 266, 249, 289, 258, 257, 247, 251, 260, 275, 241, 258, 269, 264, 243, 249, 262, 253, 256, 232, 227, 261, 255, 248, 269, 241, 256, 268, 254, 247, 254, 262, 215, 250, 238, 269, 268, 238, 235, 294, 254, 249, 251, 250, 268, 242, 251, 286, 237, 266, 260, 269, 264, 245, 250, 267, 236, 253, 251, 258, 266, 248, 274, 256, 259] payload_rows_per_dst=[5356, 5444, 5476, 5586, 5504, 5512, 5530, 5558] dup_rows_per_dst=[2542, 2806, 2644, 2774, 2772, 2716, 2630, 2686] combine_rows_per_src=[5356, 5444, 5476, 5586, 5504, 5512, 5530, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38216 dup_rows=21810 dup_rows_remote=19024 dedup_ratio=0.332794 router_tokens_per_expert=[237, 270, 264, 234, 244, 263, 256, 257, 278, 249, 267, 254, 275, 250, 277, 251, 252, 272, 263, 233, 252, 262, 241, 277, 245, 253, 262, 236, 281, 253, 233, 272, 256, 239, 278, 258, 258, 267, 269, 258, 262, 245, 275, 251, 266, 276, 231, 246, 261, 258, 264, 290, 247, 224, 259, 267, 249, 273, 244, 244, 255, 259, 244, 276, 243, 267, 248, 259, 302, 251, 256, 257, 259, 263, 256, 250, 252, 246, 229, 261, 272, 249, 258, 269, 245, 242, 244, 270, 268, 234, 248, 273, 242, 278, 242, 257, 252, 258, 242, 255, 240, 271, 249, 250, 279, 238, 244, 285, 266, 256, 241, 238, 257, 254, 237, 262, 249, 264, 248, 266, 269, 258, 247, 248, 243, 245, 258, 247] payload_rows_per_dst=[5510, 5538, 5516, 5450, 5410, 5396, 5446, 5460] dup_rows_per_dst=[2786, 2782, 2608, 2746, 2776, 2786, 2682, 2644] combine_rows_per_src=[5510, 5538, 5516, 5450, 5410, 5396, 5446, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228656 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156835120 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313441584 weight_movement_bytes=18874368 total_nvlink_bytes=332315952 weight_to_token_ratio=0.060217 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=228968 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157482600 ingress_bytes=195002368 link_bytes=195002368 token_bytes=314736232 weight_movement_bytes=37748736 total_nvlink_bytes=352484968 weight_to_token_ratio=0.119938 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868416743e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38160 dup_rows=21954 dup_rows_remote=19374 dedup_ratio=0.334991 router_tokens_per_expert=[252, 277, 258, 243, 270, 246, 259, 247, 245, 269, 267, 272, 284, 239, 255, 274, 258, 269, 253, 246, 250, 258, 237, 255, 249, 277, 231, 256, 229, 239, 238, 256, 265, 274, 279, 274, 241, 270, 258, 273, 252, 264, 233, 267, 278, 286, 242, 261, 272, 272, 232, 264, 272, 240, 290, 241, 271, 256, 267, 272, 248, 232, 253, 266, 269, 233, 234, 259, 274, 254, 264, 268, 259, 266, 236, 246, 236, 233, 273, 245, 269, 252, 242, 243, 257, 246, 282, 220, 260, 247, 261, 273, 278, 260, 257, 255, 230, 257, 239, 233, 261, 250, 254, 275, 281, 246, 257, 251, 241, 265, 268, 248, 251, 264, 266, 239, 266, 258, 253, 259, 257, 249, 269, 250, 242, 230, 259, 226] payload_rows_per_dst=[5512, 5422, 5502, 5440, 5476, 5420, 5408, 5402] dup_rows_per_dst=[2802, 2580, 2932, 2856, 2622, 2784, 2704, 2674] combine_rows_per_src=[5512, 5422, 5502, 5440, 5476, 5420, 5408, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229272 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156991384 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313753496 weight_movement_bytes=37748736 total_nvlink_bytes=351502232 weight_to_token_ratio=0.120313 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38334 dup_rows=21810 dup_rows_remote=19148 dedup_ratio=0.332794 router_tokens_per_expert=[266, 252, 254, 254, 241, 262, 280, 278, 263, 262, 260, 261, 255, 246, 256, 244, 293, 234, 236, 247, 263, 254, 251, 247, 266, 235, 264, 239, 262, 257, 282, 234, 236, 269, 246, 242, 290, 256, 278, 297, 261, 247, 257, 247, 266, 257, 238, 242, 251, 288, 257, 225, 260, 256, 275, 259, 254, 243, 235, 258, 265, 273, 267, 262, 250, 268, 265, 246, 284, 253, 263, 254, 199, 253, 266, 248, 238, 258, 260, 265, 255, 236, 248, 251, 256, 262, 260, 266, 234, 244, 233, 251, 248, 255, 252, 276, 274, 274, 258, 268, 249, 271, 251, 245, 272, 248, 267, 243, 251, 276, 249, 275, 261, 244, 266, 223, 241, 250, 277, 252, 249, 230, 263, 248, 265, 239, 271, 266] payload_rows_per_dst=[5496, 5350, 5538, 5528, 5414, 5392, 5570, 5438] dup_rows_per_dst=[2772, 2778, 2720, 2728, 2726, 2662, 2772, 2652] combine_rows_per_src=[5496, 5350, 5538, 5528, 5414, 5392, 5570, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=229504 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157663360 ingress_bytes=195182592 link_bytes=195182592 token_bytes=315097216 weight_movement_bytes=37748736 total_nvlink_bytes=352845952 weight_to_token_ratio=0.119800 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870143561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38278 dup_rows=21728 dup_rows_remote=18992 dedup_ratio=0.331543 router_tokens_per_expert=[246, 254, 262, 276, 260, 271, 250, 256, 273, 261, 235, 281, 248, 273, 242, 266, 248, 215, 280, 292, 255, 247, 238, 255, 290, 251, 276, 225, 266, 259, 283, 247, 263, 247, 249, 239, 231, 274, 235, 268, 268, 257, 241, 242, 275, 257, 247, 220, 245, 242, 249, 266, 264, 262, 282, 249, 286, 256, 250, 252, 284, 253, 248, 245, 275, 251, 262, 254, 269, 280, 254, 266, 240, 268, 234, 230, 257, 237, 244, 283, 269, 256, 273, 269, 276, 242, 254, 241, 282, 270, 240, 248, 266, 257, 231, 240, 215, 249, 274, 244, 253, 256, 279, 263, 239, 255, 229, 251, 258, 240, 269, 273, 266, 256, 282, 263, 256, 249, 227, 244, 259, 270, 253, 281, 235, 248, 244, 243] payload_rows_per_dst=[5518, 5504, 5382, 5530, 5466, 5492, 5448, 5468] dup_rows_per_dst=[2790, 2750, 2644, 2736, 2742, 2736, 2646, 2684] combine_rows_per_src=[5518, 5504, 5382, 5530, 5466, 5492, 5448, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228960 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156761696 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294432 weight_movement_bytes=18874368 total_nvlink_bytes=332168800 weight_to_token_ratio=0.060245 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=230136 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156533496 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312836856 weight_movement_bytes=37748736 total_nvlink_bytes=350585592 weight_to_token_ratio=0.120666 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229928 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245992 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314262056 weight_movement_bytes=18874368 total_nvlink_bytes=333136424 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229080 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157015768 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802456 weight_movement_bytes=37748736 total_nvlink_bytes=351551192 weight_to_token_ratio=0.120295 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38226 dup_rows=21922 dup_rows_remote=19258 dedup_ratio=0.334503 router_tokens_per_expert=[234, 243, 262, 250, 273, 256, 259, 255, 259, 287, 256, 249, 261, 231, 234, 288, 265, 272, 262, 229, 266, 237, 281, 239, 277, 265, 255, 288, 254, 236, 270, 250, 257, 258, 254, 249, 259, 259, 254, 250, 259, 268, 261, 283, 263, 232, 223, 266, 277, 225, 259, 263, 251, 263, 240, 260, 254, 265, 292, 244, 244, 249, 235, 244, 244, 257, 266, 246, 261, 266, 259, 272, 248, 225, 231, 255, 223, 280, 245, 248, 270, 232, 237, 266, 262, 241, 253, 271, 257, 263, 288, 258, 256, 248, 275, 251, 254, 229, 257, 267, 244, 269, 248, 233, 217, 280, 237, 240, 256, 248, 257, 264, 263, 241, 256, 263, 219, 259, 255, 266, 250, 293, 276, 287, 253, 264, 265, 301] payload_rows_per_dst=[5400, 5478, 5502, 5326, 5388, 5476, 5456, 5588] dup_rows_per_dst=[2794, 2814, 2688, 2804, 2664, 2780, 2544, 2834] combine_rows_per_src=[5400, 5478, 5502, 5326, 5388, 5476, 5456, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38340 dup_rows=21768 dup_rows_remote=19118 dedup_ratio=0.332153 router_tokens_per_expert=[257, 248, 233, 263, 236, 232, 255, 247, 277, 300, 259, 228, 252, 252, 250, 236, 255, 272, 258, 269, 245, 264, 261, 244, 239, 264, 234, 250, 263, 266, 266, 225, 259, 232, 273, 269, 244, 270, 244, 242, 244, 245, 251, 267, 225, 264, 250, 260, 277, 288, 275, 292, 252, 270, 250, 268, 270, 285, 268, 264, 262, 249, 282, 249, 255, 271, 260, 248, 262, 247, 246, 255, 246, 240, 242, 242, 264, 266, 247, 265, 256, 238, 258, 237, 270, 247, 230, 239, 256, 259, 235, 253, 269, 271, 251, 266, 248, 246, 248, 261, 228, 234, 284, 261, 259, 240, 257, 253, 286, 246, 291, 240, 267, 275, 235, 268, 241, 258, 246, 248, 269, 285, 239, 289, 257, 258, 257, 263] payload_rows_per_dst=[5388, 5442, 5428, 5636, 5454, 5454, 5502, 5464] dup_rows_per_dst=[2662, 2708, 2650, 2966, 2658, 2616, 2662, 2846] combine_rows_per_src=[5388, 5442, 5428, 5636, 5454, 5454, 5502, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38142 dup_rows=21792 dup_rows_remote=18948 dedup_ratio=0.332520 router_tokens_per_expert=[263, 286, 236, 264, 248, 272, 252, 278, 271, 261, 289, 245, 251, 249, 282, 241, 271, 269, 279, 245, 236, 254, 255, 271, 230, 278, 243, 246, 243, 221, 263, 261, 276, 253, 272, 264, 228, 275, 256, 265, 246, 243, 266, 242, 250, 241, 245, 235, 243, 257, 259, 254, 258, 267, 284, 244, 260, 245, 257, 254, 240, 263, 262, 247, 263, 253, 270, 238, 251, 297, 251, 236, 243, 259, 263, 251, 270, 250, 271, 235, 225, 243, 270, 248, 252, 227, 260, 246, 278, 255, 247, 270, 231, 243, 288, 268, 261, 246, 258, 240, 253, 275, 257, 225, 253, 269, 244, 258, 261, 231, 252, 258, 279, 249, 248, 266, 263, 270, 266, 291, 248, 250, 264, 244, 287, 232, 252, 262] payload_rows_per_dst=[5342, 5412, 5458, 5498, 5548, 5444, 5440, 5602] dup_rows_per_dst=[2680, 2718, 2656, 2690, 2904, 2658, 2642, 2844] combine_rows_per_src=[5342, 5412, 5458, 5498, 5548, 5444, 5440, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229936 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803632 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313377328 weight_movement_bytes=18874368 total_nvlink_bytes=332251696 weight_to_token_ratio=0.060229 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38146 dup_rows=21820 dup_rows_remote=19070 dedup_ratio=0.332947 router_tokens_per_expert=[253, 268, 290, 266, 264, 261, 258, 273, 249, 246, 236, 261, 260, 270, 260, 272, 248, 234, 261, 248, 269, 258, 254, 247, 225, 259, 248, 230, 248, 254, 247, 245, 245, 281, 239, 246, 229, 262, 283, 220, 258, 259, 269, 253, 260, 249, 256, 260, 273, 291, 217, 229, 252, 255, 255, 271, 255, 293, 247, 233, 272, 260, 245, 231, 250, 268, 260, 244, 223, 247, 253, 281, 283, 264, 279, 267, 270, 245, 242, 236, 230, 262, 296, 233, 238, 247, 238, 255, 242, 276, 256, 233, 247, 283, 266, 273, 256, 252, 300, 280, 271, 243, 238, 262, 260, 256, 253, 242, 265, 265, 272, 245, 263, 286, 262, 252, 251, 242, 255, 262, 271, 234, 234, 247, 255, 268, 240, 289] payload_rows_per_dst=[5510, 5376, 5446, 5462, 5466, 5448, 5570, 5438] dup_rows_per_dst=[2864, 2574, 2692, 2696, 2758, 2702, 2750, 2784] combine_rows_per_src=[5510, 5376, 5446, 5462, 5466, 5448, 5570, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38288 dup_rows=21814 dup_rows_remote=19092 dedup_ratio=0.332855 router_tokens_per_expert=[257, 259, 257, 275, 258, 271, 260, 237, 262, 255, 240, 263, 267, 242, 229, 229, 231, 250, 245, 276, 228, 249, 240, 262, 268, 284, 254, 276, 245, 265, 241, 255, 252, 269, 229, 268, 234, 273, 251, 245, 248, 255, 284, 263, 252, 256, 249, 235, 239, 224, 284, 263, 261, 237, 267, 270, 260, 225, 270, 235, 253, 239, 268, 242, 300, 269, 256, 275, 255, 280, 265, 254, 248, 237, 245, 282, 232, 282, 265, 284, 269, 238, 234, 268, 277, 242, 264, 243, 243, 276, 229, 260, 257, 260, 262, 247, 240, 261, 271, 238, 255, 279, 241, 273, 274, 291, 234, 253, 280, 265, 250, 276, 269, 260, 251, 252, 216, 285, 248, 244, 243, 266, 248, 263, 234, 274, 273, 233] payload_rows_per_dst=[5434, 5502, 5578, 5552, 5414, 5518, 5340, 5384] dup_rows_per_dst=[2722, 2814, 2896, 2932, 2706, 2724, 2424, 2596] combine_rows_per_src=[5434, 5502, 5578, 5552, 5414, 5518, 5340, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229832 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270472 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311112 weight_movement_bytes=18874368 total_nvlink_bytes=333185480 weight_to_token_ratio=0.060050 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=228360 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156457992 ingress_bytes=193978368 link_bytes=193978368 token_bytes=312687624 weight_movement_bytes=37748736 total_nvlink_bytes=350436360 weight_to_token_ratio=0.120723 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858605279e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38348 dup_rows=21800 dup_rows_remote=19198 dedup_ratio=0.332642 router_tokens_per_expert=[240, 243, 262, 249, 278, 245, 264, 276, 262, 280, 263, 248, 258, 232, 244, 273, 240, 235, 243, 235, 251, 249, 264, 252, 230, 246, 252, 257, 247, 235, 253, 276, 275, 262, 248, 262, 278, 246, 254, 248, 256, 286, 251, 249, 254, 269, 260, 261, 267, 244, 242, 262, 263, 258, 213, 289, 249, 246, 240, 262, 277, 263, 238, 272, 269, 261, 258, 256, 247, 278, 254, 247, 265, 251, 254, 236, 214, 238, 251, 252, 259, 250, 249, 242, 247, 272, 212, 235, 260, 277, 269, 267, 278, 256, 267, 236, 280, 263, 279, 291, 238, 264, 256, 271, 285, 237, 229, 276, 263, 257, 263, 261, 260, 257, 239, 263, 233, 261, 262, 247, 256, 273, 255, 244, 247, 268, 286, 271] payload_rows_per_dst=[5514, 5388, 5540, 5456, 5346, 5466, 5560, 5466] dup_rows_per_dst=[2720, 2602, 2778, 2654, 2716, 2686, 2866, 2778] combine_rows_per_src=[5514, 5388, 5540, 5456, 5346, 5466, 5560, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43954 payload_rows_remote=38488 dup_rows=21582 dup_rows_remote=18836 dedup_ratio=0.329315 router_tokens_per_expert=[262, 245, 249, 264, 225, 275, 252, 246, 261, 245, 267, 280, 262, 247, 228, 243, 274, 270, 272, 240, 268, 269, 242, 255, 237, 237, 228, 251, 232, 267, 267, 246, 267, 279, 252, 273, 253, 244, 241, 259, 271, 259, 243, 246, 240, 266, 249, 249, 279, 237, 273, 258, 250, 271, 251, 250, 276, 263, 284, 244, 278, 275, 253, 266, 262, 282, 256, 251, 253, 225, 257, 225, 241, 262, 263, 251, 263, 264, 235, 263, 248, 250, 285, 274, 264, 224, 261, 289, 242, 225, 252, 273, 235, 273, 244, 267, 228, 274, 249, 281, 241, 263, 272, 262, 264, 229, 264, 269, 264, 285, 251, 247, 227, 250, 240, 243, 255, 281, 271, 267, 258, 249, 251, 255, 246, 234, 258, 276] payload_rows_per_dst=[5472, 5422, 5558, 5586, 5460, 5466, 5576, 5414] dup_rows_per_dst=[2630, 2688, 2624, 2830, 2646, 2746, 2710, 2708] combine_rows_per_src=[5472, 5422, 5558, 5586, 5460, 5466, 5576, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38470 dup_rows=21598 dup_rows_remote=18900 dedup_ratio=0.329559 router_tokens_per_expert=[251, 243, 255, 243, 247, 264, 263, 280, 264, 243, 258, 266, 257, 237, 253, 288, 239, 270, 257, 254, 239, 250, 270, 268, 247, 238, 259, 257, 264, 276, 237, 238, 273, 280, 275, 267, 257, 269, 251, 253, 221, 252, 239, 248, 271, 258, 246, 244, 225, 259, 228, 266, 289, 248, 276, 246, 258, 262, 274, 268, 271, 218, 239, 256, 239, 265, 260, 254, 243, 249, 242, 266, 278, 262, 258, 251, 227, 257, 257, 268, 229, 266, 250, 261, 261, 238, 276, 260, 271, 260, 240, 277, 277, 270, 246, 247, 248, 284, 257, 249, 256, 290, 251, 270, 276, 247, 227, 264, 228, 298, 219, 244, 266, 287, 237, 247, 260, 248, 255, 239, 250, 258, 246, 248, 274, 280, 255, 243] payload_rows_per_dst=[5510, 5496, 5510, 5468, 5434, 5510, 5512, 5498] dup_rows_per_dst=[2714, 2630, 2698, 2698, 2718, 2748, 2704, 2688] combine_rows_per_src=[5510, 5496, 5510, 5468, 5434, 5510, 5512, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228864 combine_payload_bytes=156246016 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156474880 ingress_bytes=212869120 link_bytes=212869120 token_bytes=312720896 weight_movement_bytes=56623104 total_nvlink_bytes=369344000 weight_to_token_ratio=0.181066 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039607170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229520 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057168 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884816 weight_movement_bytes=18874368 total_nvlink_bytes=332759184 weight_to_token_ratio=0.060132 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=230184 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303592 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314377000 weight_movement_bytes=18874368 total_nvlink_bytes=333251368 weight_to_token_ratio=0.060037 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157646848 dispatch_meta_bytes=229296 combine_payload_bytes=157646848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157876144 ingress_bytes=176521216 link_bytes=176521216 token_bytes=315522992 weight_movement_bytes=18874368 total_nvlink_bytes=334397360 weight_to_token_ratio=0.059819 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691339438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229480 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157802600 ingress_bytes=176447488 link_bytes=176447488 token_bytes=315375720 weight_movement_bytes=18874368 total_nvlink_bytes=334250088 weight_to_token_ratio=0.059847 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690633012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38436 dup_rows=21686 dup_rows_remote=18988 dedup_ratio=0.330902 router_tokens_per_expert=[262, 254, 257, 246, 263, 225, 287, 262, 252, 248, 257, 241, 269, 263, 223, 274, 252, 244, 264, 250, 245, 264, 282, 282, 255, 247, 242, 252, 229, 259, 248, 259, 261, 262, 233, 252, 260, 255, 268, 232, 265, 270, 234, 275, 253, 253, 253, 254, 227, 269, 243, 244, 242, 239, 263, 282, 257, 258, 287, 265, 257, 244, 258, 250, 270, 244, 247, 250, 253, 253, 255, 266, 251, 244, 264, 263, 248, 251, 241, 256, 254, 243, 241, 232, 238, 228, 254, 243, 267, 253, 238, 262, 277, 243, 262, 271, 268, 253, 241, 270, 275, 250, 297, 255, 237, 288, 243, 260, 278, 261, 250, 266, 227, 298, 272, 258, 238, 235, 275, 256, 262, 257, 269, 258, 251, 297, 265, 274] payload_rows_per_dst=[5512, 5436, 5530, 5442, 5414, 5420, 5586, 5510] dup_rows_per_dst=[2654, 2712, 2630, 2728, 2698, 2592, 2798, 2874] combine_rows_per_src=[5512, 5436, 5530, 5442, 5414, 5420, 5586, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38302 dup_rows=21818 dup_rows_remote=19192 dedup_ratio=0.332916 router_tokens_per_expert=[243, 282, 234, 260, 274, 271, 254, 274, 289, 243, 256, 277, 290, 258, 284, 260, 263, 267, 274, 252, 276, 232, 234, 242, 300, 265, 263, 240, 270, 259, 264, 250, 245, 246, 234, 247, 251, 235, 242, 256, 277, 260, 264, 234, 265, 253, 244, 268, 262, 262, 244, 255, 269, 259, 251, 236, 236, 251, 258, 269, 237, 264, 263, 246, 238, 272, 264, 252, 237, 261, 254, 252, 242, 253, 241, 271, 266, 254, 226, 278, 254, 267, 257, 268, 232, 244, 252, 249, 246, 266, 269, 257, 267, 265, 261, 267, 225, 246, 251, 268, 253, 249, 251, 246, 267, 260, 272, 264, 240, 257, 266, 252, 222, 247, 245, 249, 256, 264, 247, 250, 267, 268, 250, 268, 252, 250, 224, 277] payload_rows_per_dst=[5594, 5496, 5416, 5414, 5430, 5498, 5454, 5416] dup_rows_per_dst=[2904, 2806, 2626, 2710, 2692, 2744, 2680, 2656] combine_rows_per_src=[5594, 5496, 5416, 5414, 5430, 5498, 5454, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38090 dup_rows=21948 dup_rows_remote=19124 dedup_ratio=0.334900 router_tokens_per_expert=[280, 301, 255, 270, 246, 246, 242, 264, 233, 273, 264, 268, 270, 241, 253, 266, 248, 271, 267, 223, 228, 260, 242, 274, 256, 274, 241, 236, 232, 278, 258, 257, 257, 276, 230, 264, 245, 267, 261, 266, 252, 258, 241, 251, 289, 260, 263, 227, 261, 231, 258, 243, 258, 263, 266, 235, 246, 267, 242, 253, 285, 256, 252, 242, 307, 242, 274, 273, 265, 261, 263, 256, 234, 241, 246, 237, 255, 244, 252, 216, 253, 261, 273, 256, 284, 264, 264, 261, 258, 252, 271, 277, 258, 250, 287, 254, 253, 263, 262, 251, 249, 263, 254, 228, 257, 263, 248, 272, 269, 256, 219, 235, 248, 265, 244, 277, 250, 275, 281, 256, 231, 248, 223, 268, 253, 254, 242, 240] payload_rows_per_dst=[5248, 5446, 5428, 5632, 5390, 5436, 5510, 5498] dup_rows_per_dst=[2372, 2814, 2786, 2826, 2742, 2792, 2792, 2824] combine_rows_per_src=[5248, 5446, 5428, 5632, 5390, 5436, 5510, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=229696 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157663552 ingress_bytes=176308224 link_bytes=176308224 token_bytes=315097408 weight_movement_bytes=18874368 total_nvlink_bytes=333971776 weight_to_token_ratio=0.059900 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689298653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38392 dup_rows=21774 dup_rows_remote=19156 dedup_ratio=0.332245 router_tokens_per_expert=[255, 236, 296, 266, 250, 250, 264, 255, 254, 244, 279, 275, 267, 244, 252, 257, 238, 252, 243, 234, 267, 262, 267, 267, 231, 264, 261, 238, 270, 256, 288, 264, 269, 247, 248, 244, 261, 229, 295, 266, 239, 273, 280, 265, 242, 264, 266, 259, 263, 253, 263, 260, 265, 261, 239, 263, 241, 274, 247, 240, 255, 251, 261, 262, 256, 250, 248, 288, 248, 263, 272, 242, 252, 268, 261, 244, 253, 264, 274, 259, 271, 236, 273, 250, 233, 219, 259, 247, 254, 249, 249, 234, 247, 275, 281, 271, 272, 256, 259, 252, 258, 261, 243, 264, 242, 249, 241, 226, 236, 228, 256, 251, 240, 225, 252, 285, 239, 288, 237, 263, 273, 229, 222, 257, 292, 240, 287, 264] payload_rows_per_dst=[5508, 5476, 5544, 5508, 5468, 5410, 5370, 5478] dup_rows_per_dst=[2780, 2728, 2750, 2688, 2816, 2686, 2618, 2708] combine_rows_per_src=[5508, 5476, 5544, 5508, 5468, 5410, 5370, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229976 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114968 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999960 weight_movement_bytes=18874368 total_nvlink_bytes=332874328 weight_to_token_ratio=0.060109 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38194 dup_rows=21678 dup_rows_remote=18678 dedup_ratio=0.330780 router_tokens_per_expert=[266, 229, 247, 267, 223, 238, 284, 278, 268, 266, 242, 238, 273, 239, 243, 254, 277, 272, 245, 259, 266, 257, 253, 277, 289, 264, 254, 251, 236, 253, 264, 220, 251, 282, 229, 254, 252, 237, 266, 227, 225, 242, 265, 239, 239, 278, 263, 238, 240, 260, 232, 244, 247, 270, 270, 248, 281, 205, 265, 269, 253, 251, 237, 276, 256, 287, 261, 275, 232, 245, 256, 273, 254, 271, 282, 282, 221, 240, 252, 237, 258, 253, 267, 261, 279, 284, 244, 244, 245, 250, 266, 258, 234, 249, 265, 257, 244, 249, 245, 254, 291, 270, 260, 285, 265, 254, 244, 260, 257, 271, 242, 239, 269, 267, 281, 235, 265, 251, 279, 259, 259, 234, 251, 266, 255, 277, 268, 257] payload_rows_per_dst=[5664, 5374, 5494, 5406, 5462, 5444, 5560, 5454] dup_rows_per_dst=[3000, 2584, 2646, 2730, 2864, 2540, 2700, 2614] combine_rows_per_src=[5664, 5374, 5494, 5406, 5462, 5444, 5560, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=228856 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156245496 ingress_bytes=174891008 link_bytes=174891008 token_bytes=312262136 weight_movement_bytes=18874368 total_nvlink_bytes=331136504 weight_to_token_ratio=0.060444 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675719587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38392 dup_rows=21668 dup_rows_remote=19050 dedup_ratio=0.330627 router_tokens_per_expert=[264, 275, 260, 248, 264, 248, 250, 264, 257, 293, 260, 275, 271, 224, 266, 231, 239, 251, 227, 265, 266, 229, 249, 230, 254, 277, 244, 271, 243, 284, 248, 270, 259, 271, 251, 240, 228, 245, 255, 267, 285, 261, 270, 277, 274, 267, 242, 259, 254, 252, 234, 249, 262, 274, 239, 238, 234, 219, 243, 254, 249, 249, 264, 234, 208, 262, 237, 287, 241, 261, 277, 257, 256, 259, 266, 274, 267, 278, 289, 253, 259, 254, 270, 242, 271, 263, 278, 265, 232, 279, 267, 280, 260, 249, 222, 265, 256, 300, 264, 292, 259, 283, 228, 247, 246, 231, 265, 247, 243, 228, 248, 241, 277, 257, 254, 279, 227, 243, 260, 249, 253, 241, 246, 250, 247, 271, 260, 252] payload_rows_per_dst=[5544, 5476, 5544, 5398, 5530, 5404, 5494, 5478] dup_rows_per_dst=[2756, 2618, 2758, 2498, 2814, 2908, 2662, 2654] combine_rows_per_src=[5544, 5476, 5544, 5398, 5530, 5404, 5494, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=230192 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157483824 ingress_bytes=176128000 link_bytes=176128000 token_bytes=314737456 weight_movement_bytes=18874368 total_nvlink_bytes=333611824 weight_to_token_ratio=0.059969 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687571836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38194 dup_rows=21814 dup_rows_remote=19036 dedup_ratio=0.332855 router_tokens_per_expert=[257, 250, 244, 272, 248, 229, 265, 250, 257, 255, 233, 283, 277, 275, 264, 262, 261, 262, 254, 251, 259, 287, 260, 285, 264, 268, 239, 274, 264, 262, 257, 250, 234, 236, 240, 249, 261, 260, 268, 253, 241, 262, 271, 266, 259, 252, 251, 259, 275, 249, 257, 241, 240, 255, 249, 285, 252, 279, 258, 267, 252, 266, 274, 254, 251, 254, 254, 258, 263, 241, 259, 257, 242, 280, 230, 279, 237, 241, 257, 231, 235, 271, 243, 278, 272, 257, 269, 249, 278, 267, 231, 234, 261, 254, 252, 258, 253, 260, 247, 255, 228, 264, 286, 229, 252, 260, 238, 254, 206, 259, 281, 250, 261, 247, 249, 255, 233, 271, 241, 267, 294, 247, 255, 267, 236, 265, 234, 248] payload_rows_per_dst=[5518, 5550, 5416, 5528, 5398, 5494, 5386, 5432] dup_rows_per_dst=[2724, 2844, 2708, 2778, 2670, 2724, 2658, 2708] combine_rows_per_src=[5518, 5550, 5416, 5528, 5398, 5494, 5386, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38174 dup_rows=21812 dup_rows_remote=19094 dedup_ratio=0.332825 router_tokens_per_expert=[258, 261, 253, 256, 269, 242, 269, 258, 282, 264, 224, 270, 233, 243, 273, 242, 236, 253, 267, 245, 267, 283, 241, 264, 272, 248, 280, 262, 258, 249, 265, 254, 249, 255, 275, 218, 247, 253, 232, 260, 263, 252, 269, 221, 238, 247, 250, 237, 253, 271, 248, 260, 260, 251, 259, 259, 235, 229, 240, 281, 251, 273, 279, 246, 257, 235, 220, 273, 229, 244, 254, 252, 250, 278, 251, 265, 264, 273, 281, 246, 281, 256, 243, 240, 256, 239, 250, 279, 250, 255, 274, 269, 282, 260, 236, 264, 241, 269, 287, 267, 249, 256, 267, 260, 263, 264, 287, 271, 233, 260, 248, 256, 279, 253, 247, 247, 255, 257, 258, 275, 269, 239, 251, 250, 245, 270, 231, 256] payload_rows_per_dst=[5404, 5510, 5370, 5486, 5438, 5550, 5512, 5454] dup_rows_per_dst=[2790, 2778, 2562, 2704, 2706, 2718, 2844, 2710] combine_rows_per_src=[5404, 5510, 5370, 5486, 5438, 5550, 5512, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=227488 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156670112 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313112736 weight_movement_bytes=18874368 total_nvlink_bytes=331987104 weight_to_token_ratio=0.060280 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38288 dup_rows=21790 dup_rows_remote=19096 dedup_ratio=0.332489 router_tokens_per_expert=[235, 271, 257, 251, 270, 223, 236, 285, 243, 244, 255, 255, 251, 244, 257, 270, 257, 251, 227, 276, 197, 278, 237, 261, 256, 236, 232, 262, 263, 247, 282, 268, 253, 228, 246, 256, 266, 251, 267, 260, 272, 276, 265, 277, 232, 249, 254, 259, 243, 249, 263, 261, 241, 257, 284, 281, 227, 241, 254, 269, 264, 271, 270, 263, 228, 245, 260, 234, 267, 260, 226, 250, 266, 252, 258, 246, 268, 272, 289, 255, 254, 258, 277, 251, 249, 267, 247, 242, 255, 269, 288, 254, 268, 225, 236, 277, 280, 239, 246, 238, 258, 249, 252, 255, 259, 258, 276, 254, 236, 288, 264, 223, 261, 232, 266, 258, 260, 288, 261, 262, 242, 263, 271, 253, 242, 256, 279, 280] payload_rows_per_dst=[5502, 5414, 5454, 5566, 5458, 5472, 5390, 5490] dup_rows_per_dst=[2592, 2646, 2768, 2710, 2694, 2762, 2760, 2858] combine_rows_per_src=[5502, 5414, 5454, 5566, 5458, 5472, 5390, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229768 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157483400 ingress_bytes=176128000 link_bytes=176128000 token_bytes=314737032 weight_movement_bytes=18874368 total_nvlink_bytes=333611400 weight_to_token_ratio=0.059969 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687571836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228920 combine_payload_bytes=156442624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156671544 ingress_bytes=213065728 link_bytes=213065728 token_bytes=313114168 weight_movement_bytes=56623104 total_nvlink_bytes=369737272 weight_to_token_ratio=0.180839 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041490971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229072 combine_payload_bytes=156360704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156589776 ingress_bytes=231858176 link_bytes=231858176 token_bytes=312950480 weight_movement_bytes=75497472 total_nvlink_bytes=388447952 weight_to_token_ratio=0.241244 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221550961e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38064 dup_rows=21994 dup_rows_remote=19160 dedup_ratio=0.335602 router_tokens_per_expert=[270, 239, 237, 274, 258, 273, 270, 255, 228, 274, 237, 278, 230, 258, 230, 248, 246, 242, 271, 283, 279, 265, 258, 275, 241, 230, 249, 272, 264, 255, 274, 264, 257, 261, 264, 280, 255, 250, 265, 271, 254, 230, 267, 267, 225, 278, 264, 268, 261, 271, 255, 230, 267, 261, 281, 258, 253, 240, 257, 252, 271, 251, 233, 251, 261, 253, 251, 238, 288, 270, 234, 229, 248, 246, 250, 268, 252, 258, 237, 256, 253, 288, 267, 262, 279, 255, 272, 301, 245, 260, 257, 238, 216, 269, 259, 228, 249, 246, 267, 238, 251, 257, 267, 268, 238, 256, 253, 237, 227, 268, 287, 245, 274, 257, 245, 256, 241, 286, 230, 243, 231, 238, 261, 247, 269, 250, 248, 275] payload_rows_per_dst=[5412, 5500, 5478, 5422, 5458, 5454, 5420, 5398] dup_rows_per_dst=[2706, 2836, 2834, 2762, 2620, 2844, 2688, 2704] combine_rows_per_src=[5412, 5500, 5478, 5422, 5458, 5454, 5420, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43436 payload_rows_remote=37804 dup_rows=22100 dup_rows_remote=19042 dedup_ratio=0.337219 router_tokens_per_expert=[243, 265, 260, 272, 240, 235, 257, 257, 265, 258, 243, 231, 259, 262, 246, 237, 252, 249, 242, 280, 305, 247, 222, 247, 259, 258, 273, 276, 254, 281, 256, 246, 239, 242, 235, 248, 261, 247, 243, 237, 228, 281, 261, 235, 238, 284, 276, 253, 228, 246, 271, 265, 254, 275, 248, 276, 255, 260, 227, 264, 233, 270, 267, 270, 266, 251, 278, 273, 267, 276, 257, 276, 279, 243, 252, 242, 237, 265, 258, 258, 261, 264, 237, 254, 245, 262, 254, 253, 240, 239, 254, 273, 256, 263, 258, 246, 248, 284, 252, 255, 245, 228, 261, 272, 249, 261, 262, 277, 289, 250, 265, 256, 252, 246, 277, 259, 283, 252, 246, 248, 257, 251, 260, 247, 258, 233, 243, 271] payload_rows_per_dst=[5386, 5222, 5226, 5402, 5640, 5434, 5494, 5632] dup_rows_per_dst=[2674, 2358, 2556, 2748, 2950, 2874, 2882, 3058] combine_rows_per_src=[5386, 5222, 5226, 5402, 5640, 5434, 5494, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229536 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057184 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884832 weight_movement_bytes=18874368 total_nvlink_bytes=332759200 weight_to_token_ratio=0.060132 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38306 dup_rows=21722 dup_rows_remote=19042 dedup_ratio=0.331451 router_tokens_per_expert=[243, 280, 268, 286, 278, 261, 243, 258, 259, 268, 266, 279, 231, 246, 282, 235, 247, 242, 244, 252, 258, 245, 268, 237, 261, 254, 259, 274, 236, 247, 238, 286, 265, 281, 260, 271, 249, 256, 289, 266, 275, 264, 235, 253, 271, 266, 258, 242, 265, 244, 263, 260, 235, 246, 237, 251, 238, 258, 248, 240, 271, 245, 261, 309, 231, 261, 229, 278, 261, 247, 255, 262, 253, 260, 249, 211, 251, 296, 264, 228, 250, 253, 236, 261, 266, 251, 236, 260, 260, 247, 250, 290, 258, 262, 254, 241, 282, 236, 238, 252, 236, 272, 273, 255, 281, 287, 223, 254, 260, 240, 244, 261, 308, 267, 272, 243, 246, 249, 236, 242, 252, 269, 256, 230, 229, 267, 234, 260] payload_rows_per_dst=[5532, 5428, 5536, 5440, 5466, 5488, 5508, 5416] dup_rows_per_dst=[2834, 2668, 2866, 2702, 2606, 2662, 2680, 2704] combine_rows_per_src=[5532, 5428, 5536, 5440, 5466, 5488, 5508, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228896 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156139040 ingress_bytes=212533248 link_bytes=212533248 token_bytes=312049184 weight_movement_bytes=56623104 total_nvlink_bytes=368672288 weight_to_token_ratio=0.181456 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036389009e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=154845184 dispatch_meta_bytes=227384 combine_payload_bytes=154845184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155072568 ingress_bytes=173719552 link_bytes=173719552 token_bytes=309917752 weight_movement_bytes=18874368 total_nvlink_bytes=328792120 weight_to_token_ratio=0.060901 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.664495272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38100 dup_rows=21642 dup_rows_remote=18516 dedup_ratio=0.330231 router_tokens_per_expert=[254, 251, 238, 252, 276, 279, 286, 249, 255, 237, 284, 265, 285, 273, 241, 270, 230, 269, 248, 261, 259, 253, 257, 291, 254, 289, 246, 288, 232, 262, 286, 254, 243, 263, 259, 269, 290, 244, 271, 264, 260, 236, 251, 249, 230, 246, 256, 261, 265, 258, 238, 255, 240, 248, 248, 253, 230, 262, 223, 225, 260, 257, 255, 243, 270, 292, 234, 266, 250, 234, 239, 271, 270, 254, 251, 246, 256, 254, 229, 266, 254, 256, 263, 232, 235, 259, 268, 253, 236, 236, 242, 241, 224, 254, 262, 243, 253, 277, 282, 279, 237, 259, 253, 252, 299, 252, 271, 259, 268, 249, 238, 245, 247, 272, 265, 253, 267, 268, 244, 243, 261, 247, 275, 260, 267, 263, 251, 246] payload_rows_per_dst=[5794, 5398, 5510, 5456, 5474, 5374, 5538, 5350] dup_rows_per_dst=[3126, 2706, 2680, 2458, 2826, 2542, 2672, 2632] combine_rows_per_src=[5794, 5398, 5510, 5456, 5474, 5374, 5538, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229392 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130768 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032144 weight_movement_bytes=18874368 total_nvlink_bytes=332906512 weight_to_token_ratio=0.060103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38360 dup_rows=21712 dup_rows_remote=18982 dedup_ratio=0.331299 router_tokens_per_expert=[226, 266, 243, 306, 268, 228, 255, 279, 275, 262, 255, 250, 274, 250, 257, 233, 271, 261, 247, 240, 257, 262, 253, 242, 258, 266, 251, 274, 222, 284, 261, 248, 267, 252, 243, 257, 247, 269, 236, 265, 253, 266, 272, 280, 254, 255, 248, 245, 281, 243, 237, 256, 256, 252, 260, 291, 256, 255, 245, 251, 262, 240, 247, 259, 255, 256, 240, 251, 254, 237, 270, 272, 264, 261, 222, 268, 243, 249, 297, 254, 272, 264, 246, 247, 256, 270, 244, 260, 281, 280, 253, 245, 247, 258, 246, 272, 266, 239, 251, 277, 270, 225, 249, 265, 241, 255, 243, 276, 260, 242, 253, 256, 241, 243, 260, 238, 276, 259, 259, 250, 261, 244, 240, 262, 251, 244, 265, 249] payload_rows_per_dst=[5568, 5464, 5420, 5496, 5540, 5490, 5432, 5414] dup_rows_per_dst=[2686, 2730, 2798, 2686, 2646, 2792, 2704, 2670] combine_rows_per_src=[5568, 5464, 5420, 5496, 5540, 5490, 5432, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38348 dup_rows=21710 dup_rows_remote=18904 dedup_ratio=0.331268 router_tokens_per_expert=[262, 259, 253, 271, 267, 244, 244, 284, 300, 237, 256, 255, 230, 239, 246, 248, 240, 274, 270, 255, 254, 274, 236, 225, 258, 240, 235, 283, 272, 249, 272, 267, 269, 258, 232, 230, 271, 265, 267, 249, 240, 280, 272, 255, 266, 290, 244, 243, 251, 264, 247, 257, 234, 278, 249, 262, 279, 253, 236, 275, 261, 276, 266, 254, 231, 274, 247, 280, 258, 221, 250, 242, 249, 259, 228, 263, 263, 257, 241, 263, 270, 261, 263, 256, 252, 275, 266, 259, 266, 246, 280, 250, 237, 257, 241, 243, 224, 244, 264, 250, 241, 260, 251, 250, 240, 278, 254, 247, 251, 274, 285, 259, 261, 242, 274, 295, 256, 263, 229, 251, 265, 260, 259, 248, 239, 251, 238, 245] payload_rows_per_dst=[5480, 5500, 5538, 5478, 5390, 5548, 5446, 5446] dup_rows_per_dst=[2710, 2708, 2724, 2806, 2662, 2696, 2698, 2706] combine_rows_per_src=[5480, 5500, 5538, 5478, 5390, 5548, 5446, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38458 dup_rows=21578 dup_rows_remote=18996 dedup_ratio=0.329254 router_tokens_per_expert=[219, 262, 236, 245, 254, 259, 258, 253, 237, 258, 244, 244, 261, 257, 263, 245, 270, 252, 259, 261, 265, 244, 255, 248, 217, 255, 236, 227, 280, 251, 287, 281, 268, 246, 272, 235, 240, 266, 267, 280, 258, 269, 267, 269, 271, 270, 242, 264, 262, 237, 260, 226, 271, 251, 258, 237, 283, 262, 276, 272, 262, 236, 242, 249, 256, 251, 278, 278, 241, 259, 243, 279, 257, 267, 255, 279, 270, 279, 257, 221, 243, 262, 247, 264, 236, 268, 251, 247, 263, 258, 249, 228, 243, 235, 273, 274, 276, 258, 261, 241, 229, 261, 262, 270, 271, 252, 257, 278, 240, 248, 236, 273, 256, 244, 233, 267, 259, 269, 257, 246, 267, 259, 237, 257, 261, 262, 260, 259] payload_rows_per_dst=[5452, 5472, 5576, 5484, 5488, 5500, 5472, 5514] dup_rows_per_dst=[2538, 2704, 2792, 2684, 2852, 2582, 2754, 2672] combine_rows_per_src=[5452, 5472, 5576, 5484, 5488, 5500, 5472, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=226464 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156284064 ingress_bytes=193806336 link_bytes=193806336 token_bytes=312341664 weight_movement_bytes=37748736 total_nvlink_bytes=350090400 weight_to_token_ratio=0.120857 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856956953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38216 dup_rows=21794 dup_rows_remote=19204 dedup_ratio=0.332550 router_tokens_per_expert=[270, 243, 270, 245, 277, 264, 239, 248, 236, 247, 252, 253, 259, 244, 263, 234, 241, 244, 260, 259, 287, 274, 271, 260, 242, 276, 279, 245, 262, 272, 257, 260, 249, 271, 248, 252, 263, 254, 264, 256, 266, 270, 279, 268, 250, 238, 260, 291, 279, 251, 266, 268, 250, 237, 262, 247, 255, 254, 239, 245, 255, 286, 253, 261, 250, 260, 265, 218, 267, 280, 271, 237, 250, 261, 258, 233, 256, 254, 245, 253, 259, 252, 245, 245, 243, 255, 247, 254, 269, 237, 262, 288, 262, 227, 224, 240, 277, 251, 261, 261, 240, 261, 244, 246, 252, 259, 243, 264, 234, 239, 299, 270, 240, 250, 254, 261, 249, 267, 265, 251, 239, 249, 285, 263, 248, 251, 255, 253] payload_rows_per_dst=[5364, 5538, 5554, 5418, 5526, 5426, 5462, 5454] dup_rows_per_dst=[2724, 2840, 2804, 2798, 2590, 2592, 2740, 2706] combine_rows_per_src=[5364, 5538, 5554, 5418, 5526, 5426, 5462, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229008 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157302416 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314375824 weight_movement_bytes=18874368 total_nvlink_bytes=333250192 weight_to_token_ratio=0.060038 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229368 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157351928 ingress_bytes=213745664 link_bytes=213745664 token_bytes=314474488 weight_movement_bytes=56623104 total_nvlink_bytes=371097592 weight_to_token_ratio=0.180056 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048005783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157523968 dispatch_meta_bytes=229816 combine_payload_bytes=157523968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157753784 ingress_bytes=176398336 link_bytes=176398336 token_bytes=315277752 weight_movement_bytes=18874368 total_nvlink_bytes=334152120 weight_to_token_ratio=0.059866 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690162062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38304 dup_rows=21754 dup_rows_remote=19044 dedup_ratio=0.331940 router_tokens_per_expert=[259, 260, 261, 273, 240, 279, 246, 266, 267, 259, 261, 267, 273, 246, 266, 239, 260, 251, 267, 245, 251, 243, 263, 259, 248, 250, 248, 235, 264, 274, 274, 239, 252, 249, 276, 249, 233, 254, 251, 256, 282, 265, 246, 277, 244, 261, 259, 240, 267, 273, 255, 253, 223, 229, 278, 264, 239, 254, 265, 281, 256, 231, 246, 251, 262, 239, 276, 249, 232, 281, 257, 241, 229, 257, 240, 250, 244, 278, 277, 221, 269, 250, 250, 228, 230, 291, 305, 271, 259, 262, 221, 273, 258, 276, 257, 254, 252, 282, 234, 234, 275, 257, 260, 271, 248, 282, 247, 264, 227, 273, 237, 235, 219, 260, 265, 264, 260, 248, 261, 266, 223, 259, 252, 250, 298, 250, 256, 280] payload_rows_per_dst=[5498, 5518, 5478, 5418, 5406, 5490, 5450, 5524] dup_rows_per_dst=[2826, 2624, 2710, 2712, 2660, 2818, 2706, 2698] combine_rows_per_src=[5498, 5518, 5478, 5418, 5406, 5490, 5450, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43974 payload_rows_remote=38432 dup_rows=21562 dup_rows_remote=18722 dedup_ratio=0.329010 router_tokens_per_expert=[278, 242, 247, 241, 267, 252, 260, 247, 259, 252, 259, 265, 249, 239, 248, 260, 257, 242, 293, 260, 243, 255, 239, 265, 264, 273, 230, 277, 264, 249, 252, 236, 265, 265, 223, 266, 282, 281, 265, 254, 236, 248, 220, 252, 247, 263, 251, 243, 223, 237, 273, 247, 267, 259, 244, 240, 256, 236, 247, 268, 267, 247, 208, 245, 291, 235, 252, 266, 279, 239, 266, 246, 263, 264, 269, 232, 251, 239, 252, 264, 239, 265, 266, 254, 271, 254, 284, 237, 267, 244, 245, 247, 250, 249, 214, 289, 243, 267, 230, 283, 257, 279, 282, 255, 259, 297, 276, 252, 292, 247, 268, 251, 267, 237, 275, 265, 265, 245, 265, 265, 271, 259, 300, 258, 241, 249, 249, 247] payload_rows_per_dst=[5466, 5530, 5480, 5356, 5534, 5454, 5612, 5542] dup_rows_per_dst=[2650, 2680, 2578, 2572, 2682, 2696, 2864, 2840] combine_rows_per_src=[5466, 5530, 5480, 5356, 5534, 5454, 5612, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229680 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762416 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295152 weight_movement_bytes=18874368 total_nvlink_bytes=332169520 weight_to_token_ratio=0.060245 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38374 dup_rows=21734 dup_rows_remote=19190 dedup_ratio=0.331635 router_tokens_per_expert=[254, 254, 249, 249, 241, 260, 229, 281, 235, 260, 274, 256, 240, 270, 253, 275, 254, 257, 270, 287, 267, 248, 247, 284, 285, 229, 271, 241, 241, 265, 258, 245, 255, 265, 269, 247, 268, 226, 267, 249, 242, 248, 270, 288, 227, 269, 258, 266, 240, 249, 249, 271, 258, 248, 236, 233, 270, 254, 278, 247, 252, 276, 275, 255, 261, 264, 256, 260, 266, 254, 279, 244, 252, 239, 257, 269, 260, 262, 264, 258, 285, 248, 233, 243, 253, 266, 230, 283, 263, 291, 250, 260, 241, 265, 245, 225, 266, 261, 253, 233, 230, 275, 232, 275, 236, 267, 233, 261, 233, 230, 246, 255, 243, 242, 286, 245, 250, 278, 257, 268, 231, 271, 254, 254, 259, 246, 261, 277] payload_rows_per_dst=[5422, 5530, 5488, 5482, 5532, 5472, 5428, 5448] dup_rows_per_dst=[2738, 2768, 2740, 2700, 2758, 2690, 2544, 2796] combine_rows_per_src=[5422, 5530, 5488, 5482, 5532, 5472, 5428, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229392 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122576 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015760 weight_movement_bytes=37748736 total_nvlink_bytes=351764496 weight_to_token_ratio=0.120213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=228616 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157646088 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315063560 weight_movement_bytes=18874368 total_nvlink_bytes=333937928 weight_to_token_ratio=0.059907 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38216 dup_rows=21872 dup_rows_remote=19040 dedup_ratio=0.333740 router_tokens_per_expert=[240, 257, 277, 241, 266, 241, 278, 259, 255, 259, 265, 258, 241, 243, 293, 262, 265, 260, 258, 235, 281, 270, 248, 244, 261, 246, 289, 231, 264, 247, 251, 271, 233, 222, 273, 280, 237, 259, 232, 257, 229, 215, 295, 236, 278, 260, 280, 227, 259, 257, 236, 260, 244, 277, 251, 245, 229, 271, 256, 270, 281, 243, 279, 235, 268, 259, 274, 272, 267, 249, 259, 265, 229, 258, 229, 258, 274, 268, 243, 258, 272, 276, 284, 278, 246, 250, 272, 264, 245, 257, 264, 274, 237, 259, 272, 252, 245, 246, 236, 260, 227, 264, 249, 246, 236, 244, 261, 243, 276, 265, 254, 247, 253, 290, 229, 250, 274, 275, 241, 232, 258, 247, 251, 285, 268, 254, 253, 215] payload_rows_per_dst=[5448, 5670, 5620, 5278, 5542, 5578, 5276, 5252] dup_rows_per_dst=[2832, 3134, 2900, 2348, 2718, 3032, 2504, 2404] combine_rows_per_src=[5448, 5670, 5620, 5278, 5542, 5578, 5276, 5252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=230256 combine_payload_bytes=157179904 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157410160 ingress_bytes=194928640 link_bytes=194928640 token_bytes=314590064 weight_movement_bytes=37748736 total_nvlink_bytes=352338800 weight_to_token_ratio=0.119993 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867710318e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38342 dup_rows=21684 dup_rows_remote=18988 dedup_ratio=0.330872 router_tokens_per_expert=[239, 240, 259, 259, 258, 282, 274, 199, 242, 256, 272, 269, 253, 272, 263, 298, 279, 255, 271, 243, 243, 257, 243, 252, 296, 245, 254, 277, 324, 270, 249, 266, 256, 250, 257, 233, 252, 246, 256, 245, 252, 230, 251, 253, 255, 238, 235, 247, 240, 256, 259, 277, 290, 239, 256, 256, 257, 252, 248, 267, 257, 253, 234, 262, 249, 259, 269, 240, 235, 258, 268, 267, 238, 240, 273, 265, 273, 262, 251, 264, 251, 228, 257, 262, 252, 235, 254, 263, 266, 280, 250, 263, 243, 299, 260, 244, 256, 268, 255, 270, 274, 244, 265, 241, 252, 278, 244, 249, 225, 242, 281, 273, 216, 264, 249, 243, 278, 245, 264, 266, 236, 226, 267, 240, 265, 259, 250, 247] payload_rows_per_dst=[5528, 5598, 5354, 5510, 5506, 5436, 5494, 5426] dup_rows_per_dst=[2742, 2850, 2558, 2696, 2716, 2778, 2740, 2604] combine_rows_per_src=[5528, 5598, 5354, 5510, 5506, 5436, 5494, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38164 dup_rows=21742 dup_rows_remote=18966 dedup_ratio=0.331757 router_tokens_per_expert=[257, 271, 269, 256, 275, 239, 277, 248, 271, 249, 286, 250, 227, 241, 257, 253, 257, 264, 260, 262, 258, 235, 287, 264, 274, 297, 264, 270, 275, 260, 240, 236, 264, 247, 261, 255, 248, 240, 269, 256, 245, 257, 237, 264, 246, 270, 259, 270, 243, 248, 283, 240, 270, 264, 246, 249, 274, 263, 256, 229, 242, 252, 283, 216, 250, 240, 283, 235, 232, 270, 247, 259, 256, 246, 269, 255, 256, 254, 248, 256, 261, 230, 240, 237, 276, 247, 260, 248, 277, 252, 272, 233, 241, 262, 240, 265, 278, 248, 252, 233, 244, 278, 260, 250, 269, 258, 267, 252, 257, 276, 252, 245, 259, 250, 254, 257, 245, 254, 261, 272, 250, 265, 232, 244, 255, 249, 267, 263] payload_rows_per_dst=[5498, 5630, 5494, 5400, 5390, 5442, 5474, 5466] dup_rows_per_dst=[2754, 2776, 2682, 2716, 2722, 2640, 2764, 2688] combine_rows_per_src=[5498, 5630, 5494, 5400, 5390, 5442, 5474, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38322 dup_rows=21666 dup_rows_remote=18884 dedup_ratio=0.330597 router_tokens_per_expert=[238, 301, 241, 246, 252, 269, 286, 254, 252, 252, 249, 255, 222, 240, 241, 228, 251, 252, 248, 259, 236, 274, 286, 267, 243, 257, 256, 256, 242, 231, 248, 251, 245, 243, 276, 244, 264, 259, 238, 219, 232, 256, 255, 240, 264, 250, 215, 287, 245, 254, 250, 244, 260, 244, 259, 263, 266, 268, 261, 288, 234, 276, 273, 250, 244, 240, 269, 253, 298, 246, 271, 265, 234, 254, 250, 254, 248, 287, 258, 269, 272, 265, 259, 259, 275, 268, 249, 249, 245, 273, 246, 265, 273, 250, 256, 261, 248, 258, 271, 258, 248, 259, 264, 245, 261, 262, 265, 268, 264, 224, 267, 245, 254, 258, 285, 274, 268, 243, 226, 253, 250, 268, 268, 261, 277, 275, 252, 239] payload_rows_per_dst=[5436, 5484, 5360, 5506, 5502, 5548, 5498, 5536] dup_rows_per_dst=[2616, 2630, 2614, 2764, 2778, 2782, 2716, 2766] combine_rows_per_src=[5436, 5484, 5360, 5506, 5502, 5548, 5498, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38160 dup_rows=21844 dup_rows_remote=19008 dedup_ratio=0.333313 router_tokens_per_expert=[244, 267, 254, 273, 280, 271, 256, 253, 263, 217, 219, 275, 286, 242, 245, 250, 268, 263, 285, 211, 273, 236, 220, 267, 232, 223, 249, 266, 272, 256, 232, 268, 261, 259, 246, 260, 256, 236, 253, 231, 247, 257, 242, 265, 269, 269, 259, 294, 251, 230, 251, 260, 269, 271, 271, 262, 244, 247, 232, 278, 276, 245, 246, 256, 280, 271, 258, 256, 259, 260, 251, 257, 246, 263, 263, 306, 268, 247, 223, 276, 232, 240, 259, 240, 271, 247, 261, 280, 265, 276, 238, 270, 281, 248, 231, 263, 275, 246, 292, 272, 240, 223, 276, 280, 279, 229, 265, 265, 247, 249, 258, 251, 243, 263, 227, 244, 254, 220, 238, 255, 235, 257, 256, 290, 240, 256, 284, 264] payload_rows_per_dst=[5462, 5382, 5448, 5412, 5532, 5516, 5540, 5400] dup_rows_per_dst=[2728, 2660, 2760, 2766, 2836, 2688, 2754, 2652] combine_rows_per_src=[5462, 5382, 5448, 5412, 5532, 5516, 5540, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229024 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156761760 ingress_bytes=213155840 link_bytes=213155840 token_bytes=313294496 weight_movement_bytes=56623104 total_nvlink_bytes=369917600 weight_to_token_ratio=0.180734 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042354380e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229320 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157278152 ingress_bytes=213671936 link_bytes=213671936 token_bytes=314326984 weight_movement_bytes=56623104 total_nvlink_bytes=370950088 weight_to_token_ratio=0.180141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047299358e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228520 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548264 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868008 weight_movement_bytes=18874368 total_nvlink_bytes=331742376 weight_to_token_ratio=0.060327 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38214 dup_rows=21918 dup_rows_remote=19182 dedup_ratio=0.334442 router_tokens_per_expert=[240, 249, 224, 239, 255, 245, 272, 257, 260, 256, 266, 246, 241, 256, 245, 255, 262, 280, 273, 248, 235, 259, 259, 275, 259, 262, 247, 234, 254, 259, 255, 246, 224, 263, 244, 272, 227, 243, 250, 272, 264, 256, 283, 246, 243, 265, 246, 272, 258, 239, 233, 250, 234, 263, 252, 258, 254, 280, 252, 243, 239, 268, 251, 272, 251, 259, 258, 253, 267, 283, 252, 232, 259, 249, 272, 250, 263, 246, 268, 261, 254, 267, 260, 269, 289, 279, 251, 258, 237, 258, 253, 282, 278, 237, 233, 246, 234, 267, 249, 250, 260, 267, 249, 247, 220, 241, 260, 247, 262, 298, 261, 242, 243, 257, 265, 271, 244, 263, 251, 276, 272, 265, 269, 271, 271, 265, 245, 283] payload_rows_per_dst=[5414, 5402, 5404, 5414, 5500, 5484, 5396, 5604] dup_rows_per_dst=[2598, 2812, 2736, 2678, 2746, 2818, 2712, 2818] combine_rows_per_src=[5414, 5402, 5404, 5414, 5500, 5484, 5396, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=228824 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157195736 ingress_bytes=194715648 link_bytes=194715648 token_bytes=314162648 weight_movement_bytes=37748736 total_nvlink_bytes=351911384 weight_to_token_ratio=0.120157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865669533e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38202 dup_rows=21820 dup_rows_remote=19108 dedup_ratio=0.332947 router_tokens_per_expert=[272, 263, 284, 248, 256, 266, 262, 246, 243, 260, 268, 247, 272, 254, 289, 235, 263, 225, 233, 260, 250, 244, 250, 235, 249, 245, 224, 220, 241, 243, 267, 244, 268, 274, 276, 237, 260, 241, 277, 282, 243, 254, 268, 263, 260, 264, 245, 254, 242, 238, 273, 272, 258, 235, 259, 240, 234, 284, 262, 278, 264, 268, 231, 262, 249, 246, 270, 254, 252, 253, 245, 263, 260, 265, 255, 247, 252, 263, 274, 275, 258, 240, 245, 273, 266, 247, 249, 252, 279, 257, 259, 261, 263, 256, 257, 261, 277, 232, 250, 275, 266, 265, 257, 264, 269, 260, 235, 245, 263, 243, 253, 231, 256, 255, 264, 264, 282, 236, 268, 265, 272, 245, 249, 235, 257, 251, 262, 252] payload_rows_per_dst=[5550, 5262, 5484, 5306, 5468, 5604, 5528, 5514] dup_rows_per_dst=[2780, 2418, 2848, 2564, 2778, 2852, 2868, 2712] combine_rows_per_src=[5550, 5262, 5484, 5306, 5468, 5604, 5528, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228672 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156532032 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312835392 weight_movement_bytes=18874368 total_nvlink_bytes=331709760 weight_to_token_ratio=0.060333 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38224 dup_rows=21848 dup_rows_remote=19140 dedup_ratio=0.333374 router_tokens_per_expert=[258, 244, 235, 245, 225, 236, 250, 247, 240, 239, 245, 239, 266, 277, 258, 235, 266, 270, 247, 240, 268, 239, 242, 264, 262, 229, 237, 278, 286, 258, 249, 272, 269, 272, 266, 258, 259, 249, 254, 269, 293, 244, 244, 238, 257, 264, 265, 271, 266, 246, 244, 275, 242, 283, 259, 270, 252, 281, 255, 259, 271, 237, 237, 269, 243, 273, 257, 237, 248, 270, 259, 248, 248, 264, 259, 245, 257, 246, 232, 245, 275, 264, 258, 274, 249, 268, 225, 257, 263, 266, 282, 272, 254, 251, 262, 228, 254, 234, 248, 249, 259, 249, 264, 250, 250, 241, 252, 290, 251, 267, 272, 256, 255, 249, 258, 246, 223, 259, 277, 235, 287, 284, 251, 274, 249, 260, 263, 269] payload_rows_per_dst=[5378, 5442, 5494, 5464, 5452, 5448, 5464, 5546] dup_rows_per_dst=[2500, 2772, 2850, 2828, 2610, 2848, 2708, 2732] combine_rows_per_src=[5378, 5442, 5494, 5464, 5452, 5448, 5464, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229584 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156754128 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278672 weight_movement_bytes=18874368 total_nvlink_bytes=332153040 weight_to_token_ratio=0.060248 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229240 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156704632 ingress_bytes=194224128 link_bytes=194224128 token_bytes=313180024 weight_movement_bytes=37748736 total_nvlink_bytes=350928760 weight_to_token_ratio=0.120534 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860960030e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229456 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156794960 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360464 weight_movement_bytes=18874368 total_nvlink_bytes=332234832 weight_to_token_ratio=0.060232 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38380 dup_rows=21812 dup_rows_remote=19184 dedup_ratio=0.332825 router_tokens_per_expert=[258, 251, 250, 268, 248, 268, 262, 250, 275, 218, 258, 269, 244, 253, 249, 276, 285, 284, 259, 268, 229, 248, 274, 242, 250, 233, 261, 241, 281, 261, 273, 258, 271, 268, 247, 254, 255, 279, 244, 242, 261, 274, 240, 244, 220, 258, 259, 243, 275, 254, 239, 220, 277, 272, 245, 230, 236, 261, 248, 265, 229, 234, 246, 255, 250, 274, 222, 259, 260, 260, 253, 256, 278, 250, 259, 244, 267, 247, 241, 234, 262, 275, 275, 242, 237, 257, 268, 277, 255, 244, 256, 260, 245, 246, 266, 246, 286, 258, 274, 285, 237, 266, 276, 259, 275, 234, 270, 241, 256, 257, 261, 256, 286, 241, 261, 228, 231, 256, 254, 262, 285, 258, 241, 239, 292, 260, 277, 252] payload_rows_per_dst=[5532, 5468, 5454, 5344, 5382, 5516, 5562, 5466] dup_rows_per_dst=[2662, 2826, 2664, 2628, 2726, 2706, 2820, 2780] combine_rows_per_src=[5532, 5468, 5454, 5344, 5382, 5516, 5562, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38292 dup_rows=21848 dup_rows_remote=19110 dedup_ratio=0.333374 router_tokens_per_expert=[235, 257, 276, 269, 255, 270, 267, 282, 260, 265, 283, 255, 257, 254, 255, 253, 248, 265, 272, 263, 251, 222, 270, 249, 260, 246, 231, 259, 281, 240, 269, 241, 267, 247, 249, 243, 261, 254, 271, 254, 275, 260, 256, 252, 248, 262, 249, 246, 269, 258, 273, 247, 254, 278, 253, 235, 247, 272, 257, 246, 248, 258, 267, 226, 267, 254, 272, 264, 248, 270, 281, 281, 228, 272, 244, 247, 257, 288, 262, 237, 239, 251, 244, 238, 244, 258, 263, 226, 253, 295, 245, 249, 249, 254, 246, 240, 267, 256, 238, 255, 268, 266, 256, 258, 242, 240, 235, 284, 254, 261, 246, 250, 240, 256, 259, 216, 248, 275, 263, 250, 255, 255, 248, 265, 257, 234, 286, 277] payload_rows_per_dst=[5552, 5396, 5472, 5454, 5504, 5458, 5414, 5438] dup_rows_per_dst=[2834, 2738, 2716, 2722, 2840, 2530, 2738, 2730] combine_rows_per_src=[5552, 5396, 5472, 5454, 5504, 5458, 5414, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38126 dup_rows=21944 dup_rows_remote=19092 dedup_ratio=0.334839 router_tokens_per_expert=[264, 273, 249, 250, 250, 253, 264, 273, 263, 253, 253, 281, 280, 243, 261, 245, 242, 230, 274, 248, 238, 257, 257, 287, 233, 272, 245, 260, 266, 267, 266, 259, 232, 277, 246, 259, 261, 223, 266, 284, 232, 284, 252, 259, 255, 275, 232, 234, 245, 240, 298, 269, 270, 262, 230, 255, 243, 247, 247, 250, 242, 267, 273, 277, 270, 260, 240, 243, 277, 239, 238, 259, 247, 239, 217, 256, 244, 250, 250, 230, 258, 256, 231, 257, 247, 262, 264, 268, 268, 255, 271, 256, 287, 270, 276, 261, 268, 278, 271, 246, 235, 247, 251, 264, 274, 245, 257, 243, 267, 254, 268, 249, 272, 229, 243, 261, 243, 274, 280, 250, 256, 239, 226, 247, 250, 263, 259, 271] payload_rows_per_dst=[5466, 5446, 5486, 5438, 5468, 5338, 5460, 5490] dup_rows_per_dst=[2852, 2756, 2836, 2792, 2738, 2560, 2774, 2636] combine_rows_per_src=[5466, 5446, 5486, 5438, 5468, 5338, 5460, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38108 dup_rows=21890 dup_rows_remote=19220 dedup_ratio=0.334015 router_tokens_per_expert=[277, 232, 251, 260, 225, 262, 268, 249, 262, 237, 264, 237, 288, 237, 260, 274, 252, 250, 254, 235, 276, 255, 259, 231, 238, 269, 236, 255, 257, 269, 280, 251, 289, 270, 239, 252, 266, 232, 253, 247, 255, 249, 272, 258, 274, 245, 260, 258, 234, 286, 263, 244, 263, 252, 248, 282, 243, 252, 265, 271, 259, 267, 273, 263, 270, 270, 254, 275, 254, 232, 266, 261, 260, 217, 256, 271, 243, 256, 226, 258, 257, 293, 241, 238, 250, 245, 272, 267, 281, 280, 255, 234, 241, 243, 243, 264, 271, 261, 281, 253, 257, 249, 261, 245, 266, 242, 255, 276, 270, 257, 224, 241, 266, 261, 244, 268, 263, 263, 255, 288, 246, 242, 240, 221, 258, 234, 253, 250] payload_rows_per_dst=[5464, 5398, 5458, 5502, 5366, 5538, 5506, 5414] dup_rows_per_dst=[2702, 2736, 2780, 2828, 2772, 2670, 2712, 2690] combine_rows_per_src=[5464, 5398, 5458, 5502, 5366, 5538, 5506, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38356 dup_rows=21722 dup_rows_remote=18944 dedup_ratio=0.331451 router_tokens_per_expert=[224, 249, 262, 257, 267, 235, 272, 257, 262, 283, 233, 263, 254, 237, 239, 246, 243, 260, 260, 247, 252, 270, 250, 244, 284, 251, 281, 250, 259, 267, 278, 270, 268, 256, 252, 247, 249, 270, 262, 248, 249, 252, 243, 251, 261, 245, 247, 252, 293, 275, 242, 236, 269, 265, 252, 255, 260, 261, 260, 265, 265, 237, 276, 275, 270, 248, 271, 268, 256, 246, 262, 241, 254, 224, 255, 276, 234, 263, 298, 252, 261, 236, 273, 241, 237, 273, 253, 245, 288, 223, 234, 260, 235, 274, 245, 260, 217, 245, 251, 263, 262, 268, 237, 277, 266, 258, 274, 241, 242, 246, 259, 250, 263, 253, 282, 247, 247, 254, 254, 264, 265, 252, 261, 243, 254, 257, 255, 261] payload_rows_per_dst=[5438, 5542, 5390, 5550, 5458, 5454, 5460, 5522] dup_rows_per_dst=[2642, 2790, 2714, 2822, 2778, 2622, 2652, 2702] combine_rows_per_src=[5438, 5542, 5390, 5550, 5458, 5454, 5460, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=230256 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157434736 ingress_bytes=194953216 link_bytes=194953216 token_bytes=314639216 weight_movement_bytes=37748736 total_nvlink_bytes=352387952 weight_to_token_ratio=0.119975 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867945793e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229608 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157073640 ingress_bytes=194592768 link_bytes=194592768 token_bytes=313917672 weight_movement_bytes=37748736 total_nvlink_bytes=351666408 weight_to_token_ratio=0.120250 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864492157e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228872 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156392968 ingress_bytes=231661568 link_bytes=231661568 token_bytes=312557064 weight_movement_bytes=75497472 total_nvlink_bytes=388054536 weight_to_token_ratio=0.241548 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219667160e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38300 dup_rows=21810 dup_rows_remote=19176 dedup_ratio=0.332794 router_tokens_per_expert=[246, 283, 267, 242, 247, 279, 233, 268, 223, 268, 234, 237, 276, 252, 238, 232, 243, 268, 261, 257, 234, 242, 250, 282, 272, 251, 255, 259, 264, 258, 240, 252, 228, 248, 268, 275, 246, 250, 223, 272, 249, 227, 269, 263, 246, 258, 247, 261, 236, 260, 241, 264, 265, 290, 255, 252, 251, 256, 269, 255, 250, 254, 269, 267, 260, 248, 272, 218, 279, 270, 260, 281, 258, 293, 268, 240, 270, 254, 235, 285, 238, 239, 232, 257, 260, 264, 242, 281, 240, 264, 279, 277, 253, 247, 283, 259, 245, 262, 246, 287, 252, 277, 284, 259, 273, 243, 263, 263, 275, 262, 257, 235, 237, 253, 224, 262, 266, 253, 256, 221, 272, 238, 271, 223, 256, 242, 266, 262] payload_rows_per_dst=[5350, 5494, 5426, 5534, 5534, 5528, 5510, 5350] dup_rows_per_dst=[2700, 2682, 2634, 2734, 2848, 2702, 2856, 2654] combine_rows_per_src=[5350, 5494, 5426, 5534, 5534, 5528, 5510, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38104 dup_rows=21842 dup_rows_remote=18836 dedup_ratio=0.333282 router_tokens_per_expert=[217, 278, 263, 245, 263, 256, 252, 265, 238, 265, 274, 265, 241, 244, 253, 259, 254, 242, 249, 284, 228, 248, 253, 216, 267, 245, 262, 246, 221, 274, 262, 278, 291, 249, 258, 264, 242, 251, 263, 243, 248, 269, 259, 251, 245, 253, 241, 265, 252, 267, 295, 257, 240, 272, 276, 289, 242, 271, 266, 287, 244, 250, 275, 261, 271, 266, 266, 274, 259, 258, 260, 254, 274, 261, 261, 273, 277, 232, 246, 253, 265, 229, 245, 243, 272, 271, 282, 244, 255, 246, 269, 247, 238, 272, 264, 248, 242, 250, 256, 261, 255, 240, 287, 264, 240, 264, 236, 227, 256, 256, 254, 236, 249, 269, 251, 255, 282, 233, 262, 221, 231, 272, 267, 237, 242, 267, 233, 255] payload_rows_per_dst=[5444, 5392, 5424, 5398, 5584, 5402, 5460, 5590] dup_rows_per_dst=[2712, 2666, 2760, 2546, 2784, 2778, 2590, 3006] combine_rows_per_src=[5444, 5392, 5424, 5398, 5584, 5402, 5460, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=229312 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156319680 ingress_bytes=174964736 link_bytes=174964736 token_bytes=312410048 weight_movement_bytes=18874368 total_nvlink_bytes=331284416 weight_to_token_ratio=0.060415 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676426012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229200 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157335376 ingress_bytes=213729280 link_bytes=213729280 token_bytes=314441552 weight_movement_bytes=56623104 total_nvlink_bytes=371064656 weight_to_token_ratio=0.180075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047848800e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38126 dup_rows=21944 dup_rows_remote=19184 dedup_ratio=0.334839 router_tokens_per_expert=[265, 264, 274, 260, 231, 242, 237, 253, 261, 271, 235, 259, 248, 231, 220, 226, 255, 237, 253, 241, 264, 259, 248, 269, 254, 253, 264, 246, 260, 273, 261, 232, 255, 270, 271, 257, 272, 256, 268, 240, 283, 258, 248, 280, 250, 238, 238, 246, 276, 273, 267, 283, 270, 264, 267, 252, 227, 240, 267, 257, 254, 255, 229, 252, 257, 267, 244, 284, 235, 263, 287, 259, 255, 288, 253, 269, 235, 268, 245, 267, 244, 263, 257, 267, 258, 269, 244, 249, 258, 273, 268, 266, 256, 252, 230, 279, 254, 240, 259, 257, 279, 257, 245, 239, 262, 261, 282, 249, 263, 283, 243, 240, 263, 241, 233, 233, 239, 239, 252, 253, 270, 260, 240, 263, 261, 239, 260, 291] payload_rows_per_dst=[5368, 5412, 5448, 5452, 5530, 5482, 5466, 5434] dup_rows_per_dst=[2586, 2726, 2812, 2724, 2822, 2784, 2760, 2730] combine_rows_per_src=[5368, 5412, 5448, 5452, 5530, 5482, 5466, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229904 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106704 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983504 weight_movement_bytes=18874368 total_nvlink_bytes=332857872 weight_to_token_ratio=0.060113 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=227760 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156301744 ingress_bytes=174948352 link_bytes=174948352 token_bytes=312375728 weight_movement_bytes=18874368 total_nvlink_bytes=331250096 weight_to_token_ratio=0.060422 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676269029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229240 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156393336 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312557432 weight_movement_bytes=37748736 total_nvlink_bytes=350306168 weight_to_token_ratio=0.120774 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38410 dup_rows=21626 dup_rows_remote=18976 dedup_ratio=0.329987 router_tokens_per_expert=[254, 289, 266, 247, 276, 249, 235, 248, 223, 256, 265, 265, 255, 246, 237, 254, 265, 271, 241, 233, 252, 233, 274, 282, 282, 244, 263, 232, 267, 259, 244, 235, 272, 266, 237, 277, 280, 236, 244, 252, 244, 260, 252, 270, 292, 271, 235, 241, 223, 280, 240, 265, 265, 236, 239, 263, 257, 262, 277, 247, 285, 238, 247, 251, 279, 262, 271, 234, 233, 286, 253, 266, 233, 243, 250, 279, 250, 289, 301, 271, 265, 243, 256, 270, 223, 251, 245, 262, 257, 242, 236, 248, 234, 253, 269, 234, 280, 249, 263, 249, 233, 263, 252, 254, 246, 264, 272, 252, 252, 217, 276, 272, 285, 248, 255, 279, 238, 243, 250, 264, 270, 254, 257, 249, 249, 259, 294, 246] payload_rows_per_dst=[5464, 5486, 5462, 5500, 5596, 5410, 5468, 5524] dup_rows_per_dst=[2666, 2668, 2796, 2650, 2804, 2566, 2720, 2756] combine_rows_per_src=[5464, 5486, 5462, 5500, 5596, 5410, 5468, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38134 dup_rows=21926 dup_rows_remote=19234 dedup_ratio=0.334564 router_tokens_per_expert=[271, 277, 271, 268, 265, 238, 256, 286, 273, 258, 252, 245, 265, 244, 255, 270, 252, 256, 246, 256, 256, 248, 272, 238, 274, 243, 256, 273, 283, 229, 249, 253, 245, 265, 238, 272, 234, 249, 272, 250, 248, 242, 276, 249, 230, 232, 224, 280, 273, 254, 269, 251, 237, 253, 245, 267, 260, 289, 256, 259, 265, 249, 266, 263, 264, 237, 248, 227, 262, 242, 226, 253, 259, 245, 278, 254, 248, 268, 256, 247, 275, 255, 265, 245, 265, 265, 266, 260, 240, 231, 251, 259, 294, 243, 300, 269, 265, 244, 255, 248, 249, 266, 242, 269, 247, 247, 263, 256, 263, 276, 263, 245, 255, 261, 254, 252, 266, 268, 242, 237, 226, 285, 228, 262, 279, 229, 234, 255] payload_rows_per_dst=[5474, 5476, 5400, 5474, 5384, 5542, 5426, 5434] dup_rows_per_dst=[2914, 2692, 2612, 2838, 2644, 2824, 2770, 2632] combine_rows_per_src=[5474, 5476, 5400, 5474, 5384, 5542, 5426, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38182 dup_rows=21774 dup_rows_remote=18936 dedup_ratio=0.332245 router_tokens_per_expert=[271, 269, 262, 266, 252, 264, 252, 253, 249, 277, 244, 280, 252, 247, 210, 233, 274, 284, 279, 257, 214, 277, 275, 246, 261, 258, 237, 266, 271, 271, 241, 276, 270, 284, 254, 219, 243, 252, 254, 259, 254, 262, 265, 265, 265, 249, 249, 253, 254, 285, 265, 245, 251, 260, 268, 256, 249, 253, 236, 276, 256, 232, 275, 254, 228, 242, 253, 254, 239, 283, 237, 262, 249, 243, 231, 275, 229, 292, 261, 265, 233, 243, 275, 244, 228, 275, 283, 272, 244, 281, 248, 265, 264, 255, 250, 230, 246, 249, 267, 269, 261, 257, 270, 245, 262, 239, 252, 272, 265, 281, 233, 260, 243, 252, 265, 234, 231, 260, 258, 263, 243, 231, 250, 250, 258, 246, 257, 286] payload_rows_per_dst=[5580, 5490, 5420, 5504, 5478, 5484, 5440, 5366] dup_rows_per_dst=[2838, 2892, 2552, 2684, 2608, 2696, 2816, 2688] combine_rows_per_src=[5580, 5490, 5420, 5504, 5478, 5484, 5440, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38388 dup_rows=21616 dup_rows_remote=18876 dedup_ratio=0.329834 router_tokens_per_expert=[267, 264, 265, 250, 268, 228, 253, 268, 234, 258, 255, 257, 299, 280, 217, 254, 254, 286, 245, 247, 255, 273, 270, 251, 246, 268, 265, 239, 254, 233, 259, 267, 262, 243, 276, 261, 252, 242, 273, 245, 260, 283, 277, 263, 264, 264, 245, 242, 253, 258, 231, 255, 249, 292, 253, 274, 260, 249, 271, 292, 237, 250, 244, 261, 245, 276, 259, 272, 260, 238, 237, 245, 236, 248, 268, 267, 254, 277, 283, 271, 246, 239, 275, 272, 268, 249, 248, 239, 246, 252, 263, 248, 233, 241, 248, 251, 215, 251, 265, 286, 261, 261, 285, 265, 245, 265, 238, 245, 260, 257, 235, 249, 258, 290, 237, 240, 264, 269, 248, 240, 223, 251, 250, 247, 247, 256, 247, 254] payload_rows_per_dst=[5492, 5534, 5562, 5484, 5532, 5420, 5496, 5400] dup_rows_per_dst=[2742, 2690, 2742, 2774, 2740, 2616, 2670, 2642] combine_rows_per_src=[5492, 5534, 5562, 5484, 5532, 5420, 5496, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38330 dup_rows=21752 dup_rows_remote=19076 dedup_ratio=0.331909 router_tokens_per_expert=[264, 268, 255, 259, 260, 249, 263, 223, 250, 279, 274, 246, 259, 263, 245, 248, 251, 272, 235, 259, 256, 241, 243, 264, 276, 265, 264, 262, 240, 272, 296, 250, 261, 243, 247, 262, 261, 272, 230, 255, 227, 279, 271, 244, 228, 275, 238, 252, 226, 252, 265, 259, 259, 278, 252, 244, 271, 254, 268, 268, 285, 254, 250, 255, 256, 254, 264, 257, 256, 263, 269, 259, 247, 239, 241, 249, 231, 266, 244, 299, 251, 248, 240, 266, 260, 255, 266, 242, 231, 261, 243, 254, 253, 270, 270, 255, 243, 234, 245, 283, 256, 262, 257, 290, 291, 241, 253, 242, 224, 254, 240, 248, 256, 266, 264, 286, 257, 237, 256, 246, 255, 263, 267, 216, 268, 273, 249, 251] payload_rows_per_dst=[5476, 5462, 5444, 5526, 5474, 5454, 5452, 5496] dup_rows_per_dst=[2734, 2830, 2646, 2754, 2714, 2676, 2674, 2724] combine_rows_per_src=[5476, 5462, 5444, 5526, 5474, 5454, 5452, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=229472 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156426336 ingress_bytes=175071232 link_bytes=175071232 token_bytes=312623200 weight_movement_bytes=18874368 total_nvlink_bytes=331497568 weight_to_token_ratio=0.060374 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677446405e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229544 combine_payload_bytes=157327360 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157556904 ingress_bytes=232824832 link_bytes=232824832 token_bytes=314884264 weight_movement_bytes=75497472 total_nvlink_bytes=390381736 weight_to_token_ratio=0.239763 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230812983e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228472 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156621944 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015416 weight_movement_bytes=18874368 total_nvlink_bytes=331889784 weight_to_token_ratio=0.060299 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38444 dup_rows=21676 dup_rows_remote=18914 dedup_ratio=0.330750 router_tokens_per_expert=[295, 245, 266, 276, 232, 284, 256, 260, 258, 248, 259, 241, 263, 246, 264, 253, 231, 268, 249, 267, 241, 245, 261, 268, 258, 281, 257, 299, 276, 273, 250, 244, 269, 284, 247, 235, 242, 252, 270, 260, 218, 245, 269, 234, 270, 276, 278, 240, 251, 280, 261, 244, 243, 242, 278, 266, 262, 266, 283, 269, 257, 273, 245, 220, 282, 269, 259, 242, 243, 256, 241, 245, 243, 273, 254, 281, 257, 229, 279, 253, 243, 249, 267, 270, 234, 239, 250, 232, 252, 257, 264, 236, 227, 248, 255, 243, 232, 237, 252, 263, 260, 238, 261, 249, 279, 255, 255, 250, 251, 237, 254, 265, 248, 225, 238, 273, 262, 256, 273, 258, 267, 252, 241, 222, 275, 254, 288, 283] payload_rows_per_dst=[5590, 5516, 5416, 5562, 5478, 5304, 5480, 5514] dup_rows_per_dst=[2702, 2820, 2762, 2718, 2734, 2628, 2596, 2716] combine_rows_per_src=[5590, 5516, 5416, 5562, 5478, 5304, 5480, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38142 dup_rows=21828 dup_rows_remote=19080 dedup_ratio=0.333069 router_tokens_per_expert=[280, 236, 258, 245, 261, 250, 261, 246, 254, 240, 256, 267, 248, 254, 256, 265, 237, 245, 268, 242, 256, 266, 265, 262, 252, 264, 281, 264, 250, 244, 256, 256, 226, 273, 262, 255, 270, 273, 246, 235, 253, 247, 248, 263, 265, 238, 265, 268, 239, 255, 257, 244, 283, 261, 280, 258, 247, 262, 261, 262, 244, 277, 261, 253, 251, 250, 253, 276, 269, 265, 229, 248, 262, 249, 259, 251, 219, 232, 265, 301, 261, 257, 270, 259, 260, 270, 243, 236, 241, 261, 269, 244, 283, 240, 248, 268, 273, 264, 248, 265, 257, 249, 246, 246, 257, 243, 251, 286, 252, 247, 244, 266, 255, 248, 256, 290, 242, 244, 264, 256, 237, 246, 251, 227, 269, 273, 245, 266] payload_rows_per_dst=[5516, 5196, 5434, 5462, 5348, 5694, 5492, 5566] dup_rows_per_dst=[2638, 2474, 2740, 2620, 2644, 3072, 2892, 2748] combine_rows_per_src=[5516, 5196, 5434, 5462, 5348, 5694, 5492, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229056 combine_payload_bytes=157237248 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157466304 ingress_bytes=232734720 link_bytes=232734720 token_bytes=314703552 weight_movement_bytes=75497472 total_nvlink_bytes=390201024 weight_to_token_ratio=0.239900 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229949574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38416 dup_rows=21648 dup_rows_remote=18848 dedup_ratio=0.330322 router_tokens_per_expert=[267, 250, 264, 270, 256, 247, 238, 251, 274, 235, 264, 262, 225, 264, 254, 254, 258, 293, 280, 269, 256, 275, 267, 272, 274, 246, 286, 214, 236, 233, 247, 270, 240, 252, 252, 258, 259, 271, 250, 263, 203, 263, 240, 265, 246, 259, 257, 255, 280, 254, 216, 256, 267, 254, 238, 242, 272, 256, 284, 224, 244, 271, 263, 245, 265, 249, 253, 261, 256, 288, 257, 264, 280, 245, 246, 267, 294, 229, 239, 248, 278, 281, 260, 224, 259, 253, 226, 229, 259, 225, 264, 243, 279, 247, 255, 263, 276, 246, 257, 224, 263, 271, 248, 272, 266, 257, 277, 258, 252, 268, 265, 236, 254, 235, 247, 236, 278, 249, 263, 245, 247, 273, 258, 255, 246, 265, 290, 255] payload_rows_per_dst=[5488, 5576, 5446, 5420, 5506, 5420, 5472, 5560] dup_rows_per_dst=[2662, 2776, 2620, 2712, 2776, 2670, 2800, 2632] combine_rows_per_src=[5488, 5576, 5446, 5420, 5506, 5420, 5472, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229624 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=0 egress_bytes=157229304 ingress_bytes=156999680 link_bytes=157229304 token_bytes=314228984 weight_movement_bytes=0 total_nvlink_bytes=314228984 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.506493829e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229432 combine_payload_bytes=157466624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157696056 ingress_bytes=176340992 link_bytes=176340992 token_bytes=315162680 weight_movement_bytes=18874368 total_nvlink_bytes=334037048 weight_to_token_ratio=0.059888 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689612620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156229632 dispatch_meta_bytes=228888 combine_payload_bytes=156229632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156458520 ingress_bytes=175104000 link_bytes=175104000 token_bytes=312688152 weight_movement_bytes=18874368 total_nvlink_bytes=331562520 weight_to_token_ratio=0.060362 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677760371e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229056 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157580992 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314932928 weight_movement_bytes=18874368 total_nvlink_bytes=333807296 weight_to_token_ratio=0.059931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38294 dup_rows=21794 dup_rows_remote=19012 dedup_ratio=0.332550 router_tokens_per_expert=[261, 257, 274, 285, 253, 254, 263, 260, 226, 234, 278, 251, 235, 240, 242, 270, 264, 246, 265, 253, 233, 260, 261, 277, 249, 259, 285, 251, 270, 258, 236, 248, 254, 223, 241, 273, 262, 245, 244, 248, 245, 270, 262, 287, 272, 245, 270, 280, 270, 253, 272, 250, 246, 271, 256, 232, 222, 262, 210, 264, 251, 242, 272, 266, 286, 269, 249, 249, 226, 266, 264, 230, 278, 267, 268, 264, 252, 260, 276, 267, 279, 261, 267, 254, 243, 260, 252, 261, 224, 229, 226, 260, 249, 262, 284, 255, 264, 216, 281, 262, 244, 272, 263, 270, 252, 235, 260, 262, 239, 244, 261, 247, 276, 232, 231, 258, 267, 248, 246, 277, 257, 276, 240, 267, 260, 251, 259, 256] payload_rows_per_dst=[5414, 5448, 5514, 5424, 5590, 5454, 5434, 5464] dup_rows_per_dst=[2752, 2782, 2728, 2654, 2752, 2678, 2710, 2738] combine_rows_per_src=[5414, 5448, 5514, 5424, 5590, 5454, 5434, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38118 dup_rows=21976 dup_rows_remote=19222 dedup_ratio=0.335327 router_tokens_per_expert=[259, 278, 245, 239, 240, 266, 232, 238, 250, 252, 265, 243, 227, 256, 253, 261, 271, 266, 248, 264, 264, 254, 267, 257, 272, 248, 267, 254, 263, 279, 274, 248, 253, 267, 258, 276, 260, 245, 228, 230, 247, 254, 261, 262, 271, 248, 272, 247, 239, 262, 235, 280, 252, 270, 259, 275, 244, 252, 237, 299, 249, 222, 269, 254, 254, 273, 248, 238, 246, 219, 303, 258, 250, 272, 284, 263, 283, 230, 241, 251, 258, 283, 270, 264, 236, 259, 248, 257, 236, 243, 234, 261, 243, 269, 272, 233, 271, 256, 272, 273, 260, 257, 278, 259, 263, 268, 260, 281, 223, 238, 288, 266, 237, 239, 245, 253, 239, 263, 254, 245, 250, 270, 258, 245, 260, 248, 237, 256] payload_rows_per_dst=[5382, 5504, 5494, 5442, 5414, 5408, 5554, 5362] dup_rows_per_dst=[2626, 2888, 2664, 2754, 2812, 2724, 2872, 2636] combine_rows_per_src=[5382, 5504, 5494, 5442, 5414, 5408, 5554, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38164 dup_rows=21844 dup_rows_remote=19120 dedup_ratio=0.333313 router_tokens_per_expert=[259, 259, 270, 223, 252, 265, 253, 247, 241, 277, 255, 260, 259, 259, 262, 285, 269, 241, 269, 245, 257, 266, 233, 254, 248, 234, 254, 271, 266, 247, 256, 248, 252, 255, 252, 278, 267, 280, 274, 245, 251, 282, 249, 279, 264, 237, 281, 275, 241, 233, 283, 258, 262, 236, 262, 248, 263, 287, 253, 251, 226, 281, 256, 279, 230, 260, 264, 270, 233, 274, 253, 269, 233, 257, 245, 236, 277, 248, 269, 247, 270, 261, 252, 260, 253, 275, 238, 243, 269, 250, 262, 259, 267, 250, 264, 224, 279, 276, 245, 249, 259, 256, 246, 231, 240, 241, 265, 241, 251, 255, 268, 266, 220, 277, 243, 271, 245, 225, 266, 258, 236, 249, 284, 261, 264, 252, 211, 252] payload_rows_per_dst=[5528, 5492, 5522, 5552, 5388, 5388, 5436, 5386] dup_rows_per_dst=[2724, 2624, 2920, 2896, 2764, 2668, 2606, 2642] combine_rows_per_src=[5528, 5492, 5522, 5552, 5388, 5388, 5436, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38394 dup_rows=21756 dup_rows_remote=19086 dedup_ratio=0.331970 router_tokens_per_expert=[237, 257, 233, 270, 290, 242, 235, 257, 241, 231, 254, 259, 265, 229, 241, 228, 242, 281, 247, 268, 251, 260, 253, 262, 266, 261, 289, 232, 273, 271, 228, 273, 257, 267, 261, 267, 265, 223, 279, 267, 244, 257, 263, 251, 271, 252, 246, 250, 246, 264, 261, 249, 258, 263, 252, 252, 259, 279, 255, 265, 259, 263, 271, 246, 244, 246, 269, 240, 253, 253, 224, 273, 277, 248, 246, 254, 251, 250, 234, 266, 255, 255, 227, 262, 240, 280, 252, 273, 268, 267, 254, 258, 270, 258, 247, 272, 259, 253, 234, 288, 253, 287, 255, 278, 264, 237, 261, 246, 246, 274, 266, 260, 235, 274, 266, 213, 256, 224, 257, 247, 250, 282, 268, 255, 240, 267, 252, 267] payload_rows_per_dst=[5322, 5566, 5430, 5516, 5386, 5574, 5584, 5402] dup_rows_per_dst=[2616, 2748, 2810, 2768, 2670, 2702, 2738, 2704] combine_rows_per_src=[5322, 5566, 5430, 5516, 5386, 5574, 5584, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38132 dup_rows=21920 dup_rows_remote=19274 dedup_ratio=0.334473 router_tokens_per_expert=[262, 235, 270, 243, 275, 247, 252, 239, 263, 249, 259, 293, 248, 262, 259, 261, 276, 287, 290, 234, 264, 230, 262, 260, 275, 267, 258, 265, 254, 278, 248, 252, 257, 251, 270, 248, 289, 237, 287, 288, 230, 245, 257, 228, 251, 233, 244, 232, 268, 245, 249, 274, 263, 271, 251, 254, 247, 240, 261, 253, 238, 286, 258, 258, 258, 249, 241, 241, 279, 233, 250, 265, 253, 265, 265, 238, 259, 252, 234, 269, 243, 280, 231, 242, 295, 274, 237, 254, 238, 265, 236, 260, 260, 262, 240, 248, 261, 239, 260, 235, 246, 248, 252, 252, 257, 263, 265, 258, 257, 260, 270, 269, 273, 250, 269, 259, 259, 249, 253, 261, 254, 263, 266, 251, 250, 230, 242, 251] payload_rows_per_dst=[5468, 5460, 5452, 5442, 5420, 5484, 5432, 5458] dup_rows_per_dst=[2766, 2940, 2642, 2790, 2682, 2646, 2752, 2702] combine_rows_per_src=[5468, 5460, 5452, 5442, 5420, 5484, 5432, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229224 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081448 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933672 weight_movement_bytes=37748736 total_nvlink_bytes=351682408 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229360 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156360688 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312492016 weight_movement_bytes=18874368 total_nvlink_bytes=331366384 weight_to_token_ratio=0.060400 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38360 dup_rows=21730 dup_rows_remote=19030 dedup_ratio=0.331573 router_tokens_per_expert=[252, 271, 264, 279, 241, 225, 236, 252, 247, 258, 304, 263, 268, 247, 260, 259, 262, 240, 274, 235, 254, 231, 252, 280, 218, 227, 249, 287, 254, 262, 269, 269, 241, 262, 256, 262, 226, 259, 290, 266, 253, 248, 270, 222, 249, 261, 239, 269, 256, 250, 264, 261, 231, 236, 253, 269, 241, 262, 274, 261, 228, 249, 277, 249, 223, 274, 258, 259, 250, 269, 264, 277, 284, 235, 258, 241, 239, 264, 251, 252, 248, 270, 262, 246, 252, 254, 249, 255, 255, 279, 264, 251, 255, 233, 267, 259, 236, 248, 257, 230, 256, 268, 281, 277, 294, 264, 256, 260, 234, 238, 247, 242, 237, 264, 273, 263, 271, 256, 261, 286, 265, 257, 271, 259, 241, 256, 260, 240] payload_rows_per_dst=[5538, 5488, 5446, 5480, 5434, 5486, 5434, 5500] dup_rows_per_dst=[2714, 2638, 2700, 2642, 2762, 2712, 2742, 2820] combine_rows_per_src=[5538, 5488, 5446, 5480, 5434, 5486, 5434, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229136 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156548880 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868624 weight_movement_bytes=37748736 total_nvlink_bytes=350617360 weight_to_token_ratio=0.120654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38396 dup_rows=21660 dup_rows_remote=18880 dedup_ratio=0.330505 router_tokens_per_expert=[247, 258, 248, 228, 249, 233, 256, 249, 265, 242, 273, 251, 282, 260, 258, 241, 294, 246, 256, 244, 244, 246, 252, 262, 267, 263, 233, 260, 246, 253, 251, 251, 282, 265, 240, 244, 256, 276, 281, 282, 238, 273, 252, 267, 269, 268, 244, 255, 256, 256, 246, 252, 269, 233, 237, 253, 255, 243, 246, 255, 240, 245, 256, 256, 248, 239, 230, 253, 255, 276, 238, 276, 256, 250, 291, 263, 260, 275, 272, 234, 286, 279, 288, 263, 250, 246, 232, 262, 281, 250, 235, 273, 228, 222, 247, 277, 263, 227, 258, 246, 276, 269, 269, 257, 243, 254, 258, 250, 289, 243, 286, 242, 274, 237, 283, 231, 238, 249, 266, 275, 268, 255, 258, 264, 282, 241, 253, 231] payload_rows_per_dst=[5416, 5488, 5562, 5426, 5512, 5504, 5480, 5488] dup_rows_per_dst=[2664, 2648, 2822, 2570, 2720, 2734, 2780, 2722] combine_rows_per_src=[5416, 5488, 5562, 5426, 5512, 5504, 5480, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38262 dup_rows=21876 dup_rows_remote=19140 dedup_ratio=0.333801 router_tokens_per_expert=[252, 242, 238, 308, 246, 256, 243, 235, 253, 284, 234, 244, 265, 270, 274, 244, 249, 269, 248, 243, 265, 259, 233, 227, 249, 263, 269, 264, 254, 235, 278, 263, 250, 262, 240, 271, 263, 243, 254, 211, 270, 257, 258, 257, 253, 247, 250, 230, 248, 279, 244, 273, 252, 243, 262, 259, 278, 268, 255, 249, 220, 247, 253, 266, 247, 261, 237, 259, 283, 274, 265, 283, 256, 262, 255, 281, 252, 245, 227, 274, 280, 244, 241, 262, 269, 266, 268, 270, 240, 267, 265, 264, 270, 225, 246, 255, 244, 260, 265, 235, 272, 275, 286, 269, 272, 258, 254, 256, 229, 246, 239, 281, 275, 253, 236, 252, 232, 234, 261, 261, 245, 282, 245, 282, 275, 245, 246, 242] payload_rows_per_dst=[5468, 5468, 5392, 5490, 5502, 5452, 5490, 5398] dup_rows_per_dst=[2708, 2656, 2638, 2702, 2832, 2812, 2792, 2736] combine_rows_per_src=[5468, 5468, 5392, 5490, 5502, 5452, 5490, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229920 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157491744 ingress_bytes=213884928 link_bytes=213884928 token_bytes=314753568 weight_movement_bytes=56623104 total_nvlink_bytes=371376672 weight_to_token_ratio=0.179897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049340142e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229624 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156418296 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312606968 weight_movement_bytes=18874368 total_nvlink_bytes=331481336 weight_to_token_ratio=0.060377 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229560 combine_payload_bytes=157122560 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157352120 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474680 weight_movement_bytes=18874368 total_nvlink_bytes=333349048 weight_to_token_ratio=0.060019 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229104 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157499120 ingress_bytes=232767488 link_bytes=232767488 token_bytes=314769136 weight_movement_bytes=75497472 total_nvlink_bytes=390266608 weight_to_token_ratio=0.239850 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.230263541e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229608 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156950760 ingress_bytes=213344256 link_bytes=213344256 token_bytes=313671912 weight_movement_bytes=56623104 total_nvlink_bytes=370295016 weight_to_token_ratio=0.180517 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044159689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38196 dup_rows=21790 dup_rows_remote=19022 dedup_ratio=0.332489 router_tokens_per_expert=[256, 238, 255, 272, 235, 252, 266, 267, 236, 248, 264, 254, 258, 273, 242, 260, 285, 269, 251, 259, 268, 267, 267, 267, 239, 257, 259, 260, 238, 253, 269, 251, 248, 286, 279, 286, 275, 229, 305, 249, 233, 249, 255, 264, 239, 235, 252, 252, 271, 263, 251, 284, 252, 258, 260, 252, 270, 251, 250, 232, 237, 239, 248, 271, 231, 268, 245, 258, 254, 275, 261, 263, 276, 246, 253, 255, 268, 260, 251, 253, 244, 261, 251, 262, 250, 249, 249, 246, 261, 254, 257, 244, 292, 231, 258, 253, 270, 237, 235, 253, 240, 242, 259, 261, 278, 265, 243, 257, 261, 261, 258, 254, 233, 250, 269, 254, 259, 258, 253, 260, 268, 241, 226, 254, 262, 270, 228, 270] payload_rows_per_dst=[5394, 5550, 5486, 5504, 5486, 5452, 5430, 5444] dup_rows_per_dst=[2758, 2768, 2786, 2674, 2748, 2672, 2718, 2666] combine_rows_per_src=[5394, 5550, 5486, 5504, 5486, 5452, 5430, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43936 payload_rows_remote=38468 dup_rows=21600 dup_rows_remote=18996 dedup_ratio=0.329590 router_tokens_per_expert=[234, 249, 270, 281, 238, 222, 298, 252, 282, 256, 249, 293, 271, 238, 251, 263, 269, 253, 251, 306, 239, 240, 262, 251, 241, 282, 245, 240, 280, 235, 239, 256, 277, 231, 270, 244, 258, 255, 263, 252, 254, 269, 273, 230, 236, 244, 255, 260, 264, 288, 260, 270, 278, 235, 220, 262, 236, 242, 242, 236, 246, 222, 269, 266, 227, 262, 232, 232, 257, 240, 260, 263, 266, 286, 240, 254, 251, 273, 235, 268, 256, 250, 266, 241, 257, 248, 259, 259, 275, 262, 253, 247, 251, 254, 254, 225, 255, 261, 265, 262, 253, 222, 234, 282, 266, 290, 265, 251, 241, 247, 250, 257, 264, 226, 276, 272, 278, 244, 291, 246, 242, 246, 242, 287, 256, 273, 288, 290] payload_rows_per_dst=[5496, 5442, 5520, 5468, 5500, 5406, 5516, 5588] dup_rows_per_dst=[2798, 2736, 2622, 2604, 2592, 2708, 2686, 2854] combine_rows_per_src=[5496, 5442, 5520, 5468, 5500, 5406, 5516, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38296 dup_rows=21670 dup_rows_remote=18826 dedup_ratio=0.330658 router_tokens_per_expert=[257, 252, 270, 280, 272, 264, 264, 255, 244, 284, 234, 252, 280, 269, 245, 276, 287, 260, 238, 230, 273, 274, 261, 271, 256, 243, 274, 273, 285, 253, 269, 241, 229, 260, 261, 262, 259, 250, 254, 224, 248, 284, 254, 267, 267, 285, 251, 253, 252, 253, 248, 260, 269, 247, 262, 263, 260, 258, 247, 263, 284, 220, 247, 266, 272, 264, 273, 280, 273, 234, 270, 227, 242, 253, 250, 228, 247, 259, 252, 236, 248, 225, 227, 294, 280, 240, 250, 249, 241, 252, 216, 247, 256, 276, 254, 243, 254, 281, 241, 244, 249, 245, 240, 264, 248, 257, 266, 256, 258, 278, 252, 239, 252, 259, 267, 258, 230, 250, 249, 260, 256, 245, 271, 252, 253, 241, 263, 239] payload_rows_per_dst=[5570, 5582, 5540, 5494, 5372, 5464, 5456, 5388] dup_rows_per_dst=[2844, 2798, 2738, 2704, 2670, 2532, 2688, 2696] combine_rows_per_src=[5570, 5582, 5540, 5494, 5372, 5464, 5456, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38296 dup_rows=21934 dup_rows_remote=19354 dedup_ratio=0.334686 router_tokens_per_expert=[274, 254, 255, 252, 263, 279, 257, 288, 247, 276, 241, 284, 276, 228, 264, 256, 255, 264, 234, 271, 258, 263, 258, 250, 243, 238, 250, 278, 249, 258, 245, 259, 249, 256, 255, 280, 237, 263, 267, 257, 246, 281, 250, 249, 256, 252, 235, 255, 255, 252, 261, 245, 258, 265, 253, 269, 244, 282, 260, 257, 269, 230, 263, 251, 268, 231, 263, 241, 245, 233, 259, 262, 222, 221, 228, 244, 251, 243, 267, 265, 250, 252, 250, 253, 258, 270, 263, 260, 257, 243, 242, 262, 255, 230, 270, 265, 259, 274, 259, 235, 263, 262, 252, 266, 259, 256, 260, 251, 269, 241, 253, 271, 270, 255, 250, 254, 269, 266, 259, 244, 240, 260, 255, 285, 259, 288, 247, 245] payload_rows_per_dst=[5596, 5430, 5460, 5402, 5306, 5468, 5432, 5508] dup_rows_per_dst=[2792, 2716, 2716, 2826, 2580, 2692, 2828, 2784] combine_rows_per_src=[5596, 5430, 5460, 5402, 5306, 5468, 5432, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228872 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679688 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130504 weight_movement_bytes=37748736 total_nvlink_bytes=350879240 weight_to_token_ratio=0.120553 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=229856 combine_payload_bytes=157564928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157794784 ingress_bytes=176439296 link_bytes=176439296 token_bytes=315359712 weight_movement_bytes=18874368 total_nvlink_bytes=334234080 weight_to_token_ratio=0.059850 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690554521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38300 dup_rows=21846 dup_rows_remote=19192 dedup_ratio=0.333344 router_tokens_per_expert=[261, 236, 226, 243, 279, 250, 230, 243, 280, 246, 258, 240, 278, 278, 231, 250, 260, 262, 258, 264, 258, 233, 282, 263, 262, 260, 254, 223, 254, 281, 249, 243, 262, 247, 253, 253, 235, 271, 247, 294, 266, 256, 265, 241, 232, 243, 238, 274, 233, 264, 266, 270, 234, 279, 252, 239, 275, 282, 269, 266, 274, 237, 270, 251, 242, 242, 255, 284, 266, 272, 252, 238, 277, 244, 244, 268, 273, 231, 249, 236, 277, 245, 256, 248, 265, 234, 255, 252, 249, 258, 265, 217, 254, 239, 253, 255, 250, 274, 280, 225, 255, 265, 255, 259, 290, 267, 274, 273, 253, 254, 257, 249, 271, 255, 255, 270, 238, 214, 247, 264, 258, 246, 267, 259, 261, 272, 279, 264] payload_rows_per_dst=[5416, 5442, 5432, 5482, 5488, 5390, 5510, 5530] dup_rows_per_dst=[2642, 2770, 2722, 2840, 2658, 2654, 2850, 2710] combine_rows_per_src=[5416, 5442, 5432, 5482, 5488, 5390, 5510, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38326 dup_rows=21762 dup_rows_remote=19142 dedup_ratio=0.332062 router_tokens_per_expert=[240, 251, 243, 239, 253, 267, 251, 254, 241, 267, 254, 245, 260, 261, 263, 276, 261, 270, 225, 262, 248, 244, 261, 254, 222, 229, 263, 280, 250, 273, 254, 276, 255, 256, 264, 264, 268, 219, 243, 267, 249, 264, 264, 267, 239, 264, 231, 256, 249, 259, 270, 283, 267, 266, 263, 261, 262, 241, 287, 237, 266, 244, 277, 259, 271, 233, 221, 244, 230, 253, 266, 272, 277, 252, 240, 242, 277, 261, 262, 271, 234, 281, 235, 273, 252, 244, 269, 273, 283, 265, 246, 268, 279, 272, 243, 256, 258, 247, 235, 250, 234, 256, 238, 260, 231, 244, 253, 273, 286, 272, 230, 267, 261, 286, 265, 263, 247, 259, 251, 247, 235, 259, 262, 259, 267, 246, 254, 230] payload_rows_per_dst=[5448, 5474, 5440, 5578, 5384, 5490, 5448, 5512] dup_rows_per_dst=[2682, 2670, 2700, 2804, 2760, 2856, 2620, 2670] combine_rows_per_src=[5448, 5474, 5440, 5578, 5384, 5490, 5448, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38276 dup_rows=21836 dup_rows_remote=19052 dedup_ratio=0.333191 router_tokens_per_expert=[288, 240, 254, 237, 283, 255, 250, 267, 253, 226, 260, 279, 258, 263, 251, 250, 276, 262, 275, 264, 237, 232, 249, 245, 257, 273, 243, 261, 256, 289, 254, 253, 290, 258, 246, 233, 277, 253, 272, 257, 261, 263, 241, 244, 267, 250, 247, 245, 253, 257, 245, 262, 240, 280, 258, 257, 250, 226, 242, 279, 250, 250, 265, 240, 256, 249, 265, 261, 274, 246, 249, 240, 256, 260, 249, 251, 240, 253, 249, 234, 280, 261, 256, 241, 246, 244, 264, 277, 249, 233, 241, 251, 296, 256, 267, 260, 278, 238, 270, 254, 269, 261, 249, 243, 268, 249, 255, 241, 234, 254, 265, 259, 254, 259, 263, 245, 242, 262, 265, 255, 242, 263, 260, 268, 254, 250, 286, 261] payload_rows_per_dst=[5468, 5472, 5424, 5436, 5442, 5468, 5422, 5568] dup_rows_per_dst=[2760, 2780, 2784, 2672, 2622, 2776, 2752, 2690] combine_rows_per_src=[5468, 5472, 5424, 5436, 5442, 5468, 5422, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228488 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157088904 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949320 weight_movement_bytes=18874368 total_nvlink_bytes=332823688 weight_to_token_ratio=0.060119 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38190 dup_rows=21970 dup_rows_remote=19220 dedup_ratio=0.335236 router_tokens_per_expert=[261, 257, 257, 216, 250, 259, 266, 254, 247, 270, 238, 259, 236, 252, 239, 248, 257, 284, 254, 265, 272, 249, 288, 256, 240, 242, 250, 242, 261, 278, 229, 254, 275, 252, 248, 259, 287, 242, 266, 250, 258, 266, 275, 277, 249, 226, 253, 260, 238, 252, 250, 257, 239, 261, 283, 235, 236, 242, 248, 251, 274, 240, 264, 274, 252, 254, 253, 272, 279, 272, 257, 235, 251, 234, 235, 242, 238, 253, 290, 260, 228, 285, 279, 262, 283, 268, 261, 261, 255, 260, 249, 257, 245, 273, 244, 250, 258, 260, 264, 280, 249, 251, 236, 263, 258, 240, 286, 293, 265, 239, 260, 250, 257, 270, 221, 259, 257, 250, 267, 271, 252, 265, 286, 243, 256, 243, 238, 227] payload_rows_per_dst=[5252, 5422, 5516, 5466, 5488, 5468, 5578, 5376] dup_rows_per_dst=[2410, 2820, 2770, 2622, 2666, 2850, 3082, 2750] combine_rows_per_src=[5252, 5422, 5516, 5466, 5488, 5468, 5578, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=230600 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157091016 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313951432 weight_movement_bytes=18874368 total_nvlink_bytes=332825800 weight_to_token_ratio=0.060119 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229968 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106768 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983568 weight_movement_bytes=18874368 total_nvlink_bytes=332857936 weight_to_token_ratio=0.060113 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229872 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213168 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196464 weight_movement_bytes=18874368 total_nvlink_bytes=333070832 weight_to_token_ratio=0.060072 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229312 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007808 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786304 weight_movement_bytes=37748736 total_nvlink_bytes=351535040 weight_to_token_ratio=0.120301 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229640 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156655880 ingress_bytes=213049344 link_bytes=213049344 token_bytes=313082120 weight_movement_bytes=56623104 total_nvlink_bytes=369705224 weight_to_token_ratio=0.180857 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041333987e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38274 dup_rows=21794 dup_rows_remote=19072 dedup_ratio=0.332550 router_tokens_per_expert=[273, 263, 284, 270, 257, 274, 236, 227, 268, 231, 233, 281, 277, 253, 246, 245, 253, 261, 244, 276, 270, 270, 253, 263, 257, 216, 268, 247, 257, 230, 270, 260, 252, 259, 268, 263, 287, 276, 253, 249, 251, 266, 255, 267, 248, 230, 257, 257, 261, 249, 243, 253, 277, 275, 235, 261, 246, 266, 259, 268, 248, 250, 250, 249, 223, 252, 235, 239, 241, 265, 246, 248, 237, 263, 235, 273, 267, 254, 268, 294, 244, 245, 245, 271, 276, 250, 268, 246, 265, 241, 253, 254, 260, 249, 276, 279, 260, 246, 226, 248, 236, 275, 230, 252, 265, 261, 262, 233, 252, 248, 239, 285, 262, 246, 250, 255, 255, 277, 263, 255, 260, 259, 243, 258, 267, 254, 268, 275] payload_rows_per_dst=[5510, 5468, 5422, 5446, 5408, 5484, 5422, 5582] dup_rows_per_dst=[2726, 2722, 2854, 2734, 2672, 2760, 2614, 2712] combine_rows_per_src=[5510, 5468, 5422, 5446, 5408, 5484, 5422, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38184 dup_rows=21954 dup_rows_remote=19342 dedup_ratio=0.334991 router_tokens_per_expert=[257, 253, 273, 253, 252, 279, 227, 261, 249, 264, 265, 245, 265, 258, 252, 235, 265, 260, 273, 253, 265, 249, 253, 281, 268, 258, 250, 303, 254, 274, 275, 257, 252, 258, 243, 231, 283, 260, 241, 277, 252, 268, 242, 240, 265, 270, 252, 242, 235, 256, 270, 254, 255, 237, 258, 262, 243, 258, 279, 229, 212, 288, 247, 222, 267, 262, 244, 241, 249, 238, 225, 237, 257, 244, 232, 245, 251, 253, 270, 251, 255, 239, 270, 285, 267, 234, 273, 244, 262, 247, 257, 279, 261, 256, 256, 232, 273, 274, 276, 240, 264, 244, 227, 262, 257, 273, 256, 249, 269, 278, 232, 260, 265, 249, 269, 275, 240, 281, 242, 235, 259, 256, 273, 250, 285, 251, 263, 251] payload_rows_per_dst=[5468, 5538, 5364, 5398, 5416, 5480, 5450, 5468] dup_rows_per_dst=[2708, 2938, 2788, 2612, 2516, 2754, 2818, 2820] combine_rows_per_src=[5468, 5538, 5364, 5398, 5416, 5480, 5450, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38158 dup_rows=21918 dup_rows_remote=19028 dedup_ratio=0.334442 router_tokens_per_expert=[272, 268, 265, 234, 245, 274, 274, 235, 251, 230, 277, 266, 265, 261, 250, 269, 281, 228, 262, 250, 240, 252, 257, 234, 233, 269, 225, 238, 243, 235, 248, 264, 251, 268, 239, 242, 256, 269, 272, 286, 272, 269, 257, 259, 280, 272, 239, 265, 255, 254, 252, 257, 287, 232, 287, 279, 251, 284, 240, 263, 243, 278, 253, 221, 256, 269, 259, 228, 272, 269, 261, 242, 279, 254, 268, 256, 249, 247, 263, 266, 249, 259, 258, 272, 232, 247, 259, 249, 275, 249, 222, 280, 246, 247, 242, 267, 239, 255, 280, 266, 248, 244, 260, 233, 262, 245, 236, 248, 267, 263, 253, 252, 295, 247, 259, 239, 280, 251, 281, 261, 267, 247, 229, 270, 245, 232, 253, 243] payload_rows_per_dst=[5460, 5408, 5558, 5530, 5288, 5416, 5434, 5524] dup_rows_per_dst=[2890, 2778, 2950, 2796, 2550, 2690, 2590, 2674] combine_rows_per_src=[5460, 5408, 5558, 5530, 5288, 5416, 5434, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229384 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999688 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769992 weight_movement_bytes=18874368 total_nvlink_bytes=332644360 weight_to_token_ratio=0.060154 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=230104 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156631768 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313033432 weight_movement_bytes=18874368 total_nvlink_bytes=331907800 weight_to_token_ratio=0.060295 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38344 dup_rows=21670 dup_rows_remote=18888 dedup_ratio=0.330658 router_tokens_per_expert=[254, 270, 287, 237, 270, 260, 222, 243, 247, 257, 267, 278, 265, 255, 243, 257, 286, 268, 253, 278, 260, 240, 239, 218, 253, 272, 257, 246, 252, 233, 245, 247, 228, 257, 274, 257, 269, 262, 216, 262, 238, 304, 229, 255, 228, 269, 241, 263, 235, 263, 259, 228, 260, 237, 247, 268, 271, 273, 269, 293, 247, 243, 262, 236, 238, 269, 263, 252, 275, 247, 275, 249, 296, 255, 245, 276, 266, 253, 238, 255, 222, 269, 239, 229, 271, 279, 262, 226, 245, 237, 267, 254, 293, 245, 220, 260, 236, 209, 270, 264, 282, 264, 235, 245, 235, 274, 278, 248, 253, 235, 277, 251, 244, 273, 283, 270, 233, 278, 255, 252, 264, 280, 284, 281, 300, 240, 268, 235] payload_rows_per_dst=[5506, 5456, 5482, 5476, 5522, 5412, 5402, 5610] dup_rows_per_dst=[2718, 2638, 2622, 2706, 2782, 2624, 2710, 2870] combine_rows_per_src=[5506, 5456, 5482, 5476, 5522, 5412, 5402, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38156 dup_rows=21940 dup_rows_remote=19280 dedup_ratio=0.334778 router_tokens_per_expert=[270, 281, 274, 281, 271, 263, 276, 226, 270, 253, 258, 268, 247, 266, 273, 255, 251, 258, 252, 217, 266, 256, 275, 247, 234, 248, 281, 243, 279, 250, 261, 257, 238, 250, 276, 263, 263, 250, 254, 276, 249, 243, 277, 226, 246, 237, 236, 262, 253, 251, 212, 257, 265, 245, 275, 269, 249, 242, 260, 239, 282, 266, 253, 255, 271, 254, 257, 267, 268, 258, 271, 259, 226, 261, 236, 243, 263, 273, 240, 287, 267, 253, 265, 254, 267, 258, 255, 239, 241, 264, 214, 246, 250, 252, 253, 272, 265, 237, 244, 264, 274, 244, 276, 256, 249, 241, 251, 319, 286, 240, 249, 241, 249, 251, 245, 236, 234, 254, 265, 238, 253, 259, 259, 262, 260, 238, 263, 256] payload_rows_per_dst=[5574, 5430, 5330, 5502, 5492, 5440, 5468, 5360] dup_rows_per_dst=[2890, 2720, 2762, 2644, 2776, 2660, 2804, 2684] combine_rows_per_src=[5574, 5430, 5330, 5502, 5492, 5440, 5468, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38272 dup_rows=21806 dup_rows_remote=19058 dedup_ratio=0.332733 router_tokens_per_expert=[225, 266, 246, 250, 266, 237, 233, 256, 242, 263, 243, 258, 259, 255, 287, 237, 261, 248, 250, 258, 253, 248, 265, 254, 236, 275, 257, 252, 265, 283, 250, 242, 253, 286, 263, 227, 249, 259, 262, 271, 260, 273, 248, 257, 259, 253, 280, 250, 248, 262, 263, 242, 230, 260, 227, 262, 280, 255, 255, 245, 258, 223, 253, 264, 266, 280, 253, 271, 280, 260, 251, 255, 249, 264, 276, 252, 254, 259, 276, 255, 253, 245, 234, 272, 272, 264, 241, 258, 237, 248, 266, 254, 248, 235, 251, 268, 242, 262, 263, 290, 273, 238, 265, 261, 242, 253, 244, 261, 242, 254, 266, 247, 237, 261, 251, 258, 238, 272, 256, 258, 254, 242, 258, 271, 276, 251, 287, 251] payload_rows_per_dst=[5404, 5516, 5528, 5398, 5528, 5414, 5458, 5484] dup_rows_per_dst=[2642, 2678, 2772, 2656, 2874, 2678, 2748, 2758] combine_rows_per_src=[5404, 5516, 5528, 5398, 5528, 5414, 5458, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38372 dup_rows=21670 dup_rows_remote=18890 dedup_ratio=0.330658 router_tokens_per_expert=[280, 251, 268, 262, 243, 259, 247, 264, 258, 253, 254, 259, 260, 267, 232, 265, 274, 264, 260, 259, 245, 259, 279, 231, 242, 274, 247, 231, 236, 270, 267, 262, 269, 254, 247, 265, 289, 280, 228, 283, 266, 280, 265, 226, 258, 236, 246, 245, 261, 266, 280, 256, 283, 263, 228, 252, 248, 254, 260, 259, 257, 234, 228, 228, 256, 259, 260, 271, 283, 246, 274, 238, 257, 257, 260, 271, 256, 243, 242, 274, 267, 231, 218, 245, 269, 272, 282, 252, 248, 259, 256, 251, 247, 254, 256, 240, 242, 285, 261, 250, 250, 228, 279, 241, 256, 240, 258, 286, 242, 264, 262, 248, 266, 260, 250, 234, 275, 235, 253, 245, 239, 240, 269, 281, 253, 254, 247, 265] payload_rows_per_dst=[5456, 5522, 5494, 5478, 5526, 5442, 5532, 5416] dup_rows_per_dst=[2788, 2678, 2780, 2636, 2768, 2652, 2652, 2716] combine_rows_per_src=[5456, 5522, 5494, 5478, 5526, 5442, 5532, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=228744 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156523912 ingress_bytes=175169536 link_bytes=175169536 token_bytes=312819080 weight_movement_bytes=18874368 total_nvlink_bytes=331693448 weight_to_token_ratio=0.060336 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678388305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44036 payload_rows_remote=38558 dup_rows=21500 dup_rows_remote=18816 dedup_ratio=0.328064 router_tokens_per_expert=[262, 252, 244, 256, 283, 241, 240, 249, 270, 247, 269, 255, 237, 240, 251, 256, 285, 266, 264, 228, 264, 227, 266, 285, 259, 265, 247, 276, 222, 251, 259, 266, 256, 289, 251, 251, 244, 262, 281, 243, 253, 235, 233, 220, 278, 291, 253, 272, 245, 286, 274, 266, 275, 233, 266, 261, 259, 251, 252, 277, 258, 249, 262, 231, 226, 270, 264, 282, 225, 262, 251, 255, 276, 239, 266, 255, 251, 251, 242, 250, 268, 232, 232, 241, 270, 267, 258, 246, 238, 248, 252, 242, 273, 259, 271, 243, 258, 253, 271, 252, 260, 246, 252, 275, 250, 290, 270, 246, 258, 273, 236, 262, 255, 237, 249, 259, 271, 273, 258, 235, 246, 267, 242, 270, 280, 246, 220, 264] payload_rows_per_dst=[5484, 5466, 5608, 5522, 5424, 5434, 5620, 5478] dup_rows_per_dst=[2562, 2626, 2824, 2762, 2564, 2646, 2832, 2684] combine_rows_per_src=[5484, 5466, 5608, 5522, 5424, 5434, 5620, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=228928 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157285952 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314342976 weight_movement_bytes=18874368 total_nvlink_bytes=333217344 weight_to_token_ratio=0.060044 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229744 combine_payload_bytes=156286976 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156516720 ingress_bytes=212910080 link_bytes=212910080 token_bytes=312803696 weight_movement_bytes=56623104 total_nvlink_bytes=369426800 weight_to_token_ratio=0.181018 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039999628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229320 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991432 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753544 weight_movement_bytes=18874368 total_nvlink_bytes=332627912 weight_to_token_ratio=0.060157 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229048 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157400760 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572472 weight_movement_bytes=18874368 total_nvlink_bytes=333446840 weight_to_token_ratio=0.060000 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157933568 dispatch_meta_bytes=229496 combine_payload_bytes=157933568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158163064 ingress_bytes=195682304 link_bytes=195682304 token_bytes=316096632 weight_movement_bytes=37748736 total_nvlink_bytes=353845368 weight_to_token_ratio=0.119422 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874931555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38192 dup_rows=21866 dup_rows_remote=19076 dedup_ratio=0.333649 router_tokens_per_expert=[239, 255, 241, 255, 252, 288, 252, 247, 245, 258, 253, 268, 254, 245, 240, 259, 289, 228, 242, 254, 250, 267, 279, 257, 246, 257, 245, 247, 251, 250, 234, 265, 281, 246, 250, 247, 263, 274, 253, 265, 260, 295, 272, 235, 220, 261, 241, 251, 270, 240, 256, 248, 252, 273, 255, 240, 242, 291, 292, 231, 262, 242, 273, 267, 255, 254, 255, 265, 264, 250, 292, 251, 257, 263, 277, 251, 257, 258, 244, 292, 263, 281, 240, 289, 272, 253, 233, 268, 260, 254, 242, 263, 245, 246, 249, 256, 247, 258, 254, 253, 246, 238, 225, 242, 239, 257, 274, 268, 235, 266, 271, 249, 268, 245, 275, 252, 249, 268, 261, 273, 237, 254, 241, 241, 253, 248, 260, 262] payload_rows_per_dst=[5388, 5520, 5442, 5478, 5562, 5496, 5366, 5418] dup_rows_per_dst=[2714, 2602, 2786, 2790, 2808, 2732, 2678, 2756] combine_rows_per_src=[5388, 5520, 5442, 5478, 5562, 5496, 5366, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38354 dup_rows=21682 dup_rows_remote=18960 dedup_ratio=0.330841 router_tokens_per_expert=[259, 288, 247, 283, 267, 241, 278, 254, 256, 307, 259, 224, 249, 241, 253, 263, 265, 290, 256, 259, 263, 236, 264, 248, 230, 247, 251, 260, 260, 237, 286, 259, 256, 256, 260, 244, 267, 238, 230, 243, 285, 240, 233, 259, 244, 242, 250, 278, 266, 281, 292, 257, 258, 278, 253, 257, 279, 232, 232, 287, 229, 253, 265, 256, 259, 263, 226, 249, 240, 241, 257, 266, 257, 241, 240, 249, 261, 295, 236, 247, 257, 248, 264, 268, 217, 263, 270, 256, 288, 244, 270, 251, 261, 269, 267, 231, 242, 278, 269, 263, 270, 273, 238, 277, 253, 261, 205, 247, 258, 239, 226, 233, 272, 272, 254, 262, 251, 250, 239, 275, 258, 263, 257, 250, 247, 265, 261, 229] payload_rows_per_dst=[5602, 5500, 5390, 5530, 5444, 5522, 5442, 5424] dup_rows_per_dst=[2736, 2722, 2660, 2820, 2610, 2726, 2622, 2786] combine_rows_per_src=[5602, 5500, 5390, 5530, 5444, 5522, 5442, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229072 combine_payload_bytes=156434432 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156663504 ingress_bytes=213057536 link_bytes=213057536 token_bytes=313097936 weight_movement_bytes=56623104 total_nvlink_bytes=369721040 weight_to_token_ratio=0.180848 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041412479e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229256 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157327240 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314425224 weight_movement_bytes=37748736 total_nvlink_bytes=352173960 weight_to_token_ratio=0.120056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38174 dup_rows=21838 dup_rows_remote=18890 dedup_ratio=0.333221 router_tokens_per_expert=[248, 250, 265, 260, 274, 254, 259, 261, 244, 227, 238, 256, 249, 257, 267, 257, 282, 272, 227, 249, 256, 256, 267, 232, 245, 259, 255, 249, 246, 257, 230, 255, 249, 247, 232, 242, 259, 270, 262, 259, 275, 245, 267, 253, 261, 251, 293, 226, 292, 274, 267, 266, 250, 253, 252, 255, 281, 232, 292, 267, 251, 251, 250, 270, 268, 268, 265, 256, 282, 266, 259, 248, 263, 261, 249, 277, 274, 289, 250, 270, 276, 248, 244, 259, 261, 265, 261, 255, 280, 253, 241, 298, 255, 240, 225, 256, 218, 249, 267, 234, 243, 262, 272, 227, 269, 238, 241, 261, 249, 236, 265, 252, 243, 263, 254, 275, 264, 249, 256, 233, 250, 219, 251, 248, 228, 270, 276, 247] payload_rows_per_dst=[5524, 5446, 5524, 5512, 5492, 5482, 5336, 5382] dup_rows_per_dst=[2948, 2634, 2760, 2786, 2658, 2752, 2630, 2670] combine_rows_per_src=[5524, 5446, 5524, 5512, 5492, 5482, 5336, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38196 dup_rows=21828 dup_rows_remote=19160 dedup_ratio=0.333069 router_tokens_per_expert=[272, 266, 268, 272, 240, 294, 259, 231, 247, 229, 261, 245, 274, 262, 257, 263, 233, 254, 278, 260, 271, 255, 240, 230, 272, 255, 228, 269, 242, 247, 301, 239, 248, 240, 260, 220, 267, 247, 262, 233, 287, 249, 261, 236, 253, 271, 237, 258, 254, 223, 271, 262, 271, 251, 249, 260, 251, 265, 220, 275, 274, 257, 270, 286, 268, 238, 249, 262, 265, 266, 249, 253, 257, 260, 265, 232, 229, 279, 235, 283, 258, 272, 263, 273, 277, 241, 256, 231, 238, 247, 268, 232, 251, 270, 263, 267, 245, 257, 266, 263, 252, 242, 257, 259, 249, 273, 249, 267, 258, 239, 243, 259, 275, 245, 238, 269, 273, 249, 272, 257, 272, 257, 254, 264, 252, 243, 249, 242] payload_rows_per_dst=[5492, 5446, 5406, 5472, 5512, 5436, 5442, 5502] dup_rows_per_dst=[2788, 2702, 2652, 2806, 2668, 2778, 2714, 2720] combine_rows_per_src=[5492, 5446, 5406, 5472, 5512, 5436, 5442, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38254 dup_rows=21890 dup_rows_remote=19190 dedup_ratio=0.334015 router_tokens_per_expert=[260, 253, 252, 242, 277, 258, 285, 246, 252, 270, 278, 269, 283, 239, 262, 235, 242, 249, 232, 255, 260, 280, 247, 255, 252, 248, 245, 244, 262, 256, 249, 244, 256, 238, 290, 258, 257, 244, 232, 280, 241, 261, 257, 266, 242, 261, 244, 264, 268, 252, 253, 265, 270, 254, 257, 270, 252, 260, 235, 263, 265, 232, 265, 252, 237, 267, 244, 269, 266, 265, 264, 271, 249, 275, 274, 235, 219, 260, 244, 279, 255, 242, 251, 248, 253, 276, 247, 228, 263, 246, 272, 266, 254, 255, 258, 232, 262, 252, 275, 243, 267, 264, 247, 233, 244, 252, 228, 287, 248, 272, 282, 269, 250, 276, 269, 258, 251, 260, 258, 246, 248, 246, 266, 254, 266, 238, 234, 274] payload_rows_per_dst=[5506, 5348, 5482, 5468, 5498, 5392, 5476, 5476] dup_rows_per_dst=[2816, 2692, 2700, 2758, 2738, 2700, 2774, 2712] combine_rows_per_src=[5506, 5348, 5482, 5468, 5498, 5392, 5476, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38064 dup_rows=21872 dup_rows_remote=19086 dedup_ratio=0.333740 router_tokens_per_expert=[248, 260, 289, 253, 264, 273, 260, 261, 265, 224, 273, 226, 243, 233, 276, 277, 209, 260, 251, 246, 275, 268, 229, 256, 253, 255, 245, 246, 257, 282, 262, 263, 296, 242, 270, 236, 255, 260, 214, 226, 252, 240, 270, 237, 237, 247, 238, 234, 268, 243, 284, 286, 237, 236, 263, 271, 227, 245, 287, 231, 273, 254, 248, 258, 267, 255, 264, 229, 257, 273, 270, 263, 259, 260, 263, 250, 276, 245, 220, 261, 254, 247, 252, 287, 274, 273, 249, 252, 267, 257, 276, 276, 272, 251, 257, 237, 268, 265, 282, 271, 282, 247, 249, 269, 257, 240, 263, 256, 239, 296, 243, 266, 287, 226, 214, 249, 275, 257, 256, 233, 263, 249, 270, 271, 245, 248, 236, 256] payload_rows_per_dst=[5488, 5458, 5350, 5426, 5458, 5478, 5600, 5406] dup_rows_per_dst=[2762, 2656, 2558, 2796, 2766, 2884, 2786, 2664] combine_rows_per_src=[5488, 5458, 5350, 5426, 5458, 5478, 5600, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38102 dup_rows=21900 dup_rows_remote=19198 dedup_ratio=0.334167 router_tokens_per_expert=[263, 243, 269, 267, 238, 243, 253, 270, 223, 221, 266, 271, 249, 282, 262, 229, 264, 254, 264, 262, 256, 272, 252, 242, 264, 255, 271, 260, 246, 274, 234, 251, 255, 251, 288, 253, 265, 237, 223, 276, 235, 250, 242, 265, 272, 260, 287, 259, 251, 264, 243, 264, 277, 230, 258, 260, 254, 256, 272, 235, 216, 254, 228, 264, 259, 251, 267, 244, 264, 274, 230, 260, 245, 266, 267, 252, 269, 261, 263, 239, 239, 232, 250, 232, 252, 277, 280, 241, 251, 274, 229, 238, 267, 278, 245, 242, 261, 227, 272, 264, 258, 280, 255, 264, 250, 263, 275, 250, 285, 251, 234, 261, 240, 259, 265, 260, 277, 234, 248, 269, 306, 240, 247, 283, 267, 249, 256, 266] payload_rows_per_dst=[5426, 5478, 5534, 5374, 5444, 5376, 5504, 5500] dup_rows_per_dst=[2672, 2764, 2702, 2678, 2778, 2678, 2796, 2832] combine_rows_per_src=[5426, 5478, 5534, 5374, 5444, 5376, 5504, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38208 dup_rows=21838 dup_rows_remote=19080 dedup_ratio=0.333221 router_tokens_per_expert=[274, 256, 247, 232, 244, 279, 283, 232, 270, 242, 244, 262, 254, 249, 247, 248, 259, 250, 267, 249, 254, 263, 247, 239, 254, 245, 270, 236, 259, 275, 279, 250, 270, 245, 271, 256, 249, 256, 285, 276, 260, 276, 247, 254, 253, 260, 248, 260, 234, 257, 255, 280, 220, 274, 289, 260, 239, 274, 247, 261, 245, 241, 261, 266, 257, 266, 266, 289, 255, 232, 234, 275, 219, 247, 259, 249, 283, 283, 258, 246, 254, 260, 255, 248, 277, 277, 253, 233, 269, 238, 252, 259, 258, 267, 248, 284, 261, 234, 249, 228, 267, 231, 244, 249, 243, 230, 264, 262, 245, 251, 250, 264, 252, 253, 242, 236, 245, 265, 245, 291, 271, 261, 269, 266, 255, 250, 252, 265] payload_rows_per_dst=[5394, 5462, 5502, 5472, 5324, 5662, 5392, 5490] dup_rows_per_dst=[2732, 2730, 2830, 2674, 2632, 2882, 2600, 2758] combine_rows_per_src=[5394, 5462, 5502, 5472, 5324, 5662, 5392, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229424 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156680240 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313131056 weight_movement_bytes=37748736 total_nvlink_bytes=350879792 weight_to_token_ratio=0.120553 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228256 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156588960 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312949664 weight_movement_bytes=18874368 total_nvlink_bytes=331824032 weight_to_token_ratio=0.060311 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229776 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156918160 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606544 weight_movement_bytes=18874368 total_nvlink_bytes=332480912 weight_to_token_ratio=0.060185 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228600 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156138744 ingress_bytes=174784512 link_bytes=174784512 token_bytes=312048888 weight_movement_bytes=18874368 total_nvlink_bytes=330923256 weight_to_token_ratio=0.060485 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674699195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229200 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156294992 ingress_bytes=193814528 link_bytes=193814528 token_bytes=312360784 weight_movement_bytes=37748736 total_nvlink_bytes=350109520 weight_to_token_ratio=0.120850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857035445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229152 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156729120 ingress_bytes=194248704 link_bytes=194248704 token_bytes=313229088 weight_movement_bytes=37748736 total_nvlink_bytes=350977824 weight_to_token_ratio=0.120515 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861195505e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38082 dup_rows=21958 dup_rows_remote=19264 dedup_ratio=0.335052 router_tokens_per_expert=[228, 277, 238, 260, 273, 252, 251, 252, 236, 258, 269, 275, 256, 233, 241, 291, 264, 253, 238, 261, 259, 251, 260, 269, 233, 271, 240, 280, 279, 244, 273, 267, 256, 227, 269, 254, 255, 286, 249, 280, 238, 266, 247, 258, 287, 254, 242, 235, 278, 230, 241, 242, 248, 248, 230, 237, 249, 256, 262, 267, 266, 313, 258, 270, 249, 263, 271, 244, 252, 247, 280, 252, 230, 256, 255, 263, 255, 246, 286, 245, 255, 240, 246, 273, 260, 244, 252, 269, 236, 248, 242, 258, 259, 243, 232, 250, 247, 229, 245, 269, 253, 250, 282, 268, 246, 255, 249, 231, 248, 258, 279, 282, 252, 269, 262, 269, 261, 299, 251, 264, 236, 280, 263, 252, 236, 261, 238, 253] payload_rows_per_dst=[5428, 5470, 5434, 5496, 5368, 5362, 5512, 5508] dup_rows_per_dst=[2752, 2814, 2772, 2694, 2820, 2652, 2670, 2784] combine_rows_per_src=[5428, 5470, 5434, 5496, 5368, 5362, 5512, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38436 dup_rows=21692 dup_rows_remote=19038 dedup_ratio=0.330994 router_tokens_per_expert=[256, 242, 248, 268, 254, 247, 244, 267, 267, 262, 281, 236, 271, 227, 249, 242, 246, 261, 246, 264, 262, 261, 224, 249, 261, 230, 252, 262, 237, 251, 246, 279, 251, 285, 244, 243, 256, 257, 269, 256, 275, 258, 249, 242, 252, 294, 261, 261, 264, 254, 232, 236, 257, 246, 240, 278, 242, 267, 290, 238, 232, 273, 243, 272, 275, 260, 264, 269, 254, 224, 270, 265, 272, 238, 257, 253, 273, 262, 255, 264, 248, 250, 231, 275, 239, 254, 248, 267, 244, 266, 238, 279, 262, 262, 265, 239, 284, 264, 252, 244, 257, 282, 258, 233, 248, 242, 252, 278, 260, 256, 259, 230, 268, 259, 262, 257, 245, 246, 283, 268, 263, 263, 261, 246, 248, 251, 254, 264] payload_rows_per_dst=[5502, 5408, 5496, 5382, 5538, 5438, 5520, 5560] dup_rows_per_dst=[2712, 2654, 2810, 2746, 2772, 2604, 2678, 2716] combine_rows_per_src=[5502, 5408, 5496, 5382, 5538, 5438, 5520, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=229384 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156213256 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312197128 weight_movement_bytes=18874368 total_nvlink_bytes=331071496 weight_to_token_ratio=0.060457 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=229896 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157663752 ingress_bytes=176308224 link_bytes=176308224 token_bytes=315097608 weight_movement_bytes=18874368 total_nvlink_bytes=333971976 weight_to_token_ratio=0.059900 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689298653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38222 dup_rows=21832 dup_rows_remote=19046 dedup_ratio=0.333130 router_tokens_per_expert=[259, 245, 256, 254, 232, 257, 248, 252, 282, 232, 216, 265, 248, 259, 242, 278, 233, 270, 246, 259, 244, 254, 259, 246, 236, 277, 253, 265, 244, 257, 260, 252, 257, 241, 244, 256, 253, 269, 240, 270, 284, 287, 253, 253, 245, 230, 262, 248, 270, 253, 275, 272, 258, 258, 250, 246, 255, 231, 247, 236, 272, 232, 272, 264, 267, 246, 247, 241, 257, 271, 251, 260, 269, 262, 260, 260, 302, 252, 251, 238, 269, 245, 238, 226, 276, 256, 263, 268, 232, 267, 248, 244, 270, 251, 267, 251, 239, 236, 280, 242, 268, 279, 248, 274, 247, 261, 229, 251, 254, 272, 280, 259, 271, 255, 252, 256, 261, 265, 270, 280, 266, 244, 261, 236, 262, 240, 277, 285] payload_rows_per_dst=[5390, 5440, 5492, 5442, 5482, 5444, 5502, 5512] dup_rows_per_dst=[2660, 2670, 2692, 2740, 2786, 2698, 2736, 2850] combine_rows_per_src=[5390, 5440, 5492, 5442, 5482, 5444, 5502, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38010 dup_rows=21912 dup_rows_remote=19104 dedup_ratio=0.334351 router_tokens_per_expert=[288, 277, 236, 275, 252, 267, 272, 254, 244, 259, 245, 239, 244, 240, 270, 282, 238, 267, 257, 231, 260, 240, 249, 259, 259, 244, 247, 277, 270, 237, 241, 262, 236, 256, 266, 274, 245, 274, 255, 268, 247, 257, 272, 243, 235, 271, 252, 257, 242, 257, 277, 253, 256, 258, 260, 264, 273, 251, 264, 255, 230, 242, 234, 273, 273, 251, 272, 266, 225, 268, 258, 262, 267, 258, 278, 236, 251, 255, 285, 236, 276, 236, 273, 264, 267, 247, 254, 234, 247, 247, 271, 261, 244, 263, 242, 258, 237, 253, 241, 284, 273, 222, 262, 250, 251, 264, 236, 255, 241, 258, 248, 242, 246, 261, 266, 254, 293, 251, 249, 251, 265, 288, 229, 268, 269, 267, 241, 249] payload_rows_per_dst=[5614, 5398, 5462, 5474, 5470, 5500, 5228, 5478] dup_rows_per_dst=[2808, 2684, 2754, 2828, 2812, 2876, 2396, 2754] combine_rows_per_src=[5614, 5398, 5462, 5474, 5470, 5500, 5228, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38302 dup_rows=21790 dup_rows_remote=19090 dedup_ratio=0.332489 router_tokens_per_expert=[249, 262, 262, 251, 239, 265, 248, 229, 239, 260, 274, 262, 297, 271, 274, 229, 260, 281, 247, 270, 253, 229, 258, 268, 269, 248, 270, 271, 272, 264, 222, 248, 261, 268, 257, 248, 254, 278, 224, 227, 287, 231, 258, 258, 276, 218, 274, 250, 269, 233, 246, 241, 277, 237, 280, 265, 292, 246, 265, 264, 247, 268, 279, 275, 240, 262, 257, 252, 227, 246, 253, 270, 256, 250, 257, 262, 260, 254, 262, 232, 265, 258, 232, 276, 223, 237, 269, 295, 242, 257, 249, 251, 248, 260, 265, 246, 251, 274, 281, 256, 253, 233, 237, 260, 261, 251, 255, 252, 242, 240, 281, 245, 258, 266, 260, 263, 255, 274, 290, 244, 247, 276, 231, 241, 257, 245, 250, 232] payload_rows_per_dst=[5556, 5520, 5506, 5526, 5324, 5422, 5444, 5448] dup_rows_per_dst=[2666, 2740, 2632, 2842, 2756, 2724, 2700, 2730] combine_rows_per_src=[5556, 5520, 5506, 5526, 5324, 5422, 5444, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38272 dup_rows=21608 dup_rows_remote=18814 dedup_ratio=0.329712 router_tokens_per_expert=[255, 243, 252, 254, 248, 227, 273, 268, 276, 221, 241, 243, 252, 230, 270, 266, 255, 253, 248, 269, 285, 267, 287, 265, 255, 250, 263, 229, 303, 267, 272, 268, 260, 238, 275, 275, 247, 247, 244, 274, 284, 249, 230, 245, 254, 235, 222, 230, 271, 249, 290, 298, 227, 256, 236, 239, 236, 280, 262, 230, 251, 283, 252, 267, 258, 249, 244, 261, 219, 233, 259, 255, 243, 288, 234, 262, 249, 245, 252, 275, 303, 258, 275, 271, 271, 262, 270, 280, 228, 245, 271, 263, 234, 270, 262, 262, 264, 266, 248, 262, 249, 282, 265, 262, 259, 258, 240, 248, 249, 257, 237, 259, 231, 253, 284, 249, 276, 253, 256, 266, 236, 239, 261, 249, 249, 247, 219, 253] payload_rows_per_dst=[5382, 5594, 5364, 5548, 5428, 5656, 5496, 5460] dup_rows_per_dst=[2656, 2878, 2654, 2706, 2624, 2794, 2714, 2582] combine_rows_per_src=[5382, 5594, 5364, 5548, 5428, 5656, 5496, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38484 dup_rows=21686 dup_rows_remote=19002 dedup_ratio=0.330902 router_tokens_per_expert=[274, 251, 244, 268, 241, 294, 274, 248, 262, 233, 263, 217, 249, 275, 250, 243, 272, 281, 254, 270, 241, 234, 217, 273, 251, 237, 269, 278, 258, 257, 301, 246, 248, 256, 263, 266, 246, 265, 261, 244, 248, 247, 260, 216, 247, 261, 243, 254, 256, 256, 258, 233, 289, 250, 257, 224, 243, 256, 232, 218, 259, 240, 255, 255, 288, 255, 255, 265, 235, 255, 272, 276, 236, 254, 252, 268, 246, 222, 268, 258, 255, 267, 244, 252, 229, 269, 280, 264, 266, 269, 277, 282, 258, 246, 262, 284, 254, 255, 246, 258, 270, 242, 267, 264, 265, 272, 255, 251, 242, 262, 250, 278, 221, 273, 271, 253, 263, 284, 248, 236, 259, 243, 278, 250, 252, 242, 252, 272] payload_rows_per_dst=[5468, 5488, 5366, 5390, 5502, 5584, 5536, 5516] dup_rows_per_dst=[2704, 2790, 2684, 2572, 2708, 2824, 2726, 2678] combine_rows_per_src=[5468, 5488, 5366, 5390, 5502, 5584, 5536, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38188 dup_rows=21872 dup_rows_remote=19034 dedup_ratio=0.333740 router_tokens_per_expert=[262, 243, 249, 253, 255, 268, 284, 249, 224, 277, 218, 261, 243, 247, 246, 278, 252, 265, 254, 252, 263, 258, 288, 308, 255, 257, 246, 279, 221, 216, 249, 235, 265, 268, 261, 252, 256, 254, 252, 254, 249, 247, 284, 234, 260, 261, 274, 251, 260, 248, 264, 253, 234, 273, 268, 238, 253, 237, 273, 297, 243, 261, 285, 256, 275, 246, 278, 247, 249, 270, 251, 251, 256, 263, 247, 221, 267, 258, 269, 248, 267, 244, 241, 252, 240, 236, 248, 263, 275, 262, 268, 259, 241, 266, 258, 238, 251, 254, 268, 247, 233, 246, 266, 241, 263, 244, 245, 259, 275, 255, 247, 272, 232, 259, 251, 264, 263, 262, 264, 249, 256, 253, 238, 280, 244, 287, 251, 275] payload_rows_per_dst=[5316, 5244, 5558, 5724, 5454, 5482, 5410, 5476] dup_rows_per_dst=[2498, 2402, 2986, 3112, 2712, 2654, 2670, 2838] combine_rows_per_src=[5316, 5244, 5558, 5724, 5454, 5482, 5410, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229072 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156786384 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313343696 weight_movement_bytes=37748736 total_nvlink_bytes=351092432 weight_to_token_ratio=0.120471 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=228456 combine_payload_bytes=155688960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155917416 ingress_bytes=174563328 link_bytes=174563328 token_bytes=311606376 weight_movement_bytes=18874368 total_nvlink_bytes=330480744 weight_to_token_ratio=0.060571 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672579918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229568 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114560 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999552 weight_movement_bytes=37748736 total_nvlink_bytes=351748288 weight_to_token_ratio=0.120219 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=228344 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156990456 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313752568 weight_movement_bytes=37748736 total_nvlink_bytes=351501304 weight_to_token_ratio=0.120314 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157630464 dispatch_meta_bytes=229944 combine_payload_bytes=157630464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157860408 ingress_bytes=176504832 link_bytes=176504832 token_bytes=315490872 weight_movement_bytes=18874368 total_nvlink_bytes=334365240 weight_to_token_ratio=0.059825 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691182454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228888 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156646936 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313064984 weight_movement_bytes=18874368 total_nvlink_bytes=331939352 weight_to_token_ratio=0.060289 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38190 dup_rows=21926 dup_rows_remote=19230 dedup_ratio=0.334564 router_tokens_per_expert=[265, 241, 271, 228, 246, 241, 228, 248, 243, 250, 258, 257, 263, 261, 249, 258, 268, 250, 279, 251, 272, 259, 264, 234, 238, 258, 262, 264, 265, 246, 242, 282, 266, 240, 251, 274, 261, 229, 262, 269, 260, 253, 250, 265, 237, 255, 264, 257, 245, 240, 256, 290, 282, 252, 247, 229, 252, 274, 228, 270, 261, 252, 254, 226, 275, 241, 255, 287, 270, 258, 279, 251, 241, 264, 270, 262, 247, 275, 283, 231, 258, 250, 276, 250, 258, 264, 261, 232, 243, 281, 289, 267, 232, 274, 238, 274, 265, 237, 233, 268, 242, 258, 241, 260, 262, 266, 259, 269, 259, 256, 214, 266, 246, 277, 228, 267, 294, 251, 234, 266, 246, 250, 252, 252, 272, 239, 248, 263] payload_rows_per_dst=[5358, 5414, 5486, 5420, 5670, 5452, 5404, 5406] dup_rows_per_dst=[2656, 2854, 2700, 2696, 2708, 2842, 2706, 2764] combine_rows_per_src=[5358, 5414, 5486, 5420, 5670, 5452, 5404, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38156 dup_rows=21838 dup_rows_remote=19054 dedup_ratio=0.333221 router_tokens_per_expert=[270, 238, 259, 257, 254, 272, 284, 251, 258, 259, 257, 266, 240, 247, 250, 279, 264, 271, 239, 277, 270, 273, 269, 267, 244, 250, 242, 254, 264, 270, 275, 234, 268, 260, 250, 249, 253, 248, 252, 250, 281, 259, 265, 268, 278, 281, 231, 240, 271, 236, 250, 241, 247, 269, 253, 254, 222, 271, 259, 255, 262, 256, 265, 262, 240, 240, 224, 260, 250, 253, 257, 276, 279, 249, 251, 264, 229, 247, 254, 237, 255, 254, 242, 262, 263, 254, 256, 270, 271, 269, 249, 247, 226, 242, 269, 259, 272, 255, 256, 279, 254, 251, 257, 260, 243, 258, 257, 251, 258, 256, 251, 266, 253, 226, 268, 255, 266, 275, 248, 264, 268, 245, 259, 262, 243, 251, 219, 234] payload_rows_per_dst=[5470, 5542, 5504, 5470, 5394, 5436, 5488, 5394] dup_rows_per_dst=[2812, 2784, 2762, 2676, 2626, 2740, 2760, 2678] combine_rows_per_src=[5470, 5542, 5504, 5470, 5394, 5436, 5488, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229680 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655920 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313082160 weight_movement_bytes=18874368 total_nvlink_bytes=331956528 weight_to_token_ratio=0.060286 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228840 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156515816 ingress_bytes=194035712 link_bytes=194035712 token_bytes=312802792 weight_movement_bytes=37748736 total_nvlink_bytes=350551528 weight_to_token_ratio=0.120679 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859154721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38152 dup_rows=21990 dup_rows_remote=19276 dedup_ratio=0.335541 router_tokens_per_expert=[245, 275, 255, 255, 245, 228, 256, 260, 286, 288, 261, 235, 262, 264, 264, 293, 253, 274, 253, 277, 252, 247, 240, 253, 266, 264, 238, 261, 255, 237, 255, 288, 278, 258, 254, 237, 267, 282, 261, 267, 261, 231, 277, 253, 277, 248, 251, 275, 263, 254, 243, 264, 268, 248, 272, 251, 292, 268, 262, 236, 245, 237, 260, 266, 243, 272, 213, 288, 265, 251, 256, 275, 238, 222, 238, 249, 242, 261, 250, 268, 242, 235, 284, 227, 236, 246, 236, 241, 290, 280, 259, 242, 244, 265, 255, 250, 249, 247, 258, 269, 243, 252, 257, 269, 246, 230, 270, 232, 229, 262, 271, 270, 253, 266, 243, 266, 249, 216, 239, 240, 268, 236, 266, 250, 249, 249, 278, 292] payload_rows_per_dst=[5518, 5436, 5508, 5486, 5384, 5398, 5394, 5422] dup_rows_per_dst=[2826, 2790, 2846, 2772, 2678, 2666, 2714, 2698] combine_rows_per_src=[5518, 5436, 5508, 5486, 5384, 5398, 5394, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38212 dup_rows=21812 dup_rows_remote=18942 dedup_ratio=0.332825 router_tokens_per_expert=[253, 293, 279, 269, 277, 254, 236, 248, 246, 275, 289, 267, 278, 225, 280, 270, 248, 232, 261, 239, 278, 231, 240, 264, 267, 241, 263, 266, 259, 270, 232, 275, 284, 237, 216, 225, 258, 235, 260, 254, 251, 255, 268, 256, 256, 243, 277, 275, 266, 257, 252, 235, 242, 259, 259, 238, 238, 242, 243, 253, 210, 262, 249, 238, 263, 294, 291, 249, 248, 258, 264, 245, 254, 239, 249, 248, 279, 263, 266, 281, 255, 250, 280, 239, 290, 240, 289, 281, 253, 249, 274, 244, 251, 243, 243, 258, 250, 246, 254, 265, 277, 222, 279, 257, 240, 225, 283, 284, 264, 237, 220, 235, 269, 248, 274, 232, 281, 269, 250, 252, 245, 251, 253, 257, 234, 230, 277, 280] payload_rows_per_dst=[5568, 5370, 5442, 5352, 5512, 5564, 5412, 5504] dup_rows_per_dst=[2910, 2762, 2658, 2534, 2870, 2714, 2664, 2700] combine_rows_per_src=[5568, 5370, 5442, 5352, 5512, 5564, 5412, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38326 dup_rows=21696 dup_rows_remote=18912 dedup_ratio=0.331055 router_tokens_per_expert=[263, 235, 246, 227, 277, 264, 284, 268, 279, 248, 265, 258, 239, 266, 251, 246, 262, 257, 240, 246, 224, 267, 267, 284, 286, 246, 256, 238, 263, 267, 274, 264, 244, 252, 273, 253, 286, 279, 260, 280, 263, 251, 241, 234, 275, 267, 248, 267, 280, 229, 258, 258, 266, 258, 256, 262, 249, 258, 258, 236, 264, 266, 246, 250, 264, 224, 257, 238, 266, 277, 244, 265, 267, 266, 280, 262, 279, 249, 276, 257, 250, 251, 262, 257, 242, 259, 235, 264, 255, 248, 260, 247, 265, 257, 253, 268, 270, 275, 241, 231, 241, 232, 234, 251, 258, 277, 256, 264, 245, 227, 241, 263, 232, 241, 235, 263, 250, 256, 286, 258, 264, 246, 253, 270, 231, 258, 218, 233] payload_rows_per_dst=[5514, 5530, 5474, 5586, 5604, 5480, 5422, 5230] dup_rows_per_dst=[2784, 2756, 2814, 2792, 2826, 2654, 2590, 2480] combine_rows_per_src=[5514, 5530, 5474, 5586, 5604, 5480, 5422, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38262 dup_rows=21824 dup_rows_remote=19020 dedup_ratio=0.333008 router_tokens_per_expert=[269, 241, 254, 270, 257, 260, 258, 254, 277, 252, 244, 266, 269, 248, 251, 258, 245, 229, 265, 270, 293, 269, 258, 246, 225, 244, 245, 240, 265, 267, 259, 254, 246, 270, 254, 226, 266, 246, 262, 257, 242, 271, 267, 279, 243, 244, 270, 255, 256, 231, 224, 265, 250, 251, 260, 255, 269, 249, 257, 266, 252, 262, 265, 246, 257, 276, 277, 236, 274, 263, 255, 267, 245, 269, 241, 248, 267, 256, 227, 272, 270, 279, 246, 254, 234, 264, 278, 257, 264, 258, 245, 246, 241, 249, 263, 279, 316, 236, 253, 272, 231, 265, 236, 261, 250, 281, 271, 251, 248, 263, 250, 252, 247, 248, 246, 277, 238, 235, 261, 212, 230, 270, 244, 265, 283, 245, 239, 277] payload_rows_per_dst=[5460, 5484, 5474, 5492, 5474, 5450, 5466, 5412] dup_rows_per_dst=[2796, 2664, 2722, 2624, 2786, 2804, 2806, 2622] combine_rows_per_src=[5460, 5484, 5474, 5492, 5474, 5450, 5466, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38106 dup_rows=21906 dup_rows_remote=19148 dedup_ratio=0.334259 router_tokens_per_expert=[259, 285, 238, 256, 275, 270, 243, 247, 285, 250, 271, 249, 278, 246, 272, 275, 254, 247, 240, 265, 249, 248, 261, 261, 261, 246, 261, 258, 249, 243, 251, 235, 270, 237, 271, 276, 277, 266, 247, 238, 269, 252, 293, 265, 248, 237, 249, 246, 244, 232, 249, 257, 249, 261, 253, 260, 280, 283, 281, 244, 269, 259, 263, 253, 250, 253, 264, 250, 260, 245, 257, 255, 270, 251, 235, 252, 245, 241, 243, 232, 269, 241, 244, 234, 266, 256, 265, 253, 261, 268, 299, 272, 246, 247, 250, 251, 243, 270, 251, 262, 253, 274, 231, 255, 246, 227, 251, 277, 238, 247, 286, 261, 247, 258, 265, 264, 239, 258, 270, 266, 221, 259, 294, 248, 240, 236, 269, 231] payload_rows_per_dst=[5568, 5376, 5524, 5460, 5364, 5460, 5474, 5404] dup_rows_per_dst=[2830, 2682, 2758, 2814, 2642, 2784, 2670, 2726] combine_rows_per_src=[5568, 5376, 5524, 5460, 5364, 5460, 5474, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38318 dup_rows=21670 dup_rows_remote=18908 dedup_ratio=0.330658 router_tokens_per_expert=[239, 263, 279, 255, 233, 281, 279, 276, 256, 258, 262, 256, 255, 253, 261, 245, 258, 249, 258, 244, 247, 235, 244, 254, 249, 266, 265, 278, 269, 272, 251, 250, 259, 261, 252, 263, 261, 276, 250, 247, 254, 270, 252, 275, 265, 240, 268, 248, 240, 244, 246, 247, 264, 254, 245, 237, 264, 260, 238, 269, 267, 257, 266, 254, 243, 243, 248, 270, 267, 283, 263, 227, 271, 254, 235, 287, 269, 228, 257, 273, 266, 221, 233, 241, 272, 248, 263, 263, 237, 241, 251, 245, 265, 243, 235, 269, 270, 258, 290, 247, 269, 250, 247, 278, 260, 266, 257, 268, 276, 265, 272, 246, 222, 258, 261, 238, 259, 272, 233, 234, 258, 253, 268, 235, 269, 266, 234, 245] payload_rows_per_dst=[5352, 5458, 5338, 5512, 5644, 5426, 5588, 5548] dup_rows_per_dst=[2510, 2720, 2554, 2732, 2982, 2560, 2850, 2762] combine_rows_per_src=[5352, 5458, 5338, 5512, 5644, 5426, 5588, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229712 combine_payload_bytes=156270592 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156500304 ingress_bytes=212893696 link_bytes=212893696 token_bytes=312770896 weight_movement_bytes=56623104 total_nvlink_bytes=369394000 weight_to_token_ratio=0.181037 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039842645e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228616 combine_payload_bytes=156516352 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156744968 ingress_bytes=213139456 link_bytes=213139456 token_bytes=313261320 weight_movement_bytes=56623104 total_nvlink_bytes=369884424 weight_to_token_ratio=0.180754 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042197396e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=228952 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212248 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195544 weight_movement_bytes=37748736 total_nvlink_bytes=351944280 weight_to_token_ratio=0.120144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229128 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156950280 ingress_bytes=194469888 link_bytes=194469888 token_bytes=313671432 weight_movement_bytes=37748736 total_nvlink_bytes=351420168 weight_to_token_ratio=0.120345 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863314782e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229016 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156311192 ingress_bytes=212705280 link_bytes=212705280 token_bytes=312393368 weight_movement_bytes=56623104 total_nvlink_bytes=369016472 weight_to_token_ratio=0.181256 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038037335e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38282 dup_rows=21798 dup_rows_remote=19080 dedup_ratio=0.332611 router_tokens_per_expert=[246, 285, 277, 254, 250, 241, 265, 249, 257, 246, 260, 254, 230, 262, 263, 232, 263, 246, 234, 251, 271, 263, 274, 244, 226, 270, 219, 261, 265, 243, 278, 270, 239, 255, 295, 251, 261, 242, 244, 266, 243, 278, 271, 275, 250, 258, 283, 254, 242, 246, 272, 252, 276, 265, 235, 245, 258, 248, 251, 264, 289, 221, 263, 260, 272, 268, 242, 236, 274, 244, 242, 285, 265, 241, 228, 252, 251, 276, 256, 250, 271, 252, 272, 247, 273, 239, 258, 238, 275, 246, 264, 276, 248, 259, 261, 256, 288, 222, 266, 262, 273, 247, 252, 245, 244, 254, 258, 277, 228, 255, 262, 280, 270, 258, 249, 232, 262, 249, 235, 268, 247, 265, 225, 277, 255, 230, 279, 236] payload_rows_per_dst=[5472, 5466, 5554, 5456, 5454, 5506, 5424, 5406] dup_rows_per_dst=[2670, 2690, 2776, 2718, 2710, 2764, 2802, 2668] combine_rows_per_src=[5472, 5466, 5554, 5456, 5454, 5506, 5424, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228904 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179432 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314129960 weight_movement_bytes=18874368 total_nvlink_bytes=333004328 weight_to_token_ratio=0.060085 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38178 dup_rows=21930 dup_rows_remote=19174 dedup_ratio=0.334625 router_tokens_per_expert=[242, 238, 275, 258, 258, 283, 266, 268, 260, 249, 265, 242, 254, 222, 264, 263, 274, 276, 295, 250, 240, 247, 253, 261, 249, 237, 248, 266, 235, 251, 256, 254, 277, 250, 261, 269, 267, 252, 269, 267, 250, 257, 251, 254, 233, 263, 255, 297, 246, 241, 265, 266, 233, 247, 241, 267, 241, 230, 267, 238, 263, 262, 259, 245, 270, 274, 275, 261, 253, 256, 255, 240, 248, 244, 275, 224, 256, 252, 261, 266, 257, 241, 264, 239, 249, 248, 264, 254, 242, 264, 246, 248, 252, 250, 257, 249, 250, 274, 257, 264, 267, 276, 246, 271, 244, 253, 257, 257, 268, 267, 252, 262, 277, 222, 272, 251, 247, 256, 232, 268, 248, 247, 274, 282, 259, 254, 271, 227] payload_rows_per_dst=[5478, 5428, 5542, 5366, 5452, 5338, 5508, 5494] dup_rows_per_dst=[2736, 2756, 2802, 2656, 2768, 2710, 2822, 2680] combine_rows_per_src=[5478, 5428, 5542, 5366, 5452, 5338, 5508, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229448 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157032520 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313835592 weight_movement_bytes=56623104 total_nvlink_bytes=370458696 weight_to_token_ratio=0.180423 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229408 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606496 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983584 weight_movement_bytes=18874368 total_nvlink_bytes=331857952 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38372 dup_rows=21708 dup_rows_remote=18938 dedup_ratio=0.331238 router_tokens_per_expert=[235, 218, 229, 263, 262, 265, 264, 263, 265, 249, 225, 262, 242, 291, 255, 248, 246, 232, 257, 258, 240, 250, 244, 254, 263, 269, 256, 265, 260, 247, 268, 276, 252, 278, 260, 235, 287, 265, 251, 274, 242, 232, 231, 277, 270, 283, 251, 223, 282, 263, 257, 272, 243, 293, 255, 266, 291, 262, 269, 269, 238, 271, 231, 248, 258, 261, 259, 253, 267, 242, 276, 253, 276, 257, 242, 262, 280, 245, 259, 223, 258, 265, 240, 251, 227, 266, 233, 243, 227, 281, 246, 262, 280, 266, 254, 267, 269, 281, 258, 258, 245, 272, 246, 272, 250, 259, 248, 282, 232, 254, 229, 258, 261, 258, 253, 244, 274, 268, 231, 243, 251, 243, 252, 264, 261, 253, 229, 249] payload_rows_per_dst=[5502, 5462, 5474, 5550, 5452, 5506, 5456, 5426] dup_rows_per_dst=[2570, 2708, 2748, 2870, 2774, 2626, 2770, 2642] combine_rows_per_src=[5502, 5462, 5474, 5550, 5452, 5506, 5456, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38262 dup_rows=21842 dup_rows_remote=19156 dedup_ratio=0.333282 router_tokens_per_expert=[274, 259, 247, 224, 229, 258, 257, 248, 266, 241, 252, 279, 281, 263, 272, 274, 265, 261, 282, 240, 254, 244, 228, 247, 267, 238, 266, 239, 244, 286, 266, 264, 268, 248, 258, 246, 260, 234, 265, 235, 247, 252, 251, 272, 257, 235, 250, 273, 247, 235, 227, 257, 255, 238, 250, 236, 258, 271, 277, 289, 283, 282, 255, 253, 262, 265, 266, 285, 236, 261, 266, 258, 259, 233, 240, 240, 250, 229, 267, 242, 259, 242, 256, 243, 265, 242, 245, 275, 283, 236, 269, 255, 250, 232, 252, 282, 254, 251, 281, 252, 243, 279, 262, 264, 251, 236, 273, 247, 262, 261, 279, 237, 278, 252, 270, 250, 261, 266, 242, 264, 247, 291, 251, 247, 247, 257, 232, 257] payload_rows_per_dst=[5500, 5432, 5430, 5484, 5432, 5502, 5472, 5442] dup_rows_per_dst=[2748, 2750, 2672, 2742, 2686, 2670, 2792, 2782] combine_rows_per_src=[5500, 5432, 5430, 5484, 5432, 5502, 5472, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38364 dup_rows=21818 dup_rows_remote=19208 dedup_ratio=0.332916 router_tokens_per_expert=[250, 241, 250, 220, 235, 255, 235, 284, 263, 257, 265, 243, 276, 258, 266, 237, 277, 214, 258, 244, 263, 244, 245, 267, 262, 268, 233, 265, 256, 226, 266, 262, 264, 275, 253, 249, 272, 272, 240, 276, 234, 256, 248, 280, 240, 234, 275, 267, 277, 262, 260, 251, 239, 294, 225, 288, 253, 278, 285, 256, 228, 250, 271, 232, 267, 272, 245, 245, 255, 268, 254, 223, 239, 263, 259, 238, 245, 254, 277, 259, 279, 225, 245, 250, 243, 261, 272, 226, 246, 238, 250, 215, 244, 265, 263, 260, 234, 235, 274, 288, 260, 281, 260, 275, 263, 252, 275, 250, 279, 236, 268, 251, 265, 235, 232, 271, 268, 253, 281, 243, 264, 274, 292, 261, 261, 268, 241, 264] payload_rows_per_dst=[5508, 5436, 5476, 5488, 5438, 5354, 5502, 5516] dup_rows_per_dst=[2562, 2664, 2794, 2810, 2688, 2610, 2860, 2830] combine_rows_per_src=[5508, 5436, 5476, 5488, 5438, 5354, 5502, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38272 dup_rows=21862 dup_rows_remote=19180 dedup_ratio=0.333588 router_tokens_per_expert=[267, 266, 249, 262, 253, 251, 274, 251, 253, 227, 251, 228, 249, 258, 261, 242, 241, 279, 248, 252, 264, 273, 243, 224, 282, 264, 258, 265, 239, 253, 234, 263, 234, 264, 243, 273, 235, 261, 201, 259, 250, 261, 245, 255, 265, 264, 283, 251, 273, 278, 283, 256, 255, 255, 242, 241, 226, 248, 266, 239, 248, 276, 250, 274, 236, 285, 290, 255, 273, 251, 273, 278, 248, 255, 279, 258, 267, 275, 263, 239, 282, 265, 272, 287, 257, 239, 257, 238, 269, 260, 261, 258, 256, 265, 253, 262, 253, 273, 236, 226, 253, 250, 212, 283, 234, 251, 245, 246, 248, 259, 239, 270, 281, 232, 279, 251, 234, 278, 265, 217, 264, 248, 264, 251, 263, 264, 249, 266] payload_rows_per_dst=[5402, 5474, 5378, 5488, 5606, 5568, 5356, 5402] dup_rows_per_dst=[2682, 2690, 2710, 2732, 2844, 2914, 2600, 2690] combine_rows_per_src=[5402, 5474, 5378, 5488, 5606, 5568, 5356, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38328 dup_rows=21812 dup_rows_remote=19050 dedup_ratio=0.332825 router_tokens_per_expert=[258, 252, 243, 277, 262, 256, 273, 252, 246, 273, 257, 255, 246, 275, 250, 280, 259, 254, 245, 246, 247, 243, 234, 250, 256, 257, 252, 253, 281, 252, 239, 242, 239, 251, 256, 268, 257, 268, 259, 249, 260, 280, 265, 231, 233, 249, 253, 261, 270, 270, 256, 253, 239, 265, 258, 251, 257, 269, 267, 267, 270, 256, 254, 261, 228, 247, 244, 256, 260, 237, 244, 242, 233, 267, 259, 267, 268, 259, 260, 263, 262, 273, 253, 275, 229, 294, 274, 237, 255, 239, 250, 292, 273, 231, 245, 258, 253, 274, 252, 251, 238, 241, 269, 285, 277, 269, 254, 255, 241, 235, 232, 256, 255, 240, 241, 237, 271, 266, 276, 259, 243, 271, 252, 271, 247, 262, 246, 268] payload_rows_per_dst=[5540, 5338, 5396, 5562, 5406, 5572, 5462, 5448] dup_rows_per_dst=[2770, 2682, 2762, 2764, 2662, 2708, 2702, 2762] combine_rows_per_src=[5540, 5338, 5396, 5562, 5406, 5572, 5462, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38178 dup_rows=21812 dup_rows_remote=18866 dedup_ratio=0.332825 router_tokens_per_expert=[271, 261, 283, 264, 238, 277, 266, 276, 289, 266, 264, 243, 235, 245, 264, 228, 273, 230, 258, 225, 219, 264, 233, 254, 254, 241, 225, 284, 256, 276, 246, 250, 234, 248, 241, 249, 261, 278, 276, 289, 239, 280, 265, 237, 266, 286, 271, 251, 280, 262, 260, 272, 279, 221, 245, 243, 250, 255, 257, 246, 245, 290, 263, 246, 239, 271, 252, 266, 272, 247, 270, 256, 228, 250, 237, 263, 227, 237, 227, 261, 263, 252, 237, 257, 238, 275, 274, 254, 233, 264, 232, 255, 274, 243, 268, 268, 246, 260, 243, 274, 265, 235, 280, 247, 273, 260, 246, 285, 223, 268, 260, 274, 265, 239, 250, 256, 226, 266, 275, 231, 238, 228, 271, 305, 235, 289, 262, 260] payload_rows_per_dst=[5516, 5420, 5476, 5336, 5498, 5396, 5536, 5546] dup_rows_per_dst=[2824, 2556, 2854, 2626, 2776, 2462, 2768, 2946] combine_rows_per_src=[5516, 5420, 5476, 5336, 5498, 5396, 5536, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229240 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157400952 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572664 weight_movement_bytes=18874368 total_nvlink_bytes=333447032 weight_to_token_ratio=0.060000 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229672 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950824 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671976 weight_movement_bytes=18874368 total_nvlink_bytes=332546344 weight_to_token_ratio=0.060172 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=230288 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157369232 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314508176 weight_movement_bytes=18874368 total_nvlink_bytes=333382544 weight_to_token_ratio=0.060012 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229808 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156991920 ingress_bytes=194510848 link_bytes=194510848 token_bytes=313754032 weight_movement_bytes=37748736 total_nvlink_bytes=351502768 weight_to_token_ratio=0.120313 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863707240e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229512 combine_payload_bytes=156991488 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157221000 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314212488 weight_movement_bytes=56623104 total_nvlink_bytes=370835592 weight_to_token_ratio=0.180206 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228176 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156605264 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312982352 weight_movement_bytes=18874368 total_nvlink_bytes=331856720 weight_to_token_ratio=0.060305 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43502 payload_rows_remote=38100 dup_rows=22034 dup_rows_remote=19356 dedup_ratio=0.336212 router_tokens_per_expert=[261, 242, 261, 255, 227, 268, 252, 264, 255, 282, 257, 272, 241, 279, 259, 262, 250, 273, 251, 276, 257, 241, 243, 261, 240, 235, 248, 257, 262, 252, 239, 262, 255, 270, 267, 226, 261, 250, 274, 271, 282, 238, 276, 261, 251, 269, 244, 252, 261, 276, 271, 261, 264, 238, 250, 248, 261, 269, 230, 239, 233, 247, 221, 271, 260, 265, 244, 255, 261, 255, 274, 259, 245, 256, 262, 238, 231, 246, 243, 274, 245, 259, 255, 242, 232, 247, 243, 247, 255, 281, 291, 256, 240, 264, 262, 266, 238, 252, 282, 270, 238, 279, 250, 274, 229, 269, 281, 257, 262, 233, 264, 249, 274, 253, 271, 252, 253, 250, 240, 267, 243, 256, 251, 243, 242, 278, 284, 260] payload_rows_per_dst=[5470, 5370, 5502, 5402, 5394, 5474, 5452, 5438] dup_rows_per_dst=[2804, 2724, 2792, 2678, 2742, 2696, 2802, 2796] combine_rows_per_src=[5470, 5370, 5502, 5402, 5394, 5474, 5452, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38218 dup_rows=21774 dup_rows_remote=18920 dedup_ratio=0.332245 router_tokens_per_expert=[243, 269, 263, 246, 260, 251, 245, 279, 266, 248, 238, 251, 224, 265, 261, 244, 269, 253, 273, 272, 258, 281, 236, 234, 282, 272, 273, 261, 268, 274, 236, 257, 269, 240, 247, 260, 243, 256, 256, 250, 249, 273, 255, 244, 257, 255, 252, 273, 236, 290, 285, 255, 238, 244, 277, 238, 272, 252, 279, 258, 240, 243, 254, 246, 242, 222, 247, 274, 253, 251, 236, 242, 274, 282, 252, 267, 272, 242, 233, 245, 238, 279, 258, 255, 247, 254, 282, 268, 250, 264, 265, 271, 255, 259, 258, 254, 248, 245, 268, 264, 240, 261, 264, 265, 263, 270, 253, 245, 229, 243, 262, 238, 244, 266, 256, 254, 252, 265, 226, 256, 244, 271, 258, 264, 266, 243, 260, 256] payload_rows_per_dst=[5384, 5544, 5452, 5450, 5404, 5566, 5448, 5514] dup_rows_per_dst=[2722, 2854, 2706, 2764, 2664, 2748, 2668, 2648] combine_rows_per_src=[5384, 5544, 5452, 5450, 5404, 5566, 5448, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=229824 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156287424 ingress_bytes=174931968 link_bytes=174931968 token_bytes=312345024 weight_movement_bytes=18874368 total_nvlink_bytes=331219392 weight_to_token_ratio=0.060428 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676112045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228552 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156769480 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313310408 weight_movement_bytes=18874368 total_nvlink_bytes=332184776 weight_to_token_ratio=0.060242 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38348 dup_rows=21636 dup_rows_remote=18952 dedup_ratio=0.330139 router_tokens_per_expert=[252, 277, 258, 242, 253, 257, 243, 245, 232, 270, 273, 256, 262, 276, 266, 255, 232, 257, 231, 228, 248, 274, 278, 238, 256, 252, 259, 254, 258, 261, 216, 248, 261, 251, 280, 276, 259, 247, 251, 306, 285, 280, 251, 272, 274, 267, 239, 239, 249, 246, 252, 274, 252, 256, 241, 254, 276, 254, 285, 286, 261, 250, 239, 249, 270, 246, 240, 266, 291, 263, 232, 293, 268, 229, 283, 255, 257, 228, 260, 238, 274, 255, 270, 252, 233, 258, 238, 260, 253, 241, 265, 258, 274, 265, 245, 261, 250, 227, 257, 229, 246, 263, 282, 240, 255, 252, 247, 238, 226, 226, 275, 246, 257, 238, 234, 287, 282, 264, 283, 277, 233, 272, 228, 220, 258, 267, 259, 260] payload_rows_per_dst=[5484, 5378, 5600, 5520, 5460, 5368, 5552, 5538] dup_rows_per_dst=[2702, 2602, 2876, 2776, 2778, 2518, 2684, 2700] combine_rows_per_src=[5484, 5378, 5600, 5520, 5460, 5368, 5552, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38144 dup_rows=21846 dup_rows_remote=19124 dedup_ratio=0.333344 router_tokens_per_expert=[260, 278, 245, 262, 243, 253, 252, 254, 270, 263, 247, 264, 261, 270, 260, 250, 260, 247, 246, 265, 247, 261, 275, 267, 283, 249, 255, 302, 262, 239, 267, 257, 252, 240, 262, 272, 254, 263, 242, 243, 232, 224, 246, 239, 239, 264, 280, 248, 278, 244, 253, 270, 252, 284, 252, 249, 266, 246, 241, 275, 257, 271, 255, 270, 264, 258, 239, 251, 264, 251, 301, 273, 247, 244, 240, 276, 256, 238, 277, 255, 230, 251, 257, 245, 227, 256, 232, 266, 276, 229, 252, 252, 261, 253, 270, 231, 260, 294, 262, 250, 247, 262, 224, 253, 241, 244, 269, 265, 247, 259, 247, 251, 253, 267, 237, 279, 263, 241, 244, 275, 253, 255, 247, 270, 252, 240, 260, 258] payload_rows_per_dst=[5482, 5560, 5398, 5442, 5546, 5338, 5444, 5480] dup_rows_per_dst=[2782, 2804, 2602, 2884, 2722, 2638, 2706, 2708] combine_rows_per_src=[5482, 5560, 5398, 5442, 5546, 5338, 5444, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38402 dup_rows=21630 dup_rows_remote=18794 dedup_ratio=0.330048 router_tokens_per_expert=[227, 259, 238, 248, 247, 237, 239, 255, 256, 240, 274, 258, 304, 245, 252, 230, 282, 269, 260, 275, 253, 243, 280, 232, 242, 239, 262, 251, 260, 253, 256, 237, 242, 271, 264, 258, 271, 265, 252, 263, 259, 242, 233, 280, 276, 260, 254, 230, 246, 261, 249, 251, 233, 288, 227, 236, 242, 256, 266, 234, 267, 251, 262, 273, 287, 238, 263, 303, 240, 242, 236, 241, 253, 228, 236, 254, 279, 292, 276, 249, 232, 251, 279, 234, 257, 275, 286, 248, 243, 278, 275, 249, 278, 266, 276, 243, 266, 262, 255, 258, 276, 246, 256, 248, 264, 253, 268, 248, 265, 257, 261, 234, 261, 259, 237, 299, 274, 275, 244, 239, 271, 239, 228, 243, 260, 248, 245, 277] payload_rows_per_dst=[5444, 5470, 5516, 5426, 5458, 5504, 5576, 5512] dup_rows_per_dst=[2574, 2718, 2724, 2658, 2776, 2836, 2658, 2686] combine_rows_per_src=[5444, 5470, 5516, 5426, 5458, 5504, 5576, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38248 dup_rows=21906 dup_rows_remote=19216 dedup_ratio=0.334259 router_tokens_per_expert=[260, 239, 260, 257, 242, 281, 259, 236, 252, 244, 289, 248, 260, 280, 235, 250, 272, 259, 260, 233, 243, 257, 280, 279, 262, 238, 255, 248, 255, 275, 248, 242, 256, 267, 262, 231, 283, 251, 249, 255, 218, 257, 272, 229, 267, 226, 263, 250, 235, 259, 278, 253, 263, 285, 254, 264, 243, 265, 243, 230, 252, 235, 268, 275, 267, 261, 262, 256, 247, 249, 254, 257, 261, 243, 247, 268, 258, 245, 259, 265, 266, 260, 237, 253, 255, 243, 282, 280, 266, 268, 259, 263, 289, 239, 228, 272, 221, 229, 256, 259, 255, 246, 242, 281, 252, 272, 257, 290, 266, 269, 266, 247, 250, 254, 248, 268, 263, 260, 241, 259, 259, 256, 267, 267, 239, 257, 247, 230] payload_rows_per_dst=[5524, 5448, 5382, 5404, 5508, 5440, 5458, 5466] dup_rows_per_dst=[2660, 2764, 2690, 2800, 2690, 2880, 2758, 2664] combine_rows_per_src=[5524, 5448, 5382, 5404, 5508, 5440, 5458, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38218 dup_rows=21704 dup_rows_remote=18660 dedup_ratio=0.331177 router_tokens_per_expert=[258, 243, 255, 261, 252, 252, 259, 231, 267, 228, 245, 281, 264, 260, 262, 233, 236, 273, 258, 260, 258, 266, 265, 265, 263, 253, 248, 244, 246, 239, 241, 231, 242, 257, 278, 259, 276, 267, 284, 262, 240, 250, 256, 248, 268, 268, 253, 303, 248, 292, 251, 275, 246, 236, 273, 225, 255, 285, 242, 263, 298, 281, 258, 230, 236, 255, 222, 249, 282, 241, 236, 244, 249, 239, 248, 255, 248, 258, 256, 276, 271, 265, 241, 265, 244, 278, 249, 272, 241, 260, 244, 233, 276, 285, 243, 268, 276, 268, 252, 248, 251, 260, 257, 252, 250, 238, 231, 260, 287, 252, 258, 244, 253, 253, 258, 258, 247, 253, 276, 240, 258, 226, 264, 272, 261, 279, 236, 255] payload_rows_per_dst=[5614, 5530, 5528, 5542, 5514, 5312, 5448, 5344] dup_rows_per_dst=[3044, 2692, 2796, 2742, 2620, 2402, 2720, 2688] combine_rows_per_src=[5614, 5530, 5528, 5542, 5514, 5312, 5448, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38234 dup_rows=21694 dup_rows_remote=18904 dedup_ratio=0.331024 router_tokens_per_expert=[262, 236, 226, 232, 276, 250, 249, 293, 263, 263, 268, 264, 254, 264, 237, 245, 251, 256, 272, 244, 255, 272, 248, 287, 263, 248, 253, 245, 260, 242, 233, 255, 267, 258, 279, 250, 253, 239, 284, 268, 244, 248, 266, 243, 247, 257, 258, 255, 243, 255, 233, 262, 245, 236, 256, 245, 263, 230, 259, 261, 257, 236, 287, 256, 270, 270, 258, 286, 255, 250, 246, 235, 237, 247, 231, 261, 245, 242, 270, 249, 261, 261, 238, 268, 231, 288, 251, 260, 286, 258, 239, 262, 273, 240, 280, 259, 230, 263, 245, 257, 261, 244, 257, 266, 258, 258, 233, 255, 268, 261, 237, 263, 271, 280, 246, 293, 269, 267, 263, 258, 251, 253, 274, 279, 246, 215, 271, 263] payload_rows_per_dst=[5288, 5466, 5448, 5584, 5460, 5348, 5640, 5608] dup_rows_per_dst=[2326, 2702, 2748, 2992, 2702, 2444, 2990, 2790] combine_rows_per_src=[5288, 5466, 5448, 5584, 5460, 5348, 5640, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229200 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157302608 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376016 weight_movement_bytes=18874368 total_nvlink_bytes=333250384 weight_to_token_ratio=0.060038 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229072 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156466896 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312704720 weight_movement_bytes=18874368 total_nvlink_bytes=331579088 weight_to_token_ratio=0.060358 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229856 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156893664 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313557472 weight_movement_bytes=56623104 total_nvlink_bytes=370180576 weight_to_token_ratio=0.180583 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=228784 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157523376 ingress_bytes=213917696 link_bytes=213917696 token_bytes=314817968 weight_movement_bytes=56623104 total_nvlink_bytes=371441072 weight_to_token_ratio=0.179860 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049654109e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=227512 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156768440 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313309368 weight_movement_bytes=18874368 total_nvlink_bytes=332183736 weight_to_token_ratio=0.060242 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38196 dup_rows=21868 dup_rows_remote=19222 dedup_ratio=0.333679 router_tokens_per_expert=[245, 263, 244, 265, 241, 253, 278, 242, 278, 258, 217, 277, 263, 256, 257, 243, 256, 285, 251, 234, 264, 251, 258, 256, 274, 267, 261, 268, 245, 256, 241, 264, 234, 253, 267, 273, 225, 257, 258, 262, 269, 277, 252, 246, 268, 263, 246, 244, 249, 260, 250, 251, 252, 236, 264, 252, 243, 264, 263, 259, 281, 252, 247, 236, 245, 215, 231, 278, 254, 269, 266, 254, 264, 286, 253, 273, 279, 284, 264, 280, 246, 257, 263, 269, 254, 241, 272, 238, 246, 267, 250, 249, 248, 249, 223, 266, 252, 263, 277, 248, 278, 239, 256, 252, 274, 216, 253, 274, 241, 267, 222, 294, 271, 225, 272, 245, 243, 266, 249, 271, 275, 251, 265, 262, 227, 240, 248, 255] payload_rows_per_dst=[5444, 5464, 5448, 5472, 5578, 5428, 5488, 5346] dup_rows_per_dst=[2716, 2798, 2740, 2646, 2812, 2648, 2724, 2784] combine_rows_per_src=[5444, 5464, 5448, 5472, 5578, 5428, 5488, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43410 payload_rows_remote=37970 dup_rows=22126 dup_rows_remote=19266 dedup_ratio=0.337616 router_tokens_per_expert=[283, 253, 259, 287, 273, 264, 248, 269, 257, 250, 259, 269, 252, 268, 274, 249, 255, 256, 285, 267, 245, 281, 269, 250, 241, 245, 249, 258, 257, 265, 262, 265, 243, 237, 254, 255, 230, 259, 242, 256, 270, 280, 256, 235, 260, 281, 259, 272, 228, 255, 266, 236, 245, 235, 242, 249, 232, 249, 272, 260, 261, 257, 247, 264, 255, 236, 251, 268, 237, 268, 232, 257, 245, 272, 252, 256, 243, 246, 257, 268, 272, 295, 254, 244, 244, 250, 282, 301, 243, 245, 269, 242, 254, 241, 253, 235, 253, 265, 256, 231, 250, 248, 256, 243, 268, 282, 232, 244, 260, 238, 252, 224, 271, 268, 272, 266, 242, 236, 260, 253, 281, 237, 275, 264, 263, 241, 269, 250] payload_rows_per_dst=[5534, 5440, 5334, 5400, 5406, 5446, 5396, 5454] dup_rows_per_dst=[2894, 2860, 2844, 2596, 2680, 2802, 2608, 2842] combine_rows_per_src=[5534, 5440, 5334, 5400, 5406, 5446, 5396, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228552 combine_payload_bytes=156606464 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156835016 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441480 weight_movement_bytes=37748736 total_nvlink_bytes=351190216 weight_to_token_ratio=0.120433 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229672 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156680488 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313131304 weight_movement_bytes=37748736 total_nvlink_bytes=350880040 weight_to_token_ratio=0.120552 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155525120 dispatch_meta_bytes=228944 combine_payload_bytes=155525120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155754064 ingress_bytes=174399488 link_bytes=174399488 token_bytes=311279184 weight_movement_bytes=18874368 total_nvlink_bytes=330153552 weight_to_token_ratio=0.060635 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671010084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38268 dup_rows=21734 dup_rows_remote=18976 dedup_ratio=0.331635 router_tokens_per_expert=[248, 245, 218, 237, 216, 265, 263, 288, 227, 268, 254, 265, 241, 259, 238, 261, 246, 218, 231, 261, 227, 240, 226, 267, 280, 281, 242, 257, 253, 277, 222, 261, 232, 250, 251, 245, 254, 243, 234, 280, 242, 250, 275, 284, 239, 259, 281, 245, 276, 278, 225, 253, 268, 270, 279, 278, 262, 244, 274, 275, 262, 244, 267, 291, 235, 249, 253, 237, 244, 264, 250, 226, 252, 261, 272, 252, 245, 289, 262, 247, 261, 265, 265, 239, 270, 262, 246, 286, 255, 273, 244, 249, 262, 269, 268, 244, 242, 259, 262, 274, 250, 232, 264, 258, 283, 280, 236, 272, 259, 264, 247, 264, 264, 283, 241, 246, 283, 262, 253, 252, 263, 268, 252, 243, 275, 245, 248, 256] payload_rows_per_dst=[5418, 5384, 5410, 5612, 5376, 5554, 5534, 5514] dup_rows_per_dst=[2568, 2594, 2718, 2880, 2700, 2762, 2758, 2754] combine_rows_per_src=[5418, 5384, 5410, 5612, 5376, 5554, 5534, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43944 payload_rows_remote=38504 dup_rows=21592 dup_rows_remote=18790 dedup_ratio=0.329468 router_tokens_per_expert=[243, 238, 269, 260, 236, 244, 227, 263, 271, 292, 271, 263, 249, 250, 253, 240, 231, 236, 262, 254, 250, 276, 235, 250, 263, 216, 272, 268, 273, 252, 228, 264, 277, 242, 280, 293, 247, 249, 234, 266, 257, 227, 224, 273, 259, 260, 250, 260, 223, 238, 269, 254, 250, 242, 283, 252, 275, 243, 246, 247, 286, 264, 272, 221, 260, 251, 248, 255, 232, 275, 243, 268, 272, 236, 275, 281, 267, 252, 267, 239, 266, 257, 250, 272, 228, 287, 253, 235, 270, 249, 263, 246, 279, 281, 272, 286, 232, 259, 270, 273, 267, 269, 228, 266, 260, 242, 229, 259, 255, 264, 245, 252, 261, 248, 246, 243, 261, 259, 291, 245, 257, 267, 279, 234, 254, 245, 257, 274] payload_rows_per_dst=[5496, 5426, 5516, 5434, 5440, 5612, 5502, 5518] dup_rows_per_dst=[2642, 2634, 2680, 2696, 2802, 2776, 2638, 2724] combine_rows_per_src=[5496, 5426, 5516, 5434, 5440, 5612, 5502, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38370 dup_rows=21712 dup_rows_remote=18948 dedup_ratio=0.331299 router_tokens_per_expert=[248, 265, 283, 263, 245, 262, 278, 267, 232, 287, 269, 262, 241, 241, 262, 231, 254, 248, 265, 253, 270, 253, 257, 263, 234, 271, 269, 247, 254, 250, 250, 254, 249, 272, 209, 247, 256, 303, 270, 265, 232, 250, 265, 261, 263, 253, 257, 257, 253, 262, 263, 262, 249, 262, 252, 247, 247, 264, 286, 264, 258, 271, 246, 244, 251, 238, 282, 242, 263, 269, 241, 257, 262, 225, 248, 243, 241, 272, 238, 245, 241, 259, 278, 258, 266, 245, 208, 256, 244, 290, 252, 268, 256, 234, 276, 241, 264, 233, 271, 250, 281, 265, 240, 257, 254, 223, 269, 253, 292, 234, 261, 271, 242, 259, 253, 254, 249, 256, 275, 242, 264, 265, 245, 269, 252, 266, 245, 258] payload_rows_per_dst=[5494, 5482, 5454, 5528, 5384, 5494, 5522, 5466] dup_rows_per_dst=[2778, 2702, 2764, 2732, 2650, 2650, 2714, 2722] combine_rows_per_src=[5494, 5482, 5454, 5528, 5384, 5494, 5522, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38174 dup_rows=21814 dup_rows_remote=19032 dedup_ratio=0.332855 router_tokens_per_expert=[260, 272, 280, 262, 230, 246, 269, 253, 255, 296, 252, 292, 254, 252, 273, 258, 248, 243, 268, 219, 276, 240, 262, 251, 260, 272, 274, 270, 257, 253, 253, 261, 275, 257, 283, 249, 233, 238, 264, 274, 259, 251, 243, 232, 249, 265, 258, 285, 254, 262, 272, 262, 266, 237, 249, 216, 265, 273, 265, 253, 250, 232, 233, 271, 258, 264, 229, 252, 237, 241, 242, 258, 255, 268, 265, 259, 281, 275, 258, 272, 236, 263, 256, 269, 281, 256, 259, 269, 239, 256, 253, 295, 227, 252, 294, 260, 222, 259, 256, 243, 260, 253, 261, 253, 238, 236, 231, 245, 268, 261, 233, 261, 270, 269, 248, 242, 247, 259, 259, 270, 228, 245, 233, 239, 256, 258, 240, 260] payload_rows_per_dst=[5528, 5478, 5528, 5346, 5502, 5548, 5338, 5454] dup_rows_per_dst=[2880, 2736, 2702, 2774, 2726, 2782, 2622, 2592] combine_rows_per_src=[5528, 5478, 5528, 5346, 5502, 5548, 5338, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38422 dup_rows=21740 dup_rows_remote=18982 dedup_ratio=0.331726 router_tokens_per_expert=[253, 238, 247, 270, 218, 259, 247, 251, 259, 286, 249, 248, 252, 265, 287, 231, 252, 270, 250, 286, 262, 295, 238, 278, 257, 256, 247, 229, 244, 266, 264, 265, 238, 241, 270, 252, 235, 232, 264, 246, 250, 235, 278, 253, 245, 241, 277, 261, 269, 242, 235, 253, 269, 256, 235, 205, 254, 247, 274, 250, 244, 249, 282, 252, 244, 245, 249, 263, 238, 263, 244, 261, 289, 260, 247, 292, 220, 279, 267, 260, 252, 257, 226, 254, 241, 260, 300, 257, 292, 250, 245, 256, 254, 268, 248, 274, 269, 271, 264, 257, 248, 257, 226, 266, 257, 252, 249, 285, 276, 241, 266, 252, 264, 263, 263, 246, 267, 245, 252, 254, 259, 250, 242, 263, 281, 245, 278, 252] payload_rows_per_dst=[5374, 5618, 5392, 5402, 5478, 5488, 5540, 5504] dup_rows_per_dst=[2758, 2844, 2644, 2630, 2752, 2636, 2732, 2744] combine_rows_per_src=[5374, 5618, 5392, 5402, 5478, 5488, 5540, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228976 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156974704 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720432 weight_movement_bytes=37748736 total_nvlink_bytes=351469168 weight_to_token_ratio=0.120326 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38040 dup_rows=21872 dup_rows_remote=18924 dedup_ratio=0.333740 router_tokens_per_expert=[264, 262, 249, 259, 249, 247, 262, 246, 223, 255, 249, 257, 253, 292, 258, 255, 271, 240, 246, 281, 261, 282, 251, 259, 256, 255, 250, 263, 274, 262, 273, 249, 261, 259, 243, 261, 251, 245, 243, 236, 259, 261, 238, 256, 251, 269, 252, 267, 266, 243, 243, 246, 274, 254, 224, 237, 257, 273, 243, 262, 254, 274, 264, 221, 256, 262, 252, 289, 264, 289, 252, 273, 270, 237, 243, 236, 257, 247, 241, 286, 254, 264, 251, 248, 274, 274, 243, 270, 241, 276, 241, 262, 254, 260, 244, 298, 226, 245, 237, 242, 222, 225, 221, 259, 255, 246, 256, 257, 256, 276, 242, 292, 258, 249, 276, 251, 269, 263, 248, 268, 250, 272, 244, 254, 277, 264, 259, 261] payload_rows_per_dst=[5472, 5476, 5438, 5292, 5508, 5482, 5372, 5624] dup_rows_per_dst=[2678, 2794, 2666, 2576, 2832, 2826, 2552, 2948] combine_rows_per_src=[5472, 5476, 5438, 5292, 5508, 5482, 5372, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157712384 dispatch_meta_bytes=229176 combine_payload_bytes=157712384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157941560 ingress_bytes=176586752 link_bytes=176586752 token_bytes=315653944 weight_movement_bytes=18874368 total_nvlink_bytes=334528312 weight_to_token_ratio=0.059794 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691967372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43960 payload_rows_remote=38464 dup_rows=21576 dup_rows_remote=18902 dedup_ratio=0.329224 router_tokens_per_expert=[247, 248, 250, 249, 242, 249, 259, 257, 256, 235, 257, 284, 266, 265, 270, 254, 245, 245, 256, 282, 243, 253, 280, 270, 230, 294, 258, 278, 250, 272, 275, 241, 258, 257, 279, 271, 261, 257, 275, 242, 264, 244, 250, 255, 259, 287, 232, 260, 259, 275, 251, 243, 262, 274, 241, 233, 253, 252, 246, 259, 257, 263, 258, 259, 243, 240, 298, 214, 242, 261, 258, 260, 261, 245, 269, 260, 245, 220, 271, 250, 270, 250, 222, 256, 271, 264, 259, 253, 262, 265, 247, 247, 277, 237, 245, 256, 258, 254, 261, 287, 238, 250, 226, 261, 229, 236, 244, 240, 265, 289, 239, 268, 231, 249, 269, 248, 249, 270, 274, 254, 239, 249, 274, 259, 240, 278, 273, 253] payload_rows_per_dst=[5486, 5570, 5530, 5496, 5482, 5504, 5456, 5436] dup_rows_per_dst=[2690, 2774, 2772, 2674, 2592, 2658, 2634, 2782] combine_rows_per_src=[5486, 5570, 5530, 5496, 5482, 5504, 5456, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229272 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157392792 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556312 weight_movement_bytes=18874368 total_nvlink_bytes=333430680 weight_to_token_ratio=0.060003 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228824 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156589528 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950232 weight_movement_bytes=37748736 total_nvlink_bytes=350698968 weight_to_token_ratio=0.120622 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229616 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157606128 ingress_bytes=195125248 link_bytes=195125248 token_bytes=314982640 weight_movement_bytes=37748736 total_nvlink_bytes=352731376 weight_to_token_ratio=0.119844 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869594119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38308 dup_rows=21766 dup_rows_remote=19018 dedup_ratio=0.332123 router_tokens_per_expert=[244, 272, 234, 276, 255, 241, 237, 255, 221, 265, 280, 266, 249, 255, 262, 256, 283, 234, 255, 251, 272, 242, 251, 243, 227, 267, 258, 289, 273, 277, 221, 262, 274, 263, 240, 229, 242, 266, 218, 273, 260, 263, 259, 287, 260, 257, 249, 253, 264, 230, 252, 263, 271, 277, 252, 247, 277, 273, 248, 285, 241, 291, 261, 271, 269, 249, 274, 267, 228, 242, 250, 247, 239, 247, 266, 285, 274, 243, 273, 240, 264, 254, 220, 237, 266, 254, 235, 266, 261, 245, 226, 257, 274, 265, 263, 258, 241, 274, 228, 262, 262, 242, 237, 247, 283, 238, 250, 248, 272, 292, 251, 243, 269, 252, 244, 238, 250, 265, 250, 252, 258, 279, 255, 240, 273, 271, 230, 265] payload_rows_per_dst=[5454, 5462, 5494, 5582, 5424, 5448, 5424, 5482] dup_rows_per_dst=[2682, 2748, 2692, 2824, 2762, 2642, 2716, 2700] combine_rows_per_src=[5454, 5462, 5494, 5582, 5424, 5448, 5424, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=155811840 dispatch_meta_bytes=227856 combine_payload_bytes=155811840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156039696 ingress_bytes=174686208 link_bytes=174686208 token_bytes=311851536 weight_movement_bytes=18874368 total_nvlink_bytes=330725904 weight_to_token_ratio=0.060524 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673757294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=229464 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157778008 ingress_bytes=195297280 link_bytes=195297280 token_bytes=315326552 weight_movement_bytes=37748736 total_nvlink_bytes=353075288 weight_to_token_ratio=0.119713 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871242445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38378 dup_rows=21754 dup_rows_remote=19062 dedup_ratio=0.331940 router_tokens_per_expert=[232, 277, 267, 258, 244, 268, 257, 238, 251, 253, 248, 274, 243, 240, 275, 260, 252, 243, 263, 236, 248, 261, 249, 251, 269, 256, 254, 254, 265, 243, 287, 256, 256, 226, 262, 222, 269, 276, 256, 223, 261, 256, 257, 252, 292, 273, 254, 255, 272, 273, 249, 253, 259, 247, 260, 264, 279, 249, 261, 245, 271, 283, 248, 253, 297, 279, 252, 245, 260, 229, 237, 249, 266, 261, 274, 218, 264, 267, 246, 254, 277, 238, 260, 275, 284, 249, 248, 269, 268, 264, 259, 246, 257, 239, 257, 276, 263, 263, 270, 241, 240, 249, 254, 258, 235, 258, 251, 255, 246, 265, 265, 235, 253, 259, 221, 285, 237, 268, 236, 226, 259, 278, 265, 259, 243, 265, 228, 246] payload_rows_per_dst=[5506, 5502, 5454, 5498, 5502, 5512, 5404, 5404] dup_rows_per_dst=[2664, 2672, 2726, 2834, 2694, 2820, 2692, 2652] combine_rows_per_src=[5506, 5502, 5454, 5498, 5502, 5512, 5404, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229304 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157138872 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048440 weight_movement_bytes=37748736 total_nvlink_bytes=351797176 weight_to_token_ratio=0.120200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38390 dup_rows=21658 dup_rows_remote=18896 dedup_ratio=0.330475 router_tokens_per_expert=[264, 261, 265, 268, 258, 255, 250, 241, 240, 239, 266, 282, 263, 251, 251, 242, 250, 273, 256, 265, 244, 268, 233, 249, 270, 223, 257, 245, 250, 257, 261, 277, 257, 245, 268, 239, 221, 259, 245, 252, 261, 234, 243, 253, 237, 211, 270, 242, 254, 251, 283, 258, 288, 264, 263, 284, 297, 233, 264, 261, 214, 279, 259, 259, 262, 271, 255, 234, 253, 245, 254, 267, 264, 264, 258, 278, 260, 247, 267, 246, 240, 261, 260, 251, 262, 248, 259, 246, 250, 267, 259, 263, 247, 285, 278, 251, 277, 256, 262, 226, 274, 281, 253, 248, 269, 253, 283, 264, 237, 273, 249, 283, 262, 220, 262, 274, 205, 261, 234, 257, 231, 300, 249, 239, 257, 255, 237, 263] payload_rows_per_dst=[5510, 5476, 5346, 5552, 5488, 5490, 5556, 5460] dup_rows_per_dst=[2682, 2680, 2528, 2870, 2762, 2764, 2820, 2552] combine_rows_per_src=[5510, 5476, 5346, 5552, 5488, 5490, 5556, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229760 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157426048 ingress_bytes=176070656 link_bytes=176070656 token_bytes=314622336 weight_movement_bytes=18874368 total_nvlink_bytes=333496704 weight_to_token_ratio=0.059991 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687022394e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38176 dup_rows=21868 dup_rows_remote=19088 dedup_ratio=0.333679 router_tokens_per_expert=[290, 269, 232, 268, 245, 253, 249, 267, 265, 249, 254, 257, 241, 275, 257, 264, 229, 246, 228, 253, 259, 272, 260, 266, 246, 249, 261, 265, 228, 246, 259, 276, 284, 250, 232, 265, 254, 260, 267, 241, 263, 283, 262, 274, 238, 270, 245, 248, 257, 263, 269, 240, 256, 246, 257, 280, 260, 275, 251, 256, 256, 254, 278, 265, 252, 255, 267, 244, 221, 272, 252, 257, 246, 242, 270, 271, 258, 286, 258, 267, 274, 250, 267, 235, 264, 254, 255, 263, 257, 275, 235, 259, 279, 246, 257, 257, 287, 244, 244, 258, 265, 256, 214, 275, 244, 249, 272, 244, 246, 237, 245, 272, 255, 231, 265, 248, 262, 228, 270, 254, 263, 263, 236, 227, 248, 243, 236, 265] payload_rows_per_dst=[5508, 5420, 5492, 5448, 5508, 5466, 5454, 5372] dup_rows_per_dst=[2762, 2666, 2780, 2878, 2728, 2788, 2650, 2616] combine_rows_per_src=[5508, 5420, 5492, 5448, 5508, 5466, 5454, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38362 dup_rows=21814 dup_rows_remote=19140 dedup_ratio=0.332855 router_tokens_per_expert=[276, 274, 265, 273, 259, 242, 247, 265, 266, 243, 268, 253, 269, 271, 273, 286, 242, 268, 268, 248, 261, 271, 296, 253, 248, 260, 269, 264, 244, 257, 261, 264, 257, 248, 254, 244, 278, 270, 299, 240, 252, 251, 254, 276, 268, 258, 263, 247, 248, 255, 233, 252, 246, 247, 248, 262, 256, 241, 272, 238, 219, 228, 249, 246, 266, 264, 250, 235, 257, 262, 285, 258, 266, 239, 259, 250, 252, 273, 257, 266, 285, 266, 240, 237, 254, 241, 267, 255, 268, 246, 248, 238, 223, 262, 254, 233, 269, 257, 258, 262, 234, 260, 260, 257, 250, 254, 283, 272, 280, 262, 231, 261, 235, 231, 271, 225, 227, 250, 257, 274, 248, 234, 275, 251, 232, 234, 259, 256] payload_rows_per_dst=[5568, 5568, 5538, 5330, 5482, 5360, 5548, 5328] dup_rows_per_dst=[2892, 2780, 2780, 2550, 2796, 2674, 2752, 2590] combine_rows_per_src=[5568, 5568, 5538, 5330, 5482, 5360, 5548, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38108 dup_rows=21878 dup_rows_remote=19042 dedup_ratio=0.333832 router_tokens_per_expert=[251, 244, 251, 259, 255, 273, 255, 253, 276, 276, 274, 278, 249, 253, 260, 262, 261, 279, 242, 263, 261, 244, 235, 272, 277, 257, 249, 240, 273, 250, 275, 218, 252, 255, 239, 242, 257, 248, 251, 248, 261, 275, 247, 277, 251, 263, 236, 270, 279, 263, 263, 246, 241, 246, 241, 262, 229, 262, 232, 251, 248, 259, 275, 252, 244, 230, 261, 270, 240, 252, 236, 247, 268, 236, 277, 232, 244, 240, 265, 258, 253, 249, 262, 263, 289, 266, 255, 260, 253, 243, 274, 282, 235, 241, 277, 280, 237, 269, 263, 250, 265, 272, 261, 252, 271, 248, 273, 263, 263, 279, 267, 236, 247, 255, 246, 225, 235, 251, 229, 253, 255, 269, 277, 268, 264, 261, 261, 235] payload_rows_per_dst=[5550, 5494, 5438, 5426, 5412, 5658, 5268, 5412] dup_rows_per_dst=[2836, 2846, 2668, 2672, 2588, 3106, 2512, 2650] combine_rows_per_src=[5550, 5494, 5438, 5426, 5412, 5658, 5268, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38168 dup_rows=21700 dup_rows_remote=18644 dedup_ratio=0.331116 router_tokens_per_expert=[265, 265, 220, 230, 266, 242, 230, 232, 256, 268, 261, 250, 250, 274, 269, 249, 233, 265, 262, 255, 248, 267, 264, 261, 294, 240, 258, 243, 251, 244, 280, 275, 238, 292, 273, 268, 265, 254, 263, 254, 273, 274, 252, 262, 262, 263, 230, 256, 239, 264, 247, 274, 263, 268, 229, 244, 258, 263, 243, 265, 257, 254, 262, 247, 283, 249, 245, 256, 257, 269, 265, 249, 283, 261, 233, 271, 244, 257, 251, 237, 246, 241, 261, 253, 271, 241, 228, 241, 258, 284, 259, 255, 243, 250, 256, 285, 247, 254, 245, 249, 275, 237, 242, 272, 259, 241, 246, 253, 270, 265, 268, 240, 252, 238, 250, 231, 270, 244, 270, 250, 249, 254, 239, 260, 263, 275, 271, 284] payload_rows_per_dst=[5392, 5412, 5318, 5516, 5492, 5518, 5520, 5668] dup_rows_per_dst=[2454, 2668, 2516, 2694, 2786, 2722, 2804, 3056] combine_rows_per_src=[5392, 5412, 5318, 5516, 5492, 5518, 5520, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38304 dup_rows=21714 dup_rows_remote=18986 dedup_ratio=0.331329 router_tokens_per_expert=[252, 219, 268, 264, 246, 260, 264, 260, 290, 257, 258, 269, 271, 271, 238, 285, 262, 248, 239, 264, 261, 256, 286, 271, 251, 274, 236, 253, 243, 271, 269, 247, 258, 243, 245, 260, 286, 252, 249, 249, 245, 246, 239, 299, 264, 248, 266, 231, 259, 252, 244, 242, 259, 256, 278, 234, 265, 231, 276, 283, 270, 273, 248, 253, 235, 264, 260, 276, 268, 241, 241, 256, 229, 276, 266, 270, 259, 278, 265, 257, 252, 226, 280, 282, 231, 246, 235, 234, 223, 224, 286, 274, 260, 273, 244, 230, 262, 256, 266, 268, 260, 253, 230, 251, 257, 255, 246, 239, 269, 252, 246, 263, 235, 255, 248, 276, 248, 267, 236, 258, 231, 256, 259, 222, 244, 278, 275, 260] payload_rows_per_dst=[5574, 5498, 5474, 5518, 5508, 5358, 5476, 5416] dup_rows_per_dst=[2770, 2764, 2686, 2728, 2774, 2642, 2670, 2680] combine_rows_per_src=[5574, 5498, 5474, 5518, 5508, 5358, 5476, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=229144 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157474584 ingress_bytes=194994176 link_bytes=194994176 token_bytes=314720024 weight_movement_bytes=37748736 total_nvlink_bytes=352468760 weight_to_token_ratio=0.119944 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868338251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229056 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156597952 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312966848 weight_movement_bytes=37748736 total_nvlink_bytes=350715584 weight_to_token_ratio=0.120616 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=230008 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157360760 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314491512 weight_movement_bytes=18874368 total_nvlink_bytes=333365880 weight_to_token_ratio=0.060016 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=228600 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156318968 ingress_bytes=212713472 link_bytes=212713472 token_bytes=312409336 weight_movement_bytes=56623104 total_nvlink_bytes=369032440 weight_to_token_ratio=0.181247 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038115827e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=227248 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156563376 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312899504 weight_movement_bytes=18874368 total_nvlink_bytes=331773872 weight_to_token_ratio=0.060321 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38312 dup_rows=21712 dup_rows_remote=18914 dedup_ratio=0.331299 router_tokens_per_expert=[248, 232, 255, 260, 262, 262, 282, 259, 281, 225, 258, 249, 260, 250, 278, 247, 245, 276, 263, 271, 277, 277, 272, 234, 253, 258, 272, 244, 266, 244, 242, 261, 263, 229, 252, 271, 255, 272, 246, 244, 247, 289, 251, 258, 286, 266, 192, 262, 252, 255, 252, 249, 253, 255, 244, 233, 277, 228, 269, 257, 252, 274, 279, 241, 260, 255, 247, 285, 273, 266, 242, 230, 245, 246, 251, 268, 245, 261, 277, 218, 274, 256, 244, 274, 266, 252, 249, 247, 267, 249, 268, 233, 261, 249, 253, 292, 251, 243, 255, 253, 263, 268, 256, 257, 250, 245, 239, 246, 262, 258, 258, 271, 256, 221, 249, 253, 248, 240, 254, 259, 264, 280, 278, 260, 265, 246, 262, 239] payload_rows_per_dst=[5420, 5512, 5466, 5466, 5456, 5518, 5542, 5444] dup_rows_per_dst=[2796, 2798, 2700, 2674, 2682, 2750, 2608, 2704] combine_rows_per_src=[5420, 5512, 5466, 5466, 5456, 5518, 5542, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229160 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122344 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015528 weight_movement_bytes=37748736 total_nvlink_bytes=351764264 weight_to_token_ratio=0.120213 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38326 dup_rows=21764 dup_rows_remote=18930 dedup_ratio=0.332092 router_tokens_per_expert=[276, 240, 246, 249, 238, 277, 270, 239, 247, 263, 259, 240, 257, 241, 244, 271, 257, 251, 274, 265, 280, 227, 264, 266, 247, 253, 265, 261, 244, 266, 264, 251, 285, 224, 284, 271, 256, 265, 282, 241, 261, 266, 274, 262, 242, 251, 258, 220, 229, 281, 276, 295, 236, 266, 258, 268, 265, 263, 263, 251, 272, 271, 254, 248, 244, 238, 251, 227, 241, 265, 248, 256, 232, 267, 243, 240, 261, 279, 235, 238, 252, 244, 244, 257, 266, 243, 257, 213, 274, 256, 242, 274, 268, 256, 229, 215, 261, 277, 249, 271, 243, 232, 290, 241, 264, 240, 244, 291, 269, 280, 251, 237, 276, 269, 250, 286, 256, 226, 270, 286, 259, 242, 251, 235, 247, 247, 269, 274] payload_rows_per_dst=[5428, 5476, 5518, 5530, 5402, 5438, 5446, 5534] dup_rows_per_dst=[2686, 2794, 2766, 2840, 2528, 2564, 2834, 2752] combine_rows_per_src=[5428, 5476, 5518, 5530, 5402, 5438, 5446, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228904 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157154856 ingress_bytes=232423424 link_bytes=232423424 token_bytes=314080808 weight_movement_bytes=75497472 total_nvlink_bytes=389578280 weight_to_token_ratio=0.240376 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226966889e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43518 payload_rows_remote=38034 dup_rows=22018 dup_rows_remote=19364 dedup_ratio=0.335968 router_tokens_per_expert=[255, 235, 275, 250, 277, 259, 269, 273, 242, 250, 259, 240, 304, 263, 276, 242, 288, 250, 234, 289, 271, 238, 251, 279, 255, 240, 242, 234, 247, 259, 283, 265, 266, 235, 258, 234, 251, 276, 229, 267, 240, 244, 257, 280, 249, 283, 250, 258, 267, 245, 256, 270, 279, 257, 270, 266, 268, 246, 250, 257, 243, 226, 243, 260, 242, 277, 257, 281, 268, 226, 269, 258, 262, 251, 265, 250, 222, 241, 259, 241, 257, 250, 270, 280, 269, 233, 256, 232, 247, 240, 271, 239, 234, 234, 268, 251, 254, 267, 249, 262, 257, 244, 261, 247, 270, 256, 283, 258, 271, 232, 263, 250, 243, 267, 245, 261, 242, 250, 241, 247, 264, 258, 250, 277, 256, 251, 264, 254] payload_rows_per_dst=[5534, 5416, 5368, 5414, 5484, 5406, 5502, 5394] dup_rows_per_dst=[2804, 2834, 2786, 2792, 2654, 2656, 2746, 2746] combine_rows_per_src=[5534, 5416, 5368, 5414, 5484, 5406, 5502, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229024 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212320 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195616 weight_movement_bytes=37748736 total_nvlink_bytes=351944352 weight_to_token_ratio=0.120144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38230 dup_rows=21796 dup_rows_remote=19152 dedup_ratio=0.332581 router_tokens_per_expert=[246, 227, 268, 227, 260, 211, 276, 268, 260, 265, 264, 266, 247, 264, 250, 244, 274, 245, 262, 252, 246, 260, 258, 247, 267, 252, 259, 255, 267, 234, 268, 252, 262, 294, 253, 236, 257, 251, 283, 237, 239, 275, 225, 248, 276, 256, 233, 252, 243, 259, 257, 269, 238, 237, 288, 287, 270, 250, 253, 250, 270, 249, 266, 212, 245, 249, 259, 270, 250, 265, 252, 276, 263, 270, 275, 242, 258, 257, 259, 239, 239, 259, 261, 273, 244, 262, 234, 276, 279, 267, 223, 276, 271, 251, 250, 251, 252, 230, 237, 243, 261, 250, 287, 274, 288, 274, 256, 283, 247, 256, 258, 244, 278, 249, 251, 259, 245, 268, 236, 263, 233, 246, 249, 240, 292, 236, 253, 269] payload_rows_per_dst=[5392, 5476, 5510, 5454, 5476, 5428, 5524, 5480] dup_rows_per_dst=[2694, 2720, 2644, 2742, 2782, 2804, 2756, 2654] combine_rows_per_src=[5392, 5476, 5510, 5454, 5476, 5428, 5524, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38446 dup_rows=21632 dup_rows_remote=18900 dedup_ratio=0.330078 router_tokens_per_expert=[250, 246, 219, 249, 294, 277, 258, 263, 260, 237, 246, 281, 275, 306, 261, 267, 247, 282, 254, 237, 268, 238, 254, 229, 221, 245, 263, 286, 259, 258, 251, 277, 253, 262, 255, 240, 247, 262, 206, 269, 248, 268, 297, 256, 249, 291, 265, 242, 231, 259, 264, 244, 245, 235, 275, 262, 257, 270, 250, 255, 252, 251, 226, 244, 267, 259, 257, 252, 260, 262, 279, 252, 254, 250, 252, 250, 266, 240, 255, 240, 240, 267, 246, 254, 254, 244, 284, 257, 260, 253, 269, 244, 255, 246, 282, 240, 287, 272, 238, 272, 254, 253, 271, 284, 235, 246, 269, 244, 226, 254, 237, 246, 247, 258, 264, 269, 216, 270, 265, 256, 229, 263, 260, 277, 257, 258, 239, 274] payload_rows_per_dst=[5584, 5404, 5562, 5458, 5502, 5458, 5460, 5476] dup_rows_per_dst=[2794, 2734, 2658, 2582, 2688, 2732, 2716, 2728] combine_rows_per_src=[5584, 5404, 5562, 5458, 5502, 5458, 5460, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38272 dup_rows=21916 dup_rows_remote=19214 dedup_ratio=0.334412 router_tokens_per_expert=[268, 263, 245, 260, 260, 237, 250, 244, 252, 273, 281, 266, 276, 256, 239, 250, 256, 244, 256, 262, 240, 249, 274, 233, 222, 262, 278, 281, 241, 259, 247, 250, 237, 240, 284, 238, 254, 281, 257, 238, 270, 264, 285, 260, 258, 262, 250, 254, 257, 260, 233, 252, 220, 268, 283, 241, 273, 243, 233, 277, 226, 249, 249, 269, 262, 244, 252, 236, 258, 227, 274, 265, 271, 258, 283, 254, 243, 246, 278, 258, 233, 260, 219, 261, 276, 269, 262, 271, 247, 260, 248, 277, 274, 230, 277, 252, 244, 256, 251, 253, 255, 272, 266, 261, 253, 271, 290, 251, 279, 245, 282, 250, 244, 254, 229, 247, 241, 245, 250, 255, 259, 264, 248, 246, 274, 259, 244, 266] payload_rows_per_dst=[5336, 5424, 5410, 5326, 5520, 5698, 5558, 5348] dup_rows_per_dst=[2664, 2684, 2672, 2420, 2888, 3038, 2848, 2702] combine_rows_per_src=[5336, 5424, 5410, 5326, 5520, 5698, 5558, 5348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38218 dup_rows=21756 dup_rows_remote=19034 dedup_ratio=0.331970 router_tokens_per_expert=[236, 243, 241, 274, 272, 275, 259, 235, 248, 221, 271, 249, 249, 259, 278, 266, 260, 248, 241, 228, 240, 267, 235, 257, 249, 243, 262, 254, 254, 259, 233, 247, 272, 281, 279, 256, 231, 258, 267, 258, 254, 263, 248, 238, 256, 234, 253, 267, 264, 258, 249, 248, 250, 278, 273, 266, 257, 255, 259, 253, 268, 266, 256, 270, 238, 256, 256, 243, 258, 290, 273, 247, 261, 251, 266, 256, 282, 240, 245, 255, 264, 268, 278, 268, 270, 259, 265, 248, 281, 236, 281, 250, 255, 230, 270, 260, 252, 243, 256, 252, 256, 221, 261, 252, 232, 251, 256, 276, 259, 274, 241, 276, 268, 248, 256, 255, 257, 227, 240, 246, 245, 267, 283, 248, 283, 253, 245, 251] payload_rows_per_dst=[5562, 5426, 5442, 5448, 5486, 5488, 5432, 5496] dup_rows_per_dst=[2722, 2838, 2656, 2582, 2748, 2878, 2684, 2648] combine_rows_per_src=[5562, 5426, 5442, 5448, 5486, 5488, 5432, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38216 dup_rows=21840 dup_rows_remote=18976 dedup_ratio=0.333252 router_tokens_per_expert=[276, 228, 245, 272, 264, 272, 263, 266, 262, 282, 229, 238, 288, 248, 236, 256, 256, 250, 252, 262, 254, 247, 242, 247, 229, 230, 244, 249, 252, 247, 268, 254, 233, 257, 265, 276, 245, 239, 273, 227, 252, 231, 235, 259, 242, 297, 263, 276, 264, 280, 245, 248, 247, 261, 280, 264, 275, 251, 280, 233, 264, 267, 250, 263, 273, 268, 280, 245, 232, 231, 241, 260, 274, 266, 261, 263, 252, 248, 284, 249, 241, 324, 248, 257, 283, 281, 275, 254, 248, 248, 241, 255, 206, 272, 258, 289, 243, 277, 255, 229, 255, 263, 240, 267, 251, 254, 248, 267, 237, 258, 287, 279, 236, 294, 263, 247, 257, 252, 233, 235, 259, 239, 281, 223, 240, 246, 268, 228] payload_rows_per_dst=[5496, 5364, 5440, 5480, 5554, 5544, 5474, 5344] dup_rows_per_dst=[2754, 2602, 2700, 2864, 2700, 2816, 2746, 2658] combine_rows_per_src=[5496, 5364, 5440, 5480, 5554, 5544, 5474, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=229592 combine_payload_bytes=155787264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156016856 ingress_bytes=174661632 link_bytes=174661632 token_bytes=311804120 weight_movement_bytes=18874368 total_nvlink_bytes=330678488 weight_to_token_ratio=0.060533 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673521819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229528 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156819608 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313409688 weight_movement_bytes=37748736 total_nvlink_bytes=351158424 weight_to_token_ratio=0.120445 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229384 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157704200 ingress_bytes=176349184 link_bytes=176349184 token_bytes=315179016 weight_movement_bytes=18874368 total_nvlink_bytes=334053384 weight_to_token_ratio=0.059885 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689691112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38318 dup_rows=21728 dup_rows_remote=19122 dedup_ratio=0.331543 router_tokens_per_expert=[234, 302, 229, 251, 232, 275, 259, 254, 261, 256, 276, 275, 255, 248, 261, 248, 279, 242, 266, 222, 249, 280, 227, 235, 259, 256, 261, 247, 291, 217, 249, 268, 255, 242, 274, 258, 258, 245, 266, 225, 252, 246, 258, 263, 280, 261, 249, 261, 255, 243, 233, 257, 275, 248, 242, 239, 266, 271, 242, 253, 232, 267, 252, 284, 278, 248, 258, 263, 256, 266, 265, 268, 251, 261, 266, 258, 224, 228, 257, 232, 269, 235, 243, 269, 251, 248, 284, 227, 251, 249, 261, 266, 262, 246, 275, 259, 277, 265, 261, 259, 248, 284, 275, 251, 246, 247, 257, 248, 251, 242, 273, 296, 265, 256, 256, 264, 263, 250, 243, 249, 249, 268, 245, 248, 252, 261, 241, 288] payload_rows_per_dst=[5496, 5490, 5490, 5470, 5428, 5460, 5526, 5448] dup_rows_per_dst=[2736, 2606, 2696, 2648, 2730, 2730, 2834, 2748] combine_rows_per_src=[5496, 5490, 5490, 5470, 5428, 5460, 5526, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229944 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156992056 ingress_bytes=156762112 link_bytes=156992056 token_bytes=313754168 weight_movement_bytes=0 total_nvlink_bytes=313754168 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.504220636e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229008 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156769936 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313310864 weight_movement_bytes=18874368 total_nvlink_bytes=332185232 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228768 combine_payload_bytes=156532736 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156761504 ingress_bytes=213155840 link_bytes=213155840 token_bytes=313294240 weight_movement_bytes=56623104 total_nvlink_bytes=369917344 weight_to_token_ratio=0.180735 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042354380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38172 dup_rows=21884 dup_rows_remote=19288 dedup_ratio=0.333923 router_tokens_per_expert=[257, 270, 245, 234, 257, 280, 290, 243, 231, 275, 264, 275, 268, 242, 236, 265, 286, 256, 242, 275, 262, 266, 247, 249, 240, 269, 232, 284, 237, 241, 258, 242, 273, 262, 249, 263, 259, 258, 256, 233, 248, 252, 247, 230, 275, 281, 230, 231, 272, 263, 258, 248, 229, 260, 253, 230, 276, 255, 263, 257, 265, 242, 270, 244, 227, 272, 253, 261, 251, 269, 250, 254, 280, 276, 279, 266, 220, 251, 247, 275, 266, 245, 253, 264, 273, 262, 224, 251, 256, 240, 253, 243, 264, 302, 248, 253, 234, 238, 252, 283, 250, 232, 265, 236, 241, 259, 272, 281, 246, 247, 247, 255, 258, 278, 275, 242, 268, 276, 257, 265, 255, 264, 245, 259, 247, 255, 256, 252] payload_rows_per_dst=[5522, 5434, 5438, 5450, 5462, 5438, 5480, 5428] dup_rows_per_dst=[2742, 2738, 2656, 2720, 2800, 2756, 2596, 2876] combine_rows_per_src=[5522, 5434, 5438, 5450, 5462, 5438, 5480, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229760 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157180288 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130816 weight_movement_bytes=18874368 total_nvlink_bytes=333005184 weight_to_token_ratio=0.060084 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229840 combine_payload_bytes=156352512 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156582352 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312934864 weight_movement_bytes=37748736 total_nvlink_bytes=350683600 weight_to_token_ratio=0.120628 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38348 dup_rows=21622 dup_rows_remote=18944 dedup_ratio=0.329926 router_tokens_per_expert=[241, 258, 252, 257, 230, 252, 255, 247, 260, 248, 263, 250, 263, 282, 265, 242, 290, 265, 267, 289, 264, 231, 247, 253, 230, 254, 240, 248, 243, 253, 255, 253, 243, 249, 242, 262, 297, 250, 230, 278, 258, 248, 270, 256, 265, 256, 257, 256, 263, 297, 233, 257, 267, 285, 256, 240, 236, 243, 259, 249, 271, 254, 236, 259, 257, 257, 289, 255, 259, 263, 274, 235, 254, 259, 242, 262, 247, 251, 245, 273, 294, 277, 257, 255, 272, 259, 260, 265, 245, 260, 211, 277, 247, 226, 268, 259, 232, 249, 273, 234, 271, 263, 248, 251, 248, 251, 231, 256, 273, 254, 255, 259, 249, 279, 244, 265, 230, 271, 257, 257, 277, 257, 240, 241, 251, 247, 266, 266] payload_rows_per_dst=[5504, 5444, 5516, 5504, 5566, 5440, 5436, 5504] dup_rows_per_dst=[2626, 2720, 2718, 2706, 2678, 2824, 2660, 2690] combine_rows_per_src=[5504, 5444, 5516, 5504, 5566, 5440, 5436, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38354 dup_rows=21692 dup_rows_remote=19004 dedup_ratio=0.330994 router_tokens_per_expert=[286, 251, 227, 262, 280, 251, 222, 266, 243, 264, 237, 238, 273, 227, 252, 253, 266, 254, 230, 263, 253, 259, 275, 263, 256, 241, 246, 253, 248, 265, 289, 264, 250, 269, 253, 235, 253, 269, 278, 272, 260, 225, 249, 244, 228, 259, 255, 290, 295, 251, 253, 272, 258, 275, 273, 252, 260, 251, 274, 265, 262, 261, 227, 290, 254, 264, 254, 244, 258, 235, 255, 266, 250, 246, 266, 248, 267, 257, 262, 244, 279, 279, 232, 266, 267, 245, 258, 245, 255, 230, 263, 239, 253, 252, 252, 252, 244, 239, 259, 237, 272, 238, 270, 261, 267, 255, 255, 258, 249, 277, 268, 263, 261, 280, 252, 274, 258, 265, 258, 268, 251, 236, 241, 255, 225, 238, 263, 229] payload_rows_per_dst=[5406, 5484, 5490, 5624, 5434, 5468, 5516, 5422] dup_rows_per_dst=[2658, 2766, 2688, 2814, 2706, 2666, 2708, 2686] combine_rows_per_src=[5406, 5484, 5490, 5624, 5434, 5468, 5516, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38416 dup_rows=21630 dup_rows_remote=18986 dedup_ratio=0.330048 router_tokens_per_expert=[266, 270, 279, 251, 271, 257, 265, 252, 274, 292, 283, 280, 230, 233, 256, 235, 269, 226, 257, 253, 247, 259, 241, 247, 263, 236, 232, 256, 299, 232, 246, 236, 253, 256, 282, 252, 257, 272, 250, 245, 260, 266, 263, 242, 271, 238, 281, 260, 252, 272, 251, 263, 260, 246, 254, 278, 282, 266, 234, 269, 282, 261, 261, 244, 254, 250, 265, 255, 281, 255, 237, 288, 236, 261, 240, 263, 258, 253, 241, 257, 240, 267, 223, 289, 241, 259, 275, 248, 262, 257, 250, 257, 254, 272, 242, 231, 230, 241, 265, 228, 233, 274, 236, 262, 251, 234, 277, 228, 263, 253, 273, 280, 284, 269, 235, 258, 269, 253, 257, 250, 265, 255, 245, 234, 258, 239, 244, 248] payload_rows_per_dst=[5560, 5390, 5566, 5506, 5456, 5490, 5488, 5450] dup_rows_per_dst=[2828, 2608, 2730, 2844, 2732, 2644, 2568, 2676] combine_rows_per_src=[5560, 5390, 5566, 5506, 5456, 5490, 5488, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38138 dup_rows=21800 dup_rows_remote=18844 dedup_ratio=0.332642 router_tokens_per_expert=[239, 222, 251, 266, 241, 275, 249, 229, 249, 270, 269, 261, 257, 238, 248, 254, 275, 265, 252, 248, 245, 273, 260, 264, 231, 210, 280, 268, 243, 225, 274, 266, 241, 258, 266, 252, 242, 247, 274, 219, 271, 259, 254, 268, 257, 266, 279, 257, 274, 274, 257, 267, 236, 274, 258, 240, 280, 240, 269, 236, 253, 263, 255, 263, 250, 245, 262, 249, 271, 264, 239, 281, 217, 248, 273, 260, 272, 233, 258, 273, 248, 255, 253, 241, 249, 240, 267, 246, 234, 277, 260, 253, 278, 250, 269, 264, 260, 275, 265, 281, 259, 252, 235, 245, 234, 250, 286, 267, 252, 246, 231, 225, 262, 268, 245, 260, 262, 288, 272, 249, 266, 248, 250, 245, 283, 272, 260, 250] payload_rows_per_dst=[5356, 5502, 5412, 5282, 5600, 5576, 5410, 5598] dup_rows_per_dst=[2680, 2656, 2742, 2358, 3034, 2658, 2716, 2956] combine_rows_per_src=[5356, 5502, 5412, 5282, 5600, 5576, 5410, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38248 dup_rows=21782 dup_rows_remote=18984 dedup_ratio=0.332367 router_tokens_per_expert=[221, 220, 239, 264, 276, 277, 272, 269, 264, 248, 252, 255, 270, 284, 278, 259, 242, 269, 271, 234, 257, 250, 259, 264, 238, 253, 263, 246, 246, 269, 278, 237, 256, 255, 254, 249, 235, 281, 269, 244, 239, 228, 247, 288, 266, 252, 296, 266, 268, 248, 276, 253, 277, 259, 239, 256, 246, 227, 222, 264, 268, 262, 245, 257, 260, 276, 266, 265, 248, 256, 264, 265, 238, 253, 229, 274, 245, 277, 262, 245, 234, 251, 273, 256, 279, 251, 234, 256, 274, 293, 248, 251, 255, 246, 251, 286, 289, 256, 253, 255, 235, 259, 227, 257, 260, 249, 270, 250, 264, 219, 270, 235, 250, 265, 239, 268, 275, 254, 239, 263, 228, 273, 224, 263, 242, 265, 243, 252] payload_rows_per_dst=[5506, 5586, 5350, 5446, 5416, 5592, 5458, 5400] dup_rows_per_dst=[2798, 2922, 2544, 2688, 2830, 2810, 2638, 2552] combine_rows_per_src=[5506, 5586, 5350, 5446, 5416, 5592, 5458, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38208 dup_rows=21734 dup_rows_remote=18952 dedup_ratio=0.331635 router_tokens_per_expert=[250, 255, 237, 252, 245, 279, 242, 254, 252, 232, 250, 268, 243, 232, 241, 269, 262, 264, 275, 274, 244, 224, 237, 267, 256, 245, 248, 269, 262, 227, 234, 259, 279, 286, 251, 259, 254, 292, 266, 287, 244, 268, 241, 275, 251, 249, 254, 252, 268, 265, 272, 255, 254, 270, 255, 271, 261, 246, 264, 272, 299, 254, 227, 255, 268, 279, 258, 273, 253, 270, 256, 271, 261, 247, 236, 252, 259, 251, 238, 270, 227, 249, 253, 254, 257, 249, 279, 268, 276, 235, 244, 242, 264, 247, 230, 255, 262, 267, 250, 265, 249, 264, 258, 265, 236, 249, 236, 251, 280, 251, 250, 277, 268, 246, 247, 255, 266, 255, 258, 281, 248, 273, 254, 239, 234, 267, 210, 242] payload_rows_per_dst=[5362, 5436, 5588, 5594, 5464, 5416, 5464, 5478] dup_rows_per_dst=[2640, 2658, 2828, 2782, 2820, 2642, 2756, 2608] combine_rows_per_src=[5362, 5436, 5588, 5594, 5464, 5416, 5464, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229168 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157302576 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314375984 weight_movement_bytes=18874368 total_nvlink_bytes=333250352 weight_to_token_ratio=0.060038 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229432 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157327416 ingress_bytes=194846720 link_bytes=194846720 token_bytes=314425400 weight_movement_bytes=37748736 total_nvlink_bytes=352174136 weight_to_token_ratio=0.120056 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866925400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38356 dup_rows=21802 dup_rows_remote=19124 dedup_ratio=0.332672 router_tokens_per_expert=[224, 259, 232, 231, 258, 267, 245, 287, 265, 268, 222, 274, 249, 281, 261, 266, 272, 252, 252, 256, 257, 258, 251, 252, 253, 234, 258, 264, 229, 232, 270, 238, 251, 236, 287, 273, 272, 247, 299, 237, 265, 285, 254, 236, 280, 257, 252, 261, 263, 279, 282, 240, 251, 241, 251, 275, 227, 249, 269, 274, 266, 272, 262, 260, 234, 251, 257, 238, 244, 230, 296, 259, 225, 252, 261, 248, 257, 268, 264, 258, 254, 267, 230, 227, 274, 230, 264, 278, 263, 270, 234, 252, 283, 226, 260, 245, 249, 264, 239, 271, 293, 242, 262, 279, 247, 264, 254, 227, 256, 267, 216, 276, 288, 252, 247, 264, 250, 240, 238, 262, 252, 262, 270, 262, 269, 250, 268, 219] payload_rows_per_dst=[5510, 5378, 5516, 5542, 5364, 5502, 5438, 5484] dup_rows_per_dst=[2668, 2678, 2868, 2780, 2720, 2612, 2774, 2702] combine_rows_per_src=[5510, 5378, 5516, 5542, 5364, 5502, 5438, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229608 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581544 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933480 weight_movement_bytes=18874368 total_nvlink_bytes=333807848 weight_to_token_ratio=0.059931 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=227928 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156441176 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312654424 weight_movement_bytes=18874368 total_nvlink_bytes=331528792 weight_to_token_ratio=0.060368 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228928 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156892736 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313556544 weight_movement_bytes=56623104 total_nvlink_bytes=370179648 weight_to_token_ratio=0.180583 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228640 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156728608 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313228576 weight_movement_bytes=18874368 total_nvlink_bytes=332102944 weight_to_token_ratio=0.060257 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38140 dup_rows=21906 dup_rows_remote=19028 dedup_ratio=0.334259 router_tokens_per_expert=[240, 253, 269, 293, 270, 262, 273, 257, 241, 283, 247, 246, 264, 260, 252, 259, 258, 237, 265, 262, 266, 268, 257, 251, 260, 246, 265, 264, 253, 251, 248, 296, 262, 273, 245, 241, 290, 241, 259, 268, 252, 256, 250, 268, 253, 244, 239, 251, 257, 244, 250, 262, 257, 247, 235, 259, 229, 251, 280, 247, 258, 247, 245, 253, 219, 260, 261, 251, 254, 249, 258, 259, 271, 250, 261, 235, 238, 248, 247, 253, 252, 261, 263, 256, 251, 249, 259, 244, 249, 263, 257, 239, 227, 253, 241, 235, 269, 273, 244, 270, 251, 256, 290, 257, 271, 250, 254, 238, 250, 256, 290, 265, 267, 256, 270, 230, 266, 238, 279, 299, 235, 273, 247, 283, 255, 265, 237, 242] payload_rows_per_dst=[5506, 5508, 5422, 5390, 5394, 5428, 5490, 5492] dup_rows_per_dst=[2832, 2786, 2762, 2652, 2634, 2570, 2878, 2792] combine_rows_per_src=[5506, 5508, 5422, 5390, 5394, 5428, 5490, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229920 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157336096 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314442272 weight_movement_bytes=18874368 total_nvlink_bytes=333316640 weight_to_token_ratio=0.060025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228672 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156450112 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312671552 weight_movement_bytes=37748736 total_nvlink_bytes=350420288 weight_to_token_ratio=0.120730 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38460 dup_rows=21688 dup_rows_remote=18986 dedup_ratio=0.330933 router_tokens_per_expert=[261, 256, 254, 265, 260, 228, 262, 286, 262, 255, 253, 262, 248, 231, 267, 244, 247, 284, 238, 261, 259, 245, 256, 248, 237, 297, 286, 264, 263, 231, 236, 242, 261, 264, 245, 255, 240, 254, 249, 257, 269, 270, 250, 256, 247, 239, 268, 254, 249, 244, 235, 283, 302, 280, 256, 272, 262, 238, 238, 236, 254, 235, 249, 255, 251, 243, 266, 258, 248, 262, 230, 257, 291, 242, 279, 256, 238, 239, 223, 262, 258, 248, 238, 242, 278, 275, 271, 252, 241, 259, 232, 267, 274, 274, 237, 260, 259, 270, 253, 266, 230, 261, 278, 261, 243, 258, 272, 251, 248, 275, 257, 282, 257, 252, 232, 234, 271, 251, 271, 261, 267, 236, 268, 280, 286, 250, 240, 243] payload_rows_per_dst=[5464, 5462, 5558, 5422, 5388, 5484, 5598, 5472] dup_rows_per_dst=[2724, 2726, 2598, 2754, 2702, 2728, 2730, 2726] combine_rows_per_src=[5464, 5462, 5558, 5422, 5388, 5484, 5598, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38176 dup_rows=21962 dup_rows_remote=19328 dedup_ratio=0.335114 router_tokens_per_expert=[254, 253, 247, 250, 258, 263, 259, 256, 229, 287, 260, 256, 280, 241, 280, 268, 270, 251, 242, 269, 254, 253, 247, 241, 250, 270, 253, 256, 250, 257, 262, 284, 255, 230, 246, 278, 256, 241, 270, 277, 267, 238, 254, 220, 227, 271, 236, 250, 268, 262, 277, 244, 243, 266, 210, 236, 260, 239, 230, 243, 256, 235, 229, 242, 245, 279, 264, 266, 255, 236, 262, 262, 241, 243, 259, 274, 238, 258, 291, 284, 233, 272, 266, 260, 259, 239, 281, 249, 286, 275, 252, 276, 254, 232, 253, 288, 258, 268, 281, 235, 262, 265, 256, 262, 250, 258, 256, 249, 241, 259, 283, 245, 255, 273, 255, 253, 281, 254, 250, 262, 276, 241, 238, 245, 261, 261, 246, 251] payload_rows_per_dst=[5454, 5504, 5398, 5302, 5506, 5494, 5450, 5466] dup_rows_per_dst=[2828, 2714, 2634, 2578, 2808, 2856, 2806, 2738] combine_rows_per_src=[5454, 5504, 5398, 5302, 5506, 5494, 5450, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38218 dup_rows=21838 dup_rows_remote=18998 dedup_ratio=0.333221 router_tokens_per_expert=[244, 285, 236, 249, 241, 259, 277, 284, 223, 248, 218, 244, 249, 258, 276, 250, 231, 238, 240, 276, 256, 259, 282, 271, 260, 259, 232, 248, 271, 257, 258, 211, 254, 262, 254, 251, 287, 250, 288, 276, 250, 287, 239, 247, 240, 244, 235, 263, 260, 267, 242, 270, 236, 237, 271, 236, 258, 260, 255, 243, 264, 274, 242, 253, 237, 222, 262, 238, 265, 252, 231, 237, 258, 249, 249, 278, 243, 270, 269, 255, 279, 253, 240, 245, 286, 264, 268, 285, 251, 257, 253, 249, 259, 281, 258, 271, 233, 279, 256, 259, 260, 253, 266, 283, 256, 267, 257, 265, 240, 268, 242, 232, 278, 252, 270, 262, 251, 271, 262, 277, 253, 232, 258, 249, 254, 257, 268, 259] payload_rows_per_dst=[5326, 5522, 5536, 5514, 5338, 5492, 5490, 5480] dup_rows_per_dst=[2480, 2576, 2696, 2898, 2620, 2906, 2822, 2840] combine_rows_per_src=[5326, 5522, 5536, 5514, 5338, 5492, 5490, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43988 payload_rows_remote=38546 dup_rows=21548 dup_rows_remote=18798 dedup_ratio=0.328796 router_tokens_per_expert=[257, 257, 269, 251, 256, 247, 277, 241, 269, 281, 267, 260, 258, 256, 269, 266, 267, 260, 254, 268, 240, 233, 242, 253, 251, 258, 269, 262, 256, 238, 253, 262, 270, 239, 230, 254, 248, 277, 278, 256, 248, 249, 279, 245, 259, 230, 252, 239, 272, 269, 263, 238, 253, 239, 240, 241, 250, 256, 281, 275, 273, 286, 244, 253, 255, 240, 252, 259, 239, 254, 277, 245, 273, 244, 233, 237, 247, 248, 261, 245, 250, 266, 265, 223, 312, 258, 246, 264, 275, 264, 245, 242, 240, 248, 256, 242, 278, 260, 258, 257, 253, 259, 279, 256, 250, 256, 231, 249, 255, 245, 288, 287, 245, 253, 275, 263, 235, 260, 244, 249, 284, 265, 261, 274, 238, 238, 244, 241] payload_rows_per_dst=[5656, 5502, 5490, 5484, 5406, 5442, 5600, 5408] dup_rows_per_dst=[2706, 2630, 2616, 2782, 2612, 2750, 2722, 2730] combine_rows_per_src=[5656, 5502, 5490, 5484, 5406, 5442, 5600, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38462 dup_rows=21578 dup_rows_remote=18804 dedup_ratio=0.329254 router_tokens_per_expert=[266, 264, 233, 246, 243, 243, 258, 227, 264, 279, 267, 256, 272, 255, 228, 232, 259, 262, 264, 260, 281, 253, 248, 268, 282, 255, 244, 263, 273, 279, 269, 260, 244, 222, 256, 258, 261, 245, 237, 234, 299, 245, 228, 249, 238, 263, 275, 265, 249, 242, 258, 258, 283, 261, 267, 261, 260, 268, 254, 275, 250, 242, 237, 270, 235, 247, 250, 229, 287, 254, 270, 238, 255, 255, 276, 228, 259, 257, 254, 269, 262, 264, 256, 249, 261, 262, 270, 269, 242, 259, 272, 215, 264, 276, 236, 253, 287, 261, 236, 229, 254, 248, 279, 252, 272, 267, 287, 246, 260, 234, 251, 280, 277, 240, 246, 248, 272, 240, 281, 253, 234, 256, 256, 252, 246, 264, 251, 229] payload_rows_per_dst=[5426, 5678, 5426, 5496, 5474, 5500, 5474, 5484] dup_rows_per_dst=[2640, 2762, 2612, 2774, 2652, 2720, 2812, 2606] combine_rows_per_src=[5426, 5678, 5426, 5496, 5474, 5500, 5474, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38138 dup_rows=21940 dup_rows_remote=19218 dedup_ratio=0.334778 router_tokens_per_expert=[240, 266, 245, 258, 265, 233, 251, 277, 247, 281, 261, 272, 248, 255, 232, 259, 249, 256, 256, 253, 253, 249, 287, 238, 237, 249, 275, 253, 253, 257, 259, 240, 278, 253, 246, 244, 251, 292, 259, 262, 240, 247, 253, 276, 249, 232, 255, 257, 239, 239, 246, 271, 253, 262, 239, 251, 263, 257, 248, 250, 245, 257, 262, 263, 255, 253, 267, 248, 252, 220, 229, 283, 270, 257, 284, 284, 292, 246, 259, 258, 241, 276, 267, 253, 270, 262, 250, 242, 251, 248, 252, 250, 271, 251, 254, 234, 260, 275, 269, 224, 281, 239, 261, 259, 259, 243, 274, 263, 264, 246, 229, 267, 293, 263, 235, 260, 255, 283, 282, 231, 241, 258, 255, 247, 284, 241, 254, 251] payload_rows_per_dst=[5458, 5402, 5560, 5408, 5384, 5488, 5408, 5488] dup_rows_per_dst=[2722, 2726, 2960, 2682, 2598, 2826, 2648, 2778] combine_rows_per_src=[5458, 5402, 5560, 5408, 5384, 5488, 5408, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157532160 dispatch_meta_bytes=229784 combine_payload_bytes=157532160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157761944 ingress_bytes=176406528 link_bytes=176406528 token_bytes=315294104 weight_movement_bytes=18874368 total_nvlink_bytes=334168472 weight_to_token_ratio=0.059863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690240554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=230016 combine_payload_bytes=156368896 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156598912 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312967808 weight_movement_bytes=37748736 total_nvlink_bytes=350716544 weight_to_token_ratio=0.120615 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38182 dup_rows=21830 dup_rows_remote=18960 dedup_ratio=0.333099 router_tokens_per_expert=[253, 280, 298, 254, 253, 251, 262, 266, 230, 251, 280, 251, 264, 245, 239, 240, 241, 259, 244, 274, 296, 241, 256, 260, 220, 277, 264, 270, 285, 288, 238, 284, 264, 276, 265, 263, 258, 255, 238, 265, 253, 237, 254, 261, 268, 244, 248, 257, 248, 267, 233, 254, 265, 275, 252, 246, 267, 243, 236, 273, 254, 267, 242, 267, 256, 250, 243, 238, 275, 248, 233, 254, 281, 244, 278, 251, 229, 266, 275, 264, 229, 273, 217, 266, 229, 274, 272, 265, 249, 249, 278, 242, 258, 274, 245, 237, 263, 268, 286, 244, 253, 241, 264, 228, 258, 262, 278, 278, 250, 246, 252, 247, 254, 242, 208, 264, 252, 262, 252, 259, 238, 237, 253, 257, 263, 251, 264, 243] payload_rows_per_dst=[5484, 5524, 5466, 5484, 5480, 5404, 5470, 5394] dup_rows_per_dst=[2750, 2870, 2746, 2694, 2690, 2710, 2766, 2604] combine_rows_per_src=[5484, 5524, 5466, 5484, 5480, 5404, 5470, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228864 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156769792 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313310720 weight_movement_bytes=37748736 total_nvlink_bytes=351059456 weight_to_token_ratio=0.120483 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=229376 combine_payload_bytes=157884416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158113792 ingress_bytes=176758784 link_bytes=176758784 token_bytes=315998208 weight_movement_bytes=18874368 total_nvlink_bytes=334872576 weight_to_token_ratio=0.059729 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693615698e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157540352 dispatch_meta_bytes=229064 combine_payload_bytes=157540352 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157769416 ingress_bytes=214163456 link_bytes=214163456 token_bytes=315309768 weight_movement_bytes=56623104 total_nvlink_bytes=371932872 weight_to_token_ratio=0.179579 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.052008860e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229424 combine_payload_bytes=156213248 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156442672 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312655920 weight_movement_bytes=18874368 total_nvlink_bytes=331530288 weight_to_token_ratio=0.060368 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38294 dup_rows=21798 dup_rows_remote=19088 dedup_ratio=0.332611 router_tokens_per_expert=[221, 265, 260, 280, 262, 270, 255, 248, 289, 262, 263, 272, 259, 276, 240, 254, 243, 247, 253, 243, 232, 250, 255, 237, 242, 264, 257, 244, 260, 240, 242, 248, 258, 270, 273, 239, 249, 265, 245, 266, 244, 268, 244, 248, 249, 246, 270, 262, 277, 261, 252, 263, 277, 260, 269, 266, 247, 246, 242, 262, 247, 263, 266, 215, 263, 266, 238, 264, 240, 255, 264, 283, 247, 233, 230, 287, 245, 250, 251, 288, 232, 261, 252, 266, 248, 288, 253, 260, 279, 252, 231, 263, 259, 249, 285, 253, 230, 273, 244, 242, 238, 241, 302, 233, 253, 270, 255, 260, 262, 271, 261, 242, 252, 257, 264, 239, 252, 260, 262, 267, 252, 233, 277, 266, 247, 277, 238, 271] payload_rows_per_dst=[5508, 5382, 5488, 5500, 5470, 5512, 5444, 5434] dup_rows_per_dst=[2844, 2532, 2704, 2726, 2738, 2750, 2710, 2794] combine_rows_per_src=[5508, 5382, 5488, 5500, 5470, 5512, 5444, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=228568 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622040 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313015512 weight_movement_bytes=18874368 total_nvlink_bytes=331889880 weight_to_token_ratio=0.060299 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229528 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081752 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933976 weight_movement_bytes=18874368 total_nvlink_bytes=332808344 weight_to_token_ratio=0.060122 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38260 dup_rows=21768 dup_rows_remote=18934 dedup_ratio=0.332153 router_tokens_per_expert=[273, 231, 246, 250, 284, 228, 259, 260, 272, 238, 276, 273, 252, 221, 227, 265, 262, 231, 231, 250, 272, 242, 260, 256, 238, 251, 240, 243, 242, 253, 266, 279, 239, 288, 239, 246, 253, 245, 253, 260, 250, 257, 234, 266, 244, 296, 257, 266, 248, 269, 220, 258, 256, 293, 219, 237, 248, 285, 300, 254, 264, 270, 281, 259, 233, 266, 263, 263, 262, 296, 235, 287, 280, 238, 251, 259, 265, 260, 239, 274, 285, 260, 226, 271, 244, 235, 251, 247, 244, 253, 245, 254, 266, 243, 286, 243, 253, 239, 247, 256, 263, 246, 245, 239, 237, 267, 256, 250, 282, 273, 283, 229, 252, 239, 254, 252, 256, 242, 259, 275, 234, 281, 277, 252, 260, 256, 307, 258] payload_rows_per_dst=[5364, 5430, 5462, 5546, 5508, 5464, 5490, 5504] dup_rows_per_dst=[2746, 2602, 2724, 2776, 2834, 2642, 2640, 2804] combine_rows_per_src=[5364, 5430, 5462, 5546, 5508, 5464, 5490, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38228 dup_rows=21832 dup_rows_remote=19160 dedup_ratio=0.333130 router_tokens_per_expert=[263, 294, 239, 242, 252, 226, 239, 252, 267, 232, 256, 250, 269, 261, 255, 235, 221, 237, 263, 247, 273, 276, 252, 255, 249, 248, 247, 262, 242, 241, 266, 243, 228, 259, 251, 252, 264, 234, 265, 262, 260, 248, 243, 268, 264, 261, 254, 261, 252, 267, 256, 260, 255, 269, 243, 257, 254, 270, 267, 266, 255, 244, 246, 237, 275, 227, 259, 213, 276, 297, 280, 246, 261, 265, 263, 260, 235, 266, 251, 272, 232, 237, 266, 271, 229, 228, 255, 273, 271, 301, 273, 265, 264, 262, 274, 271, 277, 244, 252, 222, 250, 267, 254, 239, 245, 253, 299, 240, 283, 244, 243, 245, 270, 254, 234, 273, 267, 263, 255, 281, 299, 274, 261, 243, 243, 254, 248, 248] payload_rows_per_dst=[5396, 5334, 5476, 5480, 5558, 5458, 5450, 5552] dup_rows_per_dst=[2668, 2710, 2672, 2716, 2734, 2886, 2664, 2782] combine_rows_per_src=[5396, 5334, 5476, 5480, 5558, 5458, 5450, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38108 dup_rows=21886 dup_rows_remote=18966 dedup_ratio=0.333954 router_tokens_per_expert=[237, 263, 239, 252, 277, 225, 256, 231, 275, 281, 273, 236, 245, 267, 242, 259, 257, 258, 239, 271, 238, 271, 258, 242, 258, 243, 267, 247, 262, 251, 238, 261, 267, 213, 248, 262, 271, 259, 272, 292, 243, 233, 259, 266, 270, 217, 240, 254, 302, 244, 227, 251, 293, 278, 247, 248, 250, 260, 229, 277, 246, 266, 231, 262, 232, 253, 287, 270, 259, 238, 260, 249, 253, 275, 249, 255, 249, 273, 247, 269, 229, 267, 244, 234, 261, 267, 274, 245, 266, 244, 253, 264, 260, 264, 300, 237, 252, 231, 284, 260, 254, 280, 260, 236, 249, 267, 241, 242, 265, 226, 256, 273, 262, 237, 287, 242, 259, 245, 260, 249, 283, 279, 250, 272, 244, 270, 268, 262] payload_rows_per_dst=[5374, 5468, 5400, 5434, 5358, 5540, 5534, 5542] dup_rows_per_dst=[2618, 2654, 2732, 2624, 2616, 2840, 2882, 2920] combine_rows_per_src=[5374, 5468, 5400, 5434, 5358, 5540, 5534, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38362 dup_rows=21686 dup_rows_remote=18946 dedup_ratio=0.330902 router_tokens_per_expert=[238, 255, 255, 232, 258, 248, 261, 213, 247, 249, 256, 270, 241, 261, 261, 267, 239, 266, 241, 250, 238, 261, 266, 273, 247, 263, 258, 261, 269, 258, 286, 247, 256, 264, 273, 276, 256, 270, 287, 277, 263, 236, 275, 243, 238, 287, 262, 253, 245, 263, 269, 259, 266, 241, 252, 264, 248, 258, 263, 253, 277, 259, 238, 259, 252, 215, 251, 276, 293, 259, 282, 243, 255, 255, 275, 245, 259, 254, 249, 253, 264, 251, 269, 232, 257, 241, 277, 257, 251, 232, 241, 270, 212, 265, 279, 259, 252, 252, 243, 237, 276, 278, 279, 246, 248, 260, 252, 242, 246, 260, 233, 259, 250, 273, 281, 250, 252, 245, 249, 251, 288, 244, 229, 286, 234, 244, 222, 269] payload_rows_per_dst=[5426, 5550, 5566, 5488, 5538, 5412, 5418, 5452] dup_rows_per_dst=[2598, 2696, 2866, 2740, 2694, 2702, 2708, 2682] combine_rows_per_src=[5426, 5550, 5566, 5488, 5538, 5412, 5418, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38290 dup_rows=21796 dup_rows_remote=19132 dedup_ratio=0.332581 router_tokens_per_expert=[259, 256, 289, 254, 289, 237, 267, 246, 273, 276, 248, 245, 271, 268, 272, 299, 280, 264, 255, 255, 249, 242, 243, 219, 241, 267, 259, 233, 250, 275, 252, 256, 282, 231, 271, 234, 247, 228, 243, 239, 267, 256, 234, 287, 246, 247, 267, 231, 253, 299, 262, 270, 274, 220, 259, 263, 283, 233, 241, 268, 247, 251, 264, 244, 284, 231, 256, 240, 245, 272, 279, 252, 228, 243, 247, 265, 279, 251, 286, 254, 261, 234, 280, 239, 273, 264, 284, 251, 254, 251, 240, 263, 235, 260, 230, 238, 259, 264, 253, 267, 247, 232, 250, 287, 230, 216, 263, 238, 269, 262, 246, 259, 250, 283, 249, 245, 254, 258, 245, 244, 256, 280, 275, 282, 246, 254, 257, 249] payload_rows_per_dst=[5572, 5486, 5370, 5500, 5504, 5450, 5426, 5432] dup_rows_per_dst=[2926, 2594, 2650, 2762, 2720, 2664, 2658, 2822] combine_rows_per_src=[5572, 5486, 5370, 5500, 5504, 5450, 5426, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228776 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156941736 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313654696 weight_movement_bytes=56623104 total_nvlink_bytes=370277800 weight_to_token_ratio=0.180527 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38292 dup_rows=21754 dup_rows_remote=19102 dedup_ratio=0.331940 router_tokens_per_expert=[254, 268, 251, 243, 260, 266, 230, 282, 246, 261, 270, 255, 249, 226, 251, 259, 254, 276, 276, 241, 246, 238, 261, 258, 231, 259, 245, 259, 270, 253, 256, 254, 274, 238, 242, 253, 256, 256, 255, 275, 232, 235, 258, 261, 252, 257, 265, 255, 274, 263, 260, 260, 246, 272, 245, 240, 248, 253, 251, 290, 272, 286, 271, 265, 250, 266, 235, 252, 237, 275, 266, 249, 251, 290, 253, 255, 285, 263, 268, 248, 265, 239, 233, 234, 225, 274, 255, 278, 246, 286, 252, 260, 217, 251, 252, 247, 233, 264, 268, 283, 233, 256, 286, 262, 266, 259, 255, 253, 296, 238, 268, 254, 240, 253, 270, 258, 224, 263, 243, 272, 279, 221, 244, 242, 237, 254, 259, 270] payload_rows_per_dst=[5490, 5472, 5586, 5428, 5528, 5368, 5548, 5362] dup_rows_per_dst=[2652, 2682, 2822, 2684, 2758, 2660, 2800, 2696] combine_rows_per_src=[5490, 5472, 5586, 5428, 5528, 5368, 5548, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229552 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156811440 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313393328 weight_movement_bytes=18874368 total_nvlink_bytes=332267696 weight_to_token_ratio=0.060226 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38350 dup_rows=21656 dup_rows_remote=18922 dedup_ratio=0.330444 router_tokens_per_expert=[259, 258, 260, 260, 235, 276, 267, 271, 268, 262, 250, 244, 255, 273, 236, 268, 276, 228, 259, 230, 266, 213, 302, 266, 274, 243, 283, 250, 283, 252, 254, 252, 255, 232, 244, 212, 248, 266, 230, 242, 260, 262, 282, 287, 253, 254, 255, 246, 268, 264, 240, 272, 277, 268, 239, 248, 250, 249, 245, 245, 296, 246, 262, 252, 266, 267, 242, 210, 254, 254, 286, 257, 275, 242, 224, 227, 288, 240, 257, 251, 242, 264, 262, 254, 239, 256, 259, 261, 286, 230, 253, 255, 257, 262, 254, 277, 257, 268, 263, 237, 256, 241, 250, 286, 251, 256, 275, 251, 269, 236, 235, 244, 269, 232, 257, 247, 254, 262, 265, 249, 259, 263, 246, 268, 265, 246, 275, 263] payload_rows_per_dst=[5504, 5530, 5538, 5464, 5436, 5422, 5508, 5478] dup_rows_per_dst=[2780, 2734, 2696, 2778, 2644, 2798, 2642, 2584] combine_rows_per_src=[5504, 5530, 5538, 5464, 5436, 5422, 5508, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229232 combine_payload_bytes=157130752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157359984 ingress_bytes=232628224 link_bytes=232628224 token_bytes=314490736 weight_movement_bytes=75497472 total_nvlink_bytes=389988208 weight_to_token_ratio=0.240063 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228929182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=228296 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156318664 ingress_bytes=193839104 link_bytes=193839104 token_bytes=312409032 weight_movement_bytes=37748736 total_nvlink_bytes=350157768 weight_to_token_ratio=0.120831 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857270920e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229688 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065528 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313901368 weight_movement_bytes=18874368 total_nvlink_bytes=332775736 weight_to_token_ratio=0.060128 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38334 dup_rows=21778 dup_rows_remote=19146 dedup_ratio=0.332306 router_tokens_per_expert=[228, 237, 251, 247, 261, 245, 277, 245, 276, 248, 235, 257, 247, 270, 249, 269, 248, 269, 267, 279, 235, 272, 248, 229, 250, 245, 250, 238, 267, 269, 263, 250, 268, 228, 254, 249, 264, 253, 242, 262, 246, 256, 240, 271, 273, 254, 261, 259, 259, 290, 243, 238, 253, 257, 274, 278, 238, 310, 240, 235, 280, 230, 251, 277, 277, 287, 266, 258, 262, 233, 274, 245, 251, 255, 243, 267, 241, 243, 248, 243, 240, 252, 246, 265, 260, 254, 254, 272, 267, 252, 241, 268, 258, 241, 251, 293, 251, 260, 259, 258, 237, 266, 254, 253, 279, 239, 227, 211, 254, 249, 276, 255, 250, 263, 272, 237, 261, 298, 265, 262, 251, 284, 246, 259, 282, 236, 249, 264] payload_rows_per_dst=[5480, 5402, 5452, 5484, 5528, 5488, 5424, 5500] dup_rows_per_dst=[2604, 2756, 2708, 2822, 2658, 2740, 2632, 2858] combine_rows_per_src=[5480, 5402, 5452, 5484, 5528, 5488, 5424, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229576 combine_payload_bytes=156844032 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157073608 ingress_bytes=194592768 link_bytes=194592768 token_bytes=313917640 weight_movement_bytes=37748736 total_nvlink_bytes=351666376 weight_to_token_ratio=0.120250 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864492157e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229088 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157310688 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392288 weight_movement_bytes=18874368 total_nvlink_bytes=333266656 weight_to_token_ratio=0.060034 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229920 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245984 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314262048 weight_movement_bytes=18874368 total_nvlink_bytes=333136416 weight_to_token_ratio=0.060059 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38044 dup_rows=21958 dup_rows_remote=19166 dedup_ratio=0.335052 router_tokens_per_expert=[282, 252, 266, 252, 266, 267, 259, 244, 259, 256, 245, 260, 224, 271, 240, 260, 278, 220, 260, 225, 244, 247, 223, 240, 251, 266, 237, 266, 268, 229, 263, 261, 257, 287, 255, 258, 248, 262, 220, 246, 257, 218, 262, 258, 279, 250, 233, 238, 243, 253, 263, 264, 286, 284, 256, 278, 255, 261, 257, 263, 257, 276, 252, 236, 265, 277, 267, 242, 228, 248, 267, 247, 284, 252, 274, 263, 289, 272, 241, 247, 258, 235, 234, 259, 264, 248, 246, 280, 275, 258, 240, 264, 254, 249, 264, 268, 245, 243, 243, 261, 237, 266, 270, 259, 276, 285, 269, 242, 291, 235, 253, 240, 260, 270, 243, 268, 232, 246, 271, 256, 244, 273, 271, 250, 223, 261, 259, 274] payload_rows_per_dst=[5480, 5320, 5382, 5528, 5534, 5448, 5512, 5374] dup_rows_per_dst=[2726, 2636, 2674, 2840, 2792, 2744, 2718, 2828] combine_rows_per_src=[5480, 5320, 5382, 5528, 5534, 5448, 5512, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43954 payload_rows_remote=38476 dup_rows=21582 dup_rows_remote=18882 dedup_ratio=0.329315 router_tokens_per_expert=[254, 273, 252, 239, 251, 249, 244, 233, 251, 262, 251, 259, 269, 259, 276, 259, 274, 251, 294, 255, 278, 262, 268, 308, 273, 265, 256, 261, 270, 277, 278, 243, 260, 279, 263, 267, 236, 264, 252, 248, 255, 250, 253, 255, 234, 253, 272, 248, 243, 263, 252, 261, 248, 225, 262, 224, 235, 252, 275, 256, 250, 242, 283, 247, 240, 243, 261, 259, 225, 232, 270, 270, 235, 248, 247, 276, 255, 244, 238, 256, 255, 248, 254, 290, 295, 251, 263, 246, 248, 266, 250, 253, 258, 252, 272, 259, 248, 257, 271, 265, 257, 263, 241, 249, 251, 259, 258, 248, 262, 240, 263, 266, 245, 260, 262, 259, 236, 241, 235, 242, 259, 246, 241, 280, 233, 273, 254, 244] payload_rows_per_dst=[5510, 5690, 5478, 5394, 5416, 5506, 5486, 5474] dup_rows_per_dst=[2652, 2936, 2700, 2642, 2582, 2814, 2710, 2546] combine_rows_per_src=[5510, 5690, 5478, 5394, 5416, 5506, 5486, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38240 dup_rows=21886 dup_rows_remote=19084 dedup_ratio=0.333954 router_tokens_per_expert=[230, 218, 263, 227, 251, 237, 250, 280, 278, 243, 264, 274, 257, 247, 275, 242, 266, 243, 257, 274, 250, 242, 242, 270, 263, 247, 272, 252, 244, 254, 256, 252, 240, 266, 263, 256, 233, 248, 260, 236, 272, 233, 251, 252, 287, 238, 248, 231, 257, 237, 246, 281, 253, 247, 235, 251, 239, 242, 281, 261, 265, 290, 277, 244, 264, 256, 303, 273, 246, 238, 284, 273, 249, 245, 284, 224, 253, 265, 278, 239, 252, 263, 255, 247, 249, 261, 263, 254, 247, 254, 250, 276, 273, 244, 254, 244, 261, 257, 259, 259, 256, 243, 283, 256, 268, 229, 247, 257, 244, 274, 245, 289, 254, 266, 263, 289, 258, 255, 234, 236, 250, 257, 270, 266, 246, 290, 239, 268] payload_rows_per_dst=[5370, 5438, 5384, 5410, 5568, 5480, 5484, 5516] dup_rows_per_dst=[2702, 2730, 2644, 2802, 2780, 2692, 2770, 2766] combine_rows_per_src=[5370, 5438, 5384, 5410, 5568, 5480, 5484, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=37984 dup_rows=21900 dup_rows_remote=19088 dedup_ratio=0.334167 router_tokens_per_expert=[251, 259, 244, 236, 264, 252, 255, 260, 275, 288, 293, 272, 248, 230, 244, 275, 256, 265, 257, 267, 242, 261, 284, 277, 264, 248, 256, 239, 219, 256, 256, 266, 264, 254, 271, 236, 228, 252, 252, 262, 280, 248, 262, 247, 234, 272, 251, 271, 266, 279, 268, 219, 228, 269, 268, 299, 256, 251, 257, 262, 283, 274, 221, 282, 255, 260, 260, 257, 282, 257, 269, 262, 242, 266, 257, 258, 258, 225, 244, 278, 239, 236, 249, 251, 255, 253, 254, 257, 270, 236, 245, 235, 259, 259, 246, 263, 237, 245, 269, 267, 248, 234, 261, 249, 279, 261, 260, 241, 270, 256, 275, 262, 270, 241, 254, 234, 235, 252, 242, 216, 261, 247, 249, 262, 264, 225, 266, 274] payload_rows_per_dst=[5552, 5392, 5430, 5198, 5544, 5368, 5500, 5652] dup_rows_per_dst=[2740, 2642, 2738, 2472, 2930, 2806, 2760, 2812] combine_rows_per_src=[5552, 5392, 5430, 5198, 5544, 5368, 5500, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38392 dup_rows=21630 dup_rows_remote=18916 dedup_ratio=0.330048 router_tokens_per_expert=[241, 281, 254, 296, 238, 266, 257, 262, 271, 266, 293, 288, 246, 263, 239, 262, 260, 254, 248, 271, 246, 241, 272, 268, 238, 244, 262, 268, 246, 222, 269, 245, 240, 217, 261, 260, 231, 243, 232, 269, 244, 248, 283, 262, 260, 242, 241, 255, 260, 231, 258, 222, 280, 256, 252, 265, 242, 256, 236, 222, 246, 246, 279, 225, 270, 285, 251, 270, 259, 257, 240, 255, 256, 236, 281, 248, 258, 271, 250, 280, 266, 256, 264, 279, 259, 275, 245, 256, 249, 254, 258, 277, 262, 234, 249, 231, 260, 276, 280, 266, 225, 251, 246, 239, 229, 239, 253, 273, 258, 252, 258, 291, 278, 236, 260, 238, 278, 262, 258, 245, 271, 252, 252, 277, 280, 250, 276, 237] payload_rows_per_dst=[5630, 5408, 5420, 5400, 5562, 5514, 5462, 5510] dup_rows_per_dst=[2816, 2700, 2556, 2552, 2772, 2714, 2730, 2790] combine_rows_per_src=[5630, 5408, 5420, 5400, 5562, 5514, 5462, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=155828224 dispatch_meta_bytes=228840 combine_payload_bytes=155828224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156057064 ingress_bytes=193576960 link_bytes=193576960 token_bytes=311885288 weight_movement_bytes=37748736 total_nvlink_bytes=349634024 weight_to_token_ratio=0.121034 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854759185e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229432 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157827128 ingress_bytes=195346432 link_bytes=195346432 token_bytes=315424824 weight_movement_bytes=37748736 total_nvlink_bytes=353173560 weight_to_token_ratio=0.119676 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871713395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38184 dup_rows=21812 dup_rows_remote=19110 dedup_ratio=0.332825 router_tokens_per_expert=[252, 280, 252, 268, 268, 218, 272, 257, 263, 248, 272, 237, 286, 240, 257, 251, 283, 273, 246, 243, 264, 244, 290, 275, 254, 245, 235, 243, 257, 265, 257, 258, 289, 283, 239, 259, 269, 239, 257, 259, 258, 252, 243, 239, 245, 256, 235, 248, 246, 236, 277, 305, 263, 262, 259, 247, 237, 248, 269, 251, 274, 254, 257, 265, 251, 254, 220, 243, 253, 266, 275, 252, 230, 258, 229, 286, 233, 262, 268, 254, 260, 236, 247, 251, 277, 243, 235, 246, 275, 246, 256, 253, 248, 225, 244, 247, 270, 236, 255, 264, 238, 245, 251, 254, 249, 263, 253, 268, 236, 264, 261, 238, 279, 258, 259, 263, 288, 263, 244, 248, 248, 259, 264, 255, 261, 290, 280, 268] payload_rows_per_dst=[5540, 5602, 5448, 5478, 5408, 5322, 5340, 5586] dup_rows_per_dst=[2702, 2840, 2692, 2822, 2660, 2656, 2572, 2868] combine_rows_per_src=[5540, 5602, 5448, 5478, 5408, 5322, 5340, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38280 dup_rows=21768 dup_rows_remote=19094 dedup_ratio=0.332153 router_tokens_per_expert=[248, 258, 247, 237, 252, 248, 265, 238, 270, 229, 268, 266, 243, 258, 263, 272, 248, 247, 261, 278, 249, 279, 223, 260, 275, 244, 254, 250, 265, 244, 272, 232, 228, 253, 275, 265, 269, 244, 239, 242, 263, 250, 241, 277, 264, 242, 242, 241, 254, 271, 252, 267, 267, 233, 267, 243, 251, 255, 266, 239, 280, 251, 264, 279, 260, 253, 257, 241, 243, 257, 274, 256, 259, 249, 259, 278, 262, 267, 243, 248, 259, 257, 270, 246, 270, 236, 272, 254, 258, 229, 244, 288, 265, 229, 284, 261, 230, 267, 265, 272, 256, 284, 268, 273, 257, 261, 268, 269, 267, 271, 238, 248, 260, 250, 262, 232, 254, 260, 225, 259, 263, 273, 253, 237, 250, 260, 249, 242] payload_rows_per_dst=[5472, 5488, 5464, 5534, 5446, 5428, 5572, 5364] dup_rows_per_dst=[2652, 2674, 2606, 2744, 2766, 2816, 2816, 2694] combine_rows_per_src=[5472, 5488, 5464, 5534, 5446, 5428, 5572, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229296 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156860336 ingress_bytes=213254144 link_bytes=213254144 token_bytes=313491376 weight_movement_bytes=56623104 total_nvlink_bytes=370114480 weight_to_token_ratio=0.180621 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043296280e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155582464 dispatch_meta_bytes=228288 combine_payload_bytes=155582464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155810752 ingress_bytes=174456832 link_bytes=174456832 token_bytes=311393216 weight_movement_bytes=18874368 total_nvlink_bytes=330267584 weight_to_token_ratio=0.060613 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671559526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229232 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157482864 ingress_bytes=195002368 link_bytes=195002368 token_bytes=314736496 weight_movement_bytes=37748736 total_nvlink_bytes=352485232 weight_to_token_ratio=0.119938 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868416743e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38444 dup_rows=21630 dup_rows_remote=18972 dedup_ratio=0.330048 router_tokens_per_expert=[238, 254, 221, 255, 261, 266, 270, 261, 279, 262, 276, 248, 249, 230, 262, 249, 250, 244, 231, 267, 246, 283, 263, 276, 254, 236, 259, 215, 273, 239, 254, 238, 229, 261, 252, 266, 264, 237, 228, 268, 292, 281, 275, 259, 245, 252, 255, 221, 269, 262, 272, 278, 267, 242, 260, 291, 264, 244, 287, 259, 275, 278, 248, 261, 258, 242, 244, 262, 257, 259, 264, 269, 264, 279, 273, 262, 227, 238, 275, 272, 233, 280, 238, 257, 223, 245, 265, 262, 284, 269, 282, 230, 260, 252, 249, 240, 251, 278, 242, 261, 237, 258, 242, 268, 258, 214, 268, 262, 257, 253, 251, 260, 270, 271, 241, 270, 238, 262, 233, 225, 248, 245, 240, 265, 256, 266, 275, 238] payload_rows_per_dst=[5518, 5400, 5454, 5516, 5604, 5496, 5462, 5456] dup_rows_per_dst=[2644, 2656, 2716, 2966, 2686, 2674, 2658, 2630] combine_rows_per_src=[5518, 5400, 5454, 5516, 5604, 5496, 5462, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229176 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156630840 ingress_bytes=213024768 link_bytes=213024768 token_bytes=313032504 weight_movement_bytes=56623104 total_nvlink_bytes=369655608 weight_to_token_ratio=0.180886 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041098512e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229496 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157024376 ingress_bytes=213417984 link_bytes=213417984 token_bytes=313819256 weight_movement_bytes=56623104 total_nvlink_bytes=370442360 weight_to_token_ratio=0.180432 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044866114e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229664 combine_payload_bytes=157466624 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157696288 ingress_bytes=195215360 link_bytes=195215360 token_bytes=315162912 weight_movement_bytes=37748736 total_nvlink_bytes=352911648 weight_to_token_ratio=0.119775 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870457528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38224 dup_rows=21810 dup_rows_remote=19192 dedup_ratio=0.332794 router_tokens_per_expert=[250, 248, 276, 264, 267, 244, 253, 267, 235, 272, 262, 233, 275, 252, 286, 252, 254, 234, 249, 257, 243, 245, 272, 272, 223, 224, 253, 257, 277, 268, 242, 242, 246, 273, 268, 251, 278, 280, 235, 241, 238, 249, 248, 249, 275, 281, 277, 248, 250, 268, 271, 255, 251, 242, 254, 278, 260, 263, 269, 270, 254, 256, 259, 257, 262, 254, 237, 239, 245, 261, 270, 272, 264, 259, 232, 237, 247, 247, 281, 253, 244, 277, 276, 253, 289, 238, 270, 235, 251, 251, 249, 236, 242, 260, 247, 271, 250, 233, 250, 239, 238, 253, 245, 244, 266, 266, 258, 254, 236, 223, 256, 278, 248, 269, 252, 269, 266, 253, 240, 261, 248, 257, 280, 286, 261, 247, 283, 268] payload_rows_per_dst=[5548, 5390, 5462, 5502, 5502, 5448, 5376, 5498] dup_rows_per_dst=[2724, 2634, 2812, 2812, 2618, 2730, 2602, 2878] combine_rows_per_src=[5548, 5390, 5462, 5502, 5502, 5448, 5376, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38170 dup_rows=21850 dup_rows_remote=19116 dedup_ratio=0.333405 router_tokens_per_expert=[238, 252, 252, 273, 261, 277, 249, 271, 232, 258, 251, 255, 238, 251, 243, 261, 255, 251, 272, 246, 261, 261, 232, 261, 264, 247, 245, 261, 283, 242, 244, 257, 251, 252, 248, 280, 261, 249, 262, 289, 282, 250, 250, 261, 264, 243, 236, 247, 256, 254, 239, 251, 242, 271, 282, 259, 256, 256, 238, 260, 250, 262, 259, 273, 253, 263, 246, 274, 244, 249, 272, 242, 277, 226, 259, 240, 254, 244, 261, 275, 267, 240, 266, 281, 268, 266, 267, 248, 240, 251, 217, 269, 248, 217, 226, 255, 260, 264, 246, 265, 265, 274, 250, 248, 276, 238, 287, 268, 253, 284, 265, 227, 237, 246, 252, 255, 256, 264, 257, 264, 262, 284, 264, 281, 238, 227, 278, 251] payload_rows_per_dst=[5386, 5514, 5516, 5480, 5444, 5448, 5400, 5498] dup_rows_per_dst=[2738, 2650, 2734, 2736, 2714, 2604, 2940, 2734] combine_rows_per_src=[5386, 5514, 5516, 5480, 5444, 5448, 5400, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43532 payload_rows_remote=38066 dup_rows=22004 dup_rows_remote=19160 dedup_ratio=0.335754 router_tokens_per_expert=[274, 267, 242, 246, 274, 234, 244, 260, 260, 253, 229, 257, 275, 254, 246, 235, 269, 265, 252, 225, 287, 257, 245, 261, 261, 246, 233, 241, 239, 256, 236, 243, 266, 233, 266, 245, 255, 271, 248, 239, 231, 257, 263, 277, 244, 267, 257, 244, 255, 239, 257, 286, 256, 249, 290, 243, 252, 275, 258, 268, 261, 243, 263, 260, 270, 287, 266, 279, 239, 270, 244, 277, 279, 275, 267, 238, 260, 250, 267, 255, 275, 253, 262, 272, 265, 298, 256, 272, 251, 247, 259, 239, 246, 254, 249, 246, 272, 275, 263, 235, 246, 251, 274, 267, 261, 234, 239, 255, 248, 237, 270, 252, 275, 246, 262, 235, 263, 266, 287, 236, 258, 239, 246, 213, 260, 247, 250, 255] payload_rows_per_dst=[5448, 5380, 5464, 5466, 5518, 5480, 5422, 5354] dup_rows_per_dst=[2652, 2652, 2662, 2844, 2928, 2808, 2736, 2722] combine_rows_per_src=[5448, 5380, 5464, 5466, 5518, 5480, 5422, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38056 dup_rows=21850 dup_rows_remote=19000 dedup_ratio=0.333405 router_tokens_per_expert=[256, 243, 256, 268, 257, 277, 255, 268, 254, 239, 244, 257, 254, 235, 259, 241, 246, 226, 274, 271, 256, 250, 271, 276, 281, 252, 276, 242, 255, 252, 232, 260, 262, 256, 258, 240, 229, 253, 240, 244, 240, 263, 292, 279, 248, 256, 275, 253, 240, 261, 272, 271, 249, 265, 236, 269, 265, 249, 271, 259, 237, 252, 247, 270, 237, 271, 246, 255, 238, 261, 250, 236, 253, 254, 270, 246, 247, 227, 247, 285, 256, 259, 239, 248, 234, 237, 256, 240, 270, 233, 235, 268, 263, 246, 254, 274, 254, 266, 268, 265, 288, 243, 291, 271, 254, 249, 250, 230, 250, 247, 268, 266, 262, 260, 254, 265, 271, 277, 267, 263, 247, 250, 244, 261, 251, 273, 282, 262] payload_rows_per_dst=[5468, 5508, 5502, 5254, 5340, 5538, 5446, 5630] dup_rows_per_dst=[2658, 2732, 2674, 2418, 2780, 2966, 2772, 2850] combine_rows_per_src=[5468, 5508, 5502, 5254, 5340, 5538, 5446, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229664 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156795168 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360672 weight_movement_bytes=18874368 total_nvlink_bytes=332235040 weight_to_token_ratio=0.060232 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229144 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573464 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917784 weight_movement_bytes=18874368 total_nvlink_bytes=331792152 weight_to_token_ratio=0.060317 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38252 dup_rows=21832 dup_rows_remote=19088 dedup_ratio=0.333130 router_tokens_per_expert=[234, 266, 273, 253, 236, 252, 252, 257, 241, 259, 258, 255, 261, 274, 241, 222, 236, 256, 259, 252, 225, 263, 250, 258, 260, 237, 258, 240, 259, 271, 260, 283, 257, 256, 281, 250, 240, 265, 285, 264, 254, 266, 275, 248, 246, 234, 250, 239, 275, 290, 278, 264, 237, 254, 262, 262, 243, 288, 254, 288, 279, 247, 247, 268, 243, 277, 263, 253, 258, 261, 232, 250, 235, 251, 264, 249, 265, 244, 260, 251, 246, 281, 242, 242, 291, 248, 259, 230, 291, 225, 257, 266, 248, 269, 244, 259, 258, 279, 255, 243, 262, 218, 247, 283, 268, 265, 269, 240, 260, 242, 252, 259, 239, 250, 255, 234, 262, 242, 289, 223, 269, 260, 234, 266, 274, 259, 257, 254] payload_rows_per_dst=[5416, 5462, 5468, 5548, 5402, 5452, 5500, 5456] dup_rows_per_dst=[2652, 2672, 2752, 2924, 2710, 2744, 2700, 2678] combine_rows_per_src=[5416, 5462, 5468, 5548, 5402, 5452, 5500, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38154 dup_rows=21886 dup_rows_remote=19188 dedup_ratio=0.333954 router_tokens_per_expert=[256, 229, 246, 250, 254, 277, 252, 250, 270, 282, 230, 243, 245, 283, 247, 274, 255, 268, 250, 259, 276, 241, 284, 267, 284, 276, 256, 256, 264, 246, 231, 237, 255, 275, 259, 293, 248, 275, 257, 269, 230, 259, 282, 233, 282, 231, 254, 257, 252, 263, 262, 232, 245, 261, 235, 258, 279, 270, 254, 263, 235, 273, 269, 248, 232, 265, 266, 259, 229, 248, 264, 249, 257, 254, 236, 257, 273, 250, 251, 237, 269, 250, 242, 256, 237, 263, 229, 268, 261, 250, 243, 269, 263, 237, 246, 265, 275, 277, 249, 242, 251, 263, 260, 280, 265, 278, 242, 278, 271, 239, 235, 269, 246, 244, 250, 242, 251, 249, 260, 278, 257, 255, 230, 251, 260, 254, 274, 222] payload_rows_per_dst=[5496, 5564, 5472, 5468, 5456, 5420, 5358, 5416] dup_rows_per_dst=[2698, 3050, 2852, 2712, 2598, 2676, 2670, 2630] combine_rows_per_src=[5496, 5564, 5472, 5468, 5456, 5420, 5358, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38306 dup_rows=21762 dup_rows_remote=19026 dedup_ratio=0.332062 router_tokens_per_expert=[239, 271, 267, 251, 248, 256, 278, 263, 241, 242, 235, 289, 260, 255, 251, 256, 266, 257, 270, 253, 230, 269, 279, 232, 276, 243, 248, 239, 260, 239, 264, 277, 260, 256, 251, 224, 268, 237, 268, 266, 267, 271, 265, 240, 234, 269, 238, 256, 290, 285, 287, 259, 261, 266, 236, 224, 280, 253, 228, 253, 250, 246, 235, 279, 246, 220, 231, 274, 258, 265, 240, 237, 223, 263, 272, 259, 273, 265, 246, 255, 277, 253, 266, 246, 251, 265, 233, 271, 263, 273, 250, 245, 268, 233, 247, 256, 271, 234, 248, 259, 250, 255, 271, 220, 276, 254, 223, 254, 267, 260, 272, 262, 255, 249, 289, 249, 256, 251, 271, 264, 278, 263, 262, 277, 270, 220, 261, 247] payload_rows_per_dst=[5552, 5468, 5448, 5474, 5398, 5530, 5426, 5478] dup_rows_per_dst=[2652, 2736, 2692, 2790, 2656, 2664, 2726, 2846] combine_rows_per_src=[5552, 5468, 5448, 5474, 5398, 5530, 5426, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228904 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156147240 ingress_bytes=193667072 link_bytes=193667072 token_bytes=312065576 weight_movement_bytes=37748736 total_nvlink_bytes=349814312 weight_to_token_ratio=0.120964 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855622594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=228224 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156105600 ingress_bytes=212500480 link_bytes=212500480 token_bytes=311982976 weight_movement_bytes=56623104 total_nvlink_bytes=368606080 weight_to_token_ratio=0.181494 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036075043e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38334 dup_rows=21838 dup_rows_remote=19176 dedup_ratio=0.333221 router_tokens_per_expert=[250, 261, 259, 248, 252, 264, 276, 276, 270, 241, 257, 232, 236, 251, 259, 258, 276, 242, 274, 246, 256, 269, 244, 234, 250, 268, 273, 265, 238, 256, 255, 264, 238, 260, 256, 268, 268, 261, 230, 238, 253, 255, 296, 270, 264, 265, 226, 257, 242, 248, 272, 232, 261, 259, 261, 265, 248, 289, 279, 266, 248, 277, 237, 271, 251, 270, 266, 250, 273, 261, 289, 256, 249, 249, 253, 256, 247, 265, 245, 253, 265, 238, 239, 257, 257, 257, 242, 247, 252, 257, 242, 243, 237, 286, 265, 260, 258, 273, 252, 234, 271, 227, 228, 257, 258, 257, 257, 253, 249, 250, 262, 227, 225, 290, 253, 248, 238, 269, 245, 255, 276, 261, 277, 257, 265, 265, 246, 248] payload_rows_per_dst=[5416, 5500, 5406, 5572, 5504, 5486, 5364, 5450] dup_rows_per_dst=[2764, 2720, 2804, 2738, 2762, 2602, 2662, 2786] combine_rows_per_src=[5416, 5500, 5406, 5572, 5504, 5486, 5364, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229360 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909552 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589744 weight_movement_bytes=18874368 total_nvlink_bytes=332464112 weight_to_token_ratio=0.060188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229368 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156508152 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312786936 weight_movement_bytes=18874368 total_nvlink_bytes=331661304 weight_to_token_ratio=0.060343 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229328 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130704 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032080 weight_movement_bytes=18874368 total_nvlink_bytes=332906448 weight_to_token_ratio=0.060103 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=230040 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157246104 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314262168 weight_movement_bytes=18874368 total_nvlink_bytes=333136536 weight_to_token_ratio=0.060059 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38264 dup_rows=21780 dup_rows_remote=19058 dedup_ratio=0.332336 router_tokens_per_expert=[264, 257, 295, 237, 237, 276, 252, 250, 250, 238, 267, 264, 274, 250, 251, 262, 254, 232, 239, 294, 268, 259, 239, 275, 242, 263, 267, 267, 252, 241, 252, 280, 243, 248, 239, 259, 251, 243, 252, 256, 251, 268, 261, 215, 233, 259, 278, 249, 271, 233, 249, 269, 278, 263, 234, 267, 244, 291, 225, 240, 245, 249, 249, 239, 271, 228, 261, 232, 247, 244, 260, 289, 249, 248, 288, 249, 271, 267, 223, 280, 268, 275, 251, 272, 250, 248, 262, 284, 268, 240, 244, 247, 276, 256, 248, 226, 272, 242, 261, 239, 250, 241, 245, 257, 222, 252, 259, 248, 269, 265, 255, 256, 257, 256, 262, 256, 255, 279, 277, 282, 245, 258, 255, 268, 276, 247, 264, 277] payload_rows_per_dst=[5498, 5442, 5402, 5460, 5492, 5508, 5420, 5534] dup_rows_per_dst=[2750, 2806, 2608, 2632, 2722, 2722, 2646, 2894] combine_rows_per_src=[5498, 5442, 5402, 5460, 5492, 5508, 5420, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38244 dup_rows=21866 dup_rows_remote=19136 dedup_ratio=0.333649 router_tokens_per_expert=[248, 256, 244, 241, 281, 250, 278, 273, 277, 281, 278, 239, 257, 266, 212, 266, 241, 269, 235, 284, 245, 240, 235, 259, 282, 249, 245, 260, 254, 255, 279, 255, 263, 253, 250, 257, 275, 251, 279, 250, 255, 265, 253, 230, 249, 256, 238, 254, 275, 246, 235, 273, 286, 261, 268, 270, 254, 293, 281, 240, 263, 266, 244, 233, 271, 247, 261, 252, 234, 226, 268, 286, 270, 254, 202, 267, 291, 230, 263, 270, 266, 277, 216, 265, 279, 240, 266, 266, 263, 263, 259, 249, 273, 264, 240, 254, 273, 253, 257, 266, 244, 259, 256, 245, 255, 277, 249, 241, 226, 258, 250, 209, 251, 240, 262, 246, 239, 261, 269, 259, 270, 237, 232, 253, 264, 255, 239, 241] payload_rows_per_dst=[5518, 5470, 5426, 5562, 5446, 5444, 5376, 5428] dup_rows_per_dst=[2776, 2704, 2730, 2814, 2738, 2836, 2660, 2608] combine_rows_per_src=[5518, 5470, 5426, 5562, 5446, 5444, 5376, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38294 dup_rows=21906 dup_rows_remote=19194 dedup_ratio=0.334259 router_tokens_per_expert=[256, 270, 220, 258, 264, 251, 251, 219, 275, 241, 268, 248, 228, 282, 281, 249, 256, 277, 262, 254, 252, 253, 247, 247, 241, 242, 233, 260, 258, 267, 255, 229, 292, 260, 251, 266, 265, 266, 253, 245, 261, 263, 262, 249, 262, 255, 284, 256, 249, 239, 243, 247, 245, 247, 251, 238, 255, 279, 236, 254, 268, 250, 269, 254, 249, 248, 238, 283, 240, 246, 273, 269, 284, 265, 239, 275, 281, 246, 291, 269, 252, 234, 253, 281, 249, 287, 242, 237, 249, 237, 233, 257, 245, 243, 262, 286, 246, 252, 265, 259, 275, 275, 247, 261, 264, 249, 273, 240, 267, 240, 269, 234, 266, 251, 243, 258, 250, 266, 235, 243, 274, 254, 265, 253, 271, 243, 276, 253] payload_rows_per_dst=[5442, 5400, 5542, 5336, 5558, 5432, 5482, 5438] dup_rows_per_dst=[2680, 2666, 2838, 2712, 2834, 2662, 2750, 2764] combine_rows_per_src=[5442, 5400, 5542, 5336, 5558, 5432, 5482, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38388 dup_rows=21624 dup_rows_remote=18994 dedup_ratio=0.329956 router_tokens_per_expert=[253, 257, 220, 277, 264, 264, 261, 271, 282, 256, 284, 269, 254, 232, 250, 223, 241, 266, 261, 262, 274, 241, 266, 261, 277, 293, 270, 231, 241, 263, 227, 247, 245, 253, 243, 248, 246, 261, 249, 262, 250, 249, 243, 249, 231, 264, 266, 260, 234, 244, 250, 252, 244, 251, 239, 258, 239, 239, 287, 261, 295, 267, 260, 247, 249, 257, 257, 251, 267, 240, 235, 281, 262, 243, 258, 268, 238, 237, 260, 222, 301, 274, 255, 264, 252, 245, 270, 258, 249, 255, 254, 261, 291, 243, 258, 291, 248, 272, 239, 285, 268, 221, 273, 250, 268, 248, 279, 260, 259, 232, 258, 261, 246, 277, 267, 260, 255, 237, 261, 237, 260, 250, 245, 256, 263, 274, 257, 232] payload_rows_per_dst=[5374, 5424, 5544, 5554, 5380, 5614, 5498, 5524] dup_rows_per_dst=[2538, 2818, 2816, 2580, 2600, 2898, 2744, 2630] combine_rows_per_src=[5374, 5424, 5544, 5554, 5380, 5614, 5498, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229288 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156958632 ingress_bytes=194478080 link_bytes=194478080 token_bytes=313687976 weight_movement_bytes=37748736 total_nvlink_bytes=351436712 weight_to_token_ratio=0.120338 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863393273e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229520 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876944 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524368 weight_movement_bytes=18874368 total_nvlink_bytes=332398736 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38152 dup_rows=21914 dup_rows_remote=19118 dedup_ratio=0.334381 router_tokens_per_expert=[266, 262, 242, 252, 240, 261, 253, 274, 253, 260, 259, 262, 256, 258, 265, 244, 242, 277, 249, 262, 249, 249, 247, 234, 248, 242, 249, 291, 240, 260, 285, 265, 238, 241, 258, 243, 277, 254, 265, 270, 248, 248, 268, 261, 290, 247, 257, 262, 256, 206, 248, 246, 242, 244, 248, 268, 267, 267, 259, 270, 262, 248, 242, 259, 256, 258, 269, 259, 223, 275, 266, 269, 267, 263, 249, 262, 276, 230, 250, 262, 247, 268, 280, 239, 240, 256, 256, 244, 249, 243, 296, 280, 253, 240, 234, 245, 235, 244, 248, 280, 251, 255, 221, 266, 241, 255, 274, 268, 249, 241, 276, 290, 277, 264, 237, 259, 265, 246, 269, 244, 256, 279, 265, 237, 268, 246, 258, 245] payload_rows_per_dst=[5470, 5546, 5458, 5422, 5458, 5484, 5366, 5418] dup_rows_per_dst=[2796, 2780, 2796, 2642, 2758, 2914, 2564, 2664] combine_rows_per_src=[5470, 5546, 5458, 5422, 5458, 5484, 5366, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229952 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157082176 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934400 weight_movement_bytes=18874368 total_nvlink_bytes=332808768 weight_to_token_ratio=0.060122 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38274 dup_rows=21848 dup_rows_remote=19116 dedup_ratio=0.333374 router_tokens_per_expert=[259, 275, 216, 286, 255, 236, 242, 242, 280, 269, 255, 268, 235, 248, 265, 256, 274, 256, 247, 261, 252, 251, 272, 278, 244, 230, 260, 241, 231, 277, 257, 242, 270, 243, 258, 249, 269, 269, 244, 238, 270, 246, 263, 236, 235, 257, 266, 253, 282, 259, 266, 239, 236, 248, 241, 281, 270, 240, 269, 241, 237, 222, 258, 255, 260, 263, 257, 253, 261, 284, 293, 241, 261, 229, 264, 234, 237, 250, 238, 267, 266, 244, 273, 264, 258, 261, 263, 265, 252, 242, 261, 244, 265, 253, 233, 270, 234, 265, 266, 249, 249, 265, 253, 246, 262, 250, 242, 258, 261, 258, 259, 284, 274, 267, 287, 265, 251, 258, 231, 269, 275, 268, 253, 282, 244, 253, 249, 265] payload_rows_per_dst=[5500, 5414, 5442, 5446, 5466, 5452, 5466, 5502] dup_rows_per_dst=[2674, 2732, 2690, 2642, 2718, 2776, 2736, 2880] combine_rows_per_src=[5500, 5414, 5442, 5446, 5466, 5452, 5466, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38364 dup_rows=21784 dup_rows_remote=19132 dedup_ratio=0.332397 router_tokens_per_expert=[262, 260, 256, 264, 267, 261, 263, 240, 248, 251, 257, 255, 248, 236, 258, 252, 276, 260, 265, 270, 255, 271, 249, 239, 284, 245, 260, 244, 249, 232, 255, 217, 251, 274, 246, 250, 270, 238, 244, 293, 238, 258, 270, 253, 256, 255, 236, 229, 292, 266, 256, 257, 234, 264, 228, 274, 266, 255, 260, 273, 277, 239, 270, 264, 254, 265, 255, 245, 245, 267, 244, 261, 261, 264, 228, 265, 245, 286, 250, 283, 267, 256, 241, 248, 249, 260, 248, 248, 265, 239, 251, 234, 250, 243, 285, 236, 258, 245, 278, 253, 225, 257, 255, 256, 265, 244, 262, 239, 271, 245, 249, 260, 290, 289, 258, 245, 252, 266, 264, 285, 238, 259, 278, 252, 275, 235, 240, 257] payload_rows_per_dst=[5496, 5464, 5474, 5550, 5498, 5388, 5400, 5482] dup_rows_per_dst=[2660, 2678, 2648, 2800, 2738, 2652, 2724, 2884] combine_rows_per_src=[5496, 5464, 5474, 5550, 5498, 5388, 5400, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229528 combine_payload_bytes=157237248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157466776 ingress_bytes=213860352 link_bytes=213860352 token_bytes=314704024 weight_movement_bytes=56623104 total_nvlink_bytes=371327128 weight_to_token_ratio=0.179925 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049104667e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38284 dup_rows=21822 dup_rows_remote=19064 dedup_ratio=0.332977 router_tokens_per_expert=[259, 257, 259, 241, 241, 251, 267, 296, 268, 246, 264, 216, 251, 249, 272, 275, 228, 265, 244, 240, 276, 257, 232, 229, 264, 261, 269, 276, 254, 256, 268, 286, 243, 271, 216, 264, 242, 253, 252, 257, 258, 257, 257, 286, 257, 239, 256, 247, 278, 237, 250, 245, 267, 268, 267, 254, 269, 268, 264, 221, 251, 268, 264, 247, 262, 263, 244, 264, 265, 288, 252, 243, 281, 245, 229, 239, 261, 236, 260, 238, 244, 271, 270, 236, 256, 290, 248, 248, 245, 267, 257, 235, 248, 249, 250, 258, 269, 255, 232, 247, 270, 258, 260, 249, 262, 249, 271, 241, 271, 256, 259, 245, 259, 231, 270, 272, 241, 255, 246, 263, 258, 227, 273, 280, 290, 268, 262, 247] payload_rows_per_dst=[5428, 5500, 5378, 5526, 5466, 5450, 5430, 5536] dup_rows_per_dst=[2796, 2678, 2732, 2710, 2674, 2726, 2758, 2748] combine_rows_per_src=[5428, 5500, 5378, 5526, 5466, 5450, 5430, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229080 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156499672 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312770264 weight_movement_bytes=37748736 total_nvlink_bytes=350519000 weight_to_token_ratio=0.120692 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229560 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999864 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770168 weight_movement_bytes=18874368 total_nvlink_bytes=332644536 weight_to_token_ratio=0.060153 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229984 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157368928 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314507872 weight_movement_bytes=18874368 total_nvlink_bytes=333382240 weight_to_token_ratio=0.060012 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229392 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040656 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851920 weight_movement_bytes=37748736 total_nvlink_bytes=351600656 weight_to_token_ratio=0.120276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38144 dup_rows=21864 dup_rows_remote=19112 dedup_ratio=0.333618 router_tokens_per_expert=[245, 245, 266, 257, 257, 253, 272, 244, 226, 238, 249, 242, 263, 277, 241, 251, 274, 297, 272, 228, 259, 216, 261, 242, 292, 285, 269, 257, 232, 274, 231, 278, 269, 223, 269, 281, 248, 264, 243, 238, 260, 235, 242, 279, 256, 276, 291, 266, 238, 247, 256, 259, 244, 265, 261, 252, 228, 266, 259, 253, 240, 255, 269, 258, 245, 266, 248, 237, 257, 236, 239, 255, 243, 284, 254, 269, 279, 277, 226, 248, 280, 277, 271, 260, 272, 217, 238, 265, 239, 289, 264, 255, 241, 245, 261, 245, 246, 259, 262, 262, 247, 243, 217, 235, 265, 250, 270, 244, 276, 259, 266, 251, 246, 281, 263, 262, 267, 269, 267, 253, 250, 269, 280, 260, 245, 231, 241, 267] payload_rows_per_dst=[5338, 5500, 5528, 5434, 5480, 5468, 5416, 5508] dup_rows_per_dst=[2714, 2834, 2752, 2666, 2646, 2770, 2688, 2794] combine_rows_per_src=[5338, 5500, 5528, 5434, 5480, 5468, 5416, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38106 dup_rows=21930 dup_rows_remote=19334 dedup_ratio=0.334625 router_tokens_per_expert=[254, 233, 258, 264, 232, 275, 274, 246, 261, 277, 240, 261, 265, 263, 264, 260, 252, 213, 259, 257, 262, 248, 253, 263, 245, 250, 234, 259, 236, 233, 232, 256, 263, 272, 273, 271, 255, 253, 240, 256, 238, 261, 242, 258, 259, 251, 255, 269, 266, 269, 237, 243, 270, 274, 253, 247, 300, 270, 241, 268, 276, 249, 252, 249, 254, 246, 260, 267, 224, 244, 283, 278, 250, 221, 271, 275, 231, 247, 244, 253, 249, 268, 250, 263, 269, 255, 264, 284, 247, 262, 264, 251, 252, 262, 271, 234, 275, 277, 249, 243, 289, 259, 261, 206, 285, 277, 236, 258, 255, 258, 256, 276, 257, 255, 253, 251, 251, 247, 253, 232, 230, 255, 246, 264, 303, 257, 245, 257] payload_rows_per_dst=[5440, 5346, 5454, 5532, 5500, 5444, 5508, 5382] dup_rows_per_dst=[2814, 2558, 2778, 2796, 2596, 2846, 2812, 2730] combine_rows_per_src=[5440, 5346, 5454, 5532, 5500, 5444, 5508, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38180 dup_rows=21868 dup_rows_remote=19126 dedup_ratio=0.333679 router_tokens_per_expert=[232, 250, 245, 217, 235, 269, 253, 243, 261, 250, 247, 249, 241, 220, 257, 262, 225, 236, 257, 238, 266, 233, 263, 267, 282, 278, 263, 251, 258, 277, 256, 249, 271, 263, 251, 262, 265, 282, 239, 251, 241, 264, 252, 235, 240, 270, 259, 263, 256, 240, 233, 254, 248, 281, 261, 274, 258, 252, 261, 257, 242, 248, 294, 256, 262, 278, 266, 287, 263, 296, 259, 259, 262, 270, 249, 274, 255, 241, 256, 257, 246, 287, 246, 241, 240, 226, 264, 271, 277, 266, 287, 244, 270, 270, 249, 254, 268, 278, 255, 244, 250, 248, 262, 275, 229, 236, 258, 256, 230, 251, 249, 280, 261, 240, 254, 249, 251, 243, 262, 240, 276, 252, 239, 280, 255, 284, 234, 254] payload_rows_per_dst=[5362, 5414, 5468, 5488, 5608, 5500, 5466, 5362] dup_rows_per_dst=[2500, 2784, 2748, 2742, 2860, 2776, 2672, 2786] combine_rows_per_src=[5362, 5414, 5468, 5488, 5608, 5500, 5466, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229024 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156466848 ingress_bytes=193986560 link_bytes=193986560 token_bytes=312704672 weight_movement_bytes=37748736 total_nvlink_bytes=350453408 weight_to_token_ratio=0.120717 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858683771e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38266 dup_rows=21744 dup_rows_remote=18980 dedup_ratio=0.331787 router_tokens_per_expert=[276, 272, 255, 284, 241, 263, 254, 225, 231, 247, 246, 281, 259, 239, 255, 271, 249, 255, 250, 285, 268, 236, 248, 245, 252, 258, 254, 250, 273, 256, 280, 244, 267, 255, 253, 225, 252, 242, 251, 284, 237, 237, 234, 263, 268, 253, 251, 299, 266, 247, 273, 258, 245, 280, 257, 272, 235, 253, 247, 248, 284, 247, 254, 251, 256, 248, 261, 239, 253, 259, 260, 262, 252, 270, 263, 266, 263, 258, 242, 260, 239, 245, 242, 263, 273, 273, 251, 264, 274, 218, 221, 246, 283, 245, 255, 270, 248, 273, 257, 258, 244, 224, 251, 263, 239, 254, 228, 251, 280, 270, 265, 284, 251, 247, 258, 259, 255, 277, 256, 265, 253, 257, 282, 260, 220, 266, 247, 262] payload_rows_per_dst=[5512, 5216, 5400, 5518, 5448, 5690, 5482, 5526] dup_rows_per_dst=[2686, 2400, 2660, 2716, 2878, 2944, 2696, 2764] combine_rows_per_src=[5512, 5216, 5400, 5518, 5448, 5690, 5482, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229760 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311936 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312394112 weight_movement_bytes=37748736 total_nvlink_bytes=350142848 weight_to_token_ratio=0.120837 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229224 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614504 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999784 weight_movement_bytes=18874368 total_nvlink_bytes=331874152 weight_to_token_ratio=0.060302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38330 dup_rows=21730 dup_rows_remote=19080 dedup_ratio=0.331573 router_tokens_per_expert=[255, 258, 269, 264, 266, 245, 262, 228, 245, 241, 232, 252, 253, 254, 278, 257, 240, 251, 242, 276, 245, 255, 249, 268, 224, 264, 254, 246, 259, 272, 261, 272, 243, 267, 272, 291, 254, 235, 300, 222, 249, 251, 275, 265, 215, 277, 233, 259, 243, 280, 269, 238, 249, 248, 242, 263, 262, 235, 288, 260, 242, 291, 252, 254, 275, 245, 272, 230, 268, 260, 259, 259, 245, 280, 240, 239, 255, 234, 279, 258, 235, 258, 253, 287, 246, 263, 268, 256, 243, 263, 282, 227, 233, 247, 266, 284, 263, 267, 253, 246, 249, 253, 241, 244, 250, 240, 270, 251, 265, 222, 238, 245, 272, 228, 257, 273, 263, 265, 257, 257, 264, 264, 247, 270, 271, 270, 253, 290] payload_rows_per_dst=[5476, 5692, 5672, 5502, 5494, 5268, 5362, 5340] dup_rows_per_dst=[2650, 2996, 3088, 2722, 2702, 2422, 2632, 2518] combine_rows_per_src=[5476, 5692, 5672, 5502, 5494, 5268, 5362, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38212 dup_rows=21830 dup_rows_remote=18996 dedup_ratio=0.333099 router_tokens_per_expert=[253, 222, 249, 252, 255, 275, 278, 256, 252, 251, 241, 271, 269, 255, 257, 260, 231, 249, 279, 264, 264, 278, 246, 274, 258, 247, 259, 268, 259, 256, 266, 266, 262, 277, 235, 261, 240, 256, 235, 241, 248, 257, 300, 260, 279, 250, 247, 235, 250, 269, 242, 240, 268, 244, 280, 268, 253, 275, 280, 249, 240, 280, 272, 240, 260, 288, 283, 269, 246, 249, 244, 263, 277, 268, 228, 244, 220, 255, 248, 255, 238, 263, 243, 238, 265, 271, 240, 239, 251, 260, 271, 273, 251, 257, 263, 254, 247, 257, 257, 269, 231, 254, 246, 248, 274, 269, 256, 218, 246, 255, 274, 266, 263, 251, 265, 249, 242, 226, 240, 247, 260, 269, 270, 258, 239, 251, 248, 256] payload_rows_per_dst=[5452, 5494, 5444, 5544, 5506, 5464, 5416, 5386] dup_rows_per_dst=[2740, 2834, 2722, 2756, 2688, 2690, 2718, 2682] combine_rows_per_src=[5452, 5494, 5444, 5544, 5506, 5464, 5416, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38360 dup_rows=21724 dup_rows_remote=19042 dedup_ratio=0.331482 router_tokens_per_expert=[279, 247, 254, 283, 278, 263, 246, 281, 260, 237, 250, 250, 216, 276, 270, 236, 273, 270, 257, 261, 253, 267, 246, 271, 264, 265, 244, 230, 257, 258, 237, 260, 295, 258, 254, 258, 264, 264, 243, 231, 255, 238, 287, 277, 264, 268, 275, 282, 240, 246, 257, 229, 234, 272, 264, 272, 257, 261, 250, 268, 271, 235, 270, 255, 254, 229, 244, 249, 284, 247, 230, 262, 275, 231, 272, 258, 236, 260, 238, 258, 302, 246, 269, 254, 245, 256, 228, 252, 252, 251, 258, 255, 249, 271, 237, 282, 257, 278, 259, 251, 309, 244, 243, 227, 239, 236, 254, 261, 267, 247, 239, 256, 242, 230, 246, 250, 236, 267, 264, 230, 222, 283, 228, 261, 280, 271, 257, 267] payload_rows_per_dst=[5518, 5476, 5572, 5472, 5402, 5508, 5452, 5412] dup_rows_per_dst=[2734, 2750, 2854, 2690, 2652, 2706, 2682, 2656] combine_rows_per_src=[5518, 5476, 5572, 5472, 5402, 5508, 5452, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38212 dup_rows=21838 dup_rows_remote=19098 dedup_ratio=0.333221 router_tokens_per_expert=[261, 256, 255, 239, 276, 273, 256, 242, 267, 273, 260, 262, 254, 262, 268, 268, 253, 258, 250, 281, 270, 257, 259, 244, 255, 237, 268, 230, 251, 238, 238, 247, 239, 279, 269, 249, 249, 256, 255, 242, 253, 251, 259, 223, 253, 221, 279, 247, 284, 256, 239, 246, 221, 255, 244, 254, 274, 254, 259, 222, 234, 275, 243, 262, 269, 243, 286, 253, 295, 233, 258, 253, 267, 247, 276, 253, 233, 265, 229, 235, 236, 235, 289, 273, 250, 234, 234, 298, 254, 253, 263, 258, 264, 266, 245, 261, 272, 267, 260, 265, 244, 272, 251, 240, 273, 258, 269, 232, 245, 241, 252, 279, 263, 279, 259, 266, 261, 274, 260, 257, 226, 270, 273, 251, 270, 260, 246, 271] payload_rows_per_dst=[5458, 5466, 5390, 5442, 5386, 5486, 5472, 5598] dup_rows_per_dst=[2886, 2606, 2658, 2602, 2804, 2740, 2768, 2774] combine_rows_per_src=[5458, 5466, 5390, 5442, 5386, 5486, 5472, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228984 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966520 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704056 weight_movement_bytes=37748736 total_nvlink_bytes=351452792 weight_to_token_ratio=0.120332 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229640 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157229320 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314229000 weight_movement_bytes=37748736 total_nvlink_bytes=351977736 weight_to_token_ratio=0.120131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229608 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157352168 ingress_bytes=194871296 link_bytes=194871296 token_bytes=314474728 weight_movement_bytes=37748736 total_nvlink_bytes=352223464 weight_to_token_ratio=0.120037 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867160876e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228832 combine_payload_bytes=156516352 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156745184 ingress_bytes=213139456 link_bytes=213139456 token_bytes=313261536 weight_movement_bytes=56623104 total_nvlink_bytes=369884640 weight_to_token_ratio=0.180753 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042197396e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=229240 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156745592 ingress_bytes=194265088 link_bytes=194265088 token_bytes=313261944 weight_movement_bytes=37748736 total_nvlink_bytes=351010680 weight_to_token_ratio=0.120502 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861352489e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38064 dup_rows=21940 dup_rows_remote=19066 dedup_ratio=0.334778 router_tokens_per_expert=[261, 266, 259, 235, 283, 255, 263, 269, 226, 245, 271, 275, 224, 267, 245, 252, 267, 268, 248, 246, 257, 258, 263, 258, 258, 272, 215, 250, 281, 253, 268, 286, 267, 272, 259, 244, 268, 257, 254, 263, 279, 280, 267, 257, 265, 244, 273, 254, 247, 212, 243, 242, 266, 243, 266, 270, 260, 252, 262, 255, 260, 229, 266, 234, 266, 256, 236, 282, 275, 244, 238, 260, 272, 280, 267, 252, 266, 231, 257, 230, 241, 234, 250, 269, 240, 238, 245, 249, 279, 260, 272, 249, 249, 235, 239, 281, 266, 244, 282, 269, 253, 255, 245, 254, 260, 258, 247, 266, 256, 244, 245, 230, 261, 286, 260, 266, 250, 228, 257, 257, 230, 241, 263, 239, 262, 273, 273, 252] payload_rows_per_dst=[5478, 5476, 5532, 5372, 5486, 5364, 5482, 5406] dup_rows_per_dst=[2714, 2820, 2874, 2642, 2738, 2696, 2666, 2790] combine_rows_per_src=[5478, 5476, 5532, 5372, 5486, 5364, 5482, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38242 dup_rows=21830 dup_rows_remote=19084 dedup_ratio=0.333099 router_tokens_per_expert=[247, 264, 257, 268, 250, 264, 275, 272, 253, 257, 234, 251, 255, 274, 237, 239, 244, 287, 247, 263, 255, 235, 270, 241, 269, 257, 249, 261, 266, 248, 263, 257, 255, 241, 245, 258, 243, 254, 286, 237, 244, 254, 255, 261, 267, 270, 265, 238, 256, 255, 270, 229, 245, 252, 207, 255, 224, 257, 257, 258, 273, 261, 256, 231, 268, 259, 255, 248, 264, 232, 281, 248, 240, 232, 286, 232, 281, 253, 269, 257, 270, 269, 265, 247, 256, 267, 233, 272, 239, 246, 244, 260, 242, 241, 269, 266, 279, 275, 288, 263, 251, 285, 273, 282, 227, 258, 294, 253, 255, 268, 265, 279, 278, 250, 242, 265, 252, 261, 219, 268, 246, 255, 235, 228, 267, 256, 243, 249] payload_rows_per_dst=[5454, 5562, 5434, 5400, 5464, 5432, 5620, 5340] dup_rows_per_dst=[2740, 2662, 2712, 2572, 2746, 2740, 2970, 2688] combine_rows_per_src=[5454, 5562, 5434, 5400, 5464, 5432, 5620, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38174 dup_rows=21898 dup_rows_remote=19070 dedup_ratio=0.334137 router_tokens_per_expert=[233, 261, 265, 268, 246, 244, 252, 243, 272, 239, 253, 267, 243, 242, 246, 254, 277, 267, 236, 228, 266, 239, 259, 271, 283, 250, 270, 259, 271, 240, 254, 265, 247, 233, 265, 248, 261, 251, 274, 268, 265, 266, 261, 270, 264, 243, 250, 271, 242, 253, 252, 261, 279, 253, 240, 258, 262, 244, 282, 250, 256, 283, 265, 266, 245, 250, 274, 249, 248, 224, 223, 256, 228, 259, 278, 252, 267, 264, 237, 249, 265, 246, 283, 266, 256, 246, 247, 271, 252, 261, 255, 258, 226, 244, 278, 259, 255, 250, 256, 277, 224, 253, 293, 249, 269, 266, 262, 234, 265, 262, 257, 256, 238, 251, 280, 223, 263, 254, 240, 249, 280, 250, 258, 255, 284, 263, 250, 240] payload_rows_per_dst=[5366, 5478, 5536, 5464, 5380, 5466, 5500, 5448] dup_rows_per_dst=[2690, 2792, 2738, 2828, 2626, 2760, 2756, 2708] combine_rows_per_src=[5366, 5478, 5536, 5464, 5380, 5466, 5500, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38050 dup_rows=21856 dup_rows_remote=18864 dedup_ratio=0.333496 router_tokens_per_expert=[260, 247, 246, 257, 262, 269, 266, 256, 270, 253, 244, 246, 230, 229, 244, 272, 224, 240, 257, 256, 259, 256, 281, 244, 252, 255, 279, 258, 239, 238, 255, 242, 257, 270, 249, 247, 259, 270, 283, 278, 244, 270, 261, 275, 241, 245, 232, 243, 242, 241, 247, 255, 264, 242, 252, 251, 263, 270, 253, 255, 276, 258, 257, 260, 239, 259, 251, 235, 273, 240, 270, 271, 253, 243, 241, 256, 257, 259, 263, 248, 279, 251, 238, 249, 259, 266, 261, 265, 260, 280, 267, 245, 257, 253, 247, 229, 237, 261, 261, 250, 262, 286, 247, 262, 235, 257, 275, 244, 274, 261, 280, 247, 290, 260, 282, 270, 272, 219, 303, 264, 227, 234, 249, 261, 245, 262, 259, 272] payload_rows_per_dst=[5420, 5336, 5276, 5728, 5344, 5524, 5422, 5630] dup_rows_per_dst=[2682, 2734, 2440, 2932, 2664, 2716, 2696, 2992] combine_rows_per_src=[5420, 5336, 5276, 5728, 5344, 5524, 5422, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228520 combine_payload_bytes=155910144 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156138664 ingress_bytes=231407616 link_bytes=231407616 token_bytes=312048808 weight_movement_bytes=75497472 total_nvlink_bytes=387546280 weight_to_token_ratio=0.241941 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217233917e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229304 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156868536 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313507768 weight_movement_bytes=37748736 total_nvlink_bytes=351256504 weight_to_token_ratio=0.120408 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228976 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156589680 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950384 weight_movement_bytes=18874368 total_nvlink_bytes=331824752 weight_to_token_ratio=0.060311 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38214 dup_rows=21774 dup_rows_remote=18812 dedup_ratio=0.332245 router_tokens_per_expert=[283, 240, 249, 279, 256, 265, 263, 246, 270, 231, 274, 257, 265, 273, 252, 282, 225, 255, 282, 247, 258, 240, 261, 255, 250, 222, 256, 278, 236, 285, 247, 274, 248, 268, 236, 271, 263, 274, 274, 218, 266, 247, 252, 254, 269, 258, 231, 238, 244, 280, 246, 262, 281, 276, 248, 254, 261, 278, 264, 253, 253, 266, 231, 244, 280, 254, 262, 262, 268, 245, 261, 276, 258, 239, 234, 290, 245, 226, 254, 247, 268, 249, 250, 254, 243, 269, 230, 252, 240, 254, 252, 266, 251, 260, 257, 246, 245, 269, 247, 247, 257, 257, 253, 269, 248, 282, 301, 264, 278, 253, 249, 247, 247, 255, 237, 232, 241, 285, 247, 245, 265, 253, 223, 239, 260, 269, 263, 235] payload_rows_per_dst=[5548, 5476, 5450, 5498, 5406, 5426, 5564, 5394] dup_rows_per_dst=[2962, 2676, 2584, 2734, 2796, 2656, 2768, 2598] combine_rows_per_src=[5548, 5476, 5450, 5498, 5406, 5426, 5564, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38278 dup_rows=21798 dup_rows_remote=18938 dedup_ratio=0.332611 router_tokens_per_expert=[270, 257, 252, 248, 235, 238, 223, 254, 248, 242, 248, 244, 248, 276, 266, 271, 299, 226, 251, 254, 228, 269, 263, 269, 247, 258, 280, 256, 254, 262, 271, 290, 247, 265, 315, 248, 254, 290, 239, 270, 281, 259, 237, 264, 234, 262, 266, 226, 264, 241, 253, 241, 264, 258, 244, 246, 242, 244, 258, 233, 246, 250, 262, 264, 264, 257, 235, 244, 250, 239, 247, 265, 292, 237, 260, 244, 260, 274, 245, 284, 233, 265, 242, 249, 267, 260, 245, 263, 268, 265, 261, 264, 252, 253, 241, 270, 243, 274, 244, 257, 262, 263, 259, 275, 269, 261, 268, 273, 280, 263, 224, 245, 228, 247, 249, 245, 253, 293, 236, 269, 239, 269, 260, 276, 256, 247, 220, 262] payload_rows_per_dst=[5406, 5518, 5522, 5432, 5496, 5434, 5460, 5470] dup_rows_per_dst=[2634, 2836, 2792, 2588, 2698, 2762, 2860, 2628] combine_rows_per_src=[5406, 5518, 5522, 5432, 5496, 5434, 5460, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=227656 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156080456 ingress_bytes=174727168 link_bytes=174727168 token_bytes=311933256 weight_movement_bytes=18874368 total_nvlink_bytes=330807624 weight_to_token_ratio=0.060508 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674149753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38222 dup_rows=21824 dup_rows_remote=19112 dedup_ratio=0.333008 router_tokens_per_expert=[233, 262, 248, 257, 276, 267, 283, 252, 249, 260, 256, 255, 254, 243, 238, 279, 272, 270, 263, 256, 249, 254, 267, 258, 258, 234, 246, 266, 263, 251, 254, 240, 240, 283, 236, 281, 239, 281, 272, 251, 234, 254, 250, 275, 285, 246, 247, 262, 263, 257, 251, 275, 241, 238, 239, 260, 239, 253, 257, 256, 247, 255, 264, 257, 282, 256, 260, 265, 249, 274, 248, 291, 251, 263, 292, 265, 236, 230, 263, 236, 277, 249, 264, 253, 221, 262, 261, 281, 268, 243, 247, 253, 261, 238, 253, 234, 283, 252, 254, 244, 249, 270, 258, 239, 238, 272, 229, 249, 260, 261, 283, 266, 250, 251, 237, 243, 270, 267, 256, 231, 264, 230, 254, 238, 240, 254, 283, 266] payload_rows_per_dst=[5494, 5490, 5522, 5386, 5480, 5422, 5480, 5438] dup_rows_per_dst=[2730, 2712, 2750, 2718, 2842, 2708, 2734, 2630] combine_rows_per_src=[5494, 5490, 5522, 5386, 5480, 5422, 5480, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38482 dup_rows=21608 dup_rows_remote=18978 dedup_ratio=0.329712 router_tokens_per_expert=[256, 284, 264, 251, 241, 242, 241, 233, 221, 240, 239, 251, 252, 261, 242, 258, 275, 267, 258, 225, 252, 252, 264, 257, 246, 253, 258, 282, 272, 278, 268, 267, 275, 273, 267, 257, 249, 293, 235, 239, 247, 261, 251, 253, 262, 271, 243, 239, 244, 273, 246, 255, 268, 259, 235, 245, 264, 283, 238, 245, 279, 244, 258, 271, 263, 261, 238, 301, 254, 251, 237, 234, 245, 272, 258, 226, 285, 278, 284, 261, 235, 251, 260, 239, 227, 251, 219, 253, 232, 283, 272, 284, 257, 269, 260, 246, 242, 257, 252, 274, 254, 227, 257, 234, 254, 263, 262, 265, 230, 257, 267, 260, 264, 257, 267, 273, 275, 231, 257, 273, 275, 217, 262, 256, 254, 246, 278, 270] payload_rows_per_dst=[5334, 5554, 5598, 5488, 5546, 5446, 5448, 5514] dup_rows_per_dst=[2618, 2794, 2632, 2726, 2750, 2630, 2662, 2796] combine_rows_per_src=[5334, 5554, 5598, 5488, 5546, 5446, 5448, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=228104 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156752648 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313277192 weight_movement_bytes=18874368 total_nvlink_bytes=332151560 weight_to_token_ratio=0.060248 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=228864 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157015552 ingress_bytes=213409792 link_bytes=213409792 token_bytes=313802240 weight_movement_bytes=56623104 total_nvlink_bytes=370425344 weight_to_token_ratio=0.180442 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044787623e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229336 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786648 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343960 weight_movement_bytes=18874368 total_nvlink_bytes=332218328 weight_to_token_ratio=0.060235 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38606 dup_rows=21620 dup_rows_remote=19106 dedup_ratio=0.329895 router_tokens_per_expert=[239, 279, 254, 279, 248, 286, 242, 263, 260, 253, 274, 253, 267, 250, 250, 272, 269, 242, 231, 268, 261, 267, 245, 258, 250, 240, 263, 270, 246, 238, 252, 257, 241, 250, 246, 222, 252, 237, 219, 257, 249, 253, 248, 244, 239, 244, 261, 250, 242, 239, 273, 282, 232, 242, 272, 265, 251, 269, 245, 256, 246, 276, 254, 291, 253, 270, 275, 300, 244, 261, 274, 248, 233, 271, 264, 243, 271, 234, 244, 268, 241, 259, 267, 247, 266, 246, 269, 240, 258, 248, 286, 253, 261, 258, 247, 279, 274, 250, 260, 268, 267, 238, 259, 252, 263, 245, 259, 251, 283, 286, 246, 242, 265, 259, 275, 240, 244, 268, 276, 224, 246, 248, 260, 247, 250, 239, 297, 236] payload_rows_per_dst=[5556, 5444, 5310, 5566, 5524, 5476, 5518, 5522] dup_rows_per_dst=[2782, 2670, 2514, 2704, 2782, 2774, 2768, 2626] combine_rows_per_src=[5556, 5444, 5310, 5566, 5524, 5476, 5518, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43368 payload_rows_remote=37940 dup_rows=22168 dup_rows_remote=19416 dedup_ratio=0.338257 router_tokens_per_expert=[247, 261, 248, 252, 290, 235, 232, 271, 241, 248, 259, 270, 245, 249, 279, 262, 245, 212, 256, 247, 258, 259, 250, 256, 242, 272, 232, 253, 230, 254, 231, 244, 249, 266, 262, 250, 264, 252, 269, 293, 243, 263, 252, 277, 258, 239, 248, 234, 260, 260, 256, 272, 273, 264, 224, 284, 259, 242, 246, 260, 261, 238, 249, 265, 245, 240, 245, 248, 252, 239, 262, 247, 232, 261, 259, 264, 280, 294, 260, 262, 253, 232, 259, 261, 267, 265, 238, 270, 266, 282, 251, 279, 255, 256, 267, 255, 232, 263, 247, 258, 266, 273, 268, 247, 272, 270, 285, 318, 263, 253, 271, 264, 225, 257, 259, 247, 244, 226, 238, 257, 255, 304, 248, 247, 239, 264, 271, 229] payload_rows_per_dst=[5400, 5278, 5438, 5458, 5428, 5462, 5546, 5358] dup_rows_per_dst=[2778, 2604, 2800, 2768, 2752, 2850, 2954, 2662] combine_rows_per_src=[5400, 5278, 5438, 5458, 5428, 5462, 5546, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157622272 dispatch_meta_bytes=229840 combine_payload_bytes=157622272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157852112 ingress_bytes=176496640 link_bytes=176496640 token_bytes=315474384 weight_movement_bytes=18874368 total_nvlink_bytes=334348752 weight_to_token_ratio=0.059829 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691103963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38294 dup_rows=21782 dup_rows_remote=18980 dedup_ratio=0.332367 router_tokens_per_expert=[207, 271, 269, 226, 261, 267, 268, 250, 267, 277, 258, 276, 238, 252, 267, 236, 259, 242, 262, 258, 253, 258, 255, 289, 260, 246, 268, 234, 264, 249, 250, 225, 244, 252, 231, 248, 272, 252, 273, 266, 236, 278, 231, 277, 250, 239, 246, 266, 245, 226, 293, 259, 251, 274, 279, 256, 291, 261, 257, 247, 229, 283, 243, 237, 281, 279, 249, 281, 265, 261, 267, 271, 224, 250, 264, 259, 262, 256, 248, 236, 274, 250, 258, 247, 250, 257, 245, 233, 277, 273, 260, 241, 280, 247, 241, 247, 232, 271, 286, 271, 225, 258, 259, 222, 252, 258, 267, 241, 251, 284, 275, 248, 238, 271, 261, 246, 267, 243, 248, 266, 241, 280, 252, 252, 232, 268, 261, 255] payload_rows_per_dst=[5454, 5440, 5462, 5460, 5512, 5446, 5492, 5488] dup_rows_per_dst=[2726, 2704, 2660, 2802, 2794, 2714, 2708, 2674] combine_rows_per_src=[5454, 5440, 5462, 5460, 5512, 5446, 5492, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38140 dup_rows=21876 dup_rows_remote=19068 dedup_ratio=0.333801 router_tokens_per_expert=[257, 254, 267, 237, 260, 272, 285, 261, 248, 253, 261, 245, 268, 242, 247, 268, 270, 237, 271, 256, 248, 239, 234, 266, 243, 249, 252, 229, 269, 264, 248, 250, 262, 271, 251, 258, 260, 236, 269, 285, 261, 256, 233, 257, 281, 261, 266, 267, 233, 272, 239, 268, 261, 262, 263, 263, 269, 253, 278, 270, 238, 263, 240, 273, 273, 270, 278, 259, 255, 266, 255, 256, 239, 272, 260, 249, 274, 233, 240, 263, 273, 255, 241, 244, 241, 251, 242, 240, 256, 245, 237, 272, 250, 259, 275, 253, 231, 259, 235, 268, 237, 264, 267, 259, 257, 234, 240, 267, 278, 262, 255, 244, 256, 250, 257, 228, 255, 256, 254, 240, 262, 240, 251, 243, 255, 287, 260, 272] payload_rows_per_dst=[5222, 5472, 5498, 5676, 5512, 5362, 5398, 5520] dup_rows_per_dst=[2392, 2864, 2850, 2896, 2772, 2706, 2588, 2808] combine_rows_per_src=[5222, 5472, 5498, 5676, 5512, 5362, 5398, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=158130176 dispatch_meta_bytes=230848 combine_payload_bytes=158130176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158361024 ingress_bytes=177004544 link_bytes=177004544 token_bytes=316491200 weight_movement_bytes=18874368 total_nvlink_bytes=335365568 weight_to_token_ratio=0.059636 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.695970449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155402240 dispatch_meta_bytes=229424 combine_payload_bytes=155402240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155631664 ingress_bytes=193150976 link_bytes=193150976 token_bytes=311033904 weight_movement_bytes=37748736 total_nvlink_bytes=348782640 weight_to_token_ratio=0.121365 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.850677616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229096 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157081320 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313933544 weight_movement_bytes=56623104 total_nvlink_bytes=370556648 weight_to_token_ratio=0.180367 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38288 dup_rows=21678 dup_rows_remote=18804 dedup_ratio=0.330780 router_tokens_per_expert=[247, 238, 245, 240, 267, 279, 286, 251, 231, 240, 253, 245, 288, 237, 247, 223, 236, 255, 266, 252, 266, 259, 249, 220, 259, 228, 269, 251, 266, 268, 249, 269, 247, 259, 262, 279, 255, 263, 267, 257, 267, 261, 257, 248, 284, 261, 275, 268, 264, 252, 266, 263, 272, 246, 247, 246, 252, 253, 249, 292, 259, 255, 273, 233, 282, 252, 270, 257, 249, 216, 257, 288, 252, 246, 257, 270, 235, 288, 283, 251, 252, 222, 258, 248, 262, 282, 259, 262, 274, 255, 243, 240, 268, 262, 267, 233, 262, 282, 265, 296, 239, 255, 261, 248, 283, 256, 250, 246, 230, 247, 241, 272, 237, 223, 242, 279, 233, 256, 269, 229, 267, 241, 260, 236, 251, 259, 233, 269] payload_rows_per_dst=[5570, 5648, 5614, 5542, 5422, 5442, 5494, 5126] dup_rows_per_dst=[2874, 2666, 3026, 2706, 2694, 2732, 2768, 2212] combine_rows_per_src=[5570, 5648, 5614, 5542, 5422, 5442, 5494, 5126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228832 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156450272 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312671712 weight_movement_bytes=18874368 total_nvlink_bytes=331546080 weight_to_token_ratio=0.060365 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38370 dup_rows=21746 dup_rows_remote=18934 dedup_ratio=0.331818 router_tokens_per_expert=[277, 250, 231, 257, 255, 240, 239, 244, 283, 265, 289, 270, 262, 264, 259, 252, 244, 245, 258, 258, 274, 239, 243, 257, 260, 252, 256, 285, 249, 243, 259, 227, 250, 266, 248, 245, 281, 247, 224, 267, 265, 285, 259, 265, 280, 279, 253, 268, 242, 245, 260, 264, 262, 248, 286, 259, 274, 260, 254, 269, 225, 235, 277, 235, 264, 269, 275, 256, 254, 212, 271, 270, 234, 247, 246, 275, 246, 257, 256, 267, 242, 248, 259, 269, 268, 225, 257, 245, 257, 212, 255, 266, 232, 242, 256, 260, 251, 253, 272, 269, 273, 260, 241, 264, 237, 278, 265, 263, 235, 240, 261, 254, 250, 257, 261, 245, 257, 245, 244, 282, 282, 257, 253, 254, 240, 257, 264, 249] payload_rows_per_dst=[5574, 5488, 5506, 5498, 5488, 5374, 5420, 5442] dup_rows_per_dst=[2700, 2610, 2858, 2692, 2710, 2612, 2812, 2752] combine_rows_per_src=[5574, 5488, 5506, 5498, 5488, 5374, 5420, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38266 dup_rows=21828 dup_rows_remote=19194 dedup_ratio=0.333069 router_tokens_per_expert=[232, 266, 269, 265, 253, 256, 237, 258, 251, 245, 253, 269, 243, 254, 258, 264, 262, 254, 237, 275, 247, 248, 247, 254, 248, 243, 247, 263, 253, 271, 247, 242, 279, 237, 276, 270, 276, 218, 245, 230, 262, 274, 267, 265, 262, 251, 236, 262, 244, 239, 259, 261, 259, 258, 237, 246, 259, 264, 264, 268, 257, 256, 236, 280, 247, 258, 276, 244, 258, 245, 271, 257, 245, 224, 263, 252, 252, 239, 263, 262, 251, 251, 243, 276, 275, 273, 262, 248, 272, 249, 257, 237, 278, 250, 248, 257, 246, 273, 249, 254, 270, 290, 245, 250, 239, 276, 248, 273, 271, 236, 257, 242, 278, 287, 244, 259, 252, 239, 249, 242, 248, 259, 263, 250, 253, 283, 270, 282] payload_rows_per_dst=[5442, 5442, 5470, 5464, 5438, 5486, 5458, 5508] dup_rows_per_dst=[2704, 2634, 2750, 2710, 2674, 2768, 2780, 2808] combine_rows_per_src=[5442, 5442, 5470, 5464, 5438, 5486, 5458, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228368 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056016 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313883664 weight_movement_bytes=18874368 total_nvlink_bytes=332758032 weight_to_token_ratio=0.060132 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43994 payload_rows_remote=38436 dup_rows=21542 dup_rows_remote=18674 dedup_ratio=0.328705 router_tokens_per_expert=[266, 259, 248, 228, 259, 259, 235, 263, 263, 249, 246, 279, 248, 285, 287, 225, 285, 259, 254, 241, 277, 275, 233, 229, 255, 261, 264, 253, 243, 259, 270, 249, 255, 256, 274, 261, 263, 256, 241, 260, 248, 270, 257, 254, 245, 260, 230, 265, 265, 251, 242, 247, 246, 274, 249, 217, 282, 245, 240, 280, 261, 229, 277, 257, 278, 263, 240, 232, 262, 245, 271, 215, 234, 237, 282, 269, 275, 242, 252, 252, 279, 263, 274, 272, 239, 283, 250, 280, 276, 252, 256, 271, 259, 235, 274, 250, 263, 236, 269, 266, 243, 245, 222, 256, 259, 278, 277, 241, 269, 234, 259, 271, 247, 246, 272, 254, 238, 247, 255, 269, 251, 276, 245, 253, 234, 251, 265, 252] payload_rows_per_dst=[5536, 5484, 5496, 5470, 5466, 5558, 5486, 5498] dup_rows_per_dst=[2662, 2730, 2694, 2654, 2632, 2868, 2690, 2612] combine_rows_per_src=[5536, 5484, 5496, 5470, 5466, 5558, 5486, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229840 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967376 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704912 weight_movement_bytes=18874368 total_nvlink_bytes=332579280 weight_to_token_ratio=0.060166 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229216 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157392736 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556256 weight_movement_bytes=18874368 total_nvlink_bytes=333430624 weight_to_token_ratio=0.060003 gemm_rows_defined=34432 gemm_rows_charged=36864 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.065972 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38310 dup_rows=21768 dup_rows_remote=19102 dedup_ratio=0.332153 router_tokens_per_expert=[257, 257, 245, 275, 256, 280, 259, 238, 255, 244, 259, 263, 244, 267, 230, 268, 219, 299, 235, 262, 274, 236, 232, 266, 247, 259, 261, 251, 273, 271, 277, 268, 266, 243, 232, 258, 258, 248, 225, 284, 270, 271, 242, 250, 256, 249, 255, 255, 235, 264, 234, 263, 257, 248, 262, 273, 270, 258, 260, 249, 241, 271, 274, 251, 260, 256, 256, 294, 253, 245, 240, 270, 255, 270, 245, 254, 281, 264, 265, 271, 250, 277, 243, 254, 233, 276, 233, 249, 246, 231, 241, 263, 246, 270, 298, 276, 276, 244, 267, 281, 236, 240, 266, 240, 241, 252, 257, 265, 245, 259, 253, 272, 262, 240, 248, 255, 233, 252, 273, 237, 236, 244, 265, 246, 244, 272, 263, 240] payload_rows_per_dst=[5454, 5462, 5458, 5600, 5522, 5442, 5432, 5398] dup_rows_per_dst=[2740, 2798, 2666, 2620, 2836, 2730, 2756, 2622] combine_rows_per_src=[5454, 5462, 5458, 5600, 5522, 5442, 5432, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38174 dup_rows=21876 dup_rows_remote=19132 dedup_ratio=0.333801 router_tokens_per_expert=[245, 277, 287, 262, 232, 282, 268, 289, 259, 243, 277, 321, 261, 279, 256, 277, 265, 272, 271, 253, 267, 230, 223, 252, 266, 246, 248, 249, 237, 258, 249, 268, 237, 223, 244, 236, 244, 255, 264, 241, 257, 237, 277, 245, 248, 289, 243, 247, 256, 278, 274, 238, 243, 245, 242, 242, 264, 226, 272, 263, 255, 268, 260, 234, 260, 253, 259, 236, 250, 278, 258, 259, 259, 250, 255, 266, 267, 241, 258, 266, 225, 257, 262, 266, 256, 231, 264, 245, 263, 248, 268, 258, 251, 264, 258, 266, 271, 246, 262, 228, 254, 238, 257, 268, 251, 284, 273, 259, 275, 252, 240, 254, 246, 263, 228, 240, 260, 254, 247, 287, 230, 274, 242, 243, 257, 245, 264, 263] payload_rows_per_dst=[5662, 5392, 5368, 5400, 5486, 5514, 5428, 5410] dup_rows_per_dst=[2968, 2716, 2606, 2720, 2744, 2650, 2796, 2676] combine_rows_per_src=[5662, 5392, 5368, 5400, 5486, 5514, 5428, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38234 dup_rows=21842 dup_rows_remote=19202 dedup_ratio=0.333282 router_tokens_per_expert=[259, 276, 266, 279, 276, 260, 253, 232, 244, 242, 265, 238, 221, 258, 258, 242, 239, 273, 243, 271, 250, 252, 257, 250, 226, 265, 248, 275, 274, 270, 279, 242, 272, 253, 249, 259, 228, 238, 271, 258, 252, 259, 262, 252, 282, 223, 250, 274, 250, 249, 258, 252, 230, 261, 238, 225, 263, 272, 253, 248, 257, 279, 267, 248, 273, 286, 257, 279, 246, 246, 233, 238, 246, 258, 287, 258, 257, 273, 249, 255, 236, 258, 229, 268, 282, 284, 261, 250, 252, 258, 241, 235, 256, 262, 269, 234, 268, 265, 232, 270, 258, 256, 228, 249, 278, 269, 257, 263, 254, 255, 252, 228, 248, 252, 274, 225, 260, 283, 246, 268, 265, 233, 274, 271, 283, 249, 262, 262] payload_rows_per_dst=[5440, 5442, 5428, 5460, 5518, 5458, 5458, 5490] dup_rows_per_dst=[2698, 2786, 2736, 2640, 2764, 2692, 2706, 2820] combine_rows_per_src=[5440, 5442, 5428, 5460, 5518, 5458, 5458, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38120 dup_rows=21964 dup_rows_remote=19202 dedup_ratio=0.335144 router_tokens_per_expert=[247, 267, 235, 280, 262, 262, 286, 257, 245, 267, 250, 270, 244, 240, 248, 286, 256, 249, 253, 261, 248, 264, 243, 262, 227, 265, 260, 247, 262, 245, 252, 253, 246, 256, 273, 260, 254, 262, 275, 254, 249, 267, 249, 252, 257, 246, 286, 288, 271, 265, 279, 272, 251, 241, 284, 238, 243, 269, 252, 278, 287, 248, 265, 256, 237, 262, 267, 252, 245, 284, 257, 242, 273, 257, 253, 268, 243, 235, 255, 266, 250, 255, 272, 257, 242, 245, 231, 262, 283, 241, 269, 250, 252, 258, 291, 248, 255, 238, 232, 261, 257, 266, 269, 260, 246, 239, 258, 247, 241, 252, 246, 271, 240, 260, 246, 248, 243, 266, 258, 264, 258, 226, 246, 247, 250, 240, 222, 248] payload_rows_per_dst=[5532, 5394, 5368, 5378, 5352, 5524, 5572, 5452] dup_rows_per_dst=[2710, 2700, 2706, 2744, 2592, 2956, 2794, 2762] combine_rows_per_src=[5532, 5394, 5368, 5378, 5352, 5524, 5572, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=228440 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157662296 ingress_bytes=195182592 link_bytes=195182592 token_bytes=315096152 weight_movement_bytes=37748736 total_nvlink_bytes=352844888 weight_to_token_ratio=0.119801 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870143561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229648 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157147408 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314065168 weight_movement_bytes=37748736 total_nvlink_bytes=351813904 weight_to_token_ratio=0.120194 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229224 combine_payload_bytes=156360704 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156589928 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950632 weight_movement_bytes=37748736 total_nvlink_bytes=350699368 weight_to_token_ratio=0.120622 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229744 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156836208 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313442672 weight_movement_bytes=37748736 total_nvlink_bytes=351191408 weight_to_token_ratio=0.120433 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38060 dup_rows=21990 dup_rows_remote=19198 dedup_ratio=0.335541 router_tokens_per_expert=[278, 269, 267, 238, 276, 282, 263, 259, 253, 252, 239, 228, 245, 253, 254, 252, 271, 252, 263, 265, 236, 263, 266, 241, 292, 259, 258, 248, 259, 236, 261, 285, 252, 281, 249, 264, 262, 281, 252, 260, 252, 254, 281, 253, 232, 241, 236, 286, 256, 290, 224, 251, 249, 246, 249, 254, 243, 238, 271, 264, 249, 245, 229, 239, 268, 279, 243, 229, 261, 277, 265, 242, 224, 269, 260, 250, 252, 237, 259, 274, 258, 229, 223, 242, 255, 247, 264, 274, 261, 246, 243, 275, 246, 258, 260, 249, 263, 281, 282, 232, 265, 260, 239, 240, 254, 246, 248, 259, 255, 252, 264, 252, 256, 270, 249, 234, 266, 266, 258, 261, 265, 265, 258, 283, 258, 260, 263, 249] payload_rows_per_dst=[5486, 5670, 5478, 5268, 5464, 5230, 5496, 5454] dup_rows_per_dst=[2792, 3076, 2794, 2678, 2726, 2394, 2816, 2714] combine_rows_per_src=[5486, 5670, 5478, 5268, 5464, 5230, 5496, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229288 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156368808 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312508328 weight_movement_bytes=18874368 total_nvlink_bytes=331382696 weight_to_token_ratio=0.060396 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38380 dup_rows=21654 dup_rows_remote=19064 dedup_ratio=0.330414 router_tokens_per_expert=[241, 279, 238, 255, 280, 240, 242, 237, 251, 272, 255, 250, 218, 262, 254, 243, 254, 264, 226, 254, 246, 282, 256, 244, 262, 272, 228, 238, 264, 248, 264, 244, 250, 240, 277, 238, 284, 257, 272, 262, 276, 255, 244, 256, 239, 260, 239, 256, 266, 271, 257, 251, 263, 262, 274, 249, 268, 258, 250, 244, 244, 241, 293, 254, 282, 246, 253, 245, 252, 243, 289, 243, 279, 252, 250, 271, 250, 250, 251, 263, 217, 252, 256, 258, 232, 249, 266, 293, 233, 258, 230, 259, 264, 263, 254, 270, 248, 264, 257, 277, 254, 266, 240, 260, 249, 261, 250, 250, 239, 295, 238, 254, 263, 261, 256, 267, 254, 264, 265, 271, 260, 264, 264, 241, 261, 255, 256, 278] payload_rows_per_dst=[5424, 5502, 5476, 5500, 5464, 5444, 5514, 5558] dup_rows_per_dst=[2610, 2590, 2734, 2790, 2774, 2664, 2690, 2802] combine_rows_per_src=[5424, 5502, 5476, 5500, 5464, 5444, 5514, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38344 dup_rows=21718 dup_rows_remote=19132 dedup_ratio=0.331390 router_tokens_per_expert=[264, 279, 257, 256, 265, 271, 240, 259, 261, 241, 239, 232, 267, 259, 247, 243, 227, 254, 274, 217, 251, 275, 241, 276, 264, 242, 286, 255, 258, 250, 236, 282, 250, 270, 252, 254, 235, 257, 285, 220, 249, 241, 282, 285, 269, 261, 250, 275, 277, 259, 236, 244, 258, 280, 225, 279, 248, 256, 253, 269, 243, 243, 272, 245, 271, 249, 296, 242, 268, 247, 246, 277, 262, 263, 267, 281, 273, 241, 269, 273, 240, 255, 247, 238, 234, 214, 233, 261, 257, 271, 245, 249, 271, 242, 264, 263, 273, 249, 245, 244, 237, 249, 250, 233, 258, 247, 252, 259, 264, 242, 274, 254, 249, 235, 263, 274, 264, 266, 251, 276, 253, 280, 263, 225, 242, 277, 235, 286] payload_rows_per_dst=[5472, 5396, 5522, 5436, 5608, 5382, 5474, 5528] dup_rows_per_dst=[2688, 2780, 2748, 2738, 2842, 2586, 2586, 2750] combine_rows_per_src=[5472, 5396, 5522, 5436, 5608, 5382, 5474, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229032 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156122792 ingress_bytes=193642496 link_bytes=193642496 token_bytes=312016552 weight_movement_bytes=37748736 total_nvlink_bytes=349765288 weight_to_token_ratio=0.120983 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855387119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229776 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434256 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638736 weight_movement_bytes=18874368 total_nvlink_bytes=333513104 weight_to_token_ratio=0.059987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38332 dup_rows=21748 dup_rows_remote=19024 dedup_ratio=0.331848 router_tokens_per_expert=[219, 230, 253, 254, 251, 264, 251, 252, 270, 242, 276, 267, 242, 265, 234, 250, 265, 264, 262, 255, 259, 271, 248, 244, 255, 261, 246, 272, 251, 226, 243, 249, 253, 269, 255, 263, 237, 246, 256, 243, 246, 245, 278, 264, 277, 248, 283, 227, 270, 272, 241, 267, 275, 259, 261, 246, 247, 227, 255, 269, 279, 278, 250, 240, 261, 263, 251, 252, 277, 235, 257, 253, 269, 230, 233, 281, 267, 258, 258, 274, 267, 254, 253, 264, 257, 279, 242, 248, 275, 282, 258, 250, 251, 241, 244, 238, 278, 258, 225, 254, 281, 278, 282, 243, 248, 246, 273, 239, 225, 275, 288, 266, 273, 254, 257, 229, 227, 249, 266, 262, 257, 241, 258, 282, 275, 246, 250, 244] payload_rows_per_dst=[5420, 5376, 5456, 5520, 5494, 5522, 5478, 5522] dup_rows_per_dst=[2620, 2766, 2724, 2752, 2744, 2684, 2840, 2618] combine_rows_per_src=[5420, 5376, 5456, 5520, 5494, 5522, 5478, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229904 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157286928 ingress_bytes=194805760 link_bytes=194805760 token_bytes=314343952 weight_movement_bytes=37748736 total_nvlink_bytes=352092688 weight_to_token_ratio=0.120087 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38248 dup_rows=21784 dup_rows_remote=19030 dedup_ratio=0.332397 router_tokens_per_expert=[280, 256, 257, 255, 268, 250, 234, 266, 241, 243, 253, 268, 282, 272, 237, 256, 250, 259, 243, 248, 255, 263, 280, 254, 247, 247, 241, 267, 265, 259, 216, 275, 262, 284, 256, 241, 244, 266, 244, 231, 248, 258, 260, 300, 279, 237, 276, 253, 262, 249, 257, 276, 246, 268, 263, 260, 245, 284, 219, 230, 257, 238, 263, 277, 251, 269, 251, 260, 264, 250, 281, 266, 245, 267, 260, 256, 229, 253, 265, 262, 250, 253, 254, 214, 259, 238, 255, 253, 243, 271, 262, 261, 269, 256, 252, 272, 271, 247, 264, 226, 236, 239, 273, 266, 248, 247, 252, 249, 250, 257, 273, 271, 252, 241, 273, 259, 249, 269, 249, 242, 246, 270, 230, 260, 248, 241, 285, 274] payload_rows_per_dst=[5514, 5386, 5508, 5496, 5504, 5446, 5418, 5480] dup_rows_per_dst=[2722, 2752, 2770, 2692, 2754, 2678, 2720, 2696] combine_rows_per_src=[5514, 5386, 5508, 5496, 5504, 5446, 5418, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38350 dup_rows=21808 dup_rows_remote=19086 dedup_ratio=0.332764 router_tokens_per_expert=[260, 258, 293, 266, 269, 246, 284, 278, 246, 245, 256, 276, 246, 261, 252, 270, 242, 266, 252, 237, 252, 259, 273, 250, 257, 252, 254, 246, 270, 247, 243, 266, 262, 233, 248, 257, 255, 258, 281, 267, 279, 250, 260, 264, 281, 271, 270, 226, 256, 256, 264, 255, 254, 232, 243, 245, 268, 262, 255, 254, 256, 225, 240, 245, 276, 243, 258, 247, 285, 251, 255, 257, 256, 273, 271, 238, 264, 259, 256, 234, 277, 258, 253, 266, 250, 285, 259, 236, 254, 232, 258, 252, 219, 214, 260, 277, 257, 247, 267, 249, 250, 257, 285, 224, 244, 236, 259, 251, 260, 248, 249, 250, 271, 270, 240, 247, 229, 261, 261, 258, 262, 270, 233, 264, 256, 267, 265, 264] payload_rows_per_dst=[5570, 5472, 5526, 5438, 5416, 5378, 5374, 5554] dup_rows_per_dst=[2842, 2660, 2798, 2582, 2830, 2722, 2692, 2682] combine_rows_per_src=[5570, 5472, 5526, 5438, 5416, 5378, 5374, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38266 dup_rows=21714 dup_rows_remote=18976 dedup_ratio=0.331329 router_tokens_per_expert=[241, 218, 262, 265, 246, 250, 230, 245, 250, 242, 266, 244, 253, 252, 264, 276, 239, 249, 260, 252, 242, 245, 270, 262, 234, 268, 265, 250, 247, 276, 250, 286, 247, 245, 267, 262, 243, 237, 259, 287, 253, 250, 245, 251, 254, 261, 284, 269, 267, 249, 261, 257, 254, 260, 263, 254, 226, 271, 258, 278, 248, 268, 268, 265, 261, 279, 265, 244, 250, 267, 267, 263, 250, 251, 234, 276, 247, 244, 268, 242, 232, 252, 261, 255, 300, 275, 248, 265, 253, 261, 254, 254, 269, 263, 256, 263, 226, 283, 236, 256, 255, 251, 287, 243, 249, 266, 254, 271, 252, 246, 240, 250, 246, 259, 264, 272, 255, 251, 261, 256, 260, 276, 223, 253, 261, 238, 255, 244] payload_rows_per_dst=[5364, 5462, 5524, 5556, 5492, 5562, 5456, 5406] dup_rows_per_dst=[2644, 2728, 2704, 2738, 2724, 2760, 2674, 2742] combine_rows_per_src=[5364, 5462, 5524, 5556, 5492, 5562, 5456, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38362 dup_rows=21746 dup_rows_remote=18962 dedup_ratio=0.331818 router_tokens_per_expert=[260, 243, 256, 275, 257, 268, 254, 294, 268, 266, 249, 269, 254, 262, 252, 278, 275, 254, 246, 270, 282, 251, 282, 265, 258, 246, 244, 268, 254, 227, 234, 235, 238, 245, 266, 281, 265, 222, 245, 275, 263, 228, 271, 226, 260, 259, 243, 216, 248, 261, 270, 219, 275, 244, 266, 265, 265, 258, 228, 289, 269, 272, 271, 261, 245, 240, 247, 229, 270, 250, 256, 253, 258, 242, 242, 244, 269, 249, 272, 252, 236, 266, 230, 247, 256, 247, 240, 240, 249, 245, 272, 264, 232, 287, 247, 256, 257, 258, 270, 262, 242, 278, 258, 270, 242, 276, 265, 241, 266, 240, 287, 258, 234, 256, 267, 249, 222, 251, 266, 289, 257, 267, 264, 264, 250, 233, 279, 258] payload_rows_per_dst=[5468, 5500, 5398, 5544, 5344, 5442, 5666, 5428] dup_rows_per_dst=[2800, 2682, 2608, 2920, 2446, 2586, 2920, 2784] combine_rows_per_src=[5468, 5500, 5398, 5544, 5344, 5442, 5666, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229424 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237296 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245168 weight_movement_bytes=18874368 total_nvlink_bytes=333119536 weight_to_token_ratio=0.060063 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229112 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156892920 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556728 weight_movement_bytes=37748736 total_nvlink_bytes=351305464 weight_to_token_ratio=0.120389 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229744 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311344 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392944 weight_movement_bytes=18874368 total_nvlink_bytes=333267312 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228968 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966504 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704040 weight_movement_bytes=37748736 total_nvlink_bytes=351452776 weight_to_token_ratio=0.120332 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229296 combine_payload_bytes=157130752 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157360048 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314490800 weight_movement_bytes=18874368 total_nvlink_bytes=333365168 weight_to_token_ratio=0.060016 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38030 dup_rows=21856 dup_rows_remote=18904 dedup_ratio=0.333496 router_tokens_per_expert=[269, 270, 262, 249, 252, 237, 244, 279, 261, 279, 258, 252, 268, 259, 272, 268, 255, 263, 273, 279, 240, 268, 243, 272, 252, 296, 233, 244, 297, 269, 302, 235, 232, 270, 286, 251, 243, 236, 256, 261, 243, 242, 257, 264, 278, 249, 251, 263, 258, 235, 270, 259, 254, 292, 259, 261, 261, 240, 247, 243, 226, 256, 240, 268, 245, 242, 276, 263, 287, 238, 241, 254, 273, 245, 253, 274, 234, 231, 252, 265, 254, 236, 268, 232, 261, 250, 269, 256, 248, 236, 245, 238, 289, 239, 254, 221, 246, 243, 271, 297, 227, 249, 254, 240, 260, 262, 235, 242, 274, 266, 262, 222, 274, 255, 267, 261, 255, 274, 231, 266, 239, 248, 247, 260, 265, 260, 241, 255] payload_rows_per_dst=[5650, 5568, 5422, 5580, 5320, 5372, 5386, 5382] dup_rows_per_dst=[2952, 2874, 2742, 2852, 2532, 2620, 2614, 2670] combine_rows_per_src=[5650, 5568, 5422, 5580, 5320, 5372, 5386, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38202 dup_rows=21898 dup_rows_remote=19170 dedup_ratio=0.334137 router_tokens_per_expert=[268, 254, 249, 228, 276, 256, 290, 235, 240, 252, 255, 287, 277, 266, 286, 279, 244, 257, 271, 241, 257, 246, 280, 247, 271, 250, 250, 247, 248, 262, 235, 276, 248, 255, 276, 250, 274, 267, 253, 268, 272, 246, 255, 220, 271, 231, 249, 236, 267, 250, 262, 275, 276, 259, 268, 270, 270, 254, 262, 237, 274, 250, 220, 272, 263, 258, 246, 266, 273, 264, 249, 259, 246, 285, 250, 260, 254, 242, 242, 250, 262, 256, 248, 254, 238, 254, 266, 249, 226, 273, 238, 232, 245, 245, 258, 247, 268, 263, 259, 274, 241, 250, 255, 284, 261, 267, 269, 242, 261, 234, 257, 232, 273, 274, 245, 243, 227, 263, 270, 270, 242, 237, 250, 244, 248, 263, 236, 251] payload_rows_per_dst=[5578, 5436, 5376, 5500, 5440, 5386, 5522, 5400] dup_rows_per_dst=[2818, 2728, 2766, 2832, 2774, 2608, 2700, 2672] combine_rows_per_src=[5578, 5436, 5376, 5500, 5440, 5386, 5522, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38438 dup_rows=21646 dup_rows_remote=18992 dedup_ratio=0.330292 router_tokens_per_expert=[259, 251, 251, 246, 234, 246, 250, 245, 237, 263, 232, 267, 261, 252, 264, 269, 241, 259, 269, 248, 265, 244, 230, 255, 263, 248, 245, 262, 294, 287, 266, 247, 256, 258, 223, 245, 257, 234, 259, 251, 252, 261, 263, 287, 273, 295, 246, 254, 253, 283, 249, 277, 251, 255, 245, 275, 246, 256, 257, 269, 239, 251, 255, 270, 270, 258, 264, 258, 294, 235, 227, 236, 250, 283, 281, 261, 247, 274, 290, 248, 264, 238, 274, 290, 272, 287, 237, 216, 217, 244, 243, 251, 261, 244, 262, 266, 244, 254, 241, 276, 275, 268, 240, 266, 260, 249, 228, 254, 226, 265, 259, 248, 250, 272, 251, 240, 259, 254, 260, 272, 264, 256, 234, 252, 265, 264, 252, 233] payload_rows_per_dst=[5392, 5474, 5452, 5584, 5576, 5448, 5452, 5512] dup_rows_per_dst=[2662, 2772, 2776, 2678, 2776, 2684, 2654, 2644] combine_rows_per_src=[5392, 5474, 5452, 5584, 5576, 5448, 5452, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=227736 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155998616 ingress_bytes=174645248 link_bytes=174645248 token_bytes=311769496 weight_movement_bytes=18874368 total_nvlink_bytes=330643864 weight_to_token_ratio=0.060539 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673364836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229488 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156704880 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313180272 weight_movement_bytes=18874368 total_nvlink_bytes=332054640 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43950 payload_rows_remote=38386 dup_rows=21586 dup_rows_remote=18908 dedup_ratio=0.329376 router_tokens_per_expert=[263, 262, 258, 274, 263, 268, 278, 245, 264, 289, 284, 267, 262, 264, 258, 247, 269, 270, 244, 264, 255, 275, 256, 259, 251, 239, 278, 261, 212, 246, 254, 241, 281, 253, 231, 247, 228, 250, 261, 273, 283, 246, 218, 259, 267, 268, 270, 286, 277, 265, 285, 246, 271, 257, 245, 239, 275, 266, 267, 256, 248, 264, 260, 253, 247, 234, 239, 247, 270, 260, 234, 304, 250, 261, 270, 243, 247, 273, 272, 252, 254, 252, 249, 255, 247, 251, 277, 250, 244, 254, 231, 250, 242, 258, 264, 244, 223, 278, 226, 246, 242, 255, 266, 272, 271, 250, 240, 266, 232, 236, 264, 259, 240, 264, 250, 262, 253, 224, 230, 262, 248, 264, 256, 250, 245, 237, 262, 255] payload_rows_per_dst=[5624, 5440, 5564, 5480, 5584, 5436, 5410, 5412] dup_rows_per_dst=[2868, 2708, 2678, 2868, 2622, 2608, 2642, 2592] combine_rows_per_src=[5624, 5440, 5564, 5480, 5584, 5436, 5410, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229720 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671768 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113816 weight_movement_bytes=18874368 total_nvlink_bytes=333988184 weight_to_token_ratio=0.059897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38256 dup_rows=21840 dup_rows_remote=19048 dedup_ratio=0.333252 router_tokens_per_expert=[245, 254, 269, 242, 260, 247, 232, 249, 246, 263, 250, 252, 263, 258, 235, 252, 261, 257, 271, 255, 259, 226, 261, 248, 260, 266, 241, 250, 235, 250, 279, 267, 271, 236, 258, 251, 218, 241, 250, 250, 257, 222, 287, 258, 289, 243, 256, 292, 260, 230, 258, 268, 284, 247, 235, 263, 268, 266, 250, 277, 251, 245, 236, 228, 276, 263, 249, 306, 244, 272, 283, 238, 253, 251, 256, 273, 241, 238, 238, 235, 236, 274, 224, 271, 274, 272, 259, 298, 273, 268, 258, 246, 260, 256, 261, 253, 250, 260, 280, 259, 258, 236, 260, 273, 256, 246, 250, 278, 250, 263, 266, 261, 246, 241, 260, 249, 270, 260, 245, 239, 265, 280, 248, 236, 255, 282, 247, 252] payload_rows_per_dst=[5432, 5386, 5514, 5434, 5440, 5542, 5528, 5420] dup_rows_per_dst=[2602, 2786, 2644, 2698, 2792, 2824, 2764, 2730] combine_rows_per_src=[5432, 5386, 5514, 5434, 5440, 5542, 5528, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38006 dup_rows=21998 dup_rows_remote=19098 dedup_ratio=0.335663 router_tokens_per_expert=[260, 231, 285, 280, 264, 246, 248, 254, 259, 252, 251, 263, 244, 260, 242, 268, 234, 232, 249, 282, 265, 236, 245, 229, 226, 275, 259, 238, 244, 257, 256, 241, 249, 229, 253, 255, 244, 262, 274, 262, 258, 260, 259, 264, 258, 264, 235, 240, 277, 256, 276, 249, 275, 255, 256, 256, 251, 286, 263, 261, 261, 250, 276, 268, 265, 256, 239, 230, 249, 256, 285, 276, 230, 242, 260, 227, 267, 243, 254, 243, 247, 243, 259, 287, 250, 245, 252, 286, 240, 233, 244, 246, 269, 269, 259, 255, 253, 277, 251, 289, 247, 270, 266, 258, 279, 277, 277, 261, 241, 277, 251, 266, 259, 262, 239, 242, 280, 245, 242, 246, 247, 235, 274, 254, 248, 267, 258, 267] payload_rows_per_dst=[5420, 5348, 5408, 5532, 5412, 5422, 5530, 5466] dup_rows_per_dst=[2794, 2588, 2724, 2900, 2632, 2746, 2950, 2664] combine_rows_per_src=[5420, 5348, 5408, 5532, 5412, 5422, 5530, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38092 dup_rows=21898 dup_rows_remote=18922 dedup_ratio=0.334137 router_tokens_per_expert=[238, 259, 241, 226, 264, 272, 276, 235, 265, 259, 240, 252, 256, 285, 254, 297, 250, 228, 253, 228, 251, 250, 237, 243, 228, 252, 295, 245, 268, 255, 256, 239, 269, 236, 266, 234, 251, 256, 255, 253, 249, 249, 233, 272, 252, 264, 253, 252, 285, 233, 269, 264, 270, 253, 247, 245, 240, 260, 242, 240, 283, 242, 245, 270, 278, 249, 284, 260, 273, 247, 261, 241, 252, 249, 270, 260, 291, 244, 253, 296, 275, 263, 257, 254, 268, 258, 231, 256, 274, 250, 260, 239, 246, 271, 240, 254, 274, 263, 248, 226, 277, 255, 280, 264, 253, 262, 239, 281, 226, 241, 250, 275, 247, 245, 268, 275, 261, 230, 267, 250, 261, 278, 242, 254, 262, 240, 267, 274] payload_rows_per_dst=[5440, 5366, 5372, 5434, 5324, 5618, 5538, 5546] dup_rows_per_dst=[2798, 2590, 2674, 2742, 2406, 2980, 2732, 2976] combine_rows_per_src=[5440, 5366, 5372, 5434, 5324, 5618, 5538, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229176 combine_payload_bytes=157229056 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157458232 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687288 weight_movement_bytes=18874368 total_nvlink_bytes=333561656 weight_to_token_ratio=0.059978 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38192 dup_rows=21776 dup_rows_remote=19044 dedup_ratio=0.332275 router_tokens_per_expert=[235, 230, 245, 262, 278, 262, 275, 246, 261, 271, 247, 253, 255, 258, 231, 262, 251, 247, 272, 248, 255, 252, 281, 266, 239, 275, 239, 274, 248, 259, 250, 243, 263, 277, 239, 229, 271, 260, 244, 246, 281, 277, 253, 256, 244, 247, 252, 267, 270, 276, 252, 254, 276, 265, 246, 231, 246, 247, 260, 261, 251, 278, 267, 248, 254, 228, 241, 236, 266, 260, 257, 268, 268, 259, 264, 258, 232, 282, 259, 271, 251, 255, 249, 276, 244, 252, 250, 271, 261, 269, 252, 273, 246, 251, 257, 293, 271, 284, 229, 228, 256, 263, 276, 273, 229, 222, 251, 273, 225, 246, 258, 252, 249, 258, 248, 222, 246, 272, 269, 265, 223, 260, 253, 255, 265, 274, 267, 249] payload_rows_per_dst=[5458, 5474, 5446, 5528, 5412, 5568, 5452, 5422] dup_rows_per_dst=[2684, 2724, 2766, 2728, 2794, 2732, 2620, 2728] combine_rows_per_src=[5458, 5474, 5446, 5528, 5412, 5568, 5452, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229216 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156925792 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622368 weight_movement_bytes=37748736 total_nvlink_bytes=351371104 weight_to_token_ratio=0.120364 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155672576 dispatch_meta_bytes=228416 combine_payload_bytes=155672576 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=155900992 ingress_bytes=231170048 link_bytes=231170048 token_bytes=311573568 weight_movement_bytes=75497472 total_nvlink_bytes=387071040 weight_to_token_ratio=0.242310 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.214957657e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228056 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156252888 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312277720 weight_movement_bytes=18874368 total_nvlink_bytes=331152088 weight_to_token_ratio=0.060441 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38384 dup_rows=21688 dup_rows_remote=19054 dedup_ratio=0.330933 router_tokens_per_expert=[261, 269, 222, 270, 260, 238, 262, 268, 272, 226, 289, 229, 247, 276, 246, 250, 250, 237, 239, 253, 251, 257, 251, 251, 244, 261, 261, 267, 267, 252, 257, 251, 258, 265, 265, 236, 250, 291, 264, 240, 258, 278, 253, 227, 259, 259, 289, 240, 239, 279, 274, 245, 265, 219, 258, 258, 248, 244, 250, 264, 266, 262, 255, 240, 259, 256, 230, 268, 276, 257, 283, 256, 255, 268, 257, 228, 255, 259, 266, 281, 255, 282, 240, 251, 246, 256, 278, 238, 259, 270, 271, 247, 261, 251, 238, 240, 241, 277, 255, 247, 253, 283, 227, 243, 268, 267, 284, 245, 229, 247, 251, 269, 260, 241, 258, 265, 273, 253, 232, 279, 263, 283, 267, 263, 251, 268, 244, 213] payload_rows_per_dst=[5478, 5464, 5554, 5484, 5478, 5448, 5448, 5494] dup_rows_per_dst=[2692, 2634, 2710, 2648, 2830, 2718, 2724, 2732] combine_rows_per_src=[5478, 5464, 5554, 5484, 5478, 5448, 5448, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38122 dup_rows=21826 dup_rows_remote=18886 dedup_ratio=0.333038 router_tokens_per_expert=[245, 263, 242, 245, 230, 285, 249, 257, 222, 261, 282, 236, 244, 254, 275, 239, 271, 274, 257, 258, 257, 236, 249, 257, 241, 254, 287, 270, 244, 263, 277, 249, 258, 237, 257, 302, 285, 263, 282, 237, 268, 278, 279, 251, 262, 268, 278, 266, 260, 255, 224, 269, 243, 244, 275, 276, 232, 243, 260, 266, 241, 225, 243, 252, 255, 249, 243, 290, 252, 241, 265, 257, 287, 250, 229, 262, 246, 266, 249, 250, 255, 244, 246, 274, 251, 238, 260, 244, 244, 256, 245, 266, 244, 252, 232, 268, 246, 244, 251, 249, 255, 240, 230, 260, 283, 238, 255, 246, 255, 258, 269, 275, 274, 266, 261, 243, 275, 258, 259, 229, 246, 248, 280, 263, 270, 258, 266, 256] payload_rows_per_dst=[5588, 5512, 5438, 5442, 5422, 5314, 5480, 5514] dup_rows_per_dst=[2940, 2936, 2634, 2574, 2760, 2564, 2628, 2790] combine_rows_per_src=[5588, 5512, 5438, 5442, 5422, 5314, 5480, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228944 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663376 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313097808 weight_movement_bytes=18874368 total_nvlink_bytes=331972176 weight_to_token_ratio=0.060283 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38042 dup_rows=21904 dup_rows_remote=19036 dedup_ratio=0.334229 router_tokens_per_expert=[249, 242, 266, 263, 235, 241, 233, 227, 261, 249, 286, 259, 288, 265, 258, 247, 285, 238, 282, 270, 249, 271, 236, 246, 258, 251, 236, 250, 268, 268, 261, 280, 243, 255, 266, 267, 241, 223, 235, 246, 264, 245, 242, 282, 247, 274, 278, 228, 240, 244, 238, 227, 243, 255, 239, 236, 248, 251, 262, 268, 255, 267, 251, 255, 259, 238, 255, 283, 268, 264, 261, 265, 274, 260, 264, 227, 240, 297, 261, 251, 247, 260, 206, 280, 248, 244, 266, 250, 261, 278, 247, 267, 243, 240, 275, 252, 277, 236, 266, 283, 253, 289, 261, 237, 275, 271, 251, 243, 284, 258, 292, 253, 270, 259, 246, 260, 284, 267, 258, 229, 235, 260, 272, 263, 250, 233, 246, 243] payload_rows_per_dst=[5406, 5478, 5440, 5316, 5548, 5406, 5590, 5448] dup_rows_per_dst=[2732, 2820, 2632, 2642, 2786, 2722, 2868, 2702] combine_rows_per_src=[5406, 5478, 5440, 5316, 5548, 5406, 5590, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229752 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157450616 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314671480 weight_movement_bytes=18874368 total_nvlink_bytes=333545848 weight_to_token_ratio=0.059981 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228032 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156375744 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312523456 weight_movement_bytes=18874368 total_nvlink_bytes=331397824 weight_to_token_ratio=0.060393 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38374 dup_rows=21632 dup_rows_remote=18818 dedup_ratio=0.330078 router_tokens_per_expert=[267, 254, 267, 275, 254, 265, 270, 255, 253, 246, 231, 249, 241, 240, 251, 242, 242, 230, 240, 249, 242, 269, 273, 244, 246, 259, 259, 254, 281, 257, 271, 260, 250, 271, 269, 263, 284, 271, 282, 258, 261, 269, 251, 232, 229, 270, 248, 264, 254, 264, 248, 292, 258, 269, 278, 269, 242, 221, 282, 241, 264, 246, 255, 233, 269, 265, 248, 255, 249, 274, 254, 258, 250, 261, 261, 220, 252, 236, 218, 244, 235, 275, 277, 265, 241, 251, 249, 285, 265, 278, 234, 241, 247, 284, 253, 278, 273, 260, 265, 262, 246, 251, 284, 293, 243, 243, 240, 243, 238, 265, 265, 266, 281, 256, 237, 242, 281, 242, 223, 263, 263, 237, 238, 259, 246, 282, 258, 227] payload_rows_per_dst=[5490, 5470, 5530, 5506, 5418, 5548, 5514, 5428] dup_rows_per_dst=[2630, 2682, 2814, 2726, 2610, 2768, 2760, 2642] combine_rows_per_src=[5490, 5470, 5530, 5506, 5418, 5548, 5514, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=228312 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156048344 ingress_bytes=212443136 link_bytes=212443136 token_bytes=311868376 weight_movement_bytes=56623104 total_nvlink_bytes=368491480 weight_to_token_ratio=0.181561 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035525601e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38346 dup_rows=21662 dup_rows_remote=18954 dedup_ratio=0.330536 router_tokens_per_expert=[224, 259, 271, 260, 271, 240, 245, 246, 231, 247, 260, 238, 251, 276, 249, 310, 253, 253, 219, 257, 254, 252, 271, 255, 258, 239, 281, 287, 273, 254, 272, 257, 260, 248, 249, 242, 266, 247, 247, 247, 265, 238, 263, 248, 260, 246, 257, 248, 243, 239, 271, 247, 292, 250, 271, 255, 239, 240, 235, 278, 268, 267, 220, 250, 243, 254, 267, 258, 259, 287, 270, 263, 248, 246, 237, 254, 252, 270, 259, 251, 237, 251, 236, 247, 259, 238, 244, 262, 234, 236, 272, 277, 234, 264, 260, 264, 256, 247, 264, 252, 272, 268, 253, 270, 273, 262, 265, 248, 240, 232, 259, 288, 256, 280, 254, 256, 262, 264, 260, 221, 268, 246, 274, 242, 299, 264, 251, 280] payload_rows_per_dst=[5472, 5462, 5494, 5370, 5528, 5438, 5548, 5562] dup_rows_per_dst=[2684, 2808, 2568, 2760, 2708, 2592, 2750, 2792] combine_rows_per_src=[5472, 5462, 5494, 5370, 5528, 5438, 5548, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38406 dup_rows=21772 dup_rows_remote=19184 dedup_ratio=0.332214 router_tokens_per_expert=[226, 239, 264, 266, 261, 244, 252, 242, 279, 242, 237, 247, 264, 259, 255, 275, 249, 258, 257, 242, 265, 248, 239, 258, 261, 240, 237, 249, 276, 239, 250, 236, 259, 264, 245, 257, 262, 278, 259, 271, 307, 260, 244, 241, 290, 250, 281, 266, 269, 222, 260, 245, 248, 258, 269, 256, 224, 223, 233, 260, 249, 257, 249, 251, 265, 250, 258, 233, 281, 257, 282, 276, 251, 248, 303, 237, 247, 235, 260, 271, 254, 247, 263, 251, 258, 279, 273, 242, 227, 254, 265, 242, 265, 262, 263, 243, 228, 260, 261, 262, 270, 252, 261, 252, 241, 258, 248, 229, 278, 269, 257, 275, 259, 275, 246, 254, 282, 246, 276, 263, 274, 252, 278, 251, 239, 277, 247, 243] payload_rows_per_dst=[5440, 5370, 5582, 5358, 5512, 5502, 5498, 5502] dup_rows_per_dst=[2664, 2638, 2886, 2588, 2796, 2674, 2704, 2822] combine_rows_per_src=[5440, 5370, 5582, 5358, 5512, 5502, 5498, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38168 dup_rows=21926 dup_rows_remote=19078 dedup_ratio=0.334564 router_tokens_per_expert=[285, 250, 249, 265, 247, 284, 257, 268, 253, 239, 260, 258, 284, 271, 266, 258, 249, 241, 274, 241, 256, 253, 270, 256, 228, 277, 235, 256, 253, 258, 243, 261, 267, 260, 276, 258, 262, 253, 244, 240, 265, 263, 257, 272, 272, 272, 252, 258, 266, 238, 255, 273, 277, 241, 284, 243, 250, 264, 256, 261, 259, 266, 247, 242, 246, 243, 255, 240, 255, 253, 227, 259, 281, 269, 288, 250, 233, 237, 255, 242, 248, 250, 259, 224, 247, 251, 248, 313, 266, 277, 253, 272, 249, 246, 250, 264, 245, 246, 264, 278, 238, 245, 253, 239, 258, 257, 235, 258, 251, 253, 259, 248, 234, 231, 281, 245, 248, 223, 256, 242, 277, 260, 249, 267, 263, 274, 246, 257] payload_rows_per_dst=[5496, 5464, 5492, 5356, 5370, 5580, 5410, 5442] dup_rows_per_dst=[2708, 2638, 2850, 2632, 2696, 2892, 2662, 2848] combine_rows_per_src=[5496, 5464, 5492, 5356, 5370, 5580, 5410, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=228768 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157408672 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314588576 weight_movement_bytes=18874368 total_nvlink_bytes=333462944 weight_to_token_ratio=0.059997 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38358 dup_rows=21700 dup_rows_remote=18926 dedup_ratio=0.331116 router_tokens_per_expert=[235, 255, 249, 233, 244, 239, 287, 237, 257, 262, 271, 251, 246, 234, 261, 256, 284, 257, 252, 249, 254, 275, 253, 273, 255, 263, 246, 274, 236, 260, 255, 231, 251, 251, 257, 228, 252, 262, 241, 250, 284, 269, 246, 261, 254, 235, 228, 277, 279, 242, 255, 255, 272, 254, 257, 261, 252, 271, 256, 272, 257, 267, 274, 230, 262, 281, 260, 266, 258, 274, 263, 250, 231, 254, 273, 243, 238, 256, 271, 248, 253, 237, 249, 243, 252, 247, 271, 272, 271, 277, 284, 265, 261, 258, 232, 254, 267, 239, 268, 251, 234, 233, 245, 272, 276, 263, 225, 235, 258, 264, 263, 285, 245, 244, 261, 265, 256, 219, 248, 264, 267, 271, 256, 254, 235, 273, 282, 262] payload_rows_per_dst=[5418, 5514, 5466, 5478, 5518, 5478, 5476, 5488] dup_rows_per_dst=[2616, 2720, 2626, 2830, 2738, 2774, 2680, 2716] combine_rows_per_src=[5418, 5514, 5466, 5478, 5518, 5478, 5476, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229200 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157294416 ingress_bytes=232562688 link_bytes=232562688 token_bytes=314359632 weight_movement_bytes=75497472 total_nvlink_bytes=389857104 weight_to_token_ratio=0.240163 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.069444 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228301248e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=230360 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157541336 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314852312 weight_movement_bytes=18874368 total_nvlink_bytes=333726680 weight_to_token_ratio=0.059947 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228984 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156565112 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312901240 weight_movement_bytes=18874368 total_nvlink_bytes=331775608 weight_to_token_ratio=0.060321 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38300 dup_rows=21798 dup_rows_remote=19070 dedup_ratio=0.332611 router_tokens_per_expert=[256, 256, 284, 269, 248, 258, 259, 279, 266, 255, 256, 264, 255, 232, 253, 246, 249, 287, 238, 252, 237, 254, 233, 251, 271, 253, 254, 265, 276, 241, 257, 265, 238, 256, 251, 240, 270, 277, 296, 267, 261, 251, 253, 274, 215, 223, 242, 236, 272, 255, 261, 270, 255, 262, 263, 245, 251, 276, 265, 295, 249, 289, 253, 261, 268, 250, 290, 282, 264, 218, 215, 252, 275, 229, 265, 259, 271, 271, 252, 244, 262, 276, 239, 254, 266, 251, 243, 286, 249, 244, 248, 265, 244, 253, 235, 228, 219, 253, 252, 242, 265, 260, 237, 254, 264, 252, 282, 254, 255, 251, 268, 231, 243, 253, 274, 263, 271, 241, 257, 245, 250, 268, 255, 270, 277, 249, 245, 229] payload_rows_per_dst=[5522, 5438, 5382, 5576, 5484, 5390, 5428, 5518] dup_rows_per_dst=[2750, 2728, 2718, 2868, 2726, 2696, 2650, 2662] combine_rows_per_src=[5522, 5438, 5382, 5576, 5484, 5390, 5428, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38216 dup_rows=21806 dup_rows_remote=19020 dedup_ratio=0.332733 router_tokens_per_expert=[256, 245, 277, 263, 267, 243, 269, 246, 268, 260, 257, 254, 234, 247, 267, 291, 257, 248, 247, 239, 268, 233, 265, 259, 252, 279, 254, 233, 273, 264, 259, 243, 243, 250, 263, 261, 254, 263, 259, 238, 266, 244, 279, 231, 272, 246, 268, 282, 243, 235, 266, 266, 266, 264, 237, 253, 246, 224, 238, 276, 248, 253, 273, 246, 262, 239, 251, 271, 236, 261, 266, 238, 245, 266, 262, 264, 255, 284, 255, 240, 252, 283, 266, 252, 254, 275, 245, 242, 242, 257, 259, 246, 262, 253, 212, 258, 252, 272, 264, 251, 239, 272, 259, 252, 241, 261, 268, 277, 257, 278, 258, 260, 247, 265, 234, 249, 240, 260, 291, 249, 261, 240, 277, 271, 263, 242, 238, 257] payload_rows_per_dst=[5514, 5420, 5516, 5404, 5378, 5466, 5548, 5484] dup_rows_per_dst=[2786, 2726, 2742, 2664, 2780, 2812, 2612, 2684] combine_rows_per_src=[5514, 5420, 5516, 5404, 5378, 5466, 5548, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229136 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157343504 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314457872 weight_movement_bytes=18874368 total_nvlink_bytes=333332240 weight_to_token_ratio=0.060022 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38462 dup_rows=21652 dup_rows_remote=19122 dedup_ratio=0.330383 router_tokens_per_expert=[277, 283, 252, 270, 285, 244, 278, 266, 234, 262, 261, 263, 237, 244, 246, 257, 256, 264, 272, 237, 284, 261, 276, 284, 262, 227, 229, 279, 261, 251, 284, 265, 265, 240, 245, 260, 255, 248, 253, 217, 228, 246, 239, 242, 257, 276, 242, 273, 266, 244, 251, 293, 259, 257, 248, 256, 295, 239, 239, 242, 249, 264, 268, 278, 277, 243, 261, 247, 281, 278, 251, 246, 243, 258, 233, 248, 278, 237, 239, 249, 255, 243, 294, 253, 268, 264, 262, 250, 253, 279, 259, 241, 247, 255, 253, 253, 233, 253, 253, 266, 238, 257, 264, 237, 247, 243, 232, 246, 259, 250, 248, 250, 258, 255, 261, 277, 249, 265, 251, 266, 249, 254, 258, 234, 263, 265, 258, 246] payload_rows_per_dst=[5494, 5576, 5386, 5574, 5408, 5584, 5422, 5440] dup_rows_per_dst=[2824, 2808, 2586, 2722, 2730, 2674, 2530, 2778] combine_rows_per_src=[5494, 5576, 5386, 5574, 5408, 5584, 5422, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229480 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106280 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983080 weight_movement_bytes=18874368 total_nvlink_bytes=332857448 weight_to_token_ratio=0.060113 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228944 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156761680 ingress_bytes=232030208 link_bytes=232030208 token_bytes=313294416 weight_movement_bytes=75497472 total_nvlink_bytes=388791888 weight_to_token_ratio=0.240979 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223199287e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38114 dup_rows=21934 dup_rows_remote=19046 dedup_ratio=0.334686 router_tokens_per_expert=[292, 265, 290, 266, 271, 283, 227, 274, 266, 256, 227, 253, 286, 231, 239, 263, 239, 267, 249, 243, 279, 267, 259, 269, 259, 233, 277, 258, 259, 238, 273, 257, 253, 261, 281, 266, 257, 261, 254, 266, 236, 260, 252, 260, 258, 285, 275, 263, 240, 242, 264, 225, 253, 261, 266, 254, 270, 243, 265, 254, 299, 261, 236, 233, 257, 272, 263, 272, 266, 258, 259, 241, 274, 219, 258, 271, 283, 263, 241, 241, 213, 275, 251, 269, 252, 263, 247, 246, 269, 240, 250, 258, 237, 263, 258, 261, 247, 250, 243, 256, 241, 272, 216, 251, 243, 257, 224, 274, 238, 252, 285, 242, 243, 255, 248, 257, 270, 242, 263, 247, 232, 226, 260, 256, 274, 238, 261, 246] payload_rows_per_dst=[5442, 5522, 5488, 5436, 5524, 5424, 5396, 5370] dup_rows_per_dst=[2936, 2730, 2888, 2696, 2752, 2680, 2586, 2666] combine_rows_per_src=[5442, 5522, 5488, 5436, 5524, 5424, 5396, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38240 dup_rows=21822 dup_rows_remote=19126 dedup_ratio=0.332977 router_tokens_per_expert=[255, 284, 258, 252, 241, 227, 238, 263, 242, 237, 234, 264, 251, 220, 273, 261, 245, 242, 258, 257, 257, 261, 277, 269, 273, 266, 261, 259, 227, 252, 255, 263, 246, 254, 246, 281, 286, 235, 256, 278, 283, 231, 245, 268, 264, 253, 267, 271, 287, 258, 261, 265, 215, 226, 243, 259, 234, 283, 262, 254, 280, 255, 252, 237, 274, 242, 254, 261, 228, 235, 251, 252, 284, 279, 267, 258, 241, 270, 242, 247, 284, 266, 259, 238, 255, 261, 236, 276, 238, 268, 267, 257, 265, 258, 255, 267, 266, 270, 261, 266, 238, 220, 261, 253, 266, 268, 242, 258, 264, 264, 269, 265, 239, 252, 270, 241, 263, 250, 239, 259, 239, 247, 260, 251, 266, 258, 247, 264] payload_rows_per_dst=[5346, 5488, 5534, 5462, 5474, 5498, 5500, 5412] dup_rows_per_dst=[2654, 2756, 2794, 2680, 2696, 2802, 2762, 2678] combine_rows_per_src=[5346, 5488, 5534, 5462, 5474, 5498, 5500, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157540352 dispatch_meta_bytes=230336 combine_payload_bytes=157540352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157770688 ingress_bytes=176414720 link_bytes=176414720 token_bytes=315311040 weight_movement_bytes=18874368 total_nvlink_bytes=334185408 weight_to_token_ratio=0.059860 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690319046e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38300 dup_rows=21786 dup_rows_remote=18984 dedup_ratio=0.332428 router_tokens_per_expert=[232, 236, 234, 262, 241, 266, 255, 255, 258, 238, 260, 258, 285, 275, 249, 250, 259, 247, 256, 260, 263, 264, 272, 273, 262, 277, 270, 247, 240, 264, 244, 238, 241, 266, 254, 241, 245, 261, 253, 271, 246, 283, 258, 257, 239, 260, 261, 267, 276, 246, 277, 267, 261, 244, 250, 242, 260, 226, 250, 259, 276, 233, 282, 277, 253, 245, 258, 250, 242, 228, 267, 269, 267, 266, 251, 242, 286, 214, 216, 262, 234, 247, 276, 258, 270, 270, 249, 261, 268, 227, 240, 242, 234, 256, 232, 240, 257, 267, 250, 265, 253, 273, 255, 287, 265, 251, 263, 263, 256, 255, 258, 265, 267, 267, 275, 243, 261, 277, 274, 254, 269, 266, 258, 270, 213, 224, 273, 255] payload_rows_per_dst=[5410, 5518, 5536, 5450, 5406, 5362, 5516, 5552] dup_rows_per_dst=[2698, 2754, 2670, 2802, 2626, 2646, 2850, 2740] combine_rows_per_src=[5410, 5518, 5536, 5450, 5406, 5362, 5516, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38242 dup_rows=21828 dup_rows_remote=19134 dedup_ratio=0.333069 router_tokens_per_expert=[228, 236, 260, 252, 235, 253, 266, 260, 265, 260, 259, 252, 267, 241, 234, 285, 252, 262, 255, 230, 249, 252, 247, 235, 277, 264, 240, 270, 256, 254, 283, 239, 249, 250, 258, 247, 259, 253, 250, 270, 270, 250, 268, 246, 281, 264, 273, 266, 255, 271, 284, 253, 213, 249, 268, 283, 244, 264, 265, 254, 242, 273, 247, 245, 250, 253, 255, 256, 246, 261, 218, 261, 264, 274, 245, 245, 241, 269, 270, 228, 291, 258, 260, 276, 275, 277, 259, 247, 249, 276, 275, 257, 258, 235, 244, 274, 240, 252, 260, 245, 286, 256, 246, 264, 237, 276, 267, 240, 225, 248, 259, 258, 263, 255, 279, 235, 254, 268, 244, 241, 227, 286, 254, 253, 276, 253, 234, 258] payload_rows_per_dst=[5314, 5460, 5438, 5516, 5368, 5596, 5550, 5466] dup_rows_per_dst=[2538, 2670, 2732, 2842, 2704, 2872, 2776, 2694] combine_rows_per_src=[5314, 5460, 5438, 5516, 5368, 5596, 5550, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228640 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156343584 ingress_bytes=193863680 link_bytes=193863680 token_bytes=312458528 weight_movement_bytes=37748736 total_nvlink_bytes=350207264 weight_to_token_ratio=0.120812 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857506395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229464 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860504 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491544 weight_movement_bytes=18874368 total_nvlink_bytes=332365912 weight_to_token_ratio=0.060207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38264 dup_rows=21802 dup_rows_remote=19176 dedup_ratio=0.332672 router_tokens_per_expert=[232, 242, 259, 244, 247, 278, 255, 274, 232, 244, 235, 231, 266, 240, 244, 257, 255, 261, 264, 276, 249, 280, 237, 255, 269, 235, 240, 278, 231, 279, 281, 273, 253, 264, 241, 254, 237, 253, 251, 264, 264, 251, 257, 245, 282, 243, 247, 245, 253, 253, 271, 256, 267, 240, 263, 240, 225, 279, 250, 249, 262, 250, 263, 256, 261, 268, 251, 285, 245, 265, 266, 264, 285, 252, 252, 267, 287, 259, 280, 266, 263, 255, 245, 235, 243, 271, 258, 256, 232, 275, 221, 261, 248, 238, 285, 262, 232, 265, 267, 264, 284, 304, 282, 252, 243, 275, 272, 253, 263, 241, 249, 252, 270, 218, 249, 231, 248, 281, 265, 249, 229, 256, 287, 237, 246, 231, 237, 264] payload_rows_per_dst=[5350, 5528, 5418, 5466, 5618, 5470, 5520, 5364] dup_rows_per_dst=[2610, 2798, 2684, 2688, 2888, 2626, 2876, 2632] combine_rows_per_src=[5350, 5528, 5418, 5466, 5618, 5470, 5520, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229136 combine_payload_bytes=156876800 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157105936 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982736 weight_movement_bytes=18874368 total_nvlink_bytes=332857104 weight_to_token_ratio=0.060113 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229504 combine_payload_bytes=156639232 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156868736 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313507968 weight_movement_bytes=56623104 total_nvlink_bytes=370131072 weight_to_token_ratio=0.180611 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38438 dup_rows=21686 dup_rows_remote=18972 dedup_ratio=0.330902 router_tokens_per_expert=[247, 259, 264, 254, 264, 257, 258, 263, 288, 258, 258, 271, 243, 260, 262, 285, 240, 256, 247, 241, 260, 236, 259, 271, 259, 247, 267, 247, 239, 255, 274, 265, 242, 273, 242, 271, 266, 266, 244, 259, 275, 248, 264, 226, 239, 289, 246, 251, 270, 274, 235, 263, 297, 276, 252, 260, 268, 261, 246, 240, 240, 290, 267, 252, 267, 242, 256, 257, 236, 264, 213, 256, 245, 253, 239, 253, 251, 246, 266, 264, 237, 260, 249, 281, 247, 238, 259, 244, 264, 260, 262, 268, 251, 220, 247, 233, 244, 248, 269, 255, 237, 238, 266, 288, 257, 266, 248, 231, 235, 235, 239, 269, 273, 279, 228, 287, 263, 248, 250, 265, 261, 277, 240, 267, 259, 268, 255, 249] payload_rows_per_dst=[5592, 5412, 5482, 5564, 5422, 5416, 5416, 5546] dup_rows_per_dst=[2790, 2714, 2720, 2818, 2594, 2624, 2634, 2792] combine_rows_per_src=[5592, 5412, 5482, 5564, 5422, 5416, 5416, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38164 dup_rows=21818 dup_rows_remote=19138 dedup_ratio=0.332916 router_tokens_per_expert=[246, 280, 261, 252, 255, 265, 255, 259, 267, 233, 255, 258, 273, 255, 259, 244, 247, 243, 259, 249, 266, 280, 278, 270, 261, 237, 231, 232, 263, 292, 290, 256, 251, 240, 243, 239, 258, 260, 276, 251, 278, 223, 262, 265, 241, 252, 255, 282, 270, 257, 249, 253, 265, 275, 237, 273, 225, 267, 248, 259, 275, 261, 238, 254, 261, 241, 254, 287, 222, 237, 248, 246, 238, 247, 281, 260, 224, 249, 262, 257, 255, 250, 257, 288, 253, 246, 251, 264, 286, 258, 245, 256, 248, 240, 270, 236, 265, 254, 247, 279, 260, 259, 230, 229, 247, 245, 259, 276, 245, 233, 256, 284, 239, 247, 285, 250, 248, 252, 277, 269, 250, 256, 259, 251, 250, 260, 252, 285] payload_rows_per_dst=[5554, 5600, 5432, 5374, 5374, 5490, 5418, 5476] dup_rows_per_dst=[2680, 2912, 2720, 2634, 2654, 2716, 2718, 2784] combine_rows_per_src=[5554, 5600, 5432, 5374, 5374, 5490, 5418, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229760 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156959104 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688448 weight_movement_bytes=18874368 total_nvlink_bytes=332562816 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15060268880 prefetch_bytes=698351616 reduce_bytes=1019215872 weight_movement_bytes=1717567488 egress_bytes=7535638352 ingress_bytes=9242198016 link_bytes=9242198016 total_nvlink_bytes=16777836368 weight_to_token_ratio=0.114046 prefetch_share_of_ingress=0.075561 dedup_ratio=0.332534 prefetch_slots=74 unused_prefetch_slots=1462 slot_utilization=0.048177 reduce_slots=54 remote_demand=37 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=108331 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.844879041e-04 prefetch_bytes_per_gemm_flop=1.394012828e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38384 dup_rows=21762 dup_rows_remote=18898 dedup_ratio=0.332062 router_tokens_per_expert=[238, 238, 243, 235, 276, 267, 259, 245, 236, 256, 242, 274, 261, 269, 248, 243, 266, 263, 230, 235, 265, 247, 250, 256, 250, 237, 238, 256, 245, 246, 251, 254, 272, 268, 261, 277, 256, 250, 279, 239, 263, 274, 275, 282, 281, 246, 282, 227, 245, 289, 268, 256, 266, 236, 252, 250, 258, 256, 272, 228, 273, 239, 259, 279, 235, 268, 269, 246, 288, 247, 245, 259, 279, 243, 246, 261, 282, 267, 256, 235, 250, 234, 248, 257, 247, 256, 270, 263, 278, 313, 222, 279, 232, 272, 252, 233, 232, 271, 233, 266, 278, 288, 266, 216, 240, 253, 258, 268, 246, 263, 282, 267, 259, 246, 273, 278, 222, 246, 255, 239, 265, 265, 271, 267, 238, 251, 212, 245] payload_rows_per_dst=[5436, 5416, 5564, 5486, 5556, 5424, 5390, 5502] dup_rows_per_dst=[2624, 2562, 2900, 2766, 2696, 2788, 2864, 2562] combine_rows_per_src=[5436, 5416, 5564, 5486, 5556, 5424, 5390, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229640 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671688 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113736 weight_movement_bytes=18874368 total_nvlink_bytes=333988104 weight_to_token_ratio=0.059897 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38524 dup_rows=21596 dup_rows_remote=18876 dedup_ratio=0.329529 router_tokens_per_expert=[242, 255, 245, 269, 256, 259, 257, 245, 237, 267, 240, 265, 215, 262, 256, 239, 268, 230, 263, 244, 241, 258, 238, 260, 282, 245, 252, 245, 246, 256, 222, 256, 259, 235, 251, 270, 249, 243, 258, 256, 251, 266, 242, 255, 245, 266, 252, 270, 222, 263, 241, 270, 273, 246, 216, 231, 249, 254, 233, 260, 281, 250, 265, 243, 259, 273, 221, 273, 271, 236, 286, 253, 249, 252, 248, 265, 296, 242, 256, 261, 254, 238, 282, 270, 248, 233, 243, 265, 274, 257, 285, 245, 259, 246, 243, 243, 270, 250, 264, 288, 259, 270, 233, 243, 275, 253, 264, 253, 269, 246, 255, 267, 281, 296, 230, 266, 289, 256, 240, 261, 290, 265, 259, 275, 276, 271, 262, 286] payload_rows_per_dst=[5450, 5406, 5416, 5438, 5538, 5574, 5498, 5620] dup_rows_per_dst=[2568, 2606, 2720, 2556, 2744, 2596, 2820, 2986] combine_rows_per_src=[5450, 5406, 5416, 5438, 5538, 5574, 5498, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229208 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156548952 ingress_bytes=212942848 link_bytes=212942848 token_bytes=312868696 weight_movement_bytes=56623104 total_nvlink_bytes=369491800 weight_to_token_ratio=0.180980 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040313595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38146 dup_rows=21850 dup_rows_remote=19122 dedup_ratio=0.333405 router_tokens_per_expert=[288, 251, 281, 273, 253, 275, 261, 240, 251, 232, 282, 257, 219, 252, 270, 269, 241, 260, 249, 228, 224, 265, 244, 238, 261, 244, 262, 232, 256, 275, 253, 250, 250, 261, 275, 228, 254, 264, 250, 241, 266, 242, 249, 260, 240, 263, 253, 275, 282, 244, 249, 261, 253, 247, 239, 250, 229, 236, 260, 245, 261, 255, 255, 269, 279, 257, 257, 246, 265, 254, 274, 245, 273, 240, 255, 251, 259, 286, 252, 241, 255, 259, 270, 253, 253, 277, 265, 224, 273, 287, 254, 229, 252, 236, 272, 240, 288, 224, 279, 267, 262, 276, 266, 268, 245, 252, 251, 273, 277, 272, 264, 270, 238, 250, 262, 241, 250, 242, 254, 274, 261, 252, 281, 246, 256, 237, 251, 264] payload_rows_per_dst=[5506, 5338, 5426, 5420, 5540, 5470, 5578, 5408] dup_rows_per_dst=[2802, 2626, 2716, 2650, 2728, 2728, 2890, 2710] combine_rows_per_src=[5506, 5338, 5426, 5420, 5540, 5470, 5578, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229128 combine_payload_bytes=157220864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157449992 ingress_bytes=194969600 link_bytes=194969600 token_bytes=314670856 weight_movement_bytes=37748736 total_nvlink_bytes=352419592 weight_to_token_ratio=0.119963 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868102776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38060 dup_rows=21922 dup_rows_remote=19150 dedup_ratio=0.334503 router_tokens_per_expert=[247, 245, 262, 260, 261, 261, 238, 244, 259, 263, 281, 260, 251, 260, 271, 272, 248, 233, 255, 244, 222, 250, 259, 269, 275, 253, 253, 256, 267, 236, 258, 248, 231, 264, 225, 279, 286, 260, 246, 263, 253, 256, 252, 250, 278, 304, 247, 266, 258, 260, 263, 269, 275, 268, 256, 261, 246, 220, 272, 247, 243, 253, 268, 249, 253, 244, 240, 264, 265, 233, 269, 269, 258, 255, 235, 272, 263, 246, 266, 282, 232, 287, 255, 255, 272, 265, 270, 240, 243, 272, 232, 268, 242, 247, 246, 275, 269, 255, 266, 270, 248, 245, 256, 253, 247, 263, 235, 252, 261, 252, 252, 257, 245, 238, 245, 272, 251, 241, 268, 228, 249, 224, 253, 242, 284, 274, 262, 267] payload_rows_per_dst=[5468, 5310, 5516, 5504, 5456, 5460, 5346, 5554] dup_rows_per_dst=[2802, 2742, 2804, 2712, 2772, 2742, 2576, 2772] combine_rows_per_src=[5468, 5310, 5516, 5504, 5456, 5460, 5346, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157794304 dispatch_meta_bytes=229600 combine_payload_bytes=157794304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158023904 ingress_bytes=195543040 link_bytes=195543040 token_bytes=315818208 weight_movement_bytes=37748736 total_nvlink_bytes=353566944 weight_to_token_ratio=0.119527 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873597196e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=229072 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156475088 ingress_bytes=175120384 link_bytes=175120384 token_bytes=312721104 weight_movement_bytes=18874368 total_nvlink_bytes=331595472 weight_to_token_ratio=0.060355 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677917355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15068843016 prefetch_bytes=736100352 reduce_bytes=943718400 weight_movement_bytes=1679818752 egress_bytes=7539928072 ingress_bytes=9208733696 link_bytes=9208733696 total_nvlink_bytes=16748661768 weight_to_token_ratio=0.111476 prefetch_share_of_ingress=0.079935 dedup_ratio=0.332325 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=50 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080584 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.838199069e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38164 dup_rows=21980 dup_rows_remote=19222 dedup_ratio=0.335388 router_tokens_per_expert=[268, 282, 263, 265, 246, 272, 261, 272, 254, 304, 251, 247, 258, 252, 259, 257, 232, 248, 256, 252, 261, 239, 237, 255, 272, 241, 241, 259, 271, 254, 244, 249, 232, 268, 260, 243, 234, 259, 239, 250, 264, 273, 246, 246, 246, 266, 260, 246, 264, 245, 244, 247, 260, 255, 267, 273, 252, 276, 255, 255, 274, 247, 276, 259, 270, 275, 266, 263, 237, 250, 260, 247, 247, 258, 271, 249, 246, 270, 248, 249, 242, 235, 251, 250, 256, 277, 236, 237, 236, 293, 267, 256, 264, 241, 278, 256, 257, 222, 252, 276, 237, 273, 287, 274, 246, 247, 245, 246, 252, 251, 286, 250, 252, 266, 241, 254, 271, 244, 246, 274, 249, 247, 255, 233, 239, 295, 252, 265] payload_rows_per_dst=[5548, 5384, 5362, 5490, 5500, 5392, 5398, 5482] dup_rows_per_dst=[2874, 2638, 2702, 2808, 2712, 2758, 2804, 2684] combine_rows_per_src=[5548, 5384, 5362, 5490, 5500, 5392, 5398, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38404 dup_rows=21688 dup_rows_remote=18874 dedup_ratio=0.330933 router_tokens_per_expert=[263, 294, 226, 254, 241, 244, 259, 250, 251, 296, 240, 262, 247, 240, 249, 272, 273, 254, 248, 284, 257, 271, 288, 269, 253, 242, 251, 258, 255, 239, 232, 255, 245, 281, 259, 273, 253, 261, 268, 266, 254, 241, 245, 274, 247, 251, 286, 252, 248, 246, 255, 255, 259, 235, 254, 260, 280, 272, 264, 241, 243, 252, 259, 248, 257, 261, 258, 245, 248, 236, 269, 244, 243, 283, 277, 234, 266, 257, 269, 263, 257, 282, 237, 248, 245, 274, 250, 242, 258, 250, 264, 246, 285, 224, 246, 258, 257, 241, 240, 229, 242, 249, 252, 233, 286, 251, 265, 266, 233, 255, 283, 265, 246, 253, 268, 258, 260, 258, 241, 266, 269, 245, 242, 259, 265, 265, 253, 253] payload_rows_per_dst=[5518, 5444, 5562, 5512, 5482, 5428, 5486, 5416] dup_rows_per_dst=[2658, 2814, 2750, 2630, 2738, 2776, 2608, 2714] combine_rows_per_src=[5518, 5444, 5562, 5512, 5482, 5428, 5486, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=228840 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156122600 ingress_bytes=193642496 link_bytes=193642496 token_bytes=312016360 weight_movement_bytes=37748736 total_nvlink_bytes=349765096 weight_to_token_ratio=0.120983 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855387119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15066105384 prefetch_bytes=773849088 reduce_bytes=717225984 weight_movement_bytes=1491075072 egress_bytes=7538558504 ingress_bytes=9018621952 link_bytes=9018621952 total_nvlink_bytes=16557180456 weight_to_token_ratio=0.098969 prefetch_share_of_ingress=0.085806 dedup_ratio=0.332455 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=38 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.800249961e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38224 dup_rows=21738 dup_rows_remote=18846 dedup_ratio=0.331696 router_tokens_per_expert=[287, 259, 245, 298, 242, 232, 273, 253, 245, 288, 254, 262, 248, 282, 290, 273, 273, 259, 241, 247, 257, 254, 256, 241, 262, 270, 254, 258, 268, 257, 266, 258, 282, 265, 231, 261, 273, 242, 219, 260, 240, 248, 237, 270, 261, 266, 269, 243, 240, 271, 251, 266, 251, 269, 269, 269, 264, 254, 252, 242, 244, 255, 232, 243, 253, 243, 262, 269, 271, 250, 249, 259, 245, 208, 261, 250, 246, 251, 262, 242, 238, 249, 260, 246, 251, 248, 243, 241, 253, 240, 271, 242, 277, 246, 252, 267, 262, 262, 266, 260, 269, 233, 238, 258, 275, 243, 266, 255, 246, 257, 262, 279, 235, 256, 276, 270, 247, 247, 254, 244, 254, 271, 265, 244, 262, 249, 267, 260] payload_rows_per_dst=[5574, 5576, 5550, 5312, 5430, 5582, 5348, 5426] dup_rows_per_dst=[2892, 2662, 2974, 2442, 2644, 2990, 2390, 2744] combine_rows_per_src=[5574, 5576, 5550, 5312, 5430, 5582, 5348, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229544 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549288 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312869032 weight_movement_bytes=18874368 total_nvlink_bytes=331743400 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229112 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157531896 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314834680 weight_movement_bytes=18874368 total_nvlink_bytes=333709048 weight_to_token_ratio=0.059950 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15032240456 prefetch_bytes=754974720 reduce_bytes=792723456 weight_movement_bytes=1547698176 egress_bytes=7521610056 ingress_bytes=9058328576 link_bytes=9058558520 total_nvlink_bytes=16579938632 weight_to_token_ratio=0.102959 prefetch_share_of_ingress=0.083346 dedup_ratio=0.332772 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=42 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080946 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.808221889e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228280 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156793784 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313359288 weight_movement_bytes=37748736 total_nvlink_bytes=351108024 weight_to_token_ratio=0.120465 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38378 dup_rows=21686 dup_rows_remote=18932 dedup_ratio=0.330902 router_tokens_per_expert=[278, 239, 262, 254, 272, 260, 252, 242, 266, 263, 245, 255, 249, 272, 230, 269, 251, 262, 238, 277, 266, 258, 274, 271, 280, 264, 254, 252, 257, 251, 274, 263, 231, 242, 258, 264, 247, 266, 226, 281, 238, 256, 264, 247, 258, 245, 248, 255, 248, 277, 270, 275, 254, 262, 242, 274, 245, 258, 249, 265, 249, 269, 252, 259, 258, 267, 260, 251, 260, 260, 254, 235, 248, 254, 256, 245, 251, 245, 268, 301, 251, 264, 264, 269, 220, 242, 253, 260, 260, 252, 255, 264, 255, 242, 244, 262, 266, 231, 268, 242, 267, 262, 254, 254, 239, 258, 262, 265, 219, 239, 239, 258, 247, 272, 274, 270, 279, 249, 217, 270, 218, 266, 256, 268, 252, 253, 250, 260] payload_rows_per_dst=[5532, 5558, 5374, 5558, 5472, 5458, 5430, 5468] dup_rows_per_dst=[2684, 2826, 2678, 2738, 2754, 2656, 2616, 2734] combine_rows_per_src=[5532, 5558, 5374, 5558, 5472, 5458, 5430, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15062599600 prefetch_bytes=849346560 reduce_bytes=434110464 weight_movement_bytes=1283457024 egress_bytes=7536805808 ingress_bytes=8809250816 link_bytes=8809250816 total_nvlink_bytes=16346056624 weight_to_token_ratio=0.085208 prefetch_share_of_ingress=0.096415 dedup_ratio=0.332497 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=23 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082538 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.758456394e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229240 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157425528 ingress_bytes=194945024 link_bytes=194945024 token_bytes=314621816 weight_movement_bytes=37748736 total_nvlink_bytes=352370552 weight_to_token_ratio=0.119981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867867301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38250 dup_rows=21712 dup_rows_remote=19020 dedup_ratio=0.331299 router_tokens_per_expert=[262, 237, 241, 256, 280, 249, 249, 224, 216, 245, 273, 241, 251, 259, 276, 259, 266, 244, 254, 275, 249, 265, 260, 253, 247, 245, 248, 239, 279, 232, 247, 275, 267, 232, 246, 257, 240, 264, 265, 277, 238, 261, 228, 261, 250, 241, 272, 240, 256, 271, 279, 244, 250, 250, 289, 255, 253, 237, 238, 277, 291, 230, 283, 257, 268, 249, 233, 233, 235, 256, 226, 255, 249, 270, 263, 256, 244, 246, 259, 279, 260, 262, 240, 264, 241, 279, 261, 271, 269, 260, 281, 251, 229, 251, 272, 242, 240, 249, 273, 268, 251, 243, 234, 262, 276, 249, 283, 258, 259, 270, 241, 279, 247, 264, 249, 231, 261, 267, 254, 278, 257, 267, 281, 274, 274, 238, 277, 265] payload_rows_per_dst=[5400, 5468, 5408, 5580, 5392, 5574, 5470, 5532] dup_rows_per_dst=[2636, 2688, 2670, 2740, 2650, 2692, 2800, 2836] combine_rows_per_src=[5400, 5468, 5408, 5580, 5392, 5574, 5470, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15053944200 prefetch_bytes=754974720 reduce_bytes=924844032 weight_movement_bytes=1679818752 egress_bytes=7532475784 ingress_bytes=9201287168 link_bytes=9201287168 total_nvlink_bytes=16733762952 weight_to_token_ratio=0.111587 prefetch_share_of_ingress=0.082051 dedup_ratio=0.332635 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=49 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080295 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.836712632e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38284 dup_rows=21652 dup_rows_remote=18794 dedup_ratio=0.330383 router_tokens_per_expert=[254, 248, 285, 253, 240, 239, 237, 260, 247, 243, 261, 236, 288, 254, 245, 259, 260, 246, 295, 280, 280, 266, 249, 264, 238, 225, 238, 265, 224, 240, 252, 243, 255, 248, 272, 250, 258, 283, 262, 290, 287, 258, 239, 271, 252, 285, 246, 226, 246, 245, 229, 244, 253, 260, 256, 265, 250, 251, 274, 250, 282, 254, 267, 247, 279, 261, 283, 272, 246, 277, 258, 242, 263, 271, 241, 272, 263, 271, 213, 292, 230, 245, 251, 234, 242, 249, 261, 250, 266, 260, 274, 270, 266, 253, 280, 243, 273, 263, 240, 255, 234, 225, 261, 254, 287, 267, 250, 250, 252, 239, 252, 254, 244, 238, 249, 249, 266, 250, 244, 247, 262, 247, 259, 259, 271, 249, 259, 272] payload_rows_per_dst=[5600, 5548, 5480, 5428, 5422, 5488, 5426, 5492] dup_rows_per_dst=[2858, 2754, 2884, 2612, 2560, 2660, 2686, 2638] combine_rows_per_src=[5600, 5548, 5480, 5428, 5422, 5488, 5426, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229080 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156901080 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313573080 weight_movement_bytes=37748736 total_nvlink_bytes=351321816 weight_to_token_ratio=0.120383 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228312 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157039576 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313850840 weight_movement_bytes=18874368 total_nvlink_bytes=332725208 weight_to_token_ratio=0.060138 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38386 dup_rows=21722 dup_rows_remote=19068 dedup_ratio=0.331451 router_tokens_per_expert=[261, 258, 270, 216, 272, 265, 262, 269, 254, 265, 258, 262, 265, 279, 297, 248, 239, 219, 231, 255, 266, 277, 265, 233, 249, 271, 250, 256, 263, 245, 246, 236, 255, 265, 275, 244, 258, 261, 266, 270, 262, 239, 234, 242, 258, 245, 282, 279, 245, 279, 250, 270, 251, 237, 256, 261, 271, 266, 229, 255, 242, 262, 261, 260, 259, 248, 262, 249, 248, 245, 274, 249, 274, 248, 251, 229, 267, 263, 268, 259, 264, 241, 271, 232, 263, 250, 263, 269, 257, 257, 235, 261, 261, 244, 224, 249, 293, 253, 264, 232, 285, 274, 240, 243, 253, 239, 275, 259, 270, 240, 238, 238, 249, 263, 262, 232, 277, 244, 273, 255, 248, 258, 270, 237, 260, 249, 262, 267] payload_rows_per_dst=[5556, 5394, 5520, 5420, 5490, 5428, 5462, 5544] dup_rows_per_dst=[2846, 2608, 2750, 2770, 2696, 2654, 2730, 2668] combine_rows_per_src=[5556, 5394, 5520, 5420, 5490, 5428, 5462, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43514 payload_rows_remote=38090 dup_rows=22022 dup_rows_remote=19280 dedup_ratio=0.336029 router_tokens_per_expert=[258, 245, 252, 263, 254, 247, 233, 221, 247, 253, 250, 255, 270, 263, 276, 276, 251, 253, 257, 241, 266, 253, 281, 239, 242, 252, 290, 259, 257, 246, 232, 265, 245, 260, 249, 240, 256, 257, 273, 285, 245, 239, 271, 242, 226, 245, 269, 243, 258, 224, 267, 267, 277, 264, 259, 252, 257, 254, 234, 238, 242, 253, 268, 259, 262, 240, 270, 260, 259, 278, 268, 275, 247, 252, 221, 243, 245, 282, 274, 260, 234, 242, 218, 265, 231, 240, 251, 258, 233, 261, 253, 269, 268, 264, 233, 238, 272, 261, 242, 267, 255, 267, 272, 253, 293, 261, 261, 281, 245, 247, 239, 263, 259, 231, 272, 255, 272, 272, 268, 264, 279, 251, 251, 277, 263, 266, 267, 283] payload_rows_per_dst=[5424, 5592, 5476, 5376, 5512, 5338, 5266, 5530] dup_rows_per_dst=[2742, 3024, 2678, 2666, 2960, 2578, 2444, 2930] combine_rows_per_src=[5424, 5592, 5476, 5376, 5512, 5338, 5266, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229816 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458872 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687928 weight_movement_bytes=18874368 total_nvlink_bytes=333562296 weight_to_token_ratio=0.059978 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=229480 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156246120 ingress_bytes=193765376 link_bytes=193765376 token_bytes=312262760 weight_movement_bytes=37748736 total_nvlink_bytes=350011496 weight_to_token_ratio=0.120888 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856564494e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15066941112 prefetch_bytes=811597824 reduce_bytes=528482304 weight_movement_bytes=1340080128 egress_bytes=7538976440 ingress_bytes=8868044800 link_bytes=8868274424 total_nvlink_bytes=16407021240 weight_to_token_ratio=0.088942 prefetch_share_of_ingress=0.091519 dedup_ratio=0.332219 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=28 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081959 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.770238376e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38242 dup_rows=21868 dup_rows_remote=19136 dedup_ratio=0.333679 router_tokens_per_expert=[257, 243, 257, 241, 251, 279, 238, 271, 252, 250, 241, 246, 266, 247, 270, 269, 265, 257, 262, 250, 260, 229, 270, 244, 293, 233, 257, 272, 237, 230, 269, 266, 247, 250, 258, 259, 259, 239, 251, 261, 252, 242, 253, 240, 260, 255, 250, 256, 229, 264, 277, 247, 228, 259, 240, 220, 217, 263, 275, 255, 263, 270, 261, 248, 242, 250, 228, 274, 264, 284, 250, 271, 263, 289, 249, 244, 246, 277, 233, 259, 233, 268, 277, 269, 268, 244, 273, 253, 263, 268, 260, 274, 282, 237, 255, 253, 247, 298, 261, 262, 242, 251, 256, 268, 261, 254, 244, 249, 260, 255, 258, 277, 255, 271, 233, 286, 263, 244, 262, 259, 235, 272, 250, 292, 261, 237, 261, 224] payload_rows_per_dst=[5426, 5588, 5422, 5438, 5606, 5362, 5524, 5302] dup_rows_per_dst=[2732, 2928, 2706, 2594, 2810, 2664, 2762, 2672] combine_rows_per_src=[5426, 5588, 5422, 5438, 5606, 5362, 5524, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229512 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156868744 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313507976 weight_movement_bytes=56623104 total_nvlink_bytes=370131080 weight_to_token_ratio=0.180611 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38170 dup_rows=21854 dup_rows_remote=19080 dedup_ratio=0.333466 router_tokens_per_expert=[254, 250, 270, 270, 225, 244, 252, 268, 238, 267, 240, 274, 250, 245, 291, 271, 251, 243, 249, 245, 214, 267, 255, 269, 289, 257, 235, 222, 270, 265, 232, 263, 255, 240, 255, 246, 231, 274, 246, 252, 244, 249, 283, 263, 231, 245, 266, 271, 233, 268, 288, 263, 242, 259, 255, 266, 239, 253, 263, 271, 257, 265, 242, 267, 254, 286, 263, 278, 241, 257, 253, 259, 275, 254, 255, 245, 259, 263, 260, 249, 245, 248, 269, 242, 244, 281, 234, 242, 258, 250, 249, 237, 248, 214, 274, 272, 255, 237, 224, 250, 277, 292, 270, 278, 241, 261, 278, 269, 271, 244, 249, 280, 253, 252, 265, 255, 248, 264, 270, 244, 250, 274, 278, 263, 255, 254, 254, 238] payload_rows_per_dst=[5512, 5684, 5306, 5414, 5372, 5622, 5510, 5262] dup_rows_per_dst=[2774, 2894, 2470, 2780, 2738, 2922, 2834, 2442] combine_rows_per_src=[5512, 5684, 5306, 5414, 5372, 5622, 5510, 5262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229000 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573320 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917640 weight_movement_bytes=18874368 total_nvlink_bytes=331792008 weight_to_token_ratio=0.060317 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15027456000 prefetch_bytes=773849088 reduce_bytes=792723456 weight_movement_bytes=1566572544 egress_bytes=7519217664 ingress_bytes=9074810880 link_bytes=9074810880 total_nvlink_bytes=16594028544 weight_to_token_ratio=0.104247 prefetch_share_of_ingress=0.085274 dedup_ratio=0.332872 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=42 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080946 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.811466099e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none + [2026-08-29 23:39:10] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 787.5 | throughput per GPU (TFLOP/s/GPU): 113.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.903749E+00 | loss scale: 1.0 | grad norm: 3.266 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38086 dup_rows=21954 dup_rows_remote=19062 dedup_ratio=0.334991 router_tokens_per_expert=[255, 259, 272, 259, 251, 233, 220, 262, 281, 235, 240, 268, 253, 252, 239, 256, 257, 258, 242, 272, 255, 236, 250, 291, 234, 253, 254, 266, 224, 283, 256, 266, 232, 273, 261, 264, 272, 215, 243, 275, 277, 259, 260, 272, 255, 276, 262, 228, 239, 247, 255, 271, 266, 249, 249, 253, 228, 242, 244, 253, 290, 273, 256, 268, 260, 250, 273, 253, 258, 250, 253, 247, 242, 247, 284, 237, 266, 260, 251, 250, 258, 277, 245, 255, 238, 255, 256, 248, 245, 235, 257, 282, 250, 269, 246, 241, 251, 274, 223, 269, 265, 288, 234, 276, 248, 263, 242, 244, 250, 265, 263, 274, 247, 284, 271, 235, 251, 257, 263, 264, 245, 263, 272, 257, 278, 271, 247, 257] payload_rows_per_dst=[5388, 5304, 5526, 5538, 5474, 5360, 5496, 5496] dup_rows_per_dst=[2682, 2556, 2722, 2962, 2688, 2682, 2770, 2892] combine_rows_per_src=[5388, 5304, 5526, 5538, 5474, 5360, 5496, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38304 dup_rows=21722 dup_rows_remote=18860 dedup_ratio=0.331451 router_tokens_per_expert=[255, 274, 272, 240, 243, 257, 242, 265, 261, 238, 251, 245, 281, 235, 268, 242, 243, 301, 269, 262, 274, 278, 252, 274, 225, 261, 250, 272, 248, 287, 251, 239, 243, 250, 242, 238, 226, 272, 252, 268, 230, 262, 246, 268, 269, 252, 259, 269, 246, 255, 245, 258, 261, 273, 259, 244, 250, 227, 270, 257, 263, 274, 256, 250, 226, 237, 254, 242, 226, 247, 231, 245, 275, 249, 245, 264, 278, 254, 280, 291, 258, 276, 277, 257, 238, 257, 268, 272, 230, 224, 267, 233, 258, 254, 248, 253, 272, 273, 261, 255, 257, 243, 259, 247, 236, 237, 242, 289, 273, 294, 254, 241, 255, 265, 268, 250, 249, 266, 268, 244, 240, 277, 257, 245, 245, 252, 280, 271] payload_rows_per_dst=[5470, 5510, 5408, 5516, 5418, 5468, 5508, 5516] dup_rows_per_dst=[2668, 2862, 2684, 2660, 2670, 2672, 2758, 2748] combine_rows_per_src=[5470, 5510, 5408, 5516, 5418, 5468, 5508, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38354 dup_rows=21770 dup_rows_remote=19052 dedup_ratio=0.332184 router_tokens_per_expert=[258, 262, 263, 249, 249, 276, 262, 257, 259, 257, 255, 247, 262, 251, 234, 269, 260, 282, 244, 276, 261, 258, 253, 230, 267, 254, 272, 276, 255, 246, 253, 243, 268, 279, 256, 259, 250, 245, 273, 249, 262, 258, 229, 255, 285, 265, 245, 249, 258, 258, 244, 258, 277, 250, 219, 256, 255, 284, 266, 241, 250, 240, 253, 254, 254, 264, 266, 299, 254, 251, 255, 243, 230, 271, 253, 255, 252, 242, 246, 242, 230, 236, 265, 249, 274, 260, 245, 250, 261, 250, 273, 290, 242, 242, 250, 248, 236, 243, 245, 254, 239, 277, 241, 279, 267, 244, 244, 267, 282, 257, 256, 268, 281, 263, 262, 232, 253, 253, 247, 259, 238, 277, 243, 248, 272, 257, 264, 248] payload_rows_per_dst=[5488, 5468, 5520, 5454, 5508, 5412, 5456, 5460] dup_rows_per_dst=[2732, 2792, 2734, 2672, 2646, 2718, 2742, 2734] combine_rows_per_src=[5488, 5468, 5520, 5454, 5508, 5412, 5456, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38448 dup_rows=21608 dup_rows_remote=18954 dedup_ratio=0.329712 router_tokens_per_expert=[264, 258, 292, 276, 257, 247, 238, 262, 248, 251, 250, 247, 243, 277, 264, 272, 255, 218, 264, 275, 244, 276, 232, 254, 243, 271, 262, 271, 279, 243, 254, 266, 274, 242, 271, 237, 268, 244, 261, 233, 263, 233, 231, 232, 279, 208, 277, 272, 279, 238, 260, 246, 267, 245, 280, 236, 240, 258, 266, 266, 252, 261, 287, 252, 295, 261, 254, 246, 255, 233, 274, 274, 250, 258, 247, 234, 244, 243, 266, 233, 277, 270, 252, 252, 273, 263, 275, 245, 259, 250, 277, 256, 239, 272, 265, 263, 259, 239, 271, 258, 241, 254, 245, 277, 248, 239, 236, 260, 232, 250, 274, 253, 260, 244, 241, 274, 238, 258, 260, 232, 250, 272, 262, 288, 220, 250, 249, 268] payload_rows_per_dst=[5510, 5482, 5478, 5538, 5480, 5528, 5440, 5472] dup_rows_per_dst=[2782, 2732, 2572, 2728, 2654, 2848, 2632, 2660] combine_rows_per_src=[5510, 5482, 5478, 5538, 5480, 5528, 5440, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38150 dup_rows=21892 dup_rows_remote=19082 dedup_ratio=0.334045 router_tokens_per_expert=[245, 269, 245, 259, 230, 253, 236, 279, 250, 245, 254, 248, 272, 264, 265, 273, 227, 231, 261, 261, 241, 260, 266, 268, 271, 242, 254, 280, 241, 249, 254, 258, 234, 236, 268, 242, 266, 257, 288, 282, 233, 252, 248, 276, 253, 272, 275, 270, 230, 232, 260, 236, 247, 237, 272, 262, 250, 273, 240, 277, 257, 244, 230, 271, 255, 264, 253, 298, 263, 256, 258, 266, 229, 251, 235, 262, 278, 243, 269, 251, 232, 220, 258, 286, 272, 290, 249, 268, 264, 261, 258, 254, 229, 248, 240, 267, 258, 266, 246, 283, 247, 275, 253, 263, 257, 246, 254, 257, 246, 276, 251, 285, 244, 273, 277, 244, 218, 262, 269, 245, 237, 237, 235, 279, 268, 251, 262, 256] payload_rows_per_dst=[5404, 5434, 5494, 5418, 5496, 5422, 5492, 5484] dup_rows_per_dst=[2770, 2694, 2810, 2618, 2766, 2770, 2834, 2630] combine_rows_per_src=[5404, 5434, 5494, 5418, 5496, 5422, 5492, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38194 dup_rows=21858 dup_rows_remote=19322 dedup_ratio=0.333527 router_tokens_per_expert=[234, 296, 257, 260, 246, 234, 256, 278, 265, 248, 263, 234, 258, 277, 266, 263, 251, 260, 259, 260, 278, 275, 246, 273, 254, 254, 257, 254, 278, 239, 227, 262, 270, 234, 265, 254, 259, 279, 267, 248, 246, 234, 252, 257, 268, 242, 263, 263, 265, 272, 240, 252, 264, 236, 244, 229, 239, 219, 253, 250, 253, 262, 276, 256, 250, 259, 262, 254, 237, 261, 255, 245, 265, 257, 253, 255, 267, 253, 226, 260, 253, 261, 256, 268, 262, 248, 257, 281, 239, 245, 240, 250, 265, 264, 277, 260, 261, 251, 264, 252, 266, 264, 262, 247, 269, 257, 241, 286, 251, 271, 267, 267, 251, 232, 257, 258, 280, 260, 239, 258, 240, 270, 234, 242, 270, 253, 256, 234] payload_rows_per_dst=[5532, 5428, 5420, 5484, 5378, 5492, 5550, 5394] dup_rows_per_dst=[2738, 2826, 2782, 2536, 2740, 2760, 2802, 2674] combine_rows_per_src=[5532, 5428, 5420, 5484, 5378, 5492, 5550, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=228592 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156228848 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312229104 weight_movement_bytes=37748736 total_nvlink_bytes=349977840 weight_to_token_ratio=0.120901 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228656 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157121840 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314015024 weight_movement_bytes=37748736 total_nvlink_bytes=351763760 weight_to_token_ratio=0.120213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229624 combine_payload_bytes=157097984 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157327608 ingress_bytes=213721088 link_bytes=213721088 token_bytes=314425592 weight_movement_bytes=56623104 total_nvlink_bytes=371048696 weight_to_token_ratio=0.180084 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047770308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=229608 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157712616 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315195624 weight_movement_bytes=18874368 total_nvlink_bytes=334069992 weight_to_token_ratio=0.059881 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228928 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156491328 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753728 weight_movement_bytes=18874368 total_nvlink_bytes=331628096 weight_to_token_ratio=0.060349 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=230064 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156672688 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313115312 weight_movement_bytes=18874368 total_nvlink_bytes=331989680 weight_to_token_ratio=0.060279 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38292 dup_rows=21734 dup_rows_remote=19028 dedup_ratio=0.331635 router_tokens_per_expert=[248, 230, 282, 245, 263, 251, 244, 260, 269, 272, 268, 273, 277, 242, 253, 285, 249, 238, 247, 254, 270, 286, 256, 291, 283, 248, 245, 250, 262, 246, 263, 273, 271, 231, 300, 293, 278, 273, 256, 254, 257, 247, 256, 247, 250, 236, 268, 252, 227, 232, 275, 240, 262, 260, 239, 250, 266, 236, 259, 269, 259, 243, 230, 235, 247, 260, 223, 276, 298, 269, 240, 242, 229, 236, 249, 270, 260, 221, 248, 231, 252, 271, 234, 260, 246, 253, 240, 249, 269, 250, 271, 256, 262, 235, 267, 271, 275, 260, 259, 260, 267, 231, 258, 261, 271, 251, 259, 245, 275, 278, 259, 229, 268, 252, 244, 282, 257, 239, 230, 261, 239, 262, 240, 250, 258, 269, 255, 265] payload_rows_per_dst=[5498, 5556, 5336, 5372, 5326, 5654, 5550, 5510] dup_rows_per_dst=[2826, 2766, 2402, 2546, 2706, 3118, 2664, 2706] combine_rows_per_src=[5498, 5556, 5336, 5372, 5326, 5654, 5550, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38322 dup_rows=21794 dup_rows_remote=19102 dedup_ratio=0.332550 router_tokens_per_expert=[247, 264, 263, 279, 269, 246, 262, 263, 242, 259, 222, 259, 251, 276, 265, 247, 243, 243, 246, 241, 250, 258, 240, 247, 240, 242, 264, 258, 281, 250, 271, 282, 242, 225, 256, 265, 266, 248, 252, 283, 241, 277, 262, 242, 259, 268, 238, 276, 246, 251, 274, 226, 255, 259, 267, 265, 263, 240, 268, 246, 273, 227, 254, 283, 245, 228, 272, 258, 267, 247, 253, 273, 238, 226, 259, 255, 250, 257, 266, 279, 245, 241, 280, 265, 274, 272, 241, 240, 275, 238, 244, 221, 251, 249, 229, 284, 273, 261, 251, 261, 250, 258, 242, 250, 271, 263, 234, 282, 242, 284, 255, 284, 224, 227, 255, 261, 233, 258, 254, 264, 256, 264, 274, 281, 280, 258, 262, 267] payload_rows_per_dst=[5440, 5420, 5486, 5432, 5414, 5508, 5526, 5516] dup_rows_per_dst=[2788, 2692, 2714, 2762, 2732, 2590, 2796, 2720] combine_rows_per_src=[5440, 5420, 5486, 5432, 5414, 5508, 5526, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38372 dup_rows=21618 dup_rows_remote=18886 dedup_ratio=0.329865 router_tokens_per_expert=[252, 249, 249, 240, 272, 255, 247, 247, 238, 254, 277, 251, 251, 260, 254, 253, 247, 264, 269, 275, 235, 272, 275, 245, 231, 227, 297, 283, 248, 256, 270, 235, 256, 276, 248, 289, 278, 264, 262, 236, 259, 251, 246, 243, 253, 252, 265, 260, 248, 249, 265, 252, 260, 246, 245, 263, 250, 259, 275, 256, 253, 241, 240, 262, 262, 257, 273, 243, 239, 261, 267, 240, 240, 240, 250, 238, 250, 251, 240, 271, 269, 264, 259, 255, 253, 253, 264, 275, 242, 281, 241, 279, 254, 237, 251, 262, 244, 249, 275, 221, 248, 285, 256, 266, 263, 237, 273, 265, 242, 264, 284, 267, 273, 231, 239, 264, 257, 258, 246, 266, 278, 249, 254, 243, 268, 256, 264, 242] payload_rows_per_dst=[5402, 5530, 5584, 5410, 5434, 5546, 5526, 5486] dup_rows_per_dst=[2696, 2728, 2692, 2718, 2610, 2732, 2752, 2690] combine_rows_per_src=[5402, 5530, 5584, 5410, 5434, 5546, 5526, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38348 dup_rows=21650 dup_rows_remote=18918 dedup_ratio=0.330353 router_tokens_per_expert=[214, 240, 267, 244, 262, 252, 277, 256, 302, 261, 239, 252, 243, 244, 261, 285, 273, 244, 252, 252, 266, 262, 242, 243, 261, 235, 233, 282, 286, 277, 250, 279, 255, 265, 272, 270, 253, 245, 233, 233, 287, 268, 219, 277, 234, 241, 263, 260, 245, 230, 274, 289, 252, 272, 268, 268, 243, 259, 243, 246, 275, 242, 244, 260, 270, 284, 287, 247, 264, 242, 238, 238, 298, 260, 243, 264, 241, 259, 225, 275, 277, 274, 233, 267, 258, 234, 236, 252, 261, 247, 259, 259, 241, 231, 271, 242, 251, 274, 235, 290, 263, 261, 243, 275, 262, 266, 240, 262, 238, 228, 236, 252, 225, 244, 284, 265, 256, 257, 237, 268, 247, 278, 260, 258, 261, 243, 243, 268] payload_rows_per_dst=[5454, 5534, 5512, 5460, 5538, 5472, 5434, 5482] dup_rows_per_dst=[2744, 2740, 2638, 2760, 2732, 2612, 2718, 2706] combine_rows_per_src=[5454, 5534, 5512, 5460, 5538, 5472, 5434, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38160 dup_rows=21866 dup_rows_remote=19116 dedup_ratio=0.333649 router_tokens_per_expert=[264, 273, 256, 242, 251, 244, 240, 269, 293, 255, 259, 259, 223, 247, 242, 265, 240, 241, 269, 264, 268, 264, 242, 254, 252, 248, 259, 250, 240, 274, 268, 254, 226, 245, 255, 251, 253, 251, 279, 266, 253, 276, 277, 257, 266, 238, 245, 292, 293, 267, 268, 243, 271, 239, 258, 253, 273, 243, 261, 254, 271, 238, 222, 272, 281, 281, 257, 268, 279, 238, 288, 234, 270, 277, 305, 286, 233, 277, 266, 230, 223, 254, 261, 239, 244, 243, 223, 238, 253, 229, 270, 262, 230, 256, 255, 258, 273, 256, 272, 251, 255, 237, 244, 272, 240, 240, 264, 256, 258, 241, 269, 257, 225, 247, 268, 264, 245, 252, 229, 212, 264, 257, 263, 238, 272, 267, 265, 282] payload_rows_per_dst=[5494, 5548, 5510, 5424, 5586, 5268, 5406, 5434] dup_rows_per_dst=[2670, 2626, 2750, 2828, 2954, 2608, 2764, 2666] combine_rows_per_src=[5494, 5548, 5510, 5424, 5586, 5268, 5406, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38312 dup_rows=21718 dup_rows_remote=18992 dedup_ratio=0.331390 router_tokens_per_expert=[248, 272, 237, 230, 274, 251, 236, 229, 252, 240, 274, 262, 253, 234, 275, 240, 233, 237, 254, 234, 255, 268, 272, 230, 267, 237, 279, 259, 250, 218, 255, 254, 288, 254, 259, 245, 248, 265, 249, 231, 249, 270, 268, 262, 264, 261, 248, 271, 240, 234, 247, 282, 246, 254, 260, 233, 279, 252, 264, 266, 246, 291, 265, 257, 280, 258, 254, 254, 264, 271, 237, 249, 276, 260, 251, 233, 248, 255, 265, 254, 266, 280, 270, 265, 260, 258, 252, 254, 243, 296, 261, 243, 246, 207, 245, 252, 253, 280, 273, 252, 227, 240, 262, 254, 269, 270, 262, 239, 277, 261, 254, 295, 275, 233, 236, 289, 264, 264, 258, 243, 246, 239, 261, 259, 272, 282, 262, 253] payload_rows_per_dst=[5394, 5400, 5530, 5506, 5464, 5510, 5500, 5514] dup_rows_per_dst=[2620, 2604, 2734, 2726, 2754, 2686, 2836, 2758] combine_rows_per_src=[5394, 5400, 5530, 5506, 5464, 5510, 5500, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229280 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073312 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917344 weight_movement_bytes=18874368 total_nvlink_bytes=332791712 weight_to_token_ratio=0.060125 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229696 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196608 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163520 weight_movement_bytes=18874368 total_nvlink_bytes=333037888 weight_to_token_ratio=0.060078 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229032 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157400744 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572456 weight_movement_bytes=18874368 total_nvlink_bytes=333446824 weight_to_token_ratio=0.060000 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229064 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157302472 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314375880 weight_movement_bytes=37748736 total_nvlink_bytes=352124616 weight_to_token_ratio=0.120075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229104 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156532464 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312835824 weight_movement_bytes=37748736 total_nvlink_bytes=350584560 weight_to_token_ratio=0.120666 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38164 dup_rows=21812 dup_rows_remote=18954 dedup_ratio=0.332825 router_tokens_per_expert=[260, 232, 257, 267, 261, 251, 261, 230, 263, 231, 251, 245, 262, 267, 290, 256, 247, 253, 237, 274, 256, 260, 248, 272, 235, 259, 239, 231, 259, 245, 271, 272, 278, 244, 225, 242, 275, 242, 234, 245, 283, 257, 274, 258, 255, 228, 248, 228, 269, 232, 245, 262, 236, 261, 283, 249, 262, 250, 237, 246, 285, 266, 252, 241, 276, 233, 261, 271, 263, 260, 240, 246, 258, 260, 266, 257, 257, 244, 245, 273, 249, 261, 265, 238, 261, 277, 278, 252, 262, 244, 233, 232, 262, 239, 252, 276, 275, 260, 266, 279, 259, 224, 280, 257, 267, 276, 255, 263, 262, 260, 260, 266, 255, 254, 271, 260, 279, 243, 273, 245, 263, 235, 240, 271, 253, 245, 269, 278] payload_rows_per_dst=[5480, 5488, 5420, 5386, 5500, 5398, 5560, 5492] dup_rows_per_dst=[2688, 2628, 2612, 2766, 2720, 2764, 2858, 2776] combine_rows_per_src=[5480, 5488, 5420, 5386, 5500, 5398, 5560, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229216 combine_payload_bytes=156925952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157155168 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314081120 weight_movement_bytes=37748736 total_nvlink_bytes=351829856 weight_to_token_ratio=0.120188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38258 dup_rows=21754 dup_rows_remote=19074 dedup_ratio=0.331940 router_tokens_per_expert=[256, 282, 259, 243, 252, 261, 240, 230, 264, 247, 228, 232, 275, 276, 266, 287, 289, 263, 233, 244, 225, 255, 265, 246, 272, 277, 257, 242, 261, 279, 222, 252, 255, 246, 253, 259, 252, 282, 244, 246, 267, 245, 261, 282, 268, 265, 235, 262, 264, 243, 259, 239, 249, 258, 246, 258, 268, 266, 275, 255, 268, 241, 281, 260, 259, 248, 236, 249, 261, 246, 255, 260, 269, 248, 238, 266, 258, 228, 256, 253, 263, 283, 273, 257, 261, 278, 245, 271, 255, 253, 239, 253, 273, 240, 219, 262, 260, 252, 253, 227, 236, 247, 271, 263, 240, 281, 232, 249, 269, 254, 260, 249, 283, 263, 255, 237, 252, 277, 250, 258, 289, 242, 247, 244, 239, 254, 279, 269] payload_rows_per_dst=[5524, 5544, 5486, 5492, 5602, 5460, 5242, 5432] dup_rows_per_dst=[2680, 2788, 2868, 2768, 2796, 2782, 2396, 2676] combine_rows_per_src=[5524, 5544, 5486, 5492, 5602, 5460, 5242, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38266 dup_rows=21858 dup_rows_remote=19118 dedup_ratio=0.333527 router_tokens_per_expert=[254, 256, 268, 265, 260, 274, 282, 281, 244, 254, 267, 233, 244, 251, 264, 248, 233, 266, 247, 235, 233, 259, 266, 238, 261, 251, 271, 278, 232, 267, 269, 270, 268, 252, 250, 258, 255, 249, 266, 243, 248, 248, 265, 267, 251, 249, 267, 269, 238, 248, 249, 255, 270, 249, 270, 247, 244, 249, 253, 257, 237, 292, 274, 274, 227, 293, 250, 246, 266, 235, 278, 250, 252, 227, 238, 248, 270, 289, 243, 254, 251, 253, 250, 239, 275, 254, 238, 217, 285, 251, 253, 260, 276, 273, 255, 239, 249, 257, 251, 291, 257, 236, 255, 257, 261, 249, 249, 260, 254, 269, 251, 249, 260, 265, 271, 237, 265, 242, 243, 254, 246, 247, 265, 264, 268, 269, 284, 226] payload_rows_per_dst=[5548, 5412, 5474, 5424, 5440, 5484, 5438, 5458] dup_rows_per_dst=[2742, 2740, 2736, 2788, 2692, 2654, 2752, 2754] combine_rows_per_src=[5548, 5412, 5474, 5424, 5440, 5484, 5438, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38174 dup_rows=21738 dup_rows_remote=18806 dedup_ratio=0.331696 router_tokens_per_expert=[235, 232, 236, 258, 269, 253, 229, 277, 271, 262, 277, 286, 245, 256, 220, 252, 276, 250, 238, 266, 269, 274, 246, 253, 275, 279, 267, 263, 245, 249, 283, 277, 265, 254, 251, 246, 253, 255, 253, 236, 245, 204, 268, 248, 241, 243, 258, 263, 248, 258, 257, 229, 270, 252, 238, 255, 284, 247, 252, 233, 271, 238, 247, 249, 237, 262, 228, 245, 234, 260, 262, 244, 294, 274, 280, 241, 242, 242, 236, 273, 258, 265, 274, 242, 272, 241, 274, 254, 246, 269, 256, 259, 233, 238, 244, 275, 268, 275, 258, 241, 264, 271, 243, 242, 263, 257, 253, 235, 232, 269, 247, 271, 256, 261, 259, 252, 282, 310, 244, 234, 267, 277, 263, 271, 252, 282, 257, 279] payload_rows_per_dst=[5370, 5522, 5328, 5384, 5496, 5582, 5492, 5624] dup_rows_per_dst=[2682, 2712, 2542, 2480, 2900, 2804, 2686, 2932] combine_rows_per_src=[5370, 5522, 5328, 5384, 5496, 5582, 5492, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228472 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156548216 ingress_bytes=212942848 link_bytes=212942848 token_bytes=312867960 weight_movement_bytes=56623104 total_nvlink_bytes=369491064 weight_to_token_ratio=0.180981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040313595e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38166 dup_rows=21868 dup_rows_remote=19080 dedup_ratio=0.333679 router_tokens_per_expert=[287, 251, 257, 252, 261, 231, 231, 248, 264, 254, 252, 244, 258, 272, 254, 240, 216, 264, 264, 281, 266, 248, 260, 250, 265, 267, 239, 248, 248, 223, 281, 256, 274, 244, 277, 264, 232, 253, 267, 271, 255, 283, 271, 240, 248, 236, 267, 258, 257, 239, 232, 253, 245, 257, 268, 245, 271, 267, 246, 262, 236, 262, 274, 252, 267, 234, 267, 242, 273, 243, 265, 251, 258, 242, 245, 259, 251, 262, 278, 245, 280, 243, 255, 255, 290, 274, 263, 249, 266, 248, 275, 272, 242, 233, 256, 244, 235, 253, 295, 246, 233, 267, 292, 235, 232, 219, 263, 267, 244, 234, 263, 251, 271, 267, 244, 268, 229, 277, 244, 265, 264, 286, 263, 260, 265, 261, 256, 254] payload_rows_per_dst=[5438, 5452, 5546, 5424, 5412, 5502, 5378, 5516] dup_rows_per_dst=[2674, 2700, 2734, 2708, 2752, 2788, 2680, 2832] combine_rows_per_src=[5438, 5452, 5546, 5424, 5412, 5502, 5378, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38278 dup_rows=21766 dup_rows_remote=19016 dedup_ratio=0.332123 router_tokens_per_expert=[297, 238, 260, 262, 263, 262, 251, 275, 236, 244, 264, 279, 247, 261, 282, 260, 234, 268, 249, 269, 260, 254, 243, 237, 264, 262, 230, 237, 247, 221, 280, 256, 268, 234, 256, 266, 279, 269, 236, 246, 269, 260, 245, 279, 266, 259, 244, 245, 246, 273, 231, 251, 232, 276, 237, 247, 265, 259, 255, 263, 269, 270, 281, 261, 299, 220, 269, 255, 279, 235, 265, 249, 248, 265, 267, 232, 233, 270, 234, 227, 261, 274, 259, 274, 247, 245, 259, 274, 273, 242, 274, 252, 281, 245, 260, 261, 280, 240, 255, 261, 229, 268, 261, 243, 254, 223, 240, 250, 272, 272, 264, 265, 250, 255, 241, 260, 242, 235, 246, 232, 283, 246, 245, 246, 262, 269, 269, 253] payload_rows_per_dst=[5524, 5402, 5492, 5482, 5516, 5516, 5446, 5392] dup_rows_per_dst=[2838, 2620, 2750, 2750, 2578, 2846, 2708, 2676] combine_rows_per_src=[5524, 5402, 5492, 5482, 5516, 5516, 5446, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38150 dup_rows=21968 dup_rows_remote=19124 dedup_ratio=0.335205 router_tokens_per_expert=[239, 230, 249, 284, 276, 274, 245, 247, 276, 254, 242, 273, 236, 265, 211, 243, 271, 263, 269, 267, 254, 253, 253, 277, 251, 240, 263, 251, 254, 260, 258, 267, 258, 247, 232, 263, 242, 231, 243, 268, 231, 242, 241, 247, 276, 256, 238, 261, 266, 246, 268, 252, 257, 269, 272, 284, 259, 286, 251, 262, 239, 261, 253, 241, 272, 251, 250, 266, 252, 267, 285, 271, 239, 252, 257, 249, 262, 263, 240, 255, 256, 235, 242, 281, 282, 245, 240, 238, 251, 239, 250, 255, 256, 242, 236, 260, 282, 199, 272, 277, 257, 272, 268, 254, 273, 273, 222, 256, 270, 264, 254, 241, 251, 245, 268, 266, 257, 242, 257, 269, 238, 285, 276, 271, 238, 276, 234, 285] payload_rows_per_dst=[5446, 5542, 5344, 5492, 5418, 5334, 5454, 5538] dup_rows_per_dst=[2642, 2760, 2608, 2840, 2844, 2682, 2814, 2778] combine_rows_per_src=[5446, 5542, 5344, 5492, 5418, 5334, 5454, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229536 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156967072 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313704608 weight_movement_bytes=37748736 total_nvlink_bytes=351453344 weight_to_token_ratio=0.120332 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=227920 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156588624 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312949328 weight_movement_bytes=37748736 total_nvlink_bytes=350698064 weight_to_token_ratio=0.120623 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229328 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934096 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313638864 weight_movement_bytes=18874368 total_nvlink_bytes=332513232 weight_to_token_ratio=0.060179 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38258 dup_rows=21800 dup_rows_remote=19072 dedup_ratio=0.332642 router_tokens_per_expert=[247, 265, 294, 247, 256, 247, 234, 280, 270, 256, 235, 260, 247, 282, 260, 262, 274, 272, 260, 272, 250, 260, 268, 251, 261, 262, 257, 261, 267, 243, 240, 240, 253, 272, 283, 280, 233, 251, 238, 244, 259, 260, 259, 248, 243, 268, 264, 261, 248, 260, 245, 297, 251, 231, 275, 270, 245, 242, 237, 253, 273, 262, 259, 255, 212, 262, 267, 259, 259, 220, 232, 251, 286, 252, 271, 224, 245, 252, 258, 253, 262, 221, 245, 260, 244, 256, 248, 272, 230, 252, 240, 235, 247, 242, 261, 255, 259, 264, 287, 265, 250, 246, 273, 264, 239, 257, 265, 284, 263, 259, 286, 258, 259, 284, 258, 260, 213, 288, 267, 245, 291, 248, 235, 264, 241, 248, 249, 227] payload_rows_per_dst=[5506, 5500, 5416, 5478, 5388, 5416, 5580, 5452] dup_rows_per_dst=[2778, 2776, 2816, 2728, 2618, 2524, 2858, 2702] combine_rows_per_src=[5506, 5500, 5416, 5478, 5388, 5416, 5580, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=228984 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156556920 ingress_bytes=175202304 link_bytes=175202304 token_bytes=312884856 weight_movement_bytes=18874368 total_nvlink_bytes=331759224 weight_to_token_ratio=0.060324 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678702272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229176 combine_payload_bytes=156786688 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157015864 ingress_bytes=213409792 link_bytes=213409792 token_bytes=313802552 weight_movement_bytes=56623104 total_nvlink_bytes=370425656 weight_to_token_ratio=0.180442 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044787623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=229096 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156491496 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753896 weight_movement_bytes=18874368 total_nvlink_bytes=331628264 weight_to_token_ratio=0.060349 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229320 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156934088 ingress_bytes=194453504 link_bytes=194453504 token_bytes=313638856 weight_movement_bytes=37748736 total_nvlink_bytes=351387592 weight_to_token_ratio=0.120357 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863157798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38034 dup_rows=21958 dup_rows_remote=18938 dedup_ratio=0.335052 router_tokens_per_expert=[275, 259, 268, 256, 279, 246, 240, 253, 256, 275, 274, 256, 267, 235, 250, 285, 258, 249, 255, 262, 309, 238, 251, 237, 265, 248, 248, 261, 257, 274, 265, 251, 250, 251, 251, 255, 248, 261, 257, 261, 255, 267, 277, 250, 276, 268, 260, 251, 242, 251, 276, 268, 282, 266, 250, 256, 251, 258, 227, 267, 276, 268, 246, 268, 252, 247, 248, 263, 226, 248, 239, 253, 233, 259, 262, 274, 276, 269, 239, 249, 256, 251, 249, 273, 264, 264, 243, 241, 254, 272, 261, 248, 234, 247, 252, 285, 250, 251, 248, 258, 247, 251, 264, 253, 224, 245, 261, 241, 244, 272, 252, 249, 253, 226, 237, 243, 263, 276, 254, 266, 247, 272, 251, 238, 231, 257, 266, 255] payload_rows_per_dst=[5298, 5496, 5428, 5540, 5468, 5606, 5198, 5544] dup_rows_per_dst=[2578, 2760, 2672, 2764, 2820, 2982, 2362, 3020] combine_rows_per_src=[5298, 5496, 5428, 5540, 5468, 5606, 5198, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38332 dup_rows=21754 dup_rows_remote=19058 dedup_ratio=0.331940 router_tokens_per_expert=[256, 257, 260, 234, 235, 265, 257, 270, 253, 263, 265, 267, 253, 232, 251, 248, 275, 246, 240, 255, 237, 252, 263, 244, 266, 242, 244, 273, 251, 241, 272, 251, 249, 245, 275, 263, 256, 250, 268, 264, 235, 294, 297, 254, 268, 271, 243, 248, 222, 263, 275, 220, 244, 287, 256, 241, 265, 267, 270, 257, 291, 230, 268, 283, 257, 252, 277, 242, 269, 250, 261, 244, 270, 248, 268, 245, 258, 259, 237, 243, 243, 275, 253, 233, 267, 280, 265, 285, 243, 263, 259, 248, 262, 238, 250, 229, 244, 237, 252, 282, 262, 240, 269, 250, 253, 238, 276, 278, 247, 250, 245, 250, 240, 255, 252, 258, 281, 248, 270, 255, 276, 242, 269, 267, 243, 260, 233, 236] payload_rows_per_dst=[5492, 5356, 5514, 5508, 5478, 5500, 5450, 5484] dup_rows_per_dst=[2640, 2748, 2834, 2770, 2682, 2698, 2696, 2686] combine_rows_per_src=[5492, 5356, 5514, 5508, 5478, 5500, 5450, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38438 dup_rows=21710 dup_rows_remote=18926 dedup_ratio=0.331268 router_tokens_per_expert=[226, 254, 238, 276, 226, 266, 280, 259, 257, 237, 253, 230, 273, 245, 269, 242, 246, 265, 270, 255, 262, 253, 261, 276, 234, 235, 257, 235, 253, 252, 263, 269, 261, 263, 262, 237, 264, 277, 274, 254, 281, 238, 259, 252, 267, 244, 238, 254, 254, 288, 261, 237, 227, 259, 271, 239, 270, 263, 247, 258, 234, 269, 265, 242, 263, 264, 273, 239, 262, 247, 267, 213, 231, 282, 273, 261, 239, 264, 251, 249, 227, 254, 299, 250, 219, 243, 268, 233, 268, 253, 234, 259, 254, 281, 256, 235, 252, 263, 270, 254, 282, 242, 278, 249, 277, 271, 255, 241, 288, 275, 284, 216, 254, 242, 246, 248, 268, 283, 258, 261, 250, 265, 242, 283, 255, 263, 252, 264] payload_rows_per_dst=[5436, 5388, 5500, 5500, 5416, 5482, 5602, 5502] dup_rows_per_dst=[2626, 2784, 2750, 2668, 2740, 2584, 2792, 2766] combine_rows_per_src=[5436, 5388, 5500, 5500, 5416, 5482, 5602, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38238 dup_rows=21856 dup_rows_remote=19170 dedup_ratio=0.333496 router_tokens_per_expert=[264, 252, 243, 271, 256, 251, 302, 266, 281, 241, 275, 261, 265, 238, 256, 252, 261, 262, 250, 272, 225, 234, 249, 280, 278, 240, 252, 204, 272, 261, 258, 268, 238, 239, 255, 261, 266, 226, 263, 246, 248, 285, 271, 275, 266, 228, 246, 258, 239, 259, 269, 254, 266, 287, 238, 245, 275, 270, 266, 244, 282, 241, 244, 261, 239, 249, 213, 270, 252, 247, 227, 235, 262, 280, 265, 255, 253, 299, 285, 259, 252, 249, 253, 280, 233, 244, 261, 263, 267, 240, 247, 244, 231, 277, 262, 261, 237, 238, 273, 237, 255, 254, 251, 236, 261, 271, 257, 264, 258, 287, 215, 283, 281, 230, 243, 235, 304, 263, 244, 257, 269, 261, 238, 258, 251, 238, 258, 256] payload_rows_per_dst=[5510, 5450, 5502, 5466, 5456, 5442, 5410, 5444] dup_rows_per_dst=[2838, 2682, 2640, 2814, 2724, 2686, 2744, 2728] combine_rows_per_src=[5510, 5450, 5502, 5466, 5456, 5442, 5410, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=227888 combine_payload_bytes=155787264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156015152 ingress_bytes=174661632 link_bytes=174661632 token_bytes=311802416 weight_movement_bytes=18874368 total_nvlink_bytes=330676784 weight_to_token_ratio=0.060533 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673521819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38276 dup_rows=21802 dup_rows_remote=19042 dedup_ratio=0.332672 router_tokens_per_expert=[299, 249, 241, 304, 229, 235, 263, 231, 247, 281, 290, 234, 268, 266, 246, 265, 258, 248, 268, 219, 285, 265, 243, 263, 248, 267, 228, 257, 265, 271, 243, 243, 273, 255, 268, 249, 279, 230, 259, 244, 247, 261, 255, 278, 265, 282, 250, 214, 261, 248, 247, 237, 270, 243, 239, 259, 253, 261, 254, 274, 231, 274, 277, 240, 255, 271, 237, 243, 242, 236, 272, 242, 247, 236, 247, 262, 283, 271, 252, 263, 249, 252, 252, 265, 249, 270, 243, 232, 276, 268, 275, 247, 257, 272, 273, 234, 270, 265, 268, 264, 274, 237, 268, 243, 257, 292, 247, 270, 277, 245, 257, 237, 239, 270, 249, 257, 265, 269, 227, 248, 253, 256, 234, 257, 261, 252, 231, 260] payload_rows_per_dst=[5496, 5438, 5458, 5384, 5470, 5478, 5566, 5444] dup_rows_per_dst=[2800, 2704, 2760, 2752, 2648, 2750, 2776, 2612] combine_rows_per_src=[5496, 5438, 5458, 5384, 5470, 5478, 5566, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229456 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671504 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113552 weight_movement_bytes=18874368 total_nvlink_bytes=333987920 weight_to_token_ratio=0.059897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38220 dup_rows=21720 dup_rows_remote=18938 dedup_ratio=0.331421 router_tokens_per_expert=[242, 245, 236, 273, 290, 273, 234, 256, 255, 259, 262, 256, 237, 250, 247, 234, 251, 247, 264, 232, 266, 267, 259, 234, 249, 226, 269, 277, 251, 259, 261, 230, 228, 261, 282, 240, 273, 234, 256, 248, 272, 245, 255, 272, 273, 265, 283, 226, 287, 266, 226, 238, 251, 257, 270, 253, 275, 258, 246, 270, 280, 249, 272, 259, 229, 274, 241, 296, 266, 274, 256, 247, 265, 230, 260, 287, 264, 238, 275, 263, 265, 268, 228, 244, 237, 253, 245, 276, 246, 259, 252, 254, 259, 232, 263, 236, 251, 264, 242, 240, 276, 249, 247, 242, 278, 267, 268, 251, 273, 252, 268, 268, 253, 259, 248, 245, 248, 291, 246, 268, 280, 250, 259, 253, 247, 240, 236, 266] payload_rows_per_dst=[5596, 5502, 5530, 5524, 5450, 5432, 5342, 5440] dup_rows_per_dst=[2782, 2786, 2770, 2790, 2676, 2602, 2576, 2738] combine_rows_per_src=[5596, 5502, 5530, 5524, 5450, 5432, 5342, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38436 dup_rows=21596 dup_rows_remote=18880 dedup_ratio=0.329529 router_tokens_per_expert=[241, 261, 251, 273, 258, 260, 245, 256, 251, 235, 266, 269, 274, 260, 264, 244, 248, 240, 273, 211, 238, 256, 248, 227, 271, 262, 242, 259, 267, 249, 223, 272, 215, 271, 267, 272, 266, 255, 274, 245, 253, 256, 254, 271, 248, 273, 277, 273, 245, 233, 247, 261, 256, 267, 250, 258, 262, 256, 279, 250, 238, 271, 265, 259, 241, 266, 281, 239, 253, 257, 240, 273, 233, 226, 238, 260, 274, 279, 289, 261, 229, 247, 258, 246, 282, 266, 271, 252, 239, 271, 258, 253, 245, 248, 231, 254, 243, 231, 280, 263, 282, 289, 246, 297, 263, 266, 246, 262, 269, 259, 244, 245, 272, 242, 270, 259, 245, 241, 254, 269, 254, 229, 252, 256, 261, 245, 281, 232] payload_rows_per_dst=[5480, 5502, 5580, 5430, 5504, 5394, 5560, 5490] dup_rows_per_dst=[2736, 2470, 2760, 2764, 2716, 2706, 2810, 2634] combine_rows_per_src=[5480, 5502, 5580, 5430, 5504, 5394, 5560, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229560 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157237432 ingress_bytes=232505344 link_bytes=232505344 token_bytes=314245304 weight_movement_bytes=75497472 total_nvlink_bytes=389742776 weight_to_token_ratio=0.240250 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227751806e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38172 dup_rows=21860 dup_rows_remote=19024 dedup_ratio=0.333557 router_tokens_per_expert=[282, 233, 238, 265, 297, 247, 245, 273, 240, 267, 272, 254, 273, 251, 236, 209, 280, 254, 243, 242, 269, 237, 280, 277, 211, 280, 241, 277, 246, 262, 245, 249, 224, 239, 271, 251, 256, 268, 268, 255, 262, 274, 231, 276, 249, 237, 259, 223, 273, 272, 254, 253, 281, 266, 189, 278, 269, 262, 285, 248, 270, 258, 264, 248, 229, 255, 253, 267, 252, 257, 252, 247, 245, 258, 289, 260, 251, 266, 248, 259, 250, 281, 252, 247, 258, 249, 248, 252, 256, 260, 251, 276, 255, 246, 245, 256, 258, 255, 247, 263, 269, 272, 256, 222, 272, 261, 256, 271, 280, 265, 244, 227, 263, 253, 264, 246, 273, 250, 261, 255, 282, 232, 237, 257, 263, 239, 230, 287] payload_rows_per_dst=[5422, 5524, 5424, 5504, 5426, 5458, 5474, 5444] dup_rows_per_dst=[2742, 2662, 2662, 2836, 2750, 2706, 2762, 2740] combine_rows_per_src=[5422, 5524, 5424, 5504, 5426, 5458, 5474, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229632 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156852480 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313475328 weight_movement_bytes=18874368 total_nvlink_bytes=332349696 weight_to_token_ratio=0.060210 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229272 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007768 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786264 weight_movement_bytes=37748736 total_nvlink_bytes=351535000 weight_to_token_ratio=0.120301 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228632 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156777752 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313326872 weight_movement_bytes=18874368 total_nvlink_bytes=332201240 weight_to_token_ratio=0.060239 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157433856 dispatch_meta_bytes=229264 combine_payload_bytes=157433856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157663120 ingress_bytes=176308224 link_bytes=176308224 token_bytes=315096976 weight_movement_bytes=18874368 total_nvlink_bytes=333971344 weight_to_token_ratio=0.059900 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689298653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228784 combine_payload_bytes=156352512 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156581296 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312933808 weight_movement_bytes=37748736 total_nvlink_bytes=350682544 weight_to_token_ratio=0.120629 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38350 dup_rows=21628 dup_rows_remote=18902 dedup_ratio=0.330017 router_tokens_per_expert=[265, 273, 242, 231, 258, 236, 260, 262, 250, 288, 249, 275, 236, 257, 287, 256, 244, 274, 256, 242, 253, 271, 259, 270, 245, 266, 261, 224, 260, 268, 243, 270, 239, 300, 269, 250, 245, 285, 264, 268, 252, 240, 228, 239, 256, 251, 246, 244, 265, 219, 243, 251, 267, 253, 246, 250, 259, 237, 254, 237, 285, 273, 269, 273, 249, 271, 255, 258, 236, 288, 238, 258, 302, 265, 252, 293, 257, 258, 226, 235, 262, 265, 255, 284, 261, 246, 229, 255, 251, 255, 276, 280, 248, 244, 271, 256, 245, 264, 254, 230, 266, 265, 221, 249, 246, 257, 264, 265, 241, 248, 228, 269, 256, 261, 248, 252, 262, 250, 258, 252, 261, 262, 259, 268, 256, 265, 235, 244] payload_rows_per_dst=[5386, 5508, 5432, 5488, 5556, 5546, 5434, 5558] dup_rows_per_dst=[2602, 2704, 2508, 2678, 2806, 2912, 2692, 2726] combine_rows_per_src=[5386, 5508, 5432, 5488, 5556, 5546, 5434, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38270 dup_rows=21848 dup_rows_remote=19166 dedup_ratio=0.333374 router_tokens_per_expert=[276, 252, 254, 276, 265, 264, 296, 282, 254, 265, 253, 265, 252, 237, 262, 246, 254, 237, 257, 236, 292, 242, 270, 256, 223, 241, 272, 230, 258, 260, 250, 272, 241, 273, 243, 279, 270, 252, 250, 275, 258, 250, 251, 254, 249, 257, 284, 250, 259, 249, 262, 241, 259, 255, 245, 262, 245, 232, 272, 256, 265, 267, 262, 246, 251, 286, 230, 259, 264, 268, 224, 256, 288, 257, 244, 248, 235, 244, 272, 247, 229, 256, 260, 252, 237, 266, 271, 254, 260, 253, 248, 254, 261, 272, 249, 234, 255, 249, 261, 291, 242, 279, 244, 262, 269, 244, 249, 250, 257, 233, 228, 271, 254, 256, 287, 228, 258, 253, 245, 275, 253, 265, 257, 244, 240, 281, 242, 255] payload_rows_per_dst=[5528, 5418, 5532, 5458, 5446, 5420, 5498, 5388] dup_rows_per_dst=[2870, 2682, 2740, 2696, 2700, 2692, 2670, 2798] combine_rows_per_src=[5528, 5418, 5532, 5458, 5446, 5420, 5498, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38186 dup_rows=21880 dup_rows_remote=18994 dedup_ratio=0.333862 router_tokens_per_expert=[259, 253, 229, 258, 243, 273, 276, 275, 262, 249, 281, 269, 233, 262, 240, 236, 258, 242, 237, 255, 246, 253, 259, 258, 288, 221, 236, 254, 263, 235, 274, 261, 250, 275, 252, 235, 282, 242, 243, 267, 266, 260, 239, 254, 267, 254, 245, 263, 253, 225, 229, 248, 247, 248, 259, 247, 246, 270, 213, 251, 278, 279, 263, 272, 274, 249, 240, 281, 260, 242, 247, 264, 238, 239, 259, 283, 242, 248, 247, 261, 248, 269, 240, 278, 244, 261, 291, 278, 267, 258, 257, 267, 277, 261, 254, 228, 273, 270, 244, 260, 278, 254, 278, 253, 254, 244, 238, 262, 256, 285, 262, 235, 247, 280, 268, 273, 261, 231, 254, 275, 280, 248, 238, 219, 281, 256, 252, 247] payload_rows_per_dst=[5422, 5418, 5502, 5400, 5416, 5470, 5504, 5524] dup_rows_per_dst=[2774, 2662, 2686, 2656, 2732, 2886, 2788, 2696] combine_rows_per_src=[5422, 5418, 5502, 5400, 5416, 5470, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38362 dup_rows=21844 dup_rows_remote=19242 dedup_ratio=0.333313 router_tokens_per_expert=[236, 243, 246, 263, 274, 262, 260, 254, 269, 261, 271, 263, 277, 261, 238, 253, 272, 255, 265, 239, 268, 256, 263, 263, 254, 256, 254, 244, 279, 269, 244, 245, 274, 270, 266, 262, 284, 263, 244, 256, 281, 272, 258, 252, 284, 246, 250, 254, 240, 230, 251, 254, 260, 276, 267, 257, 230, 253, 274, 269, 264, 276, 253, 261, 244, 257, 241, 246, 259, 253, 241, 261, 234, 265, 273, 298, 286, 253, 240, 269, 290, 245, 253, 263, 263, 264, 264, 239, 242, 239, 248, 250, 231, 248, 274, 257, 242, 256, 275, 241, 217, 251, 256, 255, 234, 252, 250, 244, 264, 268, 230, 231, 250, 278, 258, 257, 258, 235, 249, 221, 256, 248, 242, 242, 271, 269, 233, 257] payload_rows_per_dst=[5514, 5544, 5498, 5532, 5432, 5358, 5330, 5484] dup_rows_per_dst=[2748, 2708, 2934, 2698, 2808, 2782, 2602, 2564] combine_rows_per_src=[5514, 5544, 5498, 5532, 5432, 5358, 5330, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229008 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157310608 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392208 weight_movement_bytes=18874368 total_nvlink_bytes=333266576 weight_to_token_ratio=0.060034 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229744 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983664 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737584 weight_movement_bytes=18874368 total_nvlink_bytes=332611952 weight_to_token_ratio=0.060160 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38282 dup_rows=21776 dup_rows_remote=19040 dedup_ratio=0.332275 router_tokens_per_expert=[264, 235, 267, 256, 232, 237, 276, 261, 245, 252, 255, 225, 254, 252, 266, 261, 246, 257, 251, 258, 285, 254, 242, 275, 277, 272, 263, 261, 243, 274, 267, 242, 242, 246, 256, 260, 266, 252, 277, 261, 260, 231, 280, 214, 261, 272, 272, 257, 254, 287, 230, 251, 259, 274, 261, 271, 242, 262, 234, 290, 281, 270, 247, 282, 243, 253, 283, 257, 261, 250, 270, 260, 263, 263, 255, 248, 245, 243, 275, 250, 286, 246, 221, 226, 248, 260, 229, 263, 260, 253, 259, 250, 258, 245, 268, 259, 258, 249, 251, 221, 255, 260, 251, 283, 271, 259, 268, 264, 264, 263, 236, 257, 252, 271, 248, 246, 265, 238, 262, 254, 215, 235, 258, 279, 265, 239, 213, 261] payload_rows_per_dst=[5316, 5488, 5478, 5556, 5532, 5446, 5506, 5438] dup_rows_per_dst=[2760, 2846, 2736, 2834, 2706, 2616, 2714, 2564] combine_rows_per_src=[5316, 5488, 5478, 5556, 5532, 5446, 5506, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38152 dup_rows=21948 dup_rows_remote=19298 dedup_ratio=0.334900 router_tokens_per_expert=[242, 256, 266, 256, 236, 222, 254, 253, 254, 258, 271, 228, 260, 257, 251, 277, 227, 266, 255, 274, 245, 264, 252, 241, 246, 263, 247, 250, 234, 268, 286, 248, 251, 249, 256, 245, 248, 262, 239, 246, 284, 276, 281, 258, 268, 259, 264, 282, 267, 261, 269, 269, 272, 257, 240, 285, 249, 245, 260, 252, 256, 264, 268, 250, 239, 264, 250, 269, 245, 268, 245, 267, 300, 254, 227, 274, 245, 269, 221, 284, 240, 282, 249, 276, 255, 237, 270, 272, 254, 258, 259, 272, 275, 254, 218, 261, 243, 233, 260, 258, 247, 251, 259, 265, 250, 231, 241, 252, 247, 250, 243, 253, 263, 238, 249, 260, 257, 224, 248, 284, 242, 258, 255, 276, 286, 248, 253, 252] payload_rows_per_dst=[5436, 5478, 5478, 5440, 5490, 5456, 5454, 5356] dup_rows_per_dst=[2650, 2668, 2840, 2888, 2752, 2808, 2864, 2478] combine_rows_per_src=[5436, 5478, 5478, 5440, 5490, 5456, 5454, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38128 dup_rows=21882 dup_rows_remote=19202 dedup_ratio=0.333893 router_tokens_per_expert=[241, 240, 238, 265, 252, 254, 252, 258, 236, 246, 281, 269, 284, 266, 263, 228, 250, 262, 230, 255, 262, 280, 251, 255, 248, 239, 243, 243, 276, 256, 235, 255, 263, 266, 247, 254, 269, 254, 261, 252, 278, 244, 265, 260, 247, 270, 256, 269, 262, 285, 242, 234, 275, 249, 231, 275, 241, 249, 254, 257, 250, 283, 268, 248, 276, 234, 266, 279, 252, 235, 271, 266, 272, 258, 247, 241, 251, 247, 256, 282, 257, 269, 243, 258, 256, 256, 246, 248, 259, 287, 284, 234, 231, 250, 269, 269, 269, 251, 245, 243, 240, 253, 267, 230, 244, 265, 228, 251, 256, 271, 255, 266, 268, 233, 260, 243, 284, 256, 252, 268, 243, 247, 230, 263, 259, 273, 275, 260] payload_rows_per_dst=[5450, 5338, 5492, 5526, 5506, 5490, 5390, 5462] dup_rows_per_dst=[2696, 2742, 2818, 2680, 2760, 2742, 2678, 2766] combine_rows_per_src=[5450, 5338, 5492, 5526, 5506, 5490, 5390, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=230416 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157361168 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314491920 weight_movement_bytes=18874368 total_nvlink_bytes=333366288 weight_to_token_ratio=0.060015 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38178 dup_rows=21846 dup_rows_remote=18998 dedup_ratio=0.333344 router_tokens_per_expert=[278, 276, 278, 254, 267, 267, 229, 261, 256, 266, 261, 253, 262, 248, 237, 260, 247, 251, 256, 240, 262, 279, 262, 252, 210, 272, 252, 251, 273, 270, 264, 266, 245, 269, 236, 252, 287, 240, 237, 248, 261, 241, 235, 249, 274, 256, 252, 254, 232, 255, 275, 275, 269, 281, 210, 258, 223, 248, 256, 260, 226, 269, 267, 251, 286, 262, 224, 265, 261, 267, 251, 265, 272, 280, 252, 245, 279, 271, 251, 249, 247, 246, 240, 230, 262, 256, 235, 268, 239, 250, 262, 277, 250, 257, 266, 262, 262, 240, 234, 245, 263, 238, 257, 261, 268, 249, 252, 253, 261, 256, 256, 264, 265, 285, 226, 278, 276, 252, 245, 237, 271, 274, 249, 260, 249, 258, 249, 257] payload_rows_per_dst=[5448, 5468, 5384, 5458, 5512, 5438, 5486, 5496] dup_rows_per_dst=[2858, 2746, 2688, 2652, 2848, 2656, 2632, 2766] combine_rows_per_src=[5448, 5468, 5384, 5458, 5512, 5438, 5486, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228720 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156638576 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048432 weight_movement_bytes=18874368 total_nvlink_bytes=331922800 weight_to_token_ratio=0.060292 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229288 combine_payload_bytes=156803072 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157032360 ingress_bytes=232300544 link_bytes=232300544 token_bytes=313835432 weight_movement_bytes=75497472 total_nvlink_bytes=389332904 weight_to_token_ratio=0.240564 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225789514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229800 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156500392 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312770984 weight_movement_bytes=37748736 total_nvlink_bytes=350519720 weight_to_token_ratio=0.120691 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229320 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156401608 ingress_bytes=193921024 link_bytes=193921024 token_bytes=312573896 weight_movement_bytes=37748736 total_nvlink_bytes=350322632 weight_to_token_ratio=0.120767 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858055837e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228704 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156605792 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312982880 weight_movement_bytes=18874368 total_nvlink_bytes=331857248 weight_to_token_ratio=0.060305 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=38148 dup_rows=21978 dup_rows_remote=19296 dedup_ratio=0.335358 router_tokens_per_expert=[252, 257, 268, 252, 245, 253, 258, 236, 245, 260, 277, 225, 252, 261, 239, 285, 240, 246, 248, 271, 261, 253, 243, 260, 257, 249, 271, 261, 280, 254, 256, 263, 238, 248, 254, 251, 241, 291, 273, 294, 242, 241, 267, 250, 237, 258, 280, 255, 272, 278, 252, 252, 270, 235, 243, 242, 241, 251, 243, 266, 248, 282, 236, 230, 271, 283, 292, 255, 263, 256, 260, 273, 234, 252, 256, 230, 238, 258, 254, 263, 271, 265, 268, 277, 271, 269, 248, 228, 235, 262, 271, 256, 283, 245, 270, 264, 251, 252, 261, 250, 257, 277, 239, 272, 275, 247, 231, 271, 250, 241, 230, 258, 248, 266, 281, 269, 233, 240, 242, 266, 252, 265, 261, 227, 260, 251, 238, 247] payload_rows_per_dst=[5388, 5412, 5442, 5340, 5512, 5548, 5506, 5410] dup_rows_per_dst=[2742, 2734, 2798, 2782, 2764, 2858, 2618, 2682] combine_rows_per_src=[5388, 5412, 5442, 5340, 5512, 5548, 5506, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38396 dup_rows=21628 dup_rows_remote=18944 dedup_ratio=0.330017 router_tokens_per_expert=[257, 242, 261, 283, 247, 233, 271, 250, 268, 271, 245, 267, 239, 283, 242, 294, 247, 270, 243, 294, 227, 242, 261, 260, 245, 267, 246, 262, 265, 265, 257, 247, 259, 282, 241, 249, 231, 271, 242, 247, 250, 258, 257, 274, 253, 265, 216, 239, 234, 281, 272, 265, 271, 287, 243, 252, 215, 241, 251, 255, 262, 220, 258, 271, 248, 265, 249, 280, 247, 262, 245, 243, 278, 234, 244, 241, 288, 266, 243, 221, 247, 275, 271, 251, 245, 240, 228, 228, 262, 248, 269, 287, 241, 250, 243, 250, 269, 257, 271, 240, 282, 263, 270, 229, 250, 225, 279, 261, 270, 249, 274, 236, 263, 271, 251, 289, 276, 237, 296, 265, 244, 261, 254, 269, 269, 245, 237, 264] payload_rows_per_dst=[5558, 5512, 5494, 5448, 5458, 5334, 5540, 5564] dup_rows_per_dst=[2748, 2684, 2574, 2708, 2650, 2736, 2710, 2818] combine_rows_per_src=[5558, 5512, 5494, 5448, 5458, 5334, 5540, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229776 combine_payload_bytes=156254208 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156483984 ingress_bytes=212877312 link_bytes=212877312 token_bytes=312738192 weight_movement_bytes=56623104 total_nvlink_bytes=369361296 weight_to_token_ratio=0.181056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039685661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38154 dup_rows=21878 dup_rows_remote=19124 dedup_ratio=0.333832 router_tokens_per_expert=[269, 251, 229, 254, 265, 243, 266, 263, 267, 263, 255, 251, 239, 256, 273, 257, 267, 260, 267, 270, 270, 249, 234, 275, 270, 254, 244, 249, 244, 260, 248, 269, 265, 215, 270, 266, 277, 259, 266, 254, 257, 241, 265, 248, 260, 249, 267, 232, 266, 239, 258, 255, 267, 258, 260, 249, 266, 232, 263, 254, 281, 258, 265, 250, 257, 266, 265, 264, 255, 268, 245, 239, 270, 266, 246, 268, 245, 248, 283, 255, 263, 280, 266, 253, 267, 238, 223, 243, 293, 261, 249, 218, 205, 246, 255, 266, 257, 254, 264, 256, 276, 245, 270, 267, 266, 251, 261, 250, 263, 249, 250, 276, 278, 243, 255, 257, 242, 249, 233, 242, 257, 235, 259, 262, 267, 239, 266, 220] payload_rows_per_dst=[5382, 5478, 5504, 5504, 5508, 5394, 5504, 5384] dup_rows_per_dst=[2820, 2782, 2678, 2738, 2772, 2658, 2754, 2676] combine_rows_per_src=[5382, 5478, 5504, 5504, 5508, 5394, 5504, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38234 dup_rows=21858 dup_rows_remote=19186 dedup_ratio=0.333527 router_tokens_per_expert=[247, 262, 243, 261, 264, 233, 245, 279, 243, 282, 278, 274, 263, 259, 262, 252, 270, 252, 249, 252, 250, 247, 262, 239, 255, 263, 257, 255, 283, 270, 279, 250, 249, 282, 261, 265, 260, 266, 244, 248, 255, 272, 259, 264, 241, 221, 238, 250, 253, 276, 285, 245, 263, 256, 238, 255, 280, 240, 251, 277, 243, 248, 238, 234, 225, 256, 258, 260, 272, 266, 254, 271, 227, 225, 250, 269, 269, 251, 262, 256, 264, 236, 258, 232, 260, 278, 254, 238, 254, 264, 274, 267, 238, 244, 233, 264, 272, 259, 261, 242, 249, 273, 261, 250, 253, 273, 246, 256, 231, 275, 252, 268, 234, 271, 246, 254, 257, 246, 258, 236, 241, 261, 274, 263, 263, 242, 260, 275] payload_rows_per_dst=[5502, 5512, 5446, 5366, 5422, 5444, 5484, 5502] dup_rows_per_dst=[2792, 2754, 2704, 2798, 2720, 2672, 2758, 2660] combine_rows_per_src=[5502, 5512, 5446, 5366, 5422, 5444, 5484, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229360 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157499376 ingress_bytes=213893120 link_bytes=213893120 token_bytes=314769392 weight_movement_bytes=56623104 total_nvlink_bytes=371392496 weight_to_token_ratio=0.179888 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049418634e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38272 dup_rows=21712 dup_rows_remote=19062 dedup_ratio=0.331299 router_tokens_per_expert=[256, 257, 236, 264, 255, 243, 248, 236, 235, 258, 249, 258, 282, 264, 241, 239, 258, 250, 232, 266, 262, 240, 273, 247, 278, 238, 250, 237, 290, 274, 294, 273, 275, 249, 246, 246, 284, 287, 264, 244, 267, 237, 256, 263, 223, 247, 261, 252, 287, 262, 267, 264, 244, 251, 255, 270, 272, 276, 259, 272, 251, 244, 232, 242, 236, 238, 288, 240, 247, 256, 254, 294, 255, 245, 253, 252, 251, 259, 243, 259, 287, 251, 234, 269, 255, 272, 244, 255, 261, 257, 277, 234, 242, 245, 255, 250, 272, 233, 246, 262, 250, 283, 265, 253, 243, 259, 267, 262, 253, 280, 233, 257, 246, 259, 257, 238, 253, 230, 242, 261, 251, 267, 271, 256, 270, 281, 253, 225] payload_rows_per_dst=[5370, 5566, 5552, 5526, 5450, 5400, 5532, 5428] dup_rows_per_dst=[2672, 2758, 2650, 2770, 2690, 2776, 2704, 2692] combine_rows_per_src=[5370, 5566, 5552, 5526, 5450, 5400, 5532, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38232 dup_rows=21822 dup_rows_remote=19064 dedup_ratio=0.332977 router_tokens_per_expert=[291, 265, 256, 248, 263, 238, 296, 259, 248, 266, 266, 273, 243, 249, 228, 264, 256, 268, 229, 245, 256, 246, 272, 249, 280, 246, 245, 252, 247, 251, 256, 262, 261, 248, 236, 263, 272, 263, 258, 268, 237, 248, 264, 261, 257, 241, 242, 253, 275, 263, 266, 248, 267, 254, 265, 276, 247, 230, 238, 235, 273, 259, 267, 257, 227, 256, 237, 257, 259, 231, 239, 237, 254, 238, 260, 277, 305, 247, 283, 254, 252, 243, 260, 264, 224, 282, 258, 231, 227, 278, 261, 264, 239, 259, 241, 262, 268, 268, 268, 248, 278, 244, 281, 253, 237, 248, 256, 235, 262, 222, 273, 234, 255, 245, 250, 278, 281, 277, 257, 269, 263, 279, 243, 268, 232, 262, 263, 260] payload_rows_per_dst=[5466, 5524, 5426, 5482, 5460, 5370, 5442, 5544] dup_rows_per_dst=[2840, 2596, 2718, 2758, 2662, 2720, 2708, 2820] combine_rows_per_src=[5466, 5524, 5426, 5482, 5460, 5370, 5442, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38152 dup_rows=21778 dup_rows_remote=19012 dedup_ratio=0.332306 router_tokens_per_expert=[241, 257, 288, 254, 280, 275, 257, 276, 255, 271, 252, 277, 234, 261, 251, 256, 281, 245, 235, 229, 268, 253, 249, 265, 273, 278, 242, 273, 234, 251, 256, 258, 264, 251, 263, 251, 235, 258, 255, 261, 262, 276, 269, 241, 263, 254, 252, 254, 234, 248, 243, 257, 239, 242, 265, 258, 263, 270, 263, 247, 279, 257, 247, 243, 265, 239, 250, 267, 242, 254, 267, 274, 241, 241, 253, 253, 253, 222, 248, 290, 252, 290, 262, 280, 272, 241, 270, 254, 236, 242, 231, 260, 262, 245, 236, 272, 264, 249, 261, 280, 259, 233, 260, 242, 242, 268, 255, 256, 263, 241, 269, 269, 256, 252, 232, 249, 242, 281, 250, 279, 262, 237, 246, 268, 238, 241, 259, 262] payload_rows_per_dst=[5606, 5560, 5492, 5334, 5422, 5458, 5426, 5460] dup_rows_per_dst=[2766, 2856, 2726, 2538, 2696, 2752, 2796, 2648] combine_rows_per_src=[5606, 5560, 5492, 5334, 5422, 5458, 5426, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229112 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156507896 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312786680 weight_movement_bytes=37748736 total_nvlink_bytes=350535416 weight_to_token_ratio=0.120685 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229680 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156836144 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313442608 weight_movement_bytes=18874368 total_nvlink_bytes=332316976 weight_to_token_ratio=0.060216 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38318 dup_rows=21752 dup_rows_remote=19068 dedup_ratio=0.331909 router_tokens_per_expert=[242, 260, 246, 243, 236, 243, 258, 275, 263, 233, 260, 282, 241, 236, 243, 251, 291, 244, 250, 272, 242, 266, 233, 259, 269, 262, 252, 241, 270, 274, 241, 241, 288, 246, 247, 234, 251, 266, 252, 252, 265, 270, 248, 260, 262, 245, 253, 254, 265, 287, 250, 247, 252, 241, 263, 242, 264, 235, 259, 255, 271, 261, 277, 267, 242, 229, 239, 250, 256, 243, 276, 266, 245, 269, 244, 273, 283, 265, 251, 244, 235, 295, 273, 281, 278, 236, 244, 242, 243, 225, 245, 248, 261, 265, 236, 260, 290, 253, 249, 267, 236, 250, 253, 243, 255, 242, 279, 256, 262, 262, 270, 272, 265, 265, 234, 234, 250, 260, 278, 254, 251, 267, 262, 254, 279, 270, 263, 253] payload_rows_per_dst=[5488, 5438, 5442, 5478, 5466, 5442, 5518, 5512] dup_rows_per_dst=[2536, 2776, 2744, 2794, 2684, 2692, 2760, 2766] combine_rows_per_src=[5488, 5438, 5442, 5478, 5466, 5442, 5518, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229336 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991448 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753560 weight_movement_bytes=18874368 total_nvlink_bytes=332627928 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229184 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156827456 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425728 weight_movement_bytes=37748736 total_nvlink_bytes=351174464 weight_to_token_ratio=0.120439 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228656 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156499248 ingress_bytes=231768064 link_bytes=231768064 token_bytes=312769840 weight_movement_bytes=75497472 total_nvlink_bytes=388267312 weight_to_token_ratio=0.241383 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220687552e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38306 dup_rows=21802 dup_rows_remote=19056 dedup_ratio=0.332672 router_tokens_per_expert=[231, 275, 257, 253, 238, 234, 273, 244, 246, 253, 244, 233, 254, 231, 238, 237, 269, 284, 247, 282, 269, 259, 259, 230, 263, 263, 256, 252, 265, 263, 290, 244, 261, 266, 231, 251, 253, 231, 277, 260, 230, 218, 267, 255, 250, 266, 252, 264, 266, 269, 256, 263, 237, 268, 271, 302, 257, 245, 266, 221, 278, 276, 250, 249, 262, 258, 240, 260, 297, 255, 262, 231, 264, 262, 254, 275, 242, 243, 235, 267, 261, 235, 275, 253, 271, 253, 260, 269, 242, 253, 292, 236, 271, 276, 256, 296, 248, 267, 256, 258, 258, 254, 285, 270, 249, 272, 256, 262, 256, 238, 255, 243, 219, 232, 268, 243, 249, 245, 217, 281, 284, 223, 270, 238, 258, 250, 240, 276] payload_rows_per_dst=[5336, 5490, 5366, 5530, 5546, 5488, 5550, 5428] dup_rows_per_dst=[2546, 2708, 2698, 2816, 2860, 2722, 2706, 2746] combine_rows_per_src=[5336, 5490, 5366, 5530, 5546, 5488, 5550, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38250 dup_rows=21802 dup_rows_remote=18986 dedup_ratio=0.332672 router_tokens_per_expert=[258, 242, 256, 253, 249, 262, 259, 243, 265, 243, 279, 225, 246, 255, 252, 263, 247, 245, 262, 251, 260, 249, 250, 246, 264, 256, 270, 279, 269, 274, 271, 257, 263, 264, 259, 255, 245, 262, 256, 265, 258, 246, 255, 261, 254, 234, 298, 251, 251, 265, 245, 247, 277, 286, 227, 266, 283, 251, 244, 251, 243, 267, 254, 259, 265, 232, 274, 272, 265, 263, 262, 235, 236, 257, 284, 261, 246, 263, 259, 297, 250, 243, 260, 245, 260, 260, 297, 260, 219, 238, 253, 242, 251, 274, 256, 238, 283, 253, 260, 257, 248, 276, 269, 254, 263, 255, 247, 259, 245, 244, 263, 246, 250, 247, 236, 252, 240, 253, 260, 231, 254, 253, 256, 240, 250, 260, 253, 252] payload_rows_per_dst=[5402, 5484, 5502, 5450, 5530, 5442, 5506, 5418] dup_rows_per_dst=[2698, 2816, 2750, 2782, 2812, 2650, 2738, 2556] combine_rows_per_src=[5402, 5484, 5502, 5450, 5530, 5442, 5506, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229544 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157180072 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130600 weight_movement_bytes=18874368 total_nvlink_bytes=333004968 weight_to_token_ratio=0.060084 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229448 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130824 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032200 weight_movement_bytes=18874368 total_nvlink_bytes=332906568 weight_to_token_ratio=0.060103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38138 dup_rows=21840 dup_rows_remote=19090 dedup_ratio=0.333252 router_tokens_per_expert=[243, 238, 250, 253, 244, 251, 265, 253, 267, 273, 235, 254, 251, 244, 245, 290, 255, 288, 283, 221, 233, 243, 246, 264, 276, 242, 271, 252, 259, 276, 248, 258, 250, 263, 240, 252, 249, 275, 272, 249, 252, 275, 263, 259, 271, 243, 230, 264, 262, 266, 254, 267, 238, 251, 233, 283, 253, 254, 278, 270, 244, 248, 256, 248, 254, 255, 269, 275, 241, 252, 246, 254, 215, 253, 262, 264, 237, 259, 265, 240, 254, 226, 239, 243, 260, 263, 244, 249, 265, 258, 248, 261, 251, 259, 268, 260, 269, 264, 265, 274, 256, 261, 234, 262, 265, 251, 272, 256, 252, 250, 258, 265, 266, 228, 252, 234, 267, 250, 271, 253, 255, 275, 255, 269, 245, 252, 275, 295] payload_rows_per_dst=[5452, 5514, 5466, 5428, 5420, 5400, 5558, 5458] dup_rows_per_dst=[2660, 2716, 2748, 2782, 2662, 2696, 2750, 2826] combine_rows_per_src=[5452, 5514, 5466, 5428, 5420, 5400, 5558, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38260 dup_rows=21812 dup_rows_remote=19104 dedup_ratio=0.332825 router_tokens_per_expert=[253, 247, 249, 268, 259, 235, 247, 239, 237, 260, 257, 263, 255, 249, 252, 256, 264, 255, 264, 260, 254, 243, 253, 233, 246, 269, 248, 232, 259, 258, 288, 252, 240, 236, 253, 286, 263, 261, 272, 255, 258, 260, 265, 239, 268, 239, 239, 241, 285, 276, 248, 236, 272, 242, 235, 248, 275, 266, 252, 253, 272, 262, 271, 270, 243, 255, 248, 272, 253, 250, 257, 269, 285, 257, 238, 254, 276, 240, 257, 248, 243, 263, 244, 286, 237, 255, 254, 265, 276, 257, 247, 264, 249, 255, 229, 262, 286, 216, 255, 261, 249, 253, 285, 259, 276, 257, 241, 250, 269, 258, 258, 264, 275, 251, 246, 244, 268, 260, 264, 275, 262, 249, 247, 249, 261, 267, 257, 226] payload_rows_per_dst=[5380, 5372, 5460, 5552, 5526, 5464, 5500, 5470] dup_rows_per_dst=[2672, 2784, 2690, 2774, 2678, 2708, 2774, 2732] combine_rows_per_src=[5380, 5372, 5460, 5552, 5526, 5464, 5500, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228944 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156900944 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313572944 weight_movement_bytes=37748736 total_nvlink_bytes=351321680 weight_to_token_ratio=0.120383 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38272 dup_rows=21920 dup_rows_remote=19138 dedup_ratio=0.334473 router_tokens_per_expert=[231, 243, 252, 268, 260, 264, 263, 252, 269, 263, 267, 265, 234, 236, 262, 273, 240, 244, 238, 279, 271, 254, 250, 243, 258, 259, 256, 258, 247, 285, 248, 236, 264, 254, 227, 275, 255, 234, 246, 263, 271, 231, 268, 239, 249, 249, 236, 302, 275, 243, 281, 260, 285, 264, 246, 276, 260, 249, 248, 240, 237, 258, 242, 248, 270, 251, 231, 252, 238, 228, 239, 273, 249, 250, 256, 261, 251, 251, 272, 258, 270, 270, 270, 256, 272, 270, 260, 295, 254, 267, 264, 266, 251, 260, 277, 263, 268, 243, 262, 259, 251, 261, 249, 245, 262, 260, 254, 236, 246, 237, 248, 252, 251, 258, 248, 247, 244, 255, 250, 253, 278, 266, 255, 249, 250, 270, 249, 274] payload_rows_per_dst=[5438, 5468, 5344, 5522, 5482, 5538, 5390, 5434] dup_rows_per_dst=[2766, 2664, 2782, 2702, 2578, 2992, 2676, 2760] combine_rows_per_src=[5438, 5468, 5344, 5522, 5482, 5538, 5390, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38164 dup_rows=21852 dup_rows_remote=19032 dedup_ratio=0.333435 router_tokens_per_expert=[229, 266, 264, 259, 260, 247, 234, 252, 263, 255, 251, 235, 262, 281, 255, 285, 246, 256, 266, 258, 258, 260, 252, 260, 265, 248, 240, 278, 267, 244, 247, 241, 260, 239, 233, 232, 264, 274, 234, 270, 287, 253, 288, 257, 265, 269, 230, 275, 250, 269, 246, 265, 290, 247, 276, 249, 238, 278, 249, 261, 265, 246, 282, 259, 273, 252, 234, 253, 253, 254, 246, 254, 238, 280, 238, 260, 268, 239, 271, 260, 242, 242, 235, 258, 262, 258, 238, 257, 259, 245, 230, 243, 238, 259, 259, 262, 257, 252, 240, 259, 254, 260, 265, 234, 255, 270, 263, 276, 234, 250, 253, 257, 250, 243, 250, 260, 239, 277, 257, 262, 246, 242, 274, 279, 273, 258, 266, 269] payload_rows_per_dst=[5444, 5446, 5406, 5520, 5456, 5360, 5488, 5564] dup_rows_per_dst=[2752, 2726, 2854, 2820, 2690, 2614, 2670, 2726] combine_rows_per_src=[5444, 5446, 5406, 5520, 5456, 5360, 5488, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228912 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156442160 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312655408 weight_movement_bytes=18874368 total_nvlink_bytes=331529776 weight_to_token_ratio=0.060368 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229456 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156942416 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655376 weight_movement_bytes=56623104 total_nvlink_bytes=370278480 weight_to_token_ratio=0.180526 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38248 dup_rows=21778 dup_rows_remote=18974 dedup_ratio=0.332306 router_tokens_per_expert=[257, 258, 287, 259, 256, 266, 252, 256, 266, 253, 288, 264, 232, 262, 248, 248, 265, 230, 254, 258, 262, 264, 249, 244, 287, 261, 251, 268, 267, 255, 251, 227, 262, 242, 283, 247, 252, 243, 267, 260, 274, 241, 265, 274, 252, 268, 278, 260, 273, 255, 263, 256, 274, 274, 252, 226, 265, 266, 261, 241, 249, 287, 267, 263, 264, 270, 255, 249, 252, 213, 224, 238, 270, 238, 232, 219, 273, 248, 250, 272, 261, 268, 267, 239, 263, 243, 260, 216, 244, 263, 246, 269, 272, 257, 244, 251, 286, 284, 260, 249, 251, 266, 238, 268, 263, 263, 256, 238, 253, 242, 252, 249, 219, 253, 264, 279, 247, 262, 213, 276, 253, 257, 263, 232, 242, 272, 239, 264] payload_rows_per_dst=[5510, 5530, 5562, 5460, 5340, 5490, 5478, 5388] dup_rows_per_dst=[2804, 2756, 2764, 2784, 2594, 2636, 2758, 2682] combine_rows_per_src=[5510, 5530, 5562, 5460, 5340, 5490, 5478, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38128 dup_rows=21924 dup_rows_remote=19278 dedup_ratio=0.334534 router_tokens_per_expert=[238, 222, 264, 224, 246, 275, 273, 281, 254, 263, 263, 260, 258, 262, 257, 255, 239, 264, 224, 276, 259, 274, 260, 258, 260, 268, 276, 234, 234, 276, 239, 232, 246, 258, 228, 236, 265, 248, 297, 252, 279, 247, 275, 240, 256, 262, 262, 242, 263, 268, 265, 263, 254, 259, 271, 250, 239, 276, 275, 262, 259, 250, 228, 240, 246, 269, 255, 264, 264, 268, 244, 282, 266, 275, 240, 229, 232, 261, 240, 230, 246, 259, 268, 250, 253, 243, 249, 251, 272, 250, 274, 239, 262, 247, 241, 271, 267, 259, 238, 239, 278, 272, 263, 277, 247, 247, 279, 267, 234, 257, 233, 248, 253, 283, 246, 236, 262, 230, 290, 277, 259, 282, 232, 272, 252, 252, 255, 259] payload_rows_per_dst=[5464, 5392, 5450, 5488, 5484, 5366, 5464, 5504] dup_rows_per_dst=[2726, 2754, 2736, 2756, 2646, 2784, 2746, 2776] combine_rows_per_src=[5464, 5392, 5450, 5488, 5484, 5366, 5464, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38146 dup_rows=21844 dup_rows_remote=19108 dedup_ratio=0.333313 router_tokens_per_expert=[255, 278, 269, 252, 256, 270, 244, 260, 257, 254, 245, 243, 245, 234, 228, 225, 230, 279, 254, 279, 234, 250, 249, 238, 246, 250, 256, 258, 244, 262, 240, 248, 265, 290, 277, 257, 261, 262, 247, 241, 270, 262, 241, 262, 236, 260, 245, 252, 250, 245, 261, 268, 260, 236, 236, 284, 282, 247, 287, 290, 258, 249, 267, 235, 242, 268, 297, 264, 272, 262, 249, 248, 252, 259, 256, 263, 216, 269, 269, 257, 273, 273, 266, 261, 241, 238, 253, 246, 242, 235, 255, 258, 256, 253, 271, 268, 275, 260, 254, 257, 251, 252, 259, 249, 239, 252, 263, 289, 230, 267, 253, 233, 262, 290, 281, 254, 278, 267, 264, 257, 240, 241, 247, 252, 267, 227, 243, 268] payload_rows_per_dst=[5212, 5376, 5318, 5458, 5492, 5674, 5616, 5546] dup_rows_per_dst=[2432, 2658, 2514, 2862, 2794, 2912, 2936, 2736] combine_rows_per_src=[5212, 5376, 5318, 5458, 5492, 5674, 5616, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229640 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991752 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753864 weight_movement_bytes=18874368 total_nvlink_bytes=332628232 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228784 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156548528 ingress_bytes=212942848 link_bytes=212942848 token_bytes=312868272 weight_movement_bytes=56623104 total_nvlink_bytes=369491376 weight_to_token_ratio=0.180981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040313595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38330 dup_rows=21686 dup_rows_remote=18970 dedup_ratio=0.330902 router_tokens_per_expert=[253, 264, 270, 260, 253, 252, 253, 253, 250, 286, 246, 293, 233, 270, 254, 239, 246, 257, 261, 249, 282, 261, 251, 267, 264, 265, 235, 247, 244, 246, 265, 278, 273, 241, 269, 244, 227, 264, 248, 267, 252, 262, 248, 264, 268, 277, 278, 251, 240, 240, 273, 246, 274, 237, 237, 255, 267, 245, 262, 249, 262, 253, 239, 251, 238, 240, 257, 288, 272, 241, 235, 255, 253, 278, 272, 259, 237, 284, 249, 254, 235, 267, 251, 248, 242, 266, 241, 240, 257, 239, 240, 256, 238, 287, 243, 278, 247, 302, 268, 238, 253, 265, 229, 258, 258, 234, 256, 249, 257, 273, 259, 248, 283, 231, 248, 278, 221, 257, 253, 254, 252, 279, 265, 261, 257, 267, 263, 255] payload_rows_per_dst=[5554, 5520, 5476, 5476, 5506, 5404, 5462, 5452] dup_rows_per_dst=[2704, 2716, 2790, 2584, 2718, 2652, 2726, 2796] combine_rows_per_src=[5554, 5520, 5476, 5476, 5506, 5404, 5462, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229624 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156401912 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312574200 weight_movement_bytes=18874368 total_nvlink_bytes=331448568 weight_to_token_ratio=0.060384 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228888 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156892696 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313556504 weight_movement_bytes=56623104 total_nvlink_bytes=370179608 weight_to_token_ratio=0.180583 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=229016 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156475032 ingress_bytes=193994752 link_bytes=193994752 token_bytes=312721048 weight_movement_bytes=37748736 total_nvlink_bytes=350469784 weight_to_token_ratio=0.120711 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858762262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44020 payload_rows_remote=38516 dup_rows=21516 dup_rows_remote=18808 dedup_ratio=0.328308 router_tokens_per_expert=[285, 245, 249, 264, 263, 249, 246, 281, 258, 235, 276, 269, 262, 261, 238, 280, 253, 261, 254, 236, 262, 267, 279, 247, 260, 231, 225, 248, 255, 239, 258, 252, 244, 262, 271, 287, 263, 261, 262, 250, 248, 266, 244, 275, 255, 246, 235, 274, 264, 280, 235, 254, 268, 269, 231, 224, 267, 265, 256, 267, 265, 247, 248, 239, 265, 272, 248, 238, 238, 232, 226, 232, 268, 274, 290, 279, 244, 238, 262, 227, 265, 273, 244, 260, 266, 255, 254, 284, 230, 258, 241, 301, 229, 265, 235, 246, 252, 234, 230, 260, 254, 255, 243, 264, 275, 249, 265, 301, 257, 256, 243, 258, 269, 245, 228, 247, 274, 276, 246, 251, 264, 265, 268, 259, 268, 261, 247, 255] payload_rows_per_dst=[5580, 5486, 5494, 5442, 5500, 5504, 5536, 5478] dup_rows_per_dst=[2742, 2568, 2792, 2716, 2566, 2708, 2656, 2768] combine_rows_per_src=[5580, 5486, 5494, 5442, 5500, 5504, 5536, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38330 dup_rows=21784 dup_rows_remote=19066 dedup_ratio=0.332397 router_tokens_per_expert=[275, 252, 240, 264, 254, 238, 267, 241, 249, 258, 243, 257, 244, 257, 278, 255, 242, 252, 248, 213, 265, 249, 267, 250, 249, 254, 238, 278, 290, 261, 280, 268, 263, 256, 255, 262, 282, 254, 265, 266, 250, 242, 272, 254, 286, 255, 261, 250, 273, 242, 240, 264, 243, 263, 256, 277, 258, 261, 266, 264, 261, 269, 254, 239, 270, 256, 210, 271, 248, 259, 263, 244, 260, 251, 295, 264, 258, 249, 248, 253, 245, 250, 255, 257, 283, 247, 249, 252, 262, 243, 244, 272, 244, 273, 281, 234, 251, 245, 249, 249, 244, 264, 279, 222, 251, 265, 233, 275, 252, 261, 249, 281, 264, 245, 240, 236, 282, 253, 261, 243, 225, 246, 250, 240, 259, 264, 256, 265] payload_rows_per_dst=[5402, 5428, 5570, 5526, 5476, 5460, 5422, 5468] dup_rows_per_dst=[2742, 2780, 2776, 2734, 2722, 2722, 2718, 2590] combine_rows_per_src=[5402, 5428, 5570, 5526, 5476, 5460, 5422, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229200 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228880 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228560 weight_movement_bytes=18874368 total_nvlink_bytes=333102928 weight_to_token_ratio=0.060066 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157761536 dispatch_meta_bytes=229296 combine_payload_bytes=157761536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157990832 ingress_bytes=176635904 link_bytes=176635904 token_bytes=315752368 weight_movement_bytes=18874368 total_nvlink_bytes=334626736 weight_to_token_ratio=0.059776 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692438322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38088 dup_rows=21938 dup_rows_remote=19272 dedup_ratio=0.334747 router_tokens_per_expert=[274, 252, 233, 251, 279, 276, 230, 285, 274, 249, 227, 231, 263, 251, 269, 253, 264, 252, 249, 253, 252, 265, 266, 247, 261, 245, 263, 267, 254, 266, 281, 259, 265, 268, 263, 237, 225, 258, 264, 281, 243, 279, 272, 257, 258, 241, 243, 252, 246, 269, 243, 238, 260, 236, 257, 270, 240, 265, 270, 270, 276, 243, 261, 244, 268, 237, 249, 216, 266, 260, 251, 243, 259, 278, 289, 276, 277, 246, 265, 255, 240, 244, 251, 233, 264, 228, 243, 251, 259, 230, 227, 270, 250, 259, 269, 262, 246, 269, 233, 253, 241, 239, 266, 270, 270, 270, 247, 261, 231, 280, 264, 291, 226, 246, 252, 258, 240, 283, 245, 264, 250, 251, 275, 272, 241, 273, 254, 257] payload_rows_per_dst=[5434, 5454, 5500, 5510, 5432, 5300, 5488, 5480] dup_rows_per_dst=[2760, 2834, 2712, 2666, 2838, 2660, 2774, 2694] combine_rows_per_src=[5434, 5454, 5500, 5510, 5432, 5300, 5488, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38370 dup_rows=21702 dup_rows_remote=19000 dedup_ratio=0.331146 router_tokens_per_expert=[248, 262, 263, 254, 238, 262, 270, 273, 272, 277, 242, 249, 262, 270, 250, 270, 259, 271, 254, 238, 293, 273, 242, 256, 252, 258, 261, 237, 263, 253, 281, 245, 262, 263, 251, 262, 263, 250, 256, 244, 287, 237, 254, 240, 250, 230, 286, 248, 255, 245, 248, 275, 273, 268, 258, 258, 287, 276, 264, 250, 270, 252, 235, 237, 265, 286, 251, 237, 247, 248, 239, 268, 250, 232, 254, 235, 251, 256, 252, 237, 253, 251, 256, 237, 242, 259, 247, 279, 234, 280, 230, 255, 275, 275, 229, 254, 253, 239, 268, 248, 253, 243, 254, 240, 264, 261, 235, 215, 283, 263, 226, 212, 260, 243, 284, 266, 259, 252, 304, 267, 248, 254, 267, 275, 274, 257, 252, 253] payload_rows_per_dst=[5560, 5550, 5464, 5550, 5446, 5412, 5310, 5542] dup_rows_per_dst=[2764, 2722, 2702, 2752, 2570, 2700, 2604, 2888] combine_rows_per_src=[5560, 5550, 5464, 5550, 5446, 5412, 5310, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229584 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157229264 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228944 weight_movement_bytes=37748736 total_nvlink_bytes=351977680 weight_to_token_ratio=0.120131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38302 dup_rows=21622 dup_rows_remote=18852 dedup_ratio=0.329926 router_tokens_per_expert=[261, 254, 247, 237, 254, 300, 258, 253, 268, 260, 267, 274, 277, 268, 252, 224, 231, 245, 261, 240, 256, 261, 272, 271, 246, 267, 258, 230, 249, 255, 254, 235, 247, 264, 237, 246, 252, 278, 265, 266, 256, 245, 258, 273, 256, 267, 276, 235, 269, 265, 244, 273, 248, 263, 254, 264, 244, 254, 243, 255, 247, 271, 226, 235, 267, 297, 266, 282, 252, 268, 257, 258, 276, 279, 261, 226, 240, 256, 245, 261, 268, 260, 245, 244, 237, 240, 245, 241, 298, 275, 278, 239, 227, 258, 283, 261, 290, 272, 260, 248, 248, 219, 257, 254, 241, 270, 245, 258, 252, 235, 266, 248, 268, 245, 240, 261, 239, 260, 241, 244, 250, 264, 265, 250, 257, 247, 276, 247] payload_rows_per_dst=[5442, 5468, 5516, 5376, 5612, 5528, 5480, 5492] dup_rows_per_dst=[2866, 2594, 2726, 2734, 2770, 2670, 2646, 2616] combine_rows_per_src=[5442, 5468, 5516, 5376, 5612, 5528, 5480, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38180 dup_rows=21884 dup_rows_remote=18966 dedup_ratio=0.333923 router_tokens_per_expert=[269, 275, 257, 272, 243, 271, 245, 245, 246, 249, 260, 256, 261, 239, 273, 260, 271, 222, 261, 239, 252, 254, 287, 258, 278, 258, 255, 260, 245, 233, 271, 257, 274, 266, 238, 276, 247, 261, 257, 251, 257, 277, 232, 261, 273, 275, 244, 238, 237, 272, 247, 263, 240, 243, 234, 288, 250, 261, 229, 239, 282, 282, 254, 271, 250, 250, 230, 281, 258, 251, 261, 262, 257, 238, 245, 241, 253, 249, 228, 251, 273, 236, 258, 246, 249, 268, 267, 266, 242, 260, 237, 224, 244, 274, 283, 258, 258, 246, 269, 245, 259, 247, 258, 246, 261, 250, 268, 258, 244, 254, 252, 234, 259, 267, 268, 248, 253, 253, 248, 269, 269, 263, 266, 290, 256, 265, 248, 266] payload_rows_per_dst=[5358, 5658, 5420, 5480, 5368, 5456, 5440, 5472] dup_rows_per_dst=[2408, 3014, 2834, 2704, 2642, 2714, 2650, 2918] combine_rows_per_src=[5358, 5658, 5420, 5480, 5368, 5456, 5440, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38286 dup_rows=21770 dup_rows_remote=18960 dedup_ratio=0.332184 router_tokens_per_expert=[278, 275, 271, 235, 260, 272, 260, 232, 231, 259, 234, 254, 267, 235, 238, 235, 261, 254, 249, 280, 232, 233, 238, 269, 241, 261, 238, 263, 262, 245, 257, 247, 269, 272, 266, 219, 245, 267, 258, 269, 274, 268, 247, 248, 245, 264, 238, 248, 295, 273, 243, 246, 253, 253, 265, 255, 257, 262, 233, 240, 259, 267, 253, 268, 230, 239, 270, 262, 274, 260, 250, 250, 260, 243, 213, 273, 255, 269, 229, 252, 255, 240, 256, 269, 247, 254, 271, 249, 259, 265, 254, 266, 275, 293, 276, 252, 262, 252, 264, 292, 243, 256, 279, 268, 257, 249, 228, 255, 262, 258, 251, 272, 253, 250, 238, 238, 261, 253, 270, 261, 249, 254, 293, 274, 248, 256, 248, 279] payload_rows_per_dst=[5480, 5460, 5418, 5372, 5564, 5538, 5450, 5484] dup_rows_per_dst=[2810, 2602, 2782, 2654, 2836, 2824, 2666, 2596] combine_rows_per_src=[5480, 5460, 5418, 5372, 5564, 5538, 5450, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=229440 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156237888 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312246336 weight_movement_bytes=18874368 total_nvlink_bytes=331120704 weight_to_token_ratio=0.060447 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38312 dup_rows=21786 dup_rows_remote=18982 dedup_ratio=0.332428 router_tokens_per_expert=[255, 240, 229, 245, 278, 282, 278, 251, 265, 232, 275, 261, 239, 254, 253, 263, 260, 252, 277, 256, 269, 241, 256, 269, 235, 242, 221, 257, 246, 261, 276, 303, 250, 294, 252, 251, 240, 242, 245, 230, 229, 276, 271, 259, 236, 265, 272, 238, 234, 271, 250, 245, 274, 234, 264, 272, 239, 277, 264, 250, 255, 251, 244, 255, 255, 229, 269, 252, 262, 283, 253, 277, 274, 238, 268, 227, 246, 266, 255, 259, 249, 234, 249, 247, 262, 253, 282, 252, 261, 261, 234, 234, 263, 275, 251, 250, 250, 263, 268, 249, 253, 224, 269, 256, 260, 269, 289, 266, 261, 256, 246, 245, 248, 233, 245, 253, 255, 263, 259, 268, 259, 243, 253, 275, 286, 262, 280, 242] payload_rows_per_dst=[5454, 5438, 5424, 5418, 5500, 5480, 5574, 5462] dup_rows_per_dst=[2746, 2804, 2676, 2740, 2726, 2634, 2674, 2786] combine_rows_per_src=[5454, 5438, 5424, 5418, 5500, 5480, 5574, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229480 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157393000 ingress_bytes=232660992 link_bytes=232660992 token_bytes=314556520 weight_movement_bytes=75497472 total_nvlink_bytes=390053992 weight_to_token_ratio=0.240012 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229243149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=228616 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157113608 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313998600 weight_movement_bytes=37748736 total_nvlink_bytes=351747336 weight_to_token_ratio=0.120219 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228584 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156613864 ingress_bytes=213008384 link_bytes=213008384 token_bytes=312999144 weight_movement_bytes=56623104 total_nvlink_bytes=369622248 weight_to_token_ratio=0.180905 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040941529e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=228984 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048440 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313867896 weight_movement_bytes=18874368 total_nvlink_bytes=332742264 weight_to_token_ratio=0.060135 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38330 dup_rows=21768 dup_rows_remote=19032 dedup_ratio=0.332153 router_tokens_per_expert=[277, 278, 258, 263, 248, 269, 261, 278, 261, 265, 227, 251, 242, 269, 262, 252, 253, 266, 261, 226, 240, 264, 271, 225, 235, 218, 248, 283, 252, 265, 248, 251, 237, 239, 252, 264, 235, 232, 246, 257, 243, 256, 260, 253, 245, 272, 261, 260, 260, 252, 252, 226, 242, 266, 260, 294, 268, 281, 264, 240, 249, 264, 267, 249, 254, 265, 235, 240, 252, 259, 256, 254, 272, 273, 284, 253, 272, 270, 230, 242, 254, 241, 234, 278, 246, 250, 253, 286, 244, 263, 246, 267, 266, 258, 242, 259, 259, 284, 272, 237, 279, 279, 227, 250, 249, 251, 246, 277, 250, 284, 280, 245, 244, 250, 270, 262, 271, 285, 225, 255, 231, 259, 248, 270, 279, 242, 259, 238] payload_rows_per_dst=[5544, 5374, 5376, 5522, 5522, 5438, 5580, 5412] dup_rows_per_dst=[2778, 2638, 2648, 2746, 2700, 2736, 2758, 2764] combine_rows_per_src=[5544, 5374, 5376, 5522, 5522, 5438, 5580, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38286 dup_rows=21800 dup_rows_remote=19020 dedup_ratio=0.332642 router_tokens_per_expert=[231, 268, 243, 235, 249, 226, 252, 242, 267, 261, 274, 246, 263, 276, 261, 271, 250, 250, 275, 272, 245, 256, 249, 249, 278, 231, 271, 264, 255, 267, 239, 274, 277, 265, 256, 241, 246, 252, 245, 266, 255, 265, 262, 241, 236, 259, 289, 261, 261, 259, 265, 215, 267, 254, 260, 271, 253, 245, 247, 259, 258, 279, 247, 263, 240, 261, 259, 237, 255, 227, 235, 242, 245, 253, 255, 257, 252, 256, 241, 279, 256, 256, 271, 251, 264, 238, 267, 269, 266, 243, 241, 264, 278, 241, 283, 262, 239, 249, 248, 249, 239, 245, 286, 245, 259, 259, 260, 302, 234, 254, 277, 270, 250, 269, 255, 258, 250, 260, 275, 230, 255, 244, 271, 256, 270, 255, 251, 251] payload_rows_per_dst=[5488, 5492, 5458, 5480, 5408, 5500, 5450, 5460] dup_rows_per_dst=[2642, 2758, 2774, 2726, 2580, 2800, 2780, 2740] combine_rows_per_src=[5488, 5492, 5458, 5480, 5408, 5500, 5450, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229176 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157155128 ingress_bytes=175800320 link_bytes=175800320 token_bytes=314081080 weight_movement_bytes=18874368 total_nvlink_bytes=332955448 weight_to_token_ratio=0.060094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684432167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229448 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157229128 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228808 weight_movement_bytes=18874368 total_nvlink_bytes=333103176 weight_to_token_ratio=0.060066 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43504 payload_rows_remote=38060 dup_rows=22032 dup_rows_remote=19228 dedup_ratio=0.336182 router_tokens_per_expert=[250, 237, 228, 266, 268, 263, 254, 275, 251, 271, 248, 285, 272, 254, 247, 236, 258, 278, 254, 241, 246, 271, 249, 222, 251, 247, 269, 241, 249, 261, 254, 262, 266, 263, 257, 247, 253, 262, 254, 262, 260, 230, 262, 259, 272, 275, 237, 253, 273, 234, 279, 280, 219, 288, 269, 230, 233, 267, 265, 256, 250, 273, 263, 245, 244, 253, 244, 265, 223, 245, 264, 243, 234, 263, 281, 254, 244, 263, 271, 269, 260, 257, 248, 239, 266, 267, 250, 262, 271, 221, 270, 260, 271, 265, 281, 259, 278, 234, 286, 255, 254, 270, 248, 250, 238, 243, 256, 262, 264, 273, 242, 251, 249, 230, 254, 262, 271, 249, 263, 270, 234, 262, 253, 253, 256, 261, 258, 238] payload_rows_per_dst=[5424, 5382, 5450, 5444, 5392, 5500, 5440, 5472] dup_rows_per_dst=[2786, 2724, 2774, 2804, 2728, 2794, 2768, 2654] combine_rows_per_src=[5424, 5382, 5450, 5444, 5392, 5500, 5440, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229224 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157048680 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868136 weight_movement_bytes=37748736 total_nvlink_bytes=351616872 weight_to_token_ratio=0.120269 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38342 dup_rows=21660 dup_rows_remote=18872 dedup_ratio=0.330505 router_tokens_per_expert=[243, 243, 257, 234, 278, 275, 233, 280, 268, 246, 267, 245, 234, 263, 248, 293, 255, 240, 248, 247, 258, 268, 282, 251, 243, 238, 234, 242, 229, 247, 243, 257, 262, 277, 247, 267, 258, 258, 245, 254, 263, 237, 276, 267, 263, 261, 252, 274, 273, 286, 239, 252, 270, 260, 224, 267, 257, 249, 260, 276, 254, 243, 251, 256, 261, 259, 260, 277, 261, 262, 223, 280, 246, 239, 263, 284, 265, 239, 264, 245, 256, 260, 260, 240, 225, 284, 244, 250, 282, 252, 266, 261, 241, 265, 276, 228, 271, 271, 259, 263, 255, 270, 269, 258, 259, 257, 262, 235, 243, 282, 214, 271, 257, 252, 282, 245, 230, 249, 244, 253, 247, 243, 268, 266, 247, 248, 236, 277] payload_rows_per_dst=[5510, 5378, 5534, 5494, 5514, 5514, 5484, 5448] dup_rows_per_dst=[2704, 2586, 2788, 2740, 2742, 2666, 2794, 2640] combine_rows_per_src=[5510, 5378, 5534, 5494, 5514, 5514, 5484, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38432 dup_rows=21626 dup_rows_remote=19054 dedup_ratio=0.329987 router_tokens_per_expert=[271, 246, 253, 260, 249, 234, 250, 257, 254, 266, 267, 242, 236, 274, 277, 253, 263, 220, 291, 269, 229, 242, 239, 260, 260, 245, 289, 252, 258, 249, 244, 279, 248, 262, 255, 267, 282, 246, 270, 269, 249, 269, 242, 243, 253, 259, 269, 266, 276, 262, 232, 248, 286, 247, 249, 230, 256, 250, 253, 254, 261, 271, 247, 254, 258, 248, 256, 255, 239, 252, 256, 249, 217, 251, 258, 264, 273, 236, 250, 263, 226, 253, 265, 250, 254, 255, 258, 244, 237, 232, 275, 274, 244, 255, 262, 245, 233, 291, 255, 248, 241, 277, 225, 287, 260, 245, 276, 277, 252, 249, 230, 262, 263, 263, 266, 268, 235, 264, 257, 281, 263, 250, 252, 267, 274, 270, 258, 272] payload_rows_per_dst=[5530, 5508, 5486, 5502, 5478, 5386, 5500, 5520] dup_rows_per_dst=[2648, 2670, 2812, 2650, 2572, 2672, 2716, 2886] combine_rows_per_src=[5530, 5508, 5486, 5502, 5478, 5386, 5500, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229152 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156122912 ingress_bytes=174768128 link_bytes=174768128 token_bytes=312016672 weight_movement_bytes=18874368 total_nvlink_bytes=330891040 weight_to_token_ratio=0.060492 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674542211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38346 dup_rows=21728 dup_rows_remote=18936 dedup_ratio=0.331543 router_tokens_per_expert=[248, 271, 257, 283, 251, 228, 276, 271, 261, 253, 269, 265, 280, 269, 270, 269, 284, 264, 252, 255, 252, 251, 265, 257, 259, 224, 270, 246, 253, 245, 233, 255, 230, 251, 283, 277, 253, 216, 289, 233, 268, 260, 255, 245, 285, 267, 255, 234, 259, 266, 265, 240, 269, 252, 284, 241, 242, 254, 254, 265, 263, 275, 222, 230, 224, 261, 264, 253, 247, 243, 239, 290, 206, 244, 251, 260, 247, 251, 256, 244, 255, 275, 280, 262, 252, 263, 274, 258, 265, 265, 270, 242, 268, 262, 247, 244, 260, 277, 251, 261, 280, 234, 253, 260, 223, 239, 254, 262, 258, 266, 261, 244, 289, 261, 250, 268, 265, 235, 251, 279, 237, 270, 228, 249, 206, 245, 268, 254] payload_rows_per_dst=[5228, 5420, 5402, 5686, 5420, 5598, 5592, 5462] dup_rows_per_dst=[2256, 2706, 2656, 3016, 2480, 2830, 2992, 2792] combine_rows_per_src=[5228, 5420, 5402, 5686, 5420, 5598, 5592, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=37866 dup_rows=21930 dup_rows_remote=18888 dedup_ratio=0.334625 router_tokens_per_expert=[271, 291, 237, 254, 272, 269, 229, 249, 287, 295, 260, 247, 268, 278, 239, 238, 270, 239, 264, 218, 238, 255, 266, 235, 248, 274, 255, 278, 256, 284, 251, 256, 267, 243, 265, 259, 251, 243, 239, 280, 244, 249, 242, 229, 228, 238, 275, 249, 265, 255, 244, 248, 255, 263, 294, 231, 255, 235, 251, 252, 238, 221, 277, 236, 263, 235, 265, 249, 244, 262, 259, 260, 245, 254, 266, 269, 263, 260, 257, 279, 233, 263, 254, 271, 236, 259, 251, 231, 273, 275, 250, 294, 266, 275, 263, 252, 240, 272, 235, 247, 274, 288, 256, 286, 243, 228, 276, 237, 261, 261, 249, 236, 237, 256, 236, 247, 248, 257, 278, 277, 294, 244, 262, 232, 275, 244, 277, 247] payload_rows_per_dst=[5740, 5400, 5488, 5184, 5506, 5380, 5428, 5480] dup_rows_per_dst=[3042, 2782, 2778, 2434, 2754, 2728, 2670, 2742] combine_rows_per_src=[5740, 5400, 5488, 5184, 5506, 5380, 5428, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=228856 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157277688 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314326520 weight_movement_bytes=18874368 total_nvlink_bytes=333200888 weight_to_token_ratio=0.060047 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38282 dup_rows=21834 dup_rows_remote=19116 dedup_ratio=0.333160 router_tokens_per_expert=[250, 259, 238, 266, 249, 256, 259, 260, 286, 259, 252, 298, 267, 262, 265, 249, 243, 240, 271, 260, 263, 243, 259, 238, 241, 265, 248, 248, 267, 282, 258, 243, 262, 268, 259, 227, 242, 240, 256, 272, 237, 244, 257, 237, 291, 269, 254, 237, 228, 227, 269, 272, 231, 247, 234, 266, 240, 231, 256, 258, 269, 258, 266, 240, 288, 255, 274, 289, 280, 261, 252, 270, 263, 262, 240, 260, 263, 253, 297, 247, 249, 250, 279, 234, 242, 260, 244, 233, 235, 266, 240, 248, 274, 257, 238, 276, 269, 256, 233, 235, 291, 245, 267, 231, 271, 268, 252, 256, 252, 254, 258, 245, 234, 255, 244, 273, 250, 262, 258, 257, 272, 242, 282, 235, 262, 270, 254, 268] payload_rows_per_dst=[5550, 5420, 5402, 5392, 5576, 5412, 5436, 5514] dup_rows_per_dst=[2800, 2718, 2702, 2592, 2932, 2638, 2730, 2722] combine_rows_per_src=[5550, 5420, 5402, 5392, 5576, 5412, 5436, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=229944 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157647416 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315064888 weight_movement_bytes=18874368 total_nvlink_bytes=333939256 weight_to_token_ratio=0.059906 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229128 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157294344 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314359560 weight_movement_bytes=18874368 total_nvlink_bytes=333233928 weight_to_token_ratio=0.060041 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44002 payload_rows_remote=38566 dup_rows=21534 dup_rows_remote=18826 dedup_ratio=0.328583 router_tokens_per_expert=[240, 254, 306, 282, 254, 273, 270, 246, 288, 232, 233, 247, 242, 248, 266, 258, 273, 250, 238, 247, 251, 248, 276, 263, 256, 243, 246, 236, 261, 271, 271, 226, 247, 257, 272, 236, 242, 304, 262, 238, 292, 252, 232, 260, 256, 251, 278, 248, 304, 265, 250, 228, 243, 251, 265, 252, 243, 251, 259, 240, 263, 240, 256, 241, 279, 260, 243, 244, 266, 241, 264, 266, 270, 276, 252, 276, 249, 272, 238, 250, 269, 267, 254, 263, 273, 228, 270, 239, 243, 237, 249, 275, 240, 251, 266, 248, 259, 246, 282, 233, 271, 273, 250, 270, 245, 238, 257, 215, 253, 264, 262, 287, 242, 239, 229, 268, 246, 252, 236, 264, 255, 249, 257, 265, 253, 255, 290, 272] payload_rows_per_dst=[5558, 5466, 5488, 5414, 5590, 5436, 5508, 5542] dup_rows_per_dst=[2720, 2646, 2766, 2688, 2702, 2708, 2702, 2602] combine_rows_per_src=[5558, 5466, 5488, 5414, 5590, 5436, 5508, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38354 dup_rows=21642 dup_rows_remote=18838 dedup_ratio=0.330231 router_tokens_per_expert=[243, 242, 211, 257, 254, 233, 243, 270, 267, 239, 262, 270, 264, 233, 268, 236, 278, 242, 275, 244, 264, 279, 239, 238, 238, 227, 269, 264, 259, 252, 263, 284, 285, 272, 259, 248, 268, 261, 260, 267, 258, 242, 258, 252, 250, 262, 259, 251, 284, 266, 271, 260, 274, 230, 246, 247, 269, 262, 240, 250, 240, 267, 262, 247, 254, 276, 226, 256, 262, 238, 276, 239, 262, 265, 233, 242, 272, 253, 260, 263, 275, 259, 271, 258, 257, 270, 278, 251, 261, 254, 257, 242, 245, 253, 237, 263, 230, 276, 273, 254, 259, 233, 252, 257, 252, 297, 244, 262, 280, 284, 251, 268, 258, 253, 259, 232, 218, 242, 259, 274, 257, 258, 248, 212, 249, 281, 247, 267] payload_rows_per_dst=[5390, 5508, 5524, 5520, 5486, 5538, 5540, 5388] dup_rows_per_dst=[2594, 2722, 2780, 2710, 2668, 2724, 2804, 2640] combine_rows_per_src=[5390, 5508, 5524, 5520, 5486, 5538, 5540, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155099136 dispatch_meta_bytes=227016 combine_payload_bytes=155099136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155326152 ingress_bytes=173973504 link_bytes=173973504 token_bytes=310425288 weight_movement_bytes=18874368 total_nvlink_bytes=329299656 weight_to_token_ratio=0.060802 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.666928515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229592 combine_payload_bytes=156803072 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157032664 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835736 weight_movement_bytes=37748736 total_nvlink_bytes=351584472 weight_to_token_ratio=0.120282 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38266 dup_rows=21902 dup_rows_remote=19216 dedup_ratio=0.334198 router_tokens_per_expert=[256, 252, 273, 259, 254, 255, 245, 280, 243, 258, 260, 251, 237, 261, 278, 244, 255, 251, 240, 268, 246, 259, 280, 256, 253, 244, 264, 269, 292, 267, 246, 234, 269, 246, 268, 252, 247, 250, 259, 239, 277, 261, 263, 250, 250, 279, 244, 257, 249, 241, 244, 259, 228, 229, 249, 275, 242, 251, 261, 254, 252, 259, 266, 268, 274, 260, 256, 244, 240, 261, 253, 256, 274, 260, 269, 247, 255, 277, 236, 248, 213, 279, 239, 236, 232, 245, 258, 263, 259, 256, 249, 246, 265, 267, 257, 249, 274, 238, 262, 274, 248, 280, 276, 269, 233, 267, 240, 243, 247, 270, 269, 248, 250, 274, 260, 249, 258, 253, 241, 248, 256, 260, 280, 260, 268, 269, 246, 267] payload_rows_per_dst=[5478, 5464, 5468, 5368, 5456, 5362, 5544, 5494] dup_rows_per_dst=[2734, 2784, 2754, 2686, 2764, 2664, 2732, 2784] combine_rows_per_src=[5478, 5464, 5468, 5368, 5456, 5362, 5544, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157966336 dispatch_meta_bytes=229568 combine_payload_bytes=157966336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158195904 ingress_bytes=176840704 link_bytes=176840704 token_bytes=316162240 weight_movement_bytes=18874368 total_nvlink_bytes=335036608 weight_to_token_ratio=0.059698 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694400615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=228768 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157326752 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314424736 weight_movement_bytes=18874368 total_nvlink_bytes=333299104 weight_to_token_ratio=0.060028 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38326 dup_rows=21748 dup_rows_remote=18980 dedup_ratio=0.331848 router_tokens_per_expert=[291, 219, 273, 262, 244, 259, 256, 272, 249, 242, 243, 252, 286, 279, 260, 245, 234, 259, 256, 230, 252, 278, 269, 259, 227, 265, 272, 274, 261, 245, 244, 267, 287, 259, 250, 261, 289, 281, 239, 244, 267, 268, 229, 271, 250, 232, 242, 246, 250, 259, 271, 253, 259, 251, 255, 280, 271, 262, 237, 235, 237, 256, 247, 259, 248, 229, 234, 283, 255, 252, 238, 273, 276, 244, 226, 253, 263, 272, 272, 267, 262, 263, 257, 255, 264, 232, 254, 268, 262, 276, 258, 258, 266, 248, 217, 279, 245, 251, 268, 235, 257, 239, 258, 241, 245, 257, 254, 255, 240, 265, 269, 237, 256, 244, 260, 250, 261, 273, 273, 260, 248, 247, 237, 256, 256, 270, 258, 278] payload_rows_per_dst=[5516, 5452, 5462, 5460, 5494, 5450, 5418, 5536] dup_rows_per_dst=[2748, 2732, 2768, 2704, 2676, 2788, 2614, 2718] combine_rows_per_src=[5516, 5452, 5462, 5460, 5494, 5450, 5418, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229928 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967464 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313705000 weight_movement_bytes=18874368 total_nvlink_bytes=332579368 weight_to_token_ratio=0.060166 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38208 dup_rows=21876 dup_rows_remote=19176 dedup_ratio=0.333801 router_tokens_per_expert=[280, 241, 257, 229, 253, 275, 253, 263, 274, 274, 255, 242, 245, 265, 269, 254, 307, 274, 271, 250, 276, 279, 244, 277, 284, 266, 245, 232, 253, 264, 259, 230, 278, 257, 242, 265, 257, 313, 268, 279, 263, 248, 238, 258, 226, 250, 234, 267, 259, 279, 250, 250, 248, 247, 254, 237, 263, 273, 248, 221, 265, 271, 238, 234, 252, 247, 265, 263, 262, 230, 257, 237, 276, 240, 275, 260, 226, 266, 273, 247, 248, 277, 246, 247, 253, 228, 279, 267, 253, 249, 252, 268, 271, 268, 259, 248, 264, 260, 247, 229, 225, 242, 241, 237, 234, 263, 273, 249, 266, 286, 258, 235, 244, 251, 252, 240, 250, 236, 270, 250, 260, 239, 250, 245, 253, 278, 258, 274] payload_rows_per_dst=[5414, 5492, 5478, 5452, 5452, 5520, 5468, 5384] dup_rows_per_dst=[2844, 2930, 2808, 2622, 2700, 2706, 2550, 2716] combine_rows_per_src=[5414, 5492, 5478, 5452, 5452, 5520, 5468, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38226 dup_rows=21774 dup_rows_remote=19096 dedup_ratio=0.332245 router_tokens_per_expert=[261, 236, 276, 265, 239, 266, 281, 234, 263, 241, 276, 253, 261, 232, 235, 269, 283, 264, 287, 269, 244, 234, 247, 246, 254, 251, 270, 228, 235, 249, 229, 264, 273, 226, 266, 246, 256, 254, 254, 253, 224, 264, 285, 238, 272, 276, 253, 245, 282, 245, 242, 264, 258, 255, 242, 265, 239, 239, 258, 259, 250, 272, 232, 266, 231, 243, 244, 260, 298, 244, 275, 282, 256, 261, 252, 260, 232, 254, 276, 281, 242, 261, 246, 271, 262, 229, 250, 274, 253, 249, 278, 257, 255, 273, 244, 287, 257, 269, 246, 255, 252, 269, 250, 268, 246, 243, 239, 240, 252, 264, 251, 287, 259, 238, 276, 268, 252, 240, 253, 255, 241, 240, 258, 284, 286, 251, 247, 257] payload_rows_per_dst=[5476, 5414, 5434, 5408, 5518, 5534, 5442, 5536] dup_rows_per_dst=[2700, 2694, 2736, 2640, 2780, 2812, 2734, 2678] combine_rows_per_src=[5476, 5414, 5434, 5408, 5518, 5534, 5442, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38192 dup_rows=21750 dup_rows_remote=18842 dedup_ratio=0.331879 router_tokens_per_expert=[303, 260, 250, 312, 268, 269, 268, 252, 249, 268, 264, 259, 271, 268, 238, 247, 240, 267, 281, 249, 259, 233, 271, 244, 230, 253, 263, 232, 244, 254, 267, 290, 244, 275, 271, 265, 256, 228, 261, 280, 283, 220, 253, 259, 265, 238, 271, 250, 265, 248, 229, 276, 270, 250, 262, 255, 256, 262, 246, 249, 260, 257, 237, 268, 276, 279, 261, 264, 249, 263, 229, 243, 269, 242, 257, 257, 265, 260, 262, 262, 244, 244, 264, 251, 253, 269, 249, 271, 238, 266, 245, 253, 286, 255, 234, 253, 235, 264, 254, 230, 252, 235, 256, 237, 238, 258, 245, 278, 259, 240, 234, 264, 260, 256, 272, 255, 250, 260, 285, 254, 247, 237, 253, 235, 247, 253, 245, 235] payload_rows_per_dst=[5594, 5496, 5358, 5460, 5524, 5504, 5390, 5460] dup_rows_per_dst=[2908, 2864, 2870, 2720, 2614, 2646, 2568, 2560] combine_rows_per_src=[5594, 5496, 5358, 5460, 5524, 5504, 5390, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229224 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212520 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195816 weight_movement_bytes=37748736 total_nvlink_bytes=351944552 weight_to_token_ratio=0.120144 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229536 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156729504 ingress_bytes=213123072 link_bytes=213123072 token_bytes=313229472 weight_movement_bytes=56623104 total_nvlink_bytes=369852576 weight_to_token_ratio=0.180772 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042040413e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38212 dup_rows=21744 dup_rows_remote=18756 dedup_ratio=0.331787 router_tokens_per_expert=[248, 263, 274, 255, 261, 277, 237, 244, 226, 237, 257, 234, 247, 247, 267, 230, 280, 276, 267, 266, 246, 278, 216, 280, 284, 276, 244, 252, 262, 265, 249, 262, 253, 276, 259, 261, 233, 249, 261, 248, 251, 239, 244, 284, 256, 279, 252, 246, 233, 265, 230, 242, 267, 242, 235, 257, 249, 244, 264, 253, 231, 246, 269, 250, 262, 242, 245, 263, 248, 268, 273, 263, 248, 240, 260, 266, 246, 260, 268, 273, 238, 264, 233, 259, 264, 278, 249, 291, 268, 273, 244, 281, 270, 240, 251, 256, 273, 260, 266, 219, 263, 244, 265, 267, 268, 264, 273, 232, 265, 247, 261, 248, 259, 280, 252, 262, 237, 265, 247, 249, 246, 250, 253, 258, 254, 266, 264, 252] payload_rows_per_dst=[5454, 5580, 5516, 5418, 5440, 5360, 5560, 5464] dup_rows_per_dst=[2554, 2988, 2666, 2536, 2810, 2796, 2670, 2724] combine_rows_per_src=[5454, 5580, 5516, 5418, 5440, 5360, 5560, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229288 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156802984 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313376680 weight_movement_bytes=37748736 total_nvlink_bytes=351125416 weight_to_token_ratio=0.120458 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38258 dup_rows=21806 dup_rows_remote=19090 dedup_ratio=0.332733 router_tokens_per_expert=[246, 273, 239, 271, 256, 289, 235, 244, 283, 254, 273, 261, 268, 233, 225, 259, 260, 263, 274, 240, 255, 235, 234, 240, 252, 258, 263, 263, 231, 285, 262, 235, 244, 254, 262, 221, 256, 239, 292, 240, 252, 270, 261, 257, 219, 268, 246, 244, 235, 226, 247, 263, 255, 256, 260, 256, 285, 251, 297, 254, 264, 259, 217, 284, 209, 249, 260, 238, 255, 251, 259, 245, 255, 253, 240, 267, 268, 246, 280, 263, 253, 247, 253, 236, 252, 255, 297, 258, 214, 254, 258, 265, 272, 273, 265, 242, 286, 279, 236, 249, 274, 265, 260, 268, 253, 251, 265, 268, 270, 263, 263, 225, 259, 262, 295, 257, 264, 245, 249, 277, 286, 226, 271, 246, 273, 240, 271, 247] payload_rows_per_dst=[5416, 5430, 5456, 5414, 5440, 5472, 5590, 5512] dup_rows_per_dst=[2802, 2670, 2594, 2804, 2636, 2716, 2760, 2824] combine_rows_per_src=[5416, 5430, 5456, 5414, 5440, 5472, 5590, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228136 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156662568 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313097000 weight_movement_bytes=37748736 total_nvlink_bytes=350845736 weight_to_token_ratio=0.120566 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38320 dup_rows=21728 dup_rows_remote=18982 dedup_ratio=0.331543 router_tokens_per_expert=[275, 247, 276, 244, 253, 250, 267, 286, 247, 270, 254, 296, 231, 264, 271, 268, 277, 254, 240, 219, 269, 269, 247, 247, 269, 240, 253, 265, 239, 257, 274, 256, 235, 248, 240, 233, 291, 251, 253, 275, 251, 230, 253, 284, 278, 249, 241, 260, 251, 245, 263, 236, 253, 282, 238, 260, 238, 246, 263, 248, 255, 264, 256, 237, 262, 262, 277, 266, 257, 239, 240, 241, 248, 266, 244, 260, 241, 267, 266, 223, 255, 272, 257, 247, 244, 264, 258, 270, 240, 306, 280, 229, 256, 259, 275, 254, 255, 246, 268, 280, 246, 253, 269, 261, 290, 267, 275, 233, 257, 246, 233, 238, 263, 254, 225, 242, 264, 282, 242, 234, 255, 228, 265, 270, 269, 263, 235, 254] payload_rows_per_dst=[5556, 5446, 5428, 5426, 5496, 5492, 5488, 5476] dup_rows_per_dst=[2842, 2704, 2716, 2644, 2622, 2840, 2746, 2614] combine_rows_per_src=[5556, 5446, 5428, 5426, 5496, 5492, 5488, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38312 dup_rows=21764 dup_rows_remote=18938 dedup_ratio=0.332092 router_tokens_per_expert=[252, 254, 249, 254, 244, 265, 236, 277, 236, 240, 254, 251, 283, 255, 287, 248, 276, 265, 235, 264, 231, 250, 229, 247, 254, 243, 297, 254, 257, 269, 236, 242, 276, 245, 249, 259, 253, 269, 297, 243, 279, 237, 237, 263, 260, 305, 253, 256, 246, 262, 254, 259, 253, 242, 272, 242, 244, 269, 257, 273, 256, 281, 286, 247, 238, 273, 233, 242, 240, 258, 260, 240, 238, 221, 238, 253, 269, 227, 258, 241, 270, 222, 265, 263, 273, 259, 269, 272, 265, 264, 253, 252, 254, 255, 269, 243, 273, 270, 258, 280, 252, 294, 250, 272, 238, 243, 220, 273, 255, 267, 273, 274, 257, 279, 241, 241, 224, 253, 267, 224, 247, 273, 271, 266, 240, 254, 277, 227] payload_rows_per_dst=[5534, 5406, 5588, 5460, 5368, 5480, 5552, 5384] dup_rows_per_dst=[2636, 2692, 2774, 2826, 2490, 2816, 2832, 2698] combine_rows_per_src=[5534, 5406, 5588, 5460, 5368, 5480, 5552, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=227872 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156744224 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313260576 weight_movement_bytes=18874368 total_nvlink_bytes=332134944 weight_to_token_ratio=0.060251 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229392 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156934160 ingress_bytes=194453504 link_bytes=194453504 token_bytes=313638928 weight_movement_bytes=37748736 total_nvlink_bytes=351387664 weight_to_token_ratio=0.120357 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863157798e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229208 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157187928 ingress_bytes=232456192 link_bytes=232456192 token_bytes=314146648 weight_movement_bytes=75497472 total_nvlink_bytes=389644120 weight_to_token_ratio=0.240326 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227280856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229000 combine_payload_bytes=156925952 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157154952 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314080904 weight_movement_bytes=37748736 total_nvlink_bytes=351829640 weight_to_token_ratio=0.120188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38170 dup_rows=21892 dup_rows_remote=19212 dedup_ratio=0.334045 router_tokens_per_expert=[262, 258, 254, 269, 266, 254, 252, 256, 276, 261, 241, 255, 259, 268, 224, 265, 248, 266, 272, 256, 261, 263, 238, 274, 226, 248, 266, 263, 250, 246, 279, 261, 240, 276, 237, 244, 245, 269, 220, 268, 278, 251, 280, 241, 250, 250, 263, 265, 258, 259, 235, 260, 263, 261, 263, 286, 255, 260, 274, 246, 248, 248, 254, 256, 252, 260, 228, 247, 245, 269, 263, 262, 241, 261, 261, 245, 283, 249, 244, 241, 252, 253, 248, 246, 267, 258, 265, 264, 282, 261, 251, 249, 248, 260, 261, 266, 262, 255, 304, 229, 263, 249, 234, 287, 259, 230, 256, 242, 257, 269, 239, 227, 266, 232, 272, 279, 275, 249, 256, 251, 243, 232, 242, 253, 273, 257, 269, 235] payload_rows_per_dst=[5530, 5466, 5474, 5384, 5416, 5490, 5382, 5502] dup_rows_per_dst=[2710, 2768, 2680, 2868, 2686, 2772, 2742, 2666] combine_rows_per_src=[5530, 5466, 5474, 5384, 5416, 5490, 5382, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38134 dup_rows=21936 dup_rows_remote=19218 dedup_ratio=0.334717 router_tokens_per_expert=[248, 245, 292, 246, 251, 261, 278, 259, 261, 253, 259, 240, 255, 280, 250, 252, 244, 276, 247, 257, 263, 258, 266, 245, 267, 261, 246, 255, 227, 235, 255, 244, 263, 257, 268, 234, 271, 244, 240, 250, 244, 277, 265, 256, 247, 263, 269, 244, 256, 278, 272, 236, 285, 249, 281, 254, 247, 273, 236, 260, 248, 256, 234, 260, 242, 257, 248, 254, 226, 261, 252, 285, 258, 244, 260, 272, 264, 238, 252, 279, 248, 262, 269, 276, 244, 251, 260, 287, 244, 262, 263, 245, 279, 267, 253, 270, 264, 257, 253, 237, 241, 273, 274, 260, 250, 246, 234, 241, 256, 259, 282, 244, 253, 244, 242, 288, 253, 269, 207, 264, 240, 256, 273, 252, 251, 241, 270, 229] payload_rows_per_dst=[5512, 5418, 5482, 5468, 5466, 5554, 5366, 5334] dup_rows_per_dst=[2748, 2674, 2702, 2782, 2718, 2806, 2776, 2730] combine_rows_per_src=[5512, 5418, 5482, 5468, 5466, 5554, 5366, 5334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38094 dup_rows=21812 dup_rows_remote=18814 dedup_ratio=0.332825 router_tokens_per_expert=[243, 252, 268, 248, 255, 277, 259, 265, 256, 247, 243, 250, 255, 262, 246, 262, 249, 261, 261, 245, 274, 283, 268, 253, 273, 281, 252, 266, 252, 253, 259, 262, 232, 262, 247, 228, 261, 246, 246, 271, 302, 258, 255, 251, 259, 248, 258, 245, 240, 269, 233, 251, 254, 251, 267, 254, 234, 256, 230, 250, 255, 275, 250, 251, 271, 244, 248, 274, 292, 246, 253, 257, 272, 272, 220, 250, 284, 268, 233, 256, 240, 241, 259, 289, 243, 249, 240, 230, 245, 226, 253, 264, 282, 288, 271, 276, 247, 273, 257, 254, 277, 270, 237, 251, 246, 242, 269, 239, 256, 250, 258, 243, 257, 242, 244, 256, 258, 266, 286, 257, 255, 264, 228, 270, 238, 267, 252, 254] payload_rows_per_dst=[5458, 5614, 5448, 5406, 5496, 5402, 5270, 5630] dup_rows_per_dst=[2718, 2770, 2690, 2628, 2784, 2792, 2432, 2998] combine_rows_per_src=[5458, 5614, 5448, 5406, 5496, 5402, 5270, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229528 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573848 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312918168 weight_movement_bytes=18874368 total_nvlink_bytes=331792536 weight_to_token_ratio=0.060317 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38164 dup_rows=21928 dup_rows_remote=19124 dedup_ratio=0.334595 router_tokens_per_expert=[248, 240, 234, 268, 287, 259, 256, 254, 220, 244, 251, 267, 301, 265, 279, 251, 252, 266, 248, 262, 256, 255, 242, 253, 232, 242, 277, 244, 288, 249, 272, 268, 275, 267, 246, 233, 223, 254, 265, 259, 240, 224, 248, 270, 251, 284, 265, 261, 251, 244, 257, 238, 233, 266, 244, 255, 246, 252, 283, 240, 277, 230, 238, 261, 247, 258, 230, 277, 285, 252, 266, 275, 233, 264, 246, 227, 264, 259, 268, 229, 253, 259, 253, 258, 240, 268, 241, 250, 254, 249, 234, 265, 231, 283, 278, 258, 246, 250, 265, 274, 252, 266, 256, 248, 255, 260, 261, 299, 258, 279, 246, 254, 250, 250, 272, 234, 272, 270, 254, 260, 236, 263, 255, 229, 303, 243, 287, 257] payload_rows_per_dst=[5444, 5422, 5478, 5462, 5612, 5464, 5300, 5426] dup_rows_per_dst=[2804, 2790, 2692, 2760, 2922, 2750, 2624, 2586] combine_rows_per_src=[5444, 5422, 5478, 5462, 5612, 5464, 5300, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=229408 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156426272 ingress_bytes=175071232 link_bytes=175071232 token_bytes=312623136 weight_movement_bytes=18874368 total_nvlink_bytes=331497504 weight_to_token_ratio=0.060374 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677446405e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=227632 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156260656 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312293680 weight_movement_bytes=18874368 total_nvlink_bytes=331168048 weight_to_token_ratio=0.060438 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38046 dup_rows=21962 dup_rows_remote=19348 dedup_ratio=0.335114 router_tokens_per_expert=[269, 262, 248, 257, 255, 263, 262, 272, 234, 244, 232, 266, 286, 236, 274, 243, 237, 218, 286, 274, 259, 257, 224, 246, 241, 274, 247, 273, 255, 267, 233, 280, 253, 262, 237, 278, 244, 253, 266, 286, 242, 240, 271, 259, 260, 226, 251, 244, 246, 258, 266, 276, 277, 261, 259, 241, 250, 258, 267, 284, 253, 224, 272, 279, 260, 253, 261, 238, 259, 281, 254, 269, 259, 272, 257, 238, 241, 251, 261, 270, 249, 255, 249, 240, 250, 249, 234, 278, 255, 273, 259, 256, 262, 268, 257, 222, 242, 229, 250, 269, 250, 239, 241, 267, 262, 250, 270, 280, 269, 246, 252, 243, 272, 269, 249, 257, 261, 239, 274, 276, 247, 265, 258, 242, 232, 237, 268, 266] payload_rows_per_dst=[5486, 5528, 5432, 5488, 5416, 5398, 5398, 5428] dup_rows_per_dst=[2720, 2614, 2712, 2854, 2832, 2714, 2720, 2796] combine_rows_per_src=[5486, 5528, 5432, 5488, 5416, 5398, 5398, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38360 dup_rows=21702 dup_rows_remote=18898 dedup_ratio=0.331146 router_tokens_per_expert=[267, 220, 239, 261, 264, 268, 212, 247, 237, 258, 239, 227, 284, 277, 240, 268, 262, 268, 252, 281, 227, 257, 264, 267, 257, 245, 265, 253, 225, 257, 253, 267, 249, 263, 275, 271, 256, 262, 235, 229, 260, 274, 261, 253, 259, 274, 245, 256, 257, 242, 239, 281, 269, 241, 289, 266, 278, 258, 260, 256, 228, 252, 226, 254, 257, 253, 268, 250, 240, 268, 242, 251, 272, 258, 262, 255, 269, 238, 250, 250, 258, 268, 255, 266, 249, 277, 268, 265, 261, 243, 269, 243, 257, 264, 255, 241, 252, 279, 263, 285, 250, 267, 268, 256, 264, 244, 261, 226, 239, 284, 284, 262, 251, 273, 272, 239, 239, 227, 245, 240, 260, 269, 266, 266, 253, 269, 250, 217] payload_rows_per_dst=[5340, 5578, 5464, 5484, 5498, 5474, 5538, 5458] dup_rows_per_dst=[2676, 2622, 2780, 2708, 2668, 2804, 2830, 2614] combine_rows_per_src=[5340, 5578, 5464, 5484, 5498, 5474, 5538, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38374 dup_rows=21762 dup_rows_remote=19022 dedup_ratio=0.332062 router_tokens_per_expert=[265, 256, 254, 246, 263, 239, 276, 261, 258, 258, 244, 289, 280, 254, 292, 264, 253, 240, 248, 229, 234, 281, 279, 225, 246, 228, 250, 258, 274, 281, 277, 261, 237, 270, 245, 270, 249, 249, 248, 247, 268, 284, 262, 251, 265, 256, 245, 279, 262, 249, 237, 254, 268, 282, 242, 265, 264, 269, 257, 250, 267, 252, 263, 245, 283, 262, 243, 244, 242, 270, 253, 267, 277, 211, 257, 240, 246, 234, 266, 240, 240, 260, 261, 274, 242, 287, 250, 263, 246, 245, 259, 235, 244, 249, 245, 279, 248, 284, 245, 231, 236, 246, 254, 266, 261, 276, 239, 258, 255, 228, 270, 273, 251, 269, 257, 256, 243, 269, 237, 257, 240, 263, 263, 248, 263, 256, 215, 283] payload_rows_per_dst=[5586, 5410, 5536, 5480, 5384, 5450, 5400, 5528] dup_rows_per_dst=[2730, 2718, 2714, 2854, 2686, 2708, 2740, 2612] combine_rows_per_src=[5586, 5410, 5536, 5480, 5384, 5450, 5400, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38254 dup_rows=21924 dup_rows_remote=19148 dedup_ratio=0.334534 router_tokens_per_expert=[274, 281, 254, 225, 242, 236, 258, 234, 245, 267, 263, 247, 236, 273, 296, 256, 265, 254, 274, 244, 263, 250, 260, 287, 262, 249, 227, 236, 253, 250, 238, 233, 245, 264, 248, 213, 257, 284, 263, 265, 268, 248, 267, 265, 265, 235, 269, 293, 271, 243, 257, 274, 284, 286, 245, 277, 222, 247, 234, 258, 247, 222, 242, 258, 274, 265, 245, 245, 240, 270, 265, 228, 265, 267, 264, 236, 274, 250, 263, 256, 244, 254, 289, 266, 261, 239, 243, 241, 234, 277, 249, 247, 219, 262, 272, 290, 280, 253, 259, 262, 274, 252, 258, 255, 217, 277, 269, 263, 248, 257, 286, 262, 273, 259, 247, 235, 263, 254, 246, 236, 258, 258, 245, 261, 276, 252, 235, 256] payload_rows_per_dst=[5490, 5400, 5458, 5358, 5516, 5436, 5504, 5450] dup_rows_per_dst=[2684, 2690, 2840, 2776, 2698, 2738, 2840, 2658] combine_rows_per_src=[5490, 5400, 5458, 5358, 5516, 5436, 5504, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229152 combine_payload_bytes=156319744 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156548896 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868640 weight_movement_bytes=37748736 total_nvlink_bytes=350617376 weight_to_token_ratio=0.120654 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=229576 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156065992 ingress_bytes=193585152 link_bytes=193585152 token_bytes=311902408 weight_movement_bytes=37748736 total_nvlink_bytes=349651144 weight_to_token_ratio=0.121027 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854837677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229032 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157351592 ingress_bytes=213745664 link_bytes=213745664 token_bytes=314474152 weight_movement_bytes=56623104 total_nvlink_bytes=371097256 weight_to_token_ratio=0.180056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048005783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229584 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409488 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589392 weight_movement_bytes=18874368 total_nvlink_bytes=333463760 weight_to_token_ratio=0.059997 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43976 payload_rows_remote=38522 dup_rows=21560 dup_rows_remote=18922 dedup_ratio=0.328979 router_tokens_per_expert=[275, 269, 261, 240, 264, 254, 268, 248, 254, 261, 240, 219, 267, 252, 266, 246, 244, 268, 263, 285, 264, 273, 257, 247, 255, 241, 281, 253, 260, 246, 253, 263, 260, 259, 270, 264, 248, 258, 253, 260, 246, 249, 259, 258, 274, 255, 215, 218, 230, 222, 242, 288, 262, 246, 257, 244, 253, 281, 246, 256, 276, 246, 251, 264, 258, 253, 261, 256, 253, 257, 246, 259, 269, 275, 228, 236, 253, 244, 264, 278, 255, 253, 256, 246, 242, 253, 277, 258, 250, 294, 254, 242, 267, 246, 264, 262, 272, 238, 248, 265, 253, 283, 263, 261, 285, 242, 259, 264, 234, 266, 243, 250, 254, 280, 237, 253, 248, 244, 247, 252, 248, 260, 249, 255, 256, 261, 260, 282] payload_rows_per_dst=[5514, 5548, 5454, 5464, 5488, 5542, 5510, 5456] dup_rows_per_dst=[2654, 2758, 2638, 2664, 2692, 2696, 2742, 2716] combine_rows_per_src=[5514, 5548, 5454, 5464, 5488, 5542, 5510, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229608 combine_payload_bytes=156688384 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156917992 ingress_bytes=213311488 link_bytes=213311488 token_bytes=313606376 weight_movement_bytes=56623104 total_nvlink_bytes=370229480 weight_to_token_ratio=0.180555 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043845722e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44024 payload_rows_remote=38508 dup_rows=21512 dup_rows_remote=18908 dedup_ratio=0.328247 router_tokens_per_expert=[235, 238, 230, 251, 261, 260, 249, 259, 279, 250, 231, 293, 276, 258, 234, 286, 283, 263, 246, 258, 273, 287, 259, 267, 249, 266, 246, 257, 249, 300, 225, 267, 257, 225, 270, 263, 268, 233, 275, 235, 245, 255, 244, 246, 256, 253, 262, 236, 252, 227, 249, 277, 250, 294, 249, 244, 268, 254, 272, 264, 261, 263, 253, 275, 252, 262, 243, 260, 263, 217, 264, 237, 271, 267, 254, 233, 276, 247, 244, 270, 247, 261, 258, 253, 251, 252, 246, 261, 257, 243, 234, 259, 269, 215, 291, 275, 258, 242, 245, 289, 251, 251, 258, 261, 263, 269, 231, 259, 270, 231, 269, 278, 252, 247, 265, 239, 273, 248, 246, 233, 236, 250, 267, 250, 277, 251, 265, 252] payload_rows_per_dst=[5474, 5624, 5494, 5544, 5516, 5404, 5424, 5544] dup_rows_per_dst=[2706, 2766, 2552, 2760, 2604, 2740, 2826, 2558] combine_rows_per_src=[5474, 5624, 5494, 5544, 5516, 5404, 5424, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38350 dup_rows=21740 dup_rows_remote=19018 dedup_ratio=0.331726 router_tokens_per_expert=[265, 241, 265, 228, 237, 227, 269, 244, 248, 270, 281, 252, 282, 262, 290, 249, 267, 264, 256, 267, 240, 251, 256, 237, 229, 279, 257, 237, 260, 251, 262, 264, 259, 267, 283, 271, 262, 251, 240, 265, 262, 256, 288, 270, 245, 248, 248, 263, 278, 226, 274, 228, 279, 254, 230, 244, 276, 277, 266, 228, 260, 252, 299, 249, 274, 256, 242, 247, 245, 266, 272, 256, 217, 250, 277, 270, 248, 251, 228, 281, 252, 230, 286, 234, 263, 241, 248, 260, 237, 247, 269, 251, 252, 274, 265, 279, 247, 233, 242, 268, 256, 265, 250, 224, 258, 256, 232, 264, 254, 264, 266, 252, 234, 236, 276, 232, 221, 267, 268, 228, 244, 295, 282, 279, 244, 251, 267, 260] payload_rows_per_dst=[5458, 5290, 5716, 5394, 5450, 5652, 5390, 5446] dup_rows_per_dst=[2694, 2468, 3036, 2522, 2710, 2916, 2672, 2722] combine_rows_per_src=[5458, 5290, 5716, 5394, 5450, 5652, 5390, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157786112 dispatch_meta_bytes=229776 combine_payload_bytes=157786112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158015888 ingress_bytes=176660480 link_bytes=176660480 token_bytes=315802000 weight_movement_bytes=18874368 total_nvlink_bytes=334676368 weight_to_token_ratio=0.059766 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692673797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157728768 dispatch_meta_bytes=229664 combine_payload_bytes=157728768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157958432 ingress_bytes=176603136 link_bytes=176603136 token_bytes=315687200 weight_movement_bytes=18874368 total_nvlink_bytes=334561568 weight_to_token_ratio=0.059788 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692124355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229472 combine_payload_bytes=157081600 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157311072 ingress_bytes=213704704 link_bytes=213704704 token_bytes=314392672 weight_movement_bytes=56623104 total_nvlink_bytes=371015776 weight_to_token_ratio=0.180103 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047613324e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38366 dup_rows=21750 dup_rows_remote=18796 dedup_ratio=0.331879 router_tokens_per_expert=[249, 232, 249, 257, 275, 268, 250, 274, 247, 270, 253, 281, 261, 260, 252, 247, 262, 286, 253, 268, 240, 253, 237, 249, 277, 271, 268, 265, 256, 252, 244, 270, 265, 255, 266, 271, 275, 261, 250, 264, 256, 248, 265, 224, 255, 228, 235, 241, 238, 283, 247, 233, 249, 222, 255, 250, 278, 259, 253, 252, 258, 255, 271, 258, 281, 259, 220, 232, 273, 263, 273, 245, 265, 257, 275, 247, 235, 288, 244, 262, 254, 274, 276, 236, 240, 250, 259, 280, 254, 245, 268, 256, 249, 240, 273, 279, 246, 249, 240, 248, 262, 265, 260, 259, 276, 253, 273, 219, 260, 276, 229, 240, 222, 253, 252, 268, 257, 255, 224, 267, 271, 228, 237, 257, 260, 285, 258, 271] payload_rows_per_dst=[5420, 5524, 5468, 5474, 5528, 5478, 5430, 5464] dup_rows_per_dst=[2954, 2654, 2664, 2648, 2696, 2788, 2680, 2666] combine_rows_per_src=[5420, 5524, 5468, 5474, 5528, 5478, 5430, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38104 dup_rows=21944 dup_rows_remote=19188 dedup_ratio=0.334839 router_tokens_per_expert=[242, 242, 241, 234, 247, 263, 221, 267, 259, 248, 258, 256, 224, 249, 249, 252, 258, 264, 261, 236, 271, 266, 245, 266, 253, 252, 255, 292, 240, 252, 254, 257, 263, 289, 239, 229, 261, 206, 246, 255, 265, 298, 233, 267, 239, 273, 236, 262, 240, 225, 260, 241, 261, 244, 272, 261, 256, 252, 275, 248, 270, 268, 221, 268, 252, 273, 244, 255, 249, 274, 241, 267, 265, 263, 269, 232, 278, 260, 259, 253, 230, 271, 274, 276, 255, 234, 260, 260, 264, 270, 232, 256, 239, 265, 283, 276, 255, 279, 243, 291, 258, 258, 283, 256, 264, 264, 254, 279, 213, 271, 243, 267, 256, 257, 270, 271, 257, 255, 236, 260, 257, 266, 272, 273, 234, 245, 252, 253] payload_rows_per_dst=[5348, 5488, 5450, 5416, 5492, 5458, 5528, 5412] dup_rows_per_dst=[2556, 2756, 2672, 2708, 2776, 2832, 2828, 2816] combine_rows_per_src=[5348, 5488, 5450, 5416, 5492, 5458, 5528, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38274 dup_rows=21704 dup_rows_remote=18960 dedup_ratio=0.331177 router_tokens_per_expert=[236, 275, 248, 272, 269, 258, 280, 269, 261, 257, 244, 257, 251, 247, 246, 277, 272, 256, 258, 250, 258, 263, 282, 224, 262, 285, 257, 250, 227, 247, 255, 273, 240, 224, 233, 252, 250, 262, 250, 238, 265, 274, 253, 252, 227, 250, 265, 241, 269, 270, 247, 233, 254, 258, 247, 251, 267, 297, 263, 266, 259, 235, 261, 264, 250, 225, 283, 277, 244, 265, 221, 241, 247, 238, 270, 262, 266, 249, 257, 244, 280, 257, 234, 274, 237, 279, 234, 266, 281, 273, 226, 245, 282, 264, 226, 293, 239, 253, 276, 265, 283, 262, 248, 222, 236, 266, 268, 250, 231, 246, 275, 258, 282, 268, 225, 265, 264, 267, 247, 255, 267, 261, 260, 241, 279, 253, 255, 228] payload_rows_per_dst=[5576, 5516, 5412, 5538, 5370, 5558, 5450, 5412] dup_rows_per_dst=[2718, 2722, 2540, 2744, 2708, 2744, 2706, 2822] combine_rows_per_src=[5576, 5516, 5412, 5538, 5370, 5558, 5450, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38278 dup_rows=21786 dup_rows_remote=19006 dedup_ratio=0.332428 router_tokens_per_expert=[257, 265, 243, 256, 250, 255, 250, 221, 264, 253, 298, 268, 268, 271, 268, 266, 219, 241, 226, 262, 229, 258, 249, 250, 266, 260, 253, 237, 247, 247, 239, 246, 269, 255, 267, 252, 237, 257, 265, 254, 229, 237, 230, 274, 220, 253, 275, 243, 256, 248, 242, 231, 271, 256, 263, 266, 259, 242, 261, 258, 268, 275, 255, 246, 243, 274, 283, 261, 229, 266, 276, 248, 254, 259, 275, 241, 278, 259, 244, 251, 235, 257, 267, 256, 282, 244, 273, 279, 258, 250, 259, 243, 244, 253, 260, 258, 274, 271, 272, 262, 237, 291, 245, 247, 266, 249, 249, 261, 259, 233, 251, 259, 252, 278, 289, 236, 262, 271, 257, 262, 263, 239, 262, 263, 267, 262, 253, 271] payload_rows_per_dst=[5490, 5388, 5402, 5492, 5498, 5436, 5472, 5572] dup_rows_per_dst=[2816, 2470, 2632, 2702, 2784, 2800, 2780, 2802] combine_rows_per_src=[5490, 5388, 5402, 5492, 5498, 5436, 5472, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38174 dup_rows=21956 dup_rows_remote=19178 dedup_ratio=0.335022 router_tokens_per_expert=[256, 254, 267, 276, 250, 228, 281, 277, 255, 269, 282, 260, 253, 264, 232, 251, 254, 252, 216, 251, 232, 267, 253, 229, 270, 279, 269, 255, 286, 228, 274, 261, 259, 263, 244, 273, 276, 255, 285, 250, 259, 269, 249, 265, 246, 263, 252, 252, 255, 264, 254, 265, 258, 233, 250, 289, 262, 261, 240, 257, 259, 252, 255, 238, 271, 263, 239, 235, 233, 283, 249, 266, 234, 252, 281, 262, 259, 240, 265, 217, 247, 246, 250, 278, 253, 259, 267, 230, 265, 250, 256, 230, 244, 252, 256, 230, 235, 271, 277, 251, 274, 272, 250, 268, 232, 239, 262, 244, 264, 250, 256, 242, 275, 245, 262, 264, 252, 269, 271, 236, 279, 260, 238, 243, 248, 243, 285, 266] payload_rows_per_dst=[5552, 5420, 5462, 5406, 5386, 5408, 5444, 5502] dup_rows_per_dst=[2758, 2732, 2858, 2778, 2712, 2618, 2730, 2770] combine_rows_per_src=[5552, 5420, 5462, 5406, 5386, 5408, 5444, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=228648 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157375784 ingress_bytes=213770240 link_bytes=213770240 token_bytes=314522920 weight_movement_bytes=56623104 total_nvlink_bytes=371146024 weight_to_token_ratio=0.180029 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048241258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229168 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156303152 ingress_bytes=193822720 link_bytes=193822720 token_bytes=312377136 weight_movement_bytes=37748736 total_nvlink_bytes=350125872 weight_to_token_ratio=0.120843 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857113936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=228936 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999240 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769544 weight_movement_bytes=18874368 total_nvlink_bytes=332643912 weight_to_token_ratio=0.060154 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229136 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157015824 ingress_bytes=232284160 link_bytes=232284160 token_bytes=313802512 weight_movement_bytes=75497472 total_nvlink_bytes=389299984 weight_to_token_ratio=0.240589 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225632530e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38434 dup_rows=21686 dup_rows_remote=19118 dedup_ratio=0.330902 router_tokens_per_expert=[237, 262, 254, 245, 257, 263, 246, 285, 279, 289, 258, 267, 282, 263, 255, 246, 249, 278, 250, 285, 241, 258, 256, 264, 265, 238, 250, 256, 262, 294, 248, 276, 236, 253, 247, 214, 252, 251, 277, 258, 249, 234, 279, 249, 215, 260, 260, 258, 264, 252, 237, 280, 284, 263, 260, 252, 280, 272, 250, 244, 273, 241, 257, 248, 235, 263, 253, 254, 265, 233, 237, 236, 242, 270, 265, 259, 256, 249, 243, 268, 287, 235, 229, 250, 247, 252, 257, 285, 246, 243, 257, 250, 242, 247, 225, 241, 240, 264, 237, 282, 268, 256, 246, 220, 260, 259, 263, 278, 238, 265, 270, 264, 260, 262, 241, 262, 252, 267, 260, 271, 253, 268, 266, 255, 235, 281, 242, 255] payload_rows_per_dst=[5554, 5512, 5416, 5580, 5450, 5324, 5522, 5492] dup_rows_per_dst=[2822, 2828, 2568, 2734, 2606, 2662, 2698, 2768] combine_rows_per_src=[5554, 5512, 5416, 5580, 5450, 5324, 5522, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229408 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156590112 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950816 weight_movement_bytes=37748736 total_nvlink_bytes=350699552 weight_to_token_ratio=0.120622 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38232 dup_rows=21884 dup_rows_remote=19180 dedup_ratio=0.333923 router_tokens_per_expert=[270, 254, 244, 225, 261, 260, 226, 235, 240, 249, 247, 266, 268, 256, 239, 268, 278, 252, 271, 247, 248, 255, 273, 253, 258, 242, 290, 277, 260, 280, 231, 259, 227, 255, 221, 258, 272, 230, 271, 246, 283, 244, 277, 241, 250, 250, 242, 287, 229, 264, 247, 253, 260, 261, 282, 244, 266, 253, 275, 272, 233, 295, 266, 249, 258, 254, 247, 248, 261, 259, 240, 278, 239, 239, 253, 257, 271, 248, 239, 271, 248, 254, 250, 254, 258, 245, 254, 253, 246, 235, 239, 273, 242, 285, 233, 257, 271, 254, 288, 235, 246, 245, 260, 250, 254, 254, 281, 251, 247, 229, 240, 260, 259, 252, 253, 272, 268, 253, 283, 270, 284, 248, 256, 287, 264, 252, 264, 265] payload_rows_per_dst=[5384, 5556, 5426, 5506, 5420, 5394, 5420, 5546] dup_rows_per_dst=[2632, 2792, 2682, 2792, 2704, 2658, 2710, 2914] combine_rows_per_src=[5384, 5556, 5426, 5506, 5420, 5394, 5420, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38020 dup_rows=21904 dup_rows_remote=19016 dedup_ratio=0.334229 router_tokens_per_expert=[239, 242, 246, 270, 261, 257, 255, 239, 252, 263, 231, 283, 280, 288, 273, 260, 245, 262, 245, 272, 229, 248, 270, 259, 269, 253, 274, 241, 286, 242, 245, 246, 228, 281, 250, 252, 267, 278, 247, 250, 236, 265, 276, 247, 269, 271, 271, 271, 271, 244, 258, 252, 235, 244, 245, 241, 249, 250, 269, 249, 243, 252, 233, 241, 257, 263, 255, 248, 279, 262, 254, 250, 268, 266, 227, 260, 262, 277, 235, 240, 247, 240, 215, 290, 248, 242, 237, 277, 236, 231, 253, 290, 268, 263, 267, 264, 273, 275, 227, 243, 253, 252, 264, 247, 279, 251, 244, 256, 275, 238, 246, 248, 294, 264, 269, 259, 264, 253, 273, 252, 239, 258, 241, 265, 248, 270, 250, 267] payload_rows_per_dst=[5346, 5558, 5330, 5356, 5472, 5436, 5522, 5612] dup_rows_per_dst=[2516, 3030, 2680, 2582, 2740, 2708, 2760, 2888] combine_rows_per_src=[5346, 5558, 5330, 5356, 5472, 5436, 5522, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157425664 dispatch_meta_bytes=230208 combine_payload_bytes=157425664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157655872 ingress_bytes=176300032 link_bytes=176300032 token_bytes=315081536 weight_movement_bytes=18874368 total_nvlink_bytes=333955904 weight_to_token_ratio=0.059903 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689220162e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229648 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827920 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426192 weight_movement_bytes=18874368 total_nvlink_bytes=332300560 weight_to_token_ratio=0.060219 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=228144 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155958064 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311687984 weight_movement_bytes=18874368 total_nvlink_bytes=330562352 weight_to_token_ratio=0.060555 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38316 dup_rows=21812 dup_rows_remote=19062 dedup_ratio=0.332825 router_tokens_per_expert=[266, 249, 276, 243, 243, 242, 238, 242, 236, 231, 246, 285, 267, 253, 235, 316, 231, 254, 268, 265, 248, 257, 222, 244, 257, 247, 245, 247, 261, 244, 247, 263, 270, 281, 246, 250, 269, 248, 306, 247, 271, 252, 256, 249, 262, 269, 258, 245, 290, 248, 277, 267, 270, 237, 265, 263, 272, 255, 232, 247, 256, 230, 271, 287, 264, 251, 238, 214, 259, 255, 265, 267, 260, 243, 251, 263, 251, 279, 270, 256, 264, 257, 252, 264, 259, 251, 257, 239, 233, 250, 252, 263, 265, 254, 244, 281, 244, 240, 279, 224, 289, 269, 273, 271, 241, 278, 251, 254, 251, 265, 278, 215, 248, 241, 268, 246, 252, 271, 252, 236, 228, 268, 262, 257, 259, 263, 255, 255] payload_rows_per_dst=[5408, 5570, 5552, 5520, 5592, 5244, 5304, 5534] dup_rows_per_dst=[2750, 2908, 2848, 2814, 2894, 2448, 2574, 2576] combine_rows_per_src=[5408, 5570, 5552, 5520, 5592, 5244, 5304, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38326 dup_rows=21750 dup_rows_remote=19124 dedup_ratio=0.331879 router_tokens_per_expert=[228, 250, 269, 276, 230, 254, 256, 242, 250, 263, 255, 235, 242, 283, 265, 262, 270, 250, 214, 247, 222, 261, 264, 245, 240, 272, 262, 259, 241, 251, 279, 266, 246, 243, 271, 289, 258, 234, 285, 251, 232, 261, 262, 275, 234, 264, 253, 255, 264, 245, 227, 237, 258, 288, 261, 279, 250, 250, 247, 276, 249, 233, 234, 213, 261, 246, 278, 257, 276, 258, 238, 254, 247, 269, 270, 245, 272, 260, 263, 253, 230, 264, 270, 272, 262, 269, 266, 271, 264, 263, 259, 240, 281, 267, 259, 250, 247, 266, 244, 261, 249, 256, 225, 236, 259, 237, 262, 248, 249, 239, 279, 260, 263, 267, 277, 272, 246, 277, 249, 243, 274, 241, 260, 272, 287, 254, 237, 271] payload_rows_per_dst=[5424, 5460, 5448, 5412, 5502, 5594, 5426, 5520] dup_rows_per_dst=[2696, 2626, 2778, 2610, 2792, 2780, 2608, 2860] combine_rows_per_src=[5424, 5460, 5448, 5412, 5502, 5594, 5426, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38146 dup_rows=21876 dup_rows_remote=19102 dedup_ratio=0.333801 router_tokens_per_expert=[233, 265, 255, 263, 247, 273, 255, 257, 236, 259, 270, 261, 250, 246, 258, 261, 257, 265, 260, 256, 248, 251, 251, 244, 258, 282, 260, 225, 246, 249, 271, 244, 259, 250, 247, 271, 238, 263, 260, 282, 270, 248, 270, 220, 268, 264, 239, 263, 290, 259, 242, 264, 276, 258, 250, 274, 254, 244, 268, 257, 278, 267, 232, 247, 257, 220, 272, 243, 248, 233, 251, 227, 289, 279, 230, 267, 256, 254, 233, 266, 250, 251, 270, 267, 262, 244, 263, 266, 272, 271, 255, 249, 266, 250, 252, 256, 265, 247, 250, 255, 243, 267, 264, 255, 273, 225, 269, 273, 277, 246, 240, 289, 249, 287, 225, 232, 269, 266, 273, 255, 235, 248, 231, 247, 272, 244, 252, 248] payload_rows_per_dst=[5478, 5448, 5462, 5534, 5348, 5514, 5462, 5414] dup_rows_per_dst=[2700, 2686, 2762, 2786, 2702, 2774, 2814, 2652] combine_rows_per_src=[5478, 5448, 5462, 5534, 5348, 5514, 5462, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38318 dup_rows=21748 dup_rows_remote=19086 dedup_ratio=0.331848 router_tokens_per_expert=[235, 276, 251, 270, 299, 264, 275, 251, 226, 258, 252, 229, 252, 253, 255, 270, 248, 236, 248, 274, 236, 285, 256, 243, 253, 245, 268, 261, 263, 274, 260, 226, 254, 262, 250, 251, 260, 255, 266, 216, 257, 236, 263, 244, 279, 267, 269, 281, 259, 267, 254, 249, 242, 264, 272, 276, 253, 245, 265, 252, 283, 260, 244, 253, 276, 244, 227, 267, 256, 286, 280, 279, 240, 239, 250, 287, 236, 239, 253, 249, 282, 273, 243, 271, 256, 242, 213, 242, 229, 236, 260, 273, 271, 271, 256, 228, 241, 272, 265, 251, 246, 267, 263, 259, 246, 260, 236, 242, 260, 262, 242, 254, 251, 269, 249, 236, 264, 274, 248, 239, 261, 281, 275, 270, 264, 254, 233, 240] payload_rows_per_dst=[5484, 5456, 5566, 5450, 5440, 5504, 5470, 5418] dup_rows_per_dst=[2748, 2696, 2654, 2826, 2776, 2588, 2662, 2798] combine_rows_per_src=[5484, 5456, 5566, 5450, 5440, 5504, 5470, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38242 dup_rows=21828 dup_rows_remote=19054 dedup_ratio=0.333069 router_tokens_per_expert=[275, 262, 258, 243, 256, 264, 256, 261, 258, 253, 276, 229, 238, 230, 266, 286, 237, 241, 251, 247, 252, 273, 260, 248, 244, 245, 264, 243, 267, 270, 246, 259, 253, 245, 257, 246, 282, 250, 267, 248, 266, 251, 249, 241, 251, 253, 268, 257, 253, 240, 253, 238, 269, 251, 259, 270, 281, 269, 281, 243, 281, 248, 224, 260, 231, 280, 249, 262, 252, 255, 234, 270, 260, 263, 249, 265, 237, 250, 252, 244, 227, 279, 248, 279, 252, 235, 272, 264, 273, 272, 218, 273, 284, 235, 232, 228, 260, 248, 262, 257, 255, 259, 263, 273, 247, 239, 269, 255, 248, 278, 281, 256, 265, 260, 248, 262, 244, 263, 246, 271, 277, 272, 255, 234, 272, 247, 242, 274] payload_rows_per_dst=[5468, 5430, 5488, 5466, 5446, 5406, 5558, 5446] dup_rows_per_dst=[2754, 2664, 2680, 2774, 2660, 2736, 2742, 2818] combine_rows_per_src=[5468, 5430, 5488, 5466, 5446, 5406, 5558, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229512 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171848 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114184 weight_movement_bytes=18874368 total_nvlink_bytes=332988552 weight_to_token_ratio=0.060088 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229800 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213096 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196392 weight_movement_bytes=18874368 total_nvlink_bytes=333070760 weight_to_token_ratio=0.060072 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228992 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156475008 ingress_bytes=193994752 link_bytes=193994752 token_bytes=312721024 weight_movement_bytes=37748736 total_nvlink_bytes=350469760 weight_to_token_ratio=0.120711 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858762262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38474 dup_rows=21638 dup_rows_remote=18934 dedup_ratio=0.330170 router_tokens_per_expert=[285, 276, 270, 278, 239, 245, 247, 266, 243, 275, 260, 276, 243, 261, 224, 263, 276, 258, 272, 267, 265, 242, 259, 255, 245, 254, 271, 244, 268, 226, 237, 255, 266, 254, 268, 264, 245, 263, 233, 277, 237, 233, 269, 244, 264, 254, 240, 253, 282, 242, 260, 256, 277, 263, 262, 276, 263, 240, 258, 261, 259, 263, 264, 238, 220, 281, 266, 283, 277, 249, 244, 245, 222, 267, 220, 234, 268, 281, 256, 259, 239, 243, 235, 231, 297, 221, 266, 244, 252, 278, 254, 250, 260, 248, 272, 253, 250, 281, 248, 261, 282, 270, 249, 245, 276, 239, 238, 246, 237, 269, 266, 272, 229, 280, 256, 265, 259, 254, 237, 285, 221, 235, 246, 273, 248, 246, 248, 269] payload_rows_per_dst=[5504, 5478, 5424, 5606, 5420, 5466, 5526, 5474] dup_rows_per_dst=[2798, 2710, 2704, 2722, 2724, 2620, 2732, 2628] combine_rows_per_src=[5504, 5478, 5424, 5606, 5420, 5466, 5526, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229616 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157180144 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130672 weight_movement_bytes=37748736 total_nvlink_bytes=351879408 weight_to_token_ratio=0.120169 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229184 combine_payload_bytes=156639232 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156868416 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313507648 weight_movement_bytes=37748736 total_nvlink_bytes=351256384 weight_to_token_ratio=0.120408 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38228 dup_rows=21866 dup_rows_remote=19084 dedup_ratio=0.333649 router_tokens_per_expert=[257, 251, 261, 260, 265, 255, 251, 288, 258, 260, 252, 274, 255, 249, 258, 266, 266, 248, 272, 235, 265, 245, 228, 256, 253, 256, 237, 260, 250, 269, 269, 249, 257, 254, 258, 256, 252, 244, 253, 269, 249, 259, 263, 267, 273, 270, 254, 245, 258, 284, 232, 244, 256, 260, 287, 259, 246, 260, 259, 255, 271, 273, 229, 235, 274, 249, 274, 264, 275, 237, 239, 262, 232, 260, 281, 246, 250, 259, 251, 259, 243, 252, 278, 226, 258, 238, 230, 260, 275, 262, 260, 263, 240, 279, 280, 268, 227, 278, 255, 258, 252, 260, 236, 264, 244, 253, 254, 271, 232, 260, 259, 236, 260, 248, 251, 238, 245, 243, 260, 277, 238, 257, 246, 286, 237, 244, 269, 257] payload_rows_per_dst=[5508, 5424, 5464, 5528, 5442, 5482, 5398, 5424] dup_rows_per_dst=[2812, 2692, 2782, 2688, 2782, 2742, 2680, 2688] combine_rows_per_src=[5508, 5424, 5464, 5528, 5442, 5482, 5398, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38122 dup_rows=21912 dup_rows_remote=19084 dedup_ratio=0.334351 router_tokens_per_expert=[287, 280, 229, 228, 233, 244, 252, 230, 285, 255, 243, 263, 290, 241, 262, 277, 249, 241, 257, 277, 258, 261, 271, 257, 265, 244, 246, 248, 236, 221, 252, 287, 252, 268, 236, 259, 263, 254, 278, 274, 248, 255, 243, 220, 255, 257, 217, 245, 247, 257, 223, 251, 267, 255, 245, 265, 261, 279, 257, 273, 255, 254, 267, 247, 252, 247, 255, 256, 263, 277, 256, 250, 273, 271, 258, 277, 260, 243, 245, 273, 255, 265, 247, 238, 242, 256, 251, 250, 223, 240, 249, 254, 266, 244, 273, 261, 252, 287, 267, 245, 266, 245, 280, 247, 265, 268, 268, 225, 280, 244, 232, 272, 260, 234, 272, 292, 246, 253, 286, 246, 241, 245, 260, 259, 254, 279, 260, 272] payload_rows_per_dst=[5296, 5396, 5416, 5370, 5558, 5470, 5616, 5502] dup_rows_per_dst=[2674, 2744, 2632, 2586, 2754, 2786, 2908, 2828] combine_rows_per_src=[5296, 5396, 5416, 5370, 5558, 5470, 5616, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157589504 dispatch_meta_bytes=229632 combine_payload_bytes=157589504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157819136 ingress_bytes=214212608 link_bytes=214212608 token_bytes=315408640 weight_movement_bytes=56623104 total_nvlink_bytes=372031744 weight_to_token_ratio=0.179523 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.052479811e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229248 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156811136 ingress_bytes=194330624 link_bytes=194330624 token_bytes=313393024 weight_movement_bytes=37748736 total_nvlink_bytes=351141760 weight_to_token_ratio=0.120452 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861980423e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228824 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156376536 ingress_bytes=212770816 link_bytes=212770816 token_bytes=312524248 weight_movement_bytes=56623104 total_nvlink_bytes=369147352 weight_to_token_ratio=0.181180 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038665269e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38106 dup_rows=21908 dup_rows_remote=19046 dedup_ratio=0.334290 router_tokens_per_expert=[249, 246, 259, 270, 262, 248, 241, 246, 263, 255, 270, 247, 244, 244, 253, 254, 245, 244, 274, 260, 230, 237, 247, 267, 266, 284, 264, 248, 255, 235, 264, 252, 257, 272, 267, 236, 272, 270, 250, 269, 266, 248, 252, 238, 232, 284, 252, 247, 241, 267, 267, 270, 256, 255, 265, 245, 248, 273, 242, 244, 265, 226, 246, 258, 256, 237, 277, 245, 267, 242, 279, 257, 245, 230, 254, 259, 229, 239, 264, 265, 238, 265, 255, 301, 274, 249, 258, 273, 241, 268, 260, 277, 285, 276, 266, 242, 250, 269, 246, 274, 259, 261, 277, 279, 237, 271, 257, 230, 268, 259, 264, 284, 242, 273, 271, 280, 233, 237, 237, 249, 249, 261, 247, 257, 243, 233, 253, 242] payload_rows_per_dst=[5522, 5290, 5546, 5362, 5426, 5520, 5538, 5424] dup_rows_per_dst=[2862, 2582, 2808, 2764, 2686, 2784, 2832, 2590] combine_rows_per_src=[5522, 5290, 5546, 5362, 5426, 5520, 5538, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38256 dup_rows=21720 dup_rows_remote=19026 dedup_ratio=0.331421 router_tokens_per_expert=[254, 253, 221, 223, 256, 282, 258, 271, 234, 257, 262, 235, 267, 247, 242, 269, 282, 234, 256, 288, 244, 244, 275, 255, 244, 248, 253, 266, 293, 261, 237, 247, 266, 223, 238, 263, 266, 240, 238, 273, 254, 274, 251, 251, 275, 262, 279, 252, 262, 235, 244, 283, 274, 260, 253, 269, 230, 272, 252, 245, 246, 280, 271, 278, 218, 232, 253, 275, 265, 234, 228, 258, 250, 253, 276, 265, 269, 273, 253, 254, 258, 269, 233, 267, 267, 262, 245, 275, 249, 266, 247, 245, 268, 258, 284, 282, 282, 251, 257, 270, 237, 264, 243, 254, 249, 252, 253, 255, 268, 251, 262, 245, 234, 250, 252, 249, 286, 262, 250, 250, 235, 253, 243, 265, 244, 245, 246, 263] payload_rows_per_dst=[5396, 5560, 5496, 5460, 5472, 5586, 5494, 5352] dup_rows_per_dst=[2666, 2694, 2714, 2848, 2640, 2764, 2692, 2702] combine_rows_per_src=[5396, 5560, 5496, 5460, 5472, 5586, 5494, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38386 dup_rows=21724 dup_rows_remote=19104 dedup_ratio=0.331482 router_tokens_per_expert=[257, 236, 249, 271, 268, 255, 283, 294, 254, 255, 246, 251, 264, 225, 237, 267, 284, 228, 247, 276, 278, 240, 257, 238, 250, 249, 265, 247, 261, 264, 235, 269, 258, 261, 245, 253, 232, 242, 258, 253, 228, 244, 255, 232, 255, 234, 261, 250, 273, 268, 254, 247, 248, 231, 258, 245, 279, 252, 222, 255, 233, 274, 258, 272, 270, 268, 248, 258, 259, 221, 260, 257, 259, 262, 268, 261, 261, 260, 267, 266, 265, 247, 270, 239, 265, 286, 254, 239, 238, 237, 250, 256, 234, 263, 244, 236, 275, 242, 251, 234, 271, 246, 262, 264, 257, 265, 237, 279, 237, 266, 234, 269, 288, 247, 232, 260, 305, 267, 267, 269, 229, 253, 271, 272, 269, 278, 290, 284] payload_rows_per_dst=[5512, 5502, 5324, 5424, 5514, 5426, 5502, 5608] dup_rows_per_dst=[2712, 2674, 2598, 2714, 2776, 2620, 2676, 2954] combine_rows_per_src=[5512, 5502, 5324, 5424, 5514, 5426, 5502, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38220 dup_rows=21832 dup_rows_remote=19060 dedup_ratio=0.333130 router_tokens_per_expert=[236, 257, 253, 247, 281, 250, 241, 253, 260, 232, 252, 257, 263, 256, 236, 273, 268, 258, 278, 261, 236, 283, 276, 255, 262, 263, 249, 254, 233, 252, 262, 245, 240, 283, 258, 245, 246, 246, 247, 288, 265, 266, 264, 239, 257, 257, 264, 262, 269, 269, 247, 259, 265, 228, 261, 251, 271, 253, 272, 266, 257, 246, 262, 268, 246, 274, 248, 249, 248, 260, 237, 243, 263, 253, 261, 230, 280, 258, 244, 253, 249, 253, 242, 255, 276, 273, 269, 248, 240, 258, 251, 258, 253, 242, 236, 275, 238, 252, 246, 259, 270, 271, 232, 228, 265, 254, 282, 264, 261, 241, 275, 290, 250, 255, 258, 255, 266, 242, 282, 266, 241, 237, 253, 272, 257, 238, 237, 253] payload_rows_per_dst=[5394, 5506, 5508, 5482, 5394, 5474, 5484, 5462] dup_rows_per_dst=[2700, 2764, 2746, 2806, 2700, 2682, 2772, 2662] combine_rows_per_src=[5394, 5506, 5508, 5482, 5394, 5474, 5484, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38194 dup_rows=21866 dup_rows_remote=19064 dedup_ratio=0.333649 router_tokens_per_expert=[253, 257, 263, 272, 240, 265, 243, 253, 260, 274, 241, 257, 282, 267, 260, 260, 237, 248, 268, 221, 258, 246, 225, 253, 237, 245, 270, 270, 265, 267, 242, 257, 247, 253, 234, 255, 262, 271, 243, 236, 246, 254, 262, 269, 262, 210, 284, 274, 247, 262, 266, 261, 261, 257, 276, 274, 261, 255, 258, 265, 245, 218, 267, 266, 233, 258, 264, 274, 240, 279, 270, 259, 256, 261, 268, 245, 250, 261, 237, 228, 253, 273, 287, 238, 281, 259, 257, 256, 235, 262, 272, 247, 276, 244, 239, 276, 235, 232, 261, 252, 231, 242, 295, 261, 248, 265, 218, 248, 263, 271, 255, 262, 262, 282, 238, 293, 272, 242, 273, 262, 251, 255, 263, 244, 237, 247, 260, 253] payload_rows_per_dst=[5524, 5400, 5452, 5476, 5430, 5484, 5406, 5498] dup_rows_per_dst=[2770, 2618, 2672, 2802, 2736, 2826, 2672, 2770] combine_rows_per_src=[5524, 5400, 5452, 5476, 5430, 5484, 5406, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229128 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925704 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622280 weight_movement_bytes=18874368 total_nvlink_bytes=332496648 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228608 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156310784 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312392960 weight_movement_bytes=18874368 total_nvlink_bytes=331267328 weight_to_token_ratio=0.060419 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229960 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157459016 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314688072 weight_movement_bytes=18874368 total_nvlink_bytes=333562440 weight_to_token_ratio=0.059978 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229120 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156778240 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313327360 weight_movement_bytes=37748736 total_nvlink_bytes=351076096 weight_to_token_ratio=0.120477 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38204 dup_rows=21824 dup_rows_remote=19030 dedup_ratio=0.333008 router_tokens_per_expert=[269, 279, 246, 249, 275, 260, 248, 236, 259, 269, 250, 263, 264, 245, 243, 258, 256, 275, 271, 244, 257, 235, 265, 250, 233, 261, 270, 255, 279, 256, 267, 253, 252, 272, 245, 272, 255, 255, 276, 262, 215, 256, 266, 259, 268, 271, 261, 266, 234, 250, 269, 239, 248, 245, 263, 271, 282, 239, 245, 278, 275, 239, 253, 258, 264, 262, 251, 233, 239, 253, 257, 284, 255, 244, 280, 258, 262, 252, 250, 277, 264, 283, 247, 276, 260, 238, 261, 247, 251, 238, 269, 263, 243, 238, 254, 245, 271, 263, 217, 272, 253, 249, 249, 248, 238, 260, 275, 245, 258, 272, 263, 260, 256, 249, 245, 223, 247, 229, 256, 260, 279, 241, 235, 244, 282, 251, 241, 260] payload_rows_per_dst=[5462, 5492, 5508, 5470, 5464, 5446, 5470, 5400] dup_rows_per_dst=[2764, 2762, 2794, 2706, 2778, 2708, 2716, 2596] combine_rows_per_src=[5462, 5492, 5508, 5470, 5464, 5446, 5470, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38234 dup_rows=21752 dup_rows_remote=18828 dedup_ratio=0.331909 router_tokens_per_expert=[257, 251, 238, 254, 247, 254, 246, 261, 280, 269, 261, 253, 249, 247, 254, 289, 265, 274, 251, 253, 246, 234, 237, 240, 252, 258, 256, 251, 233, 259, 244, 288, 234, 269, 250, 279, 248, 267, 257, 274, 243, 262, 246, 241, 252, 273, 249, 247, 257, 262, 246, 254, 256, 227, 259, 234, 280, 257, 265, 253, 263, 264, 269, 270, 279, 251, 275, 266, 251, 286, 252, 258, 279, 240, 277, 256, 272, 254, 251, 290, 270, 243, 218, 268, 265, 274, 277, 265, 250, 255, 256, 223, 258, 256, 243, 260, 237, 257, 229, 259, 255, 283, 269, 266, 242, 251, 253, 259, 234, 255, 255, 242, 260, 224, 245, 259, 259, 249, 252, 264, 256, 271, 241, 249, 239, 263, 272, 243] payload_rows_per_dst=[5508, 5410, 5508, 5502, 5550, 5438, 5430, 5438] dup_rows_per_dst=[2712, 2672, 2674, 2730, 2924, 2724, 2662, 2654] combine_rows_per_src=[5508, 5410, 5508, 5502, 5550, 5438, 5430, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229032 combine_payload_bytes=156442624 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156671656 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114280 weight_movement_bytes=37748736 total_nvlink_bytes=350863016 weight_to_token_ratio=0.120559 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38356 dup_rows=21628 dup_rows_remote=18922 dedup_ratio=0.330017 router_tokens_per_expert=[286, 274, 276, 251, 246, 257, 259, 251, 271, 274, 254, 279, 231, 236, 254, 261, 258, 248, 267, 243, 247, 284, 268, 261, 266, 242, 248, 267, 260, 279, 265, 305, 254, 253, 267, 262, 267, 227, 264, 273, 268, 244, 252, 227, 241, 242, 259, 219, 249, 263, 247, 244, 275, 227, 246, 255, 258, 260, 261, 231, 271, 259, 245, 254, 247, 245, 248, 270, 249, 238, 253, 232, 239, 263, 240, 258, 265, 287, 268, 247, 267, 265, 252, 246, 269, 237, 269, 254, 250, 236, 236, 253, 260, 250, 256, 267, 259, 292, 269, 244, 267, 230, 270, 256, 247, 252, 272, 275, 222, 276, 209, 251, 256, 252, 275, 252, 260, 271, 281, 248, 265, 214, 246, 276, 260, 265, 240, 268] payload_rows_per_dst=[5356, 5546, 5478, 5466, 5306, 5586, 5618, 5552] dup_rows_per_dst=[2586, 2788, 2560, 2706, 2400, 2940, 2942, 2706] combine_rows_per_src=[5356, 5546, 5478, 5466, 5306, 5586, 5618, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228936 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156712520 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196104 weight_movement_bytes=18874368 total_nvlink_bytes=332070472 weight_to_token_ratio=0.060264 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228248 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156834712 ingress_bytes=213229568 link_bytes=213229568 token_bytes=313441176 weight_movement_bytes=56623104 total_nvlink_bytes=370064280 weight_to_token_ratio=0.180650 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043060805e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229112 combine_payload_bytes=157106176 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157335288 ingress_bytes=213729280 link_bytes=213729280 token_bytes=314441464 weight_movement_bytes=56623104 total_nvlink_bytes=371064568 weight_to_token_ratio=0.180075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047848800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38080 dup_rows=21982 dup_rows_remote=19152 dedup_ratio=0.335419 router_tokens_per_expert=[214, 250, 250, 276, 277, 241, 289, 258, 240, 276, 241, 285, 263, 275, 257, 238, 254, 238, 227, 263, 264, 271, 243, 257, 273, 262, 271, 259, 268, 261, 278, 263, 259, 232, 275, 272, 227, 247, 252, 259, 254, 266, 255, 256, 263, 266, 265, 249, 246, 235, 280, 259, 252, 281, 215, 235, 254, 273, 252, 275, 250, 253, 246, 274, 272, 269, 245, 246, 272, 225, 270, 260, 252, 267, 250, 250, 288, 255, 270, 252, 247, 230, 258, 281, 265, 250, 263, 252, 259, 253, 260, 254, 225, 261, 234, 264, 250, 267, 241, 220, 239, 213, 259, 245, 242, 272, 267, 251, 253, 247, 249, 226, 276, 258, 241, 251, 253, 262, 258, 261, 278, 273, 274, 264, 284, 251, 234, 251] payload_rows_per_dst=[5474, 5474, 5464, 5412, 5464, 5388, 5354, 5524] dup_rows_per_dst=[2786, 2830, 2730, 2748, 2822, 2724, 2528, 2814] combine_rows_per_src=[5474, 5474, 5464, 5412, 5464, 5388, 5354, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38256 dup_rows=21750 dup_rows_remote=19016 dedup_ratio=0.331879 router_tokens_per_expert=[260, 247, 250, 291, 239, 248, 238, 223, 252, 248, 234, 235, 262, 251, 238, 262, 247, 263, 272, 239, 266, 244, 280, 262, 245, 264, 293, 284, 226, 278, 258, 269, 254, 230, 261, 259, 251, 271, 235, 247, 273, 248, 234, 247, 244, 285, 234, 258, 230, 267, 253, 271, 237, 245, 247, 256, 244, 267, 223, 281, 272, 262, 281, 259, 248, 250, 263, 261, 249, 256, 263, 255, 259, 274, 257, 263, 257, 262, 264, 261, 272, 259, 252, 230, 255, 262, 256, 242, 271, 267, 275, 265, 255, 245, 267, 259, 230, 294, 245, 269, 280, 252, 265, 266, 233, 237, 255, 227, 255, 267, 244, 260, 272, 230, 246, 259, 250, 269, 281, 235, 254, 300, 280, 257, 249, 285, 237, 217] payload_rows_per_dst=[5406, 5522, 5408, 5488, 5520, 5530, 5424, 5488] dup_rows_per_dst=[2550, 2858, 2654, 2702, 2764, 2734, 2734, 2754] combine_rows_per_src=[5406, 5522, 5408, 5488, 5520, 5530, 5424, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38046 dup_rows=21912 dup_rows_remote=19100 dedup_ratio=0.334351 router_tokens_per_expert=[267, 262, 262, 257, 277, 257, 247, 252, 282, 270, 282, 241, 250, 252, 261, 260, 249, 256, 252, 236, 251, 253, 262, 238, 250, 245, 261, 258, 245, 263, 278, 262, 211, 245, 275, 260, 259, 251, 247, 250, 233, 249, 251, 259, 251, 263, 254, 264, 234, 231, 235, 261, 279, 278, 262, 262, 245, 232, 249, 246, 249, 255, 274, 273, 256, 277, 262, 275, 244, 242, 224, 255, 248, 258, 251, 270, 255, 250, 295, 255, 291, 261, 268, 241, 249, 254, 251, 246, 259, 239, 217, 251, 264, 263, 244, 280, 257, 252, 275, 269, 275, 283, 272, 251, 261, 252, 278, 252, 272, 267, 284, 263, 244, 242, 253, 239, 253, 237, 254, 252, 227, 244, 253, 259, 251, 270, 230, 277] payload_rows_per_dst=[5578, 5418, 5342, 5450, 5656, 5418, 5464, 5298] dup_rows_per_dst=[2812, 2768, 2702, 2680, 3002, 2738, 2718, 2492] combine_rows_per_src=[5578, 5418, 5342, 5450, 5656, 5418, 5464, 5298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38328 dup_rows=21738 dup_rows_remote=19032 dedup_ratio=0.331696 router_tokens_per_expert=[269, 275, 269, 259, 276, 242, 235, 250, 241, 220, 236, 300, 228, 268, 267, 264, 287, 263, 256, 254, 241, 259, 238, 257, 246, 233, 257, 259, 249, 236, 286, 277, 270, 256, 283, 232, 245, 247, 228, 214, 281, 236, 253, 274, 249, 237, 286, 264, 248, 267, 269, 268, 266, 241, 301, 257, 253, 270, 230, 260, 266, 266, 284, 235, 283, 241, 268, 274, 219, 250, 258, 257, 269, 267, 257, 225, 246, 246, 256, 257, 269, 273, 256, 249, 242, 260, 246, 251, 274, 256, 260, 247, 263, 279, 249, 225, 253, 255, 268, 245, 264, 252, 247, 270, 240, 302, 234, 262, 256, 263, 262, 215, 226, 261, 236, 252, 263, 262, 251, 252, 277, 253, 274, 223, 271, 254, 246, 274] payload_rows_per_dst=[5468, 5492, 5444, 5506, 5480, 5472, 5470, 5466] dup_rows_per_dst=[2730, 2704, 2666, 2856, 2666, 2726, 2706, 2684] combine_rows_per_src=[5468, 5492, 5444, 5506, 5480, 5472, 5470, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38366 dup_rows=21632 dup_rows_remote=18978 dedup_ratio=0.330078 router_tokens_per_expert=[254, 229, 248, 272, 270, 250, 253, 252, 254, 272, 257, 264, 258, 260, 264, 253, 249, 250, 269, 228, 246, 252, 239, 235, 238, 273, 274, 251, 247, 257, 250, 262, 246, 232, 234, 265, 257, 230, 265, 258, 251, 253, 283, 255, 239, 261, 277, 256, 250, 275, 271, 252, 241, 267, 252, 263, 242, 222, 251, 266, 242, 264, 274, 264, 265, 254, 254, 227, 246, 271, 291, 264, 253, 245, 262, 288, 253, 261, 249, 261, 267, 247, 258, 267, 257, 269, 257, 257, 263, 258, 245, 262, 245, 263, 275, 250, 267, 267, 246, 276, 242, 260, 228, 245, 265, 256, 253, 264, 250, 251, 238, 257, 253, 262, 254, 256, 241, 247, 272, 276, 261, 273, 230, 245, 260, 264, 283, 254] payload_rows_per_dst=[5554, 5410, 5442, 5538, 5560, 5504, 5416, 5480] dup_rows_per_dst=[2666, 2630, 2682, 2654, 2728, 2776, 2714, 2782] combine_rows_per_src=[5554, 5410, 5442, 5538, 5560, 5504, 5416, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228928 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156204608 ingress_bytes=212598784 link_bytes=212598784 token_bytes=312180288 weight_movement_bytes=56623104 total_nvlink_bytes=368803392 weight_to_token_ratio=0.181379 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037016943e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229088 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925664 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622240 weight_movement_bytes=18874368 total_nvlink_bytes=332496608 weight_to_token_ratio=0.060182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43552 payload_rows_remote=38076 dup_rows=21984 dup_rows_remote=19140 dedup_ratio=0.335449 router_tokens_per_expert=[228, 284, 272, 266, 257, 244, 253, 245, 260, 264, 248, 244, 237, 269, 243, 282, 274, 236, 265, 244, 254, 237, 264, 253, 263, 234, 258, 240, 255, 268, 247, 274, 252, 241, 285, 274, 245, 257, 278, 254, 271, 252, 268, 264, 264, 244, 257, 254, 248, 273, 267, 266, 260, 247, 265, 239, 232, 258, 265, 250, 263, 259, 240, 252, 239, 251, 256, 236, 237, 274, 247, 256, 256, 280, 241, 254, 254, 253, 257, 274, 251, 269, 229, 253, 270, 244, 246, 244, 243, 282, 273, 244, 259, 253, 245, 293, 250, 243, 228, 255, 273, 275, 240, 272, 272, 246, 279, 251, 274, 260, 234, 292, 251, 228, 257, 245, 269, 249, 250, 242, 276, 247, 232, 255, 258, 277, 269, 250] payload_rows_per_dst=[5464, 5458, 5476, 5452, 5418, 5456, 5408, 5420] dup_rows_per_dst=[2728, 2674, 2844, 2716, 2712, 2740, 2880, 2690] combine_rows_per_src=[5464, 5458, 5476, 5452, 5418, 5456, 5408, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229440 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220928 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212416 weight_movement_bytes=18874368 total_nvlink_bytes=333086784 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=228584 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156065000 ingress_bytes=193585152 link_bytes=193585152 token_bytes=311901416 weight_movement_bytes=37748736 total_nvlink_bytes=349650152 weight_to_token_ratio=0.121028 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854837677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38128 dup_rows=21810 dup_rows_remote=18904 dedup_ratio=0.332794 router_tokens_per_expert=[294, 282, 248, 244, 238, 268, 268, 246, 266, 279, 242, 261, 244, 263, 254, 247, 262, 232, 263, 252, 258, 277, 251, 233, 244, 242, 290, 274, 230, 251, 272, 245, 253, 255, 269, 237, 242, 286, 241, 266, 253, 279, 264, 244, 229, 268, 246, 259, 245, 241, 255, 241, 246, 265, 257, 254, 242, 250, 286, 232, 245, 278, 259, 244, 263, 265, 297, 260, 264, 283, 263, 245, 272, 251, 258, 256, 281, 253, 281, 260, 220, 285, 248, 250, 254, 251, 230, 289, 237, 269, 234, 252, 270, 248, 240, 249, 256, 259, 255, 251, 272, 248, 214, 263, 276, 266, 260, 239, 255, 266, 281, 236, 235, 254, 240, 250, 260, 262, 283, 252, 256, 285, 245, 231, 257, 243, 213, 276] payload_rows_per_dst=[5484, 5454, 5456, 5442, 5598, 5418, 5502, 5372] dup_rows_per_dst=[2804, 2698, 2726, 2638, 2906, 2634, 2692, 2712] combine_rows_per_src=[5484, 5454, 5456, 5442, 5598, 5418, 5502, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229376 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157376512 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314523648 weight_movement_bytes=18874368 total_nvlink_bytes=333398016 weight_to_token_ratio=0.060009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38236 dup_rows=21732 dup_rows_remote=18950 dedup_ratio=0.331604 router_tokens_per_expert=[259, 230, 264, 258, 260, 300, 246, 270, 272, 253, 254, 271, 253, 291, 266, 236, 223, 270, 264, 236, 244, 231, 274, 252, 259, 262, 254, 248, 262, 247, 253, 234, 257, 246, 257, 244, 262, 274, 287, 264, 251, 237, 265, 250, 268, 253, 264, 241, 253, 249, 262, 279, 235, 296, 253, 256, 262, 277, 258, 251, 229, 223, 235, 238, 263, 267, 283, 228, 240, 244, 256, 254, 275, 242, 266, 241, 262, 263, 220, 251, 263, 267, 250, 266, 241, 237, 262, 238, 256, 276, 250, 255, 248, 248, 240, 262, 277, 273, 244, 243, 271, 267, 267, 261, 277, 268, 261, 265, 270, 274, 257, 253, 246, 259, 256, 232, 265, 265, 250, 270, 249, 281, 252, 227, 253, 261, 251, 237] payload_rows_per_dst=[5338, 5398, 5532, 5652, 5322, 5446, 5548, 5568] dup_rows_per_dst=[2544, 2628, 2708, 2944, 2546, 2672, 2908, 2782] combine_rows_per_src=[5338, 5398, 5532, 5652, 5322, 5446, 5548, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155959296 dispatch_meta_bytes=228864 combine_payload_bytes=155959296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156188160 ingress_bytes=174833664 link_bytes=174833664 token_bytes=312147456 weight_movement_bytes=18874368 total_nvlink_bytes=331021824 weight_to_token_ratio=0.060466 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675170145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=228128 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156400416 ingress_bytes=193921024 link_bytes=193921024 token_bytes=312572704 weight_movement_bytes=37748736 total_nvlink_bytes=350321440 weight_to_token_ratio=0.120768 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858055837e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=228744 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156843400 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313458056 weight_movement_bytes=18874368 total_nvlink_bytes=332332424 weight_to_token_ratio=0.060213 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38388 dup_rows=21728 dup_rows_remote=19054 dedup_ratio=0.331543 router_tokens_per_expert=[265, 250, 284, 227, 251, 239, 254, 244, 254, 263, 251, 267, 269, 253, 249, 247, 236, 276, 235, 266, 269, 238, 235, 254, 243, 270, 234, 247, 257, 265, 271, 251, 259, 277, 265, 259, 254, 259, 275, 222, 259, 282, 268, 262, 239, 272, 290, 265, 257, 237, 244, 257, 240, 280, 286, 245, 219, 247, 269, 270, 266, 251, 252, 272, 254, 249, 269, 255, 278, 258, 270, 274, 271, 282, 238, 244, 270, 249, 260, 274, 273, 233, 255, 258, 253, 246, 268, 248, 226, 216, 242, 232, 215, 247, 254, 250, 225, 256, 267, 285, 245, 294, 266, 257, 243, 279, 245, 217, 251, 296, 267, 255, 260, 233, 281, 269, 268, 252, 249, 275, 270, 225, 244, 250, 254, 272, 245, 249] payload_rows_per_dst=[5430, 5420, 5606, 5496, 5566, 5308, 5540, 5442] dup_rows_per_dst=[2704, 2674, 2808, 2688, 2824, 2524, 2756, 2750] combine_rows_per_src=[5430, 5420, 5606, 5496, 5566, 5308, 5540, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38138 dup_rows=21790 dup_rows_remote=19038 dedup_ratio=0.332489 router_tokens_per_expert=[261, 254, 264, 268, 257, 253, 282, 273, 258, 260, 259, 254, 241, 254, 263, 245, 242, 243, 251, 237, 263, 282, 239, 224, 254, 246, 264, 235, 269, 242, 260, 240, 237, 263, 248, 284, 242, 266, 263, 270, 261, 283, 242, 254, 248, 248, 263, 250, 247, 251, 269, 261, 234, 252, 259, 244, 251, 254, 261, 237, 291, 249, 265, 244, 245, 245, 255, 245, 258, 265, 237, 259, 262, 273, 252, 250, 249, 259, 244, 247, 255, 227, 249, 263, 231, 256, 238, 266, 273, 273, 267, 283, 277, 297, 252, 273, 283, 267, 278, 255, 257, 239, 265, 251, 254, 239, 237, 232, 270, 274, 253, 250, 262, 290, 247, 254, 261, 265, 200, 241, 279, 241, 295, 270, 228, 258, 222, 298] payload_rows_per_dst=[5524, 5370, 5470, 5416, 5474, 5608, 5456, 5428] dup_rows_per_dst=[2768, 2612, 2774, 2722, 2616, 2752, 2752, 2794] combine_rows_per_src=[5524, 5370, 5470, 5416, 5474, 5608, 5456, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38244 dup_rows=21776 dup_rows_remote=19130 dedup_ratio=0.332275 router_tokens_per_expert=[257, 235, 247, 229, 255, 243, 240, 277, 249, 267, 251, 273, 250, 288, 264, 257, 264, 233, 271, 276, 241, 260, 251, 251, 271, 259, 262, 251, 252, 258, 239, 242, 263, 284, 267, 249, 280, 263, 251, 256, 253, 237, 266, 260, 265, 231, 235, 273, 244, 273, 270, 262, 242, 266, 252, 289, 234, 266, 258, 239, 253, 265, 281, 246, 267, 242, 214, 264, 300, 245, 273, 229, 289, 248, 261, 251, 229, 253, 259, 282, 249, 265, 238, 268, 244, 266, 238, 264, 269, 243, 257, 244, 249, 254, 266, 234, 261, 256, 234, 256, 257, 241, 264, 253, 276, 288, 240, 251, 254, 269, 239, 242, 254, 250, 234, 242, 276, 230, 262, 278, 262, 270, 262, 268, 262, 252, 245, 250] payload_rows_per_dst=[5496, 5400, 5514, 5514, 5508, 5402, 5516, 5410] dup_rows_per_dst=[2668, 2762, 2752, 2766, 2704, 2694, 2646, 2784] combine_rows_per_src=[5496, 5400, 5514, 5514, 5508, 5402, 5516, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38106 dup_rows=21912 dup_rows_remote=19096 dedup_ratio=0.334351 router_tokens_per_expert=[252, 248, 247, 237, 245, 252, 255, 287, 254, 285, 250, 258, 258, 247, 261, 259, 223, 250, 291, 225, 278, 260, 242, 254, 273, 263, 238, 254, 228, 255, 261, 272, 254, 258, 250, 245, 250, 259, 275, 284, 293, 238, 265, 280, 275, 260, 235, 263, 279, 256, 244, 247, 241, 255, 227, 278, 261, 276, 253, 232, 231, 252, 249, 274, 248, 287, 278, 263, 256, 272, 288, 276, 298, 269, 251, 264, 266, 237, 215, 284, 268, 262, 257, 250, 247, 247, 251, 231, 229, 247, 253, 248, 268, 268, 283, 259, 236, 250, 246, 259, 272, 255, 265, 239, 271, 246, 263, 246, 289, 238, 245, 246, 241, 261, 266, 241, 253, 264, 254, 241, 251, 260, 263, 224, 232, 250, 242, 238] payload_rows_per_dst=[5518, 5482, 5514, 5406, 5514, 5396, 5404, 5390] dup_rows_per_dst=[2816, 2652, 2910, 2704, 2846, 2684, 2728, 2572] combine_rows_per_src=[5518, 5482, 5514, 5406, 5514, 5396, 5404, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38476 dup_rows=21670 dup_rows_remote=19078 dedup_ratio=0.330658 router_tokens_per_expert=[241, 271, 242, 266, 287, 268, 250, 265, 266, 245, 239, 257, 234, 240, 267, 238, 279, 263, 259, 274, 264, 264, 237, 245, 256, 248, 254, 269, 243, 244, 276, 251, 266, 250, 261, 247, 256, 235, 248, 249, 238, 268, 258, 255, 248, 267, 262, 263, 238, 239, 254, 249, 275, 271, 251, 244, 239, 247, 234, 230, 266, 236, 269, 249, 248, 262, 255, 268, 269, 246, 242, 255, 279, 252, 259, 275, 256, 266, 259, 289, 220, 281, 259, 244, 258, 254, 258, 270, 238, 265, 260, 246, 262, 264, 269, 237, 272, 232, 218, 285, 274, 245, 280, 259, 257, 288, 223, 252, 273, 255, 259, 250, 261, 276, 222, 266, 267, 269, 263, 260, 281, 256, 271, 260, 244, 250, 255, 216] payload_rows_per_dst=[5470, 5528, 5440, 5390, 5604, 5434, 5502, 5498] dup_rows_per_dst=[2682, 2724, 2702, 2592, 2756, 2736, 2742, 2736] combine_rows_per_src=[5470, 5528, 5440, 5390, 5604, 5434, 5502, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229768 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157467016 ingress_bytes=176111616 link_bytes=176111616 token_bytes=314704264 weight_movement_bytes=18874368 total_nvlink_bytes=333578632 weight_to_token_ratio=0.059975 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687414852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38308 dup_rows=21824 dup_rows_remote=19062 dedup_ratio=0.333008 router_tokens_per_expert=[273, 276, 250, 246, 261, 288, 269, 228, 276, 261, 244, 249, 253, 262, 263, 243, 273, 263, 255, 263, 255, 262, 287, 241, 251, 248, 251, 257, 238, 251, 263, 297, 282, 268, 273, 251, 248, 252, 285, 263, 233, 247, 244, 238, 258, 259, 241, 241, 252, 242, 248, 249, 262, 284, 266, 263, 259, 232, 256, 270, 256, 263, 264, 235, 240, 264, 268, 244, 262, 256, 268, 251, 262, 278, 265, 267, 273, 268, 262, 250, 256, 247, 257, 278, 260, 235, 253, 247, 239, 245, 265, 270, 244, 279, 242, 269, 242, 288, 273, 228, 252, 270, 234, 212, 224, 246, 273, 259, 261, 244, 255, 248, 258, 273, 243, 253, 249, 266, 264, 244, 252, 226, 236, 246, 264, 270, 248, 222] payload_rows_per_dst=[5572, 5518, 5404, 5442, 5462, 5398, 5448, 5468] dup_rows_per_dst=[2712, 2792, 2762, 2760, 2894, 2774, 2570, 2560] combine_rows_per_src=[5572, 5518, 5404, 5442, 5462, 5398, 5448, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228704 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156441952 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312655200 weight_movement_bytes=18874368 total_nvlink_bytes=331529568 weight_to_token_ratio=0.060368 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229496 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876920 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524344 weight_movement_bytes=18874368 total_nvlink_bytes=332398712 weight_to_token_ratio=0.060201 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38274 dup_rows=21878 dup_rows_remote=19184 dedup_ratio=0.333832 router_tokens_per_expert=[274, 247, 267, 264, 254, 256, 230, 255, 244, 262, 236, 235, 260, 249, 268, 259, 254, 250, 279, 294, 280, 234, 294, 271, 258, 274, 236, 260, 254, 262, 260, 257, 234, 227, 263, 286, 260, 245, 257, 265, 283, 279, 240, 278, 261, 266, 257, 250, 280, 247, 262, 261, 258, 241, 256, 248, 267, 245, 247, 257, 257, 243, 272, 244, 238, 261, 237, 245, 268, 235, 243, 277, 247, 282, 264, 247, 249, 255, 246, 245, 261, 248, 269, 282, 263, 265, 226, 237, 265, 271, 239, 273, 255, 260, 246, 252, 249, 237, 247, 252, 293, 223, 260, 248, 245, 253, 296, 238, 231, 256, 256, 242, 255, 244, 255, 234, 253, 248, 255, 257, 237, 271, 262, 278, 255, 290, 252, 232] payload_rows_per_dst=[5422, 5588, 5462, 5468, 5384, 5510, 5390, 5434] dup_rows_per_dst=[2698, 2846, 2840, 2702, 2694, 2714, 2662, 2722] combine_rows_per_src=[5422, 5588, 5462, 5468, 5384, 5510, 5390, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228808 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156310984 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393160 weight_movement_bytes=37748736 total_nvlink_bytes=350141896 weight_to_token_ratio=0.120837 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=230216 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157827912 ingress_bytes=176472064 link_bytes=176472064 token_bytes=315425608 weight_movement_bytes=18874368 total_nvlink_bytes=334299976 weight_to_token_ratio=0.059838 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690868488e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38212 dup_rows=21768 dup_rows_remote=19016 dedup_ratio=0.332153 router_tokens_per_expert=[255, 254, 244, 253, 261, 273, 263, 240, 244, 276, 242, 263, 256, 251, 253, 265, 264, 242, 249, 284, 240, 249, 257, 248, 251, 258, 272, 262, 239, 270, 224, 275, 264, 240, 256, 239, 289, 246, 258, 273, 279, 239, 215, 251, 257, 245, 255, 286, 259, 249, 253, 285, 262, 248, 249, 231, 279, 253, 297, 258, 248, 251, 247, 268, 268, 253, 239, 255, 223, 257, 266, 261, 233, 273, 256, 242, 253, 250, 248, 247, 247, 235, 257, 256, 273, 258, 226, 265, 242, 262, 248, 268, 266, 268, 286, 272, 236, 257, 250, 239, 259, 248, 259, 258, 256, 266, 263, 247, 268, 260, 257, 272, 261, 244, 250, 242, 272, 266, 259, 275, 242, 272, 228, 268, 254, 247, 282, 252] payload_rows_per_dst=[5432, 5412, 5494, 5398, 5408, 5580, 5488, 5556] dup_rows_per_dst=[2754, 2756, 2600, 2512, 2640, 3042, 2712, 2752] combine_rows_per_src=[5432, 5412, 5494, 5398, 5408, 5580, 5488, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229480 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157139048 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048616 weight_movement_bytes=37748736 total_nvlink_bytes=351797352 weight_to_token_ratio=0.120200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229832 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157000136 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770440 weight_movement_bytes=37748736 total_nvlink_bytes=351519176 weight_to_token_ratio=0.120307 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228912 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156745264 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313261616 weight_movement_bytes=18874368 total_nvlink_bytes=332135984 weight_to_token_ratio=0.060251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38314 dup_rows=21766 dup_rows_remote=19114 dedup_ratio=0.332123 router_tokens_per_expert=[257, 275, 247, 273, 251, 279, 265, 249, 274, 262, 248, 243, 221, 258, 270, 235, 241, 249, 249, 253, 261, 257, 285, 258, 274, 231, 256, 237, 237, 259, 246, 261, 221, 269, 258, 247, 254, 265, 261, 289, 253, 263, 251, 263, 260, 288, 260, 280, 244, 253, 256, 245, 244, 273, 242, 264, 282, 270, 253, 257, 286, 244, 276, 233, 269, 257, 255, 256, 250, 239, 261, 241, 279, 264, 260, 251, 243, 266, 252, 270, 237, 230, 257, 259, 268, 234, 239, 259, 229, 263, 277, 264, 277, 218, 247, 277, 259, 272, 250, 241, 244, 254, 278, 248, 238, 259, 257, 232, 243, 251, 267, 260, 263, 236, 273, 280, 258, 249, 236, 241, 263, 235, 262, 283, 248, 261, 272, 242] payload_rows_per_dst=[5408, 5456, 5552, 5532, 5444, 5414, 5442, 5522] dup_rows_per_dst=[2806, 2652, 2812, 2712, 2782, 2656, 2664, 2682] combine_rows_per_src=[5408, 5456, 5552, 5532, 5444, 5414, 5442, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38382 dup_rows=21762 dup_rows_remote=19150 dedup_ratio=0.332062 router_tokens_per_expert=[252, 218, 273, 244, 267, 230, 267, 231, 259, 250, 233, 256, 242, 270, 288, 255, 242, 255, 249, 261, 248, 248, 293, 260, 256, 226, 247, 270, 278, 243, 230, 267, 246, 232, 281, 250, 237, 253, 248, 252, 267, 259, 239, 256, 263, 255, 266, 268, 243, 254, 278, 260, 251, 260, 250, 255, 248, 271, 260, 265, 270, 266, 262, 266, 232, 256, 245, 256, 234, 252, 263, 257, 275, 243, 305, 275, 263, 282, 261, 259, 249, 279, 235, 229, 285, 236, 260, 249, 264, 261, 241, 264, 232, 237, 225, 256, 257, 259, 268, 273, 267, 245, 253, 263, 261, 253, 269, 238, 250, 266, 254, 280, 274, 247, 246, 274, 259, 261, 251, 281, 281, 243, 248, 237, 250, 243, 257, 261] payload_rows_per_dst=[5418, 5470, 5468, 5492, 5500, 5392, 5514, 5520] dup_rows_per_dst=[2652, 2676, 2676, 2826, 2816, 2612, 2798, 2706] combine_rows_per_src=[5418, 5470, 5468, 5492, 5500, 5392, 5514, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38290 dup_rows=21722 dup_rows_remote=19060 dedup_ratio=0.331451 router_tokens_per_expert=[278, 279, 267, 269, 277, 251, 261, 282, 268, 240, 250, 251, 253, 271, 246, 254, 253, 266, 263, 222, 280, 258, 271, 233, 221, 275, 236, 269, 260, 245, 219, 255, 269, 222, 241, 228, 258, 259, 253, 234, 276, 272, 278, 247, 251, 267, 258, 237, 249, 261, 263, 259, 249, 260, 271, 274, 234, 277, 291, 248, 228, 242, 255, 263, 285, 246, 247, 262, 239, 246, 255, 256, 242, 255, 225, 263, 252, 265, 249, 249, 251, 266, 255, 279, 273, 250, 261, 258, 263, 242, 248, 279, 253, 248, 252, 248, 242, 267, 236, 251, 256, 298, 242, 252, 244, 273, 236, 264, 273, 258, 245, 256, 243, 264, 267, 251, 233, 253, 275, 279, 226, 248, 277, 249, 260, 267, 258, 266] payload_rows_per_dst=[5530, 5420, 5472, 5470, 5414, 5530, 5524, 5454] dup_rows_per_dst=[2864, 2632, 2628, 2778, 2658, 2722, 2662, 2778] combine_rows_per_src=[5530, 5420, 5472, 5470, 5414, 5530, 5524, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38294 dup_rows=21722 dup_rows_remote=18894 dedup_ratio=0.331451 router_tokens_per_expert=[235, 267, 273, 260, 254, 264, 261, 229, 250, 257, 257, 245, 282, 250, 262, 257, 255, 249, 245, 266, 271, 250, 239, 240, 258, 288, 253, 210, 267, 242, 260, 254, 246, 224, 277, 229, 263, 251, 247, 285, 268, 261, 265, 276, 244, 246, 259, 245, 279, 287, 275, 247, 263, 232, 254, 261, 271, 264, 260, 243, 261, 241, 272, 264, 231, 254, 262, 260, 261, 245, 272, 268, 257, 255, 252, 245, 267, 277, 252, 236, 289, 247, 265, 254, 270, 257, 255, 261, 246, 265, 252, 275, 236, 263, 242, 267, 260, 259, 248, 265, 245, 248, 256, 286, 255, 244, 250, 253, 254, 261, 269, 244, 239, 266, 239, 256, 258, 252, 255, 274, 231, 247, 252, 250, 264, 249, 258, 233] payload_rows_per_dst=[5498, 5442, 5488, 5520, 5482, 5556, 5442, 5386] dup_rows_per_dst=[2708, 2652, 2684, 2828, 2706, 2732, 2752, 2660] combine_rows_per_src=[5498, 5442, 5488, 5520, 5482, 5556, 5442, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38082 dup_rows=21846 dup_rows_remote=19078 dedup_ratio=0.333344 router_tokens_per_expert=[263, 282, 283, 246, 272, 266, 266, 248, 251, 262, 259, 226, 250, 251, 255, 246, 260, 238, 247, 257, 266, 239, 234, 251, 262, 274, 262, 253, 246, 269, 244, 273, 255, 242, 262, 271, 250, 262, 249, 252, 272, 244, 287, 260, 251, 234, 230, 246, 247, 261, 228, 267, 263, 258, 265, 258, 279, 256, 252, 271, 253, 267, 223, 236, 287, 286, 271, 270, 257, 272, 257, 248, 270, 234, 272, 243, 230, 225, 228, 232, 249, 262, 260, 246, 255, 244, 248, 241, 271, 254, 248, 237, 289, 277, 255, 245, 271, 245, 252, 255, 238, 255, 252, 278, 252, 272, 235, 246, 266, 245, 241, 262, 243, 274, 254, 275, 276, 264, 252, 254, 253, 265, 256, 258, 257, 269, 277, 261] payload_rows_per_dst=[5608, 5578, 5636, 5418, 5242, 5462, 5426, 5320] dup_rows_per_dst=[2768, 2872, 2846, 2750, 2486, 2700, 2704, 2720] combine_rows_per_src=[5608, 5578, 5636, 5418, 5242, 5462, 5426, 5320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38044 dup_rows=21860 dup_rows_remote=19002 dedup_ratio=0.333557 router_tokens_per_expert=[265, 249, 234, 237, 222, 243, 235, 248, 245, 254, 258, 247, 264, 275, 270, 263, 297, 259, 252, 270, 276, 262, 256, 300, 232, 261, 230, 241, 259, 237, 276, 258, 241, 263, 267, 266, 237, 248, 289, 277, 279, 245, 280, 248, 274, 271, 292, 268, 269, 241, 229, 235, 255, 247, 245, 252, 252, 233, 241, 243, 239, 260, 272, 250, 283, 261, 230, 254, 249, 265, 254, 243, 276, 224, 272, 255, 243, 267, 269, 267, 244, 263, 239, 266, 255, 230, 241, 284, 270, 255, 270, 244, 242, 233, 259, 272, 261, 263, 240, 246, 245, 272, 247, 274, 264, 272, 289, 211, 272, 230, 273, 244, 238, 263, 257, 261, 239, 266, 252, 246, 262, 244, 290, 253, 258, 245, 271, 258] payload_rows_per_dst=[5416, 5496, 5632, 5364, 5448, 5394, 5488, 5438] dup_rows_per_dst=[2602, 2836, 2858, 2562, 2776, 2740, 2718, 2768] combine_rows_per_src=[5416, 5496, 5632, 5364, 5448, 5394, 5488, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229712 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163856 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098000 weight_movement_bytes=18874368 total_nvlink_bytes=332972368 weight_to_token_ratio=0.060091 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=230128 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442800 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314655472 weight_movement_bytes=18874368 total_nvlink_bytes=333529840 weight_to_token_ratio=0.059984 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38264 dup_rows=21722 dup_rows_remote=18978 dedup_ratio=0.331451 router_tokens_per_expert=[264, 257, 283, 245, 238, 252, 260, 274, 236, 266, 262, 241, 261, 273, 269, 272, 247, 255, 248, 253, 252, 237, 254, 235, 281, 245, 240, 264, 205, 271, 247, 264, 227, 250, 284, 274, 267, 250, 253, 234, 262, 262, 281, 245, 271, 257, 252, 267, 249, 251, 256, 246, 281, 249, 260, 281, 266, 261, 238, 230, 267, 206, 261, 226, 267, 249, 242, 282, 248, 247, 264, 269, 245, 248, 293, 267, 235, 248, 272, 271, 248, 268, 269, 257, 254, 240, 245, 290, 243, 243, 295, 236, 278, 252, 248, 259, 252, 234, 257, 261, 262, 255, 273, 290, 269, 277, 250, 269, 271, 237, 241, 262, 266, 242, 256, 255, 248, 267, 277, 222, 238, 267, 261, 251, 230, 269, 208, 264] payload_rows_per_dst=[5516, 5382, 5494, 5458, 5550, 5478, 5510, 5426] dup_rows_per_dst=[2790, 2614, 2778, 2598, 2744, 2772, 2810, 2616] combine_rows_per_src=[5516, 5382, 5494, 5458, 5550, 5478, 5510, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229400 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157065240 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313901080 weight_movement_bytes=18874368 total_nvlink_bytes=332775448 weight_to_token_ratio=0.060128 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228752 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157080976 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933200 weight_movement_bytes=37748736 total_nvlink_bytes=351681936 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=228640 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156212512 ingress_bytes=193732608 link_bytes=193732608 token_bytes=312196384 weight_movement_bytes=37748736 total_nvlink_bytes=349945120 weight_to_token_ratio=0.120913 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856250527e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38274 dup_rows=21860 dup_rows_remote=19054 dedup_ratio=0.333557 router_tokens_per_expert=[272, 249, 239, 282, 235, 258, 255, 257, 237, 235, 278, 247, 264, 249, 280, 248, 232, 275, 234, 250, 248, 263, 225, 245, 261, 267, 237, 274, 257, 267, 242, 258, 221, 262, 266, 288, 240, 248, 253, 255, 263, 281, 274, 245, 280, 245, 268, 245, 279, 243, 247, 245, 282, 281, 285, 245, 272, 272, 253, 277, 255, 253, 224, 251, 236, 265, 275, 240, 257, 223, 270, 242, 270, 260, 266, 265, 261, 248, 267, 274, 248, 279, 286, 255, 261, 258, 255, 251, 244, 236, 250, 239, 273, 264, 252, 280, 275, 234, 253, 267, 273, 251, 250, 258, 275, 240, 251, 264, 263, 234, 238, 254, 277, 251, 256, 219, 265, 244, 232, 248, 259, 248, 239, 257, 248, 263, 273, 241] payload_rows_per_dst=[5442, 5404, 5510, 5560, 5438, 5474, 5446, 5402] dup_rows_per_dst=[2728, 2666, 2758, 2768, 2800, 2788, 2546, 2806] combine_rows_per_src=[5442, 5404, 5510, 5560, 5438, 5474, 5446, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=155828224 dispatch_meta_bytes=228184 combine_payload_bytes=155828224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156056408 ingress_bytes=212451328 link_bytes=212451328 token_bytes=311884632 weight_movement_bytes=56623104 total_nvlink_bytes=368507736 weight_to_token_ratio=0.181551 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035604092e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228968 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156958312 ingress_bytes=213352448 link_bytes=213352448 token_bytes=313687656 weight_movement_bytes=56623104 total_nvlink_bytes=370310760 weight_to_token_ratio=0.180508 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044238181e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229312 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999616 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769920 weight_movement_bytes=18874368 total_nvlink_bytes=332644288 weight_to_token_ratio=0.060154 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38194 dup_rows=21946 dup_rows_remote=19246 dedup_ratio=0.334869 router_tokens_per_expert=[266, 255, 257, 248, 222, 245, 237, 277, 277, 302, 241, 274, 249, 241, 258, 226, 278, 266, 264, 234, 239, 241, 245, 227, 237, 281, 247, 273, 259, 250, 242, 265, 223, 243, 235, 248, 253, 253, 263, 247, 241, 236, 260, 273, 268, 261, 270, 245, 252, 259, 257, 278, 271, 274, 236, 274, 252, 224, 262, 250, 253, 244, 237, 275, 256, 279, 266, 245, 254, 259, 271, 244, 257, 255, 251, 276, 253, 251, 268, 263, 280, 256, 266, 234, 222, 224, 275, 236, 262, 244, 250, 277, 289, 269, 257, 255, 282, 272, 240, 278, 245, 250, 242, 268, 268, 268, 251, 258, 248, 286, 277, 244, 262, 251, 261, 252, 272, 211, 278, 247, 262, 288, 268, 240, 261, 252, 256, 246] payload_rows_per_dst=[5432, 5396, 5396, 5446, 5438, 5440, 5586, 5456] dup_rows_per_dst=[2718, 2700, 2642, 2750, 2858, 2752, 2768, 2758] combine_rows_per_src=[5432, 5396, 5396, 5446, 5438, 5440, 5586, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38134 dup_rows=21932 dup_rows_remote=19158 dedup_ratio=0.334656 router_tokens_per_expert=[246, 235, 267, 269, 279, 254, 259, 253, 268, 255, 276, 244, 269, 263, 241, 260, 254, 230, 301, 241, 271, 258, 279, 244, 253, 275, 251, 267, 292, 251, 255, 240, 273, 232, 255, 283, 269, 257, 225, 251, 238, 231, 251, 264, 262, 247, 263, 266, 247, 251, 260, 225, 245, 226, 252, 236, 256, 260, 236, 283, 246, 276, 252, 232, 263, 247, 270, 275, 257, 254, 243, 226, 252, 250, 274, 247, 257, 278, 240, 250, 259, 259, 249, 253, 259, 238, 244, 261, 256, 250, 267, 279, 264, 276, 266, 242, 264, 242, 299, 261, 268, 247, 261, 269, 244, 234, 243, 245, 248, 243, 263, 239, 259, 273, 241, 245, 258, 256, 267, 257, 256, 279, 249, 271, 288, 244, 236, 264] payload_rows_per_dst=[5500, 5602, 5380, 5330, 5382, 5470, 5418, 5522] dup_rows_per_dst=[2776, 2722, 2754, 2636, 2784, 2774, 2722, 2764] combine_rows_per_src=[5500, 5602, 5380, 5330, 5382, 5470, 5418, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38324 dup_rows=21786 dup_rows_remote=19088 dedup_ratio=0.332428 router_tokens_per_expert=[221, 260, 274, 229, 275, 257, 229, 243, 274, 267, 245, 249, 243, 239, 253, 249, 261, 256, 228, 269, 214, 271, 244, 242, 276, 239, 277, 263, 245, 261, 241, 236, 257, 273, 243, 274, 241, 271, 265, 260, 256, 241, 242, 281, 264, 248, 250, 268, 241, 279, 263, 243, 233, 265, 262, 252, 263, 255, 248, 261, 274, 220, 257, 246, 257, 270, 276, 261, 253, 241, 238, 264, 262, 255, 255, 254, 260, 265, 245, 257, 289, 272, 268, 263, 249, 262, 290, 264, 264, 245, 262, 269, 270, 252, 245, 262, 258, 266, 244, 245, 254, 256, 244, 253, 252, 264, 211, 279, 281, 251, 228, 246, 260, 270, 269, 248, 262, 267, 269, 239, 293, 282, 235, 236, 247, 273, 261, 260] payload_rows_per_dst=[5452, 5342, 5572, 5426, 5516, 5578, 5456, 5408] dup_rows_per_dst=[2562, 2704, 2696, 2698, 2710, 2874, 2608, 2934] combine_rows_per_src=[5452, 5342, 5572, 5426, 5516, 5578, 5456, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38152 dup_rows=21844 dup_rows_remote=19158 dedup_ratio=0.333313 router_tokens_per_expert=[279, 236, 278, 241, 280, 278, 272, 266, 281, 270, 254, 263, 276, 260, 268, 257, 259, 263, 250, 247, 251, 241, 256, 227, 221, 263, 257, 250, 236, 238, 263, 270, 262, 241, 251, 270, 243, 229, 265, 271, 257, 257, 252, 261, 280, 222, 243, 260, 247, 227, 254, 247, 261, 245, 274, 266, 252, 277, 246, 255, 249, 249, 244, 252, 251, 241, 240, 233, 257, 254, 266, 250, 254, 236, 245, 246, 243, 240, 284, 238, 268, 277, 264, 278, 251, 258, 242, 252, 260, 267, 252, 262, 279, 263, 248, 283, 244, 267, 279, 278, 246, 258, 256, 246, 254, 245, 250, 263, 259, 266, 245, 257, 273, 268, 253, 243, 247, 241, 262, 261, 261, 237, 249, 277, 286, 247, 250, 258] payload_rows_per_dst=[5656, 5388, 5430, 5404, 5336, 5510, 5540, 5428] dup_rows_per_dst=[2862, 2596, 2698, 2686, 2620, 2898, 2686, 2798] combine_rows_per_src=[5656, 5388, 5430, 5404, 5336, 5510, 5540, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38106 dup_rows=21896 dup_rows_remote=19146 dedup_ratio=0.334106 router_tokens_per_expert=[244, 250, 283, 244, 256, 271, 258, 248, 249, 246, 290, 242, 260, 272, 248, 236, 257, 230, 274, 254, 260, 228, 260, 279, 231, 257, 263, 288, 267, 248, 276, 243, 236, 268, 298, 266, 238, 282, 247, 255, 265, 277, 259, 242, 243, 256, 237, 277, 250, 280, 268, 274, 247, 250, 264, 240, 307, 265, 279, 256, 255, 218, 257, 253, 271, 254, 247, 255, 266, 260, 261, 260, 260, 249, 250, 247, 241, 248, 267, 284, 252, 251, 258, 242, 251, 234, 242, 261, 265, 258, 223, 234, 226, 244, 250, 236, 260, 246, 235, 271, 266, 245, 239, 235, 243, 274, 274, 271, 273, 226, 250, 261, 234, 242, 261, 259, 241, 253, 278, 277, 257, 275, 262, 267, 268, 250, 250, 257] payload_rows_per_dst=[5534, 5454, 5538, 5512, 5442, 5288, 5420, 5452] dup_rows_per_dst=[2750, 2688, 2978, 2814, 2824, 2566, 2718, 2558] combine_rows_per_src=[5534, 5454, 5538, 5512, 5442, 5288, 5420, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229760 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156672384 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313115008 weight_movement_bytes=18874368 total_nvlink_bytes=331989376 weight_to_token_ratio=0.060279 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38360 dup_rows=21730 dup_rows_remote=19046 dedup_ratio=0.331573 router_tokens_per_expert=[268, 256, 273, 241, 251, 256, 238, 249, 271, 256, 253, 242, 252, 272, 256, 245, 258, 239, 265, 261, 267, 281, 238, 266, 253, 250, 277, 249, 270, 254, 231, 260, 253, 275, 262, 244, 268, 244, 236, 279, 269, 240, 275, 253, 258, 238, 224, 247, 230, 243, 263, 221, 252, 231, 283, 240, 259, 268, 228, 232, 257, 233, 265, 266, 264, 240, 252, 240, 245, 257, 267, 260, 232, 262, 252, 276, 241, 244, 263, 275, 243, 264, 231, 258, 244, 294, 268, 266, 268, 233, 241, 279, 252, 252, 276, 269, 253, 275, 273, 300, 234, 274, 256, 253, 275, 282, 259, 240, 261, 280, 249, 243, 272, 265, 223, 266, 249, 298, 234, 272, 256, 257, 261, 271, 268, 250, 225, 252] payload_rows_per_dst=[5430, 5504, 5446, 5368, 5418, 5490, 5602, 5548] dup_rows_per_dst=[2728, 2734, 2684, 2574, 2722, 2786, 2812, 2690] combine_rows_per_src=[5430, 5504, 5446, 5368, 5418, 5490, 5602, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=229168 combine_payload_bytes=156196864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156426032 ingress_bytes=212819968 link_bytes=212819968 token_bytes=312622896 weight_movement_bytes=56623104 total_nvlink_bytes=369246000 weight_to_token_ratio=0.181123 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039136219e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38060 dup_rows=21998 dup_rows_remote=19174 dedup_ratio=0.335663 router_tokens_per_expert=[245, 291, 280, 249, 252, 274, 249, 266, 259, 248, 264, 266, 245, 261, 248, 239, 265, 252, 255, 249, 242, 266, 256, 304, 254, 259, 237, 268, 262, 235, 245, 240, 244, 290, 276, 263, 238, 251, 242, 231, 238, 252, 223, 253, 249, 276, 249, 248, 232, 261, 276, 237, 259, 264, 248, 231, 277, 250, 243, 240, 283, 267, 246, 262, 261, 254, 246, 256, 264, 248, 265, 263, 268, 265, 255, 265, 236, 297, 246, 262, 284, 259, 254, 253, 259, 252, 280, 285, 254, 269, 236, 248, 273, 267, 243, 264, 227, 265, 256, 260, 247, 261, 264, 217, 262, 273, 264, 225, 251, 272, 272, 251, 243, 273, 275, 264, 258, 249, 255, 230, 228, 257, 259, 222, 256, 272, 248, 257] payload_rows_per_dst=[5432, 5466, 5428, 5420, 5478, 5522, 5434, 5358] dup_rows_per_dst=[2840, 2712, 2618, 2732, 2824, 2838, 2700, 2734] combine_rows_per_src=[5432, 5466, 5428, 5420, 5478, 5522, 5434, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229648 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204752 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179856 weight_movement_bytes=18874368 total_nvlink_bytes=333054224 weight_to_token_ratio=0.060075 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229008 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311184 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393360 weight_movement_bytes=37748736 total_nvlink_bytes=350142096 weight_to_token_ratio=0.120837 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229240 combine_payload_bytes=156270592 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156499832 ingress_bytes=231768064 link_bytes=231768064 token_bytes=312770424 weight_movement_bytes=75497472 total_nvlink_bytes=388267896 weight_to_token_ratio=0.241383 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220687552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38372 dup_rows=21688 dup_rows_remote=18998 dedup_ratio=0.330933 router_tokens_per_expert=[279, 229, 253, 242, 262, 246, 246, 275, 270, 224, 225, 245, 237, 290, 288, 267, 241, 268, 244, 244, 260, 269, 255, 273, 242, 280, 253, 248, 262, 239, 243, 275, 238, 267, 257, 284, 243, 255, 249, 257, 277, 263, 268, 256, 215, 248, 267, 279, 246, 225, 246, 249, 260, 261, 262, 243, 264, 256, 247, 273, 237, 266, 265, 268, 242, 243, 239, 252, 267, 259, 261, 268, 265, 269, 262, 262, 244, 279, 261, 260, 269, 265, 227, 269, 249, 255, 261, 264, 249, 275, 244, 265, 255, 254, 266, 253, 270, 241, 258, 255, 286, 278, 228, 228, 260, 240, 250, 242, 251, 262, 253, 265, 229, 258, 279, 249, 256, 252, 232, 260, 259, 269, 243, 241, 264, 272, 256, 264] payload_rows_per_dst=[5420, 5554, 5524, 5324, 5612, 5450, 5488, 5476] dup_rows_per_dst=[2736, 2638, 2722, 2526, 2940, 2790, 2646, 2690] combine_rows_per_src=[5420, 5554, 5524, 5324, 5612, 5450, 5488, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229624 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157352184 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474744 weight_movement_bytes=18874368 total_nvlink_bytes=333349112 weight_to_token_ratio=0.060019 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=228936 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156122696 ingress_bytes=212516864 link_bytes=212516864 token_bytes=312016456 weight_movement_bytes=56623104 total_nvlink_bytes=368639560 weight_to_token_ratio=0.181475 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036232026e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38264 dup_rows=21852 dup_rows_remote=19046 dedup_ratio=0.333435 router_tokens_per_expert=[256, 276, 244, 273, 247, 270, 258, 236, 275, 283, 273, 265, 233, 247, 264, 220, 253, 248, 242, 242, 263, 273, 276, 250, 243, 230, 267, 269, 288, 238, 258, 273, 278, 262, 245, 266, 225, 254, 258, 235, 276, 271, 222, 253, 246, 257, 238, 260, 253, 245, 256, 270, 250, 283, 274, 237, 244, 267, 233, 259, 251, 246, 268, 274, 256, 269, 277, 251, 257, 241, 245, 220, 255, 263, 278, 257, 245, 247, 231, 278, 228, 240, 229, 265, 250, 256, 232, 276, 249, 229, 244, 250, 267, 252, 267, 257, 262, 230, 245, 267, 265, 249, 260, 264, 281, 266, 252, 257, 271, 248, 266, 251, 290, 261, 251, 271, 246, 253, 268, 253, 243, 267, 276, 284, 272, 249, 261, 239] payload_rows_per_dst=[5438, 5420, 5408, 5548, 5426, 5380, 5534, 5530] dup_rows_per_dst=[2802, 2806, 2684, 2672, 2714, 2602, 2734, 2838] combine_rows_per_src=[5438, 5420, 5408, 5548, 5426, 5380, 5534, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229480 combine_payload_bytes=157171712 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=157401192 ingress_bytes=251543552 link_bytes=251543552 token_bytes=314572904 weight_movement_bytes=94371840 total_nvlink_bytes=408944744 weight_to_token_ratio=0.300000 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.410166548e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38240 dup_rows=21852 dup_rows_remote=19088 dedup_ratio=0.333435 router_tokens_per_expert=[259, 250, 283, 278, 232, 213, 268, 256, 268, 247, 259, 259, 250, 250, 293, 250, 260, 228, 223, 231, 276, 274, 257, 262, 246, 277, 231, 268, 270, 255, 274, 265, 252, 270, 250, 261, 220, 271, 243, 238, 271, 258, 259, 259, 243, 243, 263, 272, 273, 242, 265, 280, 241, 250, 249, 273, 283, 261, 271, 270, 259, 233, 244, 254, 217, 243, 279, 251, 265, 255, 257, 242, 259, 252, 257, 241, 273, 265, 251, 251, 276, 266, 255, 257, 272, 267, 244, 261, 254, 225, 269, 252, 256, 233, 270, 247, 242, 249, 281, 246, 266, 254, 278, 237, 245, 238, 250, 261, 246, 265, 267, 254, 247, 275, 266, 265, 258, 247, 247, 254, 261, 238, 257, 252, 227, 293, 270, 237] payload_rows_per_dst=[5526, 5450, 5498, 5470, 5424, 5444, 5426, 5446] dup_rows_per_dst=[2704, 2744, 2648, 2826, 2692, 2764, 2732, 2742] combine_rows_per_src=[5526, 5450, 5498, 5470, 5424, 5444, 5426, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38260 dup_rows=21726 dup_rows_remote=18938 dedup_ratio=0.331512 router_tokens_per_expert=[254, 260, 244, 274, 238, 248, 262, 256, 254, 235, 239, 237, 229, 270, 287, 235, 236, 277, 252, 247, 270, 263, 255, 273, 285, 247, 256, 267, 255, 248, 264, 250, 288, 243, 235, 264, 253, 270, 243, 253, 254, 255, 254, 251, 256, 249, 240, 232, 255, 241, 264, 257, 256, 294, 261, 239, 276, 272, 254, 259, 282, 241, 278, 240, 258, 262, 268, 267, 241, 258, 261, 236, 252, 258, 239, 272, 251, 274, 262, 242, 280, 273, 264, 236, 274, 270, 244, 277, 256, 242, 260, 255, 237, 247, 253, 249, 246, 268, 255, 249, 250, 267, 270, 278, 235, 248, 257, 247, 284, 256, 232, 246, 260, 251, 272, 284, 264, 222, 258, 275, 262, 250, 255, 238, 252, 240, 251, 252] payload_rows_per_dst=[5332, 5466, 5388, 5550, 5524, 5506, 5554, 5490] dup_rows_per_dst=[2712, 2824, 2692, 2788, 2678, 2728, 2622, 2682] combine_rows_per_src=[5332, 5466, 5388, 5550, 5524, 5506, 5554, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38348 dup_rows=21692 dup_rows_remote=18996 dedup_ratio=0.330994 router_tokens_per_expert=[249, 259, 273, 263, 216, 253, 245, 268, 250, 262, 251, 261, 262, 246, 268, 239, 282, 276, 252, 240, 244, 247, 273, 229, 264, 248, 237, 250, 277, 234, 260, 250, 247, 260, 281, 242, 259, 260, 244, 279, 287, 229, 250, 274, 253, 249, 220, 243, 291, 252, 228, 280, 253, 288, 311, 278, 280, 252, 281, 243, 263, 237, 225, 263, 265, 257, 286, 246, 263, 240, 285, 236, 251, 259, 253, 256, 253, 264, 239, 265, 231, 234, 258, 238, 234, 255, 267, 263, 246, 246, 237, 265, 246, 276, 242, 240, 237, 259, 256, 281, 260, 255, 260, 239, 252, 265, 282, 259, 235, 243, 272, 241, 260, 254, 255, 282, 243, 272, 278, 223, 260, 297, 254, 266, 241, 252, 275, 234] payload_rows_per_dst=[5508, 5472, 5370, 5558, 5516, 5336, 5496, 5588] dup_rows_per_dst=[2622, 2654, 2784, 2892, 2720, 2620, 2696, 2704] combine_rows_per_src=[5508, 5472, 5370, 5558, 5516, 5336, 5496, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38386 dup_rows=21726 dup_rows_remote=18996 dedup_ratio=0.331512 router_tokens_per_expert=[253, 254, 248, 245, 240, 258, 236, 238, 237, 265, 273, 269, 285, 270, 233, 243, 239, 267, 247, 234, 244, 257, 246, 279, 241, 277, 250, 238, 242, 267, 242, 266, 247, 244, 260, 262, 263, 277, 269, 217, 268, 259, 286, 254, 225, 269, 240, 266, 272, 266, 244, 253, 258, 244, 248, 254, 225, 232, 255, 282, 276, 274, 258, 246, 280, 239, 250, 233, 265, 238, 267, 273, 269, 275, 291, 298, 247, 262, 251, 262, 267, 258, 266, 255, 249, 256, 249, 276, 237, 276, 270, 251, 244, 272, 242, 253, 266, 249, 247, 245, 277, 241, 235, 273, 272, 255, 274, 266, 243, 269, 245, 250, 254, 245, 258, 250, 244, 268, 262, 267, 252, 256, 253, 246, 254, 251, 247, 257] payload_rows_per_dst=[5424, 5610, 5320, 5452, 5592, 5460, 5430, 5522] dup_rows_per_dst=[2730, 2844, 2580, 2722, 2846, 2792, 2606, 2606] combine_rows_per_src=[5424, 5610, 5320, 5452, 5592, 5460, 5430, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229240 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958584 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687928 weight_movement_bytes=18874368 total_nvlink_bytes=332562296 weight_to_token_ratio=0.060169 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43492 payload_rows_remote=38124 dup_rows=22044 dup_rows_remote=19330 dedup_ratio=0.336365 router_tokens_per_expert=[229, 251, 284, 249, 237, 270, 237, 230, 261, 238, 262, 247, 273, 255, 252, 234, 235, 245, 256, 238, 244, 244, 275, 272, 268, 260, 235, 253, 244, 237, 258, 301, 254, 263, 276, 235, 267, 273, 227, 244, 245, 263, 261, 232, 251, 253, 256, 241, 243, 262, 286, 247, 241, 272, 248, 284, 250, 268, 262, 266, 287, 256, 279, 259, 255, 253, 259, 244, 258, 256, 262, 261, 254, 259, 273, 263, 267, 288, 239, 266, 259, 254, 241, 257, 243, 268, 269, 267, 256, 252, 214, 251, 257, 268, 272, 267, 262, 279, 235, 301, 238, 261, 262, 270, 278, 259, 263, 247, 253, 270, 229, 250, 270, 280, 250, 257, 236, 277, 240, 226, 246, 268, 253, 256, 249, 258, 243, 225] payload_rows_per_dst=[5304, 5416, 5368, 5510, 5518, 5496, 5486, 5394] dup_rows_per_dst=[2714, 2714, 2714, 2910, 2796, 2694, 2828, 2674] combine_rows_per_src=[5304, 5416, 5368, 5510, 5518, 5496, 5486, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38184 dup_rows=21918 dup_rows_remote=19396 dedup_ratio=0.334442 router_tokens_per_expert=[248, 270, 228, 277, 258, 251, 224, 249, 238, 248, 279, 256, 256, 236, 255, 255, 242, 237, 265, 236, 238, 240, 250, 248, 276, 262, 237, 274, 260, 267, 239, 297, 285, 234, 241, 258, 248, 281, 249, 304, 265, 240, 258, 253, 240, 265, 247, 292, 248, 254, 240, 260, 293, 247, 266, 236, 261, 252, 280, 261, 265, 246, 257, 258, 246, 261, 225, 221, 259, 260, 271, 250, 264, 252, 255, 244, 260, 232, 232, 246, 237, 243, 259, 264, 275, 240, 259, 291, 290, 243, 258, 247, 259, 265, 271, 270, 246, 236, 246, 251, 272, 283, 252, 248, 269, 227, 257, 256, 263, 245, 296, 237, 262, 254, 256, 262, 274, 257, 279, 248, 266, 241, 243, 267, 297, 259, 234, 256] payload_rows_per_dst=[5380, 5410, 5506, 5418, 5434, 5488, 5496, 5486] dup_rows_per_dst=[2676, 2726, 2814, 2830, 2522, 2854, 2672, 2824] combine_rows_per_src=[5380, 5410, 5506, 5418, 5434, 5488, 5496, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229312 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860352 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491392 weight_movement_bytes=18874368 total_nvlink_bytes=332365760 weight_to_token_ratio=0.060207 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228792 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156941752 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313654712 weight_movement_bytes=56623104 total_nvlink_bytes=370277816 weight_to_token_ratio=0.180527 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229376 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157302784 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314376192 weight_movement_bytes=37748736 total_nvlink_bytes=352124928 weight_to_token_ratio=0.120075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229528 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157458584 ingress_bytes=194977792 link_bytes=194977792 token_bytes=314687640 weight_movement_bytes=37748736 total_nvlink_bytes=352436376 weight_to_token_ratio=0.119956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868181268e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229816 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156385720 ingress_bytes=175030272 link_bytes=175030272 token_bytes=312541624 weight_movement_bytes=18874368 total_nvlink_bytes=331415992 weight_to_token_ratio=0.060390 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677053946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38202 dup_rows=21730 dup_rows_remote=18850 dedup_ratio=0.331573 router_tokens_per_expert=[224, 250, 283, 276, 260, 237, 256, 250, 245, 279, 231, 256, 248, 272, 261, 246, 271, 250, 255, 253, 287, 257, 258, 234, 268, 263, 260, 267, 233, 252, 259, 260, 252, 251, 274, 273, 293, 240, 254, 267, 260, 269, 256, 248, 260, 270, 267, 227, 266, 260, 263, 265, 251, 272, 237, 266, 245, 272, 220, 245, 247, 244, 275, 285, 251, 255, 256, 228, 271, 255, 263, 270, 250, 287, 235, 245, 251, 223, 276, 262, 233, 241, 251, 271, 259, 252, 261, 228, 254, 264, 255, 237, 229, 242, 269, 252, 274, 247, 250, 227, 246, 247, 266, 240, 265, 246, 253, 264, 267, 253, 239, 270, 255, 271, 286, 246, 268, 285, 227, 245, 259, 243, 271, 278, 248, 264, 255, 262] payload_rows_per_dst=[5504, 5300, 5514, 5456, 5346, 5522, 5560, 5604] dup_rows_per_dst=[2644, 2588, 2808, 2650, 2416, 2870, 2874, 2880] combine_rows_per_src=[5504, 5300, 5514, 5456, 5346, 5522, 5560, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=230320 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156631984 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313033648 weight_movement_bytes=18874368 total_nvlink_bytes=331908016 weight_to_token_ratio=0.060295 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38248 dup_rows=21806 dup_rows_remote=19038 dedup_ratio=0.332733 router_tokens_per_expert=[245, 256, 262, 253, 253, 232, 259, 250, 229, 285, 263, 249, 270, 264, 253, 260, 253, 269, 252, 269, 246, 253, 252, 267, 244, 226, 255, 263, 277, 265, 262, 272, 253, 236, 250, 240, 260, 258, 262, 256, 224, 259, 274, 258, 259, 243, 287, 251, 261, 269, 246, 242, 261, 247, 250, 247, 244, 246, 244, 247, 255, 274, 255, 254, 307, 235, 231, 290, 255, 256, 278, 258, 245, 255, 277, 260, 267, 232, 261, 250, 247, 277, 273, 268, 261, 250, 269, 275, 268, 274, 261, 267, 251, 251, 253, 262, 234, 259, 245, 258, 238, 254, 253, 256, 279, 254, 255, 246, 234, 264, 264, 261, 267, 260, 255, 233, 251, 247, 246, 241, 254, 249, 241, 247, 266, 237, 270, 266] payload_rows_per_dst=[5476, 5482, 5426, 5444, 5540, 5540, 5394, 5428] dup_rows_per_dst=[2690, 2768, 2714, 2640, 2774, 2874, 2714, 2632] combine_rows_per_src=[5476, 5482, 5426, 5444, 5540, 5540, 5394, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=228208 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156703600 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313178992 weight_movement_bytes=18874368 total_nvlink_bytes=332053360 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38214 dup_rows=21944 dup_rows_remote=19160 dedup_ratio=0.334839 router_tokens_per_expert=[264, 256, 255, 275, 246, 231, 267, 297, 256, 255, 262, 243, 259, 263, 286, 244, 259, 294, 252, 254, 247, 262, 238, 242, 247, 251, 275, 240, 233, 264, 278, 275, 265, 275, 273, 232, 234, 259, 258, 267, 281, 256, 288, 280, 257, 247, 264, 251, 265, 276, 280, 234, 218, 252, 243, 283, 272, 260, 260, 251, 249, 243, 247, 248, 240, 250, 249, 240, 267, 257, 255, 263, 260, 235, 270, 279, 257, 246, 283, 251, 229, 260, 249, 267, 288, 265, 278, 250, 268, 236, 254, 263, 230, 268, 251, 271, 292, 232, 271, 253, 263, 244, 231, 249, 249, 246, 235, 256, 257, 262, 249, 240, 264, 267, 211, 246, 266, 235, 265, 250, 249, 261, 258, 218, 243, 243, 256, 240] payload_rows_per_dst=[5476, 5414, 5528, 5378, 5484, 5544, 5390, 5378] dup_rows_per_dst=[2842, 2808, 2846, 2784, 2720, 2710, 2668, 2566] combine_rows_per_src=[5476, 5414, 5528, 5378, 5484, 5544, 5390, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38424 dup_rows=21704 dup_rows_remote=19106 dedup_ratio=0.331177 router_tokens_per_expert=[249, 247, 250, 238, 240, 246, 256, 264, 269, 236, 234, 247, 271, 249, 250, 279, 257, 265, 256, 256, 264, 269, 267, 259, 254, 254, 270, 254, 251, 248, 271, 240, 277, 247, 250, 262, 262, 253, 238, 255, 283, 229, 280, 244, 250, 249, 236, 268, 225, 266, 253, 257, 242, 292, 254, 247, 256, 236, 239, 261, 249, 259, 283, 279, 242, 270, 255, 259, 235, 287, 259, 269, 255, 284, 242, 277, 260, 263, 253, 265, 234, 245, 268, 259, 249, 265, 254, 254, 237, 231, 238, 259, 278, 238, 234, 260, 272, 241, 244, 221, 263, 258, 249, 238, 260, 240, 249, 264, 281, 259, 273, 233, 245, 252, 235, 289, 287, 263, 255, 274, 273, 245, 258, 298, 269, 259, 230, 272] payload_rows_per_dst=[5476, 5480, 5458, 5466, 5504, 5408, 5486, 5554] dup_rows_per_dst=[2574, 2790, 2708, 2730, 2846, 2598, 2604, 2854] combine_rows_per_src=[5476, 5480, 5458, 5466, 5504, 5408, 5486, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38256 dup_rows=21726 dup_rows_remote=18960 dedup_ratio=0.331512 router_tokens_per_expert=[269, 237, 259, 263, 279, 259, 282, 243, 241, 257, 262, 251, 273, 227, 268, 251, 267, 235, 243, 228, 271, 232, 274, 261, 267, 246, 280, 269, 263, 269, 249, 260, 248, 268, 234, 253, 258, 240, 253, 230, 246, 255, 271, 246, 239, 240, 256, 268, 271, 261, 236, 292, 260, 253, 262, 280, 241, 237, 225, 258, 282, 271, 253, 258, 256, 260, 262, 266, 284, 260, 252, 286, 252, 258, 239, 257, 257, 254, 237, 239, 249, 266, 252, 276, 250, 262, 273, 295, 243, 235, 221, 277, 286, 255, 225, 232, 248, 239, 276, 261, 237, 279, 257, 283, 246, 257, 276, 269, 283, 262, 238, 249, 252, 253, 226, 252, 292, 230, 244, 244, 248, 260, 230, 242, 278, 252, 250, 259] payload_rows_per_dst=[5492, 5512, 5426, 5440, 5456, 5540, 5554, 5390] dup_rows_per_dst=[2750, 2716, 2584, 2840, 2782, 2654, 2766, 2634] combine_rows_per_src=[5492, 5512, 5426, 5440, 5456, 5540, 5554, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38148 dup_rows=21818 dup_rows_remote=18982 dedup_ratio=0.332916 router_tokens_per_expert=[226, 260, 303, 300, 274, 245, 257, 251, 267, 256, 257, 259, 262, 262, 229, 250, 256, 281, 259, 236, 232, 217, 266, 266, 251, 254, 272, 232, 260, 268, 258, 236, 260, 246, 283, 246, 263, 265, 271, 256, 276, 259, 261, 277, 260, 249, 246, 244, 261, 252, 251, 239, 242, 268, 267, 246, 252, 243, 260, 259, 271, 305, 241, 265, 263, 251, 239, 251, 260, 269, 251, 286, 257, 262, 244, 249, 248, 234, 232, 260, 265, 255, 263, 274, 251, 271, 248, 258, 268, 240, 256, 243, 249, 233, 254, 278, 264, 244, 256, 252, 224, 266, 235, 274, 251, 217, 248, 233, 265, 260, 255, 270, 257, 235, 234, 286, 260, 267, 260, 262, 257, 251, 253, 245, 253, 253, 269, 264] payload_rows_per_dst=[5570, 5608, 5346, 5466, 5444, 5446, 5370, 5468] dup_rows_per_dst=[2836, 2958, 2500, 2818, 2560, 2766, 2636, 2744] combine_rows_per_src=[5570, 5608, 5346, 5466, 5444, 5446, 5370, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38326 dup_rows=21780 dup_rows_remote=19160 dedup_ratio=0.332336 router_tokens_per_expert=[241, 247, 257, 249, 261, 260, 261, 240, 239, 245, 255, 240, 267, 244, 285, 230, 256, 265, 270, 276, 263, 243, 298, 234, 269, 288, 251, 263, 263, 247, 246, 251, 254, 271, 263, 265, 246, 271, 261, 256, 248, 231, 235, 256, 223, 257, 253, 235, 280, 254, 238, 231, 260, 242, 253, 276, 276, 243, 246, 287, 259, 264, 218, 260, 272, 258, 254, 265, 257, 279, 257, 285, 258, 244, 255, 249, 231, 299, 246, 228, 233, 275, 255, 257, 279, 257, 244, 246, 238, 260, 250, 249, 249, 257, 270, 265, 280, 235, 279, 251, 257, 257, 264, 266, 269, 239, 269, 248, 242, 274, 265, 248, 277, 237, 260, 250, 281, 249, 264, 237, 277, 255, 271, 253, 230, 251, 270, 226] payload_rows_per_dst=[5376, 5534, 5430, 5474, 5502, 5452, 5520, 5468] dup_rows_per_dst=[2666, 2832, 2620, 2700, 2772, 2716, 2766, 2708] combine_rows_per_src=[5376, 5534, 5430, 5474, 5502, 5452, 5520, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229144 combine_payload_bytes=156663808 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156892952 ingress_bytes=232161280 link_bytes=232161280 token_bytes=313556760 weight_movement_bytes=75497472 total_nvlink_bytes=389054232 weight_to_token_ratio=0.240778 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224455154e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38316 dup_rows=21702 dup_rows_remote=19010 dedup_ratio=0.331146 router_tokens_per_expert=[256, 256, 243, 249, 267, 270, 257, 268, 277, 262, 252, 283, 273, 250, 269, 266, 269, 247, 237, 256, 257, 268, 257, 260, 271, 231, 271, 249, 260, 266, 252, 258, 275, 281, 230, 221, 237, 237, 260, 260, 233, 282, 254, 244, 235, 255, 259, 243, 246, 264, 236, 278, 275, 235, 260, 240, 268, 262, 258, 249, 248, 290, 243, 278, 258, 266, 251, 245, 267, 245, 276, 259, 254, 268, 236, 259, 284, 250, 232, 255, 251, 239, 254, 264, 263, 257, 241, 245, 242, 264, 247, 249, 264, 246, 281, 246, 277, 270, 246, 257, 277, 246, 252, 253, 213, 248, 264, 232, 254, 258, 261, 247, 241, 276, 232, 250, 244, 244, 268, 273, 253, 264, 248, 257, 244, 281, 261, 276] payload_rows_per_dst=[5500, 5484, 5360, 5576, 5518, 5470, 5476, 5450] dup_rows_per_dst=[2896, 2734, 2652, 2684, 2692, 2636, 2634, 2774] combine_rows_per_src=[5500, 5484, 5360, 5576, 5518, 5470, 5476, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229496 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156754040 ingress_bytes=194273280 link_bytes=194273280 token_bytes=313278584 weight_movement_bytes=37748736 total_nvlink_bytes=351027320 weight_to_token_ratio=0.120496 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861430981e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157384704 dispatch_meta_bytes=230120 combine_payload_bytes=157384704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157614824 ingress_bytes=176259072 link_bytes=176259072 token_bytes=314999528 weight_movement_bytes=18874368 total_nvlink_bytes=333873896 weight_to_token_ratio=0.059919 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688827703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44030 payload_rows_remote=38364 dup_rows=21506 dup_rows_remote=18466 dedup_ratio=0.328156 router_tokens_per_expert=[247, 246, 238, 260, 235, 237, 287, 269, 235, 265, 277, 261, 228, 284, 248, 245, 261, 233, 245, 254, 247, 270, 250, 276, 272, 263, 250, 242, 253, 267, 279, 267, 276, 263, 257, 237, 274, 243, 246, 281, 278, 237, 266, 247, 282, 271, 244, 247, 241, 251, 244, 244, 271, 256, 280, 247, 259, 245, 252, 260, 274, 289, 261, 269, 247, 244, 230, 224, 253, 242, 249, 245, 212, 247, 283, 276, 234, 231, 257, 255, 253, 240, 282, 245, 276, 259, 264, 266, 264, 258, 259, 246, 260, 235, 259, 267, 280, 262, 263, 256, 264, 247, 260, 263, 236, 214, 235, 262, 217, 279, 263, 234, 246, 264, 267, 277, 252, 290, 242, 268, 275, 255, 243, 255, 273, 239, 270, 272] payload_rows_per_dst=[5504, 5454, 5584, 5516, 5422, 5426, 5458, 5666] dup_rows_per_dst=[2620, 2474, 2706, 2770, 2436, 2840, 2620, 3040] combine_rows_per_src=[5504, 5454, 5584, 5516, 5422, 5426, 5458, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=228864 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156925440 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313622016 weight_movement_bytes=56623104 total_nvlink_bytes=370245120 weight_to_token_ratio=0.180546 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228520 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156482728 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312736936 weight_movement_bytes=18874368 total_nvlink_bytes=331611304 weight_to_token_ratio=0.060352 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229944 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213240 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196536 weight_movement_bytes=18874368 total_nvlink_bytes=333070904 weight_to_token_ratio=0.060072 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229304 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157171640 ingress_bytes=213565440 link_bytes=213565440 token_bytes=314113976 weight_movement_bytes=56623104 total_nvlink_bytes=370737080 weight_to_token_ratio=0.180263 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046278965e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=227320 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157366264 ingress_bytes=194887680 link_bytes=194887680 token_bytes=314505208 weight_movement_bytes=37748736 total_nvlink_bytes=352253944 weight_to_token_ratio=0.120026 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38246 dup_rows=21824 dup_rows_remote=19106 dedup_ratio=0.333008 router_tokens_per_expert=[245, 254, 256, 282, 247, 273, 250, 260, 252, 278, 265, 272, 268, 261, 246, 267, 265, 256, 259, 270, 279, 237, 226, 276, 271, 239, 264, 264, 237, 246, 257, 246, 258, 243, 254, 247, 250, 276, 265, 248, 241, 270, 251, 254, 271, 268, 252, 281, 265, 246, 271, 262, 260, 248, 260, 230, 247, 280, 266, 265, 249, 272, 255, 237, 248, 265, 250, 230, 252, 231, 239, 244, 255, 241, 253, 242, 229, 262, 238, 256, 243, 259, 239, 270, 272, 270, 251, 264, 237, 278, 249, 260, 256, 249, 260, 254, 243, 255, 254, 230, 231, 252, 249, 266, 246, 256, 262, 251, 251, 246, 264, 253, 282, 263, 242, 244, 269, 234, 271, 253, 251, 274, 270, 263, 253, 275, 281, 278] payload_rows_per_dst=[5502, 5466, 5530, 5484, 5460, 5480, 5390, 5400] dup_rows_per_dst=[2850, 2718, 2728, 2742, 2646, 2742, 2628, 2770] combine_rows_per_src=[5502, 5466, 5530, 5484, 5460, 5480, 5390, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38444 dup_rows=21678 dup_rows_remote=19106 dedup_ratio=0.330780 router_tokens_per_expert=[261, 224, 279, 293, 267, 238, 239, 254, 258, 256, 256, 266, 263, 263, 258, 258, 243, 249, 271, 260, 265, 263, 234, 255, 272, 246, 270, 253, 264, 277, 264, 236, 282, 229, 250, 248, 263, 263, 239, 255, 262, 272, 297, 263, 262, 294, 256, 265, 247, 230, 243, 262, 276, 239, 251, 230, 242, 264, 253, 265, 247, 263, 261, 220, 274, 230, 285, 258, 255, 251, 242, 270, 267, 256, 237, 247, 251, 250, 245, 275, 266, 262, 250, 265, 237, 267, 272, 207, 236, 266, 262, 253, 256, 248, 252, 235, 254, 270, 257, 268, 253, 258, 238, 271, 257, 280, 244, 232, 278, 264, 248, 255, 231, 261, 258, 262, 260, 257, 253, 260, 242, 242, 259, 262, 239, 276, 236, 268] payload_rows_per_dst=[5500, 5516, 5568, 5414, 5454, 5466, 5526, 5414] dup_rows_per_dst=[2766, 2728, 2832, 2572, 2732, 2602, 2728, 2718] combine_rows_per_src=[5500, 5516, 5568, 5414, 5454, 5466, 5526, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38198 dup_rows=21780 dup_rows_remote=19032 dedup_ratio=0.332336 router_tokens_per_expert=[254, 244, 283, 245, 232, 240, 256, 249, 263, 272, 254, 244, 254, 268, 278, 243, 280, 249, 258, 266, 268, 263, 254, 215, 258, 256, 265, 246, 263, 249, 236, 239, 251, 258, 268, 269, 223, 258, 247, 262, 227, 288, 242, 252, 239, 241, 235, 266, 284, 254, 247, 266, 244, 250, 250, 273, 256, 251, 243, 250, 256, 227, 252, 237, 258, 254, 254, 259, 226, 260, 242, 277, 245, 270, 275, 254, 271, 243, 275, 223, 273, 287, 261, 253, 243, 237, 247, 257, 262, 264, 249, 274, 258, 267, 260, 261, 247, 301, 235, 273, 266, 266, 261, 234, 274, 285, 269, 279, 281, 282, 257, 262, 277, 259, 252, 284, 244, 263, 250, 254, 234, 251, 256, 239, 251, 256, 255, 222] payload_rows_per_dst=[5436, 5464, 5358, 5460, 5472, 5558, 5628, 5380] dup_rows_per_dst=[2722, 2666, 2694, 2620, 2700, 2748, 2916, 2714] combine_rows_per_src=[5436, 5464, 5358, 5460, 5472, 5558, 5628, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44008 payload_rows_remote=38574 dup_rows=21528 dup_rows_remote=18954 dedup_ratio=0.328491 router_tokens_per_expert=[253, 244, 263, 251, 217, 243, 239, 260, 266, 241, 299, 256, 274, 222, 268, 235, 266, 237, 262, 274, 291, 262, 257, 275, 267, 267, 256, 250, 236, 252, 264, 273, 267, 247, 256, 221, 252, 251, 258, 242, 259, 245, 264, 254, 248, 241, 249, 250, 258, 261, 247, 265, 225, 274, 259, 249, 273, 273, 260, 263, 255, 274, 248, 243, 233, 264, 266, 253, 251, 256, 259, 237, 244, 255, 237, 251, 267, 258, 260, 256, 249, 245, 254, 256, 230, 277, 239, 251, 227, 261, 248, 276, 256, 261, 279, 247, 274, 282, 271, 291, 262, 258, 241, 253, 231, 248, 267, 248, 241, 276, 240, 232, 251, 226, 224, 249, 264, 262, 284, 246, 303, 250, 271, 292, 279, 261, 275, 262] payload_rows_per_dst=[5378, 5686, 5434, 5538, 5462, 5444, 5514, 5552] dup_rows_per_dst=[2684, 2692, 2574, 2716, 2632, 2668, 2716, 2846] combine_rows_per_src=[5378, 5686, 5434, 5538, 5462, 5444, 5514, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229408 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156885024 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540640 weight_movement_bytes=37748736 total_nvlink_bytes=351289376 weight_to_token_ratio=0.120395 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38124 dup_rows=21962 dup_rows_remote=19182 dedup_ratio=0.335114 router_tokens_per_expert=[252, 261, 264, 268, 268, 260, 257, 262, 249, 238, 245, 277, 257, 279, 280, 260, 249, 241, 276, 269, 262, 254, 223, 261, 242, 240, 221, 242, 241, 240, 262, 276, 256, 271, 289, 248, 241, 253, 265, 283, 277, 290, 247, 267, 251, 276, 264, 236, 261, 269, 242, 235, 258, 250, 248, 290, 239, 230, 275, 248, 253, 234, 227, 235, 247, 258, 250, 227, 256, 258, 263, 269, 272, 258, 267, 262, 251, 267, 261, 264, 235, 241, 277, 253, 266, 236, 264, 248, 270, 227, 269, 252, 234, 252, 259, 242, 275, 239, 239, 229, 238, 277, 263, 240, 253, 277, 269, 258, 258, 247, 261, 292, 273, 287, 222, 272, 260, 245, 239, 271, 259, 234, 254, 256, 260, 267, 232, 283] payload_rows_per_dst=[5570, 5390, 5466, 5334, 5526, 5378, 5450, 5460] dup_rows_per_dst=[2784, 2608, 2962, 2654, 2734, 2672, 2780, 2768] combine_rows_per_src=[5570, 5390, 5466, 5334, 5526, 5378, 5450, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38268 dup_rows=21818 dup_rows_remote=19014 dedup_ratio=0.332916 router_tokens_per_expert=[255, 230, 252, 267, 281, 268, 241, 251, 258, 261, 245, 255, 244, 272, 268, 250, 273, 237, 258, 267, 278, 262, 255, 251, 241, 252, 240, 254, 262, 258, 276, 268, 216, 222, 252, 231, 277, 261, 269, 268, 249, 272, 248, 266, 266, 272, 251, 259, 247, 258, 249, 275, 249, 262, 270, 256, 244, 256, 254, 243, 260, 287, 257, 266, 270, 249, 238, 254, 265, 266, 267, 257, 276, 289, 256, 241, 249, 251, 263, 243, 272, 256, 253, 237, 249, 291, 245, 239, 245, 240, 240, 273, 234, 249, 266, 274, 229, 237, 237, 257, 239, 258, 239, 263, 268, 260, 258, 288, 243, 256, 263, 268, 265, 247, 254, 242, 233, 243, 256, 255, 267, 277, 257, 251, 253, 270, 249, 247] payload_rows_per_dst=[5450, 5550, 5630, 5648, 5376, 5124, 5518, 5422] dup_rows_per_dst=[2804, 2806, 3012, 2900, 2554, 2432, 2600, 2710] combine_rows_per_src=[5450, 5550, 5630, 5648, 5376, 5124, 5518, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38332 dup_rows=21724 dup_rows_remote=18952 dedup_ratio=0.331482 router_tokens_per_expert=[226, 278, 239, 248, 258, 258, 254, 229, 240, 255, 262, 255, 251, 250, 245, 258, 252, 241, 233, 275, 259, 245, 262, 262, 261, 243, 264, 235, 242, 243, 245, 234, 253, 280, 268, 248, 238, 301, 241, 290, 260, 239, 238, 279, 257, 252, 267, 261, 228, 266, 253, 230, 247, 262, 226, 283, 266, 267, 274, 236, 264, 268, 269, 258, 268, 223, 253, 265, 261, 257, 273, 270, 225, 267, 266, 249, 247, 268, 265, 269, 267, 248, 258, 245, 254, 256, 268, 261, 248, 260, 260, 258, 258, 262, 268, 291, 289, 246, 264, 255, 255, 289, 225, 246, 245, 276, 262, 237, 246, 240, 260, 243, 284, 226, 252, 257, 243, 275, 252, 230, 283, 235, 259, 267, 282, 272, 254, 260] payload_rows_per_dst=[5386, 5370, 5562, 5562, 5480, 5496, 5432, 5524] dup_rows_per_dst=[2626, 2622, 2782, 2632, 2772, 2828, 2724, 2738] combine_rows_per_src=[5386, 5370, 5562, 5562, 5480, 5496, 5432, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38288 dup_rows=21852 dup_rows_remote=19120 dedup_ratio=0.333435 router_tokens_per_expert=[247, 260, 229, 253, 255, 267, 256, 236, 235, 249, 249, 247, 257, 247, 244, 265, 267, 280, 265, 265, 253, 256, 258, 237, 255, 264, 277, 238, 236, 264, 254, 254, 246, 277, 229, 292, 256, 261, 242, 246, 245, 265, 252, 274, 269, 274, 269, 251, 263, 259, 259, 239, 257, 266, 270, 259, 255, 243, 288, 236, 262, 271, 260, 243, 216, 252, 259, 231, 272, 264, 285, 242, 270, 258, 253, 205, 264, 244, 255, 265, 288, 272, 262, 248, 253, 244, 236, 272, 258, 234, 258, 260, 241, 245, 264, 264, 253, 278, 239, 252, 268, 274, 262, 267, 279, 291, 252, 241, 259, 276, 242, 240, 245, 270, 257, 251, 239, 251, 267, 251, 243, 282, 216, 277, 263, 255, 247, 250] payload_rows_per_dst=[5382, 5492, 5330, 5498, 5592, 5466, 5528, 5396] dup_rows_per_dst=[2610, 2754, 2444, 2762, 3000, 2698, 2852, 2732] combine_rows_per_src=[5382, 5492, 5330, 5498, 5592, 5466, 5528, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=230200 combine_payload_bytes=157466624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157696824 ingress_bytes=176340992 link_bytes=176340992 token_bytes=315163448 weight_movement_bytes=18874368 total_nvlink_bytes=334037816 weight_to_token_ratio=0.059888 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689612620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228920 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156687928 ingress_bytes=194207744 link_bytes=194207744 token_bytes=313146936 weight_movement_bytes=37748736 total_nvlink_bytes=350895672 weight_to_token_ratio=0.120546 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860803047e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229128 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974856 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720584 weight_movement_bytes=18874368 total_nvlink_bytes=332594952 weight_to_token_ratio=0.060163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157999104 dispatch_meta_bytes=230112 combine_payload_bytes=157999104 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=158229216 ingress_bytes=195747840 link_bytes=195747840 token_bytes=316228320 weight_movement_bytes=37748736 total_nvlink_bytes=353977056 weight_to_token_ratio=0.119372 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.875559489e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229224 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156385128 ingress_bytes=175030272 link_bytes=175030272 token_bytes=312541032 weight_movement_bytes=18874368 total_nvlink_bytes=331415400 weight_to_token_ratio=0.060390 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677053946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229136 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237008 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314244880 weight_movement_bytes=18874368 total_nvlink_bytes=333119248 weight_to_token_ratio=0.060063 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229632 combine_payload_bytes=156827648 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157057280 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884928 weight_movement_bytes=18874368 total_nvlink_bytes=332759296 weight_to_token_ratio=0.060131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38306 dup_rows=21794 dup_rows_remote=19026 dedup_ratio=0.332550 router_tokens_per_expert=[260, 257, 246, 283, 255, 232, 253, 247, 223, 238, 243, 258, 260, 254, 260, 240, 227, 267, 262, 246, 271, 246, 258, 251, 250, 251, 285, 276, 258, 261, 234, 259, 284, 249, 264, 271, 262, 270, 263, 250, 284, 259, 260, 266, 235, 252, 272, 258, 244, 280, 256, 256, 231, 222, 268, 252, 238, 256, 281, 279, 240, 265, 263, 268, 257, 244, 266, 237, 258, 268, 261, 257, 241, 254, 247, 248, 240, 264, 247, 240, 220, 240, 264, 247, 256, 252, 277, 272, 268, 258, 235, 272, 210, 256, 250, 254, 257, 266, 257, 263, 254, 276, 293, 261, 273, 243, 254, 247, 224, 258, 261, 233, 261, 276, 252, 236, 259, 269, 270, 279, 253, 240, 271, 287, 263, 251, 238, 274] payload_rows_per_dst=[5436, 5436, 5562, 5424, 5414, 5386, 5580, 5504] dup_rows_per_dst=[2582, 2768, 2836, 2774, 2644, 2676, 2660, 2854] combine_rows_per_src=[5436, 5436, 5562, 5424, 5414, 5386, 5580, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38324 dup_rows=21678 dup_rows_remote=18944 dedup_ratio=0.330780 router_tokens_per_expert=[246, 252, 256, 256, 248, 242, 247, 256, 237, 273, 250, 274, 273, 263, 261, 244, 237, 264, 243, 254, 286, 245, 283, 223, 274, 265, 266, 269, 232, 254, 227, 261, 235, 243, 268, 244, 234, 248, 258, 255, 262, 259, 271, 291, 246, 280, 243, 294, 263, 254, 258, 241, 259, 241, 264, 253, 250, 258, 284, 270, 274, 243, 269, 253, 205, 264, 253, 243, 258, 251, 259, 255, 289, 253, 252, 249, 246, 258, 238, 249, 223, 272, 264, 268, 244, 269, 233, 252, 261, 256, 258, 265, 220, 272, 251, 250, 243, 280, 265, 252, 263, 256, 263, 277, 260, 259, 270, 251, 255, 261, 273, 250, 255, 262, 264, 254, 250, 241, 234, 270, 260, 259, 256, 261, 256, 250, 261, 251] payload_rows_per_dst=[5490, 5436, 5480, 5534, 5464, 5432, 5564, 5458] dup_rows_per_dst=[2666, 2730, 2782, 2734, 2580, 2684, 2792, 2710] combine_rows_per_src=[5490, 5436, 5480, 5534, 5464, 5432, 5564, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38214 dup_rows=21910 dup_rows_remote=19144 dedup_ratio=0.334320 router_tokens_per_expert=[275, 270, 273, 231, 263, 257, 276, 249, 268, 245, 274, 257, 250, 261, 248, 260, 270, 228, 241, 282, 257, 276, 254, 281, 260, 250, 244, 285, 228, 246, 247, 295, 249, 248, 229, 228, 274, 296, 271, 236, 251, 265, 243, 242, 239, 248, 232, 246, 250, 271, 267, 249, 239, 245, 255, 245, 259, 249, 246, 260, 240, 280, 255, 252, 248, 260, 252, 254, 262, 273, 273, 242, 243, 251, 243, 251, 267, 243, 245, 248, 251, 266, 257, 243, 250, 245, 253, 247, 243, 277, 265, 250, 286, 260, 257, 239, 269, 295, 257, 240, 298, 244, 258, 245, 260, 272, 247, 247, 232, 239, 277, 258, 252, 268, 264, 250, 241, 240, 252, 275, 268, 256, 251, 252, 267, 278, 255, 257] payload_rows_per_dst=[5564, 5506, 5348, 5442, 5420, 5412, 5468, 5466] dup_rows_per_dst=[2750, 2782, 2646, 2682, 2690, 2766, 2808, 2786] combine_rows_per_src=[5564, 5506, 5348, 5442, 5420, 5412, 5468, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229328 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130704 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032080 weight_movement_bytes=18874368 total_nvlink_bytes=332906448 weight_to_token_ratio=0.060103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38418 dup_rows=21638 dup_rows_remote=18964 dedup_ratio=0.330170 router_tokens_per_expert=[245, 277, 242, 269, 280, 281, 233, 272, 229, 259, 267, 251, 245, 259, 251, 252, 234, 237, 253, 232, 266, 270, 253, 256, 258, 251, 245, 247, 276, 253, 288, 261, 265, 266, 262, 244, 282, 252, 279, 276, 279, 240, 236, 243, 290, 230, 255, 248, 233, 239, 265, 256, 249, 263, 252, 266, 251, 261, 263, 239, 301, 308, 235, 262, 265, 286, 251, 238, 254, 262, 253, 251, 269, 275, 270, 220, 268, 257, 255, 274, 267, 240, 249, 258, 249, 255, 250, 237, 275, 260, 277, 264, 259, 243, 250, 245, 249, 265, 253, 272, 273, 245, 230, 254, 269, 240, 268, 244, 249, 265, 240, 261, 244, 234, 266, 227, 238, 255, 264, 248, 261, 239, 254, 246, 260, 244, 251, 252] payload_rows_per_dst=[5510, 5522, 5530, 5464, 5518, 5480, 5480, 5394] dup_rows_per_dst=[2714, 2638, 2764, 2822, 2778, 2676, 2674, 2572] combine_rows_per_src=[5510, 5522, 5530, 5464, 5518, 5480, 5480, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38416 dup_rows=21666 dup_rows_remote=18966 dedup_ratio=0.330597 router_tokens_per_expert=[261, 253, 247, 252, 239, 262, 282, 256, 233, 245, 267, 259, 263, 226, 257, 280, 254, 241, 240, 240, 250, 264, 275, 267, 279, 268, 276, 249, 256, 274, 253, 234, 259, 242, 248, 253, 257, 268, 248, 232, 261, 256, 275, 247, 244, 241, 276, 270, 267, 253, 277, 271, 254, 238, 248, 257, 286, 255, 269, 228, 260, 250, 230, 271, 254, 267, 256, 271, 256, 249, 276, 256, 251, 261, 267, 238, 263, 276, 251, 231, 227, 297, 244, 259, 266, 258, 263, 259, 259, 261, 260, 272, 257, 271, 253, 259, 277, 229, 266, 243, 270, 247, 251, 241, 253, 272, 267, 267, 234, 263, 254, 258, 235, 253, 251, 238, 249, 250, 254, 248, 261, 259, 272, 244, 231, 246, 247, 257] payload_rows_per_dst=[5432, 5588, 5454, 5504, 5512, 5510, 5504, 5366] dup_rows_per_dst=[2732, 2652, 2700, 2724, 2734, 2820, 2680, 2624] combine_rows_per_src=[5432, 5588, 5454, 5504, 5512, 5510, 5504, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38298 dup_rows=21690 dup_rows_remote=18800 dedup_ratio=0.330963 router_tokens_per_expert=[245, 282, 249, 257, 253, 281, 259, 270, 262, 267, 257, 261, 276, 234, 246, 277, 244, 250, 269, 289, 268, 248, 278, 236, 274, 263, 258, 251, 278, 244, 260, 268, 254, 224, 226, 238, 246, 264, 282, 252, 282, 262, 256, 263, 261, 244, 244, 242, 258, 253, 247, 272, 238, 258, 257, 267, 247, 255, 264, 242, 273, 258, 256, 263, 229, 234, 245, 240, 239, 260, 256, 255, 256, 243, 241, 246, 240, 278, 260, 276, 238, 261, 260, 242, 244, 244, 270, 279, 239, 253, 245, 263, 259, 266, 289, 258, 245, 260, 243, 251, 252, 259, 250, 255, 243, 256, 260, 240, 269, 225, 255, 249, 269, 260, 247, 265, 245, 226, 278, 268, 286, 260, 249, 232, 270, 276, 253, 262] payload_rows_per_dst=[5548, 5608, 5446, 5494, 5404, 5494, 5460, 5392] dup_rows_per_dst=[2890, 2762, 2696, 2682, 2592, 2796, 2628, 2644] combine_rows_per_src=[5548, 5608, 5446, 5494, 5404, 5494, 5460, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44000 payload_rows_remote=38504 dup_rows=21536 dup_rows_remote=18818 dedup_ratio=0.328613 router_tokens_per_expert=[218, 251, 250, 269, 238, 236, 254, 264, 241, 229, 259, 261, 261, 238, 240, 271, 273, 270, 256, 257, 268, 235, 231, 229, 231, 270, 267, 241, 242, 249, 292, 235, 252, 262, 210, 239, 272, 277, 249, 232, 260, 250, 284, 248, 260, 277, 247, 285, 245, 249, 276, 236, 269, 256, 251, 280, 289, 239, 230, 268, 273, 271, 255, 286, 261, 262, 249, 262, 257, 273, 262, 240, 248, 239, 252, 266, 264, 281, 261, 230, 238, 273, 281, 237, 266, 280, 267, 261, 244, 229, 258, 261, 237, 279, 259, 263, 255, 250, 238, 233, 250, 241, 249, 266, 275, 259, 246, 258, 274, 252, 266, 241, 264, 249, 267, 259, 249, 254, 243, 265, 248, 261, 271, 248, 276, 258, 267, 293] payload_rows_per_dst=[5374, 5500, 5532, 5568, 5496, 5494, 5460, 5576] dup_rows_per_dst=[2586, 2592, 2676, 2778, 2718, 2772, 2646, 2768] combine_rows_per_src=[5374, 5500, 5532, 5568, 5496, 5494, 5460, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229072 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157204176 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314179280 weight_movement_bytes=37748736 total_nvlink_bytes=351928016 weight_to_token_ratio=0.120150 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38114 dup_rows=21876 dup_rows_remote=19050 dedup_ratio=0.333801 router_tokens_per_expert=[255, 258, 242, 222, 268, 283, 251, 272, 251, 248, 237, 245, 254, 270, 250, 266, 258, 251, 252, 260, 252, 272, 259, 262, 279, 235, 253, 263, 254, 281, 282, 263, 227, 267, 273, 246, 248, 259, 253, 266, 264, 286, 244, 264, 253, 274, 248, 242, 239, 262, 269, 272, 244, 228, 258, 247, 254, 260, 272, 271, 240, 250, 285, 262, 229, 277, 250, 258, 256, 260, 283, 239, 226, 266, 241, 219, 237, 244, 247, 222, 247, 232, 275, 249, 269, 271, 234, 246, 269, 245, 260, 247, 253, 259, 248, 268, 272, 239, 241, 297, 272, 219, 248, 229, 263, 282, 262, 247, 266, 257, 249, 268, 283, 247, 283, 257, 258, 271, 270, 263, 240, 242, 272, 249, 274, 259, 239, 249] payload_rows_per_dst=[5404, 5566, 5410, 5228, 5568, 5482, 5456, 5546] dup_rows_per_dst=[2740, 2786, 2726, 2428, 2864, 2740, 2766, 2826] combine_rows_per_src=[5404, 5566, 5410, 5228, 5568, 5482, 5456, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229432 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156753976 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278520 weight_movement_bytes=56623104 total_nvlink_bytes=369901624 weight_to_token_ratio=0.180744 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229528 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589656 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314949784 weight_movement_bytes=18874368 total_nvlink_bytes=333824152 weight_to_token_ratio=0.059928 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229528 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157581464 ingress_bytes=195100672 link_bytes=195100672 token_bytes=314933400 weight_movement_bytes=37748736 total_nvlink_bytes=352682136 weight_to_token_ratio=0.119863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869358644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=228392 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097000 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313965608 weight_movement_bytes=18874368 total_nvlink_bytes=332839976 weight_to_token_ratio=0.060116 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157712384 dispatch_meta_bytes=229288 combine_payload_bytes=157712384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157941672 ingress_bytes=176586752 link_bytes=176586752 token_bytes=315654056 weight_movement_bytes=18874368 total_nvlink_bytes=334528424 weight_to_token_ratio=0.059794 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691967372e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228656 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156343600 ingress_bytes=193863680 link_bytes=193863680 token_bytes=312458544 weight_movement_bytes=37748736 total_nvlink_bytes=350207280 weight_to_token_ratio=0.120812 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857506395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38464 dup_rows=21670 dup_rows_remote=19006 dedup_ratio=0.330658 router_tokens_per_expert=[229, 270, 272, 279, 265, 243, 252, 242, 256, 250, 285, 269, 252, 258, 258, 263, 242, 262, 273, 256, 263, 263, 250, 263, 249, 248, 243, 260, 235, 263, 228, 235, 278, 270, 245, 254, 240, 261, 246, 234, 270, 262, 260, 247, 219, 254, 258, 276, 261, 296, 250, 251, 265, 276, 245, 233, 273, 289, 256, 287, 268, 220, 247, 248, 268, 268, 237, 249, 246, 250, 242, 245, 236, 240, 230, 243, 284, 283, 245, 271, 263, 261, 251, 224, 236, 273, 251, 241, 259, 262, 265, 252, 232, 247, 264, 262, 241, 270, 229, 277, 249, 238, 259, 266, 237, 247, 266, 266, 242, 253, 269, 269, 247, 261, 252, 261, 258, 242, 268, 261, 278, 264, 284, 278, 268, 261, 251, 261] payload_rows_per_dst=[5520, 5402, 5418, 5558, 5468, 5438, 5448, 5614] dup_rows_per_dst=[2766, 2664, 2730, 2772, 2606, 2648, 2708, 2776] combine_rows_per_src=[5520, 5402, 5418, 5558, 5468, 5438, 5448, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38158 dup_rows=21862 dup_rows_remote=19110 dedup_ratio=0.333588 router_tokens_per_expert=[251, 263, 259, 264, 244, 221, 282, 259, 237, 252, 272, 277, 259, 242, 251, 264, 280, 247, 272, 255, 235, 267, 254, 235, 250, 236, 253, 225, 257, 263, 222, 250, 256, 233, 265, 243, 274, 272, 271, 257, 267, 272, 273, 245, 248, 269, 256, 257, 283, 277, 273, 264, 275, 238, 256, 271, 265, 259, 247, 241, 242, 255, 239, 249, 223, 242, 262, 242, 248, 246, 251, 266, 239, 251, 225, 234, 258, 264, 245, 253, 243, 246, 256, 259, 275, 245, 273, 265, 258, 266, 267, 251, 248, 247, 279, 267, 270, 234, 250, 272, 266, 267, 250, 263, 262, 246, 260, 278, 256, 261, 244, 257, 239, 267, 263, 278, 275, 270, 248, 279, 258, 245, 278, 237, 288, 233, 264, 226] payload_rows_per_dst=[5452, 5378, 5432, 5516, 5380, 5488, 5504, 5524] dup_rows_per_dst=[2742, 2624, 2884, 2752, 2518, 2802, 2768, 2772] combine_rows_per_src=[5452, 5378, 5432, 5516, 5380, 5488, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=229880 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157778424 ingress_bytes=195297280 link_bytes=195297280 token_bytes=315326968 weight_movement_bytes=37748736 total_nvlink_bytes=353075704 weight_to_token_ratio=0.119713 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871242445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38422 dup_rows=21622 dup_rows_remote=18970 dedup_ratio=0.329926 router_tokens_per_expert=[253, 233, 228, 267, 242, 257, 245, 250, 257, 256, 229, 259, 244, 284, 273, 254, 250, 270, 277, 246, 271, 271, 230, 229, 242, 255, 241, 263, 251, 254, 254, 263, 256, 253, 244, 257, 225, 268, 251, 268, 268, 246, 260, 249, 209, 266, 256, 238, 244, 263, 272, 258, 253, 270, 249, 277, 253, 255, 267, 261, 259, 248, 260, 251, 261, 256, 272, 281, 235, 259, 272, 283, 246, 256, 239, 242, 256, 278, 265, 263, 264, 239, 280, 246, 284, 263, 237, 273, 251, 252, 272, 237, 235, 235, 220, 284, 299, 276, 276, 249, 256, 279, 255, 278, 229, 248, 250, 255, 247, 272, 254, 243, 264, 270, 262, 250, 237, 246, 256, 258, 281, 245, 268, 266, 260, 246, 267, 238] payload_rows_per_dst=[5432, 5390, 5432, 5540, 5628, 5492, 5530, 5470] dup_rows_per_dst=[2630, 2744, 2596, 2740, 2700, 2652, 2802, 2758] combine_rows_per_src=[5432, 5390, 5432, 5540, 5628, 5492, 5530, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38182 dup_rows=21838 dup_rows_remote=19104 dedup_ratio=0.333221 router_tokens_per_expert=[251, 255, 260, 265, 267, 247, 265, 277, 232, 223, 245, 253, 271, 239, 276, 269, 256, 260, 287, 223, 252, 262, 255, 225, 264, 252, 277, 248, 281, 281, 267, 271, 269, 243, 268, 229, 255, 244, 224, 275, 273, 240, 235, 289, 274, 279, 239, 238, 248, 252, 260, 276, 240, 234, 249, 255, 240, 237, 290, 254, 247, 247, 273, 261, 250, 262, 260, 243, 243, 291, 243, 217, 262, 275, 271, 280, 256, 256, 254, 266, 262, 275, 263, 263, 244, 272, 263, 253, 263, 229, 240, 266, 275, 253, 245, 245, 268, 260, 262, 248, 244, 254, 219, 272, 227, 276, 261, 244, 285, 280, 263, 262, 237, 251, 246, 240, 242, 254, 259, 262, 243, 261, 244, 236, 256, 253, 278, 248] payload_rows_per_dst=[5470, 5528, 5432, 5464, 5420, 5444, 5516, 5424] dup_rows_per_dst=[2720, 2794, 2716, 2662, 2838, 2778, 2734, 2596] combine_rows_per_src=[5470, 5528, 5432, 5464, 5420, 5444, 5516, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229072 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156524240 ingress_bytes=231792640 link_bytes=231792640 token_bytes=312819408 weight_movement_bytes=75497472 total_nvlink_bytes=388316880 weight_to_token_ratio=0.241345 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220923027e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38126 dup_rows=21892 dup_rows_remote=19166 dedup_ratio=0.334045 router_tokens_per_expert=[217, 269, 247, 221, 247, 255, 262, 222, 270, 240, 287, 287, 262, 255, 269, 236, 251, 264, 255, 255, 262, 234, 264, 253, 307, 258, 254, 264, 267, 255, 272, 249, 260, 257, 266, 245, 245, 283, 253, 252, 276, 240, 247, 257, 252, 260, 263, 266, 261, 261, 262, 248, 272, 275, 256, 267, 230, 253, 281, 230, 257, 249, 278, 274, 253, 243, 262, 264, 267, 223, 247, 271, 256, 288, 250, 245, 242, 277, 269, 252, 254, 233, 281, 243, 265, 256, 234, 251, 237, 253, 243, 266, 244, 254, 276, 258, 254, 250, 245, 267, 258, 224, 253, 252, 271, 247, 282, 242, 244, 259, 276, 249, 255, 249, 266, 241, 267, 245, 269, 260, 237, 248, 267, 257, 245, 263, 234, 249] payload_rows_per_dst=[5416, 5502, 5518, 5454, 5484, 5464, 5410, 5396] dup_rows_per_dst=[2676, 2826, 2726, 2854, 2734, 2632, 2736, 2708] combine_rows_per_src=[5416, 5502, 5518, 5454, 5484, 5464, 5410, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38372 dup_rows=21696 dup_rows_remote=18978 dedup_ratio=0.331055 router_tokens_per_expert=[254, 265, 249, 258, 239, 247, 264, 261, 252, 279, 250, 255, 248, 256, 243, 249, 282, 257, 256, 253, 257, 242, 258, 224, 285, 244, 279, 249, 282, 279, 262, 237, 260, 259, 259, 235, 244, 243, 280, 267, 290, 253, 274, 252, 244, 244, 253, 265, 255, 258, 269, 278, 260, 255, 279, 269, 259, 263, 262, 250, 268, 252, 220, 259, 266, 267, 236, 226, 262, 264, 288, 254, 235, 270, 264, 266, 237, 249, 256, 253, 249, 237, 260, 268, 250, 248, 265, 271, 237, 253, 250, 268, 269, 249, 263, 250, 240, 285, 242, 244, 263, 275, 235, 221, 207, 262, 263, 255, 244, 228, 237, 263, 227, 260, 294, 255, 249, 249, 266, 262, 257, 236, 245, 259, 258, 269, 272, 273] payload_rows_per_dst=[5460, 5544, 5468, 5548, 5468, 5452, 5396, 5504] dup_rows_per_dst=[2678, 2748, 2776, 2764, 2718, 2722, 2532, 2758] combine_rows_per_src=[5460, 5544, 5468, 5548, 5468, 5452, 5396, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43500 payload_rows_remote=38100 dup_rows=22036 dup_rows_remote=19244 dedup_ratio=0.336243 router_tokens_per_expert=[257, 228, 269, 266, 248, 250, 257, 275, 239, 260, 263, 245, 258, 248, 245, 273, 256, 251, 290, 254, 246, 255, 255, 285, 256, 250, 244, 227, 260, 219, 301, 247, 250, 249, 249, 271, 238, 260, 254, 267, 252, 274, 248, 237, 266, 257, 252, 258, 252, 279, 254, 274, 242, 253, 245, 285, 248, 276, 246, 288, 231, 232, 269, 260, 249, 256, 260, 277, 242, 255, 255, 236, 239, 277, 241, 264, 228, 258, 254, 279, 217, 253, 256, 248, 264, 260, 265, 245, 257, 243, 272, 253, 234, 245, 259, 249, 255, 251, 258, 269, 269, 259, 243, 278, 275, 226, 261, 255, 274, 275, 239, 261, 248, 257, 254, 247, 244, 246, 251, 278, 280, 258, 256, 256, 247, 280, 261, 274] payload_rows_per_dst=[5400, 5390, 5402, 5452, 5448, 5402, 5524, 5482] dup_rows_per_dst=[2792, 2772, 2776, 2816, 2678, 2638, 2772, 2792] combine_rows_per_src=[5400, 5390, 5402, 5452, 5448, 5402, 5524, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43550 payload_rows_remote=38052 dup_rows=21986 dup_rows_remote=19146 dedup_ratio=0.335480 router_tokens_per_expert=[238, 250, 259, 256, 269, 248, 262, 249, 237, 258, 250, 247, 252, 267, 275, 232, 257, 250, 259, 284, 263, 215, 265, 260, 233, 265, 275, 246, 260, 248, 254, 253, 246, 264, 257, 256, 241, 233, 244, 254, 252, 254, 242, 255, 258, 255, 232, 242, 263, 238, 259, 281, 244, 283, 251, 258, 234, 243, 258, 281, 264, 272, 259, 287, 259, 233, 274, 268, 281, 277, 251, 234, 267, 269, 240, 264, 262, 233, 268, 226, 237, 257, 252, 253, 283, 233, 257, 235, 250, 265, 259, 249, 269, 230, 272, 229, 250, 251, 275, 259, 253, 253, 257, 249, 295, 255, 245, 263, 291, 271, 262, 238, 256, 243, 268, 264, 264, 279, 278, 252, 258, 238, 271, 259, 260, 248, 258, 273] payload_rows_per_dst=[5348, 5520, 5320, 5516, 5486, 5400, 5462, 5498] dup_rows_per_dst=[2610, 2654, 2566, 2834, 2810, 2704, 2968, 2840] combine_rows_per_src=[5348, 5520, 5320, 5516, 5486, 5400, 5462, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229568 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157606080 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982592 weight_movement_bytes=18874368 total_nvlink_bytes=333856960 weight_to_token_ratio=0.059922 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229144 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156622616 ingress_bytes=213016576 link_bytes=213016576 token_bytes=313016088 weight_movement_bytes=56623104 total_nvlink_bytes=369639192 weight_to_token_ratio=0.180895 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041020021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229168 combine_payload_bytes=156164096 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156393264 ingress_bytes=193912832 link_bytes=193912832 token_bytes=312557360 weight_movement_bytes=37748736 total_nvlink_bytes=350306096 weight_to_token_ratio=0.120774 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857977345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229400 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157401112 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572824 weight_movement_bytes=18874368 total_nvlink_bytes=333447192 weight_to_token_ratio=0.060000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=229376 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156286976 ingress_bytes=174931968 link_bytes=174931968 token_bytes=312344576 weight_movement_bytes=18874368 total_nvlink_bytes=331218944 weight_to_token_ratio=0.060428 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676112045e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155860992 dispatch_meta_bytes=228792 combine_payload_bytes=155860992 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156089784 ingress_bytes=193609728 link_bytes=193609728 token_bytes=311950776 weight_movement_bytes=37748736 total_nvlink_bytes=349699512 weight_to_token_ratio=0.121009 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855073152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38382 dup_rows=21706 dup_rows_remote=18924 dedup_ratio=0.331207 router_tokens_per_expert=[237, 243, 260, 250, 250, 263, 294, 265, 262, 256, 243, 276, 242, 270, 228, 250, 280, 264, 244, 247, 266, 272, 242, 256, 260, 261, 257, 260, 264, 249, 258, 235, 279, 253, 253, 263, 215, 272, 255, 241, 279, 237, 281, 261, 244, 269, 295, 272, 240, 242, 254, 218, 253, 282, 254, 244, 225, 261, 257, 261, 254, 239, 252, 259, 286, 264, 265, 241, 273, 230, 251, 254, 266, 255, 255, 262, 226, 245, 260, 266, 268, 280, 237, 254, 281, 269, 262, 260, 269, 246, 277, 217, 252, 274, 251, 249, 269, 265, 258, 235, 274, 266, 242, 276, 263, 229, 268, 245, 263, 232, 277, 262, 234, 261, 233, 291, 226, 249, 249, 265, 253, 233, 256, 233, 275, 230, 265, 278] payload_rows_per_dst=[5486, 5448, 5502, 5456, 5428, 5532, 5560, 5418] dup_rows_per_dst=[2692, 2782, 2836, 2534, 2770, 2760, 2688, 2644] combine_rows_per_src=[5486, 5448, 5502, 5456, 5428, 5532, 5560, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38226 dup_rows=21840 dup_rows_remote=19152 dedup_ratio=0.333252 router_tokens_per_expert=[247, 257, 251, 250, 260, 285, 258, 240, 244, 253, 281, 237, 249, 272, 260, 244, 266, 255, 270, 269, 264, 235, 264, 250, 248, 231, 270, 257, 219, 249, 261, 244, 252, 256, 233, 260, 286, 239, 256, 234, 243, 248, 273, 252, 260, 252, 245, 248, 266, 250, 263, 260, 270, 246, 232, 272, 245, 258, 230, 244, 261, 261, 258, 263, 267, 251, 243, 244, 263, 248, 252, 255, 248, 251, 238, 269, 265, 297, 259, 255, 254, 254, 286, 259, 274, 287, 262, 240, 243, 246, 247, 253, 238, 263, 260, 279, 244, 252, 265, 270, 261, 257, 234, 251, 233, 262, 258, 266, 242, 268, 271, 261, 257, 271, 262, 254, 254, 250, 250, 304, 261, 264, 253, 243, 252, 282, 247, 263] payload_rows_per_dst=[5444, 5420, 5384, 5470, 5486, 5460, 5466, 5566] dup_rows_per_dst=[2732, 2684, 2690, 2688, 2724, 2830, 2724, 2768] combine_rows_per_src=[5444, 5420, 5384, 5470, 5486, 5460, 5466, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229224 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157441896 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654568 weight_movement_bytes=18874368 total_nvlink_bytes=333528936 weight_to_token_ratio=0.059984 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38326 dup_rows=21768 dup_rows_remote=18974 dedup_ratio=0.332153 router_tokens_per_expert=[242, 258, 290, 247, 240, 225, 228, 247, 248, 236, 267, 252, 269, 267, 268, 235, 248, 255, 229, 218, 238, 233, 271, 251, 238, 249, 258, 255, 264, 243, 251, 229, 273, 293, 252, 242, 292, 243, 261, 257, 260, 252, 267, 245, 242, 283, 265, 257, 260, 260, 251, 263, 251, 251, 265, 273, 291, 245, 266, 263, 270, 259, 252, 282, 274, 240, 257, 239, 268, 257, 254, 258, 251, 253, 248, 259, 257, 247, 253, 268, 264, 267, 247, 269, 257, 244, 273, 253, 248, 245, 251, 275, 256, 262, 234, 273, 270, 274, 231, 250, 254, 258, 256, 269, 275, 231, 278, 290, 266, 251, 258, 224, 248, 286, 250, 267, 244, 255, 258, 248, 249, 239, 273, 250, 276, 255, 265, 234] payload_rows_per_dst=[5434, 5312, 5544, 5584, 5552, 5442, 5470, 5430] dup_rows_per_dst=[2604, 2548, 2824, 2820, 2614, 2794, 2800, 2764] combine_rows_per_src=[5434, 5312, 5544, 5584, 5552, 5442, 5470, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229512 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803208 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376904 weight_movement_bytes=18874368 total_nvlink_bytes=332251272 weight_to_token_ratio=0.060229 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38264 dup_rows=21830 dup_rows_remote=19176 dedup_ratio=0.333099 router_tokens_per_expert=[249, 269, 247, 308, 280, 278, 272, 278, 271, 268, 251, 268, 270, 283, 232, 258, 231, 242, 230, 256, 264, 248, 271, 275, 262, 257, 247, 256, 257, 236, 230, 252, 244, 243, 246, 266, 268, 268, 292, 255, 260, 267, 248, 254, 256, 237, 286, 257, 229, 223, 250, 275, 240, 246, 272, 246, 255, 267, 266, 250, 258, 242, 232, 256, 230, 256, 268, 259, 243, 241, 267, 271, 248, 258, 271, 223, 255, 271, 254, 273, 273, 257, 269, 258, 253, 243, 261, 256, 254, 269, 253, 267, 282, 224, 256, 273, 248, 255, 266, 267, 237, 217, 248, 260, 264, 238, 241, 255, 266, 262, 271, 253, 247, 267, 247, 252, 260, 266, 249, 248, 264, 270, 243, 242, 255, 262, 236, 226] payload_rows_per_dst=[5582, 5440, 5464, 5388, 5466, 5474, 5442, 5450] dup_rows_per_dst=[2982, 2588, 2826, 2626, 2714, 2822, 2654, 2618] combine_rows_per_src=[5582, 5440, 5464, 5388, 5466, 5474, 5442, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38358 dup_rows=21644 dup_rows_remote=19026 dedup_ratio=0.330261 router_tokens_per_expert=[263, 258, 229, 253, 247, 266, 269, 268, 259, 242, 242, 278, 260, 265, 216, 253, 252, 247, 261, 230, 257, 262, 275, 238, 240, 292, 262, 257, 288, 288, 249, 258, 258, 290, 244, 264, 270, 247, 260, 220, 250, 241, 258, 261, 240, 271, 265, 237, 228, 270, 237, 242, 265, 262, 253, 273, 258, 230, 241, 264, 264, 242, 251, 263, 235, 241, 262, 262, 236, 260, 292, 263, 246, 262, 273, 264, 249, 255, 232, 256, 269, 288, 238, 248, 267, 243, 298, 243, 255, 267, 241, 250, 248, 258, 261, 210, 232, 276, 261, 297, 237, 246, 251, 270, 265, 248, 245, 227, 265, 247, 259, 241, 275, 244, 256, 242, 285, 247, 245, 249, 271, 267, 250, 262, 269, 305, 246, 273] payload_rows_per_dst=[5448, 5546, 5534, 5406, 5496, 5474, 5452, 5536] dup_rows_per_dst=[2688, 2766, 2618, 2680, 2680, 2694, 2682, 2836] combine_rows_per_src=[5448, 5546, 5534, 5406, 5496, 5474, 5452, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43478 payload_rows_remote=38040 dup_rows=22058 dup_rows_remote=19306 dedup_ratio=0.336578 router_tokens_per_expert=[272, 264, 267, 272, 248, 262, 277, 261, 239, 258, 269, 246, 281, 244, 248, 248, 260, 254, 238, 251, 246, 243, 250, 277, 297, 276, 258, 260, 261, 262, 259, 266, 244, 255, 245, 252, 242, 258, 241, 259, 288, 261, 256, 236, 261, 287, 240, 250, 266, 236, 253, 233, 256, 249, 265, 241, 231, 248, 270, 288, 264, 283, 262, 233, 251, 278, 250, 253, 227, 243, 256, 259, 247, 264, 271, 276, 265, 266, 231, 258, 249, 260, 242, 245, 280, 230, 258, 225, 251, 254, 271, 252, 240, 261, 251, 238, 252, 286, 242, 262, 285, 252, 274, 261, 247, 269, 267, 253, 240, 287, 257, 263, 279, 251, 252, 231, 232, 234, 258, 264, 261, 257, 237, 240, 257, 247, 258, 244] payload_rows_per_dst=[5490, 5506, 5426, 5438, 5438, 5360, 5476, 5344] dup_rows_per_dst=[2822, 2810, 2724, 2718, 2752, 2654, 2918, 2660] combine_rows_per_src=[5490, 5506, 5426, 5438, 5438, 5360, 5476, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38020 dup_rows=21884 dup_rows_remote=18792 dedup_ratio=0.333923 router_tokens_per_expert=[251, 267, 237, 264, 267, 232, 238, 245, 256, 257, 242, 245, 270, 261, 242, 256, 271, 228, 275, 250, 253, 234, 262, 234, 255, 268, 252, 257, 288, 283, 265, 274, 223, 243, 247, 245, 221, 253, 260, 256, 277, 284, 268, 242, 294, 238, 237, 243, 255, 252, 261, 252, 271, 257, 258, 266, 258, 261, 264, 271, 244, 272, 256, 236, 243, 254, 264, 271, 249, 263, 266, 278, 248, 295, 238, 258, 278, 263, 268, 247, 270, 258, 270, 267, 246, 251, 238, 266, 233, 267, 267, 251, 257, 242, 237, 263, 265, 235, 244, 251, 273, 242, 250, 269, 275, 252, 229, 254, 248, 264, 251, 238, 255, 235, 245, 247, 229, 289, 253, 267, 275, 262, 263, 259, 258, 258, 259, 264] payload_rows_per_dst=[5402, 5546, 5218, 5496, 5424, 5464, 5470, 5632] dup_rows_per_dst=[2658, 2752, 2280, 2772, 2682, 2778, 2870, 3092] combine_rows_per_src=[5402, 5546, 5218, 5496, 5424, 5464, 5470, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38254 dup_rows=21712 dup_rows_remote=18910 dedup_ratio=0.331299 router_tokens_per_expert=[241, 230, 270, 269, 251, 266, 270, 237, 251, 278, 274, 258, 270, 271, 248, 245, 247, 297, 268, 251, 274, 255, 274, 213, 273, 271, 279, 241, 240, 251, 236, 253, 240, 269, 267, 273, 231, 260, 239, 276, 264, 229, 262, 244, 272, 229, 238, 281, 253, 288, 253, 244, 264, 269, 267, 266, 250, 253, 222, 247, 241, 263, 241, 265, 238, 251, 270, 255, 255, 239, 266, 229, 254, 256, 257, 277, 256, 257, 258, 228, 268, 251, 238, 266, 247, 248, 278, 259, 240, 247, 256, 257, 285, 255, 258, 252, 286, 278, 250, 278, 235, 267, 250, 266, 246, 237, 250, 229, 241, 253, 255, 248, 239, 281, 275, 256, 260, 248, 241, 241, 258, 256, 287, 277, 265, 240, 259, 253] payload_rows_per_dst=[5570, 5574, 5398, 5544, 5706, 5430, 5332, 5270] dup_rows_per_dst=[2802, 2784, 2638, 2840, 2904, 2586, 2674, 2484] combine_rows_per_src=[5570, 5574, 5398, 5544, 5706, 5430, 5332, 5270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229200 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157212496 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314195792 weight_movement_bytes=18874368 total_nvlink_bytes=333070160 weight_to_token_ratio=0.060072 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229760 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156959104 ingress_bytes=213352448 link_bytes=213352448 token_bytes=313688448 weight_movement_bytes=56623104 total_nvlink_bytes=370311552 weight_to_token_ratio=0.180507 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044238181e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=229536 combine_payload_bytes=157114368 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157343904 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314458272 weight_movement_bytes=18874368 total_nvlink_bytes=333332640 weight_to_token_ratio=0.060022 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155811840 dispatch_meta_bytes=229384 combine_payload_bytes=155811840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156041224 ingress_bytes=174686208 link_bytes=174686208 token_bytes=311853064 weight_movement_bytes=18874368 total_nvlink_bytes=330727432 weight_to_token_ratio=0.060523 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673757294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=227248 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155957168 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311687088 weight_movement_bytes=18874368 total_nvlink_bytes=330561456 weight_to_token_ratio=0.060556 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38194 dup_rows=21852 dup_rows_remote=19090 dedup_ratio=0.333435 router_tokens_per_expert=[268, 231, 237, 249, 272, 265, 262, 263, 244, 267, 272, 260, 263, 247, 276, 268, 267, 279, 253, 258, 248, 262, 249, 286, 241, 246, 248, 250, 270, 242, 261, 266, 239, 276, 249, 258, 287, 269, 255, 241, 224, 252, 259, 270, 269, 265, 241, 270, 274, 236, 259, 250, 256, 245, 259, 267, 264, 258, 259, 259, 263, 275, 249, 269, 271, 250, 249, 255, 242, 263, 274, 219, 261, 238, 221, 284, 246, 244, 249, 254, 265, 253, 250, 265, 268, 268, 282, 241, 258, 258, 254, 245, 259, 262, 246, 251, 255, 263, 240, 235, 295, 274, 251, 235, 249, 262, 274, 243, 264, 275, 252, 263, 225, 215, 253, 257, 263, 237, 244, 241, 277, 269, 256, 261, 252, 245, 222, 240] payload_rows_per_dst=[5494, 5490, 5542, 5452, 5396, 5510, 5440, 5360] dup_rows_per_dst=[2794, 2762, 2706, 2832, 2644, 2740, 2820, 2554] combine_rows_per_src=[5494, 5490, 5542, 5452, 5396, 5510, 5440, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228656 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917040 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605424 weight_movement_bytes=18874368 total_nvlink_bytes=332479792 weight_to_token_ratio=0.060185 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38300 dup_rows=21698 dup_rows_remote=18844 dedup_ratio=0.331085 router_tokens_per_expert=[243, 236, 276, 239, 247, 284, 260, 257, 264, 269, 257, 249, 277, 246, 271, 241, 261, 267, 270, 254, 253, 275, 263, 267, 257, 244, 246, 258, 229, 247, 242, 239, 236, 251, 273, 258, 255, 268, 225, 256, 256, 246, 283, 261, 248, 267, 253, 242, 271, 277, 274, 246, 271, 271, 264, 255, 281, 241, 237, 240, 242, 261, 280, 285, 247, 239, 260, 232, 255, 270, 254, 247, 264, 257, 254, 270, 265, 263, 257, 258, 275, 255, 231, 237, 256, 241, 268, 212, 235, 258, 265, 276, 245, 229, 242, 254, 269, 286, 247, 243, 234, 263, 239, 248, 262, 252, 269, 286, 247, 256, 261, 286, 282, 277, 238, 234, 260, 254, 276, 253, 244, 265, 243, 262, 243, 246, 248, 262] payload_rows_per_dst=[5430, 5390, 5576, 5538, 5524, 5400, 5500, 5480] dup_rows_per_dst=[2802, 2754, 2580, 2854, 2660, 2558, 2796, 2694] combine_rows_per_src=[5430, 5390, 5576, 5538, 5524, 5400, 5500, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229136 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671760 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114384 weight_movement_bytes=18874368 total_nvlink_bytes=331988752 weight_to_token_ratio=0.060279 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=228576 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157105376 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313982176 weight_movement_bytes=18874368 total_nvlink_bytes=332856544 weight_to_token_ratio=0.060113 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38548 dup_rows=21660 dup_rows_remote=18978 dedup_ratio=0.330505 router_tokens_per_expert=[265, 259, 265, 254, 276, 261, 271, 241, 260, 255, 255, 249, 241, 268, 270, 260, 260, 256, 257, 261, 249, 256, 231, 262, 266, 262, 237, 247, 256, 249, 273, 256, 273, 249, 238, 267, 272, 254, 253, 239, 248, 242, 253, 264, 284, 241, 255, 282, 260, 277, 252, 272, 277, 261, 289, 222, 267, 274, 238, 239, 269, 260, 262, 263, 265, 238, 264, 247, 253, 236, 253, 254, 265, 242, 245, 248, 251, 266, 257, 225, 262, 237, 246, 267, 248, 258, 276, 255, 257, 251, 280, 273, 272, 245, 247, 228, 265, 245, 238, 251, 250, 246, 239, 267, 229, 254, 240, 259, 267, 248, 247, 260, 244, 251, 273, 250, 274, 280, 256, 249, 267, 250, 251, 275, 245, 257, 262, 244] payload_rows_per_dst=[5502, 5534, 5560, 5526, 5458, 5514, 5328, 5454] dup_rows_per_dst=[2798, 2622, 2668, 2704, 2694, 2690, 2682, 2802] combine_rows_per_src=[5502, 5534, 5560, 5526, 5458, 5514, 5328, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38218 dup_rows=21844 dup_rows_remote=19022 dedup_ratio=0.333313 router_tokens_per_expert=[279, 264, 253, 240, 279, 260, 253, 238, 275, 253, 253, 269, 266, 308, 275, 238, 275, 253, 259, 239, 261, 252, 251, 250, 271, 225, 265, 263, 276, 269, 272, 289, 242, 269, 247, 252, 242, 244, 280, 249, 261, 272, 265, 260, 256, 253, 247, 257, 242, 245, 272, 253, 294, 250, 281, 259, 238, 254, 273, 234, 235, 253, 247, 245, 244, 246, 255, 223, 224, 249, 245, 261, 259, 251, 267, 242, 250, 242, 262, 266, 277, 279, 270, 272, 276, 287, 258, 259, 241, 235, 263, 264, 244, 213, 261, 249, 242, 263, 246, 271, 248, 266, 209, 262, 244, 271, 255, 248, 256, 228, 259, 240, 270, 258, 210, 264, 248, 272, 259, 260, 276, 235, 275, 253, 248, 235, 236, 283] payload_rows_per_dst=[5584, 5538, 5514, 5406, 5334, 5474, 5392, 5450] dup_rows_per_dst=[2822, 2802, 2678, 2744, 2638, 2822, 2624, 2714] combine_rows_per_src=[5584, 5538, 5514, 5406, 5334, 5474, 5392, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38266 dup_rows=21680 dup_rows_remote=18796 dedup_ratio=0.330811 router_tokens_per_expert=[261, 234, 243, 248, 252, 264, 247, 254, 243, 253, 245, 238, 256, 255, 273, 256, 261, 263, 281, 228, 234, 260, 251, 257, 270, 240, 262, 257, 271, 261, 231, 263, 267, 258, 262, 270, 259, 249, 257, 287, 268, 286, 259, 266, 264, 279, 257, 249, 244, 245, 289, 248, 260, 234, 251, 232, 271, 285, 245, 266, 251, 244, 240, 254, 261, 278, 260, 255, 280, 247, 268, 251, 275, 243, 254, 276, 260, 261, 274, 256, 257, 275, 256, 264, 243, 228, 265, 252, 253, 247, 269, 257, 258, 254, 258, 237, 251, 251, 254, 247, 270, 236, 252, 241, 254, 272, 250, 261, 257, 255, 242, 242, 237, 257, 254, 265, 265, 283, 253, 252, 252, 250, 247, 254, 224, 258, 238, 264] payload_rows_per_dst=[5438, 5484, 5590, 5452, 5544, 5482, 5412, 5454] dup_rows_per_dst=[2606, 2696, 2884, 2666, 2854, 2664, 2658, 2652] combine_rows_per_src=[5438, 5484, 5590, 5452, 5544, 5482, 5412, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38306 dup_rows=21720 dup_rows_remote=18796 dedup_ratio=0.331421 router_tokens_per_expert=[236, 242, 256, 264, 270, 261, 251, 254, 270, 254, 234, 250, 247, 244, 277, 273, 241, 254, 229, 252, 273, 238, 261, 226, 243, 238, 246, 271, 268, 245, 238, 235, 266, 275, 246, 251, 262, 250, 256, 238, 260, 280, 254, 247, 247, 240, 242, 272, 300, 258, 273, 257, 238, 267, 243, 247, 275, 245, 252, 228, 223, 245, 248, 256, 276, 275, 266, 255, 277, 259, 271, 228, 277, 264, 245, 291, 247, 269, 247, 270, 256, 280, 267, 254, 275, 235, 236, 276, 285, 257, 239, 258, 265, 240, 244, 260, 265, 291, 279, 267, 270, 277, 232, 227, 259, 274, 261, 242, 232, 264, 249, 267, 246, 233, 253, 246, 265, 256, 288, 275, 274, 225, 229, 266, 262, 255, 251, 262] payload_rows_per_dst=[5458, 5358, 5480, 5430, 5510, 5552, 5500, 5528] dup_rows_per_dst=[2708, 2558, 2692, 2680, 2924, 2702, 2812, 2644] combine_rows_per_src=[5458, 5358, 5480, 5430, 5510, 5552, 5500, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38402 dup_rows=21664 dup_rows_remote=18942 dedup_ratio=0.330566 router_tokens_per_expert=[269, 209, 259, 295, 269, 283, 265, 252, 256, 261, 239, 248, 274, 234, 258, 264, 256, 255, 241, 243, 247, 278, 261, 240, 253, 245, 246, 233, 246, 296, 263, 238, 257, 254, 253, 276, 266, 256, 271, 267, 254, 264, 254, 229, 247, 236, 241, 268, 269, 241, 251, 261, 273, 279, 246, 258, 271, 234, 274, 278, 268, 260, 247, 270, 252, 251, 243, 236, 272, 245, 255, 284, 256, 240, 245, 258, 238, 246, 266, 242, 282, 270, 265, 266, 284, 252, 262, 266, 279, 250, 238, 252, 246, 236, 267, 277, 270, 260, 281, 253, 267, 269, 266, 270, 220, 246, 273, 232, 241, 279, 274, 234, 242, 241, 267, 279, 256, 263, 248, 235, 237, 238, 245, 236, 237, 243, 237, 259] payload_rows_per_dst=[5284, 5684, 5406, 5556, 5452, 5488, 5532, 5470] dup_rows_per_dst=[2482, 2902, 2642, 2708, 2744, 2630, 2834, 2722] combine_rows_per_src=[5284, 5684, 5406, 5556, 5452, 5488, 5532, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38088 dup_rows=21960 dup_rows_remote=19080 dedup_ratio=0.335083 router_tokens_per_expert=[275, 249, 238, 247, 239, 273, 249, 268, 259, 250, 246, 245, 255, 263, 253, 275, 264, 252, 270, 246, 259, 278, 236, 235, 242, 224, 273, 253, 261, 297, 272, 233, 260, 262, 263, 246, 280, 251, 256, 246, 269, 264, 268, 278, 243, 232, 262, 255, 248, 223, 241, 255, 251, 259, 260, 237, 285, 274, 264, 254, 278, 255, 239, 238, 248, 247, 223, 260, 257, 272, 259, 264, 254, 251, 255, 238, 254, 244, 282, 238, 271, 282, 273, 262, 265, 245, 270, 238, 259, 231, 259, 274, 239, 243, 275, 262, 245, 224, 253, 291, 265, 248, 250, 254, 252, 261, 238, 261, 250, 260, 256, 270, 239, 271, 271, 263, 268, 268, 274, 246, 251, 259, 247, 262, 243, 226, 265, 268] payload_rows_per_dst=[5488, 5444, 5556, 5410, 5458, 5456, 5324, 5440] dup_rows_per_dst=[2880, 2828, 2900, 2688, 2634, 2668, 2574, 2788] combine_rows_per_src=[5488, 5444, 5556, 5410, 5458, 5456, 5324, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157892608 dispatch_meta_bytes=230104 combine_payload_bytes=157892608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158122712 ingress_bytes=176766976 link_bytes=176766976 token_bytes=316015320 weight_movement_bytes=18874368 total_nvlink_bytes=334889688 weight_to_token_ratio=0.059726 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693694189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228960 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156769888 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313310816 weight_movement_bytes=18874368 total_nvlink_bytes=332185184 weight_to_token_ratio=0.060242 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228248 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156965784 ingress_bytes=213360640 link_bytes=213360640 token_bytes=313703320 weight_movement_bytes=56623104 total_nvlink_bytes=370326424 weight_to_token_ratio=0.180499 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044316672e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38092 dup_rows=21964 dup_rows_remote=19142 dedup_ratio=0.335144 router_tokens_per_expert=[256, 248, 261, 267, 241, 251, 255, 270, 255, 256, 254, 245, 269, 249, 279, 249, 281, 251, 259, 276, 229, 274, 276, 258, 233, 263, 252, 279, 261, 253, 266, 240, 278, 243, 250, 240, 219, 280, 252, 256, 240, 245, 275, 244, 249, 227, 243, 269, 261, 272, 248, 254, 245, 257, 259, 241, 231, 224, 243, 247, 256, 280, 255, 268, 230, 227, 296, 261, 250, 272, 224, 242, 258, 284, 288, 276, 284, 272, 260, 268, 257, 233, 260, 249, 259, 249, 255, 265, 245, 220, 261, 244, 229, 277, 262, 252, 258, 241, 278, 254, 242, 265, 243, 252, 287, 278, 249, 235, 244, 257, 242, 273, 289, 267, 260, 240, 286, 277, 238, 244, 259, 253, 250, 247, 269, 255, 255, 265] payload_rows_per_dst=[5512, 5480, 5350, 5428, 5516, 5336, 5460, 5490] dup_rows_per_dst=[2698, 2822, 2670, 2654, 2868, 2698, 2736, 2818] combine_rows_per_src=[5512, 5480, 5350, 5428, 5516, 5336, 5460, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228408 combine_payload_bytes=156901376 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=157129784 ingress_bytes=251273216 link_bytes=251273216 token_bytes=314031160 weight_movement_bytes=94371840 total_nvlink_bytes=408403000 weight_to_token_ratio=0.300517 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.407576322e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157294592 dispatch_meta_bytes=229376 combine_payload_bytes=157294592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157523968 ingress_bytes=176168960 link_bytes=176168960 token_bytes=314818560 weight_movement_bytes=18874368 total_nvlink_bytes=333692928 weight_to_token_ratio=0.059953 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687964294e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=228672 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156237120 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312245568 weight_movement_bytes=18874368 total_nvlink_bytes=331119936 weight_to_token_ratio=0.060447 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38246 dup_rows=21732 dup_rows_remote=18956 dedup_ratio=0.331604 router_tokens_per_expert=[249, 262, 261, 285, 255, 249, 265, 270, 252, 234, 261, 283, 241, 253, 265, 251, 245, 234, 264, 256, 258, 257, 245, 253, 245, 263, 233, 271, 221, 230, 255, 281, 259, 237, 269, 273, 252, 275, 273, 269, 282, 268, 259, 271, 238, 254, 270, 245, 285, 267, 241, 255, 227, 259, 253, 269, 228, 278, 264, 263, 261, 263, 278, 276, 249, 215, 257, 255, 255, 261, 248, 249, 226, 251, 275, 291, 257, 256, 221, 253, 258, 255, 256, 249, 236, 249, 309, 236, 276, 249, 252, 255, 257, 278, 246, 281, 268, 256, 271, 253, 246, 258, 253, 268, 270, 259, 269, 259, 248, 260, 266, 226, 241, 247, 241, 268, 241, 263, 257, 257, 233, 249, 231, 241, 257, 247, 265, 231] payload_rows_per_dst=[5494, 5422, 5550, 5558, 5386, 5508, 5498, 5388] dup_rows_per_dst=[2778, 2600, 2838, 2776, 2652, 2776, 2762, 2550] combine_rows_per_src=[5494, 5422, 5550, 5558, 5386, 5508, 5498, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=228936 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253768 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312278600 weight_movement_bytes=18874368 total_nvlink_bytes=331152968 weight_to_token_ratio=0.060441 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228808 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156884424 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540040 weight_movement_bytes=37748736 total_nvlink_bytes=351288776 weight_to_token_ratio=0.120395 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38464 dup_rows=21620 dup_rows_remote=19014 dedup_ratio=0.329895 router_tokens_per_expert=[267, 269, 288, 279, 238, 255, 259, 251, 273, 259, 259, 286, 269, 243, 247, 284, 255, 239, 249, 235, 259, 273, 248, 245, 280, 240, 250, 264, 267, 259, 272, 233, 278, 254, 263, 260, 282, 244, 268, 273, 257, 233, 243, 251, 287, 260, 242, 265, 231, 241, 244, 241, 249, 253, 219, 266, 270, 241, 257, 255, 254, 247, 245, 240, 243, 239, 271, 291, 234, 250, 258, 252, 249, 265, 244, 258, 265, 220, 248, 257, 261, 230, 275, 259, 273, 300, 272, 248, 260, 274, 253, 248, 259, 235, 248, 276, 250, 249, 236, 275, 259, 259, 249, 239, 237, 236, 244, 256, 249, 259, 264, 268, 259, 265, 241, 232, 225, 260, 269, 253, 262, 256, 257, 260, 272, 278, 268, 260] payload_rows_per_dst=[5588, 5526, 5578, 5382, 5404, 5534, 5452, 5452] dup_rows_per_dst=[2864, 2610, 2742, 2524, 2684, 2808, 2606, 2782] combine_rows_per_src=[5588, 5526, 5578, 5382, 5404, 5534, 5452, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38202 dup_rows=21806 dup_rows_remote=19096 dedup_ratio=0.332733 router_tokens_per_expert=[250, 247, 243, 269, 250, 242, 267, 281, 240, 267, 277, 260, 251, 231, 249, 279, 255, 255, 233, 254, 255, 223, 245, 245, 276, 257, 256, 255, 246, 279, 236, 227, 253, 266, 252, 257, 273, 257, 252, 251, 259, 274, 255, 244, 240, 303, 267, 278, 238, 262, 235, 253, 252, 237, 264, 259, 240, 248, 242, 233, 253, 269, 256, 266, 249, 256, 253, 251, 278, 250, 286, 261, 265, 267, 268, 263, 252, 265, 250, 246, 253, 250, 233, 248, 273, 271, 261, 309, 235, 260, 251, 241, 247, 263, 260, 264, 249, 244, 276, 248, 270, 278, 261, 259, 248, 256, 231, 247, 249, 256, 269, 270, 229, 274, 253, 254, 279, 261, 246, 266, 242, 266, 270, 254, 255, 234, 272, 235] payload_rows_per_dst=[5446, 5392, 5562, 5390, 5494, 5528, 5496, 5422] dup_rows_per_dst=[2760, 2602, 2800, 2624, 2826, 2710, 2726, 2758] combine_rows_per_src=[5446, 5392, 5562, 5390, 5494, 5528, 5496, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38162 dup_rows=21944 dup_rows_remote=19204 dedup_ratio=0.334839 router_tokens_per_expert=[249, 246, 250, 251, 255, 253, 270, 264, 271, 244, 248, 269, 253, 252, 244, 259, 251, 259, 250, 247, 265, 241, 265, 269, 236, 281, 274, 248, 245, 258, 259, 256, 249, 268, 231, 282, 260, 256, 257, 269, 256, 246, 275, 211, 260, 256, 262, 247, 230, 266, 255, 279, 222, 278, 281, 250, 252, 246, 255, 234, 259, 266, 245, 225, 271, 244, 243, 250, 275, 258, 225, 240, 255, 256, 236, 272, 266, 254, 249, 270, 267, 254, 252, 270, 267, 240, 240, 263, 232, 264, 268, 278, 267, 259, 261, 256, 243, 274, 229, 246, 265, 254, 272, 259, 251, 277, 246, 267, 265, 239, 287, 254, 280, 253, 268, 255, 255, 237, 261, 254, 269, 272, 241, 286, 245, 257, 260, 235] payload_rows_per_dst=[5476, 5500, 5430, 5350, 5464, 5468, 5406, 5498] dup_rows_per_dst=[2680, 2708, 2740, 2736, 2664, 2808, 2850, 2758] combine_rows_per_src=[5476, 5500, 5430, 5350, 5464, 5468, 5406, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43488 payload_rows_remote=37998 dup_rows=22048 dup_rows_remote=19386 dedup_ratio=0.336426 router_tokens_per_expert=[284, 279, 260, 259, 283, 243, 247, 264, 261, 277, 241, 259, 257, 254, 257, 228, 253, 276, 264, 272, 303, 271, 264, 250, 269, 263, 284, 262, 280, 284, 250, 276, 229, 257, 245, 264, 269, 268, 237, 260, 222, 241, 246, 232, 263, 258, 254, 273, 256, 272, 269, 246, 295, 264, 270, 263, 262, 251, 263, 255, 272, 272, 239, 245, 241, 271, 254, 226, 247, 232, 250, 264, 250, 267, 257, 259, 240, 266, 264, 258, 211, 267, 249, 251, 259, 251, 236, 266, 250, 241, 253, 250, 255, 263, 251, 272, 246, 265, 254, 263, 264, 240, 238, 272, 235, 251, 278, 260, 255, 267, 261, 277, 265, 238, 240, 240, 252, 261, 241, 223, 240, 239, 257, 232, 223, 247, 258, 229] payload_rows_per_dst=[5468, 5392, 5226, 5564, 5502, 5378, 5468, 5490] dup_rows_per_dst=[2700, 2772, 2530, 2946, 2870, 2688, 2880, 2662] combine_rows_per_src=[5468, 5392, 5226, 5564, 5502, 5378, 5468, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38066 dup_rows=21928 dup_rows_remote=19214 dedup_ratio=0.334595 router_tokens_per_expert=[241, 270, 254, 239, 256, 277, 244, 266, 251, 257, 279, 229, 279, 245, 268, 250, 262, 283, 237, 245, 247, 269, 269, 272, 245, 261, 250, 232, 266, 242, 219, 258, 246, 252, 255, 244, 280, 260, 252, 262, 236, 261, 272, 261, 232, 266, 269, 258, 276, 256, 280, 243, 277, 234, 271, 264, 235, 254, 231, 221, 253, 261, 279, 275, 237, 250, 250, 238, 265, 290, 265, 241, 267, 262, 249, 271, 245, 270, 247, 281, 266, 268, 241, 266, 260, 233, 254, 252, 252, 270, 246, 258, 253, 283, 226, 277, 266, 226, 286, 250, 273, 278, 266, 243, 259, 276, 231, 270, 247, 236, 232, 248, 241, 240, 240, 288, 247, 254, 261, 273, 252, 273, 248, 249, 264, 237, 253, 250] payload_rows_per_dst=[5426, 5456, 5508, 5404, 5542, 5420, 5432, 5420] dup_rows_per_dst=[2784, 2658, 2704, 2816, 2714, 2790, 2742, 2720] combine_rows_per_src=[5426, 5456, 5508, 5404, 5542, 5420, 5432, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=229912 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157778456 ingress_bytes=195297280 link_bytes=195297280 token_bytes=315327000 weight_movement_bytes=37748736 total_nvlink_bytes=353075736 weight_to_token_ratio=0.119713 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871242445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38304 dup_rows=21718 dup_rows_remote=18912 dedup_ratio=0.331390 router_tokens_per_expert=[265, 248, 267, 242, 258, 243, 266, 254, 244, 237, 257, 241, 257, 244, 266, 251, 251, 253, 287, 244, 249, 229, 257, 236, 260, 299, 246, 248, 279, 260, 270, 311, 274, 233, 254, 242, 237, 263, 256, 239, 271, 238, 244, 280, 273, 257, 270, 265, 255, 250, 250, 251, 250, 260, 267, 260, 253, 263, 272, 266, 280, 231, 245, 267, 287, 244, 238, 235, 243, 273, 270, 266, 237, 239, 269, 250, 273, 263, 281, 256, 228, 271, 272, 286, 244, 236, 266, 280, 276, 255, 243, 251, 277, 256, 254, 265, 263, 235, 255, 247, 259, 269, 274, 267, 247, 266, 242, 233, 258, 271, 248, 264, 242, 236, 240, 259, 267, 236, 231, 243, 253, 247, 234, 269, 246, 246, 255, 247] payload_rows_per_dst=[5514, 5466, 5538, 5486, 5496, 5522, 5588, 5208] dup_rows_per_dst=[2806, 2740, 2654, 2666, 2752, 2798, 2882, 2420] combine_rows_per_src=[5514, 5466, 5538, 5486, 5496, 5522, 5588, 5208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229192 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156704584 ingress_bytes=213098496 link_bytes=213098496 token_bytes=313179976 weight_movement_bytes=56623104 total_nvlink_bytes=369803080 weight_to_token_ratio=0.180801 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041804938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38374 dup_rows=21750 dup_rows_remote=18958 dedup_ratio=0.331879 router_tokens_per_expert=[250, 265, 231, 262, 280, 269, 250, 285, 251, 245, 235, 263, 258, 256, 244, 243, 250, 245, 271, 239, 226, 255, 265, 290, 259, 263, 258, 279, 233, 254, 287, 228, 251, 256, 241, 261, 272, 261, 260, 232, 275, 258, 256, 267, 271, 268, 229, 260, 245, 279, 265, 267, 231, 250, 236, 265, 246, 265, 246, 289, 267, 260, 269, 279, 246, 268, 263, 240, 260, 242, 254, 244, 270, 259, 258, 250, 244, 279, 254, 277, 263, 260, 265, 265, 232, 265, 246, 233, 251, 258, 281, 246, 233, 279, 252, 235, 273, 254, 266, 249, 253, 247, 240, 256, 238, 266, 250, 245, 262, 249, 236, 250, 257, 263, 269, 265, 256, 247, 251, 251, 270, 265, 269, 239, 252, 244, 235, 263] payload_rows_per_dst=[5510, 5412, 5492, 5546, 5482, 5416, 5376, 5552] dup_rows_per_dst=[2664, 2792, 2744, 2772, 2734, 2712, 2692, 2640] combine_rows_per_src=[5510, 5412, 5492, 5546, 5482, 5416, 5376, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229464 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156541016 ingress_bytes=194060288 link_bytes=194060288 token_bytes=312852568 weight_movement_bytes=37748736 total_nvlink_bytes=350601304 weight_to_token_ratio=0.120660 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859390196e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155639808 dispatch_meta_bytes=229536 combine_payload_bytes=155639808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155869344 ingress_bytes=174514176 link_bytes=174514176 token_bytes=311509152 weight_movement_bytes=18874368 total_nvlink_bytes=330383520 weight_to_token_ratio=0.060590 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672108968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38466 dup_rows=21624 dup_rows_remote=19116 dedup_ratio=0.329956 router_tokens_per_expert=[236, 280, 238, 248, 234, 260, 245, 269, 247, 255, 256, 248, 272, 261, 262, 259, 258, 292, 289, 274, 251, 261, 265, 261, 250, 266, 246, 271, 266, 255, 279, 246, 254, 256, 242, 289, 254, 235, 281, 252, 256, 255, 239, 264, 247, 257, 240, 274, 283, 231, 222, 258, 237, 244, 274, 252, 267, 253, 253, 233, 228, 242, 259, 241, 254, 241, 290, 254, 229, 248, 254, 262, 275, 259, 262, 246, 276, 261, 251, 259, 250, 245, 283, 265, 241, 254, 268, 264, 254, 247, 288, 261, 284, 248, 269, 268, 252, 256, 271, 249, 253, 285, 254, 222, 269, 231, 258, 280, 265, 248, 260, 255, 246, 248, 241, 248, 270, 277, 238, 235, 236, 255, 250, 219, 213, 273, 283, 246] payload_rows_per_dst=[5462, 5576, 5450, 5446, 5546, 5544, 5498, 5390] dup_rows_per_dst=[2678, 2884, 2740, 2508, 2696, 2834, 2718, 2566] combine_rows_per_src=[5462, 5576, 5450, 5446, 5546, 5544, 5498, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=229120 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156147456 ingress_bytes=174792704 link_bytes=174792704 token_bytes=312065792 weight_movement_bytes=18874368 total_nvlink_bytes=330940160 weight_to_token_ratio=0.060482 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674777686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228864 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122048 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015232 weight_movement_bytes=18874368 total_nvlink_bytes=332889600 weight_to_token_ratio=0.060107 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229328 combine_payload_bytes=157179904 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157409232 ingress_bytes=213803008 link_bytes=213803008 token_bytes=314589136 weight_movement_bytes=56623104 total_nvlink_bytes=371212240 weight_to_token_ratio=0.179991 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048555225e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=230328 combine_payload_bytes=157556736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157787064 ingress_bytes=176431104 link_bytes=176431104 token_bytes=315343800 weight_movement_bytes=18874368 total_nvlink_bytes=334218168 weight_to_token_ratio=0.059853 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690476029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38178 dup_rows=21884 dup_rows_remote=19166 dedup_ratio=0.333923 router_tokens_per_expert=[232, 254, 272, 267, 224, 267, 281, 247, 262, 223, 253, 255, 246, 248, 238, 251, 265, 238, 242, 236, 243, 278, 241, 272, 289, 258, 273, 258, 243, 261, 265, 241, 265, 251, 284, 258, 248, 249, 265, 238, 258, 253, 296, 295, 273, 263, 267, 284, 255, 261, 262, 224, 262, 242, 267, 240, 236, 243, 272, 242, 247, 210, 265, 257, 236, 269, 275, 260, 259, 252, 255, 280, 284, 266, 267, 243, 259, 276, 232, 252, 264, 262, 245, 253, 298, 265, 249, 262, 241, 243, 265, 278, 247, 247, 261, 231, 267, 240, 265, 252, 270, 264, 242, 248, 252, 260, 249, 263, 247, 253, 266, 258, 263, 252, 239, 265, 230, 270, 262, 245, 268, 280, 256, 229, 243, 246, 255, 238] payload_rows_per_dst=[5464, 5482, 5546, 5358, 5498, 5428, 5474, 5402] dup_rows_per_dst=[2576, 2724, 2948, 2612, 2832, 2794, 2718, 2680] combine_rows_per_src=[5464, 5482, 5546, 5358, 5498, 5428, 5474, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38280 dup_rows=21826 dup_rows_remote=19208 dedup_ratio=0.333038 router_tokens_per_expert=[246, 258, 235, 253, 272, 263, 263, 262, 269, 269, 265, 265, 238, 247, 252, 279, 244, 274, 274, 261, 274, 247, 256, 248, 261, 275, 259, 268, 243, 227, 273, 283, 266, 244, 270, 281, 240, 256, 265, 257, 249, 277, 230, 253, 242, 264, 277, 250, 270, 226, 228, 233, 245, 232, 282, 263, 253, 232, 274, 271, 275, 255, 274, 254, 249, 256, 265, 272, 269, 247, 272, 290, 251, 263, 250, 239, 244, 249, 251, 256, 263, 263, 255, 234, 223, 270, 244, 247, 249, 267, 258, 253, 245, 249, 250, 254, 258, 238, 261, 262, 218, 274, 278, 241, 263, 259, 245, 254, 240, 241, 266, 238, 251, 269, 268, 245, 222, 277, 260, 310, 246, 264, 242, 237, 210, 260, 269, 264] payload_rows_per_dst=[5462, 5538, 5476, 5430, 5474, 5430, 5418, 5482] dup_rows_per_dst=[2810, 2796, 2766, 2704, 2772, 2618, 2654, 2706] combine_rows_per_src=[5462, 5538, 5476, 5430, 5474, 5430, 5418, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38086 dup_rows=21954 dup_rows_remote=19134 dedup_ratio=0.334991 router_tokens_per_expert=[244, 261, 239, 242, 233, 255, 298, 267, 239, 266, 252, 251, 265, 265, 240, 258, 220, 284, 262, 249, 258, 259, 264, 256, 231, 229, 273, 264, 234, 248, 260, 207, 268, 272, 284, 264, 252, 260, 257, 238, 289, 274, 272, 240, 241, 241, 259, 247, 250, 270, 245, 250, 268, 240, 255, 257, 289, 277, 274, 263, 247, 233, 290, 262, 257, 256, 265, 253, 259, 251, 260, 257, 256, 254, 240, 241, 252, 240, 247, 270, 267, 260, 245, 268, 261, 270, 243, 270, 225, 213, 287, 261, 270, 234, 261, 242, 263, 261, 279, 253, 259, 265, 239, 254, 264, 242, 279, 256, 230, 291, 261, 271, 268, 252, 270, 244, 233, 260, 255, 230, 265, 242, 269, 257, 239, 268, 243, 270] payload_rows_per_dst=[5378, 5428, 5496, 5536, 5410, 5426, 5506, 5402] dup_rows_per_dst=[2772, 2568, 2820, 2804, 2706, 2728, 2828, 2728] combine_rows_per_src=[5378, 5428, 5496, 5536, 5410, 5426, 5506, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43934 payload_rows_remote=38438 dup_rows=21602 dup_rows_remote=18732 dedup_ratio=0.329620 router_tokens_per_expert=[249, 255, 234, 249, 258, 245, 250, 237, 274, 241, 267, 251, 244, 260, 246, 272, 256, 239, 265, 273, 256, 251, 274, 256, 286, 265, 238, 250, 263, 248, 241, 259, 271, 239, 252, 261, 246, 257, 233, 255, 246, 226, 275, 275, 253, 257, 256, 262, 266, 282, 237, 255, 304, 284, 259, 234, 265, 247, 269, 237, 247, 233, 259, 252, 247, 285, 251, 284, 257, 262, 257, 265, 261, 238, 260, 255, 282, 268, 252, 253, 251, 248, 256, 263, 252, 247, 256, 231, 286, 249, 236, 259, 252, 253, 261, 272, 249, 263, 252, 252, 237, 261, 259, 267, 240, 203, 261, 259, 271, 260, 238, 244, 256, 265, 269, 261, 292, 277, 260, 257, 241, 258, 228, 287, 257, 275, 247, 227] payload_rows_per_dst=[5398, 5540, 5478, 5552, 5534, 5484, 5452, 5496] dup_rows_per_dst=[2666, 2688, 2650, 2708, 2778, 2662, 2580, 2870] combine_rows_per_src=[5398, 5540, 5478, 5552, 5534, 5484, 5452, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43978 payload_rows_remote=38586 dup_rows=21558 dup_rows_remote=18994 dedup_ratio=0.328949 router_tokens_per_expert=[249, 257, 248, 268, 248, 245, 263, 237, 258, 263, 248, 280, 254, 234, 270, 254, 238, 276, 261, 226, 237, 269, 245, 248, 224, 248, 258, 276, 276, 280, 267, 270, 248, 247, 271, 277, 249, 243, 276, 247, 269, 268, 253, 220, 272, 255, 237, 266, 257, 241, 260, 236, 246, 284, 245, 239, 277, 294, 289, 256, 222, 233, 267, 245, 244, 264, 241, 223, 260, 260, 238, 259, 258, 230, 253, 234, 256, 260, 272, 226, 255, 271, 263, 260, 256, 294, 265, 252, 246, 291, 254, 237, 258, 256, 255, 252, 222, 235, 265, 255, 280, 272, 265, 247, 268, 258, 259, 263, 274, 251, 257, 265, 251, 248, 265, 263, 241, 251, 274, 245, 286, 259, 264, 254, 274, 247, 259, 244] payload_rows_per_dst=[5422, 5512, 5482, 5530, 5392, 5554, 5566, 5520] dup_rows_per_dst=[2730, 2686, 2714, 2652, 2564, 2776, 2706, 2730] combine_rows_per_src=[5422, 5512, 5482, 5530, 5392, 5554, 5566, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38106 dup_rows=21938 dup_rows_remote=19144 dedup_ratio=0.334747 router_tokens_per_expert=[263, 261, 275, 254, 259, 239, 267, 284, 261, 277, 241, 260, 242, 240, 252, 267, 253, 243, 244, 249, 261, 239, 269, 253, 238, 258, 255, 253, 240, 281, 249, 244, 242, 259, 288, 261, 264, 260, 244, 264, 264, 264, 250, 236, 233, 261, 266, 268, 259, 231, 248, 271, 275, 275, 259, 272, 283, 230, 275, 246, 264, 250, 245, 239, 255, 253, 259, 281, 257, 261, 262, 256, 238, 240, 258, 247, 252, 247, 275, 265, 241, 251, 259, 245, 249, 248, 258, 249, 250, 252, 236, 230, 282, 275, 270, 250, 228, 254, 236, 239, 273, 269, 293, 252, 254, 266, 271, 258, 257, 256, 225, 248, 264, 247, 246, 255, 289, 255, 255, 277, 289, 260, 255, 255, 256, 229, 251, 238] payload_rows_per_dst=[5492, 5514, 5476, 5560, 5432, 5434, 5312, 5378] dup_rows_per_dst=[2794, 2890, 2772, 2926, 2780, 2656, 2500, 2620] combine_rows_per_src=[5492, 5514, 5476, 5560, 5432, 5434, 5312, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229376 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606464 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983552 weight_movement_bytes=18874368 total_nvlink_bytes=331857920 weight_to_token_ratio=0.060305 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229952 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157024832 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313819712 weight_movement_bytes=37748736 total_nvlink_bytes=351568448 weight_to_token_ratio=0.120288 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38426 dup_rows=21690 dup_rows_remote=18958 dedup_ratio=0.330963 router_tokens_per_expert=[234, 267, 251, 248, 273, 231, 280, 272, 241, 279, 245, 255, 274, 240, 266, 285, 264, 276, 271, 266, 251, 285, 223, 252, 254, 298, 262, 234, 244, 262, 215, 219, 250, 251, 243, 235, 279, 237, 249, 274, 246, 254, 276, 245, 242, 263, 251, 272, 277, 269, 284, 261, 268, 251, 252, 226, 250, 249, 241, 270, 255, 261, 254, 245, 260, 242, 254, 244, 256, 270, 264, 234, 246, 256, 260, 234, 252, 236, 266, 276, 280, 289, 272, 246, 258, 272, 265, 258, 274, 273, 255, 246, 244, 270, 221, 254, 291, 265, 268, 257, 247, 276, 238, 228, 276, 279, 252, 246, 246, 284, 251, 238, 240, 248, 279, 247, 241, 247, 253, 233, 253, 237, 248, 231, 259, 267, 259, 260] payload_rows_per_dst=[5478, 5420, 5454, 5534, 5492, 5562, 5488, 5418] dup_rows_per_dst=[2804, 2732, 2680, 2692, 2608, 2792, 2796, 2586] combine_rows_per_src=[5478, 5420, 5454, 5534, 5492, 5562, 5488, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=228880 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156229136 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312229392 weight_movement_bytes=37748736 total_nvlink_bytes=349978128 weight_to_token_ratio=0.120901 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=228680 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157670728 ingress_bytes=214065152 link_bytes=214065152 token_bytes=315112776 weight_movement_bytes=56623104 total_nvlink_bytes=371735880 weight_to_token_ratio=0.179692 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051066960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38264 dup_rows=21828 dup_rows_remote=19210 dedup_ratio=0.333069 router_tokens_per_expert=[264, 219, 245, 234, 233, 244, 267, 277, 255, 263, 267, 256, 222, 253, 241, 280, 252, 265, 246, 269, 264, 269, 254, 261, 249, 284, 253, 255, 245, 225, 255, 282, 279, 250, 247, 267, 228, 262, 256, 254, 224, 258, 286, 261, 247, 225, 252, 238, 236, 260, 249, 241, 249, 259, 259, 255, 262, 240, 246, 272, 245, 245, 260, 253, 259, 255, 280, 235, 264, 263, 262, 238, 270, 247, 258, 255, 238, 246, 251, 279, 255, 260, 256, 240, 279, 240, 278, 275, 274, 253, 251, 253, 244, 266, 239, 260, 247, 230, 268, 256, 260, 257, 276, 258, 272, 251, 278, 275, 292, 254, 232, 239, 272, 279, 260, 270, 260, 257, 289, 251, 248, 274, 290, 244, 266, 233, 246, 248] payload_rows_per_dst=[5398, 5450, 5370, 5444, 5454, 5468, 5550, 5574] dup_rows_per_dst=[2642, 2806, 2698, 2618, 2746, 2778, 2740, 2800] combine_rows_per_src=[5398, 5450, 5370, 5444, 5454, 5468, 5550, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=158048256 dispatch_meta_bytes=230320 combine_payload_bytes=158048256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158278576 ingress_bytes=176922624 link_bytes=176922624 token_bytes=316326832 weight_movement_bytes=18874368 total_nvlink_bytes=335201200 weight_to_token_ratio=0.059667 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.695185532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229000 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156311176 ingress_bytes=212705280 link_bytes=212705280 token_bytes=312393352 weight_movement_bytes=56623104 total_nvlink_bytes=369016456 weight_to_token_ratio=0.181256 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038037335e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157392896 dispatch_meta_bytes=229536 combine_payload_bytes=157392896 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157622432 ingress_bytes=195141632 link_bytes=195141632 token_bytes=315015328 weight_movement_bytes=37748736 total_nvlink_bytes=352764064 weight_to_token_ratio=0.119831 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869751102e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229896 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156959240 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688584 weight_movement_bytes=18874368 total_nvlink_bytes=332562952 weight_to_token_ratio=0.060169 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38106 dup_rows=21980 dup_rows_remote=19188 dedup_ratio=0.335388 router_tokens_per_expert=[238, 277, 290, 244, 247, 269, 268, 273, 263, 252, 261, 239, 267, 247, 269, 251, 260, 241, 262, 243, 229, 243, 269, 247, 266, 258, 251, 268, 245, 248, 262, 235, 275, 239, 232, 258, 285, 262, 258, 251, 252, 273, 237, 274, 276, 249, 240, 255, 246, 260, 264, 251, 250, 281, 260, 256, 259, 253, 263, 256, 255, 287, 242, 253, 259, 256, 231, 233, 259, 232, 233, 257, 260, 233, 267, 249, 247, 286, 231, 256, 253, 217, 246, 260, 268, 261, 248, 261, 272, 247, 234, 268, 250, 246, 278, 236, 257, 239, 270, 271, 276, 257, 264, 281, 257, 229, 259, 250, 262, 279, 251, 219, 259, 281, 261, 253, 249, 260, 299, 245, 263, 267, 271, 279, 276, 230, 249, 237] payload_rows_per_dst=[5494, 5414, 5490, 5408, 5368, 5428, 5450, 5504] dup_rows_per_dst=[2718, 2738, 2742, 2864, 2610, 2662, 2792, 2854] combine_rows_per_src=[5494, 5414, 5490, 5408, 5368, 5428, 5450, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38334 dup_rows=21792 dup_rows_remote=19058 dedup_ratio=0.332520 router_tokens_per_expert=[243, 260, 259, 247, 239, 241, 253, 261, 247, 262, 252, 240, 227, 244, 251, 246, 251, 256, 243, 249, 254, 216, 277, 286, 256, 289, 274, 239, 266, 281, 249, 267, 244, 276, 251, 258, 251, 269, 237, 264, 272, 245, 243, 280, 242, 239, 263, 248, 281, 255, 251, 252, 252, 243, 237, 248, 275, 249, 249, 242, 259, 302, 245, 269, 275, 266, 262, 230, 244, 273, 238, 243, 269, 244, 275, 255, 303, 269, 261, 255, 232, 271, 265, 254, 252, 251, 234, 255, 265, 259, 250, 256, 276, 252, 251, 249, 263, 269, 247, 252, 276, 245, 265, 254, 263, 223, 239, 275, 253, 248, 281, 250, 243, 265, 254, 258, 282, 242, 226, 260, 240, 258, 266, 265, 275, 255, 244, 282] payload_rows_per_dst=[5356, 5514, 5406, 5530, 5552, 5410, 5484, 5492] dup_rows_per_dst=[2588, 2792, 2758, 2688, 2772, 2734, 2722, 2738] combine_rows_per_src=[5356, 5514, 5406, 5530, 5552, 5410, 5484, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38336 dup_rows=21766 dup_rows_remote=19106 dedup_ratio=0.332123 router_tokens_per_expert=[276, 252, 249, 247, 269, 275, 256, 265, 258, 251, 243, 218, 275, 259, 247, 253, 242, 243, 232, 248, 213, 259, 253, 243, 268, 283, 289, 278, 277, 271, 266, 265, 255, 271, 270, 269, 265, 241, 252, 237, 240, 249, 262, 225, 251, 250, 246, 264, 231, 273, 244, 258, 261, 243, 236, 252, 234, 243, 269, 243, 250, 269, 272, 266, 269, 240, 247, 272, 246, 256, 267, 248, 222, 270, 244, 265, 257, 240, 269, 251, 282, 273, 229, 244, 273, 259, 252, 246, 275, 255, 263, 265, 273, 256, 254, 290, 249, 283, 243, 244, 247, 258, 257, 247, 254, 261, 230, 263, 252, 292, 256, 243, 248, 258, 253, 245, 249, 259, 260, 265, 256, 256, 282, 287, 265, 254, 252, 234] payload_rows_per_dst=[5390, 5478, 5434, 5388, 5448, 5578, 5520, 5534] dup_rows_per_dst=[2796, 2782, 2660, 2700, 2678, 2800, 2638, 2712] combine_rows_per_src=[5390, 5478, 5434, 5388, 5448, 5578, 5520, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38126 dup_rows=21746 dup_rows_remote=18734 dedup_ratio=0.331818 router_tokens_per_expert=[245, 286, 276, 239, 217, 269, 246, 241, 236, 255, 277, 259, 278, 253, 277, 259, 257, 242, 257, 246, 243, 254, 251, 287, 252, 253, 237, 252, 252, 256, 246, 266, 271, 236, 244, 234, 266, 257, 262, 274, 244, 232, 278, 246, 245, 289, 246, 241, 247, 270, 241, 245, 240, 259, 243, 248, 236, 257, 255, 255, 264, 266, 233, 278, 273, 269, 254, 254, 264, 259, 269, 241, 265, 243, 268, 259, 247, 234, 254, 256, 268, 251, 252, 251, 283, 251, 291, 258, 272, 248, 255, 252, 256, 232, 245, 240, 215, 232, 269, 276, 238, 274, 241, 264, 273, 281, 245, 270, 256, 289, 284, 267, 253, 245, 263, 256, 260, 255, 264, 246, 260, 279, 239, 251, 264, 272, 238, 269] payload_rows_per_dst=[5388, 5460, 5386, 5382, 5464, 5484, 5562, 5664] dup_rows_per_dst=[2616, 2642, 2518, 2620, 2802, 2726, 2810, 3012] combine_rows_per_src=[5388, 5460, 5386, 5382, 5464, 5484, 5562, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38278 dup_rows=21840 dup_rows_remote=19220 dedup_ratio=0.333252 router_tokens_per_expert=[262, 273, 258, 246, 245, 240, 234, 231, 255, 297, 245, 258, 278, 250, 253, 292, 246, 225, 231, 257, 252, 250, 250, 253, 254, 240, 255, 262, 244, 274, 269, 270, 214, 255, 271, 247, 248, 233, 261, 282, 239, 239, 255, 270, 280, 253, 248, 269, 254, 271, 273, 257, 240, 235, 250, 219, 250, 247, 289, 278, 272, 238, 249, 239, 274, 256, 271, 224, 235, 273, 215, 229, 270, 258, 256, 234, 253, 251, 265, 255, 256, 261, 268, 251, 253, 255, 268, 243, 269, 267, 264, 251, 265, 244, 265, 281, 273, 257, 305, 265, 261, 227, 268, 250, 246, 262, 253, 259, 261, 258, 251, 262, 262, 254, 278, 263, 234, 276, 246, 261, 269, 279, 250, 260, 249, 257, 254, 264] payload_rows_per_dst=[5524, 5410, 5432, 5444, 5418, 5504, 5480, 5484] dup_rows_per_dst=[2710, 2654, 2696, 2678, 2620, 2818, 2836, 2828] combine_rows_per_src=[5524, 5410, 5432, 5444, 5418, 5504, 5480, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43454 payload_rows_remote=38010 dup_rows=22082 dup_rows_remote=19226 dedup_ratio=0.336945 router_tokens_per_expert=[263, 257, 217, 269, 244, 261, 266, 279, 265, 252, 240, 240, 264, 264, 266, 242, 240, 256, 249, 278, 254, 229, 285, 247, 289, 248, 243, 263, 243, 261, 247, 265, 240, 251, 266, 243, 264, 251, 268, 254, 257, 285, 264, 264, 281, 246, 240, 262, 265, 253, 270, 234, 241, 257, 232, 261, 251, 269, 263, 271, 283, 231, 251, 244, 261, 245, 265, 277, 268, 266, 252, 247, 297, 255, 247, 257, 273, 261, 243, 247, 239, 267, 260, 250, 251, 269, 228, 270, 253, 233, 276, 274, 278, 269, 240, 227, 273, 237, 259, 266, 244, 271, 249, 267, 233, 250, 270, 235, 251, 243, 237, 240, 297, 255, 261, 233, 220, 249, 265, 254, 244, 279, 266, 271, 252, 235, 257, 262] payload_rows_per_dst=[5444, 5472, 5492, 5402, 5366, 5362, 5386, 5530] dup_rows_per_dst=[2856, 2728, 2810, 2842, 2706, 2806, 2664, 2670] combine_rows_per_src=[5444, 5472, 5492, 5402, 5366, 5362, 5386, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229176 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311352 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393528 weight_movement_bytes=37748736 total_nvlink_bytes=350142264 weight_to_token_ratio=0.120837 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38162 dup_rows=21886 dup_rows_remote=19182 dedup_ratio=0.333954 router_tokens_per_expert=[250, 254, 252, 242, 245, 255, 251, 253, 275, 265, 228, 278, 260, 288, 269, 283, 270, 242, 243, 247, 251, 268, 251, 264, 269, 260, 253, 258, 268, 227, 265, 260, 262, 249, 267, 251, 265, 254, 265, 264, 242, 279, 245, 280, 242, 243, 250, 238, 245, 275, 235, 252, 220, 294, 244, 295, 277, 232, 257, 277, 270, 255, 244, 239, 257, 288, 245, 261, 255, 253, 246, 247, 257, 252, 257, 271, 235, 227, 245, 266, 250, 255, 272, 263, 246, 251, 252, 250, 229, 249, 275, 244, 237, 268, 257, 240, 272, 255, 247, 229, 247, 276, 266, 261, 268, 241, 272, 258, 246, 252, 260, 259, 229, 253, 264, 265, 229, 246, 254, 257, 249, 269, 262, 285, 257, 290, 257, 242] payload_rows_per_dst=[5496, 5488, 5466, 5430, 5398, 5388, 5496, 5488] dup_rows_per_dst=[2800, 2704, 2726, 2792, 2726, 2688, 2722, 2728] combine_rows_per_src=[5496, 5488, 5466, 5430, 5398, 5388, 5496, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229768 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157254024 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314278280 weight_movement_bytes=18874368 total_nvlink_bytes=333152648 weight_to_token_ratio=0.060056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38060 dup_rows=22002 dup_rows_remote=19242 dedup_ratio=0.335724 router_tokens_per_expert=[257, 226, 236, 251, 257, 275, 241, 263, 263, 258, 299, 281, 234, 269, 257, 238, 242, 277, 260, 267, 268, 259, 266, 246, 243, 241, 281, 251, 254, 234, 271, 251, 214, 269, 266, 247, 271, 251, 259, 224, 238, 262, 249, 269, 237, 278, 258, 245, 271, 243, 251, 262, 259, 279, 262, 226, 249, 282, 283, 243, 263, 245, 255, 244, 269, 258, 266, 249, 239, 250, 261, 258, 252, 256, 254, 282, 247, 277, 259, 252, 257, 253, 252, 290, 245, 248, 253, 260, 257, 282, 262, 257, 250, 227, 261, 281, 242, 234, 270, 250, 266, 258, 244, 293, 249, 239, 242, 225, 258, 280, 243, 269, 248, 251, 258, 240, 257, 279, 249, 268, 235, 268, 258, 224, 254, 256, 267, 260] payload_rows_per_dst=[5460, 5456, 5366, 5474, 5466, 5490, 5386, 5436] dup_rows_per_dst=[2750, 2766, 2708, 2760, 2792, 2780, 2738, 2708] combine_rows_per_src=[5460, 5456, 5366, 5474, 5466, 5490, 5386, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229568 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157245632 ingress_bytes=213639168 link_bytes=213639168 token_bytes=314261696 weight_movement_bytes=56623104 total_nvlink_bytes=370884800 weight_to_token_ratio=0.180178 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046985391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=227440 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156391536 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312555632 weight_movement_bytes=18874368 total_nvlink_bytes=331430000 weight_to_token_ratio=0.060387 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229992 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016680 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803368 weight_movement_bytes=37748736 total_nvlink_bytes=351552104 weight_to_token_ratio=0.120294 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=228944 combine_payload_bytes=155688960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155917904 ingress_bytes=174563328 link_bytes=174563328 token_bytes=311606864 weight_movement_bytes=18874368 total_nvlink_bytes=330481232 weight_to_token_ratio=0.060571 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672579918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229376 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540928 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312852480 weight_movement_bytes=18874368 total_nvlink_bytes=331726848 weight_to_token_ratio=0.060330 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229208 combine_payload_bytes=155893760 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156122968 ingress_bytes=212516864 link_bytes=212516864 token_bytes=312016728 weight_movement_bytes=56623104 total_nvlink_bytes=368639832 weight_to_token_ratio=0.181475 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036232026e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38288 dup_rows=21714 dup_rows_remote=19038 dedup_ratio=0.331329 router_tokens_per_expert=[247, 266, 254, 271, 263, 266, 277, 272, 274, 256, 246, 261, 251, 236, 276, 244, 275, 245, 260, 246, 252, 256, 255, 213, 239, 252, 253, 245, 265, 261, 234, 258, 247, 257, 253, 249, 231, 276, 271, 265, 259, 258, 259, 240, 249, 238, 272, 270, 233, 260, 286, 227, 221, 270, 235, 273, 288, 268, 261, 213, 255, 290, 237, 256, 269, 225, 260, 259, 250, 274, 233, 266, 260, 248, 267, 253, 264, 254, 249, 254, 240, 272, 247, 226, 283, 245, 245, 256, 278, 258, 240, 246, 251, 252, 246, 263, 270, 278, 275, 278, 247, 245, 255, 257, 230, 268, 261, 233, 246, 259, 251, 252, 277, 282, 257, 254, 261, 273, 283, 228, 238, 260, 279, 246, 272, 266, 268, 250] payload_rows_per_dst=[5500, 5402, 5464, 5474, 5516, 5394, 5534, 5538] dup_rows_per_dst=[2820, 2616, 2724, 2672, 2654, 2702, 2676, 2850] combine_rows_per_src=[5500, 5402, 5464, 5474, 5516, 5394, 5534, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38284 dup_rows=21736 dup_rows_remote=18980 dedup_ratio=0.331665 router_tokens_per_expert=[251, 269, 285, 260, 248, 251, 254, 250, 259, 251, 279, 256, 256, 264, 239, 285, 249, 242, 239, 241, 256, 260, 245, 246, 237, 267, 259, 272, 247, 253, 260, 243, 282, 262, 261, 253, 260, 250, 255, 255, 284, 247, 254, 264, 269, 253, 239, 248, 265, 242, 281, 241, 236, 248, 235, 264, 252, 234, 263, 276, 250, 242, 256, 279, 245, 249, 272, 267, 292, 236, 291, 272, 265, 237, 257, 245, 238, 253, 251, 249, 260, 266, 259, 253, 265, 252, 272, 278, 251, 277, 263, 239, 244, 277, 229, 229, 274, 233, 284, 232, 266, 263, 249, 245, 233, 276, 255, 261, 235, 255, 245, 271, 262, 260, 282, 263, 236, 261, 257, 243, 236, 271, 274, 244, 265, 252, 237, 242] payload_rows_per_dst=[5516, 5406, 5516, 5440, 5462, 5534, 5474, 5452] dup_rows_per_dst=[2798, 2626, 2756, 2688, 2776, 2694, 2680, 2718] combine_rows_per_src=[5516, 5406, 5516, 5440, 5462, 5534, 5474, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38258 dup_rows=21804 dup_rows_remote=19132 dedup_ratio=0.332703 router_tokens_per_expert=[253, 240, 241, 275, 267, 269, 246, 232, 256, 237, 246, 259, 251, 255, 241, 258, 265, 273, 257, 254, 244, 260, 266, 258, 257, 273, 272, 248, 245, 235, 254, 268, 248, 252, 247, 267, 261, 259, 269, 245, 224, 255, 287, 250, 238, 272, 272, 243, 270, 234, 273, 248, 260, 246, 252, 246, 242, 292, 251, 264, 254, 280, 278, 268, 268, 267, 255, 251, 271, 250, 251, 271, 269, 241, 264, 247, 249, 226, 236, 241, 263, 274, 275, 258, 246, 256, 234, 238, 252, 242, 291, 256, 227, 271, 233, 257, 289, 254, 278, 270, 254, 272, 275, 273, 243, 243, 254, 252, 278, 257, 261, 242, 242, 257, 249, 248, 244, 256, 251, 247, 276, 259, 254, 222, 257, 251, 276, 252] payload_rows_per_dst=[5352, 5494, 5480, 5556, 5428, 5474, 5512, 5436] dup_rows_per_dst=[2700, 2764, 2698, 2760, 2686, 2672, 2878, 2646] combine_rows_per_src=[5352, 5494, 5480, 5556, 5428, 5474, 5512, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38248 dup_rows=21836 dup_rows_remote=19190 dedup_ratio=0.333191 router_tokens_per_expert=[307, 248, 247, 254, 258, 228, 238, 270, 261, 247, 260, 275, 235, 286, 220, 274, 238, 257, 271, 269, 234, 249, 242, 256, 263, 251, 219, 267, 281, 248, 251, 251, 234, 285, 248, 258, 257, 264, 270, 243, 258, 301, 249, 276, 266, 246, 242, 263, 246, 236, 244, 251, 262, 266, 251, 260, 241, 256, 262, 250, 235, 270, 237, 281, 250, 255, 229, 263, 274, 255, 255, 254, 266, 262, 265, 246, 240, 273, 231, 259, 254, 240, 234, 262, 263, 279, 259, 239, 251, 307, 248, 260, 249, 259, 259, 248, 269, 246, 262, 263, 258, 276, 286, 250, 236, 277, 255, 255, 269, 258, 240, 277, 270, 233, 226, 233, 250, 239, 250, 260, 263, 281, 266, 265, 245, 276, 229, 254] payload_rows_per_dst=[5344, 5406, 5508, 5530, 5368, 5348, 5744, 5452] dup_rows_per_dst=[2538, 2688, 2812, 2900, 2786, 2384, 3082, 2646] combine_rows_per_src=[5344, 5406, 5508, 5530, 5368, 5348, 5744, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38260 dup_rows=21848 dup_rows_remote=19054 dedup_ratio=0.333374 router_tokens_per_expert=[219, 233, 269, 255, 265, 233, 264, 262, 264, 247, 286, 252, 284, 262, 244, 254, 261, 290, 240, 289, 260, 241, 268, 260, 248, 260, 264, 243, 234, 287, 261, 233, 257, 267, 266, 275, 273, 252, 255, 250, 256, 294, 240, 239, 256, 265, 249, 298, 251, 255, 236, 247, 261, 232, 256, 247, 274, 262, 259, 280, 258, 258, 262, 251, 252, 264, 262, 269, 258, 245, 256, 262, 270, 253, 256, 238, 250, 269, 267, 240, 268, 281, 264, 225, 229, 239, 256, 256, 228, 255, 240, 263, 223, 256, 253, 245, 258, 266, 242, 261, 246, 244, 258, 233, 249, 267, 274, 296, 253, 272, 243, 236, 282, 247, 257, 241, 266, 244, 249, 261, 259, 253, 262, 239, 253, 256, 233, 263] payload_rows_per_dst=[5514, 5520, 5526, 5360, 5428, 5364, 5468, 5508] dup_rows_per_dst=[2672, 2758, 2858, 2818, 2794, 2598, 2728, 2622] combine_rows_per_src=[5514, 5520, 5526, 5360, 5428, 5364, 5468, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38346 dup_rows=21794 dup_rows_remote=19008 dedup_ratio=0.332550 router_tokens_per_expert=[256, 281, 242, 262, 282, 228, 252, 252, 260, 261, 262, 246, 256, 274, 240, 245, 237, 274, 253, 252, 259, 250, 258, 258, 270, 258, 259, 243, 261, 259, 249, 251, 276, 282, 258, 258, 266, 264, 235, 256, 256, 231, 266, 260, 279, 243, 259, 274, 237, 259, 270, 263, 262, 249, 267, 281, 274, 258, 254, 277, 240, 247, 238, 259, 252, 223, 230, 251, 257, 258, 274, 241, 251, 225, 236, 276, 245, 254, 246, 269, 261, 258, 238, 283, 285, 274, 263, 249, 273, 269, 278, 239, 247, 278, 256, 261, 246, 263, 245, 263, 220, 260, 262, 250, 256, 239, 260, 276, 240, 252, 235, 277, 253, 285, 271, 251, 265, 231, 257, 253, 264, 266, 236, 239, 254, 233, 221, 257] payload_rows_per_dst=[5506, 5396, 5512, 5476, 5378, 5548, 5470, 5456] dup_rows_per_dst=[2692, 2786, 2814, 2794, 2598, 2876, 2618, 2616] combine_rows_per_src=[5506, 5396, 5512, 5476, 5378, 5548, 5470, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229304 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056952 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884600 weight_movement_bytes=18874368 total_nvlink_bytes=332758968 weight_to_token_ratio=0.060132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38222 dup_rows=21766 dup_rows_remote=18810 dedup_ratio=0.332123 router_tokens_per_expert=[248, 261, 258, 273, 256, 245, 256, 259, 257, 251, 264, 265, 264, 277, 265, 246, 254, 258, 249, 236, 248, 268, 260, 241, 249, 248, 263, 256, 258, 244, 257, 290, 244, 259, 269, 257, 246, 234, 255, 244, 247, 259, 271, 243, 276, 246, 257, 230, 279, 259, 253, 272, 277, 254, 229, 258, 256, 230, 269, 253, 263, 238, 270, 257, 270, 241, 269, 252, 257, 293, 250, 262, 236, 271, 271, 238, 255, 260, 252, 241, 259, 262, 276, 270, 290, 278, 250, 263, 252, 236, 268, 249, 271, 239, 254, 256, 237, 271, 246, 257, 235, 245, 266, 247, 255, 232, 266, 259, 261, 225, 248, 267, 272, 252, 255, 259, 255, 239, 258, 247, 263, 256, 266, 238, 244, 277, 245, 256] payload_rows_per_dst=[5548, 5434, 5464, 5562, 5474, 5488, 5316, 5484] dup_rows_per_dst=[2956, 2724, 2640, 2702, 2762, 2798, 2504, 2680] combine_rows_per_src=[5548, 5434, 5464, 5562, 5474, 5488, 5316, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229056 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040320 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851584 weight_movement_bytes=37748736 total_nvlink_bytes=351600320 weight_to_token_ratio=0.120276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38344 dup_rows=21658 dup_rows_remote=18992 dedup_ratio=0.330475 router_tokens_per_expert=[260, 269, 263, 254, 251, 265, 253, 252, 296, 261, 267, 230, 255, 254, 253, 267, 254, 245, 294, 229, 241, 266, 254, 252, 256, 251, 243, 254, 267, 241, 269, 253, 251, 266, 253, 257, 286, 256, 261, 248, 243, 247, 254, 266, 257, 239, 247, 219, 259, 279, 233, 267, 275, 219, 271, 266, 269, 245, 257, 266, 241, 233, 275, 245, 268, 253, 257, 252, 229, 268, 235, 270, 257, 260, 294, 255, 246, 233, 263, 299, 263, 237, 249, 255, 247, 279, 283, 264, 240, 239, 250, 266, 242, 265, 271, 251, 227, 263, 267, 252, 268, 264, 247, 262, 261, 237, 243, 254, 233, 251, 265, 230, 231, 241, 243, 236, 249, 278, 275, 272, 269, 292, 237, 257, 265, 258, 279, 253] payload_rows_per_dst=[5506, 5422, 5454, 5534, 5488, 5422, 5476, 5576] dup_rows_per_dst=[2794, 2716, 2646, 2666, 2790, 2780, 2572, 2694] combine_rows_per_src=[5506, 5422, 5454, 5534, 5488, 5422, 5476, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229560 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934328 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639096 weight_movement_bytes=18874368 total_nvlink_bytes=332513464 weight_to_token_ratio=0.060179 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229752 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893560 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557368 weight_movement_bytes=18874368 total_nvlink_bytes=332431736 weight_to_token_ratio=0.060194 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229256 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156942216 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655176 weight_movement_bytes=37748736 total_nvlink_bytes=351403912 weight_to_token_ratio=0.120351 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229416 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157294632 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314359848 weight_movement_bytes=18874368 total_nvlink_bytes=333234216 weight_to_token_ratio=0.060041 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=228128 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156785440 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313342752 weight_movement_bytes=37748736 total_nvlink_bytes=351091488 weight_to_token_ratio=0.120471 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229344 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286368 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343392 weight_movement_bytes=18874368 total_nvlink_bytes=333217760 weight_to_token_ratio=0.060044 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38432 dup_rows=21738 dup_rows_remote=19082 dedup_ratio=0.331696 router_tokens_per_expert=[248, 249, 267, 271, 255, 265, 255, 247, 274, 256, 264, 259, 269, 229, 271, 264, 237, 218, 233, 267, 262, 247, 249, 272, 222, 237, 264, 264, 266, 260, 273, 244, 250, 256, 257, 273, 269, 241, 259, 252, 275, 257, 242, 246, 248, 223, 258, 268, 240, 268, 267, 256, 249, 285, 265, 256, 241, 258, 260, 278, 260, 285, 252, 251, 262, 264, 254, 245, 256, 227, 264, 278, 257, 256, 258, 275, 271, 254, 234, 252, 253, 276, 268, 266, 255, 268, 250, 245, 230, 252, 234, 239, 235, 290, 270, 259, 243, 222, 248, 251, 251, 264, 263, 279, 244, 254, 251, 248, 228, 254, 266, 245, 261, 257, 259, 266, 281, 274, 249, 257, 262, 243, 260, 244, 296, 269, 234, 245] payload_rows_per_dst=[5506, 5428, 5464, 5542, 5520, 5470, 5366, 5502] dup_rows_per_dst=[2780, 2602, 2684, 2800, 2694, 2710, 2656, 2812] combine_rows_per_src=[5506, 5428, 5464, 5542, 5520, 5470, 5366, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38428 dup_rows=21640 dup_rows_remote=18970 dedup_ratio=0.330200 router_tokens_per_expert=[254, 286, 270, 277, 276, 253, 256, 250, 260, 250, 257, 271, 251, 245, 236, 265, 247, 250, 279, 224, 272, 270, 260, 228, 251, 255, 267, 249, 268, 258, 244, 262, 243, 241, 249, 254, 255, 232, 242, 285, 256, 296, 237, 269, 255, 249, 263, 243, 237, 243, 256, 284, 252, 255, 243, 247, 238, 242, 252, 241, 269, 257, 265, 250, 230, 277, 252, 238, 250, 274, 258, 258, 267, 253, 249, 268, 273, 242, 269, 233, 257, 254, 279, 267, 244, 236, 261, 251, 296, 263, 262, 257, 272, 256, 241, 247, 241, 236, 270, 246, 259, 284, 239, 252, 241, 268, 287, 258, 253, 256, 265, 243, 234, 277, 239, 217, 260, 249, 254, 270, 265, 283, 250, 273, 241, 247, 264, 272] payload_rows_per_dst=[5526, 5530, 5468, 5428, 5418, 5554, 5458, 5514] dup_rows_per_dst=[2788, 2638, 2670, 2634, 2764, 2732, 2738, 2676] combine_rows_per_src=[5526, 5530, 5468, 5428, 5418, 5554, 5458, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38304 dup_rows=21788 dup_rows_remote=19086 dedup_ratio=0.332458 router_tokens_per_expert=[220, 245, 245, 263, 258, 228, 244, 234, 252, 276, 288, 228, 294, 252, 255, 257, 279, 258, 243, 266, 280, 249, 267, 254, 247, 242, 271, 251, 267, 249, 258, 255, 251, 256, 235, 229, 257, 279, 239, 251, 265, 226, 247, 259, 266, 260, 237, 239, 246, 268, 264, 275, 240, 248, 265, 252, 253, 273, 260, 245, 239, 234, 263, 255, 246, 271, 268, 263, 270, 284, 256, 265, 255, 273, 258, 275, 266, 256, 262, 248, 237, 237, 263, 258, 250, 246, 254, 256, 239, 254, 249, 274, 272, 256, 279, 249, 262, 260, 251, 245, 265, 244, 262, 240, 285, 247, 233, 277, 291, 254, 228, 246, 254, 260, 264, 267, 286, 252, 275, 253, 262, 270, 243, 252, 256, 253, 232, 259] payload_rows_per_dst=[5368, 5506, 5412, 5456, 5536, 5444, 5462, 5564] dup_rows_per_dst=[2710, 2766, 2580, 2704, 2896, 2702, 2718, 2712] combine_rows_per_src=[5368, 5506, 5412, 5456, 5536, 5444, 5462, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38014 dup_rows=21922 dup_rows_remote=19080 dedup_ratio=0.334503 router_tokens_per_expert=[240, 294, 236, 261, 252, 246, 280, 260, 263, 259, 249, 254, 250, 250, 240, 257, 242, 249, 257, 248, 230, 247, 251, 257, 251, 263, 247, 249, 273, 238, 261, 257, 260, 250, 244, 229, 251, 253, 261, 265, 254, 249, 267, 255, 242, 255, 276, 278, 230, 249, 264, 248, 249, 265, 272, 247, 291, 280, 253, 231, 259, 255, 225, 274, 237, 277, 273, 272, 277, 247, 256, 268, 237, 253, 254, 259, 257, 262, 252, 283, 239, 250, 249, 238, 252, 256, 269, 285, 257, 240, 245, 250, 257, 245, 272, 250, 234, 226, 279, 275, 245, 240, 261, 258, 264, 258, 254, 257, 265, 246, 240, 251, 250, 262, 256, 249, 278, 266, 257, 276, 273, 267, 272, 254, 256, 280, 247, 262] payload_rows_per_dst=[5338, 5406, 5368, 5434, 5510, 5418, 5540, 5600] dup_rows_per_dst=[2564, 2602, 2724, 2750, 2904, 2690, 2846, 2842] combine_rows_per_src=[5338, 5406, 5368, 5434, 5510, 5418, 5540, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38440 dup_rows=21646 dup_rows_remote=18904 dedup_ratio=0.330292 router_tokens_per_expert=[278, 263, 263, 249, 245, 268, 255, 262, 264, 237, 278, 255, 287, 251, 252, 211, 241, 276, 258, 248, 234, 240, 269, 264, 264, 255, 252, 270, 236, 235, 252, 223, 251, 242, 273, 231, 246, 243, 293, 243, 267, 238, 256, 257, 271, 235, 261, 255, 279, 252, 230, 259, 273, 230, 231, 255, 255, 258, 261, 248, 281, 253, 280, 256, 258, 240, 251, 265, 250, 256, 265, 262, 250, 254, 244, 263, 260, 256, 257, 265, 217, 256, 268, 239, 264, 275, 256, 265, 270, 289, 249, 265, 234, 261, 240, 269, 237, 252, 283, 253, 248, 261, 238, 260, 239, 281, 271, 224, 272, 278, 258, 259, 265, 254, 265, 274, 250, 237, 267, 246, 287, 262, 253, 299, 233, 247, 268, 236] payload_rows_per_dst=[5496, 5458, 5452, 5456, 5450, 5520, 5476, 5582] dup_rows_per_dst=[2740, 2576, 2672, 2746, 2742, 2714, 2752, 2704] combine_rows_per_src=[5496, 5458, 5452, 5456, 5450, 5520, 5476, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=230056 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157647528 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315065000 weight_movement_bytes=18874368 total_nvlink_bytes=333939368 weight_to_token_ratio=0.059906 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229592 combine_payload_bytes=157401088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157630680 ingress_bytes=195149824 link_bytes=195149824 token_bytes=315031768 weight_movement_bytes=37748736 total_nvlink_bytes=352780504 weight_to_token_ratio=0.119825 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869829594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38106 dup_rows=21926 dup_rows_remote=19138 dedup_ratio=0.334564 router_tokens_per_expert=[274, 235, 251, 241, 227, 259, 276, 254, 217, 238, 268, 268, 260, 269, 249, 263, 249, 258, 274, 256, 257, 256, 255, 267, 264, 261, 251, 291, 242, 250, 243, 272, 262, 260, 239, 244, 251, 284, 267, 256, 259, 263, 272, 242, 264, 246, 257, 255, 247, 247, 249, 289, 270, 245, 258, 275, 280, 262, 254, 233, 282, 241, 279, 262, 271, 222, 266, 248, 276, 257, 243, 255, 278, 278, 254, 253, 260, 256, 223, 251, 281, 241, 260, 257, 213, 251, 265, 250, 240, 256, 268, 213, 256, 263, 248, 264, 231, 252, 268, 245, 272, 250, 241, 250, 233, 262, 259, 269, 241, 262, 252, 240, 268, 278, 251, 262, 265, 289, 231, 279, 227, 281, 275, 238, 243, 247, 260, 241] payload_rows_per_dst=[5364, 5504, 5458, 5544, 5418, 5392, 5446, 5484] dup_rows_per_dst=[2734, 2788, 2784, 2802, 2764, 2660, 2608, 2786] combine_rows_per_src=[5364, 5504, 5458, 5544, 5418, 5392, 5446, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38270 dup_rows=21818 dup_rows_remote=19138 dedup_ratio=0.332916 router_tokens_per_expert=[280, 241, 278, 242, 237, 258, 257, 269, 250, 251, 248, 250, 246, 251, 256, 270, 248, 266, 258, 242, 254, 251, 256, 261, 241, 251, 257, 282, 250, 269, 219, 274, 229, 272, 264, 311, 252, 272, 247, 281, 256, 275, 241, 237, 255, 283, 265, 278, 248, 268, 237, 268, 283, 250, 245, 251, 283, 268, 231, 246, 245, 265, 243, 233, 232, 257, 283, 255, 241, 257, 242, 248, 241, 229, 267, 248, 280, 272, 261, 234, 249, 274, 226, 245, 245, 270, 262, 243, 243, 282, 259, 246, 248, 256, 273, 254, 233, 245, 263, 260, 231, 244, 248, 252, 241, 286, 269, 262, 245, 257, 269, 267, 282, 255, 258, 276, 251, 255, 265, 285, 240, 247, 232, 241, 252, 249, 277, 264] payload_rows_per_dst=[5372, 5494, 5616, 5448, 5438, 5448, 5424, 5478] dup_rows_per_dst=[2796, 2664, 2820, 2680, 2656, 2702, 2720, 2780] combine_rows_per_src=[5372, 5494, 5616, 5448, 5438, 5448, 5424, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38242 dup_rows=21872 dup_rows_remote=19134 dedup_ratio=0.333740 router_tokens_per_expert=[273, 250, 248, 245, 251, 272, 260, 243, 263, 236, 257, 251, 271, 268, 242, 247, 253, 245, 258, 257, 258, 253, 235, 241, 246, 272, 267, 272, 260, 248, 258, 261, 249, 262, 296, 250, 249, 256, 262, 263, 278, 242, 263, 276, 261, 263, 243, 267, 260, 232, 254, 225, 241, 245, 250, 245, 266, 236, 262, 232, 272, 237, 256, 263, 277, 259, 250, 258, 253, 245, 250, 249, 253, 254, 305, 248, 255, 262, 257, 234, 269, 266, 253, 247, 264, 251, 292, 246, 274, 285, 249, 225, 265, 256, 273, 279, 252, 264, 232, 234, 276, 231, 269, 240, 252, 234, 234, 246, 280, 238, 245, 281, 260, 253, 278, 261, 257, 231, 278, 229, 268, 261, 259, 252, 282, 253, 253, 265] payload_rows_per_dst=[5422, 5426, 5478, 5336, 5532, 5534, 5396, 5540] dup_rows_per_dst=[2738, 2736, 2934, 2616, 2686, 2802, 2620, 2740] combine_rows_per_src=[5422, 5426, 5478, 5336, 5532, 5534, 5396, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=228376 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155933720 ingress_bytes=212328448 link_bytes=212328448 token_bytes=311639064 weight_movement_bytes=56623104 total_nvlink_bytes=368262168 weight_to_token_ratio=0.181695 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034426717e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229560 combine_payload_bytes=156893184 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157122744 ingress_bytes=213516288 link_bytes=213516288 token_bytes=314015928 weight_movement_bytes=56623104 total_nvlink_bytes=370639032 weight_to_token_ratio=0.180319 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045808015e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157450240 dispatch_meta_bytes=229376 combine_payload_bytes=157450240 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157679616 ingress_bytes=176324608 link_bytes=176324608 token_bytes=315129856 weight_movement_bytes=18874368 total_nvlink_bytes=334004224 weight_to_token_ratio=0.059894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689455637e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228976 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156311152 ingress_bytes=193830912 link_bytes=193830912 token_bytes=312393328 weight_movement_bytes=37748736 total_nvlink_bytes=350142064 weight_to_token_ratio=0.120837 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857192428e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229632 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983552 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737472 weight_movement_bytes=18874368 total_nvlink_bytes=332611840 weight_to_token_ratio=0.060160 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229504 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156868736 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313507968 weight_movement_bytes=56623104 total_nvlink_bytes=370131072 weight_to_token_ratio=0.180611 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38192 dup_rows=21900 dup_rows_remote=19210 dedup_ratio=0.334167 router_tokens_per_expert=[267, 242, 278, 268, 263, 285, 260, 260, 240, 266, 239, 248, 255, 258, 239, 261, 247, 248, 274, 256, 249, 255, 272, 238, 254, 253, 238, 230, 260, 256, 243, 252, 278, 253, 255, 240, 263, 275, 261, 247, 256, 272, 288, 275, 254, 257, 244, 250, 253, 269, 268, 266, 246, 266, 261, 270, 259, 264, 227, 255, 261, 262, 254, 241, 248, 253, 226, 260, 247, 258, 285, 288, 283, 225, 237, 260, 263, 258, 265, 281, 259, 268, 248, 232, 256, 225, 250, 250, 267, 260, 274, 234, 268, 260, 256, 264, 253, 239, 239, 257, 272, 262, 249, 247, 295, 247, 240, 253, 240, 251, 266, 257, 252, 260, 247, 255, 246, 254, 272, 283, 252, 248, 233, 246, 281, 237, 241, 242] payload_rows_per_dst=[5532, 5420, 5490, 5472, 5482, 5430, 5444, 5366] dup_rows_per_dst=[2726, 2630, 2846, 2772, 2792, 2712, 2690, 2732] combine_rows_per_src=[5532, 5420, 5490, 5472, 5482, 5430, 5444, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38400 dup_rows=21684 dup_rows_remote=19032 dedup_ratio=0.330872 router_tokens_per_expert=[247, 246, 255, 259, 252, 269, 259, 281, 273, 255, 237, 258, 268, 261, 248, 248, 276, 275, 264, 260, 251, 257, 242, 256, 262, 232, 229, 272, 251, 266, 252, 273, 245, 264, 247, 236, 260, 256, 228, 254, 258, 241, 266, 238, 269, 277, 249, 264, 239, 256, 250, 272, 245, 246, 256, 234, 244, 266, 258, 286, 263, 247, 241, 249, 263, 259, 261, 237, 237, 242, 250, 252, 252, 264, 238, 269, 272, 236, 250, 264, 284, 289, 246, 286, 264, 273, 266, 250, 252, 276, 242, 259, 258, 266, 271, 230, 240, 247, 250, 272, 232, 253, 248, 270, 275, 269, 252, 237, 239, 282, 233, 259, 276, 259, 274, 267, 240, 239, 249, 269, 237, 272, 262, 247, 244, 250, 260, 269] payload_rows_per_dst=[5498, 5514, 5452, 5430, 5388, 5602, 5472, 5496] dup_rows_per_dst=[2734, 2722, 2652, 2674, 2704, 2822, 2644, 2732] combine_rows_per_src=[5498, 5514, 5452, 5430, 5388, 5602, 5472, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38186 dup_rows=21866 dup_rows_remote=19130 dedup_ratio=0.333649 router_tokens_per_expert=[276, 240, 262, 237, 266, 267, 231, 283, 295, 252, 239, 253, 270, 245, 287, 277, 257, 223, 260, 247, 240, 250, 258, 281, 240, 272, 280, 257, 272, 265, 249, 248, 256, 270, 270, 257, 236, 283, 274, 258, 248, 254, 251, 243, 223, 259, 260, 281, 242, 263, 261, 258, 241, 251, 246, 272, 248, 255, 280, 241, 263, 241, 241, 245, 270, 257, 272, 260, 242, 249, 249, 264, 267, 255, 250, 291, 244, 249, 273, 236, 268, 235, 265, 253, 269, 273, 232, 237, 238, 237, 241, 258, 248, 242, 237, 236, 248, 251, 275, 245, 257, 241, 274, 254, 297, 263, 240, 259, 240, 243, 266, 263, 294, 273, 249, 268, 244, 227, 231, 252, 249, 267, 241, 267, 249, 260, 274, 260] payload_rows_per_dst=[5520, 5450, 5532, 5406, 5498, 5292, 5488, 5484] dup_rows_per_dst=[2840, 2748, 2714, 2680, 2758, 2646, 2744, 2736] combine_rows_per_src=[5520, 5450, 5532, 5406, 5498, 5292, 5488, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38454 dup_rows=21634 dup_rows_remote=19024 dedup_ratio=0.330109 router_tokens_per_expert=[259, 277, 266, 259, 225, 266, 273, 300, 255, 254, 240, 293, 242, 272, 250, 247, 242, 247, 255, 250, 260, 272, 242, 286, 280, 260, 239, 252, 256, 284, 251, 273, 248, 234, 272, 233, 260, 283, 270, 260, 260, 268, 285, 236, 260, 280, 245, 253, 264, 244, 256, 273, 241, 265, 222, 245, 272, 270, 258, 268, 260, 227, 228, 250, 273, 253, 255, 293, 245, 244, 268, 257, 242, 264, 242, 239, 274, 265, 244, 246, 229, 277, 265, 225, 233, 238, 257, 260, 253, 259, 280, 258, 249, 255, 252, 239, 249, 236, 248, 264, 264, 256, 261, 254, 255, 276, 247, 262, 264, 254, 239, 255, 247, 229, 260, 243, 249, 270, 248, 253, 259, 256, 247, 242, 274, 254, 268, 235] payload_rows_per_dst=[5560, 5540, 5534, 5370, 5478, 5448, 5488, 5484] dup_rows_per_dst=[2796, 2758, 2760, 2716, 2730, 2610, 2680, 2584] combine_rows_per_src=[5560, 5540, 5534, 5370, 5478, 5448, 5488, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229608 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156664040 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313098472 weight_movement_bytes=37748736 total_nvlink_bytes=350847208 weight_to_token_ratio=0.120565 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38188 dup_rows=21892 dup_rows_remote=19114 dedup_ratio=0.334045 router_tokens_per_expert=[255, 252, 243, 260, 276, 268, 247, 244, 243, 256, 252, 218, 265, 267, 237, 245, 276, 257, 247, 239, 265, 280, 277, 248, 234, 240, 266, 249, 272, 250, 247, 265, 261, 268, 237, 266, 259, 265, 259, 275, 254, 243, 240, 258, 235, 265, 267, 237, 271, 246, 230, 249, 233, 249, 253, 259, 272, 258, 259, 283, 231, 250, 265, 250, 293, 247, 285, 251, 246, 259, 245, 231, 250, 235, 253, 258, 307, 239, 257, 261, 246, 259, 278, 245, 252, 267, 238, 234, 253, 264, 238, 265, 240, 262, 256, 251, 278, 255, 258, 260, 239, 245, 268, 266, 279, 243, 246, 257, 285, 249, 255, 264, 265, 234, 274, 255, 245, 246, 271, 295, 262, 263, 258, 272, 233, 269, 274, 253] payload_rows_per_dst=[5440, 5486, 5408, 5438, 5456, 5418, 5498, 5500] dup_rows_per_dst=[2616, 2738, 2770, 2678, 2778, 2678, 2796, 2838] combine_rows_per_src=[5440, 5486, 5408, 5438, 5456, 5418, 5498, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229728 combine_payload_bytes=157286400 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157516128 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802528 weight_movement_bytes=37748736 total_nvlink_bytes=352551264 weight_to_token_ratio=0.119912 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38294 dup_rows=21882 dup_rows_remote=19156 dedup_ratio=0.333893 router_tokens_per_expert=[240, 242, 249, 266, 256, 274, 241, 252, 264, 255, 265, 261, 250, 245, 260, 237, 247, 258, 270, 239, 248, 230, 232, 261, 245, 275, 263, 260, 238, 286, 242, 249, 280, 245, 252, 258, 286, 255, 232, 251, 262, 231, 236, 244, 261, 218, 238, 248, 265, 251, 260, 232, 269, 236, 289, 240, 269, 219, 281, 275, 270, 254, 281, 265, 236, 269, 263, 272, 255, 256, 276, 245, 271, 269, 251, 288, 251, 281, 256, 258, 222, 273, 280, 276, 262, 248, 241, 253, 238, 244, 254, 263, 263, 264, 269, 262, 256, 254, 262, 249, 239, 243, 250, 225, 243, 270, 262, 248, 272, 270, 262, 270, 257, 252, 256, 246, 274, 237, 268, 252, 262, 243, 266, 263, 249, 252, 258, 296] payload_rows_per_dst=[5386, 5360, 5358, 5472, 5538, 5514, 5512, 5514] dup_rows_per_dst=[2728, 2726, 2636, 2840, 2856, 2710, 2638, 2748] combine_rows_per_src=[5386, 5360, 5358, 5472, 5538, 5514, 5512, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38332 dup_rows=21750 dup_rows_remote=18942 dedup_ratio=0.331879 router_tokens_per_expert=[266, 248, 239, 272, 282, 276, 243, 287, 234, 258, 254, 236, 250, 266, 255, 253, 252, 252, 250, 254, 270, 280, 281, 273, 254, 261, 259, 254, 267, 247, 263, 252, 262, 260, 246, 251, 254, 274, 246, 268, 263, 274, 267, 261, 253, 265, 272, 272, 237, 278, 272, 261, 281, 254, 273, 238, 281, 250, 266, 247, 250, 226, 259, 258, 282, 264, 255, 256, 253, 266, 238, 262, 261, 250, 245, 250, 240, 259, 239, 237, 255, 268, 269, 222, 279, 249, 228, 255, 237, 239, 250, 222, 251, 255, 234, 250, 252, 222, 245, 299, 265, 240, 255, 240, 242, 263, 274, 254, 260, 236, 254, 267, 233, 254, 260, 272, 263, 263, 245, 261, 253, 242, 257, 251, 259, 253, 238, 269] payload_rows_per_dst=[5506, 5588, 5528, 5454, 5484, 5394, 5418, 5414] dup_rows_per_dst=[2732, 2750, 2848, 2808, 2630, 2532, 2718, 2732] combine_rows_per_src=[5506, 5588, 5528, 5454, 5484, 5394, 5418, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38170 dup_rows=21774 dup_rows_remote=18866 dedup_ratio=0.332245 router_tokens_per_expert=[250, 241, 273, 256, 245, 243, 266, 268, 258, 249, 254, 260, 247, 250, 259, 261, 233, 262, 245, 231, 239, 268, 243, 247, 280, 260, 278, 289, 248, 236, 274, 287, 274, 251, 264, 266, 239, 241, 272, 256, 247, 240, 239, 252, 271, 260, 254, 260, 234, 257, 260, 224, 246, 247, 271, 241, 232, 251, 257, 262, 238, 252, 278, 271, 245, 271, 271, 280, 237, 260, 266, 287, 251, 230, 242, 255, 268, 242, 246, 260, 247, 269, 233, 284, 260, 257, 257, 257, 244, 266, 260, 254, 240, 271, 269, 273, 272, 272, 249, 239, 275, 248, 258, 265, 244, 265, 261, 254, 236, 242, 255, 229, 257, 236, 266, 247, 252, 277, 254, 270, 257, 268, 277, 258, 251, 277, 265, 233] payload_rows_per_dst=[5592, 5540, 5674, 5480, 5448, 5312, 5478, 5238] dup_rows_per_dst=[2908, 2700, 2998, 2738, 2774, 2454, 2750, 2452] combine_rows_per_src=[5592, 5540, 5674, 5480, 5448, 5312, 5478, 5238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229264 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156639120 ingress_bytes=213032960 link_bytes=213032960 token_bytes=313048976 weight_movement_bytes=56623104 total_nvlink_bytes=369672080 weight_to_token_ratio=0.180876 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041177004e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=229912 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157737496 ingress_bytes=176381952 link_bytes=176381952 token_bytes=315245080 weight_movement_bytes=18874368 total_nvlink_bytes=334119448 weight_to_token_ratio=0.059872 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690005079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229208 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156647256 ingress_bytes=194166784 link_bytes=194166784 token_bytes=313065304 weight_movement_bytes=37748736 total_nvlink_bytes=350814040 weight_to_token_ratio=0.120578 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860410588e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229800 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157082024 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934248 weight_movement_bytes=18874368 total_nvlink_bytes=332808616 weight_to_token_ratio=0.060122 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229096 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157236968 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314244840 weight_movement_bytes=37748736 total_nvlink_bytes=351993576 weight_to_token_ratio=0.120125 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228144 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156572464 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312916784 weight_movement_bytes=18874368 total_nvlink_bytes=331791152 weight_to_token_ratio=0.060318 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38326 dup_rows=21806 dup_rows_remote=19138 dedup_ratio=0.332733 router_tokens_per_expert=[243, 248, 257, 226, 250, 256, 278, 269, 259, 247, 284, 279, 237, 262, 295, 250, 231, 231, 263, 255, 240, 255, 266, 260, 241, 256, 257, 234, 259, 240, 258, 262, 255, 269, 252, 252, 261, 261, 260, 255, 245, 264, 282, 264, 276, 253, 258, 256, 242, 246, 240, 255, 232, 267, 257, 273, 271, 219, 235, 252, 262, 256, 243, 267, 261, 257, 243, 249, 259, 245, 239, 264, 278, 287, 261, 277, 281, 260, 258, 260, 227, 253, 258, 229, 248, 255, 271, 277, 234, 268, 246, 276, 279, 268, 252, 242, 247, 251, 241, 247, 239, 230, 273, 267, 270, 253, 250, 242, 273, 255, 257, 241, 284, 244, 263, 249, 263, 254, 256, 257, 274, 291, 247, 266, 270, 240, 253, 231] payload_rows_per_dst=[5404, 5400, 5568, 5436, 5532, 5452, 5404, 5534] dup_rows_per_dst=[2876, 2616, 2758, 2598, 2826, 2714, 2668, 2750] combine_rows_per_src=[5404, 5400, 5568, 5436, 5532, 5452, 5404, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44008 payload_rows_remote=38564 dup_rows=21528 dup_rows_remote=18976 dedup_ratio=0.328491 router_tokens_per_expert=[265, 268, 249, 233, 250, 249, 231, 269, 259, 259, 247, 242, 259, 235, 251, 223, 263, 228, 253, 249, 249, 261, 252, 243, 268, 250, 280, 258, 267, 236, 244, 257, 243, 244, 284, 254, 286, 239, 249, 240, 222, 265, 254, 230, 250, 271, 237, 230, 271, 219, 271, 245, 246, 252, 284, 234, 253, 266, 306, 273, 241, 240, 278, 273, 268, 266, 268, 248, 259, 253, 255, 252, 248, 256, 299, 247, 265, 262, 241, 277, 275, 259, 279, 259, 268, 240, 251, 263, 251, 257, 224, 254, 280, 252, 275, 245, 238, 249, 277, 268, 246, 267, 231, 260, 282, 250, 266, 295, 257, 247, 252, 258, 253, 264, 241, 257, 275, 252, 275, 254, 245, 252, 261, 272, 281, 259, 247, 244] payload_rows_per_dst=[5472, 5450, 5444, 5512, 5562, 5490, 5558, 5520] dup_rows_per_dst=[2506, 2666, 2552, 2792, 2766, 2774, 2728, 2744] combine_rows_per_src=[5472, 5450, 5444, 5512, 5562, 5490, 5558, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229856 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213152 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196448 weight_movement_bytes=18874368 total_nvlink_bytes=333070816 weight_to_token_ratio=0.060072 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38206 dup_rows=21750 dup_rows_remote=19040 dedup_ratio=0.331879 router_tokens_per_expert=[256, 262, 265, 265, 237, 245, 260, 247, 252, 249, 258, 283, 264, 249, 255, 254, 249, 279, 262, 228, 248, 240, 279, 260, 247, 248, 244, 253, 276, 222, 231, 231, 242, 248, 262, 245, 279, 259, 280, 266, 257, 270, 257, 250, 245, 284, 264, 258, 255, 254, 259, 292, 231, 244, 247, 283, 244, 278, 284, 242, 264, 248, 251, 263, 258, 279, 264, 258, 258, 260, 241, 293, 255, 244, 270, 271, 259, 250, 254, 259, 225, 265, 251, 232, 228, 276, 279, 260, 229, 263, 258, 244, 245, 266, 262, 238, 252, 234, 233, 252, 245, 267, 268, 263, 298, 244, 251, 231, 260, 237, 256, 269, 222, 274, 256, 273, 267, 241, 255, 239, 238, 286, 272, 266, 254, 247, 273, 248] payload_rows_per_dst=[5494, 5390, 5492, 5494, 5478, 5406, 5452, 5580] dup_rows_per_dst=[2708, 2604, 2840, 2784, 2800, 2636, 2668, 2710] combine_rows_per_src=[5494, 5390, 5492, 5494, 5478, 5406, 5452, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38152 dup_rows=21948 dup_rows_remote=19138 dedup_ratio=0.334900 router_tokens_per_expert=[269, 250, 247, 255, 257, 254, 252, 248, 239, 239, 272, 271, 274, 286, 257, 254, 246, 256, 266, 252, 260, 270, 256, 272, 223, 258, 286, 237, 278, 279, 233, 243, 247, 236, 262, 269, 239, 240, 293, 263, 272, 264, 243, 285, 236, 250, 253, 256, 271, 233, 269, 271, 272, 278, 238, 244, 262, 279, 270, 255, 249, 236, 229, 237, 229, 266, 239, 248, 243, 245, 260, 247, 268, 255, 237, 252, 291, 248, 261, 282, 270, 216, 240, 244, 287, 260, 251, 265, 279, 238, 280, 249, 257, 255, 275, 257, 268, 258, 246, 222, 262, 281, 237, 271, 218, 264, 272, 280, 283, 264, 224, 242, 274, 256, 231, 234, 250, 252, 251, 255, 239, 274, 232, 287, 250, 251, 256, 250] payload_rows_per_dst=[5402, 5492, 5514, 5498, 5410, 5436, 5434, 5402] dup_rows_per_dst=[2846, 2738, 2702, 2688, 2732, 2810, 2750, 2682] combine_rows_per_src=[5402, 5492, 5514, 5498, 5410, 5436, 5434, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38256 dup_rows=21808 dup_rows_remote=19212 dedup_ratio=0.332764 router_tokens_per_expert=[246, 238, 251, 229, 266, 255, 256, 205, 250, 253, 260, 286, 267, 247, 257, 271, 253, 276, 238, 256, 228, 236, 254, 259, 233, 255, 284, 258, 247, 265, 221, 266, 261, 281, 236, 269, 283, 272, 238, 260, 275, 274, 253, 248, 258, 261, 255, 258, 245, 285, 254, 251, 257, 272, 248, 260, 234, 271, 247, 266, 234, 265, 275, 264, 271, 254, 215, 244, 277, 245, 261, 251, 258, 257, 234, 269, 231, 279, 250, 238, 274, 255, 278, 249, 271, 267, 271, 237, 273, 280, 261, 283, 247, 264, 252, 254, 266, 244, 250, 257, 234, 244, 247, 280, 292, 270, 229, 256, 253, 257, 239, 253, 243, 241, 258, 273, 256, 263, 260, 257, 235, 238, 263, 229, 274, 259, 249, 273] payload_rows_per_dst=[5432, 5374, 5500, 5480, 5472, 5492, 5496, 5482] dup_rows_per_dst=[2642, 2684, 2864, 2776, 2596, 2940, 2646, 2660] combine_rows_per_src=[5432, 5374, 5500, 5480, 5472, 5492, 5496, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157958144 dispatch_meta_bytes=230160 combine_payload_bytes=157958144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158188304 ingress_bytes=176832512 link_bytes=176832512 token_bytes=316146448 weight_movement_bytes=18874368 total_nvlink_bytes=335020816 weight_to_token_ratio=0.059701 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.694322123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38200 dup_rows=21754 dup_rows_remote=18960 dedup_ratio=0.331940 router_tokens_per_expert=[224, 234, 233, 251, 240, 255, 249, 261, 256, 247, 253, 242, 253, 264, 276, 265, 250, 266, 290, 261, 279, 253, 250, 256, 240, 266, 281, 254, 239, 261, 282, 260, 256, 269, 249, 245, 237, 243, 269, 266, 267, 233, 241, 233, 250, 262, 279, 282, 239, 287, 267, 259, 244, 254, 249, 242, 261, 287, 252, 234, 248, 249, 247, 255, 233, 261, 246, 255, 242, 269, 249, 264, 257, 289, 252, 268, 275, 236, 255, 252, 260, 243, 237, 232, 261, 257, 237, 257, 262, 256, 255, 234, 271, 265, 253, 248, 267, 281, 275, 286, 246, 253, 269, 281, 236, 264, 269, 275, 229, 242, 244, 261, 222, 260, 265, 265, 246, 277, 235, 248, 256, 271, 247, 266, 271, 245, 267, 272] payload_rows_per_dst=[5414, 5582, 5420, 5422, 5488, 5430, 5576, 5450] dup_rows_per_dst=[2592, 2794, 2742, 2726, 2718, 2626, 2780, 2776] combine_rows_per_src=[5414, 5582, 5420, 5422, 5488, 5430, 5576, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43516 payload_rows_remote=38064 dup_rows=22020 dup_rows_remote=19164 dedup_ratio=0.335999 router_tokens_per_expert=[245, 259, 278, 263, 254, 259, 249, 252, 217, 274, 267, 253, 263, 237, 268, 238, 224, 262, 246, 264, 244, 285, 251, 233, 272, 254, 238, 261, 236, 270, 277, 226, 238, 271, 245, 244, 257, 268, 269, 261, 260, 252, 265, 254, 263, 253, 237, 251, 263, 251, 248, 248, 266, 253, 271, 266, 258, 244, 258, 263, 264, 270, 269, 262, 266, 246, 239, 246, 270, 271, 257, 240, 262, 242, 258, 251, 245, 260, 237, 244, 257, 258, 248, 277, 259, 249, 236, 263, 235, 260, 278, 264, 262, 255, 275, 269, 251, 242, 261, 281, 270, 243, 264, 268, 248, 258, 259, 264, 247, 253, 271, 238, 265, 279, 256, 246, 227, 271, 272, 264, 219, 268, 260, 243, 242, 272, 260, 266] payload_rows_per_dst=[5440, 5398, 5444, 5452, 5386, 5472, 5462, 5462] dup_rows_per_dst=[2712, 2688, 2732, 2856, 2682, 2818, 2774, 2758] combine_rows_per_src=[5440, 5398, 5444, 5452, 5386, 5472, 5462, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38430 dup_rows=21644 dup_rows_remote=18890 dedup_ratio=0.330261 router_tokens_per_expert=[269, 259, 240, 283, 295, 241, 259, 257, 240, 231, 246, 259, 255, 264, 237, 253, 280, 266, 250, 250, 238, 219, 245, 275, 245, 272, 261, 258, 241, 284, 268, 254, 256, 250, 267, 255, 267, 245, 265, 255, 248, 234, 228, 255, 280, 291, 260, 242, 245, 267, 259, 271, 238, 253, 279, 257, 278, 250, 239, 261, 250, 251, 271, 257, 267, 249, 271, 281, 247, 237, 261, 258, 257, 246, 258, 250, 256, 258, 241, 278, 221, 277, 253, 266, 267, 251, 263, 271, 248, 253, 257, 252, 260, 269, 267, 260, 248, 237, 253, 267, 275, 256, 273, 247, 247, 233, 281, 250, 257, 247, 256, 263, 264, 228, 239, 249, 233, 249, 247, 264, 253, 231, 271, 254, 256, 254, 271, 247] payload_rows_per_dst=[5462, 5646, 5508, 5434, 5366, 5638, 5402, 5436] dup_rows_per_dst=[2754, 2756, 2688, 2778, 2674, 2864, 2546, 2584] combine_rows_per_src=[5462, 5646, 5508, 5434, 5366, 5638, 5402, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228984 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156720760 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313212536 weight_movement_bytes=37748736 total_nvlink_bytes=350961272 weight_to_token_ratio=0.120521 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229160 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156499752 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312770344 weight_movement_bytes=18874368 total_nvlink_bytes=331644712 weight_to_token_ratio=0.060346 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229872 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156926448 ingress_bytes=213319680 link_bytes=213319680 token_bytes=313623024 weight_movement_bytes=56623104 total_nvlink_bytes=370246128 weight_to_token_ratio=0.180545 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043924214e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228640 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156695840 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313163040 weight_movement_bytes=18874368 total_nvlink_bytes=332037408 weight_to_token_ratio=0.060270 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228912 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156139056 ingress_bytes=193658880 link_bytes=193658880 token_bytes=312049200 weight_movement_bytes=37748736 total_nvlink_bytes=349797936 weight_to_token_ratio=0.120970 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855544102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229280 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157638560 ingress_bytes=195158016 link_bytes=195158016 token_bytes=315047840 weight_movement_bytes=37748736 total_nvlink_bytes=352796576 weight_to_token_ratio=0.119819 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869908086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38210 dup_rows=21802 dup_rows_remote=18994 dedup_ratio=0.332672 router_tokens_per_expert=[238, 250, 249, 247, 238, 247, 259, 290, 268, 251, 253, 260, 264, 256, 267, 236, 234, 273, 262, 242, 260, 238, 268, 250, 244, 250, 244, 248, 237, 255, 254, 244, 265, 249, 243, 233, 255, 231, 245, 259, 232, 273, 240, 271, 263, 245, 235, 264, 282, 241, 265, 262, 246, 282, 277, 231, 287, 277, 247, 263, 253, 234, 279, 273, 255, 236, 246, 263, 266, 285, 263, 245, 267, 225, 266, 262, 286, 261, 252, 273, 267, 277, 235, 244, 233, 258, 261, 239, 258, 217, 283, 273, 253, 252, 236, 266, 250, 254, 260, 262, 271, 278, 253, 249, 278, 253, 254, 235, 284, 283, 255, 247, 249, 284, 255, 255, 247, 238, 241, 246, 249, 256, 306, 255, 271, 252, 261, 256] payload_rows_per_dst=[5434, 5484, 5392, 5558, 5506, 5362, 5524, 5474] dup_rows_per_dst=[2712, 2522, 2614, 2840, 2796, 2742, 2808, 2768] combine_rows_per_src=[5434, 5484, 5392, 5558, 5506, 5362, 5524, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38244 dup_rows=21784 dup_rows_remote=19010 dedup_ratio=0.332397 router_tokens_per_expert=[226, 282, 249, 271, 267, 259, 265, 252, 244, 247, 260, 250, 271, 244, 271, 251, 267, 236, 253, 260, 258, 264, 264, 268, 267, 259, 254, 290, 243, 238, 244, 258, 243, 239, 278, 254, 249, 257, 263, 256, 265, 267, 263, 238, 271, 246, 294, 258, 236, 246, 247, 281, 245, 247, 231, 256, 272, 261, 251, 251, 278, 289, 233, 267, 253, 240, 244, 224, 268, 251, 259, 267, 265, 263, 251, 266, 247, 239, 258, 250, 268, 261, 232, 271, 231, 254, 246, 265, 245, 281, 261, 297, 248, 283, 256, 225, 222, 248, 252, 273, 236, 244, 268, 268, 262, 290, 266, 254, 282, 276, 247, 256, 258, 244, 256, 244, 248, 229, 265, 275, 254, 225, 251, 261, 246, 229, 274, 232] payload_rows_per_dst=[5466, 5540, 5508, 5492, 5440, 5552, 5462, 5292] dup_rows_per_dst=[2752, 2706, 2774, 2690, 2650, 2696, 2826, 2690] combine_rows_per_src=[5466, 5540, 5508, 5492, 5440, 5552, 5462, 5292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=228816 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156736976 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245136 weight_movement_bytes=18874368 total_nvlink_bytes=332119504 weight_to_token_ratio=0.060254 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38198 dup_rows=21844 dup_rows_remote=19000 dedup_ratio=0.333313 router_tokens_per_expert=[232, 247, 242, 246, 257, 247, 296, 259, 251, 238, 261, 251, 245, 240, 235, 284, 260, 283, 262, 238, 243, 231, 252, 254, 256, 237, 256, 252, 240, 225, 238, 245, 247, 247, 252, 250, 269, 236, 244, 231, 238, 272, 261, 250, 256, 234, 241, 247, 246, 274, 228, 289, 261, 243, 251, 249, 278, 240, 280, 239, 254, 248, 261, 283, 249, 253, 251, 280, 259, 262, 255, 292, 265, 282, 274, 270, 271, 289, 237, 251, 273, 246, 251, 278, 272, 261, 246, 269, 257, 242, 273, 248, 281, 259, 246, 267, 251, 262, 220, 262, 263, 263, 271, 256, 259, 260, 275, 260, 235, 271, 267, 246, 275, 271, 254, 267, 233, 279, 273, 266, 227, 244, 263, 245, 259, 265, 253, 262] payload_rows_per_dst=[5420, 5366, 5376, 5538, 5534, 5494, 5538, 5426] dup_rows_per_dst=[2642, 2578, 2574, 2710, 2946, 2844, 2704, 2846] combine_rows_per_src=[5420, 5366, 5376, 5538, 5534, 5494, 5538, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=37984 dup_rows=21884 dup_rows_remote=18764 dedup_ratio=0.333923 router_tokens_per_expert=[230, 266, 251, 236, 256, 258, 272, 287, 270, 251, 256, 266, 267, 259, 232, 271, 268, 259, 265, 261, 249, 254, 270, 232, 248, 246, 252, 227, 288, 231, 249, 238, 260, 239, 258, 272, 257, 263, 250, 278, 283, 256, 233, 267, 240, 269, 261, 257, 285, 248, 243, 280, 270, 249, 254, 252, 286, 271, 247, 247, 245, 280, 239, 254, 246, 260, 243, 260, 242, 279, 254, 246, 247, 265, 240, 262, 247, 243, 260, 234, 273, 252, 226, 215, 252, 261, 256, 274, 272, 258, 230, 267, 254, 237, 230, 252, 240, 301, 259, 253, 244, 266, 247, 258, 256, 256, 261, 252, 271, 241, 248, 278, 248, 255, 253, 251, 246, 253, 283, 237, 252, 271, 293, 263, 270, 260, 240, 267] payload_rows_per_dst=[5534, 5412, 5278, 5336, 5410, 5542, 5472, 5668] dup_rows_per_dst=[2722, 2662, 2480, 2474, 2608, 2990, 2828, 3120] combine_rows_per_src=[5534, 5412, 5278, 5336, 5410, 5542, 5472, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43960 payload_rows_remote=38500 dup_rows=21576 dup_rows_remote=18894 dedup_ratio=0.329224 router_tokens_per_expert=[249, 269, 257, 246, 253, 278, 241, 298, 262, 267, 266, 262, 245, 254, 267, 250, 263, 254, 250, 237, 263, 252, 249, 239, 254, 237, 277, 250, 265, 256, 226, 262, 270, 244, 269, 256, 265, 234, 240, 280, 248, 251, 279, 235, 256, 256, 256, 260, 241, 260, 280, 260, 261, 240, 253, 257, 273, 250, 265, 257, 279, 259, 250, 264, 270, 288, 264, 216, 262, 272, 247, 229, 230, 260, 246, 249, 254, 274, 255, 255, 252, 225, 255, 230, 234, 261, 231, 229, 281, 251, 277, 239, 256, 273, 276, 237, 266, 271, 255, 256, 265, 277, 248, 261, 250, 254, 267, 243, 257, 247, 272, 228, 262, 254, 253, 284, 259, 237, 255, 271, 261, 241, 229, 267, 270, 263, 260, 261] payload_rows_per_dst=[5538, 5506, 5488, 5560, 5460, 5364, 5536, 5508] dup_rows_per_dst=[2790, 2562, 2710, 2738, 2682, 2650, 2698, 2746] combine_rows_per_src=[5538, 5506, 5488, 5560, 5460, 5364, 5536, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38240 dup_rows=21806 dup_rows_remote=19042 dedup_ratio=0.332733 router_tokens_per_expert=[261, 282, 285, 244, 249, 262, 232, 257, 233, 264, 258, 249, 262, 265, 264, 232, 245, 264, 234, 256, 243, 280, 262, 247, 270, 252, 255, 283, 300, 250, 242, 244, 263, 261, 249, 261, 265, 272, 243, 262, 245, 270, 263, 248, 234, 260, 263, 268, 274, 265, 250, 260, 252, 261, 236, 269, 238, 252, 239, 271, 244, 265, 251, 253, 287, 247, 259, 250, 241, 252, 267, 275, 258, 278, 245, 233, 254, 272, 255, 258, 255, 274, 243, 234, 254, 233, 237, 253, 251, 262, 248, 252, 238, 264, 251, 295, 272, 243, 257, 257, 256, 259, 255, 258, 268, 249, 283, 271, 219, 258, 277, 249, 271, 249, 262, 267, 282, 278, 240, 247, 249, 251, 241, 248, 208, 235, 252, 249] payload_rows_per_dst=[5438, 5490, 5464, 5492, 5466, 5396, 5570, 5414] dup_rows_per_dst=[2760, 2764, 2790, 2668, 2796, 2692, 2692, 2644] combine_rows_per_src=[5438, 5490, 5464, 5492, 5466, 5396, 5570, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38204 dup_rows=21880 dup_rows_remote=19048 dedup_ratio=0.333862 router_tokens_per_expert=[268, 276, 254, 270, 264, 231, 271, 240, 240, 238, 271, 272, 266, 233, 254, 263, 274, 251, 266, 273, 231, 282, 257, 239, 256, 247, 235, 255, 242, 255, 254, 226, 275, 248, 273, 256, 262, 263, 260, 246, 256, 271, 260, 247, 256, 253, 238, 262, 268, 247, 244, 274, 265, 249, 274, 280, 281, 230, 238, 276, 262, 245, 259, 250, 246, 253, 273, 257, 272, 259, 294, 254, 248, 250, 242, 250, 217, 260, 271, 265, 266, 272, 248, 236, 247, 247, 257, 219, 242, 262, 249, 281, 262, 266, 281, 257, 242, 269, 273, 274, 265, 249, 244, 267, 243, 243, 260, 246, 262, 246, 257, 255, 252, 249, 252, 250, 249, 283, 283, 242, 238, 270, 244, 233, 256, 243, 237, 267] payload_rows_per_dst=[5460, 5442, 5476, 5452, 5474, 5408, 5430, 5514] dup_rows_per_dst=[2762, 2644, 2776, 2832, 2748, 2776, 2760, 2582] combine_rows_per_src=[5460, 5442, 5476, 5452, 5474, 5408, 5430, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229016 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876440 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313523864 weight_movement_bytes=18874368 total_nvlink_bytes=332398232 weight_to_token_ratio=0.060201 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38240 dup_rows=21848 dup_rows_remote=19048 dedup_ratio=0.333374 router_tokens_per_expert=[247, 271, 258, 243, 247, 265, 242, 244, 254, 262, 251, 258, 251, 259, 281, 247, 236, 252, 263, 265, 268, 260, 256, 252, 249, 249, 277, 264, 286, 256, 249, 236, 277, 254, 230, 252, 262, 246, 260, 274, 273, 255, 253, 261, 277, 249, 240, 279, 275, 255, 261, 287, 273, 260, 241, 241, 263, 224, 245, 252, 273, 232, 275, 232, 274, 255, 249, 258, 223, 252, 261, 285, 292, 264, 251, 257, 262, 275, 266, 256, 299, 272, 281, 253, 276, 247, 263, 222, 262, 231, 248, 277, 250, 289, 255, 255, 252, 257, 238, 232, 233, 225, 253, 254, 225, 260, 254, 242, 251, 269, 248, 256, 228, 244, 269, 260, 236, 240, 247, 279, 239, 252, 275, 264, 240, 233, 235, 289] payload_rows_per_dst=[5448, 5608, 5480, 5490, 5368, 5510, 5336, 5448] dup_rows_per_dst=[2800, 3024, 2730, 2688, 2596, 2836, 2562, 2612] combine_rows_per_src=[5448, 5608, 5480, 5490, 5368, 5510, 5336, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228792 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156687800 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313146808 weight_movement_bytes=18874368 total_nvlink_bytes=332021176 weight_to_token_ratio=0.060273 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=155582464 dispatch_meta_bytes=226992 combine_payload_bytes=155582464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155809456 ingress_bytes=174456832 link_bytes=174456832 token_bytes=311391920 weight_movement_bytes=18874368 total_nvlink_bytes=330266288 weight_to_token_ratio=0.060613 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671559526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229128 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156860168 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313491208 weight_movement_bytes=18874368 total_nvlink_bytes=332365576 weight_to_token_ratio=0.060207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157696000 dispatch_meta_bytes=229576 combine_payload_bytes=157696000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157925576 ingress_bytes=233193472 link_bytes=233193472 token_bytes=315621576 weight_movement_bytes=75497472 total_nvlink_bytes=391119048 weight_to_token_ratio=0.239203 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.234345110e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229008 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712592 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196176 weight_movement_bytes=37748736 total_nvlink_bytes=350944912 weight_to_token_ratio=0.120527 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229152 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860192 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491232 weight_movement_bytes=37748736 total_nvlink_bytes=351239968 weight_to_token_ratio=0.120414 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38218 dup_rows=21778 dup_rows_remote=19058 dedup_ratio=0.332306 router_tokens_per_expert=[240, 252, 275, 261, 270, 230, 253, 228, 243, 250, 247, 262, 251, 264, 242, 251, 235, 274, 229, 272, 243, 256, 269, 241, 232, 270, 269, 259, 268, 246, 301, 239, 261, 251, 264, 235, 248, 216, 255, 259, 247, 282, 253, 247, 266, 249, 253, 269, 276, 293, 246, 253, 300, 275, 301, 257, 222, 256, 248, 238, 272, 249, 266, 269, 233, 257, 237, 232, 247, 246, 262, 254, 244, 269, 251, 249, 268, 255, 257, 271, 253, 255, 265, 246, 259, 268, 255, 247, 249, 248, 235, 264, 258, 249, 284, 253, 253, 259, 277, 247, 243, 275, 244, 244, 247, 269, 272, 260, 263, 260, 273, 244, 258, 278, 249, 265, 258, 267, 272, 243, 257, 247, 275, 248, 231, 271, 256, 245] payload_rows_per_dst=[5406, 5496, 5392, 5550, 5422, 5464, 5540, 5488] dup_rows_per_dst=[2632, 2710, 2718, 2892, 2642, 2712, 2720, 2752] combine_rows_per_src=[5406, 5496, 5392, 5550, 5422, 5464, 5540, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38296 dup_rows=21726 dup_rows_remote=19018 dedup_ratio=0.331512 router_tokens_per_expert=[239, 265, 246, 261, 265, 217, 257, 262, 267, 276, 239, 241, 228, 274, 236, 245, 265, 241, 238, 244, 257, 247, 269, 239, 238, 242, 270, 254, 260, 248, 244, 234, 254, 299, 255, 231, 252, 234, 257, 274, 272, 272, 247, 249, 251, 248, 255, 261, 232, 235, 256, 260, 229, 288, 275, 249, 242, 256, 250, 255, 254, 280, 259, 263, 274, 238, 271, 255, 260, 280, 258, 253, 268, 252, 280, 266, 266, 266, 251, 272, 254, 231, 234, 230, 276, 248, 254, 237, 253, 236, 253, 270, 287, 265, 256, 255, 275, 273, 250, 262, 270, 274, 259, 242, 267, 264, 265, 259, 261, 272, 253, 258, 251, 219, 244, 278, 253, 251, 243, 274, 241, 273, 261, 240, 264, 306, 257, 258] payload_rows_per_dst=[5430, 5402, 5514, 5484, 5560, 5386, 5578, 5456] dup_rows_per_dst=[2606, 2578, 2708, 2682, 2860, 2692, 2830, 2770] combine_rows_per_src=[5430, 5402, 5514, 5484, 5560, 5386, 5578, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229104 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156770032 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313310960 weight_movement_bytes=37748736 total_nvlink_bytes=351059696 weight_to_token_ratio=0.120483 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38366 dup_rows=21660 dup_rows_remote=19048 dedup_ratio=0.330505 router_tokens_per_expert=[270, 260, 242, 243, 262, 236, 236, 261, 257, 241, 282, 265, 272, 259, 238, 237, 250, 260, 260, 258, 217, 254, 250, 242, 243, 242, 251, 287, 257, 231, 272, 248, 251, 291, 270, 280, 259, 249, 249, 255, 254, 252, 268, 247, 242, 250, 272, 255, 278, 241, 256, 260, 240, 276, 268, 268, 274, 269, 246, 248, 293, 271, 285, 257, 280, 245, 256, 288, 276, 267, 226, 251, 236, 257, 243, 239, 263, 284, 249, 233, 248, 264, 241, 237, 256, 243, 253, 247, 269, 243, 241, 248, 245, 254, 280, 292, 281, 241, 248, 268, 255, 237, 312, 231, 248, 258, 281, 268, 245, 255, 254, 264, 234, 236, 266, 253, 244, 238, 249, 231, 265, 249, 258, 278, 238, 249, 247, 276] payload_rows_per_dst=[5450, 5378, 5556, 5556, 5530, 5510, 5540, 5356] dup_rows_per_dst=[2672, 2666, 2732, 2904, 2656, 2612, 2752, 2666] combine_rows_per_src=[5450, 5378, 5556, 5556, 5530, 5510, 5540, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43948 payload_rows_remote=38506 dup_rows=21588 dup_rows_remote=18978 dedup_ratio=0.329407 router_tokens_per_expert=[248, 275, 246, 248, 264, 265, 278, 275, 265, 266, 240, 257, 279, 242, 257, 257, 240, 260, 240, 257, 250, 278, 252, 279, 245, 249, 268, 259, 277, 279, 230, 277, 259, 224, 255, 244, 254, 255, 249, 226, 242, 277, 239, 255, 257, 246, 242, 264, 250, 278, 260, 235, 259, 245, 274, 284, 230, 249, 251, 249, 252, 257, 278, 237, 260, 258, 251, 236, 274, 250, 278, 250, 231, 260, 248, 278, 250, 249, 286, 247, 267, 253, 251, 259, 255, 257, 233, 285, 275, 274, 243, 247, 279, 263, 245, 258, 245, 255, 252, 274, 273, 245, 265, 263, 266, 244, 295, 242, 259, 248, 264, 246, 242, 254, 242, 249, 268, 245, 235, 225, 264, 263, 251, 273, 244, 242, 248, 259] payload_rows_per_dst=[5516, 5444, 5480, 5426, 5524, 5500, 5616, 5442] dup_rows_per_dst=[2808, 2730, 2496, 2682, 2756, 2744, 2762, 2610] combine_rows_per_src=[5516, 5444, 5480, 5426, 5524, 5500, 5616, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229256 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157089672 ingress_bytes=232357888 link_bytes=232357888 token_bytes=313950088 weight_movement_bytes=75497472 total_nvlink_bytes=389447560 weight_to_token_ratio=0.240476 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226338956e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38294 dup_rows=21740 dup_rows_remote=18958 dedup_ratio=0.331726 router_tokens_per_expert=[225, 247, 245, 258, 267, 251, 231, 253, 249, 231, 239, 239, 273, 254, 253, 254, 261, 264, 263, 248, 246, 242, 250, 286, 251, 267, 275, 252, 248, 260, 258, 271, 278, 244, 269, 255, 246, 240, 261, 245, 253, 256, 259, 236, 247, 260, 264, 241, 262, 262, 232, 260, 251, 270, 267, 255, 272, 276, 269, 243, 259, 268, 269, 261, 268, 245, 269, 247, 258, 246, 268, 242, 271, 247, 249, 245, 275, 241, 261, 281, 263, 252, 277, 277, 264, 253, 293, 237, 261, 280, 236, 267, 234, 261, 264, 255, 277, 285, 228, 246, 261, 264, 223, 241, 258, 265, 263, 254, 266, 230, 243, 251, 238, 227, 243, 216, 232, 261, 258, 280, 261, 269, 267, 278, 272, 282, 240, 261] payload_rows_per_dst=[5396, 5502, 5418, 5510, 5524, 5546, 5426, 5474] dup_rows_per_dst=[2542, 2782, 2690, 2842, 2702, 2802, 2684, 2696] combine_rows_per_src=[5396, 5502, 5418, 5510, 5524, 5546, 5426, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38242 dup_rows=21882 dup_rows_remote=19156 dedup_ratio=0.333893 router_tokens_per_expert=[270, 226, 250, 232, 275, 259, 258, 251, 252, 269, 264, 238, 223, 247, 248, 244, 258, 240, 270, 259, 255, 259, 245, 253, 262, 266, 287, 280, 246, 250, 257, 258, 277, 241, 232, 259, 268, 259, 235, 253, 253, 228, 288, 238, 262, 265, 268, 225, 268, 268, 237, 248, 263, 223, 269, 230, 225, 263, 275, 265, 247, 249, 279, 260, 250, 253, 239, 273, 243, 284, 268, 254, 241, 259, 252, 250, 274, 260, 281, 264, 263, 241, 259, 243, 242, 246, 273, 256, 257, 260, 283, 280, 255, 271, 251, 239, 248, 281, 286, 256, 280, 274, 255, 243, 264, 228, 262, 262, 260, 250, 268, 273, 239, 250, 251, 255, 263, 267, 265, 263, 247, 237, 252, 234, 259, 242, 267, 252] payload_rows_per_dst=[5332, 5458, 5438, 5412, 5532, 5520, 5562, 5400] dup_rows_per_dst=[2680, 2832, 2664, 2726, 2758, 2718, 2818, 2686] combine_rows_per_src=[5332, 5458, 5438, 5412, 5532, 5520, 5562, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38428 dup_rows=21672 dup_rows_remote=18996 dedup_ratio=0.330688 router_tokens_per_expert=[257, 242, 233, 267, 255, 268, 281, 266, 265, 258, 270, 266, 247, 263, 245, 258, 222, 256, 244, 265, 234, 270, 243, 251, 229, 212, 267, 249, 244, 278, 279, 291, 260, 246, 255, 250, 303, 276, 265, 243, 242, 276, 257, 262, 280, 253, 257, 256, 264, 222, 263, 228, 253, 233, 262, 280, 263, 246, 261, 259, 274, 253, 244, 241, 274, 247, 251, 266, 247, 247, 265, 239, 269, 268, 233, 250, 251, 258, 270, 221, 261, 217, 230, 240, 252, 240, 282, 248, 273, 273, 269, 265, 254, 242, 302, 269, 250, 269, 255, 255, 245, 279, 275, 236, 274, 247, 261, 251, 279, 258, 218, 274, 259, 239, 267, 247, 238, 266, 255, 287, 247, 257, 274, 234, 265, 234, 271, 227] payload_rows_per_dst=[5536, 5404, 5564, 5478, 5436, 5544, 5488, 5414] dup_rows_per_dst=[2746, 2664, 2798, 2614, 2676, 2690, 2764, 2720] combine_rows_per_src=[5536, 5404, 5564, 5478, 5436, 5544, 5488, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229656 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157376792 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314523928 weight_movement_bytes=18874368 total_nvlink_bytes=333398296 weight_to_token_ratio=0.060009 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38180 dup_rows=21778 dup_rows_remote=18778 dedup_ratio=0.332306 router_tokens_per_expert=[268, 266, 254, 294, 278, 287, 270, 266, 239, 250, 249, 263, 259, 258, 269, 250, 258, 262, 256, 263, 260, 270, 278, 258, 225, 250, 236, 270, 276, 255, 253, 283, 293, 207, 248, 247, 254, 266, 254, 274, 274, 232, 260, 236, 266, 245, 259, 281, 232, 242, 227, 240, 255, 267, 244, 245, 250, 253, 242, 266, 243, 249, 261, 258, 227, 251, 277, 225, 261, 232, 240, 255, 254, 244, 280, 325, 267, 268, 260, 252, 226, 266, 252, 255, 251, 251, 256, 237, 272, 263, 272, 247, 208, 269, 263, 301, 238, 272, 236, 270, 290, 247, 272, 246, 259, 228, 227, 240, 260, 243, 232, 261, 289, 257, 233, 243, 260, 271, 228, 274, 279, 252, 247, 247, 265, 271, 221, 260] payload_rows_per_dst=[5578, 5520, 5486, 5498, 5392, 5602, 5438, 5244] dup_rows_per_dst=[3000, 2794, 2698, 2676, 2618, 2942, 2604, 2446] combine_rows_per_src=[5578, 5520, 5486, 5498, 5392, 5602, 5438, 5244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157720576 dispatch_meta_bytes=229936 combine_payload_bytes=157720576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157950512 ingress_bytes=176594944 link_bytes=176594944 token_bytes=315671088 weight_movement_bytes=18874368 total_nvlink_bytes=334545456 weight_to_token_ratio=0.059791 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692045863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229008 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081232 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933456 weight_movement_bytes=37748736 total_nvlink_bytes=351682192 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229592 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156868824 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313508056 weight_movement_bytes=56623104 total_nvlink_bytes=370131160 weight_to_token_ratio=0.180611 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229696 combine_payload_bytes=157401088 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157630784 ingress_bytes=195149824 link_bytes=195149824 token_bytes=315031872 weight_movement_bytes=37748736 total_nvlink_bytes=352780608 weight_to_token_ratio=0.119825 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869829594e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38116 dup_rows=21974 dup_rows_remote=19150 dedup_ratio=0.335297 router_tokens_per_expert=[239, 241, 260, 240, 243, 231, 228, 246, 255, 296, 277, 264, 268, 270, 243, 253, 257, 280, 276, 246, 255, 266, 254, 254, 274, 253, 252, 265, 237, 252, 266, 248, 241, 254, 258, 249, 245, 260, 262, 250, 241, 262, 255, 251, 245, 248, 258, 280, 274, 253, 261, 273, 256, 245, 259, 282, 283, 250, 276, 233, 254, 236, 259, 257, 266, 244, 249, 285, 268, 276, 262, 243, 278, 247, 231, 239, 230, 259, 263, 234, 240, 254, 233, 273, 257, 225, 236, 283, 288, 261, 270, 255, 254, 260, 241, 219, 283, 256, 275, 243, 258, 245, 245, 273, 266, 277, 232, 230, 232, 290, 268, 262, 229, 249, 282, 240, 244, 276, 286, 257, 252, 260, 274, 249, 244, 252, 268, 249] payload_rows_per_dst=[5470, 5562, 5398, 5434, 5386, 5424, 5446, 5442] dup_rows_per_dst=[2638, 2708, 2720, 2868, 2762, 2674, 2824, 2780] combine_rows_per_src=[5470, 5562, 5398, 5434, 5386, 5424, 5446, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=227832 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156613112 ingress_bytes=194134016 link_bytes=194134016 token_bytes=312998392 weight_movement_bytes=37748736 total_nvlink_bytes=350747128 weight_to_token_ratio=0.120604 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38350 dup_rows=21676 dup_rows_remote=18866 dedup_ratio=0.330750 router_tokens_per_expert=[246, 239, 258, 249, 222, 257, 262, 265, 256, 287, 249, 240, 252, 275, 255, 259, 280, 266, 236, 268, 237, 247, 252, 244, 276, 279, 262, 266, 259, 233, 252, 225, 272, 286, 240, 280, 248, 260, 265, 243, 255, 244, 249, 254, 242, 253, 295, 274, 236, 268, 251, 275, 267, 270, 243, 237, 258, 306, 261, 274, 251, 249, 262, 231, 266, 279, 281, 236, 261, 254, 275, 244, 250, 255, 240, 267, 235, 246, 258, 237, 263, 227, 254, 269, 261, 268, 240, 262, 262, 253, 236, 249, 236, 263, 250, 271, 248, 235, 259, 275, 265, 235, 246, 302, 258, 252, 260, 244, 237, 242, 263, 268, 277, 229, 258, 256, 261, 253, 236, 238, 262, 278, 260, 260, 256, 259, 264, 232] payload_rows_per_dst=[5488, 5462, 5510, 5564, 5432, 5474, 5472, 5458] dup_rows_per_dst=[2654, 2702, 2810, 2714, 2736, 2654, 2706, 2700] combine_rows_per_src=[5488, 5462, 5510, 5564, 5432, 5474, 5472, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229064 combine_payload_bytes=156123136 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156352200 ingress_bytes=193871872 link_bytes=193871872 token_bytes=312475336 weight_movement_bytes=37748736 total_nvlink_bytes=350224072 weight_to_token_ratio=0.120805 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857584887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38332 dup_rows=21816 dup_rows_remote=19082 dedup_ratio=0.332886 router_tokens_per_expert=[268, 244, 276, 246, 267, 268, 246, 272, 263, 258, 266, 257, 254, 272, 267, 252, 238, 250, 228, 256, 279, 247, 279, 252, 235, 279, 257, 257, 237, 281, 231, 246, 267, 260, 258, 268, 290, 273, 264, 237, 249, 256, 252, 234, 258, 256, 257, 265, 240, 264, 281, 271, 255, 250, 272, 245, 295, 259, 241, 244, 229, 233, 254, 262, 269, 253, 251, 247, 256, 261, 261, 253, 265, 231, 257, 252, 257, 249, 240, 265, 249, 233, 260, 267, 246, 273, 253, 250, 258, 257, 289, 238, 248, 236, 259, 245, 272, 262, 235, 246, 257, 247, 235, 235, 237, 275, 256, 253, 278, 263, 243, 266, 253, 236, 278, 246, 232, 274, 261, 262, 250, 251, 255, 261, 265, 268, 254, 267] payload_rows_per_dst=[5538, 5428, 5504, 5420, 5452, 5388, 5470, 5520] dup_rows_per_dst=[2814, 2676, 2784, 2770, 2682, 2734, 2650, 2706] combine_rows_per_src=[5538, 5428, 5504, 5420, 5452, 5388, 5470, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43406 payload_rows_remote=37862 dup_rows=22130 dup_rows_remote=19298 dedup_ratio=0.337677 router_tokens_per_expert=[246, 248, 262, 246, 277, 279, 251, 247, 253, 237, 261, 240, 286, 268, 260, 256, 267, 263, 255, 268, 251, 235, 269, 258, 279, 244, 234, 230, 263, 266, 226, 285, 260, 265, 230, 260, 261, 261, 247, 251, 269, 287, 269, 250, 259, 254, 240, 238, 280, 277, 271, 262, 271, 267, 255, 246, 257, 271, 266, 252, 260, 241, 255, 255, 254, 227, 284, 275, 262, 231, 255, 249, 240, 291, 239, 239, 231, 251, 248, 252, 296, 279, 267, 239, 276, 248, 252, 250, 242, 252, 262, 265, 260, 246, 237, 258, 260, 212, 260, 244, 256, 255, 269, 246, 243, 238, 247, 245, 255, 256, 248, 262, 250, 270, 256, 246, 253, 267, 235, 261, 244, 271, 271, 286, 246, 269, 234, 259] payload_rows_per_dst=[5430, 5316, 5422, 5562, 5314, 5378, 5440, 5544] dup_rows_per_dst=[2804, 2642, 2780, 2950, 2484, 2740, 2898, 2832] combine_rows_per_src=[5430, 5316, 5422, 5562, 5314, 5378, 5440, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228864 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157310464 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392064 weight_movement_bytes=18874368 total_nvlink_bytes=333266432 weight_to_token_ratio=0.060034 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38184 dup_rows=21784 dup_rows_remote=19114 dedup_ratio=0.332397 router_tokens_per_expert=[238, 241, 264, 260, 260, 240, 258, 230, 264, 243, 264, 279, 257, 273, 237, 248, 262, 262, 271, 248, 253, 248, 245, 257, 235, 257, 282, 239, 263, 259, 266, 272, 263, 269, 255, 280, 255, 233, 246, 271, 249, 295, 278, 264, 266, 294, 242, 247, 248, 246, 259, 232, 260, 254, 241, 236, 265, 251, 274, 273, 278, 222, 237, 216, 238, 228, 247, 264, 255, 262, 241, 251, 255, 255, 276, 251, 270, 241, 274, 281, 253, 269, 244, 255, 254, 252, 276, 278, 273, 262, 288, 246, 252, 226, 268, 272, 240, 270, 250, 249, 256, 244, 260, 251, 245, 264, 246, 230, 261, 233, 261, 255, 257, 250, 262, 231, 252, 270, 267, 275, 230, 250, 276, 253, 243, 260, 281, 265] payload_rows_per_dst=[5410, 5568, 5572, 5340, 5428, 5562, 5354, 5518] dup_rows_per_dst=[2806, 2670, 2842, 2540, 2750, 2774, 2676, 2726] combine_rows_per_src=[5410, 5568, 5572, 5340, 5428, 5562, 5354, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38302 dup_rows=21790 dup_rows_remote=19090 dedup_ratio=0.332489 router_tokens_per_expert=[256, 245, 281, 251, 240, 252, 257, 243, 244, 240, 245, 245, 264, 256, 264, 274, 260, 241, 244, 271, 252, 244, 255, 284, 264, 268, 264, 275, 275, 250, 266, 234, 214, 268, 277, 232, 254, 244, 308, 249, 258, 245, 268, 254, 264, 260, 260, 256, 265, 256, 260, 253, 248, 259, 233, 235, 258, 268, 252, 272, 256, 231, 265, 261, 245, 219, 239, 266, 254, 270, 278, 253, 256, 265, 280, 255, 280, 262, 249, 259, 245, 253, 239, 256, 253, 237, 267, 250, 272, 255, 253, 224, 268, 255, 260, 271, 223, 273, 275, 269, 240, 237, 260, 243, 248, 263, 259, 236, 266, 284, 272, 257, 239, 266, 275, 240, 273, 286, 258, 257, 239, 257, 267, 287, 240, 233, 247, 224] payload_rows_per_dst=[5496, 5482, 5508, 5444, 5494, 5470, 5432, 5420] dup_rows_per_dst=[2618, 2812, 2714, 2700, 2766, 2646, 2778, 2756] combine_rows_per_src=[5496, 5482, 5508, 5444, 5494, 5470, 5432, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38298 dup_rows=21806 dup_rows_remote=19090 dedup_ratio=0.332733 router_tokens_per_expert=[261, 229, 294, 251, 252, 258, 235, 241, 256, 239, 260, 246, 267, 262, 259, 280, 274, 286, 270, 254, 284, 260, 249, 275, 231, 267, 231, 260, 253, 280, 260, 255, 258, 251, 247, 282, 233, 222, 246, 275, 274, 247, 247, 238, 266, 245, 243, 253, 235, 258, 264, 237, 247, 253, 257, 245, 216, 272, 247, 249, 250, 275, 263, 279, 246, 245, 246, 239, 246, 284, 241, 266, 245, 276, 240, 268, 278, 253, 255, 246, 266, 264, 276, 234, 244, 262, 265, 264, 246, 247, 245, 246, 226, 263, 238, 254, 256, 265, 250, 249, 256, 252, 240, 251, 275, 263, 241, 264, 281, 239, 238, 275, 279, 266, 283, 275, 280, 236, 269, 256, 248, 264, 269, 276, 262, 245, 259, 239] payload_rows_per_dst=[5442, 5578, 5386, 5424, 5432, 5396, 5472, 5600] dup_rows_per_dst=[2738, 2800, 2668, 2670, 2716, 2684, 2718, 2812] combine_rows_per_src=[5442, 5578, 5386, 5424, 5432, 5396, 5472, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43884 payload_rows_remote=38290 dup_rows=21652 dup_rows_remote=18760 dedup_ratio=0.330383 router_tokens_per_expert=[248, 231, 264, 248, 264, 237, 258, 254, 223, 238, 250, 277, 251, 250, 266, 251, 277, 274, 277, 250, 255, 227, 261, 249, 261, 261, 241, 264, 232, 248, 281, 269, 252, 261, 236, 259, 234, 263, 243, 267, 250, 239, 277, 256, 257, 248, 256, 244, 257, 240, 240, 243, 250, 264, 243, 240, 256, 276, 236, 267, 282, 256, 268, 259, 263, 266, 244, 252, 251, 240, 235, 254, 268, 257, 283, 275, 298, 256, 275, 254, 255, 259, 250, 230, 250, 265, 230, 241, 261, 254, 238, 245, 254, 245, 267, 263, 282, 257, 255, 271, 296, 264, 260, 268, 269, 263, 241, 267, 262, 251, 249, 246, 275, 257, 278, 250, 253, 269, 268, 222, 277, 225, 259, 269, 268, 251, 247, 265] payload_rows_per_dst=[5594, 5496, 5492, 5320, 5514, 5548, 5462, 5458] dup_rows_per_dst=[2892, 2808, 2742, 2368, 2656, 2782, 2684, 2720] combine_rows_per_src=[5594, 5496, 5492, 5320, 5514, 5548, 5462, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229656 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157237528 ingress_bytes=213630976 link_bytes=213630976 token_bytes=314245400 weight_movement_bytes=56623104 total_nvlink_bytes=370868504 weight_to_token_ratio=0.180188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046906899e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155082752 dispatch_meta_bytes=228640 combine_payload_bytes=155082752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155311392 ingress_bytes=173957120 link_bytes=173957120 token_bytes=310394144 weight_movement_bytes=18874368 total_nvlink_bytes=329268512 weight_to_token_ratio=0.060808 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.666771532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229192 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156630856 ingress_bytes=213024768 link_bytes=213024768 token_bytes=313032520 weight_movement_bytes=56623104 total_nvlink_bytes=369655624 weight_to_token_ratio=0.180886 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041098512e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38330 dup_rows=21806 dup_rows_remote=19124 dedup_ratio=0.332733 router_tokens_per_expert=[257, 256, 254, 257, 259, 251, 221, 280, 259, 294, 243, 256, 243, 259, 290, 253, 273, 254, 252, 267, 284, 273, 281, 294, 263, 261, 274, 223, 226, 263, 283, 260, 249, 266, 234, 255, 252, 253, 251, 247, 265, 262, 242, 269, 271, 267, 249, 223, 233, 273, 267, 260, 275, 241, 256, 273, 233, 251, 259, 274, 262, 260, 254, 249, 260, 264, 244, 252, 252, 252, 285, 248, 246, 253, 226, 253, 270, 259, 269, 249, 244, 227, 231, 264, 250, 272, 264, 270, 268, 253, 279, 231, 249, 268, 279, 250, 247, 251, 239, 236, 251, 263, 257, 252, 255, 267, 270, 229, 244, 268, 251, 261, 222, 245, 247, 245, 261, 275, 250, 258, 259, 255, 244, 241, 245, 271, 249, 241] payload_rows_per_dst=[5504, 5588, 5476, 5504, 5414, 5444, 5400, 5400] dup_rows_per_dst=[2760, 2874, 2576, 2726, 2750, 2812, 2682, 2626] combine_rows_per_src=[5504, 5588, 5476, 5504, 5414, 5444, 5400, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229568 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114560 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999552 weight_movement_bytes=37748736 total_nvlink_bytes=351748288 weight_to_token_ratio=0.120219 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229552 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098160 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966768 weight_movement_bytes=18874368 total_nvlink_bytes=332841136 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=228200 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157064040 ingress_bytes=175710208 link_bytes=175710208 token_bytes=313899880 weight_movement_bytes=18874368 total_nvlink_bytes=332774248 weight_to_token_ratio=0.060129 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683568758e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229816 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157229496 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314229176 weight_movement_bytes=37748736 total_nvlink_bytes=351977912 weight_to_token_ratio=0.120131 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43516 payload_rows_remote=38130 dup_rows=22020 dup_rows_remote=19288 dedup_ratio=0.335999 router_tokens_per_expert=[259, 244, 241, 254, 246, 260, 253, 251, 255, 266, 256, 249, 269, 206, 260, 253, 258, 273, 246, 244, 261, 265, 241, 241, 250, 280, 275, 279, 286, 268, 242, 270, 241, 268, 245, 250, 276, 245, 219, 265, 256, 261, 257, 304, 257, 246, 231, 238, 253, 286, 230, 262, 273, 256, 263, 265, 240, 257, 287, 270, 280, 231, 229, 260, 286, 252, 239, 264, 257, 241, 242, 257, 246, 265, 274, 274, 253, 260, 257, 237, 269, 280, 243, 265, 247, 247, 286, 270, 261, 260, 248, 240, 259, 256, 257, 272, 287, 251, 231, 269, 231, 253, 273, 232, 276, 255, 234, 264, 260, 265, 241, 249, 248, 260, 239, 251, 255, 264, 257, 262, 248, 250, 241, 267, 240, 264, 248, 237] payload_rows_per_dst=[5316, 5474, 5386, 5546, 5416, 5502, 5444, 5432] dup_rows_per_dst=[2728, 2884, 2732, 2738, 2792, 2818, 2698, 2630] combine_rows_per_src=[5316, 5474, 5386, 5546, 5416, 5502, 5444, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38294 dup_rows=21848 dup_rows_remote=19108 dedup_ratio=0.333374 router_tokens_per_expert=[269, 229, 264, 257, 241, 264, 271, 229, 261, 251, 271, 249, 257, 280, 236, 235, 259, 267, 272, 244, 258, 249, 255, 242, 247, 253, 256, 228, 256, 233, 227, 261, 269, 236, 265, 245, 263, 269, 277, 261, 279, 248, 258, 264, 264, 277, 255, 255, 228, 238, 255, 274, 272, 250, 296, 273, 253, 245, 285, 233, 275, 257, 265, 265, 247, 245, 257, 234, 229, 264, 243, 257, 290, 250, 239, 271, 225, 257, 262, 247, 255, 254, 243, 261, 247, 268, 256, 248, 243, 253, 256, 252, 267, 234, 260, 270, 259, 254, 250, 279, 263, 269, 286, 251, 258, 246, 272, 282, 260, 247, 277, 241, 267, 272, 221, 227, 254, 280, 287, 243, 245, 242, 265, 235, 270, 291, 214, 257] payload_rows_per_dst=[5472, 5366, 5526, 5532, 5370, 5394, 5622, 5406] dup_rows_per_dst=[2656, 2648, 2844, 2796, 2664, 2740, 2766, 2734] combine_rows_per_src=[5472, 5366, 5526, 5532, 5370, 5394, 5622, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38352 dup_rows=21718 dup_rows_remote=19018 dedup_ratio=0.331390 router_tokens_per_expert=[272, 254, 236, 264, 272, 236, 252, 234, 261, 265, 230, 240, 273, 268, 239, 259, 282, 268, 255, 256, 259, 292, 261, 248, 276, 256, 244, 259, 256, 269, 227, 267, 239, 261, 263, 259, 237, 232, 259, 278, 268, 270, 248, 253, 257, 232, 251, 235, 257, 271, 256, 277, 249, 228, 254, 238, 276, 265, 259, 240, 246, 261, 258, 243, 249, 268, 262, 247, 269, 250, 260, 263, 244, 257, 231, 243, 263, 281, 250, 288, 240, 268, 261, 279, 266, 260, 249, 261, 255, 254, 270, 234, 247, 257, 276, 247, 269, 246, 262, 240, 271, 262, 276, 266, 261, 212, 243, 274, 260, 251, 251, 246, 256, 241, 239, 257, 272, 212, 244, 267, 263, 257, 262, 263, 241, 266, 263, 276] payload_rows_per_dst=[5260, 5462, 5462, 5438, 5560, 5598, 5572, 5466] dup_rows_per_dst=[2338, 2668, 2614, 2718, 2910, 2948, 2822, 2700] combine_rows_per_src=[5260, 5462, 5462, 5438, 5560, 5598, 5572, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38302 dup_rows=21754 dup_rows_remote=18948 dedup_ratio=0.331940 router_tokens_per_expert=[269, 250, 289, 264, 253, 276, 284, 271, 255, 245, 274, 249, 262, 275, 269, 267, 273, 270, 247, 251, 246, 297, 265, 243, 257, 268, 246, 252, 244, 260, 269, 255, 255, 246, 261, 237, 259, 239, 253, 271, 253, 244, 243, 248, 256, 271, 219, 245, 239, 263, 281, 255, 249, 231, 251, 260, 259, 232, 282, 218, 255, 244, 273, 240, 239, 242, 280, 268, 277, 280, 244, 272, 250, 237, 236, 241, 258, 287, 234, 241, 257, 266, 267, 272, 252, 257, 265, 276, 259, 219, 275, 260, 263, 253, 268, 239, 243, 264, 243, 238, 248, 261, 268, 262, 268, 239, 259, 256, 227, 274, 256, 266, 240, 230, 257, 244, 268, 270, 263, 266, 258, 252, 262, 250, 257, 250, 256, 212] payload_rows_per_dst=[5626, 5480, 5388, 5460, 5524, 5452, 5442, 5410] dup_rows_per_dst=[2878, 2806, 2612, 2836, 2648, 2612, 2702, 2660] combine_rows_per_src=[5626, 5480, 5388, 5460, 5524, 5452, 5442, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229672 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156410152 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312590632 weight_movement_bytes=18874368 total_nvlink_bytes=331465000 weight_to_token_ratio=0.060380 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38418 dup_rows=21660 dup_rows_remote=19132 dedup_ratio=0.330505 router_tokens_per_expert=[257, 271, 267, 265, 293, 277, 244, 249, 253, 265, 257, 251, 276, 290, 231, 229, 259, 242, 276, 244, 269, 291, 255, 247, 248, 257, 248, 261, 273, 251, 210, 263, 268, 254, 274, 262, 280, 260, 248, 248, 228, 263, 262, 288, 260, 256, 251, 270, 256, 262, 251, 260, 235, 254, 241, 245, 253, 255, 240, 265, 250, 237, 243, 246, 278, 286, 224, 270, 260, 252, 250, 237, 258, 241, 259, 240, 251, 257, 259, 252, 277, 280, 264, 253, 249, 254, 251, 250, 241, 260, 244, 283, 255, 301, 262, 257, 226, 274, 260, 270, 243, 265, 276, 285, 267, 241, 243, 225, 249, 248, 262, 231, 243, 257, 242, 252, 233, 276, 275, 250, 249, 254, 273, 246, 208, 223, 287, 246] payload_rows_per_dst=[5520, 5454, 5512, 5458, 5464, 5602, 5448, 5418] dup_rows_per_dst=[2830, 2734, 2832, 2528, 2684, 2760, 2682, 2610] combine_rows_per_src=[5520, 5454, 5512, 5458, 5464, 5602, 5448, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229608 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157081832 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313934056 weight_movement_bytes=56623104 total_nvlink_bytes=370557160 weight_to_token_ratio=0.180366 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38330 dup_rows=21640 dup_rows_remote=18798 dedup_ratio=0.330200 router_tokens_per_expert=[273, 254, 250, 265, 263, 251, 262, 246, 251, 230, 261, 262, 252, 235, 248, 241, 270, 248, 252, 253, 254, 240, 263, 260, 240, 246, 270, 261, 231, 263, 247, 244, 243, 252, 239, 223, 249, 261, 248, 238, 250, 230, 267, 247, 244, 258, 259, 263, 258, 240, 286, 255, 276, 257, 224, 226, 250, 265, 273, 265, 256, 265, 274, 280, 258, 255, 267, 263, 257, 264, 264, 260, 282, 260, 275, 264, 262, 274, 240, 259, 250, 227, 262, 277, 271, 240, 251, 279, 251, 245, 257, 261, 253, 283, 256, 232, 248, 244, 256, 242, 279, 250, 278, 276, 262, 259, 252, 252, 250, 259, 224, 278, 279, 264, 278, 270, 267, 258, 241, 272, 239, 261, 235, 263, 280, 251, 245, 250] payload_rows_per_dst=[5494, 5448, 5380, 5572, 5566, 5446, 5464, 5526] dup_rows_per_dst=[2594, 2636, 2562, 2728, 2842, 2744, 2754, 2780] combine_rows_per_src=[5494, 5448, 5380, 5572, 5566, 5446, 5464, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38192 dup_rows=21838 dup_rows_remote=18884 dedup_ratio=0.333221 router_tokens_per_expert=[254, 275, 236, 269, 248, 251, 261, 253, 242, 254, 269, 241, 249, 271, 229, 251, 250, 262, 221, 257, 256, 275, 263, 284, 257, 241, 266, 242, 239, 230, 222, 268, 260, 265, 231, 216, 250, 268, 274, 266, 265, 273, 244, 276, 231, 249, 234, 272, 241, 249, 257, 275, 248, 270, 242, 266, 266, 253, 292, 279, 254, 247, 220, 257, 260, 265, 250, 247, 264, 261, 230, 257, 265, 256, 248, 270, 297, 242, 286, 257, 279, 266, 249, 265, 275, 258, 252, 224, 255, 281, 243, 254, 270, 219, 255, 247, 256, 262, 242, 248, 253, 225, 257, 242, 263, 238, 272, 263, 277, 246, 234, 255, 282, 250, 254, 267, 245, 241, 274, 292, 279, 256, 260, 293, 248, 260, 264, 247] payload_rows_per_dst=[5506, 5636, 5522, 5324, 5452, 5534, 5266, 5458] dup_rows_per_dst=[2954, 2916, 2796, 2448, 2672, 2916, 2524, 2612] combine_rows_per_src=[5506, 5636, 5522, 5324, 5452, 5534, 5266, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229480 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157319272 ingress_bytes=194838528 link_bytes=194838528 token_bytes=314409064 weight_movement_bytes=37748736 total_nvlink_bytes=352157800 weight_to_token_ratio=0.120062 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866846909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38170 dup_rows=21906 dup_rows_remote=19108 dedup_ratio=0.334259 router_tokens_per_expert=[259, 253, 244, 253, 269, 259, 278, 266, 245, 271, 270, 236, 226, 237, 253, 246, 250, 245, 254, 244, 243, 260, 232, 235, 262, 251, 241, 240, 260, 278, 253, 252, 272, 275, 246, 271, 272, 262, 268, 247, 260, 262, 269, 250, 261, 254, 260, 251, 262, 277, 264, 276, 229, 261, 294, 247, 256, 252, 261, 240, 254, 256, 277, 248, 253, 272, 255, 276, 247, 259, 252, 249, 259, 271, 249, 238, 279, 258, 234, 254, 263, 254, 236, 280, 227, 279, 243, 240, 221, 271, 240, 264, 242, 257, 250, 264, 234, 242, 265, 262, 240, 260, 260, 245, 262, 272, 262, 255, 289, 258, 266, 257, 267, 283, 254, 238, 252, 261, 274, 248, 273, 242, 244, 255, 253, 251, 246, 263] payload_rows_per_dst=[5452, 5412, 5472, 5480, 5466, 5424, 5460, 5464] dup_rows_per_dst=[2678, 2588, 2888, 2828, 2744, 2638, 2798, 2744] combine_rows_per_src=[5452, 5412, 5472, 5480, 5466, 5424, 5460, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229000 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157113992 ingress_bytes=213508096 link_bytes=213508096 token_bytes=313998984 weight_movement_bytes=56623104 total_nvlink_bytes=370622088 weight_to_token_ratio=0.180329 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045729523e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=230200 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157590328 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314950456 weight_movement_bytes=18874368 total_nvlink_bytes=333824824 weight_to_token_ratio=0.059928 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=228512 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157228192 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314227872 weight_movement_bytes=37748736 total_nvlink_bytes=351976608 weight_to_token_ratio=0.120132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228304 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156662736 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313097168 weight_movement_bytes=18874368 total_nvlink_bytes=331971536 weight_to_token_ratio=0.060283 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229112 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156573432 ingress_bytes=194093056 link_bytes=194093056 token_bytes=312917752 weight_movement_bytes=37748736 total_nvlink_bytes=350666488 weight_to_token_ratio=0.120635 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859704163e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38322 dup_rows=21690 dup_rows_remote=19024 dedup_ratio=0.330963 router_tokens_per_expert=[253, 278, 241, 241, 267, 228, 264, 262, 254, 248, 265, 232, 258, 277, 253, 256, 253, 269, 284, 269, 263, 223, 240, 258, 248, 252, 256, 295, 256, 258, 258, 233, 264, 264, 251, 247, 276, 270, 259, 256, 253, 247, 270, 263, 236, 252, 236, 251, 253, 269, 273, 250, 266, 252, 244, 234, 265, 261, 239, 272, 261, 227, 247, 257, 267, 271, 236, 262, 257, 245, 264, 255, 267, 233, 280, 243, 241, 270, 256, 254, 240, 253, 266, 231, 257, 266, 250, 240, 253, 246, 260, 271, 255, 272, 252, 284, 242, 253, 262, 257, 238, 247, 274, 241, 256, 283, 218, 243, 256, 269, 252, 241, 262, 265, 265, 243, 279, 277, 261, 246, 255, 229, 255, 295, 252, 257, 268, 273] payload_rows_per_dst=[5416, 5558, 5524, 5488, 5440, 5526, 5384, 5510] dup_rows_per_dst=[2738, 2672, 2666, 2652, 2762, 2666, 2680, 2854] combine_rows_per_src=[5416, 5558, 5524, 5488, 5440, 5526, 5384, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38236 dup_rows=21752 dup_rows_remote=19044 dedup_ratio=0.331909 router_tokens_per_expert=[278, 289, 265, 246, 232, 259, 277, 277, 245, 283, 237, 263, 247, 224, 250, 247, 266, 244, 287, 241, 253, 274, 252, 259, 241, 230, 254, 295, 277, 281, 235, 252, 266, 259, 259, 264, 252, 250, 252, 274, 292, 261, 218, 228, 265, 270, 236, 244, 279, 263, 259, 245, 213, 245, 242, 253, 255, 235, 265, 244, 281, 261, 248, 244, 234, 268, 252, 238, 261, 254, 264, 246, 261, 285, 253, 277, 241, 258, 240, 247, 270, 226, 279, 264, 229, 255, 276, 264, 270, 269, 243, 270, 253, 245, 253, 262, 263, 270, 244, 260, 255, 247, 274, 258, 247, 293, 259, 261, 239, 241, 240, 237, 235, 289, 274, 267, 245, 239, 237, 265, 266, 247, 277, 238, 240, 271, 238, 263] payload_rows_per_dst=[5484, 5530, 5504, 5422, 5472, 5548, 5370, 5454] dup_rows_per_dst=[2754, 2752, 2676, 2642, 2686, 2708, 2806, 2728] combine_rows_per_src=[5484, 5530, 5504, 5422, 5472, 5548, 5370, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38270 dup_rows=21808 dup_rows_remote=19136 dedup_ratio=0.332764 router_tokens_per_expert=[239, 266, 248, 257, 261, 245, 255, 272, 267, 268, 245, 258, 244, 265, 259, 266, 262, 253, 296, 277, 234, 230, 278, 235, 239, 272, 268, 260, 233, 245, 232, 268, 254, 276, 279, 255, 240, 272, 259, 264, 259, 266, 231, 292, 276, 285, 241, 249, 232, 247, 258, 241, 246, 259, 249, 242, 270, 234, 257, 243, 240, 254, 264, 261, 255, 245, 251, 262, 209, 217, 255, 253, 277, 267, 244, 254, 265, 238, 255, 278, 233, 261, 267, 259, 264, 244, 237, 250, 255, 286, 262, 266, 263, 265, 253, 255, 263, 271, 278, 254, 263, 255, 286, 244, 237, 239, 263, 258, 272, 281, 254, 249, 252, 262, 246, 236, 269, 235, 234, 266, 264, 262, 267, 229, 248, 258, 280, 256] payload_rows_per_dst=[5302, 5444, 5590, 5374, 5562, 5466, 5532, 5458] dup_rows_per_dst=[2456, 2720, 2806, 2618, 2960, 2738, 2838, 2672] combine_rows_per_src=[5302, 5444, 5590, 5374, 5562, 5466, 5532, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229384 combine_payload_bytes=156966912 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157196296 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163208 weight_movement_bytes=18874368 total_nvlink_bytes=333037576 weight_to_token_ratio=0.060078 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38268 dup_rows=21654 dup_rows_remote=18910 dedup_ratio=0.330414 router_tokens_per_expert=[263, 272, 268, 259, 268, 282, 250, 262, 269, 242, 246, 265, 292, 257, 261, 266, 255, 249, 272, 249, 277, 224, 237, 275, 249, 252, 268, 240, 257, 235, 240, 259, 263, 237, 280, 254, 257, 243, 264, 263, 269, 234, 248, 249, 255, 251, 247, 243, 256, 251, 261, 279, 258, 246, 295, 254, 253, 231, 281, 289, 232, 296, 257, 240, 242, 259, 266, 267, 272, 236, 233, 260, 261, 236, 218, 287, 246, 247, 256, 248, 275, 248, 233, 277, 251, 250, 243, 262, 221, 251, 238, 248, 255, 258, 257, 262, 285, 251, 255, 239, 289, 282, 244, 261, 220, 257, 280, 240, 260, 263, 238, 276, 279, 251, 262, 256, 252, 243, 268, 253, 236, 251, 239, 259, 246, 272, 253, 249] payload_rows_per_dst=[5570, 5458, 5444, 5614, 5486, 5412, 5484, 5414] dup_rows_per_dst=[2874, 2618, 2670, 2744, 2582, 2646, 2796, 2724] combine_rows_per_src=[5570, 5458, 5444, 5614, 5486, 5412, 5484, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38284 dup_rows=21706 dup_rows_remote=18882 dedup_ratio=0.331207 router_tokens_per_expert=[253, 255, 273, 225, 238, 256, 241, 248, 242, 249, 259, 250, 260, 281, 263, 282, 246, 244, 256, 246, 286, 248, 264, 258, 277, 263, 284, 261, 264, 273, 255, 260, 226, 236, 262, 243, 270, 265, 266, 266, 253, 233, 238, 268, 248, 234, 280, 270, 250, 249, 261, 256, 253, 267, 266, 276, 245, 228, 254, 267, 249, 248, 246, 259, 257, 233, 246, 240, 292, 255, 237, 232, 257, 267, 239, 259, 249, 261, 253, 237, 246, 243, 264, 272, 263, 247, 215, 286, 262, 247, 285, 275, 280, 243, 241, 225, 255, 247, 245, 251, 240, 240, 273, 250, 267, 292, 266, 251, 252, 255, 245, 238, 268, 266, 247, 238, 271, 261, 249, 267, 272, 272, 256, 281, 249, 256, 282, 266] payload_rows_per_dst=[5444, 5546, 5434, 5532, 5456, 5484, 5358, 5576] dup_rows_per_dst=[2706, 2824, 2682, 2616, 2572, 2704, 2776, 2826] combine_rows_per_src=[5444, 5546, 5434, 5532, 5456, 5484, 5358, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229120 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156843776 ingress_bytes=194363392 link_bytes=194363392 token_bytes=313458432 weight_movement_bytes=37748736 total_nvlink_bytes=351207168 weight_to_token_ratio=0.120427 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862294389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229624 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983544 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737464 weight_movement_bytes=37748736 total_nvlink_bytes=351486200 weight_to_token_ratio=0.120320 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38258 dup_rows=21842 dup_rows_remote=19122 dedup_ratio=0.333282 router_tokens_per_expert=[264, 245, 270, 271, 232, 238, 268, 232, 241, 271, 248, 254, 251, 263, 243, 259, 244, 257, 265, 275, 237, 265, 237, 278, 259, 235, 263, 237, 252, 271, 265, 250, 236, 260, 259, 248, 287, 281, 273, 262, 258, 253, 261, 261, 258, 248, 272, 282, 265, 259, 265, 257, 273, 238, 255, 261, 276, 233, 282, 278, 283, 272, 247, 223, 291, 263, 261, 256, 250, 249, 271, 264, 238, 260, 255, 263, 240, 222, 268, 228, 249, 263, 245, 251, 246, 217, 254, 250, 261, 247, 265, 235, 238, 281, 245, 274, 232, 248, 254, 283, 253, 279, 253, 267, 254, 240, 267, 278, 237, 244, 232, 248, 237, 248, 247, 289, 253, 274, 236, 266, 297, 256, 252, 221, 247, 239, 271, 260] payload_rows_per_dst=[5436, 5532, 5516, 5548, 5446, 5428, 5322, 5466] dup_rows_per_dst=[2720, 3036, 2832, 2786, 2706, 2614, 2428, 2720] combine_rows_per_src=[5436, 5532, 5516, 5548, 5446, 5428, 5322, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38178 dup_rows=21868 dup_rows_remote=19188 dedup_ratio=0.333679 router_tokens_per_expert=[263, 240, 266, 258, 248, 252, 245, 228, 271, 264, 235, 276, 248, 280, 270, 264, 251, 244, 269, 238, 266, 313, 241, 283, 235, 270, 262, 251, 258, 239, 266, 257, 240, 264, 252, 280, 240, 259, 273, 265, 232, 258, 238, 241, 278, 259, 281, 254, 264, 258, 231, 241, 265, 251, 274, 242, 234, 250, 259, 239, 261, 246, 244, 277, 244, 263, 268, 258, 264, 245, 256, 237, 259, 241, 270, 265, 253, 255, 256, 251, 263, 243, 247, 256, 264, 277, 260, 238, 248, 257, 266, 253, 254, 257, 262, 261, 253, 252, 265, 234, 218, 272, 274, 275, 244, 245, 253, 253, 246, 238, 278, 266, 252, 249, 258, 282, 253, 279, 253, 277, 227, 286, 238, 262, 246, 252, 237, 259] payload_rows_per_dst=[5450, 5460, 5500, 5404, 5490, 5466, 5446, 5452] dup_rows_per_dst=[2766, 2826, 2728, 2668, 2680, 2746, 2686, 2768] combine_rows_per_src=[5450, 5460, 5500, 5404, 5490, 5466, 5446, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38344 dup_rows=21812 dup_rows_remote=19160 dedup_ratio=0.332825 router_tokens_per_expert=[233, 263, 254, 246, 265, 236, 220, 271, 271, 253, 272, 246, 239, 298, 246, 243, 259, 260, 263, 253, 260, 255, 283, 268, 271, 274, 242, 269, 232, 229, 269, 268, 231, 253, 293, 235, 259, 244, 239, 243, 259, 266, 270, 279, 234, 256, 236, 244, 240, 240, 253, 251, 258, 258, 271, 256, 255, 264, 243, 247, 263, 281, 273, 283, 240, 270, 261, 281, 243, 227, 264, 259, 250, 250, 260, 266, 263, 266, 262, 252, 275, 279, 261, 280, 251, 254, 255, 236, 268, 278, 259, 275, 230, 251, 252, 233, 264, 254, 242, 221, 234, 261, 277, 255, 252, 264, 234, 247, 275, 240, 273, 223, 249, 256, 272, 277, 256, 245, 255, 280, 263, 265, 256, 265, 249, 253, 242, 230] payload_rows_per_dst=[5430, 5512, 5462, 5508, 5408, 5512, 5380, 5512] dup_rows_per_dst=[2682, 2798, 2620, 2764, 2820, 2762, 2652, 2714] combine_rows_per_src=[5430, 5512, 5462, 5508, 5408, 5512, 5380, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228712 combine_payload_bytes=156745728 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156974440 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720168 weight_movement_bytes=18874368 total_nvlink_bytes=332594536 weight_to_token_ratio=0.060163 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228664 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157039928 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851192 weight_movement_bytes=18874368 total_nvlink_bytes=332725560 weight_to_token_ratio=0.060138 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229520 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934288 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639056 weight_movement_bytes=18874368 total_nvlink_bytes=332513424 weight_to_token_ratio=0.060179 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229464 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156606552 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312983640 weight_movement_bytes=18874368 total_nvlink_bytes=331858008 weight_to_token_ratio=0.060305 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38176 dup_rows=21940 dup_rows_remote=19236 dedup_ratio=0.334778 router_tokens_per_expert=[276, 280, 246, 257, 255, 260, 235, 238, 263, 275, 235, 254, 237, 263, 273, 262, 268, 237, 252, 230, 276, 257, 241, 250, 267, 274, 258, 244, 245, 251, 240, 258, 244, 254, 269, 225, 228, 244, 261, 266, 257, 256, 251, 258, 258, 253, 271, 267, 269, 269, 261, 260, 249, 275, 241, 251, 256, 250, 259, 288, 249, 257, 266, 289, 272, 269, 274, 256, 260, 261, 233, 251, 252, 261, 251, 267, 268, 246, 253, 265, 254, 237, 251, 236, 262, 250, 248, 262, 240, 260, 243, 236, 280, 256, 254, 250, 271, 253, 245, 299, 248, 242, 232, 304, 271, 269, 251, 253, 278, 270, 239, 236, 248, 267, 236, 250, 257, 231, 255, 252, 263, 243, 257, 258, 264, 261, 246, 253] payload_rows_per_dst=[5428, 5396, 5420, 5574, 5500, 5424, 5460, 5394] dup_rows_per_dst=[2790, 2700, 2704, 2804, 2778, 2614, 2862, 2688] combine_rows_per_src=[5428, 5396, 5420, 5574, 5500, 5424, 5460, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=230016 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157287040 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314344064 weight_movement_bytes=18874368 total_nvlink_bytes=333218432 weight_to_token_ratio=0.060044 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=37986 dup_rows=21824 dup_rows_remote=18830 dedup_ratio=0.333008 router_tokens_per_expert=[275, 261, 274, 281, 272, 242, 269, 247, 243, 241, 244, 256, 250, 281, 234, 221, 260, 256, 243, 266, 264, 279, 281, 246, 251, 230, 238, 281, 239, 248, 261, 265, 249, 248, 276, 244, 238, 281, 250, 263, 279, 254, 249, 258, 236, 266, 243, 248, 238, 283, 270, 264, 256, 271, 281, 248, 266, 258, 240, 262, 264, 259, 246, 244, 279, 249, 230, 271, 251, 242, 265, 258, 223, 270, 275, 254, 266, 258, 248, 263, 236, 269, 260, 228, 242, 237, 254, 228, 245, 277, 265, 216, 279, 250, 267, 253, 247, 255, 261, 234, 247, 236, 294, 268, 237, 253, 255, 256, 256, 261, 243, 272, 287, 252, 240, 259, 253, 271, 252, 259, 261, 254, 260, 246, 298, 225, 258, 279] payload_rows_per_dst=[5356, 5436, 5346, 5564, 5534, 5344, 5406, 5726] dup_rows_per_dst=[2414, 2780, 2702, 2734, 2786, 2668, 2746, 2994] combine_rows_per_src=[5356, 5436, 5346, 5564, 5534, 5344, 5406, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38120 dup_rows=21940 dup_rows_remote=19042 dedup_ratio=0.334778 router_tokens_per_expert=[286, 270, 289, 257, 266, 254, 258, 240, 281, 257, 262, 216, 254, 269, 267, 243, 239, 253, 260, 257, 277, 232, 249, 265, 260, 226, 261, 254, 246, 269, 266, 244, 271, 245, 255, 239, 246, 269, 261, 243, 263, 235, 227, 266, 276, 264, 268, 245, 250, 268, 280, 222, 254, 268, 236, 223, 242, 249, 256, 269, 263, 255, 224, 266, 267, 272, 242, 281, 256, 253, 269, 268, 223, 240, 284, 234, 238, 267, 237, 219, 266, 263, 264, 273, 257, 241, 286, 237, 261, 251, 272, 264, 256, 250, 293, 253, 280, 238, 237, 255, 260, 245, 246, 259, 277, 269, 247, 242, 258, 251, 257, 267, 266, 253, 293, 233, 265, 252, 236, 250, 252, 254, 240, 271, 247, 266, 278, 262] payload_rows_per_dst=[5528, 5428, 5396, 5428, 5384, 5476, 5426, 5530] dup_rows_per_dst=[2810, 2688, 2750, 2622, 2716, 2898, 2750, 2706] combine_rows_per_src=[5528, 5428, 5396, 5428, 5384, 5476, 5426, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229648 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598544 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967440 weight_movement_bytes=18874368 total_nvlink_bytes=331841808 weight_to_token_ratio=0.060308 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38370 dup_rows=21738 dup_rows_remote=19128 dedup_ratio=0.331696 router_tokens_per_expert=[282, 264, 253, 260, 282, 246, 258, 273, 279, 254, 268, 280, 298, 266, 274, 258, 262, 252, 272, 248, 284, 263, 275, 230, 239, 249, 275, 235, 243, 270, 266, 246, 276, 248, 250, 245, 228, 236, 259, 267, 244, 252, 249, 245, 280, 236, 238, 268, 276, 239, 253, 272, 229, 256, 247, 261, 266, 270, 230, 257, 255, 226, 264, 218, 265, 268, 250, 229, 260, 246, 256, 248, 247, 224, 240, 226, 239, 272, 274, 293, 280, 265, 260, 266, 236, 269, 247, 261, 263, 266, 261, 254, 275, 260, 236, 268, 267, 288, 293, 250, 267, 234, 252, 268, 248, 238, 251, 251, 206, 261, 261, 266, 259, 244, 251, 271, 260, 230, 257, 244, 252, 249, 226, 258, 260, 253, 240, 265] payload_rows_per_dst=[5580, 5438, 5504, 5428, 5454, 5568, 5450, 5376] dup_rows_per_dst=[3010, 2780, 2538, 2610, 2620, 2766, 2752, 2662] combine_rows_per_src=[5580, 5438, 5504, 5428, 5454, 5568, 5450, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=227264 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155817920 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311408576 weight_movement_bytes=18874368 total_nvlink_bytes=330282944 weight_to_token_ratio=0.060610 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228648 combine_payload_bytes=156139520 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156368168 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507688 weight_movement_bytes=18874368 total_nvlink_bytes=331382056 weight_to_token_ratio=0.060396 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38126 dup_rows=21946 dup_rows_remote=19132 dedup_ratio=0.334869 router_tokens_per_expert=[275, 241, 242, 258, 212, 271, 264, 262, 247, 229, 263, 264, 252, 251, 270, 250, 273, 295, 243, 259, 275, 237, 248, 257, 261, 252, 262, 266, 250, 247, 250, 264, 256, 255, 266, 228, 257, 276, 260, 268, 265, 268, 258, 263, 265, 260, 246, 271, 235, 255, 268, 249, 276, 251, 259, 256, 250, 247, 279, 256, 245, 261, 270, 239, 229, 253, 262, 257, 270, 261, 242, 269, 269, 252, 260, 252, 263, 261, 254, 247, 240, 268, 234, 227, 248, 274, 259, 275, 247, 214, 252, 258, 248, 246, 255, 265, 240, 267, 249, 285, 245, 272, 251, 237, 264, 275, 261, 259, 270, 266, 262, 240, 235, 274, 268, 247, 260, 241, 250, 280, 261, 248, 273, 261, 255, 233, 235, 245] payload_rows_per_dst=[5454, 5464, 5490, 5428, 5468, 5442, 5440, 5404] dup_rows_per_dst=[2648, 2814, 2834, 2764, 2734, 2578, 2846, 2728] combine_rows_per_src=[5454, 5464, 5490, 5428, 5468, 5442, 5440, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229992 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157393512 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314557032 weight_movement_bytes=37748736 total_nvlink_bytes=352305768 weight_to_token_ratio=0.120006 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38164 dup_rows=21814 dup_rows_remote=19018 dedup_ratio=0.332855 router_tokens_per_expert=[228, 255, 262, 295, 250, 257, 285, 270, 257, 252, 289, 251, 240, 259, 273, 253, 249, 257, 259, 253, 246, 250, 231, 228, 256, 256, 231, 261, 249, 235, 253, 266, 252, 257, 247, 239, 227, 267, 256, 237, 252, 254, 284, 259, 285, 256, 260, 242, 279, 262, 261, 258, 248, 251, 296, 243, 221, 253, 260, 253, 258, 250, 272, 247, 239, 245, 245, 272, 258, 269, 231, 300, 260, 257, 276, 251, 241, 240, 249, 272, 275, 259, 236, 261, 260, 263, 275, 241, 249, 245, 272, 275, 251, 265, 256, 242, 233, 262, 274, 229, 273, 258, 266, 238, 267, 273, 255, 241, 267, 268, 287, 286, 253, 255, 255, 270, 251, 264, 247, 242, 283, 236, 230, 249, 240, 257, 256, 231] payload_rows_per_dst=[5570, 5398, 5448, 5528, 5398, 5460, 5558, 5362] dup_rows_per_dst=[2782, 2562, 2700, 2696, 2812, 2790, 2796, 2676] combine_rows_per_src=[5570, 5398, 5448, 5528, 5398, 5460, 5558, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38190 dup_rows=21770 dup_rows_remote=18888 dedup_ratio=0.332184 router_tokens_per_expert=[237, 260, 249, 245, 271, 257, 232, 251, 238, 262, 249, 254, 300, 257, 252, 263, 276, 240, 271, 251, 285, 249, 269, 243, 254, 261, 249, 252, 234, 247, 243, 241, 249, 255, 242, 234, 251, 245, 253, 265, 250, 244, 264, 256, 262, 248, 268, 260, 251, 245, 266, 276, 290, 264, 236, 274, 266, 261, 264, 241, 268, 249, 269, 265, 231, 257, 249, 240, 269, 247, 268, 296, 241, 221, 251, 230, 263, 243, 269, 265, 240, 242, 279, 250, 284, 273, 243, 233, 239, 229, 270, 281, 276, 261, 274, 290, 263, 252, 243, 254, 261, 280, 274, 243, 275, 252, 251, 249, 247, 257, 238, 240, 235, 292, 267, 235, 239, 268, 264, 248, 234, 262, 250, 255, 278, 254, 261, 270] payload_rows_per_dst=[5576, 5400, 5494, 5446, 5446, 5510, 5330, 5564] dup_rows_per_dst=[2882, 2742, 2834, 2622, 2634, 2804, 2592, 2660] combine_rows_per_src=[5576, 5400, 5494, 5446, 5446, 5510, 5330, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38458 dup_rows=21642 dup_rows_remote=18936 dedup_ratio=0.330231 router_tokens_per_expert=[275, 229, 270, 261, 259, 270, 250, 236, 243, 250, 270, 257, 243, 242, 269, 254, 267, 264, 253, 257, 248, 267, 228, 238, 289, 264, 238, 256, 278, 253, 261, 254, 276, 266, 249, 261, 263, 280, 235, 266, 239, 277, 272, 244, 242, 260, 258, 253, 254, 254, 269, 256, 249, 248, 258, 261, 244, 262, 253, 268, 254, 254, 270, 245, 238, 247, 245, 264, 253, 247, 249, 235, 278, 264, 280, 241, 246, 272, 265, 247, 242, 257, 262, 262, 249, 264, 255, 278, 234, 254, 264, 263, 255, 260, 250, 275, 269, 250, 256, 250, 247, 252, 238, 278, 242, 239, 269, 258, 266, 236, 247, 223, 270, 251, 247, 261, 261, 265, 236, 257, 258, 266, 264, 235, 256, 275, 256, 262] payload_rows_per_dst=[5498, 5478, 5514, 5496, 5436, 5486, 5444, 5542] dup_rows_per_dst=[2658, 2752, 2768, 2702, 2706, 2762, 2596, 2698] combine_rows_per_src=[5498, 5478, 5514, 5496, 5436, 5486, 5444, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229032 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156393128 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557224 weight_movement_bytes=18874368 total_nvlink_bytes=331431592 weight_to_token_ratio=0.060387 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38268 dup_rows=21770 dup_rows_remote=18992 dedup_ratio=0.332184 router_tokens_per_expert=[245, 245, 257, 261, 235, 258, 246, 248, 249, 254, 266, 275, 265, 276, 252, 251, 238, 279, 272, 255, 267, 238, 252, 268, 255, 264, 255, 274, 275, 282, 251, 244, 275, 252, 260, 243, 262, 275, 281, 245, 263, 211, 264, 235, 274, 287, 257, 254, 266, 263, 244, 258, 235, 237, 251, 237, 283, 260, 258, 258, 251, 273, 237, 247, 264, 265, 224, 247, 263, 227, 267, 247, 224, 251, 250, 261, 263, 294, 250, 263, 269, 260, 239, 248, 262, 265, 254, 259, 257, 258, 251, 277, 236, 275, 257, 244, 263, 289, 231, 285, 270, 260, 269, 240, 261, 256, 248, 253, 261, 250, 239, 256, 271, 267, 234, 247, 244, 226, 267, 262, 252, 255, 266, 250, 257, 221, 256, 243] payload_rows_per_dst=[5468, 5500, 5498, 5474, 5416, 5546, 5452, 5412] dup_rows_per_dst=[2698, 2838, 2778, 2642, 2704, 2676, 2810, 2624] combine_rows_per_src=[5468, 5500, 5498, 5474, 5416, 5546, 5452, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228728 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548472 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868216 weight_movement_bytes=18874368 total_nvlink_bytes=331742584 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157523968 dispatch_meta_bytes=229576 combine_payload_bytes=157523968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157753544 ingress_bytes=176398336 link_bytes=176398336 token_bytes=315277512 weight_movement_bytes=18874368 total_nvlink_bytes=334151880 weight_to_token_ratio=0.059866 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690162062e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228312 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156654552 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313080792 weight_movement_bytes=18874368 total_nvlink_bytes=331955160 weight_to_token_ratio=0.060286 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38336 dup_rows=21628 dup_rows_remote=18648 dedup_ratio=0.330017 router_tokens_per_expert=[266, 255, 246, 241, 215, 247, 281, 261, 235, 259, 278, 230, 236, 250, 259, 278, 249, 232, 259, 255, 255, 293, 269, 254, 289, 276, 257, 273, 262, 272, 258, 257, 260, 259, 256, 271, 275, 260, 260, 244, 252, 263, 245, 241, 234, 233, 265, 267, 266, 242, 254, 258, 256, 269, 240, 277, 268, 249, 241, 247, 270, 254, 259, 220, 254, 261, 271, 250, 238, 273, 265, 237, 248, 266, 249, 265, 242, 271, 262, 241, 280, 259, 251, 254, 246, 271, 243, 266, 248, 247, 268, 271, 262, 248, 248, 259, 290, 256, 257, 235, 285, 261, 261, 279, 273, 263, 239, 225, 254, 262, 265, 232, 254, 252, 274, 254, 249, 277, 236, 227, 254, 245, 225, 266, 237, 254, 258, 253] payload_rows_per_dst=[5442, 5552, 5300, 5590, 5510, 5512, 5430, 5572] dup_rows_per_dst=[2632, 2640, 2364, 2860, 2676, 2926, 2550, 2980] combine_rows_per_src=[5442, 5552, 5300, 5590, 5510, 5512, 5430, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38474 dup_rows=21638 dup_rows_remote=18960 dedup_ratio=0.330170 router_tokens_per_expert=[260, 266, 263, 249, 271, 246, 263, 268, 277, 259, 253, 235, 260, 232, 255, 233, 232, 250, 245, 265, 261, 250, 267, 257, 255, 242, 250, 275, 247, 236, 238, 259, 278, 273, 253, 298, 240, 263, 247, 269, 268, 248, 250, 235, 250, 259, 231, 259, 275, 268, 248, 260, 271, 228, 283, 258, 270, 266, 271, 239, 279, 275, 244, 260, 255, 237, 237, 254, 256, 255, 261, 257, 247, 262, 260, 256, 273, 236, 255, 276, 250, 227, 264, 269, 262, 266, 250, 251, 276, 219, 264, 233, 267, 257, 251, 245, 259, 248, 232, 270, 257, 273, 273, 248, 261, 265, 256, 263, 239, 253, 233, 234, 251, 237, 242, 275, 234, 231, 267, 285, 255, 249, 261, 264, 288, 263, 269, 270] payload_rows_per_dst=[5408, 5446, 5504, 5666, 5490, 5424, 5462, 5498] dup_rows_per_dst=[2772, 2612, 2738, 2724, 2664, 2678, 2666, 2784] combine_rows_per_src=[5408, 5446, 5504, 5666, 5490, 5424, 5462, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229040 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156974768 ingress_bytes=213368832 link_bytes=213368832 token_bytes=313720496 weight_movement_bytes=56623104 total_nvlink_bytes=370343600 weight_to_token_ratio=0.180489 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044395164e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38108 dup_rows=21972 dup_rows_remote=19206 dedup_ratio=0.335266 router_tokens_per_expert=[245, 245, 262, 280, 243, 271, 252, 257, 260, 252, 243, 250, 258, 232, 258, 250, 251, 256, 263, 248, 276, 234, 228, 237, 242, 257, 252, 219, 261, 234, 254, 296, 253, 248, 290, 228, 265, 276, 270, 268, 260, 254, 298, 279, 236, 283, 272, 245, 247, 257, 237, 274, 286, 271, 266, 268, 246, 246, 242, 260, 242, 261, 271, 237, 247, 260, 252, 245, 267, 257, 270, 247, 226, 251, 252, 248, 234, 257, 242, 275, 237, 244, 244, 244, 276, 250, 239, 259, 229, 238, 294, 246, 250, 241, 249, 243, 262, 293, 263, 264, 263, 296, 275, 260, 263, 266, 272, 271, 261, 240, 223, 272, 237, 278, 243, 271, 279, 251, 265, 251, 247, 263, 264, 255, 270, 259, 250, 226] payload_rows_per_dst=[5436, 5364, 5546, 5456, 5398, 5330, 5572, 5462] dup_rows_per_dst=[2680, 2652, 2904, 2766, 2662, 2636, 2916, 2756] combine_rows_per_src=[5436, 5364, 5546, 5456, 5398, 5330, 5572, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=227936 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157252192 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314276448 weight_movement_bytes=18874368 total_nvlink_bytes=333150816 weight_to_token_ratio=0.060057 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157589504 dispatch_meta_bytes=229736 combine_payload_bytes=157589504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157819240 ingress_bytes=176463872 link_bytes=176463872 token_bytes=315408744 weight_movement_bytes=18874368 total_nvlink_bytes=334283112 weight_to_token_ratio=0.059841 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690789996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38374 dup_rows=21678 dup_rows_remote=18862 dedup_ratio=0.330780 router_tokens_per_expert=[247, 269, 244, 249, 244, 274, 240, 233, 281, 255, 260, 233, 255, 236, 257, 264, 260, 271, 255, 253, 267, 264, 271, 266, 277, 249, 259, 226, 257, 242, 285, 248, 236, 247, 263, 273, 275, 258, 233, 250, 267, 228, 256, 238, 281, 267, 262, 273, 241, 240, 266, 264, 251, 241, 273, 259, 254, 274, 248, 262, 269, 254, 248, 272, 249, 252, 263, 250, 263, 264, 260, 237, 256, 242, 250, 259, 257, 262, 266, 268, 281, 261, 226, 257, 261, 261, 246, 268, 263, 281, 256, 249, 258, 261, 273, 245, 224, 259, 254, 242, 253, 277, 276, 278, 253, 244, 263, 273, 281, 262, 263, 262, 231, 244, 232, 235, 255, 238, 235, 257, 253, 252, 230, 257, 227, 246, 280, 273] payload_rows_per_dst=[5412, 5484, 5508, 5502, 5500, 5568, 5520, 5364] dup_rows_per_dst=[2670, 2816, 2706, 2730, 2696, 2726, 2808, 2526] combine_rows_per_src=[5412, 5484, 5508, 5502, 5500, 5568, 5520, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=229256 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156319624 ingress_bytes=174964736 link_bytes=174964736 token_bytes=312409992 weight_movement_bytes=18874368 total_nvlink_bytes=331284360 weight_to_token_ratio=0.060415 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676426012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38278 dup_rows=21784 dup_rows_remote=18956 dedup_ratio=0.332397 router_tokens_per_expert=[263, 245, 247, 241, 252, 285, 249, 297, 249, 230, 268, 255, 254, 244, 229, 250, 271, 230, 261, 247, 257, 250, 241, 276, 241, 255, 242, 253, 247, 270, 239, 278, 248, 265, 234, 273, 296, 211, 289, 218, 262, 298, 265, 264, 250, 233, 269, 222, 256, 237, 285, 229, 265, 247, 224, 236, 246, 240, 265, 256, 274, 273, 244, 273, 261, 256, 231, 274, 230, 262, 274, 273, 269, 285, 247, 246, 275, 232, 273, 247, 274, 276, 279, 275, 278, 282, 251, 269, 238, 204, 250, 251, 269, 270, 230, 248, 254, 258, 253, 238, 243, 255, 266, 251, 310, 264, 259, 259, 245, 259, 262, 275, 253, 245, 265, 246, 251, 259, 245, 279, 278, 231, 235, 256, 268, 223, 280, 261] payload_rows_per_dst=[5430, 5454, 5464, 5466, 5500, 5552, 5474, 5412] dup_rows_per_dst=[2686, 2662, 2730, 2634, 2770, 2736, 2828, 2738] combine_rows_per_src=[5430, 5454, 5464, 5466, 5500, 5552, 5474, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38188 dup_rows=21854 dup_rows_remote=19040 dedup_ratio=0.333466 router_tokens_per_expert=[261, 237, 280, 295, 232, 271, 251, 274, 268, 254, 265, 263, 229, 243, 272, 272, 260, 248, 240, 256, 249, 260, 247, 246, 267, 270, 260, 236, 260, 246, 283, 250, 265, 236, 262, 245, 231, 272, 306, 212, 265, 252, 260, 260, 246, 261, 238, 274, 279, 262, 242, 271, 245, 262, 251, 258, 249, 247, 261, 234, 249, 237, 282, 238, 232, 272, 266, 250, 268, 277, 256, 240, 248, 245, 275, 250, 290, 260, 252, 273, 250, 272, 258, 264, 238, 249, 266, 246, 261, 262, 253, 243, 245, 257, 259, 255, 233, 266, 242, 248, 265, 239, 258, 246, 241, 240, 259, 260, 266, 260, 241, 255, 255, 264, 253, 245, 243, 264, 275, 283, 264, 234, 231, 279, 232, 276, 258, 264] payload_rows_per_dst=[5446, 5452, 5478, 5424, 5494, 5526, 5386, 5476] dup_rows_per_dst=[2888, 2704, 2692, 2710, 2814, 2630, 2652, 2764] combine_rows_per_src=[5446, 5452, 5478, 5424, 5494, 5526, 5386, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38098 dup_rows=21720 dup_rows_remote=18838 dedup_ratio=0.331421 router_tokens_per_expert=[256, 275, 242, 261, 275, 246, 252, 241, 239, 270, 233, 269, 268, 229, 232, 248, 271, 263, 249, 245, 251, 277, 258, 235, 279, 270, 274, 267, 256, 226, 243, 271, 281, 236, 241, 281, 232, 252, 257, 261, 264, 238, 260, 269, 238, 276, 237, 252, 237, 264, 228, 266, 266, 282, 265, 266, 265, 279, 264, 264, 228, 271, 260, 267, 240, 289, 310, 282, 258, 270, 246, 274, 257, 267, 226, 281, 259, 264, 241, 247, 244, 248, 297, 267, 248, 241, 250, 252, 261, 238, 246, 251, 257, 252, 247, 248, 263, 252, 246, 247, 282, 255, 246, 236, 278, 272, 247, 278, 216, 264, 252, 255, 256, 246, 228, 244, 260, 248, 236, 242, 247, 258, 252, 260, 252, 276, 252, 246] payload_rows_per_dst=[5718, 5468, 5424, 5366, 5622, 5460, 5446, 5312] dup_rows_per_dst=[2882, 2812, 2762, 2450, 2800, 2588, 2732, 2694] combine_rows_per_src=[5718, 5468, 5424, 5366, 5622, 5460, 5446, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=228944 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157408848 ingress_bytes=213803008 link_bytes=213803008 token_bytes=314588752 weight_movement_bytes=56623104 total_nvlink_bytes=371211856 weight_to_token_ratio=0.179991 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048555225e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38412 dup_rows=21628 dup_rows_remote=18928 dedup_ratio=0.330017 router_tokens_per_expert=[239, 265, 260, 256, 253, 238, 264, 297, 279, 285, 246, 272, 255, 230, 286, 240, 234, 255, 266, 241, 248, 265, 248, 247, 235, 275, 256, 252, 237, 268, 233, 293, 235, 254, 244, 254, 251, 264, 276, 250, 252, 262, 245, 263, 281, 274, 256, 237, 265, 247, 260, 271, 256, 272, 258, 253, 262, 261, 254, 250, 239, 259, 267, 252, 264, 252, 265, 246, 251, 229, 253, 259, 252, 255, 233, 299, 278, 243, 265, 241, 259, 241, 257, 289, 256, 250, 234, 247, 242, 272, 240, 243, 234, 280, 262, 230, 282, 246, 285, 260, 257, 259, 271, 263, 270, 239, 256, 242, 244, 262, 263, 234, 252, 269, 251, 264, 256, 269, 254, 266, 245, 263, 263, 220, 248, 244, 248, 260] payload_rows_per_dst=[5508, 5506, 5496, 5500, 5466, 5458, 5514, 5460] dup_rows_per_dst=[2822, 2600, 2700, 2752, 2704, 2614, 2752, 2684] combine_rows_per_src=[5508, 5506, 5496, 5500, 5466, 5458, 5514, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=228936 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157015624 ingress_bytes=213409792 link_bytes=213409792 token_bytes=313802312 weight_movement_bytes=56623104 total_nvlink_bytes=370425416 weight_to_token_ratio=0.180442 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044787623e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228912 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156646960 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313065008 weight_movement_bytes=18874368 total_nvlink_bytes=331939376 weight_to_token_ratio=0.060289 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=227744 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156277152 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312326560 weight_movement_bytes=18874368 total_nvlink_bytes=331200928 weight_to_token_ratio=0.060432 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38182 dup_rows=21884 dup_rows_remote=19092 dedup_ratio=0.333923 router_tokens_per_expert=[259, 249, 243, 270, 241, 252, 252, 221, 244, 249, 264, 268, 256, 259, 279, 238, 266, 253, 274, 262, 242, 261, 268, 272, 263, 236, 265, 242, 265, 266, 239, 242, 255, 250, 259, 250, 269, 241, 268, 229, 250, 300, 279, 244, 227, 253, 273, 254, 216, 256, 246, 264, 253, 257, 284, 262, 249, 240, 236, 254, 241, 261, 260, 256, 242, 242, 265, 251, 277, 251, 266, 271, 255, 286, 239, 255, 262, 261, 251, 248, 263, 239, 247, 297, 246, 251, 256, 251, 258, 272, 283, 241, 277, 273, 232, 265, 266, 260, 255, 235, 268, 251, 274, 263, 259, 268, 246, 263, 243, 255, 268, 249, 261, 236, 252, 267, 236, 259, 257, 253, 266, 248, 231, 287, 269, 267, 242, 245] payload_rows_per_dst=[5364, 5426, 5492, 5478, 5428, 5438, 5556, 5470] dup_rows_per_dst=[2724, 2806, 2710, 2488, 2816, 2756, 2792, 2792] combine_rows_per_src=[5364, 5426, 5492, 5478, 5428, 5438, 5556, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38050 dup_rows=22040 dup_rows_remote=19420 dedup_ratio=0.336304 router_tokens_per_expert=[252, 245, 242, 250, 271, 262, 265, 246, 240, 258, 245, 247, 273, 280, 275, 248, 257, 251, 261, 255, 287, 268, 271, 271, 261, 254, 261, 250, 265, 266, 266, 245, 269, 271, 241, 276, 278, 262, 231, 251, 280, 285, 259, 249, 242, 281, 256, 256, 243, 254, 260, 256, 252, 257, 271, 227, 254, 262, 252, 260, 234, 253, 245, 237, 249, 270, 247, 249, 266, 268, 258, 272, 241, 245, 258, 251, 281, 254, 243, 271, 251, 257, 270, 259, 235, 272, 245, 259, 235, 262, 228, 257, 248, 263, 270, 222, 250, 250, 276, 259, 273, 283, 235, 248, 253, 254, 243, 256, 251, 235, 266, 240, 253, 276, 264, 264, 271, 243, 247, 218, 275, 249, 240, 256, 256, 260, 222, 254] payload_rows_per_dst=[5458, 5578, 5512, 5348, 5400, 5446, 5426, 5328] dup_rows_per_dst=[2740, 2800, 2862, 2686, 2846, 2620, 2718, 2768] combine_rows_per_src=[5458, 5578, 5512, 5348, 5400, 5446, 5426, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229360 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157564912 ingress_bytes=213958656 link_bytes=213958656 token_bytes=314900464 weight_movement_bytes=56623104 total_nvlink_bytes=371523568 weight_to_token_ratio=0.179813 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050046568e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43512 payload_rows_remote=37992 dup_rows=22024 dup_rows_remote=19330 dedup_ratio=0.336060 router_tokens_per_expert=[244, 265, 224, 246, 268, 266, 265, 238, 248, 270, 249, 238, 232, 264, 285, 253, 256, 240, 263, 267, 262, 235, 262, 270, 251, 255, 226, 253, 267, 248, 272, 245, 254, 259, 256, 234, 229, 252, 265, 218, 283, 265, 275, 247, 257, 260, 257, 261, 224, 251, 246, 246, 254, 274, 267, 261, 250, 271, 242, 260, 258, 275, 259, 269, 262, 270, 246, 269, 245, 242, 250, 256, 256, 267, 267, 268, 263, 251, 245, 272, 255, 269, 270, 275, 267, 237, 253, 253, 272, 267, 241, 267, 274, 231, 278, 255, 297, 256, 264, 241, 272, 254, 240, 273, 249, 220, 264, 235, 269, 240, 243, 266, 256, 249, 247, 242, 249, 260, 270, 230, 267, 273, 263, 245, 260, 272, 240, 263] payload_rows_per_dst=[5366, 5360, 5502, 5520, 5404, 5442, 5450, 5468] dup_rows_per_dst=[2744, 2784, 2642, 2694, 2854, 2886, 2716, 2704] combine_rows_per_src=[5366, 5360, 5502, 5520, 5404, 5442, 5450, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=229880 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156082680 ingress_bytes=174727168 link_bytes=174727168 token_bytes=311935480 weight_movement_bytes=18874368 total_nvlink_bytes=330809848 weight_to_token_ratio=0.060507 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674149753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229096 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622568 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313016040 weight_movement_bytes=18874368 total_nvlink_bytes=331890408 weight_to_token_ratio=0.060298 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38468 dup_rows=21650 dup_rows_remote=18978 dedup_ratio=0.330353 router_tokens_per_expert=[256, 274, 250, 285, 276, 257, 239, 240, 244, 245, 265, 266, 274, 248, 267, 249, 274, 245, 287, 254, 239, 244, 230, 241, 258, 242, 262, 265, 263, 234, 257, 250, 256, 268, 240, 262, 242, 266, 278, 254, 259, 261, 279, 255, 257, 264, 275, 246, 266, 286, 261, 236, 254, 253, 247, 271, 265, 255, 252, 235, 260, 287, 265, 232, 259, 254, 242, 254, 247, 235, 280, 270, 238, 262, 268, 237, 241, 259, 231, 266, 267, 245, 237, 283, 241, 259, 269, 260, 261, 246, 253, 253, 256, 276, 259, 255, 277, 262, 275, 231, 236, 206, 270, 249, 268, 250, 255, 282, 241, 268, 253, 242, 272, 250, 245, 249, 251, 270, 257, 247, 271, 242, 244, 256, 242, 239, 263, 275] payload_rows_per_dst=[5530, 5418, 5520, 5532, 5456, 5504, 5456, 5470] dup_rows_per_dst=[2740, 2672, 2804, 2718, 2630, 2736, 2674, 2676] combine_rows_per_src=[5530, 5418, 5520, 5532, 5456, 5504, 5456, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155615232 dispatch_meta_bytes=229288 combine_payload_bytes=155615232 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=155844520 ingress_bytes=231112704 link_bytes=231112704 token_bytes=311459752 weight_movement_bytes=75497472 total_nvlink_bytes=386957224 weight_to_token_ratio=0.242399 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.214408215e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38312 dup_rows=21626 dup_rows_remote=18896 dedup_ratio=0.329987 router_tokens_per_expert=[256, 266, 248, 256, 259, 269, 258, 281, 255, 245, 262, 259, 247, 248, 230, 245, 243, 276, 257, 241, 232, 233, 286, 251, 249, 236, 262, 250, 264, 251, 265, 240, 238, 277, 234, 305, 274, 273, 286, 270, 254, 244, 272, 254, 259, 255, 260, 239, 238, 276, 252, 237, 230, 259, 252, 234, 285, 271, 256, 249, 272, 274, 268, 241, 269, 221, 228, 256, 254, 280, 252, 253, 248, 273, 279, 257, 257, 288, 232, 272, 252, 253, 247, 259, 258, 249, 251, 281, 259, 278, 244, 252, 218, 266, 238, 239, 240, 259, 272, 266, 276, 246, 246, 244, 267, 259, 230, 281, 281, 232, 281, 284, 256, 274, 261, 258, 234, 264, 225, 255, 224, 255, 217, 253, 252, 291, 257, 257] payload_rows_per_dst=[5468, 5430, 5530, 5484, 5512, 5440, 5598, 5448] dup_rows_per_dst=[2700, 2642, 2858, 2704, 2726, 2648, 2730, 2618] combine_rows_per_src=[5468, 5430, 5530, 5484, 5512, 5440, 5598, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38274 dup_rows=21752 dup_rows_remote=19048 dedup_ratio=0.331909 router_tokens_per_expert=[254, 266, 297, 260, 258, 229, 264, 252, 251, 260, 237, 233, 247, 233, 277, 263, 254, 274, 266, 256, 246, 240, 233, 246, 262, 256, 255, 247, 273, 242, 299, 250, 273, 271, 245, 245, 247, 274, 222, 252, 277, 249, 251, 272, 258, 263, 251, 270, 238, 254, 235, 245, 254, 252, 249, 247, 265, 241, 264, 215, 278, 230, 240, 261, 239, 257, 249, 253, 276, 290, 252, 240, 247, 267, 217, 272, 270, 281, 229, 268, 259, 234, 289, 287, 263, 249, 256, 271, 249, 255, 256, 281, 254, 243, 263, 258, 243, 243, 224, 258, 256, 254, 260, 247, 256, 283, 272, 262, 278, 264, 246, 285, 267, 266, 262, 250, 278, 244, 272, 277, 233, 260, 240, 218, 265, 237, 265, 261] payload_rows_per_dst=[5444, 5472, 5510, 5360, 5510, 5526, 5496, 5466] dup_rows_per_dst=[2718, 2726, 2730, 2576, 2704, 2808, 2766, 2724] combine_rows_per_src=[5444, 5472, 5510, 5360, 5510, 5526, 5496, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38270 dup_rows=21846 dup_rows_remote=19142 dedup_ratio=0.333344 router_tokens_per_expert=[238, 236, 239, 250, 240, 270, 258, 235, 250, 272, 249, 250, 246, 269, 260, 252, 282, 240, 275, 227, 276, 244, 277, 286, 273, 214, 255, 263, 231, 256, 262, 261, 272, 247, 266, 252, 266, 282, 266, 271, 297, 263, 238, 255, 240, 254, 250, 266, 240, 276, 262, 229, 283, 251, 244, 227, 283, 264, 239, 236, 274, 219, 249, 264, 238, 241, 239, 272, 236, 242, 248, 254, 247, 249, 276, 249, 237, 276, 261, 238, 276, 238, 241, 263, 249, 262, 272, 259, 224, 266, 235, 259, 268, 236, 257, 281, 248, 232, 267, 261, 259, 243, 248, 278, 270, 258, 246, 267, 263, 259, 238, 234, 302, 286, 255, 254, 234, 283, 257, 238, 258, 248, 259, 286, 254, 298, 280, 255] payload_rows_per_dst=[5420, 5556, 5540, 5542, 5314, 5254, 5458, 5606] dup_rows_per_dst=[2704, 2618, 2830, 3056, 2692, 2400, 2684, 2862] combine_rows_per_src=[5420, 5556, 5540, 5542, 5314, 5254, 5458, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=229784 combine_payload_bytes=157564928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157794712 ingress_bytes=176439296 link_bytes=176439296 token_bytes=315359640 weight_movement_bytes=18874368 total_nvlink_bytes=334234008 weight_to_token_ratio=0.059850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690554521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43524 payload_rows_remote=38050 dup_rows=22012 dup_rows_remote=19186 dedup_ratio=0.335876 router_tokens_per_expert=[268, 261, 247, 258, 258, 281, 272, 265, 255, 254, 254, 279, 233, 241, 302, 230, 232, 280, 279, 279, 252, 265, 232, 241, 280, 250, 263, 275, 260, 242, 249, 238, 273, 254, 272, 260, 252, 259, 254, 253, 255, 247, 278, 254, 256, 267, 254, 262, 293, 235, 269, 276, 256, 250, 252, 271, 266, 259, 220, 252, 261, 255, 249, 252, 252, 273, 261, 274, 264, 266, 247, 243, 276, 252, 277, 235, 262, 257, 268, 239, 232, 242, 260, 282, 217, 264, 261, 246, 271, 232, 246, 270, 250, 256, 266, 244, 238, 248, 242, 258, 255, 271, 252, 268, 248, 260, 227, 239, 270, 259, 248, 250, 255, 240, 254, 239, 261, 258, 249, 250, 245, 250, 253, 236, 265, 241, 227, 286] payload_rows_per_dst=[5492, 5472, 5474, 5418, 5450, 5436, 5398, 5384] dup_rows_per_dst=[2824, 2762, 2826, 2814, 2842, 2642, 2668, 2634] combine_rows_per_src=[5492, 5472, 5474, 5418, 5450, 5436, 5398, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228832 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157154784 ingress_bytes=213549056 link_bytes=213549056 token_bytes=314080736 weight_movement_bytes=56623104 total_nvlink_bytes=370703840 weight_to_token_ratio=0.180282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046121982e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229288 combine_payload_bytes=156770304 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156999592 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769896 weight_movement_bytes=37748736 total_nvlink_bytes=351518632 weight_to_token_ratio=0.120307 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229648 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156983568 ingress_bytes=213377024 link_bytes=213377024 token_bytes=313737488 weight_movement_bytes=56623104 total_nvlink_bytes=370360592 weight_to_token_ratio=0.180479 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044473656e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38214 dup_rows=21930 dup_rows_remote=19258 dedup_ratio=0.334625 router_tokens_per_expert=[225, 254, 244, 257, 245, 260, 262, 241, 258, 291, 261, 243, 268, 255, 250, 255, 266, 260, 253, 248, 267, 250, 240, 230, 293, 255, 240, 265, 263, 276, 278, 243, 254, 278, 287, 249, 235, 277, 256, 253, 250, 260, 239, 266, 228, 262, 253, 254, 248, 246, 264, 238, 263, 237, 255, 286, 250, 254, 245, 258, 261, 260, 276, 274, 258, 284, 225, 284, 268, 276, 241, 263, 247, 260, 252, 272, 249, 251, 253, 229, 261, 270, 265, 269, 247, 242, 222, 245, 267, 255, 237, 218, 272, 264, 249, 249, 270, 263, 241, 257, 242, 248, 270, 280, 251, 264, 263, 248, 249, 249, 273, 259, 265, 275, 248, 284, 257, 248, 241, 246, 256, 260, 236, 260, 252, 289, 224, 244] payload_rows_per_dst=[5414, 5550, 5442, 5454, 5460, 5392, 5430, 5464] dup_rows_per_dst=[2724, 2704, 2760, 2776, 2764, 2672, 2824, 2706] combine_rows_per_src=[5414, 5550, 5442, 5454, 5460, 5392, 5430, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43522 payload_rows_remote=38086 dup_rows=22014 dup_rows_remote=19250 dedup_ratio=0.335907 router_tokens_per_expert=[251, 252, 243, 307, 276, 253, 258, 265, 255, 245, 271, 261, 261, 257, 278, 273, 256, 259, 264, 235, 257, 248, 246, 243, 261, 252, 264, 246, 241, 256, 252, 251, 249, 252, 239, 248, 260, 231, 250, 285, 246, 251, 253, 253, 273, 258, 244, 233, 249, 266, 247, 251, 249, 271, 212, 257, 281, 269, 244, 262, 245, 266, 261, 239, 234, 257, 242, 267, 244, 253, 227, 285, 257, 263, 239, 261, 267, 288, 251, 230, 261, 261, 267, 267, 270, 254, 260, 261, 236, 269, 249, 267, 251, 253, 278, 239, 260, 251, 240, 252, 265, 251, 261, 260, 267, 277, 253, 270, 261, 271, 237, 275, 274, 252, 224, 257, 288, 274, 214, 242, 240, 270, 238, 258, 263, 281, 257, 246] payload_rows_per_dst=[5472, 5240, 5174, 5422, 5560, 5544, 5674, 5436] dup_rows_per_dst=[2734, 2418, 2330, 2726, 2920, 2948, 3174, 2764] combine_rows_per_src=[5472, 5240, 5174, 5422, 5560, 5544, 5674, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=228944 combine_payload_bytes=155852800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156081744 ingress_bytes=193601536 link_bytes=193601536 token_bytes=311934544 weight_movement_bytes=37748736 total_nvlink_bytes=349683280 weight_to_token_ratio=0.121015 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854994660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38450 dup_rows=21662 dup_rows_remote=19022 dedup_ratio=0.330536 router_tokens_per_expert=[249, 214, 251, 228, 247, 250, 255, 272, 249, 260, 290, 242, 288, 266, 264, 279, 281, 263, 239, 242, 260, 261, 285, 255, 276, 239, 236, 274, 271, 252, 252, 253, 259, 245, 256, 272, 245, 248, 239, 269, 242, 258, 254, 262, 287, 290, 274, 256, 252, 264, 255, 219, 237, 254, 236, 274, 283, 258, 254, 243, 253, 255, 240, 255, 262, 263, 265, 223, 252, 282, 257, 249, 250, 266, 269, 247, 230, 246, 260, 264, 269, 260, 258, 281, 247, 247, 239, 255, 259, 269, 261, 273, 250, 293, 256, 263, 225, 249, 251, 255, 226, 258, 240, 241, 247, 245, 254, 244, 242, 252, 272, 249, 285, 215, 314, 267, 250, 258, 247, 255, 254, 273, 274, 244, 240, 258, 254, 234] payload_rows_per_dst=[5468, 5576, 5486, 5424, 5532, 5556, 5356, 5476] dup_rows_per_dst=[2740, 2702, 2826, 2640, 2638, 2804, 2544, 2768] combine_rows_per_src=[5468, 5576, 5486, 5424, 5532, 5556, 5356, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38326 dup_rows=21770 dup_rows_remote=19174 dedup_ratio=0.332184 router_tokens_per_expert=[245, 273, 271, 256, 243, 264, 265, 251, 259, 248, 249, 245, 278, 272, 265, 231, 251, 248, 269, 246, 246, 263, 276, 234, 244, 257, 248, 247, 248, 248, 236, 257, 239, 246, 239, 282, 261, 292, 245, 243, 246, 282, 257, 264, 275, 266, 279, 252, 272, 274, 255, 242, 245, 264, 260, 253, 278, 232, 233, 262, 270, 254, 273, 224, 242, 247, 251, 283, 267, 283, 236, 258, 255, 260, 251, 252, 243, 269, 263, 258, 254, 252, 266, 262, 257, 266, 264, 248, 244, 270, 266, 251, 257, 253, 261, 266, 262, 280, 263, 237, 246, 243, 252, 250, 260, 246, 255, 260, 245, 245, 256, 265, 285, 242, 224, 281, 246, 269, 259, 233, 265, 265, 229, 257, 264, 219, 262, 256] payload_rows_per_dst=[5478, 5440, 5514, 5406, 5526, 5462, 5430, 5510] dup_rows_per_dst=[2752, 2596, 2822, 2776, 2710, 2812, 2700, 2602] combine_rows_per_src=[5478, 5440, 5514, 5406, 5526, 5462, 5430, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229888 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156754432 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278976 weight_movement_bytes=56623104 total_nvlink_bytes=369902080 weight_to_token_ratio=0.180743 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229344 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156229600 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312229856 weight_movement_bytes=37748736 total_nvlink_bytes=349978592 weight_to_token_ratio=0.120900 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38260 dup_rows=21910 dup_rows_remote=19316 dedup_ratio=0.334320 router_tokens_per_expert=[265, 268, 248, 244, 270, 274, 259, 239, 258, 226, 244, 253, 249, 253, 285, 266, 247, 258, 236, 294, 258, 269, 264, 253, 258, 280, 282, 249, 282, 251, 248, 252, 266, 250, 280, 263, 272, 263, 220, 228, 243, 254, 257, 269, 280, 239, 243, 276, 268, 290, 243, 267, 254, 261, 245, 226, 249, 243, 252, 254, 268, 263, 251, 241, 237, 254, 250, 248, 246, 275, 257, 236, 258, 262, 251, 281, 255, 264, 256, 267, 234, 234, 242, 239, 270, 277, 272, 236, 248, 254, 241, 285, 274, 267, 221, 235, 236, 245, 251, 286, 224, 273, 275, 278, 267, 227, 250, 261, 263, 263, 257, 268, 251, 265, 245, 259, 249, 244, 268, 247, 256, 256, 241, 259, 242, 251, 276, 249] payload_rows_per_dst=[5398, 5550, 5504, 5454, 5476, 5398, 5366, 5480] dup_rows_per_dst=[2804, 2812, 2702, 2696, 2718, 2660, 2594, 2924] combine_rows_per_src=[5398, 5550, 5504, 5454, 5476, 5398, 5366, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38386 dup_rows=21680 dup_rows_remote=19006 dedup_ratio=0.330811 router_tokens_per_expert=[247, 210, 226, 232, 264, 256, 239, 259, 277, 263, 235, 261, 254, 266, 273, 272, 249, 244, 248, 265, 281, 273, 242, 271, 255, 248, 289, 260, 262, 249, 267, 270, 279, 255, 252, 243, 288, 254, 246, 250, 257, 263, 251, 264, 226, 275, 263, 261, 270, 273, 279, 269, 261, 267, 279, 234, 248, 263, 271, 272, 243, 245, 239, 264, 282, 280, 260, 257, 256, 236, 248, 266, 237, 250, 274, 250, 255, 267, 232, 222, 232, 232, 256, 262, 257, 253, 250, 248, 240, 245, 245, 258, 267, 246, 270, 256, 254, 278, 255, 261, 273, 258, 285, 264, 252, 235, 248, 255, 241, 255, 249, 225, 262, 248, 241, 242, 254, 274, 251, 265, 230, 257, 271, 255, 270, 265, 252, 243] payload_rows_per_dst=[5448, 5546, 5526, 5552, 5470, 5398, 5484, 5432] dup_rows_per_dst=[2620, 2800, 2728, 2802, 2674, 2636, 2692, 2728] combine_rows_per_src=[5448, 5546, 5526, 5552, 5470, 5398, 5484, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=229888 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157721088 ingress_bytes=195239936 link_bytes=195239936 token_bytes=315212288 weight_movement_bytes=37748736 total_nvlink_bytes=352961024 weight_to_token_ratio=0.119757 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870693003e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=230000 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157213296 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314196592 weight_movement_bytes=56623104 total_nvlink_bytes=370819696 weight_to_token_ratio=0.180216 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38230 dup_rows=21774 dup_rows_remote=18950 dedup_ratio=0.332245 router_tokens_per_expert=[256, 275, 260, 253, 276, 232, 220, 268, 263, 251, 258, 273, 256, 241, 271, 260, 283, 277, 229, 252, 241, 230, 242, 264, 265, 262, 251, 236, 285, 239, 279, 248, 235, 239, 283, 246, 233, 252, 241, 262, 265, 261, 262, 243, 252, 233, 245, 272, 249, 266, 243, 249, 279, 262, 265, 258, 260, 234, 293, 255, 279, 268, 252, 244, 243, 222, 290, 270, 232, 223, 272, 247, 270, 242, 274, 274, 251, 244, 257, 253, 272, 262, 232, 263, 250, 264, 248, 225, 256, 262, 262, 230, 287, 242, 262, 252, 254, 267, 257, 268, 254, 238, 238, 263, 264, 260, 271, 258, 275, 255, 243, 250, 250, 268, 275, 254, 266, 241, 265, 267, 265, 264, 245, 279, 261, 246, 259, 239] payload_rows_per_dst=[5532, 5536, 5442, 5486, 5508, 5460, 5378, 5420] dup_rows_per_dst=[2824, 2832, 2722, 2826, 2620, 2678, 2534, 2738] combine_rows_per_src=[5532, 5536, 5442, 5486, 5508, 5460, 5378, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38244 dup_rows=21912 dup_rows_remote=19228 dedup_ratio=0.334351 router_tokens_per_expert=[252, 234, 247, 275, 270, 280, 248, 254, 251, 273, 263, 254, 235, 272, 269, 281, 245, 254, 265, 246, 254, 250, 242, 254, 281, 245, 255, 237, 255, 260, 274, 273, 242, 256, 272, 210, 264, 261, 242, 260, 278, 256, 255, 263, 262, 225, 246, 240, 256, 263, 264, 258, 255, 276, 269, 261, 281, 260, 266, 247, 267, 232, 273, 234, 289, 241, 272, 244, 256, 250, 248, 232, 227, 258, 269, 263, 231, 243, 232, 239, 274, 263, 241, 284, 264, 243, 233, 240, 277, 248, 239, 266, 235, 234, 229, 252, 233, 282, 284, 253, 272, 247, 248, 242, 258, 248, 243, 248, 275, 250, 259, 253, 258, 265, 247, 270, 280, 276, 275, 254, 242, 249, 286, 275, 236, 271, 266, 265] payload_rows_per_dst=[5456, 5474, 5380, 5502, 5394, 5438, 5468, 5512] dup_rows_per_dst=[2860, 2706, 2684, 2822, 2594, 2606, 2722, 2918] combine_rows_per_src=[5456, 5474, 5380, 5502, 5394, 5438, 5468, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=230304 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156943264 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313656224 weight_movement_bytes=37748736 total_nvlink_bytes=351404960 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=229568 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157458624 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314687680 weight_movement_bytes=18874368 total_nvlink_bytes=333562048 weight_to_token_ratio=0.059978 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228720 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156818800 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313408880 weight_movement_bytes=37748736 total_nvlink_bytes=351157616 weight_to_token_ratio=0.120446 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229888 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156877312 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524736 weight_movement_bytes=37748736 total_nvlink_bytes=351273472 weight_to_token_ratio=0.120401 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38250 dup_rows=21892 dup_rows_remote=19208 dedup_ratio=0.334045 router_tokens_per_expert=[244, 266, 266, 253, 243, 255, 240, 253, 265, 273, 245, 241, 276, 253, 241, 239, 230, 264, 243, 276, 287, 273, 256, 257, 270, 267, 294, 255, 277, 248, 256, 258, 251, 261, 248, 276, 277, 228, 267, 248, 257, 266, 267, 264, 272, 255, 261, 245, 253, 265, 280, 242, 256, 255, 240, 251, 260, 256, 258, 258, 270, 247, 259, 256, 251, 276, 240, 263, 264, 270, 239, 248, 252, 261, 256, 262, 238, 251, 248, 248, 243, 274, 241, 268, 247, 261, 233, 235, 242, 254, 246, 250, 276, 241, 264, 264, 255, 240, 255, 254, 243, 235, 257, 252, 240, 256, 260, 242, 233, 254, 254, 254, 263, 282, 269, 245, 245, 223, 274, 287, 266, 263, 287, 248, 262, 243, 265, 243] payload_rows_per_dst=[5468, 5520, 5484, 5442, 5456, 5394, 5390, 5490] dup_rows_per_dst=[2638, 2902, 2802, 2770, 2678, 2684, 2578, 2840] combine_rows_per_src=[5468, 5520, 5484, 5442, 5456, 5394, 5390, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38328 dup_rows=21660 dup_rows_remote=18864 dedup_ratio=0.330505 router_tokens_per_expert=[257, 232, 271, 252, 253, 250, 292, 265, 253, 267, 265, 258, 249, 243, 270, 263, 271, 257, 251, 233, 274, 251, 275, 233, 254, 235, 263, 271, 238, 276, 265, 240, 272, 244, 257, 216, 238, 264, 246, 283, 249, 248, 235, 256, 269, 238, 244, 242, 261, 285, 262, 234, 251, 281, 270, 229, 244, 261, 283, 262, 268, 241, 224, 243, 256, 245, 238, 261, 260, 263, 237, 275, 287, 262, 272, 242, 277, 221, 314, 255, 264, 261, 299, 258, 264, 248, 249, 247, 254, 269, 240, 221, 254, 275, 253, 262, 250, 248, 256, 257, 237, 267, 246, 254, 276, 252, 257, 239, 243, 278, 260, 248, 266, 279, 246, 241, 250, 237, 247, 234, 247, 261, 268, 279, 241, 255, 287, 252] payload_rows_per_dst=[5492, 5476, 5384, 5468, 5532, 5518, 5458, 5548] dup_rows_per_dst=[2788, 2698, 2618, 2730, 2610, 2720, 2700, 2796] combine_rows_per_src=[5492, 5476, 5384, 5468, 5532, 5518, 5458, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38270 dup_rows=21752 dup_rows_remote=18946 dedup_ratio=0.331909 router_tokens_per_expert=[252, 286, 274, 253, 274, 279, 249, 240, 249, 280, 256, 266, 257, 262, 241, 230, 277, 279, 259, 233, 275, 264, 255, 266, 245, 243, 252, 260, 271, 265, 252, 230, 247, 255, 215, 263, 241, 231, 245, 279, 258, 220, 258, 262, 277, 244, 231, 271, 272, 257, 265, 254, 270, 272, 282, 250, 257, 249, 242, 261, 246, 262, 263, 258, 270, 251, 253, 264, 276, 271, 265, 238, 232, 229, 257, 216, 259, 248, 278, 279, 239, 243, 291, 266, 261, 231, 278, 290, 242, 256, 260, 255, 248, 274, 261, 269, 227, 237, 285, 264, 260, 263, 245, 255, 255, 236, 276, 260, 241, 226, 268, 253, 267, 248, 240, 251, 225, 259, 269, 256, 255, 256, 247, 258, 248, 248, 266, 243] payload_rows_per_dst=[5516, 5500, 5362, 5514, 5406, 5528, 5464, 5494] dup_rows_per_dst=[2780, 2752, 2632, 2806, 2766, 2800, 2638, 2578] combine_rows_per_src=[5516, 5500, 5362, 5514, 5406, 5528, 5464, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38452 dup_rows=21620 dup_rows_remote=19030 dedup_ratio=0.329895 router_tokens_per_expert=[273, 236, 229, 248, 244, 270, 270, 262, 265, 283, 255, 265, 243, 244, 253, 236, 250, 248, 267, 238, 245, 215, 246, 235, 286, 273, 256, 275, 222, 257, 251, 263, 239, 271, 270, 248, 254, 264, 252, 259, 264, 262, 248, 247, 239, 274, 272, 278, 246, 256, 260, 267, 249, 251, 276, 284, 247, 255, 263, 240, 264, 256, 262, 263, 280, 265, 255, 262, 252, 247, 265, 262, 252, 218, 270, 288, 259, 249, 224, 260, 232, 240, 284, 240, 261, 254, 295, 258, 253, 240, 261, 236, 257, 258, 241, 255, 255, 251, 284, 242, 268, 275, 247, 269, 271, 269, 244, 254, 254, 261, 290, 218, 244, 254, 265, 270, 278, 250, 231, 262, 266, 248, 233, 258, 258, 262, 241, 240] payload_rows_per_dst=[5418, 5464, 5492, 5620, 5496, 5462, 5472, 5492] dup_rows_per_dst=[2734, 2590, 2790, 2658, 2720, 2668, 2832, 2628] combine_rows_per_src=[5418, 5464, 5492, 5620, 5496, 5462, 5472, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229832 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901832 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573832 weight_movement_bytes=18874368 total_nvlink_bytes=332448200 weight_to_token_ratio=0.060191 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=228768 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220256 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314211744 weight_movement_bytes=18874368 total_nvlink_bytes=333086112 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=228864 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156982784 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313736704 weight_movement_bytes=37748736 total_nvlink_bytes=351485440 weight_to_token_ratio=0.120320 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38272 dup_rows=21860 dup_rows_remote=19220 dedup_ratio=0.333557 router_tokens_per_expert=[252, 258, 243, 262, 240, 279, 252, 275, 256, 266, 234, 245, 220, 285, 265, 245, 208, 269, 259, 240, 243, 251, 282, 247, 253, 238, 242, 277, 254, 229, 253, 240, 279, 258, 250, 265, 263, 234, 252, 262, 226, 252, 267, 228, 256, 268, 258, 271, 256, 253, 250, 267, 278, 251, 278, 270, 264, 228, 247, 291, 268, 270, 263, 245, 236, 244, 259, 272, 260, 265, 247, 264, 270, 285, 274, 262, 241, 252, 260, 232, 242, 259, 259, 292, 245, 259, 254, 270, 232, 247, 268, 270, 295, 265, 281, 252, 284, 271, 249, 252, 225, 282, 274, 251, 236, 245, 250, 257, 259, 220, 203, 264, 244, 263, 256, 277, 238, 239, 264, 261, 286, 229, 255, 264, 271, 240, 256, 260] payload_rows_per_dst=[5466, 5406, 5464, 5578, 5402, 5554, 5404, 5402] dup_rows_per_dst=[2688, 2564, 2714, 2780, 2844, 2826, 2640, 2804] combine_rows_per_src=[5466, 5406, 5464, 5578, 5402, 5554, 5404, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38412 dup_rows=21664 dup_rows_remote=18938 dedup_ratio=0.330566 router_tokens_per_expert=[237, 256, 270, 270, 271, 248, 252, 236, 242, 271, 252, 258, 240, 288, 247, 249, 272, 250, 271, 240, 264, 242, 251, 277, 244, 258, 271, 262, 252, 253, 272, 242, 266, 271, 259, 256, 273, 241, 256, 271, 266, 245, 262, 248, 262, 253, 244, 271, 252, 277, 254, 267, 264, 264, 291, 261, 276, 241, 267, 248, 253, 250, 212, 245, 255, 286, 272, 239, 260, 253, 266, 261, 237, 248, 253, 242, 265, 248, 257, 251, 264, 269, 249, 248, 240, 247, 263, 231, 256, 264, 288, 265, 238, 236, 253, 248, 237, 239, 260, 244, 257, 258, 246, 239, 264, 271, 247, 270, 268, 244, 257, 261, 250, 256, 248, 259, 254, 268, 223, 248, 265, 240, 251, 239, 279, 267, 262, 271] payload_rows_per_dst=[5500, 5452, 5492, 5524, 5460, 5460, 5458, 5526] dup_rows_per_dst=[2674, 2790, 2796, 2720, 2726, 2658, 2666, 2634] combine_rows_per_src=[5500, 5452, 5492, 5524, 5460, 5460, 5458, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157499392 dispatch_meta_bytes=229928 combine_payload_bytes=157499392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157729320 ingress_bytes=176373760 link_bytes=176373760 token_bytes=315228712 weight_movement_bytes=18874368 total_nvlink_bytes=334103080 weight_to_token_ratio=0.059875 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689926587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38314 dup_rows=21732 dup_rows_remote=18996 dedup_ratio=0.331604 router_tokens_per_expert=[248, 259, 262, 252, 247, 269, 249, 273, 247, 232, 271, 238, 255, 256, 251, 259, 273, 249, 253, 247, 262, 263, 255, 281, 238, 243, 277, 250, 236, 270, 242, 243, 234, 241, 261, 255, 259, 253, 266, 259, 230, 263, 260, 276, 250, 249, 259, 247, 247, 253, 250, 277, 224, 272, 270, 246, 256, 253, 247, 253, 259, 254, 257, 278, 253, 227, 267, 228, 237, 256, 262, 257, 244, 263, 253, 263, 268, 273, 274, 235, 269, 294, 268, 238, 227, 236, 290, 264, 270, 247, 267, 268, 244, 253, 276, 271, 274, 262, 272, 269, 223, 245, 257, 276, 253, 276, 259, 264, 243, 262, 284, 288, 226, 257, 262, 240, 257, 248, 278, 274, 219, 243, 268, 231, 256, 259, 242, 251] payload_rows_per_dst=[5490, 5482, 5466, 5490, 5442, 5588, 5476, 5370] dup_rows_per_dst=[2736, 2718, 2580, 2690, 2678, 3014, 2664, 2652] combine_rows_per_src=[5490, 5482, 5466, 5490, 5442, 5588, 5476, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38326 dup_rows=21846 dup_rows_remote=19138 dedup_ratio=0.333344 router_tokens_per_expert=[253, 258, 251, 240, 250, 253, 245, 278, 275, 257, 249, 274, 246, 253, 256, 269, 291, 254, 246, 262, 254, 236, 261, 238, 271, 233, 226, 266, 226, 247, 239, 253, 255, 264, 248, 252, 248, 263, 271, 247, 260, 257, 258, 231, 249, 254, 237, 242, 249, 256, 271, 266, 256, 270, 279, 271, 265, 263, 257, 255, 251, 234, 253, 261, 267, 255, 260, 233, 231, 268, 229, 268, 256, 250, 259, 263, 260, 249, 269, 246, 241, 280, 263, 239, 254, 219, 246, 240, 236, 251, 264, 258, 247, 254, 257, 242, 267, 285, 269, 241, 276, 264, 264, 251, 273, 233, 287, 243, 275, 265, 253, 258, 274, 254, 279, 262, 281, 281, 245, 257, 260, 253, 268, 266, 239, 250, 262, 276] payload_rows_per_dst=[5460, 5388, 5364, 5522, 5390, 5458, 5488, 5620] dup_rows_per_dst=[2754, 2618, 2708, 2792, 2736, 2524, 2920, 2794] combine_rows_per_src=[5460, 5388, 5364, 5522, 5390, 5458, 5488, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229968 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156992080 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313754192 weight_movement_bytes=18874368 total_nvlink_bytes=332628560 weight_to_token_ratio=0.060157 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229400 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157564952 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900504 weight_movement_bytes=18874368 total_nvlink_bytes=333774872 weight_to_token_ratio=0.059938 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38340 dup_rows=21658 dup_rows_remote=18840 dedup_ratio=0.330475 router_tokens_per_expert=[290, 260, 258, 230, 235, 237, 243, 266, 252, 246, 248, 246, 259, 249, 281, 245, 241, 259, 247, 268, 252, 251, 272, 263, 237, 257, 248, 240, 261, 257, 253, 248, 236, 248, 288, 262, 254, 239, 257, 258, 273, 264, 236, 272, 250, 257, 249, 266, 243, 263, 259, 260, 251, 240, 287, 222, 245, 240, 269, 265, 252, 277, 254, 245, 261, 254, 270, 278, 253, 241, 253, 261, 242, 235, 237, 257, 265, 266, 273, 246, 264, 246, 257, 274, 292, 252, 236, 266, 265, 243, 259, 291, 233, 271, 268, 261, 246, 272, 240, 272, 251, 269, 277, 263, 274, 263, 248, 249, 284, 266, 265, 261, 263, 264, 256, 261, 227, 263, 238, 247, 242, 244, 259, 255, 258, 238, 258, 245] payload_rows_per_dst=[5450, 5478, 5512, 5450, 5492, 5538, 5532, 5426] dup_rows_per_dst=[2640, 2630, 2706, 2694, 2692, 2818, 2868, 2610] combine_rows_per_src=[5450, 5478, 5512, 5450, 5492, 5538, 5532, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38294 dup_rows=21798 dup_rows_remote=19090 dedup_ratio=0.332611 router_tokens_per_expert=[251, 267, 253, 253, 255, 265, 239, 268, 250, 271, 259, 252, 245, 227, 266, 242, 274, 275, 280, 274, 252, 267, 251, 259, 244, 250, 256, 284, 263, 262, 277, 254, 270, 257, 272, 271, 240, 266, 253, 234, 249, 269, 259, 262, 265, 239, 250, 261, 244, 232, 271, 265, 273, 233, 242, 265, 250, 228, 228, 243, 231, 226, 264, 251, 270, 234, 254, 259, 245, 253, 252, 263, 246, 256, 251, 278, 268, 256, 248, 271, 250, 264, 254, 261, 272, 233, 253, 236, 244, 275, 247, 285, 237, 255, 259, 258, 246, 230, 269, 250, 258, 271, 261, 278, 261, 269, 261, 262, 281, 239, 270, 251, 256, 268, 239, 268, 273, 244, 253, 263, 258, 283, 229, 256, 227, 239, 252, 268] payload_rows_per_dst=[5472, 5198, 5600, 5530, 5460, 5486, 5548, 5444] dup_rows_per_dst=[2654, 2444, 3138, 2660, 2748, 2680, 2766, 2708] combine_rows_per_src=[5472, 5198, 5600, 5530, 5460, 5486, 5548, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229240 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157163384 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097528 weight_movement_bytes=37748736 total_nvlink_bytes=351846264 weight_to_token_ratio=0.120182 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229856 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157213152 ingress_bytes=175857664 link_bytes=175857664 token_bytes=314196448 weight_movement_bytes=18874368 total_nvlink_bytes=333070816 weight_to_token_ratio=0.060072 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684981609e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38384 dup_rows=21676 dup_rows_remote=18926 dedup_ratio=0.330750 router_tokens_per_expert=[280, 222, 251, 252, 256, 243, 280, 253, 237, 253, 251, 269, 237, 255, 243, 244, 244, 263, 263, 254, 253, 255, 272, 291, 260, 256, 274, 287, 245, 257, 235, 288, 228, 245, 254, 268, 254, 241, 225, 221, 277, 253, 280, 262, 232, 255, 240, 256, 265, 240, 246, 251, 251, 286, 273, 248, 264, 263, 260, 262, 248, 242, 257, 257, 266, 277, 256, 244, 256, 246, 240, 246, 260, 245, 261, 266, 261, 286, 226, 255, 240, 246, 259, 287, 260, 262, 245, 257, 276, 262, 248, 244, 275, 249, 234, 265, 257, 243, 240, 265, 265, 244, 268, 263, 273, 232, 261, 256, 243, 275, 251, 275, 265, 262, 246, 254, 284, 290, 228, 248, 274, 234, 265, 258, 236, 235, 283, 268] payload_rows_per_dst=[5480, 5536, 5416, 5476, 5444, 5522, 5506, 5480] dup_rows_per_dst=[2572, 2858, 2566, 2750, 2738, 2696, 2716, 2780] combine_rows_per_src=[5480, 5536, 5416, 5476, 5444, 5522, 5506, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38250 dup_rows=21790 dup_rows_remote=18904 dedup_ratio=0.332489 router_tokens_per_expert=[261, 252, 237, 263, 248, 276, 276, 235, 278, 236, 235, 246, 263, 258, 275, 243, 277, 274, 246, 252, 266, 274, 256, 253, 252, 262, 267, 240, 267, 280, 268, 257, 268, 260, 238, 239, 246, 243, 238, 272, 252, 255, 233, 260, 263, 275, 286, 260, 260, 279, 270, 245, 258, 270, 250, 269, 275, 249, 272, 242, 282, 261, 245, 243, 251, 249, 221, 251, 253, 251, 209, 229, 258, 248, 251, 256, 262, 250, 266, 248, 248, 251, 260, 257, 272, 275, 244, 256, 283, 258, 280, 272, 226, 224, 268, 250, 238, 263, 253, 235, 257, 252, 268, 254, 263, 223, 281, 231, 251, 270, 282, 269, 266, 253, 261, 264, 264, 255, 247, 275, 222, 268, 247, 242, 263, 237, 258, 248] payload_rows_per_dst=[5504, 5496, 5550, 5456, 5320, 5490, 5482, 5448] dup_rows_per_dst=[2660, 2886, 2626, 2884, 2586, 2758, 2698, 2692] combine_rows_per_src=[5504, 5496, 5550, 5456, 5320, 5490, 5482, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=228720 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157269360 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310000 weight_movement_bytes=37748736 total_nvlink_bytes=352058736 weight_to_token_ratio=0.120100 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229536 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157081760 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933984 weight_movement_bytes=37748736 total_nvlink_bytes=351682720 weight_to_token_ratio=0.120244 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229240 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157450104 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314670968 weight_movement_bytes=18874368 total_nvlink_bytes=333545336 weight_to_token_ratio=0.059981 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38304 dup_rows=21788 dup_rows_remote=19036 dedup_ratio=0.332458 router_tokens_per_expert=[281, 264, 265, 241, 244, 254, 249, 249, 239, 245, 277, 256, 245, 241, 248, 257, 243, 241, 282, 262, 253, 246, 270, 280, 263, 268, 263, 267, 261, 264, 215, 238, 262, 275, 274, 253, 265, 269, 276, 251, 240, 268, 275, 261, 261, 243, 251, 251, 245, 251, 250, 272, 262, 270, 268, 254, 272, 269, 239, 265, 259, 258, 267, 254, 252, 264, 260, 278, 243, 278, 239, 269, 262, 215, 286, 224, 251, 268, 235, 253, 262, 228, 252, 250, 296, 278, 262, 246, 226, 258, 257, 264, 289, 227, 265, 223, 226, 230, 258, 265, 247, 260, 255, 249, 267, 225, 243, 277, 257, 270, 256, 251, 254, 267, 247, 239, 254, 240, 262, 261, 260, 254, 251, 256, 242, 247, 264, 273] payload_rows_per_dst=[5460, 5528, 5480, 5548, 5456, 5464, 5444, 5368] dup_rows_per_dst=[2650, 2704, 2746, 2762, 2698, 2702, 2752, 2774] combine_rows_per_src=[5460, 5528, 5480, 5548, 5456, 5464, 5444, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38096 dup_rows=21950 dup_rows_remote=19198 dedup_ratio=0.334930 router_tokens_per_expert=[228, 258, 258, 257, 268, 252, 251, 238, 266, 230, 262, 230, 247, 251, 252, 272, 231, 259, 243, 261, 239, 232, 256, 225, 259, 277, 245, 229, 241, 279, 245, 245, 262, 259, 285, 252, 265, 260, 283, 257, 269, 253, 254, 245, 242, 230, 258, 255, 234, 234, 258, 258, 277, 260, 267, 278, 271, 284, 255, 264, 287, 257, 266, 278, 276, 254, 248, 267, 257, 238, 259, 250, 264, 240, 265, 273, 255, 259, 240, 276, 262, 265, 260, 251, 257, 259, 263, 288, 263, 287, 253, 249, 272, 274, 243, 248, 258, 267, 263, 242, 265, 271, 231, 262, 228, 225, 245, 250, 279, 300, 223, 288, 280, 264, 221, 247, 247, 260, 217, 272, 238, 235, 263, 253, 269, 247, 253, 247] payload_rows_per_dst=[5436, 5376, 5450, 5540, 5490, 5490, 5392, 5412] dup_rows_per_dst=[2604, 2556, 2808, 2916, 2752, 2898, 2802, 2614] combine_rows_per_src=[5436, 5376, 5450, 5540, 5490, 5490, 5392, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228616 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156900616 ingress_bytes=232169472 link_bytes=232169472 token_bytes=313572616 weight_movement_bytes=75497472 total_nvlink_bytes=389070088 weight_to_token_ratio=0.240766 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224533646e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38296 dup_rows=21880 dup_rows_remote=19326 dedup_ratio=0.333862 router_tokens_per_expert=[237, 258, 256, 253, 266, 301, 262, 243, 249, 262, 248, 263, 230, 268, 267, 267, 264, 280, 282, 253, 241, 231, 263, 246, 251, 260, 253, 265, 303, 251, 283, 268, 261, 262, 252, 237, 256, 239, 245, 239, 227, 250, 254, 250, 245, 234, 246, 260, 260, 268, 267, 262, 244, 268, 249, 270, 246, 263, 224, 250, 269, 279, 250, 250, 246, 249, 244, 275, 272, 236, 295, 276, 246, 255, 259, 281, 264, 253, 255, 274, 267, 247, 264, 256, 266, 264, 238, 214, 261, 243, 270, 297, 261, 253, 257, 249, 283, 237, 273, 238, 244, 245, 247, 260, 218, 253, 252, 254, 240, 265, 256, 221, 242, 248, 241, 257, 265, 251, 264, 261, 230, 238, 250, 266, 280, 264, 268, 270] payload_rows_per_dst=[5464, 5558, 5360, 5432, 5536, 5414, 5436, 5456] dup_rows_per_dst=[2796, 2830, 2554, 2806, 2824, 2800, 2536, 2734] combine_rows_per_src=[5464, 5558, 5360, 5432, 5536, 5414, 5436, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229360 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122544 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015728 weight_movement_bytes=18874368 total_nvlink_bytes=332890096 weight_to_token_ratio=0.060106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38072 dup_rows=21926 dup_rows_remote=19092 dedup_ratio=0.334564 router_tokens_per_expert=[247, 251, 252, 258, 251, 267, 261, 271, 261, 242, 291, 271, 263, 273, 256, 246, 251, 242, 242, 225, 240, 279, 246, 258, 286, 252, 250, 229, 245, 263, 239, 269, 246, 238, 256, 269, 261, 247, 257, 257, 253, 238, 237, 264, 272, 261, 267, 235, 259, 262, 250, 240, 236, 252, 271, 273, 258, 245, 246, 274, 261, 276, 275, 252, 255, 279, 261, 275, 276, 245, 252, 232, 242, 291, 294, 264, 257, 235, 255, 261, 272, 268, 277, 260, 254, 277, 286, 255, 218, 251, 256, 259, 279, 252, 253, 237, 251, 266, 259, 277, 251, 247, 232, 239, 241, 243, 234, 263, 270, 236, 292, 253, 274, 235, 254, 231, 220, 259, 250, 247, 277, 243, 240, 273, 260, 269, 231, 258] payload_rows_per_dst=[5538, 5424, 5476, 5450, 5488, 5460, 5412, 5362] dup_rows_per_dst=[2834, 2560, 2716, 2810, 2782, 2848, 2696, 2680] combine_rows_per_src=[5538, 5424, 5476, 5450, 5488, 5460, 5412, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156041216 dispatch_meta_bytes=229176 combine_payload_bytes=156041216 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156270392 ingress_bytes=231538688 link_bytes=231538688 token_bytes=312311608 weight_movement_bytes=75497472 total_nvlink_bytes=387809080 weight_to_token_ratio=0.241738 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.218489784e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38102 dup_rows=21918 dup_rows_remote=18926 dedup_ratio=0.334442 router_tokens_per_expert=[249, 249, 256, 261, 249, 262, 258, 269, 248, 270, 250, 285, 279, 238, 254, 259, 235, 268, 269, 256, 268, 249, 247, 238, 271, 264, 270, 250, 255, 238, 270, 273, 251, 242, 262, 278, 303, 253, 262, 258, 275, 278, 242, 252, 242, 254, 256, 246, 256, 228, 240, 266, 254, 261, 256, 243, 256, 215, 230, 249, 244, 264, 245, 263, 283, 260, 254, 234, 250, 248, 232, 260, 279, 270, 254, 255, 247, 242, 261, 236, 239, 243, 247, 254, 270, 252, 230, 249, 233, 236, 267, 255, 238, 267, 250, 280, 265, 253, 255, 263, 266, 264, 258, 273, 263, 280, 229, 277, 273, 273, 259, 241, 277, 247, 263, 251, 261, 282, 253, 250, 263, 239, 255, 238, 251, 254, 257, 279] payload_rows_per_dst=[5440, 5472, 5484, 5360, 5472, 5358, 5516, 5516] dup_rows_per_dst=[2832, 2654, 2672, 2580, 2596, 2720, 2872, 2992] combine_rows_per_src=[5440, 5472, 5484, 5360, 5472, 5358, 5516, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43972 payload_rows_remote=38428 dup_rows=21564 dup_rows_remote=18938 dedup_ratio=0.329041 router_tokens_per_expert=[264, 237, 273, 269, 231, 262, 247, 256, 278, 235, 260, 280, 241, 235, 274, 254, 228, 271, 263, 264, 236, 247, 248, 270, 256, 250, 252, 278, 242, 263, 259, 258, 258, 275, 241, 253, 266, 275, 255, 253, 253, 233, 230, 254, 250, 235, 242, 258, 269, 250, 251, 254, 254, 220, 270, 249, 264, 241, 264, 243, 261, 260, 232, 268, 247, 285, 228, 264, 257, 249, 289, 252, 277, 273, 250, 280, 261, 233, 264, 241, 237, 251, 291, 232, 247, 253, 309, 252, 245, 221, 261, 272, 264, 225, 255, 270, 268, 250, 258, 250, 261, 261, 232, 230, 264, 278, 255, 272, 267, 243, 230, 241, 252, 264, 286, 257, 241, 277, 251, 258, 255, 267, 270, 253, 261, 260, 281, 278] payload_rows_per_dst=[5452, 5560, 5454, 5504, 5526, 5544, 5368, 5564] dup_rows_per_dst=[2740, 2610, 2608, 2596, 2774, 2626, 2752, 2858] combine_rows_per_src=[5452, 5560, 5454, 5504, 5526, 5544, 5368, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38238 dup_rows=21840 dup_rows_remote=19078 dedup_ratio=0.333252 router_tokens_per_expert=[262, 264, 236, 240, 257, 254, 268, 229, 256, 274, 236, 223, 247, 244, 258, 242, 251, 275, 257, 249, 266, 235, 253, 243, 261, 270, 287, 263, 228, 270, 246, 251, 275, 298, 254, 282, 236, 231, 244, 247, 245, 246, 250, 271, 250, 259, 244, 277, 248, 265, 253, 245, 253, 259, 259, 277, 268, 230, 232, 256, 259, 278, 272, 256, 258, 260, 238, 231, 265, 274, 259, 256, 267, 284, 255, 270, 291, 256, 224, 259, 266, 253, 284, 244, 241, 259, 265, 266, 293, 259, 262, 231, 259, 285, 260, 227, 258, 281, 253, 212, 268, 268, 273, 277, 220, 256, 257, 282, 249, 263, 266, 236, 265, 228, 244, 258, 236, 257, 259, 266, 270, 250, 241, 226, 241, 291, 275, 227] payload_rows_per_dst=[5392, 5422, 5514, 5458, 5508, 5512, 5456, 5434] dup_rows_per_dst=[2588, 2788, 2704, 2762, 2786, 2796, 2782, 2634] combine_rows_per_src=[5392, 5422, 5514, 5458, 5508, 5512, 5456, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=230488 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157090904 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313951320 weight_movement_bytes=18874368 total_nvlink_bytes=332825688 weight_to_token_ratio=0.060119 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=228656 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156171568 ingress_bytes=174817280 link_bytes=174817280 token_bytes=312114480 weight_movement_bytes=18874368 total_nvlink_bytes=330988848 weight_to_token_ratio=0.060473 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675013161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38404 dup_rows=21758 dup_rows_remote=19138 dedup_ratio=0.332001 router_tokens_per_expert=[273, 239, 262, 248, 260, 245, 256, 248, 244, 278, 260, 220, 260, 218, 260, 247, 271, 258, 261, 256, 233, 262, 251, 228, 235, 255, 242, 261, 273, 249, 227, 235, 257, 243, 228, 254, 258, 245, 263, 230, 247, 257, 285, 274, 255, 263, 247, 254, 275, 227, 252, 263, 236, 249, 277, 264, 249, 282, 264, 233, 241, 276, 269, 250, 260, 255, 261, 290, 268, 248, 273, 268, 255, 246, 289, 259, 237, 274, 254, 269, 253, 234, 262, 248, 272, 242, 288, 254, 235, 235, 281, 258, 244, 245, 254, 276, 256, 276, 257, 248, 234, 256, 271, 267, 271, 262, 261, 282, 245, 257, 254, 247, 274, 265, 257, 241, 252, 288, 255, 230, 284, 279, 226, 265, 252, 259, 262, 266] payload_rows_per_dst=[5384, 5374, 5474, 5490, 5598, 5496, 5454, 5508] dup_rows_per_dst=[2652, 2620, 2646, 2724, 2814, 2666, 2834, 2802] combine_rows_per_src=[5384, 5374, 5474, 5490, 5598, 5496, 5454, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=228112 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156293904 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312359696 weight_movement_bytes=18874368 total_nvlink_bytes=331234064 weight_to_token_ratio=0.060425 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229464 combine_payload_bytes=157401088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157630552 ingress_bytes=176275456 link_bytes=176275456 token_bytes=315031640 weight_movement_bytes=18874368 total_nvlink_bytes=333906008 weight_to_token_ratio=0.059913 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688984686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=229264 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156852112 ingress_bytes=175497216 link_bytes=175497216 token_bytes=313474960 weight_movement_bytes=18874368 total_nvlink_bytes=332349328 weight_to_token_ratio=0.060210 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681527974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38364 dup_rows=21654 dup_rows_remote=18918 dedup_ratio=0.330414 router_tokens_per_expert=[244, 249, 256, 228, 266, 251, 284, 267, 242, 246, 242, 267, 242, 263, 280, 251, 233, 288, 267, 245, 279, 280, 263, 247, 270, 241, 249, 264, 246, 278, 248, 233, 236, 279, 225, 282, 272, 263, 240, 260, 254, 251, 245, 243, 271, 259, 241, 264, 278, 267, 251, 264, 237, 267, 251, 246, 235, 240, 260, 228, 230, 241, 283, 271, 248, 250, 257, 246, 251, 272, 244, 265, 241, 250, 260, 257, 252, 256, 269, 271, 265, 239, 271, 247, 253, 252, 256, 249, 272, 249, 246, 263, 237, 240, 272, 246, 273, 247, 280, 283, 246, 226, 239, 244, 255, 269, 260, 244, 266, 249, 251, 295, 283, 269, 282, 270, 242, 256, 238, 259, 274, 237, 257, 262, 271, 250, 252, 250] payload_rows_per_dst=[5486, 5470, 5504, 5452, 5534, 5410, 5518, 5508] dup_rows_per_dst=[2670, 2792, 2666, 2646, 2644, 2704, 2736, 2796] combine_rows_per_src=[5486, 5470, 5504, 5452, 5534, 5410, 5518, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38380 dup_rows=21706 dup_rows_remote=18962 dedup_ratio=0.331207 router_tokens_per_expert=[264, 268, 262, 261, 245, 263, 266, 273, 250, 279, 249, 236, 245, 280, 241, 261, 274, 266, 268, 248, 243, 260, 251, 258, 255, 250, 268, 257, 249, 248, 225, 256, 260, 253, 240, 258, 242, 269, 279, 267, 243, 276, 240, 265, 262, 251, 268, 259, 279, 275, 243, 253, 279, 235, 259, 253, 271, 260, 257, 233, 239, 267, 252, 241, 269, 253, 260, 255, 235, 254, 274, 278, 272, 260, 236, 226, 250, 257, 268, 250, 270, 232, 249, 232, 254, 263, 238, 262, 248, 265, 257, 240, 253, 256, 233, 252, 267, 254, 268, 234, 273, 285, 220, 235, 259, 275, 256, 249, 259, 269, 248, 260, 275, 228, 252, 258, 268, 266, 255, 246, 240, 243, 260, 274, 246, 269, 262, 267] payload_rows_per_dst=[5566, 5510, 5510, 5412, 5450, 5354, 5498, 5530] dup_rows_per_dst=[2720, 2642, 2754, 2780, 2744, 2654, 2724, 2688] combine_rows_per_src=[5566, 5510, 5510, 5412, 5450, 5354, 5498, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=230168 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157532952 ingress_bytes=195051520 link_bytes=195051520 token_bytes=314835736 weight_movement_bytes=37748736 total_nvlink_bytes=352584472 weight_to_token_ratio=0.119900 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868887693e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229128 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157368072 ingress_bytes=194887680 link_bytes=194887680 token_bytes=314507016 weight_movement_bytes=37748736 total_nvlink_bytes=352255752 weight_to_token_ratio=0.120025 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867317859e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38328 dup_rows=21790 dup_rows_remote=19056 dedup_ratio=0.332489 router_tokens_per_expert=[241, 262, 263, 240, 266, 272, 253, 268, 264, 271, 235, 250, 273, 261, 270, 238, 249, 275, 252, 253, 279, 254, 273, 257, 257, 259, 252, 274, 256, 260, 261, 257, 251, 267, 275, 263, 257, 256, 274, 255, 251, 256, 226, 234, 253, 244, 254, 260, 266, 245, 255, 235, 262, 268, 238, 229, 287, 265, 281, 250, 249, 265, 267, 247, 233, 227, 262, 263, 242, 257, 260, 269, 257, 244, 242, 249, 269, 247, 256, 266, 262, 254, 265, 269, 251, 289, 226, 235, 264, 262, 237, 273, 263, 238, 272, 245, 248, 262, 244, 254, 234, 255, 223, 234, 241, 248, 250, 260, 261, 275, 240, 258, 267, 259, 234, 262, 297, 274, 258, 262, 268, 223, 266, 252, 241, 261, 261, 268] payload_rows_per_dst=[5456, 5490, 5418, 5484, 5486, 5526, 5338, 5548] dup_rows_per_dst=[2798, 2846, 2734, 2734, 2600, 2684, 2636, 2758] combine_rows_per_src=[5456, 5490, 5418, 5484, 5486, 5526, 5338, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229368 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157433848 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638328 weight_movement_bytes=18874368 total_nvlink_bytes=333512696 weight_to_token_ratio=0.059988 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43514 payload_rows_remote=37968 dup_rows=22022 dup_rows_remote=19176 dedup_ratio=0.336029 router_tokens_per_expert=[238, 229, 265, 252, 269, 262, 300, 232, 254, 274, 264, 270, 253, 239, 255, 242, 259, 239, 250, 258, 233, 281, 264, 266, 245, 256, 264, 258, 251, 234, 277, 254, 229, 259, 244, 242, 237, 245, 260, 266, 260, 251, 223, 234, 274, 261, 261, 246, 285, 242, 257, 241, 265, 257, 271, 256, 271, 248, 247, 238, 236, 266, 268, 234, 237, 241, 243, 267, 239, 256, 250, 263, 262, 263, 266, 215, 239, 252, 265, 240, 241, 254, 242, 250, 256, 280, 261, 251, 266, 222, 276, 280, 266, 259, 275, 253, 246, 277, 229, 252, 280, 289, 272, 255, 273, 254, 266, 253, 259, 282, 236, 253, 253, 255, 235, 249, 249, 270, 260, 310, 260, 263, 259, 280, 256, 253, 283, 266] payload_rows_per_dst=[5546, 5446, 5422, 5366, 5296, 5474, 5494, 5470] dup_rows_per_dst=[2846, 2820, 2510, 2566, 2746, 2744, 2858, 2932] combine_rows_per_src=[5546, 5446, 5422, 5366, 5296, 5474, 5494, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38310 dup_rows=21678 dup_rows_remote=18934 dedup_ratio=0.330780 router_tokens_per_expert=[261, 246, 231, 241, 254, 242, 268, 236, 284, 270, 270, 265, 253, 241, 265, 254, 257, 255, 251, 319, 257, 223, 273, 246, 242, 300, 287, 272, 244, 247, 229, 274, 268, 229, 226, 247, 267, 273, 245, 228, 250, 252, 230, 271, 253, 230, 268, 278, 257, 242, 247, 268, 265, 225, 275, 273, 244, 257, 260, 249, 261, 255, 241, 229, 219, 257, 262, 251, 246, 283, 251, 273, 274, 262, 255, 241, 261, 249, 234, 272, 247, 268, 259, 260, 275, 259, 273, 259, 265, 248, 236, 255, 261, 232, 279, 249, 246, 241, 272, 256, 258, 272, 226, 289, 233, 292, 254, 271, 256, 276, 248, 245, 237, 244, 282, 264, 232, 271, 251, 252, 254, 242, 271, 233, 264, 276, 257, 268] payload_rows_per_dst=[5452, 5526, 5398, 5390, 5492, 5588, 5464, 5548] dup_rows_per_dst=[2710, 2826, 2632, 2668, 2688, 2670, 2740, 2744] combine_rows_per_src=[5452, 5526, 5398, 5390, 5492, 5588, 5464, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43910 payload_rows_remote=38404 dup_rows=21626 dup_rows_remote=18978 dedup_ratio=0.329987 router_tokens_per_expert=[245, 238, 277, 261, 240, 249, 262, 252, 248, 279, 253, 265, 242, 245, 249, 277, 291, 257, 274, 246, 260, 223, 250, 275, 223, 249, 254, 263, 283, 248, 245, 271, 258, 249, 284, 242, 243, 266, 256, 262, 261, 270, 275, 240, 283, 269, 259, 238, 249, 230, 230, 235, 256, 257, 214, 273, 249, 254, 239, 255, 257, 234, 261, 278, 257, 272, 269, 257, 259, 260, 269, 256, 271, 268, 256, 235, 257, 231, 276, 246, 255, 269, 254, 272, 243, 249, 229, 265, 291, 256, 254, 255, 263, 252, 225, 245, 247, 262, 282, 276, 234, 230, 264, 252, 271, 261, 267, 249, 279, 272, 267, 264, 232, 230, 258, 236, 241, 257, 253, 280, 256, 277, 243, 289, 256, 245, 248, 254] payload_rows_per_dst=[5480, 5494, 5516, 5442, 5466, 5506, 5616, 5390] dup_rows_per_dst=[2684, 2730, 2794, 2500, 2812, 2648, 2738, 2720] combine_rows_per_src=[5480, 5494, 5516, 5442, 5466, 5506, 5616, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38288 dup_rows=21658 dup_rows_remote=18814 dedup_ratio=0.330475 router_tokens_per_expert=[231, 259, 258, 283, 248, 237, 269, 248, 252, 268, 231, 242, 243, 232, 281, 286, 239, 233, 254, 251, 275, 260, 247, 270, 270, 248, 265, 260, 258, 241, 281, 227, 238, 264, 269, 235, 258, 268, 277, 245, 248, 247, 245, 267, 242, 270, 247, 239, 262, 278, 269, 272, 254, 278, 292, 244, 245, 247, 250, 260, 278, 246, 270, 272, 250, 262, 250, 254, 262, 272, 277, 238, 262, 272, 283, 226, 271, 252, 238, 252, 276, 260, 261, 241, 245, 266, 225, 247, 214, 264, 278, 275, 258, 277, 246, 256, 262, 256, 257, 257, 266, 286, 231, 249, 230, 245, 292, 276, 241, 235, 239, 248, 253, 284, 230, 237, 257, 267, 241, 256, 260, 255, 241, 257, 244, 239, 263, 281] payload_rows_per_dst=[5418, 5524, 5446, 5590, 5510, 5452, 5486, 5452] dup_rows_per_dst=[2718, 2634, 2672, 2844, 2732, 2726, 2654, 2678] combine_rows_per_src=[5418, 5524, 5446, 5590, 5510, 5452, 5486, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229536 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157221024 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314212512 weight_movement_bytes=37748736 total_nvlink_bytes=351961248 weight_to_token_ratio=0.120138 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155516928 dispatch_meta_bytes=228576 combine_payload_bytes=155516928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155745504 ingress_bytes=174391296 link_bytes=174391296 token_bytes=311262432 weight_movement_bytes=18874368 total_nvlink_bytes=330136800 weight_to_token_ratio=0.060638 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670931592e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38362 dup_rows=21722 dup_rows_remote=18948 dedup_ratio=0.331451 router_tokens_per_expert=[272, 259, 264, 261, 268, 260, 235, 243, 256, 233, 230, 249, 271, 278, 258, 249, 274, 247, 254, 267, 266, 256, 235, 260, 274, 266, 245, 252, 234, 275, 262, 246, 261, 240, 249, 260, 253, 255, 281, 279, 244, 237, 231, 292, 242, 247, 254, 242, 265, 240, 255, 250, 274, 249, 234, 276, 238, 260, 246, 263, 251, 246, 252, 244, 247, 271, 275, 252, 234, 253, 267, 255, 270, 272, 271, 241, 254, 259, 263, 259, 274, 271, 268, 248, 268, 244, 241, 270, 278, 264, 245, 244, 269, 221, 245, 249, 278, 253, 274, 246, 276, 235, 254, 286, 256, 250, 261, 264, 255, 255, 241, 274, 251, 252, 283, 240, 290, 219, 245, 235, 255, 279, 244, 252, 232, 281, 237, 264] payload_rows_per_dst=[5496, 5452, 5448, 5406, 5514, 5476, 5548, 5474] dup_rows_per_dst=[2676, 2774, 2686, 2680, 2772, 2722, 2768, 2644] combine_rows_per_src=[5496, 5452, 5448, 5406, 5514, 5476, 5548, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=228976 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157146736 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314064496 weight_movement_bytes=18874368 total_nvlink_bytes=332938864 weight_to_token_ratio=0.060097 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228408 combine_payload_bytes=156827648 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157056056 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313883704 weight_movement_bytes=37748736 total_nvlink_bytes=351632440 weight_to_token_ratio=0.120263 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43522 payload_rows_remote=38060 dup_rows=22014 dup_rows_remote=19244 dedup_ratio=0.335907 router_tokens_per_expert=[230, 231, 275, 270, 248, 242, 243, 245, 253, 245, 252, 256, 240, 250, 266, 246, 259, 275, 274, 247, 236, 229, 267, 261, 274, 246, 266, 269, 252, 293, 273, 283, 230, 243, 251, 245, 262, 255, 259, 249, 277, 283, 242, 249, 269, 277, 231, 253, 286, 235, 273, 267, 252, 254, 260, 288, 250, 261, 261, 243, 251, 231, 266, 254, 239, 234, 261, 256, 259, 262, 271, 259, 256, 272, 246, 258, 253, 231, 244, 239, 229, 260, 280, 251, 253, 268, 262, 282, 235, 237, 239, 255, 292, 255, 288, 245, 259, 225, 244, 274, 273, 258, 264, 246, 220, 271, 262, 257, 271, 264, 243, 285, 243, 250, 262, 266, 274, 232, 238, 265, 271, 261, 262, 243, 264, 253, 239, 255] payload_rows_per_dst=[5378, 5558, 5418, 5404, 5404, 5488, 5462, 5410] dup_rows_per_dst=[2606, 2850, 2732, 2860, 2676, 2774, 2770, 2746] combine_rows_per_src=[5378, 5558, 5418, 5404, 5404, 5488, 5462, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229528 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532312 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835096 weight_movement_bytes=18874368 total_nvlink_bytes=333709464 weight_to_token_ratio=0.059950 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43944 payload_rows_remote=38486 dup_rows=21592 dup_rows_remote=18842 dedup_ratio=0.329468 router_tokens_per_expert=[273, 268, 287, 279, 291, 243, 253, 255, 238, 268, 248, 263, 251, 258, 283, 245, 261, 285, 259, 263, 234, 241, 233, 265, 250, 260, 277, 272, 247, 264, 274, 233, 243, 271, 261, 233, 240, 272, 242, 257, 236, 253, 267, 263, 256, 227, 256, 241, 227, 267, 264, 244, 232, 257, 267, 231, 264, 243, 256, 256, 274, 246, 283, 270, 261, 260, 274, 264, 259, 254, 249, 241, 275, 248, 249, 252, 262, 241, 272, 243, 242, 264, 257, 246, 250, 248, 245, 272, 249, 230, 226, 281, 259, 268, 247, 259, 284, 252, 256, 291, 272, 242, 250, 247, 273, 242, 261, 270, 287, 233, 237, 230, 259, 245, 264, 257, 273, 250, 226, 262, 267, 264, 253, 224, 252, 266, 274, 238] payload_rows_per_dst=[5652, 5456, 5444, 5456, 5458, 5428, 5558, 5492] dup_rows_per_dst=[2754, 2780, 2592, 2706, 2750, 2658, 2696, 2656] combine_rows_per_src=[5652, 5456, 5444, 5456, 5458, 5428, 5558, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229240 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157359992 ingress_bytes=176005120 link_bytes=176005120 token_bytes=314490744 weight_movement_bytes=18874368 total_nvlink_bytes=333365112 weight_to_token_ratio=0.060016 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686394460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229216 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156122976 ingress_bytes=212516864 link_bytes=212516864 token_bytes=312016736 weight_movement_bytes=56623104 total_nvlink_bytes=368639840 weight_to_token_ratio=0.181475 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036232026e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38164 dup_rows=21842 dup_rows_remote=19210 dedup_ratio=0.333282 router_tokens_per_expert=[261, 269, 239, 279, 230, 261, 266, 278, 230, 239, 236, 241, 278, 234, 245, 266, 257, 267, 263, 249, 251, 232, 261, 224, 268, 238, 233, 271, 249, 245, 247, 247, 255, 261, 246, 247, 253, 258, 255, 287, 268, 263, 220, 256, 268, 264, 239, 241, 233, 239, 275, 239, 261, 243, 242, 258, 255, 235, 250, 254, 261, 257, 258, 267, 244, 301, 238, 252, 254, 301, 243, 250, 279, 254, 259, 272, 253, 245, 273, 276, 254, 233, 280, 241, 244, 252, 259, 228, 263, 251, 262, 281, 271, 248, 272, 253, 254, 262, 258, 277, 250, 250, 255, 272, 278, 283, 263, 297, 242, 240, 245, 260, 278, 235, 265, 280, 279, 256, 266, 234, 267, 271, 284, 248, 236, 249, 230, 256] payload_rows_per_dst=[5428, 5346, 5530, 5420, 5490, 5394, 5602, 5484] dup_rows_per_dst=[2676, 2658, 2632, 2634, 2898, 2790, 2770, 2784] combine_rows_per_src=[5428, 5346, 5530, 5420, 5490, 5394, 5602, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157638656 dispatch_meta_bytes=229312 combine_payload_bytes=157638656 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157867968 ingress_bytes=195387392 link_bytes=195387392 token_bytes=315506624 weight_movement_bytes=37748736 total_nvlink_bytes=353255360 weight_to_token_ratio=0.119645 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872105854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38360 dup_rows=21672 dup_rows_remote=18990 dedup_ratio=0.330688 router_tokens_per_expert=[248, 253, 262, 257, 239, 262, 261, 230, 237, 255, 245, 254, 276, 258, 271, 272, 272, 288, 266, 252, 238, 271, 273, 272, 250, 286, 254, 247, 276, 270, 258, 271, 268, 252, 234, 274, 252, 278, 259, 261, 255, 268, 249, 250, 267, 229, 279, 256, 241, 259, 259, 253, 263, 275, 259, 240, 262, 264, 253, 253, 252, 237, 246, 223, 258, 261, 264, 240, 263, 284, 263, 257, 286, 236, 237, 239, 251, 251, 265, 239, 230, 218, 266, 253, 254, 238, 249, 261, 233, 246, 242, 268, 267, 251, 242, 284, 270, 249, 249, 221, 248, 260, 250, 276, 272, 264, 251, 248, 254, 260, 259, 259, 255, 258, 268, 237, 269, 247, 252, 255, 252, 258, 253, 274, 257, 252, 243, 258] payload_rows_per_dst=[5504, 5566, 5652, 5396, 5504, 5362, 5478, 5402] dup_rows_per_dst=[2682, 2922, 2868, 2656, 2684, 2642, 2702, 2516] combine_rows_per_src=[5504, 5566, 5652, 5396, 5504, 5362, 5478, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38302 dup_rows=21726 dup_rows_remote=19040 dedup_ratio=0.331512 router_tokens_per_expert=[263, 240, 244, 240, 228, 236, 263, 243, 300, 266, 265, 248, 255, 255, 247, 255, 277, 257, 255, 239, 233, 265, 261, 238, 282, 259, 260, 239, 259, 247, 263, 276, 288, 230, 273, 267, 255, 265, 255, 245, 267, 277, 281, 240, 274, 258, 257, 252, 268, 247, 273, 268, 233, 250, 258, 237, 237, 262, 286, 263, 259, 251, 286, 256, 252, 258, 269, 244, 262, 275, 243, 232, 238, 270, 270, 251, 284, 248, 253, 236, 270, 244, 249, 252, 271, 263, 277, 260, 265, 271, 234, 266, 252, 257, 246, 268, 277, 274, 242, 246, 251, 220, 260, 255, 226, 240, 258, 226, 263, 259, 237, 238, 265, 227, 266, 212, 282, 251, 271, 272, 234, 266, 242, 258, 247, 262, 277, 258] payload_rows_per_dst=[5416, 5444, 5530, 5560, 5472, 5390, 5490, 5508] dup_rows_per_dst=[2680, 2776, 2646, 2900, 2684, 2570, 2784, 2686] combine_rows_per_src=[5416, 5444, 5530, 5560, 5472, 5390, 5490, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229496 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549240 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868984 weight_movement_bytes=18874368 total_nvlink_bytes=331743352 weight_to_token_ratio=0.060327 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38428 dup_rows=21654 dup_rows_remote=18980 dedup_ratio=0.330414 router_tokens_per_expert=[267, 269, 244, 250, 264, 253, 255, 256, 280, 258, 284, 273, 249, 254, 265, 237, 244, 280, 241, 259, 254, 229, 264, 271, 232, 260, 275, 281, 249, 260, 235, 275, 237, 247, 268, 264, 292, 275, 267, 235, 258, 277, 254, 258, 264, 238, 269, 258, 256, 259, 235, 258, 265, 255, 242, 229, 244, 248, 289, 264, 258, 258, 261, 243, 268, 248, 241, 254, 251, 229, 255, 258, 257, 248, 264, 258, 257, 230, 248, 273, 260, 240, 247, 259, 269, 234, 264, 265, 262, 241, 240, 250, 244, 278, 244, 247, 268, 238, 282, 250, 247, 254, 261, 272, 251, 266, 236, 263, 267, 254, 267, 278, 244, 244, 248, 255, 227, 240, 241, 268, 250, 278, 281, 258, 245, 234, 247, 279] payload_rows_per_dst=[5460, 5442, 5572, 5454, 5442, 5470, 5604, 5438] dup_rows_per_dst=[2856, 2776, 2750, 2674, 2636, 2618, 2704, 2640] combine_rows_per_src=[5460, 5442, 5572, 5454, 5442, 5470, 5604, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38154 dup_rows=21876 dup_rows_remote=19124 dedup_ratio=0.333801 router_tokens_per_expert=[251, 273, 242, 264, 252, 263, 262, 263, 244, 229, 255, 275, 262, 288, 274, 251, 282, 257, 260, 230, 251, 250, 245, 252, 264, 277, 225, 230, 240, 249, 264, 253, 250, 248, 251, 254, 245, 241, 302, 237, 255, 236, 259, 243, 247, 250, 255, 246, 231, 264, 276, 248, 251, 261, 257, 272, 258, 258, 218, 255, 310, 258, 269, 259, 248, 241, 250, 243, 245, 229, 251, 236, 240, 280, 256, 270, 240, 272, 255, 258, 269, 258, 254, 278, 263, 249, 256, 238, 248, 257, 232, 268, 247, 270, 287, 255, 242, 265, 276, 305, 258, 233, 261, 268, 253, 256, 274, 241, 253, 236, 282, 231, 263, 256, 241, 257, 246, 256, 255, 268, 240, 237, 238, 266, 273, 300, 269, 285] payload_rows_per_dst=[5488, 5386, 5388, 5490, 5444, 5506, 5458, 5500] dup_rows_per_dst=[2808, 2672, 2650, 2800, 2584, 2752, 2810, 2800] combine_rows_per_src=[5488, 5386, 5388, 5490, 5444, 5506, 5458, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38420 dup_rows=21742 dup_rows_remote=19020 dedup_ratio=0.331757 router_tokens_per_expert=[277, 237, 243, 243, 245, 239, 250, 291, 266, 259, 258, 238, 274, 276, 259, 249, 242, 264, 254, 254, 255, 246, 252, 251, 248, 229, 258, 256, 278, 252, 240, 269, 274, 284, 270, 249, 274, 252, 242, 254, 280, 252, 271, 257, 258, 282, 261, 258, 244, 264, 263, 249, 271, 270, 255, 273, 248, 276, 270, 268, 234, 234, 248, 247, 262, 231, 263, 237, 250, 244, 252, 288, 249, 235, 273, 277, 257, 247, 257, 289, 251, 257, 280, 247, 246, 259, 239, 284, 247, 226, 254, 267, 263, 249, 266, 256, 236, 231, 233, 253, 241, 262, 250, 268, 257, 253, 250, 247, 244, 271, 278, 262, 262, 256, 250, 252, 243, 228, 238, 259, 258, 248, 266, 256, 259, 277, 252, 242] payload_rows_per_dst=[5488, 5374, 5566, 5474, 5588, 5488, 5440, 5376] dup_rows_per_dst=[2720, 2722, 2870, 2754, 2634, 2694, 2632, 2716] combine_rows_per_src=[5488, 5374, 5566, 5474, 5588, 5488, 5440, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229400 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157351960 ingress_bytes=194871296 link_bytes=194871296 token_bytes=314474520 weight_movement_bytes=37748736 total_nvlink_bytes=352223256 weight_to_token_ratio=0.120038 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867160876e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229368 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114360 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999352 weight_movement_bytes=37748736 total_nvlink_bytes=351748088 weight_to_token_ratio=0.120219 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38126 dup_rows=21836 dup_rows_remote=18968 dedup_ratio=0.333191 router_tokens_per_expert=[239, 241, 257, 279, 276, 225, 242, 249, 237, 250, 252, 254, 249, 221, 282, 246, 260, 254, 265, 225, 249, 271, 281, 236, 250, 235, 270, 263, 264, 228, 243, 263, 292, 268, 263, 234, 282, 251, 276, 256, 276, 252, 260, 252, 269, 241, 266, 248, 254, 264, 248, 258, 253, 241, 219, 268, 278, 256, 236, 276, 267, 246, 272, 285, 254, 245, 244, 230, 254, 242, 246, 230, 217, 269, 274, 221, 288, 267, 262, 268, 261, 264, 230, 269, 263, 250, 293, 257, 254, 258, 284, 250, 269, 244, 236, 292, 264, 263, 296, 267, 261, 264, 281, 273, 249, 222, 261, 258, 273, 272, 258, 259, 231, 263, 236, 246, 257, 253, 223, 252, 239, 253, 256, 250, 260, 243, 253, 284] payload_rows_per_dst=[5388, 5408, 5538, 5454, 5378, 5560, 5574, 5400] dup_rows_per_dst=[2610, 2706, 2834, 2788, 2644, 2788, 2868, 2598] combine_rows_per_src=[5388, 5408, 5538, 5454, 5378, 5560, 5574, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=229632 combine_payload_bytes=157401088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157630720 ingress_bytes=176275456 link_bytes=176275456 token_bytes=315031808 weight_movement_bytes=18874368 total_nvlink_bytes=333906176 weight_to_token_ratio=0.059913 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688984686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229112 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156507896 ingress_bytes=212901888 link_bytes=212901888 token_bytes=312786680 weight_movement_bytes=56623104 total_nvlink_bytes=369409784 weight_to_token_ratio=0.181028 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039921137e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38328 dup_rows=21736 dup_rows_remote=19028 dedup_ratio=0.331665 router_tokens_per_expert=[261, 257, 243, 235, 263, 243, 256, 238, 232, 229, 253, 247, 251, 276, 208, 242, 269, 244, 242, 258, 249, 255, 252, 286, 247, 276, 274, 274, 279, 230, 257, 254, 252, 277, 261, 254, 236, 244, 259, 288, 260, 261, 242, 256, 252, 257, 231, 259, 241, 242, 235, 247, 252, 286, 262, 266, 236, 266, 249, 266, 268, 235, 289, 246, 252, 245, 277, 248, 269, 245, 269, 269, 246, 244, 247, 252, 256, 246, 265, 260, 261, 266, 251, 264, 250, 251, 256, 262, 279, 249, 309, 279, 265, 240, 274, 282, 259, 263, 255, 260, 271, 264, 275, 280, 257, 248, 231, 251, 245, 250, 274, 259, 267, 262, 281, 241, 244, 259, 220, 245, 269, 234, 256, 244, 244, 293, 240, 244] payload_rows_per_dst=[5336, 5500, 5422, 5522, 5472, 5608, 5508, 5432] dup_rows_per_dst=[2532, 2792, 2756, 2650, 2708, 2868, 2776, 2654] combine_rows_per_src=[5336, 5500, 5422, 5522, 5472, 5608, 5508, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15034143240 prefetch_bytes=773849088 reduce_bytes=754974720 weight_movement_bytes=1528823808 egress_bytes=7522562568 ingress_bytes=9040404480 link_bytes=9040404480 total_nvlink_bytes=16562967048 weight_to_token_ratio=0.101690 prefetch_share_of_ingress=0.085599 dedup_ratio=0.332703 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=40 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080946 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.804598074e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38280 dup_rows=21854 dup_rows_remote=19148 dedup_ratio=0.333466 router_tokens_per_expert=[283, 262, 272, 232, 263, 273, 271, 253, 255, 268, 223, 253, 265, 261, 267, 219, 262, 246, 242, 258, 278, 261, 234, 257, 253, 254, 281, 242, 268, 257, 270, 262, 266, 240, 225, 268, 248, 272, 252, 241, 249, 251, 267, 246, 262, 248, 263, 256, 247, 280, 274, 257, 249, 258, 251, 287, 265, 276, 269, 247, 265, 268, 209, 266, 262, 255, 271, 279, 216, 256, 269, 240, 270, 260, 247, 265, 249, 254, 252, 257, 240, 255, 232, 270, 247, 297, 255, 266, 250, 224, 260, 279, 253, 259, 239, 240, 262, 239, 269, 246, 251, 266, 236, 264, 244, 234, 240, 265, 264, 240, 250, 246, 270, 242, 265, 280, 273, 245, 242, 253, 240, 282, 253, 244, 240, 277, 266, 245] payload_rows_per_dst=[5488, 5508, 5402, 5500, 5414, 5456, 5460, 5454] dup_rows_per_dst=[2752, 2742, 2706, 2836, 2790, 2676, 2572, 2780] combine_rows_per_src=[5488, 5508, 5402, 5500, 5414, 5456, 5460, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229760 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157598080 ingress_bytes=176242688 link_bytes=176242688 token_bytes=314966400 weight_movement_bytes=18874368 total_nvlink_bytes=333840768 weight_to_token_ratio=0.059925 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688670720e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228376 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156392472 ingress_bytes=231661568 link_bytes=231661568 token_bytes=312556568 weight_movement_bytes=75497472 total_nvlink_bytes=388054040 weight_to_token_ratio=0.241548 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219667160e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15055157520 prefetch_bytes=660602880 reduce_bytes=943718400 weight_movement_bytes=1604321280 egress_bytes=7533082896 ingress_bytes=9126395904 link_bytes=9126395904 total_nvlink_bytes=16659478800 weight_to_token_ratio=0.106563 prefetch_share_of_ingress=0.072384 dedup_ratio=0.332677 prefetch_slots=70 unused_prefetch_slots=1466 slot_utilization=0.045573 reduce_slots=50 remote_demand=35 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081380 gemm_inner_pad_upper_bound=108077 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.821763231e-04 prefetch_bytes_per_gemm_flop=1.318660783e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229424 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157220912 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212400 weight_movement_bytes=18874368 total_nvlink_bytes=333086768 weight_to_token_ratio=0.060069 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15065281464 prefetch_bytes=811597824 reduce_bytes=698351616 weight_movement_bytes=1509949440 egress_bytes=7538144184 ingress_bytes=9037086720 link_bytes=9037086720 total_nvlink_bytes=16575230904 weight_to_token_ratio=0.100227 prefetch_share_of_ingress=0.089807 dedup_ratio=0.332459 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=37 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080512 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.803935801e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229712 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157024592 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313819472 weight_movement_bytes=37748736 total_nvlink_bytes=351568208 weight_to_token_ratio=0.120288 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38078 dup_rows=21908 dup_rows_remote=19150 dedup_ratio=0.334290 router_tokens_per_expert=[268, 264, 259, 254, 250, 273, 239, 268, 261, 269, 230, 252, 268, 232, 265, 266, 236, 255, 263, 286, 277, 257, 263, 265, 244, 247, 273, 280, 255, 245, 238, 245, 245, 236, 251, 252, 265, 240, 256, 253, 252, 254, 261, 283, 252, 249, 260, 251, 259, 259, 256, 242, 247, 239, 248, 226, 289, 274, 272, 261, 250, 242, 242, 268, 288, 263, 280, 254, 253, 250, 248, 265, 253, 259, 273, 280, 261, 254, 250, 243, 250, 240, 248, 261, 285, 244, 268, 270, 249, 242, 250, 251, 226, 240, 251, 260, 230, 259, 258, 258, 255, 257, 250, 266, 243, 275, 248, 256, 242, 259, 258, 254, 263, 258, 249, 278, 244, 237, 276, 236, 281, 236, 246, 272, 268, 260, 263, 243] payload_rows_per_dst=[5550, 5668, 5604, 5246, 5512, 5350, 5246, 5452] dup_rows_per_dst=[2758, 3080, 2936, 2482, 2764, 2650, 2470, 2768] combine_rows_per_src=[5550, 5668, 5604, 5246, 5512, 5350, 5246, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38108 dup_rows=21954 dup_rows_remote=19090 dedup_ratio=0.334991 router_tokens_per_expert=[252, 296, 276, 252, 264, 236, 232, 247, 245, 235, 244, 290, 226, 284, 256, 275, 249, 239, 259, 286, 235, 255, 237, 223, 230, 245, 257, 247, 266, 255, 240, 262, 239, 245, 274, 277, 263, 239, 269, 271, 231, 253, 276, 250, 259, 239, 245, 252, 252, 264, 245, 261, 269, 236, 258, 254, 255, 243, 276, 238, 278, 236, 282, 240, 257, 241, 248, 257, 235, 270, 263, 266, 262, 235, 275, 268, 247, 270, 233, 270, 256, 272, 279, 249, 239, 272, 240, 259, 278, 282, 277, 271, 254, 235, 263, 243, 259, 249, 237, 251, 254, 250, 260, 260, 267, 261, 248, 269, 264, 250, 261, 221, 269, 283, 243, 254, 262, 264, 263, 263, 273, 229, 262, 258, 260, 272, 272, 250] payload_rows_per_dst=[5482, 5350, 5482, 5454, 5398, 5474, 5450, 5492] dup_rows_per_dst=[2738, 2620, 2682, 2720, 2796, 2864, 2672, 2862] combine_rows_per_src=[5482, 5350, 5482, 5454, 5398, 5474, 5450, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15065089648 prefetch_bytes=736100352 reduce_bytes=868220928 weight_movement_bytes=1604321280 egress_bytes=7538050672 ingress_bytes=9131360256 link_bytes=9131360256 total_nvlink_bytes=16669410928 weight_to_token_ratio=0.106493 prefetch_share_of_ingress=0.080612 dedup_ratio=0.332418 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=46 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.822754189e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=228912 combine_payload_bytes=155967488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156196400 ingress_bytes=174841856 link_bytes=174841856 token_bytes=312163888 weight_movement_bytes=18874368 total_nvlink_bytes=331038256 weight_to_token_ratio=0.060463 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675248637e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38420 dup_rows=21618 dup_rows_remote=18922 dedup_ratio=0.329865 router_tokens_per_expert=[247, 216, 240, 280, 242, 261, 254, 258, 234, 265, 235, 261, 251, 288, 245, 268, 241, 260, 263, 234, 280, 264, 254, 262, 260, 267, 226, 253, 247, 240, 292, 221, 261, 233, 257, 259, 244, 271, 268, 272, 265, 260, 263, 255, 268, 265, 245, 235, 254, 272, 263, 244, 256, 253, 264, 289, 270, 249, 261, 261, 254, 279, 253, 258, 242, 264, 252, 256, 277, 267, 250, 250, 254, 240, 261, 241, 235, 245, 232, 260, 262, 273, 266, 254, 249, 275, 262, 271, 230, 277, 230, 259, 258, 248, 238, 265, 262, 250, 255, 278, 262, 254, 229, 273, 239, 240, 239, 247, 261, 267, 263, 278, 269, 256, 252, 269, 285, 240, 244, 239, 276, 271, 247, 278, 263, 247, 246, 236] payload_rows_per_dst=[5448, 5488, 5556, 5572, 5460, 5452, 5498, 5444] dup_rows_per_dst=[2642, 2640, 2686, 2788, 2592, 2782, 2696, 2792] combine_rows_per_src=[5448, 5488, 5556, 5572, 5460, 5452, 5498, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=228792 combine_payload_bytes=156090368 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156319160 ingress_bytes=193839104 link_bytes=193839104 token_bytes=312409528 weight_movement_bytes=37748736 total_nvlink_bytes=350158264 weight_to_token_ratio=0.120831 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857270920e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38296 dup_rows=21818 dup_rows_remote=19052 dedup_ratio=0.332916 router_tokens_per_expert=[266, 233, 241, 263, 263, 261, 260, 252, 243, 275, 225, 273, 224, 254, 236, 262, 276, 249, 266, 276, 260, 282, 255, 242, 259, 266, 267, 260, 225, 258, 230, 246, 249, 246, 254, 268, 254, 251, 266, 256, 275, 247, 295, 248, 226, 254, 235, 235, 250, 237, 250, 249, 278, 276, 236, 248, 264, 254, 251, 249, 243, 239, 250, 245, 259, 275, 257, 262, 244, 254, 247, 248, 247, 215, 264, 280, 252, 246, 286, 258, 270, 255, 269, 228, 273, 253, 251, 264, 249, 244, 249, 263, 277, 257, 272, 284, 295, 251, 249, 242, 231, 265, 242, 251, 263, 237, 278, 248, 250, 247, 271, 259, 294, 265, 282, 278, 283, 249, 288, 233, 239, 254, 263, 253, 258, 259, 249, 264] payload_rows_per_dst=[5360, 5540, 5468, 5368, 5422, 5552, 5456, 5552] dup_rows_per_dst=[2702, 2694, 2650, 2670, 2766, 2764, 2702, 2870] combine_rows_per_src=[5360, 5540, 5468, 5368, 5422, 5552, 5456, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229368 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157597688 ingress_bytes=232865792 link_bytes=232865792 token_bytes=314966008 weight_movement_bytes=75497472 total_nvlink_bytes=390463480 weight_to_token_ratio=0.239700 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.231205442e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15064219288 prefetch_bytes=773849088 reduce_bytes=717225984 weight_movement_bytes=1491075072 egress_bytes=7537614488 ingress_bytes=9017679872 link_bytes=9017679872 total_nvlink_bytes=16555294360 weight_to_token_ratio=0.098981 prefetch_share_of_ingress=0.085815 dedup_ratio=0.332401 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=38 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082538 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.800061907e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229392 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157089808 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950224 weight_movement_bytes=37748736 total_nvlink_bytes=351698960 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38122 dup_rows=21830 dup_rows_remote=18942 dedup_ratio=0.333099 router_tokens_per_expert=[241, 244, 240, 259, 261, 264, 244, 263, 271, 253, 260, 245, 257, 244, 257, 250, 264, 266, 244, 273, 261, 268, 261, 254, 265, 263, 248, 246, 262, 263, 275, 239, 251, 238, 274, 260, 276, 287, 257, 246, 263, 253, 268, 252, 250, 249, 246, 255, 249, 238, 237, 225, 280, 244, 273, 246, 242, 224, 252, 269, 252, 239, 257, 266, 272, 243, 271, 255, 271, 270, 237, 272, 248, 249, 257, 251, 228, 262, 246, 242, 244, 236, 260, 253, 260, 259, 235, 274, 256, 264, 250, 267, 254, 276, 261, 253, 277, 258, 281, 247, 269, 244, 254, 255, 251, 250, 245, 274, 273, 262, 276, 268, 264, 251, 240, 271, 266, 263, 265, 258, 255, 253, 271, 234, 230, 241, 270, 253] payload_rows_per_dst=[5584, 5446, 5430, 5440, 5464, 5526, 5444, 5372] dup_rows_per_dst=[2888, 2554, 2820, 2546, 2842, 2724, 2830, 2626] combine_rows_per_src=[5584, 5446, 5430, 5440, 5464, 5526, 5444, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15069345472 prefetch_bytes=773849088 reduce_bytes=792723456 weight_movement_bytes=1566572544 egress_bytes=7540176576 ingress_bytes=9095741440 link_bytes=9095741440 total_nvlink_bytes=16635918016 weight_to_token_ratio=0.103958 prefetch_share_of_ingress=0.085078 dedup_ratio=0.332087 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=42 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081453 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.815644148e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228256 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156375968 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312523680 weight_movement_bytes=18874368 total_nvlink_bytes=331398048 weight_to_token_ratio=0.060393 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38106 dup_rows=21888 dup_rows_remote=19232 dedup_ratio=0.333984 router_tokens_per_expert=[242, 229, 254, 245, 258, 268, 240, 255, 259, 252, 275, 253, 247, 230, 270, 251, 274, 261, 263, 237, 285, 251, 263, 238, 249, 279, 235, 253, 240, 240, 243, 234, 259, 251, 246, 260, 252, 242, 258, 255, 248, 285, 230, 246, 245, 251, 270, 288, 267, 246, 258, 237, 247, 246, 247, 254, 266, 281, 251, 243, 224, 257, 244, 267, 255, 270, 250, 286, 249, 278, 257, 276, 255, 240, 247, 270, 249, 271, 264, 241, 262, 229, 246, 270, 272, 259, 285, 269, 277, 251, 270, 231, 266, 268, 285, 263, 270, 244, 259, 261, 256, 258, 249, 263, 225, 264, 249, 253, 267, 272, 249, 260, 251, 285, 226, 247, 246, 277, 244, 248, 259, 252, 259, 282, 286, 246, 261, 245] payload_rows_per_dst=[5346, 5408, 5412, 5426, 5550, 5514, 5542, 5450] dup_rows_per_dst=[2710, 2682, 2760, 2644, 2766, 2892, 2656, 2778] combine_rows_per_src=[5346, 5408, 5412, 5426, 5550, 5514, 5542, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229352 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156311528 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312393704 weight_movement_bytes=18874368 total_nvlink_bytes=331268072 weight_to_token_ratio=0.060419 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15058271280 prefetch_bytes=849346560 reduce_bytes=943718400 weight_movement_bytes=1793064960 egress_bytes=7534640176 ingress_bytes=9316696064 link_bytes=9316696064 total_nvlink_bytes=16851336240 weight_to_token_ratio=0.119075 prefetch_share_of_ingress=0.091164 dedup_ratio=0.332561 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=50 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080657 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.859749950e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43596 payload_rows_remote=38020 dup_rows=21940 dup_rows_remote=19090 dedup_ratio=0.334778 router_tokens_per_expert=[290, 244, 270, 271, 257, 233, 239, 242, 239, 257, 252, 241, 238, 233, 284, 288, 249, 256, 255, 257, 251, 270, 243, 259, 282, 254, 274, 280, 238, 265, 223, 271, 276, 248, 245, 235, 268, 271, 233, 257, 280, 266, 230, 273, 262, 231, 248, 263, 271, 238, 270, 246, 278, 237, 249, 251, 269, 233, 284, 257, 260, 255, 229, 253, 250, 280, 243, 234, 256, 251, 227, 265, 252, 256, 260, 267, 254, 245, 262, 282, 266, 268, 257, 244, 248, 251, 261, 264, 249, 229, 258, 268, 242, 265, 267, 258, 272, 282, 265, 286, 251, 247, 235, 265, 279, 252, 284, 260, 232, 260, 253, 251, 250, 252, 244, 274, 247, 250, 244, 265, 264, 226, 267, 245, 250, 266, 254, 246] payload_rows_per_dst=[5576, 5508, 5476, 5630, 5458, 5364, 5196, 5388] dup_rows_per_dst=[2850, 2788, 2696, 2968, 2710, 2826, 2402, 2700] combine_rows_per_src=[5576, 5508, 5476, 5630, 5458, 5364, 5196, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=228440 combine_payload_bytes=155729920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155958360 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311688280 weight_movement_bytes=18874368 total_nvlink_bytes=330562648 weight_to_token_ratio=0.060555 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=37960 dup_rows=21978 dup_rows_remote=18868 dedup_ratio=0.335358 router_tokens_per_expert=[258, 226, 273, 254, 255, 250, 266, 245, 252, 246, 258, 259, 265, 259, 273, 259, 250, 257, 289, 266, 249, 288, 253, 249, 262, 243, 263, 274, 255, 240, 257, 253, 265, 266, 225, 253, 254, 238, 290, 235, 277, 284, 272, 227, 247, 267, 274, 233, 227, 256, 243, 263, 260, 251, 262, 266, 254, 234, 266, 257, 248, 255, 249, 250, 254, 252, 245, 275, 264, 273, 251, 249, 249, 222, 258, 260, 270, 219, 241, 238, 235, 256, 271, 276, 241, 264, 278, 237, 282, 249, 264, 240, 245, 272, 249, 237, 255, 245, 248, 240, 280, 263, 273, 262, 259, 271, 271, 302, 269, 253, 260, 254, 267, 260, 257, 248, 241, 268, 247, 245, 252, 252, 236, 257, 245, 265, 263, 250] payload_rows_per_dst=[5598, 5538, 5618, 5412, 5470, 5250, 5246, 5426] dup_rows_per_dst=[3110, 2780, 3016, 2670, 2710, 2500, 2512, 2680] combine_rows_per_src=[5598, 5538, 5618, 5412, 5470, 5250, 5246, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155484160 dispatch_meta_bytes=227312 combine_payload_bytes=155484160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155711472 ingress_bytes=174358528 link_bytes=174358528 token_bytes=311195632 weight_movement_bytes=18874368 total_nvlink_bytes=330070000 weight_to_token_ratio=0.060651 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670617626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38242 dup_rows=21784 dup_rows_remote=18956 dedup_ratio=0.332397 router_tokens_per_expert=[267, 233, 266, 269, 241, 259, 234, 265, 259, 269, 253, 269, 258, 274, 248, 279, 278, 248, 288, 256, 254, 237, 242, 271, 255, 233, 255, 255, 244, 289, 221, 260, 236, 239, 250, 264, 223, 271, 279, 277, 260, 249, 239, 241, 254, 262, 242, 229, 268, 260, 244, 229, 232, 257, 258, 252, 252, 264, 259, 246, 260, 256, 247, 267, 274, 267, 279, 243, 294, 247, 237, 268, 248, 251, 241, 260, 223, 276, 273, 234, 279, 280, 279, 252, 260, 276, 244, 246, 265, 247, 222, 243, 257, 235, 230, 247, 266, 256, 282, 263, 276, 262, 255, 262, 286, 250, 267, 230, 266, 260, 226, 279, 256, 270, 258, 246, 249, 267, 268, 267, 283, 249, 247, 213, 261, 258, 278, 240] payload_rows_per_dst=[5510, 5554, 5444, 5478, 5420, 5456, 5604, 5286] dup_rows_per_dst=[2828, 2904, 2596, 2624, 2758, 2658, 2768, 2648] combine_rows_per_src=[5510, 5554, 5444, 5478, 5420, 5456, 5604, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228792 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868024 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507256 weight_movement_bytes=18874368 total_nvlink_bytes=332381624 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15025080544 prefetch_bytes=887095296 reduce_bytes=547356672 weight_movement_bytes=1434451968 egress_bytes=7518030048 ingress_bytes=8941502464 link_bytes=8941502464 total_nvlink_bytes=16459532512 weight_to_token_ratio=0.095471 prefetch_share_of_ingress=0.099211 dedup_ratio=0.333089 prefetch_slots=94 unused_prefetch_slots=1442 slot_utilization=0.061198 reduce_slots=29 remote_demand=47 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=109601 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.784855773e-04 prefetch_bytes_per_gemm_flop=1.770773052e-05 anomalies=none + [2026-08-29 23:39:11] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 779.8 | throughput per GPU (TFLOP/s/GPU): 114.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.844584E+00 | loss scale: 1.0 | grad norm: 3.097 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38282 dup_rows=21700 dup_rows_remote=19048 dedup_ratio=0.331116 router_tokens_per_expert=[252, 266, 259, 234, 274, 244, 266, 248, 271, 264, 253, 257, 268, 246, 255, 248, 248, 234, 244, 257, 239, 261, 274, 251, 247, 280, 256, 252, 268, 272, 263, 257, 266, 264, 255, 271, 242, 239, 235, 299, 266, 238, 271, 260, 219, 261, 241, 254, 259, 233, 234, 283, 231, 247, 275, 251, 260, 252, 293, 279, 257, 246, 255, 262, 255, 285, 235, 257, 246, 267, 248, 245, 272, 267, 263, 259, 235, 218, 251, 245, 255, 253, 259, 260, 226, 242, 260, 266, 264, 251, 279, 251, 272, 241, 247, 250, 239, 286, 226, 275, 251, 271, 239, 267, 254, 269, 258, 259, 232, 251, 244, 254, 255, 271, 271, 276, 264, 271, 267, 251, 215, 275, 277, 244, 287, 237, 235, 267] payload_rows_per_dst=[5450, 5554, 5514, 5490, 5390, 5472, 5478, 5488] dup_rows_per_dst=[2760, 2652, 2648, 2744, 2706, 2680, 2672, 2838] combine_rows_per_src=[5450, 5554, 5514, 5490, 5390, 5472, 5478, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38110 dup_rows=22002 dup_rows_remote=19278 dedup_ratio=0.335724 router_tokens_per_expert=[231, 243, 234, 279, 271, 248, 267, 257, 250, 261, 266, 229, 269, 267, 277, 281, 269, 258, 270, 243, 266, 264, 255, 281, 268, 286, 270, 245, 257, 262, 260, 272, 262, 233, 243, 267, 253, 279, 269, 220, 274, 250, 262, 237, 259, 251, 266, 249, 260, 232, 251, 257, 236, 247, 271, 242, 241, 240, 234, 251, 238, 262, 254, 247, 253, 237, 273, 266, 261, 241, 256, 246, 231, 274, 243, 254, 251, 247, 273, 257, 223, 257, 288, 210, 251, 271, 284, 234, 267, 239, 257, 247, 252, 273, 278, 285, 253, 239, 244, 251, 267, 249, 261, 267, 256, 265, 244, 256, 248, 276, 274, 255, 267, 287, 268, 263, 241, 249, 230, 242, 262, 267, 246, 269, 248, 254, 259, 239] payload_rows_per_dst=[5428, 5556, 5424, 5318, 5410, 5464, 5480, 5454] dup_rows_per_dst=[2832, 2896, 2724, 2608, 2716, 2768, 2730, 2728] combine_rows_per_src=[5428, 5556, 5424, 5318, 5410, 5464, 5480, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38230 dup_rows=21716 dup_rows_remote=18868 dedup_ratio=0.331360 router_tokens_per_expert=[266, 263, 238, 264, 245, 265, 239, 245, 263, 268, 262, 259, 269, 237, 276, 263, 239, 237, 241, 262, 242, 249, 236, 279, 267, 262, 261, 230, 254, 221, 267, 254, 274, 276, 275, 288, 274, 260, 262, 235, 258, 259, 224, 229, 261, 252, 244, 243, 233, 275, 248, 274, 236, 269, 287, 260, 272, 274, 240, 232, 231, 266, 265, 264, 270, 269, 231, 228, 269, 237, 252, 249, 256, 248, 264, 258, 263, 256, 251, 235, 236, 245, 252, 246, 263, 284, 259, 287, 253, 267, 250, 253, 273, 261, 235, 253, 255, 269, 267, 277, 249, 254, 293, 246, 261, 246, 238, 251, 263, 258, 231, 245, 227, 278, 276, 255, 242, 268, 271, 247, 263, 247, 258, 251, 246, 280, 276, 264] payload_rows_per_dst=[5516, 5434, 5318, 5562, 5568, 5416, 5416, 5590] dup_rows_per_dst=[2672, 2560, 2542, 2690, 2922, 2692, 2790, 2848] combine_rows_per_src=[5516, 5434, 5318, 5562, 5568, 5416, 5416, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38196 dup_rows=21892 dup_rows_remote=19146 dedup_ratio=0.334045 router_tokens_per_expert=[255, 265, 232, 277, 257, 245, 261, 254, 249, 254, 250, 242, 265, 280, 273, 270, 274, 248, 244, 265, 266, 235, 239, 250, 259, 263, 265, 254, 260, 268, 256, 255, 257, 275, 276, 260, 255, 242, 261, 268, 249, 273, 256, 257, 250, 272, 265, 243, 242, 271, 279, 291, 252, 260, 257, 230, 267, 254, 254, 236, 250, 245, 255, 261, 258, 229, 262, 256, 259, 244, 257, 232, 282, 250, 297, 275, 239, 270, 241, 246, 254, 232, 237, 264, 266, 244, 280, 277, 231, 255, 287, 262, 265, 272, 271, 253, 268, 264, 260, 236, 224, 225, 246, 254, 268, 265, 235, 215, 250, 234, 232, 257, 274, 271, 232, 257, 275, 264, 241, 253, 245, 273, 238, 267, 244, 250, 257, 254] payload_rows_per_dst=[5538, 5424, 5472, 5556, 5448, 5462, 5322, 5422] dup_rows_per_dst=[2720, 2778, 2846, 2652, 2746, 2838, 2544, 2768] combine_rows_per_src=[5538, 5424, 5472, 5556, 5448, 5462, 5322, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38278 dup_rows=21770 dup_rows_remote=19022 dedup_ratio=0.332184 router_tokens_per_expert=[257, 235, 252, 235, 230, 257, 246, 259, 256, 249, 261, 251, 238, 255, 258, 259, 242, 258, 251, 254, 279, 266, 241, 282, 259, 268, 275, 226, 257, 269, 251, 254, 257, 265, 242, 255, 275, 264, 250, 247, 258, 243, 268, 259, 259, 252, 239, 239, 239, 247, 264, 260, 249, 248, 273, 247, 260, 262, 259, 257, 269, 276, 267, 241, 270, 259, 227, 238, 269, 285, 263, 249, 261, 229, 259, 249, 233, 241, 247, 275, 273, 264, 236, 229, 263, 272, 284, 245, 276, 272, 249, 275, 266, 279, 281, 271, 252, 242, 280, 264, 256, 224, 243, 262, 272, 240, 242, 260, 256, 280, 269, 233, 259, 242, 246, 281, 266, 266, 230, 229, 269, 248, 256, 258, 257, 270, 252, 255] payload_rows_per_dst=[5382, 5556, 5488, 5488, 5366, 5524, 5464, 5498] dup_rows_per_dst=[2614, 2708, 2656, 2748, 2742, 2946, 2686, 2670] combine_rows_per_src=[5382, 5556, 5488, 5488, 5366, 5524, 5464, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38328 dup_rows=21866 dup_rows_remote=19312 dedup_ratio=0.333649 router_tokens_per_expert=[269, 279, 233, 265, 246, 269, 260, 261, 282, 257, 255, 233, 256, 236, 222, 241, 239, 246, 270, 247, 245, 253, 277, 263, 275, 268, 250, 254, 270, 279, 247, 255, 257, 257, 269, 243, 273, 223, 265, 250, 255, 255, 258, 237, 277, 261, 284, 297, 272, 265, 267, 259, 228, 241, 254, 261, 251, 246, 254, 249, 259, 239, 254, 254, 259, 267, 262, 260, 243, 273, 267, 265, 278, 266, 240, 250, 270, 262, 244, 253, 256, 236, 233, 252, 251, 253, 253, 246, 229, 247, 240, 254, 250, 245, 258, 245, 262, 237, 242, 245, 274, 260, 251, 269, 238, 253, 260, 277, 230, 232, 263, 282, 255, 250, 270, 279, 235, 285, 278, 232, 265, 255, 261, 247, 285, 275, 241, 257] payload_rows_per_dst=[5402, 5518, 5564, 5380, 5460, 5342, 5496, 5508] dup_rows_per_dst=[2726, 2758, 2758, 2726, 2858, 2554, 2654, 2832] combine_rows_per_src=[5402, 5518, 5564, 5380, 5460, 5342, 5496, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38268 dup_rows=21778 dup_rows_remote=19138 dedup_ratio=0.332306 router_tokens_per_expert=[242, 245, 253, 276, 252, 267, 265, 257, 284, 239, 230, 276, 282, 241, 290, 268, 270, 249, 270, 261, 249, 245, 251, 254, 248, 260, 245, 234, 230, 263, 278, 238, 257, 228, 252, 265, 258, 255, 261, 243, 252, 267, 247, 244, 255, 290, 253, 253, 264, 252, 245, 265, 227, 259, 283, 258, 257, 251, 245, 245, 263, 268, 260, 270, 246, 248, 238, 252, 223, 255, 237, 275, 232, 288, 263, 252, 280, 269, 247, 254, 207, 261, 236, 233, 283, 240, 301, 274, 281, 274, 247, 215, 269, 238, 246, 246, 276, 264, 258, 262, 231, 263, 250, 253, 244, 250, 239, 256, 254, 257, 249, 259, 252, 270, 215, 276, 258, 259, 271, 252, 268, 285, 254, 252, 288, 252, 272, 265] payload_rows_per_dst=[5520, 5418, 5424, 5538, 5420, 5426, 5490, 5522] dup_rows_per_dst=[2814, 2672, 2736, 2686, 2698, 2676, 2640, 2856] combine_rows_per_src=[5520, 5418, 5424, 5538, 5420, 5426, 5490, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229320 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032392 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835464 weight_movement_bytes=37748736 total_nvlink_bytes=351584200 weight_to_token_ratio=0.120282 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=229552 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156328112 ingress_bytes=193847296 link_bytes=193847296 token_bytes=312426672 weight_movement_bytes=37748736 total_nvlink_bytes=350175408 weight_to_token_ratio=0.120824 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857349411e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=228392 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156818472 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313408552 weight_movement_bytes=37748736 total_nvlink_bytes=351157288 weight_to_token_ratio=0.120446 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229368 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680184 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131000 weight_movement_bytes=18874368 total_nvlink_bytes=332005368 weight_to_token_ratio=0.060276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229200 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157015888 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313802576 weight_movement_bytes=18874368 total_nvlink_bytes=332676944 weight_to_token_ratio=0.060147 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=230560 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157222048 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314213536 weight_movement_bytes=56623104 total_nvlink_bytes=370836640 weight_to_token_ratio=0.180206 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229624 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156975352 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313721080 weight_movement_bytes=18874368 total_nvlink_bytes=332595448 weight_to_token_ratio=0.060163 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38334 dup_rows=21714 dup_rows_remote=18992 dedup_ratio=0.331329 router_tokens_per_expert=[237, 248, 263, 269, 273, 254, 262, 273, 296, 261, 260, 245, 249, 270, 267, 248, 253, 234, 247, 272, 242, 258, 260, 248, 238, 262, 283, 240, 257, 246, 282, 283, 249, 264, 274, 248, 266, 261, 269, 281, 261, 254, 238, 258, 269, 257, 252, 268, 234, 268, 265, 270, 230, 249, 239, 249, 288, 256, 263, 239, 220, 260, 258, 269, 255, 246, 270, 246, 271, 257, 247, 263, 265, 261, 268, 258, 225, 250, 239, 246, 251, 272, 248, 266, 238, 227, 263, 241, 251, 239, 251, 285, 254, 265, 284, 259, 260, 234, 239, 267, 254, 262, 247, 268, 236, 239, 224, 246, 240, 248, 258, 259, 261, 272, 267, 251, 254, 253, 246, 275, 263, 265, 259, 258, 251, 218, 263, 264] payload_rows_per_dst=[5526, 5488, 5556, 5496, 5368, 5542, 5360, 5486] dup_rows_per_dst=[2824, 2722, 2782, 2852, 2532, 2646, 2602, 2754] combine_rows_per_src=[5526, 5488, 5556, 5496, 5368, 5542, 5360, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38394 dup_rows=21690 dup_rows_remote=19018 dedup_ratio=0.330963 router_tokens_per_expert=[250, 270, 249, 239, 280, 245, 268, 262, 271, 259, 241, 268, 234, 253, 250, 262, 260, 234, 260, 255, 264, 246, 276, 255, 227, 250, 254, 268, 251, 276, 247, 251, 248, 269, 240, 262, 265, 259, 249, 233, 250, 238, 249, 240, 284, 266, 275, 235, 246, 258, 234, 260, 251, 265, 256, 257, 227, 258, 253, 272, 282, 271, 260, 248, 244, 246, 261, 277, 253, 254, 310, 256, 253, 224, 263, 267, 245, 268, 268, 294, 248, 283, 236, 241, 257, 252, 255, 275, 233, 273, 256, 264, 258, 260, 246, 259, 281, 257, 245, 257, 254, 246, 252, 251, 263, 230, 245, 265, 264, 252, 263, 257, 257, 248, 235, 251, 277, 242, 251, 265, 279, 262, 234, 256, 238, 271, 237, 269] payload_rows_per_dst=[5504, 5476, 5452, 5440, 5594, 5502, 5420, 5458] dup_rows_per_dst=[2698, 2672, 2672, 2756, 2772, 2690, 2744, 2686] combine_rows_per_src=[5504, 5476, 5452, 5440, 5594, 5502, 5420, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38278 dup_rows=21776 dup_rows_remote=19114 dedup_ratio=0.332275 router_tokens_per_expert=[260, 262, 242, 271, 249, 248, 267, 254, 275, 258, 231, 263, 275, 278, 275, 252, 231, 244, 278, 277, 266, 243, 237, 259, 273, 276, 270, 253, 267, 254, 282, 266, 275, 251, 250, 263, 282, 289, 251, 254, 274, 255, 247, 231, 249, 236, 250, 248, 269, 276, 266, 252, 266, 260, 264, 268, 249, 226, 245, 245, 258, 247, 250, 225, 253, 239, 260, 256, 267, 219, 263, 251, 265, 236, 281, 228, 256, 253, 258, 269, 259, 258, 275, 254, 252, 237, 232, 269, 257, 225, 255, 263, 263, 261, 227, 268, 256, 260, 280, 284, 256, 245, 240, 249, 267, 251, 245, 259, 265, 270, 228, 226, 260, 266, 230, 288, 258, 250, 240, 263, 252, 261, 244, 243, 254, 258, 255, 249] payload_rows_per_dst=[5228, 5532, 5444, 5490, 5444, 5404, 5736, 5482] dup_rows_per_dst=[2442, 2820, 2720, 2642, 2708, 2806, 2976, 2662] combine_rows_per_src=[5228, 5532, 5444, 5490, 5444, 5404, 5736, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38030 dup_rows=21858 dup_rows_remote=18968 dedup_ratio=0.333527 router_tokens_per_expert=[224, 286, 247, 257, 258, 244, 285, 247, 263, 255, 262, 249, 259, 274, 301, 252, 247, 275, 251, 264, 261, 263, 234, 246, 269, 248, 256, 261, 265, 236, 258, 229, 228, 264, 266, 246, 218, 246, 257, 266, 245, 234, 252, 272, 212, 245, 221, 261, 254, 258, 232, 225, 245, 267, 242, 254, 238, 254, 250, 260, 260, 253, 228, 245, 268, 263, 289, 268, 263, 281, 237, 248, 269, 288, 235, 263, 295, 271, 263, 268, 238, 265, 241, 244, 258, 282, 268, 263, 266, 277, 259, 256, 254, 239, 282, 251, 253, 239, 252, 266, 262, 261, 246, 262, 248, 265, 254, 237, 239, 254, 255, 302, 258, 259, 271, 274, 241, 278, 240, 271, 251, 253, 248, 262, 265, 256, 257, 253] payload_rows_per_dst=[5500, 5440, 5320, 5332, 5648, 5502, 5410, 5526] dup_rows_per_dst=[2826, 2686, 2546, 2598, 2890, 2784, 2780, 2748] combine_rows_per_src=[5500, 5440, 5320, 5332, 5648, 5502, 5410, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38340 dup_rows=21836 dup_rows_remote=19306 dedup_ratio=0.333191 router_tokens_per_expert=[257, 275, 264, 251, 236, 235, 252, 246, 295, 245, 257, 247, 260, 276, 240, 242, 255, 271, 245, 236, 220, 265, 234, 283, 235, 262, 227, 268, 250, 263, 262, 235, 248, 258, 261, 252, 262, 281, 266, 277, 277, 253, 249, 262, 247, 287, 260, 236, 249, 239, 260, 239, 231, 253, 258, 258, 251, 249, 257, 269, 246, 207, 254, 225, 266, 243, 268, 253, 260, 261, 263, 298, 279, 260, 246, 241, 275, 250, 264, 287, 283, 266, 269, 259, 254, 239, 269, 253, 253, 261, 242, 271, 264, 236, 235, 272, 259, 241, 257, 250, 270, 252, 247, 270, 247, 237, 255, 291, 252, 268, 227, 264, 263, 271, 255, 241, 256, 261, 262, 249, 249, 238, 247, 259, 271, 277, 266, 266] payload_rows_per_dst=[5428, 5356, 5538, 5360, 5560, 5494, 5464, 5500] dup_rows_per_dst=[2728, 2666, 2814, 2530, 2868, 2758, 2710, 2762] combine_rows_per_src=[5428, 5356, 5538, 5360, 5560, 5494, 5464, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43536 payload_rows_remote=38140 dup_rows=22000 dup_rows_remote=19342 dedup_ratio=0.335693 router_tokens_per_expert=[238, 251, 271, 259, 277, 257, 267, 262, 280, 264, 253, 257, 282, 241, 243, 231, 258, 264, 232, 267, 247, 276, 251, 252, 265, 279, 267, 270, 285, 223, 258, 265, 252, 259, 251, 256, 266, 266, 248, 252, 253, 287, 261, 239, 243, 243, 244, 276, 245, 266, 266, 266, 258, 244, 258, 257, 235, 248, 250, 256, 219, 272, 251, 240, 260, 268, 237, 272, 274, 266, 241, 263, 258, 250, 253, 240, 260, 264, 304, 266, 253, 255, 268, 251, 245, 230, 254, 230, 254, 244, 252, 254, 250, 242, 241, 248, 226, 251, 228, 240, 259, 235, 275, 254, 272, 254, 262, 257, 262, 247, 234, 260, 301, 273, 248, 231, 259, 252, 272, 258, 252, 267, 261, 262, 267, 267, 282, 234] payload_rows_per_dst=[5410, 5470, 5542, 5404, 5470, 5342, 5396, 5502] dup_rows_per_dst=[2834, 2848, 2650, 2658, 2882, 2600, 2658, 2870] combine_rows_per_src=[5410, 5470, 5542, 5404, 5470, 5342, 5396, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38060 dup_rows=21962 dup_rows_remote=19244 dedup_ratio=0.335114 router_tokens_per_expert=[273, 291, 245, 255, 269, 280, 253, 260, 266, 264, 273, 228, 252, 243, 269, 247, 263, 252, 249, 250, 266, 246, 286, 234, 244, 258, 254, 237, 239, 221, 247, 269, 250, 250, 249, 221, 264, 230, 259, 280, 251, 291, 268, 273, 252, 243, 294, 257, 257, 258, 243, 270, 275, 238, 261, 227, 254, 273, 254, 257, 264, 261, 254, 277, 236, 244, 234, 248, 274, 253, 259, 271, 250, 270, 283, 250, 284, 258, 278, 252, 278, 245, 254, 251, 247, 265, 282, 263, 268, 218, 279, 264, 279, 214, 245, 264, 251, 283, 267, 234, 256, 251, 246, 273, 280, 263, 230, 265, 267, 238, 236, 234, 242, 252, 254, 235, 246, 247, 255, 253, 258, 276, 249, 232, 245, 255, 247, 250] payload_rows_per_dst=[5526, 5354, 5524, 5458, 5440, 5514, 5388, 5370] dup_rows_per_dst=[2810, 2676, 2740, 2788, 2848, 2718, 2760, 2622] combine_rows_per_src=[5526, 5354, 5524, 5458, 5440, 5514, 5388, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229304 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245368 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261432 weight_movement_bytes=18874368 total_nvlink_bytes=333135800 weight_to_token_ratio=0.060059 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229648 combine_payload_bytes=157261824 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157491472 ingress_bytes=195010560 link_bytes=195010560 token_bytes=314753296 weight_movement_bytes=37748736 total_nvlink_bytes=352502032 weight_to_token_ratio=0.119931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868495235e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229568 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016256 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313802944 weight_movement_bytes=37748736 total_nvlink_bytes=351551680 weight_to_token_ratio=0.120294 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=227992 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155998872 ingress_bytes=212393984 link_bytes=212393984 token_bytes=311769752 weight_movement_bytes=56623104 total_nvlink_bytes=368392856 weight_to_token_ratio=0.181618 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.035054650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=230584 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157271224 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311864 weight_movement_bytes=18874368 total_nvlink_bytes=333186232 weight_to_token_ratio=0.060050 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229928 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156451368 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312672808 weight_movement_bytes=18874368 total_nvlink_bytes=331547176 weight_to_token_ratio=0.060365 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229216 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156122976 ingress_bytes=174768128 link_bytes=174768128 token_bytes=312016736 weight_movement_bytes=18874368 total_nvlink_bytes=330891104 weight_to_token_ratio=0.060492 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674542211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43568 payload_rows_remote=38118 dup_rows=21968 dup_rows_remote=19256 dedup_ratio=0.335205 router_tokens_per_expert=[258, 294, 253, 254, 230, 246, 234, 261, 239, 260, 254, 245, 249, 301, 262, 241, 253, 243, 264, 245, 263, 265, 237, 237, 241, 280, 229, 244, 254, 272, 243, 256, 269, 232, 260, 271, 234, 289, 258, 252, 238, 246, 288, 298, 296, 237, 262, 281, 292, 251, 257, 272, 242, 270, 259, 266, 251, 284, 230, 239, 244, 248, 249, 282, 234, 261, 263, 273, 243, 254, 259, 247, 249, 269, 264, 282, 233, 252, 227, 233, 252, 275, 276, 266, 259, 258, 239, 297, 259, 236, 250, 296, 254, 243, 262, 255, 255, 242, 260, 240, 247, 255, 250, 260, 271, 247, 251, 301, 257, 261, 274, 271, 233, 245, 259, 244, 237, 243, 246, 244, 219, 270, 232, 252, 250, 273, 241, 264] payload_rows_per_dst=[5450, 5380, 5590, 5500, 5356, 5478, 5604, 5210] dup_rows_per_dst=[2712, 2672, 2832, 2850, 2730, 2798, 3070, 2304] combine_rows_per_src=[5450, 5380, 5590, 5500, 5356, 5478, 5604, 5210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38392 dup_rows=21692 dup_rows_remote=18968 dedup_ratio=0.330994 router_tokens_per_expert=[229, 258, 291, 243, 235, 251, 275, 261, 251, 236, 243, 261, 270, 241, 261, 246, 250, 259, 250, 244, 246, 259, 261, 277, 271, 247, 257, 252, 254, 237, 244, 280, 255, 240, 249, 261, 256, 245, 268, 264, 258, 242, 260, 286, 262, 270, 247, 279, 244, 245, 257, 269, 278, 261, 264, 241, 239, 253, 251, 264, 247, 228, 267, 261, 276, 254, 259, 250, 266, 252, 266, 235, 240, 236, 256, 227, 268, 254, 266, 273, 251, 265, 272, 258, 233, 256, 267, 253, 245, 233, 281, 258, 243, 266, 268, 258, 253, 271, 246, 287, 238, 277, 271, 238, 268, 239, 236, 236, 248, 236, 264, 258, 263, 262, 256, 256, 275, 272, 260, 252, 274, 280, 265, 258, 238, 259, 262, 234] payload_rows_per_dst=[5450, 5452, 5494, 5490, 5512, 5482, 5494, 5470] dup_rows_per_dst=[2654, 2724, 2790, 2648, 2644, 2732, 2638, 2862] combine_rows_per_src=[5450, 5452, 5494, 5490, 5512, 5482, 5494, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38148 dup_rows=21932 dup_rows_remote=19170 dedup_ratio=0.334656 router_tokens_per_expert=[243, 278, 259, 287, 269, 240, 259, 265, 241, 259, 262, 237, 262, 247, 227, 246, 246, 228, 269, 273, 242, 235, 250, 262, 277, 244, 260, 238, 232, 249, 258, 265, 248, 262, 261, 230, 253, 247, 274, 244, 241, 243, 296, 268, 244, 264, 272, 262, 270, 246, 245, 248, 257, 278, 252, 263, 272, 273, 249, 274, 257, 250, 253, 257, 251, 248, 250, 267, 229, 238, 248, 264, 264, 265, 243, 259, 263, 273, 260, 257, 258, 278, 258, 265, 247, 255, 253, 240, 241, 246, 242, 240, 259, 250, 254, 263, 278, 259, 282, 251, 262, 254, 267, 279, 248, 243, 270, 254, 260, 258, 284, 300, 226, 238, 244, 267, 247, 250, 238, 270, 280, 267, 278, 246, 247, 257, 246, 228] payload_rows_per_dst=[5484, 5434, 5456, 5516, 5370, 5330, 5622, 5392] dup_rows_per_dst=[2678, 2622, 2762, 2772, 2788, 2768, 2876, 2666] combine_rows_per_src=[5484, 5434, 5456, 5516, 5370, 5330, 5622, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43922 payload_rows_remote=38438 dup_rows=21614 dup_rows_remote=18922 dedup_ratio=0.329803 router_tokens_per_expert=[244, 255, 245, 273, 273, 267, 289, 251, 239, 272, 252, 247, 270, 263, 258, 239, 272, 264, 245, 255, 274, 227, 254, 251, 245, 242, 241, 263, 281, 286, 239, 265, 287, 252, 247, 243, 251, 255, 268, 240, 271, 265, 242, 280, 261, 242, 236, 247, 257, 235, 295, 267, 253, 256, 234, 225, 246, 269, 259, 244, 262, 267, 256, 291, 243, 262, 237, 245, 273, 253, 241, 225, 294, 247, 251, 245, 254, 269, 259, 235, 274, 246, 252, 274, 247, 251, 290, 250, 273, 231, 276, 248, 263, 261, 279, 236, 263, 271, 250, 263, 229, 262, 264, 275, 243, 248, 231, 269, 256, 234, 259, 238, 265, 258, 241, 260, 237, 263, 275, 255, 282, 234, 257, 257, 256, 241, 257, 247] payload_rows_per_dst=[5460, 5524, 5504, 5434, 5426, 5562, 5528, 5484] dup_rows_per_dst=[2722, 2678, 2670, 2890, 2640, 2740, 2582, 2692] combine_rows_per_src=[5460, 5524, 5504, 5434, 5426, 5562, 5528, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38148 dup_rows=21884 dup_rows_remote=19230 dedup_ratio=0.333923 router_tokens_per_expert=[264, 249, 253, 242, 251, 255, 222, 276, 257, 280, 243, 295, 255, 232, 276, 256, 216, 269, 263, 254, 230, 250, 259, 258, 262, 257, 250, 258, 219, 263, 270, 245, 273, 229, 227, 272, 247, 232, 255, 285, 258, 281, 237, 244, 251, 269, 245, 251, 265, 253, 275, 274, 244, 242, 244, 240, 252, 261, 268, 267, 240, 225, 255, 274, 262, 237, 260, 244, 266, 247, 235, 253, 255, 255, 236, 254, 237, 233, 258, 290, 260, 261, 252, 250, 262, 258, 255, 254, 271, 270, 259, 274, 262, 247, 258, 255, 255, 253, 264, 265, 316, 246, 265, 303, 262, 242, 238, 273, 247, 241, 261, 264, 251, 284, 266, 236, 270, 270, 260, 247, 226, 268, 252, 274, 246, 262, 265, 262] payload_rows_per_dst=[5478, 5444, 5396, 5504, 5352, 5506, 5492, 5480] dup_rows_per_dst=[2734, 2602, 2716, 2654, 2692, 2790, 2898, 2798] combine_rows_per_src=[5478, 5444, 5396, 5504, 5352, 5506, 5492, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38208 dup_rows=21868 dup_rows_remote=18998 dedup_ratio=0.333679 router_tokens_per_expert=[269, 259, 261, 271, 262, 256, 270, 242, 280, 250, 260, 266, 243, 252, 291, 252, 248, 263, 262, 254, 262, 247, 267, 275, 267, 267, 247, 285, 259, 269, 250, 246, 259, 250, 250, 279, 251, 237, 248, 237, 262, 228, 253, 245, 232, 265, 261, 271, 253, 252, 256, 259, 248, 246, 274, 255, 263, 256, 240, 257, 259, 223, 256, 249, 222, 282, 252, 269, 251, 269, 276, 266, 262, 261, 266, 297, 253, 243, 249, 247, 250, 245, 257, 244, 233, 259, 248, 246, 250, 243, 252, 255, 250, 247, 233, 252, 232, 252, 271, 229, 267, 271, 244, 275, 248, 231, 242, 242, 288, 266, 274, 257, 266, 262, 266, 256, 258, 238, 258, 258, 262, 278, 263, 262, 280, 256, 225, 236] payload_rows_per_dst=[5550, 5554, 5402, 5430, 5460, 5354, 5428, 5490] dup_rows_per_dst=[2818, 2782, 2654, 2662, 2870, 2574, 2750, 2758] combine_rows_per_src=[5550, 5554, 5402, 5430, 5460, 5354, 5428, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38260 dup_rows=21810 dup_rows_remote=19200 dedup_ratio=0.332794 router_tokens_per_expert=[264, 261, 247, 245, 228, 254, 289, 255, 242, 289, 259, 241, 264, 279, 229, 235, 240, 278, 243, 277, 248, 253, 273, 258, 259, 253, 258, 256, 265, 261, 277, 241, 228, 276, 250, 259, 248, 284, 263, 265, 253, 256, 246, 228, 264, 262, 253, 251, 272, 257, 262, 239, 266, 251, 266, 242, 258, 241, 255, 249, 266, 274, 267, 275, 258, 246, 264, 271, 246, 264, 238, 286, 226, 272, 263, 243, 229, 244, 247, 252, 251, 262, 243, 226, 272, 273, 269, 230, 284, 254, 272, 262, 282, 264, 265, 237, 254, 287, 251, 236, 250, 239, 245, 262, 250, 242, 254, 254, 253, 250, 244, 267, 270, 258, 253, 252, 275, 232, 216, 264, 280, 248, 267, 260, 248, 254, 249, 262] payload_rows_per_dst=[5422, 5494, 5438, 5540, 5436, 5446, 5466, 5484] dup_rows_per_dst=[2740, 2786, 2734, 2740, 2662, 2846, 2610, 2692] combine_rows_per_src=[5422, 5494, 5438, 5540, 5436, 5446, 5466, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38200 dup_rows=21948 dup_rows_remote=19238 dedup_ratio=0.334900 router_tokens_per_expert=[251, 248, 245, 243, 239, 252, 264, 269, 257, 244, 252, 248, 266, 268, 246, 277, 243, 253, 220, 269, 271, 279, 287, 246, 241, 233, 226, 246, 271, 274, 241, 247, 267, 251, 243, 259, 241, 261, 253, 262, 245, 241, 270, 234, 262, 284, 234, 249, 258, 246, 269, 241, 261, 266, 242, 268, 245, 251, 270, 265, 249, 255, 251, 262, 265, 278, 270, 285, 243, 258, 247, 283, 245, 242, 258, 260, 243, 227, 239, 246, 288, 257, 245, 255, 254, 255, 246, 252, 245, 234, 243, 241, 271, 247, 240, 276, 267, 258, 242, 275, 257, 237, 262, 274, 251, 255, 296, 296, 255, 253, 254, 274, 265, 268, 281, 240, 292, 253, 233, 265, 241, 274, 220, 289, 268, 240, 271, 253] payload_rows_per_dst=[5470, 5440, 5358, 5428, 5392, 5388, 5534, 5578] dup_rows_per_dst=[2668, 2654, 2754, 2770, 2786, 2710, 2878, 2728] combine_rows_per_src=[5470, 5440, 5358, 5428, 5392, 5388, 5534, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229440 combine_payload_bytes=157253632 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157483072 ingress_bytes=195002368 link_bytes=195002368 token_bytes=314736704 weight_movement_bytes=37748736 total_nvlink_bytes=352485440 weight_to_token_ratio=0.119938 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868416743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229272 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483480 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737688 weight_movement_bytes=18874368 total_nvlink_bytes=331612056 weight_to_token_ratio=0.060352 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229440 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157671488 ingress_bytes=214065152 link_bytes=214065152 token_bytes=315113536 weight_movement_bytes=56623104 total_nvlink_bytes=371736640 weight_to_token_ratio=0.179691 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051066960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229496 combine_payload_bytes=156131328 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156360824 ingress_bytes=193880064 link_bytes=193880064 token_bytes=312492152 weight_movement_bytes=37748736 total_nvlink_bytes=350240888 weight_to_token_ratio=0.120799 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857663378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=228824 combine_payload_bytes=156499968 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156728792 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313228760 weight_movement_bytes=18874368 total_nvlink_bytes=332103128 weight_to_token_ratio=0.060257 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229512 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483720 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737928 weight_movement_bytes=18874368 total_nvlink_bytes=331612296 weight_to_token_ratio=0.060352 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229840 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156942800 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655760 weight_movement_bytes=56623104 total_nvlink_bytes=370278864 weight_to_token_ratio=0.180526 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229752 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156696952 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313164152 weight_movement_bytes=18874368 total_nvlink_bytes=332038520 weight_to_token_ratio=0.060270 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44046 payload_rows_remote=38514 dup_rows=21490 dup_rows_remote=18734 dedup_ratio=0.327911 router_tokens_per_expert=[276, 223, 225, 273, 230, 269, 283, 211, 262, 241, 248, 243, 261, 264, 254, 235, 257, 282, 264, 246, 270, 256, 265, 256, 256, 268, 262, 261, 219, 244, 275, 263, 270, 245, 240, 276, 282, 246, 277, 225, 281, 255, 286, 238, 256, 242, 225, 272, 251, 257, 257, 244, 284, 260, 272, 268, 270, 264, 287, 227, 277, 254, 273, 236, 292, 259, 255, 235, 255, 243, 238, 209, 253, 244, 250, 287, 260, 253, 268, 264, 255, 266, 241, 287, 265, 266, 254, 249, 238, 227, 239, 259, 260, 249, 241, 262, 234, 240, 253, 256, 272, 237, 265, 260, 242, 274, 261, 259, 279, 280, 253, 281, 238, 240, 245, 275, 259, 269, 250, 248, 220, 263, 262, 236, 273, 247, 266, 269] payload_rows_per_dst=[5468, 5532, 5548, 5494, 5536, 5492, 5572, 5404] dup_rows_per_dst=[2528, 2756, 2684, 2868, 2618, 2624, 2696, 2716] combine_rows_per_src=[5468, 5532, 5548, 5494, 5536, 5492, 5572, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38332 dup_rows=21804 dup_rows_remote=19280 dedup_ratio=0.332703 router_tokens_per_expert=[254, 279, 254, 251, 236, 258, 264, 274, 254, 269, 274, 220, 235, 255, 243, 270, 248, 253, 224, 243, 227, 277, 257, 257, 254, 259, 252, 257, 276, 247, 272, 270, 258, 266, 264, 255, 240, 271, 245, 246, 230, 247, 234, 215, 245, 251, 257, 238, 234, 255, 277, 262, 285, 268, 278, 249, 250, 272, 264, 259, 267, 264, 289, 253, 274, 290, 261, 243, 249, 266, 251, 259, 257, 248, 265, 255, 274, 256, 260, 239, 270, 249, 274, 258, 238, 247, 245, 248, 239, 263, 273, 284, 263, 261, 247, 266, 275, 297, 259, 232, 249, 238, 258, 250, 288, 268, 275, 261, 277, 264, 263, 239, 245, 247, 237, 264, 215, 259, 227, 257, 239, 260, 253, 265, 261, 235, 244, 244] payload_rows_per_dst=[5486, 5404, 5400, 5562, 5532, 5472, 5532, 5344] dup_rows_per_dst=[2694, 2742, 2524, 2890, 2762, 2778, 2854, 2560] combine_rows_per_src=[5486, 5404, 5400, 5562, 5532, 5472, 5532, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43462 payload_rows_remote=37966 dup_rows=22074 dup_rows_remote=19362 dedup_ratio=0.336823 router_tokens_per_expert=[247, 274, 275, 245, 253, 258, 256, 272, 268, 234, 235, 244, 255, 248, 250, 270, 237, 250, 272, 254, 242, 265, 262, 269, 236, 253, 263, 252, 266, 277, 270, 269, 270, 253, 251, 266, 257, 242, 245, 247, 249, 248, 273, 237, 258, 255, 264, 247, 256, 259, 239, 272, 256, 291, 257, 251, 226, 267, 273, 270, 273, 271, 256, 290, 232, 244, 255, 259, 280, 265, 262, 263, 270, 248, 254, 233, 262, 222, 265, 241, 258, 272, 262, 241, 309, 276, 247, 229, 259, 259, 270, 239, 256, 237, 261, 274, 258, 254, 242, 255, 264, 252, 231, 239, 237, 263, 241, 264, 267, 261, 264, 274, 265, 265, 279, 244, 245, 229, 236, 253, 235, 229, 265, 253, 245, 246, 254, 265] payload_rows_per_dst=[5464, 5474, 5362, 5490, 5402, 5426, 5348, 5496] dup_rows_per_dst=[2704, 2800, 2762, 2924, 2708, 2860, 2604, 2712] combine_rows_per_src=[5464, 5474, 5362, 5490, 5402, 5426, 5348, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38260 dup_rows=21824 dup_rows_remote=19126 dedup_ratio=0.333008 router_tokens_per_expert=[285, 260, 232, 276, 269, 247, 247, 288, 271, 274, 255, 264, 273, 249, 251, 237, 267, 265, 291, 233, 250, 238, 231, 265, 247, 271, 236, 252, 246, 244, 293, 266, 263, 252, 265, 234, 269, 276, 259, 232, 248, 247, 258, 244, 243, 244, 247, 240, 259, 268, 277, 268, 267, 247, 245, 265, 264, 241, 262, 264, 258, 266, 275, 247, 277, 272, 224, 230, 251, 253, 260, 282, 244, 255, 249, 250, 265, 228, 251, 275, 247, 252, 241, 249, 262, 243, 254, 254, 246, 257, 294, 252, 251, 276, 258, 255, 237, 249, 261, 240, 226, 277, 247, 277, 257, 260, 259, 268, 253, 265, 249, 250, 235, 223, 253, 244, 249, 282, 237, 265, 258, 280, 233, 240, 283, 250, 249, 288] payload_rows_per_dst=[5612, 5448, 5430, 5500, 5398, 5488, 5452, 5384] dup_rows_per_dst=[2744, 2742, 2612, 2846, 2734, 2694, 2698, 2754] combine_rows_per_src=[5612, 5448, 5430, 5500, 5398, 5488, 5452, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38244 dup_rows=21864 dup_rows_remote=19040 dedup_ratio=0.333618 router_tokens_per_expert=[254, 249, 233, 249, 280, 273, 293, 258, 261, 276, 274, 239, 257, 253, 246, 238, 258, 235, 288, 237, 250, 252, 235, 283, 276, 264, 252, 253, 249, 267, 239, 249, 229, 278, 265, 256, 281, 251, 226, 246, 222, 257, 275, 269, 261, 244, 242, 256, 260, 249, 240, 254, 228, 247, 296, 240, 266, 278, 243, 274, 241, 263, 276, 271, 230, 286, 247, 270, 236, 253, 263, 240, 251, 259, 241, 243, 222, 246, 216, 258, 234, 261, 270, 253, 264, 257, 260, 266, 242, 251, 236, 235, 276, 248, 264, 261, 252, 263, 240, 253, 235, 252, 264, 259, 268, 256, 252, 250, 271, 261, 265, 256, 274, 285, 263, 265, 246, 279, 274, 250, 304, 239, 249, 283, 249, 237, 273, 258] payload_rows_per_dst=[5510, 5484, 5396, 5428, 5320, 5460, 5442, 5632] dup_rows_per_dst=[2756, 2690, 2720, 2824, 2602, 2696, 2752, 2824] combine_rows_per_src=[5510, 5484, 5396, 5428, 5320, 5460, 5442, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38352 dup_rows=21624 dup_rows_remote=18940 dedup_ratio=0.329956 router_tokens_per_expert=[269, 254, 243, 240, 269, 293, 255, 251, 247, 260, 219, 240, 249, 252, 260, 235, 236, 254, 245, 249, 265, 261, 226, 257, 252, 248, 261, 263, 233, 245, 273, 268, 269, 240, 253, 273, 275, 272, 255, 244, 230, 227, 255, 281, 243, 280, 242, 265, 287, 266, 243, 248, 262, 274, 276, 241, 250, 242, 233, 252, 222, 257, 243, 263, 238, 245, 291, 269, 277, 233, 255, 250, 260, 253, 271, 263, 257, 252, 251, 257, 258, 250, 266, 252, 228, 252, 231, 261, 256, 269, 257, 252, 268, 266, 244, 299, 243, 285, 262, 268, 254, 268, 244, 247, 266, 261, 261, 263, 276, 268, 273, 260, 295, 278, 248, 230, 259, 261, 233, 255, 265, 241, 262, 265, 244, 255, 252, 260] payload_rows_per_dst=[5396, 5454, 5486, 5492, 5560, 5530, 5526, 5468] dup_rows_per_dst=[2676, 2618, 2722, 2626, 2684, 2688, 2872, 2738] combine_rows_per_src=[5396, 5454, 5486, 5492, 5560, 5530, 5526, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38194 dup_rows=21796 dup_rows_remote=18902 dedup_ratio=0.332581 router_tokens_per_expert=[250, 227, 249, 268, 258, 245, 227, 244, 247, 258, 279, 231, 245, 249, 276, 262, 248, 206, 284, 247, 267, 252, 259, 267, 263, 264, 260, 256, 268, 273, 260, 262, 271, 236, 261, 270, 237, 288, 277, 273, 261, 219, 271, 243, 276, 276, 257, 249, 227, 240, 269, 251, 275, 270, 249, 243, 246, 249, 251, 238, 225, 267, 251, 253, 237, 247, 253, 254, 249, 249, 257, 209, 242, 232, 266, 246, 262, 274, 266, 241, 255, 243, 255, 259, 267, 246, 282, 264, 271, 270, 267, 250, 277, 271, 256, 287, 273, 257, 275, 239, 264, 279, 249, 241, 242, 289, 267, 264, 245, 246, 251, 233, 251, 260, 253, 258, 274, 261, 253, 264, 267, 244, 263, 250, 260, 255, 263, 254] payload_rows_per_dst=[5432, 5510, 5568, 5414, 5372, 5546, 5446, 5452] dup_rows_per_dst=[2598, 2762, 2762, 2594, 2596, 2894, 2782, 2808] combine_rows_per_src=[5432, 5510, 5568, 5414, 5372, 5546, 5446, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157753344 dispatch_meta_bytes=228992 combine_payload_bytes=157753344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157982336 ingress_bytes=176627712 link_bytes=176627712 token_bytes=315735680 weight_movement_bytes=18874368 total_nvlink_bytes=334610048 weight_to_token_ratio=0.059779 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692359830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38118 dup_rows=21960 dup_rows_remote=19166 dedup_ratio=0.335083 router_tokens_per_expert=[246, 251, 274, 256, 258, 239, 269, 253, 270, 261, 233, 246, 272, 271, 266, 246, 244, 257, 229, 277, 231, 256, 226, 247, 261, 250, 227, 257, 261, 253, 244, 257, 252, 253, 261, 251, 273, 238, 278, 259, 262, 252, 251, 252, 227, 266, 270, 273, 260, 233, 252, 237, 244, 248, 270, 277, 242, 278, 259, 261, 263, 263, 239, 268, 285, 263, 238, 264, 262, 263, 263, 277, 263, 270, 279, 238, 256, 254, 269, 272, 243, 244, 249, 230, 252, 247, 274, 254, 258, 269, 286, 279, 272, 240, 251, 269, 259, 288, 247, 249, 239, 247, 246, 238, 244, 251, 276, 223, 276, 261, 256, 266, 266, 289, 259, 254, 258, 254, 257, 239, 230, 245, 259, 237, 271, 260, 241, 250] payload_rows_per_dst=[5458, 5538, 5444, 5686, 5156, 5406, 5428, 5460] dup_rows_per_dst=[2794, 2792, 2792, 3046, 2356, 2828, 2704, 2648] combine_rows_per_src=[5458, 5538, 5444, 5686, 5156, 5406, 5428, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=230448 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157238320 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314246192 weight_movement_bytes=18874368 total_nvlink_bytes=333120560 weight_to_token_ratio=0.060062 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155508736 dispatch_meta_bytes=229312 combine_payload_bytes=155508736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155738048 ingress_bytes=174383104 link_bytes=174383104 token_bytes=311246784 weight_movement_bytes=18874368 total_nvlink_bytes=330121152 weight_to_token_ratio=0.060641 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670853101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229544 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156942504 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655464 weight_movement_bytes=56623104 total_nvlink_bytes=370278568 weight_to_token_ratio=0.180526 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229136 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156876560 ingress_bytes=213270528 link_bytes=213270528 token_bytes=313523984 weight_movement_bytes=56623104 total_nvlink_bytes=370147088 weight_to_token_ratio=0.180602 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043453264e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229168 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157318960 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314408752 weight_movement_bytes=18874368 total_nvlink_bytes=333283120 weight_to_token_ratio=0.060031 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228384 combine_payload_bytes=156442624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156671008 ingress_bytes=213065728 link_bytes=213065728 token_bytes=313113632 weight_movement_bytes=56623104 total_nvlink_bytes=369736736 weight_to_token_ratio=0.180839 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041490971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229136 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156360464 ingress_bytes=212754432 link_bytes=212754432 token_bytes=312491792 weight_movement_bytes=56623104 total_nvlink_bytes=369114896 weight_to_token_ratio=0.181199 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038508286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38316 dup_rows=21736 dup_rows_remote=19018 dedup_ratio=0.331665 router_tokens_per_expert=[271, 240, 283, 256, 256, 262, 265, 237, 254, 256, 234, 238, 276, 265, 244, 259, 259, 274, 269, 239, 260, 235, 256, 223, 269, 271, 258, 277, 257, 244, 257, 253, 238, 244, 243, 231, 266, 230, 249, 268, 274, 238, 255, 244, 266, 259, 269, 282, 261, 273, 259, 259, 266, 277, 264, 267, 256, 242, 263, 251, 256, 235, 267, 256, 266, 259, 250, 255, 245, 248, 249, 260, 258, 247, 263, 253, 273, 273, 288, 272, 233, 247, 265, 269, 252, 252, 283, 257, 250, 267, 256, 246, 242, 261, 273, 265, 275, 253, 266, 252, 259, 243, 249, 262, 263, 243, 243, 240, 268, 230, 283, 243, 253, 228, 247, 248, 269, 232, 246, 238, 258, 255, 264, 248, 240, 258, 269, 261] payload_rows_per_dst=[5484, 5484, 5442, 5416, 5572, 5506, 5470, 5426] dup_rows_per_dst=[2708, 2718, 2670, 2888, 2746, 2730, 2674, 2602] combine_rows_per_src=[5484, 5484, 5442, 5416, 5572, 5506, 5470, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43524 payload_rows_remote=37974 dup_rows=22012 dup_rows_remote=19240 dedup_ratio=0.335876 router_tokens_per_expert=[280, 262, 244, 249, 256, 253, 267, 257, 253, 260, 244, 257, 241, 245, 221, 245, 256, 264, 266, 245, 231, 253, 239, 250, 265, 257, 277, 254, 262, 234, 298, 241, 234, 275, 254, 246, 265, 260, 268, 270, 249, 266, 265, 275, 250, 252, 260, 272, 264, 288, 244, 270, 289, 248, 281, 281, 261, 258, 269, 261, 259, 256, 262, 269, 273, 230, 243, 250, 259, 271, 250, 245, 267, 264, 270, 271, 247, 253, 263, 237, 241, 228, 269, 234, 240, 252, 239, 242, 268, 245, 256, 235, 270, 261, 255, 227, 270, 240, 253, 254, 255, 244, 235, 268, 242, 273, 283, 259, 236, 255, 260, 265, 273, 238, 247, 266, 254, 276, 226, 245, 266, 260, 268, 273, 220, 270, 250, 242] payload_rows_per_dst=[5364, 5512, 5550, 5596, 5382, 5324, 5394, 5402] dup_rows_per_dst=[2704, 2672, 2772, 2924, 2804, 2600, 2790, 2746] combine_rows_per_src=[5364, 5512, 5550, 5596, 5382, 5324, 5394, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=37992 dup_rows=21838 dup_rows_remote=18736 dedup_ratio=0.333221 router_tokens_per_expert=[270, 263, 272, 256, 262, 243, 243, 264, 258, 252, 271, 263, 260, 254, 252, 263, 240, 244, 234, 263, 255, 276, 255, 258, 222, 254, 280, 285, 251, 232, 278, 271, 229, 242, 247, 263, 268, 274, 260, 269, 263, 268, 260, 252, 274, 233, 276, 259, 232, 252, 235, 256, 255, 260, 246, 236, 267, 268, 250, 237, 261, 244, 250, 241, 277, 244, 262, 266, 239, 258, 249, 261, 269, 272, 271, 276, 249, 287, 249, 285, 226, 253, 250, 264, 276, 271, 251, 262, 229, 243, 259, 259, 266, 261, 238, 240, 267, 273, 258, 230, 216, 255, 247, 252, 263, 241, 256, 261, 252, 226, 278, 241, 265, 249, 242, 278, 232, 265, 248, 265, 271, 241, 248, 268, 260, 262, 259, 266] payload_rows_per_dst=[5366, 5336, 5232, 5390, 5552, 5494, 5622, 5706] dup_rows_per_dst=[2582, 2610, 2504, 2570, 2876, 2646, 2948, 3102] combine_rows_per_src=[5366, 5336, 5232, 5390, 5552, 5494, 5622, 5706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38336 dup_rows=21778 dup_rows_remote=19112 dedup_ratio=0.332306 router_tokens_per_expert=[257, 241, 244, 258, 238, 274, 270, 258, 228, 287, 258, 273, 262, 262, 250, 293, 256, 268, 253, 244, 248, 262, 253, 269, 280, 266, 250, 269, 262, 258, 253, 244, 267, 251, 273, 270, 246, 233, 280, 274, 272, 240, 250, 267, 266, 238, 261, 288, 261, 249, 250, 243, 251, 269, 250, 238, 257, 238, 269, 261, 228, 254, 265, 261, 256, 272, 252, 264, 286, 232, 256, 290, 266, 229, 228, 252, 266, 267, 265, 267, 258, 263, 274, 252, 249, 250, 240, 243, 265, 259, 275, 275, 255, 274, 247, 261, 268, 239, 228, 249, 277, 231, 235, 241, 232, 252, 226, 240, 275, 272, 246, 248, 242, 275, 243, 253, 287, 231, 250, 262, 239, 251, 240, 252, 229, 272, 238, 249] payload_rows_per_dst=[5518, 5504, 5558, 5422, 5492, 5538, 5354, 5372] dup_rows_per_dst=[2788, 2766, 2794, 2666, 2804, 2742, 2564, 2654] combine_rows_per_src=[5518, 5504, 5558, 5422, 5492, 5538, 5354, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38354 dup_rows=21684 dup_rows_remote=18802 dedup_ratio=0.330872 router_tokens_per_expert=[275, 266, 251, 264, 255, 276, 256, 228, 259, 259, 265, 244, 264, 250, 261, 264, 274, 274, 267, 244, 251, 254, 272, 260, 293, 258, 273, 255, 259, 253, 255, 240, 255, 249, 269, 266, 239, 258, 241, 294, 255, 250, 262, 256, 247, 252, 267, 252, 250, 279, 268, 250, 249, 259, 262, 259, 253, 268, 275, 270, 249, 226, 282, 274, 262, 240, 266, 234, 244, 279, 230, 238, 253, 244, 248, 257, 241, 245, 234, 247, 243, 261, 281, 239, 269, 276, 216, 272, 271, 248, 252, 257, 233, 247, 296, 265, 247, 257, 242, 241, 254, 240, 270, 260, 248, 235, 260, 263, 252, 250, 245, 230, 263, 266, 247, 270, 255, 274, 257, 244, 250, 253, 264, 215, 264, 269, 272, 219] payload_rows_per_dst=[5556, 5480, 5316, 5606, 5382, 5468, 5498, 5546] dup_rows_per_dst=[2718, 2678, 2516, 2740, 2542, 2784, 2882, 2824] combine_rows_per_src=[5556, 5480, 5316, 5606, 5382, 5468, 5498, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38260 dup_rows=21854 dup_rows_remote=19140 dedup_ratio=0.333466 router_tokens_per_expert=[258, 272, 245, 265, 245, 273, 269, 274, 280, 247, 266, 259, 246, 236, 264, 278, 255, 276, 252, 248, 251, 281, 263, 279, 242, 239, 239, 256, 250, 277, 229, 227, 261, 269, 258, 260, 258, 269, 245, 255, 250, 254, 280, 258, 258, 264, 259, 242, 267, 251, 283, 246, 242, 254, 254, 255, 253, 232, 270, 253, 275, 270, 247, 253, 226, 255, 264, 265, 274, 263, 278, 235, 249, 242, 251, 293, 252, 231, 242, 248, 240, 245, 265, 252, 239, 262, 253, 277, 266, 273, 235, 270, 268, 230, 240, 234, 243, 265, 284, 261, 221, 264, 265, 254, 254, 261, 256, 242, 234, 240, 267, 270, 279, 233, 263, 252, 267, 272, 265, 237, 251, 236, 247, 252, 240, 258, 261, 271] payload_rows_per_dst=[5492, 5430, 5572, 5444, 5422, 5368, 5398, 5556] dup_rows_per_dst=[2862, 2698, 2708, 2766, 2714, 2730, 2764, 2612] combine_rows_per_src=[5492, 5430, 5572, 5444, 5422, 5368, 5398, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229336 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157171672 ingress_bytes=194691072 link_bytes=194691072 token_bytes=314114008 weight_movement_bytes=37748736 total_nvlink_bytes=351862744 weight_to_token_ratio=0.120175 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865434058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38216 dup_rows=21886 dup_rows_remote=19100 dedup_ratio=0.333954 router_tokens_per_expert=[263, 275, 256, 286, 250, 242, 255, 231, 270, 249, 279, 259, 266, 267, 241, 273, 250, 262, 247, 295, 283, 246, 246, 293, 251, 246, 271, 239, 261, 272, 254, 223, 241, 270, 264, 260, 261, 251, 275, 253, 254, 277, 260, 274, 270, 256, 234, 261, 278, 267, 223, 278, 240, 280, 228, 249, 245, 258, 255, 245, 247, 252, 284, 233, 251, 256, 270, 258, 260, 257, 277, 267, 252, 252, 268, 244, 283, 252, 241, 273, 238, 248, 245, 291, 241, 269, 256, 257, 258, 260, 248, 272, 258, 256, 257, 256, 246, 209, 273, 281, 255, 268, 249, 241, 310, 227, 249, 222, 262, 272, 234, 207, 214, 246, 227, 232, 242, 246, 230, 255, 265, 258, 254, 277, 261, 254, 245, 262] payload_rows_per_dst=[5490, 5512, 5468, 5444, 5488, 5434, 5420, 5394] dup_rows_per_dst=[2834, 2766, 2854, 2680, 2834, 2786, 2590, 2542] combine_rows_per_src=[5490, 5512, 5468, 5444, 5488, 5434, 5420, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155615232 dispatch_meta_bytes=226912 combine_payload_bytes=155615232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155842144 ingress_bytes=174489600 link_bytes=174489600 token_bytes=311457376 weight_movement_bytes=18874368 total_nvlink_bytes=330331744 weight_to_token_ratio=0.060600 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671873493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155541504 dispatch_meta_bytes=228856 combine_payload_bytes=155541504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155770360 ingress_bytes=193290240 link_bytes=193290240 token_bytes=311311864 weight_movement_bytes=37748736 total_nvlink_bytes=349060600 weight_to_token_ratio=0.121257 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852011975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229792 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157254048 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314278304 weight_movement_bytes=37748736 total_nvlink_bytes=352027040 weight_to_token_ratio=0.120112 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=228624 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157326608 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314424592 weight_movement_bytes=18874368 total_nvlink_bytes=333298960 weight_to_token_ratio=0.060028 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38254 dup_rows=21754 dup_rows_remote=19074 dedup_ratio=0.331940 router_tokens_per_expert=[247, 250, 274, 273, 280, 215, 252, 257, 247, 229, 269, 249, 272, 265, 267, 255, 242, 260, 281, 272, 264, 279, 228, 273, 275, 264, 272, 265, 256, 250, 237, 258, 266, 271, 239, 278, 270, 258, 243, 244, 226, 232, 237, 268, 286, 275, 264, 253, 239, 278, 234, 271, 258, 258, 236, 244, 244, 235, 268, 237, 278, 246, 239, 264, 259, 273, 273, 258, 281, 240, 263, 259, 220, 274, 244, 242, 255, 259, 238, 286, 260, 264, 274, 258, 229, 241, 250, 252, 273, 249, 263, 238, 251, 243, 238, 261, 247, 262, 252, 271, 231, 235, 253, 242, 257, 274, 271, 238, 272, 242, 261, 273, 267, 243, 250, 245, 268, 270, 279, 280, 236, 257, 250, 248, 243, 231, 252, 284] payload_rows_per_dst=[5528, 5496, 5532, 5438, 5590, 5518, 5296, 5384] dup_rows_per_dst=[2680, 2850, 2876, 2620, 2916, 2790, 2444, 2578] combine_rows_per_src=[5528, 5496, 5532, 5438, 5590, 5518, 5296, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229600 combine_payload_bytes=156712960 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156942560 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655520 weight_movement_bytes=37748736 total_nvlink_bytes=351404256 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229264 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762000 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294736 weight_movement_bytes=18874368 total_nvlink_bytes=332169104 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229312 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156917696 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606080 weight_movement_bytes=37748736 total_nvlink_bytes=351354816 weight_to_token_ratio=0.120370 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43502 payload_rows_remote=38138 dup_rows=22034 dup_rows_remote=19368 dedup_ratio=0.336212 router_tokens_per_expert=[260, 246, 258, 239, 247, 262, 262, 274, 255, 252, 307, 252, 258, 250, 220, 269, 264, 258, 258, 246, 260, 218, 265, 238, 242, 264, 253, 226, 254, 240, 262, 267, 257, 256, 273, 252, 219, 231, 267, 260, 247, 268, 252, 264, 223, 245, 270, 247, 245, 287, 248, 259, 275, 248, 264, 291, 234, 267, 261, 245, 252, 274, 253, 276, 285, 279, 308, 265, 250, 269, 258, 269, 277, 271, 241, 245, 240, 268, 251, 243, 283, 237, 251, 277, 243, 237, 267, 247, 231, 275, 296, 237, 270, 237, 279, 266, 264, 231, 242, 237, 265, 250, 257, 270, 248, 222, 266, 257, 266, 251, 241, 250, 273, 270, 255, 242, 225, 239, 240, 255, 258, 255, 277, 233, 266, 263, 253, 259] payload_rows_per_dst=[5426, 5364, 5342, 5494, 5532, 5498, 5442, 5404] dup_rows_per_dst=[2796, 2666, 2720, 2864, 2906, 2768, 2592, 2722] combine_rows_per_src=[5426, 5364, 5342, 5494, 5532, 5498, 5442, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43526 payload_rows_remote=38196 dup_rows=22010 dup_rows_remote=19382 dedup_ratio=0.335846 router_tokens_per_expert=[246, 277, 252, 251, 246, 284, 285, 235, 293, 273, 253, 275, 271, 260, 250, 239, 258, 251, 257, 241, 239, 276, 265, 253, 265, 269, 261, 216, 233, 248, 272, 237, 251, 250, 273, 252, 233, 256, 240, 253, 241, 241, 248, 276, 268, 222, 256, 219, 266, 252, 252, 260, 261, 256, 232, 262, 287, 262, 219, 262, 257, 227, 242, 273, 256, 244, 270, 251, 253, 248, 276, 249, 246, 264, 268, 220, 240, 273, 228, 236, 276, 242, 278, 264, 258, 243, 272, 235, 267, 258, 260, 282, 251, 262, 259, 269, 247, 272, 258, 254, 251, 269, 238, 243, 259, 277, 258, 278, 256, 250, 247, 267, 272, 264, 262, 266, 255, 275, 240, 266, 266, 240, 258, 242, 269, 288, 235, 268] payload_rows_per_dst=[5504, 5452, 5330, 5410, 5410, 5474, 5478, 5468] dup_rows_per_dst=[2876, 2630, 2628, 2730, 2634, 2878, 2770, 2864] combine_rows_per_src=[5504, 5452, 5330, 5410, 5410, 5474, 5478, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38300 dup_rows=21776 dup_rows_remote=19000 dedup_ratio=0.332275 router_tokens_per_expert=[260, 245, 276, 275, 297, 239, 238, 253, 256, 249, 277, 245, 280, 251, 267, 257, 259, 262, 228, 245, 286, 265, 238, 241, 250, 286, 232, 249, 239, 255, 234, 246, 245, 261, 266, 275, 244, 289, 258, 280, 285, 277, 250, 250, 275, 255, 240, 218, 266, 270, 259, 250, 240, 243, 273, 264, 262, 280, 235, 254, 260, 255, 279, 262, 240, 249, 261, 244, 248, 260, 259, 258, 262, 249, 280, 241, 227, 273, 262, 241, 253, 249, 265, 232, 256, 319, 226, 277, 237, 291, 245, 241, 255, 275, 258, 238, 244, 231, 262, 262, 266, 224, 269, 254, 240, 248, 264, 263, 256, 269, 240, 247, 255, 260, 220, 283, 250, 263, 251, 269, 269, 276, 261, 238, 257, 238, 236, 232] payload_rows_per_dst=[5390, 5524, 5522, 5620, 5386, 5448, 5410, 5460] dup_rows_per_dst=[2618, 2828, 2814, 2684, 2602, 2786, 2668, 2776] combine_rows_per_src=[5390, 5524, 5522, 5620, 5386, 5448, 5410, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38066 dup_rows=21944 dup_rows_remote=19132 dedup_ratio=0.334839 router_tokens_per_expert=[247, 233, 242, 244, 242, 253, 247, 247, 216, 248, 276, 271, 269, 217, 231, 249, 265, 254, 237, 266, 273, 265, 235, 236, 270, 236, 244, 269, 283, 269, 264, 256, 264, 241, 256, 240, 240, 250, 238, 245, 264, 256, 244, 283, 248, 238, 264, 249, 295, 265, 262, 280, 271, 268, 281, 252, 258, 247, 237, 227, 259, 255, 230, 282, 255, 262, 265, 267, 243, 259, 256, 240, 275, 233, 260, 259, 231, 255, 252, 236, 245, 264, 239, 272, 279, 240, 262, 257, 257, 240, 255, 224, 269, 263, 267, 251, 266, 253, 276, 236, 247, 270, 263, 223, 255, 262, 283, 254, 252, 248, 266, 294, 277, 249, 264, 276, 272, 238, 253, 239, 277, 273, 272, 284, 263, 270, 271, 267] payload_rows_per_dst=[5302, 5444, 5408, 5526, 5450, 5412, 5474, 5576] dup_rows_per_dst=[2562, 2800, 2632, 2812, 2646, 2756, 2822, 2914] combine_rows_per_src=[5302, 5444, 5408, 5526, 5450, 5412, 5474, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38052 dup_rows=21992 dup_rows_remote=19170 dedup_ratio=0.335571 router_tokens_per_expert=[264, 273, 245, 227, 283, 252, 255, 229, 280, 258, 242, 273, 241, 261, 246, 267, 235, 238, 255, 248, 257, 253, 254, 258, 241, 262, 262, 283, 252, 248, 254, 255, 230, 253, 250, 269, 271, 268, 254, 264, 248, 263, 274, 240, 222, 280, 243, 258, 264, 251, 238, 238, 250, 226, 252, 241, 227, 269, 231, 235, 290, 240, 252, 267, 265, 268, 241, 288, 249, 259, 264, 253, 257, 255, 238, 277, 252, 268, 244, 281, 250, 287, 260, 255, 243, 247, 257, 271, 239, 259, 264, 294, 275, 244, 260, 240, 237, 282, 261, 250, 241, 260, 272, 250, 268, 260, 273, 271, 259, 247, 253, 273, 273, 282, 291, 263, 268, 257, 272, 245, 234, 258, 248, 228, 240, 245, 247, 247] payload_rows_per_dst=[5420, 5384, 5408, 5382, 5518, 5494, 5492, 5446] dup_rows_per_dst=[2772, 2726, 2766, 2560, 2800, 2796, 2822, 2750] combine_rows_per_src=[5420, 5384, 5408, 5382, 5518, 5494, 5492, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=230024 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156443272 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312656520 weight_movement_bytes=18874368 total_nvlink_bytes=331530888 weight_to_token_ratio=0.060368 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38318 dup_rows=21666 dup_rows_remote=18808 dedup_ratio=0.330597 router_tokens_per_expert=[221, 270, 256, 276, 264, 254, 246, 261, 252, 233, 252, 274, 279, 254, 277, 238, 270, 251, 258, 264, 293, 261, 250, 226, 253, 252, 267, 274, 255, 266, 272, 271, 242, 243, 278, 235, 247, 246, 259, 239, 244, 264, 227, 258, 267, 285, 254, 250, 242, 255, 264, 253, 268, 232, 246, 257, 241, 253, 266, 228, 231, 242, 263, 250, 260, 265, 252, 260, 259, 247, 249, 246, 255, 237, 266, 252, 259, 274, 224, 252, 258, 282, 241, 264, 304, 267, 282, 264, 233, 259, 263, 255, 246, 247, 258, 282, 248, 267, 235, 275, 260, 272, 240, 249, 245, 250, 265, 273, 259, 247, 252, 262, 243, 254, 249, 274, 236, 220, 244, 268, 273, 269, 235, 282, 250, 258, 275, 258] payload_rows_per_dst=[5534, 5548, 5410, 5398, 5474, 5552, 5452, 5502] dup_rows_per_dst=[2680, 2818, 2666, 2584, 2640, 2858, 2746, 2674] combine_rows_per_src=[5534, 5548, 5410, 5398, 5474, 5552, 5452, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38178 dup_rows=21876 dup_rows_remote=19124 dedup_ratio=0.333801 router_tokens_per_expert=[262, 252, 237, 281, 275, 256, 267, 263, 231, 263, 242, 231, 282, 258, 255, 243, 280, 245, 248, 240, 258, 243, 275, 273, 248, 251, 245, 267, 247, 268, 270, 270, 253, 253, 234, 266, 269, 255, 233, 241, 260, 305, 249, 233, 252, 271, 272, 259, 270, 249, 259, 264, 247, 236, 276, 287, 249, 234, 245, 262, 243, 243, 257, 226, 246, 258, 282, 243, 278, 279, 266, 252, 257, 274, 215, 234, 258, 262, 231, 282, 265, 241, 268, 266, 282, 233, 254, 247, 290, 252, 248, 240, 252, 291, 248, 251, 249, 260, 259, 272, 266, 242, 282, 254, 240, 250, 269, 223, 277, 262, 278, 235, 268, 267, 275, 227, 280, 243, 255, 247, 229, 255, 276, 255, 241, 214, 265, 230] payload_rows_per_dst=[5494, 5446, 5468, 5428, 5482, 5496, 5460, 5386] dup_rows_per_dst=[2702, 2810, 2742, 2666, 2752, 2760, 2776, 2668] combine_rows_per_src=[5494, 5446, 5468, 5428, 5482, 5496, 5460, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=230312 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156681128 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313131944 weight_movement_bytes=18874368 total_nvlink_bytes=332006312 weight_to_token_ratio=0.060276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229200 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157106000 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313982800 weight_movement_bytes=37748736 total_nvlink_bytes=351731536 weight_to_token_ratio=0.120225 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228792 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156147128 ingress_bytes=174792704 link_bytes=174792704 token_bytes=312065464 weight_movement_bytes=18874368 total_nvlink_bytes=330939832 weight_to_token_ratio=0.060482 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674777686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=155860992 dispatch_meta_bytes=228888 combine_payload_bytes=155860992 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156089880 ingress_bytes=174735360 link_bytes=174735360 token_bytes=311950872 weight_movement_bytes=18874368 total_nvlink_bytes=330825240 weight_to_token_ratio=0.060504 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674228244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228504 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157179032 ingress_bytes=232448000 link_bytes=232448000 token_bytes=314129560 weight_movement_bytes=75497472 total_nvlink_bytes=389627032 weight_to_token_ratio=0.240339 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227202364e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38178 dup_rows=21780 dup_rows_remote=18890 dedup_ratio=0.332336 router_tokens_per_expert=[254, 252, 232, 253, 250, 263, 261, 270, 234, 247, 254, 291, 232, 223, 252, 253, 268, 240, 236, 248, 267, 253, 259, 259, 266, 229, 215, 258, 247, 276, 253, 264, 253, 243, 247, 259, 262, 268, 270, 216, 262, 255, 250, 261, 264, 281, 251, 251, 250, 283, 255, 246, 245, 260, 290, 258, 265, 244, 279, 247, 239, 262, 254, 256, 251, 230, 255, 265, 279, 243, 235, 269, 268, 256, 260, 272, 239, 264, 251, 263, 240, 251, 247, 227, 265, 251, 241, 244, 249, 254, 259, 281, 264, 243, 250, 259, 262, 276, 272, 293, 256, 259, 263, 261, 264, 238, 287, 259, 256, 244, 277, 269, 295, 266, 281, 249, 239, 245, 250, 252, 264, 254, 218, 240, 255, 267, 263, 284] payload_rows_per_dst=[5578, 5398, 5636, 5220, 5720, 5380, 5360, 5464] dup_rows_per_dst=[2890, 2722, 2924, 2364, 2992, 2508, 2600, 2780] combine_rows_per_src=[5578, 5398, 5636, 5220, 5720, 5380, 5360, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229208 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156606296 ingress_bytes=194125824 link_bytes=194125824 token_bytes=312983384 weight_movement_bytes=37748736 total_nvlink_bytes=350732120 weight_to_token_ratio=0.120609 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860018130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38408 dup_rows=21722 dup_rows_remote=19020 dedup_ratio=0.331451 router_tokens_per_expert=[273, 243, 274, 232, 237, 268, 241, 273, 258, 261, 273, 280, 264, 281, 260, 260, 302, 246, 272, 233, 248, 244, 251, 282, 267, 258, 233, 235, 250, 232, 258, 243, 259, 243, 261, 280, 258, 245, 234, 270, 268, 253, 248, 257, 248, 293, 286, 246, 271, 256, 283, 259, 248, 245, 276, 269, 252, 284, 264, 240, 244, 241, 260, 259, 249, 228, 266, 261, 270, 277, 238, 252, 271, 269, 258, 270, 272, 244, 247, 245, 241, 281, 258, 231, 249, 255, 257, 288, 250, 239, 242, 275, 244, 260, 272, 257, 232, 266, 246, 244, 234, 246, 252, 240, 220, 285, 253, 224, 256, 267, 234, 276, 242, 248, 223, 245, 240, 244, 260, 268, 271, 257, 238, 241, 276, 252, 263, 277] payload_rows_per_dst=[5654, 5406, 5380, 5654, 5532, 5494, 5348, 5346] dup_rows_per_dst=[3030, 2702, 2590, 2648, 2702, 2704, 2602, 2744] combine_rows_per_src=[5654, 5406, 5380, 5654, 5532, 5494, 5348, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228272 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156605360 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312982448 weight_movement_bytes=18874368 total_nvlink_bytes=331856816 weight_to_token_ratio=0.060305 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38176 dup_rows=21906 dup_rows_remote=19122 dedup_ratio=0.334259 router_tokens_per_expert=[263, 270, 243, 239, 243, 254, 256, 243, 257, 249, 220, 223, 238, 253, 241, 216, 250, 272, 253, 240, 275, 256, 263, 262, 247, 281, 233, 266, 252, 249, 274, 264, 248, 257, 259, 263, 261, 243, 231, 251, 243, 277, 272, 240, 261, 288, 272, 253, 248, 278, 258, 268, 249, 244, 276, 243, 255, 261, 256, 244, 270, 222, 233, 229, 271, 280, 253, 234, 260, 263, 246, 275, 251, 245, 270, 263, 241, 253, 266, 242, 277, 274, 272, 269, 245, 240, 253, 251, 252, 258, 249, 258, 284, 246, 270, 256, 270, 255, 250, 267, 265, 223, 247, 266, 278, 271, 261, 235, 284, 270, 260, 241, 242, 271, 259, 271, 263, 272, 257, 265, 297, 262, 277, 267, 245, 247, 238, 227] payload_rows_per_dst=[5298, 5496, 5454, 5402, 5434, 5514, 5460, 5572] dup_rows_per_dst=[2518, 2778, 2784, 2666, 2792, 2794, 2826, 2748] combine_rows_per_src=[5298, 5496, 5454, 5402, 5434, 5514, 5460, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38266 dup_rows=21780 dup_rows_remote=18986 dedup_ratio=0.332336 router_tokens_per_expert=[245, 257, 258, 252, 245, 259, 234, 237, 256, 272, 231, 260, 286, 244, 241, 246, 267, 254, 259, 263, 263, 235, 235, 238, 258, 274, 254, 252, 250, 250, 278, 247, 271, 255, 273, 252, 260, 252, 281, 280, 263, 228, 270, 261, 244, 239, 265, 263, 246, 274, 259, 258, 252, 248, 265, 246, 257, 248, 300, 272, 267, 271, 251, 254, 250, 254, 275, 254, 255, 259, 216, 218, 254, 252, 235, 243, 260, 253, 276, 278, 272, 270, 242, 273, 261, 251, 254, 248, 279, 246, 270, 272, 259, 251, 239, 247, 213, 225, 245, 254, 258, 241, 266, 278, 261, 267, 266, 280, 252, 277, 257, 251, 260, 243, 264, 272, 264, 278, 231, 256, 262, 244, 257, 273, 240, 236, 259, 247] payload_rows_per_dst=[5456, 5422, 5490, 5536, 5372, 5488, 5502, 5490] dup_rows_per_dst=[2590, 2732, 2824, 2688, 2692, 2768, 2692, 2794] combine_rows_per_src=[5456, 5422, 5490, 5536, 5372, 5488, 5502, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38266 dup_rows=21738 dup_rows_remote=18960 dedup_ratio=0.331696 router_tokens_per_expert=[264, 244, 259, 287, 249, 212, 261, 233, 252, 263, 252, 278, 248, 260, 272, 247, 281, 249, 280, 267, 273, 258, 229, 242, 250, 238, 247, 271, 246, 243, 254, 236, 261, 260, 284, 259, 249, 271, 248, 244, 239, 257, 281, 283, 242, 258, 237, 238, 229, 254, 253, 240, 234, 273, 255, 279, 271, 240, 239, 234, 222, 281, 264, 257, 260, 260, 237, 287, 271, 253, 270, 280, 239, 247, 260, 239, 263, 266, 257, 258, 242, 276, 278, 259, 268, 243, 255, 245, 273, 258, 258, 273, 252, 255, 253, 241, 267, 239, 252, 262, 246, 279, 280, 257, 254, 256, 248, 271, 277, 226, 288, 253, 261, 253, 248, 236, 256, 260, 236, 267, 239, 269, 240, 246, 248, 263, 261, 273] payload_rows_per_dst=[5446, 5416, 5494, 5426, 5538, 5524, 5532, 5422] dup_rows_per_dst=[2716, 2712, 2728, 2624, 2756, 2734, 2778, 2690] combine_rows_per_src=[5446, 5416, 5494, 5426, 5538, 5524, 5532, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38220 dup_rows=21860 dup_rows_remote=19134 dedup_ratio=0.333557 router_tokens_per_expert=[234, 255, 264, 243, 267, 254, 262, 228, 249, 291, 222, 253, 256, 236, 264, 246, 265, 227, 230, 270, 234, 254, 253, 267, 251, 258, 250, 281, 228, 258, 236, 244, 269, 256, 257, 277, 270, 279, 268, 263, 255, 251, 254, 247, 263, 265, 254, 248, 258, 265, 285, 253, 234, 251, 263, 264, 257, 252, 252, 248, 269, 232, 244, 264, 247, 267, 250, 248, 254, 258, 273, 250, 276, 266, 273, 260, 243, 229, 250, 249, 262, 268, 276, 258, 237, 274, 284, 258, 231, 243, 262, 239, 257, 258, 238, 262, 268, 257, 253, 246, 271, 228, 266, 253, 255, 280, 280, 253, 263, 253, 252, 248, 251, 267, 266, 248, 250, 278, 265, 263, 276, 253, 268, 254, 265, 264, 225, 252] payload_rows_per_dst=[5438, 5410, 5494, 5456, 5394, 5492, 5498, 5494] dup_rows_per_dst=[2610, 2602, 2858, 2726, 2792, 2722, 2754, 2796] combine_rows_per_src=[5438, 5410, 5494, 5456, 5394, 5492, 5498, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38352 dup_rows=21750 dup_rows_remote=18972 dedup_ratio=0.331879 router_tokens_per_expert=[257, 278, 272, 275, 236, 282, 251, 218, 240, 252, 244, 256, 238, 279, 221, 253, 258, 244, 252, 258, 245, 240, 256, 242, 250, 260, 269, 244, 277, 250, 268, 266, 264, 284, 256, 262, 259, 257, 274, 238, 259, 253, 234, 258, 269, 292, 246, 267, 233, 256, 267, 236, 258, 267, 248, 262, 266, 244, 279, 280, 258, 255, 263, 233, 263, 243, 270, 238, 257, 265, 258, 259, 248, 247, 243, 257, 247, 260, 224, 239, 242, 239, 280, 254, 283, 218, 269, 242, 251, 278, 271, 250, 243, 262, 253, 271, 265, 272, 235, 239, 283, 260, 239, 259, 244, 267, 270, 241, 267, 265, 258, 252, 258, 261, 263, 253, 274, 255, 265, 258, 245, 263, 249, 247, 249, 260, 274, 246] payload_rows_per_dst=[5482, 5494, 5494, 5504, 5436, 5434, 5454, 5488] dup_rows_per_dst=[2622, 2664, 2850, 2706, 2600, 2778, 2778, 2752] combine_rows_per_src=[5482, 5494, 5494, 5504, 5436, 5434, 5454, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229712 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157548880 ingress_bytes=176193536 link_bytes=176193536 token_bytes=314868048 weight_movement_bytes=18874368 total_nvlink_bytes=333742416 weight_to_token_ratio=0.059944 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688199769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38364 dup_rows=21768 dup_rows_remote=19130 dedup_ratio=0.332153 router_tokens_per_expert=[236, 255, 276, 265, 234, 258, 255, 276, 250, 252, 243, 248, 289, 246, 242, 246, 234, 257, 257, 251, 253, 235, 274, 245, 274, 251, 275, 250, 263, 254, 265, 243, 239, 235, 282, 275, 253, 277, 256, 242, 258, 244, 269, 277, 250, 273, 243, 237, 280, 268, 268, 269, 219, 257, 276, 277, 282, 286, 252, 266, 273, 248, 260, 264, 269, 260, 240, 232, 243, 267, 236, 283, 250, 226, 249, 269, 250, 249, 254, 244, 269, 241, 240, 267, 275, 268, 253, 253, 260, 277, 277, 281, 236, 250, 261, 233, 241, 267, 268, 257, 271, 271, 243, 254, 266, 249, 247, 235, 264, 251, 284, 247, 219, 254, 291, 250, 242, 254, 248, 248, 254, 240, 242, 248, 239, 244, 258, 253] payload_rows_per_dst=[5484, 5450, 5498, 5582, 5404, 5480, 5460, 5410] dup_rows_per_dst=[2658, 2712, 2722, 2908, 2638, 2802, 2770, 2558] combine_rows_per_src=[5484, 5450, 5498, 5582, 5404, 5480, 5460, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229192 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156598088 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312966984 weight_movement_bytes=37748736 total_nvlink_bytes=350715720 weight_to_token_ratio=0.120616 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229008 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966544 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704080 weight_movement_bytes=18874368 total_nvlink_bytes=332578448 weight_to_token_ratio=0.060166 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228904 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156966440 ingress_bytes=194486272 link_bytes=194486272 token_bytes=313703976 weight_movement_bytes=37748736 total_nvlink_bytes=351452712 weight_to_token_ratio=0.120332 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863471765e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229416 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156778536 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313327656 weight_movement_bytes=56623104 total_nvlink_bytes=369950760 weight_to_token_ratio=0.180715 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229296 combine_payload_bytes=157089792 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157319088 ingress_bytes=213712896 link_bytes=213712896 token_bytes=314408880 weight_movement_bytes=56623104 total_nvlink_bytes=371031984 weight_to_token_ratio=0.180094 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047691816e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229976 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157368920 ingress_bytes=176013312 link_bytes=176013312 token_bytes=314507864 weight_movement_bytes=18874368 total_nvlink_bytes=333382232 weight_to_token_ratio=0.060012 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686472952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38320 dup_rows=21832 dup_rows_remote=19146 dedup_ratio=0.333130 router_tokens_per_expert=[257, 263, 243, 246, 251, 245, 251, 240, 272, 257, 259, 257, 248, 245, 252, 245, 280, 260, 260, 274, 269, 260, 254, 262, 255, 245, 257, 293, 235, 269, 261, 251, 261, 244, 245, 265, 239, 253, 258, 273, 262, 258, 266, 274, 245, 253, 247, 259, 254, 244, 238, 249, 250, 265, 241, 237, 273, 250, 232, 272, 262, 230, 238, 257, 260, 274, 248, 269, 261, 250, 257, 231, 245, 272, 262, 247, 273, 286, 255, 265, 255, 281, 282, 265, 260, 271, 253, 254, 262, 267, 261, 265, 262, 256, 230, 269, 220, 273, 252, 249, 287, 260, 263, 246, 241, 254, 229, 268, 264, 271, 247, 260, 268, 212, 257, 258, 258, 249, 263, 257, 260, 245, 245, 255, 230, 274, 264, 231] payload_rows_per_dst=[5384, 5580, 5506, 5340, 5386, 5550, 5518, 5440] dup_rows_per_dst=[2686, 2790, 3074, 2644, 2540, 2836, 2650, 2612] combine_rows_per_src=[5384, 5580, 5506, 5340, 5386, 5550, 5518, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38262 dup_rows=21710 dup_rows_remote=18942 dedup_ratio=0.331268 router_tokens_per_expert=[255, 267, 258, 282, 271, 271, 239, 254, 270, 244, 262, 271, 265, 266, 264, 242, 240, 272, 252, 245, 260, 243, 276, 256, 297, 282, 255, 267, 239, 264, 263, 255, 268, 245, 239, 255, 271, 251, 245, 242, 259, 260, 260, 257, 267, 255, 228, 265, 256, 252, 235, 253, 267, 273, 258, 237, 249, 240, 255, 255, 257, 270, 245, 227, 252, 266, 235, 268, 254, 245, 258, 246, 257, 262, 255, 277, 250, 246, 270, 243, 267, 246, 246, 239, 239, 234, 264, 269, 234, 275, 277, 247, 220, 252, 276, 229, 263, 267, 248, 282, 251, 274, 252, 235, 269, 248, 261, 243, 271, 266, 270, 252, 258, 266, 272, 247, 260, 254, 247, 254, 241, 247, 232, 262, 263, 264, 248, 260] payload_rows_per_dst=[5538, 5564, 5488, 5420, 5478, 5384, 5460, 5494] dup_rows_per_dst=[2824, 2768, 2646, 2638, 2690, 2644, 2844, 2656] combine_rows_per_src=[5538, 5564, 5488, 5420, 5478, 5384, 5460, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229864 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157188584 ingress_bytes=213581824 link_bytes=213581824 token_bytes=314147304 weight_movement_bytes=56623104 total_nvlink_bytes=370770408 weight_to_token_ratio=0.180244 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046435949e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43380 payload_rows_remote=37912 dup_rows=22156 dup_rows_remote=19242 dedup_ratio=0.338074 router_tokens_per_expert=[297, 248, 253, 246, 269, 239, 231, 244, 260, 271, 247, 259, 247, 266, 236, 255, 273, 270, 258, 245, 255, 271, 252, 259, 253, 243, 278, 239, 250, 267, 262, 252, 253, 285, 247, 268, 284, 249, 263, 260, 263, 242, 287, 233, 250, 273, 267, 267, 262, 250, 257, 277, 303, 236, 270, 255, 262, 248, 243, 265, 215, 263, 255, 221, 252, 275, 271, 254, 243, 264, 243, 262, 254, 267, 264, 251, 249, 220, 265, 248, 290, 244, 264, 263, 224, 239, 258, 253, 253, 250, 249, 271, 251, 254, 252, 252, 258, 267, 265, 230, 259, 236, 266, 267, 250, 250, 254, 266, 238, 280, 239, 258, 262, 256, 270, 245, 239, 272, 247, 238, 270, 255, 257, 231, 218, 270, 276, 262] payload_rows_per_dst=[5356, 5476, 5468, 5408, 5466, 5406, 5388, 5412] dup_rows_per_dst=[2780, 2778, 2914, 2756, 2698, 2728, 2778, 2724] combine_rows_per_src=[5356, 5476, 5468, 5408, 5466, 5406, 5388, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38516 dup_rows=21624 dup_rows_remote=18886 dedup_ratio=0.329956 router_tokens_per_expert=[256, 290, 264, 268, 254, 267, 260, 258, 278, 259, 277, 268, 248, 261, 243, 284, 240, 249, 253, 235, 275, 244, 279, 241, 240, 238, 255, 267, 267, 222, 235, 255, 225, 263, 257, 290, 233, 274, 247, 234, 248, 247, 249, 252, 237, 253, 252, 277, 262, 279, 266, 252, 237, 264, 275, 286, 250, 271, 240, 223, 279, 255, 258, 246, 264, 251, 273, 240, 245, 263, 284, 260, 249, 247, 279, 255, 263, 234, 247, 245, 234, 273, 263, 253, 290, 272, 254, 252, 257, 270, 261, 255, 246, 268, 278, 266, 290, 242, 254, 249, 238, 257, 236, 250, 255, 260, 260, 236, 274, 277, 238, 251, 264, 253, 238, 261, 249, 226, 244, 235, 257, 251, 238, 251, 250, 260, 244, 278] payload_rows_per_dst=[5636, 5422, 5452, 5516, 5482, 5568, 5396, 5440] dup_rows_per_dst=[2834, 2568, 2624, 2770, 2716, 2816, 2738, 2558] combine_rows_per_src=[5636, 5422, 5452, 5516, 5482, 5568, 5396, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43450 payload_rows_remote=37912 dup_rows=22086 dup_rows_remote=19000 dedup_ratio=0.337006 router_tokens_per_expert=[251, 268, 272, 253, 252, 263, 265, 238, 228, 278, 260, 237, 268, 255, 272, 248, 282, 264, 262, 286, 267, 264, 255, 260, 258, 266, 248, 272, 254, 225, 235, 272, 257, 272, 233, 243, 254, 261, 254, 241, 275, 250, 249, 253, 229, 247, 240, 272, 255, 249, 237, 244, 252, 266, 242, 259, 252, 250, 242, 230, 266, 255, 251, 235, 265, 266, 232, 259, 254, 235, 249, 265, 277, 233, 257, 236, 276, 253, 294, 282, 252, 259, 245, 269, 252, 246, 283, 261, 262, 308, 260, 234, 271, 238, 267, 267, 243, 272, 250, 301, 248, 244, 256, 272, 239, 265, 272, 234, 260, 222, 244, 272, 226, 271, 264, 250, 265, 240, 247, 259, 264, 248, 251, 262, 259, 251, 255, 262] payload_rows_per_dst=[5220, 5470, 5398, 5450, 5496, 5424, 5454, 5538] dup_rows_per_dst=[2520, 2614, 2662, 2776, 2770, 2886, 2772, 3086] combine_rows_per_src=[5220, 5470, 5398, 5450, 5496, 5424, 5454, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228816 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156949968 ingress_bytes=194469888 link_bytes=194469888 token_bytes=313671120 weight_movement_bytes=37748736 total_nvlink_bytes=351419856 weight_to_token_ratio=0.120345 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863314782e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38260 dup_rows=21762 dup_rows_remote=19010 dedup_ratio=0.332062 router_tokens_per_expert=[246, 250, 269, 252, 246, 238, 260, 266, 248, 247, 219, 235, 235, 262, 259, 238, 242, 266, 253, 278, 253, 251, 256, 265, 263, 243, 260, 252, 237, 239, 262, 250, 247, 267, 246, 249, 256, 258, 279, 258, 254, 262, 255, 244, 261, 265, 294, 255, 233, 239, 261, 268, 284, 268, 271, 232, 265, 253, 265, 248, 259, 255, 254, 232, 275, 270, 267, 255, 256, 262, 243, 230, 244, 239, 213, 270, 236, 242, 268, 255, 246, 221, 245, 264, 257, 258, 273, 256, 260, 256, 248, 277, 257, 273, 275, 267, 274, 256, 251, 257, 253, 240, 256, 264, 243, 297, 285, 276, 265, 270, 241, 263, 260, 260, 262, 261, 213, 266, 241, 272, 251, 278, 267, 256, 285, 249, 250, 271] payload_rows_per_dst=[5400, 5424, 5462, 5432, 5426, 5514, 5574, 5542] dup_rows_per_dst=[2540, 2716, 2838, 2742, 2624, 2752, 2808, 2742] combine_rows_per_src=[5400, 5424, 5462, 5432, 5426, 5514, 5574, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38160 dup_rows=21922 dup_rows_remote=19148 dedup_ratio=0.334503 router_tokens_per_expert=[263, 255, 231, 249, 245, 263, 254, 274, 235, 279, 274, 266, 274, 263, 229, 272, 252, 234, 290, 261, 262, 241, 287, 276, 242, 246, 254, 238, 246, 263, 265, 249, 266, 266, 255, 254, 263, 254, 242, 265, 272, 267, 234, 252, 223, 255, 227, 236, 278, 282, 272, 245, 253, 280, 249, 253, 249, 247, 222, 247, 234, 264, 266, 273, 273, 294, 245, 241, 246, 257, 252, 270, 231, 287, 274, 279, 246, 256, 225, 264, 251, 235, 230, 260, 237, 207, 226, 265, 238, 279, 254, 236, 279, 255, 278, 254, 222, 241, 271, 263, 255, 251, 243, 260, 244, 254, 241, 245, 270, 284, 266, 257, 279, 277, 260, 246, 262, 258, 256, 262, 259, 281, 228, 265, 247, 268, 294, 258] payload_rows_per_dst=[5570, 5488, 5384, 5454, 5458, 5338, 5396, 5526] dup_rows_per_dst=[2682, 2724, 2678, 2774, 2822, 2630, 2738, 2874] combine_rows_per_src=[5570, 5488, 5384, 5454, 5458, 5338, 5396, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38294 dup_rows=21814 dup_rows_remote=18986 dedup_ratio=0.332855 router_tokens_per_expert=[255, 287, 253, 280, 257, 269, 256, 252, 250, 252, 268, 245, 265, 261, 272, 230, 261, 271, 261, 281, 261, 260, 242, 238, 249, 274, 242, 244, 275, 224, 263, 243, 264, 253, 272, 264, 255, 251, 262, 245, 269, 244, 290, 250, 251, 240, 259, 245, 257, 254, 251, 277, 255, 266, 267, 234, 288, 245, 256, 247, 236, 256, 256, 266, 275, 260, 275, 255, 231, 265, 259, 292, 256, 247, 256, 234, 275, 240, 244, 264, 260, 256, 293, 223, 265, 243, 224, 258, 253, 296, 249, 257, 257, 266, 284, 275, 238, 245, 274, 247, 258, 249, 268, 281, 233, 248, 249, 228, 251, 279, 266, 261, 240, 253, 252, 274, 215, 254, 244, 274, 237, 235, 240, 260, 241, 219, 275, 227] payload_rows_per_dst=[5544, 5498, 5462, 5528, 5428, 5468, 5444, 5350] dup_rows_per_dst=[2760, 2680, 2766, 2694, 2828, 2850, 2706, 2530] combine_rows_per_src=[5544, 5498, 5462, 5528, 5428, 5468, 5444, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=155287552 dispatch_meta_bytes=228616 combine_payload_bytes=155287552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155516168 ingress_bytes=193036288 link_bytes=193036288 token_bytes=310803720 weight_movement_bytes=37748736 total_nvlink_bytes=348552456 weight_to_token_ratio=0.121455 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.849578732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157761536 dispatch_meta_bytes=229608 combine_payload_bytes=157761536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157991144 ingress_bytes=176635904 link_bytes=176635904 token_bytes=315752680 weight_movement_bytes=18874368 total_nvlink_bytes=334627048 weight_to_token_ratio=0.059776 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692438322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=155287552 dispatch_meta_bytes=227648 combine_payload_bytes=155287552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155515200 ingress_bytes=174161920 link_bytes=174161920 token_bytes=310802752 weight_movement_bytes=18874368 total_nvlink_bytes=329677120 weight_to_token_ratio=0.060728 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.668733824e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229080 combine_payload_bytes=156712960 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156942040 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655000 weight_movement_bytes=56623104 total_nvlink_bytes=370278104 weight_to_token_ratio=0.180527 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229232 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156532592 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312835952 weight_movement_bytes=18874368 total_nvlink_bytes=331710320 weight_to_token_ratio=0.060333 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229120 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=0 egress_bytes=157081344 ingress_bytes=156852224 link_bytes=157081344 token_bytes=313933568 weight_movement_bytes=0 total_nvlink_bytes=313933568 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.505076149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38180 dup_rows=21804 dup_rows_remote=18896 dedup_ratio=0.332703 router_tokens_per_expert=[263, 252, 279, 259, 269, 265, 277, 248, 237, 254, 258, 281, 257, 276, 221, 267, 252, 257, 278, 233, 262, 263, 252, 262, 244, 253, 265, 229, 232, 275, 241, 250, 266, 290, 246, 256, 244, 264, 244, 247, 228, 274, 241, 273, 235, 254, 281, 283, 265, 244, 239, 261, 253, 292, 272, 235, 260, 237, 259, 244, 246, 232, 262, 234, 279, 281, 256, 231, 264, 277, 267, 251, 277, 265, 233, 243, 254, 263, 260, 235, 254, 256, 254, 232, 266, 249, 273, 267, 237, 236, 264, 259, 257, 270, 258, 242, 240, 227, 282, 254, 237, 248, 225, 263, 250, 269, 244, 259, 265, 252, 289, 265, 262, 264, 267, 240, 250, 275, 272, 263, 257, 248, 260, 249, 268, 254, 251, 237] payload_rows_per_dst=[5552, 5412, 5714, 5502, 5510, 5216, 5488, 5338] dup_rows_per_dst=[2908, 2684, 2966, 2876, 2794, 2344, 2644, 2588] combine_rows_per_src=[5552, 5412, 5714, 5502, 5510, 5216, 5488, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38236 dup_rows=21880 dup_rows_remote=19196 dedup_ratio=0.333862 router_tokens_per_expert=[233, 268, 247, 266, 293, 271, 269, 281, 259, 267, 245, 266, 244, 260, 259, 255, 268, 240, 245, 283, 242, 230, 247, 267, 253, 260, 233, 260, 260, 279, 236, 249, 247, 222, 279, 253, 265, 242, 272, 290, 253, 256, 259, 260, 241, 277, 271, 245, 247, 243, 275, 233, 282, 237, 252, 239, 258, 250, 251, 244, 275, 251, 243, 239, 237, 252, 254, 263, 268, 243, 260, 220, 254, 240, 234, 277, 272, 262, 291, 243, 255, 269, 239, 244, 240, 242, 228, 291, 248, 257, 230, 268, 259, 285, 257, 262, 248, 276, 260, 260, 239, 272, 246, 282, 265, 270, 266, 254, 263, 238, 294, 252, 231, 266, 274, 256, 290, 245, 244, 241, 218, 239, 255, 295, 256, 268, 254, 221] payload_rows_per_dst=[5564, 5420, 5470, 5420, 5430, 5388, 5552, 5412] dup_rows_per_dst=[2802, 2684, 2794, 2618, 2710, 2760, 2818, 2694] combine_rows_per_src=[5564, 5420, 5470, 5420, 5430, 5388, 5552, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228304 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156613584 ingress_bytes=194134016 link_bytes=194134016 token_bytes=312998864 weight_movement_bytes=37748736 total_nvlink_bytes=350747600 weight_to_token_ratio=0.120603 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860096621e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38334 dup_rows=21756 dup_rows_remote=18914 dedup_ratio=0.331970 router_tokens_per_expert=[263, 272, 282, 267, 262, 242, 278, 232, 264, 265, 253, 254, 274, 270, 250, 244, 258, 232, 249, 249, 252, 273, 277, 231, 259, 275, 261, 243, 250, 239, 271, 239, 230, 261, 273, 275, 247, 243, 253, 286, 254, 258, 251, 256, 274, 298, 252, 233, 298, 248, 254, 244, 255, 250, 238, 264, 253, 274, 252, 261, 234, 266, 250, 249, 280, 276, 248, 264, 240, 257, 250, 256, 276, 233, 248, 228, 253, 262, 254, 265, 254, 254, 260, 264, 227, 258, 256, 241, 280, 251, 268, 251, 257, 240, 266, 287, 280, 278, 249, 258, 257, 253, 263, 253, 255, 290, 259, 265, 242, 240, 234, 239, 260, 223, 252, 261, 260, 232, 259, 258, 231, 255, 229, 250, 246, 276, 266, 227] payload_rows_per_dst=[5570, 5462, 5446, 5498, 5440, 5462, 5502, 5400] dup_rows_per_dst=[2774, 2654, 2842, 2682, 2740, 2766, 2728, 2570] combine_rows_per_src=[5570, 5462, 5446, 5498, 5440, 5462, 5502, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38412 dup_rows=21610 dup_rows_remote=18992 dedup_ratio=0.329742 router_tokens_per_expert=[298, 267, 237, 262, 271, 231, 265, 255, 279, 255, 252, 241, 241, 284, 238, 271, 256, 262, 275, 245, 219, 264, 283, 256, 248, 248, 265, 262, 236, 248, 255, 255, 256, 276, 261, 243, 267, 232, 254, 233, 225, 232, 240, 262, 243, 273, 256, 269, 257, 238, 247, 273, 267, 279, 235, 250, 253, 233, 253, 237, 281, 268, 263, 266, 245, 227, 228, 251, 266, 251, 278, 257, 271, 219, 230, 283, 263, 259, 259, 250, 263, 273, 251, 263, 259, 281, 272, 263, 290, 279, 253, 227, 218, 260, 256, 245, 252, 221, 256, 239, 267, 253, 275, 256, 258, 240, 255, 285, 232, 289, 246, 242, 268, 271, 266, 242, 284, 270, 246, 255, 225, 266, 269, 268, 272, 267, 237, 260] payload_rows_per_dst=[5502, 5548, 5364, 5550, 5424, 5544, 5514, 5480] dup_rows_per_dst=[2704, 2694, 2680, 2650, 2650, 2738, 2618, 2876] combine_rows_per_src=[5502, 5548, 5364, 5550, 5424, 5544, 5514, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229728 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156844384 ingress_bytes=175489024 link_bytes=175489024 token_bytes=313459040 weight_movement_bytes=18874368 total_nvlink_bytes=332333408 weight_to_token_ratio=0.060213 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681449482e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38340 dup_rows=21696 dup_rows_remote=18952 dedup_ratio=0.331055 router_tokens_per_expert=[269, 252, 268, 228, 257, 255, 250, 248, 287, 247, 261, 261, 251, 265, 300, 237, 252, 261, 231, 256, 236, 250, 250, 262, 257, 250, 240, 250, 221, 259, 290, 261, 267, 250, 244, 259, 289, 255, 300, 243, 252, 254, 275, 237, 248, 226, 264, 249, 263, 243, 232, 287, 249, 260, 245, 262, 259, 251, 281, 290, 274, 251, 261, 231, 293, 244, 261, 258, 244, 247, 280, 235, 237, 226, 249, 252, 256, 277, 256, 259, 254, 280, 206, 284, 269, 256, 259, 264, 279, 268, 248, 280, 243, 245, 248, 239, 274, 262, 238, 270, 279, 275, 229, 264, 265, 247, 254, 254, 252, 234, 278, 241, 233, 269, 237, 236, 275, 251, 253, 238, 257, 293, 280, 247, 236, 264, 236, 238] payload_rows_per_dst=[5552, 5448, 5536, 5524, 5468, 5500, 5442, 5370] dup_rows_per_dst=[2720, 2604, 2688, 2754, 2680, 2744, 2790, 2716] combine_rows_per_src=[5552, 5448, 5536, 5524, 5468, 5500, 5442, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38082 dup_rows=21830 dup_rows_remote=18798 dedup_ratio=0.333099 router_tokens_per_expert=[263, 265, 252, 264, 240, 249, 245, 243, 228, 278, 243, 273, 263, 261, 263, 258, 252, 236, 263, 277, 263, 223, 258, 273, 256, 270, 261, 230, 277, 262, 262, 259, 273, 260, 272, 230, 227, 248, 269, 247, 241, 237, 249, 253, 264, 259, 244, 240, 260, 245, 262, 289, 285, 262, 243, 253, 271, 264, 266, 248, 256, 243, 282, 246, 267, 257, 274, 262, 244, 263, 239, 266, 269, 250, 253, 283, 235, 239, 274, 248, 251, 248, 259, 276, 254, 257, 242, 291, 241, 274, 280, 253, 242, 249, 273, 234, 231, 250, 239, 239, 226, 245, 280, 263, 235, 276, 276, 249, 259, 266, 257, 245, 245, 260, 275, 251, 284, 247, 279, 232, 265, 257, 249, 234, 276, 262, 244, 227] payload_rows_per_dst=[5160, 5658, 5406, 5422, 5618, 5388, 5430, 5624] dup_rows_per_dst=[2308, 3066, 2620, 2798, 2856, 2506, 2644, 3032] combine_rows_per_src=[5160, 5658, 5406, 5422, 5618, 5388, 5430, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38322 dup_rows=21834 dup_rows_remote=19178 dedup_ratio=0.333160 router_tokens_per_expert=[259, 262, 261, 232, 299, 235, 254, 276, 250, 272, 248, 232, 260, 254, 270, 257, 303, 294, 235, 256, 247, 231, 233, 248, 265, 248, 269, 266, 255, 254, 257, 231, 248, 236, 272, 232, 266, 267, 266, 262, 254, 260, 290, 254, 231, 257, 266, 237, 234, 264, 255, 262, 258, 230, 252, 254, 260, 210, 229, 272, 249, 261, 270, 258, 243, 268, 257, 253, 265, 227, 265, 248, 249, 259, 242, 250, 259, 249, 220, 280, 264, 236, 252, 260, 262, 250, 281, 262, 264, 265, 257, 273, 254, 234, 252, 255, 249, 242, 260, 285, 255, 269, 256, 282, 251, 220, 243, 296, 255, 293, 254, 262, 254, 251, 259, 259, 237, 272, 270, 246, 242, 267, 253, 278, 256, 252, 251, 265] payload_rows_per_dst=[5506, 5454, 5460, 5380, 5398, 5502, 5506, 5496] dup_rows_per_dst=[2736, 2730, 2736, 2656, 2670, 2740, 2838, 2728] combine_rows_per_src=[5506, 5454, 5460, 5380, 5398, 5502, 5506, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38116 dup_rows=21992 dup_rows_remote=19250 dedup_ratio=0.335571 router_tokens_per_expert=[241, 257, 272, 249, 283, 262, 276, 275, 251, 244, 259, 259, 253, 281, 261, 251, 274, 255, 242, 255, 235, 248, 232, 238, 262, 250, 249, 238, 246, 284, 241, 245, 264, 261, 268, 244, 227, 251, 243, 247, 241, 240, 244, 248, 257, 232, 251, 249, 230, 272, 283, 245, 261, 269, 273, 257, 255, 313, 273, 287, 279, 232, 254, 250, 284, 259, 237, 255, 256, 268, 264, 235, 269, 272, 244, 229, 254, 263, 242, 254, 234, 258, 284, 259, 268, 234, 260, 266, 260, 263, 293, 243, 239, 261, 271, 252, 286, 255, 242, 244, 256, 254, 266, 243, 224, 274, 269, 252, 294, 257, 256, 250, 266, 234, 239, 252, 269, 237, 239, 272, 239, 221, 260, 257, 263, 267, 274, 259] payload_rows_per_dst=[5520, 5394, 5240, 5564, 5428, 5478, 5462, 5458] dup_rows_per_dst=[2828, 2594, 2694, 2902, 2742, 2812, 2782, 2638] combine_rows_per_src=[5520, 5394, 5240, 5564, 5428, 5478, 5462, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=228992 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245056 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261120 weight_movement_bytes=18874368 total_nvlink_bytes=333135488 weight_to_token_ratio=0.060060 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229616 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157565168 ingress_bytes=195084288 link_bytes=195084288 token_bytes=314900720 weight_movement_bytes=37748736 total_nvlink_bytes=352649456 weight_to_token_ratio=0.119875 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869201660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229168 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157269808 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310448 weight_movement_bytes=37748736 total_nvlink_bytes=352059184 weight_to_token_ratio=0.120100 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=227520 combine_payload_bytes=155983872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156211392 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312195264 weight_movement_bytes=18874368 total_nvlink_bytes=331069632 weight_to_token_ratio=0.060457 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=230000 combine_payload_bytes=156966912 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157196912 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314163824 weight_movement_bytes=56623104 total_nvlink_bytes=370786928 weight_to_token_ratio=0.180234 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229464 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352600 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475736 weight_movement_bytes=18874368 total_nvlink_bytes=331350104 weight_to_token_ratio=0.060403 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43492 payload_rows_remote=37908 dup_rows=22044 dup_rows_remote=19094 dedup_ratio=0.336365 router_tokens_per_expert=[255, 265, 280, 241, 226, 268, 270, 244, 255, 269, 236, 232, 249, 232, 238, 248, 242, 268, 275, 265, 286, 257, 237, 277, 269, 247, 227, 253, 242, 221, 261, 282, 252, 227, 260, 260, 260, 263, 235, 271, 232, 255, 252, 242, 260, 259, 244, 268, 261, 286, 274, 241, 226, 264, 252, 280, 275, 254, 268, 242, 274, 282, 238, 245, 245, 252, 269, 269, 270, 264, 248, 267, 252, 244, 284, 246, 273, 248, 229, 288, 258, 260, 253, 240, 226, 275, 226, 253, 242, 259, 243, 243, 245, 262, 254, 248, 249, 264, 294, 277, 283, 272, 235, 228, 269, 241, 273, 248, 262, 277, 267, 259, 272, 235, 247, 252, 296, 243, 257, 234, 285, 282, 238, 243, 256, 272, 251, 253] payload_rows_per_dst=[5584, 5512, 5314, 5322, 5602, 5310, 5380, 5468] dup_rows_per_dst=[2950, 3028, 2562, 2688, 2852, 2498, 2702, 2764] combine_rows_per_src=[5584, 5512, 5314, 5322, 5602, 5310, 5380, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38314 dup_rows=21702 dup_rows_remote=18910 dedup_ratio=0.331146 router_tokens_per_expert=[240, 254, 256, 249, 232, 282, 265, 266, 248, 270, 233, 258, 225, 288, 265, 225, 276, 261, 260, 249, 270, 284, 247, 262, 249, 282, 257, 260, 272, 257, 237, 233, 245, 258, 267, 226, 247, 236, 279, 260, 271, 280, 275, 303, 266, 276, 264, 248, 245, 268, 246, 258, 246, 254, 263, 246, 286, 248, 278, 259, 247, 289, 251, 245, 264, 261, 277, 261, 263, 246, 256, 267, 243, 262, 259, 249, 268, 260, 263, 240, 258, 246, 260, 255, 253, 243, 227, 245, 265, 254, 243, 270, 258, 234, 279, 248, 270, 253, 239, 279, 255, 254, 266, 269, 258, 270, 248, 233, 236, 247, 264, 265, 241, 233, 248, 260, 240, 260, 249, 252, 249, 268, 241, 233, 248, 257, 224, 240] payload_rows_per_dst=[5476, 5520, 5622, 5434, 5468, 5466, 5474, 5374] dup_rows_per_dst=[2636, 2792, 2916, 2824, 2810, 2610, 2602, 2512] combine_rows_per_src=[5476, 5520, 5622, 5434, 5468, 5466, 5474, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155271168 dispatch_meta_bytes=228008 combine_payload_bytes=155271168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155499176 ingress_bytes=174145536 link_bytes=174145536 token_bytes=310770344 weight_movement_bytes=18874368 total_nvlink_bytes=329644712 weight_to_token_ratio=0.060734 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.668576841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38276 dup_rows=21834 dup_rows_remote=19210 dedup_ratio=0.333160 router_tokens_per_expert=[262, 275, 268, 230, 241, 235, 272, 234, 255, 244, 261, 255, 250, 269, 238, 283, 263, 259, 244, 263, 234, 249, 260, 285, 252, 261, 251, 256, 276, 265, 252, 250, 258, 238, 253, 228, 248, 254, 250, 248, 261, 253, 249, 258, 249, 274, 244, 260, 298, 285, 229, 267, 244, 235, 261, 225, 258, 260, 259, 262, 270, 249, 262, 265, 260, 277, 280, 255, 262, 267, 248, 266, 245, 259, 253, 237, 223, 266, 267, 254, 277, 269, 231, 287, 241, 280, 276, 232, 265, 235, 254, 284, 224, 232, 248, 247, 262, 250, 279, 248, 243, 270, 250, 256, 247, 261, 255, 251, 267, 270, 267, 258, 245, 256, 252, 243, 258, 233, 266, 256, 252, 277, 255, 243, 276, 254, 276, 245] payload_rows_per_dst=[5406, 5500, 5426, 5524, 5416, 5484, 5452, 5494] dup_rows_per_dst=[2738, 2740, 2624, 2734, 2822, 2680, 2816, 2680] combine_rows_per_src=[5406, 5500, 5426, 5524, 5416, 5484, 5452, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38336 dup_rows=21706 dup_rows_remote=18876 dedup_ratio=0.331207 router_tokens_per_expert=[245, 268, 255, 274, 254, 263, 273, 263, 222, 232, 236, 269, 251, 257, 250, 240, 240, 267, 266, 241, 266, 239, 230, 269, 233, 274, 264, 262, 254, 254, 275, 248, 287, 250, 273, 225, 254, 271, 253, 270, 240, 258, 273, 243, 256, 267, 268, 238, 263, 269, 267, 258, 265, 258, 261, 226, 248, 261, 242, 247, 251, 253, 251, 249, 255, 249, 243, 266, 248, 231, 238, 275, 259, 243, 240, 256, 243, 264, 263, 273, 253, 253, 262, 265, 280, 249, 283, 257, 267, 262, 251, 269, 251, 246, 243, 244, 250, 280, 246, 265, 237, 239, 264, 236, 277, 258, 289, 257, 251, 290, 258, 265, 251, 257, 240, 250, 232, 266, 266, 267, 287, 270, 227, 258, 258, 256, 260, 251] payload_rows_per_dst=[5472, 5494, 5576, 5446, 5410, 5456, 5494, 5482] dup_rows_per_dst=[2632, 2670, 2676, 2692, 2682, 2814, 2830, 2710] combine_rows_per_src=[5472, 5494, 5576, 5446, 5410, 5456, 5494, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=228896 combine_payload_bytes=156934144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157163040 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097184 weight_movement_bytes=37748736 total_nvlink_bytes=351845920 weight_to_token_ratio=0.120182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38304 dup_rows=21790 dup_rows_remote=19044 dedup_ratio=0.332489 router_tokens_per_expert=[278, 270, 235, 251, 212, 255, 257, 225, 271, 264, 277, 293, 243, 253, 259, 269, 237, 275, 259, 256, 249, 297, 230, 249, 259, 265, 269, 237, 255, 302, 246, 249, 239, 253, 250, 257, 248, 255, 247, 252, 256, 230, 242, 237, 293, 243, 262, 256, 249, 253, 250, 246, 245, 265, 257, 267, 245, 232, 234, 264, 289, 295, 278, 272, 266, 271, 251, 261, 227, 269, 271, 281, 267, 265, 254, 271, 239, 245, 221, 251, 241, 248, 261, 257, 261, 253, 265, 256, 221, 268, 274, 234, 265, 241, 283, 266, 281, 271, 258, 237, 265, 241, 246, 251, 277, 239, 253, 250, 247, 270, 265, 251, 273, 258, 254, 262, 245, 260, 245, 259, 243, 243, 232, 243, 246, 290, 253, 249] payload_rows_per_dst=[5504, 5478, 5418, 5500, 5548, 5442, 5458, 5398] dup_rows_per_dst=[2720, 2790, 2622, 2782, 2672, 2746, 2746, 2712] combine_rows_per_src=[5504, 5478, 5418, 5500, 5548, 5442, 5458, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38216 dup_rows=21764 dup_rows_remote=18888 dedup_ratio=0.332092 router_tokens_per_expert=[259, 275, 247, 231, 227, 254, 234, 276, 246, 255, 274, 264, 256, 245, 259, 256, 259, 269, 251, 244, 261, 263, 263, 239, 237, 254, 243, 252, 263, 256, 261, 267, 256, 261, 254, 258, 265, 285, 232, 247, 276, 255, 276, 253, 252, 262, 242, 254, 264, 234, 239, 275, 247, 267, 293, 244, 239, 233, 244, 268, 245, 254, 285, 261, 232, 245, 255, 268, 243, 252, 249, 252, 267, 256, 261, 248, 260, 280, 256, 264, 252, 228, 270, 270, 266, 255, 262, 250, 259, 255, 239, 223, 244, 207, 239, 265, 254, 274, 270, 294, 259, 265, 281, 262, 264, 257, 224, 236, 257, 245, 242, 253, 281, 255, 253, 268, 275, 270, 251, 255, 255, 278, 277, 278, 257, 253, 244, 249] payload_rows_per_dst=[5400, 5468, 5498, 5392, 5396, 5466, 5596, 5556] dup_rows_per_dst=[2716, 2638, 2758, 2584, 2670, 2668, 2854, 2876] combine_rows_per_src=[5400, 5468, 5498, 5392, 5396, 5466, 5596, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38282 dup_rows=21744 dup_rows_remote=19084 dedup_ratio=0.331787 router_tokens_per_expert=[258, 256, 253, 266, 248, 252, 249, 253, 239, 271, 293, 255, 283, 244, 249, 257, 233, 283, 267, 242, 240, 262, 249, 236, 271, 238, 280, 276, 272, 227, 250, 269, 265, 254, 235, 243, 249, 249, 221, 261, 237, 257, 246, 247, 258, 225, 251, 241, 281, 256, 255, 246, 239, 238, 245, 239, 264, 277, 250, 244, 246, 270, 276, 259, 249, 227, 245, 259, 246, 250, 262, 243, 275, 251, 272, 261, 275, 242, 257, 269, 252, 269, 283, 232, 255, 256, 254, 246, 269, 263, 298, 252, 225, 255, 260, 282, 245, 256, 233, 254, 243, 291, 278, 266, 258, 280, 261, 260, 277, 267, 260, 248, 251, 242, 288, 257, 223, 255, 269, 269, 283, 258, 259, 262, 236, 254, 250, 256] payload_rows_per_dst=[5482, 5544, 5322, 5510, 5470, 5510, 5510, 5444] dup_rows_per_dst=[2770, 2646, 2556, 2660, 2696, 2792, 2844, 2780] combine_rows_per_src=[5482, 5544, 5322, 5510, 5470, 5510, 5510, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38224 dup_rows=21820 dup_rows_remote=18992 dedup_ratio=0.332947 router_tokens_per_expert=[286, 259, 248, 239, 255, 256, 257, 268, 261, 251, 232, 277, 265, 274, 275, 268, 270, 239, 272, 252, 244, 239, 281, 273, 262, 233, 265, 226, 271, 241, 260, 268, 251, 268, 233, 246, 268, 258, 269, 236, 240, 219, 277, 251, 226, 236, 235, 228, 244, 259, 254, 264, 231, 258, 292, 258, 288, 234, 262, 266, 272, 274, 256, 277, 267, 261, 257, 284, 267, 218, 255, 283, 262, 284, 266, 268, 225, 259, 265, 239, 265, 251, 279, 288, 252, 262, 265, 249, 263, 267, 243, 249, 243, 238, 273, 250, 250, 271, 262, 251, 240, 265, 247, 261, 251, 258, 268, 234, 262, 248, 257, 254, 280, 259, 241, 245, 240, 247, 267, 268, 253, 240, 224, 245, 228, 275, 227, 256] payload_rows_per_dst=[5494, 5480, 5298, 5526, 5492, 5504, 5502, 5420] dup_rows_per_dst=[2848, 2712, 2584, 2852, 2828, 2770, 2656, 2570] combine_rows_per_src=[5494, 5480, 5298, 5526, 5492, 5504, 5502, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229944 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008440 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786936 weight_movement_bytes=18874368 total_nvlink_bytes=332661304 weight_to_token_ratio=0.060150 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=228848 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157253104 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314277360 weight_movement_bytes=37748736 total_nvlink_bytes=352026096 weight_to_token_ratio=0.120113 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229392 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122576 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015760 weight_movement_bytes=18874368 total_nvlink_bytes=332890128 weight_to_token_ratio=0.060106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228416 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156761152 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313293888 weight_movement_bytes=18874368 total_nvlink_bytes=332168256 weight_to_token_ratio=0.060245 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229464 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157032536 ingress_bytes=232300544 link_bytes=232300544 token_bytes=313835608 weight_movement_bytes=75497472 total_nvlink_bytes=389333080 weight_to_token_ratio=0.240564 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225789514e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228864 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156794368 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313359872 weight_movement_bytes=37748736 total_nvlink_bytes=351108608 weight_to_token_ratio=0.120464 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38150 dup_rows=21806 dup_rows_remote=18910 dedup_ratio=0.332733 router_tokens_per_expert=[267, 283, 250, 242, 267, 260, 251, 276, 248, 243, 250, 228, 222, 248, 242, 251, 290, 255, 255, 287, 257, 281, 238, 278, 268, 276, 268, 273, 241, 248, 256, 267, 270, 258, 267, 261, 263, 260, 260, 239, 247, 258, 250, 264, 271, 260, 255, 242, 256, 260, 268, 249, 236, 263, 237, 267, 238, 231, 263, 254, 245, 268, 283, 246, 264, 247, 257, 267, 239, 262, 257, 274, 274, 275, 263, 280, 242, 255, 241, 243, 268, 252, 265, 254, 255, 238, 265, 250, 260, 234, 239, 231, 234, 263, 242, 287, 273, 249, 247, 235, 256, 261, 245, 235, 240, 246, 260, 270, 282, 240, 253, 234, 268, 269, 234, 242, 250, 258, 251, 266, 253, 230, 253, 271, 259, 266, 266, 274] payload_rows_per_dst=[5388, 5580, 5496, 5496, 5474, 5382, 5452, 5462] dup_rows_per_dst=[2668, 2896, 2754, 2632, 2806, 2692, 2600, 2758] combine_rows_per_src=[5388, 5580, 5496, 5496, 5474, 5382, 5452, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38224 dup_rows=21824 dup_rows_remote=18980 dedup_ratio=0.333008 router_tokens_per_expert=[288, 268, 288, 241, 245, 247, 270, 263, 266, 243, 248, 264, 268, 252, 222, 288, 234, 266, 288, 242, 278, 242, 270, 256, 239, 254, 289, 269, 245, 225, 281, 247, 281, 266, 256, 248, 240, 275, 235, 254, 263, 294, 270, 240, 244, 285, 262, 273, 257, 259, 262, 259, 255, 238, 279, 234, 246, 285, 243, 251, 253, 261, 240, 267, 257, 255, 267, 279, 262, 261, 253, 241, 277, 253, 230, 251, 238, 270, 233, 250, 248, 271, 249, 251, 261, 253, 224, 246, 241, 261, 260, 248, 257, 269, 272, 267, 258, 259, 264, 254, 283, 245, 238, 269, 240, 248, 251, 239, 235, 239, 266, 232, 287, 237, 242, 255, 261, 237, 262, 247, 262, 272, 246, 232, 250, 237, 279, 226] payload_rows_per_dst=[5488, 5522, 5492, 5460, 5496, 5372, 5434, 5448] dup_rows_per_dst=[2844, 2824, 2802, 2718, 2658, 2756, 2606, 2616] combine_rows_per_src=[5488, 5522, 5492, 5460, 5496, 5372, 5434, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228240 combine_payload_bytes=156262400 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156490640 ingress_bytes=194011136 link_bytes=194011136 token_bytes=312753040 weight_movement_bytes=37748736 total_nvlink_bytes=350501776 weight_to_token_ratio=0.120698 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858919246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=228816 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156794320 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313359824 weight_movement_bytes=37748736 total_nvlink_bytes=351108560 weight_to_token_ratio=0.120465 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38346 dup_rows=21656 dup_rows_remote=19016 dedup_ratio=0.330444 router_tokens_per_expert=[244, 253, 272, 237, 249, 257, 235, 248, 252, 258, 247, 241, 242, 245, 266, 266, 269, 240, 254, 279, 250, 264, 278, 278, 269, 253, 256, 266, 242, 245, 258, 247, 272, 302, 258, 236, 257, 246, 242, 252, 243, 274, 258, 243, 237, 263, 254, 250, 255, 251, 258, 251, 258, 253, 258, 262, 253, 248, 249, 276, 240, 265, 271, 265, 311, 269, 258, 221, 269, 248, 226, 242, 263, 283, 258, 262, 262, 284, 287, 256, 235, 254, 239, 235, 275, 236, 259, 239, 268, 279, 235, 257, 263, 300, 234, 270, 248, 258, 248, 275, 251, 279, 251, 222, 257, 259, 285, 227, 271, 272, 250, 251, 282, 273, 245, 242, 243, 246, 251, 267, 251, 263, 231, 247, 249, 247, 234, 256] payload_rows_per_dst=[5430, 5456, 5534, 5524, 5538, 5500, 5492, 5406] dup_rows_per_dst=[2594, 2840, 2640, 2702, 2860, 2656, 2716, 2648] combine_rows_per_src=[5430, 5456, 5534, 5524, 5538, 5500, 5492, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38232 dup_rows=21830 dup_rows_remote=19084 dedup_ratio=0.333099 router_tokens_per_expert=[280, 236, 234, 242, 255, 243, 238, 243, 239, 251, 249, 267, 252, 265, 280, 239, 279, 259, 244, 240, 233, 268, 265, 266, 268, 273, 262, 229, 231, 268, 251, 264, 238, 259, 249, 259, 272, 257, 239, 236, 256, 260, 235, 276, 263, 257, 270, 260, 264, 242, 235, 262, 297, 250, 270, 262, 254, 236, 250, 301, 250, 259, 248, 244, 259, 255, 253, 258, 246, 298, 271, 273, 274, 261, 259, 264, 246, 263, 279, 242, 240, 248, 280, 275, 244, 276, 236, 267, 247, 249, 254, 272, 275, 275, 258, 255, 255, 249, 262, 221, 243, 283, 266, 229, 252, 259, 259, 262, 290, 264, 242, 274, 274, 243, 224, 260, 244, 244, 240, 254, 252, 239, 256, 273, 250, 239, 254, 237] payload_rows_per_dst=[5368, 5514, 5408, 5522, 5564, 5500, 5474, 5356] dup_rows_per_dst=[2658, 2686, 2764, 2726, 2838, 2802, 2746, 2610] combine_rows_per_src=[5368, 5514, 5408, 5522, 5564, 5500, 5474, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38260 dup_rows=21714 dup_rows_remote=18940 dedup_ratio=0.331329 router_tokens_per_expert=[270, 282, 260, 233, 275, 249, 250, 257, 253, 246, 266, 266, 240, 256, 252, 234, 219, 277, 271, 260, 253, 261, 260, 248, 261, 240, 228, 228, 237, 268, 248, 240, 255, 232, 221, 243, 272, 257, 245, 237, 256, 258, 243, 247, 223, 250, 271, 228, 278, 272, 252, 260, 242, 260, 258, 289, 271, 263, 256, 276, 279, 285, 249, 254, 224, 272, 262, 281, 266, 276, 258, 269, 259, 256, 255, 219, 255, 301, 221, 246, 249, 289, 265, 266, 246, 276, 238, 244, 267, 238, 273, 282, 263, 255, 237, 276, 283, 250, 260, 256, 249, 235, 259, 249, 246, 264, 239, 272, 256, 254, 261, 245, 277, 253, 254, 247, 238, 257, 247, 239, 247, 241, 277, 271, 260, 275, 274, 279] payload_rows_per_dst=[5498, 5390, 5330, 5670, 5272, 5522, 5578, 5562] dup_rows_per_dst=[2680, 2600, 2546, 2742, 2458, 2862, 3052, 2774] combine_rows_per_src=[5498, 5390, 5330, 5670, 5272, 5522, 5578, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38370 dup_rows=21816 dup_rows_remote=18996 dedup_ratio=0.332886 router_tokens_per_expert=[242, 263, 266, 254, 286, 256, 251, 266, 262, 242, 264, 245, 250, 252, 260, 262, 239, 284, 251, 277, 250, 257, 234, 243, 248, 252, 268, 268, 230, 273, 254, 258, 229, 256, 261, 279, 241, 260, 236, 262, 255, 250, 242, 237, 246, 263, 277, 233, 253, 270, 246, 270, 262, 261, 269, 249, 271, 234, 286, 284, 244, 232, 254, 290, 265, 249, 279, 278, 265, 261, 249, 242, 257, 220, 229, 262, 283, 246, 248, 226, 274, 256, 265, 239, 226, 256, 273, 268, 237, 239, 241, 277, 268, 275, 249, 242, 241, 284, 273, 276, 249, 229, 250, 235, 274, 249, 257, 234, 242, 253, 245, 270, 250, 268, 272, 276, 236, 239, 276, 267, 281, 228, 266, 269, 231, 250, 265, 280] payload_rows_per_dst=[5454, 5506, 5402, 5532, 5460, 5350, 5442, 5574] dup_rows_per_dst=[2788, 2666, 2652, 2818, 2658, 2820, 2680, 2734] combine_rows_per_src=[5454, 5506, 5402, 5532, 5460, 5350, 5442, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38372 dup_rows=21714 dup_rows_remote=19006 dedup_ratio=0.331329 router_tokens_per_expert=[250, 232, 251, 267, 280, 257, 255, 267, 261, 292, 258, 281, 238, 281, 239, 282, 266, 223, 278, 255, 253, 256, 247, 243, 245, 231, 254, 242, 270, 280, 268, 250, 295, 261, 260, 259, 257, 238, 255, 254, 269, 262, 275, 293, 253, 250, 266, 261, 258, 247, 286, 281, 280, 272, 249, 245, 220, 231, 268, 237, 243, 252, 255, 255, 270, 225, 229, 238, 251, 248, 273, 259, 245, 245, 267, 247, 268, 242, 277, 240, 280, 228, 255, 282, 264, 256, 225, 234, 253, 255, 245, 251, 241, 268, 262, 268, 238, 247, 265, 251, 231, 238, 264, 250, 271, 278, 263, 246, 264, 246, 244, 233, 252, 267, 252, 266, 259, 272, 269, 251, 263, 252, 277, 272, 266, 241, 216, 234] payload_rows_per_dst=[5594, 5392, 5552, 5450, 5402, 5490, 5454, 5488] dup_rows_per_dst=[2788, 2730, 2864, 2708, 2646, 2644, 2604, 2730] combine_rows_per_src=[5594, 5392, 5552, 5450, 5402, 5490, 5454, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38268 dup_rows=21752 dup_rows_remote=18986 dedup_ratio=0.331909 router_tokens_per_expert=[249, 244, 264, 274, 283, 238, 258, 254, 266, 244, 277, 238, 264, 237, 246, 249, 247, 259, 256, 246, 232, 272, 256, 256, 254, 264, 273, 268, 242, 287, 258, 246, 265, 265, 267, 264, 232, 255, 258, 263, 245, 267, 256, 238, 277, 257, 247, 257, 255, 245, 272, 267, 257, 275, 255, 269, 258, 229, 276, 239, 231, 235, 242, 262, 248, 251, 283, 245, 257, 246, 258, 271, 260, 262, 249, 253, 245, 288, 254, 271, 247, 268, 283, 269, 276, 252, 238, 241, 262, 276, 242, 255, 276, 231, 256, 242, 226, 243, 250, 253, 265, 267, 276, 230, 283, 282, 270, 242, 263, 261, 234, 258, 237, 238, 252, 222, 267, 287, 241, 277, 271, 224, 256, 263, 243, 265, 242, 244] payload_rows_per_dst=[5516, 5444, 5440, 5446, 5516, 5510, 5496, 5416] dup_rows_per_dst=[2654, 2788, 2786, 2688, 2766, 2718, 2710, 2642] combine_rows_per_src=[5516, 5444, 5440, 5446, 5516, 5510, 5496, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229448 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157294664 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314359880 weight_movement_bytes=18874368 total_nvlink_bytes=333234248 weight_to_token_ratio=0.060041 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229264 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827536 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313425808 weight_movement_bytes=18874368 total_nvlink_bytes=332300176 weight_to_token_ratio=0.060220 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228800 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156941760 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313654720 weight_movement_bytes=18874368 total_nvlink_bytes=332529088 weight_to_token_ratio=0.060176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229464 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157392984 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556504 weight_movement_bytes=18874368 total_nvlink_bytes=333430872 weight_to_token_ratio=0.060003 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229512 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157401224 ingress_bytes=232669184 link_bytes=232669184 token_bytes=314572936 weight_movement_bytes=75497472 total_nvlink_bytes=390070408 weight_to_token_ratio=0.240000 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229321641e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229016 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156974744 ingress_bytes=213368832 link_bytes=213368832 token_bytes=313720472 weight_movement_bytes=56623104 total_nvlink_bytes=370343576 weight_to_token_ratio=0.180489 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044395164e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38102 dup_rows=21970 dup_rows_remote=19284 dedup_ratio=0.335236 router_tokens_per_expert=[274, 294, 259, 251, 228, 244, 258, 210, 253, 271, 279, 266, 250, 235, 257, 247, 257, 225, 261, 273, 249, 275, 273, 246, 260, 223, 271, 254, 272, 232, 252, 252, 253, 263, 261, 276, 275, 259, 238, 241, 275, 260, 244, 261, 255, 236, 269, 259, 266, 250, 256, 256, 274, 261, 251, 274, 283, 253, 231, 226, 265, 267, 233, 227, 246, 233, 272, 255, 254, 279, 268, 261, 278, 254, 257, 266, 271, 267, 239, 241, 234, 273, 231, 266, 233, 269, 257, 266, 235, 272, 258, 236, 266, 236, 261, 243, 267, 245, 233, 255, 267, 259, 250, 277, 237, 279, 249, 264, 298, 270, 247, 260, 278, 264, 259, 263, 228, 282, 269, 261, 223, 272, 273, 268, 235, 237, 233, 240] payload_rows_per_dst=[5462, 5464, 5424, 5424, 5468, 5394, 5506, 5424] dup_rows_per_dst=[2690, 2686, 2826, 2722, 2814, 2678, 2808, 2746] combine_rows_per_src=[5462, 5464, 5424, 5424, 5468, 5394, 5506, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38350 dup_rows=21676 dup_rows_remote=18896 dedup_ratio=0.330750 router_tokens_per_expert=[251, 288, 246, 255, 247, 265, 250, 269, 272, 274, 252, 255, 262, 251, 240, 254, 234, 245, 255, 245, 264, 251, 242, 259, 275, 219, 246, 254, 283, 265, 252, 274, 260, 243, 240, 251, 255, 264, 288, 265, 272, 247, 257, 262, 246, 268, 263, 254, 238, 254, 262, 229, 237, 244, 283, 245, 220, 252, 291, 269, 248, 293, 256, 252, 258, 267, 271, 246, 280, 259, 258, 248, 284, 246, 250, 250, 252, 250, 258, 264, 248, 264, 286, 259, 260, 257, 234, 255, 298, 253, 274, 241, 242, 262, 237, 234, 263, 259, 267, 246, 251, 249, 233, 267, 243, 273, 274, 242, 258, 262, 268, 245, 257, 238, 270, 265, 239, 263, 243, 256, 229, 234, 256, 253, 257, 272, 246, 243] payload_rows_per_dst=[5510, 5440, 5622, 5494, 5574, 5466, 5358, 5396] dup_rows_per_dst=[2780, 2686, 2938, 2624, 2708, 2742, 2552, 2646] combine_rows_per_src=[5510, 5440, 5622, 5494, 5574, 5466, 5358, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229544 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156295336 ingress_bytes=212688896 link_bytes=212688896 token_bytes=312361128 weight_movement_bytes=56623104 total_nvlink_bytes=368984232 weight_to_token_ratio=0.181274 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037880352e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38184 dup_rows=21826 dup_rows_remote=19028 dedup_ratio=0.333038 router_tokens_per_expert=[243, 249, 276, 270, 249, 271, 285, 249, 246, 255, 255, 258, 240, 243, 226, 252, 265, 267, 258, 245, 267, 272, 256, 279, 283, 249, 273, 266, 250, 243, 280, 258, 260, 231, 255, 262, 263, 255, 283, 276, 263, 238, 265, 243, 255, 282, 250, 281, 258, 247, 259, 245, 250, 221, 262, 229, 246, 268, 249, 232, 250, 231, 252, 256, 269, 234, 264, 281, 250, 269, 272, 280, 250, 262, 255, 247, 264, 245, 284, 230, 257, 236, 248, 269, 236, 237, 257, 231, 265, 284, 277, 264, 281, 237, 278, 273, 264, 247, 272, 240, 251, 234, 229, 242, 262, 229, 253, 240, 247, 259, 244, 256, 291, 279, 252, 263, 269, 255, 243, 255, 266, 233, 229, 288, 243, 262, 264, 226] payload_rows_per_dst=[5472, 5568, 5526, 5342, 5490, 5522, 5358, 5432] dup_rows_per_dst=[2662, 2854, 2798, 2568, 2822, 2738, 2580, 2804] combine_rows_per_src=[5472, 5568, 5526, 5342, 5490, 5522, 5358, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=228984 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157310584 ingress_bytes=213704704 link_bytes=213704704 token_bytes=314392184 weight_movement_bytes=56623104 total_nvlink_bytes=371015288 weight_to_token_ratio=0.180103 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047613324e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43858 payload_rows_remote=38328 dup_rows=21678 dup_rows_remote=18918 dedup_ratio=0.330780 router_tokens_per_expert=[270, 255, 238, 229, 258, 222, 281, 251, 252, 268, 258, 272, 260, 280, 261, 275, 246, 253, 257, 259, 285, 260, 267, 246, 224, 252, 235, 249, 243, 255, 278, 248, 225, 270, 252, 249, 238, 256, 276, 251, 257, 295, 245, 254, 255, 256, 247, 257, 261, 254, 244, 264, 250, 264, 244, 253, 270, 245, 261, 278, 244, 247, 241, 259, 281, 299, 239, 268, 264, 260, 241, 257, 234, 266, 244, 276, 265, 245, 270, 261, 241, 258, 254, 255, 253, 245, 256, 278, 271, 271, 253, 212, 259, 234, 275, 251, 284, 227, 285, 259, 245, 252, 278, 271, 280, 212, 292, 256, 274, 236, 226, 268, 260, 276, 253, 244, 260, 260, 272, 236, 239, 244, 242, 268, 252, 236, 250, 246] payload_rows_per_dst=[5464, 5476, 5456, 5450, 5584, 5458, 5530, 5440] dup_rows_per_dst=[2796, 2638, 2710, 2708, 2756, 2674, 2760, 2636] combine_rows_per_src=[5464, 5476, 5456, 5450, 5584, 5458, 5530, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43504 payload_rows_remote=38074 dup_rows=22032 dup_rows_remote=19190 dedup_ratio=0.336182 router_tokens_per_expert=[253, 260, 258, 245, 271, 284, 270, 268, 246, 226, 245, 244, 289, 260, 262, 253, 287, 250, 233, 261, 263, 236, 255, 256, 251, 251, 250, 255, 274, 246, 251, 251, 264, 248, 248, 257, 282, 266, 272, 248, 253, 283, 274, 270, 261, 236, 261, 279, 258, 253, 249, 274, 242, 275, 270, 269, 247, 249, 264, 245, 236, 273, 260, 248, 251, 263, 248, 256, 254, 270, 229, 242, 239, 247, 230, 240, 236, 251, 245, 249, 252, 245, 235, 268, 270, 248, 247, 266, 276, 270, 238, 262, 272, 247, 260, 215, 291, 300, 253, 246, 258, 252, 273, 270, 242, 245, 247, 251, 244, 253, 247, 241, 266, 260, 238, 267, 243, 239, 281, 254, 239, 267, 252, 247, 261, 266, 277, 259] payload_rows_per_dst=[5288, 5362, 5494, 5470, 5418, 5586, 5456, 5430] dup_rows_per_dst=[2654, 2582, 2870, 2754, 2666, 2882, 2782, 2842] combine_rows_per_src=[5288, 5362, 5494, 5470, 5418, 5586, 5456, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43564 payload_rows_remote=38214 dup_rows=21972 dup_rows_remote=19106 dedup_ratio=0.335266 router_tokens_per_expert=[251, 247, 268, 259, 259, 270, 251, 262, 243, 279, 246, 253, 264, 252, 260, 231, 272, 247, 256, 271, 257, 253, 252, 245, 242, 290, 277, 267, 223, 259, 270, 246, 244, 252, 280, 225, 246, 251, 250, 283, 239, 303, 247, 269, 270, 278, 253, 269, 249, 274, 249, 259, 243, 257, 291, 257, 238, 242, 289, 271, 276, 240, 248, 238, 252, 252, 219, 273, 249, 278, 271, 259, 242, 279, 224, 236, 257, 245, 262, 257, 249, 254, 251, 246, 280, 258, 252, 255, 256, 259, 243, 234, 269, 285, 277, 240, 269, 230, 279, 250, 219, 242, 268, 236, 274, 238, 250, 246, 246, 259, 230, 249, 235, 249, 270, 263, 289, 260, 258, 258, 229, 278, 258, 240, 268, 270, 255, 238] payload_rows_per_dst=[5438, 5474, 5484, 5524, 5450, 5350, 5348, 5496] dup_rows_per_dst=[2752, 2780, 2834, 2718, 2660, 2866, 2622, 2740] combine_rows_per_src=[5438, 5474, 5484, 5524, 5450, 5350, 5348, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38316 dup_rows=21748 dup_rows_remote=19124 dedup_ratio=0.331848 router_tokens_per_expert=[261, 246, 255, 249, 276, 281, 230, 264, 272, 244, 263, 256, 249, 238, 256, 260, 255, 268, 248, 241, 245, 269, 248, 258, 248, 274, 254, 241, 269, 251, 264, 264, 260, 242, 260, 256, 243, 241, 250, 261, 240, 241, 246, 274, 275, 236, 284, 248, 240, 246, 239, 278, 242, 241, 252, 250, 263, 296, 244, 255, 237, 261, 264, 240, 238, 262, 238, 273, 247, 239, 252, 269, 240, 245, 258, 249, 257, 237, 190, 276, 291, 256, 267, 279, 235, 252, 241, 249, 279, 246, 263, 243, 273, 255, 276, 246, 233, 268, 259, 237, 293, 252, 257, 252, 294, 270, 280, 237, 263, 267, 266, 254, 261, 274, 291, 244, 257, 268, 243, 274, 243, 280, 244, 251, 269, 266, 227, 271] payload_rows_per_dst=[5494, 5446, 5438, 5472, 5372, 5516, 5558, 5492] dup_rows_per_dst=[2706, 2748, 2676, 2624, 2568, 2786, 2806, 2834] combine_rows_per_src=[5494, 5446, 5438, 5472, 5372, 5516, 5558, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38128 dup_rows=21958 dup_rows_remote=19130 dedup_ratio=0.335052 router_tokens_per_expert=[268, 268, 262, 253, 226, 259, 290, 247, 258, 260, 272, 258, 294, 232, 260, 284, 233, 248, 258, 268, 254, 248, 254, 258, 270, 231, 263, 268, 236, 292, 271, 237, 255, 251, 258, 248, 252, 224, 271, 252, 241, 243, 259, 241, 259, 232, 249, 251, 235, 256, 252, 285, 255, 253, 290, 253, 255, 273, 254, 265, 256, 265, 273, 278, 252, 247, 283, 244, 258, 268, 281, 250, 238, 261, 286, 253, 254, 264, 233, 267, 245, 236, 281, 281, 228, 264, 256, 260, 230, 237, 213, 241, 260, 243, 239, 255, 253, 266, 266, 243, 249, 240, 268, 271, 269, 255, 255, 245, 267, 259, 252, 254, 244, 262, 246, 262, 278, 298, 240, 256, 270, 261, 218, 243, 239, 267, 231, 269] payload_rows_per_dst=[5538, 5496, 5300, 5530, 5450, 5328, 5486, 5450] dup_rows_per_dst=[2844, 2682, 2672, 2866, 2828, 2610, 2738, 2718] combine_rows_per_src=[5538, 5496, 5300, 5530, 5450, 5328, 5486, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228848 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156630512 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032176 weight_movement_bytes=37748736 total_nvlink_bytes=350780912 weight_to_token_ratio=0.120591 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=229056 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156180160 ingress_bytes=193699840 link_bytes=193699840 token_bytes=312131264 weight_movement_bytes=37748736 total_nvlink_bytes=349880000 weight_to_token_ratio=0.120939 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855936561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=228984 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157220472 ingress_bytes=194740224 link_bytes=194740224 token_bytes=314211960 weight_movement_bytes=37748736 total_nvlink_bytes=351960696 weight_to_token_ratio=0.120138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865905008e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229280 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156753824 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278368 weight_movement_bytes=18874368 total_nvlink_bytes=332152736 weight_to_token_ratio=0.060248 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229032 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156401320 ingress_bytes=175046656 link_bytes=175046656 token_bytes=312573608 weight_movement_bytes=18874368 total_nvlink_bytes=331447976 weight_to_token_ratio=0.060384 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677210929e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229760 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157172096 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114432 weight_movement_bytes=18874368 total_nvlink_bytes=332988800 weight_to_token_ratio=0.060088 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38262 dup_rows=21760 dup_rows_remote=19028 dedup_ratio=0.332031 router_tokens_per_expert=[273, 259, 237, 238, 249, 274, 260, 262, 270, 262, 290, 253, 274, 248, 234, 261, 246, 248, 268, 262, 240, 278, 276, 266, 257, 263, 231, 248, 271, 255, 242, 272, 224, 253, 248, 265, 255, 252, 247, 260, 263, 262, 258, 232, 225, 249, 258, 267, 249, 250, 248, 250, 252, 223, 250, 262, 238, 268, 267, 254, 245, 243, 265, 241, 251, 273, 250, 256, 244, 283, 234, 221, 247, 260, 276, 246, 249, 269, 265, 257, 236, 233, 273, 260, 280, 239, 297, 235, 229, 287, 244, 247, 248, 259, 278, 287, 264, 266, 250, 248, 284, 255, 243, 266, 275, 235, 286, 253, 228, 268, 246, 242, 277, 257, 264, 235, 272, 262, 284, 237, 262, 257, 262, 264, 264, 239, 256, 264] payload_rows_per_dst=[5468, 5514, 5446, 5414, 5474, 5412, 5452, 5596] dup_rows_per_dst=[2820, 2732, 2590, 2596, 2688, 2852, 2766, 2716] combine_rows_per_src=[5468, 5514, 5446, 5414, 5474, 5412, 5452, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38166 dup_rows=21916 dup_rows_remote=19128 dedup_ratio=0.334412 router_tokens_per_expert=[251, 267, 238, 267, 276, 242, 236, 267, 240, 269, 244, 247, 260, 263, 266, 283, 246, 238, 264, 249, 247, 254, 244, 259, 229, 256, 262, 255, 266, 224, 281, 258, 263, 258, 248, 256, 270, 243, 243, 228, 246, 284, 245, 274, 244, 271, 229, 253, 247, 255, 256, 238, 260, 270, 259, 213, 255, 267, 257, 256, 270, 253, 254, 233, 246, 245, 245, 249, 255, 226, 262, 261, 281, 286, 254, 252, 268, 271, 249, 228, 229, 249, 229, 247, 289, 271, 242, 272, 263, 242, 240, 263, 251, 249, 266, 262, 262, 239, 263, 268, 241, 266, 258, 246, 239, 230, 242, 273, 287, 265, 257, 280, 232, 278, 269, 262, 252, 284, 288, 264, 279, 262, 273, 257, 287, 245, 270, 262] payload_rows_per_dst=[5454, 5446, 5568, 5406, 5510, 5444, 5238, 5554] dup_rows_per_dst=[2788, 2618, 3072, 2670, 2646, 2684, 2464, 2974] combine_rows_per_src=[5454, 5446, 5568, 5406, 5510, 5444, 5238, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229160 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950312 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671464 weight_movement_bytes=18874368 total_nvlink_bytes=332545832 weight_to_token_ratio=0.060172 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38350 dup_rows=21724 dup_rows_remote=19034 dedup_ratio=0.331482 router_tokens_per_expert=[275, 286, 248, 271, 251, 244, 273, 271, 264, 248, 260, 267, 251, 251, 248, 255, 254, 277, 263, 252, 243, 263, 264, 241, 259, 268, 247, 266, 255, 240, 279, 274, 253, 252, 259, 252, 260, 235, 266, 240, 264, 250, 247, 279, 285, 244, 240, 250, 265, 239, 268, 272, 255, 246, 261, 266, 243, 264, 258, 271, 263, 256, 270, 252, 251, 296, 236, 260, 224, 234, 267, 230, 245, 243, 264, 271, 272, 244, 274, 259, 233, 253, 258, 249, 238, 263, 244, 279, 234, 249, 251, 265, 250, 254, 245, 277, 246, 268, 252, 254, 261, 246, 252, 264, 257, 234, 271, 236, 249, 259, 259, 248, 238, 254, 235, 259, 279, 245, 282, 278, 245, 217, 253, 233, 268, 263, 249, 269] payload_rows_per_dst=[5506, 5566, 5462, 5444, 5530, 5382, 5438, 5484] dup_rows_per_dst=[2820, 2724, 2690, 2854, 2610, 2702, 2674, 2650] combine_rows_per_src=[5506, 5566, 5462, 5444, 5530, 5382, 5438, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43508 payload_rows_remote=38002 dup_rows=22028 dup_rows_remote=19228 dedup_ratio=0.336121 router_tokens_per_expert=[272, 257, 260, 268, 261, 233, 250, 279, 277, 228, 243, 230, 246, 242, 251, 243, 269, 247, 251, 249, 255, 260, 272, 226, 231, 255, 270, 241, 227, 273, 249, 242, 255, 247, 243, 229, 260, 262, 238, 232, 260, 250, 271, 220, 261, 269, 247, 274, 294, 250, 275, 250, 232, 242, 250, 273, 269, 256, 272, 260, 253, 262, 264, 263, 288, 262, 267, 262, 283, 258, 263, 220, 259, 274, 271, 247, 240, 251, 244, 250, 249, 237, 243, 260, 270, 285, 249, 257, 264, 256, 278, 261, 259, 233, 267, 253, 259, 253, 282, 259, 256, 271, 282, 246, 259, 247, 244, 254, 281, 261, 233, 269, 247, 274, 268, 251, 270, 262, 251, 253, 246, 254, 241, 242, 273, 264, 275, 241] payload_rows_per_dst=[5238, 5390, 5352, 5470, 5562, 5482, 5508, 5506] dup_rows_per_dst=[2532, 2652, 2724, 2860, 2790, 2866, 2804, 2800] combine_rows_per_src=[5238, 5390, 5352, 5470, 5562, 5482, 5508, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38398 dup_rows=21690 dup_rows_remote=18928 dedup_ratio=0.330963 router_tokens_per_expert=[255, 262, 266, 265, 254, 305, 248, 231, 244, 268, 239, 237, 269, 242, 269, 251, 301, 254, 231, 295, 250, 269, 247, 256, 247, 264, 252, 243, 249, 224, 240, 269, 264, 249, 260, 257, 289, 247, 276, 262, 270, 269, 258, 282, 233, 292, 251, 233, 243, 249, 282, 251, 235, 265, 256, 253, 280, 230, 244, 228, 258, 252, 249, 268, 222, 269, 263, 260, 258, 251, 261, 253, 263, 272, 273, 238, 234, 248, 260, 249, 231, 251, 256, 248, 244, 278, 226, 278, 280, 239, 247, 266, 250, 285, 255, 254, 272, 256, 260, 244, 265, 272, 271, 257, 266, 221, 254, 245, 251, 241, 242, 288, 242, 266, 250, 251, 256, 252, 256, 248, 255, 285, 251, 280, 269, 261, 247, 201] payload_rows_per_dst=[5452, 5516, 5572, 5414, 5446, 5488, 5448, 5510] dup_rows_per_dst=[2758, 2666, 2812, 2672, 2702, 2688, 2762, 2630] combine_rows_per_src=[5452, 5516, 5572, 5414, 5446, 5488, 5448, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44042 payload_rows_remote=38600 dup_rows=21494 dup_rows_remote=18872 dedup_ratio=0.327972 router_tokens_per_expert=[244, 237, 258, 276, 230, 242, 270, 264, 270, 248, 272, 272, 242, 251, 235, 258, 282, 252, 278, 261, 248, 226, 249, 256, 271, 243, 260, 229, 245, 269, 290, 272, 256, 236, 235, 259, 236, 262, 238, 242, 251, 264, 263, 283, 266, 286, 265, 272, 255, 247, 241, 250, 248, 261, 268, 265, 249, 269, 246, 239, 246, 243, 285, 268, 261, 270, 263, 261, 236, 234, 265, 237, 268, 268, 279, 242, 287, 276, 248, 273, 243, 272, 233, 240, 251, 236, 251, 255, 246, 281, 252, 255, 264, 252, 255, 246, 275, 289, 229, 269, 248, 250, 255, 255, 241, 290, 261, 233, 248, 254, 231, 282, 251, 254, 246, 261, 245, 252, 284, 276, 216, 243, 242, 249, 254, 239, 253, 299] payload_rows_per_dst=[5468, 5534, 5560, 5486, 5588, 5442, 5498, 5466] dup_rows_per_dst=[2670, 2728, 2668, 2674, 2748, 2622, 2722, 2662] combine_rows_per_src=[5468, 5534, 5560, 5486, 5588, 5442, 5498, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=229176 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156557112 ingress_bytes=175202304 link_bytes=175202304 token_bytes=312885048 weight_movement_bytes=18874368 total_nvlink_bytes=331759416 weight_to_token_ratio=0.060324 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678702272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38294 dup_rows=21824 dup_rows_remote=19126 dedup_ratio=0.333008 router_tokens_per_expert=[249, 286, 283, 268, 258, 243, 275, 261, 271, 255, 257, 248, 259, 270, 249, 264, 252, 248, 227, 286, 288, 236, 250, 308, 251, 243, 231, 264, 269, 254, 241, 277, 250, 265, 256, 259, 250, 256, 254, 250, 246, 249, 258, 256, 281, 251, 270, 253, 244, 260, 267, 266, 244, 251, 241, 250, 245, 251, 245, 240, 254, 276, 282, 271, 241, 257, 262, 258, 267, 260, 270, 250, 258, 224, 250, 270, 248, 236, 245, 262, 252, 251, 222, 252, 244, 258, 250, 279, 237, 248, 267, 285, 282, 237, 252, 281, 271, 229, 244, 235, 248, 238, 252, 233, 261, 225, 276, 257, 264, 263, 267, 258, 278, 260, 258, 274, 225, 262, 236, 245, 258, 239, 282, 232, 265, 255, 262, 249] payload_rows_per_dst=[5508, 5470, 5464, 5398, 5418, 5548, 5422, 5484] dup_rows_per_dst=[2884, 2780, 2744, 2776, 2698, 2646, 2620, 2676] combine_rows_per_src=[5508, 5470, 5464, 5398, 5418, 5548, 5422, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38156 dup_rows=21910 dup_rows_remote=19130 dedup_ratio=0.334320 router_tokens_per_expert=[235, 288, 257, 251, 235, 241, 261, 268, 261, 247, 252, 252, 252, 240, 281, 252, 262, 252, 276, 258, 231, 256, 263, 230, 247, 236, 264, 254, 261, 285, 261, 258, 261, 272, 251, 257, 243, 239, 271, 302, 264, 235, 233, 247, 238, 235, 264, 285, 272, 250, 271, 263, 254, 241, 257, 270, 233, 236, 242, 266, 299, 259, 256, 256, 251, 242, 227, 267, 273, 278, 252, 231, 251, 264, 274, 252, 251, 273, 242, 238, 245, 244, 249, 268, 264, 277, 247, 258, 221, 230, 255, 269, 261, 244, 254, 274, 280, 260, 261, 229, 271, 267, 267, 248, 272, 267, 248, 262, 253, 243, 259, 254, 246, 276, 236, 249, 242, 283, 260, 251, 257, 245, 250, 245, 297, 258, 266, 251] payload_rows_per_dst=[5410, 5428, 5474, 5470, 5436, 5450, 5512, 5446] dup_rows_per_dst=[2736, 2760, 2720, 2780, 2696, 2670, 2770, 2778] combine_rows_per_src=[5410, 5428, 5474, 5470, 5436, 5450, 5512, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229536 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311136 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392736 weight_movement_bytes=18874368 total_nvlink_bytes=333267104 weight_to_token_ratio=0.060034 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155656192 dispatch_meta_bytes=228920 combine_payload_bytes=155656192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155885112 ingress_bytes=174530560 link_bytes=174530560 token_bytes=311541304 weight_movement_bytes=18874368 total_nvlink_bytes=330415672 weight_to_token_ratio=0.060584 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672265952e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229304 combine_payload_bytes=157278208 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157507512 ingress_bytes=195026944 link_bytes=195026944 token_bytes=314785720 weight_movement_bytes=37748736 total_nvlink_bytes=352534456 weight_to_token_ratio=0.119919 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868652218e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158105600 dispatch_meta_bytes=229888 combine_payload_bytes=158105600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158335488 ingress_bytes=176979968 link_bytes=176979968 token_bytes=316441088 weight_movement_bytes=18874368 total_nvlink_bytes=335315456 weight_to_token_ratio=0.059646 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.695734974e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229680 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081904 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934128 weight_movement_bytes=18874368 total_nvlink_bytes=332808496 weight_to_token_ratio=0.060122 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229144 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156516120 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312803096 weight_movement_bytes=18874368 total_nvlink_bytes=331677464 weight_to_token_ratio=0.060339 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38388 dup_rows=21660 dup_rows_remote=18992 dedup_ratio=0.330505 router_tokens_per_expert=[272, 260, 253, 262, 251, 268, 279, 251, 233, 286, 280, 271, 266, 242, 255, 283, 224, 262, 267, 273, 254, 245, 244, 244, 278, 248, 265, 253, 243, 251, 255, 272, 256, 245, 232, 263, 263, 269, 236, 257, 288, 291, 246, 257, 266, 213, 267, 257, 255, 267, 236, 243, 245, 239, 252, 255, 243, 278, 245, 248, 247, 287, 242, 252, 269, 245, 253, 239, 298, 247, 262, 257, 240, 257, 260, 245, 250, 262, 261, 277, 257, 258, 268, 251, 255, 238, 257, 244, 257, 226, 235, 241, 256, 232, 247, 278, 233, 262, 224, 264, 259, 252, 270, 263, 288, 259, 269, 239, 260, 255, 258, 246, 282, 275, 240, 272, 219, 233, 250, 265, 244, 261, 263, 293, 270, 226, 279, 243] payload_rows_per_dst=[5588, 5488, 5488, 5428, 5494, 5368, 5488, 5534] dup_rows_per_dst=[2836, 2668, 2724, 2640, 2750, 2632, 2714, 2696] combine_rows_per_src=[5588, 5488, 5488, 5428, 5494, 5368, 5488, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229520 combine_payload_bytes=157237248 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157466768 ingress_bytes=213860352 link_bytes=213860352 token_bytes=314704016 weight_movement_bytes=56623104 total_nvlink_bytes=371327120 weight_to_token_ratio=0.179925 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049104667e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38212 dup_rows=21760 dup_rows_remote=18866 dedup_ratio=0.332031 router_tokens_per_expert=[257, 257, 259, 249, 285, 252, 267, 282, 267, 249, 260, 273, 278, 241, 236, 256, 255, 253, 236, 264, 272, 238, 248, 276, 275, 249, 275, 251, 261, 242, 257, 233, 234, 275, 257, 247, 254, 235, 252, 258, 245, 257, 257, 264, 271, 284, 259, 247, 248, 263, 264, 252, 226, 242, 249, 259, 252, 237, 267, 257, 222, 266, 275, 249, 244, 249, 262, 226, 236, 296, 276, 254, 271, 259, 261, 251, 256, 259, 276, 263, 263, 247, 256, 246, 261, 240, 249, 245, 292, 256, 227, 257, 255, 235, 267, 248, 241, 266, 258, 274, 263, 233, 272, 259, 218, 266, 236, 251, 277, 226, 227, 253, 268, 265, 256, 280, 261, 254, 247, 275, 240, 237, 240, 271, 284, 266, 270, 274] payload_rows_per_dst=[5564, 5410, 5604, 5326, 5506, 5488, 5392, 5486] dup_rows_per_dst=[2894, 2638, 2764, 2554, 2772, 2600, 2648, 2890] combine_rows_per_src=[5564, 5410, 5604, 5326, 5506, 5488, 5392, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38272 dup_rows=21792 dup_rows_remote=19188 dedup_ratio=0.332520 router_tokens_per_expert=[237, 263, 262, 251, 254, 244, 266, 260, 235, 237, 266, 298, 274, 289, 256, 257, 276, 252, 260, 268, 274, 281, 267, 246, 266, 231, 267, 235, 245, 267, 240, 260, 256, 256, 254, 249, 251, 228, 272, 232, 261, 247, 241, 252, 283, 261, 254, 241, 211, 265, 264, 240, 250, 241, 259, 230, 245, 256, 255, 268, 252, 268, 267, 244, 254, 257, 270, 277, 261, 258, 256, 275, 273, 279, 264, 276, 269, 239, 256, 270, 237, 242, 227, 271, 265, 238, 254, 253, 249, 247, 298, 236, 238, 247, 237, 258, 272, 271, 285, 241, 258, 245, 275, 254, 240, 260, 254, 265, 255, 245, 288, 259, 277, 255, 248, 236, 249, 253, 258, 250, 265, 224, 248, 261, 255, 260, 251, 243] payload_rows_per_dst=[5486, 5488, 5472, 5444, 5592, 5376, 5500, 5386] dup_rows_per_dst=[2812, 2782, 2604, 2586, 2876, 2618, 2834, 2680] combine_rows_per_src=[5486, 5488, 5472, 5444, 5592, 5376, 5500, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38296 dup_rows=21716 dup_rows_remote=18810 dedup_ratio=0.331360 router_tokens_per_expert=[259, 295, 236, 254, 247, 263, 282, 262, 272, 261, 228, 280, 280, 245, 260, 242, 261, 279, 249, 270, 256, 260, 257, 262, 267, 275, 273, 258, 221, 265, 237, 291, 248, 250, 255, 256, 277, 266, 240, 265, 246, 264, 241, 256, 253, 252, 275, 234, 263, 265, 222, 255, 261, 273, 243, 230, 240, 256, 242, 285, 211, 259, 297, 260, 265, 253, 236, 271, 240, 256, 259, 253, 287, 213, 252, 259, 299, 251, 262, 212, 270, 260, 255, 285, 251, 256, 249, 256, 258, 245, 267, 219, 271, 247, 274, 218, 274, 252, 233, 228, 252, 256, 236, 244, 290, 250, 257, 257, 279, 263, 245, 246, 259, 294, 251, 261, 246, 258, 253, 255, 251, 230, 263, 260, 245, 264, 243, 237] payload_rows_per_dst=[5286, 5392, 5554, 5570, 5290, 5662, 5542, 5524] dup_rows_per_dst=[2352, 2602, 2802, 2864, 2366, 3074, 2750, 2906] combine_rows_per_src=[5286, 5392, 5554, 5570, 5290, 5662, 5542, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38256 dup_rows=21794 dup_rows_remote=19142 dedup_ratio=0.332550 router_tokens_per_expert=[254, 269, 285, 270, 261, 230, 252, 252, 239, 283, 244, 225, 263, 245, 231, 283, 250, 262, 244, 257, 265, 256, 243, 272, 271, 263, 234, 274, 244, 261, 241, 270, 255, 242, 239, 244, 233, 227, 265, 277, 262, 266, 253, 251, 271, 231, 228, 284, 260, 254, 280, 251, 290, 242, 262, 250, 250, 273, 274, 260, 266, 263, 246, 264, 279, 279, 240, 252, 251, 236, 246, 272, 266, 269, 268, 254, 261, 261, 236, 269, 248, 243, 265, 240, 238, 281, 252, 266, 228, 247, 245, 247, 241, 253, 260, 242, 253, 256, 239, 244, 236, 238, 241, 285, 300, 254, 263, 254, 243, 242, 272, 249, 276, 242, 274, 265, 255, 271, 255, 288, 281, 254, 248, 230, 233, 265, 270, 251] payload_rows_per_dst=[5488, 5496, 5368, 5500, 5332, 5540, 5486, 5532] dup_rows_per_dst=[2684, 2718, 2688, 2870, 2530, 2868, 2652, 2784] combine_rows_per_src=[5488, 5496, 5368, 5500, 5332, 5540, 5486, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38216 dup_rows=21734 dup_rows_remote=18970 dedup_ratio=0.331635 router_tokens_per_expert=[242, 259, 236, 243, 237, 288, 255, 271, 254, 245, 259, 253, 249, 238, 235, 255, 268, 243, 237, 302, 259, 236, 248, 230, 281, 295, 243, 270, 228, 243, 229, 264, 261, 292, 251, 288, 241, 265, 261, 245, 257, 244, 264, 255, 244, 263, 243, 304, 301, 249, 255, 256, 238, 250, 268, 289, 253, 248, 257, 278, 254, 274, 278, 227, 247, 272, 258, 276, 274, 266, 235, 262, 269, 278, 230, 282, 245, 234, 273, 242, 252, 229, 254, 262, 258, 268, 225, 257, 241, 247, 243, 239, 225, 280, 258, 256, 275, 275, 245, 269, 239, 265, 263, 245, 270, 241, 253, 262, 255, 246, 236, 245, 252, 257, 259, 256, 246, 277, 258, 253, 296, 241, 262, 243, 231, 252, 259, 257] payload_rows_per_dst=[5416, 5482, 5544, 5586, 5514, 5388, 5480, 5392] dup_rows_per_dst=[2622, 2670, 2812, 2764, 2772, 2600, 2688, 2806] combine_rows_per_src=[5416, 5482, 5544, 5586, 5514, 5388, 5480, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38164 dup_rows=21994 dup_rows_remote=19334 dedup_ratio=0.335602 router_tokens_per_expert=[251, 252, 256, 257, 242, 270, 246, 273, 261, 273, 245, 267, 238, 273, 260, 252, 267, 252, 259, 275, 292, 235, 219, 233, 222, 288, 284, 253, 269, 295, 263, 246, 261, 248, 243, 256, 222, 268, 258, 276, 233, 244, 244, 249, 269, 270, 266, 251, 252, 267, 273, 244, 262, 269, 244, 240, 262, 266, 273, 234, 246, 268, 235, 254, 251, 259, 257, 263, 273, 272, 268, 285, 267, 289, 237, 265, 249, 255, 270, 269, 261, 257, 249, 245, 234, 221, 261, 288, 253, 249, 235, 255, 271, 242, 252, 246, 254, 247, 251, 267, 256, 228, 244, 236, 227, 254, 276, 266, 225, 258, 257, 264, 252, 275, 257, 245, 244, 238, 273, 256, 277, 249, 237, 271, 254, 247, 284, 236] payload_rows_per_dst=[5424, 5486, 5414, 5488, 5518, 5378, 5338, 5496] dup_rows_per_dst=[2808, 2818, 2702, 2690, 2940, 2660, 2682, 2694] combine_rows_per_src=[5424, 5486, 5414, 5488, 5518, 5378, 5338, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38232 dup_rows=21788 dup_rows_remote=18828 dedup_ratio=0.332458 router_tokens_per_expert=[278, 270, 239, 247, 240, 258, 275, 259, 248, 276, 257, 266, 250, 261, 258, 256, 238, 235, 272, 285, 269, 249, 224, 262, 278, 268, 257, 230, 274, 278, 249, 223, 256, 254, 268, 253, 255, 242, 250, 249, 246, 271, 253, 245, 279, 251, 268, 266, 275, 242, 252, 270, 252, 272, 235, 228, 250, 269, 242, 262, 227, 245, 248, 280, 264, 241, 311, 285, 273, 269, 237, 253, 276, 259, 279, 210, 283, 260, 296, 242, 240, 261, 240, 254, 225, 284, 230, 258, 275, 233, 273, 264, 237, 249, 268, 261, 239, 257, 269, 253, 254, 237, 244, 250, 247, 243, 272, 255, 232, 284, 283, 244, 283, 245, 236, 249, 242, 238, 265, 261, 267, 244, 255, 254, 254, 228, 254, 256] payload_rows_per_dst=[5544, 5464, 5528, 5462, 5516, 5378, 5464, 5392] dup_rows_per_dst=[2732, 2718, 2684, 2636, 2960, 2726, 2662, 2670] combine_rows_per_src=[5544, 5464, 5528, 5462, 5516, 5378, 5464, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156516352 dispatch_meta_bytes=228312 combine_payload_bytes=156516352 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156744664 ingress_bytes=175390720 link_bytes=175390720 token_bytes=313261016 weight_movement_bytes=18874368 total_nvlink_bytes=332135384 weight_to_token_ratio=0.060251 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680507581e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229840 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991952 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313754064 weight_movement_bytes=18874368 total_nvlink_bytes=332628432 weight_to_token_ratio=0.060157 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228424 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157088840 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949256 weight_movement_bytes=18874368 total_nvlink_bytes=332823624 weight_to_token_ratio=0.060119 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229592 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926168 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622744 weight_movement_bytes=18874368 total_nvlink_bytes=332497112 weight_to_token_ratio=0.060182 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228744 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156761480 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313294216 weight_movement_bytes=18874368 total_nvlink_bytes=332168584 weight_to_token_ratio=0.060245 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229992 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549736 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312869480 weight_movement_bytes=18874368 total_nvlink_bytes=331743848 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38254 dup_rows=21782 dup_rows_remote=18920 dedup_ratio=0.332367 router_tokens_per_expert=[248, 280, 252, 279, 232, 289, 245, 238, 294, 260, 266, 268, 273, 251, 274, 243, 248, 264, 253, 271, 274, 260, 270, 270, 246, 273, 269, 220, 272, 262, 253, 276, 265, 252, 248, 268, 257, 267, 253, 273, 253, 245, 253, 238, 257, 237, 261, 264, 264, 221, 236, 241, 248, 275, 267, 246, 261, 265, 269, 278, 241, 241, 242, 214, 263, 227, 261, 224, 234, 279, 240, 261, 244, 248, 260, 247, 261, 253, 277, 255, 258, 242, 281, 248, 244, 268, 264, 262, 249, 270, 240, 271, 277, 277, 245, 239, 274, 253, 250, 249, 247, 244, 256, 285, 251, 264, 264, 243, 270, 286, 239, 284, 261, 278, 252, 256, 267, 249, 240, 224, 281, 239, 245, 243, 250, 214, 225, 243] payload_rows_per_dst=[5548, 5500, 5506, 5422, 5406, 5538, 5536, 5298] dup_rows_per_dst=[2836, 2862, 2676, 2596, 2662, 2732, 2782, 2636] combine_rows_per_src=[5548, 5500, 5506, 5422, 5406, 5538, 5536, 5298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228240 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156826512 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313424784 weight_movement_bytes=37748736 total_nvlink_bytes=351173520 weight_to_token_ratio=0.120440 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228696 combine_payload_bytes=156688384 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156917080 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605464 weight_movement_bytes=18874368 total_nvlink_bytes=332479832 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38140 dup_rows=21998 dup_rows_remote=19290 dedup_ratio=0.335663 router_tokens_per_expert=[246, 241, 252, 261, 258, 301, 258, 238, 242, 286, 258, 247, 243, 245, 244, 263, 260, 249, 274, 264, 265, 257, 264, 260, 263, 222, 235, 267, 246, 243, 267, 238, 261, 271, 266, 232, 253, 278, 244, 237, 257, 227, 245, 262, 260, 255, 251, 254, 274, 286, 274, 282, 264, 274, 257, 267, 291, 263, 294, 276, 252, 253, 250, 250, 244, 265, 252, 271, 263, 270, 247, 223, 234, 261, 252, 264, 254, 233, 238, 257, 262, 254, 280, 262, 259, 241, 228, 262, 251, 257, 256, 263, 271, 253, 254, 247, 251, 246, 295, 231, 263, 295, 250, 266, 242, 230, 255, 250, 259, 260, 251, 237, 244, 262, 241, 257, 237, 261, 241, 233, 276, 232, 263, 250, 257, 274, 264, 250] payload_rows_per_dst=[5418, 5458, 5398, 5558, 5372, 5410, 5474, 5450] dup_rows_per_dst=[2748, 2690, 2708, 3056, 2684, 2790, 2688, 2634] combine_rows_per_src=[5418, 5458, 5398, 5558, 5372, 5410, 5474, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38282 dup_rows=21690 dup_rows_remote=18978 dedup_ratio=0.330963 router_tokens_per_expert=[275, 253, 250, 261, 221, 250, 250, 255, 254, 239, 268, 271, 272, 230, 269, 268, 292, 256, 220, 243, 227, 239, 280, 270, 284, 245, 280, 267, 263, 246, 256, 271, 246, 247, 240, 256, 250, 269, 253, 246, 252, 268, 227, 262, 232, 247, 241, 279, 272, 260, 262, 282, 257, 285, 274, 256, 249, 241, 243, 252, 256, 244, 258, 260, 246, 248, 254, 227, 250, 253, 254, 236, 270, 277, 252, 281, 233, 265, 257, 292, 272, 270, 278, 257, 259, 254, 244, 247, 260, 237, 254, 261, 248, 229, 261, 243, 253, 254, 233, 261, 251, 254, 247, 294, 290, 277, 255, 256, 257, 233, 259, 256, 250, 257, 272, 252, 250, 228, 276, 266, 253, 259, 261, 256, 234, 234, 250, 280] payload_rows_per_dst=[5564, 5490, 5402, 5596, 5468, 5442, 5422, 5462] dup_rows_per_dst=[2712, 2790, 2626, 2768, 2722, 2706, 2776, 2590] combine_rows_per_src=[5564, 5490, 5402, 5596, 5468, 5442, 5422, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38338 dup_rows=21722 dup_rows_remote=18992 dedup_ratio=0.331451 router_tokens_per_expert=[267, 285, 234, 279, 277, 291, 239, 250, 250, 244, 254, 271, 268, 248, 262, 285, 263, 262, 276, 246, 270, 243, 297, 249, 235, 227, 264, 260, 249, 251, 263, 259, 257, 263, 243, 266, 244, 228, 269, 249, 260, 266, 260, 261, 256, 236, 281, 250, 223, 258, 262, 236, 280, 273, 256, 253, 262, 272, 210, 255, 252, 252, 218, 259, 241, 280, 259, 251, 259, 238, 276, 262, 241, 248, 242, 238, 240, 255, 253, 279, 246, 251, 239, 257, 232, 249, 267, 275, 243, 245, 246, 242, 228, 261, 238, 274, 232, 270, 256, 261, 249, 252, 248, 270, 258, 272, 261, 265, 269, 250, 243, 247, 268, 280, 249, 260, 241, 261, 253, 249, 255, 291, 264, 297, 265, 244, 243, 262] payload_rows_per_dst=[5554, 5502, 5488, 5358, 5472, 5436, 5476, 5528] dup_rows_per_dst=[2854, 2726, 2690, 2684, 2652, 2550, 2730, 2836] combine_rows_per_src=[5554, 5502, 5488, 5358, 5472, 5436, 5476, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38334 dup_rows=21774 dup_rows_remote=19094 dedup_ratio=0.332245 router_tokens_per_expert=[245, 257, 250, 276, 248, 264, 245, 250, 279, 254, 245, 250, 249, 276, 258, 227, 249, 252, 252, 258, 247, 237, 282, 243, 251, 246, 220, 282, 271, 248, 269, 240, 266, 262, 289, 258, 268, 277, 290, 250, 254, 220, 262, 261, 263, 233, 277, 245, 279, 269, 256, 257, 250, 257, 263, 248, 259, 244, 254, 291, 232, 229, 261, 242, 262, 269, 270, 266, 290, 283, 270, 233, 263, 241, 247, 268, 266, 258, 255, 251, 258, 259, 240, 254, 270, 271, 243, 268, 238, 268, 252, 257, 263, 255, 246, 250, 245, 251, 259, 243, 270, 241, 253, 239, 271, 259, 248, 254, 237, 249, 274, 265, 251, 270, 245, 242, 242, 256, 226, 266, 248, 252, 259, 252, 274, 261, 261, 235] payload_rows_per_dst=[5388, 5392, 5518, 5496, 5524, 5498, 5518, 5428] dup_rows_per_dst=[2664, 2586, 2832, 2648, 2870, 2802, 2692, 2680] combine_rows_per_src=[5388, 5392, 5518, 5496, 5524, 5498, 5518, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38114 dup_rows=21860 dup_rows_remote=19148 dedup_ratio=0.333557 router_tokens_per_expert=[239, 257, 248, 243, 286, 289, 228, 252, 260, 231, 247, 244, 240, 256, 254, 234, 268, 252, 269, 256, 273, 236, 262, 234, 277, 254, 273, 237, 268, 254, 233, 284, 263, 244, 232, 250, 248, 277, 248, 278, 272, 248, 265, 246, 212, 246, 266, 258, 279, 270, 253, 265, 273, 280, 246, 264, 239, 257, 236, 248, 268, 254, 247, 258, 235, 252, 269, 254, 247, 272, 267, 259, 271, 255, 270, 251, 261, 264, 248, 265, 230, 246, 253, 244, 248, 257, 275, 285, 282, 259, 247, 248, 248, 245, 261, 230, 273, 236, 252, 226, 253, 291, 244, 254, 254, 257, 255, 266, 244, 278, 264, 258, 275, 276, 254, 252, 234, 248, 222, 284, 233, 261, 265, 276, 261, 285, 249, 262] payload_rows_per_dst=[5436, 5450, 5440, 5562, 5452, 5434, 5448, 5454] dup_rows_per_dst=[2580, 2810, 2666, 2712, 2828, 2682, 2762, 2820] combine_rows_per_src=[5436, 5450, 5440, 5562, 5452, 5434, 5448, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38242 dup_rows=21682 dup_rows_remote=18874 dedup_ratio=0.330841 router_tokens_per_expert=[238, 233, 244, 279, 266, 269, 282, 237, 249, 220, 259, 251, 257, 243, 242, 281, 236, 255, 247, 235, 260, 261, 241, 256, 269, 257, 269, 282, 236, 266, 247, 303, 238, 285, 268, 205, 241, 258, 257, 261, 254, 254, 260, 250, 255, 242, 251, 274, 260, 250, 265, 237, 268, 242, 239, 248, 275, 275, 256, 245, 272, 245, 254, 245, 255, 290, 256, 240, 255, 283, 225, 261, 247, 249, 262, 275, 257, 240, 248, 248, 245, 248, 282, 249, 263, 281, 280, 276, 242, 247, 268, 260, 253, 278, 273, 265, 230, 258, 243, 244, 236, 236, 248, 249, 273, 268, 274, 271, 231, 283, 279, 242, 278, 270, 257, 267, 256, 259, 236, 244, 283, 251, 257, 237, 281, 268, 239, 220] payload_rows_per_dst=[5418, 5514, 5408, 5512, 5474, 5612, 5462, 5454] dup_rows_per_dst=[2682, 2726, 2698, 2640, 2708, 2808, 2668, 2752] combine_rows_per_src=[5418, 5514, 5408, 5512, 5474, 5612, 5462, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38178 dup_rows=21806 dup_rows_remote=19020 dedup_ratio=0.332733 router_tokens_per_expert=[257, 255, 254, 284, 210, 271, 244, 262, 244, 247, 270, 240, 251, 248, 284, 261, 244, 269, 265, 259, 233, 267, 241, 242, 243, 243, 223, 275, 261, 246, 274, 256, 272, 245, 248, 257, 251, 283, 259, 266, 250, 239, 266, 242, 260, 259, 276, 259, 216, 261, 246, 242, 251, 260, 237, 268, 243, 269, 274, 273, 261, 242, 255, 264, 284, 245, 261, 275, 260, 243, 269, 260, 279, 274, 269, 244, 243, 262, 261, 240, 251, 247, 257, 266, 253, 248, 236, 241, 258, 272, 252, 266, 266, 251, 245, 261, 256, 221, 259, 280, 256, 264, 246, 270, 257, 253, 268, 263, 268, 247, 249, 262, 243, 233, 248, 244, 257, 246, 250, 293, 266, 254, 276, 232, 284, 253, 247, 267] payload_rows_per_dst=[5410, 5358, 5450, 5490, 5552, 5468, 5522, 5480] dup_rows_per_dst=[2754, 2724, 2814, 2634, 2786, 2672, 2716, 2706] combine_rows_per_src=[5410, 5358, 5450, 5490, 5552, 5468, 5522, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=229720 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156451160 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312672600 weight_movement_bytes=18874368 total_nvlink_bytes=331546968 weight_to_token_ratio=0.060365 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229040 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032112 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835184 weight_movement_bytes=37748736 total_nvlink_bytes=351583920 weight_to_token_ratio=0.120282 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229320 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261768 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294216 weight_movement_bytes=18874368 total_nvlink_bytes=333168584 weight_to_token_ratio=0.060053 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229712 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245776 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261840 weight_movement_bytes=18874368 total_nvlink_bytes=333136208 weight_to_token_ratio=0.060059 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=229048 combine_payload_bytes=156114944 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156343992 ingress_bytes=231612416 link_bytes=231612416 token_bytes=312458936 weight_movement_bytes=75497472 total_nvlink_bytes=387956408 weight_to_token_ratio=0.241624 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219196210e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38268 dup_rows=21770 dup_rows_remote=19020 dedup_ratio=0.332184 router_tokens_per_expert=[233, 259, 267, 230, 253, 248, 236, 243, 264, 255, 252, 278, 263, 259, 240, 263, 271, 238, 263, 243, 264, 237, 274, 255, 235, 252, 250, 280, 262, 267, 277, 256, 251, 250, 251, 253, 260, 258, 275, 288, 249, 250, 262, 249, 254, 274, 250, 271, 268, 232, 255, 250, 258, 257, 246, 267, 267, 257, 247, 248, 267, 287, 246, 246, 257, 271, 256, 270, 262, 217, 257, 276, 258, 263, 275, 237, 258, 276, 267, 238, 226, 268, 230, 274, 230, 244, 246, 226, 250, 262, 244, 270, 251, 267, 263, 261, 225, 255, 271, 233, 246, 252, 261, 264, 243, 254, 292, 258, 269, 241, 243, 278, 260, 259, 263, 248, 236, 272, 250, 243, 279, 265, 247, 243, 260, 268, 265, 265] payload_rows_per_dst=[5426, 5498, 5518, 5464, 5536, 5412, 5410, 5502] dup_rows_per_dst=[2660, 2750, 2772, 2732, 2740, 2612, 2760, 2744] combine_rows_per_src=[5426, 5498, 5518, 5464, 5536, 5412, 5410, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228464 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156867696 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313506928 weight_movement_bytes=18874368 total_nvlink_bytes=332381296 weight_to_token_ratio=0.060204 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228792 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156605880 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312982968 weight_movement_bytes=18874368 total_nvlink_bytes=331857336 weight_to_token_ratio=0.060305 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229152 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156974880 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720608 weight_movement_bytes=18874368 total_nvlink_bytes=332594976 weight_to_token_ratio=0.060163 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38400 dup_rows=21762 dup_rows_remote=19066 dedup_ratio=0.332062 router_tokens_per_expert=[243, 264, 260, 251, 252, 249, 241, 243, 248, 241, 284, 250, 263, 242, 248, 239, 235, 228, 266, 244, 245, 241, 246, 253, 242, 237, 263, 260, 232, 255, 264, 227, 272, 271, 245, 275, 252, 235, 263, 266, 217, 258, 253, 231, 232, 247, 266, 252, 257, 261, 259, 285, 245, 250, 269, 276, 259, 234, 243, 251, 247, 260, 244, 270, 278, 275, 271, 245, 260, 269, 250, 255, 259, 251, 256, 260, 248, 264, 284, 274, 272, 249, 234, 250, 263, 259, 260, 269, 252, 242, 268, 247, 238, 269, 238, 276, 253, 280, 269, 231, 267, 232, 271, 269, 269, 271, 259, 242, 252, 249, 254, 254, 292, 266, 248, 304, 270, 273, 259, 253, 280, 255, 285, 248, 240, 266, 263, 258] payload_rows_per_dst=[5372, 5336, 5374, 5488, 5530, 5500, 5520, 5654] dup_rows_per_dst=[2664, 2540, 2696, 2732, 2868, 2672, 2724, 2866] combine_rows_per_src=[5372, 5336, 5374, 5488, 5530, 5500, 5520, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38262 dup_rows=21784 dup_rows_remote=19008 dedup_ratio=0.332397 router_tokens_per_expert=[258, 270, 263, 242, 256, 250, 269, 247, 303, 238, 283, 263, 266, 248, 225, 251, 261, 280, 235, 230, 268, 260, 253, 254, 243, 239, 230, 246, 267, 234, 246, 292, 246, 235, 248, 267, 252, 280, 293, 260, 266, 229, 267, 274, 242, 293, 236, 241, 267, 255, 238, 238, 259, 277, 266, 270, 256, 239, 277, 263, 252, 233, 279, 238, 269, 255, 255, 239, 263, 248, 245, 278, 251, 272, 221, 255, 251, 251, 252, 270, 265, 253, 289, 263, 254, 239, 241, 263, 241, 288, 258, 263, 227, 287, 261, 251, 246, 247, 234, 262, 259, 251, 261, 247, 251, 266, 234, 251, 282, 254, 266, 267, 233, 252, 246, 248, 265, 259, 263, 262, 267, 265, 251, 251, 261, 244, 271, 228] payload_rows_per_dst=[5490, 5446, 5492, 5444, 5472, 5524, 5476, 5408] dup_rows_per_dst=[2776, 2630, 2766, 2784, 2678, 2762, 2664, 2724] combine_rows_per_src=[5490, 5446, 5492, 5444, 5472, 5524, 5476, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38352 dup_rows=21766 dup_rows_remote=19000 dedup_ratio=0.332123 router_tokens_per_expert=[281, 251, 259, 238, 256, 244, 237, 242, 262, 266, 252, 238, 246, 262, 250, 254, 278, 257, 261, 245, 250, 261, 245, 248, 257, 226, 274, 254, 258, 272, 240, 245, 258, 251, 259, 236, 264, 264, 263, 280, 250, 263, 240, 277, 233, 248, 242, 259, 257, 247, 276, 276, 271, 249, 263, 254, 247, 232, 255, 267, 288, 261, 256, 256, 237, 264, 282, 263, 257, 239, 262, 272, 248, 237, 250, 261, 246, 241, 275, 238, 267, 305, 256, 270, 233, 235, 260, 242, 273, 261, 251, 244, 262, 270, 239, 255, 274, 234, 259, 271, 265, 256, 276, 266, 243, 234, 280, 255, 237, 250, 232, 260, 269, 261, 265, 249, 261, 232, 244, 259, 248, 235, 267, 282, 254, 258, 270, 276] payload_rows_per_dst=[5426, 5496, 5408, 5538, 5504, 5416, 5418, 5564] dup_rows_per_dst=[2650, 2646, 2766, 2772, 2640, 2830, 2766, 2696] combine_rows_per_src=[5426, 5496, 5408, 5538, 5504, 5416, 5418, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38372 dup_rows=21750 dup_rows_remote=19004 dedup_ratio=0.331879 router_tokens_per_expert=[229, 253, 236, 271, 273, 247, 273, 248, 244, 263, 242, 268, 287, 240, 258, 235, 255, 248, 252, 254, 270, 263, 254, 290, 237, 272, 253, 278, 247, 236, 249, 246, 245, 265, 258, 276, 246, 227, 257, 277, 247, 273, 277, 272, 257, 256, 272, 252, 268, 242, 260, 248, 239, 250, 238, 250, 251, 243, 227, 265, 248, 263, 248, 275, 254, 249, 278, 268, 271, 232, 281, 277, 261, 248, 293, 250, 244, 242, 253, 256, 263, 237, 257, 253, 242, 273, 249, 251, 273, 272, 260, 255, 229, 258, 248, 256, 266, 234, 269, 279, 265, 282, 256, 247, 274, 271, 235, 260, 259, 261, 242, 264, 263, 233, 240, 248, 259, 249, 255, 270, 224, 280, 255, 256, 261, 263, 227, 245] payload_rows_per_dst=[5418, 5494, 5510, 5424, 5526, 5462, 5538, 5414] dup_rows_per_dst=[2716, 2714, 2784, 2606, 2758, 2692, 2734, 2746] combine_rows_per_src=[5418, 5494, 5510, 5424, 5526, 5462, 5538, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38374 dup_rows=21784 dup_rows_remote=19154 dedup_ratio=0.332397 router_tokens_per_expert=[260, 249, 244, 278, 267, 256, 264, 251, 230, 257, 268, 269, 260, 281, 288, 265, 256, 265, 273, 258, 230, 248, 265, 260, 282, 259, 237, 243, 210, 256, 252, 249, 252, 240, 251, 214, 259, 259, 259, 259, 247, 266, 279, 253, 257, 285, 257, 280, 249, 247, 229, 244, 258, 264, 280, 273, 234, 232, 227, 281, 240, 260, 242, 244, 288, 245, 268, 238, 271, 267, 239, 258, 261, 271, 248, 244, 241, 232, 262, 252, 237, 265, 260, 258, 276, 262, 284, 259, 255, 279, 272, 247, 255, 262, 273, 267, 259, 272, 270, 250, 269, 258, 259, 245, 242, 261, 238, 262, 273, 269, 248, 245, 284, 264, 218, 240, 242, 226, 249, 256, 254, 242, 266, 265, 255, 245, 250, 245] payload_rows_per_dst=[5618, 5454, 5470, 5378, 5428, 5568, 5470, 5366] dup_rows_per_dst=[2756, 2632, 2764, 2630, 2742, 2854, 2770, 2636] combine_rows_per_src=[5618, 5454, 5470, 5378, 5428, 5568, 5470, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38094 dup_rows=21878 dup_rows_remote=19226 dedup_ratio=0.333832 router_tokens_per_expert=[279, 260, 255, 280, 235, 237, 250, 257, 245, 236, 232, 212, 246, 265, 240, 273, 255, 252, 232, 296, 283, 257, 248, 258, 246, 267, 289, 270, 260, 261, 257, 255, 227, 271, 219, 251, 246, 231, 253, 237, 264, 261, 259, 226, 244, 282, 287, 236, 269, 263, 295, 254, 253, 267, 264, 268, 259, 247, 284, 240, 234, 239, 254, 286, 237, 250, 257, 244, 250, 275, 255, 255, 272, 251, 257, 246, 249, 250, 272, 252, 278, 285, 258, 264, 263, 256, 243, 233, 257, 237, 258, 245, 237, 242, 284, 268, 280, 273, 260, 273, 275, 264, 226, 256, 249, 315, 235, 257, 227, 264, 269, 234, 268, 241, 227, 270, 237, 264, 268, 255, 279, 256, 268, 246, 241, 260, 246, 247] payload_rows_per_dst=[5332, 5554, 5328, 5506, 5430, 5564, 5472, 5472] dup_rows_per_dst=[2672, 2818, 2660, 2846, 2714, 2652, 2842, 2674] combine_rows_per_src=[5332, 5554, 5328, 5506, 5430, 5564, 5472, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38240 dup_rows=21758 dup_rows_remote=18904 dedup_ratio=0.332001 router_tokens_per_expert=[286, 271, 259, 242, 268, 230, 239, 260, 244, 251, 240, 255, 281, 259, 258, 251, 255, 243, 268, 268, 258, 236, 253, 249, 253, 234, 270, 229, 270, 252, 281, 261, 264, 259, 261, 259, 251, 254, 246, 261, 287, 265, 260, 262, 261, 236, 244, 259, 284, 273, 244, 266, 233, 253, 248, 258, 277, 213, 251, 236, 249, 255, 226, 249, 269, 249, 247, 272, 280, 276, 249, 255, 259, 262, 293, 258, 251, 272, 261, 243, 251, 255, 270, 235, 257, 243, 235, 268, 250, 231, 256, 245, 258, 266, 279, 259, 229, 243, 276, 268, 259, 240, 270, 242, 255, 282, 274, 270, 253, 283, 240, 224, 275, 272, 236, 232, 252, 251, 249, 250, 281, 254, 232, 253, 241, 289, 271, 250] payload_rows_per_dst=[5484, 5512, 5466, 5386, 5538, 5488, 5486, 5418] dup_rows_per_dst=[2704, 2648, 2792, 2644, 2854, 2628, 2730, 2758] combine_rows_per_src=[5484, 5512, 5466, 5386, 5538, 5488, 5486, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229864 combine_payload_bytes=157286400 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157516264 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802664 weight_movement_bytes=37748736 total_nvlink_bytes=352551400 weight_to_token_ratio=0.119912 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229080 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=94371840 egress_bytes=156950232 ingress_bytes=269967360 link_bytes=269967360 token_bytes=313671384 weight_movement_bytes=113246208 total_nvlink_bytes=426917592 weight_to_token_ratio=0.361035 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.586694411e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229408 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157319200 ingress_bytes=213712896 link_bytes=213712896 token_bytes=314408992 weight_movement_bytes=56623104 total_nvlink_bytes=371032096 weight_to_token_ratio=0.180094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047691816e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229504 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157401216 ingress_bytes=194920448 link_bytes=194920448 token_bytes=314572928 weight_movement_bytes=37748736 total_nvlink_bytes=352321664 weight_to_token_ratio=0.120000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867631826e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38278 dup_rows=21826 dup_rows_remote=19214 dedup_ratio=0.333038 router_tokens_per_expert=[264, 265, 277, 257, 239, 261, 273, 258, 259, 251, 241, 258, 268, 254, 244, 255, 239, 265, 232, 258, 253, 259, 238, 247, 269, 249, 257, 251, 260, 250, 247, 248, 237, 263, 252, 262, 239, 261, 252, 249, 249, 269, 264, 236, 221, 252, 237, 253, 267, 234, 271, 251, 256, 270, 256, 287, 265, 223, 273, 257, 246, 248, 267, 251, 270, 232, 266, 255, 238, 272, 267, 240, 249, 252, 275, 284, 250, 258, 254, 269, 278, 277, 263, 255, 240, 246, 253, 265, 256, 247, 250, 233, 251, 273, 252, 282, 261, 261, 251, 245, 278, 258, 260, 247, 275, 241, 260, 236, 259, 246, 271, 255, 244, 240, 240, 262, 254, 271, 279, 262, 281, 276, 274, 263, 242, 258, 250, 252] payload_rows_per_dst=[5562, 5432, 5436, 5390, 5458, 5476, 5470, 5486] dup_rows_per_dst=[2686, 2612, 2556, 2854, 2804, 2766, 2738, 2810] combine_rows_per_src=[5562, 5432, 5436, 5390, 5458, 5476, 5470, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=230112 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157410016 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589920 weight_movement_bytes=18874368 total_nvlink_bytes=333464288 weight_to_token_ratio=0.059997 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228576 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156859616 ingress_bytes=213254144 link_bytes=213254144 token_bytes=313490656 weight_movement_bytes=56623104 total_nvlink_bytes=370113760 weight_to_token_ratio=0.180621 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043296280e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=229280 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156262304 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312295328 weight_movement_bytes=18874368 total_nvlink_bytes=331169696 weight_to_token_ratio=0.060438 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229968 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016656 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803344 weight_movement_bytes=37748736 total_nvlink_bytes=351552080 weight_to_token_ratio=0.120294 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38410 dup_rows=21650 dup_rows_remote=18984 dedup_ratio=0.330353 router_tokens_per_expert=[243, 262, 240, 251, 260, 267, 242, 245, 242, 285, 242, 268, 245, 263, 263, 264, 271, 239, 235, 254, 243, 295, 250, 262, 241, 244, 241, 246, 296, 274, 265, 251, 248, 246, 288, 261, 256, 249, 267, 246, 258, 261, 244, 248, 261, 265, 239, 234, 249, 243, 232, 254, 263, 250, 257, 248, 290, 253, 275, 242, 267, 252, 236, 247, 237, 272, 261, 226, 232, 262, 263, 240, 276, 210, 285, 262, 265, 274, 283, 258, 231, 259, 241, 257, 267, 259, 262, 261, 252, 268, 269, 263, 266, 283, 247, 268, 270, 271, 265, 266, 242, 247, 261, 243, 273, 218, 267, 253, 248, 240, 254, 265, 247, 252, 267, 238, 247, 282, 242, 265, 251, 255, 265, 248, 261, 274, 254, 260] payload_rows_per_dst=[5504, 5484, 5476, 5462, 5460, 5484, 5450, 5566] dup_rows_per_dst=[2660, 2730, 2666, 2654, 2752, 2822, 2716, 2650] combine_rows_per_src=[5504, 5484, 5476, 5462, 5460, 5484, 5450, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38242 dup_rows=21858 dup_rows_remote=19088 dedup_ratio=0.333527 router_tokens_per_expert=[246, 293, 253, 230, 256, 265, 237, 281, 266, 257, 264, 264, 253, 246, 262, 251, 257, 268, 248, 242, 269, 250, 287, 255, 250, 258, 260, 241, 277, 256, 247, 250, 233, 250, 239, 271, 283, 264, 224, 250, 252, 256, 242, 251, 256, 259, 246, 264, 290, 256, 268, 251, 272, 269, 284, 222, 277, 250, 252, 273, 244, 258, 245, 288, 256, 255, 273, 273, 235, 239, 282, 255, 264, 259, 272, 256, 261, 256, 266, 251, 251, 247, 285, 258, 261, 279, 224, 252, 257, 257, 236, 253, 254, 262, 258, 247, 276, 236, 269, 249, 276, 255, 277, 257, 231, 226, 256, 255, 237, 275, 256, 272, 263, 262, 263, 225, 235, 251, 268, 242, 236, 239, 260, 256, 236, 234, 238, 245] payload_rows_per_dst=[5498, 5516, 5440, 5528, 5476, 5466, 5436, 5318] dup_rows_per_dst=[2750, 2714, 2640, 2870, 2830, 2696, 2770, 2588] combine_rows_per_src=[5498, 5516, 5440, 5528, 5476, 5466, 5436, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38282 dup_rows=21668 dup_rows_remote=18796 dedup_ratio=0.330627 router_tokens_per_expert=[246, 279, 265, 282, 276, 247, 251, 237, 263, 260, 269, 275, 266, 266, 284, 260, 257, 246, 254, 279, 274, 254, 246, 246, 228, 272, 271, 225, 289, 266, 260, 268, 248, 227, 241, 274, 275, 250, 225, 255, 231, 273, 274, 262, 246, 285, 264, 246, 236, 222, 250, 265, 257, 250, 241, 260, 250, 257, 254, 236, 252, 275, 252, 247, 261, 260, 257, 235, 251, 247, 265, 247, 261, 263, 270, 233, 250, 254, 247, 242, 250, 264, 274, 238, 247, 287, 253, 265, 262, 245, 278, 232, 255, 274, 258, 252, 264, 248, 262, 271, 268, 223, 260, 261, 245, 274, 264, 235, 262, 257, 254, 280, 244, 253, 219, 263, 243, 254, 249, 239, 218, 251, 286, 242, 274, 253, 268, 266] payload_rows_per_dst=[5586, 5574, 5514, 5384, 5360, 5510, 5518, 5422] dup_rows_per_dst=[2872, 2690, 2638, 2624, 2726, 2758, 2738, 2622] combine_rows_per_src=[5586, 5574, 5514, 5384, 5360, 5510, 5518, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38042 dup_rows=21812 dup_rows_remote=18776 dedup_ratio=0.332825 router_tokens_per_expert=[256, 252, 257, 262, 271, 264, 250, 258, 281, 277, 240, 258, 283, 250, 253, 256, 256, 278, 252, 258, 249, 256, 245, 246, 257, 263, 262, 284, 266, 250, 272, 252, 258, 263, 257, 272, 259, 236, 261, 262, 280, 268, 247, 277, 263, 259, 248, 248, 237, 268, 259, 277, 259, 253, 284, 265, 243, 244, 257, 256, 250, 236, 249, 250, 249, 243, 254, 264, 246, 262, 248, 255, 247, 241, 259, 241, 267, 247, 261, 261, 259, 265, 232, 245, 250, 273, 233, 241, 274, 240, 222, 272, 284, 222, 247, 229, 265, 249, 254, 266, 283, 278, 275, 247, 253, 241, 248, 240, 253, 243, 270, 235, 232, 270, 256, 268, 272, 264, 248, 263, 267, 255, 220, 245, 249, 246, 253, 268] payload_rows_per_dst=[5438, 5376, 5620, 5182, 5430, 5472, 5524, 5682] dup_rows_per_dst=[2592, 2632, 3002, 2398, 2660, 2788, 2704, 3036] combine_rows_per_src=[5438, 5376, 5620, 5182, 5430, 5472, 5524, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38144 dup_rows=21938 dup_rows_remote=19136 dedup_ratio=0.334747 router_tokens_per_expert=[269, 256, 285, 243, 281, 245, 257, 277, 248, 259, 256, 250, 268, 249, 247, 264, 254, 279, 270, 244, 260, 246, 253, 236, 276, 257, 262, 270, 269, 261, 289, 252, 277, 244, 267, 234, 237, 256, 262, 239, 291, 250, 253, 242, 247, 259, 262, 249, 251, 263, 209, 264, 267, 242, 299, 274, 259, 249, 252, 251, 254, 261, 269, 264, 289, 249, 275, 275, 279, 240, 245, 249, 270, 271, 230, 259, 247, 266, 231, 250, 236, 268, 259, 237, 275, 262, 251, 253, 236, 260, 237, 255, 249, 246, 256, 266, 237, 279, 249, 237, 261, 261, 250, 254, 270, 241, 236, 251, 271, 240, 249, 270, 226, 240, 258, 258, 254, 239, 230, 247, 228, 258, 285, 276, 258, 248, 242, 265] payload_rows_per_dst=[5500, 5542, 5402, 5454, 5486, 5448, 5420, 5346] dup_rows_per_dst=[2808, 2814, 2736, 2802, 2764, 2644, 2692, 2678] combine_rows_per_src=[5500, 5542, 5402, 5454, 5486, 5448, 5420, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229576 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157556936 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314884296 weight_movement_bytes=18874368 total_nvlink_bytes=333758664 weight_to_token_ratio=0.059941 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229320 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156868552 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313507784 weight_movement_bytes=37748736 total_nvlink_bytes=351256520 weight_to_token_ratio=0.120408 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38332 dup_rows=21712 dup_rows_remote=19100 dedup_ratio=0.331299 router_tokens_per_expert=[252, 266, 275, 274, 263, 241, 261, 253, 248, 251, 240, 269, 249, 250, 255, 238, 237, 255, 259, 250, 244, 239, 268, 276, 246, 251, 239, 254, 240, 264, 235, 276, 240, 277, 299, 267, 250, 240, 260, 261, 234, 246, 279, 227, 252, 249, 242, 281, 251, 258, 289, 270, 258, 275, 271, 287, 229, 234, 256, 274, 277, 264, 257, 278, 278, 230, 260, 251, 244, 269, 259, 245, 253, 262, 246, 261, 261, 244, 226, 263, 246, 262, 249, 251, 265, 266, 259, 265, 260, 251, 232, 259, 240, 264, 245, 257, 283, 271, 256, 263, 283, 244, 268, 271, 252, 260, 246, 263, 240, 269, 271, 245, 261, 237, 239, 247, 244, 251, 252, 270, 232, 258, 275, 203, 235, 265, 285, 256] payload_rows_per_dst=[5446, 5408, 5450, 5658, 5492, 5422, 5542, 5406] dup_rows_per_dst=[2724, 2658, 2758, 2798, 2612, 2720, 2828, 2614] combine_rows_per_src=[5446, 5408, 5450, 5658, 5492, 5422, 5542, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=227272 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156047304 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311867336 weight_movement_bytes=18874368 total_nvlink_bytes=330741704 weight_to_token_ratio=0.060521 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38304 dup_rows=21896 dup_rows_remote=19222 dedup_ratio=0.334106 router_tokens_per_expert=[267, 232, 286, 228, 265, 244, 246, 260, 259, 261, 235, 252, 278, 276, 230, 271, 261, 254, 249, 274, 252, 266, 286, 272, 255, 272, 256, 242, 264, 257, 272, 229, 272, 258, 264, 241, 250, 282, 258, 271, 220, 261, 287, 248, 252, 253, 264, 231, 276, 302, 264, 244, 261, 264, 253, 260, 237, 265, 277, 270, 243, 275, 263, 261, 260, 267, 247, 256, 227, 268, 248, 239, 251, 258, 238, 253, 209, 238, 281, 242, 266, 253, 245, 241, 274, 230, 251, 260, 280, 254, 258, 232, 201, 256, 254, 250, 248, 235, 251, 257, 244, 250, 278, 257, 263, 280, 265, 261, 252, 269, 243, 249, 238, 266, 273, 257, 237, 265, 261, 252, 234, 242, 257, 256, 257, 266, 260, 280] payload_rows_per_dst=[5514, 5388, 5480, 5626, 5354, 5336, 5488, 5454] dup_rows_per_dst=[2666, 2934, 2744, 2804, 2610, 2674, 2716, 2748] combine_rows_per_src=[5514, 5388, 5480, 5626, 5354, 5336, 5488, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=228312 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157031384 ingress_bytes=232300544 link_bytes=232300544 token_bytes=313834456 weight_movement_bytes=75497472 total_nvlink_bytes=389331928 weight_to_token_ratio=0.240565 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225789514e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38286 dup_rows=21644 dup_rows_remote=18922 dedup_ratio=0.330261 router_tokens_per_expert=[218, 300, 234, 256, 234, 303, 232, 244, 254, 258, 252, 280, 268, 272, 261, 253, 262, 261, 259, 268, 231, 258, 256, 244, 254, 282, 277, 248, 269, 281, 250, 264, 251, 271, 239, 250, 262, 261, 274, 283, 253, 238, 248, 267, 252, 231, 234, 244, 256, 239, 263, 251, 243, 277, 249, 271, 266, 255, 269, 257, 259, 270, 238, 240, 253, 232, 273, 258, 255, 243, 232, 248, 271, 250, 259, 235, 250, 257, 267, 220, 272, 246, 279, 268, 272, 241, 254, 234, 256, 276, 274, 236, 229, 256, 263, 258, 255, 267, 255, 246, 249, 259, 251, 270, 228, 266, 275, 284, 286, 247, 276, 251, 261, 270, 250, 248, 240, 243, 245, 246, 271, 260, 252, 245, 264, 261, 236, 250] payload_rows_per_dst=[5464, 5606, 5486, 5480, 5454, 5500, 5486, 5416] dup_rows_per_dst=[2774, 2722, 2630, 2726, 2552, 2728, 2844, 2668] combine_rows_per_src=[5464, 5606, 5486, 5480, 5454, 5500, 5486, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229120 combine_payload_bytes=156237824 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156466944 ingress_bytes=231735296 link_bytes=231735296 token_bytes=312704768 weight_movement_bytes=75497472 total_nvlink_bytes=388202240 weight_to_token_ratio=0.241434 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220373585e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229728 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237600 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245472 weight_movement_bytes=18874368 total_nvlink_bytes=333119840 weight_to_token_ratio=0.060062 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=230104 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157123288 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314016472 weight_movement_bytes=18874368 total_nvlink_bytes=332890840 weight_to_token_ratio=0.060106 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=228832 combine_payload_bytes=156819456 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157048288 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313867744 weight_movement_bytes=18874368 total_nvlink_bytes=332742112 weight_to_token_ratio=0.060135 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38304 dup_rows=21686 dup_rows_remote=18942 dedup_ratio=0.330902 router_tokens_per_expert=[248, 258, 246, 250, 257, 286, 267, 255, 254, 257, 224, 253, 268, 237, 258, 249, 239, 255, 273, 274, 243, 253, 269, 238, 229, 222, 265, 248, 222, 250, 237, 242, 246, 252, 246, 270, 263, 274, 228, 282, 269, 263, 241, 269, 254, 279, 246, 263, 255, 248, 255, 241, 277, 224, 252, 232, 265, 226, 281, 275, 255, 243, 285, 258, 268, 292, 230, 240, 261, 256, 269, 242, 238, 245, 273, 241, 241, 278, 264, 263, 268, 248, 276, 278, 271, 241, 265, 268, 246, 236, 245, 233, 279, 241, 257, 253, 246, 270, 288, 258, 231, 265, 258, 274, 271, 224, 257, 239, 264, 263, 285, 248, 242, 273, 276, 249, 282, 277, 263, 247, 266, 256, 290, 263, 244, 249, 242, 259] payload_rows_per_dst=[5474, 5346, 5546, 5436, 5508, 5468, 5496, 5576] dup_rows_per_dst=[2660, 2572, 2744, 2708, 2694, 2742, 2786, 2780] combine_rows_per_src=[5474, 5346, 5546, 5436, 5508, 5468, 5496, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38242 dup_rows=21790 dup_rows_remote=19104 dedup_ratio=0.332489 router_tokens_per_expert=[290, 243, 262, 254, 251, 278, 253, 261, 250, 276, 249, 282, 240, 245, 261, 246, 273, 256, 271, 265, 279, 230, 269, 272, 247, 280, 245, 273, 270, 254, 248, 256, 287, 245, 256, 261, 283, 255, 240, 240, 241, 232, 245, 252, 249, 280, 222, 212, 240, 237, 243, 261, 269, 238, 255, 263, 266, 259, 255, 227, 249, 255, 256, 247, 257, 255, 265, 258, 259, 242, 245, 266, 278, 260, 262, 265, 250, 254, 239, 238, 248, 243, 256, 234, 271, 269, 275, 238, 249, 255, 268, 261, 258, 261, 255, 273, 262, 264, 237, 227, 244, 273, 265, 245, 255, 276, 268, 243, 272, 259, 252, 253, 267, 244, 233, 258, 265, 262, 259, 248, 304, 263, 235, 273, 272, 261, 224, 249] payload_rows_per_dst=[5490, 5598, 5342, 5404, 5486, 5362, 5504, 5560] dup_rows_per_dst=[2792, 2778, 2658, 2636, 2700, 2866, 2686, 2674] combine_rows_per_src=[5490, 5598, 5342, 5404, 5486, 5362, 5504, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38184 dup_rows=21860 dup_rows_remote=19056 dedup_ratio=0.333557 router_tokens_per_expert=[268, 263, 280, 275, 265, 239, 264, 231, 249, 252, 247, 258, 240, 275, 282, 267, 268, 275, 262, 245, 243, 255, 256, 250, 253, 236, 240, 248, 288, 259, 249, 257, 267, 260, 257, 239, 256, 248, 277, 252, 231, 271, 234, 248, 262, 260, 279, 265, 245, 270, 265, 248, 273, 262, 238, 247, 236, 256, 256, 265, 226, 265, 244, 232, 259, 245, 247, 286, 243, 247, 277, 277, 270, 257, 241, 240, 252, 258, 264, 271, 273, 241, 257, 228, 266, 270, 224, 256, 270, 270, 261, 226, 245, 269, 266, 244, 267, 246, 246, 261, 260, 252, 248, 238, 252, 246, 262, 247, 250, 276, 256, 250, 290, 299, 263, 270, 276, 262, 239, 278, 238, 269, 226, 250, 234, 238, 252, 254] payload_rows_per_dst=[5276, 5516, 5498, 5522, 5512, 5402, 5458, 5492] dup_rows_per_dst=[2430, 2652, 2694, 2910, 2870, 2730, 2770, 2804] combine_rows_per_src=[5276, 5516, 5498, 5522, 5512, 5402, 5458, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38186 dup_rows=21870 dup_rows_remote=19104 dedup_ratio=0.333710 router_tokens_per_expert=[278, 240, 280, 284, 255, 272, 248, 243, 252, 248, 242, 241, 264, 256, 249, 264, 248, 251, 235, 224, 289, 270, 268, 255, 262, 259, 273, 251, 269, 269, 249, 247, 280, 254, 228, 274, 282, 250, 250, 262, 255, 256, 260, 253, 274, 252, 231, 247, 246, 282, 263, 262, 257, 274, 242, 249, 265, 265, 261, 241, 249, 257, 232, 260, 250, 265, 260, 252, 257, 283, 241, 237, 236, 255, 254, 256, 249, 239, 262, 251, 258, 270, 251, 264, 229, 265, 230, 272, 257, 253, 231, 250, 250, 242, 268, 262, 251, 227, 264, 214, 231, 240, 236, 251, 263, 258, 281, 271, 263, 275, 253, 253, 276, 261, 266, 271, 245, 258, 268, 259, 273, 239, 265, 270, 249, 271, 279, 240] payload_rows_per_dst=[5480, 5470, 5498, 5426, 5466, 5390, 5384, 5552] dup_rows_per_dst=[2766, 2768, 2746, 2742, 2628, 2714, 2678, 2828] combine_rows_per_src=[5480, 5470, 5498, 5426, 5466, 5390, 5384, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38180 dup_rows=21860 dup_rows_remote=19046 dedup_ratio=0.333557 router_tokens_per_expert=[257, 254, 254, 274, 263, 258, 266, 220, 270, 296, 248, 272, 248, 258, 261, 258, 255, 257, 225, 269, 246, 292, 275, 241, 246, 248, 297, 265, 252, 236, 240, 268, 271, 244, 288, 267, 255, 270, 259, 268, 265, 241, 220, 257, 224, 259, 265, 265, 281, 285, 256, 257, 260, 242, 251, 239, 256, 265, 260, 230, 235, 290, 266, 282, 249, 265, 254, 260, 249, 247, 281, 246, 268, 276, 231, 250, 256, 249, 263, 265, 228, 225, 245, 246, 253, 240, 258, 263, 265, 231, 266, 257, 254, 260, 259, 282, 255, 238, 215, 258, 268, 259, 248, 248, 228, 272, 240, 277, 242, 262, 234, 266, 247, 268, 242, 256, 236, 267, 256, 237, 268, 253, 257, 250, 256, 265, 252, 265] payload_rows_per_dst=[5476, 5494, 5512, 5496, 5468, 5356, 5390, 5484] dup_rows_per_dst=[2838, 2730, 2724, 2814, 2750, 2708, 2630, 2666] combine_rows_per_src=[5476, 5494, 5512, 5496, 5468, 5356, 5390, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=228984 combine_payload_bytes=156893184 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157122168 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015352 weight_movement_bytes=18874368 total_nvlink_bytes=332889720 weight_to_token_ratio=0.060107 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229384 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868616 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507848 weight_movement_bytes=18874368 total_nvlink_bytes=332382216 weight_to_token_ratio=0.060204 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38258 dup_rows=21864 dup_rows_remote=19202 dedup_ratio=0.333618 router_tokens_per_expert=[258, 275, 274, 271, 250, 278, 243, 248, 270, 292, 288, 238, 277, 229, 264, 260, 266, 256, 267, 236, 258, 247, 248, 240, 275, 283, 257, 271, 240, 238, 244, 251, 317, 253, 277, 225, 297, 281, 245, 235, 272, 215, 248, 238, 262, 272, 236, 268, 269, 270, 249, 242, 262, 222, 283, 258, 264, 234, 254, 257, 244, 237, 265, 267, 287, 250, 260, 243, 225, 270, 232, 264, 246, 236, 254, 270, 235, 245, 261, 260, 238, 265, 266, 255, 284, 225, 247, 253, 238, 262, 260, 248, 239, 258, 263, 266, 255, 268, 228, 271, 277, 272, 263, 246, 243, 232, 246, 228, 249, 248, 246, 225, 255, 253, 271, 267, 244, 276, 251, 260, 260, 247, 254, 241, 282, 260, 277, 258] payload_rows_per_dst=[5550, 5446, 5464, 5444, 5414, 5458, 5422, 5474] dup_rows_per_dst=[2880, 2708, 2818, 2710, 2662, 2676, 2572, 2838] combine_rows_per_src=[5550, 5446, 5464, 5444, 5414, 5458, 5422, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38126 dup_rows=21914 dup_rows_remote=19122 dedup_ratio=0.334381 router_tokens_per_expert=[265, 250, 251, 266, 217, 269, 242, 259, 263, 274, 255, 232, 226, 262, 256, 281, 271, 263, 259, 259, 262, 261, 263, 246, 255, 262, 251, 248, 232, 253, 228, 245, 254, 257, 259, 265, 259, 265, 226, 249, 267, 251, 245, 269, 271, 279, 278, 245, 236, 284, 234, 264, 252, 256, 246, 260, 244, 263, 277, 248, 272, 239, 267, 273, 251, 277, 226, 238, 275, 235, 244, 240, 266, 257, 272, 268, 283, 257, 267, 233, 262, 278, 290, 258, 284, 267, 218, 283, 265, 239, 220, 259, 248, 275, 272, 226, 266, 256, 269, 255, 252, 254, 235, 257, 256, 251, 270, 255, 249, 237, 255, 251, 232, 267, 238, 240, 268, 259, 246, 257, 265, 242, 280, 267, 262, 263, 271, 230] payload_rows_per_dst=[5438, 5446, 5444, 5494, 5432, 5496, 5472, 5400] dup_rows_per_dst=[2698, 2670, 2834, 2736, 2746, 2792, 2664, 2774] combine_rows_per_src=[5438, 5446, 5444, 5494, 5432, 5496, 5472, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228960 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156630624 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032288 weight_movement_bytes=37748736 total_nvlink_bytes=350781024 weight_to_token_ratio=0.120591 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229160 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156639016 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048872 weight_movement_bytes=37748736 total_nvlink_bytes=350797608 weight_to_token_ratio=0.120584 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38404 dup_rows=21668 dup_rows_remote=19024 dedup_ratio=0.330627 router_tokens_per_expert=[249, 246, 241, 280, 232, 266, 273, 249, 252, 261, 248, 255, 228, 244, 265, 256, 234, 268, 258, 243, 249, 244, 260, 250, 248, 274, 288, 233, 228, 250, 261, 266, 273, 240, 271, 262, 257, 259, 256, 289, 266, 229, 258, 242, 258, 243, 293, 242, 263, 277, 250, 260, 270, 248, 266, 245, 240, 257, 257, 246, 258, 261, 274, 255, 249, 270, 264, 247, 259, 236, 233, 251, 260, 257, 259, 248, 229, 234, 245, 260, 249, 254, 248, 232, 249, 236, 277, 238, 225, 243, 240, 265, 276, 297, 264, 246, 279, 259, 265, 256, 282, 263, 282, 283, 256, 219, 255, 264, 257, 254, 260, 251, 266, 237, 258, 257, 247, 230, 278, 272, 303, 263, 264, 250, 278, 250, 269, 257] payload_rows_per_dst=[5380, 5464, 5490, 5580, 5360, 5424, 5622, 5548] dup_rows_per_dst=[2710, 2644, 2786, 2674, 2642, 2654, 2748, 2810] combine_rows_per_src=[5380, 5464, 5490, 5580, 5360, 5424, 5622, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228904 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614184 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999464 weight_movement_bytes=18874368 total_nvlink_bytes=331873832 weight_to_token_ratio=0.060302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229840 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934608 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639376 weight_movement_bytes=18874368 total_nvlink_bytes=332513744 weight_to_token_ratio=0.060179 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228992 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156393088 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557184 weight_movement_bytes=18874368 total_nvlink_bytes=331431552 weight_to_token_ratio=0.060387 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229712 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532496 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835280 weight_movement_bytes=18874368 total_nvlink_bytes=333709648 weight_to_token_ratio=0.059950 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38314 dup_rows=21760 dup_rows_remote=18926 dedup_ratio=0.332031 router_tokens_per_expert=[250, 252, 255, 255, 272, 276, 265, 250, 238, 247, 251, 263, 259, 271, 248, 293, 238, 283, 261, 230, 226, 276, 270, 276, 262, 254, 270, 228, 240, 260, 268, 263, 250, 276, 265, 260, 265, 259, 275, 244, 264, 255, 228, 265, 274, 277, 251, 240, 244, 247, 269, 275, 266, 251, 234, 237, 266, 293, 234, 231, 206, 255, 252, 262, 276, 238, 268, 235, 266, 248, 275, 262, 268, 264, 244, 278, 257, 249, 240, 268, 268, 285, 262, 235, 249, 256, 258, 246, 245, 268, 229, 250, 251, 274, 247, 295, 257, 270, 260, 252, 240, 240, 232, 246, 261, 244, 257, 250, 259, 285, 262, 244, 270, 236, 257, 271, 258, 235, 234, 283, 244, 268, 246, 262, 229, 274, 227, 241] payload_rows_per_dst=[5556, 5384, 5462, 5460, 5524, 5472, 5496, 5422] dup_rows_per_dst=[2734, 2826, 2834, 2584, 2748, 2764, 2622, 2648] combine_rows_per_src=[5556, 5384, 5462, 5460, 5524, 5472, 5496, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38248 dup_rows=21692 dup_rows_remote=18760 dedup_ratio=0.330994 router_tokens_per_expert=[265, 252, 266, 276, 251, 253, 245, 258, 267, 255, 261, 273, 263, 273, 267, 245, 274, 269, 230, 243, 275, 254, 295, 270, 230, 254, 253, 219, 267, 243, 267, 248, 252, 253, 264, 232, 241, 235, 241, 263, 254, 268, 297, 236, 232, 255, 270, 241, 246, 273, 230, 265, 257, 255, 251, 254, 252, 249, 249, 259, 257, 239, 246, 250, 273, 232, 260, 273, 238, 277, 254, 267, 250, 260, 233, 258, 276, 258, 236, 265, 243, 271, 231, 263, 249, 247, 264, 252, 257, 279, 227, 302, 245, 237, 241, 262, 268, 249, 277, 282, 294, 248, 282, 264, 269, 258, 258, 272, 244, 268, 253, 278, 260, 267, 258, 260, 240, 243, 229, 250, 213, 263, 234, 250, 261, 272, 242, 255] payload_rows_per_dst=[5610, 5450, 5484, 5474, 5450, 5480, 5596, 5300] dup_rows_per_dst=[2730, 2732, 2584, 2590, 2770, 2660, 2932, 2694] combine_rows_per_src=[5610, 5450, 5484, 5474, 5450, 5480, 5596, 5300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38178 dup_rows=21820 dup_rows_remote=19026 dedup_ratio=0.332947 router_tokens_per_expert=[242, 236, 260, 246, 269, 273, 251, 262, 264, 267, 244, 264, 288, 250, 228, 247, 264, 270, 274, 260, 240, 275, 215, 223, 263, 253, 247, 253, 241, 250, 251, 249, 244, 263, 252, 230, 314, 233, 232, 250, 267, 286, 248, 238, 267, 267, 248, 289, 281, 259, 245, 296, 253, 260, 234, 255, 284, 211, 267, 286, 249, 267, 262, 255, 286, 260, 262, 265, 248, 232, 259, 246, 260, 234, 267, 234, 237, 250, 243, 281, 244, 244, 250, 258, 265, 260, 254, 263, 254, 244, 225, 247, 247, 241, 263, 252, 256, 276, 265, 242, 254, 285, 260, 241, 258, 270, 269, 254, 261, 252, 273, 274, 281, 264, 245, 261, 237, 252, 288, 248, 237, 239, 252, 242, 257, 245, 280, 264] payload_rows_per_dst=[5342, 5340, 5430, 5494, 5516, 5512, 5544, 5538] dup_rows_per_dst=[2550, 2716, 2608, 2792, 2830, 2800, 2730, 2794] combine_rows_per_src=[5342, 5340, 5430, 5494, 5516, 5512, 5544, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38198 dup_rows=21974 dup_rows_remote=19240 dedup_ratio=0.335297 router_tokens_per_expert=[273, 241, 257, 246, 264, 259, 284, 273, 269, 266, 240, 256, 245, 262, 287, 254, 253, 263, 247, 266, 248, 276, 256, 256, 292, 237, 244, 257, 271, 260, 230, 267, 250, 269, 252, 240, 279, 259, 245, 247, 247, 255, 269, 243, 259, 236, 256, 257, 239, 252, 250, 282, 224, 243, 248, 249, 270, 275, 247, 275, 234, 259, 259, 243, 251, 268, 264, 258, 245, 259, 237, 252, 247, 226, 231, 253, 238, 250, 260, 265, 269, 252, 252, 237, 253, 244, 255, 242, 230, 255, 261, 249, 258, 254, 255, 268, 259, 256, 278, 273, 277, 243, 283, 266, 247, 268, 230, 254, 245, 281, 274, 272, 272, 256, 265, 278, 278, 255, 224, 250, 259, 250, 265, 252, 272, 229, 246, 262] payload_rows_per_dst=[5518, 5484, 5416, 5364, 5336, 5384, 5554, 5506] dup_rows_per_dst=[2834, 2762, 2710, 2734, 2672, 2684, 2858, 2720] combine_rows_per_src=[5518, 5484, 5416, 5364, 5336, 5384, 5554, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38112 dup_rows=21810 dup_rows_remote=18786 dedup_ratio=0.332794 router_tokens_per_expert=[246, 256, 271, 259, 264, 276, 247, 226, 257, 274, 281, 248, 233, 232, 256, 257, 243, 268, 256, 253, 280, 267, 281, 261, 249, 260, 268, 241, 222, 252, 251, 279, 254, 277, 266, 274, 255, 260, 274, 257, 273, 247, 263, 251, 216, 246, 229, 272, 224, 260, 233, 251, 247, 248, 247, 251, 288, 238, 265, 249, 277, 259, 272, 257, 237, 269, 268, 283, 274, 235, 239, 250, 251, 261, 251, 263, 274, 284, 222, 239, 231, 249, 241, 251, 260, 272, 254, 258, 289, 243, 243, 249, 257, 269, 264, 263, 255, 247, 256, 253, 258, 254, 237, 242, 251, 249, 243, 272, 258, 270, 258, 274, 245, 258, 266, 245, 269, 269, 279, 252, 264, 244, 236, 263, 254, 263, 242, 255] payload_rows_per_dst=[5614, 5606, 5428, 5532, 5316, 5408, 5394, 5428] dup_rows_per_dst=[3024, 2838, 2800, 2858, 2412, 2778, 2502, 2598] combine_rows_per_src=[5614, 5606, 5428, 5532, 5316, 5408, 5394, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38338 dup_rows=21702 dup_rows_remote=18880 dedup_ratio=0.331146 router_tokens_per_expert=[283, 276, 248, 254, 244, 239, 274, 256, 245, 244, 245, 233, 256, 275, 232, 273, 257, 246, 230, 269, 243, 225, 277, 261, 252, 235, 235, 262, 258, 275, 269, 220, 258, 244, 247, 269, 236, 270, 267, 246, 237, 276, 272, 278, 251, 224, 281, 264, 254, 238, 268, 283, 257, 254, 272, 251, 284, 249, 234, 253, 274, 263, 246, 251, 295, 267, 287, 265, 242, 273, 268, 256, 257, 260, 259, 224, 251, 254, 247, 254, 250, 258, 267, 256, 240, 265, 240, 258, 272, 251, 236, 271, 240, 259, 288, 266, 270, 286, 267, 240, 236, 259, 259, 240, 228, 233, 276, 243, 258, 248, 256, 238, 235, 270, 244, 240, 262, 269, 254, 266, 257, 257, 261, 252, 254, 275, 256, 261] payload_rows_per_dst=[5496, 5394, 5494, 5554, 5496, 5486, 5458, 5456] dup_rows_per_dst=[2658, 2634, 2746, 2708, 2822, 2748, 2616, 2770] combine_rows_per_src=[5496, 5394, 5494, 5554, 5496, 5486, 5458, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=228960 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157163104 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097248 weight_movement_bytes=37748736 total_nvlink_bytes=351845984 weight_to_token_ratio=0.120182 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228032 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156891840 ingress_bytes=213286912 link_bytes=213286912 token_bytes=313555648 weight_movement_bytes=56623104 total_nvlink_bytes=370178752 weight_to_token_ratio=0.180584 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043610247e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=228816 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156605904 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312982992 weight_movement_bytes=18874368 total_nvlink_bytes=331857360 weight_to_token_ratio=0.060305 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38294 dup_rows=21770 dup_rows_remote=19074 dedup_ratio=0.332184 router_tokens_per_expert=[222, 250, 269, 258, 230, 265, 262, 250, 278, 245, 233, 271, 231, 270, 244, 266, 250, 248, 278, 245, 255, 250, 259, 280, 224, 261, 253, 242, 270, 268, 246, 237, 255, 257, 282, 273, 267, 257, 272, 273, 246, 257, 245, 261, 259, 253, 268, 270, 275, 275, 284, 271, 281, 245, 206, 286, 257, 245, 232, 280, 272, 239, 252, 254, 266, 263, 268, 233, 245, 262, 266, 254, 293, 261, 226, 224, 245, 253, 262, 248, 248, 291, 252, 236, 260, 261, 283, 273, 260, 239, 258, 260, 218, 243, 236, 266, 279, 252, 227, 229, 271, 248, 239, 266, 295, 241, 268, 241, 257, 283, 251, 267, 253, 269, 278, 231, 204, 268, 255, 248, 247, 256, 278, 239, 256, 256, 265, 239] payload_rows_per_dst=[5454, 5458, 5520, 5504, 5496, 5472, 5510, 5352] dup_rows_per_dst=[2634, 2674, 2870, 2804, 2642, 2696, 2718, 2732] combine_rows_per_src=[5454, 5458, 5520, 5504, 5496, 5472, 5510, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38324 dup_rows=21798 dup_rows_remote=19270 dedup_ratio=0.332611 router_tokens_per_expert=[255, 235, 270, 255, 265, 253, 281, 262, 265, 241, 239, 260, 268, 239, 286, 268, 247, 219, 267, 262, 228, 240, 240, 250, 247, 257, 233, 242, 262, 251, 255, 271, 267, 255, 221, 253, 285, 255, 276, 244, 257, 273, 223, 276, 249, 260, 268, 264, 256, 239, 248, 268, 246, 255, 257, 262, 247, 265, 262, 264, 277, 248, 273, 291, 253, 272, 252, 248, 274, 266, 249, 277, 255, 274, 243, 275, 250, 252, 249, 269, 245, 252, 284, 285, 255, 250, 256, 253, 263, 261, 262, 256, 251, 244, 256, 232, 268, 269, 256, 223, 275, 233, 217, 242, 242, 254, 252, 234, 234, 262, 251, 272, 267, 271, 244, 254, 259, 231, 258, 289, 269, 259, 240, 238, 266, 265, 270, 244] payload_rows_per_dst=[5526, 5414, 5500, 5464, 5522, 5526, 5364, 5422] dup_rows_per_dst=[2758, 2528, 2752, 2852, 2794, 2684, 2604, 2826] combine_rows_per_src=[5526, 5414, 5500, 5464, 5522, 5526, 5364, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229752 combine_payload_bytes=156459008 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156688760 ingress_bytes=213082112 link_bytes=213082112 token_bytes=313147768 weight_movement_bytes=56623104 total_nvlink_bytes=369770872 weight_to_token_ratio=0.180819 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041647954e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=227592 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156334344 ingress_bytes=174981120 link_bytes=174981120 token_bytes=312441096 weight_movement_bytes=18874368 total_nvlink_bytes=331315464 weight_to_token_ratio=0.060409 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676582996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=228872 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157261320 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314293768 weight_movement_bytes=18874368 total_nvlink_bytes=333168136 weight_to_token_ratio=0.060053 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229472 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081696 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313933920 weight_movement_bytes=18874368 total_nvlink_bytes=332808288 weight_to_token_ratio=0.060122 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=230376 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157205480 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314180584 weight_movement_bytes=18874368 total_nvlink_bytes=333054952 weight_to_token_ratio=0.060075 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43296 payload_rows_remote=37976 dup_rows=22240 dup_rows_remote=19480 dedup_ratio=0.339355 router_tokens_per_expert=[272, 246, 248, 289, 247, 245, 281, 269, 254, 267, 250, 247, 282, 250, 261, 284, 259, 240, 249, 235, 247, 261, 281, 275, 280, 276, 250, 257, 250, 262, 271, 250, 257, 245, 244, 260, 269, 245, 265, 257, 247, 247, 274, 279, 247, 242, 242, 220, 266, 257, 245, 226, 268, 260, 259, 232, 225, 267, 254, 270, 241, 260, 267, 266, 267, 236, 259, 270, 221, 247, 259, 264, 233, 225, 248, 253, 259, 250, 261, 258, 249, 245, 245, 250, 262, 226, 290, 257, 255, 274, 254, 252, 264, 266, 238, 258, 260, 248, 264, 263, 241, 278, 248, 255, 256, 248, 239, 262, 265, 271, 278, 264, 271, 250, 239, 262, 213, 258, 250, 243, 264, 233, 309, 262, 248, 249, 270, 274] payload_rows_per_dst=[5482, 5452, 5320, 5340, 5360, 5370, 5464, 5508] dup_rows_per_dst=[2902, 2834, 2760, 2786, 2660, 2800, 2816, 2682] combine_rows_per_src=[5482, 5452, 5320, 5340, 5360, 5370, 5464, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38302 dup_rows=21732 dup_rows_remote=18962 dedup_ratio=0.331604 router_tokens_per_expert=[275, 245, 274, 232, 258, 253, 259, 261, 269, 261, 263, 232, 285, 260, 256, 248, 238, 268, 279, 278, 279, 266, 256, 232, 234, 230, 276, 243, 233, 260, 248, 239, 251, 268, 211, 283, 272, 268, 262, 244, 263, 245, 260, 234, 243, 286, 239, 259, 274, 258, 259, 270, 269, 258, 245, 239, 260, 234, 252, 260, 282, 231, 254, 255, 265, 270, 283, 258, 246, 249, 265, 235, 223, 262, 257, 268, 269, 263, 252, 250, 236, 235, 259, 283, 248, 263, 245, 246, 267, 248, 276, 268, 270, 261, 243, 266, 278, 283, 257, 259, 240, 240, 256, 261, 227, 300, 273, 241, 259, 243, 252, 267, 261, 257, 257, 234, 260, 261, 261, 258, 233, 249, 257, 252, 257, 244, 250, 234] payload_rows_per_dst=[5542, 5442, 5438, 5454, 5548, 5454, 5502, 5424] dup_rows_per_dst=[2720, 2676, 2738, 2746, 2682, 2774, 2770, 2626] combine_rows_per_src=[5542, 5442, 5438, 5454, 5548, 5454, 5502, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38150 dup_rows=21848 dup_rows_remote=18936 dedup_ratio=0.333374 router_tokens_per_expert=[237, 243, 232, 248, 274, 253, 251, 247, 255, 250, 260, 245, 247, 282, 246, 283, 272, 228, 240, 252, 251, 261, 236, 244, 282, 239, 287, 266, 257, 252, 237, 269, 255, 253, 272, 238, 241, 275, 253, 264, 271, 258, 258, 241, 271, 243, 243, 217, 266, 256, 247, 270, 248, 289, 275, 233, 256, 244, 282, 240, 244, 247, 260, 240, 248, 265, 266, 262, 255, 267, 248, 259, 257, 236, 254, 256, 269, 294, 294, 250, 229, 254, 236, 256, 258, 246, 285, 259, 241, 252, 265, 260, 221, 255, 260, 241, 271, 268, 266, 233, 252, 239, 266, 289, 249, 256, 232, 274, 272, 241, 256, 264, 257, 251, 260, 287, 260, 264, 258, 226, 251, 271, 275, 277, 271, 235, 272, 251] payload_rows_per_dst=[5188, 5522, 5418, 5336, 5678, 5490, 5518, 5538] dup_rows_per_dst=[2412, 2848, 2688, 2538, 2912, 2780, 2758, 2912] combine_rows_per_src=[5188, 5522, 5418, 5336, 5678, 5490, 5518, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38252 dup_rows=21792 dup_rows_remote=19086 dedup_ratio=0.332520 router_tokens_per_expert=[263, 252, 263, 258, 274, 258, 265, 271, 245, 253, 273, 249, 276, 255, 247, 255, 243, 275, 230, 275, 271, 240, 268, 292, 271, 252, 273, 245, 281, 281, 251, 241, 275, 274, 252, 226, 264, 252, 245, 265, 247, 233, 250, 240, 260, 242, 256, 248, 259, 231, 241, 243, 249, 272, 267, 274, 254, 250, 247, 266, 267, 246, 270, 263, 274, 261, 270, 243, 270, 241, 259, 262, 247, 265, 241, 243, 274, 238, 222, 267, 256, 258, 261, 226, 237, 229, 238, 262, 303, 275, 248, 236, 267, 237, 269, 250, 258, 237, 240, 258, 249, 279, 253, 252, 240, 268, 266, 243, 245, 266, 250, 255, 216, 289, 281, 267, 259, 262, 252, 267, 221, 267, 242, 249, 248, 265, 276, 245] payload_rows_per_dst=[5552, 5570, 5356, 5492, 5406, 5408, 5434, 5526] dup_rows_per_dst=[2762, 2808, 2702, 2706, 2748, 2696, 2684, 2686] combine_rows_per_src=[5552, 5570, 5356, 5492, 5406, 5408, 5434, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38084 dup_rows=22002 dup_rows_remote=19264 dedup_ratio=0.335724 router_tokens_per_expert=[272, 229, 274, 253, 268, 278, 256, 234, 272, 256, 250, 275, 237, 236, 246, 231, 247, 240, 259, 271, 260, 262, 261, 271, 256, 265, 254, 264, 255, 269, 247, 231, 264, 249, 284, 232, 243, 239, 239, 261, 243, 237, 247, 269, 263, 265, 271, 259, 277, 270, 279, 250, 265, 270, 266, 219, 240, 260, 252, 261, 230, 260, 247, 245, 239, 268, 259, 247, 260, 257, 240, 249, 269, 258, 282, 253, 235, 248, 266, 242, 253, 281, 255, 260, 287, 268, 281, 276, 240, 241, 234, 256, 254, 264, 267, 244, 246, 280, 253, 252, 253, 263, 238, 248, 265, 250, 279, 259, 287, 252, 238, 260, 234, 241, 249, 268, 237, 263, 261, 253, 256, 262, 268, 254, 260, 273, 237, 261] payload_rows_per_dst=[5450, 5476, 5388, 5362, 5508, 5492, 5404, 5454] dup_rows_per_dst=[2738, 2820, 2742, 2694, 2840, 2830, 2638, 2700] combine_rows_per_src=[5450, 5476, 5388, 5362, 5508, 5492, 5404, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=155549696 dispatch_meta_bytes=229824 combine_payload_bytes=155549696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155779520 ingress_bytes=174424064 link_bytes=174424064 token_bytes=311329216 weight_movement_bytes=18874368 total_nvlink_bytes=330203584 weight_to_token_ratio=0.060625 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671245559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38306 dup_rows=21784 dup_rows_remote=19096 dedup_ratio=0.332397 router_tokens_per_expert=[248, 238, 280, 266, 265, 267, 253, 258, 252, 251, 277, 252, 262, 240, 232, 255, 271, 246, 263, 258, 239, 258, 241, 244, 252, 264, 258, 254, 275, 270, 258, 242, 238, 270, 264, 258, 274, 246, 255, 244, 258, 262, 266, 245, 274, 264, 256, 272, 245, 250, 265, 288, 237, 259, 265, 276, 261, 270, 275, 251, 242, 243, 250, 262, 256, 240, 243, 247, 262, 272, 247, 249, 254, 241, 256, 269, 255, 254, 240, 282, 265, 289, 258, 256, 240, 273, 248, 257, 247, 246, 245, 255, 255, 260, 217, 226, 265, 269, 275, 266, 229, 276, 257, 234, 249, 271, 258, 269, 279, 265, 236, 257, 232, 245, 239, 262, 267, 237, 265, 272, 246, 263, 234, 247, 252, 253, 272, 249] payload_rows_per_dst=[5470, 5482, 5480, 5556, 5446, 5388, 5494, 5436] dup_rows_per_dst=[2722, 2704, 2812, 2722, 2688, 2686, 2816, 2634] combine_rows_per_src=[5470, 5482, 5480, 5556, 5446, 5388, 5494, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229056 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114048 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999040 weight_movement_bytes=37748736 total_nvlink_bytes=351747776 weight_to_token_ratio=0.120219 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228344 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156490744 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753144 weight_movement_bytes=18874368 total_nvlink_bytes=331627512 weight_to_token_ratio=0.060349 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38346 dup_rows=21644 dup_rows_remote=18864 dedup_ratio=0.330261 router_tokens_per_expert=[232, 244, 239, 244, 262, 269, 279, 255, 279, 266, 259, 242, 272, 259, 225, 293, 253, 275, 251, 246, 258, 243, 266, 280, 240, 226, 261, 255, 246, 272, 257, 237, 265, 276, 272, 276, 251, 244, 272, 241, 251, 260, 247, 241, 259, 256, 255, 251, 245, 255, 255, 268, 251, 276, 259, 285, 262, 244, 280, 282, 254, 263, 259, 239, 248, 246, 246, 262, 255, 237, 276, 268, 254, 227, 244, 248, 217, 235, 282, 247, 278, 245, 259, 248, 270, 267, 265, 261, 245, 245, 265, 241, 247, 285, 256, 286, 270, 258, 252, 270, 258, 225, 227, 246, 270, 274, 239, 243, 266, 235, 242, 276, 286, 239, 256, 249, 240, 295, 252, 246, 265, 239, 237, 262, 265, 238, 261, 253] payload_rows_per_dst=[5482, 5434, 5562, 5554, 5428, 5546, 5438, 5448] dup_rows_per_dst=[2756, 2698, 2672, 2800, 2556, 2780, 2664, 2718] combine_rows_per_src=[5482, 5434, 5562, 5554, 5428, 5546, 5438, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229352 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156909544 ingress_bytes=213303296 link_bytes=213303296 token_bytes=313589736 weight_movement_bytes=56623104 total_nvlink_bytes=370212840 weight_to_token_ratio=0.180564 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043767231e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38224 dup_rows=21900 dup_rows_remote=19112 dedup_ratio=0.334167 router_tokens_per_expert=[263, 241, 273, 264, 253, 258, 242, 256, 267, 243, 272, 250, 245, 245, 254, 245, 268, 227, 249, 284, 249, 240, 283, 243, 252, 251, 254, 288, 231, 260, 252, 269, 249, 232, 258, 248, 254, 281, 268, 244, 256, 248, 272, 236, 253, 245, 257, 260, 261, 263, 264, 257, 247, 252, 255, 272, 260, 246, 252, 267, 268, 267, 238, 237, 249, 226, 267, 260, 257, 244, 262, 260, 278, 255, 230, 272, 252, 260, 246, 248, 275, 267, 261, 263, 254, 268, 265, 258, 252, 273, 283, 245, 243, 240, 250, 244, 264, 256, 251, 243, 261, 250, 250, 274, 262, 243, 256, 243, 259, 281, 256, 258, 250, 260, 259, 254, 269, 256, 230, 265, 234, 261, 272, 251, 262, 263, 259, 271] payload_rows_per_dst=[5372, 5412, 5430, 5474, 5470, 5528, 5446, 5504] dup_rows_per_dst=[2770, 2788, 2692, 2738, 2662, 2754, 2768, 2728] combine_rows_per_src=[5372, 5412, 5430, 5474, 5470, 5528, 5446, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=229392 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156221456 ingress_bytes=193740800 link_bytes=193740800 token_bytes=312213520 weight_movement_bytes=37748736 total_nvlink_bytes=349962256 weight_to_token_ratio=0.120907 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856329019e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=229608 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130984 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314032360 weight_movement_bytes=18874368 total_nvlink_bytes=332906728 weight_to_token_ratio=0.060103 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=228840 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157294056 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314359272 weight_movement_bytes=18874368 total_nvlink_bytes=333233640 weight_to_token_ratio=0.060041 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229344 combine_payload_bytes=156565504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156794848 ingress_bytes=175439872 link_bytes=175439872 token_bytes=313360352 weight_movement_bytes=18874368 total_nvlink_bytes=332234720 weight_to_token_ratio=0.060232 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680978532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38264 dup_rows=21772 dup_rows_remote=19052 dedup_ratio=0.332214 router_tokens_per_expert=[272, 248, 266, 255, 275, 234, 261, 265, 280, 241, 261, 262, 267, 277, 249, 258, 278, 255, 238, 268, 274, 241, 270, 249, 225, 254, 255, 254, 244, 237, 251, 235, 243, 255, 249, 231, 257, 258, 258, 268, 263, 251, 248, 254, 262, 271, 262, 280, 253, 261, 279, 264, 283, 246, 254, 251, 248, 243, 261, 258, 232, 283, 271, 241, 283, 236, 253, 258, 225, 237, 247, 236, 234, 263, 262, 264, 238, 255, 243, 277, 257, 252, 268, 252, 260, 245, 260, 261, 253, 244, 241, 255, 284, 239, 232, 272, 277, 259, 259, 263, 267, 256, 250, 255, 266, 263, 220, 239, 243, 291, 254, 222, 246, 265, 289, 274, 259, 260, 232, 274, 244, 271, 261, 253, 265, 251, 274, 243] payload_rows_per_dst=[5492, 5428, 5500, 5502, 5444, 5420, 5410, 5568] dup_rows_per_dst=[2850, 2628, 2720, 2754, 2578, 2730, 2758, 2754] combine_rows_per_src=[5492, 5428, 5500, 5502, 5444, 5420, 5410, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38412 dup_rows=21636 dup_rows_remote=18972 dedup_ratio=0.330139 router_tokens_per_expert=[243, 257, 245, 256, 256, 267, 271, 252, 265, 222, 284, 242, 261, 247, 257, 252, 246, 249, 246, 241, 253, 250, 274, 253, 259, 272, 255, 268, 265, 262, 254, 269, 271, 259, 244, 236, 237, 260, 260, 262, 245, 253, 262, 235, 263, 255, 253, 267, 249, 271, 233, 258, 257, 267, 265, 244, 236, 245, 241, 284, 223, 260, 257, 274, 250, 274, 232, 280, 245, 260, 252, 259, 261, 238, 273, 259, 285, 227, 250, 274, 245, 272, 261, 243, 265, 266, 232, 248, 262, 268, 255, 243, 268, 255, 277, 247, 243, 249, 230, 297, 247, 260, 249, 242, 276, 259, 235, 254, 252, 264, 260, 259, 226, 258, 258, 287, 247, 264, 264, 285, 233, 262, 245, 271, 292, 250, 255, 250] payload_rows_per_dst=[5526, 5470, 5464, 5462, 5502, 5478, 5488, 5510] dup_rows_per_dst=[2628, 2762, 2660, 2666, 2736, 2736, 2664, 2784] combine_rows_per_src=[5526, 5470, 5464, 5462, 5502, 5478, 5488, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43934 payload_rows_remote=38528 dup_rows=21602 dup_rows_remote=18950 dedup_ratio=0.329620 router_tokens_per_expert=[261, 259, 238, 261, 250, 260, 281, 247, 242, 259, 263, 259, 276, 252, 237, 226, 274, 237, 235, 235, 255, 262, 235, 265, 260, 243, 235, 246, 255, 248, 270, 248, 236, 268, 251, 238, 239, 275, 272, 243, 275, 262, 233, 274, 249, 258, 248, 276, 268, 265, 290, 281, 254, 290, 283, 249, 247, 267, 254, 266, 260, 246, 264, 246, 228, 258, 245, 229, 248, 249, 237, 240, 247, 252, 275, 263, 261, 252, 256, 262, 283, 261, 241, 235, 285, 247, 278, 255, 256, 252, 272, 263, 264, 257, 278, 297, 259, 251, 238, 256, 256, 283, 267, 271, 259, 267, 263, 238, 235, 245, 272, 256, 265, 223, 271, 244, 274, 238, 279, 244, 270, 285, 221, 244, 210, 237, 255, 265] payload_rows_per_dst=[5426, 5472, 5496, 5540, 5444, 5574, 5576, 5406] dup_rows_per_dst=[2690, 2534, 2698, 2676, 2560, 2892, 2900, 2652] combine_rows_per_src=[5426, 5472, 5496, 5540, 5444, 5574, 5576, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38326 dup_rows=21654 dup_rows_remote=18952 dedup_ratio=0.330414 router_tokens_per_expert=[241, 282, 253, 257, 260, 261, 246, 269, 277, 257, 218, 255, 278, 249, 256, 239, 275, 262, 244, 291, 247, 252, 263, 265, 265, 266, 268, 293, 270, 222, 250, 277, 271, 237, 269, 243, 267, 268, 273, 255, 244, 224, 262, 251, 258, 254, 267, 296, 252, 280, 272, 252, 274, 242, 285, 254, 256, 256, 259, 245, 261, 242, 240, 259, 240, 227, 247, 242, 251, 234, 229, 261, 231, 223, 290, 244, 297, 274, 254, 250, 260, 259, 259, 267, 245, 265, 253, 247, 250, 255, 268, 240, 253, 274, 246, 257, 246, 261, 253, 232, 259, 288, 247, 270, 274, 253, 250, 266, 248, 257, 260, 218, 231, 247, 262, 234, 255, 274, 257, 229, 251, 259, 247, 270, 276, 260, 246, 220] payload_rows_per_dst=[5452, 5526, 5496, 5556, 5456, 5526, 5468, 5402] dup_rows_per_dst=[2744, 2894, 2782, 2702, 2532, 2670, 2696, 2634] combine_rows_per_src=[5452, 5526, 5496, 5556, 5456, 5526, 5468, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43948 payload_rows_remote=38416 dup_rows=21588 dup_rows_remote=18824 dedup_ratio=0.329407 router_tokens_per_expert=[254, 226, 240, 253, 270, 263, 259, 281, 252, 226, 277, 279, 253, 275, 256, 281, 234, 250, 275, 243, 241, 259, 280, 274, 253, 251, 261, 249, 269, 267, 252, 271, 258, 240, 253, 257, 254, 286, 271, 244, 248, 246, 261, 256, 274, 257, 230, 236, 232, 246, 262, 244, 253, 270, 261, 254, 259, 249, 236, 239, 267, 248, 242, 248, 284, 262, 254, 264, 243, 230, 226, 251, 276, 238, 252, 280, 299, 260, 275, 250, 240, 245, 231, 277, 255, 263, 200, 249, 277, 248, 249, 278, 245, 258, 253, 220, 262, 234, 257, 252, 249, 274, 267, 260, 265, 273, 280, 285, 267, 251, 264, 238, 256, 244, 279, 241, 249, 261, 258, 250, 260, 250, 236, 249, 289, 256, 271, 254] payload_rows_per_dst=[5532, 5512, 5526, 5400, 5528, 5486, 5608, 5356] dup_rows_per_dst=[2764, 2740, 2616, 2620, 2760, 2766, 2748, 2574] combine_rows_per_src=[5532, 5512, 5526, 5400, 5528, 5486, 5608, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229264 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958608 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687952 weight_movement_bytes=18874368 total_nvlink_bytes=332562320 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229536 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157565088 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900640 weight_movement_bytes=18874368 total_nvlink_bytes=333775008 weight_to_token_ratio=0.059938 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38274 dup_rows=21742 dup_rows_remote=18998 dedup_ratio=0.331757 router_tokens_per_expert=[254, 260, 247, 242, 238, 260, 243, 232, 236, 252, 290, 252, 246, 288, 248, 275, 232, 229, 273, 236, 234, 252, 248, 276, 259, 263, 259, 246, 250, 271, 292, 248, 264, 242, 221, 250, 251, 254, 224, 278, 251, 263, 248, 279, 258, 248, 303, 258, 264, 253, 257, 246, 271, 225, 259, 263, 240, 238, 265, 239, 258, 264, 253, 287, 257, 270, 244, 276, 260, 285, 261, 252, 282, 243, 264, 222, 249, 262, 244, 261, 252, 234, 229, 271, 283, 277, 254, 234, 260, 246, 274, 252, 283, 268, 263, 268, 258, 248, 285, 276, 238, 254, 251, 245, 255, 231, 288, 257, 249, 264, 242, 280, 235, 252, 257, 276, 255, 246, 249, 234, 287, 257, 240, 234, 278, 287, 231, 244] payload_rows_per_dst=[5420, 5434, 5556, 5470, 5520, 5518, 5452, 5424] dup_rows_per_dst=[2706, 2702, 2628, 2694, 2744, 2778, 2790, 2700] combine_rows_per_src=[5420, 5434, 5556, 5470, 5520, 5518, 5452, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157810688 dispatch_meta_bytes=229912 combine_payload_bytes=157810688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158040600 ingress_bytes=195559424 link_bytes=195559424 token_bytes=315851288 weight_movement_bytes=37748736 total_nvlink_bytes=353600024 weight_to_token_ratio=0.119514 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873754180e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38140 dup_rows=21818 dup_rows_remote=19006 dedup_ratio=0.332916 router_tokens_per_expert=[253, 248, 243, 269, 259, 265, 253, 237, 248, 265, 273, 252, 247, 259, 248, 270, 264, 250, 245, 245, 271, 239, 258, 242, 279, 246, 240, 251, 265, 268, 256, 277, 262, 249, 243, 244, 264, 253, 258, 256, 244, 261, 263, 250, 281, 265, 275, 244, 257, 257, 289, 229, 261, 257, 259, 245, 269, 243, 242, 261, 255, 254, 259, 232, 236, 255, 258, 263, 237, 278, 236, 281, 282, 268, 239, 249, 256, 259, 253, 251, 269, 253, 250, 265, 266, 251, 276, 274, 255, 262, 248, 243, 286, 270, 261, 266, 225, 261, 249, 248, 235, 244, 238, 248, 237, 272, 257, 244, 249, 283, 239, 232, 260, 252, 266, 264, 254, 264, 268, 260, 273, 252, 266, 259, 262, 235, 226, 284] payload_rows_per_dst=[5404, 5452, 5518, 5418, 5480, 5578, 5384, 5484] dup_rows_per_dst=[2774, 2740, 2706, 2720, 2722, 2812, 2538, 2806] combine_rows_per_src=[5404, 5452, 5518, 5418, 5480, 5578, 5384, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38238 dup_rows=21908 dup_rows_remote=19272 dedup_ratio=0.334290 router_tokens_per_expert=[268, 251, 254, 260, 266, 279, 256, 268, 256, 256, 237, 251, 280, 269, 268, 232, 253, 236, 263, 252, 239, 251, 273, 239, 257, 246, 230, 265, 258, 262, 242, 247, 242, 234, 266, 256, 282, 243, 264, 254, 253, 240, 256, 282, 253, 274, 294, 276, 250, 231, 284, 243, 232, 269, 272, 240, 257, 234, 255, 259, 257, 309, 262, 238, 244, 280, 246, 242, 280, 287, 276, 253, 261, 248, 288, 249, 239, 255, 258, 246, 246, 256, 241, 244, 263, 259, 250, 270, 256, 257, 227, 262, 243, 241, 236, 257, 234, 242, 236, 244, 269, 245, 254, 255, 250, 276, 287, 250, 289, 263, 259, 257, 256, 260, 249, 248, 247, 262, 243, 252, 262, 261, 272, 251, 244, 265, 249, 252] payload_rows_per_dst=[5516, 5390, 5512, 5482, 5510, 5398, 5428, 5392] dup_rows_per_dst=[2786, 2636, 2826, 2702, 2794, 2618, 2792, 2754] combine_rows_per_src=[5516, 5390, 5512, 5482, 5510, 5398, 5428, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229112 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157212408 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314195704 weight_movement_bytes=56623104 total_nvlink_bytes=370818808 weight_to_token_ratio=0.180216 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=228960 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157580896 ingress_bytes=232849408 link_bytes=232849408 token_bytes=314932832 weight_movement_bytes=75497472 total_nvlink_bytes=390430304 weight_to_token_ratio=0.239726 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.231048458e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229088 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999392 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313769696 weight_movement_bytes=18874368 total_nvlink_bytes=332644064 weight_to_token_ratio=0.060154 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228584 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156450024 ingress_bytes=212844544 link_bytes=212844544 token_bytes=312671464 weight_movement_bytes=56623104 total_nvlink_bytes=369294568 weight_to_token_ratio=0.181095 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039371695e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=230040 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156852888 ingress_bytes=194371584 link_bytes=194371584 token_bytes=313475736 weight_movement_bytes=37748736 total_nvlink_bytes=351224472 weight_to_token_ratio=0.120420 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862372881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38230 dup_rows=21856 dup_rows_remote=19036 dedup_ratio=0.333496 router_tokens_per_expert=[236, 245, 270, 268, 255, 234, 284, 259, 263, 241, 282, 240, 275, 273, 247, 260, 259, 270, 262, 265, 262, 250, 257, 277, 264, 292, 255, 251, 268, 253, 243, 237, 282, 305, 244, 252, 271, 265, 242, 246, 240, 252, 260, 261, 243, 254, 265, 253, 264, 285, 254, 268, 255, 235, 287, 241, 257, 286, 254, 236, 269, 253, 260, 274, 244, 253, 243, 251, 234, 277, 256, 244, 260, 253, 253, 261, 226, 224, 248, 267, 264, 265, 253, 247, 285, 253, 249, 245, 256, 287, 263, 243, 249, 264, 248, 231, 247, 236, 237, 245, 264, 256, 262, 250, 242, 239, 277, 272, 239, 225, 245, 266, 248, 265, 248, 233, 238, 269, 250, 229, 246, 289, 276, 231, 269, 269, 259, 241] payload_rows_per_dst=[5492, 5540, 5450, 5484, 5336, 5510, 5392, 5476] dup_rows_per_dst=[2772, 2790, 2820, 2872, 2652, 2694, 2612, 2644] combine_rows_per_src=[5492, 5540, 5450, 5484, 5336, 5510, 5392, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38152 dup_rows=21832 dup_rows_remote=19028 dedup_ratio=0.333130 router_tokens_per_expert=[249, 269, 255, 268, 235, 253, 225, 281, 249, 263, 246, 250, 239, 249, 273, 274, 267, 225, 236, 256, 259, 281, 241, 240, 242, 251, 260, 260, 245, 257, 258, 268, 255, 253, 282, 257, 261, 255, 259, 240, 273, 241, 258, 266, 252, 254, 260, 271, 229, 252, 259, 254, 255, 282, 246, 244, 275, 251, 260, 249, 274, 250, 272, 256, 287, 238, 266, 281, 252, 230, 240, 259, 251, 255, 243, 257, 263, 236, 236, 263, 280, 266, 261, 251, 250, 246, 262, 241, 279, 227, 277, 247, 273, 267, 255, 266, 268, 262, 263, 241, 259, 257, 279, 256, 248, 252, 263, 288, 263, 260, 251, 268, 281, 262, 236, 250, 236, 249, 255, 248, 235, 263, 237, 255, 258, 250, 247, 254] payload_rows_per_dst=[5440, 5412, 5438, 5482, 5406, 5520, 5552, 5454] dup_rows_per_dst=[2716, 2680, 2836, 2734, 2708, 2776, 2804, 2578] combine_rows_per_src=[5440, 5412, 5438, 5482, 5406, 5520, 5552, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38062 dup_rows=21926 dup_rows_remote=19058 dedup_ratio=0.334564 router_tokens_per_expert=[253, 272, 289, 275, 251, 249, 261, 257, 261, 241, 265, 273, 242, 272, 265, 254, 245, 241, 256, 257, 231, 258, 273, 256, 290, 249, 277, 253, 265, 270, 253, 251, 262, 222, 233, 272, 263, 251, 258, 228, 253, 254, 279, 264, 279, 274, 253, 242, 252, 261, 257, 244, 289, 251, 243, 240, 259, 231, 257, 256, 252, 258, 252, 255, 255, 255, 252, 239, 264, 245, 281, 256, 277, 252, 227, 256, 234, 246, 263, 250, 259, 254, 267, 278, 282, 306, 234, 254, 240, 253, 260, 245, 270, 260, 230, 255, 215, 275, 236, 243, 264, 249, 254, 256, 235, 258, 260, 266, 252, 253, 261, 269, 282, 250, 251, 234, 262, 280, 240, 240, 260, 232, 228, 238, 264, 260, 302, 251] payload_rows_per_dst=[5286, 5370, 5494, 5340, 5580, 5578, 5414, 5548] dup_rows_per_dst=[2510, 2644, 2680, 2682, 2760, 3012, 2770, 2868] combine_rows_per_src=[5286, 5370, 5494, 5340, 5580, 5578, 5414, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38264 dup_rows=21738 dup_rows_remote=19012 dedup_ratio=0.331696 router_tokens_per_expert=[242, 239, 260, 261, 286, 255, 247, 275, 258, 253, 238, 267, 247, 269, 278, 240, 265, 239, 244, 268, 221, 251, 268, 257, 240, 279, 293, 239, 256, 262, 236, 262, 273, 243, 285, 247, 240, 258, 288, 248, 272, 265, 228, 224, 258, 262, 266, 257, 271, 247, 283, 234, 268, 256, 233, 253, 265, 254, 258, 270, 242, 267, 270, 259, 262, 275, 234, 236, 256, 251, 268, 250, 231, 270, 262, 257, 292, 268, 251, 247, 229, 243, 261, 262, 238, 254, 301, 240, 248, 257, 237, 234, 242, 271, 257, 255, 269, 264, 252, 247, 272, 241, 262, 220, 289, 246, 270, 241, 235, 260, 242, 246, 252, 296, 253, 300, 238, 262, 235, 270, 239, 258, 238, 287, 234, 271, 252, 249] payload_rows_per_dst=[5512, 5514, 5508, 5534, 5416, 5418, 5474, 5422] dup_rows_per_dst=[2718, 2646, 2720, 2726, 2804, 2640, 2638, 2846] combine_rows_per_src=[5512, 5514, 5508, 5534, 5416, 5418, 5474, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229064 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819144 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409224 weight_movement_bytes=18874368 total_nvlink_bytes=332283592 weight_to_token_ratio=0.060223 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228720 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156499312 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312769904 weight_movement_bytes=18874368 total_nvlink_bytes=331644272 weight_to_token_ratio=0.060346 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38006 dup_rows=21958 dup_rows_remote=19200 dedup_ratio=0.335052 router_tokens_per_expert=[276, 264, 266, 259, 263, 273, 261, 249, 257, 277, 242, 261, 267, 253, 239, 256, 249, 267, 233, 283, 271, 240, 241, 249, 273, 265, 225, 220, 244, 246, 261, 275, 264, 272, 252, 244, 264, 248, 249, 241, 265, 234, 261, 263, 244, 239, 252, 237, 241, 270, 261, 230, 276, 256, 273, 246, 250, 254, 281, 260, 258, 250, 265, 263, 254, 267, 256, 278, 252, 285, 229, 240, 249, 261, 261, 258, 264, 244, 243, 275, 263, 259, 260, 274, 256, 255, 255, 258, 268, 279, 234, 259, 270, 261, 238, 261, 304, 257, 239, 231, 266, 238, 237, 231, 241, 242, 248, 256, 263, 259, 252, 247, 256, 237, 254, 255, 252, 274, 256, 272, 247, 223, 274, 284, 241, 267, 261, 270] payload_rows_per_dst=[5572, 5656, 5354, 5440, 5442, 5516, 5162, 5436] dup_rows_per_dst=[2758, 2918, 2748, 2780, 2790, 2784, 2370, 2810] combine_rows_per_src=[5572, 5656, 5354, 5440, 5442, 5516, 5162, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38412 dup_rows=21716 dup_rows_remote=18918 dedup_ratio=0.331360 router_tokens_per_expert=[220, 262, 271, 273, 267, 244, 242, 224, 236, 263, 251, 266, 258, 255, 241, 290, 280, 277, 276, 241, 258, 290, 265, 247, 257, 256, 279, 263, 261, 261, 263, 250, 241, 251, 243, 264, 250, 270, 268, 264, 248, 237, 243, 239, 250, 250, 241, 238, 244, 262, 259, 258, 263, 269, 242, 228, 275, 262, 269, 243, 258, 274, 240, 257, 276, 259, 246, 277, 244, 245, 261, 238, 242, 222, 258, 275, 257, 271, 272, 260, 275, 247, 253, 272, 244, 267, 249, 245, 252, 277, 243, 280, 221, 260, 258, 236, 249, 277, 252, 238, 271, 265, 256, 242, 288, 274, 258, 245, 264, 236, 270, 242, 241, 246, 269, 262, 279, 231, 237, 244, 262, 286, 249, 251, 231, 258, 269, 257] payload_rows_per_dst=[5482, 5586, 5392, 5514, 5408, 5472, 5502, 5464] dup_rows_per_dst=[2644, 2862, 2602, 2692, 2798, 2686, 2752, 2680] combine_rows_per_src=[5482, 5586, 5392, 5514, 5408, 5472, 5502, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=228480 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156130432 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312032384 weight_movement_bytes=18874368 total_nvlink_bytes=330906752 weight_to_token_ratio=0.060488 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229104 combine_payload_bytes=156729344 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156958448 ingress_bytes=232226816 link_bytes=232226816 token_bytes=313687792 weight_movement_bytes=75497472 total_nvlink_bytes=389185264 weight_to_token_ratio=0.240677 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225083088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38194 dup_rows=21898 dup_rows_remote=19088 dedup_ratio=0.334137 router_tokens_per_expert=[257, 242, 281, 258, 256, 218, 260, 250, 260, 226, 258, 260, 252, 262, 259, 279, 247, 220, 250, 286, 272, 255, 256, 239, 251, 242, 261, 250, 247, 252, 231, 270, 260, 222, 260, 234, 293, 243, 253, 245, 288, 260, 259, 283, 242, 248, 238, 255, 281, 254, 250, 256, 252, 264, 260, 243, 274, 274, 264, 285, 240, 230, 239, 233, 265, 249, 274, 237, 249, 263, 261, 249, 292, 235, 261, 251, 249, 265, 234, 258, 257, 241, 264, 253, 243, 252, 257, 251, 271, 272, 282, 252, 259, 254, 245, 262, 254, 277, 260, 277, 280, 251, 264, 247, 234, 256, 242, 248, 279, 244, 272, 238, 270, 254, 238, 266, 282, 260, 284, 272, 287, 221, 267, 238, 273, 261, 224, 252] payload_rows_per_dst=[5424, 5444, 5500, 5438, 5488, 5392, 5504, 5448] dup_rows_per_dst=[2732, 2810, 2666, 2760, 2696, 2642, 2742, 2850] combine_rows_per_src=[5424, 5444, 5500, 5438, 5488, 5392, 5504, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38302 dup_rows=21834 dup_rows_remote=19110 dedup_ratio=0.333160 router_tokens_per_expert=[276, 265, 293, 271, 241, 297, 271, 260, 263, 268, 263, 239, 259, 260, 255, 247, 250, 274, 241, 275, 264, 244, 236, 261, 263, 254, 272, 249, 260, 267, 266, 248, 241, 261, 247, 255, 264, 234, 250, 226, 247, 254, 251, 259, 251, 240, 275, 277, 247, 256, 262, 261, 255, 258, 240, 253, 237, 256, 246, 269, 257, 254, 252, 258, 224, 225, 249, 256, 263, 241, 220, 244, 249, 258, 283, 266, 258, 251, 268, 254, 261, 279, 267, 241, 272, 237, 248, 241, 242, 260, 241, 265, 261, 265, 249, 233, 264, 291, 261, 253, 284, 220, 272, 257, 279, 236, 257, 254, 262, 272, 247, 269, 231, 271, 263, 267, 253, 256, 245, 248, 229, 261, 282, 293, 246, 257, 230, 242] payload_rows_per_dst=[5528, 5490, 5424, 5470, 5312, 5400, 5626, 5452] dup_rows_per_dst=[2928, 2758, 2640, 2652, 2706, 2724, 2730, 2696] combine_rows_per_src=[5528, 5490, 5424, 5470, 5312, 5400, 5626, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155672576 dispatch_meta_bytes=228824 combine_payload_bytes=155672576 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155901400 ingress_bytes=193421312 link_bytes=193421312 token_bytes=311573976 weight_movement_bytes=37748736 total_nvlink_bytes=349322712 weight_to_token_ratio=0.121155 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853267842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229320 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157564872 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900424 weight_movement_bytes=18874368 total_nvlink_bytes=333774792 weight_to_token_ratio=0.059938 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43932 payload_rows_remote=38384 dup_rows=21604 dup_rows_remote=18890 dedup_ratio=0.329651 router_tokens_per_expert=[280, 240, 265, 256, 248, 283, 256, 275, 255, 263, 258, 255, 243, 251, 220, 247, 253, 236, 256, 234, 262, 285, 241, 277, 249, 262, 226, 244, 270, 247, 254, 279, 267, 257, 240, 283, 245, 284, 251, 260, 282, 256, 252, 256, 236, 245, 259, 258, 266, 250, 224, 271, 279, 243, 237, 258, 239, 249, 259, 281, 255, 246, 271, 260, 249, 254, 273, 268, 262, 237, 241, 258, 254, 279, 256, 254, 259, 270, 250, 265, 267, 246, 268, 266, 240, 244, 265, 234, 267, 259, 266, 241, 235, 246, 268, 258, 260, 256, 223, 243, 252, 263, 281, 277, 258, 259, 230, 291, 251, 230, 251, 237, 260, 247, 269, 291, 238, 255, 245, 233, 266, 282, 258, 277, 264, 266, 249, 218] payload_rows_per_dst=[5512, 5528, 5548, 5548, 5504, 5452, 5364, 5476] dup_rows_per_dst=[2678, 2622, 2714, 2628, 2754, 2688, 2760, 2760] combine_rows_per_src=[5512, 5528, 5548, 5548, 5504, 5452, 5364, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229128 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671752 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114376 weight_movement_bytes=18874368 total_nvlink_bytes=331988744 weight_to_token_ratio=0.060279 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229648 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114640 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999632 weight_movement_bytes=18874368 total_nvlink_bytes=332874000 weight_to_token_ratio=0.060110 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38318 dup_rows=21786 dup_rows_remote=19104 dedup_ratio=0.332428 router_tokens_per_expert=[242, 278, 235, 283, 257, 286, 230, 253, 259, 265, 262, 271, 251, 240, 265, 235, 284, 236, 273, 249, 259, 244, 225, 264, 257, 258, 267, 245, 265, 244, 255, 246, 222, 280, 242, 281, 274, 283, 244, 264, 224, 236, 228, 262, 273, 260, 265, 270, 248, 247, 268, 266, 274, 254, 249, 265, 256, 273, 262, 272, 280, 279, 250, 227, 264, 288, 267, 261, 255, 231, 251, 261, 251, 250, 253, 241, 257, 243, 265, 263, 272, 251, 250, 240, 231, 248, 235, 290, 254, 245, 225, 255, 250, 262, 256, 272, 253, 265, 257, 253, 234, 236, 262, 256, 269, 240, 238, 252, 263, 264, 250, 265, 260, 244, 260, 300, 270, 271, 251, 241, 244, 262, 265, 220, 259, 239, 251, 276] payload_rows_per_dst=[5514, 5438, 5546, 5470, 5452, 5414, 5432, 5484] dup_rows_per_dst=[2710, 2704, 2670, 2870, 2750, 2658, 2682, 2742] combine_rows_per_src=[5514, 5438, 5546, 5470, 5452, 5414, 5432, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43378 payload_rows_remote=37912 dup_rows=22158 dup_rows_remote=19298 dedup_ratio=0.338104 router_tokens_per_expert=[244, 255, 248, 276, 273, 256, 248, 238, 238, 238, 273, 266, 260, 238, 242, 246, 244, 246, 269, 280, 262, 258, 241, 247, 261, 247, 242, 244, 253, 255, 283, 269, 219, 261, 248, 249, 241, 232, 269, 273, 261, 238, 252, 267, 261, 249, 246, 244, 238, 252, 253, 277, 261, 252, 254, 271, 264, 259, 280, 230, 248, 279, 251, 246, 246, 263, 275, 281, 280, 248, 255, 292, 256, 233, 253, 245, 274, 260, 264, 242, 242, 237, 238, 251, 281, 293, 289, 277, 264, 248, 239, 291, 244, 262, 261, 264, 243, 241, 260, 241, 274, 253, 270, 240, 235, 251, 238, 234, 262, 238, 249, 264, 256, 279, 275, 240, 276, 299, 273, 275, 234, 242, 240, 262, 253, 241, 261, 256] payload_rows_per_dst=[5248, 5448, 5464, 5442, 5322, 5612, 5376, 5466] dup_rows_per_dst=[2416, 2754, 2970, 2786, 2736, 3026, 2610, 2860] combine_rows_per_src=[5248, 5448, 5464, 5442, 5322, 5612, 5376, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38308 dup_rows=21758 dup_rows_remote=19086 dedup_ratio=0.332001 router_tokens_per_expert=[256, 253, 260, 287, 261, 251, 264, 261, 228, 245, 270, 244, 260, 273, 239, 264, 222, 272, 251, 253, 247, 260, 248, 265, 247, 268, 251, 266, 268, 250, 239, 241, 254, 239, 281, 248, 263, 271, 243, 248, 275, 267, 259, 234, 263, 242, 268, 254, 266, 242, 230, 238, 268, 266, 247, 265, 241, 264, 249, 268, 252, 253, 264, 258, 241, 251, 255, 264, 264, 260, 259, 260, 241, 263, 245, 262, 277, 270, 248, 260, 247, 231, 269, 273, 258, 268, 242, 280, 235, 244, 233, 258, 242, 265, 267, 277, 251, 283, 249, 272, 256, 233, 291, 265, 248, 241, 243, 284, 281, 232, 262, 241, 237, 250, 258, 249, 252, 296, 255, 253, 261, 247, 240, 243, 253, 273, 255, 261] payload_rows_per_dst=[5434, 5414, 5522, 5470, 5450, 5492, 5514, 5482] dup_rows_per_dst=[2798, 2682, 2696, 2672, 2790, 2686, 2750, 2684] combine_rows_per_src=[5434, 5414, 5522, 5470, 5450, 5492, 5514, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229096 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157449960 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314670824 weight_movement_bytes=18874368 total_nvlink_bytes=333545192 weight_to_token_ratio=0.059981 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229688 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157180216 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314130744 weight_movement_bytes=18874368 total_nvlink_bytes=333005112 weight_to_token_ratio=0.060084 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155287552 dispatch_meta_bytes=228840 combine_payload_bytes=155287552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155516392 ingress_bytes=211910656 link_bytes=211910656 token_bytes=310803944 weight_movement_bytes=56623104 total_nvlink_bytes=367427048 weight_to_token_ratio=0.182183 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.030423639e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38202 dup_rows=21838 dup_rows_remote=19072 dedup_ratio=0.333221 router_tokens_per_expert=[275, 248, 271, 261, 278, 239, 236, 279, 279, 260, 269, 249, 242, 263, 257, 225, 245, 233, 256, 249, 265, 275, 240, 274, 249, 241, 260, 275, 229, 260, 246, 265, 265, 243, 259, 265, 245, 237, 257, 265, 233, 267, 283, 257, 272, 258, 256, 257, 254, 233, 249, 242, 260, 241, 238, 247, 268, 261, 287, 265, 274, 266, 233, 277, 238, 268, 273, 262, 265, 250, 262, 250, 257, 236, 232, 244, 275, 240, 253, 249, 271, 240, 259, 263, 277, 264, 246, 264, 264, 248, 232, 268, 256, 260, 231, 234, 264, 260, 269, 260, 270, 262, 256, 258, 242, 235, 281, 246, 270, 255, 287, 240, 252, 236, 275, 270, 226, 269, 245, 263, 281, 267, 244, 255, 255, 249, 244, 244] payload_rows_per_dst=[5496, 5372, 5534, 5528, 5414, 5436, 5462, 5456] dup_rows_per_dst=[2766, 2752, 2804, 2662, 2694, 2718, 2748, 2694] combine_rows_per_src=[5496, 5372, 5534, 5528, 5414, 5436, 5462, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38072 dup_rows=21958 dup_rows_remote=19274 dedup_ratio=0.335052 router_tokens_per_expert=[257, 291, 256, 258, 260, 270, 258, 243, 255, 273, 262, 249, 259, 244, 273, 270, 255, 242, 246, 277, 267, 242, 245, 235, 247, 252, 243, 252, 244, 272, 209, 265, 241, 292, 260, 285, 237, 253, 278, 233, 256, 241, 256, 229, 246, 267, 254, 250, 274, 249, 237, 261, 247, 254, 250, 261, 241, 234, 225, 277, 254, 273, 269, 282, 266, 259, 264, 260, 244, 254, 231, 258, 248, 250, 258, 251, 241, 263, 275, 273, 285, 241, 264, 277, 259, 257, 253, 272, 277, 255, 274, 256, 246, 291, 257, 306, 247, 255, 240, 242, 256, 264, 251, 227, 270, 239, 264, 266, 247, 256, 288, 261, 256, 239, 296, 243, 247, 273, 229, 237, 234, 232, 249, 231, 278, 232, 266, 251] payload_rows_per_dst=[5548, 5312, 5368, 5464, 5506, 5558, 5480, 5342] dup_rows_per_dst=[2808, 2674, 2788, 2712, 2684, 2982, 2666, 2644] combine_rows_per_src=[5548, 5312, 5368, 5464, 5506, 5558, 5480, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229576 combine_payload_bytes=156909568 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157139144 ingress_bytes=194658304 link_bytes=194658304 token_bytes=314048712 weight_movement_bytes=37748736 total_nvlink_bytes=351797448 weight_to_token_ratio=0.120200 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865120091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38396 dup_rows=21686 dup_rows_remote=19044 dedup_ratio=0.330902 router_tokens_per_expert=[265, 248, 259, 269, 241, 272, 243, 282, 224, 252, 270, 248, 272, 219, 245, 256, 252, 263, 274, 224, 240, 254, 265, 239, 242, 263, 233, 258, 282, 245, 253, 261, 276, 260, 239, 263, 255, 270, 277, 269, 251, 261, 250, 255, 262, 254, 246, 249, 222, 298, 233, 243, 284, 258, 288, 265, 252, 240, 250, 241, 231, 254, 271, 269, 277, 283, 254, 269, 265, 271, 254, 240, 279, 256, 252, 269, 248, 286, 261, 252, 230, 217, 252, 266, 268, 252, 243, 246, 265, 225, 228, 244, 267, 245, 289, 252, 259, 241, 265, 249, 263, 214, 248, 291, 241, 250, 245, 243, 270, 243, 260, 294, 270, 254, 269, 261, 221, 272, 263, 246, 244, 260, 242, 270, 256, 264, 282, 264] payload_rows_per_dst=[5508, 5454, 5462, 5506, 5530, 5378, 5464, 5548] dup_rows_per_dst=[2728, 2642, 2706, 2692, 2902, 2600, 2688, 2728] combine_rows_per_src=[5508, 5454, 5462, 5506, 5530, 5378, 5464, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38234 dup_rows=21782 dup_rows_remote=18998 dedup_ratio=0.332367 router_tokens_per_expert=[242, 244, 274, 247, 282, 260, 266, 263, 256, 277, 284, 264, 263, 245, 262, 252, 250, 241, 257, 246, 240, 250, 247, 256, 243, 251, 250, 252, 253, 264, 280, 247, 262, 245, 245, 232, 265, 249, 245, 259, 243, 231, 256, 253, 250, 241, 257, 242, 248, 268, 260, 268, 249, 264, 266, 258, 274, 269, 272, 260, 236, 261, 266, 257, 231, 264, 296, 245, 254, 256, 245, 248, 236, 235, 264, 266, 254, 249, 262, 271, 266, 275, 250, 266, 256, 268, 280, 258, 256, 265, 255, 258, 243, 257, 266, 233, 260, 234, 263, 226, 270, 244, 249, 242, 262, 268, 277, 237, 273, 260, 258, 273, 260, 241, 243, 243, 262, 262, 256, 246, 254, 249, 281, 273, 244, 255, 257, 259] payload_rows_per_dst=[5546, 5426, 5338, 5580, 5406, 5520, 5498, 5440] dup_rows_per_dst=[2816, 2628, 2612, 2772, 2746, 2784, 2694, 2730] combine_rows_per_src=[5546, 5426, 5338, 5580, 5406, 5520, 5498, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229096 combine_payload_bytes=156475392 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156704488 ingress_bytes=231972864 link_bytes=231972864 token_bytes=313179880 weight_movement_bytes=75497472 total_nvlink_bytes=388677352 weight_to_token_ratio=0.241067 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222649845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38074 dup_rows=21930 dup_rows_remote=19114 dedup_ratio=0.334625 router_tokens_per_expert=[239, 253, 265, 282, 253, 245, 252, 247, 249, 230, 256, 226, 249, 261, 257, 282, 248, 281, 246, 277, 263, 262, 230, 264, 260, 268, 250, 254, 258, 255, 255, 260, 254, 246, 274, 253, 267, 281, 239, 280, 286, 262, 253, 245, 232, 267, 269, 266, 279, 276, 258, 266, 253, 281, 262, 264, 267, 257, 270, 251, 237, 252, 227, 226, 244, 244, 237, 254, 289, 268, 246, 251, 251, 259, 241, 261, 223, 253, 258, 244, 239, 255, 280, 251, 293, 245, 245, 256, 247, 258, 238, 257, 273, 273, 243, 265, 271, 222, 283, 269, 242, 259, 224, 266, 286, 241, 267, 247, 261, 259, 261, 244, 242, 272, 230, 269, 239, 221, 253, 259, 270, 256, 240, 246, 257, 266, 266, 262] payload_rows_per_dst=[5422, 5472, 5532, 5468, 5410, 5458, 5448, 5396] dup_rows_per_dst=[2670, 2790, 2816, 2784, 2636, 2778, 2756, 2700] combine_rows_per_src=[5422, 5472, 5532, 5468, 5410, 5458, 5448, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=229384 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156172296 ingress_bytes=193691648 link_bytes=193691648 token_bytes=312115208 weight_movement_bytes=37748736 total_nvlink_bytes=349863944 weight_to_token_ratio=0.120945 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855858069e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38420 dup_rows=21644 dup_rows_remote=18944 dedup_ratio=0.330261 router_tokens_per_expert=[240, 287, 243, 249, 239, 241, 258, 276, 247, 272, 256, 248, 254, 254, 239, 249, 262, 263, 282, 225, 278, 253, 267, 273, 250, 254, 253, 255, 236, 273, 265, 265, 252, 264, 256, 269, 223, 286, 258, 253, 252, 258, 267, 266, 264, 251, 237, 242, 267, 255, 277, 248, 267, 260, 255, 235, 260, 255, 261, 249, 280, 254, 242, 259, 242, 240, 272, 232, 253, 263, 266, 284, 264, 246, 247, 270, 249, 267, 242, 256, 236, 289, 238, 253, 233, 257, 271, 247, 262, 240, 274, 257, 272, 240, 250, 279, 263, 259, 262, 263, 257, 219, 280, 248, 274, 251, 224, 271, 267, 260, 235, 253, 261, 259, 246, 248, 255, 251, 236, 252, 241, 250, 234, 256, 263, 272, 280, 259] payload_rows_per_dst=[5460, 5566, 5466, 5536, 5492, 5440, 5472, 5460] dup_rows_per_dst=[2644, 2742, 2730, 2712, 2694, 2756, 2700, 2666] combine_rows_per_src=[5460, 5566, 5466, 5536, 5492, 5440, 5472, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=229760 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157499776 ingress_bytes=195018752 link_bytes=195018752 token_bytes=314769792 weight_movement_bytes=37748736 total_nvlink_bytes=352518528 weight_to_token_ratio=0.119925 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868573726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38120 dup_rows=21844 dup_rows_remote=19042 dedup_ratio=0.333313 router_tokens_per_expert=[241, 247, 249, 229, 257, 238, 262, 269, 300, 249, 278, 262, 270, 240, 237, 240, 259, 255, 238, 246, 251, 248, 262, 281, 258, 256, 278, 262, 269, 258, 243, 267, 265, 220, 266, 259, 239, 268, 262, 238, 266, 277, 276, 246, 276, 261, 251, 287, 255, 254, 251, 292, 269, 278, 296, 266, 260, 250, 240, 267, 269, 243, 265, 247, 222, 258, 251, 252, 251, 256, 242, 247, 238, 263, 246, 239, 261, 264, 270, 249, 252, 231, 260, 269, 270, 268, 263, 271, 257, 288, 240, 267, 229, 264, 227, 251, 241, 266, 258, 244, 250, 256, 261, 289, 236, 273, 273, 233, 243, 266, 248, 269, 249, 254, 256, 249, 250, 228, 261, 262, 247, 248, 244, 231, 248, 244, 267, 250] payload_rows_per_dst=[5466, 5320, 5510, 5518, 5352, 5476, 5478, 5572] dup_rows_per_dst=[2670, 2544, 2804, 2886, 2666, 2738, 2734, 2802] combine_rows_per_src=[5466, 5320, 5510, 5518, 5352, 5476, 5478, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228928 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156835392 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441856 weight_movement_bytes=37748736 total_nvlink_bytes=351190592 weight_to_token_ratio=0.120433 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38222 dup_rows=21788 dup_rows_remote=19042 dedup_ratio=0.332458 router_tokens_per_expert=[264, 229, 256, 254, 247, 243, 229, 233, 235, 268, 251, 292, 290, 256, 243, 248, 252, 272, 261, 239, 294, 240, 243, 244, 259, 303, 224, 251, 283, 258, 233, 260, 237, 260, 246, 238, 252, 260, 280, 253, 262, 233, 255, 279, 256, 269, 238, 272, 261, 277, 254, 272, 262, 266, 252, 259, 261, 259, 271, 236, 257, 242, 253, 254, 242, 246, 262, 247, 239, 277, 249, 262, 257, 236, 231, 255, 272, 228, 271, 284, 236, 252, 259, 275, 246, 257, 281, 269, 247, 267, 276, 247, 262, 247, 263, 278, 248, 229, 264, 244, 250, 276, 273, 259, 251, 270, 267, 221, 229, 290, 251, 241, 241, 243, 272, 260, 228, 265, 261, 298, 252, 244, 240, 258, 277, 247, 278, 241] payload_rows_per_dst=[5458, 5492, 5434, 5526, 5444, 5512, 5448, 5434] dup_rows_per_dst=[2618, 2740, 2746, 2746, 2672, 2812, 2678, 2776] combine_rows_per_src=[5458, 5492, 5434, 5526, 5444, 5512, 5448, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=228752 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156179856 ingress_bytes=193699840 link_bytes=193699840 token_bytes=312130960 weight_movement_bytes=37748736 total_nvlink_bytes=349879696 weight_to_token_ratio=0.120939 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855936561e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229456 combine_payload_bytes=157368320 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157597776 ingress_bytes=195117056 link_bytes=195117056 token_bytes=314966096 weight_movement_bytes=37748736 total_nvlink_bytes=352714832 weight_to_token_ratio=0.119850 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869515627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228648 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156368168 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507688 weight_movement_bytes=18874368 total_nvlink_bytes=331382056 weight_to_token_ratio=0.060396 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38134 dup_rows=21840 dup_rows_remote=19120 dedup_ratio=0.333252 router_tokens_per_expert=[244, 262, 267, 245, 281, 269, 259, 268, 261, 245, 223, 262, 276, 259, 240, 280, 233, 259, 255, 250, 255, 229, 245, 264, 262, 246, 248, 267, 264, 242, 249, 248, 240, 264, 252, 270, 230, 232, 252, 254, 232, 280, 250, 238, 244, 242, 265, 227, 255, 265, 275, 237, 260, 267, 289, 276, 271, 259, 247, 264, 254, 270, 260, 246, 245, 254, 246, 269, 241, 256, 258, 233, 267, 250, 264, 272, 272, 268, 263, 282, 259, 245, 241, 302, 254, 260, 267, 264, 262, 242, 255, 281, 263, 225, 225, 252, 247, 258, 268, 264, 263, 250, 263, 263, 281, 240, 238, 248, 265, 251, 256, 244, 273, 264, 245, 262, 239, 266, 251, 251, 268, 234, 238, 246, 262, 295, 261, 253] payload_rows_per_dst=[5562, 5496, 5440, 5398, 5640, 5174, 5516, 5470] dup_rows_per_dst=[2720, 2798, 2750, 2766, 3046, 2368, 2682, 2710] combine_rows_per_src=[5562, 5496, 5440, 5398, 5640, 5174, 5516, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229056 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156786368 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313343680 weight_movement_bytes=37748736 total_nvlink_bytes=351092416 weight_to_token_ratio=0.120471 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38314 dup_rows=21838 dup_rows_remote=18980 dedup_ratio=0.333221 router_tokens_per_expert=[253, 266, 281, 252, 249, 247, 278, 236, 245, 222, 285, 258, 228, 271, 261, 262, 268, 261, 244, 262, 238, 239, 294, 254, 247, 266, 252, 266, 258, 237, 267, 238, 253, 247, 256, 256, 292, 263, 277, 243, 288, 246, 266, 250, 270, 240, 274, 228, 218, 255, 241, 231, 263, 239, 238, 280, 265, 256, 254, 259, 241, 262, 248, 268, 243, 257, 256, 268, 275, 267, 276, 270, 273, 254, 227, 257, 229, 239, 250, 280, 244, 247, 249, 260, 284, 252, 250, 280, 233, 280, 244, 272, 246, 239, 257, 281, 249, 271, 251, 248, 265, 283, 249, 266, 258, 228, 274, 268, 266, 266, 255, 259, 241, 263, 246, 245, 281, 230, 224, 260, 216, 265, 260, 248, 268, 256, 279, 239] payload_rows_per_dst=[5450, 5542, 5462, 5402, 5384, 5486, 5548, 5424] dup_rows_per_dst=[2738, 2640, 2836, 2634, 2858, 2750, 2764, 2618] combine_rows_per_src=[5450, 5542, 5462, 5402, 5384, 5486, 5548, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38162 dup_rows=21818 dup_rows_remote=18950 dedup_ratio=0.332916 router_tokens_per_expert=[266, 274, 288, 244, 267, 264, 247, 269, 275, 258, 274, 238, 271, 228, 255, 286, 257, 247, 297, 250, 244, 243, 275, 256, 259, 249, 260, 236, 276, 252, 260, 257, 240, 281, 234, 254, 233, 278, 253, 256, 251, 255, 257, 253, 234, 223, 242, 216, 262, 239, 250, 249, 242, 291, 274, 248, 268, 258, 244, 244, 258, 245, 278, 238, 264, 259, 251, 263, 269, 249, 255, 250, 245, 271, 250, 280, 249, 267, 251, 234, 235, 235, 222, 282, 272, 246, 246, 254, 259, 256, 270, 256, 254, 262, 261, 274, 240, 315, 263, 277, 253, 265, 239, 243, 243, 267, 254, 244, 252, 256, 237, 240, 252, 265, 241, 235, 260, 270, 252, 260, 262, 251, 277, 264, 257, 259, 247, 267] payload_rows_per_dst=[5546, 5556, 5304, 5400, 5484, 5444, 5456, 5528] dup_rows_per_dst=[2862, 2868, 2616, 2776, 2730, 2536, 2720, 2710] combine_rows_per_src=[5546, 5556, 5304, 5400, 5484, 5444, 5456, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38290 dup_rows=21774 dup_rows_remote=18974 dedup_ratio=0.332245 router_tokens_per_expert=[261, 234, 226, 256, 239, 256, 251, 248, 258, 262, 250, 234, 236, 247, 272, 237, 260, 294, 262, 261, 253, 259, 248, 249, 228, 263, 257, 268, 286, 267, 253, 271, 258, 251, 271, 265, 280, 248, 260, 265, 229, 253, 249, 238, 252, 250, 247, 222, 258, 249, 303, 261, 245, 281, 255, 259, 230, 239, 260, 247, 251, 244, 273, 279, 242, 276, 241, 242, 230, 245, 258, 266, 271, 273, 260, 248, 252, 235, 255, 256, 273, 252, 244, 263, 258, 243, 276, 265, 262, 248, 274, 240, 257, 257, 250, 274, 251, 249, 259, 242, 270, 239, 274, 269, 253, 246, 280, 268, 220, 288, 290, 270, 304, 275, 269, 262, 266, 281, 232, 250, 228, 266, 235, 262, 259, 226, 247, 234] payload_rows_per_dst=[5324, 5552, 5434, 5524, 5460, 5472, 5536, 5460] dup_rows_per_dst=[2610, 2806, 2642, 2744, 2640, 2800, 2800, 2732] combine_rows_per_src=[5324, 5552, 5434, 5524, 5460, 5472, 5536, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=229016 combine_payload_bytes=156196864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156425880 ingress_bytes=193945600 link_bytes=193945600 token_bytes=312622744 weight_movement_bytes=37748736 total_nvlink_bytes=350371480 weight_to_token_ratio=0.120749 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858291312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38236 dup_rows=21746 dup_rows_remote=19088 dedup_ratio=0.331818 router_tokens_per_expert=[260, 260, 260, 263, 256, 238, 250, 244, 273, 258, 238, 251, 263, 275, 253, 259, 235, 235, 233, 269, 253, 283, 252, 261, 258, 259, 267, 247, 244, 241, 214, 286, 217, 239, 268, 226, 277, 245, 287, 259, 237, 251, 264, 263, 260, 260, 270, 283, 260, 245, 275, 246, 232, 240, 263, 262, 246, 248, 254, 281, 243, 240, 277, 265, 252, 261, 254, 268, 237, 274, 259, 247, 275, 256, 259, 254, 266, 255, 249, 232, 271, 273, 255, 252, 261, 276, 270, 243, 258, 259, 242, 277, 259, 245, 285, 289, 244, 275, 235, 262, 266, 250, 248, 270, 254, 265, 247, 245, 272, 283, 239, 262, 243, 267, 237, 266, 265, 253, 230, 258, 259, 249, 243, 232, 246, 262, 259, 248] payload_rows_per_dst=[5456, 5448, 5554, 5472, 5422, 5564, 5482, 5392] dup_rows_per_dst=[2746, 2626, 2658, 2682, 2774, 2866, 2752, 2642] combine_rows_per_src=[5456, 5448, 5554, 5472, 5422, 5564, 5482, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38348 dup_rows=21754 dup_rows_remote=19052 dedup_ratio=0.331940 router_tokens_per_expert=[252, 237, 230, 251, 227, 226, 263, 237, 273, 272, 296, 255, 245, 280, 226, 250, 231, 262, 258, 235, 281, 271, 257, 238, 243, 240, 207, 275, 259, 274, 255, 257, 261, 235, 252, 266, 249, 251, 271, 235, 253, 268, 253, 248, 234, 270, 274, 244, 264, 282, 291, 267, 260, 248, 266, 262, 252, 260, 272, 270, 264, 256, 255, 250, 236, 247, 246, 239, 232, 269, 301, 262, 273, 263, 262, 264, 251, 247, 276, 258, 238, 228, 253, 258, 266, 273, 244, 285, 273, 255, 243, 261, 269, 256, 244, 251, 286, 238, 278, 258, 251, 258, 276, 272, 264, 258, 223, 227, 260, 238, 242, 239, 282, 223, 240, 273, 255, 253, 262, 256, 242, 246, 257, 281, 291, 246, 271, 253] payload_rows_per_dst=[5440, 5440, 5436, 5552, 5460, 5472, 5434, 5548] dup_rows_per_dst=[2600, 2646, 2692, 2886, 2792, 2722, 2702, 2714] combine_rows_per_src=[5440, 5440, 5436, 5552, 5460, 5472, 5434, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229176 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163320 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314097464 weight_movement_bytes=18874368 total_nvlink_bytes=332971832 weight_to_token_ratio=0.060091 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38190 dup_rows=21790 dup_rows_remote=18996 dedup_ratio=0.332489 router_tokens_per_expert=[250, 261, 265, 260, 259, 250, 244, 239, 233, 244, 273, 262, 285, 257, 235, 241, 244, 254, 240, 243, 285, 244, 242, 255, 250, 261, 245, 239, 254, 268, 245, 271, 242, 279, 280, 258, 261, 271, 252, 261, 269, 262, 257, 229, 255, 266, 218, 259, 254, 257, 292, 261, 262, 265, 275, 243, 282, 275, 232, 261, 265, 254, 265, 254, 269, 272, 242, 251, 247, 262, 273, 242, 237, 266, 255, 270, 233, 258, 277, 239, 253, 240, 261, 268, 274, 268, 255, 239, 251, 263, 253, 257, 278, 241, 261, 264, 268, 268, 256, 255, 243, 243, 229, 247, 278, 278, 263, 247, 268, 254, 255, 220, 252, 229, 270, 286, 262, 256, 233, 236, 259, 270, 250, 277, 233, 273, 241, 236] payload_rows_per_dst=[5406, 5390, 5390, 5520, 5458, 5580, 5446, 5556] dup_rows_per_dst=[2630, 2690, 2624, 2790, 2588, 2976, 2698, 2794] combine_rows_per_src=[5406, 5390, 5390, 5520, 5458, 5580, 5446, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156835840 dispatch_meta_bytes=229056 combine_payload_bytes=156835840 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157064896 ingress_bytes=213458944 link_bytes=213458944 token_bytes=313900736 weight_movement_bytes=56623104 total_nvlink_bytes=370523840 weight_to_token_ratio=0.180385 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045258573e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228448 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540000 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851552 weight_movement_bytes=18874368 total_nvlink_bytes=331725920 weight_to_token_ratio=0.060330 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38152 dup_rows=21866 dup_rows_remote=19144 dedup_ratio=0.333649 router_tokens_per_expert=[237, 256, 261, 275, 251, 267, 236, 261, 262, 243, 244, 273, 263, 246, 231, 249, 262, 232, 248, 264, 242, 263, 275, 258, 244, 244, 243, 238, 254, 250, 243, 253, 247, 283, 255, 252, 261, 283, 264, 259, 268, 248, 262, 266, 249, 266, 275, 265, 264, 277, 259, 253, 259, 245, 265, 227, 255, 269, 248, 255, 250, 259, 255, 280, 246, 257, 268, 267, 284, 268, 246, 251, 230, 230, 243, 265, 252, 237, 244, 255, 260, 260, 235, 271, 256, 268, 259, 273, 287, 267, 246, 274, 249, 249, 232, 253, 248, 282, 274, 262, 253, 262, 248, 259, 229, 272, 270, 255, 238, 246, 240, 266, 278, 258, 255, 273, 243, 241, 259, 273, 253, 263, 249, 248, 266, 230, 247, 255] payload_rows_per_dst=[5442, 5314, 5530, 5518, 5376, 5512, 5474, 5504] dup_rows_per_dst=[2668, 2712, 2876, 2722, 2710, 2766, 2734, 2678] combine_rows_per_src=[5442, 5314, 5530, 5518, 5376, 5512, 5474, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229296 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156843952 ingress_bytes=194363392 link_bytes=194363392 token_bytes=313458608 weight_movement_bytes=37748736 total_nvlink_bytes=351207344 weight_to_token_ratio=0.120427 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862294389e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229600 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157303008 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314376416 weight_movement_bytes=37748736 total_nvlink_bytes=352125152 weight_to_token_ratio=0.120075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228744 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156654984 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081224 weight_movement_bytes=18874368 total_nvlink_bytes=331955592 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229184 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156499776 ingress_bytes=194019328 link_bytes=194019328 token_bytes=312770368 weight_movement_bytes=37748736 total_nvlink_bytes=350519104 weight_to_token_ratio=0.120692 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858997737e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38210 dup_rows=21890 dup_rows_remote=19120 dedup_ratio=0.334015 router_tokens_per_expert=[247, 273, 263, 264, 273, 244, 250, 254, 267, 218, 253, 248, 279, 261, 253, 247, 267, 266, 274, 255, 247, 257, 260, 235, 243, 251, 265, 258, 290, 238, 249, 246, 266, 229, 278, 277, 255, 243, 228, 243, 262, 261, 271, 261, 246, 268, 245, 249, 265, 249, 261, 252, 285, 281, 251, 245, 272, 254, 250, 233, 262, 249, 253, 255, 256, 260, 252, 243, 253, 259, 267, 239, 232, 258, 302, 253, 280, 268, 267, 284, 267, 240, 233, 260, 241, 226, 240, 288, 263, 258, 254, 248, 252, 292, 261, 222, 260, 270, 223, 231, 273, 254, 246, 229, 233, 253, 233, 273, 248, 267, 279, 253, 295, 275, 245, 248, 247, 230, 247, 272, 250, 265, 243, 273, 242, 260, 285, 254] payload_rows_per_dst=[5436, 5492, 5438, 5406, 5504, 5380, 5468, 5522] dup_rows_per_dst=[2770, 2726, 2756, 2764, 2842, 2710, 2582, 2740] combine_rows_per_src=[5436, 5492, 5438, 5406, 5504, 5380, 5468, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38176 dup_rows=21846 dup_rows_remote=19102 dedup_ratio=0.333344 router_tokens_per_expert=[256, 289, 270, 254, 250, 277, 280, 256, 234, 252, 260, 257, 280, 255, 251, 231, 249, 250, 232, 252, 235, 274, 261, 266, 265, 267, 239, 260, 275, 276, 258, 230, 270, 256, 257, 254, 276, 270, 240, 245, 241, 273, 261, 280, 241, 248, 251, 260, 257, 262, 244, 277, 266, 239, 245, 268, 242, 248, 244, 263, 258, 240, 271, 231, 220, 260, 252, 237, 257, 285, 250, 288, 251, 254, 253, 242, 275, 261, 279, 265, 254, 249, 260, 235, 266, 227, 260, 242, 252, 256, 275, 291, 276, 227, 236, 266, 260, 240, 282, 286, 252, 226, 266, 252, 228, 229, 247, 265, 255, 273, 274, 254, 261, 251, 251, 264, 270, 233, 244, 258, 259, 234, 253, 249, 262, 256, 276, 238] payload_rows_per_dst=[5572, 5424, 5522, 5464, 5514, 5378, 5446, 5370] dup_rows_per_dst=[2732, 2754, 2724, 2646, 2744, 2766, 2732, 2748] combine_rows_per_src=[5572, 5424, 5522, 5464, 5514, 5378, 5446, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229320 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156737480 ingress_bytes=213131264 link_bytes=213131264 token_bytes=313245640 weight_movement_bytes=56623104 total_nvlink_bytes=369868744 weight_to_token_ratio=0.180763 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042118905e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38184 dup_rows=21828 dup_rows_remote=19006 dedup_ratio=0.333069 router_tokens_per_expert=[253, 256, 242, 278, 240, 265, 265, 243, 252, 265, 265, 270, 275, 262, 248, 252, 258, 262, 233, 245, 230, 240, 255, 260, 278, 262, 235, 241, 263, 271, 242, 246, 256, 265, 275, 248, 260, 266, 240, 279, 241, 243, 259, 259, 249, 265, 269, 266, 254, 238, 256, 249, 250, 283, 246, 252, 259, 265, 233, 247, 237, 237, 243, 260, 260, 247, 277, 229, 254, 270, 245, 266, 241, 263, 280, 278, 282, 246, 249, 234, 267, 270, 286, 255, 278, 253, 256, 264, 264, 235, 258, 247, 260, 265, 251, 264, 264, 259, 243, 281, 289, 260, 234, 274, 242, 253, 241, 237, 267, 276, 244, 247, 269, 231, 259, 273, 246, 268, 233, 249, 264, 275, 258, 208, 242, 273, 273, 241] payload_rows_per_dst=[5526, 5398, 5520, 5318, 5508, 5524, 5456, 5458] dup_rows_per_dst=[2736, 2644, 2760, 2700, 2734, 2822, 2766, 2666] combine_rows_per_src=[5526, 5398, 5520, 5318, 5508, 5524, 5456, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38286 dup_rows=21838 dup_rows_remote=19220 dedup_ratio=0.333221 router_tokens_per_expert=[257, 238, 314, 254, 250, 235, 230, 277, 244, 234, 257, 240, 256, 261, 284, 260, 256, 277, 251, 253, 249, 259, 256, 262, 230, 257, 261, 239, 238, 272, 226, 229, 258, 253, 267, 276, 274, 247, 267, 261, 248, 270, 245, 268, 254, 255, 251, 264, 256, 217, 244, 255, 240, 269, 268, 261, 270, 240, 258, 282, 268, 253, 246, 254, 270, 257, 241, 255, 268, 246, 246, 241, 243, 273, 248, 259, 260, 273, 260, 268, 274, 228, 265, 260, 250, 246, 250, 214, 237, 248, 236, 286, 241, 253, 256, 259, 280, 276, 276, 258, 248, 243, 247, 275, 261, 246, 253, 277, 243, 290, 245, 271, 233, 300, 243, 257, 277, 243, 261, 267, 243, 266, 257, 254, 275, 254, 242, 251] payload_rows_per_dst=[5504, 5412, 5554, 5422, 5458, 5380, 5544, 5424] dup_rows_per_dst=[2678, 2618, 2762, 2740, 2758, 2626, 2834, 2822] combine_rows_per_src=[5504, 5412, 5554, 5422, 5458, 5380, 5544, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38334 dup_rows=21750 dup_rows_remote=18996 dedup_ratio=0.331879 router_tokens_per_expert=[248, 271, 252, 266, 275, 237, 260, 269, 234, 239, 273, 239, 279, 255, 260, 242, 247, 265, 258, 245, 273, 259, 245, 250, 246, 272, 245, 249, 242, 233, 285, 232, 259, 246, 242, 235, 276, 250, 270, 267, 248, 263, 271, 260, 238, 264, 246, 268, 283, 239, 261, 267, 248, 239, 257, 255, 288, 260, 262, 243, 270, 257, 262, 283, 239, 275, 294, 226, 283, 250, 291, 235, 235, 257, 247, 208, 261, 278, 249, 245, 287, 271, 263, 251, 249, 271, 254, 257, 240, 248, 276, 258, 252, 255, 230, 273, 242, 235, 266, 258, 258, 257, 264, 270, 238, 273, 253, 258, 242, 266, 259, 243, 227, 274, 259, 282, 256, 275, 268, 227, 248, 281, 249, 245, 221, 228, 268, 248] payload_rows_per_dst=[5512, 5374, 5452, 5526, 5504, 5486, 5498, 5434] dup_rows_per_dst=[2686, 2718, 2754, 2822, 2642, 2784, 2666, 2678] combine_rows_per_src=[5512, 5374, 5452, 5526, 5504, 5486, 5498, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229112 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156598008 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312966904 weight_movement_bytes=37748736 total_nvlink_bytes=350715640 weight_to_token_ratio=0.120616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38300 dup_rows=21788 dup_rows_remote=19090 dedup_ratio=0.332458 router_tokens_per_expert=[257, 239, 270, 269, 229, 273, 272, 257, 279, 225, 240, 254, 260, 263, 276, 285, 225, 222, 259, 246, 235, 261, 281, 269, 245, 267, 262, 273, 270, 239, 245, 261, 250, 262, 249, 261, 276, 245, 254, 241, 258, 264, 246, 256, 247, 231, 274, 258, 293, 286, 254, 275, 256, 264, 271, 256, 258, 281, 242, 234, 228, 246, 239, 250, 284, 277, 251, 250, 209, 243, 259, 246, 272, 243, 253, 267, 222, 237, 247, 267, 259, 246, 265, 227, 268, 267, 249, 243, 241, 268, 265, 266, 259, 232, 263, 267, 231, 272, 270, 255, 279, 245, 266, 253, 250, 216, 258, 258, 253, 236, 264, 267, 263, 263, 252, 276, 280, 235, 253, 263, 258, 262, 257, 245, 279, 251, 260, 273] payload_rows_per_dst=[5482, 5440, 5434, 5498, 5442, 5482, 5448, 5522] dup_rows_per_dst=[2814, 2680, 2710, 2768, 2612, 2688, 2698, 2818] combine_rows_per_src=[5482, 5440, 5434, 5498, 5442, 5482, 5448, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38336 dup_rows=21748 dup_rows_remote=19078 dedup_ratio=0.331848 router_tokens_per_expert=[230, 248, 246, 265, 243, 254, 276, 251, 260, 274, 244, 266, 267, 239, 263, 250, 246, 265, 233, 279, 243, 257, 269, 240, 257, 247, 272, 267, 271, 261, 237, 252, 250, 248, 268, 272, 247, 261, 278, 243, 245, 271, 281, 249, 244, 258, 282, 265, 239, 245, 268, 253, 253, 296, 252, 250, 251, 246, 250, 244, 273, 245, 244, 252, 252, 267, 257, 256, 252, 262, 263, 257, 261, 242, 285, 254, 272, 260, 254, 259, 269, 242, 253, 260, 276, 268, 266, 262, 253, 240, 280, 251, 246, 234, 260, 262, 253, 247, 202, 269, 237, 249, 251, 251, 261, 243, 262, 276, 258, 248, 221, 259, 239, 249, 250, 273, 263, 293, 224, 262, 227, 251, 264, 255, 263, 280, 265, 253] payload_rows_per_dst=[5466, 5456, 5528, 5452, 5520, 5494, 5426, 5446] dup_rows_per_dst=[2686, 2736, 2796, 2670, 2786, 2750, 2548, 2776] combine_rows_per_src=[5466, 5456, 5528, 5452, 5520, 5494, 5426, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43502 payload_rows_remote=37980 dup_rows=22034 dup_rows_remote=19178 dedup_ratio=0.336212 router_tokens_per_expert=[268, 259, 240, 261, 230, 262, 266, 250, 300, 240, 234, 266, 236, 273, 271, 257, 255, 235, 274, 254, 247, 241, 244, 242, 237, 253, 260, 266, 248, 261, 245, 255, 270, 246, 278, 282, 269, 227, 283, 254, 229, 255, 238, 240, 252, 273, 224, 255, 261, 295, 240, 231, 248, 250, 251, 258, 241, 267, 262, 274, 260, 250, 266, 235, 253, 255, 247, 267, 242, 223, 256, 263, 266, 267, 269, 242, 255, 286, 244, 274, 255, 264, 268, 219, 267, 261, 283, 244, 242, 271, 244, 264, 273, 278, 254, 256, 248, 246, 231, 246, 235, 238, 224, 261, 273, 271, 252, 271, 265, 266, 271, 272, 274, 261, 273, 280, 247, 248, 266, 261, 268, 258, 242, 247, 256, 234, 290, 247] payload_rows_per_dst=[5438, 5394, 5432, 5374, 5324, 5512, 5506, 5522] dup_rows_per_dst=[2788, 2640, 2718, 2640, 2700, 2864, 2828, 2856] combine_rows_per_src=[5438, 5394, 5432, 5374, 5324, 5512, 5506, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228760 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156630424 ingress_bytes=194150400 link_bytes=194150400 token_bytes=313032088 weight_movement_bytes=37748736 total_nvlink_bytes=350780824 weight_to_token_ratio=0.120591 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860253605e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=230024 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157049480 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868936 weight_movement_bytes=18874368 total_nvlink_bytes=332743304 weight_to_token_ratio=0.060135 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229320 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245384 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261448 weight_movement_bytes=18874368 total_nvlink_bytes=333135816 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229560 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157106360 ingress_bytes=175751168 link_bytes=175751168 token_bytes=313983160 weight_movement_bytes=18874368 total_nvlink_bytes=332857528 weight_to_token_ratio=0.060113 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683961217e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229656 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157253912 ingress_bytes=213647360 link_bytes=213647360 token_bytes=314278168 weight_movement_bytes=56623104 total_nvlink_bytes=370901272 weight_to_token_ratio=0.180169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047063882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155566080 dispatch_meta_bytes=228632 combine_payload_bytes=155566080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155794712 ingress_bytes=193314816 link_bytes=193314816 token_bytes=311360792 weight_movement_bytes=37748736 total_nvlink_bytes=349109528 weight_to_token_ratio=0.121238 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852247450e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38320 dup_rows=21646 dup_rows_remote=18864 dedup_ratio=0.330292 router_tokens_per_expert=[261, 254, 252, 263, 261, 243, 246, 253, 256, 284, 312, 258, 241, 264, 238, 231, 240, 235, 256, 235, 254, 268, 254, 269, 251, 232, 282, 228, 254, 245, 244, 267, 258, 244, 247, 265, 253, 229, 243, 268, 235, 259, 255, 248, 238, 297, 254, 274, 264, 244, 250, 246, 234, 237, 266, 234, 242, 253, 231, 262, 282, 250, 236, 290, 254, 268, 255, 261, 253, 255, 254, 255, 249, 262, 265, 263, 260, 279, 241, 242, 268, 261, 269, 239, 244, 260, 241, 312, 230, 239, 277, 265, 268, 237, 289, 228, 245, 272, 250, 264, 266, 254, 230, 265, 249, 267, 285, 260, 240, 249, 261, 270, 269, 253, 279, 269, 238, 271, 268, 243, 258, 247, 271, 273, 267, 258, 247, 268] payload_rows_per_dst=[5570, 5816, 5412, 5336, 5476, 5262, 5552, 5466] dup_rows_per_dst=[2782, 2836, 2722, 2588, 2776, 2368, 2912, 2662] combine_rows_per_src=[5570, 5816, 5412, 5336, 5476, 5262, 5552, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38298 dup_rows=21702 dup_rows_remote=19022 dedup_ratio=0.331146 router_tokens_per_expert=[252, 268, 290, 263, 230, 279, 245, 280, 262, 238, 235, 243, 248, 232, 250, 271, 235, 268, 268, 267, 258, 242, 245, 234, 261, 244, 252, 262, 286, 283, 257, 274, 255, 237, 249, 268, 265, 234, 296, 232, 256, 259, 244, 239, 276, 255, 267, 256, 247, 285, 270, 248, 245, 265, 237, 251, 256, 255, 271, 279, 277, 256, 273, 246, 237, 252, 284, 297, 230, 269, 267, 266, 247, 263, 245, 255, 260, 249, 232, 255, 240, 255, 283, 232, 276, 263, 271, 255, 245, 247, 270, 240, 267, 240, 263, 257, 265, 261, 246, 230, 260, 239, 296, 248, 273, 256, 241, 236, 220, 262, 238, 252, 265, 265, 241, 238, 247, 249, 249, 254, 232, 287, 268, 231, 245, 271, 273, 247] payload_rows_per_dst=[5418, 5580, 5468, 5510, 5536, 5436, 5440, 5446] dup_rows_per_dst=[2754, 2692, 2708, 2812, 2680, 2772, 2606, 2678] combine_rows_per_src=[5418, 5580, 5468, 5510, 5536, 5436, 5440, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228736 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157187456 ingress_bytes=194707456 link_bytes=194707456 token_bytes=314146176 weight_movement_bytes=37748736 total_nvlink_bytes=351894912 weight_to_token_ratio=0.120163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865591041e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38324 dup_rows=21712 dup_rows_remote=19064 dedup_ratio=0.331299 router_tokens_per_expert=[280, 249, 269, 277, 252, 290, 292, 238, 233, 258, 260, 250, 253, 253, 269, 236, 257, 246, 255, 232, 246, 268, 226, 273, 261, 269, 263, 268, 257, 268, 250, 241, 224, 264, 261, 249, 240, 236, 248, 216, 238, 282, 227, 253, 270, 254, 254, 219, 250, 256, 252, 267, 254, 247, 283, 260, 247, 263, 268, 249, 273, 262, 217, 232, 223, 281, 287, 248, 280, 239, 267, 276, 249, 258, 232, 263, 257, 273, 271, 270, 256, 230, 253, 261, 283, 260, 242, 229, 263, 273, 267, 275, 244, 245, 247, 246, 246, 257, 234, 266, 243, 247, 239, 305, 257, 282, 287, 241, 246, 262, 260, 249, 251, 278, 263, 239, 276, 257, 250, 271, 265, 286, 247, 299, 240, 228, 240, 255] payload_rows_per_dst=[5548, 5400, 5384, 5428, 5494, 5500, 5520, 5550] dup_rows_per_dst=[2770, 2760, 2486, 2732, 2854, 2648, 2722, 2740] combine_rows_per_src=[5548, 5400, 5384, 5428, 5494, 5500, 5520, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38314 dup_rows=21796 dup_rows_remote=19128 dedup_ratio=0.332581 router_tokens_per_expert=[262, 270, 262, 257, 260, 243, 234, 222, 265, 239, 241, 262, 249, 244, 233, 257, 249, 267, 250, 224, 279, 268, 257, 247, 271, 237, 238, 232, 255, 276, 253, 244, 235, 282, 261, 240, 249, 264, 286, 247, 245, 261, 253, 264, 232, 230, 278, 248, 270, 249, 283, 293, 237, 251, 255, 288, 225, 251, 234, 268, 259, 257, 264, 261, 238, 246, 261, 245, 279, 259, 249, 263, 221, 254, 266, 248, 252, 249, 252, 253, 253, 255, 251, 253, 251, 254, 271, 245, 266, 258, 255, 279, 259, 267, 249, 238, 272, 237, 250, 258, 257, 262, 283, 268, 258, 262, 248, 241, 258, 260, 275, 250, 304, 293, 255, 256, 268, 237, 280, 277, 285, 289, 257, 271, 224, 256, 236, 235] payload_rows_per_dst=[5402, 5426, 5448, 5536, 5394, 5472, 5486, 5576] dup_rows_per_dst=[2598, 2668, 2702, 2754, 2676, 2736, 2792, 2870] combine_rows_per_src=[5402, 5426, 5448, 5536, 5394, 5472, 5486, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38342 dup_rows=21682 dup_rows_remote=18956 dedup_ratio=0.330841 router_tokens_per_expert=[270, 234, 259, 268, 265, 263, 228, 244, 269, 237, 270, 258, 244, 253, 263, 256, 254, 261, 236, 256, 254, 271, 249, 262, 242, 232, 241, 247, 242, 269, 277, 241, 256, 257, 266, 248, 259, 272, 265, 243, 263, 215, 227, 274, 298, 271, 249, 256, 265, 251, 252, 271, 233, 247, 262, 269, 236, 221, 225, 255, 269, 244, 258, 254, 273, 240, 245, 270, 287, 260, 272, 271, 253, 279, 251, 246, 255, 233, 246, 230, 261, 284, 265, 236, 273, 269, 249, 293, 251, 263, 267, 246, 266, 260, 256, 246, 227, 243, 226, 297, 244, 212, 281, 267, 257, 289, 258, 275, 248, 256, 251, 261, 267, 272, 247, 249, 243, 262, 265, 263, 275, 288, 237, 261, 261, 234, 265, 245] payload_rows_per_dst=[5476, 5426, 5512, 5444, 5442, 5570, 5536, 5448] dup_rows_per_dst=[2686, 2642, 2726, 2580, 2780, 2800, 2648, 2820] combine_rows_per_src=[5476, 5426, 5512, 5444, 5442, 5570, 5536, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229280 combine_payload_bytes=156868608 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157097888 ingress_bytes=213491712 link_bytes=213491712 token_bytes=313966496 weight_movement_bytes=56623104 total_nvlink_bytes=370589600 weight_to_token_ratio=0.180348 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045572540e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38358 dup_rows=21918 dup_rows_remote=19374 dedup_ratio=0.334442 router_tokens_per_expert=[263, 250, 278, 248, 230, 237, 253, 275, 254, 257, 256, 242, 252, 252, 270, 269, 249, 263, 267, 276, 244, 219, 237, 279, 257, 246, 271, 266, 256, 240, 252, 255, 246, 261, 254, 260, 244, 268, 294, 272, 259, 244, 242, 265, 245, 257, 244, 290, 257, 229, 258, 251, 244, 245, 253, 263, 266, 263, 275, 256, 238, 237, 277, 234, 259, 251, 247, 270, 286, 250, 234, 232, 261, 246, 277, 266, 261, 251, 284, 260, 288, 296, 261, 257, 245, 263, 247, 249, 260, 249, 289, 263, 240, 288, 268, 284, 257, 247, 225, 234, 229, 262, 227, 251, 255, 261, 228, 246, 248, 233, 251, 248, 261, 236, 238, 273, 281, 269, 267, 225, 252, 243, 246, 253, 284, 255, 289, 258] payload_rows_per_dst=[5466, 5416, 5478, 5476, 5508, 5556, 5260, 5458] dup_rows_per_dst=[2706, 2738, 2812, 2616, 2762, 2938, 2544, 2802] combine_rows_per_src=[5466, 5416, 5478, 5476, 5508, 5556, 5260, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38242 dup_rows=21756 dup_rows_remote=18884 dedup_ratio=0.331970 router_tokens_per_expert=[249, 266, 266, 275, 254, 245, 251, 237, 264, 248, 267, 233, 244, 271, 258, 245, 258, 267, 243, 249, 256, 255, 272, 247, 263, 252, 266, 250, 236, 265, 238, 238, 222, 272, 227, 249, 250, 271, 244, 240, 253, 254, 284, 269, 263, 247, 253, 262, 267, 263, 282, 252, 264, 285, 250, 260, 251, 263, 240, 274, 258, 278, 249, 269, 235, 246, 255, 243, 263, 278, 273, 270, 242, 249, 247, 258, 239, 264, 248, 247, 261, 262, 269, 236, 262, 260, 250, 228, 254, 272, 294, 271, 245, 265, 253, 258, 268, 260, 262, 264, 224, 241, 259, 261, 240, 245, 274, 232, 279, 252, 231, 281, 261, 268, 265, 283, 251, 249, 255, 265, 233, 253, 250, 254, 245, 274, 254, 245] payload_rows_per_dst=[5452, 5422, 5456, 5538, 5456, 5482, 5512, 5462] dup_rows_per_dst=[2694, 2688, 2664, 2872, 2658, 2798, 2634, 2748] combine_rows_per_src=[5452, 5422, 5456, 5538, 5456, 5482, 5512, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38130 dup_rows=21938 dup_rows_remote=19276 dedup_ratio=0.334747 router_tokens_per_expert=[262, 277, 231, 254, 268, 246, 233, 250, 244, 254, 251, 284, 275, 247, 261, 271, 249, 249, 238, 264, 267, 260, 278, 244, 251, 267, 243, 241, 260, 259, 239, 251, 262, 267, 264, 243, 268, 237, 273, 252, 232, 262, 234, 260, 224, 277, 265, 260, 254, 237, 271, 283, 252, 259, 281, 244, 256, 260, 260, 256, 271, 260, 256, 252, 253, 251, 232, 265, 232, 245, 255, 264, 238, 285, 270, 260, 242, 248, 251, 244, 248, 254, 259, 252, 263, 251, 252, 261, 243, 253, 261, 281, 230, 271, 269, 270, 287, 228, 279, 285, 243, 242, 267, 260, 301, 258, 259, 252, 247, 240, 279, 223, 268, 263, 252, 230, 233, 251, 250, 263, 259, 256, 255, 247, 255, 276, 259, 248] payload_rows_per_dst=[5488, 5372, 5404, 5424, 5500, 5452, 5490, 5468] dup_rows_per_dst=[2728, 2748, 2756, 2710, 2740, 2784, 2810, 2662] combine_rows_per_src=[5488, 5372, 5404, 5424, 5500, 5452, 5490, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229552 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204656 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179760 weight_movement_bytes=18874368 total_nvlink_bytes=333054128 weight_to_token_ratio=0.060075 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229768 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163912 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098056 weight_movement_bytes=18874368 total_nvlink_bytes=332972424 weight_to_token_ratio=0.060091 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229192 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157278024 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314326856 weight_movement_bytes=18874368 total_nvlink_bytes=333201224 weight_to_token_ratio=0.060047 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157114368 dispatch_meta_bytes=230928 combine_payload_bytes=157114368 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157345296 ingress_bytes=175988736 link_bytes=175988736 token_bytes=314459664 weight_movement_bytes=18874368 total_nvlink_bytes=333334032 weight_to_token_ratio=0.060022 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686237476e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228504 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156867736 ingress_bytes=194387968 link_bytes=194387968 token_bytes=313506968 weight_movement_bytes=37748736 total_nvlink_bytes=351255704 weight_to_token_ratio=0.120408 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862529865e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229624 combine_payload_bytes=156180480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156410104 ingress_bytes=212803584 link_bytes=212803584 token_bytes=312590584 weight_movement_bytes=56623104 total_nvlink_bytes=369213688 weight_to_token_ratio=0.181141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038979236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38114 dup_rows=21598 dup_rows_remote=18652 dedup_ratio=0.329559 router_tokens_per_expert=[263, 267, 257, 247, 243, 255, 279, 246, 277, 259, 251, 259, 282, 260, 253, 235, 236, 248, 259, 243, 243, 260, 232, 239, 262, 246, 243, 260, 284, 261, 265, 265, 255, 284, 243, 261, 238, 252, 248, 268, 274, 240, 250, 258, 231, 270, 253, 236, 258, 247, 254, 249, 272, 249, 247, 251, 239, 251, 269, 256, 287, 272, 233, 215, 234, 250, 267, 285, 249, 274, 244, 270, 240, 271, 245, 267, 251, 268, 269, 282, 270, 232, 241, 260, 248, 245, 264, 256, 254, 223, 268, 260, 282, 238, 284, 287, 292, 265, 261, 248, 243, 257, 260, 237, 261, 253, 261, 239, 269, 247, 251, 247, 237, 282, 256, 243, 241, 261, 266, 275, 258, 252, 267, 275, 245, 268, 227, 257] payload_rows_per_dst=[5824, 5486, 5398, 5368, 5532, 5424, 5394, 5512] dup_rows_per_dst=[2946, 2748, 2558, 2490, 2864, 2656, 2628, 2708] combine_rows_per_src=[5824, 5486, 5398, 5368, 5532, 5424, 5394, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38278 dup_rows=21754 dup_rows_remote=18854 dedup_ratio=0.331940 router_tokens_per_expert=[269, 261, 242, 250, 236, 245, 228, 268, 249, 270, 261, 266, 248, 237, 232, 262, 265, 252, 263, 264, 258, 238, 252, 271, 261, 233, 255, 245, 220, 248, 247, 251, 262, 229, 259, 269, 258, 279, 255, 243, 236, 247, 255, 251, 275, 248, 264, 257, 238, 233, 265, 257, 258, 272, 240, 260, 261, 249, 212, 227, 229, 270, 267, 259, 282, 263, 281, 238, 252, 265, 269, 268, 255, 265, 263, 275, 254, 264, 265, 243, 291, 258, 261, 246, 231, 264, 280, 271, 279, 250, 227, 274, 282, 274, 257, 257, 253, 277, 261, 258, 247, 246, 246, 256, 257, 260, 266, 257, 275, 241, 263, 244, 260, 239, 251, 269, 254, 242, 252, 243, 255, 239, 256, 272, 274, 263, 285, 272] payload_rows_per_dst=[5492, 5410, 5472, 5354, 5504, 5580, 5474, 5496] dup_rows_per_dst=[2556, 2636, 2702, 2640, 2900, 2824, 2740, 2756] combine_rows_per_src=[5492, 5410, 5472, 5354, 5504, 5580, 5474, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=227064 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156342008 ingress_bytes=174989312 link_bytes=174989312 token_bytes=312456952 weight_movement_bytes=18874368 total_nvlink_bytes=331331320 weight_to_token_ratio=0.060406 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676661487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38454 dup_rows=21616 dup_rows_remote=18928 dedup_ratio=0.329834 router_tokens_per_expert=[273, 259, 262, 251, 265, 256, 236, 280, 258, 253, 278, 254, 278, 273, 203, 281, 241, 242, 261, 260, 264, 273, 274, 264, 253, 272, 263, 254, 246, 220, 273, 251, 242, 253, 266, 238, 256, 267, 270, 278, 225, 253, 267, 244, 253, 240, 228, 261, 263, 270, 277, 250, 234, 274, 255, 251, 258, 262, 255, 261, 231, 267, 245, 265, 232, 255, 292, 275, 249, 287, 251, 240, 260, 264, 275, 265, 237, 234, 240, 238, 249, 253, 291, 242, 243, 261, 262, 226, 227, 247, 288, 247, 269, 255, 252, 265, 249, 261, 272, 249, 247, 250, 233, 248, 253, 242, 249, 283, 243, 267, 221, 271, 246, 275, 232, 239, 270, 265, 279, 252, 254, 261, 258, 273, 281, 254, 243, 247] payload_rows_per_dst=[5538, 5538, 5390, 5504, 5498, 5466, 5444, 5542] dup_rows_per_dst=[2782, 2684, 2692, 2732, 2690, 2688, 2632, 2716] combine_rows_per_src=[5538, 5538, 5390, 5504, 5498, 5466, 5444, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38346 dup_rows=21836 dup_rows_remote=19308 dedup_ratio=0.333191 router_tokens_per_expert=[269, 254, 253, 228, 273, 249, 251, 267, 265, 272, 276, 270, 279, 275, 275, 232, 260, 236, 254, 254, 263, 259, 258, 238, 262, 220, 227, 270, 245, 235, 238, 222, 256, 244, 287, 276, 260, 243, 262, 260, 273, 220, 249, 270, 224, 283, 236, 270, 230, 254, 268, 263, 277, 273, 249, 264, 254, 248, 247, 280, 257, 233, 247, 291, 233, 245, 259, 240, 249, 255, 265, 260, 245, 235, 252, 253, 274, 247, 258, 275, 233, 239, 293, 240, 260, 264, 228, 237, 267, 270, 249, 270, 245, 245, 261, 274, 274, 243, 260, 262, 269, 259, 275, 253, 230, 281, 275, 277, 255, 273, 248, 252, 276, 271, 261, 242, 252, 236, 239, 270, 258, 276, 236, 249, 241, 247, 276, 255] payload_rows_per_dst=[5508, 5354, 5448, 5540, 5390, 5454, 5498, 5508] dup_rows_per_dst=[2868, 2528, 2778, 2730, 2700, 2696, 2874, 2662] combine_rows_per_src=[5508, 5354, 5448, 5540, 5390, 5454, 5498, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38350 dup_rows=21782 dup_rows_remote=19066 dedup_ratio=0.332367 router_tokens_per_expert=[267, 254, 248, 248, 249, 252, 259, 264, 262, 276, 238, 262, 264, 266, 274, 248, 262, 237, 256, 259, 239, 252, 262, 257, 251, 252, 229, 297, 249, 270, 270, 284, 237, 249, 257, 242, 247, 265, 239, 274, 272, 248, 246, 256, 269, 265, 257, 237, 261, 277, 262, 231, 259, 271, 257, 252, 234, 250, 251, 245, 251, 257, 265, 241, 293, 247, 272, 267, 274, 267, 253, 253, 241, 253, 254, 275, 246, 273, 238, 233, 258, 248, 235, 269, 274, 252, 259, 220, 252, 274, 262, 246, 242, 273, 253, 252, 266, 226, 251, 270, 289, 257, 289, 274, 256, 212, 253, 265, 248, 253, 268, 244, 262, 208, 299, 259, 238, 242, 259, 255, 257, 239, 249, 251, 249, 255, 279, 257] payload_rows_per_dst=[5462, 5498, 5404, 5434, 5496, 5440, 5498, 5522] dup_rows_per_dst=[2800, 2754, 2716, 2694, 2782, 2698, 2744, 2594] combine_rows_per_src=[5462, 5498, 5404, 5434, 5496, 5440, 5498, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=228528 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157015216 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313801904 weight_movement_bytes=18874368 total_nvlink_bytes=332676272 weight_to_token_ratio=0.060147 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38156 dup_rows=21812 dup_rows_remote=19014 dedup_ratio=0.332825 router_tokens_per_expert=[225, 253, 263, 249, 289, 240, 256, 269, 284, 241, 233, 248, 250, 247, 249, 271, 250, 276, 264, 293, 262, 288, 249, 261, 255, 252, 259, 264, 263, 255, 247, 262, 254, 230, 226, 259, 219, 250, 248, 283, 243, 269, 252, 260, 243, 250, 250, 271, 281, 219, 249, 237, 238, 245, 222, 241, 237, 239, 256, 265, 304, 259, 283, 259, 268, 263, 247, 230, 249, 220, 276, 241, 264, 269, 264, 268, 266, 254, 246, 263, 239, 274, 266, 255, 239, 261, 250, 247, 259, 255, 260, 243, 270, 280, 255, 232, 283, 289, 263, 273, 248, 277, 239, 255, 269, 234, 258, 266, 263, 253, 258, 255, 267, 264, 271, 257, 249, 242, 262, 273, 256, 251, 254, 247, 239, 264, 279, 229] payload_rows_per_dst=[5376, 5562, 5394, 5356, 5494, 5532, 5568, 5442] dup_rows_per_dst=[2648, 2838, 2620, 2712, 2682, 2748, 2798, 2766] combine_rows_per_src=[5376, 5562, 5394, 5356, 5494, 5532, 5568, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38314 dup_rows=21768 dup_rows_remote=19074 dedup_ratio=0.332153 router_tokens_per_expert=[269, 234, 266, 268, 252, 244, 236, 252, 248, 251, 255, 276, 275, 266, 275, 270, 253, 232, 231, 252, 275, 290, 275, 264, 261, 249, 242, 268, 267, 248, 271, 260, 257, 254, 258, 252, 231, 291, 256, 259, 247, 257, 247, 264, 227, 255, 242, 284, 239, 246, 252, 241, 290, 255, 253, 255, 254, 243, 265, 258, 249, 253, 266, 255, 234, 260, 268, 240, 272, 265, 262, 237, 246, 261, 268, 263, 268, 254, 217, 284, 271, 250, 243, 262, 262, 247, 236, 251, 249, 257, 246, 241, 213, 273, 231, 271, 239, 253, 245, 248, 253, 297, 247, 242, 277, 283, 235, 274, 240, 260, 253, 236, 287, 252, 267, 258, 236, 277, 248, 280, 257, 258, 271, 251, 226, 278, 253, 255] payload_rows_per_dst=[5492, 5496, 5446, 5454, 5498, 5448, 5462, 5472] dup_rows_per_dst=[2782, 2780, 2716, 2694, 2700, 2558, 2702, 2836] combine_rows_per_src=[5492, 5496, 5446, 5454, 5498, 5448, 5462, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38430 dup_rows=21598 dup_rows_remote=18780 dedup_ratio=0.329559 router_tokens_per_expert=[264, 219, 273, 270, 249, 262, 256, 251, 259, 231, 238, 278, 277, 253, 239, 260, 255, 283, 252, 215, 240, 254, 273, 252, 266, 235, 229, 226, 229, 264, 256, 234, 267, 220, 267, 280, 273, 268, 284, 255, 262, 278, 248, 260, 280, 265, 271, 217, 229, 238, 257, 270, 251, 254, 252, 269, 249, 277, 248, 250, 265, 246, 268, 271, 238, 252, 242, 264, 261, 266, 280, 217, 262, 266, 244, 271, 256, 255, 251, 252, 259, 250, 261, 262, 240, 237, 242, 275, 251, 263, 224, 273, 259, 258, 250, 275, 253, 256, 251, 280, 260, 264, 266, 260, 253, 235, 261, 263, 237, 261, 258, 261, 242, 273, 253, 284, 248, 261, 248, 269, 268, 280, 259, 251, 287, 244, 269, 226] payload_rows_per_dst=[5288, 5332, 5550, 5466, 5520, 5726, 5548, 5508] dup_rows_per_dst=[2316, 2548, 2848, 2618, 2774, 2986, 2690, 2818] combine_rows_per_src=[5288, 5332, 5550, 5466, 5520, 5726, 5548, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=229528 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157737112 ingress_bytes=233005056 link_bytes=233005056 token_bytes=315244696 weight_movement_bytes=75497472 total_nvlink_bytes=390742168 weight_to_token_ratio=0.239488 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232539801e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229664 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311264 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392864 weight_movement_bytes=18874368 total_nvlink_bytes=333267232 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=230616 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157295832 ingress_bytes=213688320 link_bytes=213688320 token_bytes=314361048 weight_movement_bytes=56623104 total_nvlink_bytes=370984152 weight_to_token_ratio=0.180121 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047456341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228680 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156515656 ingress_bytes=194035712 link_bytes=194035712 token_bytes=312802632 weight_movement_bytes=37748736 total_nvlink_bytes=350551368 weight_to_token_ratio=0.120679 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859154721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229552 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157163696 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097840 weight_movement_bytes=37748736 total_nvlink_bytes=351846576 weight_to_token_ratio=0.120181 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=228840 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157638120 ingress_bytes=195158016 link_bytes=195158016 token_bytes=315047400 weight_movement_bytes=37748736 total_nvlink_bytes=352796136 weight_to_token_ratio=0.119819 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869908086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38340 dup_rows=21752 dup_rows_remote=19022 dedup_ratio=0.331909 router_tokens_per_expert=[268, 244, 243, 262, 235, 259, 249, 247, 263, 244, 256, 244, 258, 288, 253, 267, 261, 248, 259, 257, 261, 222, 267, 246, 248, 265, 275, 263, 242, 239, 239, 269, 270, 243, 260, 257, 263, 223, 249, 274, 247, 229, 269, 277, 293, 270, 266, 275, 246, 250, 256, 248, 289, 267, 258, 261, 256, 261, 253, 215, 261, 249, 240, 231, 256, 253, 253, 243, 262, 256, 283, 253, 270, 260, 237, 230, 256, 283, 276, 250, 255, 248, 261, 278, 252, 256, 248, 233, 259, 292, 227, 257, 259, 238, 259, 234, 281, 260, 277, 251, 250, 251, 263, 258, 242, 256, 234, 244, 271, 264, 262, 266, 254, 285, 267, 246, 246, 249, 248, 241, 257, 264, 255, 259, 259, 275, 265, 244] payload_rows_per_dst=[5444, 5622, 5588, 5514, 5504, 5234, 5428, 5450] dup_rows_per_dst=[2730, 3004, 2742, 2576, 2738, 2374, 2810, 2778] combine_rows_per_src=[5444, 5622, 5588, 5514, 5504, 5234, 5428, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38144 dup_rows=21914 dup_rows_remote=19228 dedup_ratio=0.334381 router_tokens_per_expert=[252, 289, 260, 262, 277, 252, 246, 273, 261, 266, 258, 278, 255, 270, 245, 267, 261, 270, 274, 274, 236, 249, 265, 273, 251, 223, 249, 244, 253, 258, 237, 262, 274, 256, 251, 262, 249, 247, 256, 239, 267, 261, 227, 252, 261, 237, 275, 261, 256, 244, 256, 254, 254, 252, 242, 294, 256, 254, 238, 251, 250, 256, 261, 253, 267, 238, 260, 246, 242, 262, 247, 274, 282, 275, 249, 257, 231, 259, 243, 250, 232, 284, 263, 265, 259, 248, 259, 255, 252, 246, 239, 277, 273, 264, 245, 236, 249, 253, 250, 258, 256, 263, 240, 265, 272, 249, 234, 231, 273, 247, 236, 245, 225, 243, 272, 268, 257, 271, 243, 238, 248, 259, 259, 277, 283, 250, 255, 284] payload_rows_per_dst=[5502, 5418, 5438, 5448, 5478, 5436, 5416, 5486] dup_rows_per_dst=[2920, 2740, 2712, 2694, 2686, 2758, 2626, 2778] combine_rows_per_src=[5502, 5418, 5438, 5448, 5478, 5436, 5416, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229448 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157270088 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310728 weight_movement_bytes=37748736 total_nvlink_bytes=352059464 weight_to_token_ratio=0.120100 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38336 dup_rows=21768 dup_rows_remote=19190 dedup_ratio=0.332153 router_tokens_per_expert=[242, 256, 242, 253, 274, 266, 263, 261, 269, 275, 241, 234, 261, 233, 249, 303, 279, 260, 227, 278, 256, 254, 248, 250, 259, 268, 286, 241, 257, 253, 235, 250, 220, 242, 250, 293, 264, 253, 257, 236, 293, 244, 246, 231, 284, 257, 260, 264, 272, 247, 259, 238, 271, 259, 269, 271, 263, 280, 279, 263, 261, 289, 295, 249, 247, 271, 259, 227, 257, 236, 268, 260, 258, 238, 283, 239, 279, 257, 272, 238, 260, 236, 252, 244, 266, 250, 263, 253, 243, 263, 266, 245, 219, 236, 285, 224, 246, 249, 260, 265, 281, 238, 258, 283, 229, 232, 245, 247, 263, 266, 231, 238, 253, 275, 239, 254, 242, 260, 264, 256, 261, 237, 249, 264, 257, 252, 236, 262] payload_rows_per_dst=[5520, 5444, 5456, 5598, 5436, 5432, 5444, 5438] dup_rows_per_dst=[2724, 2758, 2732, 2932, 2742, 2578, 2618, 2684] combine_rows_per_src=[5520, 5444, 5456, 5598, 5436, 5432, 5444, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38432 dup_rows=21698 dup_rows_remote=18950 dedup_ratio=0.331085 router_tokens_per_expert=[244, 243, 258, 257, 242, 304, 261, 273, 238, 254, 275, 285, 255, 257, 267, 215, 286, 259, 266, 272, 278, 253, 255, 247, 221, 238, 233, 244, 255, 262, 233, 229, 262, 247, 240, 266, 264, 259, 236, 254, 268, 260, 267, 237, 268, 243, 248, 258, 260, 286, 267, 242, 257, 259, 264, 249, 232, 282, 246, 230, 257, 243, 268, 272, 252, 257, 283, 269, 280, 264, 256, 265, 285, 240, 272, 265, 247, 229, 280, 239, 227, 241, 251, 269, 240, 254, 265, 250, 251, 272, 258, 253, 275, 270, 251, 246, 235, 240, 259, 241, 232, 252, 250, 280, 279, 262, 262, 245, 245, 240, 261, 244, 268, 249, 274, 244, 257, 255, 238, 252, 257, 256, 236, 267, 268, 256, 264, 294] payload_rows_per_dst=[5562, 5406, 5406, 5480, 5556, 5496, 5384, 5548] dup_rows_per_dst=[2694, 2656, 2748, 2748, 2810, 2650, 2670, 2722] combine_rows_per_src=[5562, 5406, 5406, 5480, 5556, 5496, 5384, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156237824 dispatch_meta_bytes=229488 combine_payload_bytes=156237824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156467312 ingress_bytes=175112192 link_bytes=175112192 token_bytes=312705136 weight_movement_bytes=18874368 total_nvlink_bytes=331579504 weight_to_token_ratio=0.060358 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677838863e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38262 dup_rows=21744 dup_rows_remote=19096 dedup_ratio=0.331787 router_tokens_per_expert=[251, 242, 244, 264, 280, 288, 242, 248, 277, 261, 241, 238, 285, 272, 247, 257, 270, 264, 246, 251, 250, 239, 223, 262, 266, 302, 247, 254, 235, 263, 256, 261, 278, 263, 265, 244, 264, 258, 243, 270, 226, 257, 248, 248, 245, 246, 233, 274, 267, 268, 260, 264, 260, 247, 269, 248, 274, 278, 245, 290, 231, 249, 248, 263, 244, 246, 246, 272, 248, 264, 238, 243, 261, 239, 245, 250, 272, 268, 229, 282, 228, 270, 264, 260, 245, 275, 255, 261, 241, 259, 229, 246, 231, 262, 260, 271, 267, 228, 246, 276, 244, 261, 273, 266, 255, 252, 268, 259, 266, 271, 261, 263, 276, 235, 255, 249, 252, 260, 237, 263, 272, 250, 239, 247, 249, 280, 259, 236] payload_rows_per_dst=[5488, 5530, 5440, 5456, 5418, 5466, 5526, 5468] dup_rows_per_dst=[2786, 2648, 2684, 2866, 2676, 2648, 2786, 2650] combine_rows_per_src=[5488, 5530, 5440, 5456, 5418, 5466, 5526, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38138 dup_rows=21880 dup_rows_remote=19116 dedup_ratio=0.333862 router_tokens_per_expert=[252, 272, 264, 239, 262, 239, 292, 280, 265, 270, 261, 277, 241, 260, 260, 246, 256, 273, 283, 263, 275, 255, 248, 235, 239, 254, 233, 264, 266, 255, 256, 247, 232, 258, 273, 232, 273, 267, 248, 251, 260, 264, 278, 272, 248, 247, 266, 246, 281, 246, 266, 253, 270, 251, 273, 257, 240, 245, 279, 254, 232, 226, 264, 252, 238, 258, 249, 268, 230, 260, 260, 226, 263, 271, 247, 250, 234, 256, 248, 254, 263, 275, 278, 259, 239, 267, 259, 267, 252, 228, 242, 257, 279, 276, 269, 273, 244, 262, 264, 238, 266, 229, 261, 288, 259, 247, 251, 281, 252, 268, 267, 264, 245, 242, 223, 272, 229, 265, 239, 233, 234, 277, 228, 235, 265, 239, 263, 257] payload_rows_per_dst=[5510, 5510, 5436, 5436, 5406, 5506, 5518, 5334] dup_rows_per_dst=[2850, 2694, 2794, 2742, 2618, 2860, 2764, 2558] combine_rows_per_src=[5510, 5510, 5436, 5436, 5406, 5506, 5518, 5334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38310 dup_rows=21690 dup_rows_remote=18836 dedup_ratio=0.330963 router_tokens_per_expert=[261, 259, 262, 241, 264, 287, 251, 255, 249, 261, 258, 253, 244, 282, 251, 239, 234, 258, 250, 269, 266, 250, 282, 223, 290, 305, 261, 279, 242, 217, 225, 264, 215, 297, 225, 245, 261, 277, 242, 229, 241, 246, 250, 263, 248, 262, 210, 252, 267, 257, 261, 260, 260, 279, 274, 286, 265, 222, 251, 252, 292, 271, 247, 251, 259, 258, 267, 243, 286, 247, 265, 255, 263, 280, 262, 271, 267, 259, 243, 256, 247, 280, 221, 270, 246, 241, 267, 280, 259, 232, 253, 283, 247, 235, 265, 234, 282, 227, 246, 232, 275, 254, 276, 248, 260, 267, 256, 273, 246, 230, 260, 249, 260, 249, 258, 252, 278, 249, 268, 272, 230, 234, 259, 275, 254, 227, 248, 243] payload_rows_per_dst=[5502, 5496, 5444, 5536, 5472, 5448, 5478, 5470] dup_rows_per_dst=[2732, 2734, 2482, 2854, 2890, 2672, 2684, 2642] combine_rows_per_src=[5502, 5496, 5444, 5536, 5472, 5448, 5478, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38148 dup_rows=21856 dup_rows_remote=18944 dedup_ratio=0.333496 router_tokens_per_expert=[264, 266, 265, 287, 245, 261, 220, 261, 269, 253, 270, 237, 248, 265, 276, 229, 235, 240, 235, 268, 260, 247, 267, 249, 241, 267, 231, 243, 256, 240, 268, 247, 249, 260, 250, 233, 260, 261, 242, 275, 262, 257, 306, 252, 242, 261, 254, 249, 254, 255, 259, 283, 272, 258, 260, 257, 265, 256, 237, 249, 249, 251, 262, 276, 261, 266, 246, 248, 243, 256, 284, 253, 211, 222, 268, 231, 241, 287, 253, 235, 250, 254, 262, 263, 242, 260, 264, 260, 245, 290, 243, 272, 262, 234, 260, 246, 272, 279, 255, 251, 227, 239, 278, 246, 258, 295, 271, 264, 280, 265, 280, 264, 266, 265, 250, 285, 252, 254, 233, 271, 258, 257, 224, 261, 264, 251, 242, 233] payload_rows_per_dst=[5240, 5422, 5538, 5376, 5492, 5598, 5482, 5532] dup_rows_per_dst=[2606, 2566, 2688, 2694, 2800, 2912, 2678, 2912] combine_rows_per_src=[5240, 5422, 5538, 5376, 5492, 5598, 5482, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=230104 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157254360 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314278616 weight_movement_bytes=18874368 total_nvlink_bytes=333152984 weight_to_token_ratio=0.060056 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=229528 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157647000 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315064472 weight_movement_bytes=18874368 total_nvlink_bytes=333938840 weight_to_token_ratio=0.059906 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229432 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950584 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671736 weight_movement_bytes=18874368 total_nvlink_bytes=332546104 weight_to_token_ratio=0.060172 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=228584 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157146344 ingress_bytes=213540864 link_bytes=213540864 token_bytes=314064104 weight_movement_bytes=56623104 total_nvlink_bytes=370687208 weight_to_token_ratio=0.180292 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046043490e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=229016 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156442264 ingress_bytes=212836352 link_bytes=212836352 token_bytes=312655512 weight_movement_bytes=56623104 total_nvlink_bytes=369278616 weight_to_token_ratio=0.181104 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039293203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228368 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156482576 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312736784 weight_movement_bytes=18874368 total_nvlink_bytes=331611152 weight_to_token_ratio=0.060352 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38346 dup_rows=21802 dup_rows_remote=19198 dedup_ratio=0.332672 router_tokens_per_expert=[254, 259, 248, 243, 250, 275, 245, 264, 258, 259, 236, 233, 252, 227, 255, 238, 268, 268, 249, 244, 219, 259, 263, 265, 281, 237, 266, 247, 277, 235, 255, 254, 255, 251, 246, 281, 276, 232, 254, 246, 243, 261, 264, 241, 248, 274, 280, 256, 259, 259, 254, 249, 244, 263, 240, 257, 235, 263, 243, 281, 260, 269, 253, 233, 231, 269, 269, 237, 311, 244, 253, 260, 251, 269, 319, 253, 256, 261, 238, 268, 248, 260, 249, 239, 269, 248, 220, 239, 261, 257, 263, 237, 290, 272, 239, 266, 269, 260, 287, 259, 240, 230, 266, 277, 235, 249, 254, 281, 251, 257, 254, 257, 231, 309, 274, 273, 264, 258, 260, 254, 254, 252, 263, 258, 245, 261, 242, 245] payload_rows_per_dst=[5388, 5622, 5546, 5478, 5476, 5468, 5472, 5284] dup_rows_per_dst=[2604, 3014, 2804, 2664, 2750, 2646, 2780, 2540] combine_rows_per_src=[5388, 5622, 5546, 5478, 5476, 5468, 5472, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38294 dup_rows=21828 dup_rows_remote=19216 dedup_ratio=0.333069 router_tokens_per_expert=[242, 247, 231, 284, 298, 271, 270, 282, 255, 270, 244, 256, 263, 278, 240, 249, 295, 256, 245, 258, 262, 243, 286, 263, 260, 241, 224, 265, 235, 285, 263, 243, 264, 256, 248, 263, 276, 222, 272, 234, 269, 234, 257, 239, 274, 253, 241, 259, 260, 243, 262, 262, 265, 233, 254, 252, 264, 257, 287, 280, 262, 253, 261, 235, 246, 249, 243, 248, 254, 273, 246, 247, 250, 246, 231, 258, 271, 256, 238, 257, 263, 258, 238, 262, 246, 244, 235, 262, 275, 242, 249, 258, 247, 240, 252, 257, 218, 227, 272, 276, 272, 239, 261, 273, 242, 249, 268, 243, 288, 248, 275, 272, 269, 223, 237, 244, 263, 258, 267, 266, 256, 265, 239, 261, 239, 283, 286, 253] payload_rows_per_dst=[5522, 5486, 5478, 5496, 5414, 5372, 5496, 5444] dup_rows_per_dst=[2838, 2762, 2644, 2764, 2612, 2684, 2750, 2774] combine_rows_per_src=[5522, 5486, 5478, 5496, 5414, 5372, 5496, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=230176 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=0 egress_bytes=157295392 ingress_bytes=157065216 link_bytes=157295392 token_bytes=314360608 weight_movement_bytes=0 total_nvlink_bytes=314360608 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.507127052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38416 dup_rows=21640 dup_rows_remote=18906 dedup_ratio=0.330200 router_tokens_per_expert=[258, 271, 271, 253, 248, 265, 221, 254, 279, 242, 267, 255, 238, 265, 261, 267, 244, 238, 254, 256, 247, 270, 250, 246, 262, 249, 232, 241, 276, 236, 226, 271, 263, 251, 234, 274, 258, 229, 260, 252, 261, 265, 247, 257, 256, 259, 298, 258, 257, 275, 239, 264, 228, 258, 246, 266, 265, 277, 265, 256, 254, 247, 253, 238, 247, 241, 252, 265, 293, 250, 258, 275, 257, 271, 253, 270, 233, 256, 263, 233, 263, 226, 256, 259, 227, 250, 252, 243, 244, 271, 260, 263, 281, 268, 292, 252, 238, 246, 252, 251, 261, 263, 244, 235, 262, 268, 247, 288, 246, 295, 253, 278, 259, 247, 229, 275, 257, 254, 256, 272, 250, 266, 260, 257, 262, 260, 242, 248] payload_rows_per_dst=[5506, 5338, 5576, 5540, 5424, 5480, 5532, 5500] dup_rows_per_dst=[2724, 2658, 2668, 2636, 2810, 2734, 2722, 2688] combine_rows_per_src=[5506, 5338, 5576, 5540, 5424, 5480, 5532, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38262 dup_rows=21846 dup_rows_remote=19148 dedup_ratio=0.333344 router_tokens_per_expert=[254, 256, 299, 240, 273, 279, 264, 247, 254, 249, 256, 286, 258, 246, 222, 280, 237, 249, 261, 262, 257, 233, 267, 247, 290, 251, 256, 232, 255, 248, 245, 261, 253, 229, 250, 251, 260, 259, 263, 260, 240, 257, 256, 260, 251, 264, 236, 274, 239, 260, 249, 274, 265, 260, 276, 248, 229, 230, 257, 264, 260, 251, 267, 243, 235, 277, 232, 255, 264, 237, 274, 248, 254, 284, 251, 276, 253, 243, 280, 265, 232, 257, 250, 265, 260, 230, 260, 269, 253, 263, 258, 250, 239, 239, 283, 284, 236, 236, 268, 269, 271, 228, 270, 253, 246, 251, 227, 247, 224, 249, 263, 269, 259, 266, 261, 255, 262, 268, 295, 260, 239, 247, 241, 259, 269, 282, 266, 263] payload_rows_per_dst=[5490, 5380, 5428, 5426, 5570, 5458, 5420, 5518] dup_rows_per_dst=[2836, 2722, 2698, 2718, 2686, 2726, 2594, 2866] combine_rows_per_src=[5490, 5380, 5428, 5426, 5570, 5458, 5420, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38260 dup_rows=21740 dup_rows_remote=19052 dedup_ratio=0.331726 router_tokens_per_expert=[234, 259, 278, 274, 269, 266, 264, 236, 244, 236, 256, 226, 251, 225, 245, 251, 268, 258, 264, 262, 236, 245, 273, 261, 226, 259, 284, 256, 251, 249, 252, 268, 251, 279, 258, 246, 261, 252, 259, 285, 244, 289, 226, 243, 231, 225, 286, 239, 239, 276, 273, 257, 260, 262, 268, 251, 274, 225, 261, 294, 260, 252, 249, 280, 274, 261, 248, 254, 231, 244, 251, 275, 240, 270, 248, 237, 264, 266, 251, 283, 274, 247, 243, 259, 269, 227, 283, 271, 251, 256, 267, 256, 247, 264, 246, 272, 268, 250, 256, 239, 266, 219, 228, 253, 258, 281, 239, 273, 264, 253, 262, 267, 281, 232, 250, 267, 243, 277, 264, 257, 268, 261, 245, 270, 227, 270, 240, 230] payload_rows_per_dst=[5392, 5536, 5432, 5558, 5498, 5472, 5432, 5476] dup_rows_per_dst=[2636, 2688, 2716, 2804, 2696, 2792, 2720, 2688] combine_rows_per_src=[5392, 5536, 5432, 5558, 5498, 5472, 5432, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=230040 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157082264 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934488 weight_movement_bytes=18874368 total_nvlink_bytes=332808856 weight_to_token_ratio=0.060122 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38270 dup_rows=21846 dup_rows_remote=19092 dedup_ratio=0.333344 router_tokens_per_expert=[250, 282, 256, 227, 267, 235, 232, 260, 272, 271, 249, 234, 263, 291, 272, 266, 266, 290, 254, 266, 253, 261, 252, 258, 256, 262, 229, 246, 267, 256, 255, 264, 264, 262, 247, 240, 271, 266, 272, 265, 248, 233, 233, 254, 261, 259, 250, 275, 267, 247, 237, 247, 259, 235, 276, 268, 246, 290, 251, 251, 254, 265, 276, 242, 250, 229, 214, 272, 237, 259, 249, 268, 265, 260, 256, 250, 259, 260, 257, 274, 235, 256, 263, 253, 238, 238, 255, 253, 274, 270, 252, 259, 262, 247, 275, 246, 237, 257, 251, 270, 261, 225, 297, 256, 239, 243, 229, 273, 256, 282, 237, 274, 230, 267, 259, 253, 260, 234, 260, 250, 245, 245, 279, 257, 257, 275, 252, 250] payload_rows_per_dst=[5412, 5478, 5532, 5494, 5452, 5446, 5420, 5456] dup_rows_per_dst=[2842, 2792, 2668, 2728, 2666, 2706, 2754, 2690] combine_rows_per_src=[5412, 5478, 5532, 5494, 5452, 5446, 5420, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38340 dup_rows=21710 dup_rows_remote=19000 dedup_ratio=0.331268 router_tokens_per_expert=[259, 245, 266, 248, 263, 261, 276, 271, 238, 245, 269, 285, 255, 256, 238, 269, 265, 246, 270, 279, 238, 251, 267, 245, 254, 245, 278, 239, 261, 256, 260, 271, 237, 259, 237, 246, 266, 271, 260, 218, 269, 231, 274, 255, 262, 238, 233, 267, 265, 229, 247, 240, 252, 275, 297, 273, 260, 260, 230, 281, 253, 249, 257, 230, 263, 287, 254, 236, 251, 264, 259, 232, 264, 229, 264, 257, 274, 245, 283, 284, 239, 250, 245, 270, 259, 261, 271, 257, 252, 265, 284, 266, 234, 261, 269, 248, 255, 245, 256, 237, 254, 268, 254, 230, 260, 267, 247, 259, 226, 273, 247, 251, 291, 263, 239, 260, 265, 266, 243, 242, 271, 254, 252, 252, 241, 242, 220, 271] payload_rows_per_dst=[5440, 5568, 5418, 5486, 5498, 5510, 5448, 5458] dup_rows_per_dst=[2848, 2682, 2628, 2710, 2794, 2752, 2610, 2686] combine_rows_per_src=[5440, 5568, 5418, 5486, 5498, 5510, 5448, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38400 dup_rows=21724 dup_rows_remote=18972 dedup_ratio=0.331482 router_tokens_per_expert=[253, 257, 258, 276, 232, 248, 271, 264, 264, 244, 263, 250, 251, 274, 260, 270, 239, 227, 247, 270, 245, 269, 248, 258, 249, 228, 257, 272, 246, 248, 261, 262, 238, 269, 267, 229, 254, 286, 258, 263, 266, 258, 245, 243, 275, 276, 275, 259, 267, 245, 243, 268, 256, 243, 295, 253, 241, 267, 224, 263, 254, 261, 268, 241, 268, 264, 250, 223, 240, 254, 255, 250, 245, 264, 259, 252, 250, 256, 240, 260, 240, 237, 267, 251, 239, 268, 247, 250, 271, 277, 264, 274, 251, 263, 252, 289, 246, 226, 246, 279, 255, 247, 276, 250, 261, 274, 255, 270, 245, 263, 283, 243, 252, 247, 252, 261, 234, 251, 269, 234, 253, 264, 258, 257, 230, 273, 260, 273] payload_rows_per_dst=[5328, 5448, 5504, 5498, 5498, 5462, 5662, 5412] dup_rows_per_dst=[2466, 2604, 2818, 2704, 2594, 2818, 2968, 2752] combine_rows_per_src=[5328, 5448, 5504, 5498, 5498, 5462, 5662, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229640 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950792 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671944 weight_movement_bytes=18874368 total_nvlink_bytes=332546312 weight_to_token_ratio=0.060172 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229288 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581224 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933160 weight_movement_bytes=18874368 total_nvlink_bytes=333807528 weight_to_token_ratio=0.059931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229248 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942208 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655168 weight_movement_bytes=18874368 total_nvlink_bytes=332529536 weight_to_token_ratio=0.060176 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229448 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983368 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737288 weight_movement_bytes=18874368 total_nvlink_bytes=332611656 weight_to_token_ratio=0.060160 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229360 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157270000 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310640 weight_movement_bytes=37748736 total_nvlink_bytes=352059376 weight_to_token_ratio=0.120100 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38232 dup_rows=21658 dup_rows_remote=18832 dedup_ratio=0.330475 router_tokens_per_expert=[245, 251, 250, 240, 267, 251, 274, 251, 251, 280, 241, 242, 261, 245, 241, 257, 249, 253, 268, 269, 259, 246, 263, 243, 261, 269, 257, 240, 279, 252, 271, 280, 266, 267, 242, 245, 267, 248, 256, 256, 250, 266, 265, 263, 286, 262, 263, 230, 280, 269, 257, 275, 256, 240, 249, 273, 269, 242, 261, 248, 267, 274, 259, 254, 249, 236, 263, 266, 254, 236, 224, 267, 286, 234, 265, 283, 233, 227, 227, 264, 283, 243, 242, 274, 258, 264, 233, 229, 267, 270, 243, 263, 252, 265, 262, 241, 264, 242, 243, 268, 269, 264, 237, 246, 255, 255, 255, 253, 247, 264, 258, 255, 234, 251, 243, 276, 281, 249, 231, 260, 246, 245, 274, 266, 278, 261, 244, 240] payload_rows_per_dst=[5646, 5716, 5518, 5538, 5450, 5098, 5646, 5266] dup_rows_per_dst=[2826, 3006, 2804, 2818, 2578, 2292, 2888, 2446] combine_rows_per_src=[5646, 5716, 5518, 5538, 5450, 5098, 5646, 5266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229488 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157515888 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802288 weight_movement_bytes=37748736 total_nvlink_bytes=352551024 weight_to_token_ratio=0.119913 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38288 dup_rows=21860 dup_rows_remote=19318 dedup_ratio=0.333557 router_tokens_per_expert=[279, 260, 265, 262, 285, 251, 242, 253, 249, 242, 268, 278, 276, 242, 251, 279, 239, 235, 253, 233, 232, 271, 261, 253, 235, 271, 238, 248, 269, 270, 244, 267, 259, 269, 275, 275, 265, 269, 272, 253, 280, 234, 261, 234, 237, 286, 263, 277, 250, 237, 249, 281, 260, 236, 238, 278, 268, 254, 253, 240, 228, 260, 258, 225, 250, 242, 259, 240, 230, 254, 231, 246, 249, 254, 276, 239, 261, 250, 246, 238, 261, 285, 290, 256, 268, 259, 235, 246, 255, 240, 273, 261, 234, 246, 255, 255, 250, 249, 242, 270, 272, 243, 279, 249, 263, 259, 251, 274, 266, 247, 229, 240, 282, 263, 249, 270, 262, 242, 280, 275, 230, 261, 275, 235, 301, 269, 235, 247] payload_rows_per_dst=[5506, 5408, 5522, 5406, 5388, 5450, 5486, 5510] dup_rows_per_dst=[2858, 2630, 2896, 2624, 2542, 2788, 2680, 2842] combine_rows_per_src=[5506, 5408, 5522, 5406, 5388, 5450, 5486, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228256 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156826528 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313424800 weight_movement_bytes=18874368 total_nvlink_bytes=332299168 weight_to_token_ratio=0.060220 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38334 dup_rows=21772 dup_rows_remote=19030 dedup_ratio=0.332214 router_tokens_per_expert=[248, 263, 239, 267, 257, 237, 234, 243, 250, 255, 252, 234, 273, 244, 274, 263, 256, 235, 249, 254, 278, 272, 249, 248, 269, 253, 245, 238, 271, 287, 248, 277, 250, 258, 274, 249, 273, 252, 254, 250, 262, 234, 261, 250, 234, 269, 262, 254, 255, 242, 259, 288, 269, 271, 250, 275, 262, 254, 247, 225, 264, 235, 248, 263, 229, 255, 238, 261, 275, 255, 261, 256, 248, 254, 242, 249, 255, 247, 268, 274, 260, 272, 240, 290, 249, 268, 267, 253, 264, 253, 238, 238, 258, 256, 253, 257, 258, 236, 260, 235, 243, 305, 263, 264, 234, 224, 283, 277, 278, 253, 248, 253, 273, 240, 242, 283, 252, 223, 276, 281, 277, 265, 258, 259, 238, 235, 258, 256] payload_rows_per_dst=[5414, 5526, 5430, 5530, 5406, 5466, 5456, 5536] dup_rows_per_dst=[2652, 2732, 2742, 2684, 2728, 2766, 2772, 2696] combine_rows_per_src=[5414, 5526, 5430, 5530, 5406, 5466, 5456, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38232 dup_rows=21874 dup_rows_remote=19166 dedup_ratio=0.333771 router_tokens_per_expert=[259, 260, 219, 260, 256, 312, 223, 260, 235, 259, 261, 260, 279, 227, 272, 263, 254, 271, 243, 249, 262, 246, 244, 252, 308, 254, 272, 234, 238, 258, 240, 252, 248, 229, 232, 236, 254, 237, 229, 242, 287, 265, 267, 274, 251, 249, 257, 240, 251, 234, 281, 254, 271, 250, 294, 273, 243, 229, 278, 244, 245, 241, 258, 257, 254, 300, 248, 246, 271, 289, 268, 256, 269, 244, 247, 261, 266, 245, 269, 257, 231, 230, 250, 257, 263, 255, 263, 245, 253, 263, 243, 279, 281, 269, 248, 239, 262, 238, 246, 245, 267, 260, 241, 261, 239, 260, 255, 252, 263, 236, 273, 250, 238, 281, 250, 260, 266, 276, 271, 250, 266, 278, 252, 246, 247, 251, 273, 274] payload_rows_per_dst=[5504, 5392, 5402, 5430, 5548, 5430, 5420, 5536] dup_rows_per_dst=[2706, 2762, 2592, 2776, 2832, 2708, 2676, 2822] combine_rows_per_src=[5504, 5392, 5402, 5430, 5548, 5430, 5420, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=230424 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157058072 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885720 weight_movement_bytes=18874368 total_nvlink_bytes=332760088 weight_to_token_ratio=0.060131 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38284 dup_rows=21674 dup_rows_remote=18900 dedup_ratio=0.330719 router_tokens_per_expert=[243, 257, 273, 264, 260, 252, 259, 237, 275, 272, 257, 252, 230, 230, 255, 255, 260, 251, 272, 287, 249, 274, 265, 251, 248, 249, 245, 276, 259, 293, 263, 234, 283, 261, 269, 253, 260, 258, 254, 253, 237, 268, 225, 252, 249, 251, 249, 254, 267, 249, 264, 252, 239, 214, 258, 245, 257, 249, 246, 260, 253, 266, 280, 232, 240, 252, 249, 262, 254, 243, 246, 243, 259, 285, 250, 229, 254, 270, 274, 271, 274, 235, 234, 253, 232, 252, 263, 260, 233, 228, 268, 256, 276, 274, 263, 230, 255, 249, 271, 267, 252, 254, 255, 254, 265, 228, 260, 265, 283, 287, 245, 266, 294, 257, 257, 232, 253, 256, 245, 260, 277, 279, 231, 246, 278, 260, 272, 249] payload_rows_per_dst=[5352, 5578, 5536, 5466, 5388, 5386, 5574, 5582] dup_rows_per_dst=[2790, 2774, 2616, 2596, 2774, 2676, 2738, 2710] combine_rows_per_src=[5352, 5578, 5536, 5466, 5388, 5386, 5574, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38364 dup_rows=21726 dup_rows_remote=19020 dedup_ratio=0.331512 router_tokens_per_expert=[254, 239, 262, 248, 259, 284, 257, 250, 255, 234, 254, 247, 252, 256, 258, 246, 278, 281, 257, 264, 272, 254, 269, 254, 267, 244, 248, 275, 268, 265, 247, 257, 231, 234, 271, 239, 220, 261, 267, 248, 247, 255, 238, 263, 221, 241, 295, 298, 257, 269, 268, 253, 280, 278, 265, 234, 240, 265, 254, 250, 254, 245, 240, 256, 291, 261, 242, 240, 248, 259, 244, 279, 264, 240, 269, 267, 254, 262, 273, 256, 245, 251, 262, 267, 243, 267, 257, 246, 260, 222, 284, 257, 249, 237, 259, 246, 271, 255, 285, 255, 295, 215, 237, 274, 240, 248, 261, 271, 263, 244, 226, 236, 278, 251, 245, 243, 264, 278, 231, 266, 277, 214, 251, 277, 261, 266, 232, 265] payload_rows_per_dst=[5470, 5534, 5466, 5486, 5464, 5380, 5446, 5564] dup_rows_per_dst=[2640, 2866, 2592, 2730, 2834, 2724, 2706, 2634] combine_rows_per_src=[5470, 5534, 5466, 5486, 5464, 5380, 5446, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38258 dup_rows=21830 dup_rows_remote=19112 dedup_ratio=0.333099 router_tokens_per_expert=[261, 232, 273, 255, 258, 279, 251, 257, 265, 246, 254, 271, 241, 274, 236, 233, 250, 276, 263, 253, 252, 242, 254, 261, 264, 258, 273, 266, 263, 274, 255, 243, 238, 263, 260, 249, 277, 236, 242, 287, 257, 262, 243, 265, 252, 257, 255, 270, 262, 232, 264, 248, 240, 236, 252, 236, 225, 259, 275, 259, 291, 277, 272, 255, 252, 230, 311, 280, 245, 258, 272, 272, 256, 253, 254, 242, 271, 235, 253, 253, 240, 277, 271, 274, 260, 257, 247, 267, 217, 267, 260, 250, 249, 231, 277, 227, 275, 283, 253, 229, 283, 226, 257, 263, 244, 264, 234, 239, 296, 245, 281, 283, 256, 244, 258, 258, 270, 227, 250, 231, 260, 237, 258, 241, 266, 235, 239, 246] payload_rows_per_dst=[5442, 5540, 5524, 5448, 5486, 5442, 5486, 5338] dup_rows_per_dst=[2730, 2754, 2702, 2718, 2788, 2700, 2824, 2614] combine_rows_per_src=[5442, 5540, 5524, 5448, 5486, 5442, 5486, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38292 dup_rows=21788 dup_rows_remote=19088 dedup_ratio=0.332458 router_tokens_per_expert=[233, 253, 275, 242, 269, 257, 237, 258, 246, 281, 236, 274, 278, 288, 255, 239, 270, 276, 259, 261, 248, 240, 237, 248, 281, 269, 248, 248, 241, 271, 255, 266, 282, 261, 249, 254, 251, 236, 235, 262, 261, 236, 266, 248, 247, 252, 255, 225, 232, 221, 250, 287, 272, 271, 249, 265, 226, 236, 256, 259, 246, 281, 247, 251, 257, 256, 231, 266, 265, 259, 270, 260, 280, 225, 260, 231, 268, 264, 279, 246, 254, 254, 264, 256, 262, 286, 242, 276, 250, 264, 236, 230, 247, 256, 233, 270, 303, 258, 238, 235, 277, 278, 263, 288, 274, 258, 248, 251, 258, 265, 267, 241, 235, 263, 262, 253, 274, 258, 255, 259, 249, 251, 248, 272, 240, 238, 262, 242] payload_rows_per_dst=[5526, 5376, 5396, 5426, 5412, 5584, 5572, 5456] dup_rows_per_dst=[2716, 2604, 2644, 2672, 2812, 2832, 2808, 2700] combine_rows_per_src=[5526, 5376, 5396, 5426, 5412, 5584, 5572, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229456 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245520 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261584 weight_movement_bytes=18874368 total_nvlink_bytes=333135952 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229592 combine_payload_bytes=156598272 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156827864 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426136 weight_movement_bytes=18874368 total_nvlink_bytes=332300504 weight_to_token_ratio=0.060220 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228736 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157040000 ingress_bytes=213434368 link_bytes=213434368 token_bytes=313851264 weight_movement_bytes=56623104 total_nvlink_bytes=370474368 weight_to_token_ratio=0.180414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045023098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157138944 dispatch_meta_bytes=229536 combine_payload_bytes=157138944 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157368480 ingress_bytes=232636416 link_bytes=232636416 token_bytes=314507424 weight_movement_bytes=75497472 total_nvlink_bytes=390004896 weight_to_token_ratio=0.240050 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229007674e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229480 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934248 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639016 weight_movement_bytes=18874368 total_nvlink_bytes=332513384 weight_to_token_ratio=0.060179 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38086 dup_rows=21946 dup_rows_remote=19120 dedup_ratio=0.334869 router_tokens_per_expert=[241, 237, 232, 247, 252, 248, 258, 245, 274, 263, 236, 248, 280, 253, 248, 255, 260, 268, 278, 254, 257, 270, 287, 249, 304, 258, 283, 268, 268, 241, 275, 223, 270, 267, 252, 245, 284, 252, 252, 283, 279, 233, 254, 234, 235, 269, 258, 272, 260, 270, 253, 241, 248, 283, 232, 231, 254, 282, 260, 247, 276, 232, 254, 260, 262, 253, 247, 255, 256, 245, 271, 227, 272, 246, 273, 257, 236, 238, 257, 284, 253, 260, 259, 248, 253, 273, 260, 227, 267, 251, 246, 248, 252, 240, 247, 239, 250, 268, 265, 267, 243, 259, 256, 284, 259, 246, 266, 271, 277, 240, 256, 238, 247, 249, 257, 264, 245, 256, 245, 212, 249, 245, 265, 269, 277, 260, 268, 231] payload_rows_per_dst=[5504, 5594, 5462, 5452, 5400, 5382, 5426, 5370] dup_rows_per_dst=[2826, 2892, 2816, 2714, 2758, 2664, 2624, 2652] combine_rows_per_src=[5504, 5594, 5462, 5452, 5400, 5382, 5426, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229520 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073552 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917584 weight_movement_bytes=18874368 total_nvlink_bytes=332791952 weight_to_token_ratio=0.060125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38216 dup_rows=21868 dup_rows_remote=19034 dedup_ratio=0.333679 router_tokens_per_expert=[254, 250, 237, 249, 283, 262, 243, 269, 272, 228, 208, 274, 256, 233, 236, 263, 260, 249, 254, 256, 258, 251, 268, 256, 257, 262, 257, 281, 253, 252, 252, 251, 246, 265, 218, 266, 256, 258, 275, 267, 270, 241, 259, 245, 258, 241, 261, 255, 270, 267, 252, 265, 257, 265, 257, 282, 267, 265, 268, 235, 256, 218, 233, 270, 246, 268, 257, 267, 269, 240, 249, 265, 245, 272, 234, 245, 267, 290, 273, 256, 245, 252, 255, 254, 245, 251, 243, 268, 256, 250, 227, 262, 249, 254, 275, 284, 250, 243, 238, 277, 268, 254, 253, 260, 243, 258, 274, 264, 245, 251, 272, 241, 258, 250, 236, 259, 246, 251, 247, 282, 262, 253, 259, 274, 263, 276, 246, 260] payload_rows_per_dst=[5414, 5440, 5366, 5484, 5452, 5480, 5464, 5568] dup_rows_per_dst=[2620, 2794, 2796, 2770, 2834, 2660, 2718, 2676] combine_rows_per_src=[5414, 5440, 5366, 5484, 5452, 5480, 5464, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=228824 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156229080 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312229336 weight_movement_bytes=37748736 total_nvlink_bytes=349978072 weight_to_token_ratio=0.120901 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38244 dup_rows=21842 dup_rows_remote=19072 dedup_ratio=0.333282 router_tokens_per_expert=[261, 259, 247, 243, 246, 253, 273, 247, 256, 294, 246, 273, 237, 226, 284, 267, 256, 254, 250, 261, 249, 260, 278, 283, 255, 254, 264, 237, 242, 258, 265, 248, 239, 276, 258, 256, 279, 248, 243, 233, 283, 255, 217, 253, 252, 286, 255, 277, 262, 261, 257, 263, 256, 277, 264, 280, 219, 243, 284, 253, 278, 257, 278, 245, 262, 248, 280, 253, 226, 269, 250, 228, 265, 262, 248, 258, 239, 243, 276, 267, 282, 245, 264, 243, 230, 260, 248, 241, 251, 275, 277, 260, 263, 238, 258, 260, 239, 248, 257, 255, 236, 227, 271, 239, 277, 254, 266, 250, 234, 226, 239, 259, 254, 260, 245, 259, 273, 250, 240, 254, 230, 271, 243, 250, 290, 262, 252, 276] payload_rows_per_dst=[5424, 5504, 5450, 5530, 5460, 5480, 5392, 5454] dup_rows_per_dst=[2800, 2724, 2770, 2824, 2688, 2710, 2562, 2764] combine_rows_per_src=[5424, 5504, 5450, 5530, 5460, 5480, 5392, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38390 dup_rows=21736 dup_rows_remote=19120 dedup_ratio=0.331665 router_tokens_per_expert=[273, 295, 296, 230, 267, 244, 262, 251, 255, 271, 245, 268, 260, 231, 260, 243, 256, 256, 242, 258, 247, 290, 242, 257, 273, 253, 228, 270, 279, 251, 260, 248, 282, 249, 254, 243, 283, 262, 250, 251, 226, 257, 246, 242, 264, 286, 281, 243, 240, 238, 262, 240, 272, 256, 260, 260, 279, 228, 260, 249, 239, 268, 267, 251, 250, 273, 265, 246, 235, 260, 251, 241, 264, 233, 247, 267, 254, 260, 274, 266, 255, 232, 259, 244, 262, 232, 244, 253, 271, 248, 232, 261, 261, 256, 255, 248, 229, 241, 254, 302, 250, 277, 239, 244, 269, 254, 260, 254, 294, 250, 258, 243, 246, 268, 235, 259, 270, 255, 273, 297, 269, 225, 248, 255, 267, 263, 243, 229] payload_rows_per_dst=[5538, 5516, 5482, 5408, 5476, 5410, 5474, 5496] dup_rows_per_dst=[2764, 2704, 2756, 2730, 2696, 2616, 2762, 2708] combine_rows_per_src=[5538, 5516, 5482, 5408, 5476, 5410, 5474, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38288 dup_rows=21802 dup_rows_remote=19188 dedup_ratio=0.332672 router_tokens_per_expert=[261, 252, 256, 253, 263, 245, 244, 274, 258, 255, 263, 235, 246, 261, 250, 274, 228, 257, 246, 232, 271, 251, 256, 251, 268, 263, 250, 259, 253, 248, 244, 253, 244, 252, 247, 234, 255, 231, 241, 230, 286, 266, 274, 246, 244, 233, 232, 257, 259, 283, 238, 261, 262, 270, 244, 278, 230, 255, 273, 241, 248, 278, 271, 275, 270, 265, 228, 233, 259, 259, 259, 292, 254, 276, 278, 258, 283, 261, 267, 237, 272, 262, 261, 268, 255, 260, 229, 277, 290, 269, 254, 253, 276, 270, 257, 253, 244, 233, 271, 263, 255, 251, 240, 240, 271, 296, 254, 269, 244, 239, 260, 240, 246, 247, 261, 257, 264, 232, 263, 257, 234, 264, 269, 283, 236, 250, 242, 250] payload_rows_per_dst=[5460, 5446, 5474, 5460, 5514, 5554, 5422, 5404] dup_rows_per_dst=[2720, 2614, 2648, 2694, 2844, 2858, 2718, 2706] combine_rows_per_src=[5460, 5446, 5474, 5460, 5514, 5554, 5422, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229000 combine_payload_bytes=156532736 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156761736 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294472 weight_movement_bytes=37748736 total_nvlink_bytes=351043208 weight_to_token_ratio=0.120490 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38250 dup_rows=21776 dup_rows_remote=19028 dedup_ratio=0.332275 router_tokens_per_expert=[253, 259, 241, 263, 270, 243, 284, 230, 275, 249, 244, 237, 273, 239, 284, 264, 249, 245, 272, 212, 264, 252, 229, 253, 256, 274, 239, 259, 294, 256, 260, 244, 228, 240, 264, 268, 255, 227, 251, 236, 213, 256, 275, 254, 280, 242, 234, 247, 303, 234, 261, 251, 235, 282, 280, 255, 254, 276, 281, 274, 276, 239, 271, 259, 242, 256, 255, 271, 269, 247, 261, 240, 249, 248, 238, 236, 294, 267, 267, 280, 285, 270, 277, 246, 288, 280, 261, 241, 244, 282, 254, 283, 218, 246, 237, 231, 277, 268, 253, 222, 250, 226, 271, 272, 281, 281, 268, 254, 249, 252, 228, 277, 252, 235, 261, 263, 252, 267, 228, 229, 261, 253, 232, 263, 278, 261, 243, 231] payload_rows_per_dst=[5484, 5394, 5386, 5530, 5540, 5520, 5510, 5396] dup_rows_per_dst=[2732, 2722, 2554, 2932, 2700, 2766, 2748, 2622] combine_rows_per_src=[5484, 5394, 5386, 5530, 5540, 5520, 5510, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38326 dup_rows=21786 dup_rows_remote=19168 dedup_ratio=0.332428 router_tokens_per_expert=[269, 254, 249, 238, 277, 248, 255, 263, 257, 253, 266, 254, 245, 242, 264, 217, 233, 244, 259, 246, 250, 243, 270, 249, 283, 250, 253, 261, 271, 261, 279, 269, 285, 239, 258, 261, 254, 275, 262, 245, 246, 272, 247, 273, 254, 251, 254, 247, 259, 267, 262, 251, 248, 256, 238, 250, 235, 242, 232, 247, 268, 290, 239, 237, 241, 266, 274, 240, 241, 262, 253, 261, 260, 254, 257, 276, 247, 262, 258, 274, 252, 237, 235, 264, 252, 249, 257, 247, 275, 245, 257, 259, 249, 256, 246, 211, 254, 253, 272, 267, 261, 231, 256, 274, 276, 281, 282, 283, 278, 237, 237, 267, 250, 262, 235, 255, 285, 257, 227, 261, 228, 238, 238, 276, 281, 270, 284, 279] payload_rows_per_dst=[5428, 5466, 5420, 5424, 5482, 5372, 5648, 5510] dup_rows_per_dst=[2674, 2776, 2826, 2618, 2770, 2610, 2770, 2742] combine_rows_per_src=[5428, 5466, 5420, 5424, 5482, 5372, 5648, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38354 dup_rows=21612 dup_rows_remote=18902 dedup_ratio=0.329773 router_tokens_per_expert=[243, 273, 254, 264, 264, 257, 247, 274, 276, 284, 228, 248, 268, 253, 266, 245, 255, 253, 282, 239, 272, 237, 253, 272, 262, 237, 250, 274, 253, 260, 283, 262, 234, 239, 205, 236, 234, 273, 249, 273, 256, 257, 247, 244, 263, 273, 230, 275, 268, 261, 267, 258, 227, 266, 231, 260, 260, 263, 248, 270, 281, 228, 251, 266, 249, 262, 238, 264, 272, 227, 265, 236, 250, 244, 276, 274, 245, 271, 263, 262, 248, 253, 285, 241, 241, 229, 254, 247, 257, 253, 241, 272, 246, 240, 240, 257, 251, 270, 256, 292, 274, 257, 267, 262, 224, 259, 257, 256, 244, 275, 246, 258, 266, 257, 242, 272, 278, 256, 261, 254, 244, 258, 261, 230, 282, 280, 255, 241] payload_rows_per_dst=[5554, 5490, 5424, 5538, 5416, 5430, 5502, 5570] dup_rows_per_dst=[2734, 2798, 2552, 2672, 2612, 2740, 2794, 2710] combine_rows_per_src=[5554, 5490, 5424, 5538, 5416, 5430, 5502, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229264 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156876688 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524112 weight_movement_bytes=18874368 total_nvlink_bytes=332398480 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=230040 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157475480 ingress_bytes=176119808 link_bytes=176119808 token_bytes=314720920 weight_movement_bytes=18874368 total_nvlink_bytes=333595288 weight_to_token_ratio=0.059972 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687493344e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229904 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057552 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885200 weight_movement_bytes=18874368 total_nvlink_bytes=332759568 weight_to_token_ratio=0.060131 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38272 dup_rows=21832 dup_rows_remote=19288 dedup_ratio=0.333130 router_tokens_per_expert=[254, 284, 259, 243, 262, 229, 226, 253, 227, 247, 255, 266, 262, 246, 223, 252, 221, 273, 247, 256, 274, 253, 270, 253, 247, 237, 241, 267, 258, 256, 273, 279, 248, 272, 253, 261, 246, 259, 251, 267, 303, 245, 280, 257, 263, 262, 252, 232, 239, 260, 236, 272, 257, 260, 271, 269, 261, 266, 253, 281, 240, 259, 269, 273, 277, 267, 217, 254, 267, 266, 271, 262, 253, 253, 284, 234, 232, 221, 244, 267, 258, 261, 258, 228, 236, 277, 231, 260, 234, 257, 271, 250, 232, 254, 241, 257, 289, 247, 279, 278, 257, 266, 249, 254, 237, 254, 246, 254, 262, 260, 261, 244, 283, 235, 275, 243, 251, 268, 281, 260, 268, 260, 234, 294, 249, 232, 267, 247] payload_rows_per_dst=[5432, 5692, 5540, 5368, 5410, 5410, 5368, 5484] dup_rows_per_dst=[2544, 2938, 2970, 2544, 2728, 2600, 2698, 2810] combine_rows_per_src=[5432, 5692, 5540, 5368, 5410, 5410, 5368, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229112 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901112 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573112 weight_movement_bytes=18874368 total_nvlink_bytes=332447480 weight_to_token_ratio=0.060191 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229976 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157213272 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314196568 weight_movement_bytes=37748736 total_nvlink_bytes=351945304 weight_to_token_ratio=0.120144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229024 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157327008 ingress_bytes=213721088 link_bytes=213721088 token_bytes=314424992 weight_movement_bytes=56623104 total_nvlink_bytes=371048096 weight_to_token_ratio=0.180085 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047770308e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=230240 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156992352 ingress_bytes=213385216 link_bytes=213385216 token_bytes=313754464 weight_movement_bytes=56623104 total_nvlink_bytes=370377568 weight_to_token_ratio=0.180469 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044552148e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38362 dup_rows=21672 dup_rows_remote=19068 dedup_ratio=0.330688 router_tokens_per_expert=[255, 243, 296, 262, 241, 281, 256, 252, 262, 279, 233, 244, 264, 254, 276, 267, 247, 252, 257, 282, 259, 279, 251, 229, 238, 277, 271, 260, 278, 234, 259, 250, 251, 228, 256, 264, 252, 260, 232, 270, 276, 237, 273, 260, 242, 228, 269, 251, 252, 264, 275, 260, 252, 265, 248, 262, 246, 255, 228, 292, 237, 259, 259, 243, 247, 258, 246, 269, 288, 237, 257, 254, 274, 251, 254, 260, 240, 220, 247, 251, 235, 252, 267, 238, 253, 296, 258, 301, 246, 265, 266, 246, 241, 271, 262, 244, 247, 244, 230, 259, 251, 245, 297, 243, 248, 236, 234, 248, 231, 254, 253, 296, 252, 254, 267, 252, 261, 271, 252, 242, 236, 278, 269, 252, 248, 277, 269, 244] payload_rows_per_dst=[5462, 5494, 5444, 5450, 5502, 5536, 5454, 5522] dup_rows_per_dst=[2868, 2752, 2654, 2744, 2604, 2746, 2578, 2726] combine_rows_per_src=[5462, 5494, 5444, 5450, 5502, 5536, 5454, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43988 payload_rows_remote=38470 dup_rows=21548 dup_rows_remote=18974 dedup_ratio=0.328796 router_tokens_per_expert=[249, 236, 233, 282, 259, 261, 264, 259, 233, 257, 247, 245, 261, 253, 274, 257, 249, 242, 239, 283, 247, 251, 254, 260, 257, 258, 228, 275, 236, 234, 262, 256, 253, 271, 243, 254, 264, 226, 220, 272, 269, 275, 276, 259, 248, 237, 248, 231, 259, 242, 244, 259, 259, 254, 236, 242, 259, 261, 277, 281, 265, 257, 241, 241, 241, 264, 233, 272, 262, 259, 272, 224, 257, 268, 259, 251, 281, 294, 275, 259, 270, 253, 236, 248, 244, 257, 258, 273, 270, 266, 248, 278, 276, 284, 262, 210, 263, 265, 242, 264, 234, 238, 277, 273, 250, 235, 287, 250, 245, 253, 262, 232, 260, 245, 257, 261, 296, 261, 260, 245, 259, 265, 258, 276, 260, 259, 224, 284] payload_rows_per_dst=[5470, 5488, 5518, 5464, 5546, 5526, 5444, 5532] dup_rows_per_dst=[2670, 2574, 2574, 2690, 2796, 2740, 2696, 2808] combine_rows_per_src=[5470, 5488, 5518, 5464, 5546, 5526, 5444, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38386 dup_rows=21750 dup_rows_remote=19158 dedup_ratio=0.331879 router_tokens_per_expert=[257, 265, 257, 262, 237, 252, 240, 255, 224, 256, 242, 290, 267, 244, 251, 226, 260, 239, 285, 251, 236, 237, 279, 263, 259, 266, 245, 252, 263, 255, 221, 277, 251, 245, 267, 235, 245, 259, 264, 256, 267, 258, 246, 231, 271, 240, 264, 245, 279, 257, 241, 243, 277, 256, 295, 251, 284, 244, 243, 265, 261, 270, 259, 281, 245, 267, 234, 288, 275, 236, 277, 276, 245, 256, 234, 244, 250, 254, 260, 268, 245, 245, 254, 238, 245, 280, 224, 271, 249, 233, 258, 260, 267, 236, 258, 233, 242, 250, 252, 245, 253, 272, 249, 254, 268, 233, 241, 252, 257, 249, 259, 267, 266, 266, 237, 281, 270, 273, 253, 272, 280, 254, 279, 300, 260, 258, 242, 266] payload_rows_per_dst=[5410, 5438, 5438, 5512, 5514, 5400, 5504, 5570] dup_rows_per_dst=[2640, 2738, 2650, 2900, 2704, 2592, 2582, 2944] combine_rows_per_src=[5410, 5438, 5438, 5512, 5514, 5400, 5504, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38376 dup_rows=21676 dup_rows_remote=18978 dedup_ratio=0.330750 router_tokens_per_expert=[249, 265, 243, 241, 269, 260, 247, 220, 262, 265, 236, 235, 284, 268, 257, 249, 245, 267, 258, 239, 284, 256, 250, 229, 258, 220, 274, 289, 277, 254, 250, 241, 274, 258, 256, 247, 273, 269, 253, 219, 251, 260, 269, 247, 250, 260, 249, 264, 271, 268, 250, 241, 272, 223, 255, 263, 243, 238, 258, 287, 231, 260, 244, 263, 261, 242, 263, 278, 262, 306, 232, 254, 277, 283, 252, 229, 235, 258, 241, 275, 245, 257, 288, 280, 262, 251, 255, 235, 233, 281, 233, 244, 290, 257, 253, 250, 259, 275, 237, 251, 233, 253, 264, 250, 264, 245, 259, 257, 269, 251, 262, 270, 254, 253, 248, 273, 255, 230, 284, 252, 267, 257, 254, 250, 253, 261, 256, 253] payload_rows_per_dst=[5410, 5484, 5540, 5440, 5510, 5490, 5484, 5502] dup_rows_per_dst=[2748, 2698, 2658, 2694, 2728, 2738, 2714, 2698] combine_rows_per_src=[5410, 5484, 5540, 5440, 5510, 5490, 5484, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=229720 combine_payload_bytes=157130752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157360472 ingress_bytes=232628224 link_bytes=232628224 token_bytes=314491224 weight_movement_bytes=75497472 total_nvlink_bytes=389988696 weight_to_token_ratio=0.240062 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.228929182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38324 dup_rows=21694 dup_rows_remote=18868 dedup_ratio=0.331024 router_tokens_per_expert=[276, 234, 261, 273, 258, 249, 276, 234, 240, 265, 238, 249, 257, 252, 253, 246, 259, 243, 240, 248, 324, 268, 231, 245, 250, 244, 243, 240, 283, 229, 248, 266, 251, 265, 268, 265, 247, 278, 251, 259, 241, 266, 251, 249, 261, 273, 261, 247, 269, 264, 250, 271, 256, 229, 270, 244, 248, 277, 249, 245, 278, 233, 210, 268, 271, 230, 286, 261, 239, 266, 274, 252, 259, 267, 278, 244, 285, 253, 252, 255, 215, 210, 242, 251, 253, 276, 260, 257, 267, 260, 239, 233, 264, 243, 249, 249, 244, 262, 264, 283, 262, 252, 292, 241, 238, 276, 278, 243, 254, 278, 245, 260, 276, 250, 263, 255, 242, 259, 267, 228, 272, 269, 247, 243, 274, 289, 246, 260] payload_rows_per_dst=[5468, 5394, 5550, 5446, 5580, 5408, 5518, 5478] dup_rows_per_dst=[2654, 2728, 2716, 2676, 2764, 2528, 2826, 2802] combine_rows_per_src=[5468, 5394, 5550, 5446, 5580, 5408, 5518, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38184 dup_rows=21792 dup_rows_remote=18938 dedup_ratio=0.332520 router_tokens_per_expert=[238, 244, 258, 265, 213, 229, 239, 232, 239, 233, 228, 258, 235, 275, 256, 268, 264, 277, 254, 259, 305, 263, 257, 265, 232, 235, 252, 261, 281, 255, 257, 241, 231, 274, 251, 234, 243, 247, 231, 262, 266, 255, 262, 245, 271, 235, 242, 254, 262, 287, 239, 264, 254, 280, 294, 265, 248, 256, 258, 269, 251, 266, 255, 259, 260, 267, 254, 266, 248, 267, 224, 280, 285, 258, 240, 256, 255, 258, 238, 273, 234, 258, 266, 254, 244, 251, 269, 279, 269, 245, 240, 231, 263, 248, 256, 259, 255, 256, 236, 244, 279, 279, 236, 278, 282, 271, 285, 244, 225, 249, 268, 277, 254, 259, 268, 246, 252, 251, 261, 248, 275, 260, 233, 253, 249, 268, 257, 297] payload_rows_per_dst=[5282, 5530, 5400, 5560, 5514, 5434, 5576, 5448] dup_rows_per_dst=[2538, 2786, 2606, 2854, 2744, 2698, 2752, 2814] combine_rows_per_src=[5282, 5530, 5400, 5560, 5514, 5434, 5576, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38374 dup_rows=21610 dup_rows_remote=18900 dedup_ratio=0.329742 router_tokens_per_expert=[259, 260, 250, 261, 260, 261, 260, 243, 258, 240, 266, 251, 266, 251, 264, 240, 240, 253, 209, 250, 251, 261, 251, 253, 265, 267, 253, 263, 270, 248, 262, 250, 251, 233, 245, 233, 231, 230, 268, 269, 239, 259, 234, 259, 252, 240, 259, 296, 249, 268, 264, 283, 261, 233, 252, 269, 270, 247, 255, 251, 267, 279, 264, 266, 257, 258, 246, 290, 256, 274, 263, 274, 233, 273, 273, 269, 272, 243, 244, 242, 248, 255, 265, 263, 262, 284, 276, 260, 242, 230, 244, 218, 259, 260, 258, 251, 254, 252, 248, 284, 251, 246, 264, 267, 236, 233, 272, 253, 269, 264, 243, 253, 250, 247, 274, 264, 264, 263, 279, 255, 251, 257, 239, 258, 243, 276, 269, 236] payload_rows_per_dst=[5488, 5378, 5492, 5554, 5600, 5468, 5394, 5552] dup_rows_per_dst=[2692, 2714, 2504, 2778, 2722, 2706, 2784, 2710] combine_rows_per_src=[5488, 5378, 5492, 5554, 5600, 5468, 5394, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229776 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157802896 ingress_bytes=176447488 link_bytes=176447488 token_bytes=315376016 weight_movement_bytes=18874368 total_nvlink_bytes=334250384 weight_to_token_ratio=0.059847 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690633012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=230176 combine_payload_bytes=157229056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157459232 ingress_bytes=176103424 link_bytes=176103424 token_bytes=314688288 weight_movement_bytes=18874368 total_nvlink_bytes=333562656 weight_to_token_ratio=0.059978 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687336360e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229416 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417512 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314605608 weight_movement_bytes=18874368 total_nvlink_bytes=333479976 weight_to_token_ratio=0.059994 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38334 dup_rows=21732 dup_rows_remote=18930 dedup_ratio=0.331604 router_tokens_per_expert=[269, 231, 274, 290, 267, 239, 255, 277, 265, 273, 244, 249, 291, 233, 241, 235, 256, 261, 277, 249, 244, 274, 264, 254, 250, 266, 254, 258, 255, 231, 287, 235, 235, 233, 258, 274, 274, 258, 277, 267, 210, 249, 227, 257, 240, 268, 247, 250, 271, 278, 238, 275, 231, 279, 258, 270, 255, 255, 254, 267, 275, 249, 293, 247, 240, 244, 274, 246, 289, 251, 285, 236, 256, 268, 249, 286, 258, 248, 239, 268, 251, 240, 255, 269, 266, 245, 251, 253, 235, 261, 231, 240, 236, 244, 273, 262, 237, 246, 278, 255, 260, 266, 258, 266, 271, 247, 262, 263, 244, 236, 250, 284, 229, 245, 247, 271, 275, 284, 242, 254, 264, 250, 245, 235, 257, 230, 258, 243] payload_rows_per_dst=[5470, 5564, 5560, 5506, 5510, 5392, 5462, 5340] dup_rows_per_dst=[2802, 2882, 2722, 2668, 2764, 2626, 2784, 2484] combine_rows_per_src=[5470, 5564, 5560, 5506, 5510, 5392, 5462, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=228768 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157203872 ingress_bytes=194723840 link_bytes=194723840 token_bytes=314178976 weight_movement_bytes=37748736 total_nvlink_bytes=351927712 weight_to_token_ratio=0.120150 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865748025e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228488 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156630152 ingress_bytes=213024768 link_bytes=213024768 token_bytes=313031816 weight_movement_bytes=56623104 total_nvlink_bytes=369654920 weight_to_token_ratio=0.180886 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041098512e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229096 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157409000 ingress_bytes=194928640 link_bytes=194928640 token_bytes=314588904 weight_movement_bytes=37748736 total_nvlink_bytes=352337640 weight_to_token_ratio=0.119994 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867710318e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229056 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=157245120 ingress_bytes=251387904 link_bytes=251387904 token_bytes=314261184 weight_movement_bytes=94371840 total_nvlink_bytes=408633024 weight_to_token_ratio=0.300297 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408675206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38206 dup_rows=21768 dup_rows_remote=18892 dedup_ratio=0.332153 router_tokens_per_expert=[241, 248, 278, 247, 258, 275, 240, 227, 258, 261, 255, 265, 245, 241, 271, 255, 251, 264, 261, 252, 243, 255, 266, 258, 259, 266, 256, 266, 222, 254, 269, 261, 269, 248, 241, 264, 258, 255, 239, 256, 254, 268, 252, 286, 239, 259, 279, 283, 255, 256, 276, 254, 257, 243, 238, 271, 252, 225, 255, 288, 259, 229, 270, 263, 235, 267, 282, 277, 276, 276, 267, 253, 234, 294, 246, 259, 243, 299, 270, 241, 253, 265, 261, 237, 256, 270, 247, 257, 270, 285, 244, 247, 262, 242, 256, 242, 237, 279, 257, 268, 218, 239, 234, 277, 240, 280, 248, 257, 267, 232, 260, 266, 240, 251, 251, 246, 263, 241, 229, 270, 238, 250, 253, 244, 253, 231, 256, 271] payload_rows_per_dst=[5496, 5488, 5552, 5478, 5562, 5396, 5396, 5400] dup_rows_per_dst=[2634, 2718, 2748, 2704, 2876, 2792, 2722, 2574] combine_rows_per_src=[5496, 5488, 5552, 5478, 5562, 5396, 5396, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38254 dup_rows=21778 dup_rows_remote=19128 dedup_ratio=0.332306 router_tokens_per_expert=[235, 254, 219, 270, 222, 248, 303, 270, 276, 263, 269, 295, 262, 249, 293, 257, 240, 236, 273, 242, 266, 265, 218, 300, 258, 236, 264, 250, 272, 232, 268, 271, 254, 255, 239, 242, 226, 260, 285, 231, 268, 260, 249, 254, 238, 266, 279, 271, 256, 241, 248, 265, 251, 278, 291, 241, 244, 238, 255, 242, 254, 254, 276, 237, 259, 234, 253, 258, 247, 261, 237, 279, 267, 254, 278, 261, 273, 256, 253, 242, 232, 267, 246, 220, 274, 228, 266, 294, 242, 264, 242, 244, 242, 252, 265, 249, 240, 247, 249, 261, 259, 270, 246, 258, 265, 265, 265, 237, 259, 242, 265, 253, 251, 264, 245, 242, 253, 286, 258, 246, 269, 251, 257, 251, 253, 243, 278, 277] payload_rows_per_dst=[5540, 5440, 5504, 5436, 5478, 5392, 5456, 5512] dup_rows_per_dst=[2830, 2742, 2650, 2706, 2746, 2662, 2706, 2736] combine_rows_per_src=[5540, 5440, 5504, 5436, 5478, 5392, 5456, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228392 combine_payload_bytes=156491776 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156720168 ingress_bytes=213114880 link_bytes=213114880 token_bytes=313211944 weight_movement_bytes=56623104 total_nvlink_bytes=369835048 weight_to_token_ratio=0.180782 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041961921e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38232 dup_rows=21736 dup_rows_remote=18996 dedup_ratio=0.331665 router_tokens_per_expert=[275, 249, 274, 263, 289, 244, 269, 248, 232, 245, 254, 228, 244, 272, 245, 277, 269, 241, 266, 260, 243, 249, 253, 273, 269, 242, 236, 264, 221, 274, 244, 239, 244, 271, 252, 256, 227, 259, 265, 262, 259, 264, 227, 283, 272, 259, 244, 288, 261, 247, 250, 239, 249, 287, 240, 262, 274, 229, 253, 263, 281, 225, 278, 274, 243, 261, 244, 264, 267, 259, 250, 232, 255, 276, 249, 243, 226, 234, 269, 231, 252, 284, 242, 250, 274, 255, 261, 245, 257, 247, 266, 262, 249, 261, 269, 280, 245, 258, 281, 254, 275, 271, 245, 257, 238, 246, 251, 273, 254, 268, 263, 261, 273, 231, 258, 259, 242, 249, 274, 248, 228, 283, 277, 257, 258, 253, 243, 243] payload_rows_per_dst=[5402, 5440, 5506, 5482, 5402, 5568, 5490, 5510] dup_rows_per_dst=[2814, 2646, 2758, 2742, 2604, 2740, 2790, 2642] combine_rows_per_src=[5402, 5440, 5506, 5482, 5402, 5568, 5490, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38238 dup_rows=21774 dup_rows_remote=18970 dedup_ratio=0.332245 router_tokens_per_expert=[266, 234, 229, 255, 277, 241, 260, 273, 245, 236, 236, 270, 241, 255, 244, 246, 274, 253, 245, 247, 272, 238, 281, 273, 255, 249, 271, 262, 258, 252, 262, 272, 247, 268, 262, 250, 272, 254, 216, 278, 231, 250, 267, 266, 251, 266, 272, 246, 266, 256, 258, 276, 242, 248, 252, 268, 260, 231, 260, 229, 277, 243, 254, 254, 272, 276, 248, 264, 314, 248, 261, 265, 265, 255, 247, 280, 253, 259, 251, 245, 265, 276, 269, 218, 231, 274, 218, 280, 268, 285, 251, 239, 257, 254, 253, 260, 259, 258, 225, 259, 249, 259, 260, 271, 250, 251, 248, 277, 248, 206, 257, 252, 253, 250, 241, 269, 250, 264, 252, 256, 255, 247, 274, 262, 236, 269, 274, 244] payload_rows_per_dst=[5410, 5524, 5534, 5440, 5518, 5512, 5334, 5490] dup_rows_per_dst=[2606, 2804, 2658, 2708, 2888, 2684, 2724, 2702] combine_rows_per_src=[5410, 5524, 5534, 5440, 5518, 5512, 5334, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38162 dup_rows=21846 dup_rows_remote=19034 dedup_ratio=0.333344 router_tokens_per_expert=[263, 254, 255, 255, 272, 260, 298, 276, 263, 241, 242, 246, 261, 262, 271, 241, 255, 242, 239, 244, 252, 258, 240, 255, 241, 240, 254, 249, 255, 239, 238, 235, 244, 239, 275, 268, 234, 233, 272, 258, 231, 290, 240, 270, 241, 255, 266, 251, 258, 243, 251, 244, 232, 258, 261, 247, 250, 264, 235, 239, 273, 276, 265, 240, 253, 258, 273, 269, 270, 230, 266, 250, 245, 279, 270, 266, 243, 264, 229, 261, 280, 252, 248, 274, 230, 246, 262, 240, 256, 288, 267, 269, 258, 241, 256, 242, 273, 273, 260, 280, 262, 256, 262, 242, 273, 253, 244, 249, 249, 285, 273, 236, 265, 266, 246, 273, 281, 241, 255, 276, 239, 267, 283, 251, 277, 249, 252, 243] payload_rows_per_dst=[5500, 5336, 5476, 5398, 5486, 5478, 5528, 5488] dup_rows_per_dst=[2820, 2536, 2658, 2674, 2766, 2740, 2812, 2840] combine_rows_per_src=[5500, 5336, 5476, 5398, 5486, 5478, 5528, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38092 dup_rows=21962 dup_rows_remote=19192 dedup_ratio=0.335114 router_tokens_per_expert=[247, 241, 232, 287, 276, 251, 248, 218, 262, 263, 258, 240, 245, 249, 253, 244, 274, 255, 267, 252, 224, 255, 290, 242, 270, 243, 247, 261, 266, 233, 250, 247, 243, 271, 256, 269, 256, 276, 285, 259, 255, 243, 261, 257, 266, 287, 253, 260, 287, 271, 242, 248, 254, 254, 261, 259, 245, 287, 250, 243, 244, 279, 248, 254, 256, 266, 254, 268, 245, 249, 265, 258, 243, 259, 269, 260, 242, 254, 262, 247, 253, 239, 258, 259, 264, 241, 287, 248, 245, 264, 234, 235, 257, 280, 242, 260, 256, 277, 243, 269, 265, 245, 254, 260, 275, 258, 272, 246, 269, 261, 233, 242, 229, 257, 250, 270, 264, 248, 262, 268, 250, 261, 269, 249, 248, 255, 244, 243] payload_rows_per_dst=[5358, 5416, 5526, 5482, 5458, 5506, 5450, 5378] dup_rows_per_dst=[2670, 2736, 2868, 2770, 2736, 2626, 2800, 2756] combine_rows_per_src=[5358, 5416, 5526, 5482, 5458, 5506, 5450, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229528 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156917912 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606296 weight_movement_bytes=37748736 total_nvlink_bytes=351355032 weight_to_token_ratio=0.120370 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38338 dup_rows=21796 dup_rows_remote=19138 dedup_ratio=0.332581 router_tokens_per_expert=[254, 251, 237, 236, 249, 256, 247, 253, 227, 281, 252, 265, 238, 257, 232, 266, 271, 264, 277, 236, 263, 277, 255, 262, 228, 251, 254, 275, 273, 287, 259, 255, 258, 244, 267, 279, 270, 268, 242, 252, 252, 262, 257, 229, 242, 225, 276, 235, 268, 278, 273, 265, 265, 238, 252, 273, 267, 286, 256, 255, 279, 241, 265, 228, 258, 236, 255, 241, 260, 243, 280, 231, 285, 286, 271, 239, 240, 270, 250, 233, 266, 254, 257, 265, 275, 288, 246, 260, 262, 242, 255, 237, 286, 277, 231, 234, 278, 239, 267, 240, 276, 296, 264, 216, 218, 255, 247, 265, 246, 267, 255, 262, 246, 265, 244, 237, 276, 265, 256, 241, 219, 253, 258, 233, 262, 257, 280, 237] payload_rows_per_dst=[5362, 5496, 5292, 5526, 5562, 5498, 5602, 5402] dup_rows_per_dst=[2640, 2878, 2522, 2852, 2594, 2770, 2882, 2658] combine_rows_per_src=[5362, 5496, 5292, 5526, 5562, 5498, 5602, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38036 dup_rows=21956 dup_rows_remote=19032 dedup_ratio=0.335022 router_tokens_per_expert=[252, 242, 258, 257, 257, 231, 248, 222, 266, 242, 255, 250, 280, 277, 237, 245, 266, 245, 261, 236, 264, 241, 233, 248, 272, 263, 254, 278, 253, 253, 256, 242, 272, 270, 248, 254, 260, 272, 259, 287, 246, 260, 257, 288, 284, 245, 240, 267, 226, 269, 253, 243, 239, 264, 254, 231, 242, 257, 244, 270, 255, 272, 247, 258, 237, 258, 258, 276, 252, 252, 248, 221, 240, 267, 261, 235, 295, 231, 264, 247, 271, 253, 245, 242, 276, 272, 255, 253, 278, 251, 270, 267, 280, 267, 280, 247, 249, 238, 256, 259, 269, 265, 270, 244, 229, 238, 275, 263, 270, 265, 212, 271, 239, 245, 250, 261, 265, 241, 258, 288, 267, 276, 249, 263, 250, 244, 273, 260] payload_rows_per_dst=[5544, 5290, 5546, 5524, 5444, 5460, 5304, 5468] dup_rows_per_dst=[2924, 2508, 2942, 2864, 2542, 2954, 2432, 2790] combine_rows_per_src=[5544, 5290, 5546, 5524, 5444, 5460, 5304, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228912 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827184 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313425456 weight_movement_bytes=18874368 total_nvlink_bytes=332299824 weight_to_token_ratio=0.060220 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228832 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156851680 ingress_bytes=194371584 link_bytes=194371584 token_bytes=313474528 weight_movement_bytes=37748736 total_nvlink_bytes=351223264 weight_to_token_ratio=0.120420 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862372881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228784 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540336 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851888 weight_movement_bytes=18874368 total_nvlink_bytes=331726256 weight_to_token_ratio=0.060330 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156024832 dispatch_meta_bytes=229136 combine_payload_bytes=156024832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156253968 ingress_bytes=174899200 link_bytes=174899200 token_bytes=312278800 weight_movement_bytes=18874368 total_nvlink_bytes=331153168 weight_to_token_ratio=0.060441 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675798079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229904 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157262352 ingress_bytes=213655552 link_bytes=213655552 token_bytes=314294800 weight_movement_bytes=56623104 total_nvlink_bytes=370917904 weight_to_token_ratio=0.180159 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047142374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38310 dup_rows=21832 dup_rows_remote=19216 dedup_ratio=0.333130 router_tokens_per_expert=[266, 230, 252, 279, 286, 259, 254, 257, 237, 227, 253, 268, 242, 289, 249, 248, 249, 283, 260, 243, 266, 248, 271, 247, 238, 269, 262, 259, 271, 240, 253, 278, 260, 280, 257, 251, 291, 258, 239, 278, 253, 251, 268, 237, 262, 261, 314, 245, 297, 242, 260, 271, 258, 279, 273, 253, 255, 251, 254, 232, 260, 247, 257, 272, 271, 258, 246, 232, 232, 236, 232, 272, 258, 245, 254, 277, 242, 249, 272, 229, 255, 258, 246, 237, 250, 255, 256, 277, 261, 254, 255, 245, 243, 262, 271, 243, 268, 264, 271, 229, 259, 252, 221, 266, 248, 266, 256, 260, 236, 247, 247, 246, 259, 264, 238, 256, 238, 252, 249, 267, 257, 242, 271, 236, 258, 261, 256, 256] payload_rows_per_dst=[5460, 5498, 5534, 5564, 5394, 5446, 5400, 5408] dup_rows_per_dst=[2732, 2776, 2876, 2758, 2616, 2690, 2672, 2712] combine_rows_per_src=[5460, 5498, 5534, 5564, 5394, 5446, 5400, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=155795456 dispatch_meta_bytes=228272 combine_payload_bytes=155795456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156023728 ingress_bytes=174669824 link_bytes=174669824 token_bytes=311819184 weight_movement_bytes=18874368 total_nvlink_bytes=330693552 weight_to_token_ratio=0.060530 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673600311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38182 dup_rows=21884 dup_rows_remote=19102 dedup_ratio=0.333923 router_tokens_per_expert=[272, 257, 263, 259, 267, 234, 248, 280, 240, 270, 280, 250, 255, 253, 270, 247, 263, 241, 249, 253, 238, 232, 271, 249, 256, 251, 277, 262, 258, 253, 252, 247, 273, 256, 275, 230, 247, 258, 231, 271, 268, 265, 272, 269, 257, 236, 267, 251, 244, 255, 242, 260, 243, 300, 261, 225, 241, 271, 250, 250, 263, 287, 249, 269, 283, 245, 262, 244, 243, 244, 278, 229, 266, 266, 255, 250, 272, 240, 285, 252, 271, 256, 263, 251, 258, 248, 257, 252, 276, 275, 237, 264, 261, 225, 249, 246, 267, 244, 235, 230, 263, 254, 239, 244, 266, 264, 254, 290, 285, 257, 268, 260, 237, 244, 235, 271, 252, 262, 255, 237, 239, 261, 263, 243, 267, 261, 244, 241] payload_rows_per_dst=[5448, 5422, 5470, 5444, 5508, 5462, 5498, 5400] dup_rows_per_dst=[2842, 2682, 2782, 2776, 2720, 2716, 2742, 2624] combine_rows_per_src=[5448, 5422, 5470, 5444, 5508, 5462, 5498, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=230104 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157147864 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314065624 weight_movement_bytes=37748736 total_nvlink_bytes=351814360 weight_to_token_ratio=0.120194 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38148 dup_rows=21874 dup_rows_remote=19112 dedup_ratio=0.333771 router_tokens_per_expert=[227, 239, 247, 277, 245, 236, 283, 250, 246, 237, 277, 271, 242, 249, 226, 254, 257, 261, 293, 257, 251, 291, 271, 238, 276, 277, 255, 271, 257, 256, 250, 278, 226, 249, 281, 259, 284, 264, 264, 226, 267, 239, 255, 238, 258, 252, 249, 245, 266, 263, 245, 241, 237, 292, 266, 241, 285, 237, 270, 245, 214, 263, 259, 268, 284, 281, 250, 289, 271, 231, 265, 248, 242, 261, 237, 265, 240, 238, 247, 217, 265, 256, 290, 284, 255, 265, 264, 271, 249, 229, 244, 223, 274, 248, 272, 249, 246, 272, 259, 247, 239, 259, 242, 230, 271, 257, 262, 236, 272, 246, 283, 255, 255, 255, 282, 263, 267, 264, 257, 238, 264, 248, 252, 227, 263, 241, 255, 264] payload_rows_per_dst=[5394, 5550, 5386, 5450, 5452, 5514, 5528, 5388] dup_rows_per_dst=[2618, 2928, 2726, 2734, 2680, 2762, 2624, 2802] combine_rows_per_src=[5394, 5550, 5386, 5450, 5452, 5514, 5528, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38280 dup_rows=21840 dup_rows_remote=19018 dedup_ratio=0.333252 router_tokens_per_expert=[238, 270, 271, 260, 250, 231, 274, 274, 245, 261, 248, 257, 254, 248, 261, 249, 256, 257, 245, 268, 257, 253, 222, 230, 264, 275, 278, 294, 222, 246, 265, 287, 261, 249, 259, 251, 274, 236, 257, 253, 245, 235, 263, 271, 232, 269, 250, 266, 262, 255, 292, 274, 248, 270, 263, 238, 256, 249, 241, 256, 280, 258, 257, 255, 244, 254, 275, 257, 231, 260, 272, 234, 293, 279, 242, 233, 262, 271, 260, 220, 263, 237, 254, 247, 241, 264, 251, 259, 246, 250, 231, 306, 271, 270, 246, 255, 240, 242, 245, 251, 263, 264, 292, 232, 227, 276, 261, 223, 244, 247, 258, 239, 255, 256, 275, 255, 265, 238, 263, 257, 268, 261, 228, 269, 274, 252, 273, 262] payload_rows_per_dst=[5494, 5416, 5466, 5526, 5434, 5496, 5350, 5514] dup_rows_per_dst=[2688, 2822, 2676, 2782, 2740, 2686, 2658, 2788] combine_rows_per_src=[5494, 5416, 5466, 5526, 5434, 5496, 5350, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38124 dup_rows=21976 dup_rows_remote=19272 dedup_ratio=0.335327 router_tokens_per_expert=[266, 251, 234, 289, 275, 241, 243, 268, 252, 273, 243, 245, 239, 252, 261, 250, 242, 256, 251, 215, 270, 242, 248, 267, 296, 261, 258, 240, 245, 231, 262, 304, 253, 248, 253, 244, 267, 244, 224, 261, 235, 224, 249, 249, 252, 256, 277, 271, 272, 252, 297, 241, 256, 221, 264, 251, 252, 239, 255, 252, 264, 272, 244, 238, 238, 235, 237, 241, 276, 249, 255, 249, 262, 229, 256, 269, 285, 259, 264, 266, 234, 265, 257, 266, 276, 244, 243, 269, 243, 260, 237, 262, 240, 269, 250, 278, 268, 264, 260, 237, 271, 234, 254, 275, 258, 240, 267, 274, 259, 257, 267, 273, 263, 248, 284, 256, 262, 244, 264, 299, 267, 252, 253, 258, 251, 266, 274, 259] payload_rows_per_dst=[5446, 5410, 5316, 5436, 5452, 5410, 5516, 5574] dup_rows_per_dst=[2718, 2766, 2698, 2704, 2688, 2776, 2800, 2826] combine_rows_per_src=[5446, 5410, 5316, 5436, 5452, 5410, 5516, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38192 dup_rows=21858 dup_rows_remote=19138 dedup_ratio=0.333527 router_tokens_per_expert=[263, 254, 280, 274, 243, 244, 248, 260, 276, 286, 262, 243, 252, 292, 265, 219, 244, 251, 264, 261, 288, 271, 249, 241, 227, 265, 249, 238, 263, 252, 260, 257, 273, 239, 251, 240, 239, 271, 253, 236, 262, 248, 254, 242, 251, 265, 281, 245, 277, 249, 265, 280, 267, 288, 251, 264, 253, 250, 239, 251, 256, 237, 225, 276, 245, 262, 254, 238, 242, 292, 254, 267, 259, 273, 256, 261, 275, 246, 266, 231, 260, 273, 276, 271, 260, 269, 253, 217, 244, 298, 239, 244, 228, 240, 234, 248, 235, 258, 243, 258, 251, 251, 252, 276, 250, 250, 251, 277, 237, 274, 251, 289, 252, 249, 250, 283, 244, 260, 240, 278, 245, 277, 250, 239, 240, 266, 265, 233] payload_rows_per_dst=[5472, 5468, 5488, 5464, 5494, 5388, 5486, 5418] dup_rows_per_dst=[2850, 2692, 2612, 2792, 2748, 2720, 2720, 2724] combine_rows_per_src=[5472, 5468, 5488, 5464, 5494, 5388, 5486, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229136 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156622608 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313016080 weight_movement_bytes=18874368 total_nvlink_bytes=331890448 weight_to_token_ratio=0.060298 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38410 dup_rows=21818 dup_rows_remote=19090 dedup_ratio=0.332916 router_tokens_per_expert=[287, 260, 210, 241, 240, 248, 244, 265, 256, 282, 239, 248, 229, 258, 286, 250, 266, 258, 244, 267, 254, 252, 248, 258, 283, 239, 256, 237, 258, 253, 277, 242, 249, 264, 236, 299, 251, 248, 251, 216, 254, 234, 256, 272, 240, 224, 273, 257, 268, 276, 272, 273, 268, 251, 271, 264, 282, 249, 288, 230, 285, 261, 259, 265, 258, 263, 212, 269, 268, 267, 238, 240, 266, 264, 253, 269, 260, 285, 247, 273, 265, 254, 232, 232, 257, 269, 273, 255, 271, 283, 272, 255, 277, 253, 254, 268, 237, 269, 225, 258, 274, 247, 249, 251, 251, 253, 258, 242, 254, 272, 249, 238, 245, 248, 263, 229, 249, 267, 238, 229, 245, 244, 271, 252, 246, 252, 269, 271] payload_rows_per_dst=[5382, 5388, 5458, 5644, 5646, 5424, 5468, 5308] dup_rows_per_dst=[2526, 2706, 2590, 2970, 2796, 2656, 2846, 2728] combine_rows_per_src=[5382, 5388, 5458, 5644, 5646, 5424, 5468, 5308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38302 dup_rows=21778 dup_rows_remote=19088 dedup_ratio=0.332306 router_tokens_per_expert=[265, 250, 253, 253, 229, 242, 263, 275, 282, 220, 278, 253, 264, 222, 245, 263, 245, 246, 234, 243, 261, 257, 267, 256, 236, 242, 250, 227, 269, 237, 275, 260, 254, 283, 248, 262, 225, 247, 294, 263, 271, 251, 275, 272, 249, 248, 212, 269, 271, 264, 250, 254, 259, 300, 265, 239, 257, 260, 240, 250, 253, 252, 255, 260, 247, 265, 225, 245, 274, 256, 255, 288, 255, 243, 277, 254, 290, 286, 280, 263, 255, 275, 246, 271, 254, 273, 275, 273, 243, 248, 253, 270, 244, 255, 234, 242, 272, 272, 257, 241, 256, 262, 287, 255, 249, 243, 258, 236, 265, 260, 233, 273, 245, 279, 285, 242, 249, 259, 227, 245, 235, 250, 249, 259, 244, 232, 278, 243] payload_rows_per_dst=[5456, 5460, 5726, 5314, 5458, 5492, 5530, 5322] dup_rows_per_dst=[2690, 2720, 3120, 2394, 2728, 2730, 2822, 2574] combine_rows_per_src=[5456, 5460, 5726, 5314, 5458, 5492, 5530, 5322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229040 combine_payload_bytes=156254208 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156483248 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312737456 weight_movement_bytes=37748736 total_nvlink_bytes=350486192 weight_to_token_ratio=0.120704 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229192 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157024072 ingress_bytes=194543616 link_bytes=194543616 token_bytes=313818952 weight_movement_bytes=37748736 total_nvlink_bytes=351567688 weight_to_token_ratio=0.120288 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864021207e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229584 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156385488 ingress_bytes=193904640 link_bytes=193904640 token_bytes=312541392 weight_movement_bytes=37748736 total_nvlink_bytes=350290128 weight_to_token_ratio=0.120780 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857898853e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229320 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663752 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098184 weight_movement_bytes=18874368 total_nvlink_bytes=331972552 weight_to_token_ratio=0.060283 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=230000 combine_payload_bytes=157327360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157557360 ingress_bytes=195076096 link_bytes=195076096 token_bytes=314884720 weight_movement_bytes=37748736 total_nvlink_bytes=352633456 weight_to_token_ratio=0.119881 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869123168e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38272 dup_rows=21760 dup_rows_remote=19102 dedup_ratio=0.332031 router_tokens_per_expert=[228, 280, 269, 262, 282, 262, 252, 239, 266, 256, 234, 294, 280, 265, 260, 260, 247, 283, 252, 284, 267, 285, 276, 275, 254, 234, 237, 259, 232, 249, 236, 226, 224, 261, 248, 258, 259, 268, 222, 277, 271, 263, 292, 269, 282, 225, 241, 226, 252, 246, 280, 237, 239, 266, 263, 234, 265, 254, 247, 238, 284, 261, 258, 245, 240, 226, 256, 258, 249, 258, 283, 259, 265, 247, 231, 260, 262, 249, 264, 274, 246, 273, 233, 247, 278, 259, 280, 278, 264, 264, 260, 276, 257, 262, 245, 250, 286, 255, 237, 231, 262, 253, 235, 255, 299, 253, 272, 250, 249, 242, 241, 250, 248, 276, 223, 233, 256, 267, 217, 270, 246, 258, 243, 243, 234, 269, 279, 243] payload_rows_per_dst=[5552, 5422, 5428, 5458, 5504, 5570, 5434, 5408] dup_rows_per_dst=[2826, 2770, 2744, 2680, 2658, 2774, 2706, 2602] combine_rows_per_src=[5552, 5422, 5428, 5458, 5504, 5570, 5434, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229560 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114552 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999544 weight_movement_bytes=18874368 total_nvlink_bytes=332873912 weight_to_token_ratio=0.060110 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38384 dup_rows=21698 dup_rows_remote=19006 dedup_ratio=0.331085 router_tokens_per_expert=[245, 256, 255, 227, 258, 265, 247, 243, 244, 272, 262, 254, 249, 258, 252, 229, 238, 259, 266, 264, 283, 267, 259, 249, 260, 256, 244, 257, 298, 260, 276, 277, 269, 269, 244, 259, 281, 244, 265, 252, 255, 220, 258, 245, 257, 250, 255, 250, 246, 255, 278, 248, 247, 230, 270, 280, 228, 240, 272, 273, 262, 266, 252, 255, 253, 277, 274, 270, 281, 272, 247, 239, 262, 252, 253, 264, 248, 256, 251, 231, 244, 291, 253, 267, 262, 240, 264, 232, 248, 251, 268, 225, 272, 244, 261, 255, 262, 248, 276, 261, 232, 233, 247, 257, 242, 266, 274, 246, 250, 225, 264, 256, 235, 243, 257, 259, 248, 277, 242, 271, 292, 245, 278, 238, 230, 273, 252, 278] payload_rows_per_dst=[5446, 5596, 5454, 5510, 5484, 5460, 5400, 5488] dup_rows_per_dst=[2586, 2830, 2692, 2694, 2776, 2694, 2678, 2748] combine_rows_per_src=[5446, 5596, 5454, 5510, 5484, 5460, 5400, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229496 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991608 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753720 weight_movement_bytes=18874368 total_nvlink_bytes=332628088 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229560 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157450424 ingress_bytes=213843968 link_bytes=213843968 token_bytes=314671288 weight_movement_bytes=56623104 total_nvlink_bytes=371294392 weight_to_token_ratio=0.179944 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048947684e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38380 dup_rows=21732 dup_rows_remote=19066 dedup_ratio=0.331604 router_tokens_per_expert=[247, 256, 271, 216, 245, 262, 253, 274, 275, 256, 269, 228, 242, 244, 271, 248, 251, 247, 245, 230, 247, 247, 278, 260, 277, 227, 255, 261, 224, 260, 251, 285, 244, 246, 254, 241, 279, 246, 253, 248, 265, 264, 246, 243, 255, 256, 236, 283, 228, 265, 247, 262, 241, 236, 247, 262, 253, 260, 228, 263, 280, 273, 256, 232, 251, 272, 260, 259, 250, 282, 241, 243, 286, 238, 248, 257, 292, 263, 256, 276, 254, 270, 244, 258, 260, 258, 249, 246, 283, 257, 253, 261, 238, 259, 268, 265, 254, 274, 247, 252, 267, 289, 250, 262, 270, 231, 273, 285, 266, 234, 250, 252, 237, 258, 242, 263, 271, 279, 240, 258, 241, 250, 245, 289, 250, 277, 264, 257] payload_rows_per_dst=[5484, 5424, 5486, 5406, 5538, 5494, 5444, 5528] dup_rows_per_dst=[2630, 2666, 2632, 2660, 2810, 2752, 2868, 2714] combine_rows_per_src=[5484, 5424, 5486, 5406, 5538, 5494, 5444, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38414 dup_rows=21698 dup_rows_remote=19044 dedup_ratio=0.331085 router_tokens_per_expert=[272, 229, 270, 257, 258, 227, 264, 233, 268, 244, 239, 250, 253, 253, 239, 266, 270, 249, 283, 264, 250, 255, 273, 245, 274, 258, 236, 260, 236, 299, 296, 239, 237, 266, 273, 219, 272, 275, 258, 261, 232, 241, 234, 254, 260, 270, 251, 270, 260, 245, 270, 248, 280, 247, 263, 252, 237, 281, 252, 242, 275, 250, 272, 273, 258, 242, 237, 236, 264, 252, 278, 228, 258, 283, 229, 259, 244, 287, 245, 247, 279, 263, 222, 261, 276, 244, 246, 244, 253, 244, 267, 247, 242, 256, 229, 266, 261, 290, 298, 261, 226, 248, 269, 263, 239, 252, 243, 245, 237, 263, 234, 272, 272, 252, 281, 280, 259, 275, 241, 254, 254, 250, 248, 239, 249, 290, 247, 261] payload_rows_per_dst=[5410, 5556, 5426, 5526, 5478, 5424, 5486, 5532] dup_rows_per_dst=[2634, 2818, 2720, 2768, 2616, 2654, 2716, 2772] combine_rows_per_src=[5410, 5556, 5426, 5526, 5478, 5424, 5486, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38430 dup_rows=21722 dup_rows_remote=19052 dedup_ratio=0.331451 router_tokens_per_expert=[277, 274, 268, 266, 248, 282, 246, 253, 279, 259, 256, 254, 254, 227, 246, 274, 256, 229, 258, 263, 257, 238, 261, 270, 284, 236, 235, 261, 245, 242, 240, 255, 267, 252, 242, 272, 260, 268, 289, 267, 252, 259, 253, 253, 263, 248, 266, 259, 247, 239, 232, 248, 273, 250, 268, 264, 243, 243, 266, 220, 272, 235, 256, 271, 237, 250, 272, 240, 259, 256, 257, 253, 255, 263, 284, 246, 258, 246, 227, 256, 258, 243, 257, 254, 262, 264, 251, 249, 275, 261, 256, 240, 245, 249, 250, 263, 268, 257, 253, 270, 277, 268, 261, 267, 249, 247, 246, 249, 258, 254, 269, 233, 269, 250, 239, 249, 276, 273, 243, 267, 233, 270, 263, 237, 247, 294, 236, 270] payload_rows_per_dst=[5554, 5358, 5542, 5384, 5484, 5466, 5508, 5518] dup_rows_per_dst=[2772, 2702, 2798, 2670, 2634, 2688, 2744, 2714] combine_rows_per_src=[5554, 5358, 5542, 5384, 5484, 5466, 5508, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38422 dup_rows=21654 dup_rows_remote=18948 dedup_ratio=0.330414 router_tokens_per_expert=[261, 246, 254, 255, 268, 253, 261, 251, 261, 269, 269, 238, 252, 276, 283, 262, 289, 243, 277, 238, 233, 243, 255, 259, 262, 265, 248, 231, 252, 252, 252, 259, 285, 244, 224, 232, 233, 221, 243, 252, 251, 233, 256, 280, 260, 269, 262, 256, 242, 289, 268, 275, 255, 289, 253, 271, 254, 273, 236, 238, 244, 244, 225, 263, 248, 256, 254, 267, 243, 261, 250, 255, 271, 300, 262, 267, 258, 234, 267, 260, 238, 255, 254, 243, 243, 263, 276, 249, 260, 254, 250, 241, 269, 255, 227, 263, 268, 252, 260, 245, 235, 253, 270, 271, 255, 242, 253, 282, 234, 258, 252, 253, 255, 265, 256, 219, 275, 275, 251, 249, 270, 259, 252, 271, 259, 252, 277, 270] payload_rows_per_dst=[5558, 5456, 5426, 5458, 5550, 5468, 5460, 5506] dup_rows_per_dst=[2760, 2660, 2576, 2780, 2756, 2612, 2706, 2804] combine_rows_per_src=[5558, 5456, 5426, 5458, 5550, 5468, 5460, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38166 dup_rows=21954 dup_rows_remote=19182 dedup_ratio=0.334991 router_tokens_per_expert=[251, 268, 262, 235, 265, 256, 268, 264, 274, 256, 264, 277, 245, 252, 260, 270, 271, 263, 259, 276, 226, 251, 231, 233, 247, 263, 250, 253, 246, 247, 257, 233, 257, 290, 250, 286, 243, 231, 276, 257, 251, 265, 261, 245, 255, 240, 240, 290, 299, 231, 253, 285, 268, 247, 257, 234, 257, 252, 259, 254, 239, 254, 283, 273, 273, 263, 234, 260, 258, 270, 270, 254, 219, 247, 239, 241, 251, 266, 270, 270, 268, 243, 245, 268, 254, 246, 253, 260, 254, 268, 264, 234, 286, 223, 267, 262, 257, 254, 256, 274, 263, 232, 247, 249, 259, 256, 268, 235, 264, 254, 259, 242, 299, 259, 257, 232, 250, 251, 243, 245, 251, 246, 264, 255, 269, 227, 254, 262] payload_rows_per_dst=[5456, 5350, 5452, 5484, 5532, 5430, 5462, 5416] dup_rows_per_dst=[2812, 2712, 2744, 2806, 2672, 2760, 2676, 2772] combine_rows_per_src=[5456, 5350, 5452, 5484, 5532, 5430, 5462, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38102 dup_rows=21766 dup_rows_remote=18664 dedup_ratio=0.332123 router_tokens_per_expert=[270, 240, 241, 252, 276, 276, 260, 256, 257, 248, 237, 271, 251, 241, 255, 271, 265, 259, 227, 266, 275, 251, 252, 239, 261, 245, 261, 267, 255, 264, 236, 252, 263, 237, 243, 243, 259, 244, 259, 245, 246, 218, 254, 254, 261, 248, 256, 273, 266, 234, 273, 238, 253, 266, 268, 266, 275, 284, 264, 244, 244, 275, 275, 265, 234, 236, 231, 283, 264, 264, 247, 266, 258, 270, 255, 258, 273, 236, 265, 238, 250, 255, 270, 259, 242, 230, 248, 260, 262, 272, 273, 259, 247, 288, 277, 260, 234, 240, 270, 255, 259, 240, 242, 251, 272, 269, 246, 249, 266, 265, 251, 280, 247, 284, 255, 242, 281, 278, 261, 249, 258, 249, 250, 253, 238, 247, 238, 249] payload_rows_per_dst=[5668, 5552, 5524, 5392, 5254, 5532, 5392, 5456] dup_rows_per_dst=[3102, 2918, 2890, 2582, 2334, 2772, 2558, 2610] combine_rows_per_src=[5668, 5552, 5524, 5392, 5254, 5532, 5392, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229784 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434264 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638744 weight_movement_bytes=18874368 total_nvlink_bytes=333513112 weight_to_token_ratio=0.059987 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=229832 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157573576 ingress_bytes=213966848 link_bytes=213966848 token_bytes=314917320 weight_movement_bytes=56623104 total_nvlink_bytes=371540424 weight_to_token_ratio=0.179803 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050125059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229928 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157639208 ingress_bytes=176283648 link_bytes=176283648 token_bytes=315048488 weight_movement_bytes=18874368 total_nvlink_bytes=333922856 weight_to_token_ratio=0.059909 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689063178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229480 combine_payload_bytes=157376512 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157605992 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982504 weight_movement_bytes=18874368 total_nvlink_bytes=333856872 weight_to_token_ratio=0.059922 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=229392 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156557328 ingress_bytes=194076672 link_bytes=194076672 token_bytes=312885264 weight_movement_bytes=37748736 total_nvlink_bytes=350634000 weight_to_token_ratio=0.120647 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859547179e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38408 dup_rows=21632 dup_rows_remote=18992 dedup_ratio=0.330078 router_tokens_per_expert=[253, 257, 256, 269, 246, 284, 264, 248, 258, 245, 254, 270, 251, 253, 273, 258, 242, 272, 263, 274, 255, 241, 226, 246, 277, 257, 274, 258, 258, 270, 277, 260, 279, 232, 263, 243, 232, 236, 279, 261, 282, 237, 275, 252, 245, 273, 255, 243, 262, 252, 258, 261, 288, 264, 269, 226, 248, 239, 290, 258, 220, 215, 229, 251, 250, 259, 258, 255, 221, 269, 250, 259, 244, 250, 254, 249, 252, 250, 278, 270, 254, 250, 276, 259, 240, 278, 266, 253, 250, 261, 263, 241, 241, 255, 238, 245, 253, 283, 289, 251, 261, 271, 264, 240, 240, 249, 256, 218, 274, 241, 269, 247, 285, 281, 264, 258, 220, 229, 269, 281, 260, 263, 237, 237, 236, 252, 268, 278] payload_rows_per_dst=[5516, 5460, 5540, 5390, 5496, 5472, 5514, 5516] dup_rows_per_dst=[2762, 2840, 2634, 2670, 2640, 2668, 2698, 2720] combine_rows_per_src=[5516, 5460, 5540, 5390, 5496, 5472, 5514, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=227064 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156292856 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312358648 weight_movement_bytes=18874368 total_nvlink_bytes=331233016 weight_to_token_ratio=0.060425 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38330 dup_rows=21818 dup_rows_remote=19044 dedup_ratio=0.332916 router_tokens_per_expert=[272, 280, 266, 241, 241, 246, 249, 272, 236, 230, 232, 256, 257, 241, 262, 271, 254, 259, 242, 270, 267, 265, 253, 249, 252, 257, 252, 237, 238, 244, 256, 274, 238, 215, 269, 271, 277, 264, 269, 252, 259, 247, 287, 253, 242, 262, 238, 238, 268, 239, 245, 256, 238, 289, 270, 232, 279, 265, 273, 292, 264, 247, 236, 267, 291, 264, 240, 245, 273, 257, 238, 244, 274, 252, 234, 228, 260, 246, 278, 246, 244, 300, 268, 254, 275, 272, 262, 263, 254, 251, 242, 244, 283, 261, 252, 254, 261, 234, 245, 250, 252, 252, 238, 272, 311, 304, 265, 256, 237, 271, 274, 249, 272, 276, 282, 247, 245, 228, 225, 242, 251, 263, 255, 233, 253, 233, 247, 234] payload_rows_per_dst=[5444, 5446, 5388, 5482, 5466, 5570, 5546, 5376] dup_rows_per_dst=[2660, 2692, 2774, 2838, 2674, 2788, 2796, 2596] combine_rows_per_src=[5444, 5446, 5388, 5482, 5466, 5570, 5546, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229600 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157548768 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314867936 weight_movement_bytes=37748736 total_nvlink_bytes=352616672 weight_to_token_ratio=0.119888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229496 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157229176 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228856 weight_movement_bytes=18874368 total_nvlink_bytes=333103224 weight_to_token_ratio=0.060066 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38338 dup_rows=21748 dup_rows_remote=19050 dedup_ratio=0.331848 router_tokens_per_expert=[240, 227, 253, 275, 271, 248, 259, 243, 257, 252, 259, 243, 257, 265, 271, 243, 266, 239, 254, 261, 258, 261, 230, 285, 248, 274, 230, 257, 266, 250, 243, 252, 281, 239, 240, 253, 277, 273, 275, 266, 233, 261, 263, 258, 259, 256, 259, 224, 229, 268, 225, 243, 244, 244, 262, 241, 264, 281, 256, 271, 244, 261, 262, 262, 252, 247, 268, 256, 272, 272, 233, 260, 226, 264, 285, 265, 246, 251, 264, 244, 284, 256, 262, 278, 261, 283, 241, 244, 257, 273, 230, 247, 293, 257, 260, 249, 263, 240, 275, 229, 276, 246, 251, 255, 254, 294, 253, 255, 236, 261, 263, 247, 261, 274, 255, 267, 235, 270, 249, 258, 248, 242, 266, 273, 259, 222, 249, 251] payload_rows_per_dst=[5454, 5450, 5476, 5476, 5516, 5504, 5472, 5440] dup_rows_per_dst=[2672, 2698, 2758, 2638, 2694, 2846, 2724, 2718] combine_rows_per_src=[5454, 5450, 5476, 5476, 5516, 5504, 5472, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38418 dup_rows=21664 dup_rows_remote=19046 dedup_ratio=0.330566 router_tokens_per_expert=[258, 250, 278, 231, 271, 231, 252, 274, 242, 248, 261, 237, 289, 243, 250, 252, 262, 235, 281, 276, 241, 257, 291, 255, 251, 261, 275, 254, 251, 265, 258, 268, 262, 259, 248, 250, 274, 234, 290, 277, 277, 261, 275, 247, 232, 254, 249, 271, 269, 267, 272, 254, 292, 235, 275, 254, 244, 244, 250, 224, 230, 281, 268, 268, 225, 251, 235, 268, 253, 264, 271, 254, 255, 248, 257, 282, 275, 260, 232, 260, 238, 269, 259, 268, 236, 270, 242, 254, 275, 245, 258, 244, 246, 233, 263, 236, 254, 258, 246, 269, 236, 238, 260, 266, 250, 266, 258, 250, 259, 246, 255, 248, 265, 245, 261, 279, 223, 254, 245, 262, 249, 235, 279, 253, 257, 233, 246, 262] payload_rows_per_dst=[5464, 5544, 5530, 5528, 5426, 5454, 5512, 5414] dup_rows_per_dst=[2670, 2818, 2790, 2726, 2754, 2618, 2606, 2682] combine_rows_per_src=[5464, 5544, 5530, 5528, 5426, 5454, 5512, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38136 dup_rows=21806 dup_rows_remote=18726 dedup_ratio=0.332733 router_tokens_per_expert=[278, 245, 243, 264, 255, 261, 245, 243, 259, 234, 243, 255, 281, 275, 240, 248, 227, 255, 266, 262, 264, 283, 257, 242, 256, 275, 250, 235, 233, 279, 240, 252, 222, 265, 247, 241, 268, 277, 237, 252, 240, 248, 247, 276, 243, 257, 266, 272, 257, 260, 263, 233, 249, 262, 271, 270, 254, 266, 241, 258, 243, 230, 250, 264, 257, 269, 264, 229, 263, 257, 260, 245, 241, 267, 239, 249, 220, 240, 261, 254, 260, 265, 249, 267, 279, 259, 256, 270, 247, 276, 237, 232, 243, 277, 271, 253, 274, 270, 240, 275, 252, 260, 261, 263, 276, 264, 258, 272, 268, 244, 237, 277, 278, 255, 301, 268, 253, 263, 234, 261, 259, 262, 244, 218, 234, 289, 268, 260] payload_rows_per_dst=[5494, 5466, 5400, 5468, 5420, 5502, 5386, 5594] dup_rows_per_dst=[2644, 2686, 2716, 2674, 2610, 2780, 2616, 3080] combine_rows_per_src=[5494, 5466, 5400, 5468, 5420, 5502, 5386, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38406 dup_rows=21744 dup_rows_remote=19060 dedup_ratio=0.331787 router_tokens_per_expert=[255, 265, 282, 252, 268, 248, 238, 272, 260, 250, 264, 247, 258, 241, 257, 245, 256, 252, 251, 246, 249, 237, 255, 251, 252, 265, 244, 240, 266, 244, 274, 281, 276, 236, 248, 262, 275, 239, 254, 236, 279, 285, 261, 242, 237, 254, 265, 273, 216, 266, 262, 247, 291, 246, 251, 236, 256, 252, 253, 266, 253, 239, 243, 258, 257, 249, 274, 276, 261, 268, 271, 244, 287, 284, 250, 239, 254, 265, 230, 280, 267, 270, 266, 272, 262, 276, 236, 249, 251, 276, 252, 245, 293, 254, 255, 235, 250, 266, 238, 278, 256, 263, 240, 234, 271, 265, 248, 266, 237, 243, 242, 233, 245, 260, 278, 239, 242, 264, 250, 229, 247, 272, 234, 272, 263, 274, 267, 232] payload_rows_per_dst=[5518, 5430, 5502, 5386, 5564, 5504, 5442, 5446] dup_rows_per_dst=[2686, 2696, 2742, 2684, 2814, 2814, 2618, 2690] combine_rows_per_src=[5518, 5430, 5502, 5386, 5564, 5504, 5442, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38228 dup_rows=21766 dup_rows_remote=18996 dedup_ratio=0.332123 router_tokens_per_expert=[254, 268, 239, 269, 264, 262, 273, 250, 255, 293, 257, 237, 263, 255, 241, 264, 239, 274, 245, 265, 265, 264, 269, 262, 251, 244, 263, 235, 278, 264, 268, 276, 246, 237, 259, 234, 236, 265, 278, 259, 254, 247, 258, 278, 256, 272, 258, 259, 286, 245, 262, 248, 264, 253, 257, 223, 234, 270, 277, 276, 243, 255, 249, 256, 254, 247, 242, 241, 248, 244, 252, 273, 262, 250, 271, 262, 251, 249, 253, 216, 219, 247, 257, 260, 254, 278, 250, 257, 250, 231, 258, 256, 262, 282, 243, 282, 260, 283, 260, 240, 244, 251, 263, 252, 259, 268, 259, 283, 264, 267, 241, 262, 248, 265, 264, 238, 253, 274, 223, 235, 243, 255, 256, 243, 259, 247, 244, 264] payload_rows_per_dst=[5444, 5508, 5514, 5454, 5406, 5484, 5542, 5418] dup_rows_per_dst=[2844, 2816, 2678, 2742, 2624, 2688, 2770, 2604] combine_rows_per_src=[5444, 5508, 5514, 5454, 5406, 5484, 5542, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38090 dup_rows=21810 dup_rows_remote=18850 dedup_ratio=0.332794 router_tokens_per_expert=[263, 242, 256, 268, 249, 261, 263, 265, 276, 271, 267, 247, 237, 272, 254, 231, 247, 242, 284, 261, 258, 216, 231, 269, 266, 265, 254, 253, 278, 259, 273, 245, 290, 256, 278, 253, 270, 240, 262, 246, 256, 255, 255, 264, 230, 249, 259, 259, 236, 234, 275, 269, 249, 244, 236, 245, 240, 273, 245, 227, 271, 232, 261, 259, 277, 249, 260, 276, 264, 245, 256, 264, 271, 236, 271, 246, 267, 250, 247, 240, 265, 269, 299, 250, 250, 252, 273, 280, 248, 263, 227, 266, 226, 276, 240, 247, 241, 257, 228, 256, 244, 272, 245, 261, 253, 261, 258, 219, 250, 245, 243, 276, 264, 272, 260, 276, 256, 258, 251, 268, 260, 227, 248, 296, 271, 245, 260, 256] payload_rows_per_dst=[5636, 5388, 5478, 5374, 5478, 5482, 5434, 5456] dup_rows_per_dst=[2960, 2488, 2848, 2664, 2684, 2774, 2584, 2808] combine_rows_per_src=[5636, 5388, 5478, 5374, 5478, 5482, 5434, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229552 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157262000 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294448 weight_movement_bytes=18874368 total_nvlink_bytes=333168816 weight_to_token_ratio=0.060053 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229856 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589984 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314950112 weight_movement_bytes=18874368 total_nvlink_bytes=333824480 weight_to_token_ratio=0.059928 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=227448 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156432504 ingress_bytes=175079424 link_bytes=175079424 token_bytes=312637560 weight_movement_bytes=18874368 total_nvlink_bytes=331511928 weight_to_token_ratio=0.060371 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677524896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229864 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157540840 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314851816 weight_movement_bytes=18874368 total_nvlink_bytes=333726184 weight_to_token_ratio=0.059947 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38170 dup_rows=21792 dup_rows_remote=19128 dedup_ratio=0.332520 router_tokens_per_expert=[237, 232, 278, 263, 240, 240, 238, 264, 250, 277, 245, 286, 266, 236, 251, 263, 264, 235, 277, 251, 229, 254, 260, 243, 270, 267, 239, 258, 259, 252, 248, 232, 261, 290, 242, 232, 244, 272, 238, 249, 260, 258, 265, 242, 259, 215, 257, 238, 265, 252, 257, 257, 246, 279, 241, 274, 261, 236, 285, 258, 273, 254, 265, 224, 246, 247, 243, 262, 271, 273, 269, 270, 245, 239, 283, 260, 242, 254, 243, 272, 276, 265, 269, 238, 249, 264, 269, 244, 267, 256, 270, 284, 271, 264, 286, 257, 278, 242, 243, 250, 283, 277, 236, 236, 245, 264, 234, 270, 254, 258, 234, 274, 280, 230, 233, 246, 262, 268, 255, 266, 276, 244, 236, 262, 269, 250, 246, 266] payload_rows_per_dst=[5474, 5416, 5372, 5430, 5574, 5574, 5426, 5478] dup_rows_per_dst=[2658, 2660, 2672, 2824, 2664, 2884, 2730, 2700] combine_rows_per_src=[5474, 5416, 5372, 5430, 5574, 5574, 5426, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=228896 combine_payload_bytes=156581888 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156810784 ingress_bytes=213204992 link_bytes=213204992 token_bytes=313392672 weight_movement_bytes=56623104 total_nvlink_bytes=370015776 weight_to_token_ratio=0.180678 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042825330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=227760 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156244400 ingress_bytes=174891008 link_bytes=174891008 token_bytes=312261040 weight_movement_bytes=18874368 total_nvlink_bytes=331135408 weight_to_token_ratio=0.060444 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675719587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38298 dup_rows=21806 dup_rows_remote=19096 dedup_ratio=0.332733 router_tokens_per_expert=[274, 265, 260, 256, 270, 214, 260, 248, 267, 244, 260, 239, 284, 276, 243, 288, 223, 267, 258, 238, 272, 249, 241, 239, 246, 275, 256, 270, 227, 268, 225, 237, 254, 253, 241, 266, 257, 263, 260, 257, 247, 249, 253, 274, 232, 266, 266, 233, 281, 273, 267, 261, 249, 294, 281, 270, 267, 240, 258, 263, 250, 232, 242, 258, 254, 286, 252, 225, 265, 241, 247, 271, 259, 258, 267, 282, 246, 241, 265, 243, 235, 245, 261, 262, 244, 259, 290, 251, 266, 247, 279, 255, 269, 254, 262, 237, 277, 237, 263, 259, 258, 254, 228, 255, 241, 238, 222, 267, 248, 254, 282, 297, 255, 249, 236, 246, 256, 253, 218, 269, 285, 274, 244, 248, 256, 269, 260, 256] payload_rows_per_dst=[5466, 5450, 5432, 5562, 5476, 5484, 5454, 5406] dup_rows_per_dst=[2830, 2532, 2710, 2810, 2728, 2748, 2706, 2742] combine_rows_per_src=[5466, 5450, 5432, 5562, 5476, 5484, 5454, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229192 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573512 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917832 weight_movement_bytes=18874368 total_nvlink_bytes=331792200 weight_to_token_ratio=0.060317 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229576 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098184 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966792 weight_movement_bytes=18874368 total_nvlink_bytes=332841160 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38210 dup_rows=21832 dup_rows_remote=19042 dedup_ratio=0.333130 router_tokens_per_expert=[248, 257, 302, 276, 271, 251, 253, 272, 253, 238, 281, 269, 247, 258, 257, 265, 261, 250, 272, 256, 248, 256, 263, 255, 263, 263, 251, 258, 231, 300, 249, 266, 253, 256, 229, 269, 226, 236, 244, 273, 268, 265, 268, 239, 239, 223, 256, 242, 269, 230, 261, 232, 285, 259, 247, 265, 259, 250, 247, 248, 251, 271, 261, 246, 257, 265, 252, 246, 248, 241, 244, 252, 278, 261, 275, 277, 258, 252, 258, 265, 260, 231, 256, 240, 269, 272, 247, 256, 272, 260, 265, 252, 241, 253, 266, 251, 252, 237, 248, 272, 230, 262, 245, 250, 283, 247, 265, 278, 271, 251, 257, 254, 251, 227, 242, 244, 249, 265, 262, 268, 228, 265, 244, 246, 292, 251, 255, 250] payload_rows_per_dst=[5554, 5494, 5420, 5420, 5494, 5436, 5484, 5402] dup_rows_per_dst=[2842, 2790, 2552, 2742, 2764, 2746, 2720, 2676] combine_rows_per_src=[5554, 5494, 5420, 5420, 5494, 5436, 5484, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38106 dup_rows=21910 dup_rows_remote=19232 dedup_ratio=0.334320 router_tokens_per_expert=[261, 232, 280, 224, 291, 248, 246, 247, 256, 260, 253, 272, 236, 263, 237, 262, 273, 245, 231, 241, 261, 249, 248, 259, 257, 277, 249, 274, 290, 292, 230, 262, 261, 256, 279, 260, 255, 258, 245, 283, 248, 255, 267, 242, 262, 276, 234, 244, 244, 247, 266, 224, 279, 269, 229, 249, 269, 244, 241, 243, 271, 255, 236, 289, 253, 264, 260, 259, 262, 266, 266, 268, 247, 249, 249, 256, 255, 236, 242, 250, 235, 246, 257, 247, 232, 257, 274, 234, 283, 274, 246, 273, 269, 268, 258, 246, 249, 253, 267, 240, 248, 268, 226, 268, 286, 272, 258, 245, 253, 279, 248, 249, 255, 257, 236, 256, 276, 246, 260, 263, 232, 288, 259, 263, 253, 212, 278, 258] payload_rows_per_dst=[5384, 5488, 5492, 5470, 5418, 5520, 5454, 5400] dup_rows_per_dst=[2752, 2788, 2758, 2640, 2746, 2678, 2764, 2784] combine_rows_per_src=[5384, 5488, 5492, 5470, 5418, 5520, 5454, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38412 dup_rows=21660 dup_rows_remote=18896 dedup_ratio=0.330505 router_tokens_per_expert=[234, 255, 257, 271, 263, 255, 232, 247, 252, 257, 252, 262, 242, 249, 266, 248, 260, 254, 259, 240, 255, 248, 280, 237, 273, 274, 256, 254, 253, 257, 248, 253, 271, 257, 254, 256, 271, 262, 258, 246, 276, 249, 226, 264, 265, 268, 260, 242, 253, 272, 274, 247, 265, 258, 240, 272, 233, 286, 248, 247, 245, 252, 236, 263, 266, 262, 246, 257, 283, 240, 245, 276, 256, 284, 255, 253, 259, 255, 260, 268, 250, 229, 259, 272, 268, 283, 259, 260, 209, 228, 229, 247, 264, 268, 247, 262, 270, 235, 243, 260, 247, 250, 268, 259, 264, 247, 259, 265, 251, 259, 267, 253, 262, 256, 266, 276, 263, 248, 257, 245, 254, 252, 250, 259, 254, 260, 252, 259] payload_rows_per_dst=[5276, 5426, 5744, 5392, 5470, 5518, 5586, 5464] dup_rows_per_dst=[2298, 2760, 3016, 2642, 2700, 2724, 2756, 2764] combine_rows_per_src=[5276, 5426, 5744, 5392, 5470, 5518, 5586, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38360 dup_rows=21686 dup_rows_remote=18996 dedup_ratio=0.330902 router_tokens_per_expert=[242, 242, 244, 258, 266, 274, 245, 252, 280, 255, 268, 279, 256, 235, 288, 263, 229, 275, 288, 284, 252, 258, 280, 265, 247, 284, 263, 286, 247, 256, 244, 264, 249, 227, 240, 251, 260, 241, 269, 268, 251, 244, 251, 271, 267, 237, 242, 230, 282, 258, 246, 276, 263, 254, 246, 257, 230, 242, 255, 261, 252, 263, 243, 262, 248, 259, 276, 254, 279, 226, 236, 274, 250, 252, 259, 236, 239, 227, 216, 254, 283, 258, 244, 247, 253, 261, 251, 258, 250, 255, 237, 233, 263, 238, 272, 249, 255, 240, 275, 238, 264, 259, 226, 267, 271, 259, 268, 251, 260, 289, 241, 235, 289, 268, 266, 238, 233, 257, 261, 269, 265, 282, 287, 228, 243, 259, 261, 270] payload_rows_per_dst=[5478, 5642, 5434, 5490, 5372, 5444, 5452, 5538] dup_rows_per_dst=[2816, 2802, 2562, 2690, 2598, 2660, 2744, 2814] combine_rows_per_src=[5478, 5642, 5434, 5490, 5372, 5444, 5452, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38338 dup_rows=21836 dup_rows_remote=19178 dedup_ratio=0.333191 router_tokens_per_expert=[237, 284, 267, 259, 229, 259, 259, 296, 240, 261, 254, 266, 260, 232, 269, 245, 240, 276, 246, 241, 244, 275, 246, 272, 244, 237, 243, 267, 265, 236, 240, 246, 255, 243, 238, 231, 267, 269, 252, 238, 262, 269, 245, 261, 264, 262, 254, 254, 284, 274, 251, 284, 260, 249, 237, 262, 262, 235, 268, 245, 227, 242, 258, 275, 257, 241, 269, 268, 267, 259, 256, 306, 245, 247, 235, 253, 272, 251, 239, 257, 266, 290, 271, 254, 248, 245, 274, 277, 259, 249, 267, 277, 259, 268, 238, 255, 262, 287, 262, 245, 237, 236, 242, 253, 255, 231, 245, 257, 256, 241, 253, 248, 246, 242, 251, 263, 259, 263, 279, 275, 267, 233, 242, 278, 268, 274, 256, 231] payload_rows_per_dst=[5510, 5414, 5430, 5510, 5474, 5524, 5362, 5476] dup_rows_per_dst=[2688, 2622, 2734, 2716, 2770, 2870, 2658, 2778] combine_rows_per_src=[5510, 5414, 5430, 5510, 5474, 5524, 5362, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38276 dup_rows=21752 dup_rows_remote=19012 dedup_ratio=0.331909 router_tokens_per_expert=[265, 251, 257, 268, 235, 225, 240, 296, 277, 258, 259, 271, 249, 255, 253, 244, 288, 262, 252, 252, 245, 241, 246, 235, 264, 276, 273, 233, 242, 262, 270, 251, 286, 261, 222, 254, 272, 256, 270, 243, 247, 246, 266, 261, 233, 276, 240, 250, 253, 246, 263, 249, 241, 266, 237, 235, 286, 235, 249, 244, 249, 281, 246, 251, 232, 263, 231, 267, 226, 240, 249, 258, 265, 235, 249, 259, 253, 274, 261, 265, 257, 225, 276, 255, 250, 253, 289, 248, 243, 281, 269, 252, 288, 288, 282, 249, 294, 236, 258, 267, 264, 229, 267, 232, 247, 244, 275, 256, 263, 243, 220, 260, 266, 276, 259, 227, 275, 259, 252, 281, 242, 277, 237, 275, 263, 249, 254, 280] payload_rows_per_dst=[5508, 5482, 5614, 5218, 5478, 5532, 5424, 5528] dup_rows_per_dst=[2740, 2716, 3048, 2348, 2576, 2848, 2686, 2790] combine_rows_per_src=[5508, 5482, 5614, 5218, 5478, 5532, 5424, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229008 combine_payload_bytes=156508160 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156737168 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245328 weight_movement_bytes=37748736 total_nvlink_bytes=350994064 weight_to_token_ratio=0.120509 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=229352 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156311528 ingress_bytes=250454016 link_bytes=250454016 token_bytes=312393704 weight_movement_bytes=94371840 total_nvlink_bytes=406765544 weight_to_token_ratio=0.302093 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399727150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229232 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157564784 ingress_bytes=195084288 link_bytes=195084288 token_bytes=314900336 weight_movement_bytes=37748736 total_nvlink_bytes=352649072 weight_to_token_ratio=0.119875 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869201660e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38252 dup_rows=21826 dup_rows_remote=19114 dedup_ratio=0.333038 router_tokens_per_expert=[278, 252, 240, 264, 225, 272, 236, 275, 264, 219, 275, 258, 256, 253, 241, 245, 258, 244, 270, 253, 268, 244, 254, 274, 242, 233, 267, 249, 290, 257, 278, 259, 245, 255, 275, 250, 265, 284, 280, 265, 260, 281, 257, 256, 267, 246, 246, 245, 270, 298, 235, 261, 239, 280, 251, 265, 261, 262, 252, 249, 239, 260, 237, 231, 262, 243, 269, 239, 270, 240, 269, 246, 260, 294, 263, 292, 246, 211, 244, 237, 243, 248, 272, 258, 237, 236, 247, 270, 255, 243, 237, 269, 281, 277, 256, 264, 265, 267, 263, 245, 244, 232, 259, 245, 269, 268, 251, 247, 274, 253, 245, 237, 258, 262, 265, 257, 261, 264, 258, 225, 267, 272, 263, 217, 242, 258, 256, 241] payload_rows_per_dst=[5396, 5538, 5530, 5436, 5458, 5476, 5410, 5466] dup_rows_per_dst=[2710, 2742, 2824, 2744, 2712, 2710, 2718, 2666] combine_rows_per_src=[5396, 5538, 5530, 5436, 5458, 5476, 5410, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=230064 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157262512 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294960 weight_movement_bytes=18874368 total_nvlink_bytes=333169328 weight_to_token_ratio=0.060053 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229424 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157351984 ingress_bytes=213745664 link_bytes=213745664 token_bytes=314474544 weight_movement_bytes=56623104 total_nvlink_bytes=371097648 weight_to_token_ratio=0.180056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048005783e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38252 dup_rows=21792 dup_rows_remote=19116 dedup_ratio=0.332520 router_tokens_per_expert=[276, 282, 254, 251, 279, 247, 248, 258, 276, 253, 216, 270, 282, 269, 235, 260, 256, 267, 256, 259, 241, 254, 254, 266, 256, 248, 242, 226, 265, 229, 257, 237, 247, 251, 272, 260, 246, 271, 260, 266, 258, 263, 267, 245, 259, 254, 226, 239, 253, 237, 248, 230, 276, 268, 239, 257, 264, 256, 239, 260, 237, 260, 270, 258, 234, 227, 287, 253, 263, 235, 241, 273, 251, 230, 295, 261, 253, 259, 252, 267, 229, 253, 258, 254, 256, 291, 260, 256, 275, 258, 256, 279, 266, 254, 254, 216, 301, 257, 272, 267, 277, 247, 236, 251, 249, 267, 240, 274, 255, 250, 255, 268, 253, 255, 248, 265, 237, 259, 243, 256, 245, 279, 258, 256, 273, 260, 251, 263] payload_rows_per_dst=[5518, 5372, 5492, 5432, 5408, 5538, 5510, 5474] dup_rows_per_dst=[2794, 2654, 2676, 2672, 2754, 2692, 2822, 2728] combine_rows_per_src=[5518, 5372, 5492, 5432, 5408, 5538, 5510, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229152 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007648 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786144 weight_movement_bytes=37748736 total_nvlink_bytes=351534880 weight_to_token_ratio=0.120301 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229464 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909656 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589848 weight_movement_bytes=37748736 total_nvlink_bytes=351338584 weight_to_token_ratio=0.120376 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229472 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909664 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589856 weight_movement_bytes=18874368 total_nvlink_bytes=332464224 weight_to_token_ratio=0.060188 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38168 dup_rows=21862 dup_rows_remote=19094 dedup_ratio=0.333588 router_tokens_per_expert=[245, 246, 258, 257, 247, 211, 270, 223, 261, 243, 253, 243, 252, 238, 270, 277, 247, 249, 238, 258, 264, 241, 288, 278, 245, 253, 256, 270, 257, 238, 264, 291, 236, 256, 270, 274, 265, 262, 236, 255, 277, 265, 257, 236, 256, 267, 282, 270, 244, 271, 265, 251, 270, 253, 280, 263, 264, 251, 251, 259, 246, 244, 239, 240, 259, 257, 247, 265, 264, 234, 240, 252, 267, 264, 256, 269, 255, 225, 244, 279, 274, 253, 257, 254, 255, 273, 261, 256, 252, 266, 246, 265, 227, 249, 257, 258, 266, 257, 247, 291, 293, 236, 262, 244, 287, 264, 234, 236, 238, 240, 261, 276, 256, 263, 237, 236, 253, 249, 241, 254, 242, 268, 254, 252, 270, 255, 269, 271] payload_rows_per_dst=[5420, 5506, 5570, 5452, 5456, 5408, 5498, 5364] dup_rows_per_dst=[2568, 2768, 2758, 2730, 2698, 2798, 2766, 2776] combine_rows_per_src=[5420, 5506, 5570, 5452, 5456, 5408, 5498, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38262 dup_rows=21832 dup_rows_remote=19204 dedup_ratio=0.333130 router_tokens_per_expert=[258, 286, 269, 230, 260, 232, 254, 263, 268, 258, 250, 249, 237, 244, 259, 241, 262, 259, 262, 277, 272, 263, 249, 269, 238, 262, 290, 251, 245, 236, 252, 275, 275, 243, 265, 234, 259, 242, 245, 279, 277, 257, 267, 245, 264, 271, 273, 224, 259, 293, 245, 258, 273, 290, 256, 237, 248, 251, 264, 240, 248, 249, 264, 271, 252, 250, 250, 254, 278, 232, 243, 230, 260, 274, 249, 242, 279, 261, 258, 254, 254, 226, 268, 262, 254, 254, 262, 273, 232, 273, 230, 278, 235, 266, 248, 220, 256, 267, 270, 243, 264, 250, 267, 276, 277, 228, 268, 231, 237, 254, 288, 251, 266, 268, 253, 265, 240, 266, 256, 256, 261, 213, 246, 258, 266, 251, 238, 251] payload_rows_per_dst=[5396, 5550, 5528, 5494, 5398, 5442, 5464, 5432] dup_rows_per_dst=[2720, 2774, 2712, 2798, 2734, 2628, 2790, 2676] combine_rows_per_src=[5396, 5550, 5528, 5494, 5398, 5442, 5464, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38496 dup_rows=21654 dup_rows_remote=18978 dedup_ratio=0.330414 router_tokens_per_expert=[265, 269, 228, 230, 249, 247, 270, 269, 233, 271, 270, 238, 260, 262, 249, 253, 273, 233, 241, 277, 253, 265, 263, 254, 260, 285, 278, 286, 275, 269, 278, 265, 253, 224, 262, 236, 233, 254, 278, 271, 263, 252, 268, 265, 264, 280, 269, 268, 260, 248, 262, 264, 245, 246, 240, 245, 263, 240, 275, 266, 270, 265, 254, 246, 239, 237, 243, 249, 245, 257, 292, 299, 260, 246, 267, 257, 265, 236, 267, 269, 250, 247, 249, 271, 251, 244, 250, 272, 233, 250, 251, 233, 241, 262, 237, 267, 255, 243, 251, 258, 215, 249, 239, 251, 251, 290, 257, 270, 261, 232, 269, 263, 297, 260, 245, 263, 255, 248, 233, 229, 246, 242, 271, 263, 245, 254, 225, 255] payload_rows_per_dst=[5488, 5354, 5520, 5490, 5514, 5636, 5494, 5386] dup_rows_per_dst=[2578, 2518, 2760, 2688, 2742, 2936, 2756, 2676] combine_rows_per_src=[5488, 5354, 5520, 5490, 5514, 5636, 5494, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38162 dup_rows=21938 dup_rows_remote=19226 dedup_ratio=0.334747 router_tokens_per_expert=[267, 233, 247, 265, 226, 270, 221, 266, 255, 262, 260, 280, 248, 241, 252, 261, 258, 249, 245, 255, 256, 234, 268, 212, 268, 271, 265, 273, 252, 291, 261, 250, 256, 270, 244, 245, 252, 299, 275, 245, 253, 268, 214, 235, 252, 241, 256, 243, 262, 253, 256, 243, 261, 238, 266, 242, 269, 248, 252, 275, 267, 249, 263, 246, 281, 255, 275, 263, 245, 238, 252, 229, 279, 253, 228, 266, 265, 252, 249, 272, 264, 235, 281, 235, 255, 252, 256, 287, 226, 247, 241, 233, 264, 241, 258, 274, 258, 255, 256, 239, 254, 234, 271, 287, 240, 231, 256, 253, 287, 238, 265, 250, 235, 291, 290, 285, 233, 275, 288, 248, 272, 261, 285, 255, 265, 242, 260, 258] payload_rows_per_dst=[5456, 5460, 5434, 5462, 5418, 5412, 5436, 5520] dup_rows_per_dst=[2652, 2736, 2662, 2718, 2786, 2706, 2712, 2966] combine_rows_per_src=[5456, 5460, 5434, 5462, 5418, 5412, 5436, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38222 dup_rows=21838 dup_rows_remote=19208 dedup_ratio=0.333221 router_tokens_per_expert=[256, 258, 260, 239, 244, 257, 248, 295, 233, 278, 296, 252, 242, 255, 247, 252, 238, 280, 253, 261, 264, 284, 242, 245, 265, 257, 251, 256, 247, 246, 263, 253, 253, 271, 261, 276, 244, 275, 250, 241, 270, 253, 282, 266, 238, 274, 270, 240, 248, 251, 264, 250, 235, 264, 250, 251, 244, 270, 256, 261, 272, 218, 256, 263, 272, 251, 224, 238, 267, 260, 248, 290, 276, 249, 277, 252, 260, 235, 242, 291, 255, 243, 257, 233, 268, 264, 261, 225, 231, 257, 236, 296, 240, 234, 266, 286, 254, 243, 253, 257, 245, 254, 280, 220, 276, 253, 263, 228, 274, 254, 238, 233, 260, 286, 257, 264, 259, 262, 239, 242, 268, 270, 251, 250, 281, 255, 232, 249] payload_rows_per_dst=[5398, 5478, 5520, 5476, 5544, 5388, 5412, 5482] dup_rows_per_dst=[2826, 2732, 2808, 2630, 2720, 2716, 2638, 2768] combine_rows_per_src=[5398, 5478, 5520, 5476, 5544, 5388, 5412, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229048 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156565176 ingress_bytes=212959232 link_bytes=212959232 token_bytes=312901304 weight_movement_bytes=56623104 total_nvlink_bytes=369524408 weight_to_token_ratio=0.180962 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040470579e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38354 dup_rows=21732 dup_rows_remote=19116 dedup_ratio=0.331604 router_tokens_per_expert=[226, 221, 258, 255, 268, 265, 209, 243, 241, 282, 255, 258, 257, 245, 272, 274, 279, 255, 265, 259, 232, 257, 230, 274, 261, 237, 233, 267, 269, 222, 269, 251, 290, 262, 281, 236, 224, 258, 272, 251, 252, 242, 294, 259, 273, 229, 275, 222, 262, 233, 255, 251, 269, 266, 274, 246, 265, 247, 240, 264, 278, 264, 255, 259, 244, 278, 228, 251, 267, 252, 214, 266, 241, 252, 245, 261, 266, 240, 265, 279, 243, 252, 243, 297, 245, 258, 240, 267, 292, 260, 271, 257, 244, 245, 254, 265, 243, 277, 280, 238, 245, 256, 273, 296, 264, 230, 281, 272, 255, 229, 251, 244, 258, 263, 255, 251, 272, 242, 283, 269, 267, 256, 240, 261, 257, 246, 239, 256] payload_rows_per_dst=[5450, 5422, 5552, 5546, 5398, 5536, 5504, 5396] dup_rows_per_dst=[2616, 2698, 2688, 2824, 2586, 2964, 2756, 2600] combine_rows_per_src=[5450, 5422, 5552, 5546, 5398, 5536, 5504, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38296 dup_rows=21800 dup_rows_remote=19194 dedup_ratio=0.332642 router_tokens_per_expert=[263, 293, 258, 233, 265, 277, 254, 241, 255, 233, 256, 283, 264, 257, 254, 251, 268, 260, 251, 232, 256, 238, 238, 250, 228, 293, 234, 269, 243, 280, 288, 280, 248, 264, 258, 256, 252, 251, 256, 249, 249, 237, 256, 239, 268, 257, 265, 230, 255, 252, 261, 252, 250, 261, 266, 257, 244, 235, 261, 272, 258, 254, 259, 245, 248, 261, 233, 250, 231, 260, 253, 241, 266, 280, 241, 263, 259, 220, 290, 227, 223, 264, 267, 239, 249, 254, 249, 247, 255, 252, 282, 266, 278, 283, 231, 278, 280, 247, 220, 263, 246, 224, 242, 263, 257, 244, 286, 267, 287, 241, 263, 258, 254, 276, 248, 292, 253, 263, 265, 266, 248, 260, 273, 259, 281, 242, 234, 264] payload_rows_per_dst=[5518, 5422, 5428, 5450, 5440, 5450, 5476, 5552] dup_rows_per_dst=[2756, 2794, 2642, 2714, 2606, 2784, 2700, 2804] combine_rows_per_src=[5518, 5422, 5428, 5450, 5440, 5450, 5476, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=229864 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156951016 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313672168 weight_movement_bytes=18874368 total_nvlink_bytes=332546536 weight_to_token_ratio=0.060172 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157679616 dispatch_meta_bytes=229896 combine_payload_bytes=157679616 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157909512 ingress_bytes=214302720 link_bytes=214302720 token_bytes=315589128 weight_movement_bytes=56623104 total_nvlink_bytes=372212232 weight_to_token_ratio=0.179420 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053343220e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229552 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156541104 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312852656 weight_movement_bytes=18874368 total_nvlink_bytes=331727024 weight_to_token_ratio=0.060330 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38168 dup_rows=21958 dup_rows_remote=19132 dedup_ratio=0.335052 router_tokens_per_expert=[247, 231, 252, 280, 256, 285, 247, 257, 264, 242, 253, 259, 264, 237, 245, 234, 268, 253, 250, 278, 250, 268, 262, 253, 264, 269, 236, 244, 252, 242, 238, 277, 240, 224, 255, 261, 280, 261, 241, 251, 248, 257, 250, 289, 287, 276, 246, 252, 247, 250, 227, 254, 261, 263, 261, 248, 232, 255, 251, 260, 258, 256, 247, 248, 244, 262, 275, 222, 256, 278, 278, 274, 274, 260, 296, 268, 246, 267, 228, 252, 270, 222, 272, 275, 273, 254, 225, 260, 262, 247, 273, 283, 257, 274, 252, 289, 252, 265, 248, 243, 278, 251, 269, 258, 248, 224, 247, 272, 249, 296, 228, 264, 267, 252, 273, 257, 242, 259, 241, 249, 254, 244, 254, 238, 274, 256, 222, 233] payload_rows_per_dst=[5436, 5476, 5410, 5334, 5520, 5572, 5458, 5372] dup_rows_per_dst=[2670, 2732, 2826, 2702, 2840, 2804, 2726, 2658] combine_rows_per_src=[5436, 5476, 5410, 5334, 5520, 5572, 5458, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229720 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156787032 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313344344 weight_movement_bytes=18874368 total_nvlink_bytes=332218712 weight_to_token_ratio=0.060235 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229880 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157327864 ingress_bytes=213721088 link_bytes=213721088 token_bytes=314425848 weight_movement_bytes=56623104 total_nvlink_bytes=371048952 weight_to_token_ratio=0.180084 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047770308e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229960 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157090376 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950792 weight_movement_bytes=37748736 total_nvlink_bytes=351699528 weight_to_token_ratio=0.120238 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229200 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156565328 ingress_bytes=194084864 link_bytes=194084864 token_bytes=312901456 weight_movement_bytes=37748736 total_nvlink_bytes=350650192 weight_to_token_ratio=0.120641 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859625671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38086 dup_rows=21918 dup_rows_remote=19194 dedup_ratio=0.334442 router_tokens_per_expert=[264, 249, 242, 230, 256, 272, 258, 229, 232, 271, 245, 247, 245, 283, 282, 241, 252, 253, 278, 250, 234, 252, 282, 270, 251, 242, 255, 252, 252, 269, 281, 255, 255, 256, 238, 261, 308, 273, 253, 236, 244, 266, 251, 254, 235, 232, 268, 272, 255, 223, 250, 275, 241, 280, 237, 267, 265, 249, 280, 270, 264, 245, 269, 268, 249, 283, 251, 287, 248, 252, 259, 237, 242, 275, 248, 261, 252, 255, 251, 254, 252, 263, 248, 254, 246, 251, 272, 258, 259, 279, 242, 251, 253, 291, 246, 261, 241, 274, 244, 250, 261, 261, 241, 274, 234, 259, 252, 241, 273, 250, 238, 246, 243, 258, 247, 275, 246, 243, 273, 240, 279, 226, 242, 262, 263, 244, 274, 270] payload_rows_per_dst=[5368, 5532, 5422, 5514, 5448, 5458, 5392, 5484] dup_rows_per_dst=[2724, 2724, 2782, 2762, 2760, 2794, 2686, 2686] combine_rows_per_src=[5368, 5532, 5422, 5514, 5448, 5458, 5392, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38316 dup_rows=21658 dup_rows_remote=18814 dedup_ratio=0.330475 router_tokens_per_expert=[228, 244, 256, 256, 266, 260, 241, 250, 244, 249, 262, 293, 277, 230, 273, 263, 245, 275, 277, 257, 242, 239, 257, 244, 251, 243, 271, 249, 252, 252, 242, 240, 259, 262, 272, 230, 259, 256, 268, 274, 264, 265, 251, 267, 257, 278, 248, 251, 268, 254, 254, 258, 252, 256, 257, 256, 297, 241, 257, 270, 238, 250, 245, 249, 244, 255, 263, 271, 260, 244, 242, 265, 275, 248, 291, 252, 238, 248, 246, 281, 257, 233, 270, 249, 276, 265, 248, 259, 239, 247, 257, 254, 243, 236, 247, 247, 267, 268, 259, 237, 251, 242, 268, 257, 245, 252, 254, 240, 250, 251, 260, 257, 267, 290, 270, 287, 217, 263, 264, 244, 256, 283, 288, 242, 251, 257, 245, 245] payload_rows_per_dst=[5484, 5542, 5450, 5430, 5490, 5398, 5522, 5562] dup_rows_per_dst=[2700, 2530, 2512, 2774, 2688, 2656, 2954, 2844] combine_rows_per_src=[5484, 5542, 5450, 5430, 5490, 5398, 5522, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38406 dup_rows=21786 dup_rows_remote=19180 dedup_ratio=0.332428 router_tokens_per_expert=[262, 267, 274, 268, 263, 233, 256, 237, 270, 248, 261, 265, 285, 247, 274, 246, 284, 241, 256, 267, 269, 269, 269, 257, 248, 250, 279, 248, 241, 275, 277, 257, 287, 243, 239, 239, 263, 255, 225, 251, 256, 245, 243, 238, 259, 259, 259, 266, 247, 233, 246, 241, 277, 238, 265, 267, 269, 249, 273, 263, 258, 269, 231, 238, 254, 266, 288, 254, 297, 260, 253, 239, 249, 268, 246, 262, 254, 246, 273, 288, 276, 257, 241, 253, 231, 254, 231, 253, 254, 236, 252, 233, 225, 259, 234, 286, 263, 271, 214, 253, 240, 293, 270, 245, 254, 272, 263, 268, 241, 244, 242, 248, 269, 281, 257, 261, 245, 269, 271, 246, 252, 237, 251, 253, 248, 240, 275, 226] payload_rows_per_dst=[5520, 5526, 5418, 5438, 5536, 5344, 5486, 5482] dup_rows_per_dst=[2792, 2848, 2636, 2690, 2858, 2606, 2676, 2680] combine_rows_per_src=[5520, 5526, 5418, 5438, 5536, 5344, 5486, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38232 dup_rows=21704 dup_rows_remote=18868 dedup_ratio=0.331177 router_tokens_per_expert=[262, 230, 259, 259, 275, 253, 253, 255, 239, 271, 265, 241, 252, 271, 249, 246, 255, 274, 252, 245, 260, 256, 263, 227, 266, 268, 257, 254, 260, 255, 261, 261, 273, 240, 270, 249, 243, 275, 251, 231, 211, 267, 240, 257, 238, 264, 275, 231, 250, 274, 220, 240, 251, 256, 224, 258, 231, 235, 246, 308, 265, 269, 272, 276, 236, 294, 255, 251, 253, 285, 261, 233, 243, 274, 266, 244, 258, 263, 242, 255, 230, 259, 266, 267, 239, 241, 242, 262, 273, 251, 273, 280, 256, 248, 272, 244, 263, 276, 259, 262, 283, 242, 266, 269, 272, 298, 257, 298, 250, 238, 251, 234, 281, 255, 263, 244, 250, 255, 230, 258, 255, 250, 253, 244, 235, 277, 236, 264] payload_rows_per_dst=[5458, 5518, 5432, 5432, 5478, 5508, 5600, 5406] dup_rows_per_dst=[2702, 2710, 2598, 2718, 2748, 2698, 2836, 2694] combine_rows_per_src=[5458, 5518, 5432, 5432, 5478, 5508, 5600, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38112 dup_rows=21912 dup_rows_remote=19102 dedup_ratio=0.334351 router_tokens_per_expert=[236, 282, 257, 256, 273, 254, 250, 276, 242, 251, 271, 230, 283, 246, 241, 271, 237, 285, 249, 251, 239, 244, 250, 238, 233, 277, 247, 277, 254, 216, 243, 261, 284, 270, 260, 264, 244, 229, 259, 248, 247, 267, 250, 261, 246, 257, 249, 233, 282, 253, 246, 278, 234, 253, 280, 254, 249, 253, 282, 266, 255, 275, 239, 262, 251, 244, 254, 253, 273, 245, 250, 253, 250, 231, 242, 226, 263, 271, 277, 263, 247, 257, 277, 246, 265, 258, 252, 278, 252, 274, 263, 249, 248, 264, 249, 264, 264, 260, 268, 248, 264, 266, 253, 228, 242, 239, 241, 241, 240, 251, 277, 277, 229, 262, 247, 262, 252, 250, 251, 255, 253, 294, 257, 274, 279, 257, 301, 248] payload_rows_per_dst=[5496, 5388, 5426, 5512, 5406, 5468, 5442, 5486] dup_rows_per_dst=[2742, 2614, 2710, 2810, 2686, 2818, 2676, 2856] combine_rows_per_src=[5496, 5388, 5426, 5512, 5406, 5468, 5442, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229120 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156229376 ingress_bytes=174874624 link_bytes=174874624 token_bytes=312229632 weight_movement_bytes=18874368 total_nvlink_bytes=331104000 weight_to_token_ratio=0.060450 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675562603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=228520 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157170856 ingress_bytes=194691072 link_bytes=194691072 token_bytes=314113192 weight_movement_bytes=37748736 total_nvlink_bytes=351861928 weight_to_token_ratio=0.120176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865434058e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=230344 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157541320 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314852296 weight_movement_bytes=18874368 total_nvlink_bytes=333726664 weight_to_token_ratio=0.059947 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228400 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156826672 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313424944 weight_movement_bytes=18874368 total_nvlink_bytes=332299312 weight_to_token_ratio=0.060220 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38190 dup_rows=21848 dup_rows_remote=19172 dedup_ratio=0.333374 router_tokens_per_expert=[287, 244, 243, 273, 259, 260, 237, 270, 245, 258, 249, 264, 252, 247, 246, 249, 281, 232, 261, 246, 253, 264, 273, 275, 253, 237, 259, 255, 262, 290, 236, 257, 262, 253, 272, 264, 244, 283, 258, 274, 243, 246, 274, 276, 271, 260, 292, 265, 263, 272, 256, 256, 256, 242, 255, 264, 235, 255, 247, 262, 263, 241, 219, 264, 268, 282, 258, 269, 226, 252, 246, 271, 252, 241, 244, 254, 264, 255, 241, 264, 245, 262, 264, 238, 268, 264, 268, 261, 280, 258, 247, 241, 232, 272, 241, 268, 275, 267, 240, 247, 243, 247, 240, 267, 261, 248, 245, 272, 242, 227, 250, 229, 244, 242, 263, 252, 233, 271, 252, 238, 247, 282, 270, 248, 240, 265, 252, 269] payload_rows_per_dst=[5434, 5514, 5606, 5388, 5498, 5402, 5348, 5498] dup_rows_per_dst=[2732, 2754, 2868, 2712, 2676, 2816, 2652, 2638] combine_rows_per_src=[5434, 5514, 5606, 5388, 5498, 5402, 5348, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38216 dup_rows=21782 dup_rows_remote=18966 dedup_ratio=0.332367 router_tokens_per_expert=[270, 261, 259, 255, 248, 257, 261, 269, 270, 282, 245, 276, 246, 269, 245, 274, 238, 253, 228, 260, 265, 251, 248, 241, 266, 262, 257, 282, 245, 261, 259, 266, 259, 276, 220, 265, 280, 249, 261, 273, 284, 230, 265, 265, 242, 263, 268, 277, 271, 270, 253, 252, 247, 241, 238, 250, 270, 276, 262, 264, 235, 243, 279, 249, 254, 240, 238, 245, 242, 252, 262, 254, 262, 235, 267, 235, 250, 230, 264, 258, 232, 244, 266, 276, 253, 237, 247, 275, 256, 256, 245, 252, 266, 231, 257, 245, 260, 246, 262, 229, 253, 249, 247, 260, 259, 247, 250, 263, 269, 244, 236, 270, 253, 287, 294, 253, 242, 259, 239, 252, 257, 265, 267, 287, 245, 242, 256, 254] payload_rows_per_dst=[5572, 5416, 5538, 5414, 5406, 5444, 5442, 5522] dup_rows_per_dst=[2802, 2748, 2816, 2786, 2570, 2632, 2646, 2782] combine_rows_per_src=[5572, 5416, 5538, 5414, 5406, 5444, 5442, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38310 dup_rows=21672 dup_rows_remote=18818 dedup_ratio=0.330688 router_tokens_per_expert=[256, 227, 257, 250, 263, 272, 246, 250, 258, 247, 233, 263, 263, 242, 228, 254, 289, 248, 270, 267, 264, 258, 245, 258, 245, 247, 268, 250, 293, 245, 255, 245, 252, 223, 281, 256, 257, 278, 246, 215, 259, 246, 263, 263, 258, 253, 258, 258, 265, 255, 229, 251, 260, 245, 233, 259, 263, 270, 250, 265, 260, 262, 256, 261, 238, 258, 240, 250, 217, 231, 256, 255, 250, 282, 227, 230, 263, 259, 260, 260, 266, 281, 261, 273, 265, 275, 240, 248, 288, 253, 253, 247, 258, 295, 274, 257, 234, 271, 244, 257, 249, 299, 272, 260, 236, 257, 253, 245, 248, 251, 293, 277, 274, 242, 256, 246, 297, 252, 262, 282, 234, 279, 244, 223, 255, 242, 260, 258] payload_rows_per_dst=[5554, 5552, 5526, 5570, 5518, 5124, 5500, 5520] dup_rows_per_dst=[2854, 2744, 2768, 2760, 2828, 2276, 2750, 2692] combine_rows_per_src=[5554, 5552, 5526, 5570, 5518, 5124, 5500, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228856 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156335608 ingress_bytes=174981120 link_bytes=174981120 token_bytes=312442360 weight_movement_bytes=18874368 total_nvlink_bytes=331316728 weight_to_token_ratio=0.060409 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676582996e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229448 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156655688 ingress_bytes=213049344 link_bytes=213049344 token_bytes=313081928 weight_movement_bytes=56623104 total_nvlink_bytes=369705032 weight_to_token_ratio=0.180857 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041333987e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=228728 combine_payload_bytes=156532736 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156761464 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294200 weight_movement_bytes=37748736 total_nvlink_bytes=351042936 weight_to_token_ratio=0.120490 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=228512 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157146272 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314064032 weight_movement_bytes=18874368 total_nvlink_bytes=332938400 weight_to_token_ratio=0.060097 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38086 dup_rows=21936 dup_rows_remote=19088 dedup_ratio=0.334717 router_tokens_per_expert=[269, 243, 267, 236, 267, 233, 257, 245, 251, 243, 247, 256, 228, 246, 243, 266, 293, 265, 295, 236, 256, 241, 257, 253, 257, 246, 274, 264, 255, 247, 265, 277, 274, 261, 284, 244, 235, 280, 260, 277, 263, 250, 236, 230, 256, 248, 259, 255, 250, 254, 261, 231, 267, 264, 251, 251, 236, 272, 288, 227, 248, 227, 271, 241, 267, 243, 273, 271, 267, 237, 235, 275, 247, 286, 246, 286, 262, 269, 241, 281, 238, 250, 263, 239, 254, 263, 255, 265, 238, 258, 256, 270, 247, 264, 254, 233, 258, 246, 272, 270, 273, 254, 265, 253, 239, 251, 260, 237, 255, 231, 278, 237, 274, 268, 253, 257, 252, 253, 259, 252, 243, 254, 247, 266, 275, 247, 274, 253] payload_rows_per_dst=[5374, 5514, 5520, 5354, 5528, 5388, 5430, 5492] dup_rows_per_dst=[2620, 2848, 2704, 2724, 2844, 2712, 2722, 2762] combine_rows_per_src=[5374, 5514, 5520, 5354, 5528, 5388, 5430, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38322 dup_rows=21728 dup_rows_remote=19026 dedup_ratio=0.331543 router_tokens_per_expert=[244, 262, 241, 266, 233, 222, 260, 265, 243, 272, 252, 268, 271, 237, 243, 256, 259, 264, 246, 285, 270, 277, 258, 244, 255, 254, 238, 268, 244, 256, 266, 254, 257, 278, 240, 249, 247, 283, 263, 284, 236, 248, 252, 256, 241, 247, 239, 286, 275, 246, 274, 259, 275, 265, 260, 255, 264, 270, 261, 260, 254, 245, 275, 255, 260, 238, 236, 271, 249, 249, 254, 278, 240, 287, 260, 245, 267, 241, 234, 281, 266, 245, 255, 237, 249, 244, 262, 239, 237, 240, 240, 234, 270, 264, 248, 263, 223, 270, 297, 270, 231, 248, 260, 247, 250, 258, 264, 273, 261, 253, 260, 263, 252, 272, 260, 233, 258, 274, 263, 259, 243, 246, 249, 255, 261, 244, 249, 267] payload_rows_per_dst=[5306, 5594, 5322, 5502, 5620, 5388, 5590, 5486] dup_rows_per_dst=[2508, 2938, 2506, 2828, 2936, 2598, 2712, 2702] combine_rows_per_src=[5306, 5594, 5322, 5502, 5620, 5388, 5590, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38318 dup_rows=21774 dup_rows_remote=18904 dedup_ratio=0.332245 router_tokens_per_expert=[246, 245, 272, 257, 288, 264, 227, 259, 276, 261, 243, 262, 263, 242, 278, 245, 231, 251, 259, 257, 247, 266, 251, 238, 230, 273, 243, 261, 251, 249, 260, 256, 248, 262, 285, 245, 236, 256, 241, 275, 265, 275, 269, 248, 281, 246, 239, 294, 254, 258, 238, 258, 244, 260, 258, 240, 250, 273, 252, 260, 273, 266, 238, 293, 225, 266, 243, 267, 265, 260, 263, 251, 240, 261, 254, 247, 269, 250, 271, 242, 239, 279, 245, 240, 256, 279, 265, 279, 242, 224, 270, 253, 250, 244, 253, 272, 257, 281, 258, 277, 242, 250, 272, 258, 252, 274, 277, 251, 216, 268, 248, 276, 248, 237, 293, 277, 228, 278, 237, 236, 215, 270, 273, 230, 259, 240, 248, 247] payload_rows_per_dst=[5558, 5408, 5498, 5462, 5470, 5472, 5444, 5450] dup_rows_per_dst=[2698, 2638, 2832, 2768, 2678, 2708, 2870, 2582] combine_rows_per_src=[5558, 5408, 5498, 5462, 5470, 5472, 5444, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38246 dup_rows=21824 dup_rows_remote=19052 dedup_ratio=0.333008 router_tokens_per_expert=[230, 240, 241, 285, 235, 253, 232, 236, 247, 255, 254, 227, 247, 252, 262, 239, 268, 278, 282, 270, 245, 271, 249, 249, 257, 249, 264, 255, 276, 247, 258, 251, 291, 255, 270, 264, 292, 240, 253, 239, 246, 274, 259, 273, 258, 248, 232, 263, 253, 254, 251, 227, 261, 261, 273, 246, 236, 262, 226, 251, 250, 247, 257, 253, 253, 243, 267, 268, 237, 236, 266, 242, 264, 259, 264, 259, 242, 254, 233, 245, 247, 271, 250, 258, 254, 270, 262, 242, 268, 265, 295, 265, 230, 255, 247, 240, 250, 262, 238, 263, 221, 296, 248, 265, 301, 241, 253, 259, 267, 261, 243, 283, 258, 254, 255, 277, 260, 280, 251, 269, 278, 260, 240, 272, 263, 261, 260, 259] payload_rows_per_dst=[5300, 5552, 5506, 5414, 5440, 5466, 5532, 5502] dup_rows_per_dst=[2570, 2786, 2808, 2602, 2624, 2772, 2770, 2892] combine_rows_per_src=[5300, 5552, 5506, 5414, 5440, 5466, 5532, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=228696 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156228952 ingress_bytes=174874624 link_bytes=174874624 token_bytes=312229208 weight_movement_bytes=18874368 total_nvlink_bytes=331103576 weight_to_token_ratio=0.060450 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675562603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38236 dup_rows=21798 dup_rows_remote=19074 dedup_ratio=0.332611 router_tokens_per_expert=[266, 241, 253, 303, 262, 282, 245, 242, 238, 236, 256, 283, 277, 258, 243, 284, 274, 221, 251, 263, 251, 241, 259, 257, 264, 290, 231, 252, 275, 275, 260, 239, 273, 284, 265, 251, 284, 234, 223, 227, 241, 239, 259, 231, 251, 243, 253, 230, 250, 264, 250, 242, 258, 263, 253, 255, 273, 275, 235, 261, 285, 242, 237, 270, 255, 254, 242, 273, 251, 238, 247, 239, 261, 251, 237, 296, 252, 274, 219, 247, 254, 249, 245, 241, 230, 270, 284, 214, 262, 273, 256, 233, 267, 263, 236, 255, 249, 244, 252, 270, 249, 270, 288, 244, 252, 244, 264, 281, 253, 258, 274, 269, 263, 266, 268, 258, 262, 269, 255, 276, 237, 272, 266, 284, 250, 229, 241, 270] payload_rows_per_dst=[5536, 5510, 5338, 5502, 5410, 5406, 5494, 5542] dup_rows_per_dst=[2802, 2696, 2638, 2724, 2662, 2658, 2828, 2790] combine_rows_per_src=[5536, 5510, 5338, 5502, 5410, 5406, 5494, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229392 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157196304 ingress_bytes=194715648 link_bytes=194715648 token_bytes=314163216 weight_movement_bytes=37748736 total_nvlink_bytes=351911952 weight_to_token_ratio=0.120156 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865669533e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38408 dup_rows=21642 dup_rows_remote=19008 dedup_ratio=0.330231 router_tokens_per_expert=[243, 268, 254, 287, 256, 252, 271, 236, 247, 267, 283, 254, 256, 242, 252, 247, 228, 234, 245, 265, 270, 248, 249, 225, 245, 252, 262, 263, 250, 247, 249, 243, 260, 248, 253, 249, 270, 273, 253, 238, 297, 252, 237, 274, 274, 265, 260, 257, 248, 254, 303, 260, 238, 247, 229, 273, 243, 263, 246, 245, 260, 256, 255, 254, 261, 249, 244, 244, 269, 249, 254, 261, 244, 263, 238, 258, 248, 255, 265, 258, 256, 243, 264, 224, 269, 272, 250, 261, 261, 258, 252, 235, 257, 277, 272, 263, 278, 273, 252, 274, 277, 228, 241, 252, 271, 274, 284, 243, 268, 253, 241, 265, 259, 265, 260, 261, 255, 259, 272, 248, 259, 271, 245, 240, 256, 245, 250, 251] payload_rows_per_dst=[5454, 5412, 5482, 5464, 5486, 5520, 5602, 5474] dup_rows_per_dst=[2776, 2538, 2838, 2684, 2634, 2708, 2746, 2718] combine_rows_per_src=[5454, 5412, 5482, 5464, 5486, 5520, 5602, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43536 payload_rows_remote=38156 dup_rows=22000 dup_rows_remote=19496 dedup_ratio=0.335693 router_tokens_per_expert=[264, 260, 261, 258, 257, 227, 247, 254, 265, 253, 241, 237, 230, 267, 266, 260, 242, 257, 244, 249, 253, 273, 260, 239, 234, 246, 249, 215, 254, 257, 265, 253, 254, 261, 242, 253, 237, 246, 252, 280, 245, 223, 241, 239, 223, 261, 226, 259, 244, 275, 264, 275, 256, 234, 231, 230, 272, 281, 235, 277, 241, 286, 243, 261, 254, 273, 274, 268, 283, 244, 268, 265, 267, 256, 266, 252, 248, 284, 276, 253, 263, 266, 250, 256, 273, 263, 256, 227, 256, 274, 249, 234, 258, 241, 282, 247, 255, 266, 249, 240, 270, 242, 280, 246, 263, 275, 243, 274, 239, 262, 288, 291, 270, 243, 273, 302, 223, 247, 275, 244, 274, 279, 248, 263, 266, 246, 248, 274] payload_rows_per_dst=[5376, 5372, 5380, 5460, 5546, 5398, 5508, 5496] dup_rows_per_dst=[2718, 2608, 2504, 2750, 2916, 2792, 2858, 2854] combine_rows_per_src=[5376, 5372, 5380, 5460, 5546, 5398, 5508, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228888 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179416 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314129944 weight_movement_bytes=37748736 total_nvlink_bytes=351878680 weight_to_token_ratio=0.120169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229192 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156884808 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540424 weight_movement_bytes=56623104 total_nvlink_bytes=370163528 weight_to_token_ratio=0.180593 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38086 dup_rows=21836 dup_rows_remote=18894 dedup_ratio=0.333191 router_tokens_per_expert=[287, 279, 258, 276, 258, 250, 239, 251, 274, 258, 250, 263, 273, 265, 229, 267, 263, 238, 269, 253, 238, 255, 272, 242, 262, 246, 240, 234, 244, 265, 241, 238, 237, 265, 241, 245, 244, 252, 228, 259, 244, 246, 256, 221, 274, 241, 262, 239, 241, 241, 265, 259, 269, 277, 257, 240, 263, 256, 268, 269, 241, 248, 250, 270, 268, 284, 257, 274, 278, 270, 257, 279, 256, 243, 260, 246, 229, 268, 277, 248, 254, 285, 247, 279, 260, 237, 249, 250, 262, 237, 249, 241, 265, 240, 256, 252, 244, 273, 274, 253, 275, 229, 261, 239, 261, 231, 251, 256, 247, 259, 297, 266, 273, 259, 270, 246, 247, 265, 271, 242, 246, 281, 243, 261, 268, 258, 265, 255] payload_rows_per_dst=[5614, 5396, 5342, 5424, 5542, 5428, 5538, 5416] dup_rows_per_dst=[2942, 2604, 2566, 2666, 2846, 2698, 2694, 2820] combine_rows_per_src=[5614, 5396, 5342, 5424, 5542, 5428, 5538, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=229240 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156843896 ingress_bytes=213237760 link_bytes=213237760 token_bytes=313458552 weight_movement_bytes=56623104 total_nvlink_bytes=370081656 weight_to_token_ratio=0.180640 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043139297e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229664 combine_payload_bytes=157319168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157548832 ingress_bytes=176193536 link_bytes=176193536 token_bytes=314868000 weight_movement_bytes=18874368 total_nvlink_bytes=333742368 weight_to_token_ratio=0.059944 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688199769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=230608 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156517584 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312804560 weight_movement_bytes=18874368 total_nvlink_bytes=331678928 weight_to_token_ratio=0.060339 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=227920 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156228176 ingress_bytes=193748992 link_bytes=193748992 token_bytes=312228432 weight_movement_bytes=37748736 total_nvlink_bytes=349977168 weight_to_token_ratio=0.120901 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856407511e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38420 dup_rows=21654 dup_rows_remote=18990 dedup_ratio=0.330414 router_tokens_per_expert=[235, 255, 267, 243, 233, 244, 281, 274, 259, 274, 263, 254, 250, 238, 253, 278, 237, 258, 227, 267, 259, 265, 246, 282, 263, 219, 228, 231, 249, 247, 234, 243, 266, 273, 283, 252, 266, 252, 248, 271, 280, 260, 268, 226, 246, 245, 253, 249, 275, 252, 263, 291, 236, 224, 271, 240, 260, 248, 268, 242, 257, 244, 277, 255, 255, 268, 245, 277, 232, 255, 263, 267, 261, 237, 238, 278, 246, 265, 255, 268, 230, 266, 232, 246, 237, 254, 268, 247, 261, 265, 273, 251, 258, 261, 284, 257, 276, 278, 271, 260, 254, 271, 259, 252, 256, 255, 251, 275, 264, 260, 260, 260, 249, 255, 271, 248, 279, 237, 266, 267, 259, 243, 270, 231, 234, 280, 220, 260] payload_rows_per_dst=[5482, 5462, 5522, 5558, 5456, 5470, 5444, 5488] dup_rows_per_dst=[2720, 2664, 2754, 2648, 2764, 2710, 2744, 2650] combine_rows_per_src=[5482, 5462, 5522, 5558, 5456, 5470, 5444, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38322 dup_rows=21706 dup_rows_remote=19014 dedup_ratio=0.331207 router_tokens_per_expert=[264, 254, 242, 265, 279, 270, 273, 262, 265, 264, 245, 253, 257, 220, 265, 285, 259, 241, 266, 233, 248, 250, 283, 278, 274, 241, 238, 241, 273, 235, 273, 251, 253, 258, 265, 260, 248, 241, 246, 236, 246, 262, 242, 243, 284, 265, 248, 269, 253, 256, 241, 255, 281, 249, 246, 265, 252, 266, 309, 287, 269, 269, 245, 270, 253, 242, 233, 255, 208, 261, 264, 282, 264, 247, 262, 252, 282, 296, 240, 242, 254, 259, 269, 243, 264, 268, 262, 255, 257, 266, 242, 249, 254, 235, 258, 228, 255, 245, 259, 250, 285, 275, 240, 234, 235, 251, 289, 221, 220, 259, 243, 248, 241, 248, 244, 254, 253, 290, 245, 249, 265, 246, 276, 234, 281, 254, 246, 261] payload_rows_per_dst=[5516, 5464, 5456, 5542, 5364, 5540, 5440, 5508] dup_rows_per_dst=[2810, 2704, 2676, 2816, 2646, 2654, 2708, 2692] combine_rows_per_src=[5516, 5464, 5456, 5542, 5364, 5540, 5440, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38190 dup_rows=21820 dup_rows_remote=19044 dedup_ratio=0.332947 router_tokens_per_expert=[258, 223, 267, 233, 279, 260, 258, 259, 268, 254, 262, 240, 274, 254, 251, 252, 252, 235, 256, 229, 287, 232, 247, 281, 259, 270, 226, 237, 251, 249, 276, 268, 246, 236, 246, 236, 260, 258, 231, 259, 260, 275, 272, 299, 271, 239, 250, 260, 284, 272, 247, 266, 246, 252, 253, 255, 232, 267, 245, 260, 249, 278, 231, 249, 255, 247, 250, 228, 274, 273, 255, 244, 284, 229, 247, 254, 237, 247, 230, 229, 269, 253, 292, 243, 256, 271, 260, 246, 277, 238, 293, 276, 278, 256, 260, 258, 249, 251, 249, 298, 259, 259, 255, 254, 242, 262, 253, 295, 230, 259, 269, 267, 261, 269, 247, 245, 274, 249, 266, 256, 243, 254, 243, 259, 241, 261, 238, 271] payload_rows_per_dst=[5420, 5426, 5432, 5516, 5300, 5588, 5526, 5508] dup_rows_per_dst=[2764, 2684, 2764, 2656, 2666, 2864, 2776, 2646] combine_rows_per_src=[5420, 5426, 5432, 5516, 5300, 5588, 5526, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229640 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157597960 ingress_bytes=176242688 link_bytes=176242688 token_bytes=314966280 weight_movement_bytes=18874368 total_nvlink_bytes=333840648 weight_to_token_ratio=0.059925 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688670720e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38322 dup_rows=21754 dup_rows_remote=19080 dedup_ratio=0.331940 router_tokens_per_expert=[260, 240, 249, 275, 261, 232, 294, 274, 289, 266, 257, 248, 259, 279, 260, 260, 265, 243, 228, 246, 267, 254, 260, 253, 248, 258, 282, 266, 261, 250, 238, 255, 286, 262, 244, 250, 272, 259, 260, 243, 279, 265, 254, 251, 261, 253, 251, 258, 221, 257, 283, 253, 280, 260, 242, 216, 222, 257, 251, 265, 265, 264, 233, 255, 240, 274, 287, 258, 243, 246, 268, 275, 246, 226, 246, 264, 265, 261, 250, 245, 273, 206, 254, 267, 253, 255, 259, 249, 271, 227, 253, 268, 262, 249, 262, 259, 277, 243, 250, 266, 289, 250, 256, 234, 251, 268, 243, 263, 250, 260, 251, 251, 233, 232, 232, 271, 233, 250, 284, 279, 273, 246, 249, 247, 253, 242, 257, 275] payload_rows_per_dst=[5544, 5456, 5572, 5372, 5478, 5460, 5484, 5416] dup_rows_per_dst=[2862, 2692, 2724, 2676, 2710, 2674, 2720, 2696] combine_rows_per_src=[5544, 5456, 5572, 5372, 5478, 5460, 5484, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38322 dup_rows=21704 dup_rows_remote=19056 dedup_ratio=0.331177 router_tokens_per_expert=[280, 266, 245, 268, 241, 252, 251, 269, 256, 243, 265, 218, 283, 248, 260, 240, 235, 273, 230, 236, 283, 244, 268, 267, 246, 233, 276, 235, 268, 249, 271, 283, 231, 253, 280, 242, 250, 260, 260, 283, 244, 253, 225, 243, 239, 254, 247, 250, 228, 267, 261, 290, 238, 258, 236, 269, 256, 236, 266, 253, 289, 253, 239, 240, 266, 261, 252, 262, 268, 252, 273, 228, 279, 248, 268, 266, 249, 254, 261, 266, 230, 230, 262, 240, 269, 275, 252, 261, 269, 261, 261, 226, 257, 279, 259, 236, 259, 245, 247, 267, 259, 297, 247, 275, 281, 279, 248, 255, 265, 284, 242, 277, 215, 261, 274, 243, 275, 254, 261, 239, 238, 262, 263, 253, 257, 252, 262, 237] payload_rows_per_dst=[5402, 5498, 5430, 5510, 5502, 5454, 5610, 5426] dup_rows_per_dst=[2768, 2696, 2598, 2648, 2804, 2680, 2844, 2666] combine_rows_per_src=[5402, 5498, 5430, 5510, 5502, 5454, 5610, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229344 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196256 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163168 weight_movement_bytes=18874368 total_nvlink_bytes=333037536 weight_to_token_ratio=0.060078 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38276 dup_rows=21876 dup_rows_remote=19162 dedup_ratio=0.333801 router_tokens_per_expert=[226, 253, 240, 256, 275, 234, 271, 255, 247, 256, 259, 234, 243, 243, 270, 249, 219, 263, 225, 277, 250, 239, 240, 284, 239, 256, 247, 244, 272, 255, 234, 270, 228, 243, 262, 245, 258, 242, 257, 251, 257, 274, 269, 263, 251, 255, 225, 269, 257, 247, 234, 256, 269, 263, 239, 277, 260, 264, 258, 256, 279, 278, 266, 246, 239, 250, 281, 269, 249, 247, 244, 272, 295, 298, 282, 271, 267, 232, 256, 254, 260, 250, 265, 258, 270, 270, 224, 241, 259, 258, 263, 239, 263, 232, 260, 264, 242, 250, 266, 260, 302, 221, 265, 278, 253, 275, 265, 267, 255, 255, 262, 250, 267, 271, 278, 246, 225, 277, 234, 254, 254, 253, 252, 249, 271, 248, 253, 265] payload_rows_per_dst=[5368, 5384, 5384, 5506, 5562, 5492, 5472, 5492] dup_rows_per_dst=[2654, 2644, 2714, 2792, 2850, 2660, 2860, 2702] combine_rows_per_src=[5368, 5384, 5384, 5506, 5562, 5492, 5472, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38444 dup_rows=21608 dup_rows_remote=18832 dedup_ratio=0.329712 router_tokens_per_expert=[253, 278, 236, 235, 225, 264, 245, 260, 242, 270, 252, 247, 271, 247, 281, 245, 249, 260, 267, 269, 273, 244, 255, 272, 249, 252, 247, 239, 256, 272, 247, 234, 265, 246, 218, 244, 243, 246, 255, 246, 265, 262, 281, 246, 275, 263, 257, 248, 283, 269, 269, 228, 273, 256, 251, 238, 258, 245, 272, 244, 272, 233, 252, 232, 278, 272, 248, 279, 223, 280, 282, 263, 255, 225, 258, 256, 246, 278, 257, 230, 254, 273, 251, 259, 254, 260, 255, 270, 256, 260, 266, 252, 285, 268, 284, 255, 261, 257, 267, 249, 228, 259, 261, 226, 271, 265, 244, 274, 265, 269, 241, 281, 269, 244, 245, 261, 256, 249, 240, 251, 241, 247, 262, 257, 262, 261, 250, 252] payload_rows_per_dst=[5506, 5480, 5472, 5476, 5484, 5560, 5490, 5460] dup_rows_per_dst=[2596, 2690, 2648, 2674, 2776, 2844, 2746, 2634] combine_rows_per_src=[5506, 5480, 5472, 5476, 5484, 5560, 5490, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228936 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156655176 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081416 weight_movement_bytes=37748736 total_nvlink_bytes=350830152 weight_to_token_ratio=0.120572 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38268 dup_rows=21736 dup_rows_remote=19016 dedup_ratio=0.331665 router_tokens_per_expert=[258, 241, 227, 261, 253, 260, 286, 263, 287, 246, 272, 256, 267, 237, 253, 258, 264, 294, 260, 267, 280, 258, 271, 238, 227, 264, 255, 252, 252, 260, 255, 287, 261, 269, 237, 251, 233, 268, 246, 233, 228, 248, 233, 243, 274, 232, 248, 268, 274, 263, 246, 245, 274, 253, 241, 259, 239, 245, 269, 271, 242, 253, 252, 242, 263, 257, 255, 252, 247, 278, 257, 233, 240, 263, 257, 272, 276, 260, 262, 276, 243, 271, 240, 248, 252, 238, 248, 253, 245, 277, 269, 232, 273, 248, 262, 262, 253, 229, 229, 300, 239, 251, 267, 256, 260, 242, 267, 256, 245, 232, 281, 270, 273, 249, 249, 260, 254, 257, 218, 244, 275, 265, 253, 252, 266, 272, 269, 277] payload_rows_per_dst=[5532, 5508, 5442, 5610, 5414, 5496, 5456, 5342] dup_rows_per_dst=[2720, 2860, 2770, 2884, 2642, 2626, 2698, 2536] combine_rows_per_src=[5532, 5508, 5442, 5610, 5414, 5496, 5456, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229608 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196520 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163432 weight_movement_bytes=18874368 total_nvlink_bytes=333037800 weight_to_token_ratio=0.060078 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229512 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157196424 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314163336 weight_movement_bytes=56623104 total_nvlink_bytes=370786440 weight_to_token_ratio=0.180235 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229752 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157008248 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786744 weight_movement_bytes=18874368 total_nvlink_bytes=332661112 weight_to_token_ratio=0.060150 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229104 combine_payload_bytes=157466624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157695728 ingress_bytes=214089728 link_bytes=214089728 token_bytes=315162352 weight_movement_bytes=56623104 total_nvlink_bytes=371785456 weight_to_token_ratio=0.179663 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051302435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229136 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156974864 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720592 weight_movement_bytes=37748736 total_nvlink_bytes=351469328 weight_to_token_ratio=0.120326 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38470 dup_rows=21670 dup_rows_remote=18950 dedup_ratio=0.330658 router_tokens_per_expert=[277, 239, 256, 269, 260, 258, 264, 237, 258, 240, 265, 263, 248, 256, 232, 243, 260, 253, 241, 265, 249, 220, 242, 231, 268, 270, 239, 284, 254, 257, 278, 247, 267, 250, 228, 252, 271, 249, 233, 253, 276, 261, 224, 252, 257, 225, 262, 238, 292, 262, 292, 286, 230, 234, 278, 255, 246, 260, 234, 270, 260, 255, 243, 286, 249, 276, 259, 290, 255, 254, 261, 264, 253, 251, 245, 241, 254, 263, 237, 227, 259, 246, 283, 273, 283, 262, 245, 276, 264, 255, 265, 248, 256, 257, 253, 269, 269, 253, 243, 249, 275, 250, 260, 266, 268, 268, 240, 245, 236, 261, 283, 271, 225, 261, 261, 229, 258, 268, 233, 277, 255, 272, 257, 266, 224, 243, 256, 269] payload_rows_per_dst=[5498, 5396, 5434, 5554, 5454, 5576, 5476, 5478] dup_rows_per_dst=[2632, 2720, 2562, 2812, 2704, 2812, 2798, 2630] combine_rows_per_src=[5498, 5396, 5434, 5554, 5454, 5576, 5476, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38190 dup_rows=21796 dup_rows_remote=19004 dedup_ratio=0.332581 router_tokens_per_expert=[253, 251, 243, 283, 259, 278, 245, 266, 256, 269, 239, 253, 245, 274, 266, 278, 243, 287, 255, 232, 269, 241, 253, 261, 270, 268, 271, 262, 253, 267, 261, 265, 278, 255, 266, 289, 245, 259, 261, 242, 275, 239, 250, 222, 235, 282, 258, 267, 275, 231, 222, 281, 248, 227, 252, 250, 269, 264, 238, 250, 262, 270, 264, 255, 226, 275, 249, 247, 259, 244, 244, 257, 252, 272, 264, 275, 261, 276, 264, 241, 258, 270, 220, 255, 263, 242, 246, 243, 252, 278, 242, 260, 272, 263, 235, 270, 257, 231, 228, 247, 238, 234, 264, 253, 261, 246, 232, 279, 283, 227, 235, 251, 260, 259, 272, 243, 277, 248, 226, 270, 261, 273, 266, 261, 253, 265, 238, 258] payload_rows_per_dst=[5340, 5496, 5550, 5408, 5532, 5438, 5426, 5550] dup_rows_per_dst=[2786, 2820, 2876, 2600, 2716, 2650, 2556, 2792] combine_rows_per_src=[5340, 5496, 5550, 5408, 5532, 5438, 5426, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38064 dup_rows=21918 dup_rows_remote=19014 dedup_ratio=0.334442 router_tokens_per_expert=[239, 264, 282, 272, 252, 267, 272, 264, 265, 243, 244, 271, 233, 244, 283, 259, 257, 258, 253, 258, 259, 273, 228, 286, 250, 284, 256, 232, 269, 251, 233, 228, 246, 259, 264, 247, 236, 286, 254, 235, 266, 258, 273, 269, 292, 235, 223, 276, 283, 268, 263, 286, 251, 262, 244, 285, 235, 225, 241, 245, 263, 236, 267, 240, 294, 260, 247, 257, 262, 241, 256, 262, 261, 248, 252, 265, 255, 271, 248, 243, 259, 233, 265, 232, 232, 253, 258, 245, 250, 237, 243, 259, 250, 281, 285, 244, 274, 248, 265, 249, 241, 258, 268, 281, 275, 262, 278, 236, 280, 270, 248, 296, 259, 234, 256, 232, 238, 240, 252, 233, 236, 265, 245, 253, 241, 247, 253, 265] payload_rows_per_dst=[5442, 5402, 5538, 5462, 5452, 5422, 5554, 5346] dup_rows_per_dst=[2866, 2748, 2700, 2726, 2792, 2630, 2904, 2552] combine_rows_per_src=[5442, 5402, 5538, 5462, 5452, 5422, 5554, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38066 dup_rows=21924 dup_rows_remote=19234 dedup_ratio=0.334534 router_tokens_per_expert=[231, 229, 270, 261, 252, 235, 247, 270, 273, 231, 272, 265, 268, 245, 277, 276, 276, 259, 241, 247, 251, 241, 263, 262, 267, 239, 255, 281, 247, 245, 246, 264, 256, 283, 253, 228, 240, 253, 242, 239, 270, 248, 264, 257, 269, 223, 264, 246, 271, 257, 280, 279, 252, 282, 251, 268, 243, 239, 228, 247, 289, 274, 270, 241, 282, 244, 232, 254, 242, 255, 240, 242, 257, 267, 235, 234, 262, 235, 267, 274, 265, 259, 247, 283, 274, 264, 250, 244, 258, 261, 252, 264, 240, 244, 247, 266, 250, 246, 251, 255, 243, 254, 254, 256, 261, 258, 253, 253, 248, 260, 288, 250, 279, 268, 264, 254, 282, 277, 246, 271, 252, 259, 244, 258, 246, 252, 257, 247] payload_rows_per_dst=[5454, 5424, 5470, 5502, 5336, 5546, 5408, 5472] dup_rows_per_dst=[2750, 2744, 2600, 2840, 2708, 2690, 2752, 2840] combine_rows_per_src=[5454, 5424, 5470, 5502, 5336, 5546, 5408, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38328 dup_rows=21694 dup_rows_remote=18964 dedup_ratio=0.331024 router_tokens_per_expert=[246, 259, 238, 266, 246, 276, 218, 259, 270, 253, 259, 275, 242, 265, 287, 240, 273, 242, 253, 273, 259, 244, 278, 255, 257, 249, 252, 276, 211, 243, 252, 261, 253, 241, 234, 252, 258, 235, 295, 251, 261, 229, 267, 267, 248, 265, 248, 277, 263, 268, 243, 227, 282, 246, 264, 238, 266, 264, 274, 268, 260, 244, 255, 260, 262, 234, 277, 241, 251, 257, 232, 256, 256, 253, 245, 278, 272, 260, 269, 236, 260, 266, 222, 243, 251, 256, 246, 235, 250, 255, 252, 270, 262, 273, 256, 256, 245, 259, 256, 263, 271, 259, 259, 228, 273, 258, 241, 269, 261, 247, 249, 274, 279, 275, 290, 238, 271, 267, 272, 261, 249, 287, 240, 239, 243, 242, 255, 236] payload_rows_per_dst=[5428, 5506, 5478, 5514, 5458, 5402, 5560, 5496] dup_rows_per_dst=[2770, 2650, 2684, 2730, 2700, 2704, 2664, 2792] combine_rows_per_src=[5428, 5506, 5478, 5514, 5458, 5402, 5560, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229680 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157802800 ingress_bytes=176447488 link_bytes=176447488 token_bytes=315375920 weight_movement_bytes=18874368 total_nvlink_bytes=334250288 weight_to_token_ratio=0.059847 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690633012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228776 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655016 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081256 weight_movement_bytes=18874368 total_nvlink_bytes=331955624 weight_to_token_ratio=0.060286 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38340 dup_rows=21780 dup_rows_remote=18974 dedup_ratio=0.332336 router_tokens_per_expert=[244, 272, 244, 247, 274, 281, 275, 261, 251, 236, 242, 241, 280, 257, 241, 274, 223, 279, 217, 234, 230, 250, 253, 268, 239, 249, 265, 271, 251, 224, 246, 244, 250, 238, 261, 273, 263, 272, 265, 257, 269, 245, 229, 270, 251, 267, 249, 252, 248, 222, 234, 263, 277, 256, 286, 250, 255, 266, 265, 249, 249, 257, 259, 275, 263, 271, 257, 269, 219, 288, 254, 274, 262, 261, 245, 234, 236, 254, 263, 270, 264, 255, 255, 256, 256, 275, 234, 249, 264, 250, 270, 261, 263, 256, 260, 268, 260, 223, 279, 260, 288, 274, 276, 268, 265, 225, 264, 261, 272, 229, 264, 222, 267, 233, 262, 251, 257, 249, 268, 241, 253, 265, 266, 244, 230, 278, 285, 248] payload_rows_per_dst=[5452, 5414, 5416, 5470, 5508, 5496, 5488, 5512] dup_rows_per_dst=[2788, 2472, 2806, 2752, 2732, 2776, 2772, 2682] combine_rows_per_src=[5452, 5414, 5416, 5470, 5508, 5496, 5488, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38454 dup_rows=21744 dup_rows_remote=19210 dedup_ratio=0.331787 router_tokens_per_expert=[232, 267, 223, 256, 263, 273, 250, 274, 254, 267, 282, 226, 233, 260, 279, 293, 272, 253, 273, 252, 259, 259, 243, 271, 246, 241, 241, 270, 262, 248, 243, 243, 249, 257, 261, 243, 242, 268, 283, 246, 265, 248, 270, 270, 265, 244, 265, 237, 232, 258, 250, 252, 264, 244, 239, 261, 251, 256, 264, 229, 252, 256, 246, 245, 262, 238, 241, 246, 236, 276, 231, 221, 236, 250, 245, 276, 253, 257, 230, 238, 258, 271, 277, 254, 250, 249, 250, 267, 242, 251, 269, 224, 275, 240, 283, 236, 255, 260, 276, 253, 259, 246, 249, 267, 283, 267, 263, 281, 295, 265, 243, 254, 288, 259, 253, 280, 281, 261, 251, 231, 272, 280, 271, 252, 244, 266, 251, 260] payload_rows_per_dst=[5494, 5458, 5564, 5354, 5338, 5502, 5564, 5518] dup_rows_per_dst=[2770, 2694, 2662, 2644, 2534, 2690, 2868, 2882] combine_rows_per_src=[5494, 5458, 5564, 5354, 5338, 5502, 5564, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228312 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156138456 ingress_bytes=193658880 link_bytes=193658880 token_bytes=312048600 weight_movement_bytes=37748736 total_nvlink_bytes=349797336 weight_to_token_ratio=0.120971 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855544102e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38014 dup_rows=21830 dup_rows_remote=18802 dedup_ratio=0.333099 router_tokens_per_expert=[285, 252, 237, 262, 276, 273, 268, 264, 260, 271, 265, 273, 281, 261, 269, 242, 253, 257, 278, 267, 248, 243, 275, 253, 257, 230, 226, 233, 253, 250, 270, 257, 254, 266, 261, 238, 250, 230, 267, 249, 275, 264, 241, 257, 253, 267, 247, 260, 248, 272, 252, 270, 252, 252, 243, 273, 272, 254, 240, 246, 266, 250, 262, 256, 268, 252, 253, 262, 215, 245, 288, 246, 267, 257, 238, 261, 279, 265, 275, 254, 276, 227, 269, 264, 266, 222, 222, 268, 244, 290, 251, 234, 263, 284, 270, 259, 286, 221, 269, 295, 268, 208, 228, 244, 240, 237, 239, 241, 257, 241, 269, 228, 244, 237, 257, 263, 263, 251, 259, 259, 243, 264, 250, 262, 279, 257, 248, 251] payload_rows_per_dst=[5692, 5500, 5452, 5512, 5482, 5300, 5370, 5398] dup_rows_per_dst=[3028, 2664, 2728, 2920, 2752, 2606, 2572, 2560] combine_rows_per_src=[5692, 5500, 5452, 5512, 5482, 5300, 5370, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=229200 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156147536 ingress_bytes=193667072 link_bytes=193667072 token_bytes=312065872 weight_movement_bytes=37748736 total_nvlink_bytes=349814608 weight_to_token_ratio=0.120964 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855622594e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229168 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157220656 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314212144 weight_movement_bytes=56623104 total_nvlink_bytes=370835248 weight_to_token_ratio=0.180207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229256 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157269896 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310536 weight_movement_bytes=37748736 total_nvlink_bytes=352059272 weight_to_token_ratio=0.120100 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=230656 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157738240 ingress_bytes=176381952 link_bytes=176381952 token_bytes=315245824 weight_movement_bytes=18874368 total_nvlink_bytes=334120192 weight_to_token_ratio=0.059872 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690005079e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38210 dup_rows=21820 dup_rows_remote=19062 dedup_ratio=0.332947 router_tokens_per_expert=[230, 274, 270, 255, 255, 274, 240, 235, 268, 275, 267, 254, 257, 291, 254, 244, 241, 248, 242, 249, 276, 241, 265, 235, 243, 270, 255, 255, 249, 276, 253, 285, 237, 262, 288, 268, 282, 243, 243, 237, 234, 235, 264, 242, 300, 300, 263, 254, 252, 272, 270, 225, 231, 256, 239, 251, 232, 236, 249, 244, 234, 248, 229, 255, 244, 266, 274, 258, 252, 277, 252, 235, 261, 248, 279, 248, 233, 260, 212, 244, 251, 246, 247, 264, 274, 278, 252, 242, 281, 276, 225, 286, 251, 222, 270, 252, 274, 282, 269, 241, 251, 237, 247, 261, 262, 253, 270, 231, 223, 248, 265, 278, 284, 263, 253, 281, 265, 280, 258, 244, 255, 269, 257, 283, 262, 249, 238, 274] payload_rows_per_dst=[5480, 5506, 5502, 5380, 5424, 5408, 5468, 5548] dup_rows_per_dst=[2806, 2758, 2802, 2466, 2662, 2728, 2716, 2882] combine_rows_per_src=[5480, 5506, 5502, 5380, 5424, 5408, 5468, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38238 dup_rows=21786 dup_rows_remote=18954 dedup_ratio=0.332428 router_tokens_per_expert=[226, 267, 273, 256, 255, 248, 230, 272, 261, 258, 257, 246, 256, 249, 255, 242, 256, 258, 257, 244, 267, 223, 262, 263, 249, 271, 254, 260, 261, 263, 237, 258, 257, 262, 273, 255, 243, 265, 241, 263, 274, 265, 230, 272, 247, 246, 256, 258, 228, 260, 284, 280, 261, 275, 242, 239, 255, 271, 264, 245, 263, 260, 244, 241, 284, 256, 236, 265, 238, 248, 247, 261, 257, 255, 274, 251, 232, 247, 273, 247, 256, 241, 241, 255, 273, 269, 264, 275, 273, 254, 276, 277, 274, 236, 247, 260, 256, 259, 257, 260, 261, 245, 260, 235, 243, 269, 252, 257, 225, 233, 235, 254, 250, 264, 260, 272, 254, 256, 262, 257, 260, 285, 255, 240, 261, 264, 275, 257] payload_rows_per_dst=[5396, 5452, 5392, 5668, 5410, 5574, 5346, 5512] dup_rows_per_dst=[2630, 2714, 2658, 2796, 2732, 2768, 2656, 2832] combine_rows_per_src=[5396, 5452, 5392, 5668, 5410, 5574, 5346, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155705344 dispatch_meta_bytes=227264 combine_payload_bytes=155705344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155932608 ingress_bytes=174579712 link_bytes=174579712 token_bytes=311637952 weight_movement_bytes=18874368 total_nvlink_bytes=330512320 weight_to_token_ratio=0.060565 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672736902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38350 dup_rows=21706 dup_rows_remote=18962 dedup_ratio=0.331207 router_tokens_per_expert=[246, 235, 241, 270, 236, 255, 231, 243, 232, 260, 253, 260, 248, 261, 254, 228, 259, 259, 247, 262, 270, 254, 250, 255, 257, 275, 262, 257, 253, 235, 261, 248, 232, 266, 276, 239, 258, 245, 261, 273, 250, 251, 262, 248, 261, 247, 256, 252, 250, 252, 288, 265, 255, 259, 238, 274, 238, 261, 259, 260, 261, 265, 255, 248, 252, 240, 295, 253, 252, 262, 259, 264, 246, 252, 270, 255, 296, 259, 258, 256, 257, 279, 264, 247, 252, 265, 257, 235, 255, 261, 235, 270, 260, 257, 281, 274, 246, 259, 261, 253, 250, 242, 269, 250, 255, 256, 256, 270, 235, 272, 279, 259, 273, 240, 248, 258, 251, 257, 275, 237, 255, 254, 285, 249, 241, 260, 234, 259] payload_rows_per_dst=[5376, 5514, 5460, 5490, 5546, 5518, 5480, 5446] dup_rows_per_dst=[2530, 2694, 2694, 2766, 2792, 2780, 2744, 2706] combine_rows_per_src=[5376, 5514, 5460, 5490, 5546, 5518, 5480, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229088 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737248 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245408 weight_movement_bytes=18874368 total_nvlink_bytes=332119776 weight_to_token_ratio=0.060254 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228768 combine_payload_bytes=156622848 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156851616 ingress_bytes=213245952 link_bytes=213245952 token_bytes=313474464 weight_movement_bytes=56623104 total_nvlink_bytes=370097568 weight_to_token_ratio=0.180631 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043217789e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43982 payload_rows_remote=38380 dup_rows=21554 dup_rows_remote=18742 dedup_ratio=0.328888 router_tokens_per_expert=[265, 266, 262, 243, 272, 242, 263, 242, 256, 270, 269, 287, 234, 271, 270, 250, 258, 274, 249, 241, 228, 284, 268, 243, 285, 259, 252, 240, 235, 237, 261, 288, 263, 253, 244, 241, 280, 276, 256, 256, 249, 271, 253, 247, 248, 269, 239, 244, 271, 268, 290, 225, 246, 275, 277, 291, 269, 266, 244, 237, 223, 238, 243, 272, 268, 252, 239, 261, 251, 234, 241, 254, 246, 277, 265, 293, 262, 243, 250, 257, 264, 276, 220, 281, 282, 232, 278, 235, 300, 240, 279, 267, 253, 251, 273, 276, 253, 243, 232, 280, 209, 250, 241, 246, 238, 272, 254, 224, 250, 244, 276, 247, 229, 228, 254, 246, 248, 250, 260, 247, 248, 278, 249, 264, 250, 246, 264, 260] payload_rows_per_dst=[5580, 5434, 5496, 5546, 5490, 5602, 5428, 5406] dup_rows_per_dst=[2744, 2770, 2682, 2724, 2696, 2812, 2490, 2636] combine_rows_per_src=[5580, 5434, 5496, 5546, 5490, 5602, 5428, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38406 dup_rows=21640 dup_rows_remote=18930 dedup_ratio=0.330200 router_tokens_per_expert=[270, 256, 236, 248, 230, 259, 270, 248, 242, 275, 305, 231, 273, 256, 261, 282, 269, 268, 251, 247, 265, 257, 253, 241, 247, 236, 262, 235, 262, 272, 248, 261, 223, 243, 244, 276, 252, 272, 241, 260, 254, 281, 267, 250, 263, 267, 279, 267, 261, 244, 254, 263, 233, 272, 236, 264, 260, 264, 262, 282, 235, 257, 259, 254, 246, 256, 280, 254, 241, 266, 271, 260, 265, 249, 263, 254, 238, 279, 234, 278, 254, 276, 275, 273, 225, 251, 258, 265, 254, 242, 265, 251, 250, 287, 253, 235, 239, 268, 250, 218, 262, 261, 247, 256, 224, 240, 256, 278, 246, 250, 243, 278, 275, 244, 238, 270, 256, 255, 266, 228, 248, 270, 267, 250, 248, 253, 219, 262] payload_rows_per_dst=[5498, 5434, 5530, 5490, 5546, 5522, 5392, 5484] dup_rows_per_dst=[2786, 2714, 2748, 2710, 2722, 2706, 2640, 2614] combine_rows_per_src=[5498, 5434, 5530, 5490, 5546, 5522, 5392, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229248 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157310848 ingress_bytes=194830336 link_bytes=194830336 token_bytes=314392448 weight_movement_bytes=37748736 total_nvlink_bytes=352141184 weight_to_token_ratio=0.120069 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866768417e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38332 dup_rows=21738 dup_rows_remote=19054 dedup_ratio=0.331696 router_tokens_per_expert=[264, 240, 278, 249, 257, 267, 253, 254, 260, 272, 238, 257, 282, 254, 235, 266, 259, 261, 286, 265, 247, 262, 250, 275, 278, 256, 241, 239, 258, 273, 274, 247, 259, 268, 264, 252, 256, 254, 230, 279, 273, 257, 266, 267, 248, 239, 240, 223, 251, 263, 233, 238, 240, 274, 258, 246, 266, 237, 263, 247, 256, 265, 262, 258, 266, 259, 254, 262, 262, 253, 239, 274, 258, 246, 242, 233, 262, 237, 245, 260, 288, 259, 260, 272, 247, 236, 276, 245, 254, 255, 256, 249, 240, 265, 234, 260, 245, 282, 255, 273, 224, 255, 286, 243, 217, 247, 214, 254, 274, 247, 273, 244, 249, 251, 260, 268, 242, 266, 278, 244, 270, 261, 290, 274, 269, 240, 255, 241] payload_rows_per_dst=[5546, 5522, 5466, 5348, 5414, 5536, 5428, 5538] dup_rows_per_dst=[2706, 2820, 2684, 2766, 2690, 2656, 2638, 2778] combine_rows_per_src=[5546, 5522, 5466, 5348, 5414, 5536, 5428, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43466 payload_rows_remote=38094 dup_rows=22070 dup_rows_remote=19312 dedup_ratio=0.336761 router_tokens_per_expert=[262, 250, 272, 268, 232, 281, 275, 255, 261, 240, 236, 288, 239, 254, 271, 251, 254, 277, 261, 257, 256, 231, 231, 235, 247, 258, 259, 225, 249, 265, 283, 262, 277, 255, 256, 236, 250, 258, 254, 245, 261, 272, 260, 250, 264, 266, 269, 250, 243, 252, 264, 275, 246, 262, 233, 258, 276, 234, 263, 245, 225, 252, 240, 274, 238, 258, 247, 252, 255, 242, 279, 249, 228, 245, 254, 258, 265, 269, 268, 258, 253, 283, 243, 275, 261, 278, 243, 257, 223, 247, 254, 275, 279, 242, 220, 286, 243, 277, 264, 244, 258, 283, 251, 286, 250, 259, 249, 261, 262, 251, 284, 261, 251, 223, 282, 245, 258, 255, 267, 240, 274, 246, 263, 254, 261, 226, 243, 263] payload_rows_per_dst=[5460, 5448, 5466, 5374, 5372, 5424, 5524, 5398] dup_rows_per_dst=[2810, 2652, 2780, 2710, 2758, 2814, 2842, 2704] combine_rows_per_src=[5460, 5448, 5466, 5374, 5372, 5424, 5524, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=228488 combine_payload_bytes=157204480 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157432968 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314637448 weight_movement_bytes=18874368 total_nvlink_bytes=333511816 weight_to_token_ratio=0.059988 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157310976 dispatch_meta_bytes=229344 combine_payload_bytes=157310976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157540320 ingress_bytes=176185344 link_bytes=176185344 token_bytes=314851296 weight_movement_bytes=18874368 total_nvlink_bytes=333725664 weight_to_token_ratio=0.059947 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688121278e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38450 dup_rows=21608 dup_rows_remote=18962 dedup_ratio=0.329712 router_tokens_per_expert=[263, 246, 216, 263, 221, 279, 284, 261, 241, 254, 258, 254, 282, 221, 247, 231, 236, 264, 252, 242, 230, 270, 241, 248, 278, 253, 264, 263, 270, 271, 236, 246, 262, 257, 258, 282, 243, 264, 221, 271, 238, 258, 263, 258, 258, 262, 247, 262, 244, 247, 257, 259, 277, 281, 248, 238, 265, 268, 249, 286, 253, 259, 253, 254, 235, 254, 239, 266, 256, 261, 229, 272, 252, 262, 269, 251, 287, 254, 232, 233, 256, 261, 269, 271, 248, 251, 253, 253, 281, 246, 239, 260, 263, 259, 239, 252, 243, 260, 269, 258, 248, 272, 262, 252, 253, 231, 283, 245, 283, 268, 279, 280, 263, 250, 266, 249, 239, 234, 264, 239, 262, 250, 289, 294, 251, 258, 229, 265] payload_rows_per_dst=[5478, 5470, 5508, 5480, 5520, 5718, 5318, 5436] dup_rows_per_dst=[2646, 2658, 2700, 2796, 2584, 3050, 2488, 2686] combine_rows_per_src=[5478, 5470, 5508, 5480, 5520, 5718, 5318, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229544 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237416 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245288 weight_movement_bytes=18874368 total_nvlink_bytes=333119656 weight_to_token_ratio=0.060063 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=229624 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156262648 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312295672 weight_movement_bytes=18874368 total_nvlink_bytes=331170040 weight_to_token_ratio=0.060437 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38336 dup_rows=21822 dup_rows_remote=19216 dedup_ratio=0.332977 router_tokens_per_expert=[262, 248, 248, 271, 264, 253, 266, 254, 272, 269, 258, 262, 251, 241, 261, 262, 229, 259, 252, 240, 264, 221, 254, 241, 277, 264, 246, 248, 258, 254, 248, 237, 277, 245, 261, 263, 253, 250, 263, 253, 253, 221, 252, 230, 255, 253, 261, 285, 286, 255, 273, 249, 259, 239, 293, 243, 266, 257, 247, 213, 239, 248, 261, 234, 272, 260, 278, 249, 242, 265, 252, 267, 257, 249, 271, 266, 260, 269, 250, 253, 235, 260, 273, 290, 281, 237, 243, 258, 243, 262, 254, 260, 250, 238, 258, 257, 263, 250, 244, 258, 289, 250, 257, 257, 258, 241, 236, 261, 242, 246, 284, 262, 258, 278, 277, 251, 241, 258, 242, 263, 259, 250, 250, 257, 293, 267, 274, 222] payload_rows_per_dst=[5490, 5378, 5404, 5494, 5526, 5426, 5452, 5544] dup_rows_per_dst=[2794, 2606, 2746, 2630, 2794, 2772, 2744, 2736] combine_rows_per_src=[5490, 5378, 5404, 5494, 5526, 5426, 5452, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38170 dup_rows=21828 dup_rows_remote=19026 dedup_ratio=0.333069 router_tokens_per_expert=[291, 250, 248, 276, 266, 248, 245, 275, 275, 248, 252, 254, 264, 249, 244, 257, 246, 263, 270, 219, 288, 275, 271, 250, 238, 272, 261, 274, 224, 278, 249, 248, 274, 256, 252, 246, 270, 245, 223, 267, 255, 258, 246, 259, 281, 227, 256, 268, 275, 260, 228, 282, 249, 247, 236, 245, 256, 224, 269, 257, 262, 269, 269, 225, 264, 254, 249, 248, 259, 242, 274, 272, 249, 255, 242, 239, 251, 277, 291, 262, 246, 254, 267, 236, 246, 253, 264, 244, 295, 241, 231, 243, 268, 233, 263, 226, 233, 281, 268, 210, 272, 242, 254, 251, 262, 248, 266, 264, 283, 251, 233, 254, 247, 236, 251, 260, 253, 246, 269, 247, 261, 276, 285, 250, 265, 258, 275, 275] payload_rows_per_dst=[5406, 5512, 5396, 5568, 5410, 5422, 5456, 5538] dup_rows_per_dst=[2580, 2782, 2778, 2778, 2636, 2784, 2688, 2802] combine_rows_per_src=[5406, 5512, 5396, 5568, 5410, 5422, 5456, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=229648 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157720848 ingress_bytes=195239936 link_bytes=195239936 token_bytes=315212048 weight_movement_bytes=37748736 total_nvlink_bytes=352960784 weight_to_token_ratio=0.119757 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870693003e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38254 dup_rows=21802 dup_rows_remote=19130 dedup_ratio=0.332672 router_tokens_per_expert=[260, 255, 268, 249, 258, 281, 249, 253, 243, 265, 239, 245, 242, 234, 275, 269, 250, 267, 274, 276, 263, 265, 252, 222, 248, 243, 242, 254, 254, 260, 242, 234, 269, 294, 254, 242, 243, 243, 289, 260, 231, 264, 296, 264, 266, 233, 251, 255, 268, 254, 259, 244, 278, 276, 251, 253, 262, 262, 254, 243, 257, 242, 253, 251, 247, 276, 261, 254, 277, 279, 248, 261, 253, 262, 256, 258, 255, 266, 234, 231, 238, 277, 278, 297, 264, 244, 247, 263, 213, 238, 253, 251, 218, 265, 249, 264, 290, 225, 254, 258, 249, 228, 257, 256, 258, 249, 267, 241, 275, 258, 269, 242, 229, 292, 224, 259, 261, 252, 275, 246, 252, 244, 269, 258, 276, 270, 265, 251] payload_rows_per_dst=[5450, 5438, 5502, 5538, 5430, 5424, 5480, 5472] dup_rows_per_dst=[2720, 2654, 2806, 2676, 2806, 2694, 2672, 2774] combine_rows_per_src=[5450, 5438, 5502, 5538, 5430, 5424, 5480, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38168 dup_rows=21816 dup_rows_remote=18976 dedup_ratio=0.332886 router_tokens_per_expert=[249, 240, 253, 259, 241, 250, 250, 249, 248, 294, 253, 269, 260, 244, 287, 276, 270, 261, 248, 239, 273, 250, 283, 241, 253, 255, 243, 229, 262, 251, 240, 265, 258, 255, 259, 270, 236, 248, 243, 235, 259, 262, 244, 265, 238, 272, 251, 270, 260, 228, 269, 263, 247, 246, 240, 300, 258, 294, 241, 257, 253, 219, 250, 274, 281, 248, 268, 266, 261, 245, 244, 253, 245, 243, 262, 267, 266, 257, 247, 270, 269, 299, 253, 283, 258, 234, 273, 257, 216, 255, 279, 254, 265, 261, 271, 269, 267, 262, 259, 250, 276, 263, 241, 236, 270, 247, 291, 239, 269, 243, 255, 256, 271, 251, 215, 248, 226, 261, 254, 240, 254, 267, 237, 249, 254, 251, 239, 259] payload_rows_per_dst=[5508, 5418, 5474, 5438, 5448, 5552, 5496, 5386] dup_rows_per_dst=[2736, 2708, 2656, 2760, 2798, 2840, 2752, 2566] combine_rows_per_src=[5508, 5418, 5474, 5438, 5448, 5552, 5496, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=230208 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157254464 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314278720 weight_movement_bytes=18874368 total_nvlink_bytes=333153088 weight_to_token_ratio=0.060056 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=228784 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156573104 ingress_bytes=194093056 link_bytes=194093056 token_bytes=312917424 weight_movement_bytes=37748736 total_nvlink_bytes=350666160 weight_to_token_ratio=0.120635 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859704163e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43534 payload_rows_remote=38156 dup_rows=22002 dup_rows_remote=19344 dedup_ratio=0.335724 router_tokens_per_expert=[247, 253, 266, 283, 282, 248, 266, 281, 262, 277, 261, 254, 277, 262, 276, 259, 269, 241, 307, 274, 257, 269, 295, 244, 257, 237, 234, 259, 238, 256, 288, 217, 265, 244, 261, 243, 266, 253, 258, 260, 235, 239, 231, 247, 254, 251, 251, 261, 254, 277, 258, 268, 260, 251, 245, 242, 255, 255, 258, 231, 243, 214, 250, 257, 236, 261, 233, 270, 244, 248, 265, 270, 271, 262, 250, 249, 255, 281, 256, 261, 231, 269, 278, 256, 247, 258, 247, 269, 266, 251, 246, 243, 262, 251, 257, 264, 267, 233, 218, 278, 286, 285, 258, 238, 260, 255, 261, 250, 262, 269, 239, 248, 280, 221, 252, 258, 234, 232, 241, 254, 251, 251, 261, 272, 237, 270, 262, 245] payload_rows_per_dst=[5564, 5468, 5368, 5378, 5470, 5422, 5544, 5320] dup_rows_per_dst=[2944, 2816, 2670, 2658, 2754, 2768, 2670, 2722] combine_rows_per_src=[5564, 5468, 5368, 5378, 5470, 5422, 5544, 5320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38214 dup_rows=21906 dup_rows_remote=19210 dedup_ratio=0.334259 router_tokens_per_expert=[276, 255, 260, 259, 261, 254, 255, 230, 280, 269, 250, 260, 238, 265, 263, 266, 250, 256, 247, 250, 264, 260, 226, 273, 244, 237, 274, 285, 244, 220, 280, 228, 251, 260, 285, 260, 252, 260, 234, 225, 243, 228, 269, 257, 234, 278, 267, 253, 261, 265, 261, 250, 253, 263, 270, 266, 267, 242, 252, 263, 269, 273, 268, 271, 268, 232, 237, 235, 266, 256, 255, 261, 257, 297, 243, 257, 246, 285, 243, 262, 262, 272, 268, 239, 262, 237, 263, 236, 252, 247, 235, 276, 242, 262, 236, 237, 271, 286, 255, 258, 251, 246, 218, 241, 259, 257, 245, 251, 294, 272, 254, 272, 254, 250, 255, 252, 263, 279, 261, 234, 263, 241, 234, 251, 262, 251, 261, 272] payload_rows_per_dst=[5454, 5358, 5416, 5570, 5492, 5346, 5496, 5498] dup_rows_per_dst=[2828, 2718, 2696, 2818, 2708, 2706, 2764, 2668] combine_rows_per_src=[5454, 5358, 5416, 5570, 5492, 5346, 5496, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229536 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917920 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606304 weight_movement_bytes=18874368 total_nvlink_bytes=332480672 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228576 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156564704 ingress_bytes=194084864 link_bytes=194084864 token_bytes=312900832 weight_movement_bytes=37748736 total_nvlink_bytes=350649568 weight_to_token_ratio=0.120641 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859625671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38230 dup_rows=21884 dup_rows_remote=19324 dedup_ratio=0.333923 router_tokens_per_expert=[266, 266, 276, 276, 256, 288, 246, 245, 236, 264, 276, 243, 270, 250, 270, 244, 253, 269, 232, 274, 258, 253, 262, 270, 245, 264, 238, 247, 247, 257, 261, 285, 232, 269, 251, 273, 259, 262, 269, 256, 239, 222, 285, 252, 271, 270, 244, 251, 260, 254, 259, 241, 257, 252, 254, 253, 241, 254, 270, 253, 250, 266, 273, 264, 266, 262, 268, 227, 231, 259, 255, 256, 266, 258, 228, 251, 247, 245, 230, 242, 243, 246, 256, 274, 274, 259, 279, 282, 264, 253, 232, 246, 249, 253, 247, 255, 255, 265, 245, 265, 276, 228, 277, 228, 242, 280, 224, 255, 244, 270, 241, 262, 253, 265, 236, 266, 249, 239, 253, 277, 237, 246, 275, 263, 266, 252, 272, 266] payload_rows_per_dst=[5484, 5468, 5516, 5404, 5422, 5494, 5404, 5460] dup_rows_per_dst=[2860, 2762, 2694, 2798, 2560, 2730, 2710, 2770] combine_rows_per_src=[5484, 5468, 5516, 5404, 5422, 5494, 5404, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=230000 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156516976 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312803952 weight_movement_bytes=18874368 total_nvlink_bytes=331678320 weight_to_token_ratio=0.060339 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43922 payload_rows_remote=38394 dup_rows=21614 dup_rows_remote=18888 dedup_ratio=0.329803 router_tokens_per_expert=[294, 256, 251, 241, 271, 266, 242, 277, 252, 250, 242, 225, 234, 251, 239, 265, 258, 244, 266, 257, 236, 248, 264, 243, 235, 269, 252, 264, 239, 254, 278, 237, 250, 297, 265, 248, 260, 259, 267, 268, 269, 239, 260, 264, 250, 239, 285, 235, 278, 264, 262, 239, 257, 217, 247, 255, 254, 249, 238, 265, 262, 270, 241, 241, 246, 280, 276, 275, 255, 248, 249, 272, 247, 248, 256, 262, 240, 259, 275, 248, 263, 271, 256, 267, 237, 217, 252, 264, 296, 272, 221, 257, 249, 286, 271, 254, 269, 267, 262, 266, 266, 247, 261, 259, 232, 239, 265, 254, 246, 255, 253, 266, 258, 267, 266, 240, 252, 257, 265, 274, 255, 242, 220, 264, 275, 268, 236, 259] payload_rows_per_dst=[5528, 5624, 5684, 5382, 5564, 5300, 5350, 5490] dup_rows_per_dst=[2726, 2996, 3146, 2558, 2704, 2446, 2332, 2706] combine_rows_per_src=[5528, 5624, 5684, 5382, 5564, 5300, 5350, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229696 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156754240 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278784 weight_movement_bytes=18874368 total_nvlink_bytes=332153152 weight_to_token_ratio=0.060248 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38120 dup_rows=21808 dup_rows_remote=18958 dedup_ratio=0.332764 router_tokens_per_expert=[245, 238, 242, 260, 250, 253, 267, 271, 281, 274, 254, 261, 276, 266, 240, 258, 248, 239, 236, 241, 263, 287, 265, 253, 255, 272, 270, 283, 268, 255, 261, 261, 233, 263, 266, 254, 261, 257, 255, 237, 245, 235, 251, 249, 255, 229, 281, 259, 236, 250, 229, 271, 238, 273, 234, 256, 260, 260, 220, 279, 236, 254, 270, 244, 264, 283, 255, 273, 254, 235, 237, 253, 255, 282, 231, 245, 264, 272, 249, 246, 279, 277, 276, 247, 250, 229, 253, 267, 251, 265, 240, 249, 290, 238, 244, 233, 246, 261, 244, 259, 254, 270, 209, 246, 259, 250, 268, 275, 265, 261, 254, 275, 250, 275, 246, 251, 262, 247, 222, 274, 236, 275, 273, 260, 279, 252, 282, 269] payload_rows_per_dst=[5554, 5608, 5468, 5410, 5438, 5398, 5364, 5488] dup_rows_per_dst=[2718, 2850, 2592, 2610, 2758, 2778, 2684, 2818] combine_rows_per_src=[5554, 5608, 5468, 5410, 5438, 5398, 5364, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38226 dup_rows=21846 dup_rows_remote=19080 dedup_ratio=0.333344 router_tokens_per_expert=[237, 264, 269, 240, 276, 247, 245, 243, 256, 238, 256, 242, 238, 260, 264, 244, 262, 266, 249, 219, 260, 249, 263, 267, 253, 276, 263, 256, 249, 277, 256, 249, 281, 248, 260, 253, 276, 261, 244, 253, 276, 251, 252, 253, 246, 237, 280, 245, 251, 250, 251, 270, 251, 226, 256, 225, 247, 248, 235, 282, 239, 238, 266, 237, 237, 289, 274, 236, 250, 280, 260, 244, 254, 250, 273, 280, 266, 238, 251, 258, 264, 263, 254, 259, 250, 278, 296, 269, 262, 260, 251, 273, 247, 231, 272, 254, 257, 250, 256, 236, 255, 278, 265, 248, 248, 251, 255, 261, 255, 255, 276, 275, 262, 266, 242, 259, 255, 257, 245, 271, 239, 250, 283, 275, 259, 242, 239, 259] payload_rows_per_dst=[5390, 5500, 5540, 5372, 5440, 5472, 5512, 5464] dup_rows_per_dst=[2648, 2728, 2692, 2572, 2816, 2894, 2730, 2766] combine_rows_per_src=[5390, 5500, 5540, 5372, 5440, 5472, 5512, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=230216 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156820296 ingress_bytes=213213184 link_bytes=213213184 token_bytes=313410376 weight_movement_bytes=56623104 total_nvlink_bytes=370033480 weight_to_token_ratio=0.180668 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042903822e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229128 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157490952 ingress_bytes=176136192 link_bytes=176136192 token_bytes=314752776 weight_movement_bytes=18874368 total_nvlink_bytes=333627144 weight_to_token_ratio=0.059966 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687650327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38160 dup_rows=21748 dup_rows_remote=18938 dedup_ratio=0.331848 router_tokens_per_expert=[264, 261, 231, 273, 250, 248, 241, 278, 258, 249, 253, 255, 241, 268, 254, 257, 237, 252, 244, 247, 241, 245, 244, 248, 250, 225, 234, 253, 238, 232, 234, 258, 257, 259, 252, 250, 233, 270, 267, 264, 273, 269, 257, 252, 259, 280, 228, 230, 255, 241, 264, 261, 238, 246, 276, 271, 258, 265, 257, 241, 239, 264, 282, 273, 240, 260, 253, 278, 230, 258, 274, 268, 268, 230, 251, 266, 251, 269, 279, 266, 251, 246, 249, 259, 273, 226, 259, 252, 245, 279, 245, 244, 244, 285, 272, 276, 264, 255, 278, 274, 262, 274, 245, 264, 236, 284, 276, 251, 252, 264, 256, 284, 274, 249, 253, 260, 255, 237, 254, 260, 245, 251, 263, 272, 277, 261, 252, 246] payload_rows_per_dst=[5472, 5294, 5438, 5528, 5486, 5444, 5628, 5498] dup_rows_per_dst=[2690, 2470, 2762, 2734, 2796, 2766, 2810, 2720] combine_rows_per_src=[5472, 5294, 5438, 5528, 5486, 5444, 5628, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228312 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156367832 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507352 weight_movement_bytes=18874368 total_nvlink_bytes=331381720 weight_to_token_ratio=0.060397 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229224 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156802920 ingress_bytes=213196800 link_bytes=213196800 token_bytes=313376616 weight_movement_bytes=56623104 total_nvlink_bytes=369999720 weight_to_token_ratio=0.180687 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042746838e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43520 payload_rows_remote=38088 dup_rows=22016 dup_rows_remote=19178 dedup_ratio=0.335938 router_tokens_per_expert=[245, 266, 249, 225, 246, 248, 242, 257, 254, 261, 263, 261, 237, 251, 234, 259, 246, 237, 238, 254, 261, 268, 229, 277, 281, 256, 226, 253, 249, 263, 238, 264, 266, 278, 281, 259, 275, 276, 267, 267, 265, 269, 259, 248, 259, 247, 254, 246, 269, 264, 277, 232, 244, 252, 265, 246, 249, 271, 258, 243, 252, 281, 252, 246, 239, 265, 255, 254, 266, 255, 247, 265, 244, 271, 268, 249, 266, 269, 237, 243, 257, 255, 254, 265, 260, 245, 260, 261, 250, 268, 268, 260, 265, 256, 263, 248, 264, 261, 250, 261, 273, 255, 259, 228, 251, 265, 270, 246, 254, 248, 254, 275, 263, 247, 280, 227, 230, 275, 254, 243, 261, 254, 261, 270, 248, 272, 242, 244] payload_rows_per_dst=[5372, 5406, 5522, 5470, 5376, 5432, 5450, 5492] dup_rows_per_dst=[2624, 2674, 2910, 2732, 2810, 2838, 2778, 2650] combine_rows_per_src=[5372, 5406, 5522, 5470, 5376, 5432, 5450, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38292 dup_rows=21704 dup_rows_remote=18882 dedup_ratio=0.331177 router_tokens_per_expert=[219, 258, 235, 270, 253, 256, 266, 238, 230, 223, 259, 237, 252, 264, 250, 274, 271, 255, 242, 250, 259, 244, 253, 249, 253, 295, 249, 277, 208, 252, 249, 231, 265, 245, 236, 255, 260, 269, 283, 265, 261, 269, 254, 240, 263, 277, 235, 251, 271, 290, 270, 228, 263, 276, 264, 253, 264, 259, 270, 248, 240, 256, 264, 265, 268, 250, 257, 242, 240, 260, 219, 253, 234, 270, 278, 280, 264, 274, 259, 224, 254, 261, 257, 215, 242, 260, 262, 247, 242, 224, 254, 261, 268, 246, 265, 244, 267, 271, 264, 245, 253, 258, 261, 244, 290, 274, 252, 253, 262, 256, 257, 263, 266, 266, 255, 265, 269, 254, 272, 255, 251, 235, 241, 255, 300, 247, 300, 263] payload_rows_per_dst=[5370, 5434, 5506, 5540, 5570, 5348, 5534, 5530] dup_rows_per_dst=[2598, 2640, 2750, 2822, 2574, 2656, 2806, 2858] combine_rows_per_src=[5370, 5434, 5506, 5540, 5570, 5348, 5534, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228392 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156531752 ingress_bytes=212926464 link_bytes=212926464 token_bytes=312835112 weight_movement_bytes=56623104 total_nvlink_bytes=369458216 weight_to_token_ratio=0.181000 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040156612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38204 dup_rows=21826 dup_rows_remote=18932 dedup_ratio=0.333038 router_tokens_per_expert=[299, 279, 288, 277, 278, 257, 270, 242, 249, 272, 247, 252, 243, 270, 245, 247, 264, 221, 267, 255, 238, 251, 267, 250, 250, 234, 235, 267, 267, 252, 231, 255, 264, 265, 279, 233, 262, 263, 263, 289, 279, 245, 248, 265, 242, 253, 279, 271, 243, 251, 278, 260, 262, 251, 243, 257, 241, 248, 243, 274, 268, 249, 246, 244, 253, 268, 234, 224, 260, 261, 283, 235, 256, 240, 251, 247, 241, 272, 253, 246, 249, 264, 247, 269, 230, 258, 242, 262, 251, 247, 260, 273, 243, 253, 276, 273, 271, 278, 242, 256, 266, 249, 248, 244, 259, 240, 276, 254, 275, 266, 251, 236, 256, 246, 254, 252, 269, 247, 241, 267, 272, 276, 231, 256, 250, 239, 254, 249] payload_rows_per_dst=[5488, 5438, 5506, 5482, 5390, 5472, 5494, 5440] dup_rows_per_dst=[2942, 2570, 2894, 2634, 2658, 2722, 2728, 2678] combine_rows_per_src=[5488, 5438, 5506, 5482, 5390, 5472, 5494, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=229064 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156237512 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312245960 weight_movement_bytes=18874368 total_nvlink_bytes=331120328 weight_to_token_ratio=0.060447 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38392 dup_rows=21666 dup_rows_remote=18864 dedup_ratio=0.330597 router_tokens_per_expert=[276, 248, 245, 267, 238, 263, 241, 251, 228, 263, 246, 285, 259, 264, 243, 280, 257, 257, 259, 263, 239, 279, 268, 259, 248, 247, 246, 283, 235, 247, 245, 232, 263, 237, 241, 259, 288, 246, 237, 272, 238, 260, 262, 236, 246, 241, 273, 250, 240, 255, 231, 253, 239, 234, 267, 246, 255, 238, 274, 238, 261, 227, 234, 245, 276, 277, 247, 277, 288, 216, 273, 235, 240, 267, 284, 258, 258, 240, 247, 257, 258, 242, 257, 261, 272, 264, 280, 252, 247, 275, 251, 232, 261, 283, 244, 247, 257, 258, 265, 265, 253, 265, 250, 262, 271, 271, 261, 285, 243, 261, 275, 251, 282, 244, 267, 248, 272, 262, 268, 263, 257, 281, 264, 240, 247, 268, 270, 229] payload_rows_per_dst=[5480, 5512, 5480, 5334, 5478, 5516, 5526, 5544] dup_rows_per_dst=[2714, 2616, 2618, 2540, 2802, 2736, 2860, 2780] combine_rows_per_src=[5480, 5512, 5480, 5334, 5478, 5516, 5526, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=228696 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157072728 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313916760 weight_movement_bytes=18874368 total_nvlink_bytes=332791128 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228544 combine_payload_bytes=156483584 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156712128 ingress_bytes=213106688 link_bytes=213106688 token_bytes=313195712 weight_movement_bytes=56623104 total_nvlink_bytes=369818816 weight_to_token_ratio=0.180791 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041883429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38204 dup_rows=21836 dup_rows_remote=19168 dedup_ratio=0.333191 router_tokens_per_expert=[252, 261, 265, 248, 259, 224, 281, 246, 263, 279, 235, 263, 237, 259, 265, 245, 259, 254, 254, 259, 277, 235, 236, 264, 233, 231, 226, 238, 228, 266, 270, 277, 256, 256, 259, 245, 242, 239, 251, 267, 246, 267, 260, 288, 249, 256, 264, 261, 250, 252, 240, 283, 236, 247, 248, 263, 250, 243, 278, 260, 250, 247, 275, 260, 246, 262, 285, 260, 302, 244, 236, 261, 233, 252, 257, 290, 246, 251, 228, 276, 278, 255, 253, 268, 263, 234, 268, 261, 279, 260, 233, 244, 242, 264, 226, 244, 256, 252, 265, 264, 240, 253, 290, 248, 248, 266, 290, 250, 250, 268, 258, 260, 267, 242, 246, 276, 250, 252, 243, 250, 269, 260, 263, 289, 283, 254, 246, 242] payload_rows_per_dst=[5496, 5554, 5496, 5436, 5512, 5432, 5296, 5478] dup_rows_per_dst=[2668, 2860, 2716, 2728, 2746, 2712, 2620, 2786] combine_rows_per_src=[5496, 5554, 5496, 5436, 5512, 5432, 5296, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38410 dup_rows=21646 dup_rows_remote=18950 dedup_ratio=0.330292 router_tokens_per_expert=[256, 249, 262, 235, 275, 247, 265, 262, 258, 254, 246, 241, 274, 251, 260, 249, 262, 280, 265, 257, 253, 243, 249, 266, 267, 245, 235, 238, 263, 245, 259, 245, 224, 252, 257, 232, 248, 270, 268, 252, 262, 241, 246, 231, 273, 252, 261, 260, 265, 220, 263, 293, 279, 289, 248, 261, 225, 256, 271, 260, 258, 246, 251, 252, 233, 250, 255, 256, 237, 271, 273, 255, 256, 248, 262, 259, 270, 257, 255, 263, 260, 262, 255, 256, 273, 253, 226, 285, 244, 247, 266, 264, 284, 251, 275, 283, 253, 253, 240, 244, 235, 260, 255, 284, 263, 258, 252, 244, 272, 248, 236, 265, 267, 262, 269, 249, 240, 252, 256, 243, 274, 280, 260, 248, 237, 266, 265, 232] payload_rows_per_dst=[5492, 5480, 5466, 5506, 5526, 5490, 5418, 5512] dup_rows_per_dst=[2676, 2696, 2592, 2768, 2674, 2846, 2706, 2688] combine_rows_per_src=[5492, 5480, 5466, 5506, 5526, 5490, 5418, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38244 dup_rows=21812 dup_rows_remote=18960 dedup_ratio=0.332825 router_tokens_per_expert=[240, 245, 263, 266, 217, 266, 250, 250, 264, 228, 291, 254, 286, 243, 217, 279, 237, 272, 254, 244, 263, 243, 265, 267, 265, 276, 266, 275, 247, 238, 229, 251, 284, 265, 259, 234, 249, 258, 257, 271, 264, 236, 249, 258, 261, 216, 263, 270, 281, 240, 242, 263, 271, 255, 247, 283, 264, 234, 249, 271, 254, 249, 260, 236, 270, 241, 244, 263, 266, 240, 249, 244, 248, 235, 263, 255, 263, 232, 264, 263, 256, 267, 250, 279, 264, 250, 251, 263, 262, 261, 220, 264, 264, 237, 258, 274, 245, 237, 276, 269, 263, 259, 242, 251, 258, 240, 250, 271, 275, 243, 272, 263, 259, 270, 247, 262, 261, 253, 247, 246, 268, 280, 280, 261, 239, 266, 261, 250] payload_rows_per_dst=[5460, 5414, 5392, 5494, 5400, 5494, 5590, 5480] dup_rows_per_dst=[2658, 2770, 2524, 2704, 2682, 2746, 2876, 2852] combine_rows_per_src=[5460, 5414, 5392, 5494, 5400, 5494, 5590, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229024 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157482656 ingress_bytes=213876736 link_bytes=213876736 token_bytes=314736288 weight_movement_bytes=56623104 total_nvlink_bytes=371359392 weight_to_token_ratio=0.179907 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049261650e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38348 dup_rows=21814 dup_rows_remote=19128 dedup_ratio=0.332855 router_tokens_per_expert=[264, 251, 232, 263, 273, 262, 237, 251, 249, 267, 246, 288, 272, 225, 282, 244, 262, 224, 278, 246, 238, 281, 255, 228, 222, 222, 269, 228, 260, 243, 266, 249, 276, 259, 243, 283, 251, 252, 289, 239, 271, 237, 265, 260, 238, 227, 253, 261, 246, 262, 252, 239, 270, 277, 250, 243, 266, 296, 247, 265, 233, 254, 263, 253, 278, 253, 255, 279, 241, 263, 277, 274, 230, 257, 255, 277, 268, 254, 270, 269, 266, 289, 259, 256, 280, 244, 267, 242, 265, 255, 258, 243, 289, 249, 264, 248, 237, 256, 230, 259, 285, 252, 228, 254, 255, 240, 250, 286, 251, 281, 227, 239, 276, 226, 268, 230, 288, 265, 251, 239, 242, 267, 279, 234, 251, 233, 261, 257] payload_rows_per_dst=[5422, 5416, 5526, 5510, 5522, 5504, 5374, 5448] dup_rows_per_dst=[2790, 2526, 2682, 2722, 2878, 2844, 2686, 2686] combine_rows_per_src=[5422, 5416, 5526, 5510, 5522, 5504, 5374, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229488 combine_payload_bytes=156483584 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156713072 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196656 weight_movement_bytes=37748736 total_nvlink_bytes=350945392 weight_to_token_ratio=0.120527 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229440 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157556800 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314884160 weight_movement_bytes=18874368 total_nvlink_bytes=333758528 weight_to_token_ratio=0.059941 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=228816 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876240 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313523664 weight_movement_bytes=37748736 total_nvlink_bytes=351272400 weight_to_token_ratio=0.120402 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38082 dup_rows=21866 dup_rows_remote=19046 dedup_ratio=0.333649 router_tokens_per_expert=[246, 221, 231, 264, 255, 258, 246, 277, 246, 257, 288, 233, 233, 272, 261, 254, 255, 258, 261, 269, 255, 264, 253, 249, 261, 250, 268, 235, 253, 262, 252, 279, 245, 242, 262, 285, 245, 288, 266, 250, 234, 244, 231, 247, 253, 242, 256, 249, 266, 262, 252, 310, 238, 260, 234, 250, 285, 251, 233, 262, 225, 232, 232, 248, 260, 260, 259, 276, 247, 248, 248, 264, 262, 252, 261, 249, 253, 250, 253, 237, 275, 251, 245, 238, 251, 272, 262, 270, 234, 246, 279, 291, 253, 275, 267, 295, 237, 261, 261, 263, 238, 256, 236, 249, 235, 257, 262, 246, 264, 285, 255, 259, 250, 267, 247, 270, 259, 257, 273, 253, 264, 264, 257, 278, 269, 242, 282, 244] payload_rows_per_dst=[5362, 5458, 5498, 5392, 5440, 5588, 5406, 5526] dup_rows_per_dst=[2722, 2790, 2580, 2688, 2718, 2820, 2722, 2826] combine_rows_per_src=[5362, 5458, 5498, 5392, 5440, 5588, 5406, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38162 dup_rows=21858 dup_rows_remote=19114 dedup_ratio=0.333527 router_tokens_per_expert=[244, 246, 246, 241, 266, 241, 252, 265, 247, 235, 254, 237, 260, 281, 264, 189, 291, 261, 240, 271, 268, 258, 280, 248, 225, 230, 290, 245, 233, 241, 239, 268, 252, 262, 252, 285, 252, 262, 247, 240, 302, 286, 240, 279, 251, 283, 249, 219, 294, 249, 235, 230, 289, 255, 253, 247, 257, 283, 250, 261, 260, 256, 271, 240, 271, 257, 274, 235, 243, 256, 241, 239, 267, 258, 247, 245, 247, 295, 245, 268, 258, 284, 255, 248, 231, 271, 262, 263, 286, 256, 265, 254, 230, 271, 257, 259, 221, 297, 282, 261, 261, 255, 244, 262, 268, 237, 259, 265, 221, 269, 249, 260, 259, 249, 235, 261, 269, 242, 263, 226, 273, 251, 254, 241, 246, 238, 292, 273] payload_rows_per_dst=[5288, 5456, 5506, 5516, 5450, 5518, 5480, 5464] dup_rows_per_dst=[2648, 2720, 2816, 2744, 2726, 2782, 2742, 2680] combine_rows_per_src=[5288, 5456, 5506, 5516, 5450, 5518, 5480, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229904 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303312 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376720 weight_movement_bytes=18874368 total_nvlink_bytes=333251088 weight_to_token_ratio=0.060037 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38122 dup_rows=21976 dup_rows_remote=19236 dedup_ratio=0.335327 router_tokens_per_expert=[235, 234, 261, 269, 240, 278, 234, 268, 259, 242, 266, 247, 265, 267, 257, 236, 251, 265, 238, 216, 263, 243, 256, 241, 242, 291, 260, 280, 280, 244, 247, 248, 253, 270, 260, 268, 264, 247, 248, 253, 246, 279, 223, 218, 246, 260, 268, 286, 264, 264, 246, 253, 239, 260, 228, 248, 260, 267, 274, 241, 267, 240, 255, 256, 238, 241, 253, 245, 285, 246, 237, 252, 272, 261, 261, 249, 238, 247, 275, 238, 291, 266, 260, 245, 264, 270, 272, 224, 279, 257, 256, 250, 271, 263, 247, 280, 284, 270, 257, 259, 278, 250, 274, 226, 252, 266, 284, 222, 241, 224, 265, 257, 258, 223, 253, 238, 282, 259, 281, 247, 256, 248, 265, 261, 247, 288, 272, 274] payload_rows_per_dst=[5440, 5386, 5438, 5440, 5384, 5470, 5462, 5540] dup_rows_per_dst=[2676, 2744, 2740, 2684, 2692, 2920, 2756, 2764] combine_rows_per_src=[5440, 5386, 5438, 5440, 5384, 5470, 5462, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155983872 dispatch_meta_bytes=228512 combine_payload_bytes=155983872 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156212384 ingress_bytes=174858240 link_bytes=174858240 token_bytes=312196256 weight_movement_bytes=18874368 total_nvlink_bytes=331070624 weight_to_token_ratio=0.060457 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675405620e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38160 dup_rows=21908 dup_rows_remote=19236 dedup_ratio=0.334290 router_tokens_per_expert=[255, 259, 258, 241, 253, 261, 264, 257, 273, 254, 254, 260, 253, 271, 252, 269, 259, 243, 251, 250, 237, 259, 266, 245, 258, 249, 251, 257, 249, 239, 240, 278, 281, 284, 239, 252, 275, 269, 262, 265, 254, 248, 262, 271, 254, 292, 265, 236, 234, 247, 259, 253, 232, 281, 280, 249, 251, 266, 255, 276, 254, 235, 256, 281, 229, 247, 268, 258, 270, 251, 259, 243, 277, 256, 275, 252, 216, 262, 266, 241, 261, 256, 263, 253, 241, 273, 254, 263, 265, 247, 230, 263, 268, 214, 242, 253, 251, 255, 277, 254, 257, 226, 256, 241, 263, 241, 242, 266, 251, 242, 260, 256, 255, 229, 222, 303, 249, 295, 249, 252, 287, 277, 255, 270, 234, 233, 264, 257] payload_rows_per_dst=[5402, 5392, 5560, 5500, 5468, 5418, 5418, 5470] dup_rows_per_dst=[2866, 2670, 2858, 2718, 2672, 2674, 2658, 2792] combine_rows_per_src=[5402, 5392, 5560, 5500, 5468, 5418, 5418, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229104 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540656 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312852208 weight_movement_bytes=18874368 total_nvlink_bytes=331726576 weight_to_token_ratio=0.060330 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229432 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156377144 ingress_bytes=193896448 link_bytes=193896448 token_bytes=312524856 weight_movement_bytes=37748736 total_nvlink_bytes=350273592 weight_to_token_ratio=0.120786 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857820362e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38248 dup_rows=21788 dup_rows_remote=19008 dedup_ratio=0.332458 router_tokens_per_expert=[252, 277, 274, 262, 281, 244, 272, 271, 231, 253, 256, 283, 251, 271, 223, 234, 273, 239, 249, 262, 248, 247, 262, 257, 259, 251, 242, 269, 263, 241, 254, 247, 268, 235, 258, 260, 261, 254, 225, 269, 253, 244, 235, 266, 260, 261, 258, 269, 255, 264, 252, 263, 265, 262, 274, 252, 248, 258, 277, 254, 237, 248, 262, 262, 281, 244, 261, 252, 246, 252, 226, 251, 262, 288, 234, 265, 249, 236, 246, 239, 243, 240, 247, 265, 259, 274, 249, 229, 257, 248, 253, 256, 241, 246, 238, 258, 219, 304, 285, 260, 245, 255, 255, 269, 270, 226, 280, 232, 279, 268, 245, 275, 274, 269, 257, 253, 248, 267, 245, 258, 266, 287, 250, 253, 248, 263, 260, 261] payload_rows_per_dst=[5500, 5602, 5452, 5488, 5424, 5230, 5458, 5594] dup_rows_per_dst=[2780, 3036, 2700, 2778, 2662, 2400, 2762, 2670] combine_rows_per_src=[5500, 5602, 5452, 5488, 5424, 5230, 5458, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43550 payload_rows_remote=38156 dup_rows=21986 dup_rows_remote=19306 dedup_ratio=0.335480 router_tokens_per_expert=[273, 248, 246, 257, 272, 270, 251, 240, 258, 279, 256, 235, 241, 271, 250, 272, 255, 269, 269, 254, 230, 242, 258, 225, 246, 241, 274, 260, 255, 246, 284, 229, 247, 223, 274, 250, 284, 276, 241, 287, 244, 275, 256, 257, 266, 284, 262, 248, 245, 257, 267, 257, 269, 248, 253, 254, 232, 258, 240, 261, 255, 266, 241, 275, 254, 232, 248, 253, 270, 286, 252, 262, 259, 265, 257, 233, 252, 236, 264, 245, 247, 252, 269, 257, 267, 267, 249, 254, 255, 273, 250, 267, 248, 281, 227, 282, 246, 261, 214, 251, 255, 247, 252, 257, 231, 243, 253, 260, 267, 294, 236, 301, 256, 253, 253, 260, 248, 268, 253, 259, 257, 233, 225, 237, 268, 279, 270, 260] payload_rows_per_dst=[5432, 5394, 5506, 5392, 5406, 5502, 5474, 5444] dup_rows_per_dst=[2806, 2680, 2842, 2764, 2730, 2788, 2662, 2714] combine_rows_per_src=[5432, 5394, 5506, 5392, 5406, 5502, 5474, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38136 dup_rows=21914 dup_rows_remote=19156 dedup_ratio=0.334381 router_tokens_per_expert=[258, 228, 239, 262, 240, 268, 264, 273, 251, 259, 230, 264, 272, 248, 260, 267, 267, 279, 270, 276, 271, 253, 247, 234, 255, 282, 259, 264, 281, 255, 234, 249, 238, 244, 265, 270, 253, 243, 266, 274, 257, 256, 276, 285, 273, 257, 273, 235, 262, 218, 280, 284, 273, 269, 238, 263, 249, 232, 260, 269, 258, 245, 257, 260, 269, 269, 252, 239, 259, 250, 270, 232, 272, 277, 250, 227, 232, 259, 252, 241, 267, 251, 230, 267, 237, 248, 267, 273, 256, 243, 241, 250, 289, 246, 261, 259, 244, 248, 243, 251, 262, 257, 249, 257, 240, 257, 245, 264, 267, 256, 283, 230, 248, 268, 254, 258, 232, 273, 238, 245, 246, 242, 271, 279, 260, 245, 247, 233] payload_rows_per_dst=[5312, 5566, 5266, 5532, 5432, 5416, 5612, 5486] dup_rows_per_dst=[2612, 2894, 2558, 2836, 2668, 2588, 3000, 2758] combine_rows_per_src=[5312, 5566, 5266, 5532, 5432, 5416, 5612, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229584 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156532944 ingress_bytes=212926464 link_bytes=212926464 token_bytes=312836304 weight_movement_bytes=56623104 total_nvlink_bytes=369459408 weight_to_token_ratio=0.180999 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040156612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15053424816 prefetch_bytes=792723456 reduce_bytes=471859200 weight_movement_bytes=1264582656 egress_bytes=7532218544 ingress_bytes=8785788928 link_bytes=8785788928 total_nvlink_bytes=16318007472 weight_to_token_ratio=0.084006 prefetch_share_of_ingress=0.090228 dedup_ratio=0.333036 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=25 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082465 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.753773055e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38146 dup_rows=21844 dup_rows_remote=19052 dedup_ratio=0.333313 router_tokens_per_expert=[264, 248, 242, 252, 260, 260, 241, 275, 272, 281, 253, 263, 246, 255, 234, 239, 252, 246, 258, 243, 259, 265, 265, 262, 250, 269, 266, 262, 221, 270, 279, 240, 269, 267, 238, 293, 274, 247, 282, 260, 251, 242, 238, 247, 228, 282, 280, 234, 230, 259, 263, 255, 251, 242, 258, 243, 241, 246, 242, 261, 298, 258, 247, 250, 270, 266, 258, 254, 243, 221, 254, 261, 270, 240, 230, 266, 297, 278, 258, 276, 260, 232, 231, 229, 266, 271, 271, 264, 266, 254, 246, 265, 255, 251, 226, 262, 262, 271, 254, 241, 239, 261, 277, 249, 266, 269, 229, 240, 293, 281, 251, 261, 215, 272, 256, 267, 243, 233, 249, 261, 281, 280, 260, 229, 268, 244, 278, 229] payload_rows_per_dst=[5364, 5536, 5438, 5430, 5498, 5422, 5546, 5458] dup_rows_per_dst=[2806, 2678, 2776, 2658, 2786, 2676, 2792, 2672] combine_rows_per_src=[5364, 5536, 5438, 5430, 5498, 5422, 5546, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229024 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156892832 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556640 weight_movement_bytes=37748736 total_nvlink_bytes=351305376 weight_to_token_ratio=0.120389 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229848 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156516824 ingress_bytes=175161344 link_bytes=175161344 token_bytes=312803800 weight_movement_bytes=18874368 total_nvlink_bytes=331678168 weight_to_token_ratio=0.060339 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678309813e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44072 payload_rows_remote=38488 dup_rows=21464 dup_rows_remote=18730 dedup_ratio=0.327515 router_tokens_per_expert=[244, 209, 278, 239, 232, 246, 250, 248, 231, 243, 271, 273, 268, 277, 284, 276, 232, 239, 241, 238, 258, 264, 263, 273, 253, 232, 256, 257, 287, 258, 251, 274, 275, 270, 247, 268, 236, 269, 270, 269, 242, 252, 238, 248, 263, 233, 277, 262, 264, 251, 215, 249, 248, 254, 259, 213, 265, 243, 234, 251, 243, 269, 255, 264, 251, 260, 231, 271, 249, 229, 244, 253, 260, 261, 278, 264, 284, 265, 238, 266, 257, 295, 248, 250, 300, 244, 262, 252, 265, 253, 250, 259, 263, 255, 266, 240, 226, 265, 240, 277, 249, 272, 249, 233, 255, 254, 272, 278, 256, 286, 236, 260, 250, 282, 280, 261, 257, 228, 261, 255, 271, 271, 258, 262, 255, 275, 246, 244] payload_rows_per_dst=[5476, 5474, 5508, 5420, 5592, 5584, 5486, 5532] dup_rows_per_dst=[2662, 2678, 2730, 2534, 2616, 2734, 2730, 2780] combine_rows_per_src=[5476, 5474, 5508, 5420, 5592, 5584, 5486, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=229168 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156434224 ingress_bytes=193953792 link_bytes=193953792 token_bytes=312639280 weight_movement_bytes=37748736 total_nvlink_bytes=350388016 weight_to_token_ratio=0.120742 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858369804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38422 dup_rows=21686 dup_rows_remote=19050 dedup_ratio=0.330902 router_tokens_per_expert=[264, 267, 249, 256, 279, 267, 253, 260, 264, 256, 275, 242, 255, 241, 289, 257, 256, 244, 242, 262, 269, 257, 267, 253, 241, 268, 248, 245, 252, 227, 255, 250, 243, 238, 264, 255, 260, 262, 257, 255, 253, 255, 266, 235, 248, 259, 265, 263, 256, 254, 238, 232, 243, 263, 257, 253, 270, 228, 262, 277, 245, 252, 261, 241, 257, 257, 266, 268, 288, 244, 267, 228, 252, 259, 243, 246, 254, 247, 252, 274, 246, 245, 258, 242, 262, 254, 264, 245, 274, 237, 253, 261, 238, 255, 250, 274, 261, 275, 244, 260, 261, 269, 241, 260, 245, 250, 269, 241, 220, 269, 282, 254, 258, 265, 244, 258, 253, 268, 261, 257, 287, 274, 273, 266, 263, 255, 257, 248] payload_rows_per_dst=[5566, 5480, 5414, 5428, 5502, 5406, 5506, 5548] dup_rows_per_dst=[2782, 2592, 2742, 2636, 2702, 2710, 2696, 2826] combine_rows_per_src=[5566, 5480, 5414, 5428, 5502, 5406, 5506, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228792 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156474808 ingress_bytes=175120384 link_bytes=175120384 token_bytes=312720824 weight_movement_bytes=18874368 total_nvlink_bytes=331595192 weight_to_token_ratio=0.060355 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677917355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15063400480 prefetch_bytes=754974720 reduce_bytes=566231040 weight_movement_bytes=1321205760 egress_bytes=7537205280 ingress_bytes=8847400960 link_bytes=8847400960 total_nvlink_bytes=16384606240 weight_to_token_ratio=0.087710 prefetch_share_of_ingress=0.085333 dedup_ratio=0.332333 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=30 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081597 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.766071725e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15040144992 prefetch_bytes=830472192 reduce_bytes=736100352 weight_movement_bytes=1566572544 egress_bytes=7525566048 ingress_bytes=9081151488 link_bytes=9081151488 total_nvlink_bytes=16606717536 weight_to_token_ratio=0.104159 prefetch_share_of_ingress=0.091450 dedup_ratio=0.332785 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=39 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081308 gemm_inner_pad_upper_bound=109220 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.812731778e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157646848 dispatch_meta_bytes=228872 combine_payload_bytes=157646848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157875720 ingress_bytes=195395584 link_bytes=195395584 token_bytes=315522568 weight_movement_bytes=37748736 total_nvlink_bytes=353271304 weight_to_token_ratio=0.119639 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872184345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43994 payload_rows_remote=38546 dup_rows=21542 dup_rows_remote=18966 dedup_ratio=0.328705 router_tokens_per_expert=[247, 247, 268, 231, 268, 287, 254, 300, 257, 289, 244, 231, 260, 260, 254, 274, 266, 227, 268, 284, 245, 275, 248, 240, 240, 265, 221, 259, 260, 234, 300, 264, 264, 253, 254, 250, 239, 239, 247, 251, 244, 247, 242, 225, 289, 259, 253, 282, 262, 241, 227, 253, 286, 270, 260, 266, 262, 255, 261, 239, 282, 260, 239, 239, 246, 262, 240, 238, 238, 253, 257, 264, 243, 254, 259, 273, 243, 253, 230, 259, 263, 259, 248, 267, 248, 278, 272, 226, 298, 249, 247, 250, 271, 277, 249, 248, 235, 267, 254, 250, 248, 257, 239, 248, 287, 235, 245, 262, 247, 273, 248, 243, 244, 278, 260, 287, 230, 269, 242, 276, 269, 261, 269, 266, 279, 245, 259, 227] payload_rows_per_dst=[5532, 5486, 5494, 5498, 5448, 5518, 5438, 5580] dup_rows_per_dst=[2810, 2706, 2582, 2706, 2576, 2782, 2638, 2742] combine_rows_per_src=[5532, 5486, 5494, 5498, 5448, 5518, 5438, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229888 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157606400 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982912 weight_movement_bytes=18874368 total_nvlink_bytes=333857280 weight_to_token_ratio=0.059922 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15061165816 prefetch_bytes=792723456 reduce_bytes=717225984 weight_movement_bytes=1509949440 egress_bytes=7536084728 ingress_bytes=9035030528 link_bytes=9035030528 total_nvlink_bytes=16571115256 weight_to_token_ratio=0.100254 prefetch_share_of_ingress=0.087739 dedup_ratio=0.332336 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=38 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081453 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.803525354e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44000 payload_rows_remote=38594 dup_rows=21536 dup_rows_remote=18946 dedup_ratio=0.328613 router_tokens_per_expert=[251, 247, 259, 238, 273, 250, 254, 224, 238, 242, 262, 247, 242, 258, 257, 256, 303, 273, 238, 246, 259, 278, 238, 232, 238, 266, 251, 269, 280, 236, 239, 269, 244, 260, 233, 271, 225, 254, 243, 236, 239, 275, 265, 276, 263, 251, 250, 252, 240, 279, 254, 255, 275, 253, 276, 245, 266, 279, 264, 240, 256, 276, 252, 239, 281, 270, 241, 263, 267, 263, 254, 268, 262, 262, 291, 255, 248, 268, 269, 257, 236, 253, 278, 258, 239, 247, 274, 289, 253, 240, 253, 230, 263, 239, 249, 268, 256, 242, 292, 280, 246, 275, 236, 254, 244, 269, 248, 264, 284, 251, 257, 240, 237, 267, 263, 256, 267, 252, 252, 242, 235, 242, 258, 278, 239, 269, 225, 261] payload_rows_per_dst=[5406, 5474, 5436, 5544, 5610, 5522, 5552, 5456] dup_rows_per_dst=[2590, 2756, 2638, 2754, 2828, 2616, 2870, 2484] combine_rows_per_src=[5406, 5474, 5436, 5544, 5610, 5522, 5552, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=230048 combine_payload_bytes=157884416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158114464 ingress_bytes=176758784 link_bytes=176758784 token_bytes=315998880 weight_movement_bytes=18874368 total_nvlink_bytes=334873248 weight_to_token_ratio=0.059729 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693615698e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15054552712 prefetch_bytes=792723456 reduce_bytes=1056964608 weight_movement_bytes=1849688064 egress_bytes=7532781192 ingress_bytes=9371459584 link_bytes=9371459584 total_nvlink_bytes=16904240776 weight_to_token_ratio=0.122866 prefetch_share_of_ingress=0.084589 dedup_ratio=0.332692 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=56 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081163 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.870681556e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=158081024 dispatch_meta_bytes=230160 combine_payload_bytes=158081024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158311184 ingress_bytes=233578496 link_bytes=233578496 token_bytes=316392208 weight_movement_bytes=75497472 total_nvlink_bytes=391889680 weight_to_token_ratio=0.238620 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.238034221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15057714256 prefetch_bytes=773849088 reduce_bytes=754974720 weight_movement_bytes=1528823808 egress_bytes=7534361680 ingress_bytes=9052176384 link_bytes=9052405504 total_nvlink_bytes=16586538064 weight_to_token_ratio=0.101531 prefetch_share_of_ingress=0.085488 dedup_ratio=0.332621 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=40 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.806993656e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38420 dup_rows=21670 dup_rows_remote=18962 dedup_ratio=0.330658 router_tokens_per_expert=[270, 245, 240, 282, 271, 231, 253, 259, 254, 239, 259, 244, 241, 280, 268, 269, 248, 269, 246, 254, 242, 255, 268, 244, 289, 254, 264, 266, 265, 248, 259, 244, 255, 261, 262, 249, 274, 238, 257, 249, 246, 264, 249, 276, 266, 283, 255, 241, 269, 244, 248, 254, 255, 273, 259, 249, 247, 292, 252, 254, 289, 253, 239, 229, 257, 272, 226, 266, 247, 263, 251, 245, 293, 239, 245, 243, 275, 256, 252, 264, 239, 264, 233, 287, 205, 272, 265, 253, 265, 260, 241, 255, 258, 257, 250, 273, 272, 244, 258, 259, 260, 239, 239, 283, 244, 252, 237, 251, 238, 278, 227, 245, 266, 232, 212, 284, 267, 242, 243, 271, 247, 272, 239, 281, 265, 273, 258, 268] payload_rows_per_dst=[5456, 5442, 5494, 5516, 5520, 5446, 5454, 5538] dup_rows_per_dst=[2754, 2788, 2756, 2696, 2668, 2708, 2598, 2702] combine_rows_per_src=[5456, 5442, 5494, 5516, 5520, 5446, 5454, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=229528 combine_payload_bytes=157368320 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157597848 ingress_bytes=213991424 link_bytes=213991424 token_bytes=314966168 weight_movement_bytes=56623104 total_nvlink_bytes=371589272 weight_to_token_ratio=0.179775 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050360534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15061841808 prefetch_bytes=736100352 reduce_bytes=830472192 weight_movement_bytes=1566572544 egress_bytes=7536424848 ingress_bytes=9091989504 link_bytes=9091989504 total_nvlink_bytes=16628414352 weight_to_token_ratio=0.104009 prefetch_share_of_ingress=0.080961 dedup_ratio=0.332420 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=44 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082031 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.814895206e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43530 payload_rows_remote=37960 dup_rows=22006 dup_rows_remote=19040 dedup_ratio=0.335785 router_tokens_per_expert=[256, 276, 274, 264, 274, 245, 266, 228, 230, 268, 225, 245, 262, 241, 273, 254, 222, 233, 282, 234, 263, 277, 250, 276, 243, 243, 257, 256, 265, 235, 244, 240, 271, 294, 257, 268, 262, 248, 262, 239, 225, 255, 254, 277, 243, 248, 256, 282, 260, 273, 267, 277, 269, 258, 266, 264, 256, 228, 251, 275, 233, 267, 240, 257, 253, 250, 265, 245, 301, 254, 245, 259, 262, 254, 269, 270, 259, 262, 271, 248, 247, 245, 252, 237, 244, 257, 249, 265, 265, 249, 261, 257, 248, 247, 264, 252, 271, 280, 264, 247, 245, 283, 248, 252, 267, 243, 278, 252, 271, 242, 262, 260, 239, 240, 260, 239, 260, 230, 265, 253, 246, 243, 243, 256, 251, 249, 260, 280] payload_rows_per_dst=[5570, 5578, 5538, 5594, 5432, 5392, 5196, 5230] dup_rows_per_dst=[2966, 2982, 2962, 3012, 2588, 2676, 2498, 2322] combine_rows_per_src=[5570, 5578, 5538, 5594, 5432, 5392, 5196, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155484160 dispatch_meta_bytes=228000 combine_payload_bytes=155484160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155712160 ingress_bytes=174358528 link_bytes=174358528 token_bytes=311196320 weight_movement_bytes=18874368 total_nvlink_bytes=330070688 weight_to_token_ratio=0.060651 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670617626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38190 dup_rows=21838 dup_rows_remote=19080 dedup_ratio=0.333221 router_tokens_per_expert=[262, 247, 266, 238, 254, 258, 258, 248, 259, 275, 260, 249, 250, 258, 254, 238, 260, 249, 255, 270, 270, 245, 259, 257, 268, 281, 272, 232, 250, 236, 279, 244, 261, 257, 270, 233, 290, 242, 212, 244, 255, 238, 197, 237, 265, 245, 262, 280, 252, 263, 233, 254, 252, 243, 261, 248, 230, 270, 277, 250, 246, 257, 259, 228, 288, 253, 281, 265, 277, 282, 274, 249, 267, 301, 257, 269, 254, 261, 289, 252, 259, 291, 280, 245, 236, 247, 246, 267, 261, 226, 275, 263, 257, 241, 258, 243, 235, 227, 258, 239, 257, 267, 277, 242, 256, 253, 263, 235, 271, 254, 254, 287, 269, 241, 263, 245, 236, 271, 255, 242, 255, 265, 261, 261, 240, 268, 241, 254] payload_rows_per_dst=[5508, 5778, 5296, 5516, 5540, 5232, 5422, 5406] dup_rows_per_dst=[2758, 3058, 2680, 2768, 2742, 2476, 2728, 2628] combine_rows_per_src=[5508, 5778, 5296, 5516, 5540, 5232, 5422, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229080 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655320 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081560 weight_movement_bytes=18874368 total_nvlink_bytes=331955928 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38162 dup_rows=21924 dup_rows_remote=19186 dedup_ratio=0.334534 router_tokens_per_expert=[243, 266, 276, 275, 250, 272, 243, 253, 251, 249, 232, 279, 231, 246, 245, 274, 239, 250, 242, 252, 265, 252, 274, 261, 276, 229, 272, 244, 244, 252, 260, 270, 269, 262, 259, 239, 309, 254, 266, 272, 292, 250, 247, 282, 262, 238, 259, 263, 285, 279, 273, 259, 243, 229, 231, 243, 247, 271, 272, 253, 277, 260, 271, 267, 245, 263, 247, 295, 255, 249, 246, 245, 233, 255, 244, 254, 266, 254, 244, 258, 283, 267, 245, 242, 269, 243, 234, 251, 252, 280, 279, 260, 257, 261, 253, 247, 256, 237, 255, 236, 225, 250, 254, 246, 242, 251, 223, 244, 258, 262, 255, 257, 240, 264, 245, 266, 222, 256, 269, 248, 275, 267, 277, 245, 227, 244, 268, 278] payload_rows_per_dst=[5450, 5460, 5522, 5550, 5530, 5482, 5308, 5310] dup_rows_per_dst=[2738, 2704, 2968, 2770, 2994, 2702, 2594, 2454] combine_rows_per_src=[5450, 5460, 5522, 5550, 5530, 5482, 5308, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=229392 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156540944 ingress_bytes=212934656 link_bytes=212934656 token_bytes=312852496 weight_movement_bytes=56623104 total_nvlink_bytes=369475600 weight_to_token_ratio=0.180990 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040235103e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15037473008 prefetch_bytes=773849088 reduce_bytes=1113587712 weight_movement_bytes=1887436800 egress_bytes=7524229360 ingress_bytes=9400680448 link_bytes=9400910624 total_nvlink_bytes=16924909808 weight_to_token_ratio=0.125516 prefetch_share_of_ingress=0.082318 dedup_ratio=0.332591 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=59 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.079716 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.876560418e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none + [2026-08-29 23:39:12] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 774.4 | throughput per GPU (TFLOP/s/GPU): 115.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.802535E+00 | loss scale: 1.0 | grad norm: 3.108 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38194 dup_rows=21838 dup_rows_remote=19122 dedup_ratio=0.333221 router_tokens_per_expert=[261, 268, 254, 248, 269, 252, 234, 249, 275, 251, 249, 248, 239, 239, 252, 252, 244, 239, 283, 254, 246, 282, 235, 295, 256, 243, 276, 246, 236, 258, 252, 265, 289, 268, 263, 237, 256, 259, 268, 251, 263, 284, 235, 263, 261, 270, 253, 238, 249, 244, 295, 253, 254, 244, 242, 271, 284, 252, 250, 240, 262, 249, 237, 264, 249, 271, 237, 258, 262, 268, 261, 238, 269, 248, 273, 269, 252, 270, 237, 258, 271, 247, 259, 229, 258, 248, 233, 254, 241, 269, 247, 247, 262, 255, 274, 271, 245, 262, 268, 270, 223, 261, 267, 253, 264, 241, 258, 258, 246, 260, 245, 234, 235, 244, 303, 266, 252, 259, 276, 263, 263, 241, 268, 246, 282, 234, 239, 259] payload_rows_per_dst=[5400, 5504, 5544, 5412, 5480, 5384, 5448, 5526] dup_rows_per_dst=[2680, 2716, 2772, 2768, 2760, 2746, 2662, 2734] combine_rows_per_src=[5400, 5504, 5544, 5412, 5480, 5384, 5448, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38218 dup_rows=21822 dup_rows_remote=19078 dedup_ratio=0.332977 router_tokens_per_expert=[273, 267, 267, 264, 253, 276, 251, 226, 232, 256, 258, 260, 235, 276, 261, 267, 229, 226, 279, 232, 240, 238, 276, 272, 262, 269, 254, 292, 264, 292, 247, 246, 256, 259, 251, 241, 276, 278, 277, 268, 277, 254, 239, 256, 253, 258, 237, 240, 252, 287, 281, 240, 246, 225, 245, 272, 278, 242, 250, 239, 302, 251, 253, 269, 244, 261, 287, 284, 274, 231, 250, 215, 244, 250, 246, 246, 281, 242, 241, 263, 233, 259, 250, 257, 244, 251, 232, 237, 241, 254, 257, 240, 262, 248, 261, 244, 262, 287, 258, 249, 221, 244, 255, 269, 254, 253, 261, 277, 226, 258, 276, 255, 235, 254, 254, 248, 273, 259, 271, 252, 258, 264, 248, 246, 281, 260, 285, 254] payload_rows_per_dst=[5474, 5494, 5496, 5548, 5402, 5308, 5494, 5498] dup_rows_per_dst=[2770, 2742, 2744, 2716, 2716, 2632, 2716, 2786] combine_rows_per_src=[5474, 5494, 5496, 5548, 5402, 5308, 5494, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38150 dup_rows=21818 dup_rows_remote=18902 dedup_ratio=0.332916 router_tokens_per_expert=[276, 243, 271, 264, 272, 263, 220, 271, 244, 246, 271, 274, 275, 246, 271, 271, 239, 253, 263, 229, 266, 250, 276, 276, 260, 257, 263, 252, 250, 277, 247, 275, 252, 246, 293, 244, 262, 262, 254, 242, 254, 244, 258, 280, 241, 248, 273, 275, 280, 263, 247, 262, 265, 241, 264, 252, 235, 251, 267, 217, 226, 242, 240, 260, 224, 237, 265, 245, 264, 244, 268, 249, 248, 257, 252, 273, 257, 251, 248, 254, 266, 267, 269, 274, 287, 262, 237, 243, 215, 240, 209, 257, 246, 276, 283, 250, 245, 275, 262, 255, 254, 250, 281, 272, 244, 234, 261, 250, 261, 264, 254, 272, 245, 248, 255, 228, 251, 244, 252, 249, 237, 289, 270, 262, 276, 262, 243, 255] payload_rows_per_dst=[5458, 5416, 5416, 5288, 5452, 5528, 5592, 5568] dup_rows_per_dst=[2780, 2608, 2542, 2420, 2620, 2906, 3026, 2916] combine_rows_per_src=[5458, 5416, 5416, 5288, 5452, 5528, 5592, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38208 dup_rows=21894 dup_rows_remote=19090 dedup_ratio=0.334076 router_tokens_per_expert=[269, 288, 260, 247, 268, 231, 228, 254, 257, 271, 272, 249, 250, 278, 263, 248, 256, 229, 247, 242, 253, 258, 305, 236, 254, 251, 255, 240, 280, 247, 275, 264, 266, 261, 236, 263, 264, 241, 248, 232, 277, 246, 250, 263, 271, 219, 280, 232, 268, 268, 279, 251, 263, 268, 216, 291, 267, 266, 260, 273, 259, 255, 246, 260, 245, 258, 264, 264, 258, 257, 242, 270, 273, 239, 261, 252, 258, 247, 270, 261, 266, 276, 256, 258, 267, 242, 283, 239, 253, 253, 251, 264, 261, 222, 249, 270, 258, 234, 230, 253, 267, 259, 253, 236, 239, 266, 270, 259, 232, 232, 266, 241, 260, 263, 268, 270, 253, 246, 258, 244, 274, 237, 234, 237, 276, 240, 267, 253] payload_rows_per_dst=[5450, 5438, 5396, 5540, 5434, 5504, 5392, 5488] dup_rows_per_dst=[2816, 2746, 2702, 2840, 2804, 2716, 2598, 2672] combine_rows_per_src=[5450, 5438, 5396, 5540, 5434, 5504, 5392, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38262 dup_rows=21706 dup_rows_remote=18962 dedup_ratio=0.331207 router_tokens_per_expert=[256, 293, 273, 268, 270, 252, 269, 232, 242, 265, 256, 244, 240, 268, 255, 227, 269, 264, 272, 271, 252, 246, 279, 265, 273, 241, 253, 239, 238, 256, 264, 278, 269, 259, 237, 263, 267, 256, 269, 261, 256, 250, 229, 254, 259, 256, 252, 252, 276, 280, 235, 255, 254, 248, 282, 265, 256, 256, 254, 259, 256, 257, 261, 262, 260, 284, 267, 250, 256, 285, 258, 222, 260, 249, 250, 270, 239, 234, 252, 257, 267, 272, 283, 237, 255, 250, 253, 228, 226, 254, 236, 287, 216, 248, 279, 232, 282, 285, 241, 256, 233, 251, 289, 253, 223, 245, 235, 240, 268, 264, 263, 265, 241, 249, 274, 262, 250, 237, 244, 245, 262, 252, 266, 261, 236, 257, 236, 272] payload_rows_per_dst=[5510, 5508, 5434, 5568, 5476, 5418, 5478, 5438] dup_rows_per_dst=[2710, 2812, 2744, 2744, 2710, 2628, 2708, 2650] combine_rows_per_src=[5510, 5508, 5434, 5568, 5476, 5418, 5478, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38270 dup_rows=21754 dup_rows_remote=19028 dedup_ratio=0.331940 router_tokens_per_expert=[254, 241, 258, 290, 243, 253, 251, 247, 263, 248, 246, 260, 274, 238, 247, 259, 246, 250, 265, 257, 265, 277, 262, 242, 239, 229, 248, 289, 273, 273, 250, 262, 250, 248, 274, 265, 240, 274, 237, 271, 243, 248, 261, 253, 282, 269, 278, 220, 266, 267, 256, 237, 274, 241, 257, 254, 261, 268, 249, 269, 266, 244, 272, 272, 251, 255, 240, 273, 231, 291, 236, 262, 285, 235, 299, 257, 254, 258, 219, 248, 264, 254, 229, 267, 259, 272, 239, 274, 274, 250, 248, 248, 263, 245, 271, 262, 239, 239, 270, 288, 250, 245, 236, 245, 253, 241, 244, 246, 289, 261, 266, 240, 249, 256, 262, 276, 267, 228, 253, 237, 258, 277, 236, 236, 234, 267, 256, 246] payload_rows_per_dst=[5434, 5488, 5562, 5484, 5478, 5512, 5414, 5410] dup_rows_per_dst=[2710, 2766, 2664, 2822, 2710, 2726, 2690, 2666] combine_rows_per_src=[5434, 5488, 5562, 5484, 5478, 5512, 5414, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38284 dup_rows=21832 dup_rows_remote=19186 dedup_ratio=0.333130 router_tokens_per_expert=[236, 254, 271, 243, 269, 270, 262, 248, 265, 268, 276, 249, 274, 233, 248, 268, 242, 295, 262, 292, 258, 246, 254, 245, 254, 253, 235, 235, 277, 228, 284, 258, 254, 272, 254, 236, 262, 236, 256, 257, 263, 263, 230, 259, 296, 247, 268, 233, 252, 254, 255, 247, 280, 273, 260, 253, 238, 260, 237, 242, 273, 249, 264, 254, 276, 267, 266, 278, 253, 252, 256, 261, 257, 281, 243, 260, 214, 241, 240, 243, 269, 258, 261, 246, 263, 261, 246, 255, 255, 237, 263, 222, 292, 262, 290, 238, 265, 233, 228, 275, 261, 270, 252, 265, 238, 253, 242, 257, 257, 250, 261, 226, 260, 256, 229, 278, 253, 266, 251, 288, 259, 247, 247, 247, 257, 263, 236, 263] payload_rows_per_dst=[5480, 5494, 5470, 5416, 5440, 5498, 5420, 5486] dup_rows_per_dst=[2788, 2742, 2702, 2766, 2736, 2738, 2646, 2714] combine_rows_per_src=[5480, 5494, 5470, 5416, 5440, 5498, 5420, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229264 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156671888 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114512 weight_movement_bytes=37748736 total_nvlink_bytes=350863248 weight_to_token_ratio=0.120559 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229184 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156770112 ingress_bytes=194289664 link_bytes=194289664 token_bytes=313311040 weight_movement_bytes=37748736 total_nvlink_bytes=351059776 weight_to_token_ratio=0.120483 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861587964e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228208 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156490608 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753008 weight_movement_bytes=18874368 total_nvlink_bytes=331627376 weight_to_token_ratio=0.060349 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228896 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156950048 ingress_bytes=194469888 link_bytes=194469888 token_bytes=313671200 weight_movement_bytes=37748736 total_nvlink_bytes=351419936 weight_to_token_ratio=0.120345 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863314782e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229192 combine_payload_bytes=156499968 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156729160 ingress_bytes=175374336 link_bytes=175374336 token_bytes=313229128 weight_movement_bytes=18874368 total_nvlink_bytes=332103496 weight_to_token_ratio=0.060257 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680350598e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229192 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983112 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737032 weight_movement_bytes=18874368 total_nvlink_bytes=332611400 weight_to_token_ratio=0.060160 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229880 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157041144 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313852408 weight_movement_bytes=18874368 total_nvlink_bytes=332726776 weight_to_token_ratio=0.060138 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38298 dup_rows=21860 dup_rows_remote=19112 dedup_ratio=0.333557 router_tokens_per_expert=[239, 277, 237, 278, 264, 286, 236, 286, 279, 223, 255, 250, 253, 276, 255, 259, 256, 247, 246, 297, 257, 234, 292, 251, 239, 248, 249, 229, 243, 247, 265, 263, 261, 297, 237, 241, 281, 243, 269, 230, 255, 265, 240, 262, 269, 242, 259, 246, 269, 265, 287, 246, 238, 242, 277, 246, 255, 241, 233, 267, 245, 259, 265, 259, 265, 211, 274, 243, 263, 272, 280, 245, 257, 225, 238, 262, 248, 270, 244, 270, 245, 246, 266, 278, 251, 267, 250, 238, 260, 267, 268, 250, 239, 264, 263, 267, 231, 253, 265, 263, 247, 261, 278, 236, 234, 235, 261, 257, 283, 241, 226, 252, 263, 264, 267, 237, 259, 274, 245, 266, 277, 229, 266, 274, 243, 263, 272, 253] payload_rows_per_dst=[5486, 5378, 5444, 5518, 5484, 5484, 5378, 5504] dup_rows_per_dst=[2820, 2748, 2750, 2670, 2650, 2754, 2668, 2800] combine_rows_per_src=[5486, 5378, 5444, 5518, 5484, 5484, 5378, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38316 dup_rows=21770 dup_rows_remote=19032 dedup_ratio=0.332184 router_tokens_per_expert=[270, 252, 239, 245, 269, 254, 234, 230, 267, 255, 242, 250, 253, 272, 262, 263, 272, 232, 259, 271, 264, 240, 228, 270, 250, 263, 261, 266, 239, 247, 261, 250, 239, 264, 265, 244, 271, 271, 257, 273, 253, 272, 261, 218, 289, 245, 241, 231, 258, 266, 255, 272, 250, 256, 247, 254, 275, 276, 262, 286, 265, 233, 275, 263, 281, 265, 262, 249, 278, 273, 242, 257, 261, 274, 266, 249, 252, 254, 254, 269, 252, 271, 267, 290, 226, 244, 260, 244, 246, 264, 268, 276, 236, 265, 233, 219, 258, 248, 268, 266, 263, 275, 264, 244, 232, 262, 231, 247, 270, 243, 256, 253, 248, 236, 232, 271, 269, 258, 236, 258, 255, 263, 277, 238, 237, 245, 264, 237] payload_rows_per_dst=[5400, 5428, 5450, 5560, 5512, 5476, 5502, 5438] dup_rows_per_dst=[2714, 2718, 2738, 2826, 2860, 2646, 2658, 2610] combine_rows_per_src=[5400, 5428, 5450, 5560, 5512, 5476, 5502, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229640 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098248 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966856 weight_movement_bytes=18874368 total_nvlink_bytes=332841224 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43490 payload_rows_remote=37960 dup_rows=22046 dup_rows_remote=19076 dedup_ratio=0.336395 router_tokens_per_expert=[277, 280, 249, 258, 234, 262, 268, 255, 247, 251, 251, 263, 258, 246, 285, 253, 255, 275, 251, 275, 275, 256, 258, 234, 262, 245, 260, 263, 236, 266, 244, 269, 266, 248, 246, 267, 244, 243, 238, 244, 255, 264, 250, 263, 240, 257, 227, 233, 241, 244, 232, 275, 282, 264, 259, 269, 242, 275, 276, 254, 247, 270, 238, 286, 261, 275, 255, 269, 263, 269, 249, 270, 271, 250, 239, 274, 244, 270, 268, 267, 250, 268, 267, 249, 234, 262, 242, 252, 246, 255, 234, 246, 263, 244, 229, 220, 256, 259, 227, 245, 256, 275, 237, 248, 253, 276, 225, 261, 259, 248, 269, 288, 263, 258, 246, 242, 263, 269, 254, 260, 292, 240, 264, 273, 256, 246, 261, 244] payload_rows_per_dst=[5360, 5418, 5388, 5510, 5346, 5466, 5472, 5530] dup_rows_per_dst=[2408, 2830, 2582, 2766, 2726, 2962, 2802, 2970] combine_rows_per_src=[5360, 5418, 5388, 5510, 5346, 5466, 5472, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=38110 dup_rows=21920 dup_rows_remote=19240 dedup_ratio=0.334473 router_tokens_per_expert=[274, 242, 237, 290, 271, 289, 252, 238, 248, 224, 260, 265, 252, 250, 242, 257, 271, 231, 272, 243, 271, 248, 249, 216, 283, 266, 259, 280, 268, 285, 272, 259, 250, 267, 259, 255, 227, 261, 247, 273, 251, 267, 270, 220, 253, 282, 261, 231, 237, 233, 256, 255, 275, 245, 292, 233, 252, 255, 256, 266, 234, 274, 275, 255, 244, 255, 268, 273, 272, 267, 260, 223, 258, 267, 246, 270, 254, 285, 255, 249, 273, 244, 271, 227, 261, 234, 251, 249, 241, 261, 237, 254, 236, 248, 274, 254, 254, 244, 259, 272, 266, 258, 276, 264, 236, 237, 225, 258, 246, 251, 265, 260, 268, 243, 244, 259, 239, 267, 242, 270, 253, 264, 247, 254, 264, 257, 255, 279] payload_rows_per_dst=[5434, 5470, 5416, 5506, 5484, 5400, 5460, 5446] dup_rows_per_dst=[2748, 2876, 2732, 2680, 2808, 2630, 2682, 2764] combine_rows_per_src=[5434, 5470, 5416, 5506, 5484, 5400, 5460, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38492 dup_rows=21680 dup_rows_remote=19046 dedup_ratio=0.330811 router_tokens_per_expert=[268, 257, 268, 239, 279, 251, 255, 268, 260, 255, 264, 239, 258, 262, 268, 267, 266, 263, 239, 241, 261, 240, 256, 261, 228, 236, 259, 261, 252, 259, 259, 246, 235, 255, 277, 264, 246, 244, 238, 286, 241, 273, 250, 273, 287, 252, 233, 277, 240, 272, 267, 299, 256, 259, 257, 244, 238, 251, 264, 281, 267, 281, 268, 258, 250, 248, 248, 270, 253, 258, 260, 232, 259, 238, 245, 256, 262, 248, 241, 266, 240, 248, 245, 250, 237, 250, 259, 274, 241, 243, 256, 245, 271, 245, 250, 245, 270, 236, 239, 252, 231, 252, 266, 282, 254, 259, 263, 268, 264, 266, 269, 247, 290, 268, 246, 254, 251, 270, 215, 267, 243, 233, 252, 263, 275, 237, 258, 277] payload_rows_per_dst=[5548, 5398, 5480, 5568, 5494, 5364, 5514, 5490] dup_rows_per_dst=[2768, 2656, 2782, 2836, 2574, 2634, 2722, 2708] combine_rows_per_src=[5548, 5398, 5480, 5568, 5494, 5364, 5514, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43598 payload_rows_remote=38136 dup_rows=21938 dup_rows_remote=19048 dedup_ratio=0.334747 router_tokens_per_expert=[259, 242, 269, 276, 240, 269, 250, 256, 256, 248, 298, 258, 259, 261, 249, 244, 243, 242, 246, 245, 256, 259, 277, 255, 250, 275, 265, 241, 257, 273, 240, 272, 262, 267, 230, 253, 250, 277, 268, 282, 248, 261, 244, 250, 251, 277, 248, 283, 254, 256, 242, 250, 258, 259, 229, 259, 244, 245, 273, 241, 255, 267, 275, 239, 263, 256, 240, 233, 257, 284, 243, 231, 299, 239, 262, 277, 263, 259, 296, 274, 240, 237, 263, 249, 252, 247, 255, 258, 243, 225, 270, 262, 229, 271, 287, 234, 262, 246, 284, 250, 237, 285, 280, 268, 246, 249, 258, 270, 249, 241, 249, 250, 243, 259, 254, 284, 247, 222, 281, 244, 228, 274, 264, 239, 242, 238, 254, 246] payload_rows_per_dst=[5470, 5472, 5588, 5390, 5462, 5308, 5498, 5410] dup_rows_per_dst=[2798, 2720, 2714, 2702, 2890, 2736, 2750, 2628] combine_rows_per_src=[5470, 5472, 5588, 5390, 5462, 5308, 5498, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=38000 dup_rows=21962 dup_rows_remote=19160 dedup_ratio=0.335114 router_tokens_per_expert=[255, 235, 253, 241, 269, 247, 268, 255, 273, 246, 254, 256, 255, 291, 261, 283, 253, 252, 247, 249, 247, 262, 249, 278, 230, 260, 273, 256, 258, 277, 231, 254, 283, 252, 244, 246, 253, 251, 251, 245, 236, 284, 241, 279, 252, 254, 245, 266, 273, 260, 240, 234, 241, 273, 243, 242, 256, 263, 246, 256, 282, 252, 274, 276, 223, 274, 243, 255, 242, 253, 279, 263, 272, 269, 256, 249, 251, 241, 241, 244, 240, 230, 231, 264, 240, 247, 244, 273, 258, 265, 248, 279, 262, 254, 267, 256, 242, 233, 288, 264, 246, 256, 252, 277, 270, 270, 271, 231, 272, 281, 278, 257, 263, 232, 233, 253, 266, 245, 262, 285, 262, 229, 257, 264, 259, 236, 262, 248] payload_rows_per_dst=[5326, 5348, 5360, 5450, 5510, 5546, 5574, 5460] dup_rows_per_dst=[2660, 2804, 2804, 2772, 2600, 2868, 2802, 2652] combine_rows_per_src=[5326, 5348, 5360, 5450, 5510, 5546, 5574, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229392 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171728 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114064 weight_movement_bytes=18874368 total_nvlink_bytes=332988432 weight_to_token_ratio=0.060088 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155484160 dispatch_meta_bytes=228144 combine_payload_bytes=155484160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155712304 ingress_bytes=174358528 link_bytes=174358528 token_bytes=311196464 weight_movement_bytes=18874368 total_nvlink_bytes=330070832 weight_to_token_ratio=0.060651 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670617626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157663232 dispatch_meta_bytes=230152 combine_payload_bytes=157663232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157893384 ingress_bytes=176537600 link_bytes=176537600 token_bytes=315556616 weight_movement_bytes=18874368 total_nvlink_bytes=334430984 weight_to_token_ratio=0.059813 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691496421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=229400 combine_payload_bytes=156098560 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156327960 ingress_bytes=231596032 link_bytes=231596032 token_bytes=312426520 weight_movement_bytes=75497472 total_nvlink_bytes=387923992 weight_to_token_ratio=0.241649 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219039226e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=228736 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156433792 ingress_bytes=212828160 link_bytes=212828160 token_bytes=312638848 weight_movement_bytes=56623104 total_nvlink_bytes=369261952 weight_to_token_ratio=0.181113 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039214711e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155648000 dispatch_meta_bytes=228640 combine_payload_bytes=155648000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155876640 ingress_bytes=174522368 link_bytes=174522368 token_bytes=311524640 weight_movement_bytes=18874368 total_nvlink_bytes=330399008 weight_to_token_ratio=0.060587 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672187460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38146 dup_rows=21976 dup_rows_remote=19314 dedup_ratio=0.335327 router_tokens_per_expert=[223, 243, 242, 242, 254, 262, 227, 247, 265, 265, 274, 258, 273, 238, 255, 267, 236, 248, 241, 244, 255, 274, 273, 239, 259, 268, 238, 250, 257, 271, 243, 235, 303, 273, 247, 273, 272, 250, 240, 264, 271, 254, 275, 238, 235, 256, 264, 259, 242, 242, 258, 271, 251, 245, 264, 259, 252, 243, 267, 262, 247, 251, 271, 277, 245, 279, 283, 275, 239, 244, 245, 264, 283, 275, 259, 271, 245, 262, 254, 239, 245, 273, 246, 250, 237, 290, 247, 240, 249, 287, 252, 277, 283, 250, 253, 267, 282, 258, 264, 263, 248, 272, 244, 257, 259, 242, 237, 240, 269, 262, 264, 265, 261, 241, 248, 254, 269, 261, 255, 242, 251, 242, 262, 242, 226, 251, 238, 249] payload_rows_per_dst=[5414, 5330, 5498, 5502, 5478, 5448, 5472, 5418] dup_rows_per_dst=[2662, 2732, 2850, 2702, 2840, 2844, 2780, 2566] combine_rows_per_src=[5414, 5330, 5498, 5502, 5478, 5448, 5472, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38254 dup_rows=21750 dup_rows_remote=19084 dedup_ratio=0.331879 router_tokens_per_expert=[264, 273, 250, 265, 276, 234, 252, 247, 265, 280, 221, 241, 249, 277, 236, 244, 239, 242, 260, 270, 239, 261, 262, 230, 266, 248, 285, 241, 273, 275, 238, 270, 254, 270, 230, 261, 263, 255, 248, 264, 263, 245, 268, 248, 249, 248, 248, 275, 256, 276, 251, 260, 254, 270, 282, 232, 263, 252, 263, 289, 243, 250, 246, 249, 251, 248, 272, 285, 245, 260, 245, 253, 237, 259, 275, 260, 249, 251, 269, 265, 250, 277, 242, 253, 227, 270, 257, 250, 239, 245, 241, 239, 255, 271, 239, 261, 253, 262, 232, 280, 268, 250, 266, 237, 241, 269, 260, 261, 279, 243, 255, 282, 251, 249, 246, 260, 282, 252, 281, 260, 259, 239, 243, 246, 232, 256, 265, 271] payload_rows_per_dst=[5476, 5532, 5526, 5546, 5432, 5326, 5492, 5456] dup_rows_per_dst=[2672, 2666, 2652, 2726, 2850, 2706, 2750, 2728] combine_rows_per_src=[5476, 5532, 5526, 5546, 5432, 5326, 5492, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229352 combine_payload_bytes=156688384 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156917736 ingress_bytes=194437120 link_bytes=194437120 token_bytes=313606120 weight_movement_bytes=37748736 total_nvlink_bytes=351354856 weight_to_token_ratio=0.120370 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863000815e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38186 dup_rows=21782 dup_rows_remote=18912 dedup_ratio=0.332367 router_tokens_per_expert=[246, 265, 256, 259, 278, 220, 244, 273, 281, 222, 250, 260, 251, 256, 272, 245, 261, 273, 250, 252, 230, 266, 236, 268, 253, 253, 248, 252, 211, 267, 257, 264, 274, 266, 254, 264, 263, 237, 264, 291, 272, 230, 251, 267, 253, 293, 263, 277, 281, 221, 262, 275, 280, 272, 227, 227, 220, 272, 253, 268, 234, 261, 239, 252, 249, 255, 233, 246, 253, 262, 234, 234, 242, 254, 250, 255, 257, 264, 273, 283, 238, 266, 249, 250, 262, 278, 238, 250, 246, 252, 275, 235, 290, 246, 254, 275, 289, 251, 263, 263, 261, 258, 258, 262, 278, 271, 218, 273, 252, 264, 240, 255, 247, 260, 261, 241, 253, 241, 250, 291, 241, 244, 276, 284, 249, 273, 253, 218] payload_rows_per_dst=[5450, 5444, 5568, 5412, 5404, 5426, 5578, 5472] dup_rows_per_dst=[2706, 2638, 2870, 2676, 2684, 2782, 2734, 2692] combine_rows_per_src=[5450, 5444, 5568, 5412, 5404, 5426, 5578, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38260 dup_rows=21728 dup_rows_remote=18984 dedup_ratio=0.331543 router_tokens_per_expert=[270, 239, 290, 258, 255, 235, 264, 242, 270, 255, 263, 280, 256, 245, 244, 272, 258, 263, 265, 243, 271, 256, 261, 239, 277, 262, 271, 272, 270, 280, 283, 256, 244, 276, 260, 253, 256, 256, 248, 223, 253, 247, 270, 273, 251, 242, 217, 235, 266, 246, 282, 242, 244, 288, 267, 271, 282, 239, 271, 255, 242, 263, 268, 268, 259, 284, 257, 267, 241, 266, 248, 249, 252, 271, 230, 257, 269, 222, 261, 239, 273, 250, 253, 276, 262, 280, 247, 238, 256, 238, 275, 245, 238, 255, 297, 263, 260, 240, 271, 245, 259, 243, 258, 252, 260, 241, 235, 261, 249, 260, 224, 264, 258, 248, 218, 231, 254, 255, 271, 239, 243, 245, 265, 232, 239, 267, 232, 268] payload_rows_per_dst=[5474, 5516, 5422, 5576, 5486, 5548, 5418, 5368] dup_rows_per_dst=[2802, 2938, 2586, 2812, 2658, 2744, 2626, 2562] combine_rows_per_src=[5474, 5516, 5422, 5576, 5486, 5548, 5418, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38210 dup_rows=21784 dup_rows_remote=19058 dedup_ratio=0.332397 router_tokens_per_expert=[251, 263, 262, 256, 246, 272, 247, 246, 243, 255, 275, 253, 207, 270, 239, 249, 268, 243, 246, 236, 254, 265, 233, 258, 240, 260, 281, 273, 260, 243, 231, 253, 245, 260, 256, 246, 276, 271, 268, 244, 243, 252, 254, 270, 279, 271, 254, 264, 269, 280, 254, 251, 267, 283, 265, 237, 250, 273, 247, 234, 239, 255, 253, 277, 271, 269, 233, 235, 258, 274, 240, 224, 291, 247, 263, 233, 225, 233, 235, 262, 266, 263, 265, 270, 261, 271, 276, 248, 258, 261, 236, 244, 253, 258, 271, 228, 258, 254, 255, 246, 254, 248, 255, 297, 268, 295, 281, 242, 256, 234, 268, 245, 268, 268, 258, 246, 255, 250, 258, 275, 257, 244, 269, 244, 257, 271, 254, 251] payload_rows_per_dst=[5492, 5388, 5514, 5542, 5382, 5478, 5448, 5508] dup_rows_per_dst=[2576, 2700, 2792, 2726, 2604, 2780, 2864, 2742] combine_rows_per_src=[5492, 5388, 5514, 5542, 5382, 5478, 5448, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38056 dup_rows=21844 dup_rows_remote=18848 dedup_ratio=0.333313 router_tokens_per_expert=[257, 231, 277, 249, 246, 238, 242, 257, 271, 256, 276, 268, 250, 251, 277, 276, 268, 243, 223, 280, 295, 274, 245, 270, 273, 265, 231, 258, 240, 273, 237, 240, 250, 274, 240, 291, 259, 260, 254, 261, 241, 253, 233, 226, 237, 267, 236, 238, 243, 255, 290, 255, 223, 260, 257, 244, 255, 262, 251, 254, 263, 221, 253, 252, 241, 279, 243, 234, 257, 254, 238, 240, 265, 256, 238, 231, 247, 254, 252, 282, 234, 252, 271, 274, 245, 252, 244, 288, 260, 273, 258, 249, 245, 266, 247, 268, 260, 296, 254, 270, 274, 244, 284, 283, 259, 260, 276, 232, 263, 256, 256, 268, 244, 236, 253, 268, 228, 254, 251, 264, 273, 254, 238, 268, 280, 252, 282, 256] payload_rows_per_dst=[5442, 5432, 5346, 5336, 5392, 5554, 5554, 5636] dup_rows_per_dst=[2802, 2730, 2600, 2310, 2630, 2838, 2938, 2996] combine_rows_per_src=[5442, 5432, 5346, 5336, 5392, 5554, 5554, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38196 dup_rows=21864 dup_rows_remote=19090 dedup_ratio=0.333618 router_tokens_per_expert=[234, 249, 281, 233, 223, 278, 288, 251, 225, 252, 273, 240, 240, 263, 230, 279, 258, 259, 247, 253, 293, 275, 266, 234, 269, 277, 238, 243, 260, 258, 253, 252, 263, 244, 245, 245, 272, 254, 255, 239, 257, 278, 239, 236, 270, 281, 244, 262, 280, 244, 258, 257, 274, 275, 277, 267, 246, 267, 271, 251, 252, 244, 255, 262, 249, 228, 247, 269, 257, 241, 262, 246, 253, 268, 248, 222, 262, 249, 250, 264, 229, 240, 267, 239, 253, 258, 263, 253, 263, 271, 252, 263, 273, 235, 275, 272, 247, 278, 248, 281, 269, 258, 253, 241, 251, 261, 238, 257, 266, 269, 255, 253, 266, 264, 260, 269, 236, 243, 256, 249, 254, 270, 233, 257, 246, 253, 299, 229] payload_rows_per_dst=[5436, 5474, 5440, 5542, 5350, 5464, 5476, 5490] dup_rows_per_dst=[2642, 2796, 2728, 2818, 2680, 2748, 2774, 2678] combine_rows_per_src=[5436, 5474, 5440, 5542, 5350, 5464, 5476, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=229840 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156475856 ingress_bytes=231743488 link_bytes=231743488 token_bytes=312721872 weight_movement_bytes=75497472 total_nvlink_bytes=388219344 weight_to_token_ratio=0.241421 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220452077e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38252 dup_rows=21794 dup_rows_remote=18946 dedup_ratio=0.332550 router_tokens_per_expert=[256, 286, 255, 237, 263, 287, 258, 259, 244, 251, 265, 246, 255, 257, 225, 256, 243, 276, 238, 236, 270, 259, 253, 256, 243, 270, 248, 266, 254, 282, 269, 261, 245, 268, 258, 263, 259, 225, 267, 264, 252, 240, 266, 273, 266, 241, 264, 256, 251, 271, 246, 260, 249, 241, 249, 249, 286, 277, 233, 242, 264, 227, 253, 277, 245, 260, 280, 250, 283, 281, 228, 258, 257, 282, 252, 260, 255, 272, 249, 257, 267, 230, 256, 265, 254, 238, 246, 276, 247, 256, 232, 274, 255, 257, 243, 247, 248, 251, 289, 272, 226, 248, 226, 257, 280, 287, 261, 267, 270, 267, 238, 225, 246, 262, 274, 253, 261, 245, 241, 284, 248, 248, 231, 243, 256, 244, 242, 260] payload_rows_per_dst=[5458, 5508, 5506, 5426, 5490, 5474, 5446, 5434] dup_rows_per_dst=[2742, 2740, 2708, 2724, 2848, 2612, 2778, 2642] combine_rows_per_src=[5458, 5508, 5506, 5426, 5490, 5474, 5446, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228392 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638248 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048104 weight_movement_bytes=37748736 total_nvlink_bytes=350796840 weight_to_token_ratio=0.120584 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228976 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156941936 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313654896 weight_movement_bytes=37748736 total_nvlink_bytes=351403632 weight_to_token_ratio=0.120351 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229072 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156737232 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245392 weight_movement_bytes=37748736 total_nvlink_bytes=350994128 weight_to_token_ratio=0.120509 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=227616 combine_payload_bytes=155877376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156104992 ingress_bytes=174751744 link_bytes=174751744 token_bytes=311982368 weight_movement_bytes=18874368 total_nvlink_bytes=330856736 weight_to_token_ratio=0.060498 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674385228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229144 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156679960 ingress_bytes=213073920 link_bytes=213073920 token_bytes=313130776 weight_movement_bytes=56623104 total_nvlink_bytes=369753880 weight_to_token_ratio=0.180829 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041569463e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38292 dup_rows=21776 dup_rows_remote=19044 dedup_ratio=0.332275 router_tokens_per_expert=[288, 257, 255, 246, 254, 267, 256, 257, 252, 257, 260, 247, 261, 267, 273, 247, 225, 246, 258, 246, 256, 267, 244, 270, 281, 252, 262, 252, 249, 281, 271, 240, 245, 243, 244, 268, 257, 235, 262, 273, 263, 223, 264, 275, 271, 251, 289, 235, 250, 244, 266, 255, 247, 253, 265, 264, 233, 251, 255, 249, 244, 280, 261, 248, 263, 267, 262, 277, 258, 250, 239, 235, 235, 253, 255, 257, 265, 245, 258, 249, 246, 259, 269, 267, 274, 257, 225, 265, 260, 260, 262, 239, 267, 295, 230, 271, 266, 247, 299, 263, 244, 253, 249, 229, 254, 250, 269, 262, 272, 246, 249, 267, 234, 269, 247, 251, 230, 243, 267, 284, 214, 257, 233, 261, 229, 274, 268, 267] payload_rows_per_dst=[5498, 5468, 5492, 5462, 5426, 5542, 5436, 5436] dup_rows_per_dst=[2790, 2732, 2704, 2668, 2710, 2750, 2802, 2620] combine_rows_per_src=[5498, 5468, 5492, 5462, 5426, 5542, 5436, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228792 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156908984 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589176 weight_movement_bytes=18874368 total_nvlink_bytes=332463544 weight_to_token_ratio=0.060188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229344 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073376 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917408 weight_movement_bytes=18874368 total_nvlink_bytes=332791776 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38274 dup_rows=21892 dup_rows_remote=19348 dedup_ratio=0.334045 router_tokens_per_expert=[258, 263, 251, 258, 267, 239, 247, 255, 257, 260, 260, 271, 254, 250, 273, 245, 243, 253, 243, 262, 274, 251, 254, 238, 234, 247, 279, 263, 267, 271, 250, 270, 266, 232, 236, 247, 264, 263, 250, 215, 243, 235, 248, 239, 256, 257, 283, 223, 252, 256, 257, 268, 249, 246, 260, 271, 246, 263, 224, 282, 279, 273, 253, 253, 219, 240, 249, 276, 266, 242, 251, 265, 232, 255, 265, 260, 276, 251, 268, 269, 259, 259, 268, 266, 246, 259, 248, 262, 250, 240, 257, 266, 251, 255, 281, 265, 267, 259, 241, 263, 279, 225, 282, 262, 246, 294, 253, 259, 251, 256, 267, 262, 283, 253, 245, 248, 266, 239, 247, 267, 282, 255, 253, 265, 245, 264, 244, 234] payload_rows_per_dst=[5496, 5428, 5370, 5472, 5440, 5514, 5486, 5438] dup_rows_per_dst=[2720, 2770, 2544, 2792, 2728, 2750, 2846, 2742] combine_rows_per_src=[5496, 5428, 5370, 5472, 5440, 5514, 5486, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38384 dup_rows=21676 dup_rows_remote=18874 dedup_ratio=0.330750 router_tokens_per_expert=[249, 239, 234, 251, 250, 267, 238, 274, 238, 265, 277, 227, 280, 247, 225, 252, 248, 244, 306, 240, 259, 247, 260, 248, 267, 239, 246, 240, 266, 249, 272, 232, 242, 233, 259, 260, 264, 277, 275, 264, 258, 257, 265, 285, 233, 274, 259, 273, 257, 236, 265, 254, 243, 236, 246, 261, 244, 254, 276, 262, 265, 271, 254, 235, 275, 239, 279, 258, 265, 270, 266, 276, 235, 245, 233, 234, 256, 229, 263, 241, 251, 259, 265, 278, 249, 289, 261, 262, 273, 251, 224, 261, 273, 264, 246, 233, 246, 254, 267, 269, 242, 263, 241, 244, 233, 255, 254, 252, 283, 258, 290, 248, 268, 251, 302, 228, 256, 241, 255, 254, 288, 280, 263, 229, 251, 260, 270, 257] payload_rows_per_dst=[5442, 5486, 5584, 5454, 5436, 5476, 5456, 5526] dup_rows_per_dst=[2584, 2640, 2772, 2664, 2692, 2802, 2742, 2780] combine_rows_per_src=[5442, 5486, 5584, 5454, 5436, 5476, 5456, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38184 dup_rows=21888 dup_rows_remote=19126 dedup_ratio=0.333984 router_tokens_per_expert=[249, 255, 256, 270, 244, 230, 258, 259, 251, 285, 263, 249, 266, 237, 239, 211, 254, 229, 276, 242, 272, 257, 251, 261, 257, 255, 265, 254, 250, 248, 258, 241, 249, 248, 286, 264, 269, 278, 254, 250, 234, 262, 273, 260, 254, 242, 251, 249, 257, 287, 245, 223, 260, 264, 274, 269, 249, 262, 269, 239, 262, 260, 272, 255, 228, 271, 259, 250, 283, 258, 256, 265, 274, 247, 265, 250, 261, 247, 235, 271, 253, 279, 276, 253, 265, 291, 248, 248, 288, 221, 261, 276, 272, 273, 247, 209, 242, 276, 270, 259, 245, 244, 248, 249, 233, 231, 272, 254, 238, 235, 266, 251, 240, 277, 254, 241, 265, 245, 295, 255, 244, 250, 266, 245, 267, 247, 256, 266] payload_rows_per_dst=[5374, 5452, 5520, 5376, 5470, 5542, 5450, 5464] dup_rows_per_dst=[2670, 2688, 2726, 2680, 2770, 2778, 2814, 2762] combine_rows_per_src=[5374, 5452, 5520, 5376, 5470, 5542, 5450, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38254 dup_rows=21724 dup_rows_remote=18906 dedup_ratio=0.331482 router_tokens_per_expert=[246, 234, 248, 261, 252, 258, 248, 220, 254, 281, 232, 265, 253, 244, 268, 243, 252, 277, 255, 263, 247, 256, 256, 256, 249, 256, 267, 257, 261, 268, 239, 261, 277, 270, 244, 239, 236, 258, 239, 267, 255, 235, 243, 233, 274, 236, 240, 268, 263, 250, 243, 254, 235, 262, 262, 255, 258, 235, 272, 270, 259, 232, 240, 271, 270, 257, 253, 257, 263, 267, 236, 237, 276, 262, 249, 263, 255, 283, 232, 256, 279, 256, 263, 237, 262, 243, 256, 260, 282, 247, 268, 246, 241, 278, 270, 238, 253, 254, 253, 251, 255, 256, 276, 253, 278, 290, 291, 258, 236, 257, 271, 256, 250, 258, 257, 260, 251, 268, 296, 263, 264, 259, 261, 231, 260, 262, 251, 245] payload_rows_per_dst=[5440, 5500, 5434, 5366, 5494, 5554, 5558, 5466] dup_rows_per_dst=[2574, 2740, 2594, 2756, 2738, 2698, 2818, 2806] combine_rows_per_src=[5440, 5500, 5434, 5366, 5494, 5554, 5558, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38108 dup_rows=21880 dup_rows_remote=18832 dedup_ratio=0.333862 router_tokens_per_expert=[230, 242, 257, 263, 251, 241, 296, 266, 247, 286, 278, 248, 280, 250, 250, 284, 252, 257, 279, 255, 233, 262, 273, 261, 247, 282, 271, 264, 251, 269, 244, 231, 258, 245, 249, 268, 292, 242, 241, 239, 231, 278, 266, 258, 255, 271, 235, 228, 238, 264, 270, 255, 244, 260, 274, 259, 264, 250, 264, 247, 262, 269, 262, 241, 260, 264, 255, 257, 272, 242, 258, 268, 265, 229, 235, 251, 245, 257, 233, 277, 264, 231, 254, 267, 266, 266, 255, 270, 269, 273, 277, 233, 239, 245, 266, 256, 226, 261, 270, 242, 253, 255, 237, 280, 231, 256, 252, 245, 251, 244, 254, 270, 251, 257, 250, 236, 276, 272, 277, 277, 224, 255, 262, 258, 207, 264, 251, 246] payload_rows_per_dst=[5548, 5558, 5518, 5250, 5472, 5456, 5472, 5382] dup_rows_per_dst=[3048, 2714, 2790, 2532, 2664, 2806, 2582, 2744] combine_rows_per_src=[5548, 5558, 5518, 5250, 5472, 5456, 5472, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38192 dup_rows=21900 dup_rows_remote=19090 dedup_ratio=0.334167 router_tokens_per_expert=[253, 264, 270, 266, 244, 252, 272, 257, 243, 242, 262, 287, 266, 260, 239, 231, 241, 250, 273, 259, 259, 222, 283, 232, 234, 271, 251, 257, 247, 269, 253, 248, 276, 237, 271, 254, 275, 268, 261, 264, 255, 250, 262, 235, 253, 256, 268, 230, 257, 264, 248, 248, 254, 236, 256, 233, 285, 240, 265, 264, 268, 279, 263, 267, 268, 267, 247, 245, 266, 263, 248, 220, 256, 252, 291, 259, 252, 270, 265, 261, 249, 299, 239, 264, 232, 265, 247, 273, 255, 258, 264, 228, 264, 271, 276, 239, 242, 230, 257, 259, 233, 230, 280, 238, 222, 270, 251, 221, 269, 234, 252, 248, 288, 275, 258, 268, 249, 254, 272, 259, 247, 267, 284, 225, 252, 256, 287, 239] payload_rows_per_dst=[5454, 5426, 5506, 5444, 5434, 5488, 5366, 5518] dup_rows_per_dst=[2762, 2672, 2724, 2810, 2826, 2758, 2506, 2842] combine_rows_per_src=[5454, 5426, 5506, 5444, 5434, 5488, 5366, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38130 dup_rows=21866 dup_rows_remote=19134 dedup_ratio=0.333649 router_tokens_per_expert=[266, 267, 249, 237, 288, 268, 236, 236, 241, 271, 239, 250, 255, 259, 266, 245, 244, 232, 253, 246, 254, 265, 274, 253, 231, 239, 261, 273, 255, 238, 263, 250, 253, 262, 268, 256, 268, 261, 261, 248, 243, 262, 278, 282, 268, 260, 254, 267, 235, 280, 263, 250, 284, 268, 255, 218, 260, 250, 259, 266, 277, 280, 284, 240, 291, 266, 264, 263, 246, 240, 251, 253, 288, 273, 236, 248, 266, 250, 255, 246, 251, 245, 277, 269, 230, 235, 233, 251, 226, 268, 268, 259, 254, 267, 256, 253, 274, 262, 256, 221, 255, 263, 281, 238, 249, 260, 253, 261, 248, 234, 249, 250, 244, 267, 266, 270, 267, 247, 273, 247, 258, 235, 237, 274, 268, 251, 227, 241] payload_rows_per_dst=[5448, 5354, 5550, 5490, 5540, 5432, 5400, 5456] dup_rows_per_dst=[2698, 2708, 2832, 2848, 2732, 2652, 2708, 2688] combine_rows_per_src=[5448, 5354, 5550, 5490, 5540, 5432, 5400, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=230488 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157000792 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313771096 weight_movement_bytes=18874368 total_nvlink_bytes=332645464 weight_to_token_ratio=0.060153 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229032 combine_payload_bytes=157220864 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157449896 ingress_bytes=194969600 link_bytes=194969600 token_bytes=314670760 weight_movement_bytes=37748736 total_nvlink_bytes=352419496 weight_to_token_ratio=0.119963 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868102776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38030 dup_rows=21908 dup_rows_remote=18978 dedup_ratio=0.334290 router_tokens_per_expert=[244, 276, 271, 267, 253, 282, 238, 281, 257, 230, 262, 259, 255, 260, 262, 249, 235, 247, 246, 249, 218, 254, 234, 244, 272, 260, 245, 232, 285, 246, 232, 242, 251, 247, 250, 268, 266, 264, 242, 249, 243, 248, 238, 267, 251, 280, 248, 252, 265, 259, 270, 274, 268, 243, 253, 265, 253, 249, 264, 246, 278, 247, 242, 263, 237, 266, 286, 224, 261, 252, 242, 259, 247, 238, 254, 289, 296, 251, 259, 258, 259, 244, 246, 267, 258, 278, 295, 323, 250, 249, 270, 252, 238, 264, 278, 243, 272, 265, 243, 246, 224, 252, 292, 264, 255, 253, 259, 259, 233, 231, 252, 280, 248, 261, 260, 252, 246, 228, 288, 248, 232, 263, 257, 262, 263, 217, 259, 281] payload_rows_per_dst=[5504, 5598, 5454, 5486, 5446, 5302, 5394, 5444] dup_rows_per_dst=[2788, 2930, 2674, 2792, 2792, 2480, 2766, 2686] combine_rows_per_src=[5504, 5598, 5454, 5486, 5446, 5302, 5394, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229240 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156630904 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032568 weight_movement_bytes=18874368 total_nvlink_bytes=331906936 weight_to_token_ratio=0.060295 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228640 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917024 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605408 weight_movement_bytes=18874368 total_nvlink_bytes=332479776 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229128 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156663560 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313097992 weight_movement_bytes=37748736 total_nvlink_bytes=350846728 weight_to_token_ratio=0.120565 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=227760 combine_payload_bytes=156090368 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156318128 ingress_bytes=193839104 link_bytes=193839104 token_bytes=312408496 weight_movement_bytes=37748736 total_nvlink_bytes=350157232 weight_to_token_ratio=0.120831 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857270920e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229056 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156409536 ingress_bytes=212803584 link_bytes=212803584 token_bytes=312590016 weight_movement_bytes=56623104 total_nvlink_bytes=369213120 weight_to_token_ratio=0.181142 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038979236e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155770880 dispatch_meta_bytes=228032 combine_payload_bytes=155770880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155998912 ingress_bytes=174645248 link_bytes=174645248 token_bytes=311769792 weight_movement_bytes=18874368 total_nvlink_bytes=330644160 weight_to_token_ratio=0.060539 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673364836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38132 dup_rows=21956 dup_rows_remote=19276 dedup_ratio=0.335022 router_tokens_per_expert=[236, 257, 264, 253, 268, 267, 247, 249, 253, 233, 258, 249, 269, 253, 239, 235, 271, 212, 243, 249, 241, 248, 258, 257, 254, 247, 256, 258, 257, 224, 241, 249, 241, 247, 249, 265, 279, 234, 245, 275, 252, 274, 228, 262, 273, 243, 233, 264, 269, 263, 269, 260, 231, 244, 246, 255, 245, 266, 238, 264, 279, 241, 268, 272, 274, 290, 231, 246, 237, 246, 234, 259, 285, 270, 256, 260, 241, 259, 244, 259, 266, 245, 282, 264, 253, 279, 231, 246, 249, 282, 271, 266, 283, 267, 268, 256, 252, 245, 266, 249, 263, 288, 266, 249, 259, 259, 257, 267, 259, 279, 257, 264, 248, 235, 252, 286, 234, 254, 246, 257, 264, 244, 255, 255, 282, 285, 256, 268] payload_rows_per_dst=[5438, 5326, 5448, 5486, 5414, 5502, 5508, 5458] dup_rows_per_dst=[2622, 2674, 2680, 2734, 2768, 2844, 2850, 2784] combine_rows_per_src=[5438, 5326, 5448, 5486, 5414, 5502, 5508, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38296 dup_rows=21682 dup_rows_remote=19026 dedup_ratio=0.330841 router_tokens_per_expert=[265, 275, 250, 236, 258, 262, 240, 227, 267, 245, 229, 293, 300, 260, 264, 256, 288, 199, 256, 254, 239, 232, 281, 242, 260, 222, 257, 268, 230, 236, 240, 253, 269, 248, 241, 265, 267, 257, 264, 241, 254, 268, 259, 274, 242, 273, 251, 250, 249, 287, 241, 242, 237, 270, 255, 252, 220, 266, 271, 249, 256, 259, 263, 277, 262, 256, 242, 288, 260, 242, 239, 264, 264, 264, 282, 260, 268, 264, 247, 246, 252, 249, 267, 264, 260, 252, 253, 290, 254, 235, 259, 267, 243, 263, 250, 249, 245, 257, 245, 248, 241, 253, 303, 267, 247, 256, 225, 264, 239, 263, 247, 273, 242, 273, 264, 270, 264, 276, 258, 271, 243, 250, 271, 237, 242, 257, 259, 262] payload_rows_per_dst=[5472, 5446, 5506, 5398, 5412, 5558, 5462, 5600] dup_rows_per_dst=[2782, 2468, 2740, 2790, 2884, 2656, 2684, 2678] combine_rows_per_src=[5472, 5446, 5506, 5398, 5412, 5558, 5462, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38234 dup_rows=21754 dup_rows_remote=18970 dedup_ratio=0.331940 router_tokens_per_expert=[261, 283, 255, 240, 271, 268, 243, 241, 226, 233, 262, 226, 262, 236, 266, 245, 249, 283, 272, 251, 261, 246, 254, 270, 248, 268, 257, 276, 251, 252, 258, 248, 238, 249, 261, 238, 253, 247, 268, 274, 259, 257, 272, 253, 252, 260, 257, 245, 249, 268, 246, 242, 241, 257, 253, 287, 269, 266, 260, 262, 263, 270, 272, 283, 225, 243, 275, 255, 241, 266, 263, 246, 257, 250, 256, 266, 266, 269, 245, 252, 246, 237, 256, 235, 278, 220, 242, 252, 268, 235, 255, 261, 254, 259, 283, 260, 270, 257, 248, 259, 254, 285, 262, 237, 246, 267, 240, 234, 239, 254, 247, 261, 228, 271, 250, 248, 237, 284, 292, 272, 264, 269, 263, 255, 249, 268, 260, 249] payload_rows_per_dst=[5310, 5466, 5470, 5404, 5446, 5676, 5462, 5548] dup_rows_per_dst=[2564, 2822, 2678, 2628, 2704, 2912, 2662, 2784] combine_rows_per_src=[5310, 5466, 5470, 5404, 5446, 5676, 5462, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38314 dup_rows=21774 dup_rows_remote=19154 dedup_ratio=0.332245 router_tokens_per_expert=[240, 233, 269, 267, 258, 255, 249, 241, 263, 294, 262, 267, 291, 244, 276, 229, 246, 240, 237, 262, 250, 245, 258, 258, 255, 285, 267, 290, 281, 256, 228, 245, 268, 238, 246, 253, 269, 279, 256, 262, 269, 236, 263, 240, 264, 227, 263, 264, 265, 251, 254, 236, 241, 273, 240, 299, 271, 255, 249, 276, 250, 276, 261, 228, 252, 237, 248, 243, 270, 276, 228, 266, 232, 245, 247, 269, 258, 243, 231, 257, 254, 265, 283, 271, 273, 284, 259, 243, 249, 281, 266, 250, 218, 251, 276, 268, 268, 266, 242, 221, 271, 249, 247, 235, 234, 253, 262, 261, 250, 242, 266, 267, 253, 259, 231, 297, 239, 259, 255, 240, 248, 239, 278, 278, 251, 250, 247, 254] payload_rows_per_dst=[5526, 5472, 5456, 5510, 5396, 5540, 5448, 5414] dup_rows_per_dst=[2750, 2734, 2738, 2740, 2608, 2842, 2620, 2742] combine_rows_per_src=[5526, 5472, 5456, 5510, 5396, 5540, 5448, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38244 dup_rows=21856 dup_rows_remote=19210 dedup_ratio=0.333496 router_tokens_per_expert=[278, 243, 238, 250, 266, 249, 243, 249, 282, 243, 292, 268, 245, 251, 258, 278, 220, 288, 261, 265, 253, 246, 219, 261, 253, 251, 253, 243, 276, 238, 276, 270, 269, 231, 251, 242, 275, 277, 262, 279, 251, 254, 245, 239, 251, 251, 237, 253, 236, 267, 236, 263, 237, 283, 247, 220, 239, 256, 234, 277, 263, 246, 251, 286, 252, 264, 250, 248, 291, 267, 267, 248, 262, 251, 245, 244, 267, 247, 262, 227, 268, 257, 250, 231, 287, 245, 248, 262, 263, 276, 262, 239, 239, 289, 232, 253, 283, 264, 272, 245, 261, 250, 241, 274, 258, 289, 276, 256, 258, 249, 265, 252, 237, 260, 246, 261, 275, 250, 248, 246, 268, 237, 270, 243, 250, 245, 259, 273] payload_rows_per_dst=[5440, 5466, 5434, 5436, 5448, 5460, 5556, 5440] dup_rows_per_dst=[2826, 2680, 2700, 2646, 2736, 2742, 2830, 2696] combine_rows_per_src=[5440, 5466, 5434, 5436, 5448, 5460, 5556, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229632 combine_payload_bytes=156188672 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156418304 ingress_bytes=212811776 link_bytes=212811776 token_bytes=312606976 weight_movement_bytes=56623104 total_nvlink_bytes=369230080 weight_to_token_ratio=0.181132 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039057728e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38414 dup_rows=21596 dup_rows_remote=18756 dedup_ratio=0.329529 router_tokens_per_expert=[264, 285, 264, 237, 258, 270, 217, 229, 270, 258, 282, 241, 269, 240, 282, 263, 255, 282, 267, 271, 263, 242, 252, 252, 232, 240, 262, 249, 227, 239, 251, 233, 276, 265, 228, 271, 278, 284, 258, 272, 239, 249, 245, 265, 269, 230, 265, 245, 286, 265, 228, 247, 268, 234, 238, 273, 252, 273, 240, 252, 252, 251, 237, 258, 266, 238, 250, 257, 230, 269, 256, 223, 250, 261, 271, 236, 250, 265, 261, 272, 243, 234, 260, 257, 227, 262, 270, 265, 275, 301, 256, 283, 251, 278, 230, 264, 246, 277, 246, 259, 281, 276, 234, 265, 241, 268, 282, 232, 269, 257, 258, 261, 230, 243, 250, 257, 222, 268, 253, 253, 274, 250, 264, 264, 261, 264, 249, 264] payload_rows_per_dst=[5526, 5470, 5548, 5548, 5722, 5452, 5232, 5442] dup_rows_per_dst=[2840, 2564, 2730, 2816, 2924, 2604, 2428, 2690] combine_rows_per_src=[5526, 5470, 5548, 5548, 5722, 5452, 5232, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38264 dup_rows=21834 dup_rows_remote=19042 dedup_ratio=0.333160 router_tokens_per_expert=[270, 269, 277, 230, 228, 262, 233, 269, 267, 284, 255, 285, 238, 262, 248, 247, 240, 251, 261, 241, 264, 257, 244, 256, 236, 244, 242, 267, 266, 245, 258, 247, 258, 267, 228, 255, 234, 238, 284, 267, 281, 272, 260, 268, 268, 263, 269, 250, 277, 247, 269, 259, 245, 263, 242, 258, 257, 259, 232, 266, 249, 252, 261, 242, 259, 262, 250, 262, 259, 249, 260, 274, 251, 263, 256, 247, 256, 244, 237, 286, 264, 244, 269, 256, 261, 265, 269, 265, 267, 250, 277, 277, 246, 258, 252, 252, 247, 233, 250, 286, 211, 233, 269, 249, 269, 251, 253, 258, 250, 274, 237, 240, 250, 258, 266, 232, 250, 278, 267, 249, 229, 259, 287, 256, 248, 246, 256, 257] payload_rows_per_dst=[5472, 5386, 5540, 5436, 5438, 5504, 5388, 5538] dup_rows_per_dst=[2776, 2652, 2784, 2720, 2792, 2840, 2632, 2638] combine_rows_per_src=[5472, 5386, 5540, 5436, 5438, 5504, 5388, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229288 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157089704 ingress_bytes=194609152 link_bytes=194609152 token_bytes=313950120 weight_movement_bytes=37748736 total_nvlink_bytes=351698856 weight_to_token_ratio=0.120238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864649141e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228816 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156835280 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441744 weight_movement_bytes=37748736 total_nvlink_bytes=351190480 weight_to_token_ratio=0.120433 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43888 payload_rows_remote=38452 dup_rows=21648 dup_rows_remote=19142 dedup_ratio=0.330322 router_tokens_per_expert=[250, 267, 228, 267, 244, 243, 252, 253, 278, 247, 247, 284, 265, 286, 267, 271, 270, 242, 253, 241, 226, 273, 275, 266, 229, 225, 240, 261, 255, 233, 265, 217, 267, 257, 256, 228, 232, 260, 251, 253, 300, 241, 252, 263, 269, 279, 238, 272, 249, 262, 242, 245, 252, 272, 249, 233, 255, 270, 231, 248, 251, 250, 256, 274, 246, 254, 270, 275, 283, 253, 251, 240, 267, 259, 263, 270, 236, 260, 244, 257, 250, 267, 247, 266, 264, 258, 266, 260, 256, 255, 246, 241, 249, 259, 253, 252, 260, 263, 257, 269, 252, 273, 242, 259, 237, 277, 254, 251, 269, 253, 235, 283, 242, 270, 261, 253, 257, 251, 261, 252, 246, 276, 260, 246, 281, 272, 256, 256] payload_rows_per_dst=[5580, 5436, 5534, 5448, 5514, 5402, 5510, 5464] dup_rows_per_dst=[2718, 2506, 2702, 2630, 2742, 2776, 2758, 2816] combine_rows_per_src=[5580, 5436, 5534, 5448, 5514, 5402, 5510, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229872 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157164016 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098160 weight_movement_bytes=18874368 total_nvlink_bytes=332972528 weight_to_token_ratio=0.060091 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229816 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156877240 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524664 weight_movement_bytes=37748736 total_nvlink_bytes=351273400 weight_to_token_ratio=0.120401 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=228680 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157572424 ingress_bytes=176218112 link_bytes=176218112 token_bytes=314916168 weight_movement_bytes=18874368 total_nvlink_bytes=333790536 weight_to_token_ratio=0.059935 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688435244e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229224 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958568 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687912 weight_movement_bytes=18874368 total_nvlink_bytes=332562280 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157499392 dispatch_meta_bytes=230376 combine_payload_bytes=157499392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157729768 ingress_bytes=176373760 link_bytes=176373760 token_bytes=315229160 weight_movement_bytes=18874368 total_nvlink_bytes=334103528 weight_to_token_ratio=0.059875 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689926587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38214 dup_rows=21792 dup_rows_remote=19074 dedup_ratio=0.332520 router_tokens_per_expert=[256, 264, 262, 256, 238, 257, 268, 261, 253, 256, 269, 253, 273, 243, 258, 294, 263, 240, 281, 228, 275, 241, 258, 246, 236, 255, 253, 282, 270, 244, 265, 233, 248, 269, 267, 227, 239, 275, 279, 276, 284, 277, 255, 238, 257, 257, 240, 236, 280, 263, 268, 261, 231, 249, 264, 253, 264, 264, 269, 245, 264, 251, 263, 267, 264, 250, 252, 265, 261, 273, 247, 246, 244, 266, 245, 258, 281, 251, 257, 260, 253, 258, 254, 257, 260, 251, 258, 278, 258, 273, 262, 232, 267, 271, 238, 269, 246, 243, 269, 257, 230, 253, 265, 261, 265, 271, 242, 225, 242, 281, 250, 262, 240, 261, 250, 266, 240, 237, 236, 257, 243, 239, 274, 256, 215, 250, 233, 239] payload_rows_per_dst=[5500, 5512, 5530, 5528, 5464, 5570, 5312, 5328] dup_rows_per_dst=[2822, 2628, 2718, 2784, 2776, 2708, 2812, 2544] combine_rows_per_src=[5500, 5512, 5530, 5528, 5464, 5570, 5312, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229152 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156753696 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278240 weight_movement_bytes=18874368 total_nvlink_bytes=332152608 weight_to_token_ratio=0.060248 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38430 dup_rows=21634 dup_rows_remote=18790 dedup_ratio=0.330109 router_tokens_per_expert=[254, 239, 263, 260, 249, 272, 254, 244, 265, 238, 245, 263, 243, 262, 251, 225, 280, 230, 266, 275, 243, 249, 255, 252, 259, 230, 255, 237, 278, 263, 258, 265, 250, 260, 242, 270, 245, 253, 237, 257, 256, 234, 260, 267, 262, 248, 237, 253, 258, 264, 235, 268, 259, 254, 266, 263, 234, 259, 264, 259, 254, 268, 249, 258, 261, 261, 270, 237, 232, 244, 272, 255, 250, 264, 253, 266, 265, 244, 237, 248, 257, 262, 247, 284, 240, 259, 274, 305, 264, 270, 268, 250, 235, 275, 270, 198, 263, 276, 233, 245, 258, 238, 277, 233, 289, 247, 263, 265, 271, 258, 295, 242, 259, 266, 271, 243, 273, 254, 265, 250, 235, 246, 278, 259, 278, 268, 236, 252] payload_rows_per_dst=[5436, 5530, 5438, 5484, 5470, 5472, 5498, 5574] dup_rows_per_dst=[2618, 2660, 2624, 2740, 2648, 2844, 2808, 2692] combine_rows_per_src=[5436, 5530, 5438, 5484, 5470, 5472, 5498, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38220 dup_rows=21772 dup_rows_remote=18962 dedup_ratio=0.332214 router_tokens_per_expert=[256, 252, 252, 261, 255, 257, 244, 255, 256, 269, 245, 258, 251, 277, 289, 266, 266, 260, 211, 240, 248, 267, 222, 276, 234, 261, 256, 250, 248, 248, 244, 269, 258, 240, 267, 273, 271, 257, 241, 266, 272, 239, 244, 258, 298, 257, 233, 255, 224, 266, 258, 258, 264, 220, 238, 265, 239, 270, 257, 266, 247, 265, 281, 270, 209, 241, 262, 252, 258, 262, 260, 259, 248, 273, 256, 242, 268, 237, 259, 257, 241, 230, 265, 267, 275, 251, 269, 283, 260, 232, 236, 238, 256, 228, 268, 250, 258, 272, 251, 274, 260, 238, 261, 255, 250, 272, 273, 258, 260, 262, 267, 270, 248, 259, 263, 250, 253, 265, 259, 269, 253, 255, 255, 264, 245, 242, 258, 297] payload_rows_per_dst=[5374, 5342, 5632, 5416, 5464, 5398, 5594, 5544] dup_rows_per_dst=[2552, 2658, 2986, 2760, 2622, 2678, 2706, 2810] combine_rows_per_src=[5374, 5342, 5632, 5416, 5464, 5398, 5594, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38272 dup_rows=21808 dup_rows_remote=19118 dedup_ratio=0.332764 router_tokens_per_expert=[250, 242, 283, 270, 250, 250, 269, 277, 258, 259, 252, 257, 274, 264, 251, 247, 253, 270, 235, 295, 260, 248, 213, 264, 275, 249, 272, 235, 263, 249, 252, 245, 247, 262, 253, 246, 282, 248, 233, 237, 271, 267, 251, 245, 234, 262, 261, 267, 250, 258, 240, 259, 265, 255, 254, 249, 240, 256, 267, 248, 251, 270, 280, 231, 251, 253, 263, 248, 240, 287, 266, 261, 228, 257, 261, 268, 280, 261, 256, 248, 262, 259, 236, 260, 236, 257, 265, 251, 254, 261, 249, 253, 256, 260, 261, 246, 258, 270, 270, 240, 252, 241, 263, 252, 243, 236, 265, 247, 249, 259, 255, 273, 253, 268, 272, 226, 262, 264, 229, 257, 247, 265, 267, 266, 282, 264, 251, 258] payload_rows_per_dst=[5504, 5408, 5484, 5444, 5476, 5460, 5456, 5496] dup_rows_per_dst=[2802, 2748, 2648, 2702, 2780, 2672, 2690, 2766] combine_rows_per_src=[5504, 5408, 5484, 5444, 5476, 5460, 5456, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38288 dup_rows=21838 dup_rows_remote=19158 dedup_ratio=0.333221 router_tokens_per_expert=[233, 243, 275, 238, 274, 256, 243, 248, 246, 254, 253, 256, 248, 234, 247, 248, 272, 235, 284, 274, 251, 272, 246, 269, 272, 243, 245, 273, 257, 220, 266, 264, 264, 269, 258, 229, 241, 238, 270, 258, 238, 255, 282, 274, 277, 242, 232, 257, 233, 246, 263, 272, 233, 271, 257, 239, 248, 265, 255, 264, 243, 236, 262, 258, 279, 251, 275, 240, 272, 263, 267, 269, 262, 291, 284, 238, 255, 255, 270, 265, 269, 264, 249, 258, 237, 254, 260, 249, 263, 260, 256, 260, 236, 266, 238, 227, 235, 248, 288, 232, 221, 248, 263, 307, 253, 256, 247, 250, 274, 246, 235, 267, 241, 276, 269, 236, 283, 247, 245, 234, 268, 258, 278, 274, 237, 270, 267, 265] payload_rows_per_dst=[5352, 5516, 5416, 5410, 5634, 5398, 5468, 5504] dup_rows_per_dst=[2640, 2770, 2752, 2680, 2838, 2694, 2672, 2792] combine_rows_per_src=[5352, 5516, 5416, 5410, 5634, 5398, 5468, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=37990 dup_rows=21888 dup_rows_remote=18960 dedup_ratio=0.333984 router_tokens_per_expert=[264, 261, 253, 237, 252, 240, 236, 274, 247, 248, 268, 275, 245, 227, 242, 264, 247, 252, 281, 247, 254, 241, 270, 243, 253, 262, 240, 245, 242, 261, 253, 233, 250, 236, 250, 266, 249, 245, 267, 257, 284, 252, 236, 254, 245, 250, 267, 267, 269, 285, 241, 272, 233, 260, 267, 257, 238, 248, 257, 274, 259, 254, 242, 253, 254, 237, 257, 270, 258, 272, 242, 240, 262, 235, 262, 265, 253, 276, 241, 255, 299, 255, 239, 255, 278, 244, 266, 243, 247, 265, 245, 248, 248, 269, 285, 232, 263, 270, 242, 254, 261, 274, 267, 272, 244, 265, 242, 242, 234, 234, 270, 240, 264, 250, 224, 293, 269, 239, 283, 295, 268, 265, 283, 265, 266, 293, 244, 255] payload_rows_per_dst=[5658, 5528, 5470, 5500, 5398, 5310, 5394, 5390] dup_rows_per_dst=[2928, 2944, 2714, 2754, 2702, 2490, 2754, 2602] combine_rows_per_src=[5658, 5528, 5470, 5500, 5398, 5310, 5394, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38248 dup_rows=21702 dup_rows_remote=18858 dedup_ratio=0.331146 router_tokens_per_expert=[250, 265, 249, 275, 238, 270, 283, 238, 231, 247, 257, 235, 270, 251, 257, 256, 256, 242, 262, 270, 248, 282, 265, 278, 279, 241, 230, 254, 241, 259, 273, 268, 229, 261, 255, 232, 267, 270, 263, 243, 252, 284, 224, 270, 237, 270, 234, 227, 234, 254, 259, 239, 241, 253, 268, 242, 259, 269, 251, 240, 252, 276, 250, 230, 254, 257, 274, 262, 248, 236, 260, 276, 271, 269, 283, 239, 267, 309, 260, 250, 243, 273, 238, 264, 254, 256, 264, 267, 224, 241, 260, 259, 245, 257, 251, 254, 252, 234, 261, 287, 248, 266, 279, 263, 284, 239, 245, 256, 255, 244, 226, 246, 227, 263, 253, 251, 256, 267, 283, 264, 266, 267, 258, 257, 280, 265, 245, 261] payload_rows_per_dst=[5410, 5496, 5454, 5422, 5586, 5452, 5472, 5542] dup_rows_per_dst=[2734, 2800, 2582, 2612, 2844, 2648, 2698, 2784] combine_rows_per_src=[5410, 5496, 5454, 5422, 5586, 5452, 5472, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=228880 combine_payload_bytes=157409280 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157638160 ingress_bytes=176283648 link_bytes=176283648 token_bytes=315047440 weight_movement_bytes=18874368 total_nvlink_bytes=333921808 weight_to_token_ratio=0.059910 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689063178e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=228728 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156777848 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313326968 weight_movement_bytes=37748736 total_nvlink_bytes=351075704 weight_to_token_ratio=0.120477 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38220 dup_rows=21876 dup_rows_remote=19226 dedup_ratio=0.333801 router_tokens_per_expert=[252, 247, 266, 262, 271, 227, 251, 287, 262, 238, 267, 259, 276, 229, 260, 259, 264, 273, 251, 250, 262, 250, 217, 236, 266, 247, 248, 257, 270, 267, 236, 251, 243, 257, 263, 228, 233, 261, 258, 262, 263, 246, 250, 278, 282, 268, 244, 265, 257, 252, 283, 251, 239, 263, 259, 268, 246, 266, 265, 249, 220, 236, 248, 271, 256, 244, 261, 253, 254, 247, 260, 246, 256, 254, 250, 245, 279, 266, 244, 257, 251, 270, 259, 254, 245, 253, 267, 273, 249, 274, 265, 230, 231, 264, 275, 260, 259, 248, 287, 250, 251, 244, 247, 264, 264, 281, 265, 265, 273, 265, 266, 251, 261, 249, 263, 249, 255, 245, 265, 256, 253, 269, 242, 225, 261, 264, 273, 234] payload_rows_per_dst=[5454, 5440, 5488, 5354, 5518, 5442, 5470, 5494] dup_rows_per_dst=[2772, 2650, 2714, 2792, 2626, 2798, 2890, 2634] combine_rows_per_src=[5454, 5440, 5488, 5354, 5518, 5442, 5470, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229560 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156991672 ingress_bytes=232259584 link_bytes=232259584 token_bytes=313753784 weight_movement_bytes=75497472 total_nvlink_bytes=389251256 weight_to_token_ratio=0.240626 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225397055e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229784 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057432 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885080 weight_movement_bytes=18874368 total_nvlink_bytes=332759448 weight_to_token_ratio=0.060131 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155607040 dispatch_meta_bytes=227800 combine_payload_bytes=155607040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155834840 ingress_bytes=174481408 link_bytes=174481408 token_bytes=311441880 weight_movement_bytes=18874368 total_nvlink_bytes=330316248 weight_to_token_ratio=0.060603 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671795001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=37994 dup_rows=21996 dup_rows_remote=19076 dedup_ratio=0.335632 router_tokens_per_expert=[252, 251, 256, 279, 259, 256, 290, 240, 262, 250, 243, 272, 275, 232, 247, 232, 250, 250, 251, 255, 290, 274, 253, 240, 249, 254, 246, 242, 237, 267, 263, 251, 263, 262, 280, 269, 246, 279, 255, 248, 262, 246, 251, 293, 291, 263, 250, 275, 247, 262, 274, 250, 258, 259, 260, 233, 235, 248, 255, 253, 261, 280, 254, 241, 222, 267, 248, 234, 252, 281, 259, 242, 242, 294, 242, 243, 249, 262, 258, 251, 235, 239, 250, 248, 246, 261, 251, 259, 257, 266, 231, 267, 264, 245, 253, 255, 266, 260, 262, 256, 253, 264, 277, 267, 254, 241, 258, 294, 244, 272, 259, 243, 269, 252, 232, 242, 262, 247, 262, 254, 239, 234, 266, 272, 268, 266, 242, 247] payload_rows_per_dst=[5406, 5446, 5546, 5442, 5418, 5426, 5476, 5380] dup_rows_per_dst=[2786, 2698, 2920, 2698, 2674, 2628, 2864, 2728] combine_rows_per_src=[5406, 5446, 5546, 5442, 5418, 5426, 5476, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=228424 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156892232 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556040 weight_movement_bytes=37748736 total_nvlink_bytes=351304776 weight_to_token_ratio=0.120389 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229784 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778904 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313328024 weight_movement_bytes=18874368 total_nvlink_bytes=332202392 weight_to_token_ratio=0.060238 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155623424 dispatch_meta_bytes=228280 combine_payload_bytes=155623424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155851704 ingress_bytes=193372160 link_bytes=193372160 token_bytes=311475128 weight_movement_bytes=37748736 total_nvlink_bytes=349223864 weight_to_token_ratio=0.121193 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852796892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=37986 dup_rows=21990 dup_rows_remote=18982 dedup_ratio=0.335541 router_tokens_per_expert=[225, 244, 234, 269, 256, 249, 250, 271, 281, 279, 268, 249, 285, 282, 251, 252, 235, 257, 278, 256, 280, 267, 246, 241, 259, 261, 280, 252, 271, 232, 242, 254, 250, 264, 236, 245, 278, 254, 271, 268, 270, 247, 272, 267, 251, 234, 260, 255, 282, 237, 279, 240, 252, 263, 254, 282, 286, 240, 271, 263, 246, 247, 240, 253, 269, 276, 266, 247, 236, 246, 261, 246, 248, 241, 272, 271, 248, 280, 252, 252, 250, 289, 263, 277, 264, 250, 227, 272, 265, 261, 264, 268, 260, 265, 226, 240, 279, 258, 242, 243, 244, 240, 250, 222, 244, 278, 261, 245, 234, 272, 233, 259, 245, 265, 239, 237, 247, 242, 260, 262, 222, 252, 266, 258, 236, 277, 251, 240] payload_rows_per_dst=[5190, 5488, 5254, 5526, 5514, 5426, 5588, 5560] dup_rows_per_dst=[2530, 2734, 2488, 2744, 2708, 2856, 2922, 3008] combine_rows_per_src=[5190, 5488, 5254, 5526, 5514, 5426, 5588, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43914 payload_rows_remote=38370 dup_rows=21622 dup_rows_remote=18932 dedup_ratio=0.329926 router_tokens_per_expert=[262, 257, 276, 287, 269, 243, 235, 242, 279, 243, 260, 264, 277, 235, 234, 247, 218, 280, 253, 226, 265, 244, 261, 271, 235, 246, 271, 242, 286, 270, 238, 273, 235, 235, 243, 258, 254, 252, 237, 257, 254, 272, 238, 236, 247, 262, 238, 221, 262, 254, 285, 230, 270, 230, 247, 259, 274, 235, 274, 262, 271, 262, 241, 278, 272, 274, 259, 240, 252, 272, 259, 260, 284, 269, 286, 243, 257, 256, 251, 268, 237, 255, 252, 262, 270, 264, 254, 244, 262, 239, 262, 281, 268, 241, 276, 250, 234, 226, 241, 269, 267, 247, 250, 258, 233, 272, 272, 259, 255, 262, 252, 260, 268, 268, 251, 252, 239, 246, 243, 257, 261, 244, 258, 272, 266, 273, 262, 270] payload_rows_per_dst=[5444, 5496, 5414, 5444, 5552, 5544, 5478, 5542] dup_rows_per_dst=[2776, 2662, 2464, 2824, 2852, 2690, 2636, 2718] combine_rows_per_src=[5444, 5496, 5414, 5444, 5552, 5544, 5478, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38226 dup_rows=21886 dup_rows_remote=19180 dedup_ratio=0.333954 router_tokens_per_expert=[267, 248, 278, 240, 258, 286, 272, 258, 250, 251, 260, 290, 265, 238, 258, 255, 250, 239, 274, 255, 241, 268, 246, 254, 247, 243, 270, 271, 275, 264, 259, 228, 250, 271, 268, 240, 270, 305, 242, 232, 233, 237, 245, 250, 255, 266, 295, 253, 243, 260, 263, 275, 247, 245, 252, 254, 266, 237, 265, 255, 247, 221, 280, 254, 229, 272, 242, 262, 275, 268, 285, 246, 273, 285, 268, 253, 238, 267, 268, 223, 236, 255, 255, 258, 237, 273, 246, 297, 247, 277, 235, 253, 276, 264, 235, 281, 259, 261, 263, 277, 239, 239, 247, 249, 245, 247, 247, 262, 259, 240, 252, 253, 266, 267, 265, 237, 283, 238, 234, 261, 263, 229, 235, 260, 249, 245, 212, 272] payload_rows_per_dst=[5390, 5502, 5462, 5370, 5520, 5448, 5424, 5534] dup_rows_per_dst=[2764, 2666, 2710, 2758, 2788, 2802, 2706, 2692] combine_rows_per_src=[5390, 5502, 5462, 5370, 5520, 5448, 5424, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43662 payload_rows_remote=38186 dup_rows=21874 dup_rows_remote=19064 dedup_ratio=0.333771 router_tokens_per_expert=[228, 266, 262, 257, 253, 259, 286, 246, 286, 250, 278, 272, 268, 266, 220, 253, 267, 270, 286, 247, 280, 247, 233, 240, 232, 272, 244, 260, 289, 289, 261, 260, 287, 238, 232, 254, 257, 238, 254, 260, 271, 258, 255, 256, 233, 254, 251, 272, 254, 243, 258, 266, 271, 258, 267, 272, 265, 246, 241, 244, 263, 265, 269, 261, 200, 256, 273, 276, 256, 264, 246, 254, 272, 264, 262, 249, 248, 256, 220, 253, 254, 229, 265, 269, 263, 248, 280, 293, 261, 260, 258, 254, 242, 247, 267, 272, 247, 282, 240, 231, 234, 261, 250, 254, 232, 264, 253, 253, 259, 240, 254, 252, 280, 241, 237, 249, 250, 258, 230, 284, 241, 241, 237, 242, 241, 247, 253, 280] payload_rows_per_dst=[5480, 5520, 5438, 5476, 5454, 5500, 5360, 5434] dup_rows_per_dst=[2820, 2834, 2702, 2810, 2644, 2824, 2652, 2588] combine_rows_per_src=[5480, 5520, 5438, 5476, 5454, 5500, 5360, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38230 dup_rows=21818 dup_rows_remote=19040 dedup_ratio=0.332916 router_tokens_per_expert=[262, 262, 250, 250, 276, 263, 263, 239, 228, 243, 257, 284, 216, 275, 282, 261, 241, 249, 241, 264, 256, 259, 232, 250, 257, 240, 273, 243, 252, 206, 244, 257, 260, 250, 277, 267, 257, 272, 276, 290, 240, 243, 241, 253, 242, 259, 246, 266, 273, 276, 240, 260, 278, 261, 264, 282, 263, 240, 250, 248, 276, 243, 259, 252, 269, 274, 265, 237, 234, 249, 230, 247, 273, 263, 252, 275, 213, 253, 263, 243, 250, 255, 264, 251, 251, 259, 261, 244, 288, 249, 248, 272, 250, 243, 242, 256, 257, 234, 248, 266, 243, 276, 261, 274, 269, 268, 256, 252, 267, 262, 239, 265, 267, 290, 259, 265, 234, 226, 283, 246, 255, 252, 247, 240, 255, 276, 260, 274] payload_rows_per_dst=[5488, 5512, 5532, 5524, 5470, 5258, 5530, 5404] dup_rows_per_dst=[2778, 2904, 2702, 2806, 2782, 2420, 2744, 2682] combine_rows_per_src=[5488, 5512, 5532, 5524, 5470, 5258, 5530, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=227872 combine_payload_bytes=155590656 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155818528 ingress_bytes=174465024 link_bytes=174465024 token_bytes=311409184 weight_movement_bytes=18874368 total_nvlink_bytes=330283552 weight_to_token_ratio=0.060610 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671638018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229208 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157392728 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314556248 weight_movement_bytes=37748736 total_nvlink_bytes=352304984 weight_to_token_ratio=0.120006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43946 payload_rows_remote=38458 dup_rows=21590 dup_rows_remote=18896 dedup_ratio=0.329437 router_tokens_per_expert=[270, 268, 272, 262, 276, 239, 260, 244, 260, 248, 274, 259, 266, 256, 240, 248, 240, 237, 259, 265, 246, 251, 230, 267, 254, 244, 282, 240, 260, 241, 251, 261, 249, 271, 279, 255, 274, 286, 273, 278, 247, 232, 268, 237, 280, 256, 270, 275, 259, 251, 257, 267, 247, 250, 244, 234, 237, 220, 286, 232, 250, 265, 251, 279, 261, 266, 258, 269, 269, 236, 226, 255, 281, 276, 252, 259, 268, 232, 216, 267, 238, 254, 280, 269, 274, 242, 274, 270, 251, 257, 248, 258, 288, 280, 274, 237, 264, 234, 263, 232, 261, 262, 257, 221, 239, 252, 260, 270, 282, 248, 233, 237, 241, 269, 253, 238, 269, 255, 238, 239, 245, 252, 279, 256, 280, 235, 220, 270] payload_rows_per_dst=[5538, 5396, 5592, 5492, 5488, 5578, 5438, 5424] dup_rows_per_dst=[2746, 2660, 2868, 2566, 2694, 2810, 2592, 2654] combine_rows_per_src=[5538, 5396, 5592, 5492, 5488, 5578, 5438, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38352 dup_rows=21784 dup_rows_remote=19132 dedup_ratio=0.332397 router_tokens_per_expert=[259, 244, 267, 250, 278, 249, 273, 238, 274, 266, 257, 264, 277, 252, 266, 279, 260, 295, 253, 250, 261, 239, 243, 245, 252, 226, 234, 273, 244, 234, 268, 249, 261, 265, 260, 259, 238, 244, 246, 272, 274, 231, 254, 265, 256, 266, 260, 251, 253, 258, 247, 266, 289, 271, 256, 267, 259, 233, 260, 253, 258, 262, 263, 236, 262, 281, 236, 267, 247, 268, 238, 244, 256, 237, 254, 262, 242, 269, 258, 245, 263, 267, 287, 247, 261, 245, 258, 262, 241, 255, 246, 273, 259, 241, 258, 255, 281, 228, 245, 249, 234, 247, 274, 288, 252, 250, 258, 256, 251, 245, 249, 226, 257, 251, 268, 264, 257, 267, 241, 241, 249, 267, 232, 286, 269, 269, 246, 235] payload_rows_per_dst=[5516, 5400, 5440, 5528, 5444, 5506, 5408, 5510] dup_rows_per_dst=[2870, 2652, 2764, 2734, 2688, 2730, 2658, 2688] combine_rows_per_src=[5516, 5400, 5440, 5528, 5444, 5506, 5408, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229624 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803320 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313377016 weight_movement_bytes=18874368 total_nvlink_bytes=332251384 weight_to_token_ratio=0.060229 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229000 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156638856 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048712 weight_movement_bytes=18874368 total_nvlink_bytes=331923080 weight_to_token_ratio=0.060292 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38046 dup_rows=21982 dup_rows_remote=19180 dedup_ratio=0.335419 router_tokens_per_expert=[277, 238, 270, 240, 250, 248, 229, 237, 258, 254, 276, 248, 274, 235, 244, 254, 271, 249, 249, 256, 235, 269, 279, 289, 270, 276, 243, 274, 261, 231, 261, 241, 286, 252, 261, 256, 259, 270, 274, 251, 262, 266, 254, 237, 249, 252, 256, 270, 259, 265, 274, 262, 279, 249, 255, 252, 258, 235, 236, 275, 221, 261, 245, 248, 272, 252, 273, 280, 263, 253, 267, 263, 243, 251, 269, 265, 269, 253, 272, 252, 285, 246, 231, 265, 256, 251, 234, 227, 239, 270, 252, 238, 256, 260, 277, 256, 234, 257, 258, 256, 252, 274, 251, 243, 284, 236, 224, 278, 242, 244, 263, 263, 249, 274, 246, 255, 236, 248, 248, 268, 249, 273, 247, 251, 266, 242, 253, 249] payload_rows_per_dst=[5358, 5516, 5508, 5336, 5548, 5402, 5422, 5464] dup_rows_per_dst=[2706, 2792, 2802, 2812, 2846, 2684, 2696, 2644] combine_rows_per_src=[5358, 5516, 5508, 5336, 5548, 5402, 5422, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229080 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156819160 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313409240 weight_movement_bytes=37748736 total_nvlink_bytes=351157976 weight_to_token_ratio=0.120446 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229936 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157319728 ingress_bytes=175964160 link_bytes=175964160 token_bytes=314409520 weight_movement_bytes=18874368 total_nvlink_bytes=333283888 weight_to_token_ratio=0.060031 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686002001e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157523968 dispatch_meta_bytes=229416 combine_payload_bytes=157523968 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157753384 ingress_bytes=195272704 link_bytes=195272704 token_bytes=315277352 weight_movement_bytes=37748736 total_nvlink_bytes=353026088 weight_to_token_ratio=0.119732 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871006970e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=228904 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156065320 ingress_bytes=174710784 link_bytes=174710784 token_bytes=311901736 weight_movement_bytes=18874368 total_nvlink_bytes=330776104 weight_to_token_ratio=0.060514 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673992769e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38264 dup_rows=21808 dup_rows_remote=19088 dedup_ratio=0.332764 router_tokens_per_expert=[275, 246, 258, 251, 262, 255, 270, 255, 238, 246, 253, 243, 283, 271, 261, 239, 273, 261, 255, 263, 242, 243, 250, 252, 245, 291, 266, 264, 243, 229, 255, 236, 265, 231, 239, 258, 252, 244, 266, 262, 266, 263, 254, 276, 263, 251, 244, 248, 276, 279, 271, 257, 241, 234, 245, 253, 251, 218, 265, 242, 260, 243, 272, 243, 236, 255, 257, 273, 234, 269, 247, 266, 240, 278, 240, 294, 289, 269, 268, 265, 263, 239, 238, 240, 262, 274, 254, 275, 213, 258, 258, 256, 300, 236, 278, 251, 248, 251, 262, 255, 228, 258, 244, 257, 264, 249, 263, 269, 276, 260, 250, 264, 252, 264, 252, 235, 238, 246, 275, 265, 260, 247, 252, 256, 261, 253, 276, 257] payload_rows_per_dst=[5486, 5496, 5454, 5410, 5338, 5506, 5574, 5464] dup_rows_per_dst=[2726, 2640, 2710, 2690, 2570, 2894, 2858, 2720] combine_rows_per_src=[5486, 5496, 5454, 5410, 5338, 5506, 5574, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38266 dup_rows=21812 dup_rows_remote=19230 dedup_ratio=0.332825 router_tokens_per_expert=[261, 229, 264, 252, 244, 250, 256, 280, 245, 273, 258, 251, 263, 223, 268, 260, 277, 249, 259, 244, 248, 241, 237, 272, 258, 272, 240, 278, 256, 244, 254, 254, 232, 253, 282, 258, 246, 243, 265, 247, 265, 239, 253, 230, 232, 259, 266, 262, 260, 240, 284, 233, 265, 248, 243, 255, 248, 259, 240, 255, 262, 244, 268, 260, 300, 246, 258, 257, 268, 260, 278, 251, 266, 298, 254, 257, 266, 274, 249, 250, 251, 241, 266, 259, 244, 255, 237, 264, 233, 253, 242, 259, 248, 263, 265, 240, 247, 231, 231, 274, 267, 241, 270, 243, 255, 241, 241, 257, 273, 238, 261, 294, 280, 252, 255, 257, 295, 253, 255, 270, 267, 276, 279, 238, 259, 276, 259, 225] payload_rows_per_dst=[5466, 5450, 5398, 5396, 5560, 5458, 5476, 5520] dup_rows_per_dst=[2688, 2716, 2666, 2732, 2904, 2582, 2652, 2872] combine_rows_per_src=[5466, 5450, 5398, 5396, 5560, 5458, 5476, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38508 dup_rows=21710 dup_rows_remote=19070 dedup_ratio=0.331268 router_tokens_per_expert=[244, 231, 244, 228, 243, 254, 261, 253, 244, 259, 245, 248, 242, 267, 269, 268, 222, 257, 276, 260, 270, 271, 285, 238, 273, 263, 257, 266, 278, 243, 240, 242, 258, 257, 246, 270, 266, 272, 279, 248, 273, 240, 273, 240, 252, 301, 278, 267, 276, 257, 248, 256, 257, 245, 258, 266, 261, 272, 266, 234, 265, 281, 249, 253, 278, 262, 273, 248, 249, 270, 260, 242, 249, 242, 263, 250, 263, 225, 239, 265, 272, 248, 258, 233, 255, 275, 238, 265, 233, 241, 264, 286, 276, 265, 258, 270, 277, 213, 233, 229, 251, 277, 270, 250, 244, 239, 264, 255, 240, 235, 261, 241, 279, 264, 252, 252, 254, 244, 261, 240, 248, 258, 287, 233, 252, 249, 247, 249] payload_rows_per_dst=[5412, 5548, 5592, 5496, 5466, 5512, 5318, 5482] dup_rows_per_dst=[2588, 2734, 2848, 2792, 2690, 2762, 2640, 2656] combine_rows_per_src=[5412, 5548, 5592, 5496, 5466, 5512, 5318, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38380 dup_rows=21760 dup_rows_remote=19152 dedup_ratio=0.332031 router_tokens_per_expert=[266, 243, 257, 263, 261, 247, 254, 273, 271, 249, 251, 269, 257, 209, 232, 257, 234, 277, 277, 273, 238, 233, 247, 235, 273, 269, 259, 277, 263, 274, 229, 254, 253, 271, 245, 249, 254, 236, 247, 280, 234, 250, 274, 239, 243, 247, 256, 227, 243, 242, 238, 251, 241, 283, 242, 249, 249, 246, 218, 249, 263, 276, 250, 262, 292, 277, 234, 264, 272, 264, 281, 258, 271, 255, 264, 280, 227, 247, 263, 262, 271, 276, 274, 291, 273, 257, 270, 239, 241, 251, 249, 278, 272, 262, 250, 252, 263, 248, 236, 289, 277, 268, 250, 257, 240, 268, 291, 246, 253, 275, 259, 239, 264, 248, 239, 240, 241, 253, 266, 255, 246, 229, 210, 267, 256, 263, 273, 264] payload_rows_per_dst=[5512, 5504, 5418, 5396, 5512, 5554, 5522, 5358] dup_rows_per_dst=[2606, 2720, 2592, 2608, 2910, 2858, 2796, 2670] combine_rows_per_src=[5512, 5504, 5418, 5396, 5512, 5554, 5522, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38312 dup_rows=21640 dup_rows_remote=18730 dedup_ratio=0.330200 router_tokens_per_expert=[245, 276, 247, 242, 301, 265, 275, 282, 254, 246, 254, 251, 257, 260, 267, 279, 259, 266, 234, 239, 265, 259, 254, 245, 242, 241, 244, 254, 257, 253, 260, 253, 252, 244, 250, 249, 224, 244, 243, 271, 221, 274, 232, 282, 262, 231, 264, 275, 237, 253, 252, 270, 251, 266, 249, 232, 237, 264, 260, 257, 255, 224, 258, 255, 255, 259, 269, 255, 269, 275, 238, 229, 252, 253, 230, 242, 257, 255, 240, 253, 253, 258, 260, 267, 234, 252, 247, 250, 283, 272, 266, 254, 271, 268, 263, 284, 284, 236, 246, 272, 239, 250, 256, 243, 257, 269, 253, 253, 268, 265, 271, 271, 277, 258, 241, 286, 233, 264, 259, 276, 255, 251, 277, 251, 268, 248, 264, 250] payload_rows_per_dst=[5584, 5576, 5436, 5660, 5310, 5554, 5384, 5392] dup_rows_per_dst=[2910, 2950, 2600, 2816, 2368, 2810, 2698, 2488] combine_rows_per_src=[5584, 5576, 5436, 5660, 5310, 5554, 5384, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38376 dup_rows=21704 dup_rows_remote=18994 dedup_ratio=0.331177 router_tokens_per_expert=[256, 233, 277, 252, 303, 233, 266, 259, 258, 248, 285, 256, 270, 252, 240, 275, 238, 266, 235, 245, 288, 268, 249, 235, 247, 290, 249, 247, 238, 272, 260, 256, 264, 282, 260, 276, 236, 237, 242, 255, 261, 270, 267, 250, 245, 244, 229, 247, 253, 243, 246, 275, 238, 277, 257, 234, 255, 239, 230, 253, 227, 271, 287, 246, 232, 247, 248, 238, 264, 283, 255, 265, 283, 266, 254, 271, 286, 261, 242, 254, 262, 250, 240, 265, 265, 258, 271, 267, 240, 256, 253, 231, 249, 254, 274, 247, 249, 271, 270, 263, 269, 248, 273, 246, 268, 237, 266, 247, 261, 243, 265, 238, 243, 258, 278, 279, 261, 217, 243, 231, 269, 261, 259, 219, 236, 264, 279, 284] payload_rows_per_dst=[5566, 5456, 5452, 5446, 5506, 5418, 5490, 5498] dup_rows_per_dst=[2760, 2710, 2678, 2616, 2792, 2746, 2738, 2664] combine_rows_per_src=[5566, 5456, 5452, 5446, 5506, 5418, 5490, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229408 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156958752 ingress_bytes=213352448 link_bytes=213352448 token_bytes=313688096 weight_movement_bytes=56623104 total_nvlink_bytes=370311200 weight_to_token_ratio=0.180508 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044238181e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229984 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156967520 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313705056 weight_movement_bytes=18874368 total_nvlink_bytes=332579424 weight_to_token_ratio=0.060166 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38330 dup_rows=21836 dup_rows_remote=19152 dedup_ratio=0.333191 router_tokens_per_expert=[272, 279, 250, 257, 249, 265, 245, 259, 276, 247, 238, 253, 260, 250, 276, 261, 265, 268, 274, 223, 274, 266, 271, 263, 232, 242, 264, 260, 239, 268, 243, 261, 256, 255, 278, 242, 269, 287, 254, 256, 240, 251, 212, 247, 280, 264, 255, 251, 272, 239, 274, 266, 278, 243, 265, 265, 285, 217, 241, 260, 245, 241, 289, 257, 265, 259, 269, 255, 257, 258, 251, 277, 280, 255, 246, 247, 225, 245, 217, 221, 253, 216, 235, 232, 276, 259, 248, 294, 265, 259, 254, 268, 256, 240, 236, 250, 279, 285, 260, 232, 270, 227, 237, 262, 258, 263, 246, 251, 246, 252, 252, 248, 273, 255, 244, 278, 262, 302, 265, 262, 264, 217, 277, 238, 257, 270, 254, 230] payload_rows_per_dst=[5530, 5508, 5468, 5462, 5370, 5426, 5436, 5500] dup_rows_per_dst=[2744, 2718, 2726, 2812, 2684, 2656, 2700, 2796] combine_rows_per_src=[5530, 5508, 5468, 5462, 5370, 5426, 5436, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38352 dup_rows=21708 dup_rows_remote=18930 dedup_ratio=0.331238 router_tokens_per_expert=[238, 261, 263, 258, 245, 264, 231, 287, 258, 251, 255, 245, 246, 243, 232, 248, 264, 277, 223, 239, 238, 275, 231, 294, 240, 268, 276, 252, 289, 226, 263, 233, 277, 267, 242, 242, 263, 277, 264, 260, 249, 244, 246, 269, 265, 282, 245, 235, 261, 289, 261, 298, 276, 261, 241, 235, 257, 269, 264, 240, 259, 247, 265, 272, 277, 244, 273, 251, 289, 255, 241, 241, 252, 275, 279, 254, 276, 250, 260, 235, 263, 243, 262, 288, 265, 256, 238, 259, 240, 260, 283, 233, 256, 257, 235, 234, 247, 284, 239, 233, 251, 242, 260, 270, 249, 242, 232, 252, 232, 269, 245, 256, 284, 247, 249, 307, 251, 252, 262, 239, 251, 230, 261, 249, 241, 277, 291, 215] payload_rows_per_dst=[5432, 5456, 5476, 5524, 5492, 5542, 5404, 5502] dup_rows_per_dst=[2618, 2720, 2778, 2866, 2812, 2602, 2602, 2710] combine_rows_per_src=[5432, 5456, 5476, 5524, 5492, 5542, 5404, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157728768 dispatch_meta_bytes=230312 combine_payload_bytes=157728768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157959080 ingress_bytes=176603136 link_bytes=176603136 token_bytes=315687848 weight_movement_bytes=18874368 total_nvlink_bytes=334562216 weight_to_token_ratio=0.059788 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692124355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=230128 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434608 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314639088 weight_movement_bytes=18874368 total_nvlink_bytes=333513456 weight_to_token_ratio=0.059987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228168 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157154120 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314080072 weight_movement_bytes=37748736 total_nvlink_bytes=351828808 weight_to_token_ratio=0.120188 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229480 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417576 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314605672 weight_movement_bytes=18874368 total_nvlink_bytes=333480040 weight_to_token_ratio=0.059994 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229128 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157318920 ingress_bytes=194838528 link_bytes=194838528 token_bytes=314408712 weight_movement_bytes=37748736 total_nvlink_bytes=352157448 weight_to_token_ratio=0.120063 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866846909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229928 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=157229608 ingress_bytes=251371520 link_bytes=251371520 token_bytes=314229288 weight_movement_bytes=94371840 total_nvlink_bytes=408601128 weight_to_token_ratio=0.300328 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408518222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38204 dup_rows=21728 dup_rows_remote=18866 dedup_ratio=0.331543 router_tokens_per_expert=[281, 259, 269, 243, 259, 243, 276, 238, 265, 267, 271, 299, 253, 290, 265, 260, 296, 271, 248, 242, 257, 242, 251, 274, 261, 249, 251, 257, 245, 270, 250, 230, 260, 235, 253, 270, 277, 266, 258, 266, 267, 283, 270, 272, 245, 269, 254, 261, 264, 239, 245, 281, 236, 249, 246, 253, 233, 262, 251, 253, 256, 239, 239, 246, 243, 243, 237, 253, 254, 244, 263, 244, 265, 258, 244, 247, 257, 245, 254, 258, 282, 270, 261, 245, 246, 243, 258, 259, 268, 251, 248, 247, 233, 261, 247, 252, 257, 269, 257, 264, 260, 242, 214, 285, 248, 263, 266, 239, 245, 258, 250, 263, 288, 271, 266, 235, 247, 243, 272, 259, 258, 284, 249, 239, 215, 240, 255, 257] payload_rows_per_dst=[5560, 5514, 5382, 5400, 5386, 5506, 5456, 5604] dup_rows_per_dst=[2712, 2674, 2720, 2584, 2632, 2840, 2704, 2862] combine_rows_per_src=[5560, 5514, 5382, 5400, 5386, 5506, 5456, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38270 dup_rows=21704 dup_rows_remote=18950 dedup_ratio=0.331177 router_tokens_per_expert=[227, 260, 262, 262, 244, 293, 234, 259, 247, 254, 229, 284, 237, 231, 269, 252, 270, 284, 233, 268, 242, 281, 257, 250, 244, 251, 229, 248, 287, 285, 245, 252, 228, 260, 250, 263, 252, 256, 268, 264, 259, 256, 246, 258, 249, 268, 248, 254, 255, 254, 253, 253, 285, 256, 262, 265, 285, 282, 249, 255, 254, 249, 255, 240, 291, 253, 252, 275, 231, 253, 241, 245, 265, 248, 275, 245, 279, 241, 237, 242, 249, 240, 280, 289, 251, 247, 249, 271, 241, 259, 254, 259, 243, 282, 299, 245, 269, 258, 239, 262, 240, 235, 286, 269, 257, 242, 244, 235, 243, 269, 235, 253, 252, 263, 289, 248, 244, 267, 250, 246, 216, 241, 262, 281, 255, 243, 271, 272] payload_rows_per_dst=[5378, 5562, 5414, 5554, 5434, 5562, 5444, 5484] dup_rows_per_dst=[2710, 2690, 2744, 2750, 2712, 2754, 2628, 2716] combine_rows_per_src=[5378, 5562, 5414, 5554, 5434, 5562, 5444, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38288 dup_rows=21780 dup_rows_remote=18904 dedup_ratio=0.332336 router_tokens_per_expert=[266, 259, 236, 266, 253, 268, 255, 254, 229, 271, 223, 269, 247, 262, 269, 244, 238, 251, 264, 235, 248, 245, 258, 267, 264, 243, 248, 223, 259, 239, 259, 261, 269, 251, 248, 243, 210, 261, 271, 271, 254, 267, 260, 251, 238, 279, 249, 254, 252, 258, 284, 268, 279, 269, 264, 225, 261, 242, 262, 243, 260, 255, 253, 253, 276, 244, 245, 252, 279, 268, 237, 247, 263, 275, 256, 249, 248, 282, 276, 260, 261, 226, 260, 244, 258, 252, 258, 252, 228, 243, 271, 268, 268, 258, 254, 262, 266, 264, 272, 242, 253, 249, 271, 265, 251, 265, 250, 267, 271, 268, 269, 249, 247, 259, 253, 287, 258, 263, 242, 250, 244, 249, 274, 265, 268, 261, 234, 245] payload_rows_per_dst=[5450, 5450, 5466, 5516, 5488, 5484, 5468, 5434] dup_rows_per_dst=[2692, 2554, 2686, 2740, 2826, 2642, 2876, 2764] combine_rows_per_src=[5450, 5450, 5466, 5516, 5488, 5484, 5468, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38252 dup_rows=21846 dup_rows_remote=19104 dedup_ratio=0.333344 router_tokens_per_expert=[277, 234, 262, 255, 265, 236, 245, 237, 236, 269, 261, 240, 267, 285, 254, 254, 274, 252, 264, 262, 265, 257, 263, 259, 251, 264, 241, 252, 252, 243, 237, 256, 232, 261, 287, 240, 265, 273, 259, 252, 255, 263, 263, 285, 279, 243, 239, 272, 232, 292, 278, 254, 255, 280, 235, 260, 251, 238, 229, 269, 245, 242, 265, 265, 251, 240, 250, 246, 261, 270, 273, 256, 257, 246, 240, 246, 282, 258, 293, 265, 246, 255, 273, 235, 265, 270, 266, 228, 253, 255, 240, 277, 247, 267, 281, 232, 279, 285, 272, 243, 235, 241, 246, 274, 266, 279, 238, 263, 245, 271, 264, 256, 236, 233, 243, 247, 275, 255, 257, 262, 260, 257, 259, 257, 223, 248, 224, 224] payload_rows_per_dst=[5420, 5482, 5538, 5438, 5490, 5466, 5498, 5358] dup_rows_per_dst=[2734, 2702, 2798, 2742, 2778, 2714, 2816, 2562] combine_rows_per_src=[5420, 5482, 5538, 5438, 5490, 5466, 5498, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228280 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156711864 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313195448 weight_movement_bytes=18874368 total_nvlink_bytes=332069816 weight_to_token_ratio=0.060264 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38232 dup_rows=21818 dup_rows_remote=18864 dedup_ratio=0.332916 router_tokens_per_expert=[245, 277, 253, 231, 268, 267, 245, 263, 249, 257, 267, 284, 266, 248, 250, 263, 257, 232, 241, 259, 231, 250, 244, 258, 242, 272, 247, 242, 257, 233, 270, 277, 298, 265, 261, 241, 270, 252, 256, 256, 248, 252, 265, 257, 289, 259, 276, 260, 272, 259, 292, 257, 276, 262, 237, 248, 255, 238, 271, 236, 247, 258, 257, 250, 279, 278, 238, 268, 285, 247, 248, 254, 265, 271, 234, 263, 258, 276, 269, 230, 289, 252, 267, 254, 265, 246, 281, 279, 266, 245, 231, 234, 228, 244, 232, 234, 260, 246, 250, 252, 248, 281, 244, 240, 263, 243, 243, 271, 239, 252, 240, 248, 261, 261, 249, 250, 258, 209, 297, 262, 259, 255, 248, 254, 253, 257, 251, 249] payload_rows_per_dst=[5486, 5534, 5570, 5496, 5424, 5288, 5454, 5466] dup_rows_per_dst=[2954, 2810, 2906, 2744, 2662, 2476, 2586, 2680] combine_rows_per_src=[5486, 5534, 5570, 5496, 5424, 5288, 5454, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38216 dup_rows=21918 dup_rows_remote=19212 dedup_ratio=0.334442 router_tokens_per_expert=[240, 221, 261, 251, 264, 280, 255, 258, 236, 259, 264, 277, 234, 266, 257, 274, 281, 240, 256, 231, 260, 257, 254, 292, 260, 256, 224, 247, 249, 239, 256, 252, 271, 270, 278, 248, 230, 255, 254, 255, 269, 260, 264, 252, 274, 236, 247, 256, 258, 260, 262, 278, 267, 271, 262, 243, 251, 273, 287, 230, 254, 291, 260, 261, 246, 279, 266, 256, 238, 233, 253, 254, 234, 278, 252, 252, 245, 238, 235, 261, 267, 255, 261, 238, 244, 252, 265, 259, 247, 250, 269, 263, 253, 248, 275, 229, 225, 252, 269, 238, 267, 271, 264, 270, 239, 229, 260, 247, 268, 263, 259, 266, 277, 248, 255, 255, 252, 253, 259, 248, 277, 236, 261, 251, 250, 271, 256, 259] payload_rows_per_dst=[5444, 5402, 5472, 5556, 5420, 5410, 5464, 5450] dup_rows_per_dst=[2750, 2706, 2766, 2860, 2620, 2740, 2710, 2766] combine_rows_per_src=[5444, 5402, 5472, 5556, 5420, 5410, 5464, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38260 dup_rows=21716 dup_rows_remote=18962 dedup_ratio=0.331360 router_tokens_per_expert=[244, 247, 248, 257, 268, 249, 246, 243, 242, 279, 237, 284, 244, 233, 231, 267, 245, 267, 243, 263, 239, 263, 240, 253, 250, 237, 257, 238, 251, 255, 268, 288, 240, 252, 279, 270, 259, 247, 260, 275, 270, 253, 283, 253, 257, 221, 260, 278, 262, 298, 236, 268, 277, 247, 280, 281, 276, 296, 256, 242, 237, 265, 252, 235, 241, 245, 251, 243, 255, 268, 242, 260, 270, 248, 261, 252, 237, 277, 260, 266, 252, 252, 250, 259, 234, 246, 249, 280, 246, 264, 219, 258, 226, 255, 245, 252, 251, 262, 260, 251, 263, 266, 250, 270, 251, 260, 284, 260, 246, 243, 238, 240, 261, 273, 258, 258, 249, 257, 246, 276, 265, 257, 254, 284, 250, 271, 243, 267] payload_rows_per_dst=[5404, 5492, 5560, 5602, 5418, 5374, 5436, 5534] dup_rows_per_dst=[2634, 2622, 2754, 2814, 2734, 2600, 2754, 2804] combine_rows_per_src=[5404, 5492, 5560, 5602, 5418, 5374, 5436, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=228880 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156982800 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313736720 weight_movement_bytes=18874368 total_nvlink_bytes=332611088 weight_to_token_ratio=0.060160 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38338 dup_rows=21792 dup_rows_remote=19048 dedup_ratio=0.332520 router_tokens_per_expert=[275, 261, 255, 253, 294, 272, 286, 244, 255, 279, 256, 241, 253, 239, 224, 265, 287, 256, 225, 268, 245, 278, 257, 253, 235, 268, 267, 237, 269, 250, 258, 268, 255, 269, 247, 245, 247, 270, 261, 262, 258, 248, 251, 257, 247, 274, 263, 277, 274, 252, 258, 244, 230, 256, 252, 262, 266, 258, 242, 214, 247, 262, 267, 245, 240, 250, 260, 237, 283, 277, 248, 243, 239, 231, 280, 270, 246, 265, 224, 282, 264, 248, 253, 260, 279, 253, 233, 246, 234, 246, 255, 230, 252, 241, 265, 247, 248, 268, 262, 258, 234, 296, 288, 237, 264, 238, 248, 278, 229, 285, 272, 249, 256, 251, 233, 248, 263, 271, 252, 255, 248, 273, 283, 279, 264, 237, 238, 249] payload_rows_per_dst=[5496, 5494, 5480, 5448, 5406, 5360, 5518, 5542] dup_rows_per_dst=[2808, 2748, 2782, 2610, 2744, 2652, 2790, 2658] combine_rows_per_src=[5496, 5494, 5480, 5448, 5406, 5360, 5518, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=228768 combine_payload_bytes=156827648 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157056416 ingress_bytes=213450752 link_bytes=213450752 token_bytes=313884064 weight_movement_bytes=56623104 total_nvlink_bytes=370507168 weight_to_token_ratio=0.180395 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045180081e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229424 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909616 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589808 weight_movement_bytes=18874368 total_nvlink_bytes=332464176 weight_to_token_ratio=0.060188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=228384 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156826656 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313424928 weight_movement_bytes=37748736 total_nvlink_bytes=351173664 weight_to_token_ratio=0.120439 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228888 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156941848 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313654808 weight_movement_bytes=56623104 total_nvlink_bytes=370277912 weight_to_token_ratio=0.180527 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229712 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156762448 ingress_bytes=175407104 link_bytes=175407104 token_bytes=313295184 weight_movement_bytes=18874368 total_nvlink_bytes=332169552 weight_to_token_ratio=0.060245 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680664565e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=229544 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157261992 ingress_bytes=213655552 link_bytes=213655552 token_bytes=314294440 weight_movement_bytes=56623104 total_nvlink_bytes=370917544 weight_to_token_ratio=0.180159 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047142374e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38194 dup_rows=21858 dup_rows_remote=19090 dedup_ratio=0.333527 router_tokens_per_expert=[248, 262, 264, 276, 261, 242, 253, 255, 221, 232, 276, 240, 219, 258, 265, 254, 256, 255, 242, 257, 222, 271, 250, 241, 252, 237, 251, 259, 254, 248, 247, 249, 260, 257, 258, 254, 261, 276, 259, 236, 262, 252, 281, 237, 272, 254, 265, 225, 276, 267, 258, 280, 265, 250, 270, 272, 267, 253, 286, 253, 255, 245, 246, 285, 272, 240, 210, 241, 264, 280, 235, 270, 254, 235, 245, 263, 252, 259, 259, 246, 261, 250, 279, 263, 250, 261, 261, 232, 244, 256, 263, 260, 251, 273, 269, 228, 270, 272, 252, 254, 282, 252, 274, 293, 233, 271, 266, 244, 271, 235, 243, 254, 280, 259, 286, 279, 235, 264, 283, 240, 257, 219, 274, 258, 238, 249, 263, 238] payload_rows_per_dst=[5362, 5374, 5330, 5412, 5518, 5656, 5542, 5484] dup_rows_per_dst=[2690, 2608, 2568, 2622, 2748, 2960, 2894, 2768] combine_rows_per_src=[5362, 5374, 5330, 5412, 5518, 5656, 5542, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38286 dup_rows=21804 dup_rows_remote=19026 dedup_ratio=0.332703 router_tokens_per_expert=[280, 258, 230, 247, 241, 256, 261, 258, 269, 212, 252, 261, 244, 259, 283, 266, 266, 284, 267, 265, 250, 268, 245, 258, 252, 232, 257, 253, 231, 254, 263, 254, 258, 264, 252, 269, 249, 264, 249, 240, 264, 255, 249, 278, 258, 248, 278, 248, 256, 269, 241, 258, 254, 256, 272, 256, 247, 248, 267, 261, 270, 244, 251, 257, 230, 238, 266, 286, 275, 272, 250, 237, 261, 256, 233, 250, 246, 271, 271, 277, 264, 264, 283, 204, 248, 234, 253, 264, 292, 271, 242, 255, 272, 273, 239, 254, 275, 235, 254, 287, 278, 243, 256, 246, 243, 246, 254, 243, 261, 258, 269, 249, 282, 248, 235, 237, 267, 250, 244, 275, 241, 238, 235, 244, 257, 259, 278, 244] payload_rows_per_dst=[5486, 5472, 5474, 5472, 5492, 5446, 5500, 5390] dup_rows_per_dst=[2668, 2726, 2772, 2742, 2746, 2778, 2694, 2678] combine_rows_per_src=[5486, 5472, 5474, 5472, 5492, 5446, 5500, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38274 dup_rows=21802 dup_rows_remote=18934 dedup_ratio=0.332672 router_tokens_per_expert=[256, 271, 268, 279, 244, 255, 270, 255, 251, 248, 262, 268, 267, 256, 272, 252, 263, 258, 237, 242, 281, 257, 250, 296, 253, 260, 247, 244, 248, 256, 260, 264, 255, 257, 241, 263, 255, 255, 271, 276, 256, 230, 258, 226, 273, 262, 261, 223, 257, 272, 244, 238, 238, 252, 226, 240, 239, 227, 255, 227, 224, 285, 249, 291, 233, 277, 236, 260, 272, 233, 285, 252, 236, 248, 249, 250, 243, 265, 239, 263, 248, 229, 252, 255, 265, 259, 262, 245, 264, 260, 252, 288, 278, 258, 250, 268, 263, 240, 247, 265, 256, 264, 261, 249, 268, 248, 240, 286, 283, 283, 255, 256, 270, 286, 239, 264, 253, 282, 255, 239, 250, 270, 258, 243, 248, 253, 248, 256] payload_rows_per_dst=[5544, 5470, 5486, 5380, 5450, 5464, 5460, 5480] dup_rows_per_dst=[2804, 2762, 2638, 2548, 2632, 2802, 2868, 2748] combine_rows_per_src=[5544, 5470, 5486, 5380, 5450, 5464, 5460, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38380 dup_rows=21768 dup_rows_remote=19172 dedup_ratio=0.332153 router_tokens_per_expert=[257, 254, 242, 248, 275, 261, 278, 248, 240, 257, 255, 242, 255, 295, 260, 271, 268, 267, 260, 298, 253, 246, 249, 258, 245, 259, 266, 242, 290, 269, 274, 259, 254, 276, 259, 278, 243, 266, 250, 256, 248, 255, 268, 264, 250, 245, 265, 230, 234, 247, 269, 261, 244, 244, 251, 221, 267, 245, 238, 266, 251, 233, 274, 247, 235, 273, 263, 244, 260, 250, 258, 267, 238, 282, 257, 230, 264, 266, 286, 249, 229, 266, 262, 261, 225, 273, 272, 273, 236, 259, 303, 236, 232, 263, 237, 246, 246, 253, 270, 250, 270, 230, 272, 243, 264, 230, 272, 263, 263, 216, 253, 259, 244, 260, 273, 245, 268, 246, 265, 268, 275, 253, 254, 237, 228, 274, 260, 229] payload_rows_per_dst=[5530, 5530, 5494, 5388, 5536, 5384, 5476, 5430] dup_rows_per_dst=[2746, 2876, 2720, 2596, 2708, 2762, 2632, 2728] combine_rows_per_src=[5530, 5530, 5494, 5388, 5536, 5384, 5476, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229136 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156671760 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114384 weight_movement_bytes=37748736 total_nvlink_bytes=350863120 weight_to_token_ratio=0.120559 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38254 dup_rows=21794 dup_rows_remote=19058 dedup_ratio=0.332550 router_tokens_per_expert=[265, 277, 291, 250, 273, 252, 252, 287, 249, 254, 263, 242, 280, 269, 221, 253, 245, 243, 220, 246, 256, 258, 250, 273, 253, 274, 245, 272, 231, 260, 267, 263, 253, 247, 256, 241, 259, 270, 263, 238, 260, 274, 248, 240, 239, 250, 284, 290, 267, 267, 240, 259, 272, 258, 262, 265, 272, 268, 247, 259, 266, 251, 260, 240, 239, 256, 249, 253, 290, 267, 285, 262, 249, 260, 248, 237, 253, 257, 225, 216, 279, 235, 257, 251, 232, 281, 253, 233, 239, 265, 276, 250, 232, 269, 273, 215, 254, 240, 247, 221, 263, 265, 254, 241, 243, 263, 247, 275, 259, 269, 253, 263, 258, 243, 242, 247, 228, 271, 288, 252, 249, 283, 273, 236, 275, 260, 248, 273] payload_rows_per_dst=[5538, 5426, 5488, 5546, 5406, 5388, 5430, 5520] dup_rows_per_dst=[2818, 2686, 2736, 2760, 2686, 2692, 2684, 2732] combine_rows_per_src=[5538, 5426, 5488, 5546, 5406, 5388, 5430, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38282 dup_rows=21730 dup_rows_remote=18926 dedup_ratio=0.331573 router_tokens_per_expert=[277, 268, 253, 246, 233, 256, 253, 270, 254, 247, 256, 249, 235, 273, 252, 244, 260, 258, 236, 226, 268, 244, 244, 284, 257, 219, 268, 249, 265, 288, 235, 263, 259, 277, 259, 249, 244, 256, 244, 215, 276, 236, 253, 243, 250, 243, 263, 299, 259, 234, 255, 252, 242, 218, 253, 271, 253, 267, 269, 272, 281, 264, 263, 236, 234, 278, 252, 255, 266, 245, 280, 274, 248, 233, 244, 275, 274, 278, 264, 278, 247, 240, 259, 266, 246, 254, 251, 276, 258, 236, 247, 239, 211, 248, 265, 260, 260, 262, 259, 240, 274, 241, 257, 273, 285, 241, 278, 251, 274, 256, 280, 259, 241, 243, 266, 254, 255, 254, 247, 240, 274, 260, 309, 261, 264, 262, 244, 238] payload_rows_per_dst=[5524, 5416, 5474, 5494, 5404, 5428, 5610, 5456] dup_rows_per_dst=[2804, 2726, 2676, 2838, 2602, 2578, 2770, 2736] combine_rows_per_src=[5524, 5416, 5474, 5494, 5404, 5428, 5610, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229248 combine_payload_bytes=156819456 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157048704 ingress_bytes=175693824 link_bytes=175693824 token_bytes=313868160 weight_movement_bytes=18874368 total_nvlink_bytes=332742528 weight_to_token_ratio=0.060135 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683411775e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38256 dup_rows=21744 dup_rows_remote=19006 dedup_ratio=0.331787 router_tokens_per_expert=[276, 269, 230, 294, 262, 247, 271, 254, 245, 240, 229, 252, 260, 248, 256, 244, 257, 257, 255, 265, 282, 269, 254, 244, 251, 284, 250, 247, 254, 260, 246, 262, 260, 230, 255, 242, 259, 270, 255, 260, 258, 273, 253, 277, 236, 223, 234, 233, 240, 259, 279, 244, 245, 273, 257, 262, 251, 241, 266, 252, 250, 275, 260, 222, 252, 264, 259, 258, 245, 241, 245, 256, 274, 226, 241, 264, 290, 248, 259, 263, 258, 277, 257, 265, 241, 241, 260, 272, 247, 241, 264, 269, 240, 242, 258, 264, 252, 244, 256, 263, 278, 242, 281, 265, 268, 228, 237, 250, 239, 250, 253, 241, 255, 271, 279, 286, 294, 271, 267, 271, 253, 247, 270, 240, 251, 280, 243, 254] payload_rows_per_dst=[5432, 5536, 5396, 5474, 5442, 5546, 5454, 5512] dup_rows_per_dst=[2722, 2738, 2640, 2678, 2728, 2646, 2640, 2952] combine_rows_per_src=[5432, 5536, 5396, 5474, 5442, 5546, 5454, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38116 dup_rows=21966 dup_rows_remote=19182 dedup_ratio=0.335175 router_tokens_per_expert=[271, 270, 268, 243, 254, 219, 236, 234, 259, 254, 256, 284, 276, 260, 231, 262, 248, 246, 262, 266, 258, 249, 278, 253, 253, 266, 227, 247, 279, 256, 247, 247, 230, 261, 255, 273, 250, 249, 270, 274, 264, 256, 255, 256, 262, 243, 272, 263, 249, 264, 246, 252, 248, 262, 284, 259, 262, 261, 231, 247, 246, 270, 292, 271, 231, 264, 263, 226, 242, 276, 277, 253, 252, 264, 220, 271, 263, 269, 275, 273, 263, 252, 248, 239, 236, 229, 248, 247, 246, 258, 243, 261, 227, 262, 277, 274, 264, 276, 255, 234, 236, 266, 262, 273, 265, 254, 294, 240, 239, 243, 258, 280, 254, 223, 242, 223, 245, 279, 268, 265, 275, 268, 260, 247, 265, 250, 239, 261] payload_rows_per_dst=[5388, 5430, 5438, 5500, 5454, 5402, 5560, 5398] dup_rows_per_dst=[2766, 2734, 2828, 2788, 2784, 2618, 2718, 2730] combine_rows_per_src=[5388, 5430, 5438, 5500, 5454, 5402, 5560, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=228832 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156999136 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313769440 weight_movement_bytes=37748736 total_nvlink_bytes=351518176 weight_to_token_ratio=0.120307 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=230208 combine_payload_bytes=157204480 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157434688 ingress_bytes=213827584 link_bytes=213827584 token_bytes=314639168 weight_movement_bytes=56623104 total_nvlink_bytes=371262272 weight_to_token_ratio=0.179962 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048790700e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229248 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917632 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606016 weight_movement_bytes=18874368 total_nvlink_bytes=332480384 weight_to_token_ratio=0.060185 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=228832 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157031904 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313834976 weight_movement_bytes=18874368 total_nvlink_bytes=332709344 weight_to_token_ratio=0.060141 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229048 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156925624 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622200 weight_movement_bytes=18874368 total_nvlink_bytes=332496568 weight_to_token_ratio=0.060182 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229192 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352328 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475464 weight_movement_bytes=18874368 total_nvlink_bytes=331349832 weight_to_token_ratio=0.060403 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38372 dup_rows=21772 dup_rows_remote=19184 dedup_ratio=0.332214 router_tokens_per_expert=[263, 255, 260, 257, 255, 237, 233, 224, 279, 277, 292, 274, 283, 255, 255, 261, 246, 276, 256, 273, 242, 245, 254, 279, 244, 224, 280, 247, 269, 253, 261, 273, 263, 255, 236, 231, 242, 283, 279, 261, 255, 245, 275, 258, 241, 262, 234, 252, 248, 248, 265, 272, 246, 259, 259, 260, 269, 283, 254, 248, 256, 251, 257, 232, 270, 249, 241, 232, 270, 252, 241, 248, 245, 260, 258, 255, 259, 230, 263, 285, 301, 248, 268, 242, 238, 240, 268, 248, 237, 253, 249, 248, 275, 279, 242, 274, 251, 238, 257, 285, 248, 268, 277, 236, 260, 246, 257, 287, 272, 236, 271, 263, 267, 256, 237, 255, 275, 262, 269, 252, 227, 268, 269, 218, 224, 236, 232, 240] payload_rows_per_dst=[5550, 5424, 5490, 5498, 5420, 5320, 5670, 5392] dup_rows_per_dst=[2734, 2814, 2654, 2716, 2732, 2474, 3060, 2588] combine_rows_per_src=[5550, 5424, 5490, 5498, 5420, 5320, 5670, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38314 dup_rows=21720 dup_rows_remote=18982 dedup_ratio=0.331421 router_tokens_per_expert=[267, 255, 262, 256, 255, 238, 281, 241, 262, 226, 249, 260, 255, 263, 252, 250, 255, 254, 240, 258, 269, 239, 238, 265, 252, 247, 275, 294, 230, 263, 241, 226, 236, 254, 272, 257, 248, 277, 263, 245, 250, 223, 257, 253, 256, 258, 254, 280, 266, 252, 242, 270, 259, 286, 258, 255, 281, 266, 253, 250, 257, 271, 261, 249, 227, 293, 275, 276, 242, 255, 253, 255, 267, 251, 252, 274, 267, 242, 258, 255, 229, 257, 304, 250, 253, 241, 240, 255, 259, 263, 254, 262, 273, 293, 241, 246, 254, 251, 262, 236, 230, 271, 260, 261, 239, 258, 242, 249, 239, 230, 251, 234, 260, 247, 243, 259, 274, 261, 287, 256, 250, 286, 265, 248, 268, 252, 251, 255] payload_rows_per_dst=[5426, 5422, 5484, 5520, 5532, 5502, 5384, 5546] dup_rows_per_dst=[2718, 2670, 2682, 2832, 2752, 2738, 2550, 2778] combine_rows_per_src=[5426, 5422, 5484, 5520, 5532, 5502, 5384, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=230224 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157401936 ingress_bytes=213794816 link_bytes=213794816 token_bytes=314573648 weight_movement_bytes=56623104 total_nvlink_bytes=371196752 weight_to_token_ratio=0.180000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048476733e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43846 payload_rows_remote=38398 dup_rows=21690 dup_rows_remote=19096 dedup_ratio=0.330963 router_tokens_per_expert=[294, 259, 262, 255, 230, 238, 260, 252, 265, 281, 263, 226, 247, 258, 264, 240, 234, 266, 266, 243, 252, 248, 247, 256, 272, 263, 264, 219, 270, 283, 263, 244, 254, 246, 267, 260, 261, 248, 241, 261, 264, 271, 247, 264, 242, 254, 252, 253, 273, 249, 265, 270, 263, 271, 261, 286, 254, 259, 257, 273, 254, 239, 283, 256, 232, 244, 292, 223, 273, 239, 230, 268, 264, 237, 276, 278, 263, 258, 260, 256, 280, 255, 256, 259, 229, 257, 238, 285, 257, 228, 230, 252, 268, 245, 271, 277, 255, 284, 284, 252, 218, 275, 210, 259, 243, 251, 248, 248, 281, 246, 234, 233, 257, 261, 235, 278, 258, 255, 249, 263, 225, 269, 251, 272, 238, 241, 278, 255] payload_rows_per_dst=[5508, 5432, 5498, 5540, 5484, 5464, 5448, 5472] dup_rows_per_dst=[2680, 2748, 2672, 2886, 2702, 2710, 2594, 2698] combine_rows_per_src=[5508, 5432, 5498, 5540, 5484, 5464, 5448, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38240 dup_rows=21866 dup_rows_remote=19198 dedup_ratio=0.333649 router_tokens_per_expert=[265, 228, 243, 276, 237, 254, 242, 269, 253, 248, 251, 238, 266, 224, 253, 228, 232, 236, 232, 272, 259, 287, 239, 266, 261, 250, 277, 259, 247, 238, 252, 235, 255, 248, 258, 256, 236, 258, 271, 281, 252, 272, 304, 239, 239, 263, 275, 234, 252, 256, 247, 266, 233, 273, 262, 246, 255, 260, 271, 235, 244, 234, 249, 266, 230, 265, 258, 271, 252, 249, 247, 236, 276, 261, 270, 261, 263, 279, 270, 258, 255, 268, 271, 254, 286, 271, 266, 258, 237, 274, 260, 267, 256, 233, 261, 241, 255, 290, 277, 250, 268, 243, 272, 274, 253, 244, 267, 261, 247, 236, 264, 258, 244, 275, 226, 235, 295, 244, 254, 247, 268, 268, 259, 256, 258, 255, 267, 247] payload_rows_per_dst=[5358, 5448, 5492, 5430, 5472, 5550, 5500, 5420] dup_rows_per_dst=[2592, 2636, 2790, 2668, 2820, 2766, 2818, 2776] combine_rows_per_src=[5358, 5448, 5492, 5430, 5472, 5550, 5500, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229184 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157163328 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097472 weight_movement_bytes=37748736 total_nvlink_bytes=351846208 weight_to_token_ratio=0.120182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38246 dup_rows=21800 dup_rows_remote=19096 dedup_ratio=0.332642 router_tokens_per_expert=[273, 254, 255, 259, 231, 226, 275, 230, 246, 227, 259, 280, 270, 269, 238, 267, 251, 241, 251, 257, 235, 234, 266, 263, 237, 273, 267, 233, 281, 257, 259, 242, 229, 217, 265, 271, 271, 281, 254, 272, 271, 257, 237, 255, 253, 244, 254, 266, 237, 262, 265, 242, 269, 277, 286, 251, 250, 257, 243, 284, 255, 227, 265, 253, 267, 275, 262, 268, 238, 251, 266, 261, 236, 264, 257, 227, 226, 242, 259, 264, 246, 277, 235, 232, 257, 255, 280, 256, 259, 242, 251, 284, 258, 262, 266, 228, 282, 258, 267, 268, 243, 262, 265, 272, 277, 269, 264, 247, 257, 228, 268, 262, 242, 263, 270, 267, 261, 239, 259, 285, 261, 241, 249, 252, 259, 252, 261, 241] payload_rows_per_dst=[5428, 5464, 5490, 5472, 5476, 5444, 5492, 5470] dup_rows_per_dst=[2690, 2630, 2704, 2774, 2650, 2732, 2886, 2734] combine_rows_per_src=[5428, 5464, 5490, 5472, 5476, 5444, 5492, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38468 dup_rows=21674 dup_rows_remote=19018 dedup_ratio=0.330719 router_tokens_per_expert=[276, 211, 266, 265, 273, 257, 231, 248, 249, 242, 260, 272, 255, 265, 245, 244, 260, 246, 213, 263, 253, 249, 253, 252, 266, 242, 261, 256, 257, 255, 260, 239, 248, 261, 251, 274, 257, 256, 217, 257, 242, 265, 247, 241, 261, 277, 258, 260, 251, 260, 272, 251, 281, 252, 273, 230, 248, 252, 264, 271, 250, 252, 240, 265, 267, 241, 270, 253, 257, 241, 239, 252, 256, 246, 257, 281, 238, 282, 265, 268, 293, 281, 233, 227, 272, 262, 272, 269, 282, 236, 239, 260, 229, 226, 250, 230, 290, 257, 229, 265, 252, 269, 249, 255, 259, 279, 270, 284, 248, 259, 270, 246, 255, 292, 284, 268, 274, 241, 245, 250, 255, 248, 288, 238, 244, 228, 259, 276] payload_rows_per_dst=[5434, 5394, 5434, 5524, 5522, 5418, 5570, 5566] dup_rows_per_dst=[2684, 2656, 2710, 2700, 2704, 2704, 2792, 2724] combine_rows_per_src=[5434, 5394, 5434, 5524, 5522, 5418, 5570, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229976 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157508184 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314786392 weight_movement_bytes=18874368 total_nvlink_bytes=333660760 weight_to_token_ratio=0.059959 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38278 dup_rows=21904 dup_rows_remote=19162 dedup_ratio=0.334229 router_tokens_per_expert=[266, 255, 237, 257, 288, 275, 256, 245, 228, 278, 257, 288, 252, 250, 246, 259, 260, 280, 251, 262, 253, 238, 264, 252, 255, 263, 254, 271, 278, 249, 270, 236, 229, 236, 281, 267, 234, 237, 246, 269, 262, 234, 275, 256, 249, 227, 267, 262, 283, 278, 256, 248, 292, 271, 244, 249, 261, 260, 244, 231, 248, 229, 264, 240, 240, 273, 272, 260, 243, 241, 278, 260, 255, 251, 275, 255, 233, 246, 210, 256, 266, 259, 245, 244, 238, 246, 294, 252, 279, 267, 274, 244, 251, 263, 289, 251, 255, 252, 257, 257, 285, 249, 257, 239, 267, 234, 256, 249, 244, 252, 209, 259, 286, 265, 243, 240, 266, 237, 258, 256, 259, 262, 265, 259, 246, 264, 257, 272] payload_rows_per_dst=[5430, 5566, 5402, 5426, 5354, 5578, 5340, 5536] dup_rows_per_dst=[2844, 2706, 2660, 2770, 2742, 2746, 2702, 2734] combine_rows_per_src=[5430, 5566, 5402, 5426, 5354, 5578, 5340, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44078 payload_rows_remote=38648 dup_rows=21458 dup_rows_remote=18688 dedup_ratio=0.327423 router_tokens_per_expert=[252, 249, 238, 272, 254, 252, 268, 228, 287, 251, 268, 248, 273, 246, 228, 275, 252, 254, 254, 272, 262, 262, 257, 256, 251, 266, 245, 257, 261, 269, 253, 252, 258, 261, 246, 267, 234, 250, 248, 264, 251, 245, 249, 253, 251, 249, 247, 261, 267, 247, 244, 252, 247, 281, 239, 269, 243, 257, 227, 260, 241, 251, 323, 250, 245, 249, 255, 246, 251, 262, 276, 254, 259, 239, 255, 244, 260, 257, 276, 255, 282, 260, 257, 260, 262, 240, 252, 253, 248, 260, 245, 252, 264, 274, 254, 261, 236, 261, 247, 280, 248, 269, 242, 240, 247, 276, 270, 260, 247, 284, 279, 242, 246, 236, 268, 256, 257, 251, 252, 243, 248, 255, 262, 283, 239, 265, 234, 294] payload_rows_per_dst=[5430, 5560, 5468, 5588, 5478, 5524, 5486, 5544] dup_rows_per_dst=[2770, 2764, 2674, 2714, 2560, 2724, 2618, 2634] combine_rows_per_src=[5430, 5560, 5468, 5588, 5478, 5524, 5486, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229752 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860792 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491832 weight_movement_bytes=37748736 total_nvlink_bytes=351240568 weight_to_token_ratio=0.120414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229368 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156884984 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540600 weight_movement_bytes=18874368 total_nvlink_bytes=332414968 weight_to_token_ratio=0.060198 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=229944 combine_payload_bytes=157564928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157794872 ingress_bytes=176439296 link_bytes=176439296 token_bytes=315359800 weight_movement_bytes=18874368 total_nvlink_bytes=334234168 weight_to_token_ratio=0.059850 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690554521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43484 payload_rows_remote=37980 dup_rows=22052 dup_rows_remote=19264 dedup_ratio=0.336487 router_tokens_per_expert=[290, 268, 265, 282, 262, 252, 249, 255, 256, 257, 264, 259, 249, 226, 261, 261, 268, 254, 257, 248, 255, 241, 266, 266, 240, 266, 246, 273, 269, 256, 272, 234, 278, 282, 262, 251, 289, 260, 243, 245, 232, 250, 242, 275, 250, 253, 263, 247, 285, 227, 267, 264, 255, 276, 260, 273, 251, 258, 271, 263, 255, 262, 264, 258, 254, 245, 243, 262, 267, 220, 274, 271, 254, 247, 216, 250, 274, 221, 247, 257, 291, 272, 272, 230, 245, 269, 283, 236, 257, 235, 267, 272, 249, 238, 263, 235, 236, 215, 249, 212, 248, 274, 236, 274, 247, 250, 224, 275, 257, 264, 256, 243, 236, 253, 284, 253, 272, 266, 270, 235, 274, 278, 245, 258, 270, 227, 232, 261] payload_rows_per_dst=[5380, 5416, 5442, 5368, 5520, 5432, 5422, 5504] dup_rows_per_dst=[2626, 2806, 2916, 2680, 2814, 2864, 2558, 2788] combine_rows_per_src=[5380, 5416, 5442, 5368, 5520, 5432, 5422, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229760 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016448 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313803136 weight_movement_bytes=18874368 total_nvlink_bytes=332677504 weight_to_token_ratio=0.060147 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158302208 dispatch_meta_bytes=229344 combine_payload_bytes=158302208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158531552 ingress_bytes=196050944 link_bytes=196050944 token_bytes=316833760 weight_movement_bytes=37748736 total_nvlink_bytes=354582496 weight_to_token_ratio=0.119144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.878463682e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38366 dup_rows=21752 dup_rows_remote=19096 dedup_ratio=0.331909 router_tokens_per_expert=[248, 289, 288, 255, 242, 293, 250, 272, 215, 248, 258, 242, 291, 271, 254, 262, 269, 255, 236, 235, 249, 233, 263, 255, 233, 270, 255, 266, 254, 265, 250, 274, 261, 251, 261, 253, 268, 272, 264, 278, 249, 263, 242, 277, 249, 261, 244, 252, 271, 273, 245, 249, 247, 247, 270, 250, 262, 277, 256, 277, 259, 276, 250, 276, 254, 242, 224, 238, 264, 268, 236, 282, 246, 244, 245, 254, 233, 263, 271, 265, 238, 256, 249, 259, 255, 257, 267, 249, 255, 253, 232, 243, 253, 253, 268, 250, 239, 285, 258, 245, 239, 253, 222, 267, 277, 264, 256, 254, 261, 254, 262, 258, 251, 258, 223, 260, 253, 239, 295, 253, 251, 258, 241, 277, 251, 218, 247, 263] payload_rows_per_dst=[5510, 5446, 5494, 5524, 5540, 5418, 5428, 5424] dup_rows_per_dst=[2846, 2678, 2796, 2846, 2518, 2656, 2760, 2652] combine_rows_per_src=[5510, 5446, 5494, 5524, 5540, 5418, 5428, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155566080 dispatch_meta_bytes=228976 combine_payload_bytes=155566080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155795056 ingress_bytes=174440448 link_bytes=174440448 token_bytes=311361136 weight_movement_bytes=18874368 total_nvlink_bytes=330235504 weight_to_token_ratio=0.060619 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671402543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38158 dup_rows=21786 dup_rows_remote=18930 dedup_ratio=0.332428 router_tokens_per_expert=[257, 258, 262, 266, 246, 241, 267, 273, 258, 236, 270, 247, 257, 262, 246, 251, 248, 230, 261, 232, 257, 275, 251, 264, 274, 253, 264, 239, 214, 269, 273, 258, 235, 264, 246, 267, 258, 276, 281, 252, 250, 254, 246, 269, 253, 247, 235, 248, 255, 253, 259, 254, 260, 264, 264, 257, 232, 259, 253, 277, 263, 269, 240, 247, 254, 243, 255, 217, 244, 267, 258, 272, 257, 260, 266, 247, 247, 248, 248, 250, 232, 257, 228, 260, 246, 247, 253, 268, 257, 261, 245, 268, 241, 278, 267, 262, 272, 250, 253, 289, 264, 259, 263, 265, 275, 286, 267, 241, 259, 279, 259, 243, 244, 247, 239, 275, 259, 226, 268, 263, 281, 251, 267, 248, 250, 235, 267, 275] payload_rows_per_dst=[5468, 5482, 5438, 5466, 5396, 5452, 5592, 5456] dup_rows_per_dst=[2726, 2642, 2724, 2746, 2670, 2688, 2856, 2734] combine_rows_per_src=[5468, 5482, 5438, 5466, 5396, 5452, 5592, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38160 dup_rows=21892 dup_rows_remote=19156 dedup_ratio=0.334045 router_tokens_per_expert=[274, 243, 272, 258, 272, 244, 276, 265, 262, 245, 268, 258, 244, 241, 271, 263, 249, 268, 254, 265, 264, 235, 241, 255, 257, 234, 252, 266, 233, 267, 254, 240, 279, 267, 257, 251, 249, 236, 258, 230, 226, 252, 263, 256, 252, 245, 264, 239, 274, 260, 254, 243, 261, 273, 236, 243, 276, 260, 281, 245, 256, 253, 261, 234, 271, 263, 244, 244, 266, 282, 270, 224, 258, 259, 250, 274, 243, 256, 265, 260, 231, 254, 271, 260, 290, 252, 268, 255, 245, 247, 252, 255, 250, 238, 238, 273, 283, 292, 229, 292, 247, 252, 249, 237, 235, 275, 272, 237, 274, 253, 256, 251, 235, 259, 256, 263, 253, 264, 241, 251, 258, 261, 287, 228, 247, 256, 278, 265] payload_rows_per_dst=[5548, 5362, 5402, 5484, 5484, 5392, 5476, 5496] dup_rows_per_dst=[2764, 2706, 2646, 2736, 2774, 2766, 2792, 2708] combine_rows_per_src=[5548, 5362, 5402, 5484, 5484, 5392, 5476, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229848 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157376984 ingress_bytes=176021504 link_bytes=176021504 token_bytes=314524120 weight_movement_bytes=18874368 total_nvlink_bytes=333398488 weight_to_token_ratio=0.060009 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686551443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38294 dup_rows=21738 dup_rows_remote=19100 dedup_ratio=0.331696 router_tokens_per_expert=[252, 270, 274, 279, 214, 270, 235, 252, 284, 250, 256, 258, 258, 246, 269, 274, 258, 249, 219, 267, 234, 264, 261, 252, 287, 259, 261, 248, 238, 254, 253, 247, 256, 245, 244, 264, 236, 262, 262, 255, 222, 261, 256, 254, 254, 294, 268, 238, 238, 283, 286, 247, 258, 262, 254, 244, 241, 246, 243, 262, 254, 257, 281, 257, 251, 247, 296, 272, 226, 247, 255, 245, 252, 263, 262, 247, 259, 286, 281, 259, 267, 232, 248, 255, 245, 225, 255, 279, 276, 261, 276, 268, 254, 232, 255, 257, 254, 259, 250, 253, 258, 252, 248, 251, 248, 234, 291, 228, 287, 245, 277, 273, 266, 254, 244, 246, 244, 243, 245, 277, 270, 248, 274, 242, 273, 238, 247, 240] payload_rows_per_dst=[5484, 5388, 5504, 5486, 5450, 5508, 5516, 5462] dup_rows_per_dst=[2798, 2714, 2638, 2740, 2846, 2662, 2700, 2640] combine_rows_per_src=[5484, 5388, 5504, 5486, 5450, 5508, 5516, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=228352 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156523520 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312818688 weight_movement_bytes=37748736 total_nvlink_bytes=350567424 weight_to_token_ratio=0.120673 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38222 dup_rows=21822 dup_rows_remote=18972 dedup_ratio=0.332977 router_tokens_per_expert=[246, 242, 222, 249, 233, 251, 268, 269, 257, 264, 266, 246, 260, 261, 257, 237, 247, 248, 270, 265, 237, 259, 257, 269, 268, 247, 251, 272, 249, 281, 271, 280, 241, 264, 255, 261, 271, 242, 252, 259, 261, 255, 268, 270, 241, 261, 255, 257, 246, 220, 229, 255, 229, 250, 278, 280, 219, 255, 263, 244, 256, 280, 287, 259, 264, 248, 246, 285, 228, 260, 240, 251, 266, 250, 248, 264, 264, 222, 239, 282, 252, 240, 265, 277, 250, 273, 248, 246, 269, 281, 266, 243, 251, 253, 254, 253, 253, 220, 230, 254, 236, 263, 265, 258, 258, 271, 237, 248, 235, 267, 254, 276, 272, 290, 263, 230, 267, 254, 275, 266, 268, 245, 276, 256, 284, 282, 226, 249] payload_rows_per_dst=[5390, 5492, 5542, 5450, 5470, 5478, 5396, 5496] dup_rows_per_dst=[2666, 2850, 2684, 2650, 2644, 2764, 2654, 2910] combine_rows_per_src=[5390, 5492, 5542, 5450, 5470, 5478, 5396, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38258 dup_rows=21744 dup_rows_remote=18980 dedup_ratio=0.331787 router_tokens_per_expert=[256, 262, 269, 241, 241, 253, 271, 238, 279, 232, 251, 253, 255, 230, 283, 243, 252, 220, 240, 257, 264, 252, 253, 248, 268, 243, 237, 251, 245, 271, 234, 271, 268, 267, 256, 242, 255, 242, 270, 269, 289, 272, 245, 232, 238, 235, 256, 252, 256, 255, 245, 234, 261, 248, 257, 229, 257, 251, 259, 246, 270, 280, 285, 253, 234, 261, 252, 266, 269, 256, 278, 236, 284, 274, 268, 256, 263, 261, 247, 244, 255, 259, 269, 237, 279, 250, 246, 237, 243, 249, 264, 253, 274, 238, 261, 261, 270, 300, 255, 262, 264, 263, 215, 252, 282, 271, 243, 253, 253, 251, 249, 276, 291, 242, 249, 257, 253, 260, 256, 274, 275, 218, 268, 267, 254, 277, 252, 255] payload_rows_per_dst=[5488, 5358, 5462, 5472, 5534, 5482, 5490, 5506] dup_rows_per_dst=[2626, 2654, 2714, 2700, 2764, 2668, 2828, 2790] combine_rows_per_src=[5488, 5358, 5462, 5472, 5534, 5482, 5490, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38204 dup_rows=21804 dup_rows_remote=19012 dedup_ratio=0.332703 router_tokens_per_expert=[249, 268, 263, 255, 262, 231, 232, 260, 270, 266, 278, 263, 272, 245, 266, 272, 255, 284, 257, 269, 250, 266, 219, 283, 217, 241, 270, 264, 247, 246, 246, 252, 252, 249, 250, 219, 253, 252, 257, 245, 250, 264, 282, 274, 283, 243, 266, 258, 268, 261, 261, 257, 271, 249, 281, 266, 245, 249, 254, 289, 239, 251, 235, 260, 243, 273, 262, 263, 273, 258, 219, 273, 249, 274, 241, 273, 270, 256, 255, 256, 221, 268, 277, 253, 240, 262, 235, 264, 268, 271, 268, 224, 253, 271, 244, 251, 228, 260, 246, 249, 233, 239, 253, 271, 267, 241, 249, 258, 257, 260, 247, 256, 236, 274, 285, 263, 231, 229, 258, 278, 271, 244, 236, 264, 242, 261, 238, 285] payload_rows_per_dst=[5528, 5482, 5516, 5480, 5524, 5436, 5344, 5422] dup_rows_per_dst=[2792, 2850, 2720, 2776, 2752, 2704, 2484, 2726] combine_rows_per_src=[5528, 5482, 5516, 5480, 5524, 5436, 5344, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229264 combine_payload_bytes=156303360 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156532624 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312835984 weight_movement_bytes=37748736 total_nvlink_bytes=350584720 weight_to_token_ratio=0.120666 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229576 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081800 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934024 weight_movement_bytes=18874368 total_nvlink_bytes=332808392 weight_to_token_ratio=0.060122 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43574 payload_rows_remote=37984 dup_rows=21962 dup_rows_remote=19092 dedup_ratio=0.335114 router_tokens_per_expert=[270, 257, 254, 253, 258, 226, 278, 250, 276, 263, 275, 255, 261, 261, 237, 232, 273, 250, 252, 279, 253, 269, 252, 303, 266, 278, 220, 244, 290, 258, 258, 246, 266, 232, 231, 244, 219, 254, 251, 280, 223, 224, 251, 258, 268, 248, 260, 265, 278, 290, 257, 242, 269, 258, 265, 268, 255, 255, 235, 273, 243, 235, 251, 268, 251, 237, 263, 238, 257, 261, 247, 250, 257, 269, 256, 250, 245, 263, 237, 249, 282, 269, 243, 249, 270, 242, 240, 266, 238, 239, 237, 244, 250, 269, 262, 239, 276, 248, 257, 258, 255, 259, 246, 237, 290, 272, 259, 242, 266, 248, 260, 247, 251, 268, 262, 248, 282, 220, 253, 299, 236, 258, 298, 233, 272, 248, 267, 271] payload_rows_per_dst=[5378, 5282, 5560, 5498, 5408, 5352, 5506, 5590] dup_rows_per_dst=[2668, 2494, 2994, 2786, 2676, 2734, 2740, 2870] combine_rows_per_src=[5378, 5282, 5560, 5498, 5408, 5352, 5506, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=228776 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156786088 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313343400 weight_movement_bytes=37748736 total_nvlink_bytes=351092136 weight_to_token_ratio=0.120471 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228952 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156933720 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313638488 weight_movement_bytes=18874368 total_nvlink_bytes=332512856 weight_to_token_ratio=0.060179 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228864 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156712448 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196032 weight_movement_bytes=18874368 total_nvlink_bytes=332070400 weight_to_token_ratio=0.060264 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43548 payload_rows_remote=38118 dup_rows=21988 dup_rows_remote=19346 dedup_ratio=0.335510 router_tokens_per_expert=[226, 253, 268, 261, 222, 256, 237, 258, 289, 277, 262, 236, 269, 270, 253, 253, 241, 239, 251, 267, 241, 282, 227, 292, 259, 241, 268, 255, 253, 240, 256, 243, 243, 269, 260, 220, 284, 270, 238, 256, 245, 263, 257, 275, 236, 235, 273, 236, 270, 231, 267, 284, 265, 273, 276, 267, 266, 253, 261, 253, 247, 255, 257, 265, 225, 259, 254, 260, 259, 251, 269, 253, 288, 236, 259, 253, 263, 248, 256, 249, 232, 256, 256, 252, 246, 233, 293, 272, 248, 240, 238, 252, 276, 260, 232, 250, 247, 274, 270, 254, 246, 259, 260, 244, 281, 272, 269, 247, 260, 277, 249, 256, 266, 267, 272, 260, 249, 274, 258, 238, 246, 222, 242, 274, 240, 254, 232, 296] payload_rows_per_dst=[5488, 5396, 5336, 5490, 5440, 5430, 5504, 5464] dup_rows_per_dst=[2692, 2714, 2784, 2890, 2724, 2642, 2826, 2716] combine_rows_per_src=[5488, 5396, 5336, 5490, 5440, 5430, 5504, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155582464 dispatch_meta_bytes=228304 combine_payload_bytes=155582464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155810768 ingress_bytes=193331200 link_bytes=193331200 token_bytes=311393232 weight_movement_bytes=37748736 total_nvlink_bytes=349141968 weight_to_token_ratio=0.121225 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852404434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38398 dup_rows=21740 dup_rows_remote=19066 dedup_ratio=0.331726 router_tokens_per_expert=[298, 254, 230, 251, 268, 268, 252, 284, 231, 267, 271, 268, 241, 254, 254, 248, 267, 248, 258, 259, 236, 257, 265, 274, 285, 247, 270, 271, 256, 249, 286, 245, 253, 242, 255, 229, 248, 260, 229, 261, 274, 265, 257, 248, 277, 259, 246, 229, 281, 245, 244, 264, 256, 270, 227, 273, 244, 234, 254, 270, 249, 225, 217, 282, 238, 239, 277, 276, 282, 234, 255, 264, 254, 242, 273, 282, 278, 260, 256, 241, 259, 254, 241, 276, 268, 248, 259, 241, 259, 283, 257, 298, 243, 275, 240, 258, 274, 257, 244, 244, 237, 235, 254, 263, 260, 241, 236, 259, 253, 270, 263, 246, 264, 260, 251, 269, 236, 251, 251, 247, 246, 271, 273, 228, 241, 250, 278, 227] payload_rows_per_dst=[5520, 5486, 5458, 5500, 5500, 5462, 5398, 5472] dup_rows_per_dst=[2758, 2732, 2734, 2570, 2802, 2856, 2674, 2614] combine_rows_per_src=[5520, 5486, 5458, 5500, 5500, 5462, 5398, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38172 dup_rows=21878 dup_rows_remote=19156 dedup_ratio=0.333832 router_tokens_per_expert=[246, 255, 264, 224, 236, 267, 279, 248, 234, 263, 249, 276, 271, 237, 266, 262, 250, 235, 275, 250, 239, 255, 237, 279, 242, 258, 267, 255, 249, 269, 274, 267, 249, 246, 235, 245, 258, 233, 285, 260, 233, 271, 244, 264, 245, 234, 285, 242, 253, 234, 270, 246, 259, 265, 282, 261, 224, 279, 233, 241, 254, 262, 268, 273, 252, 250, 269, 254, 251, 258, 255, 247, 249, 269, 257, 267, 263, 243, 243, 229, 274, 259, 261, 241, 254, 261, 285, 257, 236, 278, 257, 291, 262, 241, 253, 262, 276, 247, 236, 281, 259, 234, 264, 283, 253, 236, 253, 264, 256, 238, 259, 285, 264, 252, 237, 229, 245, 294, 251, 281, 259, 243, 260, 246, 252, 257, 275, 260] payload_rows_per_dst=[5416, 5508, 5376, 5486, 5450, 5496, 5482, 5444] dup_rows_per_dst=[2738, 2694, 2682, 2722, 2662, 2848, 2766, 2766] combine_rows_per_src=[5416, 5508, 5376, 5486, 5450, 5496, 5482, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229856 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156361184 ingress_bytes=193880064 link_bytes=193880064 token_bytes=312492512 weight_movement_bytes=37748736 total_nvlink_bytes=350241248 weight_to_token_ratio=0.120799 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857663378e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38328 dup_rows=21786 dup_rows_remote=19100 dedup_ratio=0.332428 router_tokens_per_expert=[258, 277, 224, 244, 256, 270, 254, 271, 229, 262, 271, 280, 247, 273, 229, 254, 257, 274, 260, 293, 245, 258, 247, 253, 250, 269, 271, 252, 258, 251, 259, 251, 241, 243, 254, 257, 247, 262, 231, 249, 258, 256, 253, 254, 258, 247, 284, 260, 265, 252, 247, 274, 260, 241, 216, 253, 294, 220, 278, 235, 243, 238, 242, 253, 256, 242, 260, 239, 252, 237, 262, 231, 264, 243, 238, 284, 271, 265, 248, 250, 253, 297, 245, 277, 264, 245, 271, 241, 260, 247, 275, 276, 257, 238, 242, 261, 230, 247, 294, 263, 279, 256, 247, 240, 264, 249, 231, 241, 253, 287, 251, 240, 269, 230, 263, 270, 269, 268, 280, 282, 252, 260, 266, 265, 245, 249, 263, 262] payload_rows_per_dst=[5456, 5508, 5422, 5392, 5420, 5490, 5518, 5544] dup_rows_per_dst=[2742, 2788, 2686, 2630, 2664, 2808, 2626, 2842] combine_rows_per_src=[5456, 5508, 5422, 5392, 5420, 5490, 5518, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229856 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157508064 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314786272 weight_movement_bytes=18874368 total_nvlink_bytes=333660640 weight_to_token_ratio=0.059959 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38230 dup_rows=21850 dup_rows_remote=19104 dedup_ratio=0.333405 router_tokens_per_expert=[257, 247, 251, 273, 258, 250, 263, 252, 242, 249, 269, 251, 264, 240, 279, 243, 249, 265, 279, 263, 255, 259, 266, 238, 259, 255, 254, 248, 268, 268, 221, 254, 244, 254, 270, 250, 265, 272, 272, 245, 258, 248, 259, 255, 246, 271, 251, 249, 263, 254, 249, 266, 268, 248, 216, 250, 271, 261, 278, 237, 278, 248, 248, 236, 262, 249, 273, 260, 260, 258, 295, 229, 259, 253, 267, 217, 246, 278, 256, 271, 239, 243, 242, 242, 260, 257, 265, 259, 278, 261, 248, 235, 271, 253, 284, 248, 265, 267, 244, 261, 260, 243, 259, 240, 255, 264, 242, 264, 274, 265, 267, 265, 239, 235, 254, 269, 258, 256, 263, 263, 250, 254, 234, 243, 261, 240, 278, 249] payload_rows_per_dst=[5460, 5456, 5470, 5474, 5502, 5418, 5510, 5396] dup_rows_per_dst=[2716, 2746, 2748, 2668, 2764, 2752, 2760, 2696] combine_rows_per_src=[5460, 5456, 5470, 5474, 5502, 5418, 5510, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38198 dup_rows=21888 dup_rows_remote=19138 dedup_ratio=0.333984 router_tokens_per_expert=[264, 262, 246, 268, 254, 240, 250, 273, 274, 244, 235, 265, 279, 249, 258, 253, 246, 278, 232, 268, 273, 240, 260, 265, 251, 253, 237, 261, 272, 260, 245, 242, 241, 262, 262, 245, 247, 238, 281, 254, 271, 243, 268, 269, 252, 245, 270, 273, 244, 252, 253, 283, 248, 274, 251, 250, 248, 254, 239, 276, 241, 231, 260, 243, 253, 256, 253, 264, 292, 246, 241, 236, 259, 273, 264, 246, 251, 248, 244, 274, 269, 254, 247, 286, 254, 243, 248, 266, 266, 253, 262, 266, 260, 249, 241, 241, 249, 247, 265, 273, 249, 249, 234, 243, 283, 243, 266, 272, 241, 251, 262, 266, 267, 266, 232, 257, 247, 247, 262, 262, 256, 263, 273, 271, 250, 250, 251, 251] payload_rows_per_dst=[5496, 5448, 5448, 5444, 5450, 5482, 5372, 5508] dup_rows_per_dst=[2732, 2718, 2794, 2650, 2750, 2728, 2814, 2702] combine_rows_per_src=[5496, 5448, 5448, 5444, 5450, 5482, 5372, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229312 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156581824 ingress_bytes=175226880 link_bytes=175226880 token_bytes=312934336 weight_movement_bytes=18874368 total_nvlink_bytes=331808704 weight_to_token_ratio=0.060314 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678937747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43536 payload_rows_remote=38050 dup_rows=22000 dup_rows_remote=19300 dedup_ratio=0.335693 router_tokens_per_expert=[237, 241, 254, 263, 255, 274, 270, 255, 255, 233, 247, 251, 284, 258, 240, 276, 267, 283, 277, 247, 229, 231, 254, 259, 254, 238, 270, 254, 285, 273, 279, 242, 244, 281, 246, 256, 278, 228, 216, 259, 244, 264, 282, 254, 253, 245, 232, 254, 281, 237, 230, 227, 250, 257, 275, 253, 258, 233, 258, 262, 239, 250, 236, 269, 247, 251, 238, 271, 274, 236, 273, 246, 255, 258, 242, 284, 267, 264, 257, 269, 249, 277, 239, 294, 243, 247, 286, 286, 283, 257, 259, 249, 268, 240, 245, 253, 258, 246, 260, 280, 260, 248, 263, 247, 261, 242, 275, 268, 247, 282, 255, 239, 282, 229, 231, 264, 279, 263, 254, 242, 285, 251, 223, 241, 254, 250, 242, 254] payload_rows_per_dst=[5486, 5444, 5540, 5322, 5466, 5490, 5468, 5320] dup_rows_per_dst=[2700, 2840, 2912, 2470, 2798, 2926, 2794, 2560] combine_rows_per_src=[5486, 5444, 5540, 5322, 5466, 5490, 5468, 5320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229712 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157221200 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314212688 weight_movement_bytes=18874368 total_nvlink_bytes=333087056 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=38006 dup_rows=21994 dup_rows_remote=19152 dedup_ratio=0.335602 router_tokens_per_expert=[259, 246, 274, 228, 264, 248, 247, 271, 259, 249, 253, 235, 262, 265, 245, 263, 256, 276, 260, 230, 273, 268, 299, 247, 248, 258, 262, 247, 241, 270, 254, 253, 250, 243, 238, 255, 237, 233, 253, 240, 230, 282, 225, 245, 250, 258, 241, 258, 269, 239, 267, 272, 272, 249, 260, 258, 269, 256, 274, 257, 265, 267, 270, 262, 260, 242, 247, 253, 263, 274, 264, 247, 222, 281, 251, 273, 235, 269, 258, 251, 268, 257, 235, 274, 242, 236, 279, 257, 258, 259, 260, 253, 249, 271, 274, 247, 241, 254, 284, 238, 236, 242, 278, 263, 227, 255, 260, 258, 250, 262, 268, 246, 253, 242, 258, 258, 274, 266, 282, 267, 252, 276, 259, 247, 268, 230, 273, 238] payload_rows_per_dst=[5302, 5480, 5286, 5594, 5396, 5456, 5492, 5536] dup_rows_per_dst=[2618, 2804, 2590, 2818, 2762, 2804, 2756, 2842] combine_rows_per_src=[5302, 5480, 5286, 5594, 5396, 5456, 5492, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229336 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819416 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409496 weight_movement_bytes=18874368 total_nvlink_bytes=332283864 weight_to_token_ratio=0.060223 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229344 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688352 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147360 weight_movement_bytes=18874368 total_nvlink_bytes=332021728 weight_to_token_ratio=0.060273 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=155852800 dispatch_meta_bytes=229400 combine_payload_bytes=155852800 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156082200 ingress_bytes=174727168 link_bytes=174727168 token_bytes=311935000 weight_movement_bytes=18874368 total_nvlink_bytes=330809368 weight_to_token_ratio=0.060507 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674149753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38166 dup_rows=21794 dup_rows_remote=19146 dedup_ratio=0.332550 router_tokens_per_expert=[274, 269, 240, 258, 267, 238, 256, 263, 253, 243, 250, 239, 257, 235, 278, 259, 255, 241, 234, 252, 280, 282, 212, 271, 257, 252, 223, 236, 252, 235, 227, 243, 273, 256, 248, 259, 243, 261, 259, 273, 250, 239, 254, 279, 264, 252, 266, 262, 249, 256, 260, 241, 243, 293, 262, 252, 245, 268, 257, 222, 242, 255, 264, 246, 258, 277, 243, 261, 262, 272, 268, 239, 238, 294, 263, 251, 258, 230, 281, 235, 227, 252, 257, 264, 273, 223, 275, 270, 247, 258, 250, 268, 236, 262, 275, 275, 268, 253, 252, 291, 277, 246, 256, 276, 262, 252, 292, 261, 262, 242, 248, 249, 266, 264, 256, 255, 245, 265, 285, 263, 257, 250, 272, 250, 240, 268, 246, 233] payload_rows_per_dst=[5480, 5320, 5508, 5426, 5504, 5576, 5520, 5408] dup_rows_per_dst=[2678, 2584, 2768, 2684, 2756, 2648, 2854, 2822] combine_rows_per_src=[5480, 5320, 5508, 5426, 5504, 5576, 5520, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=155672576 dispatch_meta_bytes=228632 combine_payload_bytes=155672576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155901208 ingress_bytes=174546944 link_bytes=174546944 token_bytes=311573784 weight_movement_bytes=18874368 total_nvlink_bytes=330448152 weight_to_token_ratio=0.060578 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672422935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38292 dup_rows=21786 dup_rows_remote=19018 dedup_ratio=0.332428 router_tokens_per_expert=[269, 266, 232, 275, 265, 262, 253, 262, 253, 245, 259, 256, 261, 263, 257, 241, 236, 260, 294, 266, 282, 271, 241, 234, 255, 235, 238, 265, 247, 283, 276, 258, 257, 248, 258, 250, 258, 264, 262, 253, 254, 256, 259, 231, 262, 262, 269, 286, 275, 251, 264, 258, 234, 233, 266, 235, 259, 268, 273, 236, 244, 242, 261, 253, 273, 237, 235, 251, 274, 258, 244, 269, 267, 230, 237, 247, 256, 263, 255, 271, 246, 254, 249, 251, 256, 267, 240, 274, 231, 231, 263, 260, 251, 266, 218, 259, 270, 261, 277, 251, 291, 262, 255, 274, 248, 233, 241, 233, 242, 275, 253, 247, 249, 288, 250, 252, 241, 284, 242, 272, 269, 251, 290, 260, 256, 256, 233, 238] payload_rows_per_dst=[5470, 5510, 5528, 5438, 5466, 5394, 5458, 5486] dup_rows_per_dst=[2768, 2772, 2730, 2666, 2668, 2638, 2768, 2776] combine_rows_per_src=[5470, 5510, 5528, 5438, 5466, 5394, 5458, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43558 payload_rows_remote=38060 dup_rows=21978 dup_rows_remote=19240 dedup_ratio=0.335358 router_tokens_per_expert=[247, 236, 304, 270, 251, 235, 256, 238, 268, 304, 268, 245, 254, 282, 258, 250, 237, 266, 270, 261, 277, 248, 269, 278, 266, 236, 243, 258, 251, 257, 268, 248, 258, 263, 249, 273, 247, 262, 267, 255, 257, 241, 242, 257, 241, 249, 283, 260, 283, 270, 246, 247, 221, 239, 250, 246, 271, 281, 275, 246, 256, 254, 251, 282, 250, 257, 235, 246, 269, 274, 256, 209, 264, 258, 242, 255, 265, 298, 281, 265, 273, 249, 266, 237, 253, 251, 267, 257, 242, 278, 251, 263, 267, 255, 253, 267, 251, 235, 259, 251, 243, 260, 276, 258, 248, 264, 250, 253, 236, 239, 259, 236, 234, 252, 258, 257, 272, 230, 276, 268, 245, 236, 242, 241, 240, 272, 227, 226] payload_rows_per_dst=[5478, 5506, 5516, 5498, 5468, 5472, 5264, 5356] dup_rows_per_dst=[2854, 2760, 2692, 2738, 2780, 2786, 2772, 2596] combine_rows_per_src=[5478, 5506, 5516, 5498, 5468, 5472, 5264, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156327936 dispatch_meta_bytes=229248 combine_payload_bytes=156327936 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156557184 ingress_bytes=231825408 link_bytes=231825408 token_bytes=312885120 weight_movement_bytes=75497472 total_nvlink_bytes=388382592 weight_to_token_ratio=0.241295 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221236994e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43428 payload_rows_remote=38090 dup_rows=22108 dup_rows_remote=19478 dedup_ratio=0.337341 router_tokens_per_expert=[249, 249, 249, 254, 267, 270, 258, 274, 258, 252, 269, 271, 292, 240, 229, 259, 239, 250, 238, 267, 258, 260, 257, 227, 241, 258, 273, 252, 260, 271, 257, 258, 250, 252, 233, 257, 240, 242, 262, 255, 252, 249, 251, 247, 269, 235, 246, 244, 279, 270, 220, 252, 261, 270, 278, 296, 264, 263, 261, 238, 248, 264, 259, 275, 255, 248, 272, 280, 265, 241, 257, 262, 266, 280, 277, 247, 241, 253, 257, 219, 247, 246, 266, 275, 283, 264, 255, 276, 243, 245, 259, 256, 259, 241, 276, 240, 260, 272, 236, 240, 258, 252, 237, 253, 254, 263, 231, 264, 246, 256, 256, 260, 229, 259, 224, 249, 251, 286, 271, 274, 266, 260, 272, 240, 248, 239, 243, 280] payload_rows_per_dst=[5438, 5426, 5338, 5470, 5494, 5498, 5364, 5400] dup_rows_per_dst=[2842, 2706, 2630, 2926, 2746, 2764, 2712, 2782] combine_rows_per_src=[5438, 5426, 5338, 5470, 5494, 5498, 5364, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229240 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073272 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917304 weight_movement_bytes=18874368 total_nvlink_bytes=332791672 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38302 dup_rows=21650 dup_rows_remote=18828 dedup_ratio=0.330353 router_tokens_per_expert=[277, 245, 241, 282, 261, 242, 233, 253, 254, 253, 273, 258, 263, 292, 253, 247, 253, 273, 273, 285, 247, 241, 264, 248, 269, 244, 252, 285, 259, 249, 276, 285, 244, 255, 234, 273, 266, 252, 243, 246, 264, 250, 248, 261, 244, 271, 267, 235, 239, 244, 263, 249, 264, 248, 277, 254, 275, 253, 273, 238, 274, 255, 258, 260, 271, 263, 243, 266, 234, 236, 258, 279, 271, 220, 276, 241, 263, 254, 244, 244, 276, 265, 247, 255, 265, 271, 276, 277, 234, 263, 255, 253, 253, 238, 258, 272, 264, 232, 222, 252, 236, 241, 248, 245, 260, 266, 261, 263, 269, 260, 225, 233, 244, 244, 261, 223, 256, 232, 277, 249, 261, 224, 272, 266, 256, 279, 254, 265] payload_rows_per_dst=[5572, 5584, 5482, 5464, 5420, 5526, 5364, 5474] dup_rows_per_dst=[2682, 2822, 2624, 2784, 2706, 2790, 2590, 2652] combine_rows_per_src=[5572, 5584, 5482, 5464, 5420, 5526, 5364, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=155893760 dispatch_meta_bytes=229200 combine_payload_bytes=155893760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156122960 ingress_bytes=174768128 link_bytes=174768128 token_bytes=312016720 weight_movement_bytes=18874368 total_nvlink_bytes=330891088 weight_to_token_ratio=0.060492 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674542211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=230272 combine_payload_bytes=156016640 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156246912 ingress_bytes=212639744 link_bytes=212639744 token_bytes=312263552 weight_movement_bytes=56623104 total_nvlink_bytes=368886656 weight_to_token_ratio=0.181331 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037409402e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38314 dup_rows=21836 dup_rows_remote=19088 dedup_ratio=0.333191 router_tokens_per_expert=[265, 236, 259, 275, 251, 291, 270, 275, 263, 251, 259, 253, 227, 267, 236, 250, 246, 272, 244, 264, 268, 212, 259, 267, 236, 258, 229, 283, 235, 268, 277, 274, 247, 246, 270, 246, 253, 270, 230, 258, 256, 257, 281, 257, 288, 252, 239, 263, 252, 266, 244, 237, 268, 248, 255, 280, 246, 242, 225, 257, 267, 239, 252, 286, 256, 217, 247, 276, 265, 266, 263, 277, 250, 255, 258, 229, 253, 253, 248, 254, 278, 237, 257, 234, 250, 265, 249, 272, 262, 276, 271, 287, 292, 244, 240, 241, 260, 250, 245, 256, 216, 240, 252, 277, 251, 230, 270, 242, 261, 260, 253, 251, 266, 275, 252, 238, 222, 252, 260, 243, 274, 230, 280, 243, 253, 286, 291, 270] payload_rows_per_dst=[5480, 5460, 5550, 5468, 5386, 5496, 5406, 5454] dup_rows_per_dst=[2776, 2724, 2676, 2660, 2748, 2814, 2622, 2816] combine_rows_per_src=[5480, 5460, 5550, 5468, 5386, 5496, 5406, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38236 dup_rows=21672 dup_rows_remote=18982 dedup_ratio=0.330688 router_tokens_per_expert=[246, 256, 258, 258, 251, 251, 280, 257, 278, 281, 226, 278, 248, 258, 267, 255, 262, 251, 268, 255, 259, 272, 255, 248, 272, 252, 272, 264, 278, 258, 267, 252, 269, 251, 247, 263, 234, 225, 302, 280, 244, 251, 217, 280, 261, 261, 262, 235, 242, 242, 253, 244, 253, 270, 241, 259, 240, 243, 263, 305, 232, 223, 283, 240, 263, 237, 260, 255, 257, 284, 235, 258, 260, 221, 253, 258, 278, 241, 238, 287, 269, 256, 256, 248, 269, 257, 248, 258, 256, 269, 276, 273, 274, 280, 259, 262, 218, 253, 255, 249, 250, 242, 292, 246, 253, 262, 257, 238, 259, 255, 243, 261, 217, 276, 240, 260, 236, 259, 241, 246, 264, 261, 234, 259, 236, 240, 271, 252] payload_rows_per_dst=[5628, 5660, 5422, 5402, 5488, 5582, 5260, 5422] dup_rows_per_dst=[2690, 3182, 2742, 2642, 2682, 2838, 2334, 2562] combine_rows_per_src=[5628, 5660, 5422, 5402, 5488, 5582, 5260, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38226 dup_rows=21830 dup_rows_remote=19072 dedup_ratio=0.333099 router_tokens_per_expert=[257, 234, 249, 246, 245, 263, 259, 263, 280, 258, 279, 252, 239, 240, 259, 280, 281, 233, 266, 285, 259, 243, 239, 248, 266, 242, 226, 258, 251, 268, 248, 258, 265, 266, 245, 252, 287, 257, 281, 311, 247, 239, 238, 249, 271, 268, 246, 301, 248, 242, 232, 251, 244, 239, 250, 269, 251, 274, 239, 254, 246, 273, 264, 243, 260, 272, 256, 268, 236, 243, 256, 271, 265, 213, 255, 245, 245, 259, 239, 257, 266, 247, 229, 231, 237, 244, 266, 260, 247, 282, 281, 225, 244, 218, 266, 238, 284, 256, 263, 253, 271, 254, 268, 253, 261, 266, 257, 259, 276, 246, 272, 273, 264, 250, 252, 243, 249, 247, 263, 261, 260, 262, 283, 241, 264, 257, 259, 264] payload_rows_per_dst=[5310, 5420, 5508, 5464, 5482, 5402, 5640, 5480] dup_rows_per_dst=[2398, 2722, 2786, 2734, 2830, 2666, 2936, 2758] combine_rows_per_src=[5310, 5420, 5508, 5464, 5482, 5402, 5640, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=228520 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157113512 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313998504 weight_movement_bytes=18874368 total_nvlink_bytes=332872872 weight_to_token_ratio=0.060110 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229608 combine_payload_bytes=156934144 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157163752 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097896 weight_movement_bytes=37748736 total_nvlink_bytes=351846632 weight_to_token_ratio=0.120181 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=228872 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=156843528 ingress_bytes=250986496 link_bytes=250986496 token_bytes=313458184 weight_movement_bytes=94371840 total_nvlink_bytes=407830024 weight_to_token_ratio=0.301067 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.404829112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38356 dup_rows=21680 dup_rows_remote=18996 dedup_ratio=0.330811 router_tokens_per_expert=[267, 266, 246, 261, 257, 279, 279, 271, 237, 256, 257, 244, 252, 273, 241, 235, 273, 256, 252, 270, 237, 235, 272, 261, 272, 230, 268, 228, 241, 304, 280, 260, 305, 253, 251, 246, 255, 259, 259, 241, 267, 272, 248, 262, 253, 232, 235, 273, 272, 238, 244, 259, 241, 237, 243, 256, 269, 259, 220, 273, 249, 251, 272, 254, 233, 249, 276, 264, 291, 243, 272, 229, 264, 236, 248, 264, 244, 251, 228, 269, 256, 244, 251, 259, 279, 243, 282, 248, 274, 246, 264, 254, 242, 248, 260, 242, 257, 288, 259, 267, 239, 244, 250, 260, 254, 261, 267, 254, 243, 248, 255, 246, 255, 267, 253, 272, 236, 247, 277, 266, 271, 274, 226, 238, 257, 259, 247, 270] payload_rows_per_dst=[5486, 5478, 5536, 5450, 5418, 5500, 5484, 5504] dup_rows_per_dst=[2756, 2800, 2686, 2624, 2704, 2684, 2700, 2726] combine_rows_per_src=[5486, 5478, 5536, 5450, 5418, 5500, 5484, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229192 combine_payload_bytes=156573696 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156802888 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313376584 weight_movement_bytes=37748736 total_nvlink_bytes=351125320 weight_to_token_ratio=0.120458 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38334 dup_rows=21794 dup_rows_remote=19190 dedup_ratio=0.332550 router_tokens_per_expert=[265, 275, 268, 270, 269, 281, 259, 247, 251, 248, 248, 233, 252, 258, 255, 243, 280, 273, 238, 231, 266, 259, 245, 275, 252, 280, 222, 218, 255, 238, 245, 246, 259, 261, 236, 247, 280, 247, 249, 243, 283, 267, 271, 269, 246, 304, 269, 265, 256, 274, 231, 240, 256, 275, 262, 243, 261, 268, 230, 258, 281, 259, 258, 292, 265, 237, 263, 268, 258, 244, 272, 267, 226, 261, 244, 243, 228, 260, 289, 221, 281, 246, 268, 261, 225, 272, 240, 263, 258, 250, 272, 260, 264, 259, 271, 256, 248, 245, 253, 248, 254, 250, 256, 229, 249, 231, 242, 263, 247, 267, 248, 276, 248, 266, 240, 252, 237, 240, 249, 257, 260, 274, 263, 271, 249, 273, 258, 248] payload_rows_per_dst=[5492, 5378, 5504, 5536, 5504, 5494, 5408, 5426] dup_rows_per_dst=[2752, 2668, 2888, 2752, 2588, 2798, 2604, 2744] combine_rows_per_src=[5492, 5378, 5504, 5536, 5504, 5494, 5408, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38102 dup_rows=22040 dup_rows_remote=19236 dedup_ratio=0.336304 router_tokens_per_expert=[274, 250, 243, 257, 236, 264, 271, 267, 262, 246, 246, 255, 282, 274, 256, 264, 251, 266, 259, 240, 269, 254, 258, 231, 225, 284, 271, 240, 258, 251, 254, 261, 253, 245, 252, 255, 261, 236, 237, 258, 259, 254, 247, 243, 269, 232, 244, 278, 250, 226, 239, 295, 271, 281, 249, 261, 248, 274, 268, 263, 236, 238, 276, 224, 265, 235, 237, 255, 243, 272, 260, 269, 251, 262, 262, 236, 264, 269, 272, 277, 240, 264, 228, 265, 265, 222, 259, 267, 236, 249, 256, 257, 276, 258, 277, 247, 245, 277, 248, 235, 279, 258, 273, 278, 258, 257, 233, 284, 243, 249, 264, 250, 268, 244, 278, 246, 268, 270, 255, 268, 268, 265, 234, 247, 233, 239, 250, 268] payload_rows_per_dst=[5500, 5398, 5328, 5394, 5444, 5422, 5544, 5466] dup_rows_per_dst=[2794, 2746, 2718, 2804, 2814, 2710, 2718, 2736] combine_rows_per_src=[5500, 5398, 5328, 5394, 5444, 5422, 5544, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43722 payload_rows_remote=38276 dup_rows=21814 dup_rows_remote=19146 dedup_ratio=0.332855 router_tokens_per_expert=[241, 279, 252, 244, 258, 244, 223, 270, 288, 281, 264, 249, 239, 242, 303, 253, 261, 263, 248, 240, 284, 228, 266, 238, 270, 265, 260, 266, 269, 270, 277, 239, 257, 264, 256, 245, 240, 247, 234, 226, 275, 252, 260, 274, 255, 259, 260, 253, 255, 238, 270, 265, 265, 243, 246, 249, 252, 288, 278, 239, 240, 256, 252, 258, 246, 268, 227, 237, 235, 267, 228, 256, 232, 255, 251, 249, 253, 279, 244, 260, 258, 274, 266, 240, 289, 261, 235, 255, 258, 283, 296, 252, 240, 271, 272, 256, 241, 254, 234, 263, 281, 245, 229, 258, 244, 245, 241, 278, 250, 257, 225, 311, 241, 225, 253, 254, 237, 271, 268, 281, 251, 245, 252, 255, 256, 272, 280, 253] payload_rows_per_dst=[5504, 5498, 5446, 5452, 5348, 5550, 5440, 5484] dup_rows_per_dst=[2756, 2790, 2668, 2736, 2626, 2862, 2672, 2704] combine_rows_per_src=[5504, 5498, 5446, 5452, 5348, 5550, 5440, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229408 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157335584 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314441760 weight_movement_bytes=18874368 total_nvlink_bytes=333316128 weight_to_token_ratio=0.060025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=230096 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157246160 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314262224 weight_movement_bytes=18874368 total_nvlink_bytes=333136592 weight_to_token_ratio=0.060059 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38100 dup_rows=21954 dup_rows_remote=19070 dedup_ratio=0.334991 router_tokens_per_expert=[249, 281, 255, 215, 242, 245, 229, 248, 258, 273, 257, 263, 272, 257, 242, 271, 288, 270, 244, 249, 274, 232, 263, 240, 241, 276, 291, 256, 272, 250, 264, 273, 301, 255, 259, 268, 232, 269, 251, 263, 253, 279, 252, 268, 245, 246, 268, 273, 254, 240, 242, 255, 252, 281, 240, 281, 267, 265, 274, 262, 241, 245, 285, 256, 239, 255, 262, 250, 272, 226, 259, 260, 236, 260, 254, 266, 228, 229, 268, 225, 218, 258, 260, 245, 257, 256, 257, 284, 259, 277, 248, 254, 250, 237, 222, 276, 245, 268, 237, 274, 266, 261, 237, 256, 244, 263, 274, 252, 266, 253, 260, 265, 237, 256, 257, 247, 264, 248, 237, 248, 246, 235, 271, 248, 272, 268, 263, 241] payload_rows_per_dst=[5380, 5482, 5532, 5506, 5372, 5374, 5512, 5424] dup_rows_per_dst=[2734, 2884, 2832, 2774, 2606, 2742, 2730, 2652] combine_rows_per_src=[5380, 5482, 5532, 5506, 5372, 5374, 5512, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229352 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156295144 ingress_bytes=174940160 link_bytes=174940160 token_bytes=312360936 weight_movement_bytes=18874368 total_nvlink_bytes=331235304 weight_to_token_ratio=0.060425 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676190537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229688 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157008184 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786680 weight_movement_bytes=37748736 total_nvlink_bytes=351535416 weight_to_token_ratio=0.120301 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38152 dup_rows=21858 dup_rows_remote=19138 dedup_ratio=0.333527 router_tokens_per_expert=[247, 258, 241, 274, 263, 241, 265, 275, 276, 263, 255, 270, 224, 278, 252, 252, 269, 256, 254, 257, 280, 257, 254, 234, 275, 267, 245, 260, 243, 249, 280, 251, 253, 235, 269, 285, 225, 251, 248, 282, 241, 263, 281, 283, 235, 278, 250, 236, 256, 243, 260, 265, 265, 256, 243, 270, 239, 211, 269, 233, 251, 250, 229, 266, 281, 242, 254, 257, 251, 264, 252, 255, 257, 264, 252, 280, 246, 239, 292, 237, 247, 245, 252, 249, 248, 245, 280, 243, 252, 249, 234, 229, 259, 254, 252, 244, 265, 259, 271, 254, 280, 259, 266, 245, 222, 252, 266, 252, 272, 244, 283, 272, 253, 300, 253, 278, 241, 254, 237, 237, 254, 262, 257, 260, 246, 260, 255, 268] payload_rows_per_dst=[5422, 5442, 5466, 5360, 5526, 5440, 5510, 5512] dup_rows_per_dst=[2846, 2820, 2764, 2652, 2720, 2524, 2814, 2718] combine_rows_per_src=[5422, 5442, 5466, 5360, 5526, 5440, 5510, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38200 dup_rows=21760 dup_rows_remote=19024 dedup_ratio=0.332031 router_tokens_per_expert=[281, 231, 252, 262, 305, 269, 239, 255, 261, 276, 268, 227, 236, 288, 262, 244, 254, 261, 258, 278, 254, 242, 252, 252, 251, 272, 254, 267, 262, 247, 276, 241, 267, 246, 258, 273, 273, 256, 249, 247, 271, 229, 234, 232, 258, 250, 230, 228, 245, 259, 236, 270, 261, 244, 262, 269, 266, 256, 251, 255, 244, 256, 245, 237, 263, 251, 259, 264, 249, 253, 248, 276, 275, 245, 258, 263, 277, 259, 263, 229, 249, 255, 270, 238, 263, 252, 267, 250, 270, 256, 251, 258, 250, 258, 264, 227, 264, 263, 252, 231, 240, 262, 279, 256, 257, 265, 260, 242, 261, 264, 234, 264, 267, 266, 264, 248, 223, 270, 250, 254, 246, 267, 251, 265, 260, 299, 246, 254] payload_rows_per_dst=[5576, 5610, 5362, 5426, 5536, 5466, 5492, 5308] dup_rows_per_dst=[2736, 3010, 2640, 2748, 2728, 2628, 2696, 2574] combine_rows_per_src=[5576, 5610, 5362, 5426, 5536, 5466, 5492, 5308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38284 dup_rows=21768 dup_rows_remote=18826 dedup_ratio=0.332153 router_tokens_per_expert=[261, 282, 253, 231, 253, 236, 275, 272, 261, 276, 271, 242, 236, 270, 275, 271, 269, 272, 248, 270, 237, 248, 229, 247, 252, 237, 275, 244, 261, 246, 249, 273, 269, 255, 252, 249, 253, 249, 247, 270, 253, 260, 261, 257, 252, 267, 245, 282, 231, 275, 244, 256, 257, 253, 284, 263, 239, 260, 229, 257, 274, 280, 275, 220, 263, 254, 257, 237, 255, 257, 241, 305, 264, 282, 253, 273, 254, 249, 224, 257, 243, 255, 237, 269, 268, 281, 260, 249, 257, 260, 266, 281, 266, 250, 227, 261, 263, 246, 227, 264, 269, 256, 247, 248, 255, 268, 246, 279, 252, 264, 236, 256, 278, 236, 241, 248, 245, 233, 242, 245, 227, 269, 255, 241, 253, 264, 280, 240] payload_rows_per_dst=[5396, 5244, 5336, 5442, 5594, 5526, 5746, 5484] dup_rows_per_dst=[2534, 2404, 2418, 2752, 2948, 2842, 2928, 2942] combine_rows_per_src=[5396, 5244, 5336, 5442, 5594, 5526, 5746, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156057600 dispatch_meta_bytes=228680 combine_payload_bytes=156057600 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156286280 ingress_bytes=212680704 link_bytes=212680704 token_bytes=312343880 weight_movement_bytes=56623104 total_nvlink_bytes=368966984 weight_to_token_ratio=0.181284 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037801860e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229160 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156499752 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312770344 weight_movement_bytes=18874368 total_nvlink_bytes=331644712 weight_to_token_ratio=0.060346 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228896 combine_payload_bytes=156467200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156696096 ingress_bytes=231964672 link_bytes=231964672 token_bytes=313163296 weight_movement_bytes=75497472 total_nvlink_bytes=388660768 weight_to_token_ratio=0.241080 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222571353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38432 dup_rows=21620 dup_rows_remote=18910 dedup_ratio=0.329895 router_tokens_per_expert=[254, 271, 243, 259, 286, 252, 259, 239, 245, 249, 281, 267, 241, 254, 248, 253, 262, 251, 263, 289, 249, 260, 234, 238, 253, 254, 225, 251, 250, 229, 256, 254, 280, 276, 258, 271, 257, 266, 268, 251, 282, 277, 268, 266, 274, 264, 232, 258, 275, 242, 253, 232, 263, 289, 279, 276, 242, 264, 239, 245, 276, 260, 248, 240, 236, 291, 232, 275, 260, 240, 282, 255, 246, 249, 251, 250, 278, 256, 271, 256, 221, 266, 248, 252, 263, 271, 286, 278, 260, 245, 240, 249, 252, 244, 282, 240, 249, 252, 247, 240, 242, 222, 277, 252, 244, 245, 217, 269, 272, 232, 275, 237, 251, 248, 274, 256, 264, 249, 250, 235, 261, 256, 252, 232, 271, 267, 266, 249] payload_rows_per_dst=[5484, 5452, 5590, 5556, 5508, 5484, 5390, 5452] dup_rows_per_dst=[2718, 2584, 2906, 2690, 2748, 2710, 2554, 2710] combine_rows_per_src=[5484, 5452, 5590, 5556, 5508, 5484, 5390, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38416 dup_rows=21672 dup_rows_remote=19008 dedup_ratio=0.330688 router_tokens_per_expert=[249, 263, 254, 247, 249, 239, 236, 279, 258, 279, 277, 282, 256, 271, 234, 262, 252, 276, 266, 250, 249, 265, 269, 247, 269, 232, 247, 254, 263, 256, 290, 278, 265, 242, 269, 270, 261, 259, 228, 236, 266, 243, 277, 241, 253, 279, 273, 268, 277, 258, 261, 268, 269, 224, 241, 259, 239, 260, 259, 284, 243, 249, 269, 232, 270, 236, 254, 215, 234, 278, 253, 236, 247, 257, 279, 257, 271, 268, 259, 243, 264, 266, 270, 250, 247, 257, 266, 253, 241, 238, 238, 251, 262, 251, 266, 255, 250, 229, 229, 258, 271, 244, 248, 273, 269, 245, 259, 248, 264, 276, 229, 264, 254, 257, 233, 280, 244, 277, 237, 267, 274, 240, 214, 265, 252, 250, 270, 246] payload_rows_per_dst=[5546, 5510, 5506, 5540, 5462, 5440, 5448, 5412] dup_rows_per_dst=[2724, 2816, 2754, 2644, 2652, 2710, 2664, 2708] combine_rows_per_src=[5546, 5510, 5506, 5540, 5462, 5440, 5448, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228440 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157039704 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313850968 weight_movement_bytes=18874368 total_nvlink_bytes=332725336 weight_to_token_ratio=0.060138 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38370 dup_rows=21760 dup_rows_remote=19106 dedup_ratio=0.332031 router_tokens_per_expert=[289, 286, 255, 274, 259, 253, 240, 269, 242, 281, 267, 242, 267, 266, 258, 266, 274, 266, 253, 251, 241, 286, 268, 249, 270, 241, 252, 250, 253, 270, 267, 294, 257, 230, 238, 276, 274, 257, 254, 246, 271, 247, 264, 220, 256, 239, 250, 258, 240, 271, 244, 246, 267, 250, 258, 247, 268, 254, 266, 245, 240, 231, 256, 247, 260, 245, 269, 252, 238, 242, 240, 269, 245, 218, 244, 249, 274, 280, 264, 269, 243, 245, 226, 256, 265, 283, 248, 275, 259, 281, 246, 253, 260, 252, 236, 240, 256, 246, 269, 276, 268, 242, 270, 248, 252, 262, 227, 259, 250, 293, 256, 258, 247, 231, 277, 237, 243, 247, 247, 237, 261, 268, 245, 271, 257, 261, 260, 255] payload_rows_per_dst=[5566, 5498, 5452, 5406, 5460, 5498, 5478, 5418] dup_rows_per_dst=[2862, 2872, 2622, 2654, 2656, 2638, 2786, 2670] combine_rows_per_src=[5566, 5498, 5452, 5406, 5460, 5498, 5478, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38284 dup_rows=21754 dup_rows_remote=18952 dedup_ratio=0.331940 router_tokens_per_expert=[235, 232, 242, 273, 265, 279, 261, 288, 264, 236, 273, 261, 253, 253, 250, 235, 239, 244, 234, 255, 237, 238, 267, 244, 258, 251, 248, 269, 269, 276, 239, 268, 296, 248, 223, 273, 251, 242, 232, 269, 251, 260, 280, 257, 267, 264, 287, 250, 234, 221, 253, 250, 288, 255, 273, 248, 249, 243, 273, 261, 245, 235, 237, 244, 268, 246, 263, 256, 236, 292, 253, 241, 251, 238, 264, 263, 227, 244, 237, 246, 249, 241, 254, 255, 233, 273, 271, 240, 245, 242, 254, 282, 271, 270, 245, 269, 261, 263, 267, 256, 234, 277, 262, 267, 259, 287, 256, 260, 253, 280, 243, 265, 244, 280, 223, 262, 278, 248, 269, 272, 276, 248, 253, 267, 247, 288, 255, 254] payload_rows_per_dst=[5470, 5416, 5498, 5442, 5344, 5500, 5590, 5522] dup_rows_per_dst=[2730, 2656, 2802, 2576, 2706, 2688, 2790, 2806] combine_rows_per_src=[5470, 5416, 5498, 5442, 5344, 5500, 5590, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157417472 dispatch_meta_bytes=229368 combine_payload_bytes=157417472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157646840 ingress_bytes=176291840 link_bytes=176291840 token_bytes=315064312 weight_movement_bytes=18874368 total_nvlink_bytes=333938680 weight_to_token_ratio=0.059906 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689141670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229696 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157581632 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314933568 weight_movement_bytes=18874368 total_nvlink_bytes=333807936 weight_to_token_ratio=0.059931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38302 dup_rows=21752 dup_rows_remote=19008 dedup_ratio=0.331909 router_tokens_per_expert=[257, 269, 259, 247, 250, 274, 260, 235, 265, 222, 278, 272, 241, 242, 275, 252, 262, 243, 233, 254, 268, 222, 246, 271, 270, 283, 272, 269, 246, 280, 242, 252, 217, 247, 275, 259, 256, 255, 235, 260, 278, 255, 241, 251, 244, 263, 263, 230, 262, 264, 240, 242, 239, 276, 256, 261, 261, 263, 249, 245, 274, 240, 253, 263, 227, 245, 250, 263, 257, 239, 232, 255, 236, 238, 263, 252, 259, 252, 253, 259, 294, 282, 252, 242, 267, 261, 238, 260, 243, 250, 265, 259, 267, 289, 233, 270, 239, 277, 224, 280, 258, 278, 267, 285, 294, 252, 263, 277, 250, 260, 244, 239, 261, 261, 240, 260, 260, 257, 249, 284, 254, 254, 242, 263, 272, 258, 252, 234] payload_rows_per_dst=[5512, 5482, 5412, 5468, 5340, 5532, 5556, 5482] dup_rows_per_dst=[2684, 2744, 2646, 2708, 2620, 2812, 2818, 2720] combine_rows_per_src=[5512, 5482, 5412, 5468, 5340, 5532, 5556, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43692 payload_rows_remote=38254 dup_rows=21844 dup_rows_remote=18990 dedup_ratio=0.333313 router_tokens_per_expert=[267, 265, 247, 232, 236, 233, 257, 238, 276, 247, 235, 248, 277, 282, 240, 218, 273, 268, 250, 245, 234, 274, 248, 282, 285, 248, 221, 277, 256, 260, 241, 243, 264, 266, 243, 256, 261, 268, 236, 239, 257, 255, 257, 245, 282, 284, 231, 252, 264, 245, 266, 234, 259, 248, 254, 249, 259, 268, 246, 269, 275, 278, 274, 263, 245, 260, 272, 281, 243, 266, 265, 288, 280, 252, 270, 261, 224, 239, 253, 247, 224, 245, 262, 255, 239, 268, 243, 282, 272, 239, 255, 268, 255, 262, 269, 223, 250, 264, 263, 272, 238, 256, 256, 265, 232, 259, 255, 229, 259, 256, 251, 263, 239, 242, 271, 286, 233, 276, 262, 275, 268, 247, 271, 258, 255, 230, 260, 270] payload_rows_per_dst=[5350, 5424, 5442, 5574, 5438, 5458, 5460, 5546] dup_rows_per_dst=[2646, 2786, 2750, 2728, 2854, 2664, 2676, 2740] combine_rows_per_src=[5350, 5424, 5442, 5574, 5438, 5458, 5460, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229904 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157393424 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314556944 weight_movement_bytes=37748736 total_nvlink_bytes=352305680 weight_to_token_ratio=0.120006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228944 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157040208 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851472 weight_movement_bytes=18874368 total_nvlink_bytes=332725840 weight_to_token_ratio=0.060138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38254 dup_rows=21696 dup_rows_remote=19024 dedup_ratio=0.331055 router_tokens_per_expert=[266, 265, 272, 240, 271, 256, 247, 264, 253, 235, 251, 275, 270, 240, 247, 265, 255, 235, 253, 263, 234, 255, 252, 271, 247, 266, 252, 209, 233, 244, 246, 256, 259, 266, 276, 255, 265, 266, 253, 249, 267, 234, 276, 264, 242, 261, 251, 254, 244, 252, 279, 262, 282, 243, 263, 255, 259, 237, 248, 249, 252, 233, 293, 262, 230, 259, 276, 260, 251, 261, 257, 280, 277, 263, 237, 258, 256, 241, 270, 262, 273, 292, 252, 239, 299, 247, 248, 265, 269, 267, 253, 246, 256, 257, 268, 272, 234, 256, 244, 260, 279, 269, 242, 261, 265, 232, 240, 251, 265, 302, 255, 250, 245, 252, 235, 243, 248, 254, 265, 232, 245, 235, 257, 242, 264, 242, 276, 248] payload_rows_per_dst=[5586, 5446, 5530, 5556, 5580, 5402, 5402, 5338] dup_rows_per_dst=[2672, 2672, 2836, 2826, 2672, 2582, 2808, 2628] combine_rows_per_src=[5586, 5446, 5530, 5556, 5580, 5402, 5402, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38162 dup_rows=21880 dup_rows_remote=19024 dedup_ratio=0.333862 router_tokens_per_expert=[258, 265, 249, 267, 251, 249, 242, 251, 255, 290, 239, 275, 245, 242, 267, 270, 240, 269, 265, 265, 281, 259, 231, 284, 257, 236, 227, 266, 264, 252, 274, 275, 270, 247, 247, 247, 242, 265, 226, 273, 251, 252, 262, 246, 268, 245, 277, 222, 254, 251, 251, 252, 264, 249, 225, 266, 252, 273, 239, 273, 247, 243, 267, 257, 273, 261, 248, 274, 274, 240, 244, 250, 246, 254, 248, 246, 262, 247, 252, 224, 270, 275, 263, 258, 263, 236, 267, 273, 244, 263, 259, 271, 278, 242, 262, 249, 239, 231, 258, 268, 261, 250, 269, 273, 239, 232, 239, 259, 264, 245, 269, 254, 251, 257, 248, 273, 281, 260, 241, 233, 267, 261, 274, 248, 263, 255, 270, 257] payload_rows_per_dst=[5270, 5388, 5448, 5482, 5652, 5506, 5416, 5494] dup_rows_per_dst=[2528, 2650, 2636, 2644, 2996, 2852, 2718, 2856] combine_rows_per_src=[5270, 5388, 5448, 5482, 5652, 5506, 5416, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229240 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157114232 ingress_bytes=194633728 link_bytes=194633728 token_bytes=313999224 weight_movement_bytes=37748736 total_nvlink_bytes=351747960 weight_to_token_ratio=0.120219 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864884616e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=228976 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156917360 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313605744 weight_movement_bytes=18874368 total_nvlink_bytes=332480112 weight_to_token_ratio=0.060185 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229112 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156917496 ingress_bytes=213311488 link_bytes=213311488 token_bytes=313605880 weight_movement_bytes=56623104 total_nvlink_bytes=370228984 weight_to_token_ratio=0.180555 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043845722e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38246 dup_rows=21802 dup_rows_remote=19106 dedup_ratio=0.332672 router_tokens_per_expert=[261, 261, 250, 245, 268, 240, 253, 259, 256, 287, 267, 238, 273, 273, 266, 262, 284, 290, 260, 274, 268, 261, 251, 260, 261, 275, 232, 262, 269, 250, 238, 269, 232, 271, 252, 260, 267, 248, 256, 256, 274, 261, 246, 278, 265, 262, 231, 244, 243, 250, 253, 247, 266, 288, 256, 220, 247, 280, 253, 242, 245, 261, 240, 268, 253, 259, 236, 232, 247, 260, 264, 236, 267, 229, 267, 223, 239, 279, 245, 258, 263, 245, 242, 249, 258, 251, 265, 228, 266, 248, 274, 281, 270, 270, 257, 232, 245, 253, 258, 251, 260, 246, 272, 265, 237, 248, 271, 249, 279, 251, 266, 241, 215, 261, 276, 226, 274, 253, 265, 244, 265, 258, 260, 228, 256, 241, 278, 258] payload_rows_per_dst=[5468, 5490, 5482, 5452, 5456, 5440, 5488, 5458] dup_rows_per_dst=[2850, 2628, 2724, 2666, 2822, 2758, 2696, 2658] combine_rows_per_src=[5468, 5490, 5482, 5452, 5456, 5440, 5488, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38294 dup_rows=21698 dup_rows_remote=18810 dedup_ratio=0.331085 router_tokens_per_expert=[259, 268, 245, 240, 277, 259, 275, 281, 232, 301, 235, 281, 249, 243, 245, 254, 233, 254, 242, 246, 271, 242, 283, 261, 241, 260, 282, 280, 248, 256, 251, 261, 258, 254, 259, 244, 245, 277, 273, 277, 268, 272, 245, 251, 240, 261, 271, 246, 241, 262, 259, 261, 241, 254, 252, 263, 249, 263, 263, 245, 268, 252, 221, 266, 251, 254, 269, 228, 251, 258, 265, 253, 260, 263, 245, 268, 238, 236, 237, 215, 258, 265, 260, 291, 267, 250, 266, 261, 275, 267, 268, 245, 257, 266, 257, 263, 286, 263, 247, 233, 245, 221, 255, 262, 260, 238, 256, 254, 271, 217, 262, 240, 265, 262, 242, 273, 248, 230, 247, 268, 253, 256, 263, 268, 267, 262, 239, 252] payload_rows_per_dst=[5538, 5488, 5494, 5432, 5446, 5544, 5426, 5470] dup_rows_per_dst=[2750, 2734, 2788, 2688, 2536, 2888, 2594, 2720] combine_rows_per_src=[5538, 5488, 5494, 5432, 5446, 5544, 5426, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228744 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540296 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851848 weight_movement_bytes=18874368 total_nvlink_bytes=331726216 weight_to_token_ratio=0.060330 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38360 dup_rows=21784 dup_rows_remote=19142 dedup_ratio=0.332397 router_tokens_per_expert=[252, 261, 259, 238, 240, 282, 244, 268, 252, 265, 274, 273, 249, 278, 270, 261, 259, 244, 304, 244, 283, 256, 245, 257, 228, 257, 263, 273, 247, 252, 259, 233, 267, 266, 252, 272, 266, 234, 242, 284, 251, 249, 271, 260, 267, 267, 252, 266, 255, 232, 262, 240, 268, 227, 245, 248, 243, 269, 253, 253, 243, 247, 249, 283, 263, 211, 247, 276, 253, 270, 264, 255, 261, 285, 221, 230, 263, 253, 237, 257, 284, 260, 247, 252, 248, 269, 225, 254, 245, 241, 267, 242, 269, 280, 285, 256, 257, 269, 261, 226, 257, 251, 237, 242, 287, 247, 232, 283, 276, 277, 256, 265, 271, 263, 239, 229, 223, 258, 247, 290, 258, 242, 242, 271, 257, 257, 225, 250] payload_rows_per_dst=[5530, 5476, 5600, 5392, 5408, 5476, 5440, 5430] dup_rows_per_dst=[2802, 2732, 2732, 2642, 2684, 2772, 2806, 2614] combine_rows_per_src=[5530, 5476, 5600, 5392, 5408, 5476, 5440, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38198 dup_rows=21930 dup_rows_remote=19296 dedup_ratio=0.334625 router_tokens_per_expert=[219, 270, 249, 278, 224, 245, 258, 251, 263, 262, 243, 258, 270, 272, 267, 241, 275, 275, 259, 282, 235, 274, 244, 251, 253, 233, 271, 241, 257, 256, 263, 251, 252, 222, 253, 284, 265, 264, 218, 235, 255, 250, 256, 266, 246, 247, 249, 259, 261, 260, 257, 256, 260, 263, 257, 263, 266, 243, 254, 225, 241, 246, 258, 241, 259, 247, 270, 267, 251, 236, 269, 274, 258, 250, 241, 255, 294, 259, 222, 242, 259, 265, 261, 236, 263, 248, 268, 239, 255, 264, 250, 273, 253, 231, 241, 243, 258, 250, 265, 265, 242, 274, 301, 247, 270, 260, 258, 273, 273, 278, 258, 285, 271, 264, 283, 289, 257, 251, 273, 263, 237, 228, 233, 246, 235, 251, 268, 257] payload_rows_per_dst=[5350, 5468, 5408, 5368, 5470, 5496, 5554, 5492] dup_rows_per_dst=[2790, 2772, 2634, 2734, 2718, 2602, 2960, 2720] combine_rows_per_src=[5350, 5468, 5408, 5368, 5470, 5496, 5554, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229408 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885024 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540640 weight_movement_bytes=18874368 total_nvlink_bytes=332415008 weight_to_token_ratio=0.060198 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228416 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157080640 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313932864 weight_movement_bytes=37748736 total_nvlink_bytes=351681600 weight_to_token_ratio=0.120245 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38282 dup_rows=21752 dup_rows_remote=19072 dedup_ratio=0.331909 router_tokens_per_expert=[232, 269, 265, 264, 257, 267, 269, 289, 256, 236, 251, 227, 251, 236, 260, 244, 256, 244, 274, 244, 282, 262, 244, 255, 263, 246, 232, 252, 273, 241, 258, 265, 275, 242, 244, 277, 254, 240, 246, 257, 262, 252, 264, 261, 244, 251, 263, 240, 234, 282, 263, 271, 267, 250, 265, 269, 252, 259, 254, 288, 249, 265, 244, 265, 267, 265, 254, 251, 252, 259, 263, 250, 263, 223, 246, 278, 253, 264, 258, 232, 240, 264, 253, 262, 250, 269, 217, 278, 250, 241, 249, 264, 239, 247, 269, 248, 225, 260, 263, 254, 272, 238, 255, 247, 254, 232, 262, 256, 257, 271, 273, 234, 282, 261, 257, 297, 253, 260, 268, 254, 258, 244, 247, 263, 244, 266, 273, 257] payload_rows_per_dst=[5404, 5502, 5492, 5528, 5432, 5386, 5448, 5592] dup_rows_per_dst=[2742, 2680, 2652, 2826, 2724, 2694, 2658, 2776] combine_rows_per_src=[5404, 5502, 5492, 5528, 5432, 5386, 5448, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38266 dup_rows=21792 dup_rows_remote=19006 dedup_ratio=0.332520 router_tokens_per_expert=[271, 253, 270, 236, 261, 213, 268, 249, 257, 245, 265, 260, 289, 260, 235, 270, 258, 276, 225, 259, 263, 260, 273, 264, 273, 248, 290, 251, 255, 248, 241, 269, 238, 259, 270, 266, 258, 246, 264, 246, 249, 243, 265, 253, 235, 267, 222, 248, 249, 283, 281, 272, 216, 269, 256, 296, 256, 254, 241, 262, 264, 241, 239, 246, 269, 266, 276, 251, 252, 273, 254, 253, 283, 264, 213, 268, 257, 225, 271, 257, 281, 275, 277, 246, 227, 272, 276, 228, 270, 236, 255, 276, 247, 236, 229, 290, 224, 260, 245, 241, 263, 250, 237, 253, 231, 278, 274, 257, 256, 253, 255, 277, 265, 231, 220, 261, 288, 296, 251, 257, 248, 248, 255, 245, 253, 216, 258, 260] payload_rows_per_dst=[5504, 5536, 5432, 5462, 5478, 5534, 5440, 5358] dup_rows_per_dst=[2700, 2770, 2626, 2788, 2786, 2708, 2668, 2746] combine_rows_per_src=[5504, 5536, 5432, 5462, 5478, 5534, 5440, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=230008 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157352568 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314475128 weight_movement_bytes=18874368 total_nvlink_bytes=333349496 weight_to_token_ratio=0.060019 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229976 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688984 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147992 weight_movement_bytes=18874368 total_nvlink_bytes=332022360 weight_to_token_ratio=0.060273 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38300 dup_rows=21674 dup_rows_remote=18990 dedup_ratio=0.330719 router_tokens_per_expert=[247, 244, 243, 266, 289, 235, 268, 260, 281, 258, 240, 247, 255, 259, 275, 240, 243, 244, 283, 258, 239, 268, 263, 256, 250, 252, 240, 254, 249, 242, 247, 261, 237, 245, 252, 238, 251, 264, 273, 267, 255, 259, 250, 284, 251, 240, 287, 293, 254, 259, 263, 251, 292, 249, 257, 240, 257, 246, 265, 231, 257, 233, 259, 248, 261, 254, 263, 238, 252, 270, 242, 238, 263, 275, 258, 242, 285, 268, 228, 233, 248, 244, 264, 281, 277, 269, 241, 259, 263, 263, 256, 277, 266, 253, 239, 259, 268, 264, 222, 273, 274, 240, 256, 205, 257, 238, 254, 254, 294, 258, 269, 248, 230, 261, 257, 260, 248, 231, 268, 249, 265, 258, 254, 254, 265, 253, 268, 281] payload_rows_per_dst=[5562, 5416, 5546, 5414, 5390, 5552, 5500, 5482] dup_rows_per_dst=[2684, 2710, 2732, 2708, 2750, 2766, 2648, 2676] combine_rows_per_src=[5562, 5416, 5546, 5414, 5390, 5552, 5500, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38202 dup_rows=21908 dup_rows_remote=19234 dedup_ratio=0.334290 router_tokens_per_expert=[216, 264, 262, 252, 261, 271, 286, 263, 251, 261, 264, 233, 253, 260, 284, 257, 238, 258, 266, 251, 277, 248, 258, 263, 243, 251, 248, 254, 262, 277, 272, 253, 258, 265, 285, 264, 269, 235, 250, 247, 272, 261, 253, 240, 265, 257, 227, 249, 261, 282, 233, 241, 263, 248, 258, 263, 256, 242, 262, 253, 267, 268, 254, 257, 265, 263, 238, 269, 267, 237, 241, 252, 251, 228, 231, 234, 273, 273, 250, 263, 244, 279, 279, 265, 255, 265, 274, 260, 276, 235, 243, 259, 262, 278, 276, 241, 250, 268, 255, 228, 254, 263, 231, 250, 236, 255, 245, 239, 242, 285, 273, 265, 263, 230, 259, 254, 250, 231, 264, 295, 236, 227, 268, 262, 251, 250, 270, 231] payload_rows_per_dst=[5218, 5332, 5624, 5418, 5574, 5576, 5460, 5426] dup_rows_per_dst=[2486, 2468, 3008, 2660, 3068, 2884, 2660, 2674] combine_rows_per_src=[5218, 5332, 5624, 5418, 5574, 5576, 5460, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229416 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032488 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835560 weight_movement_bytes=37748736 total_nvlink_bytes=351584296 weight_to_token_ratio=0.120282 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229088 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966624 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704160 weight_movement_bytes=18874368 total_nvlink_bytes=332578528 weight_to_token_ratio=0.060166 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229160 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157105960 ingress_bytes=232374272 link_bytes=232374272 token_bytes=313982760 weight_movement_bytes=75497472 total_nvlink_bytes=389480232 weight_to_token_ratio=0.240451 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226495939e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38172 dup_rows=21816 dup_rows_remote=18980 dedup_ratio=0.332886 router_tokens_per_expert=[267, 288, 269, 255, 283, 246, 268, 251, 267, 245, 244, 240, 241, 237, 240, 260, 257, 266, 256, 238, 264, 252, 250, 243, 247, 259, 291, 252, 238, 257, 246, 251, 258, 274, 243, 269, 232, 252, 271, 252, 237, 251, 244, 273, 256, 253, 245, 285, 252, 239, 280, 267, 259, 218, 241, 229, 265, 272, 237, 261, 292, 252, 225, 274, 232, 249, 197, 259, 240, 276, 259, 239, 272, 238, 232, 248, 263, 289, 253, 246, 245, 268, 267, 279, 245, 239, 265, 264, 270, 269, 252, 266, 244, 266, 238, 269, 275, 241, 259, 273, 252, 241, 273, 267, 257, 288, 260, 232, 298, 264, 238, 274, 228, 255, 267, 270, 266, 271, 243, 232, 263, 255, 279, 263, 249, 269, 256, 246] payload_rows_per_dst=[5472, 5434, 5446, 5508, 5378, 5446, 5548, 5488] dup_rows_per_dst=[2634, 2700, 2744, 2618, 2702, 2846, 2836, 2736] combine_rows_per_src=[5472, 5434, 5446, 5508, 5378, 5446, 5548, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43510 payload_rows_remote=38086 dup_rows=22026 dup_rows_remote=19326 dedup_ratio=0.336090 router_tokens_per_expert=[257, 263, 253, 243, 267, 244, 276, 237, 261, 259, 289, 251, 275, 250, 235, 238, 257, 260, 260, 272, 246, 259, 242, 259, 268, 244, 261, 279, 261, 244, 276, 251, 284, 267, 266, 265, 240, 233, 225, 257, 283, 257, 252, 252, 228, 279, 230, 257, 278, 254, 263, 253, 276, 260, 239, 245, 258, 237, 242, 257, 246, 242, 261, 242, 235, 254, 258, 249, 246, 274, 273, 250, 262, 258, 264, 255, 249, 267, 260, 273, 248, 262, 255, 259, 268, 244, 272, 257, 255, 255, 251, 243, 243, 245, 268, 237, 257, 279, 257, 246, 265, 268, 273, 256, 242, 246, 265, 250, 257, 251, 262, 249, 235, 270, 253, 274, 234, 230, 259, 270, 237, 256, 265, 279, 240, 286, 239, 264] payload_rows_per_dst=[5374, 5480, 5392, 5444, 5540, 5424, 5426, 5430] dup_rows_per_dst=[2822, 2798, 2758, 2662, 2714, 2700, 2820, 2752] combine_rows_per_src=[5374, 5480, 5392, 5444, 5540, 5424, 5426, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229744 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156705136 ingress_bytes=194224128 link_bytes=194224128 token_bytes=313180528 weight_movement_bytes=37748736 total_nvlink_bytes=350929264 weight_to_token_ratio=0.120533 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860960030e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38272 dup_rows=21740 dup_rows_remote=19012 dedup_ratio=0.331726 router_tokens_per_expert=[245, 244, 264, 248, 246, 260, 239, 254, 269, 221, 290, 270, 295, 232, 292, 227, 243, 226, 268, 239, 267, 254, 277, 279, 230, 243, 272, 283, 276, 264, 269, 258, 267, 248, 230, 257, 265, 259, 258, 271, 237, 244, 261, 262, 255, 258, 284, 270, 255, 262, 258, 246, 220, 267, 257, 244, 242, 231, 245, 267, 260, 219, 238, 267, 248, 293, 252, 229, 262, 262, 242, 258, 274, 243, 254, 251, 244, 241, 253, 286, 242, 250, 242, 248, 261, 237, 239, 252, 254, 245, 239, 276, 264, 268, 256, 266, 260, 252, 273, 270, 238, 244, 258, 284, 249, 269, 243, 273, 268, 269, 249, 235, 257, 277, 276, 257, 261, 244, 267, 241, 249, 272, 255, 260, 273, 252, 259, 255] payload_rows_per_dst=[5390, 5574, 5524, 5394, 5460, 5368, 5552, 5534] dup_rows_per_dst=[2802, 2722, 2728, 2562, 2724, 2710, 2716, 2776] combine_rows_per_src=[5390, 5574, 5524, 5394, 5460, 5368, 5552, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38384 dup_rows=21676 dup_rows_remote=18852 dedup_ratio=0.330750 router_tokens_per_expert=[250, 274, 267, 290, 276, 244, 263, 269, 250, 286, 269, 266, 257, 285, 281, 236, 263, 269, 254, 278, 250, 246, 259, 246, 281, 252, 276, 235, 252, 246, 267, 248, 256, 242, 241, 253, 237, 235, 241, 250, 284, 213, 244, 268, 262, 242, 249, 264, 257, 263, 266, 252, 266, 244, 254, 285, 259, 281, 252, 247, 248, 273, 251, 252, 226, 202, 259, 245, 261, 244, 251, 263, 249, 247, 254, 272, 237, 244, 253, 255, 247, 257, 246, 252, 247, 268, 265, 285, 252, 249, 251, 239, 260, 240, 222, 261, 263, 260, 249, 260, 265, 280, 247, 249, 256, 208, 259, 251, 267, 258, 236, 254, 267, 245, 256, 266, 255, 255, 269, 265, 238, 257, 268, 246, 269, 259, 311, 261] payload_rows_per_dst=[5602, 5534, 5414, 5476, 5342, 5444, 5452, 5596] dup_rows_per_dst=[2924, 2710, 2548, 2824, 2582, 2638, 2672, 2778] combine_rows_per_src=[5602, 5534, 5414, 5476, 5342, 5444, 5452, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228608 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156581120 ingress_bytes=194101248 link_bytes=194101248 token_bytes=312933632 weight_movement_bytes=37748736 total_nvlink_bytes=350682368 weight_to_token_ratio=0.120629 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859782655e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156000256 dispatch_meta_bytes=229648 combine_payload_bytes=156000256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156229904 ingress_bytes=174874624 link_bytes=174874624 token_bytes=312230160 weight_movement_bytes=18874368 total_nvlink_bytes=331104528 weight_to_token_ratio=0.060450 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675562603e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229136 combine_payload_bytes=156762112 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156991248 ingress_bytes=175636480 link_bytes=175636480 token_bytes=313753360 weight_movement_bytes=18874368 total_nvlink_bytes=332627728 weight_to_token_ratio=0.060157 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682862333e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=228944 combine_payload_bytes=157220864 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157449808 ingress_bytes=213843968 link_bytes=213843968 token_bytes=314670672 weight_movement_bytes=56623104 total_nvlink_bytes=371293776 weight_to_token_ratio=0.179944 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048947684e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38112 dup_rows=21996 dup_rows_remote=19334 dedup_ratio=0.335632 router_tokens_per_expert=[268, 214, 251, 260, 276, 259, 240, 251, 267, 275, 244, 266, 290, 257, 235, 252, 265, 247, 237, 251, 272, 272, 254, 249, 255, 235, 241, 284, 251, 242, 252, 238, 253, 222, 282, 241, 262, 283, 253, 251, 246, 250, 241, 275, 274, 244, 259, 238, 289, 253, 239, 247, 275, 277, 251, 256, 257, 246, 255, 251, 267, 268, 263, 250, 246, 270, 273, 275, 270, 275, 255, 277, 255, 261, 205, 262, 262, 255, 254, 250, 238, 257, 219, 261, 260, 230, 255, 250, 250, 276, 243, 268, 252, 272, 254, 275, 236, 260, 237, 262, 262, 249, 267, 276, 285, 254, 268, 246, 267, 269, 247, 249, 264, 258, 243, 247, 248, 258, 263, 241, 247, 255, 302, 229, 272, 244, 236, 254] payload_rows_per_dst=[5476, 5428, 5440, 5494, 5422, 5402, 5418, 5460] dup_rows_per_dst=[2734, 2662, 2708, 2828, 2868, 2718, 2816, 2662] combine_rows_per_src=[5476, 5428, 5440, 5494, 5422, 5402, 5418, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38164 dup_rows=21908 dup_rows_remote=19064 dedup_ratio=0.334290 router_tokens_per_expert=[261, 239, 267, 245, 241, 263, 291, 254, 227, 250, 274, 239, 251, 288, 265, 268, 249, 282, 255, 239, 244, 260, 282, 248, 225, 253, 249, 255, 246, 250, 267, 265, 219, 245, 266, 231, 255, 254, 237, 272, 281, 236, 281, 284, 265, 233, 246, 269, 275, 263, 265, 275, 248, 271, 248, 277, 262, 243, 230, 257, 274, 207, 221, 267, 265, 275, 245, 270, 263, 259, 243, 251, 252, 268, 280, 224, 228, 271, 275, 285, 273, 261, 250, 263, 243, 266, 258, 264, 269, 275, 279, 274, 245, 239, 249, 231, 224, 251, 261, 256, 246, 244, 251, 251, 278, 247, 236, 247, 261, 258, 272, 264, 262, 235, 270, 223, 270, 272, 260, 277, 248, 242, 261, 234, 236, 242, 277, 270] payload_rows_per_dst=[5492, 5436, 5416, 5488, 5464, 5512, 5396, 5424] dup_rows_per_dst=[2754, 2702, 2732, 2678, 2844, 2766, 2698, 2734] combine_rows_per_src=[5492, 5436, 5416, 5488, 5464, 5512, 5396, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43524 payload_rows_remote=37974 dup_rows=22012 dup_rows_remote=19226 dedup_ratio=0.335876 router_tokens_per_expert=[270, 248, 241, 245, 268, 261, 262, 273, 245, 268, 261, 264, 264, 245, 258, 277, 250, 247, 259, 256, 263, 306, 268, 267, 258, 247, 267, 246, 256, 262, 265, 223, 237, 266, 247, 267, 263, 249, 262, 254, 263, 268, 269, 276, 251, 265, 254, 250, 254, 270, 240, 262, 258, 256, 260, 259, 241, 238, 262, 255, 248, 245, 263, 254, 265, 239, 260, 273, 249, 270, 258, 274, 253, 239, 244, 276, 259, 264, 243, 266, 226, 236, 255, 237, 252, 256, 255, 235, 247, 264, 278, 257, 264, 249, 258, 250, 250, 263, 262, 275, 250, 266, 238, 249, 252, 241, 244, 235, 268, 219, 262, 252, 221, 267, 246, 257, 254, 253, 240, 252, 283, 271, 251, 264, 269, 278, 235, 254] payload_rows_per_dst=[5550, 5444, 5438, 5404, 5488, 5370, 5446, 5384] dup_rows_per_dst=[2786, 2848, 2844, 2678, 2776, 2668, 2606, 2806] combine_rows_per_src=[5550, 5444, 5438, 5404, 5488, 5370, 5446, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38182 dup_rows=21674 dup_rows_remote=18754 dedup_ratio=0.330719 router_tokens_per_expert=[240, 253, 238, 251, 288, 237, 247, 255, 291, 269, 271, 267, 268, 263, 259, 279, 254, 266, 260, 269, 277, 208, 238, 261, 251, 233, 254, 286, 246, 236, 250, 270, 250, 257, 243, 237, 246, 247, 246, 258, 291, 278, 248, 250, 240, 241, 274, 266, 282, 238, 233, 263, 246, 234, 248, 248, 260, 261, 283, 240, 257, 261, 225, 286, 235, 272, 243, 263, 273, 260, 249, 275, 257, 264, 238, 243, 235, 260, 260, 253, 272, 265, 241, 250, 263, 274, 254, 267, 259, 263, 254, 266, 257, 259, 252, 235, 245, 250, 271, 251, 231, 275, 267, 260, 266, 262, 260, 258, 221, 296, 257, 249, 260, 263, 251, 231, 269, 245, 235, 228, 275, 238, 272, 280, 262, 243, 262, 252] payload_rows_per_dst=[5344, 5410, 5506, 5454, 5490, 5452, 5526, 5680] dup_rows_per_dst=[2540, 2612, 2706, 2676, 2680, 2664, 2876, 2920] combine_rows_per_src=[5344, 5410, 5506, 5454, 5490, 5452, 5526, 5680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=229784 combine_payload_bytes=156106752 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156336536 ingress_bytes=212729856 link_bytes=212729856 token_bytes=312443288 weight_movement_bytes=56623104 total_nvlink_bytes=369066392 weight_to_token_ratio=0.181227 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038272811e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228912 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548656 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868400 weight_movement_bytes=18874368 total_nvlink_bytes=331742768 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155541504 dispatch_meta_bytes=228800 combine_payload_bytes=155541504 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155770304 ingress_bytes=212164608 link_bytes=212164608 token_bytes=311311808 weight_movement_bytes=56623104 total_nvlink_bytes=367934912 weight_to_token_ratio=0.181886 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.032856882e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38378 dup_rows=21620 dup_rows_remote=18840 dedup_ratio=0.329895 router_tokens_per_expert=[241, 249, 267, 241, 254, 246, 257, 245, 243, 281, 234, 213, 248, 296, 273, 282, 246, 238, 251, 274, 260, 256, 254, 254, 283, 267, 238, 270, 269, 253, 262, 263, 251, 255, 273, 243, 263, 244, 264, 250, 252, 265, 235, 257, 277, 255, 268, 245, 247, 277, 230, 274, 274, 252, 251, 289, 283, 244, 240, 253, 265, 259, 261, 243, 226, 232, 284, 265, 252, 241, 271, 265, 273, 265, 254, 232, 272, 263, 282, 267, 246, 252, 260, 264, 282, 256, 266, 238, 255, 257, 264, 212, 257, 272, 232, 245, 291, 259, 228, 239, 244, 262, 272, 274, 237, 264, 251, 264, 283, 246, 271, 274, 258, 206, 266, 244, 254, 246, 259, 244, 268, 256, 240, 245, 243, 260, 238, 233] payload_rows_per_dst=[5454, 5558, 5530, 5532, 5440, 5448, 5538, 5416] dup_rows_per_dst=[2686, 2718, 2664, 2752, 2848, 2668, 2780, 2504] combine_rows_per_src=[5454, 5558, 5530, 5532, 5440, 5448, 5538, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38192 dup_rows=21718 dup_rows_remote=18964 dedup_ratio=0.331390 router_tokens_per_expert=[229, 264, 259, 240, 243, 242, 286, 270, 282, 264, 246, 247, 249, 283, 249, 259, 250, 260, 253, 233, 253, 286, 280, 229, 240, 246, 253, 264, 259, 257, 261, 285, 266, 261, 271, 248, 266, 265, 245, 263, 266, 252, 263, 272, 237, 246, 231, 265, 251, 255, 234, 262, 260, 264, 238, 257, 232, 232, 256, 240, 241, 260, 241, 255, 247, 258, 230, 267, 230, 265, 243, 240, 268, 263, 271, 253, 268, 244, 261, 245, 269, 261, 281, 276, 248, 247, 267, 288, 270, 284, 265, 226, 261, 252, 253, 242, 251, 225, 239, 272, 252, 261, 270, 265, 284, 223, 240, 270, 253, 283, 259, 262, 275, 259, 263, 250, 246, 266, 288, 234, 262, 245, 245, 255, 248, 241, 269, 254] payload_rows_per_dst=[5472, 5418, 5544, 5318, 5466, 5626, 5488, 5486] dup_rows_per_dst=[2752, 2800, 2690, 2638, 2640, 2754, 2730, 2714] combine_rows_per_src=[5472, 5418, 5544, 5318, 5466, 5626, 5488, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=227744 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156621216 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313014688 weight_movement_bytes=18874368 total_nvlink_bytes=331889056 weight_to_token_ratio=0.060299 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38324 dup_rows=21736 dup_rows_remote=19056 dedup_ratio=0.331665 router_tokens_per_expert=[265, 254, 264, 277, 261, 244, 248, 248, 257, 263, 256, 254, 262, 240, 248, 262, 246, 270, 238, 252, 242, 265, 246, 266, 246, 256, 240, 273, 257, 256, 253, 243, 254, 252, 286, 250, 277, 227, 257, 272, 240, 285, 247, 256, 251, 260, 213, 251, 258, 259, 263, 257, 263, 256, 267, 264, 230, 268, 259, 248, 256, 254, 242, 255, 252, 261, 264, 257, 251, 278, 253, 298, 269, 238, 256, 256, 252, 260, 233, 243, 259, 253, 258, 266, 233, 275, 247, 255, 250, 241, 250, 260, 224, 264, 232, 267, 236, 288, 248, 271, 256, 268, 243, 235, 260, 296, 245, 272, 255, 244, 254, 288, 278, 254, 249, 254, 269, 245, 266, 238, 240, 250, 256, 257, 260, 259, 281, 269] payload_rows_per_dst=[5448, 5456, 5476, 5550, 5488, 5374, 5570, 5438] dup_rows_per_dst=[2758, 2642, 2680, 2648, 2754, 2694, 2748, 2812] combine_rows_per_src=[5448, 5456, 5476, 5550, 5488, 5374, 5570, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38190 dup_rows=21832 dup_rows_remote=19128 dedup_ratio=0.333130 router_tokens_per_expert=[245, 268, 247, 260, 260, 243, 260, 284, 268, 240, 255, 244, 250, 257, 271, 258, 250, 250, 258, 244, 269, 255, 271, 239, 272, 270, 237, 252, 288, 253, 242, 237, 271, 237, 244, 262, 236, 248, 260, 282, 271, 275, 234, 252, 249, 251, 234, 252, 256, 251, 261, 262, 256, 250, 266, 265, 253, 241, 239, 260, 265, 265, 269, 250, 239, 268, 269, 266, 242, 252, 251, 243, 289, 248, 249, 276, 260, 263, 244, 242, 259, 251, 254, 234, 238, 267, 272, 253, 227, 256, 288, 258, 270, 252, 258, 247, 269, 247, 247, 285, 251, 276, 281, 263, 237, 235, 271, 265, 250, 262, 261, 249, 279, 273, 220, 247, 268, 271, 255, 248, 282, 267, 252, 256, 234, 232, 228, 258] payload_rows_per_dst=[5412, 5442, 5462, 5514, 5486, 5430, 5554, 5404] dup_rows_per_dst=[2808, 2732, 2654, 2704, 2716, 2738, 2744, 2736] combine_rows_per_src=[5412, 5442, 5462, 5514, 5486, 5430, 5554, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=228872 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157425160 ingress_bytes=194945024 link_bytes=194945024 token_bytes=314621448 weight_movement_bytes=37748736 total_nvlink_bytes=352370184 weight_to_token_ratio=0.119981 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867867301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228624 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156663056 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313097488 weight_movement_bytes=37748736 total_nvlink_bytes=350846224 weight_to_token_ratio=0.120565 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229520 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204624 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179728 weight_movement_bytes=18874368 total_nvlink_bytes=333054096 weight_to_token_ratio=0.060075 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229272 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655512 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081752 weight_movement_bytes=18874368 total_nvlink_bytes=331956120 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38152 dup_rows=21908 dup_rows_remote=19072 dedup_ratio=0.334290 router_tokens_per_expert=[253, 261, 267, 264, 263, 272, 256, 261, 237, 258, 253, 271, 254, 243, 247, 250, 224, 230, 261, 286, 258, 279, 256, 302, 265, 254, 240, 264, 247, 270, 270, 250, 258, 261, 269, 261, 248, 264, 289, 245, 278, 250, 254, 250, 266, 220, 250, 269, 239, 233, 244, 213, 250, 276, 267, 263, 241, 238, 262, 237, 246, 279, 252, 239, 271, 277, 260, 232, 255, 249, 250, 256, 264, 253, 232, 269, 255, 242, 238, 287, 272, 242, 240, 262, 259, 263, 241, 272, 266, 259, 286, 238, 258, 273, 246, 256, 254, 242, 284, 254, 250, 243, 253, 263, 273, 248, 254, 243, 228, 245, 240, 246, 283, 251, 239, 231, 261, 247, 252, 244, 255, 266, 277, 264, 251, 287, 257, 283] payload_rows_per_dst=[5450, 5476, 5534, 5348, 5442, 5492, 5336, 5550] dup_rows_per_dst=[2770, 2836, 2730, 2610, 2738, 2774, 2704, 2746] combine_rows_per_src=[5450, 5476, 5534, 5348, 5442, 5492, 5336, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38186 dup_rows=21860 dup_rows_remote=19088 dedup_ratio=0.333557 router_tokens_per_expert=[241, 255, 245, 274, 264, 252, 254, 274, 258, 260, 294, 261, 240, 236, 239, 274, 301, 270, 264, 269, 246, 256, 259, 277, 230, 248, 263, 230, 237, 274, 256, 248, 260, 233, 261, 250, 235, 277, 234, 274, 248, 265, 289, 265, 254, 242, 255, 262, 257, 245, 260, 268, 236, 260, 267, 273, 252, 272, 285, 260, 239, 243, 250, 262, 268, 254, 271, 255, 240, 260, 281, 259, 242, 259, 232, 249, 252, 245, 266, 298, 244, 269, 272, 254, 256, 257, 244, 252, 233, 268, 251, 263, 252, 236, 269, 255, 253, 246, 256, 255, 252, 256, 222, 241, 265, 270, 249, 249, 258, 273, 244, 246, 263, 247, 260, 251, 252, 266, 265, 240, 273, 275, 257, 230, 241, 240, 255, 230] payload_rows_per_dst=[5550, 5478, 5444, 5460, 5490, 5486, 5438, 5330] dup_rows_per_dst=[2692, 2778, 2764, 2798, 2772, 2664, 2632, 2760] combine_rows_per_src=[5550, 5478, 5444, 5460, 5490, 5486, 5438, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38342 dup_rows=21718 dup_rows_remote=18994 dedup_ratio=0.331390 router_tokens_per_expert=[257, 267, 259, 233, 252, 263, 261, 273, 225, 280, 255, 272, 272, 249, 262, 220, 242, 263, 262, 275, 270, 261, 287, 291, 247, 256, 276, 239, 251, 243, 268, 242, 263, 266, 244, 260, 240, 258, 261, 284, 251, 249, 240, 244, 256, 249, 269, 274, 264, 245, 251, 269, 259, 245, 270, 258, 250, 266, 233, 278, 260, 240, 288, 237, 272, 224, 254, 262, 235, 259, 267, 257, 245, 280, 283, 277, 269, 256, 252, 239, 252, 257, 249, 263, 239, 261, 258, 253, 237, 263, 246, 254, 246, 263, 243, 213, 243, 266, 261, 241, 233, 256, 235, 271, 253, 259, 250, 248, 246, 267, 261, 255, 248, 233, 246, 254, 239, 250, 247, 282, 285, 273, 267, 262, 252, 265, 243, 255] payload_rows_per_dst=[5476, 5604, 5490, 5660, 5568, 5212, 5378, 5430] dup_rows_per_dst=[2724, 2742, 2726, 3034, 2694, 2314, 2712, 2772] combine_rows_per_src=[5476, 5604, 5490, 5660, 5568, 5212, 5378, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38300 dup_rows=21848 dup_rows_remote=19224 dedup_ratio=0.333374 router_tokens_per_expert=[278, 257, 265, 261, 281, 243, 267, 258, 245, 264, 269, 264, 257, 280, 252, 244, 295, 248, 239, 251, 242, 274, 258, 265, 262, 290, 264, 291, 231, 255, 235, 255, 252, 238, 252, 248, 267, 265, 252, 265, 226, 252, 266, 258, 253, 276, 254, 245, 251, 248, 244, 257, 255, 274, 251, 228, 246, 251, 253, 229, 275, 252, 248, 272, 251, 247, 271, 242, 261, 248, 253, 280, 262, 251, 245, 247, 259, 237, 270, 279, 267, 259, 255, 243, 262, 250, 262, 261, 242, 253, 255, 275, 258, 253, 230, 267, 267, 276, 245, 242, 262, 261, 251, 236, 250, 277, 256, 274, 281, 245, 259, 258, 213, 244, 249, 222, 223, 258, 265, 242, 271, 276, 248, 256, 280, 249, 232, 262] payload_rows_per_dst=[5488, 5504, 5430, 5408, 5426, 5504, 5540, 5388] dup_rows_per_dst=[2864, 2722, 2708, 2660, 2720, 2740, 2810, 2624] combine_rows_per_src=[5488, 5504, 5430, 5408, 5426, 5504, 5540, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=228896 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156499488 ingress_bytes=212893696 link_bytes=212893696 token_bytes=312770080 weight_movement_bytes=56623104 total_nvlink_bytes=369393184 weight_to_token_ratio=0.181037 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039842645e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229096 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638952 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048808 weight_movement_bytes=37748736 total_nvlink_bytes=350797544 weight_to_token_ratio=0.120584 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229344 combine_payload_bytes=157048832 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=157278176 ingress_bytes=251420672 link_bytes=251420672 token_bytes=314327008 weight_movement_bytes=94371840 total_nvlink_bytes=408698848 weight_to_token_ratio=0.300235 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.408989172e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43546 payload_rows_remote=38192 dup_rows=21990 dup_rows_remote=19212 dedup_ratio=0.335541 router_tokens_per_expert=[273, 266, 230, 254, 228, 268, 267, 269, 244, 257, 246, 272, 242, 267, 256, 242, 275, 243, 261, 202, 260, 262, 259, 262, 238, 244, 242, 249, 245, 260, 262, 228, 248, 245, 256, 219, 242, 258, 262, 250, 278, 268, 261, 283, 252, 256, 262, 267, 244, 279, 282, 247, 245, 305, 251, 251, 268, 262, 258, 266, 233, 268, 252, 237, 262, 246, 286, 258, 263, 265, 284, 260, 241, 258, 262, 248, 272, 235, 251, 248, 239, 270, 263, 278, 239, 247, 252, 243, 260, 265, 260, 288, 257, 251, 245, 253, 245, 273, 249, 270, 252, 241, 256, 266, 262, 249, 259, 251, 244, 246, 258, 245, 276, 248, 287, 261, 218, 276, 281, 259, 256, 277, 242, 256, 249, 246, 256, 237] payload_rows_per_dst=[5452, 5382, 5448, 5490, 5470, 5490, 5354, 5460] dup_rows_per_dst=[2710, 2602, 2766, 2806, 2808, 2730, 2778, 2790] combine_rows_per_src=[5452, 5382, 5448, 5490, 5470, 5490, 5354, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38244 dup_rows=21832 dup_rows_remote=19036 dedup_ratio=0.333130 router_tokens_per_expert=[257, 263, 248, 225, 263, 254, 235, 284, 222, 285, 251, 257, 238, 277, 249, 291, 249, 233, 293, 239, 256, 281, 272, 245, 252, 271, 241, 263, 261, 258, 253, 251, 255, 252, 235, 250, 262, 266, 246, 243, 240, 271, 225, 274, 250, 258, 254, 264, 264, 226, 228, 253, 261, 250, 279, 291, 241, 228, 251, 246, 248, 257, 251, 273, 250, 237, 272, 265, 269, 258, 262, 245, 255, 250, 255, 265, 260, 253, 236, 275, 276, 235, 234, 247, 283, 253, 266, 266, 271, 272, 259, 236, 252, 246, 261, 271, 256, 261, 235, 236, 274, 275, 262, 254, 303, 260, 237, 248, 251, 265, 282, 245, 237, 281, 278, 232, 254, 245, 266, 287, 249, 252, 255, 249, 257, 214, 261, 263] payload_rows_per_dst=[5480, 5484, 5426, 5468, 5502, 5460, 5520, 5364] dup_rows_per_dst=[2718, 2752, 2664, 2626, 2712, 2796, 2768, 2796] combine_rows_per_src=[5480, 5484, 5426, 5468, 5502, 5460, 5520, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38294 dup_rows=21714 dup_rows_remote=18992 dedup_ratio=0.331329 router_tokens_per_expert=[254, 227, 249, 245, 255, 259, 253, 290, 255, 248, 258, 274, 234, 280, 238, 240, 267, 268, 234, 226, 265, 235, 243, 276, 254, 253, 265, 271, 271, 271, 251, 245, 265, 276, 285, 271, 267, 262, 226, 264, 251, 264, 234, 267, 265, 210, 246, 272, 243, 266, 243, 261, 233, 263, 263, 268, 244, 271, 228, 247, 267, 239, 265, 265, 226, 289, 264, 251, 254, 225, 251, 252, 229, 258, 233, 248, 258, 250, 239, 269, 224, 266, 289, 254, 260, 290, 236, 245, 272, 236, 307, 280, 252, 282, 239, 245, 281, 252, 256, 259, 247, 262, 241, 270, 268, 242, 230, 268, 247, 266, 271, 236, 301, 249, 274, 262, 257, 272, 260, 236, 241, 244, 234, 251, 285, 279, 237, 272] payload_rows_per_dst=[5406, 5472, 5528, 5516, 5404, 5526, 5432, 5538] dup_rows_per_dst=[2712, 2718, 2722, 2616, 2588, 2828, 2760, 2770] combine_rows_per_src=[5406, 5472, 5528, 5516, 5404, 5526, 5432, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=230096 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157106896 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313983696 weight_movement_bytes=37748736 total_nvlink_bytes=351732432 weight_to_token_ratio=0.120225 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38146 dup_rows=21912 dup_rows_remote=19028 dedup_ratio=0.334351 router_tokens_per_expert=[226, 241, 262, 259, 269, 262, 249, 248, 255, 242, 246, 249, 282, 236, 271, 279, 272, 259, 256, 266, 256, 246, 246, 230, 269, 262, 301, 271, 251, 265, 250, 255, 265, 243, 252, 233, 252, 242, 236, 258, 256, 242, 285, 264, 260, 272, 262, 264, 251, 294, 230, 252, 248, 291, 266, 295, 255, 233, 262, 255, 246, 264, 281, 258, 267, 241, 251, 244, 247, 238, 252, 308, 252, 267, 257, 251, 265, 255, 243, 258, 261, 226, 263, 250, 243, 245, 244, 239, 259, 241, 268, 234, 258, 245, 275, 257, 240, 262, 254, 277, 248, 278, 251, 255, 251, 257, 247, 255, 254, 248, 239, 264, 265, 245, 268, 260, 243, 237, 259, 252, 266, 271, 250, 251, 253, 252, 258, 256] payload_rows_per_dst=[5368, 5536, 5434, 5478, 5494, 5396, 5454, 5464] dup_rows_per_dst=[2784, 2774, 2738, 2884, 2698, 2620, 2706, 2708] combine_rows_per_src=[5368, 5536, 5434, 5478, 5494, 5396, 5454, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229120 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876544 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313523968 weight_movement_bytes=37748736 total_nvlink_bytes=351272704 weight_to_token_ratio=0.120401 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229616 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156664048 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313098480 weight_movement_bytes=18874368 total_nvlink_bytes=331972848 weight_to_token_ratio=0.060283 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229144 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157081368 ingress_bytes=213475328 link_bytes=213475328 token_bytes=313933592 weight_movement_bytes=56623104 total_nvlink_bytes=370556696 weight_to_token_ratio=0.180367 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045415556e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228696 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156474712 ingress_bytes=175120384 link_bytes=175120384 token_bytes=312720728 weight_movement_bytes=18874368 total_nvlink_bytes=331595096 weight_to_token_ratio=0.060355 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677917355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38382 dup_rows=21662 dup_rows_remote=19002 dedup_ratio=0.330536 router_tokens_per_expert=[221, 233, 254, 272, 220, 250, 250, 240, 261, 253, 288, 250, 259, 272, 263, 265, 244, 274, 255, 259, 251, 259, 241, 249, 250, 277, 241, 260, 271, 239, 253, 253, 247, 269, 241, 251, 255, 238, 280, 269, 260, 270, 263, 243, 273, 259, 253, 275, 229, 258, 292, 234, 261, 225, 277, 252, 238, 221, 246, 259, 255, 267, 256, 239, 232, 253, 286, 270, 263, 273, 250, 244, 243, 288, 267, 245, 254, 238, 263, 278, 260, 243, 263, 261, 258, 254, 267, 232, 281, 279, 257, 262, 235, 250, 270, 248, 230, 229, 241, 267, 267, 233, 238, 246, 262, 268, 295, 263, 244, 268, 267, 238, 227, 257, 244, 279, 266, 255, 229, 291, 269, 256, 269, 278, 261, 268, 258, 256] payload_rows_per_dst=[5486, 5492, 5444, 5448, 5540, 5500, 5442, 5522] dup_rows_per_dst=[2744, 2660, 2720, 2570, 2754, 2740, 2670, 2804] combine_rows_per_src=[5486, 5492, 5444, 5448, 5540, 5500, 5442, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38418 dup_rows=21704 dup_rows_remote=19068 dedup_ratio=0.331177 router_tokens_per_expert=[245, 274, 246, 254, 226, 237, 251, 269, 258, 270, 271, 249, 275, 281, 261, 229, 250, 262, 263, 281, 283, 234, 249, 234, 262, 270, 262, 233, 282, 247, 264, 240, 236, 255, 238, 270, 259, 255, 287, 271, 232, 236, 258, 241, 265, 260, 256, 221, 272, 228, 250, 231, 265, 260, 267, 222, 267, 245, 251, 273, 239, 247, 285, 261, 258, 259, 262, 257, 281, 247, 241, 226, 243, 242, 259, 230, 246, 255, 246, 273, 277, 258, 271, 264, 270, 295, 275, 267, 285, 248, 270, 267, 275, 246, 243, 259, 259, 278, 242, 210, 255, 243, 258, 275, 258, 255, 266, 268, 261, 254, 259, 245, 246, 258, 273, 255, 230, 250, 278, 257, 242, 252, 248, 248, 261, 269, 266, 239] payload_rows_per_dst=[5504, 5530, 5396, 5460, 5414, 5664, 5458, 5406] dup_rows_per_dst=[2688, 2702, 2684, 2666, 2636, 2876, 2714, 2738] combine_rows_per_src=[5504, 5530, 5396, 5460, 5414, 5664, 5458, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38112 dup_rows=21860 dup_rows_remote=18952 dedup_ratio=0.333557 router_tokens_per_expert=[270, 258, 234, 254, 253, 284, 252, 281, 259, 248, 244, 255, 279, 255, 267, 221, 250, 245, 256, 272, 256, 272, 259, 270, 259, 242, 271, 254, 280, 265, 258, 252, 239, 278, 247, 268, 248, 264, 278, 266, 223, 263, 292, 243, 264, 247, 254, 260, 269, 265, 264, 246, 271, 250, 217, 282, 238, 269, 257, 256, 270, 262, 262, 274, 268, 246, 251, 233, 251, 241, 252, 281, 248, 228, 236, 260, 261, 260, 238, 238, 271, 263, 238, 259, 253, 247, 257, 236, 257, 265, 259, 264, 245, 275, 253, 219, 273, 251, 258, 278, 258, 230, 246, 256, 232, 251, 263, 274, 250, 269, 249, 258, 262, 253, 259, 242, 269, 231, 233, 240, 259, 267, 247, 261, 268, 261, 249, 257] payload_rows_per_dst=[5564, 5524, 5516, 5452, 5356, 5496, 5360, 5408] dup_rows_per_dst=[2908, 2996, 2752, 2852, 2628, 2626, 2588, 2510] combine_rows_per_src=[5564, 5524, 5516, 5452, 5356, 5496, 5360, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38352 dup_rows=21710 dup_rows_remote=19026 dedup_ratio=0.331268 router_tokens_per_expert=[259, 253, 240, 257, 267, 243, 238, 258, 266, 255, 220, 223, 250, 240, 276, 277, 260, 222, 258, 251, 261, 276, 253, 311, 249, 261, 253, 250, 253, 242, 241, 249, 281, 269, 255, 261, 252, 282, 292, 288, 251, 265, 249, 260, 235, 276, 265, 266, 264, 256, 228, 276, 255, 253, 269, 260, 283, 258, 274, 256, 237, 291, 242, 277, 218, 267, 284, 270, 246, 269, 259, 265, 243, 259, 268, 247, 259, 252, 230, 233, 257, 252, 243, 233, 239, 274, 248, 252, 267, 243, 248, 263, 271, 278, 250, 282, 262, 252, 259, 255, 287, 249, 253, 251, 259, 223, 256, 232, 234, 232, 243, 242, 237, 235, 265, 272, 293, 227, 248, 266, 248, 270, 259, 256, 254, 245, 254, 243] payload_rows_per_dst=[5440, 5472, 5414, 5386, 5596, 5460, 5584, 5474] dup_rows_per_dst=[2604, 2708, 2552, 2716, 2784, 2740, 2922, 2684] combine_rows_per_src=[5440, 5472, 5414, 5386, 5596, 5460, 5584, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229536 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442208 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654880 weight_movement_bytes=18874368 total_nvlink_bytes=333529248 weight_to_token_ratio=0.059984 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229944 combine_payload_bytes=157360128 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157590072 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314950200 weight_movement_bytes=37748736 total_nvlink_bytes=352698936 weight_to_token_ratio=0.119856 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228256 combine_payload_bytes=156106752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156335008 ingress_bytes=193855488 link_bytes=193855488 token_bytes=312441760 weight_movement_bytes=37748736 total_nvlink_bytes=350190496 weight_to_token_ratio=0.120818 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857427903e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38298 dup_rows=21816 dup_rows_remote=19060 dedup_ratio=0.332886 router_tokens_per_expert=[240, 288, 260, 261, 291, 251, 263, 224, 255, 245, 253, 223, 267, 274, 241, 262, 247, 248, 284, 251, 259, 246, 290, 250, 242, 247, 229, 238, 252, 241, 286, 263, 264, 259, 258, 240, 260, 263, 255, 257, 232, 261, 241, 249, 287, 288, 261, 279, 268, 247, 249, 243, 277, 276, 271, 270, 257, 271, 255, 255, 257, 269, 244, 272, 269, 248, 277, 239, 248, 249, 240, 256, 254, 244, 268, 232, 280, 250, 265, 255, 250, 246, 251, 269, 249, 242, 247, 283, 257, 268, 233, 266, 271, 249, 266, 242, 262, 272, 235, 239, 252, 240, 247, 257, 246, 254, 272, 229, 238, 263, 283, 257, 251, 243, 254, 263, 245, 246, 260, 251, 253, 248, 258, 260, 246, 251, 265, 259] payload_rows_per_dst=[5440, 5506, 5532, 5484, 5438, 5422, 5436, 5462] dup_rows_per_dst=[2756, 2640, 2776, 2878, 2710, 2756, 2656, 2644] combine_rows_per_src=[5440, 5506, 5532, 5484, 5438, 5422, 5436, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38306 dup_rows=21766 dup_rows_remote=18932 dedup_ratio=0.332123 router_tokens_per_expert=[218, 247, 272, 246, 258, 260, 266, 235, 233, 308, 269, 253, 268, 250, 235, 201, 246, 272, 265, 262, 253, 287, 249, 256, 270, 258, 224, 259, 272, 243, 234, 262, 256, 276, 267, 264, 255, 268, 262, 240, 271, 249, 246, 245, 277, 261, 249, 263, 243, 244, 267, 252, 238, 250, 286, 262, 249, 266, 269, 257, 267, 252, 261, 272, 243, 257, 235, 270, 281, 256, 246, 296, 267, 229, 245, 253, 294, 255, 249, 260, 261, 257, 268, 262, 244, 239, 259, 262, 235, 233, 257, 261, 273, 250, 277, 243, 302, 248, 251, 259, 291, 254, 284, 273, 268, 261, 220, 202, 287, 242, 239, 240, 248, 257, 265, 258, 263, 253, 268, 256, 248, 244, 258, 244, 233, 221, 235, 264] payload_rows_per_dst=[5390, 5480, 5464, 5500, 5500, 5528, 5478, 5430] dup_rows_per_dst=[2648, 2744, 2834, 2770, 2772, 2634, 2764, 2600] combine_rows_per_src=[5390, 5480, 5464, 5500, 5500, 5528, 5478, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38340 dup_rows=21786 dup_rows_remote=19128 dedup_ratio=0.332428 router_tokens_per_expert=[242, 273, 250, 268, 253, 250, 262, 260, 282, 262, 245, 253, 257, 231, 243, 255, 250, 280, 222, 295, 236, 256, 263, 272, 234, 247, 215, 258, 257, 276, 243, 266, 276, 272, 242, 257, 260, 256, 262, 275, 304, 251, 273, 262, 265, 260, 249, 244, 294, 253, 244, 260, 283, 261, 240, 255, 272, 265, 255, 267, 247, 253, 236, 264, 228, 251, 261, 245, 251, 241, 222, 260, 245, 254, 243, 257, 276, 242, 260, 253, 261, 258, 257, 255, 254, 239, 229, 243, 254, 255, 251, 258, 260, 242, 265, 261, 242, 244, 237, 266, 240, 257, 255, 259, 225, 269, 245, 263, 276, 244, 269, 243, 257, 290, 250, 251, 294, 249, 250, 278, 253, 277, 248, 252, 264, 260, 253, 264] payload_rows_per_dst=[5510, 5474, 5570, 5496, 5362, 5434, 5410, 5494] dup_rows_per_dst=[2662, 2666, 2846, 2802, 2616, 2650, 2658, 2886] combine_rows_per_src=[5510, 5474, 5570, 5496, 5362, 5434, 5410, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38260 dup_rows=21760 dup_rows_remote=19034 dedup_ratio=0.332031 router_tokens_per_expert=[276, 243, 262, 295, 264, 266, 261, 247, 238, 255, 245, 234, 241, 297, 227, 258, 268, 229, 231, 272, 258, 257, 274, 269, 275, 246, 263, 240, 266, 255, 297, 227, 240, 236, 253, 260, 254, 253, 244, 262, 254, 299, 243, 262, 260, 269, 266, 239, 267, 263, 284, 264, 242, 263, 240, 269, 259, 244, 272, 247, 257, 250, 270, 230, 254, 245, 253, 266, 268, 229, 270, 222, 278, 252, 252, 263, 253, 259, 240, 256, 257, 263, 268, 256, 251, 266, 255, 271, 240, 258, 263, 238, 290, 250, 275, 255, 245, 252, 242, 252, 236, 266, 263, 272, 225, 267, 250, 240, 250, 276, 280, 240, 275, 259, 244, 256, 249, 228, 273, 228, 254, 238, 260, 265, 268, 251, 252, 245] payload_rows_per_dst=[5482, 5504, 5424, 5516, 5464, 5550, 5366, 5470] dup_rows_per_dst=[2736, 2750, 2764, 2726, 2656, 2762, 2746, 2620] combine_rows_per_src=[5482, 5504, 5424, 5516, 5464, 5550, 5366, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229512 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157319304 ingress_bytes=194838528 link_bytes=194838528 token_bytes=314409096 weight_movement_bytes=37748736 total_nvlink_bytes=352157832 weight_to_token_ratio=0.120062 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866846909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229432 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157098040 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966648 weight_movement_bytes=37748736 total_nvlink_bytes=351715384 weight_to_token_ratio=0.120232 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228952 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130328 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031704 weight_movement_bytes=37748736 total_nvlink_bytes=351780440 weight_to_token_ratio=0.120207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=229872 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270512 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311152 weight_movement_bytes=18874368 total_nvlink_bytes=333185520 weight_to_token_ratio=0.060050 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229176 combine_payload_bytes=156712960 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156942136 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655096 weight_movement_bytes=56623104 total_nvlink_bytes=370278200 weight_to_token_ratio=0.180527 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38368 dup_rows=21674 dup_rows_remote=18966 dedup_ratio=0.330719 router_tokens_per_expert=[267, 256, 237, 233, 239, 250, 273, 251, 268, 280, 262, 255, 273, 251, 271, 262, 246, 265, 259, 270, 246, 250, 248, 261, 263, 252, 265, 255, 234, 262, 278, 247, 257, 284, 275, 241, 255, 268, 251, 253, 265, 247, 280, 267, 249, 267, 277, 250, 243, 286, 259, 251, 265, 274, 258, 273, 277, 260, 266, 250, 239, 255, 247, 261, 256, 275, 241, 251, 253, 246, 234, 246, 250, 263, 263, 266, 251, 251, 262, 252, 255, 267, 249, 262, 248, 235, 258, 256, 268, 232, 269, 230, 255, 253, 297, 251, 264, 243, 265, 257, 254, 265, 247, 228, 262, 251, 239, 273, 274, 255, 256, 209, 242, 244, 244, 260, 219, 266, 226, 270, 243, 232, 273, 257, 264, 265, 238, 259] payload_rows_per_dst=[5428, 5494, 5518, 5570, 5492, 5502, 5470, 5388] dup_rows_per_dst=[2828, 2708, 2926, 2758, 2628, 2596, 2614, 2616] combine_rows_per_src=[5428, 5494, 5518, 5570, 5492, 5502, 5470, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43652 payload_rows_remote=38104 dup_rows=21884 dup_rows_remote=18974 dedup_ratio=0.333923 router_tokens_per_expert=[258, 263, 259, 224, 236, 240, 268, 240, 235, 251, 251, 258, 247, 257, 261, 259, 260, 211, 267, 262, 239, 252, 247, 232, 257, 223, 276, 265, 287, 243, 261, 250, 250, 282, 270, 254, 274, 283, 247, 221, 260, 270, 241, 217, 298, 239, 295, 280, 241, 258, 257, 268, 273, 261, 266, 246, 236, 256, 243, 254, 253, 254, 238, 261, 272, 260, 269, 282, 291, 263, 259, 261, 257, 253, 245, 269, 245, 262, 279, 262, 253, 223, 237, 254, 279, 266, 267, 247, 276, 258, 262, 252, 277, 256, 251, 261, 259, 252, 250, 261, 251, 272, 260, 237, 230, 256, 274, 272, 232, 281, 269, 271, 242, 258, 258, 254, 263, 248, 246, 249, 219, 250, 266, 245, 255, 254, 257, 244] payload_rows_per_dst=[5418, 5400, 5496, 5474, 5548, 5450, 5454, 5412] dup_rows_per_dst=[2596, 2664, 2866, 2656, 2910, 2788, 2800, 2604] combine_rows_per_src=[5418, 5400, 5496, 5474, 5548, 5450, 5454, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38114 dup_rows=21792 dup_rows_remote=18998 dedup_ratio=0.332520 router_tokens_per_expert=[255, 270, 283, 237, 253, 270, 259, 252, 256, 240, 274, 263, 253, 262, 257, 256, 237, 250, 242, 269, 235, 293, 277, 246, 242, 238, 264, 264, 256, 252, 270, 267, 254, 267, 250, 268, 242, 233, 258, 273, 255, 248, 248, 284, 266, 258, 234, 248, 247, 242, 263, 237, 240, 242, 272, 236, 253, 266, 244, 238, 221, 263, 230, 249, 231, 240, 275, 240, 256, 285, 257, 225, 251, 283, 264, 243, 298, 249, 247, 245, 236, 255, 283, 269, 295, 278, 247, 292, 245, 261, 234, 259, 244, 256, 268, 282, 222, 246, 270, 293, 254, 230, 255, 242, 262, 264, 256, 254, 266, 263, 241, 309, 301, 241, 255, 264, 246, 269, 284, 220, 253, 254, 238, 258, 240, 266, 245, 243] payload_rows_per_dst=[5630, 5456, 5534, 5520, 5380, 5520, 5282, 5422] dup_rows_per_dst=[2794, 2748, 2638, 2892, 2654, 2888, 2446, 2732] combine_rows_per_src=[5630, 5456, 5534, 5520, 5380, 5520, 5282, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43964 payload_rows_remote=38382 dup_rows=21572 dup_rows_remote=18626 dedup_ratio=0.329163 router_tokens_per_expert=[268, 257, 241, 259, 233, 256, 266, 271, 253, 248, 253, 275, 242, 255, 256, 257, 277, 236, 255, 239, 249, 229, 272, 259, 240, 251, 241, 242, 254, 254, 241, 240, 266, 206, 241, 241, 237, 243, 274, 259, 253, 278, 255, 258, 238, 275, 267, 262, 238, 277, 271, 266, 281, 266, 261, 238, 246, 268, 248, 263, 252, 230, 268, 252, 248, 268, 268, 240, 269, 258, 275, 259, 266, 269, 236, 265, 246, 295, 266, 242, 285, 264, 245, 222, 269, 253, 254, 257, 271, 262, 267, 271, 282, 276, 253, 249, 232, 267, 271, 265, 253, 258, 247, 249, 255, 292, 290, 243, 252, 255, 246, 270, 251, 271, 259, 251, 249, 234, 264, 250, 233, 255, 243, 267, 262, 245, 254, 238] payload_rows_per_dst=[5458, 5492, 5294, 5462, 5538, 5536, 5602, 5582] dup_rows_per_dst=[2610, 2578, 2336, 2756, 2656, 2824, 2866, 2946] combine_rows_per_src=[5458, 5492, 5294, 5462, 5538, 5536, 5602, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=228312 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156302296 ingress_bytes=174948352 link_bytes=174948352 token_bytes=312376280 weight_movement_bytes=18874368 total_nvlink_bytes=331250648 weight_to_token_ratio=0.060422 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676269029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229336 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384664 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314539992 weight_movement_bytes=18874368 total_nvlink_bytes=333414360 weight_to_token_ratio=0.060006 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228448 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156343392 ingress_bytes=193863680 link_bytes=193863680 token_bytes=312458336 weight_movement_bytes=37748736 total_nvlink_bytes=350207072 weight_to_token_ratio=0.120812 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857506395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43994 payload_rows_remote=38540 dup_rows=21542 dup_rows_remote=18766 dedup_ratio=0.328705 router_tokens_per_expert=[266, 283, 264, 262, 250, 235, 274, 235, 252, 230, 238, 276, 228, 272, 270, 266, 261, 273, 254, 275, 250, 259, 239, 252, 255, 276, 255, 236, 279, 249, 267, 254, 267, 238, 231, 255, 231, 240, 296, 265, 252, 257, 240, 273, 266, 265, 234, 266, 239, 229, 272, 258, 240, 255, 235, 254, 267, 261, 243, 256, 267, 285, 260, 259, 262, 246, 232, 250, 294, 271, 231, 240, 278, 260, 253, 247, 259, 266, 275, 248, 267, 269, 251, 270, 246, 248, 256, 266, 258, 235, 242, 259, 258, 276, 263, 251, 254, 279, 271, 263, 213, 268, 263, 268, 255, 273, 250, 262, 261, 226, 272, 239, 239, 256, 260, 253, 257, 250, 243, 230, 258, 255, 257, 241, 260, 242, 270, 262] payload_rows_per_dst=[5504, 5588, 5456, 5504, 5496, 5454, 5540, 5452] dup_rows_per_dst=[2698, 2680, 2696, 2656, 2728, 2776, 2694, 2614] combine_rows_per_src=[5504, 5588, 5456, 5504, 5496, 5454, 5540, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38356 dup_rows=21692 dup_rows_remote=18914 dedup_ratio=0.330994 router_tokens_per_expert=[262, 254, 224, 233, 246, 252, 235, 255, 230, 252, 234, 254, 266, 273, 261, 244, 240, 267, 251, 251, 234, 247, 269, 261, 256, 238, 260, 237, 239, 254, 261, 275, 232, 242, 275, 259, 274, 260, 256, 284, 265, 249, 240, 277, 272, 246, 251, 251, 261, 268, 262, 243, 244, 234, 251, 269, 261, 231, 220, 296, 279, 272, 276, 270, 248, 261, 250, 242, 244, 266, 254, 223, 283, 243, 263, 234, 301, 246, 253, 275, 271, 250, 271, 274, 274, 264, 249, 259, 262, 278, 257, 242, 292, 265, 266, 229, 264, 283, 259, 259, 249, 270, 261, 270, 246, 253, 261, 269, 231, 240, 245, 250, 240, 258, 256, 257, 231, 257, 235, 275, 259, 233, 276, 283, 266, 256, 266, 236] payload_rows_per_dst=[5392, 5434, 5488, 5546, 5484, 5542, 5498, 5460] dup_rows_per_dst=[2558, 2646, 2778, 2728, 2688, 2864, 2722, 2708] combine_rows_per_src=[5392, 5434, 5488, 5546, 5484, 5542, 5498, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44060 payload_rows_remote=38516 dup_rows=21476 dup_rows_remote=18736 dedup_ratio=0.327698 router_tokens_per_expert=[260, 277, 247, 277, 247, 245, 268, 246, 257, 269, 274, 257, 266, 267, 283, 253, 245, 246, 246, 269, 247, 269, 228, 236, 239, 268, 263, 239, 248, 252, 253, 236, 252, 270, 258, 234, 289, 285, 239, 256, 284, 247, 261, 232, 276, 261, 277, 265, 223, 262, 270, 290, 249, 265, 225, 243, 261, 231, 245, 284, 244, 240, 276, 251, 239, 220, 233, 247, 267, 298, 264, 265, 245, 243, 257, 264, 265, 258, 271, 267, 235, 270, 244, 261, 278, 267, 236, 232, 238, 259, 268, 257, 260, 256, 234, 262, 246, 261, 279, 275, 267, 250, 259, 267, 227, 271, 276, 257, 255, 252, 255, 245, 231, 239, 252, 251, 269, 245, 238, 275, 249, 288, 236, 243, 255, 234, 254, 285] payload_rows_per_dst=[5666, 5412, 5298, 5508, 5536, 5490, 5544, 5606] dup_rows_per_dst=[2720, 2556, 2582, 2610, 2670, 2624, 2740, 2974] combine_rows_per_src=[5666, 5412, 5298, 5508, 5536, 5490, 5544, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38370 dup_rows=21666 dup_rows_remote=18896 dedup_ratio=0.330597 router_tokens_per_expert=[255, 282, 266, 267, 258, 255, 268, 258, 268, 231, 265, 256, 237, 290, 255, 238, 255, 253, 270, 258, 249, 256, 256, 270, 257, 233, 261, 271, 252, 244, 265, 277, 256, 236, 228, 256, 214, 242, 252, 254, 257, 242, 239, 253, 270, 233, 239, 271, 250, 266, 258, 260, 268, 244, 258, 243, 236, 292, 252, 243, 259, 269, 262, 275, 253, 233, 256, 249, 247, 241, 265, 263, 268, 244, 245, 245, 261, 261, 261, 274, 286, 275, 249, 235, 240, 275, 261, 240, 274, 280, 259, 242, 230, 261, 269, 260, 234, 292, 258, 245, 233, 276, 274, 243, 253, 248, 238, 264, 279, 274, 261, 247, 249, 257, 228, 241, 291, 271, 259, 232, 243, 260, 248, 265, 271, 278, 260, 241] payload_rows_per_dst=[5534, 5468, 5310, 5500, 5496, 5580, 5462, 5520] dup_rows_per_dst=[2764, 2786, 2574, 2770, 2636, 2692, 2776, 2668] combine_rows_per_src=[5534, 5468, 5310, 5500, 5496, 5580, 5462, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=228032 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157440704 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314653376 weight_movement_bytes=18874368 total_nvlink_bytes=333527744 weight_to_token_ratio=0.059985 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157859840 dispatch_meta_bytes=229224 combine_payload_bytes=157859840 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158089064 ingress_bytes=195608576 link_bytes=195608576 token_bytes=315948904 weight_movement_bytes=37748736 total_nvlink_bytes=353697640 weight_to_token_ratio=0.119477 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.874225130e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229080 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157335256 ingress_bytes=194854912 link_bytes=194854912 token_bytes=314441432 weight_movement_bytes=37748736 total_nvlink_bytes=352190168 weight_to_token_ratio=0.120050 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867003892e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157761536 dispatch_meta_bytes=229008 combine_payload_bytes=157761536 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157990544 ingress_bytes=195510272 link_bytes=195510272 token_bytes=315752080 weight_movement_bytes=37748736 total_nvlink_bytes=353500816 weight_to_token_ratio=0.119552 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873283229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229064 combine_payload_bytes=157163520 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157392584 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314556104 weight_movement_bytes=37748736 total_nvlink_bytes=352304840 weight_to_token_ratio=0.120006 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43508 payload_rows_remote=38120 dup_rows=22028 dup_rows_remote=19178 dedup_ratio=0.336121 router_tokens_per_expert=[270, 285, 270, 247, 258, 258, 243, 259, 257, 240, 265, 246, 218, 259, 274, 267, 240, 244, 250, 260, 273, 240, 250, 266, 270, 249, 235, 251, 248, 268, 244, 239, 265, 247, 270, 245, 234, 268, 264, 269, 248, 273, 221, 244, 282, 249, 278, 268, 249, 274, 227, 270, 236, 270, 251, 251, 272, 266, 275, 284, 265, 238, 258, 249, 268, 294, 262, 267, 251, 264, 277, 260, 259, 257, 234, 247, 249, 263, 238, 236, 270, 255, 269, 260, 260, 255, 243, 249, 263, 252, 249, 259, 243, 273, 223, 243, 250, 280, 244, 263, 259, 238, 267, 262, 277, 253, 242, 233, 241, 257, 253, 288, 253, 265, 278, 218, 254, 263, 234, 277, 250, 247, 235, 274, 262, 210, 252, 294] payload_rows_per_dst=[5434, 5388, 5460, 5440, 5480, 5454, 5406, 5446] dup_rows_per_dst=[2852, 2850, 2736, 2646, 2772, 2678, 2808, 2686] combine_rows_per_src=[5434, 5388, 5460, 5440, 5480, 5454, 5406, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38246 dup_rows=21776 dup_rows_remote=18982 dedup_ratio=0.332275 router_tokens_per_expert=[240, 294, 245, 265, 251, 255, 259, 251, 237, 246, 251, 246, 242, 275, 244, 259, 258, 260, 268, 267, 253, 281, 248, 271, 260, 262, 270, 247, 272, 269, 255, 249, 270, 234, 258, 260, 268, 244, 269, 262, 267, 281, 232, 259, 262, 260, 262, 258, 261, 267, 259, 255, 273, 245, 242, 236, 252, 229, 240, 258, 242, 252, 272, 256, 276, 250, 264, 260, 249, 245, 237, 255, 237, 253, 274, 278, 284, 275, 252, 265, 259, 259, 263, 261, 269, 246, 247, 236, 238, 240, 230, 264, 276, 241, 254, 246, 246, 257, 259, 255, 251, 264, 239, 261, 297, 249, 271, 274, 262, 237, 256, 232, 255, 225, 252, 235, 288, 252, 270, 218, 236, 262, 277, 227, 241, 263, 268, 271] payload_rows_per_dst=[5440, 5602, 5498, 5380, 5514, 5440, 5454, 5432] dup_rows_per_dst=[2680, 2778, 2794, 2698, 2794, 2618, 2766, 2648] combine_rows_per_src=[5440, 5602, 5498, 5380, 5514, 5440, 5454, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38298 dup_rows=21824 dup_rows_remote=19184 dedup_ratio=0.333008 router_tokens_per_expert=[249, 260, 236, 273, 248, 261, 268, 246, 268, 223, 245, 241, 260, 243, 246, 257, 254, 258, 285, 256, 244, 233, 255, 245, 259, 256, 235, 239, 263, 256, 272, 259, 274, 240, 229, 265, 253, 284, 233, 256, 271, 283, 266, 239, 271, 248, 279, 248, 258, 267, 271, 246, 252, 259, 228, 238, 255, 252, 249, 249, 246, 257, 289, 254, 257, 256, 236, 282, 302, 246, 265, 244, 236, 251, 249, 254, 283, 255, 269, 235, 248, 230, 250, 271, 246, 245, 271, 268, 274, 258, 248, 247, 273, 254, 246, 281, 242, 270, 265, 241, 248, 264, 229, 252, 245, 225, 262, 259, 242, 279, 264, 251, 269, 268, 278, 269, 264, 279, 239, 263, 235, 255, 272, 271, 246, 254, 273, 263] payload_rows_per_dst=[5414, 5420, 5522, 5546, 5474, 5368, 5460, 5508] dup_rows_per_dst=[2640, 2718, 2900, 2888, 2760, 2558, 2616, 2744] combine_rows_per_src=[5414, 5420, 5522, 5546, 5474, 5368, 5460, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=229192 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156368712 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312508232 weight_movement_bytes=18874368 total_nvlink_bytes=331382600 weight_to_token_ratio=0.060396 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38332 dup_rows=21778 dup_rows_remote=19022 dedup_ratio=0.332306 router_tokens_per_expert=[259, 255, 264, 254, 244, 277, 258, 246, 262, 238, 227, 237, 246, 274, 282, 255, 283, 260, 241, 252, 241, 268, 267, 249, 225, 241, 286, 256, 249, 243, 269, 252, 253, 230, 238, 237, 247, 231, 267, 262, 260, 251, 233, 251, 264, 243, 258, 246, 252, 250, 248, 238, 286, 282, 276, 239, 253, 257, 254, 255, 265, 274, 258, 265, 224, 255, 271, 257, 255, 250, 234, 235, 235, 271, 243, 248, 286, 266, 269, 249, 284, 254, 255, 247, 261, 287, 246, 272, 250, 254, 275, 297, 269, 254, 273, 254, 269, 266, 236, 262, 257, 274, 278, 249, 259, 258, 247, 252, 253, 265, 227, 262, 249, 258, 231, 253, 251, 258, 270, 237, 271, 241, 266, 240, 276, 236, 273, 281] payload_rows_per_dst=[5458, 5526, 5360, 5438, 5466, 5584, 5500, 5426] dup_rows_per_dst=[2698, 2638, 2582, 2866, 2630, 2880, 2728, 2756] combine_rows_per_src=[5458, 5526, 5360, 5438, 5466, 5584, 5500, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228912 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156884528 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540144 weight_movement_bytes=37748736 total_nvlink_bytes=351288880 weight_to_token_ratio=0.120395 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38246 dup_rows=21820 dup_rows_remote=19042 dedup_ratio=0.332947 router_tokens_per_expert=[246, 240, 245, 241, 231, 247, 242, 253, 254, 252, 253, 229, 275, 255, 260, 269, 259, 267, 266, 213, 257, 256, 267, 246, 226, 237, 226, 273, 245, 261, 246, 273, 266, 273, 268, 235, 255, 242, 255, 268, 285, 262, 309, 246, 262, 224, 249, 244, 279, 292, 261, 258, 278, 273, 223, 284, 248, 234, 249, 232, 279, 245, 262, 257, 252, 242, 252, 251, 240, 251, 265, 289, 266, 267, 232, 270, 253, 293, 263, 229, 242, 255, 250, 254, 246, 248, 307, 255, 261, 252, 282, 272, 237, 236, 265, 262, 262, 270, 262, 242, 261, 259, 225, 266, 255, 254, 256, 236, 250, 263, 263, 263, 251, 248, 286, 261, 225, 257, 254, 240, 257, 249, 276, 245, 261, 291, 243, 291] payload_rows_per_dst=[5374, 5406, 5564, 5500, 5468, 5470, 5418, 5516] dup_rows_per_dst=[2610, 2630, 2722, 2808, 2762, 2778, 2756, 2754] combine_rows_per_src=[5374, 5406, 5564, 5500, 5468, 5470, 5418, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229928 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157098536 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313967144 weight_movement_bytes=37748736 total_nvlink_bytes=351715880 weight_to_token_ratio=0.120231 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38292 dup_rows=21830 dup_rows_remote=19068 dedup_ratio=0.333099 router_tokens_per_expert=[262, 250, 256, 251, 263, 291, 244, 230, 256, 259, 254, 255, 274, 241, 255, 239, 268, 242, 272, 243, 268, 265, 265, 250, 249, 254, 245, 259, 262, 275, 255, 257, 268, 267, 250, 251, 266, 269, 276, 228, 255, 251, 240, 252, 242, 250, 241, 282, 258, 252, 231, 276, 280, 253, 254, 275, 243, 274, 250, 221, 263, 274, 254, 254, 265, 236, 272, 280, 239, 245, 285, 256, 277, 231, 263, 278, 267, 284, 250, 279, 252, 240, 256, 275, 272, 259, 250, 278, 284, 252, 258, 243, 227, 247, 243, 265, 271, 253, 247, 263, 227, 264, 259, 240, 265, 215, 257, 263, 270, 260, 255, 264, 268, 251, 242, 240, 261, 226, 226, 253, 256, 253, 269, 252, 255, 244, 245, 237] payload_rows_per_dst=[5364, 5536, 5414, 5480, 5558, 5506, 5414, 5434] dup_rows_per_dst=[2796, 2722, 2762, 2744, 2856, 2696, 2732, 2522] combine_rows_per_src=[5364, 5536, 5414, 5480, 5558, 5506, 5414, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=38272 dup_rows=21878 dup_rows_remote=19206 dedup_ratio=0.333832 router_tokens_per_expert=[252, 283, 280, 259, 263, 253, 226, 280, 262, 233, 259, 260, 276, 232, 251, 276, 248, 275, 244, 267, 224, 245, 261, 257, 235, 227, 239, 254, 242, 261, 266, 245, 274, 272, 234, 243, 291, 263, 267, 222, 265, 249, 224, 233, 258, 254, 253, 270, 250, 272, 228, 274, 249, 274, 287, 263, 241, 266, 250, 272, 282, 295, 241, 262, 249, 282, 256, 242, 263, 250, 257, 261, 290, 259, 260, 258, 263, 261, 252, 249, 273, 263, 241, 263, 251, 252, 223, 242, 236, 245, 240, 257, 273, 255, 235, 290, 251, 245, 246, 255, 261, 246, 239, 253, 269, 265, 261, 244, 248, 251, 257, 238, 246, 269, 272, 278, 236, 239, 267, 256, 246, 245, 278, 267, 234, 250, 264, 288] payload_rows_per_dst=[5472, 5382, 5492, 5566, 5494, 5380, 5386, 5486] dup_rows_per_dst=[2818, 2598, 2652, 2846, 2810, 2698, 2672, 2784] combine_rows_per_src=[5472, 5382, 5492, 5566, 5494, 5380, 5386, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38234 dup_rows=21724 dup_rows_remote=18984 dedup_ratio=0.331482 router_tokens_per_expert=[264, 251, 235, 275, 263, 258, 271, 234, 267, 251, 250, 242, 234, 255, 270, 246, 271, 228, 250, 272, 269, 261, 249, 262, 238, 263, 272, 250, 240, 254, 256, 259, 293, 261, 244, 284, 238, 267, 273, 269, 242, 258, 279, 252, 249, 274, 273, 239, 289, 283, 247, 269, 263, 246, 259, 260, 243, 229, 243, 256, 266, 279, 270, 257, 227, 234, 272, 263, 245, 243, 226, 276, 249, 241, 282, 267, 304, 251, 266, 274, 243, 252, 280, 240, 221, 261, 260, 254, 270, 284, 267, 217, 249, 250, 243, 234, 286, 262, 240, 262, 226, 250, 269, 242, 248, 282, 229, 247, 273, 228, 273, 276, 243, 244, 264, 248, 223, 239, 231, 252, 261, 234, 267, 271, 264, 221, 286, 268] payload_rows_per_dst=[5494, 5460, 5510, 5578, 5478, 5380, 5484, 5428] dup_rows_per_dst=[2638, 2728, 2880, 2740, 2762, 2670, 2702, 2604] combine_rows_per_src=[5494, 5460, 5510, 5578, 5478, 5380, 5484, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229416 combine_payload_bytes=157007872 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157237288 ingress_bytes=194756608 link_bytes=194756608 token_bytes=314245160 weight_movement_bytes=37748736 total_nvlink_bytes=351993896 weight_to_token_ratio=0.120125 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866061992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229152 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156884768 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540384 weight_movement_bytes=56623104 total_nvlink_bytes=370163488 weight_to_token_ratio=0.180593 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229440 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073472 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917504 weight_movement_bytes=18874368 total_nvlink_bytes=332791872 weight_to_token_ratio=0.060125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228872 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156835336 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441800 weight_movement_bytes=37748736 total_nvlink_bytes=351190536 weight_to_token_ratio=0.120433 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156762112 dispatch_meta_bytes=229912 combine_payload_bytes=156762112 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156992024 ingress_bytes=156762112 link_bytes=156992024 token_bytes=313754136 weight_movement_bytes=0 total_nvlink_bytes=313754136 weight_to_token_ratio=0.000000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.504220329e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38424 dup_rows=21712 dup_rows_remote=19048 dedup_ratio=0.331299 router_tokens_per_expert=[233, 236, 258, 286, 264, 252, 245, 270, 273, 253, 267, 250, 265, 268, 257, 253, 244, 249, 277, 247, 258, 275, 240, 230, 258, 217, 258, 262, 263, 243, 251, 260, 267, 282, 264, 250, 259, 248, 241, 247, 276, 245, 283, 273, 240, 252, 231, 251, 231, 252, 224, 275, 275, 227, 259, 246, 268, 270, 252, 240, 273, 278, 253, 243, 249, 244, 231, 255, 282, 228, 241, 257, 289, 257, 238, 254, 273, 203, 267, 277, 229, 239, 263, 233, 256, 257, 277, 256, 271, 283, 264, 252, 266, 243, 271, 278, 225, 241, 244, 247, 243, 243, 248, 231, 281, 252, 269, 271, 256, 270, 271, 293, 272, 269, 273, 268, 241, 246, 256, 285, 249, 235, 287, 264, 256, 280, 240, 242] payload_rows_per_dst=[5480, 5400, 5476, 5446, 5458, 5514, 5448, 5602] dup_rows_per_dst=[2780, 2664, 2742, 2686, 2632, 2762, 2722, 2724] combine_rows_per_src=[5480, 5400, 5476, 5446, 5458, 5514, 5448, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38198 dup_rows=21946 dup_rows_remote=19264 dedup_ratio=0.334869 router_tokens_per_expert=[234, 255, 257, 290, 253, 258, 239, 238, 274, 264, 282, 250, 268, 253, 244, 247, 246, 268, 239, 250, 268, 242, 250, 219, 273, 268, 270, 253, 252, 277, 251, 256, 273, 266, 261, 254, 251, 264, 244, 256, 240, 252, 261, 266, 261, 270, 254, 251, 256, 259, 261, 264, 283, 245, 247, 264, 249, 223, 244, 209, 236, 253, 268, 262, 255, 213, 253, 271, 248, 264, 240, 253, 226, 271, 266, 229, 278, 231, 267, 272, 272, 227, 293, 265, 279, 272, 245, 262, 258, 253, 282, 261, 263, 248, 259, 268, 241, 254, 270, 250, 248, 247, 256, 279, 264, 240, 271, 245, 269, 232, 245, 250, 275, 269, 237, 277, 229, 261, 229, 254, 304, 229, 261, 262, 281, 283, 248, 229] payload_rows_per_dst=[5480, 5402, 5552, 5310, 5392, 5566, 5398, 5490] dup_rows_per_dst=[2732, 2762, 2696, 2736, 2682, 2848, 2724, 2766] combine_rows_per_src=[5480, 5402, 5552, 5310, 5392, 5566, 5398, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43688 payload_rows_remote=38140 dup_rows=21848 dup_rows_remote=18852 dedup_ratio=0.333374 router_tokens_per_expert=[222, 239, 241, 248, 259, 249, 238, 243, 229, 262, 258, 268, 268, 266, 265, 242, 254, 233, 268, 262, 272, 258, 254, 266, 236, 253, 239, 270, 246, 268, 271, 262, 269, 224, 250, 233, 292, 289, 281, 235, 264, 250, 266, 276, 275, 271, 252, 256, 274, 248, 251, 243, 268, 261, 272, 260, 263, 275, 275, 245, 262, 241, 265, 256, 246, 253, 266, 247, 253, 243, 262, 251, 253, 279, 228, 266, 257, 262, 262, 255, 236, 258, 273, 244, 258, 232, 271, 238, 269, 241, 267, 256, 243, 254, 223, 246, 230, 239, 260, 247, 245, 261, 239, 262, 258, 251, 268, 262, 255, 257, 271, 295, 263, 271, 249, 267, 235, 248, 260, 249, 245, 256, 256, 237, 256, 266, 282, 285] payload_rows_per_dst=[5548, 5604, 5366, 5484, 5416, 5380, 5424, 5466] dup_rows_per_dst=[2996, 2806, 2516, 2768, 2750, 2550, 2776, 2686] combine_rows_per_src=[5548, 5604, 5366, 5484, 5416, 5380, 5424, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38010 dup_rows=21914 dup_rows_remote=18930 dedup_ratio=0.334381 router_tokens_per_expert=[262, 266, 263, 247, 237, 245, 256, 270, 223, 264, 244, 231, 246, 273, 253, 256, 246, 252, 261, 236, 235, 275, 262, 240, 270, 271, 245, 239, 274, 264, 255, 241, 266, 246, 255, 249, 253, 240, 240, 297, 234, 264, 252, 261, 281, 283, 256, 266, 234, 237, 265, 273, 255, 265, 265, 240, 273, 270, 240, 287, 253, 243, 276, 262, 275, 254, 246, 282, 253, 278, 253, 271, 282, 274, 253, 264, 302, 252, 273, 250, 249, 265, 275, 269, 246, 228, 244, 267, 253, 249, 253, 262, 258, 242, 244, 246, 215, 265, 269, 235, 275, 260, 270, 258, 249, 245, 253, 247, 259, 258, 222, 224, 226, 262, 268, 244, 269, 276, 243, 265, 260, 259, 232, 285, 257, 243, 239, 241] payload_rows_per_dst=[5422, 5254, 5582, 5332, 5498, 5430, 5492, 5612] dup_rows_per_dst=[2650, 2410, 3088, 2696, 2652, 2670, 2764, 2984] combine_rows_per_src=[5422, 5254, 5582, 5332, 5498, 5430, 5492, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157384704 dispatch_meta_bytes=229888 combine_payload_bytes=157384704 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157614592 ingress_bytes=214007808 link_bytes=214007808 token_bytes=314999296 weight_movement_bytes=56623104 total_nvlink_bytes=371622400 weight_to_token_ratio=0.179756 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050517518e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=229848 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156688856 ingress_bytes=175333376 link_bytes=175333376 token_bytes=313147864 weight_movement_bytes=18874368 total_nvlink_bytes=332022232 weight_to_token_ratio=0.060273 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679958139e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38372 dup_rows=21762 dup_rows_remote=18916 dedup_ratio=0.332062 router_tokens_per_expert=[253, 294, 239, 286, 259, 251, 278, 227, 231, 255, 255, 244, 235, 266, 266, 245, 254, 251, 270, 255, 249, 261, 286, 259, 282, 243, 264, 259, 244, 259, 246, 244, 237, 283, 279, 251, 263, 260, 249, 246, 259, 230, 288, 273, 245, 251, 237, 243, 262, 252, 239, 263, 278, 248, 247, 252, 267, 249, 232, 237, 260, 261, 262, 268, 239, 284, 245, 268, 244, 222, 266, 276, 271, 275, 255, 242, 244, 257, 271, 256, 273, 248, 251, 239, 250, 270, 264, 265, 272, 275, 267, 246, 238, 264, 251, 251, 234, 249, 268, 255, 259, 266, 278, 255, 289, 265, 247, 272, 266, 246, 231, 214, 254, 241, 275, 258, 288, 236, 228, 232, 265, 246, 245, 262, 248, 276, 251, 249] payload_rows_per_dst=[5558, 5512, 5484, 5404, 5504, 5402, 5408, 5502] dup_rows_per_dst=[2610, 2740, 2704, 2750, 2726, 2846, 2780, 2606] combine_rows_per_src=[5558, 5512, 5484, 5404, 5504, 5402, 5408, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38130 dup_rows=21980 dup_rows_remote=19186 dedup_ratio=0.335388 router_tokens_per_expert=[244, 271, 276, 254, 228, 236, 227, 308, 262, 253, 257, 259, 239, 221, 253, 270, 258, 248, 247, 256, 268, 266, 248, 234, 264, 256, 257, 245, 272, 255, 252, 235, 249, 246, 265, 255, 285, 273, 245, 269, 248, 248, 237, 261, 276, 254, 235, 264, 243, 253, 224, 236, 259, 250, 242, 266, 241, 232, 230, 275, 243, 255, 277, 276, 251, 276, 244, 247, 307, 269, 292, 236, 244, 243, 250, 261, 221, 247, 256, 245, 251, 256, 271, 267, 264, 269, 230, 261, 265, 240, 261, 251, 231, 247, 254, 260, 235, 276, 284, 258, 259, 257, 282, 258, 267, 271, 249, 269, 252, 271, 231, 284, 292, 261, 241, 253, 256, 284, 255, 245, 230, 277, 260, 271, 275, 259, 277, 231] payload_rows_per_dst=[5398, 5382, 5426, 5310, 5466, 5452, 5610, 5512] dup_rows_per_dst=[2718, 2740, 2794, 2694, 2712, 2704, 2796, 2822] combine_rows_per_src=[5398, 5382, 5426, 5310, 5466, 5452, 5610, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=227968 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156449408 ingress_bytes=175095808 link_bytes=175095808 token_bytes=312670848 weight_movement_bytes=18874368 total_nvlink_bytes=331545216 weight_to_token_ratio=0.060365 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677681880e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44004 payload_rows_remote=38396 dup_rows=21532 dup_rows_remote=18846 dedup_ratio=0.328552 router_tokens_per_expert=[260, 252, 235, 268, 248, 257, 255, 258, 254, 237, 242, 254, 271, 255, 283, 276, 247, 234, 259, 260, 288, 256, 272, 262, 271, 236, 237, 245, 273, 265, 260, 278, 253, 239, 239, 286, 257, 282, 217, 229, 250, 259, 272, 259, 263, 233, 225, 253, 244, 274, 240, 253, 257, 275, 257, 264, 252, 287, 273, 246, 249, 253, 259, 264, 258, 243, 256, 279, 247, 264, 249, 286, 260, 265, 283, 226, 234, 264, 266, 249, 256, 246, 265, 272, 283, 229, 254, 225, 236, 234, 250, 257, 254, 281, 265, 244, 267, 260, 258, 237, 252, 251, 251, 260, 271, 227, 266, 224, 246, 258, 272, 241, 248, 280, 271, 275, 261, 253, 242, 288, 262, 255, 241, 235, 249, 265, 246, 265] payload_rows_per_dst=[5508, 5616, 5368, 5608, 5522, 5494, 5420, 5468] dup_rows_per_dst=[2702, 2670, 2664, 2686, 2736, 2608, 2662, 2804] combine_rows_per_src=[5508, 5616, 5368, 5608, 5522, 5494, 5420, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38320 dup_rows=21756 dup_rows_remote=19064 dedup_ratio=0.331970 router_tokens_per_expert=[233, 265, 257, 236, 261, 269, 272, 252, 263, 238, 229, 260, 227, 250, 243, 235, 280, 278, 274, 293, 282, 240, 253, 292, 265, 281, 257, 244, 280, 272, 204, 253, 236, 249, 249, 264, 268, 233, 249, 244, 246, 245, 262, 261, 242, 256, 247, 249, 264, 263, 234, 250, 269, 265, 242, 280, 269, 254, 234, 284, 243, 255, 277, 261, 277, 258, 258, 241, 272, 276, 259, 235, 272, 268, 232, 266, 263, 280, 247, 279, 242, 254, 249, 276, 259, 262, 244, 240, 249, 265, 276, 262, 264, 263, 219, 274, 265, 254, 230, 236, 263, 262, 256, 273, 240, 256, 268, 264, 254, 277, 235, 243, 245, 250, 250, 253, 256, 248, 229, 275, 254, 272, 249, 250, 232, 256, 278, 232] payload_rows_per_dst=[5388, 5568, 5370, 5518, 5640, 5394, 5460, 5442] dup_rows_per_dst=[2592, 2928, 2630, 2770, 2726, 2802, 2692, 2616] combine_rows_per_src=[5388, 5568, 5370, 5518, 5640, 5394, 5460, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=227760 combine_payload_bytes=155688960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155916720 ingress_bytes=174563328 link_bytes=174563328 token_bytes=311605680 weight_movement_bytes=18874368 total_nvlink_bytes=330480048 weight_to_token_ratio=0.060571 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672579918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229152 combine_payload_bytes=157171712 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157400864 ingress_bytes=194920448 link_bytes=194920448 token_bytes=314572576 weight_movement_bytes=37748736 total_nvlink_bytes=352321312 weight_to_token_ratio=0.120000 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867631826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156180480 dispatch_meta_bytes=229264 combine_payload_bytes=156180480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156409744 ingress_bytes=175054848 link_bytes=175054848 token_bytes=312590224 weight_movement_bytes=18874368 total_nvlink_bytes=331464592 weight_to_token_ratio=0.060381 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677289421e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=228968 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157498984 ingress_bytes=195018752 link_bytes=195018752 token_bytes=314769000 weight_movement_bytes=37748736 total_nvlink_bytes=352517736 weight_to_token_ratio=0.119925 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868573726e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229536 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157188256 ingress_bytes=175833088 link_bytes=175833088 token_bytes=314146976 weight_movement_bytes=18874368 total_nvlink_bytes=333021344 weight_to_token_ratio=0.060081 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684746134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38356 dup_rows=21718 dup_rows_remote=19108 dedup_ratio=0.331390 router_tokens_per_expert=[245, 276, 241, 274, 261, 262, 240, 253, 249, 239, 217, 259, 257, 267, 253, 249, 251, 251, 227, 254, 253, 281, 242, 249, 250, 242, 247, 242, 249, 289, 262, 247, 233, 230, 258, 237, 250, 279, 253, 259, 239, 276, 242, 273, 229, 291, 228, 282, 237, 253, 259, 241, 241, 230, 247, 255, 270, 258, 271, 244, 246, 252, 250, 267, 236, 271, 260, 244, 259, 242, 255, 245, 258, 257, 242, 268, 254, 250, 280, 224, 251, 244, 266, 252, 281, 319, 244, 267, 245, 259, 271, 274, 289, 266, 293, 258, 258, 270, 265, 252, 226, 241, 218, 275, 266, 279, 257, 256, 273, 265, 245, 259, 257, 263, 264, 271, 262, 279, 292, 272, 253, 258, 260, 261, 266, 259, 249, 215] payload_rows_per_dst=[5394, 5462, 5402, 5390, 5506, 5570, 5520, 5574] dup_rows_per_dst=[2690, 2610, 2716, 2652, 2584, 2988, 2690, 2788] combine_rows_per_src=[5394, 5462, 5402, 5390, 5506, 5570, 5520, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38342 dup_rows=21650 dup_rows_remote=18962 dedup_ratio=0.330353 router_tokens_per_expert=[246, 266, 265, 262, 244, 263, 255, 267, 240, 278, 279, 254, 271, 248, 233, 252, 256, 276, 260, 246, 256, 244, 284, 252, 271, 262, 229, 248, 236, 225, 241, 250, 252, 262, 285, 274, 244, 258, 251, 247, 269, 247, 282, 277, 242, 265, 245, 244, 298, 252, 254, 258, 260, 284, 297, 261, 260, 230, 235, 256, 261, 243, 254, 275, 248, 238, 270, 248, 283, 269, 269, 244, 258, 243, 261, 234, 288, 289, 230, 244, 233, 250, 273, 254, 252, 254, 250, 242, 249, 254, 260, 261, 282, 279, 230, 250, 251, 270, 269, 232, 270, 243, 273, 264, 251, 255, 273, 260, 267, 277, 230, 259, 255, 251, 258, 258, 268, 248, 213, 256, 249, 242, 245, 254, 221, 252, 251, 233] payload_rows_per_dst=[5492, 5436, 5548, 5524, 5544, 5474, 5524, 5344] dup_rows_per_dst=[2754, 2636, 2740, 2832, 2688, 2672, 2764, 2564] combine_rows_per_src=[5492, 5436, 5548, 5524, 5544, 5474, 5524, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38388 dup_rows=21720 dup_rows_remote=18980 dedup_ratio=0.331421 router_tokens_per_expert=[246, 233, 252, 256, 250, 250, 248, 276, 267, 264, 309, 233, 261, 277, 235, 222, 246, 254, 259, 289, 238, 248, 257, 260, 264, 252, 256, 263, 233, 248, 271, 247, 271, 244, 287, 241, 249, 290, 267, 274, 256, 241, 265, 251, 282, 263, 275, 258, 269, 259, 268, 277, 235, 248, 271, 238, 260, 240, 248, 257, 255, 227, 241, 259, 231, 229, 267, 249, 252, 267, 245, 284, 268, 246, 257, 261, 238, 254, 243, 249, 264, 215, 240, 276, 285, 260, 263, 299, 272, 267, 238, 278, 260, 242, 256, 282, 245, 285, 280, 259, 245, 269, 251, 239, 245, 252, 246, 277, 231, 251, 237, 259, 238, 258, 257, 244, 249, 242, 246, 265, 234, 233, 283, 237, 262, 245, 279, 258] payload_rows_per_dst=[5428, 5442, 5810, 5432, 5424, 5518, 5292, 5470] dup_rows_per_dst=[2740, 2728, 3146, 2662, 2656, 2724, 2474, 2590] combine_rows_per_src=[5428, 5442, 5810, 5432, 5424, 5518, 5292, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38106 dup_rows=21864 dup_rows_remote=19052 dedup_ratio=0.333618 router_tokens_per_expert=[240, 255, 256, 252, 227, 251, 249, 288, 258, 251, 263, 257, 260, 254, 293, 230, 264, 289, 255, 236, 271, 280, 261, 256, 272, 266, 266, 226, 228, 248, 267, 270, 257, 215, 231, 234, 290, 231, 266, 270, 245, 264, 249, 259, 246, 273, 256, 241, 260, 269, 269, 249, 261, 259, 268, 252, 256, 258, 300, 276, 288, 246, 259, 256, 249, 258, 226, 261, 252, 238, 277, 271, 240, 248, 243, 271, 204, 225, 251, 261, 206, 237, 264, 231, 236, 242, 276, 236, 251, 270, 254, 247, 242, 237, 264, 250, 265, 262, 251, 258, 256, 261, 284, 266, 224, 275, 238, 277, 237, 293, 274, 271, 249, 246, 257, 273, 251, 253, 252, 272, 250, 255, 280, 264, 258, 280, 281, 245] payload_rows_per_dst=[5538, 5462, 5408, 5524, 5314, 5304, 5556, 5566] dup_rows_per_dst=[2630, 2848, 2646, 2694, 2590, 2582, 3062, 2812] combine_rows_per_src=[5538, 5462, 5408, 5524, 5314, 5304, 5556, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229856 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157336032 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314442208 weight_movement_bytes=18874368 total_nvlink_bytes=333316576 weight_to_token_ratio=0.060025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229216 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157278048 ingress_bytes=213671936 link_bytes=213671936 token_bytes=314326880 weight_movement_bytes=56623104 total_nvlink_bytes=370949984 weight_to_token_ratio=0.180141 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047299358e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38066 dup_rows=21966 dup_rows_remote=19206 dedup_ratio=0.335175 router_tokens_per_expert=[242, 251, 231, 244, 257, 276, 252, 246, 248, 266, 249, 278, 241, 270, 246, 247, 251, 246, 263, 302, 249, 249, 234, 261, 276, 257, 267, 264, 253, 277, 265, 235, 257, 264, 273, 223, 266, 237, 264, 275, 242, 272, 246, 267, 272, 258, 258, 258, 264, 239, 238, 269, 235, 241, 282, 285, 284, 228, 249, 241, 222, 267, 261, 265, 247, 259, 221, 295, 231, 251, 242, 236, 243, 257, 254, 242, 254, 236, 249, 232, 268, 249, 243, 232, 252, 252, 262, 263, 260, 251, 250, 224, 237, 271, 279, 240, 256, 266, 292, 279, 265, 278, 276, 257, 273, 236, 255, 239, 228, 241, 281, 266, 246, 261, 259, 238, 253, 264, 269, 272, 263, 248, 272, 264, 274, 301, 283, 236] payload_rows_per_dst=[5350, 5516, 5504, 5438, 5320, 5384, 5512, 5546] dup_rows_per_dst=[2738, 2782, 2760, 2702, 2578, 2682, 2864, 2860] combine_rows_per_src=[5350, 5516, 5504, 5438, 5320, 5384, 5512, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38480 dup_rows=21644 dup_rows_remote=18972 dedup_ratio=0.330261 router_tokens_per_expert=[238, 275, 226, 286, 265, 280, 257, 277, 258, 244, 242, 238, 229, 243, 246, 278, 279, 238, 275, 269, 266, 270, 284, 259, 256, 277, 251, 255, 234, 248, 267, 241, 238, 268, 284, 259, 249, 252, 241, 267, 255, 237, 257, 252, 269, 246, 264, 263, 235, 239, 260, 278, 254, 267, 247, 270, 289, 268, 240, 238, 264, 281, 262, 236, 257, 261, 261, 270, 260, 269, 227, 261, 248, 251, 266, 277, 248, 295, 244, 249, 237, 250, 260, 254, 269, 253, 262, 268, 237, 251, 254, 251, 252, 247, 260, 237, 263, 259, 250, 253, 239, 239, 260, 231, 274, 245, 272, 247, 224, 251, 220, 245, 263, 256, 256, 266, 240, 264, 262, 242, 236, 247, 274, 239, 261, 253, 283, 288] payload_rows_per_dst=[5498, 5532, 5476, 5532, 5510, 5412, 5388, 5544] dup_rows_per_dst=[2666, 2806, 2726, 2724, 2778, 2672, 2556, 2716] combine_rows_per_src=[5498, 5532, 5476, 5532, 5510, 5412, 5388, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38384 dup_rows=21702 dup_rows_remote=19100 dedup_ratio=0.331146 router_tokens_per_expert=[263, 264, 265, 249, 255, 264, 275, 255, 234, 248, 256, 263, 235, 253, 259, 255, 253, 253, 247, 273, 242, 285, 223, 269, 237, 262, 245, 260, 281, 265, 261, 247, 251, 239, 250, 244, 287, 244, 266, 257, 235, 245, 262, 253, 256, 229, 257, 267, 240, 276, 258, 261, 258, 251, 262, 247, 262, 294, 285, 260, 254, 229, 265, 273, 259, 280, 291, 222, 240, 284, 250, 256, 233, 273, 262, 254, 236, 244, 248, 252, 270, 253, 286, 258, 269, 249, 238, 261, 267, 249, 262, 237, 269, 240, 276, 259, 260, 246, 273, 246, 258, 236, 232, 290, 249, 245, 261, 252, 247, 253, 227, 251, 257, 270, 278, 232, 248, 255, 269, 260, 253, 214, 267, 220, 249, 304, 265, 261] payload_rows_per_dst=[5506, 5562, 5328, 5572, 5454, 5478, 5450, 5484] dup_rows_per_dst=[2680, 2644, 2756, 2778, 2714, 2808, 2602, 2720] combine_rows_per_src=[5506, 5562, 5328, 5572, 5454, 5478, 5450, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229472 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157466720 ingress_bytes=194985984 link_bytes=194985984 token_bytes=314703968 weight_movement_bytes=37748736 total_nvlink_bytes=352452704 weight_to_token_ratio=0.119950 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868259760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38220 dup_rows=21860 dup_rows_remote=19158 dedup_ratio=0.333557 router_tokens_per_expert=[270, 276, 237, 266, 267, 276, 236, 270, 247, 249, 241, 239, 260, 260, 244, 267, 222, 270, 261, 274, 238, 281, 239, 225, 286, 266, 252, 283, 277, 265, 256, 248, 251, 264, 264, 256, 279, 270, 239, 268, 253, 257, 234, 229, 263, 251, 244, 258, 258, 236, 244, 234, 290, 275, 243, 245, 228, 271, 265, 292, 265, 267, 241, 225, 273, 259, 241, 242, 267, 259, 278, 258, 239, 257, 277, 256, 256, 260, 268, 246, 256, 250, 254, 267, 287, 280, 284, 250, 256, 255, 233, 226, 268, 255, 255, 253, 253, 265, 257, 251, 239, 223, 253, 255, 286, 252, 261, 235, 253, 257, 268, 238, 249, 248, 262, 265, 254, 228, 260, 261, 245, 249, 261, 256, 238, 258, 272, 244] payload_rows_per_dst=[5450, 5532, 5438, 5456, 5532, 5464, 5380, 5424] dup_rows_per_dst=[2760, 2754, 2722, 2702, 2740, 2794, 2712, 2676] combine_rows_per_src=[5450, 5532, 5438, 5456, 5532, 5464, 5380, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=228632 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156310808 ingress_bytes=231579648 link_bytes=231579648 token_bytes=312392984 weight_movement_bytes=75497472 total_nvlink_bytes=387890456 weight_to_token_ratio=0.241675 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.218882243e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=229088 combine_payload_bytes=155918336 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156147424 ingress_bytes=193667072 link_bytes=193667072 token_bytes=312065760 weight_movement_bytes=37748736 total_nvlink_bytes=349814496 weight_to_token_ratio=0.120964 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.855622594e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157220864 dispatch_meta_bytes=229936 combine_payload_bytes=157220864 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157450800 ingress_bytes=176095232 link_bytes=176095232 token_bytes=314671664 weight_movement_bytes=18874368 total_nvlink_bytes=333546032 weight_to_token_ratio=0.059981 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687257869e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157614080 dispatch_meta_bytes=229808 combine_payload_bytes=157614080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157843888 ingress_bytes=176488448 link_bytes=176488448 token_bytes=315457968 weight_movement_bytes=18874368 total_nvlink_bytes=334332336 weight_to_token_ratio=0.059832 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691025471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229512 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156778632 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313327752 weight_movement_bytes=56623104 total_nvlink_bytes=369950856 weight_to_token_ratio=0.180715 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38374 dup_rows=21686 dup_rows_remote=19040 dedup_ratio=0.330902 router_tokens_per_expert=[248, 267, 243, 286, 275, 266, 277, 246, 252, 250, 244, 261, 268, 218, 246, 272, 229, 252, 239, 233, 260, 263, 267, 240, 248, 277, 253, 266, 266, 261, 279, 228, 275, 247, 273, 263, 274, 227, 246, 245, 261, 239, 268, 251, 224, 258, 240, 255, 283, 231, 254, 254, 271, 242, 252, 270, 275, 255, 272, 270, 248, 272, 268, 252, 252, 252, 276, 263, 272, 263, 254, 245, 242, 236, 253, 276, 250, 232, 274, 250, 244, 233, 250, 269, 267, 250, 237, 240, 271, 285, 276, 270, 274, 276, 253, 207, 247, 280, 248, 263, 242, 249, 261, 261, 249, 255, 249, 245, 237, 249, 239, 268, 246, 248, 264, 267, 252, 296, 238, 242, 258, 285, 261, 249, 264, 261, 263, 245] payload_rows_per_dst=[5442, 5476, 5458, 5514, 5446, 5512, 5462, 5540] dup_rows_per_dst=[2796, 2646, 2634, 2824, 2734, 2692, 2622, 2738] combine_rows_per_src=[5442, 5476, 5458, 5514, 5446, 5512, 5462, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38310 dup_rows=21806 dup_rows_remote=18984 dedup_ratio=0.332733 router_tokens_per_expert=[244, 279, 257, 254, 253, 269, 251, 259, 239, 275, 269, 290, 258, 246, 266, 252, 240, 253, 244, 269, 265, 298, 261, 255, 214, 242, 258, 239, 251, 259, 248, 258, 278, 253, 261, 267, 273, 277, 240, 255, 262, 279, 227, 226, 251, 279, 285, 238, 265, 255, 257, 267, 236, 263, 286, 250, 252, 280, 264, 265, 256, 245, 260, 245, 257, 252, 234, 265, 249, 239, 251, 282, 276, 241, 274, 256, 249, 246, 276, 274, 255, 269, 240, 243, 233, 248, 252, 249, 251, 215, 247, 245, 250, 278, 259, 211, 255, 235, 230, 245, 256, 240, 231, 268, 287, 265, 249, 277, 251, 290, 246, 250, 253, 268, 256, 249, 286, 249, 264, 278, 252, 254, 255, 230, 242, 245, 277, 257] payload_rows_per_dst=[5474, 5394, 5550, 5526, 5420, 5348, 5480, 5538] dup_rows_per_dst=[2848, 2714, 2752, 2766, 2822, 2542, 2670, 2692] combine_rows_per_src=[5474, 5394, 5550, 5526, 5420, 5348, 5480, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38118 dup_rows=21768 dup_rows_remote=18892 dedup_ratio=0.332153 router_tokens_per_expert=[244, 250, 242, 270, 240, 263, 245, 264, 244, 215, 253, 262, 249, 234, 248, 259, 276, 264, 236, 261, 254, 228, 259, 260, 240, 265, 269, 244, 253, 269, 277, 236, 281, 278, 254, 290, 240, 254, 261, 246, 253, 281, 250, 241, 265, 280, 255, 258, 271, 252, 264, 230, 255, 270, 241, 263, 273, 269, 270, 265, 263, 235, 255, 268, 258, 266, 259, 236, 245, 239, 255, 244, 255, 237, 262, 245, 275, 241, 270, 239, 253, 236, 260, 264, 253, 287, 274, 210, 243, 254, 250, 252, 251, 256, 264, 234, 235, 291, 262, 276, 281, 255, 273, 264, 281, 245, 245, 265, 254, 262, 251, 227, 279, 254, 225, 258, 279, 281, 282, 254, 246, 246, 240, 248, 274, 258, 267, 239] payload_rows_per_dst=[5650, 5664, 5462, 5298, 5440, 5240, 5552, 5462] dup_rows_per_dst=[2876, 3022, 2602, 2490, 2728, 2504, 2782, 2764] combine_rows_per_src=[5650, 5664, 5462, 5298, 5440, 5240, 5552, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229656 combine_payload_bytes=157179904 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157409560 ingress_bytes=194928640 link_bytes=194928640 token_bytes=314589464 weight_movement_bytes=37748736 total_nvlink_bytes=352338200 weight_to_token_ratio=0.119994 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867710318e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38380 dup_rows=21744 dup_rows_remote=19034 dedup_ratio=0.331787 router_tokens_per_expert=[238, 238, 252, 244, 246, 273, 225, 240, 275, 241, 272, 252, 251, 276, 246, 255, 275, 228, 276, 274, 237, 257, 237, 255, 257, 254, 250, 257, 255, 255, 287, 241, 275, 253, 281, 260, 269, 262, 245, 272, 249, 262, 239, 235, 239, 266, 266, 244, 272, 242, 255, 257, 256, 266, 242, 263, 264, 265, 248, 270, 236, 274, 261, 265, 264, 273, 241, 232, 259, 239, 290, 271, 234, 269, 253, 241, 260, 256, 268, 260, 279, 267, 268, 250, 270, 255, 255, 271, 271, 245, 243, 273, 262, 238, 260, 263, 251, 255, 261, 264, 235, 263, 265, 273, 252, 243, 226, 280, 265, 256, 239, 259, 235, 242, 238, 269, 261, 251, 256, 260, 261, 266, 213, 240, 252, 258, 272, 255] payload_rows_per_dst=[5392, 5322, 5468, 5512, 5558, 5588, 5540, 5412] dup_rows_per_dst=[2656, 2702, 2692, 2668, 2802, 2762, 2752, 2710] combine_rows_per_src=[5392, 5322, 5468, 5512, 5558, 5588, 5540, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229176 combine_payload_bytes=156917760 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157146936 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314064696 weight_movement_bytes=37748736 total_nvlink_bytes=351813432 weight_to_token_ratio=0.120194 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38174 dup_rows=21888 dup_rows_remote=19134 dedup_ratio=0.333984 router_tokens_per_expert=[222, 242, 242, 252, 252, 265, 266, 261, 268, 252, 266, 253, 260, 242, 264, 246, 240, 254, 262, 240, 242, 259, 273, 247, 240, 251, 244, 263, 241, 238, 236, 246, 259, 271, 280, 255, 245, 269, 257, 242, 262, 286, 248, 261, 239, 255, 238, 247, 274, 263, 263, 264, 230, 279, 239, 280, 270, 248, 280, 240, 260, 268, 293, 233, 266, 237, 263, 231, 258, 264, 252, 253, 233, 261, 232, 282, 275, 280, 239, 258, 274, 265, 244, 253, 243, 241, 278, 265, 259, 286, 268, 252, 262, 265, 259, 285, 246, 244, 261, 283, 247, 227, 240, 225, 244, 242, 243, 251, 259, 262, 278, 247, 215, 249, 245, 266, 256, 256, 290, 253, 268, 264, 292, 247, 275, 265, 259, 259] payload_rows_per_dst=[5414, 5396, 5474, 5518, 5468, 5502, 5362, 5514] dup_rows_per_dst=[2692, 2556, 2754, 2850, 2700, 2896, 2636, 2804] combine_rows_per_src=[5414, 5396, 5474, 5518, 5468, 5502, 5362, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38244 dup_rows=21850 dup_rows_remote=19192 dedup_ratio=0.333405 router_tokens_per_expert=[269, 264, 264, 248, 247, 266, 247, 258, 268, 268, 237, 243, 254, 249, 271, 235, 255, 293, 274, 262, 274, 273, 254, 221, 276, 253, 250, 267, 263, 260, 253, 255, 259, 241, 256, 254, 263, 270, 246, 230, 249, 272, 259, 261, 272, 265, 252, 246, 264, 242, 252, 248, 248, 269, 275, 275, 281, 235, 256, 272, 288, 251, 255, 255, 264, 270, 268, 247, 250, 235, 248, 250, 243, 264, 279, 263, 243, 278, 265, 252, 251, 238, 255, 237, 249, 245, 269, 246, 234, 256, 279, 280, 261, 224, 236, 259, 232, 246, 250, 241, 262, 263, 246, 235, 247, 295, 259, 280, 212, 278, 257, 247, 244, 251, 273, 266, 248, 240, 264, 227, 263, 240, 263, 236, 265, 236, 256, 276] payload_rows_per_dst=[5468, 5560, 5502, 5538, 5474, 5366, 5442, 5336] dup_rows_per_dst=[2708, 2806, 2688, 2794, 2764, 2672, 2658, 2760] combine_rows_per_src=[5468, 5560, 5502, 5538, 5474, 5366, 5442, 5336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38292 dup_rows=21932 dup_rows_remote=19174 dedup_ratio=0.334656 router_tokens_per_expert=[259, 263, 247, 276, 245, 251, 243, 229, 271, 254, 246, 265, 242, 254, 276, 241, 241, 262, 262, 278, 231, 280, 232, 246, 251, 261, 239, 236, 258, 267, 265, 259, 280, 239, 267, 260, 269, 237, 268, 266, 213, 281, 249, 262, 261, 282, 265, 277, 249, 240, 250, 265, 261, 295, 238, 238, 242, 281, 278, 230, 221, 240, 277, 242, 269, 243, 236, 248, 262, 258, 271, 275, 279, 246, 262, 241, 258, 260, 242, 278, 269, 235, 260, 250, 233, 278, 241, 276, 250, 253, 243, 240, 253, 252, 257, 245, 252, 255, 244, 231, 259, 259, 255, 281, 259, 263, 262, 277, 256, 254, 257, 260, 238, 240, 249, 297, 258, 241, 272, 266, 255, 241, 263, 265, 244, 276, 260, 263] payload_rows_per_dst=[5464, 5408, 5568, 5416, 5502, 5312, 5472, 5462] dup_rows_per_dst=[2660, 2728, 2784, 2678, 2754, 2758, 2776, 2794] combine_rows_per_src=[5464, 5408, 5568, 5416, 5502, 5312, 5472, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38160 dup_rows=21868 dup_rows_remote=19124 dedup_ratio=0.333679 router_tokens_per_expert=[261, 241, 239, 236, 233, 266, 239, 272, 253, 239, 267, 260, 262, 249, 244, 265, 253, 306, 247, 269, 268, 257, 250, 264, 265, 259, 240, 261, 251, 246, 259, 265, 202, 277, 280, 250, 254, 245, 269, 267, 266, 250, 237, 257, 226, 262, 265, 254, 245, 269, 244, 266, 254, 250, 242, 259, 261, 285, 253, 268, 269, 234, 251, 276, 261, 241, 263, 254, 256, 254, 245, 238, 237, 229, 265, 228, 238, 267, 289, 252, 265, 266, 263, 265, 273, 253, 280, 261, 263, 258, 296, 248, 244, 276, 268, 218, 266, 259, 269, 266, 250, 248, 272, 268, 241, 271, 264, 252, 226, 242, 244, 237, 245, 251, 283, 272, 259, 244, 238, 276, 262, 258, 238, 261, 253, 246, 239, 281] payload_rows_per_dst=[5370, 5524, 5418, 5508, 5426, 5530, 5428, 5464] dup_rows_per_dst=[2682, 2796, 2704, 2744, 2608, 2864, 2722, 2748] combine_rows_per_src=[5370, 5524, 5418, 5508, 5426, 5530, 5428, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228040 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156359368 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312490696 weight_movement_bytes=18874368 total_nvlink_bytes=331365064 weight_to_token_ratio=0.060400 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229656 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434136 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638616 weight_movement_bytes=18874368 total_nvlink_bytes=333512984 weight_to_token_ratio=0.059987 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229232 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156589936 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950640 weight_movement_bytes=18874368 total_nvlink_bytes=331825008 weight_to_token_ratio=0.060311 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229744 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156877168 ingress_bytes=175521792 link_bytes=175521792 token_bytes=313524592 weight_movement_bytes=18874368 total_nvlink_bytes=332398960 weight_to_token_ratio=0.060201 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681763449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229864 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073896 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917928 weight_movement_bytes=18874368 total_nvlink_bytes=332792296 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229136 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156532496 ingress_bytes=212926464 link_bytes=212926464 token_bytes=312835856 weight_movement_bytes=56623104 total_nvlink_bytes=369458960 weight_to_token_ratio=0.180999 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040156612e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38356 dup_rows=21578 dup_rows_remote=18834 dedup_ratio=0.329254 router_tokens_per_expert=[260, 276, 242, 242, 264, 249, 241, 252, 249, 259, 271, 270, 230, 227, 246, 286, 260, 265, 251, 253, 264, 248, 257, 250, 253, 267, 272, 236, 273, 255, 256, 260, 260, 268, 266, 256, 275, 234, 254, 232, 243, 270, 258, 260, 232, 247, 266, 244, 246, 274, 236, 268, 270, 235, 265, 251, 251, 294, 233, 280, 253, 252, 256, 247, 282, 259, 257, 256, 233, 252, 250, 274, 256, 258, 267, 234, 270, 260, 290, 256, 244, 245, 250, 256, 254, 267, 272, 273, 269, 257, 244, 250, 252, 261, 245, 262, 238, 248, 261, 249, 252, 277, 263, 276, 224, 225, 268, 253, 256, 229, 249, 247, 254, 286, 255, 285, 218, 270, 248, 285, 272, 235, 262, 237, 240, 284, 265, 242] payload_rows_per_dst=[5500, 5602, 5446, 5486, 5528, 5498, 5396, 5502] dup_rows_per_dst=[2628, 2744, 2684, 2630, 2780, 2704, 2634, 2774] combine_rows_per_src=[5500, 5602, 5446, 5486, 5528, 5498, 5396, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38288 dup_rows=21722 dup_rows_remote=18972 dedup_ratio=0.331451 router_tokens_per_expert=[257, 258, 270, 249, 259, 251, 230, 237, 249, 261, 279, 277, 265, 263, 262, 284, 258, 274, 234, 255, 233, 262, 263, 251, 252, 274, 264, 260, 241, 267, 243, 243, 255, 245, 237, 257, 269, 257, 251, 265, 261, 268, 272, 240, 265, 251, 237, 258, 250, 258, 226, 236, 292, 245, 274, 259, 254, 250, 241, 283, 265, 297, 235, 260, 265, 263, 270, 293, 237, 252, 230, 262, 266, 265, 262, 275, 243, 253, 256, 246, 254, 261, 241, 239, 255, 248, 246, 281, 266, 219, 271, 263, 247, 266, 259, 273, 230, 248, 262, 241, 238, 261, 245, 274, 273, 267, 256, 233, 241, 256, 246, 288, 249, 231, 229, 255, 254, 231, 266, 263, 256, 271, 266, 256, 257, 280, 248, 232] payload_rows_per_dst=[5574, 5500, 5474, 5514, 5526, 5448, 5408, 5370] dup_rows_per_dst=[2728, 2648, 2702, 2736, 2750, 2730, 2710, 2718] combine_rows_per_src=[5574, 5500, 5474, 5514, 5526, 5448, 5408, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=228760 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157334936 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314441112 weight_movement_bytes=18874368 total_nvlink_bytes=333315480 weight_to_token_ratio=0.060025 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38176 dup_rows=21806 dup_rows_remote=18920 dedup_ratio=0.332733 router_tokens_per_expert=[258, 204, 251, 290, 274, 239, 263, 240, 241, 247, 262, 249, 273, 237, 249, 255, 270, 262, 262, 261, 276, 268, 258, 251, 289, 272, 214, 268, 255, 257, 282, 271, 264, 213, 261, 261, 204, 225, 282, 280, 257, 257, 240, 240, 246, 240, 244, 268, 266, 261, 216, 283, 250, 253, 241, 254, 280, 233, 214, 257, 250, 255, 271, 228, 279, 284, 256, 297, 242, 264, 249, 253, 241, 246, 266, 237, 233, 228, 244, 267, 262, 253, 240, 272, 249, 274, 280, 256, 269, 263, 250, 272, 240, 280, 256, 241, 255, 253, 272, 240, 260, 267, 227, 268, 249, 283, 261, 275, 274, 231, 245, 267, 226, 282, 259, 243, 262, 273, 303, 257, 286, 256, 256, 243, 241, 272, 258, 239] payload_rows_per_dst=[5554, 5600, 5384, 5476, 5306, 5464, 5454, 5492] dup_rows_per_dst=[2886, 2844, 2580, 2584, 2490, 2802, 2800, 2820] combine_rows_per_src=[5554, 5600, 5384, 5476, 5306, 5464, 5454, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38378 dup_rows=21630 dup_rows_remote=18898 dedup_ratio=0.330048 router_tokens_per_expert=[280, 251, 272, 283, 266, 234, 230, 263, 285, 257, 231, 255, 259, 258, 250, 267, 244, 255, 279, 251, 256, 236, 280, 257, 233, 257, 262, 235, 229, 257, 281, 242, 259, 253, 228, 241, 259, 227, 215, 255, 256, 256, 252, 244, 252, 267, 264, 235, 274, 243, 249, 240, 253, 267, 271, 273, 249, 232, 256, 262, 286, 250, 276, 263, 248, 267, 245, 283, 266, 252, 269, 255, 254, 259, 251, 246, 258, 267, 268, 255, 272, 246, 268, 270, 255, 262, 260, 260, 271, 268, 294, 263, 252, 259, 241, 269, 253, 252, 241, 254, 270, 280, 249, 240, 258, 253, 252, 220, 247, 239, 253, 231, 256, 261, 234, 261, 259, 255, 253, 262, 232, 288, 293, 269, 230, 268, 246, 254] payload_rows_per_dst=[5568, 5472, 5296, 5524, 5526, 5360, 5632, 5528] dup_rows_per_dst=[2714, 2636, 2630, 2764, 2760, 2504, 2890, 2732] combine_rows_per_src=[5568, 5472, 5296, 5524, 5526, 5360, 5632, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229040 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157056688 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884336 weight_movement_bytes=18874368 total_nvlink_bytes=332758704 weight_to_token_ratio=0.060132 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38174 dup_rows=21870 dup_rows_remote=19138 dedup_ratio=0.333710 router_tokens_per_expert=[268, 252, 278, 265, 283, 265, 251, 232, 245, 251, 268, 244, 267, 263, 248, 254, 245, 258, 243, 274, 243, 263, 260, 250, 262, 264, 215, 262, 259, 258, 259, 297, 257, 279, 254, 271, 230, 258, 244, 236, 285, 272, 251, 283, 256, 241, 248, 247, 266, 263, 251, 258, 233, 222, 252, 267, 210, 249, 243, 241, 247, 285, 261, 237, 261, 255, 268, 266, 275, 248, 276, 251, 247, 227, 245, 256, 241, 271, 226, 245, 274, 262, 245, 266, 286, 278, 237, 247, 251, 243, 264, 264, 259, 260, 284, 284, 279, 288, 260, 226, 252, 236, 279, 250, 251, 259, 253, 252, 274, 223, 267, 213, 228, 253, 265, 262, 245, 263, 274, 278, 253, 281, 248, 229, 270, 240, 260, 252] payload_rows_per_dst=[5488, 5442, 5492, 5352, 5398, 5602, 5478, 5414] dup_rows_per_dst=[2780, 2782, 2732, 2618, 2718, 2806, 2646, 2788] combine_rows_per_src=[5488, 5442, 5492, 5352, 5398, 5602, 5478, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38292 dup_rows=21810 dup_rows_remote=19058 dedup_ratio=0.332794 router_tokens_per_expert=[256, 246, 249, 248, 247, 247, 276, 258, 229, 245, 250, 277, 227, 261, 247, 265, 246, 246, 254, 284, 256, 254, 260, 285, 224, 246, 257, 255, 277, 232, 256, 277, 261, 240, 264, 250, 231, 251, 251, 263, 234, 278, 270, 253, 259, 267, 260, 255, 234, 246, 255, 270, 260, 264, 214, 261, 273, 234, 272, 258, 257, 288, 255, 254, 278, 266, 237, 260, 266, 300, 264, 238, 280, 249, 260, 247, 254, 262, 259, 266, 241, 257, 259, 291, 240, 271, 273, 278, 260, 257, 256, 257, 269, 236, 254, 269, 283, 253, 253, 240, 261, 243, 232, 248, 237, 252, 264, 242, 250, 271, 230, 267, 223, 260, 245, 248, 258, 269, 237, 290, 286, 232, 277, 252, 238, 233, 268, 253] payload_rows_per_dst=[5388, 5450, 5428, 5560, 5576, 5488, 5434, 5402] dup_rows_per_dst=[2668, 2768, 2746, 2630, 2796, 2714, 2752, 2736] combine_rows_per_src=[5388, 5450, 5428, 5560, 5576, 5488, 5434, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38300 dup_rows=21744 dup_rows_remote=18852 dedup_ratio=0.331787 router_tokens_per_expert=[238, 283, 230, 244, 268, 237, 263, 261, 218, 270, 255, 274, 247, 247, 228, 277, 267, 271, 279, 247, 237, 236, 255, 216, 248, 259, 230, 256, 248, 246, 254, 239, 235, 267, 228, 284, 257, 272, 265, 253, 261, 253, 255, 262, 267, 272, 263, 242, 256, 248, 253, 273, 243, 263, 273, 265, 242, 273, 273, 267, 260, 260, 259, 284, 249, 252, 238, 244, 269, 272, 245, 238, 275, 267, 255, 262, 253, 266, 229, 268, 265, 233, 298, 251, 252, 275, 253, 264, 225, 278, 233, 245, 251, 239, 293, 241, 236, 267, 247, 264, 245, 278, 251, 206, 258, 259, 264, 251, 266, 291, 234, 258, 247, 275, 264, 270, 260, 272, 252, 258, 256, 291, 244, 232, 283, 249, 237, 269] payload_rows_per_dst=[5376, 5392, 5558, 5492, 5478, 5512, 5482, 5502] dup_rows_per_dst=[2704, 2584, 2714, 2892, 2686, 2680, 2668, 2816] combine_rows_per_src=[5376, 5392, 5558, 5492, 5478, 5512, 5482, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38208 dup_rows=21886 dup_rows_remote=19120 dedup_ratio=0.333954 router_tokens_per_expert=[255, 271, 248, 273, 277, 233, 219, 250, 236, 249, 250, 263, 256, 253, 238, 248, 245, 255, 247, 249, 271, 259, 240, 248, 264, 276, 266, 265, 254, 277, 253, 250, 265, 274, 236, 257, 267, 277, 243, 250, 231, 232, 261, 255, 259, 261, 277, 250, 259, 265, 240, 253, 259, 271, 273, 259, 257, 243, 255, 256, 280, 255, 248, 222, 252, 262, 244, 261, 269, 268, 256, 256, 293, 251, 256, 255, 264, 257, 248, 248, 243, 222, 246, 257, 262, 266, 244, 250, 237, 257, 284, 271, 253, 272, 261, 279, 251, 270, 243, 246, 245, 293, 270, 242, 264, 254, 251, 266, 281, 248, 234, 279, 277, 239, 261, 259, 254, 235, 248, 285, 280, 236, 231, 259, 266, 236, 258, 235] payload_rows_per_dst=[5418, 5480, 5386, 5474, 5502, 5442, 5490, 5458] dup_rows_per_dst=[2620, 2758, 2804, 2716, 2778, 2766, 2784, 2660] combine_rows_per_src=[5418, 5480, 5386, 5474, 5502, 5442, 5490, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=228384 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156597280 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312966176 weight_movement_bytes=18874368 total_nvlink_bytes=331840544 weight_to_token_ratio=0.060308 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229104 combine_payload_bytes=157196288 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157425392 ingress_bytes=194945024 link_bytes=194945024 token_bytes=314621680 weight_movement_bytes=37748736 total_nvlink_bytes=352370416 weight_to_token_ratio=0.119981 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867867301e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229248 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156589952 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950656 weight_movement_bytes=18874368 total_nvlink_bytes=331825024 weight_to_token_ratio=0.060311 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229400 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073432 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917464 weight_movement_bytes=18874368 total_nvlink_bytes=332791832 weight_to_token_ratio=0.060125 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38076 dup_rows=21976 dup_rows_remote=19250 dedup_ratio=0.335327 router_tokens_per_expert=[272, 241, 290, 256, 264, 247, 261, 270, 256, 261, 254, 281, 262, 254, 236, 251, 270, 260, 268, 233, 263, 232, 249, 257, 273, 264, 261, 276, 250, 256, 231, 262, 253, 248, 271, 265, 245, 237, 245, 254, 273, 250, 258, 258, 251, 265, 266, 239, 286, 208, 262, 258, 239, 247, 245, 265, 274, 257, 249, 255, 272, 262, 285, 252, 250, 266, 244, 253, 245, 261, 238, 241, 267, 221, 257, 267, 243, 245, 274, 256, 250, 247, 267, 244, 251, 250, 268, 246, 246, 242, 257, 253, 266, 254, 232, 265, 248, 254, 245, 243, 276, 221, 253, 220, 260, 270, 240, 290, 290, 227, 244, 276, 245, 238, 255, 253, 261, 259, 246, 245, 288, 276, 271, 267, 275, 252, 298, 261] payload_rows_per_dst=[5526, 5484, 5422, 5474, 5328, 5348, 5444, 5534] dup_rows_per_dst=[2786, 2726, 2734, 2758, 2728, 2728, 2670, 2846] combine_rows_per_src=[5526, 5484, 5422, 5474, 5328, 5348, 5444, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=228608 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157105408 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313982208 weight_movement_bytes=37748736 total_nvlink_bytes=351730944 weight_to_token_ratio=0.120226 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156499968 dispatch_meta_bytes=229312 combine_payload_bytes=156499968 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156729280 ingress_bytes=231997440 link_bytes=231997440 token_bytes=313229248 weight_movement_bytes=75497472 total_nvlink_bytes=388726720 weight_to_token_ratio=0.241029 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222885320e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38250 dup_rows=21854 dup_rows_remote=19162 dedup_ratio=0.333466 router_tokens_per_expert=[254, 260, 229, 244, 284, 225, 252, 274, 262, 223, 262, 274, 263, 269, 264, 278, 238, 254, 284, 237, 248, 259, 277, 246, 274, 251, 230, 279, 257, 258, 277, 263, 258, 273, 240, 232, 267, 238, 258, 276, 251, 267, 254, 262, 254, 270, 219, 249, 273, 254, 252, 279, 246, 243, 230, 260, 283, 280, 234, 259, 276, 230, 283, 277, 251, 276, 258, 258, 282, 261, 227, 230, 233, 261, 255, 248, 262, 285, 230, 245, 251, 263, 252, 276, 279, 279, 260, 283, 266, 250, 250, 240, 232, 270, 249, 271, 278, 235, 226, 270, 250, 268, 242, 271, 248, 236, 280, 272, 202, 258, 260, 259, 221, 250, 253, 258, 262, 268, 229, 251, 237, 243, 274, 237, 250, 231, 278, 262] payload_rows_per_dst=[5484, 5490, 5470, 5500, 5432, 5498, 5400, 5408] dup_rows_per_dst=[2750, 2774, 2666, 2818, 2692, 2844, 2710, 2600] combine_rows_per_src=[5484, 5490, 5470, 5500, 5432, 5498, 5400, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155959296 dispatch_meta_bytes=229304 combine_payload_bytes=155959296 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156188600 ingress_bytes=212582400 link_bytes=212582400 token_bytes=312147896 weight_movement_bytes=56623104 total_nvlink_bytes=368771000 weight_to_token_ratio=0.181398 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036859960e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43492 payload_rows_remote=38104 dup_rows=22044 dup_rows_remote=19284 dedup_ratio=0.336365 router_tokens_per_expert=[275, 228, 257, 266, 221, 252, 246, 242, 240, 258, 284, 260, 254, 244, 262, 240, 254, 251, 240, 246, 258, 247, 268, 257, 259, 225, 251, 261, 257, 260, 222, 265, 256, 282, 265, 266, 247, 255, 268, 269, 256, 259, 244, 252, 269, 230, 250, 287, 257, 272, 258, 257, 230, 248, 290, 277, 279, 255, 258, 234, 272, 246, 266, 265, 302, 254, 248, 246, 252, 269, 270, 268, 273, 285, 251, 271, 261, 236, 256, 221, 286, 233, 264, 249, 239, 262, 269, 279, 267, 238, 224, 264, 244, 255, 257, 250, 251, 250, 270, 247, 256, 253, 255, 264, 263, 264, 245, 240, 254, 243, 275, 262, 247, 259, 241, 248, 278, 276, 247, 239, 257, 233, 245, 253, 259, 276, 242, 264] payload_rows_per_dst=[5374, 5356, 5466, 5548, 5448, 5480, 5432, 5388] dup_rows_per_dst=[2684, 2490, 2844, 2838, 2878, 2818, 2732, 2760] combine_rows_per_src=[5374, 5356, 5466, 5548, 5448, 5480, 5432, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38192 dup_rows=21900 dup_rows_remote=19058 dedup_ratio=0.334167 router_tokens_per_expert=[265, 259, 252, 260, 254, 270, 239, 242, 285, 240, 271, 236, 256, 273, 226, 241, 247, 247, 252, 237, 286, 255, 249, 260, 253, 245, 254, 243, 241, 243, 241, 266, 261, 285, 251, 275, 243, 238, 236, 236, 267, 252, 256, 262, 257, 258, 266, 277, 279, 259, 264, 245, 225, 279, 269, 263, 273, 259, 252, 246, 273, 239, 244, 265, 275, 245, 257, 248, 264, 281, 263, 254, 241, 263, 254, 259, 263, 281, 246, 305, 251, 257, 243, 270, 223, 243, 248, 249, 247, 289, 273, 248, 263, 252, 255, 243, 267, 256, 261, 270, 232, 257, 279, 238, 274, 239, 229, 270, 249, 231, 226, 254, 271, 257, 265, 253, 242, 249, 286, 272, 257, 256, 236, 280, 240, 259, 256, 262] payload_rows_per_dst=[5444, 5452, 5564, 5456, 5534, 5560, 5396, 5230] dup_rows_per_dst=[2842, 2642, 2832, 2760, 2860, 2858, 2668, 2438] combine_rows_per_src=[5444, 5452, 5564, 5456, 5534, 5560, 5396, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229648 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156901648 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313573648 weight_movement_bytes=37748736 total_nvlink_bytes=351322384 weight_to_token_ratio=0.120382 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43860 payload_rows_remote=38418 dup_rows=21676 dup_rows_remote=19010 dedup_ratio=0.330750 router_tokens_per_expert=[276, 269, 250, 243, 270, 253, 271, 253, 268, 264, 277, 242, 238, 231, 243, 270, 253, 246, 258, 254, 244, 267, 287, 246, 247, 244, 228, 264, 264, 281, 260, 246, 253, 227, 244, 257, 261, 264, 268, 257, 220, 249, 268, 253, 247, 260, 272, 254, 287, 241, 254, 222, 250, 242, 243, 280, 275, 257, 273, 286, 246, 265, 258, 301, 240, 240, 242, 265, 259, 261, 271, 259, 276, 256, 268, 267, 235, 250, 267, 250, 229, 259, 236, 269, 278, 246, 250, 273, 264, 238, 247, 245, 225, 297, 266, 254, 261, 243, 266, 282, 251, 224, 245, 242, 263, 273, 276, 230, 283, 248, 267, 237, 225, 253, 291, 244, 242, 235, 249, 281, 254, 245, 272, 259, 245, 266, 231, 262] payload_rows_per_dst=[5538, 5492, 5442, 5578, 5446, 5456, 5478, 5430] dup_rows_per_dst=[2698, 2686, 2666, 2782, 2766, 2696, 2704, 2678] combine_rows_per_src=[5538, 5492, 5442, 5578, 5446, 5456, 5478, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38408 dup_rows=21752 dup_rows_remote=18976 dedup_ratio=0.331909 router_tokens_per_expert=[240, 249, 248, 271, 237, 249, 276, 257, 261, 233, 230, 260, 272, 261, 281, 251, 248, 241, 265, 270, 264, 262, 262, 232, 248, 264, 275, 272, 270, 228, 246, 292, 254, 258, 242, 268, 278, 230, 251, 261, 255, 235, 261, 231, 250, 289, 271, 268, 266, 268, 294, 270, 246, 286, 246, 255, 224, 264, 294, 289, 251, 248, 238, 270, 238, 259, 223, 259, 252, 272, 255, 245, 240, 238, 230, 259, 242, 249, 257, 250, 250, 260, 239, 243, 232, 263, 276, 250, 237, 259, 251, 254, 237, 261, 285, 252, 242, 253, 263, 269, 252, 256, 237, 278, 279, 232, 278, 270, 237, 239, 255, 236, 265, 240, 291, 256, 244, 253, 267, 270, 256, 264, 259, 240, 266, 258, 257, 263] payload_rows_per_dst=[5448, 5558, 5524, 5546, 5406, 5404, 5376, 5522] dup_rows_per_dst=[2704, 2720, 2680, 2872, 2530, 2694, 2776, 2776] combine_rows_per_src=[5448, 5558, 5524, 5546, 5406, 5404, 5376, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38454 dup_rows=21670 dup_rows_remote=18950 dedup_ratio=0.330658 router_tokens_per_expert=[259, 262, 229, 260, 271, 270, 265, 258, 268, 246, 256, 257, 243, 250, 264, 265, 266, 254, 221, 249, 277, 270, 288, 247, 272, 253, 277, 259, 269, 244, 257, 261, 242, 220, 252, 235, 272, 252, 256, 284, 240, 288, 228, 273, 249, 250, 257, 245, 255, 261, 258, 268, 265, 263, 268, 245, 275, 220, 237, 256, 247, 226, 252, 270, 251, 292, 239, 237, 261, 263, 248, 265, 265, 258, 269, 242, 267, 224, 274, 264, 256, 244, 245, 245, 252, 283, 240, 266, 269, 227, 260, 265, 249, 240, 246, 254, 242, 253, 266, 255, 279, 246, 233, 248, 250, 252, 257, 270, 262, 250, 248, 248, 263, 260, 261, 246, 257, 247, 241, 266, 257, 267, 250, 273, 251, 266, 278, 270] payload_rows_per_dst=[5566, 5558, 5432, 5412, 5524, 5364, 5452, 5558] dup_rows_per_dst=[2680, 2770, 2654, 2720, 2714, 2718, 2666, 2748] combine_rows_per_src=[5566, 5558, 5432, 5412, 5524, 5364, 5452, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=229000 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156663432 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313097864 weight_movement_bytes=18874368 total_nvlink_bytes=331972232 weight_to_token_ratio=0.060283 gemm_rows_defined=33408 gemm_rows_charged=36864 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.093750 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38316 dup_rows=21746 dup_rows_remote=19042 dedup_ratio=0.331818 router_tokens_per_expert=[266, 265, 216, 258, 277, 250, 241, 269, 216, 253, 265, 247, 281, 263, 228, 252, 253, 266, 259, 264, 244, 252, 266, 240, 267, 239, 272, 249, 255, 260, 259, 256, 239, 247, 253, 271, 269, 254, 254, 253, 254, 245, 250, 257, 245, 237, 266, 263, 262, 252, 233, 266, 268, 270, 263, 226, 231, 263, 247, 236, 258, 291, 235, 274, 229, 224, 237, 248, 242, 251, 204, 278, 238, 270, 264, 302, 276, 277, 240, 255, 249, 261, 275, 257, 235, 284, 232, 214, 273, 267, 269, 245, 254, 276, 249, 249, 269, 261, 275, 263, 275, 251, 284, 239, 269, 258, 278, 282, 266, 247, 283, 243, 243, 251, 235, 275, 250, 263, 264, 261, 268, 273, 271, 243, 236, 266, 260, 262] payload_rows_per_dst=[5460, 5534, 5480, 5448, 5402, 5474, 5584, 5408] dup_rows_per_dst=[2634, 2668, 2634, 2702, 2668, 2704, 2902, 2834] combine_rows_per_src=[5460, 5534, 5480, 5448, 5402, 5474, 5584, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229552 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156303536 ingress_bytes=212697088 link_bytes=212697088 token_bytes=312377520 weight_movement_bytes=56623104 total_nvlink_bytes=369000624 weight_to_token_ratio=0.181265 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037958844e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229712 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157589840 ingress_bytes=213983232 link_bytes=213983232 token_bytes=314949968 weight_movement_bytes=56623104 total_nvlink_bytes=371573072 weight_to_token_ratio=0.179784 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050282043e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157319168 dispatch_meta_bytes=229536 combine_payload_bytes=157319168 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157548704 ingress_bytes=195067904 link_bytes=195067904 token_bytes=314867872 weight_movement_bytes=37748736 total_nvlink_bytes=352616608 weight_to_token_ratio=0.119888 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869044677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157507584 dispatch_meta_bytes=229616 combine_payload_bytes=157507584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157737200 ingress_bytes=195256320 link_bytes=195256320 token_bytes=315244784 weight_movement_bytes=37748736 total_nvlink_bytes=352993520 weight_to_token_ratio=0.119744 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870849986e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38228 dup_rows=21868 dup_rows_remote=19024 dedup_ratio=0.333679 router_tokens_per_expert=[263, 232, 259, 262, 240, 264, 282, 261, 284, 278, 266, 253, 263, 252, 250, 285, 295, 255, 241, 260, 271, 279, 259, 240, 279, 265, 268, 235, 230, 241, 254, 270, 250, 255, 230, 259, 265, 280, 243, 259, 273, 249, 278, 289, 258, 262, 260, 256, 235, 241, 278, 239, 258, 237, 226, 262, 224, 239, 254, 252, 270, 262, 276, 279, 235, 241, 242, 261, 249, 261, 258, 264, 243, 262, 244, 267, 218, 259, 272, 250, 257, 236, 277, 247, 226, 284, 250, 260, 252, 265, 251, 230, 253, 246, 263, 236, 251, 271, 255, 279, 278, 273, 254, 242, 239, 261, 251, 248, 267, 254, 261, 244, 264, 259, 259, 249, 241, 255, 269, 233, 243, 274, 245, 252, 241, 250, 245, 268] payload_rows_per_dst=[5566, 5440, 5560, 5382, 5402, 5394, 5458, 5466] dup_rows_per_dst=[2822, 2844, 2772, 2682, 2650, 2672, 2798, 2628] combine_rows_per_src=[5566, 5440, 5560, 5382, 5402, 5394, 5458, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229432 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171768 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314114104 weight_movement_bytes=18874368 total_nvlink_bytes=332988472 weight_to_token_ratio=0.060088 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38344 dup_rows=21628 dup_rows_remote=18838 dedup_ratio=0.330017 router_tokens_per_expert=[250, 263, 255, 265, 251, 232, 268, 239, 257, 259, 230, 259, 231, 254, 261, 252, 266, 266, 256, 241, 285, 258, 250, 260, 272, 276, 267, 270, 245, 267, 260, 251, 259, 261, 277, 240, 250, 243, 238, 253, 259, 279, 241, 229, 242, 257, 245, 256, 258, 256, 233, 258, 240, 239, 283, 280, 270, 244, 274, 243, 259, 271, 285, 259, 270, 264, 245, 262, 248, 281, 264, 264, 264, 260, 274, 251, 261, 267, 254, 248, 270, 274, 264, 256, 259, 229, 241, 268, 253, 272, 233, 253, 246, 244, 261, 271, 262, 262, 280, 278, 275, 278, 237, 254, 250, 261, 256, 257, 225, 242, 265, 229, 229, 250, 265, 259, 235, 240, 240, 242, 257, 248, 250, 259, 245, 255, 256, 259] payload_rows_per_dst=[5418, 5488, 5458, 5562, 5564, 5468, 5524, 5426] dup_rows_per_dst=[2634, 2892, 2600, 2742, 2790, 2720, 2698, 2552] combine_rows_per_src=[5418, 5488, 5458, 5562, 5564, 5468, 5524, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=229008 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156810896 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313392784 weight_movement_bytes=18874368 total_nvlink_bytes=332267152 weight_to_token_ratio=0.060226 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38168 dup_rows=21754 dup_rows_remote=18860 dedup_ratio=0.331940 router_tokens_per_expert=[258, 259, 256, 289, 283, 257, 234, 262, 259, 260, 234, 268, 237, 268, 283, 244, 276, 239, 242, 247, 254, 276, 246, 262, 234, 269, 238, 244, 269, 254, 251, 260, 255, 268, 269, 252, 243, 263, 257, 264, 277, 240, 255, 255, 223, 268, 251, 250, 236, 237, 301, 280, 250, 247, 226, 292, 261, 268, 225, 244, 235, 259, 269, 248, 270, 243, 248, 269, 243, 286, 251, 272, 263, 279, 248, 250, 250, 232, 280, 263, 248, 276, 261, 258, 270, 263, 242, 244, 245, 271, 248, 234, 267, 273, 250, 268, 272, 225, 254, 267, 265, 239, 263, 260, 245, 238, 261, 279, 271, 252, 258, 235, 257, 256, 234, 279, 270, 237, 275, 237, 259, 265, 236, 232, 227, 251, 248, 276] payload_rows_per_dst=[5614, 5480, 5448, 5360, 5506, 5468, 5452, 5454] dup_rows_per_dst=[2894, 2642, 2732, 2590, 2788, 2768, 2716, 2624] combine_rows_per_src=[5614, 5480, 5448, 5360, 5506, 5468, 5452, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38232 dup_rows=21796 dup_rows_remote=19216 dedup_ratio=0.332581 router_tokens_per_expert=[255, 257, 268, 260, 249, 238, 265, 255, 243, 259, 235, 267, 246, 244, 252, 238, 272, 268, 265, 251, 256, 261, 256, 276, 258, 255, 252, 269, 258, 252, 264, 295, 281, 257, 263, 238, 257, 235, 231, 263, 285, 203, 259, 252, 260, 271, 269, 250, 233, 276, 259, 231, 255, 270, 239, 263, 271, 256, 247, 251, 263, 280, 260, 246, 263, 235, 285, 242, 232, 255, 232, 273, 257, 280, 286, 251, 258, 249, 250, 279, 244, 254, 267, 263, 233, 254, 256, 270, 267, 257, 225, 290, 249, 267, 247, 242, 262, 258, 269, 261, 273, 251, 263, 230, 229, 267, 254, 258, 238, 268, 265, 253, 238, 245, 255, 227, 232, 237, 269, 264, 270, 260, 273, 278, 240, 244, 238, 274] payload_rows_per_dst=[5398, 5570, 5402, 5376, 5326, 5638, 5522, 5508] dup_rows_per_dst=[2664, 2846, 2524, 2646, 2604, 3078, 2854, 2580] combine_rows_per_src=[5398, 5570, 5402, 5376, 5326, 5638, 5522, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=228728 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157285752 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314342776 weight_movement_bytes=18874368 total_nvlink_bytes=333217144 weight_to_token_ratio=0.060044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38242 dup_rows=21822 dup_rows_remote=19056 dedup_ratio=0.332977 router_tokens_per_expert=[225, 291, 241, 232, 273, 262, 272, 263, 242, 244, 287, 247, 281, 250, 282, 259, 256, 249, 238, 261, 247, 252, 253, 265, 263, 229, 250, 251, 275, 256, 255, 269, 242, 270, 266, 232, 237, 260, 247, 255, 244, 291, 276, 293, 225, 255, 274, 252, 266, 246, 263, 251, 227, 280, 241, 251, 229, 267, 274, 241, 250, 273, 280, 241, 276, 224, 232, 260, 262, 240, 277, 254, 248, 238, 275, 274, 258, 267, 254, 247, 278, 251, 233, 270, 267, 256, 229, 261, 236, 262, 263, 268, 252, 261, 257, 241, 253, 250, 239, 242, 278, 241, 247, 205, 269, 252, 263, 232, 251, 240, 251, 279, 283, 267, 264, 258, 264, 226, 251, 276, 291, 253, 271, 263, 281, 273, 232, 233] payload_rows_per_dst=[5500, 5456, 5472, 5504, 5434, 5432, 5364, 5552] dup_rows_per_dst=[2802, 2682, 2766, 2656, 2738, 2738, 2620, 2820] combine_rows_per_src=[5500, 5456, 5472, 5504, 5434, 5432, 5364, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38190 dup_rows=21892 dup_rows_remote=19254 dedup_ratio=0.334045 router_tokens_per_expert=[230, 255, 262, 256, 248, 273, 263, 267, 240, 256, 254, 243, 264, 244, 262, 229, 239, 231, 244, 237, 256, 260, 255, 259, 260, 262, 256, 259, 254, 286, 258, 257, 270, 237, 287, 289, 274, 261, 265, 260, 265, 251, 265, 268, 245, 250, 248, 245, 267, 268, 236, 254, 261, 259, 248, 251, 266, 255, 224, 261, 269, 243, 237, 247, 260, 242, 236, 258, 259, 258, 285, 262, 258, 268, 263, 248, 268, 238, 240, 247, 274, 277, 273, 237, 243, 248, 254, 252, 267, 265, 261, 244, 255, 271, 240, 265, 257, 239, 228, 229, 277, 226, 261, 292, 242, 263, 268, 258, 249, 290, 268, 269, 241, 242, 269, 256, 271, 247, 266, 252, 248, 263, 265, 256, 250, 270, 238, 257] payload_rows_per_dst=[5368, 5408, 5564, 5454, 5428, 5494, 5520, 5408] dup_rows_per_dst=[2724, 2738, 2796, 2638, 2752, 2758, 2712, 2774] combine_rows_per_src=[5368, 5408, 5564, 5454, 5428, 5494, 5520, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38490 dup_rows=21672 dup_rows_remote=19122 dedup_ratio=0.330688 router_tokens_per_expert=[239, 243, 243, 250, 265, 241, 248, 241, 257, 233, 261, 247, 289, 272, 261, 246, 268, 260, 251, 258, 289, 243, 259, 269, 248, 271, 261, 261, 272, 238, 274, 247, 260, 266, 262, 289, 256, 251, 233, 253, 265, 256, 264, 217, 242, 279, 279, 263, 246, 245, 254, 238, 241, 243, 274, 273, 268, 239, 247, 261, 254, 282, 262, 254, 253, 243, 281, 258, 241, 247, 267, 262, 269, 273, 288, 275, 255, 271, 253, 252, 255, 252, 238, 259, 239, 258, 249, 258, 267, 269, 259, 247, 243, 285, 275, 221, 260, 265, 216, 255, 263, 285, 201, 246, 229, 230, 242, 267, 236, 229, 264, 274, 258, 262, 249, 260, 266, 256, 247, 265, 261, 279, 274, 269, 226, 264, 221, 266] payload_rows_per_dst=[5444, 5508, 5510, 5428, 5576, 5538, 5374, 5486] dup_rows_per_dst=[2628, 2830, 2760, 2734, 2782, 2628, 2550, 2760] combine_rows_per_src=[5444, 5508, 5510, 5428, 5576, 5538, 5374, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228112 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156564240 ingress_bytes=175210496 link_bytes=175210496 token_bytes=312900368 weight_movement_bytes=18874368 total_nvlink_bytes=331774736 weight_to_token_ratio=0.060321 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678780764e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38350 dup_rows=21802 dup_rows_remote=19222 dedup_ratio=0.332672 router_tokens_per_expert=[247, 253, 250, 230, 267, 287, 255, 275, 284, 260, 239, 255, 267, 263, 243, 253, 235, 242, 255, 277, 249, 268, 281, 221, 250, 259, 249, 250, 260, 236, 252, 260, 281, 278, 254, 248, 247, 268, 249, 247, 246, 260, 280, 235, 280, 267, 278, 246, 263, 226, 276, 260, 246, 260, 285, 268, 271, 260, 246, 283, 230, 263, 262, 255, 235, 252, 252, 228, 265, 275, 280, 253, 266, 238, 239, 255, 261, 229, 251, 253, 253, 241, 241, 260, 260, 233, 232, 274, 266, 257, 271, 241, 241, 252, 236, 224, 227, 274, 252, 262, 230, 244, 251, 227, 250, 270, 267, 266, 245, 282, 269, 250, 273, 245, 267, 263, 250, 263, 251, 270, 254, 285, 270, 241, 297, 271, 252, 246] payload_rows_per_dst=[5468, 5396, 5540, 5602, 5462, 5384, 5400, 5482] dup_rows_per_dst=[2788, 2692, 2788, 2706, 2602, 2580, 2732, 2914] combine_rows_per_src=[5468, 5396, 5540, 5602, 5462, 5384, 5400, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229792 combine_payload_bytes=156598272 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156828064 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313426336 weight_movement_bytes=37748736 total_nvlink_bytes=351175072 weight_to_token_ratio=0.120439 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229192 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868424 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507656 weight_movement_bytes=18874368 total_nvlink_bytes=332382024 weight_to_token_ratio=0.060204 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157655040 dispatch_meta_bytes=230448 combine_payload_bytes=157655040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157885488 ingress_bytes=176529408 link_bytes=176529408 token_bytes=315540528 weight_movement_bytes=18874368 total_nvlink_bytes=334414896 weight_to_token_ratio=0.059816 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691417930e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43866 payload_rows_remote=38320 dup_rows=21670 dup_rows_remote=18860 dedup_ratio=0.330658 router_tokens_per_expert=[241, 242, 274, 274, 262, 260, 254, 228, 239, 246, 242, 231, 254, 254, 251, 277, 262, 261, 269, 266, 250, 271, 263, 255, 255, 250, 267, 286, 265, 247, 270, 241, 284, 271, 268, 245, 261, 262, 249, 244, 264, 261, 270, 272, 239, 249, 243, 242, 251, 253, 277, 250, 272, 264, 237, 264, 270, 290, 281, 238, 228, 261, 251, 264, 249, 239, 261, 240, 226, 255, 238, 253, 268, 261, 238, 247, 249, 267, 271, 254, 265, 244, 237, 265, 254, 268, 265, 245, 258, 244, 261, 255, 246, 251, 278, 242, 262, 247, 284, 259, 250, 248, 270, 249, 260, 250, 269, 287, 238, 304, 230, 259, 231, 257, 237, 250, 266, 258, 237, 266, 246, 278, 255, 256, 243, 252, 265, 229] payload_rows_per_dst=[5436, 5546, 5422, 5566, 5366, 5516, 5504, 5510] dup_rows_per_dst=[2622, 2810, 2826, 2736, 2666, 2640, 2828, 2542] combine_rows_per_src=[5436, 5546, 5422, 5566, 5366, 5516, 5504, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229776 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156656016 ingress_bytes=213049344 link_bytes=213049344 token_bytes=313082256 weight_movement_bytes=56623104 total_nvlink_bytes=369705360 weight_to_token_ratio=0.180857 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041333987e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38226 dup_rows=21768 dup_rows_remote=18922 dedup_ratio=0.332153 router_tokens_per_expert=[246, 254, 260, 247, 265, 262, 266, 243, 277, 264, 249, 268, 243, 247, 217, 251, 213, 235, 273, 230, 235, 241, 239, 268, 243, 265, 263, 251, 228, 264, 267, 268, 256, 239, 240, 258, 257, 252, 300, 233, 263, 246, 248, 260, 262, 254, 275, 265, 255, 237, 264, 280, 278, 242, 277, 256, 262, 248, 248, 237, 252, 263, 239, 271, 253, 224, 259, 245, 243, 275, 267, 277, 252, 268, 246, 282, 278, 272, 264, 289, 263, 237, 247, 242, 233, 271, 244, 268, 244, 280, 246, 268, 257, 272, 260, 245, 252, 275, 256, 272, 269, 245, 235, 264, 278, 254, 256, 269, 252, 266, 267, 225, 250, 256, 270, 249, 256, 232, 308, 272, 269, 234, 244, 257, 257, 254, 252, 243] payload_rows_per_dst=[5446, 5368, 5494, 5530, 5542, 5420, 5458, 5510] dup_rows_per_dst=[2672, 2598, 2722, 2688, 2846, 2734, 2812, 2696] combine_rows_per_src=[5446, 5368, 5494, 5530, 5542, 5420, 5458, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=230288 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311888 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314393488 weight_movement_bytes=18874368 total_nvlink_bytes=333267856 weight_to_token_ratio=0.060034 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228720 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157187440 ingress_bytes=175833088 link_bytes=175833088 token_bytes=314146160 weight_movement_bytes=18874368 total_nvlink_bytes=333020528 weight_to_token_ratio=0.060081 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684746134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=228592 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156802288 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313375984 weight_movement_bytes=37748736 total_nvlink_bytes=351124720 weight_to_token_ratio=0.120458 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38004 dup_rows=21886 dup_rows_remote=18920 dedup_ratio=0.333954 router_tokens_per_expert=[270, 256, 244, 254, 279, 277, 281, 262, 274, 252, 234, 231, 227, 223, 270, 241, 237, 240, 252, 215, 230, 222, 261, 286, 247, 274, 261, 274, 249, 225, 257, 239, 253, 251, 264, 229, 279, 273, 254, 274, 298, 271, 239, 232, 265, 250, 252, 274, 277, 238, 242, 234, 265, 231, 287, 237, 250, 274, 259, 249, 225, 271, 248, 249, 270, 253, 235, 261, 249, 256, 281, 257, 254, 267, 251, 255, 250, 289, 251, 255, 260, 250, 260, 265, 269, 258, 247, 267, 255, 239, 283, 266, 248, 264, 234, 295, 257, 234, 278, 260, 262, 239, 274, 248, 253, 271, 262, 268, 254, 244, 272, 257, 242, 274, 260, 282, 272, 240, 274, 267, 226, 237, 273, 248, 272, 236, 266, 234] payload_rows_per_dst=[5646, 5536, 5558, 5268, 5314, 5464, 5460, 5404] dup_rows_per_dst=[2966, 2898, 2922, 2414, 2520, 2664, 2806, 2696] combine_rows_per_src=[5646, 5536, 5558, 5268, 5314, 5464, 5460, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43390 payload_rows_remote=37932 dup_rows=22146 dup_rows_remote=19418 dedup_ratio=0.337921 router_tokens_per_expert=[264, 276, 264, 284, 239, 266, 236, 243, 249, 238, 247, 257, 282, 290, 244, 234, 256, 263, 239, 224, 239, 258, 268, 278, 262, 262, 219, 264, 293, 276, 295, 255, 249, 254, 266, 255, 249, 273, 282, 253, 261, 263, 264, 238, 244, 242, 295, 250, 245, 266, 255, 276, 260, 249, 272, 267, 253, 238, 230, 263, 264, 230, 271, 275, 261, 232, 285, 267, 233, 266, 278, 264, 241, 236, 265, 244, 234, 261, 266, 261, 247, 250, 275, 234, 264, 244, 227, 252, 260, 235, 253, 266, 247, 287, 251, 246, 255, 263, 247, 268, 292, 248, 257, 254, 269, 269, 282, 252, 246, 253, 246, 246, 271, 225, 244, 256, 257, 271, 220, 247, 266, 235, 265, 241, 237, 244, 248, 246] payload_rows_per_dst=[5408, 5530, 5498, 5170, 5566, 5372, 5388, 5458] dup_rows_per_dst=[2818, 2772, 2778, 2568, 3112, 2702, 2668, 2728] combine_rows_per_src=[5408, 5530, 5498, 5170, 5566, 5372, 5388, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38268 dup_rows=21836 dup_rows_remote=19170 dedup_ratio=0.333191 router_tokens_per_expert=[257, 245, 251, 240, 256, 273, 236, 255, 260, 219, 243, 273, 265, 237, 249, 269, 242, 277, 252, 242, 236, 267, 268, 226, 274, 249, 257, 271, 266, 262, 281, 251, 259, 273, 247, 255, 266, 230, 253, 253, 268, 252, 256, 240, 271, 252, 254, 220, 228, 277, 263, 267, 266, 254, 260, 265, 257, 256, 268, 229, 259, 253, 252, 273, 251, 260, 268, 290, 263, 241, 269, 256, 259, 263, 237, 267, 271, 280, 249, 260, 260, 252, 253, 248, 276, 230, 238, 224, 241, 250, 237, 242, 245, 256, 272, 269, 243, 240, 253, 267, 222, 213, 228, 287, 267, 266, 271, 298, 262, 282, 240, 249, 242, 253, 281, 269, 262, 272, 262, 295, 258, 242, 263, 283, 237, 251, 244, 264] payload_rows_per_dst=[5462, 5430, 5432, 5436, 5514, 5438, 5492, 5496] dup_rows_per_dst=[2594, 2812, 2666, 2818, 2854, 2548, 2684, 2860] combine_rows_per_src=[5462, 5430, 5432, 5436, 5514, 5438, 5492, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38410 dup_rows=21644 dup_rows_remote=18912 dedup_ratio=0.330261 router_tokens_per_expert=[265, 286, 246, 239, 242, 276, 257, 238, 273, 251, 246, 278, 262, 279, 233, 265, 259, 250, 272, 259, 254, 270, 272, 271, 264, 221, 216, 264, 274, 234, 228, 231, 254, 266, 250, 244, 238, 252, 264, 269, 275, 283, 245, 281, 237, 238, 279, 242, 270, 272, 271, 265, 264, 266, 250, 275, 234, 259, 233, 268, 269, 270, 223, 257, 263, 269, 272, 227, 266, 243, 232, 255, 261, 237, 250, 255, 248, 241, 261, 245, 231, 227, 233, 289, 258, 273, 243, 276, 256, 261, 265, 263, 257, 289, 262, 263, 245, 235, 270, 246, 260, 258, 249, 270, 268, 261, 261, 237, 256, 254, 254, 283, 215, 253, 268, 242, 251, 253, 228, 272, 240, 288, 254, 262, 283, 243, 255, 245] payload_rows_per_dst=[5560, 5402, 5520, 5442, 5512, 5546, 5482, 5428] dup_rows_per_dst=[2712, 2676, 2714, 2808, 2580, 2746, 2732, 2676] combine_rows_per_src=[5560, 5402, 5520, 5442, 5512, 5546, 5482, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38300 dup_rows=21836 dup_rows_remote=19068 dedup_ratio=0.333191 router_tokens_per_expert=[239, 253, 247, 257, 247, 233, 272, 290, 260, 226, 253, 238, 276, 249, 250, 257, 241, 276, 254, 243, 254, 236, 271, 231, 233, 266, 253, 244, 270, 254, 264, 244, 239, 263, 275, 245, 238, 269, 251, 249, 258, 250, 238, 271, 233, 235, 256, 258, 284, 239, 226, 254, 248, 238, 273, 249, 257, 263, 264, 263, 259, 245, 269, 253, 253, 268, 279, 243, 246, 253, 261, 255, 244, 262, 257, 250, 266, 242, 241, 251, 254, 241, 244, 268, 238, 276, 239, 284, 258, 261, 241, 264, 257, 275, 267, 262, 286, 261, 259, 267, 253, 219, 287, 248, 265, 240, 264, 254, 264, 247, 291, 270, 271, 260, 257, 282, 272, 252, 269, 262, 262, 258, 259, 255, 247, 270, 238, 286] payload_rows_per_dst=[5442, 5438, 5442, 5400, 5454, 5426, 5552, 5546] dup_rows_per_dst=[2652, 2630, 2614, 2768, 2688, 2832, 2798, 2854] combine_rows_per_src=[5442, 5438, 5442, 5400, 5454, 5426, 5552, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155664384 dispatch_meta_bytes=227696 combine_payload_bytes=155664384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155892080 ingress_bytes=174538752 link_bytes=174538752 token_bytes=311556464 weight_movement_bytes=18874368 total_nvlink_bytes=330430832 weight_to_token_ratio=0.060581 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672344443e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229752 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156975480 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313721208 weight_movement_bytes=37748736 total_nvlink_bytes=351469944 weight_to_token_ratio=0.120326 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38246 dup_rows=21838 dup_rows_remote=19046 dedup_ratio=0.333221 router_tokens_per_expert=[238, 233, 261, 289, 254, 236, 262, 229, 254, 266, 251, 290, 244, 274, 252, 272, 254, 237, 270, 264, 239, 262, 248, 258, 242, 246, 265, 242, 263, 243, 269, 223, 253, 264, 263, 269, 238, 267, 237, 272, 244, 252, 241, 249, 265, 250, 251, 259, 271, 268, 269, 266, 256, 298, 286, 269, 268, 261, 231, 240, 249, 258, 254, 256, 270, 231, 254, 252, 264, 244, 246, 259, 254, 260, 249, 239, 261, 255, 258, 253, 250, 261, 253, 263, 254, 253, 250, 278, 263, 248, 257, 267, 246, 253, 256, 270, 284, 258, 257, 251, 239, 248, 256, 254, 257, 232, 272, 257, 256, 261, 276, 274, 235, 256, 249, 243, 277, 246, 239, 247, 278, 234, 234, 249, 267, 246, 284, 277] payload_rows_per_dst=[5442, 5380, 5426, 5550, 5456, 5452, 5536, 5456] dup_rows_per_dst=[2768, 2670, 2722, 2850, 2642, 2792, 2728, 2666] combine_rows_per_src=[5442, 5380, 5426, 5550, 5456, 5452, 5536, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229288 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157556648 ingress_bytes=195076096 link_bytes=195076096 token_bytes=314884008 weight_movement_bytes=37748736 total_nvlink_bytes=352632744 weight_to_token_ratio=0.119881 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869123168e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=155369472 dispatch_meta_bytes=229400 combine_payload_bytes=155369472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155598872 ingress_bytes=174243840 link_bytes=174243840 token_bytes=310968344 weight_movement_bytes=18874368 total_nvlink_bytes=329842712 weight_to_token_ratio=0.060695 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669518742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43520 payload_rows_remote=38126 dup_rows=22016 dup_rows_remote=19278 dedup_ratio=0.335938 router_tokens_per_expert=[239, 269, 261, 257, 237, 258, 269, 235, 243, 265, 266, 277, 255, 246, 274, 275, 250, 253, 252, 243, 250, 272, 270, 289, 250, 264, 245, 252, 264, 239, 240, 233, 280, 275, 232, 248, 233, 275, 260, 236, 298, 248, 250, 265, 292, 271, 273, 242, 266, 281, 259, 252, 272, 217, 234, 272, 211, 231, 281, 260, 253, 262, 245, 241, 283, 279, 267, 237, 226, 266, 263, 233, 240, 233, 241, 248, 249, 265, 265, 226, 230, 255, 250, 248, 251, 261, 227, 269, 270, 271, 257, 248, 257, 253, 247, 253, 244, 279, 259, 248, 302, 245, 280, 257, 253, 245, 289, 245, 240, 265, 261, 254, 277, 265, 237, 243, 255, 262, 243, 285, 261, 242, 251, 268, 272, 245, 277, 244] payload_rows_per_dst=[5520, 5394, 5508, 5364, 5350, 5404, 5500, 5480] dup_rows_per_dst=[2732, 2738, 2848, 2710, 2692, 2690, 2832, 2774] combine_rows_per_src=[5520, 5394, 5508, 5364, 5350, 5404, 5500, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=229472 combine_payload_bytes=156876800 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157106272 ingress_bytes=194625536 link_bytes=194625536 token_bytes=313983072 weight_movement_bytes=37748736 total_nvlink_bytes=351731808 weight_to_token_ratio=0.120225 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864806124e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43552 payload_rows_remote=38116 dup_rows=21984 dup_rows_remote=19314 dedup_ratio=0.335449 router_tokens_per_expert=[271, 251, 226, 242, 273, 251, 259, 253, 287, 230, 234, 271, 256, 238, 268, 259, 278, 274, 266, 243, 258, 240, 244, 261, 266, 263, 268, 251, 261, 256, 243, 266, 260, 279, 273, 227, 256, 240, 235, 267, 251, 262, 247, 232, 259, 261, 267, 252, 264, 231, 244, 256, 262, 255, 248, 264, 280, 262, 259, 270, 279, 243, 272, 245, 267, 266, 263, 267, 219, 232, 265, 224, 257, 264, 271, 241, 239, 243, 272, 263, 256, 248, 282, 269, 277, 259, 269, 257, 234, 249, 267, 243, 275, 250, 262, 268, 251, 267, 251, 233, 242, 228, 257, 259, 236, 277, 238, 258, 260, 280, 281, 263, 243, 272, 256, 239, 275, 265, 246, 252, 255, 253, 267, 256, 237, 256, 241, 247] payload_rows_per_dst=[5438, 5438, 5422, 5496, 5436, 5550, 5414, 5358] dup_rows_per_dst=[2700, 2838, 2714, 2772, 2670, 2780, 2748, 2762] combine_rows_per_src=[5438, 5438, 5422, 5496, 5436, 5550, 5414, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229168 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156884784 ingress_bytes=251027456 link_bytes=251027456 token_bytes=313540400 weight_movement_bytes=94371840 total_nvlink_bytes=407912240 weight_to_token_ratio=0.300988 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.405221570e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229616 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156393712 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557808 weight_movement_bytes=18874368 total_nvlink_bytes=331432176 weight_to_token_ratio=0.060387 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229720 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352856 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475992 weight_movement_bytes=18874368 total_nvlink_bytes=331350360 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38296 dup_rows=21804 dup_rows_remote=19020 dedup_ratio=0.332703 router_tokens_per_expert=[261, 285, 256, 255, 251, 248, 250, 246, 236, 281, 272, 274, 253, 246, 256, 252, 274, 235, 241, 251, 240, 281, 222, 239, 260, 249, 260, 275, 260, 241, 262, 246, 263, 262, 261, 254, 234, 278, 242, 255, 252, 271, 243, 237, 262, 274, 270, 252, 250, 268, 263, 254, 267, 239, 268, 267, 252, 273, 246, 236, 230, 271, 279, 241, 270, 250, 250, 254, 244, 249, 226, 255, 246, 255, 265, 238, 269, 261, 252, 241, 258, 261, 253, 250, 257, 251, 273, 264, 227, 261, 250, 275, 252, 269, 271, 254, 271, 264, 245, 249, 246, 261, 264, 253, 268, 247, 239, 236, 232, 258, 251, 238, 259, 277, 273, 280, 259, 247, 237, 262, 256, 256, 261, 267, 280, 260, 303, 246] payload_rows_per_dst=[5426, 5440, 5436, 5578, 5412, 5478, 5416, 5546] dup_rows_per_dst=[2818, 2632, 2784, 2630, 2638, 2774, 2628, 2900] combine_rows_per_src=[5426, 5440, 5436, 5578, 5412, 5478, 5416, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38306 dup_rows=21694 dup_rows_remote=18876 dedup_ratio=0.331024 router_tokens_per_expert=[252, 235, 248, 275, 252, 221, 272, 285, 253, 246, 222, 264, 259, 252, 263, 241, 265, 242, 251, 253, 259, 252, 240, 241, 228, 246, 253, 257, 247, 261, 261, 270, 264, 248, 255, 252, 262, 261, 277, 233, 223, 268, 284, 265, 275, 259, 224, 298, 277, 244, 256, 247, 276, 255, 263, 243, 238, 267, 257, 241, 250, 277, 263, 255, 259, 249, 240, 257, 263, 249, 231, 256, 243, 203, 246, 269, 268, 257, 271, 276, 271, 241, 258, 264, 242, 257, 249, 232, 266, 275, 262, 276, 249, 272, 249, 241, 253, 235, 248, 274, 252, 245, 239, 252, 247, 293, 260, 250, 229, 276, 272, 261, 271, 282, 264, 265, 297, 245, 236, 235, 274, 231, 288, 270, 283, 262, 251, 264] payload_rows_per_dst=[5536, 5458, 5548, 5372, 5622, 5458, 5464, 5384] dup_rows_per_dst=[2818, 2610, 2836, 2572, 2818, 2750, 2692, 2598] combine_rows_per_src=[5536, 5458, 5548, 5372, 5622, 5458, 5464, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38330 dup_rows=21740 dup_rows_remote=18998 dedup_ratio=0.331726 router_tokens_per_expert=[255, 253, 286, 259, 248, 248, 288, 279, 256, 293, 252, 242, 270, 266, 257, 228, 260, 267, 271, 274, 233, 246, 242, 248, 253, 256, 256, 262, 273, 239, 245, 291, 261, 261, 263, 251, 230, 252, 249, 267, 252, 257, 249, 255, 272, 253, 271, 294, 275, 246, 279, 261, 257, 264, 247, 240, 252, 252, 246, 261, 260, 249, 224, 244, 248, 283, 276, 239, 254, 261, 254, 246, 262, 270, 245, 232, 280, 246, 276, 259, 245, 246, 263, 245, 250, 265, 257, 254, 224, 255, 244, 254, 239, 260, 251, 241, 240, 237, 249, 255, 221, 270, 268, 263, 249, 247, 266, 248, 252, 261, 268, 256, 246, 280, 222, 248, 264, 271, 254, 261, 276, 251, 274, 235, 265, 247, 249, 261] payload_rows_per_dst=[5516, 5552, 5492, 5346, 5576, 5456, 5392, 5466] dup_rows_per_dst=[2844, 2680, 2782, 2622, 2832, 2530, 2708, 2742] combine_rows_per_src=[5516, 5552, 5492, 5346, 5576, 5456, 5392, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38232 dup_rows=21808 dup_rows_remote=19088 dedup_ratio=0.332764 router_tokens_per_expert=[304, 280, 267, 256, 253, 261, 248, 257, 253, 255, 251, 242, 249, 230, 259, 247, 248, 235, 265, 243, 226, 246, 281, 256, 245, 258, 234, 247, 223, 286, 232, 259, 258, 289, 267, 264, 262, 258, 273, 267, 249, 241, 255, 269, 238, 283, 249, 247, 278, 266, 260, 261, 242, 284, 231, 260, 230, 286, 253, 265, 253, 270, 255, 279, 255, 244, 253, 268, 227, 229, 280, 236, 253, 269, 256, 259, 251, 246, 232, 255, 224, 248, 244, 265, 226, 254, 260, 266, 270, 230, 253, 258, 274, 246, 272, 266, 261, 271, 260, 243, 269, 254, 275, 242, 274, 255, 244, 264, 238, 254, 262, 242, 262, 259, 262, 251, 271, 275, 247, 262, 254, 253, 289, 251, 249, 289, 229, 250] payload_rows_per_dst=[5444, 5418, 5566, 5498, 5366, 5450, 5496, 5490] dup_rows_per_dst=[2780, 2550, 2772, 2848, 2660, 2662, 2720, 2816] combine_rows_per_src=[5444, 5418, 5566, 5498, 5366, 5450, 5496, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38078 dup_rows=21944 dup_rows_remote=19102 dedup_ratio=0.334839 router_tokens_per_expert=[256, 289, 247, 263, 276, 220, 259, 249, 251, 260, 280, 253, 235, 237, 247, 241, 233, 212, 242, 273, 296, 254, 245, 274, 249, 258, 293, 263, 256, 255, 267, 262, 285, 243, 268, 259, 251, 243, 246, 246, 245, 269, 286, 276, 244, 253, 242, 255, 260, 241, 280, 272, 285, 262, 278, 223, 265, 269, 260, 229, 242, 256, 280, 276, 236, 257, 277, 240, 262, 252, 257, 273, 251, 265, 263, 252, 244, 265, 227, 230, 265, 236, 286, 251, 252, 279, 263, 245, 284, 236, 265, 260, 259, 249, 273, 259, 242, 253, 251, 274, 249, 253, 250, 237, 252, 286, 252, 230, 260, 253, 250, 258, 222, 239, 244, 257, 262, 259, 273, 255, 244, 236, 252, 263, 270, 244, 261, 240] payload_rows_per_dst=[5448, 5436, 5458, 5514, 5438, 5516, 5422, 5360] dup_rows_per_dst=[2678, 2828, 2764, 2842, 2664, 2808, 2678, 2682] combine_rows_per_src=[5448, 5436, 5458, 5514, 5438, 5516, 5422, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229264 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157089680 ingress_bytes=213483520 link_bytes=213483520 token_bytes=313950096 weight_movement_bytes=56623104 total_nvlink_bytes=370573200 weight_to_token_ratio=0.180357 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045494048e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38282 dup_rows=21832 dup_rows_remote=19146 dedup_ratio=0.333130 router_tokens_per_expert=[264, 226, 259, 269, 242, 248, 252, 254, 252, 263, 254, 226, 279, 256, 238, 247, 257, 257, 246, 232, 259, 268, 246, 254, 269, 279, 261, 249, 244, 287, 282, 260, 270, 233, 252, 255, 267, 275, 261, 239, 259, 249, 226, 272, 253, 231, 256, 237, 262, 238, 239, 242, 260, 288, 243, 268, 257, 270, 261, 266, 270, 239, 264, 249, 289, 283, 260, 259, 276, 266, 265, 254, 231, 233, 251, 247, 273, 262, 242, 263, 269, 257, 238, 253, 246, 250, 258, 250, 255, 255, 240, 276, 274, 255, 234, 244, 260, 267, 257, 256, 255, 238, 258, 231, 259, 250, 246, 282, 271, 252, 243, 277, 231, 261, 262, 253, 236, 251, 243, 293, 292, 258, 264, 260, 242, 253, 258, 271] payload_rows_per_dst=[5452, 5500, 5426, 5410, 5510, 5422, 5492, 5492] dup_rows_per_dst=[2606, 2800, 2644, 2822, 2798, 2686, 2712, 2764] combine_rows_per_src=[5452, 5500, 5426, 5410, 5510, 5422, 5492, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229312 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157228992 ingress_bytes=213622784 link_bytes=213622784 token_bytes=314228672 weight_movement_bytes=56623104 total_nvlink_bytes=370851776 weight_to_token_ratio=0.180197 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046828407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228728 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157130104 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314031480 weight_movement_bytes=18874368 total_nvlink_bytes=332905848 weight_to_token_ratio=0.060103 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229280 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156827552 ingress_bytes=213221376 link_bytes=213221376 token_bytes=313425824 weight_movement_bytes=56623104 total_nvlink_bytes=370048928 weight_to_token_ratio=0.180659 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042982313e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38336 dup_rows=21660 dup_rows_remote=18884 dedup_ratio=0.330505 router_tokens_per_expert=[251, 228, 259, 221, 258, 254, 249, 262, 257, 285, 267, 261, 256, 273, 230, 289, 252, 247, 259, 277, 256, 284, 277, 260, 260, 274, 260, 223, 265, 248, 254, 262, 268, 224, 241, 262, 255, 270, 226, 267, 241, 260, 251, 248, 251, 250, 263, 256, 248, 268, 261, 257, 272, 256, 264, 241, 263, 286, 246, 253, 256, 239, 268, 262, 290, 266, 252, 237, 279, 260, 251, 226, 244, 269, 266, 246, 253, 253, 265, 240, 243, 231, 267, 266, 238, 266, 281, 254, 271, 266, 259, 245, 249, 262, 245, 226, 278, 238, 255, 244, 283, 242, 274, 280, 268, 237, 238, 246, 252, 240, 246, 250, 255, 239, 260, 270, 263, 243, 261, 254, 259, 241, 282, 263, 250, 235, 271, 254] payload_rows_per_dst=[5534, 5540, 5422, 5502, 5516, 5496, 5406, 5460] dup_rows_per_dst=[2666, 2776, 2644, 2778, 2678, 2642, 2736, 2740] combine_rows_per_src=[5534, 5540, 5422, 5502, 5516, 5496, 5406, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=155967488 dispatch_meta_bytes=228720 combine_payload_bytes=155967488 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156196208 ingress_bytes=193716224 link_bytes=193716224 token_bytes=312163696 weight_movement_bytes=37748736 total_nvlink_bytes=349912432 weight_to_token_ratio=0.120926 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856093544e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38374 dup_rows=21710 dup_rows_remote=18956 dedup_ratio=0.331268 router_tokens_per_expert=[251, 252, 256, 264, 260, 242, 246, 271, 238, 280, 228, 254, 282, 267, 253, 255, 256, 250, 282, 230, 250, 236, 256, 266, 223, 248, 237, 264, 250, 241, 248, 243, 232, 276, 256, 272, 228, 270, 261, 232, 276, 277, 247, 232, 274, 248, 280, 264, 250, 241, 236, 245, 241, 237, 266, 261, 272, 248, 245, 280, 249, 260, 255, 241, 285, 246, 259, 256, 257, 266, 279, 232, 252, 227, 252, 274, 265, 255, 252, 246, 248, 228, 263, 239, 262, 282, 261, 298, 270, 254, 241, 256, 251, 266, 271, 233, 247, 263, 283, 259, 253, 240, 267, 254, 270, 241, 275, 244, 271, 262, 259, 290, 247, 265, 263, 252, 280, 231, 288, 245, 259, 271, 260, 260, 261, 236, 266, 249] payload_rows_per_dst=[5412, 5468, 5554, 5402, 5452, 5532, 5570, 5436] dup_rows_per_dst=[2786, 2492, 2696, 2652, 2754, 2714, 2786, 2830] combine_rows_per_src=[5412, 5468, 5554, 5402, 5452, 5532, 5570, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229712 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032784 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835856 weight_movement_bytes=18874368 total_nvlink_bytes=332710224 weight_to_token_ratio=0.060141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=228880 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157253136 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314277392 weight_movement_bytes=18874368 total_nvlink_bytes=333151760 weight_to_token_ratio=0.060056 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229320 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409224 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589128 weight_movement_bytes=18874368 total_nvlink_bytes=333463496 weight_to_token_ratio=0.059997 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38234 dup_rows=21828 dup_rows_remote=18972 dedup_ratio=0.333069 router_tokens_per_expert=[271, 226, 255, 274, 256, 251, 283, 253, 270, 286, 281, 256, 263, 260, 243, 242, 232, 254, 226, 245, 250, 244, 268, 238, 255, 211, 262, 231, 247, 273, 240, 258, 258, 270, 235, 234, 243, 271, 263, 253, 275, 247, 280, 277, 270, 290, 244, 255, 281, 230, 260, 263, 266, 244, 267, 231, 256, 275, 253, 229, 275, 256, 258, 283, 274, 242, 264, 256, 235, 250, 230, 276, 264, 243, 248, 243, 268, 240, 271, 245, 247, 251, 268, 251, 287, 273, 250, 256, 260, 252, 266, 262, 265, 264, 268, 231, 281, 263, 268, 272, 244, 268, 244, 243, 233, 268, 231, 248, 222, 244, 253, 288, 244, 251, 227, 266, 274, 268, 240, 265, 227, 258, 234, 240, 286, 263, 280, 279] payload_rows_per_dst=[5560, 5356, 5474, 5518, 5454, 5476, 5458, 5412] dup_rows_per_dst=[2780, 2512, 2856, 2736, 2644, 2826, 2682, 2792] combine_rows_per_src=[5560, 5356, 5474, 5518, 5454, 5476, 5458, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38122 dup_rows=21882 dup_rows_remote=19146 dedup_ratio=0.333893 router_tokens_per_expert=[266, 236, 257, 273, 246, 279, 234, 266, 253, 260, 260, 265, 269, 279, 232, 261, 267, 253, 253, 267, 266, 258, 257, 235, 255, 255, 258, 242, 253, 257, 266, 257, 277, 294, 257, 236, 240, 230, 269, 262, 240, 275, 248, 258, 225, 222, 225, 244, 283, 237, 263, 243, 257, 266, 256, 265, 243, 216, 272, 226, 277, 268, 273, 250, 270, 298, 251, 239, 244, 269, 243, 267, 251, 275, 278, 232, 260, 263, 278, 249, 257, 235, 282, 233, 267, 230, 257, 257, 256, 266, 256, 253, 257, 234, 246, 252, 267, 287, 270, 256, 267, 255, 241, 268, 256, 240, 250, 236, 261, 268, 250, 270, 269, 259, 256, 263, 277, 247, 242, 231, 226, 263, 238, 251, 278, 273, 243, 273] payload_rows_per_dst=[5368, 5404, 5278, 5370, 5700, 5386, 5616, 5532] dup_rows_per_dst=[2500, 2794, 2372, 2764, 3038, 2690, 2988, 2736] combine_rows_per_src=[5368, 5404, 5278, 5370, 5700, 5386, 5616, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38230 dup_rows=21800 dup_rows_remote=19076 dedup_ratio=0.332642 router_tokens_per_expert=[245, 266, 245, 269, 248, 247, 263, 261, 238, 267, 271, 257, 277, 260, 247, 251, 236, 255, 260, 279, 234, 261, 247, 230, 253, 255, 255, 257, 241, 225, 253, 253, 263, 296, 260, 250, 257, 273, 277, 255, 257, 273, 253, 260, 248, 248, 244, 246, 267, 251, 243, 257, 246, 272, 259, 261, 264, 260, 244, 274, 239, 282, 230, 242, 250, 244, 274, 269, 231, 255, 250, 238, 245, 266, 274, 241, 251, 282, 266, 260, 252, 264, 253, 237, 253, 271, 269, 245, 257, 263, 257, 259, 245, 260, 254, 288, 286, 237, 263, 265, 234, 260, 247, 262, 249, 263, 263, 254, 268, 252, 268, 244, 258, 274, 249, 264, 249, 224, 271, 255, 265, 264, 287, 257, 246, 232, 238, 240] payload_rows_per_dst=[5500, 5358, 5562, 5458, 5484, 5450, 5506, 5418] dup_rows_per_dst=[2724, 2630, 2758, 2724, 2708, 2804, 2724, 2728] combine_rows_per_src=[5500, 5358, 5562, 5458, 5484, 5450, 5506, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38200 dup_rows=21842 dup_rows_remote=19024 dedup_ratio=0.333282 router_tokens_per_expert=[256, 255, 258, 242, 260, 269, 254, 270, 221, 245, 250, 260, 267, 253, 276, 242, 249, 247, 271, 236, 273, 258, 240, 238, 241, 233, 271, 251, 267, 238, 248, 269, 243, 244, 259, 264, 270, 253, 260, 251, 241, 240, 269, 247, 236, 233, 264, 274, 233, 251, 262, 266, 277, 237, 255, 250, 289, 271, 282, 280, 250, 252, 258, 265, 275, 271, 254, 263, 251, 281, 253, 269, 255, 244, 253, 262, 280, 264, 248, 253, 259, 244, 234, 244, 221, 276, 258, 237, 269, 255, 240, 266, 279, 237, 275, 239, 250, 274, 265, 249, 257, 238, 251, 238, 259, 250, 292, 275, 258, 253, 227, 246, 263, 239, 275, 250, 284, 237, 249, 261, 265, 248, 269, 286, 242, 236, 264, 275] payload_rows_per_dst=[5494, 5446, 5410, 5474, 5474, 5434, 5436, 5526] dup_rows_per_dst=[2818, 2674, 2586, 2836, 2878, 2632, 2728, 2690] combine_rows_per_src=[5494, 5446, 5410, 5474, 5474, 5434, 5436, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=228824 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156835288 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313441752 weight_movement_bytes=37748736 total_nvlink_bytes=351190488 weight_to_token_ratio=0.120433 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38220 dup_rows=21908 dup_rows_remote=19144 dedup_ratio=0.334290 router_tokens_per_expert=[238, 260, 229, 256, 246, 252, 247, 265, 264, 268, 269, 237, 262, 235, 243, 267, 240, 249, 252, 244, 247, 268, 273, 271, 250, 267, 256, 246, 248, 244, 262, 270, 238, 258, 268, 276, 281, 272, 242, 281, 258, 243, 243, 277, 283, 288, 272, 246, 253, 254, 221, 265, 250, 237, 247, 256, 268, 258, 277, 249, 279, 256, 261, 255, 246, 258, 269, 247, 261, 239, 265, 246, 237, 265, 276, 252, 248, 230, 279, 235, 255, 223, 265, 246, 276, 232, 283, 271, 270, 243, 229, 271, 280, 274, 258, 263, 257, 232, 254, 249, 290, 223, 262, 255, 260, 267, 270, 255, 241, 278, 275, 244, 229, 245, 253, 251, 249, 270, 240, 247, 250, 241, 254, 247, 273, 228, 271, 279] payload_rows_per_dst=[5416, 5444, 5628, 5408, 5414, 5520, 5420, 5378] dup_rows_per_dst=[2660, 2730, 2824, 2764, 2692, 2758, 2804, 2676] combine_rows_per_src=[5416, 5444, 5628, 5408, 5414, 5520, 5420, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38152 dup_rows=21854 dup_rows_remote=19132 dedup_ratio=0.333466 router_tokens_per_expert=[268, 267, 261, 287, 241, 229, 248, 253, 258, 241, 249, 240, 251, 230, 293, 264, 238, 239, 256, 259, 279, 269, 279, 282, 264, 262, 276, 238, 271, 219, 244, 261, 258, 264, 284, 257, 269, 252, 256, 262, 263, 259, 235, 245, 251, 242, 275, 260, 278, 247, 250, 260, 261, 260, 235, 277, 255, 271, 245, 275, 258, 251, 266, 265, 237, 233, 236, 271, 254, 240, 259, 246, 279, 255, 241, 262, 267, 237, 238, 269, 257, 237, 263, 253, 238, 288, 247, 260, 250, 279, 271, 257, 256, 266, 255, 249, 243, 232, 260, 243, 264, 252, 262, 246, 253, 234, 212, 262, 259, 237, 279, 266, 233, 245, 278, 268, 241, 242, 239, 298, 247, 234, 265, 246, 268, 254, 301, 253] payload_rows_per_dst=[5488, 5528, 5476, 5540, 5392, 5530, 5312, 5416] dup_rows_per_dst=[2672, 2744, 2788, 2768, 2656, 2722, 2696, 2808] combine_rows_per_src=[5488, 5528, 5476, 5540, 5392, 5530, 5312, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=229072 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156376784 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312524496 weight_movement_bytes=18874368 total_nvlink_bytes=331398864 weight_to_token_ratio=0.060393 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=229224 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156819304 ingress_bytes=175464448 link_bytes=175464448 token_bytes=313409384 weight_movement_bytes=18874368 total_nvlink_bytes=332283752 weight_to_token_ratio=0.060223 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681214007e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38180 dup_rows=21888 dup_rows_remote=19146 dedup_ratio=0.333984 router_tokens_per_expert=[275, 259, 262, 242, 259, 240, 270, 252, 275, 241, 252, 266, 269, 297, 243, 237, 264, 265, 261, 259, 218, 266, 245, 291, 269, 251, 227, 231, 262, 275, 269, 242, 211, 232, 267, 256, 264, 266, 238, 281, 269, 242, 250, 273, 247, 260, 252, 248, 236, 253, 276, 271, 269, 252, 223, 257, 237, 222, 274, 253, 266, 269, 268, 260, 261, 250, 267, 263, 283, 240, 251, 245, 281, 270, 241, 256, 256, 269, 247, 239, 244, 265, 272, 230, 271, 267, 256, 242, 286, 267, 258, 264, 235, 267, 253, 253, 241, 251, 255, 284, 255, 239, 253, 237, 265, 281, 274, 247, 258, 271, 248, 242, 241, 235, 253, 251, 235, 255, 255, 246, 291, 254, 264, 261, 283, 236, 239, 243] payload_rows_per_dst=[5548, 5468, 5392, 5456, 5548, 5422, 5454, 5360] dup_rows_per_dst=[2730, 2742, 2720, 2716, 2690, 2818, 2748, 2724] combine_rows_per_src=[5548, 5468, 5392, 5456, 5548, 5422, 5454, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228896 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156696096 ingress_bytes=175341568 link_bytes=175341568 token_bytes=313163296 weight_movement_bytes=18874368 total_nvlink_bytes=332037664 weight_to_token_ratio=0.060270 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680036631e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38466 dup_rows=21696 dup_rows_remote=19092 dedup_ratio=0.331055 router_tokens_per_expert=[273, 275, 272, 281, 246, 266, 273, 270, 235, 273, 262, 261, 248, 278, 261, 253, 232, 236, 252, 262, 262, 241, 275, 264, 249, 226, 278, 240, 269, 282, 244, 252, 268, 272, 246, 264, 242, 229, 237, 274, 266, 259, 286, 277, 257, 254, 270, 284, 264, 243, 250, 260, 243, 245, 280, 250, 248, 264, 259, 236, 268, 235, 253, 289, 258, 226, 250, 256, 245, 244, 236, 215, 270, 257, 264, 258, 255, 266, 226, 263, 233, 277, 238, 260, 260, 263, 249, 267, 288, 276, 243, 255, 254, 246, 269, 245, 249, 253, 241, 279, 252, 254, 279, 232, 263, 231, 244, 225, 259, 273, 231, 272, 255, 275, 248, 265, 266, 266, 225, 238, 272, 263, 265, 262, 250, 230, 241, 235] payload_rows_per_dst=[5612, 5438, 5532, 5446, 5374, 5542, 5468, 5428] dup_rows_per_dst=[2842, 2690, 2838, 2728, 2604, 2704, 2606, 2684] combine_rows_per_src=[5612, 5438, 5532, 5446, 5374, 5542, 5468, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229456 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156778576 ingress_bytes=213172224 link_bytes=213172224 token_bytes=313327696 weight_movement_bytes=56623104 total_nvlink_bytes=369950800 weight_to_token_ratio=0.180715 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042511363e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229136 combine_payload_bytes=156270592 prefetch_bytes=0 reduce_bytes=0 egress_bytes=156499728 ingress_bytes=156270592 link_bytes=156499728 token_bytes=312770320 weight_movement_bytes=0 total_nvlink_bytes=312770320 weight_to_token_ratio=0.000000 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.499503391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229304 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614584 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999864 weight_movement_bytes=18874368 total_nvlink_bytes=331874232 weight_to_token_ratio=0.060302 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43556 payload_rows_remote=38164 dup_rows=21980 dup_rows_remote=19276 dedup_ratio=0.335388 router_tokens_per_expert=[258, 231, 266, 271, 259, 253, 251, 251, 264, 254, 263, 242, 270, 258, 268, 252, 273, 270, 248, 255, 232, 250, 212, 236, 238, 277, 250, 248, 239, 244, 254, 252, 238, 228, 253, 261, 253, 265, 265, 260, 255, 246, 268, 249, 255, 246, 261, 245, 271, 254, 252, 245, 277, 271, 264, 259, 267, 254, 240, 278, 250, 270, 239, 301, 249, 283, 231, 265, 255, 262, 253, 232, 255, 262, 260, 265, 276, 258, 250, 240, 266, 256, 216, 251, 235, 270, 252, 257, 261, 233, 267, 261, 255, 253, 276, 254, 235, 242, 266, 242, 236, 249, 271, 290, 289, 284, 241, 254, 267, 243, 279, 254, 245, 272, 254, 257, 252, 245, 268, 278, 250, 251, 247, 277, 272, 262, 254, 254] payload_rows_per_dst=[5520, 5328, 5392, 5472, 5418, 5436, 5486, 5504] dup_rows_per_dst=[2702, 2628, 2704, 2912, 2774, 2690, 2798, 2772] combine_rows_per_src=[5520, 5328, 5392, 5472, 5418, 5436, 5486, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=230232 combine_payload_bytes=157556736 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157786968 ingress_bytes=176431104 link_bytes=176431104 token_bytes=315343704 weight_movement_bytes=18874368 total_nvlink_bytes=334218072 weight_to_token_ratio=0.059853 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690476029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43604 payload_rows_remote=38032 dup_rows=21932 dup_rows_remote=18974 dedup_ratio=0.334656 router_tokens_per_expert=[250, 252, 275, 272, 258, 263, 255, 258, 261, 230, 276, 261, 253, 238, 269, 252, 238, 263, 240, 230, 267, 263, 247, 272, 245, 261, 242, 243, 237, 259, 267, 263, 275, 237, 249, 253, 261, 242, 257, 236, 270, 239, 248, 236, 232, 276, 246, 273, 281, 240, 252, 272, 261, 249, 275, 271, 265, 267, 278, 255, 243, 264, 259, 290, 263, 250, 232, 237, 246, 249, 268, 231, 289, 243, 249, 266, 244, 271, 263, 258, 283, 272, 251, 247, 259, 267, 242, 255, 256, 244, 236, 263, 267, 232, 251, 245, 258, 253, 252, 249, 236, 244, 295, 250, 229, 295, 250, 274, 258, 250, 275, 273, 235, 253, 246, 241, 235, 266, 279, 265, 250, 254, 279, 230, 252, 275, 270, 256] payload_rows_per_dst=[5404, 5360, 5366, 5430, 5550, 5488, 5434, 5572] dup_rows_per_dst=[2596, 2714, 2548, 2656, 2814, 2798, 2848, 2958] combine_rows_per_src=[5404, 5360, 5366, 5430, 5550, 5488, 5434, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38378 dup_rows=21640 dup_rows_remote=18914 dedup_ratio=0.330200 router_tokens_per_expert=[232, 243, 231, 275, 229, 260, 285, 249, 267, 250, 234, 272, 266, 266, 276, 252, 267, 283, 266, 250, 276, 237, 244, 266, 276, 230, 244, 248, 281, 276, 253, 261, 260, 248, 235, 259, 254, 253, 258, 222, 252, 256, 239, 264, 273, 251, 257, 258, 262, 262, 277, 253, 246, 258, 235, 252, 239, 268, 271, 267, 268, 245, 231, 300, 267, 259, 254, 243, 262, 286, 287, 258, 251, 244, 275, 248, 233, 273, 254, 256, 271, 262, 253, 252, 241, 245, 259, 244, 265, 246, 246, 252, 273, 223, 240, 261, 253, 257, 265, 250, 255, 238, 264, 266, 274, 255, 251, 273, 288, 243, 249, 241, 251, 251, 263, 234, 251, 259, 223, 242, 274, 275, 246, 246, 249, 243, 258, 280] payload_rows_per_dst=[5540, 5516, 5446, 5472, 5508, 5438, 5518, 5458] dup_rows_per_dst=[2634, 2800, 2632, 2796, 2792, 2628, 2726, 2632] combine_rows_per_src=[5540, 5516, 5446, 5472, 5508, 5438, 5518, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229760 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549504 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312869248 weight_movement_bytes=18874368 total_nvlink_bytes=331743616 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38126 dup_rows=21934 dup_rows_remote=19174 dedup_ratio=0.334686 router_tokens_per_expert=[273, 236, 254, 240, 251, 264, 235, 287, 241, 256, 269, 263, 278, 239, 265, 235, 260, 287, 263, 237, 270, 260, 268, 239, 232, 240, 261, 288, 251, 246, 261, 255, 240, 265, 241, 250, 275, 251, 268, 260, 235, 247, 270, 255, 250, 259, 268, 236, 270, 255, 279, 275, 248, 270, 270, 234, 266, 257, 239, 230, 237, 269, 250, 264, 268, 246, 242, 264, 275, 241, 241, 240, 247, 256, 268, 273, 254, 242, 261, 290, 277, 232, 236, 252, 250, 271, 245, 251, 257, 282, 279, 237, 254, 261, 264, 248, 259, 258, 244, 255, 261, 250, 256, 256, 267, 246, 263, 249, 270, 280, 241, 232, 256, 238, 221, 246, 253, 241, 235, 244, 286, 268, 286, 248, 260, 244, 287, 277] payload_rows_per_dst=[5476, 5478, 5472, 5446, 5414, 5412, 5482, 5422] dup_rows_per_dst=[2760, 2790, 2728, 2804, 2802, 2600, 2692, 2758] combine_rows_per_src=[5476, 5478, 5472, 5446, 5414, 5412, 5482, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38394 dup_rows=21694 dup_rows_remote=19000 dedup_ratio=0.331024 router_tokens_per_expert=[269, 260, 264, 245, 256, 242, 244, 242, 268, 263, 234, 212, 278, 246, 249, 288, 253, 255, 281, 275, 242, 294, 257, 263, 261, 238, 264, 255, 252, 250, 276, 231, 261, 249, 274, 244, 278, 261, 251, 264, 249, 256, 280, 257, 254, 244, 213, 269, 266, 245, 273, 242, 270, 259, 256, 257, 234, 246, 260, 250, 272, 259, 242, 240, 259, 268, 281, 235, 251, 279, 269, 273, 233, 256, 242, 274, 253, 241, 244, 255, 252, 268, 267, 273, 232, 257, 244, 262, 245, 264, 254, 277, 252, 274, 277, 239, 268, 230, 234, 254, 283, 272, 250, 244, 263, 253, 256, 248, 246, 249, 237, 268, 254, 242, 256, 242, 224, 282, 261, 251, 252, 255, 236, 280, 265, 274, 279, 228] payload_rows_per_dst=[5470, 5476, 5532, 5448, 5458, 5496, 5456, 5506] dup_rows_per_dst=[2650, 2818, 2676, 2694, 2768, 2778, 2654, 2656] combine_rows_per_src=[5470, 5476, 5532, 5448, 5458, 5496, 5456, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=228024 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156007096 ingress_bytes=174653440 link_bytes=174653440 token_bytes=311786168 weight_movement_bytes=18874368 total_nvlink_bytes=330660536 weight_to_token_ratio=0.060536 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673443327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38318 dup_rows=21666 dup_rows_remote=18858 dedup_ratio=0.330597 router_tokens_per_expert=[260, 239, 266, 253, 254, 259, 239, 273, 255, 239, 264, 241, 269, 236, 254, 233, 251, 234, 249, 244, 265, 234, 257, 248, 240, 252, 265, 284, 271, 267, 232, 267, 274, 243, 283, 278, 263, 269, 261, 226, 267, 248, 272, 242, 259, 237, 259, 256, 249, 243, 261, 262, 251, 259, 238, 260, 236, 240, 283, 236, 276, 278, 219, 242, 245, 242, 237, 249, 231, 244, 231, 259, 248, 264, 295, 255, 251, 253, 244, 278, 294, 267, 275, 234, 255, 282, 256, 235, 272, 252, 260, 258, 272, 243, 266, 259, 250, 257, 268, 279, 262, 274, 269, 251, 273, 265, 270, 258, 258, 243, 268, 206, 260, 286, 241, 251, 257, 255, 244, 258, 251, 252, 290, 273, 242, 276, 239, 272] payload_rows_per_dst=[5458, 5484, 5506, 5358, 5414, 5552, 5518, 5580] dup_rows_per_dst=[2610, 2636, 2768, 2708, 2638, 2808, 2784, 2714] combine_rows_per_src=[5458, 5484, 5506, 5358, 5414, 5552, 5518, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38066 dup_rows=21888 dup_rows_remote=19030 dedup_ratio=0.333984 router_tokens_per_expert=[249, 257, 274, 251, 250, 269, 260, 265, 243, 264, 241, 237, 266, 233, 250, 256, 276, 229, 260, 238, 234, 262, 247, 285, 256, 252, 256, 295, 241, 286, 284, 262, 239, 276, 258, 244, 283, 211, 270, 267, 264, 265, 245, 242, 258, 254, 272, 259, 271, 259, 231, 241, 234, 258, 266, 230, 262, 270, 233, 258, 269, 274, 237, 245, 259, 242, 254, 287, 251, 224, 263, 257, 265, 249, 270, 267, 253, 262, 274, 281, 253, 255, 259, 236, 254, 281, 271, 267, 261, 240, 273, 247, 240, 266, 275, 240, 247, 246, 248, 272, 272, 245, 271, 246, 254, 257, 264, 240, 246, 257, 247, 222, 252, 228, 242, 231, 286, 251, 251, 246, 261, 259, 244, 277, 284, 257, 263, 253] payload_rows_per_dst=[5470, 5582, 5478, 5354, 5448, 5484, 5372, 5460] dup_rows_per_dst=[2660, 2858, 2736, 2722, 2754, 2752, 2696, 2710] combine_rows_per_src=[5470, 5582, 5478, 5354, 5448, 5484, 5372, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157196288 dispatch_meta_bytes=229168 combine_payload_bytes=157196288 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157425456 ingress_bytes=213819392 link_bytes=213819392 token_bytes=314621744 weight_movement_bytes=56623104 total_nvlink_bytes=371244848 weight_to_token_ratio=0.179972 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048712208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=229200 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156393296 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557392 weight_movement_bytes=18874368 total_nvlink_bytes=331431760 weight_to_token_ratio=0.060387 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157261824 dispatch_meta_bytes=229576 combine_payload_bytes=157261824 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157491400 ingress_bytes=195010560 link_bytes=195010560 token_bytes=314753224 weight_movement_bytes=37748736 total_nvlink_bytes=352501960 weight_to_token_ratio=0.119931 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868495235e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38324 dup_rows=21808 dup_rows_remote=19230 dedup_ratio=0.332764 router_tokens_per_expert=[279, 236, 266, 248, 263, 268, 252, 246, 238, 239, 266, 250, 274, 278, 264, 267, 263, 256, 241, 262, 244, 259, 256, 255, 244, 268, 241, 259, 281, 282, 235, 240, 265, 264, 286, 253, 237, 254, 256, 270, 263, 293, 261, 255, 251, 250, 270, 248, 262, 245, 273, 244, 277, 282, 254, 272, 252, 261, 260, 244, 265, 236, 265, 245, 248, 267, 252, 235, 241, 265, 255, 231, 263, 257, 234, 246, 242, 268, 259, 228, 219, 255, 241, 279, 244, 247, 244, 283, 295, 259, 255, 278, 264, 223, 244, 259, 288, 234, 268, 261, 251, 256, 241, 222, 238, 256, 271, 264, 272, 271, 258, 266, 268, 256, 239, 260, 254, 260, 256, 247, 262, 233, 247, 231, 252, 257, 257, 259] payload_rows_per_dst=[5478, 5460, 5462, 5532, 5404, 5482, 5476, 5434] dup_rows_per_dst=[2790, 2712, 2890, 2742, 2578, 2696, 2758, 2642] combine_rows_per_src=[5478, 5460, 5462, 5532, 5404, 5482, 5476, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228704 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157179232 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314129760 weight_movement_bytes=37748736 total_nvlink_bytes=351878496 weight_to_token_ratio=0.120169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38104 dup_rows=21958 dup_rows_remote=19170 dedup_ratio=0.335052 router_tokens_per_expert=[267, 229, 251, 263, 236, 262, 286, 277, 249, 248, 250, 262, 287, 251, 238, 238, 259, 223, 245, 253, 273, 243, 270, 266, 266, 259, 265, 266, 263, 256, 233, 254, 246, 251, 256, 294, 268, 239, 254, 261, 264, 271, 248, 253, 253, 258, 251, 264, 283, 248, 240, 255, 262, 240, 261, 266, 240, 276, 271, 233, 265, 254, 266, 242, 250, 277, 250, 248, 252, 256, 266, 258, 262, 253, 269, 256, 253, 247, 255, 231, 250, 258, 262, 245, 283, 256, 270, 249, 256, 249, 216, 255, 242, 261, 251, 260, 254, 264, 256, 269, 268, 235, 246, 279, 264, 266, 233, 245, 242, 232, 263, 246, 271, 246, 259, 295, 244, 283, 248, 247, 259, 271, 244, 252, 258, 243, 252, 267] payload_rows_per_dst=[5462, 5396, 5474, 5498, 5464, 5436, 5362, 5486] dup_rows_per_dst=[2726, 2792, 2788, 2706, 2702, 2690, 2762, 2792] combine_rows_per_src=[5462, 5396, 5474, 5498, 5464, 5436, 5362, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228384 combine_payload_bytes=155918336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156146720 ingress_bytes=174792704 link_bytes=174792704 token_bytes=312065056 weight_movement_bytes=18874368 total_nvlink_bytes=330939424 weight_to_token_ratio=0.060482 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674777686e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=230216 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157205320 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314180424 weight_movement_bytes=18874368 total_nvlink_bytes=333054792 weight_to_token_ratio=0.060075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38210 dup_rows=21792 dup_rows_remote=18982 dedup_ratio=0.332520 router_tokens_per_expert=[274, 252, 262, 244, 260, 260, 253, 272, 270, 240, 261, 239, 221, 248, 262, 245, 222, 234, 263, 238, 255, 259, 243, 258, 282, 231, 238, 257, 261, 239, 255, 256, 273, 268, 264, 269, 244, 264, 257, 268, 267, 229, 256, 243, 237, 238, 249, 279, 250, 258, 264, 229, 256, 255, 265, 267, 233, 251, 254, 268, 238, 282, 261, 244, 231, 285, 255, 271, 234, 267, 276, 267, 257, 280, 247, 229, 287, 240, 267, 253, 273, 252, 243, 269, 276, 234, 250, 267, 274, 260, 266, 284, 248, 276, 251, 260, 275, 233, 265, 298, 260, 246, 268, 294, 269, 265, 241, 243, 242, 245, 261, 251, 270, 262, 276, 276, 264, 233, 251, 251, 236, 231, 252, 239, 250, 241, 235, 282] payload_rows_per_dst=[5372, 5414, 5482, 5342, 5640, 5454, 5506, 5534] dup_rows_per_dst=[2648, 2568, 2742, 2592, 2868, 2688, 2876, 2810] combine_rows_per_src=[5372, 5414, 5482, 5342, 5640, 5454, 5506, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=229096 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156303080 ingress_bytes=193822720 link_bytes=193822720 token_bytes=312377064 weight_movement_bytes=37748736 total_nvlink_bytes=350125800 weight_to_token_ratio=0.120843 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857113936e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43794 payload_rows_remote=38298 dup_rows=21742 dup_rows_remote=19038 dedup_ratio=0.331757 router_tokens_per_expert=[272, 257, 244, 249, 252, 256, 244, 252, 260, 262, 275, 239, 274, 255, 253, 289, 250, 266, 244, 244, 255, 244, 253, 295, 245, 255, 225, 257, 252, 262, 253, 280, 244, 258, 268, 272, 240, 252, 252, 275, 264, 265, 269, 286, 280, 270, 230, 269, 268, 283, 246, 271, 262, 248, 242, 276, 261, 262, 242, 255, 254, 226, 265, 261, 238, 270, 251, 256, 247, 261, 255, 259, 230, 277, 266, 245, 275, 263, 264, 259, 258, 214, 256, 266, 288, 273, 227, 267, 236, 254, 286, 238, 247, 267, 263, 276, 234, 264, 245, 262, 275, 245, 245, 247, 259, 272, 249, 262, 258, 255, 250, 278, 252, 258, 240, 256, 246, 254, 230, 237, 260, 234, 229, 238, 249, 244, 232, 248] payload_rows_per_dst=[5522, 5456, 5528, 5492, 5510, 5500, 5496, 5290] dup_rows_per_dst=[2744, 2704, 2860, 2752, 2722, 2732, 2704, 2524] combine_rows_per_src=[5522, 5456, 5528, 5492, 5510, 5500, 5496, 5290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38348 dup_rows=21818 dup_rows_remote=19130 dedup_ratio=0.332916 router_tokens_per_expert=[251, 254, 271, 273, 219, 256, 262, 270, 257, 258, 249, 279, 281, 261, 279, 270, 260, 283, 237, 272, 272, 248, 255, 278, 238, 252, 242, 246, 247, 279, 231, 239, 267, 266, 242, 274, 228, 278, 237, 244, 238, 254, 251, 269, 248, 293, 276, 251, 248, 257, 253, 281, 281, 253, 238, 249, 244, 266, 233, 248, 240, 241, 244, 253, 279, 245, 254, 279, 241, 267, 255, 249, 277, 237, 266, 255, 234, 269, 256, 256, 253, 283, 250, 264, 267, 261, 270, 238, 262, 265, 255, 271, 242, 249, 259, 274, 241, 237, 264, 256, 223, 265, 261, 253, 238, 241, 238, 253, 252, 240, 280, 238, 253, 232, 269, 263, 248, 225, 269, 263, 279, 245, 247, 233, 267, 289, 264, 246] payload_rows_per_dst=[5602, 5464, 5480, 5370, 5526, 5446, 5312, 5518] dup_rows_per_dst=[2778, 2694, 2752, 2688, 2712, 2880, 2648, 2666] combine_rows_per_src=[5602, 5464, 5480, 5370, 5526, 5446, 5312, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38308 dup_rows=21780 dup_rows_remote=18966 dedup_ratio=0.332336 router_tokens_per_expert=[243, 252, 239, 213, 244, 242, 239, 272, 257, 254, 240, 278, 238, 285, 281, 241, 206, 263, 251, 256, 244, 252, 262, 244, 254, 251, 257, 217, 257, 250, 259, 262, 251, 255, 267, 254, 246, 272, 252, 265, 272, 268, 254, 256, 259, 255, 252, 289, 268, 268, 257, 273, 270, 247, 240, 258, 276, 244, 261, 260, 263, 223, 275, 242, 263, 252, 251, 260, 257, 253, 266, 222, 244, 278, 252, 259, 245, 248, 231, 257, 262, 247, 281, 240, 271, 255, 260, 267, 269, 265, 289, 259, 261, 242, 259, 238, 260, 251, 256, 267, 265, 243, 232, 262, 254, 260, 269, 274, 265, 262, 268, 275, 280, 263, 258, 247, 277, 239, 277, 242, 253, 249, 278, 230, 262, 227, 272, 253] payload_rows_per_dst=[5448, 5612, 5346, 5524, 5382, 5686, 5412, 5346] dup_rows_per_dst=[2814, 2826, 2582, 2728, 2694, 2994, 2640, 2502] combine_rows_per_src=[5448, 5612, 5346, 5524, 5382, 5686, 5412, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=228768 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156736928 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245088 weight_movement_bytes=18874368 total_nvlink_bytes=332119456 weight_to_token_ratio=0.060254 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38202 dup_rows=21886 dup_rows_remote=19078 dedup_ratio=0.333954 router_tokens_per_expert=[225, 261, 250, 227, 239, 275, 229, 275, 270, 230, 276, 279, 270, 252, 280, 249, 248, 257, 258, 218, 234, 245, 285, 280, 243, 252, 247, 235, 247, 269, 268, 230, 246, 283, 241, 238, 257, 225, 231, 257, 248, 288, 255, 250, 263, 266, 249, 250, 273, 254, 263, 272, 275, 224, 255, 263, 246, 245, 243, 241, 275, 274, 247, 246, 267, 272, 277, 245, 282, 279, 242, 247, 269, 273, 224, 238, 259, 250, 267, 250, 240, 244, 262, 249, 228, 264, 262, 270, 258, 240, 241, 267, 285, 270, 264, 284, 261, 269, 263, 251, 242, 229, 257, 291, 271, 279, 250, 253, 263, 245, 253, 258, 264, 266, 267, 244, 244, 291, 248, 273, 247, 238, 262, 257, 255, 250, 268, 244] payload_rows_per_dst=[5434, 5418, 5392, 5484, 5496, 5448, 5498, 5480] dup_rows_per_dst=[2740, 2614, 2702, 2708, 2786, 2808, 2772, 2756] combine_rows_per_src=[5434, 5418, 5392, 5484, 5496, 5448, 5498, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38418 dup_rows=21636 dup_rows_remote=18978 dedup_ratio=0.330139 router_tokens_per_expert=[240, 262, 261, 254, 224, 249, 244, 231, 268, 269, 225, 266, 257, 262, 258, 265, 258, 225, 238, 272, 261, 275, 212, 271, 252, 248, 244, 255, 255, 252, 281, 271, 239, 260, 253, 260, 268, 243, 270, 263, 263, 250, 263, 257, 306, 243, 273, 239, 253, 258, 252, 242, 243, 223, 265, 259, 252, 260, 279, 266, 241, 248, 265, 241, 257, 262, 241, 284, 262, 267, 256, 244, 275, 243, 254, 250, 260, 270, 275, 255, 287, 266, 281, 251, 255, 264, 280, 293, 259, 261, 267, 256, 240, 269, 264, 255, 265, 225, 262, 235, 242, 234, 273, 260, 264, 239, 252, 247, 251, 228, 234, 249, 277, 256, 256, 260, 258, 225, 256, 247, 273, 251, 259, 255, 244, 263, 269, 254] payload_rows_per_dst=[5444, 5482, 5562, 5370, 5540, 5676, 5378, 5448] dup_rows_per_dst=[2626, 2658, 2738, 2724, 2770, 2820, 2542, 2758] combine_rows_per_src=[5444, 5482, 5562, 5370, 5540, 5676, 5378, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229344 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157097952 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966560 weight_movement_bytes=18874368 total_nvlink_bytes=332840928 weight_to_token_ratio=0.060116 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229912 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157303320 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314376728 weight_movement_bytes=37748736 total_nvlink_bytes=352125464 weight_to_token_ratio=0.120075 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229096 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157138664 ingress_bytes=175783936 link_bytes=175783936 token_bytes=314048232 weight_movement_bytes=18874368 total_nvlink_bytes=332922600 weight_to_token_ratio=0.060100 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684275184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38304 dup_rows=21800 dup_rows_remote=19130 dedup_ratio=0.332642 router_tokens_per_expert=[260, 258, 229, 274, 232, 257, 253, 215, 269, 270, 252, 252, 250, 244, 267, 232, 246, 257, 306, 250, 251, 249, 239, 258, 265, 267, 267, 251, 244, 264, 234, 247, 257, 275, 253, 246, 258, 245, 252, 263, 245, 279, 251, 253, 250, 269, 256, 239, 270, 248, 250, 261, 276, 263, 238, 285, 252, 228, 234, 259, 298, 242, 289, 256, 242, 249, 247, 262, 232, 227, 258, 279, 238, 259, 259, 242, 273, 254, 261, 269, 244, 277, 273, 274, 266, 250, 243, 240, 262, 270, 253, 247, 260, 261, 266, 259, 272, 280, 267, 287, 236, 258, 246, 246, 256, 265, 225, 263, 234, 277, 265, 256, 273, 239, 260, 264, 267, 273, 235, 254, 249, 250, 273, 247, 248, 253, 268, 237] payload_rows_per_dst=[5384, 5498, 5424, 5498, 5432, 5530, 5484, 5486] dup_rows_per_dst=[2644, 2692, 2758, 2800, 2670, 2760, 2782, 2694] combine_rows_per_src=[5384, 5498, 5424, 5498, 5432, 5530, 5484, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38178 dup_rows=21900 dup_rows_remote=19168 dedup_ratio=0.334167 router_tokens_per_expert=[259, 275, 249, 230, 237, 255, 269, 260, 240, 255, 248, 258, 259, 249, 225, 237, 265, 261, 260, 261, 257, 280, 236, 281, 261, 237, 243, 253, 263, 247, 269, 258, 245, 255, 280, 282, 274, 259, 246, 263, 248, 241, 247, 265, 265, 217, 250, 258, 260, 263, 255, 264, 240, 241, 225, 273, 252, 263, 249, 295, 269, 259, 247, 242, 273, 214, 254, 268, 266, 251, 265, 255, 248, 260, 247, 268, 277, 265, 278, 264, 255, 281, 259, 295, 214, 248, 282, 253, 245, 275, 275, 257, 255, 261, 231, 223, 269, 265, 259, 246, 239, 243, 255, 260, 234, 239, 277, 270, 260, 255, 285, 259, 242, 241, 267, 230, 249, 267, 246, 246, 276, 234, 263, 272, 245, 264, 253, 267] payload_rows_per_dst=[5354, 5500, 5458, 5472, 5540, 5416, 5504, 5392] dup_rows_per_dst=[2656, 2764, 2732, 2722, 2766, 2802, 2726, 2732] combine_rows_per_src=[5354, 5500, 5458, 5472, 5540, 5416, 5504, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229120 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156704512 ingress_bytes=194224128 link_bytes=194224128 token_bytes=313179904 weight_movement_bytes=37748736 total_nvlink_bytes=350928640 weight_to_token_ratio=0.120534 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860960030e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229584 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157589712 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314949840 weight_movement_bytes=37748736 total_nvlink_bytes=352698576 weight_to_token_ratio=0.119856 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38342 dup_rows=21808 dup_rows_remote=18968 dedup_ratio=0.332764 router_tokens_per_expert=[257, 220, 248, 228, 250, 249, 280, 236, 274, 254, 257, 240, 256, 230, 245, 234, 229, 267, 270, 302, 254, 262, 280, 264, 256, 238, 251, 242, 270, 263, 269, 286, 256, 261, 265, 246, 245, 246, 263, 275, 264, 249, 281, 247, 269, 264, 228, 264, 247, 244, 253, 232, 257, 257, 265, 260, 222, 294, 252, 258, 250, 252, 260, 284, 265, 256, 240, 264, 276, 259, 252, 263, 268, 277, 252, 280, 281, 251, 223, 251, 234, 239, 263, 243, 247, 283, 258, 263, 230, 240, 237, 253, 274, 242, 283, 252, 280, 280, 229, 242, 257, 231, 238, 268, 272, 276, 248, 252, 272, 246, 263, 246, 270, 279, 269, 257, 236, 232, 239, 260, 243, 277, 256, 262, 277, 261, 226, 254] payload_rows_per_dst=[5386, 5344, 5466, 5490, 5408, 5480, 5768, 5386] dup_rows_per_dst=[2530, 2458, 2780, 2684, 2738, 2742, 3036, 2840] combine_rows_per_src=[5386, 5344, 5466, 5490, 5408, 5480, 5768, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229736 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157122920 ingress_bytes=194641920 link_bytes=194641920 token_bytes=314016104 weight_movement_bytes=37748736 total_nvlink_bytes=351764840 weight_to_token_ratio=0.120213 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864963108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229384 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156606472 ingress_bytes=231874560 link_bytes=231874560 token_bytes=312983560 weight_movement_bytes=75497472 total_nvlink_bytes=388481032 weight_to_token_ratio=0.241219 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.221707945e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38316 dup_rows=21792 dup_rows_remote=18996 dedup_ratio=0.332520 router_tokens_per_expert=[224, 268, 272, 270, 264, 259, 273, 257, 261, 258, 255, 250, 238, 266, 256, 268, 266, 288, 249, 256, 244, 232, 240, 240, 265, 254, 241, 287, 251, 246, 246, 243, 231, 260, 251, 240, 257, 247, 266, 252, 269, 249, 262, 269, 265, 263, 242, 250, 237, 247, 248, 252, 259, 270, 267, 233, 272, 267, 256, 273, 267, 272, 254, 238, 242, 263, 253, 302, 257, 238, 262, 271, 273, 249, 271, 236, 270, 232, 255, 242, 259, 261, 301, 269, 218, 266, 269, 253, 277, 262, 239, 280, 266, 247, 262, 237, 225, 226, 244, 240, 253, 261, 273, 269, 269, 254, 230, 273, 275, 253, 273, 237, 255, 281, 261, 241, 249, 246, 246, 234, 266, 240, 263, 257, 267, 260, 254, 239] payload_rows_per_dst=[5504, 5434, 5418, 5428, 5496, 5554, 5406, 5504] dup_rows_per_dst=[2774, 2662, 2728, 2796, 2736, 2778, 2704, 2614] combine_rows_per_src=[5504, 5434, 5418, 5428, 5496, 5554, 5406, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38388 dup_rows=21654 dup_rows_remote=18852 dedup_ratio=0.330414 router_tokens_per_expert=[246, 237, 250, 260, 283, 242, 259, 265, 244, 238, 271, 274, 279, 251, 261, 241, 218, 260, 256, 278, 260, 252, 253, 247, 251, 247, 271, 260, 248, 258, 260, 268, 228, 256, 260, 247, 263, 257, 286, 267, 281, 255, 273, 268, 260, 254, 260, 258, 244, 247, 268, 261, 270, 227, 222, 236, 247, 284, 241, 259, 247, 245, 268, 271, 278, 305, 228, 259, 244, 262, 281, 243, 244, 272, 260, 235, 251, 268, 276, 261, 253, 243, 246, 285, 262, 241, 253, 241, 267, 259, 251, 244, 274, 259, 247, 250, 270, 249, 270, 279, 248, 247, 241, 234, 259, 249, 256, 235, 258, 271, 228, 208, 247, 231, 284, 296, 241, 233, 272, 259, 238, 244, 277, 275, 245, 246, 271, 267] payload_rows_per_dst=[5502, 5386, 5552, 5448, 5500, 5464, 5494, 5536] dup_rows_per_dst=[2700, 2496, 2794, 2626, 2834, 2686, 2802, 2716] combine_rows_per_src=[5502, 5386, 5552, 5448, 5500, 5464, 5494, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229240 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157278072 ingress_bytes=194797568 link_bytes=194797568 token_bytes=314326904 weight_movement_bytes=37748736 total_nvlink_bytes=352075640 weight_to_token_ratio=0.120094 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866454450e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38186 dup_rows=21866 dup_rows_remote=19040 dedup_ratio=0.333649 router_tokens_per_expert=[269, 250, 274, 265, 256, 231, 242, 260, 241, 257, 267, 280, 240, 266, 272, 278, 263, 252, 274, 246, 248, 263, 251, 270, 245, 232, 280, 253, 261, 263, 238, 270, 246, 236, 276, 222, 275, 281, 230, 281, 276, 237, 257, 276, 265, 241, 252, 253, 254, 297, 256, 243, 234, 269, 262, 262, 227, 272, 231, 260, 247, 273, 262, 275, 265, 272, 254, 254, 267, 260, 268, 246, 257, 236, 262, 259, 252, 264, 222, 277, 258, 222, 244, 260, 244, 255, 247, 231, 251, 262, 271, 225, 273, 257, 263, 257, 259, 233, 272, 239, 277, 235, 276, 259, 261, 232, 272, 265, 246, 266, 245, 268, 262, 259, 249, 253, 265, 231, 261, 263, 219, 235, 249, 249, 254, 267, 243, 284] payload_rows_per_dst=[5484, 5532, 5476, 5420, 5420, 5420, 5488, 5430] dup_rows_per_dst=[2826, 2856, 2718, 2658, 2810, 2620, 2722, 2656] combine_rows_per_src=[5484, 5532, 5476, 5420, 5420, 5420, 5488, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38250 dup_rows=21786 dup_rows_remote=18994 dedup_ratio=0.332428 router_tokens_per_expert=[263, 221, 235, 246, 252, 270, 256, 260, 277, 276, 256, 236, 264, 259, 258, 242, 269, 266, 274, 280, 251, 250, 264, 282, 249, 270, 254, 267, 272, 262, 248, 241, 258, 271, 250, 250, 285, 258, 237, 250, 254, 263, 254, 265, 258, 269, 229, 258, 251, 280, 264, 237, 249, 265, 253, 250, 227, 267, 233, 279, 259, 230, 275, 246, 255, 243, 217, 275, 244, 229, 253, 268, 250, 276, 254, 263, 249, 278, 258, 255, 251, 255, 250, 239, 273, 263, 258, 258, 269, 257, 231, 272, 264, 271, 260, 275, 249, 224, 282, 203, 279, 261, 269, 229, 230, 255, 266, 244, 251, 247, 265, 288, 259, 268, 274, 263, 251, 252, 249, 278, 262, 246, 256, 241, 255, 242, 234, 239] payload_rows_per_dst=[5504, 5516, 5578, 5386, 5470, 5500, 5380, 5416] dup_rows_per_dst=[2638, 2882, 2640, 2744, 2664, 2792, 2704, 2722] combine_rows_per_src=[5504, 5516, 5578, 5386, 5470, 5500, 5380, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38194 dup_rows=21826 dup_rows_remote=19274 dedup_ratio=0.333038 router_tokens_per_expert=[256, 275, 280, 276, 234, 231, 289, 263, 269, 276, 264, 247, 252, 252, 267, 272, 224, 239, 267, 261, 259, 245, 218, 259, 289, 258, 258, 277, 258, 248, 235, 239, 232, 258, 263, 256, 276, 272, 266, 272, 235, 283, 261, 238, 245, 246, 270, 226, 264, 248, 268, 263, 223, 257, 256, 250, 251, 256, 237, 243, 258, 267, 285, 261, 239, 254, 268, 264, 255, 260, 248, 245, 277, 267, 298, 235, 256, 238, 298, 275, 259, 219, 271, 269, 232, 237, 243, 256, 234, 289, 270, 244, 271, 262, 239, 265, 216, 256, 256, 269, 260, 255, 260, 280, 275, 231, 197, 253, 254, 241, 245, 274, 258, 255, 238, 254, 215, 274, 290, 266, 278, 262, 278, 278, 240, 250, 233, 217] payload_rows_per_dst=[5536, 5516, 5450, 5536, 5492, 5382, 5396, 5402] dup_rows_per_dst=[2870, 2552, 2748, 2638, 2862, 2738, 2648, 2770] combine_rows_per_src=[5536, 5516, 5450, 5536, 5492, 5382, 5396, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156942336 dispatch_meta_bytes=229248 combine_payload_bytes=156942336 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157171584 ingress_bytes=175816704 link_bytes=175816704 token_bytes=314113920 weight_movement_bytes=18874368 total_nvlink_bytes=332988288 weight_to_token_ratio=0.060088 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684589151e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=228960 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157466208 ingress_bytes=176111616 link_bytes=176111616 token_bytes=314703456 weight_movement_bytes=18874368 total_nvlink_bytes=333577824 weight_to_token_ratio=0.059975 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687414852e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38418 dup_rows=21662 dup_rows_remote=19082 dedup_ratio=0.330536 router_tokens_per_expert=[302, 271, 243, 252, 242, 266, 269, 241, 258, 244, 281, 266, 244, 260, 243, 261, 271, 252, 252, 246, 254, 256, 230, 266, 246, 250, 242, 288, 261, 227, 273, 254, 240, 243, 227, 238, 260, 250, 272, 272, 245, 255, 248, 250, 278, 239, 243, 258, 272, 256, 246, 236, 291, 249, 252, 261, 254, 276, 264, 242, 265, 241, 263, 232, 265, 256, 255, 268, 253, 264, 243, 228, 271, 269, 253, 251, 279, 263, 276, 273, 253, 243, 265, 248, 266, 240, 255, 256, 275, 246, 253, 259, 255, 233, 239, 260, 259, 243, 262, 260, 269, 257, 258, 273, 257, 286, 273, 248, 230, 244, 258, 264, 250, 267, 266, 239, 244, 234, 236, 253, 275, 283, 265, 252, 250, 271, 235, 265] payload_rows_per_dst=[5540, 5502, 5456, 5452, 5552, 5434, 5458, 5480] dup_rows_per_dst=[2746, 2634, 2580, 2748, 2782, 2658, 2824, 2690] combine_rows_per_src=[5540, 5502, 5456, 5452, 5552, 5434, 5458, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38288 dup_rows=21802 dup_rows_remote=19148 dedup_ratio=0.332672 router_tokens_per_expert=[258, 230, 248, 250, 264, 260, 244, 246, 258, 257, 208, 248, 266, 272, 240, 240, 290, 286, 260, 271, 264, 264, 286, 236, 268, 252, 277, 268, 257, 253, 232, 259, 267, 256, 245, 277, 276, 284, 263, 262, 237, 261, 261, 256, 235, 262, 236, 267, 225, 236, 258, 250, 245, 267, 275, 248, 223, 247, 288, 265, 229, 239, 258, 232, 259, 246, 254, 302, 234, 214, 254, 261, 230, 263, 278, 256, 270, 246, 241, 242, 248, 290, 237, 279, 261, 257, 243, 261, 263, 271, 247, 243, 272, 244, 247, 266, 280, 271, 266, 267, 248, 251, 257, 273, 232, 268, 254, 229, 251, 264, 249, 237, 279, 267, 243, 266, 255, 251, 266, 237, 246, 255, 247, 286, 258, 258, 257, 279] payload_rows_per_dst=[5340, 5566, 5536, 5406, 5446, 5494, 5424, 5522] dup_rows_per_dst=[2638, 2880, 2754, 2564, 2654, 2764, 2770, 2778] combine_rows_per_src=[5340, 5566, 5536, 5406, 5446, 5494, 5424, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=228904 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638760 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048616 weight_movement_bytes=37748736 total_nvlink_bytes=350797352 weight_to_token_ratio=0.120584 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=228976 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156900976 ingress_bytes=194420736 link_bytes=194420736 token_bytes=313572976 weight_movement_bytes=37748736 total_nvlink_bytes=351321712 weight_to_token_ratio=0.120383 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862843831e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229872 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156672496 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313115120 weight_movement_bytes=18874368 total_nvlink_bytes=331989488 weight_to_token_ratio=0.060279 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=230000 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157590128 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314950256 weight_movement_bytes=18874368 total_nvlink_bytes=333824624 weight_to_token_ratio=0.059928 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229744 combine_payload_bytes=156827648 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157057392 ingress_bytes=213450752 link_bytes=213450752 token_bytes=313885040 weight_movement_bytes=56623104 total_nvlink_bytes=370508144 weight_to_token_ratio=0.180394 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045180081e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38216 dup_rows=21800 dup_rows_remote=19038 dedup_ratio=0.332642 router_tokens_per_expert=[252, 258, 262, 265, 244, 253, 289, 269, 260, 262, 267, 248, 247, 251, 241, 263, 243, 253, 264, 260, 286, 245, 250, 241, 221, 264, 271, 278, 263, 245, 239, 258, 266, 254, 274, 244, 248, 240, 277, 260, 274, 276, 237, 245, 241, 252, 252, 232, 230, 248, 241, 258, 253, 242, 281, 254, 257, 264, 277, 255, 257, 225, 266, 285, 242, 255, 268, 253, 264, 245, 266, 237, 240, 259, 249, 243, 277, 256, 260, 250, 248, 249, 279, 260, 254, 234, 265, 241, 251, 252, 246, 253, 266, 241, 266, 262, 237, 250, 237, 253, 268, 268, 256, 239, 247, 264, 279, 254, 287, 238, 276, 278, 279, 248, 270, 237, 279, 270, 255, 250, 240, 243, 253, 235, 273, 260, 243, 294] payload_rows_per_dst=[5350, 5454, 5440, 5466, 5480, 5440, 5586, 5520] dup_rows_per_dst=[2582, 2708, 2600, 2800, 2648, 2694, 3006, 2762] combine_rows_per_src=[5350, 5454, 5440, 5466, 5480, 5440, 5586, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43576 payload_rows_remote=38042 dup_rows=21960 dup_rows_remote=19254 dedup_ratio=0.335083 router_tokens_per_expert=[270, 285, 245, 262, 268, 256, 259, 262, 244, 284, 274, 272, 244, 284, 266, 275, 258, 259, 251, 255, 225, 231, 269, 262, 283, 235, 277, 242, 284, 243, 273, 272, 263, 254, 253, 269, 290, 261, 235, 268, 240, 260, 275, 246, 261, 267, 249, 252, 251, 265, 262, 254, 254, 240, 250, 264, 276, 254, 267, 260, 247, 247, 258, 271, 261, 256, 245, 246, 276, 256, 240, 256, 270, 252, 244, 247, 254, 253, 268, 273, 222, 233, 282, 239, 264, 235, 282, 260, 252, 251, 247, 247, 245, 241, 242, 259, 281, 259, 248, 233, 251, 251, 234, 259, 262, 247, 243, 237, 252, 295, 256, 243, 243, 253, 256, 237, 222, 242, 254, 263, 253, 250, 270, 220, 254, 256, 253, 261] payload_rows_per_dst=[5546, 5428, 5542, 5534, 5514, 5326, 5372, 5314] dup_rows_per_dst=[2954, 2810, 2744, 2706, 2680, 2676, 2730, 2660] combine_rows_per_src=[5546, 5428, 5542, 5534, 5514, 5326, 5372, 5314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229016 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156761752 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294488 weight_movement_bytes=37748736 total_nvlink_bytes=351043224 weight_to_token_ratio=0.120490 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43940 payload_rows_remote=38440 dup_rows=21596 dup_rows_remote=18932 dedup_ratio=0.329529 router_tokens_per_expert=[251, 240, 250, 274, 256, 268, 256, 276, 266, 253, 271, 253, 271, 284, 266, 248, 255, 256, 245, 218, 257, 259, 219, 244, 270, 250, 255, 255, 240, 261, 242, 249, 251, 249, 258, 254, 250, 248, 277, 235, 256, 251, 267, 261, 231, 266, 270, 259, 263, 269, 262, 270, 238, 275, 233, 263, 261, 243, 238, 251, 262, 246, 216, 258, 268, 251, 259, 263, 253, 279, 249, 280, 256, 263, 273, 238, 291, 228, 271, 260, 281, 238, 245, 240, 229, 261, 288, 228, 252, 270, 266, 234, 261, 228, 261, 287, 266, 269, 266, 246, 277, 233, 237, 266, 273, 255, 253, 275, 247, 228, 261, 230, 258, 272, 271, 267, 240, 253, 247, 261, 253, 273, 263, 254, 256, 270, 266, 242] payload_rows_per_dst=[5588, 5408, 5440, 5484, 5608, 5414, 5500, 5498] dup_rows_per_dst=[2778, 2542, 2726, 2612, 2756, 2724, 2664, 2794] combine_rows_per_src=[5588, 5408, 5440, 5484, 5608, 5414, 5500, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38370 dup_rows=21720 dup_rows_remote=18878 dedup_ratio=0.331421 router_tokens_per_expert=[260, 266, 272, 268, 243, 242, 272, 249, 258, 253, 251, 260, 252, 232, 288, 244, 235, 246, 239, 250, 245, 281, 284, 226, 247, 250, 240, 241, 254, 239, 258, 254, 285, 246, 261, 243, 252, 247, 266, 250, 269, 235, 263, 264, 268, 263, 248, 260, 272, 248, 277, 250, 246, 238, 284, 264, 234, 279, 256, 238, 252, 282, 256, 254, 276, 243, 276, 255, 281, 268, 298, 263, 257, 258, 218, 237, 265, 236, 265, 242, 256, 243, 284, 271, 249, 260, 261, 280, 252, 268, 254, 225, 271, 249, 282, 262, 245, 254, 234, 273, 240, 254, 252, 275, 237, 278, 253, 277, 250, 253, 239, 252, 264, 239, 261, 264, 269, 243, 258, 241, 248, 259, 243, 231, 273, 253, 256, 246] payload_rows_per_dst=[5446, 5514, 5620, 5524, 5478, 5362, 5414, 5458] dup_rows_per_dst=[2842, 2968, 2648, 2750, 2798, 2468, 2608, 2638] combine_rows_per_src=[5446, 5514, 5620, 5524, 5478, 5362, 5414, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=229184 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156049216 ingress_bytes=174694400 link_bytes=174694400 token_bytes=311869248 weight_movement_bytes=18874368 total_nvlink_bytes=330743616 weight_to_token_ratio=0.060520 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673835786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38348 dup_rows=21840 dup_rows_remote=19144 dedup_ratio=0.333252 router_tokens_per_expert=[273, 252, 264, 258, 264, 234, 266, 292, 268, 225, 246, 238, 252, 264, 281, 267, 270, 264, 239, 262, 260, 248, 272, 255, 234, 253, 254, 242, 257, 258, 240, 276, 262, 252, 253, 249, 239, 225, 256, 262, 241, 247, 274, 259, 259, 263, 257, 287, 272, 273, 240, 268, 258, 262, 258, 237, 249, 268, 234, 236, 268, 255, 281, 265, 269, 268, 268, 230, 253, 257, 271, 266, 247, 284, 254, 257, 242, 264, 255, 250, 225, 269, 276, 250, 248, 273, 248, 244, 271, 241, 236, 257, 258, 228, 237, 261, 271, 247, 266, 239, 238, 253, 239, 245, 264, 260, 284, 242, 255, 249, 229, 271, 244, 222, 278, 254, 252, 300, 261, 254, 280, 220, 256, 239, 266, 246, 279, 271] payload_rows_per_dst=[5504, 5392, 5444, 5500, 5542, 5348, 5472, 5494] dup_rows_per_dst=[2784, 2776, 2726, 2748, 2728, 2696, 2632, 2750] combine_rows_per_src=[5504, 5392, 5444, 5500, 5542, 5348, 5472, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38334 dup_rows=21782 dup_rows_remote=18898 dedup_ratio=0.332367 router_tokens_per_expert=[240, 272, 230, 257, 222, 263, 283, 243, 246, 272, 255, 258, 238, 241, 291, 252, 272, 273, 276, 249, 253, 255, 258, 263, 253, 261, 240, 269, 266, 244, 212, 239, 244, 276, 264, 262, 275, 279, 245, 266, 301, 270, 228, 246, 241, 253, 235, 267, 254, 258, 251, 251, 262, 250, 268, 255, 276, 233, 228, 250, 244, 244, 243, 253, 264, 239, 208, 262, 273, 271, 254, 267, 241, 251, 255, 272, 261, 239, 229, 269, 269, 258, 288, 264, 249, 259, 251, 268, 258, 292, 266, 283, 256, 260, 235, 262, 267, 249, 234, 220, 254, 252, 263, 243, 248, 291, 227, 247, 260, 284, 289, 258, 253, 243, 255, 247, 273, 264, 258, 246, 256, 262, 251, 258, 249, 276, 242, 258] payload_rows_per_dst=[5412, 5478, 5420, 5394, 5496, 5592, 5502, 5460] dup_rows_per_dst=[2714, 2688, 2884, 2646, 2614, 2844, 2670, 2722] combine_rows_per_src=[5412, 5478, 5420, 5394, 5496, 5592, 5502, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38042 dup_rows=21958 dup_rows_remote=19118 dedup_ratio=0.335052 router_tokens_per_expert=[226, 241, 261, 265, 255, 246, 221, 224, 277, 273, 281, 244, 248, 241, 297, 229, 269, 259, 248, 238, 265, 271, 260, 263, 267, 287, 240, 283, 254, 239, 282, 263, 262, 265, 258, 253, 271, 244, 254, 291, 241, 256, 277, 223, 259, 279, 275, 270, 245, 229, 280, 273, 257, 265, 238, 281, 252, 255, 242, 275, 255, 240, 246, 250, 256, 240, 230, 235, 256, 268, 249, 246, 248, 253, 255, 233, 265, 245, 227, 257, 264, 259, 289, 258, 247, 216, 220, 269, 267, 252, 244, 263, 248, 268, 245, 260, 241, 247, 261, 248, 271, 267, 245, 249, 256, 263, 267, 232, 283, 260, 256, 233, 252, 230, 254, 263, 285, 267, 249, 252, 235, 272, 234, 288, 274, 282, 295, 247] payload_rows_per_dst=[5390, 5536, 5566, 5406, 5326, 5472, 5402, 5480] dup_rows_per_dst=[2668, 2840, 2790, 2760, 2600, 2666, 2756, 2878] combine_rows_per_src=[5390, 5536, 5566, 5406, 5326, 5472, 5402, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157450240 dispatch_meta_bytes=229488 combine_payload_bytes=157450240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157679728 ingress_bytes=176324608 link_bytes=176324608 token_bytes=315129968 weight_movement_bytes=18874368 total_nvlink_bytes=334004336 weight_to_token_ratio=0.059894 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689455637e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38224 dup_rows=21822 dup_rows_remote=19048 dedup_ratio=0.332977 router_tokens_per_expert=[222, 242, 238, 273, 253, 248, 269, 246, 245, 237, 261, 256, 251, 238, 242, 244, 275, 273, 261, 253, 246, 256, 233, 251, 269, 273, 231, 261, 257, 241, 253, 267, 265, 273, 268, 280, 255, 274, 238, 246, 250, 273, 244, 248, 265, 258, 264, 273, 252, 271, 251, 281, 266, 221, 236, 218, 262, 255, 223, 269, 250, 258, 265, 235, 286, 273, 266, 282, 274, 236, 236, 256, 257, 264, 279, 227, 261, 258, 234, 243, 270, 244, 246, 249, 264, 274, 243, 271, 246, 263, 254, 260, 254, 257, 260, 277, 248, 243, 257, 263, 249, 246, 276, 230, 255, 253, 277, 230, 266, 262, 280, 249, 266, 282, 270, 231, 263, 295, 263, 275, 258, 250, 219, 250, 264, 261, 230, 291] payload_rows_per_dst=[5424, 5418, 5558, 5430, 5490, 5404, 5488, 5502] dup_rows_per_dst=[2506, 2782, 2790, 2596, 2774, 2860, 2680, 2834] combine_rows_per_src=[5424, 5418, 5558, 5430, 5490, 5404, 5488, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=228992 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157392512 ingress_bytes=194912256 link_bytes=194912256 token_bytes=314556032 weight_movement_bytes=37748736 total_nvlink_bytes=352304768 weight_to_token_ratio=0.120006 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867553334e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229968 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303376 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376784 weight_movement_bytes=18874368 total_nvlink_bytes=333251152 weight_to_token_ratio=0.060037 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=228928 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157244992 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261056 weight_movement_bytes=18874368 total_nvlink_bytes=333135424 weight_to_token_ratio=0.060060 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155820032 dispatch_meta_bytes=228640 combine_payload_bytes=155820032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156048672 ingress_bytes=193568768 link_bytes=193568768 token_bytes=311868704 weight_movement_bytes=37748736 total_nvlink_bytes=349617440 weight_to_token_ratio=0.121040 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854680693e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38260 dup_rows=21822 dup_rows_remote=18880 dedup_ratio=0.332977 router_tokens_per_expert=[252, 285, 226, 262, 282, 255, 283, 251, 244, 250, 261, 273, 255, 245, 250, 260, 257, 275, 247, 258, 244, 243, 244, 279, 263, 289, 264, 272, 246, 257, 262, 241, 269, 252, 233, 294, 258, 236, 281, 238, 234, 244, 266, 258, 264, 261, 251, 231, 271, 259, 240, 273, 275, 253, 248, 253, 276, 250, 233, 251, 288, 256, 250, 255, 245, 267, 261, 246, 225, 275, 254, 275, 265, 248, 223, 234, 246, 247, 265, 233, 302, 260, 260, 251, 243, 249, 250, 256, 269, 252, 244, 252, 284, 248, 265, 255, 253, 270, 253, 245, 267, 241, 249, 266, 264, 255, 227, 261, 243, 272, 256, 242, 255, 301, 251, 224, 252, 246, 219, 273, 255, 257, 251, 248, 261, 254, 263, 269] payload_rows_per_dst=[5540, 5344, 5430, 5342, 5610, 5538, 5456, 5454] dup_rows_per_dst=[2728, 2504, 2710, 2678, 2842, 2742, 2676, 2942] combine_rows_per_src=[5540, 5344, 5430, 5342, 5610, 5538, 5456, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156565504 dispatch_meta_bytes=229088 combine_payload_bytes=156565504 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156794592 ingress_bytes=194314240 link_bytes=194314240 token_bytes=313360096 weight_movement_bytes=37748736 total_nvlink_bytes=351108832 weight_to_token_ratio=0.120464 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861823439e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38348 dup_rows=21756 dup_rows_remote=18966 dedup_ratio=0.331970 router_tokens_per_expert=[251, 242, 283, 265, 263, 242, 251, 252, 251, 279, 271, 251, 246, 251, 246, 248, 255, 278, 260, 239, 221, 240, 269, 238, 294, 246, 272, 276, 225, 234, 282, 256, 292, 243, 249, 261, 253, 255, 238, 230, 264, 231, 277, 302, 238, 245, 287, 258, 259, 266, 273, 277, 263, 248, 257, 236, 252, 240, 262, 254, 230, 245, 265, 284, 257, 250, 244, 263, 266, 246, 273, 244, 273, 268, 217, 251, 257, 226, 226, 271, 273, 281, 253, 255, 273, 274, 279, 255, 222, 242, 262, 263, 269, 244, 236, 264, 256, 265, 239, 245, 250, 237, 261, 268, 260, 263, 253, 274, 270, 256, 248, 264, 269, 244, 271, 247, 278, 234, 243, 282, 251, 232, 255, 265, 243, 246, 261, 250] payload_rows_per_dst=[5516, 5442, 5468, 5432, 5408, 5552, 5456, 5506] dup_rows_per_dst=[2668, 2728, 2778, 2790, 2656, 2738, 2762, 2636] combine_rows_per_src=[5516, 5442, 5468, 5432, 5408, 5552, 5456, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=228560 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156941520 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313654480 weight_movement_bytes=18874368 total_nvlink_bytes=332528848 weight_to_token_ratio=0.060176 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43962 payload_rows_remote=38458 dup_rows=21574 dup_rows_remote=18934 dedup_ratio=0.329193 router_tokens_per_expert=[260, 256, 243, 223, 220, 267, 250, 250, 272, 267, 256, 244, 256, 252, 273, 244, 248, 284, 258, 260, 258, 271, 248, 257, 278, 264, 253, 267, 265, 232, 263, 256, 280, 267, 263, 235, 260, 244, 267, 271, 247, 252, 222, 285, 245, 248, 250, 237, 259, 244, 252, 267, 280, 251, 288, 268, 255, 253, 271, 253, 248, 278, 271, 229, 296, 252, 266, 270, 267, 237, 249, 261, 242, 259, 273, 230, 235, 243, 244, 243, 227, 244, 281, 308, 242, 238, 252, 262, 250, 261, 284, 252, 250, 227, 242, 248, 273, 244, 260, 236, 281, 263, 276, 255, 242, 258, 236, 248, 270, 224, 256, 250, 255, 249, 290, 262, 255, 270, 264, 250, 254, 255, 254, 256, 259, 248, 261, 244] payload_rows_per_dst=[5478, 5490, 5478, 5586, 5390, 5488, 5504, 5548] dup_rows_per_dst=[2588, 2834, 2668, 2748, 2744, 2648, 2640, 2704] combine_rows_per_src=[5478, 5490, 5478, 5586, 5390, 5488, 5504, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229256 combine_payload_bytes=157073408 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157302664 ingress_bytes=194822144 link_bytes=194822144 token_bytes=314376072 weight_movement_bytes=37748736 total_nvlink_bytes=352124808 weight_to_token_ratio=0.120075 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866689925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43698 payload_rows_remote=38206 dup_rows=21838 dup_rows_remote=19014 dedup_ratio=0.333221 router_tokens_per_expert=[231, 249, 279, 270, 260, 257, 283, 252, 236, 242, 280, 276, 262, 274, 244, 260, 251, 262, 230, 242, 262, 242, 266, 266, 247, 259, 262, 262, 238, 230, 282, 249, 256, 280, 238, 243, 235, 269, 245, 241, 253, 267, 261, 226, 281, 261, 238, 259, 269, 281, 247, 245, 273, 259, 247, 254, 262, 258, 256, 276, 245, 246, 289, 271, 273, 268, 261, 259, 242, 273, 239, 235, 264, 277, 276, 271, 237, 263, 262, 249, 231, 258, 255, 234, 246, 232, 232, 270, 262, 225, 253, 266, 251, 250, 274, 225, 264, 268, 266, 239, 278, 243, 254, 230, 232, 243, 268, 257, 263, 268, 236, 244, 232, 248, 255, 263, 268, 243, 273, 267, 270, 247, 264, 285, 264, 263, 265, 259] payload_rows_per_dst=[5492, 5692, 5490, 5492, 5302, 5286, 5480, 5464] dup_rows_per_dst=[2824, 2926, 2616, 2858, 2478, 2642, 2626, 2868] combine_rows_per_src=[5492, 5692, 5490, 5492, 5302, 5286, 5480, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38170 dup_rows=21842 dup_rows_remote=19106 dedup_ratio=0.333282 router_tokens_per_expert=[264, 234, 248, 249, 275, 224, 297, 264, 251, 238, 275, 284, 260, 273, 239, 265, 246, 270, 248, 271, 265, 247, 243, 238, 261, 256, 245, 240, 239, 269, 271, 238, 234, 252, 270, 268, 263, 284, 242, 246, 248, 248, 254, 260, 264, 263, 253, 281, 272, 249, 255, 286, 246, 255, 240, 227, 244, 254, 269, 251, 247, 251, 254, 251, 260, 232, 255, 264, 273, 228, 230, 283, 255, 246, 261, 262, 266, 249, 252, 245, 255, 263, 233, 253, 214, 248, 280, 283, 254, 264, 244, 232, 260, 256, 243, 227, 269, 255, 263, 275, 246, 258, 256, 276, 227, 266, 255, 270, 257, 256, 296, 231, 284, 257, 247, 267, 261, 273, 279, 231, 238, 261, 257, 281, 243, 259, 252, 284] payload_rows_per_dst=[5516, 5402, 5524, 5440, 5450, 5342, 5474, 5546] dup_rows_per_dst=[2764, 2692, 2736, 2662, 2672, 2676, 2838, 2802] combine_rows_per_src=[5516, 5402, 5524, 5440, 5450, 5342, 5474, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38194 dup_rows=21902 dup_rows_remote=19124 dedup_ratio=0.334198 router_tokens_per_expert=[250, 280, 272, 237, 261, 255, 228, 256, 245, 244, 229, 251, 249, 277, 273, 259, 252, 257, 236, 241, 261, 277, 246, 259, 234, 246, 255, 263, 257, 265, 259, 244, 258, 277, 258, 249, 245, 289, 236, 252, 251, 272, 234, 247, 251, 232, 248, 233, 255, 258, 266, 253, 270, 246, 272, 258, 260, 267, 235, 247, 241, 259, 264, 245, 267, 266, 274, 271, 310, 263, 269, 252, 239, 270, 256, 257, 258, 256, 257, 264, 260, 241, 308, 262, 243, 239, 242, 268, 271, 259, 255, 262, 248, 238, 239, 274, 253, 263, 244, 258, 262, 235, 243, 266, 246, 253, 259, 250, 275, 269, 278, 255, 277, 265, 272, 237, 267, 256, 235, 255, 283, 238, 264, 227, 270, 226, 238, 265] payload_rows_per_dst=[5432, 5402, 5402, 5458, 5596, 5440, 5442, 5462] dup_rows_per_dst=[2700, 2702, 2662, 2734, 2862, 2778, 2776, 2688] combine_rows_per_src=[5432, 5402, 5402, 5458, 5596, 5440, 5442, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44038 payload_rows_remote=38464 dup_rows=21498 dup_rows_remote=18806 dedup_ratio=0.328033 router_tokens_per_expert=[250, 249, 272, 244, 250, 246, 245, 265, 259, 261, 268, 273, 273, 239, 297, 264, 254, 248, 280, 228, 246, 235, 246, 274, 306, 255, 260, 218, 278, 252, 292, 261, 254, 257, 271, 237, 232, 278, 230, 252, 273, 266, 241, 243, 229, 253, 250, 259, 253, 273, 255, 253, 271, 246, 265, 259, 268, 262, 273, 253, 236, 257, 281, 246, 235, 247, 268, 277, 253, 268, 263, 247, 234, 254, 271, 254, 271, 279, 253, 245, 277, 258, 237, 239, 261, 243, 257, 291, 242, 220, 248, 284, 264, 257, 272, 270, 261, 261, 244, 280, 254, 241, 271, 252, 247, 244, 236, 255, 270, 244, 247, 245, 254, 251, 276, 227, 261, 259, 291, 231, 224, 256, 241, 254, 228, 275, 246, 239] payload_rows_per_dst=[5562, 5574, 5476, 5534, 5514, 5542, 5442, 5394] dup_rows_per_dst=[2748, 2692, 2574, 2768, 2724, 2698, 2662, 2632] combine_rows_per_src=[5562, 5574, 5476, 5534, 5514, 5542, 5442, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157523968 dispatch_meta_bytes=229568 combine_payload_bytes=157523968 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157753536 ingress_bytes=233021440 link_bytes=233021440 token_bytes=315277504 weight_movement_bytes=75497472 total_nvlink_bytes=390774976 weight_to_token_ratio=0.239464 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.232696784e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38332 dup_rows=21778 dup_rows_remote=19240 dedup_ratio=0.332306 router_tokens_per_expert=[256, 237, 249, 254, 259, 283, 252, 259, 265, 236, 224, 261, 251, 246, 276, 254, 241, 252, 244, 278, 271, 272, 249, 283, 270, 301, 281, 252, 271, 235, 279, 253, 272, 241, 239, 238, 270, 229, 274, 237, 269, 257, 236, 265, 254, 233, 295, 250, 254, 261, 266, 271, 260, 212, 242, 294, 278, 246, 257, 269, 248, 245, 250, 241, 250, 265, 255, 264, 247, 238, 247, 225, 245, 254, 249, 219, 261, 272, 258, 233, 239, 273, 230, 248, 275, 280, 236, 241, 237, 263, 249, 272, 276, 284, 262, 250, 288, 252, 267, 257, 253, 252, 272, 263, 254, 256, 251, 239, 261, 261, 254, 251, 283, 276, 264, 238, 259, 232, 230, 281, 260, 255, 238, 241, 226, 281, 246, 283] payload_rows_per_dst=[5416, 5552, 5504, 5404, 5426, 5492, 5490, 5474] dup_rows_per_dst=[2708, 2912, 2614, 2784, 2538, 2738, 2772, 2712] combine_rows_per_src=[5416, 5552, 5504, 5404, 5426, 5492, 5490, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228880 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156720656 ingress_bytes=231989248 link_bytes=231989248 token_bytes=313212432 weight_movement_bytes=75497472 total_nvlink_bytes=388709904 weight_to_token_ratio=0.241042 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222806829e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38282 dup_rows=21780 dup_rows_remote=19042 dedup_ratio=0.332336 router_tokens_per_expert=[295, 275, 260, 268, 252, 246, 262, 253, 282, 260, 274, 252, 236, 234, 271, 261, 271, 257, 240, 249, 276, 293, 221, 268, 251, 268, 257, 239, 259, 231, 227, 235, 264, 272, 237, 248, 261, 239, 267, 235, 251, 274, 236, 220, 259, 247, 250, 246, 247, 244, 260, 259, 233, 265, 255, 241, 276, 257, 239, 267, 264, 247, 258, 260, 231, 261, 263, 284, 265, 271, 272, 262, 250, 262, 261, 274, 216, 235, 233, 257, 224, 249, 286, 265, 264, 231, 254, 243, 273, 296, 287, 235, 239, 258, 263, 248, 267, 257, 250, 252, 249, 288, 237, 257, 249, 261, 258, 269, 255, 267, 273, 269, 289, 251, 261, 229, 265, 245, 255, 269, 245, 267, 276, 272, 237, 237, 253, 246] payload_rows_per_dst=[5444, 5528, 5392, 5466, 5436, 5484, 5532, 5474] dup_rows_per_dst=[2768, 2706, 2620, 2678, 2758, 2728, 2784, 2738] combine_rows_per_src=[5444, 5528, 5392, 5466, 5436, 5484, 5532, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156344320 dispatch_meta_bytes=229104 combine_payload_bytes=156344320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156573424 ingress_bytes=175218688 link_bytes=175218688 token_bytes=312917744 weight_movement_bytes=18874368 total_nvlink_bytes=331792112 weight_to_token_ratio=0.060317 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678859255e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229272 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156671896 ingress_bytes=194191360 link_bytes=194191360 token_bytes=313114520 weight_movement_bytes=37748736 total_nvlink_bytes=350863256 weight_to_token_ratio=0.120559 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860646063e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=229080 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157777624 ingress_bytes=176422912 link_bytes=176422912 token_bytes=315326168 weight_movement_bytes=18874368 total_nvlink_bytes=334200536 weight_to_token_ratio=0.059857 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690397537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38234 dup_rows=21784 dup_rows_remote=19022 dedup_ratio=0.332397 router_tokens_per_expert=[256, 264, 248, 246, 283, 259, 244, 243, 262, 231, 254, 252, 250, 242, 258, 232, 244, 236, 256, 272, 249, 239, 268, 268, 265, 255, 270, 238, 248, 255, 242, 245, 262, 261, 258, 232, 243, 248, 280, 274, 279, 249, 247, 268, 285, 239, 229, 274, 258, 238, 263, 257, 280, 290, 266, 230, 272, 251, 254, 239, 263, 265, 256, 258, 258, 260, 235, 249, 253, 262, 226, 232, 260, 274, 248, 243, 268, 278, 254, 263, 275, 256, 261, 265, 227, 274, 261, 284, 242, 269, 249, 256, 275, 261, 260, 262, 253, 274, 234, 263, 234, 267, 247, 278, 248, 254, 238, 242, 244, 267, 239, 251, 260, 272, 287, 265, 262, 259, 243, 264, 261, 255, 268, 249, 255, 258, 260, 235] payload_rows_per_dst=[5414, 5438, 5484, 5518, 5514, 5524, 5400, 5460] dup_rows_per_dst=[2634, 2662, 2772, 2762, 2612, 2830, 2666, 2846] combine_rows_per_src=[5414, 5438, 5484, 5518, 5514, 5524, 5400, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=230288 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157238160 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314246032 weight_movement_bytes=18874368 total_nvlink_bytes=333120400 weight_to_token_ratio=0.060062 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229296 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032368 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835440 weight_movement_bytes=37748736 total_nvlink_bytes=351584176 weight_to_token_ratio=0.120282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38306 dup_rows=21888 dup_rows_remote=19292 dedup_ratio=0.333984 router_tokens_per_expert=[266, 291, 240, 282, 243, 275, 247, 237, 245, 304, 263, 255, 280, 275, 249, 229, 258, 260, 266, 244, 286, 267, 272, 249, 255, 260, 249, 253, 233, 231, 263, 266, 286, 257, 293, 261, 233, 275, 244, 216, 267, 232, 213, 244, 275, 264, 269, 267, 252, 235, 246, 243, 253, 286, 225, 257, 261, 269, 253, 273, 256, 254, 260, 244, 258, 272, 239, 251, 254, 275, 231, 278, 258, 241, 233, 238, 241, 239, 262, 261, 251, 249, 249, 242, 254, 249, 256, 289, 266, 287, 251, 255, 252, 256, 279, 279, 243, 245, 273, 235, 228, 268, 231, 246, 227, 286, 276, 262, 227, 253, 232, 237, 273, 264, 267, 242, 243, 249, 247, 285, 285, 253, 243, 284, 264, 260, 234, 255] payload_rows_per_dst=[5512, 5422, 5474, 5404, 5370, 5530, 5342, 5594] dup_rows_per_dst=[2850, 2802, 2718, 2730, 2692, 2798, 2596, 2702] combine_rows_per_src=[5512, 5422, 5474, 5404, 5370, 5530, 5342, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229024 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156835488 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313441952 weight_movement_bytes=18874368 total_nvlink_bytes=332316320 weight_to_token_ratio=0.060216 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38104 dup_rows=21794 dup_rows_remote=18670 dedup_ratio=0.332550 router_tokens_per_expert=[261, 260, 263, 248, 271, 255, 238, 225, 253, 286, 250, 281, 267, 257, 241, 267, 269, 286, 261, 266, 264, 256, 269, 242, 264, 252, 225, 258, 242, 284, 239, 256, 248, 237, 240, 274, 240, 274, 251, 280, 252, 246, 243, 231, 255, 254, 225, 267, 229, 274, 264, 253, 230, 292, 263, 253, 271, 275, 247, 274, 262, 265, 242, 247, 255, 280, 242, 263, 250, 242, 250, 258, 259, 259, 280, 275, 256, 226, 232, 236, 232, 250, 239, 241, 260, 265, 249, 271, 272, 273, 264, 267, 218, 248, 271, 268, 231, 268, 258, 262, 258, 228, 232, 259, 262, 273, 234, 246, 234, 254, 246, 257, 262, 258, 250, 291, 279, 246, 251, 259, 260, 274, 254, 277, 268, 253, 269, 250] payload_rows_per_dst=[5638, 5584, 5366, 5614, 5368, 5328, 5416, 5428] dup_rows_per_dst=[3124, 3000, 2362, 3052, 2746, 2332, 2588, 2590] combine_rows_per_src=[5638, 5584, 5366, 5614, 5368, 5328, 5416, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38244 dup_rows=21902 dup_rows_remote=19230 dedup_ratio=0.334198 router_tokens_per_expert=[256, 259, 245, 239, 254, 246, 265, 241, 237, 263, 257, 259, 269, 246, 262, 261, 297, 258, 279, 215, 246, 247, 264, 279, 269, 277, 247, 267, 265, 248, 273, 296, 272, 268, 242, 259, 245, 249, 229, 248, 233, 243, 267, 227, 251, 289, 248, 261, 250, 256, 271, 247, 253, 220, 266, 276, 256, 238, 251, 256, 280, 223, 274, 267, 232, 264, 284, 227, 285, 266, 243, 230, 268, 257, 264, 249, 245, 244, 258, 282, 241, 260, 259, 274, 249, 253, 243, 228, 287, 255, 256, 233, 217, 260, 258, 274, 260, 267, 267, 257, 240, 254, 231, 250, 259, 269, 265, 236, 257, 271, 240, 270, 262, 258, 240, 244, 272, 271, 273, 244, 263, 261, 265, 258, 250, 246, 265, 257] payload_rows_per_dst=[5406, 5508, 5390, 5432, 5442, 5430, 5548, 5478] dup_rows_per_dst=[2712, 2946, 2672, 2736, 2754, 2664, 2638, 2780] combine_rows_per_src=[5406, 5508, 5390, 5432, 5442, 5430, 5548, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38114 dup_rows=21930 dup_rows_remote=19114 dedup_ratio=0.334625 router_tokens_per_expert=[235, 273, 274, 240, 260, 269, 250, 276, 245, 233, 276, 284, 256, 272, 267, 253, 247, 260, 272, 258, 277, 231, 270, 256, 267, 266, 227, 254, 267, 288, 260, 254, 266, 262, 238, 240, 262, 266, 267, 251, 232, 216, 276, 256, 264, 245, 289, 267, 263, 248, 221, 267, 256, 237, 260, 268, 246, 239, 239, 245, 279, 251, 257, 241, 248, 264, 228, 245, 263, 262, 256, 223, 246, 264, 246, 262, 255, 246, 242, 282, 270, 265, 261, 235, 250, 255, 249, 254, 264, 281, 242, 261, 239, 261, 257, 255, 238, 253, 251, 267, 288, 270, 248, 271, 264, 277, 236, 232, 255, 261, 249, 232, 244, 275, 265, 249, 264, 229, 265, 283, 271, 259, 244, 239, 273, 222, 260, 272] payload_rows_per_dst=[5580, 5492, 5372, 5298, 5454, 5538, 5390, 5482] dup_rows_per_dst=[2746, 2816, 2822, 2736, 2610, 2660, 2794, 2746] combine_rows_per_src=[5580, 5492, 5372, 5298, 5454, 5538, 5390, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=230392 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157131768 ingress_bytes=175775744 link_bytes=175775744 token_bytes=314033144 weight_movement_bytes=18874368 total_nvlink_bytes=332907512 weight_to_token_ratio=0.060103 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684196692e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38188 dup_rows=21858 dup_rows_remote=19106 dedup_ratio=0.333527 router_tokens_per_expert=[243, 242, 269, 252, 252, 269, 266, 255, 272, 271, 251, 252, 225, 242, 262, 235, 242, 251, 254, 264, 246, 258, 257, 239, 244, 256, 250, 272, 262, 269, 286, 279, 238, 233, 264, 226, 240, 235, 238, 250, 268, 281, 249, 261, 281, 252, 246, 260, 258, 275, 243, 275, 231, 269, 246, 256, 285, 249, 256, 247, 265, 267, 279, 261, 244, 241, 294, 260, 254, 230, 242, 262, 248, 233, 249, 277, 246, 237, 266, 244, 246, 244, 247, 255, 301, 249, 241, 244, 285, 257, 245, 277, 242, 281, 249, 258, 284, 253, 253, 265, 247, 232, 258, 271, 241, 248, 250, 263, 254, 274, 250, 278, 277, 260, 265, 273, 244, 259, 256, 252, 256, 241, 257, 277, 236, 262, 267, 246] payload_rows_per_dst=[5452, 5472, 5420, 5468, 5424, 5490, 5488, 5464] dup_rows_per_dst=[2664, 2786, 2624, 2856, 2630, 2752, 2754, 2792] combine_rows_per_src=[5452, 5472, 5420, 5468, 5424, 5490, 5488, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156073984 dispatch_meta_bytes=227096 combine_payload_bytes=156073984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156301080 ingress_bytes=174948352 link_bytes=174948352 token_bytes=312375064 weight_movement_bytes=18874368 total_nvlink_bytes=331249432 weight_to_token_ratio=0.060422 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676269029e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43930 payload_rows_remote=38372 dup_rows=21606 dup_rows_remote=18936 dedup_ratio=0.329681 router_tokens_per_expert=[239, 243, 232, 242, 237, 257, 271, 248, 272, 234, 260, 258, 273, 266, 272, 227, 269, 260, 233, 274, 287, 245, 262, 281, 244, 203, 255, 256, 243, 254, 262, 263, 258, 260, 237, 238, 267, 255, 280, 266, 282, 238, 275, 248, 241, 249, 253, 264, 290, 239, 253, 236, 250, 256, 239, 245, 247, 231, 263, 265, 268, 261, 267, 242, 261, 262, 260, 257, 251, 249, 265, 245, 245, 255, 265, 260, 252, 254, 267, 266, 270, 279, 319, 267, 262, 230, 252, 228, 253, 249, 285, 276, 277, 246, 265, 264, 241, 242, 265, 257, 253, 249, 266, 264, 255, 251, 268, 233, 258, 253, 289, 244, 256, 242, 271, 255, 230, 268, 288, 236, 257, 240, 226, 254, 273, 260, 265, 238] payload_rows_per_dst=[5438, 5448, 5488, 5434, 5558, 5570, 5468, 5526] dup_rows_per_dst=[2624, 2734, 2734, 2670, 2670, 2874, 2708, 2592] combine_rows_per_src=[5438, 5448, 5488, 5434, 5558, 5570, 5468, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38206 dup_rows=21718 dup_rows_remote=18632 dedup_ratio=0.331390 router_tokens_per_expert=[274, 274, 262, 270, 284, 243, 259, 265, 257, 240, 273, 230, 284, 261, 256, 263, 263, 245, 238, 261, 256, 234, 217, 238, 287, 239, 271, 237, 298, 250, 253, 271, 238, 239, 259, 280, 235, 250, 266, 251, 258, 268, 269, 251, 244, 247, 264, 251, 283, 252, 275, 228, 250, 261, 245, 264, 278, 246, 237, 236, 262, 286, 269, 236, 278, 249, 301, 283, 251, 247, 260, 281, 275, 253, 264, 229, 231, 256, 258, 243, 264, 248, 250, 280, 259, 226, 253, 260, 256, 251, 260, 282, 249, 261, 250, 240, 240, 259, 230, 236, 259, 242, 263, 257, 236, 233, 273, 262, 239, 252, 258, 260, 262, 275, 264, 251, 240, 258, 258, 257, 268, 252, 266, 237, 252, 252, 274, 224] payload_rows_per_dst=[5404, 5432, 5448, 5518, 5536, 5490, 5378, 5612] dup_rows_per_dst=[2420, 2684, 2692, 2606, 2830, 2780, 2620, 3086] combine_rows_per_src=[5404, 5432, 5448, 5518, 5536, 5490, 5378, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229896 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156877320 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524744 weight_movement_bytes=37748736 total_nvlink_bytes=351273480 weight_to_token_ratio=0.120401 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=228912 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156343856 ingress_bytes=174989312 link_bytes=174989312 token_bytes=312458800 weight_movement_bytes=18874368 total_nvlink_bytes=331333168 weight_to_token_ratio=0.060406 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676661487e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38206 dup_rows=21858 dup_rows_remote=19000 dedup_ratio=0.333527 router_tokens_per_expert=[269, 233, 275, 250, 236, 271, 270, 235, 255, 248, 241, 241, 249, 274, 260, 285, 271, 285, 245, 264, 269, 236, 249, 276, 255, 256, 226, 244, 271, 233, 263, 265, 250, 261, 271, 257, 229, 301, 251, 235, 259, 236, 238, 225, 264, 245, 270, 255, 254, 261, 250, 253, 282, 285, 246, 270, 261, 255, 253, 283, 247, 255, 242, 268, 261, 261, 251, 268, 256, 227, 269, 248, 246, 248, 260, 243, 251, 228, 249, 253, 277, 235, 251, 258, 268, 279, 279, 253, 241, 282, 259, 253, 276, 247, 267, 272, 269, 286, 271, 238, 258, 252, 280, 256, 222, 271, 276, 248, 272, 257, 241, 263, 237, 287, 239, 234, 242, 259, 245, 258, 272, 232, 252, 262, 248, 245, 224, 244] payload_rows_per_dst=[5424, 5496, 5392, 5472, 5414, 5578, 5552, 5350] dup_rows_per_dst=[2760, 2720, 2702, 2858, 2624, 2816, 2768, 2610] combine_rows_per_src=[5424, 5496, 5392, 5472, 5414, 5578, 5552, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=229176 combine_payload_bytes=156418048 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156647224 ingress_bytes=250789888 link_bytes=250789888 token_bytes=313065272 weight_movement_bytes=94371840 total_nvlink_bytes=407437112 weight_to_token_ratio=0.301445 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.402945310e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229232 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157400944 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314572656 weight_movement_bytes=18874368 total_nvlink_bytes=333447024 weight_to_token_ratio=0.060000 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=227352 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156719128 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313210904 weight_movement_bytes=18874368 total_nvlink_bytes=332085272 weight_to_token_ratio=0.060261 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=228824 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156720600 ingress_bytes=213114880 link_bytes=213114880 token_bytes=313212376 weight_movement_bytes=56623104 total_nvlink_bytes=369835480 weight_to_token_ratio=0.180782 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041961921e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38360 dup_rows=21634 dup_rows_remote=18976 dedup_ratio=0.330109 router_tokens_per_expert=[246, 273, 280, 259, 276, 262, 297, 274, 247, 252, 249, 262, 271, 255, 250, 248, 269, 252, 245, 241, 251, 240, 228, 250, 239, 274, 218, 262, 274, 232, 259, 280, 260, 253, 261, 242, 259, 267, 290, 263, 261, 253, 285, 255, 261, 259, 232, 243, 260, 249, 266, 251, 264, 240, 263, 274, 248, 268, 241, 275, 253, 244, 260, 260, 278, 244, 261, 249, 264, 226, 261, 254, 280, 264, 265, 259, 250, 264, 264, 280, 247, 256, 253, 271, 261, 259, 270, 232, 250, 237, 267, 261, 245, 243, 258, 255, 258, 222, 255, 234, 237, 240, 250, 269, 270, 276, 255, 266, 264, 269, 270, 265, 240, 248, 279, 244, 245, 264, 245, 263, 255, 258, 238, 232, 215, 248, 237, 254] payload_rows_per_dst=[5534, 5426, 5458, 5460, 5572, 5474, 5542, 5436] dup_rows_per_dst=[2736, 2602, 2830, 2772, 2754, 2656, 2658, 2626] combine_rows_per_src=[5534, 5426, 5458, 5460, 5572, 5474, 5542, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38336 dup_rows=21786 dup_rows_remote=19096 dedup_ratio=0.332428 router_tokens_per_expert=[248, 259, 267, 259, 256, 271, 260, 264, 219, 250, 237, 240, 263, 265, 238, 235, 259, 228, 280, 279, 244, 243, 284, 256, 262, 219, 253, 251, 230, 227, 245, 260, 262, 262, 244, 254, 241, 267, 252, 237, 238, 273, 248, 259, 250, 268, 261, 273, 262, 253, 260, 246, 284, 281, 262, 279, 262, 283, 235, 244, 240, 248, 248, 259, 235, 228, 251, 252, 252, 236, 262, 256, 269, 267, 242, 262, 254, 261, 273, 252, 252, 275, 277, 250, 263, 246, 259, 253, 233, 241, 280, 273, 272, 265, 266, 279, 247, 234, 262, 240, 261, 267, 259, 259, 263, 251, 256, 256, 265, 248, 241, 257, 266, 259, 271, 256, 254, 243, 252, 269, 241, 272, 263, 275, 273, 252, 258, 276] payload_rows_per_dst=[5414, 5560, 5492, 5518, 5324, 5552, 5422, 5468] dup_rows_per_dst=[2690, 2926, 2646, 2774, 2332, 3064, 2710, 2644] combine_rows_per_src=[5414, 5560, 5492, 5518, 5324, 5552, 5422, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38190 dup_rows=21890 dup_rows_remote=19206 dedup_ratio=0.334015 router_tokens_per_expert=[244, 263, 241, 250, 267, 243, 260, 259, 253, 258, 259, 266, 279, 253, 249, 257, 229, 273, 262, 261, 281, 247, 251, 264, 276, 255, 235, 263, 251, 256, 236, 251, 263, 260, 246, 282, 264, 274, 227, 279, 266, 216, 261, 258, 241, 230, 242, 261, 241, 230, 279, 280, 254, 241, 234, 239, 271, 261, 267, 260, 259, 279, 272, 259, 257, 253, 232, 247, 269, 237, 275, 228, 241, 256, 266, 260, 269, 242, 254, 267, 271, 259, 263, 262, 254, 252, 272, 256, 257, 262, 270, 240, 246, 235, 266, 270, 272, 254, 255, 263, 262, 261, 262, 260, 248, 251, 264, 254, 238, 275, 243, 255, 263, 256, 240, 249, 251, 256, 242, 257, 243, 258, 250, 278, 268, 248, 262, 254] payload_rows_per_dst=[5404, 5374, 5456, 5480, 5532, 5484, 5466, 5450] dup_rows_per_dst=[2798, 2808, 2684, 2772, 2574, 2786, 2768, 2700] combine_rows_per_src=[5404, 5374, 5456, 5480, 5532, 5484, 5466, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229344 combine_payload_bytes=157122560 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157351904 ingress_bytes=213745664 link_bytes=213745664 token_bytes=314474464 weight_movement_bytes=56623104 total_nvlink_bytes=371097568 weight_to_token_ratio=0.180056 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048005783e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43786 payload_rows_remote=38446 dup_rows=21750 dup_rows_remote=19090 dedup_ratio=0.331879 router_tokens_per_expert=[235, 241, 243, 238, 237, 284, 263, 245, 266, 257, 256, 281, 252, 241, 245, 252, 229, 252, 239, 256, 287, 255, 261, 227, 244, 251, 235, 263, 257, 233, 259, 252, 252, 251, 279, 285, 254, 229, 255, 268, 246, 275, 255, 279, 245, 269, 280, 258, 270, 230, 241, 284, 249, 245, 246, 260, 263, 243, 253, 250, 257, 282, 268, 221, 250, 278, 250, 263, 293, 269, 253, 273, 265, 265, 268, 265, 230, 285, 233, 287, 248, 226, 264, 261, 248, 255, 260, 246, 267, 222, 253, 286, 261, 249, 228, 250, 233, 256, 261, 277, 258, 222, 250, 268, 272, 252, 263, 249, 269, 269, 262, 292, 262, 243, 246, 257, 292, 254, 259, 272, 233, 250, 256, 253, 253, 274, 235, 247] payload_rows_per_dst=[5536, 5340, 5506, 5398, 5510, 5424, 5536, 5536] dup_rows_per_dst=[2660, 2660, 2730, 2726, 2944, 2624, 2770, 2636] combine_rows_per_src=[5536, 5340, 5506, 5398, 5510, 5424, 5536, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38072 dup_rows=21952 dup_rows_remote=19118 dedup_ratio=0.334961 router_tokens_per_expert=[254, 275, 276, 267, 250, 278, 269, 238, 260, 269, 239, 254, 259, 271, 247, 238, 248, 237, 265, 245, 240, 270, 233, 269, 262, 246, 246, 270, 266, 226, 259, 245, 231, 252, 242, 251, 262, 246, 222, 249, 246, 253, 242, 274, 247, 249, 220, 273, 270, 247, 245, 245, 255, 274, 264, 260, 278, 261, 275, 265, 246, 261, 283, 281, 260, 244, 280, 249, 274, 253, 238, 237, 227, 253, 265, 277, 273, 242, 272, 259, 303, 260, 286, 281, 257, 246, 229, 255, 261, 217, 253, 268, 277, 246, 279, 255, 234, 267, 269, 242, 261, 279, 248, 259, 269, 242, 284, 249, 263, 253, 237, 262, 255, 266, 248, 290, 210, 242, 266, 237, 244, 265, 255, 257, 260, 248, 263, 228] payload_rows_per_dst=[5486, 5404, 5308, 5544, 5526, 5512, 5448, 5356] dup_rows_per_dst=[2802, 2650, 2610, 2876, 2680, 2834, 2788, 2712] combine_rows_per_src=[5486, 5404, 5308, 5544, 5526, 5512, 5448, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229728 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157253984 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314278240 weight_movement_bytes=37748736 total_nvlink_bytes=352026976 weight_to_token_ratio=0.120112 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229584 combine_payload_bytes=156426240 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156655824 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313082064 weight_movement_bytes=37748736 total_nvlink_bytes=350830800 weight_to_token_ratio=0.120571 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38156 dup_rows=21898 dup_rows_remote=19074 dedup_ratio=0.334137 router_tokens_per_expert=[258, 261, 261, 270, 259, 245, 258, 254, 247, 274, 270, 262, 249, 247, 249, 247, 251, 252, 255, 263, 254, 247, 273, 285, 238, 239, 237, 271, 275, 232, 248, 290, 275, 271, 263, 278, 253, 239, 240, 246, 246, 265, 237, 272, 238, 277, 255, 284, 243, 264, 263, 257, 239, 228, 276, 251, 271, 250, 268, 269, 255, 257, 239, 233, 273, 249, 269, 260, 280, 260, 257, 205, 251, 247, 259, 240, 251, 261, 285, 244, 249, 266, 248, 265, 245, 248, 254, 239, 268, 278, 244, 247, 277, 234, 268, 242, 248, 244, 283, 212, 260, 252, 263, 238, 265, 262, 261, 214, 278, 247, 270, 233, 250, 255, 289, 265, 227, 250, 252, 278, 276, 272, 255, 269, 251, 280, 241, 242] payload_rows_per_dst=[5456, 5342, 5594, 5488, 5414, 5444, 5418, 5482] dup_rows_per_dst=[2766, 2636, 2894, 2638, 2768, 2700, 2672, 2824] combine_rows_per_src=[5456, 5342, 5594, 5488, 5414, 5444, 5418, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38314 dup_rows=21634 dup_rows_remote=18816 dedup_ratio=0.330109 router_tokens_per_expert=[252, 241, 267, 258, 266, 266, 274, 236, 259, 227, 246, 263, 255, 247, 228, 239, 246, 264, 267, 265, 251, 256, 299, 259, 250, 266, 285, 248, 239, 276, 250, 241, 263, 236, 259, 273, 250, 274, 255, 260, 280, 265, 251, 240, 234, 263, 254, 244, 274, 271, 251, 254, 261, 274, 272, 236, 267, 267, 234, 247, 249, 238, 230, 277, 256, 280, 257, 254, 268, 266, 253, 250, 243, 284, 246, 267, 258, 298, 253, 270, 239, 282, 234, 234, 269, 269, 256, 253, 268, 270, 269, 273, 256, 236, 241, 250, 253, 253, 279, 266, 238, 255, 254, 248, 245, 252, 282, 234, 273, 260, 247, 258, 254, 239, 265, 266, 257, 239, 268, 258, 244, 254, 230, 239, 250, 236, 235, 246] payload_rows_per_dst=[5438, 5572, 5464, 5470, 5588, 5442, 5540, 5388] dup_rows_per_dst=[2610, 2752, 2738, 2734, 2818, 2756, 2654, 2572] combine_rows_per_src=[5438, 5572, 5464, 5470, 5588, 5442, 5540, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38270 dup_rows=21760 dup_rows_remote=19136 dedup_ratio=0.332031 router_tokens_per_expert=[236, 224, 256, 270, 245, 241, 261, 260, 266, 260, 290, 239, 258, 276, 259, 222, 245, 220, 251, 242, 250, 251, 241, 251, 261, 270, 279, 263, 260, 234, 230, 262, 263, 235, 261, 246, 275, 300, 269, 258, 234, 255, 248, 282, 267, 279, 256, 263, 261, 253, 229, 266, 267, 246, 264, 259, 277, 268, 263, 248, 214, 262, 242, 246, 237, 228, 251, 259, 268, 261, 270, 265, 256, 266, 239, 234, 226, 270, 248, 267, 256, 263, 242, 280, 268, 282, 258, 236, 273, 246, 276, 261, 250, 236, 239, 272, 241, 252, 282, 250, 241, 263, 260, 252, 252, 249, 272, 258, 253, 271, 249, 273, 253, 269, 255, 249, 264, 254, 274, 261, 250, 262, 276, 259, 269, 250, 258, 235] payload_rows_per_dst=[5424, 5404, 5516, 5506, 5422, 5468, 5514, 5522] dup_rows_per_dst=[2702, 2616, 2866, 2624, 2668, 2808, 2722, 2754] combine_rows_per_src=[5424, 5404, 5516, 5506, 5422, 5468, 5514, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=230144 combine_payload_bytes=157474816 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157704960 ingress_bytes=214097920 link_bytes=214097920 token_bytes=315179776 weight_movement_bytes=56623104 total_nvlink_bytes=371802880 weight_to_token_ratio=0.179653 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051380927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=228760 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156171672 ingress_bytes=174817280 link_bytes=174817280 token_bytes=312114584 weight_movement_bytes=18874368 total_nvlink_bytes=330988952 weight_to_token_ratio=0.060473 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675013161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228920 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156515896 ingress_bytes=212910080 link_bytes=212910080 token_bytes=312802872 weight_movement_bytes=56623104 total_nvlink_bytes=369425976 weight_to_token_ratio=0.181018 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039999628e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=228520 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157162664 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314096808 weight_movement_bytes=18874368 total_nvlink_bytes=332971176 weight_to_token_ratio=0.060091 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38164 dup_rows=21956 dup_rows_remote=19196 dedup_ratio=0.335022 router_tokens_per_expert=[242, 252, 256, 259, 254, 262, 235, 262, 232, 230, 239, 238, 248, 246, 257, 233, 247, 252, 235, 285, 279, 252, 254, 265, 283, 260, 260, 251, 263, 236, 277, 240, 262, 262, 247, 229, 254, 281, 262, 259, 252, 284, 264, 232, 247, 286, 247, 263, 281, 252, 259, 265, 227, 244, 262, 249, 234, 268, 246, 258, 277, 255, 252, 281, 247, 237, 256, 260, 250, 267, 261, 266, 252, 242, 263, 266, 241, 270, 252, 245, 212, 234, 267, 262, 270, 249, 262, 261, 266, 275, 262, 243, 237, 260, 250, 267, 260, 242, 238, 262, 267, 256, 259, 237, 258, 243, 279, 291, 248, 229, 257, 262, 240, 252, 259, 233, 266, 263, 247, 249, 282, 298, 259, 273, 269, 265, 267, 281] payload_rows_per_dst=[5340, 5472, 5446, 5484, 5430, 5468, 5416, 5524] dup_rows_per_dst=[2550, 2806, 2816, 2736, 2720, 2686, 2760, 2882] combine_rows_per_src=[5340, 5472, 5446, 5484, 5430, 5468, 5416, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229624 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156983544 ingress_bytes=175628288 link_bytes=175628288 token_bytes=313737464 weight_movement_bytes=18874368 total_nvlink_bytes=332611832 weight_to_token_ratio=0.060160 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682783841e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38156 dup_rows=21852 dup_rows_remote=19102 dedup_ratio=0.333435 router_tokens_per_expert=[271, 253, 273, 256, 263, 272, 270, 250, 248, 242, 273, 254, 253, 242, 270, 249, 235, 198, 250, 236, 265, 265, 299, 235, 275, 259, 276, 226, 245, 270, 239, 249, 238, 233, 288, 247, 305, 253, 277, 222, 259, 298, 236, 234, 265, 277, 251, 256, 247, 247, 252, 276, 251, 238, 236, 264, 266, 263, 257, 246, 249, 244, 255, 259, 259, 268, 244, 230, 292, 240, 233, 241, 260, 190, 248, 238, 257, 261, 256, 249, 248, 249, 257, 280, 282, 252, 260, 260, 228, 266, 278, 267, 252, 263, 252, 267, 266, 257, 253, 240, 258, 238, 257, 238, 256, 280, 268, 263, 259, 270, 257, 269, 238, 258, 265, 237, 255, 265, 254, 253, 267, 284, 249, 290, 238, 258, 263, 288] payload_rows_per_dst=[5526, 5326, 5528, 5446, 5314, 5498, 5510, 5536] dup_rows_per_dst=[2752, 2718, 2750, 2654, 2618, 2824, 2748, 2788] combine_rows_per_src=[5526, 5326, 5528, 5446, 5314, 5498, 5510, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38076 dup_rows=21886 dup_rows_remote=19042 dedup_ratio=0.333954 router_tokens_per_expert=[269, 247, 260, 270, 288, 259, 265, 255, 265, 218, 269, 249, 261, 252, 230, 268, 248, 255, 263, 266, 249, 251, 261, 244, 264, 246, 259, 246, 238, 257, 241, 260, 249, 267, 245, 287, 240, 238, 257, 244, 245, 269, 263, 248, 263, 263, 289, 248, 258, 263, 282, 257, 254, 244, 248, 264, 262, 278, 243, 242, 262, 227, 263, 260, 268, 244, 233, 247, 254, 244, 252, 257, 273, 285, 267, 235, 246, 266, 267, 264, 259, 232, 260, 281, 276, 246, 272, 269, 266, 238, 226, 272, 264, 275, 239, 273, 256, 272, 296, 244, 230, 257, 267, 266, 268, 274, 233, 273, 252, 250, 294, 250, 266, 278, 214, 234, 281, 255, 260, 234, 241, 228, 237, 239, 246, 227, 241, 260] payload_rows_per_dst=[5574, 5452, 5580, 5436, 5392, 5502, 5404, 5310] dup_rows_per_dst=[2844, 2662, 2850, 2778, 2812, 2650, 2718, 2572] combine_rows_per_src=[5574, 5452, 5580, 5436, 5392, 5502, 5404, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229440 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156549184 ingress_bytes=194068480 link_bytes=194068480 token_bytes=312868928 weight_movement_bytes=37748736 total_nvlink_bytes=350617664 weight_to_token_ratio=0.120654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859468688e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=229032 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156516008 ingress_bytes=212910080 link_bytes=212910080 token_bytes=312802984 weight_movement_bytes=56623104 total_nvlink_bytes=369426088 weight_to_token_ratio=0.181018 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039999628e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=155959296 dispatch_meta_bytes=228472 combine_payload_bytes=155959296 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156187768 ingress_bytes=174833664 link_bytes=174833664 token_bytes=312147064 weight_movement_bytes=18874368 total_nvlink_bytes=331021432 weight_to_token_ratio=0.060466 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675170145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38606 dup_rows=21578 dup_rows_remote=18946 dedup_ratio=0.329254 router_tokens_per_expert=[265, 267, 256, 250, 242, 252, 270, 227, 255, 265, 273, 238, 239, 266, 270, 248, 253, 269, 249, 266, 232, 264, 250, 216, 254, 267, 256, 228, 226, 238, 248, 276, 277, 219, 276, 255, 258, 269, 252, 247, 286, 249, 278, 246, 277, 258, 245, 253, 226, 278, 233, 254, 241, 271, 245, 259, 257, 243, 274, 261, 225, 279, 276, 270, 247, 278, 269, 277, 239, 230, 246, 274, 238, 231, 249, 260, 270, 251, 245, 273, 242, 256, 256, 242, 276, 249, 231, 261, 263, 253, 255, 264, 255, 260, 280, 244, 268, 259, 256, 231, 285, 223, 253, 252, 239, 285, 239, 268, 258, 241, 249, 261, 250, 265, 269, 255, 259, 264, 273, 259, 281, 273, 262, 256, 247, 263, 286, 263] payload_rows_per_dst=[5558, 5352, 5552, 5446, 5452, 5542, 5450, 5606] dup_rows_per_dst=[2608, 2632, 2738, 2738, 2702, 2632, 2684, 2844] combine_rows_per_src=[5558, 5352, 5552, 5446, 5452, 5542, 5450, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38452 dup_rows=21682 dup_rows_remote=19086 dedup_ratio=0.330841 router_tokens_per_expert=[253, 285, 261, 242, 259, 252, 253, 272, 272, 254, 239, 265, 274, 271, 251, 215, 264, 279, 248, 249, 291, 264, 257, 240, 245, 270, 263, 251, 246, 260, 285, 272, 244, 245, 227, 256, 244, 275, 237, 235, 228, 227, 261, 263, 271, 286, 278, 261, 252, 266, 279, 276, 230, 246, 267, 251, 261, 291, 263, 279, 269, 249, 254, 230, 242, 272, 266, 293, 241, 230, 241, 256, 248, 255, 244, 233, 244, 233, 255, 239, 258, 251, 269, 283, 239, 247, 230, 225, 235, 240, 255, 250, 240, 262, 247, 268, 253, 269, 272, 245, 263, 269, 276, 220, 277, 268, 261, 283, 245, 265, 257, 268, 279, 239, 250, 263, 245, 259, 235, 268, 260, 252, 242, 237, 288, 238, 252, 276] payload_rows_per_dst=[5520, 5520, 5430, 5524, 5422, 5402, 5546, 5490] dup_rows_per_dst=[2716, 2848, 2646, 2802, 2562, 2596, 2836, 2676] combine_rows_per_src=[5520, 5520, 5430, 5524, 5422, 5402, 5546, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43638 payload_rows_remote=38080 dup_rows=21898 dup_rows_remote=19148 dedup_ratio=0.334137 router_tokens_per_expert=[235, 268, 270, 262, 255, 251, 258, 265, 272, 246, 230, 266, 257, 236, 247, 260, 285, 257, 257, 271, 260, 251, 281, 250, 253, 276, 256, 260, 264, 258, 280, 264, 250, 237, 255, 271, 272, 247, 249, 236, 255, 224, 272, 250, 250, 238, 273, 261, 262, 240, 251, 252, 268, 285, 234, 235, 256, 241, 247, 262, 254, 225, 238, 245, 229, 233, 253, 246, 245, 243, 255, 255, 278, 255, 248, 248, 250, 274, 267, 274, 268, 246, 246, 268, 256, 269, 261, 248, 254, 274, 236, 263, 266, 280, 269, 286, 274, 244, 269, 260, 259, 236, 238, 267, 255, 250, 244, 288, 251, 229, 217, 255, 241, 269, 263, 239, 257, 226, 259, 299, 261, 275, 260, 289, 248, 254, 280, 233] payload_rows_per_dst=[5464, 5548, 5392, 5346, 5386, 5512, 5432, 5558] dup_rows_per_dst=[2692, 2898, 2644, 2644, 2720, 2910, 2640, 2750] combine_rows_per_src=[5464, 5548, 5392, 5346, 5386, 5512, 5432, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38438 dup_rows=21706 dup_rows_remote=19042 dedup_ratio=0.331207 router_tokens_per_expert=[238, 217, 247, 232, 260, 246, 263, 256, 262, 236, 248, 267, 233, 282, 244, 276, 248, 279, 280, 280, 266, 239, 256, 293, 240, 240, 244, 246, 256, 249, 255, 261, 254, 277, 277, 226, 269, 264, 245, 272, 271, 258, 243, 269, 272, 232, 259, 266, 274, 279, 261, 244, 287, 263, 251, 267, 245, 246, 258, 237, 257, 286, 256, 252, 233, 275, 256, 238, 262, 250, 251, 257, 241, 246, 279, 252, 231, 243, 256, 258, 289, 271, 249, 243, 268, 268, 251, 282, 264, 247, 233, 247, 281, 265, 243, 255, 251, 264, 247, 265, 263, 237, 264, 246, 283, 271, 260, 259, 238, 235, 261, 257, 242, 260, 259, 240, 249, 272, 279, 250, 246, 256, 247, 246, 232, 239, 239, 271] payload_rows_per_dst=[5454, 5498, 5540, 5534, 5392, 5516, 5500, 5396] dup_rows_per_dst=[2560, 2766, 2768, 2792, 2664, 2796, 2702, 2658] combine_rows_per_src=[5454, 5498, 5540, 5534, 5392, 5516, 5500, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38262 dup_rows=21660 dup_rows_remote=18980 dedup_ratio=0.330505 router_tokens_per_expert=[260, 239, 242, 261, 263, 273, 250, 246, 273, 286, 256, 228, 259, 250, 250, 274, 252, 270, 277, 244, 252, 259, 255, 229, 238, 234, 266, 263, 265, 271, 293, 247, 261, 241, 260, 263, 258, 268, 240, 276, 267, 289, 253, 252, 240, 237, 244, 231, 232, 254, 274, 240, 270, 260, 258, 252, 251, 249, 278, 262, 253, 266, 275, 273, 263, 261, 237, 273, 271, 269, 252, 249, 238, 261, 244, 261, 259, 254, 254, 290, 243, 259, 270, 236, 246, 269, 243, 239, 256, 235, 261, 235, 224, 251, 260, 258, 276, 223, 284, 252, 284, 270, 256, 253, 256, 263, 259, 243, 265, 224, 243, 228, 263, 256, 259, 283, 264, 260, 230, 279, 229, 252, 243, 264, 257, 259, 230, 288] payload_rows_per_dst=[5456, 5484, 5454, 5614, 5490, 5394, 5528, 5456] dup_rows_per_dst=[2764, 2746, 2706, 2680, 2782, 2576, 2630, 2776] combine_rows_per_src=[5456, 5484, 5454, 5614, 5490, 5394, 5528, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=158130176 dispatch_meta_bytes=230208 combine_payload_bytes=158130176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158360384 ingress_bytes=177004544 link_bytes=177004544 token_bytes=316490560 weight_movement_bytes=18874368 total_nvlink_bytes=335364928 weight_to_token_ratio=0.059636 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.695970449e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157499392 dispatch_meta_bytes=230152 combine_payload_bytes=157499392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157729544 ingress_bytes=176373760 link_bytes=176373760 token_bytes=315228936 weight_movement_bytes=18874368 total_nvlink_bytes=334103304 weight_to_token_ratio=0.059875 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689926587e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=228912 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156204592 ingress_bytes=231473152 link_bytes=231473152 token_bytes=312180272 weight_movement_bytes=75497472 total_nvlink_bytes=387677744 weight_to_token_ratio=0.241839 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.217861851e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229920 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671968 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315114016 weight_movement_bytes=18874368 total_nvlink_bytes=333988384 weight_to_token_ratio=0.059897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=228968 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156950120 ingress_bytes=175595520 link_bytes=175595520 token_bytes=313671272 weight_movement_bytes=18874368 total_nvlink_bytes=332545640 weight_to_token_ratio=0.060172 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682469874e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38352 dup_rows=21686 dup_rows_remote=18918 dedup_ratio=0.330902 router_tokens_per_expert=[221, 277, 266, 251, 259, 250, 261, 246, 276, 281, 231, 274, 250, 246, 263, 263, 267, 244, 237, 228, 240, 232, 244, 244, 247, 273, 258, 265, 267, 220, 245, 254, 226, 272, 269, 237, 260, 237, 262, 285, 248, 279, 253, 275, 229, 259, 261, 243, 266, 298, 244, 281, 264, 272, 226, 264, 270, 259, 245, 273, 253, 247, 265, 242, 244, 261, 242, 242, 220, 226, 277, 256, 273, 271, 250, 237, 238, 248, 273, 242, 264, 248, 252, 272, 258, 263, 222, 253, 270, 246, 234, 235, 253, 260, 260, 264, 259, 237, 264, 272, 269, 231, 249, 259, 268, 272, 278, 257, 251, 246, 276, 245, 243, 281, 282, 287, 261, 271, 258, 265, 254, 261, 312, 240, 251, 265, 270, 236] payload_rows_per_dst=[5450, 5370, 5548, 5538, 5362, 5508, 5498, 5576] dup_rows_per_dst=[2780, 2560, 2642, 2800, 2638, 2600, 2768, 2898] combine_rows_per_src=[5450, 5370, 5548, 5538, 5362, 5508, 5498, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38232 dup_rows=21858 dup_rows_remote=19122 dedup_ratio=0.333527 router_tokens_per_expert=[270, 256, 260, 294, 277, 225, 280, 257, 248, 252, 253, 257, 253, 264, 254, 256, 265, 268, 271, 250, 247, 258, 242, 283, 269, 264, 228, 245, 259, 253, 272, 241, 251, 253, 259, 263, 244, 242, 265, 261, 249, 255, 253, 252, 279, 254, 241, 270, 259, 249, 249, 254, 284, 259, 270, 234, 258, 240, 252, 238, 260, 233, 239, 254, 254, 262, 225, 248, 266, 219, 261, 249, 260, 238, 273, 291, 259, 233, 253, 276, 256, 263, 244, 238, 252, 246, 236, 279, 271, 262, 273, 247, 255, 269, 280, 282, 259, 268, 284, 276, 264, 264, 254, 259, 260, 237, 243, 254, 263, 265, 237, 250, 241, 249, 247, 246, 239, 283, 249, 247, 235, 254, 253, 265, 249, 274, 231, 255] payload_rows_per_dst=[5430, 5518, 5446, 5398, 5396, 5580, 5504, 5406] dup_rows_per_dst=[2882, 2712, 2736, 2666, 2738, 2726, 2770, 2628] combine_rows_per_src=[5430, 5518, 5446, 5398, 5396, 5580, 5504, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38196 dup_rows=21788 dup_rows_remote=19040 dedup_ratio=0.332458 router_tokens_per_expert=[252, 273, 257, 244, 260, 261, 266, 268, 248, 252, 235, 269, 227, 299, 269, 251, 257, 249, 245, 237, 277, 260, 261, 259, 260, 247, 248, 252, 246, 243, 251, 261, 250, 233, 272, 230, 282, 240, 278, 268, 264, 233, 205, 289, 238, 251, 259, 249, 261, 267, 243, 268, 241, 273, 239, 254, 271, 225, 269, 261, 233, 230, 224, 255, 266, 285, 256, 274, 241, 240, 278, 259, 244, 259, 225, 260, 255, 266, 268, 280, 262, 266, 274, 222, 239, 255, 280, 272, 261, 236, 246, 241, 251, 241, 247, 249, 263, 300, 238, 270, 271, 262, 287, 261, 273, 250, 246, 250, 266, 233, 256, 292, 259, 276, 245, 270, 267, 234, 239, 232, 266, 256, 261, 262, 246, 272, 258, 270] payload_rows_per_dst=[5552, 5434, 5584, 5456, 5578, 5068, 5642, 5434] dup_rows_per_dst=[2748, 2672, 2872, 2752, 2946, 2212, 2880, 2706] combine_rows_per_src=[5552, 5434, 5584, 5456, 5578, 5068, 5642, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157089792 dispatch_meta_bytes=229080 combine_payload_bytes=157089792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157318872 ingress_bytes=194838528 link_bytes=194838528 token_bytes=314408664 weight_movement_bytes=37748736 total_nvlink_bytes=352157400 weight_to_token_ratio=0.120063 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866846909e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229416 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827688 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313425960 weight_movement_bytes=18874368 total_nvlink_bytes=332300328 weight_to_token_ratio=0.060220 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38298 dup_rows=21796 dup_rows_remote=19060 dedup_ratio=0.332581 router_tokens_per_expert=[229, 277, 268, 258, 246, 264, 278, 254, 254, 236, 240, 276, 246, 262, 237, 265, 247, 247, 252, 255, 280, 253, 237, 251, 281, 264, 265, 251, 221, 244, 284, 257, 248, 274, 254, 266, 248, 277, 267, 246, 243, 264, 232, 252, 246, 278, 266, 276, 244, 252, 260, 273, 259, 267, 235, 247, 243, 246, 245, 280, 267, 241, 233, 282, 269, 259, 268, 244, 262, 262, 255, 247, 281, 237, 238, 278, 249, 274, 275, 243, 243, 263, 254, 277, 256, 249, 250, 248, 275, 232, 254, 263, 249, 260, 251, 236, 251, 275, 262, 255, 259, 255, 261, 225, 248, 273, 269, 244, 278, 272, 259, 252, 250, 241, 271, 238, 254, 275, 264, 262, 269, 257, 252, 217, 228, 249, 254, 258] payload_rows_per_dst=[5426, 5442, 5542, 5522, 5514, 5430, 5508, 5356] dup_rows_per_dst=[2754, 2736, 2732, 2776, 2784, 2690, 2752, 2572] combine_rows_per_src=[5426, 5442, 5542, 5522, 5514, 5430, 5508, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38284 dup_rows=21758 dup_rows_remote=19024 dedup_ratio=0.332001 router_tokens_per_expert=[230, 276, 252, 233, 242, 233, 242, 248, 223, 247, 269, 268, 271, 265, 242, 235, 250, 275, 265, 270, 237, 250, 290, 258, 232, 282, 229, 246, 273, 237, 231, 258, 262, 244, 251, 257, 242, 250, 274, 261, 263, 293, 261, 264, 223, 239, 261, 270, 259, 257, 255, 242, 288, 252, 247, 280, 267, 249, 245, 271, 235, 230, 263, 263, 226, 255, 256, 276, 257, 254, 270, 269, 255, 233, 260, 250, 214, 233, 287, 227, 280, 265, 211, 278, 256, 254, 266, 272, 272, 230, 256, 268, 268, 248, 265, 281, 253, 267, 234, 279, 263, 241, 285, 272, 243, 263, 275, 283, 257, 263, 251, 256, 258, 266, 256, 233, 245, 262, 252, 247, 256, 236, 282, 264, 266, 251, 267, 273] payload_rows_per_dst=[5396, 5404, 5392, 5490, 5452, 5600, 5550, 5494] dup_rows_per_dst=[2556, 2762, 2664, 2716, 2592, 2914, 2820, 2734] combine_rows_per_src=[5396, 5404, 5392, 5490, 5452, 5600, 5550, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38214 dup_rows=21794 dup_rows_remote=19084 dedup_ratio=0.332550 router_tokens_per_expert=[261, 256, 274, 267, 258, 207, 261, 273, 260, 240, 239, 257, 257, 250, 242, 254, 244, 266, 227, 251, 272, 248, 261, 266, 256, 270, 248, 248, 255, 254, 243, 267, 242, 255, 292, 271, 272, 245, 260, 250, 256, 256, 263, 265, 256, 268, 250, 221, 259, 264, 245, 246, 248, 233, 247, 222, 245, 226, 235, 269, 255, 205, 257, 283, 235, 251, 264, 257, 254, 269, 293, 253, 271, 236, 252, 265, 264, 254, 252, 272, 269, 269, 259, 292, 246, 221, 261, 255, 278, 263, 252, 259, 245, 243, 257, 250, 239, 256, 276, 254, 243, 260, 247, 273, 273, 261, 275, 254, 277, 226, 250, 276, 256, 262, 249, 273, 270, 266, 258, 284, 265, 255, 250, 288, 252, 228, 255, 263] payload_rows_per_dst=[5474, 5466, 5440, 5342, 5478, 5528, 5472, 5542] dup_rows_per_dst=[2638, 2686, 2804, 2536, 2806, 2710, 2808, 2806] combine_rows_per_src=[5474, 5466, 5440, 5342, 5478, 5528, 5472, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228944 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679760 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130576 weight_movement_bytes=37748736 total_nvlink_bytes=350879312 weight_to_token_ratio=0.120553 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38174 dup_rows=21778 dup_rows_remote=18998 dedup_ratio=0.332306 router_tokens_per_expert=[280, 287, 242, 255, 261, 272, 251, 256, 257, 223, 220, 227, 279, 262, 228, 234, 255, 275, 245, 231, 248, 237, 242, 255, 255, 248, 274, 240, 265, 248, 263, 272, 249, 264, 272, 240, 216, 245, 259, 290, 278, 235, 249, 253, 267, 250, 264, 247, 254, 265, 268, 283, 227, 255, 257, 281, 277, 251, 242, 277, 258, 228, 258, 246, 265, 265, 233, 265, 278, 274, 280, 274, 253, 272, 236, 254, 275, 247, 255, 256, 249, 264, 263, 248, 262, 275, 231, 269, 289, 268, 271, 264, 258, 251, 249, 268, 243, 256, 271, 270, 245, 229, 252, 267, 249, 276, 260, 273, 234, 262, 270, 271, 239, 241, 272, 240, 235, 254, 236, 246, 268, 248, 230, 259, 257, 254, 267, 241] payload_rows_per_dst=[5412, 5404, 5520, 5520, 5584, 5484, 5464, 5370] dup_rows_per_dst=[2656, 2702, 2636, 2734, 2780, 2874, 2792, 2604] combine_rows_per_src=[5412, 5404, 5520, 5520, 5584, 5484, 5464, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38348 dup_rows=21778 dup_rows_remote=19212 dedup_ratio=0.332306 router_tokens_per_expert=[268, 250, 242, 263, 261, 271, 259, 260, 248, 240, 253, 254, 273, 247, 264, 245, 277, 261, 268, 241, 234, 253, 235, 262, 275, 256, 277, 259, 268, 255, 262, 238, 234, 244, 284, 269, 234, 246, 226, 269, 271, 274, 255, 264, 235, 242, 243, 271, 278, 265, 243, 251, 224, 263, 275, 245, 254, 247, 235, 231, 248, 229, 255, 245, 255, 261, 250, 234, 247, 222, 271, 252, 276, 242, 245, 241, 258, 254, 253, 254, 241, 245, 249, 240, 242, 274, 268, 252, 252, 261, 256, 240, 255, 266, 233, 269, 255, 291, 250, 244, 283, 271, 256, 281, 233, 240, 275, 282, 239, 294, 283, 273, 262, 248, 265, 265, 258, 270, 263, 260, 233, 259, 291, 271, 263, 252, 286, 246] payload_rows_per_dst=[5440, 5512, 5444, 5410, 5424, 5406, 5590, 5532] dup_rows_per_dst=[2756, 2730, 2678, 2566, 2606, 2680, 2910, 2852] combine_rows_per_src=[5440, 5512, 5444, 5410, 5424, 5406, 5590, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229432 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098040 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966648 weight_movement_bytes=18874368 total_nvlink_bytes=332841016 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229232 combine_payload_bytes=156811264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157040496 ingress_bytes=232308736 link_bytes=232308736 token_bytes=313851760 weight_movement_bytes=75497472 total_nvlink_bytes=389349232 weight_to_token_ratio=0.240551 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.225868005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229192 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156753736 ingress_bytes=194273280 link_bytes=194273280 token_bytes=313278280 weight_movement_bytes=37748736 total_nvlink_bytes=351027016 weight_to_token_ratio=0.120496 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861430981e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38422 dup_rows=21630 dup_rows_remote=19078 dedup_ratio=0.330048 router_tokens_per_expert=[286, 264, 234, 258, 245, 260, 270, 267, 267, 252, 253, 233, 271, 256, 271, 266, 273, 278, 251, 246, 260, 251, 254, 258, 259, 257, 268, 219, 275, 274, 274, 256, 254, 236, 231, 236, 253, 229, 249, 255, 246, 279, 257, 261, 244, 281, 272, 269, 246, 266, 245, 264, 228, 267, 262, 238, 252, 251, 244, 246, 261, 249, 268, 237, 273, 235, 256, 232, 245, 226, 280, 253, 238, 259, 264, 261, 257, 256, 254, 253, 277, 249, 273, 248, 250, 248, 269, 282, 246, 274, 273, 272, 243, 237, 274, 275, 260, 254, 257, 259, 248, 244, 259, 242, 260, 244, 237, 267, 249, 240, 226, 272, 252, 253, 248, 231, 254, 257, 251, 256, 277, 262, 267, 277, 243, 267, 282, 259] payload_rows_per_dst=[5492, 5508, 5450, 5462, 5414, 5538, 5484, 5558] dup_rows_per_dst=[2814, 2798, 2654, 2586, 2670, 2842, 2552, 2714] combine_rows_per_src=[5492, 5508, 5450, 5462, 5414, 5538, 5484, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=228688 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156589392 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312950096 weight_movement_bytes=18874368 total_nvlink_bytes=331824464 weight_to_token_ratio=0.060311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=230240 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303648 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314377056 weight_movement_bytes=18874368 total_nvlink_bytes=333251424 weight_to_token_ratio=0.060037 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38214 dup_rows=21900 dup_rows_remote=19114 dedup_ratio=0.334167 router_tokens_per_expert=[271, 256, 249, 274, 235, 253, 241, 262, 274, 226, 273, 263, 271, 251, 239, 275, 288, 253, 258, 261, 265, 250, 255, 251, 278, 249, 298, 285, 261, 256, 261, 252, 261, 242, 264, 240, 232, 262, 259, 261, 261, 254, 251, 247, 265, 249, 290, 266, 235, 260, 241, 251, 273, 288, 260, 246, 251, 290, 271, 263, 246, 236, 250, 264, 236, 239, 293, 237, 246, 259, 237, 250, 270, 261, 259, 248, 233, 268, 251, 261, 229, 274, 235, 253, 240, 234, 238, 253, 238, 278, 258, 231, 253, 256, 253, 227, 241, 252, 290, 231, 249, 245, 272, 254, 263, 255, 236, 229, 254, 250, 233, 259, 254, 257, 265, 263, 265, 275, 277, 240, 271, 264, 274, 257, 243, 243, 269, 277] payload_rows_per_dst=[5490, 5512, 5422, 5474, 5486, 5338, 5392, 5522] dup_rows_per_dst=[2736, 2930, 2786, 2776, 2610, 2562, 2634, 2866] combine_rows_per_src=[5490, 5512, 5422, 5474, 5486, 5338, 5392, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=230000 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157606512 ingress_bytes=195125248 link_bytes=195125248 token_bytes=314983024 weight_movement_bytes=37748736 total_nvlink_bytes=352731760 weight_to_token_ratio=0.119844 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869594119e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229312 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156753856 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278400 weight_movement_bytes=18874368 total_nvlink_bytes=332152768 weight_to_token_ratio=0.060248 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38118 dup_rows=21846 dup_rows_remote=19102 dedup_ratio=0.333344 router_tokens_per_expert=[259, 251, 275, 272, 263, 246, 267, 266, 238, 248, 274, 250, 281, 259, 256, 252, 269, 245, 261, 266, 276, 272, 258, 260, 248, 249, 248, 264, 262, 261, 272, 240, 265, 266, 240, 285, 231, 263, 255, 267, 260, 273, 258, 230, 270, 259, 269, 256, 251, 280, 272, 235, 233, 247, 246, 244, 224, 250, 258, 274, 273, 269, 287, 271, 234, 254, 221, 244, 266, 282, 261, 238, 247, 237, 265, 239, 286, 263, 239, 228, 265, 239, 224, 244, 256, 243, 225, 275, 246, 259, 259, 265, 257, 222, 263, 249, 292, 250, 247, 233, 268, 275, 244, 271, 233, 260, 277, 246, 257, 258, 236, 251, 283, 256, 257, 253, 253, 246, 255, 254, 268, 229, 281, 245, 274, 240, 266, 246] payload_rows_per_dst=[5572, 5528, 5528, 5458, 5386, 5352, 5400, 5466] dup_rows_per_dst=[2744, 2868, 2912, 2632, 2622, 2630, 2692, 2746] combine_rows_per_src=[5572, 5528, 5528, 5458, 5386, 5352, 5400, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38154 dup_rows=21950 dup_rows_remote=19226 dedup_ratio=0.334930 router_tokens_per_expert=[260, 269, 267, 239, 240, 240, 236, 265, 234, 257, 232, 263, 258, 254, 278, 241, 259, 257, 253, 245, 257, 265, 248, 242, 240, 261, 269, 286, 265, 264, 227, 240, 249, 253, 272, 248, 270, 279, 254, 265, 272, 263, 276, 241, 261, 265, 240, 277, 258, 254, 235, 287, 278, 254, 263, 258, 262, 224, 254, 259, 264, 252, 262, 256, 253, 256, 248, 244, 249, 272, 265, 252, 247, 257, 243, 295, 261, 255, 255, 271, 250, 269, 245, 263, 254, 245, 261, 264, 234, 265, 249, 232, 281, 290, 245, 239, 286, 264, 261, 253, 277, 261, 218, 265, 243, 270, 248, 284, 257, 263, 233, 274, 233, 265, 237, 226, 246, 246, 255, 258, 238, 237, 262, 263, 280, 243, 248, 249] payload_rows_per_dst=[5378, 5432, 5522, 5464, 5468, 5448, 5480, 5394] dup_rows_per_dst=[2688, 2724, 2848, 2776, 2778, 2724, 2834, 2578] combine_rows_per_src=[5378, 5432, 5522, 5464, 5468, 5448, 5480, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=37918 dup_rows=21950 dup_rows_remote=18962 dedup_ratio=0.334930 router_tokens_per_expert=[264, 278, 252, 261, 249, 233, 280, 266, 265, 235, 254, 252, 244, 274, 256, 265, 226, 270, 258, 274, 273, 275, 266, 252, 299, 236, 268, 243, 284, 231, 235, 242, 239, 244, 246, 273, 277, 255, 246, 281, 257, 241, 240, 233, 270, 248, 244, 272, 231, 243, 267, 245, 255, 247, 275, 239, 268, 256, 263, 280, 245, 259, 259, 233, 268, 276, 266, 279, 259, 259, 244, 239, 255, 263, 258, 250, 250, 243, 239, 267, 279, 254, 257, 219, 243, 244, 255, 259, 247, 282, 279, 276, 257, 259, 263, 262, 275, 269, 227, 236, 246, 227, 251, 273, 258, 259, 261, 246, 270, 242, 254, 257, 248, 278, 246, 246, 273, 264, 264, 240, 269, 252, 233, 255, 266, 244, 246, 252] payload_rows_per_dst=[5232, 5444, 5410, 5398, 5478, 5490, 5466, 5668] dup_rows_per_dst=[2460, 2662, 2722, 2732, 2812, 2938, 2636, 2988] combine_rows_per_src=[5232, 5444, 5410, 5398, 5478, 5490, 5466, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38346 dup_rows=21788 dup_rows_remote=19236 dedup_ratio=0.332458 router_tokens_per_expert=[239, 252, 264, 261, 228, 249, 241, 276, 241, 240, 222, 293, 256, 249, 267, 280, 254, 233, 277, 240, 272, 256, 234, 263, 247, 241, 240, 265, 259, 246, 244, 294, 245, 271, 293, 240, 249, 269, 259, 248, 255, 263, 246, 272, 268, 234, 259, 250, 231, 268, 232, 284, 296, 245, 253, 239, 254, 271, 275, 265, 252, 239, 245, 272, 264, 238, 275, 233, 237, 265, 275, 247, 267, 299, 250, 265, 250, 277, 256, 246, 252, 239, 269, 253, 254, 257, 244, 245, 257, 249, 259, 215, 250, 253, 262, 219, 256, 250, 259, 262, 271, 295, 251, 256, 265, 246, 278, 239, 230, 228, 243, 274, 254, 251, 247, 256, 274, 263, 280, 260, 258, 262, 260, 253, 249, 275, 289, 248] payload_rows_per_dst=[5442, 5450, 5488, 5478, 5512, 5402, 5460, 5516] dup_rows_per_dst=[2674, 2680, 2754, 2764, 2776, 2552, 2746, 2842] combine_rows_per_src=[5442, 5450, 5488, 5478, 5512, 5402, 5460, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38248 dup_rows=21836 dup_rows_remote=19154 dedup_ratio=0.333191 router_tokens_per_expert=[237, 285, 223, 272, 266, 256, 231, 244, 244, 262, 269, 266, 262, 258, 249, 286, 245, 266, 264, 241, 276, 264, 269, 243, 240, 287, 294, 272, 254, 248, 268, 254, 272, 277, 246, 256, 228, 264, 229, 259, 254, 258, 285, 252, 271, 285, 236, 244, 249, 266, 244, 252, 273, 264, 233, 252, 242, 220, 234, 245, 263, 287, 238, 230, 229, 272, 284, 226, 253, 263, 261, 264, 254, 243, 221, 237, 250, 279, 261, 270, 243, 258, 254, 252, 267, 231, 240, 236, 267, 263, 236, 250, 234, 260, 235, 255, 240, 270, 280, 245, 249, 234, 262, 259, 260, 278, 265, 276, 242, 251, 257, 247, 258, 241, 270, 246, 276, 273, 294, 267, 234, 267, 279, 245, 264, 261, 261, 266] payload_rows_per_dst=[5534, 5470, 5478, 5362, 5452, 5354, 5524, 5526] dup_rows_per_dst=[2686, 2900, 2754, 2622, 2682, 2608, 2706, 2878] combine_rows_per_src=[5534, 5470, 5478, 5362, 5452, 5354, 5524, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=228880 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156360208 ingress_bytes=212754432 link_bytes=212754432 token_bytes=312491536 weight_movement_bytes=56623104 total_nvlink_bytes=369114640 weight_to_token_ratio=0.181199 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038508286e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43974 payload_rows_remote=38512 dup_rows=21562 dup_rows_remote=18940 dedup_ratio=0.329010 router_tokens_per_expert=[236, 242, 279, 264, 251, 265, 276, 245, 276, 257, 262, 258, 261, 271, 267, 283, 263, 256, 257, 237, 233, 248, 263, 246, 274, 253, 251, 274, 238, 262, 253, 284, 236, 249, 252, 241, 271, 254, 264, 278, 264, 248, 285, 251, 297, 256, 253, 267, 258, 234, 239, 252, 273, 251, 262, 265, 244, 249, 231, 277, 266, 225, 258, 258, 259, 234, 248, 241, 234, 283, 263, 267, 275, 237, 249, 256, 271, 249, 260, 232, 252, 238, 255, 253, 258, 238, 261, 267, 243, 248, 261, 253, 251, 255, 253, 257, 277, 235, 264, 250, 251, 286, 277, 266, 262, 229, 274, 246, 268, 254, 274, 267, 274, 247, 234, 251, 253, 243, 239, 254, 266, 259, 267, 247, 238, 224, 225, 273] payload_rows_per_dst=[5584, 5488, 5544, 5462, 5448, 5438, 5616, 5394] dup_rows_per_dst=[2802, 2696, 2788, 2622, 2668, 2648, 2744, 2594] combine_rows_per_src=[5584, 5488, 5544, 5462, 5448, 5438, 5616, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229520 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156508304 ingress_bytes=194027520 link_bytes=194027520 token_bytes=312787088 weight_movement_bytes=37748736 total_nvlink_bytes=350535824 weight_to_token_ratio=0.120685 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859076229e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43836 payload_rows_remote=38452 dup_rows=21700 dup_rows_remote=19026 dedup_ratio=0.331116 router_tokens_per_expert=[263, 263, 279, 260, 254, 241, 248, 274, 262, 260, 253, 267, 247, 243, 258, 255, 236, 243, 260, 287, 269, 274, 242, 285, 247, 270, 269, 257, 239, 255, 250, 244, 233, 235, 269, 241, 253, 238, 267, 257, 265, 254, 235, 263, 272, 279, 232, 264, 259, 261, 240, 246, 243, 264, 266, 290, 284, 245, 258, 280, 281, 219, 252, 247, 240, 228, 259, 288, 218, 252, 249, 275, 259, 254, 267, 246, 271, 256, 243, 274, 239, 282, 250, 260, 265, 281, 255, 267, 253, 271, 250, 254, 254, 244, 264, 243, 246, 284, 243, 242, 240, 249, 247, 263, 257, 241, 244, 244, 248, 259, 254, 268, 256, 255, 273, 275, 245, 284, 248, 246, 262, 257, 245, 261, 225, 265, 246, 239] payload_rows_per_dst=[5320, 5520, 5464, 5570, 5490, 5462, 5384, 5626] dup_rows_per_dst=[2648, 2734, 2650, 2700, 2668, 2802, 2674, 2824] combine_rows_per_src=[5320, 5520, 5464, 5570, 5490, 5462, 5384, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=155312128 dispatch_meta_bytes=227520 combine_payload_bytes=155312128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155539648 ingress_bytes=174186496 link_bytes=174186496 token_bytes=310851776 weight_movement_bytes=18874368 total_nvlink_bytes=329726144 weight_to_token_ratio=0.060718 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.668969300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=230328 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157295544 ingress_bytes=175939584 link_bytes=175939584 token_bytes=314360760 weight_movement_bytes=18874368 total_nvlink_bytes=333235128 weight_to_token_ratio=0.060040 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685766526e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229608 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156893416 ingress_bytes=175538176 link_bytes=175538176 token_bytes=313557224 weight_movement_bytes=18874368 total_nvlink_bytes=332431592 weight_to_token_ratio=0.060194 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681920432e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38164 dup_rows=21806 dup_rows_remote=18986 dedup_ratio=0.332733 router_tokens_per_expert=[292, 265, 263, 262, 285, 254, 252, 246, 266, 261, 281, 259, 239, 235, 264, 249, 252, 247, 261, 274, 236, 238, 283, 261, 268, 262, 242, 250, 250, 244, 244, 270, 261, 243, 288, 257, 251, 264, 267, 246, 246, 282, 272, 259, 281, 260, 243, 273, 227, 230, 274, 232, 259, 256, 248, 272, 250, 237, 283, 235, 275, 232, 290, 271, 236, 268, 260, 258, 249, 255, 270, 224, 236, 277, 264, 259, 258, 225, 252, 219, 243, 278, 252, 242, 258, 257, 277, 266, 231, 245, 261, 245, 257, 263, 221, 269, 257, 266, 257, 224, 239, 243, 272, 256, 268, 254, 248, 245, 256, 262, 261, 267, 275, 245, 281, 248, 231, 290, 236, 230, 244, 247, 277, 244, 258, 254, 276, 263] payload_rows_per_dst=[5468, 5532, 5566, 5432, 5380, 5420, 5420, 5512] dup_rows_per_dst=[2878, 2632, 2820, 2710, 2640, 2710, 2730, 2686] combine_rows_per_src=[5468, 5532, 5566, 5432, 5380, 5420, 5420, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157745152 dispatch_meta_bytes=229808 combine_payload_bytes=157745152 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157974960 ingress_bytes=176619520 link_bytes=176619520 token_bytes=315720112 weight_movement_bytes=18874368 total_nvlink_bytes=334594480 weight_to_token_ratio=0.059782 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692281338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157499392 dispatch_meta_bytes=229912 combine_payload_bytes=157499392 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157729304 ingress_bytes=214122496 link_bytes=214122496 token_bytes=315228696 weight_movement_bytes=56623104 total_nvlink_bytes=371851800 weight_to_token_ratio=0.179625 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051616402e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228600 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548344 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868088 weight_movement_bytes=18874368 total_nvlink_bytes=331742456 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43616 payload_rows_remote=37974 dup_rows=21920 dup_rows_remote=18828 dedup_ratio=0.334473 router_tokens_per_expert=[270, 249, 270, 225, 236, 240, 273, 242, 241, 278, 286, 263, 265, 231, 303, 246, 276, 245, 243, 237, 252, 281, 248, 265, 256, 255, 260, 230, 261, 262, 272, 244, 248, 234, 244, 287, 277, 249, 283, 268, 250, 251, 254, 245, 248, 249, 265, 258, 239, 262, 248, 272, 270, 261, 253, 261, 253, 222, 259, 256, 249, 238, 249, 241, 275, 248, 259, 263, 260, 274, 260, 237, 231, 257, 239, 259, 241, 269, 279, 254, 245, 241, 257, 246, 269, 230, 233, 270, 243, 293, 242, 264, 255, 240, 291, 249, 276, 266, 248, 243, 250, 279, 280, 234, 256, 262, 260, 249, 263, 272, 249, 279, 269, 239, 245, 245, 249, 272, 241, 268, 234, 242, 263, 266, 281, 255, 261, 251] payload_rows_per_dst=[5642, 5422, 5290, 5418, 5576, 5538, 5360, 5370] dup_rows_per_dst=[3092, 2752, 2432, 2698, 2848, 2972, 2598, 2528] combine_rows_per_src=[5642, 5422, 5290, 5418, 5576, 5538, 5360, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38278 dup_rows=21768 dup_rows_remote=19182 dedup_ratio=0.332153 router_tokens_per_expert=[245, 234, 280, 262, 278, 263, 279, 230, 270, 238, 246, 273, 266, 269, 248, 245, 249, 248, 239, 221, 255, 275, 253, 267, 244, 241, 257, 242, 266, 263, 244, 274, 242, 235, 245, 250, 247, 256, 261, 271, 241, 263, 249, 245, 247, 265, 253, 263, 241, 251, 266, 270, 258, 243, 279, 260, 252, 232, 274, 247, 258, 249, 262, 243, 251, 262, 275, 243, 228, 218, 259, 273, 257, 263, 281, 242, 244, 259, 314, 239, 265, 238, 281, 273, 259, 257, 248, 269, 272, 273, 240, 283, 256, 270, 259, 238, 274, 256, 233, 266, 267, 259, 269, 256, 254, 278, 246, 259, 256, 249, 250, 253, 268, 237, 259, 257, 284, 255, 227, 225, 240, 279, 251, 286, 249, 234, 257, 264] payload_rows_per_dst=[5542, 5490, 5394, 5442, 5444, 5518, 5444, 5494] dup_rows_per_dst=[2710, 2586, 2672, 2728, 2772, 2844, 2806, 2650] combine_rows_per_src=[5542, 5490, 5394, 5442, 5444, 5518, 5444, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38268 dup_rows=21720 dup_rows_remote=18936 dedup_ratio=0.331421 router_tokens_per_expert=[252, 239, 262, 265, 259, 251, 282, 266, 259, 242, 276, 267, 249, 276, 290, 282, 241, 250, 272, 255, 246, 276, 265, 227, 259, 254, 250, 236, 270, 262, 241, 249, 246, 270, 250, 241, 285, 258, 251, 252, 239, 255, 242, 271, 261, 263, 264, 262, 255, 238, 259, 261, 259, 256, 244, 255, 242, 255, 243, 284, 276, 247, 255, 262, 250, 243, 242, 233, 248, 246, 255, 277, 231, 266, 247, 261, 272, 274, 257, 276, 248, 293, 249, 236, 262, 266, 230, 218, 234, 261, 238, 235, 279, 249, 241, 254, 253, 245, 255, 277, 250, 248, 250, 252, 268, 273, 228, 270, 253, 236, 253, 256, 241, 257, 250, 269, 259, 258, 249, 266, 258, 252, 268, 277, 255, 270, 256, 274] payload_rows_per_dst=[5524, 5418, 5514, 5500, 5482, 5370, 5460, 5548] dup_rows_per_dst=[2910, 2566, 2706, 2804, 2674, 2616, 2660, 2784] combine_rows_per_src=[5524, 5418, 5514, 5500, 5482, 5370, 5460, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38476 dup_rows=21618 dup_rows_remote=18912 dedup_ratio=0.329865 router_tokens_per_expert=[256, 265, 254, 260, 264, 265, 249, 252, 265, 223, 272, 242, 252, 263, 280, 256, 251, 258, 256, 290, 247, 246, 245, 275, 249, 243, 263, 235, 262, 261, 243, 250, 251, 259, 258, 250, 248, 257, 224, 241, 261, 228, 246, 239, 226, 253, 276, 270, 255, 279, 276, 249, 258, 232, 264, 287, 272, 240, 251, 280, 246, 268, 264, 274, 263, 241, 275, 268, 284, 269, 250, 245, 250, 239, 247, 265, 254, 268, 272, 239, 236, 276, 254, 271, 250, 241, 244, 292, 260, 275, 233, 244, 258, 229, 258, 253, 273, 257, 239, 234, 247, 267, 268, 260, 261, 283, 248, 264, 245, 278, 242, 292, 241, 243, 258, 253, 263, 250, 268, 242, 251, 247, 260, 247, 239, 267, 244, 260] payload_rows_per_dst=[5518, 5514, 5434, 5594, 5522, 5442, 5532, 5362] dup_rows_per_dst=[2718, 2634, 2540, 2796, 2736, 2706, 2784, 2704] combine_rows_per_src=[5518, 5514, 5434, 5594, 5522, 5442, 5532, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155541504 dispatch_meta_bytes=227208 combine_payload_bytes=155541504 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155768712 ingress_bytes=174415872 link_bytes=174415872 token_bytes=311310216 weight_movement_bytes=18874368 total_nvlink_bytes=330184584 weight_to_token_ratio=0.060629 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671167068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38470 dup_rows=21598 dup_rows_remote=18896 dedup_ratio=0.329559 router_tokens_per_expert=[252, 263, 284, 265, 243, 256, 243, 249, 273, 262, 271, 285, 286, 284, 270, 267, 246, 238, 274, 250, 254, 249, 242, 254, 263, 254, 274, 256, 256, 232, 253, 276, 221, 252, 239, 252, 229, 264, 239, 282, 231, 264, 240, 266, 256, 263, 254, 233, 258, 251, 244, 268, 261, 259, 245, 259, 263, 226, 237, 276, 253, 264, 251, 265, 263, 242, 264, 262, 266, 249, 263, 261, 224, 286, 264, 242, 246, 270, 245, 238, 275, 245, 287, 274, 262, 252, 275, 255, 246, 253, 266, 228, 258, 269, 259, 259, 268, 259, 237, 273, 274, 243, 257, 231, 266, 261, 256, 231, 236, 223, 266, 207, 278, 261, 253, 262, 264, 270, 260, 247, 245, 254, 255, 250, 231, 285, 274, 254] payload_rows_per_dst=[5692, 5448, 5378, 5546, 5468, 5524, 5392, 5490] dup_rows_per_dst=[2814, 2694, 2592, 2614, 2702, 2802, 2584, 2796] combine_rows_per_src=[5692, 5448, 5378, 5546, 5468, 5524, 5392, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38084 dup_rows=21964 dup_rows_remote=19242 dedup_ratio=0.335144 router_tokens_per_expert=[238, 272, 262, 247, 252, 245, 239, 237, 249, 242, 242, 235, 259, 248, 241, 252, 269, 227, 253, 254, 272, 256, 255, 240, 251, 262, 271, 227, 249, 285, 252, 252, 256, 229, 246, 247, 258, 251, 276, 261, 240, 237, 243, 280, 273, 248, 247, 238, 266, 281, 233, 262, 271, 245, 288, 228, 252, 262, 261, 260, 242, 253, 246, 255, 268, 281, 240, 275, 285, 270, 242, 244, 258, 260, 240, 270, 268, 288, 247, 268, 255, 268, 255, 243, 245, 242, 275, 271, 251, 245, 256, 256, 248, 216, 236, 278, 275, 270, 254, 258, 275, 271, 262, 293, 266, 276, 271, 257, 276, 256, 230, 253, 293, 269, 254, 238, 270, 228, 260, 257, 261, 271, 260, 258, 213, 249, 241, 289] payload_rows_per_dst=[5336, 5494, 5416, 5488, 5406, 5402, 5552, 5478] dup_rows_per_dst=[2584, 2656, 2644, 2722, 3002, 2678, 2934, 2744] combine_rows_per_src=[5336, 5494, 5416, 5488, 5406, 5402, 5552, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229840 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016528 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803216 weight_movement_bytes=37748736 total_nvlink_bytes=351551952 weight_to_token_ratio=0.120294 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38468 dup_rows=21628 dup_rows_remote=18860 dedup_ratio=0.330017 router_tokens_per_expert=[257, 247, 227, 251, 266, 269, 234, 254, 244, 275, 254, 260, 266, 260, 241, 249, 285, 239, 242, 267, 271, 256, 291, 299, 269, 246, 270, 259, 255, 254, 274, 241, 246, 262, 252, 243, 225, 260, 278, 267, 238, 243, 243, 238, 280, 274, 231, 277, 214, 234, 254, 265, 254, 243, 270, 258, 241, 267, 240, 252, 270, 251, 248, 246, 268, 237, 218, 249, 285, 276, 253, 266, 251, 288, 271, 249, 245, 264, 258, 267, 239, 237, 269, 257, 256, 241, 252, 242, 229, 268, 265, 263, 269, 279, 237, 254, 238, 266, 264, 277, 260, 254, 254, 266, 239, 251, 292, 258, 232, 255, 233, 265, 255, 252, 266, 261, 258, 261, 257, 269, 250, 279, 266, 228, 235, 258, 275, 256] payload_rows_per_dst=[5474, 5660, 5422, 5460, 5528, 5462, 5440, 5462] dup_rows_per_dst=[2634, 2776, 2692, 2554, 2762, 2652, 2768, 2790] combine_rows_per_src=[5474, 5660, 5422, 5460, 5528, 5462, 5440, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38324 dup_rows=21784 dup_rows_remote=19168 dedup_ratio=0.332397 router_tokens_per_expert=[254, 263, 279, 241, 260, 257, 243, 238, 275, 252, 240, 284, 245, 247, 260, 261, 254, 288, 255, 273, 252, 270, 277, 236, 274, 246, 251, 263, 242, 243, 245, 265, 250, 236, 255, 246, 270, 248, 233, 259, 238, 245, 264, 261, 246, 259, 243, 269, 290, 230, 251, 236, 276, 224, 247, 268, 254, 242, 252, 283, 265, 256, 251, 269, 254, 261, 256, 244, 267, 281, 272, 262, 268, 262, 267, 262, 260, 252, 240, 247, 254, 241, 246, 246, 251, 254, 242, 266, 258, 262, 259, 280, 252, 264, 243, 267, 237, 243, 238, 247, 273, 251, 272, 245, 255, 259, 242, 252, 262, 259, 256, 253, 278, 249, 259, 261, 282, 268, 246, 250, 249, 260, 248, 237, 264, 269, 283, 232] payload_rows_per_dst=[5458, 5546, 5428, 5504, 5522, 5498, 5380, 5416] dup_rows_per_dst=[2740, 2722, 2616, 2684, 2788, 2672, 2708, 2854] combine_rows_per_src=[5458, 5546, 5428, 5504, 5522, 5498, 5380, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228816 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156974544 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720272 weight_movement_bytes=37748736 total_nvlink_bytes=351469008 weight_to_token_ratio=0.120326 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=229552 combine_payload_bytes=157597696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157827248 ingress_bytes=195346432 link_bytes=195346432 token_bytes=315424944 weight_movement_bytes=37748736 total_nvlink_bytes=353173680 weight_to_token_ratio=0.119676 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871713395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157573120 dispatch_meta_bytes=229464 combine_payload_bytes=157573120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157802584 ingress_bytes=176447488 link_bytes=176447488 token_bytes=315375704 weight_movement_bytes=18874368 total_nvlink_bytes=334250072 weight_to_token_ratio=0.059847 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690633012e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=155992064 dispatch_meta_bytes=229304 combine_payload_bytes=155992064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156221368 ingress_bytes=174866432 link_bytes=174866432 token_bytes=312213432 weight_movement_bytes=18874368 total_nvlink_bytes=331087800 weight_to_token_ratio=0.060453 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675484112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157564928 dispatch_meta_bytes=229312 combine_payload_bytes=157564928 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157794240 ingress_bytes=233062400 link_bytes=233062400 token_bytes=315359168 weight_movement_bytes=75497472 total_nvlink_bytes=390856640 weight_to_token_ratio=0.239402 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.233089243e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229968 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157205072 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314180176 weight_movement_bytes=18874368 total_nvlink_bytes=333054544 weight_to_token_ratio=0.060075 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38150 dup_rows=21808 dup_rows_remote=19004 dedup_ratio=0.332764 router_tokens_per_expert=[273, 264, 263, 247, 266, 250, 251, 261, 262, 254, 261, 252, 244, 245, 251, 244, 272, 249, 289, 243, 260, 240, 238, 250, 250, 272, 232, 269, 260, 248, 260, 251, 264, 256, 265, 239, 262, 253, 249, 227, 269, 278, 229, 277, 276, 280, 237, 299, 247, 246, 241, 269, 241, 234, 270, 264, 241, 262, 251, 240, 239, 264, 266, 236, 262, 293, 261, 251, 228, 252, 245, 230, 236, 230, 276, 266, 233, 281, 259, 257, 262, 272, 228, 263, 263, 267, 241, 251, 287, 238, 273, 277, 252, 250, 239, 297, 241, 237, 256, 252, 255, 258, 255, 290, 261, 241, 287, 247, 262, 259, 252, 250, 226, 269, 232, 258, 269, 257, 262, 289, 282, 232, 247, 300, 267, 249, 231, 233] payload_rows_per_dst=[5578, 5554, 5512, 5504, 5508, 5382, 5484, 5206] dup_rows_per_dst=[2804, 2952, 2808, 2842, 2742, 2614, 2652, 2394] combine_rows_per_src=[5578, 5554, 5512, 5504, 5508, 5382, 5484, 5206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38244 dup_rows=21808 dup_rows_remote=19034 dedup_ratio=0.332764 router_tokens_per_expert=[237, 254, 244, 233, 256, 272, 245, 248, 252, 251, 289, 254, 263, 246, 240, 261, 255, 249, 280, 251, 271, 259, 277, 262, 253, 246, 235, 248, 271, 249, 243, 280, 258, 276, 251, 252, 256, 249, 231, 257, 244, 271, 255, 225, 269, 275, 287, 254, 243, 246, 266, 246, 232, 232, 245, 266, 260, 267, 262, 254, 279, 264, 238, 249, 273, 258, 249, 235, 258, 267, 276, 263, 271, 249, 245, 246, 240, 277, 249, 267, 253, 230, 254, 241, 243, 262, 235, 261, 253, 238, 263, 240, 261, 229, 268, 232, 260, 248, 273, 241, 275, 269, 263, 245, 258, 240, 278, 260, 269, 273, 270, 260, 259, 237, 252, 252, 246, 266, 269, 264, 238, 257, 278, 246, 275, 274, 287, 267] payload_rows_per_dst=[5404, 5484, 5514, 5424, 5490, 5350, 5500, 5562] dup_rows_per_dst=[2686, 2774, 2706, 2674, 2756, 2576, 2864, 2772] combine_rows_per_src=[5404, 5484, 5514, 5424, 5490, 5350, 5500, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38198 dup_rows=21822 dup_rows_remote=18992 dedup_ratio=0.332977 router_tokens_per_expert=[260, 233, 243, 239, 246, 252, 273, 271, 234, 253, 256, 242, 258, 247, 253, 280, 277, 269, 263, 261, 269, 270, 259, 267, 259, 277, 286, 247, 270, 245, 249, 246, 266, 239, 266, 245, 271, 274, 266, 249, 249, 279, 226, 240, 286, 265, 261, 239, 233, 263, 229, 255, 242, 254, 241, 243, 249, 260, 244, 243, 264, 244, 250, 234, 264, 259, 258, 234, 246, 268, 267, 258, 240, 255, 252, 254, 263, 252, 247, 254, 262, 254, 255, 271, 249, 274, 256, 221, 266, 232, 245, 256, 267, 263, 265, 234, 273, 265, 271, 245, 266, 281, 250, 247, 280, 238, 269, 237, 262, 249, 263, 239, 263, 289, 281, 267, 232, 258, 258, 249, 285, 263, 267, 240, 245, 270, 241, 261] payload_rows_per_dst=[5450, 5640, 5534, 5282, 5400, 5448, 5444, 5516] dup_rows_per_dst=[2630, 2780, 2708, 2614, 2742, 2692, 2826, 2830] combine_rows_per_src=[5450, 5640, 5534, 5282, 5400, 5448, 5444, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38138 dup_rows=21924 dup_rows_remote=19042 dedup_ratio=0.334534 router_tokens_per_expert=[267, 237, 240, 263, 244, 263, 259, 263, 265, 241, 274, 272, 236, 269, 259, 262, 265, 236, 236, 241, 272, 271, 243, 268, 298, 255, 271, 258, 238, 283, 247, 268, 247, 294, 262, 252, 240, 249, 237, 232, 236, 285, 265, 238, 269, 257, 254, 253, 271, 241, 266, 238, 252, 253, 255, 218, 232, 275, 256, 250, 228, 263, 233, 265, 232, 239, 276, 257, 264, 275, 247, 239, 254, 265, 257, 239, 243, 241, 260, 243, 257, 255, 272, 243, 262, 272, 258, 277, 274, 243, 240, 266, 256, 251, 269, 283, 275, 260, 263, 230, 252, 287, 294, 276, 253, 259, 244, 239, 265, 232, 259, 250, 279, 267, 264, 236, 265, 271, 255, 275, 251, 229, 266, 243, 274, 242, 260, 214] payload_rows_per_dst=[5446, 5488, 5490, 5412, 5344, 5474, 5478, 5480] dup_rows_per_dst=[2782, 2812, 2650, 2580, 2718, 2882, 2798, 2702] combine_rows_per_src=[5446, 5488, 5490, 5412, 5344, 5474, 5478, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=228616 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156491016 ingress_bytes=175136768 link_bytes=175136768 token_bytes=312753416 weight_movement_bytes=18874368 total_nvlink_bytes=331627784 weight_to_token_ratio=0.060349 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678074338e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38152 dup_rows=21996 dup_rows_remote=19226 dedup_ratio=0.335632 router_tokens_per_expert=[259, 239, 259, 243, 238, 272, 238, 259, 228, 262, 270, 255, 207, 260, 233, 265, 259, 263, 262, 273, 233, 246, 272, 278, 286, 223, 228, 261, 252, 241, 265, 255, 253, 223, 239, 270, 230, 270, 256, 252, 294, 257, 243, 257, 279, 244, 257, 259, 255, 260, 284, 243, 239, 271, 274, 256, 237, 262, 254, 249, 216, 246, 296, 237, 252, 269, 222, 254, 256, 271, 256, 276, 279, 231, 246, 268, 223, 253, 240, 236, 278, 270, 248, 268, 295, 274, 245, 269, 258, 254, 273, 258, 237, 243, 249, 253, 251, 248, 272, 264, 276, 294, 265, 242, 237, 261, 259, 280, 255, 246, 277, 266, 272, 267, 251, 266, 260, 238, 279, 253, 261, 261, 245, 246, 243, 260, 261, 262] payload_rows_per_dst=[5318, 5460, 5452, 5388, 5406, 5538, 5544, 5434] dup_rows_per_dst=[2656, 2734, 2714, 2770, 2658, 2806, 2842, 2816] combine_rows_per_src=[5318, 5460, 5452, 5388, 5406, 5538, 5544, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38240 dup_rows=21858 dup_rows_remote=19120 dedup_ratio=0.333527 router_tokens_per_expert=[288, 262, 256, 270, 267, 245, 256, 255, 273, 263, 253, 240, 258, 240, 243, 257, 237, 265, 250, 257, 251, 249, 285, 241, 227, 253, 224, 260, 238, 252, 254, 269, 248, 256, 248, 274, 259, 237, 283, 242, 265, 241, 248, 270, 273, 240, 274, 271, 237, 281, 277, 236, 243, 267, 253, 217, 277, 272, 264, 270, 253, 256, 268, 270, 264, 260, 258, 277, 243, 253, 254, 261, 238, 251, 260, 248, 269, 254, 236, 262, 205, 260, 238, 257, 238, 254, 245, 282, 272, 260, 245, 238, 256, 291, 261, 233, 244, 259, 257, 267, 279, 257, 240, 252, 251, 234, 251, 258, 273, 286, 234, 293, 274, 258, 249, 233, 271, 233, 231, 259, 276, 252, 282, 262, 247, 267, 247, 261] payload_rows_per_dst=[5494, 5424, 5504, 5486, 5438, 5418, 5480, 5434] dup_rows_per_dst=[2758, 2600, 2754, 2796, 2738, 2652, 2790, 2770] combine_rows_per_src=[5494, 5424, 5504, 5486, 5438, 5418, 5480, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38332 dup_rows=21782 dup_rows_remote=19000 dedup_ratio=0.332367 router_tokens_per_expert=[245, 259, 251, 259, 238, 234, 237, 253, 258, 305, 234, 226, 257, 261, 280, 255, 269, 235, 249, 267, 244, 238, 253, 273, 240, 261, 229, 264, 256, 237, 251, 252, 258, 223, 219, 252, 241, 245, 253, 236, 255, 260, 265, 256, 258, 230, 247, 265, 254, 234, 251, 289, 266, 267, 263, 262, 258, 259, 262, 267, 251, 264, 262, 266, 250, 250, 267, 257, 240, 256, 282, 257, 265, 244, 271, 262, 240, 266, 265, 260, 277, 261, 256, 265, 261, 258, 283, 266, 270, 258, 269, 255, 246, 257, 272, 250, 233, 253, 270, 282, 243, 277, 236, 260, 255, 235, 254, 268, 262, 255, 264, 255, 258, 281, 248, 237, 238, 265, 249, 244, 259, 255, 279, 251, 252, 264, 265, 277] payload_rows_per_dst=[5466, 5436, 5328, 5550, 5556, 5516, 5422, 5480] dup_rows_per_dst=[2638, 2600, 2598, 2800, 2708, 2892, 2782, 2764] combine_rows_per_src=[5466, 5436, 5328, 5550, 5556, 5516, 5422, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38174 dup_rows=21908 dup_rows_remote=19236 dedup_ratio=0.334290 router_tokens_per_expert=[239, 247, 251, 282, 250, 254, 234, 236, 256, 238, 225, 290, 269, 262, 244, 248, 291, 237, 231, 247, 270, 259, 254, 259, 260, 259, 263, 272, 232, 254, 236, 279, 253, 254, 272, 221, 252, 232, 298, 261, 240, 265, 241, 254, 264, 253, 257, 246, 276, 261, 282, 259, 240, 266, 269, 267, 231, 254, 246, 233, 253, 255, 250, 268, 248, 249, 292, 275, 239, 247, 244, 216, 249, 257, 252, 258, 260, 270, 251, 278, 277, 266, 236, 272, 258, 262, 259, 288, 276, 248, 246, 261, 259, 270, 258, 273, 239, 261, 242, 247, 265, 258, 248, 250, 251, 242, 253, 279, 291, 238, 242, 276, 268, 240, 265, 242, 268, 255, 255, 242, 279, 256, 253, 249, 243, 252, 268, 256] payload_rows_per_dst=[5414, 5456, 5454, 5432, 5414, 5542, 5474, 5442] dup_rows_per_dst=[2636, 2750, 2672, 2788, 2756, 2876, 2690, 2740] combine_rows_per_src=[5414, 5456, 5454, 5432, 5414, 5542, 5474, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229112 combine_payload_bytes=156647424 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156876536 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313523960 weight_movement_bytes=37748736 total_nvlink_bytes=351272696 weight_to_token_ratio=0.120401 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228720 combine_payload_bytes=156213248 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156441968 ingress_bytes=175087616 link_bytes=175087616 token_bytes=312655216 weight_movement_bytes=18874368 total_nvlink_bytes=331529584 weight_to_token_ratio=0.060368 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677603388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156459008 dispatch_meta_bytes=228760 combine_payload_bytes=156459008 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156687768 ingress_bytes=231956480 link_bytes=231956480 token_bytes=313146776 weight_movement_bytes=75497472 total_nvlink_bytes=388644248 weight_to_token_ratio=0.241093 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222492862e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156270592 dispatch_meta_bytes=229512 combine_payload_bytes=156270592 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156500104 ingress_bytes=175144960 link_bytes=175144960 token_bytes=312770696 weight_movement_bytes=18874368 total_nvlink_bytes=331645064 weight_to_token_ratio=0.060346 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678152830e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229440 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156860480 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313491520 weight_movement_bytes=37748736 total_nvlink_bytes=351240256 weight_to_token_ratio=0.120414 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229328 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157237200 ingress_bytes=213630976 link_bytes=213630976 token_bytes=314245072 weight_movement_bytes=56623104 total_nvlink_bytes=370868176 weight_to_token_ratio=0.180188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046906899e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229640 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156590344 ingress_bytes=175235072 link_bytes=175235072 token_bytes=312951048 weight_movement_bytes=18874368 total_nvlink_bytes=331825416 weight_to_token_ratio=0.060311 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679016239e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38252 dup_rows=21856 dup_rows_remote=18996 dedup_ratio=0.333496 router_tokens_per_expert=[237, 280, 241, 253, 269, 292, 281, 251, 278, 239, 231, 251, 265, 244, 266, 245, 237, 250, 249, 266, 260, 250, 258, 254, 269, 243, 258, 249, 241, 243, 273, 249, 243, 258, 252, 236, 249, 278, 253, 246, 241, 251, 291, 275, 245, 290, 268, 256, 261, 230, 254, 263, 265, 288, 250, 260, 249, 261, 261, 248, 247, 239, 255, 250, 267, 248, 246, 245, 259, 268, 231, 262, 285, 279, 231, 260, 258, 246, 239, 262, 268, 265, 235, 274, 238, 251, 231, 220, 267, 254, 271, 279, 261, 284, 255, 246, 235, 277, 233, 265, 273, 255, 229, 264, 252, 243, 264, 251, 253, 248, 266, 240, 280, 273, 235, 290, 248, 252, 292, 257, 242, 257, 250, 251, 289, 245, 260, 229] payload_rows_per_dst=[5428, 5476, 5486, 5636, 5662, 5478, 5266, 5248] dup_rows_per_dst=[2860, 2624, 2742, 3046, 3064, 2720, 2410, 2390] combine_rows_per_src=[5428, 5476, 5486, 5636, 5662, 5478, 5266, 5248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38186 dup_rows=21800 dup_rows_remote=19092 dedup_ratio=0.332642 router_tokens_per_expert=[262, 250, 254, 258, 259, 274, 260, 256, 237, 253, 252, 268, 257, 220, 253, 245, 293, 247, 252, 242, 243, 253, 264, 264, 273, 298, 278, 242, 236, 242, 250, 252, 249, 267, 246, 254, 250, 246, 254, 227, 249, 284, 259, 251, 252, 270, 280, 225, 232, 256, 246, 281, 275, 253, 265, 255, 236, 234, 264, 236, 242, 258, 253, 234, 261, 280, 234, 285, 284, 245, 248, 247, 270, 273, 266, 247, 235, 287, 247, 269, 262, 259, 237, 257, 242, 242, 267, 261, 254, 263, 278, 265, 280, 236, 274, 270, 281, 248, 277, 260, 255, 222, 247, 246, 250, 235, 252, 244, 275, 253, 262, 257, 254, 276, 241, 259, 252, 257, 287, 248, 251, 246, 246, 263, 275, 248, 262, 244] payload_rows_per_dst=[5400, 5550, 5434, 5394, 5522, 5510, 5414, 5512] dup_rows_per_dst=[2716, 2708, 2692, 2646, 2834, 2784, 2714, 2706] combine_rows_per_src=[5400, 5550, 5434, 5394, 5522, 5510, 5414, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228992 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909184 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313589376 weight_movement_bytes=18874368 total_nvlink_bytes=332463744 weight_to_token_ratio=0.060188 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38292 dup_rows=21756 dup_rows_remote=19026 dedup_ratio=0.331970 router_tokens_per_expert=[254, 254, 254, 247, 234, 263, 256, 236, 255, 284, 255, 271, 245, 248, 247, 254, 259, 237, 237, 255, 269, 283, 264, 244, 261, 250, 275, 266, 264, 270, 228, 247, 260, 264, 249, 240, 249, 273, 257, 246, 263, 256, 254, 244, 241, 258, 262, 267, 270, 263, 262, 293, 261, 229, 234, 243, 281, 238, 253, 290, 239, 265, 254, 253, 250, 275, 268, 249, 246, 247, 247, 280, 289, 231, 236, 252, 260, 258, 252, 258, 251, 231, 257, 275, 253, 253, 271, 252, 278, 242, 250, 258, 278, 248, 277, 235, 253, 268, 254, 265, 255, 254, 253, 256, 282, 275, 239, 276, 282, 234, 235, 268, 269, 275, 255, 260, 233, 233, 238, 266, 246, 247, 241, 259, 251, 225, 275, 262] payload_rows_per_dst=[5416, 5526, 5480, 5458, 5440, 5488, 5548, 5424] dup_rows_per_dst=[2698, 2692, 2686, 2798, 2756, 2730, 2750, 2646] combine_rows_per_src=[5416, 5526, 5480, 5458, 5440, 5488, 5548, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38194 dup_rows=21868 dup_rows_remote=19168 dedup_ratio=0.333679 router_tokens_per_expert=[258, 282, 240, 244, 277, 247, 248, 270, 239, 267, 250, 247, 254, 244, 271, 265, 249, 253, 277, 276, 254, 261, 264, 246, 264, 232, 231, 252, 253, 269, 235, 285, 271, 264, 294, 255, 276, 241, 227, 277, 225, 225, 271, 278, 266, 262, 241, 272, 284, 251, 263, 241, 273, 252, 273, 264, 251, 231, 274, 290, 240, 260, 251, 258, 251, 223, 281, 264, 252, 270, 229, 260, 268, 277, 260, 249, 279, 243, 277, 265, 260, 254, 238, 232, 282, 254, 253, 257, 262, 263, 248, 260, 247, 259, 221, 251, 253, 258, 265, 244, 260, 246, 252, 221, 250, 244, 223, 269, 259, 246, 249, 253, 233, 287, 236, 254, 258, 264, 233, 254, 247, 241, 283, 290, 241, 259, 238, 264] payload_rows_per_dst=[5500, 5260, 5550, 5516, 5472, 5416, 5480, 5474] dup_rows_per_dst=[2706, 2420, 2740, 2796, 2814, 2666, 3026, 2700] combine_rows_per_src=[5500, 5260, 5550, 5516, 5472, 5416, 5480, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38266 dup_rows=21810 dup_rows_remote=19062 dedup_ratio=0.332794 router_tokens_per_expert=[244, 243, 232, 252, 243, 249, 260, 250, 295, 261, 269, 255, 235, 251, 268, 251, 221, 271, 280, 266, 284, 270, 279, 245, 242, 282, 256, 258, 265, 253, 262, 241, 268, 260, 249, 236, 266, 291, 251, 242, 262, 244, 266, 265, 255, 272, 243, 262, 245, 285, 265, 252, 251, 262, 258, 257, 241, 241, 261, 262, 254, 262, 261, 247, 258, 270, 259, 261, 262, 270, 258, 271, 272, 243, 265, 231, 262, 252, 251, 211, 251, 214, 249, 256, 262, 269, 233, 288, 268, 262, 260, 273, 275, 242, 264, 241, 237, 249, 257, 243, 283, 240, 290, 272, 250, 245, 229, 239, 272, 236, 263, 258, 270, 251, 244, 263, 232, 264, 239, 245, 231, 249, 233, 264, 269, 249, 280, 250] payload_rows_per_dst=[5434, 5484, 5474, 5460, 5538, 5460, 5502, 5374] dup_rows_per_dst=[2682, 2866, 2790, 2748, 2654, 2754, 2624, 2692] combine_rows_per_src=[5434, 5484, 5474, 5460, 5538, 5460, 5502, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43570 payload_rows_remote=38202 dup_rows=21966 dup_rows_remote=19244 dedup_ratio=0.335175 router_tokens_per_expert=[271, 217, 259, 237, 264, 271, 251, 262, 240, 266, 290, 264, 247, 249, 265, 266, 265, 252, 260, 278, 259, 272, 286, 254, 262, 256, 284, 273, 262, 242, 247, 238, 251, 253, 234, 278, 246, 239, 266, 234, 243, 270, 263, 236, 241, 241, 276, 249, 250, 260, 252, 247, 251, 267, 260, 285, 257, 248, 229, 251, 275, 279, 259, 247, 259, 261, 236, 234, 267, 265, 241, 243, 265, 240, 282, 255, 249, 239, 250, 259, 241, 263, 253, 237, 261, 230, 271, 243, 255, 245, 263, 278, 255, 264, 252, 245, 253, 258, 231, 246, 266, 260, 294, 259, 245, 252, 253, 251, 230, 234, 262, 245, 253, 257, 261, 275, 256, 245, 258, 233, 268, 264, 278, 264, 307, 250, 269, 244] payload_rows_per_dst=[5444, 5572, 5402, 5484, 5368, 5424, 5320, 5556] dup_rows_per_dst=[2794, 2808, 2638, 2750, 2722, 2688, 2758, 2808] combine_rows_per_src=[5444, 5572, 5402, 5484, 5368, 5424, 5320, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38280 dup_rows=21778 dup_rows_remote=19028 dedup_ratio=0.332306 router_tokens_per_expert=[230, 239, 260, 255, 238, 257, 270, 239, 251, 270, 247, 238, 238, 253, 264, 255, 243, 256, 272, 292, 274, 273, 256, 276, 257, 252, 284, 251, 255, 252, 270, 237, 270, 241, 251, 248, 222, 272, 236, 270, 275, 230, 238, 267, 249, 262, 260, 236, 250, 254, 266, 277, 241, 270, 230, 259, 260, 252, 267, 247, 238, 242, 258, 265, 281, 259, 269, 253, 271, 265, 262, 267, 254, 282, 265, 245, 261, 266, 233, 229, 293, 254, 281, 254, 268, 277, 237, 237, 260, 248, 263, 263, 281, 258, 252, 263, 266, 256, 260, 249, 253, 260, 243, 246, 245, 271, 240, 244, 242, 252, 219, 257, 261, 254, 263, 273, 256, 230, 265, 266, 262, 240, 237, 262, 267, 230, 277, 264] payload_rows_per_dst=[5404, 5402, 5388, 5502, 5592, 5546, 5478, 5446] dup_rows_per_dst=[2604, 2776, 2666, 2650, 2732, 2832, 2750, 2768] combine_rows_per_src=[5404, 5402, 5388, 5502, 5592, 5546, 5478, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38422 dup_rows=21610 dup_rows_remote=18934 dedup_ratio=0.329742 router_tokens_per_expert=[255, 260, 291, 230, 255, 278, 273, 241, 277, 231, 244, 266, 279, 277, 255, 268, 284, 242, 242, 243, 281, 232, 267, 274, 240, 235, 226, 235, 243, 254, 256, 269, 253, 270, 278, 237, 255, 249, 251, 234, 258, 285, 269, 237, 249, 264, 263, 238, 247, 227, 266, 270, 257, 251, 240, 264, 278, 257, 212, 274, 269, 246, 261, 265, 253, 229, 239, 255, 258, 277, 255, 261, 259, 248, 283, 263, 258, 256, 263, 239, 237, 263, 261, 247, 303, 252, 266, 276, 280, 257, 267, 245, 246, 253, 252, 258, 283, 266, 255, 241, 270, 253, 248, 276, 253, 248, 255, 244, 262, 256, 246, 258, 235, 283, 259, 267, 223, 225, 247, 258, 239, 258, 258, 244, 256, 251, 246, 269] payload_rows_per_dst=[5526, 5432, 5504, 5472, 5484, 5506, 5580, 5422] dup_rows_per_dst=[2834, 2614, 2676, 2696, 2708, 2820, 2648, 2614] combine_rows_per_src=[5526, 5432, 5504, 5472, 5484, 5506, 5580, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229112 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156638968 ingress_bytes=194158592 link_bytes=194158592 token_bytes=313048824 weight_movement_bytes=37748736 total_nvlink_bytes=350797560 weight_to_token_ratio=0.120584 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860332097e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229272 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073304 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917336 weight_movement_bytes=18874368 total_nvlink_bytes=332791704 weight_to_token_ratio=0.060125 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229448 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156672072 ingress_bytes=213065728 link_bytes=213065728 token_bytes=313114696 weight_movement_bytes=56623104 total_nvlink_bytes=369737800 weight_to_token_ratio=0.180838 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041490971e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=229312 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966848 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704384 weight_movement_bytes=18874368 total_nvlink_bytes=332578752 weight_to_token_ratio=0.060166 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229784 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156705176 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313180568 weight_movement_bytes=18874368 total_nvlink_bytes=332054936 weight_to_token_ratio=0.060267 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=229232 combine_payload_bytes=156794880 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157024112 ingress_bytes=175669248 link_bytes=175669248 token_bytes=313818992 weight_movement_bytes=18874368 total_nvlink_bytes=332693360 weight_to_token_ratio=0.060144 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683176300e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229424 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157605936 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982448 weight_movement_bytes=18874368 total_nvlink_bytes=333856816 weight_to_token_ratio=0.059922 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38190 dup_rows=21798 dup_rows_remote=18976 dedup_ratio=0.332611 router_tokens_per_expert=[261, 284, 267, 245, 252, 268, 266, 264, 266, 259, 260, 234, 250, 283, 253, 268, 271, 285, 241, 271, 267, 253, 257, 233, 289, 248, 251, 294, 228, 244, 250, 257, 223, 268, 246, 264, 242, 253, 259, 258, 257, 239, 246, 248, 239, 224, 238, 267, 265, 252, 247, 260, 253, 249, 240, 234, 260, 256, 247, 263, 285, 248, 238, 267, 243, 250, 257, 243, 245, 254, 247, 247, 261, 289, 260, 272, 245, 249, 247, 273, 254, 283, 260, 262, 257, 229, 257, 259, 268, 295, 265, 254, 244, 256, 257, 263, 241, 260, 240, 251, 270, 288, 236, 287, 261, 236, 255, 263, 265, 255, 253, 266, 253, 242, 240, 237, 245, 289, 271, 269, 246, 257, 269, 219, 246, 264, 236, 259] payload_rows_per_dst=[5548, 5582, 5572, 5478, 5474, 5290, 5380, 5414] dup_rows_per_dst=[2822, 2734, 2890, 2720, 2690, 2516, 2756, 2670] combine_rows_per_src=[5548, 5582, 5572, 5478, 5474, 5290, 5380, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15045753048 prefetch_bytes=830472192 reduce_bytes=641728512 weight_movement_bytes=1472200704 egress_bytes=7528380632 ingress_bytes=8989573120 link_bytes=8989573120 total_nvlink_bytes=16517953752 weight_to_token_ratio=0.097848 prefetch_share_of_ingress=0.092382 dedup_ratio=0.333125 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=34 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081163 gemm_inner_pad_upper_bound=109220 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.794451385e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228664 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156654904 ingress_bytes=213049344 link_bytes=213049344 token_bytes=313081144 weight_movement_bytes=56623104 total_nvlink_bytes=369704248 weight_to_token_ratio=0.180858 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041333987e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43868 payload_rows_remote=38422 dup_rows=21668 dup_rows_remote=18964 dedup_ratio=0.330627 router_tokens_per_expert=[263, 272, 263, 255, 246, 257, 260, 240, 269, 264, 238, 288, 257, 249, 253, 232, 255, 254, 236, 260, 269, 232, 264, 247, 260, 262, 248, 254, 263, 274, 263, 234, 263, 270, 244, 246, 241, 240, 251, 261, 255, 241, 236, 271, 252, 231, 251, 251, 268, 241, 266, 257, 255, 243, 250, 264, 265, 246, 265, 229, 252, 238, 244, 255, 222, 261, 251, 280, 282, 282, 250, 238, 266, 244, 251, 242, 284, 277, 262, 268, 260, 244, 232, 267, 252, 255, 245, 257, 266, 245, 241, 289, 237, 280, 243, 267, 250, 275, 265, 236, 254, 238, 259, 285, 266, 273, 247, 251, 252, 269, 290, 254, 250, 256, 266, 242, 282, 252, 227, 256, 270, 260, 243, 266, 240, 271, 272, 288] payload_rows_per_dst=[5516, 5446, 5428, 5428, 5534, 5442, 5546, 5528] dup_rows_per_dst=[2696, 2704, 2590, 2648, 2776, 2718, 2782, 2754] combine_rows_per_src=[5516, 5446, 5428, 5428, 5534, 5442, 5546, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38336 dup_rows=21820 dup_rows_remote=19104 dedup_ratio=0.332947 router_tokens_per_expert=[242, 275, 269, 234, 255, 224, 232, 276, 243, 281, 282, 266, 235, 262, 253, 250, 253, 250, 258, 257, 248, 260, 255, 240, 257, 246, 251, 259, 269, 234, 253, 244, 253, 245, 241, 254, 252, 248, 234, 249, 257, 257, 277, 252, 248, 250, 229, 278, 296, 246, 264, 266, 247, 275, 278, 249, 260, 275, 241, 265, 237, 242, 260, 276, 287, 250, 280, 245, 245, 237, 248, 254, 251, 234, 293, 238, 239, 253, 265, 256, 264, 222, 267, 241, 255, 266, 246, 266, 276, 239, 219, 246, 282, 252, 253, 254, 229, 287, 272, 243, 238, 257, 236, 255, 257, 232, 262, 282, 295, 290, 260, 262, 267, 253, 246, 276, 275, 257, 260, 263, 275, 252, 274, 257, 259, 270, 256, 234] payload_rows_per_dst=[5470, 5424, 5396, 5480, 5480, 5380, 5536, 5550] dup_rows_per_dst=[2688, 2644, 2652, 2874, 2670, 2716, 2778, 2798] combine_rows_per_src=[5470, 5424, 5396, 5480, 5480, 5380, 5536, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38248 dup_rows=21788 dup_rows_remote=19090 dedup_ratio=0.332458 router_tokens_per_expert=[272, 259, 284, 262, 245, 274, 249, 272, 252, 245, 283, 228, 237, 225, 265, 260, 243, 241, 247, 258, 264, 280, 256, 228, 245, 277, 242, 276, 267, 241, 242, 248, 227, 280, 262, 261, 254, 241, 264, 249, 246, 230, 255, 282, 268, 252, 285, 252, 253, 258, 272, 261, 224, 259, 251, 268, 243, 258, 250, 262, 259, 258, 259, 257, 254, 242, 261, 245, 237, 271, 237, 281, 277, 285, 222, 228, 233, 269, 253, 265, 251, 258, 233, 294, 290, 256, 243, 284, 247, 257, 246, 239, 267, 232, 247, 255, 261, 318, 271, 247, 267, 236, 242, 243, 258, 267, 249, 245, 268, 250, 268, 260, 252, 260, 259, 252, 254, 285, 248, 270, 253, 240, 261, 254, 243, 266, 247, 248] payload_rows_per_dst=[5436, 5288, 5524, 5412, 5430, 5594, 5564, 5500] dup_rows_per_dst=[2774, 2418, 2692, 2722, 2740, 2906, 2838, 2698] combine_rows_per_src=[5436, 5288, 5524, 5412, 5430, 5594, 5564, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38346 dup_rows=21744 dup_rows_remote=18984 dedup_ratio=0.331787 router_tokens_per_expert=[281, 260, 255, 257, 220, 283, 270, 272, 269, 246, 260, 251, 246, 251, 264, 220, 267, 267, 266, 269, 255, 269, 239, 244, 247, 281, 240, 260, 264, 285, 251, 243, 247, 246, 258, 241, 263, 256, 276, 249, 266, 249, 249, 242, 241, 266, 251, 217, 223, 249, 277, 263, 251, 269, 266, 256, 267, 256, 249, 259, 273, 237, 250, 258, 241, 281, 297, 253, 243, 280, 265, 256, 266, 254, 244, 277, 248, 230, 247, 253, 272, 259, 245, 230, 247, 237, 259, 239, 243, 236, 261, 260, 282, 264, 231, 242, 251, 248, 220, 252, 263, 240, 244, 283, 277, 263, 256, 238, 273, 248, 249, 230, 249, 280, 261, 248, 259, 252, 294, 263, 248, 271, 274, 243, 281, 273, 260, 263] payload_rows_per_dst=[5518, 5520, 5414, 5446, 5502, 5438, 5416, 5538] dup_rows_per_dst=[2692, 2774, 2620, 2760, 2768, 2576, 2654, 2900] combine_rows_per_src=[5518, 5520, 5414, 5446, 5502, 5438, 5416, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38214 dup_rows=21876 dup_rows_remote=19178 dedup_ratio=0.333801 router_tokens_per_expert=[255, 254, 297, 285, 259, 279, 257, 278, 265, 253, 224, 247, 259, 254, 247, 254, 259, 250, 231, 229, 275, 259, 259, 251, 258, 281, 262, 254, 243, 276, 240, 251, 252, 260, 268, 248, 249, 290, 242, 261, 248, 258, 248, 260, 256, 231, 254, 256, 266, 266, 258, 259, 265, 258, 258, 251, 239, 241, 246, 271, 236, 259, 278, 240, 245, 223, 263, 240, 247, 252, 258, 241, 267, 248, 273, 271, 276, 248, 264, 256, 261, 266, 275, 262, 228, 243, 266, 256, 290, 254, 266, 227, 269, 277, 279, 241, 252, 282, 236, 246, 255, 248, 268, 266, 228, 244, 260, 266, 265, 237, 270, 269, 248, 249, 261, 240, 278, 227, 245, 242, 222, 264, 270, 229, 252, 263, 280, 257] payload_rows_per_dst=[5508, 5458, 5398, 5412, 5446, 5474, 5534, 5430] dup_rows_per_dst=[2826, 2698, 2764, 2770, 2698, 2846, 2650, 2624] combine_rows_per_src=[5508, 5458, 5398, 5412, 5446, 5474, 5534, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229544 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157606056 ingress_bytes=213999616 link_bytes=213999616 token_bytes=314982568 weight_movement_bytes=56623104 total_nvlink_bytes=371605672 weight_to_token_ratio=0.179766 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050439026e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38380 dup_rows=21662 dup_rows_remote=19080 dedup_ratio=0.330536 router_tokens_per_expert=[259, 242, 270, 288, 232, 234, 255, 275, 242, 266, 234, 268, 276, 266, 238, 224, 270, 261, 274, 265, 260, 254, 265, 263, 284, 292, 264, 269, 266, 235, 256, 250, 257, 259, 251, 271, 261, 240, 257, 262, 256, 229, 230, 252, 232, 258, 244, 285, 252, 245, 255, 245, 248, 265, 244, 264, 270, 261, 252, 264, 234, 229, 302, 294, 241, 268, 263, 221, 267, 273, 247, 250, 266, 308, 262, 288, 274, 240, 248, 252, 267, 259, 280, 252, 253, 244, 245, 226, 258, 248, 237, 240, 267, 247, 251, 265, 276, 227, 260, 264, 284, 215, 231, 236, 274, 262, 264, 253, 250, 274, 218, 250, 273, 228, 250, 276, 258, 284, 242, 269, 251, 255, 234, 260, 237, 243, 252, 246] payload_rows_per_dst=[5458, 5650, 5366, 5492, 5496, 5436, 5494, 5482] dup_rows_per_dst=[2680, 2806, 2722, 2756, 2840, 2642, 2582, 2634] combine_rows_per_src=[5458, 5650, 5366, 5492, 5496, 5436, 5494, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229352 combine_payload_bytes=156663808 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156893160 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313556968 weight_movement_bytes=37748736 total_nvlink_bytes=351305704 weight_to_token_ratio=0.120389 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229760 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157254016 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314278272 weight_movement_bytes=37748736 total_nvlink_bytes=352027008 weight_to_token_ratio=0.120112 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229320 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157294536 ingress_bytes=194813952 link_bytes=194813952 token_bytes=314359752 weight_movement_bytes=37748736 total_nvlink_bytes=352108488 weight_to_token_ratio=0.120081 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866611434e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229568 combine_payload_bytes=156524544 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156754112 ingress_bytes=213147648 link_bytes=213147648 token_bytes=313278656 weight_movement_bytes=56623104 total_nvlink_bytes=369901760 weight_to_token_ratio=0.180744 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042275888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38232 dup_rows=21876 dup_rows_remote=19188 dedup_ratio=0.333801 router_tokens_per_expert=[274, 270, 247, 246, 230, 240, 257, 241, 266, 257, 246, 247, 240, 260, 244, 238, 251, 255, 230, 231, 275, 271, 264, 232, 246, 249, 268, 261, 268, 269, 247, 237, 267, 214, 275, 256, 253, 210, 276, 281, 262, 256, 243, 239, 268, 281, 263, 233, 261, 243, 265, 269, 256, 262, 235, 274, 253, 247, 259, 262, 272, 262, 267, 270, 240, 265, 270, 237, 278, 276, 234, 232, 273, 254, 232, 291, 261, 258, 258, 251, 244, 278, 224, 273, 255, 255, 246, 261, 251, 272, 224, 269, 279, 231, 250, 290, 249, 261, 272, 254, 276, 260, 260, 268, 250, 262, 275, 225, 235, 275, 264, 241, 261, 251, 238, 255, 274, 273, 256, 283, 264, 240, 260, 280, 262, 267, 247, 227] payload_rows_per_dst=[5428, 5418, 5548, 5532, 5402, 5634, 5354, 5344] dup_rows_per_dst=[2688, 2706, 2922, 2782, 2730, 3034, 2546, 2468] combine_rows_per_src=[5428, 5418, 5548, 5532, 5402, 5634, 5354, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229840 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157434320 ingress_bytes=176078848 link_bytes=176078848 token_bytes=314638800 weight_movement_bytes=18874368 total_nvlink_bytes=333513168 weight_to_token_ratio=0.059987 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687100885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229680 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156827952 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426224 weight_movement_bytes=18874368 total_nvlink_bytes=332300592 weight_to_token_ratio=0.060219 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38284 dup_rows=21738 dup_rows_remote=18948 dedup_ratio=0.331696 router_tokens_per_expert=[262, 278, 255, 252, 224, 240, 231, 257, 247, 269, 230, 238, 231, 259, 260, 239, 238, 274, 265, 287, 227, 246, 257, 261, 255, 260, 257, 258, 308, 245, 264, 250, 282, 258, 275, 261, 234, 271, 263, 280, 290, 243, 260, 235, 250, 266, 251, 259, 243, 257, 265, 243, 261, 245, 254, 254, 271, 270, 267, 243, 244, 230, 258, 246, 256, 278, 280, 260, 266, 262, 272, 263, 248, 250, 253, 277, 290, 253, 249, 262, 250, 235, 258, 248, 251, 272, 257, 249, 247, 264, 299, 255, 254, 240, 266, 234, 229, 233, 238, 257, 254, 262, 262, 249, 275, 272, 256, 269, 260, 236, 269, 238, 243, 229, 222, 254, 252, 277, 255, 263, 231, 261, 285, 263, 253, 269, 248, 253] payload_rows_per_dst=[5398, 5514, 5566, 5402, 5582, 5468, 5382, 5486] dup_rows_per_dst=[2546, 2790, 2790, 2700, 2856, 2690, 2736, 2630] combine_rows_per_src=[5398, 5514, 5566, 5402, 5582, 5468, 5382, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38210 dup_rows=21856 dup_rows_remote=19206 dedup_ratio=0.333496 router_tokens_per_expert=[271, 270, 279, 252, 228, 265, 212, 260, 234, 233, 257, 266, 241, 260, 262, 246, 235, 272, 256, 252, 250, 260, 253, 260, 282, 229, 265, 269, 236, 258, 290, 283, 278, 237, 238, 253, 276, 233, 282, 248, 265, 226, 253, 274, 227, 269, 274, 250, 275, 222, 268, 256, 271, 281, 240, 286, 247, 261, 252, 259, 246, 267, 284, 257, 228, 220, 242, 257, 222, 252, 266, 285, 251, 262, 273, 251, 267, 244, 266, 257, 269, 270, 282, 272, 254, 231, 277, 236, 237, 267, 289, 256, 265, 220, 244, 272, 235, 264, 248, 252, 261, 269, 287, 253, 233, 260, 264, 276, 270, 269, 270, 249, 226, 250, 229, 246, 271, 239, 249, 240, 280, 240, 253, 251, 252, 251, 252, 254] payload_rows_per_dst=[5478, 5430, 5448, 5542, 5316, 5514, 5482, 5470] dup_rows_per_dst=[2594, 2716, 2718, 2802, 2604, 2770, 3002, 2650] combine_rows_per_src=[5478, 5430, 5448, 5542, 5316, 5514, 5482, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43886 payload_rows_remote=38354 dup_rows=21650 dup_rows_remote=18998 dedup_ratio=0.330353 router_tokens_per_expert=[284, 254, 263, 266, 252, 239, 282, 250, 253, 260, 254, 290, 268, 259, 222, 237, 264, 269, 242, 251, 252, 247, 259, 284, 248, 289, 259, 283, 238, 250, 219, 235, 249, 256, 257, 249, 217, 243, 272, 273, 263, 262, 260, 283, 256, 241, 248, 250, 233, 249, 269, 252, 267, 259, 260, 263, 273, 277, 257, 242, 248, 258, 280, 244, 224, 224, 248, 263, 261, 254, 216, 273, 263, 241, 251, 258, 247, 270, 250, 247, 245, 248, 252, 267, 239, 279, 266, 254, 258, 273, 256, 250, 253, 240, 256, 256, 288, 252, 247, 264, 273, 217, 255, 253, 261, 293, 264, 272, 247, 252, 277, 257, 240, 261, 229, 234, 291, 244, 254, 253, 245, 278, 255, 269, 260, 239, 263, 267] payload_rows_per_dst=[5528, 5398, 5488, 5464, 5416, 5532, 5576, 5484] dup_rows_per_dst=[2738, 2780, 2670, 2798, 2564, 2652, 2768, 2680] combine_rows_per_src=[5528, 5398, 5488, 5464, 5416, 5532, 5576, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38502 dup_rows=21632 dup_rows_remote=18790 dedup_ratio=0.330078 router_tokens_per_expert=[232, 235, 223, 255, 283, 262, 268, 266, 258, 247, 265, 268, 249, 268, 258, 276, 257, 231, 266, 265, 246, 247, 278, 262, 287, 288, 262, 236, 274, 265, 256, 272, 248, 207, 242, 259, 252, 266, 242, 244, 269, 301, 277, 263, 261, 260, 261, 256, 267, 244, 267, 225, 283, 278, 254, 252, 250, 247, 280, 264, 263, 251, 255, 242, 259, 229, 256, 236, 240, 257, 256, 241, 276, 260, 268, 249, 260, 273, 261, 258, 238, 282, 258, 237, 247, 244, 250, 257, 242, 236, 248, 259, 221, 278, 260, 235, 292, 226, 245, 272, 289, 224, 227, 253, 252, 283, 248, 248, 234, 262, 264, 257, 250, 276, 271, 252, 243, 251, 229, 251, 245, 269, 266, 254, 279, 237, 265, 248] payload_rows_per_dst=[5448, 5560, 5498, 5402, 5540, 5372, 5606, 5478] dup_rows_per_dst=[2778, 2824, 2718, 2842, 2618, 2612, 2546, 2694] combine_rows_per_src=[5448, 5560, 5498, 5402, 5540, 5372, 5606, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38080 dup_rows=21902 dup_rows_remote=18902 dedup_ratio=0.334198 router_tokens_per_expert=[254, 245, 262, 247, 241, 247, 255, 274, 265, 253, 278, 257, 249, 256, 256, 256, 238, 266, 255, 272, 255, 222, 260, 262, 262, 225, 270, 264, 274, 249, 246, 244, 246, 239, 241, 271, 242, 269, 251, 271, 272, 275, 279, 238, 277, 265, 243, 255, 271, 244, 248, 253, 254, 245, 272, 247, 263, 278, 248, 252, 258, 259, 232, 262, 280, 245, 307, 282, 267, 243, 281, 243, 260, 267, 241, 273, 280, 278, 289, 241, 228, 250, 250, 252, 252, 240, 254, 259, 253, 241, 236, 265, 277, 281, 255, 273, 250, 229, 273, 226, 253, 266, 214, 257, 258, 268, 261, 227, 263, 270, 268, 252, 237, 245, 247, 251, 264, 251, 253, 249, 258, 231, 260, 244, 245, 261, 265, 250] payload_rows_per_dst=[5524, 5420, 5460, 5376, 5554, 5466, 5456, 5378] dup_rows_per_dst=[2666, 2708, 2808, 2796, 3000, 2666, 2614, 2644] combine_rows_per_src=[5524, 5420, 5460, 5376, 5554, 5466, 5456, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38274 dup_rows=21762 dup_rows_remote=19092 dedup_ratio=0.332062 router_tokens_per_expert=[296, 254, 248, 263, 278, 252, 235, 257, 265, 239, 238, 272, 243, 273, 267, 254, 245, 254, 245, 264, 244, 217, 260, 246, 248, 244, 246, 237, 264, 283, 260, 225, 230, 249, 254, 266, 237, 263, 264, 266, 266, 250, 251, 254, 221, 258, 226, 252, 277, 251, 252, 282, 255, 282, 263, 275, 258, 265, 253, 308, 250, 239, 246, 253, 283, 246, 244, 257, 246, 299, 247, 287, 260, 261, 256, 288, 269, 226, 230, 273, 249, 249, 236, 237, 255, 258, 274, 262, 234, 247, 260, 288, 252, 236, 266, 264, 237, 249, 236, 274, 248, 279, 250, 238, 251, 280, 244, 242, 266, 285, 266, 240, 259, 238, 268, 265, 245, 261, 257, 265, 259, 256, 244, 236, 266, 285, 271, 237] payload_rows_per_dst=[5444, 5362, 5412, 5558, 5558, 5428, 5500, 5512] dup_rows_per_dst=[2824, 2602, 2602, 2860, 2786, 2706, 2670, 2712] combine_rows_per_src=[5444, 5362, 5412, 5558, 5558, 5428, 5500, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=228928 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040192 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851456 weight_movement_bytes=37748736 total_nvlink_bytes=351600192 weight_to_token_ratio=0.120276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=229664 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156737824 ingress_bytes=175382528 link_bytes=175382528 token_bytes=313245984 weight_movement_bytes=18874368 total_nvlink_bytes=332120352 weight_to_token_ratio=0.060254 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680429090e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229408 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327392 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425376 weight_movement_bytes=18874368 total_nvlink_bytes=333299744 weight_to_token_ratio=0.060028 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157704192 dispatch_meta_bytes=229168 combine_payload_bytes=157704192 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157933360 ingress_bytes=214327296 link_bytes=214327296 token_bytes=315637552 weight_movement_bytes=56623104 total_nvlink_bytes=372260656 weight_to_token_ratio=0.179393 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053578695e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=155975680 dispatch_meta_bytes=227928 combine_payload_bytes=155975680 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156203608 ingress_bytes=193724416 link_bytes=193724416 token_bytes=312179288 weight_movement_bytes=37748736 total_nvlink_bytes=349928024 weight_to_token_ratio=0.120920 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.856172036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38122 dup_rows=21776 dup_rows_remote=18824 dedup_ratio=0.332275 router_tokens_per_expert=[249, 265, 245, 241, 271, 278, 283, 254, 264, 254, 282, 235, 269, 277, 250, 252, 233, 233, 227, 261, 270, 256, 247, 250, 270, 240, 260, 276, 231, 244, 268, 293, 259, 270, 259, 252, 277, 293, 231, 254, 259, 246, 261, 251, 235, 271, 261, 243, 253, 258, 258, 253, 277, 263, 259, 258, 246, 243, 249, 280, 254, 271, 268, 252, 260, 235, 281, 251, 253, 240, 240, 231, 264, 271, 234, 241, 258, 262, 254, 250, 245, 255, 292, 276, 239, 258, 272, 235, 235, 261, 245, 234, 270, 248, 267, 273, 220, 251, 237, 251, 244, 250, 270, 263, 253, 260, 253, 255, 232, 268, 259, 242, 274, 253, 244, 253, 278, 268, 243, 269, 241, 261, 253, 277, 280, 249, 241, 254] payload_rows_per_dst=[5638, 5276, 5596, 5554, 5626, 5504, 5264, 5302] dup_rows_per_dst=[2952, 2590, 2794, 2848, 2888, 2754, 2522, 2428] combine_rows_per_src=[5638, 5276, 5596, 5554, 5626, 5504, 5264, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229464 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999768 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770072 weight_movement_bytes=18874368 total_nvlink_bytes=332644440 weight_to_token_ratio=0.060153 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15059482552 prefetch_bytes=792723456 reduce_bytes=622854144 weight_movement_bytes=1415577600 egress_bytes=7535245240 ingress_bytes=8939814912 link_bytes=8939814912 total_nvlink_bytes=16475060152 weight_to_token_ratio=0.093999 prefetch_share_of_ingress=0.088673 dedup_ratio=0.332480 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=33 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081380 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.784518913e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15026217152 prefetch_bytes=792723456 reduce_bytes=868220928 weight_movement_bytes=1660944384 egress_bytes=7518601408 ingress_bytes=9168560128 link_bytes=9168560128 total_nvlink_bytes=16687161536 weight_to_token_ratio=0.110536 prefetch_share_of_ingress=0.086461 dedup_ratio=0.333221 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=46 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080223 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.830179832e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15055617216 prefetch_bytes=717225984 reduce_bytes=924844032 weight_movement_bytes=1642070016 egress_bytes=7533313216 ingress_bytes=9164374016 link_bytes=9164374016 total_nvlink_bytes=16697687232 weight_to_token_ratio=0.109067 prefetch_share_of_ingress=0.078262 dedup_ratio=0.332782 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=49 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080367 gemm_inner_pad_upper_bound=108458 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.829344222e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=227784 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156375496 ingress_bytes=175022080 link_bytes=175022080 token_bytes=312523208 weight_movement_bytes=18874368 total_nvlink_bytes=331397576 weight_to_token_ratio=0.060393 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676975454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15080116128 prefetch_bytes=773849088 reduce_bytes=736100352 weight_movement_bytes=1509949440 egress_bytes=7545565088 ingress_bytes=9044500480 link_bytes=9044730392 total_nvlink_bytes=16590065568 weight_to_token_ratio=0.100129 prefetch_share_of_ingress=0.085560 dedup_ratio=0.331905 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=39 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080729 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.805461590e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38190 dup_rows=21786 dup_rows_remote=18970 dedup_ratio=0.332428 router_tokens_per_expert=[260, 256, 241, 247, 262, 240, 224, 269, 240, 269, 270, 234, 234, 247, 243, 238, 262, 255, 258, 241, 232, 246, 238, 250, 257, 255, 277, 280, 274, 273, 264, 247, 266, 260, 234, 249, 250, 236, 256, 239, 252, 254, 272, 270, 265, 256, 265, 260, 256, 258, 268, 249, 229, 233, 264, 283, 245, 258, 256, 253, 293, 249, 256, 253, 275, 251, 255, 276, 263, 241, 237, 248, 231, 257, 267, 264, 250, 242, 302, 226, 264, 265, 267, 248, 252, 267, 266, 254, 276, 254, 264, 252, 257, 269, 270, 263, 229, 245, 238, 232, 237, 277, 244, 265, 250, 245, 243, 239, 262, 266, 272, 282, 264, 243, 240, 260, 269, 252, 251, 278, 262, 256, 268, 257, 277, 272, 277, 273] payload_rows_per_dst=[5360, 5426, 5522, 5450, 5448, 5560, 5408, 5576] dup_rows_per_dst=[2588, 2792, 2646, 2756, 2722, 2816, 2644, 2822] combine_rows_per_src=[5360, 5426, 5522, 5450, 5448, 5560, 5408, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38410 dup_rows=21658 dup_rows_remote=18902 dedup_ratio=0.330475 router_tokens_per_expert=[244, 288, 261, 263, 281, 251, 248, 268, 254, 245, 241, 260, 256, 263, 266, 271, 254, 279, 244, 240, 228, 247, 257, 281, 248, 265, 254, 264, 245, 249, 248, 253, 236, 260, 258, 284, 252, 271, 261, 260, 252, 239, 239, 248, 236, 264, 242, 267, 261, 244, 264, 257, 252, 278, 269, 265, 251, 267, 275, 253, 267, 262, 262, 305, 229, 252, 275, 269, 257, 249, 261, 257, 268, 280, 260, 227, 258, 281, 242, 247, 260, 269, 231, 270, 229, 255, 249, 251, 256, 233, 243, 259, 263, 246, 238, 268, 228, 264, 269, 263, 251, 245, 240, 260, 272, 275, 242, 225, 232, 264, 248, 258, 263, 259, 258, 278, 245, 261, 257, 257, 242, 245, 242, 229, 264, 263, 252, 268] payload_rows_per_dst=[5614, 5444, 5398, 5610, 5468, 5438, 5482, 5424] dup_rows_per_dst=[2706, 2668, 2740, 2854, 2756, 2602, 2590, 2742] combine_rows_per_src=[5614, 5444, 5398, 5610, 5468, 5438, 5482, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=228640 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156654880 ingress_bytes=194174976 link_bytes=194174976 token_bytes=313081120 weight_movement_bytes=37748736 total_nvlink_bytes=350829856 weight_to_token_ratio=0.120572 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860489080e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38160 dup_rows=21812 dup_rows_remote=18976 dedup_ratio=0.332825 router_tokens_per_expert=[274, 271, 258, 265, 234, 273, 279, 269, 264, 254, 250, 253, 243, 276, 249, 250, 253, 280, 244, 264, 284, 281, 221, 269, 256, 228, 246, 234, 254, 271, 283, 260, 296, 253, 232, 252, 263, 242, 254, 263, 269, 266, 242, 239, 276, 233, 268, 234, 257, 250, 250, 241, 245, 263, 270, 244, 246, 264, 271, 245, 227, 258, 251, 262, 256, 269, 264, 248, 267, 254, 253, 231, 235, 240, 243, 267, 256, 254, 266, 282, 245, 235, 252, 274, 252, 252, 241, 249, 281, 267, 269, 263, 245, 274, 256, 268, 258, 271, 230, 250, 241, 248, 235, 263, 249, 264, 228, 256, 259, 245, 256, 228, 261, 267, 277, 271, 239, 243, 263, 249, 267, 262, 264, 241, 256, 279, 268, 256] payload_rows_per_dst=[5564, 5644, 5650, 5350, 5262, 5396, 5364, 5494] dup_rows_per_dst=[2836, 2934, 2898, 2750, 2512, 2586, 2598, 2698] combine_rows_per_src=[5564, 5644, 5650, 5350, 5262, 5396, 5364, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229248 combine_payload_bytes=157327360 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157556608 ingress_bytes=213950464 link_bytes=213950464 token_bytes=314883968 weight_movement_bytes=56623104 total_nvlink_bytes=371507072 weight_to_token_ratio=0.179822 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.049968076e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15068264688 prefetch_bytes=754974720 reduce_bytes=811597824 weight_movement_bytes=1566572544 egress_bytes=7539636464 ingress_bytes=9095200768 link_bytes=9095429904 total_nvlink_bytes=16634837232 weight_to_token_ratio=0.103965 prefetch_share_of_ingress=0.083008 dedup_ratio=0.332230 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=43 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080367 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.815581961e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228544 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156531904 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312835264 weight_movement_bytes=18874368 total_nvlink_bytes=331709632 weight_to_token_ratio=0.060333 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15058611832 prefetch_bytes=717225984 reduce_bytes=773849088 weight_movement_bytes=1491075072 egress_bytes=7534808696 ingress_bytes=9014878208 link_bytes=9014878208 total_nvlink_bytes=16549686904 weight_to_token_ratio=0.099018 prefetch_share_of_ingress=0.079560 dedup_ratio=0.332642 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=41 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082465 gemm_inner_pad_upper_bound=108458 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.799502654e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43600 payload_rows_remote=38114 dup_rows=21936 dup_rows_remote=19162 dedup_ratio=0.334717 router_tokens_per_expert=[267, 266, 263, 253, 275, 266, 261, 235, 237, 247, 267, 263, 235, 262, 276, 256, 270, 258, 288, 270, 272, 259, 240, 261, 238, 271, 251, 266, 290, 242, 245, 269, 276, 256, 270, 280, 250, 247, 233, 240, 245, 277, 220, 230, 283, 225, 289, 243, 259, 250, 228, 227, 265, 249, 246, 262, 257, 268, 256, 250, 276, 277, 267, 273, 265, 243, 250, 238, 237, 287, 267, 260, 255, 268, 255, 273, 226, 239, 271, 289, 237, 268, 274, 274, 267, 236, 249, 253, 230, 236, 259, 251, 242, 258, 233, 258, 270, 268, 270, 268, 258, 221, 221, 275, 248, 256, 229, 269, 251, 269, 232, 236, 262, 283, 230, 234, 232, 257, 249, 267, 236, 262, 266, 283, 255, 247, 253, 270] payload_rows_per_dst=[5486, 5518, 5606, 5478, 5314, 5372, 5410, 5416] dup_rows_per_dst=[2774, 2900, 3022, 2740, 2432, 2678, 2672, 2718] combine_rows_per_src=[5486, 5518, 5606, 5478, 5314, 5372, 5410, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156114944 dispatch_meta_bytes=229104 combine_payload_bytes=156114944 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156344048 ingress_bytes=193863680 link_bytes=193863680 token_bytes=312458992 weight_movement_bytes=37748736 total_nvlink_bytes=350207728 weight_to_token_ratio=0.120812 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857506395e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38302 dup_rows=21776 dup_rows_remote=18836 dedup_ratio=0.332275 router_tokens_per_expert=[279, 232, 248, 243, 233, 254, 253, 261, 243, 240, 242, 270, 220, 263, 281, 263, 272, 261, 245, 261, 283, 255, 275, 244, 263, 258, 276, 249, 225, 265, 262, 244, 256, 261, 250, 253, 259, 272, 258, 227, 226, 269, 236, 243, 255, 282, 259, 265, 268, 275, 231, 243, 260, 272, 261, 257, 258, 227, 239, 247, 227, 268, 226, 233, 235, 261, 266, 270, 264, 255, 255, 254, 260, 263, 245, 269, 267, 257, 248, 254, 242, 251, 257, 275, 263, 249, 254, 272, 261, 258, 260, 248, 266, 231, 266, 284, 274, 274, 267, 253, 286, 241, 253, 260, 261, 234, 233, 251, 276, 265, 247, 255, 260, 264, 285, 250, 268, 280, 253, 279, 281, 260, 249, 240, 250, 256, 236, 241] payload_rows_per_dst=[5458, 5578, 5494, 5248, 5340, 5526, 5550, 5566] dup_rows_per_dst=[2940, 2784, 2804, 2342, 2750, 2708, 2710, 2738] combine_rows_per_src=[5458, 5578, 5494, 5248, 5340, 5526, 5550, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=228552 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157113544 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313998536 weight_movement_bytes=18874368 total_nvlink_bytes=332872904 weight_to_token_ratio=0.060110 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15032321320 prefetch_bytes=849346560 reduce_bytes=849346560 weight_movement_bytes=1698693120 egress_bytes=7521649960 ingress_bytes=9209364480 link_bytes=9209364480 total_nvlink_bytes=16731014440 weight_to_token_ratio=0.113003 prefetch_share_of_ingress=0.092226 dedup_ratio=0.332715 prefetch_slots=90 unused_prefetch_slots=1446 slot_utilization=0.058594 reduce_slots=45 remote_demand=45 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081091 gemm_inner_pad_upper_bound=109347 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.838324982e-04 prefetch_bytes_per_gemm_flop=1.695421007e-05 anomalies=none + [2026-08-29 23:39:12] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 776.0 | throughput per GPU (TFLOP/s/GPU): 114.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.791571E+00 | loss scale: 1.0 | grad norm: 2.968 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38164 dup_rows=21928 dup_rows_remote=19174 dedup_ratio=0.334595 router_tokens_per_expert=[241, 245, 232, 260, 246, 255, 243, 266, 244, 248, 276, 258, 260, 259, 249, 267, 267, 262, 243, 259, 240, 261, 290, 252, 270, 240, 268, 257, 264, 273, 264, 256, 250, 263, 248, 246, 278, 240, 251, 247, 279, 246, 267, 274, 268, 249, 228, 265, 258, 247, 274, 239, 220, 260, 283, 214, 272, 248, 222, 271, 263, 240, 248, 229, 245, 243, 299, 254, 246, 252, 242, 255, 239, 292, 274, 281, 248, 255, 249, 247, 276, 216, 249, 251, 242, 249, 293, 269, 253, 234, 258, 271, 239, 258, 257, 276, 247, 244, 239, 267, 281, 258, 249, 252, 263, 241, 274, 282, 250, 263, 238, 249, 261, 285, 247, 254, 276, 282, 257, 274, 243, 255, 266, 250, 254, 263, 251, 239] payload_rows_per_dst=[5474, 5530, 5444, 5390, 5400, 5390, 5440, 5540] dup_rows_per_dst=[2624, 2802, 2754, 2586, 2842, 2792, 2754, 2774] combine_rows_per_src=[5474, 5530, 5444, 5390, 5400, 5390, 5440, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43540 payload_rows_remote=38128 dup_rows=21996 dup_rows_remote=19220 dedup_ratio=0.335632 router_tokens_per_expert=[249, 250, 272, 260, 248, 260, 255, 248, 246, 275, 244, 248, 244, 264, 281, 259, 272, 244, 279, 263, 260, 241, 260, 246, 249, 236, 219, 292, 240, 264, 262, 267, 261, 251, 267, 240, 288, 241, 295, 255, 258, 250, 254, 245, 266, 248, 263, 284, 265, 285, 286, 236, 244, 266, 273, 252, 250, 243, 253, 252, 259, 235, 261, 234, 267, 277, 262, 278, 253, 253, 262, 259, 246, 242, 235, 273, 252, 259, 253, 258, 260, 246, 251, 241, 259, 242, 239, 263, 252, 247, 249, 259, 262, 233, 251, 263, 247, 254, 256, 238, 251, 258, 254, 244, 249, 263, 253, 260, 285, 265, 279, 244, 232, 251, 245, 265, 260, 239, 239, 253, 279, 249, 251, 271, 260, 253, 247, 271] payload_rows_per_dst=[5490, 5412, 5510, 5366, 5482, 5370, 5460, 5450] dup_rows_per_dst=[2716, 2776, 2822, 2822, 2776, 2664, 2740, 2680] combine_rows_per_src=[5490, 5412, 5510, 5366, 5482, 5370, 5460, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38218 dup_rows=21832 dup_rows_remote=19160 dedup_ratio=0.333130 router_tokens_per_expert=[252, 249, 256, 240, 284, 268, 269, 255, 258, 225, 266, 227, 239, 270, 281, 268, 254, 261, 243, 229, 272, 243, 267, 243, 224, 276, 262, 277, 269, 253, 245, 207, 253, 238, 259, 270, 250, 255, 223, 261, 274, 236, 268, 248, 261, 277, 246, 276, 253, 259, 255, 253, 266, 263, 266, 231, 248, 269, 238, 253, 265, 268, 269, 263, 217, 231, 258, 254, 271, 282, 263, 254, 270, 290, 237, 247, 264, 248, 235, 258, 271, 274, 237, 246, 272, 247, 298, 244, 260, 233, 243, 261, 267, 258, 267, 282, 251, 262, 243, 250, 253, 250, 234, 240, 275, 265, 302, 264, 279, 272, 252, 283, 278, 260, 225, 235, 274, 232, 249, 260, 271, 249, 234, 260, 255, 251, 247, 228] payload_rows_per_dst=[5426, 5394, 5480, 5512, 5486, 5486, 5492, 5428] dup_rows_per_dst=[2788, 2656, 2710, 2726, 2672, 2834, 2858, 2588] combine_rows_per_src=[5426, 5394, 5480, 5512, 5486, 5486, 5492, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38456 dup_rows=21646 dup_rows_remote=18924 dedup_ratio=0.330292 router_tokens_per_expert=[256, 256, 278, 247, 266, 256, 254, 269, 263, 270, 251, 235, 246, 258, 255, 249, 258, 228, 247, 249, 285, 259, 257, 309, 246, 269, 261, 270, 275, 248, 293, 263, 258, 268, 261, 240, 236, 250, 236, 242, 270, 254, 270, 236, 283, 262, 261, 263, 228, 258, 257, 270, 254, 287, 234, 252, 244, 242, 252, 223, 265, 273, 273, 265, 252, 233, 234, 246, 243, 254, 263, 221, 267, 250, 254, 244, 245, 277, 257, 265, 258, 238, 246, 243, 249, 251, 282, 261, 235, 249, 236, 248, 282, 252, 263, 285, 262, 260, 253, 266, 257, 290, 262, 250, 238, 236, 266, 244, 288, 240, 285, 255, 224, 249, 248, 238, 270, 252, 235, 272, 258, 264, 292, 244, 251, 258, 245, 240] payload_rows_per_dst=[5520, 5570, 5478, 5474, 5474, 5434, 5536, 5404] dup_rows_per_dst=[2698, 2864, 2702, 2680, 2536, 2722, 2768, 2676] combine_rows_per_src=[5520, 5570, 5478, 5474, 5474, 5434, 5536, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38176 dup_rows=21888 dup_rows_remote=19128 dedup_ratio=0.333984 router_tokens_per_expert=[251, 274, 244, 243, 269, 270, 230, 274, 245, 249, 264, 247, 268, 260, 267, 279, 259, 250, 253, 264, 266, 245, 256, 280, 256, 250, 260, 266, 242, 276, 270, 271, 282, 281, 219, 266, 258, 260, 271, 266, 272, 237, 246, 252, 287, 247, 256, 253, 225, 266, 245, 267, 258, 234, 259, 248, 244, 251, 288, 261, 276, 263, 261, 250, 239, 264, 253, 251, 255, 227, 241, 252, 237, 265, 274, 273, 240, 240, 265, 267, 221, 254, 277, 236, 256, 265, 257, 244, 239, 254, 260, 220, 267, 232, 243, 273, 261, 252, 262, 270, 264, 225, 281, 227, 272, 278, 225, 229, 259, 257, 251, 267, 245, 245, 253, 270, 259, 278, 232, 247, 278, 254, 242, 271, 247, 283, 228, 268] payload_rows_per_dst=[5378, 5288, 5560, 5556, 5452, 5454, 5488, 5472] dup_rows_per_dst=[2820, 2572, 2746, 2882, 2826, 2550, 2732, 2760] combine_rows_per_src=[5378, 5288, 5560, 5556, 5452, 5454, 5488, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38074 dup_rows=21958 dup_rows_remote=19168 dedup_ratio=0.335052 router_tokens_per_expert=[236, 261, 260, 245, 252, 246, 260, 252, 261, 287, 264, 248, 281, 237, 257, 264, 237, 232, 270, 253, 249, 244, 271, 275, 234, 256, 255, 283, 241, 258, 269, 273, 231, 248, 263, 286, 263, 259, 234, 279, 243, 245, 254, 271, 258, 263, 246, 238, 282, 257, 241, 267, 259, 274, 254, 272, 245, 261, 256, 240, 267, 245, 253, 274, 263, 258, 260, 262, 255, 264, 245, 260, 239, 268, 243, 265, 261, 245, 229, 249, 251, 271, 246, 236, 268, 287, 270, 255, 237, 247, 211, 273, 254, 260, 270, 224, 251, 259, 264, 225, 257, 256, 290, 232, 274, 222, 249, 272, 255, 221, 240, 261, 265, 264, 226, 283, 282, 275, 273, 275, 257, 251, 261, 256, 271, 245, 255, 236] payload_rows_per_dst=[5432, 5450, 5374, 5504, 5410, 5488, 5328, 5592] dup_rows_per_dst=[2790, 2750, 2788, 2790, 2722, 2632, 2728, 2758] combine_rows_per_src=[5432, 5450, 5374, 5504, 5410, 5488, 5328, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38430 dup_rows=21740 dup_rows_remote=18966 dedup_ratio=0.331726 router_tokens_per_expert=[261, 233, 234, 279, 255, 265, 247, 265, 238, 256, 238, 280, 266, 245, 247, 247, 269, 259, 260, 256, 243, 251, 261, 272, 233, 274, 286, 270, 251, 284, 266, 256, 233, 256, 251, 262, 270, 238, 272, 265, 240, 266, 229, 231, 276, 229, 248, 248, 278, 285, 237, 252, 283, 238, 213, 236, 262, 247, 265, 233, 250, 292, 249, 245, 258, 263, 262, 256, 279, 270, 260, 280, 237, 255, 269, 233, 254, 245, 244, 247, 268, 297, 254, 248, 263, 260, 242, 250, 284, 244, 243, 267, 269, 256, 274, 276, 255, 228, 254, 266, 241, 255, 260, 273, 256, 246, 255, 284, 234, 272, 241, 250, 260, 249, 270, 235, 253, 267, 253, 272, 217, 240, 238, 265, 267, 251, 252, 276] payload_rows_per_dst=[5422, 5586, 5472, 5456, 5468, 5622, 5366, 5404] dup_rows_per_dst=[2690, 2796, 2556, 2674, 2756, 2768, 2774, 2726] combine_rows_per_src=[5422, 5586, 5472, 5456, 5468, 5622, 5366, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=229352 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156549096 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312868840 weight_movement_bytes=18874368 total_nvlink_bytes=331743208 weight_to_token_ratio=0.060327 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156172288 dispatch_meta_bytes=229392 combine_payload_bytes=156172288 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156401680 ingress_bytes=231669760 link_bytes=231669760 token_bytes=312573968 weight_movement_bytes=75497472 total_nvlink_bytes=388071440 weight_to_token_ratio=0.241535 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219745652e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=229512 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156770440 ingress_bytes=175415296 link_bytes=175415296 token_bytes=313311368 weight_movement_bytes=18874368 total_nvlink_bytes=332185736 weight_to_token_ratio=0.060242 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680743057e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157515776 dispatch_meta_bytes=229520 combine_payload_bytes=157515776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157745296 ingress_bytes=176390144 link_bytes=176390144 token_bytes=315261072 weight_movement_bytes=18874368 total_nvlink_bytes=334135440 weight_to_token_ratio=0.059869 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690083570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229216 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156598112 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967008 weight_movement_bytes=18874368 total_nvlink_bytes=331841376 weight_to_token_ratio=0.060308 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155951104 dispatch_meta_bytes=228968 combine_payload_bytes=155951104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156180072 ingress_bytes=174825472 link_bytes=174825472 token_bytes=312131176 weight_movement_bytes=18874368 total_nvlink_bytes=331005544 weight_to_token_ratio=0.060469 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675091653e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229584 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157638864 ingress_bytes=195158016 link_bytes=195158016 token_bytes=315048144 weight_movement_bytes=37748736 total_nvlink_bytes=352796880 weight_to_token_ratio=0.119819 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869908086e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38382 dup_rows=21792 dup_rows_remote=19192 dedup_ratio=0.332520 router_tokens_per_expert=[277, 257, 271, 267, 237, 246, 249, 271, 243, 262, 265, 273, 247, 252, 250, 264, 252, 252, 237, 242, 273, 273, 283, 259, 250, 283, 262, 244, 265, 276, 261, 277, 251, 235, 237, 239, 241, 269, 257, 261, 246, 236, 245, 251, 260, 240, 255, 258, 277, 266, 245, 286, 240, 263, 266, 233, 243, 243, 267, 254, 250, 275, 243, 279, 264, 253, 257, 246, 267, 269, 247, 254, 263, 252, 249, 236, 257, 261, 258, 228, 278, 267, 263, 270, 239, 231, 288, 232, 264, 251, 253, 270, 253, 267, 271, 265, 258, 248, 254, 242, 266, 264, 268, 266, 236, 266, 252, 253, 254, 259, 283, 242, 230, 248, 254, 258, 264, 251, 245, 237, 244, 236, 276, 231, 276, 247, 263, 243] payload_rows_per_dst=[5506, 5572, 5362, 5504, 5398, 5536, 5492, 5374] dup_rows_per_dst=[2756, 2806, 2600, 2756, 2724, 2788, 2730, 2632] combine_rows_per_src=[5506, 5572, 5362, 5504, 5398, 5536, 5492, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43494 payload_rows_remote=38010 dup_rows=22042 dup_rows_remote=19266 dedup_ratio=0.336334 router_tokens_per_expert=[253, 260, 237, 241, 276, 266, 240, 258, 278, 271, 235, 222, 245, 248, 267, 265, 274, 253, 260, 251, 275, 250, 238, 257, 244, 257, 298, 258, 238, 257, 271, 249, 232, 254, 219, 244, 272, 290, 265, 263, 267, 257, 259, 252, 259, 226, 229, 232, 272, 269, 244, 260, 270, 231, 258, 256, 236, 272, 239, 285, 249, 250, 225, 251, 247, 289, 271, 238, 303, 281, 229, 240, 216, 242, 249, 257, 239, 277, 246, 244, 249, 267, 260, 251, 258, 265, 267, 275, 248, 234, 266, 261, 262, 274, 264, 243, 237, 266, 261, 276, 269, 254, 262, 231, 261, 242, 286, 249, 251, 258, 289, 284, 260, 234, 246, 230, 263, 278, 273, 247, 231, 259, 261, 260, 270, 247, 271, 271] payload_rows_per_dst=[5418, 5484, 5374, 5420, 5406, 5514, 5458, 5420] dup_rows_per_dst=[2706, 2776, 2766, 2714, 2730, 2774, 2794, 2782] combine_rows_per_src=[5418, 5484, 5374, 5420, 5406, 5514, 5458, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43536 payload_rows_remote=38160 dup_rows=22000 dup_rows_remote=19294 dedup_ratio=0.335693 router_tokens_per_expert=[241, 248, 250, 280, 229, 271, 245, 253, 235, 258, 263, 254, 254, 262, 281, 231, 266, 237, 246, 276, 265, 272, 257, 265, 285, 250, 240, 230, 246, 265, 269, 242, 221, 271, 255, 250, 245, 277, 273, 254, 262, 253, 282, 273, 256, 237, 262, 238, 235, 263, 240, 273, 250, 274, 264, 249, 282, 233, 269, 273, 247, 267, 245, 268, 283, 230, 251, 246, 282, 228, 264, 271, 247, 213, 250, 247, 256, 242, 280, 251, 261, 272, 264, 239, 262, 253, 224, 218, 248, 269, 290, 274, 252, 291, 277, 240, 265, 242, 225, 245, 265, 241, 281, 274, 257, 260, 251, 279, 259, 264, 238, 243, 228, 263, 269, 260, 249, 245, 273, 262, 258, 256, 257, 250, 248, 251, 259, 269] payload_rows_per_dst=[5386, 5524, 5388, 5420, 5376, 5442, 5460, 5540] dup_rows_per_dst=[2724, 2698, 2830, 2844, 2706, 2826, 2718, 2654] combine_rows_per_src=[5386, 5524, 5388, 5420, 5376, 5442, 5460, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38388 dup_rows=21736 dup_rows_remote=19030 dedup_ratio=0.331665 router_tokens_per_expert=[276, 267, 274, 237, 287, 254, 280, 250, 260, 269, 266, 266, 238, 241, 251, 262, 250, 272, 249, 277, 238, 258, 251, 248, 259, 252, 276, 281, 258, 273, 252, 280, 245, 231, 260, 268, 268, 232, 279, 246, 249, 263, 228, 254, 249, 231, 261, 271, 262, 264, 262, 237, 246, 243, 258, 266, 251, 274, 276, 254, 241, 237, 280, 265, 265, 271, 252, 251, 245, 256, 242, 262, 216, 235, 233, 235, 270, 258, 233, 229, 271, 251, 251, 258, 270, 250, 263, 256, 247, 273, 262, 229, 246, 242, 244, 246, 263, 242, 259, 249, 250, 272, 239, 275, 274, 260, 279, 267, 268, 271, 256, 271, 230, 256, 274, 232, 251, 237, 257, 245, 263, 275, 256, 248, 275, 249, 260, 250] payload_rows_per_dst=[5490, 5502, 5398, 5506, 5388, 5412, 5654, 5450] dup_rows_per_dst=[2866, 2846, 2672, 2726, 2518, 2706, 2736, 2666] combine_rows_per_src=[5490, 5502, 5398, 5506, 5388, 5412, 5654, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=37946 dup_rows=21948 dup_rows_remote=18980 dedup_ratio=0.334900 router_tokens_per_expert=[273, 268, 238, 246, 270, 241, 266, 267, 248, 258, 244, 274, 267, 252, 234, 259, 265, 277, 271, 240, 239, 253, 263, 269, 251, 280, 227, 268, 282, 271, 264, 248, 246, 271, 271, 247, 233, 257, 228, 273, 248, 278, 276, 253, 240, 259, 257, 253, 272, 240, 263, 261, 286, 251, 249, 227, 225, 256, 260, 249, 254, 267, 261, 261, 267, 243, 280, 259, 252, 252, 264, 209, 234, 274, 250, 252, 254, 266, 276, 246, 267, 282, 262, 266, 257, 249, 255, 242, 247, 219, 259, 247, 281, 268, 249, 266, 261, 247, 256, 264, 258, 234, 267, 256, 261, 271, 242, 243, 236, 237, 260, 260, 272, 253, 249, 250, 261, 244, 264, 282, 273, 235, 254, 245, 235, 246, 246, 267] payload_rows_per_dst=[5380, 5266, 5446, 5382, 5602, 5382, 5488, 5642] dup_rows_per_dst=[2560, 2448, 2734, 2782, 3096, 2592, 2768, 2968] combine_rows_per_src=[5380, 5266, 5446, 5382, 5602, 5382, 5488, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=230296 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442968 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314655640 weight_movement_bytes=18874368 total_nvlink_bytes=333530008 weight_to_token_ratio=0.059984 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155688960 dispatch_meta_bytes=229104 combine_payload_bytes=155688960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=155918064 ingress_bytes=212312064 link_bytes=212312064 token_bytes=311607024 weight_movement_bytes=56623104 total_nvlink_bytes=368230128 weight_to_token_ratio=0.181713 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.034269733e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38338 dup_rows=21800 dup_rows_remote=19180 dedup_ratio=0.332642 router_tokens_per_expert=[269, 247, 274, 271, 262, 252, 250, 277, 268, 245, 249, 262, 278, 239, 261, 228, 273, 248, 279, 268, 250, 270, 249, 276, 276, 253, 272, 247, 230, 260, 264, 258, 272, 237, 249, 249, 257, 261, 251, 237, 259, 273, 243, 258, 234, 258, 243, 258, 247, 225, 252, 249, 264, 264, 230, 259, 245, 261, 261, 261, 254, 244, 248, 245, 233, 264, 255, 241, 222, 255, 257, 264, 257, 254, 267, 289, 265, 250, 250, 229, 233, 267, 259, 281, 259, 220, 282, 245, 269, 240, 248, 260, 261, 251, 297, 263, 323, 278, 250, 265, 275, 251, 257, 265, 239, 251, 243, 244, 240, 277, 249, 255, 260, 234, 240, 260, 254, 239, 253, 279, 254, 253, 259, 253, 269, 226, 269, 264] payload_rows_per_dst=[5518, 5526, 5406, 5398, 5448, 5462, 5534, 5444] dup_rows_per_dst=[2746, 2820, 2672, 2620, 2656, 2808, 2790, 2688] combine_rows_per_src=[5518, 5526, 5406, 5398, 5448, 5462, 5534, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43898 payload_rows_remote=38418 dup_rows=21638 dup_rows_remote=18834 dedup_ratio=0.330170 router_tokens_per_expert=[254, 279, 241, 255, 267, 264, 247, 251, 247, 248, 266, 257, 269, 257, 250, 265, 274, 236, 268, 221, 265, 228, 240, 244, 249, 223, 259, 248, 260, 250, 279, 251, 226, 264, 268, 253, 253, 261, 277, 276, 249, 256, 271, 268, 258, 258, 237, 224, 263, 239, 265, 242, 265, 298, 266, 282, 277, 261, 257, 262, 250, 244, 232, 230, 246, 239, 251, 244, 261, 283, 263, 274, 229, 295, 268, 245, 247, 248, 254, 270, 249, 263, 276, 251, 257, 276, 261, 241, 280, 274, 249, 239, 246, 251, 242, 221, 282, 249, 220, 272, 269, 235, 290, 257, 258, 287, 245, 276, 258, 244, 247, 253, 262, 239, 270, 234, 252, 253, 276, 240, 253, 271, 255, 263, 263, 254, 247, 257] payload_rows_per_dst=[5506, 5334, 5458, 5552, 5624, 5508, 5480, 5436] dup_rows_per_dst=[2728, 2510, 2740, 2714, 2756, 2644, 2804, 2742] combine_rows_per_src=[5506, 5334, 5458, 5552, 5624, 5508, 5480, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=229816 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156533176 ingress_bytes=175177728 link_bytes=175177728 token_bytes=312836536 weight_movement_bytes=18874368 total_nvlink_bytes=331710904 weight_to_token_ratio=0.060333 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678466797e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=229672 combine_payload_bytes=157237248 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157466920 ingress_bytes=194985984 link_bytes=194985984 token_bytes=314704168 weight_movement_bytes=37748736 total_nvlink_bytes=352452904 weight_to_token_ratio=0.119950 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868259760e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155426816 dispatch_meta_bytes=227704 combine_payload_bytes=155426816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155654520 ingress_bytes=174301184 link_bytes=174301184 token_bytes=311081336 weight_movement_bytes=18874368 total_nvlink_bytes=329955704 weight_to_token_ratio=0.060673 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670068184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=230072 combine_payload_bytes=157032448 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157262520 ingress_bytes=175906816 link_bytes=175906816 token_bytes=314294968 weight_movement_bytes=18874368 total_nvlink_bytes=333169336 weight_to_token_ratio=0.060053 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685452559e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229008 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157589136 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314949264 weight_movement_bytes=37748736 total_nvlink_bytes=352698000 weight_to_token_ratio=0.119857 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38264 dup_rows=21916 dup_rows_remote=19158 dedup_ratio=0.334412 router_tokens_per_expert=[277, 264, 226, 280, 232, 255, 272, 258, 259, 254, 232, 265, 233, 273, 249, 262, 259, 257, 247, 256, 262, 224, 280, 273, 252, 266, 270, 237, 239, 263, 256, 265, 258, 251, 266, 226, 237, 256, 263, 275, 240, 260, 257, 263, 277, 239, 255, 234, 278, 232, 272, 245, 252, 247, 244, 246, 248, 276, 263, 258, 254, 282, 274, 250, 258, 244, 237, 258, 242, 255, 265, 271, 272, 291, 261, 277, 224, 275, 265, 257, 219, 236, 247, 268, 274, 237, 258, 258, 266, 239, 274, 274, 254, 252, 251, 274, 244, 255, 292, 266, 255, 250, 261, 257, 257, 264, 245, 273, 266, 258, 255, 277, 248, 263, 242, 272, 242, 258, 244, 280, 243, 274, 272, 224, 209, 262, 226, 226] payload_rows_per_dst=[5406, 5418, 5356, 5510, 5474, 5516, 5554, 5386] dup_rows_per_dst=[2776, 2794, 2758, 2732, 2830, 2646, 2796, 2584] combine_rows_per_src=[5406, 5418, 5356, 5510, 5474, 5516, 5554, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38056 dup_rows=21942 dup_rows_remote=19212 dedup_ratio=0.334808 router_tokens_per_expert=[282, 286, 251, 263, 266, 265, 249, 239, 263, 270, 264, 273, 248, 250, 248, 217, 243, 246, 272, 264, 235, 254, 259, 278, 259, 248, 243, 259, 257, 259, 229, 240, 250, 254, 251, 240, 229, 287, 253, 248, 241, 262, 241, 242, 226, 251, 270, 244, 279, 236, 269, 243, 281, 227, 263, 266, 269, 251, 234, 255, 241, 257, 240, 248, 264, 250, 280, 265, 229, 265, 271, 238, 243, 273, 248, 261, 288, 243, 244, 255, 260, 244, 267, 248, 254, 246, 256, 261, 241, 280, 261, 263, 275, 279, 260, 270, 259, 262, 265, 239, 239, 269, 266, 274, 241, 254, 266, 243, 258, 242, 258, 282, 289, 261, 253, 250, 283, 284, 269, 275, 265, 243, 265, 243, 265, 233, 235, 229] payload_rows_per_dst=[5538, 5402, 5426, 5414, 5430, 5512, 5356, 5516] dup_rows_per_dst=[2730, 2688, 2714, 2704, 2804, 2818, 2716, 2768] combine_rows_per_src=[5538, 5402, 5426, 5414, 5430, 5512, 5356, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38464 dup_rows=21664 dup_rows_remote=18984 dedup_ratio=0.330566 router_tokens_per_expert=[288, 246, 261, 244, 277, 252, 275, 248, 261, 237, 274, 272, 273, 246, 265, 221, 229, 260, 227, 264, 259, 257, 245, 249, 266, 256, 263, 249, 247, 269, 237, 249, 257, 233, 261, 249, 264, 249, 236, 242, 246, 248, 299, 257, 268, 243, 272, 280, 256, 245, 239, 258, 289, 257, 265, 266, 241, 252, 280, 249, 262, 255, 240, 227, 248, 237, 249, 261, 264, 243, 280, 248, 258, 260, 254, 250, 263, 258, 233, 274, 256, 265, 249, 281, 264, 260, 249, 264, 235, 269, 236, 249, 243, 267, 263, 220, 263, 248, 298, 247, 279, 231, 256, 270, 247, 244, 295, 264, 255, 265, 249, 246, 259, 252, 244, 282, 263, 262, 246, 254, 282, 282, 259, 244, 248, 244, 260, 229] payload_rows_per_dst=[5546, 5408, 5504, 5476, 5452, 5486, 5494, 5506] dup_rows_per_dst=[2734, 2680, 2704, 2686, 2708, 2654, 2784, 2714] combine_rows_per_src=[5546, 5408, 5504, 5476, 5452, 5486, 5494, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229688 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156959032 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313688376 weight_movement_bytes=18874368 total_nvlink_bytes=332562744 weight_to_token_ratio=0.060169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38298 dup_rows=21640 dup_rows_remote=18736 dedup_ratio=0.330200 router_tokens_per_expert=[250, 246, 234, 255, 249, 239, 239, 245, 262, 248, 240, 267, 254, 239, 244, 260, 266, 263, 249, 272, 260, 253, 253, 261, 230, 256, 241, 254, 251, 224, 231, 230, 280, 234, 272, 257, 243, 270, 277, 281, 259, 266, 278, 277, 247, 250, 276, 238, 263, 252, 247, 264, 288, 273, 257, 271, 247, 253, 218, 260, 252, 271, 288, 257, 208, 236, 261, 265, 246, 250, 252, 268, 277, 244, 246, 264, 250, 287, 272, 258, 276, 243, 230, 221, 241, 278, 242, 247, 250, 265, 247, 254, 265, 257, 245, 271, 286, 269, 253, 245, 246, 273, 254, 238, 274, 227, 281, 262, 237, 269, 230, 247, 260, 244, 284, 258, 289, 250, 268, 263, 293, 285, 280, 258, 252, 250, 254, 242] payload_rows_per_dst=[5326, 5458, 5618, 5506, 5488, 5430, 5472, 5598] dup_rows_per_dst=[2616, 2530, 2792, 2816, 2680, 2558, 2744, 2904] combine_rows_per_src=[5326, 5458, 5618, 5506, 5488, 5430, 5472, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38090 dup_rows=21906 dup_rows_remote=19210 dedup_ratio=0.334259 router_tokens_per_expert=[248, 246, 266, 256, 275, 235, 263, 257, 270, 260, 258, 241, 270, 258, 239, 241, 255, 269, 268, 251, 244, 279, 274, 258, 231, 246, 272, 227, 257, 274, 262, 248, 253, 249, 276, 249, 234, 265, 228, 251, 230, 254, 278, 271, 259, 243, 263, 252, 266, 259, 270, 250, 236, 265, 272, 258, 260, 250, 265, 225, 255, 247, 281, 261, 263, 268, 267, 247, 251, 272, 251, 273, 227, 251, 257, 269, 259, 255, 262, 251, 254, 266, 256, 274, 267, 254, 275, 259, 235, 243, 244, 252, 250, 255, 264, 270, 272, 268, 256, 231, 228, 278, 285, 246, 254, 257, 257, 242, 259, 234, 266, 268, 277, 237, 256, 238, 247, 256, 257, 248, 232, 252, 240, 283, 238, 265, 260, 267] payload_rows_per_dst=[5454, 5508, 5346, 5466, 5458, 5540, 5412, 5446] dup_rows_per_dst=[2712, 2722, 2764, 2774, 2788, 2696, 2790, 2660] combine_rows_per_src=[5454, 5508, 5346, 5466, 5458, 5540, 5412, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38222 dup_rows=21862 dup_rows_remote=19188 dedup_ratio=0.333588 router_tokens_per_expert=[250, 276, 229, 262, 257, 266, 278, 270, 249, 241, 249, 253, 250, 262, 250, 257, 263, 264, 246, 251, 252, 238, 243, 253, 226, 259, 240, 247, 227, 272, 242, 308, 255, 265, 282, 247, 257, 272, 273, 255, 260, 272, 230, 272, 260, 254, 257, 253, 253, 228, 241, 217, 253, 244, 250, 255, 231, 244, 263, 254, 249, 228, 241, 237, 276, 248, 243, 260, 242, 268, 251, 272, 263, 228, 254, 257, 239, 261, 254, 247, 285, 263, 240, 258, 244, 256, 262, 273, 235, 267, 233, 265, 272, 275, 270, 271, 272, 251, 279, 257, 272, 275, 252, 266, 240, 274, 264, 256, 232, 255, 259, 247, 276, 247, 231, 255, 270, 243, 269, 273, 250, 234, 275, 261, 292, 268, 272, 287] payload_rows_per_dst=[5466, 5364, 5444, 5366, 5452, 5520, 5514, 5548] dup_rows_per_dst=[2732, 2698, 2884, 2410, 2674, 2818, 2788, 2858] combine_rows_per_src=[5466, 5364, 5444, 5366, 5452, 5520, 5514, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=229792 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157778336 ingress_bytes=176422912 link_bytes=176422912 token_bytes=315326880 weight_movement_bytes=18874368 total_nvlink_bytes=334201248 weight_to_token_ratio=0.059857 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690397537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=228136 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157096744 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313965352 weight_movement_bytes=18874368 total_nvlink_bytes=332839720 weight_to_token_ratio=0.060116 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43802 payload_rows_remote=38246 dup_rows=21734 dup_rows_remote=18972 dedup_ratio=0.331635 router_tokens_per_expert=[261, 287, 258, 250, 261, 242, 275, 237, 232, 243, 264, 283, 288, 277, 275, 268, 266, 248, 268, 240, 246, 270, 248, 243, 238, 256, 241, 279, 245, 256, 238, 248, 273, 241, 246, 230, 259, 252, 252, 268, 251, 274, 234, 271, 244, 247, 271, 275, 249, 262, 270, 267, 304, 244, 223, 243, 289, 262, 250, 241, 260, 234, 258, 225, 277, 231, 244, 208, 231, 249, 253, 265, 249, 270, 235, 264, 258, 253, 258, 252, 280, 260, 227, 261, 269, 241, 273, 253, 247, 236, 259, 267, 261, 247, 234, 277, 256, 268, 236, 249, 275, 253, 264, 271, 254, 257, 283, 274, 266, 263, 234, 256, 289, 255, 260, 267, 256, 251, 263, 253, 267, 247, 257, 266, 241, 277, 228, 243] payload_rows_per_dst=[5564, 5388, 5468, 5460, 5334, 5534, 5556, 5498] dup_rows_per_dst=[2838, 2672, 2708, 2702, 2660, 2650, 2762, 2742] combine_rows_per_src=[5564, 5388, 5468, 5460, 5334, 5534, 5556, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38164 dup_rows=21768 dup_rows_remote=18912 dedup_ratio=0.332153 router_tokens_per_expert=[254, 251, 241, 260, 266, 301, 262, 242, 253, 231, 263, 248, 243, 240, 261, 236, 224, 269, 255, 253, 270, 263, 272, 260, 284, 264, 250, 254, 246, 243, 257, 249, 204, 243, 246, 230, 258, 260, 238, 243, 280, 236, 272, 266, 237, 253, 263, 270, 252, 261, 252, 274, 247, 283, 279, 258, 252, 282, 256, 286, 294, 263, 241, 250, 234, 259, 253, 244, 256, 240, 238, 256, 233, 257, 244, 265, 259, 282, 275, 255, 269, 254, 273, 224, 248, 293, 229, 219, 274, 271, 249, 247, 273, 245, 276, 255, 245, 255, 271, 249, 265, 269, 291, 253, 245, 261, 268, 255, 236, 265, 270, 261, 271, 238, 260, 256, 232, 271, 254, 251, 267, 246, 236, 258, 262, 238, 244, 282] payload_rows_per_dst=[5416, 5504, 5408, 5604, 5476, 5478, 5504, 5378] dup_rows_per_dst=[2688, 2722, 2590, 2856, 2624, 2720, 2814, 2754] combine_rows_per_src=[5416, 5504, 5408, 5604, 5476, 5478, 5504, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156016640 dispatch_meta_bytes=229200 combine_payload_bytes=156016640 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156245840 ingress_bytes=250388480 link_bytes=250388480 token_bytes=312262480 weight_movement_bytes=94371840 total_nvlink_bytes=406634320 weight_to_token_ratio=0.302220 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.399099217e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=155877376 dispatch_meta_bytes=229072 combine_payload_bytes=155877376 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156106448 ingress_bytes=212500480 link_bytes=212500480 token_bytes=311983824 weight_movement_bytes=56623104 total_nvlink_bytes=368606928 weight_to_token_ratio=0.181494 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036075043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229640 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786952 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313344264 weight_movement_bytes=18874368 total_nvlink_bytes=332218632 weight_to_token_ratio=0.060235 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38398 dup_rows=21726 dup_rows_remote=18990 dedup_ratio=0.331512 router_tokens_per_expert=[241, 254, 242, 255, 261, 270, 249, 261, 278, 246, 221, 237, 221, 270, 252, 260, 280, 289, 250, 269, 248, 260, 258, 253, 217, 258, 251, 275, 253, 259, 261, 255, 245, 276, 237, 265, 218, 276, 262, 224, 237, 258, 242, 275, 277, 281, 258, 243, 271, 233, 278, 268, 289, 267, 254, 260, 249, 290, 270, 256, 246, 254, 252, 245, 280, 246, 281, 269, 255, 262, 255, 239, 258, 275, 267, 232, 248, 263, 239, 231, 270, 239, 281, 244, 250, 277, 249, 258, 220, 239, 267, 263, 240, 241, 246, 271, 245, 253, 258, 258, 257, 260, 261, 251, 256, 249, 232, 276, 264, 244, 255, 229, 287, 255, 268, 245, 259, 240, 290, 289, 270, 231, 238, 249, 254, 218, 289, 273] payload_rows_per_dst=[5324, 5570, 5412, 5532, 5482, 5464, 5422, 5604] dup_rows_per_dst=[2712, 2702, 2736, 2832, 2718, 2646, 2674, 2706] combine_rows_per_src=[5324, 5570, 5412, 5532, 5482, 5464, 5422, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=228872 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156884488 ingress_bytes=213278720 link_bytes=213278720 token_bytes=313540104 weight_movement_bytes=56623104 total_nvlink_bytes=370163208 weight_to_token_ratio=0.180593 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043531755e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156319744 dispatch_meta_bytes=228304 combine_payload_bytes=156319744 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156548048 ingress_bytes=175194112 link_bytes=175194112 token_bytes=312867792 weight_movement_bytes=18874368 total_nvlink_bytes=331742160 weight_to_token_ratio=0.060327 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678623780e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229552 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507760 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314785968 weight_movement_bytes=18874368 total_nvlink_bytes=333660336 weight_to_token_ratio=0.059959 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43976 payload_rows_remote=38414 dup_rows=21560 dup_rows_remote=18798 dedup_ratio=0.328979 router_tokens_per_expert=[260, 260, 262, 259, 285, 263, 289, 245, 259, 271, 267, 236, 239, 261, 240, 241, 247, 267, 267, 287, 251, 254, 269, 256, 270, 263, 245, 254, 276, 266, 241, 249, 261, 248, 248, 273, 233, 277, 255, 240, 226, 202, 268, 261, 242, 277, 270, 260, 247, 281, 277, 268, 263, 237, 245, 257, 271, 250, 262, 244, 245, 235, 253, 253, 264, 250, 273, 239, 238, 265, 246, 239, 263, 226, 258, 251, 251, 274, 268, 249, 267, 279, 261, 263, 257, 261, 249, 237, 268, 276, 247, 261, 275, 263, 257, 235, 273, 256, 243, 290, 280, 228, 247, 237, 270, 254, 244, 274, 261, 243, 233, 229, 260, 259, 241, 252, 269, 276, 245, 250, 266, 245, 260, 228, 260, 256, 247, 254] payload_rows_per_dst=[5524, 5562, 5452, 5440, 5468, 5596, 5498, 5436] dup_rows_per_dst=[2750, 2762, 2630, 2736, 2640, 2716, 2626, 2700] combine_rows_per_src=[5524, 5562, 5452, 5440, 5468, 5596, 5498, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38072 dup_rows=21944 dup_rows_remote=19158 dedup_ratio=0.334839 router_tokens_per_expert=[267, 245, 223, 276, 255, 251, 253, 257, 221, 235, 256, 279, 268, 255, 260, 274, 273, 252, 249, 244, 266, 280, 254, 260, 250, 280, 262, 254, 293, 276, 276, 274, 269, 228, 244, 262, 260, 242, 277, 239, 263, 262, 226, 242, 282, 272, 260, 232, 280, 256, 264, 270, 248, 244, 235, 254, 250, 252, 274, 271, 262, 253, 239, 253, 270, 245, 279, 250, 263, 232, 255, 270, 273, 245, 263, 267, 259, 256, 243, 267, 234, 230, 240, 245, 265, 250, 259, 238, 275, 265, 271, 269, 283, 231, 235, 256, 256, 238, 263, 281, 245, 231, 264, 244, 215, 265, 256, 264, 257, 261, 254, 273, 248, 229, 269, 245, 230, 275, 248, 279, 274, 243, 219, 253, 240, 282, 228, 273] payload_rows_per_dst=[5526, 5302, 5464, 5350, 5448, 5386, 5596, 5520] dup_rows_per_dst=[2568, 2500, 2832, 2628, 2866, 2718, 3046, 2786] combine_rows_per_src=[5526, 5302, 5464, 5350, 5448, 5386, 5596, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43478 payload_rows_remote=38176 dup_rows=22058 dup_rows_remote=19360 dedup_ratio=0.336578 router_tokens_per_expert=[225, 243, 257, 247, 231, 248, 250, 279, 281, 261, 263, 284, 279, 279, 265, 241, 255, 255, 262, 254, 241, 265, 269, 261, 281, 227, 269, 272, 270, 241, 258, 300, 249, 257, 262, 258, 264, 255, 253, 258, 249, 230, 226, 248, 256, 247, 275, 262, 269, 257, 272, 277, 256, 247, 280, 253, 228, 230, 246, 232, 244, 255, 263, 270, 270, 239, 254, 263, 248, 269, 265, 244, 265, 246, 256, 265, 258, 248, 265, 242, 256, 233, 221, 254, 252, 273, 275, 261, 227, 254, 259, 240, 247, 242, 256, 250, 252, 256, 258, 245, 274, 267, 268, 227, 249, 282, 248, 266, 249, 257, 248, 257, 240, 264, 263, 245, 254, 262, 264, 277, 282, 243, 258, 256, 241, 252, 271, 255] payload_rows_per_dst=[5476, 5538, 5408, 5348, 5494, 5302, 5438, 5474] dup_rows_per_dst=[2790, 2822, 2690, 2810, 2700, 2698, 2768, 2780] combine_rows_per_src=[5476, 5538, 5408, 5348, 5494, 5302, 5438, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157343744 dispatch_meta_bytes=228848 combine_payload_bytes=157343744 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157572592 ingress_bytes=213966848 link_bytes=213966848 token_bytes=314916336 weight_movement_bytes=56623104 total_nvlink_bytes=371539440 weight_to_token_ratio=0.179804 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050125059e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38194 dup_rows=21904 dup_rows_remote=19076 dedup_ratio=0.334229 router_tokens_per_expert=[256, 248, 273, 220, 252, 260, 265, 268, 261, 259, 272, 275, 250, 252, 247, 259, 224, 266, 248, 242, 270, 272, 282, 295, 249, 263, 261, 266, 281, 238, 243, 252, 240, 260, 240, 292, 265, 257, 259, 229, 256, 223, 275, 250, 270, 270, 241, 241, 245, 244, 254, 265, 246, 254, 272, 235, 222, 255, 276, 260, 259, 250, 247, 266, 242, 234, 267, 263, 241, 255, 285, 234, 244, 264, 268, 254, 269, 267, 275, 271, 272, 239, 264, 243, 244, 268, 248, 276, 241, 256, 251, 289, 264, 269, 279, 243, 227, 254, 228, 244, 234, 246, 256, 265, 239, 289, 233, 257, 251, 243, 252, 268, 258, 246, 264, 252, 257, 250, 259, 239, 270, 263, 276, 239, 268, 244, 283, 248] payload_rows_per_dst=[5452, 5462, 5418, 5442, 5438, 5550, 5370, 5500] dup_rows_per_dst=[2782, 2842, 2718, 2658, 2828, 2742, 2602, 2732] combine_rows_per_src=[5452, 5462, 5418, 5442, 5438, 5550, 5370, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=155942912 dispatch_meta_bytes=228920 combine_payload_bytes=155942912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156171832 ingress_bytes=174817280 link_bytes=174817280 token_bytes=312114744 weight_movement_bytes=18874368 total_nvlink_bytes=330989112 weight_to_token_ratio=0.060473 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675013161e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38148 dup_rows=21942 dup_rows_remote=19166 dedup_ratio=0.334808 router_tokens_per_expert=[292, 266, 284, 264, 244, 256, 258, 258, 244, 241, 250, 234, 271, 257, 244, 261, 243, 277, 270, 264, 227, 253, 296, 259, 267, 262, 255, 282, 261, 218, 256, 241, 259, 239, 241, 242, 257, 281, 261, 268, 245, 258, 240, 286, 244, 240, 250, 263, 274, 260, 252, 279, 250, 248, 242, 244, 269, 275, 267, 275, 263, 268, 269, 246, 242, 214, 304, 254, 247, 234, 260, 266, 241, 280, 243, 236, 260, 267, 263, 260, 256, 298, 257, 277, 255, 261, 241, 241, 229, 269, 259, 242, 257, 253, 245, 261, 264, 260, 272, 235, 253, 248, 246, 270, 258, 244, 247, 251, 262, 222, 244, 263, 242, 267, 258, 240, 256, 250, 247, 273, 266, 223, 266, 254, 240, 253, 245, 267] payload_rows_per_dst=[5442, 5432, 5432, 5586, 5394, 5420, 5446, 5442] dup_rows_per_dst=[2806, 2686, 2716, 2776, 2748, 2782, 2776, 2652] combine_rows_per_src=[5442, 5432, 5432, 5586, 5394, 5420, 5446, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38236 dup_rows=21862 dup_rows_remote=18970 dedup_ratio=0.333588 router_tokens_per_expert=[264, 250, 259, 250, 247, 258, 234, 248, 257, 272, 270, 230, 228, 282, 260, 263, 227, 261, 245, 261, 232, 240, 276, 263, 236, 266, 258, 251, 238, 253, 247, 230, 247, 273, 248, 263, 266, 238, 262, 262, 256, 259, 241, 256, 264, 278, 251, 263, 291, 246, 265, 280, 240, 274, 253, 263, 243, 243, 266, 272, 239, 268, 265, 257, 264, 232, 245, 290, 258, 258, 247, 234, 266, 231, 268, 263, 231, 250, 255, 280, 265, 290, 276, 252, 261, 277, 254, 258, 263, 267, 249, 260, 261, 244, 267, 243, 259, 238, 245, 269, 243, 242, 289, 261, 237, 266, 256, 257, 247, 249, 254, 244, 265, 228, 273, 267, 257, 231, 256, 250, 247, 297, 228, 257, 258, 249, 296, 246] payload_rows_per_dst=[5464, 5374, 5470, 5438, 5502, 5544, 5418, 5464] dup_rows_per_dst=[2680, 2594, 2784, 2892, 2642, 2830, 2694, 2746] combine_rows_per_src=[5464, 5374, 5470, 5438, 5502, 5544, 5418, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=230144 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156599040 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312967936 weight_movement_bytes=18874368 total_nvlink_bytes=331842304 weight_to_token_ratio=0.060308 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38146 dup_rows=21900 dup_rows_remote=19144 dedup_ratio=0.334167 router_tokens_per_expert=[244, 257, 252, 276, 257, 256, 258, 258, 247, 263, 270, 249, 271, 263, 243, 247, 270, 246, 282, 272, 282, 243, 292, 251, 236, 255, 253, 260, 260, 307, 255, 248, 269, 260, 291, 242, 247, 251, 244, 257, 247, 266, 287, 232, 253, 246, 246, 239, 264, 253, 272, 273, 247, 252, 251, 254, 257, 261, 267, 238, 252, 275, 231, 264, 284, 262, 253, 248, 246, 271, 231, 265, 265, 247, 248, 253, 209, 266, 258, 248, 265, 241, 259, 249, 253, 231, 246, 282, 286, 249, 247, 243, 238, 241, 268, 273, 233, 260, 222, 260, 223, 228, 268, 248, 260, 287, 260, 271, 291, 232, 250, 290, 260, 240, 248, 248, 255, 279, 253, 237, 253, 259, 259, 253, 251, 234, 290, 230] payload_rows_per_dst=[5490, 5576, 5684, 5446, 5432, 5412, 5166, 5430] dup_rows_per_dst=[2756, 2848, 3006, 2752, 2676, 2730, 2464, 2668] combine_rows_per_src=[5490, 5576, 5684, 5446, 5432, 5412, 5166, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43746 payload_rows_remote=38270 dup_rows=21790 dup_rows_remote=19138 dedup_ratio=0.332489 router_tokens_per_expert=[255, 262, 275, 268, 273, 235, 253, 243, 300, 254, 231, 262, 250, 250, 268, 270, 257, 234, 266, 264, 242, 249, 239, 268, 261, 234, 256, 265, 243, 248, 267, 252, 256, 270, 259, 254, 248, 259, 262, 274, 227, 278, 242, 270, 237, 255, 231, 242, 256, 273, 236, 248, 266, 226, 282, 257, 255, 279, 259, 270, 245, 268, 254, 260, 248, 286, 247, 273, 253, 263, 246, 254, 277, 261, 254, 242, 284, 241, 266, 264, 268, 245, 249, 239, 247, 270, 277, 236, 223, 281, 236, 240, 263, 237, 222, 254, 242, 247, 244, 253, 279, 276, 297, 259, 237, 276, 248, 248, 248, 274, 259, 249, 228, 243, 252, 268, 261, 260, 279, 239, 276, 255, 254, 243, 250, 271, 247, 268] payload_rows_per_dst=[5554, 5428, 5476, 5434, 5530, 5354, 5530, 5440] dup_rows_per_dst=[2744, 2662, 2652, 2834, 2788, 2620, 2742, 2748] combine_rows_per_src=[5554, 5428, 5476, 5434, 5530, 5354, 5530, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229080 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671704 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114328 weight_movement_bytes=18874368 total_nvlink_bytes=331988696 weight_to_token_ratio=0.060279 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229256 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483464 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737672 weight_movement_bytes=18874368 total_nvlink_bytes=331612040 weight_to_token_ratio=0.060352 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=156614656 dispatch_meta_bytes=228824 combine_payload_bytes=156614656 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156843480 ingress_bytes=213237760 link_bytes=213237760 token_bytes=313458136 weight_movement_bytes=56623104 total_nvlink_bytes=370081240 weight_to_token_ratio=0.180640 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043139297e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229632 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983552 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737472 weight_movement_bytes=37748736 total_nvlink_bytes=351486208 weight_to_token_ratio=0.120320 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=229160 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156475176 ingress_bytes=212869120 link_bytes=212869120 token_bytes=312721192 weight_movement_bytes=56623104 total_nvlink_bytes=369344296 weight_to_token_ratio=0.181066 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039607170e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38188 dup_rows=21804 dup_rows_remote=18970 dedup_ratio=0.332703 router_tokens_per_expert=[267, 229, 262, 265, 269, 254, 267, 234, 280, 231, 262, 268, 245, 265, 254, 243, 257, 244, 260, 261, 246, 258, 242, 244, 272, 250, 215, 254, 291, 266, 291, 261, 267, 236, 264, 259, 261, 244, 267, 238, 250, 280, 266, 269, 259, 262, 255, 246, 238, 244, 269, 240, 251, 266, 281, 298, 243, 272, 246, 237, 250, 272, 273, 251, 237, 238, 279, 279, 227, 254, 242, 258, 256, 256, 262, 246, 273, 254, 272, 281, 251, 259, 261, 289, 255, 273, 269, 287, 240, 258, 259, 246, 256, 267, 274, 232, 231, 256, 249, 280, 245, 239, 244, 231, 248, 254, 245, 269, 258, 237, 263, 240, 234, 260, 245, 237, 273, 258, 264, 235, 267, 207, 285, 239, 257, 268, 245, 254] payload_rows_per_dst=[5504, 5544, 5426, 5564, 5344, 5536, 5404, 5410] dup_rows_per_dst=[2686, 2834, 2820, 2698, 2730, 2816, 2574, 2646] combine_rows_per_src=[5504, 5544, 5426, 5564, 5344, 5536, 5404, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43658 payload_rows_remote=37934 dup_rows=21878 dup_rows_remote=18850 dedup_ratio=0.333832 router_tokens_per_expert=[255, 275, 236, 236, 276, 273, 261, 260, 238, 230, 247, 247, 254, 269, 257, 237, 242, 237, 268, 260, 252, 231, 274, 255, 239, 270, 246, 275, 255, 265, 256, 259, 262, 257, 255, 268, 290, 252, 273, 246, 270, 250, 270, 268, 268, 287, 234, 254, 265, 251, 257, 255, 266, 237, 253, 266, 218, 248, 230, 276, 267, 265, 235, 250, 282, 247, 257, 264, 253, 256, 255, 261, 252, 263, 253, 258, 235, 254, 265, 280, 265, 254, 257, 221, 249, 270, 247, 263, 263, 237, 240, 281, 258, 231, 273, 263, 246, 253, 275, 256, 268, 260, 261, 270, 234, 260, 250, 267, 243, 237, 237, 260, 260, 258, 280, 258, 260, 266, 276, 243, 242, 248, 252, 250, 273, 275, 240, 225] payload_rows_per_dst=[5354, 5476, 5318, 5394, 5484, 5480, 5428, 5724] dup_rows_per_dst=[2532, 2692, 2550, 2684, 2890, 2776, 2726, 3028] combine_rows_per_src=[5354, 5476, 5318, 5394, 5484, 5480, 5428, 5724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38270 dup_rows=21702 dup_rows_remote=19014 dedup_ratio=0.331146 router_tokens_per_expert=[272, 267, 244, 245, 264, 273, 276, 207, 252, 234, 247, 262, 246, 272, 259, 264, 246, 250, 264, 252, 275, 240, 242, 263, 257, 248, 264, 231, 256, 270, 253, 253, 272, 254, 269, 273, 258, 256, 249, 270, 263, 240, 270, 239, 241, 241, 245, 240, 255, 248, 256, 255, 279, 233, 246, 251, 264, 250, 264, 250, 286, 283, 266, 288, 279, 249, 266, 246, 254, 237, 267, 268, 251, 255, 278, 248, 248, 267, 285, 246, 251, 277, 251, 266, 246, 265, 253, 240, 276, 255, 281, 249, 243, 252, 266, 255, 260, 261, 218, 231, 266, 238, 247, 263, 244, 249, 276, 246, 255, 249, 272, 260, 250, 258, 233, 246, 263, 281, 249, 249, 224, 277, 253, 257, 266, 231, 270, 254] payload_rows_per_dst=[5502, 5444, 5424, 5496, 5534, 5564, 5420, 5450] dup_rows_per_dst=[2666, 2684, 2736, 2852, 2754, 2688, 2650, 2672] combine_rows_per_src=[5502, 5444, 5424, 5496, 5534, 5564, 5420, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228632 combine_payload_bytes=156418048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156646680 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313064728 weight_movement_bytes=18874368 total_nvlink_bytes=331939096 weight_to_token_ratio=0.060289 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=155377664 dispatch_meta_bytes=227136 combine_payload_bytes=155377664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155604800 ingress_bytes=174252032 link_bytes=174252032 token_bytes=310982464 weight_movement_bytes=18874368 total_nvlink_bytes=329856832 weight_to_token_ratio=0.060693 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669597233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38368 dup_rows=21706 dup_rows_remote=18926 dedup_ratio=0.331207 router_tokens_per_expert=[257, 227, 277, 259, 266, 239, 215, 297, 243, 257, 250, 276, 264, 271, 266, 260, 248, 257, 246, 247, 260, 230, 255, 273, 268, 279, 264, 234, 245, 246, 244, 252, 252, 255, 253, 257, 274, 251, 259, 272, 257, 259, 255, 253, 258, 273, 255, 262, 258, 252, 251, 255, 279, 253, 257, 241, 267, 259, 266, 283, 264, 249, 270, 253, 270, 266, 255, 267, 283, 262, 241, 245, 245, 256, 260, 251, 242, 259, 250, 269, 243, 259, 256, 249, 241, 228, 241, 252, 250, 242, 266, 276, 243, 245, 250, 265, 226, 247, 253, 292, 255, 271, 286, 262, 275, 258, 279, 282, 244, 250, 239, 270, 245, 256, 247, 247, 261, 257, 240, 235, 277, 254, 235, 253, 237, 257, 252, 225] payload_rows_per_dst=[5526, 5480, 5516, 5512, 5462, 5382, 5596, 5356] dup_rows_per_dst=[2722, 2616, 2774, 2802, 2780, 2630, 2782, 2600] combine_rows_per_src=[5526, 5480, 5516, 5512, 5462, 5382, 5596, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38354 dup_rows=21782 dup_rows_remote=19068 dedup_ratio=0.332367 router_tokens_per_expert=[244, 243, 253, 244, 263, 242, 252, 282, 264, 258, 260, 247, 248, 262, 271, 274, 246, 246, 273, 239, 256, 241, 250, 221, 245, 257, 257, 275, 262, 253, 254, 255, 259, 269, 286, 265, 254, 257, 267, 257, 260, 263, 268, 240, 235, 252, 269, 246, 235, 273, 273, 283, 247, 256, 274, 225, 237, 240, 249, 263, 215, 253, 279, 251, 258, 238, 260, 256, 256, 273, 284, 252, 233, 256, 278, 244, 253, 250, 262, 258, 264, 265, 267, 256, 281, 260, 289, 246, 250, 265, 258, 264, 267, 240, 269, 287, 241, 262, 229, 252, 265, 240, 240, 271, 278, 250, 249, 232, 250, 269, 272, 257, 244, 261, 262, 270, 229, 275, 242, 242, 253, 239, 256, 262, 242, 257, 263, 238] payload_rows_per_dst=[5458, 5416, 5492, 5504, 5478, 5554, 5400, 5452] dup_rows_per_dst=[2756, 2644, 2694, 2602, 2744, 2902, 2714, 2726] combine_rows_per_src=[5458, 5416, 5492, 5504, 5478, 5554, 5400, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38336 dup_rows=21764 dup_rows_remote=19208 dedup_ratio=0.332092 router_tokens_per_expert=[255, 256, 248, 276, 254, 263, 255, 281, 247, 283, 250, 266, 233, 257, 264, 222, 239, 274, 249, 250, 249, 253, 255, 260, 272, 249, 275, 222, 257, 251, 259, 245, 248, 267, 259, 245, 245, 273, 265, 268, 265, 239, 277, 240, 258, 240, 238, 258, 258, 269, 223, 235, 217, 268, 265, 249, 228, 258, 250, 243, 275, 251, 251, 256, 270, 250, 263, 214, 282, 238, 245, 257, 245, 270, 240, 276, 250, 277, 266, 270, 259, 276, 260, 276, 260, 241, 248, 257, 259, 272, 266, 265, 250, 291, 251, 236, 253, 258, 257, 286, 271, 271, 265, 235, 265, 279, 278, 256, 233, 257, 277, 220, 243, 281, 218, 250, 265, 270, 262, 246, 255, 258, 266, 286, 251, 242, 248, 236] payload_rows_per_dst=[5478, 5454, 5390, 5436, 5532, 5498, 5526, 5458] dup_rows_per_dst=[2742, 2664, 2780, 2556, 2694, 2836, 2796, 2696] combine_rows_per_src=[5478, 5454, 5390, 5436, 5532, 5498, 5526, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38282 dup_rows=21772 dup_rows_remote=18998 dedup_ratio=0.332214 router_tokens_per_expert=[268, 226, 259, 299, 263, 266, 241, 277, 245, 264, 245, 248, 247, 247, 223, 251, 289, 253, 260, 259, 267, 264, 262, 235, 269, 281, 274, 260, 244, 278, 250, 272, 255, 251, 302, 238, 284, 254, 266, 237, 286, 234, 273, 242, 233, 256, 248, 269, 265, 257, 243, 245, 268, 236, 270, 247, 269, 266, 240, 257, 269, 253, 247, 263, 229, 268, 276, 252, 253, 237, 262, 264, 291, 265, 252, 260, 239, 252, 223, 242, 271, 235, 235, 266, 260, 229, 255, 258, 262, 252, 252, 243, 258, 253, 265, 254, 248, 235, 275, 278, 238, 245, 249, 252, 253, 264, 253, 276, 240, 263, 257, 239, 234, 251, 241, 255, 251, 265, 236, 262, 254, 227, 249, 277, 277, 265, 259, 278] payload_rows_per_dst=[5492, 5572, 5482, 5496, 5414, 5490, 5360, 5458] dup_rows_per_dst=[2646, 2862, 2774, 2694, 2716, 2606, 2770, 2704] combine_rows_per_src=[5492, 5572, 5482, 5496, 5414, 5490, 5360, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229136 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983056 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313736976 weight_movement_bytes=37748736 total_nvlink_bytes=351485712 weight_to_token_ratio=0.120320 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43954 payload_rows_remote=38392 dup_rows=21582 dup_rows_remote=18882 dedup_ratio=0.329315 router_tokens_per_expert=[264, 261, 254, 273, 252, 255, 242, 271, 258, 265, 240, 231, 260, 247, 255, 273, 273, 264, 270, 264, 281, 263, 240, 249, 246, 244, 236, 258, 249, 245, 253, 242, 249, 262, 256, 269, 272, 273, 250, 213, 285, 250, 243, 222, 248, 255, 253, 264, 251, 245, 288, 235, 242, 240, 269, 271, 228, 251, 253, 259, 271, 251, 256, 265, 274, 236, 254, 286, 268, 245, 255, 230, 258, 239, 285, 232, 252, 262, 232, 266, 220, 265, 246, 309, 279, 262, 266, 272, 261, 262, 247, 264, 257, 246, 299, 263, 252, 244, 249, 254, 252, 275, 276, 231, 276, 264, 258, 282, 266, 291, 246, 263, 246, 255, 238, 261, 276, 229, 270, 232, 234, 271, 245, 234, 231, 277, 225, 256] payload_rows_per_dst=[5562, 5678, 5492, 5444, 5636, 5554, 5194, 5394] dup_rows_per_dst=[2700, 2992, 2700, 2728, 2802, 2736, 2358, 2566] combine_rows_per_src=[5562, 5678, 5492, 5444, 5636, 5554, 5194, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229176 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157384504 ingress_bytes=194904064 link_bytes=194904064 token_bytes=314539832 weight_movement_bytes=37748736 total_nvlink_bytes=352288568 weight_to_token_ratio=0.120013 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867474842e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229688 combine_payload_bytes=157097984 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157327672 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425656 weight_movement_bytes=18874368 total_nvlink_bytes=333300024 weight_to_token_ratio=0.060028 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=230176 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157254432 ingress_bytes=194772992 link_bytes=194772992 token_bytes=314278688 weight_movement_bytes=37748736 total_nvlink_bytes=352027424 weight_to_token_ratio=0.120112 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866218975e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229120 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032192 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835264 weight_movement_bytes=37748736 total_nvlink_bytes=351584000 weight_to_token_ratio=0.120282 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38606 dup_rows=21636 dup_rows_remote=19238 dedup_ratio=0.330139 router_tokens_per_expert=[258, 277, 284, 260, 259, 267, 222, 262, 243, 256, 249, 242, 242, 253, 246, 262, 216, 251, 251, 229, 255, 224, 238, 235, 255, 264, 259, 268, 255, 241, 246, 254, 266, 250, 273, 293, 243, 258, 261, 254, 272, 270, 247, 239, 254, 274, 249, 261, 263, 252, 266, 244, 244, 259, 278, 255, 276, 275, 250, 252, 275, 251, 276, 243, 256, 275, 240, 246, 291, 219, 271, 255, 239, 228, 258, 255, 244, 260, 255, 281, 263, 249, 263, 283, 251, 267, 240, 245, 268, 240, 263, 248, 270, 245, 266, 245, 258, 253, 256, 252, 233, 250, 259, 249, 242, 252, 236, 266, 228, 269, 235, 283, 263, 238, 263, 253, 278, 252, 276, 287, 250, 262, 286, 256, 279, 250, 290, 239] payload_rows_per_dst=[5542, 5294, 5602, 5530, 5418, 5560, 5370, 5584] dup_rows_per_dst=[2812, 2398, 2726, 2788, 2728, 2652, 2672, 2860] combine_rows_per_src=[5542, 5294, 5602, 5530, 5418, 5560, 5370, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38444 dup_rows=21730 dup_rows_remote=19030 dedup_ratio=0.331573 router_tokens_per_expert=[279, 246, 265, 265, 263, 240, 214, 249, 254, 252, 275, 252, 227, 255, 242, 227, 270, 275, 262, 271, 284, 254, 251, 250, 285, 269, 246, 246, 271, 278, 279, 275, 255, 255, 272, 235, 265, 252, 257, 264, 228, 256, 279, 224, 274, 247, 255, 239, 243, 248, 259, 242, 243, 244, 244, 265, 233, 251, 257, 306, 270, 258, 272, 264, 264, 285, 245, 291, 243, 245, 273, 222, 243, 270, 274, 247, 307, 249, 212, 267, 237, 256, 250, 241, 265, 247, 230, 262, 245, 249, 233, 260, 267, 253, 259, 277, 257, 253, 276, 245, 277, 269, 250, 275, 264, 262, 258, 264, 229, 255, 246, 262, 269, 250, 277, 246, 237, 251, 229, 245, 258, 231, 267, 255, 289, 244, 250, 233] payload_rows_per_dst=[5368, 5606, 5338, 5302, 5484, 5586, 5760, 5362] dup_rows_per_dst=[2642, 2926, 2446, 2400, 2790, 2806, 3020, 2700] combine_rows_per_src=[5368, 5606, 5338, 5302, 5484, 5586, 5760, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=157253632 dispatch_meta_bytes=229096 combine_payload_bytes=157253632 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157482728 ingress_bytes=176128000 link_bytes=176128000 token_bytes=314736360 weight_movement_bytes=18874368 total_nvlink_bytes=333610728 weight_to_token_ratio=0.059969 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687571836e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43648 payload_rows_remote=38196 dup_rows=21888 dup_rows_remote=19092 dedup_ratio=0.333984 router_tokens_per_expert=[242, 233, 273, 250, 260, 255, 246, 263, 249, 263, 266, 263, 266, 241, 271, 268, 267, 271, 258, 250, 258, 269, 265, 248, 243, 263, 283, 263, 245, 244, 256, 228, 226, 266, 257, 245, 281, 248, 266, 252, 246, 262, 267, 240, 236, 249, 258, 258, 251, 289, 248, 247, 272, 238, 259, 256, 254, 270, 293, 262, 246, 271, 252, 238, 261, 257, 267, 232, 270, 262, 261, 277, 246, 290, 242, 255, 255, 270, 243, 256, 280, 251, 279, 238, 267, 268, 252, 243, 271, 220, 262, 250, 284, 241, 275, 243, 259, 224, 257, 243, 259, 269, 243, 249, 254, 248, 283, 258, 256, 255, 249, 254, 272, 243, 242, 258, 244, 247, 256, 284, 244, 243, 254, 253, 236, 236, 243, 262] payload_rows_per_dst=[5422, 5512, 5444, 5516, 5496, 5452, 5440, 5366] dup_rows_per_dst=[2796, 2710, 2670, 2776, 2792, 2796, 2680, 2668] combine_rows_per_src=[5422, 5512, 5444, 5516, 5496, 5452, 5440, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158130176 dispatch_meta_bytes=231376 combine_payload_bytes=158130176 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158361552 ingress_bytes=195878912 link_bytes=195878912 token_bytes=316491728 weight_movement_bytes=37748736 total_nvlink_bytes=354240464 weight_to_token_ratio=0.119272 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.876815356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229896 combine_payload_bytes=157466624 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157696520 ingress_bytes=176340992 link_bytes=176340992 token_bytes=315163144 weight_movement_bytes=18874368 total_nvlink_bytes=334037512 weight_to_token_ratio=0.059888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689612620e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38176 dup_rows=21840 dup_rows_remote=19228 dedup_ratio=0.333252 router_tokens_per_expert=[275, 245, 220, 255, 263, 264, 250, 238, 248, 244, 218, 243, 262, 267, 269, 271, 242, 252, 275, 279, 242, 255, 261, 255, 252, 260, 283, 266, 267, 262, 243, 265, 255, 276, 246, 253, 257, 241, 234, 234, 264, 265, 257, 250, 240, 261, 248, 282, 250, 256, 274, 255, 253, 282, 252, 262, 248, 248, 248, 237, 245, 270, 242, 275, 255, 243, 269, 245, 241, 254, 249, 245, 263, 243, 250, 257, 277, 256, 270, 249, 256, 240, 278, 259, 253, 261, 270, 257, 255, 240, 269, 242, 298, 240, 242, 263, 272, 275, 258, 263, 237, 265, 249, 250, 264, 266, 241, 270, 259, 256, 239, 263, 263, 245, 224, 239, 271, 257, 263, 255, 272, 253, 243, 258, 233, 298, 264, 263] payload_rows_per_dst=[5400, 5430, 5482, 5440, 5520, 5494, 5436, 5494] dup_rows_per_dst=[2664, 2888, 2644, 2754, 2612, 2752, 2818, 2708] combine_rows_per_src=[5400, 5430, 5482, 5440, 5520, 5494, 5436, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38294 dup_rows=21774 dup_rows_remote=19128 dedup_ratio=0.332245 router_tokens_per_expert=[273, 261, 248, 285, 253, 277, 286, 266, 265, 248, 258, 244, 264, 227, 267, 228, 260, 275, 248, 267, 257, 255, 237, 229, 260, 229, 271, 265, 255, 254, 280, 244, 272, 250, 290, 278, 276, 255, 247, 253, 272, 242, 250, 259, 276, 265, 249, 246, 253, 288, 256, 250, 222, 237, 270, 242, 250, 275, 261, 257, 287, 259, 248, 248, 279, 252, 268, 238, 268, 276, 277, 269, 232, 253, 278, 254, 259, 254, 253, 239, 233, 246, 268, 242, 249, 266, 267, 260, 244, 252, 223, 261, 254, 247, 289, 241, 251, 280, 244, 261, 256, 237, 227, 249, 272, 268, 221, 261, 236, 266, 237, 249, 243, 278, 284, 240, 243, 254, 242, 245, 235, 237, 258, 264, 246, 253, 271, 250] payload_rows_per_dst=[5488, 5476, 5570, 5446, 5512, 5428, 5468, 5374] dup_rows_per_dst=[2812, 2696, 2790, 2676, 2786, 2656, 2646, 2712] combine_rows_per_src=[5488, 5476, 5570, 5446, 5512, 5428, 5468, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38192 dup_rows=21782 dup_rows_remote=19056 dedup_ratio=0.332367 router_tokens_per_expert=[234, 241, 252, 237, 232, 253, 254, 250, 265, 246, 240, 274, 249, 233, 265, 274, 263, 266, 264, 255, 264, 280, 260, 251, 252, 266, 265, 251, 262, 277, 235, 262, 240, 246, 257, 247, 240, 244, 262, 245, 248, 272, 234, 219, 306, 250, 272, 251, 279, 275, 256, 227, 243, 270, 250, 271, 268, 239, 253, 257, 228, 285, 262, 281, 266, 236, 230, 283, 244, 232, 234, 254, 255, 235, 283, 259, 273, 229, 252, 279, 250, 280, 265, 283, 239, 222, 244, 238, 276, 275, 274, 252, 261, 277, 258, 251, 234, 250, 245, 231, 247, 277, 261, 254, 227, 264, 253, 253, 249, 272, 247, 250, 280, 269, 258, 260, 282, 279, 255, 257, 252, 260, 278, 267, 267, 257, 261, 234] payload_rows_per_dst=[5430, 5506, 5408, 5562, 5452, 5484, 5412, 5500] dup_rows_per_dst=[2568, 2840, 2658, 2726, 2636, 2806, 2616, 2932] combine_rows_per_src=[5430, 5506, 5408, 5562, 5452, 5484, 5412, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38292 dup_rows=21784 dup_rows_remote=19138 dedup_ratio=0.332397 router_tokens_per_expert=[273, 274, 231, 248, 286, 252, 293, 269, 241, 254, 250, 238, 251, 269, 215, 268, 252, 237, 263, 256, 251, 265, 271, 264, 269, 277, 251, 264, 230, 251, 254, 277, 239, 236, 270, 256, 249, 244, 261, 243, 255, 243, 249, 263, 247, 260, 256, 282, 266, 244, 246, 264, 272, 289, 243, 287, 259, 283, 243, 236, 258, 249, 235, 256, 259, 263, 236, 270, 272, 256, 275, 239, 247, 243, 241, 261, 239, 256, 246, 267, 265, 220, 246, 269, 272, 254, 257, 278, 234, 270, 243, 265, 268, 245, 251, 269, 240, 229, 264, 238, 241, 243, 252, 279, 249, 255, 271, 261, 242, 269, 272, 257, 255, 269, 263, 225, 264, 274, 258, 255, 231, 258, 236, 254, 262, 237, 293, 269] payload_rows_per_dst=[5494, 5492, 5460, 5498, 5426, 5474, 5458, 5450] dup_rows_per_dst=[2730, 2772, 2646, 2762, 2714, 2738, 2666, 2756] combine_rows_per_src=[5494, 5492, 5460, 5498, 5426, 5474, 5458, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229152 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156679968 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313130784 weight_movement_bytes=18874368 total_nvlink_bytes=332005152 weight_to_token_ratio=0.060276 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43880 payload_rows_remote=38416 dup_rows=21656 dup_rows_remote=18880 dedup_ratio=0.330444 router_tokens_per_expert=[271, 240, 268, 259, 251, 244, 264, 261, 276, 261, 267, 227, 231, 252, 269, 279, 239, 252, 282, 266, 240, 260, 274, 245, 257, 251, 260, 270, 272, 280, 259, 248, 258, 269, 258, 261, 301, 250, 250, 256, 269, 281, 257, 282, 236, 256, 235, 268, 286, 256, 229, 265, 288, 255, 249, 256, 258, 252, 248, 239, 240, 245, 252, 255, 264, 226, 250, 253, 256, 251, 252, 269, 255, 267, 266, 239, 284, 233, 259, 260, 269, 259, 248, 273, 246, 234, 251, 239, 267, 245, 237, 264, 275, 230, 273, 261, 252, 257, 264, 244, 247, 272, 285, 258, 220, 250, 260, 246, 241, 256, 253, 276, 256, 254, 230, 252, 245, 294, 261, 261, 257, 236, 251, 200, 253, 248, 242, 257] payload_rows_per_dst=[5464, 5626, 5522, 5444, 5476, 5452, 5494, 5402] dup_rows_per_dst=[2776, 2762, 2852, 2624, 2692, 2690, 2668, 2592] combine_rows_per_src=[5464, 5626, 5522, 5444, 5476, 5452, 5494, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=229616 combine_payload_bytes=156368896 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156598512 ingress_bytes=194117632 link_bytes=194117632 token_bytes=312967408 weight_movement_bytes=37748736 total_nvlink_bytes=350716144 weight_to_token_ratio=0.120616 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859939638e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229688 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157081912 ingress_bytes=175726592 link_bytes=175726592 token_bytes=313934136 weight_movement_bytes=18874368 total_nvlink_bytes=332808504 weight_to_token_ratio=0.060122 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683725742e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229720 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157073752 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313917784 weight_movement_bytes=18874368 total_nvlink_bytes=332792152 weight_to_token_ratio=0.060125 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=228992 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156663424 ingress_bytes=194183168 link_bytes=194183168 token_bytes=313097856 weight_movement_bytes=37748736 total_nvlink_bytes=350846592 weight_to_token_ratio=0.120565 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860567572e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38310 dup_rows=21766 dup_rows_remote=19010 dedup_ratio=0.332123 router_tokens_per_expert=[244, 275, 267, 256, 252, 279, 245, 242, 275, 253, 240, 246, 245, 245, 248, 278, 275, 241, 261, 240, 257, 265, 253, 258, 257, 278, 252, 298, 281, 276, 260, 239, 268, 229, 256, 270, 230, 251, 241, 245, 249, 262, 256, 264, 253, 220, 232, 248, 242, 264, 265, 256, 268, 253, 242, 294, 280, 265, 220, 244, 258, 267, 274, 270, 279, 250, 251, 230, 259, 269, 278, 275, 289, 234, 247, 263, 245, 245, 274, 260, 248, 257, 245, 240, 228, 248, 257, 253, 255, 258, 263, 284, 271, 270, 278, 240, 261, 224, 249, 274, 255, 266, 243, 285, 245, 229, 225, 280, 262, 250, 265, 266, 265, 275, 230, 240, 241, 259, 231, 275, 266, 255, 217, 261, 244, 255, 249, 266] payload_rows_per_dst=[5480, 5620, 5382, 5418, 5462, 5522, 5426, 5460] dup_rows_per_dst=[2700, 2762, 2566, 2714, 2652, 2860, 2756, 2756] combine_rows_per_src=[5480, 5620, 5382, 5418, 5462, 5522, 5426, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43704 payload_rows_remote=38148 dup_rows=21832 dup_rows_remote=19056 dedup_ratio=0.333130 router_tokens_per_expert=[258, 235, 246, 289, 276, 244, 248, 260, 241, 259, 258, 239, 279, 244, 244, 285, 258, 272, 244, 257, 259, 267, 257, 259, 252, 278, 264, 238, 297, 268, 240, 256, 249, 247, 272, 287, 257, 255, 255, 258, 264, 230, 291, 240, 243, 275, 248, 250, 236, 269, 243, 251, 265, 239, 266, 264, 255, 290, 252, 252, 242, 248, 230, 248, 315, 240, 257, 253, 245, 238, 252, 268, 252, 252, 269, 259, 274, 238, 264, 265, 270, 228, 260, 269, 264, 248, 248, 243, 262, 264, 271, 252, 245, 250, 260, 270, 276, 269, 245, 237, 242, 270, 264, 245, 270, 257, 267, 253, 264, 238, 251, 257, 242, 236, 240, 217, 266, 265, 257, 221, 239, 271, 269, 247, 238, 250, 249, 269] payload_rows_per_dst=[5462, 5556, 5452, 5392, 5530, 5466, 5494, 5352] dup_rows_per_dst=[2748, 2776, 2790, 2708, 2752, 2742, 2716, 2600] combine_rows_per_src=[5462, 5556, 5452, 5392, 5530, 5466, 5494, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229184 combine_payload_bytes=157351936 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157581120 ingress_bytes=213975040 link_bytes=213975040 token_bytes=314933056 weight_movement_bytes=56623104 total_nvlink_bytes=371556160 weight_to_token_ratio=0.179794 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050203551e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229280 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157147040 ingress_bytes=175792128 link_bytes=175792128 token_bytes=314064800 weight_movement_bytes=18874368 total_nvlink_bytes=332939168 weight_to_token_ratio=0.060097 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684353675e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228816 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156483024 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312737232 weight_movement_bytes=37748736 total_nvlink_bytes=350485968 weight_to_token_ratio=0.120704 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38064 dup_rows=21982 dup_rows_remote=19210 dedup_ratio=0.335419 router_tokens_per_expert=[245, 245, 237, 243, 264, 272, 263, 229, 241, 267, 257, 232, 242, 257, 258, 257, 266, 255, 264, 234, 255, 256, 256, 259, 273, 236, 283, 255, 257, 248, 273, 248, 289, 275, 238, 278, 234, 248, 269, 250, 288, 250, 257, 276, 227, 255, 215, 292, 266, 260, 241, 243, 258, 235, 255, 239, 248, 259, 254, 254, 272, 255, 269, 257, 244, 245, 262, 265, 261, 252, 249, 273, 242, 265, 263, 243, 253, 249, 241, 280, 263, 292, 225, 248, 254, 281, 245, 228, 270, 278, 253, 245, 267, 247, 275, 260, 264, 232, 283, 254, 240, 236, 255, 240, 237, 242, 273, 262, 290, 275, 268, 234, 249, 267, 279, 270, 270, 269, 252, 240, 241, 256, 255, 285, 253, 242, 240, 264] payload_rows_per_dst=[5314, 5514, 5464, 5450, 5388, 5490, 5376, 5558] dup_rows_per_dst=[2704, 2722, 2818, 2680, 2786, 2772, 2794, 2706] combine_rows_per_src=[5314, 5514, 5464, 5450, 5388, 5490, 5376, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43918 payload_rows_remote=38388 dup_rows=21618 dup_rows_remote=18830 dedup_ratio=0.329865 router_tokens_per_expert=[272, 258, 262, 242, 278, 282, 238, 238, 262, 251, 236, 250, 249, 261, 262, 227, 262, 255, 248, 262, 244, 255, 249, 231, 274, 252, 242, 232, 259, 270, 288, 246, 275, 284, 252, 237, 265, 248, 274, 246, 259, 232, 269, 244, 233, 283, 244, 265, 248, 268, 282, 222, 254, 254, 245, 264, 278, 278, 255, 278, 298, 269, 243, 255, 267, 250, 281, 251, 249, 237, 254, 262, 267, 243, 247, 249, 274, 293, 258, 277, 229, 239, 233, 245, 245, 260, 243, 283, 243, 256, 245, 258, 266, 258, 239, 240, 273, 265, 276, 247, 242, 266, 263, 259, 247, 259, 234, 244, 279, 231, 264, 263, 261, 263, 272, 235, 261, 238, 245, 252, 258, 249, 261, 283, 268, 239, 268, 224] payload_rows_per_dst=[5468, 5478, 5500, 5606, 5530, 5330, 5542, 5464] dup_rows_per_dst=[2668, 2660, 2720, 2776, 2788, 2634, 2682, 2690] combine_rows_per_src=[5468, 5478, 5500, 5606, 5530, 5330, 5542, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38284 dup_rows=21764 dup_rows_remote=19044 dedup_ratio=0.332092 router_tokens_per_expert=[235, 239, 268, 249, 275, 258, 232, 262, 267, 250, 253, 234, 241, 242, 248, 244, 241, 271, 254, 209, 241, 261, 273, 259, 261, 237, 259, 275, 298, 248, 242, 240, 274, 266, 232, 261, 269, 248, 246, 267, 236, 249, 235, 241, 273, 268, 278, 261, 271, 242, 274, 256, 281, 261, 265, 265, 238, 251, 254, 253, 258, 243, 270, 283, 263, 227, 235, 282, 240, 272, 265, 271, 246, 253, 230, 261, 267, 237, 246, 257, 258, 232, 267, 235, 234, 253, 243, 267, 274, 271, 253, 245, 262, 280, 232, 262, 246, 266, 235, 284, 253, 244, 262, 250, 238, 268, 254, 277, 240, 293, 253, 255, 278, 263, 226, 267, 264, 242, 260, 276, 280, 238, 274, 243, 268, 303, 262, 251] payload_rows_per_dst=[5438, 5438, 5488, 5560, 5410, 5420, 5454, 5564] dup_rows_per_dst=[2556, 2700, 2720, 2770, 2694, 2716, 2782, 2826] combine_rows_per_src=[5438, 5438, 5488, 5560, 5410, 5420, 5454, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38182 dup_rows=21846 dup_rows_remote=19130 dedup_ratio=0.333344 router_tokens_per_expert=[277, 283, 250, 245, 230, 266, 260, 254, 263, 273, 265, 248, 254, 267, 225, 243, 253, 251, 273, 295, 230, 228, 255, 244, 245, 220, 245, 269, 261, 249, 249, 251, 253, 252, 267, 259, 240, 252, 245, 256, 251, 279, 259, 245, 247, 262, 279, 285, 276, 248, 227, 276, 261, 230, 243, 249, 273, 280, 252, 233, 251, 297, 269, 247, 257, 233, 245, 260, 254, 260, 260, 256, 246, 263, 256, 241, 276, 286, 266, 245, 263, 263, 232, 264, 230, 261, 229, 244, 246, 269, 279, 257, 256, 253, 255, 265, 253, 286, 276, 254, 256, 229, 246, 280, 255, 222, 288, 241, 303, 234, 246, 246, 268, 278, 238, 265, 255, 247, 245, 267, 250, 275, 254, 253, 233, 278, 235, 278] payload_rows_per_dst=[5394, 5444, 5492, 5508, 5460, 5472, 5454, 5466] dup_rows_per_dst=[2812, 2592, 2770, 2716, 2748, 2660, 2776, 2772] combine_rows_per_src=[5394, 5444, 5492, 5508, 5460, 5472, 5454, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38256 dup_rows=21872 dup_rows_remote=19186 dedup_ratio=0.333740 router_tokens_per_expert=[251, 252, 245, 234, 271, 246, 282, 254, 260, 241, 257, 251, 301, 284, 267, 263, 299, 257, 255, 251, 227, 258, 217, 258, 239, 269, 236, 259, 251, 258, 264, 263, 278, 263, 231, 259, 259, 229, 255, 277, 262, 243, 264, 261, 221, 274, 241, 267, 266, 223, 239, 259, 264, 247, 238, 290, 266, 243, 255, 240, 271, 243, 264, 258, 246, 269, 268, 260, 268, 242, 240, 262, 270, 256, 284, 282, 256, 258, 250, 249, 262, 251, 249, 223, 259, 243, 271, 273, 249, 251, 258, 258, 257, 258, 265, 252, 251, 249, 257, 255, 256, 232, 247, 228, 240, 273, 246, 252, 249, 249, 285, 278, 259, 241, 255, 256, 264, 267, 251, 277, 256, 240, 278, 246, 248, 258, 271, 245] payload_rows_per_dst=[5504, 5414, 5426, 5432, 5504, 5502, 5408, 5474] dup_rows_per_dst=[2814, 2708, 2742, 2700, 2816, 2656, 2686, 2750] combine_rows_per_src=[5504, 5414, 5426, 5432, 5504, 5502, 5408, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=229096 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156139240 ingress_bytes=174784512 link_bytes=174784512 token_bytes=312049384 weight_movement_bytes=18874368 total_nvlink_bytes=330923752 weight_to_token_ratio=0.060485 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674699195e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=228872 combine_payload_bytes=157237248 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157466120 ingress_bytes=232734720 link_bytes=232734720 token_bytes=314703368 weight_movement_bytes=75497472 total_nvlink_bytes=390200840 weight_to_token_ratio=0.239900 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229949574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229312 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157040576 ingress_bytes=194560000 link_bytes=194560000 token_bytes=313851840 weight_movement_bytes=37748736 total_nvlink_bytes=351600576 weight_to_token_ratio=0.120276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864178191e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43766 payload_rows_remote=38218 dup_rows=21770 dup_rows_remote=19006 dedup_ratio=0.332184 router_tokens_per_expert=[274, 254, 245, 251, 256, 270, 261, 249, 272, 260, 259, 256, 264, 247, 244, 283, 232, 274, 224, 223, 241, 284, 241, 253, 278, 244, 249, 256, 251, 259, 232, 250, 259, 250, 247, 243, 258, 245, 255, 269, 247, 255, 250, 260, 263, 247, 260, 252, 284, 269, 267, 243, 242, 264, 250, 240, 241, 268, 237, 251, 254, 276, 257, 251, 281, 240, 289, 249, 265, 261, 268, 259, 247, 268, 249, 281, 245, 263, 231, 258, 240, 244, 281, 257, 275, 258, 258, 281, 249, 267, 258, 258, 246, 266, 245, 251, 265, 293, 255, 244, 238, 252, 254, 247, 250, 256, 255, 265, 232, 277, 294, 248, 233, 265, 256, 272, 241, 254, 286, 254, 241, 266, 270, 245, 242, 248, 238, 254] payload_rows_per_dst=[5548, 5554, 5558, 5416, 5322, 5334, 5524, 5510] dup_rows_per_dst=[2764, 2944, 2930, 2752, 2470, 2566, 2724, 2620] combine_rows_per_src=[5548, 5554, 5558, 5416, 5322, 5334, 5524, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38298 dup_rows=21774 dup_rows_remote=19112 dedup_ratio=0.332245 router_tokens_per_expert=[223, 258, 273, 270, 254, 244, 262, 238, 288, 233, 264, 283, 255, 248, 232, 244, 244, 238, 249, 262, 267, 252, 252, 222, 279, 276, 265, 254, 264, 255, 246, 246, 247, 235, 249, 255, 263, 266, 264, 260, 268, 236, 254, 246, 249, 284, 268, 281, 266, 267, 281, 256, 257, 255, 241, 236, 247, 270, 267, 232, 277, 257, 258, 264, 264, 273, 265, 256, 266, 222, 240, 253, 253, 254, 221, 272, 245, 259, 260, 274, 245, 259, 243, 257, 256, 244, 262, 283, 249, 241, 275, 279, 235, 240, 267, 278, 237, 272, 272, 230, 241, 261, 266, 260, 273, 260, 257, 272, 255, 266, 247, 253, 245, 268, 251, 244, 246, 245, 278, 265, 262, 235, 255, 259, 239, 262, 245, 261] payload_rows_per_dst=[5458, 5398, 5460, 5498, 5504, 5466, 5514, 5464] dup_rows_per_dst=[2680, 2744, 2790, 2732, 2650, 2786, 2730, 2662] combine_rows_per_src=[5458, 5398, 5460, 5498, 5504, 5466, 5514, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229248 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156622720 ingress_bytes=213016576 link_bytes=213016576 token_bytes=313016192 weight_movement_bytes=56623104 total_nvlink_bytes=369639296 weight_to_token_ratio=0.180895 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041020021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229768 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156926344 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622920 weight_movement_bytes=18874368 total_nvlink_bytes=332497288 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43938 payload_rows_remote=38444 dup_rows=21598 dup_rows_remote=18922 dedup_ratio=0.329559 router_tokens_per_expert=[258, 229, 262, 244, 273, 246, 238, 257, 252, 252, 252, 237, 273, 224, 247, 283, 264, 253, 265, 242, 262, 265, 280, 223, 254, 260, 239, 252, 253, 268, 239, 266, 239, 232, 255, 272, 279, 269, 273, 248, 266, 252, 291, 242, 248, 260, 273, 262, 255, 246, 273, 266, 244, 261, 274, 256, 231, 257, 257, 266, 259, 251, 255, 235, 247, 254, 243, 244, 274, 248, 278, 239, 269, 252, 254, 245, 250, 245, 253, 273, 256, 263, 242, 232, 269, 257, 263, 283, 275, 269, 248, 242, 270, 256, 249, 258, 242, 264, 287, 266, 260, 255, 269, 267, 284, 241, 244, 253, 252, 249, 264, 234, 277, 248, 250, 248, 264, 251, 283, 291, 231, 240, 257, 254, 241, 246, 249, 248] payload_rows_per_dst=[5398, 5494, 5558, 5468, 5480, 5542, 5512, 5486] dup_rows_per_dst=[2684, 2676, 2736, 2704, 2656, 2722, 2750, 2670] combine_rows_per_src=[5398, 5494, 5558, 5468, 5480, 5542, 5512, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=156540928 dispatch_meta_bytes=228896 combine_payload_bytes=156540928 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156769824 ingress_bytes=213164032 link_bytes=213164032 token_bytes=313310752 weight_movement_bytes=56623104 total_nvlink_bytes=369933856 weight_to_token_ratio=0.180725 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.042432871e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229640 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157098248 ingress_bytes=213491712 link_bytes=213491712 token_bytes=313966856 weight_movement_bytes=56623104 total_nvlink_bytes=370589960 weight_to_token_ratio=0.180347 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045572540e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38376 dup_rows=21726 dup_rows_remote=19094 dedup_ratio=0.331512 router_tokens_per_expert=[247, 242, 227, 249, 251, 257, 271, 255, 249, 243, 247, 246, 267, 238, 236, 251, 265, 233, 262, 257, 252, 264, 279, 263, 257, 267, 269, 261, 250, 229, 265, 255, 272, 259, 279, 270, 261, 280, 239, 260, 251, 237, 269, 271, 250, 264, 227, 260, 267, 224, 258, 265, 247, 237, 262, 275, 266, 253, 271, 269, 251, 288, 240, 257, 244, 296, 258, 276, 294, 252, 273, 276, 245, 259, 258, 255, 237, 239, 262, 274, 259, 233, 237, 288, 266, 262, 234, 269, 248, 255, 270, 241, 233, 263, 227, 248, 242, 248, 231, 232, 231, 257, 260, 266, 282, 279, 259, 233, 252, 254, 246, 234, 262, 248, 257, 252, 242, 271, 275, 252, 288, 265, 256, 273, 254, 251, 254, 248] payload_rows_per_dst=[5430, 5438, 5554, 5512, 5528, 5434, 5420, 5494] dup_rows_per_dst=[2522, 2818, 2744, 2748, 2868, 2632, 2592, 2802] combine_rows_per_src=[5430, 5438, 5554, 5512, 5528, 5434, 5420, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38276 dup_rows=21834 dup_rows_remote=19084 dedup_ratio=0.333160 router_tokens_per_expert=[236, 249, 228, 229, 251, 251, 278, 246, 258, 258, 292, 271, 258, 257, 232, 254, 242, 242, 260, 253, 247, 243, 261, 250, 246, 289, 258, 270, 259, 239, 250, 242, 243, 243, 278, 252, 242, 255, 268, 273, 257, 248, 262, 231, 263, 262, 248, 263, 272, 274, 239, 238, 266, 267, 257, 248, 262, 246, 270, 274, 272, 251, 242, 259, 249, 245, 255, 267, 269, 266, 248, 278, 252, 272, 269, 264, 272, 237, 245, 261, 245, 244, 238, 243, 254, 259, 248, 247, 277, 266, 277, 254, 257, 265, 269, 230, 222, 288, 251, 264, 250, 267, 249, 276, 262, 260, 259, 261, 253, 262, 237, 233, 258, 253, 274, 266, 276, 250, 253, 256, 265, 274, 265, 249, 254, 222, 241, 272] payload_rows_per_dst=[5424, 5470, 5426, 5496, 5508, 5398, 5500, 5480] dup_rows_per_dst=[2672, 2632, 2750, 2778, 2790, 2748, 2688, 2776] combine_rows_per_src=[5424, 5470, 5426, 5496, 5508, 5398, 5500, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38350 dup_rows=21732 dup_rows_remote=19040 dedup_ratio=0.331604 router_tokens_per_expert=[242, 252, 258, 276, 254, 253, 263, 267, 239, 288, 236, 236, 279, 249, 242, 235, 247, 266, 246, 253, 240, 248, 265, 279, 233, 255, 287, 248, 241, 270, 261, 265, 262, 269, 257, 248, 277, 249, 259, 299, 244, 235, 272, 272, 252, 240, 252, 263, 251, 239, 272, 259, 262, 244, 242, 271, 253, 229, 266, 241, 275, 248, 240, 264, 231, 235, 241, 255, 253, 269, 257, 216, 246, 282, 285, 274, 254, 299, 235, 241, 261, 239, 256, 256, 249, 267, 236, 262, 284, 259, 250, 282, 226, 250, 250, 265, 244, 274, 252, 278, 243, 278, 272, 239, 266, 251, 238, 283, 234, 236, 251, 250, 255, 281, 251, 272, 271, 290, 236, 231, 238, 249, 246, 269, 230, 284, 270, 262] payload_rows_per_dst=[5414, 5472, 5540, 5440, 5454, 5468, 5506, 5510] dup_rows_per_dst=[2724, 2736, 2760, 2672, 2692, 2716, 2672, 2760] combine_rows_per_src=[5414, 5472, 5540, 5440, 5454, 5468, 5506, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157466624 dispatch_meta_bytes=229464 combine_payload_bytes=157466624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157696088 ingress_bytes=176340992 link_bytes=176340992 token_bytes=315162712 weight_movement_bytes=18874368 total_nvlink_bytes=334037080 weight_to_token_ratio=0.059888 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689612620e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38380 dup_rows=21694 dup_rows_remote=18924 dedup_ratio=0.331024 router_tokens_per_expert=[246, 251, 250, 281, 223, 244, 267, 270, 270, 271, 260, 245, 279, 278, 260, 250, 246, 268, 268, 254, 247, 258, 257, 268, 241, 236, 260, 250, 252, 231, 253, 248, 260, 266, 264, 238, 252, 239, 228, 260, 269, 258, 264, 247, 251, 243, 265, 221, 286, 257, 274, 252, 244, 268, 265, 282, 256, 258, 260, 244, 273, 262, 255, 253, 255, 236, 239, 272, 248, 244, 244, 278, 234, 261, 273, 255, 253, 240, 258, 251, 236, 261, 244, 255, 247, 289, 245, 261, 224, 288, 276, 252, 272, 262, 261, 256, 253, 255, 252, 244, 255, 242, 284, 241, 251, 261, 260, 255, 267, 281, 262, 253, 262, 248, 266, 270, 288, 264, 272, 230, 238, 243, 247, 258, 253, 260, 257, 230] payload_rows_per_dst=[5574, 5460, 5372, 5552, 5510, 5454, 5462, 5458] dup_rows_per_dst=[2716, 2614, 2678, 2826, 2572, 2804, 2770, 2714] combine_rows_per_src=[5574, 5460, 5372, 5552, 5510, 5454, 5462, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43748 payload_rows_remote=38296 dup_rows=21788 dup_rows_remote=19246 dedup_ratio=0.332458 router_tokens_per_expert=[226, 269, 223, 293, 274, 256, 283, 232, 261, 240, 262, 257, 249, 268, 250, 266, 234, 276, 278, 264, 241, 264, 257, 243, 257, 263, 257, 270, 256, 266, 244, 246, 286, 254, 251, 263, 282, 261, 246, 260, 265, 246, 277, 239, 285, 244, 265, 241, 227, 259, 243, 250, 245, 248, 251, 217, 253, 242, 295, 252, 243, 253, 267, 252, 257, 251, 257, 227, 255, 266, 253, 218, 257, 248, 279, 275, 262, 255, 241, 233, 257, 276, 263, 231, 272, 286, 240, 259, 254, 244, 249, 238, 266, 228, 251, 251, 263, 274, 273, 234, 259, 295, 256, 263, 264, 269, 238, 276, 255, 241, 244, 256, 252, 247, 250, 260, 272, 245, 268, 257, 261, 239, 269, 241, 267, 262, 268, 264] payload_rows_per_dst=[5444, 5512, 5462, 5452, 5480, 5400, 5480, 5518] dup_rows_per_dst=[2774, 2720, 2868, 2542, 2588, 2730, 2840, 2726] combine_rows_per_src=[5444, 5512, 5462, 5452, 5480, 5400, 5480, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=229880 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157417976 ingress_bytes=176062464 link_bytes=176062464 token_bytes=314606072 weight_movement_bytes=18874368 total_nvlink_bytes=333480440 weight_to_token_ratio=0.059994 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686943902e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229440 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157007936 ingress_bytes=175652864 link_bytes=175652864 token_bytes=313786432 weight_movement_bytes=18874368 total_nvlink_bytes=332660800 weight_to_token_ratio=0.060150 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683019316e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157081600 dispatch_meta_bytes=229560 combine_payload_bytes=157081600 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157311160 ingress_bytes=175955968 link_bytes=175955968 token_bytes=314392760 weight_movement_bytes=18874368 total_nvlink_bytes=333267128 weight_to_token_ratio=0.060034 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685923510e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43904 payload_rows_remote=38330 dup_rows=21632 dup_rows_remote=18752 dedup_ratio=0.330078 router_tokens_per_expert=[257, 229, 244, 240, 246, 254, 258, 247, 263, 262, 237, 243, 253, 258, 226, 233, 242, 263, 258, 247, 254, 267, 263, 244, 262, 261, 277, 277, 272, 243, 250, 271, 272, 249, 233, 246, 274, 260, 261, 242, 271, 264, 258, 285, 249, 273, 277, 268, 267, 283, 221, 292, 273, 264, 277, 247, 239, 255, 265, 273, 249, 226, 281, 279, 267, 237, 218, 267, 291, 266, 266, 263, 253, 247, 294, 247, 293, 265, 257, 252, 237, 255, 254, 258, 248, 262, 263, 214, 250, 236, 281, 262, 246, 240, 245, 233, 248, 245, 222, 271, 250, 260, 267, 248, 237, 237, 257, 238, 248, 250, 294, 262, 262, 258, 253, 239, 275, 267, 237, 258, 257, 283, 243, 248, 246, 240, 261, 266] payload_rows_per_dst=[5574, 5390, 5576, 5596, 5408, 5436, 5496, 5428] dup_rows_per_dst=[2880, 2430, 2926, 2834, 2772, 2532, 2572, 2686] combine_rows_per_src=[5574, 5390, 5576, 5596, 5408, 5436, 5496, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=157204480 dispatch_meta_bytes=229216 combine_payload_bytes=157204480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157433696 ingress_bytes=194953216 link_bytes=194953216 token_bytes=314638176 weight_movement_bytes=37748736 total_nvlink_bytes=352386912 weight_to_token_ratio=0.119975 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867945793e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=230168 combine_payload_bytes=156860416 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157090584 ingress_bytes=213483520 link_bytes=213483520 token_bytes=313951000 weight_movement_bytes=56623104 total_nvlink_bytes=370574104 weight_to_token_ratio=0.180357 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045494048e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43956 payload_rows_remote=38450 dup_rows=21580 dup_rows_remote=18772 dedup_ratio=0.329285 router_tokens_per_expert=[264, 246, 271, 254, 238, 305, 255, 269, 259, 256, 270, 277, 254, 267, 255, 282, 255, 269, 283, 253, 244, 252, 286, 258, 250, 263, 235, 204, 258, 226, 249, 254, 247, 263, 254, 245, 264, 256, 268, 278, 245, 278, 225, 236, 257, 265, 270, 274, 260, 256, 274, 260, 274, 244, 241, 244, 255, 265, 241, 266, 263, 247, 276, 236, 242, 275, 278, 272, 281, 248, 254, 280, 260, 244, 258, 283, 274, 217, 231, 247, 253, 252, 258, 272, 275, 262, 230, 216, 228, 222, 268, 256, 244, 261, 280, 247, 266, 233, 260, 252, 213, 261, 246, 262, 270, 266, 217, 255, 259, 249, 241, 256, 224, 248, 251, 235, 250, 266, 254, 297, 249, 260, 251, 266, 276, 249, 269, 261] payload_rows_per_dst=[5596, 5486, 5524, 5452, 5534, 5384, 5474, 5506] dup_rows_per_dst=[2744, 2592, 2796, 2566, 2754, 2562, 2758, 2808] combine_rows_per_src=[5596, 5486, 5524, 5452, 5534, 5384, 5474, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43608 payload_rows_remote=38190 dup_rows=21928 dup_rows_remote=19126 dedup_ratio=0.334595 router_tokens_per_expert=[242, 232, 253, 263, 272, 262, 253, 258, 263, 245, 261, 250, 234, 257, 226, 249, 246, 250, 266, 236, 245, 249, 256, 240, 246, 261, 252, 254, 245, 232, 248, 289, 244, 250, 272, 234, 280, 272, 262, 245, 237, 290, 268, 255, 270, 255, 262, 277, 262, 263, 271, 265, 259, 263, 245, 292, 270, 261, 271, 235, 271, 248, 252, 235, 282, 255, 269, 267, 252, 231, 248, 223, 248, 280, 271, 230, 265, 241, 257, 260, 243, 296, 247, 286, 277, 268, 279, 256, 255, 234, 253, 258, 258, 245, 255, 261, 248, 260, 267, 248, 258, 259, 229, 259, 235, 250, 260, 282, 245, 258, 257, 245, 256, 244, 260, 246, 260, 269, 242, 264, 256, 280, 259, 236, 246, 271, 241, 257] payload_rows_per_dst=[5344, 5418, 5530, 5520, 5370, 5502, 5442, 5482] dup_rows_per_dst=[2696, 2802, 2816, 2806, 2598, 2840, 2678, 2692] combine_rows_per_src=[5344, 5418, 5530, 5520, 5370, 5502, 5442, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=228328 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228008 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314227688 weight_movement_bytes=18874368 total_nvlink_bytes=333102056 weight_to_token_ratio=0.060066 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157491200 dispatch_meta_bytes=228888 combine_payload_bytes=157491200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157720088 ingress_bytes=195239936 link_bytes=195239936 token_bytes=315211288 weight_movement_bytes=37748736 total_nvlink_bytes=352960024 weight_to_token_ratio=0.119757 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870693003e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43892 payload_rows_remote=38478 dup_rows=21644 dup_rows_remote=19028 dedup_ratio=0.330261 router_tokens_per_expert=[273, 259, 257, 238, 264, 260, 263, 269, 237, 240, 255, 267, 256, 259, 259, 251, 255, 244, 271, 242, 255, 271, 265, 256, 266, 263, 269, 251, 260, 236, 267, 251, 247, 264, 252, 259, 269, 236, 261, 232, 246, 273, 242, 242, 268, 239, 256, 245, 256, 243, 264, 272, 262, 259, 277, 250, 270, 259, 247, 276, 258, 255, 253, 266, 253, 245, 249, 251, 277, 260, 281, 268, 269, 274, 276, 257, 257, 251, 244, 257, 275, 259, 253, 239, 275, 253, 241, 242, 237, 246, 250, 274, 237, 233, 261, 240, 247, 224, 231, 266, 225, 248, 260, 249, 264, 271, 285, 280, 250, 257, 253, 276, 246, 262, 273, 245, 237, 236, 253, 240, 276, 263, 249, 254, 282, 257, 243, 255] payload_rows_per_dst=[5470, 5516, 5414, 5614, 5584, 5414, 5458, 5422] dup_rows_per_dst=[2744, 2728, 2648, 2720, 2754, 2616, 2714, 2720] combine_rows_per_src=[5470, 5516, 5414, 5614, 5584, 5414, 5458, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43588 payload_rows_remote=38178 dup_rows=21948 dup_rows_remote=19328 dedup_ratio=0.334900 router_tokens_per_expert=[244, 219, 257, 257, 228, 269, 252, 244, 257, 257, 245, 262, 278, 251, 233, 245, 229, 266, 237, 245, 255, 270, 280, 266, 284, 252, 263, 271, 253, 231, 276, 232, 281, 264, 255, 256, 212, 230, 221, 238, 273, 243, 282, 251, 238, 275, 263, 233, 254, 262, 263, 247, 236, 275, 254, 258, 289, 272, 253, 243, 263, 248, 259, 280, 247, 255, 250, 254, 268, 272, 278, 279, 302, 261, 257, 268, 273, 271, 258, 242, 258, 246, 256, 263, 232, 273, 260, 243, 237, 251, 252, 253, 265, 252, 243, 267, 261, 266, 262, 250, 284, 282, 234, 239, 264, 245, 249, 289, 244, 245, 279, 289, 258, 229, 257, 249, 223, 259, 254, 263, 268, 270, 218, 261, 256, 266, 236, 254] payload_rows_per_dst=[5410, 5508, 5410, 5530, 5524, 5378, 5450, 5378] dup_rows_per_dst=[2678, 2712, 2620, 2782, 2946, 2632, 2914, 2664] combine_rows_per_src=[5410, 5508, 5410, 5530, 5524, 5378, 5450, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156426240 dispatch_meta_bytes=229264 combine_payload_bytes=156426240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156655504 ingress_bytes=175300608 link_bytes=175300608 token_bytes=313081744 weight_movement_bytes=18874368 total_nvlink_bytes=331956112 weight_to_token_ratio=0.060286 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679644173e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43932 payload_rows_remote=38416 dup_rows=21604 dup_rows_remote=18854 dedup_ratio=0.329651 router_tokens_per_expert=[252, 240, 282, 260, 225, 242, 269, 232, 250, 260, 277, 237, 255, 245, 249, 273, 245, 277, 262, 233, 212, 246, 267, 217, 257, 240, 238, 264, 231, 258, 263, 245, 237, 253, 254, 268, 251, 272, 229, 268, 256, 267, 275, 248, 262, 252, 242, 249, 243, 228, 259, 280, 262, 242, 237, 259, 301, 226, 241, 279, 265, 215, 273, 244, 258, 229, 252, 281, 259, 263, 243, 239, 247, 271, 258, 283, 257, 263, 281, 249, 236, 261, 275, 270, 265, 262, 260, 262, 255, 248, 270, 272, 266, 287, 269, 274, 262, 236, 264, 251, 262, 269, 288, 231, 273, 260, 274, 267, 260, 241, 258, 229, 256, 275, 277, 250, 252, 250, 226, 251, 242, 263, 275, 273, 253, 248, 295, 252] payload_rows_per_dst=[5484, 5412, 5504, 5424, 5516, 5576, 5506, 5510] dup_rows_per_dst=[2612, 2498, 2662, 2684, 2750, 2888, 2744, 2766] combine_rows_per_src=[5484, 5412, 5504, 5424, 5516, 5576, 5506, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38202 dup_rows=21868 dup_rows_remote=19054 dedup_ratio=0.333679 router_tokens_per_expert=[271, 258, 237, 273, 254, 255, 245, 285, 270, 246, 256, 246, 243, 286, 236, 257, 243, 257, 259, 258, 243, 256, 256, 238, 258, 250, 233, 274, 243, 256, 259, 299, 251, 242, 283, 260, 257, 272, 251, 268, 257, 251, 236, 258, 247, 262, 275, 246, 244, 282, 225, 265, 262, 260, 256, 258, 250, 254, 269, 234, 255, 258, 289, 246, 233, 250, 263, 266, 276, 252, 245, 264, 264, 257, 272, 244, 267, 254, 244, 259, 257, 242, 271, 263, 239, 271, 263, 240, 231, 264, 233, 261, 275, 255, 285, 261, 253, 281, 239, 251, 247, 260, 255, 249, 275, 280, 249, 265, 272, 277, 245, 242, 237, 254, 275, 260, 237, 234, 225, 222, 259, 258, 250, 259, 271, 248, 252, 243] payload_rows_per_dst=[5498, 5466, 5464, 5502, 5482, 5424, 5466, 5366] dup_rows_per_dst=[2738, 2698, 2768, 2712, 2738, 2798, 2814, 2602] combine_rows_per_src=[5498, 5466, 5464, 5502, 5482, 5424, 5466, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157605888 dispatch_meta_bytes=230024 combine_payload_bytes=157605888 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157835912 ingress_bytes=195354624 link_bytes=195354624 token_bytes=315441800 weight_movement_bytes=37748736 total_nvlink_bytes=353190536 weight_to_token_ratio=0.119669 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871791887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=230024 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156607112 ingress_bytes=175251456 link_bytes=175251456 token_bytes=312984200 weight_movement_bytes=18874368 total_nvlink_bytes=331858568 weight_to_token_ratio=0.060305 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679173222e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38274 dup_rows=21816 dup_rows_remote=19120 dedup_ratio=0.332886 router_tokens_per_expert=[255, 241, 273, 283, 262, 260, 270, 290, 259, 286, 254, 236, 239, 246, 275, 246, 282, 264, 252, 237, 226, 248, 253, 249, 226, 245, 262, 256, 277, 269, 236, 253, 271, 274, 248, 221, 252, 249, 271, 226, 254, 239, 234, 253, 285, 262, 237, 265, 261, 259, 253, 256, 252, 248, 257, 251, 280, 257, 230, 261, 253, 247, 242, 264, 262, 298, 261, 265, 247, 285, 247, 254, 258, 242, 254, 240, 231, 256, 258, 256, 257, 259, 256, 247, 272, 236, 240, 247, 287, 245, 262, 255, 276, 293, 240, 266, 289, 261, 261, 246, 231, 274, 248, 252, 213, 242, 248, 230, 240, 261, 269, 238, 252, 287, 249, 274, 256, 253, 262, 260, 257, 277, 251, 274, 280, 262, 232, 265] payload_rows_per_dst=[5560, 5424, 5402, 5446, 5442, 5494, 5422, 5530] dup_rows_per_dst=[2790, 2646, 2680, 2696, 2786, 2782, 2584, 2852] combine_rows_per_src=[5560, 5424, 5402, 5446, 5442, 5494, 5422, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=229080 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157581016 ingress_bytes=195100672 link_bytes=195100672 token_bytes=314932952 weight_movement_bytes=37748736 total_nvlink_bytes=352681688 weight_to_token_ratio=0.119863 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869358644e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229024 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704416 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179808 weight_movement_bytes=18874368 total_nvlink_bytes=332054176 weight_to_token_ratio=0.060267 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38222 dup_rows=21842 dup_rows_remote=19070 dedup_ratio=0.333282 router_tokens_per_expert=[250, 267, 269, 247, 251, 266, 243, 269, 256, 248, 250, 262, 219, 253, 267, 245, 245, 276, 275, 243, 237, 245, 246, 261, 258, 244, 238, 240, 289, 248, 265, 235, 264, 252, 257, 248, 243, 243, 251, 292, 273, 246, 246, 229, 243, 256, 254, 255, 233, 249, 249, 245, 261, 273, 269, 246, 233, 271, 267, 242, 270, 268, 280, 256, 251, 252, 244, 256, 251, 272, 251, 249, 273, 258, 266, 252, 246, 265, 268, 242, 242, 262, 217, 273, 283, 271, 260, 267, 227, 260, 257, 257, 276, 264, 229, 264, 266, 254, 229, 242, 270, 288, 262, 274, 275, 271, 256, 259, 233, 249, 261, 262, 265, 241, 250, 267, 260, 260, 281, 263, 255, 274, 286, 245, 261, 254, 250, 229] payload_rows_per_dst=[5472, 5322, 5582, 5462, 5336, 5442, 5534, 5544] dup_rows_per_dst=[2772, 2704, 2726, 2714, 2644, 2776, 2768, 2738] combine_rows_per_src=[5472, 5322, 5582, 5462, 5336, 5442, 5534, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229576 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156999880 ingress_bytes=194519040 link_bytes=194519040 token_bytes=313770184 weight_movement_bytes=37748736 total_nvlink_bytes=351518920 weight_to_token_ratio=0.120307 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863785732e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38156 dup_rows=21726 dup_rows_remote=18966 dedup_ratio=0.331512 router_tokens_per_expert=[240, 258, 231, 262, 256, 251, 251, 235, 249, 257, 274, 246, 264, 230, 255, 277, 251, 237, 243, 248, 232, 259, 243, 254, 246, 273, 256, 244, 265, 240, 247, 272, 242, 253, 254, 253, 257, 260, 281, 269, 270, 275, 261, 265, 228, 265, 274, 251, 224, 250, 240, 252, 272, 243, 298, 249, 252, 268, 247, 229, 256, 284, 230, 259, 270, 248, 242, 241, 280, 239, 267, 250, 256, 273, 271, 270, 242, 276, 279, 245, 267, 232, 277, 269, 260, 286, 275, 256, 237, 283, 235, 268, 250, 280, 251, 269, 247, 264, 252, 253, 266, 266, 241, 275, 244, 219, 227, 249, 242, 273, 273, 281, 256, 260, 262, 268, 242, 245, 254, 254, 240, 284, 240, 237, 300, 260, 251, 242] payload_rows_per_dst=[5356, 5394, 5542, 5294, 5562, 5528, 5480, 5654] dup_rows_per_dst=[2716, 2626, 2774, 2588, 2688, 2862, 2712, 2760] combine_rows_per_src=[5356, 5394, 5542, 5294, 5562, 5528, 5480, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38264 dup_rows=21804 dup_rows_remote=18958 dedup_ratio=0.332703 router_tokens_per_expert=[258, 240, 237, 239, 249, 258, 233, 256, 234, 257, 256, 264, 270, 270, 250, 229, 266, 277, 273, 268, 268, 250, 267, 250, 253, 284, 230, 268, 257, 263, 251, 232, 261, 222, 245, 253, 234, 240, 257, 291, 222, 266, 252, 242, 250, 220, 244, 268, 271, 244, 268, 256, 277, 232, 241, 254, 266, 286, 276, 291, 247, 239, 275, 256, 252, 263, 270, 262, 241, 236, 266, 270, 259, 252, 249, 249, 269, 249, 260, 260, 249, 290, 282, 251, 274, 255, 273, 266, 261, 246, 231, 248, 284, 223, 248, 253, 228, 255, 236, 246, 264, 281, 248, 257, 255, 244, 238, 278, 287, 244, 263, 267, 249, 275, 252, 243, 252, 280, 267, 273, 269, 298, 233, 250, 228, 228, 267, 269] payload_rows_per_dst=[5560, 5468, 5338, 5534, 5452, 5416, 5490, 5474] dup_rows_per_dst=[2950, 2846, 2596, 2824, 2762, 2518, 2692, 2616] combine_rows_per_src=[5560, 5468, 5338, 5534, 5452, 5416, 5490, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229168 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156786480 ingress_bytes=175431680 link_bytes=175431680 token_bytes=313343792 weight_movement_bytes=18874368 total_nvlink_bytes=332218160 weight_to_token_ratio=0.060235 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680900040e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=228488 combine_payload_bytes=156286976 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156515464 ingress_bytes=194035712 link_bytes=194035712 token_bytes=312802440 weight_movement_bytes=37748736 total_nvlink_bytes=350551176 weight_to_token_ratio=0.120679 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859154721e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38446 dup_rows=21732 dup_rows_remote=19056 dedup_ratio=0.331604 router_tokens_per_expert=[267, 263, 249, 257, 244, 257, 249, 258, 253, 265, 254, 276, 257, 280, 267, 258, 267, 265, 226, 265, 282, 278, 280, 258, 252, 247, 209, 245, 251, 282, 255, 284, 228, 248, 232, 243, 262, 239, 258, 243, 254, 263, 237, 260, 283, 239, 256, 272, 261, 268, 229, 251, 256, 239, 255, 260, 275, 269, 286, 262, 276, 232, 265, 275, 236, 253, 235, 241, 244, 238, 260, 278, 225, 240, 235, 223, 241, 259, 299, 251, 243, 260, 236, 281, 254, 281, 277, 277, 258, 248, 262, 264, 272, 261, 258, 241, 251, 271, 238, 263, 261, 262, 281, 267, 274, 262, 249, 241, 231, 251, 269, 251, 254, 263, 235, 245, 245, 224, 272, 276, 278, 239, 255, 275, 246, 269, 227, 236] payload_rows_per_dst=[5520, 5542, 5358, 5548, 5336, 5550, 5538, 5412] dup_rows_per_dst=[2788, 2750, 2676, 2770, 2580, 2796, 2706, 2666] combine_rows_per_src=[5520, 5542, 5358, 5548, 5336, 5550, 5538, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38298 dup_rows=21760 dup_rows_remote=19062 dedup_ratio=0.332031 router_tokens_per_expert=[303, 269, 226, 236, 245, 263, 244, 283, 259, 244, 261, 237, 256, 201, 254, 253, 273, 261, 274, 251, 265, 260, 251, 231, 231, 219, 247, 249, 254, 258, 252, 291, 295, 251, 254, 265, 264, 252, 280, 227, 250, 235, 254, 242, 242, 253, 266, 240, 247, 237, 242, 248, 258, 248, 247, 270, 264, 254, 251, 275, 254, 282, 261, 256, 256, 268, 254, 259, 259, 236, 263, 251, 265, 262, 259, 273, 271, 241, 241, 254, 225, 242, 267, 255, 249, 256, 257, 218, 293, 291, 250, 254, 259, 274, 253, 245, 278, 262, 266, 234, 273, 267, 261, 266, 235, 236, 247, 277, 243, 268, 257, 260, 250, 239, 262, 254, 276, 236, 262, 303, 275, 252, 241, 258, 264, 276, 267, 258] payload_rows_per_dst=[5474, 5444, 5444, 5480, 5514, 5478, 5428, 5514] dup_rows_per_dst=[2594, 2690, 2696, 2708, 2710, 2698, 2832, 2832] combine_rows_per_src=[5474, 5444, 5444, 5480, 5514, 5478, 5428, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=228888 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958232 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687576 weight_movement_bytes=18874368 total_nvlink_bytes=332561944 weight_to_token_ratio=0.060169 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43952 payload_rows_remote=38492 dup_rows=21584 dup_rows_remote=18866 dedup_ratio=0.329346 router_tokens_per_expert=[255, 260, 248, 257, 238, 246, 251, 259, 258, 219, 230, 252, 247, 230, 262, 266, 284, 228, 232, 242, 268, 240, 278, 262, 249, 252, 241, 240, 258, 257, 237, 263, 268, 271, 234, 235, 237, 258, 232, 264, 275, 253, 269, 243, 283, 267, 262, 255, 247, 233, 256, 269, 256, 258, 259, 258, 232, 250, 243, 278, 272, 274, 262, 257, 256, 268, 239, 246, 260, 254, 297, 256, 255, 263, 252, 281, 232, 239, 236, 255, 254, 247, 279, 296, 269, 265, 277, 262, 258, 230, 251, 274, 247, 251, 282, 255, 275, 268, 259, 248, 232, 271, 241, 261, 251, 262, 249, 254, 248, 235, 291, 258, 246, 251, 280, 275, 248, 287, 242, 252, 241, 274, 272, 248, 263, 256, 251, 274] payload_rows_per_dst=[5380, 5488, 5476, 5514, 5460, 5594, 5538, 5502] dup_rows_per_dst=[2576, 2574, 2736, 2694, 2718, 2800, 2668, 2818] combine_rows_per_src=[5380, 5488, 5476, 5514, 5460, 5594, 5538, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38158 dup_rows=21766 dup_rows_remote=19024 dedup_ratio=0.332123 router_tokens_per_expert=[254, 244, 260, 243, 273, 269, 273, 254, 258, 243, 239, 269, 243, 242, 232, 240, 263, 259, 269, 251, 223, 252, 216, 249, 260, 246, 300, 260, 270, 270, 275, 260, 280, 263, 270, 242, 260, 232, 268, 283, 247, 262, 283, 271, 277, 243, 257, 239, 270, 237, 242, 258, 248, 256, 233, 260, 242, 283, 245, 257, 231, 245, 254, 268, 253, 276, 249, 267, 245, 241, 270, 258, 257, 239, 277, 246, 274, 253, 239, 255, 244, 266, 217, 275, 252, 255, 233, 260, 241, 222, 258, 254, 295, 246, 229, 271, 259, 254, 237, 252, 248, 245, 305, 275, 272, 227, 293, 262, 284, 256, 246, 262, 280, 265, 251, 259, 234, 251, 268, 237, 266, 276, 240, 273, 251, 253, 238, 267] payload_rows_per_dst=[5392, 5452, 5512, 5412, 5498, 5412, 5612, 5480] dup_rows_per_dst=[2680, 2794, 2842, 2646, 2700, 2624, 2742, 2738] combine_rows_per_src=[5392, 5452, 5512, 5412, 5498, 5412, 5612, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38306 dup_rows=21664 dup_rows_remote=18924 dedup_ratio=0.330566 router_tokens_per_expert=[221, 234, 298, 235, 250, 235, 233, 234, 254, 268, 280, 254, 266, 274, 258, 272, 268, 266, 256, 247, 243, 238, 256, 236, 232, 257, 238, 230, 278, 275, 229, 236, 248, 272, 297, 271, 261, 271, 245, 251, 245, 262, 241, 229, 289, 247, 263, 229, 263, 271, 269, 258, 274, 266, 249, 286, 237, 243, 247, 248, 276, 243, 280, 243, 257, 281, 253, 229, 248, 250, 232, 242, 254, 250, 264, 268, 287, 255, 240, 251, 293, 266, 255, 243, 243, 261, 261, 245, 271, 284, 248, 260, 262, 270, 268, 254, 260, 253, 258, 248, 216, 267, 254, 267, 240, 239, 238, 260, 288, 257, 251, 273, 249, 246, 251, 259, 279, 249, 242, 276, 270, 256, 242, 275, 250, 249, 257, 279] payload_rows_per_dst=[5510, 5376, 5490, 5566, 5456, 5536, 5448, 5490] dup_rows_per_dst=[2622, 2594, 2752, 2740, 2666, 2832, 2690, 2768] combine_rows_per_src=[5510, 5376, 5490, 5566, 5456, 5536, 5448, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=230008 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157704824 ingress_bytes=176349184 link_bytes=176349184 token_bytes=315179640 weight_movement_bytes=18874368 total_nvlink_bytes=334054008 weight_to_token_ratio=0.059884 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689691112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229440 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157098048 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966656 weight_movement_bytes=37748736 total_nvlink_bytes=351715392 weight_to_token_ratio=0.120232 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157663232 dispatch_meta_bytes=229432 combine_payload_bytes=157663232 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157892664 ingress_bytes=214286336 link_bytes=214286336 token_bytes=315555896 weight_movement_bytes=56623104 total_nvlink_bytes=372179000 weight_to_token_ratio=0.179439 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053186236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=228728 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156523896 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312819064 weight_movement_bytes=37748736 total_nvlink_bytes=350567800 weight_to_token_ratio=0.120673 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228920 combine_payload_bytes=156901376 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157130296 ingress_bytes=194650112 link_bytes=194650112 token_bytes=314031672 weight_movement_bytes=37748736 total_nvlink_bytes=351780408 weight_to_token_ratio=0.120207 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865041599e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38192 dup_rows=21698 dup_rows_remote=18620 dedup_ratio=0.331085 router_tokens_per_expert=[263, 254, 245, 256, 288, 255, 257, 241, 226, 253, 267, 239, 263, 243, 268, 287, 240, 273, 224, 257, 249, 243, 250, 261, 279, 295, 258, 279, 226, 260, 277, 249, 267, 257, 253, 272, 257, 245, 268, 245, 273, 257, 262, 263, 275, 249, 263, 249, 250, 257, 247, 230, 267, 235, 257, 262, 264, 249, 262, 251, 224, 269, 238, 273, 276, 253, 241, 222, 280, 278, 262, 266, 233, 253, 251, 265, 243, 282, 275, 239, 274, 233, 274, 228, 256, 267, 257, 234, 231, 276, 273, 246, 209, 290, 302, 273, 242, 260, 264, 262, 247, 274, 244, 266, 263, 233, 226, 265, 253, 261, 256, 235, 255, 261, 263, 269, 253, 242, 261, 256, 250, 261, 225, 251, 256, 270, 242, 245] payload_rows_per_dst=[5646, 5540, 5354, 5426, 5552, 5574, 5414, 5332] dup_rows_per_dst=[3078, 2776, 2452, 2634, 2898, 2672, 2638, 2550] combine_rows_per_src=[5646, 5540, 5354, 5426, 5552, 5574, 5414, 5332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43414 payload_rows_remote=38062 dup_rows=22122 dup_rows_remote=19332 dedup_ratio=0.337555 router_tokens_per_expert=[274, 237, 239, 250, 265, 254, 273, 242, 254, 280, 273, 249, 263, 224, 276, 258, 276, 267, 243, 259, 239, 254, 271, 260, 246, 253, 252, 255, 234, 251, 285, 261, 279, 276, 227, 222, 246, 274, 265, 261, 238, 229, 275, 226, 250, 270, 249, 236, 255, 258, 257, 259, 263, 240, 232, 261, 241, 259, 273, 270, 241, 252, 223, 239, 254, 260, 237, 289, 258, 286, 239, 262, 273, 279, 257, 261, 285, 280, 246, 264, 230, 232, 277, 266, 240, 277, 250, 270, 246, 259, 244, 220, 271, 276, 249, 261, 246, 260, 263, 260, 246, 280, 261, 287, 269, 245, 251, 247, 270, 259, 257, 260, 270, 241, 230, 277, 251, 258, 273, 245, 259, 233, 251, 246, 240, 253, 257, 262] payload_rows_per_dst=[5358, 5422, 5494, 5352, 5544, 5422, 5470, 5352] dup_rows_per_dst=[2504, 2662, 2912, 2694, 2982, 2716, 2862, 2790] combine_rows_per_src=[5358, 5422, 5494, 5352, 5544, 5422, 5470, 5352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38304 dup_rows=21792 dup_rows_remote=19006 dedup_ratio=0.332520 router_tokens_per_expert=[261, 263, 254, 262, 268, 250, 256, 277, 261, 257, 256, 254, 250, 243, 280, 235, 254, 273, 253, 241, 259, 257, 256, 257, 273, 248, 232, 258, 273, 260, 233, 286, 242, 240, 230, 234, 235, 253, 261, 269, 266, 250, 253, 263, 269, 266, 252, 244, 260, 276, 258, 262, 260, 267, 242, 242, 240, 251, 281, 256, 254, 235, 256, 243, 257, 273, 267, 260, 248, 250, 254, 228, 242, 254, 270, 255, 268, 243, 292, 259, 251, 248, 275, 231, 231, 276, 251, 267, 271, 255, 249, 253, 257, 237, 244, 256, 257, 293, 242, 273, 259, 255, 241, 247, 270, 267, 259, 248, 257, 248, 244, 263, 263, 261, 262, 259, 253, 257, 245, 244, 248, 284, 261, 254, 267, 285, 252, 228] payload_rows_per_dst=[5552, 5440, 5396, 5420, 5490, 5420, 5496, 5530] dup_rows_per_dst=[2702, 2786, 2658, 2746, 2750, 2684, 2750, 2716] combine_rows_per_src=[5552, 5440, 5396, 5420, 5490, 5420, 5496, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=227248 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156661680 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313096112 weight_movement_bytes=18874368 total_nvlink_bytes=331970480 weight_to_token_ratio=0.060283 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38284 dup_rows=21796 dup_rows_remote=19048 dedup_ratio=0.332581 router_tokens_per_expert=[254, 246, 258, 239, 271, 259, 269, 255, 253, 258, 265, 273, 281, 278, 253, 236, 258, 230, 243, 286, 252, 240, 240, 239, 293, 245, 253, 261, 257, 229, 245, 286, 267, 252, 260, 240, 266, 273, 248, 273, 258, 247, 273, 215, 280, 271, 252, 227, 258, 245, 293, 244, 253, 260, 241, 269, 233, 248, 263, 249, 269, 278, 295, 239, 245, 263, 239, 251, 231, 252, 233, 264, 291, 232, 245, 254, 280, 307, 275, 244, 261, 215, 246, 236, 253, 272, 254, 257, 252, 258, 274, 289, 242, 272, 277, 251, 234, 260, 271, 265, 249, 224, 239, 254, 278, 252, 264, 237, 294, 252, 241, 262, 241, 256, 263, 268, 253, 261, 249, 238, 255, 248, 226, 251, 265, 228, 267, 264] payload_rows_per_dst=[5522, 5404, 5456, 5466, 5462, 5506, 5482, 5442] dup_rows_per_dst=[2774, 2710, 2748, 2808, 2750, 2712, 2670, 2624] combine_rows_per_src=[5522, 5404, 5456, 5466, 5462, 5506, 5482, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=229576 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156131528 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312033480 weight_movement_bytes=18874368 total_nvlink_bytes=330907848 weight_to_token_ratio=0.060488 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38442 dup_rows=21658 dup_rows_remote=18910 dedup_ratio=0.330475 router_tokens_per_expert=[276, 265, 267, 253, 225, 254, 267, 240, 257, 255, 251, 271, 254, 242, 248, 258, 245, 246, 240, 228, 241, 279, 265, 231, 245, 243, 263, 271, 268, 282, 254, 255, 242, 271, 269, 227, 244, 248, 267, 252, 258, 255, 279, 231, 269, 269, 267, 273, 247, 272, 253, 256, 270, 233, 255, 262, 240, 267, 232, 267, 268, 260, 290, 256, 268, 249, 255, 254, 271, 232, 245, 277, 219, 252, 248, 251, 290, 267, 248, 262, 269, 246, 262, 256, 256, 256, 259, 284, 256, 223, 245, 279, 239, 255, 254, 253, 263, 236, 252, 248, 244, 263, 276, 271, 264, 229, 255, 286, 238, 251, 232, 266, 278, 272, 223, 254, 236, 256, 272, 279, 261, 243, 259, 253, 259, 254, 254, 273] payload_rows_per_dst=[5458, 5552, 5528, 5458, 5454, 5436, 5438, 5554] dup_rows_per_dst=[2708, 2560, 2714, 2798, 2722, 2748, 2710, 2698] combine_rows_per_src=[5458, 5552, 5528, 5458, 5454, 5436, 5438, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38240 dup_rows=21794 dup_rows_remote=18988 dedup_ratio=0.332550 router_tokens_per_expert=[249, 250, 271, 247, 251, 258, 243, 256, 273, 272, 276, 265, 247, 259, 266, 266, 242, 234, 249, 247, 267, 255, 246, 259, 219, 264, 279, 247, 247, 259, 266, 268, 278, 287, 261, 261, 272, 227, 248, 270, 251, 283, 221, 268, 262, 243, 277, 244, 265, 252, 260, 266, 248, 245, 229, 232, 244, 227, 222, 244, 262, 254, 259, 273, 248, 268, 283, 259, 226, 269, 284, 266, 250, 245, 274, 259, 244, 261, 257, 261, 263, 240, 256, 273, 237, 267, 238, 282, 242, 251, 245, 253, 256, 257, 272, 272, 238, 266, 263, 236, 264, 258, 261, 243, 253, 273, 241, 300, 252, 284, 274, 242, 254, 266, 264, 243, 246, 230, 227, 247, 275, 266, 271, 263, 236, 243, 248, 251] payload_rows_per_dst=[5568, 5348, 5504, 5402, 5502, 5526, 5532, 5360] dup_rows_per_dst=[2730, 2748, 2802, 2562, 2806, 2682, 2764, 2700] combine_rows_per_src=[5568, 5348, 5504, 5402, 5502, 5526, 5532, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229240 combine_payload_bytes=156893184 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157122424 ingress_bytes=232390656 link_bytes=232390656 token_bytes=314015608 weight_movement_bytes=75497472 total_nvlink_bytes=389513080 weight_to_token_ratio=0.240426 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226652922e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38244 dup_rows=21766 dup_rows_remote=19050 dedup_ratio=0.332123 router_tokens_per_expert=[251, 246, 248, 233, 238, 251, 255, 244, 244, 249, 257, 247, 248, 265, 239, 245, 229, 249, 237, 272, 238, 280, 256, 253, 281, 268, 245, 243, 258, 242, 263, 244, 252, 263, 260, 266, 250, 253, 284, 243, 255, 265, 289, 292, 278, 237, 278, 260, 255, 244, 263, 276, 245, 255, 266, 261, 255, 288, 259, 276, 250, 249, 247, 232, 233, 241, 262, 256, 251, 251, 229, 244, 262, 257, 258, 265, 237, 234, 270, 261, 251, 273, 266, 250, 253, 259, 256, 268, 261, 261, 269, 245, 248, 241, 265, 230, 268, 257, 259, 256, 285, 260, 240, 278, 247, 261, 247, 243, 240, 273, 243, 264, 284, 267, 264, 264, 274, 272, 271, 247, 253, 273, 290, 244, 247, 231, 255, 240] payload_rows_per_dst=[5348, 5384, 5606, 5466, 5438, 5474, 5526, 5528] dup_rows_per_dst=[2572, 2732, 2844, 2776, 2584, 2718, 2716, 2824] combine_rows_per_src=[5348, 5384, 5606, 5466, 5438, 5474, 5526, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229328 combine_payload_bytes=156811264 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157040592 ingress_bytes=175685632 link_bytes=175685632 token_bytes=313851856 weight_movement_bytes=18874368 total_nvlink_bytes=332726224 weight_to_token_ratio=0.060138 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683333283e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43676 payload_rows_remote=38292 dup_rows=21860 dup_rows_remote=19210 dedup_ratio=0.333557 router_tokens_per_expert=[260, 255, 265, 277, 248, 270, 290, 270, 252, 258, 268, 246, 234, 264, 280, 248, 241, 257, 285, 271, 262, 267, 232, 255, 257, 251, 259, 263, 264, 271, 278, 238, 270, 229, 251, 269, 264, 255, 242, 228, 253, 229, 273, 249, 240, 272, 247, 272, 231, 254, 258, 239, 237, 250, 248, 262, 232, 258, 255, 276, 257, 243, 264, 253, 250, 264, 229, 250, 263, 254, 272, 267, 263, 273, 260, 232, 236, 233, 261, 250, 252, 215, 267, 244, 302, 238, 257, 231, 267, 261, 260, 274, 291, 241, 239, 288, 260, 246, 239, 253, 249, 267, 255, 237, 265, 249, 230, 277, 252, 253, 264, 274, 255, 258, 252, 266, 257, 240, 272, 279, 266, 263, 256, 260, 269, 241, 254, 230] payload_rows_per_dst=[5518, 5486, 5458, 5384, 5398, 5524, 5362, 5546] dup_rows_per_dst=[2852, 2816, 2628, 2650, 2716, 2730, 2778, 2690] combine_rows_per_src=[5518, 5486, 5458, 5384, 5398, 5524, 5362, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157458432 dispatch_meta_bytes=229408 combine_payload_bytes=157458432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157687840 ingress_bytes=195207168 link_bytes=195207168 token_bytes=315146272 weight_movement_bytes=37748736 total_nvlink_bytes=352895008 weight_to_token_ratio=0.119782 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.870379036e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228912 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156859952 ingress_bytes=175505408 link_bytes=175505408 token_bytes=313490992 weight_movement_bytes=18874368 total_nvlink_bytes=332365360 weight_to_token_ratio=0.060207 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681606465e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156647424 dispatch_meta_bytes=229176 combine_payload_bytes=156647424 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156876600 ingress_bytes=194396160 link_bytes=194396160 token_bytes=313524024 weight_movement_bytes=37748736 total_nvlink_bytes=351272760 weight_to_token_ratio=0.120401 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862608356e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=230008 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157074040 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313918072 weight_movement_bytes=18874368 total_nvlink_bytes=332792440 weight_to_token_ratio=0.060125 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43650 payload_rows_remote=38240 dup_rows=21886 dup_rows_remote=19132 dedup_ratio=0.333954 router_tokens_per_expert=[217, 269, 266, 248, 212, 250, 238, 276, 251, 256, 291, 253, 252, 252, 276, 269, 271, 249, 255, 267, 255, 274, 258, 252, 254, 262, 262, 247, 251, 224, 269, 262, 263, 243, 268, 238, 253, 278, 247, 242, 265, 258, 247, 254, 236, 256, 258, 270, 256, 247, 239, 261, 233, 245, 252, 264, 265, 238, 279, 272, 247, 277, 239, 246, 249, 235, 268, 245, 262, 252, 237, 254, 288, 231, 241, 261, 242, 232, 287, 262, 260, 238, 276, 239, 255, 268, 270, 238, 242, 237, 252, 239, 262, 244, 284, 257, 222, 242, 271, 260, 263, 247, 261, 254, 258, 234, 295, 268, 251, 277, 268, 273, 271, 267, 251, 251, 295, 255, 264, 265, 269, 255, 277, 276, 235, 246, 250, 266] payload_rows_per_dst=[5410, 5684, 5462, 5396, 5398, 5362, 5530, 5408] dup_rows_per_dst=[2754, 3064, 2690, 2724, 2532, 2748, 2758, 2616] combine_rows_per_src=[5410, 5684, 5462, 5396, 5398, 5362, 5530, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38312 dup_rows=21716 dup_rows_remote=18790 dedup_ratio=0.331360 router_tokens_per_expert=[241, 262, 239, 272, 261, 262, 242, 259, 273, 236, 264, 268, 221, 264, 271, 251, 286, 264, 271, 256, 251, 239, 242, 251, 243, 239, 285, 262, 260, 266, 251, 255, 261, 263, 257, 249, 263, 262, 262, 242, 242, 234, 285, 280, 248, 265, 272, 256, 242, 233, 240, 263, 258, 285, 273, 249, 275, 240, 242, 247, 265, 258, 249, 261, 260, 250, 229, 249, 235, 242, 253, 256, 254, 246, 247, 267, 256, 236, 270, 250, 231, 247, 272, 241, 235, 249, 241, 261, 268, 270, 270, 238, 233, 242, 271, 263, 252, 286, 238, 244, 254, 263, 269, 250, 266, 252, 286, 262, 252, 227, 273, 277, 232, 241, 256, 278, 238, 245, 290, 263, 248, 268, 246, 275, 267, 281, 281, 248] payload_rows_per_dst=[5288, 5350, 5492, 5680, 5404, 5542, 5556, 5508] dup_rows_per_dst=[2564, 2400, 2790, 3008, 2596, 2686, 2746, 2926] combine_rows_per_src=[5288, 5350, 5492, 5680, 5404, 5542, 5556, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38140 dup_rows=21862 dup_rows_remote=18928 dedup_ratio=0.333588 router_tokens_per_expert=[227, 248, 247, 243, 246, 272, 227, 248, 260, 242, 270, 243, 258, 268, 267, 258, 237, 271, 277, 278, 254, 226, 242, 279, 257, 242, 264, 231, 247, 260, 262, 270, 273, 256, 295, 236, 260, 256, 306, 240, 267, 261, 248, 278, 270, 254, 240, 294, 293, 240, 253, 280, 248, 234, 252, 261, 255, 256, 237, 261, 245, 250, 249, 262, 251, 243, 269, 247, 275, 271, 270, 253, 243, 257, 278, 251, 249, 271, 232, 258, 264, 233, 261, 245, 265, 278, 251, 245, 253, 259, 270, 253, 248, 290, 258, 232, 248, 295, 240, 249, 244, 261, 286, 283, 252, 238, 279, 246, 245, 246, 247, 250, 237, 230, 247, 247, 240, 252, 250, 271, 282, 252, 256, 275, 229, 231, 244, 262] payload_rows_per_dst=[5416, 5474, 5534, 5452, 5542, 5468, 5398, 5390] dup_rows_per_dst=[2632, 2720, 2934, 2700, 2694, 2742, 2820, 2620] combine_rows_per_src=[5416, 5474, 5534, 5452, 5542, 5468, 5398, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38346 dup_rows=21686 dup_rows_remote=18894 dedup_ratio=0.330902 router_tokens_per_expert=[250, 261, 278, 262, 232, 260, 273, 244, 292, 256, 286, 234, 239, 230, 254, 246, 256, 275, 261, 251, 265, 262, 263, 243, 270, 261, 256, 277, 241, 256, 260, 251, 268, 242, 259, 272, 257, 239, 279, 292, 219, 258, 254, 264, 265, 269, 254, 249, 240, 251, 228, 272, 281, 226, 253, 243, 237, 247, 249, 265, 264, 260, 237, 241, 242, 248, 247, 280, 260, 251, 239, 264, 234, 245, 271, 272, 245, 243, 254, 277, 285, 225, 263, 260, 241, 261, 285, 273, 266, 251, 255, 249, 239, 288, 252, 275, 222, 246, 272, 268, 245, 257, 248, 225, 238, 260, 261, 240, 237, 257, 248, 248, 263, 244, 244, 281, 292, 256, 265, 277, 280, 289, 260, 240, 218, 269, 252, 247] payload_rows_per_dst=[5478, 5504, 5542, 5372, 5490, 5526, 5390, 5548] dup_rows_per_dst=[2716, 2792, 2738, 2616, 2670, 2794, 2554, 2806] combine_rows_per_src=[5478, 5504, 5542, 5372, 5490, 5526, 5390, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=229488 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156860528 ingress_bytes=213254144 link_bytes=213254144 token_bytes=313491568 weight_movement_bytes=56623104 total_nvlink_bytes=370114672 weight_to_token_ratio=0.180621 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043296280e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=228408 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157154360 ingress_bytes=194674688 link_bytes=194674688 token_bytes=314080312 weight_movement_bytes=37748736 total_nvlink_bytes=351829048 weight_to_token_ratio=0.120188 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865277074e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38140 dup_rows=21902 dup_rows_remote=19094 dedup_ratio=0.334198 router_tokens_per_expert=[267, 255, 225, 260, 268, 255, 290, 258, 245, 246, 230, 252, 263, 259, 228, 239, 269, 248, 231, 224, 241, 253, 269, 257, 266, 266, 260, 258, 260, 274, 257, 250, 249, 225, 244, 269, 252, 269, 266, 248, 268, 270, 258, 294, 231, 221, 253, 264, 244, 262, 266, 244, 236, 265, 247, 259, 256, 274, 251, 270, 239, 258, 264, 259, 279, 246, 263, 240, 243, 265, 248, 235, 269, 270, 264, 259, 267, 263, 265, 236, 260, 259, 268, 261, 278, 234, 245, 270, 252, 274, 253, 263, 238, 265, 266, 265, 245, 249, 267, 264, 270, 274, 252, 281, 248, 241, 250, 274, 254, 253, 229, 277, 247, 249, 264, 270, 242, 251, 253, 236, 270, 260, 246, 260, 269, 247, 266, 249] payload_rows_per_dst=[5438, 5450, 5474, 5482, 5446, 5494, 5460, 5390] dup_rows_per_dst=[2642, 2716, 2688, 2706, 2778, 2808, 2796, 2768] combine_rows_per_src=[5438, 5450, 5474, 5482, 5446, 5494, 5460, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38282 dup_rows=21830 dup_rows_remote=19266 dedup_ratio=0.333099 router_tokens_per_expert=[235, 246, 262, 272, 318, 266, 248, 249, 230, 263, 255, 261, 250, 275, 261, 261, 249, 265, 263, 249, 265, 227, 244, 278, 238, 235, 225, 236, 239, 272, 247, 265, 265, 264, 231, 247, 264, 258, 237, 261, 240, 275, 251, 245, 267, 283, 243, 267, 253, 233, 272, 232, 307, 249, 275, 254, 279, 237, 264, 250, 225, 269, 267, 259, 275, 252, 233, 242, 278, 240, 243, 264, 228, 241, 242, 255, 244, 245, 254, 258, 257, 257, 247, 269, 277, 281, 258, 237, 259, 280, 244, 269, 262, 241, 254, 248, 233, 265, 246, 265, 260, 241, 284, 284, 251, 273, 245, 247, 263, 244, 257, 251, 233, 271, 261, 211, 259, 281, 281, 255, 262, 275, 283, 280, 271, 245, 259, 226] payload_rows_per_dst=[5546, 5388, 5432, 5434, 5424, 5496, 5430, 5556] dup_rows_per_dst=[2758, 2606, 2764, 2816, 2564, 2784, 2788, 2750] combine_rows_per_src=[5546, 5388, 5432, 5434, 5424, 5496, 5430, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38192 dup_rows=21970 dup_rows_remote=19352 dedup_ratio=0.335236 router_tokens_per_expert=[281, 259, 281, 290, 260, 243, 275, 282, 248, 269, 275, 258, 264, 253, 264, 239, 247, 268, 268, 317, 277, 279, 267, 238, 268, 270, 276, 261, 293, 238, 256, 275, 239, 245, 253, 269, 250, 252, 248, 278, 229, 237, 232, 237, 220, 251, 238, 263, 248, 257, 256, 230, 268, 250, 242, 248, 297, 241, 253, 230, 224, 259, 232, 258, 252, 263, 257, 250, 277, 283, 249, 248, 241, 241, 282, 242, 236, 246, 247, 252, 284, 261, 240, 255, 273, 265, 264, 243, 279, 254, 262, 246, 277, 233, 219, 259, 269, 259, 264, 248, 242, 240, 241, 220, 213, 260, 263, 244, 270, 255, 250, 258, 256, 253, 265, 241, 294, 250, 286, 245, 253, 282, 238, 256, 242, 261, 260, 237] payload_rows_per_dst=[5544, 5560, 5318, 5382, 5472, 5450, 5374, 5466] dup_rows_per_dst=[2938, 2944, 2564, 2696, 2660, 2778, 2618, 2772] combine_rows_per_src=[5544, 5560, 5318, 5382, 5472, 5450, 5374, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228272 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156449712 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312671152 weight_movement_bytes=37748736 total_nvlink_bytes=350419888 weight_to_token_ratio=0.120730 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38232 dup_rows=21892 dup_rows_remote=19246 dedup_ratio=0.334045 router_tokens_per_expert=[255, 260, 277, 271, 255, 239, 220, 251, 244, 263, 247, 239, 241, 250, 267, 256, 267, 284, 286, 249, 279, 259, 247, 249, 241, 286, 246, 253, 269, 277, 311, 243, 256, 257, 247, 252, 282, 271, 253, 219, 244, 286, 281, 272, 263, 242, 251, 264, 246, 244, 248, 264, 256, 241, 241, 245, 267, 259, 224, 271, 284, 255, 246, 238, 254, 229, 232, 259, 251, 244, 239, 284, 265, 250, 269, 260, 254, 230, 235, 280, 269, 259, 265, 257, 257, 255, 259, 269, 278, 258, 264, 250, 244, 255, 247, 251, 231, 256, 234, 238, 243, 242, 266, 274, 277, 248, 256, 245, 238, 259, 261, 259, 257, 264, 241, 269, 244, 239, 235, 254, 278, 289, 252, 261, 245, 263, 245, 283] payload_rows_per_dst=[5434, 5506, 5566, 5412, 5368, 5448, 5442, 5468] dup_rows_per_dst=[2636, 2986, 2714, 2646, 2702, 2826, 2612, 2770] combine_rows_per_src=[5434, 5506, 5566, 5412, 5368, 5448, 5442, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=228960 combine_payload_bytes=157065216 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157294176 ingress_bytes=194813952 link_bytes=194813952 token_bytes=314359392 weight_movement_bytes=37748736 total_nvlink_bytes=352108128 weight_to_token_ratio=0.120081 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866611434e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156221440 dispatch_meta_bytes=228936 combine_payload_bytes=156221440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156450376 ingress_bytes=193970176 link_bytes=193970176 token_bytes=312671816 weight_movement_bytes=37748736 total_nvlink_bytes=350420552 weight_to_token_ratio=0.120730 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858526787e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=230192 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157033264 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313836336 weight_movement_bytes=18874368 total_nvlink_bytes=332710704 weight_to_token_ratio=0.060141 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156434432 dispatch_meta_bytes=230176 combine_payload_bytes=156434432 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156664608 ingress_bytes=175308800 link_bytes=175308800 token_bytes=313099040 weight_movement_bytes=18874368 total_nvlink_bytes=331973408 weight_to_token_ratio=0.060282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679722664e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229912 combine_payload_bytes=156598272 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156828184 ingress_bytes=175472640 link_bytes=175472640 token_bytes=313426456 weight_movement_bytes=18874368 total_nvlink_bytes=332300824 weight_to_token_ratio=0.060219 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681292499e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38106 dup_rows=21846 dup_rows_remote=18816 dedup_ratio=0.333344 router_tokens_per_expert=[270, 257, 250, 236, 264, 231, 277, 260, 223, 276, 250, 222, 267, 242, 246, 222, 252, 247, 257, 252, 244, 260, 254, 249, 277, 263, 251, 276, 250, 264, 256, 256, 244, 221, 259, 257, 264, 244, 271, 260, 253, 254, 264, 267, 238, 250, 245, 274, 245, 253, 241, 254, 234, 258, 239, 266, 266, 252, 240, 278, 274, 235, 273, 240, 268, 251, 266, 250, 258, 252, 251, 248, 282, 281, 219, 266, 254, 253, 243, 267, 266, 241, 264, 252, 246, 251, 296, 279, 278, 266, 275, 279, 271, 278, 253, 241, 263, 287, 255, 279, 276, 231, 234, 242, 269, 287, 260, 273, 266, 237, 265, 244, 278, 256, 232, 236, 249, 245, 268, 271, 270, 252, 256, 240, 231, 240, 254, 263] payload_rows_per_dst=[5376, 5500, 5372, 5392, 5468, 5404, 5594, 5584] dup_rows_per_dst=[2610, 2716, 2630, 2572, 2738, 2548, 3002, 3030] combine_rows_per_src=[5376, 5500, 5372, 5392, 5468, 5404, 5594, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=37976 dup_rows=21924 dup_rows_remote=18922 dedup_ratio=0.334534 router_tokens_per_expert=[236, 268, 246, 267, 252, 254, 254, 229, 248, 246, 272, 233, 268, 256, 235, 269, 280, 272, 258, 231, 249, 266, 271, 242, 273, 260, 249, 267, 241, 270, 269, 257, 219, 254, 275, 246, 250, 235, 260, 265, 260, 262, 251, 269, 243, 243, 230, 261, 272, 265, 266, 262, 239, 266, 255, 249, 249, 250, 228, 263, 235, 273, 232, 276, 273, 289, 267, 250, 276, 265, 225, 266, 255, 239, 245, 237, 256, 258, 256, 246, 258, 276, 251, 259, 281, 286, 262, 245, 218, 250, 271, 257, 256, 280, 253, 256, 222, 229, 262, 230, 263, 271, 235, 265, 282, 264, 266, 266, 228, 248, 247, 272, 251, 238, 263, 261, 267, 273, 282, 260, 259, 241, 245, 266, 258, 268, 279, 254] payload_rows_per_dst=[5636, 5626, 5528, 5392, 5364, 5428, 5356, 5282] dup_rows_per_dst=[3002, 2944, 2780, 2832, 2516, 2800, 2484, 2566] combine_rows_per_src=[5636, 5626, 5528, 5392, 5364, 5428, 5356, 5282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43894 payload_rows_remote=38464 dup_rows=21642 dup_rows_remote=19076 dedup_ratio=0.330231 router_tokens_per_expert=[249, 289, 293, 245, 235, 238, 256, 291, 252, 276, 269, 288, 236, 257, 247, 259, 262, 268, 243, 233, 246, 265, 279, 255, 254, 251, 259, 247, 261, 229, 241, 272, 273, 268, 252, 230, 248, 245, 260, 252, 252, 229, 228, 238, 262, 260, 250, 251, 265, 252, 262, 267, 259, 262, 262, 292, 232, 251, 260, 266, 256, 260, 290, 261, 253, 294, 231, 272, 253, 254, 261, 264, 270, 271, 256, 236, 224, 276, 235, 256, 264, 269, 286, 260, 258, 267, 246, 267, 263, 246, 249, 240, 248, 275, 224, 265, 259, 297, 274, 238, 248, 245, 279, 235, 248, 265, 238, 255, 231, 272, 238, 244, 275, 254, 252, 254, 243, 260, 274, 267, 248, 226, 266, 236, 245, 227, 249, 253] payload_rows_per_dst=[5512, 5518, 5430, 5578, 5504, 5456, 5428, 5468] dup_rows_per_dst=[2848, 2612, 2566, 2816, 2708, 2798, 2704, 2590] combine_rows_per_src=[5512, 5518, 5430, 5578, 5504, 5456, 5428, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156082176 dispatch_meta_bytes=227688 combine_payload_bytes=156082176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156309864 ingress_bytes=174956544 link_bytes=174956544 token_bytes=312392040 weight_movement_bytes=18874368 total_nvlink_bytes=331266408 weight_to_token_ratio=0.060419 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676347521e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=155549696 dispatch_meta_bytes=227592 combine_payload_bytes=155549696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155777288 ingress_bytes=174424064 link_bytes=174424064 token_bytes=311326984 weight_movement_bytes=18874368 total_nvlink_bytes=330201352 weight_to_token_ratio=0.060626 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.671245559e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38288 dup_rows=21822 dup_rows_remote=19128 dedup_ratio=0.332977 router_tokens_per_expert=[288, 235, 223, 266, 265, 259, 266, 263, 271, 249, 249, 217, 241, 252, 288, 240, 264, 266, 245, 265, 243, 247, 262, 261, 238, 249, 248, 267, 248, 249, 256, 252, 261, 253, 250, 254, 253, 285, 264, 245, 263, 262, 248, 263, 261, 252, 245, 282, 266, 277, 248, 254, 248, 259, 286, 252, 276, 236, 250, 270, 269, 255, 256, 261, 263, 251, 272, 247, 254, 239, 244, 250, 263, 271, 257, 251, 251, 272, 247, 250, 264, 254, 255, 244, 259, 255, 234, 276, 249, 275, 265, 288, 265, 253, 272, 252, 257, 277, 240, 249, 246, 251, 247, 272, 263, 241, 246, 244, 224, 271, 280, 273, 287, 243, 242, 252, 247, 249, 227, 258, 248, 270, 238, 268, 225, 245, 250, 260] payload_rows_per_dst=[5438, 5426, 5540, 5522, 5440, 5542, 5470, 5336] dup_rows_per_dst=[2706, 2694, 2742, 2804, 2724, 2778, 2692, 2682] combine_rows_per_src=[5438, 5426, 5540, 5522, 5440, 5542, 5470, 5336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38430 dup_rows=21672 dup_rows_remote=18870 dedup_ratio=0.330688 router_tokens_per_expert=[255, 242, 234, 236, 244, 279, 281, 246, 242, 275, 260, 274, 241, 262, 242, 279, 243, 276, 289, 269, 274, 286, 278, 254, 262, 250, 252, 252, 234, 258, 251, 244, 230, 263, 258, 269, 218, 262, 296, 271, 248, 288, 248, 250, 242, 234, 230, 260, 259, 287, 234, 226, 254, 270, 228, 252, 274, 267, 251, 263, 245, 273, 253, 235, 243, 271, 246, 262, 258, 272, 265, 248, 275, 242, 281, 258, 273, 246, 246, 250, 270, 245, 257, 279, 242, 249, 266, 247, 284, 268, 269, 280, 245, 225, 240, 252, 284, 274, 257, 269, 262, 241, 262, 259, 240, 236, 274, 247, 272, 251, 240, 246, 236, 274, 239, 236, 241, 247, 271, 276, 269, 242, 263, 237, 250, 278, 230, 209] payload_rows_per_dst=[5528, 5530, 5426, 5472, 5606, 5434, 5506, 5362] dup_rows_per_dst=[2656, 2814, 2708, 2670, 2666, 2802, 2722, 2634] combine_rows_per_src=[5528, 5530, 5426, 5472, 5606, 5434, 5506, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43854 payload_rows_remote=38344 dup_rows=21682 dup_rows_remote=19016 dedup_ratio=0.330841 router_tokens_per_expert=[282, 277, 241, 254, 244, 239, 253, 257, 254, 236, 273, 268, 274, 267, 262, 278, 247, 259, 244, 261, 240, 247, 274, 248, 268, 263, 240, 264, 255, 226, 276, 247, 270, 255, 242, 248, 249, 223, 253, 276, 264, 247, 269, 250, 261, 252, 250, 254, 268, 278, 267, 271, 256, 270, 243, 233, 245, 254, 229, 273, 244, 295, 275, 246, 280, 246, 237, 264, 285, 244, 275, 254, 252, 246, 258, 223, 274, 258, 264, 228, 284, 282, 262, 253, 248, 242, 252, 252, 260, 258, 250, 249, 266, 265, 236, 273, 250, 285, 260, 245, 257, 279, 241, 259, 261, 280, 243, 218, 249, 265, 253, 238, 267, 233, 285, 234, 252, 250, 253, 250, 274, 269, 251, 223, 237, 259, 256, 244] payload_rows_per_dst=[5546, 5394, 5412, 5504, 5510, 5550, 5444, 5494] dup_rows_per_dst=[2772, 2724, 2714, 2790, 2666, 2714, 2722, 2580] combine_rows_per_src=[5546, 5394, 5412, 5504, 5510, 5550, 5444, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43560 payload_rows_remote=38202 dup_rows=21976 dup_rows_remote=19394 dedup_ratio=0.335327 router_tokens_per_expert=[223, 222, 259, 275, 261, 298, 259, 288, 255, 245, 246, 239, 238, 252, 241, 264, 256, 268, 248, 243, 268, 294, 230, 243, 242, 249, 257, 222, 246, 264, 259, 260, 210, 245, 237, 250, 230, 261, 256, 259, 270, 271, 261, 271, 274, 268, 249, 308, 210, 274, 283, 269, 244, 242, 275, 254, 231, 243, 255, 258, 274, 270, 252, 247, 279, 278, 250, 257, 234, 246, 264, 270, 243, 249, 259, 237, 259, 285, 281, 268, 266, 259, 248, 283, 256, 259, 263, 247, 242, 273, 288, 259, 298, 275, 224, 274, 249, 234, 230, 258, 252, 245, 268, 254, 238, 229, 254, 254, 242, 271, 253, 239, 259, 276, 284, 230, 260, 263, 244, 264, 277, 239, 250, 235, 264, 245, 249, 271] payload_rows_per_dst=[5404, 5460, 5470, 5400, 5542, 5308, 5358, 5618] dup_rows_per_dst=[2726, 2638, 2770, 2762, 2776, 2648, 2582, 3074] combine_rows_per_src=[5404, 5460, 5470, 5400, 5542, 5308, 5358, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38064 dup_rows=21904 dup_rows_remote=19122 dedup_ratio=0.334229 router_tokens_per_expert=[265, 228, 260, 277, 288, 273, 274, 253, 270, 277, 259, 259, 251, 269, 257, 217, 233, 254, 278, 282, 255, 242, 267, 252, 249, 242, 269, 271, 257, 230, 236, 236, 263, 254, 233, 246, 247, 247, 266, 249, 254, 227, 254, 300, 259, 266, 250, 249, 264, 291, 265, 265, 268, 245, 256, 264, 273, 231, 237, 259, 257, 244, 254, 302, 271, 271, 246, 283, 244, 238, 291, 257, 276, 253, 233, 241, 258, 267, 249, 242, 277, 248, 258, 276, 238, 249, 258, 290, 250, 253, 247, 261, 272, 266, 270, 239, 245, 227, 247, 260, 246, 236, 253, 272, 271, 229, 255, 260, 246, 267, 277, 247, 257, 262, 252, 228, 273, 269, 236, 242, 227, 242, 251, 273, 250, 218, 250, 259] payload_rows_per_dst=[5552, 5388, 5434, 5568, 5438, 5510, 5380, 5362] dup_rows_per_dst=[2802, 2718, 2694, 2782, 2802, 2794, 2696, 2616] combine_rows_per_src=[5552, 5388, 5434, 5568, 5438, 5510, 5380, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157548544 dispatch_meta_bytes=230160 combine_payload_bytes=157548544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157778704 ingress_bytes=176422912 link_bytes=176422912 token_bytes=315327248 weight_movement_bytes=18874368 total_nvlink_bytes=334201616 weight_to_token_ratio=0.059856 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690397537e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229664 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057312 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313884960 weight_movement_bytes=18874368 total_nvlink_bytes=332759328 weight_to_token_ratio=0.060131 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=229200 combine_payload_bytes=157409280 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157638480 ingress_bytes=214032384 link_bytes=214032384 token_bytes=315047760 weight_movement_bytes=56623104 total_nvlink_bytes=371670864 weight_to_token_ratio=0.179729 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.050752993e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229440 combine_payload_bytes=157057024 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157286464 ingress_bytes=194805760 link_bytes=194805760 token_bytes=314343488 weight_movement_bytes=37748736 total_nvlink_bytes=352092224 weight_to_token_ratio=0.120088 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=230384 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156705776 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313181168 weight_movement_bytes=18874368 total_nvlink_bytes=332055536 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155910144 dispatch_meta_bytes=228744 combine_payload_bytes=155910144 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156138888 ingress_bytes=212533248 link_bytes=212533248 token_bytes=312049032 weight_movement_bytes=56623104 total_nvlink_bytes=368672136 weight_to_token_ratio=0.181456 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.036389009e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38334 dup_rows=21772 dup_rows_remote=18980 dedup_ratio=0.332214 router_tokens_per_expert=[257, 249, 257, 266, 254, 273, 252, 243, 244, 271, 245, 265, 265, 255, 271, 276, 246, 256, 242, 217, 264, 263, 254, 255, 242, 240, 239, 256, 273, 274, 243, 274, 221, 250, 258, 274, 260, 247, 260, 242, 251, 272, 267, 263, 260, 279, 249, 252, 261, 255, 260, 257, 244, 262, 273, 273, 248, 257, 267, 252, 254, 259, 249, 228, 248, 265, 258, 257, 265, 261, 261, 257, 233, 232, 243, 259, 256, 235, 232, 281, 280, 238, 268, 256, 242, 270, 268, 258, 226, 241, 268, 252, 252, 240, 256, 251, 270, 260, 295, 270, 258, 249, 249, 256, 244, 261, 277, 261, 247, 266, 259, 259, 247, 253, 253, 260, 264, 247, 272, 237, 272, 242, 251, 265, 268, 247, 245, 270] payload_rows_per_dst=[5536, 5420, 5424, 5434, 5384, 5502, 5634, 5430] dup_rows_per_dst=[2750, 2656, 2786, 2728, 2476, 2630, 2954, 2792] combine_rows_per_src=[5536, 5420, 5424, 5434, 5384, 5502, 5634, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=38196 dup_rows=21776 dup_rows_remote=18982 dedup_ratio=0.332275 router_tokens_per_expert=[245, 256, 247, 257, 259, 264, 230, 243, 240, 274, 283, 274, 253, 300, 290, 242, 268, 263, 241, 260, 275, 235, 269, 262, 280, 251, 277, 228, 259, 257, 262, 267, 237, 220, 284, 276, 222, 248, 265, 250, 272, 225, 236, 229, 233, 253, 267, 262, 257, 264, 255, 233, 265, 261, 258, 264, 249, 257, 241, 256, 257, 249, 237, 265, 260, 269, 235, 271, 260, 273, 262, 241, 246, 252, 251, 242, 237, 266, 268, 284, 270, 249, 267, 248, 243, 260, 252, 268, 241, 280, 248, 232, 264, 242, 245, 268, 267, 261, 260, 232, 233, 268, 237, 254, 264, 235, 271, 275, 279, 253, 287, 255, 224, 258, 249, 251, 257, 251, 241, 279, 261, 251, 238, 256, 276, 273, 266, 254] payload_rows_per_dst=[5564, 5496, 5398, 5586, 5474, 5466, 5344, 5432] dup_rows_per_dst=[2794, 2812, 2560, 2924, 2760, 2644, 2544, 2738] combine_rows_per_src=[5564, 5496, 5398, 5586, 5474, 5466, 5344, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229256 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157245320 ingress_bytes=213639168 link_bytes=213639168 token_bytes=314261384 weight_movement_bytes=56623104 total_nvlink_bytes=370884488 weight_to_token_ratio=0.180178 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046985391e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38256 dup_rows=21804 dup_rows_remote=19098 dedup_ratio=0.332703 router_tokens_per_expert=[252, 248, 249, 281, 267, 230, 235, 263, 266, 263, 277, 265, 264, 258, 264, 272, 238, 263, 258, 263, 259, 276, 265, 259, 257, 259, 257, 252, 296, 240, 245, 247, 252, 236, 283, 260, 256, 269, 250, 271, 259, 252, 249, 266, 273, 227, 241, 247, 248, 263, 250, 264, 257, 253, 272, 241, 266, 271, 219, 249, 264, 235, 238, 276, 264, 259, 253, 231, 257, 263, 260, 238, 220, 230, 234, 229, 282, 241, 257, 274, 231, 262, 263, 252, 260, 260, 249, 254, 275, 255, 261, 253, 252, 251, 250, 268, 229, 254, 252, 231, 268, 244, 268, 243, 267, 268, 251, 245, 248, 277, 259, 258, 263, 233, 258, 283, 267, 291, 253, 253, 251, 264, 240, 269, 260, 262, 260, 266] payload_rows_per_dst=[5476, 5446, 5476, 5456, 5362, 5504, 5494, 5518] dup_rows_per_dst=[2832, 2822, 2706, 2676, 2622, 2688, 2630, 2828] combine_rows_per_src=[5476, 5446, 5476, 5456, 5362, 5504, 5494, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228712 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679528 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130344 weight_movement_bytes=37748736 total_nvlink_bytes=350879080 weight_to_token_ratio=0.120553 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43758 payload_rows_remote=38228 dup_rows=21778 dup_rows_remote=18940 dedup_ratio=0.332306 router_tokens_per_expert=[244, 252, 233, 244, 248, 253, 269, 245, 241, 243, 255, 228, 256, 259, 238, 242, 237, 262, 263, 251, 251, 289, 278, 266, 274, 257, 272, 270, 235, 253, 267, 259, 265, 217, 250, 288, 249, 249, 253, 235, 253, 247, 265, 244, 264, 258, 244, 236, 238, 248, 268, 246, 255, 251, 243, 286, 266, 237, 253, 242, 277, 265, 251, 259, 230, 259, 223, 259, 239, 265, 253, 248, 255, 241, 256, 247, 298, 263, 254, 282, 248, 246, 259, 254, 249, 231, 234, 272, 239, 276, 264, 260, 256, 262, 250, 258, 269, 265, 267, 270, 267, 262, 252, 252, 249, 254, 260, 265, 260, 252, 250, 256, 290, 274, 268, 254, 247, 244, 264, 253, 259, 263, 258, 299, 278, 266, 272, 263] payload_rows_per_dst=[5426, 5530, 5466, 5480, 5450, 5426, 5360, 5620] dup_rows_per_dst=[2474, 2838, 2794, 2690, 2694, 2690, 2714, 2884] combine_rows_per_src=[5426, 5530, 5466, 5480, 5450, 5426, 5360, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43896 payload_rows_remote=38460 dup_rows=21640 dup_rows_remote=19002 dedup_ratio=0.330200 router_tokens_per_expert=[261, 265, 237, 254, 259, 263, 256, 233, 250, 256, 278, 248, 247, 233, 226, 254, 269, 235, 246, 277, 245, 253, 258, 235, 252, 257, 279, 257, 273, 265, 262, 270, 270, 230, 237, 251, 252, 258, 275, 253, 274, 243, 235, 222, 277, 243, 257, 271, 239, 255, 266, 256, 252, 247, 258, 269, 256, 270, 223, 239, 287, 283, 256, 263, 263, 261, 285, 259, 258, 260, 233, 262, 274, 240, 253, 264, 282, 262, 246, 258, 261, 242, 229, 262, 232, 233, 258, 265, 241, 286, 260, 272, 264, 249, 244, 239, 231, 262, 260, 270, 248, 248, 259, 259, 248, 287, 263, 248, 249, 269, 236, 270, 237, 278, 287, 233, 244, 265, 259, 257, 246, 264, 285, 281, 270, 242, 248, 248] payload_rows_per_dst=[5406, 5518, 5482, 5488, 5530, 5436, 5472, 5564] dup_rows_per_dst=[2634, 2748, 2614, 2750, 2790, 2638, 2742, 2724] combine_rows_per_src=[5406, 5518, 5482, 5488, 5530, 5436, 5472, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38326 dup_rows=21710 dup_rows_remote=18844 dedup_ratio=0.331268 router_tokens_per_expert=[276, 277, 261, 259, 271, 267, 242, 242, 231, 262, 250, 254, 262, 253, 259, 249, 252, 243, 251, 262, 267, 233, 258, 249, 260, 238, 296, 244, 247, 261, 232, 249, 232, 266, 257, 237, 258, 275, 258, 257, 253, 285, 259, 225, 296, 251, 270, 231, 257, 242, 232, 267, 237, 235, 219, 272, 236, 243, 251, 257, 257, 246, 245, 276, 261, 265, 264, 235, 238, 293, 225, 255, 283, 267, 284, 280, 272, 236, 279, 246, 268, 273, 281, 256, 270, 254, 252, 265, 278, 249, 249, 240, 241, 255, 271, 261, 250, 272, 233, 239, 262, 275, 250, 263, 225, 251, 257, 272, 283, 254, 244, 238, 270, 278, 266, 220, 243, 245, 242, 275, 262, 266, 277, 245, 250, 250, 260, 266] payload_rows_per_dst=[5510, 5450, 5482, 5400, 5500, 5478, 5480, 5526] dup_rows_per_dst=[2720, 2634, 2738, 2544, 2866, 2848, 2656, 2704] combine_rows_per_src=[5510, 5450, 5482, 5400, 5500, 5478, 5480, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38168 dup_rows=21866 dup_rows_remote=19012 dedup_ratio=0.333649 router_tokens_per_expert=[224, 227, 274, 248, 272, 260, 221, 245, 274, 251, 239, 237, 264, 265, 244, 254, 254, 257, 247, 249, 254, 284, 253, 250, 221, 291, 281, 255, 251, 247, 244, 267, 230, 233, 264, 264, 247, 234, 261, 249, 276, 274, 252, 225, 267, 252, 261, 252, 256, 257, 260, 247, 288, 257, 255, 247, 257, 281, 261, 251, 248, 270, 275, 268, 253, 254, 270, 269, 263, 278, 272, 255, 234, 233, 273, 223, 255, 266, 212, 248, 256, 278, 218, 256, 240, 266, 251, 241, 236, 223, 253, 254, 250, 226, 264, 269, 228, 272, 254, 282, 280, 241, 273, 264, 251, 279, 262, 271, 255, 289, 250, 246, 281, 260, 273, 268, 244, 267, 236, 239, 265, 276, 284, 256, 263, 271, 260, 266] payload_rows_per_dst=[5414, 5480, 5370, 5502, 5378, 5382, 5548, 5596] dup_rows_per_dst=[2584, 2730, 2712, 2854, 2738, 2580, 2846, 2822] combine_rows_per_src=[5414, 5480, 5370, 5502, 5378, 5382, 5548, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229416 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925992 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622568 weight_movement_bytes=18874368 total_nvlink_bytes=332496936 weight_to_token_ratio=0.060182 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43578 payload_rows_remote=38148 dup_rows=21958 dup_rows_remote=19108 dedup_ratio=0.335052 router_tokens_per_expert=[250, 249, 248, 271, 281, 237, 287, 255, 262, 243, 278, 253, 245, 235, 244, 242, 230, 250, 224, 254, 257, 255, 272, 280, 231, 287, 259, 253, 265, 240, 255, 244, 257, 257, 241, 248, 251, 257, 263, 243, 226, 265, 250, 251, 245, 238, 260, 244, 283, 248, 255, 285, 219, 256, 251, 250, 256, 252, 242, 227, 276, 259, 262, 263, 254, 254, 257, 251, 270, 263, 301, 277, 228, 265, 248, 239, 269, 254, 259, 234, 261, 268, 271, 260, 265, 268, 262, 254, 235, 254, 257, 286, 252, 245, 257, 256, 255, 237, 247, 256, 252, 262, 257, 269, 260, 271, 278, 245, 260, 240, 281, 270, 292, 258, 249, 266, 259, 265, 249, 265, 234, 253, 234, 280, 249, 252, 273, 260] payload_rows_per_dst=[5402, 5390, 5388, 5448, 5454, 5526, 5430, 5540] dup_rows_per_dst=[2758, 2722, 2604, 2720, 2792, 2776, 2850, 2736] combine_rows_per_src=[5402, 5390, 5388, 5448, 5454, 5526, 5430, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156581888 dispatch_meta_bytes=228672 combine_payload_bytes=156581888 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156810560 ingress_bytes=175456256 link_bytes=175456256 token_bytes=313392448 weight_movement_bytes=18874368 total_nvlink_bytes=332266816 weight_to_token_ratio=0.060226 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681135515e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=157532160 dispatch_meta_bytes=229848 combine_payload_bytes=157532160 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157762008 ingress_bytes=176406528 link_bytes=176406528 token_bytes=315294168 weight_movement_bytes=18874368 total_nvlink_bytes=334168536 weight_to_token_ratio=0.059863 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690240554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=228680 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157211976 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195272 weight_movement_bytes=37748736 total_nvlink_bytes=351944008 weight_to_token_ratio=0.120144 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43554 payload_rows_remote=38172 dup_rows=21982 dup_rows_remote=19172 dedup_ratio=0.335419 router_tokens_per_expert=[259, 256, 233, 256, 244, 220, 293, 216, 272, 238, 264, 267, 248, 252, 269, 269, 268, 253, 238, 268, 264, 240, 279, 270, 271, 261, 280, 236, 230, 230, 253, 249, 243, 251, 251, 249, 252, 274, 269, 273, 271, 266, 267, 240, 272, 258, 290, 240, 275, 265, 251, 256, 247, 267, 252, 242, 250, 252, 281, 254, 231, 256, 241, 278, 261, 254, 268, 275, 229, 262, 251, 230, 260, 280, 241, 286, 240, 241, 250, 245, 227, 278, 283, 265, 251, 255, 254, 276, 276, 282, 259, 241, 241, 259, 266, 243, 239, 258, 277, 262, 284, 250, 239, 250, 266, 267, 269, 248, 246, 242, 262, 240, 269, 257, 264, 234, 224, 252, 267, 240, 259, 252, 259, 220, 255, 231, 273, 274] payload_rows_per_dst=[5352, 5442, 5578, 5456, 5370, 5600, 5374, 5382] dup_rows_per_dst=[2536, 2738, 2962, 2756, 2516, 2932, 2732, 2810] combine_rows_per_src=[5352, 5442, 5578, 5456, 5370, 5600, 5374, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=228720 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156564848 ingress_bytes=212959232 link_bytes=212959232 token_bytes=312900976 weight_movement_bytes=56623104 total_nvlink_bytes=369524080 weight_to_token_ratio=0.180962 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040470579e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229024 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156483232 ingress_bytes=194002944 link_bytes=194002944 token_bytes=312737440 weight_movement_bytes=37748736 total_nvlink_bytes=350486176 weight_to_token_ratio=0.120704 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858840754e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38256 dup_rows=21836 dup_rows_remote=19022 dedup_ratio=0.333191 router_tokens_per_expert=[257, 281, 292, 253, 283, 238, 239, 261, 273, 237, 248, 256, 256, 231, 243, 272, 259, 269, 268, 267, 252, 241, 247, 260, 254, 270, 254, 255, 273, 259, 250, 232, 310, 251, 255, 254, 231, 204, 276, 228, 273, 269, 278, 273, 246, 269, 264, 253, 240, 261, 255, 259, 258, 260, 255, 237, 222, 243, 247, 246, 237, 265, 258, 263, 258, 255, 242, 246, 254, 253, 271, 235, 248, 268, 245, 238, 289, 257, 283, 252, 277, 253, 255, 250, 256, 253, 273, 266, 281, 261, 245, 264, 245, 234, 263, 248, 258, 272, 273, 252, 247, 240, 249, 259, 259, 269, 253, 250, 275, 277, 234, 223, 269, 267, 229, 272, 246, 267, 251, 264, 244, 252, 241, 265, 248, 254, 276, 245] payload_rows_per_dst=[5444, 5424, 5584, 5428, 5630, 5290, 5602, 5298] dup_rows_per_dst=[2814, 2796, 2666, 2584, 3120, 2396, 3016, 2444] combine_rows_per_src=[5444, 5424, 5584, 5428, 5630, 5290, 5602, 5298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=229376 combine_payload_bytes=156352512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156581888 ingress_bytes=175226880 link_bytes=175226880 token_bytes=312934400 weight_movement_bytes=18874368 total_nvlink_bytes=331808768 weight_to_token_ratio=0.060314 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678937747e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43518 payload_rows_remote=38020 dup_rows=22018 dup_rows_remote=19262 dedup_ratio=0.335968 router_tokens_per_expert=[270, 263, 245, 253, 258, 234, 246, 225, 243, 245, 250, 258, 245, 278, 246, 284, 254, 246, 244, 279, 280, 278, 266, 245, 250, 256, 247, 218, 270, 260, 262, 258, 254, 261, 256, 262, 264, 257, 267, 230, 281, 258, 259, 234, 292, 249, 235, 268, 286, 240, 246, 244, 253, 265, 253, 260, 239, 246, 225, 264, 252, 283, 270, 251, 258, 284, 235, 233, 267, 252, 258, 307, 267, 253, 252, 253, 249, 237, 295, 278, 239, 246, 260, 248, 233, 235, 248, 242, 263, 254, 266, 263, 236, 228, 243, 271, 260, 256, 276, 257, 285, 249, 249, 238, 267, 262, 274, 242, 267, 240, 257, 277, 266, 215, 267, 278, 256, 246, 234, 250, 250, 277, 255, 279, 257, 240, 257, 272] payload_rows_per_dst=[5462, 5432, 5498, 5392, 5512, 5324, 5480, 5418] dup_rows_per_dst=[2624, 2794, 2756, 2762, 2844, 2626, 2832, 2780] combine_rows_per_src=[5462, 5432, 5498, 5392, 5512, 5324, 5480, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229112 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156925688 ingress_bytes=232194048 link_bytes=232194048 token_bytes=313622264 weight_movement_bytes=75497472 total_nvlink_bytes=389119736 weight_to_token_ratio=0.240727 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224769121e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38146 dup_rows=21862 dup_rows_remote=18980 dedup_ratio=0.333588 router_tokens_per_expert=[265, 254, 261, 221, 252, 211, 224, 273, 271, 256, 280, 258, 255, 252, 242, 252, 236, 272, 266, 277, 251, 264, 248, 255, 261, 239, 273, 275, 269, 257, 301, 261, 251, 261, 280, 256, 245, 243, 271, 230, 272, 225, 260, 243, 259, 275, 252, 263, 239, 267, 255, 244, 264, 276, 254, 249, 238, 265, 245, 285, 246, 252, 256, 258, 253, 256, 279, 250, 259, 250, 274, 278, 273, 263, 279, 272, 256, 229, 272, 247, 273, 235, 259, 248, 252, 253, 262, 266, 262, 236, 266, 225, 254, 245, 244, 259, 259, 232, 262, 261, 267, 244, 238, 262, 269, 259, 278, 238, 276, 239, 257, 278, 256, 259, 268, 245, 228, 227, 256, 256, 257, 256, 273, 253, 233, 253, 246, 243] payload_rows_per_dst=[5424, 5528, 5462, 5446, 5546, 5416, 5514, 5338] dup_rows_per_dst=[2630, 2882, 2710, 2740, 2834, 2662, 2724, 2680] combine_rows_per_src=[5424, 5528, 5462, 5446, 5546, 5416, 5514, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38330 dup_rows=21756 dup_rows_remote=18966 dedup_ratio=0.331970 router_tokens_per_expert=[226, 239, 253, 298, 257, 241, 239, 256, 268, 285, 281, 235, 254, 257, 252, 266, 283, 267, 263, 245, 241, 263, 261, 269, 252, 249, 248, 245, 242, 239, 255, 242, 283, 256, 272, 253, 264, 250, 279, 234, 246, 247, 230, 265, 235, 240, 261, 246, 283, 234, 270, 252, 279, 220, 241, 231, 250, 278, 253, 266, 261, 237, 270, 261, 262, 236, 225, 270, 238, 247, 267, 248, 278, 254, 250, 286, 243, 257, 255, 296, 255, 257, 273, 239, 283, 235, 260, 234, 259, 243, 289, 257, 256, 265, 270, 245, 241, 246, 258, 261, 258, 272, 275, 293, 258, 232, 265, 289, 253, 261, 228, 265, 269, 258, 258, 248, 243, 235, 263, 250, 252, 261, 263, 251, 264, 255, 254, 239] payload_rows_per_dst=[5532, 5440, 5422, 5454, 5520, 5450, 5522, 5440] dup_rows_per_dst=[2682, 2688, 2700, 2718, 2704, 2790, 2788, 2686] combine_rows_per_src=[5532, 5440, 5422, 5454, 5520, 5450, 5522, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43602 payload_rows_remote=38138 dup_rows=21934 dup_rows_remote=19100 dedup_ratio=0.334686 router_tokens_per_expert=[244, 267, 250, 271, 241, 259, 249, 270, 263, 245, 272, 257, 284, 260, 247, 249, 239, 256, 264, 222, 250, 245, 283, 260, 246, 268, 277, 251, 276, 253, 223, 251, 268, 277, 257, 240, 262, 273, 256, 256, 252, 269, 243, 241, 288, 250, 257, 255, 248, 235, 257, 275, 249, 230, 253, 261, 249, 262, 238, 261, 256, 259, 261, 252, 264, 233, 260, 301, 268, 254, 259, 247, 248, 257, 232, 238, 284, 259, 268, 277, 268, 259, 234, 264, 229, 231, 276, 248, 266, 257, 241, 225, 251, 254, 256, 267, 305, 270, 263, 264, 264, 231, 259, 238, 271, 272, 253, 263, 238, 261, 264, 249, 250, 254, 255, 262, 242, 278, 220, 256, 236, 254, 274, 239, 256, 255, 268, 247] payload_rows_per_dst=[5460, 5436, 5496, 5382, 5464, 5378, 5564, 5422] dup_rows_per_dst=[2796, 2692, 2792, 2710, 2834, 2674, 2766, 2670] combine_rows_per_src=[5460, 5436, 5496, 5382, 5464, 5378, 5564, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=38360 dup_rows=21664 dup_rows_remote=19002 dedup_ratio=0.330566 router_tokens_per_expert=[281, 267, 260, 264, 284, 260, 237, 275, 257, 275, 262, 253, 281, 267, 235, 258, 257, 227, 266, 263, 253, 258, 250, 249, 246, 264, 239, 248, 241, 248, 248, 255, 244, 256, 245, 227, 258, 252, 256, 257, 240, 241, 239, 254, 252, 232, 252, 263, 277, 246, 263, 265, 297, 250, 249, 263, 228, 253, 251, 243, 241, 245, 250, 266, 267, 279, 243, 259, 267, 256, 266, 275, 259, 240, 245, 256, 236, 266, 264, 274, 238, 209, 251, 294, 234, 260, 224, 243, 215, 261, 274, 222, 248, 269, 266, 293, 274, 251, 248, 261, 282, 221, 265, 271, 240, 232, 261, 267, 248, 287, 262, 288, 257, 266, 261, 275, 250, 260, 257, 277, 231, 231, 266, 256, 267, 272, 274, 274] payload_rows_per_dst=[5612, 5428, 5376, 5512, 5518, 5410, 5472, 5544] dup_rows_per_dst=[2820, 2596, 2560, 2662, 2786, 2592, 2844, 2804] combine_rows_per_src=[5612, 5428, 5376, 5512, 5518, 5410, 5472, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155729920 dispatch_meta_bytes=229128 combine_payload_bytes=155729920 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=155959048 ingress_bytes=174604288 link_bytes=174604288 token_bytes=311688968 weight_movement_bytes=18874368 total_nvlink_bytes=330563336 weight_to_token_ratio=0.060555 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672972377e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38400 dup_rows=21688 dup_rows_remote=19042 dedup_ratio=0.330933 router_tokens_per_expert=[275, 249, 252, 259, 270, 236, 252, 262, 244, 289, 267, 245, 267, 246, 258, 228, 244, 255, 249, 236, 244, 265, 257, 242, 248, 256, 277, 267, 233, 274, 257, 228, 256, 238, 279, 261, 267, 261, 275, 253, 250, 239, 244, 270, 240, 274, 247, 246, 271, 278, 261, 274, 261, 247, 275, 264, 248, 244, 258, 277, 267, 241, 258, 257, 249, 239, 258, 242, 293, 279, 212, 256, 292, 255, 267, 283, 250, 257, 278, 272, 251, 254, 239, 278, 243, 255, 268, 268, 232, 286, 265, 247, 244, 242, 262, 251, 269, 253, 255, 253, 242, 232, 249, 260, 258, 228, 226, 267, 240, 281, 258, 276, 245, 250, 255, 252, 254, 237, 262, 274, 249, 241, 252, 250, 276, 246, 253, 246] payload_rows_per_dst=[5490, 5424, 5496, 5564, 5568, 5476, 5448, 5382] dup_rows_per_dst=[2708, 2640, 2704, 2798, 2796, 2694, 2646, 2702] combine_rows_per_src=[5490, 5424, 5496, 5564, 5568, 5476, 5448, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228504 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156474520 ingress_bytes=175120384 link_bytes=175120384 token_bytes=312720536 weight_movement_bytes=18874368 total_nvlink_bytes=331594904 weight_to_token_ratio=0.060355 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677917355e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229184 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228864 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228544 weight_movement_bytes=18874368 total_nvlink_bytes=333102912 weight_to_token_ratio=0.060066 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43756 payload_rows_remote=38276 dup_rows=21780 dup_rows_remote=19016 dedup_ratio=0.332336 router_tokens_per_expert=[225, 257, 253, 277, 255, 246, 272, 251, 265, 268, 262, 262, 266, 268, 239, 248, 254, 274, 265, 275, 227, 261, 266, 240, 278, 263, 242, 273, 256, 283, 258, 262, 295, 266, 238, 261, 263, 224, 263, 253, 278, 270, 246, 253, 261, 271, 272, 247, 246, 272, 270, 235, 254, 260, 263, 245, 255, 244, 251, 255, 239, 256, 252, 253, 242, 245, 264, 238, 248, 227, 248, 251, 279, 279, 262, 240, 258, 259, 233, 257, 241, 268, 245, 245, 231, 268, 251, 257, 264, 245, 250, 290, 257, 230, 259, 228, 235, 276, 259, 261, 281, 268, 245, 264, 256, 214, 266, 262, 256, 242, 242, 258, 242, 259, 274, 252, 271, 247, 269, 270, 233, 246, 244, 271, 237, 252, 288, 267] payload_rows_per_dst=[5446, 5496, 5448, 5490, 5478, 5430, 5488, 5480] dup_rows_per_dst=[2782, 2858, 2834, 2610, 2622, 2628, 2682, 2764] combine_rows_per_src=[5446, 5496, 5448, 5490, 5478, 5430, 5488, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156213248 dispatch_meta_bytes=228952 combine_payload_bytes=156213248 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156442200 ingress_bytes=231710720 link_bytes=231710720 token_bytes=312655448 weight_movement_bytes=75497472 total_nvlink_bytes=388152920 weight_to_token_ratio=0.241472 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220138110e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229448 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157352008 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474568 weight_movement_bytes=18874368 total_nvlink_bytes=333348936 weight_to_token_ratio=0.060019 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=229768 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157516168 ingress_bytes=195035136 link_bytes=195035136 token_bytes=314802568 weight_movement_bytes=37748736 total_nvlink_bytes=352551304 weight_to_token_ratio=0.119912 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868730710e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43552 payload_rows_remote=38032 dup_rows=21984 dup_rows_remote=19132 dedup_ratio=0.335449 router_tokens_per_expert=[268, 248, 233, 267, 264, 260, 263, 240, 231, 253, 248, 258, 250, 266, 276, 252, 236, 258, 262, 233, 248, 234, 242, 274, 279, 264, 265, 289, 240, 219, 255, 255, 252, 285, 235, 263, 261, 242, 265, 271, 230, 233, 242, 229, 253, 254, 272, 267, 265, 240, 262, 261, 239, 266, 276, 277, 264, 250, 263, 270, 251, 271, 261, 266, 248, 253, 266, 229, 246, 264, 257, 249, 259, 247, 267, 233, 255, 274, 271, 261, 271, 242, 248, 258, 243, 267, 242, 278, 263, 248, 269, 261, 270, 259, 264, 288, 246, 256, 266, 241, 228, 248, 243, 277, 259, 254, 252, 253, 262, 274, 249, 267, 254, 253, 238, 250, 247, 262, 255, 262, 261, 256, 262, 282, 256, 255, 220, 264] payload_rows_per_dst=[5520, 5512, 5402, 5458, 5292, 5454, 5404, 5510] dup_rows_per_dst=[2852, 2828, 2754, 2688, 2632, 2840, 2746, 2644] combine_rows_per_src=[5520, 5512, 5402, 5458, 5292, 5454, 5404, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229168 combine_payload_bytes=156778496 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157007664 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786160 weight_movement_bytes=37748736 total_nvlink_bytes=351534896 weight_to_token_ratio=0.120301 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43614 payload_rows_remote=38132 dup_rows=21922 dup_rows_remote=19184 dedup_ratio=0.334503 router_tokens_per_expert=[275, 239, 273, 258, 236, 256, 254, 249, 256, 286, 267, 239, 261, 247, 299, 250, 243, 242, 239, 251, 279, 253, 252, 246, 272, 251, 252, 252, 256, 276, 238, 246, 247, 240, 234, 287, 273, 238, 271, 244, 241, 257, 265, 249, 292, 277, 247, 248, 263, 279, 297, 255, 224, 278, 254, 279, 257, 248, 265, 264, 248, 257, 265, 247, 234, 284, 249, 244, 264, 245, 273, 236, 259, 263, 262, 265, 270, 269, 248, 241, 243, 245, 232, 253, 256, 276, 224, 267, 264, 251, 266, 232, 253, 252, 256, 246, 241, 250, 272, 257, 218, 277, 254, 290, 234, 263, 252, 247, 241, 268, 268, 216, 279, 267, 264, 219, 245, 267, 259, 248, 273, 242, 259, 234, 258, 242, 291, 268] payload_rows_per_dst=[5464, 5394, 5482, 5542, 5498, 5398, 5434, 5402] dup_rows_per_dst=[2826, 2702, 2738, 2818, 2714, 2634, 2662, 2828] combine_rows_per_src=[5464, 5394, 5482, 5542, 5498, 5398, 5434, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=155779072 dispatch_meta_bytes=228656 combine_payload_bytes=155779072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156007728 ingress_bytes=174653440 link_bytes=174653440 token_bytes=311786800 weight_movement_bytes=18874368 total_nvlink_bytes=330661168 weight_to_token_ratio=0.060536 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673443327e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38288 dup_rows=21842 dup_rows_remote=19206 dedup_ratio=0.333282 router_tokens_per_expert=[272, 289, 247, 218, 262, 251, 261, 260, 227, 262, 243, 264, 269, 248, 252, 258, 260, 232, 252, 260, 213, 242, 266, 265, 287, 242, 255, 284, 239, 251, 229, 244, 239, 256, 231, 245, 269, 241, 259, 278, 250, 238, 253, 260, 268, 268, 238, 244, 272, 264, 246, 266, 232, 255, 268, 264, 249, 274, 247, 298, 290, 257, 255, 240, 280, 279, 218, 239, 250, 268, 289, 222, 250, 250, 286, 254, 252, 228, 249, 250, 277, 277, 263, 295, 253, 275, 273, 254, 253, 266, 254, 267, 233, 254, 260, 267, 257, 245, 267, 246, 240, 283, 246, 244, 246, 240, 248, 279, 251, 241, 230, 242, 252, 286, 251, 273, 263, 242, 252, 240, 271, 247, 262, 264, 277, 255, 281, 244] payload_rows_per_dst=[5424, 5406, 5488, 5500, 5478, 5518, 5420, 5460] dup_rows_per_dst=[2742, 2636, 2586, 2854, 2650, 2924, 2590, 2860] combine_rows_per_src=[5424, 5406, 5488, 5500, 5478, 5518, 5420, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38186 dup_rows=21806 dup_rows_remote=19064 dedup_ratio=0.332733 router_tokens_per_expert=[261, 238, 224, 271, 240, 263, 248, 246, 261, 251, 266, 255, 260, 255, 260, 255, 250, 258, 254, 261, 261, 277, 259, 242, 269, 261, 214, 268, 257, 247, 279, 252, 243, 234, 257, 223, 267, 268, 253, 263, 286, 256, 270, 241, 270, 235, 236, 280, 258, 249, 238, 266, 264, 296, 255, 279, 254, 247, 244, 244, 252, 259, 251, 272, 269, 249, 248, 241, 249, 281, 268, 239, 250, 268, 229, 240, 267, 231, 285, 260, 284, 242, 266, 245, 262, 247, 260, 246, 256, 256, 261, 263, 241, 278, 279, 257, 278, 243, 251, 239, 274, 254, 233, 243, 233, 262, 253, 251, 289, 268, 282, 240, 265, 263, 226, 265, 258, 290, 265, 252, 255, 261, 202, 266, 243, 259, 259, 256] payload_rows_per_dst=[5436, 5456, 5456, 5488, 5466, 5544, 5400, 5484] dup_rows_per_dst=[2672, 2762, 2708, 2768, 2682, 2742, 2786, 2686] combine_rows_per_src=[5436, 5456, 5456, 5488, 5466, 5544, 5400, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43566 payload_rows_remote=38096 dup_rows=21970 dup_rows_remote=19152 dedup_ratio=0.335236 router_tokens_per_expert=[249, 264, 283, 256, 248, 252, 276, 243, 252, 277, 260, 263, 260, 239, 226, 267, 253, 269, 253, 262, 250, 243, 246, 238, 275, 258, 248, 235, 265, 257, 263, 242, 279, 263, 262, 232, 246, 247, 223, 244, 251, 270, 277, 270, 235, 263, 276, 271, 245, 266, 254, 256, 254, 251, 280, 270, 251, 252, 269, 249, 267, 276, 253, 251, 234, 283, 257, 240, 244, 227, 275, 246, 279, 263, 234, 264, 248, 260, 244, 281, 239, 257, 242, 254, 251, 255, 274, 274, 253, 254, 233, 284, 231, 265, 255, 243, 274, 260, 273, 263, 266, 270, 222, 246, 264, 268, 272, 281, 256, 242, 259, 250, 252, 267, 237, 275, 243, 265, 264, 244, 235, 241, 256, 222, 255, 258, 253, 267] payload_rows_per_dst=[5452, 5466, 5418, 5470, 5458, 5438, 5466, 5398] dup_rows_per_dst=[2778, 2648, 2800, 2818, 2700, 2690, 2866, 2670] combine_rows_per_src=[5452, 5466, 5418, 5470, 5458, 5438, 5466, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38356 dup_rows=21646 dup_rows_remote=18906 dedup_ratio=0.330292 router_tokens_per_expert=[228, 259, 257, 230, 245, 274, 269, 280, 262, 271, 225, 252, 244, 262, 282, 251, 266, 239, 231, 257, 265, 251, 234, 248, 261, 226, 267, 222, 299, 274, 221, 271, 242, 269, 259, 262, 242, 256, 264, 284, 257, 223, 250, 232, 259, 246, 256, 263, 242, 264, 279, 229, 253, 263, 236, 252, 267, 251, 265, 236, 238, 275, 262, 255, 285, 258, 269, 251, 263, 274, 244, 225, 258, 246, 276, 267, 266, 259, 247, 249, 279, 289, 249, 255, 253, 238, 238, 260, 267, 257, 236, 265, 258, 248, 245, 250, 260, 273, 260, 237, 237, 244, 253, 301, 251, 271, 260, 269, 271, 252, 252, 249, 264, 267, 251, 265, 274, 263, 243, 271, 253, 265, 256, 269, 268, 254, 244, 243] payload_rows_per_dst=[5396, 5474, 5494, 5448, 5534, 5454, 5506, 5584] dup_rows_per_dst=[2786, 2590, 2634, 2686, 2740, 2720, 2774, 2716] combine_rows_per_src=[5396, 5474, 5494, 5448, 5534, 5454, 5506, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229264 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156417936 ingress_bytes=212811776 link_bytes=212811776 token_bytes=312606608 weight_movement_bytes=56623104 total_nvlink_bytes=369229712 weight_to_token_ratio=0.181132 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.039057728e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43774 payload_rows_remote=38282 dup_rows=21762 dup_rows_remote=18900 dedup_ratio=0.332062 router_tokens_per_expert=[274, 220, 269, 238, 242, 261, 277, 247, 278, 238, 244, 268, 256, 264, 242, 265, 267, 240, 288, 274, 242, 249, 262, 254, 258, 252, 256, 241, 235, 253, 228, 235, 245, 278, 270, 282, 245, 229, 230, 278, 236, 292, 247, 250, 262, 241, 270, 259, 262, 257, 286, 243, 236, 239, 274, 244, 266, 288, 254, 266, 266, 273, 265, 265, 268, 263, 266, 250, 251, 253, 248, 255, 226, 261, 259, 279, 258, 231, 291, 257, 275, 257, 232, 254, 276, 273, 249, 233, 242, 245, 238, 235, 245, 235, 257, 259, 249, 268, 264, 274, 240, 274, 223, 288, 270, 255, 275, 245, 252, 260, 289, 251, 266, 239, 252, 254, 250, 250, 230, 269, 272, 267, 243, 277, 252, 254, 250, 230] payload_rows_per_dst=[5478, 5436, 5528, 5524, 5458, 5426, 5492, 5432] dup_rows_per_dst=[2688, 2632, 2700, 2844, 2774, 2584, 2862, 2678] combine_rows_per_src=[5478, 5436, 5528, 5524, 5458, 5426, 5492, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229976 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157057624 ingress_bytes=175702016 link_bytes=175702016 token_bytes=313885272 weight_movement_bytes=18874368 total_nvlink_bytes=332759640 weight_to_token_ratio=0.060131 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683490267e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229000 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156638856 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313048712 weight_movement_bytes=18874368 total_nvlink_bytes=331923080 weight_to_token_ratio=0.060292 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38046 dup_rows=21942 dup_rows_remote=19106 dedup_ratio=0.334808 router_tokens_per_expert=[230, 252, 256, 282, 242, 251, 290, 228, 249, 273, 262, 287, 248, 250, 245, 263, 268, 234, 281, 290, 274, 245, 254, 243, 274, 262, 241, 239, 257, 280, 275, 266, 296, 257, 279, 251, 268, 252, 273, 308, 277, 269, 224, 275, 246, 268, 280, 250, 281, 243, 262, 246, 250, 247, 270, 237, 271, 260, 247, 242, 238, 229, 240, 263, 275, 253, 243, 243, 262, 257, 238, 233, 269, 251, 257, 225, 239, 263, 242, 243, 259, 250, 269, 227, 209, 240, 244, 264, 244, 243, 247, 258, 258, 251, 245, 254, 257, 257, 253, 273, 268, 240, 260, 243, 246, 246, 249, 266, 206, 251, 276, 256, 229, 244, 293, 268, 265, 264, 259, 267, 308, 239, 231, 263, 268, 236, 291, 251] payload_rows_per_dst=[5396, 5402, 5248, 5490, 5362, 5572, 5576, 5548] dup_rows_per_dst=[2680, 2594, 2496, 2806, 2624, 2968, 2938, 2836] combine_rows_per_src=[5396, 5402, 5248, 5490, 5362, 5572, 5576, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156041216 dispatch_meta_bytes=228992 combine_payload_bytes=156041216 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156270208 ingress_bytes=212664320 link_bytes=212664320 token_bytes=312311424 weight_movement_bytes=56623104 total_nvlink_bytes=368934528 weight_to_token_ratio=0.181303 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.037644877e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157106176 dispatch_meta_bytes=229048 combine_payload_bytes=157106176 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157335224 ingress_bytes=175980544 link_bytes=175980544 token_bytes=314441400 weight_movement_bytes=18874368 total_nvlink_bytes=333315768 weight_to_token_ratio=0.060025 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686158985e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=228728 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157031800 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313834872 weight_movement_bytes=37748736 total_nvlink_bytes=351583608 weight_to_token_ratio=0.120282 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43902 payload_rows_remote=38234 dup_rows=21634 dup_rows_remote=18740 dedup_ratio=0.330109 router_tokens_per_expert=[253, 236, 278, 276, 264, 249, 262, 246, 268, 258, 238, 277, 276, 239, 253, 217, 240, 280, 263, 264, 264, 257, 252, 246, 258, 250, 221, 279, 232, 264, 275, 270, 238, 256, 274, 263, 275, 241, 262, 239, 262, 249, 250, 259, 251, 233, 228, 263, 266, 255, 252, 250, 254, 249, 240, 285, 256, 275, 264, 241, 260, 216, 259, 257, 286, 273, 244, 231, 252, 262, 263, 245, 269, 257, 266, 259, 262, 280, 267, 223, 273, 235, 265, 260, 286, 248, 250, 276, 254, 262, 261, 256, 231, 229, 250, 246, 235, 260, 235, 290, 241, 269, 264, 246, 251, 297, 255, 240, 282, 303, 228, 260, 260, 224, 229, 258, 262, 247, 236, 240, 256, 255, 248, 268, 261, 272, 273, 275] payload_rows_per_dst=[5668, 5544, 5644, 5448, 5526, 5540, 5274, 5258] dup_rows_per_dst=[2894, 2746, 3036, 2656, 2746, 2830, 2444, 2282] combine_rows_per_src=[5668, 5544, 5644, 5448, 5526, 5540, 5274, 5258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=155836416 dispatch_meta_bytes=228608 combine_payload_bytes=155836416 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156065024 ingress_bytes=193585152 link_bytes=193585152 token_bytes=311901440 weight_movement_bytes=37748736 total_nvlink_bytes=349650176 weight_to_token_ratio=0.121028 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.854837677e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44098 payload_rows_remote=38530 dup_rows=21438 dup_rows_remote=18678 dedup_ratio=0.327118 router_tokens_per_expert=[256, 228, 250, 266, 275, 264, 237, 265, 262, 260, 276, 245, 248, 254, 246, 248, 257, 255, 268, 238, 273, 251, 258, 265, 261, 253, 268, 285, 278, 247, 253, 244, 241, 230, 308, 245, 245, 267, 260, 269, 279, 255, 260, 264, 277, 250, 241, 273, 237, 250, 278, 235, 251, 248, 247, 294, 233, 269, 286, 254, 301, 253, 262, 254, 279, 247, 251, 259, 254, 244, 251, 247, 274, 242, 265, 232, 253, 252, 262, 259, 275, 228, 265, 256, 260, 262, 248, 221, 262, 246, 268, 256, 267, 229, 259, 267, 253, 220, 273, 263, 276, 255, 278, 215, 236, 262, 235, 253, 257, 245, 241, 256, 249, 271, 251, 250, 249, 272, 253, 249, 253, 249, 257, 261, 272, 231, 244, 249] payload_rows_per_dst=[5502, 5560, 5568, 5592, 5466, 5504, 5452, 5454] dup_rows_per_dst=[2658, 2748, 2760, 2712, 2676, 2634, 2584, 2666] combine_rows_per_src=[5502, 5560, 5568, 5592, 5466, 5504, 5452, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=227896 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156834360 ingress_bytes=175480832 link_bytes=175480832 token_bytes=313440824 weight_movement_bytes=18874368 total_nvlink_bytes=332315192 weight_to_token_ratio=0.060217 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681370990e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38210 dup_rows=21826 dup_rows_remote=18982 dedup_ratio=0.333038 router_tokens_per_expert=[263, 246, 254, 251, 258, 256, 287, 253, 277, 269, 232, 235, 264, 260, 255, 275, 284, 243, 286, 261, 240, 244, 249, 264, 289, 269, 282, 260, 246, 238, 260, 257, 226, 270, 242, 244, 282, 281, 255, 254, 252, 273, 232, 262, 257, 273, 235, 246, 246, 216, 258, 230, 268, 273, 226, 242, 264, 255, 269, 267, 251, 255, 257, 272, 276, 250, 249, 274, 245, 255, 268, 261, 265, 224, 288, 254, 237, 283, 244, 261, 238, 277, 233, 258, 254, 252, 279, 229, 239, 262, 234, 268, 253, 260, 262, 244, 250, 249, 282, 247, 263, 257, 262, 233, 269, 245, 268, 257, 240, 259, 222, 230, 249, 223, 248, 261, 256, 270, 265, 320, 235, 263, 278, 268, 275, 235, 245, 228] payload_rows_per_dst=[5534, 5500, 5386, 5488, 5470, 5452, 5408, 5472] dup_rows_per_dst=[2736, 2844, 2782, 2610, 2798, 2632, 2658, 2766] combine_rows_per_src=[5534, 5500, 5386, 5488, 5470, 5452, 5408, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43908 payload_rows_remote=38446 dup_rows=21628 dup_rows_remote=18970 dedup_ratio=0.330017 router_tokens_per_expert=[253, 241, 261, 249, 256, 241, 261, 263, 287, 240, 244, 270, 243, 260, 250, 229, 246, 228, 267, 276, 265, 268, 237, 252, 238, 267, 264, 267, 247, 260, 264, 264, 248, 243, 247, 239, 280, 259, 241, 276, 289, 276, 261, 260, 252, 243, 255, 244, 272, 268, 252, 262, 260, 264, 234, 249, 259, 231, 252, 248, 255, 252, 253, 268, 249, 242, 226, 227, 256, 278, 234, 281, 255, 234, 253, 288, 264, 225, 262, 256, 257, 250, 265, 256, 266, 245, 249, 242, 270, 252, 241, 234, 269, 261, 243, 260, 248, 238, 258, 246, 254, 248, 251, 264, 256, 279, 285, 264, 240, 260, 247, 275, 238, 248, 305, 264, 245, 246, 276, 251, 292, 271, 260, 259, 278, 240, 274, 268] payload_rows_per_dst=[5462, 5496, 5532, 5448, 5406, 5462, 5526, 5576] dup_rows_per_dst=[2634, 2724, 2694, 2710, 2654, 2658, 2700, 2854] combine_rows_per_src=[5462, 5496, 5532, 5448, 5406, 5462, 5526, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43678 payload_rows_remote=38206 dup_rows=21858 dup_rows_remote=19140 dedup_ratio=0.333527 router_tokens_per_expert=[251, 239, 240, 258, 250, 242, 269, 228, 252, 254, 286, 243, 250, 257, 265, 266, 239, 260, 272, 267, 252, 253, 273, 249, 231, 277, 278, 256, 266, 233, 257, 288, 274, 256, 263, 252, 257, 289, 263, 272, 274, 267, 258, 257, 273, 258, 235, 284, 240, 252, 276, 258, 231, 263, 245, 270, 251, 261, 248, 265, 269, 255, 280, 231, 252, 253, 216, 255, 267, 238, 259, 260, 227, 244, 273, 252, 274, 273, 263, 264, 286, 240, 268, 255, 256, 257, 241, 216, 263, 233, 258, 240, 235, 254, 263, 286, 292, 252, 242, 254, 279, 256, 271, 244, 267, 252, 276, 256, 231, 238, 258, 242, 241, 265, 257, 242, 279, 239, 246, 252, 247, 256, 230, 250, 261, 233, 249, 262] payload_rows_per_dst=[5404, 5530, 5582, 5472, 5428, 5420, 5460, 5382] dup_rows_per_dst=[2696, 2772, 2882, 2718, 2712, 2682, 2760, 2636] combine_rows_per_src=[5404, 5530, 5582, 5472, 5428, 5420, 5460, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38206 dup_rows=21896 dup_rows_remote=19142 dedup_ratio=0.334106 router_tokens_per_expert=[247, 281, 256, 258, 264, 271, 258, 252, 240, 270, 270, 266, 263, 262, 254, 266, 258, 252, 245, 231, 269, 243, 285, 267, 261, 271, 245, 258, 268, 245, 261, 264, 256, 250, 256, 246, 267, 273, 262, 233, 274, 264, 272, 277, 260, 271, 247, 263, 244, 247, 261, 233, 248, 254, 240, 287, 228, 255, 248, 249, 217, 240, 244, 246, 259, 272, 247, 268, 261, 244, 261, 271, 251, 276, 241, 247, 236, 261, 271, 228, 277, 254, 251, 251, 271, 249, 255, 243, 233, 246, 276, 242, 281, 284, 237, 254, 242, 232, 252, 277, 236, 243, 252, 289, 241, 266, 250, 240, 288, 266, 242, 271, 249, 255, 224, 256, 260, 260, 252, 275, 249, 264, 279, 254, 233, 265, 241, 254] payload_rows_per_dst=[5534, 5516, 5528, 5306, 5434, 5438, 5466, 5418] dup_rows_per_dst=[2822, 2730, 2814, 2576, 2754, 2770, 2708, 2722] combine_rows_per_src=[5534, 5516, 5528, 5306, 5434, 5438, 5466, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38500 dup_rows=21660 dup_rows_remote=18948 dedup_ratio=0.330505 router_tokens_per_expert=[253, 231, 271, 249, 264, 265, 270, 242, 254, 262, 271, 263, 245, 280, 249, 254, 252, 282, 264, 268, 249, 258, 265, 247, 260, 210, 278, 254, 238, 253, 245, 236, 264, 272, 257, 255, 270, 262, 232, 296, 254, 267, 244, 235, 244, 240, 274, 236, 252, 252, 269, 252, 283, 237, 228, 264, 261, 276, 265, 247, 267, 252, 278, 251, 256, 280, 233, 252, 279, 270, 249, 281, 239, 251, 228, 275, 263, 293, 238, 245, 271, 264, 288, 270, 241, 242, 292, 237, 259, 246, 225, 282, 270, 261, 242, 248, 235, 274, 258, 230, 239, 212, 274, 262, 271, 266, 228, 280, 232, 277, 266, 240, 242, 251, 279, 231, 257, 237, 238, 253, 265, 258, 271, 230, 256, 273, 263, 232] payload_rows_per_dst=[5540, 5486, 5466, 5460, 5614, 5530, 5376, 5404] dup_rows_per_dst=[2706, 2632, 2722, 2808, 2650, 2762, 2712, 2668] combine_rows_per_src=[5540, 5486, 5466, 5460, 5614, 5530, 5376, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157818880 dispatch_meta_bytes=228832 combine_payload_bytes=157818880 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158047712 ingress_bytes=195567616 link_bytes=195567616 token_bytes=315866592 weight_movement_bytes=37748736 total_nvlink_bytes=353615328 weight_to_token_ratio=0.119508 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.873832671e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43734 payload_rows_remote=38162 dup_rows=21802 dup_rows_remote=18980 dedup_ratio=0.332672 router_tokens_per_expert=[276, 262, 261, 267, 264, 245, 254, 266, 238, 239, 257, 243, 253, 257, 246, 275, 247, 257, 245, 251, 260, 244, 276, 235, 257, 223, 266, 265, 247, 254, 231, 244, 235, 222, 251, 268, 241, 289, 264, 266, 275, 280, 273, 267, 250, 227, 274, 268, 270, 283, 262, 294, 237, 258, 269, 271, 248, 244, 242, 274, 263, 243, 283, 268, 274, 263, 224, 249, 272, 262, 250, 233, 245, 255, 269, 246, 263, 259, 265, 251, 215, 234, 244, 243, 254, 270, 266, 236, 261, 253, 260, 273, 255, 229, 267, 300, 257, 250, 225, 254, 263, 261, 239, 271, 260, 263, 259, 275, 255, 249, 269, 268, 251, 247, 248, 265, 233, 277, 262, 258, 229, 246, 264, 274, 249, 263, 262, 218] payload_rows_per_dst=[5414, 5402, 5546, 5456, 5296, 5538, 5510, 5572] dup_rows_per_dst=[2792, 2602, 2754, 2760, 2506, 2780, 2786, 2822] combine_rows_per_src=[5414, 5402, 5546, 5456, 5296, 5538, 5510, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=228768 combine_payload_bytes=156508160 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156736928 ingress_bytes=194256896 link_bytes=194256896 token_bytes=313245088 weight_movement_bytes=37748736 total_nvlink_bytes=350993824 weight_to_token_ratio=0.120509 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861273997e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229664 combine_payload_bytes=157474816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157704480 ingress_bytes=176349184 link_bytes=176349184 token_bytes=315179296 weight_movement_bytes=18874368 total_nvlink_bytes=334053664 weight_to_token_ratio=0.059885 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689691112e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229384 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721160 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313212936 weight_movement_bytes=18874368 total_nvlink_bytes=332087304 weight_to_token_ratio=0.060260 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229392 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721168 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313212944 weight_movement_bytes=18874368 total_nvlink_bytes=332087312 weight_to_token_ratio=0.060260 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38254 dup_rows=21866 dup_rows_remote=19096 dedup_ratio=0.333649 router_tokens_per_expert=[264, 256, 250, 264, 246, 264, 280, 274, 233, 243, 273, 250, 223, 265, 243, 262, 246, 258, 247, 258, 247, 270, 276, 271, 263, 264, 236, 260, 258, 233, 240, 269, 225, 242, 245, 295, 274, 269, 236, 251, 277, 259, 240, 268, 240, 245, 245, 237, 249, 259, 255, 263, 277, 263, 236, 255, 301, 264, 234, 264, 252, 260, 245, 226, 236, 249, 262, 270, 243, 270, 299, 248, 267, 250, 262, 249, 240, 252, 250, 275, 251, 265, 251, 238, 240, 274, 272, 265, 263, 237, 265, 248, 255, 285, 266, 260, 224, 247, 238, 254, 259, 262, 246, 246, 274, 265, 236, 240, 250, 264, 239, 265, 267, 247, 257, 241, 245, 241, 286, 261, 254, 231, 285, 292, 284, 240, 258, 276] payload_rows_per_dst=[5416, 5502, 5382, 5484, 5450, 5516, 5386, 5534] dup_rows_per_dst=[2770, 2832, 2714, 2722, 2646, 2754, 2632, 2796] combine_rows_per_src=[5416, 5502, 5382, 5484, 5450, 5516, 5386, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=157696000 dispatch_meta_bytes=229792 combine_payload_bytes=157696000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157925792 ingress_bytes=176570368 link_bytes=176570368 token_bytes=315621792 weight_movement_bytes=18874368 total_nvlink_bytes=334496160 weight_to_token_ratio=0.059801 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691810388e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156311552 dispatch_meta_bytes=228568 combine_payload_bytes=156311552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156540120 ingress_bytes=175185920 link_bytes=175185920 token_bytes=312851672 weight_movement_bytes=18874368 total_nvlink_bytes=331726040 weight_to_token_ratio=0.060330 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678545289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229400 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156917784 ingress_bytes=213311488 link_bytes=213311488 token_bytes=313606168 weight_movement_bytes=56623104 total_nvlink_bytes=370229272 weight_to_token_ratio=0.180555 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043845722e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38214 dup_rows=21870 dup_rows_remote=19000 dedup_ratio=0.333710 router_tokens_per_expert=[275, 230, 278, 240, 250, 268, 249, 267, 256, 225, 287, 250, 249, 254, 251, 264, 272, 237, 262, 234, 292, 240, 264, 250, 252, 260, 271, 241, 234, 260, 264, 271, 290, 264, 256, 241, 255, 253, 254, 243, 250, 255, 293, 248, 263, 277, 273, 246, 278, 231, 261, 259, 230, 279, 259, 259, 264, 248, 269, 279, 246, 244, 223, 254, 227, 241, 234, 266, 258, 259, 279, 269, 259, 221, 271, 254, 250, 276, 277, 270, 289, 249, 273, 249, 242, 227, 236, 247, 270, 268, 249, 247, 246, 267, 269, 267, 220, 241, 253, 251, 283, 220, 251, 255, 233, 243, 264, 243, 292, 289, 249, 286, 248, 246, 233, 239, 256, 278, 251, 262, 265, 258, 274, 263, 223, 254, 237, 261] payload_rows_per_dst=[5446, 5432, 5452, 5466, 5484, 5448, 5440, 5498] dup_rows_per_dst=[2740, 2776, 2870, 2700, 2738, 2742, 2706, 2598] combine_rows_per_src=[5446, 5432, 5452, 5466, 5484, 5448, 5440, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38282 dup_rows=21716 dup_rows_remote=18970 dedup_ratio=0.331360 router_tokens_per_expert=[270, 249, 275, 264, 234, 250, 249, 272, 279, 244, 262, 258, 249, 229, 286, 267, 229, 275, 249, 268, 276, 259, 247, 268, 239, 244, 248, 258, 252, 273, 276, 281, 219, 241, 263, 256, 271, 255, 263, 224, 260, 274, 228, 253, 274, 243, 266, 242, 253, 262, 254, 251, 264, 259, 243, 250, 253, 279, 263, 288, 283, 252, 264, 265, 254, 260, 261, 247, 277, 254, 260, 227, 249, 267, 256, 245, 270, 232, 251, 263, 238, 238, 256, 274, 215, 261, 257, 239, 242, 239, 267, 266, 259, 236, 266, 271, 244, 275, 272, 258, 281, 240, 260, 271, 267, 260, 241, 232, 250, 248, 243, 253, 261, 245, 252, 245, 263, 273, 265, 249, 250, 245, 252, 268, 264, 265, 229, 256] payload_rows_per_dst=[5550, 5538, 5382, 5516, 5410, 5414, 5506, 5504] dup_rows_per_dst=[2724, 2746, 2682, 2850, 2736, 2634, 2684, 2660] combine_rows_per_src=[5550, 5538, 5382, 5516, 5410, 5414, 5506, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43958 payload_rows_remote=38424 dup_rows=21578 dup_rows_remote=18954 dedup_ratio=0.329254 router_tokens_per_expert=[244, 258, 239, 249, 267, 282, 260, 248, 258, 245, 261, 241, 242, 233, 255, 253, 241, 244, 258, 250, 269, 268, 264, 257, 258, 265, 251, 241, 228, 270, 232, 247, 279, 266, 272, 257, 246, 254, 263, 251, 254, 237, 268, 258, 254, 285, 266, 285, 260, 269, 227, 252, 257, 248, 235, 264, 275, 274, 251, 235, 239, 254, 285, 260, 269, 275, 272, 250, 262, 257, 254, 276, 261, 265, 276, 278, 253, 245, 259, 261, 217, 278, 267, 255, 265, 251, 272, 252, 265, 258, 225, 247, 264, 245, 271, 247, 267, 236, 248, 257, 281, 247, 273, 263, 242, 251, 273, 246, 254, 249, 268, 237, 249, 286, 280, 255, 270, 259, 223, 259, 258, 231, 266, 250, 224, 234, 245, 237] payload_rows_per_dst=[5440, 5434, 5600, 5462, 5572, 5534, 5522, 5394] dup_rows_per_dst=[2630, 2652, 2790, 2708, 2854, 2624, 2662, 2658] combine_rows_per_src=[5440, 5434, 5600, 5462, 5572, 5534, 5522, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43694 payload_rows_remote=38282 dup_rows=21842 dup_rows_remote=19164 dedup_ratio=0.333282 router_tokens_per_expert=[236, 288, 290, 278, 259, 247, 255, 276, 270, 237, 229, 236, 234, 245, 255, 254, 266, 246, 225, 254, 248, 255, 260, 253, 264, 247, 258, 240, 264, 279, 250, 259, 252, 287, 233, 243, 273, 272, 249, 258, 263, 257, 266, 264, 304, 226, 229, 236, 234, 241, 246, 251, 260, 286, 238, 240, 250, 242, 288, 245, 254, 254, 278, 238, 250, 262, 234, 258, 235, 236, 268, 267, 274, 251, 262, 272, 247, 260, 249, 241, 248, 276, 242, 263, 248, 230, 262, 261, 256, 266, 271, 269, 276, 249, 276, 242, 261, 249, 241, 252, 231, 265, 272, 254, 254, 243, 244, 262, 267, 234, 259, 272, 278, 261, 262, 263, 269, 226, 276, 255, 259, 269, 255, 301, 224, 244, 273, 278] payload_rows_per_dst=[5400, 5456, 5532, 5412, 5500, 5492, 5400, 5502] dup_rows_per_dst=[2778, 2680, 2692, 2678, 2632, 2778, 2720, 2884] combine_rows_per_src=[5400, 5456, 5532, 5412, 5500, 5492, 5400, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38320 dup_rows=21666 dup_rows_remote=18742 dedup_ratio=0.330597 router_tokens_per_expert=[277, 224, 281, 255, 256, 226, 244, 270, 250, 235, 254, 259, 253, 248, 241, 246, 259, 296, 256, 224, 266, 256, 248, 280, 253, 243, 238, 280, 272, 244, 255, 248, 225, 232, 239, 234, 244, 268, 253, 266, 260, 243, 261, 257, 290, 233, 256, 242, 256, 261, 261, 278, 239, 265, 266, 263, 258, 263, 252, 240, 249, 241, 258, 229, 286, 259, 310, 255, 251, 275, 260, 242, 234, 265, 283, 262, 262, 286, 235, 272, 259, 271, 240, 251, 272, 274, 247, 235, 237, 257, 278, 261, 248, 246, 233, 285, 245, 261, 238, 275, 240, 256, 247, 273, 274, 255, 261, 246, 239, 254, 242, 255, 257, 260, 256, 273, 277, 248, 256, 262, 257, 255, 260, 263, 255, 262, 253, 263] payload_rows_per_dst=[5454, 5502, 5346, 5550, 5550, 5498, 5466, 5504] dup_rows_per_dst=[2584, 2734, 2660, 2608, 2924, 2690, 2656, 2810] combine_rows_per_src=[5454, 5502, 5346, 5550, 5550, 5498, 5466, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=228856 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156753400 ingress_bytes=194273280 link_bytes=194273280 token_bytes=313277944 weight_movement_bytes=37748736 total_nvlink_bytes=351026680 weight_to_token_ratio=0.120496 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861430981e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38304 dup_rows=21772 dup_rows_remote=19024 dedup_ratio=0.332214 router_tokens_per_expert=[266, 250, 262, 287, 279, 255, 259, 261, 276, 260, 246, 283, 246, 240, 264, 264, 251, 287, 253, 276, 248, 242, 258, 258, 268, 293, 247, 274, 265, 210, 242, 262, 247, 267, 247, 258, 238, 228, 266, 264, 263, 235, 221, 274, 236, 263, 248, 255, 278, 251, 258, 243, 258, 256, 277, 248, 261, 258, 239, 277, 250, 268, 247, 253, 277, 261, 261, 224, 252, 255, 261, 234, 269, 249, 261, 254, 244, 262, 254, 239, 261, 261, 254, 269, 250, 224, 262, 283, 246, 245, 275, 281, 248, 247, 263, 274, 273, 260, 217, 264, 266, 268, 266, 266, 274, 250, 256, 231, 254, 240, 262, 257, 258, 278, 245, 238, 277, 253, 263, 241, 222, 220, 254, 257, 259, 226, 281, 228] payload_rows_per_dst=[5574, 5526, 5374, 5488, 5452, 5478, 5460, 5412] dup_rows_per_dst=[2822, 2742, 2646, 2756, 2662, 2808, 2748, 2588] combine_rows_per_src=[5574, 5526, 5374, 5488, 5452, 5478, 5460, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43628 payload_rows_remote=38110 dup_rows=21908 dup_rows_remote=19120 dedup_ratio=0.334290 router_tokens_per_expert=[249, 252, 255, 270, 257, 240, 251, 285, 268, 256, 245, 267, 254, 255, 256, 270, 236, 225, 264, 266, 261, 251, 249, 281, 245, 240, 280, 237, 229, 253, 241, 284, 274, 246, 237, 265, 291, 268, 264, 269, 267, 256, 243, 235, 249, 266, 239, 281, 256, 263, 258, 255, 273, 261, 245, 268, 240, 234, 257, 268, 248, 270, 249, 253, 261, 273, 234, 242, 275, 250, 235, 248, 246, 237, 300, 239, 252, 235, 270, 244, 268, 249, 243, 252, 260, 264, 248, 242, 296, 256, 255, 274, 243, 248, 235, 252, 246, 256, 276, 257, 275, 256, 246, 264, 262, 245, 239, 263, 243, 258, 238, 264, 244, 257, 279, 240, 257, 242, 249, 264, 222, 249, 261, 290, 256, 282, 285, 257] payload_rows_per_dst=[5466, 5462, 5450, 5452, 5398, 5416, 5466, 5518] dup_rows_per_dst=[2794, 2622, 2850, 2744, 2684, 2716, 2710, 2788] combine_rows_per_src=[5466, 5462, 5450, 5452, 5398, 5416, 5466, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229008 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157032080 ingress_bytes=175677440 link_bytes=175677440 token_bytes=313835152 weight_movement_bytes=18874368 total_nvlink_bytes=332709520 weight_to_token_ratio=0.060141 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683254791e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157384704 dispatch_meta_bytes=229512 combine_payload_bytes=157384704 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157614216 ingress_bytes=176259072 link_bytes=176259072 token_bytes=314998920 weight_movement_bytes=18874368 total_nvlink_bytes=333873288 weight_to_token_ratio=0.059919 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688827703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228248 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=157186968 ingress_bytes=232456192 link_bytes=232456192 token_bytes=314145688 weight_movement_bytes=75497472 total_nvlink_bytes=389643160 weight_to_token_ratio=0.240326 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227280856e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229784 combine_payload_bytes=156803072 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157032856 ingress_bytes=213426176 link_bytes=213426176 token_bytes=313835928 weight_movement_bytes=56623104 total_nvlink_bytes=370459032 weight_to_token_ratio=0.180423 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044944606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43684 payload_rows_remote=38124 dup_rows=21852 dup_rows_remote=19042 dedup_ratio=0.333435 router_tokens_per_expert=[267, 277, 257, 275, 247, 260, 248, 261, 275, 256, 242, 253, 268, 234, 236, 241, 238, 251, 269, 222, 238, 237, 242, 280, 289, 267, 250, 261, 253, 266, 280, 270, 230, 254, 268, 256, 274, 256, 264, 227, 266, 253, 254, 273, 254, 277, 230, 258, 252, 256, 250, 258, 233, 251, 281, 285, 250, 242, 237, 250, 308, 239, 273, 236, 260, 269, 275, 234, 244, 241, 247, 249, 247, 258, 247, 241, 277, 273, 251, 257, 256, 220, 243, 243, 257, 258, 250, 272, 256, 255, 267, 272, 272, 230, 257, 259, 243, 258, 257, 261, 247, 273, 260, 267, 263, 234, 267, 230, 255, 265, 243, 299, 245, 247, 227, 265, 261, 246, 258, 233, 273, 244, 271, 261, 290, 265, 272, 246] payload_rows_per_dst=[5560, 5574, 5464, 5314, 5486, 5510, 5332, 5444] dup_rows_per_dst=[2810, 2876, 2724, 2568, 2654, 2978, 2560, 2682] combine_rows_per_src=[5560, 5574, 5464, 5314, 5486, 5510, 5332, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=156893184 dispatch_meta_bytes=229312 combine_payload_bytes=156893184 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157122496 ingress_bytes=175767552 link_bytes=175767552 token_bytes=314015680 weight_movement_bytes=18874368 total_nvlink_bytes=332890048 weight_to_token_ratio=0.060106 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684118200e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156098560 dispatch_meta_bytes=228920 combine_payload_bytes=156098560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156327480 ingress_bytes=174972928 link_bytes=174972928 token_bytes=312426040 weight_movement_bytes=18874368 total_nvlink_bytes=331300408 weight_to_token_ratio=0.060412 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676504504e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=228664 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156384568 ingress_bytes=175030272 link_bytes=175030272 token_bytes=312540472 weight_movement_bytes=18874368 total_nvlink_bytes=331414840 weight_to_token_ratio=0.060390 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677053946e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43912 payload_rows_remote=38446 dup_rows=21624 dup_rows_remote=18942 dedup_ratio=0.329956 router_tokens_per_expert=[249, 242, 267, 269, 252, 244, 273, 266, 268, 258, 260, 285, 277, 229, 263, 235, 224, 253, 261, 251, 277, 256, 244, 283, 264, 269, 240, 236, 268, 275, 253, 261, 265, 219, 276, 257, 239, 284, 242, 267, 258, 263, 280, 249, 228, 245, 250, 252, 254, 289, 275, 275, 277, 265, 247, 308, 262, 266, 248, 250, 241, 256, 268, 231, 269, 244, 248, 292, 228, 243, 256, 255, 253, 263, 271, 257, 274, 265, 228, 264, 248, 261, 257, 256, 270, 254, 259, 264, 283, 244, 251, 267, 269, 244, 235, 242, 267, 262, 250, 244, 228, 273, 270, 241, 253, 257, 225, 207, 247, 290, 243, 258, 270, 238, 270, 253, 236, 225, 244, 278, 271, 247, 254, 231, 242, 254, 250, 238] payload_rows_per_dst=[5486, 5524, 5466, 5648, 5444, 5436, 5440, 5468] dup_rows_per_dst=[2788, 2706, 2682, 2776, 2776, 2772, 2590, 2534] combine_rows_per_src=[5486, 5524, 5466, 5648, 5444, 5436, 5440, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38314 dup_rows=21798 dup_rows_remote=19002 dedup_ratio=0.332611 router_tokens_per_expert=[263, 275, 263, 283, 247, 268, 269, 252, 243, 242, 234, 214, 261, 247, 259, 268, 285, 272, 249, 257, 239, 253, 258, 234, 238, 261, 264, 264, 256, 249, 269, 262, 282, 250, 249, 251, 238, 245, 253, 263, 252, 243, 247, 264, 269, 254, 242, 264, 247, 235, 254, 281, 239, 233, 232, 282, 249, 262, 270, 287, 269, 266, 282, 267, 281, 244, 259, 280, 243, 231, 236, 210, 229, 248, 262, 266, 246, 281, 293, 256, 258, 251, 227, 285, 261, 261, 255, 260, 247, 240, 267, 269, 267, 225, 239, 275, 239, 255, 251, 264, 262, 253, 231, 247, 270, 271, 246, 304, 254, 242, 260, 266, 233, 277, 275, 279, 235, 292, 251, 267, 235, 242, 260, 248, 246, 233, 268, 241] payload_rows_per_dst=[5420, 5424, 5416, 5512, 5454, 5498, 5510, 5504] dup_rows_per_dst=[2756, 2796, 2716, 2798, 2676, 2676, 2720, 2660] combine_rows_per_src=[5420, 5424, 5416, 5512, 5454, 5498, 5510, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38200 dup_rows=21914 dup_rows_remote=19292 dedup_ratio=0.334381 router_tokens_per_expert=[251, 275, 226, 272, 236, 247, 251, 276, 242, 268, 271, 257, 286, 237, 235, 280, 246, 276, 265, 268, 258, 292, 232, 239, 253, 270, 221, 281, 251, 250, 256, 271, 253, 286, 239, 260, 257, 249, 223, 227, 248, 274, 263, 258, 238, 269, 251, 266, 263, 253, 252, 259, 269, 251, 252, 280, 223, 274, 245, 246, 255, 270, 265, 283, 262, 265, 238, 243, 287, 250, 263, 275, 276, 248, 228, 244, 260, 251, 242, 264, 246, 281, 259, 268, 260, 251, 243, 236, 270, 257, 239, 264, 239, 228, 265, 216, 244, 238, 239, 268, 267, 257, 261, 258, 278, 272, 254, 262, 258, 273, 250, 243, 251, 242, 272, 255, 255, 244, 259, 279, 251, 252, 247, 237, 272, 256, 271, 245] payload_rows_per_dst=[5450, 5500, 5478, 5448, 5434, 5422, 5422, 5468] dup_rows_per_dst=[2770, 2758, 2644, 2832, 2758, 2622, 2822, 2708] combine_rows_per_src=[5450, 5500, 5478, 5448, 5434, 5422, 5422, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38154 dup_rows=21902 dup_rows_remote=19330 dedup_ratio=0.334198 router_tokens_per_expert=[276, 265, 262, 243, 247, 242, 254, 240, 294, 247, 261, 241, 274, 256, 239, 256, 270, 260, 231, 255, 265, 280, 262, 276, 252, 246, 236, 256, 267, 271, 259, 273, 236, 275, 253, 280, 253, 296, 252, 227, 245, 271, 252, 248, 276, 250, 267, 255, 256, 218, 239, 253, 245, 233, 253, 274, 251, 237, 261, 265, 253, 252, 256, 271, 289, 237, 254, 234, 258, 222, 226, 255, 259, 266, 250, 243, 276, 247, 272, 238, 254, 259, 264, 261, 281, 258, 262, 217, 270, 287, 255, 289, 260, 248, 263, 242, 259, 244, 244, 277, 246, 232, 239, 250, 248, 252, 253, 283, 270, 260, 256, 269, 257, 252, 277, 249, 253, 273, 238, 246, 254, 233, 235, 235, 260, 279, 280, 260] payload_rows_per_dst=[5406, 5490, 5404, 5432, 5480, 5534, 5462, 5426] dup_rows_per_dst=[2788, 2828, 2868, 2602, 2572, 2806, 2702, 2736] combine_rows_per_src=[5406, 5490, 5404, 5432, 5480, 5534, 5462, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38324 dup_rows=21696 dup_rows_remote=19122 dedup_ratio=0.331055 router_tokens_per_expert=[255, 257, 248, 245, 228, 275, 234, 241, 241, 240, 264, 246, 230, 288, 238, 263, 245, 227, 288, 234, 246, 280, 262, 279, 267, 240, 248, 259, 259, 270, 251, 261, 265, 235, 288, 263, 276, 232, 251, 258, 270, 273, 257, 239, 247, 256, 237, 266, 247, 270, 243, 266, 261, 244, 242, 256, 247, 245, 234, 269, 253, 264, 260, 244, 266, 264, 242, 252, 274, 256, 279, 271, 248, 268, 258, 245, 256, 266, 262, 232, 282, 247, 236, 261, 258, 272, 252, 271, 258, 279, 235, 278, 259, 267, 270, 259, 267, 245, 252, 273, 252, 263, 237, 272, 235, 270, 239, 237, 254, 229, 284, 265, 242, 259, 260, 256, 243, 275, 277, 231, 253, 264, 248, 267, 239, 279, 272, 239] payload_rows_per_dst=[5402, 5490, 5486, 5516, 5540, 5520, 5446, 5440] dup_rows_per_dst=[2584, 2742, 2740, 2574, 2738, 2848, 2702, 2768] combine_rows_per_src=[5402, 5490, 5486, 5516, 5540, 5520, 5446, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157474816 dispatch_meta_bytes=229552 combine_payload_bytes=157474816 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157704368 ingress_bytes=214097920 link_bytes=214097920 token_bytes=315179184 weight_movement_bytes=56623104 total_nvlink_bytes=371802288 weight_to_token_ratio=0.179654 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.051380927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38206 dup_rows=21868 dup_rows_remote=19088 dedup_ratio=0.333679 router_tokens_per_expert=[287, 271, 258, 260, 254, 256, 250, 251, 243, 261, 238, 250, 258, 246, 246, 281, 234, 256, 257, 230, 250, 252, 303, 261, 218, 261, 234, 261, 244, 237, 242, 242, 273, 245, 275, 234, 267, 251, 227, 230, 272, 241, 259, 251, 271, 265, 211, 214, 279, 222, 254, 258, 250, 250, 251, 225, 253, 245, 267, 270, 278, 265, 256, 259, 260, 253, 260, 261, 263, 246, 248, 245, 239, 296, 252, 249, 285, 289, 270, 276, 245, 263, 252, 253, 265, 237, 259, 256, 256, 243, 261, 294, 268, 237, 255, 280, 255, 255, 257, 256, 244, 260, 255, 263, 255, 260, 249, 266, 256, 244, 270, 276, 239, 249, 256, 257, 257, 297, 243, 253, 262, 291, 262, 282, 249, 264, 264, 246] payload_rows_per_dst=[5532, 5352, 5306, 5496, 5514, 5508, 5462, 5498] dup_rows_per_dst=[2688, 2612, 2666, 2668, 2870, 2740, 2780, 2844] combine_rows_per_src=[5532, 5352, 5306, 5496, 5514, 5508, 5462, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43928 payload_rows_remote=38416 dup_rows=21608 dup_rows_remote=18814 dedup_ratio=0.329712 router_tokens_per_expert=[220, 252, 261, 246, 233, 276, 259, 281, 275, 260, 286, 241, 234, 253, 272, 211, 233, 249, 246, 246, 254, 234, 270, 276, 277, 255, 261, 254, 277, 241, 270, 252, 267, 263, 249, 254, 261, 230, 257, 279, 299, 257, 279, 273, 279, 271, 250, 261, 254, 266, 266, 261, 245, 266, 212, 249, 264, 252, 269, 247, 260, 247, 262, 273, 264, 269, 287, 269, 239, 268, 256, 253, 228, 231, 272, 248, 257, 259, 231, 222, 222, 272, 274, 239, 267, 280, 253, 261, 241, 253, 250, 231, 277, 233, 252, 262, 276, 263, 281, 279, 246, 251, 243, 253, 255, 229, 258, 247, 256, 239, 256, 255, 243, 261, 260, 260, 253, 264, 289, 244, 243, 261, 240, 254, 252, 258, 258, 244] payload_rows_per_dst=[5176, 5560, 5514, 5590, 5528, 5596, 5452, 5512] dup_rows_per_dst=[2238, 2630, 2760, 2642, 2916, 2882, 2746, 2794] combine_rows_per_src=[5176, 5560, 5514, 5590, 5528, 5596, 5452, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=229264 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163408 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314097552 weight_movement_bytes=18874368 total_nvlink_bytes=332971920 weight_to_token_ratio=0.060091 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229968 combine_payload_bytes=156467200 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156697168 ingress_bytes=213090304 link_bytes=213090304 token_bytes=313164368 weight_movement_bytes=56623104 total_nvlink_bytes=369787472 weight_to_token_ratio=0.180810 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041726446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156278784 dispatch_meta_bytes=229936 combine_payload_bytes=156278784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156508720 ingress_bytes=175153152 link_bytes=175153152 token_bytes=312787504 weight_movement_bytes=18874368 total_nvlink_bytes=331661872 weight_to_token_ratio=0.060342 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678231322e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229176 combine_payload_bytes=156491776 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156720952 ingress_bytes=194240512 link_bytes=194240512 token_bytes=313212728 weight_movement_bytes=37748736 total_nvlink_bytes=350961464 weight_to_token_ratio=0.120521 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861117014e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=156975104 dispatch_meta_bytes=229784 combine_payload_bytes=156975104 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157204888 ingress_bytes=175849472 link_bytes=175849472 token_bytes=314179992 weight_movement_bytes=18874368 total_nvlink_bytes=333054360 weight_to_token_ratio=0.060075 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684903117e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38326 dup_rows=21672 dup_rows_remote=19108 dedup_ratio=0.330688 router_tokens_per_expert=[263, 242, 259, 242, 269, 251, 268, 253, 251, 239, 243, 272, 236, 259, 242, 262, 237, 257, 257, 260, 233, 238, 260, 269, 240, 290, 231, 282, 279, 265, 249, 260, 271, 271, 255, 250, 230, 288, 250, 260, 275, 242, 253, 253, 250, 246, 255, 243, 252, 290, 287, 265, 280, 272, 235, 242, 253, 266, 248, 256, 269, 245, 237, 245, 232, 261, 272, 295, 250, 248, 254, 264, 255, 251, 234, 266, 252, 256, 267, 229, 260, 258, 264, 273, 248, 216, 252, 258, 255, 262, 250, 275, 246, 259, 269, 234, 232, 265, 240, 269, 276, 251, 288, 239, 233, 266, 274, 273, 243, 260, 232, 235, 263, 245, 252, 258, 261, 237, 258, 270, 251, 251, 246, 278, 277, 267, 268, 253] payload_rows_per_dst=[5538, 5458, 5460, 5508, 5526, 5496, 5446, 5432] dup_rows_per_dst=[2564, 2756, 2724, 2776, 2892, 2662, 2706, 2592] combine_rows_per_src=[5538, 5458, 5460, 5508, 5526, 5496, 5446, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157351936 dispatch_meta_bytes=228920 combine_payload_bytes=157351936 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157580856 ingress_bytes=176226304 link_bytes=176226304 token_bytes=314932792 weight_movement_bytes=18874368 total_nvlink_bytes=333807160 weight_to_token_ratio=0.059931 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688513736e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43994 payload_rows_remote=38562 dup_rows=21542 dup_rows_remote=18904 dedup_ratio=0.328705 router_tokens_per_expert=[262, 238, 254, 248, 260, 278, 276, 265, 253, 274, 278, 289, 246, 255, 259, 231, 266, 265, 219, 254, 240, 237, 255, 242, 258, 253, 261, 253, 267, 265, 248, 264, 256, 253, 269, 260, 245, 245, 244, 292, 252, 235, 242, 246, 243, 262, 234, 257, 280, 239, 263, 229, 259, 270, 245, 251, 242, 258, 278, 253, 277, 257, 260, 254, 245, 235, 244, 309, 264, 268, 247, 281, 238, 275, 246, 237, 240, 285, 273, 260, 259, 269, 254, 230, 236, 262, 249, 243, 257, 264, 253, 263, 262, 265, 260, 216, 256, 217, 258, 266, 280, 275, 259, 250, 265, 267, 265, 269, 272, 241, 239, 249, 257, 253, 269, 247, 240, 255, 251, 265, 267, 238, 236, 284, 266, 262, 217, 281] payload_rows_per_dst=[5552, 5446, 5432, 5562, 5564, 5416, 5482, 5540] dup_rows_per_dst=[2780, 2648, 2638, 2668, 2730, 2668, 2774, 2636] combine_rows_per_src=[5552, 5446, 5432, 5562, 5564, 5416, 5482, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229736 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157213032 ingress_bytes=213606400 link_bytes=213606400 token_bytes=314196328 weight_movement_bytes=56623104 total_nvlink_bytes=370819432 weight_to_token_ratio=0.180216 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046671424e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43874 payload_rows_remote=38412 dup_rows=21662 dup_rows_remote=18958 dedup_ratio=0.330536 router_tokens_per_expert=[269, 247, 255, 274, 275, 263, 259, 270, 243, 239, 243, 283, 243, 278, 258, 260, 252, 264, 258, 269, 254, 244, 272, 247, 251, 270, 239, 242, 261, 253, 239, 268, 278, 255, 247, 253, 265, 248, 282, 245, 282, 271, 280, 241, 240, 282, 252, 247, 255, 261, 274, 254, 258, 240, 252, 258, 274, 232, 301, 238, 266, 245, 221, 278, 251, 278, 258, 244, 261, 228, 268, 260, 258, 253, 268, 232, 255, 266, 281, 249, 233, 254, 264, 251, 229, 230, 212, 232, 266, 257, 270, 248, 250, 249, 234, 226, 249, 264, 267, 242, 271, 270, 231, 266, 269, 230, 238, 258, 250, 267, 281, 264, 252, 281, 275, 261, 253, 259, 259, 219, 279, 253, 280, 235, 227, 278, 260, 248] payload_rows_per_dst=[5608, 5462, 5504, 5480, 5526, 5310, 5440, 5544] dup_rows_per_dst=[2710, 2704, 2832, 2734, 2694, 2500, 2794, 2694] combine_rows_per_src=[5608, 5462, 5504, 5480, 5526, 5310, 5440, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157949952 dispatch_meta_bytes=229864 combine_payload_bytes=157949952 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=158179816 ingress_bytes=214573056 link_bytes=214573056 token_bytes=316129768 weight_movement_bytes=56623104 total_nvlink_bytes=372752872 weight_to_token_ratio=0.179113 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.055933446e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43916 payload_rows_remote=38524 dup_rows=21620 dup_rows_remote=18926 dedup_ratio=0.329895 router_tokens_per_expert=[263, 244, 250, 292, 258, 252, 249, 279, 274, 255, 249, 260, 265, 261, 265, 250, 246, 236, 207, 238, 264, 242, 235, 282, 253, 266, 237, 243, 238, 286, 269, 227, 255, 251, 244, 285, 271, 272, 240, 274, 288, 241, 243, 253, 283, 274, 256, 224, 244, 259, 258, 266, 266, 237, 265, 281, 264, 275, 266, 260, 252, 254, 252, 261, 258, 265, 247, 266, 242, 256, 238, 275, 295, 239, 256, 291, 258, 244, 247, 253, 236, 264, 233, 237, 274, 233, 267, 245, 246, 244, 254, 254, 251, 265, 258, 282, 230, 250, 273, 282, 276, 270, 253, 276, 257, 237, 245, 236, 269, 268, 246, 243, 256, 243, 263, 240, 241, 246, 273, 270, 257, 251, 229, 262, 242, 254, 246, 262] payload_rows_per_dst=[5568, 5456, 5480, 5506, 5484, 5392, 5528, 5502] dup_rows_per_dst=[2764, 2482, 2828, 2814, 2776, 2694, 2694, 2568] combine_rows_per_src=[5568, 5456, 5480, 5506, 5484, 5392, 5528, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38260 dup_rows=21818 dup_rows_remote=19066 dedup_ratio=0.332916 router_tokens_per_expert=[262, 249, 248, 242, 267, 255, 237, 250, 273, 275, 259, 276, 261, 240, 232, 292, 250, 272, 239, 253, 253, 277, 244, 269, 250, 282, 283, 232, 293, 254, 248, 241, 261, 228, 257, 263, 262, 252, 227, 260, 246, 266, 239, 257, 274, 233, 233, 271, 259, 255, 244, 255, 258, 254, 262, 279, 259, 297, 238, 253, 263, 270, 242, 244, 280, 262, 258, 272, 242, 258, 238, 243, 228, 236, 269, 274, 233, 274, 267, 271, 219, 248, 280, 252, 239, 236, 257, 259, 274, 250, 259, 263, 256, 245, 251, 247, 235, 264, 261, 251, 250, 269, 263, 265, 259, 248, 241, 265, 273, 288, 234, 240, 236, 260, 263, 256, 250, 253, 271, 263, 258, 266, 249, 245, 224, 262, 283, 264] payload_rows_per_dst=[5498, 5526, 5416, 5488, 5458, 5362, 5506, 5464] dup_rows_per_dst=[2738, 2754, 2642, 2776, 2752, 2708, 2706, 2742] combine_rows_per_src=[5498, 5526, 5416, 5488, 5458, 5362, 5506, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43594 payload_rows_remote=38126 dup_rows=21942 dup_rows_remote=19082 dedup_ratio=0.334808 router_tokens_per_expert=[222, 254, 262, 273, 265, 272, 253, 235, 288, 257, 256, 265, 253, 273, 279, 243, 291, 248, 261, 238, 247, 241, 283, 299, 253, 235, 234, 247, 291, 282, 232, 251, 249, 231, 259, 258, 262, 287, 256, 246, 260, 216, 232, 254, 259, 253, 249, 265, 228, 250, 266, 265, 265, 265, 253, 250, 241, 268, 248, 240, 225, 256, 274, 258, 248, 257, 246, 261, 271, 227, 234, 239, 236, 257, 247, 244, 236, 243, 242, 243, 260, 266, 266, 237, 243, 259, 246, 247, 307, 266, 276, 290, 280, 267, 292, 219, 251, 270, 258, 275, 256, 250, 253, 248, 263, 258, 276, 276, 255, 265, 241, 269, 265, 282, 226, 283, 251, 279, 247, 257, 244, 283, 258, 240, 258, 241, 230, 237] payload_rows_per_dst=[5508, 5476, 5414, 5460, 5328, 5530, 5468, 5410] dup_rows_per_dst=[2792, 2790, 2658, 2644, 2534, 2912, 2860, 2752] combine_rows_per_src=[5508, 5476, 5414, 5460, 5328, 5530, 5468, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38376 dup_rows=21816 dup_rows_remote=19140 dedup_ratio=0.332886 router_tokens_per_expert=[276, 267, 235, 277, 255, 224, 267, 271, 274, 250, 261, 268, 254, 266, 279, 267, 244, 254, 271, 274, 254, 260, 264, 249, 248, 255, 245, 264, 257, 257, 249, 251, 251, 236, 246, 234, 306, 269, 229, 224, 255, 254, 239, 279, 247, 269, 259, 262, 239, 240, 230, 277, 245, 282, 234, 235, 269, 253, 255, 248, 223, 259, 253, 268, 251, 254, 238, 238, 254, 264, 252, 264, 279, 286, 273, 252, 249, 268, 259, 255, 244, 249, 256, 272, 266, 283, 262, 239, 247, 250, 252, 252, 271, 248, 282, 249, 244, 254, 249, 233, 278, 276, 275, 241, 258, 256, 248, 252, 278, 250, 233, 265, 259, 264, 250, 256, 230, 251, 242, 255, 255, 245, 230, 252, 277, 248, 270, 280] payload_rows_per_dst=[5538, 5456, 5416, 5344, 5520, 5476, 5524, 5446] dup_rows_per_dst=[2844, 2736, 2702, 2676, 2752, 2768, 2656, 2682] combine_rows_per_src=[5538, 5456, 5416, 5344, 5520, 5476, 5524, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38136 dup_rows=21792 dup_rows_remote=18764 dedup_ratio=0.332520 router_tokens_per_expert=[237, 266, 282, 234, 250, 245, 267, 267, 256, 251, 280, 276, 262, 242, 252, 255, 266, 267, 260, 288, 277, 245, 256, 247, 283, 241, 233, 235, 265, 257, 234, 275, 272, 256, 250, 243, 285, 234, 266, 254, 247, 254, 237, 274, 273, 281, 255, 284, 231, 243, 233, 264, 241, 246, 244, 256, 248, 267, 270, 254, 263, 270, 266, 259, 262, 243, 260, 254, 234, 284, 267, 277, 231, 265, 249, 253, 261, 248, 233, 258, 258, 278, 237, 242, 256, 258, 242, 254, 267, 244, 251, 268, 256, 256, 245, 248, 263, 245, 265, 253, 248, 266, 249, 250, 283, 271, 269, 232, 248, 220, 241, 251, 253, 258, 278, 254, 249, 272, 244, 251, 243, 273, 247, 252, 267, 269, 250, 244] payload_rows_per_dst=[5266, 5584, 5452, 5446, 5404, 5502, 5482, 5608] dup_rows_per_dst=[2530, 2674, 2810, 2664, 2764, 2618, 2704, 3028] combine_rows_per_src=[5266, 5584, 5452, 5446, 5404, 5502, 5482, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=229480 combine_payload_bytes=157335552 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157565032 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900584 weight_movement_bytes=18874368 total_nvlink_bytes=333774952 weight_to_token_ratio=0.059938 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157794304 dispatch_meta_bytes=229800 combine_payload_bytes=157794304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158024104 ingress_bytes=176668672 link_bytes=176668672 token_bytes=315818408 weight_movement_bytes=18874368 total_nvlink_bytes=334692776 weight_to_token_ratio=0.059763 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692752289e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229304 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942264 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655224 weight_movement_bytes=18874368 total_nvlink_bytes=332529592 weight_to_token_ratio=0.060176 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156164096 dispatch_meta_bytes=228832 combine_payload_bytes=156164096 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156392928 ingress_bytes=175038464 link_bytes=175038464 token_bytes=312557024 weight_movement_bytes=18874368 total_nvlink_bytes=331431392 weight_to_token_ratio=0.060387 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677132438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=157188096 dispatch_meta_bytes=230064 combine_payload_bytes=157188096 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157418160 ingress_bytes=194936832 link_bytes=194936832 token_bytes=314606256 weight_movement_bytes=37748736 total_nvlink_bytes=352354992 weight_to_token_ratio=0.119987 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867788809e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=227600 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156432656 ingress_bytes=175079424 link_bytes=175079424 token_bytes=312637712 weight_movement_bytes=18874368 total_nvlink_bytes=331512080 weight_to_token_ratio=0.060371 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677524896e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38334 dup_rows=21698 dup_rows_remote=18988 dedup_ratio=0.331085 router_tokens_per_expert=[245, 251, 283, 261, 226, 247, 228, 289, 243, 275, 230, 256, 231, 256, 250, 263, 265, 257, 237, 265, 255, 231, 232, 255, 257, 258, 254, 236, 248, 246, 251, 277, 287, 255, 262, 269, 253, 254, 261, 251, 262, 267, 250, 282, 245, 265, 236, 260, 253, 225, 244, 256, 283, 252, 240, 257, 283, 262, 253, 250, 265, 246, 252, 256, 228, 271, 259, 241, 271, 258, 265, 277, 287, 267, 232, 245, 282, 259, 253, 271, 266, 241, 239, 243, 244, 250, 260, 251, 259, 258, 240, 270, 258, 250, 268, 268, 267, 302, 241, 244, 267, 265, 258, 266, 262, 258, 251, 262, 277, 248, 263, 259, 261, 250, 284, 249, 228, 265, 276, 248, 235, 243, 239, 258, 256, 248, 250, 263] payload_rows_per_dst=[5504, 5448, 5434, 5366, 5558, 5562, 5534, 5432] dup_rows_per_dst=[2710, 2766, 2770, 2622, 2742, 2568, 2846, 2674] combine_rows_per_src=[5504, 5448, 5434, 5366, 5558, 5562, 5534, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38374 dup_rows=21758 dup_rows_remote=19008 dedup_ratio=0.332001 router_tokens_per_expert=[236, 270, 256, 277, 258, 288, 261, 263, 256, 257, 276, 245, 267, 254, 273, 252, 259, 235, 284, 262, 258, 253, 233, 236, 243, 260, 249, 266, 259, 292, 251, 241, 263, 250, 274, 230, 260, 269, 219, 261, 254, 260, 241, 238, 277, 280, 250, 251, 263, 249, 245, 260, 249, 239, 262, 251, 278, 264, 250, 273, 250, 258, 267, 224, 265, 252, 258, 252, 237, 270, 240, 296, 273, 243, 290, 266, 271, 267, 256, 238, 264, 247, 263, 266, 252, 227, 268, 254, 262, 239, 246, 258, 262, 225, 263, 235, 265, 227, 249, 252, 252, 240, 272, 237, 242, 239, 237, 281, 263, 254, 246, 251, 233, 256, 256, 250, 269, 268, 287, 270, 271, 258, 249, 255, 239, 268, 251, 247] payload_rows_per_dst=[5524, 5466, 5404, 5396, 5586, 5412, 5406, 5584] dup_rows_per_dst=[2854, 2696, 2750, 2768, 2762, 2650, 2608, 2670] combine_rows_per_src=[5524, 5466, 5404, 5396, 5586, 5412, 5406, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38344 dup_rows=21746 dup_rows_remote=19134 dedup_ratio=0.331818 router_tokens_per_expert=[252, 253, 271, 242, 263, 265, 270, 241, 267, 276, 263, 251, 257, 248, 274, 247, 264, 255, 255, 283, 233, 228, 260, 265, 250, 233, 242, 254, 252, 246, 272, 237, 265, 267, 270, 252, 262, 245, 277, 238, 261, 259, 275, 255, 260, 272, 235, 250, 239, 266, 246, 253, 248, 286, 261, 253, 272, 249, 267, 212, 251, 253, 262, 223, 283, 265, 242, 247, 249, 248, 244, 249, 259, 242, 269, 265, 270, 253, 280, 254, 259, 247, 259, 254, 268, 268, 247, 257, 228, 259, 246, 245, 273, 242, 248, 287, 261, 243, 266, 283, 245, 226, 271, 281, 292, 275, 260, 270, 229, 269, 242, 258, 240, 268, 253, 214, 264, 262, 261, 243, 261, 260, 245, 253, 242, 248, 253, 271] payload_rows_per_dst=[5558, 5446, 5542, 5422, 5498, 5454, 5492, 5378] dup_rows_per_dst=[2722, 2612, 2744, 2660, 2740, 2720, 2850, 2698] combine_rows_per_src=[5558, 5446, 5542, 5422, 5498, 5454, 5492, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=229528 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157409432 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314589336 weight_movement_bytes=18874368 total_nvlink_bytes=333463704 weight_to_token_ratio=0.059997 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229288 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245352 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261416 weight_movement_bytes=18874368 total_nvlink_bytes=333135784 weight_to_token_ratio=0.060059 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38342 dup_rows=21760 dup_rows_remote=19092 dedup_ratio=0.332031 router_tokens_per_expert=[251, 262, 255, 237, 239, 274, 245, 246, 260, 265, 253, 245, 242, 267, 225, 259, 288, 241, 252, 252, 241, 276, 270, 270, 273, 256, 240, 266, 229, 240, 228, 265, 250, 288, 289, 278, 265, 256, 252, 277, 241, 228, 243, 263, 226, 241, 240, 257, 287, 258, 263, 280, 261, 255, 261, 273, 219, 269, 251, 283, 242, 262, 235, 274, 255, 267, 245, 252, 254, 256, 263, 272, 250, 242, 273, 268, 242, 285, 252, 280, 256, 234, 246, 272, 263, 264, 247, 262, 250, 233, 258, 260, 253, 230, 261, 262, 254, 257, 255, 293, 268, 251, 239, 278, 229, 237, 236, 269, 272, 252, 276, 249, 290, 241, 245, 243, 261, 230, 260, 259, 229, 278, 251, 267, 286, 252, 226, 249] payload_rows_per_dst=[5400, 5420, 5482, 5504, 5496, 5434, 5564, 5476] dup_rows_per_dst=[2650, 2754, 2706, 2842, 2816, 2668, 2666, 2658] combine_rows_per_src=[5400, 5420, 5482, 5504, 5496, 5434, 5564, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38344 dup_rows=21716 dup_rows_remote=18878 dedup_ratio=0.331360 router_tokens_per_expert=[275, 258, 247, 249, 245, 245, 241, 263, 267, 246, 249, 262, 273, 236, 247, 299, 254, 243, 265, 264, 223, 258, 256, 280, 253, 234, 275, 268, 248, 244, 236, 265, 266, 249, 261, 282, 221, 257, 265, 236, 253, 245, 250, 228, 285, 263, 286, 263, 252, 232, 231, 239, 277, 253, 252, 286, 252, 261, 265, 249, 248, 262, 254, 249, 262, 258, 277, 260, 247, 267, 244, 238, 273, 271, 266, 252, 243, 275, 281, 243, 254, 248, 257, 265, 230, 257, 238, 256, 258, 268, 259, 288, 274, 254, 275, 269, 280, 276, 264, 264, 260, 227, 249, 241, 237, 270, 271, 229, 271, 252, 244, 240, 238, 236, 235, 234, 282, 258, 251, 239, 241, 270, 279, 247, 274, 273, 250, 239] payload_rows_per_dst=[5496, 5468, 5484, 5434, 5476, 5548, 5484, 5430] dup_rows_per_dst=[2708, 2664, 2736, 2690, 2838, 2752, 2666, 2662] combine_rows_per_src=[5496, 5468, 5484, 5434, 5476, 5548, 5484, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229912 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286936 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343960 weight_movement_bytes=18874368 total_nvlink_bytes=333218328 weight_to_token_ratio=0.060044 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43932 payload_rows_remote=38418 dup_rows=21604 dup_rows_remote=18828 dedup_ratio=0.329651 router_tokens_per_expert=[243, 275, 266, 265, 234, 296, 255, 250, 258, 273, 261, 252, 258, 272, 284, 268, 257, 250, 258, 249, 263, 260, 254, 255, 274, 300, 248, 279, 250, 265, 247, 245, 279, 255, 249, 261, 248, 223, 227, 263, 271, 254, 254, 279, 267, 272, 239, 268, 253, 264, 267, 259, 266, 254, 248, 244, 253, 245, 256, 247, 278, 259, 247, 236, 251, 259, 261, 249, 261, 262, 230, 266, 245, 252, 263, 254, 245, 233, 239, 264, 245, 261, 240, 244, 243, 233, 246, 259, 235, 228, 299, 245, 253, 230, 264, 254, 267, 271, 236, 286, 289, 249, 241, 252, 293, 267, 274, 230, 269, 226, 233, 262, 263, 247, 233, 234, 273, 258, 269, 253, 275, 245, 235, 262, 246, 242, 249, 277] payload_rows_per_dst=[5568, 5544, 5496, 5464, 5454, 5400, 5514, 5492] dup_rows_per_dst=[2852, 2764, 2722, 2688, 2614, 2558, 2776, 2630] combine_rows_per_src=[5568, 5544, 5496, 5464, 5454, 5400, 5514, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38118 dup_rows=21930 dup_rows_remote=19138 dedup_ratio=0.334625 router_tokens_per_expert=[268, 260, 266, 261, 267, 257, 262, 258, 275, 238, 254, 243, 241, 256, 232, 271, 226, 244, 252, 242, 268, 256, 284, 290, 260, 245, 245, 263, 275, 271, 270, 267, 232, 258, 250, 284, 249, 237, 272, 251, 240, 268, 272, 274, 263, 243, 246, 238, 264, 257, 228, 257, 281, 248, 266, 301, 249, 248, 259, 281, 251, 247, 256, 247, 245, 234, 245, 230, 259, 239, 265, 258, 268, 276, 241, 258, 265, 247, 260, 287, 245, 267, 249, 224, 274, 223, 242, 261, 260, 258, 275, 285, 255, 258, 254, 247, 263, 235, 256, 261, 280, 249, 263, 241, 265, 251, 255, 254, 275, 238, 277, 243, 263, 277, 258, 230, 241, 285, 247, 251, 243, 248, 278, 232, 239, 262, 224, 246] payload_rows_per_dst=[5414, 5578, 5442, 5488, 5412, 5426, 5456, 5390] dup_rows_per_dst=[2804, 2738, 2712, 2792, 2742, 2728, 2756, 2658] combine_rows_per_src=[5414, 5578, 5442, 5488, 5412, 5426, 5456, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43870 payload_rows_remote=38258 dup_rows=21666 dup_rows_remote=18870 dedup_ratio=0.330597 router_tokens_per_expert=[239, 249, 258, 249, 265, 245, 292, 242, 294, 289, 235, 246, 258, 274, 235, 268, 256, 259, 246, 245, 258, 294, 263, 248, 242, 260, 249, 235, 293, 252, 260, 272, 248, 262, 237, 264, 247, 208, 265, 262, 263, 256, 247, 272, 260, 274, 242, 257, 267, 253, 241, 228, 281, 258, 248, 254, 248, 264, 242, 234, 259, 263, 278, 264, 251, 249, 260, 270, 221, 227, 238, 265, 270, 247, 260, 251, 251, 260, 245, 272, 283, 250, 238, 261, 253, 262, 246, 243, 264, 277, 262, 232, 238, 250, 261, 242, 256, 258, 249, 277, 252, 258, 261, 254, 284, 236, 259, 243, 269, 269, 254, 250, 249, 276, 255, 251, 258, 263, 265, 245, 260, 258, 277, 268, 266, 251, 234, 248] payload_rows_per_dst=[5448, 5322, 5464, 5484, 5448, 5444, 5648, 5612] dup_rows_per_dst=[2668, 2340, 2932, 2680, 2698, 2680, 2872, 2796] combine_rows_per_src=[5448, 5322, 5464, 5484, 5448, 5444, 5648, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229736 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157278568 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314327400 weight_movement_bytes=18874368 total_nvlink_bytes=333201768 weight_to_token_ratio=0.060047 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=228888 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157285912 ingress_bytes=194805760 link_bytes=194805760 token_bytes=314342936 weight_movement_bytes=37748736 total_nvlink_bytes=352091672 weight_to_token_ratio=0.120088 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866532942e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=228984 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157589112 ingress_bytes=195108864 link_bytes=195108864 token_bytes=314949240 weight_movement_bytes=37748736 total_nvlink_bytes=352697976 weight_to_token_ratio=0.119857 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.869437135e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=156131328 dispatch_meta_bytes=229024 combine_payload_bytes=156131328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156360352 ingress_bytes=175005696 link_bytes=175005696 token_bytes=312491680 weight_movement_bytes=18874368 total_nvlink_bytes=331366048 weight_to_token_ratio=0.060400 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676818471e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=228512 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156933280 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313638048 weight_movement_bytes=18874368 total_nvlink_bytes=332512416 weight_to_token_ratio=0.060179 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38264 dup_rows=21826 dup_rows_remote=19062 dedup_ratio=0.333038 router_tokens_per_expert=[293, 250, 259, 269, 266, 284, 258, 247, 269, 274, 266, 254, 278, 242, 238, 249, 249, 239, 265, 271, 278, 260, 238, 251, 264, 272, 250, 246, 268, 262, 270, 271, 265, 259, 253, 234, 270, 274, 232, 268, 235, 253, 256, 300, 249, 246, 260, 251, 270, 247, 239, 262, 260, 257, 265, 259, 264, 270, 245, 272, 235, 256, 253, 241, 242, 253, 239, 262, 261, 257, 227, 301, 268, 260, 241, 256, 266, 253, 250, 265, 267, 264, 260, 260, 233, 256, 249, 242, 273, 231, 238, 234, 247, 275, 257, 263, 252, 265, 242, 243, 229, 263, 273, 271, 247, 248, 263, 234, 234, 249, 250, 238, 250, 268, 249, 253, 252, 244, 244, 265, 270, 251, 263, 270, 260, 222, 254, 252] payload_rows_per_dst=[5558, 5518, 5446, 5468, 5468, 5424, 5376, 5452] dup_rows_per_dst=[2834, 2790, 2764, 2722, 2734, 2674, 2626, 2682] combine_rows_per_src=[5558, 5518, 5446, 5468, 5468, 5424, 5376, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43580 payload_rows_remote=38068 dup_rows=21956 dup_rows_remote=19174 dedup_ratio=0.335022 router_tokens_per_expert=[265, 234, 264, 240, 255, 275, 264, 253, 256, 260, 256, 262, 298, 254, 239, 269, 262, 256, 274, 251, 253, 259, 264, 244, 256, 258, 264, 232, 250, 276, 255, 244, 226, 277, 286, 275, 253, 260, 259, 265, 249, 262, 277, 227, 265, 252, 238, 251, 271, 277, 262, 256, 233, 259, 250, 229, 247, 258, 263, 289, 263, 250, 234, 263, 269, 258, 257, 265, 255, 257, 255, 267, 247, 232, 242, 242, 268, 226, 226, 248, 259, 245, 252, 260, 249, 234, 276, 263, 255, 247, 226, 267, 287, 277, 248, 262, 265, 260, 248, 240, 272, 277, 304, 242, 289, 245, 238, 234, 240, 248, 272, 264, 242, 262, 254, 286, 256, 267, 235, 257, 247, 247, 221, 261, 244, 251, 256, 255] payload_rows_per_dst=[5512, 5458, 5378, 5546, 5476, 5442, 5364, 5404] dup_rows_per_dst=[2782, 2794, 2812, 2750, 2720, 2598, 2822, 2678] combine_rows_per_src=[5512, 5458, 5378, 5546, 5476, 5442, 5364, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38278 dup_rows=21856 dup_rows_remote=19068 dedup_ratio=0.333496 router_tokens_per_expert=[254, 253, 244, 276, 259, 250, 255, 224, 262, 271, 270, 235, 254, 236, 242, 266, 240, 273, 248, 260, 255, 221, 264, 296, 267, 268, 250, 235, 252, 238, 254, 274, 301, 231, 261, 232, 269, 260, 263, 264, 249, 240, 249, 269, 233, 274, 257, 250, 282, 246, 265, 227, 267, 266, 237, 243, 240, 242, 265, 250, 261, 265, 243, 260, 252, 248, 263, 271, 270, 235, 254, 260, 257, 263, 301, 254, 256, 274, 258, 267, 255, 262, 277, 236, 269, 251, 255, 266, 280, 252, 271, 255, 274, 229, 267, 231, 250, 254, 258, 265, 230, 275, 264, 237, 244, 263, 259, 266, 279, 269, 270, 221, 247, 269, 275, 244, 222, 256, 247, 293, 263, 256, 247, 246, 255, 258, 230, 236] payload_rows_per_dst=[5444, 5402, 5456, 5424, 5524, 5514, 5458, 5458] dup_rows_per_dst=[2658, 2788, 2748, 2694, 2842, 2746, 2750, 2630] combine_rows_per_src=[5444, 5402, 5456, 5424, 5524, 5514, 5458, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156729344 dispatch_meta_bytes=229304 combine_payload_bytes=156729344 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156958648 ingress_bytes=175603712 link_bytes=175603712 token_bytes=313687992 weight_movement_bytes=18874368 total_nvlink_bytes=332562360 weight_to_token_ratio=0.060169 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682548366e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=155926528 dispatch_meta_bytes=228968 combine_payload_bytes=155926528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156155496 ingress_bytes=174800896 link_bytes=174800896 token_bytes=312082024 weight_movement_bytes=18874368 total_nvlink_bytes=330956392 weight_to_token_ratio=0.060479 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674856178e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43926 payload_rows_remote=38412 dup_rows=21610 dup_rows_remote=18830 dedup_ratio=0.329742 router_tokens_per_expert=[247, 261, 298, 245, 246, 254, 239, 259, 249, 228, 267, 266, 231, 259, 261, 267, 260, 226, 278, 261, 262, 255, 270, 254, 245, 255, 243, 243, 256, 273, 262, 265, 232, 272, 269, 247, 267, 257, 258, 260, 255, 272, 258, 228, 223, 267, 249, 271, 246, 246, 255, 258, 266, 276, 246, 244, 266, 258, 259, 285, 258, 269, 244, 270, 280, 246, 262, 250, 244, 243, 266, 272, 247, 237, 248, 240, 269, 263, 278, 278, 286, 265, 237, 287, 242, 246, 265, 280, 254, 273, 239, 252, 266, 254, 246, 255, 245, 275, 267, 235, 272, 263, 267, 258, 247, 253, 237, 256, 246, 240, 265, 250, 240, 243, 252, 233, 280, 254, 260, 266, 225, 246, 248, 258, 233, 267, 252, 249] payload_rows_per_dst=[5466, 5476, 5492, 5520, 5578, 5514, 5452, 5428] dup_rows_per_dst=[2688, 2740, 2678, 2772, 2668, 2780, 2700, 2584] combine_rows_per_src=[5466, 5476, 5492, 5520, 5578, 5514, 5452, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38282 dup_rows=21766 dup_rows_remote=19064 dedup_ratio=0.332123 router_tokens_per_expert=[259, 259, 255, 268, 272, 225, 233, 282, 246, 249, 269, 246, 261, 233, 252, 217, 264, 250, 276, 251, 247, 271, 240, 272, 263, 291, 280, 245, 286, 267, 262, 257, 271, 253, 249, 254, 253, 249, 237, 256, 267, 260, 272, 263, 254, 256, 243, 266, 269, 261, 260, 252, 237, 252, 273, 229, 264, 245, 266, 250, 268, 255, 238, 262, 274, 249, 263, 231, 277, 260, 245, 255, 246, 267, 238, 252, 265, 261, 273, 239, 239, 257, 251, 238, 227, 252, 233, 295, 237, 263, 270, 260, 244, 278, 227, 262, 292, 246, 276, 260, 255, 235, 244, 251, 236, 252, 267, 250, 270, 266, 243, 244, 244, 251, 250, 253, 264, 251, 260, 266, 285, 249, 254, 251, 254, 276, 265, 248] payload_rows_per_dst=[5386, 5544, 5446, 5456, 5488, 5446, 5500, 5504] dup_rows_per_dst=[2666, 2900, 2760, 2706, 2702, 2620, 2674, 2738] combine_rows_per_src=[5386, 5544, 5446, 5456, 5488, 5446, 5500, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38382 dup_rows=21696 dup_rows_remote=18904 dedup_ratio=0.331055 router_tokens_per_expert=[252, 239, 274, 240, 260, 257, 258, 292, 261, 249, 253, 273, 252, 285, 247, 272, 264, 247, 229, 270, 273, 266, 266, 287, 257, 232, 265, 264, 252, 231, 251, 293, 241, 263, 298, 243, 278, 256, 237, 218, 242, 259, 258, 282, 258, 228, 250, 233, 272, 287, 230, 277, 263, 274, 269, 242, 225, 263, 245, 255, 258, 238, 261, 266, 267, 264, 249, 254, 243, 269, 250, 243, 260, 251, 267, 268, 263, 232, 252, 279, 267, 270, 271, 272, 248, 267, 221, 233, 270, 243, 254, 253, 252, 235, 272, 254, 258, 269, 231, 277, 243, 256, 239, 274, 224, 283, 226, 288, 249, 255, 262, 227, 259, 281, 260, 275, 252, 251, 243, 252, 266, 254, 239, 253, 225, 252, 225, 247] payload_rows_per_dst=[5590, 5490, 5448, 5458, 5552, 5448, 5448, 5406] dup_rows_per_dst=[2738, 2804, 2640, 2792, 2670, 2716, 2674, 2662] combine_rows_per_src=[5590, 5490, 5448, 5458, 5552, 5448, 5448, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43498 payload_rows_remote=38034 dup_rows=22038 dup_rows_remote=19254 dedup_ratio=0.336273 router_tokens_per_expert=[238, 261, 260, 228, 255, 238, 271, 260, 269, 234, 257, 278, 243, 253, 255, 247, 265, 248, 252, 259, 269, 258, 242, 234, 256, 245, 275, 260, 245, 283, 255, 250, 256, 266, 241, 251, 241, 243, 254, 265, 240, 242, 261, 256, 220, 244, 248, 259, 258, 240, 261, 256, 254, 231, 268, 264, 241, 247, 224, 268, 255, 264, 258, 303, 253, 242, 246, 266, 292, 268, 243, 270, 241, 234, 248, 217, 238, 250, 284, 251, 244, 230, 293, 254, 267, 282, 262, 252, 271, 254, 277, 259, 268, 277, 275, 241, 271, 240, 259, 251, 270, 255, 257, 219, 240, 248, 279, 279, 271, 251, 275, 259, 256, 254, 251, 255, 262, 245, 294, 270, 264, 263, 262, 261, 286, 240, 252, 258] payload_rows_per_dst=[5452, 5442, 5320, 5420, 5396, 5554, 5464, 5450] dup_rows_per_dst=[2642, 2750, 2654, 2764, 2690, 2858, 2784, 2896] combine_rows_per_src=[5452, 5442, 5320, 5420, 5396, 5554, 5464, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229384 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=75497472 egress_bytes=157016072 ingress_bytes=251158528 link_bytes=251158528 token_bytes=313802760 weight_movement_bytes=94371840 total_nvlink_bytes=408174600 weight_to_token_ratio=0.300736 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406477438e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38438 dup_rows=21674 dup_rows_remote=19010 dedup_ratio=0.330719 router_tokens_per_expert=[274, 264, 237, 259, 273, 262, 260, 268, 253, 285, 257, 248, 261, 279, 263, 242, 263, 244, 262, 242, 253, 257, 239, 258, 269, 269, 262, 267, 262, 249, 282, 242, 275, 235, 281, 281, 242, 241, 226, 252, 263, 269, 256, 238, 273, 256, 259, 259, 257, 252, 249, 249, 257, 247, 252, 236, 249, 246, 233, 253, 260, 247, 298, 252, 263, 264, 230, 265, 274, 244, 272, 260, 281, 269, 246, 267, 262, 256, 263, 263, 276, 263, 252, 271, 237, 220, 251, 251, 252, 246, 279, 260, 234, 250, 253, 259, 241, 264, 215, 239, 267, 304, 269, 243, 234, 264, 252, 236, 242, 236, 265, 273, 252, 243, 251, 261, 240, 234, 239, 256, 252, 252, 264, 274, 242, 271, 251, 261] payload_rows_per_dst=[5604, 5354, 5528, 5440, 5526, 5422, 5564, 5424] dup_rows_per_dst=[2766, 2658, 2684, 2634, 2832, 2724, 2712, 2664] combine_rows_per_src=[5604, 5354, 5528, 5440, 5526, 5422, 5564, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157335552 dispatch_meta_bytes=228968 combine_payload_bytes=157335552 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157564520 ingress_bytes=176209920 link_bytes=176209920 token_bytes=314900072 weight_movement_bytes=18874368 total_nvlink_bytes=333774440 weight_to_token_ratio=0.059938 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688356753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156803072 dispatch_meta_bytes=229384 combine_payload_bytes=156803072 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157032456 ingress_bytes=194551808 link_bytes=194551808 token_bytes=313835528 weight_movement_bytes=37748736 total_nvlink_bytes=351584264 weight_to_token_ratio=0.120282 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864099699e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38370 dup_rows=21710 dup_rows_remote=19064 dedup_ratio=0.331268 router_tokens_per_expert=[237, 284, 262, 240, 252, 272, 268, 255, 231, 256, 248, 256, 275, 251, 256, 252, 304, 266, 241, 249, 233, 249, 249, 257, 246, 269, 275, 253, 258, 262, 262, 249, 241, 248, 262, 254, 269, 253, 250, 230, 246, 267, 252, 265, 258, 264, 227, 265, 241, 224, 257, 248, 284, 266, 266, 245, 253, 254, 242, 245, 264, 269, 272, 266, 279, 235, 236, 265, 261, 259, 269, 264, 248, 275, 238, 239, 254, 284, 240, 271, 259, 271, 269, 239, 263, 228, 243, 226, 248, 249, 262, 239, 256, 277, 253, 227, 259, 219, 222, 255, 254, 263, 259, 265, 279, 269, 268, 256, 268, 259, 259, 244, 271, 249, 258, 260, 262, 249, 261, 284, 274, 237, 281, 245, 231, 266, 255, 297] payload_rows_per_dst=[5456, 5538, 5456, 5476, 5540, 5410, 5490, 5460] dup_rows_per_dst=[2734, 2706, 2646, 2716, 2694, 2608, 2706, 2900] combine_rows_per_src=[5456, 5538, 5456, 5476, 5540, 5410, 5490, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=229144 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157441816 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314654488 weight_movement_bytes=18874368 total_nvlink_bytes=333528856 weight_to_token_ratio=0.059984 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=155787264 dispatch_meta_bytes=229152 combine_payload_bytes=155787264 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156016416 ingress_bytes=174661632 link_bytes=174661632 token_bytes=311803680 weight_movement_bytes=18874368 total_nvlink_bytes=330678048 weight_to_token_ratio=0.060533 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.673521819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157442048 dispatch_meta_bytes=229792 combine_payload_bytes=157442048 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157671840 ingress_bytes=176316416 link_bytes=176316416 token_bytes=315113888 weight_movement_bytes=18874368 total_nvlink_bytes=333988256 weight_to_token_ratio=0.059897 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689377145e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38254 dup_rows=21768 dup_rows_remote=19002 dedup_ratio=0.332153 router_tokens_per_expert=[245, 262, 252, 238, 243, 279, 294, 267, 257, 270, 239, 251, 278, 234, 251, 274, 264, 273, 251, 208, 232, 263, 248, 251, 251, 258, 240, 246, 250, 267, 251, 252, 268, 252, 265, 277, 229, 265, 289, 214, 255, 229, 272, 273, 267, 263, 242, 257, 272, 262, 243, 281, 273, 240, 230, 253, 237, 265, 256, 284, 240, 257, 263, 255, 251, 269, 255, 266, 241, 253, 232, 247, 236, 256, 273, 262, 235, 247, 228, 273, 245, 284, 237, 250, 265, 265, 246, 239, 232, 256, 261, 257, 278, 249, 264, 271, 264, 295, 260, 238, 260, 254, 279, 249, 273, 244, 227, 226, 259, 257, 235, 246, 266, 275, 241, 265, 286, 250, 278, 270, 268, 274, 258, 250, 235, 252, 282, 262] payload_rows_per_dst=[5514, 5490, 5460, 5408, 5424, 5384, 5526, 5562] dup_rows_per_dst=[2766, 2784, 2762, 2550, 2624, 2814, 2606, 2862] combine_rows_per_src=[5514, 5490, 5460, 5408, 5424, 5384, 5526, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157163520 dispatch_meta_bytes=229736 combine_payload_bytes=157163520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157393256 ingress_bytes=176037888 link_bytes=176037888 token_bytes=314556776 weight_movement_bytes=18874368 total_nvlink_bytes=333431144 weight_to_token_ratio=0.060003 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686708427e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38112 dup_rows=21882 dup_rows_remote=18996 dedup_ratio=0.333893 router_tokens_per_expert=[262, 254, 221, 248, 238, 290, 239, 220, 246, 250, 248, 231, 268, 258, 260, 247, 259, 256, 257, 255, 255, 258, 252, 267, 263, 255, 284, 291, 291, 253, 269, 249, 248, 261, 230, 244, 260, 258, 255, 260, 249, 263, 246, 248, 233, 230, 248, 239, 281, 236, 262, 246, 233, 273, 240, 263, 286, 268, 259, 256, 256, 260, 247, 245, 236, 249, 231, 249, 280, 246, 242, 245, 273, 252, 249, 242, 232, 244, 268, 252, 266, 252, 251, 227, 279, 269, 263, 239, 253, 284, 260, 294, 250, 274, 247, 277, 249, 254, 247, 271, 261, 278, 228, 272, 276, 248, 257, 269, 282, 238, 246, 276, 264, 249, 267, 263, 274, 267, 265, 280, 235, 232, 266, 261, 273, 258, 253, 257] payload_rows_per_dst=[5318, 5542, 5382, 5454, 5380, 5564, 5492, 5522] dup_rows_per_dst=[2642, 2886, 2562, 2768, 2600, 2806, 2812, 2806] combine_rows_per_src=[5318, 5542, 5382, 5454, 5380, 5564, 5492, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229024 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156917408 ingress_bytes=232185856 link_bytes=232185856 token_bytes=313605792 weight_movement_bytes=75497472 total_nvlink_bytes=389103264 weight_to_token_ratio=0.240740 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.224690630e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43878 payload_rows_remote=38390 dup_rows=21658 dup_rows_remote=18906 dedup_ratio=0.330475 router_tokens_per_expert=[267, 263, 269, 249, 285, 260, 251, 264, 236, 247, 239, 260, 244, 233, 253, 225, 247, 264, 246, 246, 236, 250, 254, 232, 270, 249, 256, 231, 246, 253, 261, 249, 263, 241, 266, 233, 267, 280, 255, 270, 273, 251, 247, 242, 257, 257, 270, 255, 248, 246, 260, 268, 241, 249, 254, 275, 234, 261, 281, 264, 240, 277, 248, 262, 304, 258, 269, 266, 261, 286, 264, 256, 272, 241, 259, 268, 262, 297, 252, 234, 252, 251, 268, 244, 259, 231, 258, 256, 245, 286, 260, 258, 264, 268, 262, 258, 272, 250, 255, 254, 238, 278, 252, 245, 226, 221, 259, 272, 263, 259, 234, 263, 240, 253, 247, 256, 276, 261, 201, 280, 294, 264, 244, 260, 261, 253, 265, 233] payload_rows_per_dst=[5452, 5388, 5542, 5476, 5646, 5488, 5414, 5472] dup_rows_per_dst=[2638, 2592, 2712, 2740, 2852, 2752, 2668, 2704] combine_rows_per_src=[5452, 5388, 5542, 5476, 5646, 5488, 5414, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43876 payload_rows_remote=38466 dup_rows=21660 dup_rows_remote=18960 dedup_ratio=0.330505 router_tokens_per_expert=[278, 263, 297, 260, 267, 244, 241, 241, 264, 234, 231, 243, 255, 250, 263, 260, 282, 249, 259, 216, 261, 232, 253, 261, 235, 260, 237, 274, 271, 277, 248, 257, 233, 246, 260, 220, 239, 275, 275, 234, 237, 253, 256, 256, 282, 251, 275, 254, 283, 258, 251, 237, 274, 268, 271, 279, 288, 244, 272, 226, 250, 249, 251, 252, 270, 238, 288, 233, 267, 258, 267, 226, 251, 251, 238, 283, 244, 234, 246, 261, 264, 236, 273, 247, 258, 262, 264, 257, 254, 268, 262, 214, 254, 263, 244, 263, 252, 281, 267, 251, 270, 267, 248, 267, 240, 247, 260, 265, 249, 260, 269, 248, 262, 261, 257, 273, 270, 242, 255, 248, 259, 237, 250, 251, 260, 264, 252, 286] payload_rows_per_dst=[5440, 5486, 5430, 5560, 5410, 5484, 5536, 5530] dup_rows_per_dst=[2742, 2658, 2662, 2746, 2700, 2682, 2746, 2724] combine_rows_per_src=[5440, 5486, 5430, 5560, 5410, 5484, 5536, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38360 dup_rows=21672 dup_rows_remote=19028 dedup_ratio=0.330688 router_tokens_per_expert=[254, 258, 296, 261, 279, 222, 247, 270, 220, 267, 257, 269, 258, 267, 248, 262, 259, 251, 271, 261, 247, 267, 244, 255, 296, 249, 231, 279, 276, 261, 277, 261, 253, 266, 258, 256, 223, 303, 287, 261, 248, 251, 262, 254, 274, 272, 251, 254, 254, 259, 270, 282, 220, 265, 265, 252, 267, 253, 258, 246, 229, 249, 245, 260, 256, 262, 238, 243, 247, 253, 262, 276, 250, 261, 251, 263, 227, 261, 255, 251, 254, 250, 239, 254, 273, 223, 245, 284, 247, 257, 272, 242, 235, 275, 245, 267, 234, 258, 285, 247, 258, 237, 227, 237, 234, 252, 261, 256, 248, 254, 257, 259, 251, 259, 241, 277, 249, 259, 263, 261, 256, 206, 257, 248, 227, 297, 254, 274] payload_rows_per_dst=[5478, 5590, 5574, 5504, 5400, 5462, 5388, 5468] dup_rows_per_dst=[2792, 2780, 2772, 2644, 2712, 2662, 2620, 2690] combine_rows_per_src=[5478, 5590, 5574, 5504, 5400, 5462, 5388, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38180 dup_rows=21890 dup_rows_remote=19056 dedup_ratio=0.334015 router_tokens_per_expert=[256, 263, 255, 245, 244, 275, 246, 278, 259, 256, 233, 248, 242, 274, 246, 262, 267, 239, 240, 271, 250, 256, 261, 231, 261, 254, 272, 252, 251, 262, 299, 283, 261, 282, 266, 253, 231, 272, 223, 253, 271, 232, 243, 271, 265, 273, 239, 251, 250, 260, 244, 244, 275, 245, 273, 257, 261, 258, 258, 267, 257, 233, 260, 246, 266, 233, 267, 263, 269, 239, 273, 234, 271, 233, 262, 287, 261, 251, 249, 239, 268, 259, 242, 240, 244, 263, 253, 248, 249, 301, 294, 259, 276, 248, 245, 254, 260, 267, 264, 256, 251, 257, 267, 274, 276, 242, 258, 248, 280, 260, 255, 235, 260, 227, 237, 244, 240, 252, 268, 263, 252, 256, 244, 261, 234, 262, 233, 240] payload_rows_per_dst=[5426, 5482, 5488, 5420, 5452, 5538, 5466, 5374] dup_rows_per_dst=[2738, 2816, 2684, 2756, 2742, 2748, 2834, 2572] combine_rows_per_src=[5426, 5482, 5488, 5420, 5452, 5538, 5466, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38366 dup_rows=21744 dup_rows_remote=19046 dedup_ratio=0.331787 router_tokens_per_expert=[272, 280, 239, 283, 252, 253, 255, 262, 266, 268, 242, 227, 238, 282, 263, 297, 243, 257, 273, 252, 262, 269, 258, 252, 254, 254, 250, 261, 263, 255, 243, 283, 240, 262, 279, 255, 256, 267, 250, 272, 262, 225, 251, 257, 225, 262, 238, 271, 233, 234, 289, 275, 233, 236, 256, 262, 262, 276, 257, 258, 260, 240, 287, 279, 259, 249, 241, 265, 249, 262, 252, 266, 269, 258, 241, 270, 236, 258, 272, 244, 227, 244, 256, 246, 246, 241, 260, 249, 251, 281, 272, 282, 257, 241, 248, 266, 261, 261, 248, 242, 253, 247, 245, 271, 245, 241, 258, 239, 239, 284, 245, 252, 238, 267, 251, 223, 230, 227, 271, 273, 241, 283, 267, 281, 257, 255, 246, 252] payload_rows_per_dst=[5448, 5526, 5410, 5516, 5560, 5486, 5420, 5426] dup_rows_per_dst=[2674, 2732, 2734, 2758, 2858, 2648, 2642, 2698] combine_rows_per_src=[5448, 5526, 5410, 5516, 5560, 5486, 5420, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=228432 combine_payload_bytes=156106752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=156335184 ingress_bytes=231604224 link_bytes=231604224 token_bytes=312441936 weight_movement_bytes=75497472 total_nvlink_bytes=387939408 weight_to_token_ratio=0.241637 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219117718e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=229184 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157474624 ingress_bytes=194994176 link_bytes=194994176 token_bytes=314720064 weight_movement_bytes=37748736 total_nvlink_bytes=352468800 weight_to_token_ratio=0.119944 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868338251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157556736 dispatch_meta_bytes=229704 combine_payload_bytes=157556736 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157786440 ingress_bytes=176431104 link_bytes=176431104 token_bytes=315343176 weight_movement_bytes=18874368 total_nvlink_bytes=334217544 weight_to_token_ratio=0.059853 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.690476029e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38274 dup_rows=21796 dup_rows_remote=19100 dedup_ratio=0.332581 router_tokens_per_expert=[264, 254, 257, 268, 296, 232, 231, 252, 232, 266, 253, 249, 246, 259, 251, 261, 259, 264, 249, 232, 252, 266, 277, 256, 262, 269, 288, 271, 242, 281, 220, 276, 250, 235, 230, 259, 278, 245, 258, 266, 275, 254, 251, 243, 230, 278, 269, 260, 283, 234, 258, 253, 247, 249, 257, 262, 257, 270, 264, 256, 250, 269, 263, 280, 255, 257, 256, 280, 282, 255, 278, 274, 265, 263, 262, 265, 245, 222, 224, 233, 250, 237, 236, 238, 254, 251, 278, 259, 264, 244, 273, 239, 247, 263, 283, 261, 227, 270, 226, 256, 260, 240, 265, 241, 258, 258, 242, 257, 236, 259, 231, 272, 267, 262, 254, 260, 247, 266, 251, 246, 272, 241, 273, 254, 249, 242, 260, 265] payload_rows_per_dst=[5432, 5554, 5466, 5526, 5484, 5404, 5388, 5486] dup_rows_per_dst=[2710, 2774, 2696, 2778, 2748, 2750, 2608, 2732] combine_rows_per_src=[5432, 5554, 5466, 5526, 5484, 5404, 5388, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157122560 dispatch_meta_bytes=229552 combine_payload_bytes=157122560 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157352112 ingress_bytes=175996928 link_bytes=175996928 token_bytes=314474672 weight_movement_bytes=18874368 total_nvlink_bytes=333349040 weight_to_token_ratio=0.060019 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686315968e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=228944 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614224 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999504 weight_movement_bytes=18874368 total_nvlink_bytes=331873872 weight_to_token_ratio=0.060302 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=229648 combine_payload_bytes=157147136 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157376784 ingress_bytes=232644608 link_bytes=232644608 token_bytes=314523920 weight_movement_bytes=75497472 total_nvlink_bytes=390021392 weight_to_token_ratio=0.240037 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.229086166e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43840 payload_rows_remote=38346 dup_rows=21696 dup_rows_remote=18954 dedup_ratio=0.331055 router_tokens_per_expert=[256, 285, 264, 257, 246, 276, 235, 266, 263, 253, 256, 251, 265, 237, 261, 247, 293, 271, 270, 257, 235, 261, 256, 252, 253, 252, 258, 238, 239, 245, 253, 248, 280, 241, 250, 258, 259, 253, 279, 245, 231, 277, 240, 257, 261, 246, 257, 252, 286, 250, 255, 260, 257, 254, 250, 263, 259, 248, 249, 230, 277, 239, 265, 241, 247, 268, 262, 269, 250, 237, 243, 242, 245, 272, 261, 254, 238, 238, 240, 251, 265, 256, 279, 251, 266, 232, 283, 237, 235, 261, 268, 251, 298, 265, 262, 259, 257, 244, 266, 272, 252, 246, 235, 229, 260, 263, 250, 263, 241, 279, 273, 259, 258, 243, 265, 269, 241, 262, 226, 248, 240, 265, 262, 290, 264, 271, 250, 272] payload_rows_per_dst=[5494, 5414, 5466, 5506, 5402, 5740, 5346, 5472] dup_rows_per_dst=[2742, 2782, 2672, 2706, 2586, 2904, 2524, 2780] combine_rows_per_src=[5494, 5414, 5466, 5506, 5402, 5740, 5346, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=229496 combine_payload_bytes=156770304 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156999800 ingress_bytes=175644672 link_bytes=175644672 token_bytes=313770104 weight_movement_bytes=18874368 total_nvlink_bytes=332644472 weight_to_token_ratio=0.060153 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682940825e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=157065216 dispatch_meta_bytes=229200 combine_payload_bytes=157065216 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157294416 ingress_bytes=213688320 link_bytes=213688320 token_bytes=314359632 weight_movement_bytes=56623104 total_nvlink_bytes=370982736 weight_to_token_ratio=0.180122 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.047456341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38258 dup_rows=21834 dup_rows_remote=19116 dedup_ratio=0.333160 router_tokens_per_expert=[281, 283, 246, 249, 262, 270, 243, 273, 255, 249, 251, 278, 244, 284, 244, 260, 259, 251, 262, 245, 273, 243, 259, 262, 263, 253, 241, 268, 243, 251, 245, 263, 274, 289, 258, 232, 229, 258, 235, 251, 233, 250, 253, 259, 244, 279, 288, 263, 276, 265, 223, 257, 257, 281, 236, 262, 245, 258, 244, 274, 253, 260, 238, 235, 247, 261, 268, 277, 284, 247, 277, 231, 237, 264, 293, 260, 269, 253, 253, 265, 254, 248, 247, 256, 258, 262, 249, 239, 245, 256, 273, 237, 251, 265, 243, 276, 223, 264, 267, 249, 242, 243, 238, 235, 238, 247, 250, 238, 273, 245, 277, 278, 272, 267, 255, 256, 248, 241, 242, 294, 260, 245, 247, 266, 255, 242, 256, 258] payload_rows_per_dst=[5572, 5444, 5464, 5434, 5542, 5422, 5402, 5422] dup_rows_per_dst=[2772, 2718, 2726, 2694, 2830, 2696, 2612, 2786] combine_rows_per_src=[5572, 5444, 5464, 5434, 5542, 5422, 5402, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43476 payload_rows_remote=38000 dup_rows=22060 dup_rows_remote=19166 dedup_ratio=0.336609 router_tokens_per_expert=[254, 267, 244, 271, 244, 257, 266, 246, 253, 256, 255, 262, 255, 244, 261, 249, 236, 260, 286, 248, 275, 280, 278, 236, 244, 234, 255, 233, 241, 206, 247, 266, 251, 261, 244, 251, 264, 238, 270, 236, 237, 225, 287, 227, 235, 239, 283, 262, 240, 258, 251, 273, 254, 238, 251, 304, 256, 254, 258, 242, 241, 255, 249, 274, 234, 276, 243, 257, 231, 258, 257, 241, 266, 256, 260, 254, 245, 248, 226, 254, 277, 270, 245, 272, 276, 266, 278, 232, 255, 259, 259, 231, 280, 258, 256, 271, 259, 245, 264, 283, 243, 249, 236, 247, 264, 258, 249, 297, 232, 265, 279, 275, 284, 277, 259, 280, 250, 219, 253, 276, 294, 235, 256, 252, 268, 260, 271, 281] payload_rows_per_dst=[5448, 5388, 5372, 5486, 5332, 5476, 5482, 5492] dup_rows_per_dst=[2720, 2662, 2648, 2710, 2680, 2894, 2808, 2938] combine_rows_per_src=[5448, 5388, 5372, 5486, 5332, 5476, 5482, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43586 payload_rows_remote=38094 dup_rows=21950 dup_rows_remote=19356 dedup_ratio=0.334930 router_tokens_per_expert=[260, 250, 215, 269, 274, 257, 262, 259, 232, 241, 252, 243, 243, 273, 247, 268, 246, 233, 241, 270, 251, 255, 255, 243, 255, 248, 229, 265, 258, 249, 233, 265, 265, 277, 255, 241, 251, 236, 248, 257, 252, 261, 269, 242, 288, 274, 251, 227, 262, 243, 262, 246, 282, 254, 303, 257, 249, 255, 279, 268, 258, 234, 241, 239, 239, 266, 258, 267, 264, 258, 268, 235, 259, 243, 248, 242, 258, 234, 242, 262, 281, 278, 264, 250, 250, 261, 261, 248, 239, 274, 260, 246, 268, 254, 265, 245, 234, 249, 250, 232, 263, 292, 257, 228, 287, 263, 281, 285, 263, 260, 277, 265, 296, 233, 291, 272, 251, 234, 251, 255, 253, 268, 243, 243, 272, 261, 260, 245] payload_rows_per_dst=[5372, 5332, 5412, 5502, 5492, 5448, 5500, 5528] dup_rows_per_dst=[2718, 2660, 2776, 2762, 2594, 2840, 2872, 2728] combine_rows_per_src=[5372, 5332, 5412, 5502, 5492, 5448, 5500, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43950 payload_rows_remote=38390 dup_rows=21586 dup_rows_remote=18802 dedup_ratio=0.329376 router_tokens_per_expert=[247, 273, 262, 238, 256, 273, 267, 250, 257, 264, 267, 240, 263, 248, 243, 248, 242, 258, 229, 250, 242, 240, 256, 279, 273, 236, 256, 252, 275, 274, 262, 244, 276, 270, 264, 252, 258, 258, 287, 260, 253, 270, 258, 252, 263, 241, 244, 232, 263, 259, 267, 247, 253, 256, 239, 270, 260, 259, 287, 253, 246, 267, 267, 279, 251, 248, 219, 283, 272, 265, 245, 246, 234, 244, 253, 286, 244, 242, 245, 238, 230, 261, 281, 246, 260, 272, 268, 258, 264, 264, 245, 251, 281, 256, 287, 247, 234, 261, 228, 234, 264, 256, 226, 255, 263, 257, 259, 251, 271, 237, 244, 274, 272, 240, 241, 252, 271, 253, 239, 279, 257, 237, 259, 262, 280, 258, 243, 251] payload_rows_per_dst=[5492, 5530, 5542, 5560, 5372, 5558, 5378, 5518] dup_rows_per_dst=[2700, 2606, 2734, 2784, 2658, 2784, 2650, 2670] combine_rows_per_src=[5492, 5530, 5542, 5560, 5372, 5558, 5378, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43732 payload_rows_remote=38318 dup_rows=21804 dup_rows_remote=19242 dedup_ratio=0.332703 router_tokens_per_expert=[256, 262, 225, 257, 255, 238, 254, 276, 228, 254, 288, 276, 281, 255, 269, 240, 257, 278, 261, 274, 239, 251, 275, 264, 242, 243, 272, 262, 248, 256, 238, 243, 230, 240, 274, 267, 269, 257, 268, 254, 227, 263, 263, 250, 254, 271, 256, 271, 237, 230, 254, 269, 278, 263, 264, 236, 247, 283, 263, 246, 280, 281, 251, 257, 258, 276, 245, 263, 286, 254, 251, 241, 269, 243, 238, 231, 274, 234, 285, 269, 233, 260, 233, 265, 271, 257, 263, 262, 246, 255, 270, 265, 246, 243, 264, 273, 239, 252, 247, 270, 228, 248, 238, 264, 210, 259, 269, 251, 241, 262, 255, 255, 249, 289, 281, 249, 265, 258, 232, 243, 260, 246, 248, 237, 257, 252, 258, 263] payload_rows_per_dst=[5478, 5492, 5476, 5500, 5436, 5492, 5414, 5444] dup_rows_per_dst=[2750, 2714, 2752, 2778, 2798, 2720, 2562, 2730] combine_rows_per_src=[5478, 5492, 5476, 5500, 5436, 5492, 5414, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=37942 dup_rows=21954 dup_rows_remote=19190 dedup_ratio=0.334991 router_tokens_per_expert=[253, 292, 261, 239, 233, 270, 258, 264, 266, 238, 253, 270, 261, 249, 285, 254, 250, 265, 239, 263, 265, 232, 251, 252, 270, 267, 260, 235, 272, 251, 275, 259, 258, 260, 253, 250, 241, 257, 234, 228, 255, 241, 257, 240, 252, 244, 263, 243, 259, 258, 256, 250, 245, 226, 289, 257, 257, 247, 267, 273, 265, 277, 259, 251, 229, 269, 261, 248, 279, 260, 277, 273, 261, 259, 242, 244, 240, 269, 248, 242, 246, 279, 236, 245, 270, 263, 263, 266, 273, 250, 288, 268, 258, 258, 240, 242, 227, 277, 265, 263, 236, 224, 284, 226, 253, 252, 250, 230, 226, 240, 263, 260, 261, 240, 256, 270, 259, 256, 230, 275, 263, 275, 253, 264, 288, 264, 246, 282] payload_rows_per_dst=[5318, 5404, 5496, 5326, 5606, 5492, 5300, 5640] dup_rows_per_dst=[2466, 2808, 2964, 2712, 2830, 2794, 2616, 2764] combine_rows_per_src=[5318, 5404, 5496, 5326, 5606, 5492, 5300, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156704768 dispatch_meta_bytes=229496 combine_payload_bytes=156704768 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156934264 ingress_bytes=175579136 link_bytes=175579136 token_bytes=313639032 weight_movement_bytes=18874368 total_nvlink_bytes=332513400 weight_to_token_ratio=0.060179 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682312891e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43700 payload_rows_remote=38200 dup_rows=21836 dup_rows_remote=19116 dedup_ratio=0.333191 router_tokens_per_expert=[236, 233, 249, 240, 276, 271, 258, 258, 282, 271, 264, 292, 248, 255, 256, 268, 264, 267, 262, 282, 253, 244, 250, 290, 257, 250, 222, 252, 263, 268, 261, 287, 283, 256, 252, 246, 241, 250, 256, 266, 259, 253, 266, 267, 256, 234, 256, 269, 234, 241, 255, 225, 240, 258, 249, 284, 278, 260, 294, 247, 249, 238, 255, 277, 287, 257, 253, 242, 250, 264, 281, 249, 260, 249, 231, 273, 261, 255, 237, 251, 260, 231, 254, 275, 252, 249, 259, 255, 249, 258, 281, 271, 232, 260, 276, 238, 251, 247, 250, 281, 243, 274, 240, 266, 255, 237, 244, 265, 264, 263, 267, 236, 240, 235, 226, 237, 229, 224, 259, 265, 245, 270, 260, 262, 241, 255, 258, 256] payload_rows_per_dst=[5552, 5498, 5500, 5462, 5492, 5424, 5394, 5378] dup_rows_per_dst=[2762, 2846, 2720, 2706, 2708, 2776, 2772, 2546] combine_rows_per_src=[5552, 5498, 5500, 5462, 5492, 5424, 5394, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=229800 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156262824 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312295848 weight_movement_bytes=18874368 total_nvlink_bytes=331170216 weight_to_token_ratio=0.060437 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=155648000 dispatch_meta_bytes=228664 combine_payload_bytes=155648000 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155876664 ingress_bytes=193396736 link_bytes=193396736 token_bytes=311524664 weight_movement_bytes=37748736 total_nvlink_bytes=349273400 weight_to_token_ratio=0.121174 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853032367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=157245440 dispatch_meta_bytes=228768 combine_payload_bytes=157245440 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157474208 ingress_bytes=194994176 link_bytes=194994176 token_bytes=314719648 weight_movement_bytes=37748736 total_nvlink_bytes=352468384 weight_to_token_ratio=0.119944 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868338251e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=230240 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157180768 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314131296 weight_movement_bytes=18874368 total_nvlink_bytes=333005664 weight_to_token_ratio=0.060084 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=155410432 dispatch_meta_bytes=228528 combine_payload_bytes=155410432 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155638960 ingress_bytes=193159168 link_bytes=193159168 token_bytes=311049392 weight_movement_bytes=37748736 total_nvlink_bytes=348798128 weight_to_token_ratio=0.121359 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.850756108e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38196 dup_rows=21868 dup_rows_remote=19028 dedup_ratio=0.333679 router_tokens_per_expert=[248, 244, 245, 262, 273, 245, 274, 232, 285, 247, 258, 241, 301, 237, 280, 264, 261, 226, 271, 274, 279, 243, 241, 280, 273, 256, 280, 255, 259, 263, 267, 274, 244, 265, 263, 278, 250, 286, 253, 252, 264, 255, 230, 255, 267, 283, 252, 275, 235, 235, 237, 269, 253, 249, 234, 249, 266, 269, 232, 262, 248, 251, 222, 270, 272, 205, 236, 274, 264, 265, 264, 285, 259, 252, 248, 245, 268, 241, 258, 250, 257, 258, 248, 272, 273, 279, 253, 229, 255, 270, 263, 253, 240, 247, 242, 244, 264, 274, 233, 229, 294, 259, 250, 244, 257, 248, 251, 241, 229, 244, 260, 272, 263, 256, 247, 265, 278, 239, 249, 240, 263, 250, 231, 278, 249, 253, 223, 275] payload_rows_per_dst=[5472, 5574, 5462, 5450, 5382, 5462, 5442, 5424] dup_rows_per_dst=[2840, 2802, 2882, 2752, 2550, 2692, 2656, 2694] combine_rows_per_src=[5472, 5574, 5462, 5450, 5382, 5462, 5442, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229264 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696464 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163664 weight_movement_bytes=37748736 total_nvlink_bytes=350912400 weight_to_token_ratio=0.120540 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=228896 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156679712 ingress_bytes=194199552 link_bytes=194199552 token_bytes=313130528 weight_movement_bytes=37748736 total_nvlink_bytes=350879264 weight_to_token_ratio=0.120553 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860724555e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38382 dup_rows=21740 dup_rows_remote=19124 dedup_ratio=0.331726 router_tokens_per_expert=[264, 276, 269, 248, 263, 251, 236, 231, 251, 271, 260, 261, 265, 244, 277, 282, 232, 284, 248, 267, 263, 241, 255, 260, 259, 258, 227, 238, 250, 257, 236, 240, 246, 245, 251, 264, 251, 266, 219, 261, 266, 256, 239, 261, 237, 253, 240, 270, 254, 261, 267, 256, 252, 274, 272, 261, 267, 274, 257, 261, 271, 253, 260, 244, 282, 263, 260, 217, 236, 257, 284, 280, 246, 263, 256, 257, 240, 272, 241, 260, 248, 247, 255, 259, 262, 269, 261, 266, 262, 225, 235, 248, 252, 254, 242, 258, 243, 269, 263, 252, 272, 248, 229, 256, 258, 288, 244, 274, 252, 254, 233, 253, 280, 253, 264, 247, 236, 249, 239, 285, 240, 255, 287, 248, 286, 252, 278, 251] payload_rows_per_dst=[5474, 5414, 5434, 5518, 5510, 5402, 5514, 5530] dup_rows_per_dst=[2824, 2616, 2616, 2850, 2718, 2684, 2662, 2770] combine_rows_per_src=[5474, 5414, 5434, 5518, 5510, 5402, 5514, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38448 dup_rows=21822 dup_rows_remote=19232 dedup_ratio=0.332977 router_tokens_per_expert=[250, 280, 256, 260, 254, 265, 274, 254, 239, 275, 242, 253, 247, 250, 272, 266, 294, 263, 260, 277, 244, 258, 261, 253, 265, 242, 244, 252, 257, 274, 233, 261, 241, 294, 255, 240, 224, 262, 247, 287, 238, 231, 271, 249, 262, 276, 251, 265, 272, 285, 259, 262, 235, 246, 277, 261, 257, 242, 251, 249, 242, 267, 280, 277, 241, 256, 222, 229, 210, 276, 252, 240, 257, 241, 244, 246, 250, 245, 265, 254, 281, 238, 272, 270, 238, 253, 263, 271, 227, 249, 282, 239, 275, 251, 233, 259, 281, 271, 252, 274, 253, 262, 237, 265, 285, 253, 248, 262, 281, 232, 246, 250, 286, 271, 247, 241, 244, 254, 244, 228, 273, 242, 266, 251, 258, 240, 235, 277] payload_rows_per_dst=[5538, 5490, 5394, 5546, 5266, 5454, 5568, 5458] dup_rows_per_dst=[2736, 2786, 2792, 2778, 2590, 2748, 2736, 2656] combine_rows_per_src=[5538, 5490, 5394, 5546, 5266, 5454, 5568, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43526 payload_rows_remote=38158 dup_rows=22010 dup_rows_remote=19276 dedup_ratio=0.335846 router_tokens_per_expert=[256, 275, 288, 268, 265, 234, 264, 243, 243, 228, 280, 259, 252, 262, 251, 267, 229, 245, 250, 243, 255, 253, 266, 239, 286, 261, 268, 262, 263, 249, 295, 272, 267, 259, 242, 257, 259, 235, 222, 206, 243, 255, 254, 269, 264, 261, 227, 267, 255, 227, 231, 262, 220, 249, 245, 254, 260, 249, 253, 247, 274, 265, 265, 235, 248, 262, 264, 258, 269, 230, 257, 264, 264, 261, 285, 240, 263, 257, 269, 246, 237, 275, 266, 279, 265, 250, 233, 258, 218, 260, 254, 254, 261, 228, 248, 265, 240, 231, 280, 251, 258, 283, 263, 264, 247, 260, 270, 264, 277, 286, 247, 261, 248, 251, 237, 267, 284, 272, 254, 262, 266, 237, 263, 254, 285, 263, 232, 274] payload_rows_per_dst=[5524, 5474, 5360, 5334, 5486, 5368, 5502, 5478] dup_rows_per_dst=[2746, 2798, 2614, 2648, 2788, 2734, 2862, 2820] combine_rows_per_src=[5524, 5474, 5360, 5334, 5486, 5368, 5502, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38216 dup_rows=21784 dup_rows_remote=19128 dedup_ratio=0.332397 router_tokens_per_expert=[242, 243, 259, 276, 224, 295, 267, 252, 277, 245, 256, 239, 247, 242, 252, 257, 267, 255, 237, 261, 268, 269, 253, 258, 261, 276, 264, 240, 235, 235, 280, 251, 256, 236, 270, 258, 267, 266, 266, 246, 264, 258, 246, 259, 270, 251, 283, 253, 264, 266, 244, 271, 240, 277, 261, 256, 261, 253, 228, 287, 257, 257, 256, 259, 254, 241, 244, 259, 244, 273, 257, 270, 261, 246, 242, 277, 213, 268, 245, 266, 264, 263, 242, 245, 250, 257, 225, 241, 262, 230, 244, 288, 257, 254, 244, 269, 270, 269, 261, 259, 218, 275, 267, 253, 262, 253, 259, 226, 259, 274, 253, 238, 240, 285, 253, 279, 264, 254, 241, 264, 266, 258, 245, 273, 266, 205, 270, 245] payload_rows_per_dst=[5476, 5516, 5512, 5440, 5422, 5434, 5536, 5416] dup_rows_per_dst=[2652, 2704, 2804, 2834, 2698, 2636, 2656, 2800] combine_rows_per_src=[5476, 5516, 5512, 5440, 5422, 5434, 5536, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44026 payload_rows_remote=38630 dup_rows=21510 dup_rows_remote=18792 dedup_ratio=0.328217 router_tokens_per_expert=[267, 239, 275, 276, 256, 258, 264, 238, 249, 271, 260, 244, 261, 237, 248, 261, 251, 259, 276, 262, 255, 211, 250, 243, 258, 248, 254, 285, 248, 274, 257, 228, 263, 233, 266, 274, 287, 245, 246, 250, 250, 266, 268, 253, 248, 255, 242, 241, 248, 247, 258, 246, 278, 256, 266, 257, 237, 285, 245, 247, 260, 238, 250, 239, 258, 250, 280, 243, 249, 272, 275, 256, 256, 218, 239, 259, 246, 249, 257, 245, 253, 251, 244, 270, 260, 243, 281, 242, 289, 256, 279, 250, 269, 244, 278, 272, 288, 248, 265, 264, 252, 271, 259, 268, 271, 265, 251, 255, 246, 264, 260, 281, 262, 236, 268, 250, 237, 263, 241, 260, 234, 230, 240, 274, 243, 262, 250, 270] payload_rows_per_dst=[5566, 5462, 5494, 5396, 5472, 5536, 5668, 5432] dup_rows_per_dst=[2642, 2656, 2680, 2718, 2632, 2826, 2748, 2608] combine_rows_per_src=[5566, 5462, 5494, 5396, 5472, 5536, 5668, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157212672 dispatch_meta_bytes=230024 combine_payload_bytes=157212672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157442696 ingress_bytes=176087040 link_bytes=176087040 token_bytes=314655368 weight_movement_bytes=18874368 total_nvlink_bytes=333529736 weight_to_token_ratio=0.059984 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687179377e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38182 dup_rows=21896 dup_rows_remote=19242 dedup_ratio=0.334106 router_tokens_per_expert=[248, 248, 281, 231, 251, 250, 266, 264, 269, 258, 268, 246, 267, 255, 262, 273, 265, 250, 263, 256, 250, 263, 262, 260, 229, 221, 263, 247, 265, 279, 253, 261, 259, 224, 264, 280, 251, 257, 243, 233, 268, 261, 273, 246, 264, 252, 260, 231, 275, 274, 272, 286, 264, 266, 258, 259, 275, 231, 274, 268, 244, 256, 240, 256, 244, 286, 254, 251, 246, 252, 271, 254, 239, 252, 262, 259, 244, 262, 273, 259, 246, 280, 239, 249, 267, 259, 252, 278, 253, 252, 265, 247, 242, 255, 260, 281, 251, 268, 277, 262, 243, 243, 245, 248, 247, 252, 225, 234, 261, 228, 242, 267, 281, 227, 250, 265, 229, 246, 275, 224, 260, 261, 277, 225, 243, 270, 254, 267] payload_rows_per_dst=[5316, 5354, 5452, 5580, 5344, 5560, 5576, 5458] dup_rows_per_dst=[2702, 2558, 2628, 2868, 2606, 2946, 2934, 2654] combine_rows_per_src=[5316, 5354, 5452, 5580, 5344, 5560, 5576, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=157483008 dispatch_meta_bytes=230720 combine_payload_bytes=157483008 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157713728 ingress_bytes=176357376 link_bytes=176357376 token_bytes=315196736 weight_movement_bytes=18874368 total_nvlink_bytes=334071104 weight_to_token_ratio=0.059881 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689769604e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38420 dup_rows=21738 dup_rows_remote=19092 dedup_ratio=0.331696 router_tokens_per_expert=[251, 239, 255, 242, 263, 271, 259, 270, 240, 254, 273, 256, 267, 238, 263, 261, 232, 264, 272, 274, 274, 227, 261, 248, 263, 261, 263, 275, 244, 272, 276, 249, 249, 241, 288, 241, 233, 265, 248, 256, 252, 209, 241, 264, 240, 263, 266, 256, 231, 247, 266, 269, 288, 255, 265, 268, 256, 266, 264, 265, 256, 271, 243, 231, 262, 243, 261, 218, 244, 237, 263, 273, 258, 237, 270, 248, 263, 279, 278, 261, 258, 250, 271, 267, 229, 222, 233, 247, 258, 238, 253, 244, 238, 263, 244, 238, 257, 259, 234, 285, 274, 271, 268, 260, 223, 239, 275, 251, 273, 252, 247, 285, 288, 260, 260, 253, 268, 253, 262, 246, 241, 278, 273, 270, 266, 260, 230, 249] payload_rows_per_dst=[5504, 5462, 5378, 5546, 5520, 5388, 5492, 5508] dup_rows_per_dst=[2700, 2848, 2646, 2736, 2670, 2518, 2814, 2806] combine_rows_per_src=[5504, 5462, 5378, 5546, 5520, 5388, 5492, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229736 combine_payload_bytes=156295168 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156524904 ingress_bytes=212918272 link_bytes=212918272 token_bytes=312820072 weight_movement_bytes=56623104 total_nvlink_bytes=369443176 weight_to_token_ratio=0.181009 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040078120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156532736 dispatch_meta_bytes=229376 combine_payload_bytes=156532736 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156762112 ingress_bytes=194281472 link_bytes=194281472 token_bytes=313294848 weight_movement_bytes=37748736 total_nvlink_bytes=351043584 weight_to_token_ratio=0.120489 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861509472e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=158228480 dispatch_meta_bytes=229688 combine_payload_bytes=158228480 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=158458168 ingress_bytes=195977216 link_bytes=195977216 token_bytes=316686648 weight_movement_bytes=37748736 total_nvlink_bytes=354435384 weight_to_token_ratio=0.119199 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.877757257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229696 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156623168 ingress_bytes=194142208 link_bytes=194142208 token_bytes=313016640 weight_movement_bytes=37748736 total_nvlink_bytes=350765376 weight_to_token_ratio=0.120597 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860175113e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43952 payload_rows_remote=38484 dup_rows=21584 dup_rows_remote=18944 dedup_ratio=0.329346 router_tokens_per_expert=[241, 248, 244, 252, 233, 234, 254, 253, 231, 264, 269, 245, 255, 237, 262, 237, 260, 236, 255, 252, 244, 240, 270, 255, 247, 232, 273, 251, 262, 297, 237, 282, 258, 249, 242, 244, 240, 263, 269, 255, 251, 267, 249, 250, 263, 272, 242, 285, 228, 226, 258, 254, 245, 252, 258, 243, 267, 256, 263, 247, 250, 275, 225, 270, 224, 271, 256, 270, 251, 227, 256, 272, 272, 248, 263, 259, 269, 273, 268, 247, 254, 255, 244, 258, 264, 273, 263, 238, 243, 250, 284, 277, 254, 271, 247, 232, 246, 264, 254, 246, 244, 267, 255, 242, 290, 272, 275, 249, 266, 246, 261, 264, 255, 273, 244, 267, 297, 279, 270, 245, 258, 265, 274, 272, 257, 269, 247, 254] payload_rows_per_dst=[5468, 5396, 5554, 5474, 5452, 5482, 5572, 5554] dup_rows_per_dst=[2640, 2600, 2752, 2730, 2630, 2624, 2710, 2898] combine_rows_per_src=[5468, 5396, 5554, 5474, 5452, 5482, 5572, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157368320 dispatch_meta_bytes=230048 combine_payload_bytes=157368320 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157598368 ingress_bytes=176242688 link_bytes=176242688 token_bytes=314966688 weight_movement_bytes=18874368 total_nvlink_bytes=333841056 weight_to_token_ratio=0.059925 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688670720e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43612 payload_rows_remote=38158 dup_rows=21924 dup_rows_remote=19114 dedup_ratio=0.334534 router_tokens_per_expert=[255, 272, 260, 254, 260, 260, 247, 251, 275, 261, 261, 253, 250, 258, 262, 254, 252, 262, 274, 278, 260, 259, 251, 272, 235, 261, 246, 230, 260, 246, 273, 273, 222, 256, 265, 286, 251, 245, 266, 266, 275, 234, 255, 266, 266, 248, 242, 239, 282, 259, 262, 255, 272, 244, 263, 241, 236, 251, 253, 268, 254, 277, 243, 244, 266, 226, 233, 240, 240, 244, 274, 253, 265, 239, 249, 247, 287, 275, 278, 270, 242, 266, 254, 246, 262, 255, 250, 251, 239, 285, 250, 261, 242, 262, 252, 278, 245, 259, 262, 246, 262, 246, 267, 250, 281, 246, 263, 257, 237, 237, 229, 255, 249, 240, 253, 235, 260, 251, 262, 251, 268, 240, 258, 268, 243, 290, 241, 285] payload_rows_per_dst=[5472, 5454, 5420, 5466, 5426, 5468, 5384, 5522] dup_rows_per_dst=[2794, 2810, 2744, 2742, 2746, 2722, 2700, 2666] combine_rows_per_src=[5472, 5454, 5420, 5466, 5426, 5468, 5384, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=157630464 dispatch_meta_bytes=229712 combine_payload_bytes=157630464 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157860176 ingress_bytes=176504832 link_bytes=176504832 token_bytes=315490640 weight_movement_bytes=18874368 total_nvlink_bytes=334365008 weight_to_token_ratio=0.059825 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.691182454e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38278 dup_rows=21756 dup_rows_remote=19174 dedup_ratio=0.331970 router_tokens_per_expert=[225, 272, 265, 288, 280, 258, 255, 252, 254, 251, 257, 244, 250, 242, 265, 255, 268, 251, 270, 273, 285, 237, 261, 279, 232, 229, 254, 253, 251, 269, 246, 268, 242, 286, 261, 257, 230, 298, 252, 280, 248, 235, 259, 280, 258, 256, 261, 271, 237, 243, 250, 253, 268, 284, 249, 261, 265, 256, 232, 270, 242, 250, 250, 264, 241, 249, 267, 256, 252, 249, 276, 235, 274, 264, 254, 237, 230, 262, 268, 228, 266, 239, 251, 237, 249, 236, 264, 244, 254, 291, 278, 245, 244, 245, 225, 242, 223, 250, 265, 247, 267, 262, 259, 263, 268, 241, 252, 231, 261, 283, 229, 242, 265, 275, 230, 260, 287, 258, 277, 251, 271, 275, 268, 250, 264, 242, 262, 251] payload_rows_per_dst=[5472, 5476, 5524, 5474, 5502, 5400, 5432, 5500] dup_rows_per_dst=[2754, 2776, 2824, 2674, 2582, 2620, 2654, 2872] combine_rows_per_src=[5472, 5476, 5524, 5474, 5502, 5400, 5432, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38266 dup_rows=21738 dup_rows_remote=18982 dedup_ratio=0.331696 router_tokens_per_expert=[242, 258, 247, 251, 269, 248, 253, 252, 247, 267, 250, 265, 264, 280, 263, 273, 249, 237, 233, 250, 252, 247, 234, 259, 258, 270, 244, 256, 268, 285, 257, 251, 263, 250, 253, 243, 252, 240, 275, 221, 255, 243, 258, 249, 244, 258, 260, 255, 253, 244, 294, 257, 257, 248, 271, 254, 222, 269, 256, 272, 255, 256, 230, 235, 261, 255, 252, 258, 277, 244, 268, 279, 256, 267, 270, 278, 255, 285, 249, 267, 271, 306, 239, 242, 263, 250, 260, 255, 251, 239, 256, 243, 249, 260, 233, 248, 249, 251, 258, 274, 254, 265, 242, 268, 251, 247, 268, 228, 270, 278, 272, 243, 251, 256, 252, 263, 219, 270, 273, 248, 256, 247, 261, 241, 272, 242, 256, 286] payload_rows_per_dst=[5530, 5396, 5482, 5434, 5502, 5460, 5532, 5462] dup_rows_per_dst=[2728, 2704, 2556, 2712, 2888, 2670, 2756, 2724] combine_rows_per_src=[5530, 5396, 5482, 5434, 5502, 5460, 5532, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38242 dup_rows=21774 dup_rows_remote=18864 dedup_ratio=0.332245 router_tokens_per_expert=[245, 238, 239, 269, 277, 249, 281, 230, 243, 276, 263, 234, 252, 272, 269, 250, 269, 265, 228, 246, 242, 257, 239, 264, 244, 268, 255, 249, 257, 253, 261, 255, 243, 258, 254, 269, 258, 268, 264, 258, 269, 232, 238, 258, 228, 249, 238, 230, 279, 258, 255, 268, 259, 244, 245, 277, 225, 283, 257, 246, 246, 263, 253, 266, 271, 247, 240, 287, 267, 251, 247, 234, 255, 255, 258, 272, 263, 258, 257, 224, 257, 237, 257, 276, 257, 273, 267, 260, 253, 260, 272, 274, 284, 263, 269, 256, 268, 258, 232, 253, 246, 254, 236, 254, 242, 232, 288, 254, 248, 284, 234, 231, 266, 261, 275, 259, 270, 261, 272, 253, 259, 261, 251, 270, 270, 240, 246, 262] payload_rows_per_dst=[5396, 5482, 5388, 5514, 5504, 5520, 5412, 5546] dup_rows_per_dst=[2778, 2622, 2640, 2734, 2668, 2910, 2616, 2806] combine_rows_per_src=[5396, 5482, 5388, 5514, 5504, 5520, 5412, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38326 dup_rows=21774 dup_rows_remote=19134 dedup_ratio=0.332245 router_tokens_per_expert=[266, 227, 260, 284, 271, 267, 251, 245, 232, 252, 256, 254, 231, 249, 281, 282, 250, 276, 275, 262, 266, 252, 227, 262, 274, 269, 235, 234, 270, 259, 245, 283, 238, 275, 234, 261, 270, 295, 258, 234, 254, 242, 278, 244, 240, 253, 257, 274, 245, 247, 238, 256, 280, 266, 255, 248, 256, 252, 250, 226, 265, 239, 259, 256, 258, 267, 268, 263, 268, 256, 262, 254, 263, 239, 255, 276, 252, 280, 237, 256, 275, 267, 265, 246, 269, 266, 258, 243, 248, 229, 253, 255, 280, 216, 268, 238, 244, 274, 248, 235, 259, 279, 234, 244, 260, 269, 280, 263, 240, 259, 243, 250, 266, 242, 268, 264, 268, 245, 264, 231, 268, 265, 233, 268, 261, 255, 237, 230] payload_rows_per_dst=[5460, 5460, 5524, 5436, 5496, 5474, 5494, 5418] dup_rows_per_dst=[2756, 2818, 2690, 2640, 2812, 2678, 2668, 2712] combine_rows_per_src=[5460, 5460, 5524, 5436, 5496, 5474, 5494, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229088 combine_payload_bytes=156295168 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156524256 ingress_bytes=194043904 link_bytes=194043904 token_bytes=312819424 weight_movement_bytes=37748736 total_nvlink_bytes=350568160 weight_to_token_ratio=0.120673 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859233213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43812 payload_rows_remote=38296 dup_rows=21724 dup_rows_remote=18988 dedup_ratio=0.331482 router_tokens_per_expert=[273, 255, 257, 266, 237, 247, 252, 243, 252, 263, 231, 249, 254, 255, 267, 269, 242, 253, 273, 278, 280, 270, 232, 247, 251, 263, 245, 267, 229, 246, 261, 246, 232, 265, 259, 283, 258, 244, 258, 246, 270, 251, 241, 258, 271, 241, 241, 241, 250, 232, 248, 247, 257, 279, 254, 278, 247, 251, 243, 243, 255, 269, 284, 257, 264, 261, 244, 281, 263, 238, 257, 237, 248, 278, 223, 262, 268, 247, 246, 267, 258, 255, 277, 263, 238, 253, 262, 225, 222, 277, 268, 249, 245, 268, 249, 243, 263, 240, 261, 266, 249, 301, 237, 267, 251, 298, 257, 259, 249, 286, 260, 278, 230, 256, 253, 256, 249, 279, 255, 259, 272, 273, 262, 273, 256, 234, 256, 241] payload_rows_per_dst=[5398, 5290, 5524, 5494, 5484, 5504, 5602, 5516] dup_rows_per_dst=[2624, 2408, 2712, 2836, 2684, 2874, 2850, 2736] combine_rows_per_src=[5398, 5290, 5524, 5494, 5484, 5504, 5602, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43468 payload_rows_remote=37954 dup_rows=22068 dup_rows_remote=19318 dedup_ratio=0.336731 router_tokens_per_expert=[266, 250, 263, 261, 239, 260, 260, 255, 247, 238, 254, 271, 235, 253, 261, 253, 281, 254, 223, 242, 249, 251, 278, 267, 244, 239, 271, 256, 239, 274, 262, 259, 248, 257, 283, 237, 257, 267, 253, 255, 272, 233, 252, 253, 272, 257, 254, 282, 277, 253, 266, 275, 272, 234, 257, 265, 264, 246, 250, 264, 232, 251, 241, 242, 250, 277, 242, 265, 259, 243, 264, 248, 251, 256, 246, 298, 291, 269, 249, 273, 261, 240, 261, 244, 246, 235, 274, 258, 241, 219, 251, 270, 263, 297, 279, 251, 243, 273, 236, 260, 268, 265, 242, 271, 258, 233, 230, 265, 234, 275, 251, 271, 262, 228, 252, 246, 258, 256, 251, 260, 258, 258, 245, 260, 253, 247, 272, 240] payload_rows_per_dst=[5408, 5344, 5514, 5426, 5468, 5462, 5474, 5372] dup_rows_per_dst=[2724, 2834, 2750, 2752, 2894, 2718, 2676, 2720] combine_rows_per_src=[5408, 5344, 5514, 5426, 5468, 5462, 5474, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229808 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157016496 ingress_bytes=194535424 link_bytes=194535424 token_bytes=313803184 weight_movement_bytes=37748736 total_nvlink_bytes=351551920 weight_to_token_ratio=0.120294 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863942715e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228992 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966528 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313704064 weight_movement_bytes=18874368 total_nvlink_bytes=332578432 weight_to_token_ratio=0.060166 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228424 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156867656 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313506888 weight_movement_bytes=18874368 total_nvlink_bytes=332381256 weight_to_token_ratio=0.060204 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229840 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157213136 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314196432 weight_movement_bytes=37748736 total_nvlink_bytes=351945168 weight_to_token_ratio=0.120144 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38270 dup_rows=21760 dup_rows_remote=19064 dedup_ratio=0.332031 router_tokens_per_expert=[252, 235, 272, 257, 259, 256, 258, 237, 253, 269, 233, 265, 247, 273, 273, 255, 273, 255, 256, 259, 262, 272, 250, 278, 230, 274, 253, 247, 256, 279, 251, 270, 270, 259, 259, 248, 261, 277, 255, 259, 252, 263, 262, 250, 240, 250, 258, 228, 246, 264, 242, 279, 242, 242, 249, 247, 277, 269, 242, 236, 251, 219, 236, 265, 266, 258, 272, 256, 237, 264, 233, 254, 254, 259, 259, 258, 248, 258, 250, 249, 263, 243, 250, 261, 242, 249, 240, 282, 243, 265, 282, 267, 272, 249, 239, 244, 275, 260, 251, 260, 261, 231, 251, 249, 275, 250, 246, 243, 269, 272, 262, 232, 265, 291, 274, 256, 270, 274, 251, 236, 241, 240, 241, 260, 256, 258, 267, 279] payload_rows_per_dst=[5506, 5476, 5624, 5372, 5378, 5406, 5642, 5372] dup_rows_per_dst=[2696, 3012, 2764, 2488, 2772, 2648, 2956, 2424] combine_rows_per_src=[5506, 5476, 5624, 5372, 5378, 5406, 5642, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=229136 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089552 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949968 weight_movement_bytes=18874368 total_nvlink_bytes=332824336 weight_to_token_ratio=0.060119 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155459584 dispatch_meta_bytes=229088 combine_payload_bytes=155459584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155688672 ingress_bytes=174333952 link_bytes=174333952 token_bytes=311148256 weight_movement_bytes=18874368 total_nvlink_bytes=330022624 weight_to_token_ratio=0.060660 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.670382150e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43828 payload_rows_remote=38294 dup_rows=21708 dup_rows_remote=18926 dedup_ratio=0.331238 router_tokens_per_expert=[255, 274, 250, 279, 257, 250, 278, 254, 257, 254, 266, 268, 240, 276, 289, 246, 262, 249, 278, 270, 250, 252, 260, 260, 253, 256, 260, 248, 287, 262, 265, 246, 222, 258, 254, 248, 245, 288, 245, 284, 255, 265, 231, 258, 245, 274, 247, 279, 277, 241, 263, 215, 239, 257, 260, 250, 226, 267, 230, 277, 264, 261, 259, 260, 246, 243, 268, 231, 260, 265, 239, 249, 255, 246, 274, 256, 221, 268, 259, 255, 267, 263, 248, 237, 247, 259, 256, 263, 225, 278, 251, 263, 276, 257, 251, 257, 253, 218, 287, 253, 263, 237, 252, 229, 250, 247, 242, 257, 279, 254, 261, 252, 270, 223, 229, 276, 264, 272, 275, 234, 264, 262, 246, 277, 253, 253, 262, 246] payload_rows_per_dst=[5566, 5534, 5476, 5444, 5480, 5460, 5410, 5458] dup_rows_per_dst=[2820, 2782, 2720, 2648, 2590, 2736, 2658, 2754] combine_rows_per_src=[5566, 5534, 5476, 5444, 5480, 5460, 5410, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=156753920 dispatch_meta_bytes=229336 combine_payload_bytes=156753920 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156983256 ingress_bytes=194502656 link_bytes=194502656 token_bytes=313737176 weight_movement_bytes=37748736 total_nvlink_bytes=351485912 weight_to_token_ratio=0.120320 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863628749e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38194 dup_rows=21806 dup_rows_remote=18930 dedup_ratio=0.332733 router_tokens_per_expert=[232, 267, 261, 255, 224, 256, 229, 263, 224, 275, 236, 246, 237, 260, 280, 282, 241, 266, 267, 259, 237, 268, 232, 264, 247, 291, 252, 281, 281, 242, 273, 257, 257, 261, 254, 279, 261, 216, 236, 243, 238, 246, 255, 279, 251, 256, 234, 229, 255, 270, 269, 248, 271, 251, 239, 275, 260, 234, 269, 241, 247, 269, 248, 254, 263, 268, 262, 244, 233, 274, 239, 254, 276, 240, 249, 293, 277, 277, 281, 276, 252, 248, 268, 236, 231, 260, 254, 255, 249, 248, 276, 293, 251, 230, 243, 246, 246, 230, 255, 230, 229, 268, 270, 276, 274, 286, 236, 284, 264, 270, 264, 266, 262, 266, 253, 249, 246, 254, 276, 243, 262, 267, 230, 244, 259, 269, 267, 247] payload_rows_per_dst=[5368, 5530, 5418, 5438, 5536, 5476, 5512, 5452] dup_rows_per_dst=[2686, 2786, 2572, 2762, 2876, 2604, 2784, 2736] combine_rows_per_src=[5368, 5530, 5418, 5438, 5536, 5476, 5512, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=228880 combine_payload_bytes=156852224 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157081104 ingress_bytes=232349696 link_bytes=232349696 token_bytes=313933328 weight_movement_bytes=75497472 total_nvlink_bytes=389430800 weight_to_token_ratio=0.240489 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.226260464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38262 dup_rows=21902 dup_rows_remote=19294 dedup_ratio=0.334198 router_tokens_per_expert=[253, 263, 259, 248, 252, 246, 248, 246, 253, 255, 271, 265, 255, 254, 274, 247, 239, 268, 251, 266, 247, 284, 218, 247, 272, 248, 257, 266, 277, 260, 245, 243, 240, 268, 254, 249, 263, 272, 254, 260, 268, 262, 235, 263, 243, 256, 291, 259, 249, 251, 267, 296, 260, 268, 256, 270, 251, 253, 278, 261, 254, 256, 286, 253, 251, 228, 249, 249, 246, 280, 261, 260, 258, 253, 274, 242, 252, 235, 259, 256, 248, 256, 251, 256, 274, 271, 266, 252, 280, 267, 265, 263, 250, 247, 260, 238, 258, 250, 244, 249, 252, 237, 258, 264, 239, 237, 271, 261, 242, 240, 244, 244, 262, 240, 260, 264, 261, 269, 243, 256, 255, 249, 238, 242, 253, 254, 246, 266] payload_rows_per_dst=[5402, 5442, 5526, 5560, 5396, 5552, 5372, 5384] dup_rows_per_dst=[2776, 2734, 2748, 2858, 2710, 2736, 2608, 2732] combine_rows_per_src=[5402, 5442, 5526, 5560, 5396, 5552, 5372, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43572 payload_rows_remote=38070 dup_rows=21964 dup_rows_remote=19184 dedup_ratio=0.335144 router_tokens_per_expert=[243, 276, 257, 262, 255, 247, 265, 281, 269, 261, 255, 247, 268, 239, 261, 246, 230, 241, 273, 255, 244, 283, 275, 280, 266, 237, 249, 296, 243, 246, 240, 263, 249, 255, 249, 240, 255, 259, 239, 260, 259, 253, 254, 276, 256, 249, 236, 263, 252, 252, 252, 261, 239, 240, 233, 298, 273, 255, 255, 257, 252, 239, 230, 235, 250, 248, 231, 266, 273, 281, 246, 263, 260, 254, 251, 217, 233, 267, 231, 276, 261, 263, 280, 274, 263, 243, 266, 225, 251, 267, 272, 243, 244, 255, 276, 258, 250, 268, 263, 279, 253, 278, 240, 231, 238, 275, 241, 252, 247, 268, 243, 261, 285, 269, 261, 248, 235, 242, 259, 292, 262, 271, 251, 261, 259, 277, 245, 248] payload_rows_per_dst=[5434, 5520, 5438, 5362, 5374, 5502, 5412, 5530] dup_rows_per_dst=[2830, 2722, 2666, 2684, 2720, 2780, 2762, 2800] combine_rows_per_src=[5434, 5520, 5438, 5362, 5374, 5502, 5412, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=38336 dup_rows=21704 dup_rows_remote=19010 dedup_ratio=0.331177 router_tokens_per_expert=[270, 247, 249, 283, 267, 237, 245, 276, 272, 255, 235, 243, 254, 255, 270, 248, 268, 272, 260, 250, 257, 244, 282, 246, 235, 243, 255, 279, 228, 248, 264, 268, 283, 260, 279, 273, 279, 272, 274, 266, 251, 256, 249, 236, 253, 237, 265, 250, 263, 257, 261, 254, 234, 247, 278, 261, 260, 259, 231, 269, 258, 250, 270, 243, 215, 244, 267, 241, 236, 239, 277, 280, 255, 272, 249, 242, 247, 275, 272, 251, 275, 251, 262, 225, 242, 262, 257, 251, 252, 244, 253, 259, 264, 274, 275, 249, 273, 250, 267, 250, 244, 261, 260, 233, 271, 223, 240, 283, 264, 288, 261, 263, 256, 260, 259, 255, 244, 255, 241, 222, 252, 234, 236, 255, 270, 250, 256, 252] payload_rows_per_dst=[5518, 5442, 5510, 5496, 5436, 5498, 5496, 5436] dup_rows_per_dst=[2694, 2756, 2856, 2694, 2688, 2692, 2766, 2558] combine_rows_per_src=[5518, 5442, 5510, 5496, 5436, 5498, 5496, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228496 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671120 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313113744 weight_movement_bytes=18874368 total_nvlink_bytes=331988112 weight_to_token_ratio=0.060280 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43606 payload_rows_remote=38070 dup_rows=21930 dup_rows_remote=19164 dedup_ratio=0.334625 router_tokens_per_expert=[245, 256, 262, 250, 249, 278, 242, 226, 249, 242, 252, 279, 242, 291, 286, 262, 256, 240, 237, 239, 257, 250, 244, 239, 259, 255, 260, 239, 273, 250, 262, 269, 266, 259, 256, 229, 241, 273, 293, 273, 256, 236, 235, 253, 248, 246, 241, 289, 244, 263, 237, 242, 265, 262, 267, 240, 246, 262, 251, 239, 256, 270, 280, 252, 258, 264, 271, 235, 274, 250, 253, 251, 232, 260, 277, 242, 261, 245, 296, 263, 247, 270, 254, 252, 264, 257, 256, 276, 281, 260, 282, 236, 234, 256, 268, 247, 246, 267, 238, 254, 257, 281, 252, 240, 242, 264, 268, 250, 254, 252, 250, 241, 249, 294, 222, 258, 243, 298, 249, 241, 278, 287, 245, 263, 243, 258, 240, 262] payload_rows_per_dst=[5342, 5398, 5432, 5442, 5460, 5566, 5430, 5536] dup_rows_per_dst=[2754, 2660, 2760, 2768, 2818, 2714, 2690, 2766] combine_rows_per_src=[5342, 5398, 5432, 5442, 5460, 5566, 5430, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38332 dup_rows=21828 dup_rows_remote=19144 dedup_ratio=0.333069 router_tokens_per_expert=[231, 257, 246, 245, 274, 277, 246, 255, 277, 258, 244, 274, 273, 236, 239, 257, 243, 250, 262, 244, 249, 261, 252, 276, 239, 225, 240, 260, 281, 286, 256, 272, 257, 251, 270, 242, 256, 258, 243, 227, 274, 255, 248, 256, 222, 261, 270, 240, 240, 232, 239, 239, 239, 237, 229, 265, 269, 263, 234, 246, 272, 282, 272, 263, 236, 236, 300, 248, 284, 262, 250, 253, 267, 263, 238, 249, 268, 283, 252, 260, 269, 284, 269, 248, 238, 208, 240, 247, 260, 256, 227, 246, 291, 286, 265, 267, 263, 272, 253, 262, 259, 267, 261, 296, 251, 265, 230, 262, 259, 256, 259, 255, 248, 262, 281, 245, 263, 242, 230, 268, 265, 250, 273, 231, 255, 265, 266, 268] payload_rows_per_dst=[5466, 5514, 5376, 5348, 5490, 5490, 5516, 5508] dup_rows_per_dst=[2712, 2678, 2684, 2694, 2808, 2712, 2824, 2716] combine_rows_per_src=[5466, 5514, 5376, 5348, 5490, 5490, 5516, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156721152 dispatch_meta_bytes=230224 combine_payload_bytes=156721152 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156951376 ingress_bytes=213344256 link_bytes=213344256 token_bytes=313672528 weight_movement_bytes=56623104 total_nvlink_bytes=370295632 weight_to_token_ratio=0.180517 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044159689e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=229016 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156163736 ingress_bytes=174809088 link_bytes=174809088 token_bytes=312098456 weight_movement_bytes=18874368 total_nvlink_bytes=330972824 weight_to_token_ratio=0.060476 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674934670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43782 payload_rows_remote=38252 dup_rows=21754 dup_rows_remote=18852 dedup_ratio=0.331940 router_tokens_per_expert=[229, 256, 247, 234, 255, 269, 261, 269, 265, 270, 240, 259, 266, 248, 268, 270, 231, 264, 254, 258, 274, 249, 245, 246, 230, 277, 247, 238, 262, 249, 224, 258, 249, 252, 253, 243, 256, 218, 264, 252, 256, 265, 273, 284, 231, 280, 258, 273, 254, 264, 236, 233, 251, 222, 273, 276, 245, 251, 242, 274, 254, 283, 253, 235, 229, 300, 272, 249, 232, 254, 242, 272, 236, 261, 245, 250, 265, 298, 247, 291, 251, 227, 273, 267, 268, 231, 270, 255, 254, 270, 257, 291, 276, 258, 286, 257, 255, 249, 243, 248, 259, 271, 242, 251, 255, 288, 242, 261, 245, 230, 272, 275, 252, 270, 230, 252, 250, 272, 228, 274, 247, 275, 252, 238, 263, 270, 244, 266] payload_rows_per_dst=[5530, 5408, 5492, 5454, 5536, 5528, 5442, 5392] dup_rows_per_dst=[2902, 2672, 2722, 2678, 2750, 2746, 2730, 2554] combine_rows_per_src=[5530, 5408, 5492, 5454, 5536, 5528, 5442, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=157024256 dispatch_meta_bytes=229384 combine_payload_bytes=157024256 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157253640 ingress_bytes=175898624 link_bytes=175898624 token_bytes=314277896 weight_movement_bytes=18874368 total_nvlink_bytes=333152264 weight_to_token_ratio=0.060056 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685374068e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=228936 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156163656 ingress_bytes=174809088 link_bytes=174809088 token_bytes=312098376 weight_movement_bytes=18874368 total_nvlink_bytes=330972744 weight_to_token_ratio=0.060476 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674934670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157007872 dispatch_meta_bytes=229904 combine_payload_bytes=157007872 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157237776 ingress_bytes=175882240 link_bytes=175882240 token_bytes=314245648 weight_movement_bytes=18874368 total_nvlink_bytes=333120016 weight_to_token_ratio=0.060062 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685217084e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43544 payload_rows_remote=38116 dup_rows=21992 dup_rows_remote=19304 dedup_ratio=0.335571 router_tokens_per_expert=[266, 264, 250, 245, 259, 259, 250, 271, 276, 273, 261, 253, 260, 262, 242, 235, 260, 230, 247, 231, 261, 261, 240, 249, 279, 274, 250, 241, 280, 251, 268, 236, 272, 266, 269, 252, 269, 242, 237, 244, 222, 274, 235, 250, 283, 260, 261, 250, 246, 277, 256, 237, 244, 268, 231, 247, 218, 281, 285, 254, 260, 257, 253, 270, 239, 258, 248, 272, 251, 264, 257, 251, 244, 233, 272, 276, 240, 249, 251, 258, 253, 273, 242, 266, 237, 248, 253, 246, 235, 254, 251, 261, 250, 253, 272, 264, 281, 253, 251, 272, 248, 258, 294, 269, 222, 255, 266, 263, 273, 279, 240, 276, 263, 253, 239, 226, 270, 260, 248, 253, 254, 252, 287, 266, 280, 238, 247, 257] payload_rows_per_dst=[5506, 5428, 5424, 5468, 5416, 5430, 5466, 5406] dup_rows_per_dst=[2746, 2688, 2748, 2700, 2710, 2686, 2934, 2780] combine_rows_per_src=[5506, 5428, 5424, 5468, 5416, 5430, 5466, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=25 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228416 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156908608 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313588800 weight_movement_bytes=37748736 total_nvlink_bytes=351337536 weight_to_token_ratio=0.120377 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38260 dup_rows=21800 dup_rows_remote=19168 dedup_ratio=0.332642 router_tokens_per_expert=[256, 239, 254, 264, 257, 296, 247, 235, 269, 266, 252, 239, 252, 254, 267, 273, 272, 247, 241, 286, 245, 270, 257, 247, 258, 258, 266, 261, 228, 285, 243, 241, 256, 259, 245, 278, 256, 284, 237, 243, 249, 257, 283, 236, 254, 266, 266, 219, 239, 247, 273, 231, 255, 256, 314, 268, 229, 259, 264, 264, 257, 234, 266, 252, 256, 250, 280, 214, 254, 242, 252, 250, 243, 270, 240, 251, 243, 283, 259, 267, 258, 255, 253, 260, 286, 237, 234, 237, 259, 260, 253, 223, 270, 272, 253, 259, 229, 265, 255, 268, 275, 251, 248, 275, 247, 249, 276, 268, 221, 251, 257, 252, 264, 280, 259, 237, 285, 286, 262, 255, 249, 287, 240, 254, 242, 253, 246, 238] payload_rows_per_dst=[5504, 5428, 5496, 5514, 5476, 5446, 5394, 5478] dup_rows_per_dst=[2736, 2782, 2680, 2702, 2632, 2692, 2780, 2796] combine_rows_per_src=[5504, 5428, 5496, 5514, 5476, 5446, 5394, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229680 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352816 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475952 weight_movement_bytes=18874368 total_nvlink_bytes=331350320 weight_to_token_ratio=0.060403 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43744 payload_rows_remote=38298 dup_rows=21792 dup_rows_remote=19104 dedup_ratio=0.332520 router_tokens_per_expert=[252, 246, 254, 264, 247, 284, 265, 257, 257, 224, 262, 256, 273, 261, 245, 238, 241, 236, 240, 258, 249, 252, 286, 250, 252, 258, 236, 258, 263, 270, 260, 248, 244, 264, 272, 259, 261, 244, 230, 250, 263, 258, 263, 262, 259, 263, 235, 264, 249, 262, 283, 254, 290, 248, 262, 256, 256, 239, 264, 262, 270, 250, 249, 265, 267, 278, 255, 243, 245, 288, 246, 274, 235, 260, 243, 260, 274, 255, 275, 262, 253, 241, 240, 254, 221, 233, 258, 258, 243, 273, 263, 256, 239, 264, 270, 254, 272, 223, 261, 221, 257, 281, 234, 277, 252, 235, 261, 264, 267, 236, 268, 258, 245, 260, 269, 282, 272, 239, 264, 279, 257, 250, 242, 255, 258, 249, 268, 240] payload_rows_per_dst=[5454, 5406, 5422, 5586, 5518, 5386, 5446, 5526] dup_rows_per_dst=[2716, 2708, 2760, 2732, 2802, 2654, 2688, 2732] combine_rows_per_src=[5454, 5406, 5422, 5586, 5518, 5386, 5446, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38298 dup_rows=21784 dup_rows_remote=19084 dedup_ratio=0.332397 router_tokens_per_expert=[234, 241, 272, 257, 279, 251, 271, 267, 264, 236, 292, 233, 274, 249, 244, 299, 231, 283, 249, 243, 247, 252, 264, 269, 288, 233, 259, 235, 245, 239, 270, 252, 269, 262, 238, 240, 253, 277, 223, 246, 234, 251, 270, 257, 263, 278, 245, 262, 247, 266, 231, 255, 267, 244, 256, 251, 239, 260, 243, 251, 286, 231, 266, 245, 274, 254, 268, 239, 263, 285, 261, 257, 278, 217, 232, 262, 243, 237, 264, 273, 249, 271, 237, 241, 256, 248, 256, 248, 266, 269, 283, 245, 244, 262, 247, 255, 272, 244, 274, 270, 263, 266, 259, 280, 281, 242, 255, 253, 285, 254, 265, 288, 247, 241, 250, 248, 261, 254, 214, 246, 236, 259, 277, 240, 274, 265, 267, 226] payload_rows_per_dst=[5544, 5414, 5432, 5396, 5514, 5454, 5554, 5444] dup_rows_per_dst=[2782, 2704, 2704, 2680, 2700, 2700, 2948, 2566] combine_rows_per_src=[5544, 5414, 5432, 5396, 5514, 5454, 5554, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43708 payload_rows_remote=38252 dup_rows=21828 dup_rows_remote=19194 dedup_ratio=0.333069 router_tokens_per_expert=[272, 290, 235, 270, 248, 247, 252, 250, 242, 242, 238, 269, 247, 278, 246, 245, 233, 227, 247, 262, 284, 226, 252, 246, 264, 256, 260, 248, 269, 265, 244, 293, 245, 290, 247, 256, 262, 279, 257, 269, 276, 270, 254, 254, 238, 243, 245, 282, 247, 268, 243, 240, 267, 262, 287, 249, 268, 256, 232, 239, 257, 218, 268, 244, 260, 250, 267, 268, 255, 262, 278, 287, 232, 248, 273, 274, 266, 244, 241, 265, 317, 252, 252, 254, 276, 244, 240, 253, 244, 258, 250, 276, 244, 242, 257, 228, 266, 240, 275, 255, 279, 256, 246, 270, 236, 266, 257, 256, 247, 261, 241, 253, 255, 263, 240, 250, 257, 266, 242, 252, 266, 251, 229, 264, 257, 245, 272, 239] payload_rows_per_dst=[5364, 5534, 5484, 5456, 5528, 5476, 5498, 5368] dup_rows_per_dst=[2778, 2618, 2850, 2634, 2812, 2698, 2710, 2728] combine_rows_per_src=[5364, 5534, 5484, 5456, 5528, 5476, 5498, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229712 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156942672 ingress_bytes=194461696 link_bytes=194461696 token_bytes=313655632 weight_movement_bytes=37748736 total_nvlink_bytes=351404368 weight_to_token_ratio=0.120351 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863236290e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38238 dup_rows=21740 dup_rows_remote=18892 dedup_ratio=0.331726 router_tokens_per_expert=[256, 268, 258, 243, 262, 246, 238, 262, 260, 226, 248, 248, 248, 248, 271, 282, 275, 264, 255, 268, 243, 256, 245, 278, 256, 264, 276, 241, 227, 270, 258, 229, 257, 260, 253, 249, 246, 224, 235, 228, 259, 259, 271, 298, 244, 265, 238, 235, 271, 258, 309, 277, 246, 266, 272, 244, 249, 268, 264, 264, 267, 246, 231, 262, 253, 258, 245, 265, 268, 270, 277, 237, 244, 236, 260, 245, 307, 270, 268, 236, 260, 256, 261, 250, 273, 235, 246, 233, 233, 263, 262, 243, 245, 237, 263, 289, 283, 241, 253, 273, 268, 249, 235, 236, 240, 250, 262, 229, 251, 279, 270, 248, 265, 259, 240, 302, 269, 244, 259, 267, 284, 221, 270, 225, 260, 270, 258, 236] payload_rows_per_dst=[5446, 5462, 5392, 5588, 5490, 5422, 5438, 5558] dup_rows_per_dst=[2666, 2684, 2650, 2682, 2788, 2710, 2712, 2848] combine_rows_per_src=[5446, 5462, 5392, 5588, 5490, 5422, 5438, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43804 payload_rows_remote=38396 dup_rows=21732 dup_rows_remote=19142 dedup_ratio=0.331604 router_tokens_per_expert=[257, 247, 249, 261, 270, 265, 257, 268, 252, 237, 266, 254, 264, 267, 228, 249, 267, 279, 235, 288, 257, 238, 242, 250, 274, 251, 239, 241, 288, 272, 244, 280, 291, 254, 222, 257, 250, 237, 245, 257, 269, 265, 250, 238, 255, 239, 250, 220, 251, 257, 243, 255, 280, 264, 252, 251, 271, 243, 238, 268, 228, 266, 269, 290, 234, 241, 270, 260, 232, 305, 237, 261, 246, 258, 250, 239, 253, 245, 240, 275, 262, 268, 277, 292, 241, 257, 248, 265, 233, 252, 256, 260, 257, 275, 242, 252, 274, 273, 277, 273, 283, 249, 241, 258, 263, 268, 258, 261, 255, 278, 251, 234, 288, 257, 257, 231, 249, 249, 255, 227, 248, 266, 253, 268, 258, 222, 248, 252] payload_rows_per_dst=[5460, 5538, 5408, 5548, 5396, 5494, 5524, 5436] dup_rows_per_dst=[2722, 2752, 2590, 2704, 2696, 2780, 2868, 2620] combine_rows_per_src=[5460, 5538, 5408, 5548, 5396, 5494, 5524, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229608 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157098216 ingress_bytes=194617344 link_bytes=194617344 token_bytes=313966824 weight_movement_bytes=37748736 total_nvlink_bytes=351715560 weight_to_token_ratio=0.120232 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864727632e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43656 payload_rows_remote=38292 dup_rows=21880 dup_rows_remote=19180 dedup_ratio=0.333862 router_tokens_per_expert=[277, 242, 269, 239, 258, 267, 253, 259, 245, 243, 264, 238, 233, 221, 251, 227, 242, 245, 276, 273, 256, 252, 249, 236, 262, 260, 279, 279, 257, 259, 261, 255, 265, 257, 281, 247, 255, 265, 244, 280, 260, 272, 264, 253, 258, 273, 280, 259, 268, 254, 240, 272, 233, 273, 262, 254, 246, 259, 254, 258, 263, 233, 268, 256, 278, 262, 242, 262, 217, 278, 250, 239, 277, 276, 264, 253, 259, 253, 274, 246, 264, 259, 253, 216, 254, 262, 222, 275, 237, 250, 257, 233, 259, 228, 257, 228, 269, 258, 235, 249, 244, 242, 234, 278, 272, 238, 248, 268, 261, 254, 274, 282, 274, 245, 253, 225, 240, 263, 274, 254, 281, 261, 265, 270, 250, 253, 270, 267] payload_rows_per_dst=[5364, 5536, 5568, 5336, 5490, 5334, 5528, 5500] dup_rows_per_dst=[2700, 2746, 2876, 2758, 2770, 2574, 2684, 2772] combine_rows_per_src=[5364, 5536, 5568, 5336, 5490, 5334, 5528, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229528 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098136 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313966744 weight_movement_bytes=18874368 total_nvlink_bytes=332841112 weight_to_token_ratio=0.060116 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.090278 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229784 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156909976 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313590168 weight_movement_bytes=18874368 total_nvlink_bytes=332464536 weight_to_token_ratio=0.060188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156622848 dispatch_meta_bytes=228520 combine_payload_bytes=156622848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156851368 ingress_bytes=194371584 link_bytes=194371584 token_bytes=313474216 weight_movement_bytes=37748736 total_nvlink_bytes=351222952 weight_to_token_ratio=0.120421 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862372881e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157270016 dispatch_meta_bytes=230152 combine_payload_bytes=157270016 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157500168 ingress_bytes=176144384 link_bytes=176144384 token_bytes=314770184 weight_movement_bytes=18874368 total_nvlink_bytes=333644552 weight_to_token_ratio=0.059962 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687728819e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38108 dup_rows=21868 dup_rows_remote=19042 dedup_ratio=0.333679 router_tokens_per_expert=[268, 284, 250, 276, 240, 259, 252, 268, 263, 219, 276, 272, 255, 241, 265, 268, 248, 247, 260, 263, 254, 251, 256, 261, 264, 249, 295, 274, 262, 254, 272, 283, 264, 234, 271, 260, 249, 262, 274, 245, 222, 233, 250, 265, 253, 247, 248, 265, 267, 258, 273, 260, 267, 258, 260, 258, 246, 236, 234, 256, 255, 257, 236, 257, 251, 253, 241, 240, 209, 242, 228, 267, 232, 270, 238, 260, 256, 270, 240, 253, 257, 251, 272, 246, 284, 262, 255, 255, 254, 258, 254, 275, 270, 262, 253, 238, 266, 255, 235, 237, 245, 224, 255, 239, 283, 268, 273, 245, 279, 229, 299, 269, 267, 291, 257, 282, 259, 247, 258, 272, 240, 255, 239, 250, 248, 234, 250, 253] payload_rows_per_dst=[5504, 5560, 5372, 5446, 5328, 5534, 5486, 5438] dup_rows_per_dst=[2808, 2826, 2712, 2710, 2588, 2758, 2716, 2750] combine_rows_per_src=[5504, 5560, 5372, 5446, 5328, 5534, 5486, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=26 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=229888 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157073920 ingress_bytes=194592768 link_bytes=194592768 token_bytes=313917952 weight_movement_bytes=37748736 total_nvlink_bytes=351666688 weight_to_token_ratio=0.120250 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864492157e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43830 payload_rows_remote=38400 dup_rows=21706 dup_rows_remote=19136 dedup_ratio=0.331207 router_tokens_per_expert=[257, 237, 230, 253, 264, 243, 261, 239, 262, 250, 251, 273, 254, 228, 231, 252, 262, 260, 255, 245, 265, 228, 249, 267, 260, 271, 263, 248, 271, 277, 245, 234, 237, 276, 254, 270, 242, 250, 234, 253, 264, 264, 281, 240, 248, 239, 255, 243, 250, 255, 282, 264, 264, 240, 271, 268, 270, 236, 256, 263, 271, 244, 248, 237, 247, 279, 247, 250, 234, 245, 240, 259, 227, 268, 262, 268, 247, 241, 230, 256, 265, 239, 251, 285, 269, 257, 258, 254, 261, 276, 266, 282, 254, 262, 248, 237, 252, 241, 274, 256, 288, 273, 280, 252, 273, 261, 250, 240, 259, 265, 264, 263, 265, 269, 278, 242, 221, 280, 271, 249, 251, 254, 260, 280, 249, 259, 255, 276] payload_rows_per_dst=[5448, 5482, 5474, 5462, 5430, 5512, 5532, 5490] dup_rows_per_dst=[2522, 2718, 2626, 2776, 2570, 2816, 2850, 2828] combine_rows_per_src=[5448, 5482, 5474, 5462, 5430, 5512, 5532, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156090368 dispatch_meta_bytes=228600 combine_payload_bytes=156090368 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156318968 ingress_bytes=212713472 link_bytes=212713472 token_bytes=312409336 weight_movement_bytes=56623104 total_nvlink_bytes=369032440 weight_to_token_ratio=0.181247 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038115827e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38226 dup_rows=21798 dup_rows_remote=19168 dedup_ratio=0.332611 router_tokens_per_expert=[291, 264, 278, 268, 249, 275, 224, 245, 247, 242, 268, 266, 244, 267, 246, 251, 269, 235, 248, 254, 257, 220, 244, 269, 271, 249, 267, 244, 228, 269, 250, 251, 251, 243, 257, 284, 281, 268, 268, 278, 244, 268, 265, 279, 270, 241, 242, 240, 274, 251, 239, 258, 277, 231, 283, 249, 243, 263, 242, 282, 243, 265, 234, 276, 239, 257, 251, 254, 259, 259, 233, 282, 246, 269, 266, 264, 238, 244, 263, 289, 240, 242, 275, 239, 251, 252, 256, 261, 251, 263, 268, 237, 258, 237, 256, 247, 249, 248, 274, 277, 229, 253, 247, 270, 241, 258, 250, 250, 253, 256, 257, 259, 243, 285, 287, 269, 246, 240, 236, 268, 282, 280, 230, 256, 258, 270, 226, 236] payload_rows_per_dst=[5518, 5414, 5482, 5476, 5468, 5436, 5512, 5432] dup_rows_per_dst=[2732, 2636, 2876, 2744, 2758, 2630, 2630, 2792] combine_rows_per_src=[5518, 5414, 5482, 5476, 5468, 5436, 5512, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157286400 dispatch_meta_bytes=230144 combine_payload_bytes=157286400 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157516544 ingress_bytes=176160768 link_bytes=176160768 token_bytes=314802944 weight_movement_bytes=18874368 total_nvlink_bytes=333677312 weight_to_token_ratio=0.059956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687885802e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43690 payload_rows_remote=38312 dup_rows=21846 dup_rows_remote=19066 dedup_ratio=0.333344 router_tokens_per_expert=[242, 294, 260, 279, 240, 270, 259, 253, 256, 235, 253, 259, 261, 291, 281, 249, 252, 270, 236, 297, 256, 263, 273, 244, 253, 246, 243, 266, 281, 250, 242, 262, 272, 225, 270, 283, 238, 253, 244, 246, 244, 245, 239, 243, 256, 230, 246, 285, 273, 266, 245, 243, 272, 227, 273, 225, 247, 251, 274, 285, 261, 235, 243, 278, 250, 248, 259, 273, 280, 267, 249, 262, 248, 271, 243, 243, 247, 254, 244, 261, 266, 273, 251, 262, 260, 270, 255, 241, 250, 241, 234, 269, 243, 256, 248, 260, 268, 255, 261, 259, 254, 260, 238, 251, 282, 245, 261, 229, 258, 269, 232, 245, 257, 260, 246, 260, 248, 256, 253, 260, 230, 250, 265, 279, 248, 255, 274, 249] payload_rows_per_dst=[5524, 5532, 5406, 5458, 5456, 5378, 5404, 5532] dup_rows_per_dst=[2840, 2736, 2632, 2738, 2742, 2780, 2730, 2648] combine_rows_per_src=[5524, 5532, 5406, 5458, 5456, 5378, 5404, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43900 payload_rows_remote=38372 dup_rows=21636 dup_rows_remote=19024 dedup_ratio=0.330139 router_tokens_per_expert=[277, 253, 269, 260, 251, 266, 256, 251, 263, 263, 284, 262, 252, 258, 264, 258, 268, 284, 276, 261, 248, 271, 256, 251, 254, 251, 253, 257, 262, 258, 259, 252, 254, 253, 253, 240, 249, 243, 268, 260, 242, 270, 258, 246, 256, 257, 253, 268, 245, 270, 264, 237, 229, 264, 259, 258, 265, 250, 257, 251, 273, 230, 240, 252, 248, 271, 256, 247, 237, 258, 254, 236, 274, 250, 226, 269, 282, 245, 245, 272, 271, 256, 277, 281, 257, 267, 265, 246, 251, 264, 256, 253, 236, 252, 246, 272, 251, 230, 232, 263, 257, 248, 236, 283, 274, 245, 250, 242, 249, 270, 262, 216, 269, 244, 247, 259, 246, 231, 224, 274, 254, 276, 254, 264, 251, 256, 258, 271] payload_rows_per_dst=[5494, 5500, 5528, 5494, 5474, 5536, 5384, 5490] dup_rows_per_dst=[2880, 2822, 2612, 2594, 2666, 2764, 2632, 2666] combine_rows_per_src=[5494, 5500, 5528, 5494, 5474, 5536, 5384, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229576 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156803272 ingress_bytes=194322432 link_bytes=194322432 token_bytes=313376968 weight_movement_bytes=37748736 total_nvlink_bytes=351125704 weight_to_token_ratio=0.120458 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861901931e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43792 payload_rows_remote=38338 dup_rows=21744 dup_rows_remote=18906 dedup_ratio=0.331787 router_tokens_per_expert=[284, 252, 254, 249, 238, 250, 291, 240, 247, 246, 260, 247, 257, 241, 261, 271, 260, 269, 268, 275, 262, 261, 266, 256, 261, 235, 254, 264, 234, 250, 216, 270, 257, 259, 229, 290, 255, 268, 261, 259, 254, 255, 250, 256, 263, 279, 262, 269, 223, 265, 246, 292, 279, 254, 228, 274, 275, 222, 274, 259, 244, 274, 268, 269, 267, 255, 247, 257, 235, 228, 257, 231, 251, 236, 269, 237, 250, 235, 248, 262, 254, 265, 232, 279, 268, 258, 259, 250, 269, 245, 244, 234, 243, 234, 266, 249, 238, 254, 257, 255, 255, 286, 224, 252, 272, 277, 265, 279, 256, 237, 253, 260, 239, 252, 264, 296, 270, 251, 257, 236, 262, 250, 267, 273, 247, 243, 263, 263] payload_rows_per_dst=[5458, 5464, 5554, 5454, 5360, 5422, 5544, 5536] dup_rows_per_dst=[2718, 2738, 2778, 2838, 2570, 2676, 2696, 2730] combine_rows_per_src=[5458, 5464, 5554, 5454, 5360, 5422, 5544, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43724 payload_rows_remote=38234 dup_rows=21812 dup_rows_remote=19086 dedup_ratio=0.332825 router_tokens_per_expert=[259, 236, 249, 268, 271, 247, 287, 237, 255, 251, 251, 259, 263, 258, 261, 271, 248, 276, 247, 273, 229, 261, 269, 279, 241, 237, 271, 245, 250, 239, 238, 268, 266, 249, 286, 245, 261, 213, 249, 255, 246, 250, 258, 258, 251, 252, 267, 259, 241, 260, 270, 259, 250, 252, 265, 264, 251, 263, 283, 280, 281, 245, 258, 259, 270, 266, 253, 262, 253, 259, 297, 265, 238, 247, 264, 277, 244, 252, 230, 244, 238, 260, 230, 259, 267, 225, 220, 258, 280, 253, 283, 236, 262, 265, 228, 240, 221, 235, 235, 265, 244, 234, 283, 252, 260, 250, 277, 262, 259, 270, 264, 287, 253, 271, 262, 273, 239, 257, 240, 258, 264, 252, 262, 249, 252, 243, 270, 260] payload_rows_per_dst=[5474, 5462, 5402, 5546, 5488, 5364, 5498, 5490] dup_rows_per_dst=[2738, 2680, 2728, 2816, 2754, 2644, 2726, 2726] combine_rows_per_src=[5474, 5462, 5402, 5546, 5488, 5364, 5498, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38294 dup_rows=21772 dup_rows_remote=19026 dedup_ratio=0.332214 router_tokens_per_expert=[240, 247, 285, 242, 259, 269, 307, 218, 261, 252, 248, 261, 249, 250, 252, 256, 268, 285, 230, 238, 282, 222, 269, 277, 276, 227, 259, 256, 265, 270, 271, 245, 246, 232, 276, 259, 240, 276, 255, 303, 260, 267, 280, 227, 255, 270, 262, 265, 249, 243, 253, 238, 252, 250, 245, 258, 257, 284, 225, 230, 243, 255, 260, 247, 249, 248, 255, 260, 275, 232, 246, 268, 267, 258, 264, 271, 236, 247, 258, 262, 272, 248, 256, 242, 246, 254, 245, 243, 259, 251, 248, 261, 244, 273, 282, 284, 268, 267, 245, 242, 256, 260, 264, 247, 259, 253, 245, 259, 258, 264, 250, 264, 253, 290, 260, 263, 241, 255, 242, 275, 274, 240, 221, 248, 256, 239, 264, 244] payload_rows_per_dst=[5470, 5506, 5520, 5394, 5486, 5500, 5474, 5414] dup_rows_per_dst=[2746, 2774, 2826, 2584, 2886, 2716, 2524, 2716] combine_rows_per_src=[5470, 5506, 5520, 5394, 5486, 5500, 5474, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=156925952 dispatch_meta_bytes=229512 combine_payload_bytes=156925952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157155464 ingress_bytes=175800320 link_bytes=175800320 token_bytes=314081416 weight_movement_bytes=18874368 total_nvlink_bytes=332955784 weight_to_token_ratio=0.060094 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684432167e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157171712 dispatch_meta_bytes=229584 combine_payload_bytes=157171712 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157401296 ingress_bytes=176046080 link_bytes=176046080 token_bytes=314573008 weight_movement_bytes=18874368 total_nvlink_bytes=333447376 weight_to_token_ratio=0.060000 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686786918e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=157032448 dispatch_meta_bytes=228976 combine_payload_bytes=157032448 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157261424 ingress_bytes=194781184 link_bytes=194781184 token_bytes=314293872 weight_movement_bytes=37748736 total_nvlink_bytes=352042608 weight_to_token_ratio=0.120106 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866297467e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229280 combine_payload_bytes=156606464 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156835744 ingress_bytes=213229568 link_bytes=213229568 token_bytes=313442208 weight_movement_bytes=56623104 total_nvlink_bytes=370065312 weight_to_token_ratio=0.180649 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043060805e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38242 dup_rows=21784 dup_rows_remote=18918 dedup_ratio=0.332397 router_tokens_per_expert=[282, 273, 240, 253, 246, 248, 269, 245, 242, 272, 280, 276, 259, 278, 248, 249, 256, 248, 263, 292, 275, 242, 253, 285, 267, 267, 270, 250, 238, 265, 256, 261, 222, 273, 279, 229, 284, 265, 242, 271, 253, 252, 256, 249, 244, 254, 250, 234, 237, 230, 273, 241, 265, 246, 262, 246, 255, 250, 248, 268, 212, 268, 274, 257, 243, 268, 259, 237, 248, 246, 258, 270, 244, 268, 220, 246, 242, 247, 251, 237, 281, 259, 282, 239, 241, 259, 241, 275, 278, 264, 243, 263, 259, 247, 270, 263, 254, 264, 266, 302, 251, 252, 246, 245, 264, 240, 256, 264, 277, 242, 243, 234, 262, 249, 284, 230, 264, 257, 269, 249, 257, 265, 246, 236, 268, 212, 260, 275] payload_rows_per_dst=[5510, 5510, 5436, 5392, 5404, 5508, 5488, 5504] dup_rows_per_dst=[2810, 2866, 2678, 2672, 2564, 2820, 2712, 2662] combine_rows_per_src=[5510, 5510, 5436, 5392, 5404, 5508, 5488, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=27 plans=1 dispatch_payload_bytes=156852224 dispatch_meta_bytes=229280 combine_payload_bytes=156852224 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157081504 ingress_bytes=194600960 link_bytes=194600960 token_bytes=313933728 weight_movement_bytes=37748736 total_nvlink_bytes=351682464 weight_to_token_ratio=0.120244 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864570649e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38202 dup_rows=21710 dup_rows_remote=18948 dedup_ratio=0.331268 router_tokens_per_expert=[260, 256, 217, 253, 250, 276, 247, 250, 257, 279, 250, 252, 258, 275, 261, 217, 236, 271, 262, 264, 267, 252, 242, 261, 232, 294, 253, 253, 244, 273, 278, 250, 223, 277, 253, 247, 241, 281, 265, 243, 218, 233, 248, 244, 295, 246, 246, 270, 225, 265, 259, 259, 270, 241, 222, 232, 231, 266, 242, 233, 245, 274, 227, 241, 245, 259, 258, 257, 250, 284, 283, 248, 230, 292, 233, 259, 304, 282, 250, 259, 255, 261, 249, 264, 268, 262, 303, 271, 239, 285, 248, 238, 255, 265, 293, 254, 238, 252, 263, 272, 242, 274, 277, 255, 239, 249, 274, 259, 221, 249, 264, 245, 262, 252, 247, 248, 275, 256, 292, 282, 253, 269, 252, 258, 257, 229, 238, 270] payload_rows_per_dst=[5418, 5506, 5378, 5322, 5624, 5574, 5518, 5486] dup_rows_per_dst=[2698, 2758, 2682, 2542, 2762, 2846, 2628, 2794] combine_rows_per_src=[5418, 5506, 5378, 5322, 5624, 5574, 5518, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38260 dup_rows=21816 dup_rows_remote=19044 dedup_ratio=0.332886 router_tokens_per_expert=[293, 250, 233, 269, 258, 264, 266, 250, 254, 235, 237, 259, 283, 239, 249, 222, 254, 248, 241, 249, 254, 265, 243, 263, 222, 258, 265, 253, 229, 239, 238, 261, 281, 261, 264, 246, 255, 244, 253, 279, 274, 253, 293, 258, 221, 279, 281, 276, 251, 269, 226, 261, 259, 281, 269, 234, 252, 278, 263, 269, 263, 263, 249, 248, 270, 276, 256, 219, 292, 268, 238, 243, 265, 257, 263, 247, 276, 253, 247, 265, 261, 228, 249, 269, 236, 261, 227, 273, 227, 228, 258, 271, 259, 251, 267, 239, 279, 234, 255, 261, 237, 270, 251, 234, 250, 267, 254, 263, 248, 272, 295, 246, 228, 241, 256, 279, 269, 259, 232, 256, 265, 245, 255, 275, 274, 259, 274, 250] payload_rows_per_dst=[5408, 5390, 5556, 5430, 5554, 5444, 5460, 5478] dup_rows_per_dst=[2714, 2574, 2880, 2840, 2716, 2564, 2772, 2756] combine_rows_per_src=[5408, 5390, 5556, 5430, 5554, 5444, 5460, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=228640 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156867872 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507104 weight_movement_bytes=18874368 total_nvlink_bytes=332381472 weight_to_token_ratio=0.060204 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=228600 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156703992 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179384 weight_movement_bytes=18874368 total_nvlink_bytes=332053752 weight_to_token_ratio=0.060267 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43582 payload_rows_remote=38186 dup_rows=21954 dup_rows_remote=19248 dedup_ratio=0.334991 router_tokens_per_expert=[266, 288, 252, 249, 231, 236, 273, 280, 302, 215, 235, 229, 238, 224, 275, 258, 262, 259, 270, 264, 242, 256, 246, 235, 315, 268, 234, 267, 247, 230, 246, 258, 259, 230, 271, 269, 243, 244, 270, 252, 254, 261, 260, 256, 230, 242, 268, 242, 280, 262, 256, 251, 225, 254, 240, 264, 276, 267, 260, 267, 259, 260, 240, 234, 253, 252, 272, 257, 224, 259, 257, 263, 262, 261, 252, 251, 260, 255, 237, 274, 273, 259, 252, 242, 246, 280, 246, 257, 262, 256, 284, 279, 282, 254, 236, 247, 247, 246, 243, 268, 267, 252, 243, 273, 246, 271, 270, 264, 238, 256, 232, 272, 252, 257, 266, 245, 241, 291, 272, 221, 275, 281, 241, 258, 267, 265, 250, 258] payload_rows_per_dst=[5424, 5420, 5396, 5488, 5464, 5494, 5436, 5460] dup_rows_per_dst=[2678, 2778, 2706, 2702, 2714, 2816, 2740, 2820] combine_rows_per_src=[5424, 5420, 5396, 5488, 5464, 5494, 5436, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43776 payload_rows_remote=38342 dup_rows=21760 dup_rows_remote=19004 dedup_ratio=0.332031 router_tokens_per_expert=[237, 244, 265, 255, 261, 269, 235, 287, 238, 260, 226, 272, 247, 288, 270, 244, 258, 244, 223, 241, 257, 241, 248, 255, 234, 224, 246, 273, 258, 247, 280, 228, 261, 267, 258, 253, 228, 271, 260, 255, 241, 246, 244, 279, 235, 244, 236, 250, 234, 243, 273, 259, 270, 245, 268, 271, 261, 259, 242, 253, 284, 286, 257, 294, 245, 249, 256, 279, 236, 215, 243, 279, 265, 260, 239, 254, 270, 261, 251, 257, 242, 249, 257, 269, 286, 246, 253, 245, 259, 254, 253, 251, 258, 251, 270, 252, 260, 245, 258, 258, 254, 285, 270, 251, 261, 259, 266, 236, 247, 261, 271, 255, 273, 255, 274, 255, 274, 261, 256, 267, 260, 265, 248, 266, 267, 245, 270, 259] payload_rows_per_dst=[5488, 5312, 5340, 5580, 5474, 5434, 5578, 5570] dup_rows_per_dst=[2708, 2602, 2716, 2818, 2644, 2756, 2696, 2820] combine_rows_per_src=[5488, 5312, 5340, 5580, 5474, 5434, 5578, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38206 dup_rows=21896 dup_rows_remote=19280 dedup_ratio=0.334106 router_tokens_per_expert=[227, 238, 256, 261, 260, 283, 255, 260, 246, 255, 249, 272, 264, 247, 251, 256, 258, 272, 257, 262, 270, 226, 239, 256, 269, 278, 268, 254, 261, 283, 254, 242, 241, 270, 251, 259, 233, 264, 282, 250, 258, 236, 231, 255, 263, 244, 256, 277, 253, 239, 261, 243, 242, 261, 267, 234, 243, 250, 266, 277, 274, 235, 246, 234, 256, 248, 262, 239, 249, 240, 281, 250, 253, 259, 270, 274, 268, 244, 275, 233, 264, 242, 234, 257, 274, 249, 244, 246, 276, 277, 256, 253, 242, 235, 249, 264, 251, 249, 271, 253, 258, 251, 255, 236, 253, 283, 266, 258, 242, 277, 239, 260, 234, 244, 263, 240, 260, 249, 285, 269, 243, 253, 290, 266, 278, 286, 259, 260] payload_rows_per_dst=[5398, 5476, 5430, 5434, 5500, 5462, 5410, 5530] dup_rows_per_dst=[2762, 2822, 2710, 2616, 2702, 2662, 2794, 2828] combine_rows_per_src=[5398, 5476, 5430, 5434, 5500, 5462, 5410, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38146 dup_rows=21824 dup_rows_remote=18992 dedup_ratio=0.333008 router_tokens_per_expert=[245, 294, 239, 266, 242, 246, 276, 229, 258, 265, 237, 252, 230, 252, 240, 243, 272, 248, 256, 262, 259, 265, 241, 306, 257, 262, 243, 248, 234, 251, 265, 246, 248, 257, 250, 259, 235, 265, 276, 234, 263, 251, 251, 250, 284, 245, 253, 231, 246, 244, 233, 274, 244, 281, 231, 254, 255, 252, 292, 254, 279, 238, 259, 250, 275, 251, 228, 264, 288, 238, 249, 254, 262, 247, 264, 267, 249, 229, 276, 247, 286, 265, 235, 253, 289, 280, 258, 261, 261, 275, 248, 283, 284, 255, 261, 257, 270, 239, 232, 277, 250, 250, 261, 232, 262, 249, 268, 234, 263, 256, 252, 251, 260, 251, 266, 258, 271, 270, 276, 271, 266, 241, 242, 251, 254, 245, 248, 246] payload_rows_per_dst=[5354, 5348, 5378, 5462, 5668, 5322, 5614, 5566] dup_rows_per_dst=[2674, 2346, 2762, 2710, 3008, 2454, 3038, 2832] combine_rows_per_src=[5354, 5348, 5378, 5462, 5668, 5322, 5614, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229216 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156942176 ingress_bytes=213336064 link_bytes=213336064 token_bytes=313655136 weight_movement_bytes=56623104 total_nvlink_bytes=370278240 weight_to_token_ratio=0.180527 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044081197e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38048 dup_rows=22040 dup_rows_remote=19310 dedup_ratio=0.336304 router_tokens_per_expert=[274, 261, 286, 267, 239, 260, 261, 247, 276, 238, 229, 257, 259, 252, 222, 261, 246, 261, 278, 268, 242, 249, 235, 269, 275, 278, 230, 245, 255, 270, 263, 249, 286, 251, 252, 251, 237, 236, 253, 223, 227, 242, 259, 242, 282, 259, 282, 258, 262, 267, 274, 249, 235, 243, 272, 288, 275, 240, 268, 244, 254, 265, 253, 251, 238, 236, 254, 254, 227, 253, 264, 253, 247, 244, 251, 266, 299, 267, 249, 274, 260, 235, 263, 272, 249, 288, 241, 255, 273, 238, 250, 256, 287, 227, 257, 241, 286, 225, 273, 263, 224, 248, 282, 262, 258, 248, 267, 237, 262, 244, 251, 229, 247, 280, 236, 273, 246, 237, 260, 237, 279, 262, 282, 249, 250, 269, 266, 286] payload_rows_per_dst=[5448, 5570, 5386, 5432, 5254, 5550, 5336, 5520] dup_rows_per_dst=[2730, 3020, 2694, 2848, 2534, 2904, 2512, 2798] combine_rows_per_src=[5448, 5570, 5386, 5432, 5254, 5550, 5336, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=229736 combine_payload_bytes=156409856 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156639592 ingress_bytes=175284224 link_bytes=175284224 token_bytes=313049448 weight_movement_bytes=18874368 total_nvlink_bytes=331923816 weight_to_token_ratio=0.060292 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679487189e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229384 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157278216 ingress_bytes=194797568 link_bytes=194797568 token_bytes=314327048 weight_movement_bytes=37748736 total_nvlink_bytes=352075784 weight_to_token_ratio=0.120094 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866454450e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156491776 dispatch_meta_bytes=229944 combine_payload_bytes=156491776 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156721720 ingress_bytes=175366144 link_bytes=175366144 token_bytes=313213496 weight_movement_bytes=18874368 total_nvlink_bytes=332087864 weight_to_token_ratio=0.060260 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680272106e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=228552 combine_payload_bytes=156246016 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156474568 ingress_bytes=193994752 link_bytes=193994752 token_bytes=312720584 weight_movement_bytes=37748736 total_nvlink_bytes=350469320 weight_to_token_ratio=0.120711 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858762262e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38200 dup_rows=21816 dup_rows_remote=18942 dedup_ratio=0.332886 router_tokens_per_expert=[257, 264, 278, 273, 270, 227, 252, 272, 260, 235, 280, 251, 249, 256, 268, 221, 286, 259, 244, 273, 268, 261, 271, 268, 270, 252, 275, 264, 260, 248, 259, 239, 253, 259, 219, 235, 232, 251, 275, 220, 249, 239, 269, 241, 271, 264, 228, 234, 276, 255, 258, 242, 259, 275, 255, 240, 276, 238, 221, 233, 281, 255, 250, 261, 241, 263, 248, 246, 252, 244, 248, 258, 278, 257, 251, 259, 266, 245, 252, 265, 282, 240, 278, 262, 237, 263, 279, 284, 267, 276, 247, 237, 266, 251, 249, 294, 250, 233, 250, 258, 254, 245, 261, 250, 213, 300, 246, 276, 249, 250, 270, 257, 243, 275, 255, 258, 261, 264, 261, 247, 255, 258, 254, 256, 264, 257, 248, 241] payload_rows_per_dst=[5470, 5520, 5366, 5460, 5486, 5604, 5356, 5458] dup_rows_per_dst=[2756, 2874, 2512, 2690, 2660, 2820, 2768, 2736] combine_rows_per_src=[5470, 5520, 5366, 5460, 5486, 5604, 5356, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43502 payload_rows_remote=38062 dup_rows=22034 dup_rows_remote=19296 dedup_ratio=0.336212 router_tokens_per_expert=[255, 271, 257, 249, 295, 256, 267, 241, 280, 251, 241, 239, 270, 251, 244, 244, 261, 269, 263, 242, 255, 269, 263, 283, 245, 265, 261, 267, 279, 251, 256, 280, 273, 245, 263, 277, 288, 277, 249, 274, 272, 255, 271, 252, 239, 238, 242, 246, 250, 269, 272, 242, 255, 281, 229, 225, 260, 249, 238, 248, 226, 245, 228, 266, 265, 255, 282, 268, 291, 279, 248, 254, 222, 225, 248, 253, 235, 277, 232, 255, 251, 261, 257, 252, 246, 250, 233, 260, 246, 247, 259, 251, 258, 235, 277, 250, 274, 267, 264, 255, 241, 262, 241, 246, 258, 246, 252, 243, 263, 294, 281, 252, 240, 245, 250, 230, 257, 270, 280, 258, 253, 247, 245, 260, 261, 238, 239, 270] payload_rows_per_dst=[5446, 5496, 5544, 5364, 5440, 5398, 5484, 5330] dup_rows_per_dst=[2776, 2922, 2778, 2602, 2738, 2668, 2794, 2756] combine_rows_per_src=[5446, 5496, 5544, 5364, 5440, 5398, 5484, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=28 plans=1 dispatch_payload_bytes=155844608 dispatch_meta_bytes=229432 combine_payload_bytes=155844608 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156074040 ingress_bytes=250216448 link_bytes=250216448 token_bytes=311918648 weight_movement_bytes=94371840 total_nvlink_bytes=406290488 weight_to_token_ratio=0.302553 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.397450891e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38342 dup_rows=21714 dup_rows_remote=18936 dedup_ratio=0.331329 router_tokens_per_expert=[274, 241, 272, 254, 243, 271, 275, 267, 243, 263, 251, 226, 252, 261, 277, 256, 261, 282, 266, 260, 274, 271, 238, 256, 275, 233, 259, 246, 268, 259, 276, 250, 240, 224, 247, 273, 257, 244, 255, 273, 265, 260, 254, 290, 267, 221, 239, 254, 229, 276, 243, 251, 259, 266, 247, 262, 263, 223, 248, 247, 239, 266, 259, 286, 273, 257, 256, 235, 252, 268, 252, 253, 235, 276, 258, 262, 251, 261, 262, 221, 237, 267, 249, 243, 245, 264, 255, 272, 227, 264, 233, 273, 268, 258, 256, 242, 281, 246, 252, 244, 277, 262, 240, 243, 269, 267, 248, 233, 247, 256, 271, 255, 267, 252, 249, 226, 247, 240, 262, 265, 268, 268, 256, 247, 274, 294, 258, 252] payload_rows_per_dst=[5546, 5522, 5514, 5456, 5376, 5418, 5480, 5510] dup_rows_per_dst=[2706, 2826, 2612, 2672, 2768, 2612, 2778, 2740] combine_rows_per_src=[5546, 5522, 5514, 5456, 5376, 5418, 5480, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=228568 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156695768 ingress_bytes=231964672 link_bytes=231964672 token_bytes=313162968 weight_movement_bytes=75497472 total_nvlink_bytes=388660440 weight_to_token_ratio=0.241080 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222571353e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=155901952 dispatch_meta_bytes=229432 combine_payload_bytes=155901952 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156131384 ingress_bytes=174776320 link_bytes=174776320 token_bytes=312033336 weight_movement_bytes=18874368 total_nvlink_bytes=330907704 weight_to_token_ratio=0.060488 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674620703e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38344 dup_rows=21746 dup_rows_remote=18992 dedup_ratio=0.331818 router_tokens_per_expert=[254, 225, 243, 244, 270, 237, 249, 270, 251, 283, 290, 244, 261, 239, 265, 256, 253, 249, 264, 267, 259, 285, 301, 260, 241, 265, 228, 238, 230, 247, 264, 275, 274, 252, 231, 283, 243, 261, 249, 255, 238, 256, 256, 270, 272, 247, 258, 255, 243, 266, 242, 248, 257, 252, 258, 248, 233, 250, 258, 254, 241, 239, 268, 247, 268, 231, 273, 272, 265, 280, 235, 237, 289, 245, 262, 265, 257, 266, 249, 241, 285, 265, 258, 255, 268, 267, 242, 248, 236, 246, 265, 264, 270, 259, 255, 251, 236, 256, 250, 243, 266, 252, 227, 245, 231, 253, 266, 239, 258, 258, 280, 266, 275, 250, 240, 252, 250, 274, 247, 266, 256, 257, 254, 267, 255, 279, 294, 246] payload_rows_per_dst=[5492, 5472, 5446, 5362, 5496, 5572, 5472, 5478] dup_rows_per_dst=[2670, 2780, 2754, 2646, 2774, 2696, 2580, 2846] combine_rows_per_src=[5492, 5472, 5446, 5362, 5496, 5572, 5472, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38306 dup_rows=21794 dup_rows_remote=18876 dedup_ratio=0.332550 router_tokens_per_expert=[270, 269, 250, 253, 278, 275, 237, 270, 248, 259, 257, 278, 251, 225, 231, 258, 278, 263, 249, 258, 220, 250, 243, 257, 255, 243, 242, 209, 243, 247, 251, 256, 264, 266, 226, 233, 269, 250, 256, 269, 251, 268, 246, 270, 232, 264, 283, 269, 256, 254, 246, 258, 270, 258, 251, 262, 239, 283, 256, 235, 280, 243, 260, 262, 270, 277, 265, 283, 228, 227, 241, 240, 247, 248, 262, 234, 261, 284, 253, 242, 254, 270, 243, 256, 261, 285, 253, 245, 262, 248, 295, 274, 271, 262, 252, 246, 239, 251, 238, 257, 272, 275, 235, 246, 273, 234, 257, 250, 268, 252, 254, 253, 245, 263, 253, 267, 270, 270, 266, 278, 239, 236, 283, 237, 281, 243, 256, 286] payload_rows_per_dst=[5490, 5342, 5504, 5460, 5450, 5436, 5400, 5660] dup_rows_per_dst=[2728, 2586, 2728, 2766, 2674, 2918, 2708, 2686] combine_rows_per_src=[5490, 5342, 5504, 5460, 5450, 5436, 5400, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229112 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157277944 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314326776 weight_movement_bytes=18874368 total_nvlink_bytes=333201144 weight_to_token_ratio=0.060047 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43718 payload_rows_remote=38226 dup_rows=21818 dup_rows_remote=19128 dedup_ratio=0.332916 router_tokens_per_expert=[264, 243, 274, 245, 257, 263, 270, 264, 256, 237, 257, 248, 238, 252, 220, 260, 272, 252, 243, 272, 271, 262, 251, 254, 279, 249, 257, 294, 244, 242, 259, 245, 232, 245, 247, 277, 246, 263, 267, 256, 247, 283, 259, 237, 281, 283, 251, 261, 268, 285, 234, 256, 238, 248, 260, 241, 267, 246, 232, 234, 271, 264, 267, 280, 262, 261, 278, 265, 255, 250, 242, 268, 251, 234, 254, 268, 239, 241, 247, 231, 282, 265, 267, 247, 276, 275, 264, 251, 267, 267, 239, 249, 251, 243, 265, 258, 259, 246, 240, 243, 255, 244, 261, 234, 260, 241, 257, 289, 250, 243, 271, 222, 237, 262, 259, 282, 263, 278, 250, 247, 270, 266, 260, 239, 245, 269, 239, 255] payload_rows_per_dst=[5402, 5464, 5486, 5492, 5422, 5534, 5440, 5478] dup_rows_per_dst=[2694, 2828, 2784, 2690, 2670, 2798, 2590, 2764] combine_rows_per_src=[5402, 5464, 5486, 5492, 5422, 5534, 5440, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38204 dup_rows=21916 dup_rows_remote=19228 dedup_ratio=0.334412 router_tokens_per_expert=[259, 246, 252, 229, 263, 254, 263, 238, 229, 252, 247, 265, 274, 251, 265, 255, 261, 268, 270, 281, 282, 215, 244, 263, 256, 269, 256, 257, 259, 246, 241, 252, 260, 247, 255, 237, 251, 255, 219, 236, 242, 253, 263, 263, 297, 254, 237, 248, 259, 258, 260, 258, 270, 225, 267, 279, 291, 241, 252, 264, 261, 244, 238, 270, 249, 288, 261, 278, 264, 273, 257, 249, 298, 264, 257, 248, 252, 274, 264, 263, 249, 255, 237, 268, 269, 251, 269, 276, 239, 275, 250, 259, 254, 253, 263, 263, 240, 234, 266, 269, 261, 262, 257, 273, 255, 234, 254, 248, 247, 262, 232, 254, 275, 264, 247, 232, 248, 260, 243, 246, 273, 234, 244, 257, 279, 255, 244, 234] payload_rows_per_dst=[5386, 5442, 5400, 5454, 5424, 5562, 5536, 5416] dup_rows_per_dst=[2664, 2798, 2634, 2634, 2782, 2884, 2832, 2688] combine_rows_per_src=[5386, 5442, 5400, 5454, 5424, 5562, 5536, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157057024 dispatch_meta_bytes=229344 combine_payload_bytes=157057024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157286368 ingress_bytes=175931392 link_bytes=175931392 token_bytes=314343392 weight_movement_bytes=18874368 total_nvlink_bytes=333217760 weight_to_token_ratio=0.060044 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685688034e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43422 payload_rows_remote=38012 dup_rows=22114 dup_rows_remote=19350 dedup_ratio=0.337433 router_tokens_per_expert=[271, 254, 240, 233, 247, 251, 256, 276, 275, 263, 255, 230, 256, 252, 229, 298, 249, 267, 256, 233, 254, 242, 227, 224, 256, 250, 265, 267, 265, 257, 263, 272, 267, 275, 272, 270, 257, 256, 242, 274, 237, 246, 249, 258, 249, 250, 261, 271, 276, 264, 247, 238, 261, 267, 248, 233, 250, 246, 249, 250, 223, 280, 271, 258, 255, 267, 233, 237, 255, 284, 272, 256, 250, 265, 246, 241, 264, 278, 240, 241, 241, 253, 268, 248, 251, 225, 240, 257, 234, 268, 266, 243, 300, 284, 276, 276, 267, 255, 259, 252, 259, 247, 256, 257, 264, 283, 237, 247, 247, 269, 241, 260, 282, 261, 267, 242, 254, 256, 267, 252, 263, 222, 237, 264, 255, 290, 255, 259] payload_rows_per_dst=[5410, 5358, 5564, 5362, 5632, 5484, 5378, 5234] dup_rows_per_dst=[2764, 2734, 2958, 2760, 3058, 2776, 2568, 2496] combine_rows_per_src=[5410, 5358, 5564, 5362, 5632, 5484, 5378, 5234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156901376 dispatch_meta_bytes=228728 combine_payload_bytes=156901376 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157130104 ingress_bytes=213524480 link_bytes=213524480 token_bytes=314031480 weight_movement_bytes=56623104 total_nvlink_bytes=370654584 weight_to_token_ratio=0.180310 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045886507e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229416 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803112 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376808 weight_movement_bytes=18874368 total_nvlink_bytes=332251176 weight_to_token_ratio=0.060229 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=229728 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156713312 ingress_bytes=175357952 link_bytes=175357952 token_bytes=313196896 weight_movement_bytes=18874368 total_nvlink_bytes=332071264 weight_to_token_ratio=0.060264 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680193615e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43538 payload_rows_remote=38182 dup_rows=21998 dup_rows_remote=19364 dedup_ratio=0.335663 router_tokens_per_expert=[234, 240, 277, 292, 272, 265, 251, 252, 278, 292, 232, 258, 268, 260, 258, 266, 248, 249, 272, 237, 251, 257, 232, 278, 227, 253, 223, 260, 250, 230, 259, 269, 250, 249, 250, 270, 241, 276, 257, 276, 283, 279, 248, 280, 252, 240, 244, 241, 256, 280, 259, 255, 251, 241, 255, 236, 280, 220, 277, 255, 233, 283, 261, 269, 240, 271, 250, 271, 247, 268, 251, 253, 250, 258, 252, 240, 222, 259, 272, 242, 271, 260, 242, 264, 248, 237, 267, 242, 272, 297, 249, 250, 259, 253, 261, 240, 270, 280, 252, 263, 242, 264, 242, 253, 263, 235, 234, 281, 246, 251, 256, 240, 261, 280, 258, 254, 248, 254, 252, 275, 244, 263, 257, 234, 257, 274, 248, 242] payload_rows_per_dst=[5506, 5356, 5518, 5462, 5400, 5438, 5410, 5448] dup_rows_per_dst=[2884, 2634, 2754, 2760, 2692, 2786, 2734, 2754] combine_rows_per_src=[5506, 5356, 5518, 5462, 5400, 5438, 5410, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43882 payload_rows_remote=38486 dup_rows=21654 dup_rows_remote=18858 dedup_ratio=0.330414 router_tokens_per_expert=[241, 261, 248, 264, 247, 266, 248, 244, 229, 264, 245, 257, 254, 256, 261, 265, 247, 263, 266, 288, 242, 266, 260, 227, 236, 284, 258, 265, 261, 278, 277, 244, 257, 279, 270, 267, 270, 268, 247, 272, 246, 237, 263, 251, 227, 240, 232, 254, 257, 273, 262, 251, 248, 287, 238, 251, 243, 269, 253, 255, 292, 253, 253, 247, 235, 255, 227, 269, 256, 254, 278, 280, 245, 284, 231, 266, 263, 255, 229, 269, 265, 271, 279, 281, 250, 267, 245, 275, 233, 256, 275, 220, 244, 245, 247, 233, 249, 247, 255, 267, 236, 270, 253, 239, 244, 250, 268, 258, 254, 216, 258, 237, 246, 271, 286, 244, 256, 256, 253, 255, 291, 278, 257, 274, 239, 238, 258, 259] payload_rows_per_dst=[5508, 5512, 5470, 5496, 5396, 5516, 5444, 5540] dup_rows_per_dst=[2592, 2812, 2690, 2768, 2796, 2656, 2558, 2782] combine_rows_per_src=[5508, 5512, 5470, 5496, 5396, 5516, 5444, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=29 plans=1 dispatch_payload_bytes=155697152 dispatch_meta_bytes=229448 combine_payload_bytes=155697152 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155926600 ingress_bytes=193445888 link_bytes=193445888 token_bytes=311623752 weight_movement_bytes=37748736 total_nvlink_bytes=349372488 weight_to_token_ratio=0.121136 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.853503317e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43906 payload_rows_remote=38398 dup_rows=21630 dup_rows_remote=18846 dedup_ratio=0.330048 router_tokens_per_expert=[247, 265, 239, 273, 234, 284, 230, 242, 247, 263, 236, 284, 250, 236, 252, 273, 237, 253, 263, 265, 259, 254, 269, 239, 242, 258, 257, 256, 245, 252, 261, 273, 230, 272, 220, 271, 219, 251, 256, 261, 239, 243, 239, 272, 256, 240, 229, 243, 274, 253, 269, 279, 263, 240, 221, 257, 281, 248, 269, 262, 263, 239, 234, 254, 262, 246, 280, 243, 277, 276, 242, 282, 255, 258, 274, 254, 235, 244, 291, 240, 246, 236, 255, 278, 274, 250, 287, 256, 287, 260, 270, 262, 251, 251, 233, 277, 252, 275, 222, 257, 302, 275, 267, 256, 240, 256, 247, 269, 259, 261, 230, 278, 281, 249, 263, 250, 269, 273, 240, 272, 237, 265, 259, 263, 256, 233, 254, 241] payload_rows_per_dst=[5486, 5446, 5332, 5496, 5526, 5562, 5508, 5550] dup_rows_per_dst=[2624, 2720, 2550, 2716, 2792, 2784, 2784, 2660] combine_rows_per_src=[5486, 5446, 5332, 5496, 5526, 5562, 5508, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=230184 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156623656 ingress_bytes=175267840 link_bytes=175267840 token_bytes=313017128 weight_movement_bytes=18874368 total_nvlink_bytes=331891496 weight_to_token_ratio=0.060298 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679330206e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157638656 dispatch_meta_bytes=229376 combine_payload_bytes=157638656 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157868032 ingress_bytes=214261760 link_bytes=214261760 token_bytes=315506688 weight_movement_bytes=56623104 total_nvlink_bytes=372129792 weight_to_token_ratio=0.179467 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.052950761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43716 payload_rows_remote=38188 dup_rows=21820 dup_rows_remote=19018 dedup_ratio=0.332947 router_tokens_per_expert=[242, 219, 257, 238, 265, 245, 265, 224, 261, 237, 240, 228, 271, 237, 259, 259, 260, 236, 251, 259, 280, 282, 260, 291, 290, 251, 275, 267, 243, 274, 251, 283, 253, 267, 282, 278, 264, 273, 226, 283, 259, 251, 248, 235, 272, 272, 258, 244, 231, 256, 264, 260, 238, 268, 279, 278, 258, 231, 280, 273, 254, 251, 257, 264, 259, 245, 261, 248, 229, 260, 248, 257, 234, 265, 243, 263, 251, 266, 264, 244, 261, 247, 264, 259, 238, 212, 258, 226, 248, 214, 241, 273, 267, 255, 250, 272, 263, 252, 261, 252, 240, 249, 270, 275, 255, 254, 310, 260, 243, 289, 247, 246, 252, 265, 249, 228, 268, 253, 228, 252, 264, 244, 262, 240, 272, 250, 269, 277] payload_rows_per_dst=[5314, 5526, 5528, 5534, 5418, 5384, 5558, 5454] dup_rows_per_dst=[2580, 2980, 2802, 2750, 2656, 2586, 2774, 2692] combine_rows_per_src=[5314, 5526, 5528, 5534, 5418, 5384, 5558, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43622 payload_rows_remote=38102 dup_rows=21914 dup_rows_remote=19226 dedup_ratio=0.334381 router_tokens_per_expert=[304, 270, 257, 248, 272, 255, 285, 255, 241, 258, 238, 266, 273, 239, 263, 264, 249, 254, 239, 233, 270, 247, 245, 277, 264, 255, 228, 254, 244, 224, 237, 246, 262, 293, 252, 275, 257, 249, 256, 255, 276, 271, 248, 272, 256, 248, 243, 235, 236, 247, 259, 284, 259, 247, 268, 248, 260, 267, 269, 250, 269, 281, 266, 252, 266, 251, 241, 257, 269, 244, 245, 241, 251, 238, 270, 255, 251, 278, 252, 233, 251, 257, 258, 266, 266, 270, 262, 245, 260, 250, 248, 270, 241, 241, 236, 283, 254, 229, 258, 246, 228, 227, 257, 274, 251, 265, 247, 234, 253, 268, 279, 239, 249, 292, 282, 262, 262, 289, 240, 236, 236, 226, 260, 252, 255, 251, 274, 283] payload_rows_per_dst=[5504, 5340, 5484, 5564, 5426, 5520, 5338, 5446] dup_rows_per_dst=[2872, 2592, 2812, 2760, 2658, 2688, 2680, 2852] combine_rows_per_src=[5504, 5340, 5484, 5564, 5426, 5520, 5338, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38284 dup_rows=21716 dup_rows_remote=18984 dedup_ratio=0.331360 router_tokens_per_expert=[264, 239, 254, 250, 238, 261, 246, 254, 261, 265, 270, 237, 287, 241, 245, 247, 266, 262, 250, 255, 257, 295, 248, 257, 247, 237, 257, 255, 268, 261, 252, 225, 255, 245, 262, 239, 276, 243, 246, 261, 221, 253, 250, 260, 238, 260, 227, 256, 253, 269, 231, 273, 255, 243, 267, 230, 262, 255, 277, 271, 260, 257, 266, 265, 253, 259, 276, 274, 242, 263, 256, 257, 255, 261, 249, 280, 263, 261, 293, 245, 233, 247, 249, 280, 266, 255, 240, 249, 261, 255, 287, 262, 247, 254, 252, 246, 253, 271, 243, 270, 252, 248, 251, 266, 256, 273, 280, 260, 267, 248, 261, 249, 236, 251, 262, 288, 256, 247, 224, 248, 247, 242, 275, 257, 243, 287, 244, 266] payload_rows_per_dst=[5474, 5516, 5390, 5536, 5532, 5420, 5512, 5440] dup_rows_per_dst=[2644, 2668, 2594, 2732, 2842, 2746, 2784, 2706] combine_rows_per_src=[5474, 5516, 5390, 5536, 5532, 5420, 5512, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=228976 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507184 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314785392 weight_movement_bytes=18874368 total_nvlink_bytes=333659760 weight_to_token_ratio=0.059959 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38184 dup_rows=21890 dup_rows_remote=19168 dedup_ratio=0.334015 router_tokens_per_expert=[249, 239, 252, 249, 235, 263, 263, 251, 246, 255, 240, 274, 253, 245, 252, 251, 251, 273, 241, 256, 250, 241, 230, 266, 238, 276, 253, 248, 288, 239, 270, 261, 249, 267, 277, 282, 258, 254, 251, 263, 264, 250, 273, 232, 239, 273, 252, 261, 253, 254, 255, 246, 233, 272, 248, 257, 277, 249, 263, 254, 232, 269, 247, 260, 237, 240, 261, 270, 261, 242, 278, 248, 284, 274, 278, 241, 259, 247, 247, 261, 269, 292, 288, 262, 252, 237, 271, 266, 280, 279, 233, 239, 238, 242, 248, 225, 239, 256, 271, 258, 248, 253, 269, 240, 258, 263, 287, 225, 271, 290, 250, 237, 264, 257, 242, 275, 263, 270, 275, 240, 257, 225, 260, 260, 278, 245, 242, 239] payload_rows_per_dst=[5248, 5394, 5478, 5422, 5514, 5556, 5572, 5462] dup_rows_per_dst=[2474, 2834, 2812, 2716, 2742, 2730, 2860, 2722] combine_rows_per_src=[5248, 5394, 5478, 5422, 5514, 5556, 5572, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156418048 dispatch_meta_bytes=228824 combine_payload_bytes=156418048 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156646872 ingress_bytes=175292416 link_bytes=175292416 token_bytes=313064920 weight_movement_bytes=18874368 total_nvlink_bytes=331939288 weight_to_token_ratio=0.060289 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679565681e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156065792 dispatch_meta_bytes=229312 combine_payload_bytes=156065792 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156295104 ingress_bytes=193814528 link_bytes=193814528 token_bytes=312360896 weight_movement_bytes=37748736 total_nvlink_bytes=350109632 weight_to_token_ratio=0.120850 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.857035445e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156811264 dispatch_meta_bytes=229072 combine_payload_bytes=156811264 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157040336 ingress_bytes=213434368 link_bytes=213434368 token_bytes=313851600 weight_movement_bytes=56623104 total_nvlink_bytes=370474704 weight_to_token_ratio=0.180414 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.045023098e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38234 dup_rows=21768 dup_rows_remote=19074 dedup_ratio=0.332153 router_tokens_per_expert=[252, 256, 232, 251, 283, 264, 286, 227, 268, 276, 255, 270, 246, 263, 246, 239, 231, 265, 265, 268, 281, 239, 245, 254, 243, 256, 262, 245, 266, 258, 257, 255, 244, 251, 234, 255, 247, 258, 259, 241, 273, 273, 277, 263, 275, 255, 283, 249, 228, 256, 238, 248, 258, 264, 269, 259, 249, 258, 262, 274, 241, 252, 252, 240, 246, 263, 272, 224, 227, 237, 249, 280, 263, 261, 247, 271, 260, 248, 250, 240, 263, 260, 247, 255, 267, 288, 266, 235, 241, 249, 278, 253, 272, 247, 253, 270, 269, 260, 257, 273, 278, 250, 281, 254, 263, 244, 248, 259, 244, 270, 254, 246, 244, 248, 249, 251, 258, 248, 267, 244, 235, 259, 243, 267, 262, 272, 243, 257] payload_rows_per_dst=[5534, 5594, 5576, 5510, 5492, 5434, 5344, 5284] dup_rows_per_dst=[2694, 2694, 3026, 2766, 2766, 2750, 2520, 2552] combine_rows_per_src=[5534, 5594, 5576, 5510, 5492, 5434, 5344, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229408 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156631072 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032736 weight_movement_bytes=18874368 total_nvlink_bytes=331907104 weight_to_token_ratio=0.060295 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43810 payload_rows_remote=38278 dup_rows=21726 dup_rows_remote=19112 dedup_ratio=0.331512 router_tokens_per_expert=[279, 271, 293, 279, 251, 263, 272, 263, 228, 274, 247, 279, 245, 295, 269, 262, 219, 251, 256, 277, 262, 230, 253, 258, 263, 255, 269, 290, 232, 246, 249, 263, 256, 258, 268, 234, 246, 246, 268, 254, 228, 239, 252, 232, 245, 278, 258, 271, 252, 253, 248, 237, 246, 255, 250, 250, 224, 242, 257, 235, 244, 238, 276, 237, 239, 253, 211, 275, 230, 269, 250, 265, 256, 241, 247, 253, 248, 251, 235, 255, 261, 264, 251, 259, 249, 247, 256, 273, 251, 257, 259, 249, 271, 235, 289, 247, 261, 262, 236, 275, 249, 248, 262, 272, 267, 255, 248, 262, 277, 253, 274, 260, 254, 253, 235, 286, 281, 281, 241, 283, 252, 250, 275, 288, 286, 233, 245, 248] payload_rows_per_dst=[5612, 5532, 5436, 5316, 5342, 5486, 5508, 5578] dup_rows_per_dst=[2928, 2614, 2630, 2572, 2614, 2750, 2814, 2804] combine_rows_per_src=[5612, 5532, 5436, 5316, 5342, 5486, 5508, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38410 dup_rows=21766 dup_rows_remote=19166 dedup_ratio=0.332123 router_tokens_per_expert=[275, 287, 238, 264, 257, 237, 263, 270, 251, 243, 258, 261, 237, 255, 289, 277, 243, 271, 271, 256, 245, 283, 250, 278, 270, 266, 247, 272, 252, 260, 234, 242, 254, 250, 276, 248, 275, 243, 250, 270, 243, 234, 239, 250, 268, 235, 260, 253, 269, 231, 238, 275, 261, 232, 249, 235, 264, 249, 261, 237, 249, 261, 258, 253, 260, 227, 254, 258, 258, 234, 221, 242, 276, 246, 258, 230, 260, 239, 261, 256, 268, 263, 274, 245, 283, 244, 260, 264, 245, 257, 257, 241, 231, 265, 265, 260, 249, 228, 253, 273, 244, 276, 248, 280, 256, 265, 256, 260, 233, 274, 271, 285, 274, 229, 257, 305, 259, 267, 269, 268, 243, 257, 242, 247, 234, 267, 271, 254] payload_rows_per_dst=[5526, 5578, 5446, 5426, 5360, 5414, 5544, 5476] dup_rows_per_dst=[2798, 2702, 2650, 2618, 2600, 2830, 2758, 2810] combine_rows_per_src=[5526, 5578, 5446, 5426, 5360, 5414, 5544, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=30 plans=1 dispatch_payload_bytes=156606464 dispatch_meta_bytes=229232 combine_payload_bytes=156606464 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156835696 ingress_bytes=194355200 link_bytes=194355200 token_bytes=313442160 weight_movement_bytes=37748736 total_nvlink_bytes=351190896 weight_to_token_ratio=0.120433 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862215898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43796 payload_rows_remote=38330 dup_rows=21740 dup_rows_remote=18994 dedup_ratio=0.331726 router_tokens_per_expert=[260, 265, 271, 247, 275, 258, 240, 249, 259, 293, 235, 223, 295, 234, 262, 263, 254, 252, 248, 253, 258, 232, 236, 252, 271, 251, 251, 258, 266, 233, 253, 238, 253, 250, 237, 264, 235, 265, 233, 260, 236, 238, 264, 255, 236, 245, 272, 246, 250, 271, 269, 258, 263, 278, 254, 280, 273, 258, 269, 245, 241, 288, 278, 252, 231, 243, 246, 263, 275, 270, 247, 227, 250, 268, 270, 249, 264, 257, 272, 255, 278, 253, 246, 253, 247, 259, 248, 294, 275, 245, 265, 222, 288, 264, 257, 246, 221, 262, 258, 264, 245, 270, 256, 248, 278, 255, 252, 249, 256, 258, 242, 292, 277, 261, 245, 248, 259, 265, 248, 273, 254, 258, 248, 251, 259, 252, 274, 212] payload_rows_per_dst=[5444, 5398, 5388, 5536, 5478, 5608, 5466, 5478] dup_rows_per_dst=[2814, 2614, 2590, 2918, 2696, 2672, 2746, 2690] combine_rows_per_src=[5444, 5398, 5388, 5536, 5478, 5608, 5466, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229560 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016248 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313802936 weight_movement_bytes=18874368 total_nvlink_bytes=332677304 weight_to_token_ratio=0.060147 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=230304 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157557664 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314885024 weight_movement_bytes=18874368 total_nvlink_bytes=333759392 weight_to_token_ratio=0.059941 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38342 dup_rows=21774 dup_rows_remote=19116 dedup_ratio=0.332245 router_tokens_per_expert=[241, 254, 254, 269, 249, 253, 274, 256, 250, 244, 268, 259, 281, 249, 223, 240, 245, 282, 270, 253, 252, 268, 272, 247, 218, 258, 264, 252, 281, 253, 259, 281, 237, 239, 254, 242, 244, 263, 238, 247, 255, 273, 249, 272, 256, 264, 255, 251, 263, 242, 245, 254, 229, 279, 260, 269, 269, 234, 247, 242, 246, 264, 264, 227, 240, 227, 267, 249, 232, 266, 264, 237, 260, 281, 271, 247, 269, 255, 268, 251, 244, 272, 239, 282, 277, 254, 236, 264, 241, 253, 270, 260, 261, 295, 262, 245, 280, 242, 245, 237, 267, 254, 241, 265, 264, 233, 255, 281, 253, 267, 242, 270, 280, 227, 253, 258, 245, 253, 255, 231, 276, 285, 242, 252, 274, 274, 276, 260] payload_rows_per_dst=[5452, 5530, 5420, 5420, 5498, 5462, 5524, 5456] dup_rows_per_dst=[2676, 2780, 2658, 2648, 2670, 2848, 2668, 2826] combine_rows_per_src=[5452, 5530, 5420, 5420, 5498, 5462, 5524, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43666 payload_rows_remote=38202 dup_rows=21870 dup_rows_remote=19062 dedup_ratio=0.333710 router_tokens_per_expert=[255, 252, 248, 266, 248, 242, 227, 207, 283, 267, 244, 254, 256, 248, 265, 238, 248, 254, 299, 264, 282, 285, 251, 249, 242, 251, 230, 253, 266, 284, 252, 267, 278, 266, 284, 236, 252, 265, 243, 253, 229, 246, 236, 255, 281, 258, 267, 270, 240, 239, 270, 274, 259, 277, 265, 240, 252, 286, 222, 270, 261, 272, 244, 263, 233, 268, 230, 250, 269, 264, 261, 265, 263, 238, 261, 256, 259, 267, 283, 281, 261, 233, 252, 268, 258, 266, 274, 246, 221, 267, 239, 283, 249, 258, 283, 278, 243, 250, 264, 237, 226, 267, 243, 247, 275, 246, 255, 258, 273, 246, 251, 226, 252, 239, 243, 271, 274, 251, 246, 269, 258, 245, 248, 263, 240, 223, 274, 251] payload_rows_per_dst=[5384, 5546, 5490, 5460, 5450, 5464, 5430, 5442] dup_rows_per_dst=[2616, 2808, 2748, 2808, 2846, 2808, 2584, 2652] combine_rows_per_src=[5384, 5546, 5490, 5460, 5450, 5464, 5430, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43824 payload_rows_remote=38296 dup_rows=21712 dup_rows_remote=18894 dedup_ratio=0.331299 router_tokens_per_expert=[251, 250, 266, 238, 244, 263, 264, 243, 260, 248, 266, 265, 254, 247, 265, 279, 242, 254, 257, 259, 266, 250, 256, 285, 233, 252, 247, 262, 242, 262, 238, 251, 263, 269, 220, 247, 253, 254, 261, 258, 254, 234, 266, 251, 256, 257, 279, 266, 272, 252, 256, 263, 251, 272, 290, 287, 248, 268, 248, 262, 232, 252, 264, 256, 247, 266, 272, 222, 265, 261, 245, 264, 244, 230, 302, 246, 246, 258, 240, 249, 286, 278, 261, 276, 253, 283, 271, 232, 270, 277, 263, 288, 247, 254, 223, 253, 262, 255, 241, 268, 248, 246, 256, 248, 257, 253, 258, 223, 257, 254, 236, 254, 252, 246, 263, 254, 260, 238, 243, 274, 253, 281, 242, 234, 268, 247, 248, 257] payload_rows_per_dst=[5448, 5408, 5450, 5528, 5502, 5620, 5376, 5492] dup_rows_per_dst=[2758, 2704, 2726, 2818, 2612, 2810, 2656, 2628] combine_rows_per_src=[5448, 5408, 5450, 5528, 5502, 5620, 5376, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229296 combine_payload_bytes=156999680 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157228976 ingress_bytes=175874048 link_bytes=175874048 token_bytes=314228656 weight_movement_bytes=18874368 total_nvlink_bytes=333103024 weight_to_token_ratio=0.060066 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685138593e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38098 dup_rows=21876 dup_rows_remote=18994 dedup_ratio=0.333801 router_tokens_per_expert=[267, 257, 241, 241, 253, 251, 273, 267, 256, 257, 250, 259, 259, 263, 255, 241, 249, 238, 249, 286, 253, 257, 251, 280, 282, 274, 231, 258, 233, 268, 254, 251, 268, 276, 280, 230, 252, 244, 216, 252, 278, 257, 255, 259, 220, 256, 265, 276, 266, 245, 245, 282, 241, 257, 271, 263, 277, 244, 262, 277, 276, 243, 291, 248, 288, 260, 265, 259, 241, 260, 230, 253, 259, 248, 275, 266, 231, 266, 250, 264, 231, 232, 251, 263, 263, 268, 252, 271, 258, 243, 259, 259, 245, 206, 266, 263, 245, 246, 274, 250, 244, 271, 241, 281, 273, 230, 239, 279, 262, 264, 266, 263, 261, 273, 223, 265, 275, 233, 243, 289, 259, 231, 259, 248, 241, 244, 236, 239] payload_rows_per_dst=[5456, 5290, 5316, 5414, 5572, 5566, 5484, 5562] dup_rows_per_dst=[2724, 2564, 2446, 2570, 3050, 2868, 2772, 2882] combine_rows_per_src=[5456, 5290, 5316, 5414, 5572, 5566, 5484, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=229832 combine_payload_bytes=157048832 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157278664 ingress_bytes=194797568 link_bytes=194797568 token_bytes=314327496 weight_movement_bytes=37748736 total_nvlink_bytes=352076232 weight_to_token_ratio=0.120094 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866454450e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229056 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156704448 ingress_bytes=231972864 link_bytes=231972864 token_bytes=313179840 weight_movement_bytes=75497472 total_nvlink_bytes=388677312 weight_to_token_ratio=0.241067 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.222649845e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156860416 dispatch_meta_bytes=228760 combine_payload_bytes=156860416 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157089176 ingress_bytes=175734784 link_bytes=175734784 token_bytes=313949592 weight_movement_bytes=18874368 total_nvlink_bytes=332823960 weight_to_token_ratio=0.060119 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683804233e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=228368 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156277776 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312327184 weight_movement_bytes=18874368 total_nvlink_bytes=331201552 weight_to_token_ratio=0.060431 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38278 dup_rows=21826 dup_rows_remote=19138 dedup_ratio=0.333038 router_tokens_per_expert=[268, 245, 276, 244, 240, 235, 243, 255, 226, 261, 292, 251, 262, 252, 259, 235, 239, 258, 250, 281, 274, 246, 260, 258, 252, 276, 268, 257, 259, 247, 221, 237, 238, 255, 282, 292, 268, 284, 272, 266, 246, 253, 279, 242, 270, 256, 275, 236, 239, 240, 258, 278, 236, 258, 257, 250, 285, 258, 260, 272, 255, 262, 278, 243, 238, 273, 253, 261, 260, 241, 231, 261, 243, 276, 248, 267, 229, 261, 262, 214, 255, 277, 265, 261, 260, 258, 250, 252, 258, 265, 249, 287, 256, 227, 256, 254, 268, 238, 238, 228, 276, 241, 247, 258, 239, 257, 271, 261, 248, 261, 250, 254, 258, 266, 261, 270, 251, 271, 233, 228, 261, 271, 263, 269, 255, 253, 245, 260] payload_rows_per_dst=[5432, 5386, 5602, 5590, 5402, 5516, 5378, 5404] dup_rows_per_dst=[2688, 2754, 2826, 3034, 2634, 2738, 2692, 2460] combine_rows_per_src=[5432, 5386, 5602, 5590, 5402, 5516, 5378, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43620 payload_rows_remote=38220 dup_rows=21916 dup_rows_remote=19206 dedup_ratio=0.334412 router_tokens_per_expert=[235, 271, 256, 236, 235, 252, 244, 267, 271, 280, 256, 261, 253, 261, 243, 266, 259, 250, 263, 251, 253, 298, 252, 233, 257, 227, 262, 264, 216, 255, 241, 274, 266, 225, 234, 273, 266, 256, 265, 256, 250, 245, 269, 258, 278, 272, 268, 261, 270, 249, 254, 269, 248, 259, 257, 277, 277, 282, 260, 264, 250, 235, 249, 275, 217, 253, 247, 250, 266, 246, 265, 246, 259, 259, 262, 258, 233, 262, 260, 269, 235, 250, 273, 250, 249, 248, 261, 295, 260, 243, 257, 255, 261, 274, 237, 273, 236, 220, 242, 254, 270, 238, 280, 282, 236, 227, 245, 253, 258, 261, 270, 247, 273, 253, 260, 262, 252, 253, 264, 239, 245, 238, 266, 272, 274, 250, 246, 270] payload_rows_per_dst=[5434, 5400, 5508, 5492, 5402, 5464, 5438, 5482] dup_rows_per_dst=[2740, 2710, 2776, 2858, 2702, 2778, 2600, 2752] combine_rows_per_src=[5434, 5400, 5508, 5492, 5402, 5464, 5438, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43798 payload_rows_remote=38348 dup_rows=21738 dup_rows_remote=19060 dedup_ratio=0.331696 router_tokens_per_expert=[279, 257, 273, 241, 263, 251, 282, 273, 247, 239, 273, 271, 257, 256, 285, 256, 283, 246, 225, 230, 257, 253, 258, 273, 235, 254, 253, 259, 252, 266, 250, 241, 236, 267, 254, 250, 263, 270, 269, 258, 262, 259, 252, 289, 248, 260, 272, 235, 278, 249, 281, 253, 272, 242, 247, 228, 281, 271, 227, 256, 264, 256, 269, 269, 253, 252, 262, 266, 273, 264, 229, 250, 262, 263, 241, 258, 244, 234, 261, 252, 263, 277, 241, 245, 263, 280, 252, 290, 243, 270, 256, 254, 235, 230, 287, 255, 289, 267, 262, 247, 256, 273, 235, 235, 232, 245, 247, 226, 242, 265, 233, 244, 282, 245, 260, 237, 279, 271, 247, 233, 268, 241, 226, 260, 245, 246, 252, 248] payload_rows_per_dst=[5572, 5418, 5470, 5548, 5450, 5508, 5422, 5410] dup_rows_per_dst=[2834, 2652, 2818, 2738, 2678, 2774, 2574, 2670] combine_rows_per_src=[5572, 5418, 5470, 5548, 5450, 5508, 5422, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=31 plans=1 dispatch_payload_bytes=156786688 dispatch_meta_bytes=229664 combine_payload_bytes=156786688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157016352 ingress_bytes=175661056 link_bytes=175661056 token_bytes=313803040 weight_movement_bytes=18874368 total_nvlink_bytes=332677408 weight_to_token_ratio=0.060147 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683097808e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43680 payload_rows_remote=38246 dup_rows=21856 dup_rows_remote=19174 dedup_ratio=0.333496 router_tokens_per_expert=[237, 255, 272, 278, 278, 240, 273, 264, 276, 264, 263, 262, 261, 230, 281, 273, 271, 254, 272, 267, 245, 280, 245, 230, 288, 264, 248, 281, 234, 242, 268, 232, 258, 244, 260, 247, 274, 255, 239, 238, 288, 235, 254, 256, 280, 252, 240, 285, 251, 253, 265, 241, 256, 252, 244, 259, 259, 253, 254, 260, 232, 258, 241, 252, 273, 252, 224, 233, 268, 262, 244, 245, 273, 237, 261, 284, 236, 244, 257, 244, 248, 269, 218, 255, 294, 278, 255, 253, 220, 234, 233, 271, 279, 259, 228, 284, 250, 262, 237, 249, 224, 232, 274, 274, 233, 251, 248, 276, 292, 256, 244, 256, 253, 269, 264, 265, 251, 263, 270, 261, 256, 276, 264, 227, 253, 274, 222, 264] payload_rows_per_dst=[5520, 5474, 5518, 5412, 5368, 5466, 5434, 5488] dup_rows_per_dst=[2894, 2768, 2692, 2648, 2706, 2690, 2682, 2776] combine_rows_per_src=[5520, 5474, 5518, 5412, 5368, 5466, 5434, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229704 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156778824 ingress_bytes=194297856 link_bytes=194297856 token_bytes=313327944 weight_movement_bytes=37748736 total_nvlink_bytes=351076680 weight_to_token_ratio=0.120477 gemm_rows_defined=34304 gemm_rows_charged=36864 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.069444 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861666456e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157073408 dispatch_meta_bytes=229632 combine_payload_bytes=157073408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157303040 ingress_bytes=175947776 link_bytes=175947776 token_bytes=314376448 weight_movement_bytes=18874368 total_nvlink_bytes=333250816 weight_to_token_ratio=0.060037 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685845018e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43738 payload_rows_remote=38302 dup_rows=21798 dup_rows_remote=19030 dedup_ratio=0.332611 router_tokens_per_expert=[260, 257, 256, 248, 261, 263, 250, 260, 258, 258, 286, 246, 277, 252, 257, 264, 278, 255, 257, 266, 261, 256, 260, 228, 245, 236, 228, 263, 268, 225, 254, 248, 291, 262, 249, 255, 238, 272, 246, 243, 250, 271, 265, 241, 255, 258, 257, 249, 259, 279, 249, 254, 252, 282, 271, 260, 235, 238, 240, 236, 278, 260, 249, 252, 253, 267, 263, 274, 266, 263, 260, 281, 266, 251, 221, 208, 264, 242, 248, 241, 282, 255, 241, 271, 260, 237, 257, 258, 251, 285, 256, 259, 240, 236, 245, 244, 249, 280, 274, 279, 253, 264, 290, 272, 257, 271, 250, 236, 242, 266, 255, 251, 257, 277, 257, 246, 255, 257, 249, 284, 255, 244, 271, 256, 238, 242, 214, 255] payload_rows_per_dst=[5502, 5382, 5436, 5534, 5400, 5516, 5518, 5450] dup_rows_per_dst=[2804, 2674, 2768, 2654, 2736, 2638, 2860, 2664] combine_rows_per_src=[5502, 5382, 5436, 5534, 5400, 5516, 5518, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38334 dup_rows=21756 dup_rows_remote=19006 dedup_ratio=0.331970 router_tokens_per_expert=[263, 247, 275, 220, 296, 245, 240, 234, 234, 266, 272, 253, 271, 288, 229, 246, 270, 285, 252, 255, 267, 230, 265, 277, 222, 252, 239, 276, 228, 271, 289, 252, 263, 252, 257, 248, 274, 258, 263, 279, 242, 237, 252, 234, 271, 292, 270, 258, 229, 261, 259, 248, 255, 238, 263, 269, 274, 274, 245, 254, 252, 236, 269, 258, 284, 232, 248, 259, 244, 235, 270, 272, 250, 226, 255, 257, 236, 260, 257, 248, 242, 257, 264, 273, 265, 259, 250, 254, 261, 258, 266, 237, 242, 245, 276, 249, 261, 249, 256, 280, 279, 270, 257, 250, 241, 247, 251, 266, 256, 261, 255, 260, 240, 274, 218, 269, 248, 238, 253, 260, 246, 280, 259, 261, 261, 243, 239, 266] payload_rows_per_dst=[5478, 5466, 5498, 5470, 5474, 5446, 5486, 5462] dup_rows_per_dst=[2680, 2794, 2802, 2698, 2592, 2750, 2792, 2648] combine_rows_per_src=[5478, 5466, 5498, 5470, 5474, 5446, 5486, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38252 dup_rows=21906 dup_rows_remote=19168 dedup_ratio=0.334259 router_tokens_per_expert=[256, 268, 252, 240, 252, 252, 246, 225, 247, 253, 275, 275, 225, 268, 271, 230, 242, 251, 265, 267, 265, 252, 290, 248, 275, 272, 244, 228, 256, 248, 253, 258, 244, 280, 242, 262, 254, 251, 260, 247, 254, 242, 261, 248, 256, 265, 242, 262, 248, 237, 261, 242, 213, 262, 248, 275, 242, 268, 256, 267, 262, 242, 256, 279, 261, 249, 251, 263, 254, 269, 258, 269, 252, 256, 254, 260, 259, 239, 260, 250, 258, 260, 246, 244, 265, 271, 263, 251, 234, 246, 238, 254, 255, 279, 257, 252, 276, 275, 278, 250, 237, 253, 263, 248, 268, 256, 275, 256, 270, 272, 265, 242, 276, 240, 254, 258, 253, 273, 282, 269, 236, 265, 256, 278, 258, 228, 250, 254] payload_rows_per_dst=[5392, 5454, 5488, 5378, 5474, 5426, 5554, 5464] dup_rows_per_dst=[2678, 2774, 2652, 2738, 2734, 2720, 2814, 2796] combine_rows_per_src=[5392, 5454, 5488, 5378, 5474, 5426, 5554, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38120 dup_rows=21826 dup_rows_remote=19006 dedup_ratio=0.333038 router_tokens_per_expert=[262, 267, 247, 253, 264, 239, 264, 248, 261, 240, 253, 253, 266, 245, 222, 250, 258, 232, 276, 260, 260, 253, 252, 240, 234, 263, 254, 266, 260, 244, 270, 232, 283, 261, 255, 249, 247, 245, 238, 278, 243, 256, 292, 255, 254, 276, 245, 248, 258, 261, 265, 284, 258, 284, 258, 259, 243, 249, 267, 253, 256, 256, 254, 252, 263, 259, 268, 239, 238, 217, 268, 249, 260, 252, 248, 250, 271, 267, 235, 252, 253, 255, 287, 240, 298, 237, 261, 264, 257, 237, 262, 248, 244, 258, 263, 256, 264, 241, 250, 286, 248, 272, 246, 240, 259, 308, 232, 273, 258, 276, 273, 288, 237, 260, 260, 255, 245, 266, 243, 248, 254, 220, 274, 263, 238, 243, 253, 269] payload_rows_per_dst=[5316, 5306, 5414, 5544, 5512, 5404, 5624, 5590] dup_rows_per_dst=[2752, 2584, 2792, 2770, 2778, 2482, 2848, 2820] combine_rows_per_src=[5316, 5306, 5414, 5544, 5512, 5404, 5624, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229680 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156885296 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540912 weight_movement_bytes=37748736 total_nvlink_bytes=351289648 weight_to_token_ratio=0.120395 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229328 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114320 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999312 weight_movement_bytes=18874368 total_nvlink_bytes=332873680 weight_to_token_ratio=0.060110 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229360 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245424 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261488 weight_movement_bytes=18874368 total_nvlink_bytes=333135856 weight_to_token_ratio=0.060059 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229680 combine_payload_bytes=156680192 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156909872 ingress_bytes=175554560 link_bytes=175554560 token_bytes=313590064 weight_movement_bytes=18874368 total_nvlink_bytes=332464432 weight_to_token_ratio=0.060188 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682077416e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156139520 dispatch_meta_bytes=228504 combine_payload_bytes=156139520 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156368024 ingress_bytes=175013888 link_bytes=175013888 token_bytes=312507544 weight_movement_bytes=18874368 total_nvlink_bytes=331381912 weight_to_token_ratio=0.060397 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676896963e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43720 payload_rows_remote=38136 dup_rows=21816 dup_rows_remote=19110 dedup_ratio=0.332886 router_tokens_per_expert=[276, 290, 257, 258, 268, 249, 226, 274, 224, 232, 257, 261, 266, 273, 284, 246, 288, 260, 247, 248, 249, 246, 260, 249, 245, 263, 298, 283, 252, 258, 255, 269, 250, 251, 244, 259, 262, 266, 289, 235, 273, 261, 263, 253, 253, 256, 231, 287, 261, 255, 280, 245, 257, 253, 222, 282, 245, 259, 230, 264, 248, 262, 248, 222, 249, 262, 252, 255, 257, 262, 252, 272, 242, 255, 273, 252, 255, 252, 244, 248, 257, 232, 253, 242, 259, 234, 279, 255, 254, 237, 293, 260, 249, 274, 255, 269, 266, 251, 253, 274, 261, 251, 234, 243, 281, 246, 234, 268, 259, 266, 271, 262, 261, 250, 236, 274, 269, 230, 204, 262, 244, 242, 269, 235, 267, 241, 264, 239] payload_rows_per_dst=[5584, 5530, 5758, 5366, 5416, 5128, 5554, 5384] dup_rows_per_dst=[2706, 2972, 3048, 2602, 2676, 2536, 2686, 2590] combine_rows_per_src=[5584, 5530, 5758, 5366, 5416, 5128, 5554, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38340 dup_rows=21730 dup_rows_remote=18888 dedup_ratio=0.331573 router_tokens_per_expert=[246, 236, 248, 231, 239, 229, 236, 226, 248, 272, 252, 228, 250, 276, 262, 257, 266, 266, 257, 249, 261, 251, 246, 231, 270, 247, 280, 250, 288, 273, 277, 242, 263, 259, 249, 266, 257, 247, 273, 286, 264, 262, 279, 237, 292, 264, 258, 269, 276, 235, 249, 267, 258, 260, 245, 235, 249, 252, 251, 271, 251, 238, 243, 263, 238, 266, 265, 246, 261, 239, 235, 236, 267, 259, 269, 256, 260, 238, 282, 239, 250, 248, 279, 262, 242, 277, 239, 281, 290, 282, 271, 243, 251, 269, 264, 243, 231, 255, 264, 240, 226, 246, 268, 266, 278, 258, 252, 268, 256, 254, 262, 229, 265, 259, 255, 227, 271, 269, 244, 257, 249, 277, 269, 237, 264, 240, 291, 236] payload_rows_per_dst=[5340, 5466, 5564, 5466, 5410, 5640, 5404, 5516] dup_rows_per_dst=[2532, 2842, 2886, 2620, 2702, 2742, 2702, 2704] combine_rows_per_src=[5340, 5466, 5564, 5466, 5410, 5640, 5404, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38214 dup_rows=21952 dup_rows_remote=19246 dedup_ratio=0.334961 router_tokens_per_expert=[277, 275, 260, 281, 259, 251, 239, 245, 280, 235, 237, 263, 279, 259, 283, 278, 255, 233, 232, 261, 255, 259, 260, 258, 257, 293, 206, 249, 288, 270, 270, 263, 249, 268, 254, 260, 242, 257, 274, 247, 244, 273, 254, 263, 263, 247, 270, 264, 280, 255, 270, 237, 271, 225, 248, 237, 231, 254, 271, 243, 294, 230, 244, 261, 246, 244, 273, 242, 248, 268, 257, 288, 242, 286, 256, 227, 248, 242, 238, 233, 282, 282, 256, 301, 251, 259, 246, 257, 265, 239, 279, 254, 280, 264, 253, 277, 273, 235, 270, 220, 252, 272, 260, 229, 260, 230, 240, 275, 225, 225, 248, 223, 247, 256, 256, 241, 251, 236, 254, 258, 246, 265, 254, 299, 234, 258, 239, 264] payload_rows_per_dst=[5574, 5458, 5526, 5444, 5370, 5530, 5290, 5392] dup_rows_per_dst=[2828, 2760, 2732, 2658, 2706, 2960, 2584, 2724] combine_rows_per_src=[5574, 5458, 5526, 5444, 5370, 5530, 5290, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=32 plans=1 dispatch_payload_bytes=156205056 dispatch_meta_bytes=228984 combine_payload_bytes=156205056 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156434040 ingress_bytes=193953792 link_bytes=193953792 token_bytes=312639096 weight_movement_bytes=37748736 total_nvlink_bytes=350387832 weight_to_token_ratio=0.120742 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858369804e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=228912 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157269552 ingress_bytes=194789376 link_bytes=194789376 token_bytes=314310192 weight_movement_bytes=37748736 total_nvlink_bytes=352058928 weight_to_token_ratio=0.120100 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.866375958e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=229840 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156754384 ingress_bytes=175398912 link_bytes=175398912 token_bytes=313278928 weight_movement_bytes=18874368 total_nvlink_bytes=332153296 weight_to_token_ratio=0.060248 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680586073e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44034 payload_rows_remote=38462 dup_rows=21502 dup_rows_remote=18766 dedup_ratio=0.328094 router_tokens_per_expert=[258, 247, 256, 253, 272, 249, 280, 247, 244, 253, 241, 246, 226, 261, 260, 259, 241, 249, 252, 258, 222, 262, 273, 249, 244, 274, 259, 263, 249, 228, 259, 247, 261, 243, 229, 261, 222, 237, 245, 265, 269, 264, 263, 249, 280, 255, 264, 236, 250, 258, 249, 255, 257, 275, 259, 259, 265, 262, 259, 229, 246, 275, 280, 254, 250, 248, 254, 278, 278, 224, 258, 275, 243, 261, 285, 279, 252, 274, 234, 253, 276, 271, 253, 248, 260, 264, 258, 240, 265, 266, 257, 230, 264, 273, 256, 275, 290, 269, 270, 274, 247, 249, 243, 250, 247, 265, 275, 263, 251, 251, 234, 276, 278, 238, 251, 260, 256, 248, 266, 253, 261, 245, 240, 263, 233, 232, 262, 270] payload_rows_per_dst=[5394, 5438, 5426, 5598, 5542, 5610, 5572, 5454] dup_rows_per_dst=[2710, 2620, 2660, 2666, 2750, 2702, 2736, 2658] combine_rows_per_src=[5394, 5438, 5426, 5598, 5542, 5610, 5572, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38172 dup_rows=21784 dup_rows_remote=19036 dedup_ratio=0.332397 router_tokens_per_expert=[267, 260, 240, 289, 260, 257, 274, 234, 252, 255, 261, 273, 277, 258, 219, 296, 249, 250, 277, 250, 263, 241, 249, 256, 227, 269, 241, 258, 295, 276, 271, 258, 260, 259, 247, 267, 262, 255, 275, 241, 267, 251, 262, 261, 259, 264, 285, 249, 269, 263, 239, 265, 249, 263, 235, 220, 232, 272, 236, 282, 247, 244, 276, 272, 250, 268, 253, 245, 252, 252, 247, 270, 268, 240, 248, 270, 241, 244, 271, 223, 272, 283, 249, 246, 264, 256, 250, 235, 234, 257, 262, 267, 254, 249, 237, 285, 231, 238, 261, 259, 254, 255, 231, 255, 301, 264, 228, 265, 244, 250, 277, 267, 231, 246, 259, 249, 242, 254, 246, 268, 265, 263, 266, 252, 237, 238, 214, 286] payload_rows_per_dst=[5554, 5538, 5580, 5418, 5430, 5460, 5408, 5364] dup_rows_per_dst=[2790, 2722, 2748, 2710, 2654, 2740, 2752, 2668] combine_rows_per_src=[5554, 5538, 5580, 5418, 5430, 5460, 5408, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43778 payload_rows_remote=38246 dup_rows=21758 dup_rows_remote=19100 dedup_ratio=0.332001 router_tokens_per_expert=[239, 267, 248, 265, 278, 272, 274, 261, 230, 229, 243, 291, 263, 248, 243, 264, 261, 264, 272, 230, 249, 260, 248, 270, 256, 263, 271, 262, 268, 270, 259, 239, 270, 265, 255, 254, 246, 263, 266, 295, 266, 255, 269, 250, 282, 236, 257, 256, 253, 277, 233, 268, 264, 222, 226, 253, 269, 264, 263, 244, 236, 264, 258, 230, 266, 251, 265, 234, 254, 253, 243, 245, 221, 277, 256, 268, 273, 245, 259, 257, 242, 262, 255, 288, 216, 252, 239, 247, 266, 256, 274, 270, 256, 262, 234, 276, 244, 306, 264, 262, 233, 265, 254, 261, 268, 277, 246, 242, 257, 283, 254, 262, 256, 265, 216, 262, 243, 261, 240, 251, 240, 273, 263, 222, 250, 241, 221, 258] payload_rows_per_dst=[5480, 5548, 5540, 5386, 5392, 5532, 5554, 5346] dup_rows_per_dst=[2750, 2736, 2830, 2662, 2742, 2658, 2802, 2578] combine_rows_per_src=[5480, 5548, 5540, 5386, 5392, 5532, 5554, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38148 dup_rows=21894 dup_rows_remote=19170 dedup_ratio=0.334076 router_tokens_per_expert=[259, 268, 271, 239, 243, 258, 243, 253, 250, 265, 241, 236, 247, 266, 271, 241, 261, 242, 239, 255, 286, 227, 267, 256, 230, 271, 264, 241, 284, 260, 259, 271, 247, 223, 274, 232, 268, 260, 257, 246, 247, 260, 240, 251, 251, 266, 260, 236, 263, 266, 241, 235, 255, 239, 264, 275, 275, 271, 254, 252, 266, 248, 255, 250, 250, 259, 264, 255, 262, 251, 244, 261, 263, 265, 235, 241, 255, 272, 234, 255, 261, 266, 232, 260, 255, 235, 243, 264, 249, 238, 265, 258, 258, 250, 263, 273, 250, 300, 271, 267, 245, 256, 238, 258, 249, 292, 231, 239, 290, 261, 256, 269, 243, 237, 260, 253, 260, 269, 256, 232, 287, 263, 245, 280, 277, 279, 257, 271] payload_rows_per_dst=[5422, 5442, 5346, 5494, 5490, 5444, 5512, 5492] dup_rows_per_dst=[2680, 2784, 2690, 2724, 2642, 2696, 2832, 2846] combine_rows_per_src=[5422, 5442, 5346, 5494, 5490, 5444, 5512, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43728 payload_rows_remote=38222 dup_rows=21808 dup_rows_remote=19046 dedup_ratio=0.332764 router_tokens_per_expert=[247, 250, 282, 253, 272, 272, 261, 310, 256, 261, 243, 252, 258, 281, 246, 273, 266, 228, 281, 252, 291, 263, 271, 280, 266, 259, 276, 264, 248, 269, 249, 243, 262, 264, 261, 280, 249, 230, 248, 247, 266, 234, 267, 251, 242, 248, 244, 247, 240, 253, 249, 265, 259, 273, 247, 266, 249, 252, 254, 259, 273, 281, 251, 240, 254, 255, 254, 257, 266, 264, 299, 249, 227, 258, 256, 251, 225, 238, 246, 233, 257, 248, 246, 259, 246, 263, 230, 245, 260, 253, 270, 256, 253, 244, 239, 264, 254, 251, 253, 240, 267, 283, 255, 244, 234, 257, 251, 268, 216, 245, 240, 238, 263, 284, 239, 286, 256, 266, 263, 261, 290, 236, 250, 232, 256, 249, 255, 247] payload_rows_per_dst=[5342, 5576, 5524, 5542, 5374, 5440, 5424, 5506] dup_rows_per_dst=[2674, 2738, 2838, 2792, 2690, 2724, 2590, 2762] combine_rows_per_src=[5342, 5576, 5524, 5542, 5374, 5440, 5424, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=157540352 dispatch_meta_bytes=228912 combine_payload_bytes=157540352 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157769264 ingress_bytes=195289088 link_bytes=195289088 token_bytes=315309616 weight_movement_bytes=37748736 total_nvlink_bytes=353058352 weight_to_token_ratio=0.119720 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.871163953e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156352512 dispatch_meta_bytes=228832 combine_payload_bytes=156352512 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156581344 ingress_bytes=212975616 link_bytes=212975616 token_bytes=312933856 weight_movement_bytes=56623104 total_nvlink_bytes=369556960 weight_to_token_ratio=0.180943 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.072917 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040627562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229384 combine_payload_bytes=156655616 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156885000 ingress_bytes=194404352 link_bytes=194404352 token_bytes=313540616 weight_movement_bytes=37748736 total_nvlink_bytes=351289352 weight_to_token_ratio=0.120395 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862686848e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43754 payload_rows_remote=38214 dup_rows=21782 dup_rows_remote=18998 dedup_ratio=0.332367 router_tokens_per_expert=[251, 253, 264, 257, 272, 254, 246, 264, 274, 261, 253, 258, 246, 260, 260, 278, 239, 252, 249, 273, 250, 266, 267, 255, 262, 226, 254, 275, 259, 249, 248, 252, 247, 243, 261, 267, 251, 244, 273, 264, 243, 265, 243, 249, 291, 248, 289, 253, 266, 238, 245, 284, 259, 247, 255, 262, 235, 258, 252, 277, 261, 252, 247, 260, 226, 251, 262, 255, 292, 245, 250, 280, 260, 253, 250, 259, 247, 255, 279, 253, 257, 246, 233, 247, 261, 231, 264, 285, 267, 281, 265, 256, 220, 268, 249, 280, 289, 229, 254, 283, 230, 255, 244, 266, 258, 236, 254, 242, 272, 227, 276, 270, 247, 257, 243, 251, 256, 254, 273, 258, 234, 249, 241, 233, 270, 253, 239, 242] payload_rows_per_dst=[5540, 5476, 5544, 5564, 5616, 5188, 5476, 5350] dup_rows_per_dst=[2784, 2676, 2948, 2826, 2894, 2332, 2694, 2628] combine_rows_per_src=[5540, 5476, 5544, 5564, 5616, 5188, 5476, 5350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=229272 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156483480 ingress_bytes=175128576 link_bytes=175128576 token_bytes=312737688 weight_movement_bytes=18874368 total_nvlink_bytes=331612056 weight_to_token_ratio=0.060352 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677995847e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156557312 dispatch_meta_bytes=229072 combine_payload_bytes=156557312 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156786384 ingress_bytes=194306048 link_bytes=194306048 token_bytes=313343696 weight_movement_bytes=37748736 total_nvlink_bytes=351092432 weight_to_token_ratio=0.120471 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861744947e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43808 payload_rows_remote=38326 dup_rows=21728 dup_rows_remote=18990 dedup_ratio=0.331543 router_tokens_per_expert=[273, 247, 250, 277, 233, 271, 277, 255, 263, 248, 246, 244, 248, 252, 254, 238, 264, 262, 273, 271, 264, 255, 236, 262, 267, 268, 255, 252, 242, 249, 258, 232, 227, 261, 267, 233, 252, 263, 235, 239, 248, 275, 235, 258, 252, 250, 253, 260, 227, 262, 234, 248, 232, 216, 264, 257, 300, 244, 260, 272, 293, 264, 255, 265, 282, 235, 236, 256, 257, 255, 279, 270, 246, 242, 254, 280, 250, 256, 247, 260, 233, 264, 284, 252, 258, 256, 244, 258, 259, 256, 245, 245, 259, 240, 249, 250, 263, 260, 251, 245, 241, 290, 249, 268, 277, 259, 278, 252, 269, 277, 252, 269, 246, 277, 266, 301, 250, 260, 240, 243, 253, 250, 234, 267, 244, 269, 253, 271] payload_rows_per_dst=[5478, 5482, 5398, 5528, 5444, 5444, 5576, 5458] dup_rows_per_dst=[2674, 2738, 2618, 2658, 2766, 2660, 2824, 2790] combine_rows_per_src=[5478, 5482, 5398, 5528, 5444, 5444, 5576, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38196 dup_rows=21834 dup_rows_remote=19130 dedup_ratio=0.333160 router_tokens_per_expert=[238, 261, 243, 288, 287, 247, 271, 255, 258, 249, 259, 238, 247, 252, 252, 234, 263, 243, 259, 285, 258, 239, 268, 239, 244, 267, 242, 288, 254, 269, 221, 241, 257, 255, 280, 250, 270, 252, 246, 265, 233, 292, 259, 234, 231, 242, 270, 215, 236, 294, 240, 246, 248, 294, 262, 258, 278, 237, 243, 247, 249, 259, 262, 260, 280, 237, 244, 240, 280, 265, 259, 276, 249, 267, 250, 236, 234, 264, 243, 281, 255, 271, 293, 229, 242, 249, 241, 259, 256, 252, 264, 260, 248, 268, 281, 281, 240, 276, 249, 257, 263, 255, 254, 254, 251, 279, 255, 259, 257, 247, 256, 266, 256, 272, 207, 256, 283, 237, 248, 273, 260, 256, 240, 245, 252, 266, 258, 264] payload_rows_per_dst=[5458, 5400, 5458, 5544, 5506, 5460, 5440, 5436] dup_rows_per_dst=[2700, 2760, 2644, 2682, 2704, 2838, 2796, 2710] combine_rows_per_src=[5458, 5400, 5458, 5544, 5506, 5460, 5440, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=33 plans=1 dispatch_payload_bytes=156524544 dispatch_meta_bytes=228848 combine_payload_bytes=156524544 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156753392 ingress_bytes=232022016 link_bytes=232022016 token_bytes=313277936 weight_movement_bytes=75497472 total_nvlink_bytes=388775408 weight_to_token_ratio=0.240992 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.223120795e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43770 payload_rows_remote=38288 dup_rows=21766 dup_rows_remote=19068 dedup_ratio=0.332123 router_tokens_per_expert=[267, 254, 256, 252, 286, 252, 233, 303, 249, 239, 280, 257, 263, 245, 227, 270, 275, 247, 225, 264, 253, 249, 298, 274, 225, 244, 248, 256, 232, 243, 276, 269, 277, 263, 256, 242, 251, 231, 253, 259, 246, 239, 267, 236, 262, 248, 256, 266, 267, 260, 263, 267, 256, 253, 251, 268, 265, 264, 264, 264, 245, 270, 228, 264, 269, 243, 266, 283, 244, 274, 248, 253, 250, 274, 244, 253, 243, 246, 279, 250, 255, 265, 252, 268, 258, 264, 275, 268, 259, 216, 255, 251, 233, 263, 285, 251, 259, 289, 253, 253, 268, 262, 262, 242, 254, 250, 240, 269, 268, 252, 231, 238, 233, 224, 272, 265, 252, 278, 256, 267, 235, 225, 249, 267, 249, 240, 239, 278] payload_rows_per_dst=[5454, 5486, 5432, 5562, 5478, 5436, 5482, 5440] dup_rows_per_dst=[2812, 2670, 2672, 2736, 2760, 2800, 2698, 2618] combine_rows_per_src=[5454, 5486, 5432, 5562, 5478, 5436, 5482, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=229264 combine_payload_bytes=156983296 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157212560 ingress_bytes=194732032 link_bytes=194732032 token_bytes=314195856 weight_movement_bytes=37748736 total_nvlink_bytes=351944592 weight_to_token_ratio=0.120144 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865826516e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156450816 dispatch_meta_bytes=229304 combine_payload_bytes=156450816 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156680120 ingress_bytes=175325184 link_bytes=175325184 token_bytes=313130936 weight_movement_bytes=18874368 total_nvlink_bytes=332005304 weight_to_token_ratio=0.060276 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679879648e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43660 payload_rows_remote=38252 dup_rows=21876 dup_rows_remote=19096 dedup_ratio=0.333801 router_tokens_per_expert=[255, 268, 258, 218, 232, 224, 248, 235, 226, 238, 256, 249, 235, 277, 249, 225, 255, 266, 267, 263, 272, 259, 244, 240, 264, 261, 266, 250, 257, 268, 263, 243, 260, 257, 264, 247, 266, 268, 223, 256, 281, 249, 283, 254, 258, 227, 262, 239, 279, 278, 265, 263, 233, 234, 262, 227, 258, 265, 275, 271, 260, 259, 259, 256, 253, 256, 289, 265, 233, 245, 251, 232, 267, 262, 277, 251, 271, 273, 230, 244, 249, 268, 232, 292, 253, 261, 264, 240, 247, 241, 249, 237, 252, 260, 256, 233, 274, 280, 252, 262, 256, 282, 251, 289, 260, 253, 256, 245, 248, 279, 262, 272, 273, 269, 254, 264, 251, 223, 244, 267, 260, 266, 254, 257, 266, 275, 262, 260] payload_rows_per_dst=[5316, 5488, 5408, 5512, 5474, 5462, 5532, 5468] dup_rows_per_dst=[2470, 2788, 2780, 2776, 2724, 2606, 2910, 2822] combine_rows_per_src=[5316, 5488, 5408, 5512, 5474, 5462, 5532, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43820 payload_rows_remote=38314 dup_rows=21716 dup_rows_remote=18914 dedup_ratio=0.331360 router_tokens_per_expert=[251, 260, 262, 239, 265, 262, 247, 287, 229, 282, 254, 259, 233, 242, 246, 237, 235, 251, 252, 263, 253, 249, 254, 268, 251, 287, 240, 245, 283, 249, 262, 250, 235, 271, 258, 262, 237, 265, 259, 243, 257, 237, 249, 270, 242, 254, 280, 252, 246, 251, 275, 251, 292, 245, 243, 246, 255, 232, 238, 255, 261, 229, 267, 257, 243, 250, 243, 269, 267, 283, 285, 278, 262, 252, 257, 236, 245, 245, 248, 250, 252, 279, 237, 263, 277, 297, 266, 271, 216, 268, 243, 254, 256, 263, 251, 261, 264, 253, 253, 258, 275, 233, 276, 223, 260, 253, 228, 295, 267, 270, 253, 249, 252, 256, 253, 264, 240, 259, 257, 253, 245, 261, 268, 280, 269, 258, 265, 250] payload_rows_per_dst=[5430, 5470, 5524, 5446, 5472, 5506, 5472, 5500] dup_rows_per_dst=[2680, 2714, 2618, 2640, 2754, 2802, 2748, 2760] combine_rows_per_src=[5430, 5470, 5524, 5446, 5472, 5506, 5472, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43592 payload_rows_remote=38314 dup_rows=21944 dup_rows_remote=19220 dedup_ratio=0.334839 router_tokens_per_expert=[251, 235, 261, 251, 254, 251, 253, 275, 288, 259, 272, 246, 256, 252, 262, 246, 270, 263, 232, 239, 262, 259, 285, 271, 236, 257, 266, 267, 301, 261, 286, 250, 238, 265, 226, 261, 281, 255, 278, 268, 259, 266, 261, 274, 303, 261, 253, 257, 240, 256, 236, 269, 252, 261, 250, 246, 264, 255, 238, 237, 284, 246, 241, 226, 259, 244, 258, 273, 226, 246, 241, 251, 257, 251, 224, 276, 237, 247, 255, 231, 254, 215, 270, 260, 255, 253, 280, 260, 262, 233, 239, 239, 259, 274, 239, 252, 238, 254, 256, 255, 245, 283, 240, 275, 259, 263, 281, 238, 255, 249, 237, 268, 252, 293, 261, 259, 271, 242, 243, 252, 276, 241, 268, 234, 254, 261, 263, 258] payload_rows_per_dst=[5506, 5578, 5484, 5278, 5376, 5418, 5474, 5478] dup_rows_per_dst=[2718, 2832, 2928, 2724, 2576, 2670, 2718, 2778] combine_rows_per_src=[5506, 5578, 5484, 5278, 5376, 5418, 5474, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156827648 dispatch_meta_bytes=229424 combine_payload_bytes=156827648 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157057072 ingress_bytes=194576384 link_bytes=194576384 token_bytes=313884720 weight_movement_bytes=37748736 total_nvlink_bytes=351633456 weight_to_token_ratio=0.120263 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864335174e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38266 dup_rows=21722 dup_rows_remote=18936 dedup_ratio=0.331451 router_tokens_per_expert=[245, 255, 252, 237, 248, 246, 250, 233, 274, 221, 288, 272, 297, 243, 238, 223, 258, 271, 258, 245, 283, 235, 250, 279, 238, 223, 252, 234, 248, 247, 288, 244, 254, 278, 244, 255, 241, 247, 279, 276, 273, 251, 236, 257, 269, 240, 258, 237, 262, 232, 253, 232, 269, 242, 253, 251, 234, 268, 255, 251, 250, 256, 267, 242, 264, 275, 280, 258, 278, 250, 235, 289, 249, 250, 255, 247, 274, 264, 238, 266, 245, 293, 279, 278, 246, 269, 256, 263, 258, 256, 250, 248, 262, 261, 268, 258, 267, 281, 236, 253, 256, 272, 217, 273, 253, 242, 241, 269, 274, 243, 240, 264, 257, 263, 284, 239, 264, 259, 250, 258, 251, 268, 261, 249, 266, 259, 232, 278] payload_rows_per_dst=[5358, 5426, 5268, 5612, 5628, 5508, 5466, 5548] dup_rows_per_dst=[2686, 2680, 2480, 2806, 2716, 2872, 2696, 2786] combine_rows_per_src=[5358, 5426, 5268, 5612, 5628, 5508, 5466, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229392 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909584 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589776 weight_movement_bytes=37748736 total_nvlink_bytes=351338512 weight_to_token_ratio=0.120376 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=228912 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157163056 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314097200 weight_movement_bytes=18874368 total_nvlink_bytes=332971568 weight_to_token_ratio=0.060091 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38260 dup_rows=21692 dup_rows_remote=19036 dedup_ratio=0.330994 router_tokens_per_expert=[264, 239, 272, 262, 268, 244, 275, 249, 235, 257, 250, 262, 279, 226, 245, 277, 287, 259, 285, 263, 241, 241, 257, 226, 273, 218, 236, 253, 289, 254, 261, 266, 238, 253, 258, 270, 261, 261, 252, 270, 266, 269, 250, 235, 273, 240, 260, 255, 227, 248, 252, 254, 243, 250, 295, 289, 268, 255, 265, 278, 258, 229, 273, 260, 270, 261, 260, 261, 268, 237, 250, 240, 274, 246, 271, 257, 279, 248, 260, 248, 244, 243, 250, 227, 239, 236, 281, 262, 284, 257, 250, 270, 264, 243, 255, 250, 255, 230, 269, 245, 265, 246, 228, 250, 242, 272, 279, 259, 274, 270, 250, 282, 257, 246, 260, 252, 259, 222, 255, 264, 251, 234, 259, 259, 245, 240, 255, 241] payload_rows_per_dst=[5584, 5474, 5534, 5534, 5498, 5380, 5442, 5398] dup_rows_per_dst=[2656, 2792, 2688, 2754, 2762, 2730, 2710, 2600] combine_rows_per_src=[5584, 5474, 5534, 5534, 5498, 5380, 5442, 5398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=230136 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157164280 ingress_bytes=175808512 link_bytes=175808512 token_bytes=314098424 weight_movement_bytes=18874368 total_nvlink_bytes=332972792 weight_to_token_ratio=0.060091 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684510659e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38328 dup_rows=21854 dup_rows_remote=19250 dedup_ratio=0.333466 router_tokens_per_expert=[248, 236, 277, 256, 268, 237, 260, 247, 269, 261, 268, 251, 247, 244, 249, 290, 270, 253, 271, 267, 251, 265, 250, 243, 237, 235, 253, 229, 229, 253, 223, 250, 247, 251, 240, 273, 234, 251, 268, 274, 239, 263, 281, 255, 252, 265, 254, 280, 244, 267, 228, 259, 280, 232, 256, 255, 225, 263, 262, 272, 224, 236, 277, 276, 248, 254, 267, 250, 273, 235, 230, 273, 235, 264, 246, 238, 267, 221, 270, 262, 286, 274, 227, 275, 255, 232, 246, 283, 244, 292, 290, 261, 266, 243, 260, 295, 230, 275, 281, 233, 277, 267, 265, 232, 246, 265, 229, 280, 271, 278, 258, 256, 236, 255, 291, 258, 266, 265, 242, 211, 256, 262, 275, 244, 266, 264, 268, 234] payload_rows_per_dst=[5446, 5354, 5566, 5364, 5408, 5508, 5512, 5524] dup_rows_per_dst=[2770, 2604, 2688, 2748, 2780, 2828, 2774, 2662] combine_rows_per_src=[5446, 5354, 5566, 5364, 5408, 5508, 5512, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43750 payload_rows_remote=38340 dup_rows=21786 dup_rows_remote=19182 dedup_ratio=0.332428 router_tokens_per_expert=[246, 250, 262, 245, 267, 311, 270, 249, 261, 266, 242, 264, 251, 248, 250, 229, 267, 245, 232, 240, 250, 260, 248, 256, 260, 260, 264, 274, 254, 243, 280, 218, 262, 237, 273, 246, 240, 258, 239, 253, 252, 237, 273, 251, 244, 259, 275, 274, 268, 228, 257, 272, 251, 255, 286, 249, 243, 271, 237, 271, 254, 261, 243, 276, 260, 245, 256, 258, 257, 264, 245, 235, 253, 227, 261, 249, 230, 238, 256, 273, 274, 259, 256, 237, 254, 262, 281, 268, 248, 250, 257, 261, 242, 245, 253, 274, 254, 241, 262, 286, 240, 255, 280, 254, 251, 266, 256, 253, 267, 271, 298, 250, 257, 228, 276, 240, 271, 244, 288, 251, 272, 245, 246, 259, 262, 274, 257, 229] payload_rows_per_dst=[5432, 5476, 5466, 5506, 5410, 5466, 5558, 5436] dup_rows_per_dst=[2790, 2626, 2680, 2738, 2604, 2776, 2810, 2762] combine_rows_per_src=[5432, 5476, 5466, 5506, 5410, 5466, 5558, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228808 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966344 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313703880 weight_movement_bytes=18874368 total_nvlink_bytes=332578248 weight_to_token_ratio=0.060166 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2304 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=34 plans=1 dispatch_payload_bytes=156712960 dispatch_meta_bytes=229184 combine_payload_bytes=156712960 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156942144 ingress_bytes=175587328 link_bytes=175587328 token_bytes=313655104 weight_movement_bytes=18874368 total_nvlink_bytes=332529472 weight_to_token_ratio=0.060176 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682391383e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=230312 combine_payload_bytes=156991488 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157221800 ingress_bytes=175865856 link_bytes=175865856 token_bytes=314213288 weight_movement_bytes=18874368 total_nvlink_bytes=333087656 weight_to_token_ratio=0.060069 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685060101e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=230088 combine_payload_bytes=157040640 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157270728 ingress_bytes=175915008 link_bytes=175915008 token_bytes=314311368 weight_movement_bytes=18874368 total_nvlink_bytes=333185736 weight_to_token_ratio=0.060050 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685531051e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43780 payload_rows_remote=38318 dup_rows=21756 dup_rows_remote=19150 dedup_ratio=0.331970 router_tokens_per_expert=[271, 239, 249, 276, 255, 240, 270, 231, 258, 258, 281, 254, 262, 244, 269, 256, 291, 243, 254, 240, 228, 270, 253, 226, 273, 283, 264, 229, 264, 273, 242, 319, 269, 272, 235, 269, 256, 266, 294, 243, 271, 272, 238, 234, 257, 248, 244, 252, 264, 251, 273, 267, 255, 280, 239, 255, 274, 254, 243, 253, 271, 263, 248, 227, 235, 259, 270, 235, 294, 256, 250, 270, 253, 266, 241, 274, 267, 264, 239, 259, 250, 247, 247, 264, 271, 266, 240, 260, 253, 235, 262, 233, 253, 275, 237, 247, 283, 257, 264, 243, 261, 247, 219, 258, 271, 234, 256, 254, 228, 267, 239, 253, 255, 237, 260, 263, 255, 235, 254, 263, 245, 259, 241, 250, 242, 266, 262, 273] payload_rows_per_dst=[5498, 5566, 5490, 5474, 5462, 5396, 5462, 5432] dup_rows_per_dst=[2728, 2738, 2750, 2760, 2802, 2684, 2606, 2688] combine_rows_per_src=[5498, 5566, 5490, 5474, 5462, 5396, 5462, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38310 dup_rows=21830 dup_rows_remote=19180 dedup_ratio=0.333099 router_tokens_per_expert=[268, 237, 296, 275, 266, 270, 246, 247, 261, 215, 243, 224, 282, 269, 274, 275, 248, 267, 267, 238, 271, 249, 253, 255, 258, 252, 245, 277, 249, 267, 251, 267, 247, 247, 254, 263, 223, 278, 272, 242, 229, 233, 242, 267, 238, 272, 269, 247, 272, 267, 293, 252, 234, 259, 259, 246, 263, 268, 276, 275, 257, 286, 266, 248, 274, 239, 254, 257, 259, 261, 254, 269, 266, 235, 254, 264, 255, 271, 262, 251, 234, 258, 254, 229, 271, 257, 234, 230, 262, 244, 255, 251, 241, 231, 227, 256, 258, 251, 257, 254, 216, 256, 246, 274, 243, 240, 230, 271, 281, 262, 248, 307, 248, 249, 265, 272, 254, 252, 265, 252, 254, 266, 283, 249, 253, 247, 247, 253] payload_rows_per_dst=[5554, 5500, 5396, 5548, 5492, 5290, 5454, 5472] dup_rows_per_dst=[2742, 2728, 2650, 2894, 2758, 2578, 2734, 2746] combine_rows_per_src=[5554, 5500, 5396, 5548, 5492, 5290, 5454, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43818 payload_rows_remote=38314 dup_rows=21718 dup_rows_remote=18926 dedup_ratio=0.331390 router_tokens_per_expert=[260, 238, 262, 255, 265, 241, 244, 255, 265, 251, 234, 268, 246, 249, 259, 248, 258, 273, 255, 243, 251, 270, 251, 267, 251, 255, 245, 247, 252, 248, 282, 270, 267, 277, 230, 244, 256, 239, 252, 276, 253, 255, 244, 252, 281, 287, 236, 246, 255, 262, 254, 274, 234, 267, 251, 271, 263, 258, 260, 277, 254, 231, 260, 240, 258, 246, 233, 247, 263, 265, 260, 270, 242, 269, 263, 242, 237, 267, 244, 241, 248, 268, 259, 292, 236, 252, 268, 252, 262, 246, 258, 267, 216, 287, 281, 256, 233, 263, 274, 254, 244, 260, 263, 279, 268, 269, 267, 254, 251, 257, 273, 256, 284, 250, 248, 247, 257, 253, 256, 263, 250, 243, 262, 246, 259, 233, 238, 255] payload_rows_per_dst=[5414, 5520, 5480, 5488, 5450, 5504, 5530, 5432] dup_rows_per_dst=[2666, 2716, 2710, 2734, 2644, 2792, 2800, 2656] combine_rows_per_src=[5414, 5520, 5480, 5488, 5450, 5504, 5530, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38292 dup_rows=21854 dup_rows_remote=19242 dedup_ratio=0.333466 router_tokens_per_expert=[225, 254, 267, 269, 258, 266, 227, 264, 244, 251, 235, 254, 253, 263, 245, 260, 248, 260, 228, 257, 262, 263, 250, 295, 246, 239, 248, 244, 264, 292, 277, 242, 262, 255, 232, 275, 276, 263, 258, 251, 259, 254, 274, 261, 265, 240, 267, 273, 270, 258, 260, 229, 261, 260, 274, 245, 242, 275, 229, 242, 235, 248, 247, 226, 252, 255, 255, 248, 244, 244, 266, 276, 248, 246, 249, 258, 246, 249, 297, 266, 241, 268, 254, 265, 246, 224, 242, 283, 259, 268, 244, 271, 273, 236, 263, 251, 263, 295, 268, 278, 262, 255, 234, 239, 264, 275, 245, 227, 249, 262, 256, 255, 255, 259, 271, 248, 244, 275, 275, 251, 236, 261, 262, 271, 267, 249, 254, 260] payload_rows_per_dst=[5430, 5468, 5516, 5390, 5494, 5468, 5448, 5468] dup_rows_per_dst=[2640, 2762, 2814, 2612, 2704, 2708, 2806, 2808] combine_rows_per_src=[5430, 5468, 5516, 5390, 5494, 5468, 5448, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=229872 combine_payload_bytes=156950528 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157180400 ingress_bytes=194699264 link_bytes=194699264 token_bytes=314130928 weight_movement_bytes=37748736 total_nvlink_bytes=351879664 weight_to_token_ratio=0.120169 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865512550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43764 payload_rows_remote=38252 dup_rows=21772 dup_rows_remote=18868 dedup_ratio=0.332214 router_tokens_per_expert=[249, 259, 239, 239, 256, 247, 254, 257, 250, 244, 271, 252, 273, 258, 281, 275, 263, 244, 239, 252, 254, 265, 266, 252, 241, 272, 234, 257, 246, 252, 294, 251, 236, 287, 229, 266, 258, 252, 263, 241, 268, 281, 238, 273, 245, 285, 262, 265, 242, 269, 251, 259, 272, 268, 250, 237, 277, 257, 261, 297, 255, 216, 247, 260, 246, 263, 278, 241, 261, 257, 264, 260, 233, 249, 259, 246, 242, 247, 266, 267, 255, 243, 247, 248, 258, 259, 252, 286, 243, 262, 267, 250, 265, 255, 251, 253, 239, 246, 262, 270, 258, 266, 234, 235, 243, 260, 256, 247, 241, 249, 249, 252, 256, 268, 259, 246, 268, 260, 250, 246, 272, 248, 275, 249, 274, 270, 242, 252] payload_rows_per_dst=[5450, 5484, 5434, 5468, 5640, 5436, 5340, 5512] dup_rows_per_dst=[2758, 2680, 2590, 2766, 2792, 2754, 2528, 2904] combine_rows_per_src=[5450, 5484, 5434, 5468, 5640, 5436, 5340, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156917760 dispatch_meta_bytes=229960 combine_payload_bytes=156917760 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157147720 ingress_bytes=194666496 link_bytes=194666496 token_bytes=314065480 weight_movement_bytes=37748736 total_nvlink_bytes=351814216 weight_to_token_ratio=0.120194 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865198583e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156844032 dispatch_meta_bytes=230136 combine_payload_bytes=156844032 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157074168 ingress_bytes=175718400 link_bytes=175718400 token_bytes=313918200 weight_movement_bytes=18874368 total_nvlink_bytes=332792568 weight_to_token_ratio=0.060125 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683647250e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156934144 dispatch_meta_bytes=228960 combine_payload_bytes=156934144 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157163104 ingress_bytes=194682880 link_bytes=194682880 token_bytes=314097248 weight_movement_bytes=37748736 total_nvlink_bytes=351845984 weight_to_token_ratio=0.120182 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865355566e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38098 dup_rows=21850 dup_rows_remote=18808 dedup_ratio=0.333405 router_tokens_per_expert=[278, 266, 299, 270, 268, 245, 245, 258, 267, 250, 264, 257, 221, 270, 238, 246, 234, 254, 229, 240, 270, 246, 256, 250, 274, 267, 238, 271, 247, 262, 285, 280, 233, 248, 241, 251, 235, 249, 275, 272, 248, 267, 279, 250, 271, 243, 241, 241, 248, 235, 225, 249, 252, 253, 257, 250, 260, 249, 250, 245, 252, 257, 275, 287, 256, 278, 270, 272, 263, 259, 260, 263, 277, 248, 250, 246, 249, 283, 255, 237, 261, 270, 223, 262, 248, 268, 261, 264, 241, 264, 256, 270, 261, 245, 236, 257, 254, 270, 245, 269, 243, 269, 250, 265, 268, 260, 219, 265, 247, 264, 270, 257, 244, 277, 297, 249, 253, 248, 235, 263, 246, 261, 264, 247, 250, 232, 250, 251] payload_rows_per_dst=[5588, 5516, 5414, 5548, 5486, 5294, 5376, 5464] dup_rows_per_dst=[3042, 2726, 2638, 2888, 2846, 2532, 2508, 2670] combine_rows_per_src=[5588, 5516, 5414, 5548, 5486, 5294, 5376, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43686 payload_rows_remote=38256 dup_rows=21850 dup_rows_remote=19054 dedup_ratio=0.333405 router_tokens_per_expert=[238, 267, 258, 247, 272, 254, 265, 243, 255, 272, 237, 264, 268, 241, 234, 265, 268, 241, 257, 281, 238, 252, 291, 271, 260, 253, 257, 237, 265, 243, 250, 249, 249, 264, 254, 235, 245, 260, 241, 274, 255, 245, 248, 269, 285, 261, 250, 272, 265, 243, 262, 266, 248, 247, 242, 240, 237, 248, 267, 259, 226, 261, 265, 247, 246, 272, 230, 281, 258, 260, 243, 272, 240, 252, 261, 241, 255, 248, 259, 236, 260, 247, 256, 266, 273, 251, 275, 252, 255, 256, 267, 258, 272, 270, 244, 267, 240, 232, 287, 284, 279, 231, 255, 229, 268, 259, 276, 279, 230, 239, 274, 249, 266, 265, 237, 274, 273, 219, 284, 227, 250, 260, 245, 248, 249, 285, 264, 265] payload_rows_per_dst=[5520, 5430, 5476, 5432, 5430, 5488, 5508, 5402] dup_rows_per_dst=[2824, 2796, 2738, 2614, 2678, 2850, 2714, 2636] combine_rows_per_src=[5520, 5430, 5476, 5432, 5430, 5488, 5508, 5402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43668 payload_rows_remote=38246 dup_rows=21868 dup_rows_remote=19144 dedup_ratio=0.333679 router_tokens_per_expert=[250, 242, 261, 253, 279, 235, 278, 255, 236, 237, 225, 261, 275, 274, 251, 246, 287, 259, 290, 268, 273, 244, 281, 249, 225, 269, 239, 265, 267, 253, 246, 251, 234, 259, 280, 223, 262, 252, 263, 245, 248, 238, 280, 275, 248, 262, 227, 253, 253, 252, 217, 258, 265, 275, 249, 262, 273, 265, 268, 255, 268, 258, 271, 257, 260, 248, 243, 244, 272, 262, 255, 284, 256, 227, 231, 261, 243, 277, 275, 235, 251, 248, 261, 236, 244, 267, 275, 249, 249, 279, 268, 260, 245, 220, 278, 251, 266, 267, 246, 276, 252, 259, 236, 257, 269, 282, 251, 219, 261, 235, 269, 235, 279, 265, 249, 250, 251, 249, 251, 269, 251, 276, 245, 269, 279, 253, 259, 220] payload_rows_per_dst=[5420, 5472, 5450, 5520, 5422, 5490, 5464, 5430] dup_rows_per_dst=[2696, 2860, 2648, 2772, 2724, 2672, 2696, 2800] combine_rows_per_src=[5420, 5472, 5450, 5520, 5422, 5490, 5464, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=228480 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156908672 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313588864 weight_movement_bytes=37748736 total_nvlink_bytes=351337600 weight_to_token_ratio=0.120377 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=35 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=227624 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156277032 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312326440 weight_movement_bytes=18874368 total_nvlink_bytes=331200808 weight_to_token_ratio=0.060432 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43468 payload_rows_remote=37986 dup_rows=22068 dup_rows_remote=19330 dedup_ratio=0.336731 router_tokens_per_expert=[253, 251, 261, 265, 272, 244, 263, 255, 258, 244, 254, 250, 257, 248, 258, 268, 234, 243, 213, 264, 254, 260, 262, 269, 267, 239, 240, 241, 264, 253, 264, 247, 261, 261, 280, 269, 264, 261, 259, 272, 265, 232, 257, 284, 235, 250, 253, 266, 271, 226, 258, 270, 259, 235, 257, 254, 258, 259, 256, 265, 234, 247, 261, 245, 266, 284, 261, 265, 253, 221, 261, 257, 251, 246, 258, 258, 261, 248, 245, 246, 234, 248, 258, 281, 267, 254, 263, 263, 230, 244, 284, 263, 251, 256, 276, 268, 247, 257, 267, 248, 269, 264, 284, 253, 246, 259, 268, 241, 255, 245, 247, 260, 258, 270, 247, 240, 287, 264, 252, 236, 254, 234, 273, 268, 274, 254, 244, 243] payload_rows_per_dst=[5412, 5404, 5434, 5422, 5362, 5526, 5482, 5426] dup_rows_per_dst=[2790, 2624, 2904, 2688, 2800, 2754, 2738, 2770] combine_rows_per_src=[5412, 5404, 5434, 5422, 5362, 5526, 5482, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229240 combine_payload_bytes=156696576 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156925816 ingress_bytes=175570944 link_bytes=175570944 token_bytes=313622392 weight_movement_bytes=18874368 total_nvlink_bytes=332496760 weight_to_token_ratio=0.060182 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682234399e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156655616 dispatch_meta_bytes=229560 combine_payload_bytes=156655616 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156885176 ingress_bytes=175529984 link_bytes=175529984 token_bytes=313540792 weight_movement_bytes=18874368 total_nvlink_bytes=332415160 weight_to_token_ratio=0.060197 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681841941e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43674 payload_rows_remote=38232 dup_rows=21862 dup_rows_remote=19118 dedup_ratio=0.333588 router_tokens_per_expert=[248, 249, 269, 261, 263, 279, 221, 263, 261, 244, 253, 266, 254, 276, 241, 274, 240, 249, 263, 259, 256, 281, 247, 235, 288, 277, 252, 263, 249, 248, 242, 245, 249, 268, 250, 259, 274, 265, 241, 227, 262, 241, 262, 266, 253, 246, 254, 276, 247, 267, 284, 235, 251, 246, 244, 264, 258, 237, 271, 270, 261, 245, 250, 273, 261, 292, 243, 257, 245, 246, 264, 265, 264, 249, 270, 266, 253, 244, 217, 232, 258, 253, 287, 266, 270, 250, 248, 283, 255, 275, 257, 243, 246, 265, 272, 255, 265, 262, 289, 260, 259, 234, 254, 287, 245, 261, 232, 248, 257, 267, 252, 256, 225, 228, 225, 255, 272, 266, 288, 243, 237, 255, 229, 241, 255, 264, 223, 271] payload_rows_per_dst=[5496, 5432, 5442, 5438, 5512, 5544, 5454, 5356] dup_rows_per_dst=[2748, 2756, 2744, 2768, 2624, 2822, 2802, 2598] combine_rows_per_src=[5496, 5432, 5442, 5438, 5512, 5544, 5454, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43664 payload_rows_remote=38184 dup_rows=21872 dup_rows_remote=19070 dedup_ratio=0.333740 router_tokens_per_expert=[283, 245, 240, 234, 250, 248, 238, 244, 231, 270, 249, 270, 260, 266, 250, 248, 280, 241, 271, 244, 245, 248, 260, 232, 272, 252, 270, 254, 249, 266, 252, 260, 238, 277, 282, 240, 247, 269, 287, 220, 287, 270, 252, 251, 225, 256, 262, 241, 272, 242, 238, 250, 263, 255, 257, 250, 243, 253, 269, 271, 278, 264, 268, 268, 236, 252, 257, 269, 246, 208, 250, 244, 243, 260, 258, 250, 266, 250, 273, 284, 258, 232, 251, 252, 249, 268, 233, 258, 271, 283, 242, 273, 268, 259, 259, 250, 254, 254, 248, 248, 275, 284, 268, 245, 248, 242, 275, 252, 255, 265, 266, 250, 276, 315, 254, 247, 245, 264, 238, 226, 268, 257, 240, 263, 264, 250, 254, 259] payload_rows_per_dst=[5422, 5416, 5448, 5480, 5400, 5544, 5500, 5454] dup_rows_per_dst=[2630, 2776, 2760, 2802, 2692, 2668, 2758, 2786] combine_rows_per_src=[5422, 5416, 5448, 5480, 5400, 5544, 5500, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=155590656 dispatch_meta_bytes=229264 combine_payload_bytes=155590656 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=155819920 ingress_bytes=193339392 link_bytes=193339392 token_bytes=311410576 weight_movement_bytes=37748736 total_nvlink_bytes=349159312 weight_to_token_ratio=0.121219 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852482925e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43848 payload_rows_remote=38240 dup_rows=21688 dup_rows_remote=18940 dedup_ratio=0.330933 router_tokens_per_expert=[268, 260, 250, 245, 252, 250, 278, 259, 232, 264, 284, 254, 238, 274, 282, 243, 243, 266, 246, 242, 241, 286, 244, 242, 277, 254, 236, 243, 267, 247, 272, 264, 259, 248, 267, 221, 242, 250, 245, 256, 243, 255, 265, 282, 274, 250, 278, 242, 267, 250, 222, 272, 259, 279, 279, 233, 268, 235, 249, 258, 247, 270, 231, 276, 255, 260, 241, 238, 257, 232, 240, 241, 301, 256, 245, 248, 248, 264, 264, 249, 246, 277, 241, 256, 268, 253, 271, 253, 282, 251, 287, 261, 244, 287, 247, 254, 220, 230, 282, 252, 239, 277, 263, 248, 259, 243, 251, 252, 266, 249, 257, 242, 256, 280, 248, 259, 261, 249, 239, 270, 259, 283, 250, 259, 235, 248, 282, 268] payload_rows_per_dst=[5516, 5438, 5432, 5510, 5366, 5608, 5474, 5504] dup_rows_per_dst=[2750, 2702, 2722, 2680, 2712, 2748, 2586, 2788] combine_rows_per_src=[5516, 5438, 5432, 5510, 5366, 5608, 5474, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38354 dup_rows=21722 dup_rows_remote=19026 dedup_ratio=0.331451 router_tokens_per_expert=[265, 243, 255, 289, 243, 268, 251, 234, 275, 261, 270, 257, 247, 247, 250, 258, 239, 276, 258, 253, 244, 238, 264, 228, 257, 275, 253, 251, 264, 250, 267, 286, 231, 265, 278, 246, 295, 256, 246, 262, 246, 265, 242, 240, 274, 260, 252, 258, 263, 233, 277, 256, 253, 262, 289, 249, 259, 221, 289, 247, 269, 255, 254, 250, 267, 275, 244, 241, 270, 251, 259, 270, 210, 250, 236, 253, 275, 241, 240, 263, 264, 264, 268, 258, 267, 283, 252, 260, 252, 246, 286, 267, 259, 256, 259, 264, 256, 232, 255, 278, 270, 262, 281, 255, 250, 219, 260, 240, 254, 264, 251, 241, 252, 251, 260, 253, 252, 273, 230, 236, 251, 251, 235, 244, 222, 263, 252, 267] payload_rows_per_dst=[5482, 5336, 5510, 5292, 5392, 5776, 5566, 5460] dup_rows_per_dst=[2744, 2558, 2722, 2434, 2526, 3212, 2830, 2696] combine_rows_per_src=[5482, 5336, 5510, 5292, 5392, 5776, 5566, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156598272 dispatch_meta_bytes=229400 combine_payload_bytes=156598272 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156827672 ingress_bytes=194347008 link_bytes=194347008 token_bytes=313425944 weight_movement_bytes=37748736 total_nvlink_bytes=351174680 weight_to_token_ratio=0.120439 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862137406e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=229016 combine_payload_bytes=156401664 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=156630680 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032344 weight_movement_bytes=18874368 total_nvlink_bytes=331906712 weight_to_token_ratio=0.060295 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156631040 dispatch_meta_bytes=228720 combine_payload_bytes=156631040 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156859760 ingress_bytes=194379776 link_bytes=194379776 token_bytes=313490800 weight_movement_bytes=37748736 total_nvlink_bytes=351239536 weight_to_token_ratio=0.120414 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862451373e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38326 dup_rows=21674 dup_rows_remote=18906 dedup_ratio=0.330719 router_tokens_per_expert=[242, 265, 277, 263, 257, 275, 246, 254, 236, 274, 251, 286, 263, 255, 251, 257, 270, 285, 230, 225, 237, 260, 222, 239, 267, 222, 244, 264, 245, 272, 263, 258, 249, 248, 266, 266, 265, 235, 271, 258, 253, 251, 267, 243, 263, 244, 272, 246, 242, 258, 249, 273, 272, 240, 239, 245, 255, 245, 263, 276, 263, 280, 234, 274, 262, 257, 254, 254, 253, 264, 258, 272, 245, 254, 256, 273, 240, 265, 253, 260, 244, 278, 241, 260, 258, 245, 272, 239, 253, 269, 231, 238, 273, 259, 277, 255, 268, 242, 253, 249, 260, 279, 275, 249, 243, 252, 252, 246, 257, 245, 254, 264, 282, 250, 265, 245, 261, 267, 245, 247, 243, 269, 243, 245, 256, 266, 282, 242] payload_rows_per_dst=[5536, 5442, 5494, 5500, 5580, 5482, 5348, 5480] dup_rows_per_dst=[2768, 2594, 2700, 2716, 2920, 2702, 2568, 2706] combine_rows_per_src=[5536, 5442, 5494, 5500, 5580, 5482, 5348, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43850 payload_rows_remote=38366 dup_rows=21686 dup_rows_remote=18816 dedup_ratio=0.330902 router_tokens_per_expert=[229, 268, 250, 233, 259, 239, 255, 259, 269, 270, 235, 242, 251, 232, 243, 272, 277, 259, 242, 282, 257, 254, 289, 265, 256, 294, 253, 270, 253, 243, 260, 223, 244, 254, 252, 258, 240, 272, 280, 256, 248, 254, 248, 289, 256, 227, 246, 267, 277, 249, 266, 257, 238, 270, 275, 230, 292, 250, 287, 267, 260, 249, 246, 233, 232, 259, 263, 275, 292, 253, 246, 253, 266, 263, 277, 291, 262, 235, 258, 236, 224, 231, 286, 230, 253, 263, 254, 247, 256, 239, 234, 257, 243, 250, 246, 238, 276, 253, 252, 247, 246, 278, 256, 253, 268, 247, 260, 287, 292, 256, 256, 261, 231, 227, 259, 259, 271, 260, 261, 265, 230, 243, 243, 251, 281, 255, 249, 263] payload_rows_per_dst=[5392, 5484, 5462, 5556, 5522, 5398, 5600, 5436] dup_rows_per_dst=[2620, 2870, 2720, 2736, 2800, 2504, 2776, 2660] combine_rows_per_src=[5392, 5484, 5462, 5556, 5522, 5398, 5600, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43646 payload_rows_remote=38194 dup_rows=21890 dup_rows_remote=19038 dedup_ratio=0.334015 router_tokens_per_expert=[265, 255, 257, 255, 278, 270, 238, 254, 250, 232, 227, 256, 260, 253, 238, 237, 270, 260, 250, 259, 261, 253, 245, 284, 279, 251, 251, 242, 251, 248, 236, 277, 249, 272, 258, 265, 250, 249, 268, 242, 232, 243, 264, 280, 248, 271, 250, 268, 255, 258, 253, 243, 260, 243, 260, 233, 279, 265, 250, 282, 248, 252, 259, 281, 250, 280, 271, 240, 249, 273, 250, 231, 266, 249, 258, 270, 263, 260, 267, 275, 247, 257, 242, 269, 269, 259, 250, 252, 258, 302, 227, 255, 256, 294, 271, 243, 240, 239, 255, 237, 256, 246, 242, 253, 251, 253, 265, 261, 217, 220, 261, 267, 292, 251, 242, 253, 254, 251, 257, 281, 259, 254, 266, 255, 249, 270, 244, 252] payload_rows_per_dst=[5414, 5480, 5496, 5450, 5452, 5514, 5358, 5482] dup_rows_per_dst=[2636, 2754, 2722, 2792, 2852, 2788, 2568, 2778] combine_rows_per_src=[5414, 5480, 5496, 5450, 5452, 5514, 5358, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157097984 dispatch_meta_bytes=229520 combine_payload_bytes=157097984 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157327504 ingress_bytes=175972352 link_bytes=175972352 token_bytes=314425488 weight_movement_bytes=18874368 total_nvlink_bytes=333299856 weight_to_token_ratio=0.060028 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686080493e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38226 dup_rows=21806 dup_rows_remote=19066 dedup_ratio=0.332733 router_tokens_per_expert=[244, 233, 262, 254, 275, 265, 246, 259, 301, 239, 261, 270, 228, 258, 265, 229, 250, 281, 259, 263, 259, 234, 241, 270, 252, 283, 265, 244, 252, 265, 249, 256, 271, 263, 259, 271, 223, 243, 247, 234, 240, 233, 254, 256, 274, 251, 247, 288, 251, 275, 261, 252, 242, 246, 271, 268, 264, 249, 251, 275, 247, 278, 281, 269, 248, 262, 227, 248, 264, 255, 242, 279, 223, 241, 246, 247, 248, 270, 260, 276, 253, 264, 256, 258, 255, 246, 252, 262, 269, 233, 254, 249, 274, 236, 234, 287, 257, 245, 237, 280, 267, 251, 261, 254, 280, 263, 255, 273, 258, 257, 247, 237, 244, 247, 279, 233, 255, 240, 290, 271, 215, 267, 279, 240, 263, 277, 245, 237] payload_rows_per_dst=[5490, 5482, 5440, 5496, 5442, 5428, 5504, 5448] dup_rows_per_dst=[2688, 2764, 2668, 2856, 2630, 2744, 2740, 2716] combine_rows_per_src=[5490, 5482, 5440, 5496, 5442, 5428, 5504, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=36 plans=1 dispatch_payload_bytes=156983296 dispatch_meta_bytes=228928 combine_payload_bytes=156983296 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=157212224 ingress_bytes=232480768 link_bytes=232480768 token_bytes=314195520 weight_movement_bytes=75497472 total_nvlink_bytes=389692992 weight_to_token_ratio=0.240288 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.227516331e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157147136 dispatch_meta_bytes=228728 combine_payload_bytes=157147136 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157375864 ingress_bytes=213770240 link_bytes=213770240 token_bytes=314523000 weight_movement_bytes=56623104 total_nvlink_bytes=371146104 weight_to_token_ratio=0.180029 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.048241258e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=228928 combine_payload_bytes=156442624 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=156671552 ingress_bytes=213065728 link_bytes=213065728 token_bytes=313114176 weight_movement_bytes=56623104 total_nvlink_bytes=369737280 weight_to_token_ratio=0.180839 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041490971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43992 payload_rows_remote=38528 dup_rows=21544 dup_rows_remote=18888 dedup_ratio=0.328735 router_tokens_per_expert=[240, 264, 273, 255, 259, 273, 265, 278, 258, 230, 259, 276, 271, 260, 264, 272, 269, 251, 249, 237, 256, 249, 254, 272, 251, 254, 229, 242, 258, 261, 262, 258, 241, 242, 270, 286, 242, 252, 242, 256, 263, 271, 247, 235, 257, 238, 254, 264, 254, 244, 232, 253, 275, 266, 244, 261, 256, 244, 240, 241, 254, 263, 254, 255, 264, 235, 246, 257, 271, 250, 266, 258, 265, 256, 238, 273, 251, 243, 246, 243, 271, 242, 239, 244, 252, 258, 242, 275, 282, 269, 262, 251, 229, 238, 251, 242, 262, 243, 226, 277, 272, 251, 280, 260, 265, 253, 242, 271, 285, 245, 225, 281, 258, 271, 271, 265, 281, 249, 256, 273, 286, 239, 249, 251, 244, 248, 259, 276] payload_rows_per_dst=[5568, 5506, 5464, 5428, 5464, 5452, 5524, 5586] dup_rows_per_dst=[2826, 2598, 2656, 2644, 2660, 2642, 2752, 2766] combine_rows_per_src=[5568, 5506, 5464, 5428, 5464, 5452, 5524, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229168 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156802864 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376560 weight_movement_bytes=18874368 total_nvlink_bytes=332250928 weight_to_token_ratio=0.060229 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43924 payload_rows_remote=38398 dup_rows=21612 dup_rows_remote=18930 dedup_ratio=0.329773 router_tokens_per_expert=[265, 237, 257, 266, 254, 261, 252, 240, 298, 244, 278, 255, 256, 265, 254, 275, 262, 231, 268, 259, 251, 253, 270, 278, 248, 226, 271, 251, 282, 224, 276, 234, 271, 261, 253, 258, 234, 249, 249, 264, 263, 243, 234, 287, 266, 253, 220, 274, 237, 268, 259, 234, 259, 278, 281, 241, 234, 272, 257, 232, 278, 254, 252, 268, 256, 265, 222, 258, 263, 268, 264, 263, 264, 252, 253, 258, 262, 256, 233, 251, 252, 240, 259, 235, 284, 257, 237, 227, 250, 260, 253, 273, 257, 266, 268, 246, 266, 266, 239, 245, 258, 278, 245, 251, 268, 238, 271, 262, 243, 221, 245, 241, 274, 276, 264, 219, 257, 241, 263, 256, 251, 275, 267, 278, 239, 264, 265, 266] payload_rows_per_dst=[5520, 5488, 5450, 5526, 5480, 5478, 5442, 5540] dup_rows_per_dst=[2794, 2680, 2708, 2682, 2696, 2650, 2632, 2770] combine_rows_per_src=[5520, 5488, 5450, 5526, 5480, 5478, 5442, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43864 payload_rows_remote=38410 dup_rows=21672 dup_rows_remote=19024 dedup_ratio=0.330688 router_tokens_per_expert=[245, 253, 260, 254, 239, 238, 261, 265, 248, 254, 269, 269, 249, 253, 233, 274, 253, 282, 244, 288, 284, 243, 253, 257, 238, 258, 269, 265, 266, 248, 270, 220, 253, 298, 251, 256, 259, 258, 262, 258, 267, 263, 286, 253, 281, 233, 303, 244, 235, 253, 232, 229, 274, 260, 229, 242, 236, 253, 256, 282, 265, 242, 236, 258, 271, 252, 262, 242, 259, 270, 267, 238, 238, 256, 255, 278, 278, 254, 265, 242, 238, 269, 276, 244, 269, 248, 266, 252, 255, 248, 259, 232, 250, 257, 239, 249, 235, 258, 251, 254, 247, 297, 245, 246, 266, 249, 255, 250, 276, 282, 263, 242, 271, 243, 253, 270, 253, 283, 249, 253, 247, 230, 248, 230, 244, 253, 267, 271] payload_rows_per_dst=[5438, 5492, 5652, 5402, 5472, 5454, 5506, 5448] dup_rows_per_dst=[2690, 2784, 2798, 2562, 2782, 2648, 2726, 2682] combine_rows_per_src=[5438, 5492, 5652, 5402, 5472, 5454, 5506, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157810688 dispatch_meta_bytes=229664 combine_payload_bytes=157810688 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158040352 ingress_bytes=176685056 link_bytes=176685056 token_bytes=315851040 weight_movement_bytes=18874368 total_nvlink_bytes=334725408 weight_to_token_ratio=0.059757 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.692909272e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43762 payload_rows_remote=38176 dup_rows=21774 dup_rows_remote=18990 dedup_ratio=0.332245 router_tokens_per_expert=[265, 235, 246, 258, 257, 246, 256, 251, 243, 244, 257, 235, 255, 249, 266, 256, 258, 275, 233, 271, 247, 261, 261, 262, 240, 233, 246, 275, 253, 238, 236, 255, 278, 263, 273, 254, 247, 274, 234, 274, 245, 265, 250, 259, 244, 252, 251, 269, 234, 263, 271, 243, 242, 224, 266, 252, 278, 255, 271, 269, 275, 245, 279, 257, 279, 263, 256, 260, 248, 268, 243, 267, 258, 251, 249, 282, 283, 237, 258, 256, 255, 261, 241, 262, 255, 270, 245, 253, 240, 266, 248, 248, 247, 266, 267, 267, 273, 259, 260, 266, 258, 227, 279, 230, 266, 266, 255, 257, 247, 253, 257, 263, 274, 251, 255, 244, 257, 223, 275, 255, 270, 239, 262, 269, 232, 284, 262, 232] payload_rows_per_dst=[5456, 5340, 5502, 5516, 5500, 5366, 5496, 5586] dup_rows_per_dst=[2582, 2702, 2756, 2732, 2816, 2620, 2782, 2784] combine_rows_per_src=[5456, 5340, 5502, 5516, 5500, 5366, 5496, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229312 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157507520 ingress_bytes=195026944 link_bytes=195026944 token_bytes=314785728 weight_movement_bytes=37748736 total_nvlink_bytes=352534464 weight_to_token_ratio=0.119919 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868652218e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157327360 dispatch_meta_bytes=229736 combine_payload_bytes=157327360 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157557096 ingress_bytes=176201728 link_bytes=176201728 token_bytes=314884456 weight_movement_bytes=18874368 total_nvlink_bytes=333758824 weight_to_token_ratio=0.059941 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688278261e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38184 dup_rows=21910 dup_rows_remote=19018 dedup_ratio=0.334320 router_tokens_per_expert=[248, 243, 277, 268, 259, 256, 252, 235, 272, 244, 266, 280, 257, 260, 242, 271, 245, 248, 262, 244, 299, 263, 252, 251, 240, 261, 260, 251, 237, 239, 278, 258, 229, 283, 263, 266, 267, 270, 249, 253, 281, 258, 283, 259, 257, 268, 251, 250, 225, 256, 273, 270, 276, 259, 265, 246, 248, 271, 234, 232, 268, 254, 244, 267, 250, 249, 236, 256, 231, 252, 268, 258, 243, 219, 247, 239, 256, 255, 213, 242, 255, 261, 288, 238, 271, 271, 250, 258, 291, 256, 267, 244, 257, 267, 221, 262, 261, 275, 249, 266, 257, 278, 242, 253, 264, 240, 251, 229, 243, 238, 250, 257, 265, 271, 295, 251, 220, 281, 241, 251, 271, 214, 281, 281, 273, 252, 245, 259] payload_rows_per_dst=[5442, 5530, 5478, 5512, 5512, 5348, 5254, 5550] dup_rows_per_dst=[2892, 2810, 2822, 2664, 2838, 2478, 2654, 2752] combine_rows_per_src=[5442, 5530, 5478, 5512, 5512, 5348, 5254, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43654 payload_rows_remote=38200 dup_rows=21882 dup_rows_remote=19210 dedup_ratio=0.333893 router_tokens_per_expert=[242, 260, 264, 249, 250, 243, 261, 242, 243, 275, 279, 260, 244, 226, 256, 238, 259, 256, 268, 263, 274, 234, 240, 266, 279, 225, 271, 236, 231, 257, 249, 255, 235, 273, 251, 215, 280, 278, 232, 249, 253, 238, 251, 262, 254, 256, 216, 244, 262, 247, 252, 275, 246, 249, 264, 241, 279, 246, 237, 302, 268, 259, 253, 251, 251, 292, 289, 240, 250, 261, 256, 281, 222, 262, 276, 237, 281, 272, 238, 257, 262, 271, 258, 268, 245, 264, 246, 234, 278, 255, 260, 224, 284, 264, 270, 253, 255, 230, 251, 260, 258, 239, 235, 236, 246, 230, 271, 248, 279, 256, 261, 266, 287, 268, 288, 235, 258, 260, 252, 249, 274, 255, 252, 270, 295, 266, 255, 269] payload_rows_per_dst=[5426, 5454, 5396, 5454, 5488, 5462, 5428, 5546] dup_rows_per_dst=[2638, 2672, 2578, 2808, 2842, 2810, 2614, 2920] combine_rows_per_src=[5426, 5454, 5396, 5454, 5488, 5462, 5428, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43768 payload_rows_remote=38268 dup_rows=21768 dup_rows_remote=18898 dedup_ratio=0.332153 router_tokens_per_expert=[265, 232, 271, 275, 245, 228, 267, 244, 270, 245, 271, 260, 262, 269, 258, 265, 252, 228, 254, 258, 238, 266, 228, 276, 249, 238, 282, 264, 283, 253, 261, 231, 232, 266, 290, 253, 277, 266, 250, 245, 260, 252, 285, 282, 262, 250, 249, 283, 243, 260, 237, 265, 258, 250, 263, 265, 252, 295, 249, 259, 260, 254, 262, 254, 268, 265, 287, 260, 276, 263, 270, 276, 235, 231, 245, 255, 270, 252, 260, 272, 235, 242, 247, 249, 229, 273, 275, 277, 240, 290, 263, 278, 258, 259, 248, 230, 236, 235, 238, 266, 269, 248, 264, 251, 212, 272, 250, 233, 274, 253, 237, 260, 250, 251, 245, 261, 239, 238, 267, 227, 235, 263, 241, 267, 249, 220, 249, 274] payload_rows_per_dst=[5478, 5462, 5558, 5488, 5500, 5494, 5380, 5408] dup_rows_per_dst=[2776, 2660, 2846, 2764, 2870, 2692, 2616, 2544] combine_rows_per_src=[5478, 5462, 5558, 5488, 5500, 5494, 5380, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156368896 dispatch_meta_bytes=228664 combine_payload_bytes=156368896 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156597560 ingress_bytes=175243264 link_bytes=175243264 token_bytes=312966456 weight_movement_bytes=18874368 total_nvlink_bytes=331840824 weight_to_token_ratio=0.060308 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679094731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43834 payload_rows_remote=38320 dup_rows=21702 dup_rows_remote=19040 dedup_ratio=0.331146 router_tokens_per_expert=[258, 226, 267, 245, 246, 254, 263, 226, 256, 259, 273, 248, 255, 292, 281, 258, 241, 277, 260, 274, 267, 231, 282, 237, 267, 256, 291, 266, 250, 268, 217, 260, 275, 252, 248, 256, 270, 226, 236, 228, 260, 270, 247, 259, 251, 249, 252, 255, 257, 265, 223, 211, 243, 251, 239, 299, 262, 257, 255, 249, 242, 264, 228, 261, 268, 269, 256, 259, 264, 244, 258, 278, 254, 271, 261, 278, 272, 267, 250, 226, 262, 261, 252, 269, 257, 235, 272, 264, 240, 232, 249, 256, 249, 265, 261, 231, 277, 262, 243, 257, 263, 273, 251, 240, 254, 249, 279, 250, 258, 243, 242, 247, 289, 251, 237, 281, 269, 268, 274, 253, 256, 243, 226, 240, 272, 269, 275, 256] payload_rows_per_dst=[5430, 5536, 5392, 5392, 5570, 5448, 5514, 5552] dup_rows_per_dst=[2784, 2752, 2676, 2620, 2780, 2662, 2662, 2766] combine_rows_per_src=[5430, 5536, 5392, 5392, 5570, 5448, 5514, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=37 plans=1 dispatch_payload_bytes=156401664 dispatch_meta_bytes=228808 combine_payload_bytes=156401664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156630472 ingress_bytes=175276032 link_bytes=175276032 token_bytes=313032136 weight_movement_bytes=18874368 total_nvlink_bytes=331906504 weight_to_token_ratio=0.060295 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679408697e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=228664 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156974392 ingress_bytes=194494464 link_bytes=194494464 token_bytes=313720120 weight_movement_bytes=37748736 total_nvlink_bytes=351468856 weight_to_token_ratio=0.120326 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863550257e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229640 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156696840 ingress_bytes=213090304 link_bytes=213090304 token_bytes=313164040 weight_movement_bytes=56623104 total_nvlink_bytes=369787144 weight_to_token_ratio=0.180810 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041726446e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43736 payload_rows_remote=38242 dup_rows=21800 dup_rows_remote=19030 dedup_ratio=0.332642 router_tokens_per_expert=[282, 264, 286, 273, 243, 265, 256, 253, 252, 252, 267, 243, 290, 256, 288, 248, 238, 239, 269, 256, 266, 263, 262, 232, 289, 229, 235, 239, 242, 254, 265, 247, 251, 265, 247, 253, 263, 246, 289, 237, 268, 260, 263, 242, 270, 254, 264, 260, 259, 257, 256, 224, 244, 236, 259, 266, 257, 239, 236, 268, 260, 268, 253, 252, 255, 241, 251, 242, 226, 247, 246, 259, 241, 243, 264, 261, 278, 261, 259, 224, 259, 277, 257, 254, 230, 251, 249, 266, 249, 270, 277, 228, 261, 239, 253, 273, 284, 245, 272, 257, 254, 260, 254, 241, 245, 269, 270, 275, 252, 266, 250, 261, 266, 252, 253, 247, 258, 273, 272, 266, 258, 244, 257, 235, 261, 250, 236, 285] payload_rows_per_dst=[5588, 5380, 5494, 5418, 5346, 5444, 5544, 5522] dup_rows_per_dst=[2848, 2670, 2770, 2650, 2650, 2742, 2766, 2704] combine_rows_per_src=[5588, 5380, 5494, 5418, 5346, 5444, 5544, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38298 dup_rows=21826 dup_rows_remote=19156 dedup_ratio=0.333038 router_tokens_per_expert=[244, 273, 252, 259, 271, 262, 248, 261, 252, 261, 249, 251, 249, 266, 259, 263, 261, 268, 245, 264, 250, 248, 251, 263, 250, 231, 244, 295, 254, 260, 241, 275, 255, 268, 260, 265, 251, 277, 248, 236, 252, 250, 266, 270, 263, 269, 264, 268, 255, 244, 258, 269, 253, 266, 221, 264, 270, 256, 264, 231, 249, 239, 249, 253, 278, 251, 278, 273, 243, 231, 281, 250, 258, 245, 225, 242, 232, 247, 279, 236, 235, 258, 241, 294, 259, 226, 281, 251, 268, 265, 271, 259, 248, 248, 256, 255, 286, 288, 276, 229, 278, 242, 260, 251, 240, 247, 247, 270, 257, 270, 232, 265, 252, 248, 242, 271, 266, 248, 239, 260, 245, 259, 267, 243, 279, 269, 231, 224] payload_rows_per_dst=[5514, 5450, 5510, 5412, 5472, 5430, 5470, 5452] dup_rows_per_dst=[2726, 2750, 2814, 2670, 2626, 2800, 2806, 2634] combine_rows_per_src=[5514, 5450, 5510, 5412, 5472, 5430, 5470, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=229440 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157188160 ingress_bytes=175833088 link_bytes=175833088 token_bytes=314146880 weight_movement_bytes=18874368 total_nvlink_bytes=333021248 weight_to_token_ratio=0.060081 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684746134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43920 payload_rows_remote=38500 dup_rows=21616 dup_rows_remote=18988 dedup_ratio=0.329834 router_tokens_per_expert=[252, 265, 258, 243, 249, 242, 262, 294, 235, 247, 242, 261, 260, 257, 243, 227, 268, 239, 251, 284, 283, 273, 285, 272, 279, 237, 296, 239, 270, 261, 276, 246, 267, 259, 261, 232, 263, 261, 234, 272, 267, 251, 266, 251, 282, 222, 264, 232, 245, 235, 274, 264, 249, 261, 253, 244, 238, 265, 279, 250, 290, 222, 245, 269, 262, 240, 260, 239, 236, 252, 274, 270, 251, 257, 238, 260, 272, 255, 227, 244, 248, 258, 249, 270, 231, 231, 240, 280, 242, 256, 247, 247, 244, 259, 250, 272, 269, 233, 255, 252, 263, 226, 290, 264, 249, 278, 264, 250, 246, 280, 256, 269, 260, 272, 270, 223, 250, 258, 249, 262, 262, 244, 241, 254, 279, 259, 256, 261] payload_rows_per_dst=[5410, 5578, 5496, 5504, 5472, 5420, 5536, 5504] dup_rows_per_dst=[2664, 2940, 2672, 2662, 2602, 2628, 2752, 2696] combine_rows_per_src=[5410, 5578, 5496, 5504, 5472, 5420, 5536, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229088 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156868320 ingress_bytes=175513600 link_bytes=175513600 token_bytes=313507552 weight_movement_bytes=18874368 total_nvlink_bytes=332381920 weight_to_token_ratio=0.060204 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681684957e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=156868608 dispatch_meta_bytes=229816 combine_payload_bytes=156868608 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157098424 ingress_bytes=175742976 link_bytes=175742976 token_bytes=313967032 weight_movement_bytes=18874368 total_nvlink_bytes=332841400 weight_to_token_ratio=0.060116 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.683882725e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38204 dup_rows=21840 dup_rows_remote=19032 dedup_ratio=0.333252 router_tokens_per_expert=[237, 244, 245, 245, 255, 260, 251, 225, 226, 249, 259, 269, 282, 267, 260, 267, 249, 292, 255, 270, 270, 235, 274, 249, 268, 235, 256, 255, 249, 248, 266, 252, 241, 265, 260, 234, 240, 231, 269, 269, 255, 228, 261, 240, 248, 241, 225, 275, 241, 273, 272, 271, 276, 242, 271, 232, 258, 247, 259, 269, 285, 240, 262, 259, 260, 244, 259, 241, 248, 233, 270, 269, 242, 250, 270, 252, 238, 233, 246, 261, 284, 266, 266, 245, 237, 266, 243, 268, 234, 268, 264, 257, 263, 256, 270, 265, 262, 283, 264, 256, 258, 255, 264, 227, 221, 273, 291, 257, 267, 256, 257, 268, 244, 224, 257, 273, 244, 239, 260, 279, 272, 262, 264, 249, 252, 285, 262, 272] payload_rows_per_dst=[5356, 5286, 5544, 5538, 5454, 5508, 5518, 5492] dup_rows_per_dst=[2726, 2478, 2840, 2814, 2578, 2796, 2800, 2808] combine_rows_per_src=[5356, 5286, 5544, 5538, 5454, 5508, 5518, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=157696000 dispatch_meta_bytes=229952 combine_payload_bytes=157696000 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157925952 ingress_bytes=214319104 link_bytes=214319104 token_bytes=315621952 weight_movement_bytes=56623104 total_nvlink_bytes=372245056 weight_to_token_ratio=0.179402 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.053500203e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43742 payload_rows_remote=38098 dup_rows=21794 dup_rows_remote=18876 dedup_ratio=0.332550 router_tokens_per_expert=[254, 273, 243, 267, 256, 236, 250, 261, 273, 260, 258, 224, 251, 242, 248, 249, 254, 247, 254, 244, 258, 259, 252, 272, 263, 251, 242, 222, 260, 257, 267, 274, 265, 273, 262, 237, 282, 276, 256, 260, 247, 264, 261, 241, 232, 280, 259, 244, 275, 267, 250, 243, 258, 260, 231, 255, 253, 259, 259, 230, 237, 265, 258, 247, 283, 261, 244, 269, 277, 256, 259, 257, 270, 254, 266, 255, 252, 239, 259, 259, 265, 274, 236, 243, 227, 264, 264, 253, 231, 249, 251, 233, 252, 244, 245, 247, 243, 269, 266, 253, 258, 249, 269, 249, 260, 231, 235, 267, 251, 299, 269, 302, 242, 236, 271, 285, 258, 279, 256, 255, 272, 256, 264, 267, 249, 236, 272, 255] payload_rows_per_dst=[5644, 5488, 5510, 5412, 5516, 5334, 5522, 5316] dup_rows_per_dst=[2918, 2690, 2802, 2682, 2804, 2596, 2784, 2518] combine_rows_per_src=[5644, 5488, 5510, 5412, 5516, 5334, 5522, 5316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43640 payload_rows_remote=38158 dup_rows=21896 dup_rows_remote=19130 dedup_ratio=0.334106 router_tokens_per_expert=[268, 224, 262, 252, 266, 233, 247, 277, 259, 247, 270, 235, 237, 259, 273, 233, 241, 247, 280, 247, 270, 237, 249, 269, 245, 257, 266, 245, 283, 249, 261, 281, 246, 242, 253, 277, 278, 254, 248, 268, 260, 284, 257, 254, 236, 251, 265, 260, 242, 246, 240, 260, 263, 262, 252, 270, 232, 260, 271, 259, 279, 215, 253, 270, 243, 241, 276, 262, 275, 270, 273, 266, 253, 276, 251, 234, 242, 254, 251, 257, 266, 250, 252, 243, 246, 255, 282, 274, 239, 270, 272, 268, 246, 239, 246, 246, 245, 255, 268, 255, 263, 255, 260, 253, 275, 262, 262, 255, 239, 258, 238, 263, 246, 238, 264, 263, 214, 252, 285, 273, 255, 269, 267, 231, 278, 252, 250, 231] payload_rows_per_dst=[5356, 5430, 5572, 5348, 5482, 5554, 5462, 5436] dup_rows_per_dst=[2728, 2824, 2694, 2800, 2766, 2634, 2750, 2700] combine_rows_per_src=[5356, 5430, 5572, 5348, 5482, 5554, 5462, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43772 payload_rows_remote=38280 dup_rows=21764 dup_rows_remote=18968 dedup_ratio=0.332092 router_tokens_per_expert=[243, 259, 277, 247, 235, 267, 254, 235, 248, 274, 242, 247, 233, 254, 270, 288, 286, 257, 230, 221, 261, 276, 274, 257, 234, 252, 238, 285, 273, 270, 259, 271, 234, 256, 258, 244, 297, 270, 234, 249, 213, 262, 220, 261, 272, 248, 268, 239, 250, 248, 254, 271, 262, 250, 269, 250, 258, 250, 274, 265, 249, 260, 301, 261, 267, 267, 232, 235, 254, 277, 269, 277, 262, 246, 265, 255, 263, 225, 249, 249, 231, 243, 232, 261, 261, 280, 286, 250, 267, 258, 250, 264, 254, 252, 246, 236, 261, 246, 237, 253, 265, 240, 286, 258, 258, 260, 257, 288, 251, 237, 240, 247, 265, 236, 263, 244, 267, 260, 263, 268, 260, 230, 249, 254, 254, 236, 285, 273] payload_rows_per_dst=[5426, 5492, 5498, 5524, 5432, 5444, 5470, 5486] dup_rows_per_dst=[2720, 2796, 2552, 2820, 2752, 2698, 2698, 2728] combine_rows_per_src=[5426, 5492, 5498, 5524, 5432, 5444, 5470, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156483584 dispatch_meta_bytes=228944 combine_payload_bytes=156483584 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156712528 ingress_bytes=194232320 link_bytes=194232320 token_bytes=313196112 weight_movement_bytes=37748736 total_nvlink_bytes=350944848 weight_to_token_ratio=0.120527 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.861038522e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=38 plans=1 dispatch_payload_bytes=156049408 dispatch_meta_bytes=227896 combine_payload_bytes=156049408 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156277304 ingress_bytes=174923776 link_bytes=174923776 token_bytes=312326712 weight_movement_bytes=18874368 total_nvlink_bytes=331201080 weight_to_token_ratio=0.060431 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676033554e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43980 payload_rows_remote=38388 dup_rows=21556 dup_rows_remote=18774 dedup_ratio=0.328918 router_tokens_per_expert=[261, 272, 276, 242, 260, 250, 233, 232, 247, 289, 255, 229, 252, 244, 245, 249, 246, 252, 287, 244, 295, 279, 272, 239, 249, 264, 253, 259, 266, 273, 262, 234, 253, 275, 263, 242, 277, 230, 239, 247, 268, 253, 243, 222, 246, 253, 270, 281, 271, 258, 257, 246, 247, 269, 300, 245, 259, 234, 271, 227, 235, 233, 240, 254, 248, 290, 247, 244, 231, 236, 264, 273, 224, 247, 247, 264, 269, 236, 263, 241, 238, 231, 253, 277, 263, 262, 263, 267, 246, 249, 260, 262, 263, 248, 263, 261, 288, 251, 261, 253, 261, 268, 272, 264, 235, 289, 256, 272, 242, 259, 273, 243, 256, 255, 225, 244, 258, 281, 287, 246, 262, 272, 289, 246, 219, 270, 238, 285] payload_rows_per_dst=[5434, 5516, 5464, 5510, 5448, 5502, 5592, 5514] dup_rows_per_dst=[2638, 2832, 2660, 2582, 2600, 2710, 2782, 2752] combine_rows_per_src=[5434, 5516, 5464, 5510, 5448, 5502, 5592, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156295168 dispatch_meta_bytes=229152 combine_payload_bytes=156295168 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156524320 ingress_bytes=175169536 link_bytes=175169536 token_bytes=312819488 weight_movement_bytes=18874368 total_nvlink_bytes=331693856 weight_to_token_ratio=0.060336 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.678388305e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156794880 dispatch_meta_bytes=228992 combine_payload_bytes=156794880 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157023872 ingress_bytes=213417984 link_bytes=213417984 token_bytes=313818752 weight_movement_bytes=56623104 total_nvlink_bytes=370441856 weight_to_token_ratio=0.180433 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.076389 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.044866114e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38274 dup_rows=21752 dup_rows_remote=18954 dedup_ratio=0.331909 router_tokens_per_expert=[255, 264, 252, 247, 242, 227, 272, 256, 252, 268, 252, 284, 251, 273, 259, 247, 239, 263, 250, 254, 266, 270, 274, 277, 239, 218, 251, 256, 245, 263, 251, 250, 285, 253, 267, 256, 249, 270, 279, 255, 244, 277, 247, 252, 242, 250, 269, 259, 267, 236, 250, 258, 273, 247, 273, 266, 262, 263, 254, 263, 252, 229, 248, 231, 252, 221, 260, 237, 263, 256, 272, 273, 261, 258, 273, 275, 281, 237, 255, 252, 263, 264, 247, 274, 258, 233, 260, 263, 252, 259, 247, 265, 259, 247, 261, 266, 257, 262, 248, 263, 288, 267, 269, 227, 259, 258, 253, 230, 248, 271, 245, 240, 263, 266, 235, 263, 262, 252, 261, 255, 238, 244, 252, 255, 233, 276, 239, 252] payload_rows_per_dst=[5446, 5508, 5510, 5506, 5454, 5448, 5416, 5496] dup_rows_per_dst=[2756, 2624, 2798, 2638, 2798, 2788, 2754, 2596] combine_rows_per_src=[5446, 5508, 5510, 5506, 5454, 5448, 5416, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44060 payload_rows_remote=38538 dup_rows=21476 dup_rows_remote=18754 dedup_ratio=0.327698 router_tokens_per_expert=[235, 250, 242, 255, 259, 245, 252, 233, 259, 250, 242, 257, 257, 267, 279, 254, 260, 248, 230, 289, 253, 269, 261, 236, 273, 284, 276, 263, 224, 272, 277, 280, 251, 253, 267, 271, 252, 287, 261, 229, 270, 264, 256, 239, 259, 232, 285, 246, 248, 228, 259, 281, 251, 255, 256, 273, 249, 282, 255, 256, 242, 258, 259, 270, 248, 268, 260, 247, 246, 251, 269, 243, 261, 247, 262, 238, 243, 248, 263, 235, 254, 240, 227, 238, 263, 240, 269, 259, 275, 272, 245, 268, 277, 252, 260, 247, 231, 251, 288, 255, 250, 259, 248, 275, 260, 284, 232, 249, 237, 261, 254, 226, 278, 269, 264, 250, 251, 264, 260, 251, 235, 263, 254, 260, 244, 262, 239, 274] payload_rows_per_dst=[5500, 5530, 5586, 5522, 5504, 5484, 5344, 5590] dup_rows_per_dst=[2572, 2860, 2658, 2722, 2554, 2688, 2776, 2646] combine_rows_per_src=[5500, 5530, 5586, 5522, 5504, 5484, 5344, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43788 payload_rows_remote=38160 dup_rows=21748 dup_rows_remote=18874 dedup_ratio=0.331848 router_tokens_per_expert=[270, 258, 284, 277, 256, 262, 246, 222, 255, 258, 272, 259, 241, 277, 279, 256, 233, 226, 267, 239, 271, 231, 248, 275, 265, 256, 262, 256, 261, 262, 243, 266, 259, 267, 241, 229, 269, 245, 248, 228, 251, 254, 260, 242, 284, 274, 274, 266, 270, 261, 244, 245, 232, 265, 228, 260, 255, 269, 235, 224, 298, 270, 240, 247, 247, 266, 289, 223, 267, 251, 221, 266, 265, 244, 229, 268, 266, 249, 243, 284, 258, 247, 276, 269, 283, 254, 257, 286, 281, 260, 243, 263, 272, 265, 278, 259, 252, 229, 267, 271, 229, 254, 239, 226, 283, 252, 252, 281, 230, 283, 250, 241, 220, 227, 262, 237, 268, 237, 274, 251, 257, 266, 257, 251, 248, 267, 260, 251] payload_rows_per_dst=[5516, 5438, 5504, 5446, 5468, 5628, 5398, 5390] dup_rows_per_dst=[2828, 2684, 2678, 2640, 2688, 2874, 2680, 2676] combine_rows_per_src=[5516, 5438, 5504, 5446, 5468, 5628, 5398, 5390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157237248 dispatch_meta_bytes=228648 combine_payload_bytes=157237248 prefetch_bytes=0 reduce_bytes=18874368 egress_bytes=157465896 ingress_bytes=176111616 link_bytes=176111616 token_bytes=314703144 weight_movement_bytes=18874368 total_nvlink_bytes=333577512 weight_to_token_ratio=0.059975 gemm_rows_defined=33536 gemm_rows_charged=36864 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.090278 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687414852e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43464 payload_rows_remote=37862 dup_rows=22072 dup_rows_remote=19144 dedup_ratio=0.336792 router_tokens_per_expert=[243, 268, 246, 253, 264, 240, 244, 274, 268, 254, 246, 231, 265, 248, 241, 303, 260, 257, 263, 263, 265, 258, 262, 275, 229, 240, 254, 222, 265, 242, 269, 272, 259, 269, 237, 229, 295, 240, 259, 240, 236, 250, 232, 244, 238, 251, 229, 255, 268, 245, 273, 261, 254, 258, 243, 269, 252, 240, 242, 256, 253, 253, 260, 274, 252, 262, 236, 260, 273, 271, 295, 263, 245, 255, 256, 286, 237, 287, 239, 238, 229, 272, 265, 276, 262, 292, 256, 261, 292, 251, 227, 260, 248, 268, 228, 260, 266, 250, 266, 254, 258, 265, 257, 293, 249, 265, 263, 244, 237, 259, 268, 264, 249, 245, 258, 295, 256, 243, 245, 235, 241, 263, 266, 242, 244, 235, 281, 262] payload_rows_per_dst=[5442, 5420, 5270, 5246, 5480, 5432, 5572, 5602] dup_rows_per_dst=[2734, 2692, 2656, 2518, 2908, 2862, 2774, 2928] combine_rows_per_src=[5442, 5420, 5270, 5246, 5480, 5432, 5572, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156770304 dispatch_meta_bytes=228912 combine_payload_bytes=156770304 prefetch_bytes=0 reduce_bytes=94371840 egress_bytes=156999216 ingress_bytes=251142144 link_bytes=251142144 token_bytes=313769520 weight_movement_bytes=94371840 total_nvlink_bytes=408141360 weight_to_token_ratio=0.300768 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.086806 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.406320454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157851648 dispatch_meta_bytes=229168 combine_payload_bytes=157851648 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=158080816 ingress_bytes=176726016 link_bytes=176726016 token_bytes=315932464 weight_movement_bytes=18874368 total_nvlink_bytes=334806832 weight_to_token_ratio=0.059742 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.693301731e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=156303360 dispatch_meta_bytes=228136 combine_payload_bytes=156303360 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156531496 ingress_bytes=194052096 link_bytes=194052096 token_bytes=312834856 weight_movement_bytes=37748736 total_nvlink_bytes=350583592 weight_to_token_ratio=0.120667 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859311704e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38342 dup_rows=21692 dup_rows_remote=18862 dedup_ratio=0.330994 router_tokens_per_expert=[265, 271, 257, 231, 240, 276, 249, 253, 268, 242, 230, 277, 255, 268, 272, 255, 259, 264, 249, 243, 251, 246, 283, 278, 243, 266, 251, 254, 271, 239, 230, 238, 249, 261, 301, 268, 233, 234, 294, 234, 248, 247, 251, 252, 281, 232, 244, 289, 259, 275, 257, 258, 260, 246, 272, 257, 242, 266, 254, 251, 256, 275, 283, 244, 267, 243, 283, 266, 279, 235, 274, 238, 265, 258, 273, 262, 270, 248, 223, 249, 257, 223, 262, 272, 230, 232, 254, 262, 253, 268, 276, 265, 233, 240, 266, 244, 244, 255, 246, 261, 249, 241, 242, 243, 269, 245, 259, 266, 243, 254, 271, 262, 270, 236, 252, 269, 257, 264, 262, 246, 234, 249, 257, 265, 258, 263, 244, 275] payload_rows_per_dst=[5502, 5460, 5438, 5546, 5458, 5530, 5388, 5522] dup_rows_per_dst=[2830, 2676, 2704, 2764, 2782, 2544, 2712, 2680] combine_rows_per_src=[5502, 5460, 5438, 5546, 5458, 5530, 5388, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38418 dup_rows=21746 dup_rows_remote=18944 dedup_ratio=0.331818 router_tokens_per_expert=[252, 286, 245, 254, 272, 270, 234, 266, 229, 269, 260, 276, 246, 254, 243, 260, 263, 249, 235, 231, 244, 251, 257, 258, 246, 320, 246, 276, 269, 266, 231, 260, 242, 266, 246, 260, 220, 272, 281, 275, 216, 254, 253, 259, 251, 279, 272, 278, 263, 287, 268, 249, 241, 247, 252, 251, 275, 261, 245, 262, 235, 263, 271, 278, 267, 229, 258, 269, 248, 241, 249, 251, 261, 261, 256, 242, 252, 271, 256, 276, 261, 256, 249, 283, 251, 237, 254, 245, 250, 240, 257, 261, 239, 269, 238, 264, 254, 254, 248, 230, 282, 260, 248, 224, 245, 244, 249, 267, 285, 296, 257, 264, 258, 227, 243, 268, 220, 262, 254, 251, 246, 290, 242, 268, 258, 259, 228, 256] payload_rows_per_dst=[5470, 5508, 5530, 5562, 5372, 5422, 5490, 5436] dup_rows_per_dst=[2762, 2696, 2718, 2734, 2802, 2686, 2724, 2624] combine_rows_per_src=[5470, 5508, 5530, 5562, 5372, 5422, 5490, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=155082752 dispatch_meta_bytes=228024 combine_payload_bytes=155082752 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155310776 ingress_bytes=173957120 link_bytes=173957120 token_bytes=310393528 weight_movement_bytes=18874368 total_nvlink_bytes=329267896 weight_to_token_ratio=0.060808 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.666771532e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38286 dup_rows=21722 dup_rows_remote=19072 dedup_ratio=0.331451 router_tokens_per_expert=[271, 247, 284, 257, 237, 247, 255, 268, 264, 273, 262, 251, 255, 264, 243, 253, 251, 260, 242, 259, 258, 250, 281, 240, 248, 270, 253, 284, 238, 251, 266, 238, 257, 272, 275, 232, 277, 253, 264, 249, 254, 251, 249, 261, 269, 242, 254, 222, 291, 251, 239, 248, 278, 255, 261, 278, 261, 263, 264, 297, 273, 266, 250, 244, 235, 267, 247, 246, 254, 262, 279, 266, 234, 273, 271, 221, 245, 269, 236, 240, 241, 254, 235, 269, 240, 265, 268, 246, 254, 245, 245, 262, 287, 263, 248, 237, 249, 267, 264, 268, 252, 286, 248, 244, 256, 238, 242, 282, 256, 255, 258, 293, 240, 250, 241, 242, 246, 258, 246, 232, 248, 228, 273, 270, 248, 244, 239, 281] payload_rows_per_dst=[5412, 5528, 5456, 5538, 5428, 5468, 5580, 5404] dup_rows_per_dst=[2850, 2650, 2706, 2900, 2662, 2650, 2736, 2568] combine_rows_per_src=[5412, 5528, 5456, 5538, 5428, 5468, 5580, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=39 plans=1 dispatch_payload_bytes=157048832 dispatch_meta_bytes=228816 combine_payload_bytes=157048832 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157277648 ingress_bytes=175923200 link_bytes=175923200 token_bytes=314326480 weight_movement_bytes=18874368 total_nvlink_bytes=333200848 weight_to_token_ratio=0.060047 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685609543e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157360128 dispatch_meta_bytes=229448 combine_payload_bytes=157360128 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157589576 ingress_bytes=176234496 link_bytes=176234496 token_bytes=314949704 weight_movement_bytes=18874368 total_nvlink_bytes=333824072 weight_to_token_ratio=0.059928 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688592228e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43590 payload_rows_remote=38168 dup_rows=21946 dup_rows_remote=19198 dedup_ratio=0.334869 router_tokens_per_expert=[270, 260, 243, 270, 280, 272, 244, 265, 283, 260, 255, 256, 255, 270, 249, 252, 266, 264, 242, 254, 254, 260, 259, 262, 245, 255, 274, 245, 250, 272, 245, 261, 263, 264, 258, 264, 262, 228, 257, 259, 291, 257, 270, 262, 261, 255, 227, 266, 241, 267, 243, 243, 243, 240, 273, 244, 233, 264, 258, 241, 248, 285, 267, 254, 240, 258, 278, 250, 239, 272, 260, 218, 250, 257, 248, 286, 273, 253, 270, 276, 255, 239, 259, 254, 249, 254, 214, 280, 254, 265, 269, 263, 250, 260, 226, 249, 232, 256, 264, 259, 254, 248, 267, 266, 245, 257, 252, 225, 271, 283, 253, 253, 277, 244, 241, 256, 248, 243, 277, 251, 268, 248, 254, 258, 218, 234, 259, 259] payload_rows_per_dst=[5476, 5412, 5470, 5442, 5516, 5420, 5422, 5432] dup_rows_per_dst=[2892, 2804, 2818, 2646, 2740, 2660, 2748, 2638] combine_rows_per_src=[5476, 5412, 5470, 5442, 5516, 5420, 5422, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156819456 dispatch_meta_bytes=229432 combine_payload_bytes=156819456 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157048888 ingress_bytes=194568192 link_bytes=194568192 token_bytes=313868344 weight_movement_bytes=37748736 total_nvlink_bytes=351617080 weight_to_token_ratio=0.120269 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.864256682e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43814 payload_rows_remote=38362 dup_rows=21722 dup_rows_remote=19144 dedup_ratio=0.331451 router_tokens_per_expert=[237, 246, 267, 255, 261, 235, 248, 262, 305, 262, 266, 278, 224, 274, 257, 232, 266, 259, 273, 253, 238, 236, 244, 272, 276, 251, 235, 263, 250, 260, 279, 261, 236, 251, 256, 232, 225, 255, 258, 278, 247, 232, 277, 244, 258, 251, 282, 233, 267, 271, 280, 250, 277, 252, 258, 253, 268, 270, 258, 267, 269, 246, 252, 267, 237, 247, 266, 259, 263, 220, 248, 241, 229, 247, 260, 241, 254, 277, 259, 286, 246, 273, 252, 271, 254, 257, 287, 253, 269, 252, 249, 266, 258, 259, 254, 264, 249, 243, 243, 265, 276, 278, 245, 252, 273, 270, 270, 242, 251, 247, 251, 222, 254, 242, 237, 265, 246, 257, 252, 253, 260, 275, 246, 255, 254, 263, 256, 233] payload_rows_per_dst=[5540, 5468, 5452, 5528, 5400, 5580, 5464, 5382] dup_rows_per_dst=[2678, 2764, 2578, 2882, 2668, 2748, 2690, 2714] combine_rows_per_src=[5540, 5468, 5452, 5528, 5400, 5580, 5464, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38122 dup_rows=21902 dup_rows_remote=19050 dedup_ratio=0.334198 router_tokens_per_expert=[240, 247, 283, 275, 251, 270, 206, 248, 230, 240, 226, 242, 253, 245, 267, 226, 264, 232, 269, 239, 268, 246, 250, 263, 227, 253, 261, 269, 267, 245, 255, 234, 275, 262, 268, 248, 246, 270, 257, 239, 282, 246, 279, 247, 245, 252, 270, 282, 254, 281, 248, 247, 260, 277, 262, 239, 265, 283, 270, 258, 294, 251, 244, 249, 250, 245, 265, 257, 270, 227, 260, 255, 241, 310, 265, 254, 248, 243, 225, 239, 256, 241, 269, 254, 231, 254, 280, 234, 246, 264, 242, 257, 254, 249, 266, 261, 252, 255, 284, 242, 280, 285, 260, 283, 263, 251, 247, 258, 248, 275, 250, 255, 235, 290, 266, 254, 266, 248, 275, 254, 245, 276, 234, 268, 263, 243, 257, 253] payload_rows_per_dst=[5308, 5410, 5522, 5512, 5436, 5460, 5506, 5480] dup_rows_per_dst=[2590, 2674, 2814, 2852, 2672, 2656, 2870, 2774] combine_rows_per_src=[5308, 5410, 5522, 5512, 5436, 5460, 5506, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43826 payload_rows_remote=38404 dup_rows=21710 dup_rows_remote=19064 dedup_ratio=0.331268 router_tokens_per_expert=[228, 245, 256, 267, 226, 242, 280, 275, 240, 266, 234, 271, 250, 225, 257, 229, 249, 261, 247, 270, 229, 266, 260, 271, 266, 276, 264, 241, 285, 257, 238, 267, 225, 256, 285, 228, 242, 256, 258, 225, 293, 252, 276, 256, 250, 246, 239, 272, 274, 269, 247, 242, 268, 235, 274, 241, 268, 276, 270, 266, 269, 278, 260, 240, 246, 240, 276, 246, 240, 249, 279, 250, 293, 234, 257, 274, 243, 281, 243, 283, 272, 234, 246, 258, 255, 266, 246, 255, 246, 243, 256, 258, 259, 241, 251, 248, 252, 269, 289, 268, 245, 284, 245, 268, 260, 231, 268, 241, 270, 251, 259, 269, 244, 236, 263, 264, 228, 242, 254, 276, 234, 256, 240, 273, 247, 264, 274, 262] payload_rows_per_dst=[5392, 5518, 5426, 5556, 5546, 5422, 5522, 5444] dup_rows_per_dst=[2590, 2776, 2692, 2798, 2722, 2646, 2816, 2670] combine_rows_per_src=[5392, 5518, 5426, 5556, 5546, 5422, 5522, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156336128 dispatch_meta_bytes=229464 combine_payload_bytes=156336128 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156565592 ingress_bytes=212959232 link_bytes=212959232 token_bytes=312901720 weight_movement_bytes=56623104 total_nvlink_bytes=369524824 weight_to_token_ratio=0.180961 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040470579e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=38220 dup_rows=21902 dup_rows_remote=19090 dedup_ratio=0.334198 router_tokens_per_expert=[263, 263, 229, 250, 267, 293, 252, 239, 270, 240, 265, 267, 246, 251, 257, 248, 257, 242, 250, 260, 252, 246, 254, 258, 249, 243, 225, 272, 254, 270, 248, 257, 240, 272, 290, 248, 249, 269, 261, 272, 247, 254, 249, 284, 244, 273, 255, 260, 238, 265, 237, 261, 275, 248, 248, 260, 249, 267, 276, 257, 275, 284, 264, 268, 244, 269, 269, 260, 259, 267, 246, 264, 254, 245, 278, 268, 248, 239, 267, 218, 250, 229, 253, 262, 257, 246, 242, 265, 238, 243, 246, 263, 247, 244, 254, 267, 236, 242, 267, 267, 251, 250, 271, 233, 240, 245, 259, 274, 263, 264, 262, 259, 264, 247, 294, 246, 277, 269, 257, 239, 249, 246, 259, 257, 250, 236, 252, 266] payload_rows_per_dst=[5528, 5398, 5452, 5520, 5306, 5404, 5612, 5414] dup_rows_per_dst=[2672, 2676, 2724, 2824, 2512, 2728, 2954, 2812] combine_rows_per_src=[5528, 5398, 5452, 5520, 5306, 5404, 5612, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157130752 dispatch_meta_bytes=230024 combine_payload_bytes=157130752 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157360776 ingress_bytes=194879488 link_bytes=194879488 token_bytes=314491528 weight_movement_bytes=37748736 total_nvlink_bytes=352240264 weight_to_token_ratio=0.120031 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.867239367e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=157302784 dispatch_meta_bytes=229872 combine_payload_bytes=157302784 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157532656 ingress_bytes=176177152 link_bytes=176177152 token_bytes=314835440 weight_movement_bytes=18874368 total_nvlink_bytes=333709808 weight_to_token_ratio=0.059950 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688042786e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156147712 dispatch_meta_bytes=228688 combine_payload_bytes=156147712 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156376400 ingress_bytes=231645184 link_bytes=231645184 token_bytes=312524112 weight_movement_bytes=75497472 total_nvlink_bytes=388021584 weight_to_token_ratio=0.241573 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.219510177e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43562 payload_rows_remote=38000 dup_rows=21974 dup_rows_remote=19128 dedup_ratio=0.335297 router_tokens_per_expert=[282, 229, 238, 259, 254, 285, 259, 239, 268, 228, 273, 266, 270, 258, 244, 264, 304, 247, 248, 249, 249, 253, 260, 278, 230, 265, 262, 244, 245, 262, 248, 267, 251, 240, 289, 266, 243, 239, 254, 270, 232, 255, 260, 254, 244, 264, 258, 261, 225, 253, 249, 257, 237, 237, 224, 242, 242, 241, 234, 275, 276, 254, 250, 261, 262, 276, 277, 244, 222, 271, 230, 261, 225, 252, 272, 284, 263, 257, 256, 250, 298, 266, 265, 261, 263, 277, 254, 270, 248, 274, 276, 243, 245, 241, 257, 257, 279, 252, 230, 264, 279, 263, 290, 257, 246, 247, 247, 258, 266, 241, 237, 256, 258, 246, 265, 263, 244, 264, 250, 272, 257, 287, 250, 269, 240, 228, 235, 267] payload_rows_per_dst=[5562, 5316, 5462, 5556, 5366, 5472, 5414, 5414] dup_rows_per_dst=[2846, 2736, 2692, 2752, 2496, 2866, 2810, 2776] combine_rows_per_src=[5562, 5316, 5462, 5556, 5366, 5472, 5414, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43644 payload_rows_remote=38182 dup_rows=21892 dup_rows_remote=19084 dedup_ratio=0.334045 router_tokens_per_expert=[224, 232, 243, 252, 282, 274, 234, 252, 262, 263, 258, 246, 254, 259, 272, 258, 231, 261, 280, 246, 250, 237, 273, 247, 269, 246, 273, 261, 251, 279, 245, 270, 275, 272, 244, 281, 259, 251, 247, 236, 250, 264, 250, 260, 266, 255, 244, 285, 240, 252, 257, 243, 249, 241, 265, 247, 253, 262, 246, 241, 256, 220, 266, 254, 256, 264, 275, 250, 290, 250, 287, 238, 249, 249, 248, 245, 241, 245, 288, 260, 236, 271, 246, 262, 280, 258, 253, 261, 240, 252, 271, 266, 278, 248, 279, 257, 235, 241, 273, 264, 256, 258, 282, 246, 261, 237, 263, 252, 255, 240, 231, 242, 257, 268, 259, 251, 255, 241, 244, 259, 256, 261, 289, 257, 275, 250, 235, 267] payload_rows_per_dst=[5460, 5500, 5498, 5298, 5462, 5514, 5408, 5504] dup_rows_per_dst=[2670, 2738, 2780, 2686, 2808, 2802, 2664, 2744] combine_rows_per_src=[5460, 5500, 5498, 5298, 5462, 5514, 5408, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38422 dup_rows=21680 dup_rows_remote=18948 dedup_ratio=0.330811 router_tokens_per_expert=[233, 237, 253, 237, 251, 240, 256, 268, 263, 257, 228, 252, 245, 255, 251, 243, 252, 258, 263, 250, 268, 258, 256, 261, 264, 241, 261, 246, 240, 254, 243, 268, 264, 265, 287, 260, 262, 234, 257, 239, 240, 280, 262, 245, 281, 258, 240, 278, 236, 260, 247, 248, 257, 274, 228, 244, 258, 250, 257, 222, 251, 278, 276, 249, 251, 262, 275, 284, 264, 247, 279, 243, 255, 293, 249, 235, 272, 277, 259, 236, 258, 247, 280, 262, 251, 260, 274, 248, 266, 252, 235, 235, 246, 257, 239, 222, 287, 291, 258, 296, 254, 233, 241, 251, 260, 245, 265, 275, 240, 252, 266, 251, 263, 255, 245, 277, 247, 246, 247, 276, 248, 268, 282, 243, 261, 276, 228, 289] payload_rows_per_dst=[5348, 5434, 5566, 5466, 5560, 5472, 5468, 5542] dup_rows_per_dst=[2590, 2732, 2738, 2604, 2802, 2592, 2862, 2760] combine_rows_per_src=[5348, 5434, 5566, 5466, 5560, 5472, 5468, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156549120 dispatch_meta_bytes=229240 combine_payload_bytes=156549120 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156778360 ingress_bytes=175423488 link_bytes=175423488 token_bytes=313327480 weight_movement_bytes=18874368 total_nvlink_bytes=332201848 weight_to_token_ratio=0.060238 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680821548e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=40 plans=1 dispatch_payload_bytes=155648000 dispatch_meta_bytes=228512 combine_payload_bytes=155648000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155876512 ingress_bytes=174522368 link_bytes=174522368 token_bytes=311524512 weight_movement_bytes=18874368 total_nvlink_bytes=330398880 weight_to_token_ratio=0.060587 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672187460e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156393472 dispatch_meta_bytes=229064 combine_payload_bytes=156393472 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156622536 ingress_bytes=213016576 link_bytes=213016576 token_bytes=313016008 weight_movement_bytes=56623104 total_nvlink_bytes=369639112 weight_to_token_ratio=0.180895 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.041020021e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=38132 dup_rows=21912 dup_rows_remote=19164 dedup_ratio=0.334351 router_tokens_per_expert=[267, 270, 269, 261, 262, 266, 236, 259, 246, 238, 237, 270, 233, 278, 254, 228, 251, 290, 242, 244, 251, 255, 260, 254, 250, 267, 237, 246, 249, 263, 260, 241, 252, 248, 262, 234, 244, 257, 246, 224, 288, 247, 259, 256, 274, 294, 242, 268, 266, 284, 256, 262, 214, 256, 264, 237, 263, 254, 267, 273, 251, 230, 248, 256, 246, 277, 268, 273, 228, 272, 236, 262, 263, 254, 250, 279, 261, 268, 242, 234, 271, 250, 234, 247, 256, 269, 270, 252, 268, 288, 230, 252, 246, 247, 248, 278, 266, 252, 251, 234, 246, 260, 271, 261, 272, 285, 252, 228, 259, 249, 267, 267, 276, 271, 266, 268, 239, 242, 265, 256, 227, 280, 234, 267, 261, 241, 247, 279] payload_rows_per_dst=[5414, 5372, 5484, 5458, 5422, 5494, 5492, 5488] dup_rows_per_dst=[2734, 2748, 2706, 2704, 2804, 2718, 2748, 2750] combine_rows_per_src=[5414, 5372, 5484, 5458, 5422, 5494, 5492, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=157376512 dispatch_meta_bytes=229480 combine_payload_bytes=157376512 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157605992 ingress_bytes=176250880 link_bytes=176250880 token_bytes=314982504 weight_movement_bytes=18874368 total_nvlink_bytes=333856872 weight_to_token_ratio=0.059922 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.688749211e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38242 dup_rows=21866 dup_rows_remote=19222 dedup_ratio=0.333649 router_tokens_per_expert=[255, 236, 232, 256, 255, 264, 267, 261, 250, 283, 246, 253, 249, 265, 252, 257, 247, 231, 278, 253, 269, 246, 251, 262, 242, 234, 264, 266, 236, 256, 241, 237, 253, 251, 277, 233, 258, 261, 259, 266, 264, 250, 244, 237, 236, 253, 240, 254, 271, 262, 238, 269, 258, 242, 245, 244, 253, 289, 234, 283, 255, 246, 245, 250, 246, 245, 257, 263, 261, 279, 262, 280, 290, 271, 253, 275, 261, 245, 266, 263, 250, 243, 258, 246, 253, 257, 289, 251, 255, 280, 262, 257, 237, 250, 274, 272, 250, 255, 249, 261, 247, 233, 271, 227, 253, 246, 226, 262, 258, 242, 280, 237, 272, 255, 254, 250, 253, 246, 298, 251, 252, 258, 263, 305, 255, 248, 267, 279] payload_rows_per_dst=[5460, 5338, 5428, 5500, 5518, 5498, 5362, 5566] dup_rows_per_dst=[2702, 2688, 2644, 2668, 2916, 2770, 2632, 2846] combine_rows_per_src=[5460, 5338, 5428, 5500, 5518, 5498, 5362, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38088 dup_rows=21918 dup_rows_remote=19198 dedup_ratio=0.334442 router_tokens_per_expert=[255, 287, 236, 252, 267, 260, 267, 247, 278, 248, 270, 267, 266, 236, 252, 275, 264, 242, 228, 256, 251, 252, 270, 280, 245, 284, 246, 254, 239, 273, 266, 254, 288, 263, 281, 237, 247, 257, 253, 252, 256, 269, 247, 222, 254, 254, 243, 265, 268, 287, 272, 246, 262, 260, 249, 262, 264, 268, 239, 291, 260, 242, 253, 245, 273, 239, 242, 253, 272, 272, 261, 261, 256, 243, 257, 256, 247, 271, 238, 249, 271, 244, 246, 267, 274, 258, 257, 261, 249, 251, 238, 247, 257, 292, 263, 250, 236, 244, 218, 241, 249, 245, 259, 228, 275, 241, 271, 253, 237, 257, 264, 282, 256, 244, 254, 236, 237, 278, 219, 250, 269, 260, 242, 277, 256, 262, 223, 267] payload_rows_per_dst=[5474, 5430, 5466, 5446, 5500, 5530, 5360, 5412] dup_rows_per_dst=[2852, 2778, 2710, 2890, 2680, 2720, 2640, 2648] combine_rows_per_src=[5474, 5430, 5466, 5446, 5500, 5530, 5360, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38268 dup_rows=21796 dup_rows_remote=19088 dedup_ratio=0.332581 router_tokens_per_expert=[279, 252, 252, 267, 238, 272, 279, 251, 242, 251, 265, 243, 243, 272, 266, 294, 247, 262, 271, 256, 293, 255, 241, 255, 252, 265, 258, 253, 217, 227, 244, 243, 267, 255, 244, 237, 246, 274, 241, 270, 252, 234, 245, 280, 243, 265, 256, 258, 276, 251, 265, 258, 252, 247, 256, 246, 252, 258, 257, 273, 274, 250, 237, 238, 262, 244, 260, 271, 253, 254, 262, 239, 272, 271, 268, 233, 239, 274, 225, 250, 268, 262, 274, 255, 241, 230, 268, 266, 230, 261, 259, 266, 246, 265, 242, 276, 264, 286, 230, 278, 241, 228, 248, 268, 262, 275, 246, 257, 241, 245, 276, 229, 275, 271, 257, 273, 257, 245, 228, 219, 256, 263, 264, 283, 277, 263, 266, 249] payload_rows_per_dst=[5520, 5434, 5468, 5472, 5432, 5476, 5412, 5526] dup_rows_per_dst=[2812, 2644, 2666, 2708, 2722, 2742, 2736, 2766] combine_rows_per_src=[5520, 5434, 5468, 5472, 5432, 5476, 5412, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=229184 combine_payload_bytes=156188672 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156417856 ingress_bytes=175063040 link_bytes=175063040 token_bytes=312606528 weight_movement_bytes=18874368 total_nvlink_bytes=331480896 weight_to_token_ratio=0.060377 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.677367913e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156639232 dispatch_meta_bytes=229856 combine_payload_bytes=156639232 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156869088 ingress_bytes=213262336 link_bytes=213262336 token_bytes=313508320 weight_movement_bytes=56623104 total_nvlink_bytes=370131424 weight_to_token_ratio=0.180611 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.043374772e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43542 payload_rows_remote=37940 dup_rows=21994 dup_rows_remote=19034 dedup_ratio=0.335602 router_tokens_per_expert=[215, 256, 260, 245, 245, 241, 263, 268, 234, 238, 251, 224, 261, 265, 262, 274, 246, 276, 280, 241, 263, 262, 253, 240, 262, 254, 227, 257, 249, 255, 256, 277, 235, 264, 263, 262, 270, 267, 283, 291, 252, 266, 288, 250, 264, 240, 255, 246, 252, 250, 279, 258, 243, 242, 245, 252, 239, 253, 289, 243, 240, 252, 228, 251, 278, 235, 243, 245, 232, 240, 242, 273, 254, 220, 254, 243, 278, 233, 252, 256, 258, 277, 251, 278, 252, 291, 275, 228, 244, 274, 260, 268, 257, 249, 274, 239, 255, 248, 286, 235, 255, 264, 237, 252, 257, 282, 273, 275, 271, 270, 263, 258, 279, 266, 255, 258, 258, 270, 266, 229, 248, 265, 265, 266, 242, 251, 258, 246] payload_rows_per_dst=[5300, 5520, 5462, 5372, 5366, 5330, 5590, 5602] dup_rows_per_dst=[2556, 2780, 2876, 2692, 2590, 2702, 2838, 2960] combine_rows_per_src=[5300, 5520, 5462, 5372, 5366, 5330, 5590, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156008448 dispatch_meta_bytes=229144 combine_payload_bytes=156008448 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156237592 ingress_bytes=174882816 link_bytes=174882816 token_bytes=312246040 weight_movement_bytes=18874368 total_nvlink_bytes=331120408 weight_to_token_ratio=0.060447 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675641095e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156745728 dispatch_meta_bytes=229424 combine_payload_bytes=156745728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156975152 ingress_bytes=175620096 link_bytes=175620096 token_bytes=313720880 weight_movement_bytes=18874368 total_nvlink_bytes=332595248 weight_to_token_ratio=0.060163 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682705349e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38374 dup_rows=21698 dup_rows_remote=18820 dedup_ratio=0.331085 router_tokens_per_expert=[251, 266, 269, 260, 231, 256, 252, 242, 238, 267, 258, 239, 245, 249, 299, 242, 242, 254, 264, 262, 246, 234, 267, 271, 276, 262, 276, 267, 272, 248, 255, 237, 294, 252, 263, 247, 270, 266, 256, 263, 256, 245, 265, 244, 233, 254, 221, 254, 267, 241, 261, 239, 247, 258, 247, 268, 242, 256, 250, 243, 281, 289, 266, 244, 249, 280, 269, 255, 279, 266, 232, 255, 252, 275, 240, 258, 252, 281, 250, 253, 233, 242, 262, 236, 253, 254, 246, 253, 260, 241, 250, 268, 281, 262, 280, 247, 232, 247, 267, 245, 263, 244, 266, 265, 245, 260, 255, 246, 267, 275, 269, 239, 256, 257, 214, 296, 237, 261, 274, 247, 280, 253, 262, 239, 261, 255, 241, 257] payload_rows_per_dst=[5464, 5556, 5502, 5448, 5696, 5428, 5364, 5380] dup_rows_per_dst=[2878, 2710, 2664, 2750, 2990, 2494, 2626, 2586] combine_rows_per_src=[5464, 5556, 5502, 5448, 5696, 5428, 5364, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43710 payload_rows_remote=38180 dup_rows=21826 dup_rows_remote=19122 dedup_ratio=0.333038 router_tokens_per_expert=[268, 259, 255, 234, 242, 241, 233, 240, 264, 267, 268, 268, 248, 267, 252, 262, 273, 232, 231, 254, 259, 263, 261, 231, 294, 252, 264, 233, 267, 257, 244, 251, 267, 244, 242, 241, 234, 246, 228, 251, 254, 262, 248, 255, 259, 276, 253, 271, 283, 258, 267, 266, 271, 232, 266, 254, 248, 255, 271, 252, 273, 268, 250, 268, 269, 241, 234, 239, 273, 251, 244, 248, 259, 269, 269, 261, 265, 271, 263, 261, 273, 274, 246, 244, 259, 239, 254, 265, 263, 277, 265, 236, 250, 263, 247, 246, 214, 225, 263, 247, 240, 263, 247, 253, 255, 273, 259, 242, 293, 269, 265, 273, 243, 242, 256, 256, 253, 269, 245, 249, 265, 288, 259, 264, 258, 264, 276, 235] payload_rows_per_dst=[5398, 5400, 5392, 5530, 5530, 5478, 5476, 5506] dup_rows_per_dst=[2738, 2732, 2670, 2834, 2704, 2724, 2686, 2738] combine_rows_per_src=[5398, 5400, 5392, 5530, 5530, 5478, 5476, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=155402240 dispatch_meta_bytes=227896 combine_payload_bytes=155402240 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155630136 ingress_bytes=174276608 link_bytes=174276608 token_bytes=311032376 weight_movement_bytes=18874368 total_nvlink_bytes=329906744 weight_to_token_ratio=0.060683 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.669832708e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43672 payload_rows_remote=38194 dup_rows=21864 dup_rows_remote=19088 dedup_ratio=0.333618 router_tokens_per_expert=[264, 261, 236, 255, 261, 251, 275, 247, 270, 244, 244, 273, 267, 271, 242, 254, 266, 260, 271, 270, 246, 226, 262, 277, 250, 269, 282, 244, 248, 233, 260, 263, 239, 251, 271, 239, 240, 264, 246, 245, 275, 251, 262, 276, 266, 249, 249, 251, 282, 257, 257, 251, 249, 240, 257, 273, 255, 282, 256, 262, 226, 245, 235, 259, 248, 265, 251, 242, 249, 248, 259, 245, 250, 262, 262, 279, 251, 282, 264, 275, 217, 263, 231, 267, 258, 259, 252, 251, 272, 255, 239, 218, 240, 252, 275, 260, 243, 268, 248, 250, 281, 239, 266, 274, 245, 232, 263, 272, 277, 245, 244, 261, 274, 235, 251, 258, 272, 271, 231, 239, 271, 272, 258, 252, 245, 267, 261, 260] payload_rows_per_dst=[5466, 5478, 5434, 5436, 5466, 5430, 5450, 5512] dup_rows_per_dst=[2764, 2776, 2714, 2736, 2798, 2588, 2766, 2722] combine_rows_per_src=[5466, 5478, 5434, 5436, 5466, 5430, 5450, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=41 plans=1 dispatch_payload_bytes=157179904 dispatch_meta_bytes=228776 combine_payload_bytes=157179904 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157408680 ingress_bytes=176054272 link_bytes=176054272 token_bytes=314588584 weight_movement_bytes=18874368 total_nvlink_bytes=333462952 weight_to_token_ratio=0.059997 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686865410e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156385280 dispatch_meta_bytes=229208 combine_payload_bytes=156385280 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156614488 ingress_bytes=175259648 link_bytes=175259648 token_bytes=312999768 weight_movement_bytes=18874368 total_nvlink_bytes=331874136 weight_to_token_ratio=0.060302 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679251714e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43618 payload_rows_remote=38230 dup_rows=21918 dup_rows_remote=19428 dedup_ratio=0.334442 router_tokens_per_expert=[279, 267, 260, 250, 251, 265, 273, 267, 260, 282, 234, 236, 268, 256, 259, 240, 271, 248, 275, 259, 280, 261, 238, 252, 245, 261, 241, 242, 258, 278, 242, 253, 273, 234, 243, 245, 264, 256, 262, 241, 257, 267, 244, 286, 256, 285, 271, 266, 240, 264, 279, 279, 262, 241, 248, 240, 273, 240, 256, 243, 263, 270, 265, 259, 268, 277, 227, 239, 229, 259, 266, 263, 254, 319, 247, 270, 263, 250, 249, 263, 237, 259, 275, 245, 245, 260, 268, 297, 252, 259, 245, 246, 249, 268, 259, 226, 228, 260, 255, 242, 242, 242, 259, 223, 258, 242, 235, 253, 262, 231, 256, 251, 226, 276, 245, 239, 261, 248, 267, 263, 243, 261, 295, 244, 244, 264, 241, 256] payload_rows_per_dst=[5496, 5406, 5482, 5502, 5462, 5456, 5388, 5426] dup_rows_per_dst=[2798, 2802, 2818, 2742, 2824, 2724, 2490, 2720] combine_rows_per_src=[5496, 5406, 5482, 5502, 5462, 5456, 5388, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156442624 dispatch_meta_bytes=229128 combine_payload_bytes=156442624 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156671752 ingress_bytes=175316992 link_bytes=175316992 token_bytes=313114376 weight_movement_bytes=18874368 total_nvlink_bytes=331988744 weight_to_token_ratio=0.060279 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.679801156e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43696 payload_rows_remote=38250 dup_rows=21840 dup_rows_remote=19148 dedup_ratio=0.333252 router_tokens_per_expert=[257, 270, 257, 262, 245, 246, 244, 257, 242, 292, 269, 260, 259, 285, 270, 235, 275, 259, 237, 263, 244, 246, 272, 274, 268, 244, 265, 266, 252, 253, 232, 225, 277, 250, 229, 251, 249, 258, 259, 235, 264, 259, 253, 254, 265, 246, 272, 248, 245, 263, 255, 266, 219, 271, 275, 247, 280, 244, 259, 264, 233, 224, 236, 274, 253, 265, 269, 276, 236, 244, 252, 244, 259, 247, 253, 268, 268, 249, 256, 225, 256, 245, 263, 232, 239, 241, 270, 254, 278, 277, 279, 263, 221, 280, 259, 301, 237, 237, 258, 268, 257, 247, 284, 250, 265, 240, 266, 263, 284, 245, 276, 249, 254, 217, 275, 279, 261, 271, 248, 239, 264, 244, 238, 246, 236, 282, 261, 256] payload_rows_per_dst=[5542, 5454, 5446, 5470, 5390, 5484, 5464, 5446] dup_rows_per_dst=[2758, 2696, 2692, 2640, 2738, 2832, 2788, 2696] combine_rows_per_src=[5542, 5454, 5446, 5470, 5390, 5484, 5464, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43862 payload_rows_remote=38330 dup_rows=21674 dup_rows_remote=18948 dedup_ratio=0.330719 router_tokens_per_expert=[237, 270, 242, 252, 260, 268, 256, 266, 254, 261, 274, 247, 256, 265, 263, 249, 292, 251, 247, 258, 258, 268, 259, 246, 253, 258, 248, 270, 232, 263, 245, 269, 228, 252, 248, 267, 272, 252, 257, 279, 247, 241, 258, 241, 263, 220, 268, 281, 239, 243, 268, 260, 261, 249, 243, 267, 243, 266, 258, 274, 267, 248, 246, 264, 264, 249, 223, 270, 254, 265, 243, 235, 271, 280, 245, 265, 252, 247, 244, 271, 290, 237, 243, 270, 241, 251, 276, 250, 275, 253, 261, 264, 260, 255, 258, 245, 243, 249, 234, 272, 258, 274, 249, 262, 289, 211, 248, 277, 270, 247, 304, 249, 243, 265, 243, 259, 259, 277, 247, 226, 277, 235, 234, 266, 233, 235, 277, 242] payload_rows_per_dst=[5434, 5522, 5406, 5524, 5492, 5532, 5540, 5412] dup_rows_per_dst=[2806, 2712, 2742, 2668, 2664, 2726, 2732, 2624] combine_rows_per_src=[5434, 5522, 5406, 5524, 5492, 5532, 5540, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43856 payload_rows_remote=38434 dup_rows=21680 dup_rows_remote=19086 dedup_ratio=0.330811 router_tokens_per_expert=[248, 261, 234, 264, 249, 265, 287, 287, 252, 280, 255, 238, 259, 256, 254, 236, 228, 232, 273, 273, 258, 272, 257, 257, 247, 267, 266, 242, 251, 265, 222, 248, 233, 241, 250, 281, 262, 261, 260, 258, 249, 256, 258, 261, 263, 246, 236, 252, 246, 250, 251, 266, 257, 243, 269, 249, 228, 235, 262, 235, 244, 266, 243, 264, 250, 259, 259, 206, 261, 231, 265, 289, 233, 251, 266, 243, 262, 256, 279, 249, 262, 267, 246, 272, 252, 272, 292, 295, 258, 238, 265, 267, 251, 249, 255, 261, 262, 243, 243, 274, 256, 267, 267, 258, 276, 278, 244, 248, 256, 257, 270, 302, 229, 258, 254, 269, 244, 262, 251, 268, 256, 253, 227, 272, 258, 245, 266, 236] payload_rows_per_dst=[5422, 5482, 5474, 5422, 5460, 5560, 5590, 5446] dup_rows_per_dst=[2828, 2634, 2660, 2594, 2658, 2844, 2812, 2650] combine_rows_per_src=[5422, 5482, 5474, 5422, 5460, 5560, 5590, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43682 payload_rows_remote=38200 dup_rows=21854 dup_rows_remote=19084 dedup_ratio=0.333466 router_tokens_per_expert=[249, 238, 249, 242, 260, 273, 246, 256, 238, 251, 262, 250, 279, 251, 233, 262, 244, 252, 248, 254, 278, 252, 267, 261, 223, 255, 259, 258, 277, 240, 260, 266, 256, 279, 247, 262, 238, 257, 268, 254, 265, 244, 252, 269, 293, 260, 269, 292, 258, 244, 257, 243, 252, 248, 238, 274, 270, 237, 247, 253, 263, 252, 258, 276, 252, 231, 240, 250, 265, 240, 221, 235, 236, 262, 272, 258, 233, 258, 245, 250, 233, 272, 252, 284, 248, 258, 283, 241, 247, 278, 260, 257, 290, 269, 256, 246, 247, 265, 293, 246, 265, 249, 273, 247, 274, 253, 260, 245, 267, 223, 253, 271, 256, 258, 259, 258, 246, 250, 237, 238, 258, 231, 258, 258, 279, 262, 282, 277] payload_rows_per_dst=[5392, 5416, 5434, 5554, 5358, 5376, 5670, 5482] dup_rows_per_dst=[2686, 2772, 2626, 2898, 2538, 2460, 3104, 2770] combine_rows_per_src=[5392, 5416, 5434, 5554, 5358, 5376, 5670, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156590080 dispatch_meta_bytes=230632 combine_payload_bytes=156590080 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156820712 ingress_bytes=194338816 link_bytes=194338816 token_bytes=313410792 weight_movement_bytes=37748736 total_nvlink_bytes=351159528 weight_to_token_ratio=0.120445 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862058914e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156672000 dispatch_meta_bytes=229592 combine_payload_bytes=156672000 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156901592 ingress_bytes=175546368 link_bytes=175546368 token_bytes=313573592 weight_movement_bytes=18874368 total_nvlink_bytes=332447960 weight_to_token_ratio=0.060191 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681998924e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156999680 dispatch_meta_bytes=229112 combine_payload_bytes=156999680 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157228792 ingress_bytes=194748416 link_bytes=194748416 token_bytes=314228472 weight_movement_bytes=37748736 total_nvlink_bytes=351977208 weight_to_token_ratio=0.120131 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.865983500e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=157425664 dispatch_meta_bytes=230080 combine_payload_bytes=157425664 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157655744 ingress_bytes=176300032 link_bytes=176300032 token_bytes=315081408 weight_movement_bytes=18874368 total_nvlink_bytes=333955776 weight_to_token_ratio=0.059903 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.689220162e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43740 payload_rows_remote=38308 dup_rows=21796 dup_rows_remote=19046 dedup_ratio=0.332581 router_tokens_per_expert=[252, 246, 233, 260, 227, 235, 269, 240, 268, 265, 256, 276, 240, 228, 262, 259, 236, 266, 240, 248, 247, 264, 277, 275, 278, 246, 269, 300, 273, 285, 244, 251, 266, 252, 252, 263, 255, 256, 264, 234, 251, 259, 257, 235, 267, 255, 259, 263, 265, 263, 255, 254, 271, 255, 281, 278, 226, 253, 255, 263, 246, 246, 246, 234, 239, 275, 258, 252, 260, 288, 258, 268, 249, 238, 270, 258, 245, 258, 259, 253, 255, 240, 269, 249, 231, 270, 263, 247, 245, 247, 283, 237, 273, 241, 245, 247, 254, 262, 226, 244, 218, 273, 259, 248, 255, 250, 232, 281, 278, 269, 225, 275, 263, 251, 276, 266, 263, 263, 255, 270, 269, 255, 234, 246, 270, 260, 265, 249] payload_rows_per_dst=[5432, 5510, 5494, 5518, 5502, 5596, 5160, 5528] dup_rows_per_dst=[2750, 2888, 2700, 2646, 2754, 2970, 2306, 2782] combine_rows_per_src=[5432, 5510, 5494, 5518, 5502, 5596, 5160, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43712 payload_rows_remote=38256 dup_rows=21824 dup_rows_remote=19034 dedup_ratio=0.333008 router_tokens_per_expert=[259, 263, 248, 270, 257, 247, 260, 260, 267, 240, 255, 260, 263, 254, 252, 234, 262, 287, 237, 264, 250, 258, 205, 241, 247, 247, 241, 248, 262, 247, 243, 265, 253, 251, 265, 216, 255, 259, 292, 271, 256, 248, 258, 236, 246, 280, 257, 241, 275, 262, 272, 261, 266, 239, 246, 289, 233, 251, 250, 273, 268, 314, 245, 274, 261, 239, 262, 263, 286, 243, 269, 259, 251, 239, 242, 273, 268, 253, 259, 256, 236, 254, 281, 273, 245, 244, 264, 273, 272, 268, 260, 270, 248, 243, 262, 259, 254, 243, 245, 271, 247, 239, 259, 240, 274, 277, 264, 283, 260, 264, 246, 250, 233, 234, 243, 259, 250, 245, 256, 257, 253, 262, 288, 257, 255, 257, 212, 221] payload_rows_per_dst=[5504, 5412, 5496, 5574, 5456, 5480, 5476, 5314] dup_rows_per_dst=[2674, 2596, 2672, 2862, 2790, 2824, 2756, 2650] combine_rows_per_src=[5504, 5412, 5496, 5574, 5456, 5480, 5476, 5314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156467200 dispatch_meta_bytes=229136 combine_payload_bytes=156467200 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156696336 ingress_bytes=194215936 link_bytes=194215936 token_bytes=313163536 weight_movement_bytes=37748736 total_nvlink_bytes=350912272 weight_to_token_ratio=0.120540 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860881539e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43822 payload_rows_remote=38320 dup_rows=21714 dup_rows_remote=18874 dedup_ratio=0.331329 router_tokens_per_expert=[266, 233, 279, 265, 263, 244, 265, 239, 244, 270, 235, 263, 256, 269, 260, 241, 254, 233, 264, 257, 258, 266, 261, 239, 247, 254, 275, 234, 261, 243, 275, 269, 258, 249, 224, 245, 235, 244, 250, 284, 241, 274, 258, 254, 258, 261, 259, 280, 284, 251, 286, 302, 250, 252, 287, 249, 264, 238, 239, 255, 292, 264, 245, 245, 269, 262, 250, 214, 258, 245, 254, 232, 258, 272, 251, 249, 253, 262, 239, 252, 265, 249, 277, 242, 253, 264, 256, 256, 269, 255, 242, 267, 242, 269, 270, 252, 250, 248, 272, 256, 250, 238, 274, 232, 242, 285, 266, 220, 265, 236, 262, 223, 233, 244, 281, 262, 278, 239, 247, 274, 258, 249, 261, 265, 252, 262, 241, 296] payload_rows_per_dst=[5498, 5502, 5504, 5468, 5404, 5566, 5414, 5466] dup_rows_per_dst=[2686, 2840, 2644, 2776, 2636, 2690, 2624, 2818] combine_rows_per_src=[5498, 5502, 5504, 5468, 5404, 5566, 5414, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=42 plans=1 dispatch_payload_bytes=156909568 dispatch_meta_bytes=229416 combine_payload_bytes=156909568 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157138984 ingress_bytes=175783936 link_bytes=175783936 token_bytes=314048552 weight_movement_bytes=18874368 total_nvlink_bytes=332922920 weight_to_token_ratio=0.060100 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684275184e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156696576 dispatch_meta_bytes=229160 combine_payload_bytes=156696576 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156925736 ingress_bytes=194445312 link_bytes=194445312 token_bytes=313622312 weight_movement_bytes=37748736 total_nvlink_bytes=351371048 weight_to_token_ratio=0.120364 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.083333 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863079307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156958720 dispatch_meta_bytes=228776 combine_payload_bytes=156958720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157187496 ingress_bytes=175833088 link_bytes=175833088 token_bytes=314146216 weight_movement_bytes=18874368 total_nvlink_bytes=333020584 weight_to_token_ratio=0.060081 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684746134e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=38318 dup_rows=21692 dup_rows_remote=18800 dedup_ratio=0.330994 router_tokens_per_expert=[243, 244, 281, 269, 240, 260, 259, 248, 240, 263, 239, 262, 258, 264, 243, 234, 234, 272, 252, 259, 240, 259, 261, 242, 246, 241, 271, 267, 254, 263, 237, 268, 256, 264, 278, 255, 272, 270, 244, 265, 253, 245, 268, 268, 244, 258, 254, 250, 266, 267, 260, 261, 254, 260, 262, 247, 253, 229, 257, 220, 266, 255, 273, 263, 265, 265, 273, 275, 258, 267, 260, 266, 253, 281, 224, 232, 247, 286, 271, 284, 263, 254, 260, 280, 268, 248, 229, 272, 260, 267, 271, 249, 252, 244, 250, 255, 249, 241, 260, 253, 245, 244, 257, 271, 235, 267, 286, 274, 264, 235, 242, 230, 242, 239, 251, 238, 254, 251, 252, 262, 275, 253, 271, 268, 231, 243, 254, 252] payload_rows_per_dst=[5488, 5498, 5470, 5506, 5562, 5346, 5526, 5448] dup_rows_per_dst=[2606, 2634, 2818, 2680, 2852, 2586, 2892, 2624] combine_rows_per_src=[5488, 5498, 5470, 5506, 5562, 5346, 5526, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43626 payload_rows_remote=38178 dup_rows=21910 dup_rows_remote=19270 dedup_ratio=0.334320 router_tokens_per_expert=[240, 264, 256, 247, 264, 275, 239, 222, 270, 243, 253, 275, 251, 294, 271, 240, 265, 253, 263, 260, 240, 245, 242, 262, 282, 256, 262, 249, 283, 267, 258, 243, 252, 274, 254, 253, 263, 233, 250, 247, 251, 241, 258, 262, 245, 258, 269, 234, 275, 259, 267, 260, 268, 236, 237, 244, 232, 241, 257, 270, 230, 237, 286, 241, 261, 235, 285, 252, 237, 249, 246, 250, 265, 234, 274, 262, 275, 269, 277, 279, 235, 251, 256, 258, 266, 264, 258, 309, 231, 273, 232, 261, 263, 233, 251, 249, 268, 243, 288, 265, 239, 240, 251, 247, 250, 254, 252, 261, 252, 281, 238, 228, 263, 271, 247, 263, 251, 262, 257, 249, 256, 258, 268, 238, 265, 267, 255, 283] payload_rows_per_dst=[5436, 5450, 5448, 5422, 5516, 5442, 5418, 5494] dup_rows_per_dst=[2772, 2810, 2640, 2658, 2784, 2738, 2696, 2812] combine_rows_per_src=[5436, 5450, 5448, 5422, 5516, 5442, 5418, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43752 payload_rows_remote=38276 dup_rows=21784 dup_rows_remote=19010 dedup_ratio=0.332397 router_tokens_per_expert=[248, 267, 250, 237, 234, 229, 269, 231, 294, 294, 263, 257, 243, 248, 260, 253, 246, 269, 268, 270, 245, 234, 249, 232, 230, 240, 261, 266, 235, 261, 237, 258, 264, 259, 253, 246, 251, 229, 262, 278, 264, 272, 262, 264, 280, 248, 261, 252, 252, 275, 253, 240, 282, 262, 265, 287, 256, 226, 273, 241, 277, 275, 270, 254, 260, 264, 261, 272, 259, 239, 247, 243, 259, 265, 254, 258, 255, 224, 235, 257, 271, 235, 261, 259, 243, 283, 233, 235, 270, 267, 260, 249, 221, 261, 287, 290, 235, 258, 303, 244, 241, 258, 263, 241, 255, 233, 253, 250, 262, 249, 255, 247, 261, 267, 233, 248, 295, 257, 268, 248, 247, 258, 266, 253, 260, 232, 287, 253] payload_rows_per_dst=[5422, 5398, 5574, 5632, 5384, 5476, 5384, 5482] dup_rows_per_dst=[2732, 2604, 2716, 2744, 2720, 2774, 2710, 2784] combine_rows_per_src=[5422, 5398, 5574, 5632, 5384, 5476, 5384, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43630 payload_rows_remote=38174 dup_rows=21906 dup_rows_remote=19194 dedup_ratio=0.334259 router_tokens_per_expert=[249, 226, 278, 265, 250, 242, 273, 271, 257, 265, 254, 265, 276, 259, 271, 277, 225, 248, 264, 244, 254, 232, 255, 266, 275, 277, 249, 281, 263, 219, 261, 222, 239, 239, 267, 265, 236, 255, 255, 283, 274, 276, 269, 285, 258, 221, 257, 237, 262, 215, 261, 292, 264, 251, 237, 261, 246, 237, 265, 257, 257, 257, 263, 259, 251, 272, 223, 242, 234, 242, 263, 266, 271, 248, 290, 273, 248, 276, 272, 235, 246, 256, 259, 231, 267, 263, 273, 270, 271, 276, 240, 270, 271, 255, 275, 269, 242, 254, 235, 230, 240, 232, 250, 245, 253, 268, 259, 228, 254, 237, 243, 227, 254, 265, 274, 251, 267, 257, 236, 266, 265, 252, 274, 258, 288, 265, 229, 259] payload_rows_per_dst=[5522, 5398, 5516, 5456, 5436, 5492, 5302, 5508] dup_rows_per_dst=[2834, 2672, 2716, 2712, 2776, 2892, 2492, 2812] combine_rows_per_src=[5522, 5398, 5516, 5456, 5436, 5492, 5302, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43610 payload_rows_remote=38124 dup_rows=21926 dup_rows_remote=19126 dedup_ratio=0.334564 router_tokens_per_expert=[256, 256, 269, 277, 235, 254, 273, 268, 274, 268, 267, 266, 253, 267, 252, 243, 251, 244, 236, 286, 262, 250, 266, 246, 253, 270, 240, 261, 253, 262, 240, 296, 249, 234, 265, 257, 284, 259, 258, 245, 259, 253, 264, 244, 275, 240, 281, 245, 260, 247, 268, 242, 258, 263, 263, 243, 242, 269, 247, 244, 265, 248, 286, 264, 250, 283, 244, 265, 265, 226, 235, 218, 247, 247, 256, 234, 260, 229, 241, 289, 251, 263, 254, 271, 269, 274, 224, 258, 250, 266, 238, 256, 281, 279, 266, 243, 252, 240, 266, 237, 248, 263, 269, 282, 224, 231, 264, 262, 230, 258, 246, 237, 249, 277, 242, 275, 252, 257, 250, 254, 284, 277, 242, 255, 255, 265, 226, 252] payload_rows_per_dst=[5180, 5444, 5432, 5686, 5452, 5500, 5430, 5486] dup_rows_per_dst=[2296, 2788, 2792, 3066, 2870, 2726, 2588, 2800] combine_rows_per_src=[5180, 5444, 5432, 5686, 5452, 5500, 5430, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156950528 dispatch_meta_bytes=228472 combine_payload_bytes=156950528 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157179000 ingress_bytes=175824896 link_bytes=175824896 token_bytes=314129528 weight_movement_bytes=18874368 total_nvlink_bytes=333003896 weight_to_token_ratio=0.060085 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684667642e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=229792 combine_payload_bytes=156377088 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156606880 ingress_bytes=213000192 link_bytes=213000192 token_bytes=312983968 weight_movement_bytes=56623104 total_nvlink_bytes=369607072 weight_to_token_ratio=0.180914 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.040863037e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156360704 dispatch_meta_bytes=229472 combine_payload_bytes=156360704 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156590176 ingress_bytes=194109440 link_bytes=194109440 token_bytes=312950880 weight_movement_bytes=37748736 total_nvlink_bytes=350699616 weight_to_token_ratio=0.120622 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.859861146e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229144 combine_payload_bytes=156778496 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157007640 ingress_bytes=194527232 link_bytes=194527232 token_bytes=313786136 weight_movement_bytes=37748736 total_nvlink_bytes=351534872 weight_to_token_ratio=0.120301 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.863864224e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38202 dup_rows=21904 dup_rows_remote=19074 dedup_ratio=0.334229 router_tokens_per_expert=[286, 252, 265, 279, 265, 229, 261, 272, 246, 264, 256, 275, 255, 236, 247, 240, 282, 292, 268, 252, 260, 244, 237, 263, 261, 247, 268, 248, 254, 238, 240, 280, 271, 270, 270, 285, 260, 256, 245, 242, 246, 269, 252, 246, 253, 242, 260, 246, 266, 240, 259, 257, 245, 273, 245, 250, 250, 258, 227, 278, 256, 229, 262, 244, 287, 276, 266, 253, 271, 255, 251, 250, 257, 273, 255, 249, 308, 266, 290, 249, 264, 269, 249, 239, 246, 236, 245, 251, 259, 259, 254, 261, 281, 259, 238, 244, 244, 258, 249, 244, 252, 274, 242, 245, 235, 268, 215, 251, 264, 238, 261, 265, 261, 254, 228, 236, 266, 226, 239, 253, 264, 254, 256, 243, 257, 269, 260, 273] payload_rows_per_dst=[5430, 5592, 5298, 5516, 5476, 5498, 5396, 5426] dup_rows_per_dst=[2830, 2928, 2672, 2800, 2798, 2610, 2614, 2652] combine_rows_per_src=[5430, 5592, 5298, 5516, 5476, 5498, 5396, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43852 payload_rows_remote=38302 dup_rows=21684 dup_rows_remote=18984 dedup_ratio=0.330872 router_tokens_per_expert=[242, 262, 248, 244, 263, 254, 266, 229, 244, 230, 258, 259, 254, 276, 245, 269, 248, 260, 248, 259, 250, 281, 258, 256, 268, 252, 272, 234, 250, 260, 261, 268, 247, 237, 274, 264, 248, 234, 254, 269, 278, 255, 255, 282, 263, 261, 239, 260, 253, 256, 220, 232, 221, 252, 248, 258, 274, 252, 247, 221, 249, 239, 248, 254, 229, 272, 252, 271, 260, 243, 251, 272, 273, 236, 249, 291, 231, 253, 290, 240, 258, 272, 264, 277, 258, 270, 267, 265, 236, 271, 253, 283, 257, 257, 253, 231, 262, 286, 264, 254, 242, 261, 244, 243, 248, 233, 270, 242, 269, 275, 250, 247, 275, 230, 267, 261, 261, 297, 249, 267, 232, 268, 265, 264, 291, 264, 257, 233] payload_rows_per_dst=[5446, 5550, 5540, 5330, 5480, 5526, 5442, 5538] dup_rows_per_dst=[2640, 2700, 2700, 2518, 2746, 2818, 2738, 2824] combine_rows_per_src=[5446, 5550, 5540, 5330, 5480, 5526, 5442, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43552 payload_rows_remote=38116 dup_rows=21984 dup_rows_remote=19302 dedup_ratio=0.335449 router_tokens_per_expert=[267, 243, 269, 265, 274, 247, 266, 280, 265, 259, 265, 253, 270, 275, 274, 244, 284, 249, 246, 259, 251, 250, 254, 254, 264, 240, 260, 243, 243, 240, 245, 264, 245, 240, 243, 230, 248, 265, 268, 263, 258, 261, 266, 272, 278, 240, 262, 244, 265, 247, 253, 264, 266, 243, 259, 239, 277, 244, 246, 250, 265, 236, 255, 229, 261, 257, 242, 284, 248, 247, 270, 247, 246, 242, 259, 250, 253, 249, 246, 258, 258, 262, 258, 259, 282, 241, 263, 265, 263, 254, 259, 286, 238, 259, 273, 266, 250, 265, 246, 259, 249, 243, 231, 249, 256, 253, 272, 247, 242, 253, 238, 283, 256, 248, 250, 267, 239, 236, 278, 250, 250, 282, 259, 272, 257, 251, 251, 258] payload_rows_per_dst=[5516, 5372, 5478, 5346, 5436, 5530, 5422, 5452] dup_rows_per_dst=[2916, 2720, 2688, 2730, 2682, 2842, 2650, 2756] combine_rows_per_src=[5516, 5372, 5478, 5346, 5436, 5530, 5422, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156155904 dispatch_meta_bytes=229000 combine_payload_bytes=156155904 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=156384904 ingress_bytes=212779008 link_bytes=212779008 token_bytes=312540808 weight_movement_bytes=56623104 total_nvlink_bytes=369163912 weight_to_token_ratio=0.181170 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.038743761e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=43 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=229104 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156704496 ingress_bytes=175349760 link_bytes=175349760 token_bytes=313179888 weight_movement_bytes=18874368 total_nvlink_bytes=332054256 weight_to_token_ratio=0.060267 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.680115123e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156884992 dispatch_meta_bytes=229144 combine_payload_bytes=156884992 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157114136 ingress_bytes=175759360 link_bytes=175759360 token_bytes=313999128 weight_movement_bytes=18874368 total_nvlink_bytes=332873496 weight_to_token_ratio=0.060110 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684039709e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156123136 dispatch_meta_bytes=229672 combine_payload_bytes=156123136 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156352808 ingress_bytes=174997504 link_bytes=174997504 token_bytes=312475944 weight_movement_bytes=18874368 total_nvlink_bytes=331350312 weight_to_token_ratio=0.060403 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.676739979e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43584 payload_rows_remote=38094 dup_rows=21952 dup_rows_remote=19072 dedup_ratio=0.334961 router_tokens_per_expert=[276, 287, 283, 249, 238, 258, 267, 252, 277, 251, 267, 240, 266, 271, 248, 259, 240, 280, 224, 265, 261, 261, 257, 290, 225, 256, 265, 263, 263, 230, 246, 234, 224, 237, 263, 267, 233, 266, 258, 267, 239, 230, 247, 249, 257, 263, 249, 252, 254, 246, 234, 269, 265, 275, 251, 262, 274, 253, 262, 226, 270, 254, 244, 223, 228, 265, 261, 232, 280, 231, 253, 281, 248, 272, 247, 285, 240, 235, 267, 266, 233, 252, 224, 261, 233, 263, 263, 254, 283, 265, 241, 271, 250, 223, 260, 241, 263, 268, 276, 258, 257, 265, 249, 274, 260, 251, 253, 268, 248, 244, 283, 268, 288, 268, 256, 283, 264, 260, 249, 278, 279, 232, 248, 236, 240, 236, 274, 272] payload_rows_per_dst=[5470, 5372, 5392, 5492, 5486, 5376, 5490, 5506] dup_rows_per_dst=[2768, 2748, 2610, 2772, 2696, 2658, 2880, 2820] combine_rows_per_src=[5470, 5372, 5392, 5492, 5486, 5376, 5490, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43714 payload_rows_remote=38254 dup_rows=21822 dup_rows_remote=19192 dedup_ratio=0.332977 router_tokens_per_expert=[253, 273, 277, 225, 259, 266, 255, 250, 258, 257, 245, 247, 261, 263, 275, 244, 264, 261, 265, 270, 265, 265, 260, 233, 262, 272, 228, 269, 256, 241, 225, 252, 275, 243, 281, 262, 256, 231, 237, 243, 250, 252, 260, 269, 257, 249, 241, 239, 253, 261, 237, 291, 258, 261, 260, 289, 252, 257, 258, 257, 259, 258, 266, 260, 223, 258, 264, 254, 251, 278, 228, 260, 273, 255, 230, 266, 248, 252, 269, 244, 279, 304, 251, 274, 240, 276, 259, 274, 258, 228, 245, 253, 279, 254, 271, 257, 290, 267, 244, 283, 243, 212, 220, 255, 242, 244, 251, 243, 246, 228, 267, 254, 274, 268, 263, 256, 255, 268, 270, 270, 233, 250, 257, 245, 247, 269, 242, 239] payload_rows_per_dst=[5388, 5494, 5460, 5508, 5482, 5600, 5312, 5470] dup_rows_per_dst=[2828, 2682, 2630, 2846, 2624, 2804, 2666, 2742] combine_rows_per_src=[5388, 5494, 5460, 5508, 5482, 5600, 5312, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43496 payload_rows_remote=38018 dup_rows=22040 dup_rows_remote=19218 dedup_ratio=0.336304 router_tokens_per_expert=[241, 285, 267, 271, 264, 265, 281, 251, 279, 255, 257, 249, 267, 262, 274, 260, 288, 266, 281, 280, 260, 248, 230, 245, 233, 249, 244, 247, 264, 260, 244, 271, 266, 239, 268, 226, 246, 259, 270, 220, 256, 262, 259, 260, 252, 235, 247, 233, 252, 263, 262, 261, 258, 281, 254, 252, 282, 262, 254, 275, 262, 234, 244, 254, 245, 252, 273, 257, 212, 266, 266, 228, 251, 283, 263, 267, 257, 258, 271, 224, 266, 260, 266, 262, 244, 263, 264, 254, 245, 232, 246, 261, 237, 260, 270, 250, 272, 254, 251, 231, 253, 255, 256, 250, 230, 233, 266, 267, 261, 253, 250, 263, 289, 254, 230, 236, 259, 271, 253, 238, 276, 248, 263, 242, 239, 268, 252, 266] payload_rows_per_dst=[5504, 5490, 5348, 5478, 5436, 5460, 5400, 5380] dup_rows_per_dst=[2952, 2730, 2648, 2822, 2710, 2700, 2690, 2788] combine_rows_per_src=[5504, 5490, 5348, 5478, 5436, 5460, 5400, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43730 payload_rows_remote=38322 dup_rows=21806 dup_rows_remote=19184 dedup_ratio=0.332733 router_tokens_per_expert=[255, 249, 239, 280, 251, 268, 279, 263, 265, 278, 253, 260, 276, 285, 260, 283, 279, 253, 255, 252, 256, 269, 274, 281, 286, 263, 280, 250, 240, 256, 232, 254, 245, 267, 249, 223, 238, 271, 267, 247, 247, 226, 260, 262, 270, 254, 268, 257, 260, 250, 240, 267, 287, 265, 226, 249, 255, 243, 236, 233, 280, 280, 256, 240, 254, 269, 282, 250, 239, 244, 246, 229, 247, 257, 249, 258, 253, 260, 243, 249, 254, 243, 232, 240, 267, 227, 251, 249, 259, 241, 269, 252, 260, 252, 250, 269, 255, 243, 251, 267, 230, 264, 272, 265, 267, 261, 252, 260, 240, 239, 257, 272, 257, 255, 233, 263, 270, 277, 246, 246, 257, 250, 236, 273, 264, 256, 250, 254] payload_rows_per_dst=[5558, 5480, 5502, 5466, 5422, 5408, 5422, 5472] dup_rows_per_dst=[2930, 2880, 2600, 2668, 2636, 2622, 2768, 2702] combine_rows_per_src=[5558, 5480, 5502, 5466, 5422, 5408, 5422, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43816 payload_rows_remote=38202 dup_rows=21720 dup_rows_remote=18840 dedup_ratio=0.331421 router_tokens_per_expert=[263, 248, 265, 249, 264, 240, 255, 257, 267, 292, 265, 256, 228, 261, 241, 245, 263, 234, 254, 256, 231, 247, 273, 248, 274, 261, 238, 239, 265, 249, 237, 256, 249, 262, 299, 254, 239, 238, 243, 240, 251, 259, 252, 267, 260, 264, 237, 253, 239, 255, 249, 241, 253, 245, 270, 228, 257, 264, 264, 251, 269, 232, 254, 242, 276, 260, 245, 280, 240, 263, 263, 270, 243, 205, 251, 277, 258, 256, 240, 258, 277, 257, 267, 233, 277, 274, 303, 288, 248, 256, 273, 253, 244, 272, 283, 262, 241, 253, 254, 262, 234, 263, 297, 242, 225, 253, 253, 270, 278, 259, 258, 262, 249, 223, 236, 237, 238, 272, 273, 265, 275, 289, 257, 250, 253, 252, 270, 272] payload_rows_per_dst=[5482, 5420, 5440, 5390, 5520, 5550, 5400, 5614] dup_rows_per_dst=[2624, 2630, 2694, 2636, 2650, 2984, 2622, 2880] combine_rows_per_src=[5482, 5420, 5440, 5390, 5520, 5550, 5400, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156033024 dispatch_meta_bytes=228664 combine_payload_bytes=156033024 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156261688 ingress_bytes=174907392 link_bytes=174907392 token_bytes=312294712 weight_movement_bytes=18874368 total_nvlink_bytes=331169080 weight_to_token_ratio=0.060438 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.675876570e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156688384 dispatch_meta_bytes=229784 combine_payload_bytes=156688384 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156918168 ingress_bytes=175562752 link_bytes=175562752 token_bytes=313606552 weight_movement_bytes=18874368 total_nvlink_bytes=332480920 weight_to_token_ratio=0.060185 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682155907e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=155721728 dispatch_meta_bytes=228944 combine_payload_bytes=155721728 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=155950672 ingress_bytes=174596096 link_bytes=174596096 token_bytes=311672400 weight_movement_bytes=18874368 total_nvlink_bytes=330546768 weight_to_token_ratio=0.060558 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.672893885e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43726 payload_rows_remote=38226 dup_rows=21810 dup_rows_remote=19162 dedup_ratio=0.332794 router_tokens_per_expert=[266, 274, 264, 237, 263, 239, 261, 228, 256, 245, 242, 257, 286, 268, 250, 267, 267, 259, 268, 217, 257, 246, 245, 258, 243, 263, 256, 232, 270, 267, 278, 255, 249, 293, 260, 260, 267, 265, 249, 246, 263, 260, 268, 296, 254, 263, 257, 264, 272, 262, 268, 233, 262, 241, 226, 232, 251, 273, 253, 243, 264, 252, 289, 256, 252, 247, 254, 253, 296, 271, 261, 254, 263, 231, 237, 246, 252, 245, 247, 265, 284, 271, 266, 237, 264, 262, 255, 236, 237, 252, 272, 221, 237, 238, 264, 258, 238, 280, 239, 274, 264, 228, 269, 260, 282, 230, 241, 261, 234, 278, 255, 264, 274, 274, 260, 217, 250, 266, 266, 248, 260, 253, 278, 246, 241, 232, 227, 276] payload_rows_per_dst=[5464, 5422, 5528, 5476, 5500, 5466, 5432, 5438] dup_rows_per_dst=[2742, 2740, 2900, 2678, 2648, 2642, 2762, 2698] combine_rows_per_src=[5464, 5422, 5528, 5476, 5500, 5466, 5432, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43706 payload_rows_remote=38148 dup_rows=21830 dup_rows_remote=19076 dedup_ratio=0.333099 router_tokens_per_expert=[260, 269, 270, 272, 267, 277, 258, 248, 267, 231, 253, 244, 266, 266, 255, 251, 228, 286, 267, 255, 235, 255, 243, 260, 269, 249, 226, 252, 245, 233, 264, 263, 286, 233, 270, 248, 279, 263, 275, 256, 247, 266, 257, 266, 258, 256, 259, 269, 260, 228, 217, 284, 236, 294, 235, 236, 248, 263, 265, 238, 266, 263, 236, 268, 259, 267, 263, 255, 242, 276, 249, 263, 245, 264, 236, 293, 233, 280, 254, 236, 252, 269, 249, 276, 253, 277, 237, 264, 250, 257, 205, 261, 243, 265, 268, 233, 276, 249, 281, 262, 241, 262, 257, 254, 243, 258, 215, 262, 262, 284, 263, 255, 250, 263, 277, 276, 249, 242, 241, 259, 253, 254, 240, 251, 239, 257, 245, 265] payload_rows_per_dst=[5558, 5326, 5546, 5434, 5598, 5408, 5366, 5470] dup_rows_per_dst=[2754, 2734, 2830, 2640, 2856, 2706, 2658, 2652] combine_rows_per_src=[5558, 5326, 5546, 5434, 5598, 5408, 5366, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43800 payload_rows_remote=38368 dup_rows=21736 dup_rows_remote=19034 dedup_ratio=0.331665 router_tokens_per_expert=[275, 247, 262, 277, 266, 269, 266, 269, 275, 242, 262, 248, 247, 240, 239, 272, 258, 233, 244, 260, 272, 259, 265, 222, 248, 238, 274, 256, 268, 271, 241, 258, 270, 250, 248, 245, 267, 285, 250, 240, 260, 222, 272, 245, 242, 277, 271, 256, 248, 257, 258, 252, 263, 254, 230, 243, 234, 242, 277, 251, 240, 279, 253, 243, 269, 242, 261, 247, 270, 229, 263, 259, 266, 283, 253, 300, 250, 241, 250, 240, 254, 244, 269, 251, 243, 273, 240, 292, 270, 259, 280, 244, 256, 240, 262, 240, 265, 258, 248, 257, 259, 259, 238, 264, 237, 249, 275, 250, 249, 267, 251, 272, 253, 236, 255, 236, 232, 262, 253, 272, 275, 251, 254, 253, 264, 282, 254, 251] payload_rows_per_dst=[5538, 5432, 5490, 5490, 5442, 5478, 5512, 5418] dup_rows_per_dst=[2774, 2702, 2710, 2688, 2804, 2756, 2684, 2618] combine_rows_per_src=[5538, 5432, 5490, 5490, 5442, 5478, 5512, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=230024 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=37748736 egress_bytes=157196936 ingress_bytes=213590016 link_bytes=213590016 token_bytes=314163848 weight_movement_bytes=56623104 total_nvlink_bytes=370786952 weight_to_token_ratio=0.180234 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046514440e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15067076696 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7539046488 ingress_bytes=8962482176 link_bytes=8962482176 total_nvlink_bytes=16501528664 weight_to_token_ratio=0.095204 prefetch_share_of_ingress=0.084237 dedup_ratio=0.332556 prefetch_slots=80 unused_prefetch_slots=1456 slot_utilization=0.052083 reduce_slots=36 remote_demand=40 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081380 gemm_inner_pad_upper_bound=108712 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.789043633e-04 prefetch_bytes_per_gemm_flop=1.507040895e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=228168 combine_payload_bytes=156475392 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156703560 ingress_bytes=194224128 link_bytes=194224128 token_bytes=313178952 weight_movement_bytes=37748736 total_nvlink_bytes=350927688 weight_to_token_ratio=0.120534 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.860960030e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156573696 dispatch_meta_bytes=229552 combine_payload_bytes=156573696 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156803248 ingress_bytes=175448064 link_bytes=175448064 token_bytes=313376944 weight_movement_bytes=18874368 total_nvlink_bytes=332251312 weight_to_token_ratio=0.060229 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.681057023e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=44 plans=1 dispatch_payload_bytes=156254208 dispatch_meta_bytes=228896 combine_payload_bytes=156254208 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156483104 ingress_bytes=231751680 link_bytes=231751680 token_bytes=312737312 weight_movement_bytes=75497472 total_nvlink_bytes=388234784 weight_to_token_ratio=0.241409 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.220530569e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157155328 dispatch_meta_bytes=229608 combine_payload_bytes=157155328 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157384936 ingress_bytes=176029696 link_bytes=176029696 token_bytes=314540264 weight_movement_bytes=18874368 total_nvlink_bytes=333414632 weight_to_token_ratio=0.060006 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.686629935e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=44050 payload_rows_remote=38488 dup_rows=21486 dup_rows_remote=18702 dedup_ratio=0.327850 router_tokens_per_expert=[243, 226, 247, 274, 261, 247, 246, 259, 278, 245, 240, 274, 255, 263, 275, 246, 246, 244, 263, 224, 256, 234, 264, 251, 286, 245, 249, 279, 285, 262, 263, 244, 248, 283, 239, 241, 251, 266, 256, 264, 231, 266, 237, 262, 269, 257, 256, 250, 271, 262, 253, 249, 228, 251, 251, 232, 245, 259, 262, 254, 285, 259, 272, 249, 230, 258, 238, 257, 256, 258, 236, 242, 248, 268, 225, 241, 243, 245, 231, 259, 268, 261, 279, 239, 244, 268, 243, 226, 240, 288, 252, 238, 279, 279, 260, 258, 246, 240, 247, 267, 251, 243, 260, 244, 277, 262, 294, 281, 254, 272, 269, 266, 256, 267, 255, 261, 257, 241, 264, 255, 255, 260, 279, 261, 280, 271, 283, 261] payload_rows_per_dst=[5478, 5502, 5494, 5528, 5298, 5536, 5562, 5652] dup_rows_per_dst=[2680, 2688, 2658, 2636, 2572, 2708, 2784, 2760] combine_rows_per_src=[5478, 5502, 5494, 5528, 5298, 5536, 5562, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43790 payload_rows_remote=38398 dup_rows=21746 dup_rows_remote=19028 dedup_ratio=0.331818 router_tokens_per_expert=[285, 268, 270, 247, 245, 250, 250, 249, 260, 249, 253, 231, 252, 240, 246, 286, 252, 263, 258, 233, 260, 258, 251, 287, 233, 236, 260, 249, 278, 256, 254, 262, 287, 270, 249, 236, 270, 242, 251, 296, 276, 236, 281, 253, 276, 231, 280, 263, 256, 246, 258, 249, 262, 272, 267, 238, 263, 257, 255, 260, 240, 257, 218, 257, 229, 250, 260, 241, 236, 251, 242, 287, 259, 267, 262, 250, 270, 270, 251, 258, 258, 237, 247, 263, 233, 282, 258, 259, 249, 252, 285, 235, 268, 271, 232, 263, 266, 266, 272, 226, 273, 249, 268, 259, 238, 255, 231, 240, 247, 238, 237, 268, 262, 269, 261, 300, 266, 254, 267, 261, 254, 227, 271, 249, 253, 254, 253, 236] payload_rows_per_dst=[5538, 5484, 5566, 5392, 5430, 5486, 5436, 5458] dup_rows_per_dst=[2624, 2696, 2828, 2718, 2736, 2698, 2630, 2816] combine_rows_per_src=[5538, 5484, 5566, 5392, 5430, 5486, 5436, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43670 payload_rows_remote=38248 dup_rows=21866 dup_rows_remote=19138 dedup_ratio=0.333649 router_tokens_per_expert=[269, 236, 260, 239, 268, 228, 270, 264, 250, 248, 237, 287, 253, 231, 248, 262, 243, 239, 244, 274, 252, 260, 257, 255, 264, 277, 246, 245, 283, 256, 265, 244, 229, 262, 270, 263, 281, 247, 216, 269, 260, 275, 238, 278, 237, 270, 249, 267, 263, 277, 246, 242, 252, 247, 274, 270, 250, 241, 283, 261, 256, 262, 258, 260, 254, 235, 262, 243, 249, 233, 245, 262, 259, 242, 240, 258, 267, 222, 269, 254, 256, 257, 247, 248, 231, 269, 261, 252, 268, 264, 230, 259, 259, 269, 254, 251, 277, 254, 247, 287, 247, 260, 288, 235, 246, 261, 263, 263, 260, 243, 272, 284, 237, 264, 244, 268, 285, 270, 253, 277, 246, 257, 252, 235, 252, 261, 252, 252] payload_rows_per_dst=[5434, 5460, 5456, 5476, 5432, 5422, 5540, 5450] dup_rows_per_dst=[2666, 2748, 2766, 2808, 2556, 2728, 2834, 2760] combine_rows_per_src=[5434, 5460, 5456, 5476, 5432, 5422, 5540, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157646848 dispatch_meta_bytes=228760 combine_payload_bytes=157646848 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=157875608 ingress_bytes=195395584 link_bytes=195395584 token_bytes=315522456 weight_movement_bytes=37748736 total_nvlink_bytes=353271192 weight_to_token_ratio=0.119639 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.872184345e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38070 dup_rows=21752 dup_rows_remote=18898 dedup_ratio=0.331909 router_tokens_per_expert=[256, 245, 262, 223, 237, 252, 268, 244, 286, 251, 254, 279, 256, 246, 300, 272, 259, 265, 245, 267, 263, 257, 249, 259, 274, 260, 235, 254, 250, 233, 256, 247, 259, 269, 262, 283, 262, 226, 257, 252, 274, 262, 249, 269, 251, 257, 265, 252, 229, 242, 257, 274, 249, 292, 252, 284, 259, 257, 267, 244, 254, 267, 233, 255, 247, 281, 247, 235, 214, 245, 243, 273, 261, 263, 256, 248, 256, 234, 246, 237, 252, 259, 248, 243, 246, 276, 248, 243, 242, 282, 259, 239, 266, 266, 245, 231, 262, 271, 242, 251, 250, 246, 267, 271, 257, 282, 271, 260, 264, 245, 270, 236, 272, 256, 267, 275, 258, 255, 260, 243, 273, 234, 264, 261, 242, 270, 261, 233] payload_rows_per_dst=[5486, 5370, 5504, 5406, 5502, 5236, 5566, 5714] dup_rows_per_dst=[2748, 2666, 2800, 2676, 2644, 2546, 2818, 2854] combine_rows_per_src=[5486, 5370, 5504, 5406, 5502, 5236, 5566, 5714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=157278208 dispatch_meta_bytes=229704 combine_payload_bytes=157278208 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157507912 ingress_bytes=176152576 link_bytes=176152576 token_bytes=314786120 weight_movement_bytes=18874368 total_nvlink_bytes=333660488 weight_to_token_ratio=0.059959 gemm_rows_defined=33664 gemm_rows_charged=36864 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.086806 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.687807311e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43838 payload_rows_remote=38334 dup_rows=21698 dup_rows_remote=19080 dedup_ratio=0.331085 router_tokens_per_expert=[254, 255, 233, 236, 302, 271, 259, 263, 251, 258, 275, 262, 242, 260, 241, 239, 242, 243, 214, 262, 239, 256, 241, 265, 237, 275, 240, 245, 239, 262, 256, 278, 227, 243, 270, 290, 278, 277, 257, 252, 266, 244, 263, 273, 275, 246, 243, 263, 257, 275, 254, 238, 277, 259, 275, 264, 277, 220, 242, 265, 255, 255, 263, 232, 242, 256, 244, 264, 241, 274, 248, 273, 263, 242, 244, 240, 254, 281, 249, 246, 258, 234, 261, 260, 253, 242, 270, 250, 241, 272, 265, 269, 265, 250, 265, 252, 235, 255, 249, 226, 269, 269, 265, 254, 259, 249, 247, 261, 275, 263, 265, 289, 252, 251, 251, 250, 256, 296, 267, 246, 264, 251, 286, 246, 240, 218, 265, 261] payload_rows_per_dst=[5472, 5420, 5430, 5502, 5504, 5492, 5550, 5468] dup_rows_per_dst=[2730, 2568, 2904, 2714, 2618, 2722, 2710, 2732] combine_rows_per_src=[5472, 5420, 5430, 5502, 5504, 5492, 5550, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43632 payload_rows_remote=38058 dup_rows=21904 dup_rows_remote=18928 dedup_ratio=0.334229 router_tokens_per_expert=[240, 243, 271, 254, 262, 252, 277, 282, 261, 264, 278, 257, 280, 237, 254, 240, 266, 250, 279, 280, 267, 283, 271, 242, 275, 278, 252, 284, 239, 271, 256, 282, 280, 249, 231, 269, 250, 259, 276, 271, 275, 232, 252, 249, 280, 252, 261, 255, 228, 247, 235, 257, 237, 243, 272, 245, 271, 240, 269, 273, 260, 243, 269, 238, 210, 236, 250, 271, 250, 273, 252, 250, 227, 282, 251, 258, 257, 234, 282, 270, 245, 300, 274, 240, 256, 262, 250, 254, 253, 272, 253, 243, 240, 235, 276, 241, 262, 237, 225, 236, 268, 262, 261, 245, 259, 247, 249, 248, 254, 252, 238, 237, 256, 246, 265, 238, 268, 273, 215, 246, 269, 249, 242, 256, 282, 247, 241, 253] payload_rows_per_dst=[5484, 5574, 5536, 5392, 5396, 5376, 5440, 5434] dup_rows_per_dst=[2820, 2976, 2746, 2662, 2710, 2812, 2520, 2658] combine_rows_per_src=[5484, 5574, 5536, 5392, 5396, 5376, 5440, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43806 payload_rows_remote=38266 dup_rows=21730 dup_rows_remote=18940 dedup_ratio=0.331573 router_tokens_per_expert=[268, 240, 258, 274, 245, 240, 259, 256, 245, 287, 277, 271, 246, 268, 259, 235, 238, 267, 268, 227, 248, 234, 253, 245, 247, 288, 272, 285, 263, 273, 263, 246, 257, 254, 260, 259, 259, 264, 273, 265, 240, 259, 281, 280, 263, 266, 234, 223, 267, 236, 260, 258, 233, 257, 245, 235, 272, 291, 242, 246, 256, 230, 240, 252, 252, 235, 252, 278, 259, 276, 259, 276, 280, 273, 226, 253, 259, 287, 234, 235, 227, 239, 262, 256, 250, 238, 247, 269, 236, 227, 281, 274, 264, 247, 246, 247, 260, 252, 253, 244, 269, 243, 262, 258, 243, 256, 248, 260, 268, 241, 264, 275, 288, 247, 257, 278, 290, 257, 258, 260, 280, 232, 231, 246, 250, 226, 270, 256] payload_rows_per_dst=[5540, 5684, 5480, 5212, 5504, 5500, 5472, 5414] dup_rows_per_dst=[2790, 3084, 2798, 2294, 2686, 2806, 2720, 2552] combine_rows_per_src=[5540, 5684, 5480, 5212, 5504, 5500, 5472, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156663808 dispatch_meta_bytes=229544 combine_payload_bytes=156663808 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=156893352 ingress_bytes=194412544 link_bytes=194412544 token_bytes=313557160 weight_movement_bytes=37748736 total_nvlink_bytes=351305896 weight_to_token_ratio=0.120389 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862765340e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155934720 dispatch_meta_bytes=227872 combine_payload_bytes=155934720 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156162592 ingress_bytes=174809088 link_bytes=174809088 token_bytes=312097312 weight_movement_bytes=18874368 total_nvlink_bytes=330971680 weight_to_token_ratio=0.060476 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.674934670e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157016064 dispatch_meta_bytes=229656 combine_payload_bytes=157016064 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157245720 ingress_bytes=175890432 link_bytes=175890432 token_bytes=314261784 weight_movement_bytes=18874368 total_nvlink_bytes=333136152 weight_to_token_ratio=0.060059 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.685295576e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155885568 dispatch_meta_bytes=227944 combine_payload_bytes=155885568 prefetch_bytes=18874368 reduce_bytes=56623104 egress_bytes=156113512 ingress_bytes=231383040 link_bytes=231383040 token_bytes=311999080 weight_movement_bytes=75497472 total_nvlink_bytes=387496552 weight_to_token_ratio=0.241980 gemm_rows_defined=34048 gemm_rows_charged=36864 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.076389 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.216998442e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43784 payload_rows_remote=38322 dup_rows=21752 dup_rows_remote=18980 dedup_ratio=0.331909 router_tokens_per_expert=[260, 232, 257, 246, 262, 262, 292, 251, 241, 263, 271, 270, 197, 238, 274, 248, 256, 253, 247, 246, 243, 244, 291, 255, 254, 270, 275, 245, 251, 243, 253, 239, 246, 279, 263, 273, 243, 273, 258, 283, 252, 253, 251, 260, 255, 271, 269, 254, 269, 278, 283, 245, 248, 257, 239, 248, 256, 234, 262, 237, 247, 257, 261, 264, 260, 275, 269, 269, 256, 291, 247, 262, 261, 232, 252, 252, 258, 300, 246, 236, 243, 246, 260, 252, 249, 239, 236, 273, 233, 269, 240, 250, 248, 253, 263, 273, 271, 257, 226, 252, 247, 286, 236, 260, 268, 266, 269, 243, 263, 257, 242, 274, 245, 240, 244, 273, 236, 254, 256, 264, 247, 272, 241, 264, 263, 278, 228, 256] payload_rows_per_dst=[5512, 5504, 5528, 5422, 5536, 5416, 5462, 5404] dup_rows_per_dst=[2616, 2626, 2838, 2748, 2796, 2638, 2772, 2718] combine_rows_per_src=[5512, 5504, 5528, 5422, 5536, 5416, 5462, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=45 plans=1 dispatch_payload_bytes=156737536 dispatch_meta_bytes=228824 combine_payload_bytes=156737536 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=156966360 ingress_bytes=175611904 link_bytes=175611904 token_bytes=313703896 weight_movement_bytes=18874368 total_nvlink_bytes=332578264 weight_to_token_ratio=0.060166 gemm_rows_defined=33792 gemm_rows_charged=36864 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.083333 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.682626858e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43636 payload_rows_remote=38150 dup_rows=21900 dup_rows_remote=19142 dedup_ratio=0.334167 router_tokens_per_expert=[253, 294, 229, 282, 250, 264, 243, 246, 253, 236, 249, 253, 292, 281, 257, 253, 262, 250, 252, 261, 261, 239, 240, 295, 234, 239, 267, 246, 283, 255, 248, 266, 251, 234, 250, 252, 239, 275, 248, 251, 257, 243, 246, 271, 254, 256, 255, 227, 270, 289, 240, 239, 285, 248, 290, 233, 246, 239, 255, 241, 270, 246, 262, 269, 269, 237, 257, 313, 231, 230, 252, 225, 239, 246, 255, 255, 266, 271, 256, 272, 290, 248, 225, 259, 261, 247, 277, 274, 262, 267, 253, 263, 241, 251, 254, 257, 267, 267, 248, 267, 263, 265, 267, 263, 255, 255, 248, 248, 265, 230, 252, 261, 277, 237, 251, 262, 229, 250, 257, 274, 254, 251, 252, 262, 253, 234, 274, 263] payload_rows_per_dst=[5456, 5430, 5420, 5486, 5418, 5460, 5512, 5454] dup_rows_per_dst=[2814, 2766, 2598, 2758, 2730, 2798, 2730, 2706] combine_rows_per_src=[5456, 5430, 5420, 5486, 5418, 5460, 5512, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15027606952 prefetch_bytes=830472192 reduce_bytes=547356672 weight_movement_bytes=1377828864 egress_bytes=7519294888 ingress_bytes=8886140928 link_bytes=8886140928 total_nvlink_bytes=16405435816 weight_to_token_ratio=0.091687 prefetch_share_of_ingress=0.093457 dedup_ratio=0.332983 prefetch_slots=88 unused_prefetch_slots=1448 slot_utilization=0.057292 reduce_slots=29 remote_demand=44 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080440 gemm_inner_pad_upper_bound=109220 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.773804794e-04 prefetch_bytes_per_gemm_flop=1.657744985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15061451888 prefetch_bytes=717225984 reduce_bytes=717225984 weight_movement_bytes=1434451968 egress_bytes=7536231536 ingress_bytes=8959672320 link_bytes=8959672320 total_nvlink_bytes=16495903856 weight_to_token_ratio=0.095240 prefetch_share_of_ingress=0.080050 dedup_ratio=0.332584 prefetch_slots=76 unused_prefetch_slots=1460 slot_utilization=0.049479 reduce_slots=38 remote_demand=38 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.082248 gemm_inner_pad_upper_bound=108458 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.788482745e-04 prefetch_bytes_per_gemm_flop=1.431688850e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15058823128 prefetch_bytes=773849088 reduce_bytes=943718400 weight_movement_bytes=1717567488 egress_bytes=7534913496 ingress_bytes=9241477120 link_bytes=9241477120 total_nvlink_bytes=16776390616 weight_to_token_ratio=0.114057 prefetch_share_of_ingress=0.083737 dedup_ratio=0.332589 prefetch_slots=82 unused_prefetch_slots=1454 slot_utilization=0.053385 reduce_slots=50 remote_demand=41 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080657 gemm_inner_pad_upper_bound=108839 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.844735139e-04 prefetch_bytes_per_gemm_flop=1.544716917e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156966912 dispatch_meta_bytes=229208 combine_payload_bytes=156966912 prefetch_bytes=18874368 reduce_bytes=0 egress_bytes=157196120 ingress_bytes=175841280 link_bytes=175841280 token_bytes=314163032 weight_movement_bytes=18874368 total_nvlink_bytes=333037400 weight_to_token_ratio=0.060078 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.684824626e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43624 payload_rows_remote=37992 dup_rows=21912 dup_rows_remote=19106 dedup_ratio=0.334351 router_tokens_per_expert=[260, 262, 269, 255, 262, 251, 236, 272, 251, 258, 283, 243, 266, 250, 239, 236, 283, 251, 256, 280, 262, 273, 268, 254, 231, 261, 257, 250, 251, 211, 259, 259, 262, 254, 271, 280, 246, 228, 272, 236, 247, 249, 231, 256, 245, 248, 247, 289, 250, 254, 270, 244, 267, 248, 250, 247, 279, 234, 256, 247, 270, 266, 247, 267, 246, 248, 251, 263, 262, 271, 244, 251, 236, 255, 252, 267, 262, 252, 268, 256, 244, 276, 278, 251, 256, 242, 280, 279, 269, 260, 248, 294, 265, 265, 253, 259, 251, 226, 286, 264, 217, 251, 273, 224, 261, 238, 270, 257, 248, 288, 239, 240, 286, 242, 232, 240, 262, 223, 284, 237, 261, 266, 261, 262, 258, 254, 237, 271] payload_rows_per_dst=[5476, 5444, 5410, 5442, 5418, 5632, 5418, 5384] dup_rows_per_dst=[2710, 2768, 2712, 2750, 2750, 2806, 2648, 2768] combine_rows_per_src=[5476, 5444, 5410, 5442, 5418, 5632, 5418, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=156262400 dispatch_meta_bytes=229168 combine_payload_bytes=156262400 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156491568 ingress_bytes=194011136 link_bytes=194011136 token_bytes=312753968 weight_movement_bytes=37748736 total_nvlink_bytes=350502704 weight_to_token_ratio=0.120698 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.858919246e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15059908296 prefetch_bytes=792723456 reduce_bytes=641728512 weight_movement_bytes=1434451968 egress_bytes=7535457992 ingress_bytes=8958902272 link_bytes=8958902272 total_nvlink_bytes=16494360264 weight_to_token_ratio=0.095250 prefetch_share_of_ingress=0.088484 dedup_ratio=0.332544 prefetch_slots=84 unused_prefetch_slots=1452 slot_utilization=0.054688 reduce_slots=34 remote_demand=42 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108966 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.788329032e-04 prefetch_bytes_per_gemm_flop=1.582392940e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=155615232 dispatch_meta_bytes=228392 combine_payload_bytes=155615232 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=155843624 ingress_bytes=193363968 link_bytes=193363968 token_bytes=311458856 weight_movement_bytes=37748736 total_nvlink_bytes=349207592 weight_to_token_ratio=0.121200 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.079861 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.852718400e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43642 payload_rows_remote=38328 dup_rows=21894 dup_rows_remote=19148 dedup_ratio=0.334076 router_tokens_per_expert=[273, 283, 296, 249, 236, 236, 248, 252, 252, 224, 245, 254, 247, 250, 237, 248, 232, 275, 246, 263, 242, 237, 240, 237, 253, 271, 254, 240, 241, 263, 238, 245, 258, 252, 258, 270, 275, 275, 284, 234, 242, 244, 245, 256, 251, 267, 289, 246, 240, 250, 220, 278, 230, 288, 221, 257, 220, 239, 260, 274, 279, 257, 270, 243, 252, 277, 242, 267, 255, 293, 263, 254, 242, 259, 270, 291, 273, 242, 248, 250, 248, 262, 258, 279, 266, 264, 268, 267, 253, 256, 252, 274, 237, 279, 278, 274, 272, 239, 274, 248, 257, 270, 251, 246, 223, 273, 252, 251, 242, 268, 250, 278, 295, 253, 222, 270, 270, 250, 215, 267, 255, 247, 267, 249, 261, 251, 268, 262] payload_rows_per_dst=[5314, 5418, 5652, 5388, 5580, 5474, 5528, 5288] dup_rows_per_dst=[2746, 2694, 3016, 2664, 2776, 2580, 2836, 2582] combine_rows_per_src=[5314, 5418, 5652, 5388, 5580, 5474, 5528, 5288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15059142016 prefetch_bytes=811597824 reduce_bytes=679477248 weight_movement_bytes=1491075072 egress_bytes=7535076736 ingress_bytes=9015140352 link_bytes=9015140352 total_nvlink_bytes=16550217088 weight_to_token_ratio=0.099015 prefetch_share_of_ingress=0.090026 dedup_ratio=0.332628 prefetch_slots=86 unused_prefetch_slots=1450 slot_utilization=0.055990 reduce_slots=36 remote_demand=43 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.081525 gemm_inner_pad_upper_bound=109093 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.799554982e-04 prefetch_bytes_per_gemm_flop=1.620068962e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15065675200 prefetch_bytes=736100352 reduce_bytes=868220928 weight_movement_bytes=1604321280 egress_bytes=7538341312 ingress_bytes=9131655168 link_bytes=9131655168 total_nvlink_bytes=16669996480 weight_to_token_ratio=0.106489 prefetch_share_of_ingress=0.080610 dedup_ratio=0.332314 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=46 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080802 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.822813058e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=46 plans=1 dispatch_payload_bytes=156991488 dispatch_meta_bytes=229904 combine_payload_bytes=156991488 prefetch_bytes=0 reduce_bytes=56623104 egress_bytes=157221392 ingress_bytes=213614592 link_bytes=213614592 token_bytes=314212880 weight_movement_bytes=56623104 total_nvlink_bytes=370835984 weight_to_token_ratio=0.180206 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=2.046749916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43842 payload_rows_remote=38252 dup_rows=21694 dup_rows_remote=19000 dedup_ratio=0.331024 router_tokens_per_expert=[263, 256, 246, 268, 235, 240, 272, 249, 243, 249, 249, 250, 286, 256, 289, 270, 247, 257, 246, 253, 264, 254, 255, 240, 255, 282, 305, 275, 244, 262, 234, 241, 265, 248, 241, 256, 261, 257, 244, 256, 268, 248, 235, 246, 248, 244, 255, 267, 251, 259, 259, 233, 240, 263, 268, 232, 258, 254, 255, 244, 244, 246, 232, 240, 240, 282, 258, 253, 267, 249, 272, 290, 246, 241, 265, 237, 261, 282, 255, 251, 265, 254, 227, 248, 243, 261, 259, 272, 265, 252, 253, 283, 261, 254, 275, 257, 281, 236, 285, 246, 241, 251, 259, 256, 243, 254, 269, 265, 224, 274, 260, 268, 255, 243, 283, 265, 274, 248, 241, 244, 235, 277, 292, 269, 252, 274, 240, 234] payload_rows_per_dst=[5590, 5590, 5336, 5536, 5668, 5356, 5538, 5228] dup_rows_per_dst=[2694, 2762, 2582, 3004, 2980, 2546, 2686, 2440] combine_rows_per_src=[5590, 5590, 5336, 5536, 5668, 5356, 5538, 5228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=47 plans=1 dispatch_payload_bytes=156680192 dispatch_meta_bytes=229008 combine_payload_bytes=156680192 prefetch_bytes=18874368 reduce_bytes=18874368 egress_bytes=156909200 ingress_bytes=194428928 link_bytes=194428928 token_bytes=313589392 weight_movement_bytes=37748736 total_nvlink_bytes=351338128 weight_to_token_ratio=0.120376 gemm_rows_defined=34176 gemm_rows_charged=36864 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.072917 gemm_active_groups=18 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.862922323e-04 prefetch_bytes_per_gemm_flop=1.808449074e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 E=128 B=16 K=8 R=8 NvS=36832 hidden_size=2048 assignments=32768 payload_rows=43702 payload_rows_remote=38386 dup_rows=21834 dup_rows_remote=19154 dedup_ratio=0.333160 router_tokens_per_expert=[264, 259, 255, 261, 254, 255, 255, 255, 230, 268, 237, 233, 240, 247, 242, 243, 258, 252, 237, 253, 263, 260, 269, 276, 269, 220, 260, 274, 260, 263, 248, 262, 245, 231, 255, 249, 245, 263, 242, 297, 261, 236, 224, 275, 241, 259, 251, 236, 287, 230, 242, 253, 289, 259, 256, 236, 247, 257, 260, 243, 288, 273, 250, 242, 256, 289, 282, 250, 268, 258, 238, 300, 233, 272, 279, 285, 268, 253, 221, 252, 248, 263, 263, 255, 233, 248, 275, 238, 253, 246, 258, 289, 267, 257, 255, 251, 252, 238, 245, 270, 278, 271, 265, 298, 244, 232, 274, 271, 264, 253, 305, 246, 246, 235, 227, 256, 249, 237, 245, 230, 264, 253, 273, 240, 279, 258, 255, 268] payload_rows_per_dst=[5316, 5496, 5450, 5468, 5578, 5456, 5532, 5406] dup_rows_per_dst=[2680, 2752, 2570, 2756, 2830, 2742, 2880, 2624] combine_rows_per_src=[5316, 5496, 5450, 5468, 5578, 5456, 5532, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=9437184 grad_bytes_per_expert=18874368 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=48 plans=1 dispatch_payload_bytes=157229056 dispatch_meta_bytes=230160 combine_payload_bytes=157229056 prefetch_bytes=0 reduce_bytes=37748736 egress_bytes=157459216 ingress_bytes=194977792 link_bytes=194977792 token_bytes=314688272 weight_movement_bytes=37748736 total_nvlink_bytes=352437008 weight_to_token_ratio=0.119956 gemm_rows_defined=33920 gemm_rows_charged=36864 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.079861 gemm_active_groups=17 gemm_peak_group_rows=2176 gemm_flops_fwd=347892350976 gemm_flops_step=1043677052928 bytes_per_gemm_flop=1.868181268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=48 plans=48 dispatch_calls=96 combine_calls=96 prefetch_calls=96 reduce_calls=48 token_bytes=15042442528 prefetch_bytes=736100352 reduce_bytes=1075838976 weight_movement_bytes=1811939328 egress_bytes=7526716704 ingress_bytes=9327665152 link_bytes=9327665152 total_nvlink_bytes=16854381856 weight_to_token_ratio=0.120455 prefetch_share_of_ingress=0.078916 dedup_ratio=0.332673 prefetch_slots=78 unused_prefetch_slots=1458 slot_utilization=0.050781 reduce_slots=57 remote_demand=39 gemm_rows_charged=1769472 gemm_tail_pad_fraction=0.080874 gemm_inner_pad_upper_bound=108585 gemm_flops_step=50096498540544 bytes_per_gemm_flop=1.861939541e-04 prefetch_bytes_per_gemm_flop=1.469364873e-05 anomalies=none + [2026-08-29 23:39:13] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 786.5 | throughput per GPU (TFLOP/s/GPU): 113.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.763014E+00 | loss scale: 1.0 | grad norm: 2.883 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:39:13 +[rank0]:[W829 23:39:14.164605782 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W829 23:39:16.420208852 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W829 23:39:17.277963147 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W829 23:39:17.296653105 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W829 23:39:17.304895455 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W829 23:39:17.308063469 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W829 23:39:17.438834384 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W829 23:39:17.868359783 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:39:21.462000 222421 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:39:21.463000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222606 closing signal SIGTERM +W0829 23:39:21.463000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222607 closing signal SIGTERM +W0829 23:39:21.463000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222608 closing signal SIGTERM +W0829 23:39:21.464000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222609 closing signal SIGTERM +W0829 23:39:21.464000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222610 closing signal SIGTERM +W0829 23:39:21.465000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222611 closing signal SIGTERM +W0829 23:39:21.465000 222421 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 222612 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 222421 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 4a5fced99f1734da44f1d70acba0c9ba977e4460..0636e1bf088cc021f4428873e339e3c129141d07 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,3 +1,8 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir QWEN3_235B moonep natural none OK 120129 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 QWEN3_235B moonep forced none OK 120081 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1 +QWEN3_30B moonep natural none OK 116419 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1 +QWEN3_30B moonep forced none OK 116515 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 +DSV2_LITE_L24 moonep natural none OK 125157 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1 +DSV2_LITE_L24 moonep forced none OK 125633 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1 +QWEN2_57B moonep natural none OK 95855 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index ba332603fc3af4f3e6d852b6a64cf5e7f329f6b2..617d1e8a627f80a13153523b87b2fd0b668ff460 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,2 +1,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir QWEN3_235B moonep SELECTED none 0 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 +QWEN3_30B moonep SELECTED none 0 116419 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1 +DSV2_LITE_L24 moonep SELECTED none 0 125157 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv index d2b3e06ae855ed5553473eab15818a7c3f7b91ea..937dcb824b78a57f582726f2b56159ac746dd662 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,3 +1,8 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir QWEN3_235B moonep matched 16 8192 8 none none 1 OK 370.33 368.00 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 QWEN3_235B moonep matched 16 8192 8 none none+forcebal 1 OK 399.19 354.00 120081 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 1 OK 829.20 747.60 116419 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1 +QWEN3_30B moonep matched 16 4096 8 none none+forcebal 1 OK 819.26 774.40 116515 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 673.29 669.50 125157 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 674.59 672.10 125633 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1 +QWEN2_57B moonep matched 8 4096 8 none none 1 OK 238.47 236.30 95855 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1 diff --git a/tierb-fixed-moonep-full/study.log b/tierb-fixed-moonep-full/study.log index 04c7e3a23e0b9920da3c4fbb2327c26bb02a5c6f..54eb6213476e92801de754e31a53634f4bf9c08b 100644 --- a/tierb-fixed-moonep-full/study.log +++ b/tierb-fixed-moonep-full/study.log @@ -26,3 +26,27 @@ driver_repo: /workspace/blackwell_run_scripts @ 5054c89 [guard] tearing down process group (outcome=DONE) -------------------------------------------------------------------------- [23:36:36] CELL QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async (slots=16) + [guard] iter 8/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[23:38:01] CELL QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async (slots=16) + [guard] iter 9/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[23:39:25] CELL DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async (slots=8) + [guard] iter 7/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[23:40:49] CELL DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async (slots=8) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[23:42:04] CELL QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async (slots=8) + [guard] iter 6/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[23:43:17] CELL QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async (slots=8)